blob: 5c4499a8561124c480038f5167f3e34dccfe3047 [file] [log] [blame]
Kim Phillips9c4a7962008-06-23 19:50:15 +08001/*
2 * talitos - Freescale Integrated Security Engine (SEC) device driver
3 *
Kim Phillips5228f0f2011-07-15 11:21:38 +08004 * Copyright (c) 2008-2011 Freescale Semiconductor, Inc.
Kim Phillips9c4a7962008-06-23 19:50:15 +08005 *
6 * Scatterlist Crypto API glue code copied from files with the following:
7 * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au>
8 *
9 * Crypto algorithm registration code copied from hifn driver:
10 * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru>
11 * All rights reserved.
12 *
13 * This program is free software; you can redistribute it and/or modify
14 * it under the terms of the GNU General Public License as published by
15 * the Free Software Foundation; either version 2 of the License, or
16 * (at your option) any later version.
17 *
18 * This program is distributed in the hope that it will be useful,
19 * but WITHOUT ANY WARRANTY; without even the implied warranty of
20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 * GNU General Public License for more details.
22 *
23 * You should have received a copy of the GNU General Public License
24 * along with this program; if not, write to the Free Software
25 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
26 */
27
28#include <linux/kernel.h>
29#include <linux/module.h>
30#include <linux/mod_devicetable.h>
31#include <linux/device.h>
32#include <linux/interrupt.h>
33#include <linux/crypto.h>
34#include <linux/hw_random.h>
Rob Herring5af50732013-09-17 14:28:33 -050035#include <linux/of_address.h>
36#include <linux/of_irq.h>
Kim Phillips9c4a7962008-06-23 19:50:15 +080037#include <linux/of_platform.h>
38#include <linux/dma-mapping.h>
39#include <linux/io.h>
40#include <linux/spinlock.h>
41#include <linux/rtnetlink.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090042#include <linux/slab.h>
Kim Phillips9c4a7962008-06-23 19:50:15 +080043
44#include <crypto/algapi.h>
45#include <crypto/aes.h>
Lee Nipper3952f172008-07-10 18:29:18 +080046#include <crypto/des.h>
Kim Phillips9c4a7962008-06-23 19:50:15 +080047#include <crypto/sha.h>
Lee Nipper497f2e62010-05-19 19:20:36 +100048#include <crypto/md5.h>
Herbert Xue98014a2015-05-11 17:47:48 +080049#include <crypto/internal/aead.h>
Kim Phillips9c4a7962008-06-23 19:50:15 +080050#include <crypto/authenc.h>
Lee Nipper4de9d0b2009-03-29 15:52:32 +080051#include <crypto/skcipher.h>
Lee Nipperacbf7c622010-05-19 19:19:33 +100052#include <crypto/hash.h>
53#include <crypto/internal/hash.h>
Lee Nipper4de9d0b2009-03-29 15:52:32 +080054#include <crypto/scatterwalk.h>
Kim Phillips9c4a7962008-06-23 19:50:15 +080055
56#include "talitos.h"
57
LEROY Christophe922f9dc2015-04-17 16:32:07 +020058static void to_talitos_ptr(struct talitos_ptr *ptr, dma_addr_t dma_addr,
LEROY Christopheda9de142017-10-06 15:04:57 +020059 unsigned int len, bool is_sec1)
Kim Phillips81eb0242009-08-13 11:51:51 +100060{
LEROY Christopheedc6bd62015-04-17 16:31:53 +020061 ptr->ptr = cpu_to_be32(lower_32_bits(dma_addr));
LEROY Christopheda9de142017-10-06 15:04:57 +020062 if (is_sec1) {
63 ptr->len1 = cpu_to_be16(len);
64 } else {
65 ptr->len = cpu_to_be16(len);
LEROY Christophe922f9dc2015-04-17 16:32:07 +020066 ptr->eptr = upper_32_bits(dma_addr);
LEROY Christopheda9de142017-10-06 15:04:57 +020067 }
Kim Phillips81eb0242009-08-13 11:51:51 +100068}
69
Horia Geant?340ff602016-04-19 20:33:48 +030070static void copy_talitos_ptr(struct talitos_ptr *dst_ptr,
71 struct talitos_ptr *src_ptr, bool is_sec1)
72{
73 dst_ptr->ptr = src_ptr->ptr;
LEROY Christophe922f9dc2015-04-17 16:32:07 +020074 if (is_sec1) {
LEROY Christopheda9de142017-10-06 15:04:57 +020075 dst_ptr->len1 = src_ptr->len1;
LEROY Christophe922f9dc2015-04-17 16:32:07 +020076 } else {
LEROY Christopheda9de142017-10-06 15:04:57 +020077 dst_ptr->len = src_ptr->len;
78 dst_ptr->eptr = src_ptr->eptr;
LEROY Christophe922f9dc2015-04-17 16:32:07 +020079 }
LEROY Christophe538caf82015-04-17 16:31:59 +020080}
81
LEROY Christophe922f9dc2015-04-17 16:32:07 +020082static unsigned short from_talitos_ptr_len(struct talitos_ptr *ptr,
83 bool is_sec1)
LEROY Christophe538caf82015-04-17 16:31:59 +020084{
LEROY Christophe922f9dc2015-04-17 16:32:07 +020085 if (is_sec1)
86 return be16_to_cpu(ptr->len1);
87 else
88 return be16_to_cpu(ptr->len);
LEROY Christophe538caf82015-04-17 16:31:59 +020089}
90
LEROY Christopheb096b542016-06-06 13:20:34 +020091static void to_talitos_ptr_ext_set(struct talitos_ptr *ptr, u8 val,
92 bool is_sec1)
LEROY Christophe185eb792015-04-17 16:31:55 +020093{
LEROY Christophe922f9dc2015-04-17 16:32:07 +020094 if (!is_sec1)
LEROY Christopheb096b542016-06-06 13:20:34 +020095 ptr->j_extent = val;
96}
97
98static void to_talitos_ptr_ext_or(struct talitos_ptr *ptr, u8 val, bool is_sec1)
99{
100 if (!is_sec1)
101 ptr->j_extent |= val;
LEROY Christophe185eb792015-04-17 16:31:55 +0200102}
103
Kim Phillips9c4a7962008-06-23 19:50:15 +0800104/*
105 * map virtual single (contiguous) pointer to h/w descriptor pointer
106 */
107static void map_single_talitos_ptr(struct device *dev,
LEROY Christopheedc6bd62015-04-17 16:31:53 +0200108 struct talitos_ptr *ptr,
Horia Geant?42e8b0d2015-05-11 20:04:56 +0300109 unsigned int len, void *data,
Kim Phillips9c4a7962008-06-23 19:50:15 +0800110 enum dma_data_direction dir)
111{
Kim Phillips81eb0242009-08-13 11:51:51 +1000112 dma_addr_t dma_addr = dma_map_single(dev, data, len, dir);
LEROY Christophe922f9dc2015-04-17 16:32:07 +0200113 struct talitos_private *priv = dev_get_drvdata(dev);
114 bool is_sec1 = has_ftr_sec1(priv);
Kim Phillips81eb0242009-08-13 11:51:51 +1000115
LEROY Christopheda9de142017-10-06 15:04:57 +0200116 to_talitos_ptr(ptr, dma_addr, len, is_sec1);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800117}
118
119/*
120 * unmap bus single (contiguous) h/w descriptor pointer
121 */
122static void unmap_single_talitos_ptr(struct device *dev,
LEROY Christopheedc6bd62015-04-17 16:31:53 +0200123 struct talitos_ptr *ptr,
Kim Phillips9c4a7962008-06-23 19:50:15 +0800124 enum dma_data_direction dir)
125{
LEROY Christophe922f9dc2015-04-17 16:32:07 +0200126 struct talitos_private *priv = dev_get_drvdata(dev);
127 bool is_sec1 = has_ftr_sec1(priv);
128
LEROY Christopheedc6bd62015-04-17 16:31:53 +0200129 dma_unmap_single(dev, be32_to_cpu(ptr->ptr),
LEROY Christophe922f9dc2015-04-17 16:32:07 +0200130 from_talitos_ptr_len(ptr, is_sec1), dir);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800131}
132
133static int reset_channel(struct device *dev, int ch)
134{
135 struct talitos_private *priv = dev_get_drvdata(dev);
136 unsigned int timeout = TALITOS_TIMEOUT;
LEROY Christophedd3c0982015-04-17 16:32:13 +0200137 bool is_sec1 = has_ftr_sec1(priv);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800138
LEROY Christophedd3c0982015-04-17 16:32:13 +0200139 if (is_sec1) {
140 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
141 TALITOS1_CCCR_LO_RESET);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800142
LEROY Christophedd3c0982015-04-17 16:32:13 +0200143 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR_LO) &
144 TALITOS1_CCCR_LO_RESET) && --timeout)
145 cpu_relax();
146 } else {
147 setbits32(priv->chan[ch].reg + TALITOS_CCCR,
148 TALITOS2_CCCR_RESET);
149
150 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
151 TALITOS2_CCCR_RESET) && --timeout)
152 cpu_relax();
153 }
Kim Phillips9c4a7962008-06-23 19:50:15 +0800154
155 if (timeout == 0) {
156 dev_err(dev, "failed to reset channel %d\n", ch);
157 return -EIO;
158 }
159
Kim Phillips81eb0242009-08-13 11:51:51 +1000160 /* set 36-bit addressing, done writeback enable and done IRQ enable */
Kim Phillipsad42d5f2011-11-21 16:13:27 +0800161 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, TALITOS_CCCR_LO_EAE |
Kim Phillips81eb0242009-08-13 11:51:51 +1000162 TALITOS_CCCR_LO_CDWE | TALITOS_CCCR_LO_CDIE);
LEROY Christophe37b5e882017-10-06 15:05:06 +0200163 /* enable chaining descriptors */
164 if (is_sec1)
165 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
166 TALITOS_CCCR_LO_NE);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800167
Kim Phillipsfe5720e2008-10-12 20:33:14 +0800168 /* and ICCR writeback, if available */
169 if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
Kim Phillipsad42d5f2011-11-21 16:13:27 +0800170 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO,
Kim Phillipsfe5720e2008-10-12 20:33:14 +0800171 TALITOS_CCCR_LO_IWSE);
172
Kim Phillips9c4a7962008-06-23 19:50:15 +0800173 return 0;
174}
175
176static int reset_device(struct device *dev)
177{
178 struct talitos_private *priv = dev_get_drvdata(dev);
179 unsigned int timeout = TALITOS_TIMEOUT;
LEROY Christophedd3c0982015-04-17 16:32:13 +0200180 bool is_sec1 = has_ftr_sec1(priv);
181 u32 mcr = is_sec1 ? TALITOS1_MCR_SWR : TALITOS2_MCR_SWR;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800182
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800183 setbits32(priv->reg + TALITOS_MCR, mcr);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800184
LEROY Christophedd3c0982015-04-17 16:32:13 +0200185 while ((in_be32(priv->reg + TALITOS_MCR) & mcr)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800186 && --timeout)
187 cpu_relax();
188
Kim Phillips2cdba3c2011-12-12 14:59:11 -0600189 if (priv->irq[1]) {
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800190 mcr = TALITOS_MCR_RCA1 | TALITOS_MCR_RCA3;
191 setbits32(priv->reg + TALITOS_MCR, mcr);
192 }
193
Kim Phillips9c4a7962008-06-23 19:50:15 +0800194 if (timeout == 0) {
195 dev_err(dev, "failed to reset device\n");
196 return -EIO;
197 }
198
199 return 0;
200}
201
202/*
203 * Reset and initialize the device
204 */
205static int init_device(struct device *dev)
206{
207 struct talitos_private *priv = dev_get_drvdata(dev);
208 int ch, err;
LEROY Christophedd3c0982015-04-17 16:32:13 +0200209 bool is_sec1 = has_ftr_sec1(priv);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800210
211 /*
212 * Master reset
213 * errata documentation: warning: certain SEC interrupts
214 * are not fully cleared by writing the MCR:SWR bit,
215 * set bit twice to completely reset
216 */
217 err = reset_device(dev);
218 if (err)
219 return err;
220
221 err = reset_device(dev);
222 if (err)
223 return err;
224
225 /* reset channels */
226 for (ch = 0; ch < priv->num_channels; ch++) {
227 err = reset_channel(dev, ch);
228 if (err)
229 return err;
230 }
231
232 /* enable channel done and error interrupts */
LEROY Christophedd3c0982015-04-17 16:32:13 +0200233 if (is_sec1) {
234 clrbits32(priv->reg + TALITOS_IMR, TALITOS1_IMR_INIT);
235 clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT);
236 /* disable parity error check in DEU (erroneous? test vect.) */
237 setbits32(priv->reg_deu + TALITOS_EUICR, TALITOS1_DEUICR_KPE);
238 } else {
239 setbits32(priv->reg + TALITOS_IMR, TALITOS2_IMR_INIT);
240 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT);
241 }
Kim Phillips9c4a7962008-06-23 19:50:15 +0800242
Kim Phillipsfe5720e2008-10-12 20:33:14 +0800243 /* disable integrity check error interrupts (use writeback instead) */
244 if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
LEROY Christophe5fa7fa12015-04-17 16:32:11 +0200245 setbits32(priv->reg_mdeu + TALITOS_EUICR_LO,
Kim Phillipsfe5720e2008-10-12 20:33:14 +0800246 TALITOS_MDEUICR_LO_ICE);
247
Kim Phillips9c4a7962008-06-23 19:50:15 +0800248 return 0;
249}
250
251/**
252 * talitos_submit - submits a descriptor to the device for processing
253 * @dev: the SEC device to be used
Kim Phillips5228f0f2011-07-15 11:21:38 +0800254 * @ch: the SEC device channel to be used
Kim Phillips9c4a7962008-06-23 19:50:15 +0800255 * @desc: the descriptor to be processed by the device
256 * @callback: whom to call when processing is complete
257 * @context: a handle for use by caller (optional)
258 *
259 * desc must contain valid dma-mapped (bus physical) address pointers.
260 * callback must check err and feedback in descriptor header
261 * for device processing status.
262 */
Horia Geanta865d5062012-07-03 19:16:52 +0300263int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc,
264 void (*callback)(struct device *dev,
265 struct talitos_desc *desc,
266 void *context, int error),
267 void *context)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800268{
269 struct talitos_private *priv = dev_get_drvdata(dev);
270 struct talitos_request *request;
Kim Phillips5228f0f2011-07-15 11:21:38 +0800271 unsigned long flags;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800272 int head;
LEROY Christophe7d607c6a2015-04-17 16:32:09 +0200273 bool is_sec1 = has_ftr_sec1(priv);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800274
Kim Phillips4b9926282009-08-13 11:50:38 +1000275 spin_lock_irqsave(&priv->chan[ch].head_lock, flags);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800276
Kim Phillips4b9926282009-08-13 11:50:38 +1000277 if (!atomic_inc_not_zero(&priv->chan[ch].submit_count)) {
Kim Phillipsec6644d2008-07-17 20:16:40 +0800278 /* h/w fifo is full */
Kim Phillips4b9926282009-08-13 11:50:38 +1000279 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800280 return -EAGAIN;
281 }
282
Kim Phillips4b9926282009-08-13 11:50:38 +1000283 head = priv->chan[ch].head;
284 request = &priv->chan[ch].fifo[head];
Kim Phillipsec6644d2008-07-17 20:16:40 +0800285
Kim Phillips9c4a7962008-06-23 19:50:15 +0800286 /* map descriptor and save caller data */
LEROY Christophe7d607c6a2015-04-17 16:32:09 +0200287 if (is_sec1) {
288 desc->hdr1 = desc->hdr;
LEROY Christophe7d607c6a2015-04-17 16:32:09 +0200289 request->dma_desc = dma_map_single(dev, &desc->hdr1,
290 TALITOS_DESC_SIZE,
291 DMA_BIDIRECTIONAL);
292 } else {
293 request->dma_desc = dma_map_single(dev, desc,
294 TALITOS_DESC_SIZE,
295 DMA_BIDIRECTIONAL);
296 }
Kim Phillips9c4a7962008-06-23 19:50:15 +0800297 request->callback = callback;
298 request->context = context;
299
300 /* increment fifo head */
Kim Phillips4b9926282009-08-13 11:50:38 +1000301 priv->chan[ch].head = (priv->chan[ch].head + 1) & (priv->fifo_len - 1);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800302
303 smp_wmb();
304 request->desc = desc;
305
306 /* GO! */
307 wmb();
Kim Phillipsad42d5f2011-11-21 16:13:27 +0800308 out_be32(priv->chan[ch].reg + TALITOS_FF,
309 upper_32_bits(request->dma_desc));
310 out_be32(priv->chan[ch].reg + TALITOS_FF_LO,
Kim Phillipsa7524472010-09-23 15:56:38 +0800311 lower_32_bits(request->dma_desc));
Kim Phillips9c4a7962008-06-23 19:50:15 +0800312
Kim Phillips4b9926282009-08-13 11:50:38 +1000313 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800314
315 return -EINPROGRESS;
316}
Horia Geanta865d5062012-07-03 19:16:52 +0300317EXPORT_SYMBOL(talitos_submit);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800318
319/*
320 * process what was done, notify callback of error if not
321 */
322static void flush_channel(struct device *dev, int ch, int error, int reset_ch)
323{
324 struct talitos_private *priv = dev_get_drvdata(dev);
325 struct talitos_request *request, saved_req;
326 unsigned long flags;
327 int tail, status;
LEROY Christophe7d607c6a2015-04-17 16:32:09 +0200328 bool is_sec1 = has_ftr_sec1(priv);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800329
Kim Phillips4b9926282009-08-13 11:50:38 +1000330 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800331
Kim Phillips4b9926282009-08-13 11:50:38 +1000332 tail = priv->chan[ch].tail;
333 while (priv->chan[ch].fifo[tail].desc) {
LEROY Christophe7d607c6a2015-04-17 16:32:09 +0200334 __be32 hdr;
335
Kim Phillips4b9926282009-08-13 11:50:38 +1000336 request = &priv->chan[ch].fifo[tail];
Kim Phillips9c4a7962008-06-23 19:50:15 +0800337
338 /* descriptors with their done bits set don't get the error */
339 rmb();
LEROY Christophe37b5e882017-10-06 15:05:06 +0200340 if (!is_sec1)
341 hdr = request->desc->hdr;
342 else if (request->desc->next_desc)
343 hdr = (request->desc + 1)->hdr1;
344 else
345 hdr = request->desc->hdr1;
LEROY Christophe7d607c6a2015-04-17 16:32:09 +0200346
347 if ((hdr & DESC_HDR_DONE) == DESC_HDR_DONE)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800348 status = 0;
Lee Nipperca38a812008-12-20 17:09:25 +1100349 else
Kim Phillips9c4a7962008-06-23 19:50:15 +0800350 if (!error)
351 break;
352 else
353 status = error;
354
355 dma_unmap_single(dev, request->dma_desc,
LEROY Christophe7d607c6a2015-04-17 16:32:09 +0200356 TALITOS_DESC_SIZE,
Kim Phillipse938e462009-03-29 15:53:23 +0800357 DMA_BIDIRECTIONAL);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800358
359 /* copy entries so we can call callback outside lock */
360 saved_req.desc = request->desc;
361 saved_req.callback = request->callback;
362 saved_req.context = request->context;
363
364 /* release request entry in fifo */
365 smp_wmb();
366 request->desc = NULL;
367
368 /* increment fifo tail */
Kim Phillips4b9926282009-08-13 11:50:38 +1000369 priv->chan[ch].tail = (tail + 1) & (priv->fifo_len - 1);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800370
Kim Phillips4b9926282009-08-13 11:50:38 +1000371 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
Kim Phillipsec6644d2008-07-17 20:16:40 +0800372
Kim Phillips4b9926282009-08-13 11:50:38 +1000373 atomic_dec(&priv->chan[ch].submit_count);
Kim Phillipsec6644d2008-07-17 20:16:40 +0800374
Kim Phillips9c4a7962008-06-23 19:50:15 +0800375 saved_req.callback(dev, saved_req.desc, saved_req.context,
376 status);
377 /* channel may resume processing in single desc error case */
378 if (error && !reset_ch && status == error)
379 return;
Kim Phillips4b9926282009-08-13 11:50:38 +1000380 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
381 tail = priv->chan[ch].tail;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800382 }
383
Kim Phillips4b9926282009-08-13 11:50:38 +1000384 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800385}
386
387/*
388 * process completed requests for channels that have done status
389 */
LEROY Christophedd3c0982015-04-17 16:32:13 +0200390#define DEF_TALITOS1_DONE(name, ch_done_mask) \
391static void talitos1_done_##name(unsigned long data) \
392{ \
393 struct device *dev = (struct device *)data; \
394 struct talitos_private *priv = dev_get_drvdata(dev); \
395 unsigned long flags; \
396 \
397 if (ch_done_mask & 0x10000000) \
398 flush_channel(dev, 0, 0, 0); \
LEROY Christophedd3c0982015-04-17 16:32:13 +0200399 if (ch_done_mask & 0x40000000) \
400 flush_channel(dev, 1, 0, 0); \
401 if (ch_done_mask & 0x00010000) \
402 flush_channel(dev, 2, 0, 0); \
403 if (ch_done_mask & 0x00040000) \
404 flush_channel(dev, 3, 0, 0); \
405 \
LEROY Christophedd3c0982015-04-17 16:32:13 +0200406 /* At this point, all completed channels have been processed */ \
407 /* Unmask done interrupts for channels completed later on. */ \
408 spin_lock_irqsave(&priv->reg_lock, flags); \
409 clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
410 clrbits32(priv->reg + TALITOS_IMR_LO, TALITOS1_IMR_LO_INIT); \
411 spin_unlock_irqrestore(&priv->reg_lock, flags); \
412}
413
414DEF_TALITOS1_DONE(4ch, TALITOS1_ISR_4CHDONE)
LEROY Christophe9c02e282017-10-06 15:04:55 +0200415DEF_TALITOS1_DONE(ch0, TALITOS1_ISR_CH_0_DONE)
LEROY Christophedd3c0982015-04-17 16:32:13 +0200416
417#define DEF_TALITOS2_DONE(name, ch_done_mask) \
418static void talitos2_done_##name(unsigned long data) \
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800419{ \
420 struct device *dev = (struct device *)data; \
421 struct talitos_private *priv = dev_get_drvdata(dev); \
Horia Geanta511d63c2012-03-30 17:49:53 +0300422 unsigned long flags; \
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800423 \
424 if (ch_done_mask & 1) \
425 flush_channel(dev, 0, 0, 0); \
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800426 if (ch_done_mask & (1 << 2)) \
427 flush_channel(dev, 1, 0, 0); \
428 if (ch_done_mask & (1 << 4)) \
429 flush_channel(dev, 2, 0, 0); \
430 if (ch_done_mask & (1 << 6)) \
431 flush_channel(dev, 3, 0, 0); \
432 \
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800433 /* At this point, all completed channels have been processed */ \
434 /* Unmask done interrupts for channels completed later on. */ \
Horia Geanta511d63c2012-03-30 17:49:53 +0300435 spin_lock_irqsave(&priv->reg_lock, flags); \
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800436 setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
LEROY Christophedd3c0982015-04-17 16:32:13 +0200437 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS2_IMR_LO_INIT); \
Horia Geanta511d63c2012-03-30 17:49:53 +0300438 spin_unlock_irqrestore(&priv->reg_lock, flags); \
Kim Phillips9c4a7962008-06-23 19:50:15 +0800439}
LEROY Christophedd3c0982015-04-17 16:32:13 +0200440
441DEF_TALITOS2_DONE(4ch, TALITOS2_ISR_4CHDONE)
LEROY Christophe9c02e282017-10-06 15:04:55 +0200442DEF_TALITOS2_DONE(ch0, TALITOS2_ISR_CH_0_DONE)
LEROY Christophedd3c0982015-04-17 16:32:13 +0200443DEF_TALITOS2_DONE(ch0_2, TALITOS2_ISR_CH_0_2_DONE)
444DEF_TALITOS2_DONE(ch1_3, TALITOS2_ISR_CH_1_3_DONE)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800445
446/*
447 * locate current (offending) descriptor
448 */
Kim Phillips3e721ae2011-10-21 15:20:28 +0200449static u32 current_desc_hdr(struct device *dev, int ch)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800450{
451 struct talitos_private *priv = dev_get_drvdata(dev);
Horia Geantab62ffd82013-11-13 12:20:37 +0200452 int tail, iter;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800453 dma_addr_t cur_desc;
454
Horia Geantab62ffd82013-11-13 12:20:37 +0200455 cur_desc = ((u64)in_be32(priv->chan[ch].reg + TALITOS_CDPR)) << 32;
456 cur_desc |= in_be32(priv->chan[ch].reg + TALITOS_CDPR_LO);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800457
Horia Geantab62ffd82013-11-13 12:20:37 +0200458 if (!cur_desc) {
459 dev_err(dev, "CDPR is NULL, giving up search for offending descriptor\n");
460 return 0;
461 }
462
463 tail = priv->chan[ch].tail;
464
465 iter = tail;
LEROY Christophe37b5e882017-10-06 15:05:06 +0200466 while (priv->chan[ch].fifo[iter].dma_desc != cur_desc &&
467 priv->chan[ch].fifo[iter].desc->next_desc != cur_desc) {
Horia Geantab62ffd82013-11-13 12:20:37 +0200468 iter = (iter + 1) & (priv->fifo_len - 1);
469 if (iter == tail) {
Kim Phillips9c4a7962008-06-23 19:50:15 +0800470 dev_err(dev, "couldn't locate current descriptor\n");
Kim Phillips3e721ae2011-10-21 15:20:28 +0200471 return 0;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800472 }
473 }
474
LEROY Christophe37b5e882017-10-06 15:05:06 +0200475 if (priv->chan[ch].fifo[iter].desc->next_desc == cur_desc)
476 return (priv->chan[ch].fifo[iter].desc + 1)->hdr;
477
Horia Geantab62ffd82013-11-13 12:20:37 +0200478 return priv->chan[ch].fifo[iter].desc->hdr;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800479}
480
481/*
482 * user diagnostics; report root cause of error based on execution unit status
483 */
Kim Phillips3e721ae2011-10-21 15:20:28 +0200484static void report_eu_error(struct device *dev, int ch, u32 desc_hdr)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800485{
486 struct talitos_private *priv = dev_get_drvdata(dev);
487 int i;
488
Kim Phillips3e721ae2011-10-21 15:20:28 +0200489 if (!desc_hdr)
Kim Phillipsad42d5f2011-11-21 16:13:27 +0800490 desc_hdr = in_be32(priv->chan[ch].reg + TALITOS_DESCBUF);
Kim Phillips3e721ae2011-10-21 15:20:28 +0200491
492 switch (desc_hdr & DESC_HDR_SEL0_MASK) {
Kim Phillips9c4a7962008-06-23 19:50:15 +0800493 case DESC_HDR_SEL0_AFEU:
494 dev_err(dev, "AFEUISR 0x%08x_%08x\n",
LEROY Christophe5fa7fa12015-04-17 16:32:11 +0200495 in_be32(priv->reg_afeu + TALITOS_EUISR),
496 in_be32(priv->reg_afeu + TALITOS_EUISR_LO));
Kim Phillips9c4a7962008-06-23 19:50:15 +0800497 break;
498 case DESC_HDR_SEL0_DEU:
499 dev_err(dev, "DEUISR 0x%08x_%08x\n",
LEROY Christophe5fa7fa12015-04-17 16:32:11 +0200500 in_be32(priv->reg_deu + TALITOS_EUISR),
501 in_be32(priv->reg_deu + TALITOS_EUISR_LO));
Kim Phillips9c4a7962008-06-23 19:50:15 +0800502 break;
503 case DESC_HDR_SEL0_MDEUA:
504 case DESC_HDR_SEL0_MDEUB:
505 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
LEROY Christophe5fa7fa12015-04-17 16:32:11 +0200506 in_be32(priv->reg_mdeu + TALITOS_EUISR),
507 in_be32(priv->reg_mdeu + TALITOS_EUISR_LO));
Kim Phillips9c4a7962008-06-23 19:50:15 +0800508 break;
509 case DESC_HDR_SEL0_RNG:
510 dev_err(dev, "RNGUISR 0x%08x_%08x\n",
LEROY Christophe5fa7fa12015-04-17 16:32:11 +0200511 in_be32(priv->reg_rngu + TALITOS_ISR),
512 in_be32(priv->reg_rngu + TALITOS_ISR_LO));
Kim Phillips9c4a7962008-06-23 19:50:15 +0800513 break;
514 case DESC_HDR_SEL0_PKEU:
515 dev_err(dev, "PKEUISR 0x%08x_%08x\n",
LEROY Christophe5fa7fa12015-04-17 16:32:11 +0200516 in_be32(priv->reg_pkeu + TALITOS_EUISR),
517 in_be32(priv->reg_pkeu + TALITOS_EUISR_LO));
Kim Phillips9c4a7962008-06-23 19:50:15 +0800518 break;
519 case DESC_HDR_SEL0_AESU:
520 dev_err(dev, "AESUISR 0x%08x_%08x\n",
LEROY Christophe5fa7fa12015-04-17 16:32:11 +0200521 in_be32(priv->reg_aesu + TALITOS_EUISR),
522 in_be32(priv->reg_aesu + TALITOS_EUISR_LO));
Kim Phillips9c4a7962008-06-23 19:50:15 +0800523 break;
524 case DESC_HDR_SEL0_CRCU:
525 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
LEROY Christophe5fa7fa12015-04-17 16:32:11 +0200526 in_be32(priv->reg_crcu + TALITOS_EUISR),
527 in_be32(priv->reg_crcu + TALITOS_EUISR_LO));
Kim Phillips9c4a7962008-06-23 19:50:15 +0800528 break;
529 case DESC_HDR_SEL0_KEU:
530 dev_err(dev, "KEUISR 0x%08x_%08x\n",
LEROY Christophe5fa7fa12015-04-17 16:32:11 +0200531 in_be32(priv->reg_pkeu + TALITOS_EUISR),
532 in_be32(priv->reg_pkeu + TALITOS_EUISR_LO));
Kim Phillips9c4a7962008-06-23 19:50:15 +0800533 break;
534 }
535
Kim Phillips3e721ae2011-10-21 15:20:28 +0200536 switch (desc_hdr & DESC_HDR_SEL1_MASK) {
Kim Phillips9c4a7962008-06-23 19:50:15 +0800537 case DESC_HDR_SEL1_MDEUA:
538 case DESC_HDR_SEL1_MDEUB:
539 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
LEROY Christophe5fa7fa12015-04-17 16:32:11 +0200540 in_be32(priv->reg_mdeu + TALITOS_EUISR),
541 in_be32(priv->reg_mdeu + TALITOS_EUISR_LO));
Kim Phillips9c4a7962008-06-23 19:50:15 +0800542 break;
543 case DESC_HDR_SEL1_CRCU:
544 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
LEROY Christophe5fa7fa12015-04-17 16:32:11 +0200545 in_be32(priv->reg_crcu + TALITOS_EUISR),
546 in_be32(priv->reg_crcu + TALITOS_EUISR_LO));
Kim Phillips9c4a7962008-06-23 19:50:15 +0800547 break;
548 }
549
550 for (i = 0; i < 8; i++)
551 dev_err(dev, "DESCBUF 0x%08x_%08x\n",
Kim Phillipsad42d5f2011-11-21 16:13:27 +0800552 in_be32(priv->chan[ch].reg + TALITOS_DESCBUF + 8*i),
553 in_be32(priv->chan[ch].reg + TALITOS_DESCBUF_LO + 8*i));
Kim Phillips9c4a7962008-06-23 19:50:15 +0800554}
555
556/*
557 * recover from error interrupts
558 */
Kim Phillips5e718a02011-12-12 14:59:12 -0600559static void talitos_error(struct device *dev, u32 isr, u32 isr_lo)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800560{
Kim Phillips9c4a7962008-06-23 19:50:15 +0800561 struct talitos_private *priv = dev_get_drvdata(dev);
562 unsigned int timeout = TALITOS_TIMEOUT;
LEROY Christophedd3c0982015-04-17 16:32:13 +0200563 int ch, error, reset_dev = 0;
Horia Geant?42e8b0d2015-05-11 20:04:56 +0300564 u32 v_lo;
LEROY Christophedd3c0982015-04-17 16:32:13 +0200565 bool is_sec1 = has_ftr_sec1(priv);
566 int reset_ch = is_sec1 ? 1 : 0; /* only SEC2 supports continuation */
Kim Phillips9c4a7962008-06-23 19:50:15 +0800567
568 for (ch = 0; ch < priv->num_channels; ch++) {
569 /* skip channels without errors */
LEROY Christophedd3c0982015-04-17 16:32:13 +0200570 if (is_sec1) {
571 /* bits 29, 31, 17, 19 */
572 if (!(isr & (1 << (29 + (ch & 1) * 2 - (ch & 2) * 6))))
573 continue;
574 } else {
575 if (!(isr & (1 << (ch * 2 + 1))))
576 continue;
577 }
Kim Phillips9c4a7962008-06-23 19:50:15 +0800578
579 error = -EINVAL;
580
Kim Phillipsad42d5f2011-11-21 16:13:27 +0800581 v_lo = in_be32(priv->chan[ch].reg + TALITOS_CCPSR_LO);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800582
583 if (v_lo & TALITOS_CCPSR_LO_DOF) {
584 dev_err(dev, "double fetch fifo overflow error\n");
585 error = -EAGAIN;
586 reset_ch = 1;
587 }
588 if (v_lo & TALITOS_CCPSR_LO_SOF) {
589 /* h/w dropped descriptor */
590 dev_err(dev, "single fetch fifo overflow error\n");
591 error = -EAGAIN;
592 }
593 if (v_lo & TALITOS_CCPSR_LO_MDTE)
594 dev_err(dev, "master data transfer error\n");
595 if (v_lo & TALITOS_CCPSR_LO_SGDLZ)
Colin Ian King4d9b3a52016-11-01 20:14:04 -0600596 dev_err(dev, is_sec1 ? "pointer not complete error\n"
LEROY Christophedd3c0982015-04-17 16:32:13 +0200597 : "s/g data length zero error\n");
Kim Phillips9c4a7962008-06-23 19:50:15 +0800598 if (v_lo & TALITOS_CCPSR_LO_FPZ)
LEROY Christophedd3c0982015-04-17 16:32:13 +0200599 dev_err(dev, is_sec1 ? "parity error\n"
600 : "fetch pointer zero error\n");
Kim Phillips9c4a7962008-06-23 19:50:15 +0800601 if (v_lo & TALITOS_CCPSR_LO_IDH)
602 dev_err(dev, "illegal descriptor header error\n");
603 if (v_lo & TALITOS_CCPSR_LO_IEU)
LEROY Christophedd3c0982015-04-17 16:32:13 +0200604 dev_err(dev, is_sec1 ? "static assignment error\n"
605 : "invalid exec unit error\n");
Kim Phillips9c4a7962008-06-23 19:50:15 +0800606 if (v_lo & TALITOS_CCPSR_LO_EU)
Kim Phillips3e721ae2011-10-21 15:20:28 +0200607 report_eu_error(dev, ch, current_desc_hdr(dev, ch));
LEROY Christophedd3c0982015-04-17 16:32:13 +0200608 if (!is_sec1) {
609 if (v_lo & TALITOS_CCPSR_LO_GB)
610 dev_err(dev, "gather boundary error\n");
611 if (v_lo & TALITOS_CCPSR_LO_GRL)
612 dev_err(dev, "gather return/length error\n");
613 if (v_lo & TALITOS_CCPSR_LO_SB)
614 dev_err(dev, "scatter boundary error\n");
615 if (v_lo & TALITOS_CCPSR_LO_SRL)
616 dev_err(dev, "scatter return/length error\n");
617 }
Kim Phillips9c4a7962008-06-23 19:50:15 +0800618
619 flush_channel(dev, ch, error, reset_ch);
620
621 if (reset_ch) {
622 reset_channel(dev, ch);
623 } else {
Kim Phillipsad42d5f2011-11-21 16:13:27 +0800624 setbits32(priv->chan[ch].reg + TALITOS_CCCR,
LEROY Christophedd3c0982015-04-17 16:32:13 +0200625 TALITOS2_CCCR_CONT);
Kim Phillipsad42d5f2011-11-21 16:13:27 +0800626 setbits32(priv->chan[ch].reg + TALITOS_CCCR_LO, 0);
627 while ((in_be32(priv->chan[ch].reg + TALITOS_CCCR) &
LEROY Christophedd3c0982015-04-17 16:32:13 +0200628 TALITOS2_CCCR_CONT) && --timeout)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800629 cpu_relax();
630 if (timeout == 0) {
631 dev_err(dev, "failed to restart channel %d\n",
632 ch);
633 reset_dev = 1;
634 }
635 }
636 }
LEROY Christophedd3c0982015-04-17 16:32:13 +0200637 if (reset_dev || (is_sec1 && isr & ~TALITOS1_ISR_4CHERR) ||
638 (!is_sec1 && isr & ~TALITOS2_ISR_4CHERR) || isr_lo) {
639 if (is_sec1 && (isr_lo & TALITOS1_ISR_TEA_ERR))
640 dev_err(dev, "TEA error: ISR 0x%08x_%08x\n",
641 isr, isr_lo);
642 else
643 dev_err(dev, "done overflow, internal time out, or "
644 "rngu error: ISR 0x%08x_%08x\n", isr, isr_lo);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800645
646 /* purge request queues */
647 for (ch = 0; ch < priv->num_channels; ch++)
648 flush_channel(dev, ch, -EIO, 1);
649
650 /* reset and reinitialize the device */
651 init_device(dev);
652 }
653}
654
LEROY Christophedd3c0982015-04-17 16:32:13 +0200655#define DEF_TALITOS1_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \
656static irqreturn_t talitos1_interrupt_##name(int irq, void *data) \
657{ \
658 struct device *dev = data; \
659 struct talitos_private *priv = dev_get_drvdata(dev); \
660 u32 isr, isr_lo; \
661 unsigned long flags; \
662 \
663 spin_lock_irqsave(&priv->reg_lock, flags); \
664 isr = in_be32(priv->reg + TALITOS_ISR); \
665 isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \
666 /* Acknowledge interrupt */ \
667 out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
668 out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \
669 \
670 if (unlikely(isr & ch_err_mask || isr_lo & TALITOS1_IMR_LO_INIT)) { \
671 spin_unlock_irqrestore(&priv->reg_lock, flags); \
672 talitos_error(dev, isr & ch_err_mask, isr_lo); \
673 } \
674 else { \
675 if (likely(isr & ch_done_mask)) { \
676 /* mask further done interrupts. */ \
677 setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
678 /* done_task will unmask done interrupts at exit */ \
679 tasklet_schedule(&priv->done_task[tlet]); \
680 } \
681 spin_unlock_irqrestore(&priv->reg_lock, flags); \
682 } \
683 \
684 return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \
685 IRQ_NONE; \
686}
687
688DEF_TALITOS1_INTERRUPT(4ch, TALITOS1_ISR_4CHDONE, TALITOS1_ISR_4CHERR, 0)
689
690#define DEF_TALITOS2_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \
691static irqreturn_t talitos2_interrupt_##name(int irq, void *data) \
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800692{ \
693 struct device *dev = data; \
694 struct talitos_private *priv = dev_get_drvdata(dev); \
695 u32 isr, isr_lo; \
Horia Geanta511d63c2012-03-30 17:49:53 +0300696 unsigned long flags; \
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800697 \
Horia Geanta511d63c2012-03-30 17:49:53 +0300698 spin_lock_irqsave(&priv->reg_lock, flags); \
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800699 isr = in_be32(priv->reg + TALITOS_ISR); \
700 isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \
701 /* Acknowledge interrupt */ \
702 out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
703 out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \
704 \
Horia Geanta511d63c2012-03-30 17:49:53 +0300705 if (unlikely(isr & ch_err_mask || isr_lo)) { \
706 spin_unlock_irqrestore(&priv->reg_lock, flags); \
707 talitos_error(dev, isr & ch_err_mask, isr_lo); \
708 } \
709 else { \
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800710 if (likely(isr & ch_done_mask)) { \
711 /* mask further done interrupts. */ \
712 clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
713 /* done_task will unmask done interrupts at exit */ \
714 tasklet_schedule(&priv->done_task[tlet]); \
715 } \
Horia Geanta511d63c2012-03-30 17:49:53 +0300716 spin_unlock_irqrestore(&priv->reg_lock, flags); \
717 } \
Kim Phillipsc3e337f2011-11-21 16:13:27 +0800718 \
719 return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \
720 IRQ_NONE; \
Kim Phillips9c4a7962008-06-23 19:50:15 +0800721}
LEROY Christophedd3c0982015-04-17 16:32:13 +0200722
723DEF_TALITOS2_INTERRUPT(4ch, TALITOS2_ISR_4CHDONE, TALITOS2_ISR_4CHERR, 0)
724DEF_TALITOS2_INTERRUPT(ch0_2, TALITOS2_ISR_CH_0_2_DONE, TALITOS2_ISR_CH_0_2_ERR,
725 0)
726DEF_TALITOS2_INTERRUPT(ch1_3, TALITOS2_ISR_CH_1_3_DONE, TALITOS2_ISR_CH_1_3_ERR,
727 1)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800728
729/*
730 * hwrng
731 */
732static int talitos_rng_data_present(struct hwrng *rng, int wait)
733{
734 struct device *dev = (struct device *)rng->priv;
735 struct talitos_private *priv = dev_get_drvdata(dev);
736 u32 ofl;
737 int i;
738
739 for (i = 0; i < 20; i++) {
LEROY Christophe5fa7fa12015-04-17 16:32:11 +0200740 ofl = in_be32(priv->reg_rngu + TALITOS_EUSR_LO) &
Kim Phillips9c4a7962008-06-23 19:50:15 +0800741 TALITOS_RNGUSR_LO_OFL;
742 if (ofl || !wait)
743 break;
744 udelay(10);
745 }
746
747 return !!ofl;
748}
749
750static int talitos_rng_data_read(struct hwrng *rng, u32 *data)
751{
752 struct device *dev = (struct device *)rng->priv;
753 struct talitos_private *priv = dev_get_drvdata(dev);
754
755 /* rng fifo requires 64-bit accesses */
LEROY Christophe5fa7fa12015-04-17 16:32:11 +0200756 *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO);
757 *data = in_be32(priv->reg_rngu + TALITOS_EU_FIFO_LO);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800758
759 return sizeof(u32);
760}
761
762static int talitos_rng_init(struct hwrng *rng)
763{
764 struct device *dev = (struct device *)rng->priv;
765 struct talitos_private *priv = dev_get_drvdata(dev);
766 unsigned int timeout = TALITOS_TIMEOUT;
767
LEROY Christophe5fa7fa12015-04-17 16:32:11 +0200768 setbits32(priv->reg_rngu + TALITOS_EURCR_LO, TALITOS_RNGURCR_LO_SR);
769 while (!(in_be32(priv->reg_rngu + TALITOS_EUSR_LO)
770 & TALITOS_RNGUSR_LO_RD)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800771 && --timeout)
772 cpu_relax();
773 if (timeout == 0) {
774 dev_err(dev, "failed to reset rng hw\n");
775 return -ENODEV;
776 }
777
778 /* start generating */
LEROY Christophe5fa7fa12015-04-17 16:32:11 +0200779 setbits32(priv->reg_rngu + TALITOS_EUDSR_LO, 0);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800780
781 return 0;
782}
783
784static int talitos_register_rng(struct device *dev)
785{
786 struct talitos_private *priv = dev_get_drvdata(dev);
Aaron Sierra35a3bb32015-08-05 16:52:08 -0500787 int err;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800788
789 priv->rng.name = dev_driver_string(dev),
790 priv->rng.init = talitos_rng_init,
791 priv->rng.data_present = talitos_rng_data_present,
792 priv->rng.data_read = talitos_rng_data_read,
793 priv->rng.priv = (unsigned long)dev;
794
Aaron Sierra35a3bb32015-08-05 16:52:08 -0500795 err = hwrng_register(&priv->rng);
796 if (!err)
797 priv->rng_registered = true;
798
799 return err;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800800}
801
802static void talitos_unregister_rng(struct device *dev)
803{
804 struct talitos_private *priv = dev_get_drvdata(dev);
805
Aaron Sierra35a3bb32015-08-05 16:52:08 -0500806 if (!priv->rng_registered)
807 return;
808
Kim Phillips9c4a7962008-06-23 19:50:15 +0800809 hwrng_unregister(&priv->rng);
Aaron Sierra35a3bb32015-08-05 16:52:08 -0500810 priv->rng_registered = false;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800811}
812
813/*
814 * crypto alg
815 */
816#define TALITOS_CRA_PRIORITY 3000
LEROY Christophe7405c8d2016-06-06 13:20:46 +0200817/*
818 * Defines a priority for doing AEAD with descriptors type
819 * HMAC_SNOOP_NO_AFEA (HSNA) instead of type IPSEC_ESP
820 */
821#define TALITOS_CRA_PRIORITY_AEAD_HSNA (TALITOS_CRA_PRIORITY - 1)
Martin Hicks03d2c512017-05-02 09:38:35 -0400822#define TALITOS_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + SHA512_BLOCK_SIZE)
Lee Nipper3952f172008-07-10 18:29:18 +0800823#define TALITOS_MAX_IV_LENGTH 16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
Lee Nipper70bcaca2008-07-03 19:08:46 +0800824
Kim Phillips9c4a7962008-06-23 19:50:15 +0800825struct talitos_ctx {
826 struct device *dev;
Kim Phillips5228f0f2011-07-15 11:21:38 +0800827 int ch;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800828 __be32 desc_hdr_template;
829 u8 key[TALITOS_MAX_KEY_SIZE];
Lee Nipper70bcaca2008-07-03 19:08:46 +0800830 u8 iv[TALITOS_MAX_IV_LENGTH];
LEROY Christophe2e13ce02017-10-06 15:05:02 +0200831 dma_addr_t dma_key;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800832 unsigned int keylen;
833 unsigned int enckeylen;
834 unsigned int authkeylen;
LEROY Christophe37b5e882017-10-06 15:05:06 +0200835 dma_addr_t dma_buf;
LEROY Christophe49f97832017-10-06 15:05:04 +0200836 dma_addr_t dma_hw_context;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800837};
838
Lee Nipper497f2e62010-05-19 19:20:36 +1000839#define HASH_MAX_BLOCK_SIZE SHA512_BLOCK_SIZE
840#define TALITOS_MDEU_MAX_CONTEXT_SIZE TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512
841
842struct talitos_ahash_req_ctx {
Kim Phillips60f208d2010-05-19 19:21:53 +1000843 u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
Lee Nipper497f2e62010-05-19 19:20:36 +1000844 unsigned int hw_context_size;
845 u8 buf[HASH_MAX_BLOCK_SIZE];
846 u8 bufnext[HASH_MAX_BLOCK_SIZE];
Kim Phillips60f208d2010-05-19 19:21:53 +1000847 unsigned int swinit;
Lee Nipper497f2e62010-05-19 19:20:36 +1000848 unsigned int first;
849 unsigned int last;
850 unsigned int to_hash_later;
Horia Geant?42e8b0d2015-05-11 20:04:56 +0300851 unsigned int nbuf;
Lee Nipper497f2e62010-05-19 19:20:36 +1000852 struct scatterlist bufsl[2];
853 struct scatterlist *psrc;
854};
855
Horia Geant?3639ca82016-04-21 19:24:55 +0300856struct talitos_export_state {
857 u32 hw_context[TALITOS_MDEU_MAX_CONTEXT_SIZE / sizeof(u32)];
858 u8 buf[HASH_MAX_BLOCK_SIZE];
859 unsigned int swinit;
860 unsigned int first;
861 unsigned int last;
862 unsigned int to_hash_later;
863 unsigned int nbuf;
864};
865
Lee Nipper56af8cd2009-03-29 15:50:50 +0800866static int aead_setkey(struct crypto_aead *authenc,
867 const u8 *key, unsigned int keylen)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800868{
869 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
LEROY Christophe2e13ce02017-10-06 15:05:02 +0200870 struct device *dev = ctx->dev;
Mathias Krausec306a982013-10-15 13:49:34 +0200871 struct crypto_authenc_keys keys;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800872
Mathias Krausec306a982013-10-15 13:49:34 +0200873 if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800874 goto badkey;
875
Mathias Krausec306a982013-10-15 13:49:34 +0200876 if (keys.authkeylen + keys.enckeylen > TALITOS_MAX_KEY_SIZE)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800877 goto badkey;
878
LEROY Christophe2e13ce02017-10-06 15:05:02 +0200879 if (ctx->keylen)
880 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE);
881
Mathias Krausec306a982013-10-15 13:49:34 +0200882 memcpy(ctx->key, keys.authkey, keys.authkeylen);
883 memcpy(&ctx->key[keys.authkeylen], keys.enckey, keys.enckeylen);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800884
Mathias Krausec306a982013-10-15 13:49:34 +0200885 ctx->keylen = keys.authkeylen + keys.enckeylen;
886 ctx->enckeylen = keys.enckeylen;
887 ctx->authkeylen = keys.authkeylen;
LEROY Christophe2e13ce02017-10-06 15:05:02 +0200888 ctx->dma_key = dma_map_single(dev, ctx->key, ctx->keylen,
889 DMA_TO_DEVICE);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800890
891 return 0;
892
893badkey:
894 crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
895 return -EINVAL;
896}
897
898/*
Lee Nipper56af8cd2009-03-29 15:50:50 +0800899 * talitos_edesc - s/w-extended descriptor
Kim Phillips9c4a7962008-06-23 19:50:15 +0800900 * @src_nents: number of segments in input scatterlist
901 * @dst_nents: number of segments in output scatterlist
Herbert Xuaeb4c132015-07-30 17:53:22 +0800902 * @icv_ool: whether ICV is out-of-line
Horia Geanta79fd31d2012-08-02 17:16:40 +0300903 * @iv_dma: dma address of iv for checking continuity and link table
Kim Phillips9c4a7962008-06-23 19:50:15 +0800904 * @dma_len: length of dma mapped link_tbl space
LEROY Christophe6f65f6a2015-04-17 16:32:15 +0200905 * @dma_link_tbl: bus physical address of link_tbl/buf
Kim Phillips9c4a7962008-06-23 19:50:15 +0800906 * @desc: h/w descriptor
LEROY Christophe6f65f6a2015-04-17 16:32:15 +0200907 * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1) (SEC2)
908 * @buf: input and output buffeur (if {src,dst}_nents > 1) (SEC1)
Kim Phillips9c4a7962008-06-23 19:50:15 +0800909 *
910 * if decrypting (with authcheck), or either one of src_nents or dst_nents
911 * is greater than 1, an integrity check value is concatenated to the end
912 * of link_tbl data
913 */
Lee Nipper56af8cd2009-03-29 15:50:50 +0800914struct talitos_edesc {
Kim Phillips9c4a7962008-06-23 19:50:15 +0800915 int src_nents;
916 int dst_nents;
Herbert Xuaeb4c132015-07-30 17:53:22 +0800917 bool icv_ool;
Horia Geanta79fd31d2012-08-02 17:16:40 +0300918 dma_addr_t iv_dma;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800919 int dma_len;
920 dma_addr_t dma_link_tbl;
921 struct talitos_desc desc;
LEROY Christophe6f65f6a2015-04-17 16:32:15 +0200922 union {
923 struct talitos_ptr link_tbl[0];
924 u8 buf[0];
925 };
Kim Phillips9c4a7962008-06-23 19:50:15 +0800926};
927
Lee Nipper4de9d0b2009-03-29 15:52:32 +0800928static void talitos_sg_unmap(struct device *dev,
929 struct talitos_edesc *edesc,
930 struct scatterlist *src,
LEROY Christophe6a1e8d12016-06-06 13:20:38 +0200931 struct scatterlist *dst,
932 unsigned int len, unsigned int offset)
LEROY Christophe246a87c2016-06-06 13:20:36 +0200933{
934 struct talitos_private *priv = dev_get_drvdata(dev);
935 bool is_sec1 = has_ftr_sec1(priv);
LEROY Christophe6a1e8d12016-06-06 13:20:38 +0200936 unsigned int src_nents = edesc->src_nents ? : 1;
937 unsigned int dst_nents = edesc->dst_nents ? : 1;
LEROY Christophe246a87c2016-06-06 13:20:36 +0200938
LEROY Christophe6a1e8d12016-06-06 13:20:38 +0200939 if (is_sec1 && dst && dst_nents > 1) {
940 dma_sync_single_for_device(dev, edesc->dma_link_tbl + offset,
941 len, DMA_FROM_DEVICE);
942 sg_pcopy_from_buffer(dst, dst_nents, edesc->buf + offset, len,
943 offset);
944 }
945 if (src != dst) {
946 if (src_nents == 1 || !is_sec1)
947 dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
948
949 if (dst && (dst_nents == 1 || !is_sec1))
950 dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
951 } else if (src_nents == 1 || !is_sec1) {
952 dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
LEROY Christophe246a87c2016-06-06 13:20:36 +0200953 }
954}
955
Kim Phillips9c4a7962008-06-23 19:50:15 +0800956static void ipsec_esp_unmap(struct device *dev,
Lee Nipper56af8cd2009-03-29 15:50:50 +0800957 struct talitos_edesc *edesc,
Kim Phillips9c4a7962008-06-23 19:50:15 +0800958 struct aead_request *areq)
959{
LEROY Christophe549bd8b2016-06-06 13:20:40 +0200960 struct crypto_aead *aead = crypto_aead_reqtfm(areq);
961 struct talitos_ctx *ctx = crypto_aead_ctx(aead);
962 unsigned int ivsize = crypto_aead_ivsize(aead);
LEROY Christophe9a655602017-10-06 15:04:59 +0200963 bool is_ipsec_esp = edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP;
964 struct talitos_ptr *civ_ptr = &edesc->desc.ptr[is_ipsec_esp ? 2 : 3];
LEROY Christophe549bd8b2016-06-06 13:20:40 +0200965
LEROY Christophe9a655602017-10-06 15:04:59 +0200966 if (is_ipsec_esp)
LEROY Christophe549bd8b2016-06-06 13:20:40 +0200967 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6],
968 DMA_FROM_DEVICE);
LEROY Christophe9a655602017-10-06 15:04:59 +0200969 unmap_single_talitos_ptr(dev, civ_ptr, DMA_TO_DEVICE);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800970
LEROY Christophe6a1e8d12016-06-06 13:20:38 +0200971 talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->cryptlen,
972 areq->assoclen);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800973
974 if (edesc->dma_len)
975 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
976 DMA_BIDIRECTIONAL);
LEROY Christophe549bd8b2016-06-06 13:20:40 +0200977
LEROY Christophe9a655602017-10-06 15:04:59 +0200978 if (!is_ipsec_esp) {
LEROY Christophe549bd8b2016-06-06 13:20:40 +0200979 unsigned int dst_nents = edesc->dst_nents ? : 1;
980
981 sg_pcopy_to_buffer(areq->dst, dst_nents, ctx->iv, ivsize,
982 areq->assoclen + areq->cryptlen - ivsize);
983 }
Kim Phillips9c4a7962008-06-23 19:50:15 +0800984}
985
986/*
987 * ipsec_esp descriptor callbacks
988 */
989static void ipsec_esp_encrypt_done(struct device *dev,
990 struct talitos_desc *desc, void *context,
991 int err)
992{
LEROY Christophe549bd8b2016-06-06 13:20:40 +0200993 struct talitos_private *priv = dev_get_drvdata(dev);
994 bool is_sec1 = has_ftr_sec1(priv);
Kim Phillips9c4a7962008-06-23 19:50:15 +0800995 struct aead_request *areq = context;
Kim Phillips9c4a7962008-06-23 19:50:15 +0800996 struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
Herbert Xuaeb4c132015-07-30 17:53:22 +0800997 unsigned int authsize = crypto_aead_authsize(authenc);
LEROY Christophe2e13ce02017-10-06 15:05:02 +0200998 unsigned int ivsize = crypto_aead_ivsize(authenc);
Kim Phillips19bbbc62009-03-29 15:53:59 +0800999 struct talitos_edesc *edesc;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001000 struct scatterlist *sg;
1001 void *icvdata;
1002
Kim Phillips19bbbc62009-03-29 15:53:59 +08001003 edesc = container_of(desc, struct talitos_edesc, desc);
1004
Kim Phillips9c4a7962008-06-23 19:50:15 +08001005 ipsec_esp_unmap(dev, edesc, areq);
1006
1007 /* copy the generated ICV to dst */
Herbert Xuaeb4c132015-07-30 17:53:22 +08001008 if (edesc->icv_ool) {
LEROY Christophe549bd8b2016-06-06 13:20:40 +02001009 if (is_sec1)
1010 icvdata = edesc->buf + areq->assoclen + areq->cryptlen;
1011 else
1012 icvdata = &edesc->link_tbl[edesc->src_nents +
1013 edesc->dst_nents + 2];
Kim Phillips9c4a7962008-06-23 19:50:15 +08001014 sg = sg_last(areq->dst, edesc->dst_nents);
Herbert Xuaeb4c132015-07-30 17:53:22 +08001015 memcpy((char *)sg_virt(sg) + sg->length - authsize,
1016 icvdata, authsize);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001017 }
1018
LEROY Christophe2e13ce02017-10-06 15:05:02 +02001019 dma_unmap_single(dev, edesc->iv_dma, ivsize, DMA_TO_DEVICE);
1020
Kim Phillips9c4a7962008-06-23 19:50:15 +08001021 kfree(edesc);
1022
1023 aead_request_complete(areq, err);
1024}
1025
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001026static void ipsec_esp_decrypt_swauth_done(struct device *dev,
Kim Phillipse938e462009-03-29 15:53:23 +08001027 struct talitos_desc *desc,
1028 void *context, int err)
Kim Phillips9c4a7962008-06-23 19:50:15 +08001029{
1030 struct aead_request *req = context;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001031 struct crypto_aead *authenc = crypto_aead_reqtfm(req);
Herbert Xuaeb4c132015-07-30 17:53:22 +08001032 unsigned int authsize = crypto_aead_authsize(authenc);
Kim Phillips19bbbc62009-03-29 15:53:59 +08001033 struct talitos_edesc *edesc;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001034 struct scatterlist *sg;
Herbert Xuaeb4c132015-07-30 17:53:22 +08001035 char *oicv, *icv;
LEROY Christophe549bd8b2016-06-06 13:20:40 +02001036 struct talitos_private *priv = dev_get_drvdata(dev);
1037 bool is_sec1 = has_ftr_sec1(priv);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001038
Kim Phillips19bbbc62009-03-29 15:53:59 +08001039 edesc = container_of(desc, struct talitos_edesc, desc);
1040
Kim Phillips9c4a7962008-06-23 19:50:15 +08001041 ipsec_esp_unmap(dev, edesc, req);
1042
1043 if (!err) {
1044 /* auth check */
Kim Phillips9c4a7962008-06-23 19:50:15 +08001045 sg = sg_last(req->dst, edesc->dst_nents ? : 1);
Herbert Xuaeb4c132015-07-30 17:53:22 +08001046 icv = (char *)sg_virt(sg) + sg->length - authsize;
1047
1048 if (edesc->dma_len) {
LEROY Christophe549bd8b2016-06-06 13:20:40 +02001049 if (is_sec1)
1050 oicv = (char *)&edesc->dma_link_tbl +
1051 req->assoclen + req->cryptlen;
1052 else
1053 oicv = (char *)
1054 &edesc->link_tbl[edesc->src_nents +
Herbert Xuaeb4c132015-07-30 17:53:22 +08001055 edesc->dst_nents + 2];
1056 if (edesc->icv_ool)
1057 icv = oicv + authsize;
1058 } else
1059 oicv = (char *)&edesc->link_tbl[0];
1060
David Gstir79960942015-11-15 17:14:42 +01001061 err = crypto_memneq(oicv, icv, authsize) ? -EBADMSG : 0;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001062 }
1063
1064 kfree(edesc);
1065
1066 aead_request_complete(req, err);
1067}
1068
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001069static void ipsec_esp_decrypt_hwauth_done(struct device *dev,
Kim Phillipse938e462009-03-29 15:53:23 +08001070 struct talitos_desc *desc,
1071 void *context, int err)
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001072{
1073 struct aead_request *req = context;
Kim Phillips19bbbc62009-03-29 15:53:59 +08001074 struct talitos_edesc *edesc;
1075
1076 edesc = container_of(desc, struct talitos_edesc, desc);
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001077
1078 ipsec_esp_unmap(dev, edesc, req);
1079
1080 /* check ICV auth status */
Kim Phillipse938e462009-03-29 15:53:23 +08001081 if (!err && ((desc->hdr_lo & DESC_HDR_LO_ICCR1_MASK) !=
1082 DESC_HDR_LO_ICCR1_PASS))
1083 err = -EBADMSG;
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001084
1085 kfree(edesc);
1086
1087 aead_request_complete(req, err);
1088}
1089
Kim Phillips9c4a7962008-06-23 19:50:15 +08001090/*
1091 * convert scatterlist to SEC h/w link table format
1092 * stop at cryptlen bytes
1093 */
Herbert Xuaeb4c132015-07-30 17:53:22 +08001094static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count,
1095 unsigned int offset, int cryptlen,
1096 struct talitos_ptr *link_tbl_ptr)
Kim Phillips9c4a7962008-06-23 19:50:15 +08001097{
Lee Nipper70bcaca2008-07-03 19:08:46 +08001098 int n_sg = sg_count;
Herbert Xuaeb4c132015-07-30 17:53:22 +08001099 int count = 0;
Lee Nipper70bcaca2008-07-03 19:08:46 +08001100
Herbert Xuaeb4c132015-07-30 17:53:22 +08001101 while (cryptlen && sg && n_sg--) {
1102 unsigned int len = sg_dma_len(sg);
1103
1104 if (offset >= len) {
1105 offset -= len;
1106 goto next;
1107 }
1108
1109 len -= offset;
1110
1111 if (len > cryptlen)
1112 len = cryptlen;
1113
1114 to_talitos_ptr(link_tbl_ptr + count,
LEROY Christopheda9de142017-10-06 15:04:57 +02001115 sg_dma_address(sg) + offset, len, 0);
LEROY Christopheb096b542016-06-06 13:20:34 +02001116 to_talitos_ptr_ext_set(link_tbl_ptr + count, 0, 0);
Herbert Xuaeb4c132015-07-30 17:53:22 +08001117 count++;
1118 cryptlen -= len;
1119 offset = 0;
1120
1121next:
Cristian Stoica5be4d4c2015-01-20 10:06:16 +02001122 sg = sg_next(sg);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001123 }
1124
Kim Phillips9c4a7962008-06-23 19:50:15 +08001125 /* tag end of link table */
Herbert Xuaeb4c132015-07-30 17:53:22 +08001126 if (count > 0)
LEROY Christopheb096b542016-06-06 13:20:34 +02001127 to_talitos_ptr_ext_set(link_tbl_ptr + count - 1,
1128 DESC_PTR_LNKTBL_RETURN, 0);
Lee Nipper70bcaca2008-07-03 19:08:46 +08001129
Herbert Xuaeb4c132015-07-30 17:53:22 +08001130 return count;
1131}
1132
LEROY Christophe5b2cf262017-10-06 15:04:47 +02001133static int talitos_sg_map(struct device *dev, struct scatterlist *src,
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001134 unsigned int len, struct talitos_edesc *edesc,
1135 struct talitos_ptr *ptr,
1136 int sg_count, unsigned int offset, int tbl_off)
LEROY Christophe246a87c2016-06-06 13:20:36 +02001137{
LEROY Christophe246a87c2016-06-06 13:20:36 +02001138 struct talitos_private *priv = dev_get_drvdata(dev);
1139 bool is_sec1 = has_ftr_sec1(priv);
1140
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001141 if (sg_count == 1) {
LEROY Christopheda9de142017-10-06 15:04:57 +02001142 to_talitos_ptr(ptr, sg_dma_address(src) + offset, len, is_sec1);
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001143 return sg_count;
LEROY Christophe246a87c2016-06-06 13:20:36 +02001144 }
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001145 if (is_sec1) {
LEROY Christopheda9de142017-10-06 15:04:57 +02001146 to_talitos_ptr(ptr, edesc->dma_link_tbl + offset, len, is_sec1);
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001147 return sg_count;
1148 }
1149 sg_count = sg_to_link_tbl_offset(src, sg_count, offset, len,
1150 &edesc->link_tbl[tbl_off]);
1151 if (sg_count == 1) {
1152 /* Only one segment now, so no link tbl needed*/
1153 copy_talitos_ptr(ptr, &edesc->link_tbl[tbl_off], is_sec1);
1154 return sg_count;
1155 }
1156 to_talitos_ptr(ptr, edesc->dma_link_tbl +
LEROY Christopheda9de142017-10-06 15:04:57 +02001157 tbl_off * sizeof(struct talitos_ptr), len, is_sec1);
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001158 to_talitos_ptr_ext_or(ptr, DESC_PTR_LNKTBL_JUMP, is_sec1);
1159
LEROY Christophe246a87c2016-06-06 13:20:36 +02001160 return sg_count;
1161}
1162
Kim Phillips9c4a7962008-06-23 19:50:15 +08001163/*
1164 * fill in and submit ipsec_esp descriptor
1165 */
Lee Nipper56af8cd2009-03-29 15:50:50 +08001166static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,
Herbert Xuaeb4c132015-07-30 17:53:22 +08001167 void (*callback)(struct device *dev,
1168 struct talitos_desc *desc,
1169 void *context, int error))
Kim Phillips9c4a7962008-06-23 19:50:15 +08001170{
1171 struct crypto_aead *aead = crypto_aead_reqtfm(areq);
Herbert Xuaeb4c132015-07-30 17:53:22 +08001172 unsigned int authsize = crypto_aead_authsize(aead);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001173 struct talitos_ctx *ctx = crypto_aead_ctx(aead);
1174 struct device *dev = ctx->dev;
1175 struct talitos_desc *desc = &edesc->desc;
1176 unsigned int cryptlen = areq->cryptlen;
Kim Phillipse41256f2009-08-13 11:49:06 +10001177 unsigned int ivsize = crypto_aead_ivsize(aead);
Herbert Xuaeb4c132015-07-30 17:53:22 +08001178 int tbl_off = 0;
Kim Phillipsfa86a262008-07-17 20:20:06 +08001179 int sg_count, ret;
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001180 int sg_link_tbl_len;
LEROY Christophe549bd8b2016-06-06 13:20:40 +02001181 bool sync_needed = false;
1182 struct talitos_private *priv = dev_get_drvdata(dev);
1183 bool is_sec1 = has_ftr_sec1(priv);
LEROY Christophe9a655602017-10-06 15:04:59 +02001184 bool is_ipsec_esp = desc->hdr & DESC_HDR_TYPE_IPSEC_ESP;
1185 struct talitos_ptr *civ_ptr = &desc->ptr[is_ipsec_esp ? 2 : 3];
1186 struct talitos_ptr *ckey_ptr = &desc->ptr[is_ipsec_esp ? 3 : 2];
Kim Phillips9c4a7962008-06-23 19:50:15 +08001187
1188 /* hmac key */
LEROY Christophe2e13ce02017-10-06 15:05:02 +02001189 to_talitos_ptr(&desc->ptr[0], ctx->dma_key, ctx->authkeylen, is_sec1);
Horia Geanta79fd31d2012-08-02 17:16:40 +03001190
LEROY Christophe549bd8b2016-06-06 13:20:40 +02001191 sg_count = edesc->src_nents ?: 1;
1192 if (is_sec1 && sg_count > 1)
1193 sg_copy_to_buffer(areq->src, sg_count, edesc->buf,
1194 areq->assoclen + cryptlen);
1195 else
1196 sg_count = dma_map_sg(dev, areq->src, sg_count,
1197 (areq->src == areq->dst) ?
1198 DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1199
Kim Phillips9c4a7962008-06-23 19:50:15 +08001200 /* hmac data */
LEROY Christophe549bd8b2016-06-06 13:20:40 +02001201 ret = talitos_sg_map(dev, areq->src, areq->assoclen, edesc,
1202 &desc->ptr[1], sg_count, 0, tbl_off);
Horia Geanta79fd31d2012-08-02 17:16:40 +03001203
LEROY Christophe549bd8b2016-06-06 13:20:40 +02001204 if (ret > 1) {
Horia Geant?340ff602016-04-19 20:33:48 +03001205 tbl_off += ret;
LEROY Christophe549bd8b2016-06-06 13:20:40 +02001206 sync_needed = true;
Horia Geanta79fd31d2012-08-02 17:16:40 +03001207 }
1208
Kim Phillips9c4a7962008-06-23 19:50:15 +08001209 /* cipher iv */
LEROY Christophe9a655602017-10-06 15:04:59 +02001210 to_talitos_ptr(civ_ptr, edesc->iv_dma, ivsize, is_sec1);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001211
1212 /* cipher key */
LEROY Christophe2e13ce02017-10-06 15:05:02 +02001213 to_talitos_ptr(ckey_ptr, ctx->dma_key + ctx->authkeylen,
1214 ctx->enckeylen, is_sec1);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001215
1216 /*
1217 * cipher in
1218 * map and adjust cipher len to aead request cryptlen.
1219 * extent is bytes of HMAC postpended to ciphertext,
1220 * typically 12 for ipsec
1221 */
Herbert Xuaeb4c132015-07-30 17:53:22 +08001222 sg_link_tbl_len = cryptlen;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001223
LEROY Christophe9a655602017-10-06 15:04:59 +02001224 if (is_ipsec_esp) {
LEROY Christophe549bd8b2016-06-06 13:20:40 +02001225 to_talitos_ptr_ext_set(&desc->ptr[4], authsize, is_sec1);
1226
LEROY Christophe9a655602017-10-06 15:04:59 +02001227 if (desc->hdr & DESC_HDR_MODE1_MDEU_CICV)
LEROY Christophe549bd8b2016-06-06 13:20:40 +02001228 sg_link_tbl_len += authsize;
1229 }
1230
LEROY Christophefbb22132017-10-06 15:04:41 +02001231 ret = talitos_sg_map(dev, areq->src, sg_link_tbl_len, edesc,
1232 &desc->ptr[4], sg_count, areq->assoclen, tbl_off);
LEROY Christophe549bd8b2016-06-06 13:20:40 +02001233
LEROY Christopheec8c7d12017-10-06 15:04:33 +02001234 if (ret > 1) {
1235 tbl_off += ret;
LEROY Christophe549bd8b2016-06-06 13:20:40 +02001236 sync_needed = true;
Horia Geant?340ff602016-04-19 20:33:48 +03001237 }
Kim Phillips9c4a7962008-06-23 19:50:15 +08001238
1239 /* cipher out */
LEROY Christophe549bd8b2016-06-06 13:20:40 +02001240 if (areq->src != areq->dst) {
1241 sg_count = edesc->dst_nents ? : 1;
1242 if (!is_sec1 || sg_count == 1)
1243 dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE);
1244 }
Kim Phillips9c4a7962008-06-23 19:50:15 +08001245
LEROY Christophee04a61b2017-10-06 15:04:35 +02001246 ret = talitos_sg_map(dev, areq->dst, cryptlen, edesc, &desc->ptr[5],
1247 sg_count, areq->assoclen, tbl_off);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001248
LEROY Christophe9a655602017-10-06 15:04:59 +02001249 if (is_ipsec_esp)
LEROY Christophe549bd8b2016-06-06 13:20:40 +02001250 to_talitos_ptr_ext_or(&desc->ptr[5], authsize, is_sec1);
Herbert Xuaeb4c132015-07-30 17:53:22 +08001251
LEROY Christophee04a61b2017-10-06 15:04:35 +02001252 /* ICV data */
1253 if (ret > 1) {
1254 tbl_off += ret;
Herbert Xuaeb4c132015-07-30 17:53:22 +08001255 edesc->icv_ool = true;
LEROY Christophe549bd8b2016-06-06 13:20:40 +02001256 sync_needed = true;
1257
LEROY Christophe9a655602017-10-06 15:04:59 +02001258 if (is_ipsec_esp) {
LEROY Christophe549bd8b2016-06-06 13:20:40 +02001259 struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off];
1260 int offset = (edesc->src_nents + edesc->dst_nents + 2) *
1261 sizeof(struct talitos_ptr) + authsize;
1262
1263 /* Add an entry to the link table for ICV data */
LEROY Christophee04a61b2017-10-06 15:04:35 +02001264 to_talitos_ptr_ext_set(tbl_ptr - 1, 0, is_sec1);
LEROY Christophe549bd8b2016-06-06 13:20:40 +02001265 to_talitos_ptr_ext_set(tbl_ptr, DESC_PTR_LNKTBL_RETURN,
1266 is_sec1);
LEROY Christophe549bd8b2016-06-06 13:20:40 +02001267
1268 /* icv data follows link tables */
1269 to_talitos_ptr(tbl_ptr, edesc->dma_link_tbl + offset,
LEROY Christopheda9de142017-10-06 15:04:57 +02001270 authsize, is_sec1);
LEROY Christophee04a61b2017-10-06 15:04:35 +02001271 } else {
1272 dma_addr_t addr = edesc->dma_link_tbl;
1273
1274 if (is_sec1)
1275 addr += areq->assoclen + cryptlen;
1276 else
1277 addr += sizeof(struct talitos_ptr) * tbl_off;
1278
LEROY Christopheda9de142017-10-06 15:04:57 +02001279 to_talitos_ptr(&desc->ptr[6], addr, authsize, is_sec1);
LEROY Christophee04a61b2017-10-06 15:04:35 +02001280 }
LEROY Christophe9a655602017-10-06 15:04:59 +02001281 } else if (!is_ipsec_esp) {
LEROY Christophee04a61b2017-10-06 15:04:35 +02001282 ret = talitos_sg_map(dev, areq->dst, authsize, edesc,
1283 &desc->ptr[6], sg_count, areq->assoclen +
1284 cryptlen,
1285 tbl_off);
1286 if (ret > 1) {
1287 tbl_off += ret;
1288 edesc->icv_ool = true;
1289 sync_needed = true;
1290 } else {
1291 edesc->icv_ool = false;
LEROY Christophe549bd8b2016-06-06 13:20:40 +02001292 }
Horia Geant?340ff602016-04-19 20:33:48 +03001293 } else {
LEROY Christophe549bd8b2016-06-06 13:20:40 +02001294 edesc->icv_ool = false;
1295 }
1296
Kim Phillips9c4a7962008-06-23 19:50:15 +08001297 /* iv out */
LEROY Christophe9a655602017-10-06 15:04:59 +02001298 if (is_ipsec_esp)
LEROY Christophe549bd8b2016-06-06 13:20:40 +02001299 map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv,
1300 DMA_FROM_DEVICE);
1301
1302 if (sync_needed)
1303 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1304 edesc->dma_len,
1305 DMA_BIDIRECTIONAL);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001306
Kim Phillips5228f0f2011-07-15 11:21:38 +08001307 ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
Kim Phillipsfa86a262008-07-17 20:20:06 +08001308 if (ret != -EINPROGRESS) {
1309 ipsec_esp_unmap(dev, edesc, areq);
1310 kfree(edesc);
1311 }
1312 return ret;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001313}
1314
Kim Phillips9c4a7962008-06-23 19:50:15 +08001315/*
Lee Nipper56af8cd2009-03-29 15:50:50 +08001316 * allocate and map the extended descriptor
Kim Phillips9c4a7962008-06-23 19:50:15 +08001317 */
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001318static struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
1319 struct scatterlist *src,
1320 struct scatterlist *dst,
Horia Geanta79fd31d2012-08-02 17:16:40 +03001321 u8 *iv,
1322 unsigned int assoclen,
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001323 unsigned int cryptlen,
1324 unsigned int authsize,
Horia Geanta79fd31d2012-08-02 17:16:40 +03001325 unsigned int ivsize,
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001326 int icv_stashing,
Horia Geanta62293a32013-11-28 15:11:17 +02001327 u32 cryptoflags,
1328 bool encrypt)
Kim Phillips9c4a7962008-06-23 19:50:15 +08001329{
Lee Nipper56af8cd2009-03-29 15:50:50 +08001330 struct talitos_edesc *edesc;
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001331 int src_nents, dst_nents, alloc_len, dma_len, src_len, dst_len;
Horia Geanta79fd31d2012-08-02 17:16:40 +03001332 dma_addr_t iv_dma = 0;
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001333 gfp_t flags = cryptoflags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
Kim Phillips586725f2008-07-17 20:19:18 +08001334 GFP_ATOMIC;
LEROY Christophe6f65f6a2015-04-17 16:32:15 +02001335 struct talitos_private *priv = dev_get_drvdata(dev);
1336 bool is_sec1 = has_ftr_sec1(priv);
1337 int max_len = is_sec1 ? TALITOS1_MAX_DATA_LEN : TALITOS2_MAX_DATA_LEN;
LABBE Corentin8e409fe2015-11-04 21:13:34 +01001338 void *err;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001339
LEROY Christophe6f65f6a2015-04-17 16:32:15 +02001340 if (cryptlen + authsize > max_len) {
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001341 dev_err(dev, "length exceeds h/w max limit\n");
Kim Phillips9c4a7962008-06-23 19:50:15 +08001342 return ERR_PTR(-EINVAL);
1343 }
1344
Horia Geanta935e99a2013-11-19 14:57:49 +02001345 if (ivsize)
Horia Geanta79fd31d2012-08-02 17:16:40 +03001346 iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE);
1347
Horia Geanta62293a32013-11-28 15:11:17 +02001348 if (!dst || dst == src) {
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001349 src_len = assoclen + cryptlen + authsize;
1350 src_nents = sg_nents_for_len(src, src_len);
LABBE Corentin8e409fe2015-11-04 21:13:34 +01001351 if (src_nents < 0) {
1352 dev_err(dev, "Invalid number of src SG.\n");
1353 err = ERR_PTR(-EINVAL);
1354 goto error_sg;
1355 }
Horia Geanta62293a32013-11-28 15:11:17 +02001356 src_nents = (src_nents == 1) ? 0 : src_nents;
1357 dst_nents = dst ? src_nents : 0;
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001358 dst_len = 0;
Horia Geanta62293a32013-11-28 15:11:17 +02001359 } else { /* dst && dst != src*/
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001360 src_len = assoclen + cryptlen + (encrypt ? 0 : authsize);
1361 src_nents = sg_nents_for_len(src, src_len);
LABBE Corentin8e409fe2015-11-04 21:13:34 +01001362 if (src_nents < 0) {
1363 dev_err(dev, "Invalid number of src SG.\n");
1364 err = ERR_PTR(-EINVAL);
1365 goto error_sg;
1366 }
Horia Geanta62293a32013-11-28 15:11:17 +02001367 src_nents = (src_nents == 1) ? 0 : src_nents;
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001368 dst_len = assoclen + cryptlen + (encrypt ? authsize : 0);
1369 dst_nents = sg_nents_for_len(dst, dst_len);
LABBE Corentin8e409fe2015-11-04 21:13:34 +01001370 if (dst_nents < 0) {
1371 dev_err(dev, "Invalid number of dst SG.\n");
1372 err = ERR_PTR(-EINVAL);
1373 goto error_sg;
1374 }
Horia Geanta62293a32013-11-28 15:11:17 +02001375 dst_nents = (dst_nents == 1) ? 0 : dst_nents;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001376 }
1377
1378 /*
1379 * allocate space for base edesc plus the link tables,
Herbert Xuaeb4c132015-07-30 17:53:22 +08001380 * allowing for two separate entries for AD and generated ICV (+ 2),
1381 * and space for two sets of ICVs (stashed and generated)
Kim Phillips9c4a7962008-06-23 19:50:15 +08001382 */
Lee Nipper56af8cd2009-03-29 15:50:50 +08001383 alloc_len = sizeof(struct talitos_edesc);
Herbert Xuaeb4c132015-07-30 17:53:22 +08001384 if (src_nents || dst_nents) {
LEROY Christophe6f65f6a2015-04-17 16:32:15 +02001385 if (is_sec1)
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001386 dma_len = (src_nents ? src_len : 0) +
1387 (dst_nents ? dst_len : 0);
LEROY Christophe6f65f6a2015-04-17 16:32:15 +02001388 else
Herbert Xuaeb4c132015-07-30 17:53:22 +08001389 dma_len = (src_nents + dst_nents + 2) *
1390 sizeof(struct talitos_ptr) + authsize * 2;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001391 alloc_len += dma_len;
1392 } else {
1393 dma_len = 0;
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001394 alloc_len += icv_stashing ? authsize : 0;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001395 }
1396
LEROY Christophe37b5e882017-10-06 15:05:06 +02001397 /* if its a ahash, add space for a second desc next to the first one */
1398 if (is_sec1 && !dst)
1399 alloc_len += sizeof(struct talitos_desc);
1400
Kim Phillips586725f2008-07-17 20:19:18 +08001401 edesc = kmalloc(alloc_len, GFP_DMA | flags);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001402 if (!edesc) {
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001403 dev_err(dev, "could not allocate edescriptor\n");
LABBE Corentin8e409fe2015-11-04 21:13:34 +01001404 err = ERR_PTR(-ENOMEM);
1405 goto error_sg;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001406 }
LEROY Christophee4a647c2017-10-06 15:04:45 +02001407 memset(&edesc->desc, 0, sizeof(edesc->desc));
Kim Phillips9c4a7962008-06-23 19:50:15 +08001408
1409 edesc->src_nents = src_nents;
1410 edesc->dst_nents = dst_nents;
Horia Geanta79fd31d2012-08-02 17:16:40 +03001411 edesc->iv_dma = iv_dma;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001412 edesc->dma_len = dma_len;
LEROY Christophe37b5e882017-10-06 15:05:06 +02001413 if (dma_len) {
1414 void *addr = &edesc->link_tbl[0];
1415
1416 if (is_sec1 && !dst)
1417 addr += sizeof(struct talitos_desc);
1418 edesc->dma_link_tbl = dma_map_single(dev, addr,
Lee Nipper497f2e62010-05-19 19:20:36 +10001419 edesc->dma_len,
1420 DMA_BIDIRECTIONAL);
LEROY Christophe37b5e882017-10-06 15:05:06 +02001421 }
Kim Phillips9c4a7962008-06-23 19:50:15 +08001422 return edesc;
LABBE Corentin8e409fe2015-11-04 21:13:34 +01001423error_sg:
1424 if (iv_dma)
1425 dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE);
1426 return err;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001427}
1428
Horia Geanta79fd31d2012-08-02 17:16:40 +03001429static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq, u8 *iv,
Horia Geanta62293a32013-11-28 15:11:17 +02001430 int icv_stashing, bool encrypt)
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001431{
1432 struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
Herbert Xuaeb4c132015-07-30 17:53:22 +08001433 unsigned int authsize = crypto_aead_authsize(authenc);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001434 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
Horia Geanta79fd31d2012-08-02 17:16:40 +03001435 unsigned int ivsize = crypto_aead_ivsize(authenc);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001436
Herbert Xuaeb4c132015-07-30 17:53:22 +08001437 return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst,
Horia Geanta79fd31d2012-08-02 17:16:40 +03001438 iv, areq->assoclen, areq->cryptlen,
Herbert Xuaeb4c132015-07-30 17:53:22 +08001439 authsize, ivsize, icv_stashing,
Horia Geanta62293a32013-11-28 15:11:17 +02001440 areq->base.flags, encrypt);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001441}
1442
Lee Nipper56af8cd2009-03-29 15:50:50 +08001443static int aead_encrypt(struct aead_request *req)
Kim Phillips9c4a7962008-06-23 19:50:15 +08001444{
1445 struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1446 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
Lee Nipper56af8cd2009-03-29 15:50:50 +08001447 struct talitos_edesc *edesc;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001448
1449 /* allocate extended descriptor */
Horia Geanta62293a32013-11-28 15:11:17 +02001450 edesc = aead_edesc_alloc(req, req->iv, 0, true);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001451 if (IS_ERR(edesc))
1452 return PTR_ERR(edesc);
1453
1454 /* set encrypt */
Lee Nipper70bcaca2008-07-03 19:08:46 +08001455 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001456
Herbert Xuaeb4c132015-07-30 17:53:22 +08001457 return ipsec_esp(edesc, req, ipsec_esp_encrypt_done);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001458}
1459
Lee Nipper56af8cd2009-03-29 15:50:50 +08001460static int aead_decrypt(struct aead_request *req)
Kim Phillips9c4a7962008-06-23 19:50:15 +08001461{
1462 struct crypto_aead *authenc = crypto_aead_reqtfm(req);
Herbert Xuaeb4c132015-07-30 17:53:22 +08001463 unsigned int authsize = crypto_aead_authsize(authenc);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001464 struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001465 struct talitos_private *priv = dev_get_drvdata(ctx->dev);
Lee Nipper56af8cd2009-03-29 15:50:50 +08001466 struct talitos_edesc *edesc;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001467 struct scatterlist *sg;
1468 void *icvdata;
1469
1470 req->cryptlen -= authsize;
1471
1472 /* allocate extended descriptor */
Horia Geanta62293a32013-11-28 15:11:17 +02001473 edesc = aead_edesc_alloc(req, req->iv, 1, false);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001474 if (IS_ERR(edesc))
1475 return PTR_ERR(edesc);
1476
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001477 if ((priv->features & TALITOS_FTR_HW_AUTH_CHECK) &&
Kim Phillipse938e462009-03-29 15:53:23 +08001478 ((!edesc->src_nents && !edesc->dst_nents) ||
1479 priv->features & TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT)) {
Kim Phillips9c4a7962008-06-23 19:50:15 +08001480
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001481 /* decrypt and check the ICV */
Kim Phillipse938e462009-03-29 15:53:23 +08001482 edesc->desc.hdr = ctx->desc_hdr_template |
1483 DESC_HDR_DIR_INBOUND |
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001484 DESC_HDR_MODE1_MDEU_CICV;
Kim Phillips9c4a7962008-06-23 19:50:15 +08001485
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001486 /* reset integrity check result bits */
Kim Phillips9c4a7962008-06-23 19:50:15 +08001487
Herbert Xuaeb4c132015-07-30 17:53:22 +08001488 return ipsec_esp(edesc, req, ipsec_esp_decrypt_hwauth_done);
Kim Phillipsfe5720e2008-10-12 20:33:14 +08001489 }
Kim Phillipse938e462009-03-29 15:53:23 +08001490
1491 /* Have to check the ICV with software */
1492 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1493
1494 /* stash incoming ICV for later cmp with ICV generated by the h/w */
1495 if (edesc->dma_len)
Herbert Xuaeb4c132015-07-30 17:53:22 +08001496 icvdata = (char *)&edesc->link_tbl[edesc->src_nents +
1497 edesc->dst_nents + 2];
Kim Phillipse938e462009-03-29 15:53:23 +08001498 else
1499 icvdata = &edesc->link_tbl[0];
1500
1501 sg = sg_last(req->src, edesc->src_nents ? : 1);
1502
Herbert Xuaeb4c132015-07-30 17:53:22 +08001503 memcpy(icvdata, (char *)sg_virt(sg) + sg->length - authsize, authsize);
Kim Phillipse938e462009-03-29 15:53:23 +08001504
Herbert Xuaeb4c132015-07-30 17:53:22 +08001505 return ipsec_esp(edesc, req, ipsec_esp_decrypt_swauth_done);
Kim Phillips9c4a7962008-06-23 19:50:15 +08001506}
1507
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001508static int ablkcipher_setkey(struct crypto_ablkcipher *cipher,
1509 const u8 *key, unsigned int keylen)
1510{
1511 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
LEROY Christophe2e13ce02017-10-06 15:05:02 +02001512 struct device *dev = ctx->dev;
LEROY Christophef384cdc2017-10-06 15:04:37 +02001513 u32 tmp[DES_EXPKEY_WORDS];
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001514
Martin Hicks03d2c512017-05-02 09:38:35 -04001515 if (keylen > TALITOS_MAX_KEY_SIZE) {
1516 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
1517 return -EINVAL;
1518 }
1519
LEROY Christophef384cdc2017-10-06 15:04:37 +02001520 if (unlikely(crypto_ablkcipher_get_flags(cipher) &
1521 CRYPTO_TFM_REQ_WEAK_KEY) &&
1522 !des_ekey(tmp, key)) {
1523 crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_WEAK_KEY);
1524 return -EINVAL;
1525 }
1526
LEROY Christophe2e13ce02017-10-06 15:05:02 +02001527 if (ctx->keylen)
1528 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE);
1529
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001530 memcpy(&ctx->key, key, keylen);
1531 ctx->keylen = keylen;
1532
LEROY Christophe2e13ce02017-10-06 15:05:02 +02001533 ctx->dma_key = dma_map_single(dev, ctx->key, keylen, DMA_TO_DEVICE);
1534
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001535 return 0;
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001536}
1537
1538static void common_nonsnoop_unmap(struct device *dev,
1539 struct talitos_edesc *edesc,
1540 struct ablkcipher_request *areq)
1541{
1542 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
LEROY Christophe032d1972015-04-17 16:31:51 +02001543
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001544 talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->nbytes, 0);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001545 unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], DMA_TO_DEVICE);
1546
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001547 if (edesc->dma_len)
1548 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1549 DMA_BIDIRECTIONAL);
1550}
1551
1552static void ablkcipher_done(struct device *dev,
1553 struct talitos_desc *desc, void *context,
1554 int err)
1555{
1556 struct ablkcipher_request *areq = context;
Kim Phillips19bbbc62009-03-29 15:53:59 +08001557 struct talitos_edesc *edesc;
1558
1559 edesc = container_of(desc, struct talitos_edesc, desc);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001560
1561 common_nonsnoop_unmap(dev, edesc, areq);
1562
1563 kfree(edesc);
1564
1565 areq->base.complete(&areq->base, err);
1566}
1567
1568static int common_nonsnoop(struct talitos_edesc *edesc,
1569 struct ablkcipher_request *areq,
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001570 void (*callback) (struct device *dev,
1571 struct talitos_desc *desc,
1572 void *context, int error))
1573{
1574 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1575 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1576 struct device *dev = ctx->dev;
1577 struct talitos_desc *desc = &edesc->desc;
1578 unsigned int cryptlen = areq->nbytes;
Horia Geanta79fd31d2012-08-02 17:16:40 +03001579 unsigned int ivsize = crypto_ablkcipher_ivsize(cipher);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001580 int sg_count, ret;
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001581 bool sync_needed = false;
LEROY Christophe922f9dc2015-04-17 16:32:07 +02001582 struct talitos_private *priv = dev_get_drvdata(dev);
1583 bool is_sec1 = has_ftr_sec1(priv);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001584
1585 /* first DWORD empty */
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001586
1587 /* cipher iv */
LEROY Christopheda9de142017-10-06 15:04:57 +02001588 to_talitos_ptr(&desc->ptr[1], edesc->iv_dma, ivsize, is_sec1);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001589
1590 /* cipher key */
LEROY Christophe2e13ce02017-10-06 15:05:02 +02001591 to_talitos_ptr(&desc->ptr[2], ctx->dma_key, ctx->keylen, is_sec1);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001592
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001593 sg_count = edesc->src_nents ?: 1;
1594 if (is_sec1 && sg_count > 1)
1595 sg_copy_to_buffer(areq->src, sg_count, edesc->buf,
1596 cryptlen);
1597 else
1598 sg_count = dma_map_sg(dev, areq->src, sg_count,
1599 (areq->src == areq->dst) ?
1600 DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001601 /*
1602 * cipher in
1603 */
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001604 sg_count = talitos_sg_map(dev, areq->src, cryptlen, edesc,
1605 &desc->ptr[3], sg_count, 0, 0);
1606 if (sg_count > 1)
1607 sync_needed = true;
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001608
1609 /* cipher out */
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001610 if (areq->src != areq->dst) {
1611 sg_count = edesc->dst_nents ? : 1;
1612 if (!is_sec1 || sg_count == 1)
1613 dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE);
1614 }
1615
1616 ret = talitos_sg_map(dev, areq->dst, cryptlen, edesc, &desc->ptr[4],
1617 sg_count, 0, (edesc->src_nents + 1));
1618 if (ret > 1)
1619 sync_needed = true;
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001620
1621 /* iv out */
LEROY Christophea2b35aa2015-04-17 16:31:57 +02001622 map_single_talitos_ptr(dev, &desc->ptr[5], ivsize, ctx->iv,
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001623 DMA_FROM_DEVICE);
1624
1625 /* last DWORD empty */
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001626
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001627 if (sync_needed)
1628 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1629 edesc->dma_len, DMA_BIDIRECTIONAL);
1630
Kim Phillips5228f0f2011-07-15 11:21:38 +08001631 ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001632 if (ret != -EINPROGRESS) {
1633 common_nonsnoop_unmap(dev, edesc, areq);
1634 kfree(edesc);
1635 }
1636 return ret;
1637}
1638
Kim Phillipse938e462009-03-29 15:53:23 +08001639static struct talitos_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request *
Horia Geanta62293a32013-11-28 15:11:17 +02001640 areq, bool encrypt)
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001641{
1642 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1643 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
Horia Geanta79fd31d2012-08-02 17:16:40 +03001644 unsigned int ivsize = crypto_ablkcipher_ivsize(cipher);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001645
Herbert Xuaeb4c132015-07-30 17:53:22 +08001646 return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst,
Horia Geanta79fd31d2012-08-02 17:16:40 +03001647 areq->info, 0, areq->nbytes, 0, ivsize, 0,
Horia Geanta62293a32013-11-28 15:11:17 +02001648 areq->base.flags, encrypt);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001649}
1650
1651static int ablkcipher_encrypt(struct ablkcipher_request *areq)
1652{
1653 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1654 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1655 struct talitos_edesc *edesc;
1656
1657 /* allocate extended descriptor */
Horia Geanta62293a32013-11-28 15:11:17 +02001658 edesc = ablkcipher_edesc_alloc(areq, true);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001659 if (IS_ERR(edesc))
1660 return PTR_ERR(edesc);
1661
1662 /* set encrypt */
1663 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1664
Kim Phillipsfebec542011-07-15 11:21:39 +08001665 return common_nonsnoop(edesc, areq, ablkcipher_done);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001666}
1667
1668static int ablkcipher_decrypt(struct ablkcipher_request *areq)
1669{
1670 struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1671 struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1672 struct talitos_edesc *edesc;
1673
1674 /* allocate extended descriptor */
Horia Geanta62293a32013-11-28 15:11:17 +02001675 edesc = ablkcipher_edesc_alloc(areq, false);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001676 if (IS_ERR(edesc))
1677 return PTR_ERR(edesc);
1678
1679 edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1680
Kim Phillipsfebec542011-07-15 11:21:39 +08001681 return common_nonsnoop(edesc, areq, ablkcipher_done);
Lee Nipper4de9d0b2009-03-29 15:52:32 +08001682}
1683
Lee Nipper497f2e62010-05-19 19:20:36 +10001684static void common_nonsnoop_hash_unmap(struct device *dev,
1685 struct talitos_edesc *edesc,
1686 struct ahash_request *areq)
1687{
1688 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
Lee Nipper497f2e62010-05-19 19:20:36 +10001689
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001690 talitos_sg_unmap(dev, edesc, req_ctx->psrc, NULL, 0, 0);
LEROY Christophe032d1972015-04-17 16:31:51 +02001691
Lee Nipper497f2e62010-05-19 19:20:36 +10001692 if (edesc->dma_len)
1693 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1694 DMA_BIDIRECTIONAL);
1695
LEROY Christophe37b5e882017-10-06 15:05:06 +02001696 if (edesc->desc.next_desc)
1697 dma_unmap_single(dev, be32_to_cpu(edesc->desc.next_desc),
1698 TALITOS_DESC_SIZE, DMA_BIDIRECTIONAL);
Lee Nipper497f2e62010-05-19 19:20:36 +10001699}
1700
1701static void ahash_done(struct device *dev,
1702 struct talitos_desc *desc, void *context,
1703 int err)
1704{
1705 struct ahash_request *areq = context;
1706 struct talitos_edesc *edesc =
1707 container_of(desc, struct talitos_edesc, desc);
1708 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1709
1710 if (!req_ctx->last && req_ctx->to_hash_later) {
1711 /* Position any partial block for next update/final/finup */
1712 memcpy(req_ctx->buf, req_ctx->bufnext, req_ctx->to_hash_later);
Lee Nipper5e833bc2010-06-16 15:29:15 +10001713 req_ctx->nbuf = req_ctx->to_hash_later;
Lee Nipper497f2e62010-05-19 19:20:36 +10001714 }
1715 common_nonsnoop_hash_unmap(dev, edesc, areq);
1716
1717 kfree(edesc);
1718
1719 areq->base.complete(&areq->base, err);
1720}
1721
LEROY Christophe2d029052015-04-17 16:32:18 +02001722/*
1723 * SEC1 doesn't like hashing of 0 sized message, so we do the padding
1724 * ourself and submit a padded block
1725 */
LEROY Christophe5b2cf262017-10-06 15:04:47 +02001726static void talitos_handle_buggy_hash(struct talitos_ctx *ctx,
LEROY Christophe2d029052015-04-17 16:32:18 +02001727 struct talitos_edesc *edesc,
1728 struct talitos_ptr *ptr)
1729{
1730 static u8 padded_hash[64] = {
1731 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1732 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1733 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1734 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
1735 };
1736
1737 pr_err_once("Bug in SEC1, padding ourself\n");
1738 edesc->desc.hdr &= ~DESC_HDR_MODE0_MDEU_PAD;
1739 map_single_talitos_ptr(ctx->dev, ptr, sizeof(padded_hash),
1740 (char *)padded_hash, DMA_TO_DEVICE);
1741}
1742
Lee Nipper497f2e62010-05-19 19:20:36 +10001743static int common_nonsnoop_hash(struct talitos_edesc *edesc,
1744 struct ahash_request *areq, unsigned int length,
LEROY Christophe37b5e882017-10-06 15:05:06 +02001745 unsigned int offset,
Lee Nipper497f2e62010-05-19 19:20:36 +10001746 void (*callback) (struct device *dev,
1747 struct talitos_desc *desc,
1748 void *context, int error))
1749{
1750 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1751 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1752 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1753 struct device *dev = ctx->dev;
1754 struct talitos_desc *desc = &edesc->desc;
LEROY Christophe032d1972015-04-17 16:31:51 +02001755 int ret;
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001756 bool sync_needed = false;
LEROY Christophe922f9dc2015-04-17 16:32:07 +02001757 struct talitos_private *priv = dev_get_drvdata(dev);
1758 bool is_sec1 = has_ftr_sec1(priv);
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001759 int sg_count;
Lee Nipper497f2e62010-05-19 19:20:36 +10001760
1761 /* first DWORD empty */
Lee Nipper497f2e62010-05-19 19:20:36 +10001762
Kim Phillips60f208d2010-05-19 19:21:53 +10001763 /* hash context in */
1764 if (!req_ctx->first || req_ctx->swinit) {
LEROY Christophe49f97832017-10-06 15:05:04 +02001765 to_talitos_ptr(&desc->ptr[1], ctx->dma_hw_context,
1766 req_ctx->hw_context_size, is_sec1);
Kim Phillips60f208d2010-05-19 19:21:53 +10001767 req_ctx->swinit = 0;
Lee Nipper497f2e62010-05-19 19:20:36 +10001768 }
LEROY Christopheafd62fa2017-09-13 12:44:51 +02001769 /* Indicate next op is not the first. */
1770 req_ctx->first = 0;
Lee Nipper497f2e62010-05-19 19:20:36 +10001771
1772 /* HMAC key */
1773 if (ctx->keylen)
LEROY Christophe2e13ce02017-10-06 15:05:02 +02001774 to_talitos_ptr(&desc->ptr[2], ctx->dma_key, ctx->keylen,
1775 is_sec1);
Lee Nipper497f2e62010-05-19 19:20:36 +10001776
LEROY Christophe37b5e882017-10-06 15:05:06 +02001777 if (is_sec1 && req_ctx->nbuf)
1778 length -= req_ctx->nbuf;
1779
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001780 sg_count = edesc->src_nents ?: 1;
1781 if (is_sec1 && sg_count > 1)
LEROY Christophe37b5e882017-10-06 15:05:06 +02001782 sg_pcopy_to_buffer(req_ctx->psrc, sg_count,
1783 edesc->buf + sizeof(struct talitos_desc),
1784 length, req_ctx->nbuf);
1785 else if (length)
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001786 sg_count = dma_map_sg(dev, req_ctx->psrc, sg_count,
1787 DMA_TO_DEVICE);
Lee Nipper497f2e62010-05-19 19:20:36 +10001788 /*
1789 * data in
1790 */
LEROY Christophe37b5e882017-10-06 15:05:06 +02001791 if (is_sec1 && req_ctx->nbuf) {
1792 to_talitos_ptr(&desc->ptr[3], ctx->dma_buf, req_ctx->nbuf,
1793 is_sec1);
1794 } else {
1795 sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc,
1796 &desc->ptr[3], sg_count, offset, 0);
1797 if (sg_count > 1)
1798 sync_needed = true;
1799 }
Lee Nipper497f2e62010-05-19 19:20:36 +10001800
1801 /* fifth DWORD empty */
Lee Nipper497f2e62010-05-19 19:20:36 +10001802
1803 /* hash/HMAC out -or- hash context out */
1804 if (req_ctx->last)
1805 map_single_talitos_ptr(dev, &desc->ptr[5],
1806 crypto_ahash_digestsize(tfm),
LEROY Christophea2b35aa2015-04-17 16:31:57 +02001807 areq->result, DMA_FROM_DEVICE);
Lee Nipper497f2e62010-05-19 19:20:36 +10001808 else
LEROY Christophe49f97832017-10-06 15:05:04 +02001809 to_talitos_ptr(&desc->ptr[5], ctx->dma_hw_context,
1810 req_ctx->hw_context_size, is_sec1);
Lee Nipper497f2e62010-05-19 19:20:36 +10001811
1812 /* last DWORD empty */
Lee Nipper497f2e62010-05-19 19:20:36 +10001813
LEROY Christophe2d029052015-04-17 16:32:18 +02001814 if (is_sec1 && from_talitos_ptr_len(&desc->ptr[3], true) == 0)
1815 talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]);
1816
LEROY Christophe37b5e882017-10-06 15:05:06 +02001817 if (is_sec1 && req_ctx->nbuf && length) {
1818 struct talitos_desc *desc2 = desc + 1;
1819 dma_addr_t next_desc;
1820
1821 memset(desc2, 0, sizeof(*desc2));
1822 desc2->hdr = desc->hdr;
1823 desc2->hdr &= ~DESC_HDR_MODE0_MDEU_INIT;
1824 desc2->hdr1 = desc2->hdr;
1825 desc->hdr &= ~DESC_HDR_MODE0_MDEU_PAD;
1826 desc->hdr |= DESC_HDR_MODE0_MDEU_CONT;
1827 desc->hdr &= ~DESC_HDR_DONE_NOTIFY;
1828
1829 to_talitos_ptr(&desc2->ptr[1], ctx->dma_hw_context,
1830 req_ctx->hw_context_size, is_sec1);
1831
1832 copy_talitos_ptr(&desc2->ptr[2], &desc->ptr[2], is_sec1);
1833 sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc,
1834 &desc2->ptr[3], sg_count, offset, 0);
1835 if (sg_count > 1)
1836 sync_needed = true;
1837 copy_talitos_ptr(&desc2->ptr[5], &desc->ptr[5], is_sec1);
1838 if (req_ctx->last)
1839 to_talitos_ptr(&desc->ptr[5], ctx->dma_hw_context,
1840 req_ctx->hw_context_size, is_sec1);
1841
1842 next_desc = dma_map_single(dev, &desc2->hdr1, TALITOS_DESC_SIZE,
1843 DMA_BIDIRECTIONAL);
1844 desc->next_desc = cpu_to_be32(next_desc);
1845 }
1846
LEROY Christophe6a1e8d12016-06-06 13:20:38 +02001847 if (sync_needed)
1848 dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1849 edesc->dma_len, DMA_BIDIRECTIONAL);
1850
Kim Phillips5228f0f2011-07-15 11:21:38 +08001851 ret = talitos_submit(dev, ctx->ch, desc, callback, areq);
Lee Nipper497f2e62010-05-19 19:20:36 +10001852 if (ret != -EINPROGRESS) {
1853 common_nonsnoop_hash_unmap(dev, edesc, areq);
1854 kfree(edesc);
1855 }
1856 return ret;
1857}
1858
1859static struct talitos_edesc *ahash_edesc_alloc(struct ahash_request *areq,
1860 unsigned int nbytes)
1861{
1862 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1863 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1864 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
LEROY Christophe37b5e882017-10-06 15:05:06 +02001865 struct talitos_private *priv = dev_get_drvdata(ctx->dev);
1866 bool is_sec1 = has_ftr_sec1(priv);
1867
1868 if (is_sec1)
1869 nbytes -= req_ctx->nbuf;
Lee Nipper497f2e62010-05-19 19:20:36 +10001870
Herbert Xuaeb4c132015-07-30 17:53:22 +08001871 return talitos_edesc_alloc(ctx->dev, req_ctx->psrc, NULL, NULL, 0,
Horia Geanta62293a32013-11-28 15:11:17 +02001872 nbytes, 0, 0, 0, areq->base.flags, false);
Lee Nipper497f2e62010-05-19 19:20:36 +10001873}
1874
1875static int ahash_init(struct ahash_request *areq)
1876{
1877 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
LEROY Christophe49f97832017-10-06 15:05:04 +02001878 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1879 struct device *dev = ctx->dev;
Lee Nipper497f2e62010-05-19 19:20:36 +10001880 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
LEROY Christophe49f97832017-10-06 15:05:04 +02001881 unsigned int size;
LEROY Christophe37b5e882017-10-06 15:05:06 +02001882 struct talitos_private *priv = dev_get_drvdata(dev);
1883 bool is_sec1 = has_ftr_sec1(priv);
Lee Nipper497f2e62010-05-19 19:20:36 +10001884
1885 /* Initialize the context */
Lee Nipper5e833bc2010-06-16 15:29:15 +10001886 req_ctx->nbuf = 0;
Kim Phillips60f208d2010-05-19 19:21:53 +10001887 req_ctx->first = 1; /* first indicates h/w must init its context */
1888 req_ctx->swinit = 0; /* assume h/w init of context */
LEROY Christophe49f97832017-10-06 15:05:04 +02001889 size = (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
Lee Nipper497f2e62010-05-19 19:20:36 +10001890 ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
1891 : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
LEROY Christophe49f97832017-10-06 15:05:04 +02001892 req_ctx->hw_context_size = size;
Lee Nipper497f2e62010-05-19 19:20:36 +10001893
LEROY Christophe49f97832017-10-06 15:05:04 +02001894 if (ctx->dma_hw_context)
1895 dma_unmap_single(dev, ctx->dma_hw_context, size,
1896 DMA_BIDIRECTIONAL);
1897 ctx->dma_hw_context = dma_map_single(dev, req_ctx->hw_context, size,
1898 DMA_BIDIRECTIONAL);
LEROY Christophe37b5e882017-10-06 15:05:06 +02001899 if (ctx->dma_buf)
1900 dma_unmap_single(dev, ctx->dma_buf, sizeof(req_ctx->buf),
1901 DMA_TO_DEVICE);
1902 if (is_sec1)
1903 ctx->dma_buf = dma_map_single(dev, req_ctx->buf,
1904 sizeof(req_ctx->buf),
1905 DMA_TO_DEVICE);
Lee Nipper497f2e62010-05-19 19:20:36 +10001906 return 0;
1907}
1908
Kim Phillips60f208d2010-05-19 19:21:53 +10001909/*
1910 * on h/w without explicit sha224 support, we initialize h/w context
1911 * manually with sha224 constants, and tell it to run sha256.
1912 */
1913static int ahash_init_sha224_swinit(struct ahash_request *areq)
1914{
1915 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
LEROY Christophe49f97832017-10-06 15:05:04 +02001916 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1917 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1918 struct device *dev = ctx->dev;
Kim Phillips60f208d2010-05-19 19:21:53 +10001919
1920 ahash_init(areq);
1921 req_ctx->swinit = 1;/* prevent h/w initting context with sha256 values*/
1922
Kim Phillipsa7524472010-09-23 15:56:38 +08001923 req_ctx->hw_context[0] = SHA224_H0;
1924 req_ctx->hw_context[1] = SHA224_H1;
1925 req_ctx->hw_context[2] = SHA224_H2;
1926 req_ctx->hw_context[3] = SHA224_H3;
1927 req_ctx->hw_context[4] = SHA224_H4;
1928 req_ctx->hw_context[5] = SHA224_H5;
1929 req_ctx->hw_context[6] = SHA224_H6;
1930 req_ctx->hw_context[7] = SHA224_H7;
Kim Phillips60f208d2010-05-19 19:21:53 +10001931
1932 /* init 64-bit count */
1933 req_ctx->hw_context[8] = 0;
1934 req_ctx->hw_context[9] = 0;
1935
LEROY Christophe49f97832017-10-06 15:05:04 +02001936 dma_sync_single_for_device(dev, ctx->dma_hw_context,
1937 req_ctx->hw_context_size, DMA_TO_DEVICE);
1938
Kim Phillips60f208d2010-05-19 19:21:53 +10001939 return 0;
1940}
1941
Lee Nipper497f2e62010-05-19 19:20:36 +10001942static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
1943{
1944 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1945 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1946 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1947 struct talitos_edesc *edesc;
1948 unsigned int blocksize =
1949 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
1950 unsigned int nbytes_to_hash;
1951 unsigned int to_hash_later;
Lee Nipper5e833bc2010-06-16 15:29:15 +10001952 unsigned int nsg;
LABBE Corentin8e409fe2015-11-04 21:13:34 +01001953 int nents;
LEROY Christophe37b5e882017-10-06 15:05:06 +02001954 struct device *dev = ctx->dev;
1955 struct talitos_private *priv = dev_get_drvdata(dev);
1956 bool is_sec1 = has_ftr_sec1(priv);
1957 int offset = 0;
Lee Nipper497f2e62010-05-19 19:20:36 +10001958
Lee Nipper5e833bc2010-06-16 15:29:15 +10001959 if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) {
1960 /* Buffer up to one whole block */
LABBE Corentin8e409fe2015-11-04 21:13:34 +01001961 nents = sg_nents_for_len(areq->src, nbytes);
1962 if (nents < 0) {
1963 dev_err(ctx->dev, "Invalid number of src SG.\n");
1964 return nents;
1965 }
1966 sg_copy_to_buffer(areq->src, nents,
Lee Nipper5e833bc2010-06-16 15:29:15 +10001967 req_ctx->buf + req_ctx->nbuf, nbytes);
1968 req_ctx->nbuf += nbytes;
Lee Nipper497f2e62010-05-19 19:20:36 +10001969 return 0;
1970 }
1971
Lee Nipper5e833bc2010-06-16 15:29:15 +10001972 /* At least (blocksize + 1) bytes are available to hash */
1973 nbytes_to_hash = nbytes + req_ctx->nbuf;
1974 to_hash_later = nbytes_to_hash & (blocksize - 1);
1975
1976 if (req_ctx->last)
1977 to_hash_later = 0;
1978 else if (to_hash_later)
1979 /* There is a partial block. Hash the full block(s) now */
1980 nbytes_to_hash -= to_hash_later;
1981 else {
1982 /* Keep one block buffered */
1983 nbytes_to_hash -= blocksize;
1984 to_hash_later = blocksize;
1985 }
1986
1987 /* Chain in any previously buffered data */
LEROY Christophe37b5e882017-10-06 15:05:06 +02001988 if (!is_sec1 && req_ctx->nbuf) {
Lee Nipper5e833bc2010-06-16 15:29:15 +10001989 nsg = (req_ctx->nbuf < nbytes_to_hash) ? 2 : 1;
1990 sg_init_table(req_ctx->bufsl, nsg);
1991 sg_set_buf(req_ctx->bufsl, req_ctx->buf, req_ctx->nbuf);
1992 if (nsg > 1)
Dan Williamsc56f6d12015-08-07 18:15:13 +02001993 sg_chain(req_ctx->bufsl, 2, areq->src);
Lee Nipper497f2e62010-05-19 19:20:36 +10001994 req_ctx->psrc = req_ctx->bufsl;
LEROY Christophe37b5e882017-10-06 15:05:06 +02001995 } else if (is_sec1 && req_ctx->nbuf && req_ctx->nbuf < blocksize) {
1996 if (nbytes_to_hash > blocksize)
1997 offset = blocksize - req_ctx->nbuf;
1998 else
1999 offset = nbytes_to_hash - req_ctx->nbuf;
2000 nents = sg_nents_for_len(areq->src, offset);
2001 if (nents < 0) {
2002 dev_err(ctx->dev, "Invalid number of src SG.\n");
2003 return nents;
2004 }
2005 sg_copy_to_buffer(areq->src, nents,
2006 req_ctx->buf + req_ctx->nbuf, offset);
2007 req_ctx->nbuf += offset;
2008 req_ctx->psrc = areq->src;
Lee Nipper5e833bc2010-06-16 15:29:15 +10002009 } else
Lee Nipper497f2e62010-05-19 19:20:36 +10002010 req_ctx->psrc = areq->src;
Lee Nipper497f2e62010-05-19 19:20:36 +10002011
Lee Nipper5e833bc2010-06-16 15:29:15 +10002012 if (to_hash_later) {
LABBE Corentin8e409fe2015-11-04 21:13:34 +01002013 nents = sg_nents_for_len(areq->src, nbytes);
2014 if (nents < 0) {
2015 dev_err(ctx->dev, "Invalid number of src SG.\n");
2016 return nents;
2017 }
Akinobu Mitad0525722013-07-08 16:01:55 -07002018 sg_pcopy_to_buffer(areq->src, nents,
Lee Nipper5e833bc2010-06-16 15:29:15 +10002019 req_ctx->bufnext,
2020 to_hash_later,
2021 nbytes - to_hash_later);
Lee Nipper497f2e62010-05-19 19:20:36 +10002022 }
Lee Nipper5e833bc2010-06-16 15:29:15 +10002023 req_ctx->to_hash_later = to_hash_later;
Lee Nipper497f2e62010-05-19 19:20:36 +10002024
Lee Nipper5e833bc2010-06-16 15:29:15 +10002025 /* Allocate extended descriptor */
Lee Nipper497f2e62010-05-19 19:20:36 +10002026 edesc = ahash_edesc_alloc(areq, nbytes_to_hash);
2027 if (IS_ERR(edesc))
2028 return PTR_ERR(edesc);
2029
2030 edesc->desc.hdr = ctx->desc_hdr_template;
2031
2032 /* On last one, request SEC to pad; otherwise continue */
2033 if (req_ctx->last)
2034 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD;
2035 else
2036 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT;
2037
Kim Phillips60f208d2010-05-19 19:21:53 +10002038 /* request SEC to INIT hash. */
2039 if (req_ctx->first && !req_ctx->swinit)
Lee Nipper497f2e62010-05-19 19:20:36 +10002040 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT;
LEROY Christophe37b5e882017-10-06 15:05:06 +02002041 if (is_sec1)
2042 dma_sync_single_for_device(dev, ctx->dma_buf,
2043 req_ctx->nbuf, DMA_TO_DEVICE);
Lee Nipper497f2e62010-05-19 19:20:36 +10002044
2045 /* When the tfm context has a keylen, it's an HMAC.
2046 * A first or last (ie. not middle) descriptor must request HMAC.
2047 */
2048 if (ctx->keylen && (req_ctx->first || req_ctx->last))
2049 edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC;
2050
LEROY Christophe37b5e882017-10-06 15:05:06 +02002051 return common_nonsnoop_hash(edesc, areq, nbytes_to_hash, offset,
Lee Nipper497f2e62010-05-19 19:20:36 +10002052 ahash_done);
2053}
2054
2055static int ahash_update(struct ahash_request *areq)
2056{
2057 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2058
2059 req_ctx->last = 0;
2060
2061 return ahash_process_req(areq, areq->nbytes);
2062}
2063
2064static int ahash_final(struct ahash_request *areq)
2065{
2066 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2067
2068 req_ctx->last = 1;
2069
2070 return ahash_process_req(areq, 0);
2071}
2072
2073static int ahash_finup(struct ahash_request *areq)
2074{
2075 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2076
2077 req_ctx->last = 1;
2078
2079 return ahash_process_req(areq, areq->nbytes);
2080}
2081
2082static int ahash_digest(struct ahash_request *areq)
2083{
2084 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
Kim Phillips60f208d2010-05-19 19:21:53 +10002085 struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
Lee Nipper497f2e62010-05-19 19:20:36 +10002086
Kim Phillips60f208d2010-05-19 19:21:53 +10002087 ahash->init(areq);
Lee Nipper497f2e62010-05-19 19:20:36 +10002088 req_ctx->last = 1;
2089
2090 return ahash_process_req(areq, areq->nbytes);
2091}
2092
Horia Geant?3639ca82016-04-21 19:24:55 +03002093static int ahash_export(struct ahash_request *areq, void *out)
2094{
2095 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2096 struct talitos_export_state *export = out;
LEROY Christophe49f97832017-10-06 15:05:04 +02002097 struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq);
2098 struct talitos_ctx *ctx = crypto_ahash_ctx(ahash);
2099 struct device *dev = ctx->dev;
Horia Geant?3639ca82016-04-21 19:24:55 +03002100
LEROY Christophe49f97832017-10-06 15:05:04 +02002101 dma_sync_single_for_cpu(dev, ctx->dma_hw_context,
2102 req_ctx->hw_context_size, DMA_FROM_DEVICE);
Horia Geant?3639ca82016-04-21 19:24:55 +03002103 memcpy(export->hw_context, req_ctx->hw_context,
2104 req_ctx->hw_context_size);
2105 memcpy(export->buf, req_ctx->buf, req_ctx->nbuf);
2106 export->swinit = req_ctx->swinit;
2107 export->first = req_ctx->first;
2108 export->last = req_ctx->last;
2109 export->to_hash_later = req_ctx->to_hash_later;
2110 export->nbuf = req_ctx->nbuf;
2111
2112 return 0;
2113}
2114
2115static int ahash_import(struct ahash_request *areq, const void *in)
2116{
2117 struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2118 struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2119 const struct talitos_export_state *export = in;
LEROY Christophe49f97832017-10-06 15:05:04 +02002120 unsigned int size;
2121 struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
2122 struct device *dev = ctx->dev;
LEROY Christophe37b5e882017-10-06 15:05:06 +02002123 struct talitos_private *priv = dev_get_drvdata(dev);
2124 bool is_sec1 = has_ftr_sec1(priv);
Horia Geant?3639ca82016-04-21 19:24:55 +03002125
2126 memset(req_ctx, 0, sizeof(*req_ctx));
LEROY Christophe49f97832017-10-06 15:05:04 +02002127 size = (crypto_ahash_digestsize(tfm) <= SHA256_DIGEST_SIZE)
Horia Geant?3639ca82016-04-21 19:24:55 +03002128 ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
2129 : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
LEROY Christophe49f97832017-10-06 15:05:04 +02002130 req_ctx->hw_context_size = size;
2131 if (ctx->dma_hw_context)
2132 dma_unmap_single(dev, ctx->dma_hw_context, size,
2133 DMA_BIDIRECTIONAL);
2134
2135 memcpy(req_ctx->hw_context, export->hw_context, size);
2136 ctx->dma_hw_context = dma_map_single(dev, req_ctx->hw_context, size,
2137 DMA_BIDIRECTIONAL);
LEROY Christophe37b5e882017-10-06 15:05:06 +02002138 if (ctx->dma_buf)
2139 dma_unmap_single(dev, ctx->dma_buf, sizeof(req_ctx->buf),
2140 DMA_TO_DEVICE);
Horia Geant?3639ca82016-04-21 19:24:55 +03002141 memcpy(req_ctx->buf, export->buf, export->nbuf);
LEROY Christophe37b5e882017-10-06 15:05:06 +02002142 if (is_sec1)
2143 ctx->dma_buf = dma_map_single(dev, req_ctx->buf,
2144 sizeof(req_ctx->buf),
2145 DMA_TO_DEVICE);
Horia Geant?3639ca82016-04-21 19:24:55 +03002146 req_ctx->swinit = export->swinit;
2147 req_ctx->first = export->first;
2148 req_ctx->last = export->last;
2149 req_ctx->to_hash_later = export->to_hash_later;
2150 req_ctx->nbuf = export->nbuf;
2151
2152 return 0;
2153}
2154
Lee Nipper79b3a412011-11-21 16:13:25 +08002155struct keyhash_result {
2156 struct completion completion;
2157 int err;
2158};
2159
2160static void keyhash_complete(struct crypto_async_request *req, int err)
2161{
2162 struct keyhash_result *res = req->data;
2163
2164 if (err == -EINPROGRESS)
2165 return;
2166
2167 res->err = err;
2168 complete(&res->completion);
2169}
2170
2171static int keyhash(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen,
2172 u8 *hash)
2173{
2174 struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
2175
2176 struct scatterlist sg[1];
2177 struct ahash_request *req;
2178 struct keyhash_result hresult;
2179 int ret;
2180
2181 init_completion(&hresult.completion);
2182
2183 req = ahash_request_alloc(tfm, GFP_KERNEL);
2184 if (!req)
2185 return -ENOMEM;
2186
2187 /* Keep tfm keylen == 0 during hash of the long key */
2188 ctx->keylen = 0;
2189 ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
2190 keyhash_complete, &hresult);
2191
2192 sg_init_one(&sg[0], key, keylen);
2193
2194 ahash_request_set_crypt(req, sg, hash, keylen);
2195 ret = crypto_ahash_digest(req);
2196 switch (ret) {
2197 case 0:
2198 break;
2199 case -EINPROGRESS:
2200 case -EBUSY:
2201 ret = wait_for_completion_interruptible(
2202 &hresult.completion);
2203 if (!ret)
2204 ret = hresult.err;
2205 break;
2206 default:
2207 break;
2208 }
2209 ahash_request_free(req);
2210
2211 return ret;
2212}
2213
2214static int ahash_setkey(struct crypto_ahash *tfm, const u8 *key,
2215 unsigned int keylen)
2216{
2217 struct talitos_ctx *ctx = crypto_tfm_ctx(crypto_ahash_tfm(tfm));
LEROY Christophe2e13ce02017-10-06 15:05:02 +02002218 struct device *dev = ctx->dev;
Lee Nipper79b3a412011-11-21 16:13:25 +08002219 unsigned int blocksize =
2220 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
2221 unsigned int digestsize = crypto_ahash_digestsize(tfm);
2222 unsigned int keysize = keylen;
2223 u8 hash[SHA512_DIGEST_SIZE];
2224 int ret;
2225
2226 if (keylen <= blocksize)
2227 memcpy(ctx->key, key, keysize);
2228 else {
2229 /* Must get the hash of the long key */
2230 ret = keyhash(tfm, key, keylen, hash);
2231
2232 if (ret) {
2233 crypto_ahash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
2234 return -EINVAL;
2235 }
2236
2237 keysize = digestsize;
2238 memcpy(ctx->key, hash, digestsize);
2239 }
2240
LEROY Christophe2e13ce02017-10-06 15:05:02 +02002241 if (ctx->keylen)
2242 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE);
2243
Lee Nipper79b3a412011-11-21 16:13:25 +08002244 ctx->keylen = keysize;
LEROY Christophe2e13ce02017-10-06 15:05:02 +02002245 ctx->dma_key = dma_map_single(dev, ctx->key, keysize, DMA_TO_DEVICE);
Lee Nipper79b3a412011-11-21 16:13:25 +08002246
2247 return 0;
2248}
2249
2250
Kim Phillips9c4a7962008-06-23 19:50:15 +08002251struct talitos_alg_template {
Lee Nipperd5e4aae2010-05-19 19:18:38 +10002252 u32 type;
LEROY Christopheb0057762016-06-06 13:20:44 +02002253 u32 priority;
Lee Nipperd5e4aae2010-05-19 19:18:38 +10002254 union {
2255 struct crypto_alg crypto;
Lee Nipperacbf7c622010-05-19 19:19:33 +10002256 struct ahash_alg hash;
Herbert Xuaeb4c132015-07-30 17:53:22 +08002257 struct aead_alg aead;
Lee Nipperd5e4aae2010-05-19 19:18:38 +10002258 } alg;
Kim Phillips9c4a7962008-06-23 19:50:15 +08002259 __be32 desc_hdr_template;
2260};
2261
2262static struct talitos_alg_template driver_algs[] = {
Horia Geanta991155b2013-03-20 16:31:38 +02002263 /* AEAD algorithms. These use a single-pass ipsec_esp descriptor */
Lee Nipperd5e4aae2010-05-19 19:18:38 +10002264 { .type = CRYPTO_ALG_TYPE_AEAD,
Herbert Xuaeb4c132015-07-30 17:53:22 +08002265 .alg.aead = {
2266 .base = {
2267 .cra_name = "authenc(hmac(sha1),cbc(aes))",
2268 .cra_driver_name = "authenc-hmac-sha1-"
2269 "cbc-aes-talitos",
2270 .cra_blocksize = AES_BLOCK_SIZE,
2271 .cra_flags = CRYPTO_ALG_ASYNC,
2272 },
2273 .ivsize = AES_BLOCK_SIZE,
2274 .maxauthsize = SHA1_DIGEST_SIZE,
Lee Nipper56af8cd2009-03-29 15:50:50 +08002275 },
Kim Phillips9c4a7962008-06-23 19:50:15 +08002276 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2277 DESC_HDR_SEL0_AESU |
2278 DESC_HDR_MODE0_AESU_CBC |
2279 DESC_HDR_SEL1_MDEUA |
2280 DESC_HDR_MODE1_MDEU_INIT |
2281 DESC_HDR_MODE1_MDEU_PAD |
2282 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
Lee Nipper70bcaca2008-07-03 19:08:46 +08002283 },
Lee Nipperd5e4aae2010-05-19 19:18:38 +10002284 { .type = CRYPTO_ALG_TYPE_AEAD,
LEROY Christophe7405c8d2016-06-06 13:20:46 +02002285 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2286 .alg.aead = {
2287 .base = {
2288 .cra_name = "authenc(hmac(sha1),cbc(aes))",
2289 .cra_driver_name = "authenc-hmac-sha1-"
2290 "cbc-aes-talitos",
2291 .cra_blocksize = AES_BLOCK_SIZE,
2292 .cra_flags = CRYPTO_ALG_ASYNC,
2293 },
2294 .ivsize = AES_BLOCK_SIZE,
2295 .maxauthsize = SHA1_DIGEST_SIZE,
2296 },
2297 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2298 DESC_HDR_SEL0_AESU |
2299 DESC_HDR_MODE0_AESU_CBC |
2300 DESC_HDR_SEL1_MDEUA |
2301 DESC_HDR_MODE1_MDEU_INIT |
2302 DESC_HDR_MODE1_MDEU_PAD |
2303 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2304 },
2305 { .type = CRYPTO_ALG_TYPE_AEAD,
Herbert Xuaeb4c132015-07-30 17:53:22 +08002306 .alg.aead = {
2307 .base = {
2308 .cra_name = "authenc(hmac(sha1),"
2309 "cbc(des3_ede))",
2310 .cra_driver_name = "authenc-hmac-sha1-"
2311 "cbc-3des-talitos",
2312 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2313 .cra_flags = CRYPTO_ALG_ASYNC,
2314 },
2315 .ivsize = DES3_EDE_BLOCK_SIZE,
2316 .maxauthsize = SHA1_DIGEST_SIZE,
Lee Nipper56af8cd2009-03-29 15:50:50 +08002317 },
Lee Nipper70bcaca2008-07-03 19:08:46 +08002318 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2319 DESC_HDR_SEL0_DEU |
2320 DESC_HDR_MODE0_DEU_CBC |
2321 DESC_HDR_MODE0_DEU_3DES |
2322 DESC_HDR_SEL1_MDEUA |
2323 DESC_HDR_MODE1_MDEU_INIT |
2324 DESC_HDR_MODE1_MDEU_PAD |
2325 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
Lee Nipper3952f172008-07-10 18:29:18 +08002326 },
LEROY Christophe7405c8d2016-06-06 13:20:46 +02002327 { .type = CRYPTO_ALG_TYPE_AEAD,
2328 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2329 .alg.aead = {
2330 .base = {
2331 .cra_name = "authenc(hmac(sha1),"
2332 "cbc(des3_ede))",
2333 .cra_driver_name = "authenc-hmac-sha1-"
2334 "cbc-3des-talitos",
2335 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2336 .cra_flags = CRYPTO_ALG_ASYNC,
2337 },
2338 .ivsize = DES3_EDE_BLOCK_SIZE,
2339 .maxauthsize = SHA1_DIGEST_SIZE,
2340 },
2341 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2342 DESC_HDR_SEL0_DEU |
2343 DESC_HDR_MODE0_DEU_CBC |
2344 DESC_HDR_MODE0_DEU_3DES |
2345 DESC_HDR_SEL1_MDEUA |
2346 DESC_HDR_MODE1_MDEU_INIT |
2347 DESC_HDR_MODE1_MDEU_PAD |
2348 DESC_HDR_MODE1_MDEU_SHA1_HMAC,
2349 },
Horia Geanta357fb602012-07-03 19:16:53 +03002350 { .type = CRYPTO_ALG_TYPE_AEAD,
Herbert Xuaeb4c132015-07-30 17:53:22 +08002351 .alg.aead = {
2352 .base = {
2353 .cra_name = "authenc(hmac(sha224),cbc(aes))",
2354 .cra_driver_name = "authenc-hmac-sha224-"
2355 "cbc-aes-talitos",
2356 .cra_blocksize = AES_BLOCK_SIZE,
2357 .cra_flags = CRYPTO_ALG_ASYNC,
2358 },
2359 .ivsize = AES_BLOCK_SIZE,
2360 .maxauthsize = SHA224_DIGEST_SIZE,
Horia Geanta357fb602012-07-03 19:16:53 +03002361 },
2362 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2363 DESC_HDR_SEL0_AESU |
2364 DESC_HDR_MODE0_AESU_CBC |
2365 DESC_HDR_SEL1_MDEUA |
2366 DESC_HDR_MODE1_MDEU_INIT |
2367 DESC_HDR_MODE1_MDEU_PAD |
2368 DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2369 },
LEROY Christophe7405c8d2016-06-06 13:20:46 +02002370 { .type = CRYPTO_ALG_TYPE_AEAD,
2371 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2372 .alg.aead = {
2373 .base = {
2374 .cra_name = "authenc(hmac(sha224),cbc(aes))",
2375 .cra_driver_name = "authenc-hmac-sha224-"
2376 "cbc-aes-talitos",
2377 .cra_blocksize = AES_BLOCK_SIZE,
2378 .cra_flags = CRYPTO_ALG_ASYNC,
2379 },
2380 .ivsize = AES_BLOCK_SIZE,
2381 .maxauthsize = SHA224_DIGEST_SIZE,
2382 },
2383 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2384 DESC_HDR_SEL0_AESU |
2385 DESC_HDR_MODE0_AESU_CBC |
2386 DESC_HDR_SEL1_MDEUA |
2387 DESC_HDR_MODE1_MDEU_INIT |
2388 DESC_HDR_MODE1_MDEU_PAD |
2389 DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2390 },
Horia Geanta357fb602012-07-03 19:16:53 +03002391 { .type = CRYPTO_ALG_TYPE_AEAD,
Herbert Xuaeb4c132015-07-30 17:53:22 +08002392 .alg.aead = {
2393 .base = {
2394 .cra_name = "authenc(hmac(sha224),"
2395 "cbc(des3_ede))",
2396 .cra_driver_name = "authenc-hmac-sha224-"
2397 "cbc-3des-talitos",
2398 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2399 .cra_flags = CRYPTO_ALG_ASYNC,
2400 },
2401 .ivsize = DES3_EDE_BLOCK_SIZE,
2402 .maxauthsize = SHA224_DIGEST_SIZE,
Horia Geanta357fb602012-07-03 19:16:53 +03002403 },
2404 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2405 DESC_HDR_SEL0_DEU |
2406 DESC_HDR_MODE0_DEU_CBC |
2407 DESC_HDR_MODE0_DEU_3DES |
2408 DESC_HDR_SEL1_MDEUA |
2409 DESC_HDR_MODE1_MDEU_INIT |
2410 DESC_HDR_MODE1_MDEU_PAD |
2411 DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2412 },
Lee Nipperd5e4aae2010-05-19 19:18:38 +10002413 { .type = CRYPTO_ALG_TYPE_AEAD,
LEROY Christophe7405c8d2016-06-06 13:20:46 +02002414 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2415 .alg.aead = {
2416 .base = {
2417 .cra_name = "authenc(hmac(sha224),"
2418 "cbc(des3_ede))",
2419 .cra_driver_name = "authenc-hmac-sha224-"
2420 "cbc-3des-talitos",
2421 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2422 .cra_flags = CRYPTO_ALG_ASYNC,
2423 },
2424 .ivsize = DES3_EDE_BLOCK_SIZE,
2425 .maxauthsize = SHA224_DIGEST_SIZE,
2426 },
2427 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2428 DESC_HDR_SEL0_DEU |
2429 DESC_HDR_MODE0_DEU_CBC |
2430 DESC_HDR_MODE0_DEU_3DES |
2431 DESC_HDR_SEL1_MDEUA |
2432 DESC_HDR_MODE1_MDEU_INIT |
2433 DESC_HDR_MODE1_MDEU_PAD |
2434 DESC_HDR_MODE1_MDEU_SHA224_HMAC,
2435 },
2436 { .type = CRYPTO_ALG_TYPE_AEAD,
Herbert Xuaeb4c132015-07-30 17:53:22 +08002437 .alg.aead = {
2438 .base = {
2439 .cra_name = "authenc(hmac(sha256),cbc(aes))",
2440 .cra_driver_name = "authenc-hmac-sha256-"
2441 "cbc-aes-talitos",
2442 .cra_blocksize = AES_BLOCK_SIZE,
2443 .cra_flags = CRYPTO_ALG_ASYNC,
2444 },
2445 .ivsize = AES_BLOCK_SIZE,
2446 .maxauthsize = SHA256_DIGEST_SIZE,
Lee Nipper56af8cd2009-03-29 15:50:50 +08002447 },
Lee Nipper3952f172008-07-10 18:29:18 +08002448 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2449 DESC_HDR_SEL0_AESU |
2450 DESC_HDR_MODE0_AESU_CBC |
2451 DESC_HDR_SEL1_MDEUA |
2452 DESC_HDR_MODE1_MDEU_INIT |
2453 DESC_HDR_MODE1_MDEU_PAD |
2454 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2455 },
Lee Nipperd5e4aae2010-05-19 19:18:38 +10002456 { .type = CRYPTO_ALG_TYPE_AEAD,
LEROY Christophe7405c8d2016-06-06 13:20:46 +02002457 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2458 .alg.aead = {
2459 .base = {
2460 .cra_name = "authenc(hmac(sha256),cbc(aes))",
2461 .cra_driver_name = "authenc-hmac-sha256-"
2462 "cbc-aes-talitos",
2463 .cra_blocksize = AES_BLOCK_SIZE,
2464 .cra_flags = CRYPTO_ALG_ASYNC,
2465 },
2466 .ivsize = AES_BLOCK_SIZE,
2467 .maxauthsize = SHA256_DIGEST_SIZE,
2468 },
2469 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2470 DESC_HDR_SEL0_AESU |
2471 DESC_HDR_MODE0_AESU_CBC |
2472 DESC_HDR_SEL1_MDEUA |
2473 DESC_HDR_MODE1_MDEU_INIT |
2474 DESC_HDR_MODE1_MDEU_PAD |
2475 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2476 },
2477 { .type = CRYPTO_ALG_TYPE_AEAD,
Herbert Xuaeb4c132015-07-30 17:53:22 +08002478 .alg.aead = {
2479 .base = {
2480 .cra_name = "authenc(hmac(sha256),"
2481 "cbc(des3_ede))",
2482 .cra_driver_name = "authenc-hmac-sha256-"
2483 "cbc-3des-talitos",
2484 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2485 .cra_flags = CRYPTO_ALG_ASYNC,
2486 },
2487 .ivsize = DES3_EDE_BLOCK_SIZE,
2488 .maxauthsize = SHA256_DIGEST_SIZE,
Lee Nipper56af8cd2009-03-29 15:50:50 +08002489 },
Lee Nipper3952f172008-07-10 18:29:18 +08002490 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2491 DESC_HDR_SEL0_DEU |
2492 DESC_HDR_MODE0_DEU_CBC |
2493 DESC_HDR_MODE0_DEU_3DES |
2494 DESC_HDR_SEL1_MDEUA |
2495 DESC_HDR_MODE1_MDEU_INIT |
2496 DESC_HDR_MODE1_MDEU_PAD |
2497 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2498 },
Lee Nipperd5e4aae2010-05-19 19:18:38 +10002499 { .type = CRYPTO_ALG_TYPE_AEAD,
LEROY Christophe7405c8d2016-06-06 13:20:46 +02002500 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2501 .alg.aead = {
2502 .base = {
2503 .cra_name = "authenc(hmac(sha256),"
2504 "cbc(des3_ede))",
2505 .cra_driver_name = "authenc-hmac-sha256-"
2506 "cbc-3des-talitos",
2507 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2508 .cra_flags = CRYPTO_ALG_ASYNC,
2509 },
2510 .ivsize = DES3_EDE_BLOCK_SIZE,
2511 .maxauthsize = SHA256_DIGEST_SIZE,
2512 },
2513 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2514 DESC_HDR_SEL0_DEU |
2515 DESC_HDR_MODE0_DEU_CBC |
2516 DESC_HDR_MODE0_DEU_3DES |
2517 DESC_HDR_SEL1_MDEUA |
2518 DESC_HDR_MODE1_MDEU_INIT |
2519 DESC_HDR_MODE1_MDEU_PAD |
2520 DESC_HDR_MODE1_MDEU_SHA256_HMAC,
2521 },
2522 { .type = CRYPTO_ALG_TYPE_AEAD,
Herbert Xuaeb4c132015-07-30 17:53:22 +08002523 .alg.aead = {
2524 .base = {
2525 .cra_name = "authenc(hmac(sha384),cbc(aes))",
2526 .cra_driver_name = "authenc-hmac-sha384-"
2527 "cbc-aes-talitos",
2528 .cra_blocksize = AES_BLOCK_SIZE,
2529 .cra_flags = CRYPTO_ALG_ASYNC,
2530 },
2531 .ivsize = AES_BLOCK_SIZE,
2532 .maxauthsize = SHA384_DIGEST_SIZE,
Horia Geanta357fb602012-07-03 19:16:53 +03002533 },
2534 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2535 DESC_HDR_SEL0_AESU |
2536 DESC_HDR_MODE0_AESU_CBC |
2537 DESC_HDR_SEL1_MDEUB |
2538 DESC_HDR_MODE1_MDEU_INIT |
2539 DESC_HDR_MODE1_MDEU_PAD |
2540 DESC_HDR_MODE1_MDEUB_SHA384_HMAC,
2541 },
2542 { .type = CRYPTO_ALG_TYPE_AEAD,
Herbert Xuaeb4c132015-07-30 17:53:22 +08002543 .alg.aead = {
2544 .base = {
2545 .cra_name = "authenc(hmac(sha384),"
2546 "cbc(des3_ede))",
2547 .cra_driver_name = "authenc-hmac-sha384-"
2548 "cbc-3des-talitos",
2549 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2550 .cra_flags = CRYPTO_ALG_ASYNC,
2551 },
2552 .ivsize = DES3_EDE_BLOCK_SIZE,
2553 .maxauthsize = SHA384_DIGEST_SIZE,
Horia Geanta357fb602012-07-03 19:16:53 +03002554 },
2555 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2556 DESC_HDR_SEL0_DEU |
2557 DESC_HDR_MODE0_DEU_CBC |
2558 DESC_HDR_MODE0_DEU_3DES |
2559 DESC_HDR_SEL1_MDEUB |
2560 DESC_HDR_MODE1_MDEU_INIT |
2561 DESC_HDR_MODE1_MDEU_PAD |
2562 DESC_HDR_MODE1_MDEUB_SHA384_HMAC,
2563 },
2564 { .type = CRYPTO_ALG_TYPE_AEAD,
Herbert Xuaeb4c132015-07-30 17:53:22 +08002565 .alg.aead = {
2566 .base = {
2567 .cra_name = "authenc(hmac(sha512),cbc(aes))",
2568 .cra_driver_name = "authenc-hmac-sha512-"
2569 "cbc-aes-talitos",
2570 .cra_blocksize = AES_BLOCK_SIZE,
2571 .cra_flags = CRYPTO_ALG_ASYNC,
2572 },
2573 .ivsize = AES_BLOCK_SIZE,
2574 .maxauthsize = SHA512_DIGEST_SIZE,
Horia Geanta357fb602012-07-03 19:16:53 +03002575 },
2576 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2577 DESC_HDR_SEL0_AESU |
2578 DESC_HDR_MODE0_AESU_CBC |
2579 DESC_HDR_SEL1_MDEUB |
2580 DESC_HDR_MODE1_MDEU_INIT |
2581 DESC_HDR_MODE1_MDEU_PAD |
2582 DESC_HDR_MODE1_MDEUB_SHA512_HMAC,
2583 },
2584 { .type = CRYPTO_ALG_TYPE_AEAD,
Herbert Xuaeb4c132015-07-30 17:53:22 +08002585 .alg.aead = {
2586 .base = {
2587 .cra_name = "authenc(hmac(sha512),"
2588 "cbc(des3_ede))",
2589 .cra_driver_name = "authenc-hmac-sha512-"
2590 "cbc-3des-talitos",
2591 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2592 .cra_flags = CRYPTO_ALG_ASYNC,
2593 },
2594 .ivsize = DES3_EDE_BLOCK_SIZE,
2595 .maxauthsize = SHA512_DIGEST_SIZE,
Horia Geanta357fb602012-07-03 19:16:53 +03002596 },
2597 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2598 DESC_HDR_SEL0_DEU |
2599 DESC_HDR_MODE0_DEU_CBC |
2600 DESC_HDR_MODE0_DEU_3DES |
2601 DESC_HDR_SEL1_MDEUB |
2602 DESC_HDR_MODE1_MDEU_INIT |
2603 DESC_HDR_MODE1_MDEU_PAD |
2604 DESC_HDR_MODE1_MDEUB_SHA512_HMAC,
2605 },
2606 { .type = CRYPTO_ALG_TYPE_AEAD,
Herbert Xuaeb4c132015-07-30 17:53:22 +08002607 .alg.aead = {
2608 .base = {
2609 .cra_name = "authenc(hmac(md5),cbc(aes))",
2610 .cra_driver_name = "authenc-hmac-md5-"
2611 "cbc-aes-talitos",
2612 .cra_blocksize = AES_BLOCK_SIZE,
2613 .cra_flags = CRYPTO_ALG_ASYNC,
2614 },
2615 .ivsize = AES_BLOCK_SIZE,
2616 .maxauthsize = MD5_DIGEST_SIZE,
Lee Nipper56af8cd2009-03-29 15:50:50 +08002617 },
Lee Nipper3952f172008-07-10 18:29:18 +08002618 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2619 DESC_HDR_SEL0_AESU |
2620 DESC_HDR_MODE0_AESU_CBC |
2621 DESC_HDR_SEL1_MDEUA |
2622 DESC_HDR_MODE1_MDEU_INIT |
2623 DESC_HDR_MODE1_MDEU_PAD |
2624 DESC_HDR_MODE1_MDEU_MD5_HMAC,
2625 },
Lee Nipperd5e4aae2010-05-19 19:18:38 +10002626 { .type = CRYPTO_ALG_TYPE_AEAD,
LEROY Christophe7405c8d2016-06-06 13:20:46 +02002627 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2628 .alg.aead = {
2629 .base = {
2630 .cra_name = "authenc(hmac(md5),cbc(aes))",
2631 .cra_driver_name = "authenc-hmac-md5-"
2632 "cbc-aes-talitos",
2633 .cra_blocksize = AES_BLOCK_SIZE,
2634 .cra_flags = CRYPTO_ALG_ASYNC,
2635 },
2636 .ivsize = AES_BLOCK_SIZE,
2637 .maxauthsize = MD5_DIGEST_SIZE,
2638 },
2639 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2640 DESC_HDR_SEL0_AESU |
2641 DESC_HDR_MODE0_AESU_CBC |
2642 DESC_HDR_SEL1_MDEUA |
2643 DESC_HDR_MODE1_MDEU_INIT |
2644 DESC_HDR_MODE1_MDEU_PAD |
2645 DESC_HDR_MODE1_MDEU_MD5_HMAC,
2646 },
2647 { .type = CRYPTO_ALG_TYPE_AEAD,
Herbert Xuaeb4c132015-07-30 17:53:22 +08002648 .alg.aead = {
2649 .base = {
2650 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2651 .cra_driver_name = "authenc-hmac-md5-"
2652 "cbc-3des-talitos",
2653 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2654 .cra_flags = CRYPTO_ALG_ASYNC,
2655 },
2656 .ivsize = DES3_EDE_BLOCK_SIZE,
2657 .maxauthsize = MD5_DIGEST_SIZE,
Lee Nipper56af8cd2009-03-29 15:50:50 +08002658 },
Lee Nipper3952f172008-07-10 18:29:18 +08002659 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
2660 DESC_HDR_SEL0_DEU |
2661 DESC_HDR_MODE0_DEU_CBC |
2662 DESC_HDR_MODE0_DEU_3DES |
2663 DESC_HDR_SEL1_MDEUA |
2664 DESC_HDR_MODE1_MDEU_INIT |
2665 DESC_HDR_MODE1_MDEU_PAD |
2666 DESC_HDR_MODE1_MDEU_MD5_HMAC,
Lee Nipper4de9d0b2009-03-29 15:52:32 +08002667 },
LEROY Christophe7405c8d2016-06-06 13:20:46 +02002668 { .type = CRYPTO_ALG_TYPE_AEAD,
2669 .priority = TALITOS_CRA_PRIORITY_AEAD_HSNA,
2670 .alg.aead = {
2671 .base = {
2672 .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
2673 .cra_driver_name = "authenc-hmac-md5-"
2674 "cbc-3des-talitos",
2675 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2676 .cra_flags = CRYPTO_ALG_ASYNC,
2677 },
2678 .ivsize = DES3_EDE_BLOCK_SIZE,
2679 .maxauthsize = MD5_DIGEST_SIZE,
2680 },
2681 .desc_hdr_template = DESC_HDR_TYPE_HMAC_SNOOP_NO_AFEU |
2682 DESC_HDR_SEL0_DEU |
2683 DESC_HDR_MODE0_DEU_CBC |
2684 DESC_HDR_MODE0_DEU_3DES |
2685 DESC_HDR_SEL1_MDEUA |
2686 DESC_HDR_MODE1_MDEU_INIT |
2687 DESC_HDR_MODE1_MDEU_PAD |
2688 DESC_HDR_MODE1_MDEU_MD5_HMAC,
2689 },
Lee Nipper4de9d0b2009-03-29 15:52:32 +08002690 /* ABLKCIPHER algorithms. */
Lee Nipperd5e4aae2010-05-19 19:18:38 +10002691 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2692 .alg.crypto = {
LEROY Christophe5e75ae12015-12-01 12:44:15 +01002693 .cra_name = "ecb(aes)",
2694 .cra_driver_name = "ecb-aes-talitos",
2695 .cra_blocksize = AES_BLOCK_SIZE,
2696 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2697 CRYPTO_ALG_ASYNC,
2698 .cra_ablkcipher = {
2699 .min_keysize = AES_MIN_KEY_SIZE,
2700 .max_keysize = AES_MAX_KEY_SIZE,
2701 .ivsize = AES_BLOCK_SIZE,
2702 }
2703 },
2704 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2705 DESC_HDR_SEL0_AESU,
2706 },
2707 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2708 .alg.crypto = {
Lee Nipper4de9d0b2009-03-29 15:52:32 +08002709 .cra_name = "cbc(aes)",
2710 .cra_driver_name = "cbc-aes-talitos",
2711 .cra_blocksize = AES_BLOCK_SIZE,
2712 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2713 CRYPTO_ALG_ASYNC,
Lee Nipper4de9d0b2009-03-29 15:52:32 +08002714 .cra_ablkcipher = {
Lee Nipper4de9d0b2009-03-29 15:52:32 +08002715 .min_keysize = AES_MIN_KEY_SIZE,
2716 .max_keysize = AES_MAX_KEY_SIZE,
2717 .ivsize = AES_BLOCK_SIZE,
2718 }
2719 },
2720 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2721 DESC_HDR_SEL0_AESU |
2722 DESC_HDR_MODE0_AESU_CBC,
2723 },
Lee Nipperd5e4aae2010-05-19 19:18:38 +10002724 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2725 .alg.crypto = {
LEROY Christophe5e75ae12015-12-01 12:44:15 +01002726 .cra_name = "ctr(aes)",
2727 .cra_driver_name = "ctr-aes-talitos",
2728 .cra_blocksize = AES_BLOCK_SIZE,
2729 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2730 CRYPTO_ALG_ASYNC,
2731 .cra_ablkcipher = {
2732 .min_keysize = AES_MIN_KEY_SIZE,
2733 .max_keysize = AES_MAX_KEY_SIZE,
2734 .ivsize = AES_BLOCK_SIZE,
2735 }
2736 },
LEROY Christophe70d355c2017-10-06 15:04:43 +02002737 .desc_hdr_template = DESC_HDR_TYPE_AESU_CTR_NONSNOOP |
LEROY Christophe5e75ae12015-12-01 12:44:15 +01002738 DESC_HDR_SEL0_AESU |
2739 DESC_HDR_MODE0_AESU_CTR,
2740 },
2741 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2742 .alg.crypto = {
2743 .cra_name = "ecb(des)",
2744 .cra_driver_name = "ecb-des-talitos",
2745 .cra_blocksize = DES_BLOCK_SIZE,
2746 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2747 CRYPTO_ALG_ASYNC,
2748 .cra_ablkcipher = {
2749 .min_keysize = DES_KEY_SIZE,
2750 .max_keysize = DES_KEY_SIZE,
2751 .ivsize = DES_BLOCK_SIZE,
2752 }
2753 },
2754 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2755 DESC_HDR_SEL0_DEU,
2756 },
2757 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2758 .alg.crypto = {
2759 .cra_name = "cbc(des)",
2760 .cra_driver_name = "cbc-des-talitos",
2761 .cra_blocksize = DES_BLOCK_SIZE,
2762 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2763 CRYPTO_ALG_ASYNC,
2764 .cra_ablkcipher = {
2765 .min_keysize = DES_KEY_SIZE,
2766 .max_keysize = DES_KEY_SIZE,
2767 .ivsize = DES_BLOCK_SIZE,
2768 }
2769 },
2770 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2771 DESC_HDR_SEL0_DEU |
2772 DESC_HDR_MODE0_DEU_CBC,
2773 },
2774 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2775 .alg.crypto = {
2776 .cra_name = "ecb(des3_ede)",
2777 .cra_driver_name = "ecb-3des-talitos",
2778 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2779 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2780 CRYPTO_ALG_ASYNC,
2781 .cra_ablkcipher = {
2782 .min_keysize = DES3_EDE_KEY_SIZE,
2783 .max_keysize = DES3_EDE_KEY_SIZE,
2784 .ivsize = DES3_EDE_BLOCK_SIZE,
2785 }
2786 },
2787 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2788 DESC_HDR_SEL0_DEU |
2789 DESC_HDR_MODE0_DEU_3DES,
2790 },
2791 { .type = CRYPTO_ALG_TYPE_ABLKCIPHER,
2792 .alg.crypto = {
Lee Nipper4de9d0b2009-03-29 15:52:32 +08002793 .cra_name = "cbc(des3_ede)",
2794 .cra_driver_name = "cbc-3des-talitos",
2795 .cra_blocksize = DES3_EDE_BLOCK_SIZE,
2796 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
2797 CRYPTO_ALG_ASYNC,
Lee Nipper4de9d0b2009-03-29 15:52:32 +08002798 .cra_ablkcipher = {
Lee Nipper4de9d0b2009-03-29 15:52:32 +08002799 .min_keysize = DES3_EDE_KEY_SIZE,
2800 .max_keysize = DES3_EDE_KEY_SIZE,
2801 .ivsize = DES3_EDE_BLOCK_SIZE,
2802 }
2803 },
2804 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2805 DESC_HDR_SEL0_DEU |
2806 DESC_HDR_MODE0_DEU_CBC |
2807 DESC_HDR_MODE0_DEU_3DES,
Lee Nipper497f2e62010-05-19 19:20:36 +10002808 },
2809 /* AHASH algorithms. */
2810 { .type = CRYPTO_ALG_TYPE_AHASH,
2811 .alg.hash = {
Lee Nipper497f2e62010-05-19 19:20:36 +10002812 .halg.digestsize = MD5_DIGEST_SIZE,
Horia Geant?3639ca82016-04-21 19:24:55 +03002813 .halg.statesize = sizeof(struct talitos_export_state),
Lee Nipper497f2e62010-05-19 19:20:36 +10002814 .halg.base = {
2815 .cra_name = "md5",
2816 .cra_driver_name = "md5-talitos",
Martin Hicksb3988612015-03-03 08:21:34 -05002817 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
Lee Nipper497f2e62010-05-19 19:20:36 +10002818 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2819 CRYPTO_ALG_ASYNC,
Lee Nipper497f2e62010-05-19 19:20:36 +10002820 }
2821 },
2822 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2823 DESC_HDR_SEL0_MDEUA |
2824 DESC_HDR_MODE0_MDEU_MD5,
2825 },
2826 { .type = CRYPTO_ALG_TYPE_AHASH,
2827 .alg.hash = {
Lee Nipper497f2e62010-05-19 19:20:36 +10002828 .halg.digestsize = SHA1_DIGEST_SIZE,
Horia Geant?3639ca82016-04-21 19:24:55 +03002829 .halg.statesize = sizeof(struct talitos_export_state),
Lee Nipper497f2e62010-05-19 19:20:36 +10002830 .halg.base = {
2831 .cra_name = "sha1",
2832 .cra_driver_name = "sha1-talitos",
2833 .cra_blocksize = SHA1_BLOCK_SIZE,
2834 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2835 CRYPTO_ALG_ASYNC,
Lee Nipper497f2e62010-05-19 19:20:36 +10002836 }
2837 },
2838 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2839 DESC_HDR_SEL0_MDEUA |
2840 DESC_HDR_MODE0_MDEU_SHA1,
2841 },
2842 { .type = CRYPTO_ALG_TYPE_AHASH,
2843 .alg.hash = {
Kim Phillips60f208d2010-05-19 19:21:53 +10002844 .halg.digestsize = SHA224_DIGEST_SIZE,
Horia Geant?3639ca82016-04-21 19:24:55 +03002845 .halg.statesize = sizeof(struct talitos_export_state),
Kim Phillips60f208d2010-05-19 19:21:53 +10002846 .halg.base = {
2847 .cra_name = "sha224",
2848 .cra_driver_name = "sha224-talitos",
2849 .cra_blocksize = SHA224_BLOCK_SIZE,
2850 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2851 CRYPTO_ALG_ASYNC,
Kim Phillips60f208d2010-05-19 19:21:53 +10002852 }
2853 },
2854 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2855 DESC_HDR_SEL0_MDEUA |
2856 DESC_HDR_MODE0_MDEU_SHA224,
2857 },
2858 { .type = CRYPTO_ALG_TYPE_AHASH,
2859 .alg.hash = {
Lee Nipper497f2e62010-05-19 19:20:36 +10002860 .halg.digestsize = SHA256_DIGEST_SIZE,
Horia Geant?3639ca82016-04-21 19:24:55 +03002861 .halg.statesize = sizeof(struct talitos_export_state),
Lee Nipper497f2e62010-05-19 19:20:36 +10002862 .halg.base = {
2863 .cra_name = "sha256",
2864 .cra_driver_name = "sha256-talitos",
2865 .cra_blocksize = SHA256_BLOCK_SIZE,
2866 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2867 CRYPTO_ALG_ASYNC,
Lee Nipper497f2e62010-05-19 19:20:36 +10002868 }
2869 },
2870 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2871 DESC_HDR_SEL0_MDEUA |
2872 DESC_HDR_MODE0_MDEU_SHA256,
2873 },
2874 { .type = CRYPTO_ALG_TYPE_AHASH,
2875 .alg.hash = {
Lee Nipper497f2e62010-05-19 19:20:36 +10002876 .halg.digestsize = SHA384_DIGEST_SIZE,
Horia Geant?3639ca82016-04-21 19:24:55 +03002877 .halg.statesize = sizeof(struct talitos_export_state),
Lee Nipper497f2e62010-05-19 19:20:36 +10002878 .halg.base = {
2879 .cra_name = "sha384",
2880 .cra_driver_name = "sha384-talitos",
2881 .cra_blocksize = SHA384_BLOCK_SIZE,
2882 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2883 CRYPTO_ALG_ASYNC,
Lee Nipper497f2e62010-05-19 19:20:36 +10002884 }
2885 },
2886 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2887 DESC_HDR_SEL0_MDEUB |
2888 DESC_HDR_MODE0_MDEUB_SHA384,
2889 },
2890 { .type = CRYPTO_ALG_TYPE_AHASH,
2891 .alg.hash = {
Lee Nipper497f2e62010-05-19 19:20:36 +10002892 .halg.digestsize = SHA512_DIGEST_SIZE,
Horia Geant?3639ca82016-04-21 19:24:55 +03002893 .halg.statesize = sizeof(struct talitos_export_state),
Lee Nipper497f2e62010-05-19 19:20:36 +10002894 .halg.base = {
2895 .cra_name = "sha512",
2896 .cra_driver_name = "sha512-talitos",
2897 .cra_blocksize = SHA512_BLOCK_SIZE,
2898 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2899 CRYPTO_ALG_ASYNC,
Lee Nipper497f2e62010-05-19 19:20:36 +10002900 }
2901 },
2902 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2903 DESC_HDR_SEL0_MDEUB |
2904 DESC_HDR_MODE0_MDEUB_SHA512,
2905 },
Lee Nipper79b3a412011-11-21 16:13:25 +08002906 { .type = CRYPTO_ALG_TYPE_AHASH,
2907 .alg.hash = {
Lee Nipper79b3a412011-11-21 16:13:25 +08002908 .halg.digestsize = MD5_DIGEST_SIZE,
Horia Geant?3639ca82016-04-21 19:24:55 +03002909 .halg.statesize = sizeof(struct talitos_export_state),
Lee Nipper79b3a412011-11-21 16:13:25 +08002910 .halg.base = {
2911 .cra_name = "hmac(md5)",
2912 .cra_driver_name = "hmac-md5-talitos",
Martin Hicksb3988612015-03-03 08:21:34 -05002913 .cra_blocksize = MD5_HMAC_BLOCK_SIZE,
Lee Nipper79b3a412011-11-21 16:13:25 +08002914 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2915 CRYPTO_ALG_ASYNC,
Lee Nipper79b3a412011-11-21 16:13:25 +08002916 }
2917 },
2918 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2919 DESC_HDR_SEL0_MDEUA |
2920 DESC_HDR_MODE0_MDEU_MD5,
2921 },
2922 { .type = CRYPTO_ALG_TYPE_AHASH,
2923 .alg.hash = {
Lee Nipper79b3a412011-11-21 16:13:25 +08002924 .halg.digestsize = SHA1_DIGEST_SIZE,
Horia Geant?3639ca82016-04-21 19:24:55 +03002925 .halg.statesize = sizeof(struct talitos_export_state),
Lee Nipper79b3a412011-11-21 16:13:25 +08002926 .halg.base = {
2927 .cra_name = "hmac(sha1)",
2928 .cra_driver_name = "hmac-sha1-talitos",
2929 .cra_blocksize = SHA1_BLOCK_SIZE,
2930 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2931 CRYPTO_ALG_ASYNC,
Lee Nipper79b3a412011-11-21 16:13:25 +08002932 }
2933 },
2934 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2935 DESC_HDR_SEL0_MDEUA |
2936 DESC_HDR_MODE0_MDEU_SHA1,
2937 },
2938 { .type = CRYPTO_ALG_TYPE_AHASH,
2939 .alg.hash = {
Lee Nipper79b3a412011-11-21 16:13:25 +08002940 .halg.digestsize = SHA224_DIGEST_SIZE,
Horia Geant?3639ca82016-04-21 19:24:55 +03002941 .halg.statesize = sizeof(struct talitos_export_state),
Lee Nipper79b3a412011-11-21 16:13:25 +08002942 .halg.base = {
2943 .cra_name = "hmac(sha224)",
2944 .cra_driver_name = "hmac-sha224-talitos",
2945 .cra_blocksize = SHA224_BLOCK_SIZE,
2946 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2947 CRYPTO_ALG_ASYNC,
Lee Nipper79b3a412011-11-21 16:13:25 +08002948 }
2949 },
2950 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2951 DESC_HDR_SEL0_MDEUA |
2952 DESC_HDR_MODE0_MDEU_SHA224,
2953 },
2954 { .type = CRYPTO_ALG_TYPE_AHASH,
2955 .alg.hash = {
Lee Nipper79b3a412011-11-21 16:13:25 +08002956 .halg.digestsize = SHA256_DIGEST_SIZE,
Horia Geant?3639ca82016-04-21 19:24:55 +03002957 .halg.statesize = sizeof(struct talitos_export_state),
Lee Nipper79b3a412011-11-21 16:13:25 +08002958 .halg.base = {
2959 .cra_name = "hmac(sha256)",
2960 .cra_driver_name = "hmac-sha256-talitos",
2961 .cra_blocksize = SHA256_BLOCK_SIZE,
2962 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2963 CRYPTO_ALG_ASYNC,
Lee Nipper79b3a412011-11-21 16:13:25 +08002964 }
2965 },
2966 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2967 DESC_HDR_SEL0_MDEUA |
2968 DESC_HDR_MODE0_MDEU_SHA256,
2969 },
2970 { .type = CRYPTO_ALG_TYPE_AHASH,
2971 .alg.hash = {
Lee Nipper79b3a412011-11-21 16:13:25 +08002972 .halg.digestsize = SHA384_DIGEST_SIZE,
Horia Geant?3639ca82016-04-21 19:24:55 +03002973 .halg.statesize = sizeof(struct talitos_export_state),
Lee Nipper79b3a412011-11-21 16:13:25 +08002974 .halg.base = {
2975 .cra_name = "hmac(sha384)",
2976 .cra_driver_name = "hmac-sha384-talitos",
2977 .cra_blocksize = SHA384_BLOCK_SIZE,
2978 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2979 CRYPTO_ALG_ASYNC,
Lee Nipper79b3a412011-11-21 16:13:25 +08002980 }
2981 },
2982 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2983 DESC_HDR_SEL0_MDEUB |
2984 DESC_HDR_MODE0_MDEUB_SHA384,
2985 },
2986 { .type = CRYPTO_ALG_TYPE_AHASH,
2987 .alg.hash = {
Lee Nipper79b3a412011-11-21 16:13:25 +08002988 .halg.digestsize = SHA512_DIGEST_SIZE,
Horia Geant?3639ca82016-04-21 19:24:55 +03002989 .halg.statesize = sizeof(struct talitos_export_state),
Lee Nipper79b3a412011-11-21 16:13:25 +08002990 .halg.base = {
2991 .cra_name = "hmac(sha512)",
2992 .cra_driver_name = "hmac-sha512-talitos",
2993 .cra_blocksize = SHA512_BLOCK_SIZE,
2994 .cra_flags = CRYPTO_ALG_TYPE_AHASH |
2995 CRYPTO_ALG_ASYNC,
Lee Nipper79b3a412011-11-21 16:13:25 +08002996 }
2997 },
2998 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
2999 DESC_HDR_SEL0_MDEUB |
3000 DESC_HDR_MODE0_MDEUB_SHA512,
3001 }
Kim Phillips9c4a7962008-06-23 19:50:15 +08003002};
3003
3004struct talitos_crypto_alg {
3005 struct list_head entry;
3006 struct device *dev;
Lee Nipperacbf7c622010-05-19 19:19:33 +10003007 struct talitos_alg_template algt;
Kim Phillips9c4a7962008-06-23 19:50:15 +08003008};
3009
Jonas Eymann89d124c2016-04-19 20:33:47 +03003010static int talitos_init_common(struct talitos_ctx *ctx,
3011 struct talitos_crypto_alg *talitos_alg)
Kim Phillips9c4a7962008-06-23 19:50:15 +08003012{
Kim Phillips5228f0f2011-07-15 11:21:38 +08003013 struct talitos_private *priv;
Kim Phillips9c4a7962008-06-23 19:50:15 +08003014
3015 /* update context with ptr to dev */
3016 ctx->dev = talitos_alg->dev;
Kim Phillips19bbbc62009-03-29 15:53:59 +08003017
Kim Phillips5228f0f2011-07-15 11:21:38 +08003018 /* assign SEC channel to tfm in round-robin fashion */
3019 priv = dev_get_drvdata(ctx->dev);
3020 ctx->ch = atomic_inc_return(&priv->last_chan) &
3021 (priv->num_channels - 1);
3022
Kim Phillips9c4a7962008-06-23 19:50:15 +08003023 /* copy descriptor header template value */
Lee Nipperacbf7c622010-05-19 19:19:33 +10003024 ctx->desc_hdr_template = talitos_alg->algt.desc_hdr_template;
Kim Phillips9c4a7962008-06-23 19:50:15 +08003025
Kim Phillips602dba52011-07-15 11:21:39 +08003026 /* select done notification */
3027 ctx->desc_hdr_template |= DESC_HDR_DONE_NOTIFY;
3028
Lee Nipper497f2e62010-05-19 19:20:36 +10003029 return 0;
3030}
3031
Jonas Eymann89d124c2016-04-19 20:33:47 +03003032static int talitos_cra_init(struct crypto_tfm *tfm)
3033{
3034 struct crypto_alg *alg = tfm->__crt_alg;
3035 struct talitos_crypto_alg *talitos_alg;
3036 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
3037
3038 if ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_AHASH)
3039 talitos_alg = container_of(__crypto_ahash_alg(alg),
3040 struct talitos_crypto_alg,
3041 algt.alg.hash);
3042 else
3043 talitos_alg = container_of(alg, struct talitos_crypto_alg,
3044 algt.alg.crypto);
3045
3046 return talitos_init_common(ctx, talitos_alg);
3047}
3048
Herbert Xuaeb4c132015-07-30 17:53:22 +08003049static int talitos_cra_init_aead(struct crypto_aead *tfm)
Lee Nipper497f2e62010-05-19 19:20:36 +10003050{
Jonas Eymann89d124c2016-04-19 20:33:47 +03003051 struct aead_alg *alg = crypto_aead_alg(tfm);
3052 struct talitos_crypto_alg *talitos_alg;
3053 struct talitos_ctx *ctx = crypto_aead_ctx(tfm);
3054
3055 talitos_alg = container_of(alg, struct talitos_crypto_alg,
3056 algt.alg.aead);
3057
3058 return talitos_init_common(ctx, talitos_alg);
Kim Phillips9c4a7962008-06-23 19:50:15 +08003059}
3060
Lee Nipper497f2e62010-05-19 19:20:36 +10003061static int talitos_cra_init_ahash(struct crypto_tfm *tfm)
3062{
3063 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
3064
3065 talitos_cra_init(tfm);
3066
3067 ctx->keylen = 0;
3068 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
3069 sizeof(struct talitos_ahash_req_ctx));
3070
3071 return 0;
3072}
3073
LEROY Christophe2e13ce02017-10-06 15:05:02 +02003074static void talitos_cra_exit(struct crypto_tfm *tfm)
3075{
3076 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
3077 struct device *dev = ctx->dev;
3078
3079 if (ctx->keylen)
3080 dma_unmap_single(dev, ctx->dma_key, ctx->keylen, DMA_TO_DEVICE);
3081}
3082
LEROY Christophe49f97832017-10-06 15:05:04 +02003083static void talitos_cra_exit_ahash(struct crypto_tfm *tfm)
3084{
3085 struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
3086 struct device *dev = ctx->dev;
3087 unsigned int size;
3088
3089 talitos_cra_exit(tfm);
3090
3091 size = (crypto_ahash_digestsize(__crypto_ahash_cast(tfm)) <=
3092 SHA256_DIGEST_SIZE)
3093 ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
3094 : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
3095
3096 if (ctx->dma_hw_context)
3097 dma_unmap_single(dev, ctx->dma_hw_context, size,
3098 DMA_BIDIRECTIONAL);
LEROY Christophe37b5e882017-10-06 15:05:06 +02003099 if (ctx->dma_buf)
3100 dma_unmap_single(dev, ctx->dma_buf, HASH_MAX_BLOCK_SIZE,
3101 DMA_TO_DEVICE);
LEROY Christophe49f97832017-10-06 15:05:04 +02003102}
3103
Kim Phillips9c4a7962008-06-23 19:50:15 +08003104/*
3105 * given the alg's descriptor header template, determine whether descriptor
3106 * type and primary/secondary execution units required match the hw
3107 * capabilities description provided in the device tree node.
3108 */
3109static int hw_supports(struct device *dev, __be32 desc_hdr_template)
3110{
3111 struct talitos_private *priv = dev_get_drvdata(dev);
3112 int ret;
3113
3114 ret = (1 << DESC_TYPE(desc_hdr_template) & priv->desc_types) &&
3115 (1 << PRIMARY_EU(desc_hdr_template) & priv->exec_units);
3116
3117 if (SECONDARY_EU(desc_hdr_template))
3118 ret = ret && (1 << SECONDARY_EU(desc_hdr_template)
3119 & priv->exec_units);
3120
3121 return ret;
3122}
3123
Grant Likely2dc11582010-08-06 09:25:50 -06003124static int talitos_remove(struct platform_device *ofdev)
Kim Phillips9c4a7962008-06-23 19:50:15 +08003125{
3126 struct device *dev = &ofdev->dev;
3127 struct talitos_private *priv = dev_get_drvdata(dev);
3128 struct talitos_crypto_alg *t_alg, *n;
3129 int i;
3130
3131 list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) {
Lee Nipperacbf7c622010-05-19 19:19:33 +10003132 switch (t_alg->algt.type) {
3133 case CRYPTO_ALG_TYPE_ABLKCIPHER:
Lee Nipperacbf7c622010-05-19 19:19:33 +10003134 break;
Herbert Xuaeb4c132015-07-30 17:53:22 +08003135 case CRYPTO_ALG_TYPE_AEAD:
3136 crypto_unregister_aead(&t_alg->algt.alg.aead);
Lee Nipperacbf7c622010-05-19 19:19:33 +10003137 case CRYPTO_ALG_TYPE_AHASH:
3138 crypto_unregister_ahash(&t_alg->algt.alg.hash);
3139 break;
3140 }
Kim Phillips9c4a7962008-06-23 19:50:15 +08003141 list_del(&t_alg->entry);
Kim Phillips9c4a7962008-06-23 19:50:15 +08003142 }
3143
3144 if (hw_supports(dev, DESC_HDR_SEL0_RNG))
3145 talitos_unregister_rng(dev);
3146
Kim Phillipsc3e337f2011-11-21 16:13:27 +08003147 for (i = 0; i < 2; i++)
Kim Phillips2cdba3c2011-12-12 14:59:11 -06003148 if (priv->irq[i]) {
Kim Phillipsc3e337f2011-11-21 16:13:27 +08003149 free_irq(priv->irq[i], dev);
3150 irq_dispose_mapping(priv->irq[i]);
3151 }
Kim Phillips9c4a7962008-06-23 19:50:15 +08003152
Kim Phillipsc3e337f2011-11-21 16:13:27 +08003153 tasklet_kill(&priv->done_task[0]);
Kim Phillips2cdba3c2011-12-12 14:59:11 -06003154 if (priv->irq[1])
Kim Phillipsc3e337f2011-11-21 16:13:27 +08003155 tasklet_kill(&priv->done_task[1]);
Kim Phillips9c4a7962008-06-23 19:50:15 +08003156
Kim Phillips9c4a7962008-06-23 19:50:15 +08003157 return 0;
3158}
3159
3160static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev,
3161 struct talitos_alg_template
3162 *template)
3163{
Kim Phillips60f208d2010-05-19 19:21:53 +10003164 struct talitos_private *priv = dev_get_drvdata(dev);
Kim Phillips9c4a7962008-06-23 19:50:15 +08003165 struct talitos_crypto_alg *t_alg;
3166 struct crypto_alg *alg;
3167
LEROY Christophe24b92ff2017-10-06 15:04:49 +02003168 t_alg = devm_kzalloc(dev, sizeof(struct talitos_crypto_alg),
3169 GFP_KERNEL);
Kim Phillips9c4a7962008-06-23 19:50:15 +08003170 if (!t_alg)
3171 return ERR_PTR(-ENOMEM);
3172
Lee Nipperacbf7c622010-05-19 19:19:33 +10003173 t_alg->algt = *template;
3174
3175 switch (t_alg->algt.type) {
3176 case CRYPTO_ALG_TYPE_ABLKCIPHER:
Lee Nipper497f2e62010-05-19 19:20:36 +10003177 alg = &t_alg->algt.alg.crypto;
3178 alg->cra_init = talitos_cra_init;
LEROY Christophe2e13ce02017-10-06 15:05:02 +02003179 alg->cra_exit = talitos_cra_exit;
Kim Phillipsd4cd3282012-08-08 20:32:00 -05003180 alg->cra_type = &crypto_ablkcipher_type;
Kim Phillipsb286e002012-08-08 20:33:34 -05003181 alg->cra_ablkcipher.setkey = ablkcipher_setkey;
3182 alg->cra_ablkcipher.encrypt = ablkcipher_encrypt;
3183 alg->cra_ablkcipher.decrypt = ablkcipher_decrypt;
3184 alg->cra_ablkcipher.geniv = "eseqiv";
Lee Nipper497f2e62010-05-19 19:20:36 +10003185 break;
Lee Nipperacbf7c622010-05-19 19:19:33 +10003186 case CRYPTO_ALG_TYPE_AEAD:
Herbert Xuaeb4c132015-07-30 17:53:22 +08003187 alg = &t_alg->algt.alg.aead.base;
LEROY Christophe2e13ce02017-10-06 15:05:02 +02003188 alg->cra_exit = talitos_cra_exit;
Herbert Xuaeb4c132015-07-30 17:53:22 +08003189 t_alg->algt.alg.aead.init = talitos_cra_init_aead;
3190 t_alg->algt.alg.aead.setkey = aead_setkey;
3191 t_alg->algt.alg.aead.encrypt = aead_encrypt;
3192 t_alg->algt.alg.aead.decrypt = aead_decrypt;
LEROY Christophe6cda0752017-10-06 15:04:39 +02003193 if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) &&
3194 !strncmp(alg->cra_name, "authenc(hmac(sha224)", 20)) {
LEROY Christophe24b92ff2017-10-06 15:04:49 +02003195 devm_kfree(dev, t_alg);
LEROY Christophe6cda0752017-10-06 15:04:39 +02003196 return ERR_PTR(-ENOTSUPP);
3197 }
Lee Nipperacbf7c622010-05-19 19:19:33 +10003198 break;
3199 case CRYPTO_ALG_TYPE_AHASH:
3200 alg = &t_alg->algt.alg.hash.halg.base;
Lee Nipper497f2e62010-05-19 19:20:36 +10003201 alg->cra_init = talitos_cra_init_ahash;
LEROY Christophe49f97832017-10-06 15:05:04 +02003202 alg->cra_exit = talitos_cra_exit_ahash;
Kim Phillipsd4cd3282012-08-08 20:32:00 -05003203 alg->cra_type = &crypto_ahash_type;
Kim Phillipsb286e002012-08-08 20:33:34 -05003204 t_alg->algt.alg.hash.init = ahash_init;
3205 t_alg->algt.alg.hash.update = ahash_update;
3206 t_alg->algt.alg.hash.final = ahash_final;
3207 t_alg->algt.alg.hash.finup = ahash_finup;
3208 t_alg->algt.alg.hash.digest = ahash_digest;
LEROY Christophe56136632017-09-12 11:03:39 +02003209 if (!strncmp(alg->cra_name, "hmac", 4))
3210 t_alg->algt.alg.hash.setkey = ahash_setkey;
Horia Geant?3639ca82016-04-21 19:24:55 +03003211 t_alg->algt.alg.hash.import = ahash_import;
3212 t_alg->algt.alg.hash.export = ahash_export;
Kim Phillipsb286e002012-08-08 20:33:34 -05003213
Lee Nipper79b3a412011-11-21 16:13:25 +08003214 if (!(priv->features & TALITOS_FTR_HMAC_OK) &&
Kim Phillips0b2730d2011-12-12 14:59:10 -06003215 !strncmp(alg->cra_name, "hmac", 4)) {
LEROY Christophe24b92ff2017-10-06 15:04:49 +02003216 devm_kfree(dev, t_alg);
Lee Nipper79b3a412011-11-21 16:13:25 +08003217 return ERR_PTR(-ENOTSUPP);
Kim Phillips0b2730d2011-12-12 14:59:10 -06003218 }
Kim Phillips60f208d2010-05-19 19:21:53 +10003219 if (!(priv->features & TALITOS_FTR_SHA224_HWINIT) &&
Lee Nipper79b3a412011-11-21 16:13:25 +08003220 (!strcmp(alg->cra_name, "sha224") ||
3221 !strcmp(alg->cra_name, "hmac(sha224)"))) {
Kim Phillips60f208d2010-05-19 19:21:53 +10003222 t_alg->algt.alg.hash.init = ahash_init_sha224_swinit;
3223 t_alg->algt.desc_hdr_template =
3224 DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
3225 DESC_HDR_SEL0_MDEUA |
3226 DESC_HDR_MODE0_MDEU_SHA256;
3227 }
Lee Nipper497f2e62010-05-19 19:20:36 +10003228 break;
Kim Phillips1d119112010-09-23 15:55:27 +08003229 default:
3230 dev_err(dev, "unknown algorithm type %d\n", t_alg->algt.type);
LEROY Christophe24b92ff2017-10-06 15:04:49 +02003231 devm_kfree(dev, t_alg);
Kim Phillips1d119112010-09-23 15:55:27 +08003232 return ERR_PTR(-EINVAL);
Lee Nipperacbf7c622010-05-19 19:19:33 +10003233 }
Kim Phillips9c4a7962008-06-23 19:50:15 +08003234
Kim Phillips9c4a7962008-06-23 19:50:15 +08003235 alg->cra_module = THIS_MODULE;
LEROY Christopheb0057762016-06-06 13:20:44 +02003236 if (t_alg->algt.priority)
3237 alg->cra_priority = t_alg->algt.priority;
3238 else
3239 alg->cra_priority = TALITOS_CRA_PRIORITY;
Kim Phillips9c4a7962008-06-23 19:50:15 +08003240 alg->cra_alignmask = 0;
Kim Phillips9c4a7962008-06-23 19:50:15 +08003241 alg->cra_ctxsize = sizeof(struct talitos_ctx);
Nikos Mavrogiannopoulosd912bb72011-11-01 13:39:56 +01003242 alg->cra_flags |= CRYPTO_ALG_KERN_DRIVER_ONLY;
Kim Phillips9c4a7962008-06-23 19:50:15 +08003243
Kim Phillips9c4a7962008-06-23 19:50:15 +08003244 t_alg->dev = dev;
3245
3246 return t_alg;
3247}
3248
Kim Phillipsc3e337f2011-11-21 16:13:27 +08003249static int talitos_probe_irq(struct platform_device *ofdev)
3250{
3251 struct device *dev = &ofdev->dev;
3252 struct device_node *np = ofdev->dev.of_node;
3253 struct talitos_private *priv = dev_get_drvdata(dev);
3254 int err;
LEROY Christophedd3c0982015-04-17 16:32:13 +02003255 bool is_sec1 = has_ftr_sec1(priv);
Kim Phillipsc3e337f2011-11-21 16:13:27 +08003256
3257 priv->irq[0] = irq_of_parse_and_map(np, 0);
Kim Phillips2cdba3c2011-12-12 14:59:11 -06003258 if (!priv->irq[0]) {
Kim Phillipsc3e337f2011-11-21 16:13:27 +08003259 dev_err(dev, "failed to map irq\n");
3260 return -EINVAL;
3261 }
LEROY Christophedd3c0982015-04-17 16:32:13 +02003262 if (is_sec1) {
3263 err = request_irq(priv->irq[0], talitos1_interrupt_4ch, 0,
3264 dev_driver_string(dev), dev);
3265 goto primary_out;
3266 }
Kim Phillipsc3e337f2011-11-21 16:13:27 +08003267
3268 priv->irq[1] = irq_of_parse_and_map(np, 1);
3269
3270 /* get the primary irq line */
Kim Phillips2cdba3c2011-12-12 14:59:11 -06003271 if (!priv->irq[1]) {
LEROY Christophedd3c0982015-04-17 16:32:13 +02003272 err = request_irq(priv->irq[0], talitos2_interrupt_4ch, 0,
Kim Phillipsc3e337f2011-11-21 16:13:27 +08003273 dev_driver_string(dev), dev);
3274 goto primary_out;
3275 }
3276
LEROY Christophedd3c0982015-04-17 16:32:13 +02003277 err = request_irq(priv->irq[0], talitos2_interrupt_ch0_2, 0,
Kim Phillipsc3e337f2011-11-21 16:13:27 +08003278 dev_driver_string(dev), dev);
3279 if (err)
3280 goto primary_out;
3281
3282 /* get the secondary irq line */
LEROY Christophedd3c0982015-04-17 16:32:13 +02003283 err = request_irq(priv->irq[1], talitos2_interrupt_ch1_3, 0,
Kim Phillipsc3e337f2011-11-21 16:13:27 +08003284 dev_driver_string(dev), dev);
3285 if (err) {
3286 dev_err(dev, "failed to request secondary irq\n");
3287 irq_dispose_mapping(priv->irq[1]);
Kim Phillips2cdba3c2011-12-12 14:59:11 -06003288 priv->irq[1] = 0;
Kim Phillipsc3e337f2011-11-21 16:13:27 +08003289 }
3290
3291 return err;
3292
3293primary_out:
3294 if (err) {
3295 dev_err(dev, "failed to request primary irq\n");
3296 irq_dispose_mapping(priv->irq[0]);
Kim Phillips2cdba3c2011-12-12 14:59:11 -06003297 priv->irq[0] = 0;
Kim Phillipsc3e337f2011-11-21 16:13:27 +08003298 }
3299
3300 return err;
3301}
3302
Grant Likely1c48a5c2011-02-17 02:43:24 -07003303static int talitos_probe(struct platform_device *ofdev)
Kim Phillips9c4a7962008-06-23 19:50:15 +08003304{
3305 struct device *dev = &ofdev->dev;
Grant Likely61c7a082010-04-13 16:12:29 -07003306 struct device_node *np = ofdev->dev.of_node;
Kim Phillips9c4a7962008-06-23 19:50:15 +08003307 struct talitos_private *priv;
Kim Phillips9c4a7962008-06-23 19:50:15 +08003308 int i, err;
LEROY Christophe5fa7fa12015-04-17 16:32:11 +02003309 int stride;
LEROY Christophefd5ea7f2017-10-06 15:04:53 +02003310 struct resource *res;
Kim Phillips9c4a7962008-06-23 19:50:15 +08003311
LEROY Christophe24b92ff2017-10-06 15:04:49 +02003312 priv = devm_kzalloc(dev, sizeof(struct talitos_private), GFP_KERNEL);
Kim Phillips9c4a7962008-06-23 19:50:15 +08003313 if (!priv)
3314 return -ENOMEM;
3315
Kevin Haof3de9cb2014-01-28 20:17:23 +08003316 INIT_LIST_HEAD(&priv->alg_list);
3317
Kim Phillips9c4a7962008-06-23 19:50:15 +08003318 dev_set_drvdata(dev, priv);
3319
3320 priv->ofdev = ofdev;
3321
Horia Geanta511d63c2012-03-30 17:49:53 +03003322 spin_lock_init(&priv->reg_lock);
3323
LEROY Christophefd5ea7f2017-10-06 15:04:53 +02003324 res = platform_get_resource(ofdev, IORESOURCE_MEM, 0);
3325 if (!res)
3326 return -ENXIO;
3327 priv->reg = devm_ioremap(dev, res->start, resource_size(res));
Kim Phillips9c4a7962008-06-23 19:50:15 +08003328 if (!priv->reg) {
3329 dev_err(dev, "failed to of_iomap\n");
3330 err = -ENOMEM;
3331 goto err_out;
3332 }
3333
3334 /* get SEC version capabilities from device tree */
LEROY Christophefa14c6c2017-10-06 15:04:51 +02003335 of_property_read_u32(np, "fsl,num-channels", &priv->num_channels);
3336 of_property_read_u32(np, "fsl,channel-fifo-len", &priv->chfifo_len);
3337 of_property_read_u32(np, "fsl,exec-units-mask", &priv->exec_units);
3338 of_property_read_u32(np, "fsl,descriptor-types-mask",
3339 &priv->desc_types);
Kim Phillips9c4a7962008-06-23 19:50:15 +08003340
3341 if (!is_power_of_2(priv->num_channels) || !priv->chfifo_len ||
3342 !priv->exec_units || !priv->desc_types) {
3343 dev_err(dev, "invalid property data in device tree node\n");
3344 err = -EINVAL;
3345 goto err_out;
3346 }
3347
Lee Nipperf3c85bc2008-07-30 16:26:57 +08003348 if (of_device_is_compatible(np, "fsl,sec3.0"))
3349 priv->features |= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT;
3350
Kim Phillipsfe5720e2008-10-12 20:33:14 +08003351 if (of_device_is_compatible(np, "fsl,sec2.1"))
Kim Phillips60f208d2010-05-19 19:21:53 +10003352 priv->features |= TALITOS_FTR_HW_AUTH_CHECK |
Lee Nipper79b3a412011-11-21 16:13:25 +08003353 TALITOS_FTR_SHA224_HWINIT |
3354 TALITOS_FTR_HMAC_OK;
Kim Phillipsfe5720e2008-10-12 20:33:14 +08003355
LEROY Christophe21590882015-04-17 16:32:05 +02003356 if (of_device_is_compatible(np, "fsl,sec1.0"))
3357 priv->features |= TALITOS_FTR_SEC1;
3358
LEROY Christophe5fa7fa12015-04-17 16:32:11 +02003359 if (of_device_is_compatible(np, "fsl,sec1.2")) {
3360 priv->reg_deu = priv->reg + TALITOS12_DEU;
3361 priv->reg_aesu = priv->reg + TALITOS12_AESU;
3362 priv->reg_mdeu = priv->reg + TALITOS12_MDEU;
3363 stride = TALITOS1_CH_STRIDE;
3364 } else if (of_device_is_compatible(np, "fsl,sec1.0")) {
3365 priv->reg_deu = priv->reg + TALITOS10_DEU;
3366 priv->reg_aesu = priv->reg + TALITOS10_AESU;
3367 priv->reg_mdeu = priv->reg + TALITOS10_MDEU;
3368 priv->reg_afeu = priv->reg + TALITOS10_AFEU;
3369 priv->reg_rngu = priv->reg + TALITOS10_RNGU;
3370 priv->reg_pkeu = priv->reg + TALITOS10_PKEU;
3371 stride = TALITOS1_CH_STRIDE;
3372 } else {
3373 priv->reg_deu = priv->reg + TALITOS2_DEU;
3374 priv->reg_aesu = priv->reg + TALITOS2_AESU;
3375 priv->reg_mdeu = priv->reg + TALITOS2_MDEU;
3376 priv->reg_afeu = priv->reg + TALITOS2_AFEU;
3377 priv->reg_rngu = priv->reg + TALITOS2_RNGU;
3378 priv->reg_pkeu = priv->reg + TALITOS2_PKEU;
3379 priv->reg_keu = priv->reg + TALITOS2_KEU;
3380 priv->reg_crcu = priv->reg + TALITOS2_CRCU;
3381 stride = TALITOS2_CH_STRIDE;
3382 }
3383
LEROY Christophedd3c0982015-04-17 16:32:13 +02003384 err = talitos_probe_irq(ofdev);
3385 if (err)
3386 goto err_out;
3387
3388 if (of_device_is_compatible(np, "fsl,sec1.0")) {
LEROY Christophe9c02e282017-10-06 15:04:55 +02003389 if (priv->num_channels == 1)
3390 tasklet_init(&priv->done_task[0], talitos1_done_ch0,
LEROY Christophedd3c0982015-04-17 16:32:13 +02003391 (unsigned long)dev);
LEROY Christophe9c02e282017-10-06 15:04:55 +02003392 else
3393 tasklet_init(&priv->done_task[0], talitos1_done_4ch,
3394 (unsigned long)dev);
3395 } else {
3396 if (priv->irq[1]) {
LEROY Christophedd3c0982015-04-17 16:32:13 +02003397 tasklet_init(&priv->done_task[0], talitos2_done_ch0_2,
3398 (unsigned long)dev);
3399 tasklet_init(&priv->done_task[1], talitos2_done_ch1_3,
3400 (unsigned long)dev);
LEROY Christophe9c02e282017-10-06 15:04:55 +02003401 } else if (priv->num_channels == 1) {
3402 tasklet_init(&priv->done_task[0], talitos2_done_ch0,
3403 (unsigned long)dev);
3404 } else {
3405 tasklet_init(&priv->done_task[0], talitos2_done_4ch,
3406 (unsigned long)dev);
LEROY Christophedd3c0982015-04-17 16:32:13 +02003407 }
3408 }
3409
LEROY Christophe24b92ff2017-10-06 15:04:49 +02003410 priv->chan = devm_kzalloc(dev, sizeof(struct talitos_channel) *
3411 priv->num_channels, GFP_KERNEL);
Kim Phillips4b9926282009-08-13 11:50:38 +10003412 if (!priv->chan) {
3413 dev_err(dev, "failed to allocate channel management space\n");
Kim Phillips9c4a7962008-06-23 19:50:15 +08003414 err = -ENOMEM;
3415 goto err_out;
3416 }
3417
Martin Hicksf641ddd2015-03-03 08:21:33 -05003418 priv->fifo_len = roundup_pow_of_two(priv->chfifo_len);
3419
Kim Phillipsc3e337f2011-11-21 16:13:27 +08003420 for (i = 0; i < priv->num_channels; i++) {
LEROY Christophe5fa7fa12015-04-17 16:32:11 +02003421 priv->chan[i].reg = priv->reg + stride * (i + 1);
Kim Phillips2cdba3c2011-12-12 14:59:11 -06003422 if (!priv->irq[1] || !(i & 1))
Kim Phillipsc3e337f2011-11-21 16:13:27 +08003423 priv->chan[i].reg += TALITOS_CH_BASE_OFFSET;
Kim Phillipsad42d5f2011-11-21 16:13:27 +08003424
Kim Phillips4b9926282009-08-13 11:50:38 +10003425 spin_lock_init(&priv->chan[i].head_lock);
3426 spin_lock_init(&priv->chan[i].tail_lock);
Kim Phillips9c4a7962008-06-23 19:50:15 +08003427
LEROY Christophe24b92ff2017-10-06 15:04:49 +02003428 priv->chan[i].fifo = devm_kzalloc(dev,
3429 sizeof(struct talitos_request) *
3430 priv->fifo_len, GFP_KERNEL);
Kim Phillips4b9926282009-08-13 11:50:38 +10003431 if (!priv->chan[i].fifo) {
Kim Phillips9c4a7962008-06-23 19:50:15 +08003432 dev_err(dev, "failed to allocate request fifo %d\n", i);
3433 err = -ENOMEM;
3434 goto err_out;
3435 }
Kim Phillips9c4a7962008-06-23 19:50:15 +08003436
Kim Phillips4b9926282009-08-13 11:50:38 +10003437 atomic_set(&priv->chan[i].submit_count,
3438 -(priv->chfifo_len - 1));
Martin Hicksf641ddd2015-03-03 08:21:33 -05003439 }
Kim Phillips9c4a7962008-06-23 19:50:15 +08003440
Kim Phillips81eb0242009-08-13 11:51:51 +10003441 dma_set_mask(dev, DMA_BIT_MASK(36));
3442
Kim Phillips9c4a7962008-06-23 19:50:15 +08003443 /* reset and initialize the h/w */
3444 err = init_device(dev);
3445 if (err) {
3446 dev_err(dev, "failed to initialize device\n");
3447 goto err_out;
3448 }
3449
3450 /* register the RNG, if available */
3451 if (hw_supports(dev, DESC_HDR_SEL0_RNG)) {
3452 err = talitos_register_rng(dev);
3453 if (err) {
3454 dev_err(dev, "failed to register hwrng: %d\n", err);
3455 goto err_out;
3456 } else
3457 dev_info(dev, "hwrng\n");
3458 }
3459
3460 /* register crypto algorithms the device supports */
Kim Phillips9c4a7962008-06-23 19:50:15 +08003461 for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
3462 if (hw_supports(dev, driver_algs[i].desc_hdr_template)) {
3463 struct talitos_crypto_alg *t_alg;
Herbert Xuaeb4c132015-07-30 17:53:22 +08003464 struct crypto_alg *alg = NULL;
Kim Phillips9c4a7962008-06-23 19:50:15 +08003465
3466 t_alg = talitos_alg_alloc(dev, &driver_algs[i]);
3467 if (IS_ERR(t_alg)) {
3468 err = PTR_ERR(t_alg);
Kim Phillips0b2730d2011-12-12 14:59:10 -06003469 if (err == -ENOTSUPP)
Lee Nipper79b3a412011-11-21 16:13:25 +08003470 continue;
Kim Phillips9c4a7962008-06-23 19:50:15 +08003471 goto err_out;
3472 }
3473
Lee Nipperacbf7c622010-05-19 19:19:33 +10003474 switch (t_alg->algt.type) {
3475 case CRYPTO_ALG_TYPE_ABLKCIPHER:
Lee Nipperacbf7c622010-05-19 19:19:33 +10003476 err = crypto_register_alg(
3477 &t_alg->algt.alg.crypto);
Herbert Xuaeb4c132015-07-30 17:53:22 +08003478 alg = &t_alg->algt.alg.crypto;
Lee Nipperacbf7c622010-05-19 19:19:33 +10003479 break;
Herbert Xuaeb4c132015-07-30 17:53:22 +08003480
3481 case CRYPTO_ALG_TYPE_AEAD:
3482 err = crypto_register_aead(
3483 &t_alg->algt.alg.aead);
3484 alg = &t_alg->algt.alg.aead.base;
3485 break;
3486
Lee Nipperacbf7c622010-05-19 19:19:33 +10003487 case CRYPTO_ALG_TYPE_AHASH:
3488 err = crypto_register_ahash(
3489 &t_alg->algt.alg.hash);
Herbert Xuaeb4c132015-07-30 17:53:22 +08003490 alg = &t_alg->algt.alg.hash.halg.base;
Lee Nipperacbf7c622010-05-19 19:19:33 +10003491 break;
3492 }
Kim Phillips9c4a7962008-06-23 19:50:15 +08003493 if (err) {
3494 dev_err(dev, "%s alg registration failed\n",
Herbert Xuaeb4c132015-07-30 17:53:22 +08003495 alg->cra_driver_name);
LEROY Christophe24b92ff2017-10-06 15:04:49 +02003496 devm_kfree(dev, t_alg);
Horia Geanta991155b2013-03-20 16:31:38 +02003497 } else
Kim Phillips9c4a7962008-06-23 19:50:15 +08003498 list_add_tail(&t_alg->entry, &priv->alg_list);
Kim Phillips9c4a7962008-06-23 19:50:15 +08003499 }
3500 }
Kim Phillips5b859b6e2011-11-21 16:13:26 +08003501 if (!list_empty(&priv->alg_list))
3502 dev_info(dev, "%s algorithms registered in /proc/crypto\n",
3503 (char *)of_get_property(np, "compatible", NULL));
Kim Phillips9c4a7962008-06-23 19:50:15 +08003504
3505 return 0;
3506
3507err_out:
3508 talitos_remove(ofdev);
Kim Phillips9c4a7962008-06-23 19:50:15 +08003509
3510 return err;
3511}
3512
Márton Németh6c3f9752010-01-17 21:54:01 +11003513static const struct of_device_id talitos_match[] = {
LEROY Christophe0635b7db2015-04-17 16:32:20 +02003514#ifdef CONFIG_CRYPTO_DEV_TALITOS1
3515 {
3516 .compatible = "fsl,sec1.0",
3517 },
3518#endif
3519#ifdef CONFIG_CRYPTO_DEV_TALITOS2
Kim Phillips9c4a7962008-06-23 19:50:15 +08003520 {
3521 .compatible = "fsl,sec2.0",
3522 },
LEROY Christophe0635b7db2015-04-17 16:32:20 +02003523#endif
Kim Phillips9c4a7962008-06-23 19:50:15 +08003524 {},
3525};
3526MODULE_DEVICE_TABLE(of, talitos_match);
3527
Grant Likely1c48a5c2011-02-17 02:43:24 -07003528static struct platform_driver talitos_driver = {
Grant Likely40182942010-04-13 16:13:02 -07003529 .driver = {
3530 .name = "talitos",
Grant Likely40182942010-04-13 16:13:02 -07003531 .of_match_table = talitos_match,
3532 },
Kim Phillips9c4a7962008-06-23 19:50:15 +08003533 .probe = talitos_probe,
Al Viro596f1032008-11-22 17:34:24 +00003534 .remove = talitos_remove,
Kim Phillips9c4a7962008-06-23 19:50:15 +08003535};
3536
Axel Lin741e8c22011-11-26 21:26:19 +08003537module_platform_driver(talitos_driver);
Kim Phillips9c4a7962008-06-23 19:50:15 +08003538
3539MODULE_LICENSE("GPL");
3540MODULE_AUTHOR("Kim Phillips <kim.phillips@freescale.com>");
3541MODULE_DESCRIPTION("Freescale integrated security engine (SEC) driver");