blob: 56c416f39e3a28b6aee9e4c6751f3533227e54d1 [file] [log] [blame]
Kevin Barnett6c223762016-06-27 16:41:00 -05001/*
2 * driver for Microsemi PQI-based storage controllers
Kevin Barnettb805dbf2017-05-03 18:54:06 -05003 * Copyright (c) 2016-2017 Microsemi Corporation
Kevin Barnett6c223762016-06-27 16:41:00 -05004 * Copyright (c) 2016 PMC-Sierra, Inc.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; version 2 of the License.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
13 * NON INFRINGEMENT. See the GNU General Public License for more details.
14 *
15 * Questions/Comments/Bugfixes to esc.storagedev@microsemi.com
16 *
17 */
18
19#include <linux/module.h>
20#include <linux/kernel.h>
21#include <linux/pci.h>
22#include <linux/delay.h>
23#include <linux/interrupt.h>
24#include <linux/sched.h>
25#include <linux/rtc.h>
26#include <linux/bcd.h>
Kevin Barnett3c509762017-05-03 18:54:37 -050027#include <linux/reboot.h>
Kevin Barnett6c223762016-06-27 16:41:00 -050028#include <linux/cciss_ioctl.h>
Christoph Hellwig52198222016-11-01 08:12:49 -060029#include <linux/blk-mq-pci.h>
Kevin Barnett6c223762016-06-27 16:41:00 -050030#include <scsi/scsi_host.h>
31#include <scsi/scsi_cmnd.h>
32#include <scsi/scsi_device.h>
33#include <scsi/scsi_eh.h>
34#include <scsi/scsi_transport_sas.h>
35#include <asm/unaligned.h>
36#include "smartpqi.h"
37#include "smartpqi_sis.h"
38
39#if !defined(BUILD_TIMESTAMP)
40#define BUILD_TIMESTAMP
41#endif
42
Kevin Barnett699bed72016-08-31 14:55:36 -050043#define DRIVER_VERSION "0.9.13-370"
Kevin Barnett6c223762016-06-27 16:41:00 -050044#define DRIVER_MAJOR 0
45#define DRIVER_MINOR 9
Kevin Barnett699bed72016-08-31 14:55:36 -050046#define DRIVER_RELEASE 13
47#define DRIVER_REVISION 370
Kevin Barnett6c223762016-06-27 16:41:00 -050048
49#define DRIVER_NAME "Microsemi PQI Driver (v" DRIVER_VERSION ")"
50#define DRIVER_NAME_SHORT "smartpqi"
51
Kevin Barnette1d213b2017-05-03 18:53:18 -050052#define PQI_EXTRA_SGL_MEMORY (12 * sizeof(struct pqi_sg_descriptor))
53
Kevin Barnett6c223762016-06-27 16:41:00 -050054MODULE_AUTHOR("Microsemi");
55MODULE_DESCRIPTION("Driver for Microsemi Smart Family Controller version "
56 DRIVER_VERSION);
57MODULE_SUPPORTED_DEVICE("Microsemi Smart Family Controllers");
58MODULE_VERSION(DRIVER_VERSION);
59MODULE_LICENSE("GPL");
60
Kevin Barnett6c223762016-06-27 16:41:00 -050061static void pqi_take_ctrl_offline(struct pqi_ctrl_info *ctrl_info);
Kevin Barnett5f310422017-05-03 18:54:55 -050062static void pqi_ctrl_offline_worker(struct work_struct *work);
Kevin Barnett376fb882017-05-03 18:54:43 -050063static void pqi_retry_raid_bypass_requests(struct pqi_ctrl_info *ctrl_info);
Kevin Barnett6c223762016-06-27 16:41:00 -050064static int pqi_scan_scsi_devices(struct pqi_ctrl_info *ctrl_info);
65static void pqi_scan_start(struct Scsi_Host *shost);
66static void pqi_start_io(struct pqi_ctrl_info *ctrl_info,
67 struct pqi_queue_group *queue_group, enum pqi_io_path path,
68 struct pqi_io_request *io_request);
69static int pqi_submit_raid_request_synchronous(struct pqi_ctrl_info *ctrl_info,
70 struct pqi_iu_header *request, unsigned int flags,
71 struct pqi_raid_error_info *error_info, unsigned long timeout_msecs);
72static int pqi_aio_submit_io(struct pqi_ctrl_info *ctrl_info,
73 struct scsi_cmnd *scmd, u32 aio_handle, u8 *cdb,
74 unsigned int cdb_length, struct pqi_queue_group *queue_group,
Kevin Barnett376fb882017-05-03 18:54:43 -050075 struct pqi_encryption_info *encryption_info, bool raid_bypass);
Kevin Barnett6c223762016-06-27 16:41:00 -050076
77/* for flags argument to pqi_submit_raid_request_synchronous() */
78#define PQI_SYNC_FLAGS_INTERRUPTABLE 0x1
79
80static struct scsi_transport_template *pqi_sas_transport_template;
81
82static atomic_t pqi_controller_count = ATOMIC_INIT(0);
83
Kevin Barnett3c509762017-05-03 18:54:37 -050084enum pqi_lockup_action {
85 NONE,
86 REBOOT,
87 PANIC
88};
89
90static enum pqi_lockup_action pqi_lockup_action = NONE;
91
92static struct {
93 enum pqi_lockup_action action;
94 char *name;
95} pqi_lockup_actions[] = {
96 {
97 .action = NONE,
98 .name = "none",
99 },
100 {
101 .action = REBOOT,
102 .name = "reboot",
103 },
104 {
105 .action = PANIC,
106 .name = "panic",
107 },
108};
109
Kevin Barnett6a50d6a2017-05-03 18:52:52 -0500110static unsigned int pqi_supported_event_types[] = {
111 PQI_EVENT_TYPE_HOTPLUG,
112 PQI_EVENT_TYPE_HARDWARE,
113 PQI_EVENT_TYPE_PHYSICAL_DEVICE,
114 PQI_EVENT_TYPE_LOGICAL_DEVICE,
115 PQI_EVENT_TYPE_AIO_STATE_CHANGE,
116 PQI_EVENT_TYPE_AIO_CONFIG_CHANGE,
117};
118
Kevin Barnett6c223762016-06-27 16:41:00 -0500119static int pqi_disable_device_id_wildcards;
120module_param_named(disable_device_id_wildcards,
Kevin Barnettcbe0c7b2017-05-03 18:53:48 -0500121 pqi_disable_device_id_wildcards, int, 0644);
Kevin Barnett6c223762016-06-27 16:41:00 -0500122MODULE_PARM_DESC(disable_device_id_wildcards,
123 "Disable device ID wildcards.");
124
Kevin Barnett3c509762017-05-03 18:54:37 -0500125static char *pqi_lockup_action_param;
126module_param_named(lockup_action,
127 pqi_lockup_action_param, charp, 0644);
128MODULE_PARM_DESC(lockup_action, "Action to take when controller locked up.\n"
129 "\t\tSupported: none, reboot, panic\n"
130 "\t\tDefault: none");
131
Kevin Barnett6c223762016-06-27 16:41:00 -0500132static char *raid_levels[] = {
133 "RAID-0",
134 "RAID-4",
135 "RAID-1(1+0)",
136 "RAID-5",
137 "RAID-5+1",
138 "RAID-ADG",
139 "RAID-1(ADM)",
140};
141
142static char *pqi_raid_level_to_string(u8 raid_level)
143{
144 if (raid_level < ARRAY_SIZE(raid_levels))
145 return raid_levels[raid_level];
146
Kevin Barnetta9f93392017-05-03 18:55:31 -0500147 return "RAID UNKNOWN";
Kevin Barnett6c223762016-06-27 16:41:00 -0500148}
149
150#define SA_RAID_0 0
151#define SA_RAID_4 1
152#define SA_RAID_1 2 /* also used for RAID 10 */
153#define SA_RAID_5 3 /* also used for RAID 50 */
154#define SA_RAID_51 4
155#define SA_RAID_6 5 /* also used for RAID 60 */
156#define SA_RAID_ADM 6 /* also used for RAID 1+0 ADM */
157#define SA_RAID_MAX SA_RAID_ADM
158#define SA_RAID_UNKNOWN 0xff
159
160static inline void pqi_scsi_done(struct scsi_cmnd *scmd)
161{
Kevin Barnett7561a7e2017-05-03 18:52:58 -0500162 pqi_prep_for_scsi_done(scmd);
Kevin Barnett6c223762016-06-27 16:41:00 -0500163 scmd->scsi_done(scmd);
164}
165
166static inline bool pqi_scsi3addr_equal(u8 *scsi3addr1, u8 *scsi3addr2)
167{
168 return memcmp(scsi3addr1, scsi3addr2, 8) == 0;
169}
170
171static inline struct pqi_ctrl_info *shost_to_hba(struct Scsi_Host *shost)
172{
173 void *hostdata = shost_priv(shost);
174
175 return *((struct pqi_ctrl_info **)hostdata);
176}
177
178static inline bool pqi_is_logical_device(struct pqi_scsi_dev *device)
179{
180 return !device->is_physical_device;
181}
182
Kevin Barnettbd10cf02017-05-03 18:54:12 -0500183static inline bool pqi_is_external_raid_addr(u8 *scsi3addr)
184{
185 return scsi3addr[2] != 0;
186}
187
Kevin Barnett6c223762016-06-27 16:41:00 -0500188static inline bool pqi_ctrl_offline(struct pqi_ctrl_info *ctrl_info)
189{
190 return !ctrl_info->controller_online;
191}
192
193static inline void pqi_check_ctrl_health(struct pqi_ctrl_info *ctrl_info)
194{
195 if (ctrl_info->controller_online)
196 if (!sis_is_firmware_running(ctrl_info))
197 pqi_take_ctrl_offline(ctrl_info);
198}
199
200static inline bool pqi_is_hba_lunid(u8 *scsi3addr)
201{
202 return pqi_scsi3addr_equal(scsi3addr, RAID_CTLR_LUNID);
203}
204
Kevin Barnettff6abb72016-08-31 14:54:41 -0500205static inline enum pqi_ctrl_mode pqi_get_ctrl_mode(
206 struct pqi_ctrl_info *ctrl_info)
207{
208 return sis_read_driver_scratch(ctrl_info);
209}
210
211static inline void pqi_save_ctrl_mode(struct pqi_ctrl_info *ctrl_info,
212 enum pqi_ctrl_mode mode)
213{
214 sis_write_driver_scratch(ctrl_info, mode);
215}
216
Kevin Barnett7561a7e2017-05-03 18:52:58 -0500217static inline void pqi_ctrl_block_requests(struct pqi_ctrl_info *ctrl_info)
218{
219 ctrl_info->block_requests = true;
220 scsi_block_requests(ctrl_info->scsi_host);
221}
222
223static inline void pqi_ctrl_unblock_requests(struct pqi_ctrl_info *ctrl_info)
224{
225 ctrl_info->block_requests = false;
226 wake_up_all(&ctrl_info->block_requests_wait);
Kevin Barnett376fb882017-05-03 18:54:43 -0500227 pqi_retry_raid_bypass_requests(ctrl_info);
Kevin Barnett7561a7e2017-05-03 18:52:58 -0500228 scsi_unblock_requests(ctrl_info->scsi_host);
229}
230
231static inline bool pqi_ctrl_blocked(struct pqi_ctrl_info *ctrl_info)
232{
233 return ctrl_info->block_requests;
234}
235
236static unsigned long pqi_wait_if_ctrl_blocked(struct pqi_ctrl_info *ctrl_info,
237 unsigned long timeout_msecs)
238{
239 unsigned long remaining_msecs;
240
241 if (!pqi_ctrl_blocked(ctrl_info))
242 return timeout_msecs;
243
244 atomic_inc(&ctrl_info->num_blocked_threads);
245
246 if (timeout_msecs == NO_TIMEOUT) {
247 wait_event(ctrl_info->block_requests_wait,
248 !pqi_ctrl_blocked(ctrl_info));
249 remaining_msecs = timeout_msecs;
250 } else {
251 unsigned long remaining_jiffies;
252
253 remaining_jiffies =
254 wait_event_timeout(ctrl_info->block_requests_wait,
255 !pqi_ctrl_blocked(ctrl_info),
256 msecs_to_jiffies(timeout_msecs));
257 remaining_msecs = jiffies_to_msecs(remaining_jiffies);
258 }
259
260 atomic_dec(&ctrl_info->num_blocked_threads);
261
262 return remaining_msecs;
263}
264
265static inline void pqi_ctrl_busy(struct pqi_ctrl_info *ctrl_info)
266{
267 atomic_inc(&ctrl_info->num_busy_threads);
268}
269
270static inline void pqi_ctrl_unbusy(struct pqi_ctrl_info *ctrl_info)
271{
272 atomic_dec(&ctrl_info->num_busy_threads);
273}
274
275static inline void pqi_ctrl_wait_until_quiesced(struct pqi_ctrl_info *ctrl_info)
276{
277 while (atomic_read(&ctrl_info->num_busy_threads) >
278 atomic_read(&ctrl_info->num_blocked_threads))
279 usleep_range(1000, 2000);
280}
281
Kevin Barnett03b288cf2017-05-03 18:54:49 -0500282static inline bool pqi_device_offline(struct pqi_scsi_dev *device)
283{
284 return device->device_offline;
285}
286
Kevin Barnett7561a7e2017-05-03 18:52:58 -0500287static inline void pqi_device_reset_start(struct pqi_scsi_dev *device)
288{
289 device->in_reset = true;
290}
291
292static inline void pqi_device_reset_done(struct pqi_scsi_dev *device)
293{
294 device->in_reset = false;
295}
296
297static inline bool pqi_device_in_reset(struct pqi_scsi_dev *device)
298{
299 return device->in_reset;
300}
Kevin Barnett6c223762016-06-27 16:41:00 -0500301
Kevin Barnett5f310422017-05-03 18:54:55 -0500302static inline void pqi_schedule_rescan_worker_with_delay(
303 struct pqi_ctrl_info *ctrl_info, unsigned long delay)
304{
305 if (pqi_ctrl_offline(ctrl_info))
306 return;
307
308 schedule_delayed_work(&ctrl_info->rescan_work, delay);
309}
310
Kevin Barnett6c223762016-06-27 16:41:00 -0500311static inline void pqi_schedule_rescan_worker(struct pqi_ctrl_info *ctrl_info)
312{
Kevin Barnett5f310422017-05-03 18:54:55 -0500313 pqi_schedule_rescan_worker_with_delay(ctrl_info, 0);
314}
315
316#define PQI_RESCAN_WORK_DELAY (10 * HZ)
317
318static inline void pqi_schedule_rescan_worker_delayed(
319 struct pqi_ctrl_info *ctrl_info)
320{
321 pqi_schedule_rescan_worker_with_delay(ctrl_info, PQI_RESCAN_WORK_DELAY);
Kevin Barnett6c223762016-06-27 16:41:00 -0500322}
323
Kevin Barnett061ef062017-05-03 18:53:05 -0500324static inline void pqi_cancel_rescan_worker(struct pqi_ctrl_info *ctrl_info)
325{
326 cancel_delayed_work_sync(&ctrl_info->rescan_work);
327}
328
Kevin Barnett98f87662017-05-03 18:53:11 -0500329static inline u32 pqi_read_heartbeat_counter(struct pqi_ctrl_info *ctrl_info)
330{
331 if (!ctrl_info->heartbeat_counter)
332 return 0;
333
334 return readl(ctrl_info->heartbeat_counter);
335}
336
Kevin Barnett6c223762016-06-27 16:41:00 -0500337static int pqi_map_single(struct pci_dev *pci_dev,
338 struct pqi_sg_descriptor *sg_descriptor, void *buffer,
339 size_t buffer_length, int data_direction)
340{
341 dma_addr_t bus_address;
342
343 if (!buffer || buffer_length == 0 || data_direction == PCI_DMA_NONE)
344 return 0;
345
346 bus_address = pci_map_single(pci_dev, buffer, buffer_length,
347 data_direction);
348 if (pci_dma_mapping_error(pci_dev, bus_address))
349 return -ENOMEM;
350
351 put_unaligned_le64((u64)bus_address, &sg_descriptor->address);
352 put_unaligned_le32(buffer_length, &sg_descriptor->length);
353 put_unaligned_le32(CISS_SG_LAST, &sg_descriptor->flags);
354
355 return 0;
356}
357
358static void pqi_pci_unmap(struct pci_dev *pci_dev,
359 struct pqi_sg_descriptor *descriptors, int num_descriptors,
360 int data_direction)
361{
362 int i;
363
364 if (data_direction == PCI_DMA_NONE)
365 return;
366
367 for (i = 0; i < num_descriptors; i++)
368 pci_unmap_single(pci_dev,
369 (dma_addr_t)get_unaligned_le64(&descriptors[i].address),
370 get_unaligned_le32(&descriptors[i].length),
371 data_direction);
372}
373
374static int pqi_build_raid_path_request(struct pqi_ctrl_info *ctrl_info,
375 struct pqi_raid_path_request *request, u8 cmd,
376 u8 *scsi3addr, void *buffer, size_t buffer_length,
377 u16 vpd_page, int *pci_direction)
378{
379 u8 *cdb;
380 int pci_dir;
381
382 memset(request, 0, sizeof(*request));
383
384 request->header.iu_type = PQI_REQUEST_IU_RAID_PATH_IO;
385 put_unaligned_le16(offsetof(struct pqi_raid_path_request,
386 sg_descriptors[1]) - PQI_REQUEST_HEADER_LENGTH,
387 &request->header.iu_length);
388 put_unaligned_le32(buffer_length, &request->buffer_length);
389 memcpy(request->lun_number, scsi3addr, sizeof(request->lun_number));
390 request->task_attribute = SOP_TASK_ATTRIBUTE_SIMPLE;
391 request->additional_cdb_bytes_usage = SOP_ADDITIONAL_CDB_BYTES_0;
392
393 cdb = request->cdb;
394
395 switch (cmd) {
396 case INQUIRY:
397 request->data_direction = SOP_READ_FLAG;
398 cdb[0] = INQUIRY;
399 if (vpd_page & VPD_PAGE) {
400 cdb[1] = 0x1;
401 cdb[2] = (u8)vpd_page;
402 }
403 cdb[4] = (u8)buffer_length;
404 break;
405 case CISS_REPORT_LOG:
406 case CISS_REPORT_PHYS:
407 request->data_direction = SOP_READ_FLAG;
408 cdb[0] = cmd;
409 if (cmd == CISS_REPORT_PHYS)
410 cdb[1] = CISS_REPORT_PHYS_EXTENDED;
411 else
412 cdb[1] = CISS_REPORT_LOG_EXTENDED;
413 put_unaligned_be32(buffer_length, &cdb[6]);
414 break;
415 case CISS_GET_RAID_MAP:
416 request->data_direction = SOP_READ_FLAG;
417 cdb[0] = CISS_READ;
418 cdb[1] = CISS_GET_RAID_MAP;
419 put_unaligned_be32(buffer_length, &cdb[6]);
420 break;
421 case SA_CACHE_FLUSH:
422 request->data_direction = SOP_WRITE_FLAG;
423 cdb[0] = BMIC_WRITE;
424 cdb[6] = BMIC_CACHE_FLUSH;
425 put_unaligned_be16(buffer_length, &cdb[7]);
426 break;
427 case BMIC_IDENTIFY_CONTROLLER:
428 case BMIC_IDENTIFY_PHYSICAL_DEVICE:
429 request->data_direction = SOP_READ_FLAG;
430 cdb[0] = BMIC_READ;
431 cdb[6] = cmd;
432 put_unaligned_be16(buffer_length, &cdb[7]);
433 break;
434 case BMIC_WRITE_HOST_WELLNESS:
435 request->data_direction = SOP_WRITE_FLAG;
436 cdb[0] = BMIC_WRITE;
437 cdb[6] = cmd;
438 put_unaligned_be16(buffer_length, &cdb[7]);
439 break;
440 default:
441 dev_err(&ctrl_info->pci_dev->dev, "unknown command 0x%c\n",
442 cmd);
Kevin Barnett6c223762016-06-27 16:41:00 -0500443 break;
444 }
445
446 switch (request->data_direction) {
447 case SOP_READ_FLAG:
448 pci_dir = PCI_DMA_FROMDEVICE;
449 break;
450 case SOP_WRITE_FLAG:
451 pci_dir = PCI_DMA_TODEVICE;
452 break;
453 case SOP_NO_DIRECTION_FLAG:
454 pci_dir = PCI_DMA_NONE;
455 break;
456 default:
457 pci_dir = PCI_DMA_BIDIRECTIONAL;
458 break;
459 }
460
461 *pci_direction = pci_dir;
462
463 return pqi_map_single(ctrl_info->pci_dev, &request->sg_descriptors[0],
464 buffer, buffer_length, pci_dir);
465}
466
Kevin Barnett376fb882017-05-03 18:54:43 -0500467static inline void pqi_reinit_io_request(struct pqi_io_request *io_request)
468{
469 io_request->scmd = NULL;
470 io_request->status = 0;
471 io_request->error_info = NULL;
472 io_request->raid_bypass = false;
473}
474
Kevin Barnett6c223762016-06-27 16:41:00 -0500475static struct pqi_io_request *pqi_alloc_io_request(
476 struct pqi_ctrl_info *ctrl_info)
477{
478 struct pqi_io_request *io_request;
479 u16 i = ctrl_info->next_io_request_slot; /* benignly racy */
480
481 while (1) {
482 io_request = &ctrl_info->io_request_pool[i];
483 if (atomic_inc_return(&io_request->refcount) == 1)
484 break;
485 atomic_dec(&io_request->refcount);
486 i = (i + 1) % ctrl_info->max_io_slots;
487 }
488
489 /* benignly racy */
490 ctrl_info->next_io_request_slot = (i + 1) % ctrl_info->max_io_slots;
491
Kevin Barnett376fb882017-05-03 18:54:43 -0500492 pqi_reinit_io_request(io_request);
Kevin Barnett6c223762016-06-27 16:41:00 -0500493
494 return io_request;
495}
496
497static void pqi_free_io_request(struct pqi_io_request *io_request)
498{
499 atomic_dec(&io_request->refcount);
500}
501
502static int pqi_identify_controller(struct pqi_ctrl_info *ctrl_info,
503 struct bmic_identify_controller *buffer)
504{
505 int rc;
506 int pci_direction;
507 struct pqi_raid_path_request request;
508
509 rc = pqi_build_raid_path_request(ctrl_info, &request,
510 BMIC_IDENTIFY_CONTROLLER, RAID_CTLR_LUNID, buffer,
511 sizeof(*buffer), 0, &pci_direction);
512 if (rc)
513 return rc;
514
515 rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0,
516 NULL, NO_TIMEOUT);
517
518 pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1,
519 pci_direction);
520
521 return rc;
522}
523
524static int pqi_scsi_inquiry(struct pqi_ctrl_info *ctrl_info,
525 u8 *scsi3addr, u16 vpd_page, void *buffer, size_t buffer_length)
526{
527 int rc;
528 int pci_direction;
529 struct pqi_raid_path_request request;
530
531 rc = pqi_build_raid_path_request(ctrl_info, &request,
532 INQUIRY, scsi3addr, buffer, buffer_length, vpd_page,
533 &pci_direction);
534 if (rc)
535 return rc;
536
537 rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0,
538 NULL, NO_TIMEOUT);
539
540 pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1,
541 pci_direction);
542
543 return rc;
544}
545
546static int pqi_identify_physical_device(struct pqi_ctrl_info *ctrl_info,
547 struct pqi_scsi_dev *device,
548 struct bmic_identify_physical_device *buffer,
549 size_t buffer_length)
550{
551 int rc;
552 int pci_direction;
553 u16 bmic_device_index;
554 struct pqi_raid_path_request request;
555
556 rc = pqi_build_raid_path_request(ctrl_info, &request,
557 BMIC_IDENTIFY_PHYSICAL_DEVICE, RAID_CTLR_LUNID, buffer,
558 buffer_length, 0, &pci_direction);
559 if (rc)
560 return rc;
561
562 bmic_device_index = CISS_GET_DRIVE_NUMBER(device->scsi3addr);
563 request.cdb[2] = (u8)bmic_device_index;
564 request.cdb[9] = (u8)(bmic_device_index >> 8);
565
566 rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header,
567 0, NULL, NO_TIMEOUT);
568
569 pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1,
570 pci_direction);
571
572 return rc;
573}
574
575#define SA_CACHE_FLUSH_BUFFER_LENGTH 4
Kevin Barnett6c223762016-06-27 16:41:00 -0500576
577static int pqi_flush_cache(struct pqi_ctrl_info *ctrl_info)
578{
579 int rc;
580 struct pqi_raid_path_request request;
581 int pci_direction;
582 u8 *buffer;
583
584 /*
585 * Don't bother trying to flush the cache if the controller is
586 * locked up.
587 */
588 if (pqi_ctrl_offline(ctrl_info))
589 return -ENXIO;
590
591 buffer = kzalloc(SA_CACHE_FLUSH_BUFFER_LENGTH, GFP_KERNEL);
592 if (!buffer)
593 return -ENOMEM;
594
595 rc = pqi_build_raid_path_request(ctrl_info, &request,
596 SA_CACHE_FLUSH, RAID_CTLR_LUNID, buffer,
597 SA_CACHE_FLUSH_BUFFER_LENGTH, 0, &pci_direction);
598 if (rc)
599 goto out;
600
601 rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header,
Kevin Barnettd48f8fa2016-08-31 14:55:17 -0500602 0, NULL, NO_TIMEOUT);
Kevin Barnett6c223762016-06-27 16:41:00 -0500603
604 pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1,
605 pci_direction);
606
607out:
608 kfree(buffer);
609
610 return rc;
611}
612
613static int pqi_write_host_wellness(struct pqi_ctrl_info *ctrl_info,
614 void *buffer, size_t buffer_length)
615{
616 int rc;
617 struct pqi_raid_path_request request;
618 int pci_direction;
619
620 rc = pqi_build_raid_path_request(ctrl_info, &request,
621 BMIC_WRITE_HOST_WELLNESS, RAID_CTLR_LUNID, buffer,
622 buffer_length, 0, &pci_direction);
623 if (rc)
624 return rc;
625
626 rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header,
627 0, NULL, NO_TIMEOUT);
628
629 pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1,
630 pci_direction);
631
632 return rc;
633}
634
635#pragma pack(1)
636
637struct bmic_host_wellness_driver_version {
638 u8 start_tag[4];
639 u8 driver_version_tag[2];
640 __le16 driver_version_length;
641 char driver_version[32];
642 u8 end_tag[2];
643};
644
645#pragma pack()
646
647static int pqi_write_driver_version_to_host_wellness(
648 struct pqi_ctrl_info *ctrl_info)
649{
650 int rc;
651 struct bmic_host_wellness_driver_version *buffer;
652 size_t buffer_length;
653
654 buffer_length = sizeof(*buffer);
655
656 buffer = kmalloc(buffer_length, GFP_KERNEL);
657 if (!buffer)
658 return -ENOMEM;
659
660 buffer->start_tag[0] = '<';
661 buffer->start_tag[1] = 'H';
662 buffer->start_tag[2] = 'W';
663 buffer->start_tag[3] = '>';
664 buffer->driver_version_tag[0] = 'D';
665 buffer->driver_version_tag[1] = 'V';
666 put_unaligned_le16(sizeof(buffer->driver_version),
667 &buffer->driver_version_length);
Kevin Barnett061ef062017-05-03 18:53:05 -0500668 strncpy(buffer->driver_version, "Linux " DRIVER_VERSION,
Kevin Barnett6c223762016-06-27 16:41:00 -0500669 sizeof(buffer->driver_version) - 1);
670 buffer->driver_version[sizeof(buffer->driver_version) - 1] = '\0';
671 buffer->end_tag[0] = 'Z';
672 buffer->end_tag[1] = 'Z';
673
674 rc = pqi_write_host_wellness(ctrl_info, buffer, buffer_length);
675
676 kfree(buffer);
677
678 return rc;
679}
680
681#pragma pack(1)
682
683struct bmic_host_wellness_time {
684 u8 start_tag[4];
685 u8 time_tag[2];
686 __le16 time_length;
687 u8 time[8];
688 u8 dont_write_tag[2];
689 u8 end_tag[2];
690};
691
692#pragma pack()
693
694static int pqi_write_current_time_to_host_wellness(
695 struct pqi_ctrl_info *ctrl_info)
696{
697 int rc;
698 struct bmic_host_wellness_time *buffer;
699 size_t buffer_length;
700 time64_t local_time;
701 unsigned int year;
Arnd Bergmanned108582017-02-17 16:03:52 +0100702 struct tm tm;
Kevin Barnett6c223762016-06-27 16:41:00 -0500703
704 buffer_length = sizeof(*buffer);
705
706 buffer = kmalloc(buffer_length, GFP_KERNEL);
707 if (!buffer)
708 return -ENOMEM;
709
710 buffer->start_tag[0] = '<';
711 buffer->start_tag[1] = 'H';
712 buffer->start_tag[2] = 'W';
713 buffer->start_tag[3] = '>';
714 buffer->time_tag[0] = 'T';
715 buffer->time_tag[1] = 'D';
716 put_unaligned_le16(sizeof(buffer->time),
717 &buffer->time_length);
718
Arnd Bergmanned108582017-02-17 16:03:52 +0100719 local_time = ktime_get_real_seconds();
720 time64_to_tm(local_time, -sys_tz.tz_minuteswest * 60, &tm);
Kevin Barnett6c223762016-06-27 16:41:00 -0500721 year = tm.tm_year + 1900;
722
723 buffer->time[0] = bin2bcd(tm.tm_hour);
724 buffer->time[1] = bin2bcd(tm.tm_min);
725 buffer->time[2] = bin2bcd(tm.tm_sec);
726 buffer->time[3] = 0;
727 buffer->time[4] = bin2bcd(tm.tm_mon + 1);
728 buffer->time[5] = bin2bcd(tm.tm_mday);
729 buffer->time[6] = bin2bcd(year / 100);
730 buffer->time[7] = bin2bcd(year % 100);
731
732 buffer->dont_write_tag[0] = 'D';
733 buffer->dont_write_tag[1] = 'W';
734 buffer->end_tag[0] = 'Z';
735 buffer->end_tag[1] = 'Z';
736
737 rc = pqi_write_host_wellness(ctrl_info, buffer, buffer_length);
738
739 kfree(buffer);
740
741 return rc;
742}
743
744#define PQI_UPDATE_TIME_WORK_INTERVAL (24UL * 60 * 60 * HZ)
745
746static void pqi_update_time_worker(struct work_struct *work)
747{
748 int rc;
749 struct pqi_ctrl_info *ctrl_info;
750
751 ctrl_info = container_of(to_delayed_work(work), struct pqi_ctrl_info,
752 update_time_work);
753
Kevin Barnett5f310422017-05-03 18:54:55 -0500754 if (pqi_ctrl_offline(ctrl_info))
755 return;
756
Kevin Barnett6c223762016-06-27 16:41:00 -0500757 rc = pqi_write_current_time_to_host_wellness(ctrl_info);
758 if (rc)
759 dev_warn(&ctrl_info->pci_dev->dev,
760 "error updating time on controller\n");
761
762 schedule_delayed_work(&ctrl_info->update_time_work,
763 PQI_UPDATE_TIME_WORK_INTERVAL);
764}
765
766static inline void pqi_schedule_update_time_worker(
Kevin Barnett4fbebf12016-08-31 14:55:05 -0500767 struct pqi_ctrl_info *ctrl_info)
Kevin Barnett6c223762016-06-27 16:41:00 -0500768{
Kevin Barnett4fbebf12016-08-31 14:55:05 -0500769 schedule_delayed_work(&ctrl_info->update_time_work, 0);
Kevin Barnett061ef062017-05-03 18:53:05 -0500770}
771
772static inline void pqi_cancel_update_time_worker(
773 struct pqi_ctrl_info *ctrl_info)
774{
Kevin Barnett061ef062017-05-03 18:53:05 -0500775 cancel_delayed_work_sync(&ctrl_info->update_time_work);
Kevin Barnett6c223762016-06-27 16:41:00 -0500776}
777
778static int pqi_report_luns(struct pqi_ctrl_info *ctrl_info, u8 cmd,
779 void *buffer, size_t buffer_length)
780{
781 int rc;
782 int pci_direction;
783 struct pqi_raid_path_request request;
784
785 rc = pqi_build_raid_path_request(ctrl_info, &request,
786 cmd, RAID_CTLR_LUNID, buffer, buffer_length, 0, &pci_direction);
787 if (rc)
788 return rc;
789
790 rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0,
791 NULL, NO_TIMEOUT);
792
793 pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1,
794 pci_direction);
795
796 return rc;
797}
798
799static int pqi_report_phys_logical_luns(struct pqi_ctrl_info *ctrl_info, u8 cmd,
800 void **buffer)
801{
802 int rc;
803 size_t lun_list_length;
804 size_t lun_data_length;
805 size_t new_lun_list_length;
806 void *lun_data = NULL;
807 struct report_lun_header *report_lun_header;
808
809 report_lun_header = kmalloc(sizeof(*report_lun_header), GFP_KERNEL);
810 if (!report_lun_header) {
811 rc = -ENOMEM;
812 goto out;
813 }
814
815 rc = pqi_report_luns(ctrl_info, cmd, report_lun_header,
816 sizeof(*report_lun_header));
817 if (rc)
818 goto out;
819
820 lun_list_length = get_unaligned_be32(&report_lun_header->list_length);
821
822again:
823 lun_data_length = sizeof(struct report_lun_header) + lun_list_length;
824
825 lun_data = kmalloc(lun_data_length, GFP_KERNEL);
826 if (!lun_data) {
827 rc = -ENOMEM;
828 goto out;
829 }
830
831 if (lun_list_length == 0) {
832 memcpy(lun_data, report_lun_header, sizeof(*report_lun_header));
833 goto out;
834 }
835
836 rc = pqi_report_luns(ctrl_info, cmd, lun_data, lun_data_length);
837 if (rc)
838 goto out;
839
840 new_lun_list_length = get_unaligned_be32(
841 &((struct report_lun_header *)lun_data)->list_length);
842
843 if (new_lun_list_length > lun_list_length) {
844 lun_list_length = new_lun_list_length;
845 kfree(lun_data);
846 goto again;
847 }
848
849out:
850 kfree(report_lun_header);
851
852 if (rc) {
853 kfree(lun_data);
854 lun_data = NULL;
855 }
856
857 *buffer = lun_data;
858
859 return rc;
860}
861
862static inline int pqi_report_phys_luns(struct pqi_ctrl_info *ctrl_info,
863 void **buffer)
864{
865 return pqi_report_phys_logical_luns(ctrl_info, CISS_REPORT_PHYS,
866 buffer);
867}
868
869static inline int pqi_report_logical_luns(struct pqi_ctrl_info *ctrl_info,
870 void **buffer)
871{
872 return pqi_report_phys_logical_luns(ctrl_info, CISS_REPORT_LOG, buffer);
873}
874
875static int pqi_get_device_lists(struct pqi_ctrl_info *ctrl_info,
876 struct report_phys_lun_extended **physdev_list,
877 struct report_log_lun_extended **logdev_list)
878{
879 int rc;
880 size_t logdev_list_length;
881 size_t logdev_data_length;
882 struct report_log_lun_extended *internal_logdev_list;
883 struct report_log_lun_extended *logdev_data;
884 struct report_lun_header report_lun_header;
885
886 rc = pqi_report_phys_luns(ctrl_info, (void **)physdev_list);
887 if (rc)
888 dev_err(&ctrl_info->pci_dev->dev,
889 "report physical LUNs failed\n");
890
891 rc = pqi_report_logical_luns(ctrl_info, (void **)logdev_list);
892 if (rc)
893 dev_err(&ctrl_info->pci_dev->dev,
894 "report logical LUNs failed\n");
895
896 /*
897 * Tack the controller itself onto the end of the logical device list.
898 */
899
900 logdev_data = *logdev_list;
901
902 if (logdev_data) {
903 logdev_list_length =
904 get_unaligned_be32(&logdev_data->header.list_length);
905 } else {
906 memset(&report_lun_header, 0, sizeof(report_lun_header));
907 logdev_data =
908 (struct report_log_lun_extended *)&report_lun_header;
909 logdev_list_length = 0;
910 }
911
912 logdev_data_length = sizeof(struct report_lun_header) +
913 logdev_list_length;
914
915 internal_logdev_list = kmalloc(logdev_data_length +
916 sizeof(struct report_log_lun_extended), GFP_KERNEL);
917 if (!internal_logdev_list) {
918 kfree(*logdev_list);
919 *logdev_list = NULL;
920 return -ENOMEM;
921 }
922
923 memcpy(internal_logdev_list, logdev_data, logdev_data_length);
924 memset((u8 *)internal_logdev_list + logdev_data_length, 0,
925 sizeof(struct report_log_lun_extended_entry));
926 put_unaligned_be32(logdev_list_length +
927 sizeof(struct report_log_lun_extended_entry),
928 &internal_logdev_list->header.list_length);
929
930 kfree(*logdev_list);
931 *logdev_list = internal_logdev_list;
932
933 return 0;
934}
935
936static inline void pqi_set_bus_target_lun(struct pqi_scsi_dev *device,
937 int bus, int target, int lun)
938{
939 device->bus = bus;
940 device->target = target;
941 device->lun = lun;
942}
943
944static void pqi_assign_bus_target_lun(struct pqi_scsi_dev *device)
945{
946 u8 *scsi3addr;
947 u32 lunid;
Kevin Barnettbd10cf02017-05-03 18:54:12 -0500948 int bus;
949 int target;
950 int lun;
Kevin Barnett6c223762016-06-27 16:41:00 -0500951
952 scsi3addr = device->scsi3addr;
953 lunid = get_unaligned_le32(scsi3addr);
954
955 if (pqi_is_hba_lunid(scsi3addr)) {
956 /* The specified device is the controller. */
957 pqi_set_bus_target_lun(device, PQI_HBA_BUS, 0, lunid & 0x3fff);
958 device->target_lun_valid = true;
959 return;
960 }
961
962 if (pqi_is_logical_device(device)) {
Kevin Barnettbd10cf02017-05-03 18:54:12 -0500963 if (device->is_external_raid_device) {
964 bus = PQI_EXTERNAL_RAID_VOLUME_BUS;
965 target = (lunid >> 16) & 0x3fff;
966 lun = lunid & 0xff;
967 } else {
968 bus = PQI_RAID_VOLUME_BUS;
969 target = 0;
970 lun = lunid & 0x3fff;
971 }
972 pqi_set_bus_target_lun(device, bus, target, lun);
Kevin Barnett6c223762016-06-27 16:41:00 -0500973 device->target_lun_valid = true;
974 return;
975 }
976
977 /*
978 * Defer target and LUN assignment for non-controller physical devices
979 * because the SAS transport layer will make these assignments later.
980 */
981 pqi_set_bus_target_lun(device, PQI_PHYSICAL_DEVICE_BUS, 0, 0);
982}
983
984static void pqi_get_raid_level(struct pqi_ctrl_info *ctrl_info,
985 struct pqi_scsi_dev *device)
986{
987 int rc;
988 u8 raid_level;
989 u8 *buffer;
990
991 raid_level = SA_RAID_UNKNOWN;
992
993 buffer = kmalloc(64, GFP_KERNEL);
994 if (buffer) {
995 rc = pqi_scsi_inquiry(ctrl_info, device->scsi3addr,
996 VPD_PAGE | CISS_VPD_LV_DEVICE_GEOMETRY, buffer, 64);
997 if (rc == 0) {
998 raid_level = buffer[8];
999 if (raid_level > SA_RAID_MAX)
1000 raid_level = SA_RAID_UNKNOWN;
1001 }
1002 kfree(buffer);
1003 }
1004
1005 device->raid_level = raid_level;
1006}
1007
1008static int pqi_validate_raid_map(struct pqi_ctrl_info *ctrl_info,
1009 struct pqi_scsi_dev *device, struct raid_map *raid_map)
1010{
1011 char *err_msg;
1012 u32 raid_map_size;
1013 u32 r5or6_blocks_per_row;
1014 unsigned int num_phys_disks;
1015 unsigned int num_raid_map_entries;
1016
1017 raid_map_size = get_unaligned_le32(&raid_map->structure_size);
1018
1019 if (raid_map_size < offsetof(struct raid_map, disk_data)) {
1020 err_msg = "RAID map too small";
1021 goto bad_raid_map;
1022 }
1023
1024 if (raid_map_size > sizeof(*raid_map)) {
1025 err_msg = "RAID map too large";
1026 goto bad_raid_map;
1027 }
1028
1029 num_phys_disks = get_unaligned_le16(&raid_map->layout_map_count) *
1030 (get_unaligned_le16(&raid_map->data_disks_per_row) +
1031 get_unaligned_le16(&raid_map->metadata_disks_per_row));
1032 num_raid_map_entries = num_phys_disks *
1033 get_unaligned_le16(&raid_map->row_cnt);
1034
1035 if (num_raid_map_entries > RAID_MAP_MAX_ENTRIES) {
1036 err_msg = "invalid number of map entries in RAID map";
1037 goto bad_raid_map;
1038 }
1039
1040 if (device->raid_level == SA_RAID_1) {
1041 if (get_unaligned_le16(&raid_map->layout_map_count) != 2) {
1042 err_msg = "invalid RAID-1 map";
1043 goto bad_raid_map;
1044 }
1045 } else if (device->raid_level == SA_RAID_ADM) {
1046 if (get_unaligned_le16(&raid_map->layout_map_count) != 3) {
1047 err_msg = "invalid RAID-1(ADM) map";
1048 goto bad_raid_map;
1049 }
1050 } else if ((device->raid_level == SA_RAID_5 ||
1051 device->raid_level == SA_RAID_6) &&
1052 get_unaligned_le16(&raid_map->layout_map_count) > 1) {
1053 /* RAID 50/60 */
1054 r5or6_blocks_per_row =
1055 get_unaligned_le16(&raid_map->strip_size) *
1056 get_unaligned_le16(&raid_map->data_disks_per_row);
1057 if (r5or6_blocks_per_row == 0) {
1058 err_msg = "invalid RAID-5 or RAID-6 map";
1059 goto bad_raid_map;
1060 }
1061 }
1062
1063 return 0;
1064
1065bad_raid_map:
Kevin Barnettd87d5472017-05-03 18:54:00 -05001066 dev_warn(&ctrl_info->pci_dev->dev,
1067 "scsi %d:%d:%d:%d %s\n",
1068 ctrl_info->scsi_host->host_no,
1069 device->bus, device->target, device->lun, err_msg);
Kevin Barnett6c223762016-06-27 16:41:00 -05001070
1071 return -EINVAL;
1072}
1073
1074static int pqi_get_raid_map(struct pqi_ctrl_info *ctrl_info,
1075 struct pqi_scsi_dev *device)
1076{
1077 int rc;
1078 int pci_direction;
1079 struct pqi_raid_path_request request;
1080 struct raid_map *raid_map;
1081
1082 raid_map = kmalloc(sizeof(*raid_map), GFP_KERNEL);
1083 if (!raid_map)
1084 return -ENOMEM;
1085
1086 rc = pqi_build_raid_path_request(ctrl_info, &request,
1087 CISS_GET_RAID_MAP, device->scsi3addr, raid_map,
1088 sizeof(*raid_map), 0, &pci_direction);
1089 if (rc)
1090 goto error;
1091
1092 rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0,
1093 NULL, NO_TIMEOUT);
1094
1095 pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1,
1096 pci_direction);
1097
1098 if (rc)
1099 goto error;
1100
1101 rc = pqi_validate_raid_map(ctrl_info, device, raid_map);
1102 if (rc)
1103 goto error;
1104
1105 device->raid_map = raid_map;
1106
1107 return 0;
1108
1109error:
1110 kfree(raid_map);
1111
1112 return rc;
1113}
1114
Kevin Barnett588a63fe2017-05-03 18:55:25 -05001115static void pqi_get_raid_bypass_status(struct pqi_ctrl_info *ctrl_info,
Kevin Barnett6c223762016-06-27 16:41:00 -05001116 struct pqi_scsi_dev *device)
1117{
1118 int rc;
1119 u8 *buffer;
Kevin Barnett588a63fe2017-05-03 18:55:25 -05001120 u8 bypass_status;
Kevin Barnett6c223762016-06-27 16:41:00 -05001121
1122 buffer = kmalloc(64, GFP_KERNEL);
1123 if (!buffer)
1124 return;
1125
1126 rc = pqi_scsi_inquiry(ctrl_info, device->scsi3addr,
Kevin Barnett588a63fe2017-05-03 18:55:25 -05001127 VPD_PAGE | CISS_VPD_LV_BYPASS_STATUS, buffer, 64);
Kevin Barnett6c223762016-06-27 16:41:00 -05001128 if (rc)
1129 goto out;
1130
Kevin Barnett588a63fe2017-05-03 18:55:25 -05001131#define RAID_BYPASS_STATUS 4
1132#define RAID_BYPASS_CONFIGURED 0x1
1133#define RAID_BYPASS_ENABLED 0x2
Kevin Barnett6c223762016-06-27 16:41:00 -05001134
Kevin Barnett588a63fe2017-05-03 18:55:25 -05001135 bypass_status = buffer[RAID_BYPASS_STATUS];
1136 device->raid_bypass_configured =
1137 (bypass_status & RAID_BYPASS_CONFIGURED) != 0;
1138 if (device->raid_bypass_configured &&
1139 (bypass_status & RAID_BYPASS_ENABLED) &&
1140 pqi_get_raid_map(ctrl_info, device) == 0)
1141 device->raid_bypass_enabled = true;
Kevin Barnett6c223762016-06-27 16:41:00 -05001142
1143out:
1144 kfree(buffer);
1145}
1146
1147/*
1148 * Use vendor-specific VPD to determine online/offline status of a volume.
1149 */
1150
1151static void pqi_get_volume_status(struct pqi_ctrl_info *ctrl_info,
1152 struct pqi_scsi_dev *device)
1153{
1154 int rc;
1155 size_t page_length;
1156 u8 volume_status = CISS_LV_STATUS_UNAVAILABLE;
1157 bool volume_offline = true;
1158 u32 volume_flags;
1159 struct ciss_vpd_logical_volume_status *vpd;
1160
1161 vpd = kmalloc(sizeof(*vpd), GFP_KERNEL);
1162 if (!vpd)
1163 goto no_buffer;
1164
1165 rc = pqi_scsi_inquiry(ctrl_info, device->scsi3addr,
1166 VPD_PAGE | CISS_VPD_LV_STATUS, vpd, sizeof(*vpd));
1167 if (rc)
1168 goto out;
1169
1170 page_length = offsetof(struct ciss_vpd_logical_volume_status,
1171 volume_status) + vpd->page_length;
1172 if (page_length < sizeof(*vpd))
1173 goto out;
1174
1175 volume_status = vpd->volume_status;
1176 volume_flags = get_unaligned_be32(&vpd->flags);
1177 volume_offline = (volume_flags & CISS_LV_FLAGS_NO_HOST_IO) != 0;
1178
1179out:
1180 kfree(vpd);
1181no_buffer:
1182 device->volume_status = volume_status;
1183 device->volume_offline = volume_offline;
1184}
1185
1186static int pqi_get_device_info(struct pqi_ctrl_info *ctrl_info,
1187 struct pqi_scsi_dev *device)
1188{
1189 int rc;
1190 u8 *buffer;
1191
1192 buffer = kmalloc(64, GFP_KERNEL);
1193 if (!buffer)
1194 return -ENOMEM;
1195
1196 /* Send an inquiry to the device to see what it is. */
1197 rc = pqi_scsi_inquiry(ctrl_info, device->scsi3addr, 0, buffer, 64);
1198 if (rc)
1199 goto out;
1200
1201 scsi_sanitize_inquiry_string(&buffer[8], 8);
1202 scsi_sanitize_inquiry_string(&buffer[16], 16);
1203
1204 device->devtype = buffer[0] & 0x1f;
Kevin Barnettcbe0c7b2017-05-03 18:53:48 -05001205 memcpy(device->vendor, &buffer[8], sizeof(device->vendor));
1206 memcpy(device->model, &buffer[16], sizeof(device->model));
Kevin Barnett6c223762016-06-27 16:41:00 -05001207
1208 if (pqi_is_logical_device(device) && device->devtype == TYPE_DISK) {
Kevin Barnettbd10cf02017-05-03 18:54:12 -05001209 if (device->is_external_raid_device) {
1210 device->raid_level = SA_RAID_UNKNOWN;
1211 device->volume_status = CISS_LV_OK;
1212 device->volume_offline = false;
1213 } else {
1214 pqi_get_raid_level(ctrl_info, device);
Kevin Barnett588a63fe2017-05-03 18:55:25 -05001215 pqi_get_raid_bypass_status(ctrl_info, device);
Kevin Barnettbd10cf02017-05-03 18:54:12 -05001216 pqi_get_volume_status(ctrl_info, device);
1217 }
Kevin Barnett6c223762016-06-27 16:41:00 -05001218 }
1219
1220out:
1221 kfree(buffer);
1222
1223 return rc;
1224}
1225
1226static void pqi_get_physical_disk_info(struct pqi_ctrl_info *ctrl_info,
1227 struct pqi_scsi_dev *device,
1228 struct bmic_identify_physical_device *id_phys)
1229{
1230 int rc;
1231
1232 memset(id_phys, 0, sizeof(*id_phys));
1233
1234 rc = pqi_identify_physical_device(ctrl_info, device,
1235 id_phys, sizeof(*id_phys));
1236 if (rc) {
1237 device->queue_depth = PQI_PHYSICAL_DISK_DEFAULT_MAX_QUEUE_DEPTH;
1238 return;
1239 }
1240
1241 device->queue_depth =
1242 get_unaligned_le16(&id_phys->current_queue_depth_limit);
1243 device->device_type = id_phys->device_type;
1244 device->active_path_index = id_phys->active_path_number;
1245 device->path_map = id_phys->redundant_path_present_map;
1246 memcpy(&device->box,
1247 &id_phys->alternate_paths_phys_box_on_port,
1248 sizeof(device->box));
1249 memcpy(&device->phys_connector,
1250 &id_phys->alternate_paths_phys_connector,
1251 sizeof(device->phys_connector));
1252 device->bay = id_phys->phys_bay_in_box;
1253}
1254
1255static void pqi_show_volume_status(struct pqi_ctrl_info *ctrl_info,
1256 struct pqi_scsi_dev *device)
1257{
1258 char *status;
1259 static const char unknown_state_str[] =
1260 "Volume is in an unknown state (%u)";
1261 char unknown_state_buffer[sizeof(unknown_state_str) + 10];
1262
1263 switch (device->volume_status) {
1264 case CISS_LV_OK:
1265 status = "Volume online";
1266 break;
1267 case CISS_LV_FAILED:
1268 status = "Volume failed";
1269 break;
1270 case CISS_LV_NOT_CONFIGURED:
1271 status = "Volume not configured";
1272 break;
1273 case CISS_LV_DEGRADED:
1274 status = "Volume degraded";
1275 break;
1276 case CISS_LV_READY_FOR_RECOVERY:
1277 status = "Volume ready for recovery operation";
1278 break;
1279 case CISS_LV_UNDERGOING_RECOVERY:
1280 status = "Volume undergoing recovery";
1281 break;
1282 case CISS_LV_WRONG_PHYSICAL_DRIVE_REPLACED:
1283 status = "Wrong physical drive was replaced";
1284 break;
1285 case CISS_LV_PHYSICAL_DRIVE_CONNECTION_PROBLEM:
1286 status = "A physical drive not properly connected";
1287 break;
1288 case CISS_LV_HARDWARE_OVERHEATING:
1289 status = "Hardware is overheating";
1290 break;
1291 case CISS_LV_HARDWARE_HAS_OVERHEATED:
1292 status = "Hardware has overheated";
1293 break;
1294 case CISS_LV_UNDERGOING_EXPANSION:
1295 status = "Volume undergoing expansion";
1296 break;
1297 case CISS_LV_NOT_AVAILABLE:
1298 status = "Volume waiting for transforming volume";
1299 break;
1300 case CISS_LV_QUEUED_FOR_EXPANSION:
1301 status = "Volume queued for expansion";
1302 break;
1303 case CISS_LV_DISABLED_SCSI_ID_CONFLICT:
1304 status = "Volume disabled due to SCSI ID conflict";
1305 break;
1306 case CISS_LV_EJECTED:
1307 status = "Volume has been ejected";
1308 break;
1309 case CISS_LV_UNDERGOING_ERASE:
1310 status = "Volume undergoing background erase";
1311 break;
1312 case CISS_LV_READY_FOR_PREDICTIVE_SPARE_REBUILD:
1313 status = "Volume ready for predictive spare rebuild";
1314 break;
1315 case CISS_LV_UNDERGOING_RPI:
1316 status = "Volume undergoing rapid parity initialization";
1317 break;
1318 case CISS_LV_PENDING_RPI:
1319 status = "Volume queued for rapid parity initialization";
1320 break;
1321 case CISS_LV_ENCRYPTED_NO_KEY:
1322 status = "Encrypted volume inaccessible - key not present";
1323 break;
1324 case CISS_LV_UNDERGOING_ENCRYPTION:
1325 status = "Volume undergoing encryption process";
1326 break;
1327 case CISS_LV_UNDERGOING_ENCRYPTION_REKEYING:
1328 status = "Volume undergoing encryption re-keying process";
1329 break;
1330 case CISS_LV_ENCRYPTED_IN_NON_ENCRYPTED_CONTROLLER:
Kevin Barnettd87d5472017-05-03 18:54:00 -05001331 status = "Volume encrypted but encryption is disabled";
Kevin Barnett6c223762016-06-27 16:41:00 -05001332 break;
1333 case CISS_LV_PENDING_ENCRYPTION:
1334 status = "Volume pending migration to encrypted state";
1335 break;
1336 case CISS_LV_PENDING_ENCRYPTION_REKEYING:
1337 status = "Volume pending encryption rekeying";
1338 break;
1339 case CISS_LV_NOT_SUPPORTED:
1340 status = "Volume not supported on this controller";
1341 break;
1342 case CISS_LV_STATUS_UNAVAILABLE:
1343 status = "Volume status not available";
1344 break;
1345 default:
1346 snprintf(unknown_state_buffer, sizeof(unknown_state_buffer),
1347 unknown_state_str, device->volume_status);
1348 status = unknown_state_buffer;
1349 break;
1350 }
1351
1352 dev_info(&ctrl_info->pci_dev->dev,
1353 "scsi %d:%d:%d:%d %s\n",
1354 ctrl_info->scsi_host->host_no,
1355 device->bus, device->target, device->lun, status);
1356}
1357
Kevin Barnett6c223762016-06-27 16:41:00 -05001358static void pqi_rescan_worker(struct work_struct *work)
1359{
1360 struct pqi_ctrl_info *ctrl_info;
1361
1362 ctrl_info = container_of(to_delayed_work(work), struct pqi_ctrl_info,
1363 rescan_work);
1364
1365 pqi_scan_scsi_devices(ctrl_info);
1366}
1367
1368static int pqi_add_device(struct pqi_ctrl_info *ctrl_info,
1369 struct pqi_scsi_dev *device)
1370{
1371 int rc;
1372
1373 if (pqi_is_logical_device(device))
1374 rc = scsi_add_device(ctrl_info->scsi_host, device->bus,
1375 device->target, device->lun);
1376 else
1377 rc = pqi_add_sas_device(ctrl_info->sas_host, device);
1378
1379 return rc;
1380}
1381
1382static inline void pqi_remove_device(struct pqi_ctrl_info *ctrl_info,
1383 struct pqi_scsi_dev *device)
1384{
1385 if (pqi_is_logical_device(device))
1386 scsi_remove_device(device->sdev);
1387 else
1388 pqi_remove_sas_device(device);
1389}
1390
1391/* Assumes the SCSI device list lock is held. */
1392
1393static struct pqi_scsi_dev *pqi_find_scsi_dev(struct pqi_ctrl_info *ctrl_info,
1394 int bus, int target, int lun)
1395{
1396 struct pqi_scsi_dev *device;
1397
1398 list_for_each_entry(device, &ctrl_info->scsi_device_list,
1399 scsi_device_list_entry)
1400 if (device->bus == bus && device->target == target &&
1401 device->lun == lun)
1402 return device;
1403
1404 return NULL;
1405}
1406
1407static inline bool pqi_device_equal(struct pqi_scsi_dev *dev1,
1408 struct pqi_scsi_dev *dev2)
1409{
1410 if (dev1->is_physical_device != dev2->is_physical_device)
1411 return false;
1412
1413 if (dev1->is_physical_device)
1414 return dev1->wwid == dev2->wwid;
1415
1416 return memcmp(dev1->volume_id, dev2->volume_id,
1417 sizeof(dev1->volume_id)) == 0;
1418}
1419
1420enum pqi_find_result {
1421 DEVICE_NOT_FOUND,
1422 DEVICE_CHANGED,
1423 DEVICE_SAME,
1424};
1425
1426static enum pqi_find_result pqi_scsi_find_entry(struct pqi_ctrl_info *ctrl_info,
1427 struct pqi_scsi_dev *device_to_find,
1428 struct pqi_scsi_dev **matching_device)
1429{
1430 struct pqi_scsi_dev *device;
1431
1432 list_for_each_entry(device, &ctrl_info->scsi_device_list,
1433 scsi_device_list_entry) {
1434 if (pqi_scsi3addr_equal(device_to_find->scsi3addr,
1435 device->scsi3addr)) {
1436 *matching_device = device;
1437 if (pqi_device_equal(device_to_find, device)) {
1438 if (device_to_find->volume_offline)
1439 return DEVICE_CHANGED;
1440 return DEVICE_SAME;
1441 }
1442 return DEVICE_CHANGED;
1443 }
1444 }
1445
1446 return DEVICE_NOT_FOUND;
1447}
1448
Kevin Barnett6de783f2017-05-03 18:55:19 -05001449#define PQI_DEV_INFO_BUFFER_LENGTH 128
1450
Kevin Barnett6c223762016-06-27 16:41:00 -05001451static void pqi_dev_info(struct pqi_ctrl_info *ctrl_info,
1452 char *action, struct pqi_scsi_dev *device)
1453{
Kevin Barnett6de783f2017-05-03 18:55:19 -05001454 ssize_t count;
1455 char buffer[PQI_DEV_INFO_BUFFER_LENGTH];
1456
1457 count = snprintf(buffer, PQI_DEV_INFO_BUFFER_LENGTH,
1458 "%d:%d:", ctrl_info->scsi_host->host_no, device->bus);
1459
1460 if (device->target_lun_valid)
1461 count += snprintf(buffer + count,
1462 PQI_DEV_INFO_BUFFER_LENGTH - count,
1463 "%d:%d",
1464 device->target,
1465 device->lun);
1466 else
1467 count += snprintf(buffer + count,
1468 PQI_DEV_INFO_BUFFER_LENGTH - count,
1469 "-:-");
1470
1471 if (pqi_is_logical_device(device))
1472 count += snprintf(buffer + count,
1473 PQI_DEV_INFO_BUFFER_LENGTH - count,
1474 " %08x%08x",
1475 *((u32 *)&device->scsi3addr),
1476 *((u32 *)&device->scsi3addr[4]));
1477 else
1478 count += snprintf(buffer + count,
1479 PQI_DEV_INFO_BUFFER_LENGTH - count,
1480 " %016llx", device->sas_address);
1481
1482 count += snprintf(buffer + count, PQI_DEV_INFO_BUFFER_LENGTH - count,
1483 " %s %.8s %.16s ",
Kevin Barnett6c223762016-06-27 16:41:00 -05001484 scsi_device_type(device->devtype),
1485 device->vendor,
Kevin Barnett6de783f2017-05-03 18:55:19 -05001486 device->model);
1487
1488 if (pqi_is_logical_device(device)) {
1489 if (device->devtype == TYPE_DISK)
1490 count += snprintf(buffer + count,
1491 PQI_DEV_INFO_BUFFER_LENGTH - count,
1492 "SSDSmartPathCap%c En%c %-12s",
Kevin Barnett588a63fe2017-05-03 18:55:25 -05001493 device->raid_bypass_configured ? '+' : '-',
1494 device->raid_bypass_enabled ? '+' : '-',
Kevin Barnett6de783f2017-05-03 18:55:19 -05001495 pqi_raid_level_to_string(device->raid_level));
1496 } else {
1497 count += snprintf(buffer + count,
1498 PQI_DEV_INFO_BUFFER_LENGTH - count,
1499 "AIO%c", device->aio_enabled ? '+' : '-');
1500 if (device->devtype == TYPE_DISK ||
1501 device->devtype == TYPE_ZBC)
1502 count += snprintf(buffer + count,
1503 PQI_DEV_INFO_BUFFER_LENGTH - count,
1504 " qd=%-6d", device->queue_depth);
1505 }
1506
1507 dev_info(&ctrl_info->pci_dev->dev, "%s %s\n", action, buffer);
Kevin Barnett6c223762016-06-27 16:41:00 -05001508}
1509
1510/* Assumes the SCSI device list lock is held. */
1511
1512static void pqi_scsi_update_device(struct pqi_scsi_dev *existing_device,
1513 struct pqi_scsi_dev *new_device)
1514{
1515 existing_device->devtype = new_device->devtype;
1516 existing_device->device_type = new_device->device_type;
1517 existing_device->bus = new_device->bus;
1518 if (new_device->target_lun_valid) {
1519 existing_device->target = new_device->target;
1520 existing_device->lun = new_device->lun;
1521 existing_device->target_lun_valid = true;
1522 }
1523
1524 /* By definition, the scsi3addr and wwid fields are already the same. */
1525
1526 existing_device->is_physical_device = new_device->is_physical_device;
Kevin Barnettbd10cf02017-05-03 18:54:12 -05001527 existing_device->is_external_raid_device =
1528 new_device->is_external_raid_device;
Kevin Barnett6c223762016-06-27 16:41:00 -05001529 existing_device->aio_enabled = new_device->aio_enabled;
1530 memcpy(existing_device->vendor, new_device->vendor,
1531 sizeof(existing_device->vendor));
1532 memcpy(existing_device->model, new_device->model,
1533 sizeof(existing_device->model));
1534 existing_device->sas_address = new_device->sas_address;
1535 existing_device->raid_level = new_device->raid_level;
1536 existing_device->queue_depth = new_device->queue_depth;
1537 existing_device->aio_handle = new_device->aio_handle;
1538 existing_device->volume_status = new_device->volume_status;
1539 existing_device->active_path_index = new_device->active_path_index;
1540 existing_device->path_map = new_device->path_map;
1541 existing_device->bay = new_device->bay;
1542 memcpy(existing_device->box, new_device->box,
1543 sizeof(existing_device->box));
1544 memcpy(existing_device->phys_connector, new_device->phys_connector,
1545 sizeof(existing_device->phys_connector));
Kevin Barnett6c223762016-06-27 16:41:00 -05001546 existing_device->offload_to_mirror = 0;
1547 kfree(existing_device->raid_map);
1548 existing_device->raid_map = new_device->raid_map;
Kevin Barnett588a63fe2017-05-03 18:55:25 -05001549 existing_device->raid_bypass_configured =
1550 new_device->raid_bypass_configured;
1551 existing_device->raid_bypass_enabled =
1552 new_device->raid_bypass_enabled;
Kevin Barnett6c223762016-06-27 16:41:00 -05001553
1554 /* To prevent this from being freed later. */
1555 new_device->raid_map = NULL;
1556}
1557
1558static inline void pqi_free_device(struct pqi_scsi_dev *device)
1559{
1560 if (device) {
1561 kfree(device->raid_map);
1562 kfree(device);
1563 }
1564}
1565
1566/*
1567 * Called when exposing a new device to the OS fails in order to re-adjust
1568 * our internal SCSI device list to match the SCSI ML's view.
1569 */
1570
1571static inline void pqi_fixup_botched_add(struct pqi_ctrl_info *ctrl_info,
1572 struct pqi_scsi_dev *device)
1573{
1574 unsigned long flags;
1575
1576 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
1577 list_del(&device->scsi_device_list_entry);
1578 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
1579
1580 /* Allow the device structure to be freed later. */
1581 device->keep_device = false;
1582}
1583
1584static void pqi_update_device_list(struct pqi_ctrl_info *ctrl_info,
1585 struct pqi_scsi_dev *new_device_list[], unsigned int num_new_devices)
1586{
1587 int rc;
1588 unsigned int i;
1589 unsigned long flags;
1590 enum pqi_find_result find_result;
1591 struct pqi_scsi_dev *device;
1592 struct pqi_scsi_dev *next;
1593 struct pqi_scsi_dev *matching_device;
1594 struct list_head add_list;
1595 struct list_head delete_list;
1596
1597 INIT_LIST_HEAD(&add_list);
1598 INIT_LIST_HEAD(&delete_list);
1599
1600 /*
1601 * The idea here is to do as little work as possible while holding the
1602 * spinlock. That's why we go to great pains to defer anything other
1603 * than updating the internal device list until after we release the
1604 * spinlock.
1605 */
1606
1607 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
1608
1609 /* Assume that all devices in the existing list have gone away. */
1610 list_for_each_entry(device, &ctrl_info->scsi_device_list,
1611 scsi_device_list_entry)
1612 device->device_gone = true;
1613
1614 for (i = 0; i < num_new_devices; i++) {
1615 device = new_device_list[i];
1616
1617 find_result = pqi_scsi_find_entry(ctrl_info, device,
1618 &matching_device);
1619
1620 switch (find_result) {
1621 case DEVICE_SAME:
1622 /*
1623 * The newly found device is already in the existing
1624 * device list.
1625 */
1626 device->new_device = false;
1627 matching_device->device_gone = false;
1628 pqi_scsi_update_device(matching_device, device);
1629 break;
1630 case DEVICE_NOT_FOUND:
1631 /*
1632 * The newly found device is NOT in the existing device
1633 * list.
1634 */
1635 device->new_device = true;
1636 break;
1637 case DEVICE_CHANGED:
1638 /*
1639 * The original device has gone away and we need to add
1640 * the new device.
1641 */
1642 device->new_device = true;
1643 break;
Kevin Barnett6c223762016-06-27 16:41:00 -05001644 }
1645 }
1646
1647 /* Process all devices that have gone away. */
1648 list_for_each_entry_safe(device, next, &ctrl_info->scsi_device_list,
1649 scsi_device_list_entry) {
1650 if (device->device_gone) {
1651 list_del(&device->scsi_device_list_entry);
1652 list_add_tail(&device->delete_list_entry, &delete_list);
1653 }
1654 }
1655
1656 /* Process all new devices. */
1657 for (i = 0; i < num_new_devices; i++) {
1658 device = new_device_list[i];
1659 if (!device->new_device)
1660 continue;
1661 if (device->volume_offline)
1662 continue;
1663 list_add_tail(&device->scsi_device_list_entry,
1664 &ctrl_info->scsi_device_list);
1665 list_add_tail(&device->add_list_entry, &add_list);
1666 /* To prevent this device structure from being freed later. */
1667 device->keep_device = true;
1668 }
1669
Kevin Barnett6c223762016-06-27 16:41:00 -05001670 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
1671
1672 /* Remove all devices that have gone away. */
1673 list_for_each_entry_safe(device, next, &delete_list,
1674 delete_list_entry) {
Kevin Barnett6c223762016-06-27 16:41:00 -05001675 if (device->volume_offline) {
1676 pqi_dev_info(ctrl_info, "offline", device);
1677 pqi_show_volume_status(ctrl_info, device);
1678 } else {
1679 pqi_dev_info(ctrl_info, "removed", device);
1680 }
Kevin Barnett6de783f2017-05-03 18:55:19 -05001681 if (device->sdev)
1682 pqi_remove_device(ctrl_info, device);
Kevin Barnett6c223762016-06-27 16:41:00 -05001683 list_del(&device->delete_list_entry);
1684 pqi_free_device(device);
1685 }
1686
1687 /*
1688 * Notify the SCSI ML if the queue depth of any existing device has
1689 * changed.
1690 */
1691 list_for_each_entry(device, &ctrl_info->scsi_device_list,
1692 scsi_device_list_entry) {
1693 if (device->sdev && device->queue_depth !=
1694 device->advertised_queue_depth) {
1695 device->advertised_queue_depth = device->queue_depth;
1696 scsi_change_queue_depth(device->sdev,
1697 device->advertised_queue_depth);
1698 }
1699 }
1700
1701 /* Expose any new devices. */
1702 list_for_each_entry_safe(device, next, &add_list, add_list_entry) {
Kevin Barnett94086f52017-05-03 18:54:31 -05001703 if (!device->sdev) {
Kevin Barnett6de783f2017-05-03 18:55:19 -05001704 pqi_dev_info(ctrl_info, "added", device);
Kevin Barnett6c223762016-06-27 16:41:00 -05001705 rc = pqi_add_device(ctrl_info, device);
1706 if (rc) {
1707 dev_warn(&ctrl_info->pci_dev->dev,
1708 "scsi %d:%d:%d:%d addition failed, device not added\n",
1709 ctrl_info->scsi_host->host_no,
1710 device->bus, device->target,
1711 device->lun);
1712 pqi_fixup_botched_add(ctrl_info, device);
Kevin Barnett6c223762016-06-27 16:41:00 -05001713 }
1714 }
Kevin Barnett6c223762016-06-27 16:41:00 -05001715 }
1716}
1717
1718static bool pqi_is_supported_device(struct pqi_scsi_dev *device)
1719{
1720 bool is_supported = false;
1721
1722 switch (device->devtype) {
1723 case TYPE_DISK:
1724 case TYPE_ZBC:
1725 case TYPE_TAPE:
1726 case TYPE_MEDIUM_CHANGER:
1727 case TYPE_ENCLOSURE:
1728 is_supported = true;
1729 break;
1730 case TYPE_RAID:
1731 /*
1732 * Only support the HBA controller itself as a RAID
1733 * controller. If it's a RAID controller other than
Kevin Barnett376fb882017-05-03 18:54:43 -05001734 * the HBA itself (an external RAID controller, for
1735 * example), we don't support it.
Kevin Barnett6c223762016-06-27 16:41:00 -05001736 */
1737 if (pqi_is_hba_lunid(device->scsi3addr))
1738 is_supported = true;
1739 break;
1740 }
1741
1742 return is_supported;
1743}
1744
Kevin Barnett94086f52017-05-03 18:54:31 -05001745static inline bool pqi_skip_device(u8 *scsi3addr)
Kevin Barnett6c223762016-06-27 16:41:00 -05001746{
Kevin Barnett94086f52017-05-03 18:54:31 -05001747 /* Ignore all masked devices. */
1748 if (MASKED_DEVICE(scsi3addr))
Kevin Barnett6c223762016-06-27 16:41:00 -05001749 return true;
Kevin Barnett6c223762016-06-27 16:41:00 -05001750
1751 return false;
1752}
1753
Kevin Barnett6c223762016-06-27 16:41:00 -05001754static int pqi_update_scsi_devices(struct pqi_ctrl_info *ctrl_info)
1755{
1756 int i;
1757 int rc;
1758 struct list_head new_device_list_head;
1759 struct report_phys_lun_extended *physdev_list = NULL;
1760 struct report_log_lun_extended *logdev_list = NULL;
1761 struct report_phys_lun_extended_entry *phys_lun_ext_entry;
1762 struct report_log_lun_extended_entry *log_lun_ext_entry;
1763 struct bmic_identify_physical_device *id_phys = NULL;
1764 u32 num_physicals;
1765 u32 num_logicals;
1766 struct pqi_scsi_dev **new_device_list = NULL;
1767 struct pqi_scsi_dev *device;
1768 struct pqi_scsi_dev *next;
1769 unsigned int num_new_devices;
1770 unsigned int num_valid_devices;
1771 bool is_physical_device;
1772 u8 *scsi3addr;
1773 static char *out_of_memory_msg =
Kevin Barnett6de783f2017-05-03 18:55:19 -05001774 "failed to allocate memory, device discovery stopped";
Kevin Barnett6c223762016-06-27 16:41:00 -05001775
1776 INIT_LIST_HEAD(&new_device_list_head);
1777
1778 rc = pqi_get_device_lists(ctrl_info, &physdev_list, &logdev_list);
1779 if (rc)
1780 goto out;
1781
1782 if (physdev_list)
1783 num_physicals =
1784 get_unaligned_be32(&physdev_list->header.list_length)
1785 / sizeof(physdev_list->lun_entries[0]);
1786 else
1787 num_physicals = 0;
1788
1789 if (logdev_list)
1790 num_logicals =
1791 get_unaligned_be32(&logdev_list->header.list_length)
1792 / sizeof(logdev_list->lun_entries[0]);
1793 else
1794 num_logicals = 0;
1795
1796 if (num_physicals) {
1797 /*
1798 * We need this buffer for calls to pqi_get_physical_disk_info()
1799 * below. We allocate it here instead of inside
1800 * pqi_get_physical_disk_info() because it's a fairly large
1801 * buffer.
1802 */
1803 id_phys = kmalloc(sizeof(*id_phys), GFP_KERNEL);
1804 if (!id_phys) {
1805 dev_warn(&ctrl_info->pci_dev->dev, "%s\n",
1806 out_of_memory_msg);
1807 rc = -ENOMEM;
1808 goto out;
1809 }
1810 }
1811
1812 num_new_devices = num_physicals + num_logicals;
1813
1814 new_device_list = kmalloc(sizeof(*new_device_list) *
1815 num_new_devices, GFP_KERNEL);
1816 if (!new_device_list) {
1817 dev_warn(&ctrl_info->pci_dev->dev, "%s\n", out_of_memory_msg);
1818 rc = -ENOMEM;
1819 goto out;
1820 }
1821
1822 for (i = 0; i < num_new_devices; i++) {
1823 device = kzalloc(sizeof(*device), GFP_KERNEL);
1824 if (!device) {
1825 dev_warn(&ctrl_info->pci_dev->dev, "%s\n",
1826 out_of_memory_msg);
1827 rc = -ENOMEM;
1828 goto out;
1829 }
1830 list_add_tail(&device->new_device_list_entry,
1831 &new_device_list_head);
1832 }
1833
1834 device = NULL;
1835 num_valid_devices = 0;
1836
1837 for (i = 0; i < num_new_devices; i++) {
1838
1839 if (i < num_physicals) {
1840 is_physical_device = true;
1841 phys_lun_ext_entry = &physdev_list->lun_entries[i];
1842 log_lun_ext_entry = NULL;
1843 scsi3addr = phys_lun_ext_entry->lunid;
1844 } else {
1845 is_physical_device = false;
1846 phys_lun_ext_entry = NULL;
1847 log_lun_ext_entry =
1848 &logdev_list->lun_entries[i - num_physicals];
1849 scsi3addr = log_lun_ext_entry->lunid;
1850 }
1851
Kevin Barnett94086f52017-05-03 18:54:31 -05001852 if (is_physical_device && pqi_skip_device(scsi3addr))
Kevin Barnett6c223762016-06-27 16:41:00 -05001853 continue;
1854
1855 if (device)
1856 device = list_next_entry(device, new_device_list_entry);
1857 else
1858 device = list_first_entry(&new_device_list_head,
1859 struct pqi_scsi_dev, new_device_list_entry);
1860
1861 memcpy(device->scsi3addr, scsi3addr, sizeof(device->scsi3addr));
1862 device->is_physical_device = is_physical_device;
Kevin Barnettbd10cf02017-05-03 18:54:12 -05001863 if (!is_physical_device)
1864 device->is_external_raid_device =
1865 pqi_is_external_raid_addr(scsi3addr);
Kevin Barnett6c223762016-06-27 16:41:00 -05001866
1867 /* Gather information about the device. */
1868 rc = pqi_get_device_info(ctrl_info, device);
1869 if (rc == -ENOMEM) {
1870 dev_warn(&ctrl_info->pci_dev->dev, "%s\n",
1871 out_of_memory_msg);
1872 goto out;
1873 }
1874 if (rc) {
Kevin Barnett6de783f2017-05-03 18:55:19 -05001875 if (device->is_physical_device)
1876 dev_warn(&ctrl_info->pci_dev->dev,
1877 "obtaining device info failed, skipping physical device %016llx\n",
1878 get_unaligned_be64(
1879 &phys_lun_ext_entry->wwid));
1880 else
1881 dev_warn(&ctrl_info->pci_dev->dev,
1882 "obtaining device info failed, skipping logical device %08x%08x\n",
1883 *((u32 *)&device->scsi3addr),
1884 *((u32 *)&device->scsi3addr[4]));
Kevin Barnett6c223762016-06-27 16:41:00 -05001885 rc = 0;
1886 continue;
1887 }
1888
1889 if (!pqi_is_supported_device(device))
1890 continue;
1891
1892 pqi_assign_bus_target_lun(device);
1893
Kevin Barnett6c223762016-06-27 16:41:00 -05001894 if (device->is_physical_device) {
1895 device->wwid = phys_lun_ext_entry->wwid;
1896 if ((phys_lun_ext_entry->device_flags &
1897 REPORT_PHYS_LUN_DEV_FLAG_AIO_ENABLED) &&
1898 phys_lun_ext_entry->aio_handle)
1899 device->aio_enabled = true;
1900 } else {
1901 memcpy(device->volume_id, log_lun_ext_entry->volume_id,
1902 sizeof(device->volume_id));
1903 }
1904
1905 switch (device->devtype) {
1906 case TYPE_DISK:
1907 case TYPE_ZBC:
1908 case TYPE_ENCLOSURE:
1909 if (device->is_physical_device) {
1910 device->sas_address =
1911 get_unaligned_be64(&device->wwid);
1912 if (device->devtype == TYPE_DISK ||
1913 device->devtype == TYPE_ZBC) {
1914 device->aio_handle =
1915 phys_lun_ext_entry->aio_handle;
1916 pqi_get_physical_disk_info(ctrl_info,
1917 device, id_phys);
1918 }
1919 }
1920 break;
1921 }
1922
1923 new_device_list[num_valid_devices++] = device;
1924 }
1925
1926 pqi_update_device_list(ctrl_info, new_device_list, num_valid_devices);
1927
1928out:
1929 list_for_each_entry_safe(device, next, &new_device_list_head,
1930 new_device_list_entry) {
1931 if (device->keep_device)
1932 continue;
1933 list_del(&device->new_device_list_entry);
1934 pqi_free_device(device);
1935 }
1936
1937 kfree(new_device_list);
1938 kfree(physdev_list);
1939 kfree(logdev_list);
1940 kfree(id_phys);
1941
1942 return rc;
1943}
1944
1945static void pqi_remove_all_scsi_devices(struct pqi_ctrl_info *ctrl_info)
1946{
1947 unsigned long flags;
1948 struct pqi_scsi_dev *device;
Kevin Barnett6c223762016-06-27 16:41:00 -05001949
Kevin Barnetta37ef742017-05-03 18:52:22 -05001950 while (1) {
1951 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
Kevin Barnett6c223762016-06-27 16:41:00 -05001952
Kevin Barnetta37ef742017-05-03 18:52:22 -05001953 device = list_first_entry_or_null(&ctrl_info->scsi_device_list,
1954 struct pqi_scsi_dev, scsi_device_list_entry);
1955 if (device)
1956 list_del(&device->scsi_device_list_entry);
1957
1958 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock,
1959 flags);
1960
1961 if (!device)
1962 break;
1963
Kevin Barnett6c223762016-06-27 16:41:00 -05001964 if (device->sdev)
1965 pqi_remove_device(ctrl_info, device);
Kevin Barnett6c223762016-06-27 16:41:00 -05001966 pqi_free_device(device);
1967 }
Kevin Barnett6c223762016-06-27 16:41:00 -05001968}
1969
1970static int pqi_scan_scsi_devices(struct pqi_ctrl_info *ctrl_info)
1971{
1972 int rc;
1973
1974 if (pqi_ctrl_offline(ctrl_info))
1975 return -ENXIO;
1976
1977 mutex_lock(&ctrl_info->scan_mutex);
1978
1979 rc = pqi_update_scsi_devices(ctrl_info);
1980 if (rc)
Kevin Barnett5f310422017-05-03 18:54:55 -05001981 pqi_schedule_rescan_worker_delayed(ctrl_info);
Kevin Barnett6c223762016-06-27 16:41:00 -05001982
1983 mutex_unlock(&ctrl_info->scan_mutex);
1984
1985 return rc;
1986}
1987
1988static void pqi_scan_start(struct Scsi_Host *shost)
1989{
1990 pqi_scan_scsi_devices(shost_to_hba(shost));
1991}
1992
1993/* Returns TRUE if scan is finished. */
1994
1995static int pqi_scan_finished(struct Scsi_Host *shost,
1996 unsigned long elapsed_time)
1997{
1998 struct pqi_ctrl_info *ctrl_info;
1999
2000 ctrl_info = shost_priv(shost);
2001
2002 return !mutex_is_locked(&ctrl_info->scan_mutex);
2003}
2004
Kevin Barnett061ef062017-05-03 18:53:05 -05002005static void pqi_wait_until_scan_finished(struct pqi_ctrl_info *ctrl_info)
2006{
2007 mutex_lock(&ctrl_info->scan_mutex);
2008 mutex_unlock(&ctrl_info->scan_mutex);
2009}
2010
2011static void pqi_wait_until_lun_reset_finished(struct pqi_ctrl_info *ctrl_info)
2012{
2013 mutex_lock(&ctrl_info->lun_reset_mutex);
2014 mutex_unlock(&ctrl_info->lun_reset_mutex);
2015}
2016
Kevin Barnett6c223762016-06-27 16:41:00 -05002017static inline void pqi_set_encryption_info(
2018 struct pqi_encryption_info *encryption_info, struct raid_map *raid_map,
2019 u64 first_block)
2020{
2021 u32 volume_blk_size;
2022
2023 /*
2024 * Set the encryption tweak values based on logical block address.
2025 * If the block size is 512, the tweak value is equal to the LBA.
2026 * For other block sizes, tweak value is (LBA * block size) / 512.
2027 */
2028 volume_blk_size = get_unaligned_le32(&raid_map->volume_blk_size);
2029 if (volume_blk_size != 512)
2030 first_block = (first_block * volume_blk_size) / 512;
2031
2032 encryption_info->data_encryption_key_index =
2033 get_unaligned_le16(&raid_map->data_encryption_key_index);
2034 encryption_info->encrypt_tweak_lower = lower_32_bits(first_block);
2035 encryption_info->encrypt_tweak_upper = upper_32_bits(first_block);
2036}
2037
2038/*
Kevin Barnett588a63fe2017-05-03 18:55:25 -05002039 * Attempt to perform RAID bypass mapping for a logical volume I/O.
Kevin Barnett6c223762016-06-27 16:41:00 -05002040 */
2041
2042#define PQI_RAID_BYPASS_INELIGIBLE 1
2043
2044static int pqi_raid_bypass_submit_scsi_cmd(struct pqi_ctrl_info *ctrl_info,
2045 struct pqi_scsi_dev *device, struct scsi_cmnd *scmd,
2046 struct pqi_queue_group *queue_group)
2047{
2048 struct raid_map *raid_map;
2049 bool is_write = false;
2050 u32 map_index;
2051 u64 first_block;
2052 u64 last_block;
2053 u32 block_cnt;
2054 u32 blocks_per_row;
2055 u64 first_row;
2056 u64 last_row;
2057 u32 first_row_offset;
2058 u32 last_row_offset;
2059 u32 first_column;
2060 u32 last_column;
2061 u64 r0_first_row;
2062 u64 r0_last_row;
2063 u32 r5or6_blocks_per_row;
2064 u64 r5or6_first_row;
2065 u64 r5or6_last_row;
2066 u32 r5or6_first_row_offset;
2067 u32 r5or6_last_row_offset;
2068 u32 r5or6_first_column;
2069 u32 r5or6_last_column;
2070 u16 data_disks_per_row;
2071 u32 total_disks_per_row;
2072 u16 layout_map_count;
2073 u32 stripesize;
2074 u16 strip_size;
2075 u32 first_group;
2076 u32 last_group;
2077 u32 current_group;
2078 u32 map_row;
2079 u32 aio_handle;
2080 u64 disk_block;
2081 u32 disk_block_cnt;
2082 u8 cdb[16];
2083 u8 cdb_length;
2084 int offload_to_mirror;
2085 struct pqi_encryption_info *encryption_info_ptr;
2086 struct pqi_encryption_info encryption_info;
2087#if BITS_PER_LONG == 32
2088 u64 tmpdiv;
2089#endif
2090
2091 /* Check for valid opcode, get LBA and block count. */
2092 switch (scmd->cmnd[0]) {
2093 case WRITE_6:
2094 is_write = true;
2095 /* fall through */
2096 case READ_6:
kevin Barnette018ef52016-09-16 15:01:51 -05002097 first_block = (u64)(((scmd->cmnd[1] & 0x1f) << 16) |
2098 (scmd->cmnd[2] << 8) | scmd->cmnd[3]);
Kevin Barnett6c223762016-06-27 16:41:00 -05002099 block_cnt = (u32)scmd->cmnd[4];
2100 if (block_cnt == 0)
2101 block_cnt = 256;
2102 break;
2103 case WRITE_10:
2104 is_write = true;
2105 /* fall through */
2106 case READ_10:
2107 first_block = (u64)get_unaligned_be32(&scmd->cmnd[2]);
2108 block_cnt = (u32)get_unaligned_be16(&scmd->cmnd[7]);
2109 break;
2110 case WRITE_12:
2111 is_write = true;
2112 /* fall through */
2113 case READ_12:
2114 first_block = (u64)get_unaligned_be32(&scmd->cmnd[2]);
2115 block_cnt = get_unaligned_be32(&scmd->cmnd[6]);
2116 break;
2117 case WRITE_16:
2118 is_write = true;
2119 /* fall through */
2120 case READ_16:
2121 first_block = get_unaligned_be64(&scmd->cmnd[2]);
2122 block_cnt = get_unaligned_be32(&scmd->cmnd[10]);
2123 break;
2124 default:
2125 /* Process via normal I/O path. */
2126 return PQI_RAID_BYPASS_INELIGIBLE;
2127 }
2128
2129 /* Check for write to non-RAID-0. */
2130 if (is_write && device->raid_level != SA_RAID_0)
2131 return PQI_RAID_BYPASS_INELIGIBLE;
2132
2133 if (unlikely(block_cnt == 0))
2134 return PQI_RAID_BYPASS_INELIGIBLE;
2135
2136 last_block = first_block + block_cnt - 1;
2137 raid_map = device->raid_map;
2138
2139 /* Check for invalid block or wraparound. */
2140 if (last_block >= get_unaligned_le64(&raid_map->volume_blk_cnt) ||
2141 last_block < first_block)
2142 return PQI_RAID_BYPASS_INELIGIBLE;
2143
2144 data_disks_per_row = get_unaligned_le16(&raid_map->data_disks_per_row);
2145 strip_size = get_unaligned_le16(&raid_map->strip_size);
2146 layout_map_count = get_unaligned_le16(&raid_map->layout_map_count);
2147
2148 /* Calculate stripe information for the request. */
2149 blocks_per_row = data_disks_per_row * strip_size;
2150#if BITS_PER_LONG == 32
2151 tmpdiv = first_block;
2152 do_div(tmpdiv, blocks_per_row);
2153 first_row = tmpdiv;
2154 tmpdiv = last_block;
2155 do_div(tmpdiv, blocks_per_row);
2156 last_row = tmpdiv;
2157 first_row_offset = (u32)(first_block - (first_row * blocks_per_row));
2158 last_row_offset = (u32)(last_block - (last_row * blocks_per_row));
2159 tmpdiv = first_row_offset;
2160 do_div(tmpdiv, strip_size);
2161 first_column = tmpdiv;
2162 tmpdiv = last_row_offset;
2163 do_div(tmpdiv, strip_size);
2164 last_column = tmpdiv;
2165#else
2166 first_row = first_block / blocks_per_row;
2167 last_row = last_block / blocks_per_row;
2168 first_row_offset = (u32)(first_block - (first_row * blocks_per_row));
2169 last_row_offset = (u32)(last_block - (last_row * blocks_per_row));
2170 first_column = first_row_offset / strip_size;
2171 last_column = last_row_offset / strip_size;
2172#endif
2173
2174 /* If this isn't a single row/column then give to the controller. */
2175 if (first_row != last_row || first_column != last_column)
2176 return PQI_RAID_BYPASS_INELIGIBLE;
2177
2178 /* Proceeding with driver mapping. */
2179 total_disks_per_row = data_disks_per_row +
2180 get_unaligned_le16(&raid_map->metadata_disks_per_row);
2181 map_row = ((u32)(first_row >> raid_map->parity_rotation_shift)) %
2182 get_unaligned_le16(&raid_map->row_cnt);
2183 map_index = (map_row * total_disks_per_row) + first_column;
2184
2185 /* RAID 1 */
2186 if (device->raid_level == SA_RAID_1) {
2187 if (device->offload_to_mirror)
2188 map_index += data_disks_per_row;
2189 device->offload_to_mirror = !device->offload_to_mirror;
2190 } else if (device->raid_level == SA_RAID_ADM) {
2191 /* RAID ADM */
2192 /*
2193 * Handles N-way mirrors (R1-ADM) and R10 with # of drives
2194 * divisible by 3.
2195 */
2196 offload_to_mirror = device->offload_to_mirror;
2197 if (offload_to_mirror == 0) {
2198 /* use physical disk in the first mirrored group. */
2199 map_index %= data_disks_per_row;
2200 } else {
2201 do {
2202 /*
2203 * Determine mirror group that map_index
2204 * indicates.
2205 */
2206 current_group = map_index / data_disks_per_row;
2207
2208 if (offload_to_mirror != current_group) {
2209 if (current_group <
2210 layout_map_count - 1) {
2211 /*
2212 * Select raid index from
2213 * next group.
2214 */
2215 map_index += data_disks_per_row;
2216 current_group++;
2217 } else {
2218 /*
2219 * Select raid index from first
2220 * group.
2221 */
2222 map_index %= data_disks_per_row;
2223 current_group = 0;
2224 }
2225 }
2226 } while (offload_to_mirror != current_group);
2227 }
2228
2229 /* Set mirror group to use next time. */
2230 offload_to_mirror =
2231 (offload_to_mirror >= layout_map_count - 1) ?
2232 0 : offload_to_mirror + 1;
2233 WARN_ON(offload_to_mirror >= layout_map_count);
2234 device->offload_to_mirror = offload_to_mirror;
2235 /*
2236 * Avoid direct use of device->offload_to_mirror within this
2237 * function since multiple threads might simultaneously
2238 * increment it beyond the range of device->layout_map_count -1.
2239 */
2240 } else if ((device->raid_level == SA_RAID_5 ||
2241 device->raid_level == SA_RAID_6) && layout_map_count > 1) {
2242 /* RAID 50/60 */
2243 /* Verify first and last block are in same RAID group */
2244 r5or6_blocks_per_row = strip_size * data_disks_per_row;
2245 stripesize = r5or6_blocks_per_row * layout_map_count;
2246#if BITS_PER_LONG == 32
2247 tmpdiv = first_block;
2248 first_group = do_div(tmpdiv, stripesize);
2249 tmpdiv = first_group;
2250 do_div(tmpdiv, r5or6_blocks_per_row);
2251 first_group = tmpdiv;
2252 tmpdiv = last_block;
2253 last_group = do_div(tmpdiv, stripesize);
2254 tmpdiv = last_group;
2255 do_div(tmpdiv, r5or6_blocks_per_row);
2256 last_group = tmpdiv;
2257#else
2258 first_group = (first_block % stripesize) / r5or6_blocks_per_row;
2259 last_group = (last_block % stripesize) / r5or6_blocks_per_row;
2260#endif
2261 if (first_group != last_group)
2262 return PQI_RAID_BYPASS_INELIGIBLE;
2263
2264 /* Verify request is in a single row of RAID 5/6 */
2265#if BITS_PER_LONG == 32
2266 tmpdiv = first_block;
2267 do_div(tmpdiv, stripesize);
2268 first_row = r5or6_first_row = r0_first_row = tmpdiv;
2269 tmpdiv = last_block;
2270 do_div(tmpdiv, stripesize);
2271 r5or6_last_row = r0_last_row = tmpdiv;
2272#else
2273 first_row = r5or6_first_row = r0_first_row =
2274 first_block / stripesize;
2275 r5or6_last_row = r0_last_row = last_block / stripesize;
2276#endif
2277 if (r5or6_first_row != r5or6_last_row)
2278 return PQI_RAID_BYPASS_INELIGIBLE;
2279
2280 /* Verify request is in a single column */
2281#if BITS_PER_LONG == 32
2282 tmpdiv = first_block;
2283 first_row_offset = do_div(tmpdiv, stripesize);
2284 tmpdiv = first_row_offset;
2285 first_row_offset = (u32)do_div(tmpdiv, r5or6_blocks_per_row);
2286 r5or6_first_row_offset = first_row_offset;
2287 tmpdiv = last_block;
2288 r5or6_last_row_offset = do_div(tmpdiv, stripesize);
2289 tmpdiv = r5or6_last_row_offset;
2290 r5or6_last_row_offset = do_div(tmpdiv, r5or6_blocks_per_row);
2291 tmpdiv = r5or6_first_row_offset;
2292 do_div(tmpdiv, strip_size);
2293 first_column = r5or6_first_column = tmpdiv;
2294 tmpdiv = r5or6_last_row_offset;
2295 do_div(tmpdiv, strip_size);
2296 r5or6_last_column = tmpdiv;
2297#else
2298 first_row_offset = r5or6_first_row_offset =
2299 (u32)((first_block % stripesize) %
2300 r5or6_blocks_per_row);
2301
2302 r5or6_last_row_offset =
2303 (u32)((last_block % stripesize) %
2304 r5or6_blocks_per_row);
2305
2306 first_column = r5or6_first_row_offset / strip_size;
2307 r5or6_first_column = first_column;
2308 r5or6_last_column = r5or6_last_row_offset / strip_size;
2309#endif
2310 if (r5or6_first_column != r5or6_last_column)
2311 return PQI_RAID_BYPASS_INELIGIBLE;
2312
2313 /* Request is eligible */
2314 map_row =
2315 ((u32)(first_row >> raid_map->parity_rotation_shift)) %
2316 get_unaligned_le16(&raid_map->row_cnt);
2317
2318 map_index = (first_group *
2319 (get_unaligned_le16(&raid_map->row_cnt) *
2320 total_disks_per_row)) +
2321 (map_row * total_disks_per_row) + first_column;
2322 }
2323
2324 if (unlikely(map_index >= RAID_MAP_MAX_ENTRIES))
2325 return PQI_RAID_BYPASS_INELIGIBLE;
2326
2327 aio_handle = raid_map->disk_data[map_index].aio_handle;
2328 disk_block = get_unaligned_le64(&raid_map->disk_starting_blk) +
2329 first_row * strip_size +
2330 (first_row_offset - first_column * strip_size);
2331 disk_block_cnt = block_cnt;
2332
2333 /* Handle differing logical/physical block sizes. */
2334 if (raid_map->phys_blk_shift) {
2335 disk_block <<= raid_map->phys_blk_shift;
2336 disk_block_cnt <<= raid_map->phys_blk_shift;
2337 }
2338
2339 if (unlikely(disk_block_cnt > 0xffff))
2340 return PQI_RAID_BYPASS_INELIGIBLE;
2341
2342 /* Build the new CDB for the physical disk I/O. */
2343 if (disk_block > 0xffffffff) {
2344 cdb[0] = is_write ? WRITE_16 : READ_16;
2345 cdb[1] = 0;
2346 put_unaligned_be64(disk_block, &cdb[2]);
2347 put_unaligned_be32(disk_block_cnt, &cdb[10]);
2348 cdb[14] = 0;
2349 cdb[15] = 0;
2350 cdb_length = 16;
2351 } else {
2352 cdb[0] = is_write ? WRITE_10 : READ_10;
2353 cdb[1] = 0;
2354 put_unaligned_be32((u32)disk_block, &cdb[2]);
2355 cdb[6] = 0;
2356 put_unaligned_be16((u16)disk_block_cnt, &cdb[7]);
2357 cdb[9] = 0;
2358 cdb_length = 10;
2359 }
2360
2361 if (get_unaligned_le16(&raid_map->flags) &
2362 RAID_MAP_ENCRYPTION_ENABLED) {
2363 pqi_set_encryption_info(&encryption_info, raid_map,
2364 first_block);
2365 encryption_info_ptr = &encryption_info;
2366 } else {
2367 encryption_info_ptr = NULL;
2368 }
2369
2370 return pqi_aio_submit_io(ctrl_info, scmd, aio_handle,
Kevin Barnett376fb882017-05-03 18:54:43 -05002371 cdb, cdb_length, queue_group, encryption_info_ptr, true);
Kevin Barnett6c223762016-06-27 16:41:00 -05002372}
2373
2374#define PQI_STATUS_IDLE 0x0
2375
2376#define PQI_CREATE_ADMIN_QUEUE_PAIR 1
2377#define PQI_DELETE_ADMIN_QUEUE_PAIR 2
2378
2379#define PQI_DEVICE_STATE_POWER_ON_AND_RESET 0x0
2380#define PQI_DEVICE_STATE_STATUS_AVAILABLE 0x1
2381#define PQI_DEVICE_STATE_ALL_REGISTERS_READY 0x2
2382#define PQI_DEVICE_STATE_ADMIN_QUEUE_PAIR_READY 0x3
2383#define PQI_DEVICE_STATE_ERROR 0x4
2384
2385#define PQI_MODE_READY_TIMEOUT_SECS 30
2386#define PQI_MODE_READY_POLL_INTERVAL_MSECS 1
2387
2388static int pqi_wait_for_pqi_mode_ready(struct pqi_ctrl_info *ctrl_info)
2389{
2390 struct pqi_device_registers __iomem *pqi_registers;
2391 unsigned long timeout;
2392 u64 signature;
2393 u8 status;
2394
2395 pqi_registers = ctrl_info->pqi_registers;
2396 timeout = (PQI_MODE_READY_TIMEOUT_SECS * HZ) + jiffies;
2397
2398 while (1) {
2399 signature = readq(&pqi_registers->signature);
2400 if (memcmp(&signature, PQI_DEVICE_SIGNATURE,
2401 sizeof(signature)) == 0)
2402 break;
2403 if (time_after(jiffies, timeout)) {
2404 dev_err(&ctrl_info->pci_dev->dev,
2405 "timed out waiting for PQI signature\n");
2406 return -ETIMEDOUT;
2407 }
2408 msleep(PQI_MODE_READY_POLL_INTERVAL_MSECS);
2409 }
2410
2411 while (1) {
2412 status = readb(&pqi_registers->function_and_status_code);
2413 if (status == PQI_STATUS_IDLE)
2414 break;
2415 if (time_after(jiffies, timeout)) {
2416 dev_err(&ctrl_info->pci_dev->dev,
2417 "timed out waiting for PQI IDLE\n");
2418 return -ETIMEDOUT;
2419 }
2420 msleep(PQI_MODE_READY_POLL_INTERVAL_MSECS);
2421 }
2422
2423 while (1) {
2424 if (readl(&pqi_registers->device_status) ==
2425 PQI_DEVICE_STATE_ALL_REGISTERS_READY)
2426 break;
2427 if (time_after(jiffies, timeout)) {
2428 dev_err(&ctrl_info->pci_dev->dev,
2429 "timed out waiting for PQI all registers ready\n");
2430 return -ETIMEDOUT;
2431 }
2432 msleep(PQI_MODE_READY_POLL_INTERVAL_MSECS);
2433 }
2434
2435 return 0;
2436}
2437
2438static inline void pqi_aio_path_disabled(struct pqi_io_request *io_request)
2439{
2440 struct pqi_scsi_dev *device;
2441
2442 device = io_request->scmd->device->hostdata;
Kevin Barnett588a63fe2017-05-03 18:55:25 -05002443 device->raid_bypass_enabled = false;
Kevin Barnett376fb882017-05-03 18:54:43 -05002444 device->aio_enabled = false;
Kevin Barnett6c223762016-06-27 16:41:00 -05002445}
2446
Kevin Barnettd87d5472017-05-03 18:54:00 -05002447static inline void pqi_take_device_offline(struct scsi_device *sdev, char *path)
Kevin Barnett6c223762016-06-27 16:41:00 -05002448{
2449 struct pqi_ctrl_info *ctrl_info;
Kevin Barnette58081a2016-08-31 14:54:29 -05002450 struct pqi_scsi_dev *device;
Kevin Barnett6c223762016-06-27 16:41:00 -05002451
Kevin Barnett03b288cf2017-05-03 18:54:49 -05002452 device = sdev->hostdata;
2453 if (device->device_offline)
2454 return;
2455
2456 device->device_offline = true;
2457 scsi_device_set_state(sdev, SDEV_OFFLINE);
2458 ctrl_info = shost_to_hba(sdev->host);
2459 pqi_schedule_rescan_worker(ctrl_info);
2460 dev_err(&ctrl_info->pci_dev->dev, "offlined %s scsi %d:%d:%d:%d\n",
2461 path, ctrl_info->scsi_host->host_no, device->bus,
2462 device->target, device->lun);
Kevin Barnett6c223762016-06-27 16:41:00 -05002463}
2464
2465static void pqi_process_raid_io_error(struct pqi_io_request *io_request)
2466{
2467 u8 scsi_status;
2468 u8 host_byte;
2469 struct scsi_cmnd *scmd;
2470 struct pqi_raid_error_info *error_info;
2471 size_t sense_data_length;
2472 int residual_count;
2473 int xfer_count;
2474 struct scsi_sense_hdr sshdr;
2475
2476 scmd = io_request->scmd;
2477 if (!scmd)
2478 return;
2479
2480 error_info = io_request->error_info;
2481 scsi_status = error_info->status;
2482 host_byte = DID_OK;
2483
Kevin Barnettf5b63202017-05-03 18:55:07 -05002484 switch (error_info->data_out_result) {
2485 case PQI_DATA_IN_OUT_GOOD:
2486 break;
2487 case PQI_DATA_IN_OUT_UNDERFLOW:
Kevin Barnett6c223762016-06-27 16:41:00 -05002488 xfer_count =
2489 get_unaligned_le32(&error_info->data_out_transferred);
2490 residual_count = scsi_bufflen(scmd) - xfer_count;
2491 scsi_set_resid(scmd, residual_count);
2492 if (xfer_count < scmd->underflow)
2493 host_byte = DID_SOFT_ERROR;
Kevin Barnettf5b63202017-05-03 18:55:07 -05002494 break;
2495 case PQI_DATA_IN_OUT_UNSOLICITED_ABORT:
2496 case PQI_DATA_IN_OUT_ABORTED:
2497 host_byte = DID_ABORT;
2498 break;
2499 case PQI_DATA_IN_OUT_TIMEOUT:
2500 host_byte = DID_TIME_OUT;
2501 break;
2502 case PQI_DATA_IN_OUT_BUFFER_OVERFLOW:
2503 case PQI_DATA_IN_OUT_PROTOCOL_ERROR:
2504 case PQI_DATA_IN_OUT_BUFFER_ERROR:
2505 case PQI_DATA_IN_OUT_BUFFER_OVERFLOW_DESCRIPTOR_AREA:
2506 case PQI_DATA_IN_OUT_BUFFER_OVERFLOW_BRIDGE:
2507 case PQI_DATA_IN_OUT_ERROR:
2508 case PQI_DATA_IN_OUT_HARDWARE_ERROR:
2509 case PQI_DATA_IN_OUT_PCIE_FABRIC_ERROR:
2510 case PQI_DATA_IN_OUT_PCIE_COMPLETION_TIMEOUT:
2511 case PQI_DATA_IN_OUT_PCIE_COMPLETER_ABORT_RECEIVED:
2512 case PQI_DATA_IN_OUT_PCIE_UNSUPPORTED_REQUEST_RECEIVED:
2513 case PQI_DATA_IN_OUT_PCIE_ECRC_CHECK_FAILED:
2514 case PQI_DATA_IN_OUT_PCIE_UNSUPPORTED_REQUEST:
2515 case PQI_DATA_IN_OUT_PCIE_ACS_VIOLATION:
2516 case PQI_DATA_IN_OUT_PCIE_TLP_PREFIX_BLOCKED:
2517 case PQI_DATA_IN_OUT_PCIE_POISONED_MEMORY_READ:
2518 default:
2519 host_byte = DID_ERROR;
2520 break;
Kevin Barnett6c223762016-06-27 16:41:00 -05002521 }
2522
2523 sense_data_length = get_unaligned_le16(&error_info->sense_data_length);
2524 if (sense_data_length == 0)
2525 sense_data_length =
2526 get_unaligned_le16(&error_info->response_data_length);
2527 if (sense_data_length) {
2528 if (sense_data_length > sizeof(error_info->data))
2529 sense_data_length = sizeof(error_info->data);
2530
2531 if (scsi_status == SAM_STAT_CHECK_CONDITION &&
2532 scsi_normalize_sense(error_info->data,
2533 sense_data_length, &sshdr) &&
2534 sshdr.sense_key == HARDWARE_ERROR &&
2535 sshdr.asc == 0x3e &&
2536 sshdr.ascq == 0x1) {
Kevin Barnettd87d5472017-05-03 18:54:00 -05002537 pqi_take_device_offline(scmd->device, "RAID");
Kevin Barnett6c223762016-06-27 16:41:00 -05002538 host_byte = DID_NO_CONNECT;
2539 }
2540
2541 if (sense_data_length > SCSI_SENSE_BUFFERSIZE)
2542 sense_data_length = SCSI_SENSE_BUFFERSIZE;
2543 memcpy(scmd->sense_buffer, error_info->data,
2544 sense_data_length);
2545 }
2546
2547 scmd->result = scsi_status;
2548 set_host_byte(scmd, host_byte);
2549}
2550
2551static void pqi_process_aio_io_error(struct pqi_io_request *io_request)
2552{
2553 u8 scsi_status;
2554 u8 host_byte;
2555 struct scsi_cmnd *scmd;
2556 struct pqi_aio_error_info *error_info;
2557 size_t sense_data_length;
2558 int residual_count;
2559 int xfer_count;
2560 bool device_offline;
2561
2562 scmd = io_request->scmd;
2563 error_info = io_request->error_info;
2564 host_byte = DID_OK;
2565 sense_data_length = 0;
2566 device_offline = false;
2567
2568 switch (error_info->service_response) {
2569 case PQI_AIO_SERV_RESPONSE_COMPLETE:
2570 scsi_status = error_info->status;
2571 break;
2572 case PQI_AIO_SERV_RESPONSE_FAILURE:
2573 switch (error_info->status) {
2574 case PQI_AIO_STATUS_IO_ABORTED:
2575 scsi_status = SAM_STAT_TASK_ABORTED;
2576 break;
2577 case PQI_AIO_STATUS_UNDERRUN:
2578 scsi_status = SAM_STAT_GOOD;
2579 residual_count = get_unaligned_le32(
2580 &error_info->residual_count);
2581 scsi_set_resid(scmd, residual_count);
2582 xfer_count = scsi_bufflen(scmd) - residual_count;
2583 if (xfer_count < scmd->underflow)
2584 host_byte = DID_SOFT_ERROR;
2585 break;
2586 case PQI_AIO_STATUS_OVERRUN:
2587 scsi_status = SAM_STAT_GOOD;
2588 break;
2589 case PQI_AIO_STATUS_AIO_PATH_DISABLED:
2590 pqi_aio_path_disabled(io_request);
2591 scsi_status = SAM_STAT_GOOD;
2592 io_request->status = -EAGAIN;
2593 break;
2594 case PQI_AIO_STATUS_NO_PATH_TO_DEVICE:
2595 case PQI_AIO_STATUS_INVALID_DEVICE:
Kevin Barnett376fb882017-05-03 18:54:43 -05002596 if (!io_request->raid_bypass) {
2597 device_offline = true;
2598 pqi_take_device_offline(scmd->device, "AIO");
2599 host_byte = DID_NO_CONNECT;
2600 }
Kevin Barnett6c223762016-06-27 16:41:00 -05002601 scsi_status = SAM_STAT_CHECK_CONDITION;
2602 break;
2603 case PQI_AIO_STATUS_IO_ERROR:
2604 default:
2605 scsi_status = SAM_STAT_CHECK_CONDITION;
2606 break;
2607 }
2608 break;
2609 case PQI_AIO_SERV_RESPONSE_TMF_COMPLETE:
2610 case PQI_AIO_SERV_RESPONSE_TMF_SUCCEEDED:
2611 scsi_status = SAM_STAT_GOOD;
2612 break;
2613 case PQI_AIO_SERV_RESPONSE_TMF_REJECTED:
2614 case PQI_AIO_SERV_RESPONSE_TMF_INCORRECT_LUN:
2615 default:
2616 scsi_status = SAM_STAT_CHECK_CONDITION;
2617 break;
2618 }
2619
2620 if (error_info->data_present) {
2621 sense_data_length =
2622 get_unaligned_le16(&error_info->data_length);
2623 if (sense_data_length) {
2624 if (sense_data_length > sizeof(error_info->data))
2625 sense_data_length = sizeof(error_info->data);
2626 if (sense_data_length > SCSI_SENSE_BUFFERSIZE)
2627 sense_data_length = SCSI_SENSE_BUFFERSIZE;
2628 memcpy(scmd->sense_buffer, error_info->data,
2629 sense_data_length);
2630 }
2631 }
2632
2633 if (device_offline && sense_data_length == 0)
2634 scsi_build_sense_buffer(0, scmd->sense_buffer, HARDWARE_ERROR,
2635 0x3e, 0x1);
2636
2637 scmd->result = scsi_status;
2638 set_host_byte(scmd, host_byte);
2639}
2640
2641static void pqi_process_io_error(unsigned int iu_type,
2642 struct pqi_io_request *io_request)
2643{
2644 switch (iu_type) {
2645 case PQI_RESPONSE_IU_RAID_PATH_IO_ERROR:
2646 pqi_process_raid_io_error(io_request);
2647 break;
2648 case PQI_RESPONSE_IU_AIO_PATH_IO_ERROR:
2649 pqi_process_aio_io_error(io_request);
2650 break;
2651 }
2652}
2653
2654static int pqi_interpret_task_management_response(
2655 struct pqi_task_management_response *response)
2656{
2657 int rc;
2658
2659 switch (response->response_code) {
Kevin Barnettb17f0482016-08-31 14:54:17 -05002660 case SOP_TMF_COMPLETE:
2661 case SOP_TMF_FUNCTION_SUCCEEDED:
Kevin Barnett6c223762016-06-27 16:41:00 -05002662 rc = 0;
2663 break;
2664 default:
2665 rc = -EIO;
2666 break;
2667 }
2668
2669 return rc;
2670}
2671
2672static unsigned int pqi_process_io_intr(struct pqi_ctrl_info *ctrl_info,
2673 struct pqi_queue_group *queue_group)
2674{
2675 unsigned int num_responses;
2676 pqi_index_t oq_pi;
2677 pqi_index_t oq_ci;
2678 struct pqi_io_request *io_request;
2679 struct pqi_io_response *response;
2680 u16 request_id;
2681
2682 num_responses = 0;
2683 oq_ci = queue_group->oq_ci_copy;
2684
2685 while (1) {
2686 oq_pi = *queue_group->oq_pi;
2687 if (oq_pi == oq_ci)
2688 break;
2689
2690 num_responses++;
2691 response = queue_group->oq_element_array +
2692 (oq_ci * PQI_OPERATIONAL_OQ_ELEMENT_LENGTH);
2693
2694 request_id = get_unaligned_le16(&response->request_id);
2695 WARN_ON(request_id >= ctrl_info->max_io_slots);
2696
2697 io_request = &ctrl_info->io_request_pool[request_id];
2698 WARN_ON(atomic_read(&io_request->refcount) == 0);
2699
2700 switch (response->header.iu_type) {
2701 case PQI_RESPONSE_IU_RAID_PATH_IO_SUCCESS:
2702 case PQI_RESPONSE_IU_AIO_PATH_IO_SUCCESS:
2703 case PQI_RESPONSE_IU_GENERAL_MANAGEMENT:
2704 break;
2705 case PQI_RESPONSE_IU_TASK_MANAGEMENT:
2706 io_request->status =
2707 pqi_interpret_task_management_response(
2708 (void *)response);
2709 break;
2710 case PQI_RESPONSE_IU_AIO_PATH_DISABLED:
2711 pqi_aio_path_disabled(io_request);
2712 io_request->status = -EAGAIN;
2713 break;
2714 case PQI_RESPONSE_IU_RAID_PATH_IO_ERROR:
2715 case PQI_RESPONSE_IU_AIO_PATH_IO_ERROR:
2716 io_request->error_info = ctrl_info->error_buffer +
2717 (get_unaligned_le16(&response->error_index) *
2718 PQI_ERROR_BUFFER_ELEMENT_LENGTH);
2719 pqi_process_io_error(response->header.iu_type,
2720 io_request);
2721 break;
2722 default:
2723 dev_err(&ctrl_info->pci_dev->dev,
2724 "unexpected IU type: 0x%x\n",
2725 response->header.iu_type);
Kevin Barnett6c223762016-06-27 16:41:00 -05002726 break;
2727 }
2728
2729 io_request->io_complete_callback(io_request,
2730 io_request->context);
2731
2732 /*
2733 * Note that the I/O request structure CANNOT BE TOUCHED after
2734 * returning from the I/O completion callback!
2735 */
2736
2737 oq_ci = (oq_ci + 1) % ctrl_info->num_elements_per_oq;
2738 }
2739
2740 if (num_responses) {
2741 queue_group->oq_ci_copy = oq_ci;
2742 writel(oq_ci, queue_group->oq_ci);
2743 }
2744
2745 return num_responses;
2746}
2747
2748static inline unsigned int pqi_num_elements_free(unsigned int pi,
Kevin Barnettdf7a1fc2016-08-31 14:54:59 -05002749 unsigned int ci, unsigned int elements_in_queue)
Kevin Barnett6c223762016-06-27 16:41:00 -05002750{
2751 unsigned int num_elements_used;
2752
2753 if (pi >= ci)
2754 num_elements_used = pi - ci;
2755 else
2756 num_elements_used = elements_in_queue - ci + pi;
2757
2758 return elements_in_queue - num_elements_used - 1;
2759}
2760
Kevin Barnett98f87662017-05-03 18:53:11 -05002761static void pqi_send_event_ack(struct pqi_ctrl_info *ctrl_info,
Kevin Barnett6c223762016-06-27 16:41:00 -05002762 struct pqi_event_acknowledge_request *iu, size_t iu_length)
2763{
2764 pqi_index_t iq_pi;
2765 pqi_index_t iq_ci;
2766 unsigned long flags;
2767 void *next_element;
Kevin Barnett6c223762016-06-27 16:41:00 -05002768 struct pqi_queue_group *queue_group;
2769
2770 queue_group = &ctrl_info->queue_groups[PQI_DEFAULT_QUEUE_GROUP];
2771 put_unaligned_le16(queue_group->oq_id, &iu->header.response_queue_id);
2772
Kevin Barnett6c223762016-06-27 16:41:00 -05002773 while (1) {
2774 spin_lock_irqsave(&queue_group->submit_lock[RAID_PATH], flags);
2775
2776 iq_pi = queue_group->iq_pi_copy[RAID_PATH];
2777 iq_ci = *queue_group->iq_ci[RAID_PATH];
2778
2779 if (pqi_num_elements_free(iq_pi, iq_ci,
2780 ctrl_info->num_elements_per_iq))
2781 break;
2782
2783 spin_unlock_irqrestore(
2784 &queue_group->submit_lock[RAID_PATH], flags);
2785
Kevin Barnett98f87662017-05-03 18:53:11 -05002786 if (pqi_ctrl_offline(ctrl_info))
Kevin Barnett6c223762016-06-27 16:41:00 -05002787 return;
Kevin Barnett6c223762016-06-27 16:41:00 -05002788 }
2789
2790 next_element = queue_group->iq_element_array[RAID_PATH] +
2791 (iq_pi * PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
2792
2793 memcpy(next_element, iu, iu_length);
2794
2795 iq_pi = (iq_pi + 1) % ctrl_info->num_elements_per_iq;
Kevin Barnett6c223762016-06-27 16:41:00 -05002796 queue_group->iq_pi_copy[RAID_PATH] = iq_pi;
2797
2798 /*
2799 * This write notifies the controller that an IU is available to be
2800 * processed.
2801 */
2802 writel(iq_pi, queue_group->iq_pi[RAID_PATH]);
2803
2804 spin_unlock_irqrestore(&queue_group->submit_lock[RAID_PATH], flags);
Kevin Barnett6c223762016-06-27 16:41:00 -05002805}
2806
2807static void pqi_acknowledge_event(struct pqi_ctrl_info *ctrl_info,
2808 struct pqi_event *event)
2809{
2810 struct pqi_event_acknowledge_request request;
2811
2812 memset(&request, 0, sizeof(request));
2813
2814 request.header.iu_type = PQI_REQUEST_IU_ACKNOWLEDGE_VENDOR_EVENT;
2815 put_unaligned_le16(sizeof(request) - PQI_REQUEST_HEADER_LENGTH,
2816 &request.header.iu_length);
2817 request.event_type = event->event_type;
2818 request.event_id = event->event_id;
2819 request.additional_event_id = event->additional_event_id;
2820
Kevin Barnett98f87662017-05-03 18:53:11 -05002821 pqi_send_event_ack(ctrl_info, &request, sizeof(request));
Kevin Barnett6c223762016-06-27 16:41:00 -05002822}
2823
2824static void pqi_event_worker(struct work_struct *work)
2825{
2826 unsigned int i;
2827 struct pqi_ctrl_info *ctrl_info;
Kevin Barnett6a50d6a2017-05-03 18:52:52 -05002828 struct pqi_event *event;
Kevin Barnett6c223762016-06-27 16:41:00 -05002829
2830 ctrl_info = container_of(work, struct pqi_ctrl_info, event_work);
2831
Kevin Barnett7561a7e2017-05-03 18:52:58 -05002832 pqi_ctrl_busy(ctrl_info);
2833 pqi_wait_if_ctrl_blocked(ctrl_info, NO_TIMEOUT);
Kevin Barnett5f310422017-05-03 18:54:55 -05002834 if (pqi_ctrl_offline(ctrl_info))
2835 goto out;
2836
2837 pqi_schedule_rescan_worker_delayed(ctrl_info);
Kevin Barnett7561a7e2017-05-03 18:52:58 -05002838
Kevin Barnett6a50d6a2017-05-03 18:52:52 -05002839 event = ctrl_info->events;
Kevin Barnett6c223762016-06-27 16:41:00 -05002840 for (i = 0; i < PQI_NUM_SUPPORTED_EVENTS; i++) {
Kevin Barnett6a50d6a2017-05-03 18:52:52 -05002841 if (event->pending) {
2842 event->pending = false;
2843 pqi_acknowledge_event(ctrl_info, event);
Kevin Barnett6c223762016-06-27 16:41:00 -05002844 }
Kevin Barnett6a50d6a2017-05-03 18:52:52 -05002845 event++;
Kevin Barnett6c223762016-06-27 16:41:00 -05002846 }
2847
Kevin Barnett5f310422017-05-03 18:54:55 -05002848out:
Kevin Barnett7561a7e2017-05-03 18:52:58 -05002849 pqi_ctrl_unbusy(ctrl_info);
Kevin Barnett6c223762016-06-27 16:41:00 -05002850}
2851
Kevin Barnett98f87662017-05-03 18:53:11 -05002852#define PQI_HEARTBEAT_TIMER_INTERVAL (10 * HZ)
Kevin Barnett6c223762016-06-27 16:41:00 -05002853
2854static void pqi_heartbeat_timer_handler(unsigned long data)
2855{
2856 int num_interrupts;
Kevin Barnett98f87662017-05-03 18:53:11 -05002857 u32 heartbeat_count;
Kevin Barnett6c223762016-06-27 16:41:00 -05002858 struct pqi_ctrl_info *ctrl_info = (struct pqi_ctrl_info *)data;
2859
Kevin Barnett98f87662017-05-03 18:53:11 -05002860 pqi_check_ctrl_health(ctrl_info);
2861 if (pqi_ctrl_offline(ctrl_info))
Kevin Barnett061ef062017-05-03 18:53:05 -05002862 return;
2863
Kevin Barnett6c223762016-06-27 16:41:00 -05002864 num_interrupts = atomic_read(&ctrl_info->num_interrupts);
Kevin Barnett98f87662017-05-03 18:53:11 -05002865 heartbeat_count = pqi_read_heartbeat_counter(ctrl_info);
Kevin Barnett6c223762016-06-27 16:41:00 -05002866
2867 if (num_interrupts == ctrl_info->previous_num_interrupts) {
Kevin Barnett98f87662017-05-03 18:53:11 -05002868 if (heartbeat_count == ctrl_info->previous_heartbeat_count) {
2869 dev_err(&ctrl_info->pci_dev->dev,
2870 "no heartbeat detected - last heartbeat count: %u\n",
2871 heartbeat_count);
Kevin Barnett6c223762016-06-27 16:41:00 -05002872 pqi_take_ctrl_offline(ctrl_info);
2873 return;
2874 }
Kevin Barnett6c223762016-06-27 16:41:00 -05002875 } else {
Kevin Barnett98f87662017-05-03 18:53:11 -05002876 ctrl_info->previous_num_interrupts = num_interrupts;
Kevin Barnett6c223762016-06-27 16:41:00 -05002877 }
2878
Kevin Barnett98f87662017-05-03 18:53:11 -05002879 ctrl_info->previous_heartbeat_count = heartbeat_count;
Kevin Barnett6c223762016-06-27 16:41:00 -05002880 mod_timer(&ctrl_info->heartbeat_timer,
2881 jiffies + PQI_HEARTBEAT_TIMER_INTERVAL);
2882}
2883
2884static void pqi_start_heartbeat_timer(struct pqi_ctrl_info *ctrl_info)
2885{
Kevin Barnett98f87662017-05-03 18:53:11 -05002886 if (!ctrl_info->heartbeat_counter)
2887 return;
2888
Kevin Barnett6c223762016-06-27 16:41:00 -05002889 ctrl_info->previous_num_interrupts =
2890 atomic_read(&ctrl_info->num_interrupts);
Kevin Barnett98f87662017-05-03 18:53:11 -05002891 ctrl_info->previous_heartbeat_count =
2892 pqi_read_heartbeat_counter(ctrl_info);
Kevin Barnett6c223762016-06-27 16:41:00 -05002893
Kevin Barnett6c223762016-06-27 16:41:00 -05002894 ctrl_info->heartbeat_timer.expires =
2895 jiffies + PQI_HEARTBEAT_TIMER_INTERVAL;
2896 ctrl_info->heartbeat_timer.data = (unsigned long)ctrl_info;
2897 ctrl_info->heartbeat_timer.function = pqi_heartbeat_timer_handler;
Kevin Barnett061ef062017-05-03 18:53:05 -05002898 add_timer(&ctrl_info->heartbeat_timer);
Kevin Barnett6c223762016-06-27 16:41:00 -05002899}
2900
2901static inline void pqi_stop_heartbeat_timer(struct pqi_ctrl_info *ctrl_info)
2902{
Kevin Barnett98f87662017-05-03 18:53:11 -05002903 del_timer_sync(&ctrl_info->heartbeat_timer);
Kevin Barnett6c223762016-06-27 16:41:00 -05002904}
2905
Kevin Barnett6a50d6a2017-05-03 18:52:52 -05002906static inline int pqi_event_type_to_event_index(unsigned int event_type)
Kevin Barnett6c223762016-06-27 16:41:00 -05002907{
2908 int index;
2909
Kevin Barnett6a50d6a2017-05-03 18:52:52 -05002910 for (index = 0; index < ARRAY_SIZE(pqi_supported_event_types); index++)
2911 if (event_type == pqi_supported_event_types[index])
2912 return index;
Kevin Barnett6c223762016-06-27 16:41:00 -05002913
Kevin Barnett6a50d6a2017-05-03 18:52:52 -05002914 return -1;
2915}
2916
2917static inline bool pqi_is_supported_event(unsigned int event_type)
2918{
2919 return pqi_event_type_to_event_index(event_type) != -1;
Kevin Barnett6c223762016-06-27 16:41:00 -05002920}
2921
2922static unsigned int pqi_process_event_intr(struct pqi_ctrl_info *ctrl_info)
2923{
2924 unsigned int num_events;
2925 pqi_index_t oq_pi;
2926 pqi_index_t oq_ci;
2927 struct pqi_event_queue *event_queue;
2928 struct pqi_event_response *response;
Kevin Barnett6a50d6a2017-05-03 18:52:52 -05002929 struct pqi_event *event;
Kevin Barnett6c223762016-06-27 16:41:00 -05002930 int event_index;
2931
2932 event_queue = &ctrl_info->event_queue;
2933 num_events = 0;
Kevin Barnett6c223762016-06-27 16:41:00 -05002934 oq_ci = event_queue->oq_ci_copy;
2935
2936 while (1) {
2937 oq_pi = *event_queue->oq_pi;
2938 if (oq_pi == oq_ci)
2939 break;
2940
2941 num_events++;
2942 response = event_queue->oq_element_array +
2943 (oq_ci * PQI_EVENT_OQ_ELEMENT_LENGTH);
2944
2945 event_index =
2946 pqi_event_type_to_event_index(response->event_type);
2947
2948 if (event_index >= 0) {
2949 if (response->request_acknowlege) {
Kevin Barnett6a50d6a2017-05-03 18:52:52 -05002950 event = &ctrl_info->events[event_index];
2951 event->pending = true;
2952 event->event_type = response->event_type;
2953 event->event_id = response->event_id;
2954 event->additional_event_id =
Kevin Barnett6c223762016-06-27 16:41:00 -05002955 response->additional_event_id;
Kevin Barnett6c223762016-06-27 16:41:00 -05002956 }
2957 }
2958
2959 oq_ci = (oq_ci + 1) % PQI_NUM_EVENT_QUEUE_ELEMENTS;
2960 }
2961
2962 if (num_events) {
2963 event_queue->oq_ci_copy = oq_ci;
2964 writel(oq_ci, event_queue->oq_ci);
Kevin Barnett98f87662017-05-03 18:53:11 -05002965 schedule_work(&ctrl_info->event_work);
Kevin Barnett6c223762016-06-27 16:41:00 -05002966 }
2967
2968 return num_events;
2969}
2970
Kevin Barnett061ef062017-05-03 18:53:05 -05002971#define PQI_LEGACY_INTX_MASK 0x1
2972
2973static inline void pqi_configure_legacy_intx(struct pqi_ctrl_info *ctrl_info,
2974 bool enable_intx)
2975{
2976 u32 intx_mask;
2977 struct pqi_device_registers __iomem *pqi_registers;
2978 volatile void __iomem *register_addr;
2979
2980 pqi_registers = ctrl_info->pqi_registers;
2981
2982 if (enable_intx)
2983 register_addr = &pqi_registers->legacy_intx_mask_clear;
2984 else
2985 register_addr = &pqi_registers->legacy_intx_mask_set;
2986
2987 intx_mask = readl(register_addr);
2988 intx_mask |= PQI_LEGACY_INTX_MASK;
2989 writel(intx_mask, register_addr);
2990}
2991
2992static void pqi_change_irq_mode(struct pqi_ctrl_info *ctrl_info,
2993 enum pqi_irq_mode new_mode)
2994{
2995 switch (ctrl_info->irq_mode) {
2996 case IRQ_MODE_MSIX:
2997 switch (new_mode) {
2998 case IRQ_MODE_MSIX:
2999 break;
3000 case IRQ_MODE_INTX:
3001 pqi_configure_legacy_intx(ctrl_info, true);
3002 sis_disable_msix(ctrl_info);
3003 sis_enable_intx(ctrl_info);
3004 break;
3005 case IRQ_MODE_NONE:
3006 sis_disable_msix(ctrl_info);
3007 break;
3008 }
3009 break;
3010 case IRQ_MODE_INTX:
3011 switch (new_mode) {
3012 case IRQ_MODE_MSIX:
3013 pqi_configure_legacy_intx(ctrl_info, false);
3014 sis_disable_intx(ctrl_info);
3015 sis_enable_msix(ctrl_info);
3016 break;
3017 case IRQ_MODE_INTX:
3018 break;
3019 case IRQ_MODE_NONE:
3020 pqi_configure_legacy_intx(ctrl_info, false);
3021 sis_disable_intx(ctrl_info);
3022 break;
3023 }
3024 break;
3025 case IRQ_MODE_NONE:
3026 switch (new_mode) {
3027 case IRQ_MODE_MSIX:
3028 sis_enable_msix(ctrl_info);
3029 break;
3030 case IRQ_MODE_INTX:
3031 pqi_configure_legacy_intx(ctrl_info, true);
3032 sis_enable_intx(ctrl_info);
3033 break;
3034 case IRQ_MODE_NONE:
3035 break;
3036 }
3037 break;
3038 }
3039
3040 ctrl_info->irq_mode = new_mode;
3041}
3042
3043#define PQI_LEGACY_INTX_PENDING 0x1
3044
3045static inline bool pqi_is_valid_irq(struct pqi_ctrl_info *ctrl_info)
3046{
3047 bool valid_irq;
3048 u32 intx_status;
3049
3050 switch (ctrl_info->irq_mode) {
3051 case IRQ_MODE_MSIX:
3052 valid_irq = true;
3053 break;
3054 case IRQ_MODE_INTX:
3055 intx_status =
3056 readl(&ctrl_info->pqi_registers->legacy_intx_status);
3057 if (intx_status & PQI_LEGACY_INTX_PENDING)
3058 valid_irq = true;
3059 else
3060 valid_irq = false;
3061 break;
3062 case IRQ_MODE_NONE:
3063 default:
3064 valid_irq = false;
3065 break;
3066 }
3067
3068 return valid_irq;
3069}
3070
Kevin Barnett6c223762016-06-27 16:41:00 -05003071static irqreturn_t pqi_irq_handler(int irq, void *data)
3072{
3073 struct pqi_ctrl_info *ctrl_info;
3074 struct pqi_queue_group *queue_group;
3075 unsigned int num_responses_handled;
3076
3077 queue_group = data;
3078 ctrl_info = queue_group->ctrl_info;
3079
Kevin Barnett061ef062017-05-03 18:53:05 -05003080 if (!pqi_is_valid_irq(ctrl_info))
Kevin Barnett6c223762016-06-27 16:41:00 -05003081 return IRQ_NONE;
3082
3083 num_responses_handled = pqi_process_io_intr(ctrl_info, queue_group);
3084
3085 if (irq == ctrl_info->event_irq)
3086 num_responses_handled += pqi_process_event_intr(ctrl_info);
3087
3088 if (num_responses_handled)
3089 atomic_inc(&ctrl_info->num_interrupts);
3090
3091 pqi_start_io(ctrl_info, queue_group, RAID_PATH, NULL);
3092 pqi_start_io(ctrl_info, queue_group, AIO_PATH, NULL);
3093
3094 return IRQ_HANDLED;
3095}
3096
3097static int pqi_request_irqs(struct pqi_ctrl_info *ctrl_info)
3098{
Kevin Barnettd91d7822017-05-03 18:53:30 -05003099 struct pci_dev *pci_dev = ctrl_info->pci_dev;
Kevin Barnett6c223762016-06-27 16:41:00 -05003100 int i;
3101 int rc;
3102
Kevin Barnettd91d7822017-05-03 18:53:30 -05003103 ctrl_info->event_irq = pci_irq_vector(pci_dev, 0);
Kevin Barnett6c223762016-06-27 16:41:00 -05003104
3105 for (i = 0; i < ctrl_info->num_msix_vectors_enabled; i++) {
Kevin Barnettd91d7822017-05-03 18:53:30 -05003106 rc = request_irq(pci_irq_vector(pci_dev, i), pqi_irq_handler, 0,
Christoph Hellwig52198222016-11-01 08:12:49 -06003107 DRIVER_NAME_SHORT, &ctrl_info->queue_groups[i]);
Kevin Barnett6c223762016-06-27 16:41:00 -05003108 if (rc) {
Kevin Barnettd91d7822017-05-03 18:53:30 -05003109 dev_err(&pci_dev->dev,
Kevin Barnett6c223762016-06-27 16:41:00 -05003110 "irq %u init failed with error %d\n",
Kevin Barnettd91d7822017-05-03 18:53:30 -05003111 pci_irq_vector(pci_dev, i), rc);
Kevin Barnett6c223762016-06-27 16:41:00 -05003112 return rc;
3113 }
3114 ctrl_info->num_msix_vectors_initialized++;
3115 }
3116
3117 return 0;
3118}
3119
Kevin Barnett98bf0612017-05-03 18:52:28 -05003120static void pqi_free_irqs(struct pqi_ctrl_info *ctrl_info)
3121{
3122 int i;
3123
3124 for (i = 0; i < ctrl_info->num_msix_vectors_initialized; i++)
3125 free_irq(pci_irq_vector(ctrl_info->pci_dev, i),
3126 &ctrl_info->queue_groups[i]);
3127
3128 ctrl_info->num_msix_vectors_initialized = 0;
3129}
3130
Kevin Barnett6c223762016-06-27 16:41:00 -05003131static int pqi_enable_msix_interrupts(struct pqi_ctrl_info *ctrl_info)
3132{
Kevin Barnett98bf0612017-05-03 18:52:28 -05003133 int num_vectors_enabled;
Kevin Barnett6c223762016-06-27 16:41:00 -05003134
Kevin Barnett98bf0612017-05-03 18:52:28 -05003135 num_vectors_enabled = pci_alloc_irq_vectors(ctrl_info->pci_dev,
Christoph Hellwig52198222016-11-01 08:12:49 -06003136 PQI_MIN_MSIX_VECTORS, ctrl_info->num_queue_groups,
3137 PCI_IRQ_MSIX | PCI_IRQ_AFFINITY);
Kevin Barnett98bf0612017-05-03 18:52:28 -05003138 if (num_vectors_enabled < 0) {
Kevin Barnett6c223762016-06-27 16:41:00 -05003139 dev_err(&ctrl_info->pci_dev->dev,
Kevin Barnett98bf0612017-05-03 18:52:28 -05003140 "MSI-X init failed with error %d\n",
3141 num_vectors_enabled);
3142 return num_vectors_enabled;
Kevin Barnett6c223762016-06-27 16:41:00 -05003143 }
3144
Kevin Barnett98bf0612017-05-03 18:52:28 -05003145 ctrl_info->num_msix_vectors_enabled = num_vectors_enabled;
Kevin Barnett061ef062017-05-03 18:53:05 -05003146 ctrl_info->irq_mode = IRQ_MODE_MSIX;
Kevin Barnett6c223762016-06-27 16:41:00 -05003147 return 0;
3148}
3149
Kevin Barnett98bf0612017-05-03 18:52:28 -05003150static void pqi_disable_msix_interrupts(struct pqi_ctrl_info *ctrl_info)
3151{
3152 if (ctrl_info->num_msix_vectors_enabled) {
3153 pci_free_irq_vectors(ctrl_info->pci_dev);
3154 ctrl_info->num_msix_vectors_enabled = 0;
3155 }
3156}
3157
Kevin Barnett6c223762016-06-27 16:41:00 -05003158static int pqi_alloc_operational_queues(struct pqi_ctrl_info *ctrl_info)
3159{
3160 unsigned int i;
3161 size_t alloc_length;
3162 size_t element_array_length_per_iq;
3163 size_t element_array_length_per_oq;
3164 void *element_array;
3165 void *next_queue_index;
3166 void *aligned_pointer;
3167 unsigned int num_inbound_queues;
3168 unsigned int num_outbound_queues;
3169 unsigned int num_queue_indexes;
3170 struct pqi_queue_group *queue_group;
3171
3172 element_array_length_per_iq =
3173 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH *
3174 ctrl_info->num_elements_per_iq;
3175 element_array_length_per_oq =
3176 PQI_OPERATIONAL_OQ_ELEMENT_LENGTH *
3177 ctrl_info->num_elements_per_oq;
3178 num_inbound_queues = ctrl_info->num_queue_groups * 2;
3179 num_outbound_queues = ctrl_info->num_queue_groups;
3180 num_queue_indexes = (ctrl_info->num_queue_groups * 3) + 1;
3181
3182 aligned_pointer = NULL;
3183
3184 for (i = 0; i < num_inbound_queues; i++) {
3185 aligned_pointer = PTR_ALIGN(aligned_pointer,
3186 PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
3187 aligned_pointer += element_array_length_per_iq;
3188 }
3189
3190 for (i = 0; i < num_outbound_queues; i++) {
3191 aligned_pointer = PTR_ALIGN(aligned_pointer,
3192 PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
3193 aligned_pointer += element_array_length_per_oq;
3194 }
3195
3196 aligned_pointer = PTR_ALIGN(aligned_pointer,
3197 PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
3198 aligned_pointer += PQI_NUM_EVENT_QUEUE_ELEMENTS *
3199 PQI_EVENT_OQ_ELEMENT_LENGTH;
3200
3201 for (i = 0; i < num_queue_indexes; i++) {
3202 aligned_pointer = PTR_ALIGN(aligned_pointer,
3203 PQI_OPERATIONAL_INDEX_ALIGNMENT);
3204 aligned_pointer += sizeof(pqi_index_t);
3205 }
3206
3207 alloc_length = (size_t)aligned_pointer +
3208 PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT;
3209
Kevin Barnette1d213b2017-05-03 18:53:18 -05003210 alloc_length += PQI_EXTRA_SGL_MEMORY;
3211
Kevin Barnett6c223762016-06-27 16:41:00 -05003212 ctrl_info->queue_memory_base =
3213 dma_zalloc_coherent(&ctrl_info->pci_dev->dev,
3214 alloc_length,
3215 &ctrl_info->queue_memory_base_dma_handle, GFP_KERNEL);
3216
Kevin Barnettd87d5472017-05-03 18:54:00 -05003217 if (!ctrl_info->queue_memory_base)
Kevin Barnett6c223762016-06-27 16:41:00 -05003218 return -ENOMEM;
Kevin Barnett6c223762016-06-27 16:41:00 -05003219
3220 ctrl_info->queue_memory_length = alloc_length;
3221
3222 element_array = PTR_ALIGN(ctrl_info->queue_memory_base,
3223 PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
3224
3225 for (i = 0; i < ctrl_info->num_queue_groups; i++) {
3226 queue_group = &ctrl_info->queue_groups[i];
3227 queue_group->iq_element_array[RAID_PATH] = element_array;
3228 queue_group->iq_element_array_bus_addr[RAID_PATH] =
3229 ctrl_info->queue_memory_base_dma_handle +
3230 (element_array - ctrl_info->queue_memory_base);
3231 element_array += element_array_length_per_iq;
3232 element_array = PTR_ALIGN(element_array,
3233 PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
3234 queue_group->iq_element_array[AIO_PATH] = element_array;
3235 queue_group->iq_element_array_bus_addr[AIO_PATH] =
3236 ctrl_info->queue_memory_base_dma_handle +
3237 (element_array - ctrl_info->queue_memory_base);
3238 element_array += element_array_length_per_iq;
3239 element_array = PTR_ALIGN(element_array,
3240 PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
3241 }
3242
3243 for (i = 0; i < ctrl_info->num_queue_groups; i++) {
3244 queue_group = &ctrl_info->queue_groups[i];
3245 queue_group->oq_element_array = element_array;
3246 queue_group->oq_element_array_bus_addr =
3247 ctrl_info->queue_memory_base_dma_handle +
3248 (element_array - ctrl_info->queue_memory_base);
3249 element_array += element_array_length_per_oq;
3250 element_array = PTR_ALIGN(element_array,
3251 PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
3252 }
3253
3254 ctrl_info->event_queue.oq_element_array = element_array;
3255 ctrl_info->event_queue.oq_element_array_bus_addr =
3256 ctrl_info->queue_memory_base_dma_handle +
3257 (element_array - ctrl_info->queue_memory_base);
3258 element_array += PQI_NUM_EVENT_QUEUE_ELEMENTS *
3259 PQI_EVENT_OQ_ELEMENT_LENGTH;
3260
3261 next_queue_index = PTR_ALIGN(element_array,
3262 PQI_OPERATIONAL_INDEX_ALIGNMENT);
3263
3264 for (i = 0; i < ctrl_info->num_queue_groups; i++) {
3265 queue_group = &ctrl_info->queue_groups[i];
3266 queue_group->iq_ci[RAID_PATH] = next_queue_index;
3267 queue_group->iq_ci_bus_addr[RAID_PATH] =
3268 ctrl_info->queue_memory_base_dma_handle +
3269 (next_queue_index - ctrl_info->queue_memory_base);
3270 next_queue_index += sizeof(pqi_index_t);
3271 next_queue_index = PTR_ALIGN(next_queue_index,
3272 PQI_OPERATIONAL_INDEX_ALIGNMENT);
3273 queue_group->iq_ci[AIO_PATH] = next_queue_index;
3274 queue_group->iq_ci_bus_addr[AIO_PATH] =
3275 ctrl_info->queue_memory_base_dma_handle +
3276 (next_queue_index - ctrl_info->queue_memory_base);
3277 next_queue_index += sizeof(pqi_index_t);
3278 next_queue_index = PTR_ALIGN(next_queue_index,
3279 PQI_OPERATIONAL_INDEX_ALIGNMENT);
3280 queue_group->oq_pi = next_queue_index;
3281 queue_group->oq_pi_bus_addr =
3282 ctrl_info->queue_memory_base_dma_handle +
3283 (next_queue_index - ctrl_info->queue_memory_base);
3284 next_queue_index += sizeof(pqi_index_t);
3285 next_queue_index = PTR_ALIGN(next_queue_index,
3286 PQI_OPERATIONAL_INDEX_ALIGNMENT);
3287 }
3288
3289 ctrl_info->event_queue.oq_pi = next_queue_index;
3290 ctrl_info->event_queue.oq_pi_bus_addr =
3291 ctrl_info->queue_memory_base_dma_handle +
3292 (next_queue_index - ctrl_info->queue_memory_base);
3293
3294 return 0;
3295}
3296
3297static void pqi_init_operational_queues(struct pqi_ctrl_info *ctrl_info)
3298{
3299 unsigned int i;
3300 u16 next_iq_id = PQI_MIN_OPERATIONAL_QUEUE_ID;
3301 u16 next_oq_id = PQI_MIN_OPERATIONAL_QUEUE_ID;
3302
3303 /*
3304 * Initialize the backpointers to the controller structure in
3305 * each operational queue group structure.
3306 */
3307 for (i = 0; i < ctrl_info->num_queue_groups; i++)
3308 ctrl_info->queue_groups[i].ctrl_info = ctrl_info;
3309
3310 /*
3311 * Assign IDs to all operational queues. Note that the IDs
3312 * assigned to operational IQs are independent of the IDs
3313 * assigned to operational OQs.
3314 */
3315 ctrl_info->event_queue.oq_id = next_oq_id++;
3316 for (i = 0; i < ctrl_info->num_queue_groups; i++) {
3317 ctrl_info->queue_groups[i].iq_id[RAID_PATH] = next_iq_id++;
3318 ctrl_info->queue_groups[i].iq_id[AIO_PATH] = next_iq_id++;
3319 ctrl_info->queue_groups[i].oq_id = next_oq_id++;
3320 }
3321
3322 /*
3323 * Assign MSI-X table entry indexes to all queues. Note that the
3324 * interrupt for the event queue is shared with the first queue group.
3325 */
3326 ctrl_info->event_queue.int_msg_num = 0;
3327 for (i = 0; i < ctrl_info->num_queue_groups; i++)
3328 ctrl_info->queue_groups[i].int_msg_num = i;
3329
3330 for (i = 0; i < ctrl_info->num_queue_groups; i++) {
3331 spin_lock_init(&ctrl_info->queue_groups[i].submit_lock[0]);
3332 spin_lock_init(&ctrl_info->queue_groups[i].submit_lock[1]);
3333 INIT_LIST_HEAD(&ctrl_info->queue_groups[i].request_list[0]);
3334 INIT_LIST_HEAD(&ctrl_info->queue_groups[i].request_list[1]);
3335 }
3336}
3337
3338static int pqi_alloc_admin_queues(struct pqi_ctrl_info *ctrl_info)
3339{
3340 size_t alloc_length;
3341 struct pqi_admin_queues_aligned *admin_queues_aligned;
3342 struct pqi_admin_queues *admin_queues;
3343
3344 alloc_length = sizeof(struct pqi_admin_queues_aligned) +
3345 PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT;
3346
3347 ctrl_info->admin_queue_memory_base =
3348 dma_zalloc_coherent(&ctrl_info->pci_dev->dev,
3349 alloc_length,
3350 &ctrl_info->admin_queue_memory_base_dma_handle,
3351 GFP_KERNEL);
3352
3353 if (!ctrl_info->admin_queue_memory_base)
3354 return -ENOMEM;
3355
3356 ctrl_info->admin_queue_memory_length = alloc_length;
3357
3358 admin_queues = &ctrl_info->admin_queues;
3359 admin_queues_aligned = PTR_ALIGN(ctrl_info->admin_queue_memory_base,
3360 PQI_QUEUE_ELEMENT_ARRAY_ALIGNMENT);
3361 admin_queues->iq_element_array =
3362 &admin_queues_aligned->iq_element_array;
3363 admin_queues->oq_element_array =
3364 &admin_queues_aligned->oq_element_array;
3365 admin_queues->iq_ci = &admin_queues_aligned->iq_ci;
3366 admin_queues->oq_pi = &admin_queues_aligned->oq_pi;
3367
3368 admin_queues->iq_element_array_bus_addr =
3369 ctrl_info->admin_queue_memory_base_dma_handle +
3370 (admin_queues->iq_element_array -
3371 ctrl_info->admin_queue_memory_base);
3372 admin_queues->oq_element_array_bus_addr =
3373 ctrl_info->admin_queue_memory_base_dma_handle +
3374 (admin_queues->oq_element_array -
3375 ctrl_info->admin_queue_memory_base);
3376 admin_queues->iq_ci_bus_addr =
3377 ctrl_info->admin_queue_memory_base_dma_handle +
3378 ((void *)admin_queues->iq_ci -
3379 ctrl_info->admin_queue_memory_base);
3380 admin_queues->oq_pi_bus_addr =
3381 ctrl_info->admin_queue_memory_base_dma_handle +
3382 ((void *)admin_queues->oq_pi -
3383 ctrl_info->admin_queue_memory_base);
3384
3385 return 0;
3386}
3387
3388#define PQI_ADMIN_QUEUE_CREATE_TIMEOUT_JIFFIES HZ
3389#define PQI_ADMIN_QUEUE_CREATE_POLL_INTERVAL_MSECS 1
3390
3391static int pqi_create_admin_queues(struct pqi_ctrl_info *ctrl_info)
3392{
3393 struct pqi_device_registers __iomem *pqi_registers;
3394 struct pqi_admin_queues *admin_queues;
3395 unsigned long timeout;
3396 u8 status;
3397 u32 reg;
3398
3399 pqi_registers = ctrl_info->pqi_registers;
3400 admin_queues = &ctrl_info->admin_queues;
3401
3402 writeq((u64)admin_queues->iq_element_array_bus_addr,
3403 &pqi_registers->admin_iq_element_array_addr);
3404 writeq((u64)admin_queues->oq_element_array_bus_addr,
3405 &pqi_registers->admin_oq_element_array_addr);
3406 writeq((u64)admin_queues->iq_ci_bus_addr,
3407 &pqi_registers->admin_iq_ci_addr);
3408 writeq((u64)admin_queues->oq_pi_bus_addr,
3409 &pqi_registers->admin_oq_pi_addr);
3410
3411 reg = PQI_ADMIN_IQ_NUM_ELEMENTS |
3412 (PQI_ADMIN_OQ_NUM_ELEMENTS) << 8 |
3413 (admin_queues->int_msg_num << 16);
3414 writel(reg, &pqi_registers->admin_iq_num_elements);
3415 writel(PQI_CREATE_ADMIN_QUEUE_PAIR,
3416 &pqi_registers->function_and_status_code);
3417
3418 timeout = PQI_ADMIN_QUEUE_CREATE_TIMEOUT_JIFFIES + jiffies;
3419 while (1) {
3420 status = readb(&pqi_registers->function_and_status_code);
3421 if (status == PQI_STATUS_IDLE)
3422 break;
3423 if (time_after(jiffies, timeout))
3424 return -ETIMEDOUT;
3425 msleep(PQI_ADMIN_QUEUE_CREATE_POLL_INTERVAL_MSECS);
3426 }
3427
3428 /*
3429 * The offset registers are not initialized to the correct
3430 * offsets until *after* the create admin queue pair command
3431 * completes successfully.
3432 */
3433 admin_queues->iq_pi = ctrl_info->iomem_base +
3434 PQI_DEVICE_REGISTERS_OFFSET +
3435 readq(&pqi_registers->admin_iq_pi_offset);
3436 admin_queues->oq_ci = ctrl_info->iomem_base +
3437 PQI_DEVICE_REGISTERS_OFFSET +
3438 readq(&pqi_registers->admin_oq_ci_offset);
3439
3440 return 0;
3441}
3442
3443static void pqi_submit_admin_request(struct pqi_ctrl_info *ctrl_info,
3444 struct pqi_general_admin_request *request)
3445{
3446 struct pqi_admin_queues *admin_queues;
3447 void *next_element;
3448 pqi_index_t iq_pi;
3449
3450 admin_queues = &ctrl_info->admin_queues;
3451 iq_pi = admin_queues->iq_pi_copy;
3452
3453 next_element = admin_queues->iq_element_array +
3454 (iq_pi * PQI_ADMIN_IQ_ELEMENT_LENGTH);
3455
3456 memcpy(next_element, request, sizeof(*request));
3457
3458 iq_pi = (iq_pi + 1) % PQI_ADMIN_IQ_NUM_ELEMENTS;
3459 admin_queues->iq_pi_copy = iq_pi;
3460
3461 /*
3462 * This write notifies the controller that an IU is available to be
3463 * processed.
3464 */
3465 writel(iq_pi, admin_queues->iq_pi);
3466}
3467
Kevin Barnett13bede62017-05-03 18:55:13 -05003468#define PQI_ADMIN_REQUEST_TIMEOUT_SECS 60
3469
Kevin Barnett6c223762016-06-27 16:41:00 -05003470static int pqi_poll_for_admin_response(struct pqi_ctrl_info *ctrl_info,
3471 struct pqi_general_admin_response *response)
3472{
3473 struct pqi_admin_queues *admin_queues;
3474 pqi_index_t oq_pi;
3475 pqi_index_t oq_ci;
3476 unsigned long timeout;
3477
3478 admin_queues = &ctrl_info->admin_queues;
3479 oq_ci = admin_queues->oq_ci_copy;
3480
Kevin Barnett13bede62017-05-03 18:55:13 -05003481 timeout = (PQI_ADMIN_REQUEST_TIMEOUT_SECS * HZ) + jiffies;
Kevin Barnett6c223762016-06-27 16:41:00 -05003482
3483 while (1) {
3484 oq_pi = *admin_queues->oq_pi;
3485 if (oq_pi != oq_ci)
3486 break;
3487 if (time_after(jiffies, timeout)) {
3488 dev_err(&ctrl_info->pci_dev->dev,
3489 "timed out waiting for admin response\n");
3490 return -ETIMEDOUT;
3491 }
Kevin Barnett13bede62017-05-03 18:55:13 -05003492 if (!sis_is_firmware_running(ctrl_info))
3493 return -ENXIO;
Kevin Barnett6c223762016-06-27 16:41:00 -05003494 usleep_range(1000, 2000);
3495 }
3496
3497 memcpy(response, admin_queues->oq_element_array +
3498 (oq_ci * PQI_ADMIN_OQ_ELEMENT_LENGTH), sizeof(*response));
3499
3500 oq_ci = (oq_ci + 1) % PQI_ADMIN_OQ_NUM_ELEMENTS;
3501 admin_queues->oq_ci_copy = oq_ci;
3502 writel(oq_ci, admin_queues->oq_ci);
3503
3504 return 0;
3505}
3506
3507static void pqi_start_io(struct pqi_ctrl_info *ctrl_info,
3508 struct pqi_queue_group *queue_group, enum pqi_io_path path,
3509 struct pqi_io_request *io_request)
3510{
3511 struct pqi_io_request *next;
3512 void *next_element;
3513 pqi_index_t iq_pi;
3514 pqi_index_t iq_ci;
3515 size_t iu_length;
3516 unsigned long flags;
3517 unsigned int num_elements_needed;
3518 unsigned int num_elements_to_end_of_queue;
3519 size_t copy_count;
3520 struct pqi_iu_header *request;
3521
3522 spin_lock_irqsave(&queue_group->submit_lock[path], flags);
3523
Kevin Barnett376fb882017-05-03 18:54:43 -05003524 if (io_request) {
3525 io_request->queue_group = queue_group;
Kevin Barnett6c223762016-06-27 16:41:00 -05003526 list_add_tail(&io_request->request_list_entry,
3527 &queue_group->request_list[path]);
Kevin Barnett376fb882017-05-03 18:54:43 -05003528 }
Kevin Barnett6c223762016-06-27 16:41:00 -05003529
3530 iq_pi = queue_group->iq_pi_copy[path];
3531
3532 list_for_each_entry_safe(io_request, next,
3533 &queue_group->request_list[path], request_list_entry) {
3534
3535 request = io_request->iu;
3536
3537 iu_length = get_unaligned_le16(&request->iu_length) +
3538 PQI_REQUEST_HEADER_LENGTH;
3539 num_elements_needed =
3540 DIV_ROUND_UP(iu_length,
3541 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
3542
3543 iq_ci = *queue_group->iq_ci[path];
3544
3545 if (num_elements_needed > pqi_num_elements_free(iq_pi, iq_ci,
3546 ctrl_info->num_elements_per_iq))
3547 break;
3548
3549 put_unaligned_le16(queue_group->oq_id,
3550 &request->response_queue_id);
3551
3552 next_element = queue_group->iq_element_array[path] +
3553 (iq_pi * PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
3554
3555 num_elements_to_end_of_queue =
3556 ctrl_info->num_elements_per_iq - iq_pi;
3557
3558 if (num_elements_needed <= num_elements_to_end_of_queue) {
3559 memcpy(next_element, request, iu_length);
3560 } else {
3561 copy_count = num_elements_to_end_of_queue *
3562 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH;
3563 memcpy(next_element, request, copy_count);
3564 memcpy(queue_group->iq_element_array[path],
3565 (u8 *)request + copy_count,
3566 iu_length - copy_count);
3567 }
3568
3569 iq_pi = (iq_pi + num_elements_needed) %
3570 ctrl_info->num_elements_per_iq;
3571
3572 list_del(&io_request->request_list_entry);
3573 }
3574
3575 if (iq_pi != queue_group->iq_pi_copy[path]) {
3576 queue_group->iq_pi_copy[path] = iq_pi;
3577 /*
3578 * This write notifies the controller that one or more IUs are
3579 * available to be processed.
3580 */
3581 writel(iq_pi, queue_group->iq_pi[path]);
3582 }
3583
3584 spin_unlock_irqrestore(&queue_group->submit_lock[path], flags);
3585}
3586
Kevin Barnett1f37e992017-05-03 18:53:24 -05003587#define PQI_WAIT_FOR_COMPLETION_IO_TIMEOUT_SECS 10
3588
3589static int pqi_wait_for_completion_io(struct pqi_ctrl_info *ctrl_info,
3590 struct completion *wait)
3591{
3592 int rc;
Kevin Barnett1f37e992017-05-03 18:53:24 -05003593
3594 while (1) {
3595 if (wait_for_completion_io_timeout(wait,
3596 PQI_WAIT_FOR_COMPLETION_IO_TIMEOUT_SECS * HZ)) {
3597 rc = 0;
3598 break;
3599 }
3600
3601 pqi_check_ctrl_health(ctrl_info);
3602 if (pqi_ctrl_offline(ctrl_info)) {
3603 rc = -ENXIO;
3604 break;
3605 }
Kevin Barnett1f37e992017-05-03 18:53:24 -05003606 }
3607
3608 return rc;
3609}
3610
Kevin Barnett6c223762016-06-27 16:41:00 -05003611static void pqi_raid_synchronous_complete(struct pqi_io_request *io_request,
3612 void *context)
3613{
3614 struct completion *waiting = context;
3615
3616 complete(waiting);
3617}
3618
3619static int pqi_submit_raid_request_synchronous_with_io_request(
3620 struct pqi_ctrl_info *ctrl_info, struct pqi_io_request *io_request,
3621 unsigned long timeout_msecs)
3622{
3623 int rc = 0;
3624 DECLARE_COMPLETION_ONSTACK(wait);
3625
3626 io_request->io_complete_callback = pqi_raid_synchronous_complete;
3627 io_request->context = &wait;
3628
3629 pqi_start_io(ctrl_info,
3630 &ctrl_info->queue_groups[PQI_DEFAULT_QUEUE_GROUP], RAID_PATH,
3631 io_request);
3632
3633 if (timeout_msecs == NO_TIMEOUT) {
Kevin Barnett1f37e992017-05-03 18:53:24 -05003634 pqi_wait_for_completion_io(ctrl_info, &wait);
Kevin Barnett6c223762016-06-27 16:41:00 -05003635 } else {
3636 if (!wait_for_completion_io_timeout(&wait,
3637 msecs_to_jiffies(timeout_msecs))) {
3638 dev_warn(&ctrl_info->pci_dev->dev,
3639 "command timed out\n");
3640 rc = -ETIMEDOUT;
3641 }
3642 }
3643
3644 return rc;
3645}
3646
3647static int pqi_submit_raid_request_synchronous(struct pqi_ctrl_info *ctrl_info,
3648 struct pqi_iu_header *request, unsigned int flags,
3649 struct pqi_raid_error_info *error_info, unsigned long timeout_msecs)
3650{
3651 int rc;
3652 struct pqi_io_request *io_request;
3653 unsigned long start_jiffies;
3654 unsigned long msecs_blocked;
3655 size_t iu_length;
3656
3657 /*
3658 * Note that specifying PQI_SYNC_FLAGS_INTERRUPTABLE and a timeout value
3659 * are mutually exclusive.
3660 */
3661
3662 if (flags & PQI_SYNC_FLAGS_INTERRUPTABLE) {
3663 if (down_interruptible(&ctrl_info->sync_request_sem))
3664 return -ERESTARTSYS;
3665 } else {
3666 if (timeout_msecs == NO_TIMEOUT) {
3667 down(&ctrl_info->sync_request_sem);
3668 } else {
3669 start_jiffies = jiffies;
3670 if (down_timeout(&ctrl_info->sync_request_sem,
3671 msecs_to_jiffies(timeout_msecs)))
3672 return -ETIMEDOUT;
3673 msecs_blocked =
3674 jiffies_to_msecs(jiffies - start_jiffies);
3675 if (msecs_blocked >= timeout_msecs)
3676 return -ETIMEDOUT;
3677 timeout_msecs -= msecs_blocked;
3678 }
3679 }
3680
Kevin Barnett7561a7e2017-05-03 18:52:58 -05003681 pqi_ctrl_busy(ctrl_info);
3682 timeout_msecs = pqi_wait_if_ctrl_blocked(ctrl_info, timeout_msecs);
3683 if (timeout_msecs == 0) {
3684 rc = -ETIMEDOUT;
3685 goto out;
3686 }
3687
Kevin Barnett376fb882017-05-03 18:54:43 -05003688 if (pqi_ctrl_offline(ctrl_info)) {
3689 rc = -ENXIO;
3690 goto out;
3691 }
3692
Kevin Barnett6c223762016-06-27 16:41:00 -05003693 io_request = pqi_alloc_io_request(ctrl_info);
3694
3695 put_unaligned_le16(io_request->index,
3696 &(((struct pqi_raid_path_request *)request)->request_id));
3697
3698 if (request->iu_type == PQI_REQUEST_IU_RAID_PATH_IO)
3699 ((struct pqi_raid_path_request *)request)->error_index =
3700 ((struct pqi_raid_path_request *)request)->request_id;
3701
3702 iu_length = get_unaligned_le16(&request->iu_length) +
3703 PQI_REQUEST_HEADER_LENGTH;
3704 memcpy(io_request->iu, request, iu_length);
3705
3706 rc = pqi_submit_raid_request_synchronous_with_io_request(ctrl_info,
3707 io_request, timeout_msecs);
3708
3709 if (error_info) {
3710 if (io_request->error_info)
3711 memcpy(error_info, io_request->error_info,
3712 sizeof(*error_info));
3713 else
3714 memset(error_info, 0, sizeof(*error_info));
3715 } else if (rc == 0 && io_request->error_info) {
3716 u8 scsi_status;
3717 struct pqi_raid_error_info *raid_error_info;
3718
3719 raid_error_info = io_request->error_info;
3720 scsi_status = raid_error_info->status;
3721
3722 if (scsi_status == SAM_STAT_CHECK_CONDITION &&
3723 raid_error_info->data_out_result ==
3724 PQI_DATA_IN_OUT_UNDERFLOW)
3725 scsi_status = SAM_STAT_GOOD;
3726
3727 if (scsi_status != SAM_STAT_GOOD)
3728 rc = -EIO;
3729 }
3730
3731 pqi_free_io_request(io_request);
3732
Kevin Barnett7561a7e2017-05-03 18:52:58 -05003733out:
3734 pqi_ctrl_unbusy(ctrl_info);
Kevin Barnett6c223762016-06-27 16:41:00 -05003735 up(&ctrl_info->sync_request_sem);
3736
3737 return rc;
3738}
3739
3740static int pqi_validate_admin_response(
3741 struct pqi_general_admin_response *response, u8 expected_function_code)
3742{
3743 if (response->header.iu_type != PQI_RESPONSE_IU_GENERAL_ADMIN)
3744 return -EINVAL;
3745
3746 if (get_unaligned_le16(&response->header.iu_length) !=
3747 PQI_GENERAL_ADMIN_IU_LENGTH)
3748 return -EINVAL;
3749
3750 if (response->function_code != expected_function_code)
3751 return -EINVAL;
3752
3753 if (response->status != PQI_GENERAL_ADMIN_STATUS_SUCCESS)
3754 return -EINVAL;
3755
3756 return 0;
3757}
3758
3759static int pqi_submit_admin_request_synchronous(
3760 struct pqi_ctrl_info *ctrl_info,
3761 struct pqi_general_admin_request *request,
3762 struct pqi_general_admin_response *response)
3763{
3764 int rc;
3765
3766 pqi_submit_admin_request(ctrl_info, request);
3767
3768 rc = pqi_poll_for_admin_response(ctrl_info, response);
3769
3770 if (rc == 0)
3771 rc = pqi_validate_admin_response(response,
3772 request->function_code);
3773
3774 return rc;
3775}
3776
3777static int pqi_report_device_capability(struct pqi_ctrl_info *ctrl_info)
3778{
3779 int rc;
3780 struct pqi_general_admin_request request;
3781 struct pqi_general_admin_response response;
3782 struct pqi_device_capability *capability;
3783 struct pqi_iu_layer_descriptor *sop_iu_layer_descriptor;
3784
3785 capability = kmalloc(sizeof(*capability), GFP_KERNEL);
3786 if (!capability)
3787 return -ENOMEM;
3788
3789 memset(&request, 0, sizeof(request));
3790
3791 request.header.iu_type = PQI_REQUEST_IU_GENERAL_ADMIN;
3792 put_unaligned_le16(PQI_GENERAL_ADMIN_IU_LENGTH,
3793 &request.header.iu_length);
3794 request.function_code =
3795 PQI_GENERAL_ADMIN_FUNCTION_REPORT_DEVICE_CAPABILITY;
3796 put_unaligned_le32(sizeof(*capability),
3797 &request.data.report_device_capability.buffer_length);
3798
3799 rc = pqi_map_single(ctrl_info->pci_dev,
3800 &request.data.report_device_capability.sg_descriptor,
3801 capability, sizeof(*capability),
3802 PCI_DMA_FROMDEVICE);
3803 if (rc)
3804 goto out;
3805
3806 rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,
3807 &response);
3808
3809 pqi_pci_unmap(ctrl_info->pci_dev,
3810 &request.data.report_device_capability.sg_descriptor, 1,
3811 PCI_DMA_FROMDEVICE);
3812
3813 if (rc)
3814 goto out;
3815
3816 if (response.status != PQI_GENERAL_ADMIN_STATUS_SUCCESS) {
3817 rc = -EIO;
3818 goto out;
3819 }
3820
3821 ctrl_info->max_inbound_queues =
3822 get_unaligned_le16(&capability->max_inbound_queues);
3823 ctrl_info->max_elements_per_iq =
3824 get_unaligned_le16(&capability->max_elements_per_iq);
3825 ctrl_info->max_iq_element_length =
3826 get_unaligned_le16(&capability->max_iq_element_length)
3827 * 16;
3828 ctrl_info->max_outbound_queues =
3829 get_unaligned_le16(&capability->max_outbound_queues);
3830 ctrl_info->max_elements_per_oq =
3831 get_unaligned_le16(&capability->max_elements_per_oq);
3832 ctrl_info->max_oq_element_length =
3833 get_unaligned_le16(&capability->max_oq_element_length)
3834 * 16;
3835
3836 sop_iu_layer_descriptor =
3837 &capability->iu_layer_descriptors[PQI_PROTOCOL_SOP];
3838
3839 ctrl_info->max_inbound_iu_length_per_firmware =
3840 get_unaligned_le16(
3841 &sop_iu_layer_descriptor->max_inbound_iu_length);
3842 ctrl_info->inbound_spanning_supported =
3843 sop_iu_layer_descriptor->inbound_spanning_supported;
3844 ctrl_info->outbound_spanning_supported =
3845 sop_iu_layer_descriptor->outbound_spanning_supported;
3846
3847out:
3848 kfree(capability);
3849
3850 return rc;
3851}
3852
3853static int pqi_validate_device_capability(struct pqi_ctrl_info *ctrl_info)
3854{
3855 if (ctrl_info->max_iq_element_length <
3856 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH) {
3857 dev_err(&ctrl_info->pci_dev->dev,
3858 "max. inbound queue element length of %d is less than the required length of %d\n",
3859 ctrl_info->max_iq_element_length,
3860 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
3861 return -EINVAL;
3862 }
3863
3864 if (ctrl_info->max_oq_element_length <
3865 PQI_OPERATIONAL_OQ_ELEMENT_LENGTH) {
3866 dev_err(&ctrl_info->pci_dev->dev,
3867 "max. outbound queue element length of %d is less than the required length of %d\n",
3868 ctrl_info->max_oq_element_length,
3869 PQI_OPERATIONAL_OQ_ELEMENT_LENGTH);
3870 return -EINVAL;
3871 }
3872
3873 if (ctrl_info->max_inbound_iu_length_per_firmware <
3874 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH) {
3875 dev_err(&ctrl_info->pci_dev->dev,
3876 "max. inbound IU length of %u is less than the min. required length of %d\n",
3877 ctrl_info->max_inbound_iu_length_per_firmware,
3878 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
3879 return -EINVAL;
3880 }
3881
Kevin Barnett77668f42016-08-31 14:54:23 -05003882 if (!ctrl_info->inbound_spanning_supported) {
3883 dev_err(&ctrl_info->pci_dev->dev,
3884 "the controller does not support inbound spanning\n");
3885 return -EINVAL;
3886 }
3887
3888 if (ctrl_info->outbound_spanning_supported) {
3889 dev_err(&ctrl_info->pci_dev->dev,
3890 "the controller supports outbound spanning but this driver does not\n");
3891 return -EINVAL;
3892 }
3893
Kevin Barnett6c223762016-06-27 16:41:00 -05003894 return 0;
3895}
3896
3897static int pqi_delete_operational_queue(struct pqi_ctrl_info *ctrl_info,
3898 bool inbound_queue, u16 queue_id)
3899{
3900 struct pqi_general_admin_request request;
3901 struct pqi_general_admin_response response;
3902
3903 memset(&request, 0, sizeof(request));
3904 request.header.iu_type = PQI_REQUEST_IU_GENERAL_ADMIN;
3905 put_unaligned_le16(PQI_GENERAL_ADMIN_IU_LENGTH,
3906 &request.header.iu_length);
3907 if (inbound_queue)
3908 request.function_code =
3909 PQI_GENERAL_ADMIN_FUNCTION_DELETE_IQ;
3910 else
3911 request.function_code =
3912 PQI_GENERAL_ADMIN_FUNCTION_DELETE_OQ;
3913 put_unaligned_le16(queue_id,
3914 &request.data.delete_operational_queue.queue_id);
3915
3916 return pqi_submit_admin_request_synchronous(ctrl_info, &request,
3917 &response);
3918}
3919
3920static int pqi_create_event_queue(struct pqi_ctrl_info *ctrl_info)
3921{
3922 int rc;
3923 struct pqi_event_queue *event_queue;
3924 struct pqi_general_admin_request request;
3925 struct pqi_general_admin_response response;
3926
3927 event_queue = &ctrl_info->event_queue;
3928
3929 /*
3930 * Create OQ (Outbound Queue - device to host queue) to dedicate
3931 * to events.
3932 */
3933 memset(&request, 0, sizeof(request));
3934 request.header.iu_type = PQI_REQUEST_IU_GENERAL_ADMIN;
3935 put_unaligned_le16(PQI_GENERAL_ADMIN_IU_LENGTH,
3936 &request.header.iu_length);
3937 request.function_code = PQI_GENERAL_ADMIN_FUNCTION_CREATE_OQ;
3938 put_unaligned_le16(event_queue->oq_id,
3939 &request.data.create_operational_oq.queue_id);
3940 put_unaligned_le64((u64)event_queue->oq_element_array_bus_addr,
3941 &request.data.create_operational_oq.element_array_addr);
3942 put_unaligned_le64((u64)event_queue->oq_pi_bus_addr,
3943 &request.data.create_operational_oq.pi_addr);
3944 put_unaligned_le16(PQI_NUM_EVENT_QUEUE_ELEMENTS,
3945 &request.data.create_operational_oq.num_elements);
3946 put_unaligned_le16(PQI_EVENT_OQ_ELEMENT_LENGTH / 16,
3947 &request.data.create_operational_oq.element_length);
3948 request.data.create_operational_oq.queue_protocol = PQI_PROTOCOL_SOP;
3949 put_unaligned_le16(event_queue->int_msg_num,
3950 &request.data.create_operational_oq.int_msg_num);
3951
3952 rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,
3953 &response);
3954 if (rc)
3955 return rc;
3956
3957 event_queue->oq_ci = ctrl_info->iomem_base +
3958 PQI_DEVICE_REGISTERS_OFFSET +
3959 get_unaligned_le64(
3960 &response.data.create_operational_oq.oq_ci_offset);
3961
3962 return 0;
3963}
3964
Kevin Barnett061ef062017-05-03 18:53:05 -05003965static int pqi_create_queue_group(struct pqi_ctrl_info *ctrl_info,
3966 unsigned int group_number)
Kevin Barnett6c223762016-06-27 16:41:00 -05003967{
Kevin Barnett6c223762016-06-27 16:41:00 -05003968 int rc;
3969 struct pqi_queue_group *queue_group;
3970 struct pqi_general_admin_request request;
3971 struct pqi_general_admin_response response;
3972
Kevin Barnett061ef062017-05-03 18:53:05 -05003973 queue_group = &ctrl_info->queue_groups[group_number];
Kevin Barnett6c223762016-06-27 16:41:00 -05003974
3975 /*
3976 * Create IQ (Inbound Queue - host to device queue) for
3977 * RAID path.
3978 */
3979 memset(&request, 0, sizeof(request));
3980 request.header.iu_type = PQI_REQUEST_IU_GENERAL_ADMIN;
3981 put_unaligned_le16(PQI_GENERAL_ADMIN_IU_LENGTH,
3982 &request.header.iu_length);
3983 request.function_code = PQI_GENERAL_ADMIN_FUNCTION_CREATE_IQ;
3984 put_unaligned_le16(queue_group->iq_id[RAID_PATH],
3985 &request.data.create_operational_iq.queue_id);
3986 put_unaligned_le64(
3987 (u64)queue_group->iq_element_array_bus_addr[RAID_PATH],
3988 &request.data.create_operational_iq.element_array_addr);
3989 put_unaligned_le64((u64)queue_group->iq_ci_bus_addr[RAID_PATH],
3990 &request.data.create_operational_iq.ci_addr);
3991 put_unaligned_le16(ctrl_info->num_elements_per_iq,
3992 &request.data.create_operational_iq.num_elements);
3993 put_unaligned_le16(PQI_OPERATIONAL_IQ_ELEMENT_LENGTH / 16,
3994 &request.data.create_operational_iq.element_length);
3995 request.data.create_operational_iq.queue_protocol = PQI_PROTOCOL_SOP;
3996
3997 rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,
3998 &response);
3999 if (rc) {
4000 dev_err(&ctrl_info->pci_dev->dev,
4001 "error creating inbound RAID queue\n");
4002 return rc;
4003 }
4004
4005 queue_group->iq_pi[RAID_PATH] = ctrl_info->iomem_base +
4006 PQI_DEVICE_REGISTERS_OFFSET +
4007 get_unaligned_le64(
4008 &response.data.create_operational_iq.iq_pi_offset);
4009
4010 /*
4011 * Create IQ (Inbound Queue - host to device queue) for
4012 * Advanced I/O (AIO) path.
4013 */
4014 memset(&request, 0, sizeof(request));
4015 request.header.iu_type = PQI_REQUEST_IU_GENERAL_ADMIN;
4016 put_unaligned_le16(PQI_GENERAL_ADMIN_IU_LENGTH,
4017 &request.header.iu_length);
4018 request.function_code = PQI_GENERAL_ADMIN_FUNCTION_CREATE_IQ;
4019 put_unaligned_le16(queue_group->iq_id[AIO_PATH],
4020 &request.data.create_operational_iq.queue_id);
4021 put_unaligned_le64((u64)queue_group->
4022 iq_element_array_bus_addr[AIO_PATH],
4023 &request.data.create_operational_iq.element_array_addr);
4024 put_unaligned_le64((u64)queue_group->iq_ci_bus_addr[AIO_PATH],
4025 &request.data.create_operational_iq.ci_addr);
4026 put_unaligned_le16(ctrl_info->num_elements_per_iq,
4027 &request.data.create_operational_iq.num_elements);
4028 put_unaligned_le16(PQI_OPERATIONAL_IQ_ELEMENT_LENGTH / 16,
4029 &request.data.create_operational_iq.element_length);
4030 request.data.create_operational_iq.queue_protocol = PQI_PROTOCOL_SOP;
4031
4032 rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,
4033 &response);
4034 if (rc) {
4035 dev_err(&ctrl_info->pci_dev->dev,
4036 "error creating inbound AIO queue\n");
4037 goto delete_inbound_queue_raid;
4038 }
4039
4040 queue_group->iq_pi[AIO_PATH] = ctrl_info->iomem_base +
4041 PQI_DEVICE_REGISTERS_OFFSET +
4042 get_unaligned_le64(
4043 &response.data.create_operational_iq.iq_pi_offset);
4044
4045 /*
4046 * Designate the 2nd IQ as the AIO path. By default, all IQs are
4047 * assumed to be for RAID path I/O unless we change the queue's
4048 * property.
4049 */
4050 memset(&request, 0, sizeof(request));
4051 request.header.iu_type = PQI_REQUEST_IU_GENERAL_ADMIN;
4052 put_unaligned_le16(PQI_GENERAL_ADMIN_IU_LENGTH,
4053 &request.header.iu_length);
4054 request.function_code = PQI_GENERAL_ADMIN_FUNCTION_CHANGE_IQ_PROPERTY;
4055 put_unaligned_le16(queue_group->iq_id[AIO_PATH],
4056 &request.data.change_operational_iq_properties.queue_id);
4057 put_unaligned_le32(PQI_IQ_PROPERTY_IS_AIO_QUEUE,
4058 &request.data.change_operational_iq_properties.vendor_specific);
4059
4060 rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,
4061 &response);
4062 if (rc) {
4063 dev_err(&ctrl_info->pci_dev->dev,
4064 "error changing queue property\n");
4065 goto delete_inbound_queue_aio;
4066 }
4067
4068 /*
4069 * Create OQ (Outbound Queue - device to host queue).
4070 */
4071 memset(&request, 0, sizeof(request));
4072 request.header.iu_type = PQI_REQUEST_IU_GENERAL_ADMIN;
4073 put_unaligned_le16(PQI_GENERAL_ADMIN_IU_LENGTH,
4074 &request.header.iu_length);
4075 request.function_code = PQI_GENERAL_ADMIN_FUNCTION_CREATE_OQ;
4076 put_unaligned_le16(queue_group->oq_id,
4077 &request.data.create_operational_oq.queue_id);
4078 put_unaligned_le64((u64)queue_group->oq_element_array_bus_addr,
4079 &request.data.create_operational_oq.element_array_addr);
4080 put_unaligned_le64((u64)queue_group->oq_pi_bus_addr,
4081 &request.data.create_operational_oq.pi_addr);
4082 put_unaligned_le16(ctrl_info->num_elements_per_oq,
4083 &request.data.create_operational_oq.num_elements);
4084 put_unaligned_le16(PQI_OPERATIONAL_OQ_ELEMENT_LENGTH / 16,
4085 &request.data.create_operational_oq.element_length);
4086 request.data.create_operational_oq.queue_protocol = PQI_PROTOCOL_SOP;
4087 put_unaligned_le16(queue_group->int_msg_num,
4088 &request.data.create_operational_oq.int_msg_num);
4089
4090 rc = pqi_submit_admin_request_synchronous(ctrl_info, &request,
4091 &response);
4092 if (rc) {
4093 dev_err(&ctrl_info->pci_dev->dev,
4094 "error creating outbound queue\n");
4095 goto delete_inbound_queue_aio;
4096 }
4097
4098 queue_group->oq_ci = ctrl_info->iomem_base +
4099 PQI_DEVICE_REGISTERS_OFFSET +
4100 get_unaligned_le64(
4101 &response.data.create_operational_oq.oq_ci_offset);
4102
Kevin Barnett6c223762016-06-27 16:41:00 -05004103 return 0;
4104
4105delete_inbound_queue_aio:
4106 pqi_delete_operational_queue(ctrl_info, true,
4107 queue_group->iq_id[AIO_PATH]);
4108
4109delete_inbound_queue_raid:
4110 pqi_delete_operational_queue(ctrl_info, true,
4111 queue_group->iq_id[RAID_PATH]);
4112
4113 return rc;
4114}
4115
4116static int pqi_create_queues(struct pqi_ctrl_info *ctrl_info)
4117{
4118 int rc;
4119 unsigned int i;
4120
4121 rc = pqi_create_event_queue(ctrl_info);
4122 if (rc) {
4123 dev_err(&ctrl_info->pci_dev->dev,
4124 "error creating event queue\n");
4125 return rc;
4126 }
4127
4128 for (i = 0; i < ctrl_info->num_queue_groups; i++) {
Kevin Barnett061ef062017-05-03 18:53:05 -05004129 rc = pqi_create_queue_group(ctrl_info, i);
Kevin Barnett6c223762016-06-27 16:41:00 -05004130 if (rc) {
4131 dev_err(&ctrl_info->pci_dev->dev,
4132 "error creating queue group number %u/%u\n",
4133 i, ctrl_info->num_queue_groups);
4134 return rc;
4135 }
4136 }
4137
4138 return 0;
4139}
4140
4141#define PQI_REPORT_EVENT_CONFIG_BUFFER_LENGTH \
4142 (offsetof(struct pqi_event_config, descriptors) + \
4143 (PQI_MAX_EVENT_DESCRIPTORS * sizeof(struct pqi_event_descriptor)))
4144
Kevin Barnett6a50d6a2017-05-03 18:52:52 -05004145static int pqi_configure_events(struct pqi_ctrl_info *ctrl_info,
4146 bool enable_events)
Kevin Barnett6c223762016-06-27 16:41:00 -05004147{
4148 int rc;
4149 unsigned int i;
4150 struct pqi_event_config *event_config;
Kevin Barnett6a50d6a2017-05-03 18:52:52 -05004151 struct pqi_event_descriptor *event_descriptor;
Kevin Barnett6c223762016-06-27 16:41:00 -05004152 struct pqi_general_management_request request;
4153
4154 event_config = kmalloc(PQI_REPORT_EVENT_CONFIG_BUFFER_LENGTH,
4155 GFP_KERNEL);
4156 if (!event_config)
4157 return -ENOMEM;
4158
4159 memset(&request, 0, sizeof(request));
4160
4161 request.header.iu_type = PQI_REQUEST_IU_REPORT_VENDOR_EVENT_CONFIG;
4162 put_unaligned_le16(offsetof(struct pqi_general_management_request,
4163 data.report_event_configuration.sg_descriptors[1]) -
4164 PQI_REQUEST_HEADER_LENGTH, &request.header.iu_length);
4165 put_unaligned_le32(PQI_REPORT_EVENT_CONFIG_BUFFER_LENGTH,
4166 &request.data.report_event_configuration.buffer_length);
4167
4168 rc = pqi_map_single(ctrl_info->pci_dev,
4169 request.data.report_event_configuration.sg_descriptors,
4170 event_config, PQI_REPORT_EVENT_CONFIG_BUFFER_LENGTH,
4171 PCI_DMA_FROMDEVICE);
4172 if (rc)
4173 goto out;
4174
4175 rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header,
4176 0, NULL, NO_TIMEOUT);
4177
4178 pqi_pci_unmap(ctrl_info->pci_dev,
4179 request.data.report_event_configuration.sg_descriptors, 1,
4180 PCI_DMA_FROMDEVICE);
4181
4182 if (rc)
4183 goto out;
4184
Kevin Barnett6a50d6a2017-05-03 18:52:52 -05004185 for (i = 0; i < event_config->num_event_descriptors; i++) {
4186 event_descriptor = &event_config->descriptors[i];
4187 if (enable_events &&
4188 pqi_is_supported_event(event_descriptor->event_type))
4189 put_unaligned_le16(ctrl_info->event_queue.oq_id,
4190 &event_descriptor->oq_id);
4191 else
4192 put_unaligned_le16(0, &event_descriptor->oq_id);
4193 }
Kevin Barnett6c223762016-06-27 16:41:00 -05004194
4195 memset(&request, 0, sizeof(request));
4196
4197 request.header.iu_type = PQI_REQUEST_IU_SET_VENDOR_EVENT_CONFIG;
4198 put_unaligned_le16(offsetof(struct pqi_general_management_request,
4199 data.report_event_configuration.sg_descriptors[1]) -
4200 PQI_REQUEST_HEADER_LENGTH, &request.header.iu_length);
4201 put_unaligned_le32(PQI_REPORT_EVENT_CONFIG_BUFFER_LENGTH,
4202 &request.data.report_event_configuration.buffer_length);
4203
4204 rc = pqi_map_single(ctrl_info->pci_dev,
4205 request.data.report_event_configuration.sg_descriptors,
4206 event_config, PQI_REPORT_EVENT_CONFIG_BUFFER_LENGTH,
4207 PCI_DMA_TODEVICE);
4208 if (rc)
4209 goto out;
4210
4211 rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header, 0,
4212 NULL, NO_TIMEOUT);
4213
4214 pqi_pci_unmap(ctrl_info->pci_dev,
4215 request.data.report_event_configuration.sg_descriptors, 1,
4216 PCI_DMA_TODEVICE);
4217
4218out:
4219 kfree(event_config);
4220
4221 return rc;
4222}
4223
Kevin Barnett6a50d6a2017-05-03 18:52:52 -05004224static inline int pqi_enable_events(struct pqi_ctrl_info *ctrl_info)
4225{
4226 return pqi_configure_events(ctrl_info, true);
4227}
4228
4229static inline int pqi_disable_events(struct pqi_ctrl_info *ctrl_info)
4230{
4231 return pqi_configure_events(ctrl_info, false);
4232}
4233
Kevin Barnett6c223762016-06-27 16:41:00 -05004234static void pqi_free_all_io_requests(struct pqi_ctrl_info *ctrl_info)
4235{
4236 unsigned int i;
4237 struct device *dev;
4238 size_t sg_chain_buffer_length;
4239 struct pqi_io_request *io_request;
4240
4241 if (!ctrl_info->io_request_pool)
4242 return;
4243
4244 dev = &ctrl_info->pci_dev->dev;
4245 sg_chain_buffer_length = ctrl_info->sg_chain_buffer_length;
4246 io_request = ctrl_info->io_request_pool;
4247
4248 for (i = 0; i < ctrl_info->max_io_slots; i++) {
4249 kfree(io_request->iu);
4250 if (!io_request->sg_chain_buffer)
4251 break;
4252 dma_free_coherent(dev, sg_chain_buffer_length,
4253 io_request->sg_chain_buffer,
4254 io_request->sg_chain_buffer_dma_handle);
4255 io_request++;
4256 }
4257
4258 kfree(ctrl_info->io_request_pool);
4259 ctrl_info->io_request_pool = NULL;
4260}
4261
4262static inline int pqi_alloc_error_buffer(struct pqi_ctrl_info *ctrl_info)
4263{
4264 ctrl_info->error_buffer = dma_zalloc_coherent(&ctrl_info->pci_dev->dev,
4265 ctrl_info->error_buffer_length,
4266 &ctrl_info->error_buffer_dma_handle, GFP_KERNEL);
4267
4268 if (!ctrl_info->error_buffer)
4269 return -ENOMEM;
4270
4271 return 0;
4272}
4273
4274static int pqi_alloc_io_resources(struct pqi_ctrl_info *ctrl_info)
4275{
4276 unsigned int i;
4277 void *sg_chain_buffer;
4278 size_t sg_chain_buffer_length;
4279 dma_addr_t sg_chain_buffer_dma_handle;
4280 struct device *dev;
4281 struct pqi_io_request *io_request;
4282
4283 ctrl_info->io_request_pool = kzalloc(ctrl_info->max_io_slots *
4284 sizeof(ctrl_info->io_request_pool[0]), GFP_KERNEL);
4285
4286 if (!ctrl_info->io_request_pool) {
4287 dev_err(&ctrl_info->pci_dev->dev,
4288 "failed to allocate I/O request pool\n");
4289 goto error;
4290 }
4291
4292 dev = &ctrl_info->pci_dev->dev;
4293 sg_chain_buffer_length = ctrl_info->sg_chain_buffer_length;
4294 io_request = ctrl_info->io_request_pool;
4295
4296 for (i = 0; i < ctrl_info->max_io_slots; i++) {
4297 io_request->iu =
4298 kmalloc(ctrl_info->max_inbound_iu_length, GFP_KERNEL);
4299
4300 if (!io_request->iu) {
4301 dev_err(&ctrl_info->pci_dev->dev,
4302 "failed to allocate IU buffers\n");
4303 goto error;
4304 }
4305
4306 sg_chain_buffer = dma_alloc_coherent(dev,
4307 sg_chain_buffer_length, &sg_chain_buffer_dma_handle,
4308 GFP_KERNEL);
4309
4310 if (!sg_chain_buffer) {
4311 dev_err(&ctrl_info->pci_dev->dev,
4312 "failed to allocate PQI scatter-gather chain buffers\n");
4313 goto error;
4314 }
4315
4316 io_request->index = i;
4317 io_request->sg_chain_buffer = sg_chain_buffer;
4318 io_request->sg_chain_buffer_dma_handle =
4319 sg_chain_buffer_dma_handle;
4320 io_request++;
4321 }
4322
4323 return 0;
4324
4325error:
4326 pqi_free_all_io_requests(ctrl_info);
4327
4328 return -ENOMEM;
4329}
4330
4331/*
4332 * Calculate required resources that are sized based on max. outstanding
4333 * requests and max. transfer size.
4334 */
4335
4336static void pqi_calculate_io_resources(struct pqi_ctrl_info *ctrl_info)
4337{
4338 u32 max_transfer_size;
4339 u32 max_sg_entries;
4340
4341 ctrl_info->scsi_ml_can_queue =
4342 ctrl_info->max_outstanding_requests - PQI_RESERVED_IO_SLOTS;
4343 ctrl_info->max_io_slots = ctrl_info->max_outstanding_requests;
4344
4345 ctrl_info->error_buffer_length =
4346 ctrl_info->max_io_slots * PQI_ERROR_BUFFER_ELEMENT_LENGTH;
4347
Kevin Barnettd727a772017-05-03 18:54:25 -05004348 if (reset_devices)
4349 max_transfer_size = min(ctrl_info->max_transfer_size,
4350 PQI_MAX_TRANSFER_SIZE_KDUMP);
4351 else
4352 max_transfer_size = min(ctrl_info->max_transfer_size,
4353 PQI_MAX_TRANSFER_SIZE);
Kevin Barnett6c223762016-06-27 16:41:00 -05004354
4355 max_sg_entries = max_transfer_size / PAGE_SIZE;
4356
4357 /* +1 to cover when the buffer is not page-aligned. */
4358 max_sg_entries++;
4359
4360 max_sg_entries = min(ctrl_info->max_sg_entries, max_sg_entries);
4361
4362 max_transfer_size = (max_sg_entries - 1) * PAGE_SIZE;
4363
4364 ctrl_info->sg_chain_buffer_length =
Kevin Barnette1d213b2017-05-03 18:53:18 -05004365 (max_sg_entries * sizeof(struct pqi_sg_descriptor)) +
4366 PQI_EXTRA_SGL_MEMORY;
Kevin Barnett6c223762016-06-27 16:41:00 -05004367 ctrl_info->sg_tablesize = max_sg_entries;
4368 ctrl_info->max_sectors = max_transfer_size / 512;
4369}
4370
4371static void pqi_calculate_queue_resources(struct pqi_ctrl_info *ctrl_info)
4372{
Kevin Barnett6c223762016-06-27 16:41:00 -05004373 int num_queue_groups;
4374 u16 num_elements_per_iq;
4375 u16 num_elements_per_oq;
4376
Kevin Barnettd727a772017-05-03 18:54:25 -05004377 if (reset_devices) {
4378 num_queue_groups = 1;
4379 } else {
4380 int num_cpus;
4381 int max_queue_groups;
Kevin Barnett6c223762016-06-27 16:41:00 -05004382
Kevin Barnettd727a772017-05-03 18:54:25 -05004383 max_queue_groups = min(ctrl_info->max_inbound_queues / 2,
4384 ctrl_info->max_outbound_queues - 1);
4385 max_queue_groups = min(max_queue_groups, PQI_MAX_QUEUE_GROUPS);
4386
4387 num_cpus = num_online_cpus();
4388 num_queue_groups = min(num_cpus, ctrl_info->max_msix_vectors);
4389 num_queue_groups = min(num_queue_groups, max_queue_groups);
4390 }
Kevin Barnett6c223762016-06-27 16:41:00 -05004391
4392 ctrl_info->num_queue_groups = num_queue_groups;
Kevin Barnett061ef062017-05-03 18:53:05 -05004393 ctrl_info->max_hw_queue_index = num_queue_groups - 1;
Kevin Barnett6c223762016-06-27 16:41:00 -05004394
Kevin Barnett77668f42016-08-31 14:54:23 -05004395 /*
4396 * Make sure that the max. inbound IU length is an even multiple
4397 * of our inbound element length.
4398 */
4399 ctrl_info->max_inbound_iu_length =
4400 (ctrl_info->max_inbound_iu_length_per_firmware /
4401 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH) *
4402 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH;
Kevin Barnett6c223762016-06-27 16:41:00 -05004403
4404 num_elements_per_iq =
4405 (ctrl_info->max_inbound_iu_length /
4406 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
4407
4408 /* Add one because one element in each queue is unusable. */
4409 num_elements_per_iq++;
4410
4411 num_elements_per_iq = min(num_elements_per_iq,
4412 ctrl_info->max_elements_per_iq);
4413
4414 num_elements_per_oq = ((num_elements_per_iq - 1) * 2) + 1;
4415 num_elements_per_oq = min(num_elements_per_oq,
4416 ctrl_info->max_elements_per_oq);
4417
4418 ctrl_info->num_elements_per_iq = num_elements_per_iq;
4419 ctrl_info->num_elements_per_oq = num_elements_per_oq;
4420
4421 ctrl_info->max_sg_per_iu =
4422 ((ctrl_info->max_inbound_iu_length -
4423 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH) /
4424 sizeof(struct pqi_sg_descriptor)) +
4425 PQI_MAX_EMBEDDED_SG_DESCRIPTORS;
4426}
4427
4428static inline void pqi_set_sg_descriptor(
4429 struct pqi_sg_descriptor *sg_descriptor, struct scatterlist *sg)
4430{
4431 u64 address = (u64)sg_dma_address(sg);
4432 unsigned int length = sg_dma_len(sg);
4433
4434 put_unaligned_le64(address, &sg_descriptor->address);
4435 put_unaligned_le32(length, &sg_descriptor->length);
4436 put_unaligned_le32(0, &sg_descriptor->flags);
4437}
4438
4439static int pqi_build_raid_sg_list(struct pqi_ctrl_info *ctrl_info,
4440 struct pqi_raid_path_request *request, struct scsi_cmnd *scmd,
4441 struct pqi_io_request *io_request)
4442{
4443 int i;
4444 u16 iu_length;
4445 int sg_count;
4446 bool chained;
4447 unsigned int num_sg_in_iu;
4448 unsigned int max_sg_per_iu;
4449 struct scatterlist *sg;
4450 struct pqi_sg_descriptor *sg_descriptor;
4451
4452 sg_count = scsi_dma_map(scmd);
4453 if (sg_count < 0)
4454 return sg_count;
4455
4456 iu_length = offsetof(struct pqi_raid_path_request, sg_descriptors) -
4457 PQI_REQUEST_HEADER_LENGTH;
4458
4459 if (sg_count == 0)
4460 goto out;
4461
4462 sg = scsi_sglist(scmd);
4463 sg_descriptor = request->sg_descriptors;
4464 max_sg_per_iu = ctrl_info->max_sg_per_iu - 1;
4465 chained = false;
4466 num_sg_in_iu = 0;
4467 i = 0;
4468
4469 while (1) {
4470 pqi_set_sg_descriptor(sg_descriptor, sg);
4471 if (!chained)
4472 num_sg_in_iu++;
4473 i++;
4474 if (i == sg_count)
4475 break;
4476 sg_descriptor++;
4477 if (i == max_sg_per_iu) {
4478 put_unaligned_le64(
4479 (u64)io_request->sg_chain_buffer_dma_handle,
4480 &sg_descriptor->address);
4481 put_unaligned_le32((sg_count - num_sg_in_iu)
4482 * sizeof(*sg_descriptor),
4483 &sg_descriptor->length);
4484 put_unaligned_le32(CISS_SG_CHAIN,
4485 &sg_descriptor->flags);
4486 chained = true;
4487 num_sg_in_iu++;
4488 sg_descriptor = io_request->sg_chain_buffer;
4489 }
4490 sg = sg_next(sg);
4491 }
4492
4493 put_unaligned_le32(CISS_SG_LAST, &sg_descriptor->flags);
4494 request->partial = chained;
4495 iu_length += num_sg_in_iu * sizeof(*sg_descriptor);
4496
4497out:
4498 put_unaligned_le16(iu_length, &request->header.iu_length);
4499
4500 return 0;
4501}
4502
4503static int pqi_build_aio_sg_list(struct pqi_ctrl_info *ctrl_info,
4504 struct pqi_aio_path_request *request, struct scsi_cmnd *scmd,
4505 struct pqi_io_request *io_request)
4506{
4507 int i;
4508 u16 iu_length;
4509 int sg_count;
Kevin Barnetta60eec02016-08-31 14:54:11 -05004510 bool chained;
4511 unsigned int num_sg_in_iu;
4512 unsigned int max_sg_per_iu;
Kevin Barnett6c223762016-06-27 16:41:00 -05004513 struct scatterlist *sg;
4514 struct pqi_sg_descriptor *sg_descriptor;
4515
4516 sg_count = scsi_dma_map(scmd);
4517 if (sg_count < 0)
4518 return sg_count;
Kevin Barnetta60eec02016-08-31 14:54:11 -05004519
4520 iu_length = offsetof(struct pqi_aio_path_request, sg_descriptors) -
4521 PQI_REQUEST_HEADER_LENGTH;
4522 num_sg_in_iu = 0;
4523
Kevin Barnett6c223762016-06-27 16:41:00 -05004524 if (sg_count == 0)
4525 goto out;
4526
Kevin Barnetta60eec02016-08-31 14:54:11 -05004527 sg = scsi_sglist(scmd);
4528 sg_descriptor = request->sg_descriptors;
4529 max_sg_per_iu = ctrl_info->max_sg_per_iu - 1;
4530 chained = false;
4531 i = 0;
Kevin Barnett6c223762016-06-27 16:41:00 -05004532
Kevin Barnetta60eec02016-08-31 14:54:11 -05004533 while (1) {
4534 pqi_set_sg_descriptor(sg_descriptor, sg);
4535 if (!chained)
4536 num_sg_in_iu++;
4537 i++;
4538 if (i == sg_count)
4539 break;
4540 sg_descriptor++;
4541 if (i == max_sg_per_iu) {
4542 put_unaligned_le64(
4543 (u64)io_request->sg_chain_buffer_dma_handle,
4544 &sg_descriptor->address);
4545 put_unaligned_le32((sg_count - num_sg_in_iu)
4546 * sizeof(*sg_descriptor),
4547 &sg_descriptor->length);
4548 put_unaligned_le32(CISS_SG_CHAIN,
4549 &sg_descriptor->flags);
4550 chained = true;
4551 num_sg_in_iu++;
4552 sg_descriptor = io_request->sg_chain_buffer;
Kevin Barnett6c223762016-06-27 16:41:00 -05004553 }
Kevin Barnetta60eec02016-08-31 14:54:11 -05004554 sg = sg_next(sg);
Kevin Barnett6c223762016-06-27 16:41:00 -05004555 }
4556
Kevin Barnetta60eec02016-08-31 14:54:11 -05004557 put_unaligned_le32(CISS_SG_LAST, &sg_descriptor->flags);
4558 request->partial = chained;
Kevin Barnett6c223762016-06-27 16:41:00 -05004559 iu_length += num_sg_in_iu * sizeof(*sg_descriptor);
Kevin Barnetta60eec02016-08-31 14:54:11 -05004560
4561out:
Kevin Barnett6c223762016-06-27 16:41:00 -05004562 put_unaligned_le16(iu_length, &request->header.iu_length);
4563 request->num_sg_descriptors = num_sg_in_iu;
4564
4565 return 0;
4566}
4567
4568static void pqi_raid_io_complete(struct pqi_io_request *io_request,
4569 void *context)
4570{
4571 struct scsi_cmnd *scmd;
4572
4573 scmd = io_request->scmd;
4574 pqi_free_io_request(io_request);
4575 scsi_dma_unmap(scmd);
4576 pqi_scsi_done(scmd);
4577}
4578
Kevin Barnett376fb882017-05-03 18:54:43 -05004579static int pqi_raid_submit_scsi_cmd_with_io_request(
4580 struct pqi_ctrl_info *ctrl_info, struct pqi_io_request *io_request,
Kevin Barnett6c223762016-06-27 16:41:00 -05004581 struct pqi_scsi_dev *device, struct scsi_cmnd *scmd,
4582 struct pqi_queue_group *queue_group)
4583{
4584 int rc;
4585 size_t cdb_length;
Kevin Barnett6c223762016-06-27 16:41:00 -05004586 struct pqi_raid_path_request *request;
4587
Kevin Barnett6c223762016-06-27 16:41:00 -05004588 io_request->io_complete_callback = pqi_raid_io_complete;
4589 io_request->scmd = scmd;
4590
Kevin Barnett6c223762016-06-27 16:41:00 -05004591 request = io_request->iu;
4592 memset(request, 0,
4593 offsetof(struct pqi_raid_path_request, sg_descriptors));
4594
4595 request->header.iu_type = PQI_REQUEST_IU_RAID_PATH_IO;
4596 put_unaligned_le32(scsi_bufflen(scmd), &request->buffer_length);
4597 request->task_attribute = SOP_TASK_ATTRIBUTE_SIMPLE;
4598 put_unaligned_le16(io_request->index, &request->request_id);
4599 request->error_index = request->request_id;
4600 memcpy(request->lun_number, device->scsi3addr,
4601 sizeof(request->lun_number));
4602
4603 cdb_length = min_t(size_t, scmd->cmd_len, sizeof(request->cdb));
4604 memcpy(request->cdb, scmd->cmnd, cdb_length);
4605
4606 switch (cdb_length) {
4607 case 6:
4608 case 10:
4609 case 12:
4610 case 16:
4611 /* No bytes in the Additional CDB bytes field */
4612 request->additional_cdb_bytes_usage =
4613 SOP_ADDITIONAL_CDB_BYTES_0;
4614 break;
4615 case 20:
4616 /* 4 bytes in the Additional cdb field */
4617 request->additional_cdb_bytes_usage =
4618 SOP_ADDITIONAL_CDB_BYTES_4;
4619 break;
4620 case 24:
4621 /* 8 bytes in the Additional cdb field */
4622 request->additional_cdb_bytes_usage =
4623 SOP_ADDITIONAL_CDB_BYTES_8;
4624 break;
4625 case 28:
4626 /* 12 bytes in the Additional cdb field */
4627 request->additional_cdb_bytes_usage =
4628 SOP_ADDITIONAL_CDB_BYTES_12;
4629 break;
4630 case 32:
4631 default:
4632 /* 16 bytes in the Additional cdb field */
4633 request->additional_cdb_bytes_usage =
4634 SOP_ADDITIONAL_CDB_BYTES_16;
4635 break;
4636 }
4637
4638 switch (scmd->sc_data_direction) {
4639 case DMA_TO_DEVICE:
4640 request->data_direction = SOP_READ_FLAG;
4641 break;
4642 case DMA_FROM_DEVICE:
4643 request->data_direction = SOP_WRITE_FLAG;
4644 break;
4645 case DMA_NONE:
4646 request->data_direction = SOP_NO_DIRECTION_FLAG;
4647 break;
4648 case DMA_BIDIRECTIONAL:
4649 request->data_direction = SOP_BIDIRECTIONAL;
4650 break;
4651 default:
4652 dev_err(&ctrl_info->pci_dev->dev,
4653 "unknown data direction: %d\n",
4654 scmd->sc_data_direction);
Kevin Barnett6c223762016-06-27 16:41:00 -05004655 break;
4656 }
4657
4658 rc = pqi_build_raid_sg_list(ctrl_info, request, scmd, io_request);
4659 if (rc) {
4660 pqi_free_io_request(io_request);
4661 return SCSI_MLQUEUE_HOST_BUSY;
4662 }
4663
4664 pqi_start_io(ctrl_info, queue_group, RAID_PATH, io_request);
4665
4666 return 0;
4667}
4668
Kevin Barnett376fb882017-05-03 18:54:43 -05004669static inline int pqi_raid_submit_scsi_cmd(struct pqi_ctrl_info *ctrl_info,
4670 struct pqi_scsi_dev *device, struct scsi_cmnd *scmd,
4671 struct pqi_queue_group *queue_group)
4672{
4673 struct pqi_io_request *io_request;
4674
4675 io_request = pqi_alloc_io_request(ctrl_info);
4676
4677 return pqi_raid_submit_scsi_cmd_with_io_request(ctrl_info, io_request,
4678 device, scmd, queue_group);
4679}
4680
4681static inline void pqi_schedule_bypass_retry(struct pqi_ctrl_info *ctrl_info)
4682{
4683 if (!pqi_ctrl_blocked(ctrl_info))
4684 schedule_work(&ctrl_info->raid_bypass_retry_work);
4685}
4686
4687static bool pqi_raid_bypass_retry_needed(struct pqi_io_request *io_request)
4688{
4689 struct scsi_cmnd *scmd;
Kevin Barnett03b288cf2017-05-03 18:54:49 -05004690 struct pqi_scsi_dev *device;
Kevin Barnett376fb882017-05-03 18:54:43 -05004691 struct pqi_ctrl_info *ctrl_info;
4692
4693 if (!io_request->raid_bypass)
4694 return false;
4695
4696 scmd = io_request->scmd;
4697 if ((scmd->result & 0xff) == SAM_STAT_GOOD)
4698 return false;
4699 if (host_byte(scmd->result) == DID_NO_CONNECT)
4700 return false;
4701
Kevin Barnett03b288cf2017-05-03 18:54:49 -05004702 device = scmd->device->hostdata;
4703 if (pqi_device_offline(device))
4704 return false;
4705
Kevin Barnett376fb882017-05-03 18:54:43 -05004706 ctrl_info = shost_to_hba(scmd->device->host);
4707 if (pqi_ctrl_offline(ctrl_info))
4708 return false;
4709
4710 return true;
4711}
4712
4713static inline void pqi_add_to_raid_bypass_retry_list(
4714 struct pqi_ctrl_info *ctrl_info,
4715 struct pqi_io_request *io_request, bool at_head)
4716{
4717 unsigned long flags;
4718
4719 spin_lock_irqsave(&ctrl_info->raid_bypass_retry_list_lock, flags);
4720 if (at_head)
4721 list_add(&io_request->request_list_entry,
4722 &ctrl_info->raid_bypass_retry_list);
4723 else
4724 list_add_tail(&io_request->request_list_entry,
4725 &ctrl_info->raid_bypass_retry_list);
4726 spin_unlock_irqrestore(&ctrl_info->raid_bypass_retry_list_lock, flags);
4727}
4728
4729static void pqi_queued_raid_bypass_complete(struct pqi_io_request *io_request,
4730 void *context)
4731{
4732 struct scsi_cmnd *scmd;
4733
4734 scmd = io_request->scmd;
4735 pqi_free_io_request(io_request);
4736 pqi_scsi_done(scmd);
4737}
4738
4739static void pqi_queue_raid_bypass_retry(struct pqi_io_request *io_request)
4740{
4741 struct scsi_cmnd *scmd;
4742 struct pqi_ctrl_info *ctrl_info;
4743
4744 io_request->io_complete_callback = pqi_queued_raid_bypass_complete;
4745 scmd = io_request->scmd;
4746 scmd->result = 0;
4747 ctrl_info = shost_to_hba(scmd->device->host);
4748
4749 pqi_add_to_raid_bypass_retry_list(ctrl_info, io_request, false);
4750 pqi_schedule_bypass_retry(ctrl_info);
4751}
4752
4753static int pqi_retry_raid_bypass(struct pqi_io_request *io_request)
4754{
4755 struct scsi_cmnd *scmd;
4756 struct pqi_scsi_dev *device;
4757 struct pqi_ctrl_info *ctrl_info;
4758 struct pqi_queue_group *queue_group;
4759
4760 scmd = io_request->scmd;
4761 device = scmd->device->hostdata;
4762 if (pqi_device_in_reset(device)) {
4763 pqi_free_io_request(io_request);
4764 set_host_byte(scmd, DID_RESET);
4765 pqi_scsi_done(scmd);
4766 return 0;
4767 }
4768
4769 ctrl_info = shost_to_hba(scmd->device->host);
4770 queue_group = io_request->queue_group;
4771
4772 pqi_reinit_io_request(io_request);
4773
4774 return pqi_raid_submit_scsi_cmd_with_io_request(ctrl_info, io_request,
4775 device, scmd, queue_group);
4776}
4777
4778static inline struct pqi_io_request *pqi_next_queued_raid_bypass_request(
4779 struct pqi_ctrl_info *ctrl_info)
4780{
4781 unsigned long flags;
4782 struct pqi_io_request *io_request;
4783
4784 spin_lock_irqsave(&ctrl_info->raid_bypass_retry_list_lock, flags);
4785 io_request = list_first_entry_or_null(
4786 &ctrl_info->raid_bypass_retry_list,
4787 struct pqi_io_request, request_list_entry);
4788 if (io_request)
4789 list_del(&io_request->request_list_entry);
4790 spin_unlock_irqrestore(&ctrl_info->raid_bypass_retry_list_lock, flags);
4791
4792 return io_request;
4793}
4794
4795static void pqi_retry_raid_bypass_requests(struct pqi_ctrl_info *ctrl_info)
4796{
4797 int rc;
4798 struct pqi_io_request *io_request;
4799
4800 pqi_ctrl_busy(ctrl_info);
4801
4802 while (1) {
4803 if (pqi_ctrl_blocked(ctrl_info))
4804 break;
4805 io_request = pqi_next_queued_raid_bypass_request(ctrl_info);
4806 if (!io_request)
4807 break;
4808 rc = pqi_retry_raid_bypass(io_request);
4809 if (rc) {
4810 pqi_add_to_raid_bypass_retry_list(ctrl_info, io_request,
4811 true);
4812 pqi_schedule_bypass_retry(ctrl_info);
4813 break;
4814 }
4815 }
4816
4817 pqi_ctrl_unbusy(ctrl_info);
4818}
4819
4820static void pqi_raid_bypass_retry_worker(struct work_struct *work)
4821{
4822 struct pqi_ctrl_info *ctrl_info;
4823
4824 ctrl_info = container_of(work, struct pqi_ctrl_info,
4825 raid_bypass_retry_work);
4826 pqi_retry_raid_bypass_requests(ctrl_info);
4827}
4828
Kevin Barnett5f310422017-05-03 18:54:55 -05004829static void pqi_clear_all_queued_raid_bypass_retries(
4830 struct pqi_ctrl_info *ctrl_info)
Kevin Barnett376fb882017-05-03 18:54:43 -05004831{
4832 unsigned long flags;
Kevin Barnett376fb882017-05-03 18:54:43 -05004833
4834 spin_lock_irqsave(&ctrl_info->raid_bypass_retry_list_lock, flags);
Kevin Barnett5f310422017-05-03 18:54:55 -05004835 INIT_LIST_HEAD(&ctrl_info->raid_bypass_retry_list);
Kevin Barnett376fb882017-05-03 18:54:43 -05004836 spin_unlock_irqrestore(&ctrl_info->raid_bypass_retry_list_lock, flags);
4837}
4838
Kevin Barnett6c223762016-06-27 16:41:00 -05004839static void pqi_aio_io_complete(struct pqi_io_request *io_request,
4840 void *context)
4841{
4842 struct scsi_cmnd *scmd;
4843
4844 scmd = io_request->scmd;
4845 scsi_dma_unmap(scmd);
4846 if (io_request->status == -EAGAIN)
4847 set_host_byte(scmd, DID_IMM_RETRY);
Kevin Barnett376fb882017-05-03 18:54:43 -05004848 else if (pqi_raid_bypass_retry_needed(io_request)) {
4849 pqi_queue_raid_bypass_retry(io_request);
4850 return;
4851 }
Kevin Barnett6c223762016-06-27 16:41:00 -05004852 pqi_free_io_request(io_request);
4853 pqi_scsi_done(scmd);
4854}
4855
4856static inline int pqi_aio_submit_scsi_cmd(struct pqi_ctrl_info *ctrl_info,
4857 struct pqi_scsi_dev *device, struct scsi_cmnd *scmd,
4858 struct pqi_queue_group *queue_group)
4859{
4860 return pqi_aio_submit_io(ctrl_info, scmd, device->aio_handle,
Kevin Barnett376fb882017-05-03 18:54:43 -05004861 scmd->cmnd, scmd->cmd_len, queue_group, NULL, false);
Kevin Barnett6c223762016-06-27 16:41:00 -05004862}
4863
4864static int pqi_aio_submit_io(struct pqi_ctrl_info *ctrl_info,
4865 struct scsi_cmnd *scmd, u32 aio_handle, u8 *cdb,
4866 unsigned int cdb_length, struct pqi_queue_group *queue_group,
Kevin Barnett376fb882017-05-03 18:54:43 -05004867 struct pqi_encryption_info *encryption_info, bool raid_bypass)
Kevin Barnett6c223762016-06-27 16:41:00 -05004868{
4869 int rc;
4870 struct pqi_io_request *io_request;
4871 struct pqi_aio_path_request *request;
4872
4873 io_request = pqi_alloc_io_request(ctrl_info);
4874 io_request->io_complete_callback = pqi_aio_io_complete;
4875 io_request->scmd = scmd;
Kevin Barnett376fb882017-05-03 18:54:43 -05004876 io_request->raid_bypass = raid_bypass;
Kevin Barnett6c223762016-06-27 16:41:00 -05004877
4878 request = io_request->iu;
4879 memset(request, 0,
4880 offsetof(struct pqi_raid_path_request, sg_descriptors));
4881
4882 request->header.iu_type = PQI_REQUEST_IU_AIO_PATH_IO;
4883 put_unaligned_le32(aio_handle, &request->nexus_id);
4884 put_unaligned_le32(scsi_bufflen(scmd), &request->buffer_length);
4885 request->task_attribute = SOP_TASK_ATTRIBUTE_SIMPLE;
4886 put_unaligned_le16(io_request->index, &request->request_id);
4887 request->error_index = request->request_id;
4888 if (cdb_length > sizeof(request->cdb))
4889 cdb_length = sizeof(request->cdb);
4890 request->cdb_length = cdb_length;
4891 memcpy(request->cdb, cdb, cdb_length);
4892
4893 switch (scmd->sc_data_direction) {
4894 case DMA_TO_DEVICE:
4895 request->data_direction = SOP_READ_FLAG;
4896 break;
4897 case DMA_FROM_DEVICE:
4898 request->data_direction = SOP_WRITE_FLAG;
4899 break;
4900 case DMA_NONE:
4901 request->data_direction = SOP_NO_DIRECTION_FLAG;
4902 break;
4903 case DMA_BIDIRECTIONAL:
4904 request->data_direction = SOP_BIDIRECTIONAL;
4905 break;
4906 default:
4907 dev_err(&ctrl_info->pci_dev->dev,
4908 "unknown data direction: %d\n",
4909 scmd->sc_data_direction);
Kevin Barnett6c223762016-06-27 16:41:00 -05004910 break;
4911 }
4912
4913 if (encryption_info) {
4914 request->encryption_enable = true;
4915 put_unaligned_le16(encryption_info->data_encryption_key_index,
4916 &request->data_encryption_key_index);
4917 put_unaligned_le32(encryption_info->encrypt_tweak_lower,
4918 &request->encrypt_tweak_lower);
4919 put_unaligned_le32(encryption_info->encrypt_tweak_upper,
4920 &request->encrypt_tweak_upper);
4921 }
4922
4923 rc = pqi_build_aio_sg_list(ctrl_info, request, scmd, io_request);
4924 if (rc) {
4925 pqi_free_io_request(io_request);
4926 return SCSI_MLQUEUE_HOST_BUSY;
4927 }
4928
4929 pqi_start_io(ctrl_info, queue_group, AIO_PATH, io_request);
4930
4931 return 0;
4932}
4933
Kevin Barnett061ef062017-05-03 18:53:05 -05004934static inline u16 pqi_get_hw_queue(struct pqi_ctrl_info *ctrl_info,
4935 struct scsi_cmnd *scmd)
4936{
4937 u16 hw_queue;
4938
4939 hw_queue = blk_mq_unique_tag_to_hwq(blk_mq_unique_tag(scmd->request));
4940 if (hw_queue > ctrl_info->max_hw_queue_index)
4941 hw_queue = 0;
4942
4943 return hw_queue;
4944}
4945
Kevin Barnett7561a7e2017-05-03 18:52:58 -05004946/*
4947 * This function gets called just before we hand the completed SCSI request
4948 * back to the SML.
4949 */
4950
4951void pqi_prep_for_scsi_done(struct scsi_cmnd *scmd)
4952{
4953 struct pqi_scsi_dev *device;
4954
4955 device = scmd->device->hostdata;
4956 atomic_dec(&device->scsi_cmds_outstanding);
4957}
4958
Kevin Barnett6c223762016-06-27 16:41:00 -05004959static int pqi_scsi_queue_command(struct Scsi_Host *shost,
Kevin Barnett7d81d2b2016-08-31 14:55:11 -05004960 struct scsi_cmnd *scmd)
Kevin Barnett6c223762016-06-27 16:41:00 -05004961{
4962 int rc;
4963 struct pqi_ctrl_info *ctrl_info;
4964 struct pqi_scsi_dev *device;
Kevin Barnett061ef062017-05-03 18:53:05 -05004965 u16 hw_queue;
Kevin Barnett6c223762016-06-27 16:41:00 -05004966 struct pqi_queue_group *queue_group;
4967 bool raid_bypassed;
4968
4969 device = scmd->device->hostdata;
Kevin Barnett6c223762016-06-27 16:41:00 -05004970 ctrl_info = shost_to_hba(shost);
4971
Kevin Barnett7561a7e2017-05-03 18:52:58 -05004972 atomic_inc(&device->scsi_cmds_outstanding);
4973
Kevin Barnett6c223762016-06-27 16:41:00 -05004974 if (pqi_ctrl_offline(ctrl_info)) {
4975 set_host_byte(scmd, DID_NO_CONNECT);
4976 pqi_scsi_done(scmd);
4977 return 0;
4978 }
4979
Kevin Barnett7561a7e2017-05-03 18:52:58 -05004980 pqi_ctrl_busy(ctrl_info);
4981 if (pqi_ctrl_blocked(ctrl_info) || pqi_device_in_reset(device)) {
4982 rc = SCSI_MLQUEUE_HOST_BUSY;
4983 goto out;
4984 }
4985
Kevin Barnett7d81d2b2016-08-31 14:55:11 -05004986 /*
4987 * This is necessary because the SML doesn't zero out this field during
4988 * error recovery.
4989 */
4990 scmd->result = 0;
4991
Kevin Barnett061ef062017-05-03 18:53:05 -05004992 hw_queue = pqi_get_hw_queue(ctrl_info, scmd);
4993 queue_group = &ctrl_info->queue_groups[hw_queue];
Kevin Barnett6c223762016-06-27 16:41:00 -05004994
4995 if (pqi_is_logical_device(device)) {
4996 raid_bypassed = false;
Kevin Barnett588a63fe2017-05-03 18:55:25 -05004997 if (device->raid_bypass_enabled &&
Christoph Hellwig57292b52017-01-31 16:57:29 +01004998 !blk_rq_is_passthrough(scmd->request)) {
Kevin Barnett6c223762016-06-27 16:41:00 -05004999 rc = pqi_raid_bypass_submit_scsi_cmd(ctrl_info, device,
5000 scmd, queue_group);
Kevin Barnett376fb882017-05-03 18:54:43 -05005001 if (rc == 0 || rc == SCSI_MLQUEUE_HOST_BUSY)
5002 raid_bypassed = true;
Kevin Barnett6c223762016-06-27 16:41:00 -05005003 }
5004 if (!raid_bypassed)
5005 rc = pqi_raid_submit_scsi_cmd(ctrl_info, device, scmd,
5006 queue_group);
5007 } else {
5008 if (device->aio_enabled)
5009 rc = pqi_aio_submit_scsi_cmd(ctrl_info, device, scmd,
5010 queue_group);
5011 else
5012 rc = pqi_raid_submit_scsi_cmd(ctrl_info, device, scmd,
5013 queue_group);
5014 }
5015
Kevin Barnett7561a7e2017-05-03 18:52:58 -05005016out:
5017 pqi_ctrl_unbusy(ctrl_info);
5018 if (rc)
5019 atomic_dec(&device->scsi_cmds_outstanding);
5020
Kevin Barnett6c223762016-06-27 16:41:00 -05005021 return rc;
5022}
5023
Kevin Barnett7561a7e2017-05-03 18:52:58 -05005024static int pqi_wait_until_queued_io_drained(struct pqi_ctrl_info *ctrl_info,
5025 struct pqi_queue_group *queue_group)
5026{
5027 unsigned int path;
5028 unsigned long flags;
5029 bool list_is_empty;
5030
5031 for (path = 0; path < 2; path++) {
5032 while (1) {
5033 spin_lock_irqsave(
5034 &queue_group->submit_lock[path], flags);
5035 list_is_empty =
5036 list_empty(&queue_group->request_list[path]);
5037 spin_unlock_irqrestore(
5038 &queue_group->submit_lock[path], flags);
5039 if (list_is_empty)
5040 break;
5041 pqi_check_ctrl_health(ctrl_info);
5042 if (pqi_ctrl_offline(ctrl_info))
5043 return -ENXIO;
5044 usleep_range(1000, 2000);
5045 }
5046 }
5047
5048 return 0;
5049}
5050
5051static int pqi_wait_until_inbound_queues_empty(struct pqi_ctrl_info *ctrl_info)
5052{
5053 int rc;
5054 unsigned int i;
5055 unsigned int path;
5056 struct pqi_queue_group *queue_group;
5057 pqi_index_t iq_pi;
5058 pqi_index_t iq_ci;
5059
5060 for (i = 0; i < ctrl_info->num_queue_groups; i++) {
5061 queue_group = &ctrl_info->queue_groups[i];
5062
5063 rc = pqi_wait_until_queued_io_drained(ctrl_info, queue_group);
5064 if (rc)
5065 return rc;
5066
5067 for (path = 0; path < 2; path++) {
5068 iq_pi = queue_group->iq_pi_copy[path];
5069
5070 while (1) {
5071 iq_ci = *queue_group->iq_ci[path];
5072 if (iq_ci == iq_pi)
5073 break;
5074 pqi_check_ctrl_health(ctrl_info);
5075 if (pqi_ctrl_offline(ctrl_info))
5076 return -ENXIO;
5077 usleep_range(1000, 2000);
5078 }
5079 }
5080 }
5081
5082 return 0;
5083}
5084
5085static void pqi_fail_io_queued_for_device(struct pqi_ctrl_info *ctrl_info,
5086 struct pqi_scsi_dev *device)
5087{
5088 unsigned int i;
5089 unsigned int path;
5090 struct pqi_queue_group *queue_group;
5091 unsigned long flags;
5092 struct pqi_io_request *io_request;
5093 struct pqi_io_request *next;
5094 struct scsi_cmnd *scmd;
5095 struct pqi_scsi_dev *scsi_device;
5096
5097 for (i = 0; i < ctrl_info->num_queue_groups; i++) {
5098 queue_group = &ctrl_info->queue_groups[i];
5099
5100 for (path = 0; path < 2; path++) {
5101 spin_lock_irqsave(
5102 &queue_group->submit_lock[path], flags);
5103
5104 list_for_each_entry_safe(io_request, next,
5105 &queue_group->request_list[path],
5106 request_list_entry) {
5107 scmd = io_request->scmd;
5108 if (!scmd)
5109 continue;
5110
5111 scsi_device = scmd->device->hostdata;
5112 if (scsi_device != device)
5113 continue;
5114
5115 list_del(&io_request->request_list_entry);
5116 set_host_byte(scmd, DID_RESET);
5117 pqi_scsi_done(scmd);
5118 }
5119
5120 spin_unlock_irqrestore(
5121 &queue_group->submit_lock[path], flags);
5122 }
5123 }
5124}
5125
Kevin Barnett061ef062017-05-03 18:53:05 -05005126static int pqi_device_wait_for_pending_io(struct pqi_ctrl_info *ctrl_info,
5127 struct pqi_scsi_dev *device)
5128{
5129 while (atomic_read(&device->scsi_cmds_outstanding)) {
5130 pqi_check_ctrl_health(ctrl_info);
5131 if (pqi_ctrl_offline(ctrl_info))
5132 return -ENXIO;
5133 usleep_range(1000, 2000);
5134 }
5135
5136 return 0;
5137}
5138
5139static int pqi_ctrl_wait_for_pending_io(struct pqi_ctrl_info *ctrl_info)
5140{
5141 bool io_pending;
5142 unsigned long flags;
5143 struct pqi_scsi_dev *device;
5144
5145 while (1) {
5146 io_pending = false;
5147
5148 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
5149 list_for_each_entry(device, &ctrl_info->scsi_device_list,
5150 scsi_device_list_entry) {
5151 if (atomic_read(&device->scsi_cmds_outstanding)) {
5152 io_pending = true;
5153 break;
5154 }
5155 }
5156 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock,
5157 flags);
5158
5159 if (!io_pending)
5160 break;
5161
5162 pqi_check_ctrl_health(ctrl_info);
5163 if (pqi_ctrl_offline(ctrl_info))
5164 return -ENXIO;
5165
5166 usleep_range(1000, 2000);
5167 }
5168
5169 return 0;
5170}
5171
Kevin Barnett14bb2152016-08-31 14:54:35 -05005172static void pqi_lun_reset_complete(struct pqi_io_request *io_request,
Kevin Barnett6c223762016-06-27 16:41:00 -05005173 void *context)
5174{
5175 struct completion *waiting = context;
5176
5177 complete(waiting);
5178}
5179
Kevin Barnett14bb2152016-08-31 14:54:35 -05005180#define PQI_LUN_RESET_TIMEOUT_SECS 10
5181
5182static int pqi_wait_for_lun_reset_completion(struct pqi_ctrl_info *ctrl_info,
5183 struct pqi_scsi_dev *device, struct completion *wait)
5184{
5185 int rc;
Kevin Barnett14bb2152016-08-31 14:54:35 -05005186
5187 while (1) {
5188 if (wait_for_completion_io_timeout(wait,
5189 PQI_LUN_RESET_TIMEOUT_SECS * HZ)) {
5190 rc = 0;
5191 break;
5192 }
5193
5194 pqi_check_ctrl_health(ctrl_info);
5195 if (pqi_ctrl_offline(ctrl_info)) {
Kevin Barnett4e8415e2017-05-03 18:54:18 -05005196 rc = -ENXIO;
Kevin Barnett14bb2152016-08-31 14:54:35 -05005197 break;
5198 }
Kevin Barnett14bb2152016-08-31 14:54:35 -05005199 }
5200
5201 return rc;
5202}
5203
5204static int pqi_lun_reset(struct pqi_ctrl_info *ctrl_info,
Kevin Barnett6c223762016-06-27 16:41:00 -05005205 struct pqi_scsi_dev *device)
5206{
5207 int rc;
5208 struct pqi_io_request *io_request;
5209 DECLARE_COMPLETION_ONSTACK(wait);
5210 struct pqi_task_management_request *request;
5211
Kevin Barnett6c223762016-06-27 16:41:00 -05005212 io_request = pqi_alloc_io_request(ctrl_info);
Kevin Barnett14bb2152016-08-31 14:54:35 -05005213 io_request->io_complete_callback = pqi_lun_reset_complete;
Kevin Barnett6c223762016-06-27 16:41:00 -05005214 io_request->context = &wait;
5215
5216 request = io_request->iu;
5217 memset(request, 0, sizeof(*request));
5218
5219 request->header.iu_type = PQI_REQUEST_IU_TASK_MANAGEMENT;
5220 put_unaligned_le16(sizeof(*request) - PQI_REQUEST_HEADER_LENGTH,
5221 &request->header.iu_length);
5222 put_unaligned_le16(io_request->index, &request->request_id);
5223 memcpy(request->lun_number, device->scsi3addr,
5224 sizeof(request->lun_number));
5225 request->task_management_function = SOP_TASK_MANAGEMENT_LUN_RESET;
5226
5227 pqi_start_io(ctrl_info,
5228 &ctrl_info->queue_groups[PQI_DEFAULT_QUEUE_GROUP], RAID_PATH,
5229 io_request);
5230
Kevin Barnett14bb2152016-08-31 14:54:35 -05005231 rc = pqi_wait_for_lun_reset_completion(ctrl_info, device, &wait);
5232 if (rc == 0)
Kevin Barnett6c223762016-06-27 16:41:00 -05005233 rc = io_request->status;
Kevin Barnett6c223762016-06-27 16:41:00 -05005234
5235 pqi_free_io_request(io_request);
Kevin Barnett6c223762016-06-27 16:41:00 -05005236
5237 return rc;
5238}
5239
5240/* Performs a reset at the LUN level. */
5241
5242static int pqi_device_reset(struct pqi_ctrl_info *ctrl_info,
5243 struct pqi_scsi_dev *device)
5244{
5245 int rc;
5246
Kevin Barnett14bb2152016-08-31 14:54:35 -05005247 rc = pqi_lun_reset(ctrl_info, device);
Kevin Barnett061ef062017-05-03 18:53:05 -05005248 if (rc == 0)
5249 rc = pqi_device_wait_for_pending_io(ctrl_info, device);
Kevin Barnett6c223762016-06-27 16:41:00 -05005250
Kevin Barnett14bb2152016-08-31 14:54:35 -05005251 return rc == 0 ? SUCCESS : FAILED;
Kevin Barnett6c223762016-06-27 16:41:00 -05005252}
5253
5254static int pqi_eh_device_reset_handler(struct scsi_cmnd *scmd)
5255{
5256 int rc;
Kevin Barnett7561a7e2017-05-03 18:52:58 -05005257 struct Scsi_Host *shost;
Kevin Barnett6c223762016-06-27 16:41:00 -05005258 struct pqi_ctrl_info *ctrl_info;
5259 struct pqi_scsi_dev *device;
5260
Kevin Barnett7561a7e2017-05-03 18:52:58 -05005261 shost = scmd->device->host;
5262 ctrl_info = shost_to_hba(shost);
Kevin Barnett6c223762016-06-27 16:41:00 -05005263 device = scmd->device->hostdata;
5264
5265 dev_err(&ctrl_info->pci_dev->dev,
5266 "resetting scsi %d:%d:%d:%d\n",
Kevin Barnett7561a7e2017-05-03 18:52:58 -05005267 shost->host_no, device->bus, device->target, device->lun);
Kevin Barnett6c223762016-06-27 16:41:00 -05005268
Kevin Barnett7561a7e2017-05-03 18:52:58 -05005269 pqi_check_ctrl_health(ctrl_info);
5270 if (pqi_ctrl_offline(ctrl_info)) {
5271 rc = FAILED;
5272 goto out;
5273 }
Kevin Barnett6c223762016-06-27 16:41:00 -05005274
Kevin Barnett7561a7e2017-05-03 18:52:58 -05005275 mutex_lock(&ctrl_info->lun_reset_mutex);
5276
5277 pqi_ctrl_block_requests(ctrl_info);
5278 pqi_ctrl_wait_until_quiesced(ctrl_info);
5279 pqi_fail_io_queued_for_device(ctrl_info, device);
5280 rc = pqi_wait_until_inbound_queues_empty(ctrl_info);
5281 pqi_device_reset_start(device);
5282 pqi_ctrl_unblock_requests(ctrl_info);
5283
5284 if (rc)
5285 rc = FAILED;
5286 else
5287 rc = pqi_device_reset(ctrl_info, device);
5288
5289 pqi_device_reset_done(device);
5290
5291 mutex_unlock(&ctrl_info->lun_reset_mutex);
5292
5293out:
Kevin Barnett6c223762016-06-27 16:41:00 -05005294 dev_err(&ctrl_info->pci_dev->dev,
5295 "reset of scsi %d:%d:%d:%d: %s\n",
Kevin Barnett7561a7e2017-05-03 18:52:58 -05005296 shost->host_no, device->bus, device->target, device->lun,
Kevin Barnett6c223762016-06-27 16:41:00 -05005297 rc == SUCCESS ? "SUCCESS" : "FAILED");
5298
5299 return rc;
5300}
5301
5302static int pqi_slave_alloc(struct scsi_device *sdev)
5303{
5304 struct pqi_scsi_dev *device;
5305 unsigned long flags;
5306 struct pqi_ctrl_info *ctrl_info;
5307 struct scsi_target *starget;
5308 struct sas_rphy *rphy;
5309
5310 ctrl_info = shost_to_hba(sdev->host);
5311
5312 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
5313
5314 if (sdev_channel(sdev) == PQI_PHYSICAL_DEVICE_BUS) {
5315 starget = scsi_target(sdev);
5316 rphy = target_to_rphy(starget);
5317 device = pqi_find_device_by_sas_rphy(ctrl_info, rphy);
5318 if (device) {
5319 device->target = sdev_id(sdev);
5320 device->lun = sdev->lun;
5321 device->target_lun_valid = true;
5322 }
5323 } else {
5324 device = pqi_find_scsi_dev(ctrl_info, sdev_channel(sdev),
5325 sdev_id(sdev), sdev->lun);
5326 }
5327
Kevin Barnett94086f52017-05-03 18:54:31 -05005328 if (device) {
Kevin Barnett6c223762016-06-27 16:41:00 -05005329 sdev->hostdata = device;
5330 device->sdev = sdev;
5331 if (device->queue_depth) {
5332 device->advertised_queue_depth = device->queue_depth;
5333 scsi_change_queue_depth(sdev,
5334 device->advertised_queue_depth);
5335 }
5336 }
5337
5338 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
5339
5340 return 0;
5341}
5342
Christoph Hellwig52198222016-11-01 08:12:49 -06005343static int pqi_map_queues(struct Scsi_Host *shost)
5344{
5345 struct pqi_ctrl_info *ctrl_info = shost_to_hba(shost);
5346
5347 return blk_mq_pci_map_queues(&shost->tag_set, ctrl_info->pci_dev);
5348}
5349
Kevin Barnett6c223762016-06-27 16:41:00 -05005350static int pqi_getpciinfo_ioctl(struct pqi_ctrl_info *ctrl_info,
5351 void __user *arg)
5352{
5353 struct pci_dev *pci_dev;
5354 u32 subsystem_vendor;
5355 u32 subsystem_device;
5356 cciss_pci_info_struct pciinfo;
5357
5358 if (!arg)
5359 return -EINVAL;
5360
5361 pci_dev = ctrl_info->pci_dev;
5362
5363 pciinfo.domain = pci_domain_nr(pci_dev->bus);
5364 pciinfo.bus = pci_dev->bus->number;
5365 pciinfo.dev_fn = pci_dev->devfn;
5366 subsystem_vendor = pci_dev->subsystem_vendor;
5367 subsystem_device = pci_dev->subsystem_device;
5368 pciinfo.board_id = ((subsystem_device << 16) & 0xffff0000) |
5369 subsystem_vendor;
5370
5371 if (copy_to_user(arg, &pciinfo, sizeof(pciinfo)))
5372 return -EFAULT;
5373
5374 return 0;
5375}
5376
5377static int pqi_getdrivver_ioctl(void __user *arg)
5378{
5379 u32 version;
5380
5381 if (!arg)
5382 return -EINVAL;
5383
5384 version = (DRIVER_MAJOR << 28) | (DRIVER_MINOR << 24) |
5385 (DRIVER_RELEASE << 16) | DRIVER_REVISION;
5386
5387 if (copy_to_user(arg, &version, sizeof(version)))
5388 return -EFAULT;
5389
5390 return 0;
5391}
5392
5393struct ciss_error_info {
5394 u8 scsi_status;
5395 int command_status;
5396 size_t sense_data_length;
5397};
5398
5399static void pqi_error_info_to_ciss(struct pqi_raid_error_info *pqi_error_info,
5400 struct ciss_error_info *ciss_error_info)
5401{
5402 int ciss_cmd_status;
5403 size_t sense_data_length;
5404
5405 switch (pqi_error_info->data_out_result) {
5406 case PQI_DATA_IN_OUT_GOOD:
5407 ciss_cmd_status = CISS_CMD_STATUS_SUCCESS;
5408 break;
5409 case PQI_DATA_IN_OUT_UNDERFLOW:
5410 ciss_cmd_status = CISS_CMD_STATUS_DATA_UNDERRUN;
5411 break;
5412 case PQI_DATA_IN_OUT_BUFFER_OVERFLOW:
5413 ciss_cmd_status = CISS_CMD_STATUS_DATA_OVERRUN;
5414 break;
5415 case PQI_DATA_IN_OUT_PROTOCOL_ERROR:
5416 case PQI_DATA_IN_OUT_BUFFER_ERROR:
5417 case PQI_DATA_IN_OUT_BUFFER_OVERFLOW_DESCRIPTOR_AREA:
5418 case PQI_DATA_IN_OUT_BUFFER_OVERFLOW_BRIDGE:
5419 case PQI_DATA_IN_OUT_ERROR:
5420 ciss_cmd_status = CISS_CMD_STATUS_PROTOCOL_ERROR;
5421 break;
5422 case PQI_DATA_IN_OUT_HARDWARE_ERROR:
5423 case PQI_DATA_IN_OUT_PCIE_FABRIC_ERROR:
5424 case PQI_DATA_IN_OUT_PCIE_COMPLETION_TIMEOUT:
5425 case PQI_DATA_IN_OUT_PCIE_COMPLETER_ABORT_RECEIVED:
5426 case PQI_DATA_IN_OUT_PCIE_UNSUPPORTED_REQUEST_RECEIVED:
5427 case PQI_DATA_IN_OUT_PCIE_ECRC_CHECK_FAILED:
5428 case PQI_DATA_IN_OUT_PCIE_UNSUPPORTED_REQUEST:
5429 case PQI_DATA_IN_OUT_PCIE_ACS_VIOLATION:
5430 case PQI_DATA_IN_OUT_PCIE_TLP_PREFIX_BLOCKED:
5431 case PQI_DATA_IN_OUT_PCIE_POISONED_MEMORY_READ:
5432 ciss_cmd_status = CISS_CMD_STATUS_HARDWARE_ERROR;
5433 break;
5434 case PQI_DATA_IN_OUT_UNSOLICITED_ABORT:
5435 ciss_cmd_status = CISS_CMD_STATUS_UNSOLICITED_ABORT;
5436 break;
5437 case PQI_DATA_IN_OUT_ABORTED:
5438 ciss_cmd_status = CISS_CMD_STATUS_ABORTED;
5439 break;
5440 case PQI_DATA_IN_OUT_TIMEOUT:
5441 ciss_cmd_status = CISS_CMD_STATUS_TIMEOUT;
5442 break;
5443 default:
5444 ciss_cmd_status = CISS_CMD_STATUS_TARGET_STATUS;
5445 break;
5446 }
5447
5448 sense_data_length =
5449 get_unaligned_le16(&pqi_error_info->sense_data_length);
5450 if (sense_data_length == 0)
5451 sense_data_length =
5452 get_unaligned_le16(&pqi_error_info->response_data_length);
5453 if (sense_data_length)
5454 if (sense_data_length > sizeof(pqi_error_info->data))
5455 sense_data_length = sizeof(pqi_error_info->data);
5456
5457 ciss_error_info->scsi_status = pqi_error_info->status;
5458 ciss_error_info->command_status = ciss_cmd_status;
5459 ciss_error_info->sense_data_length = sense_data_length;
5460}
5461
5462static int pqi_passthru_ioctl(struct pqi_ctrl_info *ctrl_info, void __user *arg)
5463{
5464 int rc;
5465 char *kernel_buffer = NULL;
5466 u16 iu_length;
5467 size_t sense_data_length;
5468 IOCTL_Command_struct iocommand;
5469 struct pqi_raid_path_request request;
5470 struct pqi_raid_error_info pqi_error_info;
5471 struct ciss_error_info ciss_error_info;
5472
5473 if (pqi_ctrl_offline(ctrl_info))
5474 return -ENXIO;
5475 if (!arg)
5476 return -EINVAL;
5477 if (!capable(CAP_SYS_RAWIO))
5478 return -EPERM;
5479 if (copy_from_user(&iocommand, arg, sizeof(iocommand)))
5480 return -EFAULT;
5481 if (iocommand.buf_size < 1 &&
5482 iocommand.Request.Type.Direction != XFER_NONE)
5483 return -EINVAL;
5484 if (iocommand.Request.CDBLen > sizeof(request.cdb))
5485 return -EINVAL;
5486 if (iocommand.Request.Type.Type != TYPE_CMD)
5487 return -EINVAL;
5488
5489 switch (iocommand.Request.Type.Direction) {
5490 case XFER_NONE:
5491 case XFER_WRITE:
5492 case XFER_READ:
5493 break;
5494 default:
5495 return -EINVAL;
5496 }
5497
5498 if (iocommand.buf_size > 0) {
5499 kernel_buffer = kmalloc(iocommand.buf_size, GFP_KERNEL);
5500 if (!kernel_buffer)
5501 return -ENOMEM;
5502 if (iocommand.Request.Type.Direction & XFER_WRITE) {
5503 if (copy_from_user(kernel_buffer, iocommand.buf,
5504 iocommand.buf_size)) {
5505 rc = -EFAULT;
5506 goto out;
5507 }
5508 } else {
5509 memset(kernel_buffer, 0, iocommand.buf_size);
5510 }
5511 }
5512
5513 memset(&request, 0, sizeof(request));
5514
5515 request.header.iu_type = PQI_REQUEST_IU_RAID_PATH_IO;
5516 iu_length = offsetof(struct pqi_raid_path_request, sg_descriptors) -
5517 PQI_REQUEST_HEADER_LENGTH;
5518 memcpy(request.lun_number, iocommand.LUN_info.LunAddrBytes,
5519 sizeof(request.lun_number));
5520 memcpy(request.cdb, iocommand.Request.CDB, iocommand.Request.CDBLen);
5521 request.additional_cdb_bytes_usage = SOP_ADDITIONAL_CDB_BYTES_0;
5522
5523 switch (iocommand.Request.Type.Direction) {
5524 case XFER_NONE:
5525 request.data_direction = SOP_NO_DIRECTION_FLAG;
5526 break;
5527 case XFER_WRITE:
5528 request.data_direction = SOP_WRITE_FLAG;
5529 break;
5530 case XFER_READ:
5531 request.data_direction = SOP_READ_FLAG;
5532 break;
5533 }
5534
5535 request.task_attribute = SOP_TASK_ATTRIBUTE_SIMPLE;
5536
5537 if (iocommand.buf_size > 0) {
5538 put_unaligned_le32(iocommand.buf_size, &request.buffer_length);
5539
5540 rc = pqi_map_single(ctrl_info->pci_dev,
5541 &request.sg_descriptors[0], kernel_buffer,
5542 iocommand.buf_size, PCI_DMA_BIDIRECTIONAL);
5543 if (rc)
5544 goto out;
5545
5546 iu_length += sizeof(request.sg_descriptors[0]);
5547 }
5548
5549 put_unaligned_le16(iu_length, &request.header.iu_length);
5550
5551 rc = pqi_submit_raid_request_synchronous(ctrl_info, &request.header,
5552 PQI_SYNC_FLAGS_INTERRUPTABLE, &pqi_error_info, NO_TIMEOUT);
5553
5554 if (iocommand.buf_size > 0)
5555 pqi_pci_unmap(ctrl_info->pci_dev, request.sg_descriptors, 1,
5556 PCI_DMA_BIDIRECTIONAL);
5557
5558 memset(&iocommand.error_info, 0, sizeof(iocommand.error_info));
5559
5560 if (rc == 0) {
5561 pqi_error_info_to_ciss(&pqi_error_info, &ciss_error_info);
5562 iocommand.error_info.ScsiStatus = ciss_error_info.scsi_status;
5563 iocommand.error_info.CommandStatus =
5564 ciss_error_info.command_status;
5565 sense_data_length = ciss_error_info.sense_data_length;
5566 if (sense_data_length) {
5567 if (sense_data_length >
5568 sizeof(iocommand.error_info.SenseInfo))
5569 sense_data_length =
5570 sizeof(iocommand.error_info.SenseInfo);
5571 memcpy(iocommand.error_info.SenseInfo,
5572 pqi_error_info.data, sense_data_length);
5573 iocommand.error_info.SenseLen = sense_data_length;
5574 }
5575 }
5576
5577 if (copy_to_user(arg, &iocommand, sizeof(iocommand))) {
5578 rc = -EFAULT;
5579 goto out;
5580 }
5581
5582 if (rc == 0 && iocommand.buf_size > 0 &&
5583 (iocommand.Request.Type.Direction & XFER_READ)) {
5584 if (copy_to_user(iocommand.buf, kernel_buffer,
5585 iocommand.buf_size)) {
5586 rc = -EFAULT;
5587 }
5588 }
5589
5590out:
5591 kfree(kernel_buffer);
5592
5593 return rc;
5594}
5595
5596static int pqi_ioctl(struct scsi_device *sdev, int cmd, void __user *arg)
5597{
5598 int rc;
5599 struct pqi_ctrl_info *ctrl_info;
5600
5601 ctrl_info = shost_to_hba(sdev->host);
5602
5603 switch (cmd) {
5604 case CCISS_DEREGDISK:
5605 case CCISS_REGNEWDISK:
5606 case CCISS_REGNEWD:
5607 rc = pqi_scan_scsi_devices(ctrl_info);
5608 break;
5609 case CCISS_GETPCIINFO:
5610 rc = pqi_getpciinfo_ioctl(ctrl_info, arg);
5611 break;
5612 case CCISS_GETDRIVVER:
5613 rc = pqi_getdrivver_ioctl(arg);
5614 break;
5615 case CCISS_PASSTHRU:
5616 rc = pqi_passthru_ioctl(ctrl_info, arg);
5617 break;
5618 default:
5619 rc = -EINVAL;
5620 break;
5621 }
5622
5623 return rc;
5624}
5625
5626static ssize_t pqi_version_show(struct device *dev,
5627 struct device_attribute *attr, char *buffer)
5628{
5629 ssize_t count = 0;
5630 struct Scsi_Host *shost;
5631 struct pqi_ctrl_info *ctrl_info;
5632
5633 shost = class_to_shost(dev);
5634 ctrl_info = shost_to_hba(shost);
5635
5636 count += snprintf(buffer + count, PAGE_SIZE - count,
5637 " driver: %s\n", DRIVER_VERSION BUILD_TIMESTAMP);
5638
5639 count += snprintf(buffer + count, PAGE_SIZE - count,
5640 "firmware: %s\n", ctrl_info->firmware_version);
5641
5642 return count;
5643}
5644
5645static ssize_t pqi_host_rescan_store(struct device *dev,
5646 struct device_attribute *attr, const char *buffer, size_t count)
5647{
5648 struct Scsi_Host *shost = class_to_shost(dev);
5649
5650 pqi_scan_start(shost);
5651
5652 return count;
5653}
5654
Kevin Barnett3c509762017-05-03 18:54:37 -05005655static ssize_t pqi_lockup_action_show(struct device *dev,
5656 struct device_attribute *attr, char *buffer)
5657{
5658 int count = 0;
5659 unsigned int i;
5660
5661 for (i = 0; i < ARRAY_SIZE(pqi_lockup_actions); i++) {
5662 if (pqi_lockup_actions[i].action == pqi_lockup_action)
5663 count += snprintf(buffer + count, PAGE_SIZE - count,
5664 "[%s] ", pqi_lockup_actions[i].name);
5665 else
5666 count += snprintf(buffer + count, PAGE_SIZE - count,
5667 "%s ", pqi_lockup_actions[i].name);
5668 }
5669
5670 count += snprintf(buffer + count, PAGE_SIZE - count, "\n");
5671
5672 return count;
5673}
5674
5675static ssize_t pqi_lockup_action_store(struct device *dev,
5676 struct device_attribute *attr, const char *buffer, size_t count)
5677{
5678 unsigned int i;
5679 char *action_name;
5680 char action_name_buffer[32];
5681
5682 strlcpy(action_name_buffer, buffer, sizeof(action_name_buffer));
5683 action_name = strstrip(action_name_buffer);
5684
5685 for (i = 0; i < ARRAY_SIZE(pqi_lockup_actions); i++) {
5686 if (strcmp(action_name, pqi_lockup_actions[i].name) == 0) {
5687 pqi_lockup_action = pqi_lockup_actions[i].action;
5688 return count;
5689 }
5690 }
5691
5692 return -EINVAL;
5693}
5694
Kevin Barnettcbe0c7b2017-05-03 18:53:48 -05005695static DEVICE_ATTR(version, 0444, pqi_version_show, NULL);
5696static DEVICE_ATTR(rescan, 0200, NULL, pqi_host_rescan_store);
Kevin Barnett3c509762017-05-03 18:54:37 -05005697static DEVICE_ATTR(lockup_action, 0644,
5698 pqi_lockup_action_show, pqi_lockup_action_store);
Kevin Barnett6c223762016-06-27 16:41:00 -05005699
5700static struct device_attribute *pqi_shost_attrs[] = {
5701 &dev_attr_version,
5702 &dev_attr_rescan,
Kevin Barnett3c509762017-05-03 18:54:37 -05005703 &dev_attr_lockup_action,
Kevin Barnett6c223762016-06-27 16:41:00 -05005704 NULL
5705};
5706
5707static ssize_t pqi_sas_address_show(struct device *dev,
5708 struct device_attribute *attr, char *buffer)
5709{
5710 struct pqi_ctrl_info *ctrl_info;
5711 struct scsi_device *sdev;
5712 struct pqi_scsi_dev *device;
5713 unsigned long flags;
5714 u64 sas_address;
5715
5716 sdev = to_scsi_device(dev);
5717 ctrl_info = shost_to_hba(sdev->host);
5718
5719 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
5720
5721 device = sdev->hostdata;
5722 if (pqi_is_logical_device(device)) {
5723 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock,
5724 flags);
5725 return -ENODEV;
5726 }
5727 sas_address = device->sas_address;
5728
5729 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
5730
5731 return snprintf(buffer, PAGE_SIZE, "0x%016llx\n", sas_address);
5732}
5733
5734static ssize_t pqi_ssd_smart_path_enabled_show(struct device *dev,
5735 struct device_attribute *attr, char *buffer)
5736{
5737 struct pqi_ctrl_info *ctrl_info;
5738 struct scsi_device *sdev;
5739 struct pqi_scsi_dev *device;
5740 unsigned long flags;
5741
5742 sdev = to_scsi_device(dev);
5743 ctrl_info = shost_to_hba(sdev->host);
5744
5745 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
5746
5747 device = sdev->hostdata;
Kevin Barnett588a63fe2017-05-03 18:55:25 -05005748 buffer[0] = device->raid_bypass_enabled ? '1' : '0';
Kevin Barnett6c223762016-06-27 16:41:00 -05005749 buffer[1] = '\n';
5750 buffer[2] = '\0';
5751
5752 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
5753
5754 return 2;
5755}
5756
Kevin Barnetta9f93392017-05-03 18:55:31 -05005757static ssize_t pqi_raid_level_show(struct device *dev,
5758 struct device_attribute *attr, char *buffer)
5759{
5760 struct pqi_ctrl_info *ctrl_info;
5761 struct scsi_device *sdev;
5762 struct pqi_scsi_dev *device;
5763 unsigned long flags;
5764 char *raid_level;
5765
5766 sdev = to_scsi_device(dev);
5767 ctrl_info = shost_to_hba(sdev->host);
5768
5769 spin_lock_irqsave(&ctrl_info->scsi_device_list_lock, flags);
5770
5771 device = sdev->hostdata;
5772
5773 if (pqi_is_logical_device(device))
5774 raid_level = pqi_raid_level_to_string(device->raid_level);
5775 else
5776 raid_level = "N/A";
5777
5778 spin_unlock_irqrestore(&ctrl_info->scsi_device_list_lock, flags);
5779
5780 return snprintf(buffer, PAGE_SIZE, "%s\n", raid_level);
5781}
5782
Kevin Barnettcbe0c7b2017-05-03 18:53:48 -05005783static DEVICE_ATTR(sas_address, 0444, pqi_sas_address_show, NULL);
5784static DEVICE_ATTR(ssd_smart_path_enabled, 0444,
Kevin Barnett6c223762016-06-27 16:41:00 -05005785 pqi_ssd_smart_path_enabled_show, NULL);
Kevin Barnetta9f93392017-05-03 18:55:31 -05005786static DEVICE_ATTR(raid_level, 0444, pqi_raid_level_show, NULL);
Kevin Barnett6c223762016-06-27 16:41:00 -05005787
5788static struct device_attribute *pqi_sdev_attrs[] = {
5789 &dev_attr_sas_address,
5790 &dev_attr_ssd_smart_path_enabled,
Kevin Barnetta9f93392017-05-03 18:55:31 -05005791 &dev_attr_raid_level,
Kevin Barnett6c223762016-06-27 16:41:00 -05005792 NULL
5793};
5794
5795static struct scsi_host_template pqi_driver_template = {
5796 .module = THIS_MODULE,
5797 .name = DRIVER_NAME_SHORT,
5798 .proc_name = DRIVER_NAME_SHORT,
5799 .queuecommand = pqi_scsi_queue_command,
5800 .scan_start = pqi_scan_start,
5801 .scan_finished = pqi_scan_finished,
5802 .this_id = -1,
5803 .use_clustering = ENABLE_CLUSTERING,
5804 .eh_device_reset_handler = pqi_eh_device_reset_handler,
5805 .ioctl = pqi_ioctl,
5806 .slave_alloc = pqi_slave_alloc,
Christoph Hellwig52198222016-11-01 08:12:49 -06005807 .map_queues = pqi_map_queues,
Kevin Barnett6c223762016-06-27 16:41:00 -05005808 .sdev_attrs = pqi_sdev_attrs,
5809 .shost_attrs = pqi_shost_attrs,
5810};
5811
5812static int pqi_register_scsi(struct pqi_ctrl_info *ctrl_info)
5813{
5814 int rc;
5815 struct Scsi_Host *shost;
5816
5817 shost = scsi_host_alloc(&pqi_driver_template, sizeof(ctrl_info));
5818 if (!shost) {
5819 dev_err(&ctrl_info->pci_dev->dev,
5820 "scsi_host_alloc failed for controller %u\n",
5821 ctrl_info->ctrl_id);
5822 return -ENOMEM;
5823 }
5824
5825 shost->io_port = 0;
5826 shost->n_io_port = 0;
5827 shost->this_id = -1;
5828 shost->max_channel = PQI_MAX_BUS;
5829 shost->max_cmd_len = MAX_COMMAND_SIZE;
5830 shost->max_lun = ~0;
5831 shost->max_id = ~0;
5832 shost->max_sectors = ctrl_info->max_sectors;
5833 shost->can_queue = ctrl_info->scsi_ml_can_queue;
5834 shost->cmd_per_lun = shost->can_queue;
5835 shost->sg_tablesize = ctrl_info->sg_tablesize;
5836 shost->transportt = pqi_sas_transport_template;
Christoph Hellwig52198222016-11-01 08:12:49 -06005837 shost->irq = pci_irq_vector(ctrl_info->pci_dev, 0);
Kevin Barnett6c223762016-06-27 16:41:00 -05005838 shost->unique_id = shost->irq;
5839 shost->nr_hw_queues = ctrl_info->num_queue_groups;
5840 shost->hostdata[0] = (unsigned long)ctrl_info;
5841
5842 rc = scsi_add_host(shost, &ctrl_info->pci_dev->dev);
5843 if (rc) {
5844 dev_err(&ctrl_info->pci_dev->dev,
5845 "scsi_add_host failed for controller %u\n",
5846 ctrl_info->ctrl_id);
5847 goto free_host;
5848 }
5849
5850 rc = pqi_add_sas_host(shost, ctrl_info);
5851 if (rc) {
5852 dev_err(&ctrl_info->pci_dev->dev,
5853 "add SAS host failed for controller %u\n",
5854 ctrl_info->ctrl_id);
5855 goto remove_host;
5856 }
5857
5858 ctrl_info->scsi_host = shost;
5859
5860 return 0;
5861
5862remove_host:
5863 scsi_remove_host(shost);
5864free_host:
5865 scsi_host_put(shost);
5866
5867 return rc;
5868}
5869
5870static void pqi_unregister_scsi(struct pqi_ctrl_info *ctrl_info)
5871{
5872 struct Scsi_Host *shost;
5873
5874 pqi_delete_sas_host(ctrl_info);
5875
5876 shost = ctrl_info->scsi_host;
5877 if (!shost)
5878 return;
5879
5880 scsi_remove_host(shost);
5881 scsi_host_put(shost);
5882}
5883
5884#define PQI_RESET_ACTION_RESET 0x1
5885
5886#define PQI_RESET_TYPE_NO_RESET 0x0
5887#define PQI_RESET_TYPE_SOFT_RESET 0x1
5888#define PQI_RESET_TYPE_FIRM_RESET 0x2
5889#define PQI_RESET_TYPE_HARD_RESET 0x3
5890
5891static int pqi_reset(struct pqi_ctrl_info *ctrl_info)
5892{
5893 int rc;
5894 u32 reset_params;
5895
5896 reset_params = (PQI_RESET_ACTION_RESET << 5) |
5897 PQI_RESET_TYPE_HARD_RESET;
5898
5899 writel(reset_params,
5900 &ctrl_info->pqi_registers->device_reset);
5901
5902 rc = pqi_wait_for_pqi_mode_ready(ctrl_info);
5903 if (rc)
5904 dev_err(&ctrl_info->pci_dev->dev,
5905 "PQI reset failed\n");
5906
5907 return rc;
5908}
5909
5910static int pqi_get_ctrl_firmware_version(struct pqi_ctrl_info *ctrl_info)
5911{
5912 int rc;
5913 struct bmic_identify_controller *identify;
5914
5915 identify = kmalloc(sizeof(*identify), GFP_KERNEL);
5916 if (!identify)
5917 return -ENOMEM;
5918
5919 rc = pqi_identify_controller(ctrl_info, identify);
5920 if (rc)
5921 goto out;
5922
5923 memcpy(ctrl_info->firmware_version, identify->firmware_version,
5924 sizeof(identify->firmware_version));
5925 ctrl_info->firmware_version[sizeof(identify->firmware_version)] = '\0';
5926 snprintf(ctrl_info->firmware_version +
5927 strlen(ctrl_info->firmware_version),
5928 sizeof(ctrl_info->firmware_version),
5929 "-%u", get_unaligned_le16(&identify->firmware_build_number));
5930
5931out:
5932 kfree(identify);
5933
5934 return rc;
5935}
5936
Kevin Barnett98f87662017-05-03 18:53:11 -05005937static int pqi_process_config_table(struct pqi_ctrl_info *ctrl_info)
5938{
5939 u32 table_length;
5940 u32 section_offset;
5941 void __iomem *table_iomem_addr;
5942 struct pqi_config_table *config_table;
5943 struct pqi_config_table_section_header *section;
5944
5945 table_length = ctrl_info->config_table_length;
5946
5947 config_table = kmalloc(table_length, GFP_KERNEL);
5948 if (!config_table) {
5949 dev_err(&ctrl_info->pci_dev->dev,
Kevin Barnettd87d5472017-05-03 18:54:00 -05005950 "failed to allocate memory for PQI configuration table\n");
Kevin Barnett98f87662017-05-03 18:53:11 -05005951 return -ENOMEM;
5952 }
5953
5954 /*
5955 * Copy the config table contents from I/O memory space into the
5956 * temporary buffer.
5957 */
5958 table_iomem_addr = ctrl_info->iomem_base +
5959 ctrl_info->config_table_offset;
5960 memcpy_fromio(config_table, table_iomem_addr, table_length);
5961
5962 section_offset =
5963 get_unaligned_le32(&config_table->first_section_offset);
5964
5965 while (section_offset) {
5966 section = (void *)config_table + section_offset;
5967
5968 switch (get_unaligned_le16(&section->section_id)) {
5969 case PQI_CONFIG_TABLE_SECTION_HEARTBEAT:
5970 ctrl_info->heartbeat_counter = table_iomem_addr +
5971 section_offset +
5972 offsetof(struct pqi_config_table_heartbeat,
5973 heartbeat_counter);
5974 break;
5975 }
5976
5977 section_offset =
5978 get_unaligned_le16(&section->next_section_offset);
5979 }
5980
5981 kfree(config_table);
5982
5983 return 0;
5984}
5985
Kevin Barnett162d7752017-05-03 18:52:46 -05005986/* Switches the controller from PQI mode back into SIS mode. */
5987
5988static int pqi_revert_to_sis_mode(struct pqi_ctrl_info *ctrl_info)
5989{
5990 int rc;
5991
Kevin Barnett061ef062017-05-03 18:53:05 -05005992 pqi_change_irq_mode(ctrl_info, IRQ_MODE_NONE);
Kevin Barnett162d7752017-05-03 18:52:46 -05005993 rc = pqi_reset(ctrl_info);
5994 if (rc)
5995 return rc;
5996 sis_reenable_sis_mode(ctrl_info);
5997 pqi_save_ctrl_mode(ctrl_info, SIS_MODE);
5998
5999 return 0;
6000}
6001
6002/*
6003 * If the controller isn't already in SIS mode, this function forces it into
6004 * SIS mode.
6005 */
6006
6007static int pqi_force_sis_mode(struct pqi_ctrl_info *ctrl_info)
Kevin Barnettff6abb72016-08-31 14:54:41 -05006008{
6009 if (!sis_is_firmware_running(ctrl_info))
6010 return -ENXIO;
6011
Kevin Barnett162d7752017-05-03 18:52:46 -05006012 if (pqi_get_ctrl_mode(ctrl_info) == SIS_MODE)
6013 return 0;
6014
6015 if (sis_is_kernel_up(ctrl_info)) {
6016 pqi_save_ctrl_mode(ctrl_info, SIS_MODE);
6017 return 0;
Kevin Barnettff6abb72016-08-31 14:54:41 -05006018 }
6019
Kevin Barnett162d7752017-05-03 18:52:46 -05006020 return pqi_revert_to_sis_mode(ctrl_info);
Kevin Barnettff6abb72016-08-31 14:54:41 -05006021}
6022
Kevin Barnett6c223762016-06-27 16:41:00 -05006023static int pqi_ctrl_init(struct pqi_ctrl_info *ctrl_info)
6024{
6025 int rc;
6026
Kevin Barnett162d7752017-05-03 18:52:46 -05006027 rc = pqi_force_sis_mode(ctrl_info);
6028 if (rc)
6029 return rc;
Kevin Barnett6c223762016-06-27 16:41:00 -05006030
6031 /*
6032 * Wait until the controller is ready to start accepting SIS
6033 * commands.
6034 */
6035 rc = sis_wait_for_ctrl_ready(ctrl_info);
Kevin Barnett8845fdf2017-05-03 18:53:36 -05006036 if (rc)
Kevin Barnett6c223762016-06-27 16:41:00 -05006037 return rc;
Kevin Barnett6c223762016-06-27 16:41:00 -05006038
6039 /*
6040 * Get the controller properties. This allows us to determine
6041 * whether or not it supports PQI mode.
6042 */
6043 rc = sis_get_ctrl_properties(ctrl_info);
6044 if (rc) {
6045 dev_err(&ctrl_info->pci_dev->dev,
6046 "error obtaining controller properties\n");
6047 return rc;
6048 }
6049
6050 rc = sis_get_pqi_capabilities(ctrl_info);
6051 if (rc) {
6052 dev_err(&ctrl_info->pci_dev->dev,
6053 "error obtaining controller capabilities\n");
6054 return rc;
6055 }
6056
Kevin Barnettd727a772017-05-03 18:54:25 -05006057 if (reset_devices) {
6058 if (ctrl_info->max_outstanding_requests >
6059 PQI_MAX_OUTSTANDING_REQUESTS_KDUMP)
6060 ctrl_info->max_outstanding_requests =
6061 PQI_MAX_OUTSTANDING_REQUESTS_KDUMP;
6062 } else {
6063 if (ctrl_info->max_outstanding_requests >
6064 PQI_MAX_OUTSTANDING_REQUESTS)
6065 ctrl_info->max_outstanding_requests =
6066 PQI_MAX_OUTSTANDING_REQUESTS;
6067 }
Kevin Barnett6c223762016-06-27 16:41:00 -05006068
6069 pqi_calculate_io_resources(ctrl_info);
6070
6071 rc = pqi_alloc_error_buffer(ctrl_info);
6072 if (rc) {
6073 dev_err(&ctrl_info->pci_dev->dev,
6074 "failed to allocate PQI error buffer\n");
6075 return rc;
6076 }
6077
6078 /*
6079 * If the function we are about to call succeeds, the
6080 * controller will transition from legacy SIS mode
6081 * into PQI mode.
6082 */
6083 rc = sis_init_base_struct_addr(ctrl_info);
6084 if (rc) {
6085 dev_err(&ctrl_info->pci_dev->dev,
6086 "error initializing PQI mode\n");
6087 return rc;
6088 }
6089
6090 /* Wait for the controller to complete the SIS -> PQI transition. */
6091 rc = pqi_wait_for_pqi_mode_ready(ctrl_info);
6092 if (rc) {
6093 dev_err(&ctrl_info->pci_dev->dev,
6094 "transition to PQI mode failed\n");
6095 return rc;
6096 }
6097
6098 /* From here on, we are running in PQI mode. */
6099 ctrl_info->pqi_mode_enabled = true;
Kevin Barnettff6abb72016-08-31 14:54:41 -05006100 pqi_save_ctrl_mode(ctrl_info, PQI_MODE);
Kevin Barnett6c223762016-06-27 16:41:00 -05006101
Kevin Barnett98f87662017-05-03 18:53:11 -05006102 rc = pqi_process_config_table(ctrl_info);
6103 if (rc)
6104 return rc;
6105
Kevin Barnett6c223762016-06-27 16:41:00 -05006106 rc = pqi_alloc_admin_queues(ctrl_info);
6107 if (rc) {
6108 dev_err(&ctrl_info->pci_dev->dev,
Kevin Barnettd87d5472017-05-03 18:54:00 -05006109 "failed to allocate admin queues\n");
Kevin Barnett6c223762016-06-27 16:41:00 -05006110 return rc;
6111 }
6112
6113 rc = pqi_create_admin_queues(ctrl_info);
6114 if (rc) {
6115 dev_err(&ctrl_info->pci_dev->dev,
6116 "error creating admin queues\n");
6117 return rc;
6118 }
6119
6120 rc = pqi_report_device_capability(ctrl_info);
6121 if (rc) {
6122 dev_err(&ctrl_info->pci_dev->dev,
6123 "obtaining device capability failed\n");
6124 return rc;
6125 }
6126
6127 rc = pqi_validate_device_capability(ctrl_info);
6128 if (rc)
6129 return rc;
6130
6131 pqi_calculate_queue_resources(ctrl_info);
6132
6133 rc = pqi_enable_msix_interrupts(ctrl_info);
6134 if (rc)
6135 return rc;
6136
6137 if (ctrl_info->num_msix_vectors_enabled < ctrl_info->num_queue_groups) {
6138 ctrl_info->max_msix_vectors =
6139 ctrl_info->num_msix_vectors_enabled;
6140 pqi_calculate_queue_resources(ctrl_info);
6141 }
6142
6143 rc = pqi_alloc_io_resources(ctrl_info);
6144 if (rc)
6145 return rc;
6146
6147 rc = pqi_alloc_operational_queues(ctrl_info);
Kevin Barnettd87d5472017-05-03 18:54:00 -05006148 if (rc) {
6149 dev_err(&ctrl_info->pci_dev->dev,
6150 "failed to allocate operational queues\n");
Kevin Barnett6c223762016-06-27 16:41:00 -05006151 return rc;
Kevin Barnettd87d5472017-05-03 18:54:00 -05006152 }
Kevin Barnett6c223762016-06-27 16:41:00 -05006153
6154 pqi_init_operational_queues(ctrl_info);
6155
6156 rc = pqi_request_irqs(ctrl_info);
6157 if (rc)
6158 return rc;
6159
Kevin Barnett6c223762016-06-27 16:41:00 -05006160 rc = pqi_create_queues(ctrl_info);
6161 if (rc)
6162 return rc;
6163
Kevin Barnett061ef062017-05-03 18:53:05 -05006164 pqi_change_irq_mode(ctrl_info, IRQ_MODE_MSIX);
6165
6166 ctrl_info->controller_online = true;
6167 pqi_start_heartbeat_timer(ctrl_info);
Kevin Barnett6c223762016-06-27 16:41:00 -05006168
Kevin Barnett6a50d6a2017-05-03 18:52:52 -05006169 rc = pqi_enable_events(ctrl_info);
Kevin Barnett6c223762016-06-27 16:41:00 -05006170 if (rc) {
6171 dev_err(&ctrl_info->pci_dev->dev,
Kevin Barnett6a50d6a2017-05-03 18:52:52 -05006172 "error enabling events\n");
Kevin Barnett6c223762016-06-27 16:41:00 -05006173 return rc;
6174 }
6175
Kevin Barnett6c223762016-06-27 16:41:00 -05006176 /* Register with the SCSI subsystem. */
6177 rc = pqi_register_scsi(ctrl_info);
6178 if (rc)
6179 return rc;
6180
6181 rc = pqi_get_ctrl_firmware_version(ctrl_info);
6182 if (rc) {
6183 dev_err(&ctrl_info->pci_dev->dev,
6184 "error obtaining firmware version\n");
6185 return rc;
6186 }
6187
6188 rc = pqi_write_driver_version_to_host_wellness(ctrl_info);
6189 if (rc) {
6190 dev_err(&ctrl_info->pci_dev->dev,
6191 "error updating host wellness\n");
6192 return rc;
6193 }
6194
6195 pqi_schedule_update_time_worker(ctrl_info);
6196
6197 pqi_scan_scsi_devices(ctrl_info);
6198
6199 return 0;
6200}
6201
Kevin Barnett061ef062017-05-03 18:53:05 -05006202#if defined(CONFIG_PM)
6203
6204static void pqi_reinit_queues(struct pqi_ctrl_info *ctrl_info)
6205{
6206 unsigned int i;
6207 struct pqi_admin_queues *admin_queues;
6208 struct pqi_event_queue *event_queue;
6209
6210 admin_queues = &ctrl_info->admin_queues;
6211 admin_queues->iq_pi_copy = 0;
6212 admin_queues->oq_ci_copy = 0;
6213 *admin_queues->oq_pi = 0;
6214
6215 for (i = 0; i < ctrl_info->num_queue_groups; i++) {
6216 ctrl_info->queue_groups[i].iq_pi_copy[RAID_PATH] = 0;
6217 ctrl_info->queue_groups[i].iq_pi_copy[AIO_PATH] = 0;
6218 ctrl_info->queue_groups[i].oq_ci_copy = 0;
6219
6220 *ctrl_info->queue_groups[i].iq_ci[RAID_PATH] = 0;
6221 *ctrl_info->queue_groups[i].iq_ci[AIO_PATH] = 0;
6222 *ctrl_info->queue_groups[i].oq_pi = 0;
6223 }
6224
6225 event_queue = &ctrl_info->event_queue;
6226 *event_queue->oq_pi = 0;
6227 event_queue->oq_ci_copy = 0;
6228}
6229
6230static int pqi_ctrl_init_resume(struct pqi_ctrl_info *ctrl_info)
6231{
6232 int rc;
6233
6234 rc = pqi_force_sis_mode(ctrl_info);
6235 if (rc)
6236 return rc;
6237
6238 /*
6239 * Wait until the controller is ready to start accepting SIS
6240 * commands.
6241 */
6242 rc = sis_wait_for_ctrl_ready_resume(ctrl_info);
6243 if (rc)
6244 return rc;
6245
6246 /*
6247 * If the function we are about to call succeeds, the
6248 * controller will transition from legacy SIS mode
6249 * into PQI mode.
6250 */
6251 rc = sis_init_base_struct_addr(ctrl_info);
6252 if (rc) {
6253 dev_err(&ctrl_info->pci_dev->dev,
6254 "error initializing PQI mode\n");
6255 return rc;
6256 }
6257
6258 /* Wait for the controller to complete the SIS -> PQI transition. */
6259 rc = pqi_wait_for_pqi_mode_ready(ctrl_info);
6260 if (rc) {
6261 dev_err(&ctrl_info->pci_dev->dev,
6262 "transition to PQI mode failed\n");
6263 return rc;
6264 }
6265
6266 /* From here on, we are running in PQI mode. */
6267 ctrl_info->pqi_mode_enabled = true;
6268 pqi_save_ctrl_mode(ctrl_info, PQI_MODE);
6269
6270 pqi_reinit_queues(ctrl_info);
6271
6272 rc = pqi_create_admin_queues(ctrl_info);
6273 if (rc) {
6274 dev_err(&ctrl_info->pci_dev->dev,
6275 "error creating admin queues\n");
6276 return rc;
6277 }
6278
6279 rc = pqi_create_queues(ctrl_info);
6280 if (rc)
6281 return rc;
6282
6283 pqi_change_irq_mode(ctrl_info, IRQ_MODE_MSIX);
6284
6285 ctrl_info->controller_online = true;
6286 pqi_start_heartbeat_timer(ctrl_info);
6287 pqi_ctrl_unblock_requests(ctrl_info);
6288
6289 rc = pqi_enable_events(ctrl_info);
6290 if (rc) {
6291 dev_err(&ctrl_info->pci_dev->dev,
Kevin Barnettd87d5472017-05-03 18:54:00 -05006292 "error enabling events\n");
Kevin Barnett061ef062017-05-03 18:53:05 -05006293 return rc;
6294 }
6295
6296 rc = pqi_write_driver_version_to_host_wellness(ctrl_info);
6297 if (rc) {
6298 dev_err(&ctrl_info->pci_dev->dev,
6299 "error updating host wellness\n");
6300 return rc;
6301 }
6302
6303 pqi_schedule_update_time_worker(ctrl_info);
6304
6305 pqi_scan_scsi_devices(ctrl_info);
6306
6307 return 0;
6308}
6309
6310#endif /* CONFIG_PM */
6311
Kevin Barnetta81ed5f32017-05-03 18:52:34 -05006312static inline int pqi_set_pcie_completion_timeout(struct pci_dev *pci_dev,
6313 u16 timeout)
6314{
6315 return pcie_capability_clear_and_set_word(pci_dev, PCI_EXP_DEVCTL2,
6316 PCI_EXP_DEVCTL2_COMP_TIMEOUT, timeout);
6317}
6318
Kevin Barnett6c223762016-06-27 16:41:00 -05006319static int pqi_pci_init(struct pqi_ctrl_info *ctrl_info)
6320{
6321 int rc;
6322 u64 mask;
6323
6324 rc = pci_enable_device(ctrl_info->pci_dev);
6325 if (rc) {
6326 dev_err(&ctrl_info->pci_dev->dev,
6327 "failed to enable PCI device\n");
6328 return rc;
6329 }
6330
6331 if (sizeof(dma_addr_t) > 4)
6332 mask = DMA_BIT_MASK(64);
6333 else
6334 mask = DMA_BIT_MASK(32);
6335
6336 rc = dma_set_mask(&ctrl_info->pci_dev->dev, mask);
6337 if (rc) {
6338 dev_err(&ctrl_info->pci_dev->dev, "failed to set DMA mask\n");
6339 goto disable_device;
6340 }
6341
6342 rc = pci_request_regions(ctrl_info->pci_dev, DRIVER_NAME_SHORT);
6343 if (rc) {
6344 dev_err(&ctrl_info->pci_dev->dev,
6345 "failed to obtain PCI resources\n");
6346 goto disable_device;
6347 }
6348
6349 ctrl_info->iomem_base = ioremap_nocache(pci_resource_start(
6350 ctrl_info->pci_dev, 0),
6351 sizeof(struct pqi_ctrl_registers));
6352 if (!ctrl_info->iomem_base) {
6353 dev_err(&ctrl_info->pci_dev->dev,
6354 "failed to map memory for controller registers\n");
6355 rc = -ENOMEM;
6356 goto release_regions;
6357 }
6358
Kevin Barnetta81ed5f32017-05-03 18:52:34 -05006359#define PCI_EXP_COMP_TIMEOUT_65_TO_210_MS 0x6
6360
6361 /* Increase the PCIe completion timeout. */
6362 rc = pqi_set_pcie_completion_timeout(ctrl_info->pci_dev,
6363 PCI_EXP_COMP_TIMEOUT_65_TO_210_MS);
6364 if (rc) {
6365 dev_err(&ctrl_info->pci_dev->dev,
6366 "failed to set PCIe completion timeout\n");
6367 goto release_regions;
6368 }
6369
Kevin Barnett6c223762016-06-27 16:41:00 -05006370 /* Enable bus mastering. */
6371 pci_set_master(ctrl_info->pci_dev);
6372
Kevin Barnettcbe0c7b2017-05-03 18:53:48 -05006373 ctrl_info->registers = ctrl_info->iomem_base;
6374 ctrl_info->pqi_registers = &ctrl_info->registers->pqi_registers;
6375
Kevin Barnett6c223762016-06-27 16:41:00 -05006376 pci_set_drvdata(ctrl_info->pci_dev, ctrl_info);
6377
6378 return 0;
6379
6380release_regions:
6381 pci_release_regions(ctrl_info->pci_dev);
6382disable_device:
6383 pci_disable_device(ctrl_info->pci_dev);
6384
6385 return rc;
6386}
6387
6388static void pqi_cleanup_pci_init(struct pqi_ctrl_info *ctrl_info)
6389{
6390 iounmap(ctrl_info->iomem_base);
6391 pci_release_regions(ctrl_info->pci_dev);
Kevin Barnettcbe0c7b2017-05-03 18:53:48 -05006392 if (pci_is_enabled(ctrl_info->pci_dev))
6393 pci_disable_device(ctrl_info->pci_dev);
Kevin Barnett6c223762016-06-27 16:41:00 -05006394 pci_set_drvdata(ctrl_info->pci_dev, NULL);
6395}
6396
6397static struct pqi_ctrl_info *pqi_alloc_ctrl_info(int numa_node)
6398{
6399 struct pqi_ctrl_info *ctrl_info;
6400
6401 ctrl_info = kzalloc_node(sizeof(struct pqi_ctrl_info),
6402 GFP_KERNEL, numa_node);
6403 if (!ctrl_info)
6404 return NULL;
6405
6406 mutex_init(&ctrl_info->scan_mutex);
Kevin Barnett7561a7e2017-05-03 18:52:58 -05006407 mutex_init(&ctrl_info->lun_reset_mutex);
Kevin Barnett6c223762016-06-27 16:41:00 -05006408
6409 INIT_LIST_HEAD(&ctrl_info->scsi_device_list);
6410 spin_lock_init(&ctrl_info->scsi_device_list_lock);
6411
6412 INIT_WORK(&ctrl_info->event_work, pqi_event_worker);
6413 atomic_set(&ctrl_info->num_interrupts, 0);
6414
6415 INIT_DELAYED_WORK(&ctrl_info->rescan_work, pqi_rescan_worker);
6416 INIT_DELAYED_WORK(&ctrl_info->update_time_work, pqi_update_time_worker);
6417
Kevin Barnett98f87662017-05-03 18:53:11 -05006418 init_timer(&ctrl_info->heartbeat_timer);
Kevin Barnett5f310422017-05-03 18:54:55 -05006419 INIT_WORK(&ctrl_info->ctrl_offline_work, pqi_ctrl_offline_worker);
Kevin Barnett98f87662017-05-03 18:53:11 -05006420
Kevin Barnett6c223762016-06-27 16:41:00 -05006421 sema_init(&ctrl_info->sync_request_sem,
6422 PQI_RESERVED_IO_SLOTS_SYNCHRONOUS_REQUESTS);
Kevin Barnett7561a7e2017-05-03 18:52:58 -05006423 init_waitqueue_head(&ctrl_info->block_requests_wait);
Kevin Barnett6c223762016-06-27 16:41:00 -05006424
Kevin Barnett376fb882017-05-03 18:54:43 -05006425 INIT_LIST_HEAD(&ctrl_info->raid_bypass_retry_list);
6426 spin_lock_init(&ctrl_info->raid_bypass_retry_list_lock);
6427 INIT_WORK(&ctrl_info->raid_bypass_retry_work,
6428 pqi_raid_bypass_retry_worker);
6429
Kevin Barnett6c223762016-06-27 16:41:00 -05006430 ctrl_info->ctrl_id = atomic_inc_return(&pqi_controller_count) - 1;
Kevin Barnett061ef062017-05-03 18:53:05 -05006431 ctrl_info->irq_mode = IRQ_MODE_NONE;
Kevin Barnett6c223762016-06-27 16:41:00 -05006432 ctrl_info->max_msix_vectors = PQI_MAX_MSIX_VECTORS;
6433
6434 return ctrl_info;
6435}
6436
6437static inline void pqi_free_ctrl_info(struct pqi_ctrl_info *ctrl_info)
6438{
6439 kfree(ctrl_info);
6440}
6441
6442static void pqi_free_interrupts(struct pqi_ctrl_info *ctrl_info)
6443{
Kevin Barnett98bf0612017-05-03 18:52:28 -05006444 pqi_free_irqs(ctrl_info);
6445 pqi_disable_msix_interrupts(ctrl_info);
Kevin Barnett6c223762016-06-27 16:41:00 -05006446}
6447
6448static void pqi_free_ctrl_resources(struct pqi_ctrl_info *ctrl_info)
6449{
6450 pqi_stop_heartbeat_timer(ctrl_info);
6451 pqi_free_interrupts(ctrl_info);
6452 if (ctrl_info->queue_memory_base)
6453 dma_free_coherent(&ctrl_info->pci_dev->dev,
6454 ctrl_info->queue_memory_length,
6455 ctrl_info->queue_memory_base,
6456 ctrl_info->queue_memory_base_dma_handle);
6457 if (ctrl_info->admin_queue_memory_base)
6458 dma_free_coherent(&ctrl_info->pci_dev->dev,
6459 ctrl_info->admin_queue_memory_length,
6460 ctrl_info->admin_queue_memory_base,
6461 ctrl_info->admin_queue_memory_base_dma_handle);
6462 pqi_free_all_io_requests(ctrl_info);
6463 if (ctrl_info->error_buffer)
6464 dma_free_coherent(&ctrl_info->pci_dev->dev,
6465 ctrl_info->error_buffer_length,
6466 ctrl_info->error_buffer,
6467 ctrl_info->error_buffer_dma_handle);
6468 if (ctrl_info->iomem_base)
6469 pqi_cleanup_pci_init(ctrl_info);
6470 pqi_free_ctrl_info(ctrl_info);
6471}
6472
6473static void pqi_remove_ctrl(struct pqi_ctrl_info *ctrl_info)
6474{
Kevin Barnett061ef062017-05-03 18:53:05 -05006475 pqi_cancel_rescan_worker(ctrl_info);
6476 pqi_cancel_update_time_worker(ctrl_info);
Kevin Barnette57a1f92016-08-31 14:54:47 -05006477 pqi_remove_all_scsi_devices(ctrl_info);
6478 pqi_unregister_scsi(ctrl_info);
Kevin Barnett162d7752017-05-03 18:52:46 -05006479 if (ctrl_info->pqi_mode_enabled)
6480 pqi_revert_to_sis_mode(ctrl_info);
Kevin Barnett6c223762016-06-27 16:41:00 -05006481 pqi_free_ctrl_resources(ctrl_info);
6482}
6483
Kevin Barnett3c509762017-05-03 18:54:37 -05006484static void pqi_perform_lockup_action(void)
6485{
6486 switch (pqi_lockup_action) {
6487 case PANIC:
6488 panic("FATAL: Smart Family Controller lockup detected");
6489 break;
6490 case REBOOT:
6491 emergency_restart();
6492 break;
6493 case NONE:
6494 default:
6495 break;
6496 }
6497}
6498
Kevin Barnett5f310422017-05-03 18:54:55 -05006499static struct pqi_raid_error_info pqi_ctrl_offline_raid_error_info = {
6500 .data_out_result = PQI_DATA_IN_OUT_HARDWARE_ERROR,
6501 .status = SAM_STAT_CHECK_CONDITION,
6502};
6503
6504static void pqi_fail_all_outstanding_requests(struct pqi_ctrl_info *ctrl_info)
Kevin Barnett376fb882017-05-03 18:54:43 -05006505{
6506 unsigned int i;
Kevin Barnett376fb882017-05-03 18:54:43 -05006507 struct pqi_io_request *io_request;
Kevin Barnett376fb882017-05-03 18:54:43 -05006508 struct scsi_cmnd *scmd;
6509
Kevin Barnett5f310422017-05-03 18:54:55 -05006510 for (i = 0; i < ctrl_info->max_io_slots; i++) {
6511 io_request = &ctrl_info->io_request_pool[i];
6512 if (atomic_read(&io_request->refcount) == 0)
6513 continue;
Kevin Barnett376fb882017-05-03 18:54:43 -05006514
Kevin Barnett5f310422017-05-03 18:54:55 -05006515 scmd = io_request->scmd;
6516 if (scmd) {
6517 set_host_byte(scmd, DID_NO_CONNECT);
6518 } else {
6519 io_request->status = -ENXIO;
6520 io_request->error_info =
6521 &pqi_ctrl_offline_raid_error_info;
Kevin Barnett376fb882017-05-03 18:54:43 -05006522 }
Kevin Barnett5f310422017-05-03 18:54:55 -05006523
6524 io_request->io_complete_callback(io_request,
6525 io_request->context);
Kevin Barnett376fb882017-05-03 18:54:43 -05006526 }
6527}
6528
Kevin Barnett5f310422017-05-03 18:54:55 -05006529static void pqi_take_ctrl_offline_deferred(struct pqi_ctrl_info *ctrl_info)
Kevin Barnett376fb882017-05-03 18:54:43 -05006530{
Kevin Barnett5f310422017-05-03 18:54:55 -05006531 pqi_perform_lockup_action();
6532 pqi_stop_heartbeat_timer(ctrl_info);
6533 pqi_free_interrupts(ctrl_info);
6534 pqi_cancel_rescan_worker(ctrl_info);
6535 pqi_cancel_update_time_worker(ctrl_info);
6536 pqi_ctrl_wait_until_quiesced(ctrl_info);
6537 pqi_fail_all_outstanding_requests(ctrl_info);
6538 pqi_clear_all_queued_raid_bypass_retries(ctrl_info);
6539 pqi_ctrl_unblock_requests(ctrl_info);
6540}
6541
6542static void pqi_ctrl_offline_worker(struct work_struct *work)
6543{
6544 struct pqi_ctrl_info *ctrl_info;
6545
6546 ctrl_info = container_of(work, struct pqi_ctrl_info, ctrl_offline_work);
6547 pqi_take_ctrl_offline_deferred(ctrl_info);
Kevin Barnett376fb882017-05-03 18:54:43 -05006548}
6549
6550static void pqi_take_ctrl_offline(struct pqi_ctrl_info *ctrl_info)
6551{
Kevin Barnett5f310422017-05-03 18:54:55 -05006552 if (!ctrl_info->controller_online)
6553 return;
6554
Kevin Barnett376fb882017-05-03 18:54:43 -05006555 ctrl_info->controller_online = false;
Kevin Barnett5f310422017-05-03 18:54:55 -05006556 ctrl_info->pqi_mode_enabled = false;
6557 pqi_ctrl_block_requests(ctrl_info);
Kevin Barnett376fb882017-05-03 18:54:43 -05006558 sis_shutdown_ctrl(ctrl_info);
6559 pci_disable_device(ctrl_info->pci_dev);
6560 dev_err(&ctrl_info->pci_dev->dev, "controller offline\n");
Kevin Barnett5f310422017-05-03 18:54:55 -05006561 schedule_work(&ctrl_info->ctrl_offline_work);
Kevin Barnett376fb882017-05-03 18:54:43 -05006562}
6563
Kevin Barnettd91d7822017-05-03 18:53:30 -05006564static void pqi_print_ctrl_info(struct pci_dev *pci_dev,
Kevin Barnett6c223762016-06-27 16:41:00 -05006565 const struct pci_device_id *id)
6566{
6567 char *ctrl_description;
6568
Kevin Barnett37b36842017-05-03 18:55:01 -05006569 if (id->driver_data)
Kevin Barnett6c223762016-06-27 16:41:00 -05006570 ctrl_description = (char *)id->driver_data;
Kevin Barnett37b36842017-05-03 18:55:01 -05006571 else
6572 ctrl_description = "Microsemi Smart Family Controller";
Kevin Barnett6c223762016-06-27 16:41:00 -05006573
Kevin Barnettd91d7822017-05-03 18:53:30 -05006574 dev_info(&pci_dev->dev, "%s found\n", ctrl_description);
Kevin Barnett6c223762016-06-27 16:41:00 -05006575}
6576
Kevin Barnettd91d7822017-05-03 18:53:30 -05006577static int pqi_pci_probe(struct pci_dev *pci_dev,
6578 const struct pci_device_id *id)
Kevin Barnett6c223762016-06-27 16:41:00 -05006579{
6580 int rc;
6581 int node;
6582 struct pqi_ctrl_info *ctrl_info;
6583
Kevin Barnettd91d7822017-05-03 18:53:30 -05006584 pqi_print_ctrl_info(pci_dev, id);
Kevin Barnett6c223762016-06-27 16:41:00 -05006585
6586 if (pqi_disable_device_id_wildcards &&
6587 id->subvendor == PCI_ANY_ID &&
6588 id->subdevice == PCI_ANY_ID) {
Kevin Barnettd91d7822017-05-03 18:53:30 -05006589 dev_warn(&pci_dev->dev,
Kevin Barnett6c223762016-06-27 16:41:00 -05006590 "controller not probed because device ID wildcards are disabled\n");
6591 return -ENODEV;
6592 }
6593
6594 if (id->subvendor == PCI_ANY_ID || id->subdevice == PCI_ANY_ID)
Kevin Barnettd91d7822017-05-03 18:53:30 -05006595 dev_warn(&pci_dev->dev,
Kevin Barnett6c223762016-06-27 16:41:00 -05006596 "controller device ID matched using wildcards\n");
6597
Kevin Barnettd91d7822017-05-03 18:53:30 -05006598 node = dev_to_node(&pci_dev->dev);
Kevin Barnett6c223762016-06-27 16:41:00 -05006599 if (node == NUMA_NO_NODE)
Kevin Barnettd91d7822017-05-03 18:53:30 -05006600 set_dev_node(&pci_dev->dev, 0);
Kevin Barnett6c223762016-06-27 16:41:00 -05006601
6602 ctrl_info = pqi_alloc_ctrl_info(node);
6603 if (!ctrl_info) {
Kevin Barnettd91d7822017-05-03 18:53:30 -05006604 dev_err(&pci_dev->dev,
Kevin Barnett6c223762016-06-27 16:41:00 -05006605 "failed to allocate controller info block\n");
6606 return -ENOMEM;
6607 }
6608
Kevin Barnettd91d7822017-05-03 18:53:30 -05006609 ctrl_info->pci_dev = pci_dev;
Kevin Barnett6c223762016-06-27 16:41:00 -05006610
6611 rc = pqi_pci_init(ctrl_info);
6612 if (rc)
6613 goto error;
6614
6615 rc = pqi_ctrl_init(ctrl_info);
6616 if (rc)
6617 goto error;
6618
6619 return 0;
6620
6621error:
6622 pqi_remove_ctrl(ctrl_info);
6623
6624 return rc;
6625}
6626
Kevin Barnettd91d7822017-05-03 18:53:30 -05006627static void pqi_pci_remove(struct pci_dev *pci_dev)
Kevin Barnett6c223762016-06-27 16:41:00 -05006628{
6629 struct pqi_ctrl_info *ctrl_info;
6630
Kevin Barnettd91d7822017-05-03 18:53:30 -05006631 ctrl_info = pci_get_drvdata(pci_dev);
Kevin Barnett6c223762016-06-27 16:41:00 -05006632 if (!ctrl_info)
6633 return;
6634
6635 pqi_remove_ctrl(ctrl_info);
6636}
6637
Kevin Barnettd91d7822017-05-03 18:53:30 -05006638static void pqi_shutdown(struct pci_dev *pci_dev)
Kevin Barnett6c223762016-06-27 16:41:00 -05006639{
6640 int rc;
6641 struct pqi_ctrl_info *ctrl_info;
6642
Kevin Barnettd91d7822017-05-03 18:53:30 -05006643 ctrl_info = pci_get_drvdata(pci_dev);
Kevin Barnett6c223762016-06-27 16:41:00 -05006644 if (!ctrl_info)
6645 goto error;
6646
6647 /*
6648 * Write all data in the controller's battery-backed cache to
6649 * storage.
6650 */
6651 rc = pqi_flush_cache(ctrl_info);
6652 if (rc == 0)
6653 return;
6654
6655error:
Kevin Barnettd91d7822017-05-03 18:53:30 -05006656 dev_warn(&pci_dev->dev,
Kevin Barnett6c223762016-06-27 16:41:00 -05006657 "unable to flush controller cache\n");
6658}
6659
Kevin Barnett3c509762017-05-03 18:54:37 -05006660static void pqi_process_lockup_action_param(void)
6661{
6662 unsigned int i;
6663
6664 if (!pqi_lockup_action_param)
6665 return;
6666
6667 for (i = 0; i < ARRAY_SIZE(pqi_lockup_actions); i++) {
6668 if (strcmp(pqi_lockup_action_param,
6669 pqi_lockup_actions[i].name) == 0) {
6670 pqi_lockup_action = pqi_lockup_actions[i].action;
6671 return;
6672 }
6673 }
6674
6675 pr_warn("%s: invalid lockup action setting \"%s\" - supported settings: none, reboot, panic\n",
6676 DRIVER_NAME_SHORT, pqi_lockup_action_param);
6677}
6678
6679static void pqi_process_module_params(void)
6680{
6681 pqi_process_lockup_action_param();
6682}
6683
Kevin Barnett061ef062017-05-03 18:53:05 -05006684#if defined(CONFIG_PM)
6685
6686static int pqi_suspend(struct pci_dev *pci_dev, pm_message_t state)
6687{
6688 struct pqi_ctrl_info *ctrl_info;
6689
6690 ctrl_info = pci_get_drvdata(pci_dev);
6691
6692 pqi_disable_events(ctrl_info);
6693 pqi_cancel_update_time_worker(ctrl_info);
6694 pqi_cancel_rescan_worker(ctrl_info);
6695 pqi_wait_until_scan_finished(ctrl_info);
6696 pqi_wait_until_lun_reset_finished(ctrl_info);
6697 pqi_flush_cache(ctrl_info);
6698 pqi_ctrl_block_requests(ctrl_info);
6699 pqi_ctrl_wait_until_quiesced(ctrl_info);
6700 pqi_wait_until_inbound_queues_empty(ctrl_info);
6701 pqi_ctrl_wait_for_pending_io(ctrl_info);
6702 pqi_stop_heartbeat_timer(ctrl_info);
6703
6704 if (state.event == PM_EVENT_FREEZE)
6705 return 0;
6706
6707 pci_save_state(pci_dev);
6708 pci_set_power_state(pci_dev, pci_choose_state(pci_dev, state));
6709
6710 ctrl_info->controller_online = false;
6711 ctrl_info->pqi_mode_enabled = false;
6712
6713 return 0;
6714}
6715
6716static int pqi_resume(struct pci_dev *pci_dev)
6717{
6718 int rc;
6719 struct pqi_ctrl_info *ctrl_info;
6720
6721 ctrl_info = pci_get_drvdata(pci_dev);
6722
6723 if (pci_dev->current_state != PCI_D0) {
6724 ctrl_info->max_hw_queue_index = 0;
6725 pqi_free_interrupts(ctrl_info);
6726 pqi_change_irq_mode(ctrl_info, IRQ_MODE_INTX);
6727 rc = request_irq(pci_irq_vector(pci_dev, 0), pqi_irq_handler,
6728 IRQF_SHARED, DRIVER_NAME_SHORT,
6729 &ctrl_info->queue_groups[0]);
6730 if (rc) {
6731 dev_err(&ctrl_info->pci_dev->dev,
6732 "irq %u init failed with error %d\n",
6733 pci_dev->irq, rc);
6734 return rc;
6735 }
6736 pqi_start_heartbeat_timer(ctrl_info);
6737 pqi_ctrl_unblock_requests(ctrl_info);
6738 return 0;
6739 }
6740
6741 pci_set_power_state(pci_dev, PCI_D0);
6742 pci_restore_state(pci_dev);
6743
6744 return pqi_ctrl_init_resume(ctrl_info);
6745}
6746
6747#endif /* CONFIG_PM */
6748
Kevin Barnett6c223762016-06-27 16:41:00 -05006749/* Define the PCI IDs for the controllers that we support. */
6750static const struct pci_device_id pqi_pci_id_table[] = {
6751 {
6752 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
Kevin Barnett7eddabf2017-05-03 18:53:54 -05006753 0x152d, 0x8a22)
6754 },
6755 {
6756 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
6757 0x152d, 0x8a23)
6758 },
6759 {
6760 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
6761 0x152d, 0x8a24)
6762 },
6763 {
6764 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
6765 0x152d, 0x8a36)
6766 },
6767 {
6768 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
6769 0x152d, 0x8a37)
6770 },
6771 {
6772 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
Kevin Barnett6c223762016-06-27 16:41:00 -05006773 PCI_VENDOR_ID_ADAPTEC2, 0x0110)
6774 },
6775 {
6776 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
Kevin Barnett7eddabf2017-05-03 18:53:54 -05006777 PCI_VENDOR_ID_ADAPTEC2, 0x0605)
Kevin Barnett6c223762016-06-27 16:41:00 -05006778 },
6779 {
6780 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
6781 PCI_VENDOR_ID_ADAPTEC2, 0x0800)
6782 },
6783 {
6784 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
6785 PCI_VENDOR_ID_ADAPTEC2, 0x0801)
6786 },
6787 {
6788 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
6789 PCI_VENDOR_ID_ADAPTEC2, 0x0802)
6790 },
6791 {
6792 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
6793 PCI_VENDOR_ID_ADAPTEC2, 0x0803)
6794 },
6795 {
6796 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
6797 PCI_VENDOR_ID_ADAPTEC2, 0x0804)
6798 },
6799 {
6800 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
6801 PCI_VENDOR_ID_ADAPTEC2, 0x0805)
6802 },
6803 {
6804 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
Kevin Barnett7eddabf2017-05-03 18:53:54 -05006805 PCI_VENDOR_ID_ADAPTEC2, 0x0806)
6806 },
6807 {
6808 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
Kevin Barnett6c223762016-06-27 16:41:00 -05006809 PCI_VENDOR_ID_ADAPTEC2, 0x0900)
6810 },
6811 {
6812 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
6813 PCI_VENDOR_ID_ADAPTEC2, 0x0901)
6814 },
6815 {
6816 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
6817 PCI_VENDOR_ID_ADAPTEC2, 0x0902)
6818 },
6819 {
6820 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
6821 PCI_VENDOR_ID_ADAPTEC2, 0x0903)
6822 },
6823 {
6824 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
6825 PCI_VENDOR_ID_ADAPTEC2, 0x0904)
6826 },
6827 {
6828 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
6829 PCI_VENDOR_ID_ADAPTEC2, 0x0905)
6830 },
6831 {
6832 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
6833 PCI_VENDOR_ID_ADAPTEC2, 0x0906)
6834 },
6835 {
6836 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
Kevin Barnett7eddabf2017-05-03 18:53:54 -05006837 PCI_VENDOR_ID_ADAPTEC2, 0x0907)
6838 },
6839 {
6840 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
6841 PCI_VENDOR_ID_ADAPTEC2, 0x0908)
6842 },
6843 {
6844 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
6845 PCI_VENDOR_ID_ADAPTEC2, 0x1200)
6846 },
6847 {
6848 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
6849 PCI_VENDOR_ID_ADAPTEC2, 0x1201)
6850 },
6851 {
6852 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
6853 PCI_VENDOR_ID_ADAPTEC2, 0x1202)
6854 },
6855 {
6856 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
6857 PCI_VENDOR_ID_ADAPTEC2, 0x1280)
6858 },
6859 {
6860 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
6861 PCI_VENDOR_ID_ADAPTEC2, 0x1281)
6862 },
6863 {
6864 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
6865 PCI_VENDOR_ID_ADAPTEC2, 0x1300)
6866 },
6867 {
6868 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
6869 PCI_VENDOR_ID_ADAPTEC2, 0x1301)
6870 },
6871 {
6872 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
6873 PCI_VENDOR_ID_ADAPTEC2, 0x1380)
6874 },
6875 {
6876 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
6877 PCI_VENDOR_ID_HP, 0x0600)
6878 },
6879 {
6880 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
6881 PCI_VENDOR_ID_HP, 0x0601)
6882 },
6883 {
6884 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
6885 PCI_VENDOR_ID_HP, 0x0602)
6886 },
6887 {
6888 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
6889 PCI_VENDOR_ID_HP, 0x0603)
6890 },
6891 {
6892 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
6893 PCI_VENDOR_ID_HP, 0x0604)
6894 },
6895 {
6896 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
6897 PCI_VENDOR_ID_HP, 0x0606)
6898 },
6899 {
6900 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
6901 PCI_VENDOR_ID_HP, 0x0650)
6902 },
6903 {
6904 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
6905 PCI_VENDOR_ID_HP, 0x0651)
6906 },
6907 {
6908 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
6909 PCI_VENDOR_ID_HP, 0x0652)
6910 },
6911 {
6912 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
6913 PCI_VENDOR_ID_HP, 0x0653)
6914 },
6915 {
6916 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
6917 PCI_VENDOR_ID_HP, 0x0654)
6918 },
6919 {
6920 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
6921 PCI_VENDOR_ID_HP, 0x0655)
6922 },
6923 {
6924 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
6925 PCI_VENDOR_ID_HP, 0x0656)
6926 },
6927 {
6928 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
6929 PCI_VENDOR_ID_HP, 0x0657)
6930 },
6931 {
6932 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
6933 PCI_VENDOR_ID_HP, 0x0700)
6934 },
6935 {
6936 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
6937 PCI_VENDOR_ID_HP, 0x0701)
6938 },
6939 {
6940 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
Kevin Barnett6c223762016-06-27 16:41:00 -05006941 PCI_VENDOR_ID_HP, 0x1001)
6942 },
6943 {
6944 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
6945 PCI_VENDOR_ID_HP, 0x1100)
6946 },
6947 {
6948 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
6949 PCI_VENDOR_ID_HP, 0x1101)
6950 },
6951 {
6952 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
6953 PCI_VENDOR_ID_HP, 0x1102)
6954 },
6955 {
6956 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
6957 PCI_VENDOR_ID_HP, 0x1150)
6958 },
6959 {
6960 PCI_DEVICE_SUB(PCI_VENDOR_ID_ADAPTEC2, 0x028f,
6961 PCI_ANY_ID, PCI_ANY_ID)
6962 },
6963 { 0 }
6964};
6965
6966MODULE_DEVICE_TABLE(pci, pqi_pci_id_table);
6967
6968static struct pci_driver pqi_pci_driver = {
6969 .name = DRIVER_NAME_SHORT,
6970 .id_table = pqi_pci_id_table,
6971 .probe = pqi_pci_probe,
6972 .remove = pqi_pci_remove,
6973 .shutdown = pqi_shutdown,
Kevin Barnett061ef062017-05-03 18:53:05 -05006974#if defined(CONFIG_PM)
6975 .suspend = pqi_suspend,
6976 .resume = pqi_resume,
6977#endif
Kevin Barnett6c223762016-06-27 16:41:00 -05006978};
6979
6980static int __init pqi_init(void)
6981{
6982 int rc;
6983
6984 pr_info(DRIVER_NAME "\n");
6985
6986 pqi_sas_transport_template =
6987 sas_attach_transport(&pqi_sas_transport_functions);
6988 if (!pqi_sas_transport_template)
6989 return -ENODEV;
6990
Kevin Barnett3c509762017-05-03 18:54:37 -05006991 pqi_process_module_params();
6992
Kevin Barnett6c223762016-06-27 16:41:00 -05006993 rc = pci_register_driver(&pqi_pci_driver);
6994 if (rc)
6995 sas_release_transport(pqi_sas_transport_template);
6996
6997 return rc;
6998}
6999
7000static void __exit pqi_cleanup(void)
7001{
7002 pci_unregister_driver(&pqi_pci_driver);
7003 sas_release_transport(pqi_sas_transport_template);
7004}
7005
7006module_init(pqi_init);
7007module_exit(pqi_cleanup);
7008
7009static void __attribute__((unused)) verify_structures(void)
7010{
7011 BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
7012 sis_host_to_ctrl_doorbell) != 0x20);
7013 BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
7014 sis_interrupt_mask) != 0x34);
7015 BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
7016 sis_ctrl_to_host_doorbell) != 0x9c);
7017 BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
7018 sis_ctrl_to_host_doorbell_clear) != 0xa0);
7019 BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
Kevin Barnettff6abb72016-08-31 14:54:41 -05007020 sis_driver_scratch) != 0xb0);
7021 BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
Kevin Barnett6c223762016-06-27 16:41:00 -05007022 sis_firmware_status) != 0xbc);
7023 BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
7024 sis_mailbox) != 0x1000);
7025 BUILD_BUG_ON(offsetof(struct pqi_ctrl_registers,
7026 pqi_registers) != 0x4000);
7027
7028 BUILD_BUG_ON(offsetof(struct pqi_iu_header,
7029 iu_type) != 0x0);
7030 BUILD_BUG_ON(offsetof(struct pqi_iu_header,
7031 iu_length) != 0x2);
7032 BUILD_BUG_ON(offsetof(struct pqi_iu_header,
7033 response_queue_id) != 0x4);
7034 BUILD_BUG_ON(offsetof(struct pqi_iu_header,
7035 work_area) != 0x6);
7036 BUILD_BUG_ON(sizeof(struct pqi_iu_header) != 0x8);
7037
7038 BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
7039 status) != 0x0);
7040 BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
7041 service_response) != 0x1);
7042 BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
7043 data_present) != 0x2);
7044 BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
7045 reserved) != 0x3);
7046 BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
7047 residual_count) != 0x4);
7048 BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
7049 data_length) != 0x8);
7050 BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
7051 reserved1) != 0xa);
7052 BUILD_BUG_ON(offsetof(struct pqi_aio_error_info,
7053 data) != 0xc);
7054 BUILD_BUG_ON(sizeof(struct pqi_aio_error_info) != 0x10c);
7055
7056 BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
7057 data_in_result) != 0x0);
7058 BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
7059 data_out_result) != 0x1);
7060 BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
7061 reserved) != 0x2);
7062 BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
7063 status) != 0x5);
7064 BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
7065 status_qualifier) != 0x6);
7066 BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
7067 sense_data_length) != 0x8);
7068 BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
7069 response_data_length) != 0xa);
7070 BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
7071 data_in_transferred) != 0xc);
7072 BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
7073 data_out_transferred) != 0x10);
7074 BUILD_BUG_ON(offsetof(struct pqi_raid_error_info,
7075 data) != 0x14);
7076 BUILD_BUG_ON(sizeof(struct pqi_raid_error_info) != 0x114);
7077
7078 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
7079 signature) != 0x0);
7080 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
7081 function_and_status_code) != 0x8);
7082 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
7083 max_admin_iq_elements) != 0x10);
7084 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
7085 max_admin_oq_elements) != 0x11);
7086 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
7087 admin_iq_element_length) != 0x12);
7088 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
7089 admin_oq_element_length) != 0x13);
7090 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
7091 max_reset_timeout) != 0x14);
7092 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
7093 legacy_intx_status) != 0x18);
7094 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
7095 legacy_intx_mask_set) != 0x1c);
7096 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
7097 legacy_intx_mask_clear) != 0x20);
7098 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
7099 device_status) != 0x40);
7100 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
7101 admin_iq_pi_offset) != 0x48);
7102 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
7103 admin_oq_ci_offset) != 0x50);
7104 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
7105 admin_iq_element_array_addr) != 0x58);
7106 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
7107 admin_oq_element_array_addr) != 0x60);
7108 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
7109 admin_iq_ci_addr) != 0x68);
7110 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
7111 admin_oq_pi_addr) != 0x70);
7112 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
7113 admin_iq_num_elements) != 0x78);
7114 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
7115 admin_oq_num_elements) != 0x79);
7116 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
7117 admin_queue_int_msg_num) != 0x7a);
7118 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
7119 device_error) != 0x80);
7120 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
7121 error_details) != 0x88);
7122 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
7123 device_reset) != 0x90);
7124 BUILD_BUG_ON(offsetof(struct pqi_device_registers,
7125 power_action) != 0x94);
7126 BUILD_BUG_ON(sizeof(struct pqi_device_registers) != 0x100);
7127
7128 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
7129 header.iu_type) != 0);
7130 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
7131 header.iu_length) != 2);
7132 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
7133 header.work_area) != 6);
7134 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
7135 request_id) != 8);
7136 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
7137 function_code) != 10);
7138 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
7139 data.report_device_capability.buffer_length) != 44);
7140 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
7141 data.report_device_capability.sg_descriptor) != 48);
7142 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
7143 data.create_operational_iq.queue_id) != 12);
7144 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
7145 data.create_operational_iq.element_array_addr) != 16);
7146 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
7147 data.create_operational_iq.ci_addr) != 24);
7148 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
7149 data.create_operational_iq.num_elements) != 32);
7150 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
7151 data.create_operational_iq.element_length) != 34);
7152 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
7153 data.create_operational_iq.queue_protocol) != 36);
7154 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
7155 data.create_operational_oq.queue_id) != 12);
7156 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
7157 data.create_operational_oq.element_array_addr) != 16);
7158 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
7159 data.create_operational_oq.pi_addr) != 24);
7160 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
7161 data.create_operational_oq.num_elements) != 32);
7162 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
7163 data.create_operational_oq.element_length) != 34);
7164 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
7165 data.create_operational_oq.queue_protocol) != 36);
7166 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
7167 data.create_operational_oq.int_msg_num) != 40);
7168 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
7169 data.create_operational_oq.coalescing_count) != 42);
7170 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
7171 data.create_operational_oq.min_coalescing_time) != 44);
7172 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
7173 data.create_operational_oq.max_coalescing_time) != 48);
7174 BUILD_BUG_ON(offsetof(struct pqi_general_admin_request,
7175 data.delete_operational_queue.queue_id) != 12);
7176 BUILD_BUG_ON(sizeof(struct pqi_general_admin_request) != 64);
7177 BUILD_BUG_ON(FIELD_SIZEOF(struct pqi_general_admin_request,
7178 data.create_operational_iq) != 64 - 11);
7179 BUILD_BUG_ON(FIELD_SIZEOF(struct pqi_general_admin_request,
7180 data.create_operational_oq) != 64 - 11);
7181 BUILD_BUG_ON(FIELD_SIZEOF(struct pqi_general_admin_request,
7182 data.delete_operational_queue) != 64 - 11);
7183
7184 BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
7185 header.iu_type) != 0);
7186 BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
7187 header.iu_length) != 2);
7188 BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
7189 header.work_area) != 6);
7190 BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
7191 request_id) != 8);
7192 BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
7193 function_code) != 10);
7194 BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
7195 status) != 11);
7196 BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
7197 data.create_operational_iq.status_descriptor) != 12);
7198 BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
7199 data.create_operational_iq.iq_pi_offset) != 16);
7200 BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
7201 data.create_operational_oq.status_descriptor) != 12);
7202 BUILD_BUG_ON(offsetof(struct pqi_general_admin_response,
7203 data.create_operational_oq.oq_ci_offset) != 16);
7204 BUILD_BUG_ON(sizeof(struct pqi_general_admin_response) != 64);
7205
7206 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
7207 header.iu_type) != 0);
7208 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
7209 header.iu_length) != 2);
7210 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
7211 header.response_queue_id) != 4);
7212 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
7213 header.work_area) != 6);
7214 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
7215 request_id) != 8);
7216 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
7217 nexus_id) != 10);
7218 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
7219 buffer_length) != 12);
7220 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
7221 lun_number) != 16);
7222 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
7223 protocol_specific) != 24);
7224 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
7225 error_index) != 27);
7226 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
7227 cdb) != 32);
7228 BUILD_BUG_ON(offsetof(struct pqi_raid_path_request,
7229 sg_descriptors) != 64);
7230 BUILD_BUG_ON(sizeof(struct pqi_raid_path_request) !=
7231 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
7232
7233 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
7234 header.iu_type) != 0);
7235 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
7236 header.iu_length) != 2);
7237 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
7238 header.response_queue_id) != 4);
7239 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
7240 header.work_area) != 6);
7241 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
7242 request_id) != 8);
7243 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
7244 nexus_id) != 12);
7245 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
7246 buffer_length) != 16);
7247 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
7248 data_encryption_key_index) != 22);
7249 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
7250 encrypt_tweak_lower) != 24);
7251 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
7252 encrypt_tweak_upper) != 28);
7253 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
7254 cdb) != 32);
7255 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
7256 error_index) != 48);
7257 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
7258 num_sg_descriptors) != 50);
7259 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
7260 cdb_length) != 51);
7261 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
7262 lun_number) != 52);
7263 BUILD_BUG_ON(offsetof(struct pqi_aio_path_request,
7264 sg_descriptors) != 64);
7265 BUILD_BUG_ON(sizeof(struct pqi_aio_path_request) !=
7266 PQI_OPERATIONAL_IQ_ELEMENT_LENGTH);
7267
7268 BUILD_BUG_ON(offsetof(struct pqi_io_response,
7269 header.iu_type) != 0);
7270 BUILD_BUG_ON(offsetof(struct pqi_io_response,
7271 header.iu_length) != 2);
7272 BUILD_BUG_ON(offsetof(struct pqi_io_response,
7273 request_id) != 8);
7274 BUILD_BUG_ON(offsetof(struct pqi_io_response,
7275 error_index) != 10);
7276
7277 BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
7278 header.iu_type) != 0);
7279 BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
7280 header.iu_length) != 2);
7281 BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
7282 header.response_queue_id) != 4);
7283 BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
7284 request_id) != 8);
7285 BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
7286 data.report_event_configuration.buffer_length) != 12);
7287 BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
7288 data.report_event_configuration.sg_descriptors) != 16);
7289 BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
7290 data.set_event_configuration.global_event_oq_id) != 10);
7291 BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
7292 data.set_event_configuration.buffer_length) != 12);
7293 BUILD_BUG_ON(offsetof(struct pqi_general_management_request,
7294 data.set_event_configuration.sg_descriptors) != 16);
7295
7296 BUILD_BUG_ON(offsetof(struct pqi_iu_layer_descriptor,
7297 max_inbound_iu_length) != 6);
7298 BUILD_BUG_ON(offsetof(struct pqi_iu_layer_descriptor,
7299 max_outbound_iu_length) != 14);
7300 BUILD_BUG_ON(sizeof(struct pqi_iu_layer_descriptor) != 16);
7301
7302 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
7303 data_length) != 0);
7304 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
7305 iq_arbitration_priority_support_bitmask) != 8);
7306 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
7307 maximum_aw_a) != 9);
7308 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
7309 maximum_aw_b) != 10);
7310 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
7311 maximum_aw_c) != 11);
7312 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
7313 max_inbound_queues) != 16);
7314 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
7315 max_elements_per_iq) != 18);
7316 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
7317 max_iq_element_length) != 24);
7318 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
7319 min_iq_element_length) != 26);
7320 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
7321 max_outbound_queues) != 30);
7322 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
7323 max_elements_per_oq) != 32);
7324 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
7325 intr_coalescing_time_granularity) != 34);
7326 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
7327 max_oq_element_length) != 36);
7328 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
7329 min_oq_element_length) != 38);
7330 BUILD_BUG_ON(offsetof(struct pqi_device_capability,
7331 iu_layer_descriptors) != 64);
7332 BUILD_BUG_ON(sizeof(struct pqi_device_capability) != 576);
7333
7334 BUILD_BUG_ON(offsetof(struct pqi_event_descriptor,
7335 event_type) != 0);
7336 BUILD_BUG_ON(offsetof(struct pqi_event_descriptor,
7337 oq_id) != 2);
7338 BUILD_BUG_ON(sizeof(struct pqi_event_descriptor) != 4);
7339
7340 BUILD_BUG_ON(offsetof(struct pqi_event_config,
7341 num_event_descriptors) != 2);
7342 BUILD_BUG_ON(offsetof(struct pqi_event_config,
7343 descriptors) != 4);
7344
Kevin Barnett061ef062017-05-03 18:53:05 -05007345 BUILD_BUG_ON(PQI_NUM_SUPPORTED_EVENTS !=
7346 ARRAY_SIZE(pqi_supported_event_types));
7347
Kevin Barnett6c223762016-06-27 16:41:00 -05007348 BUILD_BUG_ON(offsetof(struct pqi_event_response,
7349 header.iu_type) != 0);
7350 BUILD_BUG_ON(offsetof(struct pqi_event_response,
7351 header.iu_length) != 2);
7352 BUILD_BUG_ON(offsetof(struct pqi_event_response,
7353 event_type) != 8);
7354 BUILD_BUG_ON(offsetof(struct pqi_event_response,
7355 event_id) != 10);
7356 BUILD_BUG_ON(offsetof(struct pqi_event_response,
7357 additional_event_id) != 12);
7358 BUILD_BUG_ON(offsetof(struct pqi_event_response,
7359 data) != 16);
7360 BUILD_BUG_ON(sizeof(struct pqi_event_response) != 32);
7361
7362 BUILD_BUG_ON(offsetof(struct pqi_event_acknowledge_request,
7363 header.iu_type) != 0);
7364 BUILD_BUG_ON(offsetof(struct pqi_event_acknowledge_request,
7365 header.iu_length) != 2);
7366 BUILD_BUG_ON(offsetof(struct pqi_event_acknowledge_request,
7367 event_type) != 8);
7368 BUILD_BUG_ON(offsetof(struct pqi_event_acknowledge_request,
7369 event_id) != 10);
7370 BUILD_BUG_ON(offsetof(struct pqi_event_acknowledge_request,
7371 additional_event_id) != 12);
7372 BUILD_BUG_ON(sizeof(struct pqi_event_acknowledge_request) != 16);
7373
7374 BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
7375 header.iu_type) != 0);
7376 BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
7377 header.iu_length) != 2);
7378 BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
7379 request_id) != 8);
7380 BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
7381 nexus_id) != 10);
7382 BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
7383 lun_number) != 16);
7384 BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
7385 protocol_specific) != 24);
7386 BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
7387 outbound_queue_id_to_manage) != 26);
7388 BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
7389 request_id_to_manage) != 28);
7390 BUILD_BUG_ON(offsetof(struct pqi_task_management_request,
7391 task_management_function) != 30);
7392 BUILD_BUG_ON(sizeof(struct pqi_task_management_request) != 32);
7393
7394 BUILD_BUG_ON(offsetof(struct pqi_task_management_response,
7395 header.iu_type) != 0);
7396 BUILD_BUG_ON(offsetof(struct pqi_task_management_response,
7397 header.iu_length) != 2);
7398 BUILD_BUG_ON(offsetof(struct pqi_task_management_response,
7399 request_id) != 8);
7400 BUILD_BUG_ON(offsetof(struct pqi_task_management_response,
7401 nexus_id) != 10);
7402 BUILD_BUG_ON(offsetof(struct pqi_task_management_response,
7403 additional_response_info) != 12);
7404 BUILD_BUG_ON(offsetof(struct pqi_task_management_response,
7405 response_code) != 15);
7406 BUILD_BUG_ON(sizeof(struct pqi_task_management_response) != 16);
7407
7408 BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
7409 configured_logical_drive_count) != 0);
7410 BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
7411 configuration_signature) != 1);
7412 BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
7413 firmware_version) != 5);
7414 BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
7415 extended_logical_unit_count) != 154);
7416 BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
7417 firmware_build_number) != 190);
7418 BUILD_BUG_ON(offsetof(struct bmic_identify_controller,
7419 controller_mode) != 292);
7420
Kevin Barnett1be42f42017-05-03 18:53:42 -05007421 BUILD_BUG_ON(offsetof(struct bmic_identify_physical_device,
7422 phys_bay_in_box) != 115);
7423 BUILD_BUG_ON(offsetof(struct bmic_identify_physical_device,
7424 device_type) != 120);
7425 BUILD_BUG_ON(offsetof(struct bmic_identify_physical_device,
7426 redundant_path_present_map) != 1736);
7427 BUILD_BUG_ON(offsetof(struct bmic_identify_physical_device,
7428 active_path_number) != 1738);
7429 BUILD_BUG_ON(offsetof(struct bmic_identify_physical_device,
7430 alternate_paths_phys_connector) != 1739);
7431 BUILD_BUG_ON(offsetof(struct bmic_identify_physical_device,
7432 alternate_paths_phys_box_on_port) != 1755);
7433 BUILD_BUG_ON(offsetof(struct bmic_identify_physical_device,
7434 current_queue_depth_limit) != 1796);
7435 BUILD_BUG_ON(sizeof(struct bmic_identify_physical_device) != 2560);
7436
Kevin Barnett6c223762016-06-27 16:41:00 -05007437 BUILD_BUG_ON(PQI_ADMIN_IQ_NUM_ELEMENTS > 255);
7438 BUILD_BUG_ON(PQI_ADMIN_OQ_NUM_ELEMENTS > 255);
7439 BUILD_BUG_ON(PQI_ADMIN_IQ_ELEMENT_LENGTH %
7440 PQI_QUEUE_ELEMENT_LENGTH_ALIGNMENT != 0);
7441 BUILD_BUG_ON(PQI_ADMIN_OQ_ELEMENT_LENGTH %
7442 PQI_QUEUE_ELEMENT_LENGTH_ALIGNMENT != 0);
7443 BUILD_BUG_ON(PQI_OPERATIONAL_IQ_ELEMENT_LENGTH > 1048560);
7444 BUILD_BUG_ON(PQI_OPERATIONAL_IQ_ELEMENT_LENGTH %
7445 PQI_QUEUE_ELEMENT_LENGTH_ALIGNMENT != 0);
7446 BUILD_BUG_ON(PQI_OPERATIONAL_OQ_ELEMENT_LENGTH > 1048560);
7447 BUILD_BUG_ON(PQI_OPERATIONAL_OQ_ELEMENT_LENGTH %
7448 PQI_QUEUE_ELEMENT_LENGTH_ALIGNMENT != 0);
7449
7450 BUILD_BUG_ON(PQI_RESERVED_IO_SLOTS >= PQI_MAX_OUTSTANDING_REQUESTS);
Kevin Barnettd727a772017-05-03 18:54:25 -05007451 BUILD_BUG_ON(PQI_RESERVED_IO_SLOTS >=
7452 PQI_MAX_OUTSTANDING_REQUESTS_KDUMP);
Kevin Barnett6c223762016-06-27 16:41:00 -05007453}