blob: 9454a66c12cc54deb7bee27561824b0c695d388f [file] [log] [blame]
Yishai Hadasa8b92ca2018-06-17 12:59:57 +03001// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2/*
3 * Copyright (c) 2018, Mellanox Technologies inc. All rights reserved.
4 */
5
6#include <rdma/ib_user_verbs.h>
7#include <rdma/ib_verbs.h>
8#include <rdma/uverbs_types.h>
9#include <rdma/uverbs_ioctl.h>
10#include <rdma/mlx5_user_ioctl_cmds.h>
Yishai Hadasa124edb2019-01-22 08:29:57 +020011#include <rdma/mlx5_user_ioctl_verbs.h>
Yishai Hadasa8b92ca2018-06-17 12:59:57 +030012#include <rdma/ib_umem.h>
Yishai Hadas34613eb2018-11-26 08:28:35 +020013#include <rdma/uverbs_std_types.h>
Yishai Hadasa8b92ca2018-06-17 12:59:57 +030014#include <linux/mlx5/driver.h>
15#include <linux/mlx5/fs.h>
16#include "mlx5_ib.h"
Leon Romanovsky333fbaa2020-04-04 10:40:24 +030017#include "qp.h"
Yishai Hadas75973852019-06-30 19:23:31 +030018#include <linux/xarray.h>
Yishai Hadasa8b92ca2018-06-17 12:59:57 +030019
Yishai Hadas8aa8c952018-06-17 13:00:00 +030020#define UVERBS_MODULE_NAME mlx5_ib
21#include <rdma/uverbs_named_ioctl.h>
22
Yishai Hadasef1659a2019-06-30 19:23:33 +030023static void dispatch_event_fd(struct list_head *fd_list, const void *data);
24
Yishai Hadas534fd7a2019-01-13 16:01:17 +020025enum devx_obj_flags {
26 DEVX_OBJ_FLAGS_INDIRECT_MKEY = 1 << 0,
Yishai Hadasc5ae1952019-03-06 19:21:42 +020027 DEVX_OBJ_FLAGS_DCT = 1 << 1,
Yishai Hadasef1659a2019-06-30 19:23:33 +030028 DEVX_OBJ_FLAGS_CQ = 1 << 2,
Yishai Hadas534fd7a2019-01-13 16:01:17 +020029};
30
Yishai Hadasa124edb2019-01-22 08:29:57 +020031struct devx_async_data {
32 struct mlx5_ib_dev *mdev;
33 struct list_head list;
Jason Gunthorpe93887e62020-01-08 19:21:55 +020034 struct devx_async_cmd_event_file *ev_file;
Yishai Hadasa124edb2019-01-22 08:29:57 +020035 struct mlx5_async_work cb_work;
36 u16 cmd_out_len;
37 /* must be last field in this structure */
38 struct mlx5_ib_uapi_devx_async_cmd_hdr hdr;
39};
40
Yishai Hadas5ec9d8e2019-06-30 19:23:32 +030041struct devx_async_event_data {
42 struct list_head list; /* headed in ev_file->event_list */
43 struct mlx5_ib_uapi_devx_async_event_hdr hdr;
44};
45
Yishai Hadas75973852019-06-30 19:23:31 +030046/* first level XA value data structure */
47struct devx_event {
48 struct xarray object_ids; /* second XA level, Key = object id */
49 struct list_head unaffiliated_list;
50};
51
52/* second level XA value data structure */
53struct devx_obj_event {
54 struct rcu_head rcu;
55 struct list_head obj_sub_list;
56};
57
58struct devx_event_subscription {
59 struct list_head file_list; /* headed in ev_file->
60 * subscribed_events_list
61 */
62 struct list_head xa_list; /* headed in devx_event->unaffiliated_list or
63 * devx_obj_event->obj_sub_list
64 */
65 struct list_head obj_list; /* headed in devx_object */
66 struct list_head event_list; /* headed in ev_file->event_list or in
67 * temp list via subscription
68 */
69
70 u8 is_cleaned:1;
71 u32 xa_key_level1;
72 u32 xa_key_level2;
73 struct rcu_head rcu;
74 u64 cookie;
75 struct devx_async_event_file *ev_file;
Yishai Hadas75973852019-06-30 19:23:31 +030076 struct eventfd_ctx *eventfd;
77};
78
Yishai Hadas2afc5e12019-06-30 19:23:29 +030079struct devx_async_event_file {
80 struct ib_uobject uobj;
81 /* Head of events that are subscribed to this FD */
82 struct list_head subscribed_events_list;
83 spinlock_t lock;
84 wait_queue_head_t poll_wait;
85 struct list_head event_list;
86 struct mlx5_ib_dev *dev;
87 u8 omit_data:1;
Yishai Hadas5ec9d8e2019-06-30 19:23:32 +030088 u8 is_overflow_err:1;
89 u8 is_destroyed:1;
Yishai Hadas2afc5e12019-06-30 19:23:29 +030090};
91
Yishai Hadas7efce362018-06-17 13:00:01 +030092#define MLX5_MAX_DESTROY_INBOX_SIZE_DW MLX5_ST_SZ_DW(delete_fte_in)
93struct devx_obj {
Yishai Hadasef1659a2019-06-30 19:23:33 +030094 struct mlx5_ib_dev *ib_dev;
Yishai Hadas2351776e2018-10-07 12:06:34 +030095 u64 obj_id;
Yishai Hadas7efce362018-06-17 13:00:01 +030096 u32 dinlen; /* destroy inbox length */
97 u32 dinbox[MLX5_MAX_DESTROY_INBOX_SIZE_DW];
Yishai Hadas534fd7a2019-01-13 16:01:17 +020098 u32 flags;
Yishai Hadasc5ae1952019-03-06 19:21:42 +020099 union {
100 struct mlx5_ib_devx_mr devx_mr;
101 struct mlx5_core_dct core_dct;
Yishai Hadasef1659a2019-06-30 19:23:33 +0300102 struct mlx5_core_cq core_cq;
Yevgeny Kliteynik208d70f2019-11-03 16:07:23 +0200103 u32 flow_counter_bulk_size;
Yishai Hadasc5ae1952019-03-06 19:21:42 +0200104 };
Yishai Hadas75973852019-06-30 19:23:31 +0300105 struct list_head event_sub; /* holds devx_event_subscription entries */
Yishai Hadas7efce362018-06-17 13:00:01 +0300106};
107
Yishai Hadasaeae9452018-06-17 13:00:04 +0300108struct devx_umem {
109 struct mlx5_core_dev *mdev;
110 struct ib_umem *umem;
111 u32 page_offset;
112 int page_shift;
113 int ncont;
114 u32 dinlen;
115 u32 dinbox[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)];
116};
117
118struct devx_umem_reg_cmd {
119 void *in;
120 u32 inlen;
121 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
122};
123
Jason Gunthorpe15a1b4b2018-11-25 20:51:15 +0200124static struct mlx5_ib_ucontext *
125devx_ufile2uctx(const struct uverbs_attr_bundle *attrs)
Yishai Hadas8aa8c952018-06-17 13:00:00 +0300126{
Jason Gunthorpe15a1b4b2018-11-25 20:51:15 +0200127 return to_mucontext(ib_uverbs_get_ucontext(attrs));
Yishai Hadas8aa8c952018-06-17 13:00:00 +0300128}
129
Yishai Hadasfb981532018-11-26 08:28:36 +0200130int mlx5_ib_devx_create(struct mlx5_ib_dev *dev, bool is_user)
Yishai Hadasa8b92ca2018-06-17 12:59:57 +0300131{
132 u32 in[MLX5_ST_SZ_DW(create_uctx_in)] = {0};
133 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0};
Yishai Hadas6e3722b2018-12-19 16:28:15 +0200134 void *uctx;
Yishai Hadasa8b92ca2018-06-17 12:59:57 +0300135 int err;
Yishai Hadas76dc5a82018-09-20 21:45:19 +0300136 u16 uid;
Yishai Hadasfb981532018-11-26 08:28:36 +0200137 u32 cap = 0;
Yishai Hadasa8b92ca2018-06-17 12:59:57 +0300138
Yishai Hadas6e3722b2018-12-19 16:28:15 +0200139 /* 0 means not supported */
140 if (!MLX5_CAP_GEN(dev->mdev, log_max_uctx))
Yishai Hadasa8b92ca2018-06-17 12:59:57 +0300141 return -EINVAL;
142
Yishai Hadas6e3722b2018-12-19 16:28:15 +0200143 uctx = MLX5_ADDR_OF(create_uctx_in, in, uctx);
Yishai Hadasfb981532018-11-26 08:28:36 +0200144 if (is_user && capable(CAP_NET_RAW) &&
145 (MLX5_CAP_GEN(dev->mdev, uctx_cap) & MLX5_UCTX_CAP_RAW_TX))
146 cap |= MLX5_UCTX_CAP_RAW_TX;
Ariel Levkovich33cde962019-05-05 17:07:14 +0300147 if (is_user && capable(CAP_SYS_RAWIO) &&
148 (MLX5_CAP_GEN(dev->mdev, uctx_cap) &
149 MLX5_UCTX_CAP_INTERNAL_DEV_RES))
150 cap |= MLX5_UCTX_CAP_INTERNAL_DEV_RES;
Yishai Hadasfb981532018-11-26 08:28:36 +0200151
Yishai Hadas6e3722b2018-12-19 16:28:15 +0200152 MLX5_SET(create_uctx_in, in, opcode, MLX5_CMD_OP_CREATE_UCTX);
Yishai Hadasfb981532018-11-26 08:28:36 +0200153 MLX5_SET(uctx, uctx, cap, cap);
Yishai Hadasa8b92ca2018-06-17 12:59:57 +0300154
155 err = mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out));
156 if (err)
157 return err;
158
Yishai Hadas76dc5a82018-09-20 21:45:19 +0300159 uid = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
160 return uid;
Yishai Hadasa8b92ca2018-06-17 12:59:57 +0300161}
162
Yishai Hadas76dc5a82018-09-20 21:45:19 +0300163void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid)
Yishai Hadasa8b92ca2018-06-17 12:59:57 +0300164{
Yishai Hadas6e3722b2018-12-19 16:28:15 +0200165 u32 in[MLX5_ST_SZ_DW(destroy_uctx_in)] = {0};
Yishai Hadasa8b92ca2018-06-17 12:59:57 +0300166 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0};
167
Yishai Hadas6e3722b2018-12-19 16:28:15 +0200168 MLX5_SET(destroy_uctx_in, in, opcode, MLX5_CMD_OP_DESTROY_UCTX);
169 MLX5_SET(destroy_uctx_in, in, uid, uid);
Yishai Hadasa8b92ca2018-06-17 12:59:57 +0300170
171 mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out));
172}
Yishai Hadas8aa8c952018-06-17 13:00:00 +0300173
Yishai Hadas32269442018-07-23 15:25:09 +0300174bool mlx5_ib_devx_is_flow_dest(void *obj, int *dest_id, int *dest_type)
175{
176 struct devx_obj *devx_obj = obj;
177 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode);
178
179 switch (opcode) {
180 case MLX5_CMD_OP_DESTROY_TIR:
181 *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR;
182 *dest_id = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox,
183 obj_id);
184 return true;
185
186 case MLX5_CMD_OP_DESTROY_FLOW_TABLE:
187 *dest_type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
188 *dest_id = MLX5_GET(destroy_flow_table_in, devx_obj->dinbox,
189 table_id);
190 return true;
191 default:
192 return false;
193 }
194}
195
Yevgeny Kliteynik208d70f2019-11-03 16:07:23 +0200196bool mlx5_ib_devx_is_flow_counter(void *obj, u32 offset, u32 *counter_id)
Mark Blochbfc5d832018-11-20 20:31:08 +0200197{
198 struct devx_obj *devx_obj = obj;
199 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode);
200
201 if (opcode == MLX5_CMD_OP_DEALLOC_FLOW_COUNTER) {
Yevgeny Kliteynik208d70f2019-11-03 16:07:23 +0200202
203 if (offset && offset >= devx_obj->flow_counter_bulk_size)
204 return false;
205
Mark Blochbfc5d832018-11-20 20:31:08 +0200206 *counter_id = MLX5_GET(dealloc_flow_counter_in,
207 devx_obj->dinbox,
208 flow_counter_id);
Yevgeny Kliteynik208d70f2019-11-03 16:07:23 +0200209 *counter_id += offset;
Mark Blochbfc5d832018-11-20 20:31:08 +0200210 return true;
211 }
212
213 return false;
214}
215
Yishai Hadas75973852019-06-30 19:23:31 +0300216static bool is_legacy_unaffiliated_event_num(u16 event_num)
217{
218 switch (event_num) {
219 case MLX5_EVENT_TYPE_PORT_CHANGE:
220 return true;
221 default:
222 return false;
223 }
224}
225
226static bool is_legacy_obj_event_num(u16 event_num)
227{
228 switch (event_num) {
229 case MLX5_EVENT_TYPE_PATH_MIG:
230 case MLX5_EVENT_TYPE_COMM_EST:
231 case MLX5_EVENT_TYPE_SQ_DRAINED:
232 case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
233 case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
234 case MLX5_EVENT_TYPE_CQ_ERROR:
235 case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
236 case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
237 case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
238 case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
239 case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
240 case MLX5_EVENT_TYPE_DCT_DRAINED:
241 case MLX5_EVENT_TYPE_COMP:
Yishai Hadas972d7562019-08-08 11:43:57 +0300242 case MLX5_EVENT_TYPE_DCT_KEY_VIOLATION:
243 case MLX5_EVENT_TYPE_XRQ_ERROR:
Yishai Hadas75973852019-06-30 19:23:31 +0300244 return true;
245 default:
246 return false;
247 }
248}
249
250static u16 get_legacy_obj_type(u16 opcode)
251{
252 switch (opcode) {
253 case MLX5_CMD_OP_CREATE_RQ:
254 return MLX5_EVENT_QUEUE_TYPE_RQ;
255 case MLX5_CMD_OP_CREATE_QP:
256 return MLX5_EVENT_QUEUE_TYPE_QP;
257 case MLX5_CMD_OP_CREATE_SQ:
258 return MLX5_EVENT_QUEUE_TYPE_SQ;
259 case MLX5_CMD_OP_CREATE_DCT:
260 return MLX5_EVENT_QUEUE_TYPE_DCT;
261 default:
262 return 0;
263 }
264}
265
266static u16 get_dec_obj_type(struct devx_obj *obj, u16 event_num)
267{
268 u16 opcode;
269
270 opcode = (obj->obj_id >> 32) & 0xffff;
271
272 if (is_legacy_obj_event_num(event_num))
273 return get_legacy_obj_type(opcode);
274
275 switch (opcode) {
276 case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
277 return (obj->obj_id >> 48);
278 case MLX5_CMD_OP_CREATE_RQ:
279 return MLX5_OBJ_TYPE_RQ;
280 case MLX5_CMD_OP_CREATE_QP:
281 return MLX5_OBJ_TYPE_QP;
282 case MLX5_CMD_OP_CREATE_SQ:
283 return MLX5_OBJ_TYPE_SQ;
284 case MLX5_CMD_OP_CREATE_DCT:
285 return MLX5_OBJ_TYPE_DCT;
286 case MLX5_CMD_OP_CREATE_TIR:
287 return MLX5_OBJ_TYPE_TIR;
288 case MLX5_CMD_OP_CREATE_TIS:
289 return MLX5_OBJ_TYPE_TIS;
290 case MLX5_CMD_OP_CREATE_PSV:
291 return MLX5_OBJ_TYPE_PSV;
292 case MLX5_OBJ_TYPE_MKEY:
293 return MLX5_OBJ_TYPE_MKEY;
294 case MLX5_CMD_OP_CREATE_RMP:
295 return MLX5_OBJ_TYPE_RMP;
296 case MLX5_CMD_OP_CREATE_XRC_SRQ:
297 return MLX5_OBJ_TYPE_XRC_SRQ;
298 case MLX5_CMD_OP_CREATE_XRQ:
299 return MLX5_OBJ_TYPE_XRQ;
300 case MLX5_CMD_OP_CREATE_RQT:
301 return MLX5_OBJ_TYPE_RQT;
302 case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
303 return MLX5_OBJ_TYPE_FLOW_COUNTER;
304 case MLX5_CMD_OP_CREATE_CQ:
305 return MLX5_OBJ_TYPE_CQ;
306 default:
307 return 0;
308 }
309}
310
Yishai Hadas5ec9d8e2019-06-30 19:23:32 +0300311static u16 get_event_obj_type(unsigned long event_type, struct mlx5_eqe *eqe)
312{
313 switch (event_type) {
314 case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
315 case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
316 case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
317 case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
318 case MLX5_EVENT_TYPE_PATH_MIG:
319 case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
320 case MLX5_EVENT_TYPE_COMM_EST:
321 case MLX5_EVENT_TYPE_SQ_DRAINED:
322 case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
323 case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
324 return eqe->data.qp_srq.type;
325 case MLX5_EVENT_TYPE_CQ_ERROR:
Yishai Hadas972d7562019-08-08 11:43:57 +0300326 case MLX5_EVENT_TYPE_XRQ_ERROR:
Yishai Hadas5ec9d8e2019-06-30 19:23:32 +0300327 return 0;
328 case MLX5_EVENT_TYPE_DCT_DRAINED:
Yishai Hadas972d7562019-08-08 11:43:57 +0300329 case MLX5_EVENT_TYPE_DCT_KEY_VIOLATION:
Yishai Hadas5ec9d8e2019-06-30 19:23:32 +0300330 return MLX5_EVENT_QUEUE_TYPE_DCT;
331 default:
332 return MLX5_GET(affiliated_event_header, &eqe->data, obj_type);
333 }
334}
335
Yishai Hadas75973852019-06-30 19:23:31 +0300336static u32 get_dec_obj_id(u64 obj_id)
337{
338 return (obj_id & 0xffffffff);
339}
340
Yishai Hadas2351776e2018-10-07 12:06:34 +0300341/*
342 * As the obj_id in the firmware is not globally unique the object type
343 * must be considered upon checking for a valid object id.
344 * For that the opcode of the creator command is encoded as part of the obj_id.
345 */
Yishai Hadascd5d20f2019-05-14 14:44:11 +0300346static u64 get_enc_obj_id(u32 opcode, u32 obj_id)
Yishai Hadas2351776e2018-10-07 12:06:34 +0300347{
348 return ((u64)opcode << 32) | obj_id;
349}
350
Yishai Hadas34613eb2018-11-26 08:28:35 +0200351static u64 devx_get_obj_id(const void *in)
Yishai Hadase662e142018-06-17 13:00:02 +0300352{
353 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
Yishai Hadas2351776e2018-10-07 12:06:34 +0300354 u64 obj_id;
Yishai Hadase662e142018-06-17 13:00:02 +0300355
356 switch (opcode) {
357 case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT:
358 case MLX5_CMD_OP_QUERY_GENERAL_OBJECT:
Yishai Hadascd5d20f2019-05-14 14:44:11 +0300359 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_GENERAL_OBJECT |
360 MLX5_GET(general_obj_in_cmd_hdr, in,
361 obj_type) << 16,
Yishai Hadas2351776e2018-10-07 12:06:34 +0300362 MLX5_GET(general_obj_in_cmd_hdr, in,
363 obj_id));
Yishai Hadase662e142018-06-17 13:00:02 +0300364 break;
365 case MLX5_CMD_OP_QUERY_MKEY:
Yishai Hadas2351776e2018-10-07 12:06:34 +0300366 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_MKEY,
367 MLX5_GET(query_mkey_in, in,
368 mkey_index));
Yishai Hadase662e142018-06-17 13:00:02 +0300369 break;
370 case MLX5_CMD_OP_QUERY_CQ:
Yishai Hadas2351776e2018-10-07 12:06:34 +0300371 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_CQ,
372 MLX5_GET(query_cq_in, in, cqn));
Yishai Hadase662e142018-06-17 13:00:02 +0300373 break;
374 case MLX5_CMD_OP_MODIFY_CQ:
Yishai Hadas2351776e2018-10-07 12:06:34 +0300375 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_CQ,
376 MLX5_GET(modify_cq_in, in, cqn));
Yishai Hadase662e142018-06-17 13:00:02 +0300377 break;
378 case MLX5_CMD_OP_QUERY_SQ:
Yishai Hadas2351776e2018-10-07 12:06:34 +0300379 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SQ,
380 MLX5_GET(query_sq_in, in, sqn));
Yishai Hadase662e142018-06-17 13:00:02 +0300381 break;
382 case MLX5_CMD_OP_MODIFY_SQ:
Yishai Hadas2351776e2018-10-07 12:06:34 +0300383 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SQ,
384 MLX5_GET(modify_sq_in, in, sqn));
Yishai Hadase662e142018-06-17 13:00:02 +0300385 break;
386 case MLX5_CMD_OP_QUERY_RQ:
Yishai Hadas2351776e2018-10-07 12:06:34 +0300387 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
388 MLX5_GET(query_rq_in, in, rqn));
Yishai Hadase662e142018-06-17 13:00:02 +0300389 break;
390 case MLX5_CMD_OP_MODIFY_RQ:
Yishai Hadas2351776e2018-10-07 12:06:34 +0300391 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
392 MLX5_GET(modify_rq_in, in, rqn));
Yishai Hadase662e142018-06-17 13:00:02 +0300393 break;
394 case MLX5_CMD_OP_QUERY_RMP:
Yishai Hadas2351776e2018-10-07 12:06:34 +0300395 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RMP,
396 MLX5_GET(query_rmp_in, in, rmpn));
Yishai Hadase662e142018-06-17 13:00:02 +0300397 break;
398 case MLX5_CMD_OP_MODIFY_RMP:
Yishai Hadas2351776e2018-10-07 12:06:34 +0300399 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RMP,
400 MLX5_GET(modify_rmp_in, in, rmpn));
Yishai Hadase662e142018-06-17 13:00:02 +0300401 break;
402 case MLX5_CMD_OP_QUERY_RQT:
Yishai Hadas2351776e2018-10-07 12:06:34 +0300403 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQT,
404 MLX5_GET(query_rqt_in, in, rqtn));
Yishai Hadase662e142018-06-17 13:00:02 +0300405 break;
406 case MLX5_CMD_OP_MODIFY_RQT:
Yishai Hadas2351776e2018-10-07 12:06:34 +0300407 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQT,
408 MLX5_GET(modify_rqt_in, in, rqtn));
Yishai Hadase662e142018-06-17 13:00:02 +0300409 break;
410 case MLX5_CMD_OP_QUERY_TIR:
Yishai Hadas2351776e2018-10-07 12:06:34 +0300411 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIR,
412 MLX5_GET(query_tir_in, in, tirn));
Yishai Hadase662e142018-06-17 13:00:02 +0300413 break;
414 case MLX5_CMD_OP_MODIFY_TIR:
Yishai Hadas2351776e2018-10-07 12:06:34 +0300415 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIR,
416 MLX5_GET(modify_tir_in, in, tirn));
Yishai Hadase662e142018-06-17 13:00:02 +0300417 break;
418 case MLX5_CMD_OP_QUERY_TIS:
Yishai Hadas2351776e2018-10-07 12:06:34 +0300419 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIS,
420 MLX5_GET(query_tis_in, in, tisn));
Yishai Hadase662e142018-06-17 13:00:02 +0300421 break;
422 case MLX5_CMD_OP_MODIFY_TIS:
Yishai Hadas2351776e2018-10-07 12:06:34 +0300423 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIS,
424 MLX5_GET(modify_tis_in, in, tisn));
Yishai Hadase662e142018-06-17 13:00:02 +0300425 break;
426 case MLX5_CMD_OP_QUERY_FLOW_TABLE:
Yishai Hadas2351776e2018-10-07 12:06:34 +0300427 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_FLOW_TABLE,
428 MLX5_GET(query_flow_table_in, in,
429 table_id));
Yishai Hadase662e142018-06-17 13:00:02 +0300430 break;
431 case MLX5_CMD_OP_MODIFY_FLOW_TABLE:
Yishai Hadas2351776e2018-10-07 12:06:34 +0300432 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_FLOW_TABLE,
433 MLX5_GET(modify_flow_table_in, in,
434 table_id));
Yishai Hadase662e142018-06-17 13:00:02 +0300435 break;
436 case MLX5_CMD_OP_QUERY_FLOW_GROUP:
Yishai Hadas2351776e2018-10-07 12:06:34 +0300437 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_FLOW_GROUP,
438 MLX5_GET(query_flow_group_in, in,
439 group_id));
Yishai Hadase662e142018-06-17 13:00:02 +0300440 break;
441 case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY:
Yishai Hadas2351776e2018-10-07 12:06:34 +0300442 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY,
443 MLX5_GET(query_fte_in, in,
444 flow_index));
Yishai Hadase662e142018-06-17 13:00:02 +0300445 break;
446 case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
Yishai Hadas2351776e2018-10-07 12:06:34 +0300447 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY,
448 MLX5_GET(set_fte_in, in, flow_index));
Yishai Hadase662e142018-06-17 13:00:02 +0300449 break;
450 case MLX5_CMD_OP_QUERY_Q_COUNTER:
Yishai Hadas2351776e2018-10-07 12:06:34 +0300451 obj_id = get_enc_obj_id(MLX5_CMD_OP_ALLOC_Q_COUNTER,
452 MLX5_GET(query_q_counter_in, in,
453 counter_set_id));
Yishai Hadase662e142018-06-17 13:00:02 +0300454 break;
455 case MLX5_CMD_OP_QUERY_FLOW_COUNTER:
Yishai Hadas2351776e2018-10-07 12:06:34 +0300456 obj_id = get_enc_obj_id(MLX5_CMD_OP_ALLOC_FLOW_COUNTER,
457 MLX5_GET(query_flow_counter_in, in,
458 flow_counter_id));
Yishai Hadase662e142018-06-17 13:00:02 +0300459 break;
460 case MLX5_CMD_OP_QUERY_MODIFY_HEADER_CONTEXT:
Yishai Hadas2351776e2018-10-07 12:06:34 +0300461 obj_id = get_enc_obj_id(MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT,
462 MLX5_GET(general_obj_in_cmd_hdr, in,
463 obj_id));
Yishai Hadase662e142018-06-17 13:00:02 +0300464 break;
465 case MLX5_CMD_OP_QUERY_SCHEDULING_ELEMENT:
Yishai Hadas2351776e2018-10-07 12:06:34 +0300466 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT,
467 MLX5_GET(query_scheduling_element_in,
468 in, scheduling_element_id));
Yishai Hadase662e142018-06-17 13:00:02 +0300469 break;
470 case MLX5_CMD_OP_MODIFY_SCHEDULING_ELEMENT:
Yishai Hadas2351776e2018-10-07 12:06:34 +0300471 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT,
472 MLX5_GET(modify_scheduling_element_in,
473 in, scheduling_element_id));
Yishai Hadase662e142018-06-17 13:00:02 +0300474 break;
475 case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
Yishai Hadas2351776e2018-10-07 12:06:34 +0300476 obj_id = get_enc_obj_id(MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT,
477 MLX5_GET(add_vxlan_udp_dport_in, in,
478 vxlan_udp_port));
Yishai Hadase662e142018-06-17 13:00:02 +0300479 break;
480 case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY:
Yishai Hadas2351776e2018-10-07 12:06:34 +0300481 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_L2_TABLE_ENTRY,
482 MLX5_GET(query_l2_table_entry_in, in,
483 table_index));
Yishai Hadase662e142018-06-17 13:00:02 +0300484 break;
485 case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
Yishai Hadas2351776e2018-10-07 12:06:34 +0300486 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_L2_TABLE_ENTRY,
487 MLX5_GET(set_l2_table_entry_in, in,
488 table_index));
Yishai Hadase662e142018-06-17 13:00:02 +0300489 break;
490 case MLX5_CMD_OP_QUERY_QP:
Yishai Hadas2351776e2018-10-07 12:06:34 +0300491 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
492 MLX5_GET(query_qp_in, in, qpn));
Yishai Hadase662e142018-06-17 13:00:02 +0300493 break;
494 case MLX5_CMD_OP_RST2INIT_QP:
Yishai Hadas2351776e2018-10-07 12:06:34 +0300495 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
496 MLX5_GET(rst2init_qp_in, in, qpn));
Yishai Hadase662e142018-06-17 13:00:02 +0300497 break;
Mark Zhangd246a302020-05-27 16:57:03 +0300498 case MLX5_CMD_OP_INIT2INIT_QP:
499 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
500 MLX5_GET(init2init_qp_in, in, qpn));
501 break;
Yishai Hadase662e142018-06-17 13:00:02 +0300502 case MLX5_CMD_OP_INIT2RTR_QP:
Yishai Hadas2351776e2018-10-07 12:06:34 +0300503 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
504 MLX5_GET(init2rtr_qp_in, in, qpn));
Yishai Hadase662e142018-06-17 13:00:02 +0300505 break;
506 case MLX5_CMD_OP_RTR2RTS_QP:
Yishai Hadas2351776e2018-10-07 12:06:34 +0300507 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
508 MLX5_GET(rtr2rts_qp_in, in, qpn));
Yishai Hadase662e142018-06-17 13:00:02 +0300509 break;
510 case MLX5_CMD_OP_RTS2RTS_QP:
Yishai Hadas2351776e2018-10-07 12:06:34 +0300511 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
512 MLX5_GET(rts2rts_qp_in, in, qpn));
Yishai Hadase662e142018-06-17 13:00:02 +0300513 break;
514 case MLX5_CMD_OP_SQERR2RTS_QP:
Yishai Hadas2351776e2018-10-07 12:06:34 +0300515 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
516 MLX5_GET(sqerr2rts_qp_in, in, qpn));
Yishai Hadase662e142018-06-17 13:00:02 +0300517 break;
518 case MLX5_CMD_OP_2ERR_QP:
Yishai Hadas2351776e2018-10-07 12:06:34 +0300519 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
520 MLX5_GET(qp_2err_in, in, qpn));
Yishai Hadase662e142018-06-17 13:00:02 +0300521 break;
522 case MLX5_CMD_OP_2RST_QP:
Yishai Hadas2351776e2018-10-07 12:06:34 +0300523 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
524 MLX5_GET(qp_2rst_in, in, qpn));
Yishai Hadase662e142018-06-17 13:00:02 +0300525 break;
526 case MLX5_CMD_OP_QUERY_DCT:
Yishai Hadas2351776e2018-10-07 12:06:34 +0300527 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_DCT,
528 MLX5_GET(query_dct_in, in, dctn));
Yishai Hadase662e142018-06-17 13:00:02 +0300529 break;
530 case MLX5_CMD_OP_QUERY_XRQ:
Yishai Hadas719598c2018-11-26 08:28:37 +0200531 case MLX5_CMD_OP_QUERY_XRQ_DC_PARAMS_ENTRY:
532 case MLX5_CMD_OP_QUERY_XRQ_ERROR_PARAMS:
Yishai Hadas2351776e2018-10-07 12:06:34 +0300533 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRQ,
534 MLX5_GET(query_xrq_in, in, xrqn));
Yishai Hadase662e142018-06-17 13:00:02 +0300535 break;
536 case MLX5_CMD_OP_QUERY_XRC_SRQ:
Yishai Hadas2351776e2018-10-07 12:06:34 +0300537 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRC_SRQ,
538 MLX5_GET(query_xrc_srq_in, in,
539 xrc_srqn));
Yishai Hadase662e142018-06-17 13:00:02 +0300540 break;
541 case MLX5_CMD_OP_ARM_XRC_SRQ:
Yishai Hadas2351776e2018-10-07 12:06:34 +0300542 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRC_SRQ,
543 MLX5_GET(arm_xrc_srq_in, in, xrc_srqn));
Yishai Hadase662e142018-06-17 13:00:02 +0300544 break;
545 case MLX5_CMD_OP_QUERY_SRQ:
Yishai Hadas2351776e2018-10-07 12:06:34 +0300546 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SRQ,
547 MLX5_GET(query_srq_in, in, srqn));
Yishai Hadase662e142018-06-17 13:00:02 +0300548 break;
549 case MLX5_CMD_OP_ARM_RQ:
Yishai Hadas2351776e2018-10-07 12:06:34 +0300550 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
551 MLX5_GET(arm_rq_in, in, srq_number));
Yishai Hadase662e142018-06-17 13:00:02 +0300552 break;
Yishai Hadase662e142018-06-17 13:00:02 +0300553 case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION:
Yishai Hadas2351776e2018-10-07 12:06:34 +0300554 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_DCT,
555 MLX5_GET(drain_dct_in, in, dctn));
Yishai Hadase662e142018-06-17 13:00:02 +0300556 break;
557 case MLX5_CMD_OP_ARM_XRQ:
Yishai Hadas719598c2018-11-26 08:28:37 +0200558 case MLX5_CMD_OP_SET_XRQ_DC_PARAMS_ENTRY:
Yishai Hadas8293a592019-08-08 11:43:58 +0300559 case MLX5_CMD_OP_RELEASE_XRQ_ERROR:
560 case MLX5_CMD_OP_MODIFY_XRQ:
Yishai Hadas2351776e2018-10-07 12:06:34 +0300561 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRQ,
562 MLX5_GET(arm_xrq_in, in, xrqn));
Yishai Hadase662e142018-06-17 13:00:02 +0300563 break;
Yishai Hadas719598c2018-11-26 08:28:37 +0200564 case MLX5_CMD_OP_QUERY_PACKET_REFORMAT_CONTEXT:
565 obj_id = get_enc_obj_id
566 (MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT,
567 MLX5_GET(query_packet_reformat_context_in,
568 in, packet_reformat_id));
569 break;
Yishai Hadase662e142018-06-17 13:00:02 +0300570 default:
Yishai Hadas34613eb2018-11-26 08:28:35 +0200571 obj_id = 0;
Yishai Hadase662e142018-06-17 13:00:02 +0300572 }
573
Yishai Hadas34613eb2018-11-26 08:28:35 +0200574 return obj_id;
575}
Yishai Hadase662e142018-06-17 13:00:02 +0300576
Jason Gunthorpee79c9c62019-04-01 17:08:23 -0300577static bool devx_is_valid_obj_id(struct uverbs_attr_bundle *attrs,
578 struct ib_uobject *uobj, const void *in)
Yishai Hadas34613eb2018-11-26 08:28:35 +0200579{
Jason Gunthorpee79c9c62019-04-01 17:08:23 -0300580 struct mlx5_ib_dev *dev = mlx5_udata_to_mdev(&attrs->driver_udata);
Yishai Hadas34613eb2018-11-26 08:28:35 +0200581 u64 obj_id = devx_get_obj_id(in);
582
583 if (!obj_id)
584 return false;
585
586 switch (uobj_get_object_id(uobj)) {
587 case UVERBS_OBJECT_CQ:
588 return get_enc_obj_id(MLX5_CMD_OP_CREATE_CQ,
589 to_mcq(uobj->object)->mcq.cqn) ==
590 obj_id;
591
592 case UVERBS_OBJECT_SRQ:
593 {
594 struct mlx5_core_srq *srq = &(to_msrq(uobj->object)->msrq);
Yishai Hadas34613eb2018-11-26 08:28:35 +0200595 u16 opcode;
596
597 switch (srq->common.res) {
598 case MLX5_RES_XSRQ:
599 opcode = MLX5_CMD_OP_CREATE_XRC_SRQ;
600 break;
601 case MLX5_RES_XRQ:
602 opcode = MLX5_CMD_OP_CREATE_XRQ;
603 break;
604 default:
605 if (!dev->mdev->issi)
606 opcode = MLX5_CMD_OP_CREATE_SRQ;
607 else
608 opcode = MLX5_CMD_OP_CREATE_RMP;
609 }
610
611 return get_enc_obj_id(opcode,
612 to_msrq(uobj->object)->msrq.srqn) ==
613 obj_id;
614 }
615
616 case UVERBS_OBJECT_QP:
617 {
618 struct mlx5_ib_qp *qp = to_mqp(uobj->object);
619 enum ib_qp_type qp_type = qp->ibqp.qp_type;
620
621 if (qp_type == IB_QPT_RAW_PACKET ||
Leon Romanovsky2be08c32020-04-27 18:46:13 +0300622 (qp->flags & IB_QP_CREATE_SOURCE_QPN)) {
Yishai Hadas34613eb2018-11-26 08:28:35 +0200623 struct mlx5_ib_raw_packet_qp *raw_packet_qp =
624 &qp->raw_packet_qp;
625 struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
626 struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
627
628 return (get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
629 rq->base.mqp.qpn) == obj_id ||
630 get_enc_obj_id(MLX5_CMD_OP_CREATE_SQ,
631 sq->base.mqp.qpn) == obj_id ||
632 get_enc_obj_id(MLX5_CMD_OP_CREATE_TIR,
633 rq->tirn) == obj_id ||
634 get_enc_obj_id(MLX5_CMD_OP_CREATE_TIS,
635 sq->tisn) == obj_id);
636 }
637
638 if (qp_type == MLX5_IB_QPT_DCT)
639 return get_enc_obj_id(MLX5_CMD_OP_CREATE_DCT,
640 qp->dct.mdct.mqp.qpn) == obj_id;
641
642 return get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
643 qp->ibqp.qp_num) == obj_id;
644 }
645
646 case UVERBS_OBJECT_WQ:
647 return get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
648 to_mrwq(uobj->object)->core_qp.qpn) ==
649 obj_id;
650
651 case UVERBS_OBJECT_RWQ_IND_TBL:
652 return get_enc_obj_id(MLX5_CMD_OP_CREATE_RQT,
653 to_mrwq_ind_table(uobj->object)->rqtn) ==
654 obj_id;
655
656 case MLX5_IB_OBJECT_DEVX_OBJ:
657 return ((struct devx_obj *)uobj->object)->obj_id == obj_id;
658
Yishai Hadase662e142018-06-17 13:00:02 +0300659 default:
660 return false;
661 }
Yishai Hadase662e142018-06-17 13:00:02 +0300662}
663
Yishai Hadasba1a0572018-09-20 21:39:33 +0300664static void devx_set_umem_valid(const void *in)
665{
666 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
667
668 switch (opcode) {
669 case MLX5_CMD_OP_CREATE_MKEY:
670 MLX5_SET(create_mkey_in, in, mkey_umem_valid, 1);
671 break;
672 case MLX5_CMD_OP_CREATE_CQ:
673 {
674 void *cqc;
675
676 MLX5_SET(create_cq_in, in, cq_umem_valid, 1);
677 cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
678 MLX5_SET(cqc, cqc, dbr_umem_valid, 1);
679 break;
680 }
681 case MLX5_CMD_OP_CREATE_QP:
682 {
683 void *qpc;
684
685 qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
686 MLX5_SET(qpc, qpc, dbr_umem_valid, 1);
687 MLX5_SET(create_qp_in, in, wq_umem_valid, 1);
688 break;
689 }
690
691 case MLX5_CMD_OP_CREATE_RQ:
692 {
693 void *rqc, *wq;
694
695 rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
696 wq = MLX5_ADDR_OF(rqc, rqc, wq);
697 MLX5_SET(wq, wq, dbr_umem_valid, 1);
698 MLX5_SET(wq, wq, wq_umem_valid, 1);
699 break;
700 }
701
702 case MLX5_CMD_OP_CREATE_SQ:
703 {
704 void *sqc, *wq;
705
706 sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
707 wq = MLX5_ADDR_OF(sqc, sqc, wq);
708 MLX5_SET(wq, wq, dbr_umem_valid, 1);
709 MLX5_SET(wq, wq, wq_umem_valid, 1);
710 break;
711 }
712
713 case MLX5_CMD_OP_MODIFY_CQ:
714 MLX5_SET(modify_cq_in, in, cq_umem_valid, 1);
715 break;
716
717 case MLX5_CMD_OP_CREATE_RMP:
718 {
719 void *rmpc, *wq;
720
721 rmpc = MLX5_ADDR_OF(create_rmp_in, in, ctx);
722 wq = MLX5_ADDR_OF(rmpc, rmpc, wq);
723 MLX5_SET(wq, wq, dbr_umem_valid, 1);
724 MLX5_SET(wq, wq, wq_umem_valid, 1);
725 break;
726 }
727
728 case MLX5_CMD_OP_CREATE_XRQ:
729 {
730 void *xrqc, *wq;
731
732 xrqc = MLX5_ADDR_OF(create_xrq_in, in, xrq_context);
733 wq = MLX5_ADDR_OF(xrqc, xrqc, wq);
734 MLX5_SET(wq, wq, dbr_umem_valid, 1);
735 MLX5_SET(wq, wq, wq_umem_valid, 1);
736 break;
737 }
738
739 case MLX5_CMD_OP_CREATE_XRC_SRQ:
740 {
741 void *xrc_srqc;
742
743 MLX5_SET(create_xrc_srq_in, in, xrc_srq_umem_valid, 1);
744 xrc_srqc = MLX5_ADDR_OF(create_xrc_srq_in, in,
745 xrc_srq_context_entry);
746 MLX5_SET(xrc_srqc, xrc_srqc, dbr_umem_valid, 1);
747 break;
748 }
749
750 default:
751 return;
752 }
753}
754
Yishai Hadas2351776e2018-10-07 12:06:34 +0300755static bool devx_is_obj_create_cmd(const void *in, u16 *opcode)
Yishai Hadas7efce362018-06-17 13:00:01 +0300756{
Yishai Hadas2351776e2018-10-07 12:06:34 +0300757 *opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
Yishai Hadas7efce362018-06-17 13:00:01 +0300758
Yishai Hadas2351776e2018-10-07 12:06:34 +0300759 switch (*opcode) {
Yishai Hadas7efce362018-06-17 13:00:01 +0300760 case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
761 case MLX5_CMD_OP_CREATE_MKEY:
762 case MLX5_CMD_OP_CREATE_CQ:
763 case MLX5_CMD_OP_ALLOC_PD:
764 case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN:
765 case MLX5_CMD_OP_CREATE_RMP:
766 case MLX5_CMD_OP_CREATE_SQ:
767 case MLX5_CMD_OP_CREATE_RQ:
768 case MLX5_CMD_OP_CREATE_RQT:
769 case MLX5_CMD_OP_CREATE_TIR:
770 case MLX5_CMD_OP_CREATE_TIS:
771 case MLX5_CMD_OP_ALLOC_Q_COUNTER:
772 case MLX5_CMD_OP_CREATE_FLOW_TABLE:
773 case MLX5_CMD_OP_CREATE_FLOW_GROUP:
774 case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
Mark Bloch60786f02018-08-28 14:18:46 +0300775 case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT:
Yishai Hadas7efce362018-06-17 13:00:01 +0300776 case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT:
777 case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT:
778 case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
779 case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
780 case MLX5_CMD_OP_CREATE_QP:
781 case MLX5_CMD_OP_CREATE_SRQ:
782 case MLX5_CMD_OP_CREATE_XRC_SRQ:
783 case MLX5_CMD_OP_CREATE_DCT:
784 case MLX5_CMD_OP_CREATE_XRQ:
785 case MLX5_CMD_OP_ATTACH_TO_MCG:
786 case MLX5_CMD_OP_ALLOC_XRCD:
787 return true;
788 case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
789 {
790 u16 op_mod = MLX5_GET(set_fte_in, in, op_mod);
791 if (op_mod == 0)
792 return true;
793 return false;
794 }
Max Gurtovoy8b38c532019-07-23 10:04:12 +0300795 case MLX5_CMD_OP_CREATE_PSV:
796 {
797 u8 num_psv = MLX5_GET(create_psv_in, in, num_psv);
798
799 if (num_psv == 1)
800 return true;
801 return false;
802 }
Yishai Hadas7efce362018-06-17 13:00:01 +0300803 default:
804 return false;
805 }
806}
807
Yishai Hadase662e142018-06-17 13:00:02 +0300808static bool devx_is_obj_modify_cmd(const void *in)
809{
810 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
811
812 switch (opcode) {
813 case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT:
814 case MLX5_CMD_OP_MODIFY_CQ:
815 case MLX5_CMD_OP_MODIFY_RMP:
816 case MLX5_CMD_OP_MODIFY_SQ:
817 case MLX5_CMD_OP_MODIFY_RQ:
818 case MLX5_CMD_OP_MODIFY_RQT:
819 case MLX5_CMD_OP_MODIFY_TIR:
820 case MLX5_CMD_OP_MODIFY_TIS:
821 case MLX5_CMD_OP_MODIFY_FLOW_TABLE:
822 case MLX5_CMD_OP_MODIFY_SCHEDULING_ELEMENT:
823 case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
824 case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
825 case MLX5_CMD_OP_RST2INIT_QP:
826 case MLX5_CMD_OP_INIT2RTR_QP:
Aharon Landau819f7422020-05-13 12:55:50 +0300827 case MLX5_CMD_OP_INIT2INIT_QP:
Yishai Hadase662e142018-06-17 13:00:02 +0300828 case MLX5_CMD_OP_RTR2RTS_QP:
829 case MLX5_CMD_OP_RTS2RTS_QP:
830 case MLX5_CMD_OP_SQERR2RTS_QP:
831 case MLX5_CMD_OP_2ERR_QP:
832 case MLX5_CMD_OP_2RST_QP:
833 case MLX5_CMD_OP_ARM_XRC_SRQ:
834 case MLX5_CMD_OP_ARM_RQ:
Yishai Hadase662e142018-06-17 13:00:02 +0300835 case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION:
836 case MLX5_CMD_OP_ARM_XRQ:
Yishai Hadas719598c2018-11-26 08:28:37 +0200837 case MLX5_CMD_OP_SET_XRQ_DC_PARAMS_ENTRY:
Yishai Hadas8293a592019-08-08 11:43:58 +0300838 case MLX5_CMD_OP_RELEASE_XRQ_ERROR:
839 case MLX5_CMD_OP_MODIFY_XRQ:
Yishai Hadase662e142018-06-17 13:00:02 +0300840 return true;
841 case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
842 {
843 u16 op_mod = MLX5_GET(set_fte_in, in, op_mod);
844
845 if (op_mod == 1)
846 return true;
847 return false;
848 }
849 default:
850 return false;
851 }
852}
853
854static bool devx_is_obj_query_cmd(const void *in)
855{
856 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
857
858 switch (opcode) {
859 case MLX5_CMD_OP_QUERY_GENERAL_OBJECT:
860 case MLX5_CMD_OP_QUERY_MKEY:
861 case MLX5_CMD_OP_QUERY_CQ:
862 case MLX5_CMD_OP_QUERY_RMP:
863 case MLX5_CMD_OP_QUERY_SQ:
864 case MLX5_CMD_OP_QUERY_RQ:
865 case MLX5_CMD_OP_QUERY_RQT:
866 case MLX5_CMD_OP_QUERY_TIR:
867 case MLX5_CMD_OP_QUERY_TIS:
868 case MLX5_CMD_OP_QUERY_Q_COUNTER:
869 case MLX5_CMD_OP_QUERY_FLOW_TABLE:
870 case MLX5_CMD_OP_QUERY_FLOW_GROUP:
871 case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY:
872 case MLX5_CMD_OP_QUERY_FLOW_COUNTER:
873 case MLX5_CMD_OP_QUERY_MODIFY_HEADER_CONTEXT:
874 case MLX5_CMD_OP_QUERY_SCHEDULING_ELEMENT:
875 case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY:
876 case MLX5_CMD_OP_QUERY_QP:
877 case MLX5_CMD_OP_QUERY_SRQ:
878 case MLX5_CMD_OP_QUERY_XRC_SRQ:
879 case MLX5_CMD_OP_QUERY_DCT:
880 case MLX5_CMD_OP_QUERY_XRQ:
Yishai Hadas719598c2018-11-26 08:28:37 +0200881 case MLX5_CMD_OP_QUERY_XRQ_DC_PARAMS_ENTRY:
882 case MLX5_CMD_OP_QUERY_XRQ_ERROR_PARAMS:
883 case MLX5_CMD_OP_QUERY_PACKET_REFORMAT_CONTEXT:
Yishai Hadase662e142018-06-17 13:00:02 +0300884 return true;
885 default:
886 return false;
887 }
888}
889
Yishai Hadas7e1335a2018-09-20 21:45:20 +0300890static bool devx_is_whitelist_cmd(void *in)
891{
892 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
893
894 switch (opcode) {
895 case MLX5_CMD_OP_QUERY_HCA_CAP:
896 case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT:
Maor Gottlieb56e5acd2019-03-28 15:46:24 +0200897 case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT:
Yishai Hadas7e1335a2018-09-20 21:45:20 +0300898 return true;
899 default:
900 return false;
901 }
902}
903
904static int devx_get_uid(struct mlx5_ib_ucontext *c, void *cmd_in)
905{
906 if (devx_is_whitelist_cmd(cmd_in)) {
907 struct mlx5_ib_dev *dev;
908
909 if (c->devx_uid)
910 return c->devx_uid;
911
912 dev = to_mdev(c->ibucontext.device);
913 if (dev->devx_whitelist_uid)
914 return dev->devx_whitelist_uid;
915
916 return -EOPNOTSUPP;
917 }
918
919 if (!c->devx_uid)
920 return -EINVAL;
921
Yishai Hadas7e1335a2018-09-20 21:45:20 +0300922 return c->devx_uid;
923}
Max Gurtovoyb6142602019-07-01 21:14:02 +0300924
925static bool devx_is_general_cmd(void *in, struct mlx5_ib_dev *dev)
Yishai Hadas8aa8c952018-06-17 13:00:00 +0300926{
927 u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
928
Max Gurtovoyb6142602019-07-01 21:14:02 +0300929 /* Pass all cmds for vhca_tunnel as general, tracking is done in FW */
930 if ((MLX5_CAP_GEN_64(dev->mdev, vhca_tunnel_commands) &&
931 MLX5_GET(general_obj_in_cmd_hdr, in, vhca_tunnel_id)) ||
932 (opcode >= MLX5_CMD_OP_GENERAL_START &&
933 opcode < MLX5_CMD_OP_GENERAL_END))
Yishai Hadas719598c2018-11-26 08:28:37 +0200934 return true;
935
Yishai Hadas8aa8c952018-06-17 13:00:00 +0300936 switch (opcode) {
937 case MLX5_CMD_OP_QUERY_HCA_CAP:
Yishai Hadas7e1335a2018-09-20 21:45:20 +0300938 case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT:
Maor Gottlieb56e5acd2019-03-28 15:46:24 +0200939 case MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT:
Yishai Hadas8aa8c952018-06-17 13:00:00 +0300940 case MLX5_CMD_OP_QUERY_VPORT_STATE:
941 case MLX5_CMD_OP_QUERY_ADAPTER:
942 case MLX5_CMD_OP_QUERY_ISSI:
943 case MLX5_CMD_OP_QUERY_NIC_VPORT_CONTEXT:
944 case MLX5_CMD_OP_QUERY_ROCE_ADDRESS:
945 case MLX5_CMD_OP_QUERY_VNIC_ENV:
946 case MLX5_CMD_OP_QUERY_VPORT_COUNTER:
947 case MLX5_CMD_OP_GET_DROPPED_PACKET_LOG:
948 case MLX5_CMD_OP_NOP:
949 case MLX5_CMD_OP_QUERY_CONG_STATUS:
950 case MLX5_CMD_OP_QUERY_CONG_PARAMS:
951 case MLX5_CMD_OP_QUERY_CONG_STATISTICS:
Mark Zhang7084ed32019-07-31 14:40:14 +0300952 case MLX5_CMD_OP_QUERY_LAG:
Yishai Hadas8aa8c952018-06-17 13:00:00 +0300953 return true;
954 default:
955 return false;
956 }
957}
958
Jason Gunthorpee83f0ec2018-07-25 21:40:18 -0600959static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_QUERY_EQN)(
Jason Gunthorpe15a1b4b2018-11-25 20:51:15 +0200960 struct uverbs_attr_bundle *attrs)
Yishai Hadasf6fe01b2018-06-17 13:00:05 +0300961{
Jason Gunthorpee83f0ec2018-07-25 21:40:18 -0600962 struct mlx5_ib_ucontext *c;
963 struct mlx5_ib_dev *dev;
Yishai Hadasf6fe01b2018-06-17 13:00:05 +0300964 int user_vector;
965 int dev_eqn;
966 unsigned int irqn;
967 int err;
968
969 if (uverbs_copy_from(&user_vector, attrs,
970 MLX5_IB_ATTR_DEVX_QUERY_EQN_USER_VEC))
971 return -EFAULT;
972
Jason Gunthorpe15a1b4b2018-11-25 20:51:15 +0200973 c = devx_ufile2uctx(attrs);
Jason Gunthorpee83f0ec2018-07-25 21:40:18 -0600974 if (IS_ERR(c))
975 return PTR_ERR(c);
976 dev = to_mdev(c->ibucontext.device);
977
Yishai Hadasf6fe01b2018-06-17 13:00:05 +0300978 err = mlx5_vector2eqn(dev->mdev, user_vector, &dev_eqn, &irqn);
979 if (err < 0)
980 return err;
981
982 if (uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_QUERY_EQN_DEV_EQN,
983 &dev_eqn, sizeof(dev_eqn)))
984 return -EFAULT;
985
986 return 0;
987}
988
Yishai Hadas7c043e92018-06-17 13:00:03 +0300989/*
990 *Security note:
991 * The hardware protection mechanism works like this: Each device object that
992 * is subject to UAR doorbells (QP/SQ/CQ) gets a UAR ID (called uar_page in
993 * the device specification manual) upon its creation. Then upon doorbell,
994 * hardware fetches the object context for which the doorbell was rang, and
995 * validates that the UAR through which the DB was rang matches the UAR ID
996 * of the object.
997 * If no match the doorbell is silently ignored by the hardware. Of course,
998 * the user cannot ring a doorbell on a UAR that was not mapped to it.
999 * Now in devx, as the devx kernel does not manipulate the QP/SQ/CQ command
1000 * mailboxes (except tagging them with UID), we expose to the user its UAR
1001 * ID, so it can embed it in these objects in the expected specification
1002 * format. So the only thing the user can do is hurt itself by creating a
1003 * QP/SQ/CQ with a UAR ID other than his, and then in this case other users
1004 * may ring a doorbell on its objects.
1005 * The consequence of that will be that another user can schedule a QP/SQ
1006 * of the buggy user for execution (just insert it to the hardware schedule
1007 * queue or arm its CQ for event generation), no further harm is expected.
1008 */
Jason Gunthorpee83f0ec2018-07-25 21:40:18 -06001009static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_QUERY_UAR)(
Jason Gunthorpe15a1b4b2018-11-25 20:51:15 +02001010 struct uverbs_attr_bundle *attrs)
Yishai Hadas7c043e92018-06-17 13:00:03 +03001011{
Jason Gunthorpe22fa27f2018-07-10 13:43:06 -06001012 struct mlx5_ib_ucontext *c;
1013 struct mlx5_ib_dev *dev;
Yishai Hadas7c043e92018-06-17 13:00:03 +03001014 u32 user_idx;
1015 s32 dev_idx;
1016
Jason Gunthorpe15a1b4b2018-11-25 20:51:15 +02001017 c = devx_ufile2uctx(attrs);
Jason Gunthorpe22fa27f2018-07-10 13:43:06 -06001018 if (IS_ERR(c))
1019 return PTR_ERR(c);
1020 dev = to_mdev(c->ibucontext.device);
1021
Yishai Hadas7c043e92018-06-17 13:00:03 +03001022 if (uverbs_copy_from(&user_idx, attrs,
1023 MLX5_IB_ATTR_DEVX_QUERY_UAR_USER_IDX))
1024 return -EFAULT;
1025
Jason Gunthorpe22fa27f2018-07-10 13:43:06 -06001026 dev_idx = bfregn_to_uar_index(dev, &c->bfregi, user_idx, true);
Yishai Hadas7c043e92018-06-17 13:00:03 +03001027 if (dev_idx < 0)
1028 return dev_idx;
1029
1030 if (uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_QUERY_UAR_DEV_IDX,
1031 &dev_idx, sizeof(dev_idx)))
1032 return -EFAULT;
1033
1034 return 0;
1035}
1036
Jason Gunthorpee83f0ec2018-07-25 21:40:18 -06001037static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OTHER)(
Jason Gunthorpe15a1b4b2018-11-25 20:51:15 +02001038 struct uverbs_attr_bundle *attrs)
Yishai Hadas8aa8c952018-06-17 13:00:00 +03001039{
Jason Gunthorpe22fa27f2018-07-10 13:43:06 -06001040 struct mlx5_ib_ucontext *c;
1041 struct mlx5_ib_dev *dev;
Yishai Hadas7efce362018-06-17 13:00:01 +03001042 void *cmd_in = uverbs_attr_get_alloced_ptr(
1043 attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_IN);
Yishai Hadas8aa8c952018-06-17 13:00:00 +03001044 int cmd_out_len = uverbs_attr_get_len(attrs,
1045 MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT);
1046 void *cmd_out;
1047 int err;
Yishai Hadas7e1335a2018-09-20 21:45:20 +03001048 int uid;
Yishai Hadas8aa8c952018-06-17 13:00:00 +03001049
Jason Gunthorpe15a1b4b2018-11-25 20:51:15 +02001050 c = devx_ufile2uctx(attrs);
Jason Gunthorpe22fa27f2018-07-10 13:43:06 -06001051 if (IS_ERR(c))
1052 return PTR_ERR(c);
1053 dev = to_mdev(c->ibucontext.device);
1054
Yishai Hadas7e1335a2018-09-20 21:45:20 +03001055 uid = devx_get_uid(c, cmd_in);
1056 if (uid < 0)
1057 return uid;
Yishai Hadas8aa8c952018-06-17 13:00:00 +03001058
1059 /* Only white list of some general HCA commands are allowed for this method. */
Max Gurtovoyb6142602019-07-01 21:14:02 +03001060 if (!devx_is_general_cmd(cmd_in, dev))
Yishai Hadas8aa8c952018-06-17 13:00:00 +03001061 return -EINVAL;
1062
Jason Gunthorpeb61815e2018-08-09 20:14:41 -06001063 cmd_out = uverbs_zalloc(attrs, cmd_out_len);
1064 if (IS_ERR(cmd_out))
1065 return PTR_ERR(cmd_out);
Yishai Hadas8aa8c952018-06-17 13:00:00 +03001066
Yishai Hadas7e1335a2018-09-20 21:45:20 +03001067 MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
Yishai Hadas8aa8c952018-06-17 13:00:00 +03001068 err = mlx5_cmd_exec(dev->mdev, cmd_in,
1069 uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_IN),
1070 cmd_out, cmd_out_len);
1071 if (err)
Jason Gunthorpeb61815e2018-08-09 20:14:41 -06001072 return err;
Yishai Hadas8aa8c952018-06-17 13:00:00 +03001073
Jason Gunthorpeb61815e2018-08-09 20:14:41 -06001074 return uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT, cmd_out,
1075 cmd_out_len);
Yishai Hadas8aa8c952018-06-17 13:00:00 +03001076}
1077
Yishai Hadas7efce362018-06-17 13:00:01 +03001078static void devx_obj_build_destroy_cmd(void *in, void *out, void *din,
1079 u32 *dinlen,
1080 u32 *obj_id)
1081{
1082 u16 obj_type = MLX5_GET(general_obj_in_cmd_hdr, in, obj_type);
1083 u16 uid = MLX5_GET(general_obj_in_cmd_hdr, in, uid);
1084
1085 *obj_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
1086 *dinlen = MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr);
1087
1088 MLX5_SET(general_obj_in_cmd_hdr, din, obj_id, *obj_id);
1089 MLX5_SET(general_obj_in_cmd_hdr, din, uid, uid);
1090
1091 switch (MLX5_GET(general_obj_in_cmd_hdr, in, opcode)) {
1092 case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
1093 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT);
1094 MLX5_SET(general_obj_in_cmd_hdr, din, obj_type, obj_type);
1095 break;
1096
Yishai Hadas6e3722b2018-12-19 16:28:15 +02001097 case MLX5_CMD_OP_CREATE_UMEM:
1098 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1099 MLX5_CMD_OP_DESTROY_UMEM);
1100 break;
Yishai Hadas7efce362018-06-17 13:00:01 +03001101 case MLX5_CMD_OP_CREATE_MKEY:
1102 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_MKEY);
1103 break;
1104 case MLX5_CMD_OP_CREATE_CQ:
1105 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_CQ);
1106 break;
1107 case MLX5_CMD_OP_ALLOC_PD:
1108 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DEALLOC_PD);
1109 break;
1110 case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN:
1111 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1112 MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN);
1113 break;
1114 case MLX5_CMD_OP_CREATE_RMP:
1115 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_RMP);
1116 break;
1117 case MLX5_CMD_OP_CREATE_SQ:
1118 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_SQ);
1119 break;
1120 case MLX5_CMD_OP_CREATE_RQ:
1121 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_RQ);
1122 break;
1123 case MLX5_CMD_OP_CREATE_RQT:
1124 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_RQT);
1125 break;
1126 case MLX5_CMD_OP_CREATE_TIR:
1127 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_TIR);
1128 break;
1129 case MLX5_CMD_OP_CREATE_TIS:
1130 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_TIS);
1131 break;
1132 case MLX5_CMD_OP_ALLOC_Q_COUNTER:
1133 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1134 MLX5_CMD_OP_DEALLOC_Q_COUNTER);
1135 break;
1136 case MLX5_CMD_OP_CREATE_FLOW_TABLE:
1137 *dinlen = MLX5_ST_SZ_BYTES(destroy_flow_table_in);
1138 *obj_id = MLX5_GET(create_flow_table_out, out, table_id);
1139 MLX5_SET(destroy_flow_table_in, din, other_vport,
1140 MLX5_GET(create_flow_table_in, in, other_vport));
1141 MLX5_SET(destroy_flow_table_in, din, vport_number,
1142 MLX5_GET(create_flow_table_in, in, vport_number));
1143 MLX5_SET(destroy_flow_table_in, din, table_type,
1144 MLX5_GET(create_flow_table_in, in, table_type));
1145 MLX5_SET(destroy_flow_table_in, din, table_id, *obj_id);
1146 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1147 MLX5_CMD_OP_DESTROY_FLOW_TABLE);
1148 break;
1149 case MLX5_CMD_OP_CREATE_FLOW_GROUP:
1150 *dinlen = MLX5_ST_SZ_BYTES(destroy_flow_group_in);
1151 *obj_id = MLX5_GET(create_flow_group_out, out, group_id);
1152 MLX5_SET(destroy_flow_group_in, din, other_vport,
1153 MLX5_GET(create_flow_group_in, in, other_vport));
1154 MLX5_SET(destroy_flow_group_in, din, vport_number,
1155 MLX5_GET(create_flow_group_in, in, vport_number));
1156 MLX5_SET(destroy_flow_group_in, din, table_type,
1157 MLX5_GET(create_flow_group_in, in, table_type));
1158 MLX5_SET(destroy_flow_group_in, din, table_id,
1159 MLX5_GET(create_flow_group_in, in, table_id));
1160 MLX5_SET(destroy_flow_group_in, din, group_id, *obj_id);
1161 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1162 MLX5_CMD_OP_DESTROY_FLOW_GROUP);
1163 break;
1164 case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
1165 *dinlen = MLX5_ST_SZ_BYTES(delete_fte_in);
1166 *obj_id = MLX5_GET(set_fte_in, in, flow_index);
1167 MLX5_SET(delete_fte_in, din, other_vport,
1168 MLX5_GET(set_fte_in, in, other_vport));
1169 MLX5_SET(delete_fte_in, din, vport_number,
1170 MLX5_GET(set_fte_in, in, vport_number));
1171 MLX5_SET(delete_fte_in, din, table_type,
1172 MLX5_GET(set_fte_in, in, table_type));
1173 MLX5_SET(delete_fte_in, din, table_id,
1174 MLX5_GET(set_fte_in, in, table_id));
1175 MLX5_SET(delete_fte_in, din, flow_index, *obj_id);
1176 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1177 MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY);
1178 break;
1179 case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
1180 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1181 MLX5_CMD_OP_DEALLOC_FLOW_COUNTER);
1182 break;
Mark Bloch60786f02018-08-28 14:18:46 +03001183 case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT:
Yishai Hadas7efce362018-06-17 13:00:01 +03001184 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
Mark Bloch60786f02018-08-28 14:18:46 +03001185 MLX5_CMD_OP_DEALLOC_PACKET_REFORMAT_CONTEXT);
Yishai Hadas7efce362018-06-17 13:00:01 +03001186 break;
1187 case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT:
1188 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1189 MLX5_CMD_OP_DEALLOC_MODIFY_HEADER_CONTEXT);
1190 break;
1191 case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT:
1192 *dinlen = MLX5_ST_SZ_BYTES(destroy_scheduling_element_in);
1193 *obj_id = MLX5_GET(create_scheduling_element_out, out,
1194 scheduling_element_id);
1195 MLX5_SET(destroy_scheduling_element_in, din,
1196 scheduling_hierarchy,
1197 MLX5_GET(create_scheduling_element_in, in,
1198 scheduling_hierarchy));
1199 MLX5_SET(destroy_scheduling_element_in, din,
1200 scheduling_element_id, *obj_id);
1201 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1202 MLX5_CMD_OP_DESTROY_SCHEDULING_ELEMENT);
1203 break;
1204 case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
1205 *dinlen = MLX5_ST_SZ_BYTES(delete_vxlan_udp_dport_in);
1206 *obj_id = MLX5_GET(add_vxlan_udp_dport_in, in, vxlan_udp_port);
1207 MLX5_SET(delete_vxlan_udp_dport_in, din, vxlan_udp_port, *obj_id);
1208 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1209 MLX5_CMD_OP_DELETE_VXLAN_UDP_DPORT);
1210 break;
1211 case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
1212 *dinlen = MLX5_ST_SZ_BYTES(delete_l2_table_entry_in);
1213 *obj_id = MLX5_GET(set_l2_table_entry_in, in, table_index);
1214 MLX5_SET(delete_l2_table_entry_in, din, table_index, *obj_id);
1215 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1216 MLX5_CMD_OP_DELETE_L2_TABLE_ENTRY);
1217 break;
1218 case MLX5_CMD_OP_CREATE_QP:
1219 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_QP);
1220 break;
1221 case MLX5_CMD_OP_CREATE_SRQ:
1222 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_SRQ);
1223 break;
1224 case MLX5_CMD_OP_CREATE_XRC_SRQ:
1225 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1226 MLX5_CMD_OP_DESTROY_XRC_SRQ);
1227 break;
1228 case MLX5_CMD_OP_CREATE_DCT:
1229 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_DCT);
1230 break;
1231 case MLX5_CMD_OP_CREATE_XRQ:
1232 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_XRQ);
1233 break;
1234 case MLX5_CMD_OP_ATTACH_TO_MCG:
1235 *dinlen = MLX5_ST_SZ_BYTES(detach_from_mcg_in);
1236 MLX5_SET(detach_from_mcg_in, din, qpn,
1237 MLX5_GET(attach_to_mcg_in, in, qpn));
1238 memcpy(MLX5_ADDR_OF(detach_from_mcg_in, din, multicast_gid),
1239 MLX5_ADDR_OF(attach_to_mcg_in, in, multicast_gid),
1240 MLX5_FLD_SZ_BYTES(attach_to_mcg_in, multicast_gid));
1241 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DETACH_FROM_MCG);
1242 break;
1243 case MLX5_CMD_OP_ALLOC_XRCD:
1244 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DEALLOC_XRCD);
1245 break;
Max Gurtovoy8b38c532019-07-23 10:04:12 +03001246 case MLX5_CMD_OP_CREATE_PSV:
1247 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
1248 MLX5_CMD_OP_DESTROY_PSV);
1249 MLX5_SET(destroy_psv_in, din, psvn,
1250 MLX5_GET(create_psv_out, out, psv0_index));
1251 break;
Yishai Hadas7efce362018-06-17 13:00:01 +03001252 default:
1253 /* The entry must match to one of the devx_is_obj_create_cmd */
1254 WARN_ON(true);
1255 break;
1256 }
1257}
1258
Yishai Hadas534fd7a2019-01-13 16:01:17 +02001259static int devx_handle_mkey_indirect(struct devx_obj *obj,
1260 struct mlx5_ib_dev *dev,
1261 void *in, void *out)
1262{
Yishai Hadas534fd7a2019-01-13 16:01:17 +02001263 struct mlx5_ib_devx_mr *devx_mr = &obj->devx_mr;
Yishai Hadas534fd7a2019-01-13 16:01:17 +02001264 struct mlx5_core_mkey *mkey;
1265 void *mkc;
1266 u8 key;
Yishai Hadas534fd7a2019-01-13 16:01:17 +02001267
1268 mkey = &devx_mr->mmkey;
1269 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1270 key = MLX5_GET(mkc, mkc, mkey_7_0);
1271 mkey->key = mlx5_idx_to_mkey(
1272 MLX5_GET(create_mkey_out, out, mkey_index)) | key;
1273 mkey->type = MLX5_MKEY_INDIRECT_DEVX;
1274 mkey->iova = MLX5_GET64(mkc, mkc, start_addr);
1275 mkey->size = MLX5_GET64(mkc, mkc, len);
1276 mkey->pd = MLX5_GET(mkc, mkc, pd);
1277 devx_mr->ndescs = MLX5_GET(mkc, mkc, translations_octword_size);
1278
Jason Gunthorpe806b1012019-10-09 13:09:23 -03001279 return xa_err(xa_store(&dev->odp_mkeys, mlx5_base_mkey(mkey->key), mkey,
1280 GFP_KERNEL));
Yishai Hadas534fd7a2019-01-13 16:01:17 +02001281}
1282
Yishai Hadasfa31f142019-01-13 16:01:16 +02001283static int devx_handle_mkey_create(struct mlx5_ib_dev *dev,
1284 struct devx_obj *obj,
1285 void *in, int in_len)
1286{
1287 int min_len = MLX5_BYTE_OFF(create_mkey_in, memory_key_mkey_entry) +
1288 MLX5_FLD_SZ_BYTES(create_mkey_in,
1289 memory_key_mkey_entry);
1290 void *mkc;
1291 u8 access_mode;
1292
1293 if (in_len < min_len)
1294 return -EINVAL;
1295
1296 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
1297
1298 access_mode = MLX5_GET(mkc, mkc, access_mode_1_0);
1299 access_mode |= MLX5_GET(mkc, mkc, access_mode_4_2) << 2;
1300
1301 if (access_mode == MLX5_MKC_ACCESS_MODE_KLMS ||
Yishai Hadas534fd7a2019-01-13 16:01:17 +02001302 access_mode == MLX5_MKC_ACCESS_MODE_KSM) {
1303 if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING))
1304 obj->flags |= DEVX_OBJ_FLAGS_INDIRECT_MKEY;
Yishai Hadasfa31f142019-01-13 16:01:16 +02001305 return 0;
Yishai Hadas534fd7a2019-01-13 16:01:17 +02001306 }
Yishai Hadasfa31f142019-01-13 16:01:16 +02001307
1308 MLX5_SET(create_mkey_in, in, mkey_umem_valid, 1);
1309 return 0;
1310}
1311
Yishai Hadas75973852019-06-30 19:23:31 +03001312static void devx_cleanup_subscription(struct mlx5_ib_dev *dev,
1313 struct devx_event_subscription *sub)
1314{
1315 struct devx_event *event;
1316 struct devx_obj_event *xa_val_level2;
1317
1318 if (sub->is_cleaned)
1319 return;
1320
1321 sub->is_cleaned = 1;
1322 list_del_rcu(&sub->xa_list);
1323
1324 if (list_empty(&sub->obj_list))
1325 return;
1326
1327 list_del_rcu(&sub->obj_list);
1328 /* check whether key level 1 for this obj_sub_list is empty */
1329 event = xa_load(&dev->devx_event_table.event_xa,
1330 sub->xa_key_level1);
1331 WARN_ON(!event);
1332
1333 xa_val_level2 = xa_load(&event->object_ids, sub->xa_key_level2);
1334 if (list_empty(&xa_val_level2->obj_sub_list)) {
1335 xa_erase(&event->object_ids,
1336 sub->xa_key_level2);
1337 kfree_rcu(xa_val_level2, rcu);
1338 }
1339}
1340
Yishai Hadas7efce362018-06-17 13:00:01 +03001341static int devx_obj_cleanup(struct ib_uobject *uobject,
Shamir Rabinovitcha6a37972019-03-31 19:10:04 +03001342 enum rdma_remove_reason why,
1343 struct uverbs_attr_bundle *attrs)
Yishai Hadas7efce362018-06-17 13:00:01 +03001344{
1345 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
Yishai Hadas75973852019-06-30 19:23:31 +03001346 struct mlx5_devx_event_table *devx_event_table;
Yishai Hadas7efce362018-06-17 13:00:01 +03001347 struct devx_obj *obj = uobject->object;
Yishai Hadas75973852019-06-30 19:23:31 +03001348 struct devx_event_subscription *sub_entry, *tmp;
1349 struct mlx5_ib_dev *dev;
Yishai Hadas7efce362018-06-17 13:00:01 +03001350 int ret;
1351
Yishai Hadas75973852019-06-30 19:23:31 +03001352 dev = mlx5_udata_to_mdev(&attrs->driver_udata);
Jason Gunthorpe04177912019-10-01 12:38:21 -03001353 if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY) {
1354 /*
1355 * The pagefault_single_data_segment() does commands against
1356 * the mmkey, we must wait for that to stop before freeing the
1357 * mkey, as another allocation could get the same mkey #.
1358 */
Jason Gunthorpe806b1012019-10-09 13:09:23 -03001359 xa_erase(&obj->ib_dev->odp_mkeys,
Jason Gunthorpe04177912019-10-01 12:38:21 -03001360 mlx5_base_mkey(obj->devx_mr.mmkey.key));
Jason Gunthorpe806b1012019-10-09 13:09:23 -03001361 synchronize_srcu(&dev->odp_srcu);
Jason Gunthorpe04177912019-10-01 12:38:21 -03001362 }
Yishai Hadas534fd7a2019-01-13 16:01:17 +02001363
Yishai Hadasc5ae1952019-03-06 19:21:42 +02001364 if (obj->flags & DEVX_OBJ_FLAGS_DCT)
Leon Romanovsky333fbaa2020-04-04 10:40:24 +03001365 ret = mlx5_core_destroy_dct(obj->ib_dev, &obj->core_dct);
Yishai Hadasef1659a2019-06-30 19:23:33 +03001366 else if (obj->flags & DEVX_OBJ_FLAGS_CQ)
Yishai Hadas5832fdd2019-06-30 19:23:34 +03001367 ret = mlx5_core_destroy_cq(obj->ib_dev->mdev, &obj->core_cq);
Yishai Hadasc5ae1952019-03-06 19:21:42 +02001368 else
Yishai Hadas5832fdd2019-06-30 19:23:34 +03001369 ret = mlx5_cmd_exec(obj->ib_dev->mdev, obj->dinbox,
1370 obj->dinlen, out, sizeof(out));
Yishai Hadas1c774832018-06-20 17:11:39 +03001371 if (ib_is_destroy_retryable(ret, why, uobject))
Yishai Hadas7efce362018-06-17 13:00:01 +03001372 return ret;
1373
Yishai Hadas75973852019-06-30 19:23:31 +03001374 devx_event_table = &dev->devx_event_table;
Yishai Hadas534fd7a2019-01-13 16:01:17 +02001375
Yishai Hadas75973852019-06-30 19:23:31 +03001376 mutex_lock(&devx_event_table->event_xa_lock);
1377 list_for_each_entry_safe(sub_entry, tmp, &obj->event_sub, obj_list)
1378 devx_cleanup_subscription(dev, sub_entry);
1379 mutex_unlock(&devx_event_table->event_xa_lock);
1380
Yishai Hadas7efce362018-06-17 13:00:01 +03001381 kfree(obj);
1382 return ret;
1383}
1384
Yishai Hadasef1659a2019-06-30 19:23:33 +03001385static void devx_cq_comp(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe)
1386{
1387 struct devx_obj *obj = container_of(mcq, struct devx_obj, core_cq);
1388 struct mlx5_devx_event_table *table;
1389 struct devx_event *event;
1390 struct devx_obj_event *obj_event;
1391 u32 obj_id = mcq->cqn;
1392
1393 table = &obj->ib_dev->devx_event_table;
1394 rcu_read_lock();
1395 event = xa_load(&table->event_xa, MLX5_EVENT_TYPE_COMP);
1396 if (!event)
1397 goto out;
1398
1399 obj_event = xa_load(&event->object_ids, obj_id);
1400 if (!obj_event)
1401 goto out;
1402
1403 dispatch_event_fd(&obj_event->obj_sub_list, eqe);
1404out:
1405 rcu_read_unlock();
1406}
1407
Jason Gunthorpee83f0ec2018-07-25 21:40:18 -06001408static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_CREATE)(
Jason Gunthorpe15a1b4b2018-11-25 20:51:15 +02001409 struct uverbs_attr_bundle *attrs)
Yishai Hadas7efce362018-06-17 13:00:01 +03001410{
Yishai Hadas7efce362018-06-17 13:00:01 +03001411 void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN);
1412 int cmd_out_len = uverbs_attr_get_len(attrs,
1413 MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT);
Yishai Hadasfa31f142019-01-13 16:01:16 +02001414 int cmd_in_len = uverbs_attr_get_len(attrs,
1415 MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN);
Yishai Hadas7efce362018-06-17 13:00:01 +03001416 void *cmd_out;
Jason Gunthorpec36ee462018-07-10 20:55:22 -06001417 struct ib_uobject *uobj = uverbs_attr_get_uobject(
1418 attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_HANDLE);
Shamir Rabinovitch89944452019-02-07 18:44:49 +02001419 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1420 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
Jason Gunthorpec36ee462018-07-10 20:55:22 -06001421 struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device);
Yishai Hadase8ef0902018-09-25 12:11:12 +03001422 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
Yishai Hadas7efce362018-06-17 13:00:01 +03001423 struct devx_obj *obj;
Yishai Hadascd5d20f2019-05-14 14:44:11 +03001424 u16 obj_type = 0;
Yishai Hadas7efce362018-06-17 13:00:01 +03001425 int err;
Yishai Hadas7e1335a2018-09-20 21:45:20 +03001426 int uid;
Yishai Hadas2351776e2018-10-07 12:06:34 +03001427 u32 obj_id;
1428 u16 opcode;
Yishai Hadas7efce362018-06-17 13:00:01 +03001429
Max Gurtovoyb6142602019-07-01 21:14:02 +03001430 if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id))
1431 return -EINVAL;
1432
Yishai Hadas7e1335a2018-09-20 21:45:20 +03001433 uid = devx_get_uid(c, cmd_in);
1434 if (uid < 0)
1435 return uid;
Yishai Hadas7efce362018-06-17 13:00:01 +03001436
Yishai Hadas2351776e2018-10-07 12:06:34 +03001437 if (!devx_is_obj_create_cmd(cmd_in, &opcode))
Yishai Hadas7efce362018-06-17 13:00:01 +03001438 return -EINVAL;
1439
Jason Gunthorpeb61815e2018-08-09 20:14:41 -06001440 cmd_out = uverbs_zalloc(attrs, cmd_out_len);
1441 if (IS_ERR(cmd_out))
1442 return PTR_ERR(cmd_out);
1443
Yishai Hadas7efce362018-06-17 13:00:01 +03001444 obj = kzalloc(sizeof(struct devx_obj), GFP_KERNEL);
1445 if (!obj)
1446 return -ENOMEM;
1447
Yishai Hadas7e1335a2018-09-20 21:45:20 +03001448 MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
Yishai Hadasfa31f142019-01-13 16:01:16 +02001449 if (opcode == MLX5_CMD_OP_CREATE_MKEY) {
1450 err = devx_handle_mkey_create(dev, obj, cmd_in, cmd_in_len);
1451 if (err)
1452 goto obj_free;
1453 } else {
1454 devx_set_umem_valid(cmd_in);
1455 }
Yishai Hadasba1a0572018-09-20 21:39:33 +03001456
Yishai Hadasc5ae1952019-03-06 19:21:42 +02001457 if (opcode == MLX5_CMD_OP_CREATE_DCT) {
1458 obj->flags |= DEVX_OBJ_FLAGS_DCT;
Leon Romanovsky333fbaa2020-04-04 10:40:24 +03001459 err = mlx5_core_create_dct(dev, &obj->core_dct, cmd_in,
1460 cmd_in_len, cmd_out, cmd_out_len);
Yishai Hadasef1659a2019-06-30 19:23:33 +03001461 } else if (opcode == MLX5_CMD_OP_CREATE_CQ) {
1462 obj->flags |= DEVX_OBJ_FLAGS_CQ;
1463 obj->core_cq.comp = devx_cq_comp;
1464 err = mlx5_core_create_cq(dev->mdev, &obj->core_cq,
1465 cmd_in, cmd_in_len, cmd_out,
1466 cmd_out_len);
Yishai Hadasc5ae1952019-03-06 19:21:42 +02001467 } else {
1468 err = mlx5_cmd_exec(dev->mdev, cmd_in,
1469 cmd_in_len,
1470 cmd_out, cmd_out_len);
1471 }
1472
Yishai Hadas7efce362018-06-17 13:00:01 +03001473 if (err)
Jason Gunthorpeb61815e2018-08-09 20:14:41 -06001474 goto obj_free;
Yishai Hadas7efce362018-06-17 13:00:01 +03001475
Yevgeny Kliteynik208d70f2019-11-03 16:07:23 +02001476 if (opcode == MLX5_CMD_OP_ALLOC_FLOW_COUNTER) {
1477 u8 bulk = MLX5_GET(alloc_flow_counter_in,
1478 cmd_in,
1479 flow_counter_bulk);
1480 obj->flow_counter_bulk_size = 128UL * bulk;
1481 }
1482
Yishai Hadas7efce362018-06-17 13:00:01 +03001483 uobj->object = obj;
Yishai Hadas75973852019-06-30 19:23:31 +03001484 INIT_LIST_HEAD(&obj->event_sub);
Yishai Hadasef1659a2019-06-30 19:23:33 +03001485 obj->ib_dev = dev;
Yishai Hadas2351776e2018-10-07 12:06:34 +03001486 devx_obj_build_destroy_cmd(cmd_in, cmd_out, obj->dinbox, &obj->dinlen,
1487 &obj_id);
Yishai Hadas7efce362018-06-17 13:00:01 +03001488 WARN_ON(obj->dinlen > MLX5_MAX_DESTROY_INBOX_SIZE_DW * sizeof(u32));
1489
Jason Gunthorpe04177912019-10-01 12:38:21 -03001490 err = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT, cmd_out, cmd_out_len);
1491 if (err)
1492 goto obj_destroy;
1493
1494 if (opcode == MLX5_CMD_OP_CREATE_GENERAL_OBJECT)
1495 obj_type = MLX5_GET(general_obj_in_cmd_hdr, cmd_in, obj_type);
1496 obj->obj_id = get_enc_obj_id(opcode | obj_type << 16, obj_id);
1497
Yishai Hadas534fd7a2019-01-13 16:01:17 +02001498 if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY) {
1499 err = devx_handle_mkey_indirect(obj, dev, cmd_in, cmd_out);
1500 if (err)
1501 goto obj_destroy;
1502 }
Yishai Hadas7efce362018-06-17 13:00:01 +03001503 return 0;
1504
Yishai Hadas0da4d482019-02-11 17:40:53 +02001505obj_destroy:
Yishai Hadasc5ae1952019-03-06 19:21:42 +02001506 if (obj->flags & DEVX_OBJ_FLAGS_DCT)
Leon Romanovsky333fbaa2020-04-04 10:40:24 +03001507 mlx5_core_destroy_dct(obj->ib_dev, &obj->core_dct);
Yishai Hadasef1659a2019-06-30 19:23:33 +03001508 else if (obj->flags & DEVX_OBJ_FLAGS_CQ)
Yishai Hadas5832fdd2019-06-30 19:23:34 +03001509 mlx5_core_destroy_cq(obj->ib_dev->mdev, &obj->core_cq);
Yishai Hadasc5ae1952019-03-06 19:21:42 +02001510 else
Yishai Hadas5832fdd2019-06-30 19:23:34 +03001511 mlx5_cmd_exec(obj->ib_dev->mdev, obj->dinbox, obj->dinlen, out,
Yishai Hadasc5ae1952019-03-06 19:21:42 +02001512 sizeof(out));
Yishai Hadas7efce362018-06-17 13:00:01 +03001513obj_free:
1514 kfree(obj);
1515 return err;
1516}
1517
Jason Gunthorpee83f0ec2018-07-25 21:40:18 -06001518static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_MODIFY)(
Jason Gunthorpe15a1b4b2018-11-25 20:51:15 +02001519 struct uverbs_attr_bundle *attrs)
Yishai Hadase662e142018-06-17 13:00:02 +03001520{
Yishai Hadase662e142018-06-17 13:00:02 +03001521 void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN);
1522 int cmd_out_len = uverbs_attr_get_len(attrs,
1523 MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT);
1524 struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs,
1525 MLX5_IB_ATTR_DEVX_OBJ_MODIFY_HANDLE);
Shamir Rabinovitch89944452019-02-07 18:44:49 +02001526 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1527 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1528 struct mlx5_ib_dev *mdev = to_mdev(c->ibucontext.device);
Yishai Hadase662e142018-06-17 13:00:02 +03001529 void *cmd_out;
1530 int err;
Yishai Hadas7e1335a2018-09-20 21:45:20 +03001531 int uid;
Yishai Hadase662e142018-06-17 13:00:02 +03001532
Max Gurtovoyb6142602019-07-01 21:14:02 +03001533 if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id))
1534 return -EINVAL;
1535
Yishai Hadas7e1335a2018-09-20 21:45:20 +03001536 uid = devx_get_uid(c, cmd_in);
1537 if (uid < 0)
1538 return uid;
Yishai Hadase662e142018-06-17 13:00:02 +03001539
1540 if (!devx_is_obj_modify_cmd(cmd_in))
1541 return -EINVAL;
1542
Jason Gunthorpee79c9c62019-04-01 17:08:23 -03001543 if (!devx_is_valid_obj_id(attrs, uobj, cmd_in))
Yishai Hadase662e142018-06-17 13:00:02 +03001544 return -EINVAL;
1545
Jason Gunthorpeb61815e2018-08-09 20:14:41 -06001546 cmd_out = uverbs_zalloc(attrs, cmd_out_len);
1547 if (IS_ERR(cmd_out))
1548 return PTR_ERR(cmd_out);
Yishai Hadase662e142018-06-17 13:00:02 +03001549
Yishai Hadas7e1335a2018-09-20 21:45:20 +03001550 MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
Yishai Hadasba1a0572018-09-20 21:39:33 +03001551 devx_set_umem_valid(cmd_in);
1552
Yishai Hadas34613eb2018-11-26 08:28:35 +02001553 err = mlx5_cmd_exec(mdev->mdev, cmd_in,
Yishai Hadase662e142018-06-17 13:00:02 +03001554 uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN),
1555 cmd_out, cmd_out_len);
1556 if (err)
Jason Gunthorpeb61815e2018-08-09 20:14:41 -06001557 return err;
Yishai Hadase662e142018-06-17 13:00:02 +03001558
Jason Gunthorpeb61815e2018-08-09 20:14:41 -06001559 return uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT,
1560 cmd_out, cmd_out_len);
Yishai Hadase662e142018-06-17 13:00:02 +03001561}
1562
Jason Gunthorpee83f0ec2018-07-25 21:40:18 -06001563static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_QUERY)(
Jason Gunthorpe15a1b4b2018-11-25 20:51:15 +02001564 struct uverbs_attr_bundle *attrs)
Yishai Hadase662e142018-06-17 13:00:02 +03001565{
Yishai Hadase662e142018-06-17 13:00:02 +03001566 void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN);
1567 int cmd_out_len = uverbs_attr_get_len(attrs,
1568 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT);
1569 struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs,
1570 MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE);
Shamir Rabinovitch89944452019-02-07 18:44:49 +02001571 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1572 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
Yishai Hadase662e142018-06-17 13:00:02 +03001573 void *cmd_out;
1574 int err;
Yishai Hadas7e1335a2018-09-20 21:45:20 +03001575 int uid;
Shamir Rabinovitch89944452019-02-07 18:44:49 +02001576 struct mlx5_ib_dev *mdev = to_mdev(c->ibucontext.device);
Yishai Hadase662e142018-06-17 13:00:02 +03001577
Max Gurtovoyb6142602019-07-01 21:14:02 +03001578 if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id))
1579 return -EINVAL;
1580
Yishai Hadas7e1335a2018-09-20 21:45:20 +03001581 uid = devx_get_uid(c, cmd_in);
1582 if (uid < 0)
1583 return uid;
Yishai Hadase662e142018-06-17 13:00:02 +03001584
1585 if (!devx_is_obj_query_cmd(cmd_in))
1586 return -EINVAL;
1587
Jason Gunthorpee79c9c62019-04-01 17:08:23 -03001588 if (!devx_is_valid_obj_id(attrs, uobj, cmd_in))
Yishai Hadase662e142018-06-17 13:00:02 +03001589 return -EINVAL;
1590
Jason Gunthorpeb61815e2018-08-09 20:14:41 -06001591 cmd_out = uverbs_zalloc(attrs, cmd_out_len);
1592 if (IS_ERR(cmd_out))
1593 return PTR_ERR(cmd_out);
Yishai Hadase662e142018-06-17 13:00:02 +03001594
Yishai Hadas7e1335a2018-09-20 21:45:20 +03001595 MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
Yishai Hadas34613eb2018-11-26 08:28:35 +02001596 err = mlx5_cmd_exec(mdev->mdev, cmd_in,
Yishai Hadase662e142018-06-17 13:00:02 +03001597 uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN),
1598 cmd_out, cmd_out_len);
1599 if (err)
Jason Gunthorpeb61815e2018-08-09 20:14:41 -06001600 return err;
Yishai Hadase662e142018-06-17 13:00:02 +03001601
Jason Gunthorpeb61815e2018-08-09 20:14:41 -06001602 return uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT,
1603 cmd_out, cmd_out_len);
Yishai Hadase662e142018-06-17 13:00:02 +03001604}
1605
Yishai Hadas6bf8f222019-01-22 08:29:56 +02001606struct devx_async_event_queue {
1607 spinlock_t lock;
1608 wait_queue_head_t poll_wait;
1609 struct list_head event_list;
Yishai Hadasa124edb2019-01-22 08:29:57 +02001610 atomic_t bytes_in_use;
Yishai Hadaseaebaf72019-01-22 08:29:59 +02001611 u8 is_destroyed:1;
Yishai Hadas6bf8f222019-01-22 08:29:56 +02001612};
1613
1614struct devx_async_cmd_event_file {
1615 struct ib_uobject uobj;
1616 struct devx_async_event_queue ev_queue;
Yishai Hadasa124edb2019-01-22 08:29:57 +02001617 struct mlx5_async_ctx async_ctx;
Yishai Hadas6bf8f222019-01-22 08:29:56 +02001618};
1619
1620static void devx_init_event_queue(struct devx_async_event_queue *ev_queue)
1621{
1622 spin_lock_init(&ev_queue->lock);
1623 INIT_LIST_HEAD(&ev_queue->event_list);
1624 init_waitqueue_head(&ev_queue->poll_wait);
Yishai Hadasa124edb2019-01-22 08:29:57 +02001625 atomic_set(&ev_queue->bytes_in_use, 0);
Yishai Hadaseaebaf72019-01-22 08:29:59 +02001626 ev_queue->is_destroyed = 0;
Yishai Hadas6bf8f222019-01-22 08:29:56 +02001627}
1628
1629static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC)(
1630 struct uverbs_attr_bundle *attrs)
1631{
1632 struct devx_async_cmd_event_file *ev_file;
1633
1634 struct ib_uobject *uobj = uverbs_attr_get_uobject(
1635 attrs, MLX5_IB_ATTR_DEVX_ASYNC_CMD_FD_ALLOC_HANDLE);
Jason Gunthorpee79c9c62019-04-01 17:08:23 -03001636 struct mlx5_ib_dev *mdev = mlx5_udata_to_mdev(&attrs->driver_udata);
Yishai Hadas6bf8f222019-01-22 08:29:56 +02001637
1638 ev_file = container_of(uobj, struct devx_async_cmd_event_file,
1639 uobj);
1640 devx_init_event_queue(&ev_file->ev_queue);
Yishai Hadasa124edb2019-01-22 08:29:57 +02001641 mlx5_cmd_init_async_ctx(mdev->mdev, &ev_file->async_ctx);
Yishai Hadas6bf8f222019-01-22 08:29:56 +02001642 return 0;
1643}
1644
Yishai Hadas2afc5e12019-06-30 19:23:29 +03001645static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC)(
1646 struct uverbs_attr_bundle *attrs)
1647{
1648 struct ib_uobject *uobj = uverbs_attr_get_uobject(
1649 attrs, MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_HANDLE);
1650 struct devx_async_event_file *ev_file;
1651 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1652 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1653 struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device);
1654 u32 flags;
1655 int err;
1656
1657 err = uverbs_get_flags32(&flags, attrs,
1658 MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_FLAGS,
1659 MLX5_IB_UAPI_DEVX_CR_EV_CH_FLAGS_OMIT_DATA);
1660
1661 if (err)
1662 return err;
1663
1664 ev_file = container_of(uobj, struct devx_async_event_file,
1665 uobj);
1666 spin_lock_init(&ev_file->lock);
1667 INIT_LIST_HEAD(&ev_file->event_list);
1668 init_waitqueue_head(&ev_file->poll_wait);
1669 if (flags & MLX5_IB_UAPI_DEVX_CR_EV_CH_FLAGS_OMIT_DATA)
1670 ev_file->omit_data = 1;
1671 INIT_LIST_HEAD(&ev_file->subscribed_events_list);
1672 ev_file->dev = dev;
Yishai Hadas75973852019-06-30 19:23:31 +03001673 get_device(&dev->ib_dev.dev);
Yishai Hadas2afc5e12019-06-30 19:23:29 +03001674 return 0;
1675}
1676
Yishai Hadasa124edb2019-01-22 08:29:57 +02001677static void devx_query_callback(int status, struct mlx5_async_work *context)
1678{
1679 struct devx_async_data *async_data =
1680 container_of(context, struct devx_async_data, cb_work);
Jason Gunthorpe93887e62020-01-08 19:21:55 +02001681 struct devx_async_cmd_event_file *ev_file = async_data->ev_file;
1682 struct devx_async_event_queue *ev_queue = &ev_file->ev_queue;
Yishai Hadasa124edb2019-01-22 08:29:57 +02001683 unsigned long flags;
1684
Jason Gunthorpe93887e62020-01-08 19:21:55 +02001685 /*
1686 * Note that if the struct devx_async_cmd_event_file uobj begins to be
1687 * destroyed it will block at mlx5_cmd_cleanup_async_ctx() until this
1688 * routine returns, ensuring that it always remains valid here.
1689 */
Yishai Hadasa124edb2019-01-22 08:29:57 +02001690 spin_lock_irqsave(&ev_queue->lock, flags);
1691 list_add_tail(&async_data->list, &ev_queue->event_list);
1692 spin_unlock_irqrestore(&ev_queue->lock, flags);
1693
1694 wake_up_interruptible(&ev_queue->poll_wait);
Yishai Hadasa124edb2019-01-22 08:29:57 +02001695}
1696
1697#define MAX_ASYNC_BYTES_IN_USE (1024 * 1024) /* 1MB */
1698
1699static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY)(
1700 struct uverbs_attr_bundle *attrs)
1701{
1702 void *cmd_in = uverbs_attr_get_alloced_ptr(attrs,
1703 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_CMD_IN);
1704 struct ib_uobject *uobj = uverbs_attr_get_uobject(
1705 attrs,
1706 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_HANDLE);
1707 u16 cmd_out_len;
Shamir Rabinovitch89944452019-02-07 18:44:49 +02001708 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1709 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
Yishai Hadasa124edb2019-01-22 08:29:57 +02001710 struct ib_uobject *fd_uobj;
1711 int err;
1712 int uid;
Shamir Rabinovitch89944452019-02-07 18:44:49 +02001713 struct mlx5_ib_dev *mdev = to_mdev(c->ibucontext.device);
Yishai Hadasa124edb2019-01-22 08:29:57 +02001714 struct devx_async_cmd_event_file *ev_file;
1715 struct devx_async_data *async_data;
1716
Max Gurtovoyb6142602019-07-01 21:14:02 +03001717 if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id))
1718 return -EINVAL;
1719
Yishai Hadasa124edb2019-01-22 08:29:57 +02001720 uid = devx_get_uid(c, cmd_in);
1721 if (uid < 0)
1722 return uid;
1723
1724 if (!devx_is_obj_query_cmd(cmd_in))
1725 return -EINVAL;
1726
1727 err = uverbs_get_const(&cmd_out_len, attrs,
1728 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_OUT_LEN);
1729 if (err)
1730 return err;
1731
Jason Gunthorpee79c9c62019-04-01 17:08:23 -03001732 if (!devx_is_valid_obj_id(attrs, uobj, cmd_in))
Yishai Hadasa124edb2019-01-22 08:29:57 +02001733 return -EINVAL;
1734
1735 fd_uobj = uverbs_attr_get_uobject(attrs,
1736 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_FD);
1737 if (IS_ERR(fd_uobj))
1738 return PTR_ERR(fd_uobj);
1739
1740 ev_file = container_of(fd_uobj, struct devx_async_cmd_event_file,
1741 uobj);
1742
1743 if (atomic_add_return(cmd_out_len, &ev_file->ev_queue.bytes_in_use) >
1744 MAX_ASYNC_BYTES_IN_USE) {
1745 atomic_sub(cmd_out_len, &ev_file->ev_queue.bytes_in_use);
1746 return -EAGAIN;
1747 }
1748
1749 async_data = kvzalloc(struct_size(async_data, hdr.out_data,
1750 cmd_out_len), GFP_KERNEL);
1751 if (!async_data) {
1752 err = -ENOMEM;
1753 goto sub_bytes;
1754 }
1755
1756 err = uverbs_copy_from(&async_data->hdr.wr_id, attrs,
1757 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_WR_ID);
1758 if (err)
1759 goto free_async;
1760
1761 async_data->cmd_out_len = cmd_out_len;
1762 async_data->mdev = mdev;
Jason Gunthorpe93887e62020-01-08 19:21:55 +02001763 async_data->ev_file = ev_file;
Yishai Hadasa124edb2019-01-22 08:29:57 +02001764
Yishai Hadasa124edb2019-01-22 08:29:57 +02001765 MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1766 err = mlx5_cmd_exec_cb(&ev_file->async_ctx, cmd_in,
1767 uverbs_attr_get_len(attrs,
1768 MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_CMD_IN),
1769 async_data->hdr.out_data,
1770 async_data->cmd_out_len,
1771 devx_query_callback, &async_data->cb_work);
1772
1773 if (err)
Jason Gunthorpe93887e62020-01-08 19:21:55 +02001774 goto free_async;
Yishai Hadasa124edb2019-01-22 08:29:57 +02001775
1776 return 0;
1777
Yishai Hadasa124edb2019-01-22 08:29:57 +02001778free_async:
1779 kvfree(async_data);
1780sub_bytes:
1781 atomic_sub(cmd_out_len, &ev_file->ev_queue.bytes_in_use);
1782 return err;
1783}
1784
Yishai Hadas75973852019-06-30 19:23:31 +03001785static void
1786subscribe_event_xa_dealloc(struct mlx5_devx_event_table *devx_event_table,
1787 u32 key_level1,
1788 bool is_level2,
1789 u32 key_level2)
1790{
1791 struct devx_event *event;
1792 struct devx_obj_event *xa_val_level2;
1793
1794 /* Level 1 is valid for future use, no need to free */
1795 if (!is_level2)
1796 return;
1797
1798 event = xa_load(&devx_event_table->event_xa, key_level1);
1799 WARN_ON(!event);
1800
1801 xa_val_level2 = xa_load(&event->object_ids,
1802 key_level2);
1803 if (list_empty(&xa_val_level2->obj_sub_list)) {
1804 xa_erase(&event->object_ids,
1805 key_level2);
1806 kfree_rcu(xa_val_level2, rcu);
1807 }
1808}
1809
1810static int
1811subscribe_event_xa_alloc(struct mlx5_devx_event_table *devx_event_table,
1812 u32 key_level1,
1813 bool is_level2,
1814 u32 key_level2)
1815{
1816 struct devx_obj_event *obj_event;
1817 struct devx_event *event;
1818 int err;
1819
1820 event = xa_load(&devx_event_table->event_xa, key_level1);
1821 if (!event) {
1822 event = kzalloc(sizeof(*event), GFP_KERNEL);
1823 if (!event)
1824 return -ENOMEM;
1825
1826 INIT_LIST_HEAD(&event->unaffiliated_list);
1827 xa_init(&event->object_ids);
1828
1829 err = xa_insert(&devx_event_table->event_xa,
1830 key_level1,
1831 event,
1832 GFP_KERNEL);
1833 if (err) {
1834 kfree(event);
1835 return err;
1836 }
1837 }
1838
1839 if (!is_level2)
1840 return 0;
1841
1842 obj_event = xa_load(&event->object_ids, key_level2);
1843 if (!obj_event) {
1844 obj_event = kzalloc(sizeof(*obj_event), GFP_KERNEL);
1845 if (!obj_event)
1846 /* Level1 is valid for future use, no need to free */
1847 return -ENOMEM;
1848
1849 err = xa_insert(&event->object_ids,
1850 key_level2,
1851 obj_event,
1852 GFP_KERNEL);
1853 if (err)
1854 return err;
1855 INIT_LIST_HEAD(&obj_event->obj_sub_list);
1856 }
1857
1858 return 0;
1859}
1860
1861static bool is_valid_events_legacy(int num_events, u16 *event_type_num_list,
1862 struct devx_obj *obj)
1863{
1864 int i;
1865
1866 for (i = 0; i < num_events; i++) {
1867 if (obj) {
1868 if (!is_legacy_obj_event_num(event_type_num_list[i]))
1869 return false;
1870 } else if (!is_legacy_unaffiliated_event_num(
1871 event_type_num_list[i])) {
1872 return false;
1873 }
1874 }
1875
1876 return true;
1877}
1878
1879#define MAX_SUPP_EVENT_NUM 255
1880static bool is_valid_events(struct mlx5_core_dev *dev,
1881 int num_events, u16 *event_type_num_list,
1882 struct devx_obj *obj)
1883{
1884 __be64 *aff_events;
1885 __be64 *unaff_events;
1886 int mask_entry;
1887 int mask_bit;
1888 int i;
1889
1890 if (MLX5_CAP_GEN(dev, event_cap)) {
1891 aff_events = MLX5_CAP_DEV_EVENT(dev,
1892 user_affiliated_events);
1893 unaff_events = MLX5_CAP_DEV_EVENT(dev,
1894 user_unaffiliated_events);
1895 } else {
1896 return is_valid_events_legacy(num_events, event_type_num_list,
1897 obj);
1898 }
1899
1900 for (i = 0; i < num_events; i++) {
1901 if (event_type_num_list[i] > MAX_SUPP_EVENT_NUM)
1902 return false;
1903
1904 mask_entry = event_type_num_list[i] / 64;
1905 mask_bit = event_type_num_list[i] % 64;
1906
1907 if (obj) {
1908 /* CQ completion */
1909 if (event_type_num_list[i] == 0)
1910 continue;
1911
1912 if (!(be64_to_cpu(aff_events[mask_entry]) &
1913 (1ull << mask_bit)))
1914 return false;
1915
1916 continue;
1917 }
1918
1919 if (!(be64_to_cpu(unaff_events[mask_entry]) &
1920 (1ull << mask_bit)))
1921 return false;
1922 }
1923
1924 return true;
1925}
1926
1927#define MAX_NUM_EVENTS 16
1928static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT)(
1929 struct uverbs_attr_bundle *attrs)
1930{
1931 struct ib_uobject *devx_uobj = uverbs_attr_get_uobject(
1932 attrs,
1933 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_OBJ_HANDLE);
1934 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
1935 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
1936 struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device);
1937 struct ib_uobject *fd_uobj;
1938 struct devx_obj *obj = NULL;
1939 struct devx_async_event_file *ev_file;
1940 struct mlx5_devx_event_table *devx_event_table = &dev->devx_event_table;
1941 u16 *event_type_num_list;
1942 struct devx_event_subscription *event_sub, *tmp_sub;
1943 struct list_head sub_list;
1944 int redirect_fd;
1945 bool use_eventfd = false;
1946 int num_events;
1947 int num_alloc_xa_entries = 0;
1948 u16 obj_type = 0;
1949 u64 cookie = 0;
1950 u32 obj_id = 0;
1951 int err;
1952 int i;
1953
1954 if (!c->devx_uid)
1955 return -EINVAL;
1956
1957 if (!IS_ERR(devx_uobj)) {
1958 obj = (struct devx_obj *)devx_uobj->object;
1959 if (obj)
1960 obj_id = get_dec_obj_id(obj->obj_id);
1961 }
1962
1963 fd_uobj = uverbs_attr_get_uobject(attrs,
1964 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_HANDLE);
1965 if (IS_ERR(fd_uobj))
1966 return PTR_ERR(fd_uobj);
1967
1968 ev_file = container_of(fd_uobj, struct devx_async_event_file,
1969 uobj);
1970
1971 if (uverbs_attr_is_valid(attrs,
1972 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_NUM)) {
1973 err = uverbs_copy_from(&redirect_fd, attrs,
1974 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_NUM);
1975 if (err)
1976 return err;
1977
1978 use_eventfd = true;
1979 }
1980
1981 if (uverbs_attr_is_valid(attrs,
1982 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_COOKIE)) {
1983 if (use_eventfd)
1984 return -EINVAL;
1985
1986 err = uverbs_copy_from(&cookie, attrs,
1987 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_COOKIE);
1988 if (err)
1989 return err;
1990 }
1991
1992 num_events = uverbs_attr_ptr_get_array_size(
1993 attrs, MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_TYPE_NUM_LIST,
1994 sizeof(u16));
1995
1996 if (num_events < 0)
1997 return num_events;
1998
1999 if (num_events > MAX_NUM_EVENTS)
2000 return -EINVAL;
2001
2002 event_type_num_list = uverbs_attr_get_alloced_ptr(attrs,
2003 MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_TYPE_NUM_LIST);
2004
2005 if (!is_valid_events(dev->mdev, num_events, event_type_num_list, obj))
2006 return -EINVAL;
2007
2008 INIT_LIST_HEAD(&sub_list);
2009
2010 /* Protect from concurrent subscriptions to same XA entries to allow
2011 * both to succeed
2012 */
2013 mutex_lock(&devx_event_table->event_xa_lock);
2014 for (i = 0; i < num_events; i++) {
2015 u32 key_level1;
2016
2017 if (obj)
2018 obj_type = get_dec_obj_type(obj,
2019 event_type_num_list[i]);
2020 key_level1 = event_type_num_list[i] | obj_type << 16;
2021
2022 err = subscribe_event_xa_alloc(devx_event_table,
2023 key_level1,
2024 obj,
2025 obj_id);
2026 if (err)
2027 goto err;
2028
2029 num_alloc_xa_entries++;
2030 event_sub = kzalloc(sizeof(*event_sub), GFP_KERNEL);
2031 if (!event_sub)
2032 goto err;
2033
2034 list_add_tail(&event_sub->event_list, &sub_list);
Jason Gunthorpe6898d1c2020-01-08 19:21:53 +02002035 uverbs_uobject_get(&ev_file->uobj);
Yishai Hadas75973852019-06-30 19:23:31 +03002036 if (use_eventfd) {
2037 event_sub->eventfd =
2038 eventfd_ctx_fdget(redirect_fd);
2039
Dan Carpentere7e6c632019-08-07 15:32:36 +03002040 if (IS_ERR(event_sub->eventfd)) {
Yishai Hadas75973852019-06-30 19:23:31 +03002041 err = PTR_ERR(event_sub->eventfd);
2042 event_sub->eventfd = NULL;
2043 goto err;
2044 }
2045 }
2046
2047 event_sub->cookie = cookie;
2048 event_sub->ev_file = ev_file;
Yishai Hadas75973852019-06-30 19:23:31 +03002049 /* May be needed upon cleanup the devx object/subscription */
2050 event_sub->xa_key_level1 = key_level1;
2051 event_sub->xa_key_level2 = obj_id;
2052 INIT_LIST_HEAD(&event_sub->obj_list);
2053 }
2054
2055 /* Once all the allocations and the XA data insertions were done we
2056 * can go ahead and add all the subscriptions to the relevant lists
2057 * without concern of a failure.
2058 */
2059 list_for_each_entry_safe(event_sub, tmp_sub, &sub_list, event_list) {
2060 struct devx_event *event;
2061 struct devx_obj_event *obj_event;
2062
2063 list_del_init(&event_sub->event_list);
2064
2065 spin_lock_irq(&ev_file->lock);
2066 list_add_tail_rcu(&event_sub->file_list,
2067 &ev_file->subscribed_events_list);
2068 spin_unlock_irq(&ev_file->lock);
2069
2070 event = xa_load(&devx_event_table->event_xa,
2071 event_sub->xa_key_level1);
2072 WARN_ON(!event);
2073
2074 if (!obj) {
2075 list_add_tail_rcu(&event_sub->xa_list,
2076 &event->unaffiliated_list);
2077 continue;
2078 }
2079
2080 obj_event = xa_load(&event->object_ids, obj_id);
2081 WARN_ON(!obj_event);
2082 list_add_tail_rcu(&event_sub->xa_list,
2083 &obj_event->obj_sub_list);
2084 list_add_tail_rcu(&event_sub->obj_list,
2085 &obj->event_sub);
2086 }
2087
2088 mutex_unlock(&devx_event_table->event_xa_lock);
2089 return 0;
2090
2091err:
2092 list_for_each_entry_safe(event_sub, tmp_sub, &sub_list, event_list) {
2093 list_del(&event_sub->event_list);
2094
2095 subscribe_event_xa_dealloc(devx_event_table,
2096 event_sub->xa_key_level1,
2097 obj,
2098 obj_id);
2099
2100 if (event_sub->eventfd)
2101 eventfd_ctx_put(event_sub->eventfd);
Jason Gunthorpe6898d1c2020-01-08 19:21:53 +02002102 uverbs_uobject_put(&event_sub->ev_file->uobj);
Yishai Hadas75973852019-06-30 19:23:31 +03002103 kfree(event_sub);
2104 }
2105
2106 mutex_unlock(&devx_event_table->event_xa_lock);
2107 return err;
2108}
2109
Yishai Hadasaeae9452018-06-17 13:00:04 +03002110static int devx_umem_get(struct mlx5_ib_dev *dev, struct ib_ucontext *ucontext,
2111 struct uverbs_attr_bundle *attrs,
2112 struct devx_umem *obj)
2113{
2114 u64 addr;
2115 size_t size;
Jason Gunthorpebccd0622018-07-26 16:37:14 -06002116 u32 access;
Yishai Hadasaeae9452018-06-17 13:00:04 +03002117 int npages;
2118 int err;
2119 u32 page_mask;
2120
2121 if (uverbs_copy_from(&addr, attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_ADDR) ||
Jason Gunthorpebccd0622018-07-26 16:37:14 -06002122 uverbs_copy_from(&size, attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_LEN))
Yishai Hadasaeae9452018-06-17 13:00:04 +03002123 return -EFAULT;
2124
Jason Gunthorpebccd0622018-07-26 16:37:14 -06002125 err = uverbs_get_flags32(&access, attrs,
2126 MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS,
Yishai Hadas47f07f02018-12-05 15:50:21 +02002127 IB_ACCESS_LOCAL_WRITE |
2128 IB_ACCESS_REMOTE_WRITE |
2129 IB_ACCESS_REMOTE_READ);
Jason Gunthorpebccd0622018-07-26 16:37:14 -06002130 if (err)
2131 return err;
2132
Yishai Hadasaeae9452018-06-17 13:00:04 +03002133 err = ib_check_mr_access(access);
2134 if (err)
2135 return err;
2136
Moni Shouac320e522020-01-15 14:43:31 +02002137 obj->umem = ib_umem_get(&dev->ib_dev, addr, size, access);
Yishai Hadasaeae9452018-06-17 13:00:04 +03002138 if (IS_ERR(obj->umem))
2139 return PTR_ERR(obj->umem);
2140
2141 mlx5_ib_cont_pages(obj->umem, obj->umem->address,
2142 MLX5_MKEY_PAGE_SHIFT_MASK, &npages,
2143 &obj->page_shift, &obj->ncont, NULL);
2144
2145 if (!npages) {
2146 ib_umem_release(obj->umem);
2147 return -EINVAL;
2148 }
2149
2150 page_mask = (1 << obj->page_shift) - 1;
2151 obj->page_offset = obj->umem->address & page_mask;
2152
2153 return 0;
2154}
2155
Jason Gunthorpeb61815e2018-08-09 20:14:41 -06002156static int devx_umem_reg_cmd_alloc(struct uverbs_attr_bundle *attrs,
2157 struct devx_umem *obj,
Yishai Hadasaeae9452018-06-17 13:00:04 +03002158 struct devx_umem_reg_cmd *cmd)
2159{
2160 cmd->inlen = MLX5_ST_SZ_BYTES(create_umem_in) +
2161 (MLX5_ST_SZ_BYTES(mtt) * obj->ncont);
Jason Gunthorpeb61815e2018-08-09 20:14:41 -06002162 cmd->in = uverbs_zalloc(attrs, cmd->inlen);
2163 return PTR_ERR_OR_ZERO(cmd->in);
Yishai Hadasaeae9452018-06-17 13:00:04 +03002164}
2165
2166static void devx_umem_reg_cmd_build(struct mlx5_ib_dev *dev,
2167 struct devx_umem *obj,
2168 struct devx_umem_reg_cmd *cmd)
2169{
2170 void *umem;
2171 __be64 *mtt;
2172
2173 umem = MLX5_ADDR_OF(create_umem_in, cmd->in, umem);
2174 mtt = (__be64 *)MLX5_ADDR_OF(umem, umem, mtt);
2175
Yishai Hadas6e3722b2018-12-19 16:28:15 +02002176 MLX5_SET(create_umem_in, cmd->in, opcode, MLX5_CMD_OP_CREATE_UMEM);
Yishai Hadasaeae9452018-06-17 13:00:04 +03002177 MLX5_SET64(umem, umem, num_of_mtt, obj->ncont);
2178 MLX5_SET(umem, umem, log_page_size, obj->page_shift -
2179 MLX5_ADAPTER_PAGE_SHIFT);
2180 MLX5_SET(umem, umem, page_offset, obj->page_offset);
2181 mlx5_ib_populate_pas(dev, obj->umem, obj->page_shift, mtt,
2182 (obj->umem->writable ? MLX5_IB_MTT_WRITE : 0) |
2183 MLX5_IB_MTT_READ);
2184}
2185
Jason Gunthorpee83f0ec2018-07-25 21:40:18 -06002186static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_UMEM_REG)(
Jason Gunthorpe15a1b4b2018-11-25 20:51:15 +02002187 struct uverbs_attr_bundle *attrs)
Yishai Hadasaeae9452018-06-17 13:00:04 +03002188{
Yishai Hadasaeae9452018-06-17 13:00:04 +03002189 struct devx_umem_reg_cmd cmd;
2190 struct devx_umem *obj;
Jason Gunthorpec36ee462018-07-10 20:55:22 -06002191 struct ib_uobject *uobj = uverbs_attr_get_uobject(
2192 attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE);
Yishai Hadasaeae9452018-06-17 13:00:04 +03002193 u32 obj_id;
Shamir Rabinovitch89944452019-02-07 18:44:49 +02002194 struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context(
2195 &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext);
Jason Gunthorpec36ee462018-07-10 20:55:22 -06002196 struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device);
Yishai Hadasaeae9452018-06-17 13:00:04 +03002197 int err;
2198
2199 if (!c->devx_uid)
Yishai Hadas7e1335a2018-09-20 21:45:20 +03002200 return -EINVAL;
2201
Yishai Hadasaeae9452018-06-17 13:00:04 +03002202 obj = kzalloc(sizeof(struct devx_umem), GFP_KERNEL);
2203 if (!obj)
2204 return -ENOMEM;
2205
2206 err = devx_umem_get(dev, &c->ibucontext, attrs, obj);
2207 if (err)
2208 goto err_obj_free;
2209
Jason Gunthorpeb61815e2018-08-09 20:14:41 -06002210 err = devx_umem_reg_cmd_alloc(attrs, obj, &cmd);
Yishai Hadasaeae9452018-06-17 13:00:04 +03002211 if (err)
2212 goto err_umem_release;
2213
2214 devx_umem_reg_cmd_build(dev, obj, &cmd);
2215
Yishai Hadas6e3722b2018-12-19 16:28:15 +02002216 MLX5_SET(create_umem_in, cmd.in, uid, c->devx_uid);
Yishai Hadasaeae9452018-06-17 13:00:04 +03002217 err = mlx5_cmd_exec(dev->mdev, cmd.in, cmd.inlen, cmd.out,
2218 sizeof(cmd.out));
2219 if (err)
Jason Gunthorpeb61815e2018-08-09 20:14:41 -06002220 goto err_umem_release;
Yishai Hadasaeae9452018-06-17 13:00:04 +03002221
2222 obj->mdev = dev->mdev;
2223 uobj->object = obj;
2224 devx_obj_build_destroy_cmd(cmd.in, cmd.out, obj->dinbox, &obj->dinlen, &obj_id);
Jason Gunthorpe0ac89032020-05-19 10:27:05 +03002225 uverbs_finalize_uobj_create(attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE);
Yishai Hadasaeae9452018-06-17 13:00:04 +03002226
Jason Gunthorpe0ac89032020-05-19 10:27:05 +03002227 err = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_OUT_ID, &obj_id,
2228 sizeof(obj_id));
2229 return err;
Yishai Hadasaeae9452018-06-17 13:00:04 +03002230
Yishai Hadasaeae9452018-06-17 13:00:04 +03002231err_umem_release:
2232 ib_umem_release(obj->umem);
2233err_obj_free:
2234 kfree(obj);
2235 return err;
2236}
2237
Yishai Hadasaeae9452018-06-17 13:00:04 +03002238static int devx_umem_cleanup(struct ib_uobject *uobject,
Shamir Rabinovitcha6a37972019-03-31 19:10:04 +03002239 enum rdma_remove_reason why,
2240 struct uverbs_attr_bundle *attrs)
Yishai Hadasaeae9452018-06-17 13:00:04 +03002241{
2242 struct devx_umem *obj = uobject->object;
2243 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
2244 int err;
2245
2246 err = mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, out, sizeof(out));
Yishai Hadas1c774832018-06-20 17:11:39 +03002247 if (ib_is_destroy_retryable(err, why, uobject))
Yishai Hadasaeae9452018-06-17 13:00:04 +03002248 return err;
2249
2250 ib_umem_release(obj->umem);
2251 kfree(obj);
2252 return 0;
2253}
2254
Yishai Hadas5ec9d8e2019-06-30 19:23:32 +03002255static bool is_unaffiliated_event(struct mlx5_core_dev *dev,
2256 unsigned long event_type)
2257{
2258 __be64 *unaff_events;
2259 int mask_entry;
2260 int mask_bit;
2261
2262 if (!MLX5_CAP_GEN(dev, event_cap))
2263 return is_legacy_unaffiliated_event_num(event_type);
2264
2265 unaff_events = MLX5_CAP_DEV_EVENT(dev,
2266 user_unaffiliated_events);
2267 WARN_ON(event_type > MAX_SUPP_EVENT_NUM);
2268
2269 mask_entry = event_type / 64;
2270 mask_bit = event_type % 64;
2271
2272 if (!(be64_to_cpu(unaff_events[mask_entry]) & (1ull << mask_bit)))
2273 return false;
2274
2275 return true;
2276}
2277
2278static u32 devx_get_obj_id_from_event(unsigned long event_type, void *data)
2279{
2280 struct mlx5_eqe *eqe = data;
2281 u32 obj_id = 0;
2282
2283 switch (event_type) {
2284 case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
2285 case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
2286 case MLX5_EVENT_TYPE_PATH_MIG:
2287 case MLX5_EVENT_TYPE_COMM_EST:
2288 case MLX5_EVENT_TYPE_SQ_DRAINED:
2289 case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
2290 case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
2291 case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
2292 case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
2293 case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
2294 obj_id = be32_to_cpu(eqe->data.qp_srq.qp_srq_n) & 0xffffff;
2295 break;
Yishai Hadas972d7562019-08-08 11:43:57 +03002296 case MLX5_EVENT_TYPE_XRQ_ERROR:
2297 obj_id = be32_to_cpu(eqe->data.xrq_err.type_xrqn) & 0xffffff;
2298 break;
Yishai Hadas5ec9d8e2019-06-30 19:23:32 +03002299 case MLX5_EVENT_TYPE_DCT_DRAINED:
Yishai Hadas972d7562019-08-08 11:43:57 +03002300 case MLX5_EVENT_TYPE_DCT_KEY_VIOLATION:
Yishai Hadas5ec9d8e2019-06-30 19:23:32 +03002301 obj_id = be32_to_cpu(eqe->data.dct.dctn) & 0xffffff;
2302 break;
2303 case MLX5_EVENT_TYPE_CQ_ERROR:
2304 obj_id = be32_to_cpu(eqe->data.cq_err.cqn) & 0xffffff;
2305 break;
2306 default:
2307 obj_id = MLX5_GET(affiliated_event_header, &eqe->data, obj_id);
2308 break;
2309 }
2310
2311 return obj_id;
2312}
2313
2314static int deliver_event(struct devx_event_subscription *event_sub,
2315 const void *data)
2316{
2317 struct devx_async_event_file *ev_file;
2318 struct devx_async_event_data *event_data;
2319 unsigned long flags;
2320
2321 ev_file = event_sub->ev_file;
2322
2323 if (ev_file->omit_data) {
2324 spin_lock_irqsave(&ev_file->lock, flags);
Yishai Hadasa8af8692020-02-12 09:26:32 +02002325 if (!list_empty(&event_sub->event_list) ||
2326 ev_file->is_destroyed) {
Yishai Hadas5ec9d8e2019-06-30 19:23:32 +03002327 spin_unlock_irqrestore(&ev_file->lock, flags);
2328 return 0;
2329 }
2330
2331 list_add_tail(&event_sub->event_list, &ev_file->event_list);
2332 spin_unlock_irqrestore(&ev_file->lock, flags);
2333 wake_up_interruptible(&ev_file->poll_wait);
2334 return 0;
2335 }
2336
2337 event_data = kzalloc(sizeof(*event_data) + sizeof(struct mlx5_eqe),
2338 GFP_ATOMIC);
2339 if (!event_data) {
2340 spin_lock_irqsave(&ev_file->lock, flags);
2341 ev_file->is_overflow_err = 1;
2342 spin_unlock_irqrestore(&ev_file->lock, flags);
2343 return -ENOMEM;
2344 }
2345
2346 event_data->hdr.cookie = event_sub->cookie;
2347 memcpy(event_data->hdr.out_data, data, sizeof(struct mlx5_eqe));
2348
2349 spin_lock_irqsave(&ev_file->lock, flags);
Jason Gunthorpef7c84162020-01-08 19:21:54 +02002350 if (!ev_file->is_destroyed)
2351 list_add_tail(&event_data->list, &ev_file->event_list);
2352 else
2353 kfree(event_data);
Yishai Hadas5ec9d8e2019-06-30 19:23:32 +03002354 spin_unlock_irqrestore(&ev_file->lock, flags);
2355 wake_up_interruptible(&ev_file->poll_wait);
2356
2357 return 0;
2358}
2359
2360static void dispatch_event_fd(struct list_head *fd_list,
2361 const void *data)
2362{
2363 struct devx_event_subscription *item;
2364
2365 list_for_each_entry_rcu(item, fd_list, xa_list) {
Jason Gunthorpe6898d1c2020-01-08 19:21:53 +02002366 if (item->eventfd)
Yishai Hadas5ec9d8e2019-06-30 19:23:32 +03002367 eventfd_signal(item->eventfd, 1);
Jason Gunthorpe6898d1c2020-01-08 19:21:53 +02002368 else
2369 deliver_event(item, data);
Yishai Hadas5ec9d8e2019-06-30 19:23:32 +03002370 }
2371}
2372
Yishai Hadase337dd52019-06-30 19:23:30 +03002373static int devx_event_notifier(struct notifier_block *nb,
2374 unsigned long event_type, void *data)
2375{
Yishai Hadas5ec9d8e2019-06-30 19:23:32 +03002376 struct mlx5_devx_event_table *table;
2377 struct mlx5_ib_dev *dev;
2378 struct devx_event *event;
2379 struct devx_obj_event *obj_event;
2380 u16 obj_type = 0;
2381 bool is_unaffiliated;
2382 u32 obj_id;
2383
2384 /* Explicit filtering to kernel events which may occur frequently */
2385 if (event_type == MLX5_EVENT_TYPE_CMD ||
2386 event_type == MLX5_EVENT_TYPE_PAGE_REQUEST)
2387 return NOTIFY_OK;
2388
2389 table = container_of(nb, struct mlx5_devx_event_table, devx_nb.nb);
2390 dev = container_of(table, struct mlx5_ib_dev, devx_event_table);
2391 is_unaffiliated = is_unaffiliated_event(dev->mdev, event_type);
2392
2393 if (!is_unaffiliated)
2394 obj_type = get_event_obj_type(event_type, data);
2395
2396 rcu_read_lock();
2397 event = xa_load(&table->event_xa, event_type | (obj_type << 16));
2398 if (!event) {
2399 rcu_read_unlock();
2400 return NOTIFY_DONE;
2401 }
2402
2403 if (is_unaffiliated) {
2404 dispatch_event_fd(&event->unaffiliated_list, data);
2405 rcu_read_unlock();
2406 return NOTIFY_OK;
2407 }
2408
2409 obj_id = devx_get_obj_id_from_event(event_type, data);
2410 obj_event = xa_load(&event->object_ids, obj_id);
2411 if (!obj_event) {
2412 rcu_read_unlock();
2413 return NOTIFY_DONE;
2414 }
2415
2416 dispatch_event_fd(&obj_event->obj_sub_list, data);
2417
2418 rcu_read_unlock();
2419 return NOTIFY_OK;
Yishai Hadase337dd52019-06-30 19:23:30 +03002420}
2421
2422void mlx5_ib_devx_init_event_table(struct mlx5_ib_dev *dev)
2423{
2424 struct mlx5_devx_event_table *table = &dev->devx_event_table;
2425
2426 xa_init(&table->event_xa);
2427 mutex_init(&table->event_xa_lock);
2428 MLX5_NB_INIT(&table->devx_nb, devx_event_notifier, NOTIFY_ANY);
2429 mlx5_eq_notifier_register(dev->mdev, &table->devx_nb);
2430}
2431
2432void mlx5_ib_devx_cleanup_event_table(struct mlx5_ib_dev *dev)
2433{
2434 struct mlx5_devx_event_table *table = &dev->devx_event_table;
Yishai Hadas75973852019-06-30 19:23:31 +03002435 struct devx_event_subscription *sub, *tmp;
2436 struct devx_event *event;
Yishai Hadase337dd52019-06-30 19:23:30 +03002437 void *entry;
2438 unsigned long id;
2439
2440 mlx5_eq_notifier_unregister(dev->mdev, &table->devx_nb);
Yishai Hadas75973852019-06-30 19:23:31 +03002441 mutex_lock(&dev->devx_event_table.event_xa_lock);
2442 xa_for_each(&table->event_xa, id, entry) {
2443 event = entry;
2444 list_for_each_entry_safe(sub, tmp, &event->unaffiliated_list,
2445 xa_list)
2446 devx_cleanup_subscription(dev, sub);
Yishai Hadase337dd52019-06-30 19:23:30 +03002447 kfree(entry);
Yishai Hadas75973852019-06-30 19:23:31 +03002448 }
2449 mutex_unlock(&dev->devx_event_table.event_xa_lock);
Yishai Hadase337dd52019-06-30 19:23:30 +03002450 xa_destroy(&table->event_xa);
2451}
2452
Yishai Hadas6bf8f222019-01-22 08:29:56 +02002453static ssize_t devx_async_cmd_event_read(struct file *filp, char __user *buf,
2454 size_t count, loff_t *pos)
2455{
Yishai Hadas4accbb32019-01-22 08:29:58 +02002456 struct devx_async_cmd_event_file *comp_ev_file = filp->private_data;
2457 struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue;
2458 struct devx_async_data *event;
2459 int ret = 0;
2460 size_t eventsz;
2461
2462 spin_lock_irq(&ev_queue->lock);
2463
2464 while (list_empty(&ev_queue->event_list)) {
2465 spin_unlock_irq(&ev_queue->lock);
2466
2467 if (filp->f_flags & O_NONBLOCK)
2468 return -EAGAIN;
2469
2470 if (wait_event_interruptible(
2471 ev_queue->poll_wait,
Yishai Hadaseaebaf72019-01-22 08:29:59 +02002472 (!list_empty(&ev_queue->event_list) ||
2473 ev_queue->is_destroyed))) {
Yishai Hadas4accbb32019-01-22 08:29:58 +02002474 return -ERESTARTSYS;
2475 }
Yishai Hadaseaebaf72019-01-22 08:29:59 +02002476
Yishai Hadas4accbb32019-01-22 08:29:58 +02002477 spin_lock_irq(&ev_queue->lock);
Yishai Hadasa8af8692020-02-12 09:26:32 +02002478 if (ev_queue->is_destroyed) {
2479 spin_unlock_irq(&ev_queue->lock);
2480 return -EIO;
2481 }
Yishai Hadas4accbb32019-01-22 08:29:58 +02002482 }
2483
2484 event = list_entry(ev_queue->event_list.next,
2485 struct devx_async_data, list);
2486 eventsz = event->cmd_out_len +
2487 sizeof(struct mlx5_ib_uapi_devx_async_cmd_hdr);
2488
2489 if (eventsz > count) {
2490 spin_unlock_irq(&ev_queue->lock);
2491 return -ENOSPC;
2492 }
2493
2494 list_del(ev_queue->event_list.next);
2495 spin_unlock_irq(&ev_queue->lock);
2496
2497 if (copy_to_user(buf, &event->hdr, eventsz))
2498 ret = -EFAULT;
2499 else
2500 ret = eventsz;
2501
2502 atomic_sub(event->cmd_out_len, &ev_queue->bytes_in_use);
2503 kvfree(event);
2504 return ret;
Yishai Hadas6bf8f222019-01-22 08:29:56 +02002505}
2506
Yishai Hadas6bf8f222019-01-22 08:29:56 +02002507static __poll_t devx_async_cmd_event_poll(struct file *filp,
2508 struct poll_table_struct *wait)
2509{
Yishai Hadas4accbb32019-01-22 08:29:58 +02002510 struct devx_async_cmd_event_file *comp_ev_file = filp->private_data;
2511 struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue;
2512 __poll_t pollflags = 0;
2513
2514 poll_wait(filp, &ev_queue->poll_wait, wait);
2515
2516 spin_lock_irq(&ev_queue->lock);
Yishai Hadaseaebaf72019-01-22 08:29:59 +02002517 if (ev_queue->is_destroyed)
2518 pollflags = EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
2519 else if (!list_empty(&ev_queue->event_list))
Yishai Hadas4accbb32019-01-22 08:29:58 +02002520 pollflags = EPOLLIN | EPOLLRDNORM;
2521 spin_unlock_irq(&ev_queue->lock);
2522
2523 return pollflags;
Yishai Hadas6bf8f222019-01-22 08:29:56 +02002524}
2525
Bart Van Assche1f687ed2019-03-27 16:50:48 -07002526static const struct file_operations devx_async_cmd_event_fops = {
Yishai Hadas6bf8f222019-01-22 08:29:56 +02002527 .owner = THIS_MODULE,
2528 .read = devx_async_cmd_event_read,
2529 .poll = devx_async_cmd_event_poll,
Jason Gunthorpef7c84162020-01-08 19:21:54 +02002530 .release = uverbs_uobject_fd_release,
Yishai Hadas6bf8f222019-01-22 08:29:56 +02002531 .llseek = no_llseek,
2532};
2533
Yishai Hadas2afc5e12019-06-30 19:23:29 +03002534static ssize_t devx_async_event_read(struct file *filp, char __user *buf,
2535 size_t count, loff_t *pos)
2536{
Yishai Hadas5ec9d8e2019-06-30 19:23:32 +03002537 struct devx_async_event_file *ev_file = filp->private_data;
2538 struct devx_event_subscription *event_sub;
2539 struct devx_async_event_data *uninitialized_var(event);
2540 int ret = 0;
2541 size_t eventsz;
2542 bool omit_data;
2543 void *event_data;
2544
2545 omit_data = ev_file->omit_data;
2546
2547 spin_lock_irq(&ev_file->lock);
2548
2549 if (ev_file->is_overflow_err) {
2550 ev_file->is_overflow_err = 0;
2551 spin_unlock_irq(&ev_file->lock);
2552 return -EOVERFLOW;
2553 }
2554
Yishai Hadas5ec9d8e2019-06-30 19:23:32 +03002555
2556 while (list_empty(&ev_file->event_list)) {
2557 spin_unlock_irq(&ev_file->lock);
2558
2559 if (filp->f_flags & O_NONBLOCK)
2560 return -EAGAIN;
2561
2562 if (wait_event_interruptible(ev_file->poll_wait,
2563 (!list_empty(&ev_file->event_list) ||
2564 ev_file->is_destroyed))) {
2565 return -ERESTARTSYS;
2566 }
2567
2568 spin_lock_irq(&ev_file->lock);
2569 if (ev_file->is_destroyed) {
2570 spin_unlock_irq(&ev_file->lock);
2571 return -EIO;
2572 }
2573 }
2574
2575 if (omit_data) {
2576 event_sub = list_first_entry(&ev_file->event_list,
2577 struct devx_event_subscription,
2578 event_list);
2579 eventsz = sizeof(event_sub->cookie);
2580 event_data = &event_sub->cookie;
2581 } else {
2582 event = list_first_entry(&ev_file->event_list,
2583 struct devx_async_event_data, list);
2584 eventsz = sizeof(struct mlx5_eqe) +
2585 sizeof(struct mlx5_ib_uapi_devx_async_event_hdr);
2586 event_data = &event->hdr;
2587 }
2588
2589 if (eventsz > count) {
2590 spin_unlock_irq(&ev_file->lock);
2591 return -EINVAL;
2592 }
2593
2594 if (omit_data)
2595 list_del_init(&event_sub->event_list);
2596 else
2597 list_del(&event->list);
2598
2599 spin_unlock_irq(&ev_file->lock);
2600
2601 if (copy_to_user(buf, event_data, eventsz))
2602 /* This points to an application issue, not a kernel concern */
2603 ret = -EFAULT;
2604 else
2605 ret = eventsz;
2606
2607 if (!omit_data)
2608 kfree(event);
2609 return ret;
Yishai Hadas2afc5e12019-06-30 19:23:29 +03002610}
2611
2612static __poll_t devx_async_event_poll(struct file *filp,
2613 struct poll_table_struct *wait)
2614{
Yishai Hadas5ec9d8e2019-06-30 19:23:32 +03002615 struct devx_async_event_file *ev_file = filp->private_data;
2616 __poll_t pollflags = 0;
2617
2618 poll_wait(filp, &ev_file->poll_wait, wait);
2619
2620 spin_lock_irq(&ev_file->lock);
2621 if (ev_file->is_destroyed)
2622 pollflags = EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
2623 else if (!list_empty(&ev_file->event_list))
2624 pollflags = EPOLLIN | EPOLLRDNORM;
2625 spin_unlock_irq(&ev_file->lock);
2626
2627 return pollflags;
Yishai Hadas2afc5e12019-06-30 19:23:29 +03002628}
2629
Jason Gunthorpe6898d1c2020-01-08 19:21:53 +02002630static void devx_free_subscription(struct rcu_head *rcu)
2631{
2632 struct devx_event_subscription *event_sub =
2633 container_of(rcu, struct devx_event_subscription, rcu);
2634
2635 if (event_sub->eventfd)
2636 eventfd_ctx_put(event_sub->eventfd);
2637 uverbs_uobject_put(&event_sub->ev_file->uobj);
2638 kfree(event_sub);
2639}
2640
Jason Gunthorpef7c84162020-01-08 19:21:54 +02002641static const struct file_operations devx_async_event_fops = {
2642 .owner = THIS_MODULE,
2643 .read = devx_async_event_read,
2644 .poll = devx_async_event_poll,
2645 .release = uverbs_uobject_fd_release,
2646 .llseek = no_llseek,
2647};
2648
2649static int devx_async_cmd_event_destroy_uobj(struct ib_uobject *uobj,
2650 enum rdma_remove_reason why)
Yishai Hadas2afc5e12019-06-30 19:23:29 +03002651{
Jason Gunthorpef7c84162020-01-08 19:21:54 +02002652 struct devx_async_cmd_event_file *comp_ev_file =
2653 container_of(uobj, struct devx_async_cmd_event_file,
2654 uobj);
2655 struct devx_async_event_queue *ev_queue = &comp_ev_file->ev_queue;
2656 struct devx_async_data *entry, *tmp;
2657
2658 spin_lock_irq(&ev_queue->lock);
2659 ev_queue->is_destroyed = 1;
2660 spin_unlock_irq(&ev_queue->lock);
2661 wake_up_interruptible(&ev_queue->poll_wait);
2662
2663 mlx5_cmd_cleanup_async_ctx(&comp_ev_file->async_ctx);
2664
2665 spin_lock_irq(&comp_ev_file->ev_queue.lock);
2666 list_for_each_entry_safe(entry, tmp,
Yishai Hadasa8af8692020-02-12 09:26:32 +02002667 &comp_ev_file->ev_queue.event_list, list) {
2668 list_del(&entry->list);
Jason Gunthorpef7c84162020-01-08 19:21:54 +02002669 kvfree(entry);
Yishai Hadasa8af8692020-02-12 09:26:32 +02002670 }
Jason Gunthorpef7c84162020-01-08 19:21:54 +02002671 spin_unlock_irq(&comp_ev_file->ev_queue.lock);
2672 return 0;
2673};
2674
2675static int devx_async_event_destroy_uobj(struct ib_uobject *uobj,
2676 enum rdma_remove_reason why)
2677{
2678 struct devx_async_event_file *ev_file =
2679 container_of(uobj, struct devx_async_event_file,
2680 uobj);
Yishai Hadas75973852019-06-30 19:23:31 +03002681 struct devx_event_subscription *event_sub, *event_sub_tmp;
Yishai Hadase9eec6a2019-08-08 11:15:38 +03002682 struct mlx5_ib_dev *dev = ev_file->dev;
Yishai Hadas75973852019-06-30 19:23:31 +03002683
Jason Gunthorpef7c84162020-01-08 19:21:54 +02002684 spin_lock_irq(&ev_file->lock);
2685 ev_file->is_destroyed = 1;
Yishai Hadasa8af8692020-02-12 09:26:32 +02002686
2687 /* free the pending events allocation */
2688 if (ev_file->omit_data) {
2689 struct devx_event_subscription *event_sub, *tmp;
2690
2691 list_for_each_entry_safe(event_sub, tmp, &ev_file->event_list,
2692 event_list)
2693 list_del_init(&event_sub->event_list);
2694
2695 } else {
2696 struct devx_async_event_data *entry, *tmp;
2697
2698 list_for_each_entry_safe(entry, tmp, &ev_file->event_list,
2699 list) {
2700 list_del(&entry->list);
2701 kfree(entry);
2702 }
2703 }
2704
Jason Gunthorpef7c84162020-01-08 19:21:54 +02002705 spin_unlock_irq(&ev_file->lock);
2706 wake_up_interruptible(&ev_file->poll_wait);
2707
Yishai Hadase9eec6a2019-08-08 11:15:38 +03002708 mutex_lock(&dev->devx_event_table.event_xa_lock);
Yishai Hadas75973852019-06-30 19:23:31 +03002709 /* delete the subscriptions which are related to this FD */
2710 list_for_each_entry_safe(event_sub, event_sub_tmp,
2711 &ev_file->subscribed_events_list, file_list) {
Yishai Hadase9eec6a2019-08-08 11:15:38 +03002712 devx_cleanup_subscription(dev, event_sub);
Yishai Hadas75973852019-06-30 19:23:31 +03002713 list_del_rcu(&event_sub->file_list);
2714 /* subscription may not be used by the read API any more */
Jason Gunthorpe6898d1c2020-01-08 19:21:53 +02002715 call_rcu(&event_sub->rcu, devx_free_subscription);
Yishai Hadas75973852019-06-30 19:23:31 +03002716 }
Yishai Hadase9eec6a2019-08-08 11:15:38 +03002717 mutex_unlock(&dev->devx_event_table.event_xa_lock);
Yishai Hadas75973852019-06-30 19:23:31 +03002718
Yishai Hadase9eec6a2019-08-08 11:15:38 +03002719 put_device(&dev->ib_dev.dev);
Yishai Hadas2afc5e12019-06-30 19:23:29 +03002720 return 0;
Yishai Hadas2afc5e12019-06-30 19:23:29 +03002721};
2722
Jason Gunthorpe9a119cd2018-07-04 08:50:28 +03002723DECLARE_UVERBS_NAMED_METHOD(
2724 MLX5_IB_METHOD_DEVX_UMEM_REG,
2725 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE,
2726 MLX5_IB_OBJECT_DEVX_UMEM,
2727 UVERBS_ACCESS_NEW,
Jason Gunthorpe83bb4442018-07-04 08:50:29 +03002728 UA_MANDATORY),
Jason Gunthorpe9a119cd2018-07-04 08:50:28 +03002729 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_ADDR,
2730 UVERBS_ATTR_TYPE(u64),
Jason Gunthorpe83bb4442018-07-04 08:50:29 +03002731 UA_MANDATORY),
Jason Gunthorpe9a119cd2018-07-04 08:50:28 +03002732 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_LEN,
2733 UVERBS_ATTR_TYPE(u64),
Jason Gunthorpe83bb4442018-07-04 08:50:29 +03002734 UA_MANDATORY),
Jason Gunthorpebccd0622018-07-26 16:37:14 -06002735 UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS,
2736 enum ib_access_flags),
Jason Gunthorpe9a119cd2018-07-04 08:50:28 +03002737 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_UMEM_REG_OUT_ID,
2738 UVERBS_ATTR_TYPE(u32),
Jason Gunthorpe83bb4442018-07-04 08:50:29 +03002739 UA_MANDATORY));
Yishai Hadasaeae9452018-06-17 13:00:04 +03002740
Yishai Hadas528922a2018-07-08 13:24:39 +03002741DECLARE_UVERBS_NAMED_METHOD_DESTROY(
Jason Gunthorpe9a119cd2018-07-04 08:50:28 +03002742 MLX5_IB_METHOD_DEVX_UMEM_DEREG,
2743 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_UMEM_DEREG_HANDLE,
2744 MLX5_IB_OBJECT_DEVX_UMEM,
2745 UVERBS_ACCESS_DESTROY,
Jason Gunthorpe83bb4442018-07-04 08:50:29 +03002746 UA_MANDATORY));
Yishai Hadasaeae9452018-06-17 13:00:04 +03002747
Jason Gunthorpe9a119cd2018-07-04 08:50:28 +03002748DECLARE_UVERBS_NAMED_METHOD(
2749 MLX5_IB_METHOD_DEVX_QUERY_EQN,
2750 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_QUERY_EQN_USER_VEC,
2751 UVERBS_ATTR_TYPE(u32),
Jason Gunthorpe83bb4442018-07-04 08:50:29 +03002752 UA_MANDATORY),
Jason Gunthorpe9a119cd2018-07-04 08:50:28 +03002753 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_QUERY_EQN_DEV_EQN,
2754 UVERBS_ATTR_TYPE(u32),
Jason Gunthorpe83bb4442018-07-04 08:50:29 +03002755 UA_MANDATORY));
Yishai Hadasf6fe01b2018-06-17 13:00:05 +03002756
Jason Gunthorpe9a119cd2018-07-04 08:50:28 +03002757DECLARE_UVERBS_NAMED_METHOD(
2758 MLX5_IB_METHOD_DEVX_QUERY_UAR,
2759 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_QUERY_UAR_USER_IDX,
2760 UVERBS_ATTR_TYPE(u32),
Jason Gunthorpe83bb4442018-07-04 08:50:29 +03002761 UA_MANDATORY),
Jason Gunthorpe9a119cd2018-07-04 08:50:28 +03002762 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_QUERY_UAR_DEV_IDX,
2763 UVERBS_ATTR_TYPE(u32),
Jason Gunthorpe83bb4442018-07-04 08:50:29 +03002764 UA_MANDATORY));
Yishai Hadas7c043e92018-06-17 13:00:03 +03002765
Jason Gunthorpe9a119cd2018-07-04 08:50:28 +03002766DECLARE_UVERBS_NAMED_METHOD(
2767 MLX5_IB_METHOD_DEVX_OTHER,
2768 UVERBS_ATTR_PTR_IN(
2769 MLX5_IB_ATTR_DEVX_OTHER_CMD_IN,
2770 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
Jason Gunthorpe83bb4442018-07-04 08:50:29 +03002771 UA_MANDATORY,
Jason Gunthorpe83bb4442018-07-04 08:50:29 +03002772 UA_ALLOC_AND_COPY),
Jason Gunthorpe9a119cd2018-07-04 08:50:28 +03002773 UVERBS_ATTR_PTR_OUT(
2774 MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT,
2775 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
Jason Gunthorpe540cd692018-07-04 08:50:30 +03002776 UA_MANDATORY));
Yishai Hadas8aa8c952018-06-17 13:00:00 +03002777
Jason Gunthorpe9a119cd2018-07-04 08:50:28 +03002778DECLARE_UVERBS_NAMED_METHOD(
2779 MLX5_IB_METHOD_DEVX_OBJ_CREATE,
2780 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_CREATE_HANDLE,
2781 MLX5_IB_OBJECT_DEVX_OBJ,
2782 UVERBS_ACCESS_NEW,
Jason Gunthorpe83bb4442018-07-04 08:50:29 +03002783 UA_MANDATORY),
Jason Gunthorpe9a119cd2018-07-04 08:50:28 +03002784 UVERBS_ATTR_PTR_IN(
2785 MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN,
2786 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
Jason Gunthorpe83bb4442018-07-04 08:50:29 +03002787 UA_MANDATORY,
Jason Gunthorpe83bb4442018-07-04 08:50:29 +03002788 UA_ALLOC_AND_COPY),
Jason Gunthorpe9a119cd2018-07-04 08:50:28 +03002789 UVERBS_ATTR_PTR_OUT(
2790 MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT,
2791 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
Jason Gunthorpe540cd692018-07-04 08:50:30 +03002792 UA_MANDATORY));
Yishai Hadas7efce362018-06-17 13:00:01 +03002793
Yishai Hadas528922a2018-07-08 13:24:39 +03002794DECLARE_UVERBS_NAMED_METHOD_DESTROY(
Jason Gunthorpe9a119cd2018-07-04 08:50:28 +03002795 MLX5_IB_METHOD_DEVX_OBJ_DESTROY,
2796 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_DESTROY_HANDLE,
2797 MLX5_IB_OBJECT_DEVX_OBJ,
2798 UVERBS_ACCESS_DESTROY,
Jason Gunthorpe83bb4442018-07-04 08:50:29 +03002799 UA_MANDATORY));
Yishai Hadas7efce362018-06-17 13:00:01 +03002800
Jason Gunthorpe9a119cd2018-07-04 08:50:28 +03002801DECLARE_UVERBS_NAMED_METHOD(
2802 MLX5_IB_METHOD_DEVX_OBJ_MODIFY,
2803 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_MODIFY_HANDLE,
Yishai Hadas34613eb2018-11-26 08:28:35 +02002804 UVERBS_IDR_ANY_OBJECT,
Jason Gunthorpe9a119cd2018-07-04 08:50:28 +03002805 UVERBS_ACCESS_WRITE,
Jason Gunthorpe83bb4442018-07-04 08:50:29 +03002806 UA_MANDATORY),
Jason Gunthorpe9a119cd2018-07-04 08:50:28 +03002807 UVERBS_ATTR_PTR_IN(
2808 MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN,
2809 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
Jason Gunthorpe83bb4442018-07-04 08:50:29 +03002810 UA_MANDATORY,
Jason Gunthorpe83bb4442018-07-04 08:50:29 +03002811 UA_ALLOC_AND_COPY),
Jason Gunthorpe9a119cd2018-07-04 08:50:28 +03002812 UVERBS_ATTR_PTR_OUT(
2813 MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT,
2814 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
Jason Gunthorpe540cd692018-07-04 08:50:30 +03002815 UA_MANDATORY));
Yishai Hadase662e142018-06-17 13:00:02 +03002816
Jason Gunthorpe9a119cd2018-07-04 08:50:28 +03002817DECLARE_UVERBS_NAMED_METHOD(
2818 MLX5_IB_METHOD_DEVX_OBJ_QUERY,
2819 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE,
Yishai Hadas34613eb2018-11-26 08:28:35 +02002820 UVERBS_IDR_ANY_OBJECT,
Jason Gunthorpe9a119cd2018-07-04 08:50:28 +03002821 UVERBS_ACCESS_READ,
Jason Gunthorpe83bb4442018-07-04 08:50:29 +03002822 UA_MANDATORY),
Jason Gunthorpe9a119cd2018-07-04 08:50:28 +03002823 UVERBS_ATTR_PTR_IN(
2824 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN,
2825 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
Jason Gunthorpe83bb4442018-07-04 08:50:29 +03002826 UA_MANDATORY,
Jason Gunthorpe83bb4442018-07-04 08:50:29 +03002827 UA_ALLOC_AND_COPY),
Jason Gunthorpe9a119cd2018-07-04 08:50:28 +03002828 UVERBS_ATTR_PTR_OUT(
2829 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT,
2830 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
Jason Gunthorpe540cd692018-07-04 08:50:30 +03002831 UA_MANDATORY));
Yishai Hadase662e142018-06-17 13:00:02 +03002832
Yishai Hadasa124edb2019-01-22 08:29:57 +02002833DECLARE_UVERBS_NAMED_METHOD(
2834 MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY,
2835 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE,
2836 UVERBS_IDR_ANY_OBJECT,
2837 UVERBS_ACCESS_READ,
2838 UA_MANDATORY),
2839 UVERBS_ATTR_PTR_IN(
2840 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN,
2841 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
2842 UA_MANDATORY,
2843 UA_ALLOC_AND_COPY),
2844 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_OUT_LEN,
2845 u16, UA_MANDATORY),
2846 UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_FD,
2847 MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
2848 UVERBS_ACCESS_READ,
2849 UA_MANDATORY),
2850 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_WR_ID,
2851 UVERBS_ATTR_TYPE(u64),
2852 UA_MANDATORY));
2853
Yishai Hadas75973852019-06-30 19:23:31 +03002854DECLARE_UVERBS_NAMED_METHOD(
2855 MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT,
2856 UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_HANDLE,
2857 MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,
2858 UVERBS_ACCESS_READ,
2859 UA_MANDATORY),
2860 UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_OBJ_HANDLE,
2861 MLX5_IB_OBJECT_DEVX_OBJ,
2862 UVERBS_ACCESS_READ,
2863 UA_OPTIONAL),
2864 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_TYPE_NUM_LIST,
2865 UVERBS_ATTR_MIN_SIZE(sizeof(u16)),
2866 UA_MANDATORY,
2867 UA_ALLOC_AND_COPY),
2868 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_COOKIE,
2869 UVERBS_ATTR_TYPE(u64),
2870 UA_OPTIONAL),
2871 UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_NUM,
2872 UVERBS_ATTR_TYPE(u32),
2873 UA_OPTIONAL));
2874
Jason Gunthorpe6c61d2a2018-07-04 08:50:27 +03002875DECLARE_UVERBS_GLOBAL_METHODS(MLX5_IB_OBJECT_DEVX,
Jason Gunthorpe9a119cd2018-07-04 08:50:28 +03002876 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OTHER),
2877 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_QUERY_UAR),
Yishai Hadas75973852019-06-30 19:23:31 +03002878 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_QUERY_EQN),
2879 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT));
Yishai Hadas8aa8c952018-06-17 13:00:00 +03002880
Jason Gunthorpe6c61d2a2018-07-04 08:50:27 +03002881DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_DEVX_OBJ,
Jason Gunthorpe9a119cd2018-07-04 08:50:28 +03002882 UVERBS_TYPE_ALLOC_IDR(devx_obj_cleanup),
2883 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_CREATE),
2884 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_DESTROY),
2885 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_MODIFY),
Yishai Hadasa124edb2019-01-22 08:29:57 +02002886 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_QUERY),
2887 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY));
Yishai Hadas7efce362018-06-17 13:00:01 +03002888
Jason Gunthorpe6c61d2a2018-07-04 08:50:27 +03002889DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_DEVX_UMEM,
Jason Gunthorpe9a119cd2018-07-04 08:50:28 +03002890 UVERBS_TYPE_ALLOC_IDR(devx_umem_cleanup),
2891 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_UMEM_REG),
2892 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_UMEM_DEREG));
Yishai Hadasaeae9452018-06-17 13:00:04 +03002893
Yishai Hadas6bf8f222019-01-22 08:29:56 +02002894
2895DECLARE_UVERBS_NAMED_METHOD(
2896 MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC,
2897 UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_ASYNC_CMD_FD_ALLOC_HANDLE,
2898 MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
2899 UVERBS_ACCESS_NEW,
2900 UA_MANDATORY));
2901
2902DECLARE_UVERBS_NAMED_OBJECT(
2903 MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
2904 UVERBS_TYPE_ALLOC_FD(sizeof(struct devx_async_cmd_event_file),
Jason Gunthorpef7c84162020-01-08 19:21:54 +02002905 devx_async_cmd_event_destroy_uobj,
Yishai Hadas6bf8f222019-01-22 08:29:56 +02002906 &devx_async_cmd_event_fops, "[devx_async_cmd]",
2907 O_RDONLY),
2908 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC));
2909
Yishai Hadas2afc5e12019-06-30 19:23:29 +03002910DECLARE_UVERBS_NAMED_METHOD(
2911 MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC,
2912 UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_HANDLE,
2913 MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,
2914 UVERBS_ACCESS_NEW,
2915 UA_MANDATORY),
2916 UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_FLAGS,
2917 enum mlx5_ib_uapi_devx_create_event_channel_flags,
2918 UA_MANDATORY));
2919
2920DECLARE_UVERBS_NAMED_OBJECT(
2921 MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,
2922 UVERBS_TYPE_ALLOC_FD(sizeof(struct devx_async_event_file),
Jason Gunthorpef7c84162020-01-08 19:21:54 +02002923 devx_async_event_destroy_uobj,
Yishai Hadas2afc5e12019-06-30 19:23:29 +03002924 &devx_async_event_fops, "[devx_async_event]",
2925 O_RDONLY),
2926 &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC));
2927
Jason Gunthorpe36e235c2018-11-12 22:59:53 +02002928static bool devx_is_supported(struct ib_device *device)
Yishai Hadasc59450c2018-06-17 13:00:06 +03002929{
Jason Gunthorpe36e235c2018-11-12 22:59:53 +02002930 struct mlx5_ib_dev *dev = to_mdev(device);
2931
Mark Bloch7f575102019-03-28 15:46:25 +02002932 return MLX5_CAP_GEN(dev->mdev, log_max_uctx);
Yishai Hadasc59450c2018-06-17 13:00:06 +03002933}
Jason Gunthorpe36e235c2018-11-12 22:59:53 +02002934
Jason Gunthorpe0cbf4322018-11-12 22:59:50 +02002935const struct uapi_definition mlx5_ib_devx_defs[] = {
Jason Gunthorpe36e235c2018-11-12 22:59:53 +02002936 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
2937 MLX5_IB_OBJECT_DEVX,
2938 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
2939 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
2940 MLX5_IB_OBJECT_DEVX_OBJ,
2941 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
2942 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
2943 MLX5_IB_OBJECT_DEVX_UMEM,
2944 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
Yishai Hadas6bf8f222019-01-22 08:29:56 +02002945 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
2946 MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
2947 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
Yishai Hadas2afc5e12019-06-30 19:23:29 +03002948 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
2949 MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,
2950 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
Jason Gunthorpe0cbf4322018-11-12 22:59:50 +02002951 {},
2952};