blob: 246f745cb006b36169632d0554c4d7b2070f7a55 [file] [log] [blame]
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +02001/*
2 * Testsuite for eBPF maps
3 *
4 * Copyright (c) 2014 PLUMgrid, http://plumgrid.com
5 * Copyright (c) 2016 Facebook
6 *
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of version 2 of the GNU General Public
9 * License as published by the Free Software Foundation.
10 */
11
12#include <stdio.h>
13#include <unistd.h>
14#include <errno.h>
15#include <string.h>
16#include <assert.h>
17#include <stdlib.h>
Mauricio Vasquez B43b987d2018-10-18 15:16:41 +020018#include <time.h>
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +020019
20#include <sys/wait.h>
Martin KaFai Lau6bc8529c2018-08-08 01:01:30 -070021#include <sys/socket.h>
22#include <netinet/in.h>
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +020023#include <linux/bpf.h>
24
Mickaël Salaün10ecc722017-02-10 00:21:39 +010025#include <bpf/bpf.h>
John Fastabend6f6d33f2017-08-15 22:34:22 -070026#include <bpf/libbpf.h>
Daniel Borkmannfe8d6622018-02-26 22:34:32 +010027
Daniel Borkmanne00c7b22016-11-26 01:28:09 +010028#include "bpf_util.h"
Daniel Borkmannfe8d6622018-02-26 22:34:32 +010029#include "bpf_rlimit.h"
Martin KaFai Lau51a0e302019-04-26 16:39:52 -070030#include "test_maps.h"
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +020031
Martin KaFai Lau6bc8529c2018-08-08 01:01:30 -070032#ifndef ENOTSUPP
33#define ENOTSUPP 524
34#endif
35
Stanislav Fomicheve8ddbfb42019-01-28 09:21:15 -080036static int skips;
37
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +020038static int map_flags;
39
Breno Leitaodd9cef42019-02-05 15:12:34 -020040static void test_hashmap(unsigned int task, void *data)
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +020041{
Teng Qin8fe45922017-04-24 19:00:37 -070042 long long key, next_key, first_key, value;
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +020043 int fd;
44
Mickaël Salaünf4874d02017-02-10 00:21:43 +010045 fd = bpf_create_map(BPF_MAP_TYPE_HASH, sizeof(key), sizeof(value),
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +020046 2, map_flags);
47 if (fd < 0) {
48 printf("Failed to create hashmap '%s'!\n", strerror(errno));
49 exit(1);
50 }
51
52 key = 1;
53 value = 1234;
54 /* Insert key=1 element. */
Mickaël Salaün10ecc722017-02-10 00:21:39 +010055 assert(bpf_map_update_elem(fd, &key, &value, BPF_ANY) == 0);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +020056
57 value = 0;
58 /* BPF_NOEXIST means add new element if it doesn't exist. */
Mickaël Salaün10ecc722017-02-10 00:21:39 +010059 assert(bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST) == -1 &&
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +020060 /* key=1 already exists. */
61 errno == EEXIST);
62
63 /* -1 is an invalid flag. */
Mickaël Salaün10ecc722017-02-10 00:21:39 +010064 assert(bpf_map_update_elem(fd, &key, &value, -1) == -1 &&
65 errno == EINVAL);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +020066
67 /* Check that key=1 can be found. */
Mickaël Salaüne5ff7c42017-02-10 00:21:40 +010068 assert(bpf_map_lookup_elem(fd, &key, &value) == 0 && value == 1234);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +020069
70 key = 2;
71 /* Check that key=2 is not found. */
Mickaël Salaüne5ff7c42017-02-10 00:21:40 +010072 assert(bpf_map_lookup_elem(fd, &key, &value) == -1 && errno == ENOENT);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +020073
74 /* BPF_EXIST means update existing element. */
Mickaël Salaün10ecc722017-02-10 00:21:39 +010075 assert(bpf_map_update_elem(fd, &key, &value, BPF_EXIST) == -1 &&
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +020076 /* key=2 is not there. */
77 errno == ENOENT);
78
79 /* Insert key=2 element. */
Mickaël Salaün10ecc722017-02-10 00:21:39 +010080 assert(bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST) == 0);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +020081
82 /* key=1 and key=2 were inserted, check that key=0 cannot be
83 * inserted due to max_entries limit.
84 */
85 key = 0;
Mickaël Salaün10ecc722017-02-10 00:21:39 +010086 assert(bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST) == -1 &&
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +020087 errno == E2BIG);
88
89 /* Update existing element, though the map is full. */
90 key = 1;
Mickaël Salaün10ecc722017-02-10 00:21:39 +010091 assert(bpf_map_update_elem(fd, &key, &value, BPF_EXIST) == 0);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +020092 key = 2;
Mickaël Salaün10ecc722017-02-10 00:21:39 +010093 assert(bpf_map_update_elem(fd, &key, &value, BPF_ANY) == 0);
Alexei Starovoitov8c290e62017-03-21 19:05:04 -070094 key = 3;
95 assert(bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST) == -1 &&
96 errno == E2BIG);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +020097
98 /* Check that key = 0 doesn't exist. */
99 key = 0;
Mickaël Salaüne58383b2017-02-10 00:21:41 +0100100 assert(bpf_map_delete_elem(fd, &key) == -1 && errno == ENOENT);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200101
102 /* Iterate over two elements. */
Teng Qin8fe45922017-04-24 19:00:37 -0700103 assert(bpf_map_get_next_key(fd, NULL, &first_key) == 0 &&
104 (first_key == 1 || first_key == 2));
Mickaël Salaün5f155c22017-02-10 00:21:42 +0100105 assert(bpf_map_get_next_key(fd, &key, &next_key) == 0 &&
Teng Qin8fe45922017-04-24 19:00:37 -0700106 (next_key == first_key));
Mickaël Salaün5f155c22017-02-10 00:21:42 +0100107 assert(bpf_map_get_next_key(fd, &next_key, &next_key) == 0 &&
Teng Qin8fe45922017-04-24 19:00:37 -0700108 (next_key == 1 || next_key == 2) &&
109 (next_key != first_key));
Mickaël Salaün5f155c22017-02-10 00:21:42 +0100110 assert(bpf_map_get_next_key(fd, &next_key, &next_key) == -1 &&
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200111 errno == ENOENT);
112
113 /* Delete both elements. */
114 key = 1;
Mickaël Salaüne58383b2017-02-10 00:21:41 +0100115 assert(bpf_map_delete_elem(fd, &key) == 0);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200116 key = 2;
Mickaël Salaüne58383b2017-02-10 00:21:41 +0100117 assert(bpf_map_delete_elem(fd, &key) == 0);
118 assert(bpf_map_delete_elem(fd, &key) == -1 && errno == ENOENT);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200119
120 key = 0;
121 /* Check that map is empty. */
Teng Qin8fe45922017-04-24 19:00:37 -0700122 assert(bpf_map_get_next_key(fd, NULL, &next_key) == -1 &&
123 errno == ENOENT);
Mickaël Salaün5f155c22017-02-10 00:21:42 +0100124 assert(bpf_map_get_next_key(fd, &key, &next_key) == -1 &&
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200125 errno == ENOENT);
126
127 close(fd);
128}
129
Breno Leitaodd9cef42019-02-05 15:12:34 -0200130static void test_hashmap_sizes(unsigned int task, void *data)
Alexei Starovoitov8c290e62017-03-21 19:05:04 -0700131{
132 int fd, i, j;
133
134 for (i = 1; i <= 512; i <<= 1)
135 for (j = 1; j <= 1 << 18; j <<= 1) {
136 fd = bpf_create_map(BPF_MAP_TYPE_HASH, i, j,
137 2, map_flags);
138 if (fd < 0) {
Li Zhijian80475c42018-02-22 10:34:02 +0800139 if (errno == ENOMEM)
140 return;
Alexei Starovoitov8c290e62017-03-21 19:05:04 -0700141 printf("Failed to create hashmap key=%d value=%d '%s'\n",
142 i, j, strerror(errno));
143 exit(1);
144 }
145 close(fd);
146 usleep(10); /* give kernel time to destroy */
147 }
148}
149
Breno Leitaodd9cef42019-02-05 15:12:34 -0200150static void test_hashmap_percpu(unsigned int task, void *data)
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200151{
Daniel Borkmanne00c7b22016-11-26 01:28:09 +0100152 unsigned int nr_cpus = bpf_num_possible_cpus();
Daniel Borkmannf3515b52017-04-27 01:39:35 +0200153 BPF_DECLARE_PERCPU(long, value);
Teng Qin8fe45922017-04-24 19:00:37 -0700154 long long key, next_key, first_key;
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200155 int expected_key_mask = 0;
156 int fd, i;
157
Mickaël Salaünf4874d02017-02-10 00:21:43 +0100158 fd = bpf_create_map(BPF_MAP_TYPE_PERCPU_HASH, sizeof(key),
Daniel Borkmannf3515b52017-04-27 01:39:35 +0200159 sizeof(bpf_percpu(value, 0)), 2, map_flags);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200160 if (fd < 0) {
161 printf("Failed to create hashmap '%s'!\n", strerror(errno));
162 exit(1);
163 }
164
165 for (i = 0; i < nr_cpus; i++)
Daniel Borkmannf3515b52017-04-27 01:39:35 +0200166 bpf_percpu(value, i) = i + 100;
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200167
168 key = 1;
169 /* Insert key=1 element. */
170 assert(!(expected_key_mask & key));
Mickaël Salaün10ecc722017-02-10 00:21:39 +0100171 assert(bpf_map_update_elem(fd, &key, value, BPF_ANY) == 0);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200172 expected_key_mask |= key;
173
174 /* BPF_NOEXIST means add new element if it doesn't exist. */
Mickaël Salaün10ecc722017-02-10 00:21:39 +0100175 assert(bpf_map_update_elem(fd, &key, value, BPF_NOEXIST) == -1 &&
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200176 /* key=1 already exists. */
177 errno == EEXIST);
178
179 /* -1 is an invalid flag. */
Mickaël Salaün10ecc722017-02-10 00:21:39 +0100180 assert(bpf_map_update_elem(fd, &key, value, -1) == -1 &&
181 errno == EINVAL);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200182
183 /* Check that key=1 can be found. Value could be 0 if the lookup
184 * was run from a different CPU.
185 */
Daniel Borkmannf3515b52017-04-27 01:39:35 +0200186 bpf_percpu(value, 0) = 1;
187 assert(bpf_map_lookup_elem(fd, &key, value) == 0 &&
188 bpf_percpu(value, 0) == 100);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200189
190 key = 2;
191 /* Check that key=2 is not found. */
Mickaël Salaüne5ff7c42017-02-10 00:21:40 +0100192 assert(bpf_map_lookup_elem(fd, &key, value) == -1 && errno == ENOENT);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200193
194 /* BPF_EXIST means update existing element. */
Mickaël Salaün10ecc722017-02-10 00:21:39 +0100195 assert(bpf_map_update_elem(fd, &key, value, BPF_EXIST) == -1 &&
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200196 /* key=2 is not there. */
197 errno == ENOENT);
198
199 /* Insert key=2 element. */
200 assert(!(expected_key_mask & key));
Mickaël Salaün10ecc722017-02-10 00:21:39 +0100201 assert(bpf_map_update_elem(fd, &key, value, BPF_NOEXIST) == 0);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200202 expected_key_mask |= key;
203
204 /* key=1 and key=2 were inserted, check that key=0 cannot be
205 * inserted due to max_entries limit.
206 */
207 key = 0;
Mickaël Salaün10ecc722017-02-10 00:21:39 +0100208 assert(bpf_map_update_elem(fd, &key, value, BPF_NOEXIST) == -1 &&
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200209 errno == E2BIG);
210
211 /* Check that key = 0 doesn't exist. */
Mickaël Salaüne58383b2017-02-10 00:21:41 +0100212 assert(bpf_map_delete_elem(fd, &key) == -1 && errno == ENOENT);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200213
214 /* Iterate over two elements. */
Teng Qin8fe45922017-04-24 19:00:37 -0700215 assert(bpf_map_get_next_key(fd, NULL, &first_key) == 0 &&
216 ((expected_key_mask & first_key) == first_key));
Mickaël Salaün5f155c22017-02-10 00:21:42 +0100217 while (!bpf_map_get_next_key(fd, &key, &next_key)) {
Teng Qin8fe45922017-04-24 19:00:37 -0700218 if (first_key) {
219 assert(next_key == first_key);
220 first_key = 0;
221 }
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200222 assert((expected_key_mask & next_key) == next_key);
223 expected_key_mask &= ~next_key;
224
Mickaël Salaüne5ff7c42017-02-10 00:21:40 +0100225 assert(bpf_map_lookup_elem(fd, &next_key, value) == 0);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200226
227 for (i = 0; i < nr_cpus; i++)
Daniel Borkmannf3515b52017-04-27 01:39:35 +0200228 assert(bpf_percpu(value, i) == i + 100);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200229
230 key = next_key;
231 }
232 assert(errno == ENOENT);
233
234 /* Update with BPF_EXIST. */
235 key = 1;
Mickaël Salaün10ecc722017-02-10 00:21:39 +0100236 assert(bpf_map_update_elem(fd, &key, value, BPF_EXIST) == 0);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200237
238 /* Delete both elements. */
239 key = 1;
Mickaël Salaüne58383b2017-02-10 00:21:41 +0100240 assert(bpf_map_delete_elem(fd, &key) == 0);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200241 key = 2;
Mickaël Salaüne58383b2017-02-10 00:21:41 +0100242 assert(bpf_map_delete_elem(fd, &key) == 0);
243 assert(bpf_map_delete_elem(fd, &key) == -1 && errno == ENOENT);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200244
245 key = 0;
246 /* Check that map is empty. */
Teng Qin8fe45922017-04-24 19:00:37 -0700247 assert(bpf_map_get_next_key(fd, NULL, &next_key) == -1 &&
248 errno == ENOENT);
Mickaël Salaün5f155c22017-02-10 00:21:42 +0100249 assert(bpf_map_get_next_key(fd, &key, &next_key) == -1 &&
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200250 errno == ENOENT);
251
252 close(fd);
253}
254
Lorenz Bauerbf5d68c2018-11-16 11:41:11 +0000255static int helper_fill_hashmap(int max_entries)
256{
257 int i, fd, ret;
258 long long key, value;
259
260 fd = bpf_create_map(BPF_MAP_TYPE_HASH, sizeof(key), sizeof(value),
261 max_entries, map_flags);
262 CHECK(fd < 0,
263 "failed to create hashmap",
264 "err: %s, flags: 0x%x\n", strerror(errno), map_flags);
265
266 for (i = 0; i < max_entries; i++) {
267 key = i; value = key;
268 ret = bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST);
269 CHECK(ret != 0,
270 "can't update hashmap",
271 "err: %s\n", strerror(ret));
272 }
273
274 return fd;
275}
276
Breno Leitaodd9cef42019-02-05 15:12:34 -0200277static void test_hashmap_walk(unsigned int task, void *data)
Daniel Borkmann5ecf51f2017-06-11 00:50:44 +0200278{
Alexei Starovoitov1a97cf12018-01-22 17:46:57 -0800279 int fd, i, max_entries = 1000;
Daniel Borkmann5ecf51f2017-06-11 00:50:44 +0200280 long long key, value, next_key;
281 bool next_key_valid = true;
282
Lorenz Bauerbf5d68c2018-11-16 11:41:11 +0000283 fd = helper_fill_hashmap(max_entries);
Daniel Borkmann5ecf51f2017-06-11 00:50:44 +0200284
285 for (i = 0; bpf_map_get_next_key(fd, !i ? NULL : &key,
286 &next_key) == 0; i++) {
287 key = next_key;
288 assert(bpf_map_lookup_elem(fd, &key, &value) == 0);
289 }
290
291 assert(i == max_entries);
292
293 assert(bpf_map_get_next_key(fd, NULL, &key) == 0);
294 for (i = 0; next_key_valid; i++) {
295 next_key_valid = bpf_map_get_next_key(fd, &key, &next_key) == 0;
296 assert(bpf_map_lookup_elem(fd, &key, &value) == 0);
297 value++;
298 assert(bpf_map_update_elem(fd, &key, &value, BPF_EXIST) == 0);
299 key = next_key;
300 }
301
302 assert(i == max_entries);
303
304 for (i = 0; bpf_map_get_next_key(fd, !i ? NULL : &key,
305 &next_key) == 0; i++) {
306 key = next_key;
307 assert(bpf_map_lookup_elem(fd, &key, &value) == 0);
308 assert(value - 1 == key);
309 }
310
311 assert(i == max_entries);
312 close(fd);
313}
314
Lorenz Bauerbf5d68c2018-11-16 11:41:11 +0000315static void test_hashmap_zero_seed(void)
316{
317 int i, first, second, old_flags;
318 long long key, next_first, next_second;
319
320 old_flags = map_flags;
321 map_flags |= BPF_F_ZERO_SEED;
322
323 first = helper_fill_hashmap(3);
324 second = helper_fill_hashmap(3);
325
326 for (i = 0; ; i++) {
327 void *key_ptr = !i ? NULL : &key;
328
329 if (bpf_map_get_next_key(first, key_ptr, &next_first) != 0)
330 break;
331
332 CHECK(bpf_map_get_next_key(second, key_ptr, &next_second) != 0,
333 "next_key for second map must succeed",
334 "key_ptr: %p", key_ptr);
335 CHECK(next_first != next_second,
336 "keys must match",
337 "i: %d first: %lld second: %lld\n", i,
338 next_first, next_second);
339
340 key = next_first;
341 }
342
343 map_flags = old_flags;
344 close(first);
345 close(second);
346}
347
Breno Leitaodd9cef42019-02-05 15:12:34 -0200348static void test_arraymap(unsigned int task, void *data)
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200349{
350 int key, next_key, fd;
351 long long value;
352
Mickaël Salaünf4874d02017-02-10 00:21:43 +0100353 fd = bpf_create_map(BPF_MAP_TYPE_ARRAY, sizeof(key), sizeof(value),
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200354 2, 0);
355 if (fd < 0) {
356 printf("Failed to create arraymap '%s'!\n", strerror(errno));
357 exit(1);
358 }
359
360 key = 1;
361 value = 1234;
362 /* Insert key=1 element. */
Mickaël Salaün10ecc722017-02-10 00:21:39 +0100363 assert(bpf_map_update_elem(fd, &key, &value, BPF_ANY) == 0);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200364
365 value = 0;
Mickaël Salaün10ecc722017-02-10 00:21:39 +0100366 assert(bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST) == -1 &&
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200367 errno == EEXIST);
368
369 /* Check that key=1 can be found. */
Mickaël Salaüne5ff7c42017-02-10 00:21:40 +0100370 assert(bpf_map_lookup_elem(fd, &key, &value) == 0 && value == 1234);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200371
372 key = 0;
373 /* Check that key=0 is also found and zero initialized. */
Mickaël Salaüne5ff7c42017-02-10 00:21:40 +0100374 assert(bpf_map_lookup_elem(fd, &key, &value) == 0 && value == 0);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200375
376 /* key=0 and key=1 were inserted, check that key=2 cannot be inserted
377 * due to max_entries limit.
378 */
379 key = 2;
Mickaël Salaün10ecc722017-02-10 00:21:39 +0100380 assert(bpf_map_update_elem(fd, &key, &value, BPF_EXIST) == -1 &&
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200381 errno == E2BIG);
382
383 /* Check that key = 2 doesn't exist. */
Mickaël Salaüne5ff7c42017-02-10 00:21:40 +0100384 assert(bpf_map_lookup_elem(fd, &key, &value) == -1 && errno == ENOENT);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200385
386 /* Iterate over two elements. */
Teng Qin8fe45922017-04-24 19:00:37 -0700387 assert(bpf_map_get_next_key(fd, NULL, &next_key) == 0 &&
388 next_key == 0);
Mickaël Salaün5f155c22017-02-10 00:21:42 +0100389 assert(bpf_map_get_next_key(fd, &key, &next_key) == 0 &&
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200390 next_key == 0);
Mickaël Salaün5f155c22017-02-10 00:21:42 +0100391 assert(bpf_map_get_next_key(fd, &next_key, &next_key) == 0 &&
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200392 next_key == 1);
Mickaël Salaün5f155c22017-02-10 00:21:42 +0100393 assert(bpf_map_get_next_key(fd, &next_key, &next_key) == -1 &&
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200394 errno == ENOENT);
395
396 /* Delete shouldn't succeed. */
397 key = 1;
Mickaël Salaüne58383b2017-02-10 00:21:41 +0100398 assert(bpf_map_delete_elem(fd, &key) == -1 && errno == EINVAL);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200399
400 close(fd);
401}
402
Breno Leitaodd9cef42019-02-05 15:12:34 -0200403static void test_arraymap_percpu(unsigned int task, void *data)
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200404{
Daniel Borkmanne00c7b22016-11-26 01:28:09 +0100405 unsigned int nr_cpus = bpf_num_possible_cpus();
Daniel Borkmannf3515b52017-04-27 01:39:35 +0200406 BPF_DECLARE_PERCPU(long, values);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200407 int key, next_key, fd, i;
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200408
Mickaël Salaünf4874d02017-02-10 00:21:43 +0100409 fd = bpf_create_map(BPF_MAP_TYPE_PERCPU_ARRAY, sizeof(key),
Daniel Borkmannf3515b52017-04-27 01:39:35 +0200410 sizeof(bpf_percpu(values, 0)), 2, 0);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200411 if (fd < 0) {
412 printf("Failed to create arraymap '%s'!\n", strerror(errno));
413 exit(1);
414 }
415
416 for (i = 0; i < nr_cpus; i++)
Daniel Borkmannf3515b52017-04-27 01:39:35 +0200417 bpf_percpu(values, i) = i + 100;
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200418
419 key = 1;
420 /* Insert key=1 element. */
Mickaël Salaün10ecc722017-02-10 00:21:39 +0100421 assert(bpf_map_update_elem(fd, &key, values, BPF_ANY) == 0);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200422
Daniel Borkmannf3515b52017-04-27 01:39:35 +0200423 bpf_percpu(values, 0) = 0;
Mickaël Salaün10ecc722017-02-10 00:21:39 +0100424 assert(bpf_map_update_elem(fd, &key, values, BPF_NOEXIST) == -1 &&
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200425 errno == EEXIST);
426
427 /* Check that key=1 can be found. */
Daniel Borkmannf3515b52017-04-27 01:39:35 +0200428 assert(bpf_map_lookup_elem(fd, &key, values) == 0 &&
429 bpf_percpu(values, 0) == 100);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200430
431 key = 0;
432 /* Check that key=0 is also found and zero initialized. */
Mickaël Salaüne5ff7c42017-02-10 00:21:40 +0100433 assert(bpf_map_lookup_elem(fd, &key, values) == 0 &&
Daniel Borkmannf3515b52017-04-27 01:39:35 +0200434 bpf_percpu(values, 0) == 0 &&
435 bpf_percpu(values, nr_cpus - 1) == 0);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200436
437 /* Check that key=2 cannot be inserted due to max_entries limit. */
438 key = 2;
Mickaël Salaün10ecc722017-02-10 00:21:39 +0100439 assert(bpf_map_update_elem(fd, &key, values, BPF_EXIST) == -1 &&
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200440 errno == E2BIG);
441
442 /* Check that key = 2 doesn't exist. */
Mickaël Salaüne5ff7c42017-02-10 00:21:40 +0100443 assert(bpf_map_lookup_elem(fd, &key, values) == -1 && errno == ENOENT);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200444
445 /* Iterate over two elements. */
Teng Qin8fe45922017-04-24 19:00:37 -0700446 assert(bpf_map_get_next_key(fd, NULL, &next_key) == 0 &&
447 next_key == 0);
Mickaël Salaün5f155c22017-02-10 00:21:42 +0100448 assert(bpf_map_get_next_key(fd, &key, &next_key) == 0 &&
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200449 next_key == 0);
Mickaël Salaün5f155c22017-02-10 00:21:42 +0100450 assert(bpf_map_get_next_key(fd, &next_key, &next_key) == 0 &&
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200451 next_key == 1);
Mickaël Salaün5f155c22017-02-10 00:21:42 +0100452 assert(bpf_map_get_next_key(fd, &next_key, &next_key) == -1 &&
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200453 errno == ENOENT);
454
455 /* Delete shouldn't succeed. */
456 key = 1;
Mickaël Salaüne58383b2017-02-10 00:21:41 +0100457 assert(bpf_map_delete_elem(fd, &key) == -1 && errno == EINVAL);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200458
459 close(fd);
460}
461
462static void test_arraymap_percpu_many_keys(void)
463{
Daniel Borkmanne00c7b22016-11-26 01:28:09 +0100464 unsigned int nr_cpus = bpf_num_possible_cpus();
Daniel Borkmannf3515b52017-04-27 01:39:35 +0200465 BPF_DECLARE_PERCPU(long, values);
Alexei Starovoitov8c290e62017-03-21 19:05:04 -0700466 /* nr_keys is not too large otherwise the test stresses percpu
467 * allocator more than anything else
468 */
469 unsigned int nr_keys = 2000;
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200470 int key, fd, i;
471
Mickaël Salaünf4874d02017-02-10 00:21:43 +0100472 fd = bpf_create_map(BPF_MAP_TYPE_PERCPU_ARRAY, sizeof(key),
Daniel Borkmannf3515b52017-04-27 01:39:35 +0200473 sizeof(bpf_percpu(values, 0)), nr_keys, 0);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200474 if (fd < 0) {
475 printf("Failed to create per-cpu arraymap '%s'!\n",
476 strerror(errno));
477 exit(1);
478 }
479
480 for (i = 0; i < nr_cpus; i++)
Daniel Borkmannf3515b52017-04-27 01:39:35 +0200481 bpf_percpu(values, i) = i + 10;
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200482
483 for (key = 0; key < nr_keys; key++)
Mickaël Salaün10ecc722017-02-10 00:21:39 +0100484 assert(bpf_map_update_elem(fd, &key, values, BPF_ANY) == 0);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200485
486 for (key = 0; key < nr_keys; key++) {
487 for (i = 0; i < nr_cpus; i++)
Daniel Borkmannf3515b52017-04-27 01:39:35 +0200488 bpf_percpu(values, i) = 0;
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200489
Mickaël Salaüne5ff7c42017-02-10 00:21:40 +0100490 assert(bpf_map_lookup_elem(fd, &key, values) == 0);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200491
492 for (i = 0; i < nr_cpus; i++)
Daniel Borkmannf3515b52017-04-27 01:39:35 +0200493 assert(bpf_percpu(values, i) == i + 10);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +0200494 }
495
496 close(fd);
497}
498
Breno Leitaodd9cef42019-02-05 15:12:34 -0200499static void test_devmap(unsigned int task, void *data)
John Fastabend546ac1f2017-07-17 09:28:56 -0700500{
John Fastabend81f6bf82017-07-26 17:32:07 -0700501 int fd;
John Fastabend546ac1f2017-07-17 09:28:56 -0700502 __u32 key, value;
503
504 fd = bpf_create_map(BPF_MAP_TYPE_DEVMAP, sizeof(key), sizeof(value),
505 2, 0);
506 if (fd < 0) {
Xiaozhou Liu8b6b25c2018-12-21 17:35:11 +0800507 printf("Failed to create devmap '%s'!\n", strerror(errno));
John Fastabend546ac1f2017-07-17 09:28:56 -0700508 exit(1);
509 }
510
511 close(fd);
512}
513
Breno Leitaodd9cef42019-02-05 15:12:34 -0200514static void test_queuemap(unsigned int task, void *data)
Mauricio Vasquez B43b987d2018-10-18 15:16:41 +0200515{
516 const int MAP_SIZE = 32;
517 __u32 vals[MAP_SIZE + MAP_SIZE/2], val;
518 int fd, i;
519
520 /* Fill test values to be used */
521 for (i = 0; i < MAP_SIZE + MAP_SIZE/2; i++)
522 vals[i] = rand();
523
524 /* Invalid key size */
525 fd = bpf_create_map(BPF_MAP_TYPE_QUEUE, 4, sizeof(val), MAP_SIZE,
526 map_flags);
527 assert(fd < 0 && errno == EINVAL);
528
529 fd = bpf_create_map(BPF_MAP_TYPE_QUEUE, 0, sizeof(val), MAP_SIZE,
530 map_flags);
531 /* Queue map does not support BPF_F_NO_PREALLOC */
532 if (map_flags & BPF_F_NO_PREALLOC) {
533 assert(fd < 0 && errno == EINVAL);
534 return;
535 }
536 if (fd < 0) {
537 printf("Failed to create queuemap '%s'!\n", strerror(errno));
538 exit(1);
539 }
540
541 /* Push MAP_SIZE elements */
542 for (i = 0; i < MAP_SIZE; i++)
543 assert(bpf_map_update_elem(fd, NULL, &vals[i], 0) == 0);
544
545 /* Check that element cannot be pushed due to max_entries limit */
546 assert(bpf_map_update_elem(fd, NULL, &val, 0) == -1 &&
547 errno == E2BIG);
548
549 /* Peek element */
550 assert(bpf_map_lookup_elem(fd, NULL, &val) == 0 && val == vals[0]);
551
552 /* Replace half elements */
553 for (i = MAP_SIZE; i < MAP_SIZE + MAP_SIZE/2; i++)
554 assert(bpf_map_update_elem(fd, NULL, &vals[i], BPF_EXIST) == 0);
555
556 /* Pop all elements */
557 for (i = MAP_SIZE/2; i < MAP_SIZE + MAP_SIZE/2; i++)
558 assert(bpf_map_lookup_and_delete_elem(fd, NULL, &val) == 0 &&
559 val == vals[i]);
560
561 /* Check that there are not elements left */
562 assert(bpf_map_lookup_and_delete_elem(fd, NULL, &val) == -1 &&
563 errno == ENOENT);
564
565 /* Check that non supported functions set errno to EINVAL */
566 assert(bpf_map_delete_elem(fd, NULL) == -1 && errno == EINVAL);
567 assert(bpf_map_get_next_key(fd, NULL, NULL) == -1 && errno == EINVAL);
568
569 close(fd);
570}
571
Breno Leitaodd9cef42019-02-05 15:12:34 -0200572static void test_stackmap(unsigned int task, void *data)
Mauricio Vasquez B43b987d2018-10-18 15:16:41 +0200573{
574 const int MAP_SIZE = 32;
575 __u32 vals[MAP_SIZE + MAP_SIZE/2], val;
576 int fd, i;
577
578 /* Fill test values to be used */
579 for (i = 0; i < MAP_SIZE + MAP_SIZE/2; i++)
580 vals[i] = rand();
581
582 /* Invalid key size */
583 fd = bpf_create_map(BPF_MAP_TYPE_STACK, 4, sizeof(val), MAP_SIZE,
584 map_flags);
585 assert(fd < 0 && errno == EINVAL);
586
587 fd = bpf_create_map(BPF_MAP_TYPE_STACK, 0, sizeof(val), MAP_SIZE,
588 map_flags);
589 /* Stack map does not support BPF_F_NO_PREALLOC */
590 if (map_flags & BPF_F_NO_PREALLOC) {
591 assert(fd < 0 && errno == EINVAL);
592 return;
593 }
594 if (fd < 0) {
595 printf("Failed to create stackmap '%s'!\n", strerror(errno));
596 exit(1);
597 }
598
599 /* Push MAP_SIZE elements */
600 for (i = 0; i < MAP_SIZE; i++)
601 assert(bpf_map_update_elem(fd, NULL, &vals[i], 0) == 0);
602
603 /* Check that element cannot be pushed due to max_entries limit */
604 assert(bpf_map_update_elem(fd, NULL, &val, 0) == -1 &&
605 errno == E2BIG);
606
607 /* Peek element */
608 assert(bpf_map_lookup_elem(fd, NULL, &val) == 0 && val == vals[i - 1]);
609
610 /* Replace half elements */
611 for (i = MAP_SIZE; i < MAP_SIZE + MAP_SIZE/2; i++)
612 assert(bpf_map_update_elem(fd, NULL, &vals[i], BPF_EXIST) == 0);
613
614 /* Pop all elements */
615 for (i = MAP_SIZE + MAP_SIZE/2 - 1; i >= MAP_SIZE/2; i--)
616 assert(bpf_map_lookup_and_delete_elem(fd, NULL, &val) == 0 &&
617 val == vals[i]);
618
619 /* Check that there are not elements left */
620 assert(bpf_map_lookup_and_delete_elem(fd, NULL, &val) == -1 &&
621 errno == ENOENT);
622
623 /* Check that non supported functions set errno to EINVAL */
624 assert(bpf_map_delete_elem(fd, NULL) == -1 && errno == EINVAL);
625 assert(bpf_map_get_next_key(fd, NULL, NULL) == -1 && errno == EINVAL);
626
627 close(fd);
628}
629
John Fastabend6f6d33f2017-08-15 22:34:22 -0700630#include <sys/ioctl.h>
631#include <arpa/inet.h>
632#include <sys/select.h>
633#include <linux/err.h>
634#define SOCKMAP_PARSE_PROG "./sockmap_parse_prog.o"
635#define SOCKMAP_VERDICT_PROG "./sockmap_verdict_prog.o"
John Fastabend82a86162018-03-18 12:57:31 -0700636#define SOCKMAP_TCP_MSG_PROG "./sockmap_tcp_msg_prog.o"
Breno Leitaodd9cef42019-02-05 15:12:34 -0200637static void test_sockmap(unsigned int tasks, void *data)
John Fastabend6f6d33f2017-08-15 22:34:22 -0700638{
John Fastabend82a86162018-03-18 12:57:31 -0700639 struct bpf_map *bpf_map_rx, *bpf_map_tx, *bpf_map_msg, *bpf_map_break;
640 int map_fd_msg = 0, map_fd_rx = 0, map_fd_tx = 0, map_fd_break;
John Fastabend6f6d33f2017-08-15 22:34:22 -0700641 int ports[] = {50200, 50201, 50202, 50204};
John Fastabend435bf0d2017-10-18 07:10:15 -0700642 int err, i, fd, udp, sfd[6] = {0xdeadbeef};
John Fastabend6fd28862017-08-28 07:11:05 -0700643 u8 buf[20] = {0x0, 0x5, 0x3, 0x2, 0x1, 0x0};
John Fastabend82a86162018-03-18 12:57:31 -0700644 int parse_prog, verdict_prog, msg_prog;
John Fastabend6f6d33f2017-08-15 22:34:22 -0700645 struct sockaddr_in addr;
John Fastabend82a86162018-03-18 12:57:31 -0700646 int one = 1, s, sc, rc;
John Fastabend6f6d33f2017-08-15 22:34:22 -0700647 struct bpf_object *obj;
648 struct timeval to;
649 __u32 key, value;
John Fastabend3f0d6a12017-08-28 07:12:41 -0700650 pid_t pid[tasks];
John Fastabend6f6d33f2017-08-15 22:34:22 -0700651 fd_set w;
652
653 /* Create some sockets to use with sockmap */
654 for (i = 0; i < 2; i++) {
655 sfd[i] = socket(AF_INET, SOCK_STREAM, 0);
656 if (sfd[i] < 0)
657 goto out;
658 err = setsockopt(sfd[i], SOL_SOCKET, SO_REUSEADDR,
659 (char *)&one, sizeof(one));
660 if (err) {
661 printf("failed to setsockopt\n");
662 goto out;
663 }
664 err = ioctl(sfd[i], FIONBIO, (char *)&one);
665 if (err < 0) {
666 printf("failed to ioctl\n");
667 goto out;
668 }
669 memset(&addr, 0, sizeof(struct sockaddr_in));
670 addr.sin_family = AF_INET;
671 addr.sin_addr.s_addr = inet_addr("127.0.0.1");
672 addr.sin_port = htons(ports[i]);
673 err = bind(sfd[i], (struct sockaddr *)&addr, sizeof(addr));
674 if (err < 0) {
675 printf("failed to bind: err %i: %i:%i\n",
676 err, i, sfd[i]);
677 goto out;
678 }
679 err = listen(sfd[i], 32);
680 if (err < 0) {
Colin Ian King90774a92017-08-30 18:15:25 +0100681 printf("failed to listen\n");
John Fastabend6f6d33f2017-08-15 22:34:22 -0700682 goto out;
683 }
684 }
685
686 for (i = 2; i < 4; i++) {
687 sfd[i] = socket(AF_INET, SOCK_STREAM, 0);
688 if (sfd[i] < 0)
689 goto out;
690 err = setsockopt(sfd[i], SOL_SOCKET, SO_REUSEADDR,
691 (char *)&one, sizeof(one));
692 if (err) {
693 printf("set sock opt\n");
694 goto out;
695 }
696 memset(&addr, 0, sizeof(struct sockaddr_in));
697 addr.sin_family = AF_INET;
698 addr.sin_addr.s_addr = inet_addr("127.0.0.1");
699 addr.sin_port = htons(ports[i - 2]);
700 err = connect(sfd[i], (struct sockaddr *)&addr, sizeof(addr));
701 if (err) {
Colin Ian King90774a92017-08-30 18:15:25 +0100702 printf("failed to connect\n");
John Fastabend6f6d33f2017-08-15 22:34:22 -0700703 goto out;
704 }
705 }
706
707
708 for (i = 4; i < 6; i++) {
709 sfd[i] = accept(sfd[i - 4], NULL, NULL);
710 if (sfd[i] < 0) {
711 printf("accept failed\n");
712 goto out;
713 }
714 }
715
716 /* Test sockmap with connected sockets */
717 fd = bpf_create_map(BPF_MAP_TYPE_SOCKMAP,
718 sizeof(key), sizeof(value),
719 6, 0);
720 if (fd < 0) {
Stanislav Fomicheve8ddbfb42019-01-28 09:21:15 -0800721 if (!bpf_probe_map_type(BPF_MAP_TYPE_SOCKMAP, 0)) {
722 printf("%s SKIP (unsupported map type BPF_MAP_TYPE_SOCKMAP)\n",
723 __func__);
724 skips++;
725 for (i = 0; i < 6; i++)
726 close(sfd[i]);
727 return;
728 }
729
John Fastabend6f6d33f2017-08-15 22:34:22 -0700730 printf("Failed to create sockmap %i\n", fd);
731 goto out_sockmap;
732 }
733
John Fastabend435bf0d2017-10-18 07:10:15 -0700734 /* Test update with unsupported UDP socket */
735 udp = socket(AF_INET, SOCK_DGRAM, 0);
736 i = 0;
737 err = bpf_map_update_elem(fd, &i, &udp, BPF_ANY);
738 if (!err) {
739 printf("Failed socket SOCK_DGRAM allowed '%i:%i'\n",
740 i, udp);
741 goto out_sockmap;
742 }
743
John Fastabend464bc0f2017-08-28 07:10:04 -0700744 /* Test update without programs */
John Fastabend6f6d33f2017-08-15 22:34:22 -0700745 for (i = 0; i < 6; i++) {
746 err = bpf_map_update_elem(fd, &i, &sfd[i], BPF_ANY);
John Fastabend50280272018-09-18 09:01:54 -0700747 if (i < 2 && !err) {
748 printf("Allowed update sockmap '%i:%i' not in ESTABLISHED\n",
749 i, sfd[i]);
750 goto out_sockmap;
751 } else if (i >= 2 && err) {
John Fastabend464bc0f2017-08-28 07:10:04 -0700752 printf("Failed noprog update sockmap '%i:%i'\n",
John Fastabend6f6d33f2017-08-15 22:34:22 -0700753 i, sfd[i]);
754 goto out_sockmap;
755 }
756 }
757
John Fastabend5a67da22017-09-08 14:00:49 -0700758 /* Test attaching/detaching bad fds */
John Fastabend464bc0f2017-08-28 07:10:04 -0700759 err = bpf_prog_attach(-1, fd, BPF_SK_SKB_STREAM_PARSER, 0);
John Fastabend6f6d33f2017-08-15 22:34:22 -0700760 if (!err) {
John Fastabend464bc0f2017-08-28 07:10:04 -0700761 printf("Failed invalid parser prog attach\n");
762 goto out_sockmap;
763 }
764
765 err = bpf_prog_attach(-1, fd, BPF_SK_SKB_STREAM_VERDICT, 0);
766 if (!err) {
767 printf("Failed invalid verdict prog attach\n");
John Fastabend6f6d33f2017-08-15 22:34:22 -0700768 goto out_sockmap;
769 }
770
John Fastabend82a86162018-03-18 12:57:31 -0700771 err = bpf_prog_attach(-1, fd, BPF_SK_MSG_VERDICT, 0);
772 if (!err) {
773 printf("Failed invalid msg verdict prog attach\n");
774 goto out_sockmap;
775 }
776
John Fastabend5a67da22017-09-08 14:00:49 -0700777 err = bpf_prog_attach(-1, fd, __MAX_BPF_ATTACH_TYPE, 0);
778 if (!err) {
779 printf("Failed unknown prog attach\n");
780 goto out_sockmap;
781 }
782
783 err = bpf_prog_detach(fd, BPF_SK_SKB_STREAM_PARSER);
784 if (err) {
785 printf("Failed empty parser prog detach\n");
786 goto out_sockmap;
787 }
788
789 err = bpf_prog_detach(fd, BPF_SK_SKB_STREAM_VERDICT);
790 if (err) {
791 printf("Failed empty verdict prog detach\n");
792 goto out_sockmap;
793 }
794
John Fastabend82a86162018-03-18 12:57:31 -0700795 err = bpf_prog_detach(fd, BPF_SK_MSG_VERDICT);
796 if (err) {
797 printf("Failed empty msg verdict prog detach\n");
798 goto out_sockmap;
799 }
800
John Fastabend5a67da22017-09-08 14:00:49 -0700801 err = bpf_prog_detach(fd, __MAX_BPF_ATTACH_TYPE);
802 if (!err) {
803 printf("Detach invalid prog successful\n");
804 goto out_sockmap;
805 }
806
John Fastabend6f6d33f2017-08-15 22:34:22 -0700807 /* Load SK_SKB program and Attach */
808 err = bpf_prog_load(SOCKMAP_PARSE_PROG,
809 BPF_PROG_TYPE_SK_SKB, &obj, &parse_prog);
810 if (err) {
811 printf("Failed to load SK_SKB parse prog\n");
812 goto out_sockmap;
813 }
814
John Fastabend82a86162018-03-18 12:57:31 -0700815 err = bpf_prog_load(SOCKMAP_TCP_MSG_PROG,
816 BPF_PROG_TYPE_SK_MSG, &obj, &msg_prog);
817 if (err) {
818 printf("Failed to load SK_SKB msg prog\n");
819 goto out_sockmap;
820 }
821
John Fastabend6f6d33f2017-08-15 22:34:22 -0700822 err = bpf_prog_load(SOCKMAP_VERDICT_PROG,
823 BPF_PROG_TYPE_SK_SKB, &obj, &verdict_prog);
824 if (err) {
825 printf("Failed to load SK_SKB verdict prog\n");
826 goto out_sockmap;
827 }
828
John Fastabend6fd28862017-08-28 07:11:05 -0700829 bpf_map_rx = bpf_object__find_map_by_name(obj, "sock_map_rx");
830 if (IS_ERR(bpf_map_rx)) {
831 printf("Failed to load map rx from verdict prog\n");
John Fastabend6f6d33f2017-08-15 22:34:22 -0700832 goto out_sockmap;
833 }
834
John Fastabend6fd28862017-08-28 07:11:05 -0700835 map_fd_rx = bpf_map__fd(bpf_map_rx);
836 if (map_fd_rx < 0) {
John Fastabend82a86162018-03-18 12:57:31 -0700837 printf("Failed to get map rx fd\n");
John Fastabend6f6d33f2017-08-15 22:34:22 -0700838 goto out_sockmap;
839 }
840
John Fastabend6fd28862017-08-28 07:11:05 -0700841 bpf_map_tx = bpf_object__find_map_by_name(obj, "sock_map_tx");
842 if (IS_ERR(bpf_map_tx)) {
843 printf("Failed to load map tx from verdict prog\n");
844 goto out_sockmap;
845 }
846
847 map_fd_tx = bpf_map__fd(bpf_map_tx);
848 if (map_fd_tx < 0) {
849 printf("Failed to get map tx fd\n");
850 goto out_sockmap;
851 }
852
John Fastabend82a86162018-03-18 12:57:31 -0700853 bpf_map_msg = bpf_object__find_map_by_name(obj, "sock_map_msg");
854 if (IS_ERR(bpf_map_msg)) {
855 printf("Failed to load map msg from msg_verdict prog\n");
856 goto out_sockmap;
857 }
858
859 map_fd_msg = bpf_map__fd(bpf_map_msg);
860 if (map_fd_msg < 0) {
861 printf("Failed to get map msg fd\n");
862 goto out_sockmap;
863 }
864
John Fastabend81374aa2017-08-28 07:11:43 -0700865 bpf_map_break = bpf_object__find_map_by_name(obj, "sock_map_break");
866 if (IS_ERR(bpf_map_break)) {
867 printf("Failed to load map tx from verdict prog\n");
868 goto out_sockmap;
869 }
870
871 map_fd_break = bpf_map__fd(bpf_map_break);
872 if (map_fd_break < 0) {
873 printf("Failed to get map tx fd\n");
874 goto out_sockmap;
875 }
876
877 err = bpf_prog_attach(parse_prog, map_fd_break,
878 BPF_SK_SKB_STREAM_PARSER, 0);
879 if (!err) {
880 printf("Allowed attaching SK_SKB program to invalid map\n");
881 goto out_sockmap;
882 }
883
John Fastabend6fd28862017-08-28 07:11:05 -0700884 err = bpf_prog_attach(parse_prog, map_fd_rx,
John Fastabend464bc0f2017-08-28 07:10:04 -0700885 BPF_SK_SKB_STREAM_PARSER, 0);
John Fastabend6f6d33f2017-08-15 22:34:22 -0700886 if (err) {
John Fastabend81374aa2017-08-28 07:11:43 -0700887 printf("Failed stream parser bpf prog attach\n");
John Fastabend6f6d33f2017-08-15 22:34:22 -0700888 goto out_sockmap;
889 }
890
John Fastabend6fd28862017-08-28 07:11:05 -0700891 err = bpf_prog_attach(verdict_prog, map_fd_rx,
John Fastabend464bc0f2017-08-28 07:10:04 -0700892 BPF_SK_SKB_STREAM_VERDICT, 0);
893 if (err) {
John Fastabend81374aa2017-08-28 07:11:43 -0700894 printf("Failed stream verdict bpf prog attach\n");
John Fastabend464bc0f2017-08-28 07:10:04 -0700895 goto out_sockmap;
896 }
897
John Fastabend82a86162018-03-18 12:57:31 -0700898 err = bpf_prog_attach(msg_prog, map_fd_msg, BPF_SK_MSG_VERDICT, 0);
899 if (err) {
900 printf("Failed msg verdict bpf prog attach\n");
901 goto out_sockmap;
902 }
903
John Fastabend5a67da22017-09-08 14:00:49 -0700904 err = bpf_prog_attach(verdict_prog, map_fd_rx,
905 __MAX_BPF_ATTACH_TYPE, 0);
906 if (!err) {
907 printf("Attached unknown bpf prog\n");
908 goto out_sockmap;
909 }
910
John Fastabend464bc0f2017-08-28 07:10:04 -0700911 /* Test map update elem afterwards fd lives in fd and map_fd */
John Fastabend50280272018-09-18 09:01:54 -0700912 for (i = 2; i < 6; i++) {
John Fastabend6fd28862017-08-28 07:11:05 -0700913 err = bpf_map_update_elem(map_fd_rx, &i, &sfd[i], BPF_ANY);
John Fastabend6f6d33f2017-08-15 22:34:22 -0700914 if (err) {
John Fastabend6fd28862017-08-28 07:11:05 -0700915 printf("Failed map_fd_rx update sockmap %i '%i:%i'\n",
916 err, i, sfd[i]);
917 goto out_sockmap;
918 }
919 err = bpf_map_update_elem(map_fd_tx, &i, &sfd[i], BPF_ANY);
920 if (err) {
921 printf("Failed map_fd_tx update sockmap %i '%i:%i'\n",
John Fastabend6f6d33f2017-08-15 22:34:22 -0700922 err, i, sfd[i]);
923 goto out_sockmap;
924 }
925 }
926
927 /* Test map delete elem and remove send/recv sockets */
928 for (i = 2; i < 4; i++) {
John Fastabend6fd28862017-08-28 07:11:05 -0700929 err = bpf_map_delete_elem(map_fd_rx, &i);
John Fastabend6f6d33f2017-08-15 22:34:22 -0700930 if (err) {
John Fastabend6fd28862017-08-28 07:11:05 -0700931 printf("Failed delete sockmap rx %i '%i:%i'\n",
932 err, i, sfd[i]);
933 goto out_sockmap;
934 }
935 err = bpf_map_delete_elem(map_fd_tx, &i);
936 if (err) {
937 printf("Failed delete sockmap tx %i '%i:%i'\n",
John Fastabend6f6d33f2017-08-15 22:34:22 -0700938 err, i, sfd[i]);
939 goto out_sockmap;
940 }
941 }
942
John Fastabend82a86162018-03-18 12:57:31 -0700943 /* Put sfd[2] (sending fd below) into msg map to test sendmsg bpf */
944 i = 0;
945 err = bpf_map_update_elem(map_fd_msg, &i, &sfd[2], BPF_ANY);
946 if (err) {
947 printf("Failed map_fd_msg update sockmap %i\n", err);
948 goto out_sockmap;
949 }
950
John Fastabend6f6d33f2017-08-15 22:34:22 -0700951 /* Test map send/recv */
John Fastabend6fd28862017-08-28 07:11:05 -0700952 for (i = 0; i < 2; i++) {
953 buf[0] = i;
954 buf[1] = 0x5;
955 sc = send(sfd[2], buf, 20, 0);
956 if (sc < 0) {
957 printf("Failed sockmap send\n");
958 goto out_sockmap;
959 }
960
961 FD_ZERO(&w);
962 FD_SET(sfd[3], &w);
963 to.tv_sec = 1;
964 to.tv_usec = 0;
965 s = select(sfd[3] + 1, &w, NULL, NULL, &to);
966 if (s == -1) {
967 perror("Failed sockmap select()");
968 goto out_sockmap;
969 } else if (!s) {
970 printf("Failed sockmap unexpected timeout\n");
971 goto out_sockmap;
972 }
973
974 if (!FD_ISSET(sfd[3], &w)) {
975 printf("Failed sockmap select/recv\n");
976 goto out_sockmap;
977 }
978
979 rc = recv(sfd[3], buf, sizeof(buf), 0);
980 if (rc < 0) {
981 printf("Failed sockmap recv\n");
982 goto out_sockmap;
983 }
984 }
985
986 /* Negative null entry lookup from datapath should be dropped */
987 buf[0] = 1;
988 buf[1] = 12;
989 sc = send(sfd[2], buf, 20, 0);
John Fastabend6f6d33f2017-08-15 22:34:22 -0700990 if (sc < 0) {
991 printf("Failed sockmap send\n");
992 goto out_sockmap;
993 }
994
John Fastabend464bc0f2017-08-28 07:10:04 -0700995 /* Push fd into same slot */
John Fastabend6f6d33f2017-08-15 22:34:22 -0700996 i = 2;
997 err = bpf_map_update_elem(fd, &i, &sfd[i], BPF_NOEXIST);
998 if (!err) {
John Fastabend464bc0f2017-08-28 07:10:04 -0700999 printf("Failed allowed sockmap dup slot BPF_NOEXIST\n");
John Fastabend6f6d33f2017-08-15 22:34:22 -07001000 goto out_sockmap;
1001 }
1002
1003 err = bpf_map_update_elem(fd, &i, &sfd[i], BPF_ANY);
1004 if (err) {
John Fastabend464bc0f2017-08-28 07:10:04 -07001005 printf("Failed sockmap update new slot BPF_ANY\n");
John Fastabend6f6d33f2017-08-15 22:34:22 -07001006 goto out_sockmap;
1007 }
1008
1009 err = bpf_map_update_elem(fd, &i, &sfd[i], BPF_EXIST);
1010 if (err) {
John Fastabend464bc0f2017-08-28 07:10:04 -07001011 printf("Failed sockmap update new slot BPF_EXIST\n");
John Fastabend6f6d33f2017-08-15 22:34:22 -07001012 goto out_sockmap;
1013 }
1014
John Fastabend464bc0f2017-08-28 07:10:04 -07001015 /* Delete the elems without programs */
John Fastabend50280272018-09-18 09:01:54 -07001016 for (i = 2; i < 6; i++) {
John Fastabend464bc0f2017-08-28 07:10:04 -07001017 err = bpf_map_delete_elem(fd, &i);
1018 if (err) {
1019 printf("Failed delete sockmap %i '%i:%i'\n",
1020 err, i, sfd[i]);
1021 }
1022 }
1023
1024 /* Test having multiple maps open and set with programs on same fds */
1025 err = bpf_prog_attach(parse_prog, fd,
1026 BPF_SK_SKB_STREAM_PARSER, 0);
1027 if (err) {
1028 printf("Failed fd bpf parse prog attach\n");
1029 goto out_sockmap;
1030 }
1031 err = bpf_prog_attach(verdict_prog, fd,
1032 BPF_SK_SKB_STREAM_VERDICT, 0);
1033 if (err) {
1034 printf("Failed fd bpf verdict prog attach\n");
John Fastabend6f6d33f2017-08-15 22:34:22 -07001035 goto out_sockmap;
1036 }
1037
John Fastabend464bc0f2017-08-28 07:10:04 -07001038 for (i = 4; i < 6; i++) {
1039 err = bpf_map_update_elem(fd, &i, &sfd[i], BPF_ANY);
1040 if (!err) {
1041 printf("Failed allowed duplicate programs in update ANY sockmap %i '%i:%i'\n",
1042 err, i, sfd[i]);
1043 goto out_sockmap;
1044 }
1045 err = bpf_map_update_elem(fd, &i, &sfd[i], BPF_NOEXIST);
1046 if (!err) {
1047 printf("Failed allowed duplicate program in update NOEXIST sockmap %i '%i:%i'\n",
1048 err, i, sfd[i]);
1049 goto out_sockmap;
1050 }
1051 err = bpf_map_update_elem(fd, &i, &sfd[i], BPF_EXIST);
1052 if (!err) {
1053 printf("Failed allowed duplicate program in update EXIST sockmap %i '%i:%i'\n",
1054 err, i, sfd[i]);
1055 goto out_sockmap;
1056 }
John Fastabend6f6d33f2017-08-15 22:34:22 -07001057 }
1058
John Fastabend3f0d6a12017-08-28 07:12:41 -07001059 /* Test tasks number of forked operations */
1060 for (i = 0; i < tasks; i++) {
1061 pid[i] = fork();
1062 if (pid[i] == 0) {
1063 for (i = 0; i < 6; i++) {
1064 bpf_map_delete_elem(map_fd_tx, &i);
1065 bpf_map_delete_elem(map_fd_rx, &i);
1066 bpf_map_update_elem(map_fd_tx, &i,
1067 &sfd[i], BPF_ANY);
1068 bpf_map_update_elem(map_fd_rx, &i,
1069 &sfd[i], BPF_ANY);
1070 }
1071 exit(0);
1072 } else if (pid[i] == -1) {
1073 printf("Couldn't spawn #%d process!\n", i);
1074 exit(1);
1075 }
1076 }
1077
1078 for (i = 0; i < tasks; i++) {
1079 int status;
1080
1081 assert(waitpid(pid[i], &status, 0) == pid[i]);
1082 assert(status == 0);
1083 }
1084
John Fastabend5a67da22017-09-08 14:00:49 -07001085 err = bpf_prog_detach(map_fd_rx, __MAX_BPF_ATTACH_TYPE);
1086 if (!err) {
1087 printf("Detached an invalid prog type.\n");
1088 goto out_sockmap;
1089 }
1090
1091 err = bpf_prog_detach(map_fd_rx, BPF_SK_SKB_STREAM_PARSER);
1092 if (err) {
1093 printf("Failed parser prog detach\n");
1094 goto out_sockmap;
1095 }
1096
1097 err = bpf_prog_detach(map_fd_rx, BPF_SK_SKB_STREAM_VERDICT);
1098 if (err) {
1099 printf("Failed parser prog detach\n");
1100 goto out_sockmap;
1101 }
1102
Prashant Bhole78368782018-01-23 13:30:44 +09001103 /* Test map close sockets and empty maps */
1104 for (i = 0; i < 6; i++) {
1105 bpf_map_delete_elem(map_fd_tx, &i);
1106 bpf_map_delete_elem(map_fd_rx, &i);
John Fastabend6f6d33f2017-08-15 22:34:22 -07001107 close(sfd[i]);
Prashant Bhole78368782018-01-23 13:30:44 +09001108 }
John Fastabend6f6d33f2017-08-15 22:34:22 -07001109 close(fd);
John Fastabend6fd28862017-08-28 07:11:05 -07001110 close(map_fd_rx);
John Fastabend6f6d33f2017-08-15 22:34:22 -07001111 bpf_object__close(obj);
1112 return;
1113out:
1114 for (i = 0; i < 6; i++)
1115 close(sfd[i]);
1116 printf("Failed to create sockmap '%i:%s'!\n", i, strerror(errno));
1117 exit(1);
1118out_sockmap:
Prashant Bhole78368782018-01-23 13:30:44 +09001119 for (i = 0; i < 6; i++) {
1120 if (map_fd_tx)
1121 bpf_map_delete_elem(map_fd_tx, &i);
1122 if (map_fd_rx)
1123 bpf_map_delete_elem(map_fd_rx, &i);
John Fastabend6f6d33f2017-08-15 22:34:22 -07001124 close(sfd[i]);
Prashant Bhole78368782018-01-23 13:30:44 +09001125 }
John Fastabend6f6d33f2017-08-15 22:34:22 -07001126 close(fd);
1127 exit(1);
1128}
1129
Nikita V. Shirokovb1957c92018-11-20 20:55:57 -08001130#define MAPINMAP_PROG "./test_map_in_map.o"
1131static void test_map_in_map(void)
1132{
1133 struct bpf_program *prog;
1134 struct bpf_object *obj;
1135 struct bpf_map *map;
1136 int mim_fd, fd, err;
1137 int pos = 0;
1138
1139 obj = bpf_object__open(MAPINMAP_PROG);
1140
1141 fd = bpf_create_map(BPF_MAP_TYPE_HASH, sizeof(int), sizeof(int),
1142 2, 0);
1143 if (fd < 0) {
1144 printf("Failed to create hashmap '%s'!\n", strerror(errno));
1145 exit(1);
1146 }
1147
1148 map = bpf_object__find_map_by_name(obj, "mim_array");
1149 if (IS_ERR(map)) {
1150 printf("Failed to load array of maps from test prog\n");
1151 goto out_map_in_map;
1152 }
1153 err = bpf_map__set_inner_map_fd(map, fd);
1154 if (err) {
1155 printf("Failed to set inner_map_fd for array of maps\n");
1156 goto out_map_in_map;
1157 }
1158
1159 map = bpf_object__find_map_by_name(obj, "mim_hash");
1160 if (IS_ERR(map)) {
1161 printf("Failed to load hash of maps from test prog\n");
1162 goto out_map_in_map;
1163 }
1164 err = bpf_map__set_inner_map_fd(map, fd);
1165 if (err) {
1166 printf("Failed to set inner_map_fd for hash of maps\n");
1167 goto out_map_in_map;
1168 }
1169
1170 bpf_object__for_each_program(prog, obj) {
1171 bpf_program__set_xdp(prog);
1172 }
1173 bpf_object__load(obj);
1174
1175 map = bpf_object__find_map_by_name(obj, "mim_array");
1176 if (IS_ERR(map)) {
1177 printf("Failed to load array of maps from test prog\n");
1178 goto out_map_in_map;
1179 }
1180 mim_fd = bpf_map__fd(map);
1181 if (mim_fd < 0) {
1182 printf("Failed to get descriptor for array of maps\n");
1183 goto out_map_in_map;
1184 }
1185
1186 err = bpf_map_update_elem(mim_fd, &pos, &fd, 0);
1187 if (err) {
1188 printf("Failed to update array of maps\n");
1189 goto out_map_in_map;
1190 }
1191
1192 map = bpf_object__find_map_by_name(obj, "mim_hash");
1193 if (IS_ERR(map)) {
1194 printf("Failed to load hash of maps from test prog\n");
1195 goto out_map_in_map;
1196 }
1197 mim_fd = bpf_map__fd(map);
1198 if (mim_fd < 0) {
1199 printf("Failed to get descriptor for hash of maps\n");
1200 goto out_map_in_map;
1201 }
1202
1203 err = bpf_map_update_elem(mim_fd, &pos, &fd, 0);
1204 if (err) {
1205 printf("Failed to update hash of maps\n");
1206 goto out_map_in_map;
1207 }
1208
1209 close(fd);
1210 bpf_object__close(obj);
1211 return;
1212
1213out_map_in_map:
1214 close(fd);
1215 exit(1);
1216}
1217
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +02001218#define MAP_SIZE (32 * 1024)
1219
1220static void test_map_large(void)
1221{
1222 struct bigkey {
1223 int a;
1224 char b[116];
1225 long long c;
1226 } key;
1227 int fd, i, value;
1228
Mickaël Salaünf4874d02017-02-10 00:21:43 +01001229 fd = bpf_create_map(BPF_MAP_TYPE_HASH, sizeof(key), sizeof(value),
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +02001230 MAP_SIZE, map_flags);
1231 if (fd < 0) {
1232 printf("Failed to create large map '%s'!\n", strerror(errno));
1233 exit(1);
1234 }
1235
1236 for (i = 0; i < MAP_SIZE; i++) {
1237 key = (struct bigkey) { .c = i };
1238 value = i;
1239
Mickaël Salaün10ecc722017-02-10 00:21:39 +01001240 assert(bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST) == 0);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +02001241 }
1242
1243 key.c = -1;
Mickaël Salaün10ecc722017-02-10 00:21:39 +01001244 assert(bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST) == -1 &&
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +02001245 errno == E2BIG);
1246
1247 /* Iterate through all elements. */
Teng Qin8fe45922017-04-24 19:00:37 -07001248 assert(bpf_map_get_next_key(fd, NULL, &key) == 0);
1249 key.c = -1;
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +02001250 for (i = 0; i < MAP_SIZE; i++)
Mickaël Salaün5f155c22017-02-10 00:21:42 +01001251 assert(bpf_map_get_next_key(fd, &key, &key) == 0);
1252 assert(bpf_map_get_next_key(fd, &key, &key) == -1 && errno == ENOENT);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +02001253
1254 key.c = 0;
Mickaël Salaüne5ff7c42017-02-10 00:21:40 +01001255 assert(bpf_map_lookup_elem(fd, &key, &value) == 0 && value == 0);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +02001256 key.a = 1;
Mickaël Salaüne5ff7c42017-02-10 00:21:40 +01001257 assert(bpf_map_lookup_elem(fd, &key, &value) == -1 && errno == ENOENT);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +02001258
1259 close(fd);
1260}
1261
Alexei Starovoitov1a97cf12018-01-22 17:46:57 -08001262#define run_parallel(N, FN, DATA) \
Breno Leitaodd9cef42019-02-05 15:12:34 -02001263 printf("Fork %u tasks to '" #FN "'\n", N); \
Alexei Starovoitov1a97cf12018-01-22 17:46:57 -08001264 __run_parallel(N, FN, DATA)
1265
Breno Leitaodd9cef42019-02-05 15:12:34 -02001266static void __run_parallel(unsigned int tasks,
1267 void (*fn)(unsigned int task, void *data),
Alexei Starovoitov1a97cf12018-01-22 17:46:57 -08001268 void *data)
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +02001269{
1270 pid_t pid[tasks];
1271 int i;
1272
1273 for (i = 0; i < tasks; i++) {
1274 pid[i] = fork();
1275 if (pid[i] == 0) {
1276 fn(i, data);
1277 exit(0);
1278 } else if (pid[i] == -1) {
1279 printf("Couldn't spawn #%d process!\n", i);
1280 exit(1);
1281 }
1282 }
1283
1284 for (i = 0; i < tasks; i++) {
1285 int status;
1286
1287 assert(waitpid(pid[i], &status, 0) == pid[i]);
1288 assert(status == 0);
1289 }
1290}
1291
1292static void test_map_stress(void)
1293{
1294 run_parallel(100, test_hashmap, NULL);
1295 run_parallel(100, test_hashmap_percpu, NULL);
Alexei Starovoitov8c290e62017-03-21 19:05:04 -07001296 run_parallel(100, test_hashmap_sizes, NULL);
Daniel Borkmann5ecf51f2017-06-11 00:50:44 +02001297 run_parallel(100, test_hashmap_walk, NULL);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +02001298
1299 run_parallel(100, test_arraymap, NULL);
1300 run_parallel(100, test_arraymap_percpu, NULL);
1301}
1302
1303#define TASKS 1024
1304
1305#define DO_UPDATE 1
1306#define DO_DELETE 0
1307
Breno Leitaodd9cef42019-02-05 15:12:34 -02001308static void test_update_delete(unsigned int fn, void *data)
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +02001309{
1310 int do_update = ((int *)data)[1];
1311 int fd = ((int *)data)[0];
1312 int i, key, value;
1313
1314 for (i = fn; i < MAP_SIZE; i += TASKS) {
1315 key = value = i;
1316
1317 if (do_update) {
Mickaël Salaün10ecc722017-02-10 00:21:39 +01001318 assert(bpf_map_update_elem(fd, &key, &value,
1319 BPF_NOEXIST) == 0);
1320 assert(bpf_map_update_elem(fd, &key, &value,
1321 BPF_EXIST) == 0);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +02001322 } else {
Mickaël Salaüne58383b2017-02-10 00:21:41 +01001323 assert(bpf_map_delete_elem(fd, &key) == 0);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +02001324 }
1325 }
1326}
1327
1328static void test_map_parallel(void)
1329{
1330 int i, fd, key = 0, value = 0;
1331 int data[2];
1332
Mickaël Salaünf4874d02017-02-10 00:21:43 +01001333 fd = bpf_create_map(BPF_MAP_TYPE_HASH, sizeof(key), sizeof(value),
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +02001334 MAP_SIZE, map_flags);
1335 if (fd < 0) {
1336 printf("Failed to create map for parallel test '%s'!\n",
1337 strerror(errno));
1338 exit(1);
1339 }
1340
1341 /* Use the same fd in children to add elements to this map:
1342 * child_0 adds key=0, key=1024, key=2048, ...
1343 * child_1 adds key=1, key=1025, key=2049, ...
1344 * child_1023 adds key=1023, ...
1345 */
1346 data[0] = fd;
1347 data[1] = DO_UPDATE;
Alexei Starovoitov1a97cf12018-01-22 17:46:57 -08001348 run_parallel(TASKS, test_update_delete, data);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +02001349
1350 /* Check that key=0 is already there. */
Mickaël Salaün10ecc722017-02-10 00:21:39 +01001351 assert(bpf_map_update_elem(fd, &key, &value, BPF_NOEXIST) == -1 &&
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +02001352 errno == EEXIST);
1353
1354 /* Check that all elements were inserted. */
Teng Qin8fe45922017-04-24 19:00:37 -07001355 assert(bpf_map_get_next_key(fd, NULL, &key) == 0);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +02001356 key = -1;
1357 for (i = 0; i < MAP_SIZE; i++)
Mickaël Salaün5f155c22017-02-10 00:21:42 +01001358 assert(bpf_map_get_next_key(fd, &key, &key) == 0);
1359 assert(bpf_map_get_next_key(fd, &key, &key) == -1 && errno == ENOENT);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +02001360
1361 /* Another check for all elements */
1362 for (i = 0; i < MAP_SIZE; i++) {
1363 key = MAP_SIZE - i - 1;
1364
Mickaël Salaüne5ff7c42017-02-10 00:21:40 +01001365 assert(bpf_map_lookup_elem(fd, &key, &value) == 0 &&
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +02001366 value == key);
1367 }
1368
1369 /* Now let's delete all elemenets in parallel. */
1370 data[1] = DO_DELETE;
Alexei Starovoitov1a97cf12018-01-22 17:46:57 -08001371 run_parallel(TASKS, test_update_delete, data);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +02001372
1373 /* Nothing should be left. */
1374 key = -1;
Teng Qin8fe45922017-04-24 19:00:37 -07001375 assert(bpf_map_get_next_key(fd, NULL, &key) == -1 && errno == ENOENT);
Mickaël Salaün5f155c22017-02-10 00:21:42 +01001376 assert(bpf_map_get_next_key(fd, &key, &key) == -1 && errno == ENOENT);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +02001377}
1378
Chenbo Fenge0433252017-10-18 13:00:23 -07001379static void test_map_rdonly(void)
1380{
Alexei Starovoitove27afb82017-10-22 10:29:06 -07001381 int fd, key = 0, value = 0;
Chenbo Fenge0433252017-10-18 13:00:23 -07001382
1383 fd = bpf_create_map(BPF_MAP_TYPE_HASH, sizeof(key), sizeof(value),
1384 MAP_SIZE, map_flags | BPF_F_RDONLY);
1385 if (fd < 0) {
1386 printf("Failed to create map for read only test '%s'!\n",
1387 strerror(errno));
1388 exit(1);
1389 }
1390
1391 key = 1;
1392 value = 1234;
1393 /* Insert key=1 element. */
1394 assert(bpf_map_update_elem(fd, &key, &value, BPF_ANY) == -1 &&
1395 errno == EPERM);
1396
1397 /* Check that key=2 is not found. */
1398 assert(bpf_map_lookup_elem(fd, &key, &value) == -1 && errno == ENOENT);
1399 assert(bpf_map_get_next_key(fd, &key, &value) == -1 && errno == ENOENT);
1400}
1401
1402static void test_map_wronly(void)
1403{
Alexei Starovoitove27afb82017-10-22 10:29:06 -07001404 int fd, key = 0, value = 0;
Chenbo Fenge0433252017-10-18 13:00:23 -07001405
1406 fd = bpf_create_map(BPF_MAP_TYPE_HASH, sizeof(key), sizeof(value),
1407 MAP_SIZE, map_flags | BPF_F_WRONLY);
1408 if (fd < 0) {
1409 printf("Failed to create map for read only test '%s'!\n",
1410 strerror(errno));
1411 exit(1);
1412 }
1413
1414 key = 1;
1415 value = 1234;
1416 /* Insert key=1 element. */
Alexei Starovoitove27afb82017-10-22 10:29:06 -07001417 assert(bpf_map_update_elem(fd, &key, &value, BPF_ANY) == 0);
Chenbo Fenge0433252017-10-18 13:00:23 -07001418
1419 /* Check that key=2 is not found. */
1420 assert(bpf_map_lookup_elem(fd, &key, &value) == -1 && errno == EPERM);
1421 assert(bpf_map_get_next_key(fd, &key, &value) == -1 && errno == EPERM);
1422}
1423
Martin KaFai Lau6bc8529c2018-08-08 01:01:30 -07001424static void prepare_reuseport_grp(int type, int map_fd,
1425 __s64 *fds64, __u64 *sk_cookies,
1426 unsigned int n)
1427{
1428 socklen_t optlen, addrlen;
1429 struct sockaddr_in6 s6;
1430 const __u32 index0 = 0;
1431 const int optval = 1;
1432 unsigned int i;
1433 u64 sk_cookie;
1434 __s64 fd64;
1435 int err;
1436
1437 s6.sin6_family = AF_INET6;
1438 s6.sin6_addr = in6addr_any;
1439 s6.sin6_port = 0;
1440 addrlen = sizeof(s6);
1441 optlen = sizeof(sk_cookie);
1442
1443 for (i = 0; i < n; i++) {
1444 fd64 = socket(AF_INET6, type, 0);
1445 CHECK(fd64 == -1, "socket()",
1446 "sock_type:%d fd64:%lld errno:%d\n",
1447 type, fd64, errno);
1448
1449 err = setsockopt(fd64, SOL_SOCKET, SO_REUSEPORT,
1450 &optval, sizeof(optval));
Colin Ian King26a1ccc2018-08-13 15:00:32 +01001451 CHECK(err == -1, "setsockopt(SO_REUSEPORT)",
Martin KaFai Lau6bc8529c2018-08-08 01:01:30 -07001452 "err:%d errno:%d\n", err, errno);
1453
1454 /* reuseport_array does not allow unbound sk */
1455 err = bpf_map_update_elem(map_fd, &index0, &fd64,
1456 BPF_ANY);
1457 CHECK(err != -1 || errno != EINVAL,
1458 "reuseport array update unbound sk",
1459 "sock_type:%d err:%d errno:%d\n",
1460 type, err, errno);
1461
1462 err = bind(fd64, (struct sockaddr *)&s6, sizeof(s6));
1463 CHECK(err == -1, "bind()",
1464 "sock_type:%d err:%d errno:%d\n", type, err, errno);
1465
1466 if (i == 0) {
1467 err = getsockname(fd64, (struct sockaddr *)&s6,
1468 &addrlen);
1469 CHECK(err == -1, "getsockname()",
1470 "sock_type:%d err:%d errno:%d\n",
1471 type, err, errno);
1472 }
1473
1474 err = getsockopt(fd64, SOL_SOCKET, SO_COOKIE, &sk_cookie,
1475 &optlen);
1476 CHECK(err == -1, "getsockopt(SO_COOKIE)",
1477 "sock_type:%d err:%d errno:%d\n", type, err, errno);
1478
1479 if (type == SOCK_STREAM) {
1480 /*
1481 * reuseport_array does not allow
1482 * non-listening tcp sk.
1483 */
1484 err = bpf_map_update_elem(map_fd, &index0, &fd64,
1485 BPF_ANY);
1486 CHECK(err != -1 || errno != EINVAL,
1487 "reuseport array update non-listening sk",
1488 "sock_type:%d err:%d errno:%d\n",
1489 type, err, errno);
1490 err = listen(fd64, 0);
1491 CHECK(err == -1, "listen()",
1492 "sock_type:%d, err:%d errno:%d\n",
1493 type, err, errno);
1494 }
1495
1496 fds64[i] = fd64;
1497 sk_cookies[i] = sk_cookie;
1498 }
1499}
1500
1501static void test_reuseport_array(void)
1502{
1503#define REUSEPORT_FD_IDX(err, last) ({ (err) ? last : !last; })
1504
1505 const __u32 array_size = 4, index0 = 0, index3 = 3;
1506 int types[2] = { SOCK_STREAM, SOCK_DGRAM }, type;
1507 __u64 grpa_cookies[2], sk_cookie, map_cookie;
1508 __s64 grpa_fds64[2] = { -1, -1 }, fd64 = -1;
1509 const __u32 bad_index = array_size;
1510 int map_fd, err, t, f;
1511 __u32 fds_idx = 0;
1512 int fd;
1513
1514 map_fd = bpf_create_map(BPF_MAP_TYPE_REUSEPORT_SOCKARRAY,
1515 sizeof(__u32), sizeof(__u64), array_size, 0);
1516 CHECK(map_fd == -1, "reuseport array create",
1517 "map_fd:%d, errno:%d\n", map_fd, errno);
1518
1519 /* Test lookup/update/delete with invalid index */
1520 err = bpf_map_delete_elem(map_fd, &bad_index);
1521 CHECK(err != -1 || errno != E2BIG, "reuseport array del >=max_entries",
1522 "err:%d errno:%d\n", err, errno);
1523
1524 err = bpf_map_update_elem(map_fd, &bad_index, &fd64, BPF_ANY);
1525 CHECK(err != -1 || errno != E2BIG,
1526 "reuseport array update >=max_entries",
1527 "err:%d errno:%d\n", err, errno);
1528
1529 err = bpf_map_lookup_elem(map_fd, &bad_index, &map_cookie);
1530 CHECK(err != -1 || errno != ENOENT,
1531 "reuseport array update >=max_entries",
1532 "err:%d errno:%d\n", err, errno);
1533
1534 /* Test lookup/delete non existence elem */
1535 err = bpf_map_lookup_elem(map_fd, &index3, &map_cookie);
1536 CHECK(err != -1 || errno != ENOENT,
1537 "reuseport array lookup not-exist elem",
1538 "err:%d errno:%d\n", err, errno);
1539 err = bpf_map_delete_elem(map_fd, &index3);
1540 CHECK(err != -1 || errno != ENOENT,
1541 "reuseport array del not-exist elem",
1542 "err:%d errno:%d\n", err, errno);
1543
1544 for (t = 0; t < ARRAY_SIZE(types); t++) {
1545 type = types[t];
1546
1547 prepare_reuseport_grp(type, map_fd, grpa_fds64,
1548 grpa_cookies, ARRAY_SIZE(grpa_fds64));
1549
1550 /* Test BPF_* update flags */
1551 /* BPF_EXIST failure case */
1552 err = bpf_map_update_elem(map_fd, &index3, &grpa_fds64[fds_idx],
1553 BPF_EXIST);
1554 CHECK(err != -1 || errno != ENOENT,
1555 "reuseport array update empty elem BPF_EXIST",
1556 "sock_type:%d err:%d errno:%d\n",
1557 type, err, errno);
1558 fds_idx = REUSEPORT_FD_IDX(err, fds_idx);
1559
1560 /* BPF_NOEXIST success case */
1561 err = bpf_map_update_elem(map_fd, &index3, &grpa_fds64[fds_idx],
1562 BPF_NOEXIST);
1563 CHECK(err == -1,
1564 "reuseport array update empty elem BPF_NOEXIST",
1565 "sock_type:%d err:%d errno:%d\n",
1566 type, err, errno);
1567 fds_idx = REUSEPORT_FD_IDX(err, fds_idx);
1568
1569 /* BPF_EXIST success case. */
1570 err = bpf_map_update_elem(map_fd, &index3, &grpa_fds64[fds_idx],
1571 BPF_EXIST);
1572 CHECK(err == -1,
1573 "reuseport array update same elem BPF_EXIST",
1574 "sock_type:%d err:%d errno:%d\n", type, err, errno);
1575 fds_idx = REUSEPORT_FD_IDX(err, fds_idx);
1576
1577 /* BPF_NOEXIST failure case */
1578 err = bpf_map_update_elem(map_fd, &index3, &grpa_fds64[fds_idx],
1579 BPF_NOEXIST);
1580 CHECK(err != -1 || errno != EEXIST,
1581 "reuseport array update non-empty elem BPF_NOEXIST",
1582 "sock_type:%d err:%d errno:%d\n",
1583 type, err, errno);
1584 fds_idx = REUSEPORT_FD_IDX(err, fds_idx);
1585
1586 /* BPF_ANY case (always succeed) */
1587 err = bpf_map_update_elem(map_fd, &index3, &grpa_fds64[fds_idx],
1588 BPF_ANY);
1589 CHECK(err == -1,
1590 "reuseport array update same sk with BPF_ANY",
1591 "sock_type:%d err:%d errno:%d\n", type, err, errno);
1592
1593 fd64 = grpa_fds64[fds_idx];
1594 sk_cookie = grpa_cookies[fds_idx];
1595
1596 /* The same sk cannot be added to reuseport_array twice */
1597 err = bpf_map_update_elem(map_fd, &index3, &fd64, BPF_ANY);
1598 CHECK(err != -1 || errno != EBUSY,
1599 "reuseport array update same sk with same index",
1600 "sock_type:%d err:%d errno:%d\n",
1601 type, err, errno);
1602
1603 err = bpf_map_update_elem(map_fd, &index0, &fd64, BPF_ANY);
1604 CHECK(err != -1 || errno != EBUSY,
1605 "reuseport array update same sk with different index",
1606 "sock_type:%d err:%d errno:%d\n",
1607 type, err, errno);
1608
1609 /* Test delete elem */
1610 err = bpf_map_delete_elem(map_fd, &index3);
1611 CHECK(err == -1, "reuseport array delete sk",
1612 "sock_type:%d err:%d errno:%d\n",
1613 type, err, errno);
1614
1615 /* Add it back with BPF_NOEXIST */
1616 err = bpf_map_update_elem(map_fd, &index3, &fd64, BPF_NOEXIST);
1617 CHECK(err == -1,
1618 "reuseport array re-add with BPF_NOEXIST after del",
1619 "sock_type:%d err:%d errno:%d\n", type, err, errno);
1620
1621 /* Test cookie */
1622 err = bpf_map_lookup_elem(map_fd, &index3, &map_cookie);
1623 CHECK(err == -1 || sk_cookie != map_cookie,
1624 "reuseport array lookup re-added sk",
1625 "sock_type:%d err:%d errno:%d sk_cookie:0x%llx map_cookie:0x%llxn",
1626 type, err, errno, sk_cookie, map_cookie);
1627
1628 /* Test elem removed by close() */
1629 for (f = 0; f < ARRAY_SIZE(grpa_fds64); f++)
1630 close(grpa_fds64[f]);
1631 err = bpf_map_lookup_elem(map_fd, &index3, &map_cookie);
1632 CHECK(err != -1 || errno != ENOENT,
1633 "reuseport array lookup after close()",
1634 "sock_type:%d err:%d errno:%d\n",
1635 type, err, errno);
1636 }
1637
1638 /* Test SOCK_RAW */
1639 fd64 = socket(AF_INET6, SOCK_RAW, IPPROTO_UDP);
1640 CHECK(fd64 == -1, "socket(SOCK_RAW)", "err:%d errno:%d\n",
1641 err, errno);
1642 err = bpf_map_update_elem(map_fd, &index3, &fd64, BPF_NOEXIST);
1643 CHECK(err != -1 || errno != ENOTSUPP, "reuseport array update SOCK_RAW",
1644 "err:%d errno:%d\n", err, errno);
1645 close(fd64);
1646
1647 /* Close the 64 bit value map */
1648 close(map_fd);
1649
1650 /* Test 32 bit fd */
1651 map_fd = bpf_create_map(BPF_MAP_TYPE_REUSEPORT_SOCKARRAY,
1652 sizeof(__u32), sizeof(__u32), array_size, 0);
1653 CHECK(map_fd == -1, "reuseport array create",
1654 "map_fd:%d, errno:%d\n", map_fd, errno);
1655 prepare_reuseport_grp(SOCK_STREAM, map_fd, &fd64, &sk_cookie, 1);
1656 fd = fd64;
1657 err = bpf_map_update_elem(map_fd, &index3, &fd, BPF_NOEXIST);
1658 CHECK(err == -1, "reuseport array update 32 bit fd",
1659 "err:%d errno:%d\n", err, errno);
1660 err = bpf_map_lookup_elem(map_fd, &index3, &map_cookie);
1661 CHECK(err != -1 || errno != ENOSPC,
1662 "reuseport array lookup 32 bit fd",
1663 "err:%d errno:%d\n", err, errno);
1664 close(fd);
1665 close(map_fd);
1666}
1667
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +02001668static void run_all_tests(void)
1669{
1670 test_hashmap(0, NULL);
1671 test_hashmap_percpu(0, NULL);
Daniel Borkmann5ecf51f2017-06-11 00:50:44 +02001672 test_hashmap_walk(0, NULL);
Lorenz Bauerbf5d68c2018-11-16 11:41:11 +00001673 test_hashmap_zero_seed();
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +02001674
1675 test_arraymap(0, NULL);
1676 test_arraymap_percpu(0, NULL);
1677
1678 test_arraymap_percpu_many_keys();
1679
John Fastabend81f6bf82017-07-26 17:32:07 -07001680 test_devmap(0, NULL);
John Fastabend6f6d33f2017-08-15 22:34:22 -07001681 test_sockmap(0, NULL);
John Fastabend81f6bf82017-07-26 17:32:07 -07001682
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +02001683 test_map_large();
1684 test_map_parallel();
1685 test_map_stress();
Chenbo Fenge0433252017-10-18 13:00:23 -07001686
1687 test_map_rdonly();
1688 test_map_wronly();
Martin KaFai Lau6bc8529c2018-08-08 01:01:30 -07001689
1690 test_reuseport_array();
Mauricio Vasquez B43b987d2018-10-18 15:16:41 +02001691
1692 test_queuemap(0, NULL);
1693 test_stackmap(0, NULL);
Nikita V. Shirokovb1957c92018-11-20 20:55:57 -08001694
1695 test_map_in_map();
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +02001696}
1697
Martin KaFai Lau51a0e302019-04-26 16:39:52 -07001698#define DECLARE
1699#include <map_tests/tests.h>
1700#undef DECLARE
1701
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +02001702int main(void)
1703{
Mauricio Vasquez B43b987d2018-10-18 15:16:41 +02001704 srand(time(NULL));
1705
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +02001706 map_flags = 0;
1707 run_all_tests();
1708
1709 map_flags = BPF_F_NO_PREALLOC;
1710 run_all_tests();
1711
Martin KaFai Lau51a0e302019-04-26 16:39:52 -07001712#define CALL
1713#include <map_tests/tests.h>
1714#undef CALL
1715
Stanislav Fomicheve8ddbfb42019-01-28 09:21:15 -08001716 printf("test_maps: OK, %d SKIPPED\n", skips);
Daniel Borkmann5aa5bd12016-10-17 14:28:36 +02001717 return 0;
1718}