blob: 36549357816360a4d04b6f19aec6dd8e7f2db2fa [file] [log] [blame]
Greg Kroah-Hartmanb2441312017-11-01 15:07:57 +01001// SPDX-License-Identifier: GPL-2.0
Tim Chen67afa382017-02-22 15:45:39 -08002/*
3 * Manage cache of swap slots to be used for and returned from
4 * swap.
5 *
6 * Copyright(c) 2016 Intel Corporation.
7 *
8 * Author: Tim Chen <tim.c.chen@linux.intel.com>
9 *
10 * We allocate the swap slots from the global pool and put
11 * it into local per cpu caches. This has the advantage
12 * of no needing to acquire the swap_info lock every time
13 * we need a new slot.
14 *
15 * There is also opportunity to simply return the slot
16 * to local caches without needing to acquire swap_info
17 * lock. We do not reuse the returned slots directly but
18 * move them back to the global pool in a batch. This
19 * allows the slots to coaellesce and reduce fragmentation.
20 *
21 * The swap entry allocated is marked with SWAP_HAS_CACHE
22 * flag in map_count that prevents it from being allocated
23 * again from the global pool.
24 *
25 * The swap slots cache is protected by a mutex instead of
26 * a spin lock as when we search for slots with scan_swap_map,
27 * we can possibly sleep.
28 */
29
30#include <linux/swap_slots.h>
31#include <linux/cpu.h>
32#include <linux/cpumask.h>
33#include <linux/vmalloc.h>
34#include <linux/mutex.h>
Huang Ying54f180d2017-05-08 15:57:40 -070035#include <linux/mm.h>
Bing Hand4eef932022-05-30 16:28:05 +080036#include <trace/hooks/mm.h>
Tim Chen67afa382017-02-22 15:45:39 -080037
Tim Chen67afa382017-02-22 15:45:39 -080038static DEFINE_PER_CPU(struct swap_slots_cache, swp_slots);
39static bool swap_slot_cache_active;
Huang Yingba81f832017-02-22 15:45:46 -080040bool swap_slot_cache_enabled;
Tim Chen67afa382017-02-22 15:45:39 -080041static bool swap_slot_cache_initialized;
Colin Ian King31f21da2018-08-17 15:46:54 -070042static DEFINE_MUTEX(swap_slots_cache_mutex);
Tim Chen67afa382017-02-22 15:45:39 -080043/* Serialize swap slots cache enable/disable operations */
Colin Ian King31f21da2018-08-17 15:46:54 -070044static DEFINE_MUTEX(swap_slots_cache_enable_mutex);
Tim Chen67afa382017-02-22 15:45:39 -080045
46static void __drain_swap_slots_cache(unsigned int type);
47static void deactivate_swap_slots_cache(void);
48static void reactivate_swap_slots_cache(void);
49
Zhen Leie0f3ebb2020-08-06 23:20:11 -070050#define use_swap_slot_cache (swap_slot_cache_active && swap_slot_cache_enabled)
Tim Chen67afa382017-02-22 15:45:39 -080051#define SLOTS_CACHE 0x1
52#define SLOTS_CACHE_RET 0x2
53
54static void deactivate_swap_slots_cache(void)
55{
56 mutex_lock(&swap_slots_cache_mutex);
57 swap_slot_cache_active = false;
58 __drain_swap_slots_cache(SLOTS_CACHE|SLOTS_CACHE_RET);
59 mutex_unlock(&swap_slots_cache_mutex);
60}
61
62static void reactivate_swap_slots_cache(void)
63{
64 mutex_lock(&swap_slots_cache_mutex);
65 swap_slot_cache_active = true;
66 mutex_unlock(&swap_slots_cache_mutex);
67}
68
69/* Must not be called with cpu hot plug lock */
70void disable_swap_slots_cache_lock(void)
71{
72 mutex_lock(&swap_slots_cache_enable_mutex);
73 swap_slot_cache_enabled = false;
74 if (swap_slot_cache_initialized) {
75 /* serialize with cpu hotplug operations */
76 get_online_cpus();
77 __drain_swap_slots_cache(SLOTS_CACHE|SLOTS_CACHE_RET);
78 put_online_cpus();
79 }
80}
81
82static void __reenable_swap_slots_cache(void)
83{
84 swap_slot_cache_enabled = has_usable_swap();
85}
86
87void reenable_swap_slots_cache_unlock(void)
88{
89 __reenable_swap_slots_cache();
90 mutex_unlock(&swap_slots_cache_enable_mutex);
91}
92
Bing Han06c27662022-06-10 19:40:35 +080093bool check_cache_active(void)
Tim Chen67afa382017-02-22 15:45:39 -080094{
95 long pages;
96
Zhen Leie0f3ebb2020-08-06 23:20:11 -070097 if (!swap_slot_cache_enabled)
Tim Chen67afa382017-02-22 15:45:39 -080098 return false;
99
100 pages = get_nr_swap_pages();
101 if (!swap_slot_cache_active) {
102 if (pages > num_online_cpus() *
103 THRESHOLD_ACTIVATE_SWAP_SLOTS_CACHE)
104 reactivate_swap_slots_cache();
105 goto out;
106 }
107
108 /* if global pool of slot caches too low, deactivate cache */
109 if (pages < num_online_cpus() * THRESHOLD_DEACTIVATE_SWAP_SLOTS_CACHE)
110 deactivate_swap_slots_cache();
111out:
112 return swap_slot_cache_active;
113}
Bing Han06c27662022-06-10 19:40:35 +0800114EXPORT_SYMBOL_GPL(check_cache_active);
Tim Chen67afa382017-02-22 15:45:39 -0800115
116static int alloc_swap_slot_cache(unsigned int cpu)
117{
118 struct swap_slots_cache *cache;
119 swp_entry_t *slots, *slots_ret;
Bing Hand4eef932022-05-30 16:28:05 +0800120 bool skip = false;
121 int ret = 0;
Tim Chen67afa382017-02-22 15:45:39 -0800122
123 /*
124 * Do allocation outside swap_slots_cache_mutex
Huang Ying54f180d2017-05-08 15:57:40 -0700125 * as kvzalloc could trigger reclaim and get_swap_page,
Tim Chen67afa382017-02-22 15:45:39 -0800126 * which can lock swap_slots_cache_mutex.
127 */
Bing Hand4eef932022-05-30 16:28:05 +0800128 trace_android_vh_alloc_swap_slot_cache(&per_cpu(swp_slots, cpu),
129 &ret, &skip);
130 if (skip)
131 return ret;
Kees Cook778e1cd2018-06-12 14:04:48 -0700132 slots = kvcalloc(SWAP_SLOTS_CACHE_SIZE, sizeof(swp_entry_t),
Huang Ying54f180d2017-05-08 15:57:40 -0700133 GFP_KERNEL);
Tim Chen67afa382017-02-22 15:45:39 -0800134 if (!slots)
135 return -ENOMEM;
136
Kees Cook778e1cd2018-06-12 14:04:48 -0700137 slots_ret = kvcalloc(SWAP_SLOTS_CACHE_SIZE, sizeof(swp_entry_t),
Huang Ying54f180d2017-05-08 15:57:40 -0700138 GFP_KERNEL);
Tim Chen67afa382017-02-22 15:45:39 -0800139 if (!slots_ret) {
Huang Ying54f180d2017-05-08 15:57:40 -0700140 kvfree(slots);
Tim Chen67afa382017-02-22 15:45:39 -0800141 return -ENOMEM;
142 }
143
144 mutex_lock(&swap_slots_cache_mutex);
145 cache = &per_cpu(swp_slots, cpu);
Zhen Leif90eae22020-08-06 23:20:05 -0700146 if (cache->slots || cache->slots_ret) {
Tim Chen67afa382017-02-22 15:45:39 -0800147 /* cache already allocated */
Zhen Leif90eae22020-08-06 23:20:05 -0700148 mutex_unlock(&swap_slots_cache_mutex);
149
150 kvfree(slots);
151 kvfree(slots_ret);
152
153 return 0;
154 }
155
Tim Chen67afa382017-02-22 15:45:39 -0800156 if (!cache->lock_initialized) {
157 mutex_init(&cache->alloc_lock);
158 spin_lock_init(&cache->free_lock);
159 cache->lock_initialized = true;
160 }
161 cache->nr = 0;
162 cache->cur = 0;
163 cache->n_ret = 0;
Tim Chena2e16732017-11-15 17:34:18 -0800164 /*
165 * We initialized alloc_lock and free_lock earlier. We use
166 * !cache->slots or !cache->slots_ret to know if it is safe to acquire
167 * the corresponding lock and use the cache. Memory barrier below
168 * ensures the assumption.
169 */
170 mb();
Tim Chen67afa382017-02-22 15:45:39 -0800171 cache->slots = slots;
Tim Chen67afa382017-02-22 15:45:39 -0800172 cache->slots_ret = slots_ret;
Tim Chen67afa382017-02-22 15:45:39 -0800173 mutex_unlock(&swap_slots_cache_mutex);
Tim Chen67afa382017-02-22 15:45:39 -0800174 return 0;
175}
176
177static void drain_slots_cache_cpu(unsigned int cpu, unsigned int type,
178 bool free_slots)
179{
180 struct swap_slots_cache *cache;
181 swp_entry_t *slots = NULL;
Bing Hand4eef932022-05-30 16:28:05 +0800182 bool skip = false;
Tim Chen67afa382017-02-22 15:45:39 -0800183
184 cache = &per_cpu(swp_slots, cpu);
Bing Hand4eef932022-05-30 16:28:05 +0800185 trace_android_vh_drain_slots_cache_cpu(cache, type,
186 free_slots, &skip);
187 if (skip)
188 return;
Tim Chen67afa382017-02-22 15:45:39 -0800189 if ((type & SLOTS_CACHE) && cache->slots) {
190 mutex_lock(&cache->alloc_lock);
191 swapcache_free_entries(cache->slots + cache->cur, cache->nr);
192 cache->cur = 0;
193 cache->nr = 0;
194 if (free_slots && cache->slots) {
Huang Ying54f180d2017-05-08 15:57:40 -0700195 kvfree(cache->slots);
Tim Chen67afa382017-02-22 15:45:39 -0800196 cache->slots = NULL;
197 }
198 mutex_unlock(&cache->alloc_lock);
199 }
200 if ((type & SLOTS_CACHE_RET) && cache->slots_ret) {
201 spin_lock_irq(&cache->free_lock);
202 swapcache_free_entries(cache->slots_ret, cache->n_ret);
203 cache->n_ret = 0;
204 if (free_slots && cache->slots_ret) {
205 slots = cache->slots_ret;
206 cache->slots_ret = NULL;
207 }
208 spin_unlock_irq(&cache->free_lock);
209 if (slots)
Huang Ying54f180d2017-05-08 15:57:40 -0700210 kvfree(slots);
Tim Chen67afa382017-02-22 15:45:39 -0800211 }
212}
213
214static void __drain_swap_slots_cache(unsigned int type)
215{
216 unsigned int cpu;
217
218 /*
219 * This function is called during
220 * 1) swapoff, when we have to make sure no
221 * left over slots are in cache when we remove
222 * a swap device;
223 * 2) disabling of swap slot cache, when we run low
224 * on swap slots when allocating memory and need
225 * to return swap slots to global pool.
226 *
227 * We cannot acquire cpu hot plug lock here as
228 * this function can be invoked in the cpu
229 * hot plug path:
230 * cpu_up -> lock cpu_hotplug -> cpu hotplug state callback
231 * -> memory allocation -> direct reclaim -> get_swap_page
232 * -> drain_swap_slots_cache
233 *
234 * Hence the loop over current online cpu below could miss cpu that
235 * is being brought online but not yet marked as online.
236 * That is okay as we do not schedule and run anything on a
237 * cpu before it has been marked online. Hence, we will not
238 * fill any swap slots in slots cache of such cpu.
239 * There are no slots on such cpu that need to be drained.
240 */
241 for_each_online_cpu(cpu)
242 drain_slots_cache_cpu(cpu, type, false);
243}
244
245static int free_slot_cache(unsigned int cpu)
246{
247 mutex_lock(&swap_slots_cache_mutex);
248 drain_slots_cache_cpu(cpu, SLOTS_CACHE | SLOTS_CACHE_RET, true);
249 mutex_unlock(&swap_slots_cache_mutex);
250 return 0;
251}
252
Miaohe Linf3bc52c2020-10-13 16:52:18 -0700253void enable_swap_slots_cache(void)
Tim Chen67afa382017-02-22 15:45:39 -0800254{
Tim Chen67afa382017-02-22 15:45:39 -0800255 mutex_lock(&swap_slots_cache_enable_mutex);
Zhen Leid69a9572020-08-06 23:20:08 -0700256 if (!swap_slot_cache_initialized) {
257 int ret;
258
259 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "swap_slots_cache",
260 alloc_swap_slot_cache, free_slot_cache);
261 if (WARN_ONCE(ret < 0, "Cache allocation failed (%s), operating "
262 "without swap slots cache.\n", __func__))
263 goto out_unlock;
264
265 swap_slot_cache_initialized = true;
Tim Chen67afa382017-02-22 15:45:39 -0800266 }
267
Tim Chen67afa382017-02-22 15:45:39 -0800268 __reenable_swap_slots_cache();
269out_unlock:
270 mutex_unlock(&swap_slots_cache_enable_mutex);
Tim Chen67afa382017-02-22 15:45:39 -0800271}
272
273/* called with swap slot cache's alloc lock held */
274static int refill_swap_slots_cache(struct swap_slots_cache *cache)
275{
276 if (!use_swap_slot_cache || cache->nr)
277 return 0;
278
279 cache->cur = 0;
280 if (swap_slot_cache_active)
Huang Ying5d5e8f12018-08-21 21:52:20 -0700281 cache->nr = get_swap_pages(SWAP_SLOTS_CACHE_SIZE,
282 cache->slots, 1);
Tim Chen67afa382017-02-22 15:45:39 -0800283
284 return cache->nr;
285}
286
287int free_swap_slot(swp_entry_t entry)
288{
289 struct swap_slots_cache *cache;
Bing Han7222a0b2022-06-10 18:45:31 +0800290 bool skip = false;
Tim Chen67afa382017-02-22 15:45:39 -0800291
Sebastian Andrzej Siewiorf07e0f842017-07-10 15:49:29 -0700292 cache = raw_cpu_ptr(&swp_slots);
Bing Han7222a0b2022-06-10 18:45:31 +0800293 trace_android_vh_free_swap_slot(entry, cache, &skip);
294 if (skip)
295 return 0;
Tim Chena2e16732017-11-15 17:34:18 -0800296 if (likely(use_swap_slot_cache && cache->slots_ret)) {
Tim Chen67afa382017-02-22 15:45:39 -0800297 spin_lock_irq(&cache->free_lock);
298 /* Swap slots cache may be deactivated before acquiring lock */
Sebastian Andrzej Siewiorf07e0f842017-07-10 15:49:29 -0700299 if (!use_swap_slot_cache || !cache->slots_ret) {
Tim Chen67afa382017-02-22 15:45:39 -0800300 spin_unlock_irq(&cache->free_lock);
301 goto direct_free;
302 }
303 if (cache->n_ret >= SWAP_SLOTS_CACHE_SIZE) {
304 /*
305 * Return slots to global pool.
306 * The current swap_map value is SWAP_HAS_CACHE.
307 * Set it to 0 to indicate it is available for
308 * allocation in global pool
309 */
310 swapcache_free_entries(cache->slots_ret, cache->n_ret);
311 cache->n_ret = 0;
312 }
313 cache->slots_ret[cache->n_ret++] = entry;
314 spin_unlock_irq(&cache->free_lock);
315 } else {
316direct_free:
317 swapcache_free_entries(&entry, 1);
318 }
Tim Chen67afa382017-02-22 15:45:39 -0800319
320 return 0;
321}
322
Huang Ying38d8b4e2017-07-06 15:37:18 -0700323swp_entry_t get_swap_page(struct page *page)
Tim Chen67afa382017-02-22 15:45:39 -0800324{
Wei Yang2406b762020-04-01 21:06:16 -0700325 swp_entry_t entry;
Tim Chen67afa382017-02-22 15:45:39 -0800326 struct swap_slots_cache *cache;
Bing Hand4eef932022-05-30 16:28:05 +0800327 bool found = false;
Huang Ying38d8b4e2017-07-06 15:37:18 -0700328 entry.val = 0;
329
Bing Hand4eef932022-05-30 16:28:05 +0800330 trace_android_vh_get_swap_page(page, &entry, raw_cpu_ptr(&swp_slots), &found);
331 if (found)
332 goto out;
333
Huang Ying38d8b4e2017-07-06 15:37:18 -0700334 if (PageTransHuge(page)) {
335 if (IS_ENABLED(CONFIG_THP_SWAP))
Huang Ying5d5e8f12018-08-21 21:52:20 -0700336 get_swap_pages(1, &entry, HPAGE_PMD_NR);
Tejun Heobb98f2c2018-06-07 17:05:31 -0700337 goto out;
Huang Ying38d8b4e2017-07-06 15:37:18 -0700338 }
339
Tim Chen67afa382017-02-22 15:45:39 -0800340 /*
341 * Preemption is allowed here, because we may sleep
342 * in refill_swap_slots_cache(). But it is safe, because
343 * accesses to the per-CPU data structure are protected by the
344 * mutex cache->alloc_lock.
345 *
346 * The alloc path here does not touch cache->slots_ret
347 * so cache->free_lock is not taken.
348 */
349 cache = raw_cpu_ptr(&swp_slots);
350
Tim Chena2e16732017-11-15 17:34:18 -0800351 if (likely(check_cache_active() && cache->slots)) {
Tim Chen67afa382017-02-22 15:45:39 -0800352 mutex_lock(&cache->alloc_lock);
353 if (cache->slots) {
354repeat:
355 if (cache->nr) {
Wei Yang2406b762020-04-01 21:06:16 -0700356 entry = cache->slots[cache->cur];
357 cache->slots[cache->cur++].val = 0;
Tim Chen67afa382017-02-22 15:45:39 -0800358 cache->nr--;
Wei Yang2406b762020-04-01 21:06:16 -0700359 } else if (refill_swap_slots_cache(cache)) {
360 goto repeat;
Tim Chen67afa382017-02-22 15:45:39 -0800361 }
362 }
363 mutex_unlock(&cache->alloc_lock);
364 if (entry.val)
Tejun Heobb98f2c2018-06-07 17:05:31 -0700365 goto out;
Tim Chen67afa382017-02-22 15:45:39 -0800366 }
367
Huang Ying5d5e8f12018-08-21 21:52:20 -0700368 get_swap_pages(1, &entry, 1);
Tejun Heobb98f2c2018-06-07 17:05:31 -0700369out:
370 if (mem_cgroup_try_charge_swap(page, entry)) {
371 put_swap_page(page, entry);
372 entry.val = 0;
373 }
Tim Chen67afa382017-02-22 15:45:39 -0800374 return entry;
375}