blob: 5be0e8834a5a314ba1493aea396cbeb0fd9ad2fe [file] [log] [blame]
Thomas Gleixnerc942fdd2019-05-27 08:55:06 +02001// SPDX-License-Identifier: GPL-2.0-or-later
Balbir Singh8cdea7c2008-02-07 00:13:50 -08002/* memcontrol.c - Memory Controller
3 *
4 * Copyright IBM Corporation, 2007
5 * Author Balbir Singh <balbir@linux.vnet.ibm.com>
6 *
Pavel Emelianov78fb7462008-02-07 00:13:51 -08007 * Copyright 2007 OpenVZ SWsoft Inc
8 * Author: Pavel Emelianov <xemul@openvz.org>
9 *
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -080010 * Memory thresholds
11 * Copyright (C) 2009 Nokia Corporation
12 * Author: Kirill A. Shutemov
13 *
Glauber Costa7ae1e1d2012-12-18 14:21:56 -080014 * Kernel Memory Controller
15 * Copyright (C) 2012 Parallels Inc. and Google Inc.
16 * Authors: Glauber Costa and Suleiman Souhlal
17 *
Johannes Weiner1575e682015-04-14 15:44:51 -070018 * Native page reclaim
19 * Charge lifetime sanitation
20 * Lockless page tracking & accounting
21 * Unified hierarchy configuration model
22 * Copyright (C) 2015 Red Hat, Inc., Johannes Weiner
Alex Shi6168d0d2020-12-15 12:34:29 -080023 *
24 * Per memcg lru locking
25 * Copyright (C) 2020 Alibaba, Inc, Alex Shi
Balbir Singh8cdea7c2008-02-07 00:13:50 -080026 */
27
Johannes Weiner3e32cb22014-12-10 15:42:31 -080028#include <linux/page_counter.h>
Balbir Singh8cdea7c2008-02-07 00:13:50 -080029#include <linux/memcontrol.h>
30#include <linux/cgroup.h>
Christoph Hellwiga5201102019-08-28 16:19:53 +020031#include <linux/pagewalk.h>
Ingo Molnar6e84f312017-02-08 18:51:29 +010032#include <linux/sched/mm.h>
Hugh Dickins3a4f8a02017-02-24 14:59:36 -080033#include <linux/shmem_fs.h>
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -080034#include <linux/hugetlb.h>
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -080035#include <linux/pagemap.h>
Chris Down1ff9e6e2019-03-05 15:48:09 -080036#include <linux/vm_event_item.h>
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -080037#include <linux/smp.h>
Balbir Singh8a9f3cc2008-02-07 00:13:53 -080038#include <linux/page-flags.h>
Balbir Singh66e17072008-02-07 00:13:56 -080039#include <linux/backing-dev.h>
Balbir Singh8a9f3cc2008-02-07 00:13:53 -080040#include <linux/bit_spinlock.h>
41#include <linux/rcupdate.h>
Balbir Singhe2224322009-04-02 16:57:39 -070042#include <linux/limits.h>
Paul Gortmakerb9e15ba2011-05-26 16:00:52 -040043#include <linux/export.h>
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -080044#include <linux/mutex.h>
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -070045#include <linux/rbtree.h>
Balbir Singhb6ac57d2008-04-29 01:00:19 -070046#include <linux/slab.h>
Balbir Singh66e17072008-02-07 00:13:56 -080047#include <linux/swap.h>
Daisuke Nishimura02491442010-03-10 15:22:17 -080048#include <linux/swapops.h>
Balbir Singh66e17072008-02-07 00:13:56 -080049#include <linux/spinlock.h>
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -080050#include <linux/eventfd.h>
Tejun Heo79bd9812013-11-22 18:20:42 -050051#include <linux/poll.h>
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -080052#include <linux/sort.h>
Balbir Singh66e17072008-02-07 00:13:56 -080053#include <linux/fs.h>
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -080054#include <linux/seq_file.h>
Anton Vorontsov70ddf632013-04-29 15:08:31 -070055#include <linux/vmpressure.h>
Christoph Lameterb69408e2008-10-18 20:26:14 -070056#include <linux/mm_inline.h>
Johannes Weiner5d1ea482014-12-10 15:44:55 -080057#include <linux/swap_cgroup.h>
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -080058#include <linux/cpu.h>
KAMEZAWA Hiroyuki158e0a22010-08-10 18:03:00 -070059#include <linux/oom.h>
Johannes Weiner0056f4e2013-10-31 16:34:14 -070060#include <linux/lockdep.h>
Tejun Heo79bd9812013-11-22 18:20:42 -050061#include <linux/file.h>
Tejun Heob23afb92015-11-05 18:46:11 -080062#include <linux/tracehook.h>
Chris Down0e4b01d2019-09-23 15:34:55 -070063#include <linux/psi.h>
Johannes Weinerc8713d02019-07-11 20:55:59 -070064#include <linux/seq_buf.h>
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -080065#include "internal.h"
Glauber Costad1a4c0b2011-12-11 21:47:04 +000066#include <net/sock.h>
Michal Hocko4bd2c1e2012-10-08 16:33:10 -070067#include <net/ip.h>
Qiang Huangf35c3a82013-11-12 15:08:22 -080068#include "slab.h"
Balbir Singh8cdea7c2008-02-07 00:13:50 -080069
Linus Torvalds7c0f6ba2016-12-24 11:46:01 -080070#include <linux/uaccess.h>
Balbir Singh8697d332008-02-07 00:13:59 -080071
KOSAKI Motohirocc8e9702010-08-09 17:19:57 -070072#include <trace/events/vmscan.h>
73
Tejun Heo073219e2014-02-08 10:36:58 -050074struct cgroup_subsys memory_cgrp_subsys __read_mostly;
75EXPORT_SYMBOL(memory_cgrp_subsys);
David Rientjes68ae5642012-12-12 13:51:57 -080076
Johannes Weiner7d828602016-01-14 15:20:56 -080077struct mem_cgroup *root_mem_cgroup __read_mostly;
78
Roman Gushchin37d59852020-10-17 16:13:50 -070079/* Active memory cgroup to use from an interrupt context */
80DEFINE_PER_CPU(struct mem_cgroup *, int_active_memcg);
81
Johannes Weinerf7e1cb62016-01-14 15:21:29 -080082/* Socket memory accounting disabled? */
83static bool cgroup_memory_nosocket;
84
Vladimir Davydov04823c82016-01-20 15:02:38 -080085/* Kernel memory accounting disabled? */
86static bool cgroup_memory_nokmem;
87
Johannes Weiner21afa382015-02-11 15:26:36 -080088/* Whether the swap controller is active */
Andrew Mortonc255a452012-07-31 16:43:02 -070089#ifdef CONFIG_MEMCG_SWAP
Johannes Weinereccb52e2020-06-03 16:02:11 -070090bool cgroup_memory_noswap __read_mostly;
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -080091#else
Johannes Weinereccb52e2020-06-03 16:02:11 -070092#define cgroup_memory_noswap 1
Johannes Weiner2d1c4982020-06-03 16:02:14 -070093#endif
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -080094
Tejun Heo97b27822019-08-26 09:06:56 -070095#ifdef CONFIG_CGROUP_WRITEBACK
96static DECLARE_WAIT_QUEUE_HEAD(memcg_cgwb_frn_waitq);
97#endif
98
Johannes Weiner7941d212016-01-14 15:21:23 -080099/* Whether legacy memory+swap accounting is active */
100static bool do_memsw_account(void)
101{
Johannes Weinereccb52e2020-06-03 16:02:11 -0700102 return !cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_noswap;
Johannes Weiner7941d212016-01-14 15:21:23 -0800103}
104
Kirill A. Shutemova0db00f2012-05-29 15:06:56 -0700105#define THRESHOLDS_EVENTS_TARGET 128
106#define SOFTLIMIT_EVENTS_TARGET 1024
Johannes Weinere9f89742011-03-23 16:42:37 -0700107
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700108/*
109 * Cgroups above their limits are maintained in a RB-Tree, independent of
110 * their hierarchy representation
111 */
112
Mel Gormanef8f2322016-07-28 15:46:05 -0700113struct mem_cgroup_tree_per_node {
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700114 struct rb_root rb_root;
Davidlohr Buesofa90b2f2017-09-08 16:15:21 -0700115 struct rb_node *rb_rightmost;
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700116 spinlock_t lock;
117};
118
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700119struct mem_cgroup_tree {
120 struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
121};
122
123static struct mem_cgroup_tree soft_limit_tree __read_mostly;
124
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -0700125/* for OOM */
126struct mem_cgroup_eventfd_list {
127 struct list_head list;
128 struct eventfd_ctx *eventfd;
129};
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800130
Tejun Heo79bd9812013-11-22 18:20:42 -0500131/*
132 * cgroup_event represents events which userspace want to receive.
133 */
Tejun Heo3bc942f2013-11-22 18:20:44 -0500134struct mem_cgroup_event {
Tejun Heo79bd9812013-11-22 18:20:42 -0500135 /*
Tejun Heo59b6f872013-11-22 18:20:43 -0500136 * memcg which the event belongs to.
Tejun Heo79bd9812013-11-22 18:20:42 -0500137 */
Tejun Heo59b6f872013-11-22 18:20:43 -0500138 struct mem_cgroup *memcg;
Tejun Heo79bd9812013-11-22 18:20:42 -0500139 /*
Tejun Heo79bd9812013-11-22 18:20:42 -0500140 * eventfd to signal userspace about the event.
141 */
142 struct eventfd_ctx *eventfd;
143 /*
144 * Each of these stored in a list by the cgroup.
145 */
146 struct list_head list;
147 /*
Tejun Heofba94802013-11-22 18:20:43 -0500148 * register_event() callback will be used to add new userspace
149 * waiter for changes related to this event. Use eventfd_signal()
150 * on eventfd to send notification to userspace.
151 */
Tejun Heo59b6f872013-11-22 18:20:43 -0500152 int (*register_event)(struct mem_cgroup *memcg,
Tejun Heo347c4a82013-11-22 18:20:43 -0500153 struct eventfd_ctx *eventfd, const char *args);
Tejun Heofba94802013-11-22 18:20:43 -0500154 /*
155 * unregister_event() callback will be called when userspace closes
156 * the eventfd or on cgroup removing. This callback must be set,
157 * if you want provide notification functionality.
158 */
Tejun Heo59b6f872013-11-22 18:20:43 -0500159 void (*unregister_event)(struct mem_cgroup *memcg,
Tejun Heofba94802013-11-22 18:20:43 -0500160 struct eventfd_ctx *eventfd);
161 /*
Tejun Heo79bd9812013-11-22 18:20:42 -0500162 * All fields below needed to unregister event when
163 * userspace closes eventfd.
164 */
165 poll_table pt;
166 wait_queue_head_t *wqh;
Ingo Molnarac6424b2017-06-20 12:06:13 +0200167 wait_queue_entry_t wait;
Tejun Heo79bd9812013-11-22 18:20:42 -0500168 struct work_struct remove;
169};
170
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700171static void mem_cgroup_threshold(struct mem_cgroup *memcg);
172static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800173
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -0800174/* Stuffs for move charges at task migration. */
175/*
Johannes Weiner1dfab5a2015-02-11 15:26:09 -0800176 * Types of charges to be moved.
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -0800177 */
Johannes Weiner1dfab5a2015-02-11 15:26:09 -0800178#define MOVE_ANON 0x1U
179#define MOVE_FILE 0x2U
180#define MOVE_MASK (MOVE_ANON | MOVE_FILE)
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -0800181
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -0800182/* "mc" and its members are protected by cgroup_mutex */
183static struct move_charge_struct {
Daisuke Nishimurab1dd6932010-11-24 12:57:06 -0800184 spinlock_t lock; /* for from, to */
Tejun Heo264a0ae2016-04-21 19:09:02 -0400185 struct mm_struct *mm;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -0800186 struct mem_cgroup *from;
187 struct mem_cgroup *to;
Johannes Weiner1dfab5a2015-02-11 15:26:09 -0800188 unsigned long flags;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -0800189 unsigned long precharge;
Daisuke Nishimura854ffa82010-03-10 15:22:15 -0800190 unsigned long moved_charge;
Daisuke Nishimura483c30b2010-03-10 15:22:18 -0800191 unsigned long moved_swap;
Daisuke Nishimura8033b972010-03-10 15:22:16 -0800192 struct task_struct *moving_task; /* a task moving charges */
193 wait_queue_head_t waitq; /* a waitq for other context */
194} mc = {
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -0700195 .lock = __SPIN_LOCK_UNLOCKED(mc.lock),
Daisuke Nishimura8033b972010-03-10 15:22:16 -0800196 .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
197};
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -0800198
Balbir Singh4e416952009-09-23 15:56:39 -0700199/*
200 * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft
201 * limit reclaim to prevent infinite loops, if they ever occur.
202 */
Kirill A. Shutemova0db00f2012-05-29 15:06:56 -0700203#define MEM_CGROUP_MAX_RECLAIM_LOOPS 100
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700204#define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS 2
Balbir Singh4e416952009-09-23 15:56:39 -0700205
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -0800206/* for encoding cft->private value on file */
Glauber Costa86ae53e2012-12-18 14:21:45 -0800207enum res_type {
208 _MEM,
209 _MEMSWAP,
210 _OOM_TYPE,
Glauber Costa510fc4e2012-12-18 14:21:47 -0800211 _KMEM,
Vladimir Davydovd55f90b2016-01-20 15:02:44 -0800212 _TCP,
Glauber Costa86ae53e2012-12-18 14:21:45 -0800213};
214
Kirill A. Shutemova0db00f2012-05-29 15:06:56 -0700215#define MEMFILE_PRIVATE(x, val) ((x) << 16 | (val))
216#define MEMFILE_TYPE(val) ((val) >> 16 & 0xffff)
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -0800217#define MEMFILE_ATTR(val) ((val) & 0xffff)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -0700218/* Used for OOM nofiier */
219#define OOM_CONTROL (0)
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -0800220
Kirill Tkhaib05706f2018-08-17 15:47:33 -0700221/*
222 * Iteration constructs for visiting all cgroups (under a tree). If
223 * loops are exited prematurely (break), mem_cgroup_iter_break() must
224 * be used for reference counting.
225 */
226#define for_each_mem_cgroup_tree(iter, root) \
227 for (iter = mem_cgroup_iter(root, NULL, NULL); \
228 iter != NULL; \
229 iter = mem_cgroup_iter(root, iter, NULL))
230
231#define for_each_mem_cgroup(iter) \
232 for (iter = mem_cgroup_iter(NULL, NULL, NULL); \
233 iter != NULL; \
234 iter = mem_cgroup_iter(NULL, iter, NULL))
235
Tetsuo Handa7775fac2019-03-05 15:46:47 -0800236static inline bool should_force_charge(void)
237{
238 return tsk_is_oom_victim(current) || fatal_signal_pending(current) ||
239 (current->flags & PF_EXITING);
240}
241
Anton Vorontsov70ddf632013-04-29 15:08:31 -0700242/* Some nice accessors for the vmpressure. */
243struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg)
244{
245 if (!memcg)
246 memcg = root_mem_cgroup;
247 return &memcg->vmpressure;
248}
249
250struct cgroup_subsys_state *vmpressure_to_css(struct vmpressure *vmpr)
251{
252 return &container_of(vmpr, struct mem_cgroup, vmpressure)->css;
253}
254
Kirill Tkhai84c07d12018-08-17 15:47:25 -0700255#ifdef CONFIG_MEMCG_KMEM
Roman Gushchinbf4f0592020-08-06 23:20:49 -0700256extern spinlock_t css_set_lock;
257
Roman Gushchinc1a660d2021-02-24 12:03:58 -0800258static int __memcg_kmem_charge(struct mem_cgroup *memcg, gfp_t gfp,
259 unsigned int nr_pages);
260static void __memcg_kmem_uncharge(struct mem_cgroup *memcg,
261 unsigned int nr_pages);
262
Roman Gushchinbf4f0592020-08-06 23:20:49 -0700263static void obj_cgroup_release(struct percpu_ref *ref)
264{
265 struct obj_cgroup *objcg = container_of(ref, struct obj_cgroup, refcnt);
266 struct mem_cgroup *memcg;
267 unsigned int nr_bytes;
268 unsigned int nr_pages;
269 unsigned long flags;
270
271 /*
272 * At this point all allocated objects are freed, and
273 * objcg->nr_charged_bytes can't have an arbitrary byte value.
274 * However, it can be PAGE_SIZE or (x * PAGE_SIZE).
275 *
276 * The following sequence can lead to it:
277 * 1) CPU0: objcg == stock->cached_objcg
278 * 2) CPU1: we do a small allocation (e.g. 92 bytes),
279 * PAGE_SIZE bytes are charged
280 * 3) CPU1: a process from another memcg is allocating something,
281 * the stock if flushed,
282 * objcg->nr_charged_bytes = PAGE_SIZE - 92
283 * 5) CPU0: we do release this object,
284 * 92 bytes are added to stock->nr_bytes
285 * 6) CPU0: stock is flushed,
286 * 92 bytes are added to objcg->nr_charged_bytes
287 *
288 * In the result, nr_charged_bytes == PAGE_SIZE.
289 * This page will be uncharged in obj_cgroup_release().
290 */
291 nr_bytes = atomic_read(&objcg->nr_charged_bytes);
292 WARN_ON_ONCE(nr_bytes & (PAGE_SIZE - 1));
293 nr_pages = nr_bytes >> PAGE_SHIFT;
294
295 spin_lock_irqsave(&css_set_lock, flags);
296 memcg = obj_cgroup_memcg(objcg);
297 if (nr_pages)
298 __memcg_kmem_uncharge(memcg, nr_pages);
299 list_del(&objcg->list);
300 mem_cgroup_put(memcg);
301 spin_unlock_irqrestore(&css_set_lock, flags);
302
303 percpu_ref_exit(ref);
304 kfree_rcu(objcg, rcu);
305}
306
307static struct obj_cgroup *obj_cgroup_alloc(void)
308{
309 struct obj_cgroup *objcg;
310 int ret;
311
312 objcg = kzalloc(sizeof(struct obj_cgroup), GFP_KERNEL);
313 if (!objcg)
314 return NULL;
315
316 ret = percpu_ref_init(&objcg->refcnt, obj_cgroup_release, 0,
317 GFP_KERNEL);
318 if (ret) {
319 kfree(objcg);
320 return NULL;
321 }
322 INIT_LIST_HEAD(&objcg->list);
323 return objcg;
324}
325
326static void memcg_reparent_objcgs(struct mem_cgroup *memcg,
327 struct mem_cgroup *parent)
328{
329 struct obj_cgroup *objcg, *iter;
330
331 objcg = rcu_replace_pointer(memcg->objcg, NULL, true);
332
333 spin_lock_irq(&css_set_lock);
334
335 /* Move active objcg to the parent's list */
336 xchg(&objcg->memcg, parent);
337 css_get(&parent->css);
338 list_add(&objcg->list, &parent->objcg_list);
339
340 /* Move already reparented objcgs to the parent's list */
341 list_for_each_entry(iter, &memcg->objcg_list, list) {
342 css_get(&parent->css);
343 xchg(&iter->memcg, parent);
344 css_put(&memcg->css);
345 }
346 list_splice(&memcg->objcg_list, &parent->objcg_list);
347
348 spin_unlock_irq(&css_set_lock);
349
350 percpu_ref_kill(&objcg->refcnt);
351}
352
Glauber Costa55007d82012-12-18 14:22:38 -0800353/*
Roman Gushchin98556092020-08-06 23:21:10 -0700354 * This will be used as a shrinker list's index.
Li Zefanb8627832013-09-23 16:56:47 +0800355 * The main reason for not using cgroup id for this:
356 * this works better in sparse environments, where we have a lot of memcgs,
357 * but only a few kmem-limited. Or also, if we have, for instance, 200
358 * memcgs, and none but the 200th is kmem-limited, we'd have to have a
359 * 200 entry array for that.
Glauber Costa55007d82012-12-18 14:22:38 -0800360 *
Vladimir Davydovdbcf73e2015-02-12 14:58:57 -0800361 * The current size of the caches array is stored in memcg_nr_cache_ids. It
362 * will double each time we have to increase it.
Glauber Costa55007d82012-12-18 14:22:38 -0800363 */
Vladimir Davydovdbcf73e2015-02-12 14:58:57 -0800364static DEFINE_IDA(memcg_cache_ida);
365int memcg_nr_cache_ids;
Glauber Costa749c5412012-12-18 14:23:01 -0800366
Vladimir Davydov05257a12015-02-12 14:59:01 -0800367/* Protects memcg_nr_cache_ids */
368static DECLARE_RWSEM(memcg_cache_ids_sem);
369
370void memcg_get_cache_ids(void)
371{
372 down_read(&memcg_cache_ids_sem);
373}
374
375void memcg_put_cache_ids(void)
376{
377 up_read(&memcg_cache_ids_sem);
378}
379
Glauber Costa55007d82012-12-18 14:22:38 -0800380/*
381 * MIN_SIZE is different than 1, because we would like to avoid going through
382 * the alloc/free process all the time. In a small machine, 4 kmem-limited
383 * cgroups is a reasonable guess. In the future, it could be a parameter or
384 * tunable, but that is strictly not necessary.
385 *
Li Zefanb8627832013-09-23 16:56:47 +0800386 * MAX_SIZE should be as large as the number of cgrp_ids. Ideally, we could get
Glauber Costa55007d82012-12-18 14:22:38 -0800387 * this constant directly from cgroup, but it is understandable that this is
388 * better kept as an internal representation in cgroup.c. In any case, the
Li Zefanb8627832013-09-23 16:56:47 +0800389 * cgrp_id space is not getting any smaller, and we don't have to necessarily
Glauber Costa55007d82012-12-18 14:22:38 -0800390 * increase ours as well if it increases.
391 */
392#define MEMCG_CACHES_MIN_SIZE 4
Li Zefanb8627832013-09-23 16:56:47 +0800393#define MEMCG_CACHES_MAX_SIZE MEM_CGROUP_ID_MAX
Glauber Costa55007d82012-12-18 14:22:38 -0800394
Glauber Costad7f25f82012-12-18 14:22:40 -0800395/*
396 * A lot of the calls to the cache allocation functions are expected to be
Roman Gushchin272911a2020-08-06 23:21:17 -0700397 * inlined by the compiler. Since the calls to memcg_slab_pre_alloc_hook() are
Glauber Costad7f25f82012-12-18 14:22:40 -0800398 * conditional to this static branch, we'll have to allow modules that does
399 * kmem_cache_alloc and the such to see this symbol as well
400 */
Johannes Weineref129472016-01-14 15:21:34 -0800401DEFINE_STATIC_KEY_FALSE(memcg_kmem_enabled_key);
Glauber Costad7f25f82012-12-18 14:22:40 -0800402EXPORT_SYMBOL(memcg_kmem_enabled_key);
Yang Shi0a432dc2019-09-23 15:38:12 -0700403#endif
Tejun Heo17cc4df2017-02-22 15:41:36 -0800404
Kirill Tkhai0a4465d2018-08-17 15:47:37 -0700405static int memcg_shrinker_map_size;
406static DEFINE_MUTEX(memcg_shrinker_map_mutex);
407
408static void memcg_free_shrinker_map_rcu(struct rcu_head *head)
409{
410 kvfree(container_of(head, struct memcg_shrinker_map, rcu));
411}
412
413static int memcg_expand_one_shrinker_map(struct mem_cgroup *memcg,
414 int size, int old_size)
415{
416 struct memcg_shrinker_map *new, *old;
417 int nid;
418
419 lockdep_assert_held(&memcg_shrinker_map_mutex);
420
421 for_each_node(nid) {
422 old = rcu_dereference_protected(
423 mem_cgroup_nodeinfo(memcg, nid)->shrinker_map, true);
424 /* Not yet online memcg */
425 if (!old)
426 return 0;
427
Kirill Tkhai86daf942020-04-01 21:06:33 -0700428 new = kvmalloc_node(sizeof(*new) + size, GFP_KERNEL, nid);
Kirill Tkhai0a4465d2018-08-17 15:47:37 -0700429 if (!new)
430 return -ENOMEM;
431
432 /* Set all old bits, clear all new bits */
433 memset(new->map, (int)0xff, old_size);
434 memset((void *)new->map + old_size, 0, size - old_size);
435
436 rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_map, new);
437 call_rcu(&old->rcu, memcg_free_shrinker_map_rcu);
438 }
439
440 return 0;
441}
442
443static void memcg_free_shrinker_maps(struct mem_cgroup *memcg)
444{
445 struct mem_cgroup_per_node *pn;
446 struct memcg_shrinker_map *map;
447 int nid;
448
449 if (mem_cgroup_is_root(memcg))
450 return;
451
452 for_each_node(nid) {
453 pn = mem_cgroup_nodeinfo(memcg, nid);
454 map = rcu_dereference_protected(pn->shrinker_map, true);
Yang Li8a260162021-02-24 12:04:05 -0800455 kvfree(map);
Kirill Tkhai0a4465d2018-08-17 15:47:37 -0700456 rcu_assign_pointer(pn->shrinker_map, NULL);
457 }
458}
459
460static int memcg_alloc_shrinker_maps(struct mem_cgroup *memcg)
461{
462 struct memcg_shrinker_map *map;
463 int nid, size, ret = 0;
464
465 if (mem_cgroup_is_root(memcg))
466 return 0;
467
468 mutex_lock(&memcg_shrinker_map_mutex);
469 size = memcg_shrinker_map_size;
470 for_each_node(nid) {
Kirill Tkhai86daf942020-04-01 21:06:33 -0700471 map = kvzalloc_node(sizeof(*map) + size, GFP_KERNEL, nid);
Kirill Tkhai0a4465d2018-08-17 15:47:37 -0700472 if (!map) {
473 memcg_free_shrinker_maps(memcg);
474 ret = -ENOMEM;
475 break;
476 }
477 rcu_assign_pointer(memcg->nodeinfo[nid]->shrinker_map, map);
478 }
479 mutex_unlock(&memcg_shrinker_map_mutex);
480
481 return ret;
482}
483
484int memcg_expand_shrinker_maps(int new_id)
485{
486 int size, old_size, ret = 0;
487 struct mem_cgroup *memcg;
488
489 size = DIV_ROUND_UP(new_id + 1, BITS_PER_LONG) * sizeof(unsigned long);
490 old_size = memcg_shrinker_map_size;
491 if (size <= old_size)
492 return 0;
493
494 mutex_lock(&memcg_shrinker_map_mutex);
495 if (!root_mem_cgroup)
496 goto unlock;
497
498 for_each_mem_cgroup(memcg) {
499 if (mem_cgroup_is_root(memcg))
500 continue;
501 ret = memcg_expand_one_shrinker_map(memcg, size, old_size);
Vasily Averin75866af2020-02-20 20:04:18 -0800502 if (ret) {
503 mem_cgroup_iter_break(NULL, memcg);
Kirill Tkhai0a4465d2018-08-17 15:47:37 -0700504 goto unlock;
Vasily Averin75866af2020-02-20 20:04:18 -0800505 }
Kirill Tkhai0a4465d2018-08-17 15:47:37 -0700506 }
507unlock:
508 if (!ret)
509 memcg_shrinker_map_size = size;
510 mutex_unlock(&memcg_shrinker_map_mutex);
511 return ret;
512}
Kirill Tkhaifae91d62018-08-17 15:48:10 -0700513
514void memcg_set_shrinker_bit(struct mem_cgroup *memcg, int nid, int shrinker_id)
515{
516 if (shrinker_id >= 0 && memcg && !mem_cgroup_is_root(memcg)) {
517 struct memcg_shrinker_map *map;
518
519 rcu_read_lock();
520 map = rcu_dereference(memcg->nodeinfo[nid]->shrinker_map);
Kirill Tkhaif90280d2018-08-17 15:48:25 -0700521 /* Pairs with smp mb in shrink_slab() */
522 smp_mb__before_atomic();
Kirill Tkhaifae91d62018-08-17 15:48:10 -0700523 set_bit(shrinker_id, map->map);
524 rcu_read_unlock();
525 }
526}
527
Tejun Heoad7fa852015-05-27 20:00:02 -0400528/**
529 * mem_cgroup_css_from_page - css of the memcg associated with a page
530 * @page: page of interest
531 *
532 * If memcg is bound to the default hierarchy, css of the memcg associated
533 * with @page is returned. The returned css remains associated with @page
534 * until it is released.
535 *
536 * If memcg is bound to a traditional hierarchy, the css of root_mem_cgroup
537 * is returned.
Tejun Heoad7fa852015-05-27 20:00:02 -0400538 */
539struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page)
540{
541 struct mem_cgroup *memcg;
542
Roman Gushchinbcfe06b2020-12-01 13:58:27 -0800543 memcg = page_memcg(page);
Tejun Heoad7fa852015-05-27 20:00:02 -0400544
Tejun Heo9e10a132015-09-18 11:56:28 -0400545 if (!memcg || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
Tejun Heoad7fa852015-05-27 20:00:02 -0400546 memcg = root_mem_cgroup;
547
Tejun Heoad7fa852015-05-27 20:00:02 -0400548 return &memcg->css;
549}
550
Vladimir Davydov2fc04522015-09-09 15:35:28 -0700551/**
552 * page_cgroup_ino - return inode number of the memcg a page is charged to
553 * @page: the page
554 *
555 * Look up the closest online ancestor of the memory cgroup @page is charged to
556 * and return its inode number or 0 if @page is not charged to any cgroup. It
557 * is safe to call this function without holding a reference to @page.
558 *
559 * Note, this function is inherently racy, because there is nothing to prevent
560 * the cgroup inode from getting torn down and potentially reallocated a moment
561 * after page_cgroup_ino() returns, so it only should be used by callers that
562 * do not care (such as procfs interfaces).
563 */
564ino_t page_cgroup_ino(struct page *page)
565{
566 struct mem_cgroup *memcg;
567 unsigned long ino = 0;
568
569 rcu_read_lock();
Roman Gushchinbcfe06b2020-12-01 13:58:27 -0800570 memcg = page_memcg_check(page);
Roman Gushchin286e04b2020-08-06 23:20:52 -0700571
Vladimir Davydov2fc04522015-09-09 15:35:28 -0700572 while (memcg && !(memcg->css.flags & CSS_ONLINE))
573 memcg = parent_mem_cgroup(memcg);
574 if (memcg)
575 ino = cgroup_ino(memcg->css.cgroup);
576 rcu_read_unlock();
577 return ino;
578}
579
Mel Gormanef8f2322016-07-28 15:46:05 -0700580static struct mem_cgroup_per_node *
581mem_cgroup_page_nodeinfo(struct mem_cgroup *memcg, struct page *page)
Balbir Singhf64c3f52009-09-23 15:56:37 -0700582{
Johannes Weiner97a6c372011-03-23 16:42:27 -0700583 int nid = page_to_nid(page);
Balbir Singhf64c3f52009-09-23 15:56:37 -0700584
Mel Gormanef8f2322016-07-28 15:46:05 -0700585 return memcg->nodeinfo[nid];
Balbir Singhf64c3f52009-09-23 15:56:37 -0700586}
587
Mel Gormanef8f2322016-07-28 15:46:05 -0700588static struct mem_cgroup_tree_per_node *
589soft_limit_tree_node(int nid)
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700590{
Mel Gormanef8f2322016-07-28 15:46:05 -0700591 return soft_limit_tree.rb_tree_per_node[nid];
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700592}
593
Mel Gormanef8f2322016-07-28 15:46:05 -0700594static struct mem_cgroup_tree_per_node *
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700595soft_limit_tree_from_page(struct page *page)
596{
597 int nid = page_to_nid(page);
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700598
Mel Gormanef8f2322016-07-28 15:46:05 -0700599 return soft_limit_tree.rb_tree_per_node[nid];
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700600}
601
Mel Gormanef8f2322016-07-28 15:46:05 -0700602static void __mem_cgroup_insert_exceeded(struct mem_cgroup_per_node *mz,
603 struct mem_cgroup_tree_per_node *mctz,
Johannes Weiner3e32cb22014-12-10 15:42:31 -0800604 unsigned long new_usage_in_excess)
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700605{
606 struct rb_node **p = &mctz->rb_root.rb_node;
607 struct rb_node *parent = NULL;
Mel Gormanef8f2322016-07-28 15:46:05 -0700608 struct mem_cgroup_per_node *mz_node;
Davidlohr Buesofa90b2f2017-09-08 16:15:21 -0700609 bool rightmost = true;
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700610
611 if (mz->on_tree)
612 return;
613
614 mz->usage_in_excess = new_usage_in_excess;
615 if (!mz->usage_in_excess)
616 return;
617 while (*p) {
618 parent = *p;
Mel Gormanef8f2322016-07-28 15:46:05 -0700619 mz_node = rb_entry(parent, struct mem_cgroup_per_node,
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700620 tree_node);
Davidlohr Buesofa90b2f2017-09-08 16:15:21 -0700621 if (mz->usage_in_excess < mz_node->usage_in_excess) {
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700622 p = &(*p)->rb_left;
Davidlohr Buesofa90b2f2017-09-08 16:15:21 -0700623 rightmost = false;
Miaohe Lin378876b2020-12-14 19:06:28 -0800624 } else {
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700625 p = &(*p)->rb_right;
Miaohe Lin378876b2020-12-14 19:06:28 -0800626 }
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700627 }
Davidlohr Buesofa90b2f2017-09-08 16:15:21 -0700628
629 if (rightmost)
630 mctz->rb_rightmost = &mz->tree_node;
631
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700632 rb_link_node(&mz->tree_node, parent, p);
633 rb_insert_color(&mz->tree_node, &mctz->rb_root);
634 mz->on_tree = true;
635}
636
Mel Gormanef8f2322016-07-28 15:46:05 -0700637static void __mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz,
638 struct mem_cgroup_tree_per_node *mctz)
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700639{
640 if (!mz->on_tree)
641 return;
Davidlohr Buesofa90b2f2017-09-08 16:15:21 -0700642
643 if (&mz->tree_node == mctz->rb_rightmost)
644 mctz->rb_rightmost = rb_prev(&mz->tree_node);
645
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700646 rb_erase(&mz->tree_node, &mctz->rb_root);
647 mz->on_tree = false;
648}
649
Mel Gormanef8f2322016-07-28 15:46:05 -0700650static void mem_cgroup_remove_exceeded(struct mem_cgroup_per_node *mz,
651 struct mem_cgroup_tree_per_node *mctz)
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700652{
Johannes Weiner0a31bc92014-08-08 14:19:22 -0700653 unsigned long flags;
654
655 spin_lock_irqsave(&mctz->lock, flags);
Johannes Weinercf2c8122014-06-06 14:38:21 -0700656 __mem_cgroup_remove_exceeded(mz, mctz);
Johannes Weiner0a31bc92014-08-08 14:19:22 -0700657 spin_unlock_irqrestore(&mctz->lock, flags);
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700658}
659
Johannes Weiner3e32cb22014-12-10 15:42:31 -0800660static unsigned long soft_limit_excess(struct mem_cgroup *memcg)
661{
662 unsigned long nr_pages = page_counter_read(&memcg->memory);
Jason Low4db0c3c2015-04-15 16:14:08 -0700663 unsigned long soft_limit = READ_ONCE(memcg->soft_limit);
Johannes Weiner3e32cb22014-12-10 15:42:31 -0800664 unsigned long excess = 0;
665
666 if (nr_pages > soft_limit)
667 excess = nr_pages - soft_limit;
668
669 return excess;
670}
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700671
672static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page)
673{
Johannes Weiner3e32cb22014-12-10 15:42:31 -0800674 unsigned long excess;
Mel Gormanef8f2322016-07-28 15:46:05 -0700675 struct mem_cgroup_per_node *mz;
676 struct mem_cgroup_tree_per_node *mctz;
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700677
Jianyu Zhane2318752014-06-06 14:38:20 -0700678 mctz = soft_limit_tree_from_page(page);
Laurent Dufourbfc72282017-03-09 16:17:06 -0800679 if (!mctz)
680 return;
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700681 /*
682 * Necessary to update all ancestors when hierarchy is used.
683 * because their event counter is not touched.
684 */
685 for (; memcg; memcg = parent_mem_cgroup(memcg)) {
Mel Gormanef8f2322016-07-28 15:46:05 -0700686 mz = mem_cgroup_page_nodeinfo(memcg, page);
Johannes Weiner3e32cb22014-12-10 15:42:31 -0800687 excess = soft_limit_excess(memcg);
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700688 /*
689 * We have to update the tree if mz is on RB-tree or
690 * mem is over its softlimit.
691 */
692 if (excess || mz->on_tree) {
Johannes Weiner0a31bc92014-08-08 14:19:22 -0700693 unsigned long flags;
694
695 spin_lock_irqsave(&mctz->lock, flags);
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700696 /* if on-tree, remove it */
697 if (mz->on_tree)
Johannes Weinercf2c8122014-06-06 14:38:21 -0700698 __mem_cgroup_remove_exceeded(mz, mctz);
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700699 /*
700 * Insert again. mz->usage_in_excess will be updated.
701 * If excess is 0, no tree ops.
702 */
Johannes Weinercf2c8122014-06-06 14:38:21 -0700703 __mem_cgroup_insert_exceeded(mz, mctz, excess);
Johannes Weiner0a31bc92014-08-08 14:19:22 -0700704 spin_unlock_irqrestore(&mctz->lock, flags);
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700705 }
706 }
707}
708
709static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg)
710{
Mel Gormanef8f2322016-07-28 15:46:05 -0700711 struct mem_cgroup_tree_per_node *mctz;
712 struct mem_cgroup_per_node *mz;
713 int nid;
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700714
Jianyu Zhane2318752014-06-06 14:38:20 -0700715 for_each_node(nid) {
Mel Gormanef8f2322016-07-28 15:46:05 -0700716 mz = mem_cgroup_nodeinfo(memcg, nid);
717 mctz = soft_limit_tree_node(nid);
Laurent Dufourbfc72282017-03-09 16:17:06 -0800718 if (mctz)
719 mem_cgroup_remove_exceeded(mz, mctz);
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700720 }
721}
722
Mel Gormanef8f2322016-07-28 15:46:05 -0700723static struct mem_cgroup_per_node *
724__mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700725{
Mel Gormanef8f2322016-07-28 15:46:05 -0700726 struct mem_cgroup_per_node *mz;
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700727
728retry:
729 mz = NULL;
Davidlohr Buesofa90b2f2017-09-08 16:15:21 -0700730 if (!mctz->rb_rightmost)
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700731 goto done; /* Nothing to reclaim from */
732
Davidlohr Buesofa90b2f2017-09-08 16:15:21 -0700733 mz = rb_entry(mctz->rb_rightmost,
734 struct mem_cgroup_per_node, tree_node);
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700735 /*
736 * Remove the node now but someone else can add it back,
737 * we will to add it back at the end of reclaim to its correct
738 * position in the tree.
739 */
Johannes Weinercf2c8122014-06-06 14:38:21 -0700740 __mem_cgroup_remove_exceeded(mz, mctz);
Johannes Weiner3e32cb22014-12-10 15:42:31 -0800741 if (!soft_limit_excess(mz->memcg) ||
Shakeel Butt8965aa22020-04-01 21:07:10 -0700742 !css_tryget(&mz->memcg->css))
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700743 goto retry;
744done:
745 return mz;
746}
747
Mel Gormanef8f2322016-07-28 15:46:05 -0700748static struct mem_cgroup_per_node *
749mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_node *mctz)
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700750{
Mel Gormanef8f2322016-07-28 15:46:05 -0700751 struct mem_cgroup_per_node *mz;
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700752
Johannes Weiner0a31bc92014-08-08 14:19:22 -0700753 spin_lock_irq(&mctz->lock);
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700754 mz = __mem_cgroup_largest_soft_limit_node(mctz);
Johannes Weiner0a31bc92014-08-08 14:19:22 -0700755 spin_unlock_irq(&mctz->lock);
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700756 return mz;
757}
758
Johannes Weinerdb9adbc2019-05-14 15:47:09 -0700759/**
760 * __mod_memcg_state - update cgroup memory statistics
761 * @memcg: the memory cgroup
762 * @idx: the stat item - can be enum memcg_stat_item or enum node_stat_item
763 * @val: delta to add to the counter, can be negative
764 */
765void __mod_memcg_state(struct mem_cgroup *memcg, int idx, int val)
766{
Roman Gushchinea426c22020-08-06 23:20:35 -0700767 long x, threshold = MEMCG_CHARGE_BATCH;
Johannes Weinerdb9adbc2019-05-14 15:47:09 -0700768
769 if (mem_cgroup_disabled())
770 return;
771
Roman Gushchin772616b2020-08-11 18:30:21 -0700772 if (memcg_stat_item_in_bytes(idx))
Roman Gushchinea426c22020-08-06 23:20:35 -0700773 threshold <<= PAGE_SHIFT;
774
Johannes Weinerdb9adbc2019-05-14 15:47:09 -0700775 x = val + __this_cpu_read(memcg->vmstats_percpu->stat[idx]);
Roman Gushchinea426c22020-08-06 23:20:35 -0700776 if (unlikely(abs(x) > threshold)) {
Johannes Weiner42a30032019-05-14 15:47:12 -0700777 struct mem_cgroup *mi;
778
Yafang Shao766a4c12019-07-16 16:26:06 -0700779 /*
780 * Batch local counters to keep them in sync with
781 * the hierarchical ones.
782 */
783 __this_cpu_add(memcg->vmstats_local->stat[idx], x);
Johannes Weiner42a30032019-05-14 15:47:12 -0700784 for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
785 atomic_long_add(x, &mi->vmstats[idx]);
Johannes Weinerdb9adbc2019-05-14 15:47:09 -0700786 x = 0;
787 }
788 __this_cpu_write(memcg->vmstats_percpu->stat[idx], x);
789}
790
Johannes Weiner42a30032019-05-14 15:47:12 -0700791static struct mem_cgroup_per_node *
792parent_nodeinfo(struct mem_cgroup_per_node *pn, int nid)
793{
794 struct mem_cgroup *parent;
795
796 parent = parent_mem_cgroup(pn->memcg);
797 if (!parent)
798 return NULL;
799 return mem_cgroup_nodeinfo(parent, nid);
800}
801
Roman Gushchineedc4e52020-08-06 23:20:32 -0700802void __mod_memcg_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx,
803 int val)
804{
805 struct mem_cgroup_per_node *pn;
806 struct mem_cgroup *memcg;
Roman Gushchinea426c22020-08-06 23:20:35 -0700807 long x, threshold = MEMCG_CHARGE_BATCH;
Roman Gushchineedc4e52020-08-06 23:20:32 -0700808
809 pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
810 memcg = pn->memcg;
811
812 /* Update memcg */
813 __mod_memcg_state(memcg, idx, val);
814
815 /* Update lruvec */
816 __this_cpu_add(pn->lruvec_stat_local->count[idx], val);
817
Roman Gushchinea426c22020-08-06 23:20:35 -0700818 if (vmstat_item_in_bytes(idx))
819 threshold <<= PAGE_SHIFT;
820
Roman Gushchineedc4e52020-08-06 23:20:32 -0700821 x = val + __this_cpu_read(pn->lruvec_stat_cpu->count[idx]);
Roman Gushchinea426c22020-08-06 23:20:35 -0700822 if (unlikely(abs(x) > threshold)) {
Roman Gushchineedc4e52020-08-06 23:20:32 -0700823 pg_data_t *pgdat = lruvec_pgdat(lruvec);
824 struct mem_cgroup_per_node *pi;
825
826 for (pi = pn; pi; pi = parent_nodeinfo(pi, pgdat->node_id))
827 atomic_long_add(x, &pi->lruvec_stat[idx]);
828 x = 0;
829 }
830 __this_cpu_write(pn->lruvec_stat_cpu->count[idx], x);
831}
832
Johannes Weinerdb9adbc2019-05-14 15:47:09 -0700833/**
834 * __mod_lruvec_state - update lruvec memory statistics
835 * @lruvec: the lruvec
836 * @idx: the stat item
837 * @val: delta to add to the counter, can be negative
838 *
839 * The lruvec is the intersection of the NUMA node and a cgroup. This
840 * function updates the all three counters that are affected by a
841 * change of state at this level: per-node, per-cgroup, per-lruvec.
842 */
843void __mod_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx,
844 int val)
845{
Johannes Weinerdb9adbc2019-05-14 15:47:09 -0700846 /* Update node */
Roman Gushchineedc4e52020-08-06 23:20:32 -0700847 __mod_node_page_state(lruvec_pgdat(lruvec), idx, val);
Johannes Weinerdb9adbc2019-05-14 15:47:09 -0700848
Roman Gushchineedc4e52020-08-06 23:20:32 -0700849 /* Update memcg and lruvec */
850 if (!mem_cgroup_disabled())
851 __mod_memcg_lruvec_state(lruvec, idx, val);
Johannes Weinerdb9adbc2019-05-14 15:47:09 -0700852}
853
Shakeel Buttc47d5032020-12-14 19:07:14 -0800854void __mod_lruvec_page_state(struct page *page, enum node_stat_item idx,
855 int val)
856{
857 struct page *head = compound_head(page); /* rmap on tail pages */
Linus Torvaldsd635a692020-12-15 13:22:29 -0800858 struct mem_cgroup *memcg = page_memcg(head);
Shakeel Buttc47d5032020-12-14 19:07:14 -0800859 pg_data_t *pgdat = page_pgdat(page);
860 struct lruvec *lruvec;
861
862 /* Untracked pages have no memcg, no lruvec. Update only the node */
Linus Torvaldsd635a692020-12-15 13:22:29 -0800863 if (!memcg) {
Shakeel Buttc47d5032020-12-14 19:07:14 -0800864 __mod_node_page_state(pgdat, idx, val);
865 return;
866 }
867
Linus Torvaldsd635a692020-12-15 13:22:29 -0800868 lruvec = mem_cgroup_lruvec(memcg, pgdat);
Shakeel Buttc47d5032020-12-14 19:07:14 -0800869 __mod_lruvec_state(lruvec, idx, val);
870}
Shakeel Buttf0c0c112020-12-14 19:07:17 -0800871EXPORT_SYMBOL(__mod_lruvec_page_state);
Shakeel Buttc47d5032020-12-14 19:07:14 -0800872
Muchun Songda3ceef2020-12-14 19:07:04 -0800873void __mod_lruvec_kmem_state(void *p, enum node_stat_item idx, int val)
Roman Gushchinec9f0232019-08-13 15:37:41 -0700874{
Roman Gushchin4f103c62020-04-01 21:06:36 -0700875 pg_data_t *pgdat = page_pgdat(virt_to_page(p));
Roman Gushchinec9f0232019-08-13 15:37:41 -0700876 struct mem_cgroup *memcg;
877 struct lruvec *lruvec;
878
879 rcu_read_lock();
Roman Gushchin4f103c62020-04-01 21:06:36 -0700880 memcg = mem_cgroup_from_obj(p);
Roman Gushchinec9f0232019-08-13 15:37:41 -0700881
Muchun Song8faeb1f2020-11-21 22:17:12 -0800882 /*
883 * Untracked pages have no memcg, no lruvec. Update only the
884 * node. If we reparent the slab objects to the root memcg,
885 * when we free the slab object, we need to update the per-memcg
886 * vmstats to keep it correct for the root memcg.
887 */
888 if (!memcg) {
Roman Gushchinec9f0232019-08-13 15:37:41 -0700889 __mod_node_page_state(pgdat, idx, val);
890 } else {
Johannes Weiner867e5e12019-11-30 17:55:34 -0800891 lruvec = mem_cgroup_lruvec(memcg, pgdat);
Roman Gushchinec9f0232019-08-13 15:37:41 -0700892 __mod_lruvec_state(lruvec, idx, val);
893 }
894 rcu_read_unlock();
895}
896
Johannes Weinerdb9adbc2019-05-14 15:47:09 -0700897/**
898 * __count_memcg_events - account VM events in a cgroup
899 * @memcg: the memory cgroup
900 * @idx: the event item
901 * @count: the number of events that occured
902 */
903void __count_memcg_events(struct mem_cgroup *memcg, enum vm_event_item idx,
904 unsigned long count)
905{
906 unsigned long x;
907
908 if (mem_cgroup_disabled())
909 return;
910
911 x = count + __this_cpu_read(memcg->vmstats_percpu->events[idx]);
912 if (unlikely(x > MEMCG_CHARGE_BATCH)) {
Johannes Weiner42a30032019-05-14 15:47:12 -0700913 struct mem_cgroup *mi;
914
Yafang Shao766a4c12019-07-16 16:26:06 -0700915 /*
916 * Batch local counters to keep them in sync with
917 * the hierarchical ones.
918 */
919 __this_cpu_add(memcg->vmstats_local->events[idx], x);
Johannes Weiner42a30032019-05-14 15:47:12 -0700920 for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
921 atomic_long_add(x, &mi->vmevents[idx]);
Johannes Weinerdb9adbc2019-05-14 15:47:09 -0700922 x = 0;
923 }
924 __this_cpu_write(memcg->vmstats_percpu->events[idx], x);
925}
926
Johannes Weiner42a30032019-05-14 15:47:12 -0700927static unsigned long memcg_events(struct mem_cgroup *memcg, int event)
Johannes Weinere9f89742011-03-23 16:42:37 -0700928{
Chris Down871789d2019-05-14 15:46:57 -0700929 return atomic_long_read(&memcg->vmevents[event]);
Johannes Weinere9f89742011-03-23 16:42:37 -0700930}
931
Johannes Weiner42a30032019-05-14 15:47:12 -0700932static unsigned long memcg_events_local(struct mem_cgroup *memcg, int event)
933{
Johannes Weiner815744d2019-06-13 15:55:46 -0700934 long x = 0;
935 int cpu;
936
937 for_each_possible_cpu(cpu)
938 x += per_cpu(memcg->vmstats_local->events[event], cpu);
939 return x;
Johannes Weiner42a30032019-05-14 15:47:12 -0700940}
941
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700942static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
David Rientjesb070e652013-05-07 16:18:09 -0700943 struct page *page,
Johannes Weiner3fba69a2020-06-03 16:01:31 -0700944 int nr_pages)
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800945{
KAMEZAWA Hiroyukie401f172011-01-20 14:44:23 -0800946 /* pagein of a big page is an event. So, ignore page size */
947 if (nr_pages > 0)
Johannes Weinerc9019e92018-01-31 16:16:37 -0800948 __count_memcg_events(memcg, PGPGIN, 1);
KAMEZAWA Hiroyuki3751d602011-02-01 15:52:45 -0800949 else {
Johannes Weinerc9019e92018-01-31 16:16:37 -0800950 __count_memcg_events(memcg, PGPGOUT, 1);
KAMEZAWA Hiroyuki3751d602011-02-01 15:52:45 -0800951 nr_pages = -nr_pages; /* for event */
952 }
KAMEZAWA Hiroyukie401f172011-01-20 14:44:23 -0800953
Chris Down871789d2019-05-14 15:46:57 -0700954 __this_cpu_add(memcg->vmstats_percpu->nr_page_events, nr_pages);
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800955}
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800956
Johannes Weinerf53d7ce2012-01-12 17:18:23 -0800957static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
958 enum mem_cgroup_events_target target)
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -0800959{
Johannes Weiner7a159cc2011-03-23 16:42:38 -0700960 unsigned long val, next;
961
Chris Down871789d2019-05-14 15:46:57 -0700962 val = __this_cpu_read(memcg->vmstats_percpu->nr_page_events);
963 next = __this_cpu_read(memcg->vmstats_percpu->targets[target]);
Johannes Weiner7a159cc2011-03-23 16:42:38 -0700964 /* from time_after() in jiffies.h */
Michal Hocko6a1a8b82017-07-10 15:48:53 -0700965 if ((long)(next - val) < 0) {
Johannes Weinerf53d7ce2012-01-12 17:18:23 -0800966 switch (target) {
967 case MEM_CGROUP_TARGET_THRESH:
968 next = val + THRESHOLDS_EVENTS_TARGET;
969 break;
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700970 case MEM_CGROUP_TARGET_SOFTLIMIT:
971 next = val + SOFTLIMIT_EVENTS_TARGET;
972 break;
Johannes Weinerf53d7ce2012-01-12 17:18:23 -0800973 default:
974 break;
975 }
Chris Down871789d2019-05-14 15:46:57 -0700976 __this_cpu_write(memcg->vmstats_percpu->targets[target], next);
Johannes Weinerf53d7ce2012-01-12 17:18:23 -0800977 return true;
Johannes Weiner7a159cc2011-03-23 16:42:38 -0700978 }
Johannes Weinerf53d7ce2012-01-12 17:18:23 -0800979 return false;
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -0800980}
981
982/*
983 * Check events in order.
984 *
985 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700986static void memcg_check_events(struct mem_cgroup *memcg, struct page *page)
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -0800987{
988 /* threshold event is triggered in finer grain than soft limit */
Johannes Weinerf53d7ce2012-01-12 17:18:23 -0800989 if (unlikely(mem_cgroup_event_ratelimit(memcg,
990 MEM_CGROUP_TARGET_THRESH))) {
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700991 bool do_softlimit;
Johannes Weinerf53d7ce2012-01-12 17:18:23 -0800992
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700993 do_softlimit = mem_cgroup_event_ratelimit(memcg,
994 MEM_CGROUP_TARGET_SOFTLIMIT);
Johannes Weinerf53d7ce2012-01-12 17:18:23 -0800995 mem_cgroup_threshold(memcg);
Andrew Mortonbb4cc1a82013-09-24 15:27:40 -0700996 if (unlikely(do_softlimit))
997 mem_cgroup_update_tree(memcg, page);
Johannes Weiner0a31bc92014-08-08 14:19:22 -0700998 }
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -0800999}
1000
Balbir Singhcf475ad2008-04-29 01:00:16 -07001001struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
Pavel Emelianov78fb7462008-02-07 00:13:51 -08001002{
Balbir Singh31a78f22008-09-28 23:09:31 +01001003 /*
1004 * mm_update_next_owner() may clear mm->owner to NULL
1005 * if it races with swapoff, page migration, etc.
1006 * So this can be called with p == NULL.
1007 */
1008 if (unlikely(!p))
1009 return NULL;
1010
Tejun Heo073219e2014-02-08 10:36:58 -05001011 return mem_cgroup_from_css(task_css(p, memory_cgrp_id));
Pavel Emelianov78fb7462008-02-07 00:13:51 -08001012}
Michal Hocko33398cf2015-09-08 15:01:02 -07001013EXPORT_SYMBOL(mem_cgroup_from_task);
Pavel Emelianov78fb7462008-02-07 00:13:51 -08001014
Shakeel Buttd46eb14b2018-08-17 15:46:39 -07001015/**
1016 * get_mem_cgroup_from_mm: Obtain a reference on given mm_struct's memcg.
1017 * @mm: mm from which memcg should be extracted. It can be NULL.
1018 *
1019 * Obtain a reference on mm->memcg and returns it if successful. Otherwise
1020 * root_mem_cgroup is returned. However if mem_cgroup is disabled, NULL is
1021 * returned.
1022 */
1023struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm)
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08001024{
Shakeel Buttd46eb14b2018-08-17 15:46:39 -07001025 struct mem_cgroup *memcg;
1026
1027 if (mem_cgroup_disabled())
1028 return NULL;
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07001029
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08001030 rcu_read_lock();
1031 do {
Michal Hocko6f6acb02014-05-22 11:54:19 -07001032 /*
1033 * Page cache insertions can happen withou an
1034 * actual mm context, e.g. during disk probing
1035 * on boot, loopback IO, acct() writes etc.
1036 */
1037 if (unlikely(!mm))
Johannes Weinerdf381972014-04-07 15:37:43 -07001038 memcg = root_mem_cgroup;
Michal Hocko6f6acb02014-05-22 11:54:19 -07001039 else {
1040 memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
1041 if (unlikely(!memcg))
1042 memcg = root_mem_cgroup;
1043 }
Roman Gushchin00d484f2019-11-15 17:34:43 -08001044 } while (!css_tryget(&memcg->css));
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08001045 rcu_read_unlock();
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001046 return memcg;
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08001047}
Shakeel Buttd46eb14b2018-08-17 15:46:39 -07001048EXPORT_SYMBOL(get_mem_cgroup_from_mm);
1049
Roman Gushchin37d59852020-10-17 16:13:50 -07001050static __always_inline struct mem_cgroup *active_memcg(void)
1051{
1052 if (in_interrupt())
1053 return this_cpu_read(int_active_memcg);
1054 else
1055 return current->active_memcg;
1056}
1057
1058static __always_inline struct mem_cgroup *get_active_memcg(void)
1059{
1060 struct mem_cgroup *memcg;
1061
1062 rcu_read_lock();
1063 memcg = active_memcg();
Muchun Song1685bde2021-02-24 12:04:22 -08001064 /* remote memcg must hold a ref. */
1065 if (memcg && WARN_ON_ONCE(!css_tryget(&memcg->css)))
1066 memcg = root_mem_cgroup;
Roman Gushchin37d59852020-10-17 16:13:50 -07001067 rcu_read_unlock();
1068
1069 return memcg;
1070}
1071
Roman Gushchin4127c652020-10-17 16:13:53 -07001072static __always_inline bool memcg_kmem_bypass(void)
1073{
1074 /* Allow remote memcg charging from any context. */
1075 if (unlikely(active_memcg()))
1076 return false;
1077
1078 /* Memcg to charge can't be determined. */
1079 if (in_interrupt() || !current->mm || (current->flags & PF_KTHREAD))
1080 return true;
1081
1082 return false;
1083}
1084
Shakeel Buttf745c6f2018-08-17 15:46:44 -07001085/**
Roman Gushchin37d59852020-10-17 16:13:50 -07001086 * If active memcg is set, do not fallback to current->mm->memcg.
Shakeel Buttd46eb14b2018-08-17 15:46:39 -07001087 */
1088static __always_inline struct mem_cgroup *get_mem_cgroup_from_current(void)
1089{
Roman Gushchin279c3392020-10-17 16:13:44 -07001090 if (memcg_kmem_bypass())
1091 return NULL;
1092
Roman Gushchin37d59852020-10-17 16:13:50 -07001093 if (unlikely(active_memcg()))
1094 return get_active_memcg();
Shakeel Buttd46eb14b2018-08-17 15:46:39 -07001095
Shakeel Buttd46eb14b2018-08-17 15:46:39 -07001096 return get_mem_cgroup_from_mm(current->mm);
1097}
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08001098
Johannes Weiner56600482012-01-12 17:17:59 -08001099/**
1100 * mem_cgroup_iter - iterate over memory cgroup hierarchy
1101 * @root: hierarchy root
1102 * @prev: previously returned memcg, NULL on first invocation
1103 * @reclaim: cookie for shared reclaim walks, NULL for full walks
1104 *
1105 * Returns references to children of the hierarchy below @root, or
1106 * @root itself, or %NULL after a full round-trip.
1107 *
1108 * Caller must pass the return value in @prev on subsequent
1109 * invocations for reference counting, or use mem_cgroup_iter_break()
1110 * to cancel a hierarchy walk before the round-trip is complete.
1111 *
Miaohe Lin05bdc522020-10-13 16:52:45 -07001112 * Reclaimers can specify a node in @reclaim to divide up the memcgs
1113 * in the hierarchy among all concurrent reclaimers operating on the
1114 * same node.
Johannes Weiner56600482012-01-12 17:17:59 -08001115 */
Andrew Morton694fbc02013-09-24 15:27:37 -07001116struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root,
Johannes Weiner56600482012-01-12 17:17:59 -08001117 struct mem_cgroup *prev,
Andrew Morton694fbc02013-09-24 15:27:37 -07001118 struct mem_cgroup_reclaim_cookie *reclaim)
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07001119{
Kees Cook3f649ab2020-06-03 13:09:38 -07001120 struct mem_cgroup_reclaim_iter *iter;
Johannes Weiner5ac8fb32014-12-10 15:42:39 -08001121 struct cgroup_subsys_state *css = NULL;
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001122 struct mem_cgroup *memcg = NULL;
Johannes Weiner5ac8fb32014-12-10 15:42:39 -08001123 struct mem_cgroup *pos = NULL;
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001124
Andrew Morton694fbc02013-09-24 15:27:37 -07001125 if (mem_cgroup_disabled())
1126 return NULL;
Johannes Weiner56600482012-01-12 17:17:59 -08001127
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -07001128 if (!root)
1129 root = root_mem_cgroup;
1130
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001131 if (prev && !reclaim)
Johannes Weiner5ac8fb32014-12-10 15:42:39 -08001132 pos = prev;
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001133
Michal Hocko542f85f2013-04-29 15:07:15 -07001134 rcu_read_lock();
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001135
Johannes Weiner5ac8fb32014-12-10 15:42:39 -08001136 if (reclaim) {
Mel Gormanef8f2322016-07-28 15:46:05 -07001137 struct mem_cgroup_per_node *mz;
Johannes Weiner527a5ec2012-01-12 17:17:55 -08001138
Mel Gormanef8f2322016-07-28 15:46:05 -07001139 mz = mem_cgroup_nodeinfo(root, reclaim->pgdat->node_id);
Yafang Shao9da83f32019-11-30 17:50:03 -08001140 iter = &mz->iter;
Michal Hocko5f578162013-04-29 15:07:17 -07001141
Johannes Weiner5ac8fb32014-12-10 15:42:39 -08001142 if (prev && reclaim->generation != iter->generation)
Michal Hocko542f85f2013-04-29 15:07:15 -07001143 goto out_unlock;
Johannes Weiner5ac8fb32014-12-10 15:42:39 -08001144
Vladimir Davydov6df38682015-12-29 14:54:10 -08001145 while (1) {
Jason Low4db0c3c2015-04-15 16:14:08 -07001146 pos = READ_ONCE(iter->position);
Vladimir Davydov6df38682015-12-29 14:54:10 -08001147 if (!pos || css_tryget(&pos->css))
1148 break;
Johannes Weiner5ac8fb32014-12-10 15:42:39 -08001149 /*
Vladimir Davydov6df38682015-12-29 14:54:10 -08001150 * css reference reached zero, so iter->position will
1151 * be cleared by ->css_released. However, we should not
1152 * rely on this happening soon, because ->css_released
1153 * is called from a work queue, and by busy-waiting we
1154 * might block it. So we clear iter->position right
1155 * away.
Johannes Weiner5ac8fb32014-12-10 15:42:39 -08001156 */
Vladimir Davydov6df38682015-12-29 14:54:10 -08001157 (void)cmpxchg(&iter->position, pos, NULL);
1158 }
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001159 }
Johannes Weiner5ac8fb32014-12-10 15:42:39 -08001160
1161 if (pos)
1162 css = &pos->css;
1163
1164 for (;;) {
1165 css = css_next_descendant_pre(css, &root->css);
1166 if (!css) {
1167 /*
1168 * Reclaimers share the hierarchy walk, and a
1169 * new one might jump in right at the end of
1170 * the hierarchy - make sure they see at least
1171 * one group and restart from the beginning.
1172 */
1173 if (!prev)
1174 continue;
1175 break;
1176 }
1177
1178 /*
1179 * Verify the css and acquire a reference. The root
1180 * is provided by the caller, so we know it's alive
1181 * and kicking, and don't take an extra reference.
1182 */
1183 memcg = mem_cgroup_from_css(css);
1184
1185 if (css == &root->css)
1186 break;
1187
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08001188 if (css_tryget(css))
1189 break;
Johannes Weiner5ac8fb32014-12-10 15:42:39 -08001190
1191 memcg = NULL;
1192 }
1193
1194 if (reclaim) {
Johannes Weiner5ac8fb32014-12-10 15:42:39 -08001195 /*
Vladimir Davydov6df38682015-12-29 14:54:10 -08001196 * The position could have already been updated by a competing
1197 * thread, so check that the value hasn't changed since we read
1198 * it to avoid reclaiming from the same cgroup twice.
Johannes Weiner5ac8fb32014-12-10 15:42:39 -08001199 */
Vladimir Davydov6df38682015-12-29 14:54:10 -08001200 (void)cmpxchg(&iter->position, pos, memcg);
1201
Johannes Weiner5ac8fb32014-12-10 15:42:39 -08001202 if (pos)
1203 css_put(&pos->css);
1204
1205 if (!memcg)
1206 iter->generation++;
1207 else if (!prev)
1208 reclaim->generation = iter->generation;
1209 }
1210
Michal Hocko542f85f2013-04-29 15:07:15 -07001211out_unlock:
1212 rcu_read_unlock();
Michal Hockoc40046f2013-04-29 15:07:14 -07001213 if (prev && prev != root)
1214 css_put(&prev->css);
1215
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001216 return memcg;
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001217}
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001218
Johannes Weiner56600482012-01-12 17:17:59 -08001219/**
1220 * mem_cgroup_iter_break - abort a hierarchy walk prematurely
1221 * @root: hierarchy root
1222 * @prev: last visited hierarchy member as returned by mem_cgroup_iter()
1223 */
1224void mem_cgroup_iter_break(struct mem_cgroup *root,
1225 struct mem_cgroup *prev)
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001226{
1227 if (!root)
1228 root = root_mem_cgroup;
1229 if (prev && prev != root)
1230 css_put(&prev->css);
1231}
1232
Miles Chen54a83d62019-08-13 15:37:28 -07001233static void __invalidate_reclaim_iterators(struct mem_cgroup *from,
1234 struct mem_cgroup *dead_memcg)
Vladimir Davydov6df38682015-12-29 14:54:10 -08001235{
Vladimir Davydov6df38682015-12-29 14:54:10 -08001236 struct mem_cgroup_reclaim_iter *iter;
Mel Gormanef8f2322016-07-28 15:46:05 -07001237 struct mem_cgroup_per_node *mz;
1238 int nid;
Vladimir Davydov6df38682015-12-29 14:54:10 -08001239
Miles Chen54a83d62019-08-13 15:37:28 -07001240 for_each_node(nid) {
1241 mz = mem_cgroup_nodeinfo(from, nid);
Yafang Shao9da83f32019-11-30 17:50:03 -08001242 iter = &mz->iter;
1243 cmpxchg(&iter->position, dead_memcg, NULL);
Vladimir Davydov6df38682015-12-29 14:54:10 -08001244 }
1245}
1246
Miles Chen54a83d62019-08-13 15:37:28 -07001247static void invalidate_reclaim_iterators(struct mem_cgroup *dead_memcg)
1248{
1249 struct mem_cgroup *memcg = dead_memcg;
1250 struct mem_cgroup *last;
1251
1252 do {
1253 __invalidate_reclaim_iterators(memcg, dead_memcg);
1254 last = memcg;
1255 } while ((memcg = parent_mem_cgroup(memcg)));
1256
1257 /*
1258 * When cgruop1 non-hierarchy mode is used,
1259 * parent_mem_cgroup() does not walk all the way up to the
1260 * cgroup root (root_mem_cgroup). So we have to handle
1261 * dead_memcg from cgroup root separately.
1262 */
1263 if (last != root_mem_cgroup)
1264 __invalidate_reclaim_iterators(root_mem_cgroup,
1265 dead_memcg);
1266}
1267
Johannes Weiner925b7672012-01-12 17:18:15 -08001268/**
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -07001269 * mem_cgroup_scan_tasks - iterate over tasks of a memory cgroup hierarchy
1270 * @memcg: hierarchy root
1271 * @fn: function to call for each task
1272 * @arg: argument passed to @fn
1273 *
1274 * This function iterates over tasks attached to @memcg or to any of its
1275 * descendants and calls @fn for each task. If @fn returns a non-zero
1276 * value, the function breaks the iteration loop and returns the value.
1277 * Otherwise, it will iterate over all tasks and return 0.
1278 *
1279 * This function must not be called for the root memory cgroup.
1280 */
1281int mem_cgroup_scan_tasks(struct mem_cgroup *memcg,
1282 int (*fn)(struct task_struct *, void *), void *arg)
1283{
1284 struct mem_cgroup *iter;
1285 int ret = 0;
1286
1287 BUG_ON(memcg == root_mem_cgroup);
1288
1289 for_each_mem_cgroup_tree(iter, memcg) {
1290 struct css_task_iter it;
1291 struct task_struct *task;
1292
Tetsuo Handaf168a9a2019-07-11 21:00:20 -07001293 css_task_iter_start(&iter->css, CSS_TASK_ITER_PROCS, &it);
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -07001294 while (!ret && (task = css_task_iter_next(&it)))
1295 ret = fn(task, arg);
1296 css_task_iter_end(&it);
1297 if (ret) {
1298 mem_cgroup_iter_break(memcg, iter);
1299 break;
1300 }
1301 }
1302 return ret;
1303}
1304
Alex Shi6168d0d2020-12-15 12:34:29 -08001305#ifdef CONFIG_DEBUG_VM
1306void lruvec_memcg_debug(struct lruvec *lruvec, struct page *page)
1307{
1308 struct mem_cgroup *memcg;
1309
1310 if (mem_cgroup_disabled())
1311 return;
1312
1313 memcg = page_memcg(page);
1314
1315 if (!memcg)
1316 VM_BUG_ON_PAGE(lruvec_memcg(lruvec) != root_mem_cgroup, page);
1317 else
1318 VM_BUG_ON_PAGE(lruvec_memcg(lruvec) != memcg, page);
1319}
1320#endif
1321
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -07001322/**
Alex Shi6168d0d2020-12-15 12:34:29 -08001323 * lock_page_lruvec - lock and return lruvec for a given page.
1324 * @page: the page
1325 *
Alex Shid7e3aba2021-02-24 12:03:47 -08001326 * These functions are safe to use under any of the following conditions:
1327 * - page locked
1328 * - PageLRU cleared
1329 * - lock_page_memcg()
1330 * - page->_refcount is zero
Alex Shi6168d0d2020-12-15 12:34:29 -08001331 */
1332struct lruvec *lock_page_lruvec(struct page *page)
1333{
1334 struct lruvec *lruvec;
1335 struct pglist_data *pgdat = page_pgdat(page);
1336
Alex Shi6168d0d2020-12-15 12:34:29 -08001337 lruvec = mem_cgroup_page_lruvec(page, pgdat);
1338 spin_lock(&lruvec->lru_lock);
Alex Shi6168d0d2020-12-15 12:34:29 -08001339
1340 lruvec_memcg_debug(lruvec, page);
1341
1342 return lruvec;
1343}
1344
1345struct lruvec *lock_page_lruvec_irq(struct page *page)
1346{
1347 struct lruvec *lruvec;
1348 struct pglist_data *pgdat = page_pgdat(page);
1349
Alex Shi6168d0d2020-12-15 12:34:29 -08001350 lruvec = mem_cgroup_page_lruvec(page, pgdat);
1351 spin_lock_irq(&lruvec->lru_lock);
Alex Shi6168d0d2020-12-15 12:34:29 -08001352
1353 lruvec_memcg_debug(lruvec, page);
1354
1355 return lruvec;
1356}
1357
1358struct lruvec *lock_page_lruvec_irqsave(struct page *page, unsigned long *flags)
1359{
1360 struct lruvec *lruvec;
1361 struct pglist_data *pgdat = page_pgdat(page);
1362
Alex Shi6168d0d2020-12-15 12:34:29 -08001363 lruvec = mem_cgroup_page_lruvec(page, pgdat);
1364 spin_lock_irqsave(&lruvec->lru_lock, *flags);
Alex Shi6168d0d2020-12-15 12:34:29 -08001365
1366 lruvec_memcg_debug(lruvec, page);
1367
1368 return lruvec;
1369}
1370
1371/**
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001372 * mem_cgroup_update_lru_size - account for adding or removing an lru page
1373 * @lruvec: mem_cgroup per zone lru vector
1374 * @lru: index of lru list the page is sitting on
Michal Hockob4536f0c82017-01-10 16:58:04 -08001375 * @zid: zone id of the accounted pages
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001376 * @nr_pages: positive when adding or negative when removing
Johannes Weiner925b7672012-01-12 17:18:15 -08001377 *
Hugh Dickinsca707232016-05-19 17:12:35 -07001378 * This function must be called under lru_lock, just before a page is added
1379 * to or just after a page is removed from an lru list (that ordering being
1380 * so as to allow it to check that lru_size 0 is consistent with list_empty).
Johannes Weiner925b7672012-01-12 17:18:15 -08001381 */
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001382void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
Michal Hockob4536f0c82017-01-10 16:58:04 -08001383 int zid, int nr_pages)
Johannes Weiner925b7672012-01-12 17:18:15 -08001384{
Mel Gormanef8f2322016-07-28 15:46:05 -07001385 struct mem_cgroup_per_node *mz;
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001386 unsigned long *lru_size;
Hugh Dickinsca707232016-05-19 17:12:35 -07001387 long size;
Johannes Weiner925b7672012-01-12 17:18:15 -08001388
1389 if (mem_cgroup_disabled())
1390 return;
1391
Mel Gormanef8f2322016-07-28 15:46:05 -07001392 mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
Michal Hockob4536f0c82017-01-10 16:58:04 -08001393 lru_size = &mz->lru_zone_size[zid][lru];
Hugh Dickinsca707232016-05-19 17:12:35 -07001394
1395 if (nr_pages < 0)
1396 *lru_size += nr_pages;
1397
1398 size = *lru_size;
Michal Hockob4536f0c82017-01-10 16:58:04 -08001399 if (WARN_ONCE(size < 0,
1400 "%s(%p, %d, %d): lru_size %ld\n",
1401 __func__, lruvec, lru, nr_pages, size)) {
Hugh Dickinsca707232016-05-19 17:12:35 -07001402 VM_BUG_ON(1);
1403 *lru_size = 0;
1404 }
1405
1406 if (nr_pages > 0)
1407 *lru_size += nr_pages;
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08001408}
KAMEZAWA Hiroyuki544122e2009-01-07 18:08:34 -08001409
Johannes Weiner19942822011-02-01 15:52:43 -08001410/**
Johannes Weiner9d11ea92011-03-23 16:42:21 -07001411 * mem_cgroup_margin - calculate chargeable space of a memory cgroup
Wanpeng Lidad75572012-06-20 12:53:01 -07001412 * @memcg: the memory cgroup
Johannes Weiner19942822011-02-01 15:52:43 -08001413 *
Johannes Weiner9d11ea92011-03-23 16:42:21 -07001414 * Returns the maximum amount of memory @mem can be charged with, in
Johannes Weiner7ec99d62011-03-23 16:42:36 -07001415 * pages.
Johannes Weiner19942822011-02-01 15:52:43 -08001416 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001417static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg)
Johannes Weiner19942822011-02-01 15:52:43 -08001418{
Johannes Weiner3e32cb22014-12-10 15:42:31 -08001419 unsigned long margin = 0;
1420 unsigned long count;
1421 unsigned long limit;
Johannes Weiner9d11ea92011-03-23 16:42:21 -07001422
Johannes Weiner3e32cb22014-12-10 15:42:31 -08001423 count = page_counter_read(&memcg->memory);
Roman Gushchinbbec2e12018-06-07 17:06:18 -07001424 limit = READ_ONCE(memcg->memory.max);
Johannes Weiner3e32cb22014-12-10 15:42:31 -08001425 if (count < limit)
1426 margin = limit - count;
1427
Johannes Weiner7941d212016-01-14 15:21:23 -08001428 if (do_memsw_account()) {
Johannes Weiner3e32cb22014-12-10 15:42:31 -08001429 count = page_counter_read(&memcg->memsw);
Roman Gushchinbbec2e12018-06-07 17:06:18 -07001430 limit = READ_ONCE(memcg->memsw.max);
Kaixu Xia1c4448e2020-06-01 21:49:36 -07001431 if (count < limit)
Johannes Weiner3e32cb22014-12-10 15:42:31 -08001432 margin = min(margin, limit - count);
Li RongQingcbedbac2016-05-27 14:27:43 -07001433 else
1434 margin = 0;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08001435 }
1436
1437 return margin;
Johannes Weiner19942822011-02-01 15:52:43 -08001438}
1439
KAMEZAWA Hiroyuki619d0942012-03-21 16:34:23 -07001440/*
Qiang Huangbdcbb652014-06-04 16:08:21 -07001441 * A routine for checking "mem" is under move_account() or not.
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001442 *
Qiang Huangbdcbb652014-06-04 16:08:21 -07001443 * Checking a cgroup is mc.from or mc.to or under hierarchy of
1444 * moving cgroups. This is for waiting at high-memory pressure
1445 * caused by "move".
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001446 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001447static bool mem_cgroup_under_move(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001448{
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07001449 struct mem_cgroup *from;
1450 struct mem_cgroup *to;
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001451 bool ret = false;
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07001452 /*
1453 * Unlike task_move routines, we access mc.to, mc.from not under
1454 * mutual exclusion by cgroup_mutex. Here, we take spinlock instead.
1455 */
1456 spin_lock(&mc.lock);
1457 from = mc.from;
1458 to = mc.to;
1459 if (!from)
1460 goto unlock;
Michal Hocko3e920412011-07-26 16:08:29 -07001461
Johannes Weiner2314b422014-12-10 15:44:33 -08001462 ret = mem_cgroup_is_descendant(from, memcg) ||
1463 mem_cgroup_is_descendant(to, memcg);
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07001464unlock:
1465 spin_unlock(&mc.lock);
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001466 return ret;
1467}
1468
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001469static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001470{
1471 if (mc.moving_task && current != mc.moving_task) {
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001472 if (mem_cgroup_under_move(memcg)) {
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001473 DEFINE_WAIT(wait);
1474 prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
1475 /* moving charge context might have finished. */
1476 if (mc.moving_task)
1477 schedule();
1478 finish_wait(&mc.waitq, &wait);
1479 return true;
1480 }
1481 }
1482 return false;
1483}
1484
Muchun Song5f9a4f42020-10-13 16:52:59 -07001485struct memory_stat {
1486 const char *name;
Muchun Song5f9a4f42020-10-13 16:52:59 -07001487 unsigned int idx;
1488};
1489
Muchun Song57b28472021-02-24 12:03:31 -08001490static const struct memory_stat memory_stats[] = {
Muchun Songfff66b72021-02-24 12:03:43 -08001491 { "anon", NR_ANON_MAPPED },
1492 { "file", NR_FILE_PAGES },
1493 { "kernel_stack", NR_KERNEL_STACK_KB },
1494 { "pagetables", NR_PAGETABLE },
1495 { "percpu", MEMCG_PERCPU_B },
1496 { "sock", MEMCG_SOCK },
1497 { "shmem", NR_SHMEM },
1498 { "file_mapped", NR_FILE_MAPPED },
1499 { "file_dirty", NR_FILE_DIRTY },
1500 { "file_writeback", NR_WRITEBACK },
Shakeel Buttb6038942021-02-24 12:03:55 -08001501#ifdef CONFIG_SWAP
1502 { "swapcached", NR_SWAPCACHE },
1503#endif
Muchun Song5f9a4f42020-10-13 16:52:59 -07001504#ifdef CONFIG_TRANSPARENT_HUGEPAGE
Muchun Songfff66b72021-02-24 12:03:43 -08001505 { "anon_thp", NR_ANON_THPS },
1506 { "file_thp", NR_FILE_THPS },
1507 { "shmem_thp", NR_SHMEM_THPS },
Muchun Song5f9a4f42020-10-13 16:52:59 -07001508#endif
Muchun Songfff66b72021-02-24 12:03:43 -08001509 { "inactive_anon", NR_INACTIVE_ANON },
1510 { "active_anon", NR_ACTIVE_ANON },
1511 { "inactive_file", NR_INACTIVE_FILE },
1512 { "active_file", NR_ACTIVE_FILE },
1513 { "unevictable", NR_UNEVICTABLE },
1514 { "slab_reclaimable", NR_SLAB_RECLAIMABLE_B },
1515 { "slab_unreclaimable", NR_SLAB_UNRECLAIMABLE_B },
Muchun Song5f9a4f42020-10-13 16:52:59 -07001516
1517 /* The memory events */
Muchun Songfff66b72021-02-24 12:03:43 -08001518 { "workingset_refault_anon", WORKINGSET_REFAULT_ANON },
1519 { "workingset_refault_file", WORKINGSET_REFAULT_FILE },
1520 { "workingset_activate_anon", WORKINGSET_ACTIVATE_ANON },
1521 { "workingset_activate_file", WORKINGSET_ACTIVATE_FILE },
1522 { "workingset_restore_anon", WORKINGSET_RESTORE_ANON },
1523 { "workingset_restore_file", WORKINGSET_RESTORE_FILE },
1524 { "workingset_nodereclaim", WORKINGSET_NODERECLAIM },
Muchun Song5f9a4f42020-10-13 16:52:59 -07001525};
1526
Muchun Songfff66b72021-02-24 12:03:43 -08001527/* Translate stat items to the correct unit for memory.stat output */
1528static int memcg_page_state_unit(int item)
1529{
1530 switch (item) {
1531 case MEMCG_PERCPU_B:
1532 case NR_SLAB_RECLAIMABLE_B:
1533 case NR_SLAB_UNRECLAIMABLE_B:
1534 case WORKINGSET_REFAULT_ANON:
1535 case WORKINGSET_REFAULT_FILE:
1536 case WORKINGSET_ACTIVATE_ANON:
1537 case WORKINGSET_ACTIVATE_FILE:
1538 case WORKINGSET_RESTORE_ANON:
1539 case WORKINGSET_RESTORE_FILE:
1540 case WORKINGSET_NODERECLAIM:
1541 return 1;
1542 case NR_KERNEL_STACK_KB:
1543 return SZ_1K;
1544 default:
1545 return PAGE_SIZE;
1546 }
1547}
1548
1549static inline unsigned long memcg_page_state_output(struct mem_cgroup *memcg,
1550 int item)
1551{
1552 return memcg_page_state(memcg, item) * memcg_page_state_unit(item);
1553}
1554
Johannes Weinerc8713d02019-07-11 20:55:59 -07001555static char *memory_stat_format(struct mem_cgroup *memcg)
1556{
1557 struct seq_buf s;
1558 int i;
Johannes Weiner71cd3112017-05-03 14:55:13 -07001559
Johannes Weinerc8713d02019-07-11 20:55:59 -07001560 seq_buf_init(&s, kmalloc(PAGE_SIZE, GFP_KERNEL), PAGE_SIZE);
1561 if (!s.buffer)
1562 return NULL;
1563
1564 /*
1565 * Provide statistics on the state of the memory subsystem as
1566 * well as cumulative event counters that show past behavior.
1567 *
1568 * This list is ordered following a combination of these gradients:
1569 * 1) generic big picture -> specifics and details
1570 * 2) reflecting userspace activity -> reflecting kernel heuristics
1571 *
1572 * Current memory state:
1573 */
1574
Muchun Song5f9a4f42020-10-13 16:52:59 -07001575 for (i = 0; i < ARRAY_SIZE(memory_stats); i++) {
1576 u64 size;
Johannes Weinerc8713d02019-07-11 20:55:59 -07001577
Muchun Songfff66b72021-02-24 12:03:43 -08001578 size = memcg_page_state_output(memcg, memory_stats[i].idx);
Muchun Song5f9a4f42020-10-13 16:52:59 -07001579 seq_buf_printf(&s, "%s %llu\n", memory_stats[i].name, size);
Johannes Weinerc8713d02019-07-11 20:55:59 -07001580
Muchun Song5f9a4f42020-10-13 16:52:59 -07001581 if (unlikely(memory_stats[i].idx == NR_SLAB_UNRECLAIMABLE_B)) {
Muchun Songfff66b72021-02-24 12:03:43 -08001582 size += memcg_page_state_output(memcg,
1583 NR_SLAB_RECLAIMABLE_B);
Muchun Song5f9a4f42020-10-13 16:52:59 -07001584 seq_buf_printf(&s, "slab %llu\n", size);
1585 }
1586 }
Johannes Weinerc8713d02019-07-11 20:55:59 -07001587
1588 /* Accumulated memory events */
1589
Konstantin Khlebnikovebc5d83d2019-12-04 16:49:53 -08001590 seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGFAULT),
1591 memcg_events(memcg, PGFAULT));
1592 seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGMAJFAULT),
1593 memcg_events(memcg, PGMAJFAULT));
Konstantin Khlebnikovebc5d83d2019-12-04 16:49:53 -08001594 seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGREFILL),
1595 memcg_events(memcg, PGREFILL));
Johannes Weinerc8713d02019-07-11 20:55:59 -07001596 seq_buf_printf(&s, "pgscan %lu\n",
1597 memcg_events(memcg, PGSCAN_KSWAPD) +
1598 memcg_events(memcg, PGSCAN_DIRECT));
1599 seq_buf_printf(&s, "pgsteal %lu\n",
1600 memcg_events(memcg, PGSTEAL_KSWAPD) +
1601 memcg_events(memcg, PGSTEAL_DIRECT));
Konstantin Khlebnikovebc5d83d2019-12-04 16:49:53 -08001602 seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGACTIVATE),
1603 memcg_events(memcg, PGACTIVATE));
1604 seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGDEACTIVATE),
1605 memcg_events(memcg, PGDEACTIVATE));
1606 seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGLAZYFREE),
1607 memcg_events(memcg, PGLAZYFREE));
1608 seq_buf_printf(&s, "%s %lu\n", vm_event_name(PGLAZYFREED),
1609 memcg_events(memcg, PGLAZYFREED));
Johannes Weinerc8713d02019-07-11 20:55:59 -07001610
1611#ifdef CONFIG_TRANSPARENT_HUGEPAGE
Konstantin Khlebnikovebc5d83d2019-12-04 16:49:53 -08001612 seq_buf_printf(&s, "%s %lu\n", vm_event_name(THP_FAULT_ALLOC),
Johannes Weinerc8713d02019-07-11 20:55:59 -07001613 memcg_events(memcg, THP_FAULT_ALLOC));
Konstantin Khlebnikovebc5d83d2019-12-04 16:49:53 -08001614 seq_buf_printf(&s, "%s %lu\n", vm_event_name(THP_COLLAPSE_ALLOC),
Johannes Weinerc8713d02019-07-11 20:55:59 -07001615 memcg_events(memcg, THP_COLLAPSE_ALLOC));
1616#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1617
1618 /* The above should easily fit into one page */
1619 WARN_ON_ONCE(seq_buf_has_overflowed(&s));
1620
1621 return s.buffer;
1622}
Johannes Weiner71cd3112017-05-03 14:55:13 -07001623
Sha Zhengju58cf1882013-02-22 16:32:05 -08001624#define K(x) ((x) << (PAGE_SHIFT-10))
Balbir Singhe2224322009-04-02 16:57:39 -07001625/**
yuzhoujianf0c867d2018-12-28 00:36:10 -08001626 * mem_cgroup_print_oom_context: Print OOM information relevant to
1627 * memory controller.
Balbir Singhe2224322009-04-02 16:57:39 -07001628 * @memcg: The memory cgroup that went over limit
1629 * @p: Task that is going to be killed
1630 *
1631 * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
1632 * enabled
1633 */
yuzhoujianf0c867d2018-12-28 00:36:10 -08001634void mem_cgroup_print_oom_context(struct mem_cgroup *memcg, struct task_struct *p)
1635{
1636 rcu_read_lock();
1637
1638 if (memcg) {
1639 pr_cont(",oom_memcg=");
1640 pr_cont_cgroup_path(memcg->css.cgroup);
1641 } else
1642 pr_cont(",global_oom");
1643 if (p) {
1644 pr_cont(",task_memcg=");
1645 pr_cont_cgroup_path(task_cgroup(p, memory_cgrp_id));
1646 }
1647 rcu_read_unlock();
1648}
1649
1650/**
1651 * mem_cgroup_print_oom_meminfo: Print OOM memory information relevant to
1652 * memory controller.
1653 * @memcg: The memory cgroup that went over limit
1654 */
1655void mem_cgroup_print_oom_meminfo(struct mem_cgroup *memcg)
Balbir Singhe2224322009-04-02 16:57:39 -07001656{
Johannes Weinerc8713d02019-07-11 20:55:59 -07001657 char *buf;
Balbir Singhe2224322009-04-02 16:57:39 -07001658
Johannes Weiner3e32cb22014-12-10 15:42:31 -08001659 pr_info("memory: usage %llukB, limit %llukB, failcnt %lu\n",
1660 K((u64)page_counter_read(&memcg->memory)),
Chris Down15b42562020-04-01 21:07:20 -07001661 K((u64)READ_ONCE(memcg->memory.max)), memcg->memory.failcnt);
Johannes Weinerc8713d02019-07-11 20:55:59 -07001662 if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
1663 pr_info("swap: usage %llukB, limit %llukB, failcnt %lu\n",
1664 K((u64)page_counter_read(&memcg->swap)),
Chris Down32d087c2020-04-01 21:07:30 -07001665 K((u64)READ_ONCE(memcg->swap.max)), memcg->swap.failcnt);
Johannes Weinerc8713d02019-07-11 20:55:59 -07001666 else {
1667 pr_info("memory+swap: usage %llukB, limit %llukB, failcnt %lu\n",
1668 K((u64)page_counter_read(&memcg->memsw)),
1669 K((u64)memcg->memsw.max), memcg->memsw.failcnt);
1670 pr_info("kmem: usage %llukB, limit %llukB, failcnt %lu\n",
1671 K((u64)page_counter_read(&memcg->kmem)),
1672 K((u64)memcg->kmem.max), memcg->kmem.failcnt);
Sha Zhengju58cf1882013-02-22 16:32:05 -08001673 }
Johannes Weinerc8713d02019-07-11 20:55:59 -07001674
1675 pr_info("Memory cgroup stats for ");
1676 pr_cont_cgroup_path(memcg->css.cgroup);
1677 pr_cont(":");
1678 buf = memory_stat_format(memcg);
1679 if (!buf)
1680 return;
1681 pr_info("%s", buf);
1682 kfree(buf);
Balbir Singhe2224322009-04-02 16:57:39 -07001683}
1684
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07001685/*
David Rientjesa63d83f2010-08-09 17:19:46 -07001686 * Return the memory (and swap, if configured) limit for a memcg.
1687 */
Roman Gushchinbbec2e12018-06-07 17:06:18 -07001688unsigned long mem_cgroup_get_max(struct mem_cgroup *memcg)
David Rientjesa63d83f2010-08-09 17:19:46 -07001689{
Waiman Long8d387a52020-10-13 16:52:52 -07001690 unsigned long max = READ_ONCE(memcg->memory.max);
David Rientjesa63d83f2010-08-09 17:19:46 -07001691
Waiman Long8d387a52020-10-13 16:52:52 -07001692 if (cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
1693 if (mem_cgroup_swappiness(memcg))
1694 max += min(READ_ONCE(memcg->swap.max),
1695 (unsigned long)total_swap_pages);
1696 } else { /* v1 */
1697 if (mem_cgroup_swappiness(memcg)) {
1698 /* Calculate swap excess capacity from memsw limit */
1699 unsigned long swap = READ_ONCE(memcg->memsw.max) - max;
Michal Hocko9a5a8f12012-11-16 14:14:49 -08001700
Waiman Long8d387a52020-10-13 16:52:52 -07001701 max += min(swap, (unsigned long)total_swap_pages);
1702 }
Michal Hocko9a5a8f12012-11-16 14:14:49 -08001703 }
Roman Gushchinbbec2e12018-06-07 17:06:18 -07001704 return max;
David Rientjesa63d83f2010-08-09 17:19:46 -07001705}
1706
Chris Down9783aa92019-10-06 17:58:32 -07001707unsigned long mem_cgroup_size(struct mem_cgroup *memcg)
1708{
1709 return page_counter_read(&memcg->memory);
1710}
1711
Johannes Weinerb6e6edc2016-03-17 14:20:28 -07001712static bool mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,
David Rientjes19965462012-12-11 16:00:26 -08001713 int order)
David Rientjes9cbb78b2012-07-31 16:43:44 -07001714{
David Rientjes6e0fc462015-09-08 15:00:36 -07001715 struct oom_control oc = {
1716 .zonelist = NULL,
1717 .nodemask = NULL,
Vladimir Davydov2a966b72016-07-26 15:22:33 -07001718 .memcg = memcg,
David Rientjes6e0fc462015-09-08 15:00:36 -07001719 .gfp_mask = gfp_mask,
1720 .order = order,
David Rientjes6e0fc462015-09-08 15:00:36 -07001721 };
Yafang Shao1378b372020-08-06 23:22:08 -07001722 bool ret = true;
David Rientjes9cbb78b2012-07-31 16:43:44 -07001723
Tetsuo Handa7775fac2019-03-05 15:46:47 -08001724 if (mutex_lock_killable(&oom_lock))
1725 return true;
Yafang Shao1378b372020-08-06 23:22:08 -07001726
1727 if (mem_cgroup_margin(memcg) >= (1 << order))
1728 goto unlock;
1729
Tetsuo Handa7775fac2019-03-05 15:46:47 -08001730 /*
1731 * A few threads which were not waiting at mutex_lock_killable() can
1732 * fail to bail out. Therefore, check again after holding oom_lock.
1733 */
1734 ret = should_force_charge() || out_of_memory(&oc);
Yafang Shao1378b372020-08-06 23:22:08 -07001735
1736unlock:
Johannes Weinerdc564012015-06-24 16:57:19 -07001737 mutex_unlock(&oom_lock);
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -07001738 return ret;
David Rientjes9cbb78b2012-07-31 16:43:44 -07001739}
1740
Andrew Morton0608f432013-09-24 15:27:41 -07001741static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg,
Mel Gormanef8f2322016-07-28 15:46:05 -07001742 pg_data_t *pgdat,
Andrew Morton0608f432013-09-24 15:27:41 -07001743 gfp_t gfp_mask,
1744 unsigned long *total_scanned)
Balbir Singh6d61ef42009-01-07 18:08:06 -08001745{
Andrew Morton0608f432013-09-24 15:27:41 -07001746 struct mem_cgroup *victim = NULL;
1747 int total = 0;
1748 int loop = 0;
1749 unsigned long excess;
1750 unsigned long nr_scanned;
1751 struct mem_cgroup_reclaim_cookie reclaim = {
Mel Gormanef8f2322016-07-28 15:46:05 -07001752 .pgdat = pgdat,
Andrew Morton0608f432013-09-24 15:27:41 -07001753 };
Johannes Weiner9d11ea92011-03-23 16:42:21 -07001754
Johannes Weiner3e32cb22014-12-10 15:42:31 -08001755 excess = soft_limit_excess(root_memcg);
Balbir Singh6d61ef42009-01-07 18:08:06 -08001756
Andrew Morton0608f432013-09-24 15:27:41 -07001757 while (1) {
1758 victim = mem_cgroup_iter(root_memcg, victim, &reclaim);
1759 if (!victim) {
1760 loop++;
1761 if (loop >= 2) {
1762 /*
1763 * If we have not been able to reclaim
1764 * anything, it might because there are
1765 * no reclaimable pages under this hierarchy
1766 */
1767 if (!total)
1768 break;
1769 /*
1770 * We want to do more targeted reclaim.
1771 * excess >> 2 is not to excessive so as to
1772 * reclaim too much, nor too less that we keep
1773 * coming back to reclaim from this cgroup
1774 */
1775 if (total >= (excess >> 2) ||
1776 (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS))
1777 break;
1778 }
1779 continue;
1780 }
Mel Gormana9dd0a82016-07-28 15:46:02 -07001781 total += mem_cgroup_shrink_node(victim, gfp_mask, false,
Mel Gormanef8f2322016-07-28 15:46:05 -07001782 pgdat, &nr_scanned);
Andrew Morton0608f432013-09-24 15:27:41 -07001783 *total_scanned += nr_scanned;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08001784 if (!soft_limit_excess(root_memcg))
Andrew Morton0608f432013-09-24 15:27:41 -07001785 break;
Balbir Singh6d61ef42009-01-07 18:08:06 -08001786 }
Andrew Morton0608f432013-09-24 15:27:41 -07001787 mem_cgroup_iter_break(root_memcg, victim);
1788 return total;
Balbir Singh6d61ef42009-01-07 18:08:06 -08001789}
1790
Johannes Weiner0056f4e2013-10-31 16:34:14 -07001791#ifdef CONFIG_LOCKDEP
1792static struct lockdep_map memcg_oom_lock_dep_map = {
1793 .name = "memcg_oom_lock",
1794};
1795#endif
1796
Johannes Weinerfb2a6fc2013-09-12 15:13:43 -07001797static DEFINE_SPINLOCK(memcg_oom_lock);
1798
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001799/*
1800 * Check OOM-Killer is already running under our hierarchy.
1801 * If someone is running, return false.
1802 */
Johannes Weinerfb2a6fc2013-09-12 15:13:43 -07001803static bool mem_cgroup_oom_trylock(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001804{
Michal Hocko79dfdac2011-07-26 16:08:23 -07001805 struct mem_cgroup *iter, *failed = NULL;
KAMEZAWA Hiroyukia636b322009-01-07 18:08:08 -08001806
Johannes Weinerfb2a6fc2013-09-12 15:13:43 -07001807 spin_lock(&memcg_oom_lock);
1808
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001809 for_each_mem_cgroup_tree(iter, memcg) {
Johannes Weiner23751be2011-08-25 15:59:16 -07001810 if (iter->oom_lock) {
Michal Hocko79dfdac2011-07-26 16:08:23 -07001811 /*
1812 * this subtree of our hierarchy is already locked
1813 * so we cannot give a lock.
1814 */
Michal Hocko79dfdac2011-07-26 16:08:23 -07001815 failed = iter;
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001816 mem_cgroup_iter_break(memcg, iter);
1817 break;
Johannes Weiner23751be2011-08-25 15:59:16 -07001818 } else
1819 iter->oom_lock = true;
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001820 }
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001821
Johannes Weinerfb2a6fc2013-09-12 15:13:43 -07001822 if (failed) {
1823 /*
1824 * OK, we failed to lock the whole subtree so we have
1825 * to clean up what we set up to the failing subtree
1826 */
1827 for_each_mem_cgroup_tree(iter, memcg) {
1828 if (iter == failed) {
1829 mem_cgroup_iter_break(memcg, iter);
1830 break;
1831 }
1832 iter->oom_lock = false;
Michal Hocko79dfdac2011-07-26 16:08:23 -07001833 }
Johannes Weiner0056f4e2013-10-31 16:34:14 -07001834 } else
1835 mutex_acquire(&memcg_oom_lock_dep_map, 0, 1, _RET_IP_);
Johannes Weinerfb2a6fc2013-09-12 15:13:43 -07001836
1837 spin_unlock(&memcg_oom_lock);
1838
1839 return !failed;
KAMEZAWA Hiroyukia636b322009-01-07 18:08:08 -08001840}
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07001841
Johannes Weinerfb2a6fc2013-09-12 15:13:43 -07001842static void mem_cgroup_oom_unlock(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07001843{
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001844 struct mem_cgroup *iter;
1845
Johannes Weinerfb2a6fc2013-09-12 15:13:43 -07001846 spin_lock(&memcg_oom_lock);
Qian Cai5facae42019-09-19 12:09:40 -04001847 mutex_release(&memcg_oom_lock_dep_map, _RET_IP_);
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001848 for_each_mem_cgroup_tree(iter, memcg)
Michal Hocko79dfdac2011-07-26 16:08:23 -07001849 iter->oom_lock = false;
Johannes Weinerfb2a6fc2013-09-12 15:13:43 -07001850 spin_unlock(&memcg_oom_lock);
Michal Hocko79dfdac2011-07-26 16:08:23 -07001851}
1852
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001853static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg)
Michal Hocko79dfdac2011-07-26 16:08:23 -07001854{
1855 struct mem_cgroup *iter;
1856
Tejun Heoc2b42d32015-06-24 16:58:23 -07001857 spin_lock(&memcg_oom_lock);
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001858 for_each_mem_cgroup_tree(iter, memcg)
Tejun Heoc2b42d32015-06-24 16:58:23 -07001859 iter->under_oom++;
1860 spin_unlock(&memcg_oom_lock);
Michal Hocko79dfdac2011-07-26 16:08:23 -07001861}
1862
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001863static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg)
Michal Hocko79dfdac2011-07-26 16:08:23 -07001864{
1865 struct mem_cgroup *iter;
1866
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001867 /*
Miaohe Lin7a52d4d82020-10-13 16:53:05 -07001868 * Be careful about under_oom underflows becase a child memcg
1869 * could have been added after mem_cgroup_mark_under_oom.
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001870 */
Tejun Heoc2b42d32015-06-24 16:58:23 -07001871 spin_lock(&memcg_oom_lock);
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001872 for_each_mem_cgroup_tree(iter, memcg)
Tejun Heoc2b42d32015-06-24 16:58:23 -07001873 if (iter->under_oom > 0)
1874 iter->under_oom--;
1875 spin_unlock(&memcg_oom_lock);
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07001876}
1877
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001878static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);
1879
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07001880struct oom_wait_info {
Hugh Dickinsd79154b2012-03-21 16:34:18 -07001881 struct mem_cgroup *memcg;
Ingo Molnarac6424b2017-06-20 12:06:13 +02001882 wait_queue_entry_t wait;
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07001883};
1884
Ingo Molnarac6424b2017-06-20 12:06:13 +02001885static int memcg_oom_wake_function(wait_queue_entry_t *wait,
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07001886 unsigned mode, int sync, void *arg)
1887{
Hugh Dickinsd79154b2012-03-21 16:34:18 -07001888 struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg;
1889 struct mem_cgroup *oom_wait_memcg;
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07001890 struct oom_wait_info *oom_wait_info;
1891
1892 oom_wait_info = container_of(wait, struct oom_wait_info, wait);
Hugh Dickinsd79154b2012-03-21 16:34:18 -07001893 oom_wait_memcg = oom_wait_info->memcg;
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07001894
Johannes Weiner2314b422014-12-10 15:44:33 -08001895 if (!mem_cgroup_is_descendant(wake_memcg, oom_wait_memcg) &&
1896 !mem_cgroup_is_descendant(oom_wait_memcg, wake_memcg))
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07001897 return 0;
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07001898 return autoremove_wake_function(wait, mode, sync, arg);
1899}
1900
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001901static void memcg_oom_recover(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07001902{
Tejun Heoc2b42d32015-06-24 16:58:23 -07001903 /*
1904 * For the following lockless ->under_oom test, the only required
1905 * guarantee is that it must see the state asserted by an OOM when
1906 * this function is called as a result of userland actions
1907 * triggered by the notification of the OOM. This is trivially
1908 * achieved by invoking mem_cgroup_mark_under_oom() before
1909 * triggering notification.
1910 */
1911 if (memcg && memcg->under_oom)
Tejun Heof4b90b702015-06-24 16:58:21 -07001912 __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07001913}
1914
Michal Hocko29ef6802018-08-17 15:47:11 -07001915enum oom_status {
1916 OOM_SUCCESS,
1917 OOM_FAILED,
1918 OOM_ASYNC,
1919 OOM_SKIPPED
1920};
1921
1922static enum oom_status mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order)
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001923{
Michal Hocko7056d3a2018-12-28 00:39:57 -08001924 enum oom_status ret;
1925 bool locked;
1926
Michal Hocko29ef6802018-08-17 15:47:11 -07001927 if (order > PAGE_ALLOC_COSTLY_ORDER)
1928 return OOM_SKIPPED;
1929
Roman Gushchin7a1adfd2018-10-26 15:09:48 -07001930 memcg_memory_event(memcg, MEMCG_OOM);
1931
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001932 /*
Johannes Weiner49426422013-10-16 13:46:59 -07001933 * We are in the middle of the charge context here, so we
1934 * don't want to block when potentially sitting on a callstack
1935 * that holds all kinds of filesystem and mm locks.
1936 *
Michal Hocko29ef6802018-08-17 15:47:11 -07001937 * cgroup1 allows disabling the OOM killer and waiting for outside
1938 * handling until the charge can succeed; remember the context and put
1939 * the task to sleep at the end of the page fault when all locks are
1940 * released.
Johannes Weiner49426422013-10-16 13:46:59 -07001941 *
Michal Hocko29ef6802018-08-17 15:47:11 -07001942 * On the other hand, in-kernel OOM killer allows for an async victim
1943 * memory reclaim (oom_reaper) and that means that we are not solely
1944 * relying on the oom victim to make a forward progress and we can
1945 * invoke the oom killer here.
1946 *
1947 * Please note that mem_cgroup_out_of_memory might fail to find a
1948 * victim and then we have to bail out from the charge path.
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001949 */
Michal Hocko29ef6802018-08-17 15:47:11 -07001950 if (memcg->oom_kill_disable) {
1951 if (!current->in_user_fault)
1952 return OOM_SKIPPED;
1953 css_get(&memcg->css);
1954 current->memcg_in_oom = memcg;
1955 current->memcg_oom_gfp_mask = mask;
1956 current->memcg_oom_order = order;
1957
1958 return OOM_ASYNC;
1959 }
1960
Michal Hocko7056d3a2018-12-28 00:39:57 -08001961 mem_cgroup_mark_under_oom(memcg);
Michal Hocko29ef6802018-08-17 15:47:11 -07001962
Michal Hocko7056d3a2018-12-28 00:39:57 -08001963 locked = mem_cgroup_oom_trylock(memcg);
1964
1965 if (locked)
1966 mem_cgroup_oom_notify(memcg);
1967
1968 mem_cgroup_unmark_under_oom(memcg);
1969 if (mem_cgroup_out_of_memory(memcg, mask, order))
1970 ret = OOM_SUCCESS;
1971 else
1972 ret = OOM_FAILED;
1973
1974 if (locked)
1975 mem_cgroup_oom_unlock(memcg);
1976
1977 return ret;
Johannes Weiner49426422013-10-16 13:46:59 -07001978}
1979
1980/**
1981 * mem_cgroup_oom_synchronize - complete memcg OOM handling
1982 * @handle: actually kill/wait or just clean up the OOM state
1983 *
1984 * This has to be called at the end of a page fault if the memcg OOM
1985 * handler was enabled.
1986 *
1987 * Memcg supports userspace OOM handling where failed allocations must
1988 * sleep on a waitqueue until the userspace task resolves the
1989 * situation. Sleeping directly in the charge context with all kinds
1990 * of locks held is not a good idea, instead we remember an OOM state
1991 * in the task and mem_cgroup_oom_synchronize() has to be called at
1992 * the end of the page fault to complete the OOM handling.
1993 *
1994 * Returns %true if an ongoing memcg OOM situation was detected and
1995 * completed, %false otherwise.
1996 */
1997bool mem_cgroup_oom_synchronize(bool handle)
1998{
Tejun Heo626ebc42015-11-05 18:46:09 -08001999 struct mem_cgroup *memcg = current->memcg_in_oom;
Johannes Weiner49426422013-10-16 13:46:59 -07002000 struct oom_wait_info owait;
2001 bool locked;
2002
2003 /* OOM is global, do not handle */
2004 if (!memcg)
2005 return false;
2006
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -07002007 if (!handle)
Johannes Weiner49426422013-10-16 13:46:59 -07002008 goto cleanup;
2009
2010 owait.memcg = memcg;
2011 owait.wait.flags = 0;
2012 owait.wait.func = memcg_oom_wake_function;
2013 owait.wait.private = current;
Ingo Molnar2055da92017-06-20 12:06:46 +02002014 INIT_LIST_HEAD(&owait.wait.entry);
Johannes Weiner49426422013-10-16 13:46:59 -07002015
2016 prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
Johannes Weinerfb2a6fc2013-09-12 15:13:43 -07002017 mem_cgroup_mark_under_oom(memcg);
2018
2019 locked = mem_cgroup_oom_trylock(memcg);
2020
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07002021 if (locked)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002022 mem_cgroup_oom_notify(memcg);
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08002023
Johannes Weinerfb2a6fc2013-09-12 15:13:43 -07002024 if (locked && !memcg->oom_kill_disable) {
2025 mem_cgroup_unmark_under_oom(memcg);
Johannes Weiner49426422013-10-16 13:46:59 -07002026 finish_wait(&memcg_oom_waitq, &owait.wait);
Tejun Heo626ebc42015-11-05 18:46:09 -08002027 mem_cgroup_out_of_memory(memcg, current->memcg_oom_gfp_mask,
2028 current->memcg_oom_order);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07002029 } else {
Johannes Weiner3812c8c2013-09-12 15:13:44 -07002030 schedule();
Johannes Weiner49426422013-10-16 13:46:59 -07002031 mem_cgroup_unmark_under_oom(memcg);
2032 finish_wait(&memcg_oom_waitq, &owait.wait);
2033 }
2034
2035 if (locked) {
Johannes Weinerfb2a6fc2013-09-12 15:13:43 -07002036 mem_cgroup_oom_unlock(memcg);
2037 /*
2038 * There is no guarantee that an OOM-lock contender
2039 * sees the wakeups triggered by the OOM kill
2040 * uncharges. Wake any sleepers explicitely.
2041 */
2042 memcg_oom_recover(memcg);
2043 }
Johannes Weiner49426422013-10-16 13:46:59 -07002044cleanup:
Tejun Heo626ebc42015-11-05 18:46:09 -08002045 current->memcg_in_oom = NULL;
Johannes Weiner3812c8c2013-09-12 15:13:44 -07002046 css_put(&memcg->css);
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08002047 return true;
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07002048}
2049
Johannes Weinerd7365e72014-10-29 14:50:48 -07002050/**
Roman Gushchin3d8b38e2018-08-21 21:53:54 -07002051 * mem_cgroup_get_oom_group - get a memory cgroup to clean up after OOM
2052 * @victim: task to be killed by the OOM killer
2053 * @oom_domain: memcg in case of memcg OOM, NULL in case of system-wide OOM
2054 *
2055 * Returns a pointer to a memory cgroup, which has to be cleaned up
2056 * by killing all belonging OOM-killable tasks.
2057 *
2058 * Caller has to call mem_cgroup_put() on the returned non-NULL memcg.
2059 */
2060struct mem_cgroup *mem_cgroup_get_oom_group(struct task_struct *victim,
2061 struct mem_cgroup *oom_domain)
2062{
2063 struct mem_cgroup *oom_group = NULL;
2064 struct mem_cgroup *memcg;
2065
2066 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
2067 return NULL;
2068
2069 if (!oom_domain)
2070 oom_domain = root_mem_cgroup;
2071
2072 rcu_read_lock();
2073
2074 memcg = mem_cgroup_from_task(victim);
2075 if (memcg == root_mem_cgroup)
2076 goto out;
2077
2078 /*
Roman Gushchin48fe2672020-04-01 21:07:39 -07002079 * If the victim task has been asynchronously moved to a different
2080 * memory cgroup, we might end up killing tasks outside oom_domain.
2081 * In this case it's better to ignore memory.group.oom.
2082 */
2083 if (unlikely(!mem_cgroup_is_descendant(memcg, oom_domain)))
2084 goto out;
2085
2086 /*
Roman Gushchin3d8b38e2018-08-21 21:53:54 -07002087 * Traverse the memory cgroup hierarchy from the victim task's
2088 * cgroup up to the OOMing cgroup (or root) to find the
2089 * highest-level memory cgroup with oom.group set.
2090 */
2091 for (; memcg; memcg = parent_mem_cgroup(memcg)) {
2092 if (memcg->oom_group)
2093 oom_group = memcg;
2094
2095 if (memcg == oom_domain)
2096 break;
2097 }
2098
2099 if (oom_group)
2100 css_get(&oom_group->css);
2101out:
2102 rcu_read_unlock();
2103
2104 return oom_group;
2105}
2106
2107void mem_cgroup_print_oom_group(struct mem_cgroup *memcg)
2108{
2109 pr_info("Tasks in ");
2110 pr_cont_cgroup_path(memcg->css.cgroup);
2111 pr_cont(" are going to be killed due to memory.oom.group set\n");
2112}
2113
2114/**
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08002115 * lock_page_memcg - lock a page and memcg binding
Johannes Weiner81f8c3a2016-03-15 14:57:04 -07002116 * @page: the page
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07002117 *
Johannes Weiner81f8c3a2016-03-15 14:57:04 -07002118 * This function protects unlocked LRU pages from being moved to
Johannes Weiner739f79f2017-08-18 15:15:48 -07002119 * another cgroup.
2120 *
Johannes Weiner1c824a62021-04-29 22:55:32 -07002121 * It ensures lifetime of the locked memcg. Caller is responsible
2122 * for the lifetime of the page.
Balbir Singhd69b0422009-06-17 16:26:34 -07002123 */
Johannes Weiner1c824a62021-04-29 22:55:32 -07002124void lock_page_memcg(struct page *page)
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07002125{
Johannes Weiner9da7b522020-06-03 16:01:51 -07002126 struct page *head = compound_head(page); /* rmap on tail pages */
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07002127 struct mem_cgroup *memcg;
Johannes Weiner6de22612015-02-11 15:25:01 -08002128 unsigned long flags;
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07002129
Johannes Weiner6de22612015-02-11 15:25:01 -08002130 /*
2131 * The RCU lock is held throughout the transaction. The fast
2132 * path can get away without acquiring the memcg->move_lock
2133 * because page moving starts with an RCU grace period.
Johannes Weiner739f79f2017-08-18 15:15:48 -07002134 */
Johannes Weinerd7365e72014-10-29 14:50:48 -07002135 rcu_read_lock();
2136
2137 if (mem_cgroup_disabled())
Johannes Weiner1c824a62021-04-29 22:55:32 -07002138 return;
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07002139again:
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08002140 memcg = page_memcg(head);
Johannes Weiner29833312014-12-10 15:44:02 -08002141 if (unlikely(!memcg))
Johannes Weiner1c824a62021-04-29 22:55:32 -07002142 return;
Johannes Weinerd7365e72014-10-29 14:50:48 -07002143
Alex Shi20ad50d2020-12-15 12:33:51 -08002144#ifdef CONFIG_PROVE_LOCKING
2145 local_irq_save(flags);
2146 might_lock(&memcg->move_lock);
2147 local_irq_restore(flags);
2148#endif
2149
Qiang Huangbdcbb652014-06-04 16:08:21 -07002150 if (atomic_read(&memcg->moving_account) <= 0)
Johannes Weiner1c824a62021-04-29 22:55:32 -07002151 return;
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07002152
Johannes Weiner6de22612015-02-11 15:25:01 -08002153 spin_lock_irqsave(&memcg->move_lock, flags);
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08002154 if (memcg != page_memcg(head)) {
Johannes Weiner6de22612015-02-11 15:25:01 -08002155 spin_unlock_irqrestore(&memcg->move_lock, flags);
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07002156 goto again;
2157 }
Johannes Weiner6de22612015-02-11 15:25:01 -08002158
2159 /*
Johannes Weiner1c824a62021-04-29 22:55:32 -07002160 * When charge migration first begins, we can have multiple
2161 * critical sections holding the fast-path RCU lock and one
2162 * holding the slowpath move_lock. Track the task who has the
2163 * move_lock for unlock_page_memcg().
Johannes Weiner6de22612015-02-11 15:25:01 -08002164 */
2165 memcg->move_lock_task = current;
2166 memcg->move_lock_flags = flags;
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07002167}
Johannes Weiner81f8c3a2016-03-15 14:57:04 -07002168EXPORT_SYMBOL(lock_page_memcg);
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07002169
Johannes Weiner1c824a62021-04-29 22:55:32 -07002170static void __unlock_page_memcg(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07002171{
Johannes Weiner6de22612015-02-11 15:25:01 -08002172 if (memcg && memcg->move_lock_task == current) {
2173 unsigned long flags = memcg->move_lock_flags;
2174
2175 memcg->move_lock_task = NULL;
2176 memcg->move_lock_flags = 0;
2177
2178 spin_unlock_irqrestore(&memcg->move_lock, flags);
2179 }
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07002180
Johannes Weinerd7365e72014-10-29 14:50:48 -07002181 rcu_read_unlock();
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07002182}
Johannes Weiner739f79f2017-08-18 15:15:48 -07002183
2184/**
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08002185 * unlock_page_memcg - unlock a page and memcg binding
Johannes Weiner739f79f2017-08-18 15:15:48 -07002186 * @page: the page
2187 */
2188void unlock_page_memcg(struct page *page)
2189{
Johannes Weiner9da7b522020-06-03 16:01:51 -07002190 struct page *head = compound_head(page);
2191
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08002192 __unlock_page_memcg(page_memcg(head));
Johannes Weiner739f79f2017-08-18 15:15:48 -07002193}
Johannes Weiner81f8c3a2016-03-15 14:57:04 -07002194EXPORT_SYMBOL(unlock_page_memcg);
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07002195
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002196struct memcg_stock_pcp {
2197 struct mem_cgroup *cached; /* this never be root cgroup */
Johannes Weiner11c9ea42011-03-23 16:42:34 -07002198 unsigned int nr_pages;
Roman Gushchinbf4f0592020-08-06 23:20:49 -07002199
2200#ifdef CONFIG_MEMCG_KMEM
2201 struct obj_cgroup *cached_objcg;
2202 unsigned int nr_bytes;
2203#endif
2204
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002205 struct work_struct work;
KAMEZAWA Hiroyuki26fe6162011-06-15 15:08:45 -07002206 unsigned long flags;
Kirill A. Shutemova0db00f2012-05-29 15:06:56 -07002207#define FLUSHING_CACHED_CHARGE 0
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002208};
2209static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
Michal Hocko9f50fad2011-08-09 11:56:26 +02002210static DEFINE_MUTEX(percpu_charge_mutex);
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002211
Roman Gushchinbf4f0592020-08-06 23:20:49 -07002212#ifdef CONFIG_MEMCG_KMEM
2213static void drain_obj_stock(struct memcg_stock_pcp *stock);
2214static bool obj_stock_flush_required(struct memcg_stock_pcp *stock,
2215 struct mem_cgroup *root_memcg);
2216
2217#else
2218static inline void drain_obj_stock(struct memcg_stock_pcp *stock)
2219{
2220}
2221static bool obj_stock_flush_required(struct memcg_stock_pcp *stock,
2222 struct mem_cgroup *root_memcg)
2223{
2224 return false;
2225}
2226#endif
2227
Suleiman Souhlala0956d52012-12-18 14:21:36 -08002228/**
2229 * consume_stock: Try to consume stocked charge on this cpu.
2230 * @memcg: memcg to consume from.
2231 * @nr_pages: how many pages to charge.
2232 *
2233 * The charges will only happen if @memcg matches the current cpu's memcg
2234 * stock, and at least @nr_pages are available in that stock. Failure to
2235 * service an allocation will refill the stock.
2236 *
2237 * returns true if successful, false otherwise.
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002238 */
Suleiman Souhlala0956d52012-12-18 14:21:36 -08002239static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002240{
2241 struct memcg_stock_pcp *stock;
Johannes Weinerdb2ba40c2016-09-19 14:44:36 -07002242 unsigned long flags;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08002243 bool ret = false;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002244
Johannes Weinera983b5e2018-01-31 16:16:45 -08002245 if (nr_pages > MEMCG_CHARGE_BATCH)
Johannes Weiner3e32cb22014-12-10 15:42:31 -08002246 return ret;
Suleiman Souhlala0956d52012-12-18 14:21:36 -08002247
Johannes Weinerdb2ba40c2016-09-19 14:44:36 -07002248 local_irq_save(flags);
2249
2250 stock = this_cpu_ptr(&memcg_stock);
Johannes Weiner3e32cb22014-12-10 15:42:31 -08002251 if (memcg == stock->cached && stock->nr_pages >= nr_pages) {
Suleiman Souhlala0956d52012-12-18 14:21:36 -08002252 stock->nr_pages -= nr_pages;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08002253 ret = true;
2254 }
Johannes Weinerdb2ba40c2016-09-19 14:44:36 -07002255
2256 local_irq_restore(flags);
2257
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002258 return ret;
2259}
2260
2261/*
Johannes Weiner3e32cb22014-12-10 15:42:31 -08002262 * Returns stocks cached in percpu and reset cached information.
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002263 */
2264static void drain_stock(struct memcg_stock_pcp *stock)
2265{
2266 struct mem_cgroup *old = stock->cached;
2267
Johannes Weiner1a3e1f42020-08-06 23:20:45 -07002268 if (!old)
2269 return;
2270
Johannes Weiner11c9ea42011-03-23 16:42:34 -07002271 if (stock->nr_pages) {
Johannes Weiner3e32cb22014-12-10 15:42:31 -08002272 page_counter_uncharge(&old->memory, stock->nr_pages);
Johannes Weiner7941d212016-01-14 15:21:23 -08002273 if (do_memsw_account())
Johannes Weiner3e32cb22014-12-10 15:42:31 -08002274 page_counter_uncharge(&old->memsw, stock->nr_pages);
Johannes Weiner11c9ea42011-03-23 16:42:34 -07002275 stock->nr_pages = 0;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002276 }
Johannes Weiner1a3e1f42020-08-06 23:20:45 -07002277
2278 css_put(&old->css);
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002279 stock->cached = NULL;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002280}
2281
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002282static void drain_local_stock(struct work_struct *dummy)
2283{
Johannes Weinerdb2ba40c2016-09-19 14:44:36 -07002284 struct memcg_stock_pcp *stock;
2285 unsigned long flags;
2286
Michal Hocko72f01842017-10-03 16:14:53 -07002287 /*
2288 * The only protection from memory hotplug vs. drain_stock races is
2289 * that we always operate on local CPU stock here with IRQ disabled
2290 */
Johannes Weinerdb2ba40c2016-09-19 14:44:36 -07002291 local_irq_save(flags);
2292
2293 stock = this_cpu_ptr(&memcg_stock);
Roman Gushchinbf4f0592020-08-06 23:20:49 -07002294 drain_obj_stock(stock);
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002295 drain_stock(stock);
KAMEZAWA Hiroyuki26fe6162011-06-15 15:08:45 -07002296 clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
Johannes Weinerdb2ba40c2016-09-19 14:44:36 -07002297
2298 local_irq_restore(flags);
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002299}
2300
2301/*
Johannes Weiner3e32cb22014-12-10 15:42:31 -08002302 * Cache charges(val) to local per_cpu area.
Greg Thelen320cc512010-03-15 15:27:28 +01002303 * This will be consumed by consume_stock() function, later.
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002304 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002305static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002306{
Johannes Weinerdb2ba40c2016-09-19 14:44:36 -07002307 struct memcg_stock_pcp *stock;
2308 unsigned long flags;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002309
Johannes Weinerdb2ba40c2016-09-19 14:44:36 -07002310 local_irq_save(flags);
2311
2312 stock = this_cpu_ptr(&memcg_stock);
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002313 if (stock->cached != memcg) { /* reset if necessary */
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002314 drain_stock(stock);
Johannes Weiner1a3e1f42020-08-06 23:20:45 -07002315 css_get(&memcg->css);
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002316 stock->cached = memcg;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002317 }
Johannes Weiner11c9ea42011-03-23 16:42:34 -07002318 stock->nr_pages += nr_pages;
Johannes Weinerdb2ba40c2016-09-19 14:44:36 -07002319
Johannes Weinera983b5e2018-01-31 16:16:45 -08002320 if (stock->nr_pages > MEMCG_CHARGE_BATCH)
Roman Gushchin475d0482017-09-08 16:13:09 -07002321 drain_stock(stock);
2322
Johannes Weinerdb2ba40c2016-09-19 14:44:36 -07002323 local_irq_restore(flags);
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002324}
2325
2326/*
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002327 * Drains all per-CPU charge caches for given root_memcg resp. subtree
Johannes Weiner6d3d6aa2014-12-10 15:42:50 -08002328 * of the hierarchy under it.
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002329 */
Johannes Weiner6d3d6aa2014-12-10 15:42:50 -08002330static void drain_all_stock(struct mem_cgroup *root_memcg)
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002331{
KAMEZAWA Hiroyuki26fe6162011-06-15 15:08:45 -07002332 int cpu, curcpu;
Michal Hockod38144b2011-07-26 16:08:28 -07002333
Johannes Weiner6d3d6aa2014-12-10 15:42:50 -08002334 /* If someone's already draining, avoid adding running more workers. */
2335 if (!mutex_trylock(&percpu_charge_mutex))
2336 return;
Michal Hocko72f01842017-10-03 16:14:53 -07002337 /*
2338 * Notify other cpus that system-wide "drain" is running
2339 * We do not care about races with the cpu hotplug because cpu down
2340 * as well as workers from this path always operate on the local
2341 * per-cpu data. CPU up doesn't touch memcg_stock at all.
2342 */
Johannes Weiner5af12d02011-08-25 15:59:07 -07002343 curcpu = get_cpu();
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002344 for_each_online_cpu(cpu) {
2345 struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002346 struct mem_cgroup *memcg;
Roman Gushchine1a366b2019-09-23 15:34:58 -07002347 bool flush = false;
KAMEZAWA Hiroyuki26fe6162011-06-15 15:08:45 -07002348
Roman Gushchine1a366b2019-09-23 15:34:58 -07002349 rcu_read_lock();
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002350 memcg = stock->cached;
Roman Gushchine1a366b2019-09-23 15:34:58 -07002351 if (memcg && stock->nr_pages &&
2352 mem_cgroup_is_descendant(memcg, root_memcg))
2353 flush = true;
Roman Gushchinbf4f0592020-08-06 23:20:49 -07002354 if (obj_stock_flush_required(stock, root_memcg))
2355 flush = true;
Roman Gushchine1a366b2019-09-23 15:34:58 -07002356 rcu_read_unlock();
2357
2358 if (flush &&
2359 !test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) {
Michal Hockod1a05b62011-07-26 16:08:27 -07002360 if (cpu == curcpu)
2361 drain_local_stock(&stock->work);
2362 else
2363 schedule_work_on(cpu, &stock->work);
2364 }
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002365 }
Johannes Weiner5af12d02011-08-25 15:59:07 -07002366 put_cpu();
Michal Hocko9f50fad2011-08-09 11:56:26 +02002367 mutex_unlock(&percpu_charge_mutex);
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002368}
2369
Sebastian Andrzej Siewior308167f2016-11-03 15:49:59 +01002370static int memcg_hotplug_cpu_dead(unsigned int cpu)
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002371{
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002372 struct memcg_stock_pcp *stock;
Johannes Weinera3d4c052021-04-29 22:56:11 -07002373 struct mem_cgroup *memcg;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002374
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002375 stock = &per_cpu(memcg_stock, cpu);
2376 drain_stock(stock);
Johannes Weinera983b5e2018-01-31 16:16:45 -08002377
2378 for_each_mem_cgroup(memcg) {
Johannes Weinera3d4c052021-04-29 22:56:11 -07002379 struct memcg_vmstats_percpu *statc;
Johannes Weinera983b5e2018-01-31 16:16:45 -08002380 int i;
2381
Johannes Weinera3d4c052021-04-29 22:56:11 -07002382 statc = per_cpu_ptr(memcg->vmstats_percpu, cpu);
2383
Johannes Weinera983b5e2018-01-31 16:16:45 -08002384 for (i = 0; i < MEMCG_NR_STAT; i++) {
2385 int nid;
Johannes Weinera983b5e2018-01-31 16:16:45 -08002386
Johannes Weinera3d4c052021-04-29 22:56:11 -07002387 if (statc->stat[i]) {
2388 mod_memcg_state(memcg, i, statc->stat[i]);
2389 statc->stat[i] = 0;
2390 }
Johannes Weinera983b5e2018-01-31 16:16:45 -08002391
2392 if (i >= NR_VM_NODE_STAT_ITEMS)
2393 continue;
2394
2395 for_each_node(nid) {
Johannes Weinera3d4c052021-04-29 22:56:11 -07002396 struct batched_lruvec_stat *lstatc;
Johannes Weinera983b5e2018-01-31 16:16:45 -08002397 struct mem_cgroup_per_node *pn;
Johannes Weinera3d4c052021-04-29 22:56:11 -07002398 long x;
Johannes Weinera983b5e2018-01-31 16:16:45 -08002399
2400 pn = mem_cgroup_nodeinfo(memcg, nid);
Johannes Weinera3d4c052021-04-29 22:56:11 -07002401 lstatc = per_cpu_ptr(pn->lruvec_stat_cpu, cpu);
2402
2403 x = lstatc->count[i];
2404 lstatc->count[i] = 0;
2405
2406 if (x) {
Johannes Weiner42a30032019-05-14 15:47:12 -07002407 do {
2408 atomic_long_add(x, &pn->lruvec_stat[i]);
2409 } while ((pn = parent_nodeinfo(pn, nid)));
Johannes Weinera3d4c052021-04-29 22:56:11 -07002410 }
Johannes Weinera983b5e2018-01-31 16:16:45 -08002411 }
2412 }
2413
Johannes Weinere27be242018-04-10 16:29:45 -07002414 for (i = 0; i < NR_VM_EVENT_ITEMS; i++) {
Johannes Weinera3d4c052021-04-29 22:56:11 -07002415 if (statc->events[i]) {
2416 count_memcg_events(memcg, i, statc->events[i]);
2417 statc->events[i] = 0;
2418 }
Johannes Weinera983b5e2018-01-31 16:16:45 -08002419 }
2420 }
2421
Sebastian Andrzej Siewior308167f2016-11-03 15:49:59 +01002422 return 0;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002423}
2424
Chris Downb3ff9292020-08-06 23:21:54 -07002425static unsigned long reclaim_high(struct mem_cgroup *memcg,
2426 unsigned int nr_pages,
2427 gfp_t gfp_mask)
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08002428{
Chris Downb3ff9292020-08-06 23:21:54 -07002429 unsigned long nr_reclaimed = 0;
2430
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08002431 do {
Johannes Weinere22c6ed2020-08-06 23:22:15 -07002432 unsigned long pflags;
2433
Jakub Kicinskid1663a92020-06-01 21:49:49 -07002434 if (page_counter_read(&memcg->memory) <=
2435 READ_ONCE(memcg->memory.high))
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08002436 continue;
Johannes Weinere22c6ed2020-08-06 23:22:15 -07002437
Johannes Weinere27be242018-04-10 16:29:45 -07002438 memcg_memory_event(memcg, MEMCG_HIGH);
Johannes Weinere22c6ed2020-08-06 23:22:15 -07002439
2440 psi_memstall_enter(&pflags);
Chris Downb3ff9292020-08-06 23:21:54 -07002441 nr_reclaimed += try_to_free_mem_cgroup_pages(memcg, nr_pages,
2442 gfp_mask, true);
Johannes Weinere22c6ed2020-08-06 23:22:15 -07002443 psi_memstall_leave(&pflags);
Chris Down4bf17302020-04-06 20:03:30 -07002444 } while ((memcg = parent_mem_cgroup(memcg)) &&
2445 !mem_cgroup_is_root(memcg));
Chris Downb3ff9292020-08-06 23:21:54 -07002446
2447 return nr_reclaimed;
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08002448}
2449
2450static void high_work_func(struct work_struct *work)
2451{
2452 struct mem_cgroup *memcg;
2453
2454 memcg = container_of(work, struct mem_cgroup, high_work);
Johannes Weinera983b5e2018-01-31 16:16:45 -08002455 reclaim_high(memcg, MEMCG_CHARGE_BATCH, GFP_KERNEL);
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08002456}
2457
Tejun Heob23afb92015-11-05 18:46:11 -08002458/*
Chris Down0e4b01d2019-09-23 15:34:55 -07002459 * Clamp the maximum sleep time per allocation batch to 2 seconds. This is
2460 * enough to still cause a significant slowdown in most cases, while still
2461 * allowing diagnostics and tracing to proceed without becoming stuck.
2462 */
2463#define MEMCG_MAX_HIGH_DELAY_JIFFIES (2UL*HZ)
2464
2465/*
2466 * When calculating the delay, we use these either side of the exponentiation to
2467 * maintain precision and scale to a reasonable number of jiffies (see the table
2468 * below.
2469 *
2470 * - MEMCG_DELAY_PRECISION_SHIFT: Extra precision bits while translating the
2471 * overage ratio to a delay.
Randy Dunlapac5ddd02020-08-11 18:33:02 -07002472 * - MEMCG_DELAY_SCALING_SHIFT: The number of bits to scale down the
Chris Down0e4b01d2019-09-23 15:34:55 -07002473 * proposed penalty in order to reduce to a reasonable number of jiffies, and
2474 * to produce a reasonable delay curve.
2475 *
2476 * MEMCG_DELAY_SCALING_SHIFT just happens to be a number that produces a
2477 * reasonable delay curve compared to precision-adjusted overage, not
2478 * penalising heavily at first, but still making sure that growth beyond the
2479 * limit penalises misbehaviour cgroups by slowing them down exponentially. For
2480 * example, with a high of 100 megabytes:
2481 *
2482 * +-------+------------------------+
2483 * | usage | time to allocate in ms |
2484 * +-------+------------------------+
2485 * | 100M | 0 |
2486 * | 101M | 6 |
2487 * | 102M | 25 |
2488 * | 103M | 57 |
2489 * | 104M | 102 |
2490 * | 105M | 159 |
2491 * | 106M | 230 |
2492 * | 107M | 313 |
2493 * | 108M | 409 |
2494 * | 109M | 518 |
2495 * | 110M | 639 |
2496 * | 111M | 774 |
2497 * | 112M | 921 |
2498 * | 113M | 1081 |
2499 * | 114M | 1254 |
2500 * | 115M | 1439 |
2501 * | 116M | 1638 |
2502 * | 117M | 1849 |
2503 * | 118M | 2000 |
2504 * | 119M | 2000 |
2505 * | 120M | 2000 |
2506 * +-------+------------------------+
2507 */
2508 #define MEMCG_DELAY_PRECISION_SHIFT 20
2509 #define MEMCG_DELAY_SCALING_SHIFT 14
2510
Jakub Kicinski8a5dbc62020-06-01 21:49:42 -07002511static u64 calculate_overage(unsigned long usage, unsigned long high)
2512{
2513 u64 overage;
2514
2515 if (usage <= high)
2516 return 0;
2517
2518 /*
2519 * Prevent division by 0 in overage calculation by acting as if
2520 * it was a threshold of 1 page
2521 */
2522 high = max(high, 1UL);
2523
2524 overage = usage - high;
2525 overage <<= MEMCG_DELAY_PRECISION_SHIFT;
2526 return div64_u64(overage, high);
2527}
2528
2529static u64 mem_find_max_overage(struct mem_cgroup *memcg)
2530{
2531 u64 overage, max_overage = 0;
2532
2533 do {
2534 overage = calculate_overage(page_counter_read(&memcg->memory),
Jakub Kicinskid1663a92020-06-01 21:49:49 -07002535 READ_ONCE(memcg->memory.high));
Jakub Kicinski8a5dbc62020-06-01 21:49:42 -07002536 max_overage = max(overage, max_overage);
2537 } while ((memcg = parent_mem_cgroup(memcg)) &&
2538 !mem_cgroup_is_root(memcg));
2539
2540 return max_overage;
2541}
2542
Jakub Kicinski4b82ab42020-06-01 21:49:52 -07002543static u64 swap_find_max_overage(struct mem_cgroup *memcg)
2544{
2545 u64 overage, max_overage = 0;
2546
2547 do {
2548 overage = calculate_overage(page_counter_read(&memcg->swap),
2549 READ_ONCE(memcg->swap.high));
2550 if (overage)
2551 memcg_memory_event(memcg, MEMCG_SWAP_HIGH);
2552 max_overage = max(overage, max_overage);
2553 } while ((memcg = parent_mem_cgroup(memcg)) &&
2554 !mem_cgroup_is_root(memcg));
2555
2556 return max_overage;
2557}
2558
Chris Down0e4b01d2019-09-23 15:34:55 -07002559/*
Chris Downe26733e2020-03-21 18:22:23 -07002560 * Get the number of jiffies that we should penalise a mischievous cgroup which
2561 * is exceeding its memory.high by checking both it and its ancestors.
Tejun Heob23afb92015-11-05 18:46:11 -08002562 */
Chris Downe26733e2020-03-21 18:22:23 -07002563static unsigned long calculate_high_delay(struct mem_cgroup *memcg,
Jakub Kicinski8a5dbc62020-06-01 21:49:42 -07002564 unsigned int nr_pages,
2565 u64 max_overage)
Tejun Heob23afb92015-11-05 18:46:11 -08002566{
Chris Downe26733e2020-03-21 18:22:23 -07002567 unsigned long penalty_jiffies;
Chris Downe26733e2020-03-21 18:22:23 -07002568
2569 if (!max_overage)
2570 return 0;
Chris Down0e4b01d2019-09-23 15:34:55 -07002571
2572 /*
Chris Down0e4b01d2019-09-23 15:34:55 -07002573 * We use overage compared to memory.high to calculate the number of
2574 * jiffies to sleep (penalty_jiffies). Ideally this value should be
2575 * fairly lenient on small overages, and increasingly harsh when the
2576 * memcg in question makes it clear that it has no intention of stopping
2577 * its crazy behaviour, so we exponentially increase the delay based on
2578 * overage amount.
2579 */
Chris Downe26733e2020-03-21 18:22:23 -07002580 penalty_jiffies = max_overage * max_overage * HZ;
2581 penalty_jiffies >>= MEMCG_DELAY_PRECISION_SHIFT;
2582 penalty_jiffies >>= MEMCG_DELAY_SCALING_SHIFT;
Chris Down0e4b01d2019-09-23 15:34:55 -07002583
2584 /*
2585 * Factor in the task's own contribution to the overage, such that four
2586 * N-sized allocations are throttled approximately the same as one
2587 * 4N-sized allocation.
2588 *
2589 * MEMCG_CHARGE_BATCH pages is nominal, so work out how much smaller or
2590 * larger the current charge patch is than that.
2591 */
Jakub Kicinskiff144e62020-06-01 21:49:45 -07002592 return penalty_jiffies * nr_pages / MEMCG_CHARGE_BATCH;
Chris Downe26733e2020-03-21 18:22:23 -07002593}
2594
2595/*
2596 * Scheduled by try_charge() to be executed from the userland return path
2597 * and reclaims memory over the high limit.
2598 */
2599void mem_cgroup_handle_over_high(void)
2600{
2601 unsigned long penalty_jiffies;
2602 unsigned long pflags;
Chris Downb3ff9292020-08-06 23:21:54 -07002603 unsigned long nr_reclaimed;
Chris Downe26733e2020-03-21 18:22:23 -07002604 unsigned int nr_pages = current->memcg_nr_pages_over_high;
Chris Downd977aa92020-08-06 23:21:58 -07002605 int nr_retries = MAX_RECLAIM_RETRIES;
Chris Downe26733e2020-03-21 18:22:23 -07002606 struct mem_cgroup *memcg;
Chris Downb3ff9292020-08-06 23:21:54 -07002607 bool in_retry = false;
Chris Downe26733e2020-03-21 18:22:23 -07002608
2609 if (likely(!nr_pages))
2610 return;
2611
2612 memcg = get_mem_cgroup_from_mm(current->mm);
Chris Downe26733e2020-03-21 18:22:23 -07002613 current->memcg_nr_pages_over_high = 0;
2614
Chris Downb3ff9292020-08-06 23:21:54 -07002615retry_reclaim:
2616 /*
2617 * The allocating task should reclaim at least the batch size, but for
2618 * subsequent retries we only want to do what's necessary to prevent oom
2619 * or breaching resource isolation.
2620 *
2621 * This is distinct from memory.max or page allocator behaviour because
2622 * memory.high is currently batched, whereas memory.max and the page
2623 * allocator run every time an allocation is made.
2624 */
2625 nr_reclaimed = reclaim_high(memcg,
2626 in_retry ? SWAP_CLUSTER_MAX : nr_pages,
2627 GFP_KERNEL);
2628
Chris Downe26733e2020-03-21 18:22:23 -07002629 /*
2630 * memory.high is breached and reclaim is unable to keep up. Throttle
2631 * allocators proactively to slow down excessive growth.
2632 */
Jakub Kicinski8a5dbc62020-06-01 21:49:42 -07002633 penalty_jiffies = calculate_high_delay(memcg, nr_pages,
2634 mem_find_max_overage(memcg));
Chris Down0e4b01d2019-09-23 15:34:55 -07002635
Jakub Kicinski4b82ab42020-06-01 21:49:52 -07002636 penalty_jiffies += calculate_high_delay(memcg, nr_pages,
2637 swap_find_max_overage(memcg));
2638
Chris Down0e4b01d2019-09-23 15:34:55 -07002639 /*
Jakub Kicinskiff144e62020-06-01 21:49:45 -07002640 * Clamp the max delay per usermode return so as to still keep the
2641 * application moving forwards and also permit diagnostics, albeit
2642 * extremely slowly.
2643 */
2644 penalty_jiffies = min(penalty_jiffies, MEMCG_MAX_HIGH_DELAY_JIFFIES);
2645
2646 /*
Chris Down0e4b01d2019-09-23 15:34:55 -07002647 * Don't sleep if the amount of jiffies this memcg owes us is so low
2648 * that it's not even worth doing, in an attempt to be nice to those who
2649 * go only a small amount over their memory.high value and maybe haven't
2650 * been aggressively reclaimed enough yet.
2651 */
2652 if (penalty_jiffies <= HZ / 100)
2653 goto out;
2654
2655 /*
Chris Downb3ff9292020-08-06 23:21:54 -07002656 * If reclaim is making forward progress but we're still over
2657 * memory.high, we want to encourage that rather than doing allocator
2658 * throttling.
2659 */
2660 if (nr_reclaimed || nr_retries--) {
2661 in_retry = true;
2662 goto retry_reclaim;
2663 }
2664
2665 /*
Chris Down0e4b01d2019-09-23 15:34:55 -07002666 * If we exit early, we're guaranteed to die (since
2667 * schedule_timeout_killable sets TASK_KILLABLE). This means we don't
2668 * need to account for any ill-begotten jiffies to pay them off later.
2669 */
2670 psi_memstall_enter(&pflags);
2671 schedule_timeout_killable(penalty_jiffies);
2672 psi_memstall_leave(&pflags);
2673
2674out:
2675 css_put(&memcg->css);
Tejun Heob23afb92015-11-05 18:46:11 -08002676}
2677
Johannes Weiner00501b52014-08-08 14:19:20 -07002678static int try_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
2679 unsigned int nr_pages)
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002680{
Johannes Weinera983b5e2018-01-31 16:16:45 -08002681 unsigned int batch = max(MEMCG_CHARGE_BATCH, nr_pages);
Chris Downd977aa92020-08-06 23:21:58 -07002682 int nr_retries = MAX_RECLAIM_RETRIES;
Johannes Weiner6539cc02014-08-06 16:05:42 -07002683 struct mem_cgroup *mem_over_limit;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08002684 struct page_counter *counter;
Johannes Weinere22c6ed2020-08-06 23:22:15 -07002685 enum oom_status oom_status;
Johannes Weiner6539cc02014-08-06 16:05:42 -07002686 unsigned long nr_reclaimed;
Johannes Weinerb70a2a22014-10-09 15:28:56 -07002687 bool may_swap = true;
2688 bool drained = false;
Johannes Weinere22c6ed2020-08-06 23:22:15 -07002689 unsigned long pflags;
KAMEZAWA Hiroyukia636b322009-01-07 18:08:08 -08002690
Johannes Weinerce00a962014-09-05 08:43:57 -04002691 if (mem_cgroup_is_root(memcg))
Tejun Heo10d53c72015-11-05 18:46:17 -08002692 return 0;
Johannes Weiner6539cc02014-08-06 16:05:42 -07002693retry:
Michal Hockob6b6cc72014-04-07 15:37:44 -07002694 if (consume_stock(memcg, nr_pages))
Tejun Heo10d53c72015-11-05 18:46:17 -08002695 return 0;
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002696
Johannes Weiner7941d212016-01-14 15:21:23 -08002697 if (!do_memsw_account() ||
Johannes Weiner6071ca52015-11-05 18:50:26 -08002698 page_counter_try_charge(&memcg->memsw, batch, &counter)) {
2699 if (page_counter_try_charge(&memcg->memory, batch, &counter))
Johannes Weiner6539cc02014-08-06 16:05:42 -07002700 goto done_restock;
Johannes Weiner7941d212016-01-14 15:21:23 -08002701 if (do_memsw_account())
Johannes Weiner3e32cb22014-12-10 15:42:31 -08002702 page_counter_uncharge(&memcg->memsw, batch);
2703 mem_over_limit = mem_cgroup_from_counter(counter, memory);
Johannes Weiner3fbe7242014-10-09 15:28:54 -07002704 } else {
Johannes Weiner3e32cb22014-12-10 15:42:31 -08002705 mem_over_limit = mem_cgroup_from_counter(counter, memsw);
Johannes Weinerb70a2a22014-10-09 15:28:56 -07002706 may_swap = false;
Johannes Weiner3fbe7242014-10-09 15:28:54 -07002707 }
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002708
Johannes Weiner6539cc02014-08-06 16:05:42 -07002709 if (batch > nr_pages) {
2710 batch = nr_pages;
2711 goto retry;
2712 }
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002713
Johannes Weiner06b078f2014-08-06 16:05:44 -07002714 /*
Johannes Weiner869712f2019-11-05 21:17:13 -08002715 * Memcg doesn't have a dedicated reserve for atomic
2716 * allocations. But like the global atomic pool, we need to
2717 * put the burden of reclaim on regular allocation requests
2718 * and let these go through as privileged allocations.
2719 */
2720 if (gfp_mask & __GFP_ATOMIC)
2721 goto force;
2722
2723 /*
Johannes Weiner06b078f2014-08-06 16:05:44 -07002724 * Unlike in global OOM situations, memcg is not in a physical
2725 * memory shortage. Allow dying and OOM-killed tasks to
2726 * bypass the last charges so that they can exit quickly and
2727 * free their memory.
2728 */
Tetsuo Handa7775fac2019-03-05 15:46:47 -08002729 if (unlikely(should_force_charge()))
Tejun Heo10d53c72015-11-05 18:46:17 -08002730 goto force;
Johannes Weiner06b078f2014-08-06 16:05:44 -07002731
Johannes Weiner89a28482016-10-27 17:46:56 -07002732 /*
2733 * Prevent unbounded recursion when reclaim operations need to
2734 * allocate memory. This might exceed the limits temporarily,
2735 * but we prefer facilitating memory reclaim and getting back
2736 * under the limit over triggering OOM kills in these cases.
2737 */
2738 if (unlikely(current->flags & PF_MEMALLOC))
2739 goto force;
2740
Johannes Weiner06b078f2014-08-06 16:05:44 -07002741 if (unlikely(task_in_memcg_oom(current)))
2742 goto nomem;
2743
Mel Gormand0164ad2015-11-06 16:28:21 -08002744 if (!gfpflags_allow_blocking(gfp_mask))
Johannes Weiner6539cc02014-08-06 16:05:42 -07002745 goto nomem;
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002746
Johannes Weinere27be242018-04-10 16:29:45 -07002747 memcg_memory_event(mem_over_limit, MEMCG_MAX);
Johannes Weiner241994ed2015-02-11 15:26:06 -08002748
Johannes Weinere22c6ed2020-08-06 23:22:15 -07002749 psi_memstall_enter(&pflags);
Johannes Weinerb70a2a22014-10-09 15:28:56 -07002750 nr_reclaimed = try_to_free_mem_cgroup_pages(mem_over_limit, nr_pages,
2751 gfp_mask, may_swap);
Johannes Weinere22c6ed2020-08-06 23:22:15 -07002752 psi_memstall_leave(&pflags);
Johannes Weiner6539cc02014-08-06 16:05:42 -07002753
Johannes Weiner61e02c72014-08-06 16:08:16 -07002754 if (mem_cgroup_margin(mem_over_limit) >= nr_pages)
Johannes Weiner6539cc02014-08-06 16:05:42 -07002755 goto retry;
Johannes Weiner28c34c22014-08-06 16:05:47 -07002756
Johannes Weinerb70a2a22014-10-09 15:28:56 -07002757 if (!drained) {
Johannes Weiner6d3d6aa2014-12-10 15:42:50 -08002758 drain_all_stock(mem_over_limit);
Johannes Weinerb70a2a22014-10-09 15:28:56 -07002759 drained = true;
2760 goto retry;
2761 }
2762
Johannes Weiner28c34c22014-08-06 16:05:47 -07002763 if (gfp_mask & __GFP_NORETRY)
2764 goto nomem;
Johannes Weiner6539cc02014-08-06 16:05:42 -07002765 /*
2766 * Even though the limit is exceeded at this point, reclaim
2767 * may have been able to free some pages. Retry the charge
2768 * before killing the task.
2769 *
2770 * Only for regular pages, though: huge pages are rather
2771 * unlikely to succeed so close to the limit, and we fall back
2772 * to regular pages anyway in case of failure.
2773 */
Johannes Weiner61e02c72014-08-06 16:08:16 -07002774 if (nr_reclaimed && nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER))
Johannes Weiner6539cc02014-08-06 16:05:42 -07002775 goto retry;
2776 /*
2777 * At task move, charge accounts can be doubly counted. So, it's
2778 * better to wait until the end of task_move if something is going on.
2779 */
2780 if (mem_cgroup_wait_acct_move(mem_over_limit))
2781 goto retry;
2782
Johannes Weiner9b130612014-08-06 16:05:51 -07002783 if (nr_retries--)
2784 goto retry;
2785
Shakeel Butt38d38492019-07-11 20:55:48 -07002786 if (gfp_mask & __GFP_RETRY_MAYFAIL)
Michal Hocko29ef6802018-08-17 15:47:11 -07002787 goto nomem;
2788
Johannes Weiner6539cc02014-08-06 16:05:42 -07002789 if (fatal_signal_pending(current))
Tejun Heo10d53c72015-11-05 18:46:17 -08002790 goto force;
Johannes Weiner6539cc02014-08-06 16:05:42 -07002791
Michal Hocko29ef6802018-08-17 15:47:11 -07002792 /*
2793 * keep retrying as long as the memcg oom killer is able to make
2794 * a forward progress or bypass the charge if the oom killer
2795 * couldn't make any progress.
2796 */
2797 oom_status = mem_cgroup_oom(mem_over_limit, gfp_mask,
Jerome Marchand3608de02015-11-05 18:47:29 -08002798 get_order(nr_pages * PAGE_SIZE));
Michal Hocko29ef6802018-08-17 15:47:11 -07002799 switch (oom_status) {
2800 case OOM_SUCCESS:
Chris Downd977aa92020-08-06 23:21:58 -07002801 nr_retries = MAX_RECLAIM_RETRIES;
Michal Hocko29ef6802018-08-17 15:47:11 -07002802 goto retry;
2803 case OOM_FAILED:
2804 goto force;
2805 default:
2806 goto nomem;
2807 }
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002808nomem:
Johannes Weiner6d1fdc42014-04-07 15:37:45 -07002809 if (!(gfp_mask & __GFP_NOFAIL))
Johannes Weiner3168ecb2013-10-31 16:34:13 -07002810 return -ENOMEM;
Tejun Heo10d53c72015-11-05 18:46:17 -08002811force:
2812 /*
2813 * The allocation either can't fail or will lead to more memory
2814 * being freed very soon. Allow memory usage go over the limit
2815 * temporarily by force charging it.
2816 */
2817 page_counter_charge(&memcg->memory, nr_pages);
Johannes Weiner7941d212016-01-14 15:21:23 -08002818 if (do_memsw_account())
Tejun Heo10d53c72015-11-05 18:46:17 -08002819 page_counter_charge(&memcg->memsw, nr_pages);
Tejun Heo10d53c72015-11-05 18:46:17 -08002820
2821 return 0;
Johannes Weiner6539cc02014-08-06 16:05:42 -07002822
2823done_restock:
2824 if (batch > nr_pages)
2825 refill_stock(memcg, batch - nr_pages);
Tejun Heob23afb92015-11-05 18:46:11 -08002826
Johannes Weiner241994ed2015-02-11 15:26:06 -08002827 /*
Tejun Heob23afb92015-11-05 18:46:11 -08002828 * If the hierarchy is above the normal consumption range, schedule
2829 * reclaim on returning to userland. We can perform reclaim here
Mel Gorman71baba42015-11-06 16:28:28 -08002830 * if __GFP_RECLAIM but let's always punt for simplicity and so that
Tejun Heob23afb92015-11-05 18:46:11 -08002831 * GFP_KERNEL can consistently be used during reclaim. @memcg is
2832 * not recorded as it most likely matches current's and won't
2833 * change in the meantime. As high limit is checked again before
2834 * reclaim, the cost of mismatch is negligible.
Johannes Weiner241994ed2015-02-11 15:26:06 -08002835 */
2836 do {
Jakub Kicinski4b82ab42020-06-01 21:49:52 -07002837 bool mem_high, swap_high;
2838
2839 mem_high = page_counter_read(&memcg->memory) >
2840 READ_ONCE(memcg->memory.high);
2841 swap_high = page_counter_read(&memcg->swap) >
2842 READ_ONCE(memcg->swap.high);
2843
2844 /* Don't bother a random interrupted task */
2845 if (in_interrupt()) {
2846 if (mem_high) {
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08002847 schedule_work(&memcg->high_work);
2848 break;
2849 }
Jakub Kicinski4b82ab42020-06-01 21:49:52 -07002850 continue;
2851 }
2852
2853 if (mem_high || swap_high) {
2854 /*
2855 * The allocating tasks in this cgroup will need to do
2856 * reclaim or be throttled to prevent further growth
2857 * of the memory or swap footprints.
2858 *
2859 * Target some best-effort fairness between the tasks,
2860 * and distribute reclaim work and delay penalties
2861 * based on how much each task is actually allocating.
2862 */
Vladimir Davydov9516a182015-12-11 13:40:24 -08002863 current->memcg_nr_pages_over_high += batch;
Tejun Heob23afb92015-11-05 18:46:11 -08002864 set_notify_resume(current);
2865 break;
2866 }
Johannes Weiner241994ed2015-02-11 15:26:06 -08002867 } while ((memcg = parent_mem_cgroup(memcg)));
Tejun Heo10d53c72015-11-05 18:46:17 -08002868
2869 return 0;
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002870}
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002871
Johannes Weinerf0e45fb2020-06-03 16:02:07 -07002872#if defined(CONFIG_MEMCG_KMEM) || defined(CONFIG_MMU)
Johannes Weiner00501b52014-08-08 14:19:20 -07002873static void cancel_charge(struct mem_cgroup *memcg, unsigned int nr_pages)
Daisuke Nishimuraa3032a22009-12-15 16:47:10 -08002874{
Johannes Weinerce00a962014-09-05 08:43:57 -04002875 if (mem_cgroup_is_root(memcg))
2876 return;
2877
Johannes Weiner3e32cb22014-12-10 15:42:31 -08002878 page_counter_uncharge(&memcg->memory, nr_pages);
Johannes Weiner7941d212016-01-14 15:21:23 -08002879 if (do_memsw_account())
Johannes Weiner3e32cb22014-12-10 15:42:31 -08002880 page_counter_uncharge(&memcg->memsw, nr_pages);
KAMEZAWA Hiroyukid01dd172012-05-29 15:07:03 -07002881}
Johannes Weinerf0e45fb2020-06-03 16:02:07 -07002882#endif
KAMEZAWA Hiroyukid01dd172012-05-29 15:07:03 -07002883
Johannes Weinerd9eb1ea2020-06-03 16:02:24 -07002884static void commit_charge(struct page *page, struct mem_cgroup *memcg)
Johannes Weiner0a31bc92014-08-08 14:19:22 -07002885{
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08002886 VM_BUG_ON_PAGE(page_memcg(page), page);
Johannes Weiner0a31bc92014-08-08 14:19:22 -07002887 /*
Alex Shia5eb0112020-12-14 19:06:42 -08002888 * Any of the following ensures page's memcg stability:
Johannes Weiner0a31bc92014-08-08 14:19:22 -07002889 *
Johannes Weinera0b5b412020-06-03 16:02:27 -07002890 * - the page lock
2891 * - LRU isolation
2892 * - lock_page_memcg()
2893 * - exclusive reference
Johannes Weiner0a31bc92014-08-08 14:19:22 -07002894 */
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08002895 page->memcg_data = (unsigned long)memcg;
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002896}
2897
Kirill Tkhai84c07d12018-08-17 15:47:25 -07002898#ifdef CONFIG_MEMCG_KMEM
Roman Gushchin10befea2020-08-06 23:21:27 -07002899int memcg_alloc_page_obj_cgroups(struct page *page, struct kmem_cache *s,
Roman Gushchin2e9bd482021-02-24 12:03:11 -08002900 gfp_t gfp, bool new_page)
Roman Gushchin10befea2020-08-06 23:21:27 -07002901{
2902 unsigned int objects = objs_per_slab_page(s, page);
Roman Gushchin2e9bd482021-02-24 12:03:11 -08002903 unsigned long memcg_data;
Roman Gushchin10befea2020-08-06 23:21:27 -07002904 void *vec;
2905
2906 vec = kcalloc_node(objects, sizeof(struct obj_cgroup *), gfp,
2907 page_to_nid(page));
2908 if (!vec)
2909 return -ENOMEM;
2910
Roman Gushchin2e9bd482021-02-24 12:03:11 -08002911 memcg_data = (unsigned long) vec | MEMCG_DATA_OBJCGS;
2912 if (new_page) {
2913 /*
2914 * If the slab page is brand new and nobody can yet access
2915 * it's memcg_data, no synchronization is required and
2916 * memcg_data can be simply assigned.
2917 */
2918 page->memcg_data = memcg_data;
2919 } else if (cmpxchg(&page->memcg_data, 0, memcg_data)) {
2920 /*
2921 * If the slab page is already in use, somebody can allocate
2922 * and assign obj_cgroups in parallel. In this case the existing
2923 * objcg vector should be reused.
2924 */
Roman Gushchin10befea2020-08-06 23:21:27 -07002925 kfree(vec);
Roman Gushchin2e9bd482021-02-24 12:03:11 -08002926 return 0;
2927 }
Roman Gushchin10befea2020-08-06 23:21:27 -07002928
Roman Gushchin2e9bd482021-02-24 12:03:11 -08002929 kmemleak_not_leak(vec);
Roman Gushchin10befea2020-08-06 23:21:27 -07002930 return 0;
2931}
2932
Roman Gushchin8380ce42020-03-28 19:17:25 -07002933/*
2934 * Returns a pointer to the memory cgroup to which the kernel object is charged.
2935 *
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08002936 * A passed kernel object can be a slab object or a generic kernel page, so
2937 * different mechanisms for getting the memory cgroup pointer should be used.
2938 * In certain cases (e.g. kernel stacks or large kmallocs with SLUB) the caller
2939 * can not know for sure how the kernel object is implemented.
2940 * mem_cgroup_from_obj() can be safely used in such cases.
2941 *
Roman Gushchin8380ce42020-03-28 19:17:25 -07002942 * The caller must ensure the memcg lifetime, e.g. by taking rcu_read_lock(),
2943 * cgroup_mutex, etc.
2944 */
2945struct mem_cgroup *mem_cgroup_from_obj(void *p)
2946{
2947 struct page *page;
2948
2949 if (mem_cgroup_disabled())
2950 return NULL;
2951
2952 page = virt_to_head_page(p);
2953
2954 /*
Roman Gushchin98556092020-08-06 23:21:10 -07002955 * Slab objects are accounted individually, not per-page.
2956 * Memcg membership data for each individual object is saved in
2957 * the page->obj_cgroups.
Roman Gushchin8380ce42020-03-28 19:17:25 -07002958 */
Roman Gushchin270c6a72020-12-01 13:58:28 -08002959 if (page_objcgs_check(page)) {
Roman Gushchin98556092020-08-06 23:21:10 -07002960 struct obj_cgroup *objcg;
2961 unsigned int off;
2962
2963 off = obj_to_index(page->slab_cache, page, p);
Roman Gushchin270c6a72020-12-01 13:58:28 -08002964 objcg = page_objcgs(page)[off];
Roman Gushchin10befea2020-08-06 23:21:27 -07002965 if (objcg)
2966 return obj_cgroup_memcg(objcg);
2967
2968 return NULL;
Roman Gushchin98556092020-08-06 23:21:10 -07002969 }
Roman Gushchin8380ce42020-03-28 19:17:25 -07002970
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08002971 /*
2972 * page_memcg_check() is used here, because page_has_obj_cgroups()
2973 * check above could fail because the object cgroups vector wasn't set
2974 * at that moment, but it can be set concurrently.
2975 * page_memcg_check(page) will guarantee that a proper memory
2976 * cgroup pointer or NULL will be returned.
2977 */
2978 return page_memcg_check(page);
Roman Gushchin8380ce42020-03-28 19:17:25 -07002979}
2980
Roman Gushchinbf4f0592020-08-06 23:20:49 -07002981__always_inline struct obj_cgroup *get_obj_cgroup_from_current(void)
2982{
2983 struct obj_cgroup *objcg = NULL;
2984 struct mem_cgroup *memcg;
2985
Roman Gushchin279c3392020-10-17 16:13:44 -07002986 if (memcg_kmem_bypass())
2987 return NULL;
2988
Roman Gushchinbf4f0592020-08-06 23:20:49 -07002989 rcu_read_lock();
Roman Gushchin37d59852020-10-17 16:13:50 -07002990 if (unlikely(active_memcg()))
2991 memcg = active_memcg();
Roman Gushchinbf4f0592020-08-06 23:20:49 -07002992 else
2993 memcg = mem_cgroup_from_task(current);
2994
2995 for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg)) {
2996 objcg = rcu_dereference(memcg->objcg);
2997 if (objcg && obj_cgroup_tryget(objcg))
2998 break;
Muchun Song2f7659a2020-12-14 19:06:31 -08002999 objcg = NULL;
Roman Gushchinbf4f0592020-08-06 23:20:49 -07003000 }
3001 rcu_read_unlock();
3002
3003 return objcg;
3004}
3005
Vladimir Davydovf3bb3042014-10-09 15:28:45 -07003006static int memcg_alloc_cache_id(void)
Glauber Costa55007d82012-12-18 14:22:38 -08003007{
Vladimir Davydovf3bb3042014-10-09 15:28:45 -07003008 int id, size;
3009 int err;
Glauber Costa55007d82012-12-18 14:22:38 -08003010
Vladimir Davydovdbcf73e2015-02-12 14:58:57 -08003011 id = ida_simple_get(&memcg_cache_ida,
Vladimir Davydovf3bb3042014-10-09 15:28:45 -07003012 0, MEMCG_CACHES_MAX_SIZE, GFP_KERNEL);
3013 if (id < 0)
3014 return id;
3015
Vladimir Davydovdbcf73e2015-02-12 14:58:57 -08003016 if (id < memcg_nr_cache_ids)
Vladimir Davydovf3bb3042014-10-09 15:28:45 -07003017 return id;
3018
3019 /*
3020 * There's no space for the new id in memcg_caches arrays,
3021 * so we have to grow them.
3022 */
Vladimir Davydov05257a12015-02-12 14:59:01 -08003023 down_write(&memcg_cache_ids_sem);
Vladimir Davydovf3bb3042014-10-09 15:28:45 -07003024
3025 size = 2 * (id + 1);
Glauber Costa55007d82012-12-18 14:22:38 -08003026 if (size < MEMCG_CACHES_MIN_SIZE)
3027 size = MEMCG_CACHES_MIN_SIZE;
3028 else if (size > MEMCG_CACHES_MAX_SIZE)
3029 size = MEMCG_CACHES_MAX_SIZE;
3030
Roman Gushchin98556092020-08-06 23:21:10 -07003031 err = memcg_update_all_list_lrus(size);
Vladimir Davydov60d3fd32015-02-12 14:59:10 -08003032 if (!err)
Vladimir Davydov05257a12015-02-12 14:59:01 -08003033 memcg_nr_cache_ids = size;
3034
3035 up_write(&memcg_cache_ids_sem);
3036
Vladimir Davydovf3bb3042014-10-09 15:28:45 -07003037 if (err) {
Vladimir Davydovdbcf73e2015-02-12 14:58:57 -08003038 ida_simple_remove(&memcg_cache_ida, id);
Vladimir Davydovf3bb3042014-10-09 15:28:45 -07003039 return err;
3040 }
3041 return id;
3042}
3043
3044static void memcg_free_cache_id(int id)
3045{
Vladimir Davydovdbcf73e2015-02-12 14:58:57 -08003046 ida_simple_remove(&memcg_cache_ida, id);
Glauber Costa55007d82012-12-18 14:22:38 -08003047}
3048
Vladimir Davydov45264772016-07-26 15:24:21 -07003049/**
Roman Gushchin4b13f642020-04-01 21:06:56 -07003050 * __memcg_kmem_charge: charge a number of kernel pages to a memcg
Roman Gushchin10eaec22020-04-01 21:06:39 -07003051 * @memcg: memory cgroup to charge
Vladimir Davydov45264772016-07-26 15:24:21 -07003052 * @gfp: reclaim mode
Roman Gushchin92d0510c2020-04-01 21:06:49 -07003053 * @nr_pages: number of pages to charge
Vladimir Davydov45264772016-07-26 15:24:21 -07003054 *
3055 * Returns 0 on success, an error code on failure.
3056 */
Roman Gushchinc1a660d2021-02-24 12:03:58 -08003057static int __memcg_kmem_charge(struct mem_cgroup *memcg, gfp_t gfp,
3058 unsigned int nr_pages)
Vladimir Davydovf3ccb2c42015-11-05 18:49:01 -08003059{
Vladimir Davydovf3ccb2c42015-11-05 18:49:01 -08003060 struct page_counter *counter;
Johannes Weiner6071ca52015-11-05 18:50:26 -08003061 int ret;
Vladimir Davydovf3ccb2c42015-11-05 18:49:01 -08003062
Vladimir Davydovf3ccb2c42015-11-05 18:49:01 -08003063 ret = try_charge(memcg, gfp, nr_pages);
Johannes Weiner52c29b02016-01-20 15:02:35 -08003064 if (ret)
Vladimir Davydovf3ccb2c42015-11-05 18:49:01 -08003065 return ret;
Johannes Weiner52c29b02016-01-20 15:02:35 -08003066
3067 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) &&
3068 !page_counter_try_charge(&memcg->kmem, nr_pages, &counter)) {
Michal Hockoe55d9d92019-09-25 16:45:53 -07003069
3070 /*
3071 * Enforce __GFP_NOFAIL allocation because callers are not
3072 * prepared to see failures and likely do not have any failure
3073 * handling code.
3074 */
3075 if (gfp & __GFP_NOFAIL) {
3076 page_counter_charge(&memcg->kmem, nr_pages);
3077 return 0;
3078 }
Johannes Weiner52c29b02016-01-20 15:02:35 -08003079 cancel_charge(memcg, nr_pages);
3080 return -ENOMEM;
Vladimir Davydovf3ccb2c42015-11-05 18:49:01 -08003081 }
Vladimir Davydovf3ccb2c42015-11-05 18:49:01 -08003082 return 0;
3083}
3084
Vladimir Davydov45264772016-07-26 15:24:21 -07003085/**
Roman Gushchin4b13f642020-04-01 21:06:56 -07003086 * __memcg_kmem_uncharge: uncharge a number of kernel pages from a memcg
3087 * @memcg: memcg to uncharge
3088 * @nr_pages: number of pages to uncharge
3089 */
Roman Gushchinc1a660d2021-02-24 12:03:58 -08003090static void __memcg_kmem_uncharge(struct mem_cgroup *memcg, unsigned int nr_pages)
Roman Gushchin4b13f642020-04-01 21:06:56 -07003091{
3092 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
3093 page_counter_uncharge(&memcg->kmem, nr_pages);
3094
Roman Gushchin3de7d4f2021-01-23 21:01:07 -08003095 refill_stock(memcg, nr_pages);
Roman Gushchin4b13f642020-04-01 21:06:56 -07003096}
3097
3098/**
Roman Gushchinf4b00ea2020-04-01 21:06:46 -07003099 * __memcg_kmem_charge_page: charge a kmem page to the current memory cgroup
Vladimir Davydov45264772016-07-26 15:24:21 -07003100 * @page: page to charge
3101 * @gfp: reclaim mode
3102 * @order: allocation order
3103 *
3104 * Returns 0 on success, an error code on failure.
3105 */
Roman Gushchinf4b00ea2020-04-01 21:06:46 -07003106int __memcg_kmem_charge_page(struct page *page, gfp_t gfp, int order)
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08003107{
3108 struct mem_cgroup *memcg;
Vladimir Davydovfcff7d72016-03-17 14:17:29 -07003109 int ret = 0;
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08003110
Shakeel Buttd46eb14b2018-08-17 15:46:39 -07003111 memcg = get_mem_cgroup_from_current();
Roman Gushchin279c3392020-10-17 16:13:44 -07003112 if (memcg && !mem_cgroup_is_root(memcg)) {
Roman Gushchin4b13f642020-04-01 21:06:56 -07003113 ret = __memcg_kmem_charge(memcg, gfp, 1 << order);
Roman Gushchin4d96ba32019-07-11 20:56:31 -07003114 if (!ret) {
Roman Gushchin18b2db32020-12-01 13:58:30 -08003115 page->memcg_data = (unsigned long)memcg |
3116 MEMCG_DATA_KMEM;
Johannes Weiner1a3e1f42020-08-06 23:20:45 -07003117 return 0;
Roman Gushchin4d96ba32019-07-11 20:56:31 -07003118 }
Roman Gushchin279c3392020-10-17 16:13:44 -07003119 css_put(&memcg->css);
Vladimir Davydovc4159a72016-08-08 23:03:12 +03003120 }
Vladimir Davydovd05e83a2015-11-05 18:48:59 -08003121 return ret;
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08003122}
Roman Gushchin49a18ea2019-07-11 20:56:13 -07003123
3124/**
Roman Gushchinf4b00ea2020-04-01 21:06:46 -07003125 * __memcg_kmem_uncharge_page: uncharge a kmem page
Vladimir Davydov45264772016-07-26 15:24:21 -07003126 * @page: page to uncharge
3127 * @order: allocation order
3128 */
Roman Gushchinf4b00ea2020-04-01 21:06:46 -07003129void __memcg_kmem_uncharge_page(struct page *page, int order)
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08003130{
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08003131 struct mem_cgroup *memcg = page_memcg(page);
Vladimir Davydovf3ccb2c42015-11-05 18:49:01 -08003132 unsigned int nr_pages = 1 << order;
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08003133
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08003134 if (!memcg)
3135 return;
3136
Sasha Levin309381fea2014-01-23 15:52:54 -08003137 VM_BUG_ON_PAGE(mem_cgroup_is_root(memcg), page);
Roman Gushchin4b13f642020-04-01 21:06:56 -07003138 __memcg_kmem_uncharge(memcg, nr_pages);
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08003139 page->memcg_data = 0;
Johannes Weiner1a3e1f42020-08-06 23:20:45 -07003140 css_put(&memcg->css);
Vladimir Davydov60d3fd32015-02-12 14:59:10 -08003141}
Roman Gushchinbf4f0592020-08-06 23:20:49 -07003142
3143static bool consume_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes)
3144{
3145 struct memcg_stock_pcp *stock;
3146 unsigned long flags;
3147 bool ret = false;
3148
3149 local_irq_save(flags);
3150
3151 stock = this_cpu_ptr(&memcg_stock);
3152 if (objcg == stock->cached_objcg && stock->nr_bytes >= nr_bytes) {
3153 stock->nr_bytes -= nr_bytes;
3154 ret = true;
3155 }
3156
3157 local_irq_restore(flags);
3158
3159 return ret;
3160}
3161
3162static void drain_obj_stock(struct memcg_stock_pcp *stock)
3163{
3164 struct obj_cgroup *old = stock->cached_objcg;
3165
3166 if (!old)
3167 return;
3168
3169 if (stock->nr_bytes) {
3170 unsigned int nr_pages = stock->nr_bytes >> PAGE_SHIFT;
3171 unsigned int nr_bytes = stock->nr_bytes & (PAGE_SIZE - 1);
3172
3173 if (nr_pages) {
3174 rcu_read_lock();
3175 __memcg_kmem_uncharge(obj_cgroup_memcg(old), nr_pages);
3176 rcu_read_unlock();
3177 }
3178
3179 /*
3180 * The leftover is flushed to the centralized per-memcg value.
3181 * On the next attempt to refill obj stock it will be moved
3182 * to a per-cpu stock (probably, on an other CPU), see
3183 * refill_obj_stock().
3184 *
3185 * How often it's flushed is a trade-off between the memory
3186 * limit enforcement accuracy and potential CPU contention,
3187 * so it might be changed in the future.
3188 */
3189 atomic_add(nr_bytes, &old->nr_charged_bytes);
3190 stock->nr_bytes = 0;
3191 }
3192
3193 obj_cgroup_put(old);
3194 stock->cached_objcg = NULL;
3195}
3196
3197static bool obj_stock_flush_required(struct memcg_stock_pcp *stock,
3198 struct mem_cgroup *root_memcg)
3199{
3200 struct mem_cgroup *memcg;
3201
3202 if (stock->cached_objcg) {
3203 memcg = obj_cgroup_memcg(stock->cached_objcg);
3204 if (memcg && mem_cgroup_is_descendant(memcg, root_memcg))
3205 return true;
3206 }
3207
3208 return false;
3209}
3210
3211static void refill_obj_stock(struct obj_cgroup *objcg, unsigned int nr_bytes)
3212{
3213 struct memcg_stock_pcp *stock;
3214 unsigned long flags;
3215
3216 local_irq_save(flags);
3217
3218 stock = this_cpu_ptr(&memcg_stock);
3219 if (stock->cached_objcg != objcg) { /* reset if necessary */
3220 drain_obj_stock(stock);
3221 obj_cgroup_get(objcg);
3222 stock->cached_objcg = objcg;
3223 stock->nr_bytes = atomic_xchg(&objcg->nr_charged_bytes, 0);
3224 }
3225 stock->nr_bytes += nr_bytes;
3226
3227 if (stock->nr_bytes > PAGE_SIZE)
3228 drain_obj_stock(stock);
3229
3230 local_irq_restore(flags);
3231}
3232
3233int obj_cgroup_charge(struct obj_cgroup *objcg, gfp_t gfp, size_t size)
3234{
3235 struct mem_cgroup *memcg;
3236 unsigned int nr_pages, nr_bytes;
3237 int ret;
3238
3239 if (consume_obj_stock(objcg, size))
3240 return 0;
3241
3242 /*
3243 * In theory, memcg->nr_charged_bytes can have enough
3244 * pre-charged bytes to satisfy the allocation. However,
3245 * flushing memcg->nr_charged_bytes requires two atomic
3246 * operations, and memcg->nr_charged_bytes can't be big,
3247 * so it's better to ignore it and try grab some new pages.
3248 * memcg->nr_charged_bytes will be flushed in
3249 * refill_obj_stock(), called from this function or
3250 * independently later.
3251 */
3252 rcu_read_lock();
Muchun Songeefbfa72020-12-14 19:06:35 -08003253retry:
Roman Gushchinbf4f0592020-08-06 23:20:49 -07003254 memcg = obj_cgroup_memcg(objcg);
Muchun Songeefbfa72020-12-14 19:06:35 -08003255 if (unlikely(!css_tryget(&memcg->css)))
3256 goto retry;
Roman Gushchinbf4f0592020-08-06 23:20:49 -07003257 rcu_read_unlock();
3258
3259 nr_pages = size >> PAGE_SHIFT;
3260 nr_bytes = size & (PAGE_SIZE - 1);
3261
3262 if (nr_bytes)
3263 nr_pages += 1;
3264
3265 ret = __memcg_kmem_charge(memcg, gfp, nr_pages);
3266 if (!ret && nr_bytes)
3267 refill_obj_stock(objcg, PAGE_SIZE - nr_bytes);
3268
3269 css_put(&memcg->css);
3270 return ret;
3271}
3272
3273void obj_cgroup_uncharge(struct obj_cgroup *objcg, size_t size)
3274{
3275 refill_obj_stock(objcg, size);
3276}
3277
Kirill Tkhai84c07d12018-08-17 15:47:25 -07003278#endif /* CONFIG_MEMCG_KMEM */
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08003279
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08003280/*
Zhou Guanghuibe6c8982021-03-12 21:08:30 -08003281 * Because page_memcg(head) is not set on tails, set it now.
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08003282 */
Zhou Guanghuibe6c8982021-03-12 21:08:30 -08003283void split_page_memcg(struct page *head, unsigned int nr)
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08003284{
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08003285 struct mem_cgroup *memcg = page_memcg(head);
KAMEZAWA Hiroyukie94c8a92012-01-12 17:18:20 -08003286 int i;
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08003287
Zhou Guanghuibe6c8982021-03-12 21:08:30 -08003288 if (mem_cgroup_disabled() || !memcg)
KAMEZAWA Hiroyuki3d37c4a2011-01-25 15:07:28 -08003289 return;
David Rientjesb070e652013-05-07 16:18:09 -07003290
Zhou Guanghuibe6c8982021-03-12 21:08:30 -08003291 for (i = 1; i < nr; i++)
3292 head[i].memcg_data = head->memcg_data;
3293 css_get_many(&memcg->css, nr - 1);
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08003294}
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08003295
Andrew Mortonc255a452012-07-31 16:43:02 -07003296#ifdef CONFIG_MEMCG_SWAP
Daisuke Nishimura02491442010-03-10 15:22:17 -08003297/**
3298 * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
3299 * @entry: swap entry to be moved
3300 * @from: mem_cgroup which the entry is moved from
3301 * @to: mem_cgroup which the entry is moved to
3302 *
3303 * It succeeds only when the swap_cgroup's record for this entry is the same
3304 * as the mem_cgroup's id of @from.
3305 *
3306 * Returns 0 on success, -EINVAL on failure.
3307 *
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003308 * The caller must have charged to @to, IOW, called page_counter_charge() about
Daisuke Nishimura02491442010-03-10 15:22:17 -08003309 * both res and memsw, and called css_get().
3310 */
3311static int mem_cgroup_move_swap_account(swp_entry_t entry,
Hugh Dickinse91cbb42012-05-29 15:06:51 -07003312 struct mem_cgroup *from, struct mem_cgroup *to)
Daisuke Nishimura02491442010-03-10 15:22:17 -08003313{
3314 unsigned short old_id, new_id;
3315
Li Zefan34c00c32013-09-23 16:56:01 +08003316 old_id = mem_cgroup_id(from);
3317 new_id = mem_cgroup_id(to);
Daisuke Nishimura02491442010-03-10 15:22:17 -08003318
3319 if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
Johannes Weinerc9019e92018-01-31 16:16:37 -08003320 mod_memcg_state(from, MEMCG_SWAP, -1);
3321 mod_memcg_state(to, MEMCG_SWAP, 1);
Daisuke Nishimura02491442010-03-10 15:22:17 -08003322 return 0;
3323 }
3324 return -EINVAL;
3325}
3326#else
3327static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
Hugh Dickinse91cbb42012-05-29 15:06:51 -07003328 struct mem_cgroup *from, struct mem_cgroup *to)
Daisuke Nishimura02491442010-03-10 15:22:17 -08003329{
3330 return -EINVAL;
3331}
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003332#endif
3333
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003334static DEFINE_MUTEX(memcg_max_mutex);
Daisuke Nishimuraf212ad72011-03-23 16:42:25 -07003335
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003336static int mem_cgroup_resize_max(struct mem_cgroup *memcg,
3337 unsigned long max, bool memsw)
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003338{
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003339 bool enlarge = false;
Shakeel Buttbb4a7ea2018-06-07 17:07:27 -07003340 bool drained = false;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003341 int ret;
Yu Zhaoc054a782018-01-31 16:20:02 -08003342 bool limits_invariant;
3343 struct page_counter *counter = memsw ? &memcg->memsw : &memcg->memory;
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07003344
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003345 do {
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003346 if (signal_pending(current)) {
3347 ret = -EINTR;
3348 break;
3349 }
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003350
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003351 mutex_lock(&memcg_max_mutex);
Yu Zhaoc054a782018-01-31 16:20:02 -08003352 /*
3353 * Make sure that the new limit (memsw or memory limit) doesn't
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003354 * break our basic invariant rule memory.max <= memsw.max.
Yu Zhaoc054a782018-01-31 16:20:02 -08003355 */
Chris Down15b42562020-04-01 21:07:20 -07003356 limits_invariant = memsw ? max >= READ_ONCE(memcg->memory.max) :
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003357 max <= memcg->memsw.max;
Yu Zhaoc054a782018-01-31 16:20:02 -08003358 if (!limits_invariant) {
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003359 mutex_unlock(&memcg_max_mutex);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003360 ret = -EINVAL;
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003361 break;
3362 }
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003363 if (max > counter->max)
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003364 enlarge = true;
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003365 ret = page_counter_set_max(counter, max);
3366 mutex_unlock(&memcg_max_mutex);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003367
3368 if (!ret)
3369 break;
3370
Shakeel Buttbb4a7ea2018-06-07 17:07:27 -07003371 if (!drained) {
3372 drain_all_stock(memcg);
3373 drained = true;
3374 continue;
3375 }
3376
Andrey Ryabinin1ab5c052018-01-31 16:20:37 -08003377 if (!try_to_free_mem_cgroup_pages(memcg, 1,
3378 GFP_KERNEL, !memsw)) {
3379 ret = -EBUSY;
3380 break;
3381 }
3382 } while (true);
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003383
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07003384 if (!ret && enlarge)
3385 memcg_oom_recover(memcg);
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003386
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003387 return ret;
3388}
3389
Mel Gormanef8f2322016-07-28 15:46:05 -07003390unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
Andrew Morton0608f432013-09-24 15:27:41 -07003391 gfp_t gfp_mask,
3392 unsigned long *total_scanned)
3393{
3394 unsigned long nr_reclaimed = 0;
Mel Gormanef8f2322016-07-28 15:46:05 -07003395 struct mem_cgroup_per_node *mz, *next_mz = NULL;
Andrew Morton0608f432013-09-24 15:27:41 -07003396 unsigned long reclaimed;
3397 int loop = 0;
Mel Gormanef8f2322016-07-28 15:46:05 -07003398 struct mem_cgroup_tree_per_node *mctz;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003399 unsigned long excess;
Andrew Morton0608f432013-09-24 15:27:41 -07003400 unsigned long nr_scanned;
3401
3402 if (order > 0)
3403 return 0;
3404
Mel Gormanef8f2322016-07-28 15:46:05 -07003405 mctz = soft_limit_tree_node(pgdat->node_id);
Michal Hockod6507ff2016-08-02 14:02:37 -07003406
3407 /*
3408 * Do not even bother to check the largest node if the root
3409 * is empty. Do it lockless to prevent lock bouncing. Races
3410 * are acceptable as soft limit is best effort anyway.
3411 */
Laurent Dufourbfc72282017-03-09 16:17:06 -08003412 if (!mctz || RB_EMPTY_ROOT(&mctz->rb_root))
Michal Hockod6507ff2016-08-02 14:02:37 -07003413 return 0;
3414
Andrew Morton0608f432013-09-24 15:27:41 -07003415 /*
3416 * This loop can run a while, specially if mem_cgroup's continuously
3417 * keep exceeding their soft limit and putting the system under
3418 * pressure
3419 */
3420 do {
3421 if (next_mz)
3422 mz = next_mz;
3423 else
3424 mz = mem_cgroup_largest_soft_limit_node(mctz);
3425 if (!mz)
3426 break;
3427
3428 nr_scanned = 0;
Mel Gormanef8f2322016-07-28 15:46:05 -07003429 reclaimed = mem_cgroup_soft_reclaim(mz->memcg, pgdat,
Andrew Morton0608f432013-09-24 15:27:41 -07003430 gfp_mask, &nr_scanned);
3431 nr_reclaimed += reclaimed;
3432 *total_scanned += nr_scanned;
Johannes Weiner0a31bc92014-08-08 14:19:22 -07003433 spin_lock_irq(&mctz->lock);
Vladimir Davydovbc2f2e72014-12-10 15:43:40 -08003434 __mem_cgroup_remove_exceeded(mz, mctz);
Andrew Morton0608f432013-09-24 15:27:41 -07003435
3436 /*
3437 * If we failed to reclaim anything from this memory cgroup
3438 * it is time to move on to the next cgroup
3439 */
3440 next_mz = NULL;
Vladimir Davydovbc2f2e72014-12-10 15:43:40 -08003441 if (!reclaimed)
3442 next_mz = __mem_cgroup_largest_soft_limit_node(mctz);
3443
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003444 excess = soft_limit_excess(mz->memcg);
Andrew Morton0608f432013-09-24 15:27:41 -07003445 /*
3446 * One school of thought says that we should not add
3447 * back the node to the tree if reclaim returns 0.
3448 * But our reclaim could return 0, simply because due
3449 * to priority we are exposing a smaller subset of
3450 * memory to reclaim from. Consider this as a longer
3451 * term TODO.
3452 */
3453 /* If excess == 0, no tree ops */
Johannes Weinercf2c8122014-06-06 14:38:21 -07003454 __mem_cgroup_insert_exceeded(mz, mctz, excess);
Johannes Weiner0a31bc92014-08-08 14:19:22 -07003455 spin_unlock_irq(&mctz->lock);
Andrew Morton0608f432013-09-24 15:27:41 -07003456 css_put(&mz->memcg->css);
3457 loop++;
3458 /*
3459 * Could not reclaim anything and there are no more
3460 * mem cgroups to try or we seem to be looping without
3461 * reclaiming anything.
3462 */
3463 if (!nr_reclaimed &&
3464 (next_mz == NULL ||
3465 loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
3466 break;
3467 } while (!nr_reclaimed);
3468 if (next_mz)
3469 css_put(&next_mz->memcg->css);
3470 return nr_reclaimed;
3471}
3472
Tejun Heoea280e72014-05-16 13:22:48 -04003473/*
Greg Thelen51038172016-05-20 16:58:18 -07003474 * Reclaims as many pages from the given memcg as possible.
Michal Hockoc26251f2012-10-26 13:37:28 +02003475 *
3476 * Caller is responsible for holding css reference for memcg.
3477 */
3478static int mem_cgroup_force_empty(struct mem_cgroup *memcg)
3479{
Chris Downd977aa92020-08-06 23:21:58 -07003480 int nr_retries = MAX_RECLAIM_RETRIES;
Michal Hockoc26251f2012-10-26 13:37:28 +02003481
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003482 /* we call try-to-free pages for make this cgroup empty */
3483 lru_add_drain_all();
Junaid Shahidd12c60f2018-06-07 17:07:31 -07003484
3485 drain_all_stock(memcg);
3486
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003487 /* try to free all pages in this cgroup */
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003488 while (nr_retries && page_counter_read(&memcg->memory)) {
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003489 int progress;
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003490
Michal Hockoc26251f2012-10-26 13:37:28 +02003491 if (signal_pending(current))
3492 return -EINTR;
3493
Johannes Weinerb70a2a22014-10-09 15:28:56 -07003494 progress = try_to_free_mem_cgroup_pages(memcg, 1,
3495 GFP_KERNEL, true);
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003496 if (!progress) {
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003497 nr_retries--;
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003498 /* maybe some writeback is necessary */
Jens Axboe8aa7e842009-07-09 14:52:32 +02003499 congestion_wait(BLK_RW_ASYNC, HZ/10);
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003500 }
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003501
3502 }
Michal Hockoab5196c2012-10-26 13:37:32 +02003503
3504 return 0;
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08003505}
3506
Tejun Heo6770c642014-05-13 12:16:21 -04003507static ssize_t mem_cgroup_force_empty_write(struct kernfs_open_file *of,
3508 char *buf, size_t nbytes,
3509 loff_t off)
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003510{
Tejun Heo6770c642014-05-13 12:16:21 -04003511 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
Michal Hockoc26251f2012-10-26 13:37:28 +02003512
Michal Hockod8423012012-10-26 13:37:29 +02003513 if (mem_cgroup_is_root(memcg))
3514 return -EINVAL;
Tejun Heo6770c642014-05-13 12:16:21 -04003515 return mem_cgroup_force_empty(memcg) ?: nbytes;
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003516}
3517
Tejun Heo182446d2013-08-08 20:11:24 -04003518static u64 mem_cgroup_hierarchy_read(struct cgroup_subsys_state *css,
3519 struct cftype *cft)
Balbir Singh18f59ea2009-01-07 18:08:07 -08003520{
Roman Gushchinbef86202020-12-14 19:06:49 -08003521 return 1;
Balbir Singh18f59ea2009-01-07 18:08:07 -08003522}
3523
Tejun Heo182446d2013-08-08 20:11:24 -04003524static int mem_cgroup_hierarchy_write(struct cgroup_subsys_state *css,
3525 struct cftype *cft, u64 val)
Balbir Singh18f59ea2009-01-07 18:08:07 -08003526{
Roman Gushchinbef86202020-12-14 19:06:49 -08003527 if (val == 1)
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08003528 return 0;
Glauber Costa567fb432012-07-31 16:43:07 -07003529
Roman Gushchinbef86202020-12-14 19:06:49 -08003530 pr_warn_once("Non-hierarchical mode is deprecated. "
3531 "Please report your usecase to linux-mm@kvack.org if you "
3532 "depend on this functionality.\n");
Glauber Costa567fb432012-07-31 16:43:07 -07003533
Roman Gushchinbef86202020-12-14 19:06:49 -08003534 return -EINVAL;
Balbir Singh18f59ea2009-01-07 18:08:07 -08003535}
3536
Andrew Morton6f646152015-11-06 16:28:58 -08003537static unsigned long mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
Johannes Weinerce00a962014-09-05 08:43:57 -04003538{
Johannes Weiner42a30032019-05-14 15:47:12 -07003539 unsigned long val;
Johannes Weinerce00a962014-09-05 08:43:57 -04003540
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003541 if (mem_cgroup_is_root(memcg)) {
Johannes Weiner0d1c2072020-06-03 16:01:54 -07003542 val = memcg_page_state(memcg, NR_FILE_PAGES) +
Johannes Weinerbe5d0a72020-06-03 16:01:57 -07003543 memcg_page_state(memcg, NR_ANON_MAPPED);
Johannes Weiner42a30032019-05-14 15:47:12 -07003544 if (swap)
3545 val += memcg_page_state(memcg, MEMCG_SWAP);
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003546 } else {
Johannes Weinerce00a962014-09-05 08:43:57 -04003547 if (!swap)
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003548 val = page_counter_read(&memcg->memory);
Johannes Weinerce00a962014-09-05 08:43:57 -04003549 else
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003550 val = page_counter_read(&memcg->memsw);
Johannes Weinerce00a962014-09-05 08:43:57 -04003551 }
Michal Hockoc12176d2015-11-05 18:50:29 -08003552 return val;
Johannes Weinerce00a962014-09-05 08:43:57 -04003553}
3554
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003555enum {
3556 RES_USAGE,
3557 RES_LIMIT,
3558 RES_MAX_USAGE,
3559 RES_FAILCNT,
3560 RES_SOFT_LIMIT,
3561};
Johannes Weinerce00a962014-09-05 08:43:57 -04003562
Tejun Heo791badb2013-12-05 12:28:02 -05003563static u64 mem_cgroup_read_u64(struct cgroup_subsys_state *css,
Johannes Weiner05b84302014-08-06 16:05:59 -07003564 struct cftype *cft)
Balbir Singh8cdea7c2008-02-07 00:13:50 -08003565{
Tejun Heo182446d2013-08-08 20:11:24 -04003566 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003567 struct page_counter *counter;
Tejun Heoaf36f902012-04-01 12:09:55 -07003568
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003569 switch (MEMFILE_TYPE(cft->private)) {
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003570 case _MEM:
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003571 counter = &memcg->memory;
Glauber Costa510fc4e2012-12-18 14:21:47 -08003572 break;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003573 case _MEMSWAP:
3574 counter = &memcg->memsw;
3575 break;
3576 case _KMEM:
3577 counter = &memcg->kmem;
3578 break;
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08003579 case _TCP:
Johannes Weiner0db15292016-01-20 15:02:50 -08003580 counter = &memcg->tcpmem;
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08003581 break;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003582 default:
3583 BUG();
3584 }
3585
3586 switch (MEMFILE_ATTR(cft->private)) {
3587 case RES_USAGE:
3588 if (counter == &memcg->memory)
Michal Hockoc12176d2015-11-05 18:50:29 -08003589 return (u64)mem_cgroup_usage(memcg, false) * PAGE_SIZE;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003590 if (counter == &memcg->memsw)
Michal Hockoc12176d2015-11-05 18:50:29 -08003591 return (u64)mem_cgroup_usage(memcg, true) * PAGE_SIZE;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003592 return (u64)page_counter_read(counter) * PAGE_SIZE;
3593 case RES_LIMIT:
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003594 return (u64)counter->max * PAGE_SIZE;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003595 case RES_MAX_USAGE:
3596 return (u64)counter->watermark * PAGE_SIZE;
3597 case RES_FAILCNT:
3598 return counter->failcnt;
3599 case RES_SOFT_LIMIT:
3600 return (u64)memcg->soft_limit * PAGE_SIZE;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003601 default:
3602 BUG();
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003603 }
Balbir Singh8cdea7c2008-02-07 00:13:50 -08003604}
Glauber Costa510fc4e2012-12-18 14:21:47 -08003605
Roman Gushchin4a87e2a2020-01-13 16:29:16 -08003606static void memcg_flush_percpu_vmstats(struct mem_cgroup *memcg)
Roman Gushchinc350a992019-08-24 17:54:47 -07003607{
Roman Gushchin4a87e2a2020-01-13 16:29:16 -08003608 unsigned long stat[MEMCG_NR_STAT] = {0};
Roman Gushchinc350a992019-08-24 17:54:47 -07003609 struct mem_cgroup *mi;
3610 int node, cpu, i;
Roman Gushchinc350a992019-08-24 17:54:47 -07003611
3612 for_each_online_cpu(cpu)
Roman Gushchin4a87e2a2020-01-13 16:29:16 -08003613 for (i = 0; i < MEMCG_NR_STAT; i++)
Shakeel Butt6c1c2802019-08-30 16:04:53 -07003614 stat[i] += per_cpu(memcg->vmstats_percpu->stat[i], cpu);
Roman Gushchinc350a992019-08-24 17:54:47 -07003615
3616 for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
Roman Gushchin4a87e2a2020-01-13 16:29:16 -08003617 for (i = 0; i < MEMCG_NR_STAT; i++)
Roman Gushchinc350a992019-08-24 17:54:47 -07003618 atomic_long_add(stat[i], &mi->vmstats[i]);
3619
3620 for_each_node(node) {
3621 struct mem_cgroup_per_node *pn = memcg->nodeinfo[node];
3622 struct mem_cgroup_per_node *pi;
3623
Roman Gushchin4a87e2a2020-01-13 16:29:16 -08003624 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
Roman Gushchinc350a992019-08-24 17:54:47 -07003625 stat[i] = 0;
3626
3627 for_each_online_cpu(cpu)
Roman Gushchin4a87e2a2020-01-13 16:29:16 -08003628 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
Shakeel Butt6c1c2802019-08-30 16:04:53 -07003629 stat[i] += per_cpu(
3630 pn->lruvec_stat_cpu->count[i], cpu);
Roman Gushchinc350a992019-08-24 17:54:47 -07003631
3632 for (pi = pn; pi; pi = parent_nodeinfo(pi, node))
Roman Gushchin4a87e2a2020-01-13 16:29:16 -08003633 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
Roman Gushchinc350a992019-08-24 17:54:47 -07003634 atomic_long_add(stat[i], &pi->lruvec_stat[i]);
3635 }
3636}
3637
Roman Gushchinbb65f892019-08-24 17:54:50 -07003638static void memcg_flush_percpu_vmevents(struct mem_cgroup *memcg)
3639{
3640 unsigned long events[NR_VM_EVENT_ITEMS];
3641 struct mem_cgroup *mi;
3642 int cpu, i;
3643
3644 for (i = 0; i < NR_VM_EVENT_ITEMS; i++)
3645 events[i] = 0;
3646
3647 for_each_online_cpu(cpu)
3648 for (i = 0; i < NR_VM_EVENT_ITEMS; i++)
Shakeel Butt6c1c2802019-08-30 16:04:53 -07003649 events[i] += per_cpu(memcg->vmstats_percpu->events[i],
3650 cpu);
Roman Gushchinbb65f892019-08-24 17:54:50 -07003651
3652 for (mi = memcg; mi; mi = parent_mem_cgroup(mi))
3653 for (i = 0; i < NR_VM_EVENT_ITEMS; i++)
3654 atomic_long_add(events[i], &mi->vmevents[i]);
3655}
3656
Kirill Tkhai84c07d12018-08-17 15:47:25 -07003657#ifdef CONFIG_MEMCG_KMEM
Johannes Weiner567e9ab2016-01-20 15:02:24 -08003658static int memcg_online_kmem(struct mem_cgroup *memcg)
Vladimir Davydovd6441632014-01-23 15:53:09 -08003659{
Roman Gushchinbf4f0592020-08-06 23:20:49 -07003660 struct obj_cgroup *objcg;
Vladimir Davydovd6441632014-01-23 15:53:09 -08003661 int memcg_id;
3662
Vladimir Davydovb313aee2016-03-17 14:18:27 -07003663 if (cgroup_memory_nokmem)
3664 return 0;
3665
Vladimir Davydov2a4db7e2015-02-12 14:59:32 -08003666 BUG_ON(memcg->kmemcg_id >= 0);
Johannes Weiner567e9ab2016-01-20 15:02:24 -08003667 BUG_ON(memcg->kmem_state);
Vladimir Davydovd6441632014-01-23 15:53:09 -08003668
Vladimir Davydovf3bb3042014-10-09 15:28:45 -07003669 memcg_id = memcg_alloc_cache_id();
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08003670 if (memcg_id < 0)
3671 return memcg_id;
Vladimir Davydovd6441632014-01-23 15:53:09 -08003672
Roman Gushchinbf4f0592020-08-06 23:20:49 -07003673 objcg = obj_cgroup_alloc();
3674 if (!objcg) {
3675 memcg_free_cache_id(memcg_id);
3676 return -ENOMEM;
3677 }
3678 objcg->memcg = memcg;
3679 rcu_assign_pointer(memcg->objcg, objcg);
3680
Roman Gushchind648bcc2020-08-06 23:20:28 -07003681 static_branch_enable(&memcg_kmem_enabled_key);
3682
Vladimir Davydov900a38f2014-12-12 16:55:10 -08003683 memcg->kmemcg_id = memcg_id;
Johannes Weiner567e9ab2016-01-20 15:02:24 -08003684 memcg->kmem_state = KMEM_ONLINE;
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08003685
3686 return 0;
Vladimir Davydovd6441632014-01-23 15:53:09 -08003687}
3688
Johannes Weiner8e0a8912016-01-20 15:02:26 -08003689static void memcg_offline_kmem(struct mem_cgroup *memcg)
3690{
3691 struct cgroup_subsys_state *css;
3692 struct mem_cgroup *parent, *child;
3693 int kmemcg_id;
3694
3695 if (memcg->kmem_state != KMEM_ONLINE)
3696 return;
Roman Gushchin98556092020-08-06 23:21:10 -07003697
Johannes Weiner8e0a8912016-01-20 15:02:26 -08003698 memcg->kmem_state = KMEM_ALLOCATED;
3699
Johannes Weiner8e0a8912016-01-20 15:02:26 -08003700 parent = parent_mem_cgroup(memcg);
3701 if (!parent)
3702 parent = root_mem_cgroup;
3703
Roman Gushchinbf4f0592020-08-06 23:20:49 -07003704 memcg_reparent_objcgs(memcg, parent);
Roman Gushchinfb2f2b02019-07-11 20:56:34 -07003705
3706 kmemcg_id = memcg->kmemcg_id;
3707 BUG_ON(kmemcg_id < 0);
3708
Johannes Weiner8e0a8912016-01-20 15:02:26 -08003709 /*
3710 * Change kmemcg_id of this cgroup and all its descendants to the
3711 * parent's id, and then move all entries from this cgroup's list_lrus
3712 * to ones of the parent. After we have finished, all list_lrus
3713 * corresponding to this cgroup are guaranteed to remain empty. The
3714 * ordering is imposed by list_lru_node->lock taken by
3715 * memcg_drain_all_list_lrus().
3716 */
Tejun Heo3a06bb72016-06-03 14:55:44 -07003717 rcu_read_lock(); /* can be called from css_free w/o cgroup_mutex */
Johannes Weiner8e0a8912016-01-20 15:02:26 -08003718 css_for_each_descendant_pre(css, &memcg->css) {
3719 child = mem_cgroup_from_css(css);
3720 BUG_ON(child->kmemcg_id != kmemcg_id);
3721 child->kmemcg_id = parent->kmemcg_id;
Johannes Weiner8e0a8912016-01-20 15:02:26 -08003722 }
Tejun Heo3a06bb72016-06-03 14:55:44 -07003723 rcu_read_unlock();
3724
Kirill Tkhai9bec5c32018-08-17 15:47:58 -07003725 memcg_drain_all_list_lrus(kmemcg_id, parent);
Johannes Weiner8e0a8912016-01-20 15:02:26 -08003726
3727 memcg_free_cache_id(kmemcg_id);
3728}
3729
3730static void memcg_free_kmem(struct mem_cgroup *memcg)
3731{
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08003732 /* css_alloc() failed, offlining didn't happen */
3733 if (unlikely(memcg->kmem_state == KMEM_ONLINE))
3734 memcg_offline_kmem(memcg);
Johannes Weiner8e0a8912016-01-20 15:02:26 -08003735}
Vladimir Davydovd6441632014-01-23 15:53:09 -08003736#else
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08003737static int memcg_online_kmem(struct mem_cgroup *memcg)
Johannes Weiner127424c2016-01-20 15:02:32 -08003738{
3739 return 0;
3740}
3741static void memcg_offline_kmem(struct mem_cgroup *memcg)
3742{
3743}
3744static void memcg_free_kmem(struct mem_cgroup *memcg)
3745{
3746}
Kirill Tkhai84c07d12018-08-17 15:47:25 -07003747#endif /* CONFIG_MEMCG_KMEM */
Johannes Weiner127424c2016-01-20 15:02:32 -08003748
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003749static int memcg_update_kmem_max(struct mem_cgroup *memcg,
3750 unsigned long max)
Johannes Weiner127424c2016-01-20 15:02:32 -08003751{
Vladimir Davydovb313aee2016-03-17 14:18:27 -07003752 int ret;
Johannes Weiner127424c2016-01-20 15:02:32 -08003753
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003754 mutex_lock(&memcg_max_mutex);
3755 ret = page_counter_set_max(&memcg->kmem, max);
3756 mutex_unlock(&memcg_max_mutex);
Johannes Weiner127424c2016-01-20 15:02:32 -08003757 return ret;
3758}
Glauber Costa510fc4e2012-12-18 14:21:47 -08003759
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003760static int memcg_update_tcp_max(struct mem_cgroup *memcg, unsigned long max)
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08003761{
3762 int ret;
3763
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003764 mutex_lock(&memcg_max_mutex);
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08003765
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003766 ret = page_counter_set_max(&memcg->tcpmem, max);
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08003767 if (ret)
3768 goto out;
3769
Johannes Weiner0db15292016-01-20 15:02:50 -08003770 if (!memcg->tcpmem_active) {
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08003771 /*
3772 * The active flag needs to be written after the static_key
3773 * update. This is what guarantees that the socket activation
Johannes Weiner2d758072016-10-07 17:00:58 -07003774 * function is the last one to run. See mem_cgroup_sk_alloc()
3775 * for details, and note that we don't mark any socket as
3776 * belonging to this memcg until that flag is up.
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08003777 *
3778 * We need to do this, because static_keys will span multiple
3779 * sites, but we can't control their order. If we mark a socket
3780 * as accounted, but the accounting functions are not patched in
3781 * yet, we'll lose accounting.
3782 *
Johannes Weiner2d758072016-10-07 17:00:58 -07003783 * We never race with the readers in mem_cgroup_sk_alloc(),
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08003784 * because when this value change, the code to process it is not
3785 * patched in yet.
3786 */
3787 static_branch_inc(&memcg_sockets_enabled_key);
Johannes Weiner0db15292016-01-20 15:02:50 -08003788 memcg->tcpmem_active = true;
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08003789 }
3790out:
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003791 mutex_unlock(&memcg_max_mutex);
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08003792 return ret;
3793}
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08003794
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003795/*
3796 * The user of this function is...
3797 * RES_LIMIT.
3798 */
Tejun Heo451af502014-05-13 12:16:21 -04003799static ssize_t mem_cgroup_write(struct kernfs_open_file *of,
3800 char *buf, size_t nbytes, loff_t off)
Balbir Singh8cdea7c2008-02-07 00:13:50 -08003801{
Tejun Heo451af502014-05-13 12:16:21 -04003802 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003803 unsigned long nr_pages;
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003804 int ret;
3805
Tejun Heo451af502014-05-13 12:16:21 -04003806 buf = strstrip(buf);
Johannes Weiner650c5e52015-02-11 15:26:03 -08003807 ret = page_counter_memparse(buf, "-1", &nr_pages);
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003808 if (ret)
3809 return ret;
Tejun Heoaf36f902012-04-01 12:09:55 -07003810
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003811 switch (MEMFILE_ATTR(of_cft(of)->private)) {
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003812 case RES_LIMIT:
Balbir Singh4b3bde42009-09-23 15:56:32 -07003813 if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
3814 ret = -EINVAL;
3815 break;
3816 }
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003817 switch (MEMFILE_TYPE(of_cft(of)->private)) {
3818 case _MEM:
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003819 ret = mem_cgroup_resize_max(memcg, nr_pages, false);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003820 break;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003821 case _MEMSWAP:
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003822 ret = mem_cgroup_resize_max(memcg, nr_pages, true);
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003823 break;
3824 case _KMEM:
Michal Hocko01581152019-09-23 15:37:22 -07003825 pr_warn_once("kmem.limit_in_bytes is deprecated and will be removed. "
3826 "Please report your usecase to linux-mm@kvack.org if you "
3827 "depend on this functionality.\n");
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003828 ret = memcg_update_kmem_max(memcg, nr_pages);
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003829 break;
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08003830 case _TCP:
Roman Gushchinbbec2e12018-06-07 17:06:18 -07003831 ret = memcg_update_tcp_max(memcg, nr_pages);
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08003832 break;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003833 }
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003834 break;
Balbir Singh296c81d2009-09-23 15:56:36 -07003835 case RES_SOFT_LIMIT:
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003836 memcg->soft_limit = nr_pages;
3837 ret = 0;
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003838 break;
3839 }
Tejun Heo451af502014-05-13 12:16:21 -04003840 return ret ?: nbytes;
Balbir Singh8cdea7c2008-02-07 00:13:50 -08003841}
3842
Tejun Heo6770c642014-05-13 12:16:21 -04003843static ssize_t mem_cgroup_reset(struct kernfs_open_file *of, char *buf,
3844 size_t nbytes, loff_t off)
Pavel Emelyanovc84872e2008-04-29 01:00:17 -07003845{
Tejun Heo6770c642014-05-13 12:16:21 -04003846 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003847 struct page_counter *counter;
Pavel Emelyanovc84872e2008-04-29 01:00:17 -07003848
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003849 switch (MEMFILE_TYPE(of_cft(of)->private)) {
3850 case _MEM:
3851 counter = &memcg->memory;
3852 break;
3853 case _MEMSWAP:
3854 counter = &memcg->memsw;
3855 break;
3856 case _KMEM:
3857 counter = &memcg->kmem;
3858 break;
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08003859 case _TCP:
Johannes Weiner0db15292016-01-20 15:02:50 -08003860 counter = &memcg->tcpmem;
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08003861 break;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003862 default:
3863 BUG();
3864 }
Tejun Heoaf36f902012-04-01 12:09:55 -07003865
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003866 switch (MEMFILE_ATTR(of_cft(of)->private)) {
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07003867 case RES_MAX_USAGE:
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003868 page_counter_reset_watermark(counter);
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07003869 break;
3870 case RES_FAILCNT:
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003871 counter->failcnt = 0;
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07003872 break;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08003873 default:
3874 BUG();
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07003875 }
Balbir Singhf64c3f52009-09-23 15:56:37 -07003876
Tejun Heo6770c642014-05-13 12:16:21 -04003877 return nbytes;
Pavel Emelyanovc84872e2008-04-29 01:00:17 -07003878}
3879
Tejun Heo182446d2013-08-08 20:11:24 -04003880static u64 mem_cgroup_move_charge_read(struct cgroup_subsys_state *css,
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08003881 struct cftype *cft)
3882{
Tejun Heo182446d2013-08-08 20:11:24 -04003883 return mem_cgroup_from_css(css)->move_charge_at_immigrate;
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08003884}
3885
Daisuke Nishimura02491442010-03-10 15:22:17 -08003886#ifdef CONFIG_MMU
Tejun Heo182446d2013-08-08 20:11:24 -04003887static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08003888 struct cftype *cft, u64 val)
3889{
Tejun Heo182446d2013-08-08 20:11:24 -04003890 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08003891
Johannes Weiner1dfab5a2015-02-11 15:26:09 -08003892 if (val & ~MOVE_MASK)
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08003893 return -EINVAL;
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08003894
Glauber Costaee5e8472013-02-22 16:34:50 -08003895 /*
3896 * No kind of locking is needed in here, because ->can_attach() will
3897 * check this value once in the beginning of the process, and then carry
3898 * on with stale data. This means that changes to this value will only
3899 * affect task migrations starting after the change.
3900 */
3901 memcg->move_charge_at_immigrate = val;
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08003902 return 0;
3903}
Daisuke Nishimura02491442010-03-10 15:22:17 -08003904#else
Tejun Heo182446d2013-08-08 20:11:24 -04003905static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
Daisuke Nishimura02491442010-03-10 15:22:17 -08003906 struct cftype *cft, u64 val)
3907{
3908 return -ENOSYS;
3909}
3910#endif
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08003911
Ying Han406eb0c2011-05-26 16:25:37 -07003912#ifdef CONFIG_NUMA
Johannes Weiner113b7df2019-05-13 17:18:11 -07003913
3914#define LRU_ALL_FILE (BIT(LRU_INACTIVE_FILE) | BIT(LRU_ACTIVE_FILE))
3915#define LRU_ALL_ANON (BIT(LRU_INACTIVE_ANON) | BIT(LRU_ACTIVE_ANON))
3916#define LRU_ALL ((1 << NR_LRU_LISTS) - 1)
3917
3918static unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
Shakeel Buttdd8657b2020-06-03 15:56:24 -07003919 int nid, unsigned int lru_mask, bool tree)
Johannes Weiner113b7df2019-05-13 17:18:11 -07003920{
Johannes Weiner867e5e12019-11-30 17:55:34 -08003921 struct lruvec *lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid));
Johannes Weiner113b7df2019-05-13 17:18:11 -07003922 unsigned long nr = 0;
3923 enum lru_list lru;
3924
3925 VM_BUG_ON((unsigned)nid >= nr_node_ids);
3926
3927 for_each_lru(lru) {
3928 if (!(BIT(lru) & lru_mask))
3929 continue;
Shakeel Buttdd8657b2020-06-03 15:56:24 -07003930 if (tree)
3931 nr += lruvec_page_state(lruvec, NR_LRU_BASE + lru);
3932 else
3933 nr += lruvec_page_state_local(lruvec, NR_LRU_BASE + lru);
Johannes Weiner113b7df2019-05-13 17:18:11 -07003934 }
3935 return nr;
3936}
3937
3938static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg,
Shakeel Buttdd8657b2020-06-03 15:56:24 -07003939 unsigned int lru_mask,
3940 bool tree)
Johannes Weiner113b7df2019-05-13 17:18:11 -07003941{
3942 unsigned long nr = 0;
3943 enum lru_list lru;
3944
3945 for_each_lru(lru) {
3946 if (!(BIT(lru) & lru_mask))
3947 continue;
Shakeel Buttdd8657b2020-06-03 15:56:24 -07003948 if (tree)
3949 nr += memcg_page_state(memcg, NR_LRU_BASE + lru);
3950 else
3951 nr += memcg_page_state_local(memcg, NR_LRU_BASE + lru);
Johannes Weiner113b7df2019-05-13 17:18:11 -07003952 }
3953 return nr;
3954}
3955
Tejun Heo2da8ca82013-12-05 12:28:04 -05003956static int memcg_numa_stat_show(struct seq_file *m, void *v)
Ying Han406eb0c2011-05-26 16:25:37 -07003957{
Greg Thelen25485de2013-11-12 15:07:40 -08003958 struct numa_stat {
3959 const char *name;
3960 unsigned int lru_mask;
3961 };
3962
3963 static const struct numa_stat stats[] = {
3964 { "total", LRU_ALL },
3965 { "file", LRU_ALL_FILE },
3966 { "anon", LRU_ALL_ANON },
3967 { "unevictable", BIT(LRU_UNEVICTABLE) },
3968 };
3969 const struct numa_stat *stat;
Ying Han406eb0c2011-05-26 16:25:37 -07003970 int nid;
Chris Downaa9694b2019-03-05 15:45:52 -08003971 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
Ying Han406eb0c2011-05-26 16:25:37 -07003972
Greg Thelen25485de2013-11-12 15:07:40 -08003973 for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
Shakeel Buttdd8657b2020-06-03 15:56:24 -07003974 seq_printf(m, "%s=%lu", stat->name,
3975 mem_cgroup_nr_lru_pages(memcg, stat->lru_mask,
3976 false));
3977 for_each_node_state(nid, N_MEMORY)
3978 seq_printf(m, " N%d=%lu", nid,
3979 mem_cgroup_node_nr_lru_pages(memcg, nid,
3980 stat->lru_mask, false));
Greg Thelen25485de2013-11-12 15:07:40 -08003981 seq_putc(m, '\n');
Ying Han406eb0c2011-05-26 16:25:37 -07003982 }
Ying Han406eb0c2011-05-26 16:25:37 -07003983
Ying Han071aee12013-11-12 15:07:41 -08003984 for (stat = stats; stat < stats + ARRAY_SIZE(stats); stat++) {
Ying Han406eb0c2011-05-26 16:25:37 -07003985
Shakeel Buttdd8657b2020-06-03 15:56:24 -07003986 seq_printf(m, "hierarchical_%s=%lu", stat->name,
3987 mem_cgroup_nr_lru_pages(memcg, stat->lru_mask,
3988 true));
3989 for_each_node_state(nid, N_MEMORY)
3990 seq_printf(m, " N%d=%lu", nid,
3991 mem_cgroup_node_nr_lru_pages(memcg, nid,
3992 stat->lru_mask, true));
Ying Han071aee12013-11-12 15:07:41 -08003993 seq_putc(m, '\n');
Ying Han406eb0c2011-05-26 16:25:37 -07003994 }
Ying Han406eb0c2011-05-26 16:25:37 -07003995
Ying Han406eb0c2011-05-26 16:25:37 -07003996 return 0;
3997}
3998#endif /* CONFIG_NUMA */
3999
Johannes Weinerc8713d02019-07-11 20:55:59 -07004000static const unsigned int memcg1_stats[] = {
Johannes Weiner0d1c2072020-06-03 16:01:54 -07004001 NR_FILE_PAGES,
Johannes Weinerbe5d0a72020-06-03 16:01:57 -07004002 NR_ANON_MAPPED,
Johannes Weiner468c3982020-06-03 16:02:01 -07004003#ifdef CONFIG_TRANSPARENT_HUGEPAGE
4004 NR_ANON_THPS,
4005#endif
Johannes Weinerc8713d02019-07-11 20:55:59 -07004006 NR_SHMEM,
4007 NR_FILE_MAPPED,
4008 NR_FILE_DIRTY,
4009 NR_WRITEBACK,
4010 MEMCG_SWAP,
4011};
4012
4013static const char *const memcg1_stat_names[] = {
4014 "cache",
4015 "rss",
Johannes Weiner468c3982020-06-03 16:02:01 -07004016#ifdef CONFIG_TRANSPARENT_HUGEPAGE
Johannes Weinerc8713d02019-07-11 20:55:59 -07004017 "rss_huge",
Johannes Weiner468c3982020-06-03 16:02:01 -07004018#endif
Johannes Weinerc8713d02019-07-11 20:55:59 -07004019 "shmem",
4020 "mapped_file",
4021 "dirty",
4022 "writeback",
4023 "swap",
4024};
4025
Johannes Weinerdf0e53d2017-05-03 14:55:10 -07004026/* Universal VM events cgroup1 shows, original sort order */
Greg Thelen8dd53fd2018-06-07 17:07:23 -07004027static const unsigned int memcg1_events[] = {
Johannes Weinerdf0e53d2017-05-03 14:55:10 -07004028 PGPGIN,
4029 PGPGOUT,
4030 PGFAULT,
4031 PGMAJFAULT,
4032};
4033
Tejun Heo2da8ca82013-12-05 12:28:04 -05004034static int memcg_stat_show(struct seq_file *m, void *v)
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08004035{
Chris Downaa9694b2019-03-05 15:45:52 -08004036 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
Johannes Weiner3e32cb22014-12-10 15:42:31 -08004037 unsigned long memory, memsw;
Johannes Weineraf7c4b02012-05-29 15:07:08 -07004038 struct mem_cgroup *mi;
4039 unsigned int i;
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08004040
Johannes Weiner71cd3112017-05-03 14:55:13 -07004041 BUILD_BUG_ON(ARRAY_SIZE(memcg1_stat_names) != ARRAY_SIZE(memcg1_stats));
Rickard Strandqvist70bc0682014-12-12 16:56:41 -08004042
Johannes Weiner71cd3112017-05-03 14:55:13 -07004043 for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
Johannes Weiner468c3982020-06-03 16:02:01 -07004044 unsigned long nr;
4045
Johannes Weiner71cd3112017-05-03 14:55:13 -07004046 if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account())
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07004047 continue;
Johannes Weiner468c3982020-06-03 16:02:01 -07004048 nr = memcg_page_state_local(memcg, memcg1_stats[i]);
Johannes Weiner468c3982020-06-03 16:02:01 -07004049 seq_printf(m, "%s %lu\n", memcg1_stat_names[i], nr * PAGE_SIZE);
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07004050 }
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08004051
Johannes Weinerdf0e53d2017-05-03 14:55:10 -07004052 for (i = 0; i < ARRAY_SIZE(memcg1_events); i++)
Konstantin Khlebnikovebc5d83d2019-12-04 16:49:53 -08004053 seq_printf(m, "%s %lu\n", vm_event_name(memcg1_events[i]),
Johannes Weiner205b20c2019-05-14 15:47:06 -07004054 memcg_events_local(memcg, memcg1_events[i]));
Johannes Weineraf7c4b02012-05-29 15:07:08 -07004055
4056 for (i = 0; i < NR_LRU_LISTS; i++)
Konstantin Khlebnikovebc5d83d2019-12-04 16:49:53 -08004057 seq_printf(m, "%s %lu\n", lru_list_name(i),
Johannes Weiner205b20c2019-05-14 15:47:06 -07004058 memcg_page_state_local(memcg, NR_LRU_BASE + i) *
Johannes Weiner21d89d12019-05-13 17:18:08 -07004059 PAGE_SIZE);
Johannes Weineraf7c4b02012-05-29 15:07:08 -07004060
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07004061 /* Hierarchical information */
Johannes Weiner3e32cb22014-12-10 15:42:31 -08004062 memory = memsw = PAGE_COUNTER_MAX;
4063 for (mi = memcg; mi; mi = parent_mem_cgroup(mi)) {
Chris Down15b42562020-04-01 21:07:20 -07004064 memory = min(memory, READ_ONCE(mi->memory.max));
4065 memsw = min(memsw, READ_ONCE(mi->memsw.max));
KAMEZAWA Hiroyukifee7b542009-01-07 18:08:26 -08004066 }
Johannes Weiner3e32cb22014-12-10 15:42:31 -08004067 seq_printf(m, "hierarchical_memory_limit %llu\n",
4068 (u64)memory * PAGE_SIZE);
Johannes Weiner7941d212016-01-14 15:21:23 -08004069 if (do_memsw_account())
Johannes Weiner3e32cb22014-12-10 15:42:31 -08004070 seq_printf(m, "hierarchical_memsw_limit %llu\n",
4071 (u64)memsw * PAGE_SIZE);
KOSAKI Motohiro7f016ee2009-01-07 18:08:22 -08004072
Shakeel Butt8de7ecc62018-08-21 21:53:17 -07004073 for (i = 0; i < ARRAY_SIZE(memcg1_stats); i++) {
zhongjiang-ali7de2e9f2020-11-01 17:07:30 -08004074 unsigned long nr;
4075
Johannes Weiner71cd3112017-05-03 14:55:13 -07004076 if (memcg1_stats[i] == MEMCG_SWAP && !do_memsw_account())
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07004077 continue;
zhongjiang-ali7de2e9f2020-11-01 17:07:30 -08004078 nr = memcg_page_state(memcg, memcg1_stats[i]);
Shakeel Butt8de7ecc62018-08-21 21:53:17 -07004079 seq_printf(m, "total_%s %llu\n", memcg1_stat_names[i],
zhongjiang-ali7de2e9f2020-11-01 17:07:30 -08004080 (u64)nr * PAGE_SIZE);
Johannes Weineraf7c4b02012-05-29 15:07:08 -07004081 }
4082
Shakeel Butt8de7ecc62018-08-21 21:53:17 -07004083 for (i = 0; i < ARRAY_SIZE(memcg1_events); i++)
Konstantin Khlebnikovebc5d83d2019-12-04 16:49:53 -08004084 seq_printf(m, "total_%s %llu\n",
4085 vm_event_name(memcg1_events[i]),
Yafang Shaodd923992019-07-11 20:52:11 -07004086 (u64)memcg_events(memcg, memcg1_events[i]));
Johannes Weineraf7c4b02012-05-29 15:07:08 -07004087
Shakeel Butt8de7ecc62018-08-21 21:53:17 -07004088 for (i = 0; i < NR_LRU_LISTS; i++)
Konstantin Khlebnikovebc5d83d2019-12-04 16:49:53 -08004089 seq_printf(m, "total_%s %llu\n", lru_list_name(i),
Johannes Weiner42a30032019-05-14 15:47:12 -07004090 (u64)memcg_page_state(memcg, NR_LRU_BASE + i) *
4091 PAGE_SIZE);
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07004092
KOSAKI Motohiro7f016ee2009-01-07 18:08:22 -08004093#ifdef CONFIG_DEBUG_VM
KOSAKI Motohiro7f016ee2009-01-07 18:08:22 -08004094 {
Mel Gormanef8f2322016-07-28 15:46:05 -07004095 pg_data_t *pgdat;
4096 struct mem_cgroup_per_node *mz;
Johannes Weiner1431d4d2020-06-03 16:02:53 -07004097 unsigned long anon_cost = 0;
4098 unsigned long file_cost = 0;
KOSAKI Motohiro7f016ee2009-01-07 18:08:22 -08004099
Mel Gormanef8f2322016-07-28 15:46:05 -07004100 for_each_online_pgdat(pgdat) {
4101 mz = mem_cgroup_nodeinfo(memcg, pgdat->node_id);
KOSAKI Motohiro7f016ee2009-01-07 18:08:22 -08004102
Johannes Weiner1431d4d2020-06-03 16:02:53 -07004103 anon_cost += mz->lruvec.anon_cost;
4104 file_cost += mz->lruvec.file_cost;
Mel Gormanef8f2322016-07-28 15:46:05 -07004105 }
Johannes Weiner1431d4d2020-06-03 16:02:53 -07004106 seq_printf(m, "anon_cost %lu\n", anon_cost);
4107 seq_printf(m, "file_cost %lu\n", file_cost);
KOSAKI Motohiro7f016ee2009-01-07 18:08:22 -08004108 }
4109#endif
4110
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08004111 return 0;
4112}
4113
Tejun Heo182446d2013-08-08 20:11:24 -04004114static u64 mem_cgroup_swappiness_read(struct cgroup_subsys_state *css,
4115 struct cftype *cft)
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08004116{
Tejun Heo182446d2013-08-08 20:11:24 -04004117 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08004118
KAMEZAWA Hiroyuki1f4c0252011-07-26 16:08:21 -07004119 return mem_cgroup_swappiness(memcg);
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08004120}
4121
Tejun Heo182446d2013-08-08 20:11:24 -04004122static int mem_cgroup_swappiness_write(struct cgroup_subsys_state *css,
4123 struct cftype *cft, u64 val)
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08004124{
Tejun Heo182446d2013-08-08 20:11:24 -04004125 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
Li Zefan068b38c2009-01-15 13:51:26 -08004126
Johannes Weiner3dae7fe2014-06-04 16:07:01 -07004127 if (val > 100)
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08004128 return -EINVAL;
4129
Shakeel Butta4792032021-04-29 22:56:05 -07004130 if (!mem_cgroup_is_root(memcg))
Johannes Weiner3dae7fe2014-06-04 16:07:01 -07004131 memcg->swappiness = val;
4132 else
4133 vm_swappiness = val;
Li Zefan068b38c2009-01-15 13:51:26 -08004134
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08004135 return 0;
4136}
4137
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004138static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
4139{
4140 struct mem_cgroup_threshold_ary *t;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08004141 unsigned long usage;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004142 int i;
4143
4144 rcu_read_lock();
4145 if (!swap)
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004146 t = rcu_dereference(memcg->thresholds.primary);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004147 else
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004148 t = rcu_dereference(memcg->memsw_thresholds.primary);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004149
4150 if (!t)
4151 goto unlock;
4152
Johannes Weinerce00a962014-09-05 08:43:57 -04004153 usage = mem_cgroup_usage(memcg, swap);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004154
4155 /*
Sha Zhengju748dad32012-05-29 15:06:57 -07004156 * current_threshold points to threshold just below or equal to usage.
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004157 * If it's not true, a threshold was crossed after last
4158 * call of __mem_cgroup_threshold().
4159 */
Phil Carmody5407a562010-05-26 14:42:42 -07004160 i = t->current_threshold;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004161
4162 /*
4163 * Iterate backward over array of thresholds starting from
4164 * current_threshold and check if a threshold is crossed.
4165 * If none of thresholds below usage is crossed, we read
4166 * only one element of the array here.
4167 */
4168 for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
4169 eventfd_signal(t->entries[i].eventfd, 1);
4170
4171 /* i = current_threshold + 1 */
4172 i++;
4173
4174 /*
4175 * Iterate forward over array of thresholds starting from
4176 * current_threshold+1 and check if a threshold is crossed.
4177 * If none of thresholds above usage is crossed, we read
4178 * only one element of the array here.
4179 */
4180 for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
4181 eventfd_signal(t->entries[i].eventfd, 1);
4182
4183 /* Update current_threshold */
Phil Carmody5407a562010-05-26 14:42:42 -07004184 t->current_threshold = i - 1;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004185unlock:
4186 rcu_read_unlock();
4187}
4188
4189static void mem_cgroup_threshold(struct mem_cgroup *memcg)
4190{
Kirill A. Shutemovad4ca5f2010-10-07 12:59:27 -07004191 while (memcg) {
4192 __mem_cgroup_threshold(memcg, false);
Johannes Weiner7941d212016-01-14 15:21:23 -08004193 if (do_memsw_account())
Kirill A. Shutemovad4ca5f2010-10-07 12:59:27 -07004194 __mem_cgroup_threshold(memcg, true);
4195
4196 memcg = parent_mem_cgroup(memcg);
4197 }
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004198}
4199
4200static int compare_thresholds(const void *a, const void *b)
4201{
4202 const struct mem_cgroup_threshold *_a = a;
4203 const struct mem_cgroup_threshold *_b = b;
4204
Greg Thelen2bff24a2013-09-11 14:23:08 -07004205 if (_a->threshold > _b->threshold)
4206 return 1;
4207
4208 if (_a->threshold < _b->threshold)
4209 return -1;
4210
4211 return 0;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004212}
4213
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004214static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004215{
4216 struct mem_cgroup_eventfd_list *ev;
4217
Michal Hocko2bcf2e92014-07-30 16:08:33 -07004218 spin_lock(&memcg_oom_lock);
4219
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004220 list_for_each_entry(ev, &memcg->oom_notify, list)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004221 eventfd_signal(ev->eventfd, 1);
Michal Hocko2bcf2e92014-07-30 16:08:33 -07004222
4223 spin_unlock(&memcg_oom_lock);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004224 return 0;
4225}
4226
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004227static void mem_cgroup_oom_notify(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004228{
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07004229 struct mem_cgroup *iter;
4230
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004231 for_each_mem_cgroup_tree(iter, memcg)
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07004232 mem_cgroup_oom_notify_cb(iter);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004233}
4234
Tejun Heo59b6f872013-11-22 18:20:43 -05004235static int __mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
Tejun Heo347c4a82013-11-22 18:20:43 -05004236 struct eventfd_ctx *eventfd, const char *args, enum res_type type)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004237{
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004238 struct mem_cgroup_thresholds *thresholds;
4239 struct mem_cgroup_threshold_ary *new;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08004240 unsigned long threshold;
4241 unsigned long usage;
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004242 int i, size, ret;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004243
Johannes Weiner650c5e52015-02-11 15:26:03 -08004244 ret = page_counter_memparse(args, "-1", &threshold);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004245 if (ret)
4246 return ret;
4247
4248 mutex_lock(&memcg->thresholds_lock);
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004249
Johannes Weiner05b84302014-08-06 16:05:59 -07004250 if (type == _MEM) {
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004251 thresholds = &memcg->thresholds;
Johannes Weinerce00a962014-09-05 08:43:57 -04004252 usage = mem_cgroup_usage(memcg, false);
Johannes Weiner05b84302014-08-06 16:05:59 -07004253 } else if (type == _MEMSWAP) {
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004254 thresholds = &memcg->memsw_thresholds;
Johannes Weinerce00a962014-09-05 08:43:57 -04004255 usage = mem_cgroup_usage(memcg, true);
Johannes Weiner05b84302014-08-06 16:05:59 -07004256 } else
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004257 BUG();
4258
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004259 /* Check if a threshold crossed before adding a new one */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004260 if (thresholds->primary)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004261 __mem_cgroup_threshold(memcg, type == _MEMSWAP);
4262
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004263 size = thresholds->primary ? thresholds->primary->size + 1 : 1;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004264
4265 /* Allocate memory for new array of thresholds */
Gustavo A. R. Silva67b80462019-03-05 15:44:05 -08004266 new = kmalloc(struct_size(new, entries, size), GFP_KERNEL);
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004267 if (!new) {
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004268 ret = -ENOMEM;
4269 goto unlock;
4270 }
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004271 new->size = size;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004272
4273 /* Copy thresholds (if any) to new array */
Gustavo A. R. Silvae90342e2020-10-13 16:52:36 -07004274 if (thresholds->primary)
4275 memcpy(new->entries, thresholds->primary->entries,
4276 flex_array_size(new, entries, size - 1));
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004277
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004278 /* Add new threshold */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004279 new->entries[size - 1].eventfd = eventfd;
4280 new->entries[size - 1].threshold = threshold;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004281
4282 /* Sort thresholds. Registering of new threshold isn't time-critical */
Gustavo A. R. Silva61e604e62020-10-13 16:52:39 -07004283 sort(new->entries, size, sizeof(*new->entries),
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004284 compare_thresholds, NULL);
4285
4286 /* Find current threshold */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004287 new->current_threshold = -1;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004288 for (i = 0; i < size; i++) {
Sha Zhengju748dad32012-05-29 15:06:57 -07004289 if (new->entries[i].threshold <= usage) {
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004290 /*
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004291 * new->current_threshold will not be used until
4292 * rcu_assign_pointer(), so it's safe to increment
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004293 * it here.
4294 */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004295 ++new->current_threshold;
Sha Zhengju748dad32012-05-29 15:06:57 -07004296 } else
4297 break;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004298 }
4299
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004300 /* Free old spare buffer and save old primary buffer as spare */
4301 kfree(thresholds->spare);
4302 thresholds->spare = thresholds->primary;
4303
4304 rcu_assign_pointer(thresholds->primary, new);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004305
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07004306 /* To be sure that nobody uses thresholds */
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004307 synchronize_rcu();
4308
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004309unlock:
4310 mutex_unlock(&memcg->thresholds_lock);
4311
4312 return ret;
4313}
4314
Tejun Heo59b6f872013-11-22 18:20:43 -05004315static int mem_cgroup_usage_register_event(struct mem_cgroup *memcg,
Tejun Heo347c4a82013-11-22 18:20:43 -05004316 struct eventfd_ctx *eventfd, const char *args)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004317{
Tejun Heo59b6f872013-11-22 18:20:43 -05004318 return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEM);
Tejun Heo347c4a82013-11-22 18:20:43 -05004319}
4320
Tejun Heo59b6f872013-11-22 18:20:43 -05004321static int memsw_cgroup_usage_register_event(struct mem_cgroup *memcg,
Tejun Heo347c4a82013-11-22 18:20:43 -05004322 struct eventfd_ctx *eventfd, const char *args)
4323{
Tejun Heo59b6f872013-11-22 18:20:43 -05004324 return __mem_cgroup_usage_register_event(memcg, eventfd, args, _MEMSWAP);
Tejun Heo347c4a82013-11-22 18:20:43 -05004325}
4326
Tejun Heo59b6f872013-11-22 18:20:43 -05004327static void __mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
Tejun Heo347c4a82013-11-22 18:20:43 -05004328 struct eventfd_ctx *eventfd, enum res_type type)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004329{
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004330 struct mem_cgroup_thresholds *thresholds;
4331 struct mem_cgroup_threshold_ary *new;
Johannes Weiner3e32cb22014-12-10 15:42:31 -08004332 unsigned long usage;
Chunguang Xu7d366652020-03-21 18:22:10 -07004333 int i, j, size, entries;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004334
4335 mutex_lock(&memcg->thresholds_lock);
Johannes Weiner05b84302014-08-06 16:05:59 -07004336
4337 if (type == _MEM) {
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004338 thresholds = &memcg->thresholds;
Johannes Weinerce00a962014-09-05 08:43:57 -04004339 usage = mem_cgroup_usage(memcg, false);
Johannes Weiner05b84302014-08-06 16:05:59 -07004340 } else if (type == _MEMSWAP) {
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004341 thresholds = &memcg->memsw_thresholds;
Johannes Weinerce00a962014-09-05 08:43:57 -04004342 usage = mem_cgroup_usage(memcg, true);
Johannes Weiner05b84302014-08-06 16:05:59 -07004343 } else
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004344 BUG();
4345
Anton Vorontsov371528c2012-02-24 05:14:46 +04004346 if (!thresholds->primary)
4347 goto unlock;
4348
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004349 /* Check if a threshold crossed before removing */
4350 __mem_cgroup_threshold(memcg, type == _MEMSWAP);
4351
4352 /* Calculate new number of threshold */
Chunguang Xu7d366652020-03-21 18:22:10 -07004353 size = entries = 0;
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004354 for (i = 0; i < thresholds->primary->size; i++) {
4355 if (thresholds->primary->entries[i].eventfd != eventfd)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004356 size++;
Chunguang Xu7d366652020-03-21 18:22:10 -07004357 else
4358 entries++;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004359 }
4360
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004361 new = thresholds->spare;
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07004362
Chunguang Xu7d366652020-03-21 18:22:10 -07004363 /* If no items related to eventfd have been cleared, nothing to do */
4364 if (!entries)
4365 goto unlock;
4366
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004367 /* Set thresholds array to NULL if we don't have thresholds */
4368 if (!size) {
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004369 kfree(new);
4370 new = NULL;
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07004371 goto swap_buffers;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004372 }
4373
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004374 new->size = size;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004375
4376 /* Copy thresholds and find current threshold */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004377 new->current_threshold = -1;
4378 for (i = 0, j = 0; i < thresholds->primary->size; i++) {
4379 if (thresholds->primary->entries[i].eventfd == eventfd)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004380 continue;
4381
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004382 new->entries[j] = thresholds->primary->entries[i];
Sha Zhengju748dad32012-05-29 15:06:57 -07004383 if (new->entries[j].threshold <= usage) {
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004384 /*
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004385 * new->current_threshold will not be used
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004386 * until rcu_assign_pointer(), so it's safe to increment
4387 * it here.
4388 */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004389 ++new->current_threshold;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004390 }
4391 j++;
4392 }
4393
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07004394swap_buffers:
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004395 /* Swap primary and spare array */
4396 thresholds->spare = thresholds->primary;
Sha Zhengju8c757762012-05-10 13:01:45 -07004397
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004398 rcu_assign_pointer(thresholds->primary, new);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004399
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07004400 /* To be sure that nobody uses thresholds */
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004401 synchronize_rcu();
Martijn Coenen6611d8d2016-01-15 16:57:49 -08004402
4403 /* If all events are unregistered, free the spare array */
4404 if (!new) {
4405 kfree(thresholds->spare);
4406 thresholds->spare = NULL;
4407 }
Anton Vorontsov371528c2012-02-24 05:14:46 +04004408unlock:
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004409 mutex_unlock(&memcg->thresholds_lock);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004410}
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08004411
Tejun Heo59b6f872013-11-22 18:20:43 -05004412static void mem_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
Tejun Heo347c4a82013-11-22 18:20:43 -05004413 struct eventfd_ctx *eventfd)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004414{
Tejun Heo59b6f872013-11-22 18:20:43 -05004415 return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEM);
Tejun Heo347c4a82013-11-22 18:20:43 -05004416}
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004417
Tejun Heo59b6f872013-11-22 18:20:43 -05004418static void memsw_cgroup_usage_unregister_event(struct mem_cgroup *memcg,
Tejun Heo347c4a82013-11-22 18:20:43 -05004419 struct eventfd_ctx *eventfd)
4420{
Tejun Heo59b6f872013-11-22 18:20:43 -05004421 return __mem_cgroup_usage_unregister_event(memcg, eventfd, _MEMSWAP);
Tejun Heo347c4a82013-11-22 18:20:43 -05004422}
4423
Tejun Heo59b6f872013-11-22 18:20:43 -05004424static int mem_cgroup_oom_register_event(struct mem_cgroup *memcg,
Tejun Heo347c4a82013-11-22 18:20:43 -05004425 struct eventfd_ctx *eventfd, const char *args)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004426{
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004427 struct mem_cgroup_eventfd_list *event;
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004428
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004429 event = kmalloc(sizeof(*event), GFP_KERNEL);
4430 if (!event)
4431 return -ENOMEM;
4432
Michal Hocko1af8efe2011-07-26 16:08:24 -07004433 spin_lock(&memcg_oom_lock);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004434
4435 event->eventfd = eventfd;
4436 list_add(&event->list, &memcg->oom_notify);
4437
4438 /* already in OOM ? */
Tejun Heoc2b42d32015-06-24 16:58:23 -07004439 if (memcg->under_oom)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004440 eventfd_signal(eventfd, 1);
Michal Hocko1af8efe2011-07-26 16:08:24 -07004441 spin_unlock(&memcg_oom_lock);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004442
4443 return 0;
4444}
4445
Tejun Heo59b6f872013-11-22 18:20:43 -05004446static void mem_cgroup_oom_unregister_event(struct mem_cgroup *memcg,
Tejun Heo347c4a82013-11-22 18:20:43 -05004447 struct eventfd_ctx *eventfd)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004448{
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004449 struct mem_cgroup_eventfd_list *ev, *tmp;
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004450
Michal Hocko1af8efe2011-07-26 16:08:24 -07004451 spin_lock(&memcg_oom_lock);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004452
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004453 list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) {
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004454 if (ev->eventfd == eventfd) {
4455 list_del(&ev->list);
4456 kfree(ev);
4457 }
4458 }
4459
Michal Hocko1af8efe2011-07-26 16:08:24 -07004460 spin_unlock(&memcg_oom_lock);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004461}
4462
Tejun Heo2da8ca82013-12-05 12:28:04 -05004463static int mem_cgroup_oom_control_read(struct seq_file *sf, void *v)
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004464{
Chris Downaa9694b2019-03-05 15:45:52 -08004465 struct mem_cgroup *memcg = mem_cgroup_from_seq(sf);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004466
Tejun Heo791badb2013-12-05 12:28:02 -05004467 seq_printf(sf, "oom_kill_disable %d\n", memcg->oom_kill_disable);
Tejun Heoc2b42d32015-06-24 16:58:23 -07004468 seq_printf(sf, "under_oom %d\n", (bool)memcg->under_oom);
Roman Gushchinfe6bdfc2018-06-14 15:28:05 -07004469 seq_printf(sf, "oom_kill %lu\n",
4470 atomic_long_read(&memcg->memory_events[MEMCG_OOM_KILL]));
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004471 return 0;
4472}
4473
Tejun Heo182446d2013-08-08 20:11:24 -04004474static int mem_cgroup_oom_control_write(struct cgroup_subsys_state *css,
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004475 struct cftype *cft, u64 val)
4476{
Tejun Heo182446d2013-08-08 20:11:24 -04004477 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004478
4479 /* cannot set to root cgroup and only 0 and 1 are allowed */
Shakeel Butta4792032021-04-29 22:56:05 -07004480 if (mem_cgroup_is_root(memcg) || !((val == 0) || (val == 1)))
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004481 return -EINVAL;
4482
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004483 memcg->oom_kill_disable = val;
KAMEZAWA Hiroyuki4d845eb2010-06-29 15:05:18 -07004484 if (!val)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004485 memcg_oom_recover(memcg);
Johannes Weiner3dae7fe2014-06-04 16:07:01 -07004486
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004487 return 0;
4488}
4489
Tejun Heo52ebea72015-05-22 17:13:37 -04004490#ifdef CONFIG_CGROUP_WRITEBACK
4491
Tejun Heo3a8e9ac2019-08-29 15:47:19 -07004492#include <trace/events/writeback.h>
4493
Tejun Heo841710a2015-05-22 18:23:33 -04004494static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
4495{
4496 return wb_domain_init(&memcg->cgwb_domain, gfp);
4497}
4498
4499static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
4500{
4501 wb_domain_exit(&memcg->cgwb_domain);
4502}
4503
Tejun Heo2529bb32015-05-22 18:23:34 -04004504static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
4505{
4506 wb_domain_size_changed(&memcg->cgwb_domain);
4507}
4508
Tejun Heo841710a2015-05-22 18:23:33 -04004509struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb)
4510{
4511 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
4512
4513 if (!memcg->css.parent)
4514 return NULL;
4515
4516 return &memcg->cgwb_domain;
4517}
4518
Greg Thelen0b3d6e62019-04-05 18:39:18 -07004519/*
4520 * idx can be of type enum memcg_stat_item or node_stat_item.
4521 * Keep in sync with memcg_exact_page().
4522 */
4523static unsigned long memcg_exact_page_state(struct mem_cgroup *memcg, int idx)
4524{
Chris Down871789d2019-05-14 15:46:57 -07004525 long x = atomic_long_read(&memcg->vmstats[idx]);
Greg Thelen0b3d6e62019-04-05 18:39:18 -07004526 int cpu;
4527
4528 for_each_online_cpu(cpu)
Chris Down871789d2019-05-14 15:46:57 -07004529 x += per_cpu_ptr(memcg->vmstats_percpu, cpu)->stat[idx];
Greg Thelen0b3d6e62019-04-05 18:39:18 -07004530 if (x < 0)
4531 x = 0;
4532 return x;
4533}
4534
Tejun Heoc2aa7232015-05-22 18:23:35 -04004535/**
4536 * mem_cgroup_wb_stats - retrieve writeback related stats from its memcg
4537 * @wb: bdi_writeback in question
Tejun Heoc5edf9c2015-09-29 13:04:26 -04004538 * @pfilepages: out parameter for number of file pages
4539 * @pheadroom: out parameter for number of allocatable pages according to memcg
Tejun Heoc2aa7232015-05-22 18:23:35 -04004540 * @pdirty: out parameter for number of dirty pages
4541 * @pwriteback: out parameter for number of pages under writeback
4542 *
Tejun Heoc5edf9c2015-09-29 13:04:26 -04004543 * Determine the numbers of file, headroom, dirty, and writeback pages in
4544 * @wb's memcg. File, dirty and writeback are self-explanatory. Headroom
4545 * is a bit more involved.
Tejun Heoc2aa7232015-05-22 18:23:35 -04004546 *
Tejun Heoc5edf9c2015-09-29 13:04:26 -04004547 * A memcg's headroom is "min(max, high) - used". In the hierarchy, the
4548 * headroom is calculated as the lowest headroom of itself and the
4549 * ancestors. Note that this doesn't consider the actual amount of
4550 * available memory in the system. The caller should further cap
4551 * *@pheadroom accordingly.
Tejun Heoc2aa7232015-05-22 18:23:35 -04004552 */
Tejun Heoc5edf9c2015-09-29 13:04:26 -04004553void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages,
4554 unsigned long *pheadroom, unsigned long *pdirty,
4555 unsigned long *pwriteback)
Tejun Heoc2aa7232015-05-22 18:23:35 -04004556{
4557 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
4558 struct mem_cgroup *parent;
Tejun Heoc2aa7232015-05-22 18:23:35 -04004559
Greg Thelen0b3d6e62019-04-05 18:39:18 -07004560 *pdirty = memcg_exact_page_state(memcg, NR_FILE_DIRTY);
Tejun Heoc2aa7232015-05-22 18:23:35 -04004561
Greg Thelen0b3d6e62019-04-05 18:39:18 -07004562 *pwriteback = memcg_exact_page_state(memcg, NR_WRITEBACK);
Johannes Weiner21d89d12019-05-13 17:18:08 -07004563 *pfilepages = memcg_exact_page_state(memcg, NR_INACTIVE_FILE) +
4564 memcg_exact_page_state(memcg, NR_ACTIVE_FILE);
Tejun Heoc5edf9c2015-09-29 13:04:26 -04004565 *pheadroom = PAGE_COUNTER_MAX;
Tejun Heoc2aa7232015-05-22 18:23:35 -04004566
Tejun Heoc2aa7232015-05-22 18:23:35 -04004567 while ((parent = parent_mem_cgroup(memcg))) {
Chris Down15b42562020-04-01 21:07:20 -07004568 unsigned long ceiling = min(READ_ONCE(memcg->memory.max),
Jakub Kicinskid1663a92020-06-01 21:49:49 -07004569 READ_ONCE(memcg->memory.high));
Tejun Heoc2aa7232015-05-22 18:23:35 -04004570 unsigned long used = page_counter_read(&memcg->memory);
4571
Tejun Heoc5edf9c2015-09-29 13:04:26 -04004572 *pheadroom = min(*pheadroom, ceiling - min(ceiling, used));
Tejun Heoc2aa7232015-05-22 18:23:35 -04004573 memcg = parent;
4574 }
Tejun Heoc2aa7232015-05-22 18:23:35 -04004575}
4576
Tejun Heo97b27822019-08-26 09:06:56 -07004577/*
4578 * Foreign dirty flushing
4579 *
4580 * There's an inherent mismatch between memcg and writeback. The former
4581 * trackes ownership per-page while the latter per-inode. This was a
4582 * deliberate design decision because honoring per-page ownership in the
4583 * writeback path is complicated, may lead to higher CPU and IO overheads
4584 * and deemed unnecessary given that write-sharing an inode across
4585 * different cgroups isn't a common use-case.
4586 *
4587 * Combined with inode majority-writer ownership switching, this works well
4588 * enough in most cases but there are some pathological cases. For
4589 * example, let's say there are two cgroups A and B which keep writing to
4590 * different but confined parts of the same inode. B owns the inode and
4591 * A's memory is limited far below B's. A's dirty ratio can rise enough to
4592 * trigger balance_dirty_pages() sleeps but B's can be low enough to avoid
4593 * triggering background writeback. A will be slowed down without a way to
4594 * make writeback of the dirty pages happen.
4595 *
4596 * Conditions like the above can lead to a cgroup getting repatedly and
4597 * severely throttled after making some progress after each
4598 * dirty_expire_interval while the underyling IO device is almost
4599 * completely idle.
4600 *
4601 * Solving this problem completely requires matching the ownership tracking
4602 * granularities between memcg and writeback in either direction. However,
4603 * the more egregious behaviors can be avoided by simply remembering the
4604 * most recent foreign dirtying events and initiating remote flushes on
4605 * them when local writeback isn't enough to keep the memory clean enough.
4606 *
4607 * The following two functions implement such mechanism. When a foreign
4608 * page - a page whose memcg and writeback ownerships don't match - is
4609 * dirtied, mem_cgroup_track_foreign_dirty() records the inode owning
4610 * bdi_writeback on the page owning memcg. When balance_dirty_pages()
4611 * decides that the memcg needs to sleep due to high dirty ratio, it calls
4612 * mem_cgroup_flush_foreign() which queues writeback on the recorded
4613 * foreign bdi_writebacks which haven't expired. Both the numbers of
4614 * recorded bdi_writebacks and concurrent in-flight foreign writebacks are
4615 * limited to MEMCG_CGWB_FRN_CNT.
4616 *
4617 * The mechanism only remembers IDs and doesn't hold any object references.
4618 * As being wrong occasionally doesn't matter, updates and accesses to the
4619 * records are lockless and racy.
4620 */
4621void mem_cgroup_track_foreign_dirty_slowpath(struct page *page,
4622 struct bdi_writeback *wb)
4623{
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08004624 struct mem_cgroup *memcg = page_memcg(page);
Tejun Heo97b27822019-08-26 09:06:56 -07004625 struct memcg_cgwb_frn *frn;
4626 u64 now = get_jiffies_64();
4627 u64 oldest_at = now;
4628 int oldest = -1;
4629 int i;
4630
Tejun Heo3a8e9ac2019-08-29 15:47:19 -07004631 trace_track_foreign_dirty(page, wb);
4632
Tejun Heo97b27822019-08-26 09:06:56 -07004633 /*
4634 * Pick the slot to use. If there is already a slot for @wb, keep
4635 * using it. If not replace the oldest one which isn't being
4636 * written out.
4637 */
4638 for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++) {
4639 frn = &memcg->cgwb_frn[i];
4640 if (frn->bdi_id == wb->bdi->id &&
4641 frn->memcg_id == wb->memcg_css->id)
4642 break;
4643 if (time_before64(frn->at, oldest_at) &&
4644 atomic_read(&frn->done.cnt) == 1) {
4645 oldest = i;
4646 oldest_at = frn->at;
4647 }
4648 }
4649
4650 if (i < MEMCG_CGWB_FRN_CNT) {
4651 /*
4652 * Re-using an existing one. Update timestamp lazily to
4653 * avoid making the cacheline hot. We want them to be
4654 * reasonably up-to-date and significantly shorter than
4655 * dirty_expire_interval as that's what expires the record.
4656 * Use the shorter of 1s and dirty_expire_interval / 8.
4657 */
4658 unsigned long update_intv =
4659 min_t(unsigned long, HZ,
4660 msecs_to_jiffies(dirty_expire_interval * 10) / 8);
4661
4662 if (time_before64(frn->at, now - update_intv))
4663 frn->at = now;
4664 } else if (oldest >= 0) {
4665 /* replace the oldest free one */
4666 frn = &memcg->cgwb_frn[oldest];
4667 frn->bdi_id = wb->bdi->id;
4668 frn->memcg_id = wb->memcg_css->id;
4669 frn->at = now;
4670 }
4671}
4672
4673/* issue foreign writeback flushes for recorded foreign dirtying events */
4674void mem_cgroup_flush_foreign(struct bdi_writeback *wb)
4675{
4676 struct mem_cgroup *memcg = mem_cgroup_from_css(wb->memcg_css);
4677 unsigned long intv = msecs_to_jiffies(dirty_expire_interval * 10);
4678 u64 now = jiffies_64;
4679 int i;
4680
4681 for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++) {
4682 struct memcg_cgwb_frn *frn = &memcg->cgwb_frn[i];
4683
4684 /*
4685 * If the record is older than dirty_expire_interval,
4686 * writeback on it has already started. No need to kick it
4687 * off again. Also, don't start a new one if there's
4688 * already one in flight.
4689 */
4690 if (time_after64(frn->at, now - intv) &&
4691 atomic_read(&frn->done.cnt) == 1) {
4692 frn->at = 0;
Tejun Heo3a8e9ac2019-08-29 15:47:19 -07004693 trace_flush_foreign(wb, frn->bdi_id, frn->memcg_id);
Tejun Heo97b27822019-08-26 09:06:56 -07004694 cgroup_writeback_by_id(frn->bdi_id, frn->memcg_id, 0,
4695 WB_REASON_FOREIGN_FLUSH,
4696 &frn->done);
4697 }
4698 }
4699}
4700
Tejun Heo841710a2015-05-22 18:23:33 -04004701#else /* CONFIG_CGROUP_WRITEBACK */
4702
4703static int memcg_wb_domain_init(struct mem_cgroup *memcg, gfp_t gfp)
4704{
4705 return 0;
4706}
4707
4708static void memcg_wb_domain_exit(struct mem_cgroup *memcg)
4709{
4710}
4711
Tejun Heo2529bb32015-05-22 18:23:34 -04004712static void memcg_wb_domain_size_changed(struct mem_cgroup *memcg)
4713{
4714}
4715
Tejun Heo52ebea72015-05-22 17:13:37 -04004716#endif /* CONFIG_CGROUP_WRITEBACK */
4717
Tejun Heo79bd9812013-11-22 18:20:42 -05004718/*
Tejun Heo3bc942f2013-11-22 18:20:44 -05004719 * DO NOT USE IN NEW FILES.
4720 *
4721 * "cgroup.event_control" implementation.
4722 *
4723 * This is way over-engineered. It tries to support fully configurable
4724 * events for each user. Such level of flexibility is completely
4725 * unnecessary especially in the light of the planned unified hierarchy.
4726 *
4727 * Please deprecate this and replace with something simpler if at all
4728 * possible.
4729 */
4730
4731/*
Tejun Heo79bd9812013-11-22 18:20:42 -05004732 * Unregister event and free resources.
4733 *
4734 * Gets called from workqueue.
4735 */
Tejun Heo3bc942f2013-11-22 18:20:44 -05004736static void memcg_event_remove(struct work_struct *work)
Tejun Heo79bd9812013-11-22 18:20:42 -05004737{
Tejun Heo3bc942f2013-11-22 18:20:44 -05004738 struct mem_cgroup_event *event =
4739 container_of(work, struct mem_cgroup_event, remove);
Tejun Heo59b6f872013-11-22 18:20:43 -05004740 struct mem_cgroup *memcg = event->memcg;
Tejun Heo79bd9812013-11-22 18:20:42 -05004741
4742 remove_wait_queue(event->wqh, &event->wait);
4743
Tejun Heo59b6f872013-11-22 18:20:43 -05004744 event->unregister_event(memcg, event->eventfd);
Tejun Heo79bd9812013-11-22 18:20:42 -05004745
4746 /* Notify userspace the event is going away. */
4747 eventfd_signal(event->eventfd, 1);
4748
4749 eventfd_ctx_put(event->eventfd);
4750 kfree(event);
Tejun Heo59b6f872013-11-22 18:20:43 -05004751 css_put(&memcg->css);
Tejun Heo79bd9812013-11-22 18:20:42 -05004752}
4753
4754/*
Linus Torvaldsa9a08842018-02-11 14:34:03 -08004755 * Gets called on EPOLLHUP on eventfd when user closes it.
Tejun Heo79bd9812013-11-22 18:20:42 -05004756 *
4757 * Called with wqh->lock held and interrupts disabled.
4758 */
Ingo Molnarac6424b2017-06-20 12:06:13 +02004759static int memcg_event_wake(wait_queue_entry_t *wait, unsigned mode,
Tejun Heo3bc942f2013-11-22 18:20:44 -05004760 int sync, void *key)
Tejun Heo79bd9812013-11-22 18:20:42 -05004761{
Tejun Heo3bc942f2013-11-22 18:20:44 -05004762 struct mem_cgroup_event *event =
4763 container_of(wait, struct mem_cgroup_event, wait);
Tejun Heo59b6f872013-11-22 18:20:43 -05004764 struct mem_cgroup *memcg = event->memcg;
Al Viro3ad6f932017-07-03 20:14:56 -04004765 __poll_t flags = key_to_poll(key);
Tejun Heo79bd9812013-11-22 18:20:42 -05004766
Linus Torvaldsa9a08842018-02-11 14:34:03 -08004767 if (flags & EPOLLHUP) {
Tejun Heo79bd9812013-11-22 18:20:42 -05004768 /*
4769 * If the event has been detached at cgroup removal, we
4770 * can simply return knowing the other side will cleanup
4771 * for us.
4772 *
4773 * We can't race against event freeing since the other
4774 * side will require wqh->lock via remove_wait_queue(),
4775 * which we hold.
4776 */
Tejun Heofba94802013-11-22 18:20:43 -05004777 spin_lock(&memcg->event_list_lock);
Tejun Heo79bd9812013-11-22 18:20:42 -05004778 if (!list_empty(&event->list)) {
4779 list_del_init(&event->list);
4780 /*
4781 * We are in atomic context, but cgroup_event_remove()
4782 * may sleep, so we have to call it in workqueue.
4783 */
4784 schedule_work(&event->remove);
4785 }
Tejun Heofba94802013-11-22 18:20:43 -05004786 spin_unlock(&memcg->event_list_lock);
Tejun Heo79bd9812013-11-22 18:20:42 -05004787 }
4788
4789 return 0;
4790}
4791
Tejun Heo3bc942f2013-11-22 18:20:44 -05004792static void memcg_event_ptable_queue_proc(struct file *file,
Tejun Heo79bd9812013-11-22 18:20:42 -05004793 wait_queue_head_t *wqh, poll_table *pt)
4794{
Tejun Heo3bc942f2013-11-22 18:20:44 -05004795 struct mem_cgroup_event *event =
4796 container_of(pt, struct mem_cgroup_event, pt);
Tejun Heo79bd9812013-11-22 18:20:42 -05004797
4798 event->wqh = wqh;
4799 add_wait_queue(wqh, &event->wait);
4800}
4801
4802/*
Tejun Heo3bc942f2013-11-22 18:20:44 -05004803 * DO NOT USE IN NEW FILES.
4804 *
Tejun Heo79bd9812013-11-22 18:20:42 -05004805 * Parse input and register new cgroup event handler.
4806 *
4807 * Input must be in format '<event_fd> <control_fd> <args>'.
4808 * Interpretation of args is defined by control file implementation.
4809 */
Tejun Heo451af502014-05-13 12:16:21 -04004810static ssize_t memcg_write_event_control(struct kernfs_open_file *of,
4811 char *buf, size_t nbytes, loff_t off)
Tejun Heo79bd9812013-11-22 18:20:42 -05004812{
Tejun Heo451af502014-05-13 12:16:21 -04004813 struct cgroup_subsys_state *css = of_css(of);
Tejun Heofba94802013-11-22 18:20:43 -05004814 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
Tejun Heo3bc942f2013-11-22 18:20:44 -05004815 struct mem_cgroup_event *event;
Tejun Heo79bd9812013-11-22 18:20:42 -05004816 struct cgroup_subsys_state *cfile_css;
4817 unsigned int efd, cfd;
4818 struct fd efile;
4819 struct fd cfile;
Tejun Heofba94802013-11-22 18:20:43 -05004820 const char *name;
Tejun Heo79bd9812013-11-22 18:20:42 -05004821 char *endp;
4822 int ret;
4823
Tejun Heo451af502014-05-13 12:16:21 -04004824 buf = strstrip(buf);
4825
4826 efd = simple_strtoul(buf, &endp, 10);
Tejun Heo79bd9812013-11-22 18:20:42 -05004827 if (*endp != ' ')
4828 return -EINVAL;
Tejun Heo451af502014-05-13 12:16:21 -04004829 buf = endp + 1;
Tejun Heo79bd9812013-11-22 18:20:42 -05004830
Tejun Heo451af502014-05-13 12:16:21 -04004831 cfd = simple_strtoul(buf, &endp, 10);
Tejun Heo79bd9812013-11-22 18:20:42 -05004832 if ((*endp != ' ') && (*endp != '\0'))
4833 return -EINVAL;
Tejun Heo451af502014-05-13 12:16:21 -04004834 buf = endp + 1;
Tejun Heo79bd9812013-11-22 18:20:42 -05004835
4836 event = kzalloc(sizeof(*event), GFP_KERNEL);
4837 if (!event)
4838 return -ENOMEM;
4839
Tejun Heo59b6f872013-11-22 18:20:43 -05004840 event->memcg = memcg;
Tejun Heo79bd9812013-11-22 18:20:42 -05004841 INIT_LIST_HEAD(&event->list);
Tejun Heo3bc942f2013-11-22 18:20:44 -05004842 init_poll_funcptr(&event->pt, memcg_event_ptable_queue_proc);
4843 init_waitqueue_func_entry(&event->wait, memcg_event_wake);
4844 INIT_WORK(&event->remove, memcg_event_remove);
Tejun Heo79bd9812013-11-22 18:20:42 -05004845
4846 efile = fdget(efd);
4847 if (!efile.file) {
4848 ret = -EBADF;
4849 goto out_kfree;
4850 }
4851
4852 event->eventfd = eventfd_ctx_fileget(efile.file);
4853 if (IS_ERR(event->eventfd)) {
4854 ret = PTR_ERR(event->eventfd);
4855 goto out_put_efile;
4856 }
4857
4858 cfile = fdget(cfd);
4859 if (!cfile.file) {
4860 ret = -EBADF;
4861 goto out_put_eventfd;
4862 }
4863
4864 /* the process need read permission on control file */
4865 /* AV: shouldn't we check that it's been opened for read instead? */
Christian Brauner02f92b32021-01-21 14:19:22 +01004866 ret = file_permission(cfile.file, MAY_READ);
Tejun Heo79bd9812013-11-22 18:20:42 -05004867 if (ret < 0)
4868 goto out_put_cfile;
4869
Tejun Heo79bd9812013-11-22 18:20:42 -05004870 /*
Tejun Heofba94802013-11-22 18:20:43 -05004871 * Determine the event callbacks and set them in @event. This used
4872 * to be done via struct cftype but cgroup core no longer knows
4873 * about these events. The following is crude but the whole thing
4874 * is for compatibility anyway.
Tejun Heo3bc942f2013-11-22 18:20:44 -05004875 *
4876 * DO NOT ADD NEW FILES.
Tejun Heofba94802013-11-22 18:20:43 -05004877 */
Al Virob5830432014-10-31 01:22:04 -04004878 name = cfile.file->f_path.dentry->d_name.name;
Tejun Heofba94802013-11-22 18:20:43 -05004879
4880 if (!strcmp(name, "memory.usage_in_bytes")) {
4881 event->register_event = mem_cgroup_usage_register_event;
4882 event->unregister_event = mem_cgroup_usage_unregister_event;
4883 } else if (!strcmp(name, "memory.oom_control")) {
4884 event->register_event = mem_cgroup_oom_register_event;
4885 event->unregister_event = mem_cgroup_oom_unregister_event;
4886 } else if (!strcmp(name, "memory.pressure_level")) {
4887 event->register_event = vmpressure_register_event;
4888 event->unregister_event = vmpressure_unregister_event;
4889 } else if (!strcmp(name, "memory.memsw.usage_in_bytes")) {
Tejun Heo347c4a82013-11-22 18:20:43 -05004890 event->register_event = memsw_cgroup_usage_register_event;
4891 event->unregister_event = memsw_cgroup_usage_unregister_event;
Tejun Heofba94802013-11-22 18:20:43 -05004892 } else {
4893 ret = -EINVAL;
4894 goto out_put_cfile;
4895 }
4896
4897 /*
Tejun Heob5557c42013-11-22 18:20:42 -05004898 * Verify @cfile should belong to @css. Also, remaining events are
4899 * automatically removed on cgroup destruction but the removal is
4900 * asynchronous, so take an extra ref on @css.
Tejun Heo79bd9812013-11-22 18:20:42 -05004901 */
Al Virob5830432014-10-31 01:22:04 -04004902 cfile_css = css_tryget_online_from_dir(cfile.file->f_path.dentry->d_parent,
Tejun Heoec903c02014-05-13 12:11:01 -04004903 &memory_cgrp_subsys);
Tejun Heo79bd9812013-11-22 18:20:42 -05004904 ret = -EINVAL;
Tejun Heo5a17f542014-02-11 11:52:47 -05004905 if (IS_ERR(cfile_css))
Tejun Heo79bd9812013-11-22 18:20:42 -05004906 goto out_put_cfile;
Tejun Heo5a17f542014-02-11 11:52:47 -05004907 if (cfile_css != css) {
4908 css_put(cfile_css);
4909 goto out_put_cfile;
4910 }
Tejun Heo79bd9812013-11-22 18:20:42 -05004911
Tejun Heo451af502014-05-13 12:16:21 -04004912 ret = event->register_event(memcg, event->eventfd, buf);
Tejun Heo79bd9812013-11-22 18:20:42 -05004913 if (ret)
4914 goto out_put_css;
4915
Christoph Hellwig9965ed172018-03-05 07:26:05 -08004916 vfs_poll(efile.file, &event->pt);
Tejun Heo79bd9812013-11-22 18:20:42 -05004917
Tejun Heofba94802013-11-22 18:20:43 -05004918 spin_lock(&memcg->event_list_lock);
4919 list_add(&event->list, &memcg->event_list);
4920 spin_unlock(&memcg->event_list_lock);
Tejun Heo79bd9812013-11-22 18:20:42 -05004921
4922 fdput(cfile);
4923 fdput(efile);
4924
Tejun Heo451af502014-05-13 12:16:21 -04004925 return nbytes;
Tejun Heo79bd9812013-11-22 18:20:42 -05004926
4927out_put_css:
Tejun Heob5557c42013-11-22 18:20:42 -05004928 css_put(css);
Tejun Heo79bd9812013-11-22 18:20:42 -05004929out_put_cfile:
4930 fdput(cfile);
4931out_put_eventfd:
4932 eventfd_ctx_put(event->eventfd);
4933out_put_efile:
4934 fdput(efile);
4935out_kfree:
4936 kfree(event);
4937
4938 return ret;
4939}
4940
Johannes Weiner241994ed2015-02-11 15:26:06 -08004941static struct cftype mem_cgroup_legacy_files[] = {
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004942 {
Balbir Singh0eea1032008-02-07 00:13:57 -08004943 .name = "usage_in_bytes",
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004944 .private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
Tejun Heo791badb2013-12-05 12:28:02 -05004945 .read_u64 = mem_cgroup_read_u64,
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004946 },
4947 {
Pavel Emelyanovc84872e2008-04-29 01:00:17 -07004948 .name = "max_usage_in_bytes",
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004949 .private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
Tejun Heo6770c642014-05-13 12:16:21 -04004950 .write = mem_cgroup_reset,
Tejun Heo791badb2013-12-05 12:28:02 -05004951 .read_u64 = mem_cgroup_read_u64,
Pavel Emelyanovc84872e2008-04-29 01:00:17 -07004952 },
4953 {
Balbir Singh0eea1032008-02-07 00:13:57 -08004954 .name = "limit_in_bytes",
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004955 .private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
Tejun Heo451af502014-05-13 12:16:21 -04004956 .write = mem_cgroup_write,
Tejun Heo791badb2013-12-05 12:28:02 -05004957 .read_u64 = mem_cgroup_read_u64,
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004958 },
4959 {
Balbir Singh296c81d2009-09-23 15:56:36 -07004960 .name = "soft_limit_in_bytes",
4961 .private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
Tejun Heo451af502014-05-13 12:16:21 -04004962 .write = mem_cgroup_write,
Tejun Heo791badb2013-12-05 12:28:02 -05004963 .read_u64 = mem_cgroup_read_u64,
Balbir Singh296c81d2009-09-23 15:56:36 -07004964 },
4965 {
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004966 .name = "failcnt",
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004967 .private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
Tejun Heo6770c642014-05-13 12:16:21 -04004968 .write = mem_cgroup_reset,
Tejun Heo791badb2013-12-05 12:28:02 -05004969 .read_u64 = mem_cgroup_read_u64,
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004970 },
Balbir Singh8697d332008-02-07 00:13:59 -08004971 {
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08004972 .name = "stat",
Tejun Heo2da8ca82013-12-05 12:28:04 -05004973 .seq_show = memcg_stat_show,
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08004974 },
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08004975 {
4976 .name = "force_empty",
Tejun Heo6770c642014-05-13 12:16:21 -04004977 .write = mem_cgroup_force_empty_write,
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08004978 },
Balbir Singh18f59ea2009-01-07 18:08:07 -08004979 {
4980 .name = "use_hierarchy",
4981 .write_u64 = mem_cgroup_hierarchy_write,
4982 .read_u64 = mem_cgroup_hierarchy_read,
4983 },
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08004984 {
Tejun Heo3bc942f2013-11-22 18:20:44 -05004985 .name = "cgroup.event_control", /* XXX: for compat */
Tejun Heo451af502014-05-13 12:16:21 -04004986 .write = memcg_write_event_control,
Tejun Heo7dbdb192015-09-18 17:54:23 -04004987 .flags = CFTYPE_NO_PREFIX | CFTYPE_WORLD_WRITABLE,
Tejun Heo79bd9812013-11-22 18:20:42 -05004988 },
4989 {
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08004990 .name = "swappiness",
4991 .read_u64 = mem_cgroup_swappiness_read,
4992 .write_u64 = mem_cgroup_swappiness_write,
4993 },
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08004994 {
4995 .name = "move_charge_at_immigrate",
4996 .read_u64 = mem_cgroup_move_charge_read,
4997 .write_u64 = mem_cgroup_move_charge_write,
4998 },
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004999 {
5000 .name = "oom_control",
Tejun Heo2da8ca82013-12-05 12:28:04 -05005001 .seq_show = mem_cgroup_oom_control_read,
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07005002 .write_u64 = mem_cgroup_oom_control_write,
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07005003 .private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL),
5004 },
Anton Vorontsov70ddf632013-04-29 15:08:31 -07005005 {
5006 .name = "pressure_level",
Anton Vorontsov70ddf632013-04-29 15:08:31 -07005007 },
Ying Han406eb0c2011-05-26 16:25:37 -07005008#ifdef CONFIG_NUMA
5009 {
5010 .name = "numa_stat",
Tejun Heo2da8ca82013-12-05 12:28:04 -05005011 .seq_show = memcg_numa_stat_show,
Ying Han406eb0c2011-05-26 16:25:37 -07005012 },
5013#endif
Glauber Costa510fc4e2012-12-18 14:21:47 -08005014 {
5015 .name = "kmem.limit_in_bytes",
5016 .private = MEMFILE_PRIVATE(_KMEM, RES_LIMIT),
Tejun Heo451af502014-05-13 12:16:21 -04005017 .write = mem_cgroup_write,
Tejun Heo791badb2013-12-05 12:28:02 -05005018 .read_u64 = mem_cgroup_read_u64,
Glauber Costa510fc4e2012-12-18 14:21:47 -08005019 },
5020 {
5021 .name = "kmem.usage_in_bytes",
5022 .private = MEMFILE_PRIVATE(_KMEM, RES_USAGE),
Tejun Heo791badb2013-12-05 12:28:02 -05005023 .read_u64 = mem_cgroup_read_u64,
Glauber Costa510fc4e2012-12-18 14:21:47 -08005024 },
5025 {
5026 .name = "kmem.failcnt",
5027 .private = MEMFILE_PRIVATE(_KMEM, RES_FAILCNT),
Tejun Heo6770c642014-05-13 12:16:21 -04005028 .write = mem_cgroup_reset,
Tejun Heo791badb2013-12-05 12:28:02 -05005029 .read_u64 = mem_cgroup_read_u64,
Glauber Costa510fc4e2012-12-18 14:21:47 -08005030 },
5031 {
5032 .name = "kmem.max_usage_in_bytes",
5033 .private = MEMFILE_PRIVATE(_KMEM, RES_MAX_USAGE),
Tejun Heo6770c642014-05-13 12:16:21 -04005034 .write = mem_cgroup_reset,
Tejun Heo791badb2013-12-05 12:28:02 -05005035 .read_u64 = mem_cgroup_read_u64,
Glauber Costa510fc4e2012-12-18 14:21:47 -08005036 },
Yafang Shaoa87425a2020-04-01 21:06:30 -07005037#if defined(CONFIG_MEMCG_KMEM) && \
5038 (defined(CONFIG_SLAB) || defined(CONFIG_SLUB_DEBUG))
Glauber Costa749c5412012-12-18 14:23:01 -08005039 {
5040 .name = "kmem.slabinfo",
Vladimir Davydovb0475012014-12-10 15:44:19 -08005041 .seq_show = memcg_slab_show,
Glauber Costa749c5412012-12-18 14:23:01 -08005042 },
5043#endif
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08005044 {
5045 .name = "kmem.tcp.limit_in_bytes",
5046 .private = MEMFILE_PRIVATE(_TCP, RES_LIMIT),
5047 .write = mem_cgroup_write,
5048 .read_u64 = mem_cgroup_read_u64,
5049 },
5050 {
5051 .name = "kmem.tcp.usage_in_bytes",
5052 .private = MEMFILE_PRIVATE(_TCP, RES_USAGE),
5053 .read_u64 = mem_cgroup_read_u64,
5054 },
5055 {
5056 .name = "kmem.tcp.failcnt",
5057 .private = MEMFILE_PRIVATE(_TCP, RES_FAILCNT),
5058 .write = mem_cgroup_reset,
5059 .read_u64 = mem_cgroup_read_u64,
5060 },
5061 {
5062 .name = "kmem.tcp.max_usage_in_bytes",
5063 .private = MEMFILE_PRIVATE(_TCP, RES_MAX_USAGE),
5064 .write = mem_cgroup_reset,
5065 .read_u64 = mem_cgroup_read_u64,
5066 },
Tejun Heo6bc10342012-04-01 12:09:55 -07005067 { }, /* terminate */
Tejun Heoaf36f902012-04-01 12:09:55 -07005068};
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08005069
Johannes Weiner73f576c2016-07-20 15:44:57 -07005070/*
5071 * Private memory cgroup IDR
5072 *
5073 * Swap-out records and page cache shadow entries need to store memcg
5074 * references in constrained space, so we maintain an ID space that is
5075 * limited to 16 bit (MEM_CGROUP_ID_MAX), limiting the total number of
5076 * memory-controlled cgroups to 64k.
5077 *
Ethon Paulb8f29352020-06-04 16:49:28 -07005078 * However, there usually are many references to the offline CSS after
Johannes Weiner73f576c2016-07-20 15:44:57 -07005079 * the cgroup has been destroyed, such as page cache or reclaimable
5080 * slab objects, that don't need to hang on to the ID. We want to keep
5081 * those dead CSS from occupying IDs, or we might quickly exhaust the
5082 * relatively small ID space and prevent the creation of new cgroups
5083 * even when there are much fewer than 64k cgroups - possibly none.
5084 *
5085 * Maintain a private 16-bit ID space for memcg, and allow the ID to
5086 * be freed and recycled when it's no longer needed, which is usually
5087 * when the CSS is offlined.
5088 *
5089 * The only exception to that are records of swapped out tmpfs/shmem
5090 * pages that need to be attributed to live ancestors on swapin. But
5091 * those references are manageable from userspace.
5092 */
5093
5094static DEFINE_IDR(mem_cgroup_idr);
5095
Kirill Tkhai7e97de02018-08-02 15:36:01 -07005096static void mem_cgroup_id_remove(struct mem_cgroup *memcg)
5097{
5098 if (memcg->id.id > 0) {
5099 idr_remove(&mem_cgroup_idr, memcg->id.id);
5100 memcg->id.id = 0;
5101 }
5102}
5103
Vincenzo Frascinoc1514c02020-04-01 21:07:13 -07005104static void __maybe_unused mem_cgroup_id_get_many(struct mem_cgroup *memcg,
5105 unsigned int n)
Johannes Weiner73f576c2016-07-20 15:44:57 -07005106{
Kirill Tkhai1c2d4792018-10-26 15:09:28 -07005107 refcount_add(n, &memcg->id.ref);
Johannes Weiner73f576c2016-07-20 15:44:57 -07005108}
5109
Vladimir Davydov615d66c2016-08-11 15:33:03 -07005110static void mem_cgroup_id_put_many(struct mem_cgroup *memcg, unsigned int n)
Johannes Weiner73f576c2016-07-20 15:44:57 -07005111{
Kirill Tkhai1c2d4792018-10-26 15:09:28 -07005112 if (refcount_sub_and_test(n, &memcg->id.ref)) {
Kirill Tkhai7e97de02018-08-02 15:36:01 -07005113 mem_cgroup_id_remove(memcg);
Johannes Weiner73f576c2016-07-20 15:44:57 -07005114
5115 /* Memcg ID pins CSS */
5116 css_put(&memcg->css);
5117 }
5118}
5119
Vladimir Davydov615d66c2016-08-11 15:33:03 -07005120static inline void mem_cgroup_id_put(struct mem_cgroup *memcg)
5121{
5122 mem_cgroup_id_put_many(memcg, 1);
5123}
5124
Johannes Weiner73f576c2016-07-20 15:44:57 -07005125/**
5126 * mem_cgroup_from_id - look up a memcg from a memcg id
5127 * @id: the memcg id to look up
5128 *
5129 * Caller must hold rcu_read_lock().
5130 */
5131struct mem_cgroup *mem_cgroup_from_id(unsigned short id)
5132{
5133 WARN_ON_ONCE(!rcu_read_lock_held());
5134 return idr_find(&mem_cgroup_idr, id);
5135}
5136
Mel Gormanef8f2322016-07-28 15:46:05 -07005137static int alloc_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08005138{
5139 struct mem_cgroup_per_node *pn;
Mel Gormanef8f2322016-07-28 15:46:05 -07005140 int tmp = node;
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08005141 /*
5142 * This routine is called against possible nodes.
5143 * But it's BUG to call kmalloc() against offline node.
5144 *
5145 * TODO: this routine can waste much memory for nodes which will
5146 * never be onlined. It's better to use memory hotplug callback
5147 * function.
5148 */
KAMEZAWA Hiroyuki41e33552008-04-08 17:41:54 -07005149 if (!node_state(node, N_NORMAL_MEMORY))
5150 tmp = -1;
Jesper Juhl17295c82011-01-13 15:47:42 -08005151 pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08005152 if (!pn)
5153 return 1;
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08005154
Roman Gushchin3e38e0a2020-08-11 18:30:25 -07005155 pn->lruvec_stat_local = alloc_percpu_gfp(struct lruvec_stat,
5156 GFP_KERNEL_ACCOUNT);
Johannes Weiner815744d2019-06-13 15:55:46 -07005157 if (!pn->lruvec_stat_local) {
5158 kfree(pn);
5159 return 1;
5160 }
5161
Muchun Songf3344ad2021-02-24 12:03:15 -08005162 pn->lruvec_stat_cpu = alloc_percpu_gfp(struct batched_lruvec_stat,
Roman Gushchin3e38e0a2020-08-11 18:30:25 -07005163 GFP_KERNEL_ACCOUNT);
Johannes Weinera983b5e2018-01-31 16:16:45 -08005164 if (!pn->lruvec_stat_cpu) {
Johannes Weiner815744d2019-06-13 15:55:46 -07005165 free_percpu(pn->lruvec_stat_local);
Johannes Weiner00f3ca22017-07-06 15:40:52 -07005166 kfree(pn);
5167 return 1;
5168 }
5169
Mel Gormanef8f2322016-07-28 15:46:05 -07005170 lruvec_init(&pn->lruvec);
5171 pn->usage_in_excess = 0;
5172 pn->on_tree = false;
5173 pn->memcg = memcg;
5174
Johannes Weiner54f72fe2013-07-08 15:59:49 -07005175 memcg->nodeinfo[node] = pn;
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08005176 return 0;
5177}
5178
Mel Gormanef8f2322016-07-28 15:46:05 -07005179static void free_mem_cgroup_per_node_info(struct mem_cgroup *memcg, int node)
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08005180{
Johannes Weiner00f3ca22017-07-06 15:40:52 -07005181 struct mem_cgroup_per_node *pn = memcg->nodeinfo[node];
5182
Michal Hocko4eaf4312018-04-10 16:29:52 -07005183 if (!pn)
5184 return;
5185
Johannes Weinera983b5e2018-01-31 16:16:45 -08005186 free_percpu(pn->lruvec_stat_cpu);
Johannes Weiner815744d2019-06-13 15:55:46 -07005187 free_percpu(pn->lruvec_stat_local);
Johannes Weiner00f3ca22017-07-06 15:40:52 -07005188 kfree(pn);
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08005189}
5190
Tahsin Erdogan40e952f2017-03-09 16:17:26 -08005191static void __mem_cgroup_free(struct mem_cgroup *memcg)
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005192{
5193 int node;
5194
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005195 for_each_node(node)
Mel Gormanef8f2322016-07-28 15:46:05 -07005196 free_mem_cgroup_per_node_info(memcg, node);
Chris Down871789d2019-05-14 15:46:57 -07005197 free_percpu(memcg->vmstats_percpu);
Johannes Weiner815744d2019-06-13 15:55:46 -07005198 free_percpu(memcg->vmstats_local);
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005199 kfree(memcg);
5200}
5201
Tahsin Erdogan40e952f2017-03-09 16:17:26 -08005202static void mem_cgroup_free(struct mem_cgroup *memcg)
5203{
5204 memcg_wb_domain_exit(memcg);
Shakeel Butt7961eee2019-11-05 21:16:21 -08005205 /*
5206 * Flush percpu vmstats and vmevents to guarantee the value correctness
5207 * on parent's and all ancestor levels.
5208 */
Roman Gushchin4a87e2a2020-01-13 16:29:16 -08005209 memcg_flush_percpu_vmstats(memcg);
Shakeel Butt7961eee2019-11-05 21:16:21 -08005210 memcg_flush_percpu_vmevents(memcg);
Tahsin Erdogan40e952f2017-03-09 16:17:26 -08005211 __mem_cgroup_free(memcg);
5212}
5213
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07005214static struct mem_cgroup *mem_cgroup_alloc(void)
5215{
Hugh Dickinsd79154b2012-03-21 16:34:18 -07005216 struct mem_cgroup *memcg;
Alexey Dobriyanb9726c22019-03-05 15:48:26 -08005217 unsigned int size;
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005218 int node;
Tejun Heo97b27822019-08-26 09:06:56 -07005219 int __maybe_unused i;
Yafang Shao11d67612020-05-07 18:35:43 -07005220 long error = -ENOMEM;
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07005221
Vladimir Davydov8ff69e22014-01-23 15:52:52 -08005222 size = sizeof(struct mem_cgroup);
5223 size += nr_node_ids * sizeof(struct mem_cgroup_per_node *);
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07005224
Vladimir Davydov8ff69e22014-01-23 15:52:52 -08005225 memcg = kzalloc(size, GFP_KERNEL);
Hugh Dickinsd79154b2012-03-21 16:34:18 -07005226 if (!memcg)
Yafang Shao11d67612020-05-07 18:35:43 -07005227 return ERR_PTR(error);
Dan Carpentere7bbcdf2010-03-23 13:35:12 -07005228
Johannes Weiner73f576c2016-07-20 15:44:57 -07005229 memcg->id.id = idr_alloc(&mem_cgroup_idr, NULL,
5230 1, MEM_CGROUP_ID_MAX,
5231 GFP_KERNEL);
Yafang Shao11d67612020-05-07 18:35:43 -07005232 if (memcg->id.id < 0) {
5233 error = memcg->id.id;
Johannes Weiner73f576c2016-07-20 15:44:57 -07005234 goto fail;
Yafang Shao11d67612020-05-07 18:35:43 -07005235 }
Johannes Weiner73f576c2016-07-20 15:44:57 -07005236
Roman Gushchin3e38e0a2020-08-11 18:30:25 -07005237 memcg->vmstats_local = alloc_percpu_gfp(struct memcg_vmstats_percpu,
5238 GFP_KERNEL_ACCOUNT);
Johannes Weiner815744d2019-06-13 15:55:46 -07005239 if (!memcg->vmstats_local)
5240 goto fail;
5241
Roman Gushchin3e38e0a2020-08-11 18:30:25 -07005242 memcg->vmstats_percpu = alloc_percpu_gfp(struct memcg_vmstats_percpu,
5243 GFP_KERNEL_ACCOUNT);
Chris Down871789d2019-05-14 15:46:57 -07005244 if (!memcg->vmstats_percpu)
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005245 goto fail;
Pavel Emelianov78fb7462008-02-07 00:13:51 -08005246
Bob Liu3ed28fa2012-01-12 17:19:04 -08005247 for_each_node(node)
Mel Gormanef8f2322016-07-28 15:46:05 -07005248 if (alloc_mem_cgroup_per_node_info(memcg, node))
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005249 goto fail;
Balbir Singhf64c3f52009-09-23 15:56:37 -07005250
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005251 if (memcg_wb_domain_init(memcg, GFP_KERNEL))
5252 goto fail;
Balbir Singh28dbc4b2009-01-07 18:08:05 -08005253
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08005254 INIT_WORK(&memcg->high_work, high_work_func);
Glauber Costad142e3e2013-02-22 16:34:52 -08005255 INIT_LIST_HEAD(&memcg->oom_notify);
Glauber Costad142e3e2013-02-22 16:34:52 -08005256 mutex_init(&memcg->thresholds_lock);
5257 spin_lock_init(&memcg->move_lock);
Anton Vorontsov70ddf632013-04-29 15:08:31 -07005258 vmpressure_init(&memcg->vmpressure);
Tejun Heofba94802013-11-22 18:20:43 -05005259 INIT_LIST_HEAD(&memcg->event_list);
5260 spin_lock_init(&memcg->event_list_lock);
Johannes Weinerd886f4e2016-01-20 15:02:47 -08005261 memcg->socket_pressure = jiffies;
Kirill Tkhai84c07d12018-08-17 15:47:25 -07005262#ifdef CONFIG_MEMCG_KMEM
Vladimir Davydov900a38f2014-12-12 16:55:10 -08005263 memcg->kmemcg_id = -1;
Roman Gushchinbf4f0592020-08-06 23:20:49 -07005264 INIT_LIST_HEAD(&memcg->objcg_list);
Vladimir Davydov900a38f2014-12-12 16:55:10 -08005265#endif
Tejun Heo52ebea72015-05-22 17:13:37 -04005266#ifdef CONFIG_CGROUP_WRITEBACK
5267 INIT_LIST_HEAD(&memcg->cgwb_list);
Tejun Heo97b27822019-08-26 09:06:56 -07005268 for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++)
5269 memcg->cgwb_frn[i].done =
5270 __WB_COMPLETION_INIT(&memcg_cgwb_frn_waitq);
Tejun Heo52ebea72015-05-22 17:13:37 -04005271#endif
Yang Shi87eaceb2019-09-23 15:38:15 -07005272#ifdef CONFIG_TRANSPARENT_HUGEPAGE
5273 spin_lock_init(&memcg->deferred_split_queue.split_queue_lock);
5274 INIT_LIST_HEAD(&memcg->deferred_split_queue.split_queue);
5275 memcg->deferred_split_queue.split_queue_len = 0;
5276#endif
Johannes Weiner73f576c2016-07-20 15:44:57 -07005277 idr_replace(&mem_cgroup_idr, memcg, memcg->id.id);
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005278 return memcg;
5279fail:
Kirill Tkhai7e97de02018-08-02 15:36:01 -07005280 mem_cgroup_id_remove(memcg);
Tahsin Erdogan40e952f2017-03-09 16:17:26 -08005281 __mem_cgroup_free(memcg);
Yafang Shao11d67612020-05-07 18:35:43 -07005282 return ERR_PTR(error);
Glauber Costad142e3e2013-02-22 16:34:52 -08005283}
5284
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005285static struct cgroup_subsys_state * __ref
5286mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
Glauber Costad142e3e2013-02-22 16:34:52 -08005287{
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005288 struct mem_cgroup *parent = mem_cgroup_from_css(parent_css);
Roman Gushchinb87d8ce2020-10-17 16:13:40 -07005289 struct mem_cgroup *memcg, *old_memcg;
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005290 long error = -ENOMEM;
Glauber Costad142e3e2013-02-22 16:34:52 -08005291
Roman Gushchinb87d8ce2020-10-17 16:13:40 -07005292 old_memcg = set_active_memcg(parent);
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005293 memcg = mem_cgroup_alloc();
Roman Gushchinb87d8ce2020-10-17 16:13:40 -07005294 set_active_memcg(old_memcg);
Yafang Shao11d67612020-05-07 18:35:43 -07005295 if (IS_ERR(memcg))
5296 return ERR_CAST(memcg);
Li Zefan4219b2d2013-09-23 16:56:29 +08005297
Jakub Kicinskid1663a92020-06-01 21:49:49 -07005298 page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX);
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005299 memcg->soft_limit = PAGE_COUNTER_MAX;
Jakub Kicinski4b82ab42020-06-01 21:49:52 -07005300 page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX);
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005301 if (parent) {
5302 memcg->swappiness = mem_cgroup_swappiness(parent);
5303 memcg->oom_kill_disable = parent->oom_kill_disable;
Roman Gushchinbef86202020-12-14 19:06:49 -08005304
Johannes Weiner3e32cb22014-12-10 15:42:31 -08005305 page_counter_init(&memcg->memory, &parent->memory);
Vladimir Davydov37e84352016-01-20 15:02:56 -08005306 page_counter_init(&memcg->swap, &parent->swap);
Johannes Weiner3e32cb22014-12-10 15:42:31 -08005307 page_counter_init(&memcg->kmem, &parent->kmem);
Johannes Weiner0db15292016-01-20 15:02:50 -08005308 page_counter_init(&memcg->tcpmem, &parent->tcpmem);
Balbir Singh18f59ea2009-01-07 18:08:07 -08005309 } else {
Roman Gushchinbef86202020-12-14 19:06:49 -08005310 page_counter_init(&memcg->memory, NULL);
5311 page_counter_init(&memcg->swap, NULL);
5312 page_counter_init(&memcg->kmem, NULL);
5313 page_counter_init(&memcg->tcpmem, NULL);
Vladimir Davydovd6441632014-01-23 15:53:09 -08005314
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005315 root_mem_cgroup = memcg;
5316 return &memcg->css;
5317 }
5318
Roman Gushchinbef86202020-12-14 19:06:49 -08005319 /* The following stuff does not apply to the root */
Vladimir Davydovb313aee2016-03-17 14:18:27 -07005320 error = memcg_online_kmem(memcg);
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005321 if (error)
5322 goto fail;
Johannes Weiner127424c2016-01-20 15:02:32 -08005323
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08005324 if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket)
Johannes Weineref129472016-01-14 15:21:34 -08005325 static_branch_inc(&memcg_sockets_enabled_key);
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08005326
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005327 return &memcg->css;
5328fail:
Kirill Tkhai7e97de02018-08-02 15:36:01 -07005329 mem_cgroup_id_remove(memcg);
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005330 mem_cgroup_free(memcg);
Yafang Shao11d67612020-05-07 18:35:43 -07005331 return ERR_PTR(error);
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005332}
5333
Johannes Weiner73f576c2016-07-20 15:44:57 -07005334static int mem_cgroup_css_online(struct cgroup_subsys_state *css)
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005335{
Vladimir Davydov58fa2a52016-10-07 16:57:29 -07005336 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5337
Kirill Tkhai0a4465d2018-08-17 15:47:37 -07005338 /*
5339 * A memcg must be visible for memcg_expand_shrinker_maps()
5340 * by the time the maps are allocated. So, we allocate maps
5341 * here, when for_each_mem_cgroup() can't skip it.
5342 */
5343 if (memcg_alloc_shrinker_maps(memcg)) {
5344 mem_cgroup_id_remove(memcg);
5345 return -ENOMEM;
5346 }
5347
Johannes Weiner73f576c2016-07-20 15:44:57 -07005348 /* Online state pins memcg ID, memcg ID pins CSS */
Kirill Tkhai1c2d4792018-10-26 15:09:28 -07005349 refcount_set(&memcg->id.ref, 1);
Johannes Weiner73f576c2016-07-20 15:44:57 -07005350 css_get(css);
Johannes Weiner2f7dd7a2014-10-02 16:16:57 -07005351 return 0;
Balbir Singh8cdea7c2008-02-07 00:13:50 -08005352}
5353
Tejun Heoeb954192013-08-08 20:11:23 -04005354static void mem_cgroup_css_offline(struct cgroup_subsys_state *css)
KAMEZAWA Hiroyukidf878fb2008-02-07 00:14:28 -08005355{
Tejun Heoeb954192013-08-08 20:11:23 -04005356 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
Tejun Heo3bc942f2013-11-22 18:20:44 -05005357 struct mem_cgroup_event *event, *tmp;
Tejun Heo79bd9812013-11-22 18:20:42 -05005358
5359 /*
5360 * Unregister events and notify userspace.
5361 * Notify userspace about cgroup removing only after rmdir of cgroup
5362 * directory to avoid race between userspace and kernelspace.
5363 */
Tejun Heofba94802013-11-22 18:20:43 -05005364 spin_lock(&memcg->event_list_lock);
5365 list_for_each_entry_safe(event, tmp, &memcg->event_list, list) {
Tejun Heo79bd9812013-11-22 18:20:42 -05005366 list_del_init(&event->list);
5367 schedule_work(&event->remove);
5368 }
Tejun Heofba94802013-11-22 18:20:43 -05005369 spin_unlock(&memcg->event_list_lock);
KAMEZAWA Hiroyukiec64f512009-04-02 16:57:26 -07005370
Roman Gushchinbf8d5d52018-06-07 17:07:46 -07005371 page_counter_set_min(&memcg->memory, 0);
Roman Gushchin23067152018-06-07 17:06:22 -07005372 page_counter_set_low(&memcg->memory, 0);
Roman Gushchin63677c742017-09-06 16:21:47 -07005373
Johannes Weiner567e9ab2016-01-20 15:02:24 -08005374 memcg_offline_kmem(memcg);
Tejun Heo52ebea72015-05-22 17:13:37 -04005375 wb_memcg_offline(memcg);
Johannes Weiner73f576c2016-07-20 15:44:57 -07005376
Roman Gushchin591edfb2018-10-26 15:03:23 -07005377 drain_all_stock(memcg);
5378
Johannes Weiner73f576c2016-07-20 15:44:57 -07005379 mem_cgroup_id_put(memcg);
KAMEZAWA Hiroyukidf878fb2008-02-07 00:14:28 -08005380}
5381
Vladimir Davydov6df38682015-12-29 14:54:10 -08005382static void mem_cgroup_css_released(struct cgroup_subsys_state *css)
5383{
5384 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5385
5386 invalidate_reclaim_iterators(memcg);
5387}
5388
Tejun Heoeb954192013-08-08 20:11:23 -04005389static void mem_cgroup_css_free(struct cgroup_subsys_state *css)
Balbir Singh8cdea7c2008-02-07 00:13:50 -08005390{
Tejun Heoeb954192013-08-08 20:11:23 -04005391 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
Tejun Heo97b27822019-08-26 09:06:56 -07005392 int __maybe_unused i;
Daisuke Nishimurac268e992009-01-15 13:51:13 -08005393
Tejun Heo97b27822019-08-26 09:06:56 -07005394#ifdef CONFIG_CGROUP_WRITEBACK
5395 for (i = 0; i < MEMCG_CGWB_FRN_CNT; i++)
5396 wb_wait_for_completion(&memcg->cgwb_frn[i].done);
5397#endif
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08005398 if (cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_nosocket)
Johannes Weineref129472016-01-14 15:21:34 -08005399 static_branch_dec(&memcg_sockets_enabled_key);
Johannes Weiner3893e302016-01-20 15:02:29 -08005400
Johannes Weiner0db15292016-01-20 15:02:50 -08005401 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg->tcpmem_active)
Vladimir Davydovd55f90b2016-01-20 15:02:44 -08005402 static_branch_dec(&memcg_sockets_enabled_key);
Johannes Weiner3893e302016-01-20 15:02:29 -08005403
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005404 vmpressure_cleanup(&memcg->vmpressure);
5405 cancel_work_sync(&memcg->high_work);
5406 mem_cgroup_remove_from_trees(memcg);
Kirill Tkhai0a4465d2018-08-17 15:47:37 -07005407 memcg_free_shrinker_maps(memcg);
Johannes Weinerd886f4e2016-01-20 15:02:47 -08005408 memcg_free_kmem(memcg);
Johannes Weiner0b8f73e2016-01-20 15:02:53 -08005409 mem_cgroup_free(memcg);
Balbir Singh8cdea7c2008-02-07 00:13:50 -08005410}
5411
Tejun Heo1ced9532014-07-08 18:02:57 -04005412/**
5413 * mem_cgroup_css_reset - reset the states of a mem_cgroup
5414 * @css: the target css
5415 *
5416 * Reset the states of the mem_cgroup associated with @css. This is
5417 * invoked when the userland requests disabling on the default hierarchy
5418 * but the memcg is pinned through dependency. The memcg should stop
5419 * applying policies and should revert to the vanilla state as it may be
5420 * made visible again.
5421 *
5422 * The current implementation only resets the essential configurations.
5423 * This needs to be expanded to cover all the visible parts.
5424 */
5425static void mem_cgroup_css_reset(struct cgroup_subsys_state *css)
5426{
5427 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
5428
Roman Gushchinbbec2e12018-06-07 17:06:18 -07005429 page_counter_set_max(&memcg->memory, PAGE_COUNTER_MAX);
5430 page_counter_set_max(&memcg->swap, PAGE_COUNTER_MAX);
Roman Gushchinbbec2e12018-06-07 17:06:18 -07005431 page_counter_set_max(&memcg->kmem, PAGE_COUNTER_MAX);
5432 page_counter_set_max(&memcg->tcpmem, PAGE_COUNTER_MAX);
Roman Gushchinbf8d5d52018-06-07 17:07:46 -07005433 page_counter_set_min(&memcg->memory, 0);
Roman Gushchin23067152018-06-07 17:06:22 -07005434 page_counter_set_low(&memcg->memory, 0);
Jakub Kicinskid1663a92020-06-01 21:49:49 -07005435 page_counter_set_high(&memcg->memory, PAGE_COUNTER_MAX);
Johannes Weiner24d404d2015-01-08 14:32:35 -08005436 memcg->soft_limit = PAGE_COUNTER_MAX;
Jakub Kicinski4b82ab42020-06-01 21:49:52 -07005437 page_counter_set_high(&memcg->swap, PAGE_COUNTER_MAX);
Tejun Heo2529bb32015-05-22 18:23:34 -04005438 memcg_wb_domain_size_changed(memcg);
Tejun Heo1ced9532014-07-08 18:02:57 -04005439}
5440
Daisuke Nishimura02491442010-03-10 15:22:17 -08005441#ifdef CONFIG_MMU
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005442/* Handlers for move charge at task migration. */
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08005443static int mem_cgroup_do_precharge(unsigned long count)
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005444{
Johannes Weiner05b84302014-08-06 16:05:59 -07005445 int ret;
Johannes Weiner9476db92014-08-06 16:05:55 -07005446
Mel Gormand0164ad2015-11-06 16:28:21 -08005447 /* Try a single bulk charge without reclaim first, kswapd may wake */
5448 ret = try_charge(mc.to, GFP_KERNEL & ~__GFP_DIRECT_RECLAIM, count);
Johannes Weiner9476db92014-08-06 16:05:55 -07005449 if (!ret) {
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08005450 mc.precharge += count;
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08005451 return ret;
5452 }
Johannes Weiner9476db92014-08-06 16:05:55 -07005453
David Rientjes36745342017-01-24 15:18:10 -08005454 /* Try charges one by one with reclaim, but do not retry */
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08005455 while (count--) {
David Rientjes36745342017-01-24 15:18:10 -08005456 ret = try_charge(mc.to, GFP_KERNEL | __GFP_NORETRY, 1);
KAMEZAWA Hiroyuki38c5d722012-01-12 17:19:01 -08005457 if (ret)
KAMEZAWA Hiroyuki38c5d722012-01-12 17:19:01 -08005458 return ret;
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08005459 mc.precharge++;
Johannes Weiner9476db92014-08-06 16:05:55 -07005460 cond_resched();
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08005461 }
Johannes Weiner9476db92014-08-06 16:05:55 -07005462 return 0;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005463}
5464
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005465union mc_target {
5466 struct page *page;
Daisuke Nishimura02491442010-03-10 15:22:17 -08005467 swp_entry_t ent;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005468};
5469
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005470enum mc_target_type {
Naoya Horiguchi8d32ff82012-03-21 16:34:27 -07005471 MC_TARGET_NONE = 0,
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005472 MC_TARGET_PAGE,
Daisuke Nishimura02491442010-03-10 15:22:17 -08005473 MC_TARGET_SWAP,
Jérôme Glissec733a822017-09-08 16:11:54 -07005474 MC_TARGET_DEVICE,
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005475};
5476
Daisuke Nishimura90254a62010-05-26 14:42:38 -07005477static struct page *mc_handle_present_pte(struct vm_area_struct *vma,
5478 unsigned long addr, pte_t ptent)
5479{
Christoph Hellwig25b29952019-06-13 22:50:49 +02005480 struct page *page = vm_normal_page(vma, addr, ptent);
Daisuke Nishimura90254a62010-05-26 14:42:38 -07005481
5482 if (!page || !page_mapped(page))
5483 return NULL;
5484 if (PageAnon(page)) {
Johannes Weiner1dfab5a2015-02-11 15:26:09 -08005485 if (!(mc.flags & MOVE_ANON))
Daisuke Nishimura90254a62010-05-26 14:42:38 -07005486 return NULL;
Johannes Weiner1dfab5a2015-02-11 15:26:09 -08005487 } else {
5488 if (!(mc.flags & MOVE_FILE))
5489 return NULL;
5490 }
Daisuke Nishimura90254a62010-05-26 14:42:38 -07005491 if (!get_page_unless_zero(page))
5492 return NULL;
5493
5494 return page;
5495}
5496
Jérôme Glissec733a822017-09-08 16:11:54 -07005497#if defined(CONFIG_SWAP) || defined(CONFIG_DEVICE_PRIVATE)
Daisuke Nishimura90254a62010-05-26 14:42:38 -07005498static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
Li RongQing48406ef2016-07-26 15:22:14 -07005499 pte_t ptent, swp_entry_t *entry)
Daisuke Nishimura90254a62010-05-26 14:42:38 -07005500{
Daisuke Nishimura90254a62010-05-26 14:42:38 -07005501 struct page *page = NULL;
5502 swp_entry_t ent = pte_to_swp_entry(ptent);
5503
Ralph Campbell9a137152020-10-13 16:53:13 -07005504 if (!(mc.flags & MOVE_ANON))
Daisuke Nishimura90254a62010-05-26 14:42:38 -07005505 return NULL;
Jérôme Glissec733a822017-09-08 16:11:54 -07005506
5507 /*
5508 * Handle MEMORY_DEVICE_PRIVATE which are ZONE_DEVICE page belonging to
5509 * a device and because they are not accessible by CPU they are store
5510 * as special swap entry in the CPU page table.
5511 */
5512 if (is_device_private_entry(ent)) {
5513 page = device_private_entry_to_page(ent);
5514 /*
5515 * MEMORY_DEVICE_PRIVATE means ZONE_DEVICE page and which have
5516 * a refcount of 1 when free (unlike normal page)
5517 */
5518 if (!page_ref_add_unless(page, 1, 1))
5519 return NULL;
5520 return page;
5521 }
5522
Ralph Campbell9a137152020-10-13 16:53:13 -07005523 if (non_swap_entry(ent))
5524 return NULL;
5525
KAMEZAWA Hiroyuki4b913552012-05-29 15:06:51 -07005526 /*
5527 * Because lookup_swap_cache() updates some statistics counter,
5528 * we call find_get_page() with swapper_space directly.
5529 */
Huang Yingf6ab1f72016-10-07 17:00:21 -07005530 page = find_get_page(swap_address_space(ent), swp_offset(ent));
Johannes Weiner2d1c4982020-06-03 16:02:14 -07005531 entry->val = ent.val;
Daisuke Nishimura90254a62010-05-26 14:42:38 -07005532
5533 return page;
5534}
KAMEZAWA Hiroyuki4b913552012-05-29 15:06:51 -07005535#else
5536static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
Li RongQing48406ef2016-07-26 15:22:14 -07005537 pte_t ptent, swp_entry_t *entry)
KAMEZAWA Hiroyuki4b913552012-05-29 15:06:51 -07005538{
5539 return NULL;
5540}
5541#endif
Daisuke Nishimura90254a62010-05-26 14:42:38 -07005542
Daisuke Nishimura87946a72010-05-26 14:42:39 -07005543static struct page *mc_handle_file_pte(struct vm_area_struct *vma,
5544 unsigned long addr, pte_t ptent, swp_entry_t *entry)
5545{
Daisuke Nishimura87946a72010-05-26 14:42:39 -07005546 if (!vma->vm_file) /* anonymous vma */
5547 return NULL;
Johannes Weiner1dfab5a2015-02-11 15:26:09 -08005548 if (!(mc.flags & MOVE_FILE))
Daisuke Nishimura87946a72010-05-26 14:42:39 -07005549 return NULL;
5550
Daisuke Nishimura87946a72010-05-26 14:42:39 -07005551 /* page is moved even if it's not RSS of this task(page-faulted). */
Hugh Dickinsaa3b1892011-08-03 16:21:24 -07005552 /* shmem/tmpfs may report page out on swap: account for that too. */
Matthew Wilcox (Oracle)f5df8632020-10-13 16:51:21 -07005553 return find_get_incore_page(vma->vm_file->f_mapping,
5554 linear_page_index(vma, addr));
Daisuke Nishimura87946a72010-05-26 14:42:39 -07005555}
5556
Chen Gangb1b0dea2015-04-14 15:47:35 -07005557/**
5558 * mem_cgroup_move_account - move account of the page
5559 * @page: the page
Li RongQing25843c22016-07-26 15:26:56 -07005560 * @compound: charge the page as compound or small page
Chen Gangb1b0dea2015-04-14 15:47:35 -07005561 * @from: mem_cgroup which the page is moved from.
5562 * @to: mem_cgroup which the page is moved to. @from != @to.
5563 *
Kirill A. Shutemov3ac808f2016-01-15 16:53:07 -08005564 * The caller must make sure the page is not on LRU (isolate_page() is useful.)
Chen Gangb1b0dea2015-04-14 15:47:35 -07005565 *
5566 * This function doesn't do "charge" to new cgroup and doesn't do "uncharge"
5567 * from old cgroup.
5568 */
5569static int mem_cgroup_move_account(struct page *page,
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -08005570 bool compound,
Chen Gangb1b0dea2015-04-14 15:47:35 -07005571 struct mem_cgroup *from,
5572 struct mem_cgroup *to)
5573{
Konstantin Khlebnikovae8af432019-10-18 20:20:11 -07005574 struct lruvec *from_vec, *to_vec;
5575 struct pglist_data *pgdat;
Matthew Wilcox (Oracle)6c357842020-08-14 17:30:37 -07005576 unsigned int nr_pages = compound ? thp_nr_pages(page) : 1;
Chen Gangb1b0dea2015-04-14 15:47:35 -07005577 int ret;
5578
5579 VM_BUG_ON(from == to);
5580 VM_BUG_ON_PAGE(PageLRU(page), page);
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -08005581 VM_BUG_ON(compound && !PageTransHuge(page));
Chen Gangb1b0dea2015-04-14 15:47:35 -07005582
5583 /*
Johannes Weiner6a93ca82016-03-15 14:57:19 -07005584 * Prevent mem_cgroup_migrate() from looking at
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08005585 * page's memory cgroup of its source page while we change it.
Chen Gangb1b0dea2015-04-14 15:47:35 -07005586 */
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -08005587 ret = -EBUSY;
Chen Gangb1b0dea2015-04-14 15:47:35 -07005588 if (!trylock_page(page))
5589 goto out;
5590
5591 ret = -EINVAL;
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08005592 if (page_memcg(page) != from)
Chen Gangb1b0dea2015-04-14 15:47:35 -07005593 goto out_unlock;
5594
Konstantin Khlebnikovae8af432019-10-18 20:20:11 -07005595 pgdat = page_pgdat(page);
Johannes Weiner867e5e12019-11-30 17:55:34 -08005596 from_vec = mem_cgroup_lruvec(from, pgdat);
5597 to_vec = mem_cgroup_lruvec(to, pgdat);
Konstantin Khlebnikovae8af432019-10-18 20:20:11 -07005598
Johannes Weinerabb242f2020-06-03 16:01:28 -07005599 lock_page_memcg(page);
Chen Gangb1b0dea2015-04-14 15:47:35 -07005600
Johannes Weinerbe5d0a72020-06-03 16:01:57 -07005601 if (PageAnon(page)) {
5602 if (page_mapped(page)) {
5603 __mod_lruvec_state(from_vec, NR_ANON_MAPPED, -nr_pages);
5604 __mod_lruvec_state(to_vec, NR_ANON_MAPPED, nr_pages);
Johannes Weiner468c3982020-06-03 16:02:01 -07005605 if (PageTransHuge(page)) {
Muchun Song69473e52021-02-24 12:03:23 -08005606 __mod_lruvec_state(from_vec, NR_ANON_THPS,
5607 -nr_pages);
5608 __mod_lruvec_state(to_vec, NR_ANON_THPS,
5609 nr_pages);
Johannes Weiner468c3982020-06-03 16:02:01 -07005610 }
Johannes Weinerbe5d0a72020-06-03 16:01:57 -07005611 }
5612 } else {
Johannes Weiner0d1c2072020-06-03 16:01:54 -07005613 __mod_lruvec_state(from_vec, NR_FILE_PAGES, -nr_pages);
5614 __mod_lruvec_state(to_vec, NR_FILE_PAGES, nr_pages);
5615
5616 if (PageSwapBacked(page)) {
5617 __mod_lruvec_state(from_vec, NR_SHMEM, -nr_pages);
5618 __mod_lruvec_state(to_vec, NR_SHMEM, nr_pages);
5619 }
5620
Johannes Weiner49e50d22020-06-03 16:01:47 -07005621 if (page_mapped(page)) {
5622 __mod_lruvec_state(from_vec, NR_FILE_MAPPED, -nr_pages);
5623 __mod_lruvec_state(to_vec, NR_FILE_MAPPED, nr_pages);
5624 }
Chen Gangb1b0dea2015-04-14 15:47:35 -07005625
Johannes Weiner49e50d22020-06-03 16:01:47 -07005626 if (PageDirty(page)) {
5627 struct address_space *mapping = page_mapping(page);
Greg Thelenc4843a72015-05-22 17:13:16 -04005628
Christoph Hellwigf56753a2020-09-24 08:51:40 +02005629 if (mapping_can_writeback(mapping)) {
Johannes Weiner49e50d22020-06-03 16:01:47 -07005630 __mod_lruvec_state(from_vec, NR_FILE_DIRTY,
5631 -nr_pages);
5632 __mod_lruvec_state(to_vec, NR_FILE_DIRTY,
5633 nr_pages);
5634 }
Greg Thelenc4843a72015-05-22 17:13:16 -04005635 }
5636 }
5637
Chen Gangb1b0dea2015-04-14 15:47:35 -07005638 if (PageWriteback(page)) {
Konstantin Khlebnikovae8af432019-10-18 20:20:11 -07005639 __mod_lruvec_state(from_vec, NR_WRITEBACK, -nr_pages);
5640 __mod_lruvec_state(to_vec, NR_WRITEBACK, nr_pages);
Chen Gangb1b0dea2015-04-14 15:47:35 -07005641 }
5642
5643 /*
Johannes Weinerabb242f2020-06-03 16:01:28 -07005644 * All state has been migrated, let's switch to the new memcg.
5645 *
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08005646 * It is safe to change page's memcg here because the page
Johannes Weinerabb242f2020-06-03 16:01:28 -07005647 * is referenced, charged, isolated, and locked: we can't race
5648 * with (un)charging, migration, LRU putback, or anything else
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08005649 * that would rely on a stable page's memory cgroup.
Johannes Weinerabb242f2020-06-03 16:01:28 -07005650 *
5651 * Note that lock_page_memcg is a memcg lock, not a page lock,
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08005652 * to save space. As soon as we switch page's memory cgroup to a
Johannes Weinerabb242f2020-06-03 16:01:28 -07005653 * new memcg that isn't locked, the above state can change
5654 * concurrently again. Make sure we're truly done with it.
Chen Gangb1b0dea2015-04-14 15:47:35 -07005655 */
Johannes Weinerabb242f2020-06-03 16:01:28 -07005656 smp_mb();
Chen Gangb1b0dea2015-04-14 15:47:35 -07005657
Johannes Weiner1a3e1f42020-08-06 23:20:45 -07005658 css_get(&to->css);
5659 css_put(&from->css);
5660
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08005661 page->memcg_data = (unsigned long)to;
Yang Shi87eaceb2019-09-23 15:38:15 -07005662
Johannes Weinerabb242f2020-06-03 16:01:28 -07005663 __unlock_page_memcg(from);
Chen Gangb1b0dea2015-04-14 15:47:35 -07005664
5665 ret = 0;
5666
5667 local_irq_disable();
Johannes Weiner3fba69a2020-06-03 16:01:31 -07005668 mem_cgroup_charge_statistics(to, page, nr_pages);
Chen Gangb1b0dea2015-04-14 15:47:35 -07005669 memcg_check_events(to, page);
Johannes Weiner3fba69a2020-06-03 16:01:31 -07005670 mem_cgroup_charge_statistics(from, page, -nr_pages);
Chen Gangb1b0dea2015-04-14 15:47:35 -07005671 memcg_check_events(from, page);
5672 local_irq_enable();
5673out_unlock:
5674 unlock_page(page);
5675out:
5676 return ret;
5677}
5678
Li RongQing7cf78062016-05-27 14:27:46 -07005679/**
5680 * get_mctgt_type - get target type of moving charge
5681 * @vma: the vma the pte to be checked belongs
5682 * @addr: the address corresponding to the pte to be checked
5683 * @ptent: the pte to be checked
5684 * @target: the pointer the target page or swap ent will be stored(can be NULL)
5685 *
5686 * Returns
5687 * 0(MC_TARGET_NONE): if the pte is not a target for move charge.
5688 * 1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for
5689 * move charge. if @target is not NULL, the page is stored in target->page
5690 * with extra refcnt got(Callers should handle it).
5691 * 2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a
5692 * target for charge migration. if @target is not NULL, the entry is stored
5693 * in target->ent.
Christoph Hellwig25b29952019-06-13 22:50:49 +02005694 * 3(MC_TARGET_DEVICE): like MC_TARGET_PAGE but page is MEMORY_DEVICE_PRIVATE
5695 * (so ZONE_DEVICE page and thus not on the lru).
Jérôme Glissedf6ad692017-09-08 16:12:24 -07005696 * For now we such page is charge like a regular page would be as for all
5697 * intent and purposes it is just special memory taking the place of a
5698 * regular page.
Jérôme Glissec733a822017-09-08 16:11:54 -07005699 *
5700 * See Documentations/vm/hmm.txt and include/linux/hmm.h
Li RongQing7cf78062016-05-27 14:27:46 -07005701 *
5702 * Called with pte lock held.
5703 */
5704
Naoya Horiguchi8d32ff82012-03-21 16:34:27 -07005705static enum mc_target_type get_mctgt_type(struct vm_area_struct *vma,
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005706 unsigned long addr, pte_t ptent, union mc_target *target)
5707{
Daisuke Nishimura02491442010-03-10 15:22:17 -08005708 struct page *page = NULL;
Naoya Horiguchi8d32ff82012-03-21 16:34:27 -07005709 enum mc_target_type ret = MC_TARGET_NONE;
Daisuke Nishimura02491442010-03-10 15:22:17 -08005710 swp_entry_t ent = { .val = 0 };
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005711
Daisuke Nishimura90254a62010-05-26 14:42:38 -07005712 if (pte_present(ptent))
5713 page = mc_handle_present_pte(vma, addr, ptent);
5714 else if (is_swap_pte(ptent))
Li RongQing48406ef2016-07-26 15:22:14 -07005715 page = mc_handle_swap_pte(vma, ptent, &ent);
Kirill A. Shutemov0661a332015-02-10 14:10:04 -08005716 else if (pte_none(ptent))
Daisuke Nishimura87946a72010-05-26 14:42:39 -07005717 page = mc_handle_file_pte(vma, addr, ptent, &ent);
Daisuke Nishimura90254a62010-05-26 14:42:38 -07005718
5719 if (!page && !ent.val)
Naoya Horiguchi8d32ff82012-03-21 16:34:27 -07005720 return ret;
Daisuke Nishimura02491442010-03-10 15:22:17 -08005721 if (page) {
Daisuke Nishimura02491442010-03-10 15:22:17 -08005722 /*
Johannes Weiner0a31bc92014-08-08 14:19:22 -07005723 * Do only loose check w/o serialization.
Johannes Weiner1306a852014-12-10 15:44:52 -08005724 * mem_cgroup_move_account() checks the page is valid or
Johannes Weiner0a31bc92014-08-08 14:19:22 -07005725 * not under LRU exclusion.
Daisuke Nishimura02491442010-03-10 15:22:17 -08005726 */
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08005727 if (page_memcg(page) == mc.from) {
Daisuke Nishimura02491442010-03-10 15:22:17 -08005728 ret = MC_TARGET_PAGE;
Christoph Hellwig25b29952019-06-13 22:50:49 +02005729 if (is_device_private_page(page))
Jérôme Glissec733a822017-09-08 16:11:54 -07005730 ret = MC_TARGET_DEVICE;
Daisuke Nishimura02491442010-03-10 15:22:17 -08005731 if (target)
5732 target->page = page;
5733 }
5734 if (!ret || !target)
5735 put_page(page);
5736 }
Huang Ying3e14a572017-09-06 16:22:37 -07005737 /*
5738 * There is a swap entry and a page doesn't exist or isn't charged.
5739 * But we cannot move a tail-page in a THP.
5740 */
5741 if (ent.val && !ret && (!page || !PageTransCompound(page)) &&
Li Zefan34c00c32013-09-23 16:56:01 +08005742 mem_cgroup_id(mc.from) == lookup_swap_cgroup_id(ent)) {
KAMEZAWA Hiroyuki7f0f1542010-05-11 14:06:58 -07005743 ret = MC_TARGET_SWAP;
5744 if (target)
5745 target->ent = ent;
Daisuke Nishimura02491442010-03-10 15:22:17 -08005746 }
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005747 return ret;
5748}
5749
Naoya Horiguchi12724852012-03-21 16:34:28 -07005750#ifdef CONFIG_TRANSPARENT_HUGEPAGE
5751/*
Huang Yingd6810d72017-09-06 16:22:45 -07005752 * We don't consider PMD mapped swapping or file mapped pages because THP does
5753 * not support them for now.
Naoya Horiguchi12724852012-03-21 16:34:28 -07005754 * Caller should make sure that pmd_trans_huge(pmd) is true.
5755 */
5756static enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
5757 unsigned long addr, pmd_t pmd, union mc_target *target)
5758{
5759 struct page *page = NULL;
Naoya Horiguchi12724852012-03-21 16:34:28 -07005760 enum mc_target_type ret = MC_TARGET_NONE;
5761
Zi Yan84c3fc42017-09-08 16:11:01 -07005762 if (unlikely(is_swap_pmd(pmd))) {
5763 VM_BUG_ON(thp_migration_supported() &&
5764 !is_pmd_migration_entry(pmd));
5765 return ret;
5766 }
Naoya Horiguchi12724852012-03-21 16:34:28 -07005767 page = pmd_page(pmd);
Sasha Levin309381fea2014-01-23 15:52:54 -08005768 VM_BUG_ON_PAGE(!page || !PageHead(page), page);
Johannes Weiner1dfab5a2015-02-11 15:26:09 -08005769 if (!(mc.flags & MOVE_ANON))
Naoya Horiguchi12724852012-03-21 16:34:28 -07005770 return ret;
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08005771 if (page_memcg(page) == mc.from) {
Naoya Horiguchi12724852012-03-21 16:34:28 -07005772 ret = MC_TARGET_PAGE;
5773 if (target) {
5774 get_page(page);
5775 target->page = page;
5776 }
5777 }
5778 return ret;
5779}
5780#else
5781static inline enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
5782 unsigned long addr, pmd_t pmd, union mc_target *target)
5783{
5784 return MC_TARGET_NONE;
5785}
5786#endif
5787
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005788static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
5789 unsigned long addr, unsigned long end,
5790 struct mm_walk *walk)
5791{
Naoya Horiguchi26bcd642015-02-11 15:27:57 -08005792 struct vm_area_struct *vma = walk->vma;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005793 pte_t *pte;
5794 spinlock_t *ptl;
5795
Kirill A. Shutemovb6ec57f2016-01-21 16:40:25 -08005796 ptl = pmd_trans_huge_lock(pmd, vma);
5797 if (ptl) {
Jérôme Glissec733a822017-09-08 16:11:54 -07005798 /*
5799 * Note their can not be MC_TARGET_DEVICE for now as we do not
Christoph Hellwig25b29952019-06-13 22:50:49 +02005800 * support transparent huge page with MEMORY_DEVICE_PRIVATE but
5801 * this might change.
Jérôme Glissec733a822017-09-08 16:11:54 -07005802 */
Naoya Horiguchi12724852012-03-21 16:34:28 -07005803 if (get_mctgt_type_thp(vma, addr, *pmd, NULL) == MC_TARGET_PAGE)
5804 mc.precharge += HPAGE_PMD_NR;
Kirill A. Shutemovbf929152013-11-14 14:30:54 -08005805 spin_unlock(ptl);
Andrea Arcangeli1a5a9902012-03-21 16:33:42 -07005806 return 0;
Naoya Horiguchi12724852012-03-21 16:34:28 -07005807 }
Dave Hansen03319322011-03-22 16:32:56 -07005808
Andrea Arcangeli45f83ce2012-03-28 14:42:40 -07005809 if (pmd_trans_unstable(pmd))
5810 return 0;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005811 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
5812 for (; addr != end; pte++, addr += PAGE_SIZE)
Naoya Horiguchi8d32ff82012-03-21 16:34:27 -07005813 if (get_mctgt_type(vma, addr, *pte, NULL))
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005814 mc.precharge++; /* increment precharge temporarily */
5815 pte_unmap_unlock(pte - 1, ptl);
5816 cond_resched();
5817
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005818 return 0;
5819}
5820
Christoph Hellwig7b86ac32019-08-28 16:19:54 +02005821static const struct mm_walk_ops precharge_walk_ops = {
5822 .pmd_entry = mem_cgroup_count_precharge_pte_range,
5823};
5824
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005825static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
5826{
5827 unsigned long precharge;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005828
Michel Lespinassed8ed45c2020-06-08 21:33:25 -07005829 mmap_read_lock(mm);
Christoph Hellwig7b86ac32019-08-28 16:19:54 +02005830 walk_page_range(mm, 0, mm->highest_vm_end, &precharge_walk_ops, NULL);
Michel Lespinassed8ed45c2020-06-08 21:33:25 -07005831 mmap_read_unlock(mm);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005832
5833 precharge = mc.precharge;
5834 mc.precharge = 0;
5835
5836 return precharge;
5837}
5838
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005839static int mem_cgroup_precharge_mc(struct mm_struct *mm)
5840{
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08005841 unsigned long precharge = mem_cgroup_count_precharge(mm);
5842
5843 VM_BUG_ON(mc.moving_task);
5844 mc.moving_task = current;
5845 return mem_cgroup_do_precharge(precharge);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005846}
5847
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08005848/* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */
5849static void __mem_cgroup_clear_mc(void)
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005850{
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07005851 struct mem_cgroup *from = mc.from;
5852 struct mem_cgroup *to = mc.to;
5853
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005854 /* we must uncharge all the leftover precharges from mc.to */
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08005855 if (mc.precharge) {
Johannes Weiner00501b52014-08-08 14:19:20 -07005856 cancel_charge(mc.to, mc.precharge);
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08005857 mc.precharge = 0;
5858 }
5859 /*
5860 * we didn't uncharge from mc.from at mem_cgroup_move_account(), so
5861 * we must uncharge here.
5862 */
5863 if (mc.moved_charge) {
Johannes Weiner00501b52014-08-08 14:19:20 -07005864 cancel_charge(mc.from, mc.moved_charge);
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08005865 mc.moved_charge = 0;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005866 }
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08005867 /* we must fixup refcnts and charges */
5868 if (mc.moved_swap) {
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08005869 /* uncharge swap account from the old cgroup */
Johannes Weinerce00a962014-09-05 08:43:57 -04005870 if (!mem_cgroup_is_root(mc.from))
Johannes Weiner3e32cb22014-12-10 15:42:31 -08005871 page_counter_uncharge(&mc.from->memsw, mc.moved_swap);
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08005872
Vladimir Davydov615d66c2016-08-11 15:33:03 -07005873 mem_cgroup_id_put_many(mc.from, mc.moved_swap);
5874
Johannes Weiner05b84302014-08-06 16:05:59 -07005875 /*
Johannes Weiner3e32cb22014-12-10 15:42:31 -08005876 * we charged both to->memory and to->memsw, so we
5877 * should uncharge to->memory.
Johannes Weiner05b84302014-08-06 16:05:59 -07005878 */
Johannes Weinerce00a962014-09-05 08:43:57 -04005879 if (!mem_cgroup_is_root(mc.to))
Johannes Weiner3e32cb22014-12-10 15:42:31 -08005880 page_counter_uncharge(&mc.to->memory, mc.moved_swap);
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08005881
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08005882 mc.moved_swap = 0;
5883 }
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08005884 memcg_oom_recover(from);
5885 memcg_oom_recover(to);
5886 wake_up_all(&mc.waitq);
5887}
5888
5889static void mem_cgroup_clear_mc(void)
5890{
Tejun Heo264a0ae2016-04-21 19:09:02 -04005891 struct mm_struct *mm = mc.mm;
5892
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08005893 /*
5894 * we must clear moving_task before waking up waiters at the end of
5895 * task migration.
5896 */
5897 mc.moving_task = NULL;
5898 __mem_cgroup_clear_mc();
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07005899 spin_lock(&mc.lock);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005900 mc.from = NULL;
5901 mc.to = NULL;
Tejun Heo264a0ae2016-04-21 19:09:02 -04005902 mc.mm = NULL;
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07005903 spin_unlock(&mc.lock);
Tejun Heo264a0ae2016-04-21 19:09:02 -04005904
5905 mmput(mm);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005906}
5907
Tejun Heo1f7dd3e52015-12-03 10:18:21 -05005908static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005909{
Tejun Heo1f7dd3e52015-12-03 10:18:21 -05005910 struct cgroup_subsys_state *css;
Ross Zwislereed67d72015-12-23 14:53:27 -07005911 struct mem_cgroup *memcg = NULL; /* unneeded init to make gcc happy */
Tejun Heo9f2115f2015-09-08 15:01:10 -07005912 struct mem_cgroup *from;
Tejun Heo4530edd2015-09-11 15:00:19 -04005913 struct task_struct *leader, *p;
Tejun Heo9f2115f2015-09-08 15:01:10 -07005914 struct mm_struct *mm;
Johannes Weiner1dfab5a2015-02-11 15:26:09 -08005915 unsigned long move_flags;
Tejun Heo9f2115f2015-09-08 15:01:10 -07005916 int ret = 0;
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005917
Tejun Heo1f7dd3e52015-12-03 10:18:21 -05005918 /* charge immigration isn't supported on the default hierarchy */
5919 if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
Tejun Heo9f2115f2015-09-08 15:01:10 -07005920 return 0;
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005921
Tejun Heo4530edd2015-09-11 15:00:19 -04005922 /*
5923 * Multi-process migrations only happen on the default hierarchy
5924 * where charge immigration is not used. Perform charge
5925 * immigration if @tset contains a leader and whine if there are
5926 * multiple.
5927 */
5928 p = NULL;
Tejun Heo1f7dd3e52015-12-03 10:18:21 -05005929 cgroup_taskset_for_each_leader(leader, css, tset) {
Tejun Heo4530edd2015-09-11 15:00:19 -04005930 WARN_ON_ONCE(p);
5931 p = leader;
Tejun Heo1f7dd3e52015-12-03 10:18:21 -05005932 memcg = mem_cgroup_from_css(css);
Tejun Heo4530edd2015-09-11 15:00:19 -04005933 }
5934 if (!p)
5935 return 0;
5936
Tejun Heo1f7dd3e52015-12-03 10:18:21 -05005937 /*
5938 * We are now commited to this value whatever it is. Changes in this
5939 * tunable will only affect upcoming migrations, not the current one.
5940 * So we need to save it, and keep it going.
5941 */
5942 move_flags = READ_ONCE(memcg->move_charge_at_immigrate);
5943 if (!move_flags)
5944 return 0;
5945
Tejun Heo9f2115f2015-09-08 15:01:10 -07005946 from = mem_cgroup_from_task(p);
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005947
Tejun Heo9f2115f2015-09-08 15:01:10 -07005948 VM_BUG_ON(from == memcg);
Johannes Weiner247b1442014-12-10 15:44:11 -08005949
Tejun Heo9f2115f2015-09-08 15:01:10 -07005950 mm = get_task_mm(p);
5951 if (!mm)
5952 return 0;
5953 /* We move charges only when we move a owner of the mm */
5954 if (mm->owner == p) {
5955 VM_BUG_ON(mc.from);
5956 VM_BUG_ON(mc.to);
5957 VM_BUG_ON(mc.precharge);
5958 VM_BUG_ON(mc.moved_charge);
5959 VM_BUG_ON(mc.moved_swap);
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005960
Tejun Heo9f2115f2015-09-08 15:01:10 -07005961 spin_lock(&mc.lock);
Tejun Heo264a0ae2016-04-21 19:09:02 -04005962 mc.mm = mm;
Tejun Heo9f2115f2015-09-08 15:01:10 -07005963 mc.from = from;
5964 mc.to = memcg;
5965 mc.flags = move_flags;
5966 spin_unlock(&mc.lock);
5967 /* We set mc.moving_task later */
5968
5969 ret = mem_cgroup_precharge_mc(mm);
5970 if (ret)
5971 mem_cgroup_clear_mc();
Tejun Heo264a0ae2016-04-21 19:09:02 -04005972 } else {
5973 mmput(mm);
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005974 }
5975 return ret;
5976}
5977
Tejun Heo1f7dd3e52015-12-03 10:18:21 -05005978static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005979{
Johannes Weiner4e2f2452014-12-10 15:44:08 -08005980 if (mc.to)
5981 mem_cgroup_clear_mc();
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005982}
5983
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005984static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
5985 unsigned long addr, unsigned long end,
5986 struct mm_walk *walk)
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005987{
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005988 int ret = 0;
Naoya Horiguchi26bcd642015-02-11 15:27:57 -08005989 struct vm_area_struct *vma = walk->vma;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005990 pte_t *pte;
5991 spinlock_t *ptl;
Naoya Horiguchi12724852012-03-21 16:34:28 -07005992 enum mc_target_type target_type;
5993 union mc_target target;
5994 struct page *page;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005995
Kirill A. Shutemovb6ec57f2016-01-21 16:40:25 -08005996 ptl = pmd_trans_huge_lock(pmd, vma);
5997 if (ptl) {
Hugh Dickins62ade862012-05-18 11:28:34 -07005998 if (mc.precharge < HPAGE_PMD_NR) {
Kirill A. Shutemovbf929152013-11-14 14:30:54 -08005999 spin_unlock(ptl);
Naoya Horiguchi12724852012-03-21 16:34:28 -07006000 return 0;
6001 }
6002 target_type = get_mctgt_type_thp(vma, addr, *pmd, &target);
6003 if (target_type == MC_TARGET_PAGE) {
6004 page = target.page;
6005 if (!isolate_lru_page(page)) {
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -08006006 if (!mem_cgroup_move_account(page, true,
Johannes Weiner1306a852014-12-10 15:44:52 -08006007 mc.from, mc.to)) {
Naoya Horiguchi12724852012-03-21 16:34:28 -07006008 mc.precharge -= HPAGE_PMD_NR;
6009 mc.moved_charge += HPAGE_PMD_NR;
6010 }
6011 putback_lru_page(page);
6012 }
6013 put_page(page);
Jérôme Glissec733a822017-09-08 16:11:54 -07006014 } else if (target_type == MC_TARGET_DEVICE) {
6015 page = target.page;
6016 if (!mem_cgroup_move_account(page, true,
6017 mc.from, mc.to)) {
6018 mc.precharge -= HPAGE_PMD_NR;
6019 mc.moved_charge += HPAGE_PMD_NR;
6020 }
6021 put_page(page);
Naoya Horiguchi12724852012-03-21 16:34:28 -07006022 }
Kirill A. Shutemovbf929152013-11-14 14:30:54 -08006023 spin_unlock(ptl);
Andrea Arcangeli1a5a9902012-03-21 16:33:42 -07006024 return 0;
Naoya Horiguchi12724852012-03-21 16:34:28 -07006025 }
6026
Andrea Arcangeli45f83ce2012-03-28 14:42:40 -07006027 if (pmd_trans_unstable(pmd))
6028 return 0;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006029retry:
6030 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
6031 for (; addr != end; addr += PAGE_SIZE) {
6032 pte_t ptent = *(pte++);
Jérôme Glissec733a822017-09-08 16:11:54 -07006033 bool device = false;
Daisuke Nishimura02491442010-03-10 15:22:17 -08006034 swp_entry_t ent;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006035
6036 if (!mc.precharge)
6037 break;
6038
Naoya Horiguchi8d32ff82012-03-21 16:34:27 -07006039 switch (get_mctgt_type(vma, addr, ptent, &target)) {
Jérôme Glissec733a822017-09-08 16:11:54 -07006040 case MC_TARGET_DEVICE:
6041 device = true;
Joe Perchese4a9bc52020-04-06 20:08:39 -07006042 fallthrough;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006043 case MC_TARGET_PAGE:
6044 page = target.page;
Kirill A. Shutemov53f92632016-01-15 16:53:42 -08006045 /*
6046 * We can have a part of the split pmd here. Moving it
6047 * can be done but it would be too convoluted so simply
6048 * ignore such a partial THP and keep it in original
6049 * memcg. There should be somebody mapping the head.
6050 */
6051 if (PageTransCompound(page))
6052 goto put;
Jérôme Glissec733a822017-09-08 16:11:54 -07006053 if (!device && isolate_lru_page(page))
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006054 goto put;
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -08006055 if (!mem_cgroup_move_account(page, false,
6056 mc.from, mc.to)) {
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006057 mc.precharge--;
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08006058 /* we uncharge from mc.from later. */
6059 mc.moved_charge++;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006060 }
Jérôme Glissec733a822017-09-08 16:11:54 -07006061 if (!device)
6062 putback_lru_page(page);
Naoya Horiguchi8d32ff82012-03-21 16:34:27 -07006063put: /* get_mctgt_type() gets the page */
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006064 put_page(page);
6065 break;
Daisuke Nishimura02491442010-03-10 15:22:17 -08006066 case MC_TARGET_SWAP:
6067 ent = target.ent;
Hugh Dickinse91cbb42012-05-29 15:06:51 -07006068 if (!mem_cgroup_move_swap_account(ent, mc.from, mc.to)) {
Daisuke Nishimura02491442010-03-10 15:22:17 -08006069 mc.precharge--;
Hugh Dickins8d22a932020-07-23 21:15:24 -07006070 mem_cgroup_id_get_many(mc.to, 1);
6071 /* we fixup other refcnts and charges later. */
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08006072 mc.moved_swap++;
6073 }
Daisuke Nishimura02491442010-03-10 15:22:17 -08006074 break;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006075 default:
6076 break;
6077 }
6078 }
6079 pte_unmap_unlock(pte - 1, ptl);
6080 cond_resched();
6081
6082 if (addr != end) {
6083 /*
6084 * We have consumed all precharges we got in can_attach().
6085 * We try charge one by one, but don't do any additional
6086 * charges to mc.to if we have failed in charge once in attach()
6087 * phase.
6088 */
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08006089 ret = mem_cgroup_do_precharge(1);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006090 if (!ret)
6091 goto retry;
6092 }
6093
6094 return ret;
6095}
6096
Christoph Hellwig7b86ac32019-08-28 16:19:54 +02006097static const struct mm_walk_ops charge_walk_ops = {
6098 .pmd_entry = mem_cgroup_move_charge_pte_range,
6099};
6100
Tejun Heo264a0ae2016-04-21 19:09:02 -04006101static void mem_cgroup_move_charge(void)
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006102{
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006103 lru_add_drain_all();
Johannes Weiner312722c2014-12-10 15:44:25 -08006104 /*
Johannes Weiner81f8c3a2016-03-15 14:57:04 -07006105 * Signal lock_page_memcg() to take the memcg's move_lock
6106 * while we're moving its pages to another memcg. Then wait
6107 * for already started RCU-only updates to finish.
Johannes Weiner312722c2014-12-10 15:44:25 -08006108 */
6109 atomic_inc(&mc.from->moving_account);
6110 synchronize_rcu();
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08006111retry:
Michel Lespinassed8ed45c2020-06-08 21:33:25 -07006112 if (unlikely(!mmap_read_trylock(mc.mm))) {
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08006113 /*
Michel Lespinassec1e8d7c2020-06-08 21:33:54 -07006114 * Someone who are holding the mmap_lock might be waiting in
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08006115 * waitq. So we cancel all extra charges, wake up all waiters,
6116 * and retry. Because we cancel precharges, we might not be able
6117 * to move enough charges, but moving charge is a best-effort
6118 * feature anyway, so it wouldn't be a big problem.
6119 */
6120 __mem_cgroup_clear_mc();
6121 cond_resched();
6122 goto retry;
6123 }
Naoya Horiguchi26bcd642015-02-11 15:27:57 -08006124 /*
6125 * When we have consumed all precharges and failed in doing
6126 * additional charge, the page walk just aborts.
6127 */
Christoph Hellwig7b86ac32019-08-28 16:19:54 +02006128 walk_page_range(mc.mm, 0, mc.mm->highest_vm_end, &charge_walk_ops,
6129 NULL);
James Morse0247f3f2016-10-07 17:00:12 -07006130
Michel Lespinassed8ed45c2020-06-08 21:33:25 -07006131 mmap_read_unlock(mc.mm);
Johannes Weiner312722c2014-12-10 15:44:25 -08006132 atomic_dec(&mc.from->moving_account);
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08006133}
6134
Tejun Heo264a0ae2016-04-21 19:09:02 -04006135static void mem_cgroup_move_task(void)
Balbir Singh67e465a2008-02-07 00:13:54 -08006136{
Tejun Heo264a0ae2016-04-21 19:09:02 -04006137 if (mc.to) {
6138 mem_cgroup_move_charge();
KOSAKI Motohiroa4336582011-06-15 15:08:13 -07006139 mem_cgroup_clear_mc();
Tejun Heo264a0ae2016-04-21 19:09:02 -04006140 }
Balbir Singh67e465a2008-02-07 00:13:54 -08006141}
Daisuke Nishimura5cfb80a2010-03-23 13:35:11 -07006142#else /* !CONFIG_MMU */
Tejun Heo1f7dd3e52015-12-03 10:18:21 -05006143static int mem_cgroup_can_attach(struct cgroup_taskset *tset)
Daisuke Nishimura5cfb80a2010-03-23 13:35:11 -07006144{
6145 return 0;
6146}
Tejun Heo1f7dd3e52015-12-03 10:18:21 -05006147static void mem_cgroup_cancel_attach(struct cgroup_taskset *tset)
Daisuke Nishimura5cfb80a2010-03-23 13:35:11 -07006148{
6149}
Tejun Heo264a0ae2016-04-21 19:09:02 -04006150static void mem_cgroup_move_task(void)
Daisuke Nishimura5cfb80a2010-03-23 13:35:11 -07006151{
6152}
6153#endif
Balbir Singh67e465a2008-02-07 00:13:54 -08006154
Chris Down677dc972019-03-05 15:45:55 -08006155static int seq_puts_memcg_tunable(struct seq_file *m, unsigned long value)
6156{
6157 if (value == PAGE_COUNTER_MAX)
6158 seq_puts(m, "max\n");
6159 else
6160 seq_printf(m, "%llu\n", (u64)value * PAGE_SIZE);
6161
6162 return 0;
6163}
6164
Johannes Weiner241994ed2015-02-11 15:26:06 -08006165static u64 memory_current_read(struct cgroup_subsys_state *css,
6166 struct cftype *cft)
6167{
Johannes Weinerf5fc3c5d2015-11-05 18:50:23 -08006168 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
6169
6170 return (u64)page_counter_read(&memcg->memory) * PAGE_SIZE;
Johannes Weiner241994ed2015-02-11 15:26:06 -08006171}
6172
Roman Gushchinbf8d5d52018-06-07 17:07:46 -07006173static int memory_min_show(struct seq_file *m, void *v)
6174{
Chris Down677dc972019-03-05 15:45:55 -08006175 return seq_puts_memcg_tunable(m,
6176 READ_ONCE(mem_cgroup_from_seq(m)->memory.min));
Roman Gushchinbf8d5d52018-06-07 17:07:46 -07006177}
6178
6179static ssize_t memory_min_write(struct kernfs_open_file *of,
6180 char *buf, size_t nbytes, loff_t off)
6181{
6182 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6183 unsigned long min;
6184 int err;
6185
6186 buf = strstrip(buf);
6187 err = page_counter_memparse(buf, "max", &min);
6188 if (err)
6189 return err;
6190
6191 page_counter_set_min(&memcg->memory, min);
6192
6193 return nbytes;
6194}
6195
Johannes Weiner241994ed2015-02-11 15:26:06 -08006196static int memory_low_show(struct seq_file *m, void *v)
6197{
Chris Down677dc972019-03-05 15:45:55 -08006198 return seq_puts_memcg_tunable(m,
6199 READ_ONCE(mem_cgroup_from_seq(m)->memory.low));
Johannes Weiner241994ed2015-02-11 15:26:06 -08006200}
6201
6202static ssize_t memory_low_write(struct kernfs_open_file *of,
6203 char *buf, size_t nbytes, loff_t off)
6204{
6205 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6206 unsigned long low;
6207 int err;
6208
6209 buf = strstrip(buf);
Johannes Weinerd2973692015-02-27 15:52:04 -08006210 err = page_counter_memparse(buf, "max", &low);
Johannes Weiner241994ed2015-02-11 15:26:06 -08006211 if (err)
6212 return err;
6213
Roman Gushchin23067152018-06-07 17:06:22 -07006214 page_counter_set_low(&memcg->memory, low);
Johannes Weiner241994ed2015-02-11 15:26:06 -08006215
6216 return nbytes;
6217}
6218
6219static int memory_high_show(struct seq_file *m, void *v)
6220{
Jakub Kicinskid1663a92020-06-01 21:49:49 -07006221 return seq_puts_memcg_tunable(m,
6222 READ_ONCE(mem_cgroup_from_seq(m)->memory.high));
Johannes Weiner241994ed2015-02-11 15:26:06 -08006223}
6224
6225static ssize_t memory_high_write(struct kernfs_open_file *of,
6226 char *buf, size_t nbytes, loff_t off)
6227{
6228 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
Chris Downd977aa92020-08-06 23:21:58 -07006229 unsigned int nr_retries = MAX_RECLAIM_RETRIES;
Johannes Weiner8c8c3832019-11-30 17:50:09 -08006230 bool drained = false;
Johannes Weiner241994ed2015-02-11 15:26:06 -08006231 unsigned long high;
6232 int err;
6233
6234 buf = strstrip(buf);
Johannes Weinerd2973692015-02-27 15:52:04 -08006235 err = page_counter_memparse(buf, "max", &high);
Johannes Weiner241994ed2015-02-11 15:26:06 -08006236 if (err)
6237 return err;
6238
Johannes Weinere82553c2021-02-09 13:42:28 -08006239 page_counter_set_high(&memcg->memory, high);
6240
Johannes Weiner8c8c3832019-11-30 17:50:09 -08006241 for (;;) {
6242 unsigned long nr_pages = page_counter_read(&memcg->memory);
6243 unsigned long reclaimed;
Johannes Weiner588083b2016-03-17 14:20:25 -07006244
Johannes Weiner8c8c3832019-11-30 17:50:09 -08006245 if (nr_pages <= high)
6246 break;
6247
6248 if (signal_pending(current))
6249 break;
6250
6251 if (!drained) {
6252 drain_all_stock(memcg);
6253 drained = true;
6254 continue;
6255 }
6256
6257 reclaimed = try_to_free_mem_cgroup_pages(memcg, nr_pages - high,
6258 GFP_KERNEL, true);
6259
6260 if (!reclaimed && !nr_retries--)
6261 break;
6262 }
6263
Johannes Weiner19ce33a2020-08-06 23:22:12 -07006264 memcg_wb_domain_size_changed(memcg);
Johannes Weiner241994ed2015-02-11 15:26:06 -08006265 return nbytes;
6266}
6267
6268static int memory_max_show(struct seq_file *m, void *v)
6269{
Chris Down677dc972019-03-05 15:45:55 -08006270 return seq_puts_memcg_tunable(m,
6271 READ_ONCE(mem_cgroup_from_seq(m)->memory.max));
Johannes Weiner241994ed2015-02-11 15:26:06 -08006272}
6273
6274static ssize_t memory_max_write(struct kernfs_open_file *of,
6275 char *buf, size_t nbytes, loff_t off)
6276{
6277 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
Chris Downd977aa92020-08-06 23:21:58 -07006278 unsigned int nr_reclaims = MAX_RECLAIM_RETRIES;
Johannes Weinerb6e6edc2016-03-17 14:20:28 -07006279 bool drained = false;
Johannes Weiner241994ed2015-02-11 15:26:06 -08006280 unsigned long max;
6281 int err;
6282
6283 buf = strstrip(buf);
Johannes Weinerd2973692015-02-27 15:52:04 -08006284 err = page_counter_memparse(buf, "max", &max);
Johannes Weiner241994ed2015-02-11 15:26:06 -08006285 if (err)
6286 return err;
6287
Roman Gushchinbbec2e12018-06-07 17:06:18 -07006288 xchg(&memcg->memory.max, max);
Johannes Weinerb6e6edc2016-03-17 14:20:28 -07006289
6290 for (;;) {
6291 unsigned long nr_pages = page_counter_read(&memcg->memory);
6292
6293 if (nr_pages <= max)
6294 break;
6295
Johannes Weiner7249c9f2019-11-30 17:50:06 -08006296 if (signal_pending(current))
Johannes Weinerb6e6edc2016-03-17 14:20:28 -07006297 break;
Johannes Weinerb6e6edc2016-03-17 14:20:28 -07006298
6299 if (!drained) {
6300 drain_all_stock(memcg);
6301 drained = true;
6302 continue;
6303 }
6304
6305 if (nr_reclaims) {
6306 if (!try_to_free_mem_cgroup_pages(memcg, nr_pages - max,
6307 GFP_KERNEL, true))
6308 nr_reclaims--;
6309 continue;
6310 }
6311
Johannes Weinere27be242018-04-10 16:29:45 -07006312 memcg_memory_event(memcg, MEMCG_OOM);
Johannes Weinerb6e6edc2016-03-17 14:20:28 -07006313 if (!mem_cgroup_out_of_memory(memcg, GFP_KERNEL, 0))
6314 break;
6315 }
Johannes Weiner241994ed2015-02-11 15:26:06 -08006316
Tejun Heo2529bb32015-05-22 18:23:34 -04006317 memcg_wb_domain_size_changed(memcg);
Johannes Weiner241994ed2015-02-11 15:26:06 -08006318 return nbytes;
6319}
6320
Shakeel Butt1e577f92019-07-11 20:55:55 -07006321static void __memory_events_show(struct seq_file *m, atomic_long_t *events)
6322{
6323 seq_printf(m, "low %lu\n", atomic_long_read(&events[MEMCG_LOW]));
6324 seq_printf(m, "high %lu\n", atomic_long_read(&events[MEMCG_HIGH]));
6325 seq_printf(m, "max %lu\n", atomic_long_read(&events[MEMCG_MAX]));
6326 seq_printf(m, "oom %lu\n", atomic_long_read(&events[MEMCG_OOM]));
6327 seq_printf(m, "oom_kill %lu\n",
6328 atomic_long_read(&events[MEMCG_OOM_KILL]));
6329}
6330
Johannes Weiner241994ed2015-02-11 15:26:06 -08006331static int memory_events_show(struct seq_file *m, void *v)
6332{
Chris Downaa9694b2019-03-05 15:45:52 -08006333 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
Johannes Weiner241994ed2015-02-11 15:26:06 -08006334
Shakeel Butt1e577f92019-07-11 20:55:55 -07006335 __memory_events_show(m, memcg->memory_events);
6336 return 0;
6337}
Johannes Weiner241994ed2015-02-11 15:26:06 -08006338
Shakeel Butt1e577f92019-07-11 20:55:55 -07006339static int memory_events_local_show(struct seq_file *m, void *v)
6340{
6341 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
6342
6343 __memory_events_show(m, memcg->memory_events_local);
Johannes Weiner241994ed2015-02-11 15:26:06 -08006344 return 0;
6345}
6346
Johannes Weiner587d9f72016-01-20 15:03:19 -08006347static int memory_stat_show(struct seq_file *m, void *v)
6348{
Chris Downaa9694b2019-03-05 15:45:52 -08006349 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
Johannes Weinerc8713d02019-07-11 20:55:59 -07006350 char *buf;
Johannes Weiner587d9f72016-01-20 15:03:19 -08006351
Johannes Weinerc8713d02019-07-11 20:55:59 -07006352 buf = memory_stat_format(memcg);
6353 if (!buf)
6354 return -ENOMEM;
6355 seq_puts(m, buf);
6356 kfree(buf);
Johannes Weiner587d9f72016-01-20 15:03:19 -08006357 return 0;
6358}
6359
Muchun Song5f9a4f42020-10-13 16:52:59 -07006360#ifdef CONFIG_NUMA
Muchun Songfff66b72021-02-24 12:03:43 -08006361static inline unsigned long lruvec_page_state_output(struct lruvec *lruvec,
6362 int item)
6363{
6364 return lruvec_page_state(lruvec, item) * memcg_page_state_unit(item);
6365}
6366
Muchun Song5f9a4f42020-10-13 16:52:59 -07006367static int memory_numa_stat_show(struct seq_file *m, void *v)
6368{
6369 int i;
6370 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
6371
6372 for (i = 0; i < ARRAY_SIZE(memory_stats); i++) {
6373 int nid;
6374
6375 if (memory_stats[i].idx >= NR_VM_NODE_STAT_ITEMS)
6376 continue;
6377
6378 seq_printf(m, "%s", memory_stats[i].name);
6379 for_each_node_state(nid, N_MEMORY) {
6380 u64 size;
6381 struct lruvec *lruvec;
6382
6383 lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(nid));
Muchun Songfff66b72021-02-24 12:03:43 -08006384 size = lruvec_page_state_output(lruvec,
6385 memory_stats[i].idx);
Muchun Song5f9a4f42020-10-13 16:52:59 -07006386 seq_printf(m, " N%d=%llu", nid, size);
6387 }
6388 seq_putc(m, '\n');
6389 }
6390
6391 return 0;
6392}
6393#endif
6394
Roman Gushchin3d8b38e2018-08-21 21:53:54 -07006395static int memory_oom_group_show(struct seq_file *m, void *v)
6396{
Chris Downaa9694b2019-03-05 15:45:52 -08006397 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
Roman Gushchin3d8b38e2018-08-21 21:53:54 -07006398
6399 seq_printf(m, "%d\n", memcg->oom_group);
6400
6401 return 0;
6402}
6403
6404static ssize_t memory_oom_group_write(struct kernfs_open_file *of,
6405 char *buf, size_t nbytes, loff_t off)
6406{
6407 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
6408 int ret, oom_group;
6409
6410 buf = strstrip(buf);
6411 if (!buf)
6412 return -EINVAL;
6413
6414 ret = kstrtoint(buf, 0, &oom_group);
6415 if (ret)
6416 return ret;
6417
6418 if (oom_group != 0 && oom_group != 1)
6419 return -EINVAL;
6420
6421 memcg->oom_group = oom_group;
6422
6423 return nbytes;
6424}
6425
Johannes Weiner241994ed2015-02-11 15:26:06 -08006426static struct cftype memory_files[] = {
6427 {
6428 .name = "current",
Johannes Weinerf5fc3c5d2015-11-05 18:50:23 -08006429 .flags = CFTYPE_NOT_ON_ROOT,
Johannes Weiner241994ed2015-02-11 15:26:06 -08006430 .read_u64 = memory_current_read,
6431 },
6432 {
Roman Gushchinbf8d5d52018-06-07 17:07:46 -07006433 .name = "min",
6434 .flags = CFTYPE_NOT_ON_ROOT,
6435 .seq_show = memory_min_show,
6436 .write = memory_min_write,
6437 },
6438 {
Johannes Weiner241994ed2015-02-11 15:26:06 -08006439 .name = "low",
6440 .flags = CFTYPE_NOT_ON_ROOT,
6441 .seq_show = memory_low_show,
6442 .write = memory_low_write,
6443 },
6444 {
6445 .name = "high",
6446 .flags = CFTYPE_NOT_ON_ROOT,
6447 .seq_show = memory_high_show,
6448 .write = memory_high_write,
6449 },
6450 {
6451 .name = "max",
6452 .flags = CFTYPE_NOT_ON_ROOT,
6453 .seq_show = memory_max_show,
6454 .write = memory_max_write,
6455 },
6456 {
6457 .name = "events",
6458 .flags = CFTYPE_NOT_ON_ROOT,
Tejun Heo472912a2015-09-18 18:01:59 -04006459 .file_offset = offsetof(struct mem_cgroup, events_file),
Johannes Weiner241994ed2015-02-11 15:26:06 -08006460 .seq_show = memory_events_show,
6461 },
Johannes Weiner587d9f72016-01-20 15:03:19 -08006462 {
Shakeel Butt1e577f92019-07-11 20:55:55 -07006463 .name = "events.local",
6464 .flags = CFTYPE_NOT_ON_ROOT,
6465 .file_offset = offsetof(struct mem_cgroup, events_local_file),
6466 .seq_show = memory_events_local_show,
6467 },
6468 {
Johannes Weiner587d9f72016-01-20 15:03:19 -08006469 .name = "stat",
Johannes Weiner587d9f72016-01-20 15:03:19 -08006470 .seq_show = memory_stat_show,
6471 },
Muchun Song5f9a4f42020-10-13 16:52:59 -07006472#ifdef CONFIG_NUMA
6473 {
6474 .name = "numa_stat",
6475 .seq_show = memory_numa_stat_show,
6476 },
6477#endif
Roman Gushchin3d8b38e2018-08-21 21:53:54 -07006478 {
6479 .name = "oom.group",
6480 .flags = CFTYPE_NOT_ON_ROOT | CFTYPE_NS_DELEGATABLE,
6481 .seq_show = memory_oom_group_show,
6482 .write = memory_oom_group_write,
6483 },
Johannes Weiner241994ed2015-02-11 15:26:06 -08006484 { } /* terminate */
6485};
6486
Tejun Heo073219e2014-02-08 10:36:58 -05006487struct cgroup_subsys memory_cgrp_subsys = {
Tejun Heo92fb9742012-11-19 08:13:38 -08006488 .css_alloc = mem_cgroup_css_alloc,
Glauber Costad142e3e2013-02-22 16:34:52 -08006489 .css_online = mem_cgroup_css_online,
Tejun Heo92fb9742012-11-19 08:13:38 -08006490 .css_offline = mem_cgroup_css_offline,
Vladimir Davydov6df38682015-12-29 14:54:10 -08006491 .css_released = mem_cgroup_css_released,
Tejun Heo92fb9742012-11-19 08:13:38 -08006492 .css_free = mem_cgroup_css_free,
Tejun Heo1ced9532014-07-08 18:02:57 -04006493 .css_reset = mem_cgroup_css_reset,
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08006494 .can_attach = mem_cgroup_can_attach,
6495 .cancel_attach = mem_cgroup_cancel_attach,
Tejun Heo264a0ae2016-04-21 19:09:02 -04006496 .post_attach = mem_cgroup_move_task,
Johannes Weiner241994ed2015-02-11 15:26:06 -08006497 .dfl_cftypes = memory_files,
6498 .legacy_cftypes = mem_cgroup_legacy_files,
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08006499 .early_init = 0,
Balbir Singh8cdea7c2008-02-07 00:13:50 -08006500};
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -08006501
Johannes Weinerbc50bcc2020-04-01 21:07:03 -07006502/*
6503 * This function calculates an individual cgroup's effective
6504 * protection which is derived from its own memory.min/low, its
6505 * parent's and siblings' settings, as well as the actual memory
6506 * distribution in the tree.
6507 *
6508 * The following rules apply to the effective protection values:
6509 *
6510 * 1. At the first level of reclaim, effective protection is equal to
6511 * the declared protection in memory.min and memory.low.
6512 *
6513 * 2. To enable safe delegation of the protection configuration, at
6514 * subsequent levels the effective protection is capped to the
6515 * parent's effective protection.
6516 *
6517 * 3. To make complex and dynamic subtrees easier to configure, the
6518 * user is allowed to overcommit the declared protection at a given
6519 * level. If that is the case, the parent's effective protection is
6520 * distributed to the children in proportion to how much protection
6521 * they have declared and how much of it they are utilizing.
6522 *
6523 * This makes distribution proportional, but also work-conserving:
6524 * if one cgroup claims much more protection than it uses memory,
6525 * the unused remainder is available to its siblings.
6526 *
6527 * 4. Conversely, when the declared protection is undercommitted at a
6528 * given level, the distribution of the larger parental protection
6529 * budget is NOT proportional. A cgroup's protection from a sibling
6530 * is capped to its own memory.min/low setting.
6531 *
Johannes Weiner8a931f82020-04-01 21:07:07 -07006532 * 5. However, to allow protecting recursive subtrees from each other
6533 * without having to declare each individual cgroup's fixed share
6534 * of the ancestor's claim to protection, any unutilized -
6535 * "floating" - protection from up the tree is distributed in
6536 * proportion to each cgroup's *usage*. This makes the protection
6537 * neutral wrt sibling cgroups and lets them compete freely over
6538 * the shared parental protection budget, but it protects the
6539 * subtree as a whole from neighboring subtrees.
6540 *
6541 * Note that 4. and 5. are not in conflict: 4. is about protecting
6542 * against immediate siblings whereas 5. is about protecting against
6543 * neighboring subtrees.
Johannes Weinerbc50bcc2020-04-01 21:07:03 -07006544 */
6545static unsigned long effective_protection(unsigned long usage,
Johannes Weiner8a931f82020-04-01 21:07:07 -07006546 unsigned long parent_usage,
Johannes Weinerbc50bcc2020-04-01 21:07:03 -07006547 unsigned long setting,
6548 unsigned long parent_effective,
6549 unsigned long siblings_protected)
6550{
6551 unsigned long protected;
Johannes Weiner8a931f82020-04-01 21:07:07 -07006552 unsigned long ep;
Johannes Weinerbc50bcc2020-04-01 21:07:03 -07006553
6554 protected = min(usage, setting);
6555 /*
6556 * If all cgroups at this level combined claim and use more
6557 * protection then what the parent affords them, distribute
6558 * shares in proportion to utilization.
6559 *
6560 * We are using actual utilization rather than the statically
6561 * claimed protection in order to be work-conserving: claimed
6562 * but unused protection is available to siblings that would
6563 * otherwise get a smaller chunk than what they claimed.
6564 */
6565 if (siblings_protected > parent_effective)
6566 return protected * parent_effective / siblings_protected;
6567
6568 /*
6569 * Ok, utilized protection of all children is within what the
6570 * parent affords them, so we know whatever this child claims
6571 * and utilizes is effectively protected.
6572 *
6573 * If there is unprotected usage beyond this value, reclaim
6574 * will apply pressure in proportion to that amount.
6575 *
6576 * If there is unutilized protection, the cgroup will be fully
6577 * shielded from reclaim, but we do return a smaller value for
6578 * protection than what the group could enjoy in theory. This
6579 * is okay. With the overcommit distribution above, effective
6580 * protection is always dependent on how memory is actually
6581 * consumed among the siblings anyway.
6582 */
Johannes Weiner8a931f82020-04-01 21:07:07 -07006583 ep = protected;
6584
6585 /*
6586 * If the children aren't claiming (all of) the protection
6587 * afforded to them by the parent, distribute the remainder in
6588 * proportion to the (unprotected) memory of each cgroup. That
6589 * way, cgroups that aren't explicitly prioritized wrt each
6590 * other compete freely over the allowance, but they are
6591 * collectively protected from neighboring trees.
6592 *
6593 * We're using unprotected memory for the weight so that if
6594 * some cgroups DO claim explicit protection, we don't protect
6595 * the same bytes twice.
Johannes Weinercd324ed2020-06-25 20:30:16 -07006596 *
6597 * Check both usage and parent_usage against the respective
6598 * protected values. One should imply the other, but they
6599 * aren't read atomically - make sure the division is sane.
Johannes Weiner8a931f82020-04-01 21:07:07 -07006600 */
6601 if (!(cgrp_dfl_root.flags & CGRP_ROOT_MEMORY_RECURSIVE_PROT))
6602 return ep;
Johannes Weinercd324ed2020-06-25 20:30:16 -07006603 if (parent_effective > siblings_protected &&
6604 parent_usage > siblings_protected &&
6605 usage > protected) {
Johannes Weiner8a931f82020-04-01 21:07:07 -07006606 unsigned long unclaimed;
6607
6608 unclaimed = parent_effective - siblings_protected;
6609 unclaimed *= usage - protected;
6610 unclaimed /= parent_usage - siblings_protected;
6611
6612 ep += unclaimed;
6613 }
6614
6615 return ep;
Johannes Weinerbc50bcc2020-04-01 21:07:03 -07006616}
6617
Johannes Weiner241994ed2015-02-11 15:26:06 -08006618/**
Roman Gushchinbf8d5d52018-06-07 17:07:46 -07006619 * mem_cgroup_protected - check if memory consumption is in the normal range
Sean Christopherson34c81052017-07-10 15:48:05 -07006620 * @root: the top ancestor of the sub-tree being checked
Johannes Weiner241994ed2015-02-11 15:26:06 -08006621 * @memcg: the memory cgroup to check
6622 *
Roman Gushchin23067152018-06-07 17:06:22 -07006623 * WARNING: This function is not stateless! It can only be used as part
6624 * of a top-down tree iteration, not for isolated queries.
Johannes Weiner241994ed2015-02-11 15:26:06 -08006625 */
Chris Down45c7f7e2020-08-06 23:22:05 -07006626void mem_cgroup_calculate_protection(struct mem_cgroup *root,
6627 struct mem_cgroup *memcg)
Johannes Weiner241994ed2015-02-11 15:26:06 -08006628{
Johannes Weiner8a931f82020-04-01 21:07:07 -07006629 unsigned long usage, parent_usage;
Roman Gushchin23067152018-06-07 17:06:22 -07006630 struct mem_cgroup *parent;
6631
Johannes Weiner241994ed2015-02-11 15:26:06 -08006632 if (mem_cgroup_disabled())
Chris Down45c7f7e2020-08-06 23:22:05 -07006633 return;
Johannes Weiner241994ed2015-02-11 15:26:06 -08006634
Sean Christopherson34c81052017-07-10 15:48:05 -07006635 if (!root)
6636 root = root_mem_cgroup;
Yafang Shao22f74962020-08-06 23:22:01 -07006637
6638 /*
6639 * Effective values of the reclaim targets are ignored so they
6640 * can be stale. Have a look at mem_cgroup_protection for more
6641 * details.
6642 * TODO: calculation should be more robust so that we do not need
6643 * that special casing.
6644 */
Sean Christopherson34c81052017-07-10 15:48:05 -07006645 if (memcg == root)
Chris Down45c7f7e2020-08-06 23:22:05 -07006646 return;
Johannes Weiner241994ed2015-02-11 15:26:06 -08006647
Roman Gushchin23067152018-06-07 17:06:22 -07006648 usage = page_counter_read(&memcg->memory);
Roman Gushchinbf8d5d52018-06-07 17:07:46 -07006649 if (!usage)
Chris Down45c7f7e2020-08-06 23:22:05 -07006650 return;
Sean Christopherson34c81052017-07-10 15:48:05 -07006651
Roman Gushchinbf8d5d52018-06-07 17:07:46 -07006652 parent = parent_mem_cgroup(memcg);
Roman Gushchindf2a4192018-06-14 15:26:17 -07006653 /* No parent means a non-hierarchical mode on v1 memcg */
6654 if (!parent)
Chris Down45c7f7e2020-08-06 23:22:05 -07006655 return;
Roman Gushchindf2a4192018-06-14 15:26:17 -07006656
Johannes Weinerbc50bcc2020-04-01 21:07:03 -07006657 if (parent == root) {
Chris Downc3d53202020-04-01 21:07:27 -07006658 memcg->memory.emin = READ_ONCE(memcg->memory.min);
Chris Down03960e32020-06-25 20:30:22 -07006659 memcg->memory.elow = READ_ONCE(memcg->memory.low);
Chris Down45c7f7e2020-08-06 23:22:05 -07006660 return;
Roman Gushchinbf8d5d52018-06-07 17:07:46 -07006661 }
6662
Johannes Weiner8a931f82020-04-01 21:07:07 -07006663 parent_usage = page_counter_read(&parent->memory);
6664
Chris Downb3a78222020-04-01 21:07:33 -07006665 WRITE_ONCE(memcg->memory.emin, effective_protection(usage, parent_usage,
Chris Downc3d53202020-04-01 21:07:27 -07006666 READ_ONCE(memcg->memory.min),
6667 READ_ONCE(parent->memory.emin),
Chris Downb3a78222020-04-01 21:07:33 -07006668 atomic_long_read(&parent->memory.children_min_usage)));
Roman Gushchin23067152018-06-07 17:06:22 -07006669
Chris Downb3a78222020-04-01 21:07:33 -07006670 WRITE_ONCE(memcg->memory.elow, effective_protection(usage, parent_usage,
Chris Down03960e32020-06-25 20:30:22 -07006671 READ_ONCE(memcg->memory.low),
6672 READ_ONCE(parent->memory.elow),
Chris Downb3a78222020-04-01 21:07:33 -07006673 atomic_long_read(&parent->memory.children_low_usage)));
Johannes Weiner241994ed2015-02-11 15:26:06 -08006674}
6675
Johannes Weiner00501b52014-08-08 14:19:20 -07006676/**
Johannes Weinerf0e45fb2020-06-03 16:02:07 -07006677 * mem_cgroup_charge - charge a newly allocated page to a cgroup
Johannes Weiner00501b52014-08-08 14:19:20 -07006678 * @page: page to charge
6679 * @mm: mm context of the victim
6680 * @gfp_mask: reclaim mode
Johannes Weiner00501b52014-08-08 14:19:20 -07006681 *
6682 * Try to charge @page to the memcg that @mm belongs to, reclaiming
6683 * pages according to @gfp_mask if necessary.
6684 *
Johannes Weinerf0e45fb2020-06-03 16:02:07 -07006685 * Returns 0 on success. Otherwise, an error code is returned.
Johannes Weiner00501b52014-08-08 14:19:20 -07006686 */
Johannes Weinerd9eb1ea2020-06-03 16:02:24 -07006687int mem_cgroup_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask)
Johannes Weiner00501b52014-08-08 14:19:20 -07006688{
Matthew Wilcox (Oracle)6c357842020-08-14 17:30:37 -07006689 unsigned int nr_pages = thp_nr_pages(page);
Johannes Weiner00501b52014-08-08 14:19:20 -07006690 struct mem_cgroup *memcg = NULL;
Johannes Weiner00501b52014-08-08 14:19:20 -07006691 int ret = 0;
6692
6693 if (mem_cgroup_disabled())
6694 goto out;
6695
6696 if (PageSwapCache(page)) {
Johannes Weiner2d1c4982020-06-03 16:02:14 -07006697 swp_entry_t ent = { .val = page_private(page), };
6698 unsigned short id;
6699
Johannes Weiner00501b52014-08-08 14:19:20 -07006700 /*
6701 * Every swap fault against a single page tries to charge the
6702 * page, bail as early as possible. shmem_unuse() encounters
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08006703 * already charged pages, too. page and memcg binding is
6704 * protected by the page lock, which serializes swap cache
6705 * removal, which in turn serializes uncharging.
Johannes Weiner00501b52014-08-08 14:19:20 -07006706 */
Vladimir Davydove993d902015-09-09 15:35:35 -07006707 VM_BUG_ON_PAGE(!PageLocked(page), page);
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08006708 if (page_memcg(compound_head(page)))
Johannes Weiner00501b52014-08-08 14:19:20 -07006709 goto out;
Vladimir Davydove993d902015-09-09 15:35:35 -07006710
Johannes Weiner2d1c4982020-06-03 16:02:14 -07006711 id = lookup_swap_cgroup_id(ent);
6712 rcu_read_lock();
6713 memcg = mem_cgroup_from_id(id);
6714 if (memcg && !css_tryget_online(&memcg->css))
6715 memcg = NULL;
6716 rcu_read_unlock();
Johannes Weiner00501b52014-08-08 14:19:20 -07006717 }
6718
Johannes Weiner00501b52014-08-08 14:19:20 -07006719 if (!memcg)
6720 memcg = get_mem_cgroup_from_mm(mm);
6721
6722 ret = try_charge(memcg, gfp_mask, nr_pages);
Johannes Weinerf0e45fb2020-06-03 16:02:07 -07006723 if (ret)
6724 goto out_put;
Johannes Weiner00501b52014-08-08 14:19:20 -07006725
Johannes Weiner1a3e1f42020-08-06 23:20:45 -07006726 css_get(&memcg->css);
Johannes Weinerd9eb1ea2020-06-03 16:02:24 -07006727 commit_charge(page, memcg);
Johannes Weiner6abb5a82014-08-08 14:19:33 -07006728
Johannes Weiner6abb5a82014-08-08 14:19:33 -07006729 local_irq_disable();
Johannes Weiner3fba69a2020-06-03 16:01:31 -07006730 mem_cgroup_charge_statistics(memcg, page, nr_pages);
Johannes Weiner6abb5a82014-08-08 14:19:33 -07006731 memcg_check_events(memcg, page);
6732 local_irq_enable();
Johannes Weiner00501b52014-08-08 14:19:20 -07006733
Muchun Songcae3af62021-02-24 12:04:19 -08006734 /*
6735 * Cgroup1's unified memory+swap counter has been charged with the
6736 * new swapcache page, finish the transfer by uncharging the swap
6737 * slot. The swap slot would also get uncharged when it dies, but
6738 * it can stick around indefinitely and we'd count the page twice
6739 * the entire time.
6740 *
6741 * Cgroup2 has separate resource counters for memory and swap,
6742 * so this is a non-issue here. Memory and swap charge lifetimes
6743 * correspond 1:1 to page and swap slot lifetimes: we charge the
6744 * page to memory here, and uncharge swap when the slot is freed.
6745 */
6746 if (do_memsw_account() && PageSwapCache(page)) {
Johannes Weiner00501b52014-08-08 14:19:20 -07006747 swp_entry_t entry = { .val = page_private(page) };
6748 /*
6749 * The swap entry might not get freed for a long time,
6750 * let's not wait for it. The page already received a
6751 * memory+swap charge, drop the swap entry duplicate.
6752 */
Huang Ying38d8b4e2017-07-06 15:37:18 -07006753 mem_cgroup_uncharge_swap(entry, nr_pages);
Johannes Weiner00501b52014-08-08 14:19:20 -07006754 }
Johannes Weiner00501b52014-08-08 14:19:20 -07006755
Johannes Weinerf0e45fb2020-06-03 16:02:07 -07006756out_put:
6757 css_put(&memcg->css);
6758out:
6759 return ret;
Johannes Weiner3fea5a42020-06-03 16:01:41 -07006760}
6761
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006762struct uncharge_gather {
6763 struct mem_cgroup *memcg;
Johannes Weiner9f762db2020-06-03 16:01:44 -07006764 unsigned long nr_pages;
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006765 unsigned long pgpgout;
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006766 unsigned long nr_kmem;
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006767 struct page *dummy_page;
6768};
6769
6770static inline void uncharge_gather_clear(struct uncharge_gather *ug)
Johannes Weiner747db952014-08-08 14:19:24 -07006771{
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006772 memset(ug, 0, sizeof(*ug));
6773}
6774
6775static void uncharge_batch(const struct uncharge_gather *ug)
6776{
Johannes Weiner747db952014-08-08 14:19:24 -07006777 unsigned long flags;
6778
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006779 if (!mem_cgroup_is_root(ug->memcg)) {
Johannes Weiner9f762db2020-06-03 16:01:44 -07006780 page_counter_uncharge(&ug->memcg->memory, ug->nr_pages);
Johannes Weiner7941d212016-01-14 15:21:23 -08006781 if (do_memsw_account())
Johannes Weiner9f762db2020-06-03 16:01:44 -07006782 page_counter_uncharge(&ug->memcg->memsw, ug->nr_pages);
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006783 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && ug->nr_kmem)
6784 page_counter_uncharge(&ug->memcg->kmem, ug->nr_kmem);
6785 memcg_oom_recover(ug->memcg);
Johannes Weinerce00a962014-09-05 08:43:57 -04006786 }
Johannes Weiner747db952014-08-08 14:19:24 -07006787
6788 local_irq_save(flags);
Johannes Weinerc9019e92018-01-31 16:16:37 -08006789 __count_memcg_events(ug->memcg, PGPGOUT, ug->pgpgout);
Johannes Weiner9f762db2020-06-03 16:01:44 -07006790 __this_cpu_add(ug->memcg->vmstats_percpu->nr_page_events, ug->nr_pages);
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006791 memcg_check_events(ug->memcg, ug->dummy_page);
Johannes Weiner747db952014-08-08 14:19:24 -07006792 local_irq_restore(flags);
Michal Hockof1796542020-09-04 16:35:24 -07006793
6794 /* drop reference from uncharge_page */
6795 css_put(&ug->memcg->css);
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006796}
6797
6798static void uncharge_page(struct page *page, struct uncharge_gather *ug)
6799{
Johannes Weiner9f762db2020-06-03 16:01:44 -07006800 unsigned long nr_pages;
6801
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006802 VM_BUG_ON_PAGE(PageLRU(page), page);
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006803
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08006804 if (!page_memcg(page))
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006805 return;
6806
6807 /*
6808 * Nobody should be changing or seriously looking at
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08006809 * page_memcg(page) at this point, we have fully
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006810 * exclusive access to the page.
6811 */
6812
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08006813 if (ug->memcg != page_memcg(page)) {
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006814 if (ug->memcg) {
6815 uncharge_batch(ug);
6816 uncharge_gather_clear(ug);
6817 }
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08006818 ug->memcg = page_memcg(page);
Michal Hockof1796542020-09-04 16:35:24 -07006819
6820 /* pairs with css_put in uncharge_batch */
6821 css_get(&ug->memcg->css);
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006822 }
6823
Johannes Weiner9f762db2020-06-03 16:01:44 -07006824 nr_pages = compound_nr(page);
6825 ug->nr_pages += nr_pages;
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006826
Roman Gushchin18b2db32020-12-01 13:58:30 -08006827 if (PageMemcgKmem(page))
Johannes Weiner9f762db2020-06-03 16:01:44 -07006828 ug->nr_kmem += nr_pages;
Roman Gushchin18b2db32020-12-01 13:58:30 -08006829 else
6830 ug->pgpgout++;
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006831
6832 ug->dummy_page = page;
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08006833 page->memcg_data = 0;
Johannes Weiner1a3e1f42020-08-06 23:20:45 -07006834 css_put(&ug->memcg->css);
Johannes Weiner747db952014-08-08 14:19:24 -07006835}
6836
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006837/**
6838 * mem_cgroup_uncharge - uncharge a page
6839 * @page: page to uncharge
6840 *
Johannes Weinerf0e45fb2020-06-03 16:02:07 -07006841 * Uncharge a page previously charged with mem_cgroup_charge().
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006842 */
6843void mem_cgroup_uncharge(struct page *page)
6844{
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006845 struct uncharge_gather ug;
6846
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006847 if (mem_cgroup_disabled())
6848 return;
6849
Johannes Weiner747db952014-08-08 14:19:24 -07006850 /* Don't touch page->lru of any random page, pre-check: */
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08006851 if (!page_memcg(page))
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006852 return;
6853
Jérôme Glissea9d5ade2017-09-08 16:11:50 -07006854 uncharge_gather_clear(&ug);
6855 uncharge_page(page, &ug);
6856 uncharge_batch(&ug);
Johannes Weiner747db952014-08-08 14:19:24 -07006857}
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006858
Johannes Weiner747db952014-08-08 14:19:24 -07006859/**
6860 * mem_cgroup_uncharge_list - uncharge a list of page
6861 * @page_list: list of pages to uncharge
6862 *
6863 * Uncharge a list of pages previously charged with
Johannes Weinerf0e45fb2020-06-03 16:02:07 -07006864 * mem_cgroup_charge().
Johannes Weiner747db952014-08-08 14:19:24 -07006865 */
6866void mem_cgroup_uncharge_list(struct list_head *page_list)
6867{
Muchun Songc41a40b2021-02-24 12:04:08 -08006868 struct uncharge_gather ug;
6869 struct page *page;
6870
Johannes Weiner747db952014-08-08 14:19:24 -07006871 if (mem_cgroup_disabled())
6872 return;
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006873
Muchun Songc41a40b2021-02-24 12:04:08 -08006874 uncharge_gather_clear(&ug);
6875 list_for_each_entry(page, page_list, lru)
6876 uncharge_page(page, &ug);
6877 if (ug.memcg)
6878 uncharge_batch(&ug);
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006879}
6880
6881/**
Johannes Weiner6a93ca82016-03-15 14:57:19 -07006882 * mem_cgroup_migrate - charge a page's replacement
6883 * @oldpage: currently circulating page
6884 * @newpage: replacement page
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006885 *
Johannes Weiner6a93ca82016-03-15 14:57:19 -07006886 * Charge @newpage as a replacement page for @oldpage. @oldpage will
6887 * be uncharged upon free.
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006888 *
6889 * Both pages must be locked, @newpage->mapping must be set up.
6890 */
Johannes Weiner6a93ca82016-03-15 14:57:19 -07006891void mem_cgroup_migrate(struct page *oldpage, struct page *newpage)
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006892{
Johannes Weiner29833312014-12-10 15:44:02 -08006893 struct mem_cgroup *memcg;
Johannes Weiner44b7a8d2016-01-20 15:03:16 -08006894 unsigned int nr_pages;
Tejun Heod93c4132016-06-24 14:49:54 -07006895 unsigned long flags;
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006896
6897 VM_BUG_ON_PAGE(!PageLocked(oldpage), oldpage);
6898 VM_BUG_ON_PAGE(!PageLocked(newpage), newpage);
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006899 VM_BUG_ON_PAGE(PageAnon(oldpage) != PageAnon(newpage), newpage);
Johannes Weiner6abb5a82014-08-08 14:19:33 -07006900 VM_BUG_ON_PAGE(PageTransHuge(oldpage) != PageTransHuge(newpage),
6901 newpage);
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006902
6903 if (mem_cgroup_disabled())
6904 return;
6905
6906 /* Page cache replacement: new page already charged? */
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08006907 if (page_memcg(newpage))
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006908 return;
6909
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08006910 memcg = page_memcg(oldpage);
Alex Shia4055882020-12-18 14:01:31 -08006911 VM_WARN_ON_ONCE_PAGE(!memcg, oldpage);
Johannes Weiner29833312014-12-10 15:44:02 -08006912 if (!memcg)
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006913 return;
6914
Johannes Weiner44b7a8d2016-01-20 15:03:16 -08006915 /* Force-charge the new page. The old one will be freed soon */
Matthew Wilcox (Oracle)6c357842020-08-14 17:30:37 -07006916 nr_pages = thp_nr_pages(newpage);
Johannes Weiner44b7a8d2016-01-20 15:03:16 -08006917
6918 page_counter_charge(&memcg->memory, nr_pages);
6919 if (do_memsw_account())
6920 page_counter_charge(&memcg->memsw, nr_pages);
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006921
Johannes Weiner1a3e1f42020-08-06 23:20:45 -07006922 css_get(&memcg->css);
Johannes Weinerd9eb1ea2020-06-03 16:02:24 -07006923 commit_charge(newpage, memcg);
Johannes Weiner44b7a8d2016-01-20 15:03:16 -08006924
Tejun Heod93c4132016-06-24 14:49:54 -07006925 local_irq_save(flags);
Johannes Weiner3fba69a2020-06-03 16:01:31 -07006926 mem_cgroup_charge_statistics(memcg, newpage, nr_pages);
Johannes Weiner44b7a8d2016-01-20 15:03:16 -08006927 memcg_check_events(memcg, newpage);
Tejun Heod93c4132016-06-24 14:49:54 -07006928 local_irq_restore(flags);
Johannes Weiner0a31bc92014-08-08 14:19:22 -07006929}
6930
Johannes Weineref129472016-01-14 15:21:34 -08006931DEFINE_STATIC_KEY_FALSE(memcg_sockets_enabled_key);
Johannes Weiner11092082016-01-14 15:21:26 -08006932EXPORT_SYMBOL(memcg_sockets_enabled_key);
6933
Johannes Weiner2d758072016-10-07 17:00:58 -07006934void mem_cgroup_sk_alloc(struct sock *sk)
Johannes Weiner11092082016-01-14 15:21:26 -08006935{
6936 struct mem_cgroup *memcg;
6937
Johannes Weiner2d758072016-10-07 17:00:58 -07006938 if (!mem_cgroup_sockets_enabled)
6939 return;
6940
Shakeel Butte876ecc2020-03-09 22:16:05 -07006941 /* Do not associate the sock with unrelated interrupted task's memcg. */
6942 if (in_interrupt())
6943 return;
6944
Johannes Weiner11092082016-01-14 15:21:26 -08006945 rcu_read_lock();
6946 memcg = mem_cgroup_from_task(current);
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08006947 if (memcg == root_mem_cgroup)
6948 goto out;
Johannes Weiner0db15292016-01-20 15:02:50 -08006949 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && !memcg->tcpmem_active)
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08006950 goto out;
Shakeel Butt8965aa22020-04-01 21:07:10 -07006951 if (css_tryget(&memcg->css))
Johannes Weiner11092082016-01-14 15:21:26 -08006952 sk->sk_memcg = memcg;
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08006953out:
Johannes Weiner11092082016-01-14 15:21:26 -08006954 rcu_read_unlock();
6955}
Johannes Weiner11092082016-01-14 15:21:26 -08006956
Johannes Weiner2d758072016-10-07 17:00:58 -07006957void mem_cgroup_sk_free(struct sock *sk)
Johannes Weiner11092082016-01-14 15:21:26 -08006958{
Johannes Weiner2d758072016-10-07 17:00:58 -07006959 if (sk->sk_memcg)
6960 css_put(&sk->sk_memcg->css);
Johannes Weiner11092082016-01-14 15:21:26 -08006961}
6962
6963/**
6964 * mem_cgroup_charge_skmem - charge socket memory
6965 * @memcg: memcg to charge
6966 * @nr_pages: number of pages to charge
6967 *
6968 * Charges @nr_pages to @memcg. Returns %true if the charge fit within
6969 * @memcg's configured limit, %false if the charge had to be forced.
6970 */
6971bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
6972{
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08006973 gfp_t gfp_mask = GFP_KERNEL;
Johannes Weiner11092082016-01-14 15:21:26 -08006974
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08006975 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
Johannes Weiner0db15292016-01-20 15:02:50 -08006976 struct page_counter *fail;
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08006977
Johannes Weiner0db15292016-01-20 15:02:50 -08006978 if (page_counter_try_charge(&memcg->tcpmem, nr_pages, &fail)) {
6979 memcg->tcpmem_pressure = 0;
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08006980 return true;
6981 }
Johannes Weiner0db15292016-01-20 15:02:50 -08006982 page_counter_charge(&memcg->tcpmem, nr_pages);
6983 memcg->tcpmem_pressure = 1;
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08006984 return false;
Johannes Weiner11092082016-01-14 15:21:26 -08006985 }
Johannes Weinerd886f4e2016-01-20 15:02:47 -08006986
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08006987 /* Don't block in the packet receive path */
6988 if (in_softirq())
6989 gfp_mask = GFP_NOWAIT;
6990
Johannes Weinerc9019e92018-01-31 16:16:37 -08006991 mod_memcg_state(memcg, MEMCG_SOCK, nr_pages);
Johannes Weinerb2807f02016-01-20 15:03:22 -08006992
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08006993 if (try_charge(memcg, gfp_mask, nr_pages) == 0)
6994 return true;
6995
6996 try_charge(memcg, gfp_mask|__GFP_NOFAIL, nr_pages);
Johannes Weiner11092082016-01-14 15:21:26 -08006997 return false;
6998}
6999
7000/**
7001 * mem_cgroup_uncharge_skmem - uncharge socket memory
Mike Rapoportb7701a52018-02-06 15:42:13 -08007002 * @memcg: memcg to uncharge
7003 * @nr_pages: number of pages to uncharge
Johannes Weiner11092082016-01-14 15:21:26 -08007004 */
7005void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
7006{
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08007007 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
Johannes Weiner0db15292016-01-20 15:02:50 -08007008 page_counter_uncharge(&memcg->tcpmem, nr_pages);
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08007009 return;
7010 }
Johannes Weinerd886f4e2016-01-20 15:02:47 -08007011
Johannes Weinerc9019e92018-01-31 16:16:37 -08007012 mod_memcg_state(memcg, MEMCG_SOCK, -nr_pages);
Johannes Weinerb2807f02016-01-20 15:03:22 -08007013
Roman Gushchin475d0482017-09-08 16:13:09 -07007014 refill_stock(memcg, nr_pages);
Johannes Weiner11092082016-01-14 15:21:26 -08007015}
7016
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08007017static int __init cgroup_memory(char *s)
7018{
7019 char *token;
7020
7021 while ((token = strsep(&s, ",")) != NULL) {
7022 if (!*token)
7023 continue;
7024 if (!strcmp(token, "nosocket"))
7025 cgroup_memory_nosocket = true;
Vladimir Davydov04823c82016-01-20 15:02:38 -08007026 if (!strcmp(token, "nokmem"))
7027 cgroup_memory_nokmem = true;
Johannes Weinerf7e1cb62016-01-14 15:21:29 -08007028 }
7029 return 0;
7030}
7031__setup("cgroup.memory=", cgroup_memory);
Johannes Weiner11092082016-01-14 15:21:26 -08007032
Michal Hocko2d110852013-02-22 16:34:43 -08007033/*
Michal Hocko10813122013-02-22 16:35:41 -08007034 * subsys_initcall() for memory controller.
7035 *
Sebastian Andrzej Siewior308167f2016-11-03 15:49:59 +01007036 * Some parts like memcg_hotplug_cpu_dead() have to be initialized from this
7037 * context because of lock dependencies (cgroup_lock -> cpu hotplug) but
7038 * basically everything that doesn't depend on a specific mem_cgroup structure
7039 * should be initialized from here.
Michal Hocko2d110852013-02-22 16:34:43 -08007040 */
7041static int __init mem_cgroup_init(void)
7042{
Johannes Weiner95a045f2015-02-11 15:26:33 -08007043 int cpu, node;
7044
Muchun Songf3344ad2021-02-24 12:03:15 -08007045 /*
7046 * Currently s32 type (can refer to struct batched_lruvec_stat) is
7047 * used for per-memcg-per-cpu caching of per-node statistics. In order
7048 * to work fine, we should make sure that the overfill threshold can't
7049 * exceed S32_MAX / PAGE_SIZE.
7050 */
7051 BUILD_BUG_ON(MEMCG_CHARGE_BATCH > S32_MAX / PAGE_SIZE);
7052
Sebastian Andrzej Siewior308167f2016-11-03 15:49:59 +01007053 cpuhp_setup_state_nocalls(CPUHP_MM_MEMCQ_DEAD, "mm/memctrl:dead", NULL,
7054 memcg_hotplug_cpu_dead);
Johannes Weiner95a045f2015-02-11 15:26:33 -08007055
7056 for_each_possible_cpu(cpu)
7057 INIT_WORK(&per_cpu_ptr(&memcg_stock, cpu)->work,
7058 drain_local_stock);
7059
7060 for_each_node(node) {
7061 struct mem_cgroup_tree_per_node *rtpn;
Johannes Weiner95a045f2015-02-11 15:26:33 -08007062
7063 rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL,
7064 node_online(node) ? node : NUMA_NO_NODE);
7065
Mel Gormanef8f2322016-07-28 15:46:05 -07007066 rtpn->rb_root = RB_ROOT;
Davidlohr Buesofa90b2f2017-09-08 16:15:21 -07007067 rtpn->rb_rightmost = NULL;
Mel Gormanef8f2322016-07-28 15:46:05 -07007068 spin_lock_init(&rtpn->lock);
Johannes Weiner95a045f2015-02-11 15:26:33 -08007069 soft_limit_tree.rb_tree_per_node[node] = rtpn;
7070 }
7071
Michal Hocko2d110852013-02-22 16:34:43 -08007072 return 0;
7073}
7074subsys_initcall(mem_cgroup_init);
Johannes Weiner21afa382015-02-11 15:26:36 -08007075
7076#ifdef CONFIG_MEMCG_SWAP
Arnd Bergmann358c07f2016-08-25 15:17:08 -07007077static struct mem_cgroup *mem_cgroup_id_get_online(struct mem_cgroup *memcg)
7078{
Kirill Tkhai1c2d4792018-10-26 15:09:28 -07007079 while (!refcount_inc_not_zero(&memcg->id.ref)) {
Arnd Bergmann358c07f2016-08-25 15:17:08 -07007080 /*
7081 * The root cgroup cannot be destroyed, so it's refcount must
7082 * always be >= 1.
7083 */
7084 if (WARN_ON_ONCE(memcg == root_mem_cgroup)) {
7085 VM_BUG_ON(1);
7086 break;
7087 }
7088 memcg = parent_mem_cgroup(memcg);
7089 if (!memcg)
7090 memcg = root_mem_cgroup;
7091 }
7092 return memcg;
7093}
7094
Johannes Weiner21afa382015-02-11 15:26:36 -08007095/**
7096 * mem_cgroup_swapout - transfer a memsw charge to swap
7097 * @page: page whose memsw charge to transfer
7098 * @entry: swap entry to move the charge to
7099 *
7100 * Transfer the memsw charge of @page to @entry.
7101 */
7102void mem_cgroup_swapout(struct page *page, swp_entry_t entry)
7103{
Vladimir Davydov1f47b612016-08-11 15:33:00 -07007104 struct mem_cgroup *memcg, *swap_memcg;
Huang Yingd6810d72017-09-06 16:22:45 -07007105 unsigned int nr_entries;
Johannes Weiner21afa382015-02-11 15:26:36 -08007106 unsigned short oldid;
7107
7108 VM_BUG_ON_PAGE(PageLRU(page), page);
7109 VM_BUG_ON_PAGE(page_count(page), page);
7110
Alex Shi76358ab2020-12-18 14:01:28 -08007111 if (mem_cgroup_disabled())
7112 return;
7113
Johannes Weiner2d1c4982020-06-03 16:02:14 -07007114 if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
Johannes Weiner21afa382015-02-11 15:26:36 -08007115 return;
7116
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08007117 memcg = page_memcg(page);
Johannes Weiner21afa382015-02-11 15:26:36 -08007118
Alex Shia4055882020-12-18 14:01:31 -08007119 VM_WARN_ON_ONCE_PAGE(!memcg, page);
Johannes Weiner21afa382015-02-11 15:26:36 -08007120 if (!memcg)
7121 return;
7122
Vladimir Davydov1f47b612016-08-11 15:33:00 -07007123 /*
7124 * In case the memcg owning these pages has been offlined and doesn't
7125 * have an ID allocated to it anymore, charge the closest online
7126 * ancestor for the swap instead and transfer the memory+swap charge.
7127 */
7128 swap_memcg = mem_cgroup_id_get_online(memcg);
Matthew Wilcox (Oracle)6c357842020-08-14 17:30:37 -07007129 nr_entries = thp_nr_pages(page);
Huang Yingd6810d72017-09-06 16:22:45 -07007130 /* Get references for the tail pages, too */
7131 if (nr_entries > 1)
7132 mem_cgroup_id_get_many(swap_memcg, nr_entries - 1);
7133 oldid = swap_cgroup_record(entry, mem_cgroup_id(swap_memcg),
7134 nr_entries);
Johannes Weiner21afa382015-02-11 15:26:36 -08007135 VM_BUG_ON_PAGE(oldid, page);
Johannes Weinerc9019e92018-01-31 16:16:37 -08007136 mod_memcg_state(swap_memcg, MEMCG_SWAP, nr_entries);
Johannes Weiner21afa382015-02-11 15:26:36 -08007137
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08007138 page->memcg_data = 0;
Johannes Weiner21afa382015-02-11 15:26:36 -08007139
7140 if (!mem_cgroup_is_root(memcg))
Huang Yingd6810d72017-09-06 16:22:45 -07007141 page_counter_uncharge(&memcg->memory, nr_entries);
Johannes Weiner21afa382015-02-11 15:26:36 -08007142
Johannes Weiner2d1c4982020-06-03 16:02:14 -07007143 if (!cgroup_memory_noswap && memcg != swap_memcg) {
Vladimir Davydov1f47b612016-08-11 15:33:00 -07007144 if (!mem_cgroup_is_root(swap_memcg))
Huang Yingd6810d72017-09-06 16:22:45 -07007145 page_counter_charge(&swap_memcg->memsw, nr_entries);
7146 page_counter_uncharge(&memcg->memsw, nr_entries);
Vladimir Davydov1f47b612016-08-11 15:33:00 -07007147 }
7148
Sebastian Andrzej Siewiorce9ce662015-09-04 15:47:50 -07007149 /*
7150 * Interrupts should be disabled here because the caller holds the
Matthew Wilcoxb93b0162018-04-10 16:36:56 -07007151 * i_pages lock which is taken with interrupts-off. It is
Sebastian Andrzej Siewiorce9ce662015-09-04 15:47:50 -07007152 * important here to have the interrupts disabled because it is the
Matthew Wilcoxb93b0162018-04-10 16:36:56 -07007153 * only synchronisation we have for updating the per-CPU variables.
Sebastian Andrzej Siewiorce9ce662015-09-04 15:47:50 -07007154 */
7155 VM_BUG_ON(!irqs_disabled());
Johannes Weiner3fba69a2020-06-03 16:01:31 -07007156 mem_cgroup_charge_statistics(memcg, page, -nr_entries);
Johannes Weiner21afa382015-02-11 15:26:36 -08007157 memcg_check_events(memcg, page);
Johannes Weiner73f576c2016-07-20 15:44:57 -07007158
Johannes Weiner1a3e1f42020-08-06 23:20:45 -07007159 css_put(&memcg->css);
Johannes Weiner21afa382015-02-11 15:26:36 -08007160}
7161
Huang Ying38d8b4e2017-07-06 15:37:18 -07007162/**
7163 * mem_cgroup_try_charge_swap - try charging swap space for a page
Vladimir Davydov37e84352016-01-20 15:02:56 -08007164 * @page: page being added to swap
7165 * @entry: swap entry to charge
7166 *
Huang Ying38d8b4e2017-07-06 15:37:18 -07007167 * Try to charge @page's memcg for the swap space at @entry.
Vladimir Davydov37e84352016-01-20 15:02:56 -08007168 *
7169 * Returns 0 on success, -ENOMEM on failure.
7170 */
7171int mem_cgroup_try_charge_swap(struct page *page, swp_entry_t entry)
7172{
Matthew Wilcox (Oracle)6c357842020-08-14 17:30:37 -07007173 unsigned int nr_pages = thp_nr_pages(page);
Vladimir Davydov37e84352016-01-20 15:02:56 -08007174 struct page_counter *counter;
Huang Ying38d8b4e2017-07-06 15:37:18 -07007175 struct mem_cgroup *memcg;
Vladimir Davydov37e84352016-01-20 15:02:56 -08007176 unsigned short oldid;
7177
Alex Shi76358ab2020-12-18 14:01:28 -08007178 if (mem_cgroup_disabled())
7179 return 0;
7180
Johannes Weiner2d1c4982020-06-03 16:02:14 -07007181 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys))
Vladimir Davydov37e84352016-01-20 15:02:56 -08007182 return 0;
7183
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08007184 memcg = page_memcg(page);
Vladimir Davydov37e84352016-01-20 15:02:56 -08007185
Alex Shia4055882020-12-18 14:01:31 -08007186 VM_WARN_ON_ONCE_PAGE(!memcg, page);
Vladimir Davydov37e84352016-01-20 15:02:56 -08007187 if (!memcg)
7188 return 0;
7189
Tejun Heof3a53a32018-06-07 17:05:35 -07007190 if (!entry.val) {
7191 memcg_memory_event(memcg, MEMCG_SWAP_FAIL);
Tejun Heobb98f2c2018-06-07 17:05:31 -07007192 return 0;
Tejun Heof3a53a32018-06-07 17:05:35 -07007193 }
Tejun Heobb98f2c2018-06-07 17:05:31 -07007194
Vladimir Davydov1f47b612016-08-11 15:33:00 -07007195 memcg = mem_cgroup_id_get_online(memcg);
Vladimir Davydov37e84352016-01-20 15:02:56 -08007196
Johannes Weiner2d1c4982020-06-03 16:02:14 -07007197 if (!cgroup_memory_noswap && !mem_cgroup_is_root(memcg) &&
Huang Ying38d8b4e2017-07-06 15:37:18 -07007198 !page_counter_try_charge(&memcg->swap, nr_pages, &counter)) {
Tejun Heof3a53a32018-06-07 17:05:35 -07007199 memcg_memory_event(memcg, MEMCG_SWAP_MAX);
7200 memcg_memory_event(memcg, MEMCG_SWAP_FAIL);
Vladimir Davydov1f47b612016-08-11 15:33:00 -07007201 mem_cgroup_id_put(memcg);
7202 return -ENOMEM;
7203 }
7204
Huang Ying38d8b4e2017-07-06 15:37:18 -07007205 /* Get references for the tail pages, too */
7206 if (nr_pages > 1)
7207 mem_cgroup_id_get_many(memcg, nr_pages - 1);
7208 oldid = swap_cgroup_record(entry, mem_cgroup_id(memcg), nr_pages);
Vladimir Davydov37e84352016-01-20 15:02:56 -08007209 VM_BUG_ON_PAGE(oldid, page);
Johannes Weinerc9019e92018-01-31 16:16:37 -08007210 mod_memcg_state(memcg, MEMCG_SWAP, nr_pages);
Vladimir Davydov37e84352016-01-20 15:02:56 -08007211
Vladimir Davydov37e84352016-01-20 15:02:56 -08007212 return 0;
7213}
7214
Johannes Weiner21afa382015-02-11 15:26:36 -08007215/**
Huang Ying38d8b4e2017-07-06 15:37:18 -07007216 * mem_cgroup_uncharge_swap - uncharge swap space
Johannes Weiner21afa382015-02-11 15:26:36 -08007217 * @entry: swap entry to uncharge
Huang Ying38d8b4e2017-07-06 15:37:18 -07007218 * @nr_pages: the amount of swap space to uncharge
Johannes Weiner21afa382015-02-11 15:26:36 -08007219 */
Huang Ying38d8b4e2017-07-06 15:37:18 -07007220void mem_cgroup_uncharge_swap(swp_entry_t entry, unsigned int nr_pages)
Johannes Weiner21afa382015-02-11 15:26:36 -08007221{
7222 struct mem_cgroup *memcg;
7223 unsigned short id;
7224
Huang Ying38d8b4e2017-07-06 15:37:18 -07007225 id = swap_cgroup_record(entry, 0, nr_pages);
Johannes Weiner21afa382015-02-11 15:26:36 -08007226 rcu_read_lock();
Vladimir Davydovadbe4272015-04-15 16:13:00 -07007227 memcg = mem_cgroup_from_id(id);
Johannes Weiner21afa382015-02-11 15:26:36 -08007228 if (memcg) {
Johannes Weiner2d1c4982020-06-03 16:02:14 -07007229 if (!cgroup_memory_noswap && !mem_cgroup_is_root(memcg)) {
Vladimir Davydov37e84352016-01-20 15:02:56 -08007230 if (cgroup_subsys_on_dfl(memory_cgrp_subsys))
Huang Ying38d8b4e2017-07-06 15:37:18 -07007231 page_counter_uncharge(&memcg->swap, nr_pages);
Vladimir Davydov37e84352016-01-20 15:02:56 -08007232 else
Huang Ying38d8b4e2017-07-06 15:37:18 -07007233 page_counter_uncharge(&memcg->memsw, nr_pages);
Vladimir Davydov37e84352016-01-20 15:02:56 -08007234 }
Johannes Weinerc9019e92018-01-31 16:16:37 -08007235 mod_memcg_state(memcg, MEMCG_SWAP, -nr_pages);
Huang Ying38d8b4e2017-07-06 15:37:18 -07007236 mem_cgroup_id_put_many(memcg, nr_pages);
Johannes Weiner21afa382015-02-11 15:26:36 -08007237 }
7238 rcu_read_unlock();
7239}
7240
Vladimir Davydovd8b38432016-01-20 15:03:07 -08007241long mem_cgroup_get_nr_swap_pages(struct mem_cgroup *memcg)
7242{
7243 long nr_swap_pages = get_nr_swap_pages();
7244
Johannes Weinereccb52e2020-06-03 16:02:11 -07007245 if (cgroup_memory_noswap || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
Vladimir Davydovd8b38432016-01-20 15:03:07 -08007246 return nr_swap_pages;
7247 for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg))
7248 nr_swap_pages = min_t(long, nr_swap_pages,
Roman Gushchinbbec2e12018-06-07 17:06:18 -07007249 READ_ONCE(memcg->swap.max) -
Vladimir Davydovd8b38432016-01-20 15:03:07 -08007250 page_counter_read(&memcg->swap));
7251 return nr_swap_pages;
7252}
7253
Vladimir Davydov5ccc5ab2016-01-20 15:03:10 -08007254bool mem_cgroup_swap_full(struct page *page)
7255{
7256 struct mem_cgroup *memcg;
7257
7258 VM_BUG_ON_PAGE(!PageLocked(page), page);
7259
7260 if (vm_swap_full())
7261 return true;
Johannes Weinereccb52e2020-06-03 16:02:11 -07007262 if (cgroup_memory_noswap || !cgroup_subsys_on_dfl(memory_cgrp_subsys))
Vladimir Davydov5ccc5ab2016-01-20 15:03:10 -08007263 return false;
7264
Roman Gushchinbcfe06b2020-12-01 13:58:27 -08007265 memcg = page_memcg(page);
Vladimir Davydov5ccc5ab2016-01-20 15:03:10 -08007266 if (!memcg)
7267 return false;
7268
Jakub Kicinski4b82ab42020-06-01 21:49:52 -07007269 for (; memcg != root_mem_cgroup; memcg = parent_mem_cgroup(memcg)) {
7270 unsigned long usage = page_counter_read(&memcg->swap);
7271
7272 if (usage * 2 >= READ_ONCE(memcg->swap.high) ||
7273 usage * 2 >= READ_ONCE(memcg->swap.max))
Vladimir Davydov5ccc5ab2016-01-20 15:03:10 -08007274 return true;
Jakub Kicinski4b82ab42020-06-01 21:49:52 -07007275 }
Vladimir Davydov5ccc5ab2016-01-20 15:03:10 -08007276
7277 return false;
7278}
7279
Johannes Weinereccb52e2020-06-03 16:02:11 -07007280static int __init setup_swap_account(char *s)
Johannes Weiner21afa382015-02-11 15:26:36 -08007281{
7282 if (!strcmp(s, "1"))
Kaixu Xia5ab92902020-12-14 19:07:07 -08007283 cgroup_memory_noswap = false;
Johannes Weiner21afa382015-02-11 15:26:36 -08007284 else if (!strcmp(s, "0"))
Kaixu Xia5ab92902020-12-14 19:07:07 -08007285 cgroup_memory_noswap = true;
Johannes Weiner21afa382015-02-11 15:26:36 -08007286 return 1;
7287}
Johannes Weinereccb52e2020-06-03 16:02:11 -07007288__setup("swapaccount=", setup_swap_account);
Johannes Weiner21afa382015-02-11 15:26:36 -08007289
Vladimir Davydov37e84352016-01-20 15:02:56 -08007290static u64 swap_current_read(struct cgroup_subsys_state *css,
7291 struct cftype *cft)
7292{
7293 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
7294
7295 return (u64)page_counter_read(&memcg->swap) * PAGE_SIZE;
7296}
7297
Jakub Kicinski4b82ab42020-06-01 21:49:52 -07007298static int swap_high_show(struct seq_file *m, void *v)
7299{
7300 return seq_puts_memcg_tunable(m,
7301 READ_ONCE(mem_cgroup_from_seq(m)->swap.high));
7302}
7303
7304static ssize_t swap_high_write(struct kernfs_open_file *of,
7305 char *buf, size_t nbytes, loff_t off)
7306{
7307 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
7308 unsigned long high;
7309 int err;
7310
7311 buf = strstrip(buf);
7312 err = page_counter_memparse(buf, "max", &high);
7313 if (err)
7314 return err;
7315
7316 page_counter_set_high(&memcg->swap, high);
7317
7318 return nbytes;
7319}
7320
Vladimir Davydov37e84352016-01-20 15:02:56 -08007321static int swap_max_show(struct seq_file *m, void *v)
7322{
Chris Down677dc972019-03-05 15:45:55 -08007323 return seq_puts_memcg_tunable(m,
7324 READ_ONCE(mem_cgroup_from_seq(m)->swap.max));
Vladimir Davydov37e84352016-01-20 15:02:56 -08007325}
7326
7327static ssize_t swap_max_write(struct kernfs_open_file *of,
7328 char *buf, size_t nbytes, loff_t off)
7329{
7330 struct mem_cgroup *memcg = mem_cgroup_from_css(of_css(of));
7331 unsigned long max;
7332 int err;
7333
7334 buf = strstrip(buf);
7335 err = page_counter_memparse(buf, "max", &max);
7336 if (err)
7337 return err;
7338
Tejun Heobe091022018-06-07 17:09:21 -07007339 xchg(&memcg->swap.max, max);
Vladimir Davydov37e84352016-01-20 15:02:56 -08007340
7341 return nbytes;
7342}
7343
Tejun Heof3a53a32018-06-07 17:05:35 -07007344static int swap_events_show(struct seq_file *m, void *v)
7345{
Chris Downaa9694b2019-03-05 15:45:52 -08007346 struct mem_cgroup *memcg = mem_cgroup_from_seq(m);
Tejun Heof3a53a32018-06-07 17:05:35 -07007347
Jakub Kicinski4b82ab42020-06-01 21:49:52 -07007348 seq_printf(m, "high %lu\n",
7349 atomic_long_read(&memcg->memory_events[MEMCG_SWAP_HIGH]));
Tejun Heof3a53a32018-06-07 17:05:35 -07007350 seq_printf(m, "max %lu\n",
7351 atomic_long_read(&memcg->memory_events[MEMCG_SWAP_MAX]));
7352 seq_printf(m, "fail %lu\n",
7353 atomic_long_read(&memcg->memory_events[MEMCG_SWAP_FAIL]));
7354
7355 return 0;
7356}
7357
Vladimir Davydov37e84352016-01-20 15:02:56 -08007358static struct cftype swap_files[] = {
7359 {
7360 .name = "swap.current",
7361 .flags = CFTYPE_NOT_ON_ROOT,
7362 .read_u64 = swap_current_read,
7363 },
7364 {
Jakub Kicinski4b82ab42020-06-01 21:49:52 -07007365 .name = "swap.high",
7366 .flags = CFTYPE_NOT_ON_ROOT,
7367 .seq_show = swap_high_show,
7368 .write = swap_high_write,
7369 },
7370 {
Vladimir Davydov37e84352016-01-20 15:02:56 -08007371 .name = "swap.max",
7372 .flags = CFTYPE_NOT_ON_ROOT,
7373 .seq_show = swap_max_show,
7374 .write = swap_max_write,
7375 },
Tejun Heof3a53a32018-06-07 17:05:35 -07007376 {
7377 .name = "swap.events",
7378 .flags = CFTYPE_NOT_ON_ROOT,
7379 .file_offset = offsetof(struct mem_cgroup, swap_events_file),
7380 .seq_show = swap_events_show,
7381 },
Vladimir Davydov37e84352016-01-20 15:02:56 -08007382 { } /* terminate */
7383};
7384
Johannes Weinereccb52e2020-06-03 16:02:11 -07007385static struct cftype memsw_files[] = {
Johannes Weiner21afa382015-02-11 15:26:36 -08007386 {
7387 .name = "memsw.usage_in_bytes",
7388 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
7389 .read_u64 = mem_cgroup_read_u64,
7390 },
7391 {
7392 .name = "memsw.max_usage_in_bytes",
7393 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
7394 .write = mem_cgroup_reset,
7395 .read_u64 = mem_cgroup_read_u64,
7396 },
7397 {
7398 .name = "memsw.limit_in_bytes",
7399 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
7400 .write = mem_cgroup_write,
7401 .read_u64 = mem_cgroup_read_u64,
7402 },
7403 {
7404 .name = "memsw.failcnt",
7405 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
7406 .write = mem_cgroup_reset,
7407 .read_u64 = mem_cgroup_read_u64,
7408 },
7409 { }, /* terminate */
7410};
7411
Bhupesh Sharma82ff1652020-07-23 21:15:21 -07007412/*
7413 * If mem_cgroup_swap_init() is implemented as a subsys_initcall()
7414 * instead of a core_initcall(), this could mean cgroup_memory_noswap still
7415 * remains set to false even when memcg is disabled via "cgroup_disable=memory"
7416 * boot parameter. This may result in premature OOPS inside
7417 * mem_cgroup_get_nr_swap_pages() function in corner cases.
7418 */
Johannes Weiner21afa382015-02-11 15:26:36 -08007419static int __init mem_cgroup_swap_init(void)
7420{
Johannes Weiner2d1c4982020-06-03 16:02:14 -07007421 /* No memory control -> no swap control */
7422 if (mem_cgroup_disabled())
7423 cgroup_memory_noswap = true;
7424
7425 if (cgroup_memory_noswap)
Johannes Weinereccb52e2020-06-03 16:02:11 -07007426 return 0;
7427
7428 WARN_ON(cgroup_add_dfl_cftypes(&memory_cgrp_subsys, swap_files));
7429 WARN_ON(cgroup_add_legacy_cftypes(&memory_cgrp_subsys, memsw_files));
7430
Johannes Weiner21afa382015-02-11 15:26:36 -08007431 return 0;
7432}
Bhupesh Sharma82ff1652020-07-23 21:15:21 -07007433core_initcall(mem_cgroup_swap_init);
Johannes Weiner21afa382015-02-11 15:26:36 -08007434
7435#endif /* CONFIG_MEMCG_SWAP */