blob: d99b71bc2c667a512541d33894ec073ff1bb6c91 [file] [log] [blame]
Balbir Singh8cdea7c2008-02-07 00:13:50 -08001/* memcontrol.h - Memory Controller
2 *
3 * Copyright IBM Corporation, 2007
4 * Author Balbir Singh <balbir@linux.vnet.ibm.com>
5 *
Pavel Emelianov78fb7462008-02-07 00:13:51 -08006 * Copyright 2007 OpenVZ SWsoft Inc
7 * Author: Pavel Emelianov <xemul@openvz.org>
8 *
Balbir Singh8cdea7c2008-02-07 00:13:50 -08009 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License as published by
11 * the Free Software Foundation; either version 2 of the License, or
12 * (at your option) any later version.
13 *
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU General Public License for more details.
18 */
19
20#ifndef _LINUX_MEMCONTROL_H
21#define _LINUX_MEMCONTROL_H
Hirokazu Takahashif8d665422009-01-07 18:08:02 -080022#include <linux/cgroup.h>
Ying Han456f9982011-05-26 16:25:38 -070023#include <linux/vm_event_item.h>
Glauber Costa7ae1e1d2012-12-18 14:21:56 -080024#include <linux/hardirq.h>
Glauber Costaa8964b92012-12-18 14:22:09 -080025#include <linux/jump_label.h>
Michal Hocko33398cf2015-09-08 15:01:02 -070026#include <linux/page_counter.h>
27#include <linux/vmpressure.h>
28#include <linux/eventfd.h>
Johannes Weiner00f3ca22017-07-06 15:40:52 -070029#include <linux/mm.h>
30#include <linux/vmstat.h>
Michal Hocko33398cf2015-09-08 15:01:02 -070031#include <linux/writeback.h>
Johannes Weinerfdf1cdb2016-03-15 14:57:25 -070032#include <linux/page-flags.h>
Ying Han456f9982011-05-26 16:25:38 -070033
Pavel Emelianov78fb7462008-02-07 00:13:51 -080034struct mem_cgroup;
Balbir Singh8697d332008-02-07 00:13:59 -080035struct page;
36struct mm_struct;
Glauber Costa2633d7a2012-12-18 14:22:34 -080037struct kmem_cache;
Pavel Emelianov78fb7462008-02-07 00:13:51 -080038
Johannes Weiner71cd3112017-05-03 14:55:13 -070039/* Cgroup-specific page state, on top of universal node page state */
40enum memcg_stat_item {
41 MEMCG_CACHE = NR_VM_NODE_STAT_ITEMS,
42 MEMCG_RSS,
43 MEMCG_RSS_HUGE,
44 MEMCG_SWAP,
45 MEMCG_SOCK,
46 /* XXX: why are these zone and not node counters? */
47 MEMCG_KERNEL_STACK_KB,
Johannes Weinerb2807f02016-01-20 15:03:22 -080048 MEMCG_NR_STAT,
Greg Thelen2a7106f2011-01-13 15:47:37 -080049};
50
Johannes Weinere27be242018-04-10 16:29:45 -070051enum memcg_memory_event {
52 MEMCG_LOW,
Johannes Weiner71cd3112017-05-03 14:55:13 -070053 MEMCG_HIGH,
54 MEMCG_MAX,
55 MEMCG_OOM,
Johannes Weinere27be242018-04-10 16:29:45 -070056 MEMCG_NR_MEMORY_EVENTS,
Johannes Weiner71cd3112017-05-03 14:55:13 -070057};
58
Johannes Weiner56600482012-01-12 17:17:59 -080059struct mem_cgroup_reclaim_cookie {
Mel Gormanef8f2322016-07-28 15:46:05 -070060 pg_data_t *pgdat;
Johannes Weiner56600482012-01-12 17:17:59 -080061 int priority;
62 unsigned int generation;
63};
64
Johannes Weiner71cd3112017-05-03 14:55:13 -070065#ifdef CONFIG_MEMCG
66
67#define MEM_CGROUP_ID_SHIFT 16
68#define MEM_CGROUP_ID_MAX USHRT_MAX
69
70struct mem_cgroup_id {
71 int id;
72 atomic_t ref;
73};
74
Michal Hocko33398cf2015-09-08 15:01:02 -070075/*
76 * Per memcg event counter is incremented at every pagein/pageout. With THP,
77 * it will be incremated by the number of pages. This counter is used for
78 * for trigger some periodic events. This is straightforward and better
79 * than using jiffies etc. to handle periodic memcg event.
80 */
81enum mem_cgroup_events_target {
82 MEM_CGROUP_TARGET_THRESH,
83 MEM_CGROUP_TARGET_SOFTLIMIT,
84 MEM_CGROUP_TARGET_NUMAINFO,
85 MEM_CGROUP_NTARGETS,
86};
87
Michal Hocko33398cf2015-09-08 15:01:02 -070088struct mem_cgroup_stat_cpu {
Johannes Weinerb2807f02016-01-20 15:03:22 -080089 long count[MEMCG_NR_STAT];
Johannes Weinere27be242018-04-10 16:29:45 -070090 unsigned long events[NR_VM_EVENT_ITEMS];
Michal Hocko33398cf2015-09-08 15:01:02 -070091 unsigned long nr_page_events;
92 unsigned long targets[MEM_CGROUP_NTARGETS];
93};
94
95struct mem_cgroup_reclaim_iter {
96 struct mem_cgroup *position;
97 /* scan generation, increased every round-trip */
98 unsigned int generation;
99};
100
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700101struct lruvec_stat {
102 long count[NR_VM_NODE_STAT_ITEMS];
103};
104
Michal Hocko33398cf2015-09-08 15:01:02 -0700105/*
106 * per-zone information in memory controller.
107 */
Mel Gormanef8f2322016-07-28 15:46:05 -0700108struct mem_cgroup_per_node {
Michal Hocko33398cf2015-09-08 15:01:02 -0700109 struct lruvec lruvec;
Johannes Weinera983b5e2018-01-31 16:16:45 -0800110
111 struct lruvec_stat __percpu *lruvec_stat_cpu;
112 atomic_long_t lruvec_stat[NR_VM_NODE_STAT_ITEMS];
113
Michal Hockob4536f0c82017-01-10 16:58:04 -0800114 unsigned long lru_zone_size[MAX_NR_ZONES][NR_LRU_LISTS];
Michal Hocko33398cf2015-09-08 15:01:02 -0700115
116 struct mem_cgroup_reclaim_iter iter[DEF_PRIORITY + 1];
117
118 struct rb_node tree_node; /* RB tree node */
119 unsigned long usage_in_excess;/* Set to the value by which */
120 /* the soft limit is exceeded*/
121 bool on_tree;
Andrey Ryabinine3c1ac52018-04-10 16:28:03 -0700122 bool congested; /* memcg has many dirty pages */
123 /* backed by a congested BDI */
124
Michal Hocko33398cf2015-09-08 15:01:02 -0700125 struct mem_cgroup *memcg; /* Back pointer, we cannot */
126 /* use container_of */
127};
128
Michal Hocko33398cf2015-09-08 15:01:02 -0700129struct mem_cgroup_threshold {
130 struct eventfd_ctx *eventfd;
131 unsigned long threshold;
132};
133
134/* For threshold */
135struct mem_cgroup_threshold_ary {
136 /* An array index points to threshold just below or equal to usage. */
137 int current_threshold;
138 /* Size of entries[] */
139 unsigned int size;
140 /* Array of thresholds */
141 struct mem_cgroup_threshold entries[0];
142};
143
144struct mem_cgroup_thresholds {
145 /* Primary thresholds array */
146 struct mem_cgroup_threshold_ary *primary;
147 /*
148 * Spare threshold array.
149 * This is needed to make mem_cgroup_unregister_event() "never fail".
150 * It must be able to store at least primary->size - 1 entries.
151 */
152 struct mem_cgroup_threshold_ary *spare;
153};
154
Johannes Weiner567e9ab2016-01-20 15:02:24 -0800155enum memcg_kmem_state {
156 KMEM_NONE,
157 KMEM_ALLOCATED,
158 KMEM_ONLINE,
159};
160
Michal Hocko33398cf2015-09-08 15:01:02 -0700161/*
162 * The memory controller data structure. The memory controller controls both
163 * page cache and RSS per cgroup. We would eventually like to provide
164 * statistics based on the statistics developed by Rik Van Riel for clock-pro,
165 * to help the administrator determine what knobs to tune.
166 */
167struct mem_cgroup {
168 struct cgroup_subsys_state css;
169
Johannes Weiner73f576c2016-07-20 15:44:57 -0700170 /* Private memcg ID. Used to ID objects that outlive the cgroup */
171 struct mem_cgroup_id id;
172
Michal Hocko33398cf2015-09-08 15:01:02 -0700173 /* Accounted resources */
174 struct page_counter memory;
Vladimir Davydov37e84352016-01-20 15:02:56 -0800175 struct page_counter swap;
Johannes Weiner0db15292016-01-20 15:02:50 -0800176
177 /* Legacy consumer-oriented counters */
Michal Hocko33398cf2015-09-08 15:01:02 -0700178 struct page_counter memsw;
179 struct page_counter kmem;
Johannes Weiner0db15292016-01-20 15:02:50 -0800180 struct page_counter tcpmem;
Michal Hocko33398cf2015-09-08 15:01:02 -0700181
182 /* Normal memory consumption range */
183 unsigned long low;
184 unsigned long high;
185
Johannes Weinerf7e1cb62016-01-14 15:21:29 -0800186 /* Range enforcement for interrupt charges */
187 struct work_struct high_work;
188
Michal Hocko33398cf2015-09-08 15:01:02 -0700189 unsigned long soft_limit;
190
191 /* vmpressure notifications */
192 struct vmpressure vmpressure;
193
Michal Hocko33398cf2015-09-08 15:01:02 -0700194 /*
195 * Should the accounting and control be hierarchical, per subtree?
196 */
197 bool use_hierarchy;
198
199 /* protected by memcg_oom_lock */
200 bool oom_lock;
201 int under_oom;
202
203 int swappiness;
204 /* OOM-Killer disable */
205 int oom_kill_disable;
206
Johannes Weinere27be242018-04-10 16:29:45 -0700207 /* memory.events */
208 atomic_long_t memory_events[MEMCG_NR_MEMORY_EVENTS];
Tejun Heo472912a2015-09-18 18:01:59 -0400209 struct cgroup_file events_file;
210
Michal Hocko33398cf2015-09-08 15:01:02 -0700211 /* protect arrays of thresholds */
212 struct mutex thresholds_lock;
213
214 /* thresholds for memory usage. RCU-protected */
215 struct mem_cgroup_thresholds thresholds;
216
217 /* thresholds for mem+swap usage. RCU-protected */
218 struct mem_cgroup_thresholds memsw_thresholds;
219
220 /* For oom notifier event fd */
221 struct list_head oom_notify;
222
223 /*
224 * Should we move charges of a task when a task is moved into this
225 * mem_cgroup ? And what type of charges should we move ?
226 */
227 unsigned long move_charge_at_immigrate;
228 /*
229 * set > 0 if pages under this cgroup are moving to other cgroup.
230 */
231 atomic_t moving_account;
232 /* taken only while moving_account > 0 */
233 spinlock_t move_lock;
234 struct task_struct *move_lock_task;
235 unsigned long move_lock_flags;
Johannes Weinera983b5e2018-01-31 16:16:45 -0800236
Johannes Weinere27be242018-04-10 16:29:45 -0700237 /* memory.stat */
Johannes Weinera983b5e2018-01-31 16:16:45 -0800238 struct mem_cgroup_stat_cpu __percpu *stat_cpu;
239 atomic_long_t stat[MEMCG_NR_STAT];
Johannes Weinere27be242018-04-10 16:29:45 -0700240 atomic_long_t events[NR_VM_EVENT_ITEMS];
Michal Hocko33398cf2015-09-08 15:01:02 -0700241
Johannes Weinerd886f4e2016-01-20 15:02:47 -0800242 unsigned long socket_pressure;
243
244 /* Legacy tcp memory accounting */
Johannes Weiner0db15292016-01-20 15:02:50 -0800245 bool tcpmem_active;
246 int tcpmem_pressure;
Johannes Weinerd886f4e2016-01-20 15:02:47 -0800247
Johannes Weiner127424c2016-01-20 15:02:32 -0800248#ifndef CONFIG_SLOB
Michal Hocko33398cf2015-09-08 15:01:02 -0700249 /* Index in the kmem_cache->memcg_params.memcg_caches array */
250 int kmemcg_id;
Johannes Weiner567e9ab2016-01-20 15:02:24 -0800251 enum memcg_kmem_state kmem_state;
Tejun Heobc2791f2017-02-22 15:41:21 -0800252 struct list_head kmem_caches;
Michal Hocko33398cf2015-09-08 15:01:02 -0700253#endif
254
255 int last_scanned_node;
256#if MAX_NUMNODES > 1
257 nodemask_t scan_nodes;
258 atomic_t numainfo_events;
259 atomic_t numainfo_updating;
260#endif
261
262#ifdef CONFIG_CGROUP_WRITEBACK
263 struct list_head cgwb_list;
264 struct wb_domain cgwb_domain;
265#endif
266
267 /* List of events which userspace want to receive */
268 struct list_head event_list;
269 spinlock_t event_list_lock;
270
271 struct mem_cgroup_per_node *nodeinfo[0];
272 /* WARNING: nodeinfo must be the last member here */
273};
Johannes Weiner7d828602016-01-14 15:20:56 -0800274
Johannes Weinera983b5e2018-01-31 16:16:45 -0800275/*
276 * size of first charge trial. "32" comes from vmscan.c's magic value.
277 * TODO: maybe necessary to use big numbers in big irons.
278 */
279#define MEMCG_CHARGE_BATCH 32U
280
Johannes Weiner7d828602016-01-14 15:20:56 -0800281extern struct mem_cgroup *root_mem_cgroup;
Tejun Heo56161632015-05-22 17:13:20 -0400282
Johannes Weiner23047a92016-03-15 14:57:16 -0700283static inline bool mem_cgroup_disabled(void)
284{
285 return !cgroup_subsys_enabled(memory_cgrp_subsys);
286}
287
Johannes Weiner241994ed2015-02-11 15:26:06 -0800288bool mem_cgroup_low(struct mem_cgroup *root, struct mem_cgroup *memcg);
289
Johannes Weiner00501b52014-08-08 14:19:20 -0700290int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm,
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -0800291 gfp_t gfp_mask, struct mem_cgroup **memcgp,
292 bool compound);
Johannes Weiner00501b52014-08-08 14:19:20 -0700293void mem_cgroup_commit_charge(struct page *page, struct mem_cgroup *memcg,
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -0800294 bool lrucare, bool compound);
295void mem_cgroup_cancel_charge(struct page *page, struct mem_cgroup *memcg,
296 bool compound);
Johannes Weiner0a31bc92014-08-08 14:19:22 -0700297void mem_cgroup_uncharge(struct page *page);
Johannes Weiner747db952014-08-08 14:19:24 -0700298void mem_cgroup_uncharge_list(struct list_head *page_list);
Johannes Weiner0a31bc92014-08-08 14:19:22 -0700299
Johannes Weiner6a93ca82016-03-15 14:57:19 -0700300void mem_cgroup_migrate(struct page *oldpage, struct page *newpage);
Johannes Weiner0a31bc92014-08-08 14:19:22 -0700301
Mel Gormanef8f2322016-07-28 15:46:05 -0700302static struct mem_cgroup_per_node *
303mem_cgroup_nodeinfo(struct mem_cgroup *memcg, int nid)
Johannes Weiner55779ec2016-07-28 15:45:10 -0700304{
Mel Gormanef8f2322016-07-28 15:46:05 -0700305 return memcg->nodeinfo[nid];
Johannes Weiner55779ec2016-07-28 15:45:10 -0700306}
307
308/**
Mel Gormana9dd0a82016-07-28 15:46:02 -0700309 * mem_cgroup_lruvec - get the lru list vector for a node or a memcg zone
310 * @node: node of the wanted lruvec
Johannes Weiner55779ec2016-07-28 15:45:10 -0700311 * @memcg: memcg of the wanted lruvec
312 *
Mel Gormana9dd0a82016-07-28 15:46:02 -0700313 * Returns the lru list vector holding pages for a given @node or a given
314 * @memcg and @zone. This can be the node lruvec, if the memory controller
Johannes Weiner55779ec2016-07-28 15:45:10 -0700315 * is disabled.
316 */
Mel Gormana9dd0a82016-07-28 15:46:02 -0700317static inline struct lruvec *mem_cgroup_lruvec(struct pglist_data *pgdat,
Mel Gormanef8f2322016-07-28 15:46:05 -0700318 struct mem_cgroup *memcg)
Johannes Weiner55779ec2016-07-28 15:45:10 -0700319{
Mel Gormanef8f2322016-07-28 15:46:05 -0700320 struct mem_cgroup_per_node *mz;
Johannes Weiner55779ec2016-07-28 15:45:10 -0700321 struct lruvec *lruvec;
322
323 if (mem_cgroup_disabled()) {
Mel Gormana9dd0a82016-07-28 15:46:02 -0700324 lruvec = node_lruvec(pgdat);
Johannes Weiner55779ec2016-07-28 15:45:10 -0700325 goto out;
326 }
327
Mel Gormanef8f2322016-07-28 15:46:05 -0700328 mz = mem_cgroup_nodeinfo(memcg, pgdat->node_id);
Johannes Weiner55779ec2016-07-28 15:45:10 -0700329 lruvec = &mz->lruvec;
330out:
331 /*
332 * Since a node can be onlined after the mem_cgroup was created,
Mel Gorman599d0c92016-07-28 15:45:31 -0700333 * we have to be prepared to initialize lruvec->pgdat here;
Johannes Weiner55779ec2016-07-28 15:45:10 -0700334 * and if offlined then reonlined, we need to reinitialize it.
335 */
Mel Gormanef8f2322016-07-28 15:46:05 -0700336 if (unlikely(lruvec->pgdat != pgdat))
337 lruvec->pgdat = pgdat;
Johannes Weiner55779ec2016-07-28 15:45:10 -0700338 return lruvec;
339}
340
Mel Gorman599d0c92016-07-28 15:45:31 -0700341struct lruvec *mem_cgroup_page_lruvec(struct page *, struct pglist_data *);
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -0800342
Johannes Weiner2314b422014-12-10 15:44:33 -0800343bool task_in_mem_cgroup(struct task_struct *task, struct mem_cgroup *memcg);
Michal Hocko64219992015-09-08 15:01:07 -0700344struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p);
Vladimir Davydove993d902015-09-09 15:35:35 -0700345
Michal Hocko33398cf2015-09-08 15:01:02 -0700346static inline
347struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *css){
348 return css ? container_of(css, struct mem_cgroup, css) : NULL;
349}
350
Johannes Weiner8e8ae642016-01-14 15:21:32 -0800351#define mem_cgroup_from_counter(counter, member) \
352 container_of(counter, struct mem_cgroup, member)
353
Michal Hocko33398cf2015-09-08 15:01:02 -0700354struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *,
355 struct mem_cgroup *,
356 struct mem_cgroup_reclaim_cookie *);
357void mem_cgroup_iter_break(struct mem_cgroup *, struct mem_cgroup *);
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -0700358int mem_cgroup_scan_tasks(struct mem_cgroup *,
359 int (*)(struct task_struct *, void *), void *);
Michal Hocko33398cf2015-09-08 15:01:02 -0700360
Johannes Weiner23047a92016-03-15 14:57:16 -0700361static inline unsigned short mem_cgroup_id(struct mem_cgroup *memcg)
362{
363 if (mem_cgroup_disabled())
364 return 0;
365
Johannes Weiner73f576c2016-07-20 15:44:57 -0700366 return memcg->id.id;
Johannes Weiner23047a92016-03-15 14:57:16 -0700367}
Johannes Weiner73f576c2016-07-20 15:44:57 -0700368struct mem_cgroup *mem_cgroup_from_id(unsigned short id);
Johannes Weiner23047a92016-03-15 14:57:16 -0700369
Roman Gushchin22621852017-07-06 15:40:25 -0700370static inline struct mem_cgroup *lruvec_memcg(struct lruvec *lruvec)
371{
372 struct mem_cgroup_per_node *mz;
373
374 if (mem_cgroup_disabled())
375 return NULL;
376
377 mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
378 return mz->memcg;
379}
380
Johannes Weiner8e8ae642016-01-14 15:21:32 -0800381/**
382 * parent_mem_cgroup - find the accounting parent of a memcg
383 * @memcg: memcg whose parent to find
384 *
385 * Returns the parent memcg, or NULL if this is the root or the memory
386 * controller is in legacy no-hierarchy mode.
387 */
388static inline struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg)
389{
390 if (!memcg->memory.parent)
391 return NULL;
392 return mem_cgroup_from_counter(memcg->memory.parent, memory);
393}
394
Michal Hocko33398cf2015-09-08 15:01:02 -0700395static inline bool mem_cgroup_is_descendant(struct mem_cgroup *memcg,
396 struct mem_cgroup *root)
397{
398 if (root == memcg)
399 return true;
400 if (!root->use_hierarchy)
401 return false;
402 return cgroup_is_descendant(memcg->css.cgroup, root->css.cgroup);
403}
Glauber Costae1aab162011-12-11 21:47:03 +0000404
Johannes Weiner2314b422014-12-10 15:44:33 -0800405static inline bool mm_match_cgroup(struct mm_struct *mm,
406 struct mem_cgroup *memcg)
Lai Jiangshan2e4d4092009-01-07 18:08:07 -0800407{
Johannes Weiner587af302012-10-08 16:34:12 -0700408 struct mem_cgroup *task_memcg;
Johannes Weiner413918b2014-12-10 15:44:30 -0800409 bool match = false;
Johannes Weinerc3ac9a82012-05-29 15:06:25 -0700410
Lai Jiangshan2e4d4092009-01-07 18:08:07 -0800411 rcu_read_lock();
Johannes Weiner587af302012-10-08 16:34:12 -0700412 task_memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
Johannes Weiner413918b2014-12-10 15:44:30 -0800413 if (task_memcg)
Johannes Weiner2314b422014-12-10 15:44:33 -0800414 match = mem_cgroup_is_descendant(task_memcg, memcg);
Lai Jiangshan2e4d4092009-01-07 18:08:07 -0800415 rcu_read_unlock();
Johannes Weinerc3ac9a82012-05-29 15:06:25 -0700416 return match;
Lai Jiangshan2e4d4092009-01-07 18:08:07 -0800417}
Balbir Singh8a9f3cc2008-02-07 00:13:53 -0800418
Michal Hocko64219992015-09-08 15:01:07 -0700419struct cgroup_subsys_state *mem_cgroup_css_from_page(struct page *page);
Vladimir Davydov2fc04522015-09-09 15:35:28 -0700420ino_t page_cgroup_ino(struct page *page);
Wu Fengguangd3242362009-12-16 12:19:59 +0100421
Vladimir Davydoveb01aaa2016-01-20 15:03:02 -0800422static inline bool mem_cgroup_online(struct mem_cgroup *memcg)
423{
424 if (mem_cgroup_disabled())
425 return true;
426 return !!(memcg->css.flags & CSS_ONLINE);
427}
428
KAMEZAWA Hiroyuki58ae83d2008-02-07 00:14:32 -0800429/*
430 * For memory reclaim.
431 */
Ying Han889976d2011-05-26 16:25:33 -0700432int mem_cgroup_select_victim_node(struct mem_cgroup *memcg);
Michal Hocko33398cf2015-09-08 15:01:02 -0700433
434void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
Michal Hockob4536f0c82017-01-10 16:58:04 -0800435 int zid, int nr_pages);
Michal Hocko33398cf2015-09-08 15:01:02 -0700436
Vladimir Davydov0a6b76d2016-03-17 14:18:42 -0700437unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
438 int nid, unsigned int lru_mask);
439
Michal Hocko33398cf2015-09-08 15:01:02 -0700440static inline
441unsigned long mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru)
442{
Mel Gormanef8f2322016-07-28 15:46:05 -0700443 struct mem_cgroup_per_node *mz;
Michal Hockob4536f0c82017-01-10 16:58:04 -0800444 unsigned long nr_pages = 0;
445 int zid;
Michal Hocko33398cf2015-09-08 15:01:02 -0700446
Mel Gormanef8f2322016-07-28 15:46:05 -0700447 mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
Michal Hockob4536f0c82017-01-10 16:58:04 -0800448 for (zid = 0; zid < MAX_NR_ZONES; zid++)
449 nr_pages += mz->lru_zone_size[zid][lru];
450 return nr_pages;
451}
452
453static inline
454unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec,
455 enum lru_list lru, int zone_idx)
456{
457 struct mem_cgroup_per_node *mz;
458
459 mz = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
460 return mz->lru_zone_size[zone_idx][lru];
Michal Hocko33398cf2015-09-08 15:01:02 -0700461}
462
Tejun Heob23afb92015-11-05 18:46:11 -0800463void mem_cgroup_handle_over_high(void);
464
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -0700465unsigned long mem_cgroup_get_limit(struct mem_cgroup *memcg);
466
Michal Hocko64219992015-09-08 15:01:07 -0700467void mem_cgroup_print_oom_info(struct mem_cgroup *memcg,
468 struct task_struct *p);
KAMEZAWA Hiroyuki58ae83d2008-02-07 00:14:32 -0800469
Johannes Weiner49426422013-10-16 13:46:59 -0700470static inline void mem_cgroup_oom_enable(void)
Johannes Weiner519e5242013-09-12 15:13:42 -0700471{
Tejun Heo626ebc42015-11-05 18:46:09 -0800472 WARN_ON(current->memcg_may_oom);
473 current->memcg_may_oom = 1;
Johannes Weiner519e5242013-09-12 15:13:42 -0700474}
475
Johannes Weiner49426422013-10-16 13:46:59 -0700476static inline void mem_cgroup_oom_disable(void)
Johannes Weiner519e5242013-09-12 15:13:42 -0700477{
Tejun Heo626ebc42015-11-05 18:46:09 -0800478 WARN_ON(!current->memcg_may_oom);
479 current->memcg_may_oom = 0;
Johannes Weiner519e5242013-09-12 15:13:42 -0700480}
481
Johannes Weiner3812c8c2013-09-12 15:13:44 -0700482static inline bool task_in_memcg_oom(struct task_struct *p)
483{
Tejun Heo626ebc42015-11-05 18:46:09 -0800484 return p->memcg_in_oom;
Johannes Weiner3812c8c2013-09-12 15:13:44 -0700485}
486
Johannes Weiner49426422013-10-16 13:46:59 -0700487bool mem_cgroup_oom_synchronize(bool wait);
Johannes Weiner3812c8c2013-09-12 15:13:44 -0700488
Andrew Mortonc255a452012-07-31 16:43:02 -0700489#ifdef CONFIG_MEMCG_SWAP
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -0800490extern int do_swap_account;
491#endif
Hirokazu Takahashif8d665422009-01-07 18:08:02 -0800492
Johannes Weiner739f79f2017-08-18 15:15:48 -0700493struct mem_cgroup *lock_page_memcg(struct page *page);
494void __unlock_page_memcg(struct mem_cgroup *memcg);
Johannes Weiner62cccb82016-03-15 14:57:22 -0700495void unlock_page_memcg(struct page *page);
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -0700496
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -0700497/* idx can be of type enum memcg_stat_item or node_stat_item */
Johannes Weinerccda7f42017-05-03 14:55:16 -0700498static inline unsigned long memcg_page_state(struct mem_cgroup *memcg,
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -0700499 int idx)
Johannes Weiner2a2e4882017-05-03 14:55:03 -0700500{
Johannes Weinera983b5e2018-01-31 16:16:45 -0800501 long x = atomic_long_read(&memcg->stat[idx]);
502#ifdef CONFIG_SMP
503 if (x < 0)
504 x = 0;
505#endif
506 return x;
Johannes Weiner2a2e4882017-05-03 14:55:03 -0700507}
508
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -0700509/* idx can be of type enum memcg_stat_item or node_stat_item */
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700510static inline void __mod_memcg_state(struct mem_cgroup *memcg,
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -0700511 int idx, int val)
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700512{
Johannes Weinera983b5e2018-01-31 16:16:45 -0800513 long x;
514
515 if (mem_cgroup_disabled())
516 return;
517
518 x = val + __this_cpu_read(memcg->stat_cpu->count[idx]);
519 if (unlikely(abs(x) > MEMCG_CHARGE_BATCH)) {
520 atomic_long_add(x, &memcg->stat[idx]);
521 x = 0;
522 }
523 __this_cpu_write(memcg->stat_cpu->count[idx], x);
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700524}
525
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -0700526/* idx can be of type enum memcg_stat_item or node_stat_item */
Johannes Weinerccda7f42017-05-03 14:55:16 -0700527static inline void mod_memcg_state(struct mem_cgroup *memcg,
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -0700528 int idx, int val)
Johannes Weiner2a2e4882017-05-03 14:55:03 -0700529{
Johannes Weinerc3cc3912018-02-21 14:45:24 -0800530 unsigned long flags;
531
532 local_irq_save(flags);
Johannes Weinera983b5e2018-01-31 16:16:45 -0800533 __mod_memcg_state(memcg, idx, val);
Johannes Weinerc3cc3912018-02-21 14:45:24 -0800534 local_irq_restore(flags);
Johannes Weiner2a2e4882017-05-03 14:55:03 -0700535}
536
Michal Hocko33398cf2015-09-08 15:01:02 -0700537/**
Johannes Weinerccda7f42017-05-03 14:55:16 -0700538 * mod_memcg_page_state - update page state statistics
Johannes Weiner62cccb82016-03-15 14:57:22 -0700539 * @page: the page
Michal Hocko33398cf2015-09-08 15:01:02 -0700540 * @idx: page state item to account
541 * @val: number of pages (positive or negative)
542 *
Johannes Weinerfdf1cdb2016-03-15 14:57:25 -0700543 * The @page must be locked or the caller must use lock_page_memcg()
544 * to prevent double accounting when the page is concurrently being
545 * moved to another memcg:
Johannes Weiner81f8c3a2016-03-15 14:57:04 -0700546 *
Johannes Weinerfdf1cdb2016-03-15 14:57:25 -0700547 * lock_page(page) or lock_page_memcg(page)
Johannes Weiner81f8c3a2016-03-15 14:57:04 -0700548 * if (TestClearPageState(page))
Johannes Weinerccda7f42017-05-03 14:55:16 -0700549 * mod_memcg_page_state(page, state, -1);
Johannes Weinerfdf1cdb2016-03-15 14:57:25 -0700550 * unlock_page(page) or unlock_page_memcg(page)
Johannes Weiner2a2e4882017-05-03 14:55:03 -0700551 *
552 * Kernel pages are an exception to this, since they'll never move.
Michal Hocko33398cf2015-09-08 15:01:02 -0700553 */
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700554static inline void __mod_memcg_page_state(struct page *page,
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -0700555 int idx, int val)
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700556{
557 if (page->mem_cgroup)
558 __mod_memcg_state(page->mem_cgroup, idx, val);
559}
560
Johannes Weinerccda7f42017-05-03 14:55:16 -0700561static inline void mod_memcg_page_state(struct page *page,
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -0700562 int idx, int val)
Michal Hocko33398cf2015-09-08 15:01:02 -0700563{
Johannes Weiner62cccb82016-03-15 14:57:22 -0700564 if (page->mem_cgroup)
Johannes Weinerccda7f42017-05-03 14:55:16 -0700565 mod_memcg_state(page->mem_cgroup, idx, val);
Michal Hocko33398cf2015-09-08 15:01:02 -0700566}
567
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700568static inline unsigned long lruvec_page_state(struct lruvec *lruvec,
569 enum node_stat_item idx)
Greg Thelen2a7106f2011-01-13 15:47:37 -0800570{
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700571 struct mem_cgroup_per_node *pn;
Johannes Weinera983b5e2018-01-31 16:16:45 -0800572 long x;
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700573
574 if (mem_cgroup_disabled())
575 return node_page_state(lruvec_pgdat(lruvec), idx);
576
577 pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
Johannes Weinera983b5e2018-01-31 16:16:45 -0800578 x = atomic_long_read(&pn->lruvec_stat[idx]);
579#ifdef CONFIG_SMP
580 if (x < 0)
581 x = 0;
582#endif
583 return x;
Greg Thelen2a7106f2011-01-13 15:47:37 -0800584}
585
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700586static inline void __mod_lruvec_state(struct lruvec *lruvec,
587 enum node_stat_item idx, int val)
Greg Thelen2a7106f2011-01-13 15:47:37 -0800588{
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700589 struct mem_cgroup_per_node *pn;
Johannes Weinera983b5e2018-01-31 16:16:45 -0800590 long x;
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700591
Johannes Weiner28454262018-01-31 16:16:41 -0800592 /* Update node */
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700593 __mod_node_page_state(lruvec_pgdat(lruvec), idx, val);
Johannes Weiner28454262018-01-31 16:16:41 -0800594
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700595 if (mem_cgroup_disabled())
596 return;
Johannes Weiner28454262018-01-31 16:16:41 -0800597
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700598 pn = container_of(lruvec, struct mem_cgroup_per_node, lruvec);
Johannes Weiner28454262018-01-31 16:16:41 -0800599
600 /* Update memcg */
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700601 __mod_memcg_state(pn->memcg, idx, val);
Johannes Weiner28454262018-01-31 16:16:41 -0800602
603 /* Update lruvec */
Johannes Weinera983b5e2018-01-31 16:16:45 -0800604 x = val + __this_cpu_read(pn->lruvec_stat_cpu->count[idx]);
605 if (unlikely(abs(x) > MEMCG_CHARGE_BATCH)) {
606 atomic_long_add(x, &pn->lruvec_stat[idx]);
607 x = 0;
608 }
609 __this_cpu_write(pn->lruvec_stat_cpu->count[idx], x);
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700610}
611
612static inline void mod_lruvec_state(struct lruvec *lruvec,
613 enum node_stat_item idx, int val)
614{
Johannes Weinerc3cc3912018-02-21 14:45:24 -0800615 unsigned long flags;
616
617 local_irq_save(flags);
Johannes Weiner28454262018-01-31 16:16:41 -0800618 __mod_lruvec_state(lruvec, idx, val);
Johannes Weinerc3cc3912018-02-21 14:45:24 -0800619 local_irq_restore(flags);
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700620}
621
622static inline void __mod_lruvec_page_state(struct page *page,
623 enum node_stat_item idx, int val)
624{
Johannes Weiner28454262018-01-31 16:16:41 -0800625 pg_data_t *pgdat = page_pgdat(page);
626 struct lruvec *lruvec;
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700627
Johannes Weiner28454262018-01-31 16:16:41 -0800628 /* Untracked pages have no memcg, no lruvec. Update only the node */
629 if (!page->mem_cgroup) {
630 __mod_node_page_state(pgdat, idx, val);
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700631 return;
Johannes Weiner28454262018-01-31 16:16:41 -0800632 }
633
634 lruvec = mem_cgroup_lruvec(pgdat, page->mem_cgroup);
635 __mod_lruvec_state(lruvec, idx, val);
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700636}
637
638static inline void mod_lruvec_page_state(struct page *page,
639 enum node_stat_item idx, int val)
640{
Johannes Weinerc3cc3912018-02-21 14:45:24 -0800641 unsigned long flags;
642
643 local_irq_save(flags);
Johannes Weiner28454262018-01-31 16:16:41 -0800644 __mod_lruvec_page_state(page, idx, val);
Johannes Weinerc3cc3912018-02-21 14:45:24 -0800645 local_irq_restore(flags);
Greg Thelen2a7106f2011-01-13 15:47:37 -0800646}
647
Mel Gormanef8f2322016-07-28 15:46:05 -0700648unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
Andrew Morton0608f432013-09-24 15:27:41 -0700649 gfp_t gfp_mask,
650 unsigned long *total_scanned);
David Rientjesa63d83f2010-08-09 17:19:46 -0700651
Johannes Weinerc9019e92018-01-31 16:16:37 -0800652static inline void __count_memcg_events(struct mem_cgroup *memcg,
Johannes Weinere27be242018-04-10 16:29:45 -0700653 enum vm_event_item idx,
654 unsigned long count)
Johannes Weinerc9019e92018-01-31 16:16:37 -0800655{
Johannes Weinera983b5e2018-01-31 16:16:45 -0800656 unsigned long x;
657
658 if (mem_cgroup_disabled())
659 return;
660
661 x = count + __this_cpu_read(memcg->stat_cpu->events[idx]);
662 if (unlikely(x > MEMCG_CHARGE_BATCH)) {
663 atomic_long_add(x, &memcg->events[idx]);
664 x = 0;
665 }
666 __this_cpu_write(memcg->stat_cpu->events[idx], x);
Johannes Weinerc9019e92018-01-31 16:16:37 -0800667}
668
Roman Gushchin22621852017-07-06 15:40:25 -0700669static inline void count_memcg_events(struct mem_cgroup *memcg,
Johannes Weinere27be242018-04-10 16:29:45 -0700670 enum vm_event_item idx,
671 unsigned long count)
Roman Gushchin22621852017-07-06 15:40:25 -0700672{
Johannes Weinerc3cc3912018-02-21 14:45:24 -0800673 unsigned long flags;
674
675 local_irq_save(flags);
Johannes Weinera983b5e2018-01-31 16:16:45 -0800676 __count_memcg_events(memcg, idx, count);
Johannes Weinerc3cc3912018-02-21 14:45:24 -0800677 local_irq_restore(flags);
Roman Gushchin22621852017-07-06 15:40:25 -0700678}
679
680static inline void count_memcg_page_event(struct page *page,
Johannes Weinere27be242018-04-10 16:29:45 -0700681 enum vm_event_item idx)
Roman Gushchin22621852017-07-06 15:40:25 -0700682{
683 if (page->mem_cgroup)
684 count_memcg_events(page->mem_cgroup, idx, 1);
685}
686
687static inline void count_memcg_event_mm(struct mm_struct *mm,
688 enum vm_event_item idx)
David Rientjes68ae5642012-12-12 13:51:57 -0800689{
Michal Hocko33398cf2015-09-08 15:01:02 -0700690 struct mem_cgroup *memcg;
691
David Rientjes68ae5642012-12-12 13:51:57 -0800692 if (mem_cgroup_disabled())
693 return;
Michal Hocko33398cf2015-09-08 15:01:02 -0700694
695 rcu_read_lock();
696 memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
Konstantin Khlebnikov8e675f72017-07-06 15:40:28 -0700697 if (likely(memcg)) {
Johannes Weinerc9019e92018-01-31 16:16:37 -0800698 count_memcg_events(memcg, idx, 1);
Konstantin Khlebnikov8e675f72017-07-06 15:40:28 -0700699 if (idx == OOM_KILL)
700 cgroup_file_notify(&memcg->events_file);
701 }
Michal Hocko33398cf2015-09-08 15:01:02 -0700702 rcu_read_unlock();
David Rientjes68ae5642012-12-12 13:51:57 -0800703}
Johannes Weinerc9019e92018-01-31 16:16:37 -0800704
Johannes Weinere27be242018-04-10 16:29:45 -0700705static inline void memcg_memory_event(struct mem_cgroup *memcg,
706 enum memcg_memory_event event)
Johannes Weinerc9019e92018-01-31 16:16:37 -0800707{
Johannes Weinere27be242018-04-10 16:29:45 -0700708 atomic_long_inc(&memcg->memory_events[event]);
Johannes Weinerc9019e92018-01-31 16:16:37 -0800709 cgroup_file_notify(&memcg->events_file);
710}
711
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -0800712#ifdef CONFIG_TRANSPARENT_HUGEPAGE
KAMEZAWA Hiroyukie94c8a92012-01-12 17:18:20 -0800713void mem_cgroup_split_huge_fixup(struct page *head);
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -0800714#endif
715
Andrew Mortonc255a452012-07-31 16:43:02 -0700716#else /* CONFIG_MEMCG */
Johannes Weiner23047a92016-03-15 14:57:16 -0700717
718#define MEM_CGROUP_ID_SHIFT 0
719#define MEM_CGROUP_ID_MAX 0
720
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -0800721struct mem_cgroup;
722
Johannes Weiner23047a92016-03-15 14:57:16 -0700723static inline bool mem_cgroup_disabled(void)
724{
725 return true;
726}
727
Johannes Weinere27be242018-04-10 16:29:45 -0700728static inline void memcg_memory_event(struct mem_cgroup *memcg,
729 enum memcg_memory_event event)
Johannes Weiner241994ed2015-02-11 15:26:06 -0800730{
731}
732
733static inline bool mem_cgroup_low(struct mem_cgroup *root,
734 struct mem_cgroup *memcg)
735{
736 return false;
737}
738
Johannes Weiner00501b52014-08-08 14:19:20 -0700739static inline int mem_cgroup_try_charge(struct page *page, struct mm_struct *mm,
740 gfp_t gfp_mask,
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -0800741 struct mem_cgroup **memcgp,
742 bool compound)
Balbir Singh8a9f3cc2008-02-07 00:13:53 -0800743{
Johannes Weiner00501b52014-08-08 14:19:20 -0700744 *memcgp = NULL;
Balbir Singh8a9f3cc2008-02-07 00:13:53 -0800745 return 0;
746}
747
Johannes Weiner00501b52014-08-08 14:19:20 -0700748static inline void mem_cgroup_commit_charge(struct page *page,
749 struct mem_cgroup *memcg,
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -0800750 bool lrucare, bool compound)
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -0800751{
752}
753
Johannes Weiner00501b52014-08-08 14:19:20 -0700754static inline void mem_cgroup_cancel_charge(struct page *page,
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -0800755 struct mem_cgroup *memcg,
756 bool compound)
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -0800757{
758}
759
Johannes Weiner0a31bc92014-08-08 14:19:22 -0700760static inline void mem_cgroup_uncharge(struct page *page)
761{
762}
763
Johannes Weiner747db952014-08-08 14:19:24 -0700764static inline void mem_cgroup_uncharge_list(struct list_head *page_list)
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -0800765{
766}
767
Johannes Weiner6a93ca82016-03-15 14:57:19 -0700768static inline void mem_cgroup_migrate(struct page *old, struct page *new)
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -0700769{
770}
771
Mel Gormana9dd0a82016-07-28 15:46:02 -0700772static inline struct lruvec *mem_cgroup_lruvec(struct pglist_data *pgdat,
Mel Gormanef8f2322016-07-28 15:46:05 -0700773 struct mem_cgroup *memcg)
Johannes Weiner925b7672012-01-12 17:18:15 -0800774{
Mel Gormana9dd0a82016-07-28 15:46:02 -0700775 return node_lruvec(pgdat);
Johannes Weiner925b7672012-01-12 17:18:15 -0800776}
777
Hugh Dickinsfa9add62012-05-29 15:07:09 -0700778static inline struct lruvec *mem_cgroup_page_lruvec(struct page *page,
Mel Gorman599d0c92016-07-28 15:45:31 -0700779 struct pglist_data *pgdat)
Minchan Kim3f58a822011-03-22 16:32:53 -0700780{
Mel Gorman599d0c92016-07-28 15:45:31 -0700781 return &pgdat->lruvec;
Balbir Singh66e17072008-02-07 00:13:56 -0800782}
783
Johannes Weiner587af302012-10-08 16:34:12 -0700784static inline bool mm_match_cgroup(struct mm_struct *mm,
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700785 struct mem_cgroup *memcg)
Balbir Singhbed71612008-02-07 00:14:01 -0800786{
Johannes Weiner587af302012-10-08 16:34:12 -0700787 return true;
Balbir Singhbed71612008-02-07 00:14:01 -0800788}
789
David Rientjesffbdccf2013-07-03 15:01:23 -0700790static inline bool task_in_mem_cgroup(struct task_struct *task,
791 const struct mem_cgroup *memcg)
David Rientjes4c4a2212008-02-07 00:14:06 -0800792{
David Rientjesffbdccf2013-07-03 15:01:23 -0700793 return true;
David Rientjes4c4a2212008-02-07 00:14:06 -0800794}
795
Johannes Weiner56600482012-01-12 17:17:59 -0800796static inline struct mem_cgroup *
797mem_cgroup_iter(struct mem_cgroup *root,
798 struct mem_cgroup *prev,
799 struct mem_cgroup_reclaim_cookie *reclaim)
800{
801 return NULL;
802}
803
804static inline void mem_cgroup_iter_break(struct mem_cgroup *root,
805 struct mem_cgroup *prev)
806{
807}
808
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -0700809static inline int mem_cgroup_scan_tasks(struct mem_cgroup *memcg,
810 int (*fn)(struct task_struct *, void *), void *arg)
811{
812 return 0;
813}
814
Johannes Weiner23047a92016-03-15 14:57:16 -0700815static inline unsigned short mem_cgroup_id(struct mem_cgroup *memcg)
Hirokazu Takahashif8d665422009-01-07 18:08:02 -0800816{
Johannes Weiner23047a92016-03-15 14:57:16 -0700817 return 0;
818}
819
820static inline struct mem_cgroup *mem_cgroup_from_id(unsigned short id)
821{
822 WARN_ON_ONCE(id);
823 /* XXX: This should always return root_mem_cgroup */
824 return NULL;
Hirokazu Takahashif8d665422009-01-07 18:08:02 -0800825}
KAMEZAWA Hiroyukia636b322009-01-07 18:08:08 -0800826
Roman Gushchin22621852017-07-06 15:40:25 -0700827static inline struct mem_cgroup *lruvec_memcg(struct lruvec *lruvec)
828{
829 return NULL;
830}
831
Vladimir Davydoveb01aaa2016-01-20 15:03:02 -0800832static inline bool mem_cgroup_online(struct mem_cgroup *memcg)
KOSAKI Motohiro14797e22009-01-07 18:08:18 -0800833{
Yaowei Bai13308ca2015-11-05 18:47:40 -0800834 return true;
KOSAKI Motohiro14797e22009-01-07 18:08:18 -0800835}
836
KOSAKI Motohiroa3d8e052009-01-07 18:08:19 -0800837static inline unsigned long
Hugh Dickins4d7dcca2012-05-29 15:07:08 -0700838mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru)
KOSAKI Motohiroa3d8e052009-01-07 18:08:19 -0800839{
840 return 0;
841}
Michal Hockob4536f0c82017-01-10 16:58:04 -0800842static inline
843unsigned long mem_cgroup_get_zone_lru_size(struct lruvec *lruvec,
844 enum lru_list lru, int zone_idx)
845{
846 return 0;
847}
KOSAKI Motohiroa3d8e052009-01-07 18:08:19 -0800848
Vladimir Davydov0a6b76d2016-03-17 14:18:42 -0700849static inline unsigned long
850mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
851 int nid, unsigned int lru_mask)
852{
853 return 0;
854}
855
Vladimir Davydov7c5f64f2016-10-07 16:57:23 -0700856static inline unsigned long mem_cgroup_get_limit(struct mem_cgroup *memcg)
857{
858 return 0;
859}
860
Balbir Singhe2224322009-04-02 16:57:39 -0700861static inline void
862mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
863{
864}
865
Johannes Weiner739f79f2017-08-18 15:15:48 -0700866static inline struct mem_cgroup *lock_page_memcg(struct page *page)
867{
868 return NULL;
869}
870
871static inline void __unlock_page_memcg(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -0700872{
873}
874
Johannes Weiner62cccb82016-03-15 14:57:22 -0700875static inline void unlock_page_memcg(struct page *page)
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -0700876{
877}
878
Tejun Heob23afb92015-11-05 18:46:11 -0800879static inline void mem_cgroup_handle_over_high(void)
880{
881}
882
Johannes Weiner49426422013-10-16 13:46:59 -0700883static inline void mem_cgroup_oom_enable(void)
Johannes Weiner519e5242013-09-12 15:13:42 -0700884{
885}
886
Johannes Weiner49426422013-10-16 13:46:59 -0700887static inline void mem_cgroup_oom_disable(void)
Johannes Weiner519e5242013-09-12 15:13:42 -0700888{
889}
890
Johannes Weiner3812c8c2013-09-12 15:13:44 -0700891static inline bool task_in_memcg_oom(struct task_struct *p)
892{
893 return false;
894}
895
Johannes Weiner49426422013-10-16 13:46:59 -0700896static inline bool mem_cgroup_oom_synchronize(bool wait)
Johannes Weiner3812c8c2013-09-12 15:13:44 -0700897{
898 return false;
899}
900
Johannes Weinerccda7f42017-05-03 14:55:16 -0700901static inline unsigned long memcg_page_state(struct mem_cgroup *memcg,
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -0700902 int idx)
Johannes Weiner2a2e4882017-05-03 14:55:03 -0700903{
904 return 0;
905}
906
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700907static inline void __mod_memcg_state(struct mem_cgroup *memcg,
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -0700908 int idx,
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700909 int nr)
910{
911}
912
Johannes Weinerccda7f42017-05-03 14:55:16 -0700913static inline void mod_memcg_state(struct mem_cgroup *memcg,
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -0700914 int idx,
Johannes Weinerccda7f42017-05-03 14:55:16 -0700915 int nr)
Johannes Weiner2a2e4882017-05-03 14:55:03 -0700916{
917}
918
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700919static inline void __mod_memcg_page_state(struct page *page,
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -0700920 int idx,
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700921 int nr)
Johannes Weiner2a2e4882017-05-03 14:55:03 -0700922{
923}
924
Johannes Weinerccda7f42017-05-03 14:55:16 -0700925static inline void mod_memcg_page_state(struct page *page,
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -0700926 int idx,
Johannes Weinerccda7f42017-05-03 14:55:16 -0700927 int nr)
Johannes Weiner553af432017-03-31 15:11:50 -0700928{
929}
930
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700931static inline unsigned long lruvec_page_state(struct lruvec *lruvec,
932 enum node_stat_item idx)
Greg Thelen2a7106f2011-01-13 15:47:37 -0800933{
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700934 return node_page_state(lruvec_pgdat(lruvec), idx);
Greg Thelen2a7106f2011-01-13 15:47:37 -0800935}
936
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700937static inline void __mod_lruvec_state(struct lruvec *lruvec,
938 enum node_stat_item idx, int val)
Balbir Singhd69b0422009-06-17 16:26:34 -0700939{
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700940 __mod_node_page_state(lruvec_pgdat(lruvec), idx, val);
941}
942
943static inline void mod_lruvec_state(struct lruvec *lruvec,
944 enum node_stat_item idx, int val)
945{
946 mod_node_page_state(lruvec_pgdat(lruvec), idx, val);
947}
948
949static inline void __mod_lruvec_page_state(struct page *page,
950 enum node_stat_item idx, int val)
951{
952 __mod_node_page_state(page_pgdat(page), idx, val);
953}
954
955static inline void mod_lruvec_page_state(struct page *page,
956 enum node_stat_item idx, int val)
957{
958 mod_node_page_state(page_pgdat(page), idx, val);
Balbir Singhd69b0422009-06-17 16:26:34 -0700959}
960
Balbir Singh4e416952009-09-23 15:56:39 -0700961static inline
Mel Gormanef8f2322016-07-28 15:46:05 -0700962unsigned long mem_cgroup_soft_limit_reclaim(pg_data_t *pgdat, int order,
Andrew Morton0608f432013-09-24 15:27:41 -0700963 gfp_t gfp_mask,
964 unsigned long *total_scanned)
Balbir Singh4e416952009-09-23 15:56:39 -0700965{
Andrew Morton0608f432013-09-24 15:27:41 -0700966 return 0;
Balbir Singh4e416952009-09-23 15:56:39 -0700967}
968
KAMEZAWA Hiroyukie94c8a92012-01-12 17:18:20 -0800969static inline void mem_cgroup_split_huge_fixup(struct page *head)
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -0800970{
971}
972
Roman Gushchin22621852017-07-06 15:40:25 -0700973static inline void count_memcg_events(struct mem_cgroup *memcg,
974 enum vm_event_item idx,
975 unsigned long count)
976{
977}
978
979static inline void count_memcg_page_event(struct page *page,
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -0700980 int idx)
Roman Gushchin22621852017-07-06 15:40:25 -0700981{
982}
983
Ying Han456f9982011-05-26 16:25:38 -0700984static inline
Roman Gushchin22621852017-07-06 15:40:25 -0700985void count_memcg_event_mm(struct mm_struct *mm, enum vm_event_item idx)
Ying Han456f9982011-05-26 16:25:38 -0700986{
987}
Andrew Mortonc255a452012-07-31 16:43:02 -0700988#endif /* CONFIG_MEMCG */
Pavel Emelianov78fb7462008-02-07 00:13:51 -0800989
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -0700990/* idx can be of type enum memcg_stat_item or node_stat_item */
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700991static inline void __inc_memcg_state(struct mem_cgroup *memcg,
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -0700992 int idx)
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700993{
994 __mod_memcg_state(memcg, idx, 1);
995}
996
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -0700997/* idx can be of type enum memcg_stat_item or node_stat_item */
Johannes Weiner00f3ca22017-07-06 15:40:52 -0700998static inline void __dec_memcg_state(struct mem_cgroup *memcg,
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -0700999 int idx)
Johannes Weiner00f3ca22017-07-06 15:40:52 -07001000{
1001 __mod_memcg_state(memcg, idx, -1);
1002}
1003
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -07001004/* idx can be of type enum memcg_stat_item or node_stat_item */
Johannes Weiner00f3ca22017-07-06 15:40:52 -07001005static inline void __inc_memcg_page_state(struct page *page,
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -07001006 int idx)
Johannes Weiner00f3ca22017-07-06 15:40:52 -07001007{
1008 __mod_memcg_page_state(page, idx, 1);
1009}
1010
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -07001011/* idx can be of type enum memcg_stat_item or node_stat_item */
Johannes Weiner00f3ca22017-07-06 15:40:52 -07001012static inline void __dec_memcg_page_state(struct page *page,
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -07001013 int idx)
Johannes Weiner00f3ca22017-07-06 15:40:52 -07001014{
1015 __mod_memcg_page_state(page, idx, -1);
1016}
1017
1018static inline void __inc_lruvec_state(struct lruvec *lruvec,
1019 enum node_stat_item idx)
1020{
1021 __mod_lruvec_state(lruvec, idx, 1);
1022}
1023
1024static inline void __dec_lruvec_state(struct lruvec *lruvec,
1025 enum node_stat_item idx)
1026{
1027 __mod_lruvec_state(lruvec, idx, -1);
1028}
1029
1030static inline void __inc_lruvec_page_state(struct page *page,
1031 enum node_stat_item idx)
1032{
1033 __mod_lruvec_page_state(page, idx, 1);
1034}
1035
1036static inline void __dec_lruvec_page_state(struct page *page,
1037 enum node_stat_item idx)
1038{
1039 __mod_lruvec_page_state(page, idx, -1);
1040}
1041
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -07001042/* idx can be of type enum memcg_stat_item or node_stat_item */
Johannes Weiner00f3ca22017-07-06 15:40:52 -07001043static inline void inc_memcg_state(struct mem_cgroup *memcg,
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -07001044 int idx)
Johannes Weiner00f3ca22017-07-06 15:40:52 -07001045{
1046 mod_memcg_state(memcg, idx, 1);
1047}
1048
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -07001049/* idx can be of type enum memcg_stat_item or node_stat_item */
Johannes Weiner00f3ca22017-07-06 15:40:52 -07001050static inline void dec_memcg_state(struct mem_cgroup *memcg,
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -07001051 int idx)
Johannes Weiner00f3ca22017-07-06 15:40:52 -07001052{
1053 mod_memcg_state(memcg, idx, -1);
1054}
1055
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -07001056/* idx can be of type enum memcg_stat_item or node_stat_item */
Johannes Weiner00f3ca22017-07-06 15:40:52 -07001057static inline void inc_memcg_page_state(struct page *page,
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -07001058 int idx)
Johannes Weiner00f3ca22017-07-06 15:40:52 -07001059{
1060 mod_memcg_page_state(page, idx, 1);
1061}
1062
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -07001063/* idx can be of type enum memcg_stat_item or node_stat_item */
Johannes Weiner00f3ca22017-07-06 15:40:52 -07001064static inline void dec_memcg_page_state(struct page *page,
Matthias Kaehlcke04fecbf2017-09-06 16:22:09 -07001065 int idx)
Johannes Weiner00f3ca22017-07-06 15:40:52 -07001066{
1067 mod_memcg_page_state(page, idx, -1);
1068}
1069
1070static inline void inc_lruvec_state(struct lruvec *lruvec,
1071 enum node_stat_item idx)
1072{
1073 mod_lruvec_state(lruvec, idx, 1);
1074}
1075
1076static inline void dec_lruvec_state(struct lruvec *lruvec,
1077 enum node_stat_item idx)
1078{
1079 mod_lruvec_state(lruvec, idx, -1);
1080}
1081
1082static inline void inc_lruvec_page_state(struct page *page,
1083 enum node_stat_item idx)
1084{
1085 mod_lruvec_page_state(page, idx, 1);
1086}
1087
1088static inline void dec_lruvec_page_state(struct page *page,
1089 enum node_stat_item idx)
1090{
1091 mod_lruvec_page_state(page, idx, -1);
1092}
1093
Tejun Heo52ebea72015-05-22 17:13:37 -04001094#ifdef CONFIG_CGROUP_WRITEBACK
Tejun Heo841710a2015-05-22 18:23:33 -04001095
Tejun Heo52ebea72015-05-22 17:13:37 -04001096struct list_head *mem_cgroup_cgwb_list(struct mem_cgroup *memcg);
Tejun Heo841710a2015-05-22 18:23:33 -04001097struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb);
Tejun Heoc5edf9c2015-09-29 13:04:26 -04001098void mem_cgroup_wb_stats(struct bdi_writeback *wb, unsigned long *pfilepages,
1099 unsigned long *pheadroom, unsigned long *pdirty,
1100 unsigned long *pwriteback);
Tejun Heo841710a2015-05-22 18:23:33 -04001101
1102#else /* CONFIG_CGROUP_WRITEBACK */
1103
1104static inline struct wb_domain *mem_cgroup_wb_domain(struct bdi_writeback *wb)
1105{
1106 return NULL;
1107}
1108
Tejun Heoc2aa7232015-05-22 18:23:35 -04001109static inline void mem_cgroup_wb_stats(struct bdi_writeback *wb,
Tejun Heoc5edf9c2015-09-29 13:04:26 -04001110 unsigned long *pfilepages,
1111 unsigned long *pheadroom,
Tejun Heoc2aa7232015-05-22 18:23:35 -04001112 unsigned long *pdirty,
1113 unsigned long *pwriteback)
1114{
1115}
1116
Tejun Heo841710a2015-05-22 18:23:33 -04001117#endif /* CONFIG_CGROUP_WRITEBACK */
Tejun Heo52ebea72015-05-22 17:13:37 -04001118
Glauber Costae1aab162011-12-11 21:47:03 +00001119struct sock;
Johannes Weinerbaac50b2016-01-14 15:21:17 -08001120bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages);
1121void mem_cgroup_uncharge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages);
Johannes Weinerd886f4e2016-01-20 15:02:47 -08001122#ifdef CONFIG_MEMCG
Johannes Weineref129472016-01-14 15:21:34 -08001123extern struct static_key_false memcg_sockets_enabled_key;
1124#define mem_cgroup_sockets_enabled static_branch_unlikely(&memcg_sockets_enabled_key)
Johannes Weiner2d758072016-10-07 17:00:58 -07001125void mem_cgroup_sk_alloc(struct sock *sk);
1126void mem_cgroup_sk_free(struct sock *sk);
Johannes Weinerbaac50b2016-01-14 15:21:17 -08001127static inline bool mem_cgroup_under_socket_pressure(struct mem_cgroup *memcg)
Johannes Weinere8056052016-01-14 15:21:14 -08001128{
Johannes Weiner0db15292016-01-20 15:02:50 -08001129 if (!cgroup_subsys_on_dfl(memory_cgrp_subsys) && memcg->tcpmem_pressure)
Johannes Weiner8e8ae642016-01-14 15:21:32 -08001130 return true;
Johannes Weiner8e8ae642016-01-14 15:21:32 -08001131 do {
1132 if (time_before(jiffies, memcg->socket_pressure))
1133 return true;
1134 } while ((memcg = parent_mem_cgroup(memcg)));
1135 return false;
Johannes Weinere8056052016-01-14 15:21:14 -08001136}
1137#else
Johannes Weiner80e95fe2016-01-14 15:21:20 -08001138#define mem_cgroup_sockets_enabled 0
Johannes Weiner2d758072016-10-07 17:00:58 -07001139static inline void mem_cgroup_sk_alloc(struct sock *sk) { };
1140static inline void mem_cgroup_sk_free(struct sock *sk) { };
Johannes Weinerbaac50b2016-01-14 15:21:17 -08001141static inline bool mem_cgroup_under_socket_pressure(struct mem_cgroup *memcg)
Johannes Weinere8056052016-01-14 15:21:14 -08001142{
1143 return false;
1144}
1145#endif
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08001146
Vladimir Davydov45264772016-07-26 15:24:21 -07001147struct kmem_cache *memcg_kmem_get_cache(struct kmem_cache *cachep);
1148void memcg_kmem_put_cache(struct kmem_cache *cachep);
1149int memcg_kmem_charge_memcg(struct page *page, gfp_t gfp, int order,
1150 struct mem_cgroup *memcg);
1151int memcg_kmem_charge(struct page *page, gfp_t gfp, int order);
1152void memcg_kmem_uncharge(struct page *page, int order);
1153
Johannes Weiner127424c2016-01-20 15:02:32 -08001154#if defined(CONFIG_MEMCG) && !defined(CONFIG_SLOB)
Johannes Weineref129472016-01-14 15:21:34 -08001155extern struct static_key_false memcg_kmem_enabled_key;
Tejun Heo17cc4df2017-02-22 15:41:36 -08001156extern struct workqueue_struct *memcg_kmem_cache_wq;
Glauber Costa749c5412012-12-18 14:23:01 -08001157
Vladimir Davydovdbcf73e2015-02-12 14:58:57 -08001158extern int memcg_nr_cache_ids;
Michal Hocko64219992015-09-08 15:01:07 -07001159void memcg_get_cache_ids(void);
1160void memcg_put_cache_ids(void);
Glauber Costaebe945c2012-12-18 14:23:10 -08001161
1162/*
1163 * Helper macro to loop through all memcg-specific caches. Callers must still
1164 * check if the cache is valid (it is either valid or NULL).
1165 * the slab_mutex must be held when looping through those caches
1166 */
Glauber Costa749c5412012-12-18 14:23:01 -08001167#define for_each_memcg_cache_index(_idx) \
Vladimir Davydovdbcf73e2015-02-12 14:58:57 -08001168 for ((_idx) = 0; (_idx) < memcg_nr_cache_ids; (_idx)++)
Glauber Costa749c5412012-12-18 14:23:01 -08001169
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08001170static inline bool memcg_kmem_enabled(void)
1171{
Johannes Weineref129472016-01-14 15:21:34 -08001172 return static_branch_unlikely(&memcg_kmem_enabled_key);
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08001173}
1174
1175/*
Jesper Dangaard Brouer9f706d62016-03-15 14:54:03 -07001176 * helper for accessing a memcg's index. It will be used as an index in the
Michal Hocko33398cf2015-09-08 15:01:02 -07001177 * child cache array in kmem_cache, and also to derive its name. This function
1178 * will return -1 when this is not a kmem-limited memcg.
1179 */
1180static inline int memcg_cache_id(struct mem_cgroup *memcg)
1181{
1182 return memcg ? memcg->kmemcg_id : -1;
1183}
Vladimir Davydov5722d092014-04-07 15:39:24 -07001184
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08001185#else
Glauber Costa749c5412012-12-18 14:23:01 -08001186#define for_each_memcg_cache_index(_idx) \
1187 for (; NULL; )
1188
Glauber Costab9ce5ef2012-12-18 14:22:46 -08001189static inline bool memcg_kmem_enabled(void)
1190{
1191 return false;
1192}
1193
Glauber Costa2633d7a2012-12-18 14:22:34 -08001194static inline int memcg_cache_id(struct mem_cgroup *memcg)
1195{
1196 return -1;
1197}
1198
Vladimir Davydov05257a12015-02-12 14:59:01 -08001199static inline void memcg_get_cache_ids(void)
1200{
1201}
1202
1203static inline void memcg_put_cache_ids(void)
1204{
1205}
1206
Johannes Weiner127424c2016-01-20 15:02:32 -08001207#endif /* CONFIG_MEMCG && !CONFIG_SLOB */
1208
Balbir Singh8cdea7c2008-02-07 00:13:50 -08001209#endif /* _LINUX_MEMCONTROL_H */