blob: e84f764b9dcdb60440c4a0a87c04def9daf3900d [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/fs/nfs/direct.c
3 *
4 * Copyright (C) 2003 by Chuck Lever <cel@netapp.com>
5 *
6 * High-performance uncached I/O for the Linux NFS client
7 *
8 * There are important applications whose performance or correctness
9 * depends on uncached access to file data. Database clusters
Chuck Lever88467052006-03-20 13:44:34 -050010 * (multiple copies of the same instance running on separate hosts)
Linus Torvalds1da177e2005-04-16 15:20:36 -070011 * implement their own cache coherency protocol that subsumes file
Chuck Lever88467052006-03-20 13:44:34 -050012 * system cache protocols. Applications that process datasets
13 * considerably larger than the client's memory do not always benefit
14 * from a local cache. A streaming video server, for instance, has no
Linus Torvalds1da177e2005-04-16 15:20:36 -070015 * need to cache the contents of a file.
16 *
17 * When an application requests uncached I/O, all read and write requests
18 * are made directly to the server; data stored or fetched via these
19 * requests is not cached in the Linux page cache. The client does not
20 * correct unaligned requests from applications. All requested bytes are
21 * held on permanent storage before a direct write system call returns to
22 * an application.
23 *
24 * Solaris implements an uncached I/O facility called directio() that
25 * is used for backups and sequential I/O to very large files. Solaris
26 * also supports uncaching whole NFS partitions with "-o forcedirectio,"
27 * an undocumented mount option.
28 *
29 * Designed by Jeff Kimmel, Chuck Lever, and Trond Myklebust, with
30 * help from Andrew Morton.
31 *
32 * 18 Dec 2001 Initial implementation for 2.4 --cel
33 * 08 Jul 2002 Version for 2.4.19, with bug fixes --trondmy
34 * 08 Jun 2003 Port to 2.5 APIs --cel
35 * 31 Mar 2004 Handle direct I/O without VFS support --cel
36 * 15 Sep 2004 Parallel async reads --cel
Chuck Lever88467052006-03-20 13:44:34 -050037 * 04 May 2005 support O_DIRECT with aio --cel
Linus Torvalds1da177e2005-04-16 15:20:36 -070038 *
39 */
40
Linus Torvalds1da177e2005-04-16 15:20:36 -070041#include <linux/errno.h>
42#include <linux/sched.h>
43#include <linux/kernel.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070044#include <linux/file.h>
45#include <linux/pagemap.h>
46#include <linux/kref.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090047#include <linux/slab.h>
Konstantin Khlebnikov7ec10f22011-02-22 00:28:34 +030048#include <linux/task_io_accounting_ops.h>
Peng Tao62965562012-09-25 14:55:57 +080049#include <linux/module.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070050
51#include <linux/nfs_fs.h>
52#include <linux/nfs_page.h>
53#include <linux/sunrpc/clnt.h>
54
Linus Torvalds1da177e2005-04-16 15:20:36 -070055#include <asm/uaccess.h>
Arun Sharma600634972011-07-26 16:09:06 -070056#include <linux/atomic.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070057
Trond Myklebust8d5658c2007-04-10 09:26:35 -040058#include "internal.h"
Chuck Lever91d5b472006-03-20 13:44:14 -050059#include "iostat.h"
Fred Isaman1763da12012-04-20 14:47:57 -040060#include "pnfs.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070061
Linus Torvalds1da177e2005-04-16 15:20:36 -070062#define NFSDBG_FACILITY NFSDBG_VFS
Linus Torvalds1da177e2005-04-16 15:20:36 -070063
Christoph Lametere18b8902006-12-06 20:33:20 -080064static struct kmem_cache *nfs_direct_cachep;
Linus Torvalds1da177e2005-04-16 15:20:36 -070065
66/*
67 * This represents a set of asynchronous requests that we're waiting on
68 */
69struct nfs_direct_req {
70 struct kref kref; /* release manager */
Chuck Lever15ce4a02006-03-20 13:44:34 -050071
72 /* I/O parameters */
Trond Myklebusta8881f52006-03-20 13:44:36 -050073 struct nfs_open_context *ctx; /* file open context info */
Trond Myklebustf11ac8d2010-06-25 16:35:53 -040074 struct nfs_lock_context *l_ctx; /* Lock context info */
Chuck Lever99514f82006-03-20 13:44:30 -050075 struct kiocb * iocb; /* controlling i/o request */
Chuck Lever88467052006-03-20 13:44:34 -050076 struct inode * inode; /* target file of i/o */
Chuck Lever15ce4a02006-03-20 13:44:34 -050077
78 /* completion state */
Trond Myklebust607f31e2006-06-28 16:52:45 -040079 atomic_t io_count; /* i/os we're waiting for */
Chuck Lever15ce4a02006-03-20 13:44:34 -050080 spinlock_t lock; /* protect completion state */
Chuck Lever15ce4a02006-03-20 13:44:34 -050081 ssize_t count, /* bytes actually processed */
Peng Tao35754bc2012-09-25 14:55:57 +080082 bytes_left, /* bytes left to be sent */
Linus Torvalds1da177e2005-04-16 15:20:36 -070083 error; /* any reported error */
Trond Myklebustd72b7a62006-03-20 13:44:43 -050084 struct completion completion; /* wait for i/o completion */
Trond Myklebustfad61492006-03-20 13:44:36 -050085
86 /* commit state */
Fred Isaman1763da12012-04-20 14:47:57 -040087 struct nfs_mds_commit_info mds_cinfo; /* Storage for cinfo */
88 struct pnfs_ds_commit_info ds_cinfo; /* Storage for cinfo */
89 struct work_struct work;
Trond Myklebustfad61492006-03-20 13:44:36 -050090 int flags;
91#define NFS_ODIRECT_DO_COMMIT (1) /* an unstable reply was received */
92#define NFS_ODIRECT_RESCHED_WRITES (2) /* write verification failed */
93 struct nfs_writeverf verf; /* unstable write verifier */
Linus Torvalds1da177e2005-04-16 15:20:36 -070094};
95
Fred Isaman1763da12012-04-20 14:47:57 -040096static const struct nfs_pgio_completion_ops nfs_direct_write_completion_ops;
97static const struct nfs_commit_completion_ops nfs_direct_commit_completion_ops;
Trond Myklebustfad61492006-03-20 13:44:36 -050098static void nfs_direct_write_complete(struct nfs_direct_req *dreq, struct inode *inode);
Fred Isaman1763da12012-04-20 14:47:57 -040099static void nfs_direct_write_schedule_work(struct work_struct *work);
Trond Myklebust607f31e2006-06-28 16:52:45 -0400100
101static inline void get_dreq(struct nfs_direct_req *dreq)
102{
103 atomic_inc(&dreq->io_count);
104}
105
106static inline int put_dreq(struct nfs_direct_req *dreq)
107{
108 return atomic_dec_and_test(&dreq->io_count);
109}
110
Weston Andros Adamson5002c582014-05-15 11:56:54 -0400111/*
112 * nfs_direct_select_verf - select the right verifier
113 * @dreq - direct request possibly spanning multiple servers
114 * @ds_clp - nfs_client of data server or NULL if MDS / non-pnfs
115 * @ds_idx - index of data server in data server list, only valid if ds_clp set
116 *
117 * returns the correct verifier to use given the role of the server
118 */
119static struct nfs_writeverf *
120nfs_direct_select_verf(struct nfs_direct_req *dreq,
121 struct nfs_client *ds_clp,
122 int ds_idx)
123{
124 struct nfs_writeverf *verfp = &dreq->verf;
125
126#ifdef CONFIG_NFS_V4_1
127 if (ds_clp) {
128 /* pNFS is in use, use the DS verf */
129 if (ds_idx >= 0 && ds_idx < dreq->ds_cinfo.nbuckets)
130 verfp = &dreq->ds_cinfo.buckets[ds_idx].direct_verf;
131 else
132 WARN_ON_ONCE(1);
133 }
134#endif
135 return verfp;
136}
137
138
139/*
140 * nfs_direct_set_hdr_verf - set the write/commit verifier
141 * @dreq - direct request possibly spanning multiple servers
142 * @hdr - pageio header to validate against previously seen verfs
143 *
144 * Set the server's (MDS or DS) "seen" verifier
145 */
146static void nfs_direct_set_hdr_verf(struct nfs_direct_req *dreq,
147 struct nfs_pgio_header *hdr)
148{
149 struct nfs_writeverf *verfp;
150
Weston Andros Adamsond45f60c2014-06-09 11:48:35 -0400151 verfp = nfs_direct_select_verf(dreq, hdr->ds_clp,
152 hdr->ds_idx);
Weston Andros Adamson5002c582014-05-15 11:56:54 -0400153 WARN_ON_ONCE(verfp->committed >= 0);
154 memcpy(verfp, &hdr->verf, sizeof(struct nfs_writeverf));
155 WARN_ON_ONCE(verfp->committed < 0);
156}
157
158/*
159 * nfs_direct_cmp_hdr_verf - compare verifier for pgio header
160 * @dreq - direct request possibly spanning multiple servers
161 * @hdr - pageio header to validate against previously seen verf
162 *
163 * set the server's "seen" verf if not initialized.
164 * returns result of comparison between @hdr->verf and the "seen"
165 * verf of the server used by @hdr (DS or MDS)
166 */
167static int nfs_direct_set_or_cmp_hdr_verf(struct nfs_direct_req *dreq,
168 struct nfs_pgio_header *hdr)
169{
170 struct nfs_writeverf *verfp;
171
Weston Andros Adamsond45f60c2014-06-09 11:48:35 -0400172 verfp = nfs_direct_select_verf(dreq, hdr->ds_clp,
173 hdr->ds_idx);
Weston Andros Adamson5002c582014-05-15 11:56:54 -0400174 if (verfp->committed < 0) {
175 nfs_direct_set_hdr_verf(dreq, hdr);
176 return 0;
177 }
178 return memcmp(verfp, &hdr->verf, sizeof(struct nfs_writeverf));
179}
180
Weston Andros Adamson5002c582014-05-15 11:56:54 -0400181/*
182 * nfs_direct_cmp_commit_data_verf - compare verifier for commit data
183 * @dreq - direct request possibly spanning multiple servers
184 * @data - commit data to validate against previously seen verf
185 *
186 * returns result of comparison between @data->verf and the verf of
187 * the server used by @data (DS or MDS)
188 */
189static int nfs_direct_cmp_commit_data_verf(struct nfs_direct_req *dreq,
190 struct nfs_commit_data *data)
191{
192 struct nfs_writeverf *verfp;
193
194 verfp = nfs_direct_select_verf(dreq, data->ds_clp,
195 data->ds_commit_index);
196 WARN_ON_ONCE(verfp->committed < 0);
197 return memcmp(verfp, &data->verf, sizeof(struct nfs_writeverf));
198}
Weston Andros Adamson5002c582014-05-15 11:56:54 -0400199
Linus Torvalds1da177e2005-04-16 15:20:36 -0700200/**
Chuck Leverb8a32e22006-03-20 13:44:28 -0500201 * nfs_direct_IO - NFS address space operation for direct I/O
202 * @rw: direction (read or write)
203 * @iocb: target I/O control block
204 * @iov: array of vectors that define I/O buffer
205 * @pos: offset in file to begin the operation
206 * @nr_segs: size of iovec array
207 *
208 * The presence of this routine in the address space ops vector means
Mel Gormana564b8f2012-07-31 16:45:12 -0700209 * the NFS client supports direct I/O. However, for most direct IO, we
210 * shunt off direct read and write requests before the VFS gets them,
211 * so this method is only ever called for swap.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700212 */
Al Virod8d3d942014-03-04 21:27:34 -0500213ssize_t nfs_direct_IO(int rw, struct kiocb *iocb, struct iov_iter *iter, loff_t pos)
Chuck Leverb8a32e22006-03-20 13:44:28 -0500214{
Mel Gormana564b8f2012-07-31 16:45:12 -0700215#ifndef CONFIG_NFS_SWAP
Al Viro6de14722013-09-16 10:53:17 -0400216 dprintk("NFS: nfs_direct_IO (%pD) off/no(%Ld/%lu) EINVAL\n",
Al Virod8d3d942014-03-04 21:27:34 -0500217 iocb->ki_filp, (long long) pos, iter->nr_segs);
Chuck Leverb8a32e22006-03-20 13:44:28 -0500218
219 return -EINVAL;
Mel Gormana564b8f2012-07-31 16:45:12 -0700220#else
Mel Gormana564b8f2012-07-31 16:45:12 -0700221 VM_BUG_ON(iocb->ki_nbytes != PAGE_SIZE);
222
Martin K. Petersene19a8a02014-10-14 09:00:44 -0600223 if (rw == READ)
224 return nfs_file_direct_read(iocb, iter, pos);
225 return nfs_file_direct_write(iocb, iter, pos);
Mel Gormana564b8f2012-07-31 16:45:12 -0700226#endif /* CONFIG_NFS_SWAP */
Chuck Leverb8a32e22006-03-20 13:44:28 -0500227}
228
Chuck Lever749e1462007-05-19 17:22:46 -0400229static void nfs_direct_release_pages(struct page **pages, unsigned int npages)
Chuck Lever9c93ab72006-06-20 12:56:31 -0400230{
Chuck Lever749e1462007-05-19 17:22:46 -0400231 unsigned int i;
Trond Myklebust607f31e2006-06-28 16:52:45 -0400232 for (i = 0; i < npages; i++)
233 page_cache_release(pages[i]);
Trond Myklebust6b45d852006-03-20 13:44:43 -0500234}
235
Fred Isaman1763da12012-04-20 14:47:57 -0400236void nfs_init_cinfo_from_dreq(struct nfs_commit_info *cinfo,
237 struct nfs_direct_req *dreq)
238{
239 cinfo->lock = &dreq->lock;
240 cinfo->mds = &dreq->mds_cinfo;
241 cinfo->ds = &dreq->ds_cinfo;
242 cinfo->dreq = dreq;
243 cinfo->completion_ops = &nfs_direct_commit_completion_ops;
244}
245
Chuck Lever93619e52006-03-20 13:44:31 -0500246static inline struct nfs_direct_req *nfs_direct_req_alloc(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700247{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700248 struct nfs_direct_req *dreq;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700249
Trond Myklebust292f3ee2012-04-30 18:31:49 -0400250 dreq = kmem_cache_zalloc(nfs_direct_cachep, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700251 if (!dreq)
252 return NULL;
253
254 kref_init(&dreq->kref);
Trond Myklebust607f31e2006-06-28 16:52:45 -0400255 kref_get(&dreq->kref);
Trond Myklebustd72b7a62006-03-20 13:44:43 -0500256 init_completion(&dreq->completion);
Fred Isaman1763da12012-04-20 14:47:57 -0400257 INIT_LIST_HEAD(&dreq->mds_cinfo.list);
Weston Andros Adamson5002c582014-05-15 11:56:54 -0400258 dreq->verf.committed = NFS_INVALID_STABLE_HOW; /* not set yet */
Fred Isaman1763da12012-04-20 14:47:57 -0400259 INIT_WORK(&dreq->work, nfs_direct_write_schedule_work);
Chuck Lever15ce4a02006-03-20 13:44:34 -0500260 spin_lock_init(&dreq->lock);
Chuck Lever93619e52006-03-20 13:44:31 -0500261
262 return dreq;
263}
264
Trond Myklebustb4946ff2007-05-30 12:58:00 -0400265static void nfs_direct_req_free(struct kref *kref)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700266{
267 struct nfs_direct_req *dreq = container_of(kref, struct nfs_direct_req, kref);
Trond Myklebusta8881f52006-03-20 13:44:36 -0500268
Peng Tao8c393f92014-11-05 22:36:50 +0800269 nfs_free_pnfs_ds_cinfo(&dreq->ds_cinfo);
Trond Myklebustf11ac8d2010-06-25 16:35:53 -0400270 if (dreq->l_ctx != NULL)
271 nfs_put_lock_context(dreq->l_ctx);
Trond Myklebusta8881f52006-03-20 13:44:36 -0500272 if (dreq->ctx != NULL)
273 put_nfs_open_context(dreq->ctx);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700274 kmem_cache_free(nfs_direct_cachep, dreq);
275}
276
Trond Myklebustb4946ff2007-05-30 12:58:00 -0400277static void nfs_direct_req_release(struct nfs_direct_req *dreq)
278{
279 kref_put(&dreq->kref, nfs_direct_req_free);
280}
281
Peng Tao62965562012-09-25 14:55:57 +0800282ssize_t nfs_dreq_bytes_left(struct nfs_direct_req *dreq)
283{
284 return dreq->bytes_left;
285}
286EXPORT_SYMBOL_GPL(nfs_dreq_bytes_left);
287
Chuck Leverd4cc9482006-03-20 13:44:28 -0500288/*
Chuck Leverbc0fb202006-03-20 13:44:31 -0500289 * Collects and returns the final error value/byte-count.
290 */
291static ssize_t nfs_direct_wait(struct nfs_direct_req *dreq)
292{
Chuck Lever15ce4a02006-03-20 13:44:34 -0500293 ssize_t result = -EIOCBQUEUED;
Chuck Leverbc0fb202006-03-20 13:44:31 -0500294
295 /* Async requests don't wait here */
296 if (dreq->iocb)
297 goto out;
298
Matthew Wilcox150030b2007-12-06 16:24:39 -0500299 result = wait_for_completion_killable(&dreq->completion);
Chuck Leverbc0fb202006-03-20 13:44:31 -0500300
301 if (!result)
Chuck Lever15ce4a02006-03-20 13:44:34 -0500302 result = dreq->error;
Chuck Leverbc0fb202006-03-20 13:44:31 -0500303 if (!result)
Chuck Lever15ce4a02006-03-20 13:44:34 -0500304 result = dreq->count;
Chuck Leverbc0fb202006-03-20 13:44:31 -0500305
306out:
Chuck Leverbc0fb202006-03-20 13:44:31 -0500307 return (ssize_t) result;
308}
309
310/*
Trond Myklebust607f31e2006-06-28 16:52:45 -0400311 * Synchronous I/O uses a stack-allocated iocb. Thus we can't trust
312 * the iocb is still valid here if this is a synchronous request.
Chuck Lever63ab46a2006-03-20 13:44:31 -0500313 */
Christoph Hellwig9811cd52013-11-14 08:50:28 -0800314static void nfs_direct_complete(struct nfs_direct_req *dreq, bool write)
Chuck Lever63ab46a2006-03-20 13:44:31 -0500315{
Christoph Hellwig9811cd52013-11-14 08:50:28 -0800316 struct inode *inode = dreq->inode;
317
Christoph Hellwig2a009ec2013-11-14 08:50:29 -0800318 if (dreq->iocb && write) {
Christoph Hellwig9811cd52013-11-14 08:50:28 -0800319 loff_t pos = dreq->iocb->ki_pos + dreq->count;
Christoph Hellwig2a009ec2013-11-14 08:50:29 -0800320
321 spin_lock(&inode->i_lock);
322 if (i_size_read(inode) < pos)
323 i_size_write(inode, pos);
324 spin_unlock(&inode->i_lock);
325 }
326
Christoph Hellwig1f90ee22013-11-14 08:50:30 -0800327 if (write)
Christoph Hellwig2a009ec2013-11-14 08:50:29 -0800328 nfs_zap_mapping(inode, inode->i_mapping);
Christoph Hellwig1f90ee22013-11-14 08:50:30 -0800329
330 inode_dio_done(inode);
Christoph Hellwig2a009ec2013-11-14 08:50:29 -0800331
332 if (dreq->iocb) {
Chuck Lever15ce4a02006-03-20 13:44:34 -0500333 long res = (long) dreq->error;
Chuck Lever63ab46a2006-03-20 13:44:31 -0500334 if (!res)
Chuck Lever15ce4a02006-03-20 13:44:34 -0500335 res = (long) dreq->count;
Chuck Lever63ab46a2006-03-20 13:44:31 -0500336 aio_complete(dreq->iocb, res, 0);
Trond Myklebustd72b7a62006-03-20 13:44:43 -0500337 }
Christoph Hellwig2a009ec2013-11-14 08:50:29 -0800338
Trond Myklebustd72b7a62006-03-20 13:44:43 -0500339 complete_all(&dreq->completion);
Chuck Lever63ab46a2006-03-20 13:44:31 -0500340
Trond Myklebustb4946ff2007-05-30 12:58:00 -0400341 nfs_direct_req_release(dreq);
Chuck Lever63ab46a2006-03-20 13:44:31 -0500342}
343
Trond Myklebust1385b812012-05-04 13:54:24 -0400344static void nfs_direct_readpage_release(struct nfs_page *req)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700345{
Niels de Vos1e8968c2013-12-17 18:20:16 +0100346 dprintk("NFS: direct read done (%s/%llu %d@%lld)\n",
Fred Isaman584aa812012-04-20 14:47:51 -0400347 req->wb_context->dentry->d_inode->i_sb->s_id,
Niels de Vos1e8968c2013-12-17 18:20:16 +0100348 (unsigned long long)NFS_FILEID(req->wb_context->dentry->d_inode),
Fred Isaman584aa812012-04-20 14:47:51 -0400349 req->wb_bytes,
350 (long long)req_offset(req));
351 nfs_release_request(req);
Trond Myklebustfdd1e742008-04-15 16:33:58 -0400352}
353
Fred Isaman584aa812012-04-20 14:47:51 -0400354static void nfs_direct_read_completion(struct nfs_pgio_header *hdr)
Trond Myklebustfdd1e742008-04-15 16:33:58 -0400355{
Fred Isaman584aa812012-04-20 14:47:51 -0400356 unsigned long bytes = 0;
357 struct nfs_direct_req *dreq = hdr->dreq;
Trond Myklebustfdd1e742008-04-15 16:33:58 -0400358
Fred Isaman584aa812012-04-20 14:47:51 -0400359 if (test_bit(NFS_IOHDR_REDO, &hdr->flags))
360 goto out_put;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700361
Chuck Lever15ce4a02006-03-20 13:44:34 -0500362 spin_lock(&dreq->lock);
Fred Isaman584aa812012-04-20 14:47:51 -0400363 if (test_bit(NFS_IOHDR_ERROR, &hdr->flags) && (hdr->good_bytes == 0))
364 dreq->error = hdr->error;
365 else
366 dreq->count += hdr->good_bytes;
367 spin_unlock(&dreq->lock);
Trond Myklebust607f31e2006-06-28 16:52:45 -0400368
Trond Myklebust4bd8b012012-05-01 12:49:58 -0400369 while (!list_empty(&hdr->pages)) {
370 struct nfs_page *req = nfs_list_entry(hdr->pages.next);
371 struct page *page = req->wb_page;
Fred Isaman584aa812012-04-20 14:47:51 -0400372
Jeff Laytonbe7e9852012-12-12 12:36:31 -0500373 if (!PageCompound(page) && bytes < hdr->good_bytes)
374 set_page_dirty(page);
Trond Myklebust4bd8b012012-05-01 12:49:58 -0400375 bytes += req->wb_bytes;
376 nfs_list_remove_request(req);
377 nfs_direct_readpage_release(req);
Fred Isaman584aa812012-04-20 14:47:51 -0400378 }
379out_put:
Trond Myklebust607f31e2006-06-28 16:52:45 -0400380 if (put_dreq(dreq))
Christoph Hellwig9811cd52013-11-14 08:50:28 -0800381 nfs_direct_complete(dreq, false);
Fred Isaman584aa812012-04-20 14:47:51 -0400382 hdr->release(hdr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700383}
384
Trond Myklebust3e9e0ca32012-04-30 13:40:06 -0400385static void nfs_read_sync_pgio_error(struct list_head *head)
Fred Isamancd841602012-04-20 14:47:44 -0400386{
Fred Isaman584aa812012-04-20 14:47:51 -0400387 struct nfs_page *req;
Fred Isamancd841602012-04-20 14:47:44 -0400388
Fred Isaman584aa812012-04-20 14:47:51 -0400389 while (!list_empty(head)) {
390 req = nfs_list_entry(head->next);
391 nfs_list_remove_request(req);
392 nfs_release_request(req);
393 }
Fred Isamancd841602012-04-20 14:47:44 -0400394}
395
Fred Isaman584aa812012-04-20 14:47:51 -0400396static void nfs_direct_pgio_init(struct nfs_pgio_header *hdr)
397{
398 get_dreq(hdr->dreq);
399}
400
401static const struct nfs_pgio_completion_ops nfs_direct_read_completion_ops = {
Trond Myklebust3e9e0ca32012-04-30 13:40:06 -0400402 .error_cleanup = nfs_read_sync_pgio_error,
Fred Isaman584aa812012-04-20 14:47:51 -0400403 .init_hdr = nfs_direct_pgio_init,
404 .completion = nfs_direct_read_completion,
405};
406
Chuck Leverd4cc9482006-03-20 13:44:28 -0500407/*
Trond Myklebust607f31e2006-06-28 16:52:45 -0400408 * For each rsize'd chunk of the user's buffer, dispatch an NFS READ
409 * operation. If nfs_readdata_alloc() or get_user_pages() fails,
410 * bail and stop sending more reads. Read length accounting is
411 * handled automatically by nfs_direct_read_result(). Otherwise, if
412 * no requests have been sent, just return an error.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700413 */
Al Viro91f79c42014-03-21 04:58:33 -0400414
415static ssize_t nfs_direct_read_schedule_iovec(struct nfs_direct_req *dreq,
416 struct iov_iter *iter,
417 loff_t pos)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700418{
Al Viro91f79c42014-03-21 04:58:33 -0400419 struct nfs_pageio_descriptor desc;
420 struct inode *inode = dreq->inode;
421 ssize_t result = -EINVAL;
422 size_t requested_bytes = 0;
423 size_t rsize = max_t(size_t, NFS_SERVER(inode)->rsize, PAGE_SIZE);
Chuck Lever82b145c2006-06-20 12:57:03 -0400424
Linus Torvalds16b90572014-06-12 10:30:18 -0700425 nfs_pageio_init_read(&desc, dreq->inode, false,
Al Viro91f79c42014-03-21 04:58:33 -0400426 &nfs_direct_read_completion_ops);
427 get_dreq(dreq);
428 desc.pg_dreq = dreq;
429 atomic_inc(&inode->i_dio_count);
430
431 while (iov_iter_count(iter)) {
432 struct page **pagevec;
Chuck Lever5dd602f2006-03-20 13:44:29 -0500433 size_t bytes;
Al Viro91f79c42014-03-21 04:58:33 -0400434 size_t pgbase;
435 unsigned npages, i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700436
Al Viro91f79c42014-03-21 04:58:33 -0400437 result = iov_iter_get_pages_alloc(iter, &pagevec,
438 rsize, &pgbase);
439 if (result < 0)
Trond Myklebust607f31e2006-06-28 16:52:45 -0400440 break;
Al Viro91f79c42014-03-21 04:58:33 -0400441
442 bytes = result;
443 iov_iter_advance(iter, bytes);
444 npages = (result + pgbase + PAGE_SIZE - 1) / PAGE_SIZE;
Fred Isaman584aa812012-04-20 14:47:51 -0400445 for (i = 0; i < npages; i++) {
446 struct nfs_page *req;
Trond Myklebustbf5fc402012-05-04 13:47:16 -0400447 unsigned int req_len = min_t(size_t, bytes, PAGE_SIZE - pgbase);
Fred Isaman584aa812012-04-20 14:47:51 -0400448 /* XXX do we need to do the eof zeroing found in async_filler? */
Weston Andros Adamson2bfc6e52014-05-15 11:56:45 -0400449 req = nfs_create_request(dreq->ctx, pagevec[i], NULL,
Fred Isaman584aa812012-04-20 14:47:51 -0400450 pgbase, req_len);
451 if (IS_ERR(req)) {
Fred Isaman584aa812012-04-20 14:47:51 -0400452 result = PTR_ERR(req);
453 break;
454 }
455 req->wb_index = pos >> PAGE_SHIFT;
456 req->wb_offset = pos & ~PAGE_MASK;
Al Viro91f79c42014-03-21 04:58:33 -0400457 if (!nfs_pageio_add_request(&desc, req)) {
458 result = desc.pg_error;
Fred Isaman584aa812012-04-20 14:47:51 -0400459 nfs_release_request(req);
Fred Isaman584aa812012-04-20 14:47:51 -0400460 break;
461 }
462 pgbase = 0;
463 bytes -= req_len;
Al Viro91f79c42014-03-21 04:58:33 -0400464 requested_bytes += req_len;
Fred Isaman584aa812012-04-20 14:47:51 -0400465 pos += req_len;
Peng Tao35754bc2012-09-25 14:55:57 +0800466 dreq->bytes_left -= req_len;
Fred Isaman584aa812012-04-20 14:47:51 -0400467 }
Trond Myklebust6d747432012-04-30 13:27:31 -0400468 nfs_direct_release_pages(pagevec, npages);
Al Viro91f79c42014-03-21 04:58:33 -0400469 kvfree(pagevec);
Chuck Lever19f73782007-11-12 12:16:47 -0500470 if (result < 0)
471 break;
Chuck Lever19f73782007-11-12 12:16:47 -0500472 }
473
Fred Isaman584aa812012-04-20 14:47:51 -0400474 nfs_pageio_complete(&desc);
475
Chuck Lever839f7ad2011-01-21 15:54:57 +0000476 /*
477 * If no bytes were started, return the error, and let the
478 * generic layer handle the completion.
479 */
480 if (requested_bytes == 0) {
Christoph Hellwig1f90ee22013-11-14 08:50:30 -0800481 inode_dio_done(inode);
Chuck Lever839f7ad2011-01-21 15:54:57 +0000482 nfs_direct_req_release(dreq);
483 return result < 0 ? result : -EIO;
484 }
485
Chuck Lever19f73782007-11-12 12:16:47 -0500486 if (put_dreq(dreq))
Christoph Hellwig9811cd52013-11-14 08:50:28 -0800487 nfs_direct_complete(dreq, false);
Chuck Lever839f7ad2011-01-21 15:54:57 +0000488 return 0;
Chuck Lever19f73782007-11-12 12:16:47 -0500489}
490
Christoph Hellwig14a3ec792013-11-14 08:50:31 -0800491/**
492 * nfs_file_direct_read - file direct read operation for NFS files
493 * @iocb: target I/O control block
Al Viro619d30b2014-03-04 21:53:33 -0500494 * @iter: vector of user buffers into which to read data
Christoph Hellwig14a3ec792013-11-14 08:50:31 -0800495 * @pos: byte offset in file where reading starts
496 *
497 * We use this function for direct reads instead of calling
498 * generic_file_aio_read() in order to avoid gfar's check to see if
499 * the request starts before the end of the file. For that check
500 * to work, we must generate a GETATTR before each direct read, and
501 * even then there is a window between the GETATTR and the subsequent
502 * READ where the file size could change. Our preference is simply
503 * to do all reads the application wants, and the server will take
504 * care of managing the end of file boundary.
505 *
506 * This function also eliminates unnecessarily updating the file's
507 * atime locally, as the NFS server sets the file's atime, and this
508 * client must read the updated atime from the server back into its
509 * cache.
510 */
Al Viro619d30b2014-03-04 21:53:33 -0500511ssize_t nfs_file_direct_read(struct kiocb *iocb, struct iov_iter *iter,
Martin K. Petersene19a8a02014-10-14 09:00:44 -0600512 loff_t pos)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700513{
Christoph Hellwig14a3ec792013-11-14 08:50:31 -0800514 struct file *file = iocb->ki_filp;
515 struct address_space *mapping = file->f_mapping;
516 struct inode *inode = mapping->host;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700517 struct nfs_direct_req *dreq;
Trond Myklebustb3c54de2012-08-13 17:15:50 -0400518 struct nfs_lock_context *l_ctx;
Christoph Hellwig14a3ec792013-11-14 08:50:31 -0800519 ssize_t result = -EINVAL;
Al Viroa6cbcd42014-03-04 22:38:00 -0500520 size_t count = iov_iter_count(iter);
Christoph Hellwig14a3ec792013-11-14 08:50:31 -0800521 nfs_add_stats(mapping->host, NFSIOS_DIRECTREADBYTES, count);
522
523 dfprintk(FILE, "NFS: direct read(%pD2, %zd@%Ld)\n",
524 file, count, (long long) pos);
525
526 result = 0;
527 if (!count)
528 goto out;
529
Christoph Hellwigd0b98752013-11-14 08:50:33 -0800530 mutex_lock(&inode->i_mutex);
Christoph Hellwig14a3ec792013-11-14 08:50:31 -0800531 result = nfs_sync_mapping(mapping);
532 if (result)
Christoph Hellwigd0b98752013-11-14 08:50:33 -0800533 goto out_unlock;
Christoph Hellwig14a3ec792013-11-14 08:50:31 -0800534
535 task_io_account_read(count);
536
537 result = -ENOMEM;
Trond Myklebust607f31e2006-06-28 16:52:45 -0400538 dreq = nfs_direct_req_alloc();
Trond Myklebustf11ac8d2010-06-25 16:35:53 -0400539 if (dreq == NULL)
Christoph Hellwigd0b98752013-11-14 08:50:33 -0800540 goto out_unlock;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700541
Chuck Lever91d5b472006-03-20 13:44:14 -0500542 dreq->inode = inode;
Al Viro619d30b2014-03-04 21:53:33 -0500543 dreq->bytes_left = count;
Trond Myklebustcd3758e2007-08-10 17:44:32 -0400544 dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp));
Trond Myklebustb3c54de2012-08-13 17:15:50 -0400545 l_ctx = nfs_get_lock_context(dreq->ctx);
546 if (IS_ERR(l_ctx)) {
547 result = PTR_ERR(l_ctx);
Trond Myklebustf11ac8d2010-06-25 16:35:53 -0400548 goto out_release;
Trond Myklebustb3c54de2012-08-13 17:15:50 -0400549 }
550 dreq->l_ctx = l_ctx;
Chuck Lever487b8372006-03-20 13:44:30 -0500551 if (!is_sync_kiocb(iocb))
552 dreq->iocb = iocb;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700553
Al Viro619d30b2014-03-04 21:53:33 -0500554 NFS_I(inode)->read_io += count;
Al Viro91f79c42014-03-21 04:58:33 -0400555 result = nfs_direct_read_schedule_iovec(dreq, iter, pos);
Christoph Hellwigd0b98752013-11-14 08:50:33 -0800556
557 mutex_unlock(&inode->i_mutex);
558
Christoph Hellwig14a3ec792013-11-14 08:50:31 -0800559 if (!result) {
Trond Myklebust607f31e2006-06-28 16:52:45 -0400560 result = nfs_direct_wait(dreq);
Christoph Hellwig14a3ec792013-11-14 08:50:31 -0800561 if (result > 0)
562 iocb->ki_pos = pos + result;
563 }
Christoph Hellwigd0b98752013-11-14 08:50:33 -0800564
565 nfs_direct_req_release(dreq);
566 return result;
567
Trond Myklebustf11ac8d2010-06-25 16:35:53 -0400568out_release:
Trond Myklebustb4946ff2007-05-30 12:58:00 -0400569 nfs_direct_req_release(dreq);
Christoph Hellwigd0b98752013-11-14 08:50:33 -0800570out_unlock:
571 mutex_unlock(&inode->i_mutex);
Trond Myklebustf11ac8d2010-06-25 16:35:53 -0400572out:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700573 return result;
574}
575
Tom Haynes085d1e32014-12-11 13:04:55 -0500576static void
577nfs_direct_write_scan_commit_list(struct inode *inode,
578 struct list_head *list,
579 struct nfs_commit_info *cinfo)
580{
581 spin_lock(cinfo->lock);
582#ifdef CONFIG_NFS_V4_1
583 if (cinfo->ds != NULL && cinfo->ds->nwritten != 0)
584 NFS_SERVER(inode)->pnfs_curr_ld->recover_commit_reqs(list, cinfo);
585#endif
586 nfs_scan_commit_list(&cinfo->mds->list, list, cinfo, 0);
587 spin_unlock(cinfo->lock);
588}
589
Trond Myklebustfad61492006-03-20 13:44:36 -0500590static void nfs_direct_write_reschedule(struct nfs_direct_req *dreq)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700591{
Fred Isaman1763da12012-04-20 14:47:57 -0400592 struct nfs_pageio_descriptor desc;
593 struct nfs_page *req, *tmp;
594 LIST_HEAD(reqs);
595 struct nfs_commit_info cinfo;
596 LIST_HEAD(failed);
597
598 nfs_init_cinfo_from_dreq(&cinfo, dreq);
Tom Haynes085d1e32014-12-11 13:04:55 -0500599 nfs_direct_write_scan_commit_list(dreq->inode, &reqs, &cinfo);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700600
Trond Myklebustfad61492006-03-20 13:44:36 -0500601 dreq->count = 0;
Trond Myklebust607f31e2006-06-28 16:52:45 -0400602 get_dreq(dreq);
Trond Myklebustfad61492006-03-20 13:44:36 -0500603
Christoph Hellwiga20c93e2014-04-16 15:07:21 +0200604 nfs_pageio_init_write(&desc, dreq->inode, FLUSH_STABLE, false,
Fred Isaman1763da12012-04-20 14:47:57 -0400605 &nfs_direct_write_completion_ops);
606 desc.pg_dreq = dreq;
Trond Myklebust607f31e2006-06-28 16:52:45 -0400607
Fred Isaman1763da12012-04-20 14:47:57 -0400608 list_for_each_entry_safe(req, tmp, &reqs, wb_list) {
609 if (!nfs_pageio_add_request(&desc, req)) {
Trond Myklebust4035c242012-07-08 10:24:10 -0400610 nfs_list_remove_request(req);
Fred Isaman1763da12012-04-20 14:47:57 -0400611 nfs_list_add_request(req, &failed);
612 spin_lock(cinfo.lock);
613 dreq->flags = 0;
614 dreq->error = -EIO;
615 spin_unlock(cinfo.lock);
616 }
Trond Myklebust5a695da2012-06-19 13:39:14 -0400617 nfs_release_request(req);
Fred Isaman1763da12012-04-20 14:47:57 -0400618 }
619 nfs_pageio_complete(&desc);
Trond Myklebust607f31e2006-06-28 16:52:45 -0400620
Trond Myklebust4035c242012-07-08 10:24:10 -0400621 while (!list_empty(&failed)) {
622 req = nfs_list_entry(failed.next);
623 nfs_list_remove_request(req);
Trond Myklebust1d1afcb2012-05-09 14:04:55 -0400624 nfs_unlock_and_release_request(req);
Trond Myklebust4035c242012-07-08 10:24:10 -0400625 }
Trond Myklebust607f31e2006-06-28 16:52:45 -0400626
627 if (put_dreq(dreq))
Fred Isaman1763da12012-04-20 14:47:57 -0400628 nfs_direct_write_complete(dreq, dreq->inode);
Trond Myklebustfad61492006-03-20 13:44:36 -0500629}
630
Fred Isaman1763da12012-04-20 14:47:57 -0400631static void nfs_direct_commit_complete(struct nfs_commit_data *data)
Trond Myklebustfad61492006-03-20 13:44:36 -0500632{
Fred Isaman0b7c0152012-04-20 14:47:39 -0400633 struct nfs_direct_req *dreq = data->dreq;
Fred Isaman1763da12012-04-20 14:47:57 -0400634 struct nfs_commit_info cinfo;
635 struct nfs_page *req;
Trond Myklebustc9d8f892008-04-15 16:56:39 -0400636 int status = data->task.tk_status;
637
Fred Isaman1763da12012-04-20 14:47:57 -0400638 nfs_init_cinfo_from_dreq(&cinfo, dreq);
Trond Myklebustc9d8f892008-04-15 16:56:39 -0400639 if (status < 0) {
Trond Myklebust60fa3f72007-04-14 19:11:52 -0400640 dprintk("NFS: %5u commit failed with error %d.\n",
Fred Isaman1763da12012-04-20 14:47:57 -0400641 data->task.tk_pid, status);
Trond Myklebustfad61492006-03-20 13:44:36 -0500642 dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
Weston Andros Adamson5002c582014-05-15 11:56:54 -0400643 } else if (nfs_direct_cmp_commit_data_verf(dreq, data)) {
Trond Myklebustc9d8f892008-04-15 16:56:39 -0400644 dprintk("NFS: %5u commit verify failed\n", data->task.tk_pid);
Trond Myklebustfad61492006-03-20 13:44:36 -0500645 dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
646 }
647
Trond Myklebustc9d8f892008-04-15 16:56:39 -0400648 dprintk("NFS: %5u commit returned %d\n", data->task.tk_pid, status);
Fred Isaman1763da12012-04-20 14:47:57 -0400649 while (!list_empty(&data->pages)) {
650 req = nfs_list_entry(data->pages.next);
651 nfs_list_remove_request(req);
652 if (dreq->flags == NFS_ODIRECT_RESCHED_WRITES) {
653 /* Note the rewrite will go through mds */
654 nfs_mark_request_commit(req, NULL, &cinfo);
Fred Isaman906369e2012-06-08 16:48:33 -0400655 } else
656 nfs_release_request(req);
Trond Myklebust1d1afcb2012-05-09 14:04:55 -0400657 nfs_unlock_and_release_request(req);
Fred Isaman1763da12012-04-20 14:47:57 -0400658 }
659
660 if (atomic_dec_and_test(&cinfo.mds->rpcs_out))
661 nfs_direct_write_complete(dreq, data->inode);
Trond Myklebustfad61492006-03-20 13:44:36 -0500662}
663
Fred Isaman1763da12012-04-20 14:47:57 -0400664static void nfs_direct_error_cleanup(struct nfs_inode *nfsi)
665{
666 /* There is no lock to clear */
667}
668
669static const struct nfs_commit_completion_ops nfs_direct_commit_completion_ops = {
670 .completion = nfs_direct_commit_complete,
671 .error_cleanup = nfs_direct_error_cleanup,
Trond Myklebustfad61492006-03-20 13:44:36 -0500672};
673
674static void nfs_direct_commit_schedule(struct nfs_direct_req *dreq)
675{
Fred Isaman1763da12012-04-20 14:47:57 -0400676 int res;
677 struct nfs_commit_info cinfo;
678 LIST_HEAD(mds_list);
Trond Myklebustfad61492006-03-20 13:44:36 -0500679
Fred Isaman1763da12012-04-20 14:47:57 -0400680 nfs_init_cinfo_from_dreq(&cinfo, dreq);
681 nfs_scan_commit(dreq->inode, &mds_list, &cinfo);
682 res = nfs_generic_commit_list(dreq->inode, &mds_list, 0, &cinfo);
683 if (res < 0) /* res == -ENOMEM */
684 nfs_direct_write_reschedule(dreq);
Trond Myklebustfad61492006-03-20 13:44:36 -0500685}
686
Fred Isaman1763da12012-04-20 14:47:57 -0400687static void nfs_direct_write_schedule_work(struct work_struct *work)
Trond Myklebustfad61492006-03-20 13:44:36 -0500688{
Fred Isaman1763da12012-04-20 14:47:57 -0400689 struct nfs_direct_req *dreq = container_of(work, struct nfs_direct_req, work);
Trond Myklebustfad61492006-03-20 13:44:36 -0500690 int flags = dreq->flags;
691
692 dreq->flags = 0;
693 switch (flags) {
694 case NFS_ODIRECT_DO_COMMIT:
695 nfs_direct_commit_schedule(dreq);
696 break;
697 case NFS_ODIRECT_RESCHED_WRITES:
698 nfs_direct_write_reschedule(dreq);
699 break;
700 default:
Christoph Hellwig9811cd52013-11-14 08:50:28 -0800701 nfs_direct_complete(dreq, true);
Trond Myklebustfad61492006-03-20 13:44:36 -0500702 }
703}
704
Fred Isaman1763da12012-04-20 14:47:57 -0400705static void nfs_direct_write_complete(struct nfs_direct_req *dreq, struct inode *inode)
Trond Myklebustfad61492006-03-20 13:44:36 -0500706{
Fred Isaman1763da12012-04-20 14:47:57 -0400707 schedule_work(&dreq->work); /* Calls nfs_direct_write_schedule_work */
Trond Myklebustfad61492006-03-20 13:44:36 -0500708}
Fred Isaman1763da12012-04-20 14:47:57 -0400709
Fred Isaman1763da12012-04-20 14:47:57 -0400710static void nfs_direct_write_completion(struct nfs_pgio_header *hdr)
711{
712 struct nfs_direct_req *dreq = hdr->dreq;
713 struct nfs_commit_info cinfo;
Weston Andros Adamsonc65e6252014-06-09 11:48:36 -0400714 bool request_commit = false;
Fred Isaman1763da12012-04-20 14:47:57 -0400715 struct nfs_page *req = nfs_list_entry(hdr->pages.next);
716
717 if (test_bit(NFS_IOHDR_REDO, &hdr->flags))
718 goto out_put;
719
720 nfs_init_cinfo_from_dreq(&cinfo, dreq);
721
722 spin_lock(&dreq->lock);
723
724 if (test_bit(NFS_IOHDR_ERROR, &hdr->flags)) {
725 dreq->flags = 0;
726 dreq->error = hdr->error;
727 }
Weston Andros Adamsonc65e6252014-06-09 11:48:36 -0400728 if (dreq->error == 0) {
Fred Isaman1763da12012-04-20 14:47:57 -0400729 dreq->count += hdr->good_bytes;
Weston Andros Adamsonc65e6252014-06-09 11:48:36 -0400730 if (nfs_write_need_commit(hdr)) {
Fred Isaman1763da12012-04-20 14:47:57 -0400731 if (dreq->flags == NFS_ODIRECT_RESCHED_WRITES)
Weston Andros Adamsonc65e6252014-06-09 11:48:36 -0400732 request_commit = true;
Fred Isaman1763da12012-04-20 14:47:57 -0400733 else if (dreq->flags == 0) {
Weston Andros Adamson5002c582014-05-15 11:56:54 -0400734 nfs_direct_set_hdr_verf(dreq, hdr);
Weston Andros Adamsonc65e6252014-06-09 11:48:36 -0400735 request_commit = true;
Fred Isaman1763da12012-04-20 14:47:57 -0400736 dreq->flags = NFS_ODIRECT_DO_COMMIT;
737 } else if (dreq->flags == NFS_ODIRECT_DO_COMMIT) {
Weston Andros Adamsonc65e6252014-06-09 11:48:36 -0400738 request_commit = true;
739 if (nfs_direct_set_or_cmp_hdr_verf(dreq, hdr))
Weston Andros Adamson5002c582014-05-15 11:56:54 -0400740 dreq->flags =
741 NFS_ODIRECT_RESCHED_WRITES;
Fred Isaman1763da12012-04-20 14:47:57 -0400742 }
743 }
744 }
745 spin_unlock(&dreq->lock);
746
747 while (!list_empty(&hdr->pages)) {
Weston Andros Adamson2bfc6e52014-05-15 11:56:45 -0400748
Fred Isaman1763da12012-04-20 14:47:57 -0400749 req = nfs_list_entry(hdr->pages.next);
750 nfs_list_remove_request(req);
Weston Andros Adamsonc65e6252014-06-09 11:48:36 -0400751 if (request_commit) {
Trond Myklebust04277082012-05-09 13:54:53 -0400752 kref_get(&req->wb_kref);
Fred Isaman1763da12012-04-20 14:47:57 -0400753 nfs_mark_request_commit(req, hdr->lseg, &cinfo);
Fred Isaman1763da12012-04-20 14:47:57 -0400754 }
Trond Myklebust1d1afcb2012-05-09 14:04:55 -0400755 nfs_unlock_and_release_request(req);
Fred Isaman1763da12012-04-20 14:47:57 -0400756 }
757
758out_put:
759 if (put_dreq(dreq))
760 nfs_direct_write_complete(dreq, hdr->inode);
761 hdr->release(hdr);
762}
763
Trond Myklebust3e9e0ca32012-04-30 13:40:06 -0400764static void nfs_write_sync_pgio_error(struct list_head *head)
765{
766 struct nfs_page *req;
767
768 while (!list_empty(head)) {
769 req = nfs_list_entry(head->next);
770 nfs_list_remove_request(req);
Trond Myklebust1d1afcb2012-05-09 14:04:55 -0400771 nfs_unlock_and_release_request(req);
Trond Myklebust3e9e0ca32012-04-30 13:40:06 -0400772 }
773}
774
Fred Isaman1763da12012-04-20 14:47:57 -0400775static const struct nfs_pgio_completion_ops nfs_direct_write_completion_ops = {
Trond Myklebust3e9e0ca32012-04-30 13:40:06 -0400776 .error_cleanup = nfs_write_sync_pgio_error,
Fred Isaman1763da12012-04-20 14:47:57 -0400777 .init_hdr = nfs_direct_pgio_init,
778 .completion = nfs_direct_write_completion,
779};
780
Al Viro91f79c42014-03-21 04:58:33 -0400781
782/*
783 * NB: Return the value of the first error return code. Subsequent
784 * errors after the first one are ignored.
785 */
786/*
787 * For each wsize'd chunk of the user's buffer, dispatch an NFS WRITE
788 * operation. If nfs_writedata_alloc() or get_user_pages() fails,
789 * bail and stop sending more writes. Write length accounting is
790 * handled automatically by nfs_direct_write_result(). Otherwise, if
791 * no requests have been sent, just return an error.
792 */
Chuck Lever19f73782007-11-12 12:16:47 -0500793static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq,
Al Viro619d30b2014-03-04 21:53:33 -0500794 struct iov_iter *iter,
Al Viro91f79c42014-03-21 04:58:33 -0400795 loff_t pos)
Chuck Lever19f73782007-11-12 12:16:47 -0500796{
Fred Isaman1763da12012-04-20 14:47:57 -0400797 struct nfs_pageio_descriptor desc;
Trond Myklebust1d59d612012-05-31 12:22:33 -0400798 struct inode *inode = dreq->inode;
Chuck Lever19f73782007-11-12 12:16:47 -0500799 ssize_t result = 0;
800 size_t requested_bytes = 0;
Al Viro91f79c42014-03-21 04:58:33 -0400801 size_t wsize = max_t(size_t, NFS_SERVER(inode)->wsize, PAGE_SIZE);
Chuck Lever19f73782007-11-12 12:16:47 -0500802
Christoph Hellwiga20c93e2014-04-16 15:07:21 +0200803 nfs_pageio_init_write(&desc, inode, FLUSH_COND_STABLE, false,
Fred Isaman1763da12012-04-20 14:47:57 -0400804 &nfs_direct_write_completion_ops);
805 desc.pg_dreq = dreq;
Chuck Lever19f73782007-11-12 12:16:47 -0500806 get_dreq(dreq);
Trond Myklebust1d59d612012-05-31 12:22:33 -0400807 atomic_inc(&inode->i_dio_count);
Chuck Lever19f73782007-11-12 12:16:47 -0500808
Al Viro91f79c42014-03-21 04:58:33 -0400809 NFS_I(inode)->write_io += iov_iter_count(iter);
810 while (iov_iter_count(iter)) {
811 struct page **pagevec;
812 size_t bytes;
813 size_t pgbase;
814 unsigned npages, i;
815
816 result = iov_iter_get_pages_alloc(iter, &pagevec,
817 wsize, &pgbase);
Chuck Lever19f73782007-11-12 12:16:47 -0500818 if (result < 0)
819 break;
Al Viro91f79c42014-03-21 04:58:33 -0400820
821 bytes = result;
822 iov_iter_advance(iter, bytes);
823 npages = (result + pgbase + PAGE_SIZE - 1) / PAGE_SIZE;
824 for (i = 0; i < npages; i++) {
825 struct nfs_page *req;
826 unsigned int req_len = min_t(size_t, bytes, PAGE_SIZE - pgbase);
827
Linus Torvalds16b90572014-06-12 10:30:18 -0700828 req = nfs_create_request(dreq->ctx, pagevec[i], NULL,
Al Viro91f79c42014-03-21 04:58:33 -0400829 pgbase, req_len);
830 if (IS_ERR(req)) {
831 result = PTR_ERR(req);
832 break;
833 }
834 nfs_lock_request(req);
835 req->wb_index = pos >> PAGE_SHIFT;
836 req->wb_offset = pos & ~PAGE_MASK;
837 if (!nfs_pageio_add_request(&desc, req)) {
838 result = desc.pg_error;
839 nfs_unlock_and_release_request(req);
840 break;
841 }
842 pgbase = 0;
843 bytes -= req_len;
844 requested_bytes += req_len;
845 pos += req_len;
846 dreq->bytes_left -= req_len;
847 }
848 nfs_direct_release_pages(pagevec, npages);
849 kvfree(pagevec);
850 if (result < 0)
Chuck Lever19f73782007-11-12 12:16:47 -0500851 break;
Chuck Lever19f73782007-11-12 12:16:47 -0500852 }
Fred Isaman1763da12012-04-20 14:47:57 -0400853 nfs_pageio_complete(&desc);
Chuck Lever19f73782007-11-12 12:16:47 -0500854
Chuck Lever839f7ad2011-01-21 15:54:57 +0000855 /*
856 * If no bytes were started, return the error, and let the
857 * generic layer handle the completion.
858 */
859 if (requested_bytes == 0) {
Trond Myklebust1d59d612012-05-31 12:22:33 -0400860 inode_dio_done(inode);
Chuck Lever839f7ad2011-01-21 15:54:57 +0000861 nfs_direct_req_release(dreq);
862 return result < 0 ? result : -EIO;
863 }
864
Chuck Lever19f73782007-11-12 12:16:47 -0500865 if (put_dreq(dreq))
866 nfs_direct_write_complete(dreq, dreq->inode);
Chuck Lever839f7ad2011-01-21 15:54:57 +0000867 return 0;
Chuck Lever19f73782007-11-12 12:16:47 -0500868}
869
Linus Torvalds1da177e2005-04-16 15:20:36 -0700870/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700871 * nfs_file_direct_write - file direct write operation for NFS files
872 * @iocb: target I/O control block
Al Viro619d30b2014-03-04 21:53:33 -0500873 * @iter: vector of user buffers from which to write data
Chuck Lever88467052006-03-20 13:44:34 -0500874 * @pos: byte offset in file where writing starts
Linus Torvalds1da177e2005-04-16 15:20:36 -0700875 *
876 * We use this function for direct writes instead of calling
877 * generic_file_aio_write() in order to avoid taking the inode
878 * semaphore and updating the i_size. The NFS server will set
879 * the new i_size and this client must read the updated size
880 * back into its cache. We let the server do generic write
881 * parameter checking and report problems.
882 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700883 * We eliminate local atime updates, see direct read above.
884 *
885 * We avoid unnecessary page cache invalidations for normal cached
886 * readers of this file.
887 *
888 * Note that O_APPEND is not supported for NFS direct writes, as there
889 * is no atomic O_APPEND write facility in the NFS protocol.
890 */
Al Viro619d30b2014-03-04 21:53:33 -0500891ssize_t nfs_file_direct_write(struct kiocb *iocb, struct iov_iter *iter,
Martin K. Petersene19a8a02014-10-14 09:00:44 -0600892 loff_t pos)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700893{
Christoph Hellwig22cd1bf2013-11-14 08:50:32 -0800894 ssize_t result = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700895 struct file *file = iocb->ki_filp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700896 struct address_space *mapping = file->f_mapping;
Christoph Hellwig22cd1bf2013-11-14 08:50:32 -0800897 struct inode *inode = mapping->host;
898 struct nfs_direct_req *dreq;
899 struct nfs_lock_context *l_ctx;
Christoph Hellwiga9ab5e82013-11-14 08:50:34 -0800900 loff_t end;
Al Viroa6cbcd42014-03-04 22:38:00 -0500901 size_t count = iov_iter_count(iter);
Christoph Hellwiga9ab5e82013-11-14 08:50:34 -0800902 end = (pos + count - 1) >> PAGE_CACHE_SHIFT;
903
Chuck Leverc216fd72007-11-12 12:16:52 -0500904 nfs_add_stats(mapping->host, NFSIOS_DIRECTWRITTENBYTES, count);
905
Al Viro6de14722013-09-16 10:53:17 -0400906 dfprintk(FILE, "NFS: direct write(%pD2, %zd@%Ld)\n",
907 file, count, (long long) pos);
Badari Pulavarty027445c2006-09-30 23:28:46 -0700908
Christoph Hellwig22cd1bf2013-11-14 08:50:32 -0800909 result = generic_write_checks(file, &pos, &count, 0);
910 if (result)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700911 goto out;
Chuck Leverce1a8e62005-11-30 18:08:17 -0500912
Christoph Hellwig22cd1bf2013-11-14 08:50:32 -0800913 result = -EINVAL;
Chuck Leverce1a8e62005-11-30 18:08:17 -0500914 if ((ssize_t) count < 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700915 goto out;
Christoph Hellwig22cd1bf2013-11-14 08:50:32 -0800916 result = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700917 if (!count)
918 goto out;
Chuck Leverce1a8e62005-11-30 18:08:17 -0500919
Christoph Hellwiga9ab5e82013-11-14 08:50:34 -0800920 mutex_lock(&inode->i_mutex);
921
Christoph Hellwig22cd1bf2013-11-14 08:50:32 -0800922 result = nfs_sync_mapping(mapping);
923 if (result)
Christoph Hellwiga9ab5e82013-11-14 08:50:34 -0800924 goto out_unlock;
925
926 if (mapping->nrpages) {
927 result = invalidate_inode_pages2_range(mapping,
928 pos >> PAGE_CACHE_SHIFT, end);
929 if (result)
930 goto out_unlock;
931 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700932
Konstantin Khlebnikov7ec10f22011-02-22 00:28:34 +0300933 task_io_account_write(count);
934
Christoph Hellwig22cd1bf2013-11-14 08:50:32 -0800935 result = -ENOMEM;
936 dreq = nfs_direct_req_alloc();
937 if (!dreq)
Christoph Hellwiga9ab5e82013-11-14 08:50:34 -0800938 goto out_unlock;
Chuck Lever9eafa8c2006-03-20 13:44:33 -0500939
Christoph Hellwig22cd1bf2013-11-14 08:50:32 -0800940 dreq->inode = inode;
941 dreq->bytes_left = count;
942 dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp));
943 l_ctx = nfs_get_lock_context(dreq->ctx);
944 if (IS_ERR(l_ctx)) {
945 result = PTR_ERR(l_ctx);
946 goto out_release;
Fred Isaman1763da12012-04-20 14:47:57 -0400947 }
Christoph Hellwig22cd1bf2013-11-14 08:50:32 -0800948 dreq->l_ctx = l_ctx;
949 if (!is_sync_kiocb(iocb))
950 dreq->iocb = iocb;
951
Al Viro91f79c42014-03-21 04:58:33 -0400952 result = nfs_direct_write_schedule_iovec(dreq, iter, pos);
Christoph Hellwiga9ab5e82013-11-14 08:50:34 -0800953
954 if (mapping->nrpages) {
955 invalidate_inode_pages2_range(mapping,
956 pos >> PAGE_CACHE_SHIFT, end);
957 }
958
959 mutex_unlock(&inode->i_mutex);
960
Christoph Hellwig22cd1bf2013-11-14 08:50:32 -0800961 if (!result) {
962 result = nfs_direct_wait(dreq);
963 if (result > 0) {
964 struct inode *inode = mapping->host;
965
966 iocb->ki_pos = pos + result;
967 spin_lock(&inode->i_lock);
968 if (i_size_read(inode) < iocb->ki_pos)
969 i_size_write(inode, iocb->ki_pos);
970 spin_unlock(&inode->i_lock);
971 }
972 }
Christoph Hellwiga9ab5e82013-11-14 08:50:34 -0800973 nfs_direct_req_release(dreq);
974 return result;
975
Christoph Hellwig22cd1bf2013-11-14 08:50:32 -0800976out_release:
977 nfs_direct_req_release(dreq);
Christoph Hellwiga9ab5e82013-11-14 08:50:34 -0800978out_unlock:
979 mutex_unlock(&inode->i_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700980out:
Christoph Hellwig22cd1bf2013-11-14 08:50:32 -0800981 return result;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700982}
983
Chuck Lever88467052006-03-20 13:44:34 -0500984/**
985 * nfs_init_directcache - create a slab cache for nfs_direct_req structures
986 *
987 */
David Howellsf7b422b2006-06-09 09:34:33 -0400988int __init nfs_init_directcache(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700989{
990 nfs_direct_cachep = kmem_cache_create("nfs_direct_cache",
991 sizeof(struct nfs_direct_req),
Paul Jacksonfffb60f2006-03-24 03:16:06 -0800992 0, (SLAB_RECLAIM_ACCOUNT|
993 SLAB_MEM_SPREAD),
Paul Mundt20c2df82007-07-20 10:11:58 +0900994 NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700995 if (nfs_direct_cachep == NULL)
996 return -ENOMEM;
997
998 return 0;
999}
1000
Chuck Lever88467052006-03-20 13:44:34 -05001001/**
David Howellsf7b422b2006-06-09 09:34:33 -04001002 * nfs_destroy_directcache - destroy the slab cache for nfs_direct_req structures
Chuck Lever88467052006-03-20 13:44:34 -05001003 *
1004 */
David Brownell266bee82006-06-27 12:59:15 -07001005void nfs_destroy_directcache(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001006{
Alexey Dobriyan1a1d92c2006-09-27 01:49:40 -07001007 kmem_cache_destroy(nfs_direct_cachep);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001008}