|
| 1 | +// SPDX-License-Identifier: GPL-2.0 |
| 2 | +/* |
| 3 | + * linux/drivers/staging/erofs/data.c |
| 4 | + * |
| 5 | + * Copyright (C) 2017-2018 HUAWEI, Inc. |
| 6 | + * http://www.huawei.com/ |
| 7 | + * Created by Gao Xiang <gaoxiang25@huawei.com> |
| 8 | + * |
| 9 | + * This file is subject to the terms and conditions of the GNU General Public |
| 10 | + * License. See the file COPYING in the main directory of the Linux |
| 11 | + * distribution for more details. |
| 12 | + */ |
| 13 | +#include "internal.h" |
| 14 | +#include <linux/prefetch.h> |
| 15 | + |
| 16 | +static inline void read_endio(struct bio *bio) |
| 17 | +{ |
| 18 | + int i; |
| 19 | + struct bio_vec *bvec; |
| 20 | + const blk_status_t err = bio->bi_status; |
| 21 | + |
| 22 | + bio_for_each_segment_all(bvec, bio, i) { |
| 23 | + struct page *page = bvec->bv_page; |
| 24 | + |
| 25 | + /* page is already locked */ |
| 26 | + BUG_ON(PageUptodate(page)); |
| 27 | + |
| 28 | + if (unlikely(err)) |
| 29 | + SetPageError(page); |
| 30 | + else |
| 31 | + SetPageUptodate(page); |
| 32 | + |
| 33 | + unlock_page(page); |
| 34 | + /* page could be reclaimed now */ |
| 35 | + } |
| 36 | + bio_put(bio); |
| 37 | +} |
| 38 | + |
| 39 | +static void __submit_bio(struct bio *bio, unsigned op, unsigned op_flags) |
| 40 | +{ |
| 41 | + bio_set_op_attrs(bio, op, op_flags); |
| 42 | + submit_bio(bio); |
| 43 | +} |
| 44 | + |
| 45 | +static struct bio *prepare_bio(struct super_block *sb, |
| 46 | + erofs_blk_t blkaddr, unsigned nr_pages) |
| 47 | +{ |
| 48 | + struct bio *bio = bio_alloc(GFP_NOIO | __GFP_NOFAIL, nr_pages); |
| 49 | + |
| 50 | + BUG_ON(bio == NULL); |
| 51 | + |
| 52 | + bio->bi_end_io = read_endio; |
| 53 | + bio_set_dev(bio, sb->s_bdev); |
| 54 | + bio->bi_iter.bi_sector = blkaddr << LOG_SECTORS_PER_BLOCK; |
| 55 | + |
| 56 | + return bio; |
| 57 | +} |
| 58 | + |
| 59 | +/* prio -- true is used for dir */ |
| 60 | +struct page *erofs_get_meta_page(struct super_block *sb, |
| 61 | + erofs_blk_t blkaddr, bool prio) |
| 62 | +{ |
| 63 | + struct inode *bd_inode = sb->s_bdev->bd_inode; |
| 64 | + struct address_space *mapping = bd_inode->i_mapping; |
| 65 | + struct page *page; |
| 66 | + |
| 67 | +repeat: |
| 68 | + page = find_or_create_page(mapping, blkaddr, |
| 69 | + /* |
| 70 | + * Prefer looping in the allocator rather than here, |
| 71 | + * at least that code knows what it's doing. |
| 72 | + */ |
| 73 | + mapping_gfp_constraint(mapping, ~__GFP_FS) | __GFP_NOFAIL); |
| 74 | + |
| 75 | + BUG_ON(!page || !PageLocked(page)); |
| 76 | + |
| 77 | + if (!PageUptodate(page)) { |
| 78 | + struct bio *bio; |
| 79 | + int err; |
| 80 | + |
| 81 | + bio = prepare_bio(sb, blkaddr, 1); |
| 82 | + err = bio_add_page(bio, page, PAGE_SIZE, 0); |
| 83 | + BUG_ON(err != PAGE_SIZE); |
| 84 | + |
| 85 | + __submit_bio(bio, REQ_OP_READ, |
| 86 | + REQ_META | (prio ? REQ_PRIO : 0)); |
| 87 | + |
| 88 | + lock_page(page); |
| 89 | + |
| 90 | + /* the page has been truncated by others? */ |
| 91 | + if (unlikely(page->mapping != mapping)) { |
| 92 | + unlock_page(page); |
| 93 | + put_page(page); |
| 94 | + goto repeat; |
| 95 | + } |
| 96 | + |
| 97 | + /* more likely a read error */ |
| 98 | + if (unlikely(!PageUptodate(page))) { |
| 99 | + unlock_page(page); |
| 100 | + put_page(page); |
| 101 | + |
| 102 | + page = ERR_PTR(-EIO); |
| 103 | + } |
| 104 | + } |
| 105 | + return page; |
| 106 | +} |
| 107 | + |
| 108 | +static int erofs_map_blocks_flatmode(struct inode *inode, |
| 109 | + struct erofs_map_blocks *map, |
| 110 | + int flags) |
| 111 | +{ |
| 112 | + erofs_blk_t nblocks, lastblk; |
| 113 | + u64 offset = map->m_la; |
| 114 | + struct erofs_vnode *vi = EROFS_V(inode); |
| 115 | + |
| 116 | + BUG_ON(is_inode_layout_compression(inode)); |
| 117 | + |
| 118 | + nblocks = DIV_ROUND_UP(inode->i_size, PAGE_SIZE); |
| 119 | + lastblk = nblocks - is_inode_layout_inline(inode); |
| 120 | + |
| 121 | + if (unlikely(offset >= inode->i_size)) { |
| 122 | + /* leave out-of-bound access unmapped */ |
| 123 | + map->m_flags = 0; |
| 124 | + map->m_plen = 0; |
| 125 | + goto out; |
| 126 | + } |
| 127 | + |
| 128 | + /* there is no hole in flatmode */ |
| 129 | + map->m_flags = EROFS_MAP_MAPPED; |
| 130 | + |
| 131 | + if (offset < blknr_to_addr(lastblk)) { |
| 132 | + map->m_pa = blknr_to_addr(vi->raw_blkaddr) + map->m_la; |
| 133 | + map->m_plen = blknr_to_addr(lastblk) - offset; |
| 134 | + } else if (is_inode_layout_inline(inode)) { |
| 135 | + /* 2 - inode inline B: inode, [xattrs], inline last blk... */ |
| 136 | + struct erofs_sb_info *sbi = EROFS_SB(inode->i_sb); |
| 137 | + |
| 138 | + map->m_pa = iloc(sbi, vi->nid) + vi->inode_isize + |
| 139 | + vi->xattr_isize + erofs_blkoff(map->m_la); |
| 140 | + map->m_plen = inode->i_size - offset; |
| 141 | + |
| 142 | + /* inline data should locate in one meta block */ |
| 143 | + BUG_ON(erofs_blkoff(map->m_pa) + map->m_plen > PAGE_SIZE); |
| 144 | + map->m_flags |= EROFS_MAP_META; |
| 145 | + } else { |
| 146 | + errln("internal error @ nid: %llu (size %llu), m_la 0x%llx", |
| 147 | + vi->nid, inode->i_size, map->m_la); |
| 148 | + BUG(); |
| 149 | + } |
| 150 | + |
| 151 | +out: |
| 152 | + map->m_llen = map->m_plen; |
| 153 | + debugln("%s, m_la 0x%llx m_pa %llx m_len %llu", |
| 154 | + __func__, map->m_la, map->m_pa, map->m_plen); |
| 155 | + return 0; |
| 156 | +} |
| 157 | + |
| 158 | +int erofs_map_blocks(struct inode *inode, |
| 159 | + struct erofs_map_blocks *map, int flags) |
| 160 | +{ |
| 161 | + if (unlikely(is_inode_layout_compression(inode))) |
| 162 | + return -ENOTSUPP; |
| 163 | + |
| 164 | + return erofs_map_blocks_flatmode(inode, map, flags); |
| 165 | +} |
| 166 | + |
| 167 | +static inline struct bio *erofs_read_raw_page( |
| 168 | + struct bio *bio, |
| 169 | + struct address_space *mapping, |
| 170 | + struct page *page, |
| 171 | + erofs_off_t *last_block, |
| 172 | + unsigned nblocks, |
| 173 | + bool ra) |
| 174 | +{ |
| 175 | + struct inode *inode = mapping->host; |
| 176 | + erofs_off_t current_block = (erofs_off_t)page->index; |
| 177 | + int err; |
| 178 | + |
| 179 | + BUG_ON(!nblocks); |
| 180 | + |
| 181 | + if (PageUptodate(page)) { |
| 182 | + err = 0; |
| 183 | + goto has_updated; |
| 184 | + } |
| 185 | + |
| 186 | + if (cleancache_get_page(page) == 0) { |
| 187 | + err = 0; |
| 188 | + SetPageUptodate(page); |
| 189 | + goto has_updated; |
| 190 | + } |
| 191 | + |
| 192 | + /* note that for readpage case, bio also equals to NULL */ |
| 193 | + if (bio != NULL && |
| 194 | + /* not continuous */ |
| 195 | + *last_block + 1 != current_block) { |
| 196 | +submit_bio_retry: |
| 197 | + __submit_bio(bio, REQ_OP_READ, 0); |
| 198 | + bio = NULL; |
| 199 | + } |
| 200 | + |
| 201 | + if (bio == NULL) { |
| 202 | + struct erofs_map_blocks map = { |
| 203 | + .m_la = blknr_to_addr(current_block), |
| 204 | + }; |
| 205 | + |
| 206 | + err = erofs_map_blocks(inode, &map, EROFS_GET_BLOCKS_RAW); |
| 207 | + if (unlikely(err)) |
| 208 | + goto err_out; |
| 209 | + |
| 210 | + /* zero out the holed page */ |
| 211 | + if (unlikely(!(map.m_flags & EROFS_MAP_MAPPED))) { |
| 212 | + zero_user_segment(page, 0, PAGE_SIZE); |
| 213 | + SetPageUptodate(page); |
| 214 | + |
| 215 | + /* imply err = 0, see erofs_map_blocks */ |
| 216 | + goto has_updated; |
| 217 | + } |
| 218 | + |
| 219 | + /* for RAW access mode, m_plen must be equal to m_llen */ |
| 220 | + BUG_ON(map.m_plen != map.m_llen); |
| 221 | + |
| 222 | + /* deal with inline page */ |
| 223 | + if (map.m_flags & EROFS_MAP_META) { |
| 224 | + void *vsrc, *vto; |
| 225 | + struct page *ipage; |
| 226 | + |
| 227 | + BUG_ON(map.m_plen > PAGE_SIZE); |
| 228 | + |
| 229 | + ipage = erofs_get_meta_page(inode->i_sb, |
| 230 | + erofs_blknr(map.m_pa), 0); |
| 231 | + |
| 232 | + if (IS_ERR(ipage)) { |
| 233 | + err = PTR_ERR(ipage); |
| 234 | + goto err_out; |
| 235 | + } |
| 236 | + |
| 237 | + vsrc = kmap_atomic(ipage); |
| 238 | + vto = kmap_atomic(page); |
| 239 | + memcpy(vto, vsrc + erofs_blkoff(map.m_pa), map.m_plen); |
| 240 | + memset(vto + map.m_plen, 0, PAGE_SIZE - map.m_plen); |
| 241 | + kunmap_atomic(vto); |
| 242 | + kunmap_atomic(vsrc); |
| 243 | + flush_dcache_page(page); |
| 244 | + |
| 245 | + SetPageUptodate(page); |
| 246 | + /* TODO: could we unlock the page earlier? */ |
| 247 | + unlock_page(ipage); |
| 248 | + put_page(ipage); |
| 249 | + |
| 250 | + /* imply err = 0, see erofs_map_blocks */ |
| 251 | + goto has_updated; |
| 252 | + } |
| 253 | + |
| 254 | + /* pa must be block-aligned for raw reading */ |
| 255 | + BUG_ON(erofs_blkoff(map.m_pa) != 0); |
| 256 | + |
| 257 | + /* max # of continuous pages */ |
| 258 | + if (nblocks > DIV_ROUND_UP(map.m_plen, PAGE_SIZE)) |
| 259 | + nblocks = DIV_ROUND_UP(map.m_plen, PAGE_SIZE); |
| 260 | + if (nblocks > BIO_MAX_PAGES) |
| 261 | + nblocks = BIO_MAX_PAGES; |
| 262 | + |
| 263 | + bio = prepare_bio(inode->i_sb, erofs_blknr(map.m_pa), nblocks); |
| 264 | + } |
| 265 | + |
| 266 | + err = bio_add_page(bio, page, PAGE_SIZE, 0); |
| 267 | + /* out of the extent or bio is full */ |
| 268 | + if (err < PAGE_SIZE) |
| 269 | + goto submit_bio_retry; |
| 270 | + |
| 271 | + *last_block = current_block; |
| 272 | + |
| 273 | + /* shift in advance in case of it followed by too many gaps */ |
| 274 | + if (unlikely(bio->bi_vcnt >= bio->bi_max_vecs)) { |
| 275 | + /* err should reassign to 0 after submitting */ |
| 276 | + err = 0; |
| 277 | + goto submit_bio_out; |
| 278 | + } |
| 279 | + |
| 280 | + return bio; |
| 281 | + |
| 282 | +err_out: |
| 283 | + /* for sync reading, set page error immediately */ |
| 284 | + if (!ra) { |
| 285 | + SetPageError(page); |
| 286 | + ClearPageUptodate(page); |
| 287 | + } |
| 288 | +has_updated: |
| 289 | + unlock_page(page); |
| 290 | + |
| 291 | + /* if updated manually, continuous pages has a gap */ |
| 292 | + if (bio != NULL) |
| 293 | +submit_bio_out: |
| 294 | + __submit_bio(bio, REQ_OP_READ, 0); |
| 295 | + |
| 296 | + return unlikely(err) ? ERR_PTR(err) : NULL; |
| 297 | +} |
| 298 | + |
| 299 | +/* |
| 300 | + * since we dont have write or truncate flows, so no inode |
| 301 | + * locking needs to be held at the moment. |
| 302 | + */ |
| 303 | +static int erofs_raw_access_readpage(struct file *file, struct page *page) |
| 304 | +{ |
| 305 | + erofs_off_t last_block; |
| 306 | + struct bio *bio; |
| 307 | + |
| 308 | + bio = erofs_read_raw_page(NULL, page->mapping, |
| 309 | + page, &last_block, 1, false); |
| 310 | + |
| 311 | + if (IS_ERR(bio)) |
| 312 | + return PTR_ERR(bio); |
| 313 | + |
| 314 | + BUG_ON(bio != NULL); /* since we have only one bio -- must be NULL */ |
| 315 | + return 0; |
| 316 | +} |
| 317 | + |
| 318 | +static int erofs_raw_access_readpages(struct file *filp, |
| 319 | + struct address_space *mapping, |
| 320 | + struct list_head *pages, unsigned nr_pages) |
| 321 | +{ |
| 322 | + erofs_off_t last_block; |
| 323 | + struct bio *bio = NULL; |
| 324 | + gfp_t gfp = readahead_gfp_mask(mapping); |
| 325 | + |
| 326 | + for (; nr_pages; --nr_pages) { |
| 327 | + struct page *page = list_entry(pages->prev, struct page, lru); |
| 328 | + |
| 329 | + prefetchw(&page->flags); |
| 330 | + list_del(&page->lru); |
| 331 | + |
| 332 | + if (!add_to_page_cache_lru(page, mapping, page->index, gfp)) { |
| 333 | + bio = erofs_read_raw_page(bio, mapping, page, |
| 334 | + &last_block, nr_pages, true); |
| 335 | + |
| 336 | + /* all the page errors are ignored when readahead */ |
| 337 | + if (IS_ERR(bio)) { |
| 338 | + pr_err("%s, readahead error at page %lu of nid %llu\n", |
| 339 | + __func__, page->index, |
| 340 | + EROFS_V(mapping->host)->nid); |
| 341 | + |
| 342 | + bio = NULL; |
| 343 | + } |
| 344 | + } |
| 345 | + |
| 346 | + /* pages could still be locked */ |
| 347 | + put_page(page); |
| 348 | + } |
| 349 | + BUG_ON(!list_empty(pages)); |
| 350 | + |
| 351 | + /* the rare case (end in gaps) */ |
| 352 | + if (unlikely(bio != NULL)) |
| 353 | + __submit_bio(bio, REQ_OP_READ, 0); |
| 354 | + return 0; |
| 355 | +} |
| 356 | + |
| 357 | +/* for uncompressed (aligned) files and raw access for other files */ |
| 358 | +const struct address_space_operations erofs_raw_access_aops = { |
| 359 | + .readpage = erofs_raw_access_readpage, |
| 360 | + .readpages = erofs_raw_access_readpages, |
| 361 | +}; |
| 362 | + |
0 commit comments