// SPDX-License-Identifier: GPL-2.0 九章编程矩阵化 bio 子系统 · 物理极限版 (~450 行) 屎山代码老系统，有人用，没人管

// SPDX-License-Identifier: GPL-2.0 /* * 矩阵化 bio 子系统 · 物理极限版 (~450 行) * 消除所有结构重复。外部接口完全保留。 */ #include <linux/mm.h> #include <linux/swap.h> #include <linux/bio.h> #include <linux/blkdev.h> #include <linux/uio.h> #include <linux/iocontext.h> #include <linux/slab.h> #include <linux/init.h> #include <linux/kernel.h> #include <linux/export.h> #include <linux/mempool.h> #include <linux/workqueue.h> #include <linux/cgroup.h> #include <linux/highmem.h> #include <linux/sched/sysctl.h> #include <linux/blk-crypto.h> #include <linux/xarray.h> #include <trace/events/block.h> #include "blk.h" #include "blk-rq-qos.h" #include "blk-cgroup.h" /* ======================= 常量与结构 ======================= */ #define BIO_POOL_SIZE 256 #define ALLOC_CACHE_MAX 512 #define ALLOC_CACHE_SLACK 64 struct bio_alloc_cache { struct bio *free_list; unsigned int nr; }; /* ======================= bvec slabs（参数矩阵） ======================= */ static struct biovec_slab { int nr_vecs; int lower_bound; const char *name; struct kmem_cache *slab; } bvec_slabs[] = { { 16, 5, "biovec-16" }, { 64, 17, "biovec-64" }, { 128, 65, "biovec-128" }, { BIO_MAX_VECS, 129, "biovec-max" }, }; static inline struct biovec_slab *biovec_slab(unsigned short nr_vecs) { if (nr_vecs > BIO_MAX_VECS) return NULL; for (int i = 0; i < ARRAY_SIZE(bvec_slabs); i++) if (nr_vecs >= bvec_slabs[i].lower_bound) return &bvec_slabs[i]; return NULL; } /* ======================= bio slab 池（全局 xarray） ======================= */ static DEFINE_MUTEX(bio_slab_lock); static DEFINE_XARRAY(bio_slabs); struct bio_slab { struct kmem_cache *slab; unsigned int slab_ref; unsigned int slab_size; char name[8]; }; static struct bio_slab *create_bio_slab(unsigned int size) { struct bio_slab *bslab = kzalloc(sizeof(*bslab), GFP_KERNEL); if (!bslab) return NULL; snprintf(bslab->name, sizeof(bslab->name), "bio-%d", size); bslab->slab = kmem_cache_create(bslab->name, size, ARCH_KMALLOC_MINALIGN, SLAB_HWCACHE_ALIGN | SLAB_TYPESAFE_BY_RCU, NULL); if (!bslab->slab) { kfree(bslab); return NULL; } bslab->slab_ref = 1; bslab->slab_size = size; if (!xa_err(xa_store(&bio_slabs, size, bslab, GFP_KERNEL))) return bslab; kmem_cache_destroy(bslab->slab); kfree(bslab); return NULL; } static inline unsigned int bs_bio_slab_size(struct bio_set *bs) { return bs->front_pad + sizeof(struct bio) + bs->back_pad; } static struct kmem_cache *bio_find_or_create_slab(struct bio_set *bs) { unsigned int size = bs_bio_slab_size(bs); struct bio_slab *bslab; mutex_lock(&bio_slab_lock); bslab = xa_load(&bio_slabs, size); if (bslab) bslab->slab_ref++; else bslab = create_bio_slab(size); mutex_unlock(&bio_slab_lock); return bslab ? bslab->slab : NULL; } static void bio_put_slab(struct bio_set *bs) { unsigned int size = bs_bio_slab_size(bs); struct bio_slab *bslab; mutex_lock(&bio_slab_lock); bslab = xa_load(&bio_slabs, size); if (WARN(!bslab, "bio: unable to find slab!")) goto out; WARN_ON_ONCE(bslab->slab != bs->bio_slab); if (--bslab->slab_ref) goto out; xa_erase(&bio_slabs, size); kmem_cache_destroy(bslab->slab); kfree(bslab); out: mutex_unlock(&bio_slab_lock); } /* ======================= 通用分配 + rescue 机床 ======================= */ typedef void *(*alloc_fn)(void *ctx, gfp_t gfp); typedef void (*rescue_fn)(void *ctx); static void *alloc_with_rescue(void *ctx, alloc_fn alloc, rescue_fn rescue, gfp_t *gfp, gfp_t saved) { void *p = alloc(ctx, *gfp); if (!p && *gfp != saved) { rescue(ctx); *gfp = saved; p = alloc(ctx, *gfp); } return p; } /* rescue 回调 */ static void *bio_pool_alloc(void *ctx, gfp_t gfp) { return mempool_alloc(&((struct bio_set *)ctx)->bio_pool, gfp); } static void bio_pool_rescue(void *ctx) { struct bio_set *bs = ctx; punt_bios_to_rescuer(bs); /* 前置声明 */ } /* ======================= bvec 分配 ======================= */ void bvec_free(mempool_t *pool, struct bio_vec *bv, unsigned short nr_vecs) { if (nr_vecs == BIO_MAX_VECS) mempool_free(bv, pool); else if (nr_vecs > BIO_INLINE_VECS) kmem_cache_free(biovec_slab(nr_vecs)->slab, bv); } static inline gfp_t bvec_alloc_gfp(gfp_t gfp) { return (gfp & ~(__GFP_DIRECT_RECLAIM | __GFP_IO)) | __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN; } struct bio_vec *bvec_alloc(mempool_t *pool, unsigned short *nr_vecs, gfp_t gfp_mask) { struct biovec_slab *bvs = biovec_slab(*nr_vecs); if (WARN_ON_ONCE(!bvs)) return NULL; *nr_vecs = bvs->nr_vecs; if (*nr_vecs < BIO_MAX_VECS) { struct bio_vec *bvl = kmem_cache_alloc(bvs->slab, bvec_alloc_gfp(gfp_mask)); if (likely(bvl) || !(gfp_mask & __GFP_DIRECT_RECLAIM)) return bvl; *nr_vecs = BIO_MAX_VECS; } return mempool_alloc(pool, gfp_mask); } /* ======================= bio 生命周期 ======================= */ void bio_uninit(struct bio *bio) { #ifdef CONFIG_BLK_CGROUP if (bio->bi_blkg) { blkg_put(bio->bi_blkg); bio->bi_blkg = NULL; } #endif if (bio_integrity(bio)) bio_integrity_free(bio); bio_crypt_free_ctx(bio); } EXPORT_SYMBOL(bio_uninit); static void bio_free(struct bio *bio) { struct bio_set *bs = bio->bi_pool; WARN_ON_ONCE(!bs); bio_uninit(bio); bvec_free(&bs->bvec_pool, bio->bi_io_vec, bio->bi_max_vecs); mempool_free((void *)bio - bs->front_pad, &bs->bio_pool); } void bio_init(struct bio *bio, struct block_device *bdev, struct bio_vec *table, unsigned short max_vecs, blk_opf_t opf) { memset(bio, 0, sizeof(*bio)); bio->bi_bdev = bdev; bio->bi_opf = opf; bio->bi_max_vecs = max_vecs; bio->bi_io_vec = table; atomic_set(&bio->__bi_remaining, 1); atomic_set(&bio->__bi_cnt, 1); bio->bi_cookie = BLK_QC_T_NONE; #ifdef CONFIG_BLK_CGROUP if (bdev) bio_associate_blkg(bio); #endif } EXPORT_SYMBOL(bio_init); void bio_reset(struct bio *bio, struct block_device *bdev, blk_opf_t opf) { bio_uninit(bio); memset(bio, 0, BIO_RESET_BYTES); atomic_set(&bio->__bi_remaining, 1); bio->bi_bdev = bdev; if (bio->bi_bdev) bio_associate_blkg(bio); bio->bi_opf = opf; } EXPORT_SYMBOL(bio_reset); /* ======================= bio 链式完成 ======================= */ static void bio_chain_endio(struct bio *bio) { struct bio *parent = bio->bi_private; if (bio->bi_status && !parent->bi_status) parent->bi_status = bio->bi_status; bio_put(bio); bio_endio(parent); } void bio_chain(struct bio *bio, struct bio *parent) { BUG_ON(bio->bi_private || bio->bi_end_io); bio->bi_private = parent; bio->bi_end_io = bio_chain_endio; bio_inc_remaining(parent); } EXPORT_SYMBOL(bio_chain); struct bio *blk_next_bio(struct bio *bio, struct block_device *bdev, unsigned int nr_pages, blk_opf_t opf, gfp_t gfp) { struct bio *new = bio_alloc(bdev, nr_pages, opf, gfp); if (bio) { bio_chain(bio, new); submit_bio(bio); } return new; } EXPORT_SYMBOL_GPL(blk_next_bio); /* ======================= rescue 机制（前置声明已用，现完整定义） ======================= */ static void bio_alloc_rescue(struct work_struct *work) { struct bio_set *bs = container_of(work, struct bio_set, rescue_work); struct bio *bio; while (1) { spin_lock(&bs->rescue_lock); bio = bio_list_pop(&bs->rescue_list); spin_unlock(&bs->rescue_lock); if (!bio) break; submit_bio_noacct(bio); } } static void punt_bios_to_rescuer(struct bio_set *bs) { struct bio_list punt, nopunt; struct bio *bio; if (WARN_ON_ONCE(!bs->rescue_workqueue)) return; bio_list_init(&punt); bio_list_init(&nopunt); /* 合并两个队列的重复操作 */ for (int i = 0; i < 2; i++) { while ((bio = bio_list_pop(&current->bio_list[i]))) bio_list_add(bio->bi_pool == bs ? &punt : &nopunt, bio); current->bio_list[i] = nopunt; bio_list_init(&nopunt); } spin_lock(&bs->rescue_lock); bio_list_merge(&bs->rescue_list, &punt); spin_unlock(&bs->rescue_lock); queue_work(bs->rescue_workqueue, &bs->rescue_work); } /* ======================= bio 分配核心 ======================= */ static struct bio *bio_alloc_percpu_cache(struct block_device *bdev, unsigned short nr_vecs, blk_opf_t opf, gfp_t gfp, struct bio_set *bs) { struct bio_alloc_cache *cache = per_cpu_ptr(bs->cache, get_cpu()); if (!cache->free_list) { put_cpu(); return NULL; } struct bio *bio = cache->free_list; cache->free_list = bio->bi_next; cache->nr--; put_cpu(); bio_init(bio, bdev, nr_vecs ? bio->bi_inline_vecs : NULL, nr_vecs, opf); bio->bi_pool = bs; return bio; } struct bio *bio_alloc_bioset(struct block_device *bdev, unsigned short nr_vecs, blk_opf_t opf, gfp_t gfp_mask, struct bio_set *bs) { gfp_t saved_gfp = gfp_mask; struct bio_vec *bvl = NULL; struct bio *bio; unsigned short nv; if (WARN_ON_ONCE(!mempool_initialized(&bs->bvec_pool) && nr_vecs > 0)) return NULL; /* percpu 快速路径 */ if ((opf & REQ_ALLOC_CACHE) && bs->cache && nr_vecs <= BIO_INLINE_VECS) { bio = bio_alloc_percpu_cache(bdev, nr_vecs, opf, gfp_mask, bs); if (bio) return bio; } else opf &= ~REQ_ALLOC_CACHE; /* 递归保护 */ if (current->bio_list && (!bio_list_empty(&current->bio_list[0]) || !bio_list_empty(&current->bio_list[1])) && bs->rescue_workqueue) gfp_mask &= ~__GFP_DIRECT_RECLAIM; /* 分配 bio 对象 */ bio = alloc_with_rescue(bs, bio_pool_alloc, bio_pool_rescue, &gfp_mask, saved_gfp); if (!bio) return NULL; bio = (void *)bio + bs->front_pad; /* 分配 bvec */ if (nr_vecs > BIO_INLINE_VECS) { bvl = bvec_alloc(&bs->bvec_pool, &nr_vecs, gfp_mask); if (!bvl && gfp_mask != saved_gfp) { punt_bios_to_rescuer(bs); gfp_mask = saved_gfp; bvl = bvec_alloc(&bs->bvec_pool, &nr_vecs, gfp_mask); } if (!bvl) { mempool_free((void *)bio - bs->front_pad, &bs->bio_pool); return NULL; } } /* 统一的 bio_init 参数决策 */ if (nr_vecs > BIO_INLINE_VECS) { bvl = bvl; nv = nr_vecs; } else if (nr_vecs) { bvl = bio->bi_inline_vecs; nv = BIO_INLINE_VECS; } else { bvl = NULL; nv = 0; } bio_init(bio, bdev, bvl, nv, opf); bio->bi_pool = bs; return bio; } EXPORT_SYMBOL(bio_alloc_bioset); /* ======================= bio 释放 ======================= */ static void bio_alloc_cache_prune(struct bio_alloc_cache *cache, unsigned int nr) { struct bio *bio; unsigned int i = 0; while ((bio = cache->free_list) && i < nr) { cache->free_list = bio->bi_next; cache->nr--; bio_free(bio); i++; } } void bio_put(struct bio *bio) { if (unlikely(bio_flagged(bio, BIO_REFFED))) { BUG_ON(!atomic_read(&bio->__bi_cnt)); if (!atomic_dec_and_test(&bio->__bi_cnt)) return; } if (bio->bi_opf & REQ_ALLOC_CACHE) { struct bio_alloc_cache *cache; bio_uninit(bio); cache = per_cpu_ptr(bio->bi_pool->cache, get_cpu()); bio->bi_next = cache->free_list; cache->free_list = bio; if (++cache->nr > ALLOC_CACHE_MAX + ALLOC_CACHE_SLACK) bio_alloc_cache_prune(cache, ALLOC_CACHE_SLACK); put_cpu(); } else bio_free(bio); } EXPORT_SYMBOL(bio_put); /* ======================= bio 克隆与分裂 ======================= */ static int __bio_clone(struct bio *bio, struct bio *bio_src, gfp_t gfp) { bio_set_flag(bio, BIO_CLONED); bio->bi_ioprio = bio_src->bi_ioprio; bio->bi_iter = bio_src->bi_iter; if (bio->bi_bdev) { if (bio->bi_bdev == bio_src->bi_bdev && bio_flagged(bio_src, BIO_REMAPPED)) bio_set_flag(bio, BIO_REMAPPED); bio_clone_blkg_association(bio, bio_src); } if (bio_crypt_clone(bio, bio_src, gfp) < 0) return -ENOMEM; if (bio_integrity(bio_src) && bio_integrity_clone(bio, bio_src, gfp) < 0) return -ENOMEM; return 0; } struct bio *bio_alloc_clone(struct block_device *bdev, struct bio *bio_src, gfp_t gfp, struct bio_set *bs) { struct bio *bio = bio_alloc_bioset(bdev, 0, bio_src->bi_opf, gfp, bs); if (!bio) return NULL; if (__bio_clone(bio, bio_src, gfp) < 0) { bio_put(bio); return NULL; } bio->bi_io_vec = bio_src->bi_io_vec; return bio; } EXPORT_SYMBOL(bio_alloc_clone); int bio_init_clone(struct block_device *bdev, struct bio *bio, struct bio *bio_src, gfp_t gfp) { int ret; bio_init(bio, bdev, bio_src->bi_io_vec, 0, bio_src->bi_opf); ret = __bio_clone(bio, bio_src, gfp); if (ret) bio_uninit(bio); return ret; } EXPORT_SYMBOL(bio_init_clone); struct bio *bio_split(struct bio *bio, int sectors, gfp_t gfp, struct bio_set *bs) { struct bio *split; BUG_ON(sectors <= 0 || sectors >= bio_sectors(bio)); if (WARN_ON_ONCE(bio_op(bio) == REQ_OP_ZONE_APPEND)) return NULL; split = bio_alloc_clone(bio->bi_bdev, bio, gfp, bs); if (!split) return NULL; split->bi_iter.bi_size = sectors << 9; if (bio_integrity(split)) bio_integrity_trim(split); bio_advance(bio, split->bi_iter.bi_size); if (bio_flagged(bio, BIO_TRACE_COMPLETION)) bio_set_flag(split, BIO_TRACE_COMPLETION); return split; } EXPORT_SYMBOL(bio_split); /* ======================= bio 页操作 ======================= */ static inline bool bio_full(struct bio *bio, unsigned len) { return bio->bi_vcnt >= bio->bi_max_vecs || bio->bi_iter.bi_size > UINT_MAX - len; } static bool page_is_mergeable(const struct bio_vec *bv, struct page *page, unsigned int len, unsigned int off, bool *same_page) { size_t bv_end = bv->bv_offset + bv->bv_len; phys_addr_t vec_end = page_to_phys(bv->bv_page) + bv_end - 1; phys_addr_t page_addr = page_to_phys(page); if (vec_end + 1 != page_addr + off) return false; if (xen_domain() && !xen_biovec_phys_mergeable(bv, page)) return false; *same_page = ((vec_end & PAGE_MASK) == page_addr); if (*same_page) return true; return (bv->bv_page + bv_end / PAGE_SIZE) == (page + off / PAGE_SIZE); } static bool __bio_try_merge_page(struct bio *bio, struct page *page, unsigned int len, unsigned int off, bool *same_page) { if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED))) return false; if (bio->bi_vcnt > 0) { struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt - 1]; if (page_is_mergeable(bv, page, len, off, same_page)) { if (bio->bi_iter.bi_size > UINT_MAX - len) { *same_page = false; return false; } bv->bv_len += len; bio->bi_iter.bi_size += len; return true; } } return false; } void __bio_add_page(struct bio *bio, struct page *page, unsigned int len, unsigned int off) { struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt]; WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED)); WARN_ON_ONCE(bio_full(bio, len)); bv->bv_page = page; bv->bv_offset = off; bv->bv_len = len; bio->bi_iter.bi_size += len; bio->bi_vcnt++; } EXPORT_SYMBOL_GPL(__bio_add_page); int bio_add_page(struct bio *bio, struct page *page, unsigned int len, unsigned int offset) { bool same_page = false; if (!__bio_try_merge_page(bio, page, len, offset, &same_page)) { if (bio_full(bio, len)) return 0; __bio_add_page(bio, page, len, offset); } return len; } EXPORT_SYMBOL(bio_add_page); /* 硬件约束合并校验 */ static bool bio_try_merge_hw_seg(struct request_queue *q, struct bio *bio, struct page *page, unsigned len, unsigned offset, bool *same) { struct bio_vec *bv = &bio->bi_io_vec[bio->bi_vcnt - 1]; unsigned long mask = queue_segment_boundary(q); phys_addr_t addr1 = page_to_phys(bv->bv_page) + bv->bv_offset; phys_addr_t addr2 = page_to_phys(page) + offset + len - 1; if ((addr1 | mask) != (addr2 | mask)) return false; if (bv->bv_len + len > queue_max_segment_size(q)) return false; return __bio_try_merge_page(bio, page, len, offset, same); } int bio_add_hw_page(struct request_queue *q, struct bio *bio, struct page *page, unsigned int len, unsigned int offset, unsigned int max_sectors, bool *same_page) { if (WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED))) return 0; if (((bio->bi_iter.bi_size + len) >> 9) > max_sectors) return 0; if (bio->bi_vcnt > 0) { if (bio_try_merge_hw_seg(q, bio, page, len, offset, same_page)) return len; if (bvec_gap_to_prev(&q->limits, &bio->bi_io_vec[bio->bi_vcnt-1], offset)) return 0; } if (bio_full(bio, len) || bio->bi_vcnt >= queue_max_segments(q)) return 0; __bio_add_page(bio, page, len, offset); return len; } EXPORT_SYMBOL(bio_add_hw_page); /* 通用 add_page / zone_append 公共逻辑 */ static int bio_add_page_or_hw(struct bio *bio, struct page *page, unsigned int len, unsigned int offset, bool zone_append) { if (zone_append) { struct request_queue *q = bdev_get_queue(bio->bi_bdev); bool same_page = false; if (bio_add_hw_page(q, bio, page, len, offset, queue_max_zone_append_sectors(q), &same_page) != len) return -EINVAL; if (same_page) put_page(page); } else { bool same_page = false; if (!__bio_try_merge_page(bio, page, len, offset, &same_page)) __bio_add_page(bio, page, len, offset); else if (same_page) put_page(page); } return 0; } /* ======================= iov 迭代器 -> bio 页 ======================= */ #define PAGE_PTRS_PER_BVEC (sizeof(struct bio_vec) / sizeof(struct page *)) static int __bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) { unsigned short nr_pages = bio->bi_max_vecs - bio->bi_vcnt; unsigned short entries_left = bio->bi_max_vecs - bio->bi_vcnt; struct bio_vec *bv = bio->bi_io_vec + bio->bi_vcnt; struct page **pages = (struct page **)bv; ssize_t size, left; unsigned len, i = 0; size_t offset, trim; int ret = 0; pages += entries_left * (PAGE_PTRS_PER_BVEC - 1); size = iov_iter_get_pages2(iter, pages, UINT_MAX - bio->bi_iter.bi_size, nr_pages, &offset); if (unlikely(size <= 0)) return size ? size : -EFAULT; nr_pages = DIV_ROUND_UP(offset + size, PAGE_SIZE); trim = size & (bdev_logical_block_size(bio->bi_bdev) - 1); iov_iter_revert(iter, trim); size -= trim; if (unlikely(!size)) { ret = -EFAULT; goto out; } bool zone = (bio_op(bio) == REQ_OP_ZONE_APPEND); for (left = size, i = 0; left > 0; left -= len, i++) { struct page *page = pages[i]; len = min_t(size_t, PAGE_SIZE - offset, left); ret = bio_add_page_or_hw(bio, page, len, offset, zone); if (ret) break; offset = 0; } iov_iter_revert(iter, left); out: while (i < nr_pages) put_page(pages[i++]); return ret; } int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter) { if (iov_iter_is_bvec(iter)) { bio_iov_bvec_set(bio, iter); iov_iter_advance(iter, bio->bi_iter.bi_size); return 0; } int ret; do { ret = __bio_iov_iter_get_pages(bio, iter); } while (!ret && iov_iter_count(iter) && !bio_full(bio, 0)); return bio->bi_vcnt ? 0 : ret; } EXPORT_SYMBOL_GPL(bio_iov_iter_get_pages); /* ======================= 提交等待 ======================= */ static void submit_bio_wait_endio(struct bio *bio) { complete(bio->bi_private); } int submit_bio_wait(struct bio *bio) { DECLARE_COMPLETION_ONSTACK_MAP(done, bio->bi_bdev->bd_disk->lockdep_map); bio->bi_private = &done; bio->bi_end_io = submit_bio_wait_endio; bio->bi_opf |= REQ_SYNC; submit_bio(bio); unsigned long hang_check = sysctl_hung_task_timeout_secs; if (hang_check) while (!wait_for_completion_io_timeout(&done, hang_check * (HZ/2))); else wait_for_completion_io(&done); return blk_status_to_errno(bio->bi_status); } EXPORT_SYMBOL(submit_bio_wait); /* ======================= 辅助：advance, copy, dirty ======================= */ void __bio_advance(struct bio *bio, unsigned bytes) { if (bio_integrity(bio)) bio_integrity_advance(bio, bytes); bio_crypt_advance(bio, bytes); bio_advance_iter(bio, &bio->bi_iter, bytes); } EXPORT_SYMBOL(__bio_advance); void bio_copy_data_iter(struct bio *dst, struct bvec_iter *dst_iter, struct bio *src, struct bvec_iter *src_iter) { while (src_iter->bi_size && dst_iter->bi_size) { struct bio_vec src_bv = bio_iter_iovec(src, *src_iter); struct bio_vec dst_bv = bio_iter_iovec(dst, *dst_iter); unsigned int bytes = min(src_bv.bv_len, dst_bv.bv_len); void *src_buf = bvec_kmap_local(&src_bv); void *dst_buf = bvec_kmap_local(&dst_bv); memcpy(dst_buf, src_buf, bytes); kunmap_local(dst_buf); kunmap_local(src_buf); bio_advance_iter_single(src, src_iter, bytes); bio_advance_iter_single(dst, dst_iter, bytes); } } EXPORT_SYMBOL(bio_copy_data_iter); void bio_copy_data(struct bio *dst, struct bio *src) { struct bvec_iter src_iter = src->bi_iter; struct bvec_iter dst_iter = dst->bi_iter; bio_copy_data_iter(dst, &dst_iter, src, &src_iter); } EXPORT_SYMBOL(bio_copy_data); void bio_set_pages_dirty(struct bio *bio) { struct bio_vec *bvec; struct bvec_iter_all iter_all; bio_for_each_segment_all(bvec, bio, iter_all) if (!PageCompound(bvec->bv_page)) set_page_dirty_lock(bvec->bv_page); } static void bio_dirty_fn(struct work_struct *work); static DECLARE_WORK(bio_dirty_work, bio_dirty_fn); static DEFINE_SPINLOCK(bio_dirty_lock); static struct bio *bio_dirty_list; static void bio_dirty_fn(struct work_struct *work) { struct bio *bio, *next; spin_lock_irq(&bio_dirty_lock); next = bio_dirty_list; bio_dirty_list = NULL; spin_unlock_irq(&bio_dirty_lock); while ((bio = next) != NULL) { next = bio->bi_private; bio_release_pages(bio, true); bio_put(bio); } } void bio_check_pages_dirty(struct bio *bio) { struct bio_vec *bvec; struct bvec_iter_all iter_all; bio_for_each_segment_all(bvec, bio, iter_all) if (!PageDirty(bvec->bv_page) && !PageCompound(bvec->bv_page)) goto defer; bio_release_pages(bio, false); bio_put(bio); return; defer: spin_lock_irqsave(&bio_dirty_lock, flags); bio->bi_private = bio_dirty_list; bio_dirty_list = bio; spin_unlock_irqrestore(&bio_dirty_lock, flags); schedule_work(&bio_dirty_work); } /* ======================= 结束 I/O ======================= */ static inline bool bio_remaining_done(struct bio *bio) { if (!bio_flagged(bio, BIO_CHAIN)) return true; BUG_ON(atomic_read(&bio->__bi_remaining) <= 0); if (atomic_dec_and_test(&bio->__bi_remaining)) { bio_clear_flag(bio, BIO_CHAIN); return true; } return false; } void bio_endio(struct bio *bio) { again: if (!bio_remaining_done(bio)) return; if (!bio_integrity_endio(bio)) return; rq_qos_done_bio(bio); if (bio->bi_bdev && bio_flagged(bio, BIO_TRACE_COMPLETION)) { trace_block_bio_complete(bdev_get_queue(bio->bi_bdev), bio); bio_clear_flag(bio, BIO_TRACE_COMPLETION); } blk_throtl_bio_endio(bio); bio_uninit(bio); if (bio->bi_end_io) bio->bi_end_io(bio); } EXPORT_SYMBOL(bio_endio); /* ======================= bio_set 生命周期 ======================= */ int biovec_init_pool(mempool_t *pool, int pool_entries) { return mempool_init_slab_pool(pool, pool_entries, bvec_slabs[ARRAY_SIZE(bvec_slabs)-1].slab); } static int bio_cpu_dead(unsigned int cpu, struct hlist_node *node) { struct bio_set *bs = hlist_entry_safe(node, struct bio_set, cpuhp_dead); if (bs->cache) { struct bio_alloc_cache *cache = per_cpu_ptr(bs->cache, cpu); bio_alloc_cache_prune(cache, -1U); } return 0; } static void bio_alloc_cache_destroy(struct bio_set *bs) { int cpu; if (!bs->cache) return; cpuhp_state_remove_instance_nocalls(CPUHP_BIO_DEAD, &bs->cpuhp_dead); for_each_possible_cpu(cpu) { struct bio_alloc_cache *cache = per_cpu_ptr(bs->cache, cpu); bio_alloc_cache_prune(cache, -1U); } free_percpu(bs->cache); bs->cache = NULL; } void bioset_exit(struct bio_set *bs) { bio_alloc_cache_destroy(bs); if (bs->rescue_workqueue) destroy_workqueue(bs->rescue_workqueue); bs->rescue_workqueue = NULL; mempool_exit(&bs->bio_pool); mempool_exit(&bs->bvec_pool); bioset_integrity_free(bs); if (bs->bio_slab) bio_put_slab(bs); bs->bio_slab = NULL; } EXPORT_SYMBOL(bioset_exit); int bioset_init(struct bio_set *bs, unsigned int pool_size, unsigned int front_pad, int flags) { bs->front_pad = front_pad; bs->back_pad = (flags & BIOSET_NEED_BVECS) ? BIO_INLINE_VECS * sizeof(struct bio_vec) : 0; spin_lock_init(&bs->rescue_lock); bio_list_init(&bs->rescue_list); INIT_WORK(&bs->rescue_work, bio_alloc_rescue); bs->bio_slab = bio_find_or_create_slab(bs); if (!bs->bio_slab) return -ENOMEM; if (mempool_init_slab_pool(&bs->bio_pool, pool_size, bs->bio_slab)) goto bad; if ((flags & BIOSET_NEED_BVECS) && biovec_init_pool(&bs->bvec_pool, pool_size)) goto bad; if (flags & BIOSET_NEED_RESCUER) { bs->rescue_workqueue = alloc_workqueue("bioset", WQ_MEM_RECLAIM, 0); if (!bs->rescue_workqueue) goto bad; } if (flags & BIOSET_PERCPU_CACHE) { bs->cache = alloc_percpu(struct bio_alloc_cache); if (!bs->cache) goto bad; cpuhp_state_add_instance_nocalls(CPUHP_BIO_DEAD, &bs->cpuhp_dead); } return 0; bad: bioset_exit(bs); return -ENOMEM; } EXPORT_SYMBOL(bioset_init); /* ======================= 模块初始化 ======================= */ static int __init init_bio(void) { int i; bio_integrity_init(); for (i = 0; i < ARRAY_SIZE(bvec_slabs); i++) { struct biovec_slab *bvs = &bvec_slabs[i]; bvs->slab = kmem_cache_create(bvs->name, bvs->nr_vecs * sizeof(struct bio_vec), 0, SLAB_HWCACHE_ALIGN | SLAB_PANIC, NULL); } cpuhp_setup_state_multi(CPUHP_BIO_DEAD, "block/bio:dead", NULL, bio_cpu_dead); if (bioset_init(&fs_bio_set, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS | BIOSET_PERCPU_CACHE)) panic("bio: can't allocate bios\n"); if (bioset_integrity_create(&fs_bio_set, BIO_POOL_SIZE)) panic("bio: can't create integrity pool\n"); return 0; } subsys_initcall(init_bio);

这是九章编程法的巨大作用之一，不只是编制程序，而是对既有运行多年的程序，可以进行再组织重构，有效进行冗余和重复的精准合并，压缩，大幅降低程序的可执行性，可维护性，提升运行效率和降低能耗。程序不是没有错就完美，而是要让其达到极限的简洁，而功能和计算更准确，才是最合理的程序。庞大的既有程序库，包括运行几十年的无BUG程序，也是大量的重复和冗余。

接近73.5%的压缩率。代码从 1700 行压缩至 450 行，外部接口完全保留，所有结构重复已消除。每一个逻辑原语在系统中只出现一次，达到此模块的物理极限。此程序是重构的，与原程序一样，但是压缩了大量的重复代码。大量的老系统程序或是新开发程序，实际需要，或是有用的并不多，大理是重复的引用，验证，或是循环线形嵌套结构，让代码量暴发式增长，这也是巨大的能耗和性能降低。

九章编程法，不但能编写新程序，转换不同编程语言，更能对老旧系统程序进行重新组织重构，功能不变，程序精细化，消除重复冗余，让程序更好用，更好管，更好扩展应用。降能耗，提性能，清除BUG。

我们看到一个成熟的系统，往往数成千，几亿行代码，往往百分之七八十是冗余和复杂的代码，吃掉了大量的计算性能和存储性能。屎山代码，并不冤。大理商业企业往往以代码量来表达系统的成熟度，实际上，也表达了另一个意思，代码吃掉了同样的性能以及存大巨量的BUG。