diff --git a/Makefile b/Makefile index 3c9a295..b9cc28a 100644 --- a/Makefile +++ b/Makefile @@ -80,7 +80,8 @@ LIB_SRCS := \ format.c \ open.c \ resize.c \ - stats.c + stats.c \ + util.c BIN_SRCS := \ cbd.c KMOD_SRCS := \ @@ -98,7 +99,7 @@ LIB_OBJDIR := $(OUT_OBJ)/$(LIB_NAME) LIB_OBJS := $(addprefix $(LIB_OBJDIR)/,$(LIB_SRCS:.c=$(OBJ_EXT))) BIN_OBJDIR := $(OUT_OBJ)/$(BIN_NAME) BIN_OBJS := $(addprefix $(BIN_OBJDIR)/,$(BIN_SRCS:.c=$(OBJ_EXT))) -EXE_LIBS := -ldevmapper +EXE_LIBS := -ldevmapper -llz4 -lz KMOD_OBJDIR := $(OUT_OBJ)/$(KMOD_NAME) # Transforms diff --git a/cbd/cbd.c b/cbd/cbd.c index 33b4f33..084e02e 100644 --- a/cbd/cbd.c +++ b/cbd/cbd.c @@ -102,6 +102,7 @@ usage(void) " close [opts] Close an opened compressed device\n" " check [opts] Check and repair a compressed device\n" " -f --force Force check even if device seems clean\n" + " -F --full-check Do a full check, including verifying data\n" " -n --assume-no Assume \"no\" no all questions\n" " -y --assume-yes Assume \"yes\" to all questions\n" " resize [opts] Resize an existing compressed device\n" @@ -266,15 +267,17 @@ do_stats(int argc, char** argv) static int do_check(int argc, char** argv) { - static const char short_opts[] = "fny"; + static const char short_opts[] = "fFny"; static const struct option long_opts[] = { { "force", no_argument, NULL, 'f' }, + { "full-check", no_argument, NULL, 'F' }, { "assume-no", no_argument, NULL, 'n' }, { "assume-yes", no_argument, NULL, 'y' }, { NULL, no_argument, NULL, 0 } }; char opt; bool force = false; + bool full_check = false; tristate_t auto_response = t_none; const char* dev; @@ -284,6 +287,9 @@ do_check(int argc, char** argv) case 'f': force = true; break; + case 'F': + full_check = true; + break; case 'n': if (auto_response == t_true) { fprintf(stderr, "Cannot specify both -y and -n\n"); @@ -307,7 +313,7 @@ do_check(int argc, char** argv) } dev = argv[optind++]; - cbd_check(dev, force, auto_response); + cbd_check(dev, force, auto_response, full_check); return 0; } @@ -379,13 +385,15 @@ main(int argc, char** argv) { NULL, 0, NULL, 0 } }; char opt; - int verbose_level = 0; /* XXX: make global or ...? */ const char* cmd; uint n; progname = argv[0]; + /* Disable stdout buffering. */ + setbuf(stdout, NULL); + while ((opt = getopt_long(argc, argv, short_opts, long_opts, NULL)) != -1) { switch (opt) { case 'v': diff --git a/dm-compress/compress.c b/dm-compress/compress.c index 5ed274b..f09ea12 100644 --- a/dm-compress/compress.c +++ b/dm-compress/compress.c @@ -52,8 +52,8 @@ struct compress { struct dm_dev* dev; - struct cbd_params params; - struct cbd_stats stats; + struct compress_params kparams; + struct compress_stats kstats; struct lbdcache* lc; struct workqueue_struct* io_workq; @@ -91,15 +91,14 @@ compress_read_header(struct compress* c) pblkbuf = page_address(pblkpage); iopagev[0] = pblkpage; - header.params.priv = c->dev->bdev; - ret = pblk_read_wait(&header.params, 0, 1, iopagev); + c->kparams.dev = c->dev->bdev; + ret = pblk_read_wait(&c->kparams, 0, 1, iopagev); if (ret) { printk(KERN_ERR "%s: failed to read header\n", __func__); goto out; } memset(&header, 0, sizeof(header)); cbd_header_get(pblkbuf, &header); - header.params.priv = c->dev->bdev; if (memcmp(header.magic, CBD_MAGIC, sizeof(header.magic)) != 0) { printk(KERN_ERR "%s: bad magic\n", __func__); @@ -157,8 +156,8 @@ compress_read_header(struct compress* c) printk(KERN_INFO " nr_zones=%u\n", (unsigned int)header.params.nr_zones); printk(KERN_INFO " lblk_per_zone=%u\n", (unsigned int)header.params.lblk_per_zone); - memcpy(&c->params, &header.params, sizeof(header.params)); - memcpy(&c->stats, &header.stats, sizeof(header.stats)); + memcpy(&c->kparams.params, &header.params, sizeof(header.params)); + memcpy(&c->kstats.stats, &header.stats, sizeof(header.stats)); out: cbd_free_page(pblkpage); @@ -183,11 +182,11 @@ compress_write_header(struct compress* c) memcpy(header.magic, CBD_MAGIC, sizeof(header.magic)); header.version_major = CBD_VERSION_MAJOR; header.version_minor = CBD_VERSION_MINOR; - memcpy(&header.params, &c->params, sizeof(header.params)); - memcpy(&header.stats, &c->stats, sizeof(header.stats)); + memcpy(&header.params, &c->kparams.params, sizeof(header.params)); + memcpy(&header.stats, &c->kstats.stats, sizeof(header.stats)); cbd_header_put(pblkbuf, &header); iopagev[0] = pblkpage; - ret = pblk_write_wait(&c->params, 0, 1, iopagev); + ret = pblk_write_wait(&c->kparams, 0, 1, iopagev); if (ret) { printk(KERN_ERR "%s: failed to write header\n", __func__); } @@ -202,7 +201,7 @@ compress_read(struct compress *c, struct bio *bio) struct lbd* lbd = NULL; struct bio_vec bv; struct bvec_iter iter; - u32 lblk_per_sector = lblk_per_pblk(&c->params) * PBLK_PER_SECTOR; + u32 lblk_per_sector = lblk_per_pblk(&c->kparams.params) * PBLK_PER_SECTOR; bio_for_each_segment(bv, bio, iter) { u64 lblk = iter.bi_sector / lblk_per_sector; @@ -229,7 +228,7 @@ compress_write(struct compress *c, struct bio *bio) struct lbd* lbd = NULL; struct bio_vec bv; struct bvec_iter iter; - u32 lblk_per_sector = lblk_per_pblk(&c->params) * PBLK_PER_SECTOR; + u32 lblk_per_sector = lblk_per_pblk(&c->kparams.params) * PBLK_PER_SECTOR; bio_for_each_segment(bv, bio, iter) { u64 lblk = iter.bi_sector / lblk_per_sector; @@ -388,27 +387,27 @@ compress_ctr(struct dm_target *ti, unsigned int argc, char **argv) if (ret) { goto err; } - if (c->params.flags & CBD_FLAG_DIRTY) { + if (c->kparams.params.flags & CBD_FLAG_DIRTY) { printk(KERN_INFO "Warning: device was not properly closed\n"); } if (dm_table_get_mode(ti->table) & FMODE_WRITE) { - u16 save_flags = c->params.flags; - c->params.flags |= CBD_FLAG_DIRTY; + u16 save_flags = c->kparams.params.flags; + c->kparams.params.flags |= CBD_FLAG_DIRTY; ret = compress_write_header(c); - c->params.flags = save_flags; + c->kparams.params.flags = save_flags; if (ret) { goto err; } } /* XXX: validate minumum pblk using zone_off(max_zone+1) */ - if (c->params.nr_pblk > backing_nr_pblks) { + if (c->kparams.params.nr_pblk > backing_nr_pblks) { printk(KERN_ERR "%s: bad nr_pblk\n", __func__); ret = -EINVAL; goto err; } - if (c->params.nr_zones > zone_for_pblk(&c->params, backing_nr_pblks)) { + if (c->kparams.params.nr_zones > zone_for_pblk(&c->kparams.params, backing_nr_pblks)) { printk(KERN_ERR "%s: bad nr_zones\n", __func__); ret = -EINVAL; goto err; @@ -422,7 +421,7 @@ compress_ctr(struct dm_target *ti, unsigned int argc, char **argv) ret = -ENOMEM; goto err; } - if (!lbdcache_ctr(c->lc, &c->params, &c->stats, cache_pages)) { + if (!lbdcache_ctr(c->lc, &c->kparams, &c->kstats, cache_pages)) { printk(KERN_ERR "Failed to init logical block cache\n"); ret = -ENOMEM; goto err; @@ -476,7 +475,7 @@ compress_map(struct dm_target *ti, struct bio *bio) struct compress *c = ti->private; struct compress_io *cio; - if (c->params.flags & CBD_FLAG_ERROR) { + if (c->kparams.params.flags & CBD_FLAG_ERROR) { bio->bi_status = BLK_STS_IOERR; bio_endio(bio); return DM_MAPIO_SUBMITTED; /* XXXX: DM_MAPIO_KILL? */ diff --git a/dm-compress/lbatpage.c b/dm-compress/lbatpage.c index c96a1a0..c2157ce 100644 --- a/dm-compress/lbatpage.c +++ b/dm-compress/lbatpage.c @@ -36,14 +36,14 @@ struct lbatpage { unsigned int ref; struct mutex lock; - struct cbd_params* params; + struct compress_params* kparams; struct page* page; u8* buf; bool dirty; }; static bool -lbatpage_ctr(struct lbatpage* lp, struct cbd_params* params) +lbatpage_ctr(struct lbatpage* lp, struct compress_params* kparams) { memset(lp, 0, sizeof(struct lbatpage)); INIT_LIST_HEAD(&lp->list); @@ -51,7 +51,7 @@ lbatpage_ctr(struct lbatpage* lp, struct cbd_params* params) mutex_init(&lp->reflock); lp->ref = 0; mutex_init(&lp->lock); - lp->params = params; + lp->kparams = kparams; lp->page = cbd_alloc_page(); if (!lp->page) { return false; @@ -91,7 +91,7 @@ lbatpage_flush(struct lbatpage* lp) goto unlock; } iopagev[0] = lp->page; - pblk_write(lp->params, lp->pblk, 1, iopagev); + pblk_write(lp->kparams, lp->pblk, 1, iopagev); mutex_unlock(&lp->lock); return ret; @@ -110,7 +110,7 @@ lbatpage_read(struct lbatpage* lp) struct page* pagev[1]; pagev[0] = lp->page; - ret = pblk_read_wait(lp->params, lp->pblk, 1, pagev); + ret = pblk_read_wait(lp->kparams, lp->pblk, 1, pagev); return ret; } @@ -152,7 +152,6 @@ lbatpage_put_buf(struct lbatpage* lp) } struct lbatpagecache { - struct cbd_params* params; struct mutex cache_lock; struct list_head cache_head; unsigned int cache_len; @@ -167,14 +166,13 @@ lbatpagecache_size(void) bool lbatpagecache_ctr(struct lbatpagecache* lpc, - struct cbd_params* params, u32 cache_pages) + struct compress_params* kparams, u32 cache_pages) { struct lbatpage* cache; u32 cache_len; u32 n; memset(lpc, 0, sizeof(struct lbatpagecache)); - lpc->params = params; /* lbatpagecache gets 15/32 of cache pages */ cache_len = (cache_pages * 15 / 32); @@ -192,7 +190,7 @@ lbatpagecache_ctr(struct lbatpagecache* lpc, lpc->cache_len = cache_len; lpc->cache = cache; for (n = 0; n < cache_len; ++n) { - if (!lbatpage_ctr(&cache[n], lpc->params)) { + if (!lbatpage_ctr(&cache[n], kparams)) { return false; } list_add_tail(&cache[n].list, &lpc->cache_head); @@ -221,7 +219,6 @@ lbatpagecache_dtr(struct lbatpagecache* lpc) lpc->cache = NULL; lpc->cache_len = 0; INIT_LIST_HEAD(&lpc->cache_head); - lpc->params = NULL; } struct lbatpage* diff --git a/dm-compress/lbatview.c b/dm-compress/lbatview.c index 55a3340..ce83345 100644 --- a/dm-compress/lbatview.c +++ b/dm-compress/lbatview.c @@ -36,17 +36,19 @@ struct lbatview { unsigned int ref; struct mutex lock; - struct cbd_params* params; - struct pbatcache* pbatcache; - struct pbat* pbat; + struct compress_params* kparams; + struct compress_stats* kstats; + struct pbatcache* pc; struct lbatpagecache* lpc; + struct pbat* pbat; struct lbatpage* pages[2]; }; static bool lbatview_ctr(struct lbatview* lv, - struct cbd_params* params, - struct pbatcache* pbatcache, + struct compress_params* kparams, + struct compress_stats* kstats, + struct pbatcache* pc, struct lbatpagecache* lpc) { memset(lv, 0, sizeof(struct lbatview)); @@ -55,10 +57,11 @@ lbatview_ctr(struct lbatview* lv, mutex_init(&lv->reflock); lv->ref = 0; mutex_init(&lv->lock); - lv->params = params; - lv->pbatcache = pbatcache; - lv->pbat = NULL; + lv->kparams = kparams; + lv->kstats = kstats; + lv->pc = pc; lv->lpc = lpc; + lv->pbat = NULL; lv->pages[0] = lv->pages[1] = NULL; return true; @@ -67,13 +70,14 @@ lbatview_ctr(struct lbatview* lv, static void lbatview_dtr(struct lbatview* lv) { - pbatcache_put(lv->pbatcache, lv->pbat); - lv->pbat = NULL; lbatpagecache_put(lv->lpc, lv->pages[1]); lv->pages[1] = NULL; lbatpagecache_put(lv->lpc, lv->pages[0]); lv->pages[0] = NULL; + pbatcache_put(lv->pc, lv->pbat); + lv->pbat = NULL; lv->lpc = NULL; + lv->pc = NULL; } static int @@ -97,7 +101,7 @@ lbatview_flush(struct lbatview* lv) } lv->pages[0] = NULL; } - err = pbatcache_put(lv->pbatcache, lv->pbat); + err = pbatcache_put(lv->pc, lv->pbat); lv->pbat = NULL; if (err) { ret = err; @@ -144,7 +148,7 @@ static u64 lbatview_alloc_pblk(struct lbatview* lv) { int ret = 0; - u32 zone = zone_for_pblk(lv->params, lv->pblk); + u32 zone = zone_for_pblk(&lv->kparams->params, lv->pblk); u64 pblk; u32 zone_off; struct pbat* pbat; @@ -154,53 +158,53 @@ lbatview_alloc_pblk(struct lbatview* lv) if (pblk != PBLK_NONE) { return pblk; } - ret = pbatcache_put(lv->pbatcache, lv->pbat); + ret = pbatcache_put(lv->pc, lv->pbat); lv->pbat = NULL; if (ret) { printk(KERN_ERR "%s: pbatcache_put failed\n", __func__); return PBLK_NONE; } } - pbat = pbatcache_get(lv->pbatcache, zone, true); + pbat = pbatcache_get(lv->pc, zone, true); if (pbat) { pblk = pbat_alloc(pbat); if (pblk != PBLK_NONE) { lv->pbat = pbat; return pblk; } - ret = pbatcache_put(lv->pbatcache, pbat); + ret = pbatcache_put(lv->pc, pbat); if (ret) { printk(KERN_ERR "%s: pbatcache_put failed\n", __func__); return PBLK_NONE; } } for (zone_off = 1; - zone_off <= zone || zone + zone_off < lv->params->nr_zones; + zone_off <= zone || zone + zone_off < lv->kparams->params.nr_zones; ++zone_off) { if (zone_off <= zone) { - pbat = pbatcache_get(lv->pbatcache, zone - zone_off, true); + pbat = pbatcache_get(lv->pc, zone - zone_off, true); if (pbat) { pblk = pbat_alloc(pbat); if (pblk != PBLK_NONE) { lv->pbat = pbat; return pblk; } - ret = pbatcache_put(lv->pbatcache, pbat); + ret = pbatcache_put(lv->pc, pbat); if (ret) { printk(KERN_ERR "%s: pbatcache_put failed\n", __func__); return PBLK_NONE; } } } - if (zone + zone_off < lv->params->nr_zones) { - pbat = pbatcache_get(lv->pbatcache, zone + zone_off, true); + if (zone + zone_off < lv->kparams->params.nr_zones) { + pbat = pbatcache_get(lv->pc, zone + zone_off, true); if (pbat) { pblk = pbat_alloc(pbat); if (pblk != PBLK_NONE) { lv->pbat = pbat; return pblk; } - ret = pbatcache_put(lv->pbatcache, pbat); + ret = pbatcache_put(lv->pc, pbat); if (ret) { printk(KERN_ERR "%s: pbatcache_put failed\n", __func__); return PBLK_NONE; @@ -217,16 +221,16 @@ static int lbatview_free_pblk(struct lbatview* lv, u64 pblk) { int ret = 0; - u32 zone = zone_for_pblk(lv->params, lv->pblk); + u32 zone = zone_for_pblk(&lv->kparams->params, lv->pblk); u32 pblk_zone; struct pbat* pbat; - pblk_zone = zone_for_pblk(lv->params, pblk); - if (pblk_zone == ZONE_NONE || pblk_zone >= lv->params->nr_zones) { + pblk_zone = zone_for_pblk(&lv->kparams->params, pblk); + if (pblk_zone == ZONE_NONE || pblk_zone >= lv->kparams->params.nr_zones) { printk(KERN_ERR "%s: pblk=%lu: zone out of bounds\n", __func__, (unsigned long)pblk); return -EINVAL; } - pbat = pbatcache_get(lv->pbatcache, pblk_zone, false); + pbat = pbatcache_get(lv->pc, pblk_zone, false); if (!pbat) { printk(KERN_ERR "%s: pbatcache_get failed\n", __func__); return -EINVAL; @@ -235,14 +239,14 @@ lbatview_free_pblk(struct lbatview* lv, u64 pblk) BUG_ON(ret != 0); if (lv->pbat && pbat_zone(lv->pbat) != zone && pblk_zone == zone) { printk(KERN_INFO "%s: freed block %lu in zone %u switching back\n", __func__, (unsigned long)pblk, zone); - ret = pbatcache_put(lv->pbatcache, lv->pbat); + ret = pbatcache_put(lv->pc, lv->pbat); if (ret) { printk(KERN_ERR "%s: pbatcache_put failed\n", __func__); } lv->pbat = pbat; } else { - ret = pbatcache_put(lv->pbatcache, pbat); + ret = pbatcache_put(lv->pc, pbat); if (ret) { printk(KERN_ERR "%s: pbatcache_put failed\n", __func__); } @@ -254,13 +258,13 @@ lbatview_free_pblk(struct lbatview* lv, u64 pblk) static u32 lbatview_elem_off(struct lbatview* lv, u64 lblk) { - u32 lv_zone = zone_for_pblk(lv->params, lv->pblk); + u32 lv_zone = zone_for_pblk(&lv->kparams->params, lv->pblk); /* The relative lblk in the zone. */ - u32 zone_rel_lblk = lblk - (lv_zone * lv->params->lblk_per_zone); + u32 zone_rel_lblk = lblk - (lv_zone * lv->kparams->params.lblk_per_zone); /* The offset of the element in the (full) lbat. */ - u32 lbat_elem_off = zone_rel_lblk * lba_len(lv->params); + u32 lbat_elem_off = zone_rel_lblk * lba_len(&lv->kparams->params); /* The offset of the first view pblk. */ - u32 lbatview_off = PBLK_SIZE * (lv->pblk - lbat_off(lv->params, lv_zone)); + u32 lbatview_off = PBLK_SIZE * (lv->pblk - lbat_off(&lv->kparams->params, lv_zone)); return lbat_elem_off - lbatview_off; } @@ -318,7 +322,7 @@ lbatview_wmem(struct lbatview* lv, u32 off, u32 len, void* buf) } int -lbatview_elem_realloc(struct lbatview* lv, u64 lblk, u32 len, struct cbd_stats* stats) +lbatview_elem_realloc(struct lbatview* lv, u64 lblk, u32 len) { int ret = 0; int err; @@ -335,19 +339,19 @@ lbatview_elem_realloc(struct lbatview* lv, u64 lblk, u32 len, struct cbd_stats* mutex_lock(&lv->lock); elem_off = lbatview_elem_off(lv, lblk); req_nalloc = (len == CBD_UNCOMPRESSED) ? - lblk_per_pblk(lv->params) : + lblk_per_pblk(&lv->kparams->params) : DIV_ROUND_UP(len, PBLK_SIZE); elem_lelen = 0; - lbatview_rmem(lv, elem_off, lba_elem_len_bytes(lv->params), &elem_lelen); + lbatview_rmem(lv, elem_off, lba_elem_len_bytes(&lv->kparams->params), &elem_lelen); elem_len = __le32_to_cpu(elem_lelen); cur_nalloc = (elem_len == CBD_UNCOMPRESSED) ? - lblk_per_pblk(lv->params) : + lblk_per_pblk(&lv->kparams->params) : DIV_ROUND_UP(elem_len, PBLK_SIZE); old_nalloc = cur_nalloc; while (cur_nalloc < req_nalloc) { - off = elem_off + lba_elem_len_bytes(lv->params) + - cur_nalloc * lba_elem_pblk_bytes(lv->params); + off = elem_off + lba_elem_len_bytes(&lv->kparams->params) + + cur_nalloc * lba_elem_pblk_bytes(&lv->kparams->params); pblk = lbatview_alloc_pblk(lv); if (pblk == PBLK_NONE) { printk(KERN_ERR "%s: lbatview_alloc_pblk failed\n", __func__); @@ -356,16 +360,16 @@ lbatview_elem_realloc(struct lbatview* lv, u64 lblk, u32 len, struct cbd_stats* goto do_free; } elem_lepblk = __cpu_to_le64(pblk); - lbatview_wmem(lv, off, lba_elem_pblk_bytes(lv->params), &elem_lepblk); + lbatview_wmem(lv, off, lba_elem_pblk_bytes(&lv->kparams->params), &elem_lepblk); ++cur_nalloc; } do_free: while (cur_nalloc > req_nalloc) { --cur_nalloc; - off = elem_off + lba_elem_len_bytes(lv->params) + - cur_nalloc * lba_elem_pblk_bytes(lv->params); + off = elem_off + lba_elem_len_bytes(&lv->kparams->params) + + cur_nalloc * lba_elem_pblk_bytes(&lv->kparams->params); elem_lepblk = 0; - lbatview_rmem(lv, off, lba_elem_pblk_bytes(lv->params), &elem_lepblk); + lbatview_rmem(lv, off, lba_elem_pblk_bytes(&lv->kparams->params), &elem_lepblk); pblk = __le64_to_cpu(elem_lepblk); err = lbatview_free_pblk(lv, pblk); if (err) { @@ -376,8 +380,20 @@ do_free: if (!ret) { elem_lelen = __cpu_to_le32(len); - lbatview_wmem(lv, elem_off, lba_elem_len_bytes(lv->params), &elem_lelen); - stats->pblk_alloc += req_nalloc - old_nalloc; + lbatview_wmem(lv, elem_off, lba_elem_len_bytes(&lv->kparams->params), &elem_lelen); + mutex_lock(&lv->kstats->lock); + lv->kstats->stats.pblk_alloc += req_nalloc - old_nalloc; + if (old_nalloc == 0) { + if (req_nalloc != 0) { + ++lv->kstats->stats.lblk_alloc; + } + } + else { + if (req_nalloc == 0) { + --lv->kstats->stats.lblk_alloc; + } + } + mutex_unlock(&lv->kstats->lock); } mutex_unlock(&lv->lock); @@ -393,7 +409,7 @@ lbatview_elem_len(struct lbatview* lv, u64 lblk) mutex_lock(&lv->lock); off = lbatview_elem_off(lv, lblk); elem_lelen = 0; - lbatview_rmem(lv, off, lba_elem_len_bytes(lv->params), &elem_lelen); + lbatview_rmem(lv, off, lba_elem_len_bytes(&lv->kparams->params), &elem_lelen); mutex_unlock(&lv->lock); return __le32_to_cpu(elem_lelen); @@ -409,19 +425,19 @@ lbatview_elem_pblk(struct lbatview* lv, u64 lblk, u32 idx) mutex_lock(&lv->lock); off = lbatview_elem_off(lv, lblk) + - lba_elem_len_bytes(lv->params) + - idx * lba_elem_pblk_bytes(lv->params); + lba_elem_len_bytes(&lv->kparams->params) + + idx * lba_elem_pblk_bytes(&lv->kparams->params); elem_lepblk = 0; - lbatview_rmem(lv, off, lba_elem_pblk_bytes(lv->params), &elem_lepblk); + lbatview_rmem(lv, off, lba_elem_pblk_bytes(&lv->kparams->params), &elem_lepblk); mutex_unlock(&lv->lock); pblk = __le64_to_cpu(elem_lepblk); - pblk_zone = zone_for_pblk(lv->params, pblk); - if (pblk_zone == ZONE_NONE || pblk_zone >= lv->params->nr_zones) { + pblk_zone = zone_for_pblk(&lv->kparams->params, pblk); + if (pblk_zone == ZONE_NONE || pblk_zone >= lv->kparams->params.nr_zones) { printk(KERN_ERR "%s: pblk %lu out of range at lblk=%lu n=%u\n", __func__, (unsigned long)pblk, (unsigned long)lblk, idx); return PBLK_NONE; } - if (pblk < zone_data_off(lv->params, pblk_zone)) { + if (pblk < zone_data_off(&lv->kparams->params, pblk_zone)) { printk(KERN_ERR "%s: pblk in metadata at lblk=%lu n=%u\n", __func__, (unsigned long)pblk, idx); return PBLK_NONE; @@ -448,26 +464,27 @@ lbatviewcache_size(void) bool lbatviewcache_ctr(struct lbatviewcache* lvc, - struct cbd_params* params, u32 cache_pages) + struct compress_params* kparams, struct compress_stats* stats, + u32 cache_pages) { struct lbatview* cache; u32 cache_len; u32 n; memset(lvc, 0, sizeof(struct lbatviewcache)); - lvc->params = params; + lvc->params = &kparams->params; lvc->pc = kmalloc(pbatcache_size(), GFP_KERNEL); if (!lvc->pc) { return false; } - if (!pbatcache_ctr(lvc->pc, params, cache_pages)) { + if (!pbatcache_ctr(lvc->pc, kparams, cache_pages)) { return false; } lvc->lpc = kmalloc(lbatpagecache_size(), GFP_KERNEL); if (!lvc->lpc) { return false; } - if (!lbatpagecache_ctr(lvc->lpc, params, cache_pages)) { + if (!lbatpagecache_ctr(lvc->lpc, kparams, cache_pages)) { return false; } /* lbatviewcache gets one entry per lbatpage (XXX: 5/6?) */ @@ -486,7 +503,7 @@ lbatviewcache_ctr(struct lbatviewcache* lvc, lvc->cache_len = cache_len; lvc->cache = cache; for (n = 0; n < cache_len; ++n) { - if (!lbatview_ctr(&cache[n], lvc->params, lvc->pc, lvc->lpc)) { + if (!lbatview_ctr(&cache[n], kparams, stats, lvc->pc, lvc->lpc)) { return false; } list_add_tail(&cache[n].list, &lvc->cache_head); diff --git a/dm-compress/lbd.c b/dm-compress/lbd.c index 423b7a6..e10f80f 100644 --- a/dm-compress/lbd.c +++ b/dm-compress/lbd.c @@ -39,7 +39,7 @@ struct lbd { unsigned int ref; struct mutex lock; - struct cbd_params* params; + struct compress_params* kparams; struct lbatviewcache* lvc; struct lbatview* lv; void* percpu; @@ -99,9 +99,9 @@ lblk_compress_lz4(struct lbd* lbd) state = lblk_get_compress_state(lbd->percpu, cpu); BUG_ON(state == NULL); clen = LZ4_compress_fast(lbd->buf, state->buf, - PBLK_SIZE * lblk_per_pblk(lbd->params), - PBLK_SIZE * (lblk_per_pblk(lbd->params) - 1), - lbd->params->compression, state->lz4_workmem); + PBLK_SIZE * lblk_per_pblk(&lbd->kparams->params), + PBLK_SIZE * (lblk_per_pblk(&lbd->kparams->params) - 1), + lbd->kparams->params.compression, state->lz4_workmem); if (clen <= 0) { put_cpu(); return 0; @@ -118,7 +118,7 @@ lblk_decompress_lz4(struct lbd* lbd) int ret; int cpu; struct lblk_compress_state* state; - u32 dlen = PBLK_SIZE * lblk_per_pblk(lbd->params); + u32 dlen = PBLK_SIZE * lblk_per_pblk(&lbd->kparams->params); cpu = get_cpu(); state = lblk_get_compress_state(lbd->percpu, cpu); @@ -155,9 +155,9 @@ lblk_compress_zlib(struct lbd* lbd) ret = zlib_deflateReset(stream); BUG_ON(ret != Z_OK); stream->next_in = lbd->buf; - stream->avail_in = PBLK_SIZE * lblk_per_pblk(lbd->params); + stream->avail_in = PBLK_SIZE * lblk_per_pblk(&lbd->kparams->params); stream->next_out = state->buf; - stream->avail_out = PBLK_SIZE * (lblk_per_pblk(lbd->params) - 1); + stream->avail_out = PBLK_SIZE * (lblk_per_pblk(&lbd->kparams->params) - 1); ret = zlib_deflate(stream, Z_FINISH); if (ret != Z_STREAM_END) { put_cpu(); @@ -177,7 +177,7 @@ lblk_decompress_zlib(struct lbd* lbd) int cpu; struct lblk_compress_state* state; z_stream* stream; - u32 dlen = PBLK_SIZE * lblk_per_pblk(lbd->params); + u32 dlen = PBLK_SIZE * lblk_per_pblk(&lbd->kparams->params); cpu = get_cpu(); state = lblk_get_compress_state(lbd->percpu, cpu); @@ -217,12 +217,12 @@ static size_t lblk_compress(struct lbd* lbd) { #ifdef COMPRESS_HAVE_LZ4 - if (lbd->params->algorithm == CBD_ALG_LZ4) { + if (lbd->kparams->params.algorithm == CBD_ALG_LZ4) { return lblk_compress_lz4(lbd); } #endif #ifdef COMPRESS_HAVE_ZLIB - if (lbd->params->algorithm == CBD_ALG_ZLIB) { + if (lbd->kparams->params.algorithm == CBD_ALG_ZLIB) { return lblk_compress_zlib(lbd); } #endif @@ -236,12 +236,12 @@ static bool lblk_decompress(struct lbd* lbd) { #ifdef COMPRESS_HAVE_LZ4 - if (lbd->params->algorithm == CBD_ALG_LZ4) { + if (lbd->kparams->params.algorithm == CBD_ALG_LZ4) { return lblk_decompress_lz4(lbd); } #endif #ifdef COMPRESS_HAVE_ZLIB - if (lbd->params->algorithm == CBD_ALG_ZLIB) { + if (lbd->kparams->params.algorithm == CBD_ALG_ZLIB) { return lblk_decompress_zlib(lbd); } #endif @@ -250,11 +250,11 @@ lblk_decompress(struct lbd* lbd) static bool lbd_ctr(struct lbd* lbd, - struct cbd_params* params, + struct compress_params* kparams, struct lbatviewcache* lvc, void* percpu) { - u32 nr_pages = lblk_per_pblk(params); + u32 nr_pages = lblk_per_pblk(&kparams->params); memset(lbd, 0, sizeof(struct lbd)); INIT_LIST_HEAD(&lbd->lru_list); @@ -264,7 +264,7 @@ lbd_ctr(struct lbd* lbd, mutex_init(&lbd->reflock); lbd->ref = 0; mutex_init(&lbd->lock); - lbd->params = params; + lbd->kparams = kparams; lbd->lvc = lvc; lbd->lv = NULL; lbd->percpu = percpu; @@ -287,7 +287,7 @@ lbd_ctr(struct lbd* lbd, static void lbd_dtr(struct lbd* lbd) { - u32 nr_pages = lblk_per_pblk(lbd->params); + u32 nr_pages = lblk_per_pblk(&lbd->kparams->params); if (lbatviewcache_put(lbd->lvc, lbd->lv) != 0) { printk(KERN_ERR "%s: lbatviewcache_put failed\n", __func__); @@ -310,13 +310,13 @@ lbd_error(struct lbd* lbd) } static int -lbd_flush(struct lbd* lbd, struct cbd_stats* stats) +lbd_flush(struct lbd* lbd) { int ret = 0; int err; u32 n; u64 pblk; - u32 nr_pages = lblk_per_pblk(lbd->params); + u32 nr_pages = lblk_per_pblk(&lbd->kparams->params); u32 count; struct page* iopagev[1]; @@ -329,9 +329,9 @@ lbd_flush(struct lbd* lbd, struct cbd_stats* stats) goto unlock; } - if (lblk_is_zeros(lbd->params, lbd)) { + if (lblk_is_zeros(&lbd->kparams->params, lbd)) { lbd->c_len = CBD_UNCOMPRESSED; - ret = lbatview_elem_realloc(lbd->lv, lbd->lblk, 0, stats); + ret = lbatview_elem_realloc(lbd->lv, lbd->lblk, 0); goto unlock; } lbd->c_len = lblk_compress(lbd); @@ -344,20 +344,20 @@ lbd_flush(struct lbd* lbd, struct cbd_stats* stats) } else { lbd->c_len = CBD_UNCOMPRESSED; - count = lblk_per_pblk(lbd->params); + count = lblk_per_pblk(&lbd->kparams->params); } - ret = lbatview_elem_realloc(lbd->lv, lbd->lblk, lbd->c_len, stats); + ret = lbatview_elem_realloc(lbd->lv, lbd->lblk, lbd->c_len); if (ret) { - lbd->params->flags |= CBD_FLAG_ERROR; + lbd->kparams->params.flags |= CBD_FLAG_ERROR; goto unlock; } for (n = 0; n < count; ++n) { pblk = lbatview_elem_pblk(lbd->lv, lbd->lblk, n); BUG_ON(pblk == PBLK_NONE); iopagev[0] = lbd->pagev[n]; - pblk_write(lbd->params, pblk, 1, iopagev); + pblk_write(lbd->kparams, pblk, 1, iopagev); } - while (n < lblk_per_pblk(lbd->params)) { + while (n < lblk_per_pblk(&lbd->kparams->params)) { unlock_page(lbd->pagev[n]); ++n; } @@ -394,11 +394,11 @@ lbd_read(struct lbd* lbd) } lbd->c_len = lbatview_elem_len(lbd->lv, lbd->lblk); if (lbd->c_len == 0) { - memset(lbd->buf, 0, PBLK_SIZE * lblk_per_pblk(lbd->params)); + memset(lbd->buf, 0, PBLK_SIZE * lblk_per_pblk(&lbd->kparams->params)); } else { count = (lbd->c_len == CBD_UNCOMPRESSED) ? - lblk_per_pblk(lbd->params) : + lblk_per_pblk(&lbd->kparams->params) : DIV_ROUND_UP(lbd->c_len, PBLK_SIZE); for (n = 0; n < count; ++n) { pblk = lbatview_elem_pblk(lbd->lv, lbd->lblk, n); @@ -408,7 +408,7 @@ lbd_read(struct lbd* lbd) } iopagev[0] = lbd->pagev[n]; /* XXX: Issue non-blocking reads? */ - ret = pblk_read_wait(lbd->params, pblk, 1, iopagev); + ret = pblk_read_wait(lbd->kparams, pblk, 1, iopagev); if (ret) { goto out; } @@ -431,7 +431,7 @@ static int lbd_reset(struct lbd* lbd, u64 lblk) { int ret = 0; - u32 nr_pages = lblk_per_pblk(lbd->params); + u32 nr_pages = lblk_per_pblk(&lbd->kparams->params); u32 n; if (lbd->lv) { printk(KERN_ERR "%s: lbatview leak\n", __func__); } @@ -485,7 +485,7 @@ void lbd_data_read(struct lbd* lbd, u32 off, u32 len, u8* buf) { /* XXX: convert to BUG_ON */ - if (off + len > PBLK_SIZE * lblk_per_pblk(lbd->params)) { + if (off + len > PBLK_SIZE * lblk_per_pblk(&lbd->kparams->params)) { printk(KERN_ERR "%s: out of bounds\n", __func__); return; } @@ -498,7 +498,7 @@ void lbd_data_write(struct lbd* lbd, u32 off, u32 len, const u8* buf) { /* XXX: convert to BUG_ON */ - if (off + len > PBLK_SIZE * lblk_per_pblk(lbd->params)) { + if (off + len > PBLK_SIZE * lblk_per_pblk(&lbd->kparams->params)) { printk(KERN_ERR "%s: out of bounds\n", __func__); return; } @@ -511,7 +511,6 @@ lbd_data_write(struct lbd* lbd, u32 off, u32 len, const u8* buf) struct lbdcache { struct cbd_params* params; - struct cbd_stats* stats; void* percpu; struct lbatviewcache* lvc; struct mutex cache_lock; @@ -605,7 +604,7 @@ lbdcache_free_compress_state(void* percpu, const struct cbd_params* params, int bool lbdcache_ctr(struct lbdcache* lc, - struct cbd_params* params, struct cbd_stats* stats, + struct compress_params* kparams, struct compress_stats* kstats, u32 cache_pages) { int cpu; @@ -614,11 +613,10 @@ lbdcache_ctr(struct lbdcache* lc, u32 n; memset(lc, 0, sizeof(struct lbdcache)); - lc->params = params; - lc->stats = stats; + lc->params = &kparams->params; lc->percpu = alloc_percpu(void*); for (cpu = 0; cpu < num_online_cpus(); ++cpu) { - if (!lbdcache_alloc_compress_state(lc->percpu, params, cpu)) { + if (!lbdcache_alloc_compress_state(lc->percpu, lc->params, cpu)) { return false; } } @@ -626,12 +624,12 @@ lbdcache_ctr(struct lbdcache* lc, if (!lc->lvc) { return false; } - if (!lbatviewcache_ctr(lc->lvc, params, cache_pages)) { + if (!lbatviewcache_ctr(lc->lvc, kparams, kstats, cache_pages)) { return false; } /* lbdcache gets 1/2 of cache_pages */ - cache_len = (cache_pages * 1 / 2) / lblk_per_pblk(params); + cache_len = (cache_pages * 1 / 2) / lblk_per_pblk(lc->params); if (!cache_len) { printk(KERN_ERR "%s: Cache too small\n", __func__); return false; @@ -646,7 +644,7 @@ lbdcache_ctr(struct lbdcache* lc, lc->cache_len = cache_len; lc->cache = cache; for (n = 0; n < cache_len; ++n) { - if (!lbd_ctr(&cache[n], params, lc->lvc, lc->percpu)) { + if (!lbd_ctr(&cache[n], kparams, lc->lvc, lc->percpu)) { return false; } list_add_tail(&cache[n].lru_list, &lc->cache_head); @@ -669,7 +667,7 @@ lbdcache_dtr(struct lbdcache* lc) cancel_delayed_work_sync(&lc->flush_dwork); flush_delayed_work(&lc->flush_dwork); list_for_each_entry(lbd, &lc->flush_head, flush_list) { - ret = lbd_flush(lbd, lc->stats); + ret = lbd_flush(lbd); if (ret) { printk(KERN_ERR "%s: lbd_flush failed\n", __func__); } @@ -727,7 +725,7 @@ lbdcache_flush(struct work_struct* work) mutex_unlock(&lc->flush_lock); list_for_each_entry(lbd, &flushq, flush_list) { lbd->ref = 0; - ret = lbd_flush(lbd, lc->stats); + ret = lbd_flush(lbd); if (ret) { printk(KERN_ERR "%s: lbd_flush failed\n", __func__); } @@ -801,7 +799,7 @@ lbdcache_get(struct lbdcache* lc, u64 lblk) mutex_unlock(&lc->cache_lock); mutex_unlock(&lc->flush_lock); lbd->ref = 0; - ret = lbd_flush(lbd, lc->stats); + ret = lbd_flush(lbd); if (ret) { printk(KERN_ERR "%s: lbd_flush failed\n", __func__); } diff --git a/dm-compress/pbat.c b/dm-compress/pbat.c index 2c067c2..b4547a9 100644 --- a/dm-compress/pbat.c +++ b/dm-compress/pbat.c @@ -36,7 +36,7 @@ struct pbat { unsigned int ref; struct mutex lock; - struct cbd_params* params; + struct compress_params* kparams; bool full; u32 last_alloc; struct page** pagev; @@ -45,9 +45,9 @@ struct pbat { static bool pbat_ctr(struct pbat* pbat, - struct cbd_params* params) + struct compress_params* kparams) { - u32 nr_pages = pbat_len(params); + u32 nr_pages = pbat_len(&kparams->params); memset(pbat, 0, sizeof(struct pbat)); INIT_LIST_HEAD(&pbat->list); @@ -55,7 +55,7 @@ pbat_ctr(struct pbat* pbat, mutex_init(&pbat->reflock); pbat->ref = 0; mutex_init(&pbat->lock); - pbat->params = params; + pbat->kparams = kparams; pbat->full = false; pbat->last_alloc = 0; pbat->pagev = kzalloc(nr_pages * sizeof(struct page*), GFP_KERNEL); @@ -81,7 +81,7 @@ pbat_ctr(struct pbat* pbat, static void pbat_dtr(struct pbat* pbat) { - u32 nr_pages = pbat_len(pbat->params); + u32 nr_pages = pbat_len(&pbat->kparams->params); u32 n; for (n = 0; n < nr_pages; ++n) { @@ -106,7 +106,7 @@ static int pbat_flush(struct pbat* pbat) { int ret = 0; - u32 nr_pages = pbat_len(pbat->params); + u32 nr_pages = pbat_len(&pbat->kparams->params); u32 n; u64 pblk; @@ -118,8 +118,8 @@ pbat_flush(struct pbat* pbat) ret = -EIO; goto unlock; } - pblk = pbat_off(pbat->params, pbat->zone); - pblk_write(pbat->params, pblk, nr_pages, pbat->pagev); + pblk = pbat_off(&pbat->kparams->params, pbat->zone); + pblk_write(pbat->kparams, pblk, nr_pages, pbat->pagev); mutex_unlock(&pbat->lock); return ret; @@ -137,15 +137,15 @@ static int pbat_read(struct pbat* pbat) { int ret = 0; - u32 nr_pages = pbat_len(pbat->params); + u32 nr_pages = pbat_len(&pbat->kparams->params); u64 pblk; /* XXX: can't happen because pbatcache will not use a page with an error */ if (PageError(pbat->pagev[0])) { return -EIO; } - pblk = pbat_off(pbat->params, pbat->zone); - ret = pblk_read_wait(pbat->params, pblk, nr_pages, pbat->pagev); + pblk = pbat_off(&pbat->kparams->params, pbat->zone); + ret = pblk_read_wait(pbat->kparams, pblk, nr_pages, pbat->pagev); return ret; } @@ -154,7 +154,7 @@ static int pbat_reset(struct pbat* pbat, u32 zone) { int ret = 0; - u32 nr_pages = pbat_len(pbat->params); + u32 nr_pages = pbat_len(&pbat->kparams->params); u32 n; for (n = 0; n < nr_pages; ++n) { @@ -186,7 +186,7 @@ pbat_zone(struct pbat* pbat) u64 pbat_alloc(struct pbat* pbat) { - u32 pblk_count = pbat_len(pbat->params) * PBLK_SIZE_BITS; + u32 pblk_count = pbat_len(&pbat->kparams->params) * PBLK_SIZE_BITS; u32 idx; u64 pblk; @@ -202,7 +202,7 @@ pbat_alloc(struct pbat* pbat) goto out; } pbat->last_alloc = idx; - pblk = idx + zone_data_off(pbat->params, pbat->zone); + pblk = idx + zone_data_off(&pbat->kparams->params, pbat->zone); SetPageDirty(pbat->pagev[0]); out: @@ -213,17 +213,17 @@ out: int pbat_free(struct pbat* pbat, u64 pblk) { - u32 pblk_count = pbat_len(pbat->params) * PBLK_SIZE_BITS; + u32 pblk_count = pbat_len(&pbat->kparams->params) * PBLK_SIZE_BITS; u32 zone; u32 idx; - zone = zone_for_pblk(pbat->params, pblk); + zone = zone_for_pblk(&pbat->kparams->params, pblk); BUG_ON(zone != pbat->zone); - if (pblk < zone_data_off(pbat->params, zone)) { + if (pblk < zone_data_off(&pbat->kparams->params, zone)) { printk(KERN_ERR "%s: pblk in metadata\n", __func__); return -EINVAL; } - idx = pblk - zone_data_off(pbat->params, zone); + idx = pblk - zone_data_off(&pbat->kparams->params, zone); BUG_ON(idx >= pblk_count); mutex_lock(&pbat->lock); cbd_bitmap_free(pbat->buf, idx); @@ -235,7 +235,6 @@ pbat_free(struct pbat* pbat, u64 pblk) } struct pbatcache { - struct cbd_params* params; struct mutex cache_lock; struct list_head cache_head; unsigned int cache_len; @@ -251,17 +250,16 @@ pbatcache_size(void) bool pbatcache_ctr(struct pbatcache* pc, - struct cbd_params* params, u32 cache_pages) + struct compress_params* kparams, u32 cache_pages) { struct pbat* cache; u32 cache_len; u32 n; memset(pc, 0, sizeof(struct pbatcache)); - pc->params = params; /* pbatcache gets 1/32 of cache_pages */ - cache_len = (cache_pages * 1 / 32) / pbat_len(params); + cache_len = (cache_pages * 1 / 32) / pbat_len(&kparams->params); if (!cache_len) { printk(KERN_ERR "%s: Cache too small\n", __func__); return false; @@ -276,12 +274,12 @@ pbatcache_ctr(struct pbatcache* pc, pc->cache_len = cache_len; pc->cache = cache; for (n = 0; n < cache_len; ++n) { - if (!pbat_ctr(&cache[n], pc->params)) { + if (!pbat_ctr(&cache[n], kparams)) { return false; } list_add_tail(&cache[n].list, &pc->cache_head); } - pc->full = kzalloc(DIV_ROUND_UP(params->nr_zones, BITS_PER_BYTE), GFP_KERNEL); + pc->full = kzalloc(DIV_ROUND_UP(kparams->params.nr_zones, BITS_PER_BYTE), GFP_KERNEL); return true; } @@ -307,7 +305,6 @@ pbatcache_dtr(struct pbatcache* pc) pc->cache = NULL; pc->cache_len = 0; INIT_LIST_HEAD(&pc->cache_head); - pc->params = NULL; } struct pbat* diff --git a/dm-compress/util.c b/dm-compress/util.c index 1b242b2..6b2f535 100644 --- a/dm-compress/util.c +++ b/dm-compress/util.c @@ -123,13 +123,13 @@ pblk_io_prepare(struct block_device* bdev, unsigned int op, } int -pblk_read_wait(struct cbd_params* params, +pblk_read_wait(struct compress_params* kparams, u64 pblk, u32 count, struct page** pagev) { int ret; struct bio* bio; - bio = pblk_io_prepare(params->priv, REQ_OP_READ, pblk, count, pagev); + bio = pblk_io_prepare(kparams->dev, REQ_OP_READ, pblk, count, pagev); if (!bio) { printk(KERN_ERR "%s: out of memory\n", __func__); return -ENOMEM; @@ -144,13 +144,13 @@ pblk_read_wait(struct cbd_params* params, } int -pblk_write_wait(struct cbd_params* params, +pblk_write_wait(struct compress_params* kparams, u64 pblk, u32 count, struct page** pagev) { int ret; struct bio* bio; - bio = pblk_io_prepare(params->priv, REQ_OP_WRITE, pblk, count, pagev); + bio = pblk_io_prepare(kparams->dev, REQ_OP_WRITE, pblk, count, pagev); if (!bio) { printk(KERN_ERR "%s: out of memory\n", __func__); return -ENOMEM; @@ -158,7 +158,7 @@ pblk_write_wait(struct cbd_params* params, ret = submit_bio_wait(bio); if (ret) { printk(KERN_ERR "%s: submit_bio_wait failed: %d\n", __func__, ret); - params->flags |= CBD_FLAG_ERROR; + kparams->params.flags |= CBD_FLAG_ERROR; } bio_put(bio); @@ -168,13 +168,13 @@ pblk_write_wait(struct cbd_params* params, void pblk_write_endio(struct bio* bio) { - struct cbd_params* params = bio->bi_private; + struct compress_params* kparams = bio->bi_private; u32 n; struct page* page; BUG_ON(!bio); if (bio->bi_status != BLK_STS_OK) { - params->flags |= CBD_FLAG_ERROR; + kparams->params.flags |= CBD_FLAG_ERROR; for (n = 0; n < bio->bi_max_vecs; ++n) { page = bio->bi_io_vec[n].bv_page; SetPageError(page); @@ -189,16 +189,16 @@ pblk_write_endio(struct bio* bio) } void -pblk_write(struct cbd_params* params, +pblk_write(struct compress_params* kparams, u64 pblk, u32 count, struct page** pagev) { struct bio* bio; u32 n; - bio = pblk_io_prepare(params->priv, REQ_OP_WRITE, pblk, count, pagev); + bio = pblk_io_prepare(kparams->dev, REQ_OP_WRITE, pblk, count, pagev); if (!bio) { printk(KERN_ERR "%s: out of memory\n", __func__); - params->flags |= CBD_FLAG_ERROR; + kparams->params.flags |= CBD_FLAG_ERROR; for (n = 0; n < count; ++n) { SetPageError(pagev[n]); unlock_page(pagev[n]); @@ -206,7 +206,7 @@ pblk_write(struct cbd_params* params, return; } bio->bi_end_io = pblk_write_endio; - bio->bi_private = params; + bio->bi_private = kparams; submit_bio(bio); } diff --git a/include/cbdutil.h b/include/cbdutil.h index 3cdf67c..2af81f1 100644 --- a/include/cbdutil.h +++ b/include/cbdutil.h @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -34,14 +35,12 @@ #include #include -static inline void __attribute__((noreturn)) -error(const char* fmt, ...) -{ - va_list ap; +extern uint verbose_level; - va_start(ap, fmt); - vfprintf(stderr, fmt, ap); - exit(EXIT_FAILURE); -} +void __attribute__((noreturn)) + error(const char* fmt, ...); +int verbose(uint level, const char* fmt, ...); + +bool ask_user_bool(const char* fmt, ...); #endif diff --git a/include/libcbd.h b/include/libcbd.h index 9b258be..160b631 100644 --- a/include/libcbd.h +++ b/include/libcbd.h @@ -62,7 +62,8 @@ int cbd_stats(const char* dev, struct cbd_stats* stats); int cbd_check(const char* dev, bool force, - tristate_t auto_response); + tristate_t auto_response, + bool full_check); int cbd_resize(const char* dev, uint64_t lsize); diff --git a/include/linux/dm-compress.h b/include/linux/dm-compress.h index fd71080..929a346 100644 --- a/include/linux/dm-compress.h +++ b/include/linux/dm-compress.h @@ -48,11 +48,11 @@ struct cbd_params { u64 nr_pblk; u32 nr_zones; u32 lblk_per_zone; - void* priv; }; struct cbd_stats { u64 pblk_alloc; + u64 lblk_alloc; }; struct cbd_header { @@ -352,6 +352,7 @@ cbd_header_get(const u8* buf, struct cbd_header* header) header->params.lblk_per_zone = get32_le(&buf); buf += 32; /* Reserved */ header->stats.pblk_alloc = get64_le(&buf); + header->stats.lblk_alloc = get64_le(&buf); } static inline void @@ -370,6 +371,7 @@ cbd_header_put(u8* buf, const struct cbd_header* header) put32_le(&buf, header->params.lblk_per_zone); buf += 32; /* Reserved */ put64_le(&buf, header->stats.pblk_alloc); + put64_le(&buf, header->stats.lblk_alloc); } static inline u32 @@ -509,6 +511,16 @@ lba_put(const struct cbd_params* params, #define COMPRESS_FLUSH_DELAY (HZ / 10) +struct compress_params { + struct block_device* dev; + struct cbd_params params; +}; + +struct compress_stats { + struct mutex lock; + struct cbd_stats stats; +}; + typedef void (*pblk_endio_t)(struct bio*); /* Single page allocator */ @@ -524,11 +536,11 @@ bool cbd_alloc_pagev(struct page** pagev, size_t len); void cbd_free_pagev(struct page** pagev, size_t len); /* Core low-level I/O */ -int pblk_read_wait(struct cbd_params* params, +int pblk_read_wait(struct compress_params* kparams, u64 pblk, u32 count, struct page** pagev); -int pblk_write_wait(struct cbd_params* params, +int pblk_write_wait(struct compress_params* kparams, u64 pblk, u32 count, struct page** pagev); -void pblk_write(struct cbd_params* params, +void pblk_write(struct compress_params* kparams, u64 pblk, u32 count, struct page** pagev); struct pbat; @@ -539,7 +551,7 @@ int pbat_free(struct pbat* pbat, u64 pblk); struct pbatcache; size_t pbatcache_size(void); bool pbatcache_ctr(struct pbatcache* pbatcache, - struct cbd_params* params, u32 cache_pages); + struct compress_params* kparams, u32 cache_pages); void pbatcache_dtr(struct pbatcache* pbatcache); struct pbat* pbatcache_get(struct pbatcache* pbatcache, u32 zone, bool avail); @@ -553,21 +565,22 @@ void lbatpage_put_buf(struct lbatpage* lp); struct lbatpagecache; size_t lbatpagecache_size(void); bool lbatpagecache_ctr(struct lbatpagecache* lpc, - struct cbd_params* params, u32 cache_pages); + struct compress_params* kparams, u32 cache_pages); void lbatpagecache_dtr(struct lbatpagecache* lpc); struct lbatpage* lbatpagecache_get(struct lbatpagecache* lpc, u64 pblk); int lbatpagecache_put(struct lbatpagecache* lpc, struct lbatpage* lpi); struct lbatview; -int lbatview_elem_realloc(struct lbatview* lv, u64 lblk, u32 len, struct cbd_stats* stats); +int lbatview_elem_realloc(struct lbatview* lv, u64 lblk, u32 len); u32 lbatview_elem_len(struct lbatview* lv, u64 lblk); u64 lbatview_elem_pblk(struct lbatview* lv, u64 lblk, u32 idx); struct lbatviewcache; size_t lbatviewcache_size(void); bool lbatviewcache_ctr(struct lbatviewcache* lvc, - struct cbd_params* params, u32 cache_pages); + struct compress_params* kparams, struct compress_stats* kstats, + u32 cache_pages); void lbatviewcache_dtr(struct lbatviewcache* lvc); struct lbatview* lbatviewcache_get(struct lbatviewcache* lvc, u64 lblk); @@ -581,7 +594,7 @@ void lbd_data_write(struct lbd* lbd, u32 off, u32 len, const u8* buf); struct lbdcache; size_t lbdcache_size(void); bool lbdcache_ctr(struct lbdcache* lc, - struct cbd_params* params, struct cbd_stats* stats, + struct compress_params* kparams, struct compress_stats* kstats, u32 cache_pages); void lbdcache_dtr(struct lbdcache* lc); struct lbd* diff --git a/libcbd/check.c b/libcbd/check.c index 16c472b..a5d55bc 100644 --- a/libcbd/check.c +++ b/libcbd/check.c @@ -2,10 +2,37 @@ #include -struct zone_metadata +#include +/* XXX */ +typedef off_t off64_t; +#include + +/* + * Problem: + * - Memory usage is rather high (1.5gb for 700gb device). + * - Startup is quite long. + * Fix: + * - Do not read lbat at start, read in check_lbat(). + * - Do not read pbat at start, read in check_pbat(). + * + * Update stats as we go (pblk_alloc etc.) + * + * With the above, struct zone_metadata name is not really accurate. + * Name it something else, like...??? + */ + +struct check_state { - u8* pbat; - u8* lbat; + int fd; + bool check_lblk_data; + bool clean; + u64 pblk_alloc; + u64 lblk_alloc; + u8** pbatv; + + u8* compress_buf; + u8* lz4_workmem; + z_stream zlib_dstream; }; static void @@ -31,6 +58,53 @@ pblk_read(int fd, u64 pblk, u32 count, u8* data) } } +static void +pblk_write(int fd, u64 pblk, u32 count, const u8* data) +{ + off_t pos; + size_t remain; + ssize_t ret; + + pos = lseek(fd, pblk * PBLK_SIZE, SEEK_SET); + if (pos == (off_t)-1) { + error("Failed to seek\n"); + } + + remain = count * PBLK_SIZE; + while (remain) { + ret = write(fd, data, remain); + if (ret <= 0) { + error("Failed to write\n"); + } + remain -= ret; + data += ret; + } +} + +static void +pbat_read(int fd, const struct cbd_params* params, u32 zone, u8* data) +{ + pblk_read(fd, pbat_off(params, zone), pbat_len(params), data); +} + +static void +pbat_write(int fd, const struct cbd_params* params, u32 zone, const u8* data) +{ + pblk_write(fd, pbat_off(params, zone), pbat_len(params), data); +} + +static void +lbat_read(int fd, const struct cbd_params* params, u32 zone, u8* data) +{ + pblk_read(fd, lbat_off(params, zone), lbat_len(params), data); +} + +static void +lbat_write(int fd, const struct cbd_params* params, u32 zone, const u8* data) +{ + pblk_write(fd, lbat_off(params, zone), lbat_len(params), data); +} + static void check_header(const struct cbd_header* header) { @@ -56,180 +130,330 @@ check_header(const struct cbd_header* header) } } -static void -check_one_lblk(const struct cbd_params* params, - u32 zone, - u32 lblk, - const struct zone_metadata* zm, - u8** pblk_used) +static bool +check_decompress_lz4(struct check_state* state, const struct cbd_params* params, u8* buf, u32 clen) { - struct lba* lba; - u8* lba_buf; + int ret; + u32 dlen = PBLK_SIZE * lblk_per_pblk(params); + + ret = LZ4_decompress_safe((const char*)buf, (char*)state->compress_buf, clen, dlen); + if (ret != dlen) { + return false; + } + + return true; +} + +static bool +check_decompress_zlib(struct check_state* state, const struct cbd_params* params, u8* buf, u32 clen) +{ + int ret; + z_stream* stream; + u32 dlen = PBLK_SIZE * lblk_per_pblk(params); + + stream = &state->zlib_dstream; + ret = inflateReset(stream); + assert(ret == Z_OK); + stream->next_in = buf; + stream->avail_in = clen; + stream->next_out = state->compress_buf; + stream->avail_out = dlen; + ret = inflate(stream, Z_SYNC_FLUSH); + if (ret == Z_OK && !stream->avail_in && stream->avail_out) { + u8 zerostuff = 0; + stream->next_in = &zerostuff; + stream->avail_in = 1; + ret = inflate(stream, Z_FINISH); + } + if (ret != Z_STREAM_END || stream->total_out != dlen) { + return false; + } + + return true; +} + +static bool +check_decompress(struct check_state* state, const struct cbd_params* params, u8* buf, u32 clen) +{ + bool ret = false; + + switch (params->algorithm) { + case CBD_ALG_LZ4: + ret = check_decompress_lz4(state, params, buf, clen); + break; + case CBD_ALG_ZLIB: + ret = check_decompress_zlib(state, params, buf, clen); + break; + default: + ret = false; + } + + return ret; +} + +static bool +check_lblk_data(struct check_state* state, + const struct cbd_params* params, + u64 lblk, u8* lba) +{ + bool ret = false; + u8* data; + u8* buf; + u32 len; u32 n_alloc; u32 n; u64 pblk; - lba = calloc(1, offsetof(struct lba, pblk[lblk_per_pblk(params)])); - lba_buf = zm->lbat + lblk * lba_len(params); - lba_get(params, lba_buf, lba); - if (lba->len) { - if (lba->len == CBD_UNCOMPRESSED) { - printf(" lblk[%u]: UNCOMPRESSED\n", lblk); - } - else { - printf(" lblk[%u]: len=%u\n", lblk, lba->len); + data = calloc(PBLK_SIZE, lblk_per_pblk(params)); + buf = lba; + len = lba_len_get(params, buf); + if (len == 0 || len == CBD_UNCOMPRESSED) { + return true; + } + n_alloc = DIV_ROUND_UP(len, PBLK_SIZE); + for (n = 0; n < n_alloc; ++n) { + pblk = lba_pblk_get(params, buf, n); + pblk_read(state->fd, pblk, 1, data + n * PBLK_SIZE); + } + ret = check_decompress(state, params, data, len); + free(data); + if (!ret) { + if (ask_user_bool("lblk %u: failed to decompress. Clear?", lblk)) { + memset(lba, 0, lba_len(params)); + return true; } + state->clean = false; + } + + return false; +} + +static bool +check_lblk_alloc(struct check_state* state, + const struct cbd_params* params, + u64 lblk, u8* lba) +{ + u8* buf = lba; + u32 len; + u32 n_alloc; + u32 n; + u64 pblk; + u32 pblk_zone; + u32 pblk_off; + + len = lba_len_get(params, buf); + if (!len) { + verbose(2, " lblk[%u]: EMPTY\n", lblk); + return false; + } + if (len == CBD_UNCOMPRESSED) { + verbose(2, " lblk[%u]: UNCOMPRESSED\n", lblk); } else { - printf(" lblk[%u]: EMPTY\n", lblk); + verbose(2, " lblk[%u]: len=%u\n", lblk, len); } - if (lba->len > PBLK_SIZE * lblk_per_pblk(params)) { - printf(" :E: Length out of bounds\n"); - return; + if (len > PBLK_SIZE * lblk_per_pblk(params)) { + if (ask_user_bool("lblk %u: length %u out of bounds. Clear?", lblk, len)) { + memset(lba, 0, lba_len(params)); + return true; + } + state->clean = false; + return false; } - n_alloc = (lba->len == CBD_UNCOMPRESSED) ? + n_alloc = (len == CBD_UNCOMPRESSED) ? lblk_per_pblk(params) : - DIV_ROUND_UP(lba->len, PBLK_SIZE); + DIV_ROUND_UP(len, PBLK_SIZE); for (n = 0; n < n_alloc; ++n) { - u32 pblk_zone; - u32 rel_pblk; - pblk = lba->pblk[n]; + pblk = lba_pblk_get(params, buf, n); if (pblk < CBD_HEADER_BLOCKS) { - printf(" [%u] :E: Alloc in header: %lu\n", n, pblk); + verbose(2, " [%u] :E: Alloc in header: %lu\n", n, pblk); + if (ask_user_bool("lblk %u: alloc %u in header. Clear?", lblk, n)) { + memset(lba, 0, lba_len(params)); + return true; + } + state->clean = false; continue; } - pblk_zone = (pblk - CBD_HEADER_BLOCKS) / zone_len(params); - if (pblk_zone >= params->nr_zones) { - printf(" [%u] :E: Alloc beyond end: %lu\n", n, pblk); + pblk_zone = zone_for_pblk(params, pblk); + if (pblk_zone == ZONE_NONE || pblk_zone >= params->nr_zones) { + verbose(2, " [%u] :E: Alloc beyond end: %lu\n", n, pblk); + if (ask_user_bool("lblk %u: alloc %u beyond end. Clear?", lblk, n)) { + memset(lba, 0, lba_len(params)); + return true; + } + state->clean = false; continue; } if (pblk < zone_data_off(params, pblk_zone)) { - printf(" [%u] :E: Alloc in metadata: %lu\n", n, pblk); + verbose(2, " [%u] :E: Alloc in metadata: %lu\n", n, pblk); + if (ask_user_bool("lblk %u alloc in medatada. Clear?", lblk)) { + memset(lba, 0, lba_len(params)); + return true; + } + state->clean = false; continue; } - rel_pblk = pblk - zone_data_off(params, pblk_zone); - /* XXX: Cannot happen? */ - if (rel_pblk >= pbat_len(params) * PBLK_SIZE_BITS) { - printf(" [%u] :E: Alloc out of zone: %lu\n", n, pblk); + pblk_off = pblk - zone_data_off(params, pblk_zone); + verbose(3, " [%u] pblk=%lu\n", n, (unsigned long)pblk); + if (cbd_bitmap_isset(state->pbatv[pblk_zone], pblk_off)) { + verbose(2, " [%u] :E: Duplicate allocation for pblk %lu\n", n, (unsigned long)pblk); + if (ask_user_bool("lblk %u duplicate alloc for pblk %lu. Clear?", lblk, pblk)) { + memset(lba, 0, lba_len(params)); + return true; + } + state->clean = false; continue; } - printf(" [%u] pblk=%lu pblk_zone=%u rel_pblk=%u\n", n, - (unsigned long)pblk, pblk_zone, rel_pblk); - if (pblk_used[pblk_zone][rel_pblk/8] & (1 << (rel_pblk % 8))) { - printf(" [%u] :E: Duplicate allocation for zone %u block %u\n", - n, pblk_zone, rel_pblk); - continue; - } - pblk_used[pblk_zone][rel_pblk/8] |= (1 << (rel_pblk % 8)); + cbd_bitmap_set(state->pbatv[pblk_zone], pblk_off); } - free(lba); + return false; } static void -check_one_zone(const struct cbd_params* params, - u32 zone, - const struct zone_metadata* zm, - u8** pblk_used) +check_lbat(struct check_state* state, const struct cbd_params* params) { - u32 lblk_alloc_len; - u32 n; - bool zone_empty; - u32 lblk; - - printf("Zone %u: lbat=[%lu..%lu] alloc=[%lu .. %lu]\n", - (unsigned int)zone, - (unsigned long)zone_off(params, zone), - (unsigned long)(zone_data_off(params, zone) - 1), - (unsigned long)zone_data_off(params, zone), - (unsigned long)zone_off(params, zone + 1)); - zone_empty = true; - lblk_alloc_len = params->lblk_per_zone * lba_len(params); - for (n = 0; n < lblk_alloc_len; ++n) { - if (zm->lbat[n]) { - zone_empty = false; - break; - } - } - if (zone_empty) { - printf(" [empty]\n"); - return; - } - for (lblk = 0; lblk < params->lblk_per_zone; ++lblk) { - check_one_lblk(params, zone, lblk, zm, pblk_used); - } -} - -static void -check_zone_metadata(const struct cbd_params* params, - const struct zone_metadata* zmvec) -{ - u8** pblk_used; u32 zone; - pblk_used = calloc(params->nr_zones, sizeof(void*)); for (zone = 0; zone < params->nr_zones; ++zone) { - pblk_used[zone] = calloc(pbat_len(params), PBLK_SIZE); - } + u8* lbat = calloc(PBLK_SIZE, lbat_len(params)); + bool zone_empty = true; + bool changed = false; + u32 n; - for (zone = 0; zone < params->nr_zones; ++zone) { - check_one_zone(params, zone, &zmvec[zone], pblk_used); - } + lbat_read(state->fd, params, zone, lbat); - for (zone = 0; zone < params->nr_zones; ++zone) { - free(pblk_used[zone]); + verbose(2, "Zone %u: lbat=[%lu..%lu] alloc=[%lu .. %lu]\n", + (unsigned int)zone, + (unsigned long)zone_off(params, zone), + (unsigned long)(zone_data_off(params, zone) - 1), + (unsigned long)zone_data_off(params, zone), + (unsigned long)zone_off(params, zone + 1)); + for (n = 0; n < params->lblk_per_zone; ++n) { + u8* buf = lbat + n * lba_len(params); + if (lba_len_get(params, buf) != 0) { + zone_empty = false; + break; + } + } + if (zone_empty) { + verbose(2, " [empty]\n"); + continue; + } + for (n = 0; n < params->lblk_per_zone; ++n) { + u64 lblk = zone * params->lblk_per_zone + n; + u8* buf = lbat + n * lba_len(params); + u32 len; + if (check_lblk_alloc(state, params, lblk, buf)) { + changed = true; + } + if (state->check_lblk_data) { + if (check_lblk_data(state, params, lblk, buf)) { + changed = true; + } + } + len = lba_len_get(params, buf); + if (len != 0) { + ++state->lblk_alloc; + state->pblk_alloc += DIV_ROUND_UP(len, PBLK_SIZE); + } + } + if (changed) { + lbat_write(state->fd, params, zone, lbat); + } + free(lbat); } - free(pblk_used); +} + +static void +check_pbat(struct check_state* state, const struct cbd_params* params) +{ + u32 zone; + u8* pbat; + + pbat = malloc(PBLK_SIZE * pbat_len(params)); + for (zone = 0; zone < params->nr_zones; ++zone) { + bool changed = false; + pbat_read(state->fd, params, zone, pbat); + if (memcmp(pbat, state->pbatv[zone], PBLK_SIZE * pbat_len(params)) != 0) { + if (ask_user_bool("zone %u has incorrect pbat. Fix?", zone)) { + memcpy(pbat, state->pbatv[zone], PBLK_SIZE * pbat_len(params)); + changed = true; + } + else { + state->clean = false; + } + if (changed) { + pbat_write(state->fd, params, zone, state->pbatv[zone]); + } + } + } + free(pbat); } int cbd_check(const char* dev, - bool force, - tristate_t auto_response) + bool force, + tristate_t auto_response, + bool full_check) { - int devfd; + struct check_state state; struct cbd_header header; uint8_t pblkbuf[PBLK_SIZE]; - struct zone_metadata* zmvec; - u32 zone; + u32 n; - devfd = open(dev, O_RDONLY); - if (devfd < 0) { + memset(&state, 0, sizeof(state)); + state.fd = open(dev, O_RDWR); + if (state.fd < 0) { error("Cannot open device\n"); } + state.check_lblk_data = full_check; + state.clean = true; - pblk_read(devfd, 0, 1, pblkbuf); + verbose(1, "Reading header\n"); + pblk_read(state.fd, 0, 1, pblkbuf); cbd_header_get(pblkbuf, &header); + verbose(1, "Checking header\n"); check_header(&header); if (!force && !(header.params.flags & CBD_FLAG_DIRTY)) { printf("%s: clean\n", dev); - close(devfd); + close(state.fd); return 0; } - zmvec = calloc(header.params.nr_zones, sizeof(struct zone_metadata)); - for (zone = 0; zone < header.params.nr_zones; ++zone) { - zmvec[zone].pbat = calloc(pbat_len(&header.params), PBLK_SIZE); - pblk_read(devfd, - pbat_off(&header.params, zone), - pbat_len(&header.params), - zmvec[zone].pbat); - zmvec[zone].lbat = calloc(lbat_len(&header.params), PBLK_SIZE); - pblk_read(devfd, - lbat_off(&header.params, zone), - lbat_len(&header.params), - zmvec[zone].lbat); + state.pbatv = calloc(header.params.nr_zones, sizeof(u8*)); + for (n = 0; n < header.params.nr_zones; ++n) { + state.pbatv[n] = calloc(PBLK_SIZE, pbat_len(&header.params)); } - check_zone_metadata(&header.params, zmvec); + verbose(1, "Checking lbat\n"); + check_lbat(&state, &header.params); + verbose(1, "Checking pbat\n"); + check_pbat(&state, &header.params); - for (zone = 0; zone < header.params.nr_zones; ++zone) { - free(zmvec[zone].lbat); - free(zmvec[zone].pbat); + for (n = 0; n < header.params.nr_zones; ++n) { + free(state.pbatv[n]); } - free(zmvec); + free(state.pbatv); - close(devfd); + if (state.clean) { + if (state.pblk_alloc != header.stats.pblk_alloc) { + verbose(1, "pblk alloc incorrect (%lu expected, %lu found), fixing\n", + (unsigned long)state.pblk_alloc, + (unsigned long)header.stats.pblk_alloc); + header.stats.pblk_alloc = state.pblk_alloc; + } + if (state.lblk_alloc != header.stats.lblk_alloc) { + verbose(1, "lblk alloc incorrect (%lu expected, %lu found), fixing\n", + (unsigned long)state.lblk_alloc, + (unsigned long)header.stats.lblk_alloc); + header.stats.lblk_alloc = state.lblk_alloc; + } + header.params.flags &= ~(CBD_FLAG_ERROR | CBD_FLAG_DIRTY); + cbd_header_put(pblkbuf, &header); + pblk_write(state.fd, 0, 1, pblkbuf); + } + close(state.fd); return 0; } diff --git a/libcbd/util.c b/libcbd/util.c new file mode 100644 index 0000000..7427a4b --- /dev/null +++ b/libcbd/util.c @@ -0,0 +1,65 @@ +#include + +#include + +uint verbose_level; + +void __attribute__((noreturn)) +error(const char* fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + exit(EXIT_FAILURE); +} + +int +verbose(uint level, const char* fmt, ...) +{ + int ret = 0; + va_list ap; + + if (verbose_level >= level) { + va_start(ap, fmt); + vprintf(fmt, ap); + } + + return ret; +} + +bool +ask_user_bool(const char* fmt, ...) +{ + va_list ap; + char* line = NULL; + size_t len = 0; + int ret; + bool answer; + + va_start(ap, fmt); +again: + vprintf(fmt, ap); + printf(" [y/n]? "); + getline(&line, &len, stdin); + if (ret > 0) { + switch (line[0]) { + case 'y': + case 'Y': + answer = true; + break; + case 'n': + case 'N': + answer = false; + break; + default: + ret = -1; + } + } + free(line); + if (ret <= 0) { + goto again; + } + + return answer; +}