WIP: Implement zone caches
This commit is contained in:
parent
07f4221a5f
commit
24401f9a68
|
|
@ -50,13 +50,11 @@ struct dm_compress_io {
|
|||
struct work_struct work;
|
||||
};
|
||||
|
||||
struct dm_compress
|
||||
{
|
||||
struct dm_dev* dev;
|
||||
bool io_failed;
|
||||
|
||||
struct cbd_params params;
|
||||
struct zone_cache {
|
||||
u32 zone;
|
||||
struct mutex lock;
|
||||
|
||||
/* Compression working memory */
|
||||
u8* lz4_wrkmem;
|
||||
u8* lz4_cbuf;
|
||||
|
||||
|
|
@ -78,11 +76,26 @@ struct dm_compress
|
|||
|
||||
void* lblk;
|
||||
|
||||
};
|
||||
|
||||
struct dm_compress
|
||||
{
|
||||
struct dm_dev* dev;
|
||||
bool io_failed;
|
||||
|
||||
struct cbd_params params;
|
||||
struct mutex zc_lock;
|
||||
unsigned int nr_zc;
|
||||
struct zone_cache* zcache;
|
||||
|
||||
/* Queueing stuff */
|
||||
struct workqueue_struct* io_queue;
|
||||
struct mutex io_lock;
|
||||
};
|
||||
|
||||
/* Forward decls */
|
||||
static struct zone_cache* zone_cache_get(struct dm_compress*, u32);
|
||||
static int zone_cache_put(struct dm_compress*, struct zone_cache*);
|
||||
|
||||
static inline int
|
||||
memcmpz(const void* buf, size_t len)
|
||||
{
|
||||
|
|
@ -252,68 +265,65 @@ blkdev_pblk_write(struct dm_compress* dc, u64 pblk, u32 count, void *data)
|
|||
**************************************/
|
||||
|
||||
static int
|
||||
pblk_alloc_write(struct dm_compress* dc)
|
||||
pblk_alloc_write(struct dm_compress* dc, struct zone_cache* zc)
|
||||
{
|
||||
u64 pblk;
|
||||
u32 count;
|
||||
void* pg;
|
||||
|
||||
BUG_ON(dc->pblk_alloc_idx == ZONE_NONE);
|
||||
pblk = pblk_alloc_off(&dc->params, dc->pblk_alloc_idx);
|
||||
BUG_ON(zc->pblk_alloc_idx == ZONE_NONE);
|
||||
pblk = pblk_alloc_off(&dc->params, zc->pblk_alloc_idx);
|
||||
count = pblk_alloc_len(&dc->params);
|
||||
|
||||
pg = compress_alloc_pages(PBLK_SIZE);
|
||||
if (!pg) {
|
||||
return -ENOMEM;
|
||||
}
|
||||
memcpy(pg, dc->pblk_alloc, count * PBLK_SIZE);
|
||||
memcpy(pg, zc->pblk_alloc, count * PBLK_SIZE);
|
||||
blkdev_pblk_write(dc, pblk, count, pg);
|
||||
|
||||
dc->pblk_alloc_dirty = false;
|
||||
zc->pblk_alloc_dirty = false;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int
|
||||
pblk_alloc_flush(struct dm_compress* dc)
|
||||
pblk_alloc_flush(struct dm_compress* dc, struct zone_cache* zc)
|
||||
{
|
||||
int ret;
|
||||
int ret = 0;
|
||||
|
||||
if (dc->pblk_alloc_dirty) {
|
||||
ret = pblk_alloc_write(dc);
|
||||
if (ret) {
|
||||
return ret;
|
||||
}
|
||||
if (zc->pblk_alloc_dirty) {
|
||||
ret = pblk_alloc_write(dc, zc);
|
||||
}
|
||||
|
||||
return 0;
|
||||
return ret;
|
||||
}
|
||||
|
||||
/* Read zone physical block alloc bitmap */
|
||||
static int
|
||||
pblk_alloc_read(struct dm_compress* dc, u32 idx)
|
||||
pblk_alloc_read(struct dm_compress* dc, struct zone_cache* zc)
|
||||
{
|
||||
int ret;
|
||||
u64 pblk;
|
||||
u32 count;
|
||||
|
||||
if (dc->pblk_alloc_idx == idx) {
|
||||
if (zc->pblk_alloc_idx == zc->zone) {
|
||||
return 0;
|
||||
}
|
||||
ret = pblk_alloc_flush(dc);
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
pblk = pblk_alloc_off(&dc->params, idx);
|
||||
count = pblk_alloc_len(&dc->params);
|
||||
|
||||
ret = blkdev_pblk_read(dc, pblk, count, dc->pblk_alloc);
|
||||
ret = pblk_alloc_flush(dc, zc);
|
||||
if (ret) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
dc->pblk_alloc_idx = idx;
|
||||
pblk = pblk_alloc_off(&dc->params, zc->zone);
|
||||
count = pblk_alloc_len(&dc->params);
|
||||
|
||||
ret = blkdev_pblk_read(dc, pblk, count, zc->pblk_alloc);
|
||||
if (ret) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
zc->pblk_alloc_idx = zc->zone;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -323,30 +333,36 @@ pblk_alloc_read(struct dm_compress* dc, u32 idx)
|
|||
* XXX: get rid of this function and use pblk_alloc directly in lblk_write().
|
||||
*/
|
||||
static u64
|
||||
pblk_alloc_get(struct dm_compress* dc, u32 zone_hint)
|
||||
pblk_alloc_get(struct dm_compress* dc, struct zone_cache* zc_hint)
|
||||
{
|
||||
u32 zone_pblk_count = pblk_alloc_len(&dc->params) * PBLK_SIZE_BITS;
|
||||
struct zone_cache* zc;
|
||||
u32 zone;
|
||||
u32 idx;
|
||||
|
||||
zone = zone_hint;
|
||||
zc = zc_hint;
|
||||
zone = zc->zone;
|
||||
/* XXX: check both forward and backward */
|
||||
do {
|
||||
if (pblk_alloc_read(dc, zone) != 0) {
|
||||
if (pblk_alloc_read(dc, zc) != 0) {
|
||||
printk(KERN_ERR " pblk_alloc_read failed\n");
|
||||
return 0;
|
||||
}
|
||||
idx = cbd_bitmap_alloc(dc->pblk_alloc, zone_pblk_count);
|
||||
idx = cbd_bitmap_alloc(zc->pblk_alloc, zone_pblk_count);
|
||||
if (idx != zone_pblk_count) {
|
||||
dc->pblk_alloc_dirty = true;
|
||||
zc->pblk_alloc_dirty = true;
|
||||
if (zc != zc_hint) {
|
||||
zone_cache_put(dc, zc);
|
||||
}
|
||||
return zone_data_off(&dc->params, zone) + idx;
|
||||
}
|
||||
++zone;
|
||||
if (zone == dc->params.nr_zones) {
|
||||
zone = 0;
|
||||
}
|
||||
zc = zone_cache_get(dc, zone);
|
||||
}
|
||||
while (zone != zone_hint);
|
||||
while (zc != zc_hint);
|
||||
printk(KERN_ERR "%s: fail, all zones full\n", __func__);
|
||||
|
||||
return 0;
|
||||
|
|
@ -357,9 +373,10 @@ pblk_alloc_get(struct dm_compress* dc, u32 zone_hint)
|
|||
* XXX: get rid of this function and use pblk_free directly in lblk_write().
|
||||
*/
|
||||
static int
|
||||
pblk_alloc_put(struct dm_compress* dc, u64 pblk)
|
||||
pblk_alloc_put(struct dm_compress* dc, struct zone_cache* zc, u64 pblk)
|
||||
{
|
||||
u32 zone_pblk_count = pblk_alloc_len(&dc->params) * PBLK_SIZE_BITS;
|
||||
bool put_zone = false;
|
||||
u32 zone;
|
||||
u32 idx;
|
||||
int ret;
|
||||
|
|
@ -382,14 +399,25 @@ pblk_alloc_put(struct dm_compress* dc, u64 pblk)
|
|||
printk(KERN_ERR "%s: pblk index out of bounds\n", __func__);
|
||||
return -EINVAL;
|
||||
}
|
||||
ret = pblk_alloc_read(dc, zone);
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
cbd_bitmap_free(dc->pblk_alloc, idx);
|
||||
dc->pblk_alloc_dirty = true;
|
||||
|
||||
return 0;
|
||||
if (zone != zc->zone) {
|
||||
zc = zone_cache_get(dc, zone);
|
||||
put_zone = true;
|
||||
}
|
||||
|
||||
ret = pblk_alloc_read(dc, zc);
|
||||
if (ret) {
|
||||
goto out_put;
|
||||
}
|
||||
cbd_bitmap_free(zc->pblk_alloc, idx);
|
||||
zc->pblk_alloc_dirty = true;
|
||||
|
||||
out_put:
|
||||
if (put_zone) {
|
||||
zone_cache_put(dc, zc);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**************************************
|
||||
|
|
@ -397,7 +425,7 @@ pblk_alloc_put(struct dm_compress* dc, u64 pblk)
|
|||
**************************************/
|
||||
|
||||
static int
|
||||
lblk_alloc_elem_write(struct dm_compress* dc)
|
||||
lblk_alloc_elem_write(struct dm_compress* dc, struct zone_cache* zc)
|
||||
{
|
||||
u32 zone;
|
||||
u32 zone_lblk;
|
||||
|
|
@ -409,32 +437,32 @@ lblk_alloc_elem_write(struct dm_compress* dc)
|
|||
u8* buf;
|
||||
void* pg;
|
||||
|
||||
BUG_ON(dc->lblk_alloc_elem_lblk == LBLK_NONE);
|
||||
BUG_ON(dc->lblk_alloc_pblk == PBLK_NONE);
|
||||
BUG_ON(dc->lblk_alloc_len == 0);
|
||||
BUG_ON(zc->lblk_alloc_elem_lblk == LBLK_NONE);
|
||||
BUG_ON(zc->lblk_alloc_pblk == PBLK_NONE);
|
||||
BUG_ON(zc->lblk_alloc_len == 0);
|
||||
|
||||
zone = dc->lblk_alloc_elem_lblk / dc->params.lblk_per_zone;
|
||||
zone_lblk = dc->lblk_alloc_elem_lblk - (zone * dc->params.lblk_per_zone);
|
||||
zone = zc->lblk_alloc_elem_lblk / dc->params.lblk_per_zone;
|
||||
zone_lblk = zc->lblk_alloc_elem_lblk - (zc->zone * dc->params.lblk_per_zone);
|
||||
elem_off = lblk_alloc_elem_len(&dc->params) * zone_lblk;
|
||||
elem_end = elem_off + lblk_alloc_elem_len(&dc->params);
|
||||
rel_pblk = elem_off / PBLK_SIZE;
|
||||
count = dc->lblk_alloc_len;
|
||||
pblk = dc->lblk_alloc_pblk;
|
||||
buf = dc->lblk_alloc + (elem_off - rel_pblk * PBLK_SIZE);
|
||||
lblk_alloc_elem_put(&dc->params, buf, dc->lblk_alloc_elem);
|
||||
count = zc->lblk_alloc_len;
|
||||
pblk = zc->lblk_alloc_pblk;
|
||||
buf = zc->lblk_alloc + (elem_off - rel_pblk * PBLK_SIZE);
|
||||
lblk_alloc_elem_put(&dc->params, buf, zc->lblk_alloc_elem);
|
||||
|
||||
pg = compress_alloc_pages(count * PBLK_SIZE);
|
||||
if (!pg) {
|
||||
return -ENOMEM;
|
||||
}
|
||||
memcpy(pg, dc->lblk_alloc, count * PBLK_SIZE);
|
||||
memcpy(pg, zc->lblk_alloc, count * PBLK_SIZE);
|
||||
blkdev_pblk_write(dc, pblk, count, pg);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int
|
||||
lblk_alloc_elem_read(struct dm_compress* dc, u64 lblk)
|
||||
lblk_alloc_elem_read(struct dm_compress* dc, struct zone_cache* zc, u64 lblk)
|
||||
{
|
||||
int ret;
|
||||
u32 zone;
|
||||
|
|
@ -446,7 +474,7 @@ lblk_alloc_elem_read(struct dm_compress* dc, u64 lblk)
|
|||
u64 pblk;
|
||||
u8* buf;
|
||||
|
||||
if (dc->lblk_alloc_elem_lblk == lblk) {
|
||||
if (zc->lblk_alloc_elem_lblk == lblk) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -457,17 +485,17 @@ lblk_alloc_elem_read(struct dm_compress* dc, u64 lblk)
|
|||
rel_pblk = elem_off / PBLK_SIZE;
|
||||
count = 1 + (elem_end - 1) / PBLK_SIZE - (elem_off / PBLK_SIZE);
|
||||
pblk = lblk_alloc_off(&dc->params, zone) + rel_pblk;
|
||||
if (dc->lblk_alloc_pblk != pblk || dc->lblk_alloc_len < count) {
|
||||
ret = blkdev_pblk_read(dc, pblk, count, dc->lblk_alloc);
|
||||
if (zc->lblk_alloc_pblk != pblk || zc->lblk_alloc_len < count) {
|
||||
ret = blkdev_pblk_read(dc, pblk, count, zc->lblk_alloc);
|
||||
if (ret != 0) {
|
||||
return ret;
|
||||
}
|
||||
dc->lblk_alloc_pblk = pblk;
|
||||
dc->lblk_alloc_len = count;
|
||||
zc->lblk_alloc_pblk = pblk;
|
||||
zc->lblk_alloc_len = count;
|
||||
}
|
||||
buf = dc->lblk_alloc + (elem_off - rel_pblk * PBLK_SIZE);
|
||||
lblk_alloc_elem_get(&dc->params, buf, dc->lblk_alloc_elem);
|
||||
dc->lblk_alloc_elem_lblk = lblk;
|
||||
buf = zc->lblk_alloc + (elem_off - rel_pblk * PBLK_SIZE);
|
||||
lblk_alloc_elem_get(&dc->params, buf, zc->lblk_alloc_elem);
|
||||
zc->lblk_alloc_elem_lblk = lblk;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -482,15 +510,15 @@ lblk_alloc_elem_read(struct dm_compress* dc, u64 lblk)
|
|||
* Returns number of bytes in cbuf or 0 for failure.
|
||||
*/
|
||||
static size_t
|
||||
lblk_compress(struct dm_compress* dc)
|
||||
lblk_compress(struct cbd_params* params, struct zone_cache* zc)
|
||||
{
|
||||
int ret;
|
||||
void *dbuf = dc->lblk;
|
||||
u32 dlen = PBLK_SIZE * lblk_per_pblk(&dc->params);
|
||||
void *cbuf = dc->lz4_cbuf;
|
||||
u32 clen = PBLK_SIZE * lblk_per_pblk(&dc->params);
|
||||
void *dbuf = zc->lblk;
|
||||
u32 dlen = PBLK_SIZE * lblk_per_pblk(params);
|
||||
void *cbuf = zc->lz4_cbuf;
|
||||
u32 clen = PBLK_SIZE * lblk_per_pblk(params);
|
||||
|
||||
ret = LZ4_compress_default(dbuf, cbuf, dlen, clen, dc->lz4_wrkmem);
|
||||
ret = LZ4_compress_default(dbuf, cbuf, dlen, clen, zc->lz4_wrkmem);
|
||||
if (ret <= 0) {
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -504,12 +532,12 @@ lblk_compress(struct dm_compress* dc)
|
|||
* Returns 0 for success, <0 for failure.
|
||||
*/
|
||||
static int
|
||||
lblk_decompress(struct dm_compress* dc, u32 clen)
|
||||
lblk_decompress(struct cbd_params* params, struct zone_cache* zc, u32 clen)
|
||||
{
|
||||
int ret;
|
||||
void *cbuf = dc->lz4_cbuf;
|
||||
void *dbuf = dc->lblk;
|
||||
u32 dlen = PBLK_SIZE * lblk_per_pblk(&dc->params);
|
||||
void *cbuf = zc->lz4_cbuf;
|
||||
void *dbuf = zc->lblk;
|
||||
u32 dlen = PBLK_SIZE * lblk_per_pblk(params);
|
||||
|
||||
ret = LZ4_decompress_safe(cbuf, dbuf, clen, dlen);
|
||||
if (ret != dlen) {
|
||||
|
|
@ -521,7 +549,7 @@ lblk_decompress(struct dm_compress* dc, u32 clen)
|
|||
}
|
||||
|
||||
static int
|
||||
lblk_write(struct dm_compress* dc)
|
||||
lblk_write(struct dm_compress* dc, struct zone_cache* zc)
|
||||
{
|
||||
int ret;
|
||||
u32 zone;
|
||||
|
|
@ -533,51 +561,51 @@ lblk_write(struct dm_compress* dc)
|
|||
u32 n;
|
||||
u64 pblk;
|
||||
|
||||
zone = dc->lblk_num / dc->params.lblk_per_zone;
|
||||
zone_lblk = dc->lblk_num - (zone * dc->params.lblk_per_zone);
|
||||
elem_buf = dc->lblk_alloc + zone_lblk * lblk_alloc_elem_len(&dc->params);
|
||||
zone = zc->zone;
|
||||
zone_lblk = zc->lblk_num - (zone * dc->params.lblk_per_zone);
|
||||
elem_buf = zc->lblk_alloc + zone_lblk * lblk_alloc_elem_len(&dc->params);
|
||||
|
||||
/* We must have a cached lblk elem */
|
||||
BUG_ON(dc->lblk_alloc_elem_lblk == LBLK_NONE);
|
||||
BUG_ON(zc->lblk_alloc_elem_lblk == LBLK_NONE);
|
||||
|
||||
d_len = PBLK_SIZE * lblk_per_pblk(&dc->params);
|
||||
#ifdef CBD_DETECT_ZERO_BLOCKS
|
||||
if (memcmpz(dc->lblk, d_len) == 0) {
|
||||
if (memcmpz(zc->lblk, d_len) == 0) {
|
||||
#else
|
||||
if (0) {
|
||||
#endif
|
||||
c_len = 0;
|
||||
c_buf = NULL;
|
||||
dc->lblk_alloc_elem->len = 0;
|
||||
zc->lblk_alloc_elem->len = 0;
|
||||
}
|
||||
else {
|
||||
c_len = lblk_compress(dc);
|
||||
c_len = lblk_compress(&dc->params, zc);
|
||||
if (c_len > 0) {
|
||||
size_t c_blkrem = c_len % PBLK_SIZE;
|
||||
if (c_blkrem) {
|
||||
memset(dc->lz4_cbuf + c_len, 0, c_blkrem);
|
||||
memset(zc->lz4_cbuf + c_len, 0, c_blkrem);
|
||||
}
|
||||
c_buf = dc->lz4_cbuf;
|
||||
dc->lblk_alloc_elem->len = c_len;
|
||||
c_buf = zc->lz4_cbuf;
|
||||
zc->lblk_alloc_elem->len = c_len;
|
||||
}
|
||||
else {
|
||||
c_len = d_len;
|
||||
c_buf = dc->lblk;
|
||||
dc->lblk_alloc_elem->len = CBD_UNCOMPRESSED;
|
||||
c_buf = zc->lblk;
|
||||
zc->lblk_alloc_elem->len = CBD_UNCOMPRESSED;
|
||||
}
|
||||
}
|
||||
|
||||
for (n = 0; n < lblk_per_pblk(&dc->params); ++n) {
|
||||
if (c_len > PBLK_SIZE * n) {
|
||||
void* pg;
|
||||
pblk = dc->lblk_alloc_elem->pblk[n];
|
||||
pblk = zc->lblk_alloc_elem->pblk[n];
|
||||
if (!pblk) {
|
||||
pblk = pblk_alloc_get(dc, zone);
|
||||
pblk = pblk_alloc_get(dc, zc);
|
||||
if (pblk == 0) {
|
||||
printk(KERN_ERR " pblk_alloc_get failed\n");
|
||||
return -ENOSPC;
|
||||
}
|
||||
dc->lblk_alloc_elem->pblk[n] = pblk;
|
||||
zc->lblk_alloc_elem->pblk[n] = pblk;
|
||||
}
|
||||
pg = compress_alloc_pages(PBLK_SIZE);
|
||||
if (!pg) {
|
||||
|
|
@ -588,10 +616,10 @@ lblk_write(struct dm_compress* dc)
|
|||
c_buf += PBLK_SIZE;
|
||||
}
|
||||
else {
|
||||
pblk = dc->lblk_alloc_elem->pblk[n];
|
||||
pblk = zc->lblk_alloc_elem->pblk[n];
|
||||
if (pblk) {
|
||||
dc->lblk_alloc_elem->pblk[n] = 0;
|
||||
ret = pblk_alloc_put(dc, pblk);
|
||||
zc->lblk_alloc_elem->pblk[n] = 0;
|
||||
ret = pblk_alloc_put(dc, zc, pblk);
|
||||
if (ret != 0) {
|
||||
printk(KERN_ERR " pblk_alloc_put failed\n");
|
||||
return ret;
|
||||
|
|
@ -600,29 +628,29 @@ lblk_write(struct dm_compress* dc)
|
|||
}
|
||||
}
|
||||
|
||||
ret = lblk_alloc_elem_write(dc);
|
||||
ret = lblk_alloc_elem_write(dc, zc);
|
||||
if (ret != 0) {
|
||||
printk(KERN_ERR " lblk_alloc_elem_write failed\n");
|
||||
return ret;
|
||||
}
|
||||
ret = pblk_alloc_flush(dc);
|
||||
ret = pblk_alloc_flush(dc, zc);
|
||||
if (ret != 0) {
|
||||
printk(KERN_ERR " pblk_alloc_flush failed\n");
|
||||
return ret;
|
||||
}
|
||||
|
||||
dc->lblk_dirty = false;
|
||||
zc->lblk_dirty = false;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int
|
||||
lblk_flush(struct dm_compress* dc)
|
||||
lblk_flush(struct dm_compress* dc, struct zone_cache* zc)
|
||||
{
|
||||
int ret;
|
||||
|
||||
if (dc->lblk_dirty) {
|
||||
ret = lblk_write(dc);
|
||||
if (zc->lblk_dirty) {
|
||||
ret = lblk_write(dc, zc);
|
||||
if (ret) {
|
||||
return ret;
|
||||
}
|
||||
|
|
@ -632,7 +660,7 @@ lblk_flush(struct dm_compress* dc)
|
|||
}
|
||||
|
||||
static int
|
||||
lblk_read(struct dm_compress* dc, u64 idx)
|
||||
lblk_read(struct dm_compress* dc, struct zone_cache* zc, u64 idx)
|
||||
{
|
||||
int ret;
|
||||
u32 zone;
|
||||
|
|
@ -641,27 +669,27 @@ lblk_read(struct dm_compress* dc, u64 idx)
|
|||
u32 c_len;
|
||||
u64 pblk;
|
||||
|
||||
if (dc->lblk_num == idx) {
|
||||
if (zc->lblk_num == idx) {
|
||||
return 0;
|
||||
}
|
||||
ret = lblk_flush(dc);
|
||||
ret = lblk_flush(dc, zc);
|
||||
if (ret) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
zone = idx / dc->params.lblk_per_zone;
|
||||
zone_lblk = idx - (zone * dc->params.lblk_per_zone);
|
||||
elem_buf = dc->lblk_alloc + zone_lblk * lblk_alloc_elem_len(&dc->params);
|
||||
elem_buf = zc->lblk_alloc + zone_lblk * lblk_alloc_elem_len(&dc->params);
|
||||
|
||||
ret = lblk_alloc_elem_read(dc, idx);
|
||||
ret = lblk_alloc_elem_read(dc, zc, idx);
|
||||
if (ret != 0) {
|
||||
printk(KERN_ERR " lblk_alloc_elem_read failed\n");
|
||||
return ret;
|
||||
}
|
||||
|
||||
c_len = dc->lblk_alloc_elem->len;
|
||||
c_len = zc->lblk_alloc_elem->len;
|
||||
if (c_len == 0) {
|
||||
memset(dc->lblk, 0, PBLK_SIZE * lblk_per_pblk(&dc->params));
|
||||
memset(zc->lblk, 0, PBLK_SIZE * lblk_per_pblk(&dc->params));
|
||||
}
|
||||
else {
|
||||
bool is_compressed = true;
|
||||
|
|
@ -673,9 +701,9 @@ lblk_read(struct dm_compress* dc, u64 idx)
|
|||
is_compressed = false;
|
||||
c_len = d_len;
|
||||
}
|
||||
p = dc->lz4_cbuf;
|
||||
p = zc->lz4_cbuf;
|
||||
for (n = 0; n * PBLK_SIZE < c_len; ++n, p += PBLK_SIZE) {
|
||||
pblk = dc->lblk_alloc_elem->pblk[n];
|
||||
pblk = zc->lblk_alloc_elem->pblk[n];
|
||||
BUG_ON(pblk == 0);
|
||||
ret = blkdev_pblk_read(dc, pblk, 1, p);
|
||||
if (ret != 0) {
|
||||
|
|
@ -683,89 +711,169 @@ lblk_read(struct dm_compress* dc, u64 idx)
|
|||
}
|
||||
}
|
||||
if (is_compressed) {
|
||||
if (lblk_decompress(dc, c_len) != 0) {
|
||||
if (lblk_decompress(&dc->params, zc, c_len) != 0) {
|
||||
printk(KERN_ERR " decompress failed\n");
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
else {
|
||||
memcpy(dc->lblk, dc->lz4_cbuf, d_len);
|
||||
memcpy(zc->lblk, zc->lz4_cbuf, d_len);
|
||||
}
|
||||
}
|
||||
|
||||
dc->lblk_num = idx;
|
||||
zc->lblk_num = idx;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**************************************
|
||||
* Main functions
|
||||
* Zone cache functions
|
||||
**************************************/
|
||||
|
||||
static void
|
||||
compress_free_buffers(struct dm_compress* dc)
|
||||
zone_cache_reset(struct zone_cache* zc, u32 zone)
|
||||
{
|
||||
compress_free_pages(dc->lblk, PBLK_SIZE * lblk_per_pblk(&dc->params));
|
||||
dc->lblk = NULL;
|
||||
|
||||
kfree(dc->lblk_alloc_elem);
|
||||
dc->lblk_alloc_elem = NULL;
|
||||
|
||||
compress_free_pages(dc->lblk_alloc, PBLK_SIZE * 2);
|
||||
dc->lblk_alloc = NULL;
|
||||
|
||||
compress_free_pages(dc->pblk_alloc, PBLK_SIZE * pblk_alloc_len(&dc->params));
|
||||
dc->pblk_alloc = NULL;
|
||||
|
||||
compress_free_pages(dc->lz4_cbuf, PBLK_SIZE * lblk_per_pblk(&dc->params));
|
||||
dc->lz4_cbuf = NULL;
|
||||
|
||||
kfree(dc->lz4_wrkmem);
|
||||
dc->lz4_wrkmem = NULL;
|
||||
zc->zone = zone;
|
||||
zc->pblk_alloc_idx = ZONE_NONE;
|
||||
zc->pblk_alloc_dirty = false;
|
||||
zc->lblk_alloc_pblk = PBLK_NONE;
|
||||
zc->lblk_alloc_len = 0;
|
||||
zc->lblk_alloc_elem_lblk = LBLK_NONE;
|
||||
zc->lblk_num = LBLK_NONE;
|
||||
zc->lblk_dirty = false;
|
||||
}
|
||||
|
||||
/*
|
||||
* XXX: Many of the below (all except lz4 buffers) are used in bio operations
|
||||
* and should be page aligned. We always get page aligned buffers because of
|
||||
* the way kmalloc() works, but that is technically not guaranteed.
|
||||
*/
|
||||
static int
|
||||
compress_alloc_buffers(struct dm_compress* dc)
|
||||
zone_cache_flush(struct dm_compress* dc, struct zone_cache* zc)
|
||||
{
|
||||
dc->lz4_wrkmem = kmalloc(LZ4_compressBound(PBLK_SIZE * lblk_per_pblk(&dc->params)), GFP_KERNEL);
|
||||
if (!dc->lz4_wrkmem) {
|
||||
int ret;
|
||||
|
||||
ret = lblk_flush(dc, zc);
|
||||
if (ret) {
|
||||
return ret;
|
||||
}
|
||||
ret = pblk_alloc_flush(dc, zc);
|
||||
if (ret) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static struct zone_cache*
|
||||
zone_cache_get(struct dm_compress* dc, u32 zone)
|
||||
{
|
||||
struct zone_cache* zc;
|
||||
u32 idx;
|
||||
|
||||
//printk(KERN_INFO "%s: zone=%u\n", __func__, (unsigned int)zone);
|
||||
|
||||
mutex_lock(&dc->zc_lock);
|
||||
for (idx = 0; idx < dc->nr_zc; ++idx) {
|
||||
zc = &dc->zcache[idx];
|
||||
if (zc->zone == zone) {
|
||||
mutex_lock(&zc->lock);
|
||||
goto out;
|
||||
}
|
||||
}
|
||||
for (idx = 0; idx < dc->nr_zc; ++idx) {
|
||||
zc = &dc->zcache[idx];
|
||||
if (zc->zone == ZONE_NONE) {
|
||||
zone_cache_reset(zc, zone);
|
||||
mutex_lock(&zc->lock);
|
||||
goto out;
|
||||
}
|
||||
}
|
||||
for (idx = 0; idx < dc->nr_zc; ++idx) {
|
||||
zc = &dc->zcache[idx];
|
||||
if (mutex_trylock(&zc->lock) == 1) {
|
||||
zone_cache_reset(zc, zone);
|
||||
goto out;
|
||||
}
|
||||
}
|
||||
printk(KERN_ERR "%s: Cannot get zone %u\n", __func__, (unsigned int)zone);
|
||||
zc = NULL;
|
||||
|
||||
out:
|
||||
mutex_unlock(&dc->zc_lock);
|
||||
return zc;
|
||||
}
|
||||
|
||||
static int
|
||||
zone_cache_put(struct dm_compress* dc, struct zone_cache* zc)
|
||||
{
|
||||
int ret;
|
||||
|
||||
//printk(KERN_INFO "%s: zone=%u\n", __func__, (unsigned int)zc->zone);
|
||||
|
||||
ret = zone_cache_flush(dc, zc);
|
||||
mutex_unlock(&zc->lock);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void
|
||||
zone_cache_dtr(struct dm_compress* dc, struct zone_cache* zc)
|
||||
{
|
||||
compress_free_pages(zc->lblk, PBLK_SIZE * lblk_per_pblk(&dc->params));
|
||||
zc->lblk = NULL;
|
||||
|
||||
kfree(zc->lblk_alloc_elem);
|
||||
zc->lblk_alloc_elem = NULL;
|
||||
|
||||
compress_free_pages(zc->lblk_alloc, PBLK_SIZE * 2);
|
||||
zc->lblk_alloc = NULL;
|
||||
|
||||
compress_free_pages(zc->pblk_alloc, PBLK_SIZE * pblk_alloc_len(&dc->params));
|
||||
zc->pblk_alloc = NULL;
|
||||
|
||||
compress_free_pages(zc->lz4_cbuf, PBLK_SIZE * lblk_per_pblk(&dc->params));
|
||||
zc->lz4_cbuf = NULL;
|
||||
|
||||
kfree(zc->lz4_wrkmem);
|
||||
zc->lz4_wrkmem = NULL;
|
||||
}
|
||||
|
||||
static int
|
||||
zone_cache_ctr(struct dm_compress* dc, struct zone_cache* zc)
|
||||
{
|
||||
zc->zone = ZONE_NONE;
|
||||
mutex_init(&zc->lock);
|
||||
|
||||
zc->lz4_wrkmem = kmalloc(LZ4_compressBound(PBLK_SIZE * lblk_per_pblk(&dc->params)), GFP_KERNEL);
|
||||
if (!zc->lz4_wrkmem) {
|
||||
printk(KERN_ERR "%s: Failed to alloc lz4_wrkmem\n", __func__);
|
||||
goto out_nomem;
|
||||
}
|
||||
dc->lz4_cbuf = compress_alloc_pages(PBLK_SIZE * lblk_per_pblk(&dc->params));
|
||||
if (!dc->lz4_cbuf) {
|
||||
zc->lz4_cbuf = compress_alloc_pages(PBLK_SIZE * lblk_per_pblk(&dc->params));
|
||||
if (!zc->lz4_cbuf) {
|
||||
printk(KERN_ERR "%s: Failed to alloc lz4_cmem\n", __func__);
|
||||
goto out_nomem;
|
||||
}
|
||||
dc->pblk_alloc_idx = ZONE_NONE;
|
||||
dc->pblk_alloc_dirty = false;
|
||||
dc->pblk_alloc = compress_alloc_pages(PBLK_SIZE * pblk_alloc_len(&dc->params));
|
||||
if (!dc->pblk_alloc) {
|
||||
zc->pblk_alloc_idx = ZONE_NONE;
|
||||
zc->pblk_alloc_dirty = false;
|
||||
zc->pblk_alloc = compress_alloc_pages(PBLK_SIZE * pblk_alloc_len(&dc->params));
|
||||
if (!zc->pblk_alloc) {
|
||||
printk(KERN_ERR "%s: Failed to alloc pblk_alloc\n", __func__);
|
||||
goto out_nomem;
|
||||
}
|
||||
dc->lblk_alloc_pblk = PBLK_NONE;
|
||||
dc->lblk_alloc_len = 0;
|
||||
dc->lblk_alloc = compress_alloc_pages(PBLK_SIZE * 2);
|
||||
if (!dc->lblk_alloc) {
|
||||
zc->lblk_alloc_pblk = PBLK_NONE;
|
||||
zc->lblk_alloc_len = 0;
|
||||
zc->lblk_alloc = compress_alloc_pages(PBLK_SIZE * 2);
|
||||
if (!zc->lblk_alloc) {
|
||||
printk(KERN_ERR "%s: Failed to alloc lblk_alloc\n", __func__);
|
||||
goto out_nomem;
|
||||
}
|
||||
dc->lblk_alloc_elem_lblk = LBLK_NONE;
|
||||
dc->lblk_alloc_elem = kmalloc(offsetof(struct lblk_alloc_elem, pblk[lblk_per_pblk(&dc->params)]), GFP_KERNEL);
|
||||
if (!dc->lblk_alloc_elem) {
|
||||
zc->lblk_alloc_elem_lblk = LBLK_NONE;
|
||||
zc->lblk_alloc_elem = kmalloc(offsetof(struct lblk_alloc_elem, pblk[lblk_per_pblk(&dc->params)]), GFP_KERNEL);
|
||||
if (!zc->lblk_alloc_elem) {
|
||||
printk(KERN_ERR "%s: Failed to alloc lblk_alloc_elem\n", __func__);
|
||||
goto out_nomem;
|
||||
}
|
||||
dc->lblk_num = LBLK_NONE;
|
||||
dc->lblk_dirty = false;
|
||||
dc->lblk = compress_alloc_pages(PBLK_SIZE * lblk_per_pblk(&dc->params));
|
||||
if (!dc->lblk) {
|
||||
zc->lblk_num = LBLK_NONE;
|
||||
zc->lblk_dirty = false;
|
||||
zc->lblk = compress_alloc_pages(PBLK_SIZE * lblk_per_pblk(&dc->params));
|
||||
if (!zc->lblk) {
|
||||
printk(KERN_ERR "%s: Failed to alloc lblk\n", __func__);
|
||||
goto out_nomem;
|
||||
}
|
||||
|
|
@ -773,10 +881,14 @@ compress_alloc_buffers(struct dm_compress* dc)
|
|||
return 0;
|
||||
|
||||
out_nomem:
|
||||
compress_free_buffers(dc);
|
||||
zone_cache_dtr(dc, zc);
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
/**************************************
|
||||
* Main functions
|
||||
**************************************/
|
||||
|
||||
static int
|
||||
compress_open(struct dm_compress* dc, u64 dev_nr_pblks)
|
||||
{
|
||||
|
|
@ -784,6 +896,7 @@ compress_open(struct dm_compress* dc, u64 dev_nr_pblks)
|
|||
u8 *pblkbuf;
|
||||
struct cbd_header header;
|
||||
u64 max_nr_zones;
|
||||
unsigned int n;
|
||||
|
||||
pblkbuf = kmalloc(PBLK_SIZE, GFP_KERNEL);
|
||||
if (!pblkbuf) {
|
||||
|
|
@ -846,22 +959,30 @@ compress_open(struct dm_compress* dc, u64 dev_nr_pblks)
|
|||
|
||||
memcpy(&dc->params, &header.params, sizeof(header.params));
|
||||
|
||||
err = compress_alloc_buffers(dc);
|
||||
if (err) {
|
||||
printk(KERN_ERR "%s: failed to alloc buffers\n", __func__);
|
||||
mutex_init(&dc->zc_lock);
|
||||
dc->nr_zc = min(2 * num_online_cpus(), dc->params.nr_zones);
|
||||
dc->zcache = kmalloc(dc->nr_zc * sizeof(struct zone_cache), GFP_KERNEL);
|
||||
if (!dc->zcache) {
|
||||
printk(KERN_ERR "%s: out of memory\n", __func__);
|
||||
goto out;
|
||||
}
|
||||
for (n = 0; n < dc->nr_zc; ++n) {
|
||||
err = zone_cache_ctr(dc, &dc->zcache[n]);
|
||||
if (err) {
|
||||
printk(KERN_ERR "%s: failed to init zone cache\n", __func__);
|
||||
goto out;
|
||||
}
|
||||
}
|
||||
|
||||
dc->io_queue = alloc_workqueue("kcompress_io", WQ_HIGHPRI | WQ_MEM_RECLAIM, 1);
|
||||
if (!dc->io_queue) {
|
||||
printk(KERN_ERR "%s: failed to alloc io_queue\n", __func__);
|
||||
compress_free_buffers(dc);
|
||||
return -ENOMEM;
|
||||
err = -ENOMEM;
|
||||
goto out;
|
||||
}
|
||||
|
||||
mutex_init(&dc->io_lock);
|
||||
|
||||
out:
|
||||
/* XXX: cleanup on error */
|
||||
kfree(pblkbuf);
|
||||
|
||||
return err;
|
||||
|
|
@ -879,22 +1000,29 @@ compress_read(struct dm_compress *dc, struct bio *bio)
|
|||
bio_for_each_segment(bv, bio, iter) {
|
||||
sector_t lblk = iter.bi_sector / lblk_per_sector;
|
||||
u32 lblk_off = (iter.bi_sector - lblk * lblk_per_sector) * SECTOR_SIZE;
|
||||
u32 zone = lblk / dc->params.lblk_per_zone;
|
||||
struct zone_cache* zc = NULL;
|
||||
unsigned long flags;
|
||||
char* data;
|
||||
|
||||
zc = zone_cache_get(dc, zone);
|
||||
|
||||
/* Ensure the data is within the logical block */
|
||||
if (lblk_off + bv.bv_len > lblk_len) {
|
||||
printk(KERN_ERR "%s: logical block bounds exceeded\n", __func__);
|
||||
return -EIO;
|
||||
}
|
||||
/* BUG_ON(lblk_off + bv.bv_offset + bv.bv_len > PBLK_SIZE + lblk_per_pblk(dc)); */
|
||||
ret = lblk_read(dc, lblk);
|
||||
ret = lblk_read(dc, zc, lblk);
|
||||
if (ret) {
|
||||
zone_cache_put(dc, zc);
|
||||
return ret;
|
||||
}
|
||||
data = bvec_kmap_irq(&bv, &flags);
|
||||
memcpy(data, dc->lblk + lblk_off, bv.bv_len);
|
||||
memcpy(data, zc->lblk + lblk_off, bv.bv_len);
|
||||
bvec_kunmap_irq(data, &flags);
|
||||
|
||||
zone_cache_put(dc, zc);
|
||||
}
|
||||
|
||||
return 0;
|
||||
|
|
@ -912,9 +1040,13 @@ compress_write(struct dm_compress *dc, struct bio *bio)
|
|||
bio_for_each_segment(bv, bio, iter) {
|
||||
sector_t lblk = iter.bi_sector / lblk_per_sector;
|
||||
u32 lblk_off = (iter.bi_sector - lblk * lblk_per_sector) * SECTOR_SIZE;
|
||||
u32 zone = lblk / dc->params.lblk_per_zone;
|
||||
struct zone_cache* zc = NULL;
|
||||
unsigned long flags;
|
||||
char* data;
|
||||
|
||||
zc = zone_cache_get(dc, zone);
|
||||
|
||||
/* Ensure the data is within the logical block */
|
||||
if (lblk_off + bv.bv_len > lblk_len) {
|
||||
printk(KERN_ERR "%s logical block bounds exceeded\n", __func__);
|
||||
|
|
@ -924,18 +1056,17 @@ compress_write(struct dm_compress *dc, struct bio *bio)
|
|||
return -EIO;
|
||||
}
|
||||
/* BUG_ON(lblk_off + bv.bv_offset + bv.bv_len > PBLK_SIZE + lblk_per_pblk(dc)); */
|
||||
ret = lblk_read(dc, lblk);
|
||||
ret = lblk_read(dc, zc, lblk);
|
||||
if (ret) {
|
||||
zone_cache_put(dc, zc);
|
||||
return ret;
|
||||
}
|
||||
data = bvec_kmap_irq(&bv, &flags);
|
||||
memcpy(dc->lblk + lblk_off, data, bv.bv_len);
|
||||
memcpy(zc->lblk + lblk_off, data, bv.bv_len);
|
||||
bvec_kunmap_irq(data, &flags);
|
||||
dc->lblk_dirty = true;
|
||||
}
|
||||
ret = lblk_flush(dc);
|
||||
if (ret) {
|
||||
return ret;
|
||||
zc->lblk_dirty = true;
|
||||
|
||||
zone_cache_put(dc, zc);
|
||||
}
|
||||
|
||||
return 0;
|
||||
|
|
@ -947,8 +1078,6 @@ static void compress_io(struct dm_compress_io* io)
|
|||
struct dm_compress* dc = io->dc;
|
||||
struct bio* bio = io->bio;
|
||||
|
||||
mutex_lock(&dc->io_lock);
|
||||
|
||||
switch (bio_op(bio)) {
|
||||
case REQ_OP_READ:
|
||||
ret = compress_read(dc, bio);
|
||||
|
|
@ -964,8 +1093,6 @@ static void compress_io(struct dm_compress_io* io)
|
|||
printk(KERN_ERR "%s: failed, ret=%d\n", __func__, ret);
|
||||
}
|
||||
|
||||
mutex_unlock(&dc->io_lock);
|
||||
|
||||
bio->bi_status = (ret == 0 ? BLK_STS_OK : BLK_STS_IOERR); /* XXX */
|
||||
bio_endio(bio);
|
||||
}
|
||||
|
|
@ -1072,11 +1199,17 @@ static void
|
|||
compress_dtr(struct dm_target *ti)
|
||||
{
|
||||
struct dm_compress *dc;
|
||||
unsigned int n;
|
||||
|
||||
printk(KERN_INFO "%s: enter\n", __func__);
|
||||
|
||||
dc = (struct dm_compress *)ti->private;
|
||||
compress_free_buffers(dc);
|
||||
if (dc->zcache) {
|
||||
for (n = 0; n < dc->nr_zc; ++n) {
|
||||
zone_cache_dtr(dc, &dc->zcache[n]);
|
||||
}
|
||||
kfree(dc->zcache);
|
||||
}
|
||||
if (dc->io_queue) {
|
||||
destroy_workqueue(dc->io_queue);
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue