tdm
/
cbd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use per-cpu lz4 state

This commit is contained in:
Tom Marshall 2019-10-30 12:34:21 -07:00
parent 290b1873a0
commit eeafc209a5
4 changed files with 109 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
dm-compress/compress.c
View File

@ -53,6 +53,7 @@ struct compress
struct dm_dev* dev;
struct cbd_params params;
void* percpu;
struct lbdcache* lc;
struct workqueue_struct* io_workq;
@ -150,6 +151,7 @@ compress_open(struct compress* c, u64 dev_nr_pblks)
printk(KERN_INFO " lblk_per_zone=%u\n", (unsigned int)header.params.lblk_per_zone);
memcpy(&c->params, &header.params, sizeof(header.params));
c->percpu = alloc_percpu(void*);
c->lc = kmalloc(lbdcache_size(), GFP_KERNEL);
if (!c->lc) {
@ -157,7 +159,7 @@ compress_open(struct compress* c, u64 dev_nr_pblks)
printk(KERN_ERR "Failed to alloc lbdcache\n");
goto out;
}
if (!lbdcache_ctr(c->lc, &c->params)) {
if (!lbdcache_ctr(c->lc, &c->params, c->percpu)) {
err = -ENOMEM;
printk(KERN_ERR "Failed to init logical block cache\n");
goto out;
@ -404,6 +406,7 @@ compress_dtr(struct dm_target *ti)
c = ti->private;
lbdcache_dtr(c->lc);
kfree(c->lc);
free_percpu(c->percpu);
if (c->io_workq) {
destroy_workqueue(c->io_workq);
}

130
dm-compress/lbd.c
View File

@ -39,9 +39,7 @@ struct lbd {
struct cbd_params* params;
struct lbatviewcache* lvc;
struct lbatview* lv;
u8* lz4_wrkmem;
struct page* lz4_cpages;
u8* lz4_cbuf;
void* percpu;
struct page* pages;
u8* buf;
};
@ -93,6 +91,42 @@ lblk_is_zeros(struct cbd_params* params, struct lbd* lbd)
#endif
}
struct lz4_state {
u8* wrkmem;
struct page* pages;
u8* buf;
};
static struct lz4_state*
lblk_get_lz4_state(struct lbd* lbd, int cpu)
{
struct lz4_state** statep;
struct lz4_state* state;
statep = per_cpu_ptr(lbd->percpu, cpu);
if (*statep) {
return *statep;
}
state = kmalloc(sizeof(struct lz4_state), GFP_NOWAIT);
if (!state) {
printk(KERN_ERR "%s: failed to alloc state\n", __func__);
return NULL;
}
state->wrkmem = kmalloc(LZ4_compressBound(PBLK_SIZE * lblk_per_pblk(lbd->params)), GFP_NOWAIT);
state->pages = cbd_alloc_pages_nowait(lblk_per_pblk(lbd->params));
if (!state->wrkmem || !state->pages) {
kfree(state->wrkmem);
cbd_free_pages(state->pages, lblk_per_pblk(lbd->params));
kfree(state);
printk(KERN_ERR "%s: failed to alloc buffers\n", __func__);
return NULL;
}
state->buf = page_address(state->pages);
*statep = state;
return state;
}
/*
* Compress dc->lblk into dc->lz4_cbuf
*
@ -101,18 +135,29 @@ lblk_is_zeros(struct cbd_params* params, struct lbd* lbd)
static size_t
lblk_compress(struct lbd* lbd)
{
int ret;
void *dbuf = lbd->buf;
u32 dlen = PBLK_SIZE * lblk_per_pblk(lbd->params);
void *cbuf = lbd->lz4_cbuf;
u32 clen = PBLK_SIZE * (lblk_per_pblk(lbd->params) - 1);
int clen;
int cpu;
struct lz4_state* state;
ret = LZ4_compress_default(dbuf, cbuf, dlen, clen, lbd->lz4_wrkmem);
if (ret <= 0) {
cpu = get_cpu();
state = lblk_get_lz4_state(lbd, cpu);
if (!state) {
put_cpu();
return 0;
}
clen = LZ4_compress_default(lbd->buf,
state->buf,
PBLK_SIZE * lblk_per_pblk(lbd->params),
PBLK_SIZE * (lblk_per_pblk(lbd->params) - 1),
state->wrkmem);
if (clen <= 0) {
put_cpu();
return 0;
}
memcpy(lbd->buf, state->buf, clen);
put_cpu();
return (size_t)ret;
return (size_t)clen;
}
/*
@ -124,15 +169,26 @@ static int
lblk_decompress(struct lbd* lbd, u32 clen)
{
int ret;
void *cbuf = lbd->lz4_cbuf;
void *dbuf = lbd->buf;
int cpu;
struct lz4_state* state;
u32 dlen = PBLK_SIZE * lblk_per_pblk(lbd->params);
ret = LZ4_decompress_safe(cbuf, dbuf, clen, dlen);
if (ret != dlen) {
printk(KERN_ERR "%s: failed, ret=%d (expected %u)\n", __func__, ret, (unsigned int)dlen);
cpu = get_cpu();
state = lblk_get_lz4_state(lbd, cpu);
if (!state) {
put_cpu();
return -1;
}
ret = LZ4_decompress_safe(lbd->buf,
state->buf,
clen,
dlen);
if (ret != dlen) {
put_cpu();
return -1;
}
memcpy(lbd->buf, state->buf, dlen);
put_cpu();
return 0;
}
@ -140,7 +196,8 @@ lblk_decompress(struct lbd* lbd, u32 clen)
static bool
lbd_ctr(struct lbd* lbd,
struct cbd_params* params,
struct lbatviewcache* lvc)
struct lbatviewcache* lvc,
void* percpu)
{
memset(lbd, 0, sizeof(struct lbd));
lbd->lblk = LBLK_NONE;
@ -151,15 +208,7 @@ lbd_ctr(struct lbd* lbd,
lbd->params = params;
lbd->lvc = lvc;
lbd->lv = NULL;
lbd->lz4_wrkmem = kmalloc(LZ4_compressBound(PBLK_SIZE * lblk_per_pblk(lbd->params)), GFP_KERNEL);
if (!lbd->lz4_wrkmem) {
return false;
}
lbd->lz4_cpages = cbd_alloc_pages(lblk_per_pblk(lbd->params));
if (!lbd->lz4_cpages) {
return false;
}
lbd->lz4_cbuf = page_address(lbd->lz4_cpages);
lbd->percpu = percpu;
lbd->pages = cbd_alloc_pages(lblk_per_pblk(lbd->params));
if (!lbd->pages) {
return false;
@ -175,15 +224,12 @@ lbd_dtr(struct lbd* lbd)
if (lbatviewcache_put(lbd->lvc, lbd->lv) != 0) {
printk(KERN_ERR "%s: lbatviewcache_put failed\n", __func__);
}
lbd->lv = NULL;
lbd->buf = NULL;
cbd_free_pages(lbd->pages, lblk_per_pblk(lbd->params));
lbd->pages = NULL;
lbd->buf = NULL;
cbd_free_pages(lbd->lz4_cpages, lblk_per_pblk(lbd->params));
lbd->lz4_cpages = NULL;
lbd->lz4_cbuf = NULL;
kfree(lbd->lz4_wrkmem);
lbd->lz4_wrkmem = NULL;
lbd->percpu = NULL;
lbd->lv = NULL;
lbd->lvc = NULL;
}
static void
@ -223,22 +269,19 @@ lbd_flush(struct lbd* lbd)
if (lblk_is_zeros(lbd->params, lbd)) {
c_len = 0;
elem_len = 0;
p = NULL;
}
else {
c_len = lblk_compress(lbd);
if (c_len > 0) {
size_t c_blkrem = c_len % PBLK_SIZE;
if (c_blkrem) {
memset(lbd->lz4_cbuf + c_len, 0, c_blkrem);
memset(lbd->buf + c_len, 0, c_blkrem);
}
elem_len = c_len;
p = lbd->lz4_cbuf;
}
else {
c_len = PBLK_SIZE * lblk_per_pblk(lbd->params);
elem_len = CBD_UNCOMPRESSED;
p = lbd->buf;
}
}
@ -246,6 +289,7 @@ lbd_flush(struct lbd* lbd)
if (ret) {
goto out;
}
p = lbd->buf;
for (n = 0; n * PBLK_SIZE < c_len; ++n, p += PBLK_SIZE) {
pblk = lbatview_elem_pblk(lbd->lv, lbd->lblk, n);
if (pblk == PBLK_NONE) {
@ -322,7 +366,7 @@ lbd_read(struct lbd* lbd)
is_compressed = false;
c_len = d_len;
}
p = lbd->lz4_cbuf;
p = lbd->buf;
for (n = 0; n * PBLK_SIZE < c_len; ++n, p += PBLK_SIZE) {
pblk = lbatview_elem_pblk(lbd->lv, lbd->lblk, n);
if (pblk == PBLK_NONE) {
@ -343,9 +387,6 @@ lbd_read(struct lbd* lbd)
goto out;
}
}
else {
memcpy(lbd->buf, lbd->lz4_cbuf, d_len);
}
}
lbd->state = CACHE_STATE_CLEAN;
@ -387,6 +428,7 @@ struct lbdcache
{
struct mutex lock;
struct cbd_params* params;
void* percpu;
struct lbatviewcache* lvc;
unsigned int len;
struct lbd** cache;
@ -423,7 +465,7 @@ lbdcache_realloc(struct lbdcache* lc, unsigned int len)
return false;
}
cache[n++] = lbd;
if (!lbd_ctr(lbd, lc->params, lc->lvc)) {
if (!lbd_ctr(lbd, lc->params, lc->lvc, lc->percpu)) {
return false;
}
}
@ -433,11 +475,13 @@ lbdcache_realloc(struct lbdcache* lc, unsigned int len)
bool
lbdcache_ctr(struct lbdcache* lc,
struct cbd_params* params)
struct cbd_params* params,
void* percpu)
{
memset(lc, 0, sizeof(struct lbdcache));
mutex_init(&lc->lock);
lc->params = params;
lc->percpu = percpu;
lc->lvc = kzalloc(lbatviewcache_size(), GFP_KERNEL);
if (!lc->lvc) {
return false;

12
dm-compress/util.c
View File

@ -39,6 +39,12 @@ cbd_alloc_page(void)
return alloc_page(GFP_KERNEL);
}
struct page*
cbd_alloc_page_nowait(void)
{
return alloc_page(GFP_NOWAIT);
}
void
cbd_free_page(struct page* page)
{
@ -51,6 +57,12 @@ cbd_alloc_pages(size_t len)
return alloc_pages(GFP_KERNEL, get_order(len * PAGE_SIZE));
}
struct page*
cbd_alloc_pages_nowait(size_t len)
{
return alloc_pages(GFP_NOWAIT, get_order(len * PAGE_SIZE));
}
void
cbd_free_pages(struct page* pages, size_t len)
{

7
include/linux/dm-compress.h
View File

@ -457,10 +457,14 @@ typedef void (*pblk_endio_t)(struct bio*);
/* Single page allocator */
struct page*
cbd_alloc_page(void);
struct page*
cbd_alloc_page_nowait(void);
void cbd_free_page(struct page* page);
/* Multiple page allocator */
struct page*
cbd_alloc_pages(size_t len);
struct page*
cbd_alloc_pages_nowait(size_t len);
void cbd_free_pages(struct page* pages, size_t len);
/* Vector page allocator */
bool cbd_alloc_pagev(struct page** pagev, size_t len);
@ -544,7 +548,8 @@ void lbd_data_write(struct lbd* lbd, u32 off, u32 len, const u8* buf);
struct lbdcache;
size_t lbdcache_size(void);
bool lbdcache_ctr(struct lbdcache* lc,
struct cbd_params* params);
struct cbd_params* params,
void* percpu);
void lbdcache_dtr(struct lbdcache* lc);
struct lbd*
lbdcache_get(struct lbdcache* lc, u64 lblk);