Use per-cpu lz4 state
This commit is contained in:
parent
290b1873a0
commit
eeafc209a5
|
@ -53,6 +53,7 @@ struct compress
|
|||
struct dm_dev* dev;
|
||||
|
||||
struct cbd_params params;
|
||||
void* percpu;
|
||||
struct lbdcache* lc;
|
||||
|
||||
struct workqueue_struct* io_workq;
|
||||
|
@ -150,6 +151,7 @@ compress_open(struct compress* c, u64 dev_nr_pblks)
|
|||
printk(KERN_INFO " lblk_per_zone=%u\n", (unsigned int)header.params.lblk_per_zone);
|
||||
|
||||
memcpy(&c->params, &header.params, sizeof(header.params));
|
||||
c->percpu = alloc_percpu(void*);
|
||||
|
||||
c->lc = kmalloc(lbdcache_size(), GFP_KERNEL);
|
||||
if (!c->lc) {
|
||||
|
@ -157,7 +159,7 @@ compress_open(struct compress* c, u64 dev_nr_pblks)
|
|||
printk(KERN_ERR "Failed to alloc lbdcache\n");
|
||||
goto out;
|
||||
}
|
||||
if (!lbdcache_ctr(c->lc, &c->params)) {
|
||||
if (!lbdcache_ctr(c->lc, &c->params, c->percpu)) {
|
||||
err = -ENOMEM;
|
||||
printk(KERN_ERR "Failed to init logical block cache\n");
|
||||
goto out;
|
||||
|
@ -404,6 +406,7 @@ compress_dtr(struct dm_target *ti)
|
|||
c = ti->private;
|
||||
lbdcache_dtr(c->lc);
|
||||
kfree(c->lc);
|
||||
free_percpu(c->percpu);
|
||||
if (c->io_workq) {
|
||||
destroy_workqueue(c->io_workq);
|
||||
}
|
||||
|
|
|
@ -39,9 +39,7 @@ struct lbd {
|
|||
struct cbd_params* params;
|
||||
struct lbatviewcache* lvc;
|
||||
struct lbatview* lv;
|
||||
u8* lz4_wrkmem;
|
||||
struct page* lz4_cpages;
|
||||
u8* lz4_cbuf;
|
||||
void* percpu;
|
||||
struct page* pages;
|
||||
u8* buf;
|
||||
};
|
||||
|
@ -93,6 +91,42 @@ lblk_is_zeros(struct cbd_params* params, struct lbd* lbd)
|
|||
#endif
|
||||
}
|
||||
|
||||
struct lz4_state {
|
||||
u8* wrkmem;
|
||||
struct page* pages;
|
||||
u8* buf;
|
||||
};
|
||||
|
||||
static struct lz4_state*
|
||||
lblk_get_lz4_state(struct lbd* lbd, int cpu)
|
||||
{
|
||||
struct lz4_state** statep;
|
||||
struct lz4_state* state;
|
||||
|
||||
statep = per_cpu_ptr(lbd->percpu, cpu);
|
||||
if (*statep) {
|
||||
return *statep;
|
||||
}
|
||||
state = kmalloc(sizeof(struct lz4_state), GFP_NOWAIT);
|
||||
if (!state) {
|
||||
printk(KERN_ERR "%s: failed to alloc state\n", __func__);
|
||||
return NULL;
|
||||
}
|
||||
state->wrkmem = kmalloc(LZ4_compressBound(PBLK_SIZE * lblk_per_pblk(lbd->params)), GFP_NOWAIT);
|
||||
state->pages = cbd_alloc_pages_nowait(lblk_per_pblk(lbd->params));
|
||||
if (!state->wrkmem || !state->pages) {
|
||||
kfree(state->wrkmem);
|
||||
cbd_free_pages(state->pages, lblk_per_pblk(lbd->params));
|
||||
kfree(state);
|
||||
printk(KERN_ERR "%s: failed to alloc buffers\n", __func__);
|
||||
return NULL;
|
||||
}
|
||||
state->buf = page_address(state->pages);
|
||||
*statep = state;
|
||||
|
||||
return state;
|
||||
}
|
||||
|
||||
/*
|
||||
* Compress dc->lblk into dc->lz4_cbuf
|
||||
*
|
||||
|
@ -101,18 +135,29 @@ lblk_is_zeros(struct cbd_params* params, struct lbd* lbd)
|
|||
static size_t
|
||||
lblk_compress(struct lbd* lbd)
|
||||
{
|
||||
int ret;
|
||||
void *dbuf = lbd->buf;
|
||||
u32 dlen = PBLK_SIZE * lblk_per_pblk(lbd->params);
|
||||
void *cbuf = lbd->lz4_cbuf;
|
||||
u32 clen = PBLK_SIZE * (lblk_per_pblk(lbd->params) - 1);
|
||||
int clen;
|
||||
int cpu;
|
||||
struct lz4_state* state;
|
||||
|
||||
ret = LZ4_compress_default(dbuf, cbuf, dlen, clen, lbd->lz4_wrkmem);
|
||||
if (ret <= 0) {
|
||||
cpu = get_cpu();
|
||||
state = lblk_get_lz4_state(lbd, cpu);
|
||||
if (!state) {
|
||||
put_cpu();
|
||||
return 0;
|
||||
}
|
||||
clen = LZ4_compress_default(lbd->buf,
|
||||
state->buf,
|
||||
PBLK_SIZE * lblk_per_pblk(lbd->params),
|
||||
PBLK_SIZE * (lblk_per_pblk(lbd->params) - 1),
|
||||
state->wrkmem);
|
||||
if (clen <= 0) {
|
||||
put_cpu();
|
||||
return 0;
|
||||
}
|
||||
memcpy(lbd->buf, state->buf, clen);
|
||||
put_cpu();
|
||||
|
||||
return (size_t)ret;
|
||||
return (size_t)clen;
|
||||
}
|
||||
|
||||
/*
|
||||
|
@ -124,15 +169,26 @@ static int
|
|||
lblk_decompress(struct lbd* lbd, u32 clen)
|
||||
{
|
||||
int ret;
|
||||
void *cbuf = lbd->lz4_cbuf;
|
||||
void *dbuf = lbd->buf;
|
||||
int cpu;
|
||||
struct lz4_state* state;
|
||||
u32 dlen = PBLK_SIZE * lblk_per_pblk(lbd->params);
|
||||
|
||||
ret = LZ4_decompress_safe(cbuf, dbuf, clen, dlen);
|
||||
if (ret != dlen) {
|
||||
printk(KERN_ERR "%s: failed, ret=%d (expected %u)\n", __func__, ret, (unsigned int)dlen);
|
||||
cpu = get_cpu();
|
||||
state = lblk_get_lz4_state(lbd, cpu);
|
||||
if (!state) {
|
||||
put_cpu();
|
||||
return -1;
|
||||
}
|
||||
ret = LZ4_decompress_safe(lbd->buf,
|
||||
state->buf,
|
||||
clen,
|
||||
dlen);
|
||||
if (ret != dlen) {
|
||||
put_cpu();
|
||||
return -1;
|
||||
}
|
||||
memcpy(lbd->buf, state->buf, dlen);
|
||||
put_cpu();
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
@ -140,7 +196,8 @@ lblk_decompress(struct lbd* lbd, u32 clen)
|
|||
static bool
|
||||
lbd_ctr(struct lbd* lbd,
|
||||
struct cbd_params* params,
|
||||
struct lbatviewcache* lvc)
|
||||
struct lbatviewcache* lvc,
|
||||
void* percpu)
|
||||
{
|
||||
memset(lbd, 0, sizeof(struct lbd));
|
||||
lbd->lblk = LBLK_NONE;
|
||||
|
@ -151,15 +208,7 @@ lbd_ctr(struct lbd* lbd,
|
|||
lbd->params = params;
|
||||
lbd->lvc = lvc;
|
||||
lbd->lv = NULL;
|
||||
lbd->lz4_wrkmem = kmalloc(LZ4_compressBound(PBLK_SIZE * lblk_per_pblk(lbd->params)), GFP_KERNEL);
|
||||
if (!lbd->lz4_wrkmem) {
|
||||
return false;
|
||||
}
|
||||
lbd->lz4_cpages = cbd_alloc_pages(lblk_per_pblk(lbd->params));
|
||||
if (!lbd->lz4_cpages) {
|
||||
return false;
|
||||
}
|
||||
lbd->lz4_cbuf = page_address(lbd->lz4_cpages);
|
||||
lbd->percpu = percpu;
|
||||
lbd->pages = cbd_alloc_pages(lblk_per_pblk(lbd->params));
|
||||
if (!lbd->pages) {
|
||||
return false;
|
||||
|
@ -175,15 +224,12 @@ lbd_dtr(struct lbd* lbd)
|
|||
if (lbatviewcache_put(lbd->lvc, lbd->lv) != 0) {
|
||||
printk(KERN_ERR "%s: lbatviewcache_put failed\n", __func__);
|
||||
}
|
||||
lbd->lv = NULL;
|
||||
lbd->buf = NULL;
|
||||
cbd_free_pages(lbd->pages, lblk_per_pblk(lbd->params));
|
||||
lbd->pages = NULL;
|
||||
lbd->buf = NULL;
|
||||
cbd_free_pages(lbd->lz4_cpages, lblk_per_pblk(lbd->params));
|
||||
lbd->lz4_cpages = NULL;
|
||||
lbd->lz4_cbuf = NULL;
|
||||
kfree(lbd->lz4_wrkmem);
|
||||
lbd->lz4_wrkmem = NULL;
|
||||
lbd->percpu = NULL;
|
||||
lbd->lv = NULL;
|
||||
lbd->lvc = NULL;
|
||||
}
|
||||
|
||||
static void
|
||||
|
@ -223,22 +269,19 @@ lbd_flush(struct lbd* lbd)
|
|||
if (lblk_is_zeros(lbd->params, lbd)) {
|
||||
c_len = 0;
|
||||
elem_len = 0;
|
||||
p = NULL;
|
||||
}
|
||||
else {
|
||||
c_len = lblk_compress(lbd);
|
||||
if (c_len > 0) {
|
||||
size_t c_blkrem = c_len % PBLK_SIZE;
|
||||
if (c_blkrem) {
|
||||
memset(lbd->lz4_cbuf + c_len, 0, c_blkrem);
|
||||
memset(lbd->buf + c_len, 0, c_blkrem);
|
||||
}
|
||||
elem_len = c_len;
|
||||
p = lbd->lz4_cbuf;
|
||||
}
|
||||
else {
|
||||
c_len = PBLK_SIZE * lblk_per_pblk(lbd->params);
|
||||
elem_len = CBD_UNCOMPRESSED;
|
||||
p = lbd->buf;
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -246,6 +289,7 @@ lbd_flush(struct lbd* lbd)
|
|||
if (ret) {
|
||||
goto out;
|
||||
}
|
||||
p = lbd->buf;
|
||||
for (n = 0; n * PBLK_SIZE < c_len; ++n, p += PBLK_SIZE) {
|
||||
pblk = lbatview_elem_pblk(lbd->lv, lbd->lblk, n);
|
||||
if (pblk == PBLK_NONE) {
|
||||
|
@ -322,7 +366,7 @@ lbd_read(struct lbd* lbd)
|
|||
is_compressed = false;
|
||||
c_len = d_len;
|
||||
}
|
||||
p = lbd->lz4_cbuf;
|
||||
p = lbd->buf;
|
||||
for (n = 0; n * PBLK_SIZE < c_len; ++n, p += PBLK_SIZE) {
|
||||
pblk = lbatview_elem_pblk(lbd->lv, lbd->lblk, n);
|
||||
if (pblk == PBLK_NONE) {
|
||||
|
@ -343,9 +387,6 @@ lbd_read(struct lbd* lbd)
|
|||
goto out;
|
||||
}
|
||||
}
|
||||
else {
|
||||
memcpy(lbd->buf, lbd->lz4_cbuf, d_len);
|
||||
}
|
||||
}
|
||||
lbd->state = CACHE_STATE_CLEAN;
|
||||
|
||||
|
@ -387,6 +428,7 @@ struct lbdcache
|
|||
{
|
||||
struct mutex lock;
|
||||
struct cbd_params* params;
|
||||
void* percpu;
|
||||
struct lbatviewcache* lvc;
|
||||
unsigned int len;
|
||||
struct lbd** cache;
|
||||
|
@ -423,7 +465,7 @@ lbdcache_realloc(struct lbdcache* lc, unsigned int len)
|
|||
return false;
|
||||
}
|
||||
cache[n++] = lbd;
|
||||
if (!lbd_ctr(lbd, lc->params, lc->lvc)) {
|
||||
if (!lbd_ctr(lbd, lc->params, lc->lvc, lc->percpu)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
@ -433,11 +475,13 @@ lbdcache_realloc(struct lbdcache* lc, unsigned int len)
|
|||
|
||||
bool
|
||||
lbdcache_ctr(struct lbdcache* lc,
|
||||
struct cbd_params* params)
|
||||
struct cbd_params* params,
|
||||
void* percpu)
|
||||
{
|
||||
memset(lc, 0, sizeof(struct lbdcache));
|
||||
mutex_init(&lc->lock);
|
||||
lc->params = params;
|
||||
lc->percpu = percpu;
|
||||
lc->lvc = kzalloc(lbatviewcache_size(), GFP_KERNEL);
|
||||
if (!lc->lvc) {
|
||||
return false;
|
||||
|
|
|
@ -39,6 +39,12 @@ cbd_alloc_page(void)
|
|||
return alloc_page(GFP_KERNEL);
|
||||
}
|
||||
|
||||
struct page*
|
||||
cbd_alloc_page_nowait(void)
|
||||
{
|
||||
return alloc_page(GFP_NOWAIT);
|
||||
}
|
||||
|
||||
void
|
||||
cbd_free_page(struct page* page)
|
||||
{
|
||||
|
@ -51,6 +57,12 @@ cbd_alloc_pages(size_t len)
|
|||
return alloc_pages(GFP_KERNEL, get_order(len * PAGE_SIZE));
|
||||
}
|
||||
|
||||
struct page*
|
||||
cbd_alloc_pages_nowait(size_t len)
|
||||
{
|
||||
return alloc_pages(GFP_NOWAIT, get_order(len * PAGE_SIZE));
|
||||
}
|
||||
|
||||
void
|
||||
cbd_free_pages(struct page* pages, size_t len)
|
||||
{
|
||||
|
|
|
@ -457,10 +457,14 @@ typedef void (*pblk_endio_t)(struct bio*);
|
|||
/* Single page allocator */
|
||||
struct page*
|
||||
cbd_alloc_page(void);
|
||||
struct page*
|
||||
cbd_alloc_page_nowait(void);
|
||||
void cbd_free_page(struct page* page);
|
||||
/* Multiple page allocator */
|
||||
struct page*
|
||||
cbd_alloc_pages(size_t len);
|
||||
struct page*
|
||||
cbd_alloc_pages_nowait(size_t len);
|
||||
void cbd_free_pages(struct page* pages, size_t len);
|
||||
/* Vector page allocator */
|
||||
bool cbd_alloc_pagev(struct page** pagev, size_t len);
|
||||
|
@ -544,7 +548,8 @@ void lbd_data_write(struct lbd* lbd, u32 off, u32 len, const u8* buf);
|
|||
struct lbdcache;
|
||||
size_t lbdcache_size(void);
|
||||
bool lbdcache_ctr(struct lbdcache* lc,
|
||||
struct cbd_params* params);
|
||||
struct cbd_params* params,
|
||||
void* percpu);
|
||||
void lbdcache_dtr(struct lbdcache* lc);
|
||||
struct lbd*
|
||||
lbdcache_get(struct lbdcache* lc, u64 lblk);
|
||||
|
|
Loading…
Reference in New Issue