Implement cbd tune plus some cleanup and rearranging

Still need to implement cbd tune --repack.
This commit is contained in:
Tom Marshall 2019-11-18 00:29:25 +01:00
parent 8b08b0c2c1
commit 2fab692cdf
8 changed files with 261 additions and 165 deletions

View File

@ -81,6 +81,7 @@ LIB_SRCS := \
open.c \
resize.c \
stats.c \
tune.c \
util.c
BIN_SRCS := \
cbd.c

142
cbd/cbd.c
View File

@ -154,12 +154,12 @@ usage(void)
" format [opts] <device> Create (format) a compressed device\n"
" -P --pbat-size Physical block allocation table size [1]\n"
" -S --pysical-size Physical size [device size]\n"
" -Z --compress-level Compression level [1]\n"
" -c --compress-factor Compression factor [2.0]\n"
" -l --logical-blksize Logical block size\n"
" -p --physical-blksize Physical block size\n"
" -s --size Logical size\n"
" -z --compress-alg Compression algorithm [lz4]\n"
" -Z --compress-level Compression level [1]\n"
" --detect-zeros Detect and free zero blocks at runtime\n"
" --profile Set -p -l -z -Z automatically\n"
" --full-init Fully init device (no lazy init)\n"
@ -173,18 +173,28 @@ usage(void)
" archive: 4k pblk, 256k lblk, zlib level 9\n"
"\n"
" open [opts] <device> <name> Open an existing compressed device\n"
" -c --cache-pages Set cache pages\n"
" -s --sync Open in synchronous mode\n"
" -c --cache-pages Set cache pages\n"
" -s --sync Open in synchronous mode\n"
"\n"
" create [opts] <device> <name> Alias for open\n"
"\n"
" close [opts] <name> Close an opened compressed device\n"
"\n"
" check [opts] <device> Check and repair a compressed device\n"
" -f --force Force check even if device seems clean\n"
" -F --full-check Do a full check, including verifying data\n"
" -F --full-check Do a full check, including verifying data\n"
" -n --assume-no Assume \"no\" no all questions\n"
" -y --assume-yes Assume \"yes\" to all questions\n"
"\n"
" resize [opts] <device> Resize an existing compressed device\n"
" -s --size New logical size [use all]\n"
" -s --size New logical size [use all]\n"
"\n"
" stats [opts] <name> Show device statistics\n"
"\n"
" tune [opts] <name> Change compression and/or repack\n"
" -r --repack Repack data blocks\n"
" -Z --compress-level Compression level\n"
" --detect-zeros Detect and free zeroed blocks\n"
"\n");
exit(1);
}
@ -192,16 +202,16 @@ usage(void)
static int
do_format(int argc, char** argv)
{
static const char short_opts[] = "P:S:c:l:p:s:z:Z:";
static const char short_opts[] = "P:S:Z:c:l:p:s:z:";
static const struct option long_opts[] = {
{ "pbat-size", required_argument, NULL, 'P' },
{ "physical-size", required_argument, NULL, 'S' },
{ "compress-level", required_argument, NULL, 'Z' },
{ "compress-factor", required_argument, NULL, 'c' },
{ "logical-blksize", required_argument, NULL, 'l' },
{ "physical-blksize", required_argument, NULL, 'p' },
{ "size", required_argument, NULL, 's' },
{ "compress-alg", required_argument, NULL, 'z' },
{ "compress-level", required_argument, NULL, 'Z' },
{ "detect-zeros", optional_argument, NULL, 0x1 },
{ "profile", required_argument, NULL, 0x2 },
{ "full-init", no_argument, NULL, 0x3 },
@ -242,6 +252,15 @@ do_format(int argc, char** argv)
}
psize = optval;
break;
case 'Z':
if (!parse_numeric_arg(optarg, &optval)) {
error("Failed to parse \"%s\"\n", optarg);
}
if (optval < 1 || optval > 9) {
error("Compression level \"%s\" out of bounds\n", optarg);
}
level = optval;
break;
case 'c':
error("Implement me!\n");
break;
@ -281,15 +300,6 @@ do_format(int argc, char** argv)
error("Invalid compression algorithm \"%s\"\n", optarg);
}
break;
case 'Z':
if (!parse_numeric_arg(optarg, &optval)) {
error("Failed to parse \"%s\"\n", optarg);
}
if (optval < 1 || optval > 9) {
error("Compression level \"%s\" out of bounds\n", optarg);
}
level = optval;
break;
case 0x1:
optval = 1;
if (optarg) {
@ -401,25 +411,6 @@ do_close(int argc, char** argv)
return 0;
}
static int
do_stats(int argc, char** argv)
{
const char* name;
struct cbd_stats stats;
if (argc != 2) {
usage();
}
name = argv[1];
cbd_stats(name, &stats);
printf("Stats:\n");
printf("pblk used: %lu\n", (unsigned long)stats.pblk_used);
printf("lblk used: %lu\n", (unsigned long)stats.lblk_used);
return 0;
}
static int
do_check(int argc, char** argv)
{
@ -515,6 +506,84 @@ do_resize(int argc, char** argv)
return 0;
}
static int
do_stats(int argc, char** argv)
{
const char* name;
struct cbd_stats stats;
if (argc != 2) {
usage();
}
name = argv[1];
cbd_stats(name, &stats);
printf("Stats:\n");
printf("pblk used: %lu\n", (unsigned long)stats.pblk_used);
printf("lblk used: %lu\n", (unsigned long)stats.lblk_used);
return 0;
}
static int
do_tune(int argc, char** argv)
{
static const char short_opts[] = "rzZ:";
static const struct option long_opts[] = {
{ "repack", no_argument, NULL, 'r' },
{ "compress-level", required_argument, NULL, 'Z' },
{ "detect-zeros", optional_argument, NULL, 0x1 },
{ NULL, no_argument, NULL, 0 }
};
char opt;
uint64_t optval;
uint level = 0;
bool repack = false;
tristate_t zero_detect = t_none;
char dev[PATH_MAX];
while ((opt = getopt_long(argc, argv, short_opts, long_opts, NULL)) != -1) {
switch (opt) {
case 'r':
repack = true;
break;
case 'Z':
if (!parse_numeric_arg(optarg, &optval)) {
error("Failed to parse \"%s\"\n", optarg);
}
if (optval < 1 || optval > 9) {
error("Compression level \"%s\" out of bounds\n", optarg);
}
level = optval;
break;
case 0x1:
optval = 1;
if (optarg) {
if (!parse_boolean_arg(optarg, &optval)) {
error("Failed to parse \"%s\"\n", optarg);
}
}
zero_detect = (optval ? t_true : t_false);
break;
default:
usage();
}
}
if (argc - optind != 1) {
usage();
}
strcpy(dev, argv[optind]);
if (dev[0] != '/') {
sprintf(dev, "/dev/mapper/%s", argv[optind]);
}
++optind;
cbd_tune(dev, zero_detect, level, repack);
return 0;
}
struct cmd_dispatch
{
const char* cmd;
@ -527,9 +596,10 @@ static struct cmd_dispatch dispatch[] =
{ "open", do_open },
{ "create", do_open },
{ "close", do_close },
{ "stats", do_stats },
{ "check", do_check },
{ "resize", do_resize },
{ "stats", do_stats },
{ "tune", do_tune },
};
int

View File

@ -43,4 +43,9 @@ int verbose(uint level, const char* fmt, ...);
bool ask_user_bool(tristate_t auto_response, const char* fmt, ...);
void pblk_read(int fd, u32 pblk_size, u64 pblk, u32 count, u8* data);
void pblk_write(int fd, u32 pblk_size, u64 pblk, u32 count, const u8* data);
void cbd_check_header(const struct cbd_header* header);
#endif

View File

@ -65,25 +65,26 @@ typedef enum {
t_none
} tristate_t;
int cbd_format(const char* dev,
uint16_t flags,
enum cbd_alg alg, uint level,
uint8_t pshift, uint8_t lshift,
uint8_t pbatshift,
uint64_t psize, uint64_t lsize,
bool full_init);
int cbd_open(const char* dev,
const char* name,
uint64_t cache_pages, bool sync);
int cbd_close(const char* name);
int cbd_stats(const char* dev, struct cbd_stats* stats);
int cbd_check(const char* dev,
bool force,
tristate_t auto_response,
bool full_check);
int cbd_resize(const char* dev,
uint64_t lsize);
int cbd_format(const char* dev,
uint16_t flags,
enum cbd_alg alg, uint level,
uint8_t pshift, uint8_t lshift,
uint8_t pbatshift,
uint64_t psize, uint64_t lsize,
bool full_init);
int cbd_open(const char* dev,
const char* name,
uint64_t cache_pages, bool sync);
int cbd_close(const char* name);
int cbd_stats(const char* dev,
struct cbd_stats* stats);
int cbd_check(const char* dev,
bool force,
tristate_t auto_response,
bool full_check);
int cbd_resize(const char* dev,
uint64_t lsize);
int cbd_tune(const char* dev,
tristate_t zero_detect, uint level, bool repack);
#endif

View File

@ -21,52 +21,6 @@ struct check_state
z_stream zlib_dstream;
};
static void
pblk_read(int fd, u32 pblk_size, u64 pblk, u32 count, u8* data)
{
off_t pos;
size_t remain;
ssize_t ret;
pos = lseek(fd, pblk * pblk_size, SEEK_SET);
if (pos == (off_t)-1) {
error("Failed to seek\n");
}
remain = count * pblk_size;
while (remain) {
ret = read(fd, data, remain);
if (ret <= 0) {
error("Failed to read\n");
}
remain -= ret;
data += ret;
}
}
static void
pblk_write(int fd, u32 pblk_size, u64 pblk, u32 count, const u8* data)
{
off_t pos;
size_t remain;
ssize_t ret;
pos = lseek(fd, pblk * pblk_size, SEEK_SET);
if (pos == (off_t)-1) {
error("Failed to seek\n");
}
remain = count * pblk_size;
while (remain) {
ret = write(fd, data, remain);
if (ret <= 0) {
error("Failed to write\n");
}
remain -= ret;
data += ret;
}
}
static void
pbat_read(int fd, const struct cbd_params* params, u32 zone, u8* data)
{
@ -91,37 +45,6 @@ lbat_write(int fd, const struct cbd_params* params, u32 zone, const u8* data)
pblk_write(fd, pblk_size(params), lbat_off(params, zone), lbat_len(params), data);
}
static void
check_header(struct cbd_header* header)
{
enum cbd_alg alg = cbd_compression_alg_get(&header->params);
u8 level = cbd_compression_level_get(&header->params);
if (memcmp(header->magic, CBD_MAGIC, sizeof(header->magic))) {
error("Bad magic\n");
}
if (header->version_major != CBD_VERSION_MAJOR) {
error("Bad major version\n");
}
if (header->version_minor != CBD_VERSION_MINOR) {
error("Bad major version\n");
}
if (alg == CBD_ALG_NONE || alg >= CBD_ALG_MAX) {
error("Bad algorithm\n");
}
if (level < 1 || level > 9) {
error("Bad compression\n");
}
if (header->params.lblk_shift < LBLK_SHIFT_MIN ||
header->params.lblk_shift >= LBLK_SHIFT_MAX) {
error("Bad logical block shift\n");
}
if (header->params.init_zones > header->params.nr_zones) {
verbose(1, "init_zones incorrect, fixing\n");
header->params.init_zones = header->params.nr_zones;
}
}
static bool
check_decompress_lz4(struct check_state* state, const struct cbd_params* params, u8* buf, u32 clen)
{
@ -407,7 +330,11 @@ cbd_check(const char* dev,
pblk_read(state.fd, SECTOR_SIZE, 0, 1, buf);
cbd_header_get(buf, &header);
verbose(1, "Checking header\n");
check_header(&header);
cbd_check_header(&header);
if (header.params.init_zones > header.params.nr_zones) {
verbose(1, "init_zones incorrect, fixing\n");
header.params.init_zones = header.params.nr_zones;
}
if (!force && !(header.params.flags & CBD_FLAG_DIRTY)) {
printf("%s: clean\n", dev);
close(state.fd);

View File

@ -2,29 +2,6 @@
#include <cbdutil.h>
static void
pblk_write(int fd, u32 pblk_size, u64 pblk, u32 count, const u8* data)
{
off_t pos;
size_t remain;
ssize_t ret;
pos = lseek(fd, pblk * pblk_size, SEEK_SET);
if (pos == (off_t)-1) {
error("Failed to seek\n");
}
remain = count * pblk_size;
while (remain) {
ret = write(fd, data, remain);
if (ret <= 0) {
error("Failed to write\n");
}
remain -= ret;
data += ret;
}
}
#define FORMAT_FLAGS_MASK \
(CBD_FLAG_DETECT_ZEROS)
@ -37,15 +14,15 @@ cbd_format(const char* dev,
uint64_t psize, uint64_t lsize,
bool full_init)
{
int devfd;
int fd;
uint pblk_size;
uint lblk_size;
uint32_t est_zone_len;
struct cbd_header header;
uint8_t header_buf[PAGE_SIZE];
devfd = open(dev, O_RDWR);
if (devfd < 0) {
fd = open(dev, O_RDWR);
if (fd < 0) {
error("Cannot open device\n");
}
@ -80,7 +57,7 @@ cbd_format(const char* dev,
}
if (!psize) {
off_t pos;
pos = lseek(devfd, 0, SEEK_END);
pos = lseek(fd, 0, SEEK_END);
if (pos == (off_t)-1) {
error("Cannot seek device\n");
}
@ -146,7 +123,7 @@ cbd_format(const char* dev,
memset(header_buf, 0, sizeof(header_buf));
cbd_header_put(header_buf, &header);
pblk_write(devfd, pblk_size, 0, 1, header_buf);
pblk_write(fd, pblk_size, 0, 1, header_buf);
if (full_init) {
uint32_t nr_pblk = zone_metadata_len(&header.params);
@ -161,20 +138,20 @@ cbd_format(const char* dev,
next_write = time(NULL) + 5;
while (header.params.init_zones < header.params.nr_zones) {
pblk = zone_off(&header.params, header.params.init_zones);
pblk_write(devfd, pblk_size, pblk, nr_pblk, data_buf);
pblk_write(fd, pblk_size, pblk, nr_pblk, data_buf);
++header.params.init_zones;
now = time(NULL);
if (now >= next_write) {
printf("Initialized %u/%u zones\n",
header.params.init_zones, header.params.nr_zones);
cbd_header_put(header_buf, &header);
pblk_write(devfd, pblk_size, 0, 1, header_buf);
pblk_write(fd, pblk_size, 0, 1, header_buf);
next_write = now + 5;
}
}
free(data_buf);
cbd_header_put(header_buf, &header);
pblk_write(devfd, pblk_size, 0, 1, header_buf);
pblk_write(fd, pblk_size, 0, 1, header_buf);
}
return 0;

42
libcbd/tune.c Normal file
View File

@ -0,0 +1,42 @@
#include <libcbd.h>
#include <libdevmapper.h>
#include <cbdutil.h>
int
cbd_tune(const char* dev,
tristate_t zero_detect, uint level, bool repack)
{
int fd;
uint8_t header_buf[SECTOR_SIZE];
struct cbd_header header;
fd = open(dev, O_RDWR);
if (fd < 0) {
error("Cannot open device");
}
pblk_read(fd, SECTOR_SIZE, 0, 1, header_buf);
cbd_header_get(header_buf, &header);
cbd_check_header(&header);
if (zero_detect != t_none) {
if (zero_detect == t_true) {
header.params.flags |= CBD_FLAG_DETECT_ZEROS;
}
else {
header.params.flags &= ~CBD_FLAG_DETECT_ZEROS;
}
}
if (level) {
cbd_compression_level_put(&header.params, level);
}
cbd_header_put(header_buf, &header);
pblk_write(fd, SECTOR_SIZE, 0, 1, header_buf);
if (repack) {
error("repack not supported yet\n");
}
return 0;
}

View File

@ -72,3 +72,76 @@ again:
return answer;
}
void
pblk_read(int fd, u32 pblk_size, u64 pblk, u32 count, u8* data)
{
off_t pos;
size_t remain;
ssize_t ret;
pos = lseek(fd, pblk * pblk_size, SEEK_SET);
if (pos == (off_t)-1) {
error("Failed to seek\n");
}
remain = count * pblk_size;
while (remain) {
ret = read(fd, data, remain);
if (ret <= 0) {
error("Failed to read\n");
}
remain -= ret;
data += ret;
}
}
void
pblk_write(int fd, u32 pblk_size, u64 pblk, u32 count, const u8* data)
{
off_t pos;
size_t remain;
ssize_t ret;
pos = lseek(fd, pblk * pblk_size, SEEK_SET);
if (pos == (off_t)-1) {
error("Failed to seek\n");
}
remain = count * pblk_size;
while (remain) {
ret = write(fd, data, remain);
if (ret <= 0) {
error("Failed to write\n");
}
remain -= ret;
data += ret;
}
}
void
cbd_check_header(const struct cbd_header* header)
{
enum cbd_alg alg = cbd_compression_alg_get(&header->params);
u8 level = cbd_compression_level_get(&header->params);
if (memcmp(header->magic, CBD_MAGIC, sizeof(header->magic))) {
error("Bad magic\n");
}
if (header->version_major != CBD_VERSION_MAJOR) {
error("Bad major version\n");
}
if (header->version_minor != CBD_VERSION_MINOR) {
error("Bad major version\n");
}
if (alg == CBD_ALG_NONE || alg >= CBD_ALG_MAX) {
error("Bad algorithm\n");
}
if (level < 1 || level > 9) {
error("Bad compression\n");
}
if (header->params.lblk_shift < LBLK_SHIFT_MIN ||
header->params.lblk_shift >= LBLK_SHIFT_MAX) {
error("Bad logical block shift\n");
}
}