Skip to content

Commit

Permalink
Fix public issue google#26: Take memory allocation and reallocation e…
Browse files Browse the repository at this point in the history 
…ntirely out of the

Measure() loop. This gives all algorithms a small speed boost, except Snappy which
already didn't do reallocation (so the measurements were slightly biased in its
favor).

R=csilvers
DELTA=92  (69 added, 9 deleted, 14 changed)


Revision created by MOE tool push_codebase.
MOE_MIGRATION=1151


git-svn-id: https://snappy.googlecode.com/svn/trunk@24 03e5f5b5-db94-4691-08a0-1a8bf15f6143
  • Loading branch information
[email protected] committed Mar 30, 2011
1 parent cc333c1 commit f67bcaa
Showing 1 changed file with 83 additions and 23 deletions.
106 changes: 83 additions & 23 deletions snappy_unittest.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,13 +128,58 @@ const char* names[] = {
"ZLIB", "LZO", "LIBLZF", "QUICKLZ", "FASTLZ", "SNAPPY",
};

// Returns true if we successfully compressed, false otherwise
static size_t MinimumRequiredOutputSpace(size_t input_size,
CompressorType comp) {
switch (comp) {
#ifdef ZLIB_VERSION
case ZLIB:
return ZLib::MinCompressbufSize(input_size);
#endif // ZLIB_VERSION

#ifdef LZO_VERSION
case LZO:
return input_size + input_size/64 + 16 + 3;
#endif // LZO_VERSION

#ifdef LZF_VERSION
case LIBLZF:
return input_size;
#endif // LZF_VERSION

#ifdef QLZ_VERSION_MAJOR
case QUICKLZ:
return input_size + 36000; // 36000 is used for scratch.
#endif // QLZ_VERSION_MAJOR

#ifdef FASTLZ_VERSION
case FASTLZ:
return max(static_cast<int>(ceil(input_size * 1.05)), 66);
#endif // FASTLZ_VERSION

case SNAPPY:
return snappy::MaxCompressedLength(input_size);

default:
LOG(FATAL) << "Unknown compression type number " << comp;
}
}

// Returns true if we successfully compressed, false otherwise.
//
// If compressed_is_preallocated is set, do not resize the compressed buffer.
// This is typically what you want for a benchmark, in order to not spend
// time in the memory allocator. If you do set this flag, however,
// "compressed" must be preinitialized to at least MinCompressbufSize(comp)
// number of bytes, and may contain junk bytes at the end after return.
static bool Compress(const char* input, size_t input_size, CompressorType comp,
string* compressed) {
string* compressed, bool compressed_is_preallocated) {
if (!compressed_is_preallocated) {
compressed->resize(MinimumRequiredOutputSpace(input_size, comp));
}

switch (comp) {
#ifdef ZLIB_VERSION
case ZLIB: {
compressed->resize(ZLib::MinCompressbufSize(input_size));
ZLib zlib;
uLongf destlen = compressed->size();
int ret = zlib.Compress(
Expand All @@ -143,14 +188,15 @@ static bool Compress(const char* input, size_t input_size, CompressorType comp,
reinterpret_cast<const Bytef*>(input),
input_size);
CHECK_EQ(Z_OK, ret);
compressed->resize(destlen);
if (!compressed_is_preallocated) {
compressed->resize(destlen);
}
return true;
}
#endif // ZLIB_VERSION

#ifdef LZO_VERSION
case LZO: {
compressed->resize(input_size + input_size/64 + 16 + 3);
unsigned char* mem = new unsigned char[LZO1X_1_15_MEM_COMPRESS];
lzo_uint destlen;
int ret = lzo1x_1_15_compress(
Expand All @@ -161,57 +207,60 @@ static bool Compress(const char* input, size_t input_size, CompressorType comp,
mem);
CHECK_EQ(LZO_E_OK, ret);
delete[] mem;
compressed->resize(destlen);
if (!compressed_is_preallocated) {
compressed->resize(destlen);
}
break;
}
#endif // LZO_VERSION

#ifdef LZF_VERSION
case LIBLZF: {
compressed->resize(input_size - 1);
int destlen = lzf_compress(input,
input_size,
string_as_array(compressed),
input_size - 1);
input_size);
if (destlen == 0) {
// lzf *can* cause lots of blowup when compressing, so they
// recommend to limit outsize to insize, and just not compress
// if it's bigger. Ideally, we'd just swap input and output.
compressed->assign(input, input_size);
destlen = input_size;
}
compressed->resize(destlen);
if (!compressed_is_preallocated) {
compressed->resize(destlen);
}
break;
}
#endif // LZF_VERSION

#ifdef QLZ_VERSION_MAJOR
case QUICKLZ: {
compressed->resize(input_size + 36000); // 36000 is used for scratch
qlz_state_compress *state_compress = new qlz_state_compress;
int destlen = qlz_compress(input,
string_as_array(compressed),
input_size,
state_compress);
delete state_compress;
CHECK_NE(0, destlen);
compressed->resize(destlen);
if (!compressed_is_preallocated) {
compressed->resize(destlen);
}
break;
}
#endif // QLZ_VERSION_MAJOR

#ifdef FASTLZ_VERSION
case FASTLZ: {
int compressed_size = max(static_cast<int>(ceil(input_size * 1.05)),
66);
compressed->resize(compressed_size);
// Use level 1 compression since we mostly care about speed.
int destlen = fastlz_compress_level(
1,
input,
input_size,
string_as_array(compressed));
compressed->resize(destlen);
if (!compressed_is_preallocated) {
compressed->resize(destlen);
}
CHECK_NE(destlen, 0);
break;
}
Expand All @@ -223,7 +272,9 @@ static bool Compress(const char* input, size_t input_size, CompressorType comp,
string_as_array(compressed),
&destlen);
CHECK_LE(destlen, snappy::MaxCompressedLength(input_size));
compressed->resize(destlen);
if (!compressed_is_preallocated) {
compressed->resize(destlen);
}
break;
}

Expand Down Expand Up @@ -351,13 +402,12 @@ static void Measure(const char* data,
input[b] = data+input_start;
input_length[b] = input_limit-input_start;

// Pre-grow the output buffer so we don't measure stupid string
// append time
compressed[b].resize(block_size * 2);
// Pre-grow the output buffer so we don't measure string append time.
compressed[b].resize(MinimumRequiredOutputSpace(block_size, comp));
}

// First, try one trial compression to make sure the code is compiled in
if (!Compress(input[0], input_length[0], comp, &compressed[0])) {
if (!Compress(input[0], input_length[0], comp, &compressed[0], true)) {
LOG(WARNING) << "Skipping " << names[comp] << ": "
<< "library not compiled in";
return;
Expand All @@ -366,12 +416,23 @@ static void Measure(const char* data,
for (int run = 0; run < kRuns; run++) {
CycleTimer ctimer, utimer;

for (int b = 0; b < num_blocks; b++) {
// Pre-grow the output buffer so we don't measure string append time.
compressed[b].resize(MinimumRequiredOutputSpace(block_size, comp));
}

ctimer.Start();
for (int b = 0; b < num_blocks; b++)
for (int i = 0; i < repeats; i++)
Compress(input[b], input_length[b], comp, &compressed[b]);
Compress(input[b], input_length[b], comp, &compressed[b], true);
ctimer.Stop();

// Compress once more, with resizing, so we don't leave junk
// at the end that will confuse the decompressor.
for (int b = 0; b < num_blocks; b++) {
Compress(input[b], input_length[b], comp, &compressed[b], false);
}

for (int b = 0; b < num_blocks; b++) {
output[b].resize(input_length[b]);
}
Expand Down Expand Up @@ -911,8 +972,7 @@ static void CompressFile(const char* fname) {
File::ReadFileToStringOrDie(fname, &fullinput);

string compressed;
compressed.resize(fullinput.size() * 2);
Compress(fullinput.data(), fullinput.size(), SNAPPY, &compressed);
Compress(fullinput.data(), fullinput.size(), SNAPPY, &compressed, false);

File::WriteStringToFileOrDie(compressed,
string(fname).append(".comp").c_str());
Expand Down

0 comments on commit f67bcaa

Please sign in to comment.