From 806be49c750347eb78f4d94bb21ad37aa9121f93 Mon Sep 17 00:00:00 2001 From: Lode Vandevenne Date: Wed, 13 Mar 2013 00:26:58 +0100 Subject: More options for Zopfli binary --- deflate.c | 6 ++++++ gzip_container.c | 4 ++-- lz77.c | 48 +++++++++++++++++++++++++++++++----------------- squeeze.c | 3 +++ util.c | 1 + zlib_container.c | 4 ++-- zopfli.h | 22 ++++++++++++++++++++-- zopfli_bin.c | 55 +++++++++++++++++++++++++++---------------------------- zopfli_lib.c | 18 ++++++++++++++++-- 9 files changed, 108 insertions(+), 53 deletions(-) diff --git a/deflate.c b/deflate.c index 1ce25bb..713c6ae 100644 --- a/deflate.c +++ b/deflate.c @@ -689,4 +689,10 @@ void ZopfliDeflate(const ZopfliOptions* options, int btype, int final, i += size; } #endif + if (options->verbose) { + fprintf(stderr, + "Original Size: %d, Deflate: %d, Compression: %f%% Removed\n", + (int)insize, (int)*outsize, + 100.0 * (double)(insize - *outsize) / (double)insize); + } } diff --git a/gzip_container.c b/gzip_container.c index b68667f..8a062f2 100644 --- a/gzip_container.c +++ b/gzip_container.c @@ -110,8 +110,8 @@ void ZopfliGzipCompress(const ZopfliOptions* options, if (options->verbose) { fprintf(stderr, - "Original Size: %d, Compressed: %d, Compression: %f%% Removed\n", + "Original Size: %d, Gzip: %d, Compression: %f%% Removed\n", (int)insize, (int)*outsize, - 100.0f * (float)(insize - *outsize) / (float)insize); + 100.0 * (double)(insize - *outsize) / (double)insize); } } diff --git a/lz77.c b/lz77.c index a9dc4d3..26186b4 100644 --- a/lz77.c +++ b/lz77.c @@ -64,16 +64,30 @@ void ZopfliStoreLitLenDist(unsigned short length, unsigned short dist, } /* -Gets the value of the length given the distance. Typically, the value of the -length is the length, but if the distance is very long, decrease the value of -the length a bit to make up for the fact that long distances use large amounts -of extra bits. +Gets a score of the length given the distance. Typically, the score of the +length is the length itself, but if the distance is very long, decrease the +score of the length a bit to make up for the fact that long distances use large +amounts of extra bits. + +This is not an accurate score, it is a heuristic only for the greedy LZ77 +implementation. More accurate cost models are employed later. Making this +heuristic more accurate may hurt rather than improve compression. + +The two direct uses of this heuristic are: +-avoid using a length of 3 in combination with a long distance. This only has + an effect if length == 3. +-make a slightly better choice between the two options of the lazy matching. + +Indirectly, this affects: +-the block split points if the default of block splitting first is used, in a + rather unpredictable way +-the first zopfli run, so it affects the chance of the first run being closer + to the optimal output */ -static int GetLengthValue(int length, int distance) { +static int GetLengthScore(int length, int distance) { /* - At distance > 1024, using length 3 is no longer good, due to the large amount - of extra bits for the distance code. distance > 1024 uses 9+ extra bits, and - this seems to be the sweet spot. + At 1024, the distance uses 9+ extra bits and this seems to be the sweet spot + on tested files. */ return distance > 1024 ? length - 1 : length; } @@ -355,7 +369,7 @@ void ZopfliLZ77Greedy(ZopfliBlockState* s, const unsigned char* in, size_t i = 0, j; unsigned short leng; unsigned short dist; - int lengvalue; + int lengthscore; size_t windowstart = instart > ZOPFLI_WINDOW_SIZE ? instart - ZOPFLI_WINDOW_SIZE : 0; unsigned short dummysublen[259]; @@ -367,7 +381,7 @@ void ZopfliLZ77Greedy(ZopfliBlockState* s, const unsigned char* in, /* Lazy matching. */ unsigned prev_length = 0; unsigned prev_match = 0; - int prevlengvalue; + int prevlengthscore; int match_available = 0; #endif @@ -384,16 +398,16 @@ void ZopfliLZ77Greedy(ZopfliBlockState* s, const unsigned char* in, ZopfliFindLongestMatch(s, h, in, i, inend, ZOPFLI_MAX_MATCH, dummysublen, &dist, &leng); - lengvalue = GetLengthValue(leng, dist); + lengthscore = GetLengthScore(leng, dist); #ifdef ZOPFLI_LAZY_MATCHING /* Lazy matching. */ - prevlengvalue = GetLengthValue(prev_length, prev_match); + prevlengthscore = GetLengthScore(prev_length, prev_match); if (match_available) { match_available = 0; - if (lengvalue > prevlengvalue + 1) { + if (lengthscore > prevlengthscore + 1) { ZopfliStoreLitLenDist(in[i - 1], 0, store); - if (lengvalue >= ZOPFLI_MIN_MATCH && lengvalue < ZOPFLI_MAX_MATCH) { + if (lengthscore >= ZOPFLI_MIN_MATCH && leng < ZOPFLI_MAX_MATCH) { match_available = 1; prev_length = leng; prev_match = dist; @@ -403,7 +417,7 @@ void ZopfliLZ77Greedy(ZopfliBlockState* s, const unsigned char* in, /* Add previous to output. */ leng = prev_length; dist = prev_match; - lengvalue = prevlengvalue; + lengthscore = prevlengthscore; /* Add to output. */ ZopfliVerifyLenDist(in, inend, i - 1, dist, leng); ZopfliStoreLitLenDist(leng, dist, store); @@ -415,7 +429,7 @@ void ZopfliLZ77Greedy(ZopfliBlockState* s, const unsigned char* in, continue; } } - else if (lengvalue >= ZOPFLI_MIN_MATCH && leng < ZOPFLI_MAX_MATCH) { + else if (lengthscore >= ZOPFLI_MIN_MATCH && leng < ZOPFLI_MAX_MATCH) { match_available = 1; prev_length = leng; prev_match = dist; @@ -425,7 +439,7 @@ void ZopfliLZ77Greedy(ZopfliBlockState* s, const unsigned char* in, #endif /* Add to output. */ - if (lengvalue >= ZOPFLI_MIN_MATCH) { + if (lengthscore >= ZOPFLI_MIN_MATCH) { ZopfliVerifyLenDist(in, inend, i, dist, leng); ZopfliStoreLitLenDist(leng, dist, store); } else { diff --git a/squeeze.c b/squeeze.c index 7558b34..09e7e2e 100644 --- a/squeeze.c +++ b/squeeze.c @@ -486,6 +486,9 @@ void ZopfliLZ77Optimal(ZopfliBlockState *s, ¤tstore); cost = ZopfliCalculateBlockSize(currentstore.litlens, currentstore.dists, 0, currentstore.size, 2); + if (s->options->verbose_more || (s->options->verbose && cost < bestcost)) { + fprintf(stderr, "Iteration %d: %d bit\n", i, (int) cost); + } if (cost < bestcost) { /* Copy to the output store. */ ZopfliCopyLZ77Store(¤tstore, store); diff --git a/util.c b/util.c index b7ee2a0..d207145 100644 --- a/util.c +++ b/util.c @@ -205,6 +205,7 @@ int ZopfliGetLengthSymbol(int l) { void ZopfliInitOptions(ZopfliOptions* options) { options->verbose = 0; + options->verbose_more = 0; options->numiterations = 15; options->blocksplitting = 1; options->blocksplittinglast = 0; diff --git a/zlib_container.c b/zlib_container.c index 9a21231..5b7d0aa 100644 --- a/zlib_container.c +++ b/zlib_container.c @@ -72,8 +72,8 @@ void ZopfliZlibCompress(const ZopfliOptions* options, if (options->verbose) { fprintf(stderr, - "Original Size: %d, Compressed: %d, Compression: %f%% Removed\n", + "Original Size: %d, Zlib: %d, Compression: %f%% Removed\n", (int)insize, (int)*outsize, - 100.0f * (float)(insize - *outsize) / (float)insize); + 100.0 * (double)(insize - *outsize) / (double)insize); } } diff --git a/zopfli.h b/zopfli.h index f520b1e..1388501 100644 --- a/zopfli.h +++ b/zopfli.h @@ -1,8 +1,23 @@ /* -Copyright 2013 Google Inc. All Rights Reserved. -Author: lode@google.com (Lode Vandevenne) +Copyright 2011 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Author: lode.vandevenne@gmail.com (Lode Vandevenne) +Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) */ + #ifndef UTIL_COMPRESSION_ZOPFLI_INTERNAL_ZOPFLI_H_ #define UTIL_COMPRESSION_ZOPFLI_INTERNAL_ZOPFLI_H_ @@ -15,6 +30,9 @@ typedef struct ZopfliOptions { /* Whether to print output */ int verbose; + /* Whether to print more detailed output */ + int verbose_more; + /* Maximum amount of times to rerun forward and backward pass to optimize LZ77 compression cost. Good values: 10, 15 for small files, 5 for files over diff --git a/zopfli_bin.c b/zopfli_bin.c index c649cf6..8a147ef 100644 --- a/zopfli_bin.c +++ b/zopfli_bin.c @@ -135,44 +135,43 @@ int main(int argc, char* argv[]) { ZopfliInitOptions(&options); for (i = 1; i < argc; i++) { - if (StringsEqual(argv[i], "-v")) options.verbose = 1; - else if (StringsEqual(argv[i], "-c")) output_to_stdout = 1; - else if (StringsEqual(argv[i], "--deflate")) { + const char* arg = argv[i]; + if (StringsEqual(arg, "-v")) options.verbose = 1; + else if (StringsEqual(arg, "-c")) output_to_stdout = 1; + else if (StringsEqual(arg, "--deflate")) { output_type = ZOPFLI_FORMAT_DEFLATE; } - else if (StringsEqual(argv[i], "--zlib")) output_type = ZOPFLI_FORMAT_ZLIB; - else if (StringsEqual(argv[i], "--gzip")) output_type = ZOPFLI_FORMAT_GZIP; - else if (StringsEqual(argv[i], "--i5")) options.numiterations = 5; - else if (StringsEqual(argv[i], "--i10")) options.numiterations = 10; - else if (StringsEqual(argv[i], "--i15")) options.numiterations = 15; - else if (StringsEqual(argv[i], "--i25")) options.numiterations = 25; - else if (StringsEqual(argv[i], "--i50")) options.numiterations = 50; - else if (StringsEqual(argv[i], "--i100")) options.numiterations = 100; - else if (StringsEqual(argv[i], "--i250")) options.numiterations = 250; - else if (StringsEqual(argv[i], "--i500")) options.numiterations = 500; - else if (StringsEqual(argv[i], "--i1000")) options.numiterations = 1000; - else if (StringsEqual(argv[i], "-h")) { - fprintf(stderr, "Usage: zopfli [OPTION]... FILE\n" + else if (StringsEqual(arg, "--zlib")) output_type = ZOPFLI_FORMAT_ZLIB; + else if (StringsEqual(arg, "--gzip")) output_type = ZOPFLI_FORMAT_GZIP; + else if (StringsEqual(arg, "--splitlast")) options.blocksplittinglast = 1; + else if (arg[0] == '-' && arg[1] == '-' && arg[2] == 'i' + && arg[3] >= '0' && arg[3] <= '9') { + options.numiterations = atoi(arg + 3); + } + else if (StringsEqual(arg, "-h")) { + fprintf(stderr, + "Usage: zopfli [OPTION]... FILE\n" " -h gives this help\n" " -c write the result on standard output, instead of disk" " filename + '.gz'\n" " -v verbose mode\n" - " --gzip output to gzip format (default)\n" - " --deflate output to deflate format instead of gzip\n" - " --zlib output to zlib format instead of gzip\n"); - fprintf(stderr, " --i5 less compression, but faster\n" - " --i10 less compression, but faster\n" - " --i15 default compression, 15 iterations\n" - " --i25 more compression, but slower\n" - " --i50 more compression, but slower\n" - " --i100 more compression, but slower\n" - " --i250 more compression, but slower\n" - " --i500 more compression, but slower\n" - " --i1000 more compression, but slower\n"); + " --i# perform # iterations (default 15). More gives" + " more compression but is slower." + " Examples: --i10, --i50, --i1000\n"); + fprintf(stderr, + " --gzip output to gzip format (default)\n" + " --zlib output to zlib format instead of gzip\n" + " --deflate output to deflate format instead of gzip\n" + " --splitlast do block splitting last instead of first\n"); return 0; } } + if (options.numiterations < 1) { + fprintf(stderr, "Error: must have 1 or more iterations"); + return 0; + } + for (i = 1; i < argc; i++) { if (argv[i][0] != '-') { char* outfilename; diff --git a/zopfli_lib.c b/zopfli_lib.c index 561d017..b2280e8 100644 --- a/zopfli_lib.c +++ b/zopfli_lib.c @@ -1,6 +1,20 @@ /* -Copyright 2013 Google Inc. All Rights Reserved. -Author: lode@google.com (Lode Vandevenne) +Copyright 2011 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Author: lode.vandevenne@gmail.com (Lode Vandevenne) +Author: jyrki.alakuijala@gmail.com (Jyrki Alakuijala) */ #include "zopfli.h" -- cgit v1.2.3