diff options
author | Xin Li <delphij@google.com> | 2023-08-14 15:39:08 -0700 |
---|---|---|
committer | Xin Li <delphij@google.com> | 2023-08-14 15:39:08 -0700 |
commit | 831b09764f663425d81949afc407805598537908 (patch) | |
tree | bc2d1560d1cf9dfb9d8c113d6578dec84a456656 | |
parent | 5696cff206f8466cc82a8a094693a6900c7a9440 (diff) | |
parent | 351203a83eb83cc2d96c21a9488554b1e1ba6c2c (diff) | |
download | tflite-support-tmp_amf_298295554.tar.gz |
Merge Android U (ab/10368041)tmp_amf_298295554
Bug: 291102124
Merged-In: Iec90380aa990facae4a83a73e00f2f4adbb1b646
Change-Id: I7f9c998119bb00b35bacbe568b5adf26a1fd2854
106 files changed, 38812 insertions, 422 deletions
@@ -16,6 +16,9 @@ package { // to attach the license to, and including a comment whether the files may be // used in the current project. // See: http://go/android-license-faq + +// TODO (ag/17748161): Split into smaller localized build files as necessary. + license { name: "external_tflite-support_license", visibility: [":__subpackages__"], @@ -29,6 +32,16 @@ license { ], } +cc_defaults { + name: "tflite_support_defaults", + cflags: [ + "-Wno-ignored-qualifiers", + "-Wno-missing-field-initializers", + "-Wno-unused-parameter", + "-Wno-unused-variable", + ], +} + cc_library_static { name: "tflite_support", sdk_version: "current", @@ -64,12 +77,391 @@ cc_library_static { ], apex_available: [ "//apex_available:platform", + "com.android.adservices", "com.android.extservices", ], visibility: ["//external/libtextclassifier:__subpackages__"], } +java_library { + name: "tflite_support_java", + sdk_version: "current", + min_sdk_version: "30", + srcs: [ + "tensorflow_lite_support/java/src/java/org/tensorflow/lite/support/**/*.java", + ], + libs: [ + "guava-android-annotation-stubs", + ], + static_libs: [ + "tensorflowlite_java", + ], + apex_available: [ + "//apex_available:platform", + "com.android.adservices", + "com.android.extservices", + ], +} + +cc_library_static { + name: "tflite_support_tokenizers", + srcs: [ + "tensorflow_lite_support/cc/task/core/task_utils.cc", + "tensorflow_lite_support/cc/text/tokenizers/regex_tokenizer.cc", + "tensorflow_lite_support/cc/text/tokenizers/tokenizer_utils.cc", + "tensorflow_lite_support/cc/utils/common_utils.cc", + ], + defaults: ["tflite_support_defaults"], + stl: "libc++_static", + sdk_version: "current", + min_sdk_version: "30", + export_include_dirs: ["."], + header_libs: [ + // TODO (ag/17748161): Create target for just TFLite headers and use here. + "tensorflow_headers", + "flatbuffer_headers", + "libtextclassifier_flatbuffer_headers", + ], + generated_headers: ["tflite_support_metadata_schema"], + static_libs: [ + "libtextclassifier_bert_tokenizer", + "libtextclassifier_abseil", + "libtflite_static", + ], + apex_available: [ + "//apex_available:platform", + "com.android.adservices", + "com.android.extservices", + ], +} + +cc_test { + name: "tflite_support_tokenizer_tests", + srcs: [ + "tensorflow_lite_support/cc/test/text/regex_tokenizer_test.cc", + ], + defaults: ["tflite_support_defaults"], + data: ["tensorflow_lite_support/cc/test/testdata/task/text/**/*.txt"], + stl: "libc++_static", + sdk_version: "current", + min_sdk_version: "30", + static_libs: [ + "tflite_support_tokenizers", + "libtextclassifier_abseil", + "libgmock_ndk", + "libbase_ndk", + ], + header_libs: [ + // TODO (ag/17748161): Create target for just TFLite headers and use here. + "tensorflow_headers", + "flatbuffer_headers", + ], + generated_headers: ["tflite_support_metadata_schema"], +} + +cc_library_static { + name: "tflite_support_classifiers", + srcs: [ + "tensorflow_lite_support/cc/task/text/nlclassifier/**/*.cc", + "tensorflow_lite_support/cc/common.cc", + "tensorflow_lite_support/cc/port/default/tflite_wrapper.cc", + "tensorflow_lite_support/cc/port/default/statusor.cc", + ], + defaults: ["tflite_support_defaults"], + stl: "libc++_static", + sdk_version: "current", + min_sdk_version: "30", + export_include_dirs: ["."], + header_libs: [ + // TODO (ag/17748161): Create target for just TFLite headers and use here. + "tensorflow_headers", + "flatbuffer_headers", + ], + generated_headers: ["tflite_support_metadata_schema"], + static_libs: [ + "libtextclassifier_abseil", + "libtflite_static", + "tflite_configuration_proto", + "tflite_support_task_core_proto", + "tflite_support_tokenizers", + "tflite_support_metadata_extractor", + ], + apex_available: [ + "//apex_available:platform", + "com.android.adservices", + "com.android.extservices", + ], +} + +java_library { + name: "tflite_support_classifiers_java", + sdk_version: "current", + min_sdk_version: "30", + srcs: [ + "tensorflow_lite_support/java/src/java/org/tensorflow/lite/task/text/nlclassifier/**/*.java", + ], + static_libs: [ + "auto_value_annotations", + "tflite_support_java", + "tflite_support_base_task_api_java", + ], + plugins: [ + "auto_value_plugin", + ], + apex_available: [ + "//apex_available:platform", + "com.android.adservices", + "com.android.extservices", + ], +} + +cc_library_shared { + name: "libtflite_support_classifiers_native", + srcs: [ + "tensorflow_lite_support/java/src/native/task/text/nlclassifier/bert_nlclassifier/bert_nl_classifier_jni.cc", + "tensorflow_lite_support/java/src/native/task/text/nlclassifier/nl_classifier_jni.cc", + "tensorflow_lite_support/java/src/native/task/text/nlclassifier/nl_classifier_jni_utils.cc", + "tensorflow_lite_support/java/src/native/task/core/rbtml_op_resolver.cc", + "tensorflow_lite_support/custom_ops/kernel/unsorted_segment.cc", + "tensorflow_lite_support/cc/utils/jni_utils.cc", + ], + version_script: "tensorflow_lite_support/java/tflite_version_script.lds", + shared_libs: ["liblog"], + static_libs: [ + "libprotobuf-cpp-lite-ndk", + "libtextclassifier_abseil", + "libtextclassifier_bert_tokenizer", + "libtflite_static", + "tflite_configuration_proto", + "tflite_support_classifiers", + "tflite_support_libz", + "tflite_support_metadata_extractor", + "tflite_support_task_core", + "tflite_support_task_core_proto", + "tflite_support_tokenizers", + ], + header_libs: [ + // TODO (ag/17748161): Create target for just TFLite headers and use here. + "tensorflow_headers", + "flatbuffer_headers", + "jni_headers", + "libtextclassifier_flatbuffer_headers", + ], + generated_headers: [ + "libtflite_mutable_schema", + "tflite_support_metadata_schema", + ], + defaults: ["tflite_support_defaults"], + stl: "libc++_static", + sdk_version: "current", + min_sdk_version: "30", + export_include_dirs: ["."], + apex_available: [ + "//apex_available:platform", + "com.android.adservices", + "com.android.extservices", + ], +} + +java_library { + name: "tflite_support_base_task_api_java", + sdk_version: "current", + min_sdk_version: "30", + srcs: [ + "tensorflow_lite_support/java/src/java/org/tensorflow/lite/task/core/*.java", + ], + optimize: { + proguard_flags_files: ["proguard.flags"], + }, + apex_available: [ + "//apex_available:platform", + "com.android.adservices", + "com.android.extservices", + ], +} + +cc_library_static { + name: "tflite_support_metadata_extractor", + srcs: [ + "tensorflow_lite_support/metadata/cc/metadata_extractor.cc", + "tensorflow_lite_support/metadata/cc/utils/zip_readonly_mem_file.cc", + ], + defaults: ["tflite_support_defaults"], + stl: "libc++_static", + sdk_version: "current", + min_sdk_version: "30", + export_include_dirs: ["."], + header_libs: [ + // TODO (ag/17748161): Create target for just TFLite headers and use here. + "tensorflow_headers", + "flatbuffer_headers", + "libtextclassifier_flatbuffer_headers", + ], + generated_headers: [ + "libtflite_mutable_schema", + "tflite_support_metadata_schema", + ], + static_libs: [ + "libtextclassifier_abseil", + "tflite_support_libz", + ], + apex_available: [ + "//apex_available:platform", + "com.android.adservices", + "com.android.extservices", + ], +} + +cc_library_static { + name: "tflite_support_task_core", + srcs: ["tensorflow_lite_support/cc/task/core/**/*.cc"], + defaults: ["tflite_support_defaults"], + stl: "libc++_static", + sdk_version: "current", + min_sdk_version: "30", + export_include_dirs: ["."], + header_libs: [ + // TODO (ag/17748161): Create target for just TFLite headers and use here. + "tensorflow_headers", + "flatbuffer_headers", + ], + generated_headers: [ + "libtflite_mutable_schema", + "tflite_support_metadata_schema", + ], + static_libs: [ + "libtextclassifier_abseil", + "tflite_configuration_proto", + "tflite_support_task_core_proto", + ], + apex_available: [ + "//apex_available:platform", + "com.android.adservices", + "com.android.extservices", + ], +} + +cc_test { + name: "tflite_support_classifier_tests", + srcs: [ + "tensorflow_lite_support/cc/test/task/text/bert_nl_classifier_test.cc", + ], + data: ["tensorflow_lite_support/cc/test/testdata/task/text/**/*.tflite"], + defaults: [ + "tflite_support_defaults", + ], + stl: "libc++_static", + sdk_version: "current", + min_sdk_version: "30", + shared_libs: [ + "liblog", + ], + static_libs: [ + "libprotobuf-cpp-lite-ndk", + "libtextclassifier_abseil", + "libtextclassifier_bert_tokenizer", + "libtflite_static", + "libgmock_ndk", + "libbase_ndk", + "tflite_configuration_proto", + "tflite_support_classifiers", + "tflite_support_libz", + "tflite_support_metadata_extractor", + "tflite_support_task_core", + "tflite_support_task_core_proto", + "tflite_support_tokenizers", + ], + header_libs: [ + // TODO (ag/17748161): Create target for just TFLite headers and use here. + "tensorflow_headers", + "flatbuffer_headers", + "liblog_headers", + "libtextclassifier_flatbuffer_headers", + ], + generated_headers: [ + "libtflite_mutable_schema", + "tflite_support_metadata_schema", + ], +} + +android_test { + name: "TfliteSupportClassifierTests", + srcs: [ + "tensorflow_lite_support/java/src/javatests/org/tensorflow/lite/task/text/nlclassifier/BertNLClassifierTest.java", + ], + asset_dirs: [ + "tensorflow_lite_support/java/src/javatests/testdata/task/text", + ], + defaults: ["modules-utils-testable-device-config-defaults"], + manifest: "tensorflow_lite_support/java/AndroidManifest.xml", + sdk_version: "module_current", + min_sdk_version: "30", + static_libs: [ + "androidx.test.core", + "tensorflowlite_java", + "truth-prebuilt", + "tflite_support_classifiers_java", + "tflite_support_test_utils_java", + ], + libs: [ + "android.test.base", + "android.test.mock.stubs", + ], + test_suites: [ + "general-tests", + ], + jni_libs: [ + "libtflite_support_classifiers_native", + ], + aaptflags: [ + // Avoid compression on tflite files as the Interpreter + // can not load compressed flat buffer formats. + // (*appt compresses all assets into the apk by default) + // See https://elinux.org/Android_aapt for more detail. + "-0 .tflite", + ], +} + +java_library_static { + name: "tflite_support_test_utils_java", + sdk_version: "module_current", + min_sdk_version: "30", + srcs: [ + "tensorflow_lite_support/java/src/javatests/org/tensorflow/lite/task/core/TestUtils.java", + ], + static_libs: [ + "apache-commons-compress", + "guava", + ], +} + +cc_library_static { + name: "tflite_support_task_core_proto", + proto: { + type: "lite", + canonical_path_from_root: false, + export_proto_headers: true, + }, + stl: "libc++", + srcs: ["tensorflow_lite_support/cc/task/core/proto/**/*.proto"], + sdk_version: "current", + min_sdk_version: "30", + apex_available: [ + "//apex_available:platform", + "com.android.adservices", + "com.android.extservices", + ], +} + +// Add header file targets. +genrule { + name: "tflite_support_metadata_schema", + srcs: ["tensorflow_lite_support/metadata/metadata_schema.fbs"], + out: ["tensorflow_lite_support/metadata/metadata_schema_generated.h"], + defaults: ["tflite_support_fbgen"], +} + // ----------------- // Generate headers with FlatBuffer schema compiler. @@ -1,3 +1,2 @@ -licha@google.com -toki@google.com -tonymak@google.com +fban@google.com +luwa@google.com
\ No newline at end of file diff --git a/proguard.flags b/proguard.flags new file mode 100644 index 00000000..566e53fd --- /dev/null +++ b/proguard.flags @@ -0,0 +1,10 @@ +# Stop proguard from stripping away code used by tensorflow JNI library. +-keepclassmembers class org.tensorflow.lite.NativeInterpreterWrapper { + private long inferenceDurationNanoseconds; +} + +-keep class org.tensorflow.lite.annotations.UsedByReflection +-keep @org.tensorflow.lite.annotations.UsedByReflection class * +-keepclassmembers class * { + @org.tensorflow.lite.annotations.UsedByReflection *; +}
\ No newline at end of file diff --git a/tensorflow_lite_support/cc/port/default/statusor.cc b/tensorflow_lite_support/cc/port/default/statusor.cc index 5cf1196a..11c1babe 100644 --- a/tensorflow_lite_support/cc/port/default/statusor.cc +++ b/tensorflow_lite_support/cc/port/default/statusor.cc @@ -18,7 +18,8 @@ limitations under the License. #include <utility> -#include <glog/logging.h> +#include "absl/base/internal/raw_logging.h" +#include "absl/status/status.h" #include "absl/strings/str_cat.h" namespace tflite { @@ -40,23 +41,32 @@ namespace internal_statusor { void Helper::HandleInvalidStatusCtorArg(absl::Status* status) { const char* kMessage = "An OK status is not a valid constructor argument to StatusOr<T>"; - LOG(DFATAL) << kMessage; +#ifdef NDEBUG + ABSL_INTERNAL_LOG(ERROR, kMessage); +#else + ABSL_INTERNAL_LOG(FATAL, kMessage); +#endif // In optimized builds, we will fall back to ::util::error::INTERNAL. *status = absl::InternalError(kMessage); } void Helper::Crash(const absl::Status& status) { - LOG(FATAL) << "Attempting to fetch value instead of handling error " - << status; - _exit(1); + ABSL_INTERNAL_LOG( + FATAL, + absl::StrCat("Attempting to fetch value instead of handling error ", + status.ToString())); + _Exit(1); } void ThrowBadStatusOrAccess(absl::Status status) { #ifdef ABSL_HAVE_EXCEPTIONS throw BadStatusOrAccess(std::move(status)); #else - LOG(FATAL) << "Attempting to fetch value instead of handling error " - << status; + ABSL_INTERNAL_LOG( + FATAL, + absl::StrCat("Attempting to fetch value instead of handling error ", + status.ToString())); + std::abort(); #endif } diff --git a/tensorflow_lite_support/cc/task/core/tflite_engine.cc b/tensorflow_lite_support/cc/task/core/tflite_engine.cc index cf923f6a..bb711ede 100644 --- a/tensorflow_lite_support/cc/task/core/tflite_engine.cc +++ b/tensorflow_lite_support/cc/task/core/tflite_engine.cc @@ -21,7 +21,6 @@ limitations under the License. #include "absl/strings/str_cat.h" #include "tensorflow/lite/builtin_ops.h" #include "tensorflow/lite/stderr_reporter.h" -#include "tensorflow/lite/tools/verifier.h" #include "tensorflow_lite_support/cc/common.h" #include "tensorflow_lite_support/cc/port/status_macros.h" #include "tensorflow_lite_support/cc/task/core/external_file_handler.h" @@ -59,19 +58,13 @@ int TfLiteEngine::ErrorReporter::Report(const char* format, va_list args) { return std::vsnprintf(error_message, sizeof(error_message), format, args); } -bool TfLiteEngine::Verifier::Verify(const char* data, int length, - tflite::ErrorReporter* reporter) { - return tflite::Verify(data, length, *op_resolver_, reporter); -} - #if TFLITE_USE_C_API TfLiteEngine::TfLiteEngine(std::unique_ptr<tflite::OpResolver> resolver) : model_(nullptr, TfLiteModelDelete), - resolver_(std::move(resolver)), - verifier_(resolver_.get()) {} + resolver_(std::move(resolver)) {} #else TfLiteEngine::TfLiteEngine(std::unique_ptr<tflite::OpResolver> resolver) - : model_(), resolver_(std::move(resolver)), verifier_(resolver_.get()) {} + : model_(), resolver_(std::move(resolver)) {} #endif std::vector<TfLiteTensor*> TfLiteEngine::GetInputs() { @@ -96,10 +89,8 @@ std::vector<const TfLiteTensor*> TfLiteEngine::GetOutputs() { return tensors; } -// The following function is adapted from the code in -// tflite::FlatBufferModel::VerifyAndBuildFromBuffer. -void TfLiteEngine::VerifyAndBuildModelFromBuffer(const char* buffer_data, - size_t buffer_size) { +// The following function is adapted from the code in tflite::FlatBufferModel::BuildFromBuffer. +void TfLiteEngine::BuildModelFromBuffer(const char* buffer_data, size_t buffer_size) { #if TFLITE_USE_C_API // First verify with the base flatbuffers verifier. // This verifies that the model is a valid flatbuffer model. @@ -111,24 +102,19 @@ void TfLiteEngine::VerifyAndBuildModelFromBuffer(const char* buffer_data, model_ = nullptr; return; } - // Next verify with the extra verifier. This verifies that the model only - // uses operators supported by the OpResolver. - if (!verifier_.Verify(buffer_data, buffer_size, &error_reporter_)) { - model_ = nullptr; - return; - } // Build the model. model_.reset(TfLiteModelCreate(buffer_data, buffer_size)); #else - model_ = tflite::FlatBufferModel::VerifyAndBuildFromBuffer( - buffer_data, buffer_size, &verifier_, &error_reporter_); + // Warning: This branch of the if-statement lacks a verification step for the model. + model_ = tflite::FlatBufferModel::BuildFromBuffer( + buffer_data, buffer_size, &error_reporter_); #endif } absl::Status TfLiteEngine::InitializeFromModelFileHandler() { const char* buffer_data = model_file_handler_->GetFileContent().data(); size_t buffer_size = model_file_handler_->GetFileContent().size(); - VerifyAndBuildModelFromBuffer(buffer_data, buffer_size); + BuildModelFromBuffer(buffer_data, buffer_size); if (model_ == nullptr) { // To be replaced with a proper switch-case when TF Lite model builder // returns a `TfLiteStatus` code capturing this type of error. diff --git a/tensorflow_lite_support/cc/task/core/tflite_engine.h b/tensorflow_lite_support/cc/task/core/tflite_engine.h index 30f239da..3ab6abea 100644 --- a/tensorflow_lite_support/cc/task/core/tflite_engine.h +++ b/tensorflow_lite_support/cc/task/core/tflite_engine.h @@ -38,8 +38,6 @@ limitations under the License. // replacement for the features of tflite::support::TfLiteInterpreterWrapper. #if TFLITE_USE_C_API #include "tensorflow/lite/c/c_api.h" -#include "tensorflow/lite/core/api/verifier.h" -#include "tensorflow/lite/tools/verifier.h" #else #include "tensorflow/lite/interpreter.h" #include "tensorflow/lite/model.h" @@ -190,24 +188,8 @@ class TfLiteEngine { ErrorReporter error_reporter_; private: - // Direct wrapper around tflite::TfLiteVerifier which checks the integrity of - // the FlatBuffer data provided as input. - class Verifier : public tflite::TfLiteVerifier { - public: - explicit Verifier(const tflite::OpResolver* op_resolver) - : op_resolver_(op_resolver) {} - bool Verify(const char* data, int length, - tflite::ErrorReporter* reporter) override; - // The OpResolver to be used to build the TF Lite interpreter. - const tflite::OpResolver* op_resolver_; - }; - - // Verifies that the supplied buffer refers to a valid flatbuffer model, - // and that it uses only operators that are supported by the OpResolver - // that was passed to the TfLiteEngine constructor, and then builds - // the model from the buffer and stores it in 'model_'. - void VerifyAndBuildModelFromBuffer(const char* buffer_data, - size_t buffer_size); + // Builds the model from the buffer and stores it in 'model_'. + void BuildModelFromBuffer(const char* buffer_data, size_t buffer_size); // Gets the buffer from the file handler; verifies and builds the model // from the buffer; if successful, sets 'model_metadata_extractor_' to be @@ -229,9 +211,6 @@ class TfLiteEngine { // actual implementation. Defaults to TF Lite BuiltinOpResolver. std::unique_ptr<tflite::OpResolver> resolver_; - // Extra verifier for FlatBuffer input data. - Verifier verifier_; - // ExternalFile and corresponding ExternalFileHandler for models loaded from // disk or file descriptor. ExternalFile external_file_; diff --git a/tensorflow_lite_support/cc/task/text/nlclassifier/bert_nl_classifier.cc b/tensorflow_lite_support/cc/task/text/nlclassifier/bert_nl_classifier.cc index d689c9e8..4e4e999d 100644 --- a/tensorflow_lite_support/cc/task/text/nlclassifier/bert_nl_classifier.cc +++ b/tensorflow_lite_support/cc/task/text/nlclassifier/bert_nl_classifier.cc @@ -55,6 +55,9 @@ namespace { constexpr char kIdsTensorName[] = "ids"; constexpr char kMaskTensorName[] = "mask"; constexpr char kSegmentIdsTensorName[] = "segment_ids"; +constexpr int kIdsTensorIndex = 0; +constexpr int kMaskTensorIndex = 1; +constexpr int kSegmentIdsTensorIndex = 2; constexpr char kScoreTensorName[] = "probability"; constexpr char kClassificationToken[] = "[CLS]"; constexpr char kSeparator[] = "[SEP]"; @@ -78,39 +81,46 @@ absl::Status BertNLClassifier::Preprocess( TokenizerResult input_tokenize_results; input_tokenize_results = tokenizer_->Tokenize(processed_input); - // 2 accounts for [CLS], [SEP] - absl::Span<const std::string> query_tokens = - absl::MakeSpan(input_tokenize_results.subwords.data(), - input_tokenize_results.subwords.data() + - std::min(static_cast<size_t>(kMaxSeqLen - 2), - input_tokenize_results.subwords.size())); - - std::vector<std::string> tokens; - tokens.reserve(2 + query_tokens.size()); - // Start of generating the features. - tokens.push_back(kClassificationToken); - // For query input. - for (const auto& query_token : query_tokens) { - tokens.push_back(query_token); + // Offset by 2 to account for [CLS] and [SEP] + int input_tokens_size = + static_cast<int>(input_tokenize_results.subwords.size()) + 2; + int input_tensor_length = input_tokens_size; + if (!input_tensors_are_dynamic_) { + input_tokens_size = std::min(kMaxSeqLen, input_tokens_size); + input_tensor_length = kMaxSeqLen; + } else { + GetTfLiteEngine()->interpreter()->ResizeInputTensorStrict(kIdsTensorIndex, + {1, input_tensor_length}); + GetTfLiteEngine()->interpreter()->ResizeInputTensorStrict(kMaskTensorIndex, + {1, input_tensor_length}); + GetTfLiteEngine()->interpreter()->ResizeInputTensorStrict(kSegmentIdsTensorIndex, + {1, input_tensor_length}); + GetTfLiteEngine()->interpreter()->AllocateTensors(); } - // For Separation. - tokens.push_back(kSeparator); - std::vector<int> input_ids(kMaxSeqLen, 0); - std::vector<int> input_mask(kMaxSeqLen, 0); + std::vector<std::string> input_tokens; + input_tokens.reserve(input_tokens_size); + input_tokens.push_back(std::string(kClassificationToken)); + for (int i = 0; i < input_tokens_size - 2; ++i) { + input_tokens.push_back(std::move(input_tokenize_results.subwords[i])); + } + input_tokens.push_back(std::string(kSeparator)); + + std::vector<int> input_ids(input_tensor_length, 0); + std::vector<int> input_mask(input_tensor_length, 0); // Convert tokens back into ids and set mask - for (int i = 0; i < tokens.size(); ++i) { - tokenizer_->LookupId(tokens[i], &input_ids[i]); + for (int i = 0; i < input_tokens.size(); ++i) { + tokenizer_->LookupId(input_tokens[i], &input_ids[i]); input_mask[i] = 1; } - // |<-----------kMaxSeqLen---------->| + // |<--------input_tensor_length------->| // input_ids [CLS] s1 s2... sn [SEP] 0 0... 0 // input_masks 1 1 1... 1 1 0 0... 0 // segment_ids 0 0 0... 0 0 0 0... 0 PopulateTensor(input_ids, ids_tensor); PopulateTensor(input_mask, mask_tensor); - PopulateTensor(std::vector<int>(kMaxSeqLen, 0), segment_ids_tensor); + PopulateTensor(std::vector<int>(input_tensor_length, 0), segment_ids_tensor); return absl::OkStatus(); } @@ -189,6 +199,73 @@ absl::Status BertNLClassifier::InitializeFromMetadata() { TrySetLabelFromMetadata( GetMetadataExtractor()->GetOutputTensorMetadata(kOutputTensorIndex)) .IgnoreError(); + + auto* input_tensor_metadatas = + GetMetadataExtractor()->GetInputTensorMetadata(); + const auto& input_tensors = GetInputTensors(); + const auto& ids_tensor = *FindTensorByName(input_tensors, input_tensor_metadatas, + kIdsTensorName); + const auto& mask_tensor = *FindTensorByName(input_tensors, input_tensor_metadatas, + kMaskTensorName); + const auto& segment_ids_tensor = *FindTensorByName(input_tensors, input_tensor_metadatas, + kSegmentIdsTensorName); + if (ids_tensor.dims->size != 2 || mask_tensor.dims->size != 2 || + segment_ids_tensor.dims->size != 2) { + return CreateStatusWithPayload( + absl::StatusCode::kInternal, + absl::StrFormat( + "The three input tensors in Bert models are expected to have dim " + "2, but got ids_tensor (%d), mask_tensor (%d), segment_ids_tensor " + "(%d).", + ids_tensor.dims->size, mask_tensor.dims->size, + segment_ids_tensor.dims->size), + TfLiteSupportStatus::kInvalidInputTensorDimensionsError); + } + if (ids_tensor.dims->data[0] != 1 || mask_tensor.dims->data[0] != 1 || + segment_ids_tensor.dims->data[0] != 1) { + return CreateStatusWithPayload( + absl::StatusCode::kInternal, + absl::StrFormat( + "The three input tensors in Bert models are expected to have same " + "batch size 1, but got ids_tensor (%d), mask_tensor (%d), " + "segment_ids_tensor (%d).", + ids_tensor.dims->data[0], mask_tensor.dims->data[0], + segment_ids_tensor.dims->data[0]), + TfLiteSupportStatus::kInvalidInputTensorSizeError); + } + if (ids_tensor.dims->data[1] != mask_tensor.dims->data[1] || + ids_tensor.dims->data[1] != segment_ids_tensor.dims->data[1]) { + return CreateStatusWithPayload( + absl::StatusCode::kInternal, + absl::StrFormat("The three input tensors in Bert models are " + "expected to have same length, but got ids_tensor " + "(%d), mask_tensor (%d), segment_ids_tensor (%d).", + ids_tensor.dims->data[1], mask_tensor.dims->data[1], + segment_ids_tensor.dims->data[1]), + TfLiteSupportStatus::kInvalidInputTensorSizeError); + } + + // If some tensor does not have a size 2 dims_signature, then we + // assume the input is not dynamic. + if (ids_tensor.dims_signature->size != 2 || + mask_tensor.dims_signature->size != 2 || + segment_ids_tensor.dims_signature->size != 2) { + return absl::OkStatus(); + } + + if (ids_tensor.dims_signature->data[1] == -1 && + mask_tensor.dims_signature->data[1] == -1 && + segment_ids_tensor.dims_signature->data[1] == -1) { + input_tensors_are_dynamic_ = true; + } else if (ids_tensor.dims_signature->data[1] == -1 || + mask_tensor.dims_signature->data[1] == -1 || + segment_ids_tensor.dims_signature->data[1] == -1) { + return CreateStatusWithPayload( + absl::StatusCode::kInternal, + "Input tensors contain a mix of static and dynamic tensors", + TfLiteSupportStatus::kInvalidInputTensorSizeError); + } + return absl::OkStatus(); } diff --git a/tensorflow_lite_support/cc/task/text/nlclassifier/bert_nl_classifier.h b/tensorflow_lite_support/cc/task/text/nlclassifier/bert_nl_classifier.h index 0c709ee0..7a4b0587 100644 --- a/tensorflow_lite_support/cc/task/text/nlclassifier/bert_nl_classifier.h +++ b/tensorflow_lite_support/cc/task/text/nlclassifier/bert_nl_classifier.h @@ -95,6 +95,7 @@ class BertNLClassifier : public NLClassifier { absl::Status InitializeFromMetadata(); std::unique_ptr<tflite::support::text::tokenizer::Tokenizer> tokenizer_; + bool input_tensors_are_dynamic_ = false; }; } // namespace nlclassifier diff --git a/tensorflow_lite_support/cc/task/text/nlclassifier/nl_classifier.cc b/tensorflow_lite_support/cc/task/text/nlclassifier/nl_classifier.cc index 1643e3e0..d322a151 100644 --- a/tensorflow_lite_support/cc/task/text/nlclassifier/nl_classifier.cc +++ b/tensorflow_lite_support/cc/task/text/nlclassifier/nl_classifier.cc @@ -63,6 +63,7 @@ using ::tflite::task::core::PopulateTensor; namespace { constexpr int kRegexTokenizerInputTensorIndex = 0; constexpr int kRegexTokenizerProcessUnitIndex = 0; +constexpr char kNoVersionInfo[] = "NO_VERSION_INFO"; StatusOr<absl::string_view> CheckAndLoadFirstAssociatedFile( const flatbuffers::Vector<flatbuffers::Offset<tflite::AssociatedFile>>* @@ -81,7 +82,7 @@ StatusOr<absl::string_view> CheckAndLoadFirstAssociatedFile( return vocab_buffer; } -StatusOr<std::unique_ptr<Tokenizer>> CreateRegexTokenizerFromProcessUnit( +StatusOr<std::unique_ptr<RegexTokenizer>> CreateRegexTokenizerFromProcessUnit( const tflite::ProcessUnit* tokenizer_process_unit, const tflite::metadata::ModelMetadataExtractor* metadata_extractor) { if (metadata_extractor == nullptr || tokenizer_process_unit == nullptr) { @@ -169,6 +170,11 @@ absl::Status NLClassifier::TrySetLabelFromMetadata( labels_vector_ = absl::make_unique<std::vector<std::string>>(LoadVocabFromBuffer( label_buffer.value().data(), label_buffer.value().size())); + if (associated_file->version() == nullptr) { + labels_version_ = kNoVersionInfo; + } else { + labels_version_ = associated_file->version()->str(); + } return absl::OkStatus(); } else { return CreateStatusWithPayload( @@ -179,9 +185,29 @@ absl::Status NLClassifier::TrySetLabelFromMetadata( } std::vector<Category> NLClassifier::Classify(const std::string& text) { - // The NLClassifier implementation for Preprocess() and Postprocess() never - // returns errors: just call value(). - return Infer(text).value(); + StatusOr<std::vector<Category>> infer_result = ClassifyText(text); + if (!infer_result.ok()) { + return {}; + } + return infer_result.value(); +} + +StatusOr<std::vector<Category>> NLClassifier::ClassifyText( + const std::string& text) { + return Infer(text); +} + +std::string NLClassifier::GetModelVersion() const { + tflite::support::StatusOr<std::string> model_version = + GetMetadataExtractor()->GetModelVersion(); + if (model_version.ok()) { + return model_version.value(); + } + return kNoVersionInfo; +} + +std::string NLClassifier::GetLabelsVersion() const { + return labels_version_; } absl::Status NLClassifier::Preprocess( @@ -447,7 +473,7 @@ bool NLClassifier::HasRegexTokenizerMetadata() { absl::Status NLClassifier::SetupRegexTokenizer() { ASSIGN_OR_RETURN( - std::unique_ptr<Tokenizer> base_tokenizer, + tokenizer_, CreateRegexTokenizerFromProcessUnit( GetMetadataExtractor() ->GetInputTensorMetadata(kRegexTokenizerInputTensorIndex) @@ -455,9 +481,6 @@ absl::Status NLClassifier::SetupRegexTokenizer() { ->Get(kRegexTokenizerProcessUnitIndex), GetMetadataExtractor())); - tokenizer_ = std::unique_ptr<RegexTokenizer>( - dynamic_cast<RegexTokenizer*>(base_tokenizer.release())); - return absl::OkStatus(); } diff --git a/tensorflow_lite_support/cc/task/text/nlclassifier/nl_classifier.h b/tensorflow_lite_support/cc/task/text/nlclassifier/nl_classifier.h index 2a9573a1..8e70ba4c 100644 --- a/tensorflow_lite_support/cc/task/text/nlclassifier/nl_classifier.h +++ b/tensorflow_lite_support/cc/task/text/nlclassifier/nl_classifier.h @@ -109,9 +109,20 @@ class NLClassifier : public core::BaseTaskApi<std::vector<core::Category>, std::unique_ptr<tflite::OpResolver> resolver = absl::make_unique<tflite::ops::builtin::BuiltinOpResolver>()); - // Performs classification on a string input, returns classified results. + // DEPRECATED (unannotated for backward compatibility). Prefer using `ClassifyText`. std::vector<core::Category> Classify(const std::string& text); + // Performs classification on a string input, returns classified results or an + // error. + tflite::support::StatusOr<std::vector<core::Category>> ClassifyText( + const std::string& text); + + // Gets the model version, or "NO_VERSION_INFO" in case there is no version. + std::string GetModelVersion() const; + + // Gets the labels version, or "NO_VERSION_INFO" in case there is no version. + std::string GetLabelsVersion() const; + protected: static constexpr int kOutputTensorIndex = 0; static constexpr int kOutputTensorLabelFileIndex = 0; @@ -170,6 +181,9 @@ class NLClassifier : public core::BaseTaskApi<std::vector<core::Category>, // labels vector initialized from output tensor's associated file, if one // exists. std::unique_ptr<std::vector<std::string>> labels_vector_; + // labels version assigned from output tensor's associated file metadata, + // if one exists. + std::string labels_version_; std::unique_ptr<tflite::support::text::tokenizer::RegexTokenizer> tokenizer_; }; diff --git a/tensorflow_lite_support/cc/test/task/text/bert_nl_classifier_test.cc b/tensorflow_lite_support/cc/test/task/text/bert_nl_classifier_test.cc new file mode 100644 index 00000000..23a3ddca --- /dev/null +++ b/tensorflow_lite_support/cc/test/task/text/bert_nl_classifier_test.cc @@ -0,0 +1,171 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_lite_support/cc/task/text/nlclassifier/bert_nl_classifier.h" + +#include <fcntl.h> + +#include "android-base/file.h" +#include "tensorflow_lite_support/cc/port/gmock.h" +#include "tensorflow_lite_support/cc/port/gtest.h" +#include "tensorflow_lite_support/cc/task/core/task_utils.h" + +namespace tflite { +namespace task { +namespace text { +namespace nlclassifier { + +namespace { + +using ::android::base::GetExecutableDirectory; +using ::testing::HasSubstr; +using ::tflite::support::kTfLiteSupportPayload; +using ::tflite::support::StatusOr; +using ::tflite::support::TfLiteSupportStatus; +using ::tflite::task::core::Category; +using ::tflite::task::core::LoadBinaryContent; + +constexpr char kTestModelPath[] = + "/tensorflow_lite_support/cc/test/testdata/task/text/" + "test_model_nl_classifier_bert.tflite"; + +constexpr char kInvalidModelPath[] = "i/do/not/exist.tflite"; + +constexpr int kMaxSeqLen = 128; + +TEST(BertNLClassifierTest, TestNLClassifierCreationFilePath) { + std::string test_model_path = absl::StrCat(GetExecutableDirectory(), kTestModelPath); + StatusOr<std::unique_ptr<BertNLClassifier>> classifier = + BertNLClassifier::CreateFromFile(test_model_path); + EXPECT_TRUE(classifier.ok()); +} + +TEST(BertNLClassifierTest, TestNLClassifierCreationBinary) { + std::string test_model_path = absl::StrCat(GetExecutableDirectory(), kTestModelPath); + std::string model_buffer = LoadBinaryContent(test_model_path.c_str()); + StatusOr<std::unique_ptr<BertNLClassifier>> classifier = + BertNLClassifier::CreateFromBuffer(model_buffer.data(), model_buffer.size()); + EXPECT_TRUE(classifier.ok()); +} + +TEST(BertNLClassifierTest, TestNLClassifierCreationFailure) { + StatusOr<std::unique_ptr<BertNLClassifier>> classifier = + BertNLClassifier::CreateFromFile(kInvalidModelPath); + + EXPECT_EQ(classifier.status().code(), absl::StatusCode::kNotFound); + EXPECT_THAT(classifier.status().message(), + HasSubstr("Unable to open file at i/do/not/exist.tflite")); + EXPECT_THAT(classifier.status().GetPayload(kTfLiteSupportPayload), + testing::Optional(absl::Cord( + absl::StrCat(TfLiteSupportStatus::kFileNotFoundError)))); +} + +Category* GetCategoryWithClassName(const std::string& class_name, + std::vector<Category>& categories) { + for (Category& category : categories) { + if (category.class_name == class_name) { + return &category; + } + } + return nullptr; +} + +void verify_classifier(std::unique_ptr<BertNLClassifier> classifier, + bool verify_positive) { + if (verify_positive) { + tflite::support::StatusOr<std::vector<core::Category>> results = + classifier->ClassifyText("unflinchingly bleak and desperate"); + + EXPECT_TRUE(results.ok()); + EXPECT_GT(GetCategoryWithClassName("negative", results.value())->score, + GetCategoryWithClassName("positive", results.value())->score); + } else { + tflite::support::StatusOr<std::vector<core::Category>> results = + classifier->ClassifyText("it's a charming and often affecting journey"); + + EXPECT_TRUE(results.ok()); + EXPECT_GT(GetCategoryWithClassName("positive", results.value())->score, + GetCategoryWithClassName("negative", results.value())->score); + } +} + +TEST(BertNLClassifierTest, TestNLClassifier_ClassifyNegative) { + std::string test_model_path = absl::StrCat(GetExecutableDirectory(), kTestModelPath); + std::string model_buffer = LoadBinaryContent(test_model_path.c_str()); + StatusOr<std::unique_ptr<BertNLClassifier>> classifier = + BertNLClassifier::CreateFromBuffer(model_buffer.data(), model_buffer.size()); + EXPECT_TRUE(classifier.ok()); + + verify_classifier(std::move(*classifier), false); +} + +TEST(BertNLClassifierTest, TestNLClassifier_ClassifyPositive) { + std::string test_model_path = absl::StrCat(GetExecutableDirectory(), kTestModelPath); + std::string model_buffer = LoadBinaryContent(test_model_path.c_str()); + StatusOr<std::unique_ptr<BertNLClassifier>> classifier = + BertNLClassifier::CreateFromBuffer(model_buffer.data(), model_buffer.size()); + EXPECT_TRUE(classifier.ok()); + + verify_classifier(std::move(*classifier), true); +} + +TEST(BertNLClassifierTest, TestNLClassifierFd_ClassifyPositive) { + std::string test_model_path = absl::StrCat(GetExecutableDirectory(), kTestModelPath); + StatusOr<std::unique_ptr<BertNLClassifier>> classifier = + BertNLClassifier::CreateFromFd(open(test_model_path.c_str(), O_RDONLY)); + EXPECT_TRUE(classifier.ok()); + + verify_classifier(std::move(*classifier), false); +} + +TEST(BertNLClassifierTest, TestNLClassifierFd_ClassifyNegative) { + std::string test_model_path = absl::StrCat(GetExecutableDirectory(), kTestModelPath); + StatusOr<std::unique_ptr<BertNLClassifier>> classifier = + BertNLClassifier::CreateFromFd(open(test_model_path.c_str(), O_RDONLY)); + EXPECT_TRUE(classifier.ok()); + + verify_classifier(std::move(*classifier), true); +} + +// BertNLClassifier limits the input sequence to kMaxSeqLen, test when input is +// longer than this the classifier still works correctly. +TEST(BertNLClassifierTest, TestNLClassifier_ClassifyLongPositive_notOOB) { + std::string test_model_path = absl::StrCat(GetExecutableDirectory(), kTestModelPath); + std::string model_buffer = LoadBinaryContent(test_model_path.c_str()); + std::stringstream ss_for_positive_review; + ss_for_positive_review + << "it's a charming and often affecting journey and this is a long"; + for (int i = 0; i < kMaxSeqLen; ++i) { + ss_for_positive_review << " long"; + } + ss_for_positive_review << " movie review"; + StatusOr<std::unique_ptr<BertNLClassifier>> classifier = + BertNLClassifier::CreateFromBuffer(model_buffer.data(), model_buffer.size()); + EXPECT_TRUE(classifier.ok()); + + tflite::support::StatusOr<std::vector<core::Category>> results = + classifier.value()->ClassifyText(ss_for_positive_review.str()); + + EXPECT_TRUE(results.ok()); + EXPECT_GT(GetCategoryWithClassName("positive", results.value())->score, + GetCategoryWithClassName("negative", results.value())->score); +} + +} // namespace + +} // namespace nlclassifier +} // namespace text +} // namespace task +} // namespace tflite diff --git a/tensorflow_lite_support/cc/test/testdata/task/text/empty_vocab_for_regex_tokenizer.txt b/tensorflow_lite_support/cc/test/testdata/task/text/empty_vocab_for_regex_tokenizer.txt new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/tensorflow_lite_support/cc/test/testdata/task/text/empty_vocab_for_regex_tokenizer.txt diff --git a/tensorflow_lite_support/cc/test/testdata/task/text/test_model_nl_classifier_bert.tflite b/tensorflow_lite_support/cc/test/testdata/task/text/test_model_nl_classifier_bert.tflite Binary files differnew file mode 100644 index 00000000..97a32da4 --- /dev/null +++ b/tensorflow_lite_support/cc/test/testdata/task/text/test_model_nl_classifier_bert.tflite diff --git a/tensorflow_lite_support/cc/test/testdata/task/text/vocab_for_regex_tokenizer.txt b/tensorflow_lite_support/cc/test/testdata/task/text/vocab_for_regex_tokenizer.txt new file mode 100644 index 00000000..0a27d7c6 --- /dev/null +++ b/tensorflow_lite_support/cc/test/testdata/task/text/vocab_for_regex_tokenizer.txt @@ -0,0 +1,10000 @@ +<PAD> 0 +<START> 1 +<UNKNOWN> 2 +<UNUSED> 3 +the 4 +and 5 +a 6 +of 7 +to 8 +is 9 +br 10 +in 11 +it 12 +i 13 +this 14 +that 15 +was 16 +as 17 +for 18 +with 19 +movie 20 +but 21 +film 22 +on 23 +not 24 +you 25 +are 26 +his 27 +have 28 +he 29 +be 30 +one 31 +all 32 +at 33 +by 34 +an 35 +they 36 +who 37 +so 38 +from 39 +like 40 +her 41 +or 42 +just 43 +about 44 +it's 45 +out 46 +has 47 +if 48 +some 49 +there 50 +what 51 +good 52 +more 53 +when 54 +very 55 +up 56 +no 57 +time 58 +she 59 +even 60 +my 61 +would 62 +which 63 +only 64 +story 65 +really 66 +see 67 +their 68 +had 69 +can 70 +were 71 +me 72 +well 73 +than 74 +we 75 +much 76 +been 77 +bad 78 +get 79 +will 80 +do 81 +also 82 +into 83 +people 84 +other 85 +first 86 +great 87 +because 88 +how 89 +him 90 +most 91 +don't 92 +made 93 +its 94 +then 95 +way 96 +make 97 +them 98 +too 99 +could 100 +any 101 +movies 102 +after 103 +think 104 +characters 105 +watch 106 +two 107 +films 108 +character 109 +seen 110 +many 111 +being 112 +life 113 +plot 114 +never 115 +acting 116 +little 117 +best 118 +love 119 +over 120 +where 121 +did 122 +show 123 +know 124 +off 125 +ever 126 +does 127 +better 128 +your 129 +end 130 +still 131 +man 132 +here 133 +these 134 +say 135 +scene 136 +while 137 +why 138 +scenes 139 +go 140 +such 141 +something 142 +through 143 +should 144 +back 145 +i'm 146 +real 147 +those 148 +watching 149 +now 150 +though 151 +doesn't 152 +years 153 +old 154 +thing 155 +actors 156 +work 157 +10 158 +before 159 +another 160 +didn't 161 +new 162 +funny 163 +nothing 164 +actually 165 +makes 166 +director 167 +look 168 +find 169 +going 170 +few 171 +same 172 +part 173 +again 174 +every 175 +lot 176 +cast 177 +us 178 +quite 179 +down 180 +want 181 +world 182 +things 183 +pretty 184 +young 185 +seems 186 +around 187 +got 188 +horror 189 +however 190 +can't 191 +fact 192 +take 193 +big 194 +enough 195 +long 196 +thought 197 +that's 198 +both 199 +between 200 +series 201 +give 202 +may 203 +original 204 +own 205 +action 206 +i've 207 +right 208 +without 209 +always 210 +times 211 +comedy 212 +point 213 +gets 214 +must 215 +come 216 +role 217 +isn't 218 +saw 219 +almost 220 +interesting 221 +least 222 +family 223 +done 224 +there's 225 +whole 226 +bit 227 +music 228 +script 229 +far 230 +making 231 +guy 232 +anything 233 +minutes 234 +feel 235 +last 236 +since 237 +might 238 +performance 239 +he's 240 +2 241 +probably 242 +kind 243 +am 244 +away 245 +yet 246 +rather 247 +tv 248 +worst 249 +girl 250 +day 251 +sure 252 +fun 253 +hard 254 +woman 255 +played 256 +each 257 +found 258 +anyone 259 +having 260 +although 261 +especially 262 +our 263 +believe 264 +course 265 +comes 266 +looking 267 +screen 268 +trying 269 +set 270 +goes 271 +looks 272 +place 273 +book 274 +different 275 +put 276 +ending 277 +money 278 +maybe 279 +once 280 +sense 281 +reason 282 +true 283 +actor 284 +everything 285 +wasn't 286 +shows 287 +dvd 288 +three 289 +worth 290 +year 291 +job 292 +main 293 +someone 294 +together 295 +watched 296 +play 297 +american 298 +plays 299 +1 300 +said 301 +effects 302 +later 303 +takes 304 +instead 305 +seem 306 +beautiful 307 +john 308 +himself 309 +version 310 +audience 311 +high 312 +house 313 +night 314 +during 315 +everyone 316 +left 317 +special 318 +seeing 319 +half 320 +excellent 321 +wife 322 +star 323 +shot 324 +war 325 +idea 326 +nice 327 +black 328 +less 329 +mind 330 +simply 331 +read 332 +second 333 +else 334 +you're 335 +father 336 +fan 337 +poor 338 +help 339 +completely 340 +death 341 +3 342 +used 343 +home 344 +either 345 +short 346 +line 347 +given 348 +men 349 +top 350 +dead 351 +budget 352 +try 353 +performances 354 +wrong 355 +classic 356 +boring 357 +enjoy 358 +need 359 +rest 360 +use 361 +kids 362 +hollywood 363 +low 364 +production 365 +until 366 +along 367 +full 368 +friends 369 +camera 370 +truly 371 +women 372 +awful 373 +video 374 +next 375 +tell 376 +remember 377 +couple 378 +stupid 379 +start 380 +stars 381 +perhaps 382 +sex 383 +mean 384 +came 385 +recommend 386 +let 387 +moments 388 +wonderful 389 +episode 390 +understand 391 +small 392 +face 393 +terrible 394 +playing 395 +school 396 +getting 397 +written 398 +doing 399 +often 400 +keep 401 +early 402 +name 403 +perfect 404 +style 405 +human 406 +definitely 407 +gives 408 +others 409 +itself 410 +lines 411 +live 412 +become 413 +dialogue 414 +person 415 +lost 416 +finally 417 +piece 418 +head 419 +case 420 +felt 421 +yes 422 +liked 423 +supposed 424 +title 425 +couldn't 426 +absolutely 427 +white 428 +against 429 +boy 430 +picture 431 +sort 432 +worse 433 +certainly 434 +went 435 +entire 436 +waste 437 +cinema 438 +problem 439 +hope 440 +entertaining 441 +she's 442 +mr 443 +overall 444 +evil 445 +called 446 +loved 447 +based 448 +oh 449 +several 450 +fans 451 +mother 452 +drama 453 +beginning 454 +killer 455 +lives 456 +5 457 +direction 458 +care 459 +already 460 +becomes 461 +laugh 462 +example 463 +friend 464 +dark 465 +despite 466 +under 467 +seemed 468 +throughout 469 +4 470 +turn 471 +unfortunately 472 +wanted 473 +i'd 474 +– 475 +children 476 +final 477 +fine 478 +history 479 +amazing 480 +sound 481 +guess 482 +heart 483 +totally 484 +lead 485 +humor 486 +writing 487 +michael 488 +quality 489 +you'll 490 +close 491 +son 492 +guys 493 +wants 494 +works 495 +behind 496 +tries 497 +art 498 +side 499 +game 500 +past 501 +able 502 +b 503 +days 504 +turns 505 +child 506 +they're 507 +hand 508 +flick 509 +enjoyed 510 +act 511 +genre 512 +town 513 +favorite 514 +soon 515 +kill 516 +starts 517 +sometimes 518 +car 519 +gave 520 +run 521 +late 522 +eyes 523 +actress 524 +etc 525 +directed 526 +horrible 527 +won't 528 +viewer 529 +brilliant 530 +parts 531 +self 532 +themselves 533 +hour 534 +expect 535 +thinking 536 +stories 537 +stuff 538 +girls 539 +obviously 540 +blood 541 +decent 542 +city 543 +voice 544 +highly 545 +myself 546 +feeling 547 +fight 548 +except 549 +slow 550 +matter 551 +type 552 +anyway 553 +kid 554 +roles 555 +killed 556 +heard 557 +god 558 +age 559 +says 560 +moment 561 +took 562 +leave 563 +writer 564 +strong 565 +cannot 566 +violence 567 +police 568 +hit 569 +stop 570 +happens 571 +particularly 572 +known 573 +involved 574 +happened 575 +extremely 576 +daughter 577 +obvious 578 +told 579 +chance 580 +living 581 +coming 582 +lack 583 +alone 584 +experience 585 +wouldn't 586 +including 587 +murder 588 +attempt 589 +s 590 +please 591 +james 592 +happen 593 +wonder 594 +crap 595 +ago 596 +brother 597 +film's 598 +gore 599 +none 600 +complete 601 +interest 602 +score 603 +group 604 +cut 605 +simple 606 +save 607 +ok 608 +hell 609 +looked 610 +career 611 +number 612 +song 613 +possible 614 +seriously 615 +annoying 616 +shown 617 +exactly 618 +sad 619 +running 620 +musical 621 +serious 622 +taken 623 +yourself 624 +whose 625 +released 626 +cinematography 627 +david 628 +scary 629 +ends 630 +english 631 +hero 632 +usually 633 +hours 634 +reality 635 +opening 636 +i'll 637 +across 638 +today 639 +jokes 640 +light 641 +hilarious 642 +somewhat 643 +usual 644 +started 645 +cool 646 +ridiculous 647 +body 648 +relationship 649 +view 650 +level 651 +opinion 652 +change 653 +happy 654 +middle 655 +taking 656 +wish 657 +husband 658 +finds 659 +saying 660 +order 661 +talking 662 +ones 663 +documentary 664 +shots 665 +huge 666 +novel 667 +female 668 +mostly 669 +robert 670 +power 671 +episodes 672 +room 673 +important 674 +rating 675 +talent 676 +five 677 +major 678 +turned 679 +strange 680 +word 681 +modern 682 +call 683 +apparently 684 +disappointed 685 +single 686 +events 687 +due 688 +four 689 +songs 690 +basically 691 +attention 692 +7 693 +knows 694 +clearly 695 +supporting 696 +knew 697 +british 698 +television 699 +comic 700 +non 701 +fast 702 +earth 703 +country 704 +future 705 +cheap 706 +class 707 +thriller 708 +8 709 +silly 710 +king 711 +problems 712 +aren't 713 +easily 714 +words 715 +tells 716 +miss 717 +jack 718 +local 719 +sequence 720 +bring 721 +entertainment 722 +paul 723 +beyond 724 +upon 725 +whether 726 +predictable 727 +moving 728 +similar 729 +straight 730 +romantic 731 +sets 732 +review 733 +falls 734 +oscar 735 +mystery 736 +enjoyable 737 +needs 738 +appears 739 +talk 740 +rock 741 +george 742 +giving 743 +eye 744 +richard 745 +within 746 +ten 747 +animation 748 +message 749 +theater 750 +near 751 +above 752 +dull 753 +nearly 754 +sequel 755 +theme 756 +points 757 +' 758 +stand 759 +mention 760 +lady 761 +bunch 762 +add 763 +feels 764 +herself 765 +release 766 +red 767 +team 768 +storyline 769 +surprised 770 +ways 771 +using 772 +named 773 +haven't 774 +lots 775 +easy 776 +fantastic 777 +begins 778 +actual 779 +working 780 +effort 781 +york 782 +die 783 +hate 784 +french 785 +minute 786 +tale 787 +clear 788 +stay 789 +9 790 +elements 791 +feature 792 +among 793 +follow 794 +comments 795 +re 796 +viewers 797 +avoid 798 +sister 799 +showing 800 +typical 801 +editing 802 +what's 803 +famous 804 +tried 805 +sorry 806 +dialog 807 +check 808 +fall 809 +period 810 +season 811 +form 812 +certain 813 +filmed 814 +weak 815 +soundtrack 816 +means 817 +buy 818 +material 819 +somehow 820 +realistic 821 +figure 822 +crime 823 +doubt 824 +gone 825 +peter 826 +tom 827 +kept 828 +viewing 829 +t 830 +general 831 +leads 832 +greatest 833 +space 834 +lame 835 +suspense 836 +dance 837 +imagine 838 +brought 839 +third 840 +atmosphere 841 +hear 842 +particular 843 +sequences 844 +whatever 845 +parents 846 +move 847 +lee 848 +indeed 849 +learn 850 +rent 851 +de 852 +eventually 853 +note 854 +deal 855 +average 856 +reviews 857 +wait 858 +forget 859 +japanese 860 +sexual 861 +poorly 862 +premise 863 +okay 864 +zombie 865 +surprise 866 +believable 867 +stage 868 +possibly 869 +sit 870 +who's 871 +decided 872 +expected 873 +you've 874 +subject 875 +nature 876 +became 877 +difficult 878 +free 879 +killing 880 +screenplay 881 +truth 882 +romance 883 +dr 884 +nor 885 +reading 886 +needed 887 +question 888 +leaves 889 +street 890 +20 891 +meets 892 +hot 893 +unless 894 +begin 895 +baby 896 +superb 897 +credits 898 +imdb 899 +otherwise 900 +write 901 +shame 902 +let's 903 +situation 904 +dramatic 905 +memorable 906 +directors 907 +earlier 908 +meet 909 +disney 910 +open 911 +dog 912 +badly 913 +joe 914 +male 915 +weird 916 +acted 917 +forced 918 +laughs 919 +sci 920 +emotional 921 +older 922 +realize 923 +fi 924 +dream 925 +society 926 +writers 927 +interested 928 +footage 929 +forward 930 +comment 931 +crazy 932 +deep 933 +sounds 934 +plus 935 +beauty 936 +whom 937 +america 938 +fantasy 939 +directing 940 +keeps 941 +ask 942 +development 943 +features 944 +air 945 +quickly 946 +mess 947 +creepy 948 +towards 949 +perfectly 950 +mark 951 +worked 952 +box 953 +cheesy 954 +unique 955 +setting 956 +hands 957 +plenty 958 +result 959 +previous 960 +brings 961 +effect 962 +e 963 +total 964 +personal 965 +incredibly 966 +rate 967 +fire 968 +monster 969 +business 970 +leading 971 +apart 972 +casting 973 +admit 974 +joke 975 +powerful 976 +appear 977 +background 978 +telling 979 +girlfriend 980 +meant 981 +christmas 982 +hardly 983 +present 984 +battle 985 +potential 986 +create 987 +bill 988 +break 989 +pay 990 +masterpiece 991 +gay 992 +political 993 +return 994 +dumb 995 +fails 996 +fighting 997 +various 998 +era 999 +portrayed 1000 +co 1001 +cop 1002 +secret 1003 +inside 1004 +outside 1005 +nudity 1006 +reasons 1007 +ideas 1008 +twist 1009 +western 1010 +front 1011 +missing 1012 +boys 1013 +match 1014 +deserves 1015 +jane 1016 +expecting 1017 +fairly 1018 +villain 1019 +talented 1020 +married 1021 +ben 1022 +success 1023 +william 1024 +unlike 1025 +rich 1026 +attempts 1027 +spoilers 1028 +list 1029 +manages 1030 +social 1031 +odd 1032 +recently 1033 +remake 1034 +flat 1035 +cute 1036 +further 1037 +sadly 1038 +copy 1039 +wrote 1040 +agree 1041 +doctor 1042 +cold 1043 +plain 1044 +following 1045 +mentioned 1046 +sweet 1047 +incredible 1048 +missed 1049 +pure 1050 +crew 1051 +office 1052 +wasted 1053 +ended 1054 +produced 1055 +gun 1056 +filmmakers 1057 +large 1058 +caught 1059 +revenge 1060 +filled 1061 +pace 1062 +popular 1063 +waiting 1064 +'the 1065 +members 1066 +science 1067 +decides 1068 +considering 1069 +hold 1070 +public 1071 +cartoon 1072 +party 1073 +tension 1074 +created 1075 +slightly 1076 +uses 1077 +convincing 1078 +compared 1079 +la 1080 +familiar 1081 +neither 1082 +mary 1083 +spent 1084 +sees 1085 +6 1086 +suddenly 1087 +30 1088 +intelligent 1089 +escape 1090 +scott 1091 +fear 1092 +water 1093 +brothers 1094 +d 1095 +clever 1096 +entirely 1097 +kills 1098 +choice 1099 +bored 1100 +language 1101 +moves 1102 +spirit 1103 +laughing 1104 +dancing 1105 +we're 1106 +value 1107 +cover 1108 +credit 1109 +state 1110 +island 1111 +successful 1112 +trouble 1113 +visual 1114 +violent 1115 +ultimately 1116 +century 1117 +singing 1118 +15 1119 +concept 1120 +basic 1121 +italian 1122 +positive 1123 +german 1124 +animated 1125 +biggest 1126 +exciting 1127 +speak 1128 +runs 1129 +store 1130 +died 1131 +cat 1132 +consider 1133 +effective 1134 +walk 1135 +recent 1136 +depth 1137 +former 1138 +amusing 1139 +control 1140 +common 1141 +spend 1142 +band 1143 +appreciate 1144 +zombies 1145 +portrayal 1146 +force 1147 +c 1148 +pointless 1149 +rated 1150 +books 1151 +focus 1152 +hair 1153 +adventure 1154 +younger 1155 +solid 1156 +trash 1157 +adult 1158 +impressive 1159 +follows 1160 +respect 1161 +bizarre 1162 +tone 1163 +law 1164 +super 1165 +amount 1166 +impossible 1167 +mad 1168 +company 1169 +college 1170 +van 1171 +prison 1172 +weren't 1173 +conclusion 1174 +chemistry 1175 +win 1176 +showed 1177 +recommended 1178 +slasher 1179 +producers 1180 +culture 1181 +studio 1182 +fit 1183 +starring 1184 +heavy 1185 +situations 1186 +project 1187 +makers 1188 +trip 1189 +awesome 1190 +accent 1191 +considered 1192 +disturbing 1193 +changed 1194 +sick 1195 +failed 1196 +decide 1197 +somewhere 1198 +won 1199 +leaving 1200 +barely 1201 +honest 1202 +cause 1203 +questions 1204 +shooting 1205 +u 1206 +longer 1207 +post 1208 +f 1209 +anti 1210 +tough 1211 +aside 1212 +ghost 1213 +fake 1214 +cult 1215 +thanks 1216 +meaning 1217 +images 1218 +fiction 1219 +charming 1220 +audiences 1221 +computer 1222 +tony 1223 +brain 1224 +planet 1225 +south 1226 +literally 1227 +generally 1228 +touch 1229 +steve 1230 +stick 1231 +likes 1232 +ex 1233 +values 1234 +pathetic 1235 +magic 1236 +involving 1237 +surprisingly 1238 +alive 1239 +jim 1240 +immediately 1241 +grade 1242 +yeah 1243 +garbage 1244 +100 1245 +dad 1246 +bought 1247 +military 1248 +natural 1249 +camp 1250 +aspect 1251 +honestly 1252 +adaptation 1253 +utterly 1254 +detective 1255 +ability 1256 +fair 1257 +shoot 1258 +smith 1259 +explain 1260 +pick 1261 +genius 1262 +west 1263 +glad 1264 +frank 1265 +sitting 1266 +appearance 1267 +pictures 1268 +week 1269 +motion 1270 +appeal 1271 +army 1272 +standard 1273 +attack 1274 +knowing 1275 +personally 1276 +catch 1277 +drive 1278 +sexy 1279 +normal 1280 +rare 1281 +nowhere 1282 +added 1283 +sam 1284 +humour 1285 +walking 1286 +remains 1287 +purpose 1288 +edge 1289 +comedies 1290 +thinks 1291 +loud 1292 +beautifully 1293 +thank 1294 +silent 1295 +taste 1296 +unbelievable 1297 +naked 1298 +twists 1299 +master 1300 +touching 1301 +subtle 1302 +terms 1303 +date 1304 +equally 1305 +dreams 1306 +terrific 1307 +channel 1308 +drawn 1309 +mood 1310 +journey 1311 +door 1312 +chase 1313 +fully 1314 +complex 1315 +london 1316 +key 1317 +wow 1318 +managed 1319 +road 1320 +narrative 1321 +laughable 1322 +mistake 1323 +bottom 1324 +producer 1325 +themes 1326 +movie's 1327 +pieces 1328 +likely 1329 +climax 1330 +g 1331 +disappointing 1332 +club 1333 +lovely 1334 +harry 1335 +blue 1336 +nobody 1337 +excuse 1338 +outstanding 1339 +soldiers 1340 +issues 1341 +stewart 1342 +constantly 1343 +award 1344 +pass 1345 +thus 1346 +plan 1347 +surely 1348 +marriage 1349 +painful 1350 +justice 1351 +costumes 1352 +presented 1353 +batman 1354 +80's 1355 +innocent 1356 +soul 1357 +wild 1358 +noir 1359 +cinematic 1360 +spoiler 1361 +vampire 1362 +finish 1363 +slowly 1364 +ride 1365 +gang 1366 +contains 1367 +christopher 1368 +presence 1369 +places 1370 +besides 1371 +government 1372 +details 1373 +train 1374 +central 1375 +thrown 1376 +manner 1377 +chris 1378 +historical 1379 +stunning 1380 +photography 1381 +charm 1382 +hoping 1383 +impression 1384 +scenery 1385 +speaking 1386 +disappointment 1387 +loves 1388 +animals 1389 +you'd 1390 +developed 1391 +drug 1392 +smart 1393 +charles 1394 +indian 1395 +numbers 1396 +mysterious 1397 +expectations 1398 +color 1399 +hey 1400 +exception 1401 +throw 1402 +minor 1403 +ahead 1404 +double 1405 +track 1406 +stands 1407 +suppose 1408 +aspects 1409 +boss 1410 +woods 1411 +sent 1412 +festival 1413 +bother 1414 +cry 1415 +church 1416 +feelings 1417 +critics 1418 +green 1419 +brief 1420 +acts 1421 +opera 1422 +filming 1423 +mainly 1424 +support 1425 +emotion 1426 +element 1427 +held 1428 +fascinating 1429 +building 1430 +million 1431 +boyfriend 1432 +names 1433 +opportunity 1434 +serial 1435 +intended 1436 +forever 1437 +emotions 1438 +available 1439 +victim 1440 +charlie 1441 +dies 1442 +changes 1443 +compelling 1444 +bed 1445 +six 1446 +born 1447 +happening 1448 +bar 1449 +paris 1450 +likable 1451 +lived 1452 +twice 1453 +falling 1454 +hotel 1455 +zero 1456 +puts 1457 +tired 1458 +image 1459 +pain 1460 +lover 1461 +everybody 1462 +giant 1463 +offer 1464 +shock 1465 +spot 1466 +suggest 1467 +j 1468 +henry 1469 +include 1470 +confused 1471 +trailer 1472 +adults 1473 +difference 1474 +student 1475 +fresh 1476 +followed 1477 +bruce 1478 +r 1479 +kelly 1480 +hasn't 1481 +appeared 1482 +approach 1483 +victims 1484 +christian 1485 +fellow 1486 +hurt 1487 +impact 1488 +putting 1489 +gorgeous 1490 +step 1491 +sub 1492 +mix 1493 +event 1494 +notice 1495 +murders 1496 +share 1497 +laughed 1498 +confusing 1499 +content 1500 +mediocre 1501 +11 1502 +lacks 1503 +direct 1504 +supposedly 1505 +summer 1506 +actresses 1507 +flaws 1508 +porn 1509 +system 1510 +page 1511 +holes 1512 +wall 1513 +billy 1514 +moral 1515 +jerry 1516 +worthy 1517 +creative 1518 +relationships 1519 +rape 1520 +tragedy 1521 +race 1522 +thin 1523 +lighting 1524 +helps 1525 +random 1526 +answer 1527 +gem 1528 +funniest 1529 +ii 1530 +americans 1531 +jones 1532 +merely 1533 +proves 1534 +wondering 1535 +alien 1536 +students 1537 +ray 1538 +paid 1539 +al 1540 +land 1541 +seven 1542 +damn 1543 +agent 1544 +delivers 1545 +imagination 1546 +park 1547 +childhood 1548 +flying 1549 +hospital 1550 +forgotten 1551 +90 1552 +standards 1553 +flicks 1554 +impressed 1555 +finding 1556 +absolute 1557 +ugly 1558 +beat 1559 +jean 1560 +don 1561 +thoroughly 1562 +ms 1563 +attractive 1564 +ground 1565 +negative 1566 +wise 1567 +provides 1568 +latter 1569 +50 1570 +stuck 1571 +extreme 1572 +seemingly 1573 +seconds 1574 +becoming 1575 +winning 1576 +addition 1577 +reminded 1578 +tragic 1579 +offers 1580 +inspired 1581 +count 1582 +fell 1583 +thats 1584 +lose 1585 +affair 1586 +turning 1587 +folks 1588 +detail 1589 +faces 1590 +cliché 1591 +design 1592 +martin 1593 +collection 1594 +afraid 1595 +intense 1596 +fashion 1597 +pull 1598 +hidden 1599 +industry 1600 +man's 1601 +allen 1602 +apartment 1603 +o 1604 +quick 1605 +nasty 1606 +arthur 1607 +adds 1608 +area 1609 +rented 1610 +alan 1611 +angry 1612 +personality 1613 +artistic 1614 +length 1615 +shouldn't 1616 +therefore 1617 +information 1618 +chinese 1619 +brian 1620 +shocking 1621 +location 1622 +ready 1623 +professional 1624 +lets 1625 +animal 1626 +anymore 1627 +games 1628 +teen 1629 +states 1630 +soldier 1631 +listen 1632 +mom 1633 +describe 1634 +lord 1635 +news 1636 +picked 1637 +led 1638 +wooden 1639 +favourite 1640 +dirty 1641 +mouth 1642 +asks 1643 +food 1644 +deliver 1645 +onto 1646 +martial 1647 +bond 1648 +clothes 1649 +wars 1650 +struggle 1651 +queen 1652 +redeeming 1653 +stone 1654 +jason 1655 +scientist 1656 +p 1657 +wearing 1658 +ed 1659 +stephen 1660 +compare 1661 +castle 1662 +intelligence 1663 +creature 1664 +cross 1665 +sleep 1666 +teenage 1667 +allowed 1668 +wonderfully 1669 +necessary 1670 +carry 1671 +drugs 1672 +40 1673 +tears 1674 +fox 1675 +criminal 1676 +rip 1677 +helped 1678 +member 1679 +desperate 1680 +moved 1681 +sight 1682 +cgi 1683 +trust 1684 +deeply 1685 +roll 1686 +includes 1687 +willing 1688 +whatsoever 1689 +disaster 1690 +12 1691 +machine 1692 +ship 1693 +treat 1694 +began 1695 +mid 1696 +uncle 1697 +grace 1698 +phone 1699 +70's 1700 +williams 1701 +commentary 1702 +build 1703 +accident 1704 +captain 1705 +realized 1706 +plane 1707 +energy 1708 +station 1709 +warning 1710 +epic 1711 +davis 1712 +rarely 1713 +humans 1714 +loving 1715 +theatre 1716 +comedic 1717 +witch 1718 +pop 1719 +suicide 1720 +dying 1721 +powers 1722 +filmmaker 1723 +independent 1724 +introduced 1725 +nightmare 1726 +extra 1727 +engaging 1728 +actions 1729 +character's 1730 +superior 1731 +unusual 1732 +arts 1733 +apparent 1734 +suit 1735 +religious 1736 +heroes 1737 +danny 1738 +remarkable 1739 +artist 1740 +allow 1741 +pleasure 1742 +continue 1743 +unnecessary 1744 +x 1745 +ring 1746 +returns 1747 +physical 1748 +sky 1749 +teacher 1750 +pre 1751 +mental 1752 +watchable 1753 +provide 1754 +absurd 1755 +tim 1756 +memory 1757 +grand 1758 +technical 1759 +normally 1760 +wedding 1761 +desire 1762 +limited 1763 +anywhere 1764 +scared 1765 +russian 1766 +surprising 1767 +douglas 1768 +finished 1769 +brutal 1770 +skip 1771 +vision 1772 +process 1773 +intriguing 1774 +bloody 1775 +media 1776 +holds 1777 +exist 1778 +accept 1779 +nicely 1780 +suspect 1781 +000 1782 +jump 1783 +twenty 1784 +paced 1785 +wanting 1786 +search 1787 +cops 1788 +torture 1789 +growing 1790 +reminds 1791 +jr 1792 +according 1793 +pacing 1794 +legend 1795 +soft 1796 +passion 1797 +andy 1798 +player 1799 +hated 1800 +bits 1801 +fred 1802 +asked 1803 +faith 1804 +joy 1805 +johnny 1806 +clichés 1807 +jeff 1808 +academy 1809 +dressed 1810 +pilot 1811 +eddie 1812 +constant 1813 +anybody 1814 +ill 1815 +deserved 1816 +horse 1817 +gold 1818 +drunk 1819 +joan 1820 +blame 1821 +originally 1822 +explanation 1823 +dangerous 1824 +instance 1825 +smile 1826 +heaven 1827 +heads 1828 +sat 1829 +community 1830 +england 1831 +superman 1832 +deserve 1833 +issue 1834 +nonsense 1835 +met 1836 +dick 1837 +lies 1838 +capture 1839 +gotten 1840 +toward 1841 +kevin 1842 +somebody 1843 +soap 1844 +field 1845 +lovers 1846 +plots 1847 +taylor 1848 +mixed 1849 +players 1850 +nick 1851 +explained 1852 +record 1853 +fail 1854 +creating 1855 +vhs 1856 +knowledge 1857 +quiet 1858 +unknown 1859 +fights 1860 +starting 1861 +friendship 1862 +accurate 1863 +whilst 1864 +guns 1865 +price 1866 +adam 1867 +kate 1868 +hadn't 1869 +sucks 1870 +ball 1871 +river 1872 +floor 1873 +european 1874 +spanish 1875 +wide 1876 +cable 1877 +radio 1878 +fu 1879 +cars 1880 +jackson 1881 +realism 1882 +memories 1883 +moon 1884 +finest 1885 +heroine 1886 +aware 1887 +loose 1888 +eating 1889 +featuring 1890 +prince 1891 +lacking 1892 +responsible 1893 +saved 1894 +keeping 1895 +empty 1896 +understanding 1897 +japan 1898 +treated 1899 +eat 1900 +results 1901 +cuts 1902 +ice 1903 +bland 1904 +terribly 1905 +pulled 1906 +saving 1907 +below 1908 +officer 1909 +villains 1910 +candy 1911 +broken 1912 +sign 1913 +ladies 1914 +hopes 1915 +rubbish 1916 +delightful 1917 +vs 1918 +judge 1919 +witty 1920 +manage 1921 +fat 1922 +mine 1923 +gene 1924 +noticed 1925 +included 1926 +bright 1927 +months 1928 +forces 1929 +screaming 1930 +higher 1931 +kinda 1932 +wind 1933 +tarzan 1934 +cage 1935 +hits 1936 +loss 1937 +today's 1938 +monsters 1939 +youth 1940 +sing 1941 +numerous 1942 +partner 1943 +conflict 1944 +whenever 1945 +humanity 1946 +concerned 1947 +pretentious 1948 +fate 1949 +singer 1950 +dealing 1951 +mike 1952 +driving 1953 +jesus 1954 +private 1955 +talents 1956 +discovered 1957 +naturally 1958 +skills 1959 +unfunny 1960 +opposite 1961 +finale 1962 +bigger 1963 +v 1964 +ann 1965 +international 1966 +dated 1967 +kick 1968 +ups 1969 +prove 1970 +perspective 1971 +morning 1972 +mission 1973 +discover 1974 +portray 1975 +blonde 1976 +here's 1977 +loses 1978 +locations 1979 +visit 1980 +ordinary 1981 +bank 1982 +m 1983 +humorous 1984 +werewolf 1985 +streets 1986 +psychological 1987 +regular 1988 +reviewers 1989 +received 1990 +kong 1991 +w 1992 +edited 1993 +gags 1994 +ass 1995 +luck 1996 +curious 1997 +gary 1998 +continues 1999 +magnificent 2000 +13 2001 +we've 2002 +behavior 2003 +captured 2004 +jimmy 2005 +satire 2006 +survive 2007 +context 2008 +visually 2009 +breaks 2010 +existence 2011 +shallow 2012 +opens 2013 +l 2014 +mrs 2015 +debut 2016 +advice 2017 +calls 2018 +sea 2019 +foot 2020 +morgan 2021 +shop 2022 +h 2023 +murdered 2024 +connection 2025 +core 2026 +essentially 2027 +current 2028 +revealed 2029 +director's 2030 +corny 2031 +remembered 2032 +deals 2033 +blind 2034 +frankly 2035 +occasionally 2036 +lesson 2037 +genuine 2038 +scream 2039 +traditional 2040 +they've 2041 +lucky 2042 +identity 2043 +dimensional 2044 +african 2045 +bob 2046 +anthony 2047 +efforts 2048 +sean 2049 +golden 2050 +learned 2051 +segment 2052 +stock 2053 +window 2054 +cameo 2055 +owner 2056 +visuals 2057 +versions 2058 +village 2059 +albert 2060 +develop 2061 +santa 2062 +formula 2063 +miles 2064 +keaton 2065 +one's 2066 +sucked 2067 +decade 2068 +buddy 2069 +genuinely 2070 +grown 2071 +references 2072 +suffering 2073 +boat 2074 +lewis 2075 +unexpected 2076 +favor 2077 +study 2078 +washington 2079 +allows 2080 +program 2081 +national 2082 +grew 2083 +80s 2084 +proved 2085 +meanwhile 2086 +overly 2087 +ages 2088 +board 2089 +standing 2090 +logic 2091 +desert 2092 +spectacular 2093 +awkward 2094 +ultimate 2095 +comparison 2096 +reaction 2097 +rob 2098 +sheer 2099 +jennifer 2100 +reach 2101 +thomas 2102 +unable 2103 +failure 2104 +brilliantly 2105 +travel 2106 +grant 2107 +ford 2108 +vampires 2109 +types 2110 +parody 2111 +gangster 2112 +devil 2113 +steal 2114 +brown 2115 +passed 2116 +sudden 2117 +stereotypes 2118 +sake 2119 +flesh 2120 +leader 2121 +frame 2122 +bear 2123 +strength 2124 +speed 2125 +creates 2126 +eric 2127 +awards 2128 +laughter 2129 +dan 2130 +technology 2131 +delivered 2132 +author 2133 +bet 2134 +kung 2135 +crappy 2136 +wood 2137 +site 2138 +broadway 2139 +insane 2140 +trek 2141 +executed 2142 +relief 2143 +lake 2144 +hitler 2145 +gonna 2146 +discovers 2147 +emotionally 2148 +painfully 2149 +dreadful 2150 +marie 2151 +utter 2152 +commercial 2153 +decision 2154 +code 2155 +steven 2156 +fault 2157 +anime 2158 +majority 2159 +anne 2160 +round 2161 +pair 2162 +robin 2163 +caused 2164 +bomb 2165 +families 2166 +psycho 2167 +driven 2168 +attitude 2169 +clean 2170 +built 2171 +gratuitous 2172 +harris 2173 +native 2174 +luke 2175 +entertained 2176 +graphic 2177 +ran 2178 +killers 2179 +meeting 2180 +test 2181 +simon 2182 +flashbacks 2183 +underrated 2184 +nevertheless 2185 +model 2186 +seasons 2187 +asian 2188 +foreign 2189 +hill 2190 +levels 2191 +obsessed 2192 +evening 2193 +feet 2194 +halloween 2195 +vehicle 2196 +barbara 2197 +relate 2198 +treatment 2199 +rise 2200 +practically 2201 +range 2202 +endless 2203 +freedom 2204 +costs 2205 +religion 2206 +gory 2207 +cash 2208 +described 2209 +wit 2210 +pleasant 2211 +aged 2212 +ancient 2213 +tape 2214 +reviewer 2215 +center 2216 +president 2217 +chosen 2218 +lynch 2219 +product 2220 +combination 2221 +send 2222 +fly 2223 +seat 2224 +sell 2225 +70s 2226 +irritating 2227 +exploitation 2228 +excited 2229 +stopped 2230 +hearing 2231 +rescue 2232 +fill 2233 +howard 2234 +portrays 2235 +gordon 2236 +assume 2237 +parker 2238 +classics 2239 +pity 2240 +0 2241 +produce 2242 +hunter 2243 +breaking 2244 +dry 2245 +fame 2246 +anna 2247 +generation 2248 +sheriff 2249 +capable 2250 +believes 2251 +handsome 2252 +theatrical 2253 +asking 2254 +sports 2255 +largely 2256 +choose 2257 +theaters 2258 +sympathetic 2259 +extras 2260 +proper 2261 +ruined 2262 +cares 2263 +contrived 2264 +portraying 2265 +drew 2266 +individual 2267 +embarrassing 2268 +rules 2269 +unrealistic 2270 +learns 2271 +warm 2272 +victor 2273 +daniel 2274 +marry 2275 +appealing 2276 +safe 2277 +dubbed 2278 +depressing 2279 +canadian 2280 +freddy 2281 +shakespeare 2282 +recall 2283 +chick 2284 +uk 2285 +winner 2286 +hearted 2287 +contrast 2288 +sequels 2289 +involves 2290 +par 2291 +woody 2292 +crowd 2293 +matters 2294 +k 2295 +correct 2296 +chief 2297 +costume 2298 +haunting 2299 +paper 2300 +research 2301 +vote 2302 +strongly 2303 +heck 2304 +nominated 2305 +grow 2306 +clue 2307 +claim 2308 +facts 2309 +eight 2310 +protagonist 2311 +matt 2312 +rose 2313 +evidence 2314 +joseph 2315 +appropriate 2316 +disgusting 2317 +excitement 2318 +football 2319 +lousy 2320 +germany 2321 +cost 2322 +france 2323 +saturday 2324 +priest 2325 +talks 2326 +substance 2327 +losing 2328 +patrick 2329 +destroy 2330 +circumstances 2331 +tedious 2332 +training 2333 +thoughts 2334 +hunt 2335 +market 2336 +scare 2337 +voices 2338 +promise 2339 +naive 2340 +bringing 2341 +amateurish 2342 +teenager 2343 +angel 2344 +walter 2345 +captures 2346 +convinced 2347 +hanging 2348 +satisfying 2349 +bodies 2350 +united 2351 +fits 2352 +tend 2353 +jackie 2354 +trilogy 2355 +roy 2356 +horribly 2357 +lower 2358 +asleep 2359 +virtually 2360 +baseball 2361 +robot 2362 +hopefully 2363 +rental 2364 +alex 2365 +com 2366 +factor 2367 +haunted 2368 +teenagers 2369 +hall 2370 +walks 2371 +spoil 2372 +creatures 2373 +amateur 2374 +relatively 2375 +steals 2376 +mask 2377 +welcome 2378 +cinderella 2379 +covered 2380 +ryan 2381 +danger 2382 +europe 2383 +insult 2384 +category 2385 +continuity 2386 +mini 2387 +unlikely 2388 +drag 2389 +sinatra 2390 +skin 2391 +contemporary 2392 +louis 2393 +semi 2394 +viewed 2395 +fare 2396 +north 2397 +influence 2398 +depicted 2399 +handled 2400 +target 2401 +oliver 2402 +offensive 2403 +hat 2404 +initial 2405 +nancy 2406 +scale 2407 +lawyer 2408 +tiny 2409 +cutting 2410 +unfortunate 2411 +holding 2412 +witness 2413 +shocked 2414 +africa 2415 +remain 2416 +believed 2417 +fool 2418 +inner 2419 +politics 2420 +hide 2421 +reporter 2422 +presents 2423 +section 2424 +movement 2425 +provided 2426 +surreal 2427 +promising 2428 +designed 2429 +makeup 2430 +max 2431 +qualities 2432 +liners 2433 +refreshing 2434 +australian 2435 +source 2436 +14 2437 +structure 2438 +closer 2439 +drop 2440 +forgettable 2441 +touches 2442 +welles 2443 +display 2444 +angles 2445 +pile 2446 +fairy 2447 +repeated 2448 +till 2449 +texas 2450 +wayne 2451 +claims 2452 +previously 2453 +faced 2454 +sharp 2455 +deaths 2456 +ruin 2457 +accents 2458 +surprises 2459 +universal 2460 +degree 2461 +focused 2462 +propaganda 2463 +plans 2464 +serves 2465 +speaks 2466 +supernatural 2467 +highlight 2468 +service 2469 +peace 2470 +chose 2471 +related 2472 +cartoons 2473 +adventures 2474 +erotic 2475 +25 2476 +roger 2477 +suffers 2478 +blow 2479 +weekend 2480 +sisters 2481 +granted 2482 +mainstream 2483 +latest 2484 +weeks 2485 +prime 2486 +crash 2487 +cant 2488 +professor 2489 +experiences 2490 +speech 2491 +print 2492 +lesbian 2493 +harsh 2494 +deadly 2495 +veteran 2496 +mistakes 2497 +edward 2498 +routine 2499 +whoever 2500 +notch 2501 +uninteresting 2502 +realizes 2503 +invisible 2504 +combined 2505 +sympathy 2506 +accidentally 2507 +kim 2508 +twisted 2509 +brave 2510 +colors 2511 +dollars 2512 +security 2513 +draw 2514 +dogs 2515 +nude 2516 +rain 2517 +universe 2518 +struggling 2519 +dozen 2520 +teens 2521 +convince 2522 +guilty 2523 +path 2524 +appreciated 2525 +atrocious 2526 +mountain 2527 +treasure 2528 +walked 2529 +columbo 2530 +irish 2531 +frightening 2532 +would've 2533 +committed 2534 +aliens 2535 +technically 2536 +recognize 2537 +cowboy 2538 +blah 2539 +birth 2540 +enter 2541 +gritty 2542 +enemy 2543 +aka 2544 +spy 2545 +changing 2546 +magical 2547 +anderson 2548 +princess 2549 +department 2550 +gas 2551 +occasional 2552 +friday 2553 +sword 2554 +directly 2555 +false 2556 +massive 2557 +surface 2558 +narration 2559 +legendary 2560 +featured 2561 +victoria 2562 +anger 2563 +offered 2564 +paint 2565 +performed 2566 +moore 2567 +explains 2568 +abuse 2569 +suspenseful 2570 +vietnam 2571 +kinds 2572 +terror 2573 +experienced 2574 +friendly 2575 +subtitles 2576 +reputation 2577 +crying 2578 +hong 2579 +sorts 2580 +passing 2581 +junk 2582 +beach 2583 +multiple 2584 +forest 2585 +stolen 2586 +everywhere 2587 +figures 2588 +forth 2589 +statement 2590 +exact 2591 +powell 2592 +variety 2593 +required 2594 +clark 2595 +reveal 2596 +donald 2597 +regret 2598 +conversation 2599 +prior 2600 +darkness 2601 +remotely 2602 +execution 2603 +theory 2604 +trapped 2605 +proud 2606 +belief 2607 +urban 2608 +russell 2609 +lonely 2610 +placed 2611 +downright 2612 +wilson 2613 +san 2614 +fictional 2615 +melodrama 2616 +spends 2617 +insight 2618 +court 2619 +effectively 2620 +listening 2621 +grave 2622 +express 2623 +demons 2624 +crude 2625 +figured 2626 +bothered 2627 +abandoned 2628 +scares 2629 +network 2630 +unconvincing 2631 +jobs 2632 +hired 2633 +revolution 2634 +favorites 2635 +jon 2636 +wear 2637 +minds 2638 +metal 2639 +worthwhile 2640 +emma 2641 +california 2642 +dean 2643 +buying 2644 +blockbuster 2645 +lifetime 2646 +bus 2647 +paying 2648 +pulls 2649 +account 2650 +angle 2651 +happiness 2652 +von 2653 +blown 2654 +afternoon 2655 +imagery 2656 +rights 2657 +driver 2658 +alright 2659 +rolling 2660 +matrix 2661 +mexican 2662 +productions 2663 +amazed 2664 +idiot 2665 +rings 2666 +cultural 2667 +status 2668 +delivery 2669 +thankfully 2670 +grim 2671 +reveals 2672 +rule 2673 +stayed 2674 +handed 2675 +alice 2676 +stays 2677 +scenario 2678 +focuses 2679 +ha 2680 +significant 2681 +quest 2682 +rough 2683 +starred 2684 +examples 2685 +julia 2686 +jungle 2687 +sir 2688 +indie 2689 +lights 2690 +mere 2691 +views 2692 +murphy 2693 +shadow 2694 +sarah 2695 +bore 2696 +con 2697 +teeth 2698 +heavily 2699 +mature 2700 +device 2701 +table 2702 +skill 2703 +interview 2704 +caine 2705 +tight 2706 +necessarily 2707 +he'd 2708 +ron 2709 +sunday 2710 +clichéd 2711 +suffer 2712 +mexico 2713 +china 2714 +achieve 2715 +spite 2716 +understood 2717 +format 2718 +artists 2719 +position 2720 +initially 2721 +closing 2722 +campy 2723 +desperately 2724 +bound 2725 +fabulous 2726 +dress 2727 +sensitive 2728 +mgm 2729 +destroyed 2730 +hip 2731 +complicated 2732 +burns 2733 +demon 2734 +summary 2735 +seek 2736 +faithful 2737 +forgot 2738 +sun 2739 +decades 2740 +breath 2741 +gross 2742 +pitt 2743 +bourne 2744 +ghosts 2745 +titanic 2746 +cruel 2747 +murderer 2748 +stereotypical 2749 +deeper 2750 +lisa 2751 +facial 2752 +renting 2753 +ignore 2754 +pregnant 2755 +league 2756 +answers 2757 +racist 2758 +un 2759 +helping 2760 +ludicrous 2761 +beloved 2762 +flashback 2763 +slapstick 2764 +sleeping 2765 +17 2766 +dude 2767 +cell 2768 +musicals 2769 +fourth 2770 +wing 2771 +intellectual 2772 +beast 2773 +sounded 2774 +settings 2775 +environment 2776 +suck 2777 +critical 2778 +drinking 2779 +nazi 2780 +reminiscent 2781 +brad 2782 +calling 2783 +lugosi 2784 +dragon 2785 +description 2786 +susan 2787 +prefer 2788 +amazingly 2789 +task 2790 +mildly 2791 +pacino 2792 +disbelief 2793 +encounter 2794 +regarding 2795 +larry 2796 +inept 2797 +greater 2798 +learning 2799 +arms 2800 +dennis 2801 +extraordinary 2802 +turkey 2803 +storytelling 2804 +funnier 2805 +julie 2806 +halfway 2807 +ain't 2808 +expert 2809 +base 2810 +criticism 2811 +quirky 2812 +father's 2813 +leslie 2814 +warned 2815 +cabin 2816 +flight 2817 +titles 2818 +criminals 2819 +johnson 2820 +raw 2821 +praise 2822 +depiction 2823 +screening 2824 +throwing 2825 +extent 2826 +expression 2827 +kiss 2828 +jail 2829 +studios 2830 +freeman 2831 +truck 2832 +convey 2833 +originality 2834 +chan 2835 +entertain 2836 +choices 2837 +spoof 2838 +notorious 2839 +tree 2840 +raised 2841 +touched 2842 +children's 2843 +rachel 2844 +punch 2845 +experiment 2846 +daughters 2847 +prepared 2848 +comical 2849 +spoken 2850 +people's 2851 +timing 2852 +india 2853 +headed 2854 +purely 2855 +could've 2856 +basis 2857 +hoffman 2858 +bollywood 2859 +chilling 2860 +michelle 2861 +underground 2862 +dollar 2863 +via 2864 +picks 2865 +lie 2866 +inspiration 2867 +novels 2868 +wave 2869 +elizabeth 2870 +introduction 2871 +weapons 2872 +trick 2873 +lazy 2874 +jessica 2875 +graphics 2876 +breathtaking 2877 +notable 2878 +stomach 2879 +succeeds 2880 +term 2881 +crafted 2882 +join 2883 +throws 2884 +handle 2885 +strangely 2886 +properly 2887 +toy 2888 +nowadays 2889 +christ 2890 +sidney 2891 +reference 2892 +adding 2893 +claire 2894 +serve 2895 +ratings 2896 +locked 2897 +honor 2898 +wears 2899 +sitcom 2900 +ted 2901 +authentic 2902 +foster 2903 +regard 2904 +everyday 2905 +causes 2906 +maria 2907 +provoking 2908 +charge 2909 +protect 2910 +lesser 2911 +hitchcock 2912 +caring 2913 +mouse 2914 +mirror 2915 +bat 2916 +fallen 2917 +carrying 2918 +bitter 2919 +jewish 2920 +established 2921 +pet 2922 +amongst 2923 +east 2924 +shut 2925 +guard 2926 +midnight 2927 +sleazy 2928 +southern 2929 +determined 2930 +ned 2931 +challenge 2932 +daily 2933 +obnoxious 2934 +nonetheless 2935 +cases 2936 +carried 2937 +carries 2938 +wins 2939 +alas 2940 +remote 2941 +embarrassed 2942 +gruesome 2943 +hole 2944 +2006 2945 +lane 2946 +attempting 2947 +westerns 2948 +escapes 2949 +sinister 2950 +confusion 2951 +nation 2952 +tales 2953 +ironic 2954 +tradition 2955 +interpretation 2956 +arrives 2957 +busy 2958 +replaced 2959 +risk 2960 +enjoying 2961 +sold 2962 +essential 2963 +needless 2964 +aunt 2965 +hardy 2966 +burt 2967 +goofy 2968 +mass 2969 +obsession 2970 +minded 2971 +balance 2972 +flow 2973 +clips 2974 +existent 2975 +successfully 2976 +legs 2977 +presentation 2978 +screenwriter 2979 +jumps 2980 +exists 2981 +attacked 2982 +blair 2983 +laid 2984 +mentally 2985 +bbc 2986 +seeking 2987 +raise 2988 +topic 2989 +oddly 2990 +warner 2991 +inspector 2992 +horrific 2993 +fortunately 2994 +shape 2995 +marvelous 2996 +usa 2997 +intentions 2998 +buck 2999 +retarded 3000 +madness 3001 +stupidity 3002 +stops 3003 +text 3004 +stylish 3005 +stanley 3006 +che 3007 +rival 3008 +served 3009 +workers 3010 +maker 3011 +sides 3012 +ashamed 3013 +shower 3014 +packed 3015 +comedian 3016 +thrilling 3017 +wwii 3018 +interviews 3019 +nine 3020 +laura 3021 +frequently 3022 +upper 3023 +mob 3024 +mansion 3025 +bridge 3026 +remind 3027 +tongue 3028 +navy 3029 +wanna 3030 +contain 3031 +albeit 3032 +intensity 3033 +attacks 3034 +vacation 3035 +thief 3036 +delight 3037 +manager 3038 +chair 3039 +sum 3040 +hence 3041 +80 3042 +cheese 3043 +drives 3044 +2001 3045 +expressions 3046 +struggles 3047 +flawed 3048 +poignant 3049 +angels 3050 +personalities 3051 +rogers 3052 +riding 3053 +revolves 3054 +refuses 3055 +adapted 3056 +opened 3057 +greatly 3058 +credibility 3059 +philip 3060 +cooper 3061 +glass 3062 +pitch 3063 +tracy 3064 +1950s 3065 +jay 3066 +torn 3067 +dinner 3068 +bette 3069 +18 3070 +cynical 3071 +upset 3072 +pool 3073 +sin 3074 +tour 3075 +2000 3076 +internet 3077 +suspects 3078 +advantage 3079 +lessons 3080 +warn 3081 +lion 3082 +overcome 3083 +credible 3084 +wishes 3085 +thousands 3086 +spin 3087 +miller 3088 +racism 3089 +90's 3090 +mindless 3091 +wealthy 3092 +innocence 3093 +tense 3094 +broke 3095 +bugs 3096 +happily 3097 +catholic 3098 +guessing 3099 +trial 3100 +lucy 3101 +hood 3102 +hundreds 3103 +trite 3104 +physically 3105 +thrillers 3106 +cook 3107 +fish 3108 +alike 3109 +dubbing 3110 +fbi 3111 +crisis 3112 +per 3113 +pride 3114 +succeed 3115 +controversial 3116 +suffered 3117 +reed 3118 +bag 3119 +technique 3120 +wasting 3121 +dislike 3122 +medical 3123 +sexuality 3124 +countries 3125 +perform 3126 +patient 3127 +stranger 3128 +enjoyment 3129 +corner 3130 +arm 3131 +glimpse 3132 +gripping 3133 +reunion 3134 +franchise 3135 +holmes 3136 +ensemble 3137 +separate 3138 +hundred 3139 +lincoln 3140 +60's 3141 +sings 3142 +noble 3143 +shines 3144 +whereas 3145 +tied 3146 +ourselves 3147 +uncomfortable 3148 +infamous 3149 +neat 3150 +atmospheric 3151 +millions 3152 +shorts 3153 +contact 3154 +card 3155 +hint 3156 +pack 3157 +courage 3158 +irony 3159 +exceptional 3160 +plastic 3161 +storm 3162 +drink 3163 +ralph 3164 +searching 3165 +oscars 3166 +scripts 3167 +connected 3168 +italy 3169 +proof 3170 +sandler 3171 +snow 3172 +lying 3173 +flash 3174 +nose 3175 +curse 3176 +helen 3177 +sentimental 3178 +mst3k 3179 +grey 3180 +aired 3181 +holiday 3182 +steps 3183 +hills 3184 +performers 3185 +letting 3186 +chasing 3187 +suggests 3188 +dancer 3189 +tune 3190 +meaningful 3191 +idiotic 3192 +knife 3193 +quote 3194 +weapon 3195 +plague 3196 +sons 3197 +entry 3198 +kurt 3199 +fortune 3200 +cameos 3201 +consists 3202 +perfection 3203 +lovable 3204 +hoped 3205 +troubled 3206 +thousand 3207 +hiding 3208 +develops 3209 +unforgettable 3210 +accepted 3211 +noted 3212 +portrait 3213 +dear 3214 +equal 3215 +bettie 3216 +assistant 3217 +stretch 3218 +woman's 3219 +saves 3220 +colorful 3221 +annoyed 3222 +larger 3223 +attraction 3224 +condition 3225 +miscast 3226 +chases 3227 +brooks 3228 +virgin 3229 +spots 3230 +basement 3231 +host 3232 +dialogs 3233 +shoots 3234 +gain 3235 +horses 3236 +guilt 3237 +protagonists 3238 +oil 3239 +terrifying 3240 +month 3241 +cousin 3242 +neighborhood 3243 +vincent 3244 +pg 3245 +belongs 3246 +stealing 3247 +16 3248 +nelson 3249 +worry 3250 +burning 3251 +concert 3252 +ad 3253 +zone 3254 +strip 3255 +appearing 3256 +worlds 3257 +object 3258 +split 3259 +repeat 3260 +hang 3261 +boredom 3262 +destruction 3263 +thirty 3264 +redemption 3265 +hunting 3266 +encounters 3267 +imaginative 3268 +expensive 3269 +eerie 3270 +cube 3271 +seagal 3272 +jake 3273 +pie 3274 +competent 3275 +homeless 3276 +concerns 3277 +andrew 3278 +flaw 3279 +closely 3280 +bo 3281 +ultra 3282 +factory 3283 +1st 3284 +multi 3285 +civil 3286 +dramas 3287 +gag 3288 +stunts 3289 +wake 3290 +guts 3291 +sends 3292 +60 3293 +sutherland 3294 +glory 3295 +knock 3296 +matthau 3297 +massacre 3298 +letter 3299 +elsewhere 3300 +achieved 3301 +dig 3302 +checking 3303 +widmark 3304 +hooked 3305 +complaint 3306 +neck 3307 +endearing 3308 +segments 3309 +shark 3310 +sullivan 3311 +rushed 3312 +virus 3313 +ripped 3314 +charisma 3315 +incoherent 3316 +dragged 3317 +beating 3318 +dentist 3319 +essence 3320 +bears 3321 +profound 3322 +library 3323 +weight 3324 +tear 3325 +crimes 3326 +arnold 3327 +dare 3328 +appearances 3329 +solve 3330 +trade 3331 +pat 3332 +24 3333 +stanwyck 3334 +colour 3335 +teach 3336 +dorothy 3337 +roberts 3338 +rocks 3339 +fest 3340 +spell 3341 +catherine 3342 +dealt 3343 +stan 3344 +fitting 3345 +hitting 3346 +striking 3347 +pro 3348 +2005 3349 +tribute 3350 +tricks 3351 +60s 3352 +battles 3353 +believing 3354 +briefly 3355 +countless 3356 +fashioned 3357 +loser 3358 +goal 3359 +gothic 3360 +noise 3361 +techniques 3362 +n 3363 +videos 3364 +health 3365 +thumbs 3366 +attempted 3367 +scientists 3368 +st 3369 +painting 3370 +baker 3371 +strikes 3372 +inspiring 3373 +huh 3374 +sexually 3375 +birthday 3376 +secretary 3377 +curtis 3378 +jeremy 3379 +covers 3380 +pointed 3381 +slight 3382 +specific 3383 +tea 3384 +hearts 3385 +unintentionally 3386 +denzel 3387 +horrendous 3388 +charismatic 3389 +silver 3390 +surrounded 3391 +surrounding 3392 +reactions 3393 +branagh 3394 +importance 3395 +rochester 3396 +admittedly 3397 +carefully 3398 +jerk 3399 +tons 3400 +hype 3401 +relevant 3402 +they'd 3403 +walls 3404 +stood 3405 +eyed 3406 +bible 3407 +corrupt 3408 +rush 3409 +stunt 3410 +revelation 3411 +smoking 3412 +magazine 3413 +lloyd 3414 +kicks 3415 +karloff 3416 +stronger 3417 +grows 3418 +mild 3419 +hamlet 3420 +represents 3421 +dawn 3422 +andrews 3423 +intention 3424 +easier 3425 +enters 3426 +spending 3427 +scooby 3428 +fired 3429 +killings 3430 +stated 3431 +chances 3432 +shall 3433 +brand 3434 +exercise 3435 +university 3436 +increasingly 3437 +row 3438 +disagree 3439 +cardboard 3440 +winter 3441 +comics 3442 +requires 3443 +dropped 3444 +associated 3445 +world's 3446 +chuck 3447 +iii 3448 +medium 3449 +bush 3450 +projects 3451 +bride 3452 +occurs 3453 +korean 3454 +inevitable 3455 +messages 3456 +brando 3457 +le 3458 +strike 3459 +poverty 3460 +forgive 3461 +performing 3462 +stiff 3463 +attached 3464 +drags 3465 +luckily 3466 +ian 3467 +identify 3468 +1970s 3469 +gift 3470 +bobby 3471 +acceptable 3472 +resolution 3473 +eva 3474 +typically 3475 +canada 3476 +guest 3477 +nuclear 3478 +elvis 3479 +toilet 3480 +strictly 3481 +vague 3482 +spike 3483 +contract 3484 +hire 3485 +1980s 3486 +thrills 3487 +selling 3488 +hudson 3489 +homage 3490 +lab 3491 +boll 3492 +mafia 3493 +depression 3494 +sophisticated 3495 +fifteen 3496 +disease 3497 +allowing 3498 +brilliance 3499 +investigation 3500 +continued 3501 +struck 3502 +insulting 3503 +worker 3504 +instantly 3505 +useless 3506 +breasts 3507 +barry 3508 +jesse 3509 +sally 3510 +afterwards 3511 +chaplin 3512 +britain 3513 +carter 3514 +executive 3515 +handful 3516 +importantly 3517 +godfather 3518 +estate 3519 +hanks 3520 +pleased 3521 +overlooked 3522 +evident 3523 +burn 3524 +gotta 3525 +wreck 3526 +nights 3527 +2002 3528 +beings 3529 +ego 3530 +kidnapped 3531 +presumably 3532 +competition 3533 +press 3534 +partly 3535 +digital 3536 +shining 3537 +commit 3538 +tremendous 3539 +raped 3540 +menacing 3541 +silence 3542 +talked 3543 +derek 3544 +worthless 3545 +jamie 3546 +realise 3547 +ambitious 3548 +meat 3549 +wondered 3550 +photographed 3551 +sacrifice 3552 +arrested 3553 +buried 3554 +burton 3555 +threatening 3556 +smooth 3557 +aforementioned 3558 +superbly 3559 +boxing 3560 +kane 3561 +flawless 3562 +regardless 3563 +fears 3564 +creation 3565 +shy 3566 +heat 3567 +highlights 3568 +savage 3569 +persona 3570 +frustrated 3571 +drivel 3572 +conspiracy 3573 +individuals 3574 +wonders 3575 +listed 3576 +appalling 3577 +doc 3578 +'s 3579 +spiritual 3580 +pushed 3581 +returning 3582 +jumping 3583 +elvira 3584 +cox 3585 +corpse 3586 +size 3587 +characterization 3588 +bullets 3589 +walken 3590 +generous 3591 +string 3592 +rex 3593 +doors 3594 +pleasantly 3595 +bucks 3596 +relative 3597 +45 3598 +outrageous 3599 +kudos 3600 +planning 3601 +ticket 3602 +achievement 3603 +accomplished 3604 +miserably 3605 +monkey 3606 +beaten 3607 +neighbor 3608 +distant 3609 +fatal 3610 +repetitive 3611 +accused 3612 +picking 3613 +ironically 3614 +consequences 3615 +curiosity 3616 +union 3617 +admire 3618 +guide 3619 +splendid 3620 +prevent 3621 +reynolds 3622 +border 3623 +attracted 3624 +butt 3625 +clues 3626 +trap 3627 +notes 3628 +chain 3629 +opposed 3630 +watches 3631 +samurai 3632 +shortly 3633 +heston 3634 +twin 3635 +cole 3636 +glover 3637 +slightest 3638 +response 3639 +beer 3640 +territory 3641 +spooky 3642 +diamond 3643 +rap 3644 +horrors 3645 +20th 3646 +cup 3647 +dire 3648 +spirited 3649 +melodramatic 3650 +lucas 3651 +flynn 3652 +los 3653 +piano 3654 +push 3655 +revealing 3656 +spoiled 3657 +uninspired 3658 +ritter 3659 +convoluted 3660 +pulling 3661 +ken 3662 +root 3663 +they'll 3664 +streisand 3665 +motivation 3666 +directorial 3667 +installment 3668 +precious 3669 +titled 3670 +logical 3671 +documentaries 3672 +spring 3673 +lacked 3674 +suits 3675 +tall 3676 +subplot 3677 +mate 3678 +timeless 3679 +hatred 3680 +throat 3681 +blows 3682 +jealous 3683 +creators 3684 +blank 3685 +farce 3686 +spielberg 3687 +slap 3688 +ward 3689 +carol 3690 +subsequent 3691 +cared 3692 +mile 3693 +exaggerated 3694 +duke 3695 +morality 3696 +liberal 3697 +francisco 3698 +indians 3699 +psychotic 3700 +overdone 3701 +psychiatrist 3702 +astaire 3703 +intrigued 3704 +jet 3705 +blob 3706 +50's 3707 +conceived 3708 +fx 3709 +neil 3710 +aimed 3711 +remaining 3712 +doo 3713 +ignored 3714 +elderly 3715 +reasonably 3716 +mitchell 3717 +failing 3718 +sole 3719 +obscure 3720 +drunken 3721 +minimal 3722 +temple 3723 +progress 3724 +fancy 3725 +captivating 3726 +repeatedly 3727 +wes 3728 +tunes 3729 +shoes 3730 +grandmother 3731 +cia 3732 +nurse 3733 +marks 3734 +notably 3735 +emily 3736 +soviet 3737 +shirt 3738 +explore 3739 +smoke 3740 +souls 3741 +pushing 3742 +argument 3743 +distance 3744 +warrior 3745 +outcome 3746 +reduced 3747 +loosely 3748 +scientific 3749 +goldberg 3750 +gradually 3751 +bleak 3752 +timothy 3753 +manhattan 3754 +idiots 3755 +restaurant 3756 +scripted 3757 +misses 3758 +explicit 3759 +providing 3760 +elaborate 3761 +poster 3762 +lou 3763 +dignity 3764 +carpenter 3765 +norman 3766 +rid 3767 +turner 3768 +show's 3769 +davies 3770 +draws 3771 +discussion 3772 +exposed 3773 +mel 3774 +sticks 3775 +kenneth 3776 +definite 3777 +darker 3778 +laurel 3779 +intent 3780 +1950's 3781 +returned 3782 +superhero 3783 +sloppy 3784 +cried 3785 +worried 3786 +childish 3787 +shadows 3788 +craig 3789 +cruise 3790 +hysterical 3791 +imagined 3792 +reasonable 3793 +editor 3794 +ah 3795 +birds 3796 +horrid 3797 +areas 3798 +wicked 3799 +gentle 3800 +wannabe 3801 +alexander 3802 +thick 3803 +contrary 3804 +joey 3805 +empire 3806 +connect 3807 +discovery 3808 +unbearable 3809 +tortured 3810 +screams 3811 +fever 3812 +unbelievably 3813 +1930s 3814 +disc 3815 +99 3816 +load 3817 +heroic 3818 +absence 3819 +reached 3820 +ho 3821 +choreography 3822 +triumph 3823 +complain 3824 +annie 3825 +broad 3826 +improved 3827 +concerning 3828 +brazil 3829 +movements 3830 +2003 3831 +2004 3832 +dave 3833 +folk 3834 +eve 3835 +purple 3836 +commercials 3837 +futuristic 3838 +vicious 3839 +gray 3840 +freak 3841 +threat 3842 +cusack 3843 +extended 3844 +citizen 3845 +stole 3846 +anyways 3847 +glenn 3848 +existed 3849 +cheek 3850 +broadcast 3851 +photographer 3852 +translation 3853 +arrive 3854 +differences 3855 +displays 3856 +critic 3857 +slave 3858 +landscape 3859 +occurred 3860 +builds 3861 +drawing 3862 +incident 3863 +warren 3864 +burned 3865 +involvement 3866 +styles 3867 +bathroom 3868 +machines 3869 +narrator 3870 +antics 3871 +he'll 3872 +fisher 3873 +swear 3874 +australia 3875 +matthew 3876 +resembles 3877 +lily 3878 +overrated 3879 +currently 3880 +symbolism 3881 +ought 3882 +bare 3883 +audio 3884 +web 3885 +farm 3886 +contained 3887 +greek 3888 +affected 3889 +blend 3890 +q 3891 +recognized 3892 +duo 3893 +genres 3894 +population 3895 +carrie 3896 +ranks 3897 +demands 3898 +we'll 3899 +abc 3900 +prom 3901 +altogether 3902 +superficial 3903 +kitchen 3904 +pseudo 3905 +sunshine 3906 +sadness 3907 +secrets 3908 +bone 3909 +website 3910 +receive 3911 +popcorn 3912 +threw 3913 +craft 3914 +enjoys 3915 +occur 3916 +twelve 3917 +block 3918 +girl's 3919 +proceedings 3920 +dynamic 3921 +daring 3922 +swedish 3923 +argue 3924 +bite 3925 +wolf 3926 +adequate 3927 +investigate 3928 +harder 3929 +ruth 3930 +ridiculously 3931 +tap 3932 +dinosaurs 3933 +hugh 3934 +synopsis 3935 +beats 3936 +carrey 3937 +explosion 3938 +foul 3939 +merit 3940 +suited 3941 +holy 3942 +staged 3943 +journalist 3944 +pretend 3945 +composed 3946 +cagney 3947 +robots 3948 +giallo 3949 +aging 3950 +fay 3951 +sadistic 3952 +engaged 3953 +escaped 3954 +juvenile 3955 +rambo 3956 +ireland 3957 +conversations 3958 +thugs 3959 +modesty 3960 +selfish 3961 +margaret 3962 +dialogues 3963 +ease 3964 +cameras 3965 +tame 3966 +leg 3967 +rural 3968 +comfortable 3969 +nazis 3970 +clothing 3971 +innovative 3972 +terry 3973 +thrill 3974 +2nd 3975 +dancers 3976 +brosnan 3977 +explosions 3978 +bin 3979 +rage 3980 +overwhelming 3981 +jazz 3982 +vivid 3983 +coherent 3984 +bullet 3985 +odds 3986 +mountains 3987 +kidding 3988 +versus 3989 +lit 3990 +offering 3991 +mother's 3992 +trio 3993 +newspaper 3994 +pulp 3995 +ellen 3996 +dawson 3997 +bird 3998 +buddies 3999 +combat 4000 +dracula 4001 +lol 4002 +grab 4003 +orders 4004 +staff 4005 +nearby 4006 +cats 4007 +wealth 4008 +unpleasant 4009 +staying 4010 +devoted 4011 +centered 4012 +errors 4013 +disturbed 4014 +bell 4015 +atlantis 4016 +snake 4017 +felix 4018 +damage 4019 +clint 4020 +lust 4021 +groups 4022 +banned 4023 +blowing 4024 +fighter 4025 +removed 4026 +react 4027 +conventional 4028 +kapoor 4029 +intrigue 4030 +possessed 4031 +cringe 4032 +eyre 4033 +liking 4034 +implausible 4035 +philosophy 4036 +producing 4037 +abilities 4038 +seventies 4039 +bang 4040 +murderous 4041 +deliberately 4042 +gandhi 4043 +tommy 4044 +meaningless 4045 +subjects 4046 +lips 4047 +ingredients 4048 +mildred 4049 +perry 4050 +warming 4051 +causing 4052 +possibility 4053 +detailed 4054 +walker 4055 +garden 4056 +prostitute 4057 +nightmares 4058 +cameron 4059 +flop 4060 +influenced 4061 +spare 4062 +unwatchable 4063 +undoubtedly 4064 +celluloid 4065 +relies 4066 +resemblance 4067 +neo 4068 +parent 4069 +falk 4070 +uneven 4071 +unintentional 4072 +eccentric 4073 +mistaken 4074 +distracting 4075 +careers 4076 +yesterday 4077 +forbidden 4078 +panic 4079 +crack 4080 +brains 4081 +highest 4082 +occasion 4083 +signs 4084 +focusing 4085 +hollow 4086 +explored 4087 +aid 4088 +cary 4089 +scheme 4090 +shine 4091 +it'll 4092 +kirk 4093 +bedroom 4094 +satisfied 4095 +rat 4096 +passes 4097 +survival 4098 +coffee 4099 +furthermore 4100 +primary 4101 +succeeded 4102 +politically 4103 +pays 4104 +apes 4105 +stiller 4106 +dating 4107 +defeat 4108 +sport 4109 +catches 4110 +mickey 4111 +clown 4112 +roman 4113 +discuss 4114 +karen 4115 +clumsy 4116 +chaos 4117 +financial 4118 +official 4119 +trees 4120 +explaining 4121 +models 4122 +spirits 4123 +carl 4124 +jeffrey 4125 +duty 4126 +whale 4127 +funeral 4128 +secondly 4129 +sentence 4130 +2007 4131 +classes 4132 +sidekick 4133 +tracks 4134 +props 4135 +travels 4136 +flies 4137 +remarkably 4138 +smaller 4139 +wallace 4140 +awake 4141 +1996 4142 +brady 4143 +blatant 4144 +decisions 4145 +afford 4146 +notion 4147 +recorded 4148 +glorious 4149 +enterprise 4150 +maggie 4151 +consistently 4152 +toys 4153 +offended 4154 +officers 4155 +danes 4156 +backdrop 4157 +beneath 4158 +masters 4159 +measure 4160 +endings 4161 +doomed 4162 +mysteries 4163 +lifestyle 4164 +houses 4165 +portion 4166 +primarily 4167 +satan 4168 +hates 4169 +devoid 4170 +impress 4171 +outer 4172 +generic 4173 +dutch 4174 +punk 4175 +lyrics 4176 +yellow 4177 +eastwood 4178 +exotic 4179 +represent 4180 +instant 4181 +desperation 4182 +mixture 4183 +settle 4184 +frustration 4185 +unfolds 4186 +goodness 4187 +wives 4188 +directs 4189 +fetched 4190 +ape 4191 +cheating 4192 +dozens 4193 +rebel 4194 +cuba 4195 +paulie 4196 +enormous 4197 +revolutionary 4198 +hints 4199 +shelf 4200 +brooklyn 4201 +florida 4202 +dances 4203 +motives 4204 +destiny 4205 +1999 4206 +donna 4207 +hardcore 4208 +mill 4209 +wrestling 4210 +subtlety 4211 +forty 4212 +describes 4213 +drops 4214 +blake 4215 +stinker 4216 +doll 4217 +painted 4218 +fond 4219 +linda 4220 +principal 4221 +rank 4222 +ideal 4223 +kennedy 4224 +hammer 4225 +montage 4226 +hollywood's 4227 +tie 4228 +disjointed 4229 +3rd 4230 +reaches 4231 +amy 4232 +immensely 4233 +ginger 4234 +judging 4235 +companion 4236 +communist 4237 +urge 4238 +winds 4239 +developing 4240 +trailers 4241 +cliff 4242 +lawrence 4243 +stellar 4244 +topless 4245 +circle 4246 +surviving 4247 +avoided 4248 +relations 4249 +bold 4250 +hideous 4251 +voight 4252 +closet 4253 +et 4254 +surfing 4255 +melting 4256 +soccer 4257 +edie 4258 +matches 4259 +backgrounds 4260 +planned 4261 +enemies 4262 +advance 4263 +bull 4264 +authority 4265 +crush 4266 +outfit 4267 +emphasis 4268 +method 4269 +terrorist 4270 +senseless 4271 +pig 4272 +uwe 4273 +simplistic 4274 +benefit 4275 +adorable 4276 +eighties 4277 +ruthless 4278 +godzilla 4279 +blew 4280 +countryside 4281 +specifically 4282 +wont 4283 +performer 4284 +hbo 4285 +traveling 4286 +todd 4287 +practice 4288 +diane 4289 +fix 4290 +faster 4291 +1980 4292 +commented 4293 +sh 4294 +loyal 4295 +saga 4296 +ties 4297 +disappear 4298 +awe 4299 +earned 4300 +buff 4301 +rick 4302 +loads 4303 +link 4304 +angeles 4305 +corruption 4306 +forms 4307 +menace 4308 +miserable 4309 +claimed 4310 +vast 4311 +coach 4312 +divorce 4313 +hal 4314 +gadget 4315 +chorus 4316 +limits 4317 +cure 4318 +introduces 4319 +cards 4320 +solo 4321 +blues 4322 +splatter 4323 +april 4324 +endure 4325 +riveting 4326 +dedicated 4327 +tender 4328 +winters 4329 +illogical 4330 +choreographed 4331 +disappeared 4332 +unsettling 4333 +waters 4334 +guessed 4335 +lemmon 4336 +involve 4337 +transformation 4338 +depressed 4339 +rooms 4340 +lasted 4341 +displayed 4342 +weakest 4343 +leonard 4344 +philosophical 4345 +racial 4346 +interaction 4347 +arrogant 4348 +tag 4349 +rocket 4350 +similarities 4351 +hurts 4352 +thoughtful 4353 +realizing 4354 +harvey 4355 +justify 4356 +hook 4357 +survivors 4358 +represented 4359 +pot 4360 +possibilities 4361 +wore 4362 +disappoint 4363 +voiced 4364 +kicked 4365 +abysmal 4366 +hamilton 4367 +buffs 4368 +safety 4369 +widow 4370 +ears 4371 +nomination 4372 +trashy 4373 +honesty 4374 +stereotype 4375 +severe 4376 +formulaic 4377 +moody 4378 +similarly 4379 +stress 4380 +pan 4381 +chased 4382 +isolated 4383 +blond 4384 +stinks 4385 +mario 4386 +passionate 4387 +finger 4388 +shirley 4389 +march 4390 +hank 4391 +improve 4392 +mann 4393 +understandable 4394 +characters' 4395 +considerable 4396 +scope 4397 +holly 4398 +diana 4399 +grasp 4400 +command 4401 +solely 4402 +'em 4403 +concern 4404 +treats 4405 +akshay 4406 +promised 4407 +colonel 4408 +jonathan 4409 +faults 4410 +helicopter 4411 +inventive 4412 +sounding 4413 +quotes 4414 +trained 4415 +switch 4416 +celebrity 4417 +tad 4418 +swimming 4419 +orson 4420 +education 4421 +aids 4422 +nail 4423 +judy 4424 +cg 4425 +user 4426 +nervous 4427 +nostalgic 4428 +daddy 4429 +alert 4430 +amanda 4431 +facing 4432 +comparing 4433 +unhappy 4434 +preview 4435 +report 4436 +bonus 4437 +purchase 4438 +chess 4439 +wet 4440 +lately 4441 +horrifying 4442 +agrees 4443 +thru 4444 +dolls 4445 +cinematographer 4446 +ignorant 4447 +species 4448 +seed 4449 +consistent 4450 +downhill 4451 +corporate 4452 +photos 4453 +confidence 4454 +letters 4455 +berlin 4456 +dinosaur 4457 +rotten 4458 +taught 4459 +fooled 4460 +laws 4461 +nicholson 4462 +namely 4463 +shake 4464 +waited 4465 +wished 4466 +embarrassment 4467 +everyone's 4468 +boot 4469 +pretending 4470 +reaching 4471 +someone's 4472 +transfer 4473 +sits 4474 +armed 4475 +del 4476 +dub 4477 +defend 4478 +hart 4479 +35 4480 +constructed 4481 +mall 4482 +poetic 4483 +motivations 4484 +inane 4485 +behave 4486 +tonight 4487 +staring 4488 +humble 4489 +snl 4490 +elephant 4491 +agents 4492 +oz 4493 +grandfather 4494 +writes 4495 +relation 4496 +hop 4497 +delivering 4498 +fonda 4499 +edgar 4500 +cave 4501 +artificial 4502 +grinch 4503 +sappy 4504 +prize 4505 +1972 4506 +useful 4507 +buildings 4508 +li 4509 +cake 4510 +eager 4511 +closest 4512 +suitable 4513 +raising 4514 +destroying 4515 +combine 4516 +beatty 4517 +pants 4518 +cleverly 4519 +ballet 4520 +convincingly 4521 +porno 4522 +1990 4523 +miike 4524 +affect 4525 +engage 4526 +cd 4527 +conservative 4528 +wound 4529 +arrived 4530 +stevens 4531 +alcoholic 4532 +valuable 4533 +ya 4534 +reads 4535 +scottish 4536 +elegant 4537 +vegas 4538 +chest 4539 +charlotte 4540 +climactic 4541 +tiresome 4542 +z 4543 +conflicts 4544 +babe 4545 +vengeance 4546 +square 4547 +bath 4548 +secretly 4549 +airport 4550 +campbell 4551 +kingdom 4552 +september 4553 +inferior 4554 +1968 4555 +latin 4556 +plant 4557 +button 4558 +museum 4559 +maintain 4560 +wrapped 4561 +kicking 4562 +cheated 4563 +global 4564 +robbery 4565 +virginia 4566 +wells 4567 +waves 4568 +stilted 4569 +blunt 4570 +lena 4571 +boom 4572 +access 4573 +raymond 4574 +1960s 4575 +catching 4576 +nicholas 4577 +yelling 4578 +scarecrow 4579 +beliefs 4580 +paranoia 4581 +christians 4582 +vice 4583 +jumped 4584 +lay 4585 +iron 4586 +steel 4587 +lowest 4588 +reflect 4589 +closed 4590 +mummy 4591 +transition 4592 +advertising 4593 +vulnerable 4594 +abusive 4595 +1970's 4596 +spoke 4597 +plight 4598 +mars 4599 +spread 4600 +adams 4601 +wizard 4602 +poetry 4603 +im 4604 +sandra 4605 +germans 4606 +pokemon 4607 +progresses 4608 +70 4609 +00 4610 +hung 4611 +questionable 4612 +remarks 4613 +airplane 4614 +centers 4615 +potentially 4616 +bottle 4617 +chicago 4618 +guarantee 4619 +couples 4620 +messed 4621 +catchy 4622 +slick 4623 +gangsters 4624 +misery 4625 +blade 4626 +designs 4627 +construction 4628 +ethan 4629 +desired 4630 +miracle 4631 +carradine 4632 +firstly 4633 +scores 4634 +wandering 4635 +greedy 4636 +recognition 4637 +understated 4638 +restored 4639 +complexity 4640 +madonna 4641 +attitudes 4642 +rendition 4643 +hunters 4644 +intentionally 4645 +experiments 4646 +ruby 4647 +alongside 4648 +vaguely 4649 +inappropriate 4650 +copies 4651 +operation 4652 +brutally 4653 +taxi 4654 +amounts 4655 +stooges 4656 +joined 4657 +pearl 4658 +demand 4659 +crocodile 4660 +depicts 4661 +purchased 4662 +acid 4663 +myers 4664 +exploration 4665 +advise 4666 +illegal 4667 +balls 4668 +king's 4669 +gundam 4670 +disney's 4671 +gender 4672 +lengthy 4673 +survived 4674 +hopper 4675 +niro 4676 +advanced 4677 +simplicity 4678 +bela 4679 +parallel 4680 +ocean 4681 +slaughter 4682 +rising 4683 +witnesses 4684 +chicks 4685 +streep 4686 +visible 4687 +nostalgia 4688 +arguably 4689 +careful 4690 +intimate 4691 +online 4692 +floating 4693 +rubber 4694 +june 4695 +illness 4696 +resources 4697 +khan 4698 +jaw 4699 +newly 4700 +witches 4701 +showcase 4702 +signed 4703 +opinions 4704 +dust 4705 +eaten 4706 +civilization 4707 +shelley 4708 +incomprehensible 4709 +invasion 4710 +lee's 4711 +monkeys 4712 +resort 4713 +literature 4714 +junior 4715 +likewise 4716 +homosexual 4717 +family's 4718 +viewings 4719 +sue 4720 +wisdom 4721 +matched 4722 +amitabh 4723 +edition 4724 +witnessed 4725 +visits 4726 +mistress 4727 +1983 4728 +demented 4729 +basketball 4730 +neighbors 4731 +macy 4732 +fascinated 4733 +dreary 4734 +suspicious 4735 +accompanied 4736 +worn 4737 +mail 4738 +challenging 4739 +doom 4740 +ensues 4741 +manipulative 4742 +robinson 4743 +classical 4744 +olivier 4745 +agreed 4746 +appreciation 4747 +franco 4748 +montana 4749 +troops 4750 +capturing 4751 +alternate 4752 +bands 4753 +twilight 4754 +ridden 4755 +responsibility 4756 +proceeds 4757 +chapter 4758 +jenny 4759 +prisoners 4760 +pops 4761 +analysis 4762 +subplots 4763 +lively 4764 +nuts 4765 +prisoner 4766 +incompetent 4767 +damon 4768 +sellers 4769 +mayor 4770 +rats 4771 +simpson 4772 +90s 4773 +persons 4774 +feed 4775 +descent 4776 +reel 4777 +bay 4778 +assault 4779 +losers 4780 +widely 4781 +rabbit 4782 +smiling 4783 +relatives 4784 +excessive 4785 +defined 4786 +satisfy 4787 +solution 4788 +legal 4789 +molly 4790 +arrival 4791 +overacting 4792 +equivalent 4793 +iran 4794 +pit 4795 +masterful 4796 +capital 4797 +richardson 4798 +compelled 4799 +plausible 4800 +stale 4801 +scrooge 4802 +cities 4803 +francis 4804 +enthusiasm 4805 +lone 4806 +parties 4807 +tomatoes 4808 +channels 4809 +hilariously 4810 +rocky 4811 +crucial 4812 +dropping 4813 +unit 4814 +waitress 4815 +domestic 4816 +attorney 4817 +bakshi 4818 +serving 4819 +wrap 4820 +jaws 4821 +historically 4822 +3d 4823 +defense 4824 +hello 4825 +greed 4826 +1973 4827 +priceless 4828 +sincere 4829 +warmth 4830 +paltrow 4831 +gerard 4832 +tends 4833 +god's 4834 +patients 4835 +creep 4836 +counter 4837 +dalton 4838 +kay 4839 +whats 4840 +louise 4841 +peoples 4842 +exceptionally 4843 +nyc 4844 +pal 4845 +seeks 4846 +terrorists 4847 +lumet 4848 +morris 4849 +ninja 4850 +randomly 4851 +frequent 4852 +despair 4853 +irrelevant 4854 +dressing 4855 +pursuit 4856 +prequel 4857 +creativity 4858 +imitation 4859 +bumbling 4860 +hyde 4861 +property 4862 +muslim 4863 +wishing 4864 +richards 4865 +bargain 4866 +50s 4867 +creator 4868 +calm 4869 +bacall 4870 +gabriel 4871 +mentioning 4872 +rangers 4873 +methods 4874 +earl 4875 +royal 4876 +butler 4877 +justin 4878 +psychic 4879 +chooses 4880 +belong 4881 +der 4882 +photo 4883 +polanski 4884 +mundane 4885 +specially 4886 +mighty 4887 +homer 4888 +ear 4889 +masterpieces 4890 +generated 4891 +leo 4892 +improvement 4893 +poem 4894 +ham 4895 +cliche 4896 +marty 4897 +caliber 4898 +mentions 4899 +minimum 4900 +showdown 4901 +borrowed 4902 +elm 4903 +icon 4904 +brenda 4905 +polished 4906 +1984 4907 +mechanical 4908 +overlook 4909 +loaded 4910 +map 4911 +recording 4912 +craven 4913 +tiger 4914 +roth 4915 +awfully 4916 +suffice 4917 +troubles 4918 +introduce 4919 +equipment 4920 +ashley 4921 +wendy 4922 +pamela 4923 +empathy 4924 +phantom 4925 +betty 4926 +resident 4927 +unreal 4928 +ruins 4929 +performs 4930 +promises 4931 +monk 4932 +iraq 4933 +hippie 4934 +purposes 4935 +marketing 4936 +angela 4937 +keith 4938 +sink 4939 +gifted 4940 +opportunities 4941 +garbo 4942 +assigned 4943 +feminist 4944 +household 4945 +wacky 4946 +alfred 4947 +absent 4948 +sneak 4949 +popularity 4950 +trail 4951 +inducing 4952 +moronic 4953 +wounded 4954 +receives 4955 +willis 4956 +unseen 4957 +stretched 4958 +fulci 4959 +unaware 4960 +dimension 4961 +dolph 4962 +definition 4963 +testament 4964 +educational 4965 +survivor 4966 +attend 4967 +clip 4968 +contest 4969 +petty 4970 +13th 4971 +christy 4972 +respected 4973 +resist 4974 +year's 4975 +album 4976 +expressed 4977 +randy 4978 +quit 4979 +phony 4980 +unoriginal 4981 +punishment 4982 +activities 4983 +suspend 4984 +rolled 4985 +eastern 4986 +1933 4987 +instinct 4988 +distinct 4989 +championship 4990 +tech 4991 +doubts 4992 +interests 4993 +exposure 4994 +travesty 4995 +israel 4996 +sixties 4997 +pink 4998 +orange 4999 +resulting 5000 +spain 5001 +bergman 5002 +1987 5003 +verhoeven 5004 +distribution 5005 +laughably 5006 +depicting 5007 +kissing 5008 +tooth 5009 +shed 5010 +kubrick 5011 +pin 5012 +nonsensical 5013 +roots 5014 +assumed 5015 +swim 5016 +whoopi 5017 +domino 5018 +heights 5019 +spock 5020 +inevitably 5021 +abraham 5022 +stunned 5023 +businessman 5024 +correctly 5025 +deceased 5026 +buffalo 5027 +wholly 5028 +underlying 5029 +dud 5030 +othello 5031 +unpredictable 5032 +package 5033 +hopeless 5034 +teaching 5035 +valley 5036 +uplifting 5037 +peters 5038 +integrity 5039 +1993 5040 +biography 5041 +yard 5042 +brutality 5043 +america's 5044 +trademark 5045 +retired 5046 +shaw 5047 +reflection 5048 +maniac 5049 +– 5050 +meryl 5051 +accuracy 5052 +sid 5053 +compassion 5054 +dreck 5055 +2008 5056 +edgy 5057 +greatness 5058 +assassin 5059 +greg 5060 +palace 5061 +suggested 5062 +patience 5063 +landscapes 5064 +1971 5065 +mankind 5066 +supported 5067 +merits 5068 +directions 5069 +fed 5070 +romero 5071 +spider 5072 +mtv 5073 +metaphor 5074 +masses 5075 +puppet 5076 +seldom 5077 +wife's 5078 +loyalty 5079 +deaf 5080 +grayson 5081 +strangers 5082 +3000 5083 +passable 5084 +checked 5085 +connery 5086 +confess 5087 +shaky 5088 +drake 5089 +eugene 5090 +significance 5091 +pierce 5092 +unfair 5093 +maid 5094 +indulgent 5095 +comfort 5096 +orleans 5097 +willie 5098 +glasses 5099 +pressure 5100 +alec 5101 +composer 5102 +marion 5103 +nicole 5104 +tribe 5105 +fought 5106 +technicolor 5107 +watson 5108 +dee 5109 +emperor 5110 +adaptations 5111 +romp 5112 +peak 5113 +conditions 5114 +grabs 5115 +exchange 5116 +fury 5117 +immediate 5118 +women's 5119 +timon 5120 +omen 5121 +generations 5122 +barrymore 5123 +resemble 5124 +1995 5125 +1997 5126 +confrontation 5127 +landing 5128 +frustrating 5129 +demise 5130 +spacey 5131 +lackluster 5132 +disliked 5133 +kyle 5134 +y 5135 +victory 5136 +wretched 5137 +… 5138 +farrell 5139 +we'd 5140 +respectively 5141 +crazed 5142 +din 5143 +expedition 5144 +chicken 5145 +cannibal 5146 +conscious 5147 +experimental 5148 +astonishing 5149 +inability 5150 +examination 5151 +wilderness 5152 +tube 5153 +blast 5154 +nerd 5155 +legacy 5156 +companies 5157 +subjected 5158 +ships 5159 +rises 5160 +invented 5161 +stuart 5162 +ambiguous 5163 +grief 5164 +rave 5165 +cracking 5166 +unexpectedly 5167 +scotland 5168 +stargate 5169 +milk 5170 +singers 5171 +darren 5172 +billed 5173 +tripe 5174 +ordered 5175 +furious 5176 +flair 5177 +griffith 5178 +refused 5179 +fascination 5180 +tastes 5181 +owen 5182 +frightened 5183 +amused 5184 +masks 5185 +females 5186 +graham 5187 +rates 5188 +simultaneously 5189 +senses 5190 +walsh 5191 +marc 5192 +simmons 5193 +shanghai 5194 +premiere 5195 +remained 5196 +warriors 5197 +1936 5198 +josh 5199 +antwone 5200 +difficulties 5201 +shoulders 5202 +femme 5203 +alternative 5204 +sentiment 5205 +relax 5206 +ollie 5207 +leon 5208 +rooney 5209 +objective 5210 +deranged 5211 +alcohol 5212 +austin 5213 +sissy 5214 +tank 5215 +dysfunctional 5216 +vulgar 5217 +stumbled 5218 +desires 5219 +replace 5220 +dixon 5221 +claus 5222 +joel 5223 +hears 5224 +coast 5225 +poison 5226 +addicted 5227 +slice 5228 +lundgren 5229 +parade 5230 +gather 5231 +appropriately 5232 +abused 5233 +cream 5234 +challenged 5235 +awhile 5236 +tacky 5237 +interactions 5238 +function 5239 +pun 5240 +bud 5241 +filling 5242 +primitive 5243 +fishing 5244 +raises 5245 +infected 5246 +musicians 5247 +precisely 5248 +caricatures 5249 +karl 5250 +underneath 5251 +ross 5252 +alicia 5253 +prey 5254 +fingers 5255 +nephew 5256 +crystal 5257 +skull 5258 +remakes 5259 +favour 5260 +wildly 5261 +phil 5262 +phrase 5263 +julian 5264 +sopranos 5265 +complaints 5266 +presenting 5267 +noises 5268 +19th 5269 +twins 5270 +les 5271 +ramones 5272 +lands 5273 +joins 5274 +wakes 5275 +require 5276 +fifty 5277 +items 5278 +frankenstein 5279 +nathan 5280 +christianity 5281 +reid 5282 +accomplish 5283 +22 5284 +dana 5285 +wang 5286 +breed 5287 +millionaire 5288 +sums 5289 +knocked 5290 +teaches 5291 +literary 5292 +loneliness 5293 +fiancé 5294 +complaining 5295 +silliness 5296 +sharon 5297 +celebration 5298 +gentleman 5299 +ustinov 5300 +husband's 5301 +exposition 5302 +choppy 5303 +altman 5304 +minus 5305 +amusement 5306 +sugar 5307 +husbands 5308 +framed 5309 +other's 5310 +andre 5311 +unlikable 5312 +sunny 5313 +roommate 5314 +stark 5315 +absurdity 5316 +rifle 5317 +electric 5318 +posters 5319 +aspiring 5320 +conscience 5321 +fields 5322 +hackneyed 5323 +downey 5324 +buster 5325 +edit 5326 +straightforward 5327 +misleading 5328 +carell 5329 +murdering 5330 +credited 5331 +sung 5332 +releases 5333 +muddled 5334 +raines 5335 +coincidence 5336 +unfold 5337 +rude 5338 +charged 5339 +weakness 5340 +quietly 5341 +pitiful 5342 +marshall 5343 +objects 5344 +shared 5345 +inexplicably 5346 +automatically 5347 +heartfelt 5348 +agenda 5349 +dresses 5350 +trend 5351 +acclaimed 5352 +blacks 5353 +murray 5354 +beverly 5355 +asylum 5356 +belushi 5357 +en 5358 +moreover 5359 +shoddy 5360 +bernard 5361 +teachers 5362 +devices 5363 +cattle 5364 +preston 5365 +dont 5366 +grotesque 5367 +visited 5368 +discovering 5369 +roof 5370 +spark 5371 +realised 5372 +handling 5373 +adopted 5374 +bread 5375 +haired 5376 +ethnic 5377 +encourage 5378 +lock 5379 +conviction 5380 +imaginable 5381 +fog 5382 +crawford 5383 +firm 5384 +servant 5385 +invites 5386 +dirt 5387 +cancer 5388 +fantasies 5389 +rely 5390 +biased 5391 +occasions 5392 +dose 5393 +industrial 5394 +harm 5395 +hungry 5396 +vance 5397 +kansas 5398 +active 5399 +preposterous 5400 +profanity 5401 +positively 5402 +prepare 5403 +ladder 5404 +sketch 5405 +alison 5406 +controlled 5407 +squad 5408 +outfits 5409 +deniro 5410 +canyon 5411 +babies 5412 +frankie 5413 +referred 5414 +kumar 5415 +regarded 5416 +designer 5417 +1988 5418 +paradise 5419 +comedians 5420 +russia 5421 +fido 5422 +provocative 5423 +behaviour 5424 +region 5425 +1930's 5426 +baldwin 5427 +laurence 5428 +translated 5429 +tracking 5430 +clock 5431 +1939 5432 +chills 5433 +hawke 5434 +cue 5435 +heist 5436 +citizens 5437 +da 5438 +1978 5439 +mode 5440 +hk 5441 +counts 5442 +riot 5443 +uncut 5444 +musician 5445 +accepts 5446 +shoulder 5447 +heartbreaking 5448 +secondary 5449 +option 5450 +75 5451 +roller 5452 +1980's 5453 +fathers 5454 +mclaglen 5455 +hopelessly 5456 +tasteless 5457 +bye 5458 +challenges 5459 +bitch 5460 +additional 5461 +backs 5462 +should've 5463 +swing 5464 +betrayal 5465 +labor 5466 +lush 5467 +morbid 5468 +abrupt 5469 +gambling 5470 +historic 5471 +iv 5472 +insurance 5473 +1986 5474 +fade 5475 +screens 5476 +bike 5477 +damme 5478 +pages 5479 +nut 5480 +admirable 5481 +rejected 5482 +skits 5483 +lip 5484 +ignorance 5485 +chainsaw 5486 +cassidy 5487 +suspension 5488 +respective 5489 +nod 5490 +chuckle 5491 +recommendation 5492 +guitar 5493 +youngest 5494 +reign 5495 +1970 5496 +biko 5497 +severely 5498 +affection 5499 +coaster 5500 +visiting 5501 +kid's 5502 +darn 5503 +refer 5504 +boxer 5505 +naughty 5506 +macarthur 5507 +deserted 5508 +amazon 5509 +paramount 5510 +files 5511 +corpses 5512 +realm 5513 +nemesis 5514 +1979 5515 +sabrina 5516 +address 5517 +beware 5518 +shares 5519 +tomorrow 5520 +prejudice 5521 +el 5522 +guaranteed 5523 +wwe 5524 +sooner 5525 +reluctant 5526 +1989 5527 +invited 5528 +aim 5529 +dickens 5530 +evidently 5531 +lindsay 5532 +hyped 5533 +penny 5534 +praised 5535 +jews 5536 +sympathize 5537 +barrel 5538 +disappears 5539 +guests 5540 +anticipation 5541 +conventions 5542 +outs 5543 +tail 5544 +deleted 5545 +freaks 5546 +rome 5547 +indication 5548 +bunny 5549 +actor's 5550 +19 5551 +fist 5552 +mayhem 5553 +1969 5554 +policeman 5555 +cannon 5556 +thread 5557 +basinger 5558 +bridget 5559 +selection 5560 +palma 5561 +inconsistent 5562 +saint 5563 +stopping 5564 +gut 5565 +burst 5566 +visions 5567 +angst 5568 +daughter's 5569 +beside 5570 +reader 5571 +sentinel 5572 +nails 5573 +promote 5574 +weaknesses 5575 +heading 5576 +www 5577 +venture 5578 +malone 5579 +misguided 5580 +1960's 5581 +muppet 5582 +uh 5583 +drove 5584 +overlong 5585 +gal 5586 +cope 5587 +mccoy 5588 +threatens 5589 +iconic 5590 +rita 5591 +stages 5592 +underworld 5593 +adolescent 5594 +tip 5595 +previews 5596 +depending 5597 +hammy 5598 +behold 5599 +steady 5600 +circus 5601 +filler 5602 +conveys 5603 +glowing 5604 +vader 5605 +shades 5606 +acceptance 5607 +psychology 5608 +bent 5609 +banal 5610 +receiving 5611 +palance 5612 +reflects 5613 +cruelty 5614 +guy's 5615 +tyler 5616 +insipid 5617 +posted 5618 +hack 5619 +curly 5620 +sassy 5621 +nicolas 5622 +harmless 5623 +morally 5624 +affairs 5625 +macho 5626 +understands 5627 +fluff 5628 +demonstrates 5629 +exceptions 5630 +bow 5631 +investigating 5632 +widescreen 5633 +30's 5634 +remade 5635 +studies 5636 +records 5637 +bros 5638 +unexplained 5639 +sirk 5640 +oldest 5641 +firing 5642 +vein 5643 +explores 5644 +completed 5645 +eternal 5646 +marvel 5647 +preachy 5648 +triple 5649 +schlock 5650 +min 5651 +employed 5652 +campaign 5653 +difficulty 5654 +strongest 5655 +gregory 5656 +grainy 5657 +popping 5658 +disguise 5659 +filth 5660 +dates 5661 +obligatory 5662 +robbins 5663 +terrified 5664 +portrayals 5665 +commander 5666 +hokey 5667 +emerges 5668 +confident 5669 +connections 5670 +lifted 5671 +artsy 5672 +height 5673 +entitled 5674 +outing 5675 +rukh 5676 +hopkins 5677 +pounds 5678 +sending 5679 +hapless 5680 +physics 5681 +phenomenon 5682 +assuming 5683 +unrelated 5684 +kitty 5685 +repeating 5686 +stores 5687 +attract 5688 +fifties 5689 +assured 5690 +clan 5691 +insists 5692 +interestingly 5693 +patricia 5694 +mentality 5695 +knight 5696 +1981 5697 +bug 5698 +paxton 5699 +pole 5700 +hughes 5701 +communicate 5702 +sox 5703 +rhythm 5704 +nolan 5705 +bitten 5706 +despicable 5707 +slimy 5708 +predict 5709 +recognizable 5710 +rounded 5711 +shakespeare's 5712 +gate 5713 +1945 5714 +recycled 5715 +conclude 5716 +casual 5717 +disgusted 5718 +comparisons 5719 +zombi 5720 +couch 5721 +offs 5722 +vital 5723 +representation 5724 +rod 5725 +duck 5726 +martha 5727 +danish 5728 +yawn 5729 +studying 5730 +1976 5731 +clarke 5732 +woo 5733 +route 5734 +prominent 5735 +tarantino 5736 +legends 5737 +paintings 5738 +suitably 5739 +someday 5740 +snakes 5741 +absorbed 5742 +stairs 5743 +redeem 5744 +gear 5745 +shortcomings 5746 +agency 5747 +tempted 5748 +rapist 5749 +inexplicable 5750 +locals 5751 +http 5752 +clueless 5753 +pleasing 5754 +vibrant 5755 +independence 5756 +marries 5757 +clad 5758 +charms 5759 +rendered 5760 +heartwarming 5761 +melody 5762 +shouting 5763 +wig 5764 +defeated 5765 +friend's 5766 +stack 5767 +lois 5768 +novak 5769 +coup 5770 +globe 5771 +soup 5772 +claustrophobic 5773 +eats 5774 +flashy 5775 +trivia 5776 +spinal 5777 +thompson 5778 +considerably 5779 +forcing 5780 +befriends 5781 +grudge 5782 +chavez 5783 +net 5784 +shopping 5785 +gems 5786 +claiming 5787 +foxx 5788 +muppets 5789 +discussing 5790 +boston 5791 +ingenious 5792 +flowers 5793 +harold 5794 +feeding 5795 +eternity 5796 +norm 5797 +sharing 5798 +meg 5799 +quinn 5800 +election 5801 +camcorder 5802 +limit 5803 +genie 5804 +daniels 5805 +quaid 5806 +bacon 5807 +runner 5808 +tierney 5809 +champion 5810 +stallone 5811 +minister 5812 +publicity 5813 +static 5814 +springer 5815 +info 5816 +screw 5817 +inhabitants 5818 +'70s 5819 +renaissance 5820 +carla 5821 +screwed 5822 +delicate 5823 +marlon 5824 +weather 5825 +deserving 5826 +incidentally 5827 +depends 5828 +winchester 5829 +boyle 5830 +gina 5831 +immature 5832 +lift 5833 +wings 5834 +partners 5835 +rope 5836 +ace 5837 +phillips 5838 +kathryn 5839 +elite 5840 +pete 5841 +brother's 5842 +glamorous 5843 +transformed 5844 +blatantly 5845 +symbolic 5846 +traffic 5847 +belt 5848 +strings 5849 +excess 5850 +stalker 5851 +smiles 5852 +ton 5853 +politician 5854 +keen 5855 +esther 5856 +ambition 5857 +surgery 5858 +ants 5859 +audrey 5860 +housewife 5861 +ish 5862 +lasting 5863 +allen's 5864 +dvds 5865 +schools 5866 +concepts 5867 +hilarity 5868 +newman 5869 +shaking 5870 +28 5871 +programs 5872 +frames 5873 +coupled 5874 +cheer 5875 +disorder 5876 +salt 5877 +beatles 5878 +fuller 5879 +shorter 5880 +voted 5881 +toronto 5882 +raj 5883 +1940 5884 +exploring 5885 +debate 5886 +yeti 5887 +layers 5888 +fontaine 5889 +backwards 5890 +continually 5891 +feat 5892 +georges 5893 +organized 5894 +destined 5895 +bombs 5896 +differently 5897 +nope 5898 +bend 5899 +towers 5900 +mothers 5901 +partially 5902 +outdated 5903 +punches 5904 +stumbles 5905 +bully 5906 +threatened 5907 +thrilled 5908 +leigh 5909 +charlton 5910 +wax 5911 +bondage 5912 +kolchak 5913 +spree 5914 +assassination 5915 +doctors 5916 +remove 5917 +claude 5918 +europa 5919 +wire 5920 +leather 5921 +messy 5922 +item 5923 +institution 5924 +departure 5925 +centre 5926 +else's 5927 +detectives 5928 +triangle 5929 +lifeless 5930 +handles 5931 +hides 5932 +wanders 5933 +dudley 5934 +accurately 5935 +duration 5936 +hum 5937 +harrison 5938 +damaged 5939 +satirical 5940 +1950 5941 +minority 5942 +suggestion 5943 +insightful 5944 +hangs 5945 +btw 5946 +preferred 5947 +sorely 5948 +windows 5949 +formed 5950 +profession 5951 +boy's 5952 +commenting 5953 +newer 5954 +landed 5955 +colin 5956 +tenant 5957 +goers 5958 +gunga 5959 +uniformly 5960 +neurotic 5961 +trials 5962 +authorities 5963 +oriented 5964 +swept 5965 +northern 5966 +computers 5967 +dylan 5968 +racing 5969 +kline 5970 +95 5971 +vocal 5972 +steele 5973 +1990s 5974 +viewer's 5975 +bridges 5976 +proving 5977 +entered 5978 +demonic 5979 +natives 5980 +seeming 5981 +brendan 5982 +reeves 5983 +obtain 5984 +rear 5985 +evolution 5986 +ie 5987 +christine 5988 +token 5989 +elevator 5990 +braveheart 5991 +garner 5992 +ripping 5993 +refuse 5994 +firmly 5995 +outright 5996 +mermaid 5997 +exquisite 5998 +mutual 5999 +posey 6000 +biblical 6001 +disastrous 6002 +sleaze 6003 +bars 6004 +helpful 6005 +wendigo 6006 +eleven 6007 +choosing 6008 +neatly 6009 +engrossing 6010 +kidman 6011 +freddy's 6012 +earn 6013 +tops 6014 +uma 6015 +anton 6016 +justified 6017 +wtf 6018 +demanding 6019 +mannerisms 6020 +inspire 6021 +speeches 6022 +containing 6023 +pacific 6024 +myth 6025 +sleeps 6026 +reliable 6027 +fifth 6028 +gillian 6029 +setup 6030 +vile 6031 +cookie 6032 +4th 6033 +hitler's 6034 +bowl 6035 +she'll 6036 +sincerely 6037 +tapes 6038 +vanessa 6039 +insanity 6040 +casts 6041 +ratso 6042 +brooding 6043 +disgrace 6044 +luis 6045 +helpless 6046 +1991 6047 +mirrors 6048 +label 6049 +emerge 6050 +kent 6051 +altered 6052 +forgiven 6053 +predecessor 6054 +heels 6055 +skit 6056 +contempt 6057 +activity 6058 +crossing 6059 +describing 6060 +1985 6061 +duvall 6062 +rampage 6063 +healthy 6064 +knightley 6065 +mercy 6066 +undead 6067 +cemetery 6068 +spies 6069 +mesmerizing 6070 +homicide 6071 +cons 6072 +frontal 6073 +ariel 6074 +restrained 6075 +valentine 6076 +approaches 6077 +startling 6078 +cerebral 6079 +vain 6080 +rooting 6081 +destroys 6082 +preparing 6083 +subtly 6084 +1977 6085 +1974 6086 +jordan 6087 +hats 6088 +grateful 6089 +pc 6090 +boasts 6091 +gere 6092 +regards 6093 +creek 6094 +survives 6095 +mixing 6096 +realities 6097 +conan 6098 +topics 6099 +educated 6100 +shaped 6101 +insights 6102 +melissa 6103 +carey 6104 +tunnel 6105 +artwork 6106 +hulk 6107 +hartley 6108 +radical 6109 +deny 6110 +modest 6111 +unlikeable 6112 +compete 6113 +1994 6114 +sometime 6115 +statue 6116 +grounds 6117 +weaker 6118 +seedy 6119 +mitch 6120 +breakfast 6121 +inspirational 6122 +jess 6123 +hugely 6124 +leaders 6125 +coat 6126 +miami 6127 +scariest 6128 +owners 6129 +casino 6130 +miniseries 6131 +freeze 6132 +akin 6133 +timberlake 6134 +deer 6135 +jared 6136 +bulk 6137 +conrad 6138 +wardrobe 6139 +poker 6140 +crashes 6141 +hers 6142 +rapidly 6143 +applaud 6144 +tara 6145 +nominations 6146 +wrenching 6147 +votes 6148 +contribution 6149 +candidate 6150 +loretta 6151 +affects 6152 +homes 6153 +cinemas 6154 +dubious 6155 +child's 6156 +stare 6157 +banter 6158 +exploits 6159 +advertised 6160 +21st 6161 +guards 6162 +vastly 6163 +relentless 6164 +disguised 6165 +masterfully 6166 +critique 6167 +dim 6168 +located 6169 +refers 6170 +narrow 6171 +des 6172 +washed 6173 +origin 6174 +puppets 6175 +addict 6176 +internal 6177 +error 6178 +disgust 6179 +injured 6180 +cartoonish 6181 +bronson 6182 +gods 6183 +alvin 6184 +30s 6185 +shell 6186 +owes 6187 +repulsive 6188 +gimmick 6189 +boris 6190 +linear 6191 +randolph 6192 +photographs 6193 +rides 6194 +ingrid 6195 +scifi 6196 +abruptly 6197 +limitations 6198 +joker 6199 +youthful 6200 +dandy 6201 +unsure 6202 +dazzling 6203 +gained 6204 +arab 6205 +detract 6206 +underwear 6207 +christina 6208 +caricature 6209 +bloom 6210 +continuing 6211 +lasts 6212 +inaccurate 6213 +where's 6214 +swallow 6215 +standout 6216 +motive 6217 +nations 6218 +convicted 6219 +bravo 6220 +youtube 6221 +nolte 6222 +lauren 6223 +holocaust 6224 +vehicles 6225 +bones 6226 +thirties 6227 +audition 6228 +factors 6229 +headache 6230 +growth 6231 +natured 6232 +mason 6233 +expertly 6234 +spine 6235 +hires 6236 +zizek 6237 +undeniably 6238 +bates 6239 +excellently 6240 +highway 6241 +nina 6242 +screenwriters 6243 +buzz 6244 +chronicles 6245 +insults 6246 +corn 6247 +stunningly 6248 +dread 6249 +homosexuality 6250 +perception 6251 +antonio 6252 +lukas 6253 +reward 6254 +decline 6255 +son's 6256 +las 6257 +mol 6258 +unsuspecting 6259 +strengths 6260 +convinces 6261 +spit 6262 +entering 6263 +natalie 6264 +tossed 6265 +toni 6266 +colours 6267 +ronald 6268 +mathieu 6269 +implied 6270 +teams 6271 +resolved 6272 +tower 6273 +entirety 6274 +confront 6275 +wander 6276 +derivative 6277 +missile 6278 +definitive 6279 +gates 6280 +supply 6281 +bachelor 6282 +anyone's 6283 +divorced 6284 +attenborough 6285 +males 6286 +promptly 6287 +painter 6288 +sinking 6289 +polly 6290 +origins 6291 +endlessly 6292 +nerves 6293 +1959 6294 +wagner 6295 +carmen 6296 +judd 6297 +poe 6298 +walt 6299 +unimaginative 6300 +anil 6301 +mice 6302 +1940s 6303 +confronted 6304 +200 6305 +lend 6306 +authenticity 6307 +siblings 6308 +longest 6309 +repressed 6310 +alexandre 6311 +span 6312 +sergeant 6313 +stardom 6314 +cassavetes 6315 +vividly 6316 +salvation 6317 +yep 6318 +jacket 6319 +users 6320 +jarring 6321 +enhanced 6322 +puerto 6323 +colleagues 6324 +referring 6325 +jedi 6326 +tokyo 6327 +niece 6328 +published 6329 +jackson's 6330 +mates 6331 +cbs 6332 +damned 6333 +sgt 6334 +delicious 6335 +uniform 6336 +dominated 6337 +judgment 6338 +juliet 6339 +accessible 6340 +bsg 6341 +exterior 6342 +misfortune 6343 +zane 6344 +phillip 6345 +ally 6346 +giants 6347 +netflix 6348 +energetic 6349 +austen 6350 +unattractive 6351 +devil's 6352 +mobile 6353 +underwater 6354 +stalking 6355 +disabled 6356 +depict 6357 +offbeat 6358 +earnest 6359 +servants 6360 +jill 6361 +bruno 6362 +cliches 6363 +crisp 6364 +nerve 6365 +peck 6366 +wounds 6367 +hepburn 6368 +terminator 6369 +sized 6370 +suburban 6371 +depths 6372 +buys 6373 +hindi 6374 +sticking 6375 +literal 6376 +playboy 6377 +gable 6378 +meandering 6379 +belly 6380 +sensible 6381 +lighter 6382 +21 6383 +stranded 6384 +yokai 6385 +pray 6386 +mutant 6387 +sale 6388 +exit 6389 +estranged 6390 +anyhow 6391 +identical 6392 +foolish 6393 +eventual 6394 +errol 6395 +separated 6396 +bashing 6397 +cushing 6398 +soylent 6399 +antonioni 6400 +galaxy 6401 +glued 6402 +imo 6403 +tormented 6404 +syndrome 6405 +biting 6406 +dragons 6407 +macabre 6408 +dealer 6409 +filthy 6410 +residents 6411 +victorian 6412 +witchcraft 6413 +cents 6414 +improbable 6415 +inherent 6416 +alley 6417 +lester 6418 +readers 6419 +scratch 6420 +pirate 6421 +cher 6422 +pickford 6423 +astounding 6424 +devastating 6425 +breathing 6426 +clash 6427 +approaching 6428 +severed 6429 +owned 6430 +interact 6431 +cleaning 6432 +characteristics 6433 +expects 6434 +guinness 6435 +dismal 6436 +sniper 6437 +lance 6438 +sand 6439 +respectable 6440 +budgets 6441 +sought 6442 +scoop 6443 +slide 6444 +butch 6445 +nightclub 6446 +yours 6447 +blooded 6448 +she'd 6449 +appeals 6450 +ebert 6451 +harriet 6452 +farmer 6453 +stylized 6454 +owns 6455 +noticeable 6456 +kurosawa 6457 +dustin 6458 +id 6459 +balanced 6460 +fragile 6461 +sublime 6462 +salman 6463 +answered 6464 +penn 6465 +amrita 6466 +adore 6467 +logan 6468 +demonstrate 6469 +concentrate 6470 +exploit 6471 +races 6472 +laden 6473 +psychopath 6474 +affleck 6475 +1982 6476 +garland 6477 +worms 6478 +23 6479 +filmmaking 6480 +pattern 6481 +habit 6482 +incapable 6483 +isolation 6484 +fatale 6485 +decidedly 6486 +steam 6487 +jules 6488 +ford's 6489 +asia 6490 +possess 6491 +senior 6492 +reminder 6493 +cheaply 6494 +principals 6495 +immortal 6496 +christie 6497 +monty 6498 +sf 6499 +evelyn 6500 +denis 6501 +corporation 6502 +turd 6503 +soderbergh 6504 +deliverance 6505 +subway 6506 +potter 6507 +breakdown 6508 +flimsy 6509 +packs 6510 +judged 6511 +wisely 6512 +moe 6513 +bogus 6514 +enthusiastic 6515 +cries 6516 +conveyed 6517 +escaping 6518 +plotting 6519 +wilder 6520 +pale 6521 +deliberate 6522 +dvd's 6523 +informed 6524 +promoted 6525 +axe 6526 +flashes 6527 +cypher 6528 +tremendously 6529 +esquire 6530 +1944 6531 +feast 6532 +glaring 6533 +irene 6534 +spectacle 6535 +chopped 6536 +cyborg 6537 +assembled 6538 +drinks 6539 +dump 6540 +celebrated 6541 +quarter 6542 +boyer 6543 +clara 6544 +arguing 6545 +selected 6546 +numbing 6547 +romeo 6548 +volume 6549 +truman 6550 +combines 6551 +embrace 6552 +troma 6553 +expose 6554 +laurie 6555 +kidnapping 6556 +debt 6557 +contribute 6558 +ominous 6559 +jodie 6560 +magician 6561 +o'hara 6562 +conveniently 6563 +outline 6564 +excruciatingly 6565 +accounts 6566 +pound 6567 +pixar 6568 +pierre 6569 +hackman 6570 +lightning 6571 +absorbing 6572 +copied 6573 +clone 6574 +lola 6575 +ugh 6576 +burke 6577 +cecil 6578 +jan 6579 +mitchum 6580 +jealousy 6581 +advised 6582 +40s 6583 +ensure 6584 +collect 6585 +rewarding 6586 +updated 6587 +freaky 6588 +attacking 6589 +rescued 6590 +lex 6591 +1975 6592 +dilemma 6593 +colored 6594 +beowulf 6595 +hi 6596 +melvyn 6597 +ps 6598 +pocket 6599 +passengers 6600 +accepting 6601 +sydney 6602 +classy 6603 +whiny 6604 +loy 6605 +experiencing 6606 +exorcist 6607 +destructive 6608 +300 6609 +goods 6610 +spencer 6611 +corbett 6612 +shepherd 6613 +reports 6614 +expectation 6615 +sophie 6616 +sentimentality 6617 +pause 6618 +sidewalk 6619 +karate 6620 +quantum 6621 +intricate 6622 +tax 6623 +scarface 6624 +crippled 6625 +longing 6626 +nbc 6627 +reeve 6628 +vintage 6629 +crown 6630 +1998 6631 +quentin 6632 +obsessive 6633 +immense 6634 +knocks 6635 +bounty 6636 +indiana 6637 +adaption 6638 +delighted 6639 +er 6640 +naschy 6641 +liam 6642 +establish 6643 +addiction 6644 +europeans 6645 +tool 6646 +stroke 6647 +overblown 6648 +goldblum 6649 +jaded 6650 +pursue 6651 +sucker 6652 +slip 6653 +theories 6654 +rookie 6655 +havoc 6656 +1953 6657 +anticipated 6658 +dukes 6659 +principle 6660 +voyage 6661 +gamera 6662 +swearing 6663 +unsatisfying 6664 +wonderland 6665 +frontier 6666 +parallels 6667 +crashing 6668 +downs 6669 +incorrect 6670 +erika 6671 +aggressive 6672 +divine 6673 +paula 6674 +dashing 6675 +turmoil 6676 +suspected 6677 +aided 6678 +grass 6679 +story's 6680 +distract 6681 +cape 6682 +snuff 6683 +bach 6684 +comprehend 6685 +werewolves 6686 +masterson 6687 +resulted 6688 +miranda 6689 +tendency 6690 +fright 6691 +spaghetti 6692 +goals 6693 +rainy 6694 +reviewing 6695 +juliette 6696 +establishment 6697 +redundant 6698 +switched 6699 +taped 6700 +sarcastic 6701 +arguments 6702 +rider 6703 +peaceful 6704 +barbra 6705 +butcher 6706 +shootout 6707 +bubble 6708 +routines 6709 +demonstrated 6710 +spice 6711 +backed 6712 +polish 6713 +cultures 6714 +parsons 6715 +distress 6716 +hero's 6717 +chill 6718 +morons 6719 +slugs 6720 +subtext 6721 +ultimatum 6722 +intentional 6723 +virtual 6724 +morals 6725 +cutter 6726 +hayworth 6727 +mouthed 6728 +fleshed 6729 +fascist 6730 +dramatically 6731 +passage 6732 +realization 6733 +slaves 6734 +gentlemen 6735 +liu 6736 +hyper 6737 +peculiar 6738 +avoiding 6739 +lavish 6740 +adrian 6741 +vanilla 6742 +boiled 6743 +admired 6744 +thieves 6745 +moron 6746 +sixth 6747 +'cause 6748 +arranged 6749 +climb 6750 +horny 6751 +approached 6752 +alleged 6753 +pumbaa 6754 +predictably 6755 +wielding 6756 +armstrong 6757 +commitment 6758 +seymour 6759 +serum 6760 +odyssey 6761 +hybrid 6762 +messing 6763 +begging 6764 +alter 6765 +establishing 6766 +toby 6767 +whining 6768 +canceled 6769 +collective 6770 +define 6771 +dame 6772 +bikini 6773 +afterward 6774 +mystical 6775 +tourist 6776 +furniture 6777 +fairbanks 6778 +casper 6779 +revolt 6780 +remembering 6781 +exploding 6782 +consideration 6783 +arrest 6784 +inmates 6785 +1934 6786 +shift 6787 +aiming 6788 +samantha 6789 +puzzle 6790 +ghetto 6791 +arc 6792 +traits 6793 +apply 6794 +olds 6795 +sang 6796 +distraction 6797 +hateful 6798 +fools 6799 +anytime 6800 +reviewed 6801 +enhance 6802 +lunch 6803 +coke 6804 +upside 6805 +papers 6806 +insist 6807 +medieval 6808 +wine 6809 +vega 6810 +insomnia 6811 +arriving 6812 +keaton's 6813 +phenomenal 6814 +fills 6815 +graveyard 6816 +stella 6817 +exploited 6818 +writer's 6819 +acquired 6820 +strict 6821 +slapped 6822 +jewel 6823 +thelma 6824 +mcqueen 6825 +pedestrian 6826 +cal 6827 +anthology 6828 +vince 6829 +mythology 6830 +consciousness 6831 +kinnear 6832 +life's 6833 +carnage 6834 +courtroom 6835 +tolerable 6836 +populated 6837 +huston 6838 +contributed 6839 +poses 6840 +actors' 6841 +optimistic 6842 +verdict 6843 +rebellious 6844 +trace 6845 +whites 6846 +commits 6847 +kelly's 6848 +mouths 6849 +stream 6850 +respects 6851 +leap 6852 +sickening 6853 +puppy 6854 +overboard 6855 +diverse 6856 +monologue 6857 +tuned 6858 +corman 6859 +gypo 6860 +skilled 6861 +seasoned 6862 +settled 6863 +horrified 6864 +remembers 6865 +relentlessly 6866 +dj 6867 +— 6868 +jersey 6869 +psychologist 6870 +borders 6871 +lethal 6872 +tony's 6873 +shoe 6874 +smash 6875 +taboo 6876 +wiped 6877 +excuses 6878 +crosses 6879 +salesman 6880 +ritual 6881 +mormon 6882 +achieves 6883 +thunderbirds 6884 +scored 6885 +vanity 6886 +pad 6887 +aussie 6888 +explodes 6889 +ira 6890 +dynamics 6891 +preminger 6892 +franklin 6893 +verbal 6894 +feminine 6895 +policy 6896 +flavor 6897 +expense 6898 +suggesting 6899 +trains 6900 +instincts 6901 +nuances 6902 +dumber 6903 +flock 6904 +feeble 6905 +deanna 6906 +hoot 6907 +cuban 6908 +kathy 6909 +possession 6910 +document 6911 +cohen 6912 +foundation 6913 +diary 6914 +guinea 6915 +covering 6916 +vomit 6917 +readily 6918 +fluid 6919 +cigarette 6920 +tactics 6921 +deliciously 6922 +seductive 6923 +circles 6924 +phase 6925 +themed 6926 +busey 6927 +marilyn 6928 +amidst 6929 +posing 6930 +lean 6931 +cooking 6932 +deputy 6933 +duel 6934 +brainless 6935 +mute 6936 +meantime 6937 +unsympathetic 6938 +wheel 6939 +update 6940 +immigrant 6941 +weary 6942 +basket 6943 +attending 6944 +mortal 6945 +clive 6946 +regularly 6947 +delightfully 6948 +possesses 6949 +newcomer 6950 +porter 6951 +invention 6952 +sources 6953 +wash 6954 +contestants 6955 +shockingly 6956 +wheelchair 6957 +stephanie 6958 +ritchie 6959 +wong 6960 +pushes 6961 +ricky 6962 +audience's 6963 +einstein 6964 +controlling 6965 +mama 6966 +encountered 6967 +pathos 6968 +zorro 6969 +mysteriously 6970 +korea 6971 +bachchan 6972 +jury 6973 +keys 6974 +skinny 6975 +sells 6976 +satisfaction 6977 +romances 6978 +meal 6979 +explosive 6980 +defies 6981 +drab 6982 +clerk 6983 +pfeiffer 6984 +sunrise 6985 +symbol 6986 +pirates 6987 +otto 6988 +novelty 6989 +jacques 6990 +void 6991 +herbert 6992 +narrated 6993 +lionel 6994 +targets 6995 +august 6996 +razor 6997 +rivers 6998 +admitted 6999 +mum 7000 +sundance 7001 +lends 7002 +cliched 7003 +screwball 7004 +serials 7005 +neglected 7006 +olivia 7007 +truths 7008 +sided 7009 +steer 7010 +flower 7011 +indifferent 7012 +dumped 7013 +lucille 7014 +mole 7015 +products 7016 +beg 7017 +releasing 7018 +niven 7019 +stewart's 7020 +ordeal 7021 +darth 7022 +um 7023 +crosby 7024 +statements 7025 +followers 7026 +psyche 7027 +excruciating 7028 +noteworthy 7029 +swinging 7030 +deed 7031 +aftermath 7032 +ranch 7033 +consist 7034 +embarrassingly 7035 +unusually 7036 +convention 7037 +shifts 7038 +produces 7039 +motorcycle 7040 +tickets 7041 +wider 7042 +longoria 7043 +gwyneth 7044 +employee 7045 +instances 7046 +parking 7047 +intact 7048 +starters 7049 +rapid 7050 +arrow 7051 +thurman 7052 +debbie 7053 +dumbest 7054 +wastes 7055 +sarandon 7056 +economic 7057 +israeli 7058 +additionally 7059 +fanatic 7060 +planes 7061 +pursued 7062 +legitimate 7063 +discussed 7064 +forties 7065 +introducing 7066 +anxious 7067 +cannes 7068 +biker 7069 +deciding 7070 +sanders 7071 +fuzzy 7072 +agony 7073 +alot 7074 +assignment 7075 +stones 7076 +scorsese 7077 +caron 7078 +degrees 7079 +medicine 7080 +hannah 7081 +reverse 7082 +inaccuracies 7083 +july 7084 +attended 7085 +gilbert 7086 +forgetting 7087 +jane's 7088 +gielgud 7089 +angie 7090 +milo 7091 +laputa 7092 +branagh's 7093 +motions 7094 +auto 7095 +controversy 7096 +grandma 7097 +cunningham 7098 +professionals 7099 +criticize 7100 +kidnap 7101 +artistry 7102 +sarcasm 7103 +fishburne 7104 +brow 7105 +bogart 7106 +columbia 7107 +incidents 7108 +vera 7109 +meteor 7110 +georgia 7111 +arty 7112 +freaking 7113 +hadley 7114 +suspicion 7115 +scott's 7116 +coffin 7117 +juan 7118 +crossed 7119 +idol 7120 +grip 7121 +obstacles 7122 +mentor 7123 +consequently 7124 +begs 7125 +stating 7126 +ambitions 7127 +muslims 7128 +executives 7129 +daisy 7130 +manners 7131 +warns 7132 +1948 7133 +jolie 7134 +arquette 7135 +distracted 7136 +centuries 7137 +abound 7138 +jose 7139 +factual 7140 +goodbye 7141 +trigger 7142 +breast 7143 +invite 7144 +tcm 7145 +unanswered 7146 +indicate 7147 +shepard 7148 +session 7149 +daylight 7150 +minnelli 7151 +cindy 7152 +funding 7153 +pains 7154 +predator 7155 +flames 7156 +fried 7157 +scripting 7158 +rational 7159 +stabbed 7160 +collette 7161 +'i 7162 +compliment 7163 +hooker 7164 +cliffhanger 7165 +inclusion 7166 +debra 7167 +roughly 7168 +moss 7169 +1967 7170 +awakening 7171 +viewpoint 7172 +kazan 7173 +rejects 7174 +toned 7175 +sentences 7176 +denise 7177 +originals 7178 +cycle 7179 +informative 7180 +pros 7181 +harlow 7182 +stern 7183 +corey 7184 +stalked 7185 +foil 7186 +plodding 7187 +varied 7188 +sweden 7189 +detroit 7190 +misunderstood 7191 +clay 7192 +relevance 7193 +depictions 7194 +blamed 7195 +paints 7196 +pointing 7197 +click 7198 +stance 7199 +protest 7200 +chamber 7201 +robbers 7202 +gooding 7203 +soprano 7204 +likeable 7205 +exclusively 7206 +slim 7207 +campus 7208 +haines 7209 +cheadle 7210 +cap 7211 +cab 7212 +rambling 7213 +paranoid 7214 +seats 7215 +frances 7216 +rowlands 7217 +101 7218 +consequence 7219 +murky 7220 +abandon 7221 +gap 7222 +berkeley 7223 +ruining 7224 +stink 7225 +denouement 7226 +penelope 7227 +intro 7228 +abortion 7229 +tomei 7230 +replies 7231 +antagonist 7232 +gloria 7233 +stardust 7234 +tomb 7235 +gallery 7236 +bug's 7237 +determination 7238 +40's 7239 +c'mon 7240 +translate 7241 +bait 7242 +killer's 7243 +eagerly 7244 +relating 7245 +iranian 7246 +rips 7247 +momentum 7248 +uncanny 7249 +frozen 7250 +begun 7251 +generate 7252 +uniforms 7253 +intensely 7254 +dreamy 7255 +martian 7256 +festivals 7257 +grabbed 7258 +mock 7259 +jenna 7260 +che's 7261 +schedule 7262 +surroundings 7263 +coma 7264 +imaginary 7265 +schneider 7266 +gus 7267 +foremost 7268 +composition 7269 +robertson 7270 +politicians 7271 +services 7272 +hysterically 7273 +snowman 7274 +maureen 7275 +omar 7276 +republic 7277 +lurking 7278 +pans 7279 +alliance 7280 +hostel 7281 +diner 7282 +sheen 7283 +injury 7284 +rupert 7285 +hippies 7286 +rosario 7287 +chamberlain 7288 +ww2 7289 +scenarios 7290 +participants 7291 +realistically 7292 +communication 7293 +kris 7294 +sg 7295 +kathleen 7296 +brat 7297 +redneck 7298 +launch 7299 +therapy 7300 +quasi 7301 +miyazaki 7302 +hmmm 7303 +85 7304 +faux 7305 +geisha 7306 +bauer 7307 +mick 7308 +enigmatic 7309 +1951 7310 +phones 7311 +shaggy 7312 +hostage 7313 +destination 7314 +lens 7315 +glimpses 7316 +1943 7317 +lastly 7318 +rehash 7319 +gestures 7320 +shotgun 7321 +casablanca 7322 +dismiss 7323 +sights 7324 +periods 7325 +burnt 7326 +bats 7327 +resembling 7328 +charlie's 7329 +apt 7330 +linked 7331 +widowed 7332 +dominic 7333 +glance 7334 +cow 7335 +tho 7336 +traps 7337 +curiously 7338 +heath 7339 +envy 7340 +playwright 7341 +gigantic 7342 +paths 7343 +bleed 7344 +ambiguity 7345 +gaps 7346 +bosses 7347 +hayes 7348 +sterling 7349 +necessity 7350 +comeback 7351 +sketches 7352 +sondra 7353 +ignoring 7354 +revolving 7355 +apocalyptic 7356 +reiser 7357 +sailor 7358 +saloon 7359 +frantic 7360 +resistance 7361 +pegg 7362 +overs 7363 +precise 7364 +herman 7365 +rounds 7366 +arkin 7367 +gloomy 7368 +pressed 7369 +haunt 7370 +1992 7371 +enchanted 7372 +iturbi 7373 +fuel 7374 +blaise 7375 +mabel 7376 +laboratory 7377 +county 7378 +veterans 7379 +studied 7380 +cheers 7381 +bearing 7382 +eh 7383 +sunset 7384 +reflected 7385 +rolls 7386 +investigator 7387 +adele 7388 +pen 7389 +maintains 7390 +capacity 7391 +kubrick's 7392 +unstable 7393 +avid 7394 +midst 7395 +man' 7396 +qualify 7397 +bonnie 7398 +person's 7399 +mins 7400 +geek 7401 +nun 7402 +jude 7403 +angelina 7404 +galactica 7405 +sufficient 7406 +substantial 7407 +incest 7408 +handicapped 7409 +trier 7410 +ample 7411 +doctor's 7412 +warden 7413 +supreme 7414 +hinted 7415 +slashers 7416 +rewarded 7417 +rice 7418 +complications 7419 +trauma 7420 +biopic 7421 +sebastian 7422 +'80s 7423 +characterizations 7424 +awareness 7425 +popped 7426 +sparks 7427 +vignettes 7428 +psychedelic 7429 +unclear 7430 +kells 7431 +tightly 7432 +existing 7433 +du 7434 +entrance 7435 +offend 7436 +goldie 7437 +guardian 7438 +collins 7439 +targeted 7440 +talky 7441 +extensive 7442 +ny 7443 +benefits 7444 +epics 7445 +pilots 7446 +payoff 7447 +stadium 7448 +october 7449 +stake 7450 +characterisation 7451 +applied 7452 +applies 7453 +pivotal 7454 +lowe 7455 +gathering 7456 +marisa 7457 +brent 7458 +upcoming 7459 +1963 7460 +overbearing 7461 +eli 7462 +occult 7463 +joking 7464 +ol' 7465 +graduate 7466 +beckinsale 7467 +nuanced 7468 +homicidal 7469 +addressed 7470 +evans 7471 +lunatic 7472 +parrot 7473 +edith 7474 +revival 7475 +convict 7476 +ignores 7477 +safely 7478 +plate 7479 +sour 7480 +turkish 7481 +favourites 7482 +ajay 7483 +boundaries 7484 +northam 7485 +profile 7486 +russ 7487 +skeptical 7488 +frog 7489 +invested 7490 +repeats 7491 +bias 7492 +'60s 7493 +drowned 7494 +iq 7495 +diversity 7496 +outlandish 7497 +nightmarish 7498 +dynamite 7499 +unfolding 7500 +convent 7501 +clooney 7502 +observations 7503 +johansson 7504 +1955 7505 +enchanting 7506 +tire 7507 +stabbing 7508 +disco 7509 +excellence 7510 +27 7511 +clunky 7512 +valid 7513 +array 7514 +engine 7515 +sammo 7516 +doug 7517 +sly 7518 +interior 7519 +resolve 7520 +hating 7521 +olsen 7522 +interviewed 7523 +chong 7524 +protection 7525 +maximum 7526 +nauseating 7527 +versa 7528 +apocalypse 7529 +exploitative 7530 +observation 7531 +murderers 7532 +questioning 7533 +gosh 7534 +stereotyped 7535 +flag 7536 +shore 7537 +pose 7538 +acknowledge 7539 +fruit 7540 +caretaker 7541 +rosemary's 7542 +interpretations 7543 +shin 7544 +stations 7545 +flavia 7546 +nutshell 7547 +announced 7548 +assure 7549 +silverman 7550 +duh 7551 +sonny 7552 +1958 7553 +blockbusters 7554 +pornography 7555 +vivian 7556 +sensibility 7557 +courtesy 7558 +battlestar 7559 +macdonald 7560 +boots 7561 +brides 7562 +reunite 7563 +brooke 7564 +controls 7565 +masked 7566 +phantasm 7567 +prophecy 7568 +slower 7569 +relying 7570 +sweat 7571 +divided 7572 +mannered 7573 +marked 7574 +witnessing 7575 +girlfriends 7576 +snipes 7577 +fortunate 7578 +watcher 7579 +brett 7580 +ernie 7581 +villainous 7582 +strung 7583 +rebels 7584 +candle 7585 +counting 7586 +mccarthy 7587 +rodriguez 7588 +bonham 7589 +portuguese 7590 +daytime 7591 +rea 7592 +insert 7593 +misty 7594 +displaying 7595 +substitute 7596 +satanic 7597 +wayans 7598 +magically 7599 +sincerity 7600 +owl 7601 +cocaine 7602 +spotlight 7603 +inter 7604 +chewing 7605 +lopez 7606 +chiba 7607 +progressed 7608 +entries 7609 +demille 7610 +chuckles 7611 +climbing 7612 +26 7613 +chaotic 7614 +criticized 7615 +confined 7616 +sanity 7617 +goat 7618 +unhinged 7619 +bittersweet 7620 +collar 7621 +realises 7622 +peril 7623 +bust 7624 +smell 7625 +turtle 7626 +wartime 7627 +admits 7628 +commanding 7629 +evokes 7630 +beard 7631 +seduce 7632 +harrowing 7633 +janet 7634 +phoenix 7635 +stiles 7636 +interrupted 7637 +whore 7638 +shocks 7639 +inadvertently 7640 +jar 7641 +wright 7642 +fart 7643 +resume 7644 +lynch's 7645 +needing 7646 +delirious 7647 +upstairs 7648 +obscurity 7649 +famed 7650 +palm 7651 +weekly 7652 +replacement 7653 +monotonous 7654 +smug 7655 +preaching 7656 +projected 7657 +randall 7658 +enduring 7659 +hmm 7660 +organization 7661 +landmark 7662 +thereby 7663 +fundamental 7664 +ripoff 7665 +rightly 7666 +ins 7667 +chew 7668 +slavery 7669 +unnatural 7670 +arrogance 7671 +waking 7672 +manipulation 7673 +jagger 7674 +reserved 7675 +blazing 7676 +finishes 7677 +somethings 7678 +observe 7679 +raging 7680 +thrust 7681 +trivial 7682 +madsen 7683 +carlos 7684 +samuel 7685 +tones 7686 +commendable 7687 +crushed 7688 +similarity 7689 +deemed 7690 +choir 7691 +imagining 7692 +unappealing 7693 +understatement 7694 +apple 7695 +discipline 7696 +thailand 7697 +colleague 7698 +convenient 7699 +rendering 7700 +hines 7701 +cena 7702 +mandy 7703 +testing 7704 +motel 7705 +subsequently 7706 +fassbinder 7707 +reluctantly 7708 +platform 7709 +men's 7710 +egyptian 7711 +aesthetic 7712 +hooper 7713 +accompanying 7714 +protective 7715 +penned 7716 +fetish 7717 +kirsten 7718 +herd 7719 +layered 7720 +scarecrows 7721 +incestuous 7722 +thunder 7723 +boogie 7724 +participate 7725 +forgiveness 7726 +baddies 7727 +hardened 7728 +forgets 7729 +comparable 7730 +combs 7731 +understandably 7732 +shahid 7733 +laying 7734 +marine 7735 +recover 7736 +scheming 7737 +cancelled 7738 +vargas 7739 +stumble 7740 +celebrities 7741 +merry 7742 +russo 7743 +frost 7744 +unfamiliar 7745 +madeleine 7746 +isabelle 7747 +crooks 7748 +python 7749 +filmography 7750 +explode 7751 +sylvia 7752 +article 7753 +climatic 7754 +achievements 7755 +conductor 7756 +pizza 7757 +reminding 7758 +remark 7759 +lo 7760 +gackt 7761 +traumatic 7762 +benjamin 7763 +stuffed 7764 +accidental 7765 +travis 7766 +govinda 7767 +must've 7768 +quintessential 7769 +deathtrap 7770 +cheerful 7771 +hostile 7772 +orchestra 7773 +ninety 7774 +gorilla 7775 +marcel 7776 +cameraman 7777 +shred 7778 +sholay 7779 +wrestler 7780 +customers 7781 +hallmark 7782 +beers 7783 +glossy 7784 +despise 7785 +anita 7786 +goings 7787 +spontaneous 7788 +1932 7789 +fleet 7790 +shameless 7791 +charges 7792 +camping 7793 +finishing 7794 +district 7795 +sins 7796 +dallas 7797 +file 7798 +yell 7799 +serbian 7800 +myrna 7801 +wholesome 7802 +titular 7803 +boo 7804 +o'brien 7805 +implies 7806 +sack 7807 +flip 7808 +salvage 7809 +annoy 7810 +restraint 7811 +imho 7812 +creations 7813 +affecting 7814 +pornographic 7815 +spoiling 7816 +bonanza 7817 +ala 7818 +raid 7819 +raunchy 7820 +sales 7821 +cheering 7822 +captivated 7823 +je 7824 +espionage 7825 +license 7826 +defining 7827 +beforehand 7828 +se 7829 +conclusions 7830 +bakshi's 7831 +hawn 7832 +sherlock 7833 +caprica 7834 +ruled 7835 +unconventional 7836 +diego 7837 +awry 7838 +verge 7839 +krueger 7840 +grin 7841 +whimsical 7842 +ideals 7843 +meyer 7844 +surround 7845 +characteristic 7846 +digging 7847 +shameful 7848 +coolest 7849 +philo 7850 +cells 7851 +reagan 7852 +seattle 7853 +infinitely 7854 +sickness 7855 +excels 7856 +2009 7857 +novelist 7858 +1946 7859 +burial 7860 +fades 7861 +faded 7862 +shannon 7863 +traditions 7864 +fraud 7865 +perverted 7866 +sheets 7867 +voodoo 7868 +desk 7869 +abundance 7870 +flashing 7871 +hunted 7872 +betrayed 7873 +admission 7874 +gershwin 7875 +rampant 7876 +relaxed 7877 +fires 7878 +polar 7879 +kindly 7880 +tits 7881 +melancholy 7882 +drowning 7883 +semblance 7884 +temper 7885 +cracks 7886 +tide 7887 +oblivious 7888 +miraculously 7889 +clarity 7890 +elliott 7891 +inserted 7892 +considers 7893 +constraints 7894 +drift 7895 +sunk 7896 +distributed 7897 +unnecessarily 7898 +welles' 7899 +flows 7900 +sexist 7901 +beckham 7902 +summed 7903 +henchmen 7904 +tools 7905 +transparent 7906 +devotion 7907 +hitchcock's 7908 +earliest 7909 +scarlett 7910 +dangerously 7911 +taut 7912 +dafoe 7913 +dreaming 7914 +seth 7915 +prop 7916 +cain 7917 +wesley 7918 +adapt 7919 +openly 7920 +sane 7921 +hugo 7922 +creasy 7923 +chops 7924 +pitched 7925 +juice 7926 +riff 7927 +blandings 7928 +shah 7929 +screened 7930 +tashan 7931 +meredith 7932 +doyle 7933 +mud 7934 +zodiac 7935 +regime 7936 +irritated 7937 +eagle 7938 +paycheck 7939 +egypt 7940 +spiral 7941 +letdown 7942 +wherever 7943 +madison 7944 +deeds 7945 +robotic 7946 +faint 7947 +outrageously 7948 +sheep 7949 +elsa 7950 +baron 7951 +overtones 7952 +searched 7953 +unleashed 7954 +sporting 7955 +lennon 7956 +gangs 7957 +dahmer 7958 +peggy 7959 +vapid 7960 +heap 7961 +circa 7962 +simpsons 7963 +slater 7964 +permanent 7965 +voyager 7966 +presidential 7967 +compensate 7968 +deepest 7969 +reject 7970 +uneasy 7971 +ghastly 7972 +gretchen 7973 +sophia 7974 +warehouse 7975 +switching 7976 +cedric 7977 +lara 7978 +evoke 7979 +flame 7980 +automatic 7981 +submarine 7982 +plug 7983 +programme 7984 +sucking 7985 +pursuing 7986 +avoids 7987 +assistance 7988 +assumes 7989 +orphan 7990 +mart 7991 +practical 7992 +joining 7993 +failures 7994 +liner 7995 +garfield 7996 +dwight 7997 +slut 7998 +oprah 7999 +committing 8000 +intend 8001 +ealing 8002 +shirts 8003 +locke 8004 +admirer 8005 +awaiting 8006 +ram 8007 +fritz 8008 +melbourne 8009 +contestant 8010 +timmy 8011 +rivals 8012 +buffy 8013 +clouds 8014 +ambiance 8015 +babes 8016 +ensue 8017 +coburn 8018 +occupied 8019 +sergio 8020 +sitcoms 8021 +variation 8022 +censorship 8023 +ferrell 8024 +radiation 8025 +snap 8026 +underdeveloped 8027 +takashi 8028 +hobgoblins 8029 +finney 8030 +listened 8031 +fiancée 8032 +complained 8033 +pauline 8034 +kinski 8035 +alarm 8036 +engineer 8037 +chloe 8038 +proceed 8039 +demeanor 8040 +suzanne 8041 +battlefield 8042 +rebellion 8043 +criticisms 8044 +remainder 8045 +ghostly 8046 +spaceship 8047 +howling 8048 +motivated 8049 +joint 8050 +carpenter's 8051 +fodder 8052 +bert 8053 +dominate 8054 +monks 8055 +dragging 8056 +inclined 8057 +upbeat 8058 +encouraged 8059 +networks 8060 +han 8061 +loren 8062 +brazilian 8063 +atlantic 8064 +flowing 8065 +progression 8066 +tess 8067 +meek 8068 +darkly 8069 +disappearance 8070 +colman 8071 +crashed 8072 +caper 8073 +solved 8074 +fairness 8075 +distinction 8076 +sensual 8077 +feinstone 8078 +sho 8079 +warrant 8080 +grease 8081 +visitor 8082 +marijuana 8083 +sections 8084 +avenge 8085 +tv's 8086 +croc 8087 +sober 8088 +badness 8089 +who've 8090 +ninjas 8091 +myrtle 8092 +runaway 8093 +helmet 8094 +scratching 8095 +quaint 8096 +busby 8097 +defending 8098 +buttons 8099 +artemisia 8100 +cloak 8101 +noting 8102 +confuse 8103 +experts 8104 +whip 8105 +borrow 8106 +barney 8107 +garage 8108 +happenings 8109 +mega 8110 +1990's 8111 +disregard 8112 +bean 8113 +aaron 8114 +edges 8115 +diving 8116 +investment 8117 +wee 8118 +electronic 8119 +gena 8120 +gypsy 8121 +suave 8122 +mustache 8123 +toxic 8124 +mira 8125 +bartender 8126 +prologue 8127 +transport 8128 +atrocity 8129 +everett 8130 +bernsen 8131 +notices 8132 +jo 8133 +boogeyman 8134 +knees 8135 +1966 8136 +1000 8137 +robbed 8138 +epitome 8139 +bennett 8140 +vcr 8141 +who'd 8142 +'a 8143 +detached 8144 +brit 8145 +hometown 8146 +jack's 8147 +prone 8148 +enormously 8149 +gilliam 8150 +jackman 8151 +dom 8152 +impending 8153 +bloodbath 8154 +mister 8155 +macmurray 8156 +vigilante 8157 +offense 8158 +prostitutes 8159 +fashions 8160 +idealistic 8161 +pigs 8162 +abomination 8163 +carpet 8164 +battling 8165 +principles 8166 +paz 8167 +pretends 8168 +awarded 8169 +admiration 8170 +incidental 8171 +tin 8172 +pairing 8173 +woefully 8174 +chip 8175 +classmates 8176 +timed 8177 +budding 8178 +gandolfini 8179 +revolver 8180 +liberty 8181 +associate 8182 +padding 8183 +colony 8184 +zelah 8185 +drum 8186 +vincenzo 8187 +secure 8188 +palestinian 8189 +girls' 8190 +blames 8191 +torment 8192 +kids' 8193 +framing 8194 +tackle 8195 +tended 8196 +peers 8197 +policemen 8198 +facility 8199 +ostensibly 8200 +harron 8201 +prank 8202 +lindy 8203 +bimbo 8204 +1957 8205 +saints 8206 +capote 8207 +shrek 8208 +breathe 8209 +nineties 8210 +worrying 8211 +believability 8212 +paragraph 8213 +mediocrity 8214 +influences 8215 +reported 8216 +conveying 8217 +programming 8218 +stoned 8219 +val 8220 +barnes 8221 +sharks 8222 +unravel 8223 +courageous 8224 +deck 8225 +giovanna 8226 +grating 8227 +britney 8228 +distinctive 8229 +blondell 8230 +spoofs 8231 +brush 8232 +effortlessly 8233 +riders 8234 +midget 8235 +annoyance 8236 +counterparts 8237 +economy 8238 +rivalry 8239 +stab 8240 +knights 8241 +socially 8242 +symbols 8243 +bodyguard 8244 +qualifies 8245 +connie 8246 +acclaim 8247 +managing 8248 +vibe 8249 +monroe 8250 +frat 8251 +baked 8252 +combining 8253 +martians 8254 +boobs 8255 +prostitution 8256 +closure 8257 +senator 8258 +outset 8259 +magazines 8260 +respond 8261 +interiors 8262 +division 8263 +slam 8264 +celebrate 8265 +elected 8266 +zu 8267 +monica 8268 +dillinger 8269 +brashear 8270 +cohesive 8271 +clinic 8272 +gig 8273 +tacked 8274 +coward 8275 +parodies 8276 +greene 8277 +billing 8278 +weirdness 8279 +dunst 8280 +rourke 8281 +manipulated 8282 +concentration 8283 +sinks 8284 +dreyfuss 8285 +asset 8286 +duchovny 8287 +superstar 8288 +clyde 8289 +december 8290 +pompous 8291 +fabric 8292 +placement 8293 +gibson 8294 +bless 8295 +boards 8296 +troopers 8297 +reese 8298 +goodman 8299 +transplant 8300 +shocker 8301 +examine 8302 +chock 8303 +scarlet 8304 +informs 8305 +responds 8306 +collapse 8307 +data 8308 +swiss 8309 +reasoning 8310 +confines 8311 +categories 8312 +injustice 8313 +laser 8314 +dish 8315 +employees 8316 +smith's 8317 +em 8318 +gasp 8319 +sacrifices 8320 +maurice 8321 +worship 8322 +screenplays 8323 +tolerate 8324 +pee 8325 +overshadowed 8326 +dern 8327 +reunited 8328 +brick 8329 +loner 8330 +holt 8331 +sites 8332 +uncertain 8333 +theatres 8334 +morse 8335 +yells 8336 +sibling 8337 +cheech 8338 +butchered 8339 +mae 8340 +ernest 8341 +sensibilities 8342 +500 8343 +ali 8344 +irving 8345 +castro 8346 +influential 8347 +terrorism 8348 +strained 8349 +derived 8350 +chandler 8351 +slept 8352 +perspectives 8353 +bleeding 8354 +madman 8355 +1942 8356 +inconsistencies 8357 +sensitivity 8358 +jam 8359 +hans 8360 +sustain 8361 +systems 8362 +armor 8363 +burgess 8364 +fiery 8365 +queens 8366 +katie 8367 +gruff 8368 +ewoks 8369 +faye 8370 +tramp 8371 +brandon 8372 +lighthearted 8373 +inform 8374 +cursed 8375 +retro 8376 +250 8377 +malden 8378 +cody 8379 +spelled 8380 +manic 8381 +labeled 8382 +perverse 8383 +collector 8384 +drain 8385 +shelter 8386 +spade 8387 +fallon 8388 +ang 8389 +gino 8390 +kareena 8391 +depardieu 8392 +apollo 8393 +officially 8394 +playful 8395 +informer 8396 +banks 8397 +retirement 8398 +booth 8399 +replacing 8400 +transforms 8401 +surrender 8402 +shield 8403 +jigsaw 8404 +fiend 8405 +predecessors 8406 +judgement 8407 +bing 8408 +englund 8409 +ads 8410 +damsel 8411 +stirring 8412 +structured 8413 +patty 8414 +poet 8415 +signature 8416 +tolerance 8417 +bites 8418 +dash 8419 +seriousness 8420 +casted 8421 +mercifully 8422 +edison 8423 +advances 8424 +padded 8425 +czech 8426 +lingering 8427 +sensational 8428 +crowded 8429 +bigfoot 8430 +captive 8431 +plotted 8432 +premiered 8433 +dictator 8434 +locale 8435 +bastard 8436 +manga 8437 +fighters 8438 +sophistication 8439 +lifts 8440 +yarn 8441 +spelling 8442 +uptight 8443 +farrah 8444 +drummer 8445 +amid 8446 +kidnaps 8447 +peaks 8448 +drastically 8449 +cringing 8450 +coop 8451 +dealers 8452 +geoffrey 8453 +rousing 8454 +supermarket 8455 +standpoint 8456 +thereafter 8457 +portions 8458 +latino 8459 +henchman 8460 +berenger 8461 +slash 8462 +sandy 8463 +lurid 8464 +coal 8465 +interplay 8466 +stares 8467 +willingly 8468 +mines 8469 +ss 8470 +ceremony 8471 +inexperienced 8472 +awfulness 8473 +condemned 8474 +benny 8475 +alba 8476 +mythical 8477 +spotted 8478 +sara 8479 +fierce 8480 +thereof 8481 +bloodshed 8482 +enthralling 8483 +geniuses 8484 +lars 8485 +rant 8486 +theodore 8487 +heather 8488 +echoes 8489 +maintaining 8490 +bombed 8491 +bitchy 8492 +fiasco 8493 +powered 8494 +tina 8495 +ossessione 8496 +worm 8497 +godard 8498 +observed 8499 +staging 8500 +attendant 8501 +anxiety 8502 +villa 8503 +varying 8504 +stepmother 8505 +aircraft 8506 +david's 8507 +justification 8508 +identified 8509 +downfall 8510 +anguish 8511 +shoved 8512 +allan 8513 +bliss 8514 +caution 8515 +transported 8516 +impressions 8517 +miike's 8518 +alexandra 8519 +shout 8520 +functions 8521 +imitate 8522 +norris 8523 +dwarf 8524 +nearest 8525 +funky 8526 +drugged 8527 +stabs 8528 +marrying 8529 +hallucinations 8530 +allies 8531 +communism 8532 +fixed 8533 +sorrow 8534 +orlando 8535 +register 8536 +surf 8537 +scarier 8538 +freed 8539 +tasty 8540 +baddie 8541 +vet 8542 +attic 8543 +representing 8544 +widower 8545 +cunning 8546 +plagued 8547 +hunky 8548 +apartheid 8549 +cockney 8550 +luc 8551 +islands 8552 +fur 8553 +emphasize 8554 +confession 8555 +ceiling 8556 +hairy 8557 +warhols 8558 +stricken 8559 +presume 8560 +rosenstrasse 8561 +meadows 8562 +distorted 8563 +virtue 8564 +natali 8565 +forrest 8566 +starship 8567 +lampoon 8568 +depend 8569 +marvin 8570 +mixes 8571 +jewelry 8572 +correctness 8573 +nest 8574 +myra 8575 +rockets 8576 +russians 8577 +glenda 8578 +byron 8579 +sammy 8580 +grandpa 8581 +monday 8582 +entertains 8583 +adultery 8584 +egg 8585 +massey 8586 +drawings 8587 +travolta 8588 +tricked 8589 +abu 8590 +bio 8591 +lin 8592 +fagin 8593 +cowardly 8594 +overwrought 8595 +determine 8596 +throne 8597 +ratio 8598 +tsui 8599 +paired 8600 +cannibals 8601 +fuss 8602 +client 8603 +animator 8604 +hurry 8605 +romania 8606 +foreboding 8607 +pub 8608 +earns 8609 +bon 8610 +gen 8611 +della 8612 +photograph 8613 +pecker 8614 +censors 8615 +groundbreaking 8616 +predicted 8617 +crooked 8618 +engagement 8619 +arnie 8620 +torturing 8621 +towns 8622 +intellectually 8623 +bald 8624 +finely 8625 +confirmed 8626 +natasha 8627 +hale 8628 +chemical 8629 +spells 8630 +loony 8631 +richly 8632 +edmund 8633 +groove 8634 +vaudeville 8635 +bills 8636 +ma 8637 +millennium 8638 +gladiator 8639 +icy 8640 +irrational 8641 +ballroom 8642 +daria 8643 +conflicted 8644 +clarence 8645 +subdued 8646 +sigh 8647 +artistically 8648 +keanu 8649 +laced 8650 +potent 8651 +representative 8652 +gently 8653 +reckless 8654 +dopey 8655 +jerky 8656 +deborah 8657 +decency 8658 +grossly 8659 +predictability 8660 +consumed 8661 +belle 8662 +blessed 8663 +parks 8664 +curtain 8665 +dukakis 8666 +federal 8667 +analyze 8668 +echo 8669 +contributes 8670 +accomplishment 8671 +cheesiness 8672 +romanian 8673 +almighty 8674 +continuously 8675 +gathered 8676 +dive 8677 +undercover 8678 +diaz 8679 +profoundly 8680 +identities 8681 +crypt 8682 +downbeat 8683 +1949 8684 +gusto 8685 +missions 8686 +sasquatch 8687 +locate 8688 +borrows 8689 +maturity 8690 +harbor 8691 +denial 8692 +emmy 8693 +arch 8694 +animations 8695 +airing 8696 +superfluous 8697 +lists 8698 +officials 8699 +steaming 8700 +operate 8701 +threads 8702 +significantly 8703 +aniston 8704 +goldsworthy 8705 +anchors 8706 +disappoints 8707 +collaboration 8708 +trusted 8709 +lays 8710 +sync 8711 +1920s 8712 +wrongly 8713 +lindsey 8714 +optimism 8715 +vertigo 8716 +abroad 8717 +judges 8718 +continent 8719 +lizard 8720 +muni 8721 +helena 8722 +hartley's 8723 +zeta 8724 +denying 8725 +proportions 8726 +winners 8727 +ll 8728 +monologues 8729 +gravity 8730 +forbes 8731 +launched 8732 +robbing 8733 +mash 8734 +mocking 8735 +confronts 8736 +mutants 8737 +beetle 8738 +nifty 8739 +fence 8740 +horn 8741 +luxury 8742 +athletic 8743 +imprisoned 8744 +scriptwriter 8745 +mack 8746 +handy 8747 +pia 8748 +uninspiring 8749 +rhyme 8750 +1964 8751 +promoting 8752 +73 8753 +flew 8754 +98 8755 +corbin 8756 +chevy 8757 +mobster 8758 +altman's 8759 +extraordinarily 8760 +applause 8761 +abstract 8762 +switches 8763 +garde 8764 +icons 8765 +showcases 8766 +intelligently 8767 +capitalism 8768 +developments 8769 +lions 8770 +hanzo 8771 +hypnotic 8772 +temptation 8773 +dedication 8774 +opposition 8775 +sensation 8776 +kristofferson 8777 +barton 8778 +lds 8779 +bothers 8780 +satisfactory 8781 +nora 8782 +genetic 8783 +moonstruck 8784 +illustrate 8785 +notwithstanding 8786 +elephants 8787 +stripper 8788 +grendel 8789 +fulfilling 8790 +languages 8791 +hilton 8792 +autobiography 8793 +pleasures 8794 +lightweight 8795 +increasing 8796 +preferably 8797 +shifting 8798 +bearable 8799 +prefers 8800 +idiocy 8801 +heroin 8802 +manipulate 8803 +uncredited 8804 +sheridan 8805 +conniving 8806 +surgeon 8807 +nonexistent 8808 +deservedly 8809 +clutter 8810 +bullies 8811 +penalty 8812 +scattered 8813 +owe 8814 +lawn 8815 +upbringing 8816 +increase 8817 +oblivion 8818 +fanning 8819 +shiny 8820 +cynicism 8821 +kings 8822 +hazzard 8823 +preacher 8824 +ongoing 8825 +luthor 8826 +sister's 8827 +quirks 8828 +michaels 8829 +transitions 8830 +ravishing 8831 +reno 8832 +corridors 8833 +shady 8834 +cloth 8835 +liotta 8836 +spinning 8837 +sleeper 8838 +auteur 8839 +plummer 8840 +appalled 8841 +reportedly 8842 +dodgy 8843 +todays 8844 +harilal 8845 +kilmer 8846 +blackmail 8847 +toss 8848 +distinctly 8849 +violently 8850 +ebay 8851 +limp 8852 +marines 8853 +lesbians 8854 +vaughn 8855 +bart 8856 +knocking 8857 +palma's 8858 +boost 8859 +aboard 8860 +defy 8861 +civilians 8862 +brunette 8863 +fewer 8864 +cinematographic 8865 +liberties 8866 +shrill 8867 +youngsters 8868 +strain 8869 +hammerhead 8870 +inhabit 8871 +thug 8872 +dyke 8873 +euro 8874 +cassie 8875 +fellini 8876 +puzzled 8877 +chop 8878 +sweeping 8879 +throats 8880 +thirds 8881 +billion 8882 +witted 8883 +operating 8884 +atomic 8885 +lt 8886 +supportive 8887 +henderson 8888 +profit 8889 +prolific 8890 +sore 8891 +virginity 8892 +sleepy 8893 +golf 8894 +outlaw 8895 +unnerving 8896 +expresses 8897 +mills 8898 +forsythe 8899 +authors 8900 +behaving 8901 +visconti 8902 +efficient 8903 +visceral 8904 +glow 8905 +jones' 8906 +melinda 8907 +muscle 8908 +pepper 8909 +heavenly 8910 +unwilling 8911 +1965 8912 +roach 8913 +marcus 8914 +tables 8915 +shelves 8916 +dunne 8917 +tedium 8918 +illustrated 8919 +explanations 8920 +snowy 8921 +patriotic 8922 +alcoholism 8923 +whipped 8924 +ledger 8925 +slaughtered 8926 +redford 8927 +percent 8928 +rapes 8929 +disasters 8930 +dickinson 8931 +examined 8932 +cradle 8933 +fleeing 8934 +healing 8935 +lightly 8936 +nerdy 8937 +torch 8938 +rodney 8939 +believer 8940 +teddy 8941 +meyers 8942 +lorre 8943 +denver 8944 +dangers 8945 +architect 8946 +vulnerability 8947 +knives 8948 +dillon 8949 +goo 8950 +numbingly 8951 +inch 8952 +compositions 8953 +flipping 8954 +amoral 8955 +wrath 8956 +rack 8957 +imply 8958 +bonds 8959 +pistol 8960 +perceived 8961 +aura 8962 +tobe 8963 +seventh 8964 +verhoeven's 8965 +insignificant 8966 +simpler 8967 +shatner 8968 +mac 8969 +kornbluth 8970 +barbarian 8971 +zoom 8972 +proudly 8973 +hawaii 8974 +hustler 8975 +penguin 8976 +supports 8977 +thumb 8978 +segal 8979 +fulfill 8980 +bothering 8981 +jurassic 8982 +compromise 8983 +annoyingly 8984 +kenny 8985 +scandal 8986 +overtly 8987 +fleeting 8988 +metropolis 8989 +guru 8990 +rotting 8991 +sixteen 8992 +deadpan 8993 +retrieve 8994 +moderately 8995 +chat 8996 +lang 8997 +simon's 8998 +illusion 8999 +heartless 9000 +backwoods 9001 +climate 9002 +righteous 9003 +beth 9004 +grisly 9005 +prejudices 9006 +immigrants 9007 +alienation 9008 +muscular 9009 +astonishingly 9010 +doses 9011 +traveled 9012 +happier 9013 +electricity 9014 +succession 9015 +cousins 9016 +mandatory 9017 +dental 9018 +breakthrough 9019 +freaked 9020 +clockwork 9021 +ursula 9022 +recurring 9023 +notions 9024 +mechanic 9025 +recovering 9026 +zhang 9027 +comprised 9028 +coverage 9029 +elder 9030 +afghanistan 9031 +trendy 9032 +keeper 9033 +hungarian 9034 +attributes 9035 +brennan 9036 +protecting 9037 +priests 9038 +aztec 9039 +ranger 9040 +recipe 9041 +vienna 9042 +ogre 9043 +farnsworth 9044 +tasks 9045 +romero's 9046 +purse 9047 +subtitled 9048 +lansbury 9049 +pickup 9050 +pals 9051 +unconscious 9052 +animators 9053 +legion 9054 +meanings 9055 +needlessly 9056 +sleuth 9057 +association 9058 +slips 9059 +doris 9060 +pond 9061 +improvised 9062 +relates 9063 +mcdowell 9064 +volumes 9065 +ranging 9066 +zany 9067 +irresistible 9068 +elisha 9069 +herrings 9070 +coppola 9071 +prolonged 9072 +relaxing 9073 +1931 9074 +1938 9075 +rudd 9076 +heir 9077 +innuendo 9078 +urgency 9079 +bloke 9080 +flamboyant 9081 +muriel 9082 +prophet 9083 +reruns 9084 +christensen 9085 +lure 9086 +cracker 9087 +levy 9088 +shakespearean 9089 +encourages 9090 +mockery 9091 +swords 9092 +penis 9093 +pam 9094 +welcomed 9095 +rugged 9096 +academic 9097 +honeymoon 9098 +climbs 9099 +snatch 9100 +overwhelmed 9101 +gays 9102 +roommates 9103 +jolly 9104 +heavens 9105 +placing 9106 +watered 9107 +fable 9108 +zealand 9109 +carnival 9110 +gee 9111 +archer 9112 +locales 9113 +thorn 9114 +smarmy 9115 +kiddie 9116 +farewell 9117 +cheat 9118 +hopeful 9119 +backdrops 9120 +treating 9121 +kamal 9122 +irresponsible 9123 +behalf 9124 +benoit 9125 +unemployed 9126 +backyard 9127 +norton 9128 +stumbling 9129 +theirs 9130 +anonymous 9131 +temporary 9132 +distinguished 9133 +moore's 9134 +inhabited 9135 +wwi 9136 +eastwood's 9137 +pranks 9138 +custody 9139 +yearning 9140 +interspersed 9141 +agatha 9142 +chocolate 9143 +hug 9144 +guided 9145 +martino 9146 +steamy 9147 +feared 9148 +opponents 9149 +crawl 9150 +mans 9151 +jew 9152 +bombing 9153 +assortment 9154 +poke 9155 +imitating 9156 +management 9157 +keitel 9158 +frenzy 9159 +mcadams 9160 +architecture 9161 +spitting 9162 +48 9163 +hector 9164 +fitzgerald 9165 +rko 9166 +redgrave 9167 +induced 9168 +plants 9169 +rusty 9170 +janitor 9171 +weaver 9172 +recreate 9173 +islam 9174 +rogue 9175 +roads 9176 +rewrite 9177 +dodge 9178 +balloon 9179 +honey 9180 +neeson 9181 +conquest 9182 +slug 9183 +wolves 9184 +neglect 9185 +shawn 9186 +concentrated 9187 +tested 9188 +existential 9189 +expanded 9190 +worldwide 9191 +truthful 9192 +unlucky 9193 +liz 9194 +compassionate 9195 +limbs 9196 +impeccable 9197 +dogma 9198 +shattering 9199 +sailors 9200 +peterson 9201 +jock 9202 +rizzo 9203 +kalifornia 9204 +mcdermott 9205 +versatile 9206 +400 9207 +michael's 9208 +naval 9209 +burden 9210 +cheung 9211 +largest 9212 +culkin 9213 +retelling 9214 +muted 9215 +leaps 9216 +theo 9217 +passive 9218 +bucket 9219 +pertwee 9220 +eddy 9221 +rapture 9222 +continuous 9223 +gage 9224 +stretches 9225 +giggle 9226 +marx 9227 +concludes 9228 +stalks 9229 +amok 9230 +adequately 9231 +melt 9232 +stature 9233 +counted 9234 +borderline 9235 +mastermind 9236 +boxes 9237 +posh 9238 +taker 9239 +counterpart 9240 +izzard 9241 +straw 9242 +toe 9243 +shamelessly 9244 +crenna 9245 +tango 9246 +pour 9247 +behaves 9248 +sematary 9249 +expand 9250 +azumi 9251 +country's 9252 +stimulating 9253 +grady 9254 +expressing 9255 +payne 9256 +crass 9257 +intellect 9258 +booker 9259 +dani 9260 +parents' 9261 +lotr 9262 +miyazaki's 9263 +wits 9264 +waving 9265 +traumatized 9266 +illiterate 9267 +chan's 9268 +puzzling 9269 +splitting 9270 +subtleties 9271 +seduction 9272 +condescending 9273 +rebecca 9274 +inherited 9275 +seal 9276 +consisted 9277 +stubborn 9278 +didnt 9279 +lieutenant 9280 +slows 9281 +john's 9282 +glee 9283 +honorable 9284 +'73 9285 +valerie 9286 +smoothly 9287 +poo 9288 +evolved 9289 +darling 9290 +planted 9291 +mold 9292 +supremacy 9293 +opener 9294 +seuss 9295 +craven's 9296 +celine 9297 +hesitate 9298 +conception 9299 +supporters 9300 +revolting 9301 +practices 9302 +orgy 9303 +cheaper 9304 +town's 9305 +forgivable 9306 +nutty 9307 +speechless 9308 +nailed 9309 +associates 9310 +platoon 9311 +disdain 9312 +waits 9313 +knox 9314 +it´s 9315 +collecting 9316 +alligator 9317 +hispanic 9318 +mutated 9319 +woven 9320 +hardest 9321 +lubitsch 9322 +january 9323 +apprentice 9324 +uber 9325 +sarne 9326 +pets 9327 +fawcett 9328 +marred 9329 +elevate 9330 +drivers 9331 +creepiness 9332 +revive 9333 +harlem 9334 +vivah 9335 +kindness 9336 +marathon 9337 +bishop 9338 +gannon 9339 +carole 9340 +brits 9341 +submit 9342 +embarrass 9343 +boyfriends 9344 +dreadfully 9345 +oppressive 9346 +discernible 9347 +intruder 9348 +tourists 9349 +conduct 9350 +rehearsal 9351 +bolivia 9352 +astronaut 9353 +joanna 9354 +grounded 9355 +sessions 9356 +cocktail 9357 +stir 9358 +gimmicks 9359 +archive 9360 +stereotyping 9361 +aweigh 9362 +18th 9363 +undeveloped 9364 +rico 9365 +concentrates 9366 +bruckheimer 9367 +psychiatric 9368 +incompetence 9369 +villagers 9370 +customs 9371 +alienate 9372 +slew 9373 +footsteps 9374 +approximately 9375 +discussions 9376 +blink 9377 +vault 9378 +transformers 9379 +sloane 9380 +choke 9381 +infidelity 9382 +relied 9383 +undertaker 9384 +lovingly 9385 +casually 9386 +luzhin 9387 +disappearing 9388 +historians 9389 +shaolin 9390 +mastroianni 9391 +midler 9392 +atrocities 9393 +bash 9394 +inc 9395 +hedy 9396 +drums 9397 +bonding 9398 +entertainer 9399 +revelations 9400 +holland 9401 +floriane 9402 +downtown 9403 +denied 9404 +connor 9405 +stupidest 9406 +tel 9407 +sinatra's 9408 +lyrical 9409 +woke 9410 +knack 9411 +dripping 9412 +saddest 9413 +loathing 9414 +insects 9415 +hoover 9416 +apologize 9417 +premises 9418 +elmer 9419 +screamed 9420 +lecture 9421 +skipping 9422 +bursts 9423 +noam 9424 +passions 9425 +cocky 9426 +prevalent 9427 +regrets 9428 +suspended 9429 +shack 9430 +democracy 9431 +overacts 9432 +enhances 9433 +deathstalker 9434 +1960 9435 +choreographer 9436 +keeler 9437 +cillian 9438 +contemplate 9439 +smarter 9440 +marlene 9441 +philadelphia 9442 +sammi 9443 +kingsley 9444 +micheal 9445 +mpaa 9446 +duryea 9447 +creeps 9448 +capsule 9449 +converted 9450 +zabriskie 9451 +perceive 9452 +confronting 9453 +administration 9454 +arizona 9455 +viggo 9456 +ecstasy 9457 +candidates 9458 +branch 9459 +passenger 9460 +benson 9461 +sans 9462 +victoria's 9463 +callahan 9464 +intestines 9465 +swamp 9466 +sparse 9467 +request 9468 +overseas 9469 +bass 9470 +surpasses 9471 +organs 9472 +rohmer 9473 +montages 9474 +joshua 9475 +ella 9476 +maguire 9477 +rhys 9478 +cloud 9479 +stripped 9480 +rushes 9481 +kentucky 9482 +tensions 9483 +mom's 9484 +operas 9485 +chapters 9486 +monstrous 9487 +usage 9488 +fugitive 9489 +shaun 9490 +slipped 9491 +documents 9492 +email 9493 +classified 9494 +norwegian 9495 +reception 9496 +ash 9497 +sacrificed 9498 +switzerland 9499 +rightfully 9500 +cruella 9501 +psychologically 9502 +bury 9503 +liar 9504 +clumsily 9505 +crow 9506 +mindset 9507 +untrue 9508 +barker 9509 +lange 9510 +toro 9511 +ahmad 9512 +wipe 9513 +sixty 9514 +brink 9515 +insanely 9516 +mourning 9517 +vets 9518 +wu 9519 +1956 9520 +restless 9521 +loop 9522 +fanatics 9523 +rests 9524 +guevara 9525 +connecting 9526 +city's 9527 +friendships 9528 +satellite 9529 +empathize 9530 +surfers 9531 +immersed 9532 +mostel 9533 +squeeze 9534 +backing 9535 +admirably 9536 +confirm 9537 +equals 9538 +vengeful 9539 +pauses 9540 +snippets 9541 +mamet 9542 +that'll 9543 +anchorman 9544 +dense 9545 +strikingly 9546 +daphne 9547 +misplaced 9548 +1941 9549 +streak 9550 +shrink 9551 +garnered 9552 +breathless 9553 +hiv 9554 +delve 9555 +grain 9556 +spectrum 9557 +dusty 9558 +durbin 9559 +locks 9560 +november 9561 +o'neill 9562 +crook 9563 +render 9564 +participation 9565 +deception 9566 +replay 9567 +apartments 9568 +sr 9569 +lawyers 9570 +requisite 9571 +telly 9572 +basil 9573 +kinky 9574 +assist 9575 +spectacularly 9576 +scantily 9577 +prevented 9578 +obscene 9579 +reincarnation 9580 +morgana 9581 +bout 9582 +looney 9583 +adventurous 9584 +sykes 9585 +maverick 9586 +lucio 9587 +travelling 9588 +diabolical 9589 +capt 9590 +promotion 9591 +partial 9592 +eater 9593 +dime 9594 +bathing 9595 +criminally 9596 +underdog 9597 +interpret 9598 +suggestive 9599 +springs 9600 +graves 9601 +spielberg's 9602 +technological 9603 +wan 9604 +cortez 9605 +proverbial 9606 +granger 9607 +phrases 9608 +societies 9609 +thankful 9610 +palette 9611 +outrage 9612 +betrays 9613 +lung 9614 +marquis 9615 +ing 9616 +regal 9617 +oriental 9618 +duties 9619 +whacked 9620 +kerr 9621 +documented 9622 +700 9623 +stoic 9624 +fairytale 9625 +listing 9626 +acknowledged 9627 +allison 9628 +matching 9629 +longtime 9630 +garcia 9631 +elliot 9632 +33 9633 +adopt 9634 +flea 9635 +carlito's 9636 +1940's 9637 +coleman 9638 +draft 9639 +witless 9640 +kramer 9641 +haha 9642 +lap 9643 +alternately 9644 +1930 9645 +sentenced 9646 +harry's 9647 +daisies 9648 +overt 9649 +mining 9650 +stepped 9651 +eliminate 9652 +chains 9653 +regain 9654 +nuance 9655 +italians 9656 +hurting 9657 +honour 9658 +sealed 9659 +societal 9660 +indifference 9661 +lombard 9662 +teamed 9663 +cathy 9664 +its' 9665 +unfinished 9666 +floors 9667 +downside 9668 +tucker 9669 +paperhouse 9670 +compound 9671 +eggs 9672 +underused 9673 +incarnation 9674 +hunk 9675 +goer 9676 +presumed 9677 +caruso 9678 +interpreted 9679 +colourful 9680 +stills 9681 +caroline 9682 +keyboard 9683 +claw 9684 +snappy 9685 +camps 9686 +crop 9687 +sheet 9688 +overnight 9689 +dung 9690 +booze 9691 +risks 9692 +rub 9693 +oddball 9694 +exhibit 9695 +anchor 9696 +fireworks 9697 +batwoman 9698 +gesture 9699 +skinned 9700 +undertones 9701 +achieving 9702 +lanza 9703 +goofs 9704 +flee 9705 +recalls 9706 +stable 9707 +fantastically 9708 +exposing 9709 +shakes 9710 +addressing 9711 +prototype 9712 +carface 9713 +hes 9714 +competently 9715 +retain 9716 +schemes 9717 +hogan 9718 +voting 9719 +episodic 9720 +occurring 9721 +topped 9722 +1954 9723 +norma 9724 +chore 9725 +chang 9726 +shouts 9727 +rainer 9728 +colonial 9729 +recreation 9730 +forum 9731 +companions 9732 +apologies 9733 +insulted 9734 +holidays 9735 +throwaway 9736 +tepid 9737 +darkest 9738 +pulse 9739 +pita 9740 +superiors 9741 +grumpy 9742 +illustrates 9743 +sweetheart 9744 +showtime 9745 +aiello 9746 +btk 9747 +cbc 9748 +baseketball 9749 +horizon 9750 +eliminated 9751 +weirdo 9752 +welch 9753 +stepping 9754 +leno 9755 +beau 9756 +affections 9757 +leopold 9758 +inheritance 9759 +masturbation 9760 +itchy 9761 +locker 9762 +universally 9763 +shadowy 9764 +employ 9765 +skywalker 9766 +grips 9767 +gardens 9768 +sorvino 9769 +expertise 9770 +irwin 9771 +t'aime 9772 +babysitter 9773 +bryan 9774 +positions 9775 +coarse 9776 +tremors 9777 +iceberg 9778 +monumental 9779 +thinner 9780 +allegedly 9781 +dominick 9782 +allied 9783 +bogdanovich 9784 +raving 9785 +supplies 9786 +kaufman 9787 +sacred 9788 +shootings 9789 +primal 9790 +hiring 9791 +hockey 9792 +flamenco 9793 +thirteen 9794 +carlito 9795 +polite 9796 +exudes 9797 +gaining 9798 +darius 9799 +quarters 9800 +willem 9801 +crummy 9802 +duff 9803 +sorta 9804 +rigid 9805 +eponymous 9806 +smitten 9807 +attributed 9808 +variations 9809 +mischievous 9810 +unborn 9811 +wayne's 9812 +circuit 9813 +integrated 9814 +unimpressive 9815 +carson 9816 +150 9817 +siege 9818 +endured 9819 +surrogate 9820 +gifts 9821 +practicing 9822 +disgruntled 9823 +drifter 9824 +renowned 9825 +chef 9826 +operatic 9827 +maiden 9828 +frenetic 9829 +wal 9830 +roaring 9831 +author's 9832 +wondrous 9833 +greta 9834 +gamut 9835 +marital 9836 +gym 9837 +offerings 9838 +zatoichi 9839 +emerged 9840 +exaggeration 9841 +planets 9842 +raft 9843 +connolly 9844 +mcintire 9845 +strangest 9846 +marvellous 9847 +runtime 9848 +misfire 9849 +extremes 9850 +swift 9851 +seinfeld 9852 +jackass 9853 +harmony 9854 +plantation 9855 +bravery 9856 +pavarotti 9857 +catastrophe 9858 +malcolm 9859 +portman 9860 +solving 9861 +albums 9862 +winston 9863 +corky 9864 +allegory 9865 +spears 9866 +saif 9867 +goof 9868 +outta 9869 +virtues 9870 +monstrosity 9871 +ideology 9872 +edits 9873 +celebrating 9874 +adapting 9875 +ferry 9876 +desolate 9877 +jessie 9878 +inflicted 9879 +rocker 9880 +projection 9881 +irs 9882 +cambodia 9883 +enthralled 9884 +ensuing 9885 +leia 9886 +o'toole 9887 +transferred 9888 +exposes 9889 +competing 9890 +yourselves 9891 +sentiments 9892 +kisses 9893 +stray 9894 +turgid 9895 +declares 9896 +nuns 9897 +mercilessly 9898 +it'd 9899 +exceedingly 9900 +ted's 9901 +insecure 9902 +ben's 9903 +tanks 9904 +kusturica 9905 +spaces 9906 +spliced 9907 +sheila 9908 +crowds 9909 +balcony 9910 +menu 9911 +lamas 9912 +diver 9913 +secluded 9914 +integral 9915 +redeemed 9916 +halt 9917 +decapitated 9918 +stealth 9919 +budgeted 9920 +voters 9921 +overweight 9922 +praying 9923 +stevenson 9924 +cleveland 9925 +stakes 9926 +mattei 9927 +charity 9928 +stalk 9929 +olympia 9930 +olympic 9931 +aspirations 9932 +decoration 9933 +slack 9934 +bullying 9935 +bum 9936 +mo 9937 +capitalize 9938 +jameson 9939 +skimpy 9940 +wicker 9941 +starving 9942 +frenchman 9943 +frye 9944 +ate 9945 +monastery 9946 +wb 9947 +hayden 9948 +banana 9949 +grandparents 9950 +vacuous 9951 +willy 9952 +darkman 9953 +neutral 9954 +rumors 9955 +somber 9956 +aunts 9957 +amateurs 9958 +radar 9959 +ounce 9960 +bagdad 9961 +stud 9962 +closeups 9963 +insisted 9964 +jed 9965 +geeky 9966 +64 9967 +aims 9968 +complains 9969 +ewan 9970 +exhausted 9971 +day's 9972 +weaves 9973 +gladly 9974 +misogynistic 9975 +soles 9976 +michel 9977 +uniquely 9978 +interminable 9979 +aristocrat 9980 +paul's 9981 +everybody's 9982 +avant 9983 +answering 9984 +smallest 9985 +contacts 9986 +enlightenment 9987 +murphy's 9988 +employs 9989 +unforgivable 9990 +punchline 9991 +culminating 9992 +talentless 9993 +grabbing 9994 +soulless 9995 +unfairly 9996 +grail 9997 +retrospect 9998 +edged 9999 diff --git a/tensorflow_lite_support/cc/test/text/regex_tokenizer_test.cc b/tensorflow_lite_support/cc/test/text/regex_tokenizer_test.cc new file mode 100644 index 00000000..4106c12a --- /dev/null +++ b/tensorflow_lite_support/cc/test/text/regex_tokenizer_test.cc @@ -0,0 +1,138 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_lite_support/cc/text/tokenizers/regex_tokenizer.h" + +#include "android-base/file.h" +#include "tensorflow_lite_support/cc/port/gmock.h" +#include "tensorflow_lite_support/cc/port/gtest.h" +#include "tensorflow_lite_support/cc/task/core/task_utils.h" + +namespace tflite { +namespace support { +namespace text { +namespace tokenizer { + +using ::android::base::GetExecutableDirectory; +using ::testing::ElementsAre; +using ::tflite::task::core::LoadBinaryContent; + +namespace { +constexpr char kTestRegexVocabSubPath[] = + "/tensorflow_lite_support/cc/test/testdata/task/text/" + "vocab_for_regex_tokenizer.txt"; + +constexpr char kTestRegexEmptyVocabSubPath[] = + "/tensorflow_lite_support/cc/test/testdata/task/text/" + "empty_vocab_for_regex_tokenizer.txt"; + +constexpr char kRegex[] = "[^\\w\\']+"; + +TEST(RegexTokenizerTest, TestTokenize) { + std::string test_regex_vocab_path = + absl::StrCat(GetExecutableDirectory(), kTestRegexVocabSubPath); + auto tokenizer = + absl::make_unique<RegexTokenizer>(kRegex, test_regex_vocab_path); + auto results = tokenizer->Tokenize("good morning, i'm your teacher.\n"); + EXPECT_THAT(results.subwords, + ElementsAre("good", "morning", "i'm", "your", "teacher")); +} + +TEST(RegexTokenizerTest, TestTokenizeFromFileBuffer) { + std::string test_regex_vocab_path = + absl::StrCat(GetExecutableDirectory(), kTestRegexVocabSubPath); + std::string buffer = LoadBinaryContent(test_regex_vocab_path.c_str()); + auto tokenizer = + absl::make_unique<RegexTokenizer>(kRegex, buffer.data(), buffer.size()); + auto results = tokenizer->Tokenize("good morning, i'm your teacher.\n"); + EXPECT_THAT(results.subwords, + ElementsAre("good", "morning", "i'm", "your", "teacher")); +} + +TEST(RegexTokenizerTest, TestLookupId) { + std::string test_regex_vocab_path = + absl::StrCat(GetExecutableDirectory(), kTestRegexVocabSubPath); + auto tokenizer = + absl::make_unique<RegexTokenizer>(kRegex, test_regex_vocab_path); + std::vector<std::string> subwords = {"good", "morning", "i'm", "your", + "teacher"}; + std::vector<int> true_ids = {52, 1972, 146, 129, 1750}; + int id; + for (int i = 0; i < subwords.size(); i++) { + ASSERT_TRUE(tokenizer->LookupId(subwords[i], &id)); + ASSERT_EQ(id, true_ids[i]); + } +} + +TEST(RegexTokenizerTest, TestLookupWord) { + std::string test_regex_vocab_path = + absl::StrCat(GetExecutableDirectory(), kTestRegexVocabSubPath); + auto tokenizer = + absl::make_unique<RegexTokenizer>(kRegex, test_regex_vocab_path); + std::vector<int> ids = {52, 1972, 146, 129, 1750}; + std::vector<std::string> subwords = {"good", "morning", "i'm", "your", + "teacher"}; + absl::string_view result; + for (int i = 0; i < ids.size(); i++) { + ASSERT_TRUE(tokenizer->LookupWord(ids[i], &result)); + ASSERT_EQ(result, subwords[i]); + } +} + +TEST(RegexTokenizerTest, TestGetSpecialTokens) { + // The vocab the following tokens: + // <PAD> 0 + // <START> 1 + // <UNKNOWN> 2 + std::string test_regex_vocab_path = + absl::StrCat(GetExecutableDirectory(), kTestRegexVocabSubPath); + auto tokenizer = + absl::make_unique<RegexTokenizer>(kRegex, test_regex_vocab_path); + + int start_token; + ASSERT_TRUE(tokenizer->GetStartToken(&start_token)); + ASSERT_EQ(start_token, 1); + + int pad_token; + ASSERT_TRUE(tokenizer->GetPadToken(&pad_token)); + ASSERT_EQ(pad_token, 0); + + int unknown_token; + ASSERT_TRUE(tokenizer->GetUnknownToken(&unknown_token)); + ASSERT_EQ(unknown_token, 2); +} + +TEST(RegexTokenizerTest, TestGetSpecialTokensFailure) { + std::string test_regex_empty_vocab_path = + absl::StrCat(GetExecutableDirectory(), kTestRegexEmptyVocabSubPath); + auto tokenizer = + absl::make_unique<RegexTokenizer>(kRegex, test_regex_empty_vocab_path); + + int start_token; + ASSERT_FALSE(tokenizer->GetStartToken(&start_token)); + + int pad_token; + ASSERT_FALSE(tokenizer->GetPadToken(&pad_token)); + + int unknown_token; + ASSERT_FALSE(tokenizer->GetUnknownToken(&unknown_token)); +} + +} // namespace + +} // namespace tokenizer +} // namespace text +} // namespace support +} // namespace tflite diff --git a/tensorflow_lite_support/cc/text/tokenizers/BUILD b/tensorflow_lite_support/cc/text/tokenizers/BUILD deleted file mode 100644 index 3ad8da2f..00000000 --- a/tensorflow_lite_support/cc/text/tokenizers/BUILD +++ /dev/null @@ -1,191 +0,0 @@ -# This package contains C++ support libraries that Java libraries can invoke. -load("@build_bazel_rules_android//android:rules.bzl", "android_library") -load( - "@org_tensorflow//tensorflow/lite:build_def.bzl", - "tflite_copts", - "tflite_jni_binary", -) - -package( - default_visibility = ["//tensorflow_lite_support:users"], - licenses = ["notice"], # Apache 2.0 -) - -cc_library( - name = "tokenizer", - hdrs = [ - "tokenizer.h", - ], - deps = [ - "@com_google_absl//absl/strings", - ], -) - -cc_library( - name = "tokenizer_jni_lib", - srcs = [ - "tokenizer_jni_lib.cc", - ], - hdrs = [ - "tokenizer_jni_lib.h", - ], - deps = [ - ":tokenizer", - "//tensorflow_lite_support/cc/utils:jni_utils", - "@org_tensorflow//tensorflow/lite/java/jni", - ], -) - -cc_library( - name = "bert_tokenizer", - srcs = [ - "bert_tokenizer.cc", - ], - hdrs = [ - "bert_tokenizer.h", - ], - deps = [ - ":tokenizer", - "//tensorflow_lite_support/cc/port:integral_types", - "//tensorflow_lite_support/cc/utils:common_utils", - "@com_google_absl//absl/container:flat_hash_map", - "@com_googlesource_code_re2//:re2", - "@org_tensorflow_text//tensorflow_text/core/kernels:regex_split", - "@org_tensorflow_text//tensorflow_text/core/kernels:wordpiece_tokenizer", - ], -) - -cc_library( - name = "bert_tokenizer_jni_lib", - srcs = [ - "bert_tokenizer_jni.cc", - ], - copts = tflite_copts(), - linkopts = [ - "-lm", - "-ldl", - ], - deps = [ - ":bert_tokenizer", - ":tokenizer_jni_lib", - "//tensorflow_lite_support/cc/utils:jni_utils", - "@com_google_absl//absl/memory", - "@org_tensorflow//tensorflow/lite/java/jni", - ], - alwayslink = 1, -) - -tflite_jni_binary( - name = "libbert_tokenizer_jni.so", - deps = [ - ":bert_tokenizer_jni_lib", - ], -) - -cc_library( - name = "bert_tokenizer_runtime", - srcs = ["libbert_tokenizer_jni.so"], - alwayslink = 1, -) - -android_library( - name = "bert_tokenizer_jni", - custom_package = "org.tensorflow.lite.support.text", - manifest = "DummyManifest.xml", - resource_files = [], - deps = [ - ":bert_tokenizer_runtime", # build_cleaner: skip - ], -) - -cc_library( - name = "sentencepiece_tokenizer", - hdrs = [ - "sentencepiece_tokenizer.h", - ], - deps = [ - ":tokenizer", - "@com_google_sentencepiece//src:sentencepiece_processor", - ], -) - -cc_library( - name = "sentencepiece_jni_lib", - srcs = [ - "sentencepiece_jni.cc", - ], - copts = tflite_copts(), - linkopts = [ - "-lm", - "-ldl", - ], - deps = [ - ":sentencepiece_tokenizer", - ":tokenizer_jni_lib", - "//tensorflow_lite_support/cc/utils:jni_utils", - "@com_google_absl//absl/memory", - "@com_google_absl//absl/strings", - "@org_tensorflow//tensorflow/lite/java/jni", - ], - alwayslink = 1, -) - -cc_library( - name = "sentencepiece_runtime", - srcs = ["libsentencepiece_jni.so"], - alwayslink = 1, -) - -tflite_jni_binary( - name = "libsentencepiece_jni.so", - deps = [ - ":sentencepiece_jni_lib", - ], -) - -android_library( - name = "sentencepiece_jni", - custom_package = "org.tensorflow.lite.support.text", - manifest = "DummyManifest.xml", - resource_files = [], - deps = [ - ":sentencepiece_runtime", # build_cleaner: skip - ], -) - -cc_library( - name = "tokenizer_utils", - srcs = ["tokenizer_utils.cc"], - hdrs = [ - "tokenizer_utils.h", - ], - deps = [ - ":bert_tokenizer", - ":regex_tokenizer", - ":sentencepiece_tokenizer", - ":tokenizer", - "//tensorflow_lite_support/cc:common", - "//tensorflow_lite_support/cc/port:status_macros", - "//tensorflow_lite_support/cc/port:statusor", - "//tensorflow_lite_support/metadata:metadata_schema_cc", - "//tensorflow_lite_support/metadata/cc:metadata_extractor", - "@com_google_absl//absl/status", - ], -) - -cc_library( - name = "regex_tokenizer", - srcs = [ - "regex_tokenizer.cc", - ], - hdrs = [ - "regex_tokenizer.h", - ], - deps = [ - ":tokenizer", - "//tensorflow_lite_support/cc/utils:common_utils", - "@com_google_absl//absl/container:node_hash_map", - "@com_google_absl//absl/strings", - "@com_googlesource_code_re2//:re2", - ], -) diff --git a/tensorflow_lite_support/cc/text/tokenizers/regex_tokenizer.cc b/tensorflow_lite_support/cc/text/tokenizers/regex_tokenizer.cc index 38aff880..a97d20e2 100644 --- a/tensorflow_lite_support/cc/text/tokenizers/regex_tokenizer.cc +++ b/tensorflow_lite_support/cc/text/tokenizers/regex_tokenizer.cc @@ -16,6 +16,7 @@ limitations under the License. #include "tensorflow_lite_support/cc/text/tokenizers/regex_tokenizer.h" #include <iostream> +#include <regex> #include "absl/strings/str_cat.h" #include "absl/strings/substitute.h" @@ -40,10 +41,6 @@ void buildIndexTokenMap( } // namespace -// RE2::FindAndConsume requires the delim_re_ to have a matching group in order -// to capture the matched delimiter length. Surround the regex with a -// parenthesis to create a matching group, it's fine if the regex is already -// surrounded by parenthesis. RegexTokenizer::RegexTokenizer(const std::string& regex_pattern, const std::string& path_to_vocab) : delim_re_{absl::Substitute("($0)", regex_pattern)}, @@ -61,29 +58,22 @@ RegexTokenizer::RegexTokenizer(const std::string& regex_pattern, } TokenizerResult RegexTokenizer::Tokenize(const std::string& input) { - absl::string_view leftover(input.data()); - absl::string_view last_end = leftover; - TokenizerResult result; // Keep looking for split points until we have reached the end of the input. - absl::string_view extracted_delim_token; - while (RE2::FindAndConsume(&leftover, delim_re_, &extracted_delim_token)) { - absl::string_view token(last_end.data(), - extracted_delim_token.data() - last_end.data()); - bool has_non_empty_token = token.length() > 0; - - last_end = leftover; - - // Mark the end of the previous token, only if there was something. - if (has_non_empty_token) { - result.subwords.push_back(std::string(token)); + // TODO (ag/17748161): Using smatch here introduces inefficient string copying; optimize if necessary. + std::string leftover = input; + std::smatch token; + while(std::regex_search(leftover, token, delim_re_)) { + if (token.length() > 0) { + result.subwords.push_back(token.prefix().str()); } + leftover = token.suffix().str(); } // Close the last token. if (!leftover.empty()) { - result.subwords.push_back(std::string(leftover)); + result.subwords.push_back(leftover); } return result; diff --git a/tensorflow_lite_support/cc/text/tokenizers/regex_tokenizer.h b/tensorflow_lite_support/cc/text/tokenizers/regex_tokenizer.h index c53ae496..3afafb79 100644 --- a/tensorflow_lite_support/cc/text/tokenizers/regex_tokenizer.h +++ b/tensorflow_lite_support/cc/text/tokenizers/regex_tokenizer.h @@ -16,8 +16,8 @@ limitations under the License. #ifndef TENSORFLOW_LITE_SUPPORT_CC_TEXT_TOKENIZERS_REGEX_TOKENIZER_H_ #define TENSORFLOW_LITE_SUPPORT_CC_TEXT_TOKENIZERS_REGEX_TOKENIZER_H_ +#include <regex> #include "absl/container/node_hash_map.h" -#include "re2/re2.h" #include "tensorflow_lite_support/cc/text/tokenizers/tokenizer.h" namespace tflite { @@ -46,7 +46,7 @@ class RegexTokenizer : public Tokenizer { bool GetUnknownToken(int* unknown_token); private: - RE2 delim_re_; + std::regex delim_re_; absl::node_hash_map<std::string, int> token_index_map_; absl::node_hash_map<int, absl::string_view> index_token_map_; }; diff --git a/tensorflow_lite_support/cc/text/tokenizers/tokenizer_utils.cc b/tensorflow_lite_support/cc/text/tokenizers/tokenizer_utils.cc index 3e81c478..7e7eb69c 100644 --- a/tensorflow_lite_support/cc/text/tokenizers/tokenizer_utils.cc +++ b/tensorflow_lite_support/cc/text/tokenizers/tokenizer_utils.cc @@ -1,4 +1,4 @@ -/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,18 +18,15 @@ limitations under the License. #include "absl/status/status.h" #include "tensorflow_lite_support/cc/common.h" #include "tensorflow_lite_support/cc/port/status_macros.h" -#include "tensorflow_lite_support/cc/text/tokenizers/bert_tokenizer.h" #include "tensorflow_lite_support/cc/text/tokenizers/regex_tokenizer.h" -#include "tensorflow_lite_support/cc/text/tokenizers/sentencepiece_tokenizer.h" #include "tensorflow_lite_support/metadata/metadata_schema_generated.h" +#include "utils/bert_tokenizer.h" namespace tflite { namespace support { namespace text { namespace tokenizer { - -using ::tflite::ProcessUnit; using ::tflite::SentencePieceTokenizerOptions; using ::tflite::support::CreateStatusWithPayload; using ::tflite::support::StatusOr; @@ -71,18 +68,8 @@ StatusOr<std::unique_ptr<Tokenizer>> CreateTokenizerFromProcessUnit( ASSIGN_OR_RETURN(absl::string_view vocab_buffer, CheckAndLoadFirstAssociatedFile(options->vocab_file(), metadata_extractor)); - return absl::make_unique<BertTokenizer>(vocab_buffer.data(), - vocab_buffer.size()); - } - case ProcessUnitOptions_SentencePieceTokenizerOptions: { - const tflite::SentencePieceTokenizerOptions* options = - tokenizer_process_unit->options_as<SentencePieceTokenizerOptions>(); - ASSIGN_OR_RETURN(absl::string_view model_buffer, - CheckAndLoadFirstAssociatedFile( - options->sentencePiece_model(), metadata_extractor)); - // TODO(b/160647204): Extract sentence piece model vocabulary - return absl::make_unique<SentencePieceTokenizer>(model_buffer.data(), - model_buffer.size()); + return absl::make_unique<libtextclassifier3::BertTokenizer>( + vocab_buffer.data(), vocab_buffer.size()); } case ProcessUnitOptions_RegexTokenizerOptions: { const tflite::RegexTokenizerOptions* options = diff --git a/tensorflow_lite_support/custom_ops/kernel/BUILD b/tensorflow_lite_support/custom_ops/kernel/BUILD index b9b11de9..a55dcb95 100644 --- a/tensorflow_lite_support/custom_ops/kernel/BUILD +++ b/tensorflow_lite_support/custom_ops/kernel/BUILD @@ -144,3 +144,83 @@ py_test( "@absl_py//absl/testing:parameterized", ], ) + +cc_library( + name = "unsorted_segment", + srcs = ["unsorted_segment.cc"], + hdrs = ["unsorted_segment.h"], + deps = [ + "@org_tensorflow//tensorflow/lite/c:common", + "@org_tensorflow//tensorflow/lite/kernels:kernel_util", + "@org_tensorflow//tensorflow/lite/kernels/internal:reference_base", + "@org_tensorflow//tensorflow/lite/kernels/internal:tensor", + ], +) + +cc_test( + name = "unsorted_segment_prod_test", + size = "small", + srcs = [ + "unsorted_segment_prod_test.cc", + "unsorted_segment_test.cc", + "unsorted_segment_test.h", + ], + deps = [ + "@com_google_googletest//:gtest_main", + "@org_tensorflow//tensorflow/lite/c:common", + "@org_tensorflow//tensorflow/lite/kernels:test_main", + "@org_tensorflow//tensorflow/lite/kernels:test_util", + "@org_tensorflow//tensorflow/lite/schema:schema_fbs", + ], +) + +cc_test( + name = "unsorted_segment_max_test", + size = "small", + srcs = [ + "unsorted_segment_max_test.cc", + "unsorted_segment_test.cc", + "unsorted_segment_test.h", + ], + deps = [ + "@com_google_googletest//:gtest_main", + "@org_tensorflow//tensorflow/lite/c:common", + "@org_tensorflow//tensorflow/lite/kernels:test_main", + "@org_tensorflow//tensorflow/lite/kernels:test_util", + "@org_tensorflow//tensorflow/lite/schema:schema_fbs", + ], +) + +cc_test( + name = "unsorted_segment_sum_test", + size = "small", + srcs = [ + "unsorted_segment_sum_test.cc", + "unsorted_segment_test.cc", + "unsorted_segment_test.h", + ], + deps = [ + "@com_google_googletest//:gtest_main", + "@org_tensorflow//tensorflow/lite/c:common", + "@org_tensorflow//tensorflow/lite/kernels:test_main", + "@org_tensorflow//tensorflow/lite/kernels:test_util", + "@org_tensorflow//tensorflow/lite/schema:schema_fbs", + ], +) + +cc_test( + name = "unsorted_segment_min_test", + size = "small", + srcs = [ + "unsorted_segment_sum_test.cc", + "unsorted_segment_test.cc", + "unsorted_segment_test.h", + ], + deps = [ + "@com_google_googletest//:gtest_main", + "@org_tensorflow//tensorflow/lite/c:common", + "@org_tensorflow//tensorflow/lite/kernels:test_main", + "@org_tensorflow//tensorflow/lite/kernels:test_util", + "@org_tensorflow//tensorflow/lite/schema:schema_fbs", + ], +) diff --git a/tensorflow_lite_support/custom_ops/kernel/unsorted_segment.cc b/tensorflow_lite_support/custom_ops/kernel/unsorted_segment.cc new file mode 100644 index 00000000..15972ed6 --- /dev/null +++ b/tensorflow_lite_support/custom_ops/kernel/unsorted_segment.cc @@ -0,0 +1,295 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include <stdint.h> + +#include <algorithm> +#include <functional> + +#include "tensorflow_lite_support/custom_ops/kernel/unsorted_segment.h" + +#include "tensorflow/lite/c/common.h" +#include "tensorflow/lite/kernels/internal/tensor_ctypes.h" +#include "tensorflow/lite/kernels/kernel_util.h" + +namespace tflite { +namespace ops { +namespace custom { +namespace unsorted_segment { + +enum SegmentType { + kSegmentMax, + kSegmentMin, + kSegmentProd, + kSegmentSum, +}; + +static const int kInputDataTensor = 0; +static const int kInputSegmentIdsTensor = 1; +static const int kInputNumSegmentsTensor = 2; +static const int kOutputTensor = 0; + +inline bool IsConstantOrPersistentTensor(const TfLiteTensor* tensor) { + return tflite::IsConstantTensor(tensor) || + (tensor->allocation_type == kTfLitePersistentRo); +} + +template <typename T, template <typename T2> typename Op> +void UnsortedSegmentRef(const tflite::RuntimeShape& input_shape, + const T* input_data, + const tflite::RuntimeShape& segment_ids_shape, + const int32_t* segment_ids_data, + const tflite::RuntimeShape& output_shape, + T* output_data) { + for (int i = 0; i < output_shape.FlatSize(); ++i) { + output_data[i] = Op<T>::kInitialValue; + } + Op<T> op; + int segment_flat_size = 1; + for (int i = 1; i < output_shape.DimensionsCount(); ++i) { + segment_flat_size *= output_shape.Dims(i); + } + for (int i = 0; i < segment_ids_shape.FlatSize(); i++) { + int output_index = segment_ids_data[i]; + if (output_index < 0) continue; + for (int j = 0; j < segment_flat_size; ++j) { + output_data[output_index * segment_flat_size + j] = + op(output_data[output_index * segment_flat_size + j], + input_data[i * segment_flat_size + j]); + } + } +} + +TfLiteStatus ResizeOutputTensor(TfLiteContext* context, + const TfLiteTensor* data, + const TfLiteTensor* segment_ids, + const TfLiteTensor* num_segments, + TfLiteTensor* output) { + // The shape of segment_ids is permitted to be any non-empty prefix of + // the input data's shape. The shape of output's first dimension is always + // equal to num_segments. The remaining dimensions of output's shape are then + // taken to be the suffix of input shape after rank(segment_ids)th position. + // Public facing tensorflow erroneously describe unsorted_segment ops as only + // supporting segment_ids of rank 1, however tensorflow implementation + // supports higher dimensional segment_ids as described. + const int segment_ids_rank = tflite::NumDimensions(segment_ids); + const int data_rank = tflite::NumDimensions(data); + TF_LITE_ENSURE(context, segment_ids_rank <= data_rank); + for (int i = 0; i < segment_ids_rank; ++i) { + // segment_ids shape must be prefix of data shape. + TF_LITE_ENSURE_EQ(context, segment_ids->dims->data[i], data->dims->data[i]); + } + TF_LITE_ENSURE(context, (num_segments->dims->size == 1 && + num_segments->dims->data[0] == 1) || + num_segments->dims->size == 0); + // num_segments can be thought of as number of buckets (segments) in output, + // where each segment is the reduction of all elements mapped to that + // segment_ids. The shape of said elements is the respective + // suffix of the data shape. + int32_t num_segments_ = tflite::GetTensorData<int32_t>(num_segments)[0]; + const int num_segment_ids = tflite::NumElements(segment_ids); + int max_index = -1; + for (int i = 0; i < num_segment_ids; i++) { + max_index = std::max(tflite::GetTensorData<int32_t>(segment_ids)[i], max_index); + } + // num_segments_ must be at greater than max_index else would map elements + // to non existent output segments. + TF_LITE_ENSURE(context, max_index < num_segments_); + const int output_rank = data_rank - segment_ids_rank + 1; + TfLiteIntArray* output_shape = TfLiteIntArrayCreate(output_rank); + output_shape->data[0] = num_segments_; + // output_shape[1:] should be data_shape[Rank(segment_ids):] + for (int i = segment_ids_rank; i < data_rank; ++i) { + output_shape->data[i - segment_ids_rank + 1] = data->dims->data[i]; + } + return context->ResizeTensor(context, output, output_shape); +} + +TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { + TF_LITE_ENSURE_EQ(context, tflite::NumInputs(node), 3); + TF_LITE_ENSURE_EQ(context, tflite::NumOutputs(node), 1); + const TfLiteTensor* data; + TF_LITE_ENSURE_OK(context, + tflite::GetInputSafe(context, node, kInputDataTensor, &data)); + const TfLiteTensor* segment_ids; + TF_LITE_ENSURE_OK(context, tflite::GetInputSafe(context, node, kInputSegmentIdsTensor, + &segment_ids)); + const TfLiteTensor* num_segments; + TF_LITE_ENSURE_OK( + context, + tflite::GetInputSafe(context, node, kInputNumSegmentsTensor, &num_segments)); + TfLiteTensor* output; + TF_LITE_ENSURE_OK(context, + tflite::GetOutputSafe(context, node, kOutputTensor, &output)); + TF_LITE_ENSURE(context, + data->type == kTfLiteInt32 || data->type == kTfLiteFloat32); + TF_LITE_ENSURE_EQ(context, segment_ids->type, kTfLiteInt32); + TF_LITE_ENSURE_EQ(context, num_segments->type, kTfLiteInt32); + + if (tflite::IsDynamicTensor(data) || !IsConstantOrPersistentTensor(segment_ids) || + !IsConstantOrPersistentTensor(num_segments)) { + tflite::SetTensorToDynamic(output); + return kTfLiteOk; + } + return ResizeOutputTensor(context, data, segment_ids, num_segments, output); +} + +template <typename T> +struct SegmenMax { + inline T operator()(const T& a, const T& b) const { return std::max(a, b); } + static constexpr T kInitialValue = std::numeric_limits<T>::lowest(); +}; + +template <typename T> +struct SegmenMin { + inline T operator()(const T& a, const T& b) const { return std::min(a, b); } + static constexpr T kInitialValue = std::numeric_limits<T>::max(); +}; + +template <typename T> +struct SegmenProd { + inline T operator()(const T& a, const T& b) const { return a * b; } + static constexpr T kInitialValue = T(1); +}; + +template <typename T> +struct SegmenSum { + inline T operator()(const T& a, const T& b) const { return a + b; } + static constexpr T kInitialValue = T(0); +}; + +template <typename T> +TfLiteStatus EvalType(TfLiteContext* context, const tflite::RuntimeShape& input_shape, + const T* input_data, + const tflite::RuntimeShape& segment_ids_shape, + const int32_t* segment_ids_data, + const tflite::RuntimeShape& output_shape, T* output_data, + SegmentType segment_type) { + switch (segment_type) { + case kSegmentProd: + unsorted_segment::UnsortedSegmentRef<T, SegmenProd>( + input_shape, input_data, segment_ids_shape, segment_ids_data, + output_shape, output_data); + break; + case kSegmentMax: + unsorted_segment::UnsortedSegmentRef<T, SegmenMax>( + input_shape, input_data, segment_ids_shape, segment_ids_data, + output_shape, output_data); + break; + case kSegmentSum: + unsorted_segment::UnsortedSegmentRef<T, SegmenSum>( + input_shape, input_data, segment_ids_shape, segment_ids_data, + output_shape, output_data); + break; + case kSegmentMin: + unsorted_segment::UnsortedSegmentRef<T, SegmenMin>( + input_shape, input_data, segment_ids_shape, segment_ids_data, + output_shape, output_data); + break; + default: + TF_LITE_KERNEL_LOG(context, "Not recognized segment type: %d", + segment_type); + return kTfLiteError; + } + return kTfLiteOk; +} + +TfLiteStatus EvalGeneric(TfLiteContext* context, TfLiteNode* node, + SegmentType segment_type) { + const TfLiteTensor* data; + TF_LITE_ENSURE_OK(context, + tflite::GetInputSafe(context, node, kInputDataTensor, &data)); + const TfLiteTensor* segment_ids; + TF_LITE_ENSURE_OK(context, tflite::GetInputSafe(context, node, kInputSegmentIdsTensor, + &segment_ids)); + const TfLiteTensor* num_segments; + TF_LITE_ENSURE_OK( + context, + tflite::GetInputSafe(context, node, kInputNumSegmentsTensor, &num_segments)); + TfLiteTensor* output; + TF_LITE_ENSURE_OK(context, + tflite::GetOutputSafe(context, node, kOutputTensor, &output)); + + if (tflite::IsDynamicTensor(output)) { + TF_LITE_ENSURE_OK(context, ResizeOutputTensor(context, data, segment_ids, + num_segments, output)); + } + TF_LITE_ENSURE_EQ(context, tflite::GetTensorShape(data).Dims(0), + tflite::GetTensorShape(segment_ids).Dims(0)); + +#define TF_LITE_UNSORTED_SEGMENT(dtype) \ + EvalType<dtype>(context, tflite::GetTensorShape(data), tflite::GetTensorData<dtype>(data), \ + tflite::GetTensorShape(segment_ids), \ + tflite::GetTensorData<int32_t>(segment_ids), tflite::GetTensorShape(output), \ + tflite::GetTensorData<dtype>(output), segment_type); + switch (data->type) { + case kTfLiteInt32: + TF_LITE_UNSORTED_SEGMENT(int32_t); + break; + case kTfLiteFloat32: + TF_LITE_UNSORTED_SEGMENT(float); + break; + default: + TF_LITE_KERNEL_LOG( + context, "Currently UnsortedSegment doesn't support data type: %s", + TfLiteTypeGetName(data->type)); + return kTfLiteError; + } +#undef TF_LITE_UNSORTED_SEGMENT + return kTfLiteOk; +} + +TfLiteStatus EvalProd(TfLiteContext* context, TfLiteNode* node) { + return EvalGeneric(context, node, kSegmentProd); +} +TfLiteStatus EvalMax(TfLiteContext* context, TfLiteNode* node) { + return EvalGeneric(context, node, kSegmentMax); +} +TfLiteStatus EvalSum(TfLiteContext* context, TfLiteNode* node) { + return EvalGeneric(context, node, kSegmentSum); +} +TfLiteStatus EvalMin(TfLiteContext* context, TfLiteNode* node) { + return EvalGeneric(context, node, kSegmentMin); +} + +} // namespace unsorted_segment + +TfLiteRegistration* Register_UNSORTED_SEGMENT_PROD() { + static TfLiteRegistration r = {nullptr, nullptr, unsorted_segment::Prepare, + unsorted_segment::EvalProd}; + return &r; +} + +TfLiteRegistration* Register_UNSORTED_SEGMENT_MAX() { + static TfLiteRegistration r = {nullptr, nullptr, unsorted_segment::Prepare, + unsorted_segment::EvalMax}; + return &r; +} + +TfLiteRegistration* Register_UNSORTED_SEGMENT_SUM() { + static TfLiteRegistration r = {nullptr, nullptr, unsorted_segment::Prepare, + unsorted_segment::EvalSum}; + return &r; +} + +TfLiteRegistration* Register_UNSORTED_SEGMENT_MIN() { + static TfLiteRegistration r = {nullptr, nullptr, unsorted_segment::Prepare, + unsorted_segment::EvalMin}; + return &r; +} + +} // namespace custom +} // namespace ops +} // namespace tflite
\ No newline at end of file diff --git a/tensorflow_lite_support/custom_ops/kernel/unsorted_segment.h b/tensorflow_lite_support/custom_ops/kernel/unsorted_segment.h new file mode 100644 index 00000000..fb8fd798 --- /dev/null +++ b/tensorflow_lite_support/custom_ops/kernel/unsorted_segment.h @@ -0,0 +1,31 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_LITE_SUPPORT_CUSTOM_OPS_KERNEL_UNSORTED_SEGMENT_H_ +#define TENSORFLOW_LITE_SUPPORT_CUSTOM_OPS_KERNEL_UNSORTED_SEGMENT_H_ + +#include "tensorflow/lite/kernels/kernel_util.h" + +namespace tflite { +namespace ops { +namespace custom { + +TfLiteRegistration* Register_UNSORTED_SEGMENT_PROD(); + +} // namespace custom +} // namespace ops +} // namespace tflite + +#endif //TENSORFLOW_LITE_SUPPORT_CUSTOM_OPS_KERNEL_UNSORTED_SEGMENT_H_
\ No newline at end of file diff --git a/tensorflow_lite_support/custom_ops/kernel/unsorted_segment_max_test.cc b/tensorflow_lite_support/custom_ops/kernel/unsorted_segment_max_test.cc new file mode 100644 index 00000000..10baefc0 --- /dev/null +++ b/tensorflow_lite_support/custom_ops/kernel/unsorted_segment_max_test.cc @@ -0,0 +1,137 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include <limits.h> +#include <stdint.h> + +#include <initializer_list> +#include <vector> + +#include "testing/base/public/gunit.h" +#include "tensorflow/lite/kernels/test_util.h" +#include "tensorflow/lite/kernels/unsorted_segment_test.h" +#include "tensorflow/lite/schema/schema_generated.h" + +namespace tflite { +namespace { + +using ::testing::ElementsAreArray; + +template <typename T> +class UnsortedSegmentMaxModel : public UnsortedSegmentModel<T> { + public: + UnsortedSegmentMaxModel(const TensorData& data, const TensorData& segment_ids, + const TensorData& num_segments) + : UnsortedSegmentModel<T>(data, segment_ids, num_segments, + BuiltinOperator_UNSORTED_SEGMENT_MAX, + BuiltinOptions_NONE) {} + + explicit UnsortedSegmentMaxModel( + const TensorData& data, const std::initializer_list<int>& segment_id_data, + const std::initializer_list<int>& segment_id_shape, + const std::initializer_list<int>& num_segments_data, + const std::initializer_list<int>& num_segments_shape) + : UnsortedSegmentModel<T>(data, segment_id_data, segment_id_shape, + num_segments_data, num_segments_shape, + BuiltinOperator_UNSORTED_SEGMENT_MAX, + BuiltinOptions_NONE) {} +}; + +INSTANTIATE_TEST_SUITE_P(UnsortedSegmentMaxTestP, UnsortedSegmentTest, + testing::Values(BuiltinOperator_UNSORTED_SEGMENT_MAX)); + +TEST(UnsortedSegmentMaxModelTest, Int32Test_Simple) { + UnsortedSegmentMaxModel<int32_t> model({TensorType_INT32, {6}}, + {TensorType_INT32, {6}}, + {TensorType_INT32, {1}}); + model.PopulateTensor<int32_t>(model.data(), {5, 1, 7, 2, 3, 4}); + model.PopulateTensor<int32_t>(model.segment_ids(), {0, 0, 1, 1, 0, 1}); + model.PopulateTensor<int32_t>(model.num_segments(), {2}); + ASSERT_EQ(model.Invoke(), kTfLiteOk); + EXPECT_THAT(model.GetOutput(), ElementsAreArray({5, 7})); + EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2})); +} + +TEST(UnsortedSegmentMaxModelTest, Int32Test_Simple2D) { + UnsortedSegmentMaxModel<int32_t> model({TensorType_INT32, {3, 4}}, + {TensorType_INT32, {3}}, + {TensorType_INT32, {1}}); + model.PopulateTensor<int32_t>(model.data(), + {1, 2, 3, 4, 5, 6, 7, 8, 4, 3, 2, 1}); + model.PopulateTensor<int32_t>(model.segment_ids(), {0, 1, 0}); + model.PopulateTensor<int32_t>(model.num_segments(), {2}); + ASSERT_EQ(model.Invoke(), kTfLiteOk); + EXPECT_THAT(model.GetOutput(), ElementsAreArray({4, 3, 3, 4, 5, 6, 7, 8})); + EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2, 4})); +} + +TEST(UnsortedSegmentMaxModelTest, FloatTest_Simple) { + UnsortedSegmentMaxModel<float> model({TensorType_FLOAT32, {8}}, + {TensorType_INT32, {8}}, + {TensorType_INT32, {1}}); + model.PopulateTensor<float>(model.data(), + {1.0, 2.0, 3.0, 4.0, 4.0, 3.0, 2.0, 1.0}); + model.PopulateTensor<int32_t>(model.segment_ids(), {1, 0, 1, 7, 7, 7, 7, 7}); + model.PopulateTensor<int32_t>(model.num_segments(), {8}); + ASSERT_EQ(model.Invoke(), kTfLiteOk); + EXPECT_THAT(model.GetOutput(), + ElementsAreArray(ArrayFloatNear( + {2.0, 3.0, std::numeric_limits<float>::lowest(), + std::numeric_limits<float>::lowest(), + std::numeric_limits<float>::lowest(), + std::numeric_limits<float>::lowest(), + std::numeric_limits<float>::lowest(), 4.0}))); + EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({8})); +} + +TEST(UnsortedSegmentMaxModelTest, FloatTest_Simple2D) { + UnsortedSegmentMaxModel<float> model({TensorType_FLOAT32, {3, 4}}, + {TensorType_INT32, {3}}, + {TensorType_INT32, {1}}); + model.PopulateTensor<float>(model.data(), {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, + 8.0, 4.0, 3.0, 2.0, 1.0}); + model.PopulateTensor<int32_t>(model.segment_ids(), {0, 1, 0}); + model.PopulateTensor<int32_t>(model.num_segments(), {2}); + ASSERT_EQ(model.Invoke(), kTfLiteOk); + EXPECT_THAT(model.GetOutput(), + ElementsAreArray( + ArrayFloatNear({4.0, 3.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0}))); + EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2, 4})); +} + +TEST(UnsortedSegmentMaxModelTest, SegmentsAreNegative) { + UnsortedSegmentMaxModel<int32_t> model({TensorType_INT32, {2, 2}}, + {TensorType_INT32, {2}}, + {TensorType_INT32, {1}}); + model.PopulateTensor<int32_t>(model.data(), {1, 2, 3, 4}); + model.PopulateTensor<int32_t>(model.segment_ids(), {-1, -1}); + model.PopulateTensor<int32_t>(model.num_segments(), {1}); + ASSERT_EQ(model.Invoke(), kTfLiteOk); + EXPECT_THAT(model.GetOutput(), + ElementsAreArray({std::numeric_limits<int32_t>::lowest(), + std::numeric_limits<int32_t>::lowest()})); + EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({1, 2})); +} + +TEST(UnsortedSegmentMaxModelTest, ConstParamenterTest) { + UnsortedSegmentMaxModel<int32_t> model({TensorType_INT32, {3, 2}}, {0, 1, 0}, + {3}, {2}, {1}); + model.PopulateTensor<int32_t>(model.data(), {1, 2, 3, 4, 5, 6}); + ASSERT_EQ(model.Invoke(), kTfLiteOk); + EXPECT_THAT(model.GetOutput(), ElementsAreArray({5, 6, 3, 4})); + EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2, 2})); +} + +} // namespace +} // namespace tflite
\ No newline at end of file diff --git a/tensorflow_lite_support/custom_ops/kernel/unsorted_segment_min_test.cc b/tensorflow_lite_support/custom_ops/kernel/unsorted_segment_min_test.cc new file mode 100644 index 00000000..bb4d4141 --- /dev/null +++ b/tensorflow_lite_support/custom_ops/kernel/unsorted_segment_min_test.cc @@ -0,0 +1,137 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include <limits.h> +#include <stdint.h> + +#include <initializer_list> +#include <vector> + +#include "testing/base/public/gunit.h" +#include "tensorflow/lite/kernels/test_util.h" +#include "tensorflow/lite/kernels/unsorted_segment_test.h" +#include "tensorflow/lite/schema/schema_generated.h" + +namespace tflite { +namespace { + +using ::testing::ElementsAreArray; + +template <typename T> +class UnsortedSegmentMinModel : public UnsortedSegmentModel<T> { + public: + UnsortedSegmentMinModel(const TensorData& data, const TensorData& segment_ids, + const TensorData& num_segments) + : UnsortedSegmentModel<T>(data, segment_ids, num_segments, + BuiltinOperator_UNSORTED_SEGMENT_MIN, + BuiltinOptions_NONE) {} + + explicit UnsortedSegmentMinModel( + const TensorData& data, const std::initializer_list<int>& segment_id_data, + const std::initializer_list<int>& segment_id_shape, + const std::initializer_list<int>& num_segments_data, + const std::initializer_list<int>& num_segments_shape) + : UnsortedSegmentModel<T>(data, segment_id_data, segment_id_shape, + num_segments_data, num_segments_shape, + BuiltinOperator_UNSORTED_SEGMENT_MIN, + BuiltinOptions_NONE) {} +}; + +INSTANTIATE_TEST_SUITE_P(UnsortedSegmentMinTestP, UnsortedSegmentTest, + testing::Values(BuiltinOperator_UNSORTED_SEGMENT_MIN)); + +TEST(UnsortedSegmentMinModelTest, Int32Test_Simple) { + UnsortedSegmentMinModel<int32_t> model({TensorType_INT32, {6}}, + {TensorType_INT32, {6}}, + {TensorType_INT32, {1}}); + model.PopulateTensor<int32_t>(model.data(), {5, 3, 7, 8, 6, 4}); + model.PopulateTensor<int32_t>(model.segment_ids(), {0, 0, 1, 1, 0, 1}); + model.PopulateTensor<int32_t>(model.num_segments(), {2}); + ASSERT_EQ(model.Invoke(), kTfLiteOk); + EXPECT_THAT(model.GetOutput(), ElementsAreArray({3, 4})); + EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2})); +} + +TEST(UnsortedSegmentMinModelTest, Int32Test_Simple2D) { + UnsortedSegmentMinModel<int32_t> model({TensorType_INT32, {3, 4}}, + {TensorType_INT32, {3}}, + {TensorType_INT32, {1}}); + model.PopulateTensor<int32_t>(model.data(), + {1, 2, 3, 4, 5, 6, 7, 8, 4, 3, 2, 1}); + model.PopulateTensor<int32_t>(model.segment_ids(), {0, 1, 0}); + model.PopulateTensor<int32_t>(model.num_segments(), {2}); + ASSERT_EQ(model.Invoke(), kTfLiteOk); + EXPECT_THAT(model.GetOutput(), ElementsAreArray({1, 2, 2, 1, 5, 6, 7, 8})); + EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2, 4})); +} + +TEST(UnsortedSegmentMinModelTest, FloatTest_Simple) { + UnsortedSegmentMinModel<float> model({TensorType_FLOAT32, {8}}, + {TensorType_INT32, {8}}, + {TensorType_INT32, {1}}); + model.PopulateTensor<float>(model.data(), + {1.0, 2.0, 3.0, 4.0, 4.0, 3.0, 2.0, 1.0}); + model.PopulateTensor<int32_t>(model.segment_ids(), {1, 0, 1, 7, 7, 7, 7, 7}); + model.PopulateTensor<int32_t>(model.num_segments(), {8}); + ASSERT_EQ(model.Invoke(), kTfLiteOk); + EXPECT_THAT( + model.GetOutput(), + ElementsAreArray(ArrayFloatNear( + {2.0, 1.0, std::numeric_limits<float>::max(), + std::numeric_limits<float>::max(), std::numeric_limits<float>::max(), + std::numeric_limits<float>::max(), std::numeric_limits<float>::max(), + 1.0}))); + EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({8})); +} + +TEST(UnsortedSegmentMinModelTest, FloatTest_Simple2D) { + UnsortedSegmentMinModel<float> model({TensorType_FLOAT32, {3, 4}}, + {TensorType_INT32, {3}}, + {TensorType_INT32, {1}}); + model.PopulateTensor<float>(model.data(), {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, + 8.0, 4.0, 3.0, 2.0, 1.0}); + model.PopulateTensor<int32_t>(model.segment_ids(), {0, 1, 0}); + model.PopulateTensor<int32_t>(model.num_segments(), {2}); + ASSERT_EQ(model.Invoke(), kTfLiteOk); + EXPECT_THAT(model.GetOutput(), + ElementsAreArray( + ArrayFloatNear({1.0, 2.0, 2.0, 1.0, 5.0, 6.0, 7.0, 8.0}))); + EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2, 4})); +} + +TEST(UnsortedSegmentMinModelTest, SegmentsAreNegative) { + UnsortedSegmentMinModel<int32_t> model({TensorType_INT32, {2, 2}}, + {TensorType_INT32, {2}}, + {TensorType_INT32, {1}}); + model.PopulateTensor<int32_t>(model.data(), {1, 2, 3, 4}); + model.PopulateTensor<int32_t>(model.segment_ids(), {-1, -1}); + model.PopulateTensor<int32_t>(model.num_segments(), {1}); + ASSERT_EQ(model.Invoke(), kTfLiteOk); + EXPECT_THAT(model.GetOutput(), + ElementsAreArray({std::numeric_limits<int32_t>::max(), + std::numeric_limits<int32_t>::max()})); + EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({1, 2})); +} + +TEST(UnsortedSegmentMinModelTest, ConstParamenterTest) { + UnsortedSegmentMinModel<int32_t> model({TensorType_INT32, {3, 2}}, {0, 1, 0}, + {3}, {2}, {1}); + model.PopulateTensor<int32_t>(model.data(), {1, 2, 3, 4, 5, 6}); + ASSERT_EQ(model.Invoke(), kTfLiteOk); + EXPECT_THAT(model.GetOutput(), ElementsAreArray({1, 2, 3, 4})); + EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2, 2})); +} + +} // namespace +} // namespace tflite
\ No newline at end of file diff --git a/tensorflow_lite_support/custom_ops/kernel/unsorted_segment_prod_test.cc b/tensorflow_lite_support/custom_ops/kernel/unsorted_segment_prod_test.cc new file mode 100644 index 00000000..524d45a0 --- /dev/null +++ b/tensorflow_lite_support/custom_ops/kernel/unsorted_segment_prod_test.cc @@ -0,0 +1,122 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include <stdint.h> + +#include <vector> + +#include "testing/base/public/gunit.h" +#include "tensorflow/lite/kernels/test_util.h" +#include "tensorflow/lite/kernels/unsorted_segment_test.h" +#include "tensorflow/lite/schema/schema_generated.h" + +namespace tflite { +namespace { + +using ::testing::ElementsAreArray; + +template <typename T> +class UnsortedSegmentProdModel : public UnsortedSegmentModel<T> { + public: + UnsortedSegmentProdModel(const TensorData& data, + const TensorData& segment_ids, + const TensorData& num_segments) + : UnsortedSegmentModel<T>(data, segment_ids, num_segments, + BuiltinOperator_UNSORTED_SEGMENT_PROD, + BuiltinOptions_UnsortedSegmentProdOptions) {} + + explicit UnsortedSegmentProdModel( + const TensorData& data, const std::initializer_list<int>& segment_id_data, + const std::initializer_list<int>& segment_id_shape, + const std::initializer_list<int>& num_segments_data, + const std::initializer_list<int>& num_segments_shape) + : UnsortedSegmentModel<T>(data, segment_id_data, segment_id_shape, + num_segments_data, num_segments_shape, + BuiltinOperator_UNSORTED_SEGMENT_PROD, + BuiltinOptions_UnsortedSegmentProdOptions) {} +}; + +INSTANTIATE_TEST_SUITE_P( + UnsortedSegmentProdTestP, UnsortedSegmentTest, + testing::Values(BuiltinOperator_UNSORTED_SEGMENT_PROD)); + +TEST(UnsortedSegmentProdModelTest, Int32Test_Simple) { + UnsortedSegmentProdModel<int32_t> model({TensorType_INT32, {8}}, + {TensorType_INT32, {8}}, + {TensorType_INT32, {1}}); + model.PopulateTensor<int32_t>(model.data(), {1, 2, 3, 4, 4, 3, 2, 1}); + model.PopulateTensor<int32_t>(model.segment_ids(), {1, 0, 1, 7, 7, 7, 7, 7}); + model.PopulateTensor<int32_t>(model.num_segments(), {8}); + ASSERT_EQ(model.Invoke(), kTfLiteOk); + EXPECT_THAT(model.GetOutput(), ElementsAreArray({2, 3, 1, 1, 1, 1, 1, 96})); + EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({8})); +} + +TEST(UnsortedSegmentProdModelTest, TestSkipNegSegmentId) { + UnsortedSegmentProdModel<int32_t> model({TensorType_INT32, {8}}, + {TensorType_INT32, {8}}, + {TensorType_INT32, {1}}); + model.PopulateTensor<int32_t>(model.data(), {1, 2, 3, 4, 4, 3, 2, 1}); + model.PopulateTensor<int32_t>(model.segment_ids(), {1, 0, 1, 7, 7, 7, 7, -1}); + model.PopulateTensor<int32_t>(model.num_segments(), {8}); + ASSERT_EQ(model.Invoke(), kTfLiteOk); + EXPECT_THAT(model.GetOutput(), ElementsAreArray({2, 3, 1, 1, 1, 1, 1, 96})); + EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({8})); +} + +TEST(UnsortedSegmentProdModelTest, Int32Test_Simple2D) { + UnsortedSegmentProdModel<int32_t> model({TensorType_INT32, {3, 4}}, + {TensorType_INT32, {3}}, + {TensorType_INT32, {1}}); + model.PopulateTensor<int32_t>(model.data(), + {1, 2, 3, 4, 5, 6, 7, 8, 4, 3, 2, 1}); + model.PopulateTensor<int32_t>(model.segment_ids(), {0, 1, 0}); + model.PopulateTensor<int32_t>(model.num_segments(), {2}); + ASSERT_EQ(model.Invoke(), kTfLiteOk); + EXPECT_THAT(model.GetOutput(), ElementsAreArray({4, 6, 6, 4, 5, 6, 7, 8})); + EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2, 4})); +} + +TEST(UnsortedSegmentProdModelTest, FloatTest_Simple) { + UnsortedSegmentProdModel<float> model({TensorType_FLOAT32, {8}}, + {TensorType_INT32, {8}}, + {TensorType_INT32, {1}}); + model.PopulateTensor<float>(model.data(), + {1.0, 2.0, 3.0, 4.0, 4.0, 3.0, 2.0, 1.0}); + model.PopulateTensor<int32_t>(model.segment_ids(), {1, 0, 1, 7, 7, 7, 7, 7}); + model.PopulateTensor<int32_t>(model.num_segments(), {8}); + ASSERT_EQ(model.Invoke(), kTfLiteOk); + EXPECT_THAT(model.GetOutput(), + ElementsAreArray( + ArrayFloatNear({2.0, 3.0, 1.0, 1.0, 1.0, 1.0, 1.0, 96.0}))); + EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({8})); +} + +TEST(UnsortedSegmentProdModelTest, FloatTest_Simple2D) { + UnsortedSegmentProdModel<float> model({TensorType_FLOAT32, {3, 4}}, + {TensorType_INT32, {3}}, + {TensorType_INT32, {1}}); + model.PopulateTensor<float>(model.data(), {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, + 8.0, 4.0, 3.0, 2.0, 1.0}); + model.PopulateTensor<int32_t>(model.segment_ids(), {0, 1, 0}); + model.PopulateTensor<int32_t>(model.num_segments(), {2}); + ASSERT_EQ(model.Invoke(), kTfLiteOk); + EXPECT_THAT(model.GetOutput(), + ElementsAreArray( + ArrayFloatNear({4.0, 6.0, 6.0, 4.0, 5.0, 6.0, 7.0, 8.0}))); + EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2, 4})); +} + +} // namespace +} // namespace tflite
\ No newline at end of file diff --git a/tensorflow_lite_support/custom_ops/kernel/unsorted_segment_sum_test.cc b/tensorflow_lite_support/custom_ops/kernel/unsorted_segment_sum_test.cc new file mode 100644 index 00000000..2451c169 --- /dev/null +++ b/tensorflow_lite_support/custom_ops/kernel/unsorted_segment_sum_test.cc @@ -0,0 +1,145 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include <limits.h> +#include <stdint.h> + +#include <initializer_list> +#include <vector> + +#include "testing/base/public/gunit.h" +#include "tensorflow/lite/kernels/test_util.h" +#include "tensorflow/lite/kernels/unsorted_segment_test.h" +#include "tensorflow/lite/schema/schema_generated.h" + +namespace tflite { +namespace { + +using ::testing::ElementsAreArray; + +template <typename T> +class UnsortedSegmentSumModel : public UnsortedSegmentModel<T> { + public: + UnsortedSegmentSumModel(const TensorData& data, const TensorData& segment_ids, + const TensorData& num_segments) + : UnsortedSegmentModel<T>(data, segment_ids, num_segments, + BuiltinOperator_UNSORTED_SEGMENT_SUM, + BuiltinOptions_NONE) {} + + explicit UnsortedSegmentSumModel( + const TensorData& data, const std::initializer_list<int>& segment_id_data, + const std::initializer_list<int>& segment_id_shape, + const std::initializer_list<int>& num_segments_data, + const std::initializer_list<int>& num_segments_shape) + : UnsortedSegmentModel<T>(data, segment_id_data, segment_id_shape, + num_segments_data, num_segments_shape, + BuiltinOperator_UNSORTED_SEGMENT_SUM, + BuiltinOptions_NONE) {} +}; + +INSTANTIATE_TEST_SUITE_P(UnsortedSegmentSumTestP, UnsortedSegmentTest, + testing::Values(BuiltinOperator_UNSORTED_SEGMENT_SUM)); + +TEST(UnsortedSegmentSumModelTest, Int32Test_Simple) { + UnsortedSegmentSumModel<int32_t> model({TensorType_INT32, {7}}, + {TensorType_INT32, {7}}, + {TensorType_INT32, {1}}); + model.PopulateTensor<int32_t>(model.data(), {5, 1, 7, 2, 3, 4, 10}); + model.PopulateTensor<int32_t>(model.segment_ids(), {0, 0, 1, 1, 0, 1, 0}); + model.PopulateTensor<int32_t>(model.num_segments(), {2}); + ASSERT_EQ(model.Invoke(), kTfLiteOk); + EXPECT_THAT(model.GetOutput(), ElementsAreArray({19, 13})); + EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2})); +} + +TEST(UnsortedSegmentSumModelTest, Int32Test_Simple2D) { + UnsortedSegmentSumModel<int32_t> model({TensorType_INT32, {3, 4}}, + {TensorType_INT32, {3}}, + {TensorType_INT32, {1}}); + model.PopulateTensor<int32_t>(model.data(), + {1, 2, 3, 4, 5, 6, 7, 8, 4, 3, 2, 1}); + model.PopulateTensor<int32_t>(model.segment_ids(), {0, 1, 0}); + model.PopulateTensor<int32_t>(model.num_segments(), {2}); + ASSERT_EQ(model.Invoke(), kTfLiteOk); + EXPECT_THAT(model.GetOutput(), ElementsAreArray({5, 5, 5, 5, 5, 6, 7, 8})); + EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2, 4})); +} + +TEST(UnsortedSegmentSumModelTest, FloatTest_Simple) { + UnsortedSegmentSumModel<float> model({TensorType_FLOAT32, {6}}, + {TensorType_INT32, {6}}, + {TensorType_INT32, {1}}); + model.PopulateTensor<float>(model.data(), {1.0, 2.0, 3.0, 4.0, 4.0, 3.0}); + model.PopulateTensor<int32_t>(model.segment_ids(), {1, 0, 1, 7, 7, 7}); + model.PopulateTensor<int32_t>(model.num_segments(), {8}); + ASSERT_EQ(model.Invoke(), kTfLiteOk); + EXPECT_THAT(model.GetOutput(), + ElementsAreArray( + ArrayFloatNear({2.0, 4.0, 0.0, 0.0, 0.0, 0.0, 0.0, 11.0}))); + EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({8})); +} + +TEST(UnsortedSegmentSumModelTest, FloatTest_Simple2D) { + UnsortedSegmentSumModel<float> model({TensorType_FLOAT32, {3, 4}}, + {TensorType_INT32, {3}}, + {TensorType_INT32, {1}}); + model.PopulateTensor<float>(model.data(), {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, + 8.0, 4.0, 3.0, 2.0, 1.0}); + model.PopulateTensor<int32_t>(model.segment_ids(), {0, 1, 0}); + model.PopulateTensor<int32_t>(model.num_segments(), {2}); + ASSERT_EQ(model.Invoke(), kTfLiteOk); + EXPECT_THAT(model.GetOutput(), + ElementsAreArray( + ArrayFloatNear({5.0, 5.0, 5.0, 5.0, 5.0, 6.0, 7.0, 8.0}))); + EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2, 4})); +} + +TEST(UnsortedSegmentSumModelTest, AllNegativeSegmentIdsZeroTensor) { + UnsortedSegmentSumModel<int32_t> model({TensorType_INT32, {2, 2}}, + {TensorType_INT32, {2}}, + {TensorType_INT32, {1}}); + model.PopulateTensor<int32_t>(model.data(), {1, 2, 3, 4}); + model.PopulateTensor<int32_t>(model.segment_ids(), {-1, -1}); + model.PopulateTensor<int32_t>(model.num_segments(), {1}); + ASSERT_EQ(model.Invoke(), kTfLiteOk); + EXPECT_THAT(model.GetOutput(), ElementsAreArray({0, 0})); + EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({1, 2})); +} + +TEST(UnsortedSegmentSumModelTest, SomeNegativeSegmentIdsIgnored) { + UnsortedSegmentSumModel<int32_t> model({TensorType_INT32, {4}}, + {TensorType_INT32, {4}}, + {TensorType_INT32, {1}}); + model.PopulateTensor<int32_t>(model.data(), {1, 2, 3, 4}); + model.PopulateTensor<int32_t>(model.segment_ids(), {-1, 0, -1, 1}); + model.PopulateTensor<int32_t>(model.num_segments(), {2}); + ASSERT_EQ(model.Invoke(), kTfLiteOk); + EXPECT_THAT(model.GetOutput(), ElementsAreArray({2, 4})); + EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({2})); +} + +TEST(UnsortedSegmentSumModelTest, + NumSegmentsGreaterThanNumIdsPadsWithZeroTensors) { + UnsortedSegmentSumModel<int32_t> model({TensorType_INT32, {2, 2}}, + {TensorType_INT32, {2}}, + {TensorType_INT32, {1}}); + model.PopulateTensor<int32_t>(model.data(), {1, 2, 3, 4}); + model.PopulateTensor<int32_t>(model.segment_ids(), {0, 1}); + model.PopulateTensor<int32_t>(model.num_segments(), {3}); + ASSERT_EQ(model.Invoke(), kTfLiteOk); + EXPECT_THAT(model.GetOutput(), ElementsAreArray({1, 2, 3, 4, 0, 0})); + EXPECT_THAT(model.GetOutputShape(), ElementsAreArray({3, 2})); +} +} // namespace +} // namespace tflite
\ No newline at end of file diff --git a/tensorflow_lite_support/custom_ops/kernel/unsorted_segment_test.cc b/tensorflow_lite_support/custom_ops/kernel/unsorted_segment_test.cc new file mode 100644 index 00000000..6e398374 --- /dev/null +++ b/tensorflow_lite_support/custom_ops/kernel/unsorted_segment_test.cc @@ -0,0 +1,130 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "third_party/tensorflow/lite/kernels/unsorted_segment_test.h" + +#include <limits.h> +#include <stdint.h> + +#include <initializer_list> +#include <vector> + +#include "testing/base/public/gunit.h" +#include "tensorflow/lite/kernels/test_util.h" +#include "tensorflow/lite/schema/schema_generated.h" + +namespace tflite { +namespace { + +TEST_P(UnsortedSegmentTest, SegmentIdsSizeNotEqualToDataFirstDimensionFails) { + UnsortedSegmentModel<int32_t> model = + getModel({TensorType_INT32, {3, 2}}, {TensorType_INT32, {2}}, + {TensorType_INT32, {1}}); + model.PopulateTensor<int32_t>(model.data(), {1, 2, 3, 4, 5, 6}); + model.PopulateTensor<int32_t>(model.segment_ids(), {0, 1}); + model.PopulateTensor<int32_t>(model.num_segments(), {2}); + ASSERT_EQ(model.Invoke(), kTfLiteError); +} +TEST_P(UnsortedSegmentTest, + LargestSegmentIdPlusOneGreaterThanNumSegmentsFails) { + UnsortedSegmentModel<int32_t> model = + getModel({TensorType_INT32, {2, 2}}, {TensorType_INT32, {2}}, + {TensorType_INT32, {1}}); + model.PopulateTensor<int32_t>(model.data(), {1, 2, 3, 4}); + model.PopulateTensor<int32_t>(model.segment_ids(), {0, 1}); + model.PopulateTensor<int32_t>(model.num_segments(), {1}); + ASSERT_EQ(model.Invoke(), kTfLiteError); +} +TEST_P(UnsortedSegmentTest, NumSegmentsNotScalarShapeFails) { + UnsortedSegmentModel<int32_t> model = + getModel({TensorType_INT32, {3, 2}}, {TensorType_INT32, {3}}, + {TensorType_INT32, {2}}); + model.PopulateTensor<int32_t>(model.data(), {1, 2, 3, 4, 5, 6}); + model.PopulateTensor<int32_t>(model.segment_ids(), {0, 1, 0}); + model.PopulateTensor<int32_t>(model.num_segments(), {2, 1}); + ASSERT_EQ(model.Invoke(), kTfLiteError); +} +TEST_P(UnsortedSegmentTest, Rank2SegIdsNotPrefixFails) { + UnsortedSegmentModel<int32_t> model = + getModel({TensorType_INT32, {2, 2, 2}}, {TensorType_INT32, {2, 1}}, + {TensorType_INT32, {1}}); + model.PopulateTensor<int32_t>(model.data(), {1, 2, 3, 4, 5, 6}); + model.PopulateTensor<int32_t>(model.segment_ids(), {1, 1}); + model.PopulateTensor<int32_t>(model.num_segments(), {3}); + ASSERT_EQ(model.Invoke(), kTfLiteError); +} +TEST_P(UnsortedSegmentTest, Rank2SegIdsHasShapeNumSegDataShapeSuffix) { + UnsortedSegmentModel<int32_t> model = + getModel({TensorType_INT32, {2, 2, 2}}, {TensorType_INT32, {2, 2}}, + {TensorType_INT32, {1}}); + model.PopulateTensor<int32_t>(model.data(), {1, 2, 3, 4, 5, 6}); + model.PopulateTensor<int32_t>(model.segment_ids(), {1, 2, 0, 8}); + model.PopulateTensor<int32_t>(model.num_segments(), {10}); + ASSERT_EQ(model.Invoke(), kTfLiteOk); + EXPECT_THAT(model.GetOutputShape(), testing::ElementsAreArray({10, 2})); +} +TEST_P(UnsortedSegmentTest, Rank2SegIdsHasShapeNumSegDataShapeSuffixConst) { + UnsortedSegmentModel<int32_t> model = getConstModel( + {TensorType_INT32, {2, 2, 2}}, {1, 2, -1, -1}, {2, 2}, {3}, {1}); + model.PopulateTensor<int32_t>(model.data(), {1, 2, 3, 4, 5, 6}); + ASSERT_EQ(model.Invoke(), kTfLiteOk); + EXPECT_THAT(model.GetOutputShape(), testing::ElementsAreArray({3, 2})); +} +TEST_P(UnsortedSegmentTest, SegIdsHasSameShapeAsData2d) { + UnsortedSegmentModel<int32_t> model = + getModel({TensorType_INT32, {2, 2}}, {TensorType_INT32, {2, 2}}, + {TensorType_INT32, {1}}); + model.PopulateTensor<int32_t>(model.data(), {1, 2, 3, 4}); + model.PopulateTensor<int32_t>(model.segment_ids(), {0, 1, 5, 2, 4}); + model.PopulateTensor<int32_t>(model.num_segments(), {10}); + ASSERT_EQ(model.Invoke(), kTfLiteOk); + EXPECT_THAT(model.GetOutputShape(), testing::ElementsAreArray({10})); +} +TEST_P(UnsortedSegmentTest, SegIdsHasSameShapeAsData2dConst) { + UnsortedSegmentModel<int32_t> model = + getConstModel({TensorType_INT32, {2, 2}}, {1, 1, 1, 1}, {2, 2}, {3}, {1}); + model.PopulateTensor<int32_t>(model.data(), {1, 2, 3, 4}); + ASSERT_EQ(model.Invoke(), kTfLiteOk); + EXPECT_THAT(model.GetOutputShape(), testing::ElementsAreArray({3})); +} +TEST_P(UnsortedSegmentTest, SegIdsHasSameShapeAsData3d) { + UnsortedSegmentModel<int32_t> model = + getModel({TensorType_INT32, {2, 2, 2}}, {TensorType_INT32, {2, 2, 2}}, + {TensorType_INT32, {1}}); + model.PopulateTensor<int32_t>(model.data(), {1, 2, 3, 4, 5, 6, 7, 8}); + model.PopulateTensor<int32_t>(model.segment_ids(), {1, 2, 3, 4, 5, 6, 7, 8}); + model.PopulateTensor<int32_t>(model.num_segments(), {10}); + ASSERT_EQ(model.Invoke(), kTfLiteOk); + EXPECT_THAT(model.GetOutputShape(), testing::ElementsAreArray({10})); +} +TEST_P(UnsortedSegmentTest, SegIdsHasSameShapeAsData3dConst) { + UnsortedSegmentModel<int32_t> model = + getConstModel({TensorType_INT32, {2, 2, 2}}, {0, 1, 2, -1, 3, -1, 4, -1}, + {2, 2, 2}, {8}, {1}); + model.PopulateTensor<int32_t>(model.data(), {1, 1, 1, 1, 1, 1}); + ASSERT_EQ(model.Invoke(), kTfLiteOk); + EXPECT_THAT(model.GetOutputShape(), testing::ElementsAreArray({8})); +} +TEST_P(UnsortedSegmentTest, Data5dHasShapeNumSegDataShapeSuffix) { + UnsortedSegmentModel<int32_t> model = + getModel({TensorType_INT32, {2, 1, 2, 1, 2}}, {TensorType_INT32, {2, 1}}, + {TensorType_INT32, {1}}); + model.PopulateTensor<int32_t>(model.data(), {1, 2, 3, 4, 5, 6, 7, 8}); + model.PopulateTensor(model.segment_ids(), {0, 1}); + model.PopulateTensor(model.num_segments(), {10}); + ASSERT_EQ(model.Invoke(), kTfLiteOk); + EXPECT_THAT(model.GetOutputShape(), testing::ElementsAreArray({10, 2, 1, 2})); +} +} // namespace +} // namespace tflite
\ No newline at end of file diff --git a/tensorflow_lite_support/custom_ops/kernel/unsorted_segment_test.h b/tensorflow_lite_support/custom_ops/kernel/unsorted_segment_test.h new file mode 100644 index 00000000..6a5b0f64 --- /dev/null +++ b/tensorflow_lite_support/custom_ops/kernel/unsorted_segment_test.h @@ -0,0 +1,97 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_LITE_KERNELS_UNSORTED_SEGMENT_TEST_H_ +#define TENSORFLOW_LITE_KERNELS_UNSORTED_SEGMENT_TEST_H_ + +#include <limits.h> +#include <stdint.h> + +#include <initializer_list> +#include <iostream> +#include <ostream> +#include <vector> + +#include "testing/base/public/gunit.h" +#include "tensorflow/lite/core/c/common.h" +#include "tensorflow/lite/kernels/test_util.h" +#include "tensorflow/lite/schema/schema_generated.h" + +namespace tflite { +template <typename T> +class UnsortedSegmentModel : public SingleOpModel { + public: + UnsortedSegmentModel(const TensorData& data, const TensorData& segment_ids, + const TensorData& num_segments, const BuiltinOperator op, + const BuiltinOptions options) { + data_id_ = AddInput(data); + segment_ids_id_ = AddInput(segment_ids); + num_segments_id_ = AddInput(num_segments); + output_id_ = AddOutput(data.type); + SetBuiltinOp(op, options, 0); + BuildInterpreter({GetShape(data_id_), GetShape(segment_ids_id_), + GetShape(num_segments_id_)}); + } + + explicit UnsortedSegmentModel( + const TensorData& data, const std::initializer_list<int>& segment_id_data, + const std::initializer_list<int>& segment_id_shape, + const std::initializer_list<int>& num_segments_data, + const std::initializer_list<int>& num_segments_shape, + const BuiltinOperator op, const BuiltinOptions options) { + data_id_ = AddInput(data); + segment_ids_id_ = + AddConstInput(TensorType_INT32, segment_id_data, segment_id_shape); + num_segments_id_ = + AddConstInput(TensorType_INT32, num_segments_data, num_segments_shape); + output_id_ = AddOutput(data.type); + SetBuiltinOp(op, options, 0); + BuildInterpreter({GetShape(data_id_), GetShape(segment_ids_id_), + GetShape(num_segments_id_)}); + } + + int data() const { return data_id_; } + int segment_ids() const { return segment_ids_id_; } + int num_segments() const { return num_segments_id_; } + std::vector<T> GetOutput() { return ExtractVector<T>(output_id_); } + std::vector<int32_t> GetOutputShape() { return GetTensorShape(output_id_); } + + protected: + int data_id_; + int segment_ids_id_; + int num_segments_id_; + int output_id_; +}; + +class UnsortedSegmentTest : public ::testing::TestWithParam<BuiltinOperator> { + public: + UnsortedSegmentModel<int32_t> getModel(const TensorData& data, + const TensorData& segment_ids, + const TensorData& num_segments) { + return UnsortedSegmentModel<int32_t>(data, segment_ids, num_segments, + GetParam(), BuiltinOptions_NONE); + } + UnsortedSegmentModel<int32_t> getConstModel( + const TensorData& data, const std::initializer_list<int>& segment_id_data, + const std::initializer_list<int>& segment_id_shape, + const std::initializer_list<int>& num_segments_data, + const std::initializer_list<int>& num_segments_shape) { + return UnsortedSegmentModel<int32_t>( + data, segment_id_data, segment_id_shape, num_segments_data, + num_segments_shape, GetParam(), BuiltinOptions_NONE); + } +}; + +} // namespace tflite +#endif // TENSORFLOW_LITE_KERNELS_UNSORTED_SEGMENT_TEST_H_
\ No newline at end of file diff --git a/tensorflow_lite_support/java/AndroidManifest.xml b/tensorflow_lite_support/java/AndroidManifest.xml index 14909296..c36eb383 100644 --- a/tensorflow_lite_support/java/AndroidManifest.xml +++ b/tensorflow_lite_support/java/AndroidManifest.xml @@ -2,4 +2,8 @@ <manifest xmlns:android="http://schemas.android.com/apk/res/android" package="org.tensorflow.lite.support"> <uses-sdk android:minSdkVersion="19" android:targetSdkVersion="29"/> + <instrumentation + android:name="androidx.test.runner.AndroidJUnitRunner" + android:targetPackage="org.tensorflow.lite.support" > + </instrumentation> </manifest> diff --git a/tensorflow_lite_support/java/src/java/org/tensorflow/lite/support/label/Category.java b/tensorflow_lite_support/java/src/java/org/tensorflow/lite/support/label/Category.java index 5b043a9f..cce42399 100644 --- a/tensorflow_lite_support/java/src/java/org/tensorflow/lite/support/label/Category.java +++ b/tensorflow_lite_support/java/src/java/org/tensorflow/lite/support/label/Category.java @@ -40,8 +40,8 @@ public final class Category { return new Category(label, displayName, score); } - @UsedByReflection("TFLiteSupport/Task") /** Constructs a {@link Category} object with an empty displayName. */ + @UsedByReflection("TFLiteSupport/Task") public Category(String label, float score) { this(label, /*displayName=*/ "", score); } diff --git a/tensorflow_lite_support/java/src/java/org/tensorflow/lite/task/text/nlclassifier/BertNLClassifier.java b/tensorflow_lite_support/java/src/java/org/tensorflow/lite/task/text/nlclassifier/BertNLClassifier.java index 90bea370..65cac78e 100644 --- a/tensorflow_lite_support/java/src/java/org/tensorflow/lite/task/text/nlclassifier/BertNLClassifier.java +++ b/tensorflow_lite_support/java/src/java/org/tensorflow/lite/task/text/nlclassifier/BertNLClassifier.java @@ -42,7 +42,7 @@ import org.tensorflow.lite.task.core.TaskJniUtils.EmptyHandleProvider; * </ul> */ public class BertNLClassifier extends BaseTaskApi { - private static final String BERT_NL_CLASSIFIER_NATIVE_LIBNAME = "task_text_jni"; + private static final String BERT_NL_CLASSIFIER_NATIVE_LIBNAME = "tflite_support_classifiers_native"; /** * Constructor to initialize the JNI with a pointer from C++. @@ -122,12 +122,36 @@ public class BertNLClassifier extends BaseTaskApi { return classifyNative(getNativeHandle(), text); } + /** + * Gets the model version from the model metadata, + * or "NO_VERSION_INFO" in case there is no version. + * + * @return The model version. + */ + public String getModelVersion() { + return getModelVersionNative(getNativeHandle()); + } + + /** + * Gets the labels version from the model metadata, + * or "NO_VERSION_INFO" in case there is no version. + * + * @return The labels version. + */ + public String getLabelsVersion() { + return getLabelsVersionNative(getNativeHandle()); + } + private static native long initJniWithByteBuffer(ByteBuffer modelBuffer); private static native long initJniWithFileDescriptor(int fd); private static native List<Category> classifyNative(long nativeHandle, String text); + private static native String getModelVersionNative(long nativeHandle); + + private static native String getLabelsVersionNative(long nativeHandle); + @Override protected void deinit(long nativeHandle) { deinitJni(nativeHandle); diff --git a/tensorflow_lite_support/java/src/java/org/tensorflow/lite/task/text/nlclassifier/NLClassifier.java b/tensorflow_lite_support/java/src/java/org/tensorflow/lite/task/text/nlclassifier/NLClassifier.java index 2bc20d8c..642fab5b 100644 --- a/tensorflow_lite_support/java/src/java/org/tensorflow/lite/task/text/nlclassifier/NLClassifier.java +++ b/tensorflow_lite_support/java/src/java/org/tensorflow/lite/task/text/nlclassifier/NLClassifier.java @@ -126,7 +126,7 @@ public class NLClassifier extends BaseTaskApi { } } - private static final String NL_CLASSIFIER_NATIVE_LIBNAME = "task_text_jni"; + private static final String NL_CLASSIFIER_NATIVE_LIBNAME = "tflite_support_classifiers_native"; /** * Constructor to initialize the JNI with a pointer from C++. diff --git a/tensorflow_lite_support/java/src/java/org/tensorflow/lite/task/text/qa/BertQuestionAnswerer.java b/tensorflow_lite_support/java/src/java/org/tensorflow/lite/task/text/qa/BertQuestionAnswerer.java index 76f562ef..7c019d21 100644 --- a/tensorflow_lite_support/java/src/java/org/tensorflow/lite/task/text/qa/BertQuestionAnswerer.java +++ b/tensorflow_lite_support/java/src/java/org/tensorflow/lite/task/text/qa/BertQuestionAnswerer.java @@ -28,7 +28,7 @@ import org.tensorflow.lite.task.core.TaskJniUtils.MultipleBuffersHandleProvider; /** Task API for BertQA models. */ public class BertQuestionAnswerer extends BaseTaskApi implements QuestionAnswerer { - private static final String BERT_QUESTION_ANSWERER_NATIVE_LIBNAME = "task_text_jni"; + private static final String BERT_QUESTION_ANSWERER_NATIVE_LIBNAME = "tflite_support_classifiers_native"; private BertQuestionAnswerer(long nativeHandle) { super(nativeHandle); diff --git a/tensorflow_lite_support/java/src/javatests/org/tensorflow/lite/task/core/TestUtils.java b/tensorflow_lite_support/java/src/javatests/org/tensorflow/lite/task/core/TestUtils.java new file mode 100644 index 00000000..4c6f369d --- /dev/null +++ b/tensorflow_lite_support/java/src/javatests/org/tensorflow/lite/task/core/TestUtils.java @@ -0,0 +1,50 @@ +package org.tensorflow.lite.task.core; + +import android.content.Context; +import android.content.res.AssetManager; + +import com.google.common.io.ByteStreams; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +/** Helper class for the Java test in Task Libary. */ +public final class TestUtils { + + /** + * Loads the file and create a {@link File} object by reading a file from the asset directory. + * Simulates downloading or reading a file that's not precompiled with the app. + * + * @return a {@link File} object for the model. + */ + public static File loadFile(Context context, String fileName) { + File target = new File(context.getFilesDir(), fileName); + try (InputStream is = context.getAssets().open(fileName); + FileOutputStream os = new FileOutputStream(target)) { + ByteStreams.copy(is, os); + } catch (IOException e) { + throw new AssertionError("Failed to load model file at " + fileName, e); + } + return target; + } + + /** + * Reads a file into a direct {@link ByteBuffer} object from the asset directory. + * + * @return a {@link ByteBuffer} object for the file. + */ + public static ByteBuffer loadToDirectByteBuffer(Context context, String fileName) + throws IOException { + AssetManager assetManager = context.getAssets(); + InputStream inputStream = assetManager.open(fileName); + byte[] bytes = ByteStreams.toByteArray(inputStream); + + ByteBuffer buffer = ByteBuffer.allocateDirect(bytes.length).order(ByteOrder.nativeOrder()); + buffer.put(bytes); + return buffer; + } +}
\ No newline at end of file diff --git a/tensorflow_lite_support/java/src/javatests/org/tensorflow/lite/task/text/nlclassifier/BertNLClassifierTest.java b/tensorflow_lite_support/java/src/javatests/org/tensorflow/lite/task/text/nlclassifier/BertNLClassifierTest.java new file mode 100644 index 00000000..71d8fee2 --- /dev/null +++ b/tensorflow_lite_support/java/src/javatests/org/tensorflow/lite/task/text/nlclassifier/BertNLClassifierTest.java @@ -0,0 +1,128 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +package org.tensorflow.lite.task.text.nlclassifier; + +import static com.google.common.truth.Truth.assertThat; + +import androidx.test.core.app.ApplicationProvider; + +import java.io.IOException; +import java.util.List; + +import org.junit.Test; +import org.tensorflow.lite.support.label.Category; +import org.tensorflow.lite.task.core.TestUtils; + +/** Test for {@link BertNLClassifier}. */ +public class BertNLClassifierTest { + private static final String MODEL_FILE = "bert_nl_classifier.tflite"; + // A classifier model with dynamic input tensors. Provided by the Android Rubidium team. + private static final String DYNAMIC_INPUT_MODEL_FILE = "rb_v4_model.tflite"; + + Category findCategoryWithLabel(List<Category> list, String label) { + return list.stream() + .filter(category -> label.equals(category.getLabel())) + .findAny() + .orElse(null); + } + + @Test + public void createFromPath_verifyResults() throws IOException { + verifyResults( + BertNLClassifier.createFromFile(ApplicationProvider.getApplicationContext(), + MODEL_FILE)); + } + + @Test + public void createFromFile_verifyResults() throws IOException { + verifyResults( + BertNLClassifier.createFromFile( + TestUtils.loadFile(ApplicationProvider.getApplicationContext(), + MODEL_FILE))); + } + + @Test + public void classify_succeedsWithModelFile() throws IOException { + verifyResults( + BertNLClassifier.createFromFile( + ApplicationProvider.getApplicationContext(), MODEL_FILE)); + } + + @Test + public void classify_succeedsWithModelBuffer() throws IOException { + verifyResults( + BertNLClassifier.createFromBuffer( + TestUtils.loadToDirectByteBuffer( + ApplicationProvider.getApplicationContext(), MODEL_FILE))); + } + + @Test + public void classify_succeedsWithDynamicInputModelBuffer() throws IOException { + verifyDynamicInputResults( + BertNLClassifier.createFromBuffer( + TestUtils.loadToDirectByteBuffer( + ApplicationProvider.getApplicationContext(), + DYNAMIC_INPUT_MODEL_FILE))); + } + + @Test + public void getModelVersion_succeedsWithVersionInMetadata() throws IOException { + BertNLClassifier classifier = BertNLClassifier.createFromFile( + ApplicationProvider.getApplicationContext(), MODEL_FILE); + + assertThat(classifier.getModelVersion()).isEqualTo("v1"); + } + + @Test + public void getModelVersion_succeedsWithDynamicInputModelVersion() throws IOException { + BertNLClassifier classifier = BertNLClassifier.createFromFile( + ApplicationProvider.getApplicationContext(), DYNAMIC_INPUT_MODEL_FILE); + + assertThat(classifier.getModelVersion()).isEqualTo("4"); + } + + @Test + public void getLabelsVersion_succeedsWithNoVersionInMetadata() throws IOException { + BertNLClassifier classifier = BertNLClassifier.createFromFile( + ApplicationProvider.getApplicationContext(), MODEL_FILE); + + assertThat(classifier.getLabelsVersion()).isEqualTo("NO_VERSION_INFO"); + } + + @Test + public void getLabelsVersion_succeedsWithDynamicInputLabelsVersion() throws IOException { + BertNLClassifier classifier = BertNLClassifier.createFromFile( + ApplicationProvider.getApplicationContext(), DYNAMIC_INPUT_MODEL_FILE); + + assertThat(classifier.getLabelsVersion()).isEqualTo("2"); + } + + private void verifyResults(BertNLClassifier classifier) { + List<Category> negativeResults = classifier.classify("unflinchingly bleak and desperate"); + assertThat(findCategoryWithLabel(negativeResults, "negative").getScore()) + .isGreaterThan(findCategoryWithLabel(negativeResults, "positive").getScore()); + + List<Category> positiveResults = + classifier.classify("it's a charming and often affecting journey"); + assertThat(findCategoryWithLabel(positiveResults, "positive").getScore()) + .isGreaterThan(findCategoryWithLabel(positiveResults, "negative").getScore()); + } + + private void verifyDynamicInputResults(BertNLClassifier classifier) { + List<Category> topics = classifier.classify("FooBarBaz"); + assertThat(topics.size()).isEqualTo(446); + } +} diff --git a/tensorflow_lite_support/java/src/javatests/testdata/task/text/bert_nl_classifier.tflite b/tensorflow_lite_support/java/src/javatests/testdata/task/text/bert_nl_classifier.tflite Binary files differnew file mode 100644 index 00000000..97a32da4 --- /dev/null +++ b/tensorflow_lite_support/java/src/javatests/testdata/task/text/bert_nl_classifier.tflite diff --git a/tensorflow_lite_support/java/src/javatests/testdata/task/text/rb_v4_model.tflite b/tensorflow_lite_support/java/src/javatests/testdata/task/text/rb_v4_model.tflite Binary files differnew file mode 100644 index 00000000..7265ba27 --- /dev/null +++ b/tensorflow_lite_support/java/src/javatests/testdata/task/text/rb_v4_model.tflite diff --git a/tensorflow_lite_support/java/src/native/task/core/BUILD b/tensorflow_lite_support/java/src/native/task/core/BUILD index d4dd7ab3..65e9e30c 100644 --- a/tensorflow_lite_support/java/src/native/task/core/BUILD +++ b/tensorflow_lite_support/java/src/native/task/core/BUILD @@ -3,7 +3,7 @@ package( licenses = ["notice"], # Apache 2.0 ) -# Default provider for BuiltInOpResover. Create your own target, overwrite the +# Default provider for BuiltInOpResolver. Create your own target, overwrite the # function to provide a MutableOpResolver for customized OPs and/or a subset of # builtin OPs. cc_library( @@ -14,3 +14,13 @@ cc_library( "@org_tensorflow//tensorflow/lite/kernels:builtin_ops", ], ) + +cc_library( + name = "rbtml_op_resolver", + srcs = ["rbtml_op_resolver.cc"], + deps = [ + "//tensorflow_lite_support/custom_ops/kernel:unsorted_segment", + "@org_tensorflow//tensorflow/lite:framework", + "@org_tensorflow//tensorflow/lite/kernels:builtin_ops", + ], +) diff --git a/tensorflow_lite_support/java/src/native/task/core/minimal_op_resolver.cc b/tensorflow_lite_support/java/src/native/task/core/minimal_op_resolver.cc new file mode 100644 index 00000000..32d1054d --- /dev/null +++ b/tensorflow_lite_support/java/src/native/task/core/minimal_op_resolver.cc @@ -0,0 +1,89 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include <memory> + +#include "tensorflow/lite/kernels/builtin_op_kernels.h" +#include "tensorflow/lite/op_resolver.h" + +namespace tflite { +namespace task { + +// Create a minimal MutableOpResolver to provide only +// the ops required by the bert_nl_classifier and rb_model for BertNLClassifier. +std::unique_ptr<MutableOpResolver> CreateOpResolver() { + MutableOpResolver resolver; + resolver.AddBuiltin(::tflite::BuiltinOperator_RESHAPE, + ::tflite::ops::builtin::Register_RESHAPE()); + resolver.AddBuiltin(::tflite::BuiltinOperator_GATHER, + ::tflite::ops::builtin::Register_GATHER()); + resolver.AddBuiltin(::tflite::BuiltinOperator_STRIDED_SLICE, + ::tflite::ops::builtin::Register_STRIDED_SLICE()); + resolver.AddBuiltin(::tflite::BuiltinOperator_FULLY_CONNECTED, + ::tflite::ops::builtin::Register_FULLY_CONNECTED()); + resolver.AddBuiltin(::tflite::BuiltinOperator_CAST, + ::tflite::ops::builtin::Register_CAST()); + resolver.AddBuiltin(::tflite::BuiltinOperator_MUL, + ::tflite::ops::builtin::Register_MUL()); + resolver.AddBuiltin(::tflite::BuiltinOperator_ADD, + ::tflite::ops::builtin::Register_ADD()); + resolver.AddBuiltin(::tflite::BuiltinOperator_TRANSPOSE, + ::tflite::ops::builtin::Register_TRANSPOSE()); + resolver.AddBuiltin(::tflite::BuiltinOperator_SPLIT, + ::tflite::ops::builtin::Register_SPLIT()); + resolver.AddBuiltin(::tflite::BuiltinOperator_PACK, + ::tflite::ops::builtin::Register_PACK()); + resolver.AddBuiltin(::tflite::BuiltinOperator_SOFTMAX, + ::tflite::ops::builtin::Register_SOFTMAX()); + resolver.AddBuiltin(::tflite::BuiltinOperator_EXPAND_DIMS, + ::tflite::ops::builtin::Register_EXPAND_DIMS()); + resolver.AddBuiltin(::tflite::BuiltinOperator_SHAPE, + ::tflite::ops::builtin::Register_SHAPE()); + resolver.AddBuiltin(::tflite::BuiltinOperator_FILL, + ::tflite::ops::builtin::Register_FILL()); + resolver.AddBuiltin(::tflite::BuiltinOperator_SUB, + ::tflite::ops::builtin::Register_SUB()); + resolver.AddBuiltin(::tflite::BuiltinOperator_MEAN, + ::tflite::ops::builtin::Register_MEAN()); + resolver.AddBuiltin(::tflite::BuiltinOperator_SQUARED_DIFFERENCE, + ::tflite::ops::builtin::Register_SQUARED_DIFFERENCE()); + resolver.AddBuiltin(::tflite::BuiltinOperator_RSQRT, + ::tflite::ops::builtin::Register_RSQRT()); + resolver.AddBuiltin(::tflite::BuiltinOperator_BATCH_MATMUL, + ::tflite::ops::builtin::Register_BATCH_MATMUL()); + resolver.AddBuiltin(::tflite::BuiltinOperator_GELU, + ::tflite::ops::builtin::Register_GELU()); + resolver.AddBuiltin(::tflite::BuiltinOperator_TANH, + ::tflite::ops::builtin::Register_TANH()); + resolver.AddBuiltin(::tflite::BuiltinOperator_LOGISTIC, + ::tflite::ops::builtin::Register_LOGISTIC()); + resolver.AddBuiltin(::tflite::BuiltinOperator_SLICE, + ::tflite::ops::builtin::Register_SLICE()); + // Needed for the test bert_nl_classifier model. + resolver.AddBuiltin(::tflite::BuiltinOperator_PAD, + ::tflite::ops::builtin::Register_PAD()); + resolver.AddBuiltin(::tflite::BuiltinOperator_CONCATENATION, + ::tflite::ops::builtin::Register_CONCATENATION()); + resolver.AddBuiltin(::tflite::BuiltinOperator_FULLY_CONNECTED, + ::tflite::ops::builtin::Register_FULLY_CONNECTED(), + /*version=*/9); + resolver.AddBuiltin(::tflite::BuiltinOperator_DEQUANTIZE, + ::tflite::ops::builtin::Register_DEQUANTIZE(), + /*version=*/2); + return std::make_unique<MutableOpResolver>(resolver); +} + +} // namespace task +} // namespace tflite
\ No newline at end of file diff --git a/tensorflow_lite_support/java/src/native/task/core/rbtml_op_resolver.cc b/tensorflow_lite_support/java/src/native/task/core/rbtml_op_resolver.cc new file mode 100644 index 00000000..c35501cb --- /dev/null +++ b/tensorflow_lite_support/java/src/native/task/core/rbtml_op_resolver.cc @@ -0,0 +1,40 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include <memory> + +#include "tensorflow/lite/kernels/register.h" + +#include "tensorflow_lite_support/custom_ops/kernel/unsorted_segment.h" + +namespace tflite { +namespace task { +// Create a custom op resolver to provide the unsorted_segment_prod op +// required by the bert_nl_classifier and rb_model for BertNLClassifier. +std::unique_ptr<tflite::OpResolver> CreateOpResolver() { // NOLINT + std::unique_ptr<tflite::ops::builtin::BuiltinOpResolver> resolver( + new tflite::ops::builtin::BuiltinOpResolver); + // "UnsortedSegmentProd" is the name used by unsorted_segment_prod op when + // when converting SavedModel to tflite using the size optimization approach. + resolver->AddCustom("UnsortedSegmentProd", + tflite::ops::custom::Register_UNSORTED_SEGMENT_PROD()); + // "FlexUnsortedSegmentProd" is the name used by unsorted_segment_prod op when + // when converting SavedModel to tflite using the the other approaches. + resolver->AddCustom("FlexUnsortedSegmentProd", + tflite::ops::custom::Register_UNSORTED_SEGMENT_PROD()); + return std::unique_ptr<tflite::OpResolver>(std::move(resolver)); +} + +} // namespace task +} // namespace tflite
\ No newline at end of file diff --git a/tensorflow_lite_support/java/src/native/task/text/nlclassifier/bert_nlclassifier/bert_nl_classifier_jni.cc b/tensorflow_lite_support/java/src/native/task/text/nlclassifier/bert_nlclassifier/bert_nl_classifier_jni.cc index 1edb3507..7781a4a8 100644 --- a/tensorflow_lite_support/java/src/native/task/text/nlclassifier/bert_nlclassifier/bert_nl_classifier_jni.cc +++ b/tensorflow_lite_support/java/src/native/task/text/nlclassifier/bert_nlclassifier/bert_nl_classifier_jni.cc @@ -14,11 +14,18 @@ limitations under the License. ==============================================================================*/ #include <jni.h> - +#include "tensorflow/lite/op_resolver.h" #include "tensorflow_lite_support/cc/task/text/nlclassifier/bert_nl_classifier.h" #include "tensorflow_lite_support/cc/utils/jni_utils.h" #include "tensorflow_lite_support/java/src/native/task/text/nlclassifier/nl_classifier_jni_utils.h" +namespace tflite { +namespace task { +// To be provided by a link-time library +extern std::unique_ptr<OpResolver> CreateOpResolver(); +} // namespace task +} // namespace tflite + namespace { using ::tflite::support::utils::GetMappedFileBuffer; @@ -27,6 +34,8 @@ using ::tflite::support::utils::kInvalidPointer; using ::tflite::support::utils::ThrowException; using ::tflite::task::text::nlclassifier::BertNLClassifier; using ::tflite::task::text::nlclassifier::RunClassifier; +using ::tflite::task::text::nlclassifier::GetModelVersionNative; +using ::tflite::task::text::nlclassifier::GetLabelsVersionNative; extern "C" JNIEXPORT void JNICALL Java_org_tensorflow_lite_task_text_nlclassifier_BertNLClassifier_deinitJni( @@ -39,7 +48,8 @@ Java_org_tensorflow_lite_task_text_nlclassifier_BertNLClassifier_initJniWithByte JNIEnv* env, jclass thiz, jobject model_buffer) { auto model = GetMappedFileBuffer(env, model_buffer); tflite::support::StatusOr<std::unique_ptr<BertNLClassifier>> status = - BertNLClassifier::CreateFromBuffer(model.data(), model.size()); + BertNLClassifier::CreateFromBuffer(model.data(), model.size(), + tflite::task::CreateOpResolver()); if (status.ok()) { return reinterpret_cast<jlong>(status->release()); } else { @@ -54,7 +64,7 @@ extern "C" JNIEXPORT jlong JNICALL Java_org_tensorflow_lite_task_text_nlclassifier_BertNLClassifier_initJniWithFileDescriptor( JNIEnv* env, jclass thiz, jint fd) { tflite::support::StatusOr<std::unique_ptr<BertNLClassifier>> status = - BertNLClassifier::CreateFromFd(fd); + BertNLClassifier::CreateFromFd(fd, tflite::task::CreateOpResolver()); if (status.ok()) { return reinterpret_cast<jlong>(status->release()); } else { @@ -71,4 +81,16 @@ Java_org_tensorflow_lite_task_text_nlclassifier_BertNLClassifier_classifyNative( return RunClassifier(env, native_handle, text); } +extern "C" JNIEXPORT jstring JNICALL +Java_org_tensorflow_lite_task_text_nlclassifier_BertNLClassifier_getModelVersionNative( + JNIEnv* env, jclass clazz, jlong native_handle) { + return GetModelVersionNative(env, native_handle); +} + +extern "C" JNIEXPORT jstring JNICALL +Java_org_tensorflow_lite_task_text_nlclassifier_BertNLClassifier_getLabelsVersionNative( + JNIEnv* env, jclass clazz, jlong native_handle) { + return GetLabelsVersionNative(env, native_handle); +} + } // namespace diff --git a/tensorflow_lite_support/java/src/native/task/text/nlclassifier/nl_classifier_jni_utils.cc b/tensorflow_lite_support/java/src/native/task/text/nlclassifier/nl_classifier_jni_utils.cc index c358bee1..d2f6e7ca 100644 --- a/tensorflow_lite_support/java/src/native/task/text/nlclassifier/nl_classifier_jni_utils.cc +++ b/tensorflow_lite_support/java/src/native/task/text/nlclassifier/nl_classifier_jni_utils.cc @@ -23,6 +23,9 @@ namespace task { namespace text { namespace nlclassifier { +using ::tflite::support::utils::kAssertionError; +using ::tflite::support::utils::kInvalidPointer; +using ::tflite::support::utils::ThrowException; using ::tflite::support::utils::ConvertVectorToArrayList; using ::tflite::support::utils::JStringToString; using ::tflite::task::core::Category; @@ -31,14 +34,21 @@ using ::tflite::task::text::nlclassifier::NLClassifier; jobject RunClassifier(JNIEnv* env, jlong native_handle, jstring text) { auto* nl_classifier = reinterpret_cast<NLClassifier*>(native_handle); - auto results = nl_classifier->Classify(JStringToString(env, text)); + auto results = nl_classifier->ClassifyText(JStringToString(env, text)); + if (!results.ok()) { + ThrowException(env, kAssertionError, + "Error occurred when running classifier: %s", + results.status().message().data()); + return env->ExceptionOccurred(); + } + jclass category_class = env->FindClass("org/tensorflow/lite/support/label/Category"); jmethodID category_init = env->GetMethodID(category_class, "<init>", "(Ljava/lang/String;F)V"); return ConvertVectorToArrayList<Category>( - env, results, + env, results.value(), [env, category_class, category_init](const Category& category) { jstring class_name = env->NewStringUTF(category.class_name.data()); // Convert double to float as Java interface exposes float as scores. @@ -50,6 +60,16 @@ jobject RunClassifier(JNIEnv* env, jlong native_handle, jstring text) { }); } +jstring GetModelVersionNative(JNIEnv* env, jlong native_handle) { + auto* nl_classifier = reinterpret_cast<NLClassifier*>(native_handle); + return env->NewStringUTF(nl_classifier->GetModelVersion().c_str()); +} + +jstring GetLabelsVersionNative(JNIEnv* env, jlong native_handle) { + auto* nl_classifier = reinterpret_cast<NLClassifier*>(native_handle); + return env->NewStringUTF(nl_classifier->GetLabelsVersion().c_str()); +} + } // namespace nlclassifier } // namespace text } // namespace task diff --git a/tensorflow_lite_support/java/src/native/task/text/nlclassifier/nl_classifier_jni_utils.h b/tensorflow_lite_support/java/src/native/task/text/nlclassifier/nl_classifier_jni_utils.h index 2c59ab50..c21eda70 100644 --- a/tensorflow_lite_support/java/src/native/task/text/nlclassifier/nl_classifier_jni_utils.h +++ b/tensorflow_lite_support/java/src/native/task/text/nlclassifier/nl_classifier_jni_utils.h @@ -25,6 +25,10 @@ namespace nlclassifier { jobject RunClassifier(JNIEnv* env, jlong native_handle, jstring text); +jstring GetModelVersionNative(JNIEnv* env, jlong native_handle); + +jstring GetLabelsVersionNative(JNIEnv* env, jlong native_handle); + } // namespace nlclassifier } // namespace text } // namespace task diff --git a/tensorflow_lite_support/java/tflite_version_script.lds b/tensorflow_lite_support/java/tflite_version_script.lds new file mode 100644 index 00000000..604c923a --- /dev/null +++ b/tensorflow_lite_support/java/tflite_version_script.lds @@ -0,0 +1,10 @@ +VERS_1.0 { + # Export JNI and native C symbols. + global: + Java_*; + + # Hide everything else. + local: + *; +}; + diff --git a/tensorflow_lite_support/metadata/cc/metadata_extractor.cc b/tensorflow_lite_support/metadata/cc/metadata_extractor.cc index cf4edaa7..c2d85bc0 100644 --- a/tensorflow_lite_support/metadata/cc/metadata_extractor.cc +++ b/tensorflow_lite_support/metadata/cc/metadata_extractor.cc @@ -1,4 +1,4 @@ -/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -15,24 +15,21 @@ limitations under the License. #include "tensorflow_lite_support/metadata/cc/metadata_extractor.h" -#include <functional> +#include <string> #include "absl/memory/memory.h" #include "absl/status/status.h" #include "absl/strings/str_format.h" -#include "flatbuffers/flatbuffers.h" // from @flatbuffers -#include "lib/zip.h" // from @org_libzip +#include "absl/strings/string_view.h" +#include "flatbuffers/flatbuffers.h" +#include "contrib/minizip/ioapi.h" +#include "contrib/minizip/unzip.h" #include "tensorflow/lite/schema/schema_generated.h" #include "tensorflow_lite_support/cc/common.h" #include "tensorflow_lite_support/cc/port/status_macros.h" +#include "tensorflow_lite_support/metadata/cc/utils/zip_readonly_mem_file.h" #include "tensorflow_lite_support/metadata/metadata_schema_generated.h" -#if TFLITE_USE_C_API -#include "tensorflow/lite/c/c_api.h" -#else -#include "tensorflow/lite/model_builder.h" -#endif - namespace tflite { namespace metadata { @@ -46,26 +43,6 @@ using ::tflite::TensorMetadata; using ::tflite::support::CreateStatusWithPayload; using ::tflite::support::TfLiteSupportStatus; -// Helper class that takes a callback function, and invokes it in its -// destructor. -class SimpleCleanUp { - public: - explicit SimpleCleanUp(std::function<void()> callback) - : callback_(std::move(callback)) {} - - ~SimpleCleanUp() { - if (callback_ != nullptr) callback_(); - } - - // Use `std::move(simple_cleanup).Cancel()` to prevent the callback from ever - // executing at all. Once a SimpleCleanUp object has been `std::move(...)`-ed, - // it may not be read from again. - void Cancel() && { callback_ = nullptr; } - - private: - std::function<void()> callback_; -}; - // Util to get item from src_vector specified by index. template <typename T> const T* GetItemFromVector( @@ -75,6 +52,70 @@ const T* GetItemFromVector( } return src_vector->Get(index); } + +// Wrapper function around calls to unzip to avoid repeating conversion logic +// from error code to Status. +absl::Status UnzipErrorToStatus(int error) { + if (error != UNZ_OK) { + return CreateStatusWithPayload( + StatusCode::kUnknown, "Unable to read associated file in zip archive.", + TfLiteSupportStatus::kMetadataAssociatedFileZipError); + } + return absl::OkStatus(); +} + +// Stores a file name, position in zip buffer and size. +struct ZipFileInfo { + std::string name; + ZPOS64_T position; + ZPOS64_T size; +}; + +// Returns the ZipFileInfo corresponding to the current file in the provided +// unzFile object. +tflite::support::StatusOr<ZipFileInfo> GetCurrentZipFileInfo(const unzFile& zf) { + // Open file in raw mode, as data is expected to be uncompressed. + int method; + RETURN_IF_ERROR(UnzipErrorToStatus( + unzOpenCurrentFile2(zf, &method, /*level=*/nullptr, /*raw=*/1))); + if (method != Z_NO_COMPRESSION) { + return CreateStatusWithPayload( + StatusCode::kUnknown, "Expected uncompressed zip archive.", + TfLiteSupportStatus::kMetadataAssociatedFileZipError); + } + + // Get file info a first time to get filename size. + unz_file_info64 file_info; + RETURN_IF_ERROR(UnzipErrorToStatus(unzGetCurrentFileInfo64( + zf, &file_info, /*szFileName=*/nullptr, /*szFileNameBufferSize=*/0, + /*extraField=*/nullptr, /*extraFieldBufferSize=*/0, + /*szComment=*/nullptr, /*szCommentBufferSize=*/0))); + + // Second call to get file name. + auto file_name_size = file_info.size_filename; + char* c_file_name = (char*)malloc(file_name_size); + RETURN_IF_ERROR(UnzipErrorToStatus(unzGetCurrentFileInfo64( + zf, &file_info, c_file_name, file_name_size, + /*extraField=*/nullptr, /*extraFieldBufferSize=*/0, + /*szComment=*/nullptr, /*szCommentBufferSize=*/0))); + std::string file_name = std::string(c_file_name, file_name_size); + free(c_file_name); + + // Get position in file. + auto position = unzGetCurrentFileZStreamPos64(zf); + if (position == 0) { + return CreateStatusWithPayload( + StatusCode::kUnknown, "Unable to read file in zip archive.", + TfLiteSupportStatus::kMetadataAssociatedFileZipError); + } + ZipFileInfo result = {.name = file_name, + .position = position, + .size = file_info.uncompressed_size}; + + // Close file and return. + RETURN_IF_ERROR(UnzipErrorToStatus(unzCloseCurrentFile(zf))); + return result; +} } // namespace /* static */ @@ -155,6 +196,9 @@ absl::Status ModelMetadataExtractor::InitFromModelBuffer( // Look for the "TFLITE_METADATA" field, if any. for (int i = 0; i < model_->metadata()->size(); ++i) { const auto metadata = model_->metadata()->Get(i); + if (!metadata->name()) { + continue; + } if (metadata->name()->str() != kMetadataBufferName) { continue; } @@ -191,71 +235,45 @@ absl::Status ModelMetadataExtractor::InitFromModelBuffer( absl::Status ModelMetadataExtractor::ExtractAssociatedFiles( const char* buffer_data, size_t buffer_size) { - // Setup libzip error reporting. - zip_error_t error; - zip_error_init(&error); - auto zip_error_cleanup = SimpleCleanUp([&error] { zip_error_fini(&error); }); - - // Initialize zip source. - zip_source_t* src = - zip_source_buffer_create(buffer_data, buffer_size, /*freep=*/0, &error); - if (src == nullptr) { - return CreateStatusWithPayload( - StatusCode::kUnknown, - absl::StrFormat("Can't create zip source from model buffer: %s", - zip_error_strerror(&error)), - TfLiteSupportStatus::kMetadataAssociatedFileZipError); - } - auto zip_source_cleanup = SimpleCleanUp([src] { zip_source_free(src); }); - - // Try opening zip source. - zip* zip_archive = zip_open_from_source(src, /*flags=*/0, &error); - if (zip_archive == nullptr) { + // Create in-memory read-only zip file. + ZipReadOnlyMemFile mem_file = ZipReadOnlyMemFile(buffer_data, buffer_size); + // Open zip. + unzFile zf = unzOpen2_64(/*path=*/nullptr, &mem_file.GetFileFunc64Def()); + if (zf == nullptr) { // It's OK if it fails: this means there are no associated files with this // model. return absl::OkStatus(); } - auto zip_archive_cleanup = - SimpleCleanUp([zip_archive] { zip_close(zip_archive); }); - // As per the documentation [1] for zip_source_free, it should not be called - // after a successful call to zip_open_from_source. - // - // [1]: https://libzip.org/documentation/zip_source_free.html - std::move(zip_source_cleanup).Cancel(); - - const int num_files = zip_get_num_entries(zip_archive, /*flags=*/0); - for (int index = 0; index < num_files; ++index) { - // Get file stats. - struct zip_stat zip_file_stat; - zip_stat_init(&zip_file_stat); - zip_stat_index(zip_archive, index, /*flags=*/0, &zip_file_stat); - absl::string_view filename = zip_file_stat.name; - const auto unzip_filesize = zip_file_stat.size; - - // Open file. - zip_file* zip_file = zip_fopen_index(zip_archive, index, /*flags=*/0); - if (zip_file == nullptr) { - return CreateStatusWithPayload( - StatusCode::kUnknown, - absl::StrFormat("Unable to open associated file with name: %s", - zip_file_stat.name), - TfLiteSupportStatus::kMetadataAssociatedFileZipError); - } - auto zip_file_cleanup = SimpleCleanUp([zip_file] { zip_fclose(zip_file); }); + // Get number of files. + unz_global_info global_info; + if (unzGetGlobalInfo(zf, &global_info) != UNZ_OK) { + return CreateStatusWithPayload( + StatusCode::kUnknown, "Unable to get zip archive info.", + TfLiteSupportStatus::kMetadataAssociatedFileZipError); + } - // Unzip file. - char* unzip_buffer = new char[unzip_filesize]; - auto unzip_buffer_cleanup = - SimpleCleanUp([unzip_buffer] { delete[] unzip_buffer; }); - if (zip_fread(zip_file, unzip_buffer, unzip_filesize) != unzip_filesize) { + // Browse through files in archive. + if (global_info.number_entry > 0) { + int error = unzGoToFirstFile(zf); + while (error == UNZ_OK) { + ASSIGN_OR_RETURN(auto zip_file_info, GetCurrentZipFileInfo(zf)); + // Store result in map. + associated_files_[zip_file_info.name] = absl::string_view( + buffer_data + zip_file_info.position, zip_file_info.size); + error = unzGoToNextFile(zf); + } + if (error != UNZ_END_OF_LIST_OF_FILE) { return CreateStatusWithPayload( StatusCode::kUnknown, - absl::StrFormat("Unzipping failed for file: %s.", filename), + "Unable to read associated file in zip archive.", TfLiteSupportStatus::kMetadataAssociatedFileZipError); } - - // Copy file contents in map. - associated_files_[filename] = std::string(unzip_buffer, unzip_filesize); + } + // Close zip. + if (unzClose(zf) != UNZ_OK) { + return CreateStatusWithPayload( + StatusCode::kUnknown, "Unable to close zip archive.", + TfLiteSupportStatus::kMetadataAssociatedFileZipError); } return absl::OkStatus(); } @@ -272,6 +290,23 @@ ModelMetadataExtractor::GetAssociatedFile(const std::string& filename) const { return it->second; } +tflite::support::StatusOr<std::string> +ModelMetadataExtractor::GetModelVersion() const { + if (model_metadata_ == nullptr) { + return CreateStatusWithPayload( + StatusCode::kFailedPrecondition, + "No model metadata", + TfLiteSupportStatus::kMetadataNotFoundError); + } + if (model_metadata_->version() == nullptr) { + return CreateStatusWithPayload( + StatusCode::kNotFound, + "No version in model metadata", + TfLiteSupportStatus::kMetadataNotFoundError); + } + return model_metadata_->version()->str(); +} + const flatbuffers::Vector<flatbuffers::Offset<tflite::TensorMetadata>>* ModelMetadataExtractor::GetInputTensorMetadata() const { if (model_metadata_ == nullptr || @@ -363,4 +398,4 @@ int ModelMetadataExtractor::GetOutputProcessUnitsCount() const { } } // namespace metadata -} // namespace tflite +} // namespace tflite
\ No newline at end of file diff --git a/tensorflow_lite_support/metadata/cc/metadata_extractor.h b/tensorflow_lite_support/metadata/cc/metadata_extractor.h index 8eafe932..bff308ab 100644 --- a/tensorflow_lite_support/metadata/cc/metadata_extractor.h +++ b/tensorflow_lite_support/metadata/cc/metadata_extractor.h @@ -1,4 +1,4 @@ -/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -15,6 +15,8 @@ limitations under the License. #ifndef TENSORFLOW_LITE_SUPPORT_METADATA_CC_METADATA_EXTRACTOR_H_ #define TENSORFLOW_LITE_SUPPORT_METADATA_CC_METADATA_EXTRACTOR_H_ +#include <string> + #include "absl/container/flat_hash_map.h" #include "absl/status/status.h" #include "absl/strings/string_view.h" @@ -78,6 +80,10 @@ class ModelMetadataExtractor { tflite::support::StatusOr<absl::string_view> GetAssociatedFile( const std::string& filename) const; + // Gets the model version from the model metadata. An error is returned if + // either the metadata does not exist or no model version is present in it. + tflite::support::StatusOr<std::string> GetModelVersion() const; + // Note: all methods below retrieves metadata of the *first* subgraph as // default. @@ -146,9 +152,9 @@ class ModelMetadataExtractor { // Pointer to the extracted ModelMetadata, if any. const tflite::ModelMetadata* model_metadata_{nullptr}; // The files associated with the ModelMetadata, as a map with the filename - // (corresponding to a basename, e.g. "labels.txt") as key and the file - // contents as value. - absl::flat_hash_map<std::string, std::string> associated_files_; + // (corresponding to a basename, e.g. "labels.txt") as key and a pointer to + // the file contents as value. + absl::flat_hash_map<std::string, absl::string_view> associated_files_; }; } // namespace metadata diff --git a/tensorflow_lite_support/metadata/cc/metadata_version.cc b/tensorflow_lite_support/metadata/cc/metadata_version.cc index 7679f6c4..baceca91 100644 --- a/tensorflow_lite_support/metadata/cc/metadata_version.cc +++ b/tensorflow_lite_support/metadata/cc/metadata_version.cc @@ -45,6 +45,9 @@ enum class SchemaMembers { kSubGraphMetadataInputTensorGroups = 5, kSubGraphMetadataOutputTensorGroups = 6, kProcessUnitOptionsRegexTokenizerOptions = 7, + kContentPropertiesAudioProperties = 8, + kAssociatedFileTypeScannIndexFile = 9, + kAssociatedFileVersion = 10, }; // Helper class to compare semantic versions in terms of three integers, major, @@ -107,6 +110,12 @@ Version GetMemberVersion(SchemaMembers member) { return Version(1, 2, 0); case SchemaMembers::kProcessUnitOptionsRegexTokenizerOptions: return Version(1, 2, 1); + case SchemaMembers::kContentPropertiesAudioProperties: + return Version(1, 3, 0); + case SchemaMembers::kAssociatedFileTypeScannIndexFile: + return Version(1, 4, 0); + case SchemaMembers::kAssociatedFileVersion: + return Version(1, 4, 1); default: // Should never happen. TFLITE_LOG(FATAL) << "Unsupported schema member: " @@ -148,6 +157,18 @@ void UpdateMinimumVersionForTable<tflite::AssociatedFile>( GetMemberVersion(SchemaMembers::kAssociatedFileTypeVocabulary), min_version); } + + if (table->type() == AssociatedFileType_SCANN_INDEX_FILE) { + UpdateMinimumVersion( + GetMemberVersion(SchemaMembers::kAssociatedFileTypeScannIndexFile), + min_version); + } + + if (table->version() != nullptr) { + UpdateMinimumVersion( + GetMemberVersion(SchemaMembers::kAssociatedFileVersion), + min_version); + } } template <> @@ -177,6 +198,19 @@ void UpdateMinimumVersionForTable<tflite::ProcessUnit>( } template <> +void UpdateMinimumVersionForTable<tflite::Content>(const tflite::Content* table, + Version* min_version) { + if (table == nullptr) return; + + // Checks the ContenProperties field. + if (table->content_properties_type() == ContentProperties_AudioProperties) { + UpdateMinimumVersion( + GetMemberVersion(SchemaMembers::kContentPropertiesAudioProperties), + min_version); + } +} + +template <> void UpdateMinimumVersionForTable<tflite::TensorMetadata>( const tflite::TensorMetadata* table, Version* min_version) { if (table == nullptr) return; @@ -188,6 +222,9 @@ void UpdateMinimumVersionForTable<tflite::TensorMetadata>( // Checks the process_units field. UpdateMinimumVersionForArray<tflite::ProcessUnit>(table->process_units(), min_version); + + // Check the content field. + UpdateMinimumVersionForTable<tflite::Content>(table->content(), min_version); } template <> diff --git a/tensorflow_lite_support/metadata/cc/utils/zip_readonly_mem_file.cc b/tensorflow_lite_support/metadata/cc/utils/zip_readonly_mem_file.cc new file mode 100644 index 00000000..448031b1 --- /dev/null +++ b/tensorflow_lite_support/metadata/cc/utils/zip_readonly_mem_file.cc @@ -0,0 +1,119 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_lite_support/metadata/cc/utils/zip_readonly_mem_file.h" + +#include <algorithm> +#include <cstdio> + +#include "absl/strings/string_view.h" +#include "contrib/minizip/ioapi.h" + +namespace tflite { +namespace metadata { + +ZipReadOnlyMemFile::ZipReadOnlyMemFile(const char* buffer, size_t size) + : data_(buffer, size), offset_(0) { + zlib_filefunc64_def_.zopen64_file = OpenFile; + zlib_filefunc64_def_.zread_file = ReadFile; + zlib_filefunc64_def_.zwrite_file = WriteFile; + zlib_filefunc64_def_.ztell64_file = TellFile; + zlib_filefunc64_def_.zseek64_file = SeekFile; + zlib_filefunc64_def_.zclose_file = CloseFile; + zlib_filefunc64_def_.zerror_file = ErrorFile; + zlib_filefunc64_def_.opaque = this; +} + +zlib_filefunc64_def& ZipReadOnlyMemFile::GetFileFunc64Def() { + return zlib_filefunc64_def_; +} + +/* static */ +voidpf ZipReadOnlyMemFile::OpenFile(voidpf opaque, const void* filename, + int mode) { + // Result is never used, but needs to be non-null for `zipOpen2` not to fail. + return opaque; +} + +/* static */ +uLong ZipReadOnlyMemFile::ReadFile(voidpf opaque, voidpf stream, void* buf, + uLong size) { + auto* mem_file = static_cast<ZipReadOnlyMemFile*>(opaque); + if (mem_file->offset_ < 0 || mem_file->Size() < mem_file->offset_) { + return 0; + } + if (mem_file->offset_ + size > mem_file->Size()) { + size = mem_file->Size() - mem_file->offset_; + } + memcpy(buf, + static_cast<const char*>(mem_file->data_.data()) + mem_file->offset_, + size); + mem_file->offset_ += size; + return size; +} + +/* static */ +uLong ZipReadOnlyMemFile::WriteFile(voidpf opaque, voidpf stream, + const void* buf, uLong size) { + // File is not writable. + return 0; +} + +/* static */ +ZPOS64_T ZipReadOnlyMemFile::TellFile(voidpf opaque, voidpf stream) { + return static_cast<ZipReadOnlyMemFile*>(opaque)->offset_; +} + +/* static */ +long ZipReadOnlyMemFile::SeekFile // NOLINT + (voidpf opaque, voidpf stream, ZPOS64_T offset, int origin) { + auto* mem_file = static_cast<ZipReadOnlyMemFile*>(opaque); + switch (origin) { + case SEEK_SET: + mem_file->offset_ = offset; + return 0; + case SEEK_CUR: + if (mem_file->offset_ + offset < 0 || + mem_file->offset_ + offset > mem_file->Size()) { + return -1; + } + mem_file->offset_ += offset; + return 0; + case SEEK_END: + if (mem_file->Size() - offset < 0 || + mem_file->Size() - offset > mem_file->Size()) { + return -1; + } + mem_file->offset_ = offset + mem_file->Size(); + return 0; + default: + return -1; + } +} + +/* static */ +int ZipReadOnlyMemFile::CloseFile(voidpf opaque, voidpf stream) { + // Nothing to do. + return 0; +} + +/* static */ +int ZipReadOnlyMemFile::ErrorFile(voidpf opaque, voidpf stream) { + // Unused. + return 0; +} + +} // namespace metadata +} // namespace tflite
\ No newline at end of file diff --git a/tensorflow_lite_support/metadata/cc/utils/zip_readonly_mem_file.h b/tensorflow_lite_support/metadata/cc/utils/zip_readonly_mem_file.h new file mode 100644 index 00000000..bdb82845 --- /dev/null +++ b/tensorflow_lite_support/metadata/cc/utils/zip_readonly_mem_file.h @@ -0,0 +1,73 @@ +/* Copyright 2022 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_LITE_SUPPORT_METADATA_CC_UTILS_ZIP_MEM_FILE_H_ +#define TENSORFLOW_LITE_SUPPORT_METADATA_CC_UTILS_ZIP_MEM_FILE_H_ + +#include <cstdlib> + +#include "absl/strings/string_view.h" +#include "contrib/minizip/ioapi.h" + +namespace tflite { +namespace metadata { + +// In-memory read-only zip file implementation. +// +// Adapted from [1], with a few key differences: +// * backed by an `absl::string_view` instead of malloc-ed C buffers, +// * supports opening the file for reading through `unzOpen2_64`. +// +// This class is NOT thread-safe. +// +// [1]: +// https://github.com/google/libkml/blob/master/third_party/zlib-1.2.3/contrib/minizip/iomem_simple.c +class ZipReadOnlyMemFile { + public: + // Constructs an in-memory read-only zip file from a buffer. Does not copy or + // take ownership over the provided buffer: the caller is responsible for + // ensuring the buffer outlives this object. + ZipReadOnlyMemFile(const char* buffer, size_t size); + // Provides access to the `zlib_filefunc64_def` implementation for the + // in-memory zip file. + zlib_filefunc64_def& GetFileFunc64Def(); + + private: + // The string view backing the in-memory file. + absl::string_view data_; + // The current offset in the file. + ZPOS64_T offset_; + // The `zlib_filefunc64_def` implementation for this in-memory zip file. + zlib_filefunc64_def zlib_filefunc64_def_; + + // Convenience function to access the current data size. + size_t Size() const { return data_.size(); } + + // The file function implementations used in the `zlib_filefunc64_def`. + static voidpf OpenFile(voidpf opaque, const void* filename, int mode); + static uLong ReadFile(voidpf opaque, voidpf stream, void* buf, uLong size); + static uLong WriteFile(voidpf opaque, voidpf stream, const void* buf, + uLong size); + static ZPOS64_T TellFile(voidpf opaque, voidpf stream); + static long SeekFile // NOLINT + (voidpf opaque, voidpf stream, ZPOS64_T offset, int origin); + static int CloseFile(voidpf opaque, voidpf stream); + static int ErrorFile(voidpf opaque, voidpf stream); +}; + +} // namespace metadata +} // namespace tflite + +#endif // TENSORFLOW_LITE_SUPPORT_METADATA_CC_UTILS_ZIP_MEM_FILE_H_
\ No newline at end of file diff --git a/tensorflow_lite_support/metadata/metadata_schema.fbs b/tensorflow_lite_support/metadata/metadata_schema.fbs index 8faae0a8..35b3bd94 100644 --- a/tensorflow_lite_support/metadata/metadata_schema.fbs +++ b/tensorflow_lite_support/metadata/metadata_schema.fbs @@ -50,7 +50,7 @@ namespace tflite; // for which they were added. // // LINT.IfChange -// Schema Semantic version: 1.2.1 +// Schema Semantic version: 1.4.1 // LINT.ThenChange(//tensorflow_lite_support/\ // metadata/java/src/java/org/tensorflow/lite/support/metadata/\ // MetadataParser.java) @@ -69,6 +69,9 @@ file_identifier "M001"; // 1.2.0 - Added input_tensor_group to SubGraphMetadata. // Added output_tensor_group to SubGraphMetadata. // 1.2.1 - Added RegexTokenizerOptions to ProcessUnitOptions. +// 1.3.0 - Added AudioProperties to ContentProperties. +// 1.4.0 - Added SCANN_INDEX_FILE type to AssociatedFileType. +// 1.4.1 - Added version to AssociatedFile. // File extension of any written files. file_extension "tflitemeta"; @@ -80,10 +83,12 @@ enum AssociatedFileType : byte { // Files such as readme.txt. DESCRIPTIONS = 1, - // Contains labels that annotate certain axis of the tensor. For example, + // Contains a list of labels (characters separated by "\n" or in lines) that + // annotate certain axis of the tensor. For example, // the label file in image classification. Those labels annotate the // the output tensor, such that each value in the output tensor is the - // probability of that corresponding category specified by the label. + // probability of that corresponding category specified by the label. See the + // example label file used in image classification [1]. // // <Codegen usage>: // If an output tensor has an associated file as TENSOR_AXIS_LABELS, return @@ -92,12 +97,16 @@ enum AssociatedFileType : byte { // If multiple files of the same type are present, the first one is used by // default; additional ones are to be distinguished from one another by their // specified locale. + // + // [1]: https://github.com/tensorflow/tflite-support/blob/master/tensorflow_lite_support/metadata/python/tests/testdata/image_classifier/labels.txt TENSOR_AXIS_LABELS = 2, - // Contains labels that tensor values correspond to. For example, in + // Contains a list of labels (characters separated by "\n" or in lines) that + // tensor values correspond to. For example, in // the object detection model, one of the output tensors is the detected // classes. And each value in the tensor refers to the index of label in the - // category label file. + // category label file. See the example label file used in object detection + // [1]. // // <Codegen usage>: // If an output tensor has an associated file as TENSOR_VALUE_LABELS, convert @@ -105,25 +114,41 @@ enum AssociatedFileType : byte { // If multiple files of the same type are present, the first one is used by // default; additional ones are to be distinguished from one another by their // specified locale. + // + // [1]: https://github.com/tensorflow/tflite-support/blob/master/tensorflow_lite_support/metadata/python/tests/testdata/object_detector/labelmap.txt TENSOR_VALUE_LABELS = 3, // Contains sigmoid-based score calibration parameters, formatted as CSV. // Lines contain for each index of an output tensor the scale, slope, offset // and (optional) min_score parameters to be used for sigmoid fitting (in this - // order and in `strtof`-compatible [1] format). + // order and in `strtof`-compatible [1] format). Scale should be a + // non-negative value. // A line may be left empty to default calibrated scores for this index to // default_score. // In summary, each line should thus contain 0, 3 or 4 comma-separated values. // + // See the example score calibration file used in image classification [2]. + // // See documentation for ScoreCalibrationOptions for details. // // [1]: https://en.cppreference.com/w/c/string/byte/strtof + // [2]: https://github.com/tensorflow/tflite-support/blob/master/tensorflow_lite_support/metadata/python/tests/testdata/image_classifier/score_calibration.txt TENSOR_AXIS_SCORE_CALIBRATION = 4, // Contains a list of unique words (characters separated by "\n" or in lines) // that help to convert natural language words to embedding vectors. + // + // See the example vocab file used in text classification [1]. + // + // [1]: https://github.com/tensorflow/tflite-support/blob/master/tensorflow_lite_support/metadata/python/tests/testdata/nl_classifier/vocab.txt // Added in: 1.0.1 VOCABULARY = 5, + + // TODO(b/222351186): introduce the ScaNN index file with links once the code + // is released. + // Contains on-devide ScaNN index file with LevelDB format. + // Added in: 1.4.0 + SCANN_INDEX_FILE = 6, } table AssociatedFile { @@ -152,6 +177,10 @@ table AssociatedFile { // Leverage this in order to specify e.g multiple label files translated in // different languages. locale:string; + + // Version of the file specified by model creators. + // Added in: 1.4.1 + version:string; } // The basic content type for all tensors. @@ -232,6 +261,16 @@ enum BoundingBoxType : byte { } +// The properties for audio tensors. +// Added in: 1.3.0 +table AudioProperties { + // The sample rate in Hz when the audio was captured. + sample_rate:uint; + + // The channel count of the audio. + channels:uint; +} + enum CoordinateType : byte { // The coordinates are float values from 0 to 1. RATIO = 0, @@ -267,6 +306,8 @@ union ContentProperties { FeatureProperties, ImageProperties, BoundingBoxProperties, + // Added in: 1.3.0 + AudioProperties, } table ValueRange { @@ -412,6 +453,9 @@ enum ScoreTransformationType : byte { // An AssociatedFile with type TANSOR_AXIS_SCORE_CALIBRATION specifying the // index-specific parameters must be associated with the corresponding // TensorMetadata for score calibration be applied. +// +// See the example score calibration file used in image classification [1]. +// [1]: https://github.com/tensorflow/tflite-support/blob/master/tensorflow_lite_support/metadata/python/tests/testdata/image_classifier/score_calibration.txt table ScoreCalibrationOptions { // The function to use for transforming the uncalibrated score before // applying score calibration. diff --git a/third_party/zlib/Android.bp b/third_party/zlib/Android.bp new file mode 100644 index 00000000..2da5f53c --- /dev/null +++ b/third_party/zlib/Android.bp @@ -0,0 +1,228 @@ +// Fork of external/zlib to external/tflite-support/third_party/zlib. +// TODO(b/233151429): Clean up external/tflite-support/third_party/zlib so that it contains the +// minimum set of files needed to support `BertNLClassifier`. + +package { + default_applicable_licenses: [ + "external_tflite-support_third_party_zlib_license", + ], +} + +// Added automatically by a large-scale-change that took the approach of +// 'apply every license found to every target'. While this makes sure we respect +// every license restriction, it may not be entirely correct. +// +// e.g. GPL in an MIT project might only apply to the contrib/ directory. +// +// Please consider splitting the single license below into multiple licenses, +// taking care not to lose any license_kind information, and overriding the +// default license using the 'licenses: [...]' property on targets as needed. +// +// For unused files, consider creating a 'fileGroup' with "//visibility:private" +// to attach the license to, and including a comment whether the files may be +// used in the current project. +// See: http://go/android-license-faq +license { + name: "external_tflite-support_third_party_zlib_license", + visibility: [":__subpackages__"], + license_kinds: [ + "SPDX-license-identifier-BSD", + "SPDX-license-identifier-Zlib", + ], + license_text: [ + "LICENSE", + ], +} + +srcs_opt = [ + "adler32_simd.c", + // See https://chromium-review.googlesource.com/749732. + // TODO: causes `atest org.apache.harmony.tests.java.util.zip.DeflaterTest` failures. + // "contrib/optimizations/inffast_chunk.c", + // "contrib/optimizations/inflate.c", + // This file doesn't build for non-neon, so it can't be in the main srcs. + "crc32_simd.c", +] + +cflags_arm = [ + // Testing with zlib_bench shows -O3 is a win for ARM but a bit of a wash + // for x86, so match the BUILD file in only enabling this for ARM. + "-O3", + // We need a non-NEON libz.a for the NDK, and cpu_features.c won't build + // without this. + "-DCPU_NO_SIMD", +] +cflags_arm_neon = [ + // Undo the -DCPU_NO_SIMD from the generic (non-NEON) ARM flags. + "-UCPU_NO_SIMD", + // We no longer support non-Neon platform builds, but the NDK just has one libz. + "-DADLER32_SIMD_NEON", + // TODO: causes `atest org.apache.harmony.tests.java.util.zip.DeflaterTest` failures. + // "-DINFLATE_CHUNK_SIMD_NEON", + // HWCAP_CRC32 is checked at runtime, so it's okay to turn crc32 + // acceleration on for both 32- and 64-bit. + "-DCRC32_ARMV8_CRC32", +] +cflags_arm64 = cflags_arm + cflags_arm_neon + +// The *host* x86 configuration (with *lower* CPU feature requirements). +cflags_x86 = [ + // See ARMV8_OS_LINUX above. + "-DX86_NOT_WINDOWS", + // TODO: see arm above. + // "-DINFLATE_CHUNK_SIMD_SSE2", + // Android's host CPU feature requirements are *lower* than the + // corresponding device CPU feature requirements, so it's easier to just + // say "no SIMD for you" rather than specificially disable SSSE3. + // We should have a conversation about that, but not until we at least have + // data on how many Studio users have CPUs that don't make the grade... + // https://issuetracker.google.com/171235570 + "-DCPU_NO_SIMD", +] +// The *device* x86 configuration (with *higher* CPU feature requirements). +cflags_android_x86 = [ + // Android's x86/x86-64 ABI includes SSE2 and SSSE3. + "-UCPU_NO_SIMD", + "-DADLER32_SIMD_SSSE3", + // PCLMUL isn't in the ABI, but it won't actually be used unless CPUID + // reports that the processor really does have the instruction. + "-mpclmul", + "-DCRC32_SIMD_SSE42_PCLMUL", +] +srcs_x86 = [ + "crc_folding.c", + "fill_window_sse.c", +] + srcs_opt + +// This optimization is applicable to arm64 and x86-64. +cflags_64 = ["-DINFLATE_CHUNK_READ_64LE"] + +tflite_support_libz_srcs = [ + "adler32.c", + "compress.c", + "cpu_features.c", + "crc32.c", + "deflate.c", + "gzclose.c", + "gzlib.c", + "gzread.c", + "gzwrite.c", + "infback.c", + "inffast.c", + "inflate.c", + "inftrees.c", + "trees.c", + "uncompr.c", + "zutil.c", + "contrib/minizip/ioapi.c", + "contrib/minizip/unzip.c", +] + +cc_defaults { + name: "tflite_support_libz_defaults", + + cflags: [ + // We do support hidden visibility, so turn that on. + "-DHAVE_HIDDEN", + // We do support const, so turn that on. + "-DZLIB_CONST", + // Enable -O3 as per chromium. + "-O3", + "-Wall", + "-Werror", + "-Wno-unused", + "-Wno-unused-parameter", + ], + stl: "none", + export_include_dirs: ["."], + srcs: tflite_support_libz_srcs, + + arch: { + arm: { + // TODO: This is to work around b/24465209. Remove after root cause + // is fixed. + pack_relocations: false, + ldflags: ["-Wl,--hash-style=both"], + + cflags: cflags_arm, + neon: { + cflags: cflags_arm_neon, + srcs: srcs_opt, + }, + }, + arm64: { + cflags: cflags_arm64 + cflags_64, + srcs: srcs_opt, + }, + x86: { + cflags: cflags_x86, + srcs: srcs_x86, + }, + x86_64: { + cflags: cflags_x86 + cflags_64, + srcs: srcs_x86, + }, + }, + target: { + android_arm: { + cflags: [ + // Since we're building for the platform, we claim to be Linux rather than + // Android so we use getauxval() directly instead of the NDK + // android_getCpuFeatures which isn't available to us anyway. + "-DARMV8_OS_LINUX", + ], + }, + android_x86: { + cflags: cflags_android_x86, + }, + android_x86_64: { + cflags: cflags_android_x86, + }, + darwin_arm64: { + cflags: [ + "-DARMV8_OS_MACOS", + ], + }, + linux_arm64: { + cflags: [ + // Since we're building for the platform, we claim to be Linux rather than + // Android so we use getauxval() directly instead of the NDK + // android_getCpuFeatures which isn't available to us anyway. + "-DARMV8_OS_LINUX", + ], + }, + }, +} + +cc_library_static { + name: "tflite_support_libz", + defaults: ["tflite_support_libz_defaults"], + + host_supported: true, + unique_host_soname: true, + static_ndk_lib: true, + sdk_version: "current", + min_sdk_version: "30", + + vendor_available: true, + product_available: true, + ramdisk_available: true, + vendor_ramdisk_available: true, + recovery_available: true, + native_bridge_supported: true, + + target: { + linux_bionic: { + enabled: true, + }, + windows: { + enabled: true, + }, + }, + + apex_available: [ + "//apex_available:platform", + "com.android.adservices", + "com.android.extservices", + ], +} diff --git a/third_party/zlib/LICENSE b/third_party/zlib/LICENSE new file mode 100644 index 00000000..9f056865 --- /dev/null +++ b/third_party/zlib/LICENSE @@ -0,0 +1,19 @@ +version 1.2.11, January 15th, 2017 + +Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. +2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. +3. This notice may not be removed or altered from any source distribution. diff --git a/third_party/zlib/adler32.c b/third_party/zlib/adler32.c new file mode 100644 index 00000000..8f8fbb90 --- /dev/null +++ b/third_party/zlib/adler32.c @@ -0,0 +1,214 @@ +/* adler32.c -- compute the Adler-32 checksum of a data stream + * Copyright (C) 1995-2011, 2016 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +/* @(#) $Id$ */ + +#include "zutil.h" + +local uLong adler32_combine_ OF((uLong adler1, uLong adler2, z_off64_t len2)); + +#define BASE 65521U /* largest prime smaller than 65536 */ +#define NMAX 5552 +/* NMAX is the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1 */ + +#define DO1(buf,i) {adler += (buf)[i]; sum2 += adler;} +#define DO2(buf,i) DO1(buf,i); DO1(buf,i+1); +#define DO4(buf,i) DO2(buf,i); DO2(buf,i+2); +#define DO8(buf,i) DO4(buf,i); DO4(buf,i+4); +#define DO16(buf) DO8(buf,0); DO8(buf,8); + +/* use NO_DIVIDE if your processor does not do division in hardware -- + try it both ways to see which is faster */ +#ifdef NO_DIVIDE +/* note that this assumes BASE is 65521, where 65536 % 65521 == 15 + (thank you to John Reiser for pointing this out) */ +# define CHOP(a) \ + do { \ + unsigned long tmp = a >> 16; \ + a &= 0xffffUL; \ + a += (tmp << 4) - tmp; \ + } while (0) +# define MOD28(a) \ + do { \ + CHOP(a); \ + if (a >= BASE) a -= BASE; \ + } while (0) +# define MOD(a) \ + do { \ + CHOP(a); \ + MOD28(a); \ + } while (0) +# define MOD63(a) \ + do { /* this assumes a is not negative */ \ + z_off64_t tmp = a >> 32; \ + a &= 0xffffffffL; \ + a += (tmp << 8) - (tmp << 5) + tmp; \ + tmp = a >> 16; \ + a &= 0xffffL; \ + a += (tmp << 4) - tmp; \ + tmp = a >> 16; \ + a &= 0xffffL; \ + a += (tmp << 4) - tmp; \ + if (a >= BASE) a -= BASE; \ + } while (0) +#else +# define MOD(a) a %= BASE +# define MOD28(a) a %= BASE +# define MOD63(a) a %= BASE +#endif + +#include "cpu_features.h" +#if defined(ADLER32_SIMD_SSSE3) || defined(ADLER32_SIMD_NEON) +#include "adler32_simd.h" +#endif + +/* ========================================================================= */ +uLong ZEXPORT adler32_z(adler, buf, len) + uLong adler; + const Bytef *buf; + z_size_t len; +{ + unsigned long sum2; + unsigned n; + +#if defined(ADLER32_SIMD_SSSE3) + if (buf != Z_NULL && len >= 64 && x86_cpu_enable_ssse3) + return adler32_simd_(adler, buf, len); +#elif defined(ADLER32_SIMD_NEON) + if (buf != Z_NULL && len >= 64) + return adler32_simd_(adler, buf, len); +#endif + + /* split Adler-32 into component sums */ + sum2 = (adler >> 16) & 0xffff; + adler &= 0xffff; + + /* in case user likes doing a byte at a time, keep it fast */ + if (len == 1) { + adler += buf[0]; + if (adler >= BASE) + adler -= BASE; + sum2 += adler; + if (sum2 >= BASE) + sum2 -= BASE; + return adler | (sum2 << 16); + } + +#if defined(ADLER32_SIMD_SSSE3) + /* + * Use SSSE3 to compute the adler32. Since this routine can be + * freely used, check CPU features here. zlib convention is to + * call adler32(0, NULL, 0), before making calls to adler32(). + * So this is a good early (and infrequent) place to cache CPU + * features for those later, more interesting adler32() calls. + */ + if (buf == Z_NULL) { + if (!len) /* Assume user is calling adler32(0, NULL, 0); */ + cpu_check_features(); + return 1L; + } +#else + /* initial Adler-32 value (deferred check for len == 1 speed) */ + if (buf == Z_NULL) + return 1L; +#endif + + /* in case short lengths are provided, keep it somewhat fast */ + if (len < 16) { + while (len--) { + adler += *buf++; + sum2 += adler; + } + if (adler >= BASE) + adler -= BASE; + MOD28(sum2); /* only added so many BASE's */ + return adler | (sum2 << 16); + } + + /* do length NMAX blocks -- requires just one modulo operation */ + while (len >= NMAX) { + len -= NMAX; + n = NMAX / 16; /* NMAX is divisible by 16 */ + do { + DO16(buf); /* 16 sums unrolled */ + buf += 16; + } while (--n); + MOD(adler); + MOD(sum2); + } + + /* do remaining bytes (less than NMAX, still just one modulo) */ + if (len) { /* avoid modulos if none remaining */ + while (len >= 16) { + len -= 16; + DO16(buf); + buf += 16; + } + while (len--) { + adler += *buf++; + sum2 += adler; + } + MOD(adler); + MOD(sum2); + } + + /* return recombined sums */ + return adler | (sum2 << 16); +} + +/* ========================================================================= */ +uLong ZEXPORT adler32(adler, buf, len) + uLong adler; + const Bytef *buf; + uInt len; +{ + return adler32_z(adler, buf, len); +} + +/* ========================================================================= */ +local uLong adler32_combine_(adler1, adler2, len2) + uLong adler1; + uLong adler2; + z_off64_t len2; +{ + unsigned long sum1; + unsigned long sum2; + unsigned rem; + + /* for negative len, return invalid adler32 as a clue for debugging */ + if (len2 < 0) + return 0xffffffffUL; + + /* the derivation of this formula is left as an exercise for the reader */ + MOD63(len2); /* assumes len2 >= 0 */ + rem = (unsigned)len2; + sum1 = adler1 & 0xffff; + sum2 = rem * sum1; + MOD(sum2); + sum1 += (adler2 & 0xffff) + BASE - 1; + sum2 += ((adler1 >> 16) & 0xffff) + ((adler2 >> 16) & 0xffff) + BASE - rem; + if (sum1 >= BASE) sum1 -= BASE; + if (sum1 >= BASE) sum1 -= BASE; + if (sum2 >= ((unsigned long)BASE << 1)) sum2 -= ((unsigned long)BASE << 1); + if (sum2 >= BASE) sum2 -= BASE; + return sum1 | (sum2 << 16); +} + +/* ========================================================================= */ +uLong ZEXPORT adler32_combine(adler1, adler2, len2) + uLong adler1; + uLong adler2; + z_off_t len2; +{ + return adler32_combine_(adler1, adler2, len2); +} + +uLong ZEXPORT adler32_combine64(adler1, adler2, len2) + uLong adler1; + uLong adler2; + z_off64_t len2; +{ + return adler32_combine_(adler1, adler2, len2); +} diff --git a/third_party/zlib/adler32_simd.c b/third_party/zlib/adler32_simd.c new file mode 100644 index 00000000..1354915c --- /dev/null +++ b/third_party/zlib/adler32_simd.c @@ -0,0 +1,366 @@ +/* adler32_simd.c + * + * Copyright 2017 The Chromium Authors. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the Chromium source repository LICENSE file. + * + * Per http://en.wikipedia.org/wiki/Adler-32 the adler32 A value (aka s1) is + * the sum of N input data bytes D1 ... DN, + * + * A = A0 + D1 + D2 + ... + DN + * + * where A0 is the initial value. + * + * SSE2 _mm_sad_epu8() can be used for byte sums (see http://bit.ly/2wpUOeD, + * for example) and accumulating the byte sums can use SSE shuffle-adds (see + * the "Integer" section of http://bit.ly/2erPT8t for details). Arm NEON has + * similar instructions. + * + * The adler32 B value (aka s2) sums the A values from each step: + * + * B0 + (A0 + D1) + (A0 + D1 + D2) + ... + (A0 + D1 + D2 + ... + DN) or + * + * B0 + N.A0 + N.D1 + (N-1).D2 + (N-2).D3 + ... + (N-(N-1)).DN + * + * B0 being the initial value. For 32 bytes (ideal for garden-variety SIMD): + * + * B = B0 + 32.A0 + [D1 D2 D3 ... D32] x [32 31 30 ... 1]. + * + * Adjacent blocks of 32 input bytes can be iterated with the expressions to + * compute the adler32 s1 s2 of M >> 32 input bytes [1]. + * + * As M grows, the s1 s2 sums grow. If left unchecked, they would eventually + * overflow the precision of their integer representation (bad). However, s1 + * and s2 also need to be computed modulo the adler BASE value (reduced). If + * at most NMAX bytes are processed before a reduce, s1 s2 _cannot_ overflow + * a uint32_t type (the NMAX constraint) [2]. + * + * [1] the iterative equations for s2 contain constant factors; these can be + * hoisted from the n-blocks do loop of the SIMD code. + * + * [2] zlib adler32_z() uses this fact to implement NMAX-block-based updates + * of the adler s1 s2 of uint32_t type (see adler32.c). + */ + +#include "adler32_simd.h" + +/* Definitions from adler32.c: largest prime smaller than 65536 */ +#define BASE 65521U +/* NMAX is the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1 */ +#define NMAX 5552 + +#if defined(ADLER32_SIMD_SSSE3) + +#include <tmmintrin.h> + +uint32_t ZLIB_INTERNAL adler32_simd_( /* SSSE3 */ + uint32_t adler, + const unsigned char *buf, + z_size_t len) +{ + /* + * Split Adler-32 into component sums. + */ + uint32_t s1 = adler & 0xffff; + uint32_t s2 = adler >> 16; + + /* + * Process the data in blocks. + */ + const unsigned BLOCK_SIZE = 1 << 5; + + z_size_t blocks = len / BLOCK_SIZE; + len -= blocks * BLOCK_SIZE; + + while (blocks) + { + unsigned n = NMAX / BLOCK_SIZE; /* The NMAX constraint. */ + if (n > blocks) + n = (unsigned) blocks; + blocks -= n; + + const __m128i tap1 = + _mm_setr_epi8(32,31,30,29,28,27,26,25,24,23,22,21,20,19,18,17); + const __m128i tap2 = + _mm_setr_epi8(16,15,14,13,12,11,10, 9, 8, 7, 6, 5, 4, 3, 2, 1); + const __m128i zero = + _mm_setr_epi8( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + const __m128i ones = + _mm_set_epi16( 1, 1, 1, 1, 1, 1, 1, 1); + + /* + * Process n blocks of data. At most NMAX data bytes can be + * processed before s2 must be reduced modulo BASE. + */ + __m128i v_ps = _mm_set_epi32(0, 0, 0, s1 * n); + __m128i v_s2 = _mm_set_epi32(0, 0, 0, s2); + __m128i v_s1 = _mm_set_epi32(0, 0, 0, 0); + + do { + /* + * Load 32 input bytes. + */ + const __m128i bytes1 = _mm_loadu_si128((__m128i*)(buf)); + const __m128i bytes2 = _mm_loadu_si128((__m128i*)(buf + 16)); + + /* + * Add previous block byte sum to v_ps. + */ + v_ps = _mm_add_epi32(v_ps, v_s1); + + /* + * Horizontally add the bytes for s1, multiply-adds the + * bytes by [ 32, 31, 30, ... ] for s2. + */ + v_s1 = _mm_add_epi32(v_s1, _mm_sad_epu8(bytes1, zero)); + const __m128i mad1 = _mm_maddubs_epi16(bytes1, tap1); + v_s2 = _mm_add_epi32(v_s2, _mm_madd_epi16(mad1, ones)); + + v_s1 = _mm_add_epi32(v_s1, _mm_sad_epu8(bytes2, zero)); + const __m128i mad2 = _mm_maddubs_epi16(bytes2, tap2); + v_s2 = _mm_add_epi32(v_s2, _mm_madd_epi16(mad2, ones)); + + buf += BLOCK_SIZE; + + } while (--n); + + v_s2 = _mm_add_epi32(v_s2, _mm_slli_epi32(v_ps, 5)); + + /* + * Sum epi32 ints v_s1(s2) and accumulate in s1(s2). + */ + +#define S23O1 _MM_SHUFFLE(2,3,0,1) /* A B C D -> B A D C */ +#define S1O32 _MM_SHUFFLE(1,0,3,2) /* A B C D -> C D A B */ + + v_s1 = _mm_add_epi32(v_s1, _mm_shuffle_epi32(v_s1, S23O1)); + v_s1 = _mm_add_epi32(v_s1, _mm_shuffle_epi32(v_s1, S1O32)); + + s1 += _mm_cvtsi128_si32(v_s1); + + v_s2 = _mm_add_epi32(v_s2, _mm_shuffle_epi32(v_s2, S23O1)); + v_s2 = _mm_add_epi32(v_s2, _mm_shuffle_epi32(v_s2, S1O32)); + + s2 = _mm_cvtsi128_si32(v_s2); + +#undef S23O1 +#undef S1O32 + + /* + * Reduce. + */ + s1 %= BASE; + s2 %= BASE; + } + + /* + * Handle leftover data. + */ + if (len) { + if (len >= 16) { + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + + len -= 16; + } + + while (len--) { + s2 += (s1 += *buf++); + } + + if (s1 >= BASE) + s1 -= BASE; + s2 %= BASE; + } + + /* + * Return the recombined sums. + */ + return s1 | (s2 << 16); +} + +#elif defined(ADLER32_SIMD_NEON) + +#include <arm_neon.h> + +uint32_t ZLIB_INTERNAL adler32_simd_( /* NEON */ + uint32_t adler, + const unsigned char *buf, + z_size_t len) +{ + /* + * Split Adler-32 into component sums. + */ + uint32_t s1 = adler & 0xffff; + uint32_t s2 = adler >> 16; + + /* + * Serially compute s1 & s2, until the data is 16-byte aligned. + */ + if ((uintptr_t)buf & 15) { + while ((uintptr_t)buf & 15) { + s2 += (s1 += *buf++); + --len; + } + + if (s1 >= BASE) + s1 -= BASE; + s2 %= BASE; + } + + /* + * Process the data in blocks. + */ + const unsigned BLOCK_SIZE = 1 << 5; + + z_size_t blocks = len / BLOCK_SIZE; + len -= blocks * BLOCK_SIZE; + + while (blocks) + { + unsigned n = NMAX / BLOCK_SIZE; /* The NMAX constraint. */ + if (n > blocks) + n = (unsigned) blocks; + blocks -= n; + + /* + * Process n blocks of data. At most NMAX data bytes can be + * processed before s2 must be reduced modulo BASE. + */ + uint32x4_t v_s2 = (uint32x4_t) { 0, 0, 0, s1 * n }; + uint32x4_t v_s1 = (uint32x4_t) { 0, 0, 0, 0 }; + + uint16x8_t v_column_sum_1 = vdupq_n_u16(0); + uint16x8_t v_column_sum_2 = vdupq_n_u16(0); + uint16x8_t v_column_sum_3 = vdupq_n_u16(0); + uint16x8_t v_column_sum_4 = vdupq_n_u16(0); + + do { + /* + * Load 32 input bytes. + */ + const uint8x16_t bytes1 = vld1q_u8((uint8_t*)(buf)); + const uint8x16_t bytes2 = vld1q_u8((uint8_t*)(buf + 16)); + + /* + * Add previous block byte sum to v_s2. + */ + v_s2 = vaddq_u32(v_s2, v_s1); + + /* + * Horizontally add the bytes for s1. + */ + v_s1 = vpadalq_u16(v_s1, vpadalq_u8(vpaddlq_u8(bytes1), bytes2)); + + /* + * Vertically add the bytes for s2. + */ + v_column_sum_1 = vaddw_u8(v_column_sum_1, vget_low_u8 (bytes1)); + v_column_sum_2 = vaddw_u8(v_column_sum_2, vget_high_u8(bytes1)); + v_column_sum_3 = vaddw_u8(v_column_sum_3, vget_low_u8 (bytes2)); + v_column_sum_4 = vaddw_u8(v_column_sum_4, vget_high_u8(bytes2)); + + buf += BLOCK_SIZE; + + } while (--n); + + v_s2 = vshlq_n_u32(v_s2, 5); + + /* + * Multiply-add bytes by [ 32, 31, 30, ... ] for s2. + */ + v_s2 = vmlal_u16(v_s2, vget_low_u16 (v_column_sum_1), + (uint16x4_t) { 32, 31, 30, 29 }); + v_s2 = vmlal_u16(v_s2, vget_high_u16(v_column_sum_1), + (uint16x4_t) { 28, 27, 26, 25 }); + v_s2 = vmlal_u16(v_s2, vget_low_u16 (v_column_sum_2), + (uint16x4_t) { 24, 23, 22, 21 }); + v_s2 = vmlal_u16(v_s2, vget_high_u16(v_column_sum_2), + (uint16x4_t) { 20, 19, 18, 17 }); + v_s2 = vmlal_u16(v_s2, vget_low_u16 (v_column_sum_3), + (uint16x4_t) { 16, 15, 14, 13 }); + v_s2 = vmlal_u16(v_s2, vget_high_u16(v_column_sum_3), + (uint16x4_t) { 12, 11, 10, 9 }); + v_s2 = vmlal_u16(v_s2, vget_low_u16 (v_column_sum_4), + (uint16x4_t) { 8, 7, 6, 5 }); + v_s2 = vmlal_u16(v_s2, vget_high_u16(v_column_sum_4), + (uint16x4_t) { 4, 3, 2, 1 }); + + /* + * Sum epi32 ints v_s1(s2) and accumulate in s1(s2). + */ + uint32x2_t sum1 = vpadd_u32(vget_low_u32(v_s1), vget_high_u32(v_s1)); + uint32x2_t sum2 = vpadd_u32(vget_low_u32(v_s2), vget_high_u32(v_s2)); + uint32x2_t s1s2 = vpadd_u32(sum1, sum2); + + s1 += vget_lane_u32(s1s2, 0); + s2 += vget_lane_u32(s1s2, 1); + + /* + * Reduce. + */ + s1 %= BASE; + s2 %= BASE; + } + + /* + * Handle leftover data. + */ + if (len) { + if (len >= 16) { + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + s2 += (s1 += *buf++); + + len -= 16; + } + + while (len--) { + s2 += (s1 += *buf++); + } + + if (s1 >= BASE) + s1 -= BASE; + s2 %= BASE; + } + + /* + * Return the recombined sums. + */ + return s1 | (s2 << 16); +} + +#endif /* ADLER32_SIMD_SSSE3 */ diff --git a/third_party/zlib/adler32_simd.h b/third_party/zlib/adler32_simd.h new file mode 100644 index 00000000..52bb14d1 --- /dev/null +++ b/third_party/zlib/adler32_simd.h @@ -0,0 +1,16 @@ +/* adler32_simd.h + * + * Copyright 2017 The Chromium Authors. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the Chromium source repository LICENSE file. + */ + +#include <stdint.h> + +#include "zconf.h" +#include "zutil.h" + +uint32_t ZLIB_INTERNAL adler32_simd_( + uint32_t adler, + const unsigned char *buf, + z_size_t len); diff --git a/third_party/zlib/chromeconf.h b/third_party/zlib/chromeconf.h new file mode 100644 index 00000000..5ecf29ed --- /dev/null +++ b/third_party/zlib/chromeconf.h @@ -0,0 +1,199 @@ +/* Copyright 2017 The Chromium Authors. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. */ + +#ifndef THIRD_PARTY_ZLIB_CHROMECONF_H_ +#define THIRD_PARTY_ZLIB_CHROMECONF_H_ + +#if defined(COMPONENT_BUILD) +#if defined(WIN32) +#if defined(ZLIB_IMPLEMENTATION) +#define ZEXTERN __declspec(dllexport) +#else +#define ZEXTERN __declspec(dllimport) +#endif +#elif defined(ZLIB_IMPLEMENTATION) +#define ZEXTERN __attribute__((visibility("default"))) +#endif +#endif + +/* Rename all zlib names with a Cr_z_ prefix. This is based on the Z_PREFIX + * option from zconf.h, but with a custom prefix. Where zconf.h would rename + * both a macro and its underscore-suffixed internal implementation (such as + * deflateInit2 and deflateInit2_), only the implementation is renamed here. + * The Byte type is also omitted. + * + * To generate this list, run + * sed -rn -e 's/^# *define +([^ ]+) +(z_[^ ]+)$/#define \1 Cr_\2/p' zconf.h + * (use -E instead of -r on macOS). + * + * gzread is also addressed by modifications in gzread.c and zlib.h. */ + +#define Z_CR_PREFIX_SET + +#define _dist_code Cr_z__dist_code +#define _length_code Cr_z__length_code +#define _tr_align Cr_z__tr_align +#define _tr_flush_bits Cr_z__tr_flush_bits +#define _tr_flush_block Cr_z__tr_flush_block +#define _tr_init Cr_z__tr_init +#define _tr_stored_block Cr_z__tr_stored_block +#define _tr_tally Cr_z__tr_tally +#define adler32 Cr_z_adler32 +#define adler32_combine Cr_z_adler32_combine +#define adler32_combine64 Cr_z_adler32_combine64 +#define adler32_z Cr_z_adler32_z +#define compress Cr_z_compress +#define compress2 Cr_z_compress2 +#define compressBound Cr_z_compressBound +#define crc32 Cr_z_crc32 +#define crc32_combine Cr_z_crc32_combine +#define crc32_combine64 Cr_z_crc32_combine64 +#define crc32_z Cr_z_crc32_z +#define deflate Cr_z_deflate +#define deflateBound Cr_z_deflateBound +#define deflateCopy Cr_z_deflateCopy +#define deflateEnd Cr_z_deflateEnd +#define deflateGetDictionary Cr_z_deflateGetDictionary +/* #undef deflateInit */ +/* #undef deflateInit2 */ +#define deflateInit2_ Cr_z_deflateInit2_ +#define deflateInit_ Cr_z_deflateInit_ +#define deflateParams Cr_z_deflateParams +#define deflatePending Cr_z_deflatePending +#define deflatePrime Cr_z_deflatePrime +#define deflateReset Cr_z_deflateReset +#define deflateResetKeep Cr_z_deflateResetKeep +#define deflateSetDictionary Cr_z_deflateSetDictionary +#define deflateSetHeader Cr_z_deflateSetHeader +#define deflateTune Cr_z_deflateTune +#define deflate_copyright Cr_z_deflate_copyright +#define get_crc_table Cr_z_get_crc_table +#define gz_error Cr_z_gz_error +#define gz_intmax Cr_z_gz_intmax +#define gz_strwinerror Cr_z_gz_strwinerror +#define gzbuffer Cr_z_gzbuffer +#define gzclearerr Cr_z_gzclearerr +#define gzclose Cr_z_gzclose +#define gzclose_r Cr_z_gzclose_r +#define gzclose_w Cr_z_gzclose_w +#define gzdirect Cr_z_gzdirect +#define gzdopen Cr_z_gzdopen +#define gzeof Cr_z_gzeof +#define gzerror Cr_z_gzerror +#define gzflush Cr_z_gzflush +#define gzfread Cr_z_gzfread +#define gzfwrite Cr_z_gzfwrite +#define gzgetc Cr_z_gzgetc +#define gzgetc_ Cr_z_gzgetc_ +#define gzgets Cr_z_gzgets +#define gzoffset Cr_z_gzoffset +#define gzoffset64 Cr_z_gzoffset64 +#define gzopen Cr_z_gzopen +#define gzopen64 Cr_z_gzopen64 +#define gzopen_w Cr_z_gzopen_w +#define gzprintf Cr_z_gzprintf +#define gzputc Cr_z_gzputc +#define gzputs Cr_z_gzputs +#define gzread Cr_z_gzread +#define gzrewind Cr_z_gzrewind +#define gzseek Cr_z_gzseek +#define gzseek64 Cr_z_gzseek64 +#define gzsetparams Cr_z_gzsetparams +#define gztell Cr_z_gztell +#define gztell64 Cr_z_gztell64 +#define gzungetc Cr_z_gzungetc +#define gzvprintf Cr_z_gzvprintf +#define gzwrite Cr_z_gzwrite +#define inflate Cr_z_inflate +#define inflateBack Cr_z_inflateBack +#define inflateBackEnd Cr_z_inflateBackEnd +/* #undef inflateBackInit */ +#define inflateBackInit_ Cr_z_inflateBackInit_ +#define inflateCodesUsed Cr_z_inflateCodesUsed +#define inflateCopy Cr_z_inflateCopy +#define inflateEnd Cr_z_inflateEnd +#define inflateGetDictionary Cr_z_inflateGetDictionary +#define inflateGetHeader Cr_z_inflateGetHeader +/* #undef inflateInit */ +/* #undef inflateInit2 */ +#define inflateInit2_ Cr_z_inflateInit2_ +#define inflateInit_ Cr_z_inflateInit_ +#define inflateMark Cr_z_inflateMark +#define inflatePrime Cr_z_inflatePrime +#define inflateReset Cr_z_inflateReset +#define inflateReset2 Cr_z_inflateReset2 +#define inflateResetKeep Cr_z_inflateResetKeep +#define inflateSetDictionary Cr_z_inflateSetDictionary +#define inflateSync Cr_z_inflateSync +#define inflateSyncPoint Cr_z_inflateSyncPoint +#define inflateUndermine Cr_z_inflateUndermine +#define inflateValidate Cr_z_inflateValidate +#define inflate_copyright Cr_z_inflate_copyright +#define inflate_fast Cr_z_inflate_fast +#define inflate_table Cr_z_inflate_table +#define uncompress Cr_z_uncompress +#define uncompress2 Cr_z_uncompress2 +#define zError Cr_z_zError +#define zcalloc Cr_z_zcalloc +#define zcfree Cr_z_zcfree +#define zlibCompileFlags Cr_z_zlibCompileFlags +#define zlibVersion Cr_z_zlibVersion +/* #undef Byte */ +#define Bytef Cr_z_Bytef +#define alloc_func Cr_z_alloc_func +#define charf Cr_z_charf +#define free_func Cr_z_free_func +#define gzFile Cr_z_gzFile +#define gz_header Cr_z_gz_header +#define gz_headerp Cr_z_gz_headerp +#define in_func Cr_z_in_func +#define intf Cr_z_intf +#define out_func Cr_z_out_func +#define uInt Cr_z_uInt +#define uIntf Cr_z_uIntf +#define uLong Cr_z_uLong +#define uLongf Cr_z_uLongf +#define voidp Cr_z_voidp +#define voidpc Cr_z_voidpc +#define voidpf Cr_z_voidpf +#define gz_header_s Cr_z_gz_header_s +/* #undef internal_state */ +/* #undef z_off64_t */ + +/* An exported symbol that isn't handled by Z_PREFIX in zconf.h */ +#define z_errmsg Cr_z_z_errmsg + +/* Symbols added in simd.patch */ +#define copy_with_crc Cr_z_copy_with_crc +#define crc_finalize Cr_z_crc_finalize +#define crc_fold_512to32 Cr_z_crc_fold_512to32 +#define crc_fold_copy Cr_z_crc_fold_copy +#define crc_fold_init Cr_z_crc_fold_init +#define crc_reset Cr_z_crc_reset +#define fill_window_sse Cr_z_fill_window_sse +#define deflate_read_buf Cr_z_deflate_read_buf +#define x86_check_features Cr_z_x86_check_features +#define x86_cpu_enable_simd Cr_z_x86_cpu_enable_simd + +/* Symbols added by adler_simd.c */ +#define adler32_simd_ Cr_z_adler32_simd_ +#define x86_cpu_enable_ssse3 Cr_z_x86_cpu_enable_ssse3 + +/* Symbols added by contrib/optimizations/inffast_chunk */ +#define inflate_fast_chunk_ Cr_z_inflate_fast_chunk_ + +/* Symbols added by crc32_simd.c */ +#define crc32_sse42_simd_ Cr_z_crc32_sse42_simd_ + +/* Symbols added by armv8_crc32 */ +#define arm_cpu_enable_crc32 Cr_z_arm_cpu_enable_crc32 +#define arm_cpu_enable_pmull Cr_z_arm_cpu_enable_pmull +#define arm_check_features Cr_z_arm_check_features +#define armv8_crc32_little Cr_z_armv8_crc32_little + +/* Symbols added by cpu_features.c */ +#define cpu_check_features Cr_z_cpu_check_features +#define x86_cpu_enable_sse2 Cr_z_x86_cpu_enable_sse2 + +#endif /* THIRD_PARTY_ZLIB_CHROMECONF_H_ */ diff --git a/third_party/zlib/compress.c b/third_party/zlib/compress.c new file mode 100644 index 00000000..6e851733 --- /dev/null +++ b/third_party/zlib/compress.c @@ -0,0 +1,96 @@ +/* compress.c -- compress a memory buffer + * Copyright (C) 1995-2005, 2014, 2016 Jean-loup Gailly, Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +/* @(#) $Id$ */ + +#define ZLIB_INTERNAL +#include "zlib.h" + +/* =========================================================================== + Compresses the source buffer into the destination buffer. The level + parameter has the same meaning as in deflateInit. sourceLen is the byte + length of the source buffer. Upon entry, destLen is the total size of the + destination buffer, which must be at least 0.1% larger than sourceLen plus + 12 bytes. Upon exit, destLen is the actual size of the compressed buffer. + + compress2 returns Z_OK if success, Z_MEM_ERROR if there was not enough + memory, Z_BUF_ERROR if there was not enough room in the output buffer, + Z_STREAM_ERROR if the level parameter is invalid. +*/ +int ZEXPORT compress2 (dest, destLen, source, sourceLen, level) + Bytef *dest; + uLongf *destLen; + const Bytef *source; + uLong sourceLen; + int level; +{ + z_stream stream; + int err; + const uInt max = (uInt)-1; + uLong left; + + left = *destLen; + *destLen = 0; + + stream.zalloc = (alloc_func)0; + stream.zfree = (free_func)0; + stream.opaque = (voidpf)0; + + err = deflateInit(&stream, level); + if (err != Z_OK) return err; + + stream.next_out = dest; + stream.avail_out = 0; + stream.next_in = (z_const Bytef *)source; + stream.avail_in = 0; + + do { + if (stream.avail_out == 0) { + stream.avail_out = left > (uLong)max ? max : (uInt)left; + left -= stream.avail_out; + } + if (stream.avail_in == 0) { + stream.avail_in = sourceLen > (uLong)max ? max : (uInt)sourceLen; + sourceLen -= stream.avail_in; + } + err = deflate(&stream, sourceLen ? Z_NO_FLUSH : Z_FINISH); + } while (err == Z_OK); + + *destLen = stream.total_out; + deflateEnd(&stream); + return err == Z_STREAM_END ? Z_OK : err; +} + +/* =========================================================================== + */ +int ZEXPORT compress (dest, destLen, source, sourceLen) + Bytef *dest; + uLongf *destLen; + const Bytef *source; + uLong sourceLen; +{ + return compress2(dest, destLen, source, sourceLen, Z_DEFAULT_COMPRESSION); +} + +/* =========================================================================== + If the default memLevel or windowBits for deflateInit() is changed, then + this function needs to be updated. + */ +uLong ZEXPORT compressBound (sourceLen) + uLong sourceLen; +{ + sourceLen = sourceLen + (sourceLen >> 12) + (sourceLen >> 14) + + (sourceLen >> 25) + 13; + /* FIXME(cavalcantii): usage of CRC32 Castagnoli as a hash function + * for the hash table of symbols used for compression has a side effect + * where for compression level [4, 5] it will increase the output buffer size + * by 0.1% (i.e. less than 1%) for a high entropy input (i.e. random data). + * To avoid a scenario where client code would fail, for safety we increase + * the expected output size by 0.8% (i.e. 8x more than the worst scenario). + * See: http://crbug.com/990489 + */ + sourceLen += sourceLen >> 7; // Equivalent to 1.0078125 + return sourceLen; +} diff --git a/third_party/zlib/contrib/minizip/crypt.h b/third_party/zlib/contrib/minizip/crypt.h new file mode 100644 index 00000000..1e9e8200 --- /dev/null +++ b/third_party/zlib/contrib/minizip/crypt.h @@ -0,0 +1,131 @@ +/* crypt.h -- base code for crypt/uncrypt ZIPfile + + + Version 1.01e, February 12th, 2005 + + Copyright (C) 1998-2005 Gilles Vollant + + This code is a modified version of crypting code in Infozip distribution + + The encryption/decryption parts of this source code (as opposed to the + non-echoing password parts) were originally written in Europe. The + whole source package can be freely distributed, including from the USA. + (Prior to January 2000, re-export from the US was a violation of US law.) + + This encryption code is a direct transcription of the algorithm from + Roger Schlafly, described by Phil Katz in the file appnote.txt. This + file (appnote.txt) is distributed with the PKZIP program (even in the + version without encryption capabilities). + + If you don't need crypting in your application, just define symbols + NOCRYPT and NOUNCRYPT. + + This code support the "Traditional PKWARE Encryption". + + The new AES encryption added on Zip format by Winzip (see the page + http://www.winzip.com/aes_info.htm ) and PKWare PKZip 5.x Strong + Encryption is not supported. +*/ + +#define CRC32(c, b) ((*(pcrc_32_tab+(((int)(c) ^ (b)) & 0xff))) ^ ((c) >> 8)) + +/*********************************************************************** + * Return the next byte in the pseudo-random sequence + */ +static int decrypt_byte(unsigned long* pkeys, const z_crc_t* pcrc_32_tab) +{ + unsigned temp; /* POTENTIAL BUG: temp*(temp^1) may overflow in an + * unpredictable manner on 16-bit systems; not a problem + * with any known compiler so far, though */ + + temp = ((unsigned)(*(pkeys+2)) & 0xffff) | 2; + return (int)(((temp * (temp ^ 1)) >> 8) & 0xff); +} + +/*********************************************************************** + * Update the encryption keys with the next byte of plain text + */ +static int update_keys(unsigned long* pkeys,const z_crc_t* pcrc_32_tab,int c) +{ + (*(pkeys+0)) = CRC32((*(pkeys+0)), c); + (*(pkeys+1)) += (*(pkeys+0)) & 0xff; + (*(pkeys+1)) = (*(pkeys+1)) * 134775813L + 1; + { + register int keyshift = (int)((*(pkeys+1)) >> 24); + (*(pkeys+2)) = CRC32((*(pkeys+2)), keyshift); + } + return c; +} + + +/*********************************************************************** + * Initialize the encryption keys and the random header according to + * the given password. + */ +static void init_keys(const char* passwd,unsigned long* pkeys,const z_crc_t* pcrc_32_tab) +{ + *(pkeys+0) = 305419896L; + *(pkeys+1) = 591751049L; + *(pkeys+2) = 878082192L; + while (*passwd != '\0') { + update_keys(pkeys,pcrc_32_tab,(int)*passwd); + passwd++; + } +} + +#define zdecode(pkeys,pcrc_32_tab,c) \ + (update_keys(pkeys,pcrc_32_tab,c ^= decrypt_byte(pkeys,pcrc_32_tab))) + +#define zencode(pkeys,pcrc_32_tab,c,t) \ + (t=decrypt_byte(pkeys,pcrc_32_tab), update_keys(pkeys,pcrc_32_tab,c), t^(c)) + +#ifdef INCLUDECRYPTINGCODE_IFCRYPTALLOWED + +#define RAND_HEAD_LEN 12 + /* "last resort" source for second part of crypt seed pattern */ +# ifndef ZCR_SEED2 +# define ZCR_SEED2 3141592654UL /* use PI as default pattern */ +# endif + +static int crypthead(const char* passwd, /* password string */ + unsigned char* buf, /* where to write header */ + int bufSize, + unsigned long* pkeys, + const z_crc_t* pcrc_32_tab, + unsigned long crcForCrypting) +{ + int n; /* index in random header */ + int t; /* temporary */ + int c; /* random byte */ + unsigned char header[RAND_HEAD_LEN-2]; /* random header */ + static unsigned calls = 0; /* ensure different random header each time */ + + if (bufSize<RAND_HEAD_LEN) + return 0; + + /* First generate RAND_HEAD_LEN-2 random bytes. We encrypt the + * output of rand() to get less predictability, since rand() is + * often poorly implemented. + */ + if (++calls == 1) + { + srand((unsigned)(time(NULL) ^ ZCR_SEED2)); + } + init_keys(passwd, pkeys, pcrc_32_tab); + for (n = 0; n < RAND_HEAD_LEN-2; n++) + { + c = (rand() >> 7) & 0xff; + header[n] = (unsigned char)zencode(pkeys, pcrc_32_tab, c, t); + } + /* Encrypt random header (last two bytes is high word of crc) */ + init_keys(passwd, pkeys, pcrc_32_tab); + for (n = 0; n < RAND_HEAD_LEN-2; n++) + { + buf[n] = (unsigned char)zencode(pkeys, pcrc_32_tab, header[n], t); + } + buf[n++] = (unsigned char)zencode(pkeys, pcrc_32_tab, (int)(crcForCrypting >> 16) & 0xff, t); + buf[n++] = (unsigned char)zencode(pkeys, pcrc_32_tab, (int)(crcForCrypting >> 24) & 0xff, t); + return n; +} + +#endif diff --git a/third_party/zlib/contrib/minizip/ioapi.c b/third_party/zlib/contrib/minizip/ioapi.c new file mode 100644 index 00000000..543910b5 --- /dev/null +++ b/third_party/zlib/contrib/minizip/ioapi.c @@ -0,0 +1,247 @@ +/* ioapi.h -- IO base function header for compress/uncompress .zip + part of the MiniZip project - ( http://www.winimage.com/zLibDll/minizip.html ) + + Copyright (C) 1998-2010 Gilles Vollant (minizip) ( http://www.winimage.com/zLibDll/minizip.html ) + + Modifications for Zip64 support + Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) + + For more info read MiniZip_info.txt + +*/ + +#if defined(_WIN32) && (!(defined(_CRT_SECURE_NO_WARNINGS))) + #define _CRT_SECURE_NO_WARNINGS +#endif + +#if defined(__APPLE__) || defined(__Fuchsia__) || defined(IOAPI_NO_64) +// In darwin and perhaps other BSD variants off_t is a 64 bit value, hence no need for specific 64 bit functions +#define FOPEN_FUNC(filename, mode) fopen(filename, mode) +#define FTELLO_FUNC(stream) ftello(stream) +#define FSEEKO_FUNC(stream, offset, origin) fseeko(stream, offset, origin) +#else +#define FOPEN_FUNC(filename, mode) fopen64(filename, mode) +#define FTELLO_FUNC(stream) ftello64(stream) +#define FSEEKO_FUNC(stream, offset, origin) fseeko64(stream, offset, origin) +#endif + + +#include "ioapi.h" + +voidpf call_zopen64 (const zlib_filefunc64_32_def* pfilefunc,const void*filename,int mode) +{ + if (pfilefunc->zfile_func64.zopen64_file != NULL) + return (*(pfilefunc->zfile_func64.zopen64_file)) (pfilefunc->zfile_func64.opaque,filename,mode); + else + { + return (*(pfilefunc->zopen32_file))(pfilefunc->zfile_func64.opaque,(const char*)filename,mode); + } +} + +long call_zseek64 (const zlib_filefunc64_32_def* pfilefunc,voidpf filestream, ZPOS64_T offset, int origin) +{ + if (pfilefunc->zfile_func64.zseek64_file != NULL) + return (*(pfilefunc->zfile_func64.zseek64_file)) (pfilefunc->zfile_func64.opaque,filestream,offset,origin); + else + { + uLong offsetTruncated = (uLong)offset; + if (offsetTruncated != offset) + return -1; + else + return (*(pfilefunc->zseek32_file))(pfilefunc->zfile_func64.opaque,filestream,offsetTruncated,origin); + } +} + +ZPOS64_T call_ztell64 (const zlib_filefunc64_32_def* pfilefunc,voidpf filestream) +{ + if (pfilefunc->zfile_func64.zseek64_file != NULL) + return (*(pfilefunc->zfile_func64.ztell64_file)) (pfilefunc->zfile_func64.opaque,filestream); + else + { + uLong tell_uLong = (*(pfilefunc->ztell32_file))(pfilefunc->zfile_func64.opaque,filestream); + if ((tell_uLong) == MAXU32) + return (ZPOS64_T)-1; + else + return tell_uLong; + } +} + +void fill_zlib_filefunc64_32_def_from_filefunc32(zlib_filefunc64_32_def* p_filefunc64_32,const zlib_filefunc_def* p_filefunc32) +{ + p_filefunc64_32->zfile_func64.zopen64_file = NULL; + p_filefunc64_32->zopen32_file = p_filefunc32->zopen_file; + p_filefunc64_32->zfile_func64.zerror_file = p_filefunc32->zerror_file; + p_filefunc64_32->zfile_func64.zread_file = p_filefunc32->zread_file; + p_filefunc64_32->zfile_func64.zwrite_file = p_filefunc32->zwrite_file; + p_filefunc64_32->zfile_func64.ztell64_file = NULL; + p_filefunc64_32->zfile_func64.zseek64_file = NULL; + p_filefunc64_32->zfile_func64.zclose_file = p_filefunc32->zclose_file; + p_filefunc64_32->zfile_func64.zerror_file = p_filefunc32->zerror_file; + p_filefunc64_32->zfile_func64.opaque = p_filefunc32->opaque; + p_filefunc64_32->zseek32_file = p_filefunc32->zseek_file; + p_filefunc64_32->ztell32_file = p_filefunc32->ztell_file; +} + + + +static voidpf ZCALLBACK fopen_file_func OF((voidpf opaque, const char* filename, int mode)); +static uLong ZCALLBACK fread_file_func OF((voidpf opaque, voidpf stream, void* buf, uLong size)); +static uLong ZCALLBACK fwrite_file_func OF((voidpf opaque, voidpf stream, const void* buf,uLong size)); +static ZPOS64_T ZCALLBACK ftell64_file_func OF((voidpf opaque, voidpf stream)); +static long ZCALLBACK fseek64_file_func OF((voidpf opaque, voidpf stream, ZPOS64_T offset, int origin)); +static int ZCALLBACK fclose_file_func OF((voidpf opaque, voidpf stream)); +static int ZCALLBACK ferror_file_func OF((voidpf opaque, voidpf stream)); + +static voidpf ZCALLBACK fopen_file_func (voidpf opaque, const char* filename, int mode) +{ + FILE* file = NULL; + const char* mode_fopen = NULL; + if ((mode & ZLIB_FILEFUNC_MODE_READWRITEFILTER)==ZLIB_FILEFUNC_MODE_READ) + mode_fopen = "rb"; + else + if (mode & ZLIB_FILEFUNC_MODE_EXISTING) + mode_fopen = "r+b"; + else + if (mode & ZLIB_FILEFUNC_MODE_CREATE) + mode_fopen = "wb"; + + if ((filename!=NULL) && (mode_fopen != NULL)) + file = fopen(filename, mode_fopen); + return file; +} + +static voidpf ZCALLBACK fopen64_file_func (voidpf opaque, const void* filename, int mode) +{ + FILE* file = NULL; + const char* mode_fopen = NULL; + if ((mode & ZLIB_FILEFUNC_MODE_READWRITEFILTER)==ZLIB_FILEFUNC_MODE_READ) + mode_fopen = "rb"; + else + if (mode & ZLIB_FILEFUNC_MODE_EXISTING) + mode_fopen = "r+b"; + else + if (mode & ZLIB_FILEFUNC_MODE_CREATE) + mode_fopen = "wb"; + + if ((filename!=NULL) && (mode_fopen != NULL)) + file = FOPEN_FUNC((const char*)filename, mode_fopen); + return file; +} + + +static uLong ZCALLBACK fread_file_func (voidpf opaque, voidpf stream, void* buf, uLong size) +{ + uLong ret; + ret = (uLong)fread(buf, 1, (size_t)size, (FILE *)stream); + return ret; +} + +static uLong ZCALLBACK fwrite_file_func (voidpf opaque, voidpf stream, const void* buf, uLong size) +{ + uLong ret; + ret = (uLong)fwrite(buf, 1, (size_t)size, (FILE *)stream); + return ret; +} + +static long ZCALLBACK ftell_file_func (voidpf opaque, voidpf stream) +{ + long ret; + ret = ftell((FILE *)stream); + return ret; +} + + +static ZPOS64_T ZCALLBACK ftell64_file_func (voidpf opaque, voidpf stream) +{ + ZPOS64_T ret; + ret = FTELLO_FUNC((FILE *)stream); + return ret; +} + +static long ZCALLBACK fseek_file_func (voidpf opaque, voidpf stream, uLong offset, int origin) +{ + int fseek_origin=0; + long ret; + switch (origin) + { + case ZLIB_FILEFUNC_SEEK_CUR : + fseek_origin = SEEK_CUR; + break; + case ZLIB_FILEFUNC_SEEK_END : + fseek_origin = SEEK_END; + break; + case ZLIB_FILEFUNC_SEEK_SET : + fseek_origin = SEEK_SET; + break; + default: return -1; + } + ret = 0; + if (fseek((FILE *)stream, offset, fseek_origin) != 0) + ret = -1; + return ret; +} + +static long ZCALLBACK fseek64_file_func (voidpf opaque, voidpf stream, ZPOS64_T offset, int origin) +{ + int fseek_origin=0; + long ret; + switch (origin) + { + case ZLIB_FILEFUNC_SEEK_CUR : + fseek_origin = SEEK_CUR; + break; + case ZLIB_FILEFUNC_SEEK_END : + fseek_origin = SEEK_END; + break; + case ZLIB_FILEFUNC_SEEK_SET : + fseek_origin = SEEK_SET; + break; + default: return -1; + } + ret = 0; + + if(FSEEKO_FUNC((FILE *)stream, offset, fseek_origin) != 0) + ret = -1; + + return ret; +} + + +static int ZCALLBACK fclose_file_func (voidpf opaque, voidpf stream) +{ + int ret; + ret = fclose((FILE *)stream); + return ret; +} + +static int ZCALLBACK ferror_file_func (voidpf opaque, voidpf stream) +{ + int ret; + ret = ferror((FILE *)stream); + return ret; +} + +void fill_fopen_filefunc (pzlib_filefunc_def) + zlib_filefunc_def* pzlib_filefunc_def; +{ + pzlib_filefunc_def->zopen_file = fopen_file_func; + pzlib_filefunc_def->zread_file = fread_file_func; + pzlib_filefunc_def->zwrite_file = fwrite_file_func; + pzlib_filefunc_def->ztell_file = ftell_file_func; + pzlib_filefunc_def->zseek_file = fseek_file_func; + pzlib_filefunc_def->zclose_file = fclose_file_func; + pzlib_filefunc_def->zerror_file = ferror_file_func; + pzlib_filefunc_def->opaque = NULL; +} + +void fill_fopen64_filefunc (zlib_filefunc64_def* pzlib_filefunc_def) +{ + pzlib_filefunc_def->zopen64_file = fopen64_file_func; + pzlib_filefunc_def->zread_file = fread_file_func; + pzlib_filefunc_def->zwrite_file = fwrite_file_func; + pzlib_filefunc_def->ztell64_file = ftell64_file_func; + pzlib_filefunc_def->zseek64_file = fseek64_file_func; + pzlib_filefunc_def->zclose_file = fclose_file_func; + pzlib_filefunc_def->zerror_file = ferror_file_func; + pzlib_filefunc_def->opaque = NULL; +} diff --git a/third_party/zlib/contrib/minizip/ioapi.h b/third_party/zlib/contrib/minizip/ioapi.h new file mode 100644 index 00000000..8dcbdb06 --- /dev/null +++ b/third_party/zlib/contrib/minizip/ioapi.h @@ -0,0 +1,208 @@ +/* ioapi.h -- IO base function header for compress/uncompress .zip + part of the MiniZip project - ( http://www.winimage.com/zLibDll/minizip.html ) + + Copyright (C) 1998-2010 Gilles Vollant (minizip) ( http://www.winimage.com/zLibDll/minizip.html ) + + Modifications for Zip64 support + Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) + + For more info read MiniZip_info.txt + + Changes + + Oct-2009 - Defined ZPOS64_T to fpos_t on windows and u_int64_t on linux. (might need to find a better why for this) + Oct-2009 - Change to fseeko64, ftello64 and fopen64 so large files would work on linux. + More if/def section may be needed to support other platforms + Oct-2009 - Defined fxxxx64 calls to normal fopen/ftell/fseek so they would compile on windows. + (but you should use iowin32.c for windows instead) + +*/ + +#ifndef _ZLIBIOAPI64_H +#define _ZLIBIOAPI64_H + +#if (!defined(_WIN32)) && (!defined(WIN32)) && (!defined(__APPLE__)) + + // Linux needs this to support file operation on files larger then 4+GB + // But might need better if/def to select just the platforms that needs them. + + #ifndef __USE_FILE_OFFSET64 + #define __USE_FILE_OFFSET64 + #endif + #ifndef __USE_LARGEFILE64 + #define __USE_LARGEFILE64 + #endif + #ifndef _LARGEFILE64_SOURCE + #define _LARGEFILE64_SOURCE + #endif + #ifndef _FILE_OFFSET_BIT + #define _FILE_OFFSET_BIT 64 + #endif + +#endif + +#include <stdio.h> +#include <stdlib.h> +#include "zlib.h" + +#if defined(USE_FILE32API) +#define fopen64 fopen +#define ftello64 ftell +#define fseeko64 fseek +#else +#ifdef __FreeBSD__ +#define fopen64 fopen +#define ftello64 ftello +#define fseeko64 fseeko +#endif +#ifdef _MSC_VER + #define fopen64 fopen + #if (_MSC_VER >= 1400) && (!(defined(NO_MSCVER_FILE64_FUNC))) + #define ftello64 _ftelli64 + #define fseeko64 _fseeki64 + #else // old MSC + #define ftello64 ftell + #define fseeko64 fseek + #endif +#endif +#endif + +/* +#ifndef ZPOS64_T + #ifdef _WIN32 + #define ZPOS64_T fpos_t + #else + #include <stdint.h> + #define ZPOS64_T uint64_t + #endif +#endif +*/ + +#ifdef HAVE_MINIZIP64_CONF_H +#include "mz64conf.h" +#endif + +/* a type choosen by DEFINE */ +#ifdef HAVE_64BIT_INT_CUSTOM +typedef 64BIT_INT_CUSTOM_TYPE ZPOS64_T; +#else +#ifdef HAS_STDINT_H +#include "stdint.h" +typedef uint64_t ZPOS64_T; +#else + +/* Maximum unsigned 32-bit value used as placeholder for zip64 */ +#define MAXU32 0xffffffff + +#if defined(_MSC_VER) || defined(__BORLANDC__) +typedef unsigned __int64 ZPOS64_T; +#else +typedef unsigned long long int ZPOS64_T; +#endif +#endif +#endif + + + +#ifdef __cplusplus +extern "C" { +#endif + + +#define ZLIB_FILEFUNC_SEEK_CUR (1) +#define ZLIB_FILEFUNC_SEEK_END (2) +#define ZLIB_FILEFUNC_SEEK_SET (0) + +#define ZLIB_FILEFUNC_MODE_READ (1) +#define ZLIB_FILEFUNC_MODE_WRITE (2) +#define ZLIB_FILEFUNC_MODE_READWRITEFILTER (3) + +#define ZLIB_FILEFUNC_MODE_EXISTING (4) +#define ZLIB_FILEFUNC_MODE_CREATE (8) + + +#ifndef ZCALLBACK + #if (defined(WIN32) || defined(_WIN32) || defined (WINDOWS) || defined (_WINDOWS)) && defined(CALLBACK) && defined (USEWINDOWS_CALLBACK) + #define ZCALLBACK CALLBACK + #else + #define ZCALLBACK + #endif +#endif + + + + +typedef voidpf (ZCALLBACK *open_file_func) OF((voidpf opaque, const char* filename, int mode)); +typedef uLong (ZCALLBACK *read_file_func) OF((voidpf opaque, voidpf stream, void* buf, uLong size)); +typedef uLong (ZCALLBACK *write_file_func) OF((voidpf opaque, voidpf stream, const void* buf, uLong size)); +typedef int (ZCALLBACK *close_file_func) OF((voidpf opaque, voidpf stream)); +typedef int (ZCALLBACK *testerror_file_func) OF((voidpf opaque, voidpf stream)); + +typedef long (ZCALLBACK *tell_file_func) OF((voidpf opaque, voidpf stream)); +typedef long (ZCALLBACK *seek_file_func) OF((voidpf opaque, voidpf stream, uLong offset, int origin)); + + +/* here is the "old" 32 bits structure structure */ +typedef struct zlib_filefunc_def_s +{ + open_file_func zopen_file; + read_file_func zread_file; + write_file_func zwrite_file; + tell_file_func ztell_file; + seek_file_func zseek_file; + close_file_func zclose_file; + testerror_file_func zerror_file; + voidpf opaque; +} zlib_filefunc_def; + +typedef ZPOS64_T (ZCALLBACK *tell64_file_func) OF((voidpf opaque, voidpf stream)); +typedef long (ZCALLBACK *seek64_file_func) OF((voidpf opaque, voidpf stream, ZPOS64_T offset, int origin)); +typedef voidpf (ZCALLBACK *open64_file_func) OF((voidpf opaque, const void* filename, int mode)); + +typedef struct zlib_filefunc64_def_s +{ + open64_file_func zopen64_file; + read_file_func zread_file; + write_file_func zwrite_file; + tell64_file_func ztell64_file; + seek64_file_func zseek64_file; + close_file_func zclose_file; + testerror_file_func zerror_file; + voidpf opaque; +} zlib_filefunc64_def; + +void fill_fopen64_filefunc OF((zlib_filefunc64_def* pzlib_filefunc_def)); +void fill_fopen_filefunc OF((zlib_filefunc_def* pzlib_filefunc_def)); + +/* now internal definition, only for zip.c and unzip.h */ +typedef struct zlib_filefunc64_32_def_s +{ + zlib_filefunc64_def zfile_func64; + open_file_func zopen32_file; + tell_file_func ztell32_file; + seek_file_func zseek32_file; +} zlib_filefunc64_32_def; + + +#define ZREAD64(filefunc,filestream,buf,size) ((*((filefunc).zfile_func64.zread_file)) ((filefunc).zfile_func64.opaque,filestream,buf,size)) +#define ZWRITE64(filefunc,filestream,buf,size) ((*((filefunc).zfile_func64.zwrite_file)) ((filefunc).zfile_func64.opaque,filestream,buf,size)) +//#define ZTELL64(filefunc,filestream) ((*((filefunc).ztell64_file)) ((filefunc).opaque,filestream)) +//#define ZSEEK64(filefunc,filestream,pos,mode) ((*((filefunc).zseek64_file)) ((filefunc).opaque,filestream,pos,mode)) +#define ZCLOSE64(filefunc,filestream) ((*((filefunc).zfile_func64.zclose_file)) ((filefunc).zfile_func64.opaque,filestream)) +#define ZERROR64(filefunc,filestream) ((*((filefunc).zfile_func64.zerror_file)) ((filefunc).zfile_func64.opaque,filestream)) + +voidpf call_zopen64 OF((const zlib_filefunc64_32_def* pfilefunc,const void*filename,int mode)); +long call_zseek64 OF((const zlib_filefunc64_32_def* pfilefunc,voidpf filestream, ZPOS64_T offset, int origin)); +ZPOS64_T call_ztell64 OF((const zlib_filefunc64_32_def* pfilefunc,voidpf filestream)); + +void fill_zlib_filefunc64_32_def_from_filefunc32(zlib_filefunc64_32_def* p_filefunc64_32,const zlib_filefunc_def* p_filefunc32); + +#define ZOPEN64(filefunc,filename,mode) (call_zopen64((&(filefunc)),(filename),(mode))) +#define ZTELL64(filefunc,filestream) (call_ztell64((&(filefunc)),(filestream))) +#define ZSEEK64(filefunc,filestream,pos,mode) (call_zseek64((&(filefunc)),(filestream),(pos),(mode))) + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/third_party/zlib/contrib/minizip/iowin32.c b/third_party/zlib/contrib/minizip/iowin32.c new file mode 100644 index 00000000..c6bc314b --- /dev/null +++ b/third_party/zlib/contrib/minizip/iowin32.c @@ -0,0 +1,467 @@ +/* iowin32.c -- IO base function header for compress/uncompress .zip + Version 1.1, February 14h, 2010 + part of the MiniZip project - ( http://www.winimage.com/zLibDll/minizip.html ) + + Copyright (C) 1998-2010 Gilles Vollant (minizip) ( http://www.winimage.com/zLibDll/minizip.html ) + + Modifications for Zip64 support + Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) + + For more info read MiniZip_info.txt + +*/ + +#include <stdlib.h> + +#include "zlib.h" +#include "ioapi.h" +#include "iowin32.h" + +#ifndef INVALID_HANDLE_VALUE +#define INVALID_HANDLE_VALUE (0xFFFFFFFF) +#endif + +#ifndef INVALID_SET_FILE_POINTER +#define INVALID_SET_FILE_POINTER ((DWORD)-1) +#endif + + +#ifdef _WIN32_WINNT +#undef _WIN32_WINNT +#define _WIN32_WINNT 0x601 +#endif + +#if !defined(IOWIN32_USING_WINRT_API) +#if defined(WINAPI_FAMILY) && (WINAPI_FAMILY == WINAPI_FAMILY_APP) +// Windows Store or Universal Windows Platform +#define IOWIN32_USING_WINRT_API 1 +#endif +#endif + +voidpf ZCALLBACK win32_open_file_func OF((voidpf opaque, const char* filename, int mode)); +uLong ZCALLBACK win32_read_file_func OF((voidpf opaque, voidpf stream, void* buf, uLong size)); +uLong ZCALLBACK win32_write_file_func OF((voidpf opaque, voidpf stream, const void* buf, uLong size)); +ZPOS64_T ZCALLBACK win32_tell64_file_func OF((voidpf opaque, voidpf stream)); +long ZCALLBACK win32_seek64_file_func OF((voidpf opaque, voidpf stream, ZPOS64_T offset, int origin)); +int ZCALLBACK win32_close_file_func OF((voidpf opaque, voidpf stream)); +int ZCALLBACK win32_error_file_func OF((voidpf opaque, voidpf stream)); + +typedef struct +{ + HANDLE hf; + int error; +} WIN32FILE_IOWIN; + + +static void win32_translate_open_mode(int mode, + DWORD* lpdwDesiredAccess, + DWORD* lpdwCreationDisposition, + DWORD* lpdwShareMode, + DWORD* lpdwFlagsAndAttributes) +{ + *lpdwDesiredAccess = *lpdwShareMode = *lpdwFlagsAndAttributes = *lpdwCreationDisposition = 0; + + if ((mode & ZLIB_FILEFUNC_MODE_READWRITEFILTER)==ZLIB_FILEFUNC_MODE_READ) + { + *lpdwDesiredAccess = GENERIC_READ; + *lpdwCreationDisposition = OPEN_EXISTING; + *lpdwShareMode = FILE_SHARE_READ; + } + else if (mode & ZLIB_FILEFUNC_MODE_EXISTING) + { + *lpdwDesiredAccess = GENERIC_WRITE | GENERIC_READ; + *lpdwCreationDisposition = OPEN_EXISTING; + } + else if (mode & ZLIB_FILEFUNC_MODE_CREATE) + { + *lpdwDesiredAccess = GENERIC_WRITE | GENERIC_READ; + *lpdwCreationDisposition = CREATE_ALWAYS; + } +} + +static voidpf win32_build_iowin(HANDLE hFile) +{ + voidpf ret=NULL; + + if ((hFile != NULL) && (hFile != INVALID_HANDLE_VALUE)) + { + WIN32FILE_IOWIN w32fiow; + w32fiow.hf = hFile; + w32fiow.error = 0; + ret = malloc(sizeof(WIN32FILE_IOWIN)); + + if (ret==NULL) + CloseHandle(hFile); + else + *((WIN32FILE_IOWIN*)ret) = w32fiow; + } + return ret; +} + +voidpf ZCALLBACK win32_open64_file_func (voidpf opaque,const void* filename,int mode) +{ + const char* mode_fopen = NULL; + DWORD dwDesiredAccess,dwCreationDisposition,dwShareMode,dwFlagsAndAttributes ; + HANDLE hFile = NULL; + + win32_translate_open_mode(mode,&dwDesiredAccess,&dwCreationDisposition,&dwShareMode,&dwFlagsAndAttributes); + +#ifdef IOWIN32_USING_WINRT_API +#ifdef UNICODE + if ((filename!=NULL) && (dwDesiredAccess != 0)) + hFile = CreateFile2((LPCTSTR)filename, dwDesiredAccess, dwShareMode, dwCreationDisposition, NULL); +#else + if ((filename!=NULL) && (dwDesiredAccess != 0)) + { + WCHAR filenameW[FILENAME_MAX + 0x200 + 1]; + MultiByteToWideChar(CP_ACP,0,(const char*)filename,-1,filenameW,FILENAME_MAX + 0x200); + hFile = CreateFile2(filenameW, dwDesiredAccess, dwShareMode, dwCreationDisposition, NULL); + } +#endif +#else + if ((filename!=NULL) && (dwDesiredAccess != 0)) + hFile = CreateFile((LPCTSTR)filename, dwDesiredAccess, dwShareMode, NULL, dwCreationDisposition, dwFlagsAndAttributes, NULL); +#endif + + return win32_build_iowin(hFile); +} + + +voidpf ZCALLBACK win32_open64_file_funcA (voidpf opaque,const void* filename,int mode) +{ + const char* mode_fopen = NULL; + DWORD dwDesiredAccess,dwCreationDisposition,dwShareMode,dwFlagsAndAttributes ; + HANDLE hFile = NULL; + + win32_translate_open_mode(mode,&dwDesiredAccess,&dwCreationDisposition,&dwShareMode,&dwFlagsAndAttributes); + +#ifdef IOWIN32_USING_WINRT_API + if ((filename!=NULL) && (dwDesiredAccess != 0)) + { + WCHAR filenameW[FILENAME_MAX + 0x200 + 1]; + MultiByteToWideChar(CP_ACP,0,(const char*)filename,-1,filenameW,FILENAME_MAX + 0x200); + hFile = CreateFile2(filenameW, dwDesiredAccess, dwShareMode, dwCreationDisposition, NULL); + } +#else + if ((filename!=NULL) && (dwDesiredAccess != 0)) + hFile = CreateFileA((LPCSTR)filename, dwDesiredAccess, dwShareMode, NULL, dwCreationDisposition, dwFlagsAndAttributes, NULL); +#endif + + return win32_build_iowin(hFile); +} + + +voidpf ZCALLBACK win32_open64_file_funcW (voidpf opaque,const void* filename,int mode) +{ + const char* mode_fopen = NULL; + DWORD dwDesiredAccess,dwCreationDisposition,dwShareMode,dwFlagsAndAttributes ; + HANDLE hFile = NULL; + + win32_translate_open_mode(mode,&dwDesiredAccess,&dwCreationDisposition,&dwShareMode,&dwFlagsAndAttributes); + +#ifdef IOWIN32_USING_WINRT_API + if ((filename!=NULL) && (dwDesiredAccess != 0)) + hFile = CreateFile2((LPCWSTR)filename, dwDesiredAccess, dwShareMode, dwCreationDisposition,NULL); +#else + if ((filename!=NULL) && (dwDesiredAccess != 0)) + hFile = CreateFileW((LPCWSTR)filename, dwDesiredAccess, dwShareMode, NULL, dwCreationDisposition, dwFlagsAndAttributes, NULL); +#endif + + return win32_build_iowin(hFile); +} + + +voidpf ZCALLBACK win32_open_file_func (voidpf opaque,const char* filename,int mode) +{ + const char* mode_fopen = NULL; + DWORD dwDesiredAccess,dwCreationDisposition,dwShareMode,dwFlagsAndAttributes ; + HANDLE hFile = NULL; + + win32_translate_open_mode(mode,&dwDesiredAccess,&dwCreationDisposition,&dwShareMode,&dwFlagsAndAttributes); + +#ifdef IOWIN32_USING_WINRT_API +#ifdef UNICODE + if ((filename!=NULL) && (dwDesiredAccess != 0)) + hFile = CreateFile2((LPCTSTR)filename, dwDesiredAccess, dwShareMode, dwCreationDisposition, NULL); +#else + if ((filename!=NULL) && (dwDesiredAccess != 0)) + { + WCHAR filenameW[FILENAME_MAX + 0x200 + 1]; + MultiByteToWideChar(CP_ACP,0,(const char*)filename,-1,filenameW,FILENAME_MAX + 0x200); + hFile = CreateFile2(filenameW, dwDesiredAccess, dwShareMode, dwCreationDisposition, NULL); + } +#endif +#else + if ((filename!=NULL) && (dwDesiredAccess != 0)) + hFile = CreateFile((LPCTSTR)filename, dwDesiredAccess, dwShareMode, NULL, dwCreationDisposition, dwFlagsAndAttributes, NULL); +#endif + + return win32_build_iowin(hFile); +} + + +uLong ZCALLBACK win32_read_file_func (voidpf opaque, voidpf stream, void* buf,uLong size) +{ + uLong ret=0; + HANDLE hFile = NULL; + if (stream!=NULL) + hFile = ((WIN32FILE_IOWIN*)stream) -> hf; + + if (hFile != NULL) + { + if (!ReadFile(hFile, buf, size, &ret, NULL)) + { + DWORD dwErr = GetLastError(); + if (dwErr == ERROR_HANDLE_EOF) + dwErr = 0; + ((WIN32FILE_IOWIN*)stream) -> error=(int)dwErr; + } + } + + return ret; +} + + +uLong ZCALLBACK win32_write_file_func (voidpf opaque,voidpf stream,const void* buf,uLong size) +{ + uLong ret=0; + HANDLE hFile = NULL; + if (stream!=NULL) + hFile = ((WIN32FILE_IOWIN*)stream) -> hf; + + if (hFile != NULL) + { + if (!WriteFile(hFile, buf, size, &ret, NULL)) + { + DWORD dwErr = GetLastError(); + if (dwErr == ERROR_HANDLE_EOF) + dwErr = 0; + ((WIN32FILE_IOWIN*)stream) -> error=(int)dwErr; + } + } + + return ret; +} + +static BOOL MySetFilePointerEx(HANDLE hFile, LARGE_INTEGER pos, LARGE_INTEGER *newPos, DWORD dwMoveMethod) +{ +#ifdef IOWIN32_USING_WINRT_API + return SetFilePointerEx(hFile, pos, newPos, dwMoveMethod); +#else + LONG lHigh = pos.HighPart; + DWORD dwNewPos = SetFilePointer(hFile, pos.LowPart, &lHigh, dwMoveMethod); + BOOL fOk = TRUE; + if (dwNewPos == 0xFFFFFFFF) + if (GetLastError() != NO_ERROR) + fOk = FALSE; + if ((newPos != NULL) && (fOk)) + { + newPos->LowPart = dwNewPos; + newPos->HighPart = lHigh; + } + return fOk; +#endif +} + +long ZCALLBACK win32_tell_file_func (voidpf opaque,voidpf stream) +{ + long ret=-1; + HANDLE hFile = NULL; + if (stream!=NULL) + hFile = ((WIN32FILE_IOWIN*)stream) -> hf; + if (hFile != NULL) + { + LARGE_INTEGER pos; + pos.QuadPart = 0; + + if (!MySetFilePointerEx(hFile, pos, &pos, FILE_CURRENT)) + { + DWORD dwErr = GetLastError(); + ((WIN32FILE_IOWIN*)stream) -> error=(int)dwErr; + ret = -1; + } + else + ret=(long)pos.LowPart; + } + return ret; +} + +ZPOS64_T ZCALLBACK win32_tell64_file_func (voidpf opaque, voidpf stream) +{ + ZPOS64_T ret= (ZPOS64_T)-1; + HANDLE hFile = NULL; + if (stream!=NULL) + hFile = ((WIN32FILE_IOWIN*)stream)->hf; + + if (hFile) + { + LARGE_INTEGER pos; + pos.QuadPart = 0; + + if (!MySetFilePointerEx(hFile, pos, &pos, FILE_CURRENT)) + { + DWORD dwErr = GetLastError(); + ((WIN32FILE_IOWIN*)stream) -> error=(int)dwErr; + ret = (ZPOS64_T)-1; + } + else + ret=pos.QuadPart; + } + return ret; +} + + +long ZCALLBACK win32_seek_file_func (voidpf opaque,voidpf stream,uLong offset,int origin) +{ + DWORD dwMoveMethod=0xFFFFFFFF; + HANDLE hFile = NULL; + + long ret=-1; + if (stream!=NULL) + hFile = ((WIN32FILE_IOWIN*)stream) -> hf; + switch (origin) + { + case ZLIB_FILEFUNC_SEEK_CUR : + dwMoveMethod = FILE_CURRENT; + break; + case ZLIB_FILEFUNC_SEEK_END : + dwMoveMethod = FILE_END; + break; + case ZLIB_FILEFUNC_SEEK_SET : + dwMoveMethod = FILE_BEGIN; + break; + default: return -1; + } + + if (hFile != NULL) + { + LARGE_INTEGER pos; + pos.QuadPart = offset; + if (!MySetFilePointerEx(hFile, pos, NULL, dwMoveMethod)) + { + DWORD dwErr = GetLastError(); + ((WIN32FILE_IOWIN*)stream) -> error=(int)dwErr; + ret = -1; + } + else + ret=0; + } + return ret; +} + +long ZCALLBACK win32_seek64_file_func (voidpf opaque, voidpf stream,ZPOS64_T offset,int origin) +{ + DWORD dwMoveMethod=0xFFFFFFFF; + HANDLE hFile = NULL; + long ret=-1; + + if (stream!=NULL) + hFile = ((WIN32FILE_IOWIN*)stream)->hf; + + switch (origin) + { + case ZLIB_FILEFUNC_SEEK_CUR : + dwMoveMethod = FILE_CURRENT; + break; + case ZLIB_FILEFUNC_SEEK_END : + dwMoveMethod = FILE_END; + break; + case ZLIB_FILEFUNC_SEEK_SET : + dwMoveMethod = FILE_BEGIN; + break; + default: return -1; + } + + if (hFile) + { + LARGE_INTEGER pos; + pos.QuadPart = offset; + if (!MySetFilePointerEx(hFile, pos, NULL, dwMoveMethod)) + { + DWORD dwErr = GetLastError(); + ((WIN32FILE_IOWIN*)stream) -> error=(int)dwErr; + ret = -1; + } + else + ret=0; + } + return ret; +} + +int ZCALLBACK win32_close_file_func (voidpf opaque, voidpf stream) +{ + int ret=-1; + + if (stream!=NULL) + { + HANDLE hFile; + hFile = ((WIN32FILE_IOWIN*)stream) -> hf; + if (hFile != NULL) + { + CloseHandle(hFile); + ret=0; + } + free(stream); + } + return ret; +} + +int ZCALLBACK win32_error_file_func (voidpf opaque,voidpf stream) +{ + int ret=-1; + if (stream!=NULL) + { + ret = ((WIN32FILE_IOWIN*)stream) -> error; + } + return ret; +} + +void fill_win32_filefunc (zlib_filefunc_def* pzlib_filefunc_def) +{ + pzlib_filefunc_def->zopen_file = win32_open_file_func; + pzlib_filefunc_def->zread_file = win32_read_file_func; + pzlib_filefunc_def->zwrite_file = win32_write_file_func; + pzlib_filefunc_def->ztell_file = win32_tell_file_func; + pzlib_filefunc_def->zseek_file = win32_seek_file_func; + pzlib_filefunc_def->zclose_file = win32_close_file_func; + pzlib_filefunc_def->zerror_file = win32_error_file_func; + pzlib_filefunc_def->opaque = NULL; +} + +void fill_win32_filefunc64(zlib_filefunc64_def* pzlib_filefunc_def) +{ + pzlib_filefunc_def->zopen64_file = win32_open64_file_func; + pzlib_filefunc_def->zread_file = win32_read_file_func; + pzlib_filefunc_def->zwrite_file = win32_write_file_func; + pzlib_filefunc_def->ztell64_file = win32_tell64_file_func; + pzlib_filefunc_def->zseek64_file = win32_seek64_file_func; + pzlib_filefunc_def->zclose_file = win32_close_file_func; + pzlib_filefunc_def->zerror_file = win32_error_file_func; + pzlib_filefunc_def->opaque = NULL; +} + + +void fill_win32_filefunc64A(zlib_filefunc64_def* pzlib_filefunc_def) +{ + pzlib_filefunc_def->zopen64_file = win32_open64_file_funcA; + pzlib_filefunc_def->zread_file = win32_read_file_func; + pzlib_filefunc_def->zwrite_file = win32_write_file_func; + pzlib_filefunc_def->ztell64_file = win32_tell64_file_func; + pzlib_filefunc_def->zseek64_file = win32_seek64_file_func; + pzlib_filefunc_def->zclose_file = win32_close_file_func; + pzlib_filefunc_def->zerror_file = win32_error_file_func; + pzlib_filefunc_def->opaque = NULL; +} + + +void fill_win32_filefunc64W(zlib_filefunc64_def* pzlib_filefunc_def) +{ + pzlib_filefunc_def->zopen64_file = win32_open64_file_funcW; + pzlib_filefunc_def->zread_file = win32_read_file_func; + pzlib_filefunc_def->zwrite_file = win32_write_file_func; + pzlib_filefunc_def->ztell64_file = win32_tell64_file_func; + pzlib_filefunc_def->zseek64_file = win32_seek64_file_func; + pzlib_filefunc_def->zclose_file = win32_close_file_func; + pzlib_filefunc_def->zerror_file = win32_error_file_func; + pzlib_filefunc_def->opaque = NULL; +} diff --git a/third_party/zlib/contrib/minizip/iowin32.h b/third_party/zlib/contrib/minizip/iowin32.h new file mode 100644 index 00000000..0ca0969a --- /dev/null +++ b/third_party/zlib/contrib/minizip/iowin32.h @@ -0,0 +1,28 @@ +/* iowin32.h -- IO base function header for compress/uncompress .zip + Version 1.1, February 14h, 2010 + part of the MiniZip project - ( http://www.winimage.com/zLibDll/minizip.html ) + + Copyright (C) 1998-2010 Gilles Vollant (minizip) ( http://www.winimage.com/zLibDll/minizip.html ) + + Modifications for Zip64 support + Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) + + For more info read MiniZip_info.txt + +*/ + +#include <windows.h> + + +#ifdef __cplusplus +extern "C" { +#endif + +void fill_win32_filefunc OF((zlib_filefunc_def* pzlib_filefunc_def)); +void fill_win32_filefunc64 OF((zlib_filefunc64_def* pzlib_filefunc_def)); +void fill_win32_filefunc64A OF((zlib_filefunc64_def* pzlib_filefunc_def)); +void fill_win32_filefunc64W OF((zlib_filefunc64_def* pzlib_filefunc_def)); + +#ifdef __cplusplus +} +#endif diff --git a/third_party/zlib/contrib/minizip/miniunz.c b/third_party/zlib/contrib/minizip/miniunz.c new file mode 100644 index 00000000..08737f68 --- /dev/null +++ b/third_party/zlib/contrib/minizip/miniunz.c @@ -0,0 +1,659 @@ +/* + miniunz.c + Version 1.1, February 14h, 2010 + sample part of the MiniZip project - ( http://www.winimage.com/zLibDll/minizip.html ) + + Copyright (C) 1998-2010 Gilles Vollant (minizip) ( http://www.winimage.com/zLibDll/minizip.html ) + + Modifications of Unzip for Zip64 + Copyright (C) 2007-2008 Even Rouault + + Modifications for Zip64 support on both zip and unzip + Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) +*/ + +#if (!defined(_WIN32)) && (!defined(WIN32)) && (!defined(__APPLE__)) && (!defined(__ANDROID_API__)) + #ifndef __USE_FILE_OFFSET64 + #define __USE_FILE_OFFSET64 + #endif + #ifndef __USE_LARGEFILE64 + #define __USE_LARGEFILE64 + #endif + #ifndef _LARGEFILE64_SOURCE + #define _LARGEFILE64_SOURCE + #endif + #ifndef _FILE_OFFSET_BIT + #define _FILE_OFFSET_BIT 64 + #endif +#endif + +#if defined(__APPLE__) || defined(__Fuchsia__) || defined(__ANDROID_API__) +// In darwin and perhaps other BSD variants off_t is a 64 bit value, hence no need for specific 64 bit functions +#define FOPEN_FUNC(filename, mode) fopen(filename, mode) +#define FTELLO_FUNC(stream) ftello(stream) +#define FSEEKO_FUNC(stream, offset, origin) fseeko(stream, offset, origin) +#else +#define FOPEN_FUNC(filename, mode) fopen64(filename, mode) +#define FTELLO_FUNC(stream) ftello64(stream) +#define FSEEKO_FUNC(stream, offset, origin) fseeko64(stream, offset, origin) +#endif + + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include <errno.h> +#include <fcntl.h> +#include <sys/stat.h> + +#ifdef _WIN32 +# include <direct.h> +# include <io.h> +#else +# include <unistd.h> +# include <utime.h> +#endif + + +#include "unzip.h" + +#define CASESENSITIVITY (0) +#define WRITEBUFFERSIZE (8192) +#define MAXFILENAME (256) + +#ifdef _WIN32 +#define USEWIN32IOAPI +#include "iowin32.h" +#endif +/* + mini unzip, demo of unzip package + + usage : + Usage : miniunz [-exvlo] file.zip [file_to_extract] [-d extractdir] + + list the file in the zipfile, and print the content of FILE_ID.ZIP or README.TXT + if it exists +*/ + + +/* change_file_date : change the date/time of a file + filename : the filename of the file where date/time must be modified + dosdate : the new date at the MSDos format (4 bytes) + tmu_date : the SAME new date at the tm_unz format */ +void change_file_date(filename,dosdate,tmu_date) + const char *filename; + uLong dosdate; + tm_unz tmu_date; +{ +#ifdef _WIN32 + HANDLE hFile; + FILETIME ftm,ftLocal,ftCreate,ftLastAcc,ftLastWrite; + + hFile = CreateFileA(filename,GENERIC_READ | GENERIC_WRITE, + 0,NULL,OPEN_EXISTING,0,NULL); + GetFileTime(hFile,&ftCreate,&ftLastAcc,&ftLastWrite); + DosDateTimeToFileTime((WORD)(dosdate>>16),(WORD)dosdate,&ftLocal); + LocalFileTimeToFileTime(&ftLocal,&ftm); + SetFileTime(hFile,&ftm,&ftLastAcc,&ftm); + CloseHandle(hFile); +#else +#if defined(unix) || defined(__APPLE__) || defined(__Fuchsia__) || defined(__ANDROID_API__) + struct utimbuf ut; + struct tm newdate; + newdate.tm_sec = tmu_date.tm_sec; + newdate.tm_min=tmu_date.tm_min; + newdate.tm_hour=tmu_date.tm_hour; + newdate.tm_mday=tmu_date.tm_mday; + newdate.tm_mon=tmu_date.tm_mon; + if (tmu_date.tm_year > 1900) + newdate.tm_year=tmu_date.tm_year - 1900; + else + newdate.tm_year=tmu_date.tm_year ; + newdate.tm_isdst=-1; + + ut.actime=ut.modtime=mktime(&newdate); + utime(filename,&ut); +#endif +#endif +} + + +/* mymkdir and change_file_date are not 100 % portable + As I don't know well Unix, I wait feedback for the unix portion */ + +int mymkdir(dirname) + const char* dirname; +{ + int ret=0; +#if defined(_WIN32) + ret = _mkdir(dirname); +#elif defined(unix) || defined(__APPLE__) || defined(__Fuchsia__) || defined(__ANDROID_API__) + ret = mkdir (dirname,0775); +#endif + return ret; +} + +int makedir (newdir) + char *newdir; +{ + char *buffer ; + char *p; + int len = (int)strlen(newdir); + + if (len <= 0) + return 0; + + buffer = (char*)malloc(len+1); + if (buffer==NULL) + { + printf("Error allocating memory\n"); + return UNZ_INTERNALERROR; + } + strcpy(buffer,newdir); + + if (buffer[len-1] == '/') { + buffer[len-1] = '\0'; + } + if (mymkdir(buffer) == 0) + { + free(buffer); + return 1; + } + + p = buffer+1; + while (1) + { + char hold; + + while(*p && *p != '\\' && *p != '/') + p++; + hold = *p; + *p = 0; + if ((mymkdir(buffer) == -1) && (errno == ENOENT)) + { + printf("couldn't create directory %s\n",buffer); + free(buffer); + return 0; + } + if (hold == 0) + break; + *p++ = hold; + } + free(buffer); + return 1; +} + +void do_banner() +{ + printf("MiniUnz 1.01b, demo of zLib + Unz package written by Gilles Vollant\n"); + printf("more info at http://www.winimage.com/zLibDll/unzip.html\n\n"); +} + +void do_help() +{ + printf("Usage : miniunz [-e] [-x] [-v] [-l] [-o] [-p password] file.zip [file_to_extr.] [-d extractdir]\n\n" \ + " -e Extract without pathname (junk paths)\n" \ + " -x Extract with pathname\n" \ + " -v list files\n" \ + " -l list files\n" \ + " -d directory to extract into\n" \ + " -o overwrite files without prompting\n" \ + " -p extract crypted file using password\n\n"); +} + +void Display64BitsSize(ZPOS64_T n, int size_char) +{ + /* to avoid compatibility problem , we do here the conversion */ + char number[21]; + int offset=19; + int pos_string = 19; + number[20]=0; + for (;;) { + number[offset]=(char)((n%10)+'0'); + if (number[offset] != '0') + pos_string=offset; + n/=10; + if (offset==0) + break; + offset--; + } + { + int size_display_string = 19-pos_string; + while (size_char > size_display_string) + { + size_char--; + printf(" "); + } + } + + printf("%s",&number[pos_string]); +} + +int do_list(uf) + unzFile uf; +{ + uLong i; + unz_global_info64 gi; + int err; + + err = unzGetGlobalInfo64(uf,&gi); + if (err!=UNZ_OK) + printf("error %d with zipfile in unzGetGlobalInfo \n",err); + printf(" Length Method Size Ratio Date Time CRC-32 Name\n"); + printf(" ------ ------ ---- ----- ---- ---- ------ ----\n"); + for (i=0;i<gi.number_entry;i++) + { + char filename_inzip[256]; + unz_file_info64 file_info; + uLong ratio=0; + const char *string_method; + char charCrypt=' '; + err = unzGetCurrentFileInfo64(uf,&file_info,filename_inzip,sizeof(filename_inzip),NULL,0,NULL,0); + if (err!=UNZ_OK) + { + printf("error %d with zipfile in unzGetCurrentFileInfo\n",err); + break; + } + if (file_info.uncompressed_size>0) + ratio = (uLong)((file_info.compressed_size*100)/file_info.uncompressed_size); + + /* display a '*' if the file is crypted */ + if ((file_info.flag & 1) != 0) + charCrypt='*'; + + if (file_info.compression_method==0) + string_method="Stored"; + else + if (file_info.compression_method==Z_DEFLATED) + { + uInt iLevel=(uInt)((file_info.flag & 0x6)/2); + if (iLevel==0) + string_method="Defl:N"; + else if (iLevel==1) + string_method="Defl:X"; + else if ((iLevel==2) || (iLevel==3)) + string_method="Defl:F"; /* 2:fast , 3 : extra fast*/ + } + else + if (file_info.compression_method==Z_BZIP2ED) + { + string_method="BZip2 "; + } + else + string_method="Unkn. "; + + Display64BitsSize(file_info.uncompressed_size,7); + printf(" %6s%c",string_method,charCrypt); + Display64BitsSize(file_info.compressed_size,7); + printf(" %3lu%% %2.2lu-%2.2lu-%2.2lu %2.2lu:%2.2lu %8.8lx %s\n", + ratio, + (uLong)file_info.tmu_date.tm_mon + 1, + (uLong)file_info.tmu_date.tm_mday, + (uLong)file_info.tmu_date.tm_year % 100, + (uLong)file_info.tmu_date.tm_hour,(uLong)file_info.tmu_date.tm_min, + (uLong)file_info.crc,filename_inzip); + if ((i+1)<gi.number_entry) + { + err = unzGoToNextFile(uf); + if (err!=UNZ_OK) + { + printf("error %d with zipfile in unzGoToNextFile\n",err); + break; + } + } + } + + return 0; +} + + +int do_extract_currentfile(uf,popt_extract_without_path,popt_overwrite,password) + unzFile uf; + const int* popt_extract_without_path; + int* popt_overwrite; + const char* password; +{ + char filename_inzip[256]; + char* filename_withoutpath; + char* p; + int err=UNZ_OK; + FILE *fout=NULL; + void* buf; + uInt size_buf; + + unz_file_info64 file_info; + uLong ratio=0; + err = unzGetCurrentFileInfo64(uf,&file_info,filename_inzip,sizeof(filename_inzip),NULL,0,NULL,0); + + if (err!=UNZ_OK) + { + printf("error %d with zipfile in unzGetCurrentFileInfo\n",err); + return err; + } + + size_buf = WRITEBUFFERSIZE; + buf = (void*)malloc(size_buf); + if (buf==NULL) + { + printf("Error allocating memory\n"); + return UNZ_INTERNALERROR; + } + + p = filename_withoutpath = filename_inzip; + while ((*p) != '\0') + { + if (((*p)=='/') || ((*p)=='\\')) + filename_withoutpath = p+1; + p++; + } + + if ((*filename_withoutpath)=='\0') + { + if ((*popt_extract_without_path)==0) + { + printf("creating directory: %s\n",filename_inzip); + mymkdir(filename_inzip); + } + } + else + { + const char* write_filename; + int skip=0; + + if ((*popt_extract_without_path)==0) + write_filename = filename_inzip; + else + write_filename = filename_withoutpath; + + err = unzOpenCurrentFilePassword(uf,password); + if (err!=UNZ_OK) + { + printf("error %d with zipfile in unzOpenCurrentFilePassword\n",err); + } + + if (((*popt_overwrite)==0) && (err==UNZ_OK)) + { + char rep=0; + FILE* ftestexist; + ftestexist = FOPEN_FUNC(write_filename,"rb"); + if (ftestexist!=NULL) + { + fclose(ftestexist); + do + { + char answer[128]; + int ret; + + printf("The file %s exists. Overwrite ? [y]es, [n]o, [A]ll: ",write_filename); + ret = scanf("%1s",answer); + if (ret != 1) + { + exit(EXIT_FAILURE); + } + rep = answer[0] ; + if ((rep>='a') && (rep<='z')) + rep -= 0x20; + } + while ((rep!='Y') && (rep!='N') && (rep!='A')); + } + + if (rep == 'N') + skip = 1; + + if (rep == 'A') + *popt_overwrite=1; + } + + if ((skip==0) && (err==UNZ_OK)) + { + fout=FOPEN_FUNC(write_filename,"wb"); + /* some zipfile don't contain directory alone before file */ + if ((fout==NULL) && ((*popt_extract_without_path)==0) && + (filename_withoutpath!=(char*)filename_inzip)) + { + char c=*(filename_withoutpath-1); + *(filename_withoutpath-1)='\0'; + makedir(write_filename); + *(filename_withoutpath-1)=c; + fout=FOPEN_FUNC(write_filename,"wb"); + } + + if (fout==NULL) + { + printf("error opening %s\n",write_filename); + } + } + + if (fout!=NULL) + { + printf(" extracting: %s\n",write_filename); + + do + { + err = unzReadCurrentFile(uf,buf,size_buf); + if (err<0) + { + printf("error %d with zipfile in unzReadCurrentFile\n",err); + break; + } + if (err>0) + if (fwrite(buf,err,1,fout)!=1) + { + printf("error in writing extracted file\n"); + err=UNZ_ERRNO; + break; + } + } + while (err>0); + if (fout) + fclose(fout); + + if (err==0) + change_file_date(write_filename,file_info.dosDate, + file_info.tmu_date); + } + + if (err==UNZ_OK) + { + err = unzCloseCurrentFile (uf); + if (err!=UNZ_OK) + { + printf("error %d with zipfile in unzCloseCurrentFile\n",err); + } + } + else + unzCloseCurrentFile(uf); /* don't lose the error */ + } + + free(buf); + return err; +} + + +int do_extract(uf,opt_extract_without_path,opt_overwrite,password) + unzFile uf; + int opt_extract_without_path; + int opt_overwrite; + const char* password; +{ + uLong i; + unz_global_info64 gi; + int err; + FILE* fout=NULL; + + err = unzGetGlobalInfo64(uf,&gi); + if (err!=UNZ_OK) + printf("error %d with zipfile in unzGetGlobalInfo \n",err); + + for (i=0;i<gi.number_entry;i++) + { + if (do_extract_currentfile(uf,&opt_extract_without_path, + &opt_overwrite, + password) != UNZ_OK) + break; + + if ((i+1)<gi.number_entry) + { + err = unzGoToNextFile(uf); + if (err!=UNZ_OK) + { + printf("error %d with zipfile in unzGoToNextFile\n",err); + break; + } + } + } + + return 0; +} + +int do_extract_onefile(uf,filename,opt_extract_without_path,opt_overwrite,password) + unzFile uf; + const char* filename; + int opt_extract_without_path; + int opt_overwrite; + const char* password; +{ + int err = UNZ_OK; + if (unzLocateFile(uf,filename,CASESENSITIVITY)!=UNZ_OK) + { + printf("file %s not found in the zipfile\n",filename); + return 2; + } + + if (do_extract_currentfile(uf,&opt_extract_without_path, + &opt_overwrite, + password) == UNZ_OK) + return 0; + else + return 1; +} + + +int main(argc,argv) + int argc; + char *argv[]; +{ + const char *zipfilename=NULL; + const char *filename_to_extract=NULL; + const char *password=NULL; + char filename_try[MAXFILENAME+16] = ""; + int i; + int ret_value=0; + int opt_do_list=0; + int opt_do_extract=1; + int opt_do_extract_withoutpath=0; + int opt_overwrite=0; + int opt_extractdir=0; + const char *dirname=NULL; + unzFile uf=NULL; + + do_banner(); + if (argc==1) + { + do_help(); + return 0; + } + else + { + for (i=1;i<argc;i++) + { + if ((*argv[i])=='-') + { + const char *p=argv[i]+1; + + while ((*p)!='\0') + { + char c=*(p++);; + if ((c=='l') || (c=='L')) + opt_do_list = 1; + if ((c=='v') || (c=='V')) + opt_do_list = 1; + if ((c=='x') || (c=='X')) + opt_do_extract = 1; + if ((c=='e') || (c=='E')) + opt_do_extract = opt_do_extract_withoutpath = 1; + if ((c=='o') || (c=='O')) + opt_overwrite=1; + if ((c=='d') || (c=='D')) + { + opt_extractdir=1; + dirname=argv[i+1]; + } + + if (((c=='p') || (c=='P')) && (i+1<argc)) + { + password=argv[i+1]; + i++; + } + } + } + else + { + if (zipfilename == NULL) + zipfilename = argv[i]; + else if ((filename_to_extract==NULL) && (!opt_extractdir)) + filename_to_extract = argv[i] ; + } + } + } + + if (zipfilename!=NULL) + { + +# ifdef USEWIN32IOAPI + zlib_filefunc64_def ffunc; +# endif + + strncpy(filename_try, zipfilename,MAXFILENAME-1); + /* strncpy doesnt append the trailing NULL, of the string is too long. */ + filename_try[ MAXFILENAME ] = '\0'; + +# ifdef USEWIN32IOAPI + fill_win32_filefunc64A(&ffunc); + uf = unzOpen2_64(zipfilename,&ffunc); +# else + uf = unzOpen64(zipfilename); +# endif + if (uf==NULL) + { + strcat(filename_try,".zip"); +# ifdef USEWIN32IOAPI + uf = unzOpen2_64(filename_try,&ffunc); +# else + uf = unzOpen64(filename_try); +# endif + } + } + + if (uf==NULL) + { + printf("Cannot open %s or %s.zip\n",zipfilename,zipfilename); + return 1; + } + printf("%s opened\n",filename_try); + + if (opt_do_list==1) + ret_value = do_list(uf); + else if (opt_do_extract==1) + { +#ifdef _WIN32 + if (opt_extractdir && _chdir(dirname)) +#else + if (opt_extractdir && chdir(dirname)) +#endif + { + printf("Error changing into %s, aborting\n", dirname); + exit(-1); + } + + if (filename_to_extract == NULL) + ret_value = do_extract(uf, opt_do_extract_withoutpath, opt_overwrite, password); + else + ret_value = do_extract_onefile(uf, filename_to_extract, opt_do_extract_withoutpath, opt_overwrite, password); + } + + unzClose(uf); + + return ret_value; +} diff --git a/third_party/zlib/contrib/minizip/minizip.c b/third_party/zlib/contrib/minizip/minizip.c new file mode 100644 index 00000000..b794953c --- /dev/null +++ b/third_party/zlib/contrib/minizip/minizip.c @@ -0,0 +1,519 @@ +/* + minizip.c + Version 1.1, February 14h, 2010 + sample part of the MiniZip project - ( http://www.winimage.com/zLibDll/minizip.html ) + + Copyright (C) 1998-2010 Gilles Vollant (minizip) ( http://www.winimage.com/zLibDll/minizip.html ) + + Modifications of Unzip for Zip64 + Copyright (C) 2007-2008 Even Rouault + + Modifications for Zip64 support on both zip and unzip + Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) +*/ + +#if (!defined(_WIN32)) && (!defined(WIN32)) && (!defined(__APPLE__)) && (!defined(__ANDROID_API__)) + #ifndef __USE_FILE_OFFSET64 + #define __USE_FILE_OFFSET64 + #endif + #ifndef __USE_LARGEFILE64 + #define __USE_LARGEFILE64 + #endif + #ifndef _LARGEFILE64_SOURCE + #define _LARGEFILE64_SOURCE + #endif + #ifndef _FILE_OFFSET_BIT + #define _FILE_OFFSET_BIT 64 + #endif +#endif + +#if defined(__APPLE__) || defined(__Fuchsia__) || defined(__ANDROID_API__) +// In darwin and perhaps other BSD variants off_t is a 64 bit value, hence no need for specific 64 bit functions +#define FOPEN_FUNC(filename, mode) fopen(filename, mode) +#define FTELLO_FUNC(stream) ftello(stream) +#define FSEEKO_FUNC(stream, offset, origin) fseeko(stream, offset, origin) +#else +#define FOPEN_FUNC(filename, mode) fopen64(filename, mode) +#define FTELLO_FUNC(stream) ftello64(stream) +#define FSEEKO_FUNC(stream, offset, origin) fseeko64(stream, offset, origin) +#endif + + + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include <errno.h> +#include <fcntl.h> + +#ifdef _WIN32 +# include <direct.h> +# include <io.h> +#else +# include <unistd.h> +# include <utime.h> +# include <sys/types.h> +# include <sys/stat.h> +#endif + +#include "zip.h" + +#ifdef _WIN32 + #define USEWIN32IOAPI + #include "iowin32.h" +#endif + + + +#define WRITEBUFFERSIZE (16384) +#define MAXFILENAME (256) + +#ifdef _WIN32 +uLong filetime(f, tmzip, dt) + char *f; /* name of file to get info on */ + tm_zip *tmzip; /* return value: access, modific. and creation times */ + uLong *dt; /* dostime */ +{ + int ret = 0; + { + FILETIME ftLocal; + HANDLE hFind; + WIN32_FIND_DATAA ff32; + + hFind = FindFirstFileA(f,&ff32); + if (hFind != INVALID_HANDLE_VALUE) + { + FileTimeToLocalFileTime(&(ff32.ftLastWriteTime),&ftLocal); + FileTimeToDosDateTime(&ftLocal,((LPWORD)dt)+1,((LPWORD)dt)+0); + FindClose(hFind); + ret = 1; + } + } + return ret; +} +#else +#if defined(unix) || defined(__APPLE__) || defined(__Fuchsia__) || defined(__ANDROID_API__) +uLong filetime(f, tmzip, dt) + char *f; /* name of file to get info on */ + tm_zip *tmzip; /* return value: access, modific. and creation times */ + uLong *dt; /* dostime */ +{ + int ret=0; + struct stat s; /* results of stat() */ + struct tm* filedate; + time_t tm_t=0; + + if (strcmp(f,"-")!=0) + { + char name[MAXFILENAME+1]; + int len = strlen(f); + if (len > MAXFILENAME) + len = MAXFILENAME; + + strncpy(name, f,MAXFILENAME-1); + /* strncpy doesnt append the trailing NULL, of the string is too long. */ + name[ MAXFILENAME ] = '\0'; + + if (name[len - 1] == '/') + name[len - 1] = '\0'; + /* not all systems allow stat'ing a file with / appended */ + if (stat(name,&s)==0) + { + tm_t = s.st_mtime; + ret = 1; + } + } + filedate = localtime(&tm_t); + + tmzip->tm_sec = filedate->tm_sec; + tmzip->tm_min = filedate->tm_min; + tmzip->tm_hour = filedate->tm_hour; + tmzip->tm_mday = filedate->tm_mday; + tmzip->tm_mon = filedate->tm_mon ; + tmzip->tm_year = filedate->tm_year; + + return ret; +} +#else +uLong filetime(f, tmzip, dt) + char *f; /* name of file to get info on */ + tm_zip *tmzip; /* return value: access, modific. and creation times */ + uLong *dt; /* dostime */ +{ + return 0; +} +#endif +#endif + + + + +int check_exist_file(filename) + const char* filename; +{ + FILE* ftestexist; + int ret = 1; + ftestexist = FOPEN_FUNC(filename,"rb"); + if (ftestexist==NULL) + ret = 0; + else + fclose(ftestexist); + return ret; +} + +void do_banner() +{ + printf("MiniZip 1.1, demo of zLib + MiniZip64 package, written by Gilles Vollant\n"); + printf("more info on MiniZip at http://www.winimage.com/zLibDll/minizip.html\n\n"); +} + +void do_help() +{ + printf("Usage : minizip [-o] [-a] [-0 to -9] [-p password] [-j] file.zip [files_to_add]\n\n" \ + " -o Overwrite existing file.zip\n" \ + " -a Append to existing file.zip\n" \ + " -0 Store only\n" \ + " -1 Compress faster\n" \ + " -9 Compress better\n\n" \ + " -j exclude path. store only the file name.\n\n"); +} + +/* calculate the CRC32 of a file, + because to encrypt a file, we need known the CRC32 of the file before */ +int getFileCrc(const char* filenameinzip,void*buf,unsigned long size_buf,unsigned long* result_crc) +{ + unsigned long calculate_crc=0; + int err=ZIP_OK; + FILE * fin = FOPEN_FUNC(filenameinzip,"rb"); + + unsigned long size_read = 0; + unsigned long total_read = 0; + if (fin==NULL) + { + err = ZIP_ERRNO; + } + + if (err == ZIP_OK) + do + { + err = ZIP_OK; + size_read = (int)fread(buf,1,size_buf,fin); + if (size_read < size_buf) + if (feof(fin)==0) + { + printf("error in reading %s\n",filenameinzip); + err = ZIP_ERRNO; + } + + if (size_read>0) + calculate_crc = crc32(calculate_crc,buf,size_read); + total_read += size_read; + + } while ((err == ZIP_OK) && (size_read>0)); + + if (fin) + fclose(fin); + + *result_crc=calculate_crc; + printf("file %s crc %lx\n", filenameinzip, calculate_crc); + return err; +} + +int isLargeFile(const char* filename) +{ + int largeFile = 0; + ZPOS64_T pos = 0; + FILE* pFile = FOPEN_FUNC(filename, "rb"); + + if(pFile != NULL) + { + int n = FSEEKO_FUNC(pFile, 0, SEEK_END); + pos = FTELLO_FUNC(pFile); + + printf("File : %s is %lld bytes\n", filename, pos); + + if(pos >= 0xffffffff) + largeFile = 1; + + fclose(pFile); + } + + return largeFile; +} + +int main(argc,argv) + int argc; + char *argv[]; +{ + int i; + int opt_overwrite=0; + int opt_compress_level=Z_DEFAULT_COMPRESSION; + int opt_exclude_path=0; + int zipfilenamearg = 0; + char filename_try[MAXFILENAME+16]; + int zipok; + int err=0; + int size_buf=0; + void* buf=NULL; + const char* password=NULL; + + + do_banner(); + if (argc==1) + { + do_help(); + return 0; + } + else + { + for (i=1;i<argc;i++) + { + if ((*argv[i])=='-') + { + const char *p=argv[i]+1; + + while ((*p)!='\0') + { + char c=*(p++);; + if ((c=='o') || (c=='O')) + opt_overwrite = 1; + if ((c=='a') || (c=='A')) + opt_overwrite = 2; + if ((c>='0') && (c<='9')) + opt_compress_level = c-'0'; + if ((c=='j') || (c=='J')) + opt_exclude_path = 1; + + if (((c=='p') || (c=='P')) && (i+1<argc)) + { + password=argv[i+1]; + i++; + } + } + } + else + { + if (zipfilenamearg == 0) + { + zipfilenamearg = i ; + } + } + } + } + + size_buf = WRITEBUFFERSIZE; + buf = (void*)malloc(size_buf); + if (buf==NULL) + { + printf("Error allocating memory\n"); + return ZIP_INTERNALERROR; + } + + if (zipfilenamearg==0) + { + zipok=0; + } + else + { + int i,len; + int dot_found=0; + + zipok = 1 ; + strncpy(filename_try, argv[zipfilenamearg],MAXFILENAME-1); + /* strncpy doesnt append the trailing NULL, of the string is too long. */ + filename_try[ MAXFILENAME ] = '\0'; + + len=(int)strlen(filename_try); + for (i=0;i<len;i++) + if (filename_try[i]=='.') + dot_found=1; + + if (dot_found==0) + strcat(filename_try,".zip"); + + if (opt_overwrite==2) + { + /* if the file don't exist, we not append file */ + if (check_exist_file(filename_try)==0) + opt_overwrite=1; + } + else + if (opt_overwrite==0) + if (check_exist_file(filename_try)!=0) + { + char rep=0; + do + { + char answer[128]; + int ret; + printf("The file %s exists. Overwrite ? [y]es, [n]o, [a]ppend : ",filename_try); + ret = scanf("%1s",answer); + if (ret != 1) + { + exit(EXIT_FAILURE); + } + rep = answer[0] ; + if ((rep>='a') && (rep<='z')) + rep -= 0x20; + } + while ((rep!='Y') && (rep!='N') && (rep!='A')); + if (rep=='N') + zipok = 0; + if (rep=='A') + opt_overwrite = 2; + } + } + + if (zipok==1) + { + zipFile zf; + int errclose; +# ifdef USEWIN32IOAPI + zlib_filefunc64_def ffunc; + fill_win32_filefunc64A(&ffunc); + zf = zipOpen2_64(filename_try,(opt_overwrite==2) ? 2 : 0,NULL,&ffunc); +# else + zf = zipOpen64(filename_try,(opt_overwrite==2) ? 2 : 0); +# endif + + if (zf == NULL) + { + printf("error opening %s\n",filename_try); + err= ZIP_ERRNO; + } + else + printf("creating %s\n",filename_try); + + for (i=zipfilenamearg+1;(i<argc) && (err==ZIP_OK);i++) + { + if (!((((*(argv[i]))=='-') || ((*(argv[i]))=='/')) && + ((argv[i][1]=='o') || (argv[i][1]=='O') || + (argv[i][1]=='a') || (argv[i][1]=='A') || + (argv[i][1]=='p') || (argv[i][1]=='P') || + ((argv[i][1]>='0') || (argv[i][1]<='9'))) && + (strlen(argv[i]) == 2))) + { + FILE * fin; + int size_read; + const char* filenameinzip = argv[i]; + const char *savefilenameinzip; + zip_fileinfo zi; + unsigned long crcFile=0; + int zip64 = 0; + + zi.tmz_date.tm_sec = zi.tmz_date.tm_min = zi.tmz_date.tm_hour = + zi.tmz_date.tm_mday = zi.tmz_date.tm_mon = zi.tmz_date.tm_year = 0; + zi.dosDate = 0; + zi.internal_fa = 0; + zi.external_fa = 0; + filetime(filenameinzip,&zi.tmz_date,&zi.dosDate); + +/* + err = zipOpenNewFileInZip(zf,filenameinzip,&zi, + NULL,0,NULL,0,NULL / * comment * /, + (opt_compress_level != 0) ? Z_DEFLATED : 0, + opt_compress_level); +*/ + if ((password != NULL) && (err==ZIP_OK)) + err = getFileCrc(filenameinzip,buf,size_buf,&crcFile); + + zip64 = isLargeFile(filenameinzip); + + /* The path name saved, should not include a leading slash. */ + /*if it did, windows/xp and dynazip couldn't read the zip file. */ + savefilenameinzip = filenameinzip; + while( savefilenameinzip[0] == '\\' || savefilenameinzip[0] == '/' ) + { + savefilenameinzip++; + } + + /*should the zip file contain any path at all?*/ + if( opt_exclude_path ) + { + const char *tmpptr; + const char *lastslash = 0; + for( tmpptr = savefilenameinzip; *tmpptr; tmpptr++) + { + if( *tmpptr == '\\' || *tmpptr == '/') + { + lastslash = tmpptr; + } + } + if( lastslash != NULL ) + { + savefilenameinzip = lastslash+1; // base filename follows last slash. + } + } + + /**/ + err = zipOpenNewFileInZip3_64(zf,savefilenameinzip,&zi, + NULL,0,NULL,0,NULL /* comment*/, + (opt_compress_level != 0) ? Z_DEFLATED : 0, + opt_compress_level,0, + /* -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY, */ + -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY, + password,crcFile, zip64); + + if (err != ZIP_OK) + printf("error in opening %s in zipfile\n",filenameinzip); + else + { + fin = FOPEN_FUNC(filenameinzip,"rb"); + if (fin==NULL) + { + err=ZIP_ERRNO; + printf("error in opening %s for reading\n",filenameinzip); + } + } + + if (err == ZIP_OK) + do + { + err = ZIP_OK; + size_read = (int)fread(buf,1,size_buf,fin); + if (size_read < size_buf) + if (feof(fin)==0) + { + printf("error in reading %s\n",filenameinzip); + err = ZIP_ERRNO; + } + + if (size_read>0) + { + err = zipWriteInFileInZip (zf,buf,size_read); + if (err<0) + { + printf("error in writing %s in the zipfile\n", + filenameinzip); + } + + } + } while ((err == ZIP_OK) && (size_read>0)); + + if (fin) + fclose(fin); + + if (err<0) + err=ZIP_ERRNO; + else + { + err = zipCloseFileInZip(zf); + if (err!=ZIP_OK) + printf("error in closing %s in the zipfile\n", + filenameinzip); + } + } + } + errclose = zipClose(zf,NULL); + if (errclose != ZIP_OK) + printf("error in closing %s\n",filename_try); + } + else + { + do_help(); + } + + free(buf); + return 0; +} diff --git a/third_party/zlib/contrib/minizip/mztools.c b/third_party/zlib/contrib/minizip/mztools.c new file mode 100644 index 00000000..8bf9cca3 --- /dev/null +++ b/third_party/zlib/contrib/minizip/mztools.c @@ -0,0 +1,291 @@ +/* + Additional tools for Minizip + Code: Xavier Roche '2004 + License: Same as ZLIB (www.gzip.org) +*/ + +/* Code */ +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include "third_party/zlib/zlib.h" +#include "unzip.h" + +#define READ_8(adr) ((unsigned char)*(adr)) +#define READ_16(adr) ( READ_8(adr) | (READ_8(adr+1) << 8) ) +#define READ_32(adr) ( READ_16(adr) | (READ_16((adr)+2) << 16) ) + +#define WRITE_8(buff, n) do { \ + *((unsigned char*)(buff)) = (unsigned char) ((n) & 0xff); \ +} while(0) +#define WRITE_16(buff, n) do { \ + WRITE_8((unsigned char*)(buff), n); \ + WRITE_8(((unsigned char*)(buff)) + 1, (n) >> 8); \ +} while(0) +#define WRITE_32(buff, n) do { \ + WRITE_16((unsigned char*)(buff), (n) & 0xffff); \ + WRITE_16((unsigned char*)(buff) + 2, (n) >> 16); \ +} while(0) + +extern int ZEXPORT unzRepair(file, fileOut, fileOutTmp, nRecovered, bytesRecovered) +const char* file; +const char* fileOut; +const char* fileOutTmp; +uLong* nRecovered; +uLong* bytesRecovered; +{ + int err = Z_OK; + FILE* fpZip = fopen(file, "rb"); + FILE* fpOut = fopen(fileOut, "wb"); + FILE* fpOutCD = fopen(fileOutTmp, "wb"); + if (fpZip != NULL && fpOut != NULL) { + int entries = 0; + uLong totalBytes = 0; + char header[30]; + char filename[1024]; + char extra[1024]; + int offset = 0; + int offsetCD = 0; + while ( fread(header, 1, 30, fpZip) == 30 ) { + int currentOffset = offset; + + /* File entry */ + if (READ_32(header) == 0x04034b50) { + unsigned int version = READ_16(header + 4); + unsigned int gpflag = READ_16(header + 6); + unsigned int method = READ_16(header + 8); + unsigned int filetime = READ_16(header + 10); + unsigned int filedate = READ_16(header + 12); + unsigned int crc = READ_32(header + 14); /* crc */ + unsigned int cpsize = READ_32(header + 18); /* compressed size */ + unsigned int uncpsize = READ_32(header + 22); /* uncompressed sz */ + unsigned int fnsize = READ_16(header + 26); /* file name length */ + unsigned int extsize = READ_16(header + 28); /* extra field length */ + filename[0] = extra[0] = '\0'; + + /* Header */ + if (fwrite(header, 1, 30, fpOut) == 30) { + offset += 30; + } else { + err = Z_ERRNO; + break; + } + + /* Filename */ + if (fnsize > 0) { + if (fnsize < sizeof(filename)) { + if (fread(filename, 1, fnsize, fpZip) == fnsize) { + if (fwrite(filename, 1, fnsize, fpOut) == fnsize) { + offset += fnsize; + } else { + err = Z_ERRNO; + break; + } + } else { + err = Z_ERRNO; + break; + } + } else { + err = Z_ERRNO; + break; + } + } else { + err = Z_STREAM_ERROR; + break; + } + + /* Extra field */ + if (extsize > 0) { + if (extsize < sizeof(extra)) { + if (fread(extra, 1, extsize, fpZip) == extsize) { + if (fwrite(extra, 1, extsize, fpOut) == extsize) { + offset += extsize; + } else { + err = Z_ERRNO; + break; + } + } else { + err = Z_ERRNO; + break; + } + } else { + err = Z_ERRNO; + break; + } + } + + /* Data */ + { + int dataSize = cpsize; + if (dataSize == 0) { + dataSize = uncpsize; + } + if (dataSize > 0) { + char* data = malloc(dataSize); + if (data != NULL) { + if ((int)fread(data, 1, dataSize, fpZip) == dataSize) { + if ((int)fwrite(data, 1, dataSize, fpOut) == dataSize) { + offset += dataSize; + totalBytes += dataSize; + } else { + err = Z_ERRNO; + } + } else { + err = Z_ERRNO; + } + free(data); + if (err != Z_OK) { + break; + } + } else { + err = Z_MEM_ERROR; + break; + } + } + } + + /* Central directory entry */ + { + char header[46]; + char* comment = ""; + int comsize = (int) strlen(comment); + WRITE_32(header, 0x02014b50); + WRITE_16(header + 4, version); + WRITE_16(header + 6, version); + WRITE_16(header + 8, gpflag); + WRITE_16(header + 10, method); + WRITE_16(header + 12, filetime); + WRITE_16(header + 14, filedate); + WRITE_32(header + 16, crc); + WRITE_32(header + 20, cpsize); + WRITE_32(header + 24, uncpsize); + WRITE_16(header + 28, fnsize); + WRITE_16(header + 30, extsize); + WRITE_16(header + 32, comsize); + WRITE_16(header + 34, 0); /* disk # */ + WRITE_16(header + 36, 0); /* int attrb */ + WRITE_32(header + 38, 0); /* ext attrb */ + WRITE_32(header + 42, currentOffset); + /* Header */ + if (fwrite(header, 1, 46, fpOutCD) == 46) { + offsetCD += 46; + + /* Filename */ + if (fnsize > 0) { + if (fwrite(filename, 1, fnsize, fpOutCD) == fnsize) { + offsetCD += fnsize; + } else { + err = Z_ERRNO; + break; + } + } else { + err = Z_STREAM_ERROR; + break; + } + + /* Extra field */ + if (extsize > 0) { + if (fwrite(extra, 1, extsize, fpOutCD) == extsize) { + offsetCD += extsize; + } else { + err = Z_ERRNO; + break; + } + } + + /* Comment field */ + if (comsize > 0) { + if ((int)fwrite(comment, 1, comsize, fpOutCD) == comsize) { + offsetCD += comsize; + } else { + err = Z_ERRNO; + break; + } + } + + + } else { + err = Z_ERRNO; + break; + } + } + + /* Success */ + entries++; + + } else { + break; + } + } + + /* Final central directory */ + { + int entriesZip = entries; + char header[22]; + char* comment = ""; // "ZIP File recovered by zlib/minizip/mztools"; + int comsize = (int) strlen(comment); + if (entriesZip > 0xffff) { + entriesZip = 0xffff; + } + WRITE_32(header, 0x06054b50); + WRITE_16(header + 4, 0); /* disk # */ + WRITE_16(header + 6, 0); /* disk # */ + WRITE_16(header + 8, entriesZip); /* hack */ + WRITE_16(header + 10, entriesZip); /* hack */ + WRITE_32(header + 12, offsetCD); /* size of CD */ + WRITE_32(header + 16, offset); /* offset to CD */ + WRITE_16(header + 20, comsize); /* comment */ + + /* Header */ + if (fwrite(header, 1, 22, fpOutCD) == 22) { + + /* Comment field */ + if (comsize > 0) { + if ((int)fwrite(comment, 1, comsize, fpOutCD) != comsize) { + err = Z_ERRNO; + } + } + + } else { + err = Z_ERRNO; + } + } + + /* Final merge (file + central directory) */ + fclose(fpOutCD); + if (err == Z_OK) { + fpOutCD = fopen(fileOutTmp, "rb"); + if (fpOutCD != NULL) { + int nRead; + char buffer[8192]; + while ( (nRead = (int)fread(buffer, 1, sizeof(buffer), fpOutCD)) > 0) { + if ((int)fwrite(buffer, 1, nRead, fpOut) != nRead) { + err = Z_ERRNO; + break; + } + } + fclose(fpOutCD); + } + } + + /* Close */ + fclose(fpZip); + fclose(fpOut); + + /* Wipe temporary file */ + (void)remove(fileOutTmp); + + /* Number of recovered entries */ + if (err == Z_OK) { + if (nRecovered != NULL) { + *nRecovered = entries; + } + if (bytesRecovered != NULL) { + *bytesRecovered = totalBytes; + } + } + } else { + err = Z_STREAM_ERROR; + } + return err; +} diff --git a/third_party/zlib/contrib/minizip/mztools.h b/third_party/zlib/contrib/minizip/mztools.h new file mode 100644 index 00000000..f295ffed --- /dev/null +++ b/third_party/zlib/contrib/minizip/mztools.h @@ -0,0 +1,37 @@ +/* + Additional tools for Minizip + Code: Xavier Roche '2004 + License: Same as ZLIB (www.gzip.org) +*/ + +#ifndef _zip_tools_H +#define _zip_tools_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef _ZLIB_H +#include "third_party/zlib/zlib.h" +#endif + +#include "unzip.h" + +/* Repair a ZIP file (missing central directory) + file: file to recover + fileOut: output file after recovery + fileOutTmp: temporary file name used for recovery +*/ +extern int ZEXPORT unzRepair(const char* file, + const char* fileOut, + const char* fileOutTmp, + uLong* nRecovered, + uLong* bytesRecovered); + + +#ifdef __cplusplus +} +#endif + + +#endif diff --git a/third_party/zlib/contrib/minizip/unzip.c b/third_party/zlib/contrib/minizip/unzip.c new file mode 100644 index 00000000..63e3ccb6 --- /dev/null +++ b/third_party/zlib/contrib/minizip/unzip.c @@ -0,0 +1,2117 @@ +/* unzip.c -- IO for uncompress .zip files using zlib + Version 1.1, February 14h, 2010 + part of the MiniZip project - ( http://www.winimage.com/zLibDll/minizip.html ) + + Copyright (C) 1998-2010 Gilles Vollant (minizip) ( http://www.winimage.com/zLibDll/minizip.html ) + + Modifications of Unzip for Zip64 + Copyright (C) 2007-2008 Even Rouault + + Modifications for Zip64 support on both zip and unzip + Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) + + For more info read MiniZip_info.txt + + + ------------------------------------------------------------------------------------ + Decryption code comes from crypt.c by Info-ZIP but has been greatly reduced in terms of + compatibility with older software. The following is from the original crypt.c. + Code woven in by Terry Thorsen 1/2003. + + Copyright (c) 1990-2000 Info-ZIP. All rights reserved. + + See the accompanying file LICENSE, version 2000-Apr-09 or later + (the contents of which are also included in zip.h) for terms of use. + If, for some reason, all these files are missing, the Info-ZIP license + also may be found at: ftp://ftp.info-zip.org/pub/infozip/license.html + + crypt.c (full version) by Info-ZIP. Last revised: [see crypt.h] + + The encryption/decryption parts of this source code (as opposed to the + non-echoing password parts) were originally written in Europe. The + whole source package can be freely distributed, including from the USA. + (Prior to January 2000, re-export from the US was a violation of US law.) + + This encryption code is a direct transcription of the algorithm from + Roger Schlafly, described by Phil Katz in the file appnote.txt. This + file (appnote.txt) is distributed with the PKZIP program (even in the + version without encryption capabilities). + + ------------------------------------------------------------------------------------ + + Changes in unzip.c + + 2007-2008 - Even Rouault - Addition of cpl_unzGetCurrentFileZStreamPos + 2007-2008 - Even Rouault - Decoration of symbol names unz* -> cpl_unz* + 2007-2008 - Even Rouault - Remove old C style function prototypes + 2007-2008 - Even Rouault - Add unzip support for ZIP64 + + Copyright (C) 2007-2008 Even Rouault + + + Oct-2009 - Mathias Svensson - Removed cpl_* from symbol names (Even Rouault added them but since this is now moved to a new project (minizip64) I renamed them again). + Oct-2009 - Mathias Svensson - Fixed problem if uncompressed size was > 4G and compressed size was <4G + should only read the compressed/uncompressed size from the Zip64 format if + the size from normal header was 0xFFFFFFFF + Oct-2009 - Mathias Svensson - Applied some bug fixes from paches recived from Gilles Vollant + Oct-2009 - Mathias Svensson - Applied support to unzip files with compression mathod BZIP2 (bzip2 lib is required) + Patch created by Daniel Borca + + Jan-2010 - back to unzip and minizip 1.0 name scheme, with compatibility layer + + Copyright (C) 1998 - 2010 Gilles Vollant, Even Rouault, Mathias Svensson + +*/ + + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "zlib.h" +#include "unzip.h" + +#ifdef STDC +# include <stddef.h> +# include <string.h> +# include <stdlib.h> +#endif +#ifdef NO_ERRNO_H + extern int errno; +#else +# include <errno.h> +#endif + + +#ifndef local +# define local static +#endif +/* compile with -Dlocal if your debugger can't find static symbols */ + + +#ifndef CASESENSITIVITYDEFAULT_NO +# if !defined(unix) && !defined(CASESENSITIVITYDEFAULT_YES) +# define CASESENSITIVITYDEFAULT_NO +# endif +#endif + + +#ifndef UNZ_BUFSIZE +#define UNZ_BUFSIZE (16384) +#endif + +#ifndef UNZ_MAXFILENAMEINZIP +#define UNZ_MAXFILENAMEINZIP (256) +#endif + +#ifndef ALLOC +# define ALLOC(size) (malloc(size)) +#endif +#ifndef TRYFREE +# define TRYFREE(p) {if (p) free(p);} +#endif + +#define SIZECENTRALDIRITEM (0x2e) +#define SIZEZIPLOCALHEADER (0x1e) + + +const char unz_copyright[] = + " unzip 1.01 Copyright 1998-2004 Gilles Vollant - http://www.winimage.com/zLibDll"; + +/* unz_file_info_interntal contain internal info about a file in zipfile*/ +typedef struct unz_file_info64_internal_s +{ + ZPOS64_T offset_curfile;/* relative offset of local header 8 bytes */ +} unz_file_info64_internal; + + +/* file_in_zip_read_info_s contain internal information about a file in zipfile, + when reading and decompress it */ +typedef struct +{ + char *read_buffer; /* internal buffer for compressed data */ + z_stream stream; /* zLib stream structure for inflate */ + +#ifdef HAVE_BZIP2 + bz_stream bstream; /* bzLib stream structure for bziped */ +#endif + + ZPOS64_T pos_in_zipfile; /* position in byte on the zipfile, for fseek*/ + uLong stream_initialised; /* flag set if stream structure is initialised*/ + + ZPOS64_T offset_local_extrafield;/* offset of the local extra field */ + uInt size_local_extrafield;/* size of the local extra field */ + ZPOS64_T pos_local_extrafield; /* position in the local extra field in read*/ + ZPOS64_T total_out_64; + + uLong crc32; /* crc32 of all data uncompressed */ + uLong crc32_wait; /* crc32 we must obtain after decompress all */ + ZPOS64_T rest_read_compressed; /* number of byte to be decompressed */ + ZPOS64_T rest_read_uncompressed;/*number of byte to be obtained after decomp*/ + zlib_filefunc64_32_def z_filefunc; + voidpf filestream; /* io structore of the zipfile */ + uLong compression_method; /* compression method (0==store) */ + ZPOS64_T byte_before_the_zipfile;/* byte before the zipfile, (>0 for sfx)*/ + int raw; +} file_in_zip64_read_info_s; + + +/* unz64_s contain internal information about the zipfile +*/ +typedef struct +{ + zlib_filefunc64_32_def z_filefunc; + int is64bitOpenFunction; + voidpf filestream; /* io structore of the zipfile */ + unz_global_info64 gi; /* public global information */ + ZPOS64_T byte_before_the_zipfile;/* byte before the zipfile, (>0 for sfx)*/ + ZPOS64_T num_file; /* number of the current file in the zipfile*/ + ZPOS64_T pos_in_central_dir; /* pos of the current file in the central dir*/ + ZPOS64_T current_file_ok; /* flag about the usability of the current file*/ + ZPOS64_T central_pos; /* position of the beginning of the central dir*/ + + ZPOS64_T size_central_dir; /* size of the central directory */ + ZPOS64_T offset_central_dir; /* offset of start of central directory with + respect to the starting disk number */ + + unz_file_info64 cur_file_info; /* public info about the current file in zip*/ + unz_file_info64_internal cur_file_info_internal; /* private info about it*/ + file_in_zip64_read_info_s* pfile_in_zip_read; /* structure about the current + file if we are decompressing it */ + int encrypted; + + int isZip64; + +# ifndef NOUNCRYPT + unsigned long keys[3]; /* keys defining the pseudo-random sequence */ + const z_crc_t* pcrc_32_tab; +# endif +} unz64_s; + + +#ifndef NOUNCRYPT +#include "crypt.h" +#endif + +/* =========================================================================== + Read a byte from a gz_stream; update next_in and avail_in. Return EOF + for end of file. + IN assertion: the stream s has been successfully opened for reading. +*/ + + +local int unz64local_getByte OF(( + const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream, + int *pi)); + +local int unz64local_getByte(const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, int *pi) +{ + unsigned char c; + int err = (int)ZREAD64(*pzlib_filefunc_def,filestream,&c,1); + if (err==1) + { + *pi = (int)c; + return UNZ_OK; + } + else + { + if (ZERROR64(*pzlib_filefunc_def,filestream)) + return UNZ_ERRNO; + else + return UNZ_EOF; + } +} + + +/* =========================================================================== + Reads a long in LSB order from the given gz_stream. Sets +*/ +local int unz64local_getShort OF(( + const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream, + uLong *pX)); + +local int unz64local_getShort (const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream, + uLong *pX) +{ + uLong x ; + int i = 0; + int err; + + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x = (uLong)i; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((uLong)i)<<8; + + if (err==UNZ_OK) + *pX = x; + else + *pX = 0; + return err; +} + +local int unz64local_getLong OF(( + const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream, + uLong *pX)); + +local int unz64local_getLong (const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream, + uLong *pX) +{ + uLong x ; + int i = 0; + int err; + + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x = (uLong)i; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((uLong)i)<<8; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((uLong)i)<<16; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((uLong)i)<<24; + + if (err==UNZ_OK) + *pX = x; + else + *pX = 0; + return err; +} + +local int unz64local_getLong64 OF(( + const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream, + ZPOS64_T *pX)); + + +local int unz64local_getLong64 (const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream, + ZPOS64_T *pX) +{ + ZPOS64_T x ; + int i = 0; + int err; + + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x = (ZPOS64_T)i; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((ZPOS64_T)i)<<8; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((ZPOS64_T)i)<<16; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((ZPOS64_T)i)<<24; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((ZPOS64_T)i)<<32; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((ZPOS64_T)i)<<40; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((ZPOS64_T)i)<<48; + + if (err==UNZ_OK) + err = unz64local_getByte(pzlib_filefunc_def,filestream,&i); + x |= ((ZPOS64_T)i)<<56; + + if (err==UNZ_OK) + *pX = x; + else + *pX = 0; + return err; +} + +/* My own strcmpi / strcasecmp */ +local int strcmpcasenosensitive_internal (const char* fileName1, const char* fileName2) +{ + for (;;) + { + char c1=*(fileName1++); + char c2=*(fileName2++); + if ((c1>='a') && (c1<='z')) + c1 -= 0x20; + if ((c2>='a') && (c2<='z')) + c2 -= 0x20; + if (c1=='\0') + return ((c2=='\0') ? 0 : -1); + if (c2=='\0') + return 1; + if (c1<c2) + return -1; + if (c1>c2) + return 1; + } +} + + +#ifdef CASESENSITIVITYDEFAULT_NO +#define CASESENSITIVITYDEFAULTVALUE 2 +#else +#define CASESENSITIVITYDEFAULTVALUE 1 +#endif + +#ifndef STRCMPCASENOSENTIVEFUNCTION +#define STRCMPCASENOSENTIVEFUNCTION strcmpcasenosensitive_internal +#endif + +/* + Compare two filename (fileName1,fileName2). + If iCaseSenisivity = 1, comparision is case sensitivity (like strcmp) + If iCaseSenisivity = 2, comparision is not case sensitivity (like strcmpi + or strcasecmp) + If iCaseSenisivity = 0, case sensitivity is defaut of your operating system + (like 1 on Unix, 2 on Windows) + +*/ +extern int ZEXPORT unzStringFileNameCompare (const char* fileName1, + const char* fileName2, + int iCaseSensitivity) + +{ + if (iCaseSensitivity==0) + iCaseSensitivity=CASESENSITIVITYDEFAULTVALUE; + + if (iCaseSensitivity==1) + return strcmp(fileName1,fileName2); + + return STRCMPCASENOSENTIVEFUNCTION(fileName1,fileName2); +} + +#ifndef BUFREADCOMMENT +#define BUFREADCOMMENT (0x400) +#endif + +/* + Locate the Central directory of a zipfile (at the end, just before + the global comment) +*/ +local ZPOS64_T unz64local_SearchCentralDir OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream)); +local ZPOS64_T unz64local_SearchCentralDir(const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream) +{ + unsigned char* buf; + ZPOS64_T uSizeFile; + ZPOS64_T uBackRead; + ZPOS64_T uMaxBack=0xffff; /* maximum size of global comment */ + ZPOS64_T uPosFound=0; + + if (ZSEEK64(*pzlib_filefunc_def,filestream,0,ZLIB_FILEFUNC_SEEK_END) != 0) + return 0; + + + uSizeFile = ZTELL64(*pzlib_filefunc_def,filestream); + + if (uMaxBack>uSizeFile) + uMaxBack = uSizeFile; + + buf = (unsigned char*)ALLOC(BUFREADCOMMENT+4); + if (buf==NULL) + return 0; + + uBackRead = 4; + while (uBackRead<uMaxBack) + { + uLong uReadSize; + ZPOS64_T uReadPos ; + int i; + if (uBackRead+BUFREADCOMMENT>uMaxBack) + uBackRead = uMaxBack; + else + uBackRead+=BUFREADCOMMENT; + uReadPos = uSizeFile-uBackRead ; + + uReadSize = ((BUFREADCOMMENT+4) < (uSizeFile-uReadPos)) ? + (BUFREADCOMMENT+4) : (uLong)(uSizeFile-uReadPos); + if (ZSEEK64(*pzlib_filefunc_def,filestream,uReadPos,ZLIB_FILEFUNC_SEEK_SET)!=0) + break; + + if (ZREAD64(*pzlib_filefunc_def,filestream,buf,uReadSize)!=uReadSize) + break; + + for (i=(int)uReadSize-3; (i--)>0;) + if (((*(buf+i))==0x50) && ((*(buf+i+1))==0x4b) && + ((*(buf+i+2))==0x05) && ((*(buf+i+3))==0x06)) + { + uPosFound = uReadPos+i; + break; + } + + if (uPosFound!=0) + break; + } + TRYFREE(buf); + return uPosFound; +} + + +/* + Locate the Central directory 64 of a zipfile (at the end, just before + the global comment) +*/ +local ZPOS64_T unz64local_SearchCentralDir64 OF(( + const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream)); + +local ZPOS64_T unz64local_SearchCentralDir64(const zlib_filefunc64_32_def* pzlib_filefunc_def, + voidpf filestream) +{ + unsigned char* buf; + ZPOS64_T uSizeFile; + ZPOS64_T uBackRead; + ZPOS64_T uMaxBack=0xffff; /* maximum size of global comment */ + ZPOS64_T uPosFound=0; + uLong uL; + ZPOS64_T relativeOffset; + + if (ZSEEK64(*pzlib_filefunc_def,filestream,0,ZLIB_FILEFUNC_SEEK_END) != 0) + return 0; + + + uSizeFile = ZTELL64(*pzlib_filefunc_def,filestream); + + if (uMaxBack>uSizeFile) + uMaxBack = uSizeFile; + + buf = (unsigned char*)ALLOC(BUFREADCOMMENT+4); + if (buf==NULL) + return 0; + + uBackRead = 4; + while (uBackRead<uMaxBack) + { + uLong uReadSize; + ZPOS64_T uReadPos; + int i; + if (uBackRead+BUFREADCOMMENT>uMaxBack) + uBackRead = uMaxBack; + else + uBackRead+=BUFREADCOMMENT; + uReadPos = uSizeFile-uBackRead ; + + uReadSize = ((BUFREADCOMMENT+4) < (uSizeFile-uReadPos)) ? + (BUFREADCOMMENT+4) : (uLong)(uSizeFile-uReadPos); + if (ZSEEK64(*pzlib_filefunc_def,filestream,uReadPos,ZLIB_FILEFUNC_SEEK_SET)!=0) + break; + + if (ZREAD64(*pzlib_filefunc_def,filestream,buf,uReadSize)!=uReadSize) + break; + + for (i=(int)uReadSize-3; (i--)>0;) + if (((*(buf+i))==0x50) && ((*(buf+i+1))==0x4b) && + ((*(buf+i+2))==0x06) && ((*(buf+i+3))==0x07)) + { + uPosFound = uReadPos+i; + break; + } + + if (uPosFound!=0) + break; + } + TRYFREE(buf); + if (uPosFound == 0) + return 0; + + /* Zip64 end of central directory locator */ + if (ZSEEK64(*pzlib_filefunc_def,filestream, uPosFound,ZLIB_FILEFUNC_SEEK_SET)!=0) + return 0; + + /* the signature, already checked */ + if (unz64local_getLong(pzlib_filefunc_def,filestream,&uL)!=UNZ_OK) + return 0; + + /* number of the disk with the start of the zip64 end of central directory */ + if (unz64local_getLong(pzlib_filefunc_def,filestream,&uL)!=UNZ_OK) + return 0; + if (uL != 0) + return 0; + + /* relative offset of the zip64 end of central directory record */ + if (unz64local_getLong64(pzlib_filefunc_def,filestream,&relativeOffset)!=UNZ_OK) + return 0; + + /* total number of disks */ + if (unz64local_getLong(pzlib_filefunc_def,filestream,&uL)!=UNZ_OK) + return 0; + if (uL != 1) + return 0; + + /* Goto end of central directory record */ + if (ZSEEK64(*pzlib_filefunc_def,filestream, relativeOffset,ZLIB_FILEFUNC_SEEK_SET)!=0) + return 0; + + /* the signature */ + if (unz64local_getLong(pzlib_filefunc_def,filestream,&uL)!=UNZ_OK) + return 0; + + if (uL != 0x06064b50) + return 0; + + return relativeOffset; +} + +/* + Open a Zip file. path contain the full pathname (by example, + on a Windows NT computer "c:\\test\\zlib114.zip" or on an Unix computer + "zlib/zlib114.zip". + If the zipfile cannot be opened (file doesn't exist or in not valid), the + return value is NULL. + Else, the return value is a unzFile Handle, usable with other function + of this unzip package. +*/ +local unzFile unzOpenInternal (const void *path, + zlib_filefunc64_32_def* pzlib_filefunc64_32_def, + int is64bitOpenFunction) +{ + unz64_s us; + unz64_s *s; + ZPOS64_T central_pos; + uLong uL; + + uLong number_disk; /* number of the current dist, used for + spaning ZIP, unsupported, always 0*/ + uLong number_disk_with_CD; /* number the the disk with central dir, used + for spaning ZIP, unsupported, always 0*/ + ZPOS64_T number_entry_CD; /* total number of entries in + the central dir + (same than number_entry on nospan) */ + + int err=UNZ_OK; + + if (unz_copyright[0]!=' ') + return NULL; + + us.z_filefunc.zseek32_file = NULL; + us.z_filefunc.ztell32_file = NULL; + if (pzlib_filefunc64_32_def==NULL) + fill_fopen64_filefunc(&us.z_filefunc.zfile_func64); + else + us.z_filefunc = *pzlib_filefunc64_32_def; + us.is64bitOpenFunction = is64bitOpenFunction; + + + + us.filestream = ZOPEN64(us.z_filefunc, + path, + ZLIB_FILEFUNC_MODE_READ | + ZLIB_FILEFUNC_MODE_EXISTING); + if (us.filestream==NULL) + return NULL; + + central_pos = unz64local_SearchCentralDir64(&us.z_filefunc,us.filestream); + if (central_pos) + { + uLong uS; + ZPOS64_T uL64; + + us.isZip64 = 1; + + if (ZSEEK64(us.z_filefunc, us.filestream, + central_pos,ZLIB_FILEFUNC_SEEK_SET)!=0) + err=UNZ_ERRNO; + + /* the signature, already checked */ + if (unz64local_getLong(&us.z_filefunc, us.filestream,&uL)!=UNZ_OK) + err=UNZ_ERRNO; + + /* size of zip64 end of central directory record */ + if (unz64local_getLong64(&us.z_filefunc, us.filestream,&uL64)!=UNZ_OK) + err=UNZ_ERRNO; + + /* version made by */ + if (unz64local_getShort(&us.z_filefunc, us.filestream,&uS)!=UNZ_OK) + err=UNZ_ERRNO; + + /* version needed to extract */ + if (unz64local_getShort(&us.z_filefunc, us.filestream,&uS)!=UNZ_OK) + err=UNZ_ERRNO; + + /* number of this disk */ + if (unz64local_getLong(&us.z_filefunc, us.filestream,&number_disk)!=UNZ_OK) + err=UNZ_ERRNO; + + /* number of the disk with the start of the central directory */ + if (unz64local_getLong(&us.z_filefunc, us.filestream,&number_disk_with_CD)!=UNZ_OK) + err=UNZ_ERRNO; + + /* total number of entries in the central directory on this disk */ + if (unz64local_getLong64(&us.z_filefunc, us.filestream,&us.gi.number_entry)!=UNZ_OK) + err=UNZ_ERRNO; + + /* total number of entries in the central directory */ + if (unz64local_getLong64(&us.z_filefunc, us.filestream,&number_entry_CD)!=UNZ_OK) + err=UNZ_ERRNO; + + if ((number_entry_CD!=us.gi.number_entry) || + (number_disk_with_CD!=0) || + (number_disk!=0)) + err=UNZ_BADZIPFILE; + + /* size of the central directory */ + if (unz64local_getLong64(&us.z_filefunc, us.filestream,&us.size_central_dir)!=UNZ_OK) + err=UNZ_ERRNO; + + /* offset of start of central directory with respect to the + starting disk number */ + if (unz64local_getLong64(&us.z_filefunc, us.filestream,&us.offset_central_dir)!=UNZ_OK) + err=UNZ_ERRNO; + + us.gi.size_comment = 0; + } + else + { + central_pos = unz64local_SearchCentralDir(&us.z_filefunc,us.filestream); + if (central_pos==0) + err=UNZ_ERRNO; + + us.isZip64 = 0; + + if (ZSEEK64(us.z_filefunc, us.filestream, + central_pos,ZLIB_FILEFUNC_SEEK_SET)!=0) + err=UNZ_ERRNO; + + /* the signature, already checked */ + if (unz64local_getLong(&us.z_filefunc, us.filestream,&uL)!=UNZ_OK) + err=UNZ_ERRNO; + + /* number of this disk */ + if (unz64local_getShort(&us.z_filefunc, us.filestream,&number_disk)!=UNZ_OK) + err=UNZ_ERRNO; + + /* number of the disk with the start of the central directory */ + if (unz64local_getShort(&us.z_filefunc, us.filestream,&number_disk_with_CD)!=UNZ_OK) + err=UNZ_ERRNO; + + /* total number of entries in the central dir on this disk */ + if (unz64local_getShort(&us.z_filefunc, us.filestream,&uL)!=UNZ_OK) + err=UNZ_ERRNO; + us.gi.number_entry = uL; + + /* total number of entries in the central dir */ + if (unz64local_getShort(&us.z_filefunc, us.filestream,&uL)!=UNZ_OK) + err=UNZ_ERRNO; + number_entry_CD = uL; + + if ((number_entry_CD!=us.gi.number_entry) || + (number_disk_with_CD!=0) || + (number_disk!=0)) + err=UNZ_BADZIPFILE; + + /* size of the central directory */ + if (unz64local_getLong(&us.z_filefunc, us.filestream,&uL)!=UNZ_OK) + err=UNZ_ERRNO; + us.size_central_dir = uL; + + /* offset of start of central directory with respect to the + starting disk number */ + if (unz64local_getLong(&us.z_filefunc, us.filestream,&uL)!=UNZ_OK) + err=UNZ_ERRNO; + us.offset_central_dir = uL; + + /* zipfile comment length */ + if (unz64local_getShort(&us.z_filefunc, us.filestream,&us.gi.size_comment)!=UNZ_OK) + err=UNZ_ERRNO; + } + + if ((central_pos<us.offset_central_dir+us.size_central_dir) && + (err==UNZ_OK)) + err=UNZ_BADZIPFILE; + + if (err!=UNZ_OK) + { + ZCLOSE64(us.z_filefunc, us.filestream); + return NULL; + } + + us.byte_before_the_zipfile = central_pos - + (us.offset_central_dir+us.size_central_dir); + us.central_pos = central_pos; + us.pfile_in_zip_read = NULL; + us.encrypted = 0; + + + s=(unz64_s*)ALLOC(sizeof(unz64_s)); + if( s != NULL) + { + *s=us; + unzGoToFirstFile((unzFile)s); + } + return (unzFile)s; +} + + +extern unzFile ZEXPORT unzOpen2 (const char *path, + zlib_filefunc_def* pzlib_filefunc32_def) +{ + if (pzlib_filefunc32_def != NULL) + { + zlib_filefunc64_32_def zlib_filefunc64_32_def_fill; + fill_zlib_filefunc64_32_def_from_filefunc32(&zlib_filefunc64_32_def_fill,pzlib_filefunc32_def); + return unzOpenInternal(path, &zlib_filefunc64_32_def_fill, 0); + } + else + return unzOpenInternal(path, NULL, 0); +} + +extern unzFile ZEXPORT unzOpen2_64 (const void *path, + zlib_filefunc64_def* pzlib_filefunc_def) +{ + if (pzlib_filefunc_def != NULL) + { + zlib_filefunc64_32_def zlib_filefunc64_32_def_fill; + zlib_filefunc64_32_def_fill.zfile_func64 = *pzlib_filefunc_def; + zlib_filefunc64_32_def_fill.ztell32_file = NULL; + zlib_filefunc64_32_def_fill.zseek32_file = NULL; + return unzOpenInternal(path, &zlib_filefunc64_32_def_fill, 1); + } + else + return unzOpenInternal(path, NULL, 1); +} + +extern unzFile ZEXPORT unzOpen (const char *path) +{ + return unzOpenInternal(path, NULL, 0); +} + +extern unzFile ZEXPORT unzOpen64 (const void *path) +{ + return unzOpenInternal(path, NULL, 1); +} + +/* + Close a ZipFile opened with unzOpen. + If there is files inside the .Zip opened with unzOpenCurrentFile (see later), + these files MUST be closed with unzCloseCurrentFile before call unzClose. + return UNZ_OK if there is no problem. */ +extern int ZEXPORT unzClose (unzFile file) +{ + unz64_s* s; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + + if (s->pfile_in_zip_read!=NULL) + unzCloseCurrentFile(file); + + ZCLOSE64(s->z_filefunc, s->filestream); + TRYFREE(s); + return UNZ_OK; +} + + +/* + Write info about the ZipFile in the *pglobal_info structure. + No preparation of the structure is needed + return UNZ_OK if there is no problem. */ +extern int ZEXPORT unzGetGlobalInfo64 (unzFile file, unz_global_info64* pglobal_info) +{ + unz64_s* s; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + *pglobal_info=s->gi; + return UNZ_OK; +} + +extern int ZEXPORT unzGetGlobalInfo (unzFile file, unz_global_info* pglobal_info32) +{ + unz64_s* s; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + /* to do : check if number_entry is not truncated */ + pglobal_info32->number_entry = (uLong)s->gi.number_entry; + pglobal_info32->size_comment = s->gi.size_comment; + return UNZ_OK; +} +/* + Translate date/time from Dos format to tm_unz (readable more easilty) +*/ +local void unz64local_DosDateToTmuDate (ZPOS64_T ulDosDate, tm_unz* ptm) +{ + ZPOS64_T uDate; + uDate = (ZPOS64_T)(ulDosDate>>16); + ptm->tm_mday = (uInt)(uDate&0x1f) ; + ptm->tm_mon = (uInt)((((uDate)&0x1E0)/0x20)-1) ; + ptm->tm_year = (uInt)(((uDate&0x0FE00)/0x0200)+1980) ; + + ptm->tm_hour = (uInt) ((ulDosDate &0xF800)/0x800); + ptm->tm_min = (uInt) ((ulDosDate&0x7E0)/0x20) ; + ptm->tm_sec = (uInt) (2*(ulDosDate&0x1f)) ; +} + +/* + Get Info about the current file in the zipfile, with internal only info +*/ +local int unz64local_GetCurrentFileInfoInternal OF((unzFile file, + unz_file_info64 *pfile_info, + unz_file_info64_internal + *pfile_info_internal, + char *szFileName, + uLong fileNameBufferSize, + void *extraField, + uLong extraFieldBufferSize, + char *szComment, + uLong commentBufferSize)); + +local int unz64local_GetCurrentFileInfoInternal (unzFile file, + unz_file_info64 *pfile_info, + unz_file_info64_internal + *pfile_info_internal, + char *szFileName, + uLong fileNameBufferSize, + void *extraField, + uLong extraFieldBufferSize, + char *szComment, + uLong commentBufferSize) +{ + unz64_s* s; + unz_file_info64 file_info; + unz_file_info64_internal file_info_internal; + int err=UNZ_OK; + uLong uMagic; + long lSeek=0; + uLong uL; + + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + if (ZSEEK64(s->z_filefunc, s->filestream, + s->pos_in_central_dir+s->byte_before_the_zipfile, + ZLIB_FILEFUNC_SEEK_SET)!=0) + err=UNZ_ERRNO; + + + /* we check the magic */ + if (err==UNZ_OK) + { + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uMagic) != UNZ_OK) + err=UNZ_ERRNO; + else if (uMagic!=0x02014b50) + err=UNZ_BADZIPFILE; + } + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.version) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.version_needed) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.flag) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.compression_method) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&file_info.dosDate) != UNZ_OK) + err=UNZ_ERRNO; + + unz64local_DosDateToTmuDate(file_info.dosDate,&file_info.tmu_date); + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&file_info.crc) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uL) != UNZ_OK) + err=UNZ_ERRNO; + file_info.compressed_size = uL; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uL) != UNZ_OK) + err=UNZ_ERRNO; + file_info.uncompressed_size = uL; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.size_filename) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.size_file_extra) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.size_file_comment) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.disk_num_start) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&file_info.internal_fa) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&file_info.external_fa) != UNZ_OK) + err=UNZ_ERRNO; + + // relative offset of local header + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uL) != UNZ_OK) + err=UNZ_ERRNO; + file_info_internal.offset_curfile = uL; + + lSeek+=file_info.size_filename; + if ((err==UNZ_OK) && (szFileName!=NULL)) + { + uLong uSizeRead ; + if (file_info.size_filename<fileNameBufferSize) + { + *(szFileName+file_info.size_filename)='\0'; + uSizeRead = file_info.size_filename; + } + else + uSizeRead = fileNameBufferSize; + + if ((file_info.size_filename>0) && (fileNameBufferSize>0)) + if (ZREAD64(s->z_filefunc, s->filestream,szFileName,uSizeRead)!=uSizeRead) + err=UNZ_ERRNO; + lSeek -= uSizeRead; + } + + // Read extrafield + if ((err==UNZ_OK) && (extraField!=NULL)) + { + ZPOS64_T uSizeRead ; + if (file_info.size_file_extra<extraFieldBufferSize) + uSizeRead = file_info.size_file_extra; + else + uSizeRead = extraFieldBufferSize; + + if (lSeek!=0) + { + if (ZSEEK64(s->z_filefunc, s->filestream,lSeek,ZLIB_FILEFUNC_SEEK_CUR)==0) + lSeek=0; + else + err=UNZ_ERRNO; + } + + if ((file_info.size_file_extra>0) && (extraFieldBufferSize>0)) + if (ZREAD64(s->z_filefunc, s->filestream,extraField,(uLong)uSizeRead)!=uSizeRead) + err=UNZ_ERRNO; + + lSeek += file_info.size_file_extra - (uLong)uSizeRead; + } + else + lSeek += file_info.size_file_extra; + + + if ((err==UNZ_OK) && (file_info.size_file_extra != 0)) + { + uLong acc = 0; + + // since lSeek now points to after the extra field we need to move back + lSeek -= file_info.size_file_extra; + + if (lSeek!=0) + { + if (ZSEEK64(s->z_filefunc, s->filestream,lSeek,ZLIB_FILEFUNC_SEEK_CUR)==0) + lSeek=0; + else + err=UNZ_ERRNO; + } + + while(acc < file_info.size_file_extra) + { + uLong headerId; + uLong dataSize; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&headerId) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&dataSize) != UNZ_OK) + err=UNZ_ERRNO; + + /* ZIP64 extra fields */ + if (headerId == 0x0001) + { + uLong uL; + + if(file_info.uncompressed_size == MAXU32) + { + if (unz64local_getLong64(&s->z_filefunc, s->filestream,&file_info.uncompressed_size) != UNZ_OK) + err=UNZ_ERRNO; + } + + if(file_info.compressed_size == MAXU32) + { + if (unz64local_getLong64(&s->z_filefunc, s->filestream,&file_info.compressed_size) != UNZ_OK) + err=UNZ_ERRNO; + } + + if(file_info_internal.offset_curfile == MAXU32) + { + /* Relative Header offset */ + if (unz64local_getLong64(&s->z_filefunc, s->filestream,&file_info_internal.offset_curfile) != UNZ_OK) + err=UNZ_ERRNO; + } + + if(file_info.disk_num_start == MAXU32) + { + /* Disk Start Number */ + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uL) != UNZ_OK) + err=UNZ_ERRNO; + } + + } + else + { + if (ZSEEK64(s->z_filefunc, s->filestream,dataSize,ZLIB_FILEFUNC_SEEK_CUR)!=0) + err=UNZ_ERRNO; + } + + acc += 2 + 2 + dataSize; + } + } + + if ((err==UNZ_OK) && (szComment!=NULL)) + { + uLong uSizeRead ; + if (file_info.size_file_comment<commentBufferSize) + { + *(szComment+file_info.size_file_comment)='\0'; + uSizeRead = file_info.size_file_comment; + } + else + uSizeRead = commentBufferSize; + + if (lSeek!=0) + { + if (ZSEEK64(s->z_filefunc, s->filestream,lSeek,ZLIB_FILEFUNC_SEEK_CUR)==0) + lSeek=0; + else + err=UNZ_ERRNO; + } + + if ((file_info.size_file_comment>0) && (commentBufferSize>0)) + if (ZREAD64(s->z_filefunc, s->filestream,szComment,uSizeRead)!=uSizeRead) + err=UNZ_ERRNO; + lSeek+=file_info.size_file_comment - uSizeRead; + } + else + lSeek+=file_info.size_file_comment; + + + if ((err==UNZ_OK) && (pfile_info!=NULL)) + *pfile_info=file_info; + + if ((err==UNZ_OK) && (pfile_info_internal!=NULL)) + *pfile_info_internal=file_info_internal; + + return err; +} + + + +/* + Write info about the ZipFile in the *pglobal_info structure. + No preparation of the structure is needed + return UNZ_OK if there is no problem. +*/ +extern int ZEXPORT unzGetCurrentFileInfo64 (unzFile file, + unz_file_info64 * pfile_info, + char * szFileName, uLong fileNameBufferSize, + void *extraField, uLong extraFieldBufferSize, + char* szComment, uLong commentBufferSize) +{ + return unz64local_GetCurrentFileInfoInternal(file,pfile_info,NULL, + szFileName,fileNameBufferSize, + extraField,extraFieldBufferSize, + szComment,commentBufferSize); +} + +extern int ZEXPORT unzGetCurrentFileInfo (unzFile file, + unz_file_info * pfile_info, + char * szFileName, uLong fileNameBufferSize, + void *extraField, uLong extraFieldBufferSize, + char* szComment, uLong commentBufferSize) +{ + int err; + unz_file_info64 file_info64; + err = unz64local_GetCurrentFileInfoInternal(file,&file_info64,NULL, + szFileName,fileNameBufferSize, + extraField,extraFieldBufferSize, + szComment,commentBufferSize); + if ((err==UNZ_OK) && (pfile_info != NULL)) + { + pfile_info->version = file_info64.version; + pfile_info->version_needed = file_info64.version_needed; + pfile_info->flag = file_info64.flag; + pfile_info->compression_method = file_info64.compression_method; + pfile_info->dosDate = file_info64.dosDate; + pfile_info->crc = file_info64.crc; + + pfile_info->size_filename = file_info64.size_filename; + pfile_info->size_file_extra = file_info64.size_file_extra; + pfile_info->size_file_comment = file_info64.size_file_comment; + + pfile_info->disk_num_start = file_info64.disk_num_start; + pfile_info->internal_fa = file_info64.internal_fa; + pfile_info->external_fa = file_info64.external_fa; + + pfile_info->tmu_date = file_info64.tmu_date, + + + pfile_info->compressed_size = (uLong)file_info64.compressed_size; + pfile_info->uncompressed_size = (uLong)file_info64.uncompressed_size; + + } + return err; +} +/* + Set the current file of the zipfile to the first file. + return UNZ_OK if there is no problem +*/ +extern int ZEXPORT unzGoToFirstFile (unzFile file) +{ + int err=UNZ_OK; + unz64_s* s; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + s->pos_in_central_dir=s->offset_central_dir; + s->num_file=0; + err=unz64local_GetCurrentFileInfoInternal(file,&s->cur_file_info, + &s->cur_file_info_internal, + NULL,0,NULL,0,NULL,0); + s->current_file_ok = (err == UNZ_OK); + return err; +} + +/* + Set the current file of the zipfile to the next file. + return UNZ_OK if there is no problem + return UNZ_END_OF_LIST_OF_FILE if the actual file was the latest. +*/ +extern int ZEXPORT unzGoToNextFile (unzFile file) +{ + unz64_s* s; + int err; + + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + if (!s->current_file_ok) + return UNZ_END_OF_LIST_OF_FILE; + if (s->gi.number_entry != 0xffff) /* 2^16 files overflow hack */ + if (s->num_file+1==s->gi.number_entry) + return UNZ_END_OF_LIST_OF_FILE; + + s->pos_in_central_dir += SIZECENTRALDIRITEM + s->cur_file_info.size_filename + + s->cur_file_info.size_file_extra + s->cur_file_info.size_file_comment ; + s->num_file++; + err = unz64local_GetCurrentFileInfoInternal(file,&s->cur_file_info, + &s->cur_file_info_internal, + NULL,0,NULL,0,NULL,0); + s->current_file_ok = (err == UNZ_OK); + return err; +} + + +/* + Try locate the file szFileName in the zipfile. + For the iCaseSensitivity signification, see unzStringFileNameCompare + + return value : + UNZ_OK if the file is found. It becomes the current file. + UNZ_END_OF_LIST_OF_FILE if the file is not found +*/ +extern int ZEXPORT unzLocateFile (unzFile file, const char *szFileName, int iCaseSensitivity) +{ + unz64_s* s; + int err; + + /* We remember the 'current' position in the file so that we can jump + * back there if we fail. + */ + unz_file_info64 cur_file_infoSaved; + unz_file_info64_internal cur_file_info_internalSaved; + ZPOS64_T num_fileSaved; + ZPOS64_T pos_in_central_dirSaved; + + + if (file==NULL) + return UNZ_PARAMERROR; + + if (strlen(szFileName)>=UNZ_MAXFILENAMEINZIP) + return UNZ_PARAMERROR; + + s=(unz64_s*)file; + if (!s->current_file_ok) + return UNZ_END_OF_LIST_OF_FILE; + + /* Save the current state */ + num_fileSaved = s->num_file; + pos_in_central_dirSaved = s->pos_in_central_dir; + cur_file_infoSaved = s->cur_file_info; + cur_file_info_internalSaved = s->cur_file_info_internal; + + err = unzGoToFirstFile(file); + + while (err == UNZ_OK) + { + char szCurrentFileName[UNZ_MAXFILENAMEINZIP+1]; + err = unzGetCurrentFileInfo64(file,NULL, + szCurrentFileName,sizeof(szCurrentFileName)-1, + NULL,0,NULL,0); + if (err == UNZ_OK) + { + if (unzStringFileNameCompare(szCurrentFileName, + szFileName,iCaseSensitivity)==0) + return UNZ_OK; + err = unzGoToNextFile(file); + } + } + + /* We failed, so restore the state of the 'current file' to where we + * were. + */ + s->num_file = num_fileSaved ; + s->pos_in_central_dir = pos_in_central_dirSaved ; + s->cur_file_info = cur_file_infoSaved; + s->cur_file_info_internal = cur_file_info_internalSaved; + return err; +} + + +/* +/////////////////////////////////////////// +// Contributed by Ryan Haksi (mailto://cryogen@infoserve.net) +// I need random access +// +// Further optimization could be realized by adding an ability +// to cache the directory in memory. The goal being a single +// comprehensive file read to put the file I need in a memory. +*/ + +/* +typedef struct unz_file_pos_s +{ + ZPOS64_T pos_in_zip_directory; // offset in file + ZPOS64_T num_of_file; // # of file +} unz_file_pos; +*/ + +extern int ZEXPORT unzGetFilePos64(unzFile file, unz64_file_pos* file_pos) +{ + unz64_s* s; + + if (file==NULL || file_pos==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + if (!s->current_file_ok) + return UNZ_END_OF_LIST_OF_FILE; + + file_pos->pos_in_zip_directory = s->pos_in_central_dir; + file_pos->num_of_file = s->num_file; + + return UNZ_OK; +} + +extern int ZEXPORT unzGetFilePos( + unzFile file, + unz_file_pos* file_pos) +{ + unz64_file_pos file_pos64; + int err = unzGetFilePos64(file,&file_pos64); + if (err==UNZ_OK) + { + file_pos->pos_in_zip_directory = (uLong)file_pos64.pos_in_zip_directory; + file_pos->num_of_file = (uLong)file_pos64.num_of_file; + } + return err; +} + +extern int ZEXPORT unzGoToFilePos64(unzFile file, const unz64_file_pos* file_pos) +{ + unz64_s* s; + int err; + + if (file==NULL || file_pos==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + + /* jump to the right spot */ + s->pos_in_central_dir = file_pos->pos_in_zip_directory; + s->num_file = file_pos->num_of_file; + + /* set the current file */ + err = unz64local_GetCurrentFileInfoInternal(file,&s->cur_file_info, + &s->cur_file_info_internal, + NULL,0,NULL,0,NULL,0); + /* return results */ + s->current_file_ok = (err == UNZ_OK); + return err; +} + +extern int ZEXPORT unzGoToFilePos( + unzFile file, + unz_file_pos* file_pos) +{ + unz64_file_pos file_pos64; + if (file_pos == NULL) + return UNZ_PARAMERROR; + + file_pos64.pos_in_zip_directory = file_pos->pos_in_zip_directory; + file_pos64.num_of_file = file_pos->num_of_file; + return unzGoToFilePos64(file,&file_pos64); +} + +/* +// Unzip Helper Functions - should be here? +/////////////////////////////////////////// +*/ + +/* + Read the local header of the current zipfile + Check the coherency of the local header and info in the end of central + directory about this file + store in *piSizeVar the size of extra info in local header + (filename and size of extra field data) +*/ +local int unz64local_CheckCurrentFileCoherencyHeader (unz64_s* s, uInt* piSizeVar, + ZPOS64_T * poffset_local_extrafield, + uInt * psize_local_extrafield) +{ + uLong uMagic,uData,uFlags; + uLong size_filename; + uLong size_extra_field; + int err=UNZ_OK; + + *piSizeVar = 0; + *poffset_local_extrafield = 0; + *psize_local_extrafield = 0; + + if (ZSEEK64(s->z_filefunc, s->filestream,s->cur_file_info_internal.offset_curfile + + s->byte_before_the_zipfile,ZLIB_FILEFUNC_SEEK_SET)!=0) + return UNZ_ERRNO; + + + if (err==UNZ_OK) + { + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uMagic) != UNZ_OK) + err=UNZ_ERRNO; + else if (uMagic!=0x04034b50) + err=UNZ_BADZIPFILE; + } + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&uData) != UNZ_OK) + err=UNZ_ERRNO; +/* + else if ((err==UNZ_OK) && (uData!=s->cur_file_info.wVersion)) + err=UNZ_BADZIPFILE; +*/ + if (unz64local_getShort(&s->z_filefunc, s->filestream,&uFlags) != UNZ_OK) + err=UNZ_ERRNO; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&uData) != UNZ_OK) + err=UNZ_ERRNO; + else if ((err==UNZ_OK) && (uData!=s->cur_file_info.compression_method)) + err=UNZ_BADZIPFILE; + + if ((err==UNZ_OK) && (s->cur_file_info.compression_method!=0) && +/* #ifdef HAVE_BZIP2 */ + (s->cur_file_info.compression_method!=Z_BZIP2ED) && +/* #endif */ + (s->cur_file_info.compression_method!=Z_DEFLATED)) + err=UNZ_BADZIPFILE; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uData) != UNZ_OK) /* date/time */ + err=UNZ_ERRNO; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uData) != UNZ_OK) /* crc */ + err=UNZ_ERRNO; + else if ((err==UNZ_OK) && (uData!=s->cur_file_info.crc) && ((uFlags & 8)==0)) + err=UNZ_BADZIPFILE; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uData) != UNZ_OK) /* size compr */ + err=UNZ_ERRNO; + else if (uData != 0xFFFFFFFF && (err==UNZ_OK) && (uData!=s->cur_file_info.compressed_size) && ((uFlags & 8)==0)) + err=UNZ_BADZIPFILE; + + if (unz64local_getLong(&s->z_filefunc, s->filestream,&uData) != UNZ_OK) /* size uncompr */ + err=UNZ_ERRNO; + else if (uData != 0xFFFFFFFF && (err==UNZ_OK) && (uData!=s->cur_file_info.uncompressed_size) && ((uFlags & 8)==0)) + err=UNZ_BADZIPFILE; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&size_filename) != UNZ_OK) + err=UNZ_ERRNO; + else if ((err==UNZ_OK) && (size_filename!=s->cur_file_info.size_filename)) + err=UNZ_BADZIPFILE; + + *piSizeVar += (uInt)size_filename; + + if (unz64local_getShort(&s->z_filefunc, s->filestream,&size_extra_field) != UNZ_OK) + err=UNZ_ERRNO; + *poffset_local_extrafield= s->cur_file_info_internal.offset_curfile + + SIZEZIPLOCALHEADER + size_filename; + *psize_local_extrafield = (uInt)size_extra_field; + + *piSizeVar += (uInt)size_extra_field; + + return err; +} + +/* + Open for reading data the current file in the zipfile. + If there is no error and the file is opened, the return value is UNZ_OK. +*/ +extern int ZEXPORT unzOpenCurrentFile3 (unzFile file, int* method, + int* level, int raw, const char* password) +{ + int err=UNZ_OK; + uInt iSizeVar; + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + ZPOS64_T offset_local_extrafield; /* offset of the local extra field */ + uInt size_local_extrafield; /* size of the local extra field */ +# ifndef NOUNCRYPT + char source[12]; +# else + if (password != NULL) + return UNZ_PARAMERROR; +# endif + + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + if (!s->current_file_ok) + return UNZ_PARAMERROR; + + if (s->pfile_in_zip_read != NULL) + unzCloseCurrentFile(file); + + if (unz64local_CheckCurrentFileCoherencyHeader(s,&iSizeVar, &offset_local_extrafield,&size_local_extrafield)!=UNZ_OK) + return UNZ_BADZIPFILE; + + pfile_in_zip_read_info = (file_in_zip64_read_info_s*)ALLOC(sizeof(file_in_zip64_read_info_s)); + if (pfile_in_zip_read_info==NULL) + return UNZ_INTERNALERROR; + + pfile_in_zip_read_info->read_buffer=(char*)ALLOC(UNZ_BUFSIZE); + pfile_in_zip_read_info->offset_local_extrafield = offset_local_extrafield; + pfile_in_zip_read_info->size_local_extrafield = size_local_extrafield; + pfile_in_zip_read_info->pos_local_extrafield=0; + pfile_in_zip_read_info->raw=raw; + + if (pfile_in_zip_read_info->read_buffer==NULL) + { + TRYFREE(pfile_in_zip_read_info); + return UNZ_INTERNALERROR; + } + + pfile_in_zip_read_info->stream_initialised=0; + + if (method!=NULL) + *method = (int)s->cur_file_info.compression_method; + + if (level!=NULL) + { + *level = 6; + switch (s->cur_file_info.flag & 0x06) + { + case 6 : *level = 1; break; + case 4 : *level = 2; break; + case 2 : *level = 9; break; + } + } + + if ((s->cur_file_info.compression_method!=0) && +/* #ifdef HAVE_BZIP2 */ + (s->cur_file_info.compression_method!=Z_BZIP2ED) && +/* #endif */ + (s->cur_file_info.compression_method!=Z_DEFLATED)) + + err=UNZ_BADZIPFILE; + + pfile_in_zip_read_info->crc32_wait=s->cur_file_info.crc; + pfile_in_zip_read_info->crc32=0; + pfile_in_zip_read_info->total_out_64=0; + pfile_in_zip_read_info->compression_method = s->cur_file_info.compression_method; + pfile_in_zip_read_info->filestream=s->filestream; + pfile_in_zip_read_info->z_filefunc=s->z_filefunc; + pfile_in_zip_read_info->byte_before_the_zipfile=s->byte_before_the_zipfile; + + pfile_in_zip_read_info->stream.total_out = 0; + + if ((s->cur_file_info.compression_method==Z_BZIP2ED) && (!raw)) + { +#ifdef HAVE_BZIP2 + pfile_in_zip_read_info->bstream.bzalloc = (void *(*) (void *, int, int))0; + pfile_in_zip_read_info->bstream.bzfree = (free_func)0; + pfile_in_zip_read_info->bstream.opaque = (voidpf)0; + pfile_in_zip_read_info->bstream.state = (voidpf)0; + + pfile_in_zip_read_info->stream.zalloc = (alloc_func)0; + pfile_in_zip_read_info->stream.zfree = (free_func)0; + pfile_in_zip_read_info->stream.opaque = (voidpf)0; + pfile_in_zip_read_info->stream.next_in = (voidpf)0; + pfile_in_zip_read_info->stream.avail_in = 0; + + err=BZ2_bzDecompressInit(&pfile_in_zip_read_info->bstream, 0, 0); + if (err == Z_OK) + pfile_in_zip_read_info->stream_initialised=Z_BZIP2ED; + else + { + TRYFREE(pfile_in_zip_read_info); + return err; + } +#else + pfile_in_zip_read_info->raw=1; +#endif + } + else if ((s->cur_file_info.compression_method==Z_DEFLATED) && (!raw)) + { + pfile_in_zip_read_info->stream.zalloc = (alloc_func)0; + pfile_in_zip_read_info->stream.zfree = (free_func)0; + pfile_in_zip_read_info->stream.opaque = (voidpf)0; + pfile_in_zip_read_info->stream.next_in = 0; + pfile_in_zip_read_info->stream.avail_in = 0; + + err=inflateInit2(&pfile_in_zip_read_info->stream, -MAX_WBITS); + if (err == Z_OK) + pfile_in_zip_read_info->stream_initialised=Z_DEFLATED; + else + { + TRYFREE(pfile_in_zip_read_info); + return err; + } + /* windowBits is passed < 0 to tell that there is no zlib header. + * Note that in this case inflate *requires* an extra "dummy" byte + * after the compressed stream in order to complete decompression and + * return Z_STREAM_END. + * In unzip, i don't wait absolutely Z_STREAM_END because I known the + * size of both compressed and uncompressed data + */ + } + pfile_in_zip_read_info->rest_read_compressed = + s->cur_file_info.compressed_size ; + pfile_in_zip_read_info->rest_read_uncompressed = + s->cur_file_info.uncompressed_size ; + + + pfile_in_zip_read_info->pos_in_zipfile = + s->cur_file_info_internal.offset_curfile + SIZEZIPLOCALHEADER + + iSizeVar; + + pfile_in_zip_read_info->stream.avail_in = (uInt)0; + + s->pfile_in_zip_read = pfile_in_zip_read_info; + s->encrypted = 0; + +# ifndef NOUNCRYPT + if (password != NULL) + { + int i; + s->pcrc_32_tab = get_crc_table(); + init_keys(password,s->keys,s->pcrc_32_tab); + if (ZSEEK64(s->z_filefunc, s->filestream, + s->pfile_in_zip_read->pos_in_zipfile + + s->pfile_in_zip_read->byte_before_the_zipfile, + SEEK_SET)!=0) + return UNZ_INTERNALERROR; + if(ZREAD64(s->z_filefunc, s->filestream,source, 12)<12) + return UNZ_INTERNALERROR; + + for (i = 0; i<12; i++) + zdecode(s->keys,s->pcrc_32_tab,source[i]); + + s->pfile_in_zip_read->pos_in_zipfile+=12; + s->pfile_in_zip_read->rest_read_compressed-=12; + s->encrypted=1; + } +# endif + + + return UNZ_OK; +} + +extern int ZEXPORT unzOpenCurrentFile (unzFile file) +{ + return unzOpenCurrentFile3(file, NULL, NULL, 0, NULL); +} + +extern int ZEXPORT unzOpenCurrentFilePassword (unzFile file, const char* password) +{ + return unzOpenCurrentFile3(file, NULL, NULL, 0, password); +} + +extern int ZEXPORT unzOpenCurrentFile2 (unzFile file, int* method, int* level, int raw) +{ + return unzOpenCurrentFile3(file, method, level, raw, NULL); +} + +/** Addition for GDAL : START */ + +extern ZPOS64_T ZEXPORT unzGetCurrentFileZStreamPos64( unzFile file) +{ + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + s=(unz64_s*)file; + if (file==NULL) + return 0; //UNZ_PARAMERROR; + pfile_in_zip_read_info=s->pfile_in_zip_read; + if (pfile_in_zip_read_info==NULL) + return 0; //UNZ_PARAMERROR; + return pfile_in_zip_read_info->pos_in_zipfile + + pfile_in_zip_read_info->byte_before_the_zipfile; +} + +/** Addition for GDAL : END */ + +/* + Read bytes from the current file. + buf contain buffer where data must be copied + len the size of buf. + + return the number of byte copied if somes bytes are copied + return 0 if the end of file was reached + return <0 with error code if there is an error + (UNZ_ERRNO for IO error, or zLib error for uncompress error) +*/ +extern int ZEXPORT unzReadCurrentFile (unzFile file, voidp buf, unsigned len) +{ + int err=UNZ_OK; + uInt iRead = 0; + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + pfile_in_zip_read_info=s->pfile_in_zip_read; + + if (pfile_in_zip_read_info==NULL) + return UNZ_PARAMERROR; + + + if (pfile_in_zip_read_info->read_buffer == NULL) + return UNZ_END_OF_LIST_OF_FILE; + if (len==0) + return 0; + + pfile_in_zip_read_info->stream.next_out = (Bytef*)buf; + + pfile_in_zip_read_info->stream.avail_out = (uInt)len; + + if ((len>pfile_in_zip_read_info->rest_read_compressed+ + pfile_in_zip_read_info->stream.avail_in) && + (pfile_in_zip_read_info->raw)) + pfile_in_zip_read_info->stream.avail_out = + (uInt)pfile_in_zip_read_info->rest_read_compressed+ + pfile_in_zip_read_info->stream.avail_in; + + while (pfile_in_zip_read_info->stream.avail_out>0) + { + if ((pfile_in_zip_read_info->stream.avail_in==0) && + (pfile_in_zip_read_info->rest_read_compressed>0)) + { + uInt uReadThis = UNZ_BUFSIZE; + if (pfile_in_zip_read_info->rest_read_compressed<uReadThis) + uReadThis = (uInt)pfile_in_zip_read_info->rest_read_compressed; + if (uReadThis == 0) + return UNZ_EOF; + if (ZSEEK64(pfile_in_zip_read_info->z_filefunc, + pfile_in_zip_read_info->filestream, + pfile_in_zip_read_info->pos_in_zipfile + + pfile_in_zip_read_info->byte_before_the_zipfile, + ZLIB_FILEFUNC_SEEK_SET)!=0) + return UNZ_ERRNO; + if (ZREAD64(pfile_in_zip_read_info->z_filefunc, + pfile_in_zip_read_info->filestream, + pfile_in_zip_read_info->read_buffer, + uReadThis)!=uReadThis) + return UNZ_ERRNO; + + +# ifndef NOUNCRYPT + if(s->encrypted) + { + uInt i; + for(i=0;i<uReadThis;i++) + pfile_in_zip_read_info->read_buffer[i] = + zdecode(s->keys,s->pcrc_32_tab, + pfile_in_zip_read_info->read_buffer[i]); + } +# endif + + + pfile_in_zip_read_info->pos_in_zipfile += uReadThis; + + pfile_in_zip_read_info->rest_read_compressed-=uReadThis; + + pfile_in_zip_read_info->stream.next_in = + (Bytef*)pfile_in_zip_read_info->read_buffer; + pfile_in_zip_read_info->stream.avail_in = (uInt)uReadThis; + } + + if ((pfile_in_zip_read_info->compression_method==0) || (pfile_in_zip_read_info->raw)) + { + uInt uDoCopy,i ; + + if ((pfile_in_zip_read_info->stream.avail_in == 0) && + (pfile_in_zip_read_info->rest_read_compressed == 0)) + return (iRead==0) ? UNZ_EOF : iRead; + + if (pfile_in_zip_read_info->stream.avail_out < + pfile_in_zip_read_info->stream.avail_in) + uDoCopy = pfile_in_zip_read_info->stream.avail_out ; + else + uDoCopy = pfile_in_zip_read_info->stream.avail_in ; + + for (i=0;i<uDoCopy;i++) + *(pfile_in_zip_read_info->stream.next_out+i) = + *(pfile_in_zip_read_info->stream.next_in+i); + + pfile_in_zip_read_info->total_out_64 = pfile_in_zip_read_info->total_out_64 + uDoCopy; + + pfile_in_zip_read_info->crc32 = crc32(pfile_in_zip_read_info->crc32, + pfile_in_zip_read_info->stream.next_out, + uDoCopy); + pfile_in_zip_read_info->rest_read_uncompressed-=uDoCopy; + pfile_in_zip_read_info->stream.avail_in -= uDoCopy; + pfile_in_zip_read_info->stream.avail_out -= uDoCopy; + pfile_in_zip_read_info->stream.next_out += uDoCopy; + pfile_in_zip_read_info->stream.next_in += uDoCopy; + pfile_in_zip_read_info->stream.total_out += uDoCopy; + iRead += uDoCopy; + } + else if (pfile_in_zip_read_info->compression_method==Z_BZIP2ED) + { +#ifdef HAVE_BZIP2 + uLong uTotalOutBefore,uTotalOutAfter; + const Bytef *bufBefore; + uLong uOutThis; + + pfile_in_zip_read_info->bstream.next_in = (char*)pfile_in_zip_read_info->stream.next_in; + pfile_in_zip_read_info->bstream.avail_in = pfile_in_zip_read_info->stream.avail_in; + pfile_in_zip_read_info->bstream.total_in_lo32 = pfile_in_zip_read_info->stream.total_in; + pfile_in_zip_read_info->bstream.total_in_hi32 = 0; + pfile_in_zip_read_info->bstream.next_out = (char*)pfile_in_zip_read_info->stream.next_out; + pfile_in_zip_read_info->bstream.avail_out = pfile_in_zip_read_info->stream.avail_out; + pfile_in_zip_read_info->bstream.total_out_lo32 = pfile_in_zip_read_info->stream.total_out; + pfile_in_zip_read_info->bstream.total_out_hi32 = 0; + + uTotalOutBefore = pfile_in_zip_read_info->bstream.total_out_lo32; + bufBefore = (const Bytef *)pfile_in_zip_read_info->bstream.next_out; + + err=BZ2_bzDecompress(&pfile_in_zip_read_info->bstream); + + uTotalOutAfter = pfile_in_zip_read_info->bstream.total_out_lo32; + uOutThis = uTotalOutAfter-uTotalOutBefore; + + pfile_in_zip_read_info->total_out_64 = pfile_in_zip_read_info->total_out_64 + uOutThis; + + pfile_in_zip_read_info->crc32 = crc32(pfile_in_zip_read_info->crc32,bufBefore, (uInt)(uOutThis)); + pfile_in_zip_read_info->rest_read_uncompressed -= uOutThis; + iRead += (uInt)(uTotalOutAfter - uTotalOutBefore); + + pfile_in_zip_read_info->stream.next_in = (Bytef*)pfile_in_zip_read_info->bstream.next_in; + pfile_in_zip_read_info->stream.avail_in = pfile_in_zip_read_info->bstream.avail_in; + pfile_in_zip_read_info->stream.total_in = pfile_in_zip_read_info->bstream.total_in_lo32; + pfile_in_zip_read_info->stream.next_out = (Bytef*)pfile_in_zip_read_info->bstream.next_out; + pfile_in_zip_read_info->stream.avail_out = pfile_in_zip_read_info->bstream.avail_out; + pfile_in_zip_read_info->stream.total_out = pfile_in_zip_read_info->bstream.total_out_lo32; + + if (err==BZ_STREAM_END) + return (iRead==0) ? UNZ_EOF : iRead; + if (err!=BZ_OK) + break; +#endif + } // end Z_BZIP2ED + else + { + ZPOS64_T uTotalOutBefore,uTotalOutAfter; + const Bytef *bufBefore; + ZPOS64_T uOutThis; + int flush=Z_SYNC_FLUSH; + + uTotalOutBefore = pfile_in_zip_read_info->stream.total_out; + bufBefore = pfile_in_zip_read_info->stream.next_out; + + /* + if ((pfile_in_zip_read_info->rest_read_uncompressed == + pfile_in_zip_read_info->stream.avail_out) && + (pfile_in_zip_read_info->rest_read_compressed == 0)) + flush = Z_FINISH; + */ + err=inflate(&pfile_in_zip_read_info->stream,flush); + + if ((err>=0) && (pfile_in_zip_read_info->stream.msg!=NULL)) + err = Z_DATA_ERROR; + + uTotalOutAfter = pfile_in_zip_read_info->stream.total_out; + uOutThis = uTotalOutAfter-uTotalOutBefore; + + pfile_in_zip_read_info->total_out_64 = pfile_in_zip_read_info->total_out_64 + uOutThis; + + pfile_in_zip_read_info->crc32 = + crc32(pfile_in_zip_read_info->crc32,bufBefore, + (uInt)(uOutThis)); + + pfile_in_zip_read_info->rest_read_uncompressed -= + uOutThis; + + iRead += (uInt)(uTotalOutAfter - uTotalOutBefore); + + if (err==Z_STREAM_END) + return (iRead==0) ? UNZ_EOF : iRead; + if (err!=Z_OK) + break; + } + } + + if (err==Z_OK) + return iRead; + return err; +} + + +/* + Give the current position in uncompressed data +*/ +extern z_off_t ZEXPORT unztell (unzFile file) +{ + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + pfile_in_zip_read_info=s->pfile_in_zip_read; + + if (pfile_in_zip_read_info==NULL) + return UNZ_PARAMERROR; + + return (z_off_t)pfile_in_zip_read_info->stream.total_out; +} + +extern ZPOS64_T ZEXPORT unztell64 (unzFile file) +{ + + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + if (file==NULL) + return (ZPOS64_T)-1; + s=(unz64_s*)file; + pfile_in_zip_read_info=s->pfile_in_zip_read; + + if (pfile_in_zip_read_info==NULL) + return (ZPOS64_T)-1; + + return pfile_in_zip_read_info->total_out_64; +} + + +/* + return 1 if the end of file was reached, 0 elsewhere +*/ +extern int ZEXPORT unzeof (unzFile file) +{ + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + pfile_in_zip_read_info=s->pfile_in_zip_read; + + if (pfile_in_zip_read_info==NULL) + return UNZ_PARAMERROR; + + if (pfile_in_zip_read_info->rest_read_uncompressed == 0) + return 1; + else + return 0; +} + + + +/* +Read extra field from the current file (opened by unzOpenCurrentFile) +This is the local-header version of the extra field (sometimes, there is +more info in the local-header version than in the central-header) + + if buf==NULL, it return the size of the local extra field that can be read + + if buf!=NULL, len is the size of the buffer, the extra header is copied in + buf. + the return value is the number of bytes copied in buf, or (if <0) + the error code +*/ +extern int ZEXPORT unzGetLocalExtrafield (unzFile file, voidp buf, unsigned len) +{ + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + uInt read_now; + ZPOS64_T size_to_read; + + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + pfile_in_zip_read_info=s->pfile_in_zip_read; + + if (pfile_in_zip_read_info==NULL) + return UNZ_PARAMERROR; + + size_to_read = (pfile_in_zip_read_info->size_local_extrafield - + pfile_in_zip_read_info->pos_local_extrafield); + + if (buf==NULL) + return (int)size_to_read; + + if (len>size_to_read) + read_now = (uInt)size_to_read; + else + read_now = (uInt)len ; + + if (read_now==0) + return 0; + + if (ZSEEK64(pfile_in_zip_read_info->z_filefunc, + pfile_in_zip_read_info->filestream, + pfile_in_zip_read_info->offset_local_extrafield + + pfile_in_zip_read_info->pos_local_extrafield, + ZLIB_FILEFUNC_SEEK_SET)!=0) + return UNZ_ERRNO; + + if (ZREAD64(pfile_in_zip_read_info->z_filefunc, + pfile_in_zip_read_info->filestream, + buf,read_now)!=read_now) + return UNZ_ERRNO; + + return (int)read_now; +} + +/* + Close the file in zip opened with unzOpenCurrentFile + Return UNZ_CRCERROR if all the file was read but the CRC is not good +*/ +extern int ZEXPORT unzCloseCurrentFile (unzFile file) +{ + int err=UNZ_OK; + + unz64_s* s; + file_in_zip64_read_info_s* pfile_in_zip_read_info; + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + pfile_in_zip_read_info=s->pfile_in_zip_read; + + if (pfile_in_zip_read_info==NULL) + return UNZ_PARAMERROR; + + + if ((pfile_in_zip_read_info->rest_read_uncompressed == 0) && + (!pfile_in_zip_read_info->raw)) + { + if (pfile_in_zip_read_info->crc32 != pfile_in_zip_read_info->crc32_wait) + err=UNZ_CRCERROR; + } + + + TRYFREE(pfile_in_zip_read_info->read_buffer); + pfile_in_zip_read_info->read_buffer = NULL; + if (pfile_in_zip_read_info->stream_initialised == Z_DEFLATED) + inflateEnd(&pfile_in_zip_read_info->stream); +#ifdef HAVE_BZIP2 + else if (pfile_in_zip_read_info->stream_initialised == Z_BZIP2ED) + BZ2_bzDecompressEnd(&pfile_in_zip_read_info->bstream); +#endif + + + pfile_in_zip_read_info->stream_initialised = 0; + TRYFREE(pfile_in_zip_read_info); + + s->pfile_in_zip_read=NULL; + + return err; +} + + +/* + Get the global comment string of the ZipFile, in the szComment buffer. + uSizeBuf is the size of the szComment buffer. + return the number of byte copied or an error code <0 +*/ +extern int ZEXPORT unzGetGlobalComment (unzFile file, char * szComment, uLong uSizeBuf) +{ + unz64_s* s; + uLong uReadThis ; + if (file==NULL) + return (int)UNZ_PARAMERROR; + s=(unz64_s*)file; + + uReadThis = uSizeBuf; + if (uReadThis>s->gi.size_comment) + uReadThis = s->gi.size_comment; + + if (ZSEEK64(s->z_filefunc,s->filestream,s->central_pos+22,ZLIB_FILEFUNC_SEEK_SET)!=0) + return UNZ_ERRNO; + + if (uReadThis>0) + { + *szComment='\0'; + if (ZREAD64(s->z_filefunc,s->filestream,szComment,uReadThis)!=uReadThis) + return UNZ_ERRNO; + } + + if ((szComment != NULL) && (uSizeBuf > s->gi.size_comment)) + *(szComment+s->gi.size_comment)='\0'; + return (int)uReadThis; +} + +/* Additions by RX '2004 */ +extern ZPOS64_T ZEXPORT unzGetOffset64(unzFile file) +{ + unz64_s* s; + + if (file==NULL) + return 0; //UNZ_PARAMERROR; + s=(unz64_s*)file; + if (!s->current_file_ok) + return 0; + if (s->gi.number_entry != 0 && s->gi.number_entry != 0xffff) + if (s->num_file==s->gi.number_entry) + return 0; + return s->pos_in_central_dir; +} + +extern uLong ZEXPORT unzGetOffset (unzFile file) +{ + ZPOS64_T offset64; + + if (file==NULL) + return 0; //UNZ_PARAMERROR; + offset64 = unzGetOffset64(file); + return (uLong)offset64; +} + +extern int ZEXPORT unzSetOffset64(unzFile file, ZPOS64_T pos) +{ + unz64_s* s; + int err; + + if (file==NULL) + return UNZ_PARAMERROR; + s=(unz64_s*)file; + + s->pos_in_central_dir = pos; + s->num_file = s->gi.number_entry; /* hack */ + err = unz64local_GetCurrentFileInfoInternal(file,&s->cur_file_info, + &s->cur_file_info_internal, + NULL,0,NULL,0,NULL,0); + s->current_file_ok = (err == UNZ_OK); + return err; +} + +extern int ZEXPORT unzSetOffset (unzFile file, uLong pos) +{ + return unzSetOffset64(file,pos); +} diff --git a/third_party/zlib/contrib/minizip/unzip.h b/third_party/zlib/contrib/minizip/unzip.h new file mode 100644 index 00000000..2104e391 --- /dev/null +++ b/third_party/zlib/contrib/minizip/unzip.h @@ -0,0 +1,437 @@ +/* unzip.h -- IO for uncompress .zip files using zlib + Version 1.1, February 14h, 2010 + part of the MiniZip project - ( http://www.winimage.com/zLibDll/minizip.html ) + + Copyright (C) 1998-2010 Gilles Vollant (minizip) ( http://www.winimage.com/zLibDll/minizip.html ) + + Modifications of Unzip for Zip64 + Copyright (C) 2007-2008 Even Rouault + + Modifications for Zip64 support on both zip and unzip + Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) + + For more info read MiniZip_info.txt + + --------------------------------------------------------------------------------- + + Condition of use and distribution are the same than zlib : + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + --------------------------------------------------------------------------------- + + Changes + + See header of unzip64.c + +*/ + +#ifndef _unz64_H +#define _unz64_H + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef _ZLIB_H +#include "zlib.h" +#endif + +#ifndef _ZLIBIOAPI_H +#include "ioapi.h" +#endif + +#ifdef HAVE_BZIP2 +#include "bzlib.h" +#endif + +#define Z_BZIP2ED 12 + +#if defined(STRICTUNZIP) || defined(STRICTZIPUNZIP) +/* like the STRICT of WIN32, we define a pointer that cannot be converted + from (void*) without cast */ +typedef struct TagunzFile__ { int unused; } unzFile__; +typedef unzFile__ *unzFile; +#else +typedef voidp unzFile; +#endif + + +#define UNZ_OK (0) +#define UNZ_END_OF_LIST_OF_FILE (-100) +#define UNZ_ERRNO (Z_ERRNO) +#define UNZ_EOF (0) +#define UNZ_PARAMERROR (-102) +#define UNZ_BADZIPFILE (-103) +#define UNZ_INTERNALERROR (-104) +#define UNZ_CRCERROR (-105) + +/* tm_unz contain date/time info */ +typedef struct tm_unz_s +{ + uInt tm_sec; /* seconds after the minute - [0,59] */ + uInt tm_min; /* minutes after the hour - [0,59] */ + uInt tm_hour; /* hours since midnight - [0,23] */ + uInt tm_mday; /* day of the month - [1,31] */ + uInt tm_mon; /* months since January - [0,11] */ + uInt tm_year; /* years - [1980..2044] */ +} tm_unz; + +/* unz_global_info structure contain global data about the ZIPfile + These data comes from the end of central dir */ +typedef struct unz_global_info64_s +{ + ZPOS64_T number_entry; /* total number of entries in + the central dir on this disk */ + uLong size_comment; /* size of the global comment of the zipfile */ +} unz_global_info64; + +typedef struct unz_global_info_s +{ + uLong number_entry; /* total number of entries in + the central dir on this disk */ + uLong size_comment; /* size of the global comment of the zipfile */ +} unz_global_info; + +/* unz_file_info contain information about a file in the zipfile */ +typedef struct unz_file_info64_s +{ + uLong version; /* version made by 2 bytes */ + uLong version_needed; /* version needed to extract 2 bytes */ + uLong flag; /* general purpose bit flag 2 bytes */ + uLong compression_method; /* compression method 2 bytes */ + uLong dosDate; /* last mod file date in Dos fmt 4 bytes */ + uLong crc; /* crc-32 4 bytes */ + ZPOS64_T compressed_size; /* compressed size 8 bytes */ + ZPOS64_T uncompressed_size; /* uncompressed size 8 bytes */ + uLong size_filename; /* filename length 2 bytes */ + uLong size_file_extra; /* extra field length 2 bytes */ + uLong size_file_comment; /* file comment length 2 bytes */ + + uLong disk_num_start; /* disk number start 2 bytes */ + uLong internal_fa; /* internal file attributes 2 bytes */ + uLong external_fa; /* external file attributes 4 bytes */ + + tm_unz tmu_date; +} unz_file_info64; + +typedef struct unz_file_info_s +{ + uLong version; /* version made by 2 bytes */ + uLong version_needed; /* version needed to extract 2 bytes */ + uLong flag; /* general purpose bit flag 2 bytes */ + uLong compression_method; /* compression method 2 bytes */ + uLong dosDate; /* last mod file date in Dos fmt 4 bytes */ + uLong crc; /* crc-32 4 bytes */ + uLong compressed_size; /* compressed size 4 bytes */ + uLong uncompressed_size; /* uncompressed size 4 bytes */ + uLong size_filename; /* filename length 2 bytes */ + uLong size_file_extra; /* extra field length 2 bytes */ + uLong size_file_comment; /* file comment length 2 bytes */ + + uLong disk_num_start; /* disk number start 2 bytes */ + uLong internal_fa; /* internal file attributes 2 bytes */ + uLong external_fa; /* external file attributes 4 bytes */ + + tm_unz tmu_date; +} unz_file_info; + +extern int ZEXPORT unzStringFileNameCompare OF ((const char* fileName1, + const char* fileName2, + int iCaseSensitivity)); +/* + Compare two filename (fileName1,fileName2). + If iCaseSenisivity = 1, comparision is case sensitivity (like strcmp) + If iCaseSenisivity = 2, comparision is not case sensitivity (like strcmpi + or strcasecmp) + If iCaseSenisivity = 0, case sensitivity is defaut of your operating system + (like 1 on Unix, 2 on Windows) +*/ + + +extern unzFile ZEXPORT unzOpen OF((const char *path)); +extern unzFile ZEXPORT unzOpen64 OF((const void *path)); +/* + Open a Zip file. path contain the full pathname (by example, + on a Windows XP computer "c:\\zlib\\zlib113.zip" or on an Unix computer + "zlib/zlib113.zip". + If the zipfile cannot be opened (file don't exist or in not valid), the + return value is NULL. + Else, the return value is a unzFile Handle, usable with other function + of this unzip package. + the "64" function take a const void* pointer, because the path is just the + value passed to the open64_file_func callback. + Under Windows, if UNICODE is defined, using fill_fopen64_filefunc, the path + is a pointer to a wide unicode string (LPCTSTR is LPCWSTR), so const char* + does not describe the reality +*/ + + +extern unzFile ZEXPORT unzOpen2 OF((const char *path, + zlib_filefunc_def* pzlib_filefunc_def)); +/* + Open a Zip file, like unzOpen, but provide a set of file low level API + for read/write the zip file (see ioapi.h) +*/ + +extern unzFile ZEXPORT unzOpen2_64 OF((const void *path, + zlib_filefunc64_def* pzlib_filefunc_def)); +/* + Open a Zip file, like unz64Open, but provide a set of file low level API + for read/write the zip file (see ioapi.h) +*/ + +extern int ZEXPORT unzClose OF((unzFile file)); +/* + Close a ZipFile opened with unzOpen. + If there is files inside the .Zip opened with unzOpenCurrentFile (see later), + these files MUST be closed with unzCloseCurrentFile before call unzClose. + return UNZ_OK if there is no problem. */ + +extern int ZEXPORT unzGetGlobalInfo OF((unzFile file, + unz_global_info *pglobal_info)); + +extern int ZEXPORT unzGetGlobalInfo64 OF((unzFile file, + unz_global_info64 *pglobal_info)); +/* + Write info about the ZipFile in the *pglobal_info structure. + No preparation of the structure is needed + return UNZ_OK if there is no problem. */ + + +extern int ZEXPORT unzGetGlobalComment OF((unzFile file, + char *szComment, + uLong uSizeBuf)); +/* + Get the global comment string of the ZipFile, in the szComment buffer. + uSizeBuf is the size of the szComment buffer. + return the number of byte copied or an error code <0 +*/ + + +/***************************************************************************/ +/* Unzip package allow you browse the directory of the zipfile */ + +extern int ZEXPORT unzGoToFirstFile OF((unzFile file)); +/* + Set the current file of the zipfile to the first file. + return UNZ_OK if there is no problem +*/ + +extern int ZEXPORT unzGoToNextFile OF((unzFile file)); +/* + Set the current file of the zipfile to the next file. + return UNZ_OK if there is no problem + return UNZ_END_OF_LIST_OF_FILE if the actual file was the latest. +*/ + +extern int ZEXPORT unzLocateFile OF((unzFile file, + const char *szFileName, + int iCaseSensitivity)); +/* + Try locate the file szFileName in the zipfile. + For the iCaseSensitivity signification, see unzStringFileNameCompare + + return value : + UNZ_OK if the file is found. It becomes the current file. + UNZ_END_OF_LIST_OF_FILE if the file is not found +*/ + + +/* ****************************************** */ +/* Ryan supplied functions */ +/* unz_file_info contain information about a file in the zipfile */ +typedef struct unz_file_pos_s +{ + uLong pos_in_zip_directory; /* offset in zip file directory */ + uLong num_of_file; /* # of file */ +} unz_file_pos; + +extern int ZEXPORT unzGetFilePos( + unzFile file, + unz_file_pos* file_pos); + +extern int ZEXPORT unzGoToFilePos( + unzFile file, + unz_file_pos* file_pos); + +typedef struct unz64_file_pos_s +{ + ZPOS64_T pos_in_zip_directory; /* offset in zip file directory */ + ZPOS64_T num_of_file; /* # of file */ +} unz64_file_pos; + +extern int ZEXPORT unzGetFilePos64( + unzFile file, + unz64_file_pos* file_pos); + +extern int ZEXPORT unzGoToFilePos64( + unzFile file, + const unz64_file_pos* file_pos); + +/* ****************************************** */ + +extern int ZEXPORT unzGetCurrentFileInfo64 OF((unzFile file, + unz_file_info64 *pfile_info, + char *szFileName, + uLong fileNameBufferSize, + void *extraField, + uLong extraFieldBufferSize, + char *szComment, + uLong commentBufferSize)); + +extern int ZEXPORT unzGetCurrentFileInfo OF((unzFile file, + unz_file_info *pfile_info, + char *szFileName, + uLong fileNameBufferSize, + void *extraField, + uLong extraFieldBufferSize, + char *szComment, + uLong commentBufferSize)); +/* + Get Info about the current file + if pfile_info!=NULL, the *pfile_info structure will contain somes info about + the current file + if szFileName!=NULL, the filemane string will be copied in szFileName + (fileNameBufferSize is the size of the buffer) + if extraField!=NULL, the extra field information will be copied in extraField + (extraFieldBufferSize is the size of the buffer). + This is the Central-header version of the extra field + if szComment!=NULL, the comment string of the file will be copied in szComment + (commentBufferSize is the size of the buffer) +*/ + + +/** Addition for GDAL : START */ + +extern ZPOS64_T ZEXPORT unzGetCurrentFileZStreamPos64 OF((unzFile file)); + +/** Addition for GDAL : END */ + + +/***************************************************************************/ +/* for reading the content of the current zipfile, you can open it, read data + from it, and close it (you can close it before reading all the file) + */ + +extern int ZEXPORT unzOpenCurrentFile OF((unzFile file)); +/* + Open for reading data the current file in the zipfile. + If there is no error, the return value is UNZ_OK. +*/ + +extern int ZEXPORT unzOpenCurrentFilePassword OF((unzFile file, + const char* password)); +/* + Open for reading data the current file in the zipfile. + password is a crypting password + If there is no error, the return value is UNZ_OK. +*/ + +extern int ZEXPORT unzOpenCurrentFile2 OF((unzFile file, + int* method, + int* level, + int raw)); +/* + Same than unzOpenCurrentFile, but open for read raw the file (not uncompress) + if raw==1 + *method will receive method of compression, *level will receive level of + compression + note : you can set level parameter as NULL (if you did not want known level, + but you CANNOT set method parameter as NULL +*/ + +extern int ZEXPORT unzOpenCurrentFile3 OF((unzFile file, + int* method, + int* level, + int raw, + const char* password)); +/* + Same than unzOpenCurrentFile, but open for read raw the file (not uncompress) + if raw==1 + *method will receive method of compression, *level will receive level of + compression + note : you can set level parameter as NULL (if you did not want known level, + but you CANNOT set method parameter as NULL +*/ + + +extern int ZEXPORT unzCloseCurrentFile OF((unzFile file)); +/* + Close the file in zip opened with unzOpenCurrentFile + Return UNZ_CRCERROR if all the file was read but the CRC is not good +*/ + +extern int ZEXPORT unzReadCurrentFile OF((unzFile file, + voidp buf, + unsigned len)); +/* + Read bytes from the current file (opened by unzOpenCurrentFile) + buf contain buffer where data must be copied + len the size of buf. + + return the number of byte copied if somes bytes are copied + return 0 if the end of file was reached + return <0 with error code if there is an error + (UNZ_ERRNO for IO error, or zLib error for uncompress error) +*/ + +extern z_off_t ZEXPORT unztell OF((unzFile file)); + +extern ZPOS64_T ZEXPORT unztell64 OF((unzFile file)); +/* + Give the current position in uncompressed data +*/ + +extern int ZEXPORT unzeof OF((unzFile file)); +/* + return 1 if the end of file was reached, 0 elsewhere +*/ + +extern int ZEXPORT unzGetLocalExtrafield OF((unzFile file, + voidp buf, + unsigned len)); +/* + Read extra field from the current file (opened by unzOpenCurrentFile) + This is the local-header version of the extra field (sometimes, there is + more info in the local-header version than in the central-header) + + if buf==NULL, it return the size of the local extra field + + if buf!=NULL, len is the size of the buffer, the extra header is copied in + buf. + the return value is the number of bytes copied in buf, or (if <0) + the error code +*/ + +/***************************************************************************/ + +/* Get the current file offset */ +extern ZPOS64_T ZEXPORT unzGetOffset64 (unzFile file); +extern uLong ZEXPORT unzGetOffset (unzFile file); + +/* Set the current file offset */ +extern int ZEXPORT unzSetOffset64 (unzFile file, ZPOS64_T pos); +extern int ZEXPORT unzSetOffset (unzFile file, uLong pos); + + + +#ifdef __cplusplus +} +#endif + +#endif /* _unz64_H */ diff --git a/third_party/zlib/contrib/minizip/zip.c b/third_party/zlib/contrib/minizip/zip.c new file mode 100644 index 00000000..65c0c725 --- /dev/null +++ b/third_party/zlib/contrib/minizip/zip.c @@ -0,0 +1,2007 @@ +/* zip.c -- IO on .zip files using zlib + Version 1.1, February 14h, 2010 + part of the MiniZip project - ( http://www.winimage.com/zLibDll/minizip.html ) + + Copyright (C) 1998-2010 Gilles Vollant (minizip) ( http://www.winimage.com/zLibDll/minizip.html ) + + Modifications for Zip64 support + Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) + + For more info read MiniZip_info.txt + + Changes + Oct-2009 - Mathias Svensson - Remove old C style function prototypes + Oct-2009 - Mathias Svensson - Added Zip64 Support when creating new file archives + Oct-2009 - Mathias Svensson - Did some code cleanup and refactoring to get better overview of some functions. + Oct-2009 - Mathias Svensson - Added zipRemoveExtraInfoBlock to strip extra field data from its ZIP64 data + It is used when recreting zip archive with RAW when deleting items from a zip. + ZIP64 data is automatically added to items that needs it, and existing ZIP64 data need to be removed. + Oct-2009 - Mathias Svensson - Added support for BZIP2 as compression mode (bzip2 lib is required) + Jan-2010 - back to unzip and minizip 1.0 name scheme, with compatibility layer + +*/ + + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include "third_party/zlib/zlib.h" +#include "zip.h" + +#ifdef STDC +# include <stddef.h> +# include <string.h> +# include <stdlib.h> +#endif +#ifdef NO_ERRNO_H + extern int errno; +#else +# include <errno.h> +#endif + + +#ifndef local +# define local static +#endif +/* compile with -Dlocal if your debugger can't find static symbols */ + +#ifndef VERSIONMADEBY +# define VERSIONMADEBY (0x0) /* platform depedent */ +#endif + +#ifndef Z_BUFSIZE +#define Z_BUFSIZE (64*1024) //(16384) +#endif + +#ifndef Z_MAXFILENAMEINZIP +#define Z_MAXFILENAMEINZIP (256) +#endif + +#ifndef ALLOC +# define ALLOC(size) (malloc(size)) +#endif +#ifndef TRYFREE +# define TRYFREE(p) {if (p) free(p);} +#endif + +/* +#define SIZECENTRALDIRITEM (0x2e) +#define SIZEZIPLOCALHEADER (0x1e) +*/ + +/* I've found an old Unix (a SunOS 4.1.3_U1) without all SEEK_* defined.... */ + + +// NOT sure that this work on ALL platform +#define MAKEULONG64(a, b) ((ZPOS64_T)(((unsigned long)(a)) | ((ZPOS64_T)((unsigned long)(b))) << 32)) + +#ifndef SEEK_CUR +#define SEEK_CUR 1 +#endif + +#ifndef SEEK_END +#define SEEK_END 2 +#endif + +#ifndef SEEK_SET +#define SEEK_SET 0 +#endif + +#ifndef DEF_MEM_LEVEL +#if MAX_MEM_LEVEL >= 8 +# define DEF_MEM_LEVEL 8 +#else +# define DEF_MEM_LEVEL MAX_MEM_LEVEL +#endif +#endif +const char zip_copyright[] =" zip 1.01 Copyright 1998-2004 Gilles Vollant - http://www.winimage.com/zLibDll"; + + +#define SIZEDATA_INDATABLOCK (4096-(4*4)) + +#define LOCALHEADERMAGIC (0x04034b50) +#define CENTRALHEADERMAGIC (0x02014b50) +#define ENDHEADERMAGIC (0x06054b50) +#define ZIP64ENDHEADERMAGIC (0x6064b50) +#define ZIP64ENDLOCHEADERMAGIC (0x7064b50) + +#define FLAG_LOCALHEADER_OFFSET (0x06) +#define CRC_LOCALHEADER_OFFSET (0x0e) + +#define SIZECENTRALHEADER (0x2e) /* 46 */ + +typedef struct linkedlist_datablock_internal_s +{ + struct linkedlist_datablock_internal_s* next_datablock; + uLong avail_in_this_block; + uLong filled_in_this_block; + uLong unused; /* for future use and alignment */ + unsigned char data[SIZEDATA_INDATABLOCK]; +} linkedlist_datablock_internal; + +typedef struct linkedlist_data_s +{ + linkedlist_datablock_internal* first_block; + linkedlist_datablock_internal* last_block; +} linkedlist_data; + + +typedef struct +{ + z_stream stream; /* zLib stream structure for inflate */ +#ifdef HAVE_BZIP2 + bz_stream bstream; /* bzLib stream structure for bziped */ +#endif + + int stream_initialised; /* 1 is stream is initialised */ + uInt pos_in_buffered_data; /* last written byte in buffered_data */ + + ZPOS64_T pos_local_header; /* offset of the local header of the file + currenty writing */ + char* central_header; /* central header data for the current file */ + uLong size_centralExtra; + uLong size_centralheader; /* size of the central header for cur file */ + uLong size_centralExtraFree; /* Extra bytes allocated to the centralheader but that are not used */ + uLong flag; /* flag of the file currently writing */ + + int method; /* compression method of file currenty wr.*/ + int raw; /* 1 for directly writing raw data */ + Byte buffered_data[Z_BUFSIZE];/* buffer contain compressed data to be writ*/ + uLong dosDate; + uLong crc32; + int encrypt; + int zip64; /* Add ZIP64 extened information in the extra field */ + ZPOS64_T pos_zip64extrainfo; + ZPOS64_T totalCompressedData; + ZPOS64_T totalUncompressedData; +#ifndef NOCRYPT + unsigned long keys[3]; /* keys defining the pseudo-random sequence */ + const z_crc_t* pcrc_32_tab; + int crypt_header_size; +#endif +} curfile64_info; + +typedef struct +{ + zlib_filefunc64_32_def z_filefunc; + voidpf filestream; /* io structore of the zipfile */ + linkedlist_data central_dir;/* datablock with central dir in construction*/ + int in_opened_file_inzip; /* 1 if a file in the zip is currently writ.*/ + curfile64_info ci; /* info on the file curretly writing */ + + ZPOS64_T begin_pos; /* position of the beginning of the zipfile */ + ZPOS64_T add_position_when_writing_offset; + ZPOS64_T number_entry; + +#ifndef NO_ADDFILEINEXISTINGZIP + char *globalcomment; +#endif + +} zip64_internal; + + +#ifndef NOCRYPT +#define INCLUDECRYPTINGCODE_IFCRYPTALLOWED +#include "crypt.h" +#endif + +local linkedlist_datablock_internal* allocate_new_datablock() +{ + linkedlist_datablock_internal* ldi; + ldi = (linkedlist_datablock_internal*) + ALLOC(sizeof(linkedlist_datablock_internal)); + if (ldi!=NULL) + { + ldi->next_datablock = NULL ; + ldi->filled_in_this_block = 0 ; + ldi->avail_in_this_block = SIZEDATA_INDATABLOCK ; + } + return ldi; +} + +local void free_datablock(linkedlist_datablock_internal* ldi) +{ + while (ldi!=NULL) + { + linkedlist_datablock_internal* ldinext = ldi->next_datablock; + TRYFREE(ldi); + ldi = ldinext; + } +} + +local void init_linkedlist(linkedlist_data* ll) +{ + ll->first_block = ll->last_block = NULL; +} + +local void free_linkedlist(linkedlist_data* ll) +{ + free_datablock(ll->first_block); + ll->first_block = ll->last_block = NULL; +} + + +local int add_data_in_datablock(linkedlist_data* ll, const void* buf, uLong len) +{ + linkedlist_datablock_internal* ldi; + const unsigned char* from_copy; + + if (ll==NULL) + return ZIP_INTERNALERROR; + + if (ll->last_block == NULL) + { + ll->first_block = ll->last_block = allocate_new_datablock(); + if (ll->first_block == NULL) + return ZIP_INTERNALERROR; + } + + ldi = ll->last_block; + from_copy = (unsigned char*)buf; + + while (len>0) + { + uInt copy_this; + uInt i; + unsigned char* to_copy; + + if (ldi->avail_in_this_block==0) + { + ldi->next_datablock = allocate_new_datablock(); + if (ldi->next_datablock == NULL) + return ZIP_INTERNALERROR; + ldi = ldi->next_datablock ; + ll->last_block = ldi; + } + + if (ldi->avail_in_this_block < len) + copy_this = (uInt)ldi->avail_in_this_block; + else + copy_this = (uInt)len; + + to_copy = &(ldi->data[ldi->filled_in_this_block]); + + for (i=0;i<copy_this;i++) + *(to_copy+i)=*(from_copy+i); + + ldi->filled_in_this_block += copy_this; + ldi->avail_in_this_block -= copy_this; + from_copy += copy_this ; + len -= copy_this; + } + return ZIP_OK; +} + + + +/****************************************************************************/ + +#ifndef NO_ADDFILEINEXISTINGZIP +/* =========================================================================== + Inputs a long in LSB order to the given file + nbByte == 1, 2 ,4 or 8 (byte, short or long, ZPOS64_T) +*/ + +local int zip64local_putValue OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, ZPOS64_T x, int nbByte)); +local int zip64local_putValue (const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, ZPOS64_T x, int nbByte) +{ + unsigned char buf[8]; + int n; + for (n = 0; n < nbByte; n++) + { + buf[n] = (unsigned char)(x & 0xff); + x >>= 8; + } + if (x != 0) + { /* data overflow - hack for ZIP64 (X Roche) */ + for (n = 0; n < nbByte; n++) + { + buf[n] = 0xff; + } + } + + if (ZWRITE64(*pzlib_filefunc_def,filestream,buf,nbByte)!=(uLong)nbByte) + return ZIP_ERRNO; + else + return ZIP_OK; +} + +local void zip64local_putValue_inmemory OF((void* dest, ZPOS64_T x, int nbByte)); +local void zip64local_putValue_inmemory (void* dest, ZPOS64_T x, int nbByte) +{ + unsigned char* buf=(unsigned char*)dest; + int n; + for (n = 0; n < nbByte; n++) { + buf[n] = (unsigned char)(x & 0xff); + x >>= 8; + } + + if (x != 0) + { /* data overflow - hack for ZIP64 */ + for (n = 0; n < nbByte; n++) + { + buf[n] = 0xff; + } + } +} + +/****************************************************************************/ + + +local uLong zip64local_TmzDateToDosDate(const tm_zip* ptm) +{ + uLong year = (uLong)ptm->tm_year; + if (year>=1980) + year-=1980; + else if (year>=80) + year-=80; + return + (uLong) (((ptm->tm_mday) + (32 * (ptm->tm_mon+1)) + (512 * year)) << 16) | + ((ptm->tm_sec/2) + (32* ptm->tm_min) + (2048 * (uLong)ptm->tm_hour)); +} + + +/****************************************************************************/ + +local int zip64local_getByte OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, int *pi)); + +local int zip64local_getByte(const zlib_filefunc64_32_def* pzlib_filefunc_def,voidpf filestream,int* pi) +{ + unsigned char c; + int err = (int)ZREAD64(*pzlib_filefunc_def,filestream,&c,1); + if (err==1) + { + *pi = (int)c; + return ZIP_OK; + } + else + { + if (ZERROR64(*pzlib_filefunc_def,filestream)) + return ZIP_ERRNO; + else + return ZIP_EOF; + } +} + + +/* =========================================================================== + Reads a long in LSB order from the given gz_stream. Sets +*/ +local int zip64local_getShort OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, uLong *pX)); + +local int zip64local_getShort (const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, uLong* pX) +{ + uLong x ; + int i = 0; + int err; + + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x = (uLong)i; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((uLong)i)<<8; + + if (err==ZIP_OK) + *pX = x; + else + *pX = 0; + return err; +} + +local int zip64local_getLong OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, uLong *pX)); + +local int zip64local_getLong (const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, uLong* pX) +{ + uLong x ; + int i = 0; + int err; + + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x = (uLong)i; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((uLong)i)<<8; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((uLong)i)<<16; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((uLong)i)<<24; + + if (err==ZIP_OK) + *pX = x; + else + *pX = 0; + return err; +} + +local int zip64local_getLong64 OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, ZPOS64_T *pX)); + + +local int zip64local_getLong64 (const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream, ZPOS64_T *pX) +{ + ZPOS64_T x; + int i = 0; + int err; + + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x = (ZPOS64_T)i; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((ZPOS64_T)i)<<8; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((ZPOS64_T)i)<<16; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((ZPOS64_T)i)<<24; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((ZPOS64_T)i)<<32; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((ZPOS64_T)i)<<40; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((ZPOS64_T)i)<<48; + + if (err==ZIP_OK) + err = zip64local_getByte(pzlib_filefunc_def,filestream,&i); + x += ((ZPOS64_T)i)<<56; + + if (err==ZIP_OK) + *pX = x; + else + *pX = 0; + + return err; +} + +#ifndef BUFREADCOMMENT +#define BUFREADCOMMENT (0x400) +#endif +/* + Locate the Central directory of a zipfile (at the end, just before + the global comment) +*/ +local ZPOS64_T zip64local_SearchCentralDir OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream)); + +local ZPOS64_T zip64local_SearchCentralDir(const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream) +{ + unsigned char* buf; + ZPOS64_T uSizeFile; + ZPOS64_T uBackRead; + ZPOS64_T uMaxBack=0xffff; /* maximum size of global comment */ + ZPOS64_T uPosFound=0; + + if (ZSEEK64(*pzlib_filefunc_def,filestream,0,ZLIB_FILEFUNC_SEEK_END) != 0) + return 0; + + + uSizeFile = ZTELL64(*pzlib_filefunc_def,filestream); + + if (uMaxBack>uSizeFile) + uMaxBack = uSizeFile; + + buf = (unsigned char*)ALLOC(BUFREADCOMMENT+4); + if (buf==NULL) + return 0; + + uBackRead = 4; + while (uBackRead<uMaxBack) + { + uLong uReadSize; + ZPOS64_T uReadPos ; + int i; + if (uBackRead+BUFREADCOMMENT>uMaxBack) + uBackRead = uMaxBack; + else + uBackRead+=BUFREADCOMMENT; + uReadPos = uSizeFile-uBackRead ; + + uReadSize = ((BUFREADCOMMENT+4) < (uSizeFile-uReadPos)) ? + (BUFREADCOMMENT+4) : (uLong)(uSizeFile-uReadPos); + if (ZSEEK64(*pzlib_filefunc_def,filestream,uReadPos,ZLIB_FILEFUNC_SEEK_SET)!=0) + break; + + if (ZREAD64(*pzlib_filefunc_def,filestream,buf,uReadSize)!=uReadSize) + break; + + for (i=(int)uReadSize-3; (i--)>0;) + if (((*(buf+i))==0x50) && ((*(buf+i+1))==0x4b) && + ((*(buf+i+2))==0x05) && ((*(buf+i+3))==0x06)) + { + uPosFound = uReadPos+i; + break; + } + + if (uPosFound!=0) + break; + } + TRYFREE(buf); + return uPosFound; +} + +/* +Locate the End of Zip64 Central directory locator and from there find the CD of a zipfile (at the end, just before +the global comment) +*/ +local ZPOS64_T zip64local_SearchCentralDir64 OF((const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream)); + +local ZPOS64_T zip64local_SearchCentralDir64(const zlib_filefunc64_32_def* pzlib_filefunc_def, voidpf filestream) +{ + unsigned char* buf; + ZPOS64_T uSizeFile; + ZPOS64_T uBackRead; + ZPOS64_T uMaxBack=0xffff; /* maximum size of global comment */ + ZPOS64_T uPosFound=0; + uLong uL; + ZPOS64_T relativeOffset; + + if (ZSEEK64(*pzlib_filefunc_def,filestream,0,ZLIB_FILEFUNC_SEEK_END) != 0) + return 0; + + uSizeFile = ZTELL64(*pzlib_filefunc_def,filestream); + + if (uMaxBack>uSizeFile) + uMaxBack = uSizeFile; + + buf = (unsigned char*)ALLOC(BUFREADCOMMENT+4); + if (buf==NULL) + return 0; + + uBackRead = 4; + while (uBackRead<uMaxBack) + { + uLong uReadSize; + ZPOS64_T uReadPos; + int i; + if (uBackRead+BUFREADCOMMENT>uMaxBack) + uBackRead = uMaxBack; + else + uBackRead+=BUFREADCOMMENT; + uReadPos = uSizeFile-uBackRead ; + + uReadSize = ((BUFREADCOMMENT+4) < (uSizeFile-uReadPos)) ? + (BUFREADCOMMENT+4) : (uLong)(uSizeFile-uReadPos); + if (ZSEEK64(*pzlib_filefunc_def,filestream,uReadPos,ZLIB_FILEFUNC_SEEK_SET)!=0) + break; + + if (ZREAD64(*pzlib_filefunc_def,filestream,buf,uReadSize)!=uReadSize) + break; + + for (i=(int)uReadSize-3; (i--)>0;) + { + // Signature "0x07064b50" Zip64 end of central directory locater + if (((*(buf+i))==0x50) && ((*(buf+i+1))==0x4b) && ((*(buf+i+2))==0x06) && ((*(buf+i+3))==0x07)) + { + uPosFound = uReadPos+i; + break; + } + } + + if (uPosFound!=0) + break; + } + + TRYFREE(buf); + if (uPosFound == 0) + return 0; + + /* Zip64 end of central directory locator */ + if (ZSEEK64(*pzlib_filefunc_def,filestream, uPosFound,ZLIB_FILEFUNC_SEEK_SET)!=0) + return 0; + + /* the signature, already checked */ + if (zip64local_getLong(pzlib_filefunc_def,filestream,&uL)!=ZIP_OK) + return 0; + + /* number of the disk with the start of the zip64 end of central directory */ + if (zip64local_getLong(pzlib_filefunc_def,filestream,&uL)!=ZIP_OK) + return 0; + if (uL != 0) + return 0; + + /* relative offset of the zip64 end of central directory record */ + if (zip64local_getLong64(pzlib_filefunc_def,filestream,&relativeOffset)!=ZIP_OK) + return 0; + + /* total number of disks */ + if (zip64local_getLong(pzlib_filefunc_def,filestream,&uL)!=ZIP_OK) + return 0; + if (uL != 1) + return 0; + + /* Goto Zip64 end of central directory record */ + if (ZSEEK64(*pzlib_filefunc_def,filestream, relativeOffset,ZLIB_FILEFUNC_SEEK_SET)!=0) + return 0; + + /* the signature */ + if (zip64local_getLong(pzlib_filefunc_def,filestream,&uL)!=ZIP_OK) + return 0; + + if (uL != 0x06064b50) // signature of 'Zip64 end of central directory' + return 0; + + return relativeOffset; +} + +int LoadCentralDirectoryRecord(zip64_internal* pziinit) +{ + int err=ZIP_OK; + ZPOS64_T byte_before_the_zipfile;/* byte before the zipfile, (>0 for sfx)*/ + + ZPOS64_T size_central_dir; /* size of the central directory */ + ZPOS64_T offset_central_dir; /* offset of start of central directory */ + ZPOS64_T central_pos; + uLong uL; + + uLong number_disk; /* number of the current dist, used for + spaning ZIP, unsupported, always 0*/ + uLong number_disk_with_CD; /* number the the disk with central dir, used + for spaning ZIP, unsupported, always 0*/ + ZPOS64_T number_entry; + ZPOS64_T number_entry_CD; /* total number of entries in + the central dir + (same than number_entry on nospan) */ + uLong VersionMadeBy; + uLong VersionNeeded; + uLong size_comment; + + int hasZIP64Record = 0; + + // check first if we find a ZIP64 record + central_pos = zip64local_SearchCentralDir64(&pziinit->z_filefunc,pziinit->filestream); + if(central_pos > 0) + { + hasZIP64Record = 1; + } + else if(central_pos == 0) + { + central_pos = zip64local_SearchCentralDir(&pziinit->z_filefunc,pziinit->filestream); + } + +/* disable to allow appending to empty ZIP archive + if (central_pos==0) + err=ZIP_ERRNO; +*/ + + if(hasZIP64Record) + { + ZPOS64_T sizeEndOfCentralDirectory; + if (ZSEEK64(pziinit->z_filefunc, pziinit->filestream, central_pos, ZLIB_FILEFUNC_SEEK_SET) != 0) + err=ZIP_ERRNO; + + /* the signature, already checked */ + if (zip64local_getLong(&pziinit->z_filefunc, pziinit->filestream,&uL)!=ZIP_OK) + err=ZIP_ERRNO; + + /* size of zip64 end of central directory record */ + if (zip64local_getLong64(&pziinit->z_filefunc, pziinit->filestream, &sizeEndOfCentralDirectory)!=ZIP_OK) + err=ZIP_ERRNO; + + /* version made by */ + if (zip64local_getShort(&pziinit->z_filefunc, pziinit->filestream, &VersionMadeBy)!=ZIP_OK) + err=ZIP_ERRNO; + + /* version needed to extract */ + if (zip64local_getShort(&pziinit->z_filefunc, pziinit->filestream, &VersionNeeded)!=ZIP_OK) + err=ZIP_ERRNO; + + /* number of this disk */ + if (zip64local_getLong(&pziinit->z_filefunc, pziinit->filestream,&number_disk)!=ZIP_OK) + err=ZIP_ERRNO; + + /* number of the disk with the start of the central directory */ + if (zip64local_getLong(&pziinit->z_filefunc, pziinit->filestream,&number_disk_with_CD)!=ZIP_OK) + err=ZIP_ERRNO; + + /* total number of entries in the central directory on this disk */ + if (zip64local_getLong64(&pziinit->z_filefunc, pziinit->filestream, &number_entry)!=ZIP_OK) + err=ZIP_ERRNO; + + /* total number of entries in the central directory */ + if (zip64local_getLong64(&pziinit->z_filefunc, pziinit->filestream,&number_entry_CD)!=ZIP_OK) + err=ZIP_ERRNO; + + if ((number_entry_CD!=number_entry) || (number_disk_with_CD!=0) || (number_disk!=0)) + err=ZIP_BADZIPFILE; + + /* size of the central directory */ + if (zip64local_getLong64(&pziinit->z_filefunc, pziinit->filestream,&size_central_dir)!=ZIP_OK) + err=ZIP_ERRNO; + + /* offset of start of central directory with respect to the + starting disk number */ + if (zip64local_getLong64(&pziinit->z_filefunc, pziinit->filestream,&offset_central_dir)!=ZIP_OK) + err=ZIP_ERRNO; + + // TODO.. + // read the comment from the standard central header. + size_comment = 0; + } + else + { + // Read End of central Directory info + if (ZSEEK64(pziinit->z_filefunc, pziinit->filestream, central_pos,ZLIB_FILEFUNC_SEEK_SET)!=0) + err=ZIP_ERRNO; + + /* the signature, already checked */ + if (zip64local_getLong(&pziinit->z_filefunc, pziinit->filestream,&uL)!=ZIP_OK) + err=ZIP_ERRNO; + + /* number of this disk */ + if (zip64local_getShort(&pziinit->z_filefunc, pziinit->filestream,&number_disk)!=ZIP_OK) + err=ZIP_ERRNO; + + /* number of the disk with the start of the central directory */ + if (zip64local_getShort(&pziinit->z_filefunc, pziinit->filestream,&number_disk_with_CD)!=ZIP_OK) + err=ZIP_ERRNO; + + /* total number of entries in the central dir on this disk */ + number_entry = 0; + if (zip64local_getShort(&pziinit->z_filefunc, pziinit->filestream, &uL)!=ZIP_OK) + err=ZIP_ERRNO; + else + number_entry = uL; + + /* total number of entries in the central dir */ + number_entry_CD = 0; + if (zip64local_getShort(&pziinit->z_filefunc, pziinit->filestream, &uL)!=ZIP_OK) + err=ZIP_ERRNO; + else + number_entry_CD = uL; + + if ((number_entry_CD!=number_entry) || (number_disk_with_CD!=0) || (number_disk!=0)) + err=ZIP_BADZIPFILE; + + /* size of the central directory */ + size_central_dir = 0; + if (zip64local_getLong(&pziinit->z_filefunc, pziinit->filestream, &uL)!=ZIP_OK) + err=ZIP_ERRNO; + else + size_central_dir = uL; + + /* offset of start of central directory with respect to the starting disk number */ + offset_central_dir = 0; + if (zip64local_getLong(&pziinit->z_filefunc, pziinit->filestream, &uL)!=ZIP_OK) + err=ZIP_ERRNO; + else + offset_central_dir = uL; + + + /* zipfile global comment length */ + if (zip64local_getShort(&pziinit->z_filefunc, pziinit->filestream, &size_comment)!=ZIP_OK) + err=ZIP_ERRNO; + } + + if ((central_pos<offset_central_dir+size_central_dir) && + (err==ZIP_OK)) + err=ZIP_BADZIPFILE; + + if (err!=ZIP_OK) + { + ZCLOSE64(pziinit->z_filefunc, pziinit->filestream); + return ZIP_ERRNO; + } + + if (size_comment>0) + { + pziinit->globalcomment = (char*)ALLOC(size_comment+1); + if (pziinit->globalcomment) + { + size_comment = ZREAD64(pziinit->z_filefunc, pziinit->filestream, pziinit->globalcomment,size_comment); + pziinit->globalcomment[size_comment]=0; + } + } + + byte_before_the_zipfile = central_pos - (offset_central_dir+size_central_dir); + pziinit->add_position_when_writing_offset = byte_before_the_zipfile; + + { + ZPOS64_T size_central_dir_to_read = size_central_dir; + size_t buf_size = SIZEDATA_INDATABLOCK; + void* buf_read = (void*)ALLOC(buf_size); + if (ZSEEK64(pziinit->z_filefunc, pziinit->filestream, offset_central_dir + byte_before_the_zipfile, ZLIB_FILEFUNC_SEEK_SET) != 0) + err=ZIP_ERRNO; + + while ((size_central_dir_to_read>0) && (err==ZIP_OK)) + { + ZPOS64_T read_this = SIZEDATA_INDATABLOCK; + if (read_this > size_central_dir_to_read) + read_this = size_central_dir_to_read; + + if (ZREAD64(pziinit->z_filefunc, pziinit->filestream,buf_read,(uLong)read_this) != read_this) + err=ZIP_ERRNO; + + if (err==ZIP_OK) + err = add_data_in_datablock(&pziinit->central_dir,buf_read, (uLong)read_this); + + size_central_dir_to_read-=read_this; + } + TRYFREE(buf_read); + } + pziinit->begin_pos = byte_before_the_zipfile; + pziinit->number_entry = number_entry_CD; + + if (ZSEEK64(pziinit->z_filefunc, pziinit->filestream, offset_central_dir+byte_before_the_zipfile,ZLIB_FILEFUNC_SEEK_SET) != 0) + err=ZIP_ERRNO; + + return err; +} + + +#endif /* !NO_ADDFILEINEXISTINGZIP*/ + + +/************************************************************/ +extern zipFile ZEXPORT zipOpen3 (const void *pathname, int append, zipcharpc* globalcomment, zlib_filefunc64_32_def* pzlib_filefunc64_32_def) +{ + zip64_internal ziinit; + zip64_internal* zi; + int err=ZIP_OK; + + ziinit.z_filefunc.zseek32_file = NULL; + ziinit.z_filefunc.ztell32_file = NULL; + if (pzlib_filefunc64_32_def==NULL) + fill_fopen64_filefunc(&ziinit.z_filefunc.zfile_func64); + else + ziinit.z_filefunc = *pzlib_filefunc64_32_def; + + ziinit.filestream = ZOPEN64(ziinit.z_filefunc, + pathname, + (append == APPEND_STATUS_CREATE) ? + (ZLIB_FILEFUNC_MODE_READ | ZLIB_FILEFUNC_MODE_WRITE | ZLIB_FILEFUNC_MODE_CREATE) : + (ZLIB_FILEFUNC_MODE_READ | ZLIB_FILEFUNC_MODE_WRITE | ZLIB_FILEFUNC_MODE_EXISTING)); + + if (ziinit.filestream == NULL) + return NULL; + + if (append == APPEND_STATUS_CREATEAFTER) + ZSEEK64(ziinit.z_filefunc,ziinit.filestream,0,SEEK_END); + + ziinit.begin_pos = ZTELL64(ziinit.z_filefunc,ziinit.filestream); + ziinit.in_opened_file_inzip = 0; + ziinit.ci.stream_initialised = 0; + ziinit.number_entry = 0; + ziinit.add_position_when_writing_offset = 0; + init_linkedlist(&(ziinit.central_dir)); + + + + zi = (zip64_internal*)ALLOC(sizeof(zip64_internal)); + if (zi==NULL) + { + ZCLOSE64(ziinit.z_filefunc,ziinit.filestream); + return NULL; + } + + /* now we add file in a zipfile */ +# ifndef NO_ADDFILEINEXISTINGZIP + ziinit.globalcomment = NULL; + if (append == APPEND_STATUS_ADDINZIP) + { + // Read and Cache Central Directory Records + err = LoadCentralDirectoryRecord(&ziinit); + } + + if (globalcomment) + { + *globalcomment = ziinit.globalcomment; + } +# endif /* !NO_ADDFILEINEXISTINGZIP*/ + + if (err != ZIP_OK) + { +# ifndef NO_ADDFILEINEXISTINGZIP + TRYFREE(ziinit.globalcomment); +# endif /* !NO_ADDFILEINEXISTINGZIP*/ + TRYFREE(zi); + return NULL; + } + else + { + *zi = ziinit; + return (zipFile)zi; + } +} + +extern zipFile ZEXPORT zipOpen2 (const char *pathname, int append, zipcharpc* globalcomment, zlib_filefunc_def* pzlib_filefunc32_def) +{ + if (pzlib_filefunc32_def != NULL) + { + zlib_filefunc64_32_def zlib_filefunc64_32_def_fill; + fill_zlib_filefunc64_32_def_from_filefunc32(&zlib_filefunc64_32_def_fill,pzlib_filefunc32_def); + return zipOpen3(pathname, append, globalcomment, &zlib_filefunc64_32_def_fill); + } + else + return zipOpen3(pathname, append, globalcomment, NULL); +} + +extern zipFile ZEXPORT zipOpen2_64 (const void *pathname, int append, zipcharpc* globalcomment, zlib_filefunc64_def* pzlib_filefunc_def) +{ + if (pzlib_filefunc_def != NULL) + { + zlib_filefunc64_32_def zlib_filefunc64_32_def_fill; + zlib_filefunc64_32_def_fill.zfile_func64 = *pzlib_filefunc_def; + zlib_filefunc64_32_def_fill.ztell32_file = NULL; + zlib_filefunc64_32_def_fill.zseek32_file = NULL; + return zipOpen3(pathname, append, globalcomment, &zlib_filefunc64_32_def_fill); + } + else + return zipOpen3(pathname, append, globalcomment, NULL); +} + + + +extern zipFile ZEXPORT zipOpen (const char* pathname, int append) +{ + return zipOpen3((const void*)pathname,append,NULL,NULL); +} + +extern zipFile ZEXPORT zipOpen64 (const void* pathname, int append) +{ + return zipOpen3(pathname,append,NULL,NULL); +} + +int Write_LocalFileHeader(zip64_internal* zi, const char* filename, uInt size_extrafield_local, const void* extrafield_local) +{ + /* write the local header */ + int err; + uInt size_filename = (uInt)strlen(filename); + uInt size_extrafield = size_extrafield_local; + + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)LOCALHEADERMAGIC, 4); + + if (err==ZIP_OK) + { + if(zi->ci.zip64) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)45,2);/* version needed to extract */ + else + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)20,2);/* version needed to extract */ + } + + if (err==ZIP_OK) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)zi->ci.flag,2); + + if (err==ZIP_OK) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)zi->ci.method,2); + + if (err==ZIP_OK) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)zi->ci.dosDate,4); + + // CRC / Compressed size / Uncompressed size will be filled in later and rewritten later + if (err==ZIP_OK) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,4); /* crc 32, unknown */ + if (err==ZIP_OK) + { + if(zi->ci.zip64) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0xFFFFFFFF,4); /* compressed size, unknown */ + else + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,4); /* compressed size, unknown */ + } + if (err==ZIP_OK) + { + if(zi->ci.zip64) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0xFFFFFFFF,4); /* uncompressed size, unknown */ + else + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,4); /* uncompressed size, unknown */ + } + + if (err==ZIP_OK) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)size_filename,2); + + if(zi->ci.zip64) + { + size_extrafield += 20; + } + + if (err==ZIP_OK) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)size_extrafield,2); + + if ((err==ZIP_OK) && (size_filename > 0)) + { + if (ZWRITE64(zi->z_filefunc,zi->filestream,filename,size_filename)!=size_filename) + err = ZIP_ERRNO; + } + + if ((err==ZIP_OK) && (size_extrafield_local > 0)) + { + if (ZWRITE64(zi->z_filefunc, zi->filestream, extrafield_local, size_extrafield_local) != size_extrafield_local) + err = ZIP_ERRNO; + } + + + if ((err==ZIP_OK) && (zi->ci.zip64)) + { + // write the Zip64 extended info + short HeaderID = 1; + short DataSize = 16; + ZPOS64_T CompressedSize = 0; + ZPOS64_T UncompressedSize = 0; + + // Remember position of Zip64 extended info for the local file header. (needed when we update size after done with file) + zi->ci.pos_zip64extrainfo = ZTELL64(zi->z_filefunc,zi->filestream); + + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, (short)HeaderID,2); + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, (short)DataSize,2); + + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, (ZPOS64_T)UncompressedSize,8); + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, (ZPOS64_T)CompressedSize,8); + } + + return err; +} + +/* + NOTE. + When writing RAW the ZIP64 extended information in extrafield_local and extrafield_global needs to be stripped + before calling this function it can be done with zipRemoveExtraInfoBlock + + It is not done here because then we need to realloc a new buffer since parameters are 'const' and I want to minimize + unnecessary allocations. + */ +extern int ZEXPORT zipOpenNewFileInZip4_64 (zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void* extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int raw, + int windowBits,int memLevel, int strategy, + const char* password, uLong crcForCrypting, + uLong versionMadeBy, uLong flagBase, int zip64) +{ + zip64_internal* zi; + uInt size_filename; + uInt size_comment; + uInt i; + int err = ZIP_OK; + +# ifdef NOCRYPT + (crcForCrypting); + if (password != NULL) + return ZIP_PARAMERROR; +# endif + + if (file == NULL) + return ZIP_PARAMERROR; + +#ifdef HAVE_BZIP2 + if ((method!=0) && (method!=Z_DEFLATED) && (method!=Z_BZIP2ED)) + return ZIP_PARAMERROR; +#else + if ((method!=0) && (method!=Z_DEFLATED)) + return ZIP_PARAMERROR; +#endif + + zi = (zip64_internal*)file; + + if (zi->in_opened_file_inzip == 1) + { + err = zipCloseFileInZip (file); + if (err != ZIP_OK) + return err; + } + + if (filename==NULL) + filename="-"; + + if (comment==NULL) + size_comment = 0; + else + size_comment = (uInt)strlen(comment); + + size_filename = (uInt)strlen(filename); + + if (zipfi == NULL) + zi->ci.dosDate = 0; + else + { + if (zipfi->dosDate != 0) + zi->ci.dosDate = zipfi->dosDate; + else + zi->ci.dosDate = zip64local_TmzDateToDosDate(&zipfi->tmz_date); + } + + zi->ci.flag = flagBase; + if ((level==8) || (level==9)) + zi->ci.flag |= 2; + if (level==2) + zi->ci.flag |= 4; + if (level==1) + zi->ci.flag |= 6; + if (password != NULL) + zi->ci.flag |= 1; + + zi->ci.crc32 = 0; + zi->ci.method = method; + zi->ci.encrypt = 0; + zi->ci.stream_initialised = 0; + zi->ci.pos_in_buffered_data = 0; + zi->ci.raw = raw; + zi->ci.pos_local_header = ZTELL64(zi->z_filefunc,zi->filestream); + + zi->ci.size_centralheader = SIZECENTRALHEADER + size_filename + size_extrafield_global + size_comment; + zi->ci.size_centralExtraFree = 32; // Extra space we have reserved in case we need to add ZIP64 extra info data + + zi->ci.central_header = (char*)ALLOC((uInt)zi->ci.size_centralheader + zi->ci.size_centralExtraFree); + + zi->ci.size_centralExtra = size_extrafield_global; + zip64local_putValue_inmemory(zi->ci.central_header,(uLong)CENTRALHEADERMAGIC,4); + /* version info */ + zip64local_putValue_inmemory(zi->ci.central_header+4,(uLong)versionMadeBy,2); + zip64local_putValue_inmemory(zi->ci.central_header+6,(uLong)20,2); + zip64local_putValue_inmemory(zi->ci.central_header+8,(uLong)zi->ci.flag,2); + zip64local_putValue_inmemory(zi->ci.central_header+10,(uLong)zi->ci.method,2); + zip64local_putValue_inmemory(zi->ci.central_header+12,(uLong)zi->ci.dosDate,4); + zip64local_putValue_inmemory(zi->ci.central_header+16,(uLong)0,4); /*crc*/ + zip64local_putValue_inmemory(zi->ci.central_header+20,(uLong)0,4); /*compr size*/ + zip64local_putValue_inmemory(zi->ci.central_header+24,(uLong)0,4); /*uncompr size*/ + zip64local_putValue_inmemory(zi->ci.central_header+28,(uLong)size_filename,2); + zip64local_putValue_inmemory(zi->ci.central_header+30,(uLong)size_extrafield_global,2); + zip64local_putValue_inmemory(zi->ci.central_header+32,(uLong)size_comment,2); + zip64local_putValue_inmemory(zi->ci.central_header+34,(uLong)0,2); /*disk nm start*/ + + if (zipfi==NULL) + zip64local_putValue_inmemory(zi->ci.central_header+36,(uLong)0,2); + else + zip64local_putValue_inmemory(zi->ci.central_header+36,(uLong)zipfi->internal_fa,2); + + if (zipfi==NULL) + zip64local_putValue_inmemory(zi->ci.central_header+38,(uLong)0,4); + else + zip64local_putValue_inmemory(zi->ci.central_header+38,(uLong)zipfi->external_fa,4); + + if(zi->ci.pos_local_header >= 0xffffffff) + zip64local_putValue_inmemory(zi->ci.central_header+42,(uLong)0xffffffff,4); + else + zip64local_putValue_inmemory(zi->ci.central_header+42,(uLong)zi->ci.pos_local_header - zi->add_position_when_writing_offset,4); + + for (i=0;i<size_filename;i++) + *(zi->ci.central_header+SIZECENTRALHEADER+i) = *(filename+i); + + for (i=0;i<size_extrafield_global;i++) + *(zi->ci.central_header+SIZECENTRALHEADER+size_filename+i) = + *(((const char*)extrafield_global)+i); + + for (i=0;i<size_comment;i++) + *(zi->ci.central_header+SIZECENTRALHEADER+size_filename+ + size_extrafield_global+i) = *(comment+i); + if (zi->ci.central_header == NULL) + return ZIP_INTERNALERROR; + + zi->ci.zip64 = zip64; + zi->ci.totalCompressedData = 0; + zi->ci.totalUncompressedData = 0; + zi->ci.pos_zip64extrainfo = 0; + + err = Write_LocalFileHeader(zi, filename, size_extrafield_local, extrafield_local); + +#ifdef HAVE_BZIP2 + zi->ci.bstream.avail_in = (uInt)0; + zi->ci.bstream.avail_out = (uInt)Z_BUFSIZE; + zi->ci.bstream.next_out = (char*)zi->ci.buffered_data; + zi->ci.bstream.total_in_hi32 = 0; + zi->ci.bstream.total_in_lo32 = 0; + zi->ci.bstream.total_out_hi32 = 0; + zi->ci.bstream.total_out_lo32 = 0; +#endif + + zi->ci.stream.avail_in = (uInt)0; + zi->ci.stream.avail_out = (uInt)Z_BUFSIZE; + zi->ci.stream.next_out = zi->ci.buffered_data; + zi->ci.stream.total_in = 0; + zi->ci.stream.total_out = 0; + zi->ci.stream.data_type = Z_BINARY; + +#ifdef HAVE_BZIP2 + if ((err==ZIP_OK) && (zi->ci.method == Z_DEFLATED || zi->ci.method == Z_BZIP2ED) && (!zi->ci.raw)) +#else + if ((err==ZIP_OK) && (zi->ci.method == Z_DEFLATED) && (!zi->ci.raw)) +#endif + { + if(zi->ci.method == Z_DEFLATED) + { + zi->ci.stream.zalloc = (alloc_func)0; + zi->ci.stream.zfree = (free_func)0; + zi->ci.stream.opaque = (voidpf)0; + + if (windowBits>0) + windowBits = -windowBits; + + err = deflateInit2(&zi->ci.stream, level, Z_DEFLATED, windowBits, memLevel, strategy); + + if (err==Z_OK) + zi->ci.stream_initialised = Z_DEFLATED; + } + else if(zi->ci.method == Z_BZIP2ED) + { +#ifdef HAVE_BZIP2 + // Init BZip stuff here + zi->ci.bstream.bzalloc = 0; + zi->ci.bstream.bzfree = 0; + zi->ci.bstream.opaque = (voidpf)0; + + err = BZ2_bzCompressInit(&zi->ci.bstream, level, 0,35); + if(err == BZ_OK) + zi->ci.stream_initialised = Z_BZIP2ED; +#endif + } + + } + +# ifndef NOCRYPT + zi->ci.crypt_header_size = 0; + if ((err==Z_OK) && (password != NULL)) + { + unsigned char bufHead[RAND_HEAD_LEN]; + unsigned int sizeHead; + zi->ci.encrypt = 1; + zi->ci.pcrc_32_tab = get_crc_table(); + /*init_keys(password,zi->ci.keys,zi->ci.pcrc_32_tab);*/ + + sizeHead=crypthead(password,bufHead,RAND_HEAD_LEN,zi->ci.keys,zi->ci.pcrc_32_tab,crcForCrypting); + zi->ci.crypt_header_size = sizeHead; + + if (ZWRITE64(zi->z_filefunc,zi->filestream,bufHead,sizeHead) != sizeHead) + err = ZIP_ERRNO; + } +# endif + + if (err==Z_OK) + zi->in_opened_file_inzip = 1; + return err; +} + +extern int ZEXPORT zipOpenNewFileInZip4 (zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void* extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int raw, + int windowBits,int memLevel, int strategy, + const char* password, uLong crcForCrypting, + uLong versionMadeBy, uLong flagBase) +{ + return zipOpenNewFileInZip4_64 (file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, raw, + windowBits, memLevel, strategy, + password, crcForCrypting, versionMadeBy, flagBase, 0); +} + +extern int ZEXPORT zipOpenNewFileInZip3 (zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void* extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int raw, + int windowBits,int memLevel, int strategy, + const char* password, uLong crcForCrypting) +{ + return zipOpenNewFileInZip4_64 (file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, raw, + windowBits, memLevel, strategy, + password, crcForCrypting, VERSIONMADEBY, 0, 0); +} + +extern int ZEXPORT zipOpenNewFileInZip3_64(zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void* extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int raw, + int windowBits,int memLevel, int strategy, + const char* password, uLong crcForCrypting, int zip64) +{ + return zipOpenNewFileInZip4_64 (file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, raw, + windowBits, memLevel, strategy, + password, crcForCrypting, VERSIONMADEBY, 0, zip64); +} + +extern int ZEXPORT zipOpenNewFileInZip2(zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void* extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int raw) +{ + return zipOpenNewFileInZip4_64 (file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, raw, + -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY, + NULL, 0, VERSIONMADEBY, 0, 0); +} + +extern int ZEXPORT zipOpenNewFileInZip2_64(zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void* extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int raw, int zip64) +{ + return zipOpenNewFileInZip4_64 (file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, raw, + -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY, + NULL, 0, VERSIONMADEBY, 0, zip64); +} + +extern int ZEXPORT zipOpenNewFileInZip64 (zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void*extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level, int zip64) +{ + return zipOpenNewFileInZip4_64 (file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, 0, + -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY, + NULL, 0, VERSIONMADEBY, 0, zip64); +} + +extern int ZEXPORT zipOpenNewFileInZip (zipFile file, const char* filename, const zip_fileinfo* zipfi, + const void* extrafield_local, uInt size_extrafield_local, + const void*extrafield_global, uInt size_extrafield_global, + const char* comment, int method, int level) +{ + return zipOpenNewFileInZip4_64 (file, filename, zipfi, + extrafield_local, size_extrafield_local, + extrafield_global, size_extrafield_global, + comment, method, level, 0, + -MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY, + NULL, 0, VERSIONMADEBY, 0, 0); +} + +local int zip64FlushWriteBuffer(zip64_internal* zi) +{ + int err=ZIP_OK; + + if (zi->ci.encrypt != 0) + { +#ifndef NOCRYPT + uInt i; + int t; + for (i=0;i<zi->ci.pos_in_buffered_data;i++) + zi->ci.buffered_data[i] = zencode(zi->ci.keys, zi->ci.pcrc_32_tab, zi->ci.buffered_data[i],t); +#endif + } + + if (ZWRITE64(zi->z_filefunc,zi->filestream,zi->ci.buffered_data,zi->ci.pos_in_buffered_data) != zi->ci.pos_in_buffered_data) + err = ZIP_ERRNO; + + zi->ci.totalCompressedData += zi->ci.pos_in_buffered_data; + +#ifdef HAVE_BZIP2 + if(zi->ci.method == Z_BZIP2ED) + { + zi->ci.totalUncompressedData += zi->ci.bstream.total_in_lo32; + zi->ci.bstream.total_in_lo32 = 0; + zi->ci.bstream.total_in_hi32 = 0; + } + else +#endif + { + zi->ci.totalUncompressedData += zi->ci.stream.total_in; + zi->ci.stream.total_in = 0; + } + + + zi->ci.pos_in_buffered_data = 0; + + return err; +} + +extern int ZEXPORT zipWriteInFileInZip (zipFile file,const void* buf,unsigned int len) +{ + zip64_internal* zi; + int err=ZIP_OK; + + if (file == NULL) + return ZIP_PARAMERROR; + zi = (zip64_internal*)file; + + if (zi->in_opened_file_inzip == 0) + return ZIP_PARAMERROR; + + zi->ci.crc32 = crc32(zi->ci.crc32,buf,(uInt)len); + +#ifdef HAVE_BZIP2 + if(zi->ci.method == Z_BZIP2ED && (!zi->ci.raw)) + { + zi->ci.bstream.next_in = (void*)buf; + zi->ci.bstream.avail_in = len; + err = BZ_RUN_OK; + + while ((err==BZ_RUN_OK) && (zi->ci.bstream.avail_in>0)) + { + if (zi->ci.bstream.avail_out == 0) + { + if (zip64FlushWriteBuffer(zi) == ZIP_ERRNO) + err = ZIP_ERRNO; + zi->ci.bstream.avail_out = (uInt)Z_BUFSIZE; + zi->ci.bstream.next_out = (char*)zi->ci.buffered_data; + } + + + if(err != BZ_RUN_OK) + break; + + if ((zi->ci.method == Z_BZIP2ED) && (!zi->ci.raw)) + { + uLong uTotalOutBefore_lo = zi->ci.bstream.total_out_lo32; +// uLong uTotalOutBefore_hi = zi->ci.bstream.total_out_hi32; + err=BZ2_bzCompress(&zi->ci.bstream, BZ_RUN); + + zi->ci.pos_in_buffered_data += (uInt)(zi->ci.bstream.total_out_lo32 - uTotalOutBefore_lo) ; + } + } + + if(err == BZ_RUN_OK) + err = ZIP_OK; + } + else +#endif + { + zi->ci.stream.next_in = (Bytef*)buf; + zi->ci.stream.avail_in = len; + + while ((err==ZIP_OK) && (zi->ci.stream.avail_in>0)) + { + if (zi->ci.stream.avail_out == 0) + { + if (zip64FlushWriteBuffer(zi) == ZIP_ERRNO) + err = ZIP_ERRNO; + zi->ci.stream.avail_out = (uInt)Z_BUFSIZE; + zi->ci.stream.next_out = zi->ci.buffered_data; + } + + + if(err != ZIP_OK) + break; + + if ((zi->ci.method == Z_DEFLATED) && (!zi->ci.raw)) + { + uLong uTotalOutBefore = zi->ci.stream.total_out; + err=deflate(&zi->ci.stream, Z_NO_FLUSH); + if(uTotalOutBefore > zi->ci.stream.total_out) + { + int bBreak = 0; + bBreak++; + } + + zi->ci.pos_in_buffered_data += (uInt)(zi->ci.stream.total_out - uTotalOutBefore) ; + } + else + { + uInt copy_this,i; + if (zi->ci.stream.avail_in < zi->ci.stream.avail_out) + copy_this = zi->ci.stream.avail_in; + else + copy_this = zi->ci.stream.avail_out; + + for (i = 0; i < copy_this; i++) + *(((char*)zi->ci.stream.next_out)+i) = + *(((const char*)zi->ci.stream.next_in)+i); + { + zi->ci.stream.avail_in -= copy_this; + zi->ci.stream.avail_out-= copy_this; + zi->ci.stream.next_in+= copy_this; + zi->ci.stream.next_out+= copy_this; + zi->ci.stream.total_in+= copy_this; + zi->ci.stream.total_out+= copy_this; + zi->ci.pos_in_buffered_data += copy_this; + } + } + }// while(...) + } + + return err; +} + +extern int ZEXPORT zipCloseFileInZipRaw (zipFile file, uLong uncompressed_size, uLong crc32) +{ + return zipCloseFileInZipRaw64 (file, uncompressed_size, crc32); +} + +extern int ZEXPORT zipCloseFileInZipRaw64 (zipFile file, ZPOS64_T uncompressed_size, uLong crc32) +{ + zip64_internal* zi; + ZPOS64_T compressed_size; + uLong invalidValue = 0xffffffff; + short datasize = 0; + int err=ZIP_OK; + + if (file == NULL) + return ZIP_PARAMERROR; + zi = (zip64_internal*)file; + + if (zi->in_opened_file_inzip == 0) + return ZIP_PARAMERROR; + zi->ci.stream.avail_in = 0; + + if ((zi->ci.method == Z_DEFLATED) && (!zi->ci.raw)) + { + while (err==ZIP_OK) + { + uLong uTotalOutBefore; + if (zi->ci.stream.avail_out == 0) + { + if (zip64FlushWriteBuffer(zi) == ZIP_ERRNO) + err = ZIP_ERRNO; + zi->ci.stream.avail_out = (uInt)Z_BUFSIZE; + zi->ci.stream.next_out = zi->ci.buffered_data; + } + uTotalOutBefore = zi->ci.stream.total_out; + err=deflate(&zi->ci.stream, Z_FINISH); + zi->ci.pos_in_buffered_data += (uInt)(zi->ci.stream.total_out - uTotalOutBefore) ; + } + } + else if ((zi->ci.method == Z_BZIP2ED) && (!zi->ci.raw)) + { +#ifdef HAVE_BZIP2 + err = BZ_FINISH_OK; + while (err==BZ_FINISH_OK) + { + uLong uTotalOutBefore; + if (zi->ci.bstream.avail_out == 0) + { + if (zip64FlushWriteBuffer(zi) == ZIP_ERRNO) + err = ZIP_ERRNO; + zi->ci.bstream.avail_out = (uInt)Z_BUFSIZE; + zi->ci.bstream.next_out = (char*)zi->ci.buffered_data; + } + uTotalOutBefore = zi->ci.bstream.total_out_lo32; + err=BZ2_bzCompress(&zi->ci.bstream, BZ_FINISH); + if(err == BZ_STREAM_END) + err = Z_STREAM_END; + + zi->ci.pos_in_buffered_data += (uInt)(zi->ci.bstream.total_out_lo32 - uTotalOutBefore); + } + + if(err == BZ_FINISH_OK) + err = ZIP_OK; +#endif + } + + if (err==Z_STREAM_END) + err=ZIP_OK; /* this is normal */ + + if ((zi->ci.pos_in_buffered_data>0) && (err==ZIP_OK)) + { + if (zip64FlushWriteBuffer(zi)==ZIP_ERRNO) + err = ZIP_ERRNO; + } + + if ((zi->ci.method == Z_DEFLATED) && (!zi->ci.raw)) + { + int tmp_err = deflateEnd(&zi->ci.stream); + if (err == ZIP_OK) + err = tmp_err; + zi->ci.stream_initialised = 0; + } +#ifdef HAVE_BZIP2 + else if((zi->ci.method == Z_BZIP2ED) && (!zi->ci.raw)) + { + int tmperr = BZ2_bzCompressEnd(&zi->ci.bstream); + if (err==ZIP_OK) + err = tmperr; + zi->ci.stream_initialised = 0; + } +#endif + + if (!zi->ci.raw) + { + crc32 = (uLong)zi->ci.crc32; + uncompressed_size = zi->ci.totalUncompressedData; + } + compressed_size = zi->ci.totalCompressedData; + +# ifndef NOCRYPT + compressed_size += zi->ci.crypt_header_size; +# endif + + // update Current Item crc and sizes, + if(compressed_size >= 0xffffffff || uncompressed_size >= 0xffffffff || zi->ci.pos_local_header >= 0xffffffff) + { + /*version Made by*/ + zip64local_putValue_inmemory(zi->ci.central_header+4,(uLong)45,2); + /*version needed*/ + zip64local_putValue_inmemory(zi->ci.central_header+6,(uLong)45,2); + + } + + zip64local_putValue_inmemory(zi->ci.central_header+16,crc32,4); /*crc*/ + + + if(compressed_size >= 0xffffffff) + zip64local_putValue_inmemory(zi->ci.central_header+20, invalidValue,4); /*compr size*/ + else + zip64local_putValue_inmemory(zi->ci.central_header+20, compressed_size,4); /*compr size*/ + + /// set internal file attributes field + if (zi->ci.stream.data_type == Z_ASCII) + zip64local_putValue_inmemory(zi->ci.central_header+36,(uLong)Z_ASCII,2); + + if(uncompressed_size >= 0xffffffff) + zip64local_putValue_inmemory(zi->ci.central_header+24, invalidValue,4); /*uncompr size*/ + else + zip64local_putValue_inmemory(zi->ci.central_header+24, uncompressed_size,4); /*uncompr size*/ + + // Add ZIP64 extra info field for uncompressed size + if(uncompressed_size >= 0xffffffff) + datasize += 8; + + // Add ZIP64 extra info field for compressed size + if(compressed_size >= 0xffffffff) + datasize += 8; + + // Add ZIP64 extra info field for relative offset to local file header of current file + if(zi->ci.pos_local_header >= 0xffffffff) + datasize += 8; + + if(datasize > 0) + { + char* p = NULL; + + if((uLong)(datasize + 4) > zi->ci.size_centralExtraFree) + { + // we can not write more data to the buffer that we have room for. + return ZIP_BADZIPFILE; + } + + p = zi->ci.central_header + zi->ci.size_centralheader; + + // Add Extra Information Header for 'ZIP64 information' + zip64local_putValue_inmemory(p, 0x0001, 2); // HeaderID + p += 2; + zip64local_putValue_inmemory(p, datasize, 2); // DataSize + p += 2; + + if(uncompressed_size >= 0xffffffff) + { + zip64local_putValue_inmemory(p, uncompressed_size, 8); + p += 8; + } + + if(compressed_size >= 0xffffffff) + { + zip64local_putValue_inmemory(p, compressed_size, 8); + p += 8; + } + + if(zi->ci.pos_local_header >= 0xffffffff) + { + zip64local_putValue_inmemory(p, zi->ci.pos_local_header, 8); + p += 8; + } + + // Update how much extra free space we got in the memory buffer + // and increase the centralheader size so the new ZIP64 fields are included + // ( 4 below is the size of HeaderID and DataSize field ) + zi->ci.size_centralExtraFree -= datasize + 4; + zi->ci.size_centralheader += datasize + 4; + + // Update the extra info size field + zi->ci.size_centralExtra += datasize + 4; + zip64local_putValue_inmemory(zi->ci.central_header+30,(uLong)zi->ci.size_centralExtra,2); + } + + if (err==ZIP_OK) + err = add_data_in_datablock(&zi->central_dir, zi->ci.central_header, (uLong)zi->ci.size_centralheader); + + free(zi->ci.central_header); + + if (err==ZIP_OK) + { + // Update the LocalFileHeader with the new values. + + ZPOS64_T cur_pos_inzip = ZTELL64(zi->z_filefunc,zi->filestream); + + if (ZSEEK64(zi->z_filefunc,zi->filestream, zi->ci.pos_local_header + 14,ZLIB_FILEFUNC_SEEK_SET)!=0) + err = ZIP_ERRNO; + + if (err==ZIP_OK) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,crc32,4); /* crc 32, unknown */ + + if(uncompressed_size >= 0xffffffff || compressed_size >= 0xffffffff ) + { + if(zi->ci.pos_zip64extrainfo > 0) + { + // Update the size in the ZIP64 extended field. + if (ZSEEK64(zi->z_filefunc,zi->filestream, zi->ci.pos_zip64extrainfo + 4,ZLIB_FILEFUNC_SEEK_SET)!=0) + err = ZIP_ERRNO; + + if (err==ZIP_OK) /* compressed size, unknown */ + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, uncompressed_size, 8); + + if (err==ZIP_OK) /* uncompressed size, unknown */ + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, compressed_size, 8); + } + else + err = ZIP_BADZIPFILE; // Caller passed zip64 = 0, so no room for zip64 info -> fatal + } + else + { + if (err==ZIP_OK) /* compressed size, unknown */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,compressed_size,4); + + if (err==ZIP_OK) /* uncompressed size, unknown */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,uncompressed_size,4); + } + + if (ZSEEK64(zi->z_filefunc,zi->filestream, cur_pos_inzip,ZLIB_FILEFUNC_SEEK_SET)!=0) + err = ZIP_ERRNO; + } + + zi->number_entry ++; + zi->in_opened_file_inzip = 0; + + return err; +} + +extern int ZEXPORT zipCloseFileInZip (zipFile file) +{ + return zipCloseFileInZipRaw (file,0,0); +} + +int Write_Zip64EndOfCentralDirectoryLocator(zip64_internal* zi, ZPOS64_T zip64eocd_pos_inzip) +{ + int err = ZIP_OK; + ZPOS64_T pos = zip64eocd_pos_inzip - zi->add_position_when_writing_offset; + + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)ZIP64ENDLOCHEADERMAGIC,4); + + /*num disks*/ + if (err==ZIP_OK) /* number of the disk with the start of the central directory */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,4); + + /*relative offset*/ + if (err==ZIP_OK) /* Relative offset to the Zip64EndOfCentralDirectory */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream, pos,8); + + /*total disks*/ /* Do not support spawning of disk so always say 1 here*/ + if (err==ZIP_OK) /* number of the disk with the start of the central directory */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)1,4); + + return err; +} + +int Write_Zip64EndOfCentralDirectoryRecord(zip64_internal* zi, uLong size_centraldir, ZPOS64_T centraldir_pos_inzip) +{ + int err = ZIP_OK; + + uLong Zip64DataSize = 44; + + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)ZIP64ENDHEADERMAGIC,4); + + if (err==ZIP_OK) /* size of this 'zip64 end of central directory' */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(ZPOS64_T)Zip64DataSize,8); // why ZPOS64_T of this ? + + if (err==ZIP_OK) /* version made by */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)45,2); + + if (err==ZIP_OK) /* version needed */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)45,2); + + if (err==ZIP_OK) /* number of this disk */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,4); + + if (err==ZIP_OK) /* number of the disk with the start of the central directory */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,4); + + if (err==ZIP_OK) /* total number of entries in the central dir on this disk */ + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, zi->number_entry, 8); + + if (err==ZIP_OK) /* total number of entries in the central dir */ + err = zip64local_putValue(&zi->z_filefunc, zi->filestream, zi->number_entry, 8); + + if (err==ZIP_OK) /* size of the central directory */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(ZPOS64_T)size_centraldir,8); + + if (err==ZIP_OK) /* offset of start of central directory with respect to the starting disk number */ + { + ZPOS64_T pos = centraldir_pos_inzip - zi->add_position_when_writing_offset; + err = zip64local_putValue(&zi->z_filefunc,zi->filestream, (ZPOS64_T)pos,8); + } + return err; +} +int Write_EndOfCentralDirectoryRecord(zip64_internal* zi, uLong size_centraldir, ZPOS64_T centraldir_pos_inzip) +{ + int err = ZIP_OK; + + /*signature*/ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)ENDHEADERMAGIC,4); + + if (err==ZIP_OK) /* number of this disk */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,2); + + if (err==ZIP_OK) /* number of the disk with the start of the central directory */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0,2); + + if (err==ZIP_OK) /* total number of entries in the central dir on this disk */ + { + { + if(zi->number_entry >= 0xFFFF) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0xffff,2); // use value in ZIP64 record + else + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)zi->number_entry,2); + } + } + + if (err==ZIP_OK) /* total number of entries in the central dir */ + { + if(zi->number_entry >= 0xFFFF) + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)0xffff,2); // use value in ZIP64 record + else + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)zi->number_entry,2); + } + + if (err==ZIP_OK) /* size of the central directory */ + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)size_centraldir,4); + + if (err==ZIP_OK) /* offset of start of central directory with respect to the starting disk number */ + { + ZPOS64_T pos = centraldir_pos_inzip - zi->add_position_when_writing_offset; + if(pos >= 0xffffffff) + { + err = zip64local_putValue(&zi->z_filefunc,zi->filestream, (uLong)0xffffffff,4); + } + else + err = zip64local_putValue(&zi->z_filefunc,zi->filestream, (uLong)(centraldir_pos_inzip - zi->add_position_when_writing_offset),4); + } + + return err; +} + +int Write_GlobalComment(zip64_internal* zi, const char* global_comment) +{ + int err = ZIP_OK; + uInt size_global_comment = 0; + + if(global_comment != NULL) + size_global_comment = (uInt)strlen(global_comment); + + err = zip64local_putValue(&zi->z_filefunc,zi->filestream,(uLong)size_global_comment,2); + + if (err == ZIP_OK && size_global_comment > 0) + { + if (ZWRITE64(zi->z_filefunc,zi->filestream, global_comment, size_global_comment) != size_global_comment) + err = ZIP_ERRNO; + } + return err; +} + +extern int ZEXPORT zipClose (zipFile file, const char* global_comment) +{ + zip64_internal* zi; + int err = 0; + uLong size_centraldir = 0; + ZPOS64_T centraldir_pos_inzip; + ZPOS64_T pos; + + if (file == NULL) + return ZIP_PARAMERROR; + + zi = (zip64_internal*)file; + + if (zi->in_opened_file_inzip == 1) + { + err = zipCloseFileInZip (file); + } + +#ifndef NO_ADDFILEINEXISTINGZIP + if (global_comment==NULL) + global_comment = zi->globalcomment; +#endif + + centraldir_pos_inzip = ZTELL64(zi->z_filefunc,zi->filestream); + + if (err==ZIP_OK) + { + linkedlist_datablock_internal* ldi = zi->central_dir.first_block; + while (ldi!=NULL) + { + if ((err==ZIP_OK) && (ldi->filled_in_this_block>0)) + { + if (ZWRITE64(zi->z_filefunc,zi->filestream, ldi->data, ldi->filled_in_this_block) != ldi->filled_in_this_block) + err = ZIP_ERRNO; + } + + size_centraldir += ldi->filled_in_this_block; + ldi = ldi->next_datablock; + } + } + free_linkedlist(&(zi->central_dir)); + + pos = centraldir_pos_inzip - zi->add_position_when_writing_offset; + if(pos >= 0xffffffff || zi->number_entry > 0xFFFF) + { + ZPOS64_T Zip64EOCDpos = ZTELL64(zi->z_filefunc,zi->filestream); + Write_Zip64EndOfCentralDirectoryRecord(zi, size_centraldir, centraldir_pos_inzip); + + Write_Zip64EndOfCentralDirectoryLocator(zi, Zip64EOCDpos); + } + + if (err==ZIP_OK) + err = Write_EndOfCentralDirectoryRecord(zi, size_centraldir, centraldir_pos_inzip); + + if(err == ZIP_OK) + err = Write_GlobalComment(zi, global_comment); + + if (ZCLOSE64(zi->z_filefunc,zi->filestream) != 0) + if (err == ZIP_OK) + err = ZIP_ERRNO; + +#ifndef NO_ADDFILEINEXISTINGZIP + TRYFREE(zi->globalcomment); +#endif + TRYFREE(zi); + + return err; +} + +extern int ZEXPORT zipRemoveExtraInfoBlock (char* pData, int* dataLen, short sHeader) +{ + char* p = pData; + int size = 0; + char* pNewHeader; + char* pTmp; + short header; + short dataSize; + + int retVal = ZIP_OK; + + if(pData == NULL || *dataLen < 4) + return ZIP_PARAMERROR; + + pNewHeader = (char*)ALLOC(*dataLen); + pTmp = pNewHeader; + + while(p < (pData + *dataLen)) + { + header = *(short*)p; + dataSize = *(((short*)p)+1); + + if( header == sHeader ) // Header found. + { + p += dataSize + 4; // skip it. do not copy to temp buffer + } + else + { + // Extra Info block should not be removed, So copy it to the temp buffer. + memcpy(pTmp, p, dataSize + 4); + p += dataSize + 4; + size += dataSize + 4; + } + + } + + if(size < *dataLen) + { + // clean old extra info block. + memset(pData,0, *dataLen); + + // copy the new extra info block over the old + if(size > 0) + memcpy(pData, pNewHeader, size); + + // set the new extra info size + *dataLen = size; + + retVal = ZIP_OK; + } + else + retVal = ZIP_ERRNO; + + TRYFREE(pNewHeader); + + return retVal; +} diff --git a/third_party/zlib/contrib/minizip/zip.h b/third_party/zlib/contrib/minizip/zip.h new file mode 100644 index 00000000..8c06c0aa --- /dev/null +++ b/third_party/zlib/contrib/minizip/zip.h @@ -0,0 +1,362 @@ +/* zip.h -- IO on .zip files using zlib + Version 1.1, February 14h, 2010 + part of the MiniZip project - ( http://www.winimage.com/zLibDll/minizip.html ) + + Copyright (C) 1998-2010 Gilles Vollant (minizip) ( http://www.winimage.com/zLibDll/minizip.html ) + + Modifications for Zip64 support + Copyright (C) 2009-2010 Mathias Svensson ( http://result42.com ) + + For more info read MiniZip_info.txt + + --------------------------------------------------------------------------- + + Condition of use and distribution are the same than zlib : + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + --------------------------------------------------------------------------- + + Changes + + See header of zip.h + +*/ + +#ifndef _zip12_H +#define _zip12_H + +#ifdef __cplusplus +extern "C" { +#endif + +//#define HAVE_BZIP2 + +#ifndef _ZLIB_H +#include "third_party/zlib/zlib.h" +#endif + +#ifndef _ZLIBIOAPI_H +#include "ioapi.h" +#endif + +#ifdef HAVE_BZIP2 +#include "bzlib.h" +#endif + +#define Z_BZIP2ED 12 + +#if defined(STRICTZIP) || defined(STRICTZIPUNZIP) +/* like the STRICT of WIN32, we define a pointer that cannot be converted + from (void*) without cast */ +typedef struct TagzipFile__ { int unused; } zipFile__; +typedef zipFile__ *zipFile; +#else +typedef voidp zipFile; +#endif + +#define ZIP_OK (0) +#define ZIP_EOF (0) +#define ZIP_ERRNO (Z_ERRNO) +#define ZIP_PARAMERROR (-102) +#define ZIP_BADZIPFILE (-103) +#define ZIP_INTERNALERROR (-104) + +#ifndef DEF_MEM_LEVEL +# if MAX_MEM_LEVEL >= 8 +# define DEF_MEM_LEVEL 8 +# else +# define DEF_MEM_LEVEL MAX_MEM_LEVEL +# endif +#endif +/* default memLevel */ + +/* tm_zip contain date/time info */ +typedef struct tm_zip_s +{ + uInt tm_sec; /* seconds after the minute - [0,59] */ + uInt tm_min; /* minutes after the hour - [0,59] */ + uInt tm_hour; /* hours since midnight - [0,23] */ + uInt tm_mday; /* day of the month - [1,31] */ + uInt tm_mon; /* months since January - [0,11] */ + uInt tm_year; /* years - [1980..2044] */ +} tm_zip; + +typedef struct +{ + tm_zip tmz_date; /* date in understandable format */ + uLong dosDate; /* if dos_date == 0, tmu_date is used */ +/* uLong flag; */ /* general purpose bit flag 2 bytes */ + + uLong internal_fa; /* internal file attributes 2 bytes */ + uLong external_fa; /* external file attributes 4 bytes */ +} zip_fileinfo; + +typedef const char* zipcharpc; + + +#define APPEND_STATUS_CREATE (0) +#define APPEND_STATUS_CREATEAFTER (1) +#define APPEND_STATUS_ADDINZIP (2) + +extern zipFile ZEXPORT zipOpen OF((const char *pathname, int append)); +extern zipFile ZEXPORT zipOpen64 OF((const void *pathname, int append)); +/* + Create a zipfile. + pathname contain on Windows XP a filename like "c:\\zlib\\zlib113.zip" or on + an Unix computer "zlib/zlib113.zip". + if the file pathname exist and append==APPEND_STATUS_CREATEAFTER, the zip + will be created at the end of the file. + (useful if the file contain a self extractor code) + if the file pathname exist and append==APPEND_STATUS_ADDINZIP, we will + add files in existing zip (be sure you don't add file that doesn't exist) + If the zipfile cannot be opened, the return value is NULL. + Else, the return value is a zipFile Handle, usable with other function + of this zip package. +*/ + +/* Note : there is no delete function into a zipfile. + If you want delete file into a zipfile, you must open a zipfile, and create another + Of couse, you can use RAW reading and writing to copy the file you did not want delte +*/ + +extern zipFile ZEXPORT zipOpen2 OF((const char *pathname, + int append, + zipcharpc* globalcomment, + zlib_filefunc_def* pzlib_filefunc_def)); + +extern zipFile ZEXPORT zipOpen2_64 OF((const void *pathname, + int append, + zipcharpc* globalcomment, + zlib_filefunc64_def* pzlib_filefunc_def)); + +extern int ZEXPORT zipOpenNewFileInZip OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level)); + +extern int ZEXPORT zipOpenNewFileInZip64 OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int zip64)); + +/* + Open a file in the ZIP for writing. + filename : the filename in zip (if NULL, '-' without quote will be used + *zipfi contain supplemental information + if extrafield_local!=NULL and size_extrafield_local>0, extrafield_local + contains the extrafield data the the local header + if extrafield_global!=NULL and size_extrafield_global>0, extrafield_global + contains the extrafield data the the local header + if comment != NULL, comment contain the comment string + method contain the compression method (0 for store, Z_DEFLATED for deflate) + level contain the level of compression (can be Z_DEFAULT_COMPRESSION) + zip64 is set to 1 if a zip64 extended information block should be added to the local file header. + this MUST be '1' if the uncompressed size is >= 0xffffffff. + +*/ + + +extern int ZEXPORT zipOpenNewFileInZip2 OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int raw)); + + +extern int ZEXPORT zipOpenNewFileInZip2_64 OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int raw, + int zip64)); +/* + Same than zipOpenNewFileInZip, except if raw=1, we write raw file + */ + +extern int ZEXPORT zipOpenNewFileInZip3 OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int raw, + int windowBits, + int memLevel, + int strategy, + const char* password, + uLong crcForCrypting)); + +extern int ZEXPORT zipOpenNewFileInZip3_64 OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int raw, + int windowBits, + int memLevel, + int strategy, + const char* password, + uLong crcForCrypting, + int zip64 + )); + +/* + Same than zipOpenNewFileInZip2, except + windowBits,memLevel,,strategy : see parameter strategy in deflateInit2 + password : crypting password (NULL for no crypting) + crcForCrypting : crc of file to compress (needed for crypting) + */ + +extern int ZEXPORT zipOpenNewFileInZip4 OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int raw, + int windowBits, + int memLevel, + int strategy, + const char* password, + uLong crcForCrypting, + uLong versionMadeBy, + uLong flagBase + )); + + +extern int ZEXPORT zipOpenNewFileInZip4_64 OF((zipFile file, + const char* filename, + const zip_fileinfo* zipfi, + const void* extrafield_local, + uInt size_extrafield_local, + const void* extrafield_global, + uInt size_extrafield_global, + const char* comment, + int method, + int level, + int raw, + int windowBits, + int memLevel, + int strategy, + const char* password, + uLong crcForCrypting, + uLong versionMadeBy, + uLong flagBase, + int zip64 + )); +/* + Same than zipOpenNewFileInZip4, except + versionMadeBy : value for Version made by field + flag : value for flag field (compression level info will be added) + */ + + +extern int ZEXPORT zipWriteInFileInZip OF((zipFile file, + const void* buf, + unsigned len)); +/* + Write data in the zipfile +*/ + +extern int ZEXPORT zipCloseFileInZip OF((zipFile file)); +/* + Close the current file in the zipfile +*/ + +extern int ZEXPORT zipCloseFileInZipRaw OF((zipFile file, + uLong uncompressed_size, + uLong crc32)); + +extern int ZEXPORT zipCloseFileInZipRaw64 OF((zipFile file, + ZPOS64_T uncompressed_size, + uLong crc32)); + +/* + Close the current file in the zipfile, for file opened with + parameter raw=1 in zipOpenNewFileInZip2 + uncompressed_size and crc32 are value for the uncompressed size +*/ + +extern int ZEXPORT zipClose OF((zipFile file, + const char* global_comment)); +/* + Close the zipfile +*/ + + +extern int ZEXPORT zipRemoveExtraInfoBlock OF((char* pData, int* dataLen, short sHeader)); +/* + zipRemoveExtraInfoBlock - Added by Mathias Svensson + + Remove extra information block from a extra information data for the local file header or central directory header + + It is needed to remove ZIP64 extra information blocks when before data is written if using RAW mode. + + 0x0001 is the signature header for the ZIP64 extra information blocks + + usage. + Remove ZIP64 Extra information from a central director extra field data + zipRemoveExtraInfoBlock(pCenDirExtraFieldData, &nCenDirExtraFieldDataLen, 0x0001); + + Remove ZIP64 Extra information from a Local File Header extra field data + zipRemoveExtraInfoBlock(pLocalHeaderExtraFieldData, &nLocalHeaderExtraFieldDataLen, 0x0001); +*/ + +#ifdef __cplusplus +} +#endif + +#endif /* _zip64_H */ diff --git a/third_party/zlib/contrib/optimizations/chunkcopy.h b/third_party/zlib/contrib/optimizations/chunkcopy.h new file mode 100644 index 00000000..9c0b7cb0 --- /dev/null +++ b/third_party/zlib/contrib/optimizations/chunkcopy.h @@ -0,0 +1,477 @@ +/* chunkcopy.h -- fast chunk copy and set operations + * Copyright (C) 2017 ARM, Inc. + * Copyright 2017 The Chromium Authors. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the Chromium source repository LICENSE file. + */ + +#ifndef CHUNKCOPY_H +#define CHUNKCOPY_H + +#include <stdint.h> +#include "zutil.h" + +#define Z_STATIC_ASSERT(name, assert) typedef char name[(assert) ? 1 : -1] + +#if __STDC_VERSION__ >= 199901L +#define Z_RESTRICT restrict +#else +#define Z_RESTRICT +#endif + +#if defined(__clang__) || defined(__GNUC__) || defined(__llvm__) +#define Z_BUILTIN_MEMCPY __builtin_memcpy +#else +#define Z_BUILTIN_MEMCPY zmemcpy +#endif + +#if defined(INFLATE_CHUNK_SIMD_NEON) +#include <arm_neon.h> +typedef uint8x16_t z_vec128i_t; +#elif defined(INFLATE_CHUNK_SIMD_SSE2) +#include <emmintrin.h> +typedef __m128i z_vec128i_t; +#else +#error chunkcopy.h inflate chunk SIMD is not defined for your build target +#endif + +/* + * chunk copy type: the z_vec128i_t type size should be exactly 128-bits + * and equal to CHUNKCOPY_CHUNK_SIZE. + */ +#define CHUNKCOPY_CHUNK_SIZE sizeof(z_vec128i_t) + +Z_STATIC_ASSERT(vector_128_bits_wide, + CHUNKCOPY_CHUNK_SIZE == sizeof(int8_t) * 16); + +/* + * Ask the compiler to perform a wide, unaligned load with a machine + * instruction appropriate for the z_vec128i_t type. + */ +static inline z_vec128i_t loadchunk( + const unsigned char FAR* s) { + z_vec128i_t v; + Z_BUILTIN_MEMCPY(&v, s, sizeof(v)); + return v; +} + +/* + * Ask the compiler to perform a wide, unaligned store with a machine + * instruction appropriate for the z_vec128i_t type. + */ +static inline void storechunk( + unsigned char FAR* d, + const z_vec128i_t v) { + Z_BUILTIN_MEMCPY(d, &v, sizeof(v)); +} + +/* + * Perform a memcpy-like operation, assuming that length is non-zero and that + * it's OK to overwrite at least CHUNKCOPY_CHUNK_SIZE bytes of output even if + * the length is shorter than this. + * + * It also guarantees that it will properly unroll the data if the distance + * between `out` and `from` is at least CHUNKCOPY_CHUNK_SIZE, which we rely on + * in chunkcopy_relaxed(). + * + * Aside from better memory bus utilisation, this means that short copies + * (CHUNKCOPY_CHUNK_SIZE bytes or fewer) will fall straight through the loop + * without iteration, which will hopefully make the branch prediction more + * reliable. + */ +static inline unsigned char FAR* chunkcopy_core( + unsigned char FAR* out, + const unsigned char FAR* from, + unsigned len) { + const int bump = (--len % CHUNKCOPY_CHUNK_SIZE) + 1; + storechunk(out, loadchunk(from)); + out += bump; + from += bump; + len /= CHUNKCOPY_CHUNK_SIZE; + while (len-- > 0) { + storechunk(out, loadchunk(from)); + out += CHUNKCOPY_CHUNK_SIZE; + from += CHUNKCOPY_CHUNK_SIZE; + } + return out; +} + +/* + * Like chunkcopy_core(), but avoid writing beyond of legal output. + * + * Accepts an additional pointer to the end of safe output. A generic safe + * copy would use (out + len), but it's normally the case that the end of the + * output buffer is beyond the end of the current copy, and this can still be + * exploited. + */ +static inline unsigned char FAR* chunkcopy_core_safe( + unsigned char FAR* out, + const unsigned char FAR* from, + unsigned len, + unsigned char FAR* limit) { + Assert(out + len <= limit, "chunk copy exceeds safety limit"); + if ((limit - out) < (ptrdiff_t)CHUNKCOPY_CHUNK_SIZE) { + const unsigned char FAR* Z_RESTRICT rfrom = from; + Assert((uintptr_t)out - (uintptr_t)from >= len, + "invalid restrict in chunkcopy_core_safe"); + Assert((uintptr_t)from - (uintptr_t)out >= len, + "invalid restrict in chunkcopy_core_safe"); + if (len & 8) { + Z_BUILTIN_MEMCPY(out, rfrom, 8); + out += 8; + rfrom += 8; + } + if (len & 4) { + Z_BUILTIN_MEMCPY(out, rfrom, 4); + out += 4; + rfrom += 4; + } + if (len & 2) { + Z_BUILTIN_MEMCPY(out, rfrom, 2); + out += 2; + rfrom += 2; + } + if (len & 1) { + *out++ = *rfrom++; + } + return out; + } + return chunkcopy_core(out, from, len); +} + +/* + * Perform short copies until distance can be rewritten as being at least + * CHUNKCOPY_CHUNK_SIZE. + * + * Assumes it's OK to overwrite at least the first 2*CHUNKCOPY_CHUNK_SIZE + * bytes of output even if the copy is shorter than this. This assumption + * holds within zlib inflate_fast(), which starts every iteration with at + * least 258 bytes of output space available (258 being the maximum length + * output from a single token; see inffast.c). + */ +static inline unsigned char FAR* chunkunroll_relaxed( + unsigned char FAR* out, + unsigned FAR* dist, + unsigned FAR* len) { + const unsigned char FAR* from = out - *dist; + while (*dist < *len && *dist < CHUNKCOPY_CHUNK_SIZE) { + storechunk(out, loadchunk(from)); + out += *dist; + *len -= *dist; + *dist += *dist; + } + return out; +} + +#if defined(INFLATE_CHUNK_SIMD_NEON) +/* + * v_load64_dup(): load *src as an unaligned 64-bit int and duplicate it in + * every 64-bit component of the 128-bit result (64-bit int splat). + */ +static inline z_vec128i_t v_load64_dup(const void* src) { + return vcombine_u8(vld1_u8(src), vld1_u8(src)); +} + +/* + * v_load32_dup(): load *src as an unaligned 32-bit int and duplicate it in + * every 32-bit component of the 128-bit result (32-bit int splat). + */ +static inline z_vec128i_t v_load32_dup(const void* src) { + int32_t i32; + Z_BUILTIN_MEMCPY(&i32, src, sizeof(i32)); + return vreinterpretq_u8_s32(vdupq_n_s32(i32)); +} + +/* + * v_load16_dup(): load *src as an unaligned 16-bit int and duplicate it in + * every 16-bit component of the 128-bit result (16-bit int splat). + */ +static inline z_vec128i_t v_load16_dup(const void* src) { + int16_t i16; + Z_BUILTIN_MEMCPY(&i16, src, sizeof(i16)); + return vreinterpretq_u8_s16(vdupq_n_s16(i16)); +} + +/* + * v_load8_dup(): load the 8-bit int *src and duplicate it in every 8-bit + * component of the 128-bit result (8-bit int splat). + */ +static inline z_vec128i_t v_load8_dup(const void* src) { + return vld1q_dup_u8((const uint8_t*)src); +} + +/* + * v_store_128(): store the 128-bit vec in a memory destination (that might + * not be 16-byte aligned) void* out. + */ +static inline void v_store_128(void* out, const z_vec128i_t vec) { + vst1q_u8(out, vec); +} + +#elif defined(INFLATE_CHUNK_SIMD_SSE2) +/* + * v_load64_dup(): load *src as an unaligned 64-bit int and duplicate it in + * every 64-bit component of the 128-bit result (64-bit int splat). + */ +static inline z_vec128i_t v_load64_dup(const void* src) { + int64_t i64; + Z_BUILTIN_MEMCPY(&i64, src, sizeof(i64)); + return _mm_set1_epi64x(i64); +} + +/* + * v_load32_dup(): load *src as an unaligned 32-bit int and duplicate it in + * every 32-bit component of the 128-bit result (32-bit int splat). + */ +static inline z_vec128i_t v_load32_dup(const void* src) { + int32_t i32; + Z_BUILTIN_MEMCPY(&i32, src, sizeof(i32)); + return _mm_set1_epi32(i32); +} + +/* + * v_load16_dup(): load *src as an unaligned 16-bit int and duplicate it in + * every 16-bit component of the 128-bit result (16-bit int splat). + */ +static inline z_vec128i_t v_load16_dup(const void* src) { + int16_t i16; + Z_BUILTIN_MEMCPY(&i16, src, sizeof(i16)); + return _mm_set1_epi16(i16); +} + +/* + * v_load8_dup(): load the 8-bit int *src and duplicate it in every 8-bit + * component of the 128-bit result (8-bit int splat). + */ +static inline z_vec128i_t v_load8_dup(const void* src) { + return _mm_set1_epi8(*(const char*)src); +} + +/* + * v_store_128(): store the 128-bit vec in a memory destination (that might + * not be 16-byte aligned) void* out. + */ +static inline void v_store_128(void* out, const z_vec128i_t vec) { + _mm_storeu_si128((__m128i*)out, vec); +} +#endif + +/* + * Perform an overlapping copy which behaves as a memset() operation, but + * supporting periods other than one, and assume that length is non-zero and + * that it's OK to overwrite at least CHUNKCOPY_CHUNK_SIZE*3 bytes of output + * even if the length is shorter than this. + */ +static inline unsigned char FAR* chunkset_core( + unsigned char FAR* out, + unsigned period, + unsigned len) { + z_vec128i_t v; + const int bump = ((len - 1) % sizeof(v)) + 1; + + switch (period) { + case 1: + v = v_load8_dup(out - 1); + v_store_128(out, v); + out += bump; + len -= bump; + while (len > 0) { + v_store_128(out, v); + out += sizeof(v); + len -= sizeof(v); + } + return out; + case 2: + v = v_load16_dup(out - 2); + v_store_128(out, v); + out += bump; + len -= bump; + if (len > 0) { + v = v_load16_dup(out - 2); + do { + v_store_128(out, v); + out += sizeof(v); + len -= sizeof(v); + } while (len > 0); + } + return out; + case 4: + v = v_load32_dup(out - 4); + v_store_128(out, v); + out += bump; + len -= bump; + if (len > 0) { + v = v_load32_dup(out - 4); + do { + v_store_128(out, v); + out += sizeof(v); + len -= sizeof(v); + } while (len > 0); + } + return out; + case 8: + v = v_load64_dup(out - 8); + v_store_128(out, v); + out += bump; + len -= bump; + if (len > 0) { + v = v_load64_dup(out - 8); + do { + v_store_128(out, v); + out += sizeof(v); + len -= sizeof(v); + } while (len > 0); + } + return out; + } + out = chunkunroll_relaxed(out, &period, &len); + return chunkcopy_core(out, out - period, len); +} + +/* + * Perform a memcpy-like operation, but assume that length is non-zero and that + * it's OK to overwrite at least CHUNKCOPY_CHUNK_SIZE bytes of output even if + * the length is shorter than this. + * + * Unlike chunkcopy_core() above, no guarantee is made regarding the behaviour + * of overlapping buffers, regardless of the distance between the pointers. + * This is reflected in the `restrict`-qualified pointers, allowing the + * compiler to re-order loads and stores. + */ +static inline unsigned char FAR* chunkcopy_relaxed( + unsigned char FAR* Z_RESTRICT out, + const unsigned char FAR* Z_RESTRICT from, + unsigned len) { + Assert((uintptr_t)out - (uintptr_t)from >= len, + "invalid restrict in chunkcopy_relaxed"); + Assert((uintptr_t)from - (uintptr_t)out >= len, + "invalid restrict in chunkcopy_relaxed"); + return chunkcopy_core(out, from, len); +} + +/* + * Like chunkcopy_relaxed(), but avoid writing beyond of legal output. + * + * Unlike chunkcopy_core_safe() above, no guarantee is made regarding the + * behaviour of overlapping buffers, regardless of the distance between the + * pointers. This is reflected in the `restrict`-qualified pointers, allowing + * the compiler to re-order loads and stores. + * + * Accepts an additional pointer to the end of safe output. A generic safe + * copy would use (out + len), but it's normally the case that the end of the + * output buffer is beyond the end of the current copy, and this can still be + * exploited. + */ +static inline unsigned char FAR* chunkcopy_safe( + unsigned char FAR* out, + const unsigned char FAR* Z_RESTRICT from, + unsigned len, + unsigned char FAR* limit) { + Assert(out + len <= limit, "chunk copy exceeds safety limit"); + Assert((uintptr_t)out - (uintptr_t)from >= len, + "invalid restrict in chunkcopy_safe"); + Assert((uintptr_t)from - (uintptr_t)out >= len, + "invalid restrict in chunkcopy_safe"); + + return chunkcopy_core_safe(out, from, len, limit); +} + +/* + * Perform chunky copy within the same buffer, where the source and destination + * may potentially overlap. + * + * Assumes that len > 0 on entry, and that it's safe to write at least + * CHUNKCOPY_CHUNK_SIZE*3 bytes to the output. + */ +static inline unsigned char FAR* chunkcopy_lapped_relaxed( + unsigned char FAR* out, + unsigned dist, + unsigned len) { + if (dist < len && dist < CHUNKCOPY_CHUNK_SIZE) { + return chunkset_core(out, dist, len); + } + return chunkcopy_core(out, out - dist, len); +} + +/* + * Behave like chunkcopy_lapped_relaxed(), but avoid writing beyond of legal + * output. + * + * Accepts an additional pointer to the end of safe output. A generic safe + * copy would use (out + len), but it's normally the case that the end of the + * output buffer is beyond the end of the current copy, and this can still be + * exploited. + */ +static inline unsigned char FAR* chunkcopy_lapped_safe( + unsigned char FAR* out, + unsigned dist, + unsigned len, + unsigned char FAR* limit) { + Assert(out + len <= limit, "chunk copy exceeds safety limit"); + if ((limit - out) < (ptrdiff_t)(3 * CHUNKCOPY_CHUNK_SIZE)) { + /* TODO(cavalcantii): try harder to optimise this */ + while (len-- > 0) { + *out = *(out - dist); + out++; + } + return out; + } + return chunkcopy_lapped_relaxed(out, dist, len); +} + +/* TODO(cavalcanti): see crbug.com/1110083. */ +static inline unsigned char FAR* chunkcopy_safe_ugly(unsigned char FAR* out, + unsigned dist, + unsigned len, + unsigned char FAR* limit) { +#if defined(__GNUC__) && !defined(__clang__) + /* Speed is the same as using chunkcopy_safe + w/ GCC on ARM (tested gcc 6.3 and 7.5) and avoids + undefined behavior. + */ + return chunkcopy_core_safe(out, out - dist, len, limit); +#elif defined(__clang__) && defined(ARMV8_OS_ANDROID) && !defined(__aarch64__) + /* Seems to perform better on 32bit (i.e. Android). */ + return chunkcopy_core_safe(out, out - dist, len, limit); +#else + /* Seems to perform better on 64bit. */ + return chunkcopy_lapped_safe(out, dist, len, limit); +#endif +} + +/* + * The chunk-copy code above deals with writing the decoded DEFLATE data to + * the output with SIMD methods to increase decode speed. Reading the input + * to the DEFLATE decoder with a wide, SIMD method can also increase decode + * speed. This option is supported on little endian machines, and reads the + * input data in 64-bit (8 byte) chunks. + */ + +#ifdef INFLATE_CHUNK_READ_64LE +/* + * Buffer the input in a uint64_t (8 bytes) in the wide input reading case. + */ +typedef uint64_t inflate_holder_t; + +/* + * Ask the compiler to perform a wide, unaligned load of a uint64_t using a + * machine instruction appropriate for the uint64_t type. + */ +static inline inflate_holder_t read64le(const unsigned char FAR *in) { + inflate_holder_t input; + Z_BUILTIN_MEMCPY(&input, in, sizeof(input)); + return input; +} +#else +/* + * Otherwise, buffer the input bits using zlib's default input buffer type. + */ +typedef unsigned long inflate_holder_t; + +#endif /* INFLATE_CHUNK_READ_64LE */ + +#undef Z_STATIC_ASSERT +#undef Z_RESTRICT +#undef Z_BUILTIN_MEMCPY + +#endif /* CHUNKCOPY_H */ diff --git a/third_party/zlib/contrib/optimizations/inffast_chunk.c b/third_party/zlib/contrib/optimizations/inffast_chunk.c new file mode 100644 index 00000000..4bacbc46 --- /dev/null +++ b/third_party/zlib/contrib/optimizations/inffast_chunk.c @@ -0,0 +1,359 @@ +/* inffast_chunk.c -- fast decoding + * Copyright (C) 1995-2017 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +#include "zutil.h" +#include "inftrees.h" +#include "inflate.h" +#include "contrib/optimizations/inffast_chunk.h" +#include "contrib/optimizations/chunkcopy.h" + +#ifdef ASMINF +# pragma message("Assembler code may have bugs -- use at your own risk") +#else + +/* + Decode literal, length, and distance codes and write out the resulting + literal and match bytes until either not enough input or output is + available, an end-of-block is encountered, or a data error is encountered. + When large enough input and output buffers are supplied to inflate(), for + example, a 16K input buffer and a 64K output buffer, more than 95% of the + inflate() execution time is spent in this routine. + + Entry assumptions: + + state->mode == LEN + strm->avail_in >= INFLATE_FAST_MIN_INPUT (6 or 8 bytes) + strm->avail_out >= INFLATE_FAST_MIN_OUTPUT (258 bytes) + start >= strm->avail_out + state->bits < 8 + (state->hold >> state->bits) == 0 + strm->next_out[0..strm->avail_out] does not overlap with + strm->next_in[0..strm->avail_in] + strm->state->window is allocated with an additional + CHUNKCOPY_CHUNK_SIZE-1 bytes of padding beyond strm->state->wsize + + On return, state->mode is one of: + + LEN -- ran out of enough output space or enough available input + TYPE -- reached end of block code, inflate() to interpret next block + BAD -- error in block data + + Notes: + + INFLATE_FAST_MIN_INPUT: 6 or 8 bytes + + - The maximum input bits used by a length/distance pair is 15 bits for the + length code, 5 bits for the length extra, 15 bits for the distance code, + and 13 bits for the distance extra. This totals 48 bits, or six bytes. + Therefore if strm->avail_in >= 6, then there is enough input to avoid + checking for available input while decoding. + + - The wide input data reading option reads 64 input bits at a time. Thus, + if strm->avail_in >= 8, then there is enough input to avoid checking for + available input while decoding. Reading consumes the input with: + + hold |= read64le(in) << bits; + in += 6; + bits += 48; + + reporting 6 bytes of new input because |bits| is 0..15 (2 bytes rounded + up, worst case) and 6 bytes is enough to decode as noted above. At exit, + hold &= (1U << bits) - 1 drops excess input to keep the invariant: + + (state->hold >> state->bits) == 0 + + INFLATE_FAST_MIN_OUTPUT: 258 bytes + + - The maximum bytes that a single length/distance pair can output is 258 + bytes, which is the maximum length that can be coded. inflate_fast() + requires strm->avail_out >= 258 for each loop to avoid checking for + available output space while decoding. + */ +void ZLIB_INTERNAL inflate_fast_chunk_(strm, start) +z_streamp strm; +unsigned start; /* inflate()'s starting value for strm->avail_out */ +{ + struct inflate_state FAR *state; + z_const unsigned char FAR *in; /* local strm->next_in */ + z_const unsigned char FAR *last; /* have enough input while in < last */ + unsigned char FAR *out; /* local strm->next_out */ + unsigned char FAR *beg; /* inflate()'s initial strm->next_out */ + unsigned char FAR *end; /* while out < end, enough space available */ + unsigned char FAR *limit; /* safety limit for chunky copies */ +#ifdef INFLATE_STRICT + unsigned dmax; /* maximum distance from zlib header */ +#endif + unsigned wsize; /* window size or zero if not using window */ + unsigned whave; /* valid bytes in the window */ + unsigned wnext; /* window write index */ + unsigned char FAR *window; /* allocated sliding window, if wsize != 0 */ + inflate_holder_t hold; /* local strm->hold */ + unsigned bits; /* local strm->bits */ + code const FAR *lcode; /* local strm->lencode */ + code const FAR *dcode; /* local strm->distcode */ + unsigned lmask; /* mask for first level of length codes */ + unsigned dmask; /* mask for first level of distance codes */ + code here; /* retrieved table entry */ + unsigned op; /* code bits, operation, extra bits, or */ + /* window position, window bytes to copy */ + unsigned len; /* match length, unused bytes */ + unsigned dist; /* match distance */ + unsigned char FAR *from; /* where to copy match from */ + + /* copy state to local variables */ + state = (struct inflate_state FAR *)strm->state; + in = strm->next_in; + last = in + (strm->avail_in - (INFLATE_FAST_MIN_INPUT - 1)); + out = strm->next_out; + beg = out - (start - strm->avail_out); + end = out + (strm->avail_out - (INFLATE_FAST_MIN_OUTPUT - 1)); + limit = out + strm->avail_out; +#ifdef INFLATE_STRICT + dmax = state->dmax; +#endif + wsize = state->wsize; + whave = state->whave; + wnext = (state->wnext == 0 && whave >= wsize) ? wsize : state->wnext; + window = state->window; + hold = state->hold; + bits = state->bits; + lcode = state->lencode; + dcode = state->distcode; + lmask = (1U << state->lenbits) - 1; + dmask = (1U << state->distbits) - 1; + + /* decode literals and length/distances until end-of-block or not enough + input data or output space */ + do { + if (bits < 15) { +#ifdef INFLATE_CHUNK_READ_64LE + hold |= read64le(in) << bits; + in += 6; + bits += 48; +#else + hold += (unsigned long)(*in++) << bits; + bits += 8; + hold += (unsigned long)(*in++) << bits; + bits += 8; +#endif + } + here = lcode[hold & lmask]; + dolen: + op = (unsigned)(here.bits); + hold >>= op; + bits -= op; + op = (unsigned)(here.op); + if (op == 0) { /* literal */ + Tracevv((stderr, here.val >= 0x20 && here.val < 0x7f ? + "inflate: literal '%c'\n" : + "inflate: literal 0x%02x\n", here.val)); + *out++ = (unsigned char)(here.val); + } + else if (op & 16) { /* length base */ + len = (unsigned)(here.val); + op &= 15; /* number of extra bits */ + if (op) { + if (bits < op) { +#ifdef INFLATE_CHUNK_READ_64LE + hold |= read64le(in) << bits; + in += 6; + bits += 48; +#else + hold += (unsigned long)(*in++) << bits; + bits += 8; +#endif + } + len += (unsigned)hold & ((1U << op) - 1); + hold >>= op; + bits -= op; + } + Tracevv((stderr, "inflate: length %u\n", len)); + if (bits < 15) { +#ifdef INFLATE_CHUNK_READ_64LE + hold |= read64le(in) << bits; + in += 6; + bits += 48; +#else + hold += (unsigned long)(*in++) << bits; + bits += 8; + hold += (unsigned long)(*in++) << bits; + bits += 8; +#endif + } + here = dcode[hold & dmask]; + dodist: + op = (unsigned)(here.bits); + hold >>= op; + bits -= op; + op = (unsigned)(here.op); + if (op & 16) { /* distance base */ + dist = (unsigned)(here.val); + op &= 15; /* number of extra bits */ + if (bits < op) { +#ifdef INFLATE_CHUNK_READ_64LE + hold |= read64le(in) << bits; + in += 6; + bits += 48; +#else + hold += (unsigned long)(*in++) << bits; + bits += 8; + if (bits < op) { + hold += (unsigned long)(*in++) << bits; + bits += 8; + } +#endif + } + dist += (unsigned)hold & ((1U << op) - 1); +#ifdef INFLATE_STRICT + if (dist > dmax) { + strm->msg = (char *)"invalid distance too far back"; + state->mode = BAD; + break; + } +#endif + hold >>= op; + bits -= op; + Tracevv((stderr, "inflate: distance %u\n", dist)); + op = (unsigned)(out - beg); /* max distance in output */ + if (dist > op) { /* see if copy from window */ + op = dist - op; /* distance back in window */ + if (op > whave) { + if (state->sane) { + strm->msg = + (char *)"invalid distance too far back"; + state->mode = BAD; + break; + } +#ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR + if (len <= op - whave) { + do { + *out++ = 0; + } while (--len); + continue; + } + len -= op - whave; + do { + *out++ = 0; + } while (--op > whave); + if (op == 0) { + from = out - dist; + do { + *out++ = *from++; + } while (--len); + continue; + } +#endif + } + from = window; + if (wnext >= op) { /* contiguous in window */ + from += wnext - op; + } + else { /* wrap around window */ + op -= wnext; + from += wsize - op; + if (op < len) { /* some from end of window */ + len -= op; + out = chunkcopy_safe(out, from, op, limit); + from = window; /* more from start of window */ + op = wnext; + /* This (rare) case can create a situation where + the first chunkcopy below must be checked. + */ + } + } + if (op < len) { /* still need some from output */ + out = chunkcopy_safe(out, from, op, limit); + len -= op; + /* When dist is small the amount of data that can be + copied from the window is also small, and progress + towards the dangerous end of the output buffer is + also small. This means that for trivial memsets and + for chunkunroll_relaxed() a safety check is + unnecessary. However, these conditions may not be + entered at all, and in that case it's possible that + the main copy is near the end. + */ + out = chunkunroll_relaxed(out, &dist, &len); + out = chunkcopy_safe_ugly(out, dist, len, limit); + } else { + /* from points to window, so there is no risk of + overlapping pointers requiring memset-like behaviour + */ + out = chunkcopy_safe(out, from, len, limit); + } + } + else { + /* Whole reference is in range of current output. No + range checks are necessary because we start with room + for at least 258 bytes of output, so unroll and roundoff + operations can write beyond `out+len` so long as they + stay within 258 bytes of `out`. + */ + out = chunkcopy_lapped_relaxed(out, dist, len); + } + } + else if ((op & 64) == 0) { /* 2nd level distance code */ + here = dcode[here.val + (hold & ((1U << op) - 1))]; + goto dodist; + } + else { + strm->msg = (char *)"invalid distance code"; + state->mode = BAD; + break; + } + } + else if ((op & 64) == 0) { /* 2nd level length code */ + here = lcode[here.val + (hold & ((1U << op) - 1))]; + goto dolen; + } + else if (op & 32) { /* end-of-block */ + Tracevv((stderr, "inflate: end of block\n")); + state->mode = TYPE; + break; + } + else { + strm->msg = (char *)"invalid literal/length code"; + state->mode = BAD; + break; + } + } while (in < last && out < end); + + /* return unused bytes (on entry, bits < 8, so in won't go too far back) */ + len = bits >> 3; + in -= len; + bits -= len << 3; + hold &= (1U << bits) - 1; + + /* update state and return */ + strm->next_in = in; + strm->next_out = out; + strm->avail_in = (unsigned)(in < last ? + (INFLATE_FAST_MIN_INPUT - 1) + (last - in) : + (INFLATE_FAST_MIN_INPUT - 1) - (in - last)); + strm->avail_out = (unsigned)(out < end ? + (INFLATE_FAST_MIN_OUTPUT - 1) + (end - out) : + (INFLATE_FAST_MIN_OUTPUT - 1) - (out - end)); + state->hold = hold; + state->bits = bits; + + Assert((state->hold >> state->bits) == 0, "invalid input data state"); + return; +} + +/* + inflate_fast() speedups that turned out slower (on a PowerPC G3 750CXe): + - Using bit fields for code structure + - Different op definition to avoid & for extra bits (do & for table bits) + - Three separate decoding do-loops for direct, window, and wnext == 0 + - Special case for distance > 1 copies to do overlapped load and store copy + - Explicit branch predictions (based on measured branch probabilities) + - Deferring match copy and interspersed it with decoding subsequent codes + - Swapping literal/length else + - Swapping window/direct else + - Larger unrolled copy loops (three is about right) + - Moving len -= 3 statement into middle of loop + */ + +#endif /* !ASMINF */ diff --git a/third_party/zlib/contrib/optimizations/inffast_chunk.h b/third_party/zlib/contrib/optimizations/inffast_chunk.h new file mode 100644 index 00000000..39c771b8 --- /dev/null +++ b/third_party/zlib/contrib/optimizations/inffast_chunk.h @@ -0,0 +1,26 @@ +/* inffast_chunk.h -- header to use inffast_chunk.c + * Copyright (C) 1995-2003, 2010 Mark Adler + * Copyright (C) 2017 ARM, Inc. + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +/* WARNING: this file should *not* be used by applications. It is + part of the implementation of the compression library and is + subject to change. Applications should only use zlib.h. + */ + +#include "inffast.h" + +/* INFLATE_FAST_MIN_INPUT: the minimum number of input bytes needed so that + we can safely call inflate_fast() with only one up-front bounds check. One + length/distance code pair (15 bits for the length code, 5 bits for length + extra, 15 bits for the distance code, 13 bits for distance extra) requires + reading up to 48 input bits (6 bytes). The wide input data reading option + requires a little endian machine, and reads 64 input bits (8 bytes). +*/ +#ifdef INFLATE_CHUNK_READ_64LE +#undef INFLATE_FAST_MIN_INPUT +#define INFLATE_FAST_MIN_INPUT 8 +#endif + +void ZLIB_INTERNAL inflate_fast_chunk_ OF((z_streamp strm, unsigned start)); diff --git a/third_party/zlib/contrib/optimizations/inflate.c b/third_party/zlib/contrib/optimizations/inflate.c new file mode 100644 index 00000000..81d558bd --- /dev/null +++ b/third_party/zlib/contrib/optimizations/inflate.c @@ -0,0 +1,1583 @@ +/* inflate.c -- zlib decompression + * Copyright (C) 1995-2016 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +/* + * Change history: + * + * 1.2.beta0 24 Nov 2002 + * - First version -- complete rewrite of inflate to simplify code, avoid + * creation of window when not needed, minimize use of window when it is + * needed, make inffast.c even faster, implement gzip decoding, and to + * improve code readability and style over the previous zlib inflate code + * + * 1.2.beta1 25 Nov 2002 + * - Use pointers for available input and output checking in inffast.c + * - Remove input and output counters in inffast.c + * - Change inffast.c entry and loop from avail_in >= 7 to >= 6 + * - Remove unnecessary second byte pull from length extra in inffast.c + * - Unroll direct copy to three copies per loop in inffast.c + * + * 1.2.beta2 4 Dec 2002 + * - Change external routine names to reduce potential conflicts + * - Correct filename to inffixed.h for fixed tables in inflate.c + * - Make hbuf[] unsigned char to match parameter type in inflate.c + * - Change strm->next_out[-state->offset] to *(strm->next_out - state->offset) + * to avoid negation problem on Alphas (64 bit) in inflate.c + * + * 1.2.beta3 22 Dec 2002 + * - Add comments on state->bits assertion in inffast.c + * - Add comments on op field in inftrees.h + * - Fix bug in reuse of allocated window after inflateReset() + * - Remove bit fields--back to byte structure for speed + * - Remove distance extra == 0 check in inflate_fast()--only helps for lengths + * - Change post-increments to pre-increments in inflate_fast(), PPC biased? + * - Add compile time option, POSTINC, to use post-increments instead (Intel?) + * - Make MATCH copy in inflate() much faster for when inflate_fast() not used + * - Use local copies of stream next and avail values, as well as local bit + * buffer and bit count in inflate()--for speed when inflate_fast() not used + * + * 1.2.beta4 1 Jan 2003 + * - Split ptr - 257 statements in inflate_table() to avoid compiler warnings + * - Move a comment on output buffer sizes from inffast.c to inflate.c + * - Add comments in inffast.c to introduce the inflate_fast() routine + * - Rearrange window copies in inflate_fast() for speed and simplification + * - Unroll last copy for window match in inflate_fast() + * - Use local copies of window variables in inflate_fast() for speed + * - Pull out common wnext == 0 case for speed in inflate_fast() + * - Make op and len in inflate_fast() unsigned for consistency + * - Add FAR to lcode and dcode declarations in inflate_fast() + * - Simplified bad distance check in inflate_fast() + * - Added inflateBackInit(), inflateBack(), and inflateBackEnd() in new + * source file infback.c to provide a call-back interface to inflate for + * programs like gzip and unzip -- uses window as output buffer to avoid + * window copying + * + * 1.2.beta5 1 Jan 2003 + * - Improved inflateBack() interface to allow the caller to provide initial + * input in strm. + * - Fixed stored blocks bug in inflateBack() + * + * 1.2.beta6 4 Jan 2003 + * - Added comments in inffast.c on effectiveness of POSTINC + * - Typecasting all around to reduce compiler warnings + * - Changed loops from while (1) or do {} while (1) to for (;;), again to + * make compilers happy + * - Changed type of window in inflateBackInit() to unsigned char * + * + * 1.2.beta7 27 Jan 2003 + * - Changed many types to unsigned or unsigned short to avoid warnings + * - Added inflateCopy() function + * + * 1.2.0 9 Mar 2003 + * - Changed inflateBack() interface to provide separate opaque descriptors + * for the in() and out() functions + * - Changed inflateBack() argument and in_func typedef to swap the length + * and buffer address return values for the input function + * - Check next_in and next_out for Z_NULL on entry to inflate() + * + * The history for versions after 1.2.0 are in ChangeLog in zlib distribution. + */ + +#include "zutil.h" +#include "inftrees.h" +#include "inflate.h" +#include "contrib/optimizations/inffast_chunk.h" +#include "contrib/optimizations/chunkcopy.h" + +#ifdef MAKEFIXED +# ifndef BUILDFIXED +# define BUILDFIXED +# endif +#endif + +/* function prototypes */ +local int inflateStateCheck OF((z_streamp strm)); +local void fixedtables OF((struct inflate_state FAR *state)); +local int updatewindow OF((z_streamp strm, const unsigned char FAR *end, + unsigned copy)); +#ifdef BUILDFIXED + void makefixed OF((void)); +#endif +local unsigned syncsearch OF((unsigned FAR *have, const unsigned char FAR *buf, + unsigned len)); + +local int inflateStateCheck(strm) +z_streamp strm; +{ + struct inflate_state FAR *state; + if (strm == Z_NULL || + strm->zalloc == (alloc_func)0 || strm->zfree == (free_func)0) + return 1; + state = (struct inflate_state FAR *)strm->state; + if (state == Z_NULL || state->strm != strm || + state->mode < HEAD || state->mode > SYNC) + return 1; + return 0; +} + +int ZEXPORT inflateResetKeep(strm) +z_streamp strm; +{ + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + strm->total_in = strm->total_out = state->total = 0; + strm->msg = Z_NULL; + if (state->wrap) /* to support ill-conceived Java test suite */ + strm->adler = state->wrap & 1; + state->mode = HEAD; + state->last = 0; + state->havedict = 0; + state->dmax = 32768U; + state->head = Z_NULL; + state->hold = 0; + state->bits = 0; + state->lencode = state->distcode = state->next = state->codes; + state->sane = 1; + state->back = -1; + Tracev((stderr, "inflate: reset\n")); + return Z_OK; +} + +int ZEXPORT inflateReset(strm) +z_streamp strm; +{ + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + state->wsize = 0; + state->whave = 0; + state->wnext = 0; + return inflateResetKeep(strm); +} + +int ZEXPORT inflateReset2(strm, windowBits) +z_streamp strm; +int windowBits; +{ + int wrap; + struct inflate_state FAR *state; + + /* get the state */ + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + + /* extract wrap request from windowBits parameter */ + if (windowBits < 0) { + wrap = 0; + windowBits = -windowBits; + } + else { + wrap = (windowBits >> 4) + 5; +#ifdef GUNZIP + if (windowBits < 48) + windowBits &= 15; +#endif + } + + /* set number of window bits, free window if different */ + if (windowBits && (windowBits < 8 || windowBits > 15)) + return Z_STREAM_ERROR; + if (state->window != Z_NULL && state->wbits != (unsigned)windowBits) { + ZFREE(strm, state->window); + state->window = Z_NULL; + } + + /* update state and reset the rest of it */ + state->wrap = wrap; + state->wbits = (unsigned)windowBits; + return inflateReset(strm); +} + +int ZEXPORT inflateInit2_(strm, windowBits, version, stream_size) +z_streamp strm; +int windowBits; +const char *version; +int stream_size; +{ + int ret; + struct inflate_state FAR *state; + + if (version == Z_NULL || version[0] != ZLIB_VERSION[0] || + stream_size != (int)(sizeof(z_stream))) + return Z_VERSION_ERROR; + if (strm == Z_NULL) return Z_STREAM_ERROR; + strm->msg = Z_NULL; /* in case we return an error */ + if (strm->zalloc == (alloc_func)0) { +#ifdef Z_SOLO + return Z_STREAM_ERROR; +#else + strm->zalloc = zcalloc; + strm->opaque = (voidpf)0; +#endif + } + if (strm->zfree == (free_func)0) +#ifdef Z_SOLO + return Z_STREAM_ERROR; +#else + strm->zfree = zcfree; +#endif + state = (struct inflate_state FAR *) + ZALLOC(strm, 1, sizeof(struct inflate_state)); + if (state == Z_NULL) return Z_MEM_ERROR; + Tracev((stderr, "inflate: allocated\n")); + strm->state = (struct internal_state FAR *)state; + state->strm = strm; + state->window = Z_NULL; + state->mode = HEAD; /* to pass state test in inflateReset2() */ + state->check = 1L; /* 1L is the result of adler32() zero length data */ + ret = inflateReset2(strm, windowBits); + if (ret != Z_OK) { + ZFREE(strm, state); + strm->state = Z_NULL; + } + return ret; +} + +int ZEXPORT inflateInit_(strm, version, stream_size) +z_streamp strm; +const char *version; +int stream_size; +{ + return inflateInit2_(strm, DEF_WBITS, version, stream_size); +} + +int ZEXPORT inflatePrime(strm, bits, value) +z_streamp strm; +int bits; +int value; +{ + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + if (bits < 0) { + state->hold = 0; + state->bits = 0; + return Z_OK; + } + if (bits > 16 || state->bits + (uInt)bits > 32) return Z_STREAM_ERROR; + value &= (1L << bits) - 1; + state->hold += (unsigned)value << state->bits; + state->bits += (uInt)bits; + return Z_OK; +} + +/* + Return state with length and distance decoding tables and index sizes set to + fixed code decoding. Normally this returns fixed tables from inffixed.h. + If BUILDFIXED is defined, then instead this routine builds the tables the + first time it's called, and returns those tables the first time and + thereafter. This reduces the size of the code by about 2K bytes, in + exchange for a little execution time. However, BUILDFIXED should not be + used for threaded applications, since the rewriting of the tables and virgin + may not be thread-safe. + */ +local void fixedtables(state) +struct inflate_state FAR *state; +{ +#ifdef BUILDFIXED + static int virgin = 1; + static code *lenfix, *distfix; + static code fixed[544]; + + /* build fixed huffman tables if first call (may not be thread safe) */ + if (virgin) { + unsigned sym, bits; + static code *next; + + /* literal/length table */ + sym = 0; + while (sym < 144) state->lens[sym++] = 8; + while (sym < 256) state->lens[sym++] = 9; + while (sym < 280) state->lens[sym++] = 7; + while (sym < 288) state->lens[sym++] = 8; + next = fixed; + lenfix = next; + bits = 9; + inflate_table(LENS, state->lens, 288, &(next), &(bits), state->work); + + /* distance table */ + sym = 0; + while (sym < 32) state->lens[sym++] = 5; + distfix = next; + bits = 5; + inflate_table(DISTS, state->lens, 32, &(next), &(bits), state->work); + + /* do this just once */ + virgin = 0; + } +#else /* !BUILDFIXED */ +# include "inffixed.h" +#endif /* BUILDFIXED */ + state->lencode = lenfix; + state->lenbits = 9; + state->distcode = distfix; + state->distbits = 5; +} + +#ifdef MAKEFIXED +#include <stdio.h> + +/* + Write out the inffixed.h that is #include'd above. Defining MAKEFIXED also + defines BUILDFIXED, so the tables are built on the fly. makefixed() writes + those tables to stdout, which would be piped to inffixed.h. A small program + can simply call makefixed to do this: + + void makefixed(void); + + int main(void) + { + makefixed(); + return 0; + } + + Then that can be linked with zlib built with MAKEFIXED defined and run: + + a.out > inffixed.h + */ +void makefixed() +{ + unsigned low, size; + struct inflate_state state; + + fixedtables(&state); + puts(" /* inffixed.h -- table for decoding fixed codes"); + puts(" * Generated automatically by makefixed()."); + puts(" */"); + puts(""); + puts(" /* WARNING: this file should *not* be used by applications."); + puts(" It is part of the implementation of this library and is"); + puts(" subject to change. Applications should only use zlib.h."); + puts(" */"); + puts(""); + size = 1U << 9; + printf(" static const code lenfix[%u] = {", size); + low = 0; + for (;;) { + if ((low % 7) == 0) printf("\n "); + printf("{%u,%u,%d}", (low & 127) == 99 ? 64 : state.lencode[low].op, + state.lencode[low].bits, state.lencode[low].val); + if (++low == size) break; + putchar(','); + } + puts("\n };"); + size = 1U << 5; + printf("\n static const code distfix[%u] = {", size); + low = 0; + for (;;) { + if ((low % 6) == 0) printf("\n "); + printf("{%u,%u,%d}", state.distcode[low].op, state.distcode[low].bits, + state.distcode[low].val); + if (++low == size) break; + putchar(','); + } + puts("\n };"); +} +#endif /* MAKEFIXED */ + +/* + Update the window with the last wsize (normally 32K) bytes written before + returning. If window does not exist yet, create it. This is only called + when a window is already in use, or when output has been written during this + inflate call, but the end of the deflate stream has not been reached yet. + It is also called to create a window for dictionary data when a dictionary + is loaded. + + Providing output buffers larger than 32K to inflate() should provide a speed + advantage, since only the last 32K of output is copied to the sliding window + upon return from inflate(), and since all distances after the first 32K of + output will fall in the output data, making match copies simpler and faster. + The advantage may be dependent on the size of the processor's data caches. + */ +local int updatewindow(strm, end, copy) +z_streamp strm; +const Bytef *end; +unsigned copy; +{ + struct inflate_state FAR *state; + unsigned dist; + + state = (struct inflate_state FAR *)strm->state; + + /* if it hasn't been done already, allocate space for the window */ + if (state->window == Z_NULL) { + unsigned wsize = 1U << state->wbits; + state->window = (unsigned char FAR *) + ZALLOC(strm, wsize + CHUNKCOPY_CHUNK_SIZE, + sizeof(unsigned char)); + if (state->window == Z_NULL) return 1; +#ifdef INFLATE_CLEAR_UNUSED_UNDEFINED + /* Copies from the overflow portion of this buffer are undefined and + may cause analysis tools to raise a warning if we don't initialize + it. However, this undefined data overwrites other undefined data + and is subsequently either overwritten or left deliberately + undefined at the end of decode; so there's really no point. + */ + zmemzero(state->window + wsize, CHUNKCOPY_CHUNK_SIZE); +#endif + } + + /* if window not in use yet, initialize */ + if (state->wsize == 0) { + state->wsize = 1U << state->wbits; + state->wnext = 0; + state->whave = 0; + } + + /* copy state->wsize or less output bytes into the circular window */ + if (copy >= state->wsize) { + zmemcpy(state->window, end - state->wsize, state->wsize); + state->wnext = 0; + state->whave = state->wsize; + } + else { + dist = state->wsize - state->wnext; + if (dist > copy) dist = copy; + zmemcpy(state->window + state->wnext, end - copy, dist); + copy -= dist; + if (copy) { + zmemcpy(state->window, end - copy, copy); + state->wnext = copy; + state->whave = state->wsize; + } + else { + state->wnext += dist; + if (state->wnext == state->wsize) state->wnext = 0; + if (state->whave < state->wsize) state->whave += dist; + } + } + return 0; +} + +/* Macros for inflate(): */ + +/* check function to use adler32() for zlib or crc32() for gzip */ +#ifdef GUNZIP +# define UPDATE(check, buf, len) \ + (state->flags ? crc32(check, buf, len) : adler32(check, buf, len)) +#else +# define UPDATE(check, buf, len) adler32(check, buf, len) +#endif + +/* check macros for header crc */ +#ifdef GUNZIP +# define CRC2(check, word) \ + do { \ + hbuf[0] = (unsigned char)(word); \ + hbuf[1] = (unsigned char)((word) >> 8); \ + check = crc32(check, hbuf, 2); \ + } while (0) + +# define CRC4(check, word) \ + do { \ + hbuf[0] = (unsigned char)(word); \ + hbuf[1] = (unsigned char)((word) >> 8); \ + hbuf[2] = (unsigned char)((word) >> 16); \ + hbuf[3] = (unsigned char)((word) >> 24); \ + check = crc32(check, hbuf, 4); \ + } while (0) +#endif + +/* Load registers with state in inflate() for speed */ +#define LOAD() \ + do { \ + put = strm->next_out; \ + left = strm->avail_out; \ + next = strm->next_in; \ + have = strm->avail_in; \ + hold = state->hold; \ + bits = state->bits; \ + } while (0) + +/* Restore state from registers in inflate() */ +#define RESTORE() \ + do { \ + strm->next_out = put; \ + strm->avail_out = left; \ + strm->next_in = next; \ + strm->avail_in = have; \ + state->hold = hold; \ + state->bits = bits; \ + } while (0) + +/* Clear the input bit accumulator */ +#define INITBITS() \ + do { \ + hold = 0; \ + bits = 0; \ + } while (0) + +/* Get a byte of input into the bit accumulator, or return from inflate() + if there is no input available. */ +#define PULLBYTE() \ + do { \ + if (have == 0) goto inf_leave; \ + have--; \ + hold += (unsigned long)(*next++) << bits; \ + bits += 8; \ + } while (0) + +/* Assure that there are at least n bits in the bit accumulator. If there is + not enough available input to do that, then return from inflate(). */ +#define NEEDBITS(n) \ + do { \ + while (bits < (unsigned)(n)) \ + PULLBYTE(); \ + } while (0) + +/* Return the low n bits of the bit accumulator (n < 16) */ +#define BITS(n) \ + ((unsigned)hold & ((1U << (n)) - 1)) + +/* Remove n bits from the bit accumulator */ +#define DROPBITS(n) \ + do { \ + hold >>= (n); \ + bits -= (unsigned)(n); \ + } while (0) + +/* Remove zero to seven bits as needed to go to a byte boundary */ +#define BYTEBITS() \ + do { \ + hold >>= bits & 7; \ + bits -= bits & 7; \ + } while (0) + +/* + inflate() uses a state machine to process as much input data and generate as + much output data as possible before returning. The state machine is + structured roughly as follows: + + for (;;) switch (state) { + ... + case STATEn: + if (not enough input data or output space to make progress) + return; + ... make progress ... + state = STATEm; + break; + ... + } + + so when inflate() is called again, the same case is attempted again, and + if the appropriate resources are provided, the machine proceeds to the + next state. The NEEDBITS() macro is usually the way the state evaluates + whether it can proceed or should return. NEEDBITS() does the return if + the requested bits are not available. The typical use of the BITS macros + is: + + NEEDBITS(n); + ... do something with BITS(n) ... + DROPBITS(n); + + where NEEDBITS(n) either returns from inflate() if there isn't enough + input left to load n bits into the accumulator, or it continues. BITS(n) + gives the low n bits in the accumulator. When done, DROPBITS(n) drops + the low n bits off the accumulator. INITBITS() clears the accumulator + and sets the number of available bits to zero. BYTEBITS() discards just + enough bits to put the accumulator on a byte boundary. After BYTEBITS() + and a NEEDBITS(8), then BITS(8) would return the next byte in the stream. + + NEEDBITS(n) uses PULLBYTE() to get an available byte of input, or to return + if there is no input available. The decoding of variable length codes uses + PULLBYTE() directly in order to pull just enough bytes to decode the next + code, and no more. + + Some states loop until they get enough input, making sure that enough + state information is maintained to continue the loop where it left off + if NEEDBITS() returns in the loop. For example, want, need, and keep + would all have to actually be part of the saved state in case NEEDBITS() + returns: + + case STATEw: + while (want < need) { + NEEDBITS(n); + keep[want++] = BITS(n); + DROPBITS(n); + } + state = STATEx; + case STATEx: + + As shown above, if the next state is also the next case, then the break + is omitted. + + A state may also return if there is not enough output space available to + complete that state. Those states are copying stored data, writing a + literal byte, and copying a matching string. + + When returning, a "goto inf_leave" is used to update the total counters, + update the check value, and determine whether any progress has been made + during that inflate() call in order to return the proper return code. + Progress is defined as a change in either strm->avail_in or strm->avail_out. + When there is a window, goto inf_leave will update the window with the last + output written. If a goto inf_leave occurs in the middle of decompression + and there is no window currently, goto inf_leave will create one and copy + output to the window for the next call of inflate(). + + In this implementation, the flush parameter of inflate() only affects the + return code (per zlib.h). inflate() always writes as much as possible to + strm->next_out, given the space available and the provided input--the effect + documented in zlib.h of Z_SYNC_FLUSH. Furthermore, inflate() always defers + the allocation of and copying into a sliding window until necessary, which + provides the effect documented in zlib.h for Z_FINISH when the entire input + stream available. So the only thing the flush parameter actually does is: + when flush is set to Z_FINISH, inflate() cannot return Z_OK. Instead it + will return Z_BUF_ERROR if it has not reached the end of the stream. + */ + +int ZEXPORT inflate(strm, flush) +z_streamp strm; +int flush; +{ + struct inflate_state FAR *state; + z_const unsigned char FAR *next; /* next input */ + unsigned char FAR *put; /* next output */ + unsigned have, left; /* available input and output */ + unsigned long hold; /* bit buffer */ + unsigned bits; /* bits in bit buffer */ + unsigned in, out; /* save starting available input and output */ + unsigned copy; /* number of stored or match bytes to copy */ + unsigned char FAR *from; /* where to copy match bytes from */ + code here; /* current decoding table entry */ + code last; /* parent table entry */ + unsigned len; /* length to copy for repeats, bits to drop */ + int ret; /* return code */ +#ifdef GUNZIP + unsigned char hbuf[4]; /* buffer for gzip header crc calculation */ +#endif + static const unsigned short order[19] = /* permutation of code lengths */ + {16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15}; + + if (inflateStateCheck(strm) || strm->next_out == Z_NULL || + (strm->next_in == Z_NULL && strm->avail_in != 0)) + return Z_STREAM_ERROR; + + state = (struct inflate_state FAR *)strm->state; + if (state->mode == TYPE) state->mode = TYPEDO; /* skip check */ + LOAD(); + in = have; + out = left; + ret = Z_OK; + for (;;) + switch (state->mode) { + case HEAD: + if (state->wrap == 0) { + state->mode = TYPEDO; + break; + } + NEEDBITS(16); +#ifdef GUNZIP + if ((state->wrap & 2) && hold == 0x8b1f) { /* gzip header */ + if (state->wbits == 0) + state->wbits = 15; + state->check = crc32(0L, Z_NULL, 0); + CRC2(state->check, hold); + INITBITS(); + state->mode = FLAGS; + break; + } + state->flags = 0; /* expect zlib header */ + if (state->head != Z_NULL) + state->head->done = -1; + if (!(state->wrap & 1) || /* check if zlib header allowed */ +#else + if ( +#endif + ((BITS(8) << 8) + (hold >> 8)) % 31) { + strm->msg = (char *)"incorrect header check"; + state->mode = BAD; + break; + } + if (BITS(4) != Z_DEFLATED) { + strm->msg = (char *)"unknown compression method"; + state->mode = BAD; + break; + } + DROPBITS(4); + len = BITS(4) + 8; + if (state->wbits == 0) + state->wbits = len; + if (len > 15 || len > state->wbits) { + strm->msg = (char *)"invalid window size"; + state->mode = BAD; + break; + } + state->dmax = 1U << len; + Tracev((stderr, "inflate: zlib header ok\n")); + strm->adler = state->check = adler32(0L, Z_NULL, 0); + state->mode = hold & 0x200 ? DICTID : TYPE; + INITBITS(); + break; +#ifdef GUNZIP + case FLAGS: + NEEDBITS(16); + state->flags = (int)(hold); + if ((state->flags & 0xff) != Z_DEFLATED) { + strm->msg = (char *)"unknown compression method"; + state->mode = BAD; + break; + } + if (state->flags & 0xe000) { + strm->msg = (char *)"unknown header flags set"; + state->mode = BAD; + break; + } + if (state->head != Z_NULL) + state->head->text = (int)((hold >> 8) & 1); + if ((state->flags & 0x0200) && (state->wrap & 4)) + CRC2(state->check, hold); + INITBITS(); + state->mode = TIME; + case TIME: + NEEDBITS(32); + if (state->head != Z_NULL) + state->head->time = hold; + if ((state->flags & 0x0200) && (state->wrap & 4)) + CRC4(state->check, hold); + INITBITS(); + state->mode = OS; + case OS: + NEEDBITS(16); + if (state->head != Z_NULL) { + state->head->xflags = (int)(hold & 0xff); + state->head->os = (int)(hold >> 8); + } + if ((state->flags & 0x0200) && (state->wrap & 4)) + CRC2(state->check, hold); + INITBITS(); + state->mode = EXLEN; + case EXLEN: + if (state->flags & 0x0400) { + NEEDBITS(16); + state->length = (unsigned)(hold); + if (state->head != Z_NULL) + state->head->extra_len = (unsigned)hold; + if ((state->flags & 0x0200) && (state->wrap & 4)) + CRC2(state->check, hold); + INITBITS(); + } + else if (state->head != Z_NULL) + state->head->extra = Z_NULL; + state->mode = EXTRA; + case EXTRA: + if (state->flags & 0x0400) { + copy = state->length; + if (copy > have) copy = have; + if (copy) { + if (state->head != Z_NULL && + state->head->extra != Z_NULL) { + len = state->head->extra_len - state->length; + zmemcpy(state->head->extra + len, next, + len + copy > state->head->extra_max ? + state->head->extra_max - len : copy); + } + if ((state->flags & 0x0200) && (state->wrap & 4)) + state->check = crc32(state->check, next, copy); + have -= copy; + next += copy; + state->length -= copy; + } + if (state->length) goto inf_leave; + } + state->length = 0; + state->mode = NAME; + case NAME: + if (state->flags & 0x0800) { + if (have == 0) goto inf_leave; + copy = 0; + do { + len = (unsigned)(next[copy++]); + if (state->head != Z_NULL && + state->head->name != Z_NULL && + state->length < state->head->name_max) + state->head->name[state->length++] = (Bytef)len; + } while (len && copy < have); + if ((state->flags & 0x0200) && (state->wrap & 4)) + state->check = crc32(state->check, next, copy); + have -= copy; + next += copy; + if (len) goto inf_leave; + } + else if (state->head != Z_NULL) + state->head->name = Z_NULL; + state->length = 0; + state->mode = COMMENT; + case COMMENT: + if (state->flags & 0x1000) { + if (have == 0) goto inf_leave; + copy = 0; + do { + len = (unsigned)(next[copy++]); + if (state->head != Z_NULL && + state->head->comment != Z_NULL && + state->length < state->head->comm_max) + state->head->comment[state->length++] = (Bytef)len; + } while (len && copy < have); + if ((state->flags & 0x0200) && (state->wrap & 4)) + state->check = crc32(state->check, next, copy); + have -= copy; + next += copy; + if (len) goto inf_leave; + } + else if (state->head != Z_NULL) + state->head->comment = Z_NULL; + state->mode = HCRC; + case HCRC: + if (state->flags & 0x0200) { + NEEDBITS(16); + if ((state->wrap & 4) && hold != (state->check & 0xffff)) { + strm->msg = (char *)"header crc mismatch"; + state->mode = BAD; + break; + } + INITBITS(); + } + if (state->head != Z_NULL) { + state->head->hcrc = (int)((state->flags >> 9) & 1); + state->head->done = 1; + } + strm->adler = state->check = crc32(0L, Z_NULL, 0); + state->mode = TYPE; + break; +#endif + case DICTID: + NEEDBITS(32); + strm->adler = state->check = ZSWAP32(hold); + INITBITS(); + state->mode = DICT; + case DICT: + if (state->havedict == 0) { + RESTORE(); + return Z_NEED_DICT; + } + strm->adler = state->check = adler32(0L, Z_NULL, 0); + state->mode = TYPE; + case TYPE: + if (flush == Z_BLOCK || flush == Z_TREES) goto inf_leave; + case TYPEDO: + if (state->last) { + BYTEBITS(); + state->mode = CHECK; + break; + } + NEEDBITS(3); + state->last = BITS(1); + DROPBITS(1); + switch (BITS(2)) { + case 0: /* stored block */ + Tracev((stderr, "inflate: stored block%s\n", + state->last ? " (last)" : "")); + state->mode = STORED; + break; + case 1: /* fixed block */ + fixedtables(state); + Tracev((stderr, "inflate: fixed codes block%s\n", + state->last ? " (last)" : "")); + state->mode = LEN_; /* decode codes */ + if (flush == Z_TREES) { + DROPBITS(2); + goto inf_leave; + } + break; + case 2: /* dynamic block */ + Tracev((stderr, "inflate: dynamic codes block%s\n", + state->last ? " (last)" : "")); + state->mode = TABLE; + break; + case 3: + strm->msg = (char *)"invalid block type"; + state->mode = BAD; + } + DROPBITS(2); + break; + case STORED: + BYTEBITS(); /* go to byte boundary */ + NEEDBITS(32); + if ((hold & 0xffff) != ((hold >> 16) ^ 0xffff)) { + strm->msg = (char *)"invalid stored block lengths"; + state->mode = BAD; + break; + } + state->length = (unsigned)hold & 0xffff; + Tracev((stderr, "inflate: stored length %u\n", + state->length)); + INITBITS(); + state->mode = COPY_; + if (flush == Z_TREES) goto inf_leave; + case COPY_: + state->mode = COPY; + case COPY: + copy = state->length; + if (copy) { + if (copy > have) copy = have; + if (copy > left) copy = left; + if (copy == 0) goto inf_leave; + zmemcpy(put, next, copy); + have -= copy; + next += copy; + left -= copy; + put += copy; + state->length -= copy; + break; + } + Tracev((stderr, "inflate: stored end\n")); + state->mode = TYPE; + break; + case TABLE: + NEEDBITS(14); + state->nlen = BITS(5) + 257; + DROPBITS(5); + state->ndist = BITS(5) + 1; + DROPBITS(5); + state->ncode = BITS(4) + 4; + DROPBITS(4); +#ifndef PKZIP_BUG_WORKAROUND + if (state->nlen > 286 || state->ndist > 30) { + strm->msg = (char *)"too many length or distance symbols"; + state->mode = BAD; + break; + } +#endif + Tracev((stderr, "inflate: table sizes ok\n")); + state->have = 0; + state->mode = LENLENS; + case LENLENS: + while (state->have < state->ncode) { + NEEDBITS(3); + state->lens[order[state->have++]] = (unsigned short)BITS(3); + DROPBITS(3); + } + while (state->have < 19) + state->lens[order[state->have++]] = 0; + state->next = state->codes; + state->lencode = (const code FAR *)(state->next); + state->lenbits = 7; + ret = inflate_table(CODES, state->lens, 19, &(state->next), + &(state->lenbits), state->work); + if (ret) { + strm->msg = (char *)"invalid code lengths set"; + state->mode = BAD; + break; + } + Tracev((stderr, "inflate: code lengths ok\n")); + state->have = 0; + state->mode = CODELENS; + case CODELENS: + while (state->have < state->nlen + state->ndist) { + for (;;) { + here = state->lencode[BITS(state->lenbits)]; + if ((unsigned)(here.bits) <= bits) break; + PULLBYTE(); + } + if (here.val < 16) { + DROPBITS(here.bits); + state->lens[state->have++] = here.val; + } + else { + if (here.val == 16) { + NEEDBITS(here.bits + 2); + DROPBITS(here.bits); + if (state->have == 0) { + strm->msg = (char *)"invalid bit length repeat"; + state->mode = BAD; + break; + } + len = state->lens[state->have - 1]; + copy = 3 + BITS(2); + DROPBITS(2); + } + else if (here.val == 17) { + NEEDBITS(here.bits + 3); + DROPBITS(here.bits); + len = 0; + copy = 3 + BITS(3); + DROPBITS(3); + } + else { + NEEDBITS(here.bits + 7); + DROPBITS(here.bits); + len = 0; + copy = 11 + BITS(7); + DROPBITS(7); + } + if (state->have + copy > state->nlen + state->ndist) { + strm->msg = (char *)"invalid bit length repeat"; + state->mode = BAD; + break; + } + while (copy--) + state->lens[state->have++] = (unsigned short)len; + } + } + + /* handle error breaks in while */ + if (state->mode == BAD) break; + + /* check for end-of-block code (better have one) */ + if (state->lens[256] == 0) { + strm->msg = (char *)"invalid code -- missing end-of-block"; + state->mode = BAD; + break; + } + + /* build code tables -- note: do not change the lenbits or distbits + values here (9 and 6) without reading the comments in inftrees.h + concerning the ENOUGH constants, which depend on those values */ + state->next = state->codes; + state->lencode = (const code FAR *)(state->next); + state->lenbits = 9; + ret = inflate_table(LENS, state->lens, state->nlen, &(state->next), + &(state->lenbits), state->work); + if (ret) { + strm->msg = (char *)"invalid literal/lengths set"; + state->mode = BAD; + break; + } + state->distcode = (const code FAR *)(state->next); + state->distbits = 6; + ret = inflate_table(DISTS, state->lens + state->nlen, state->ndist, + &(state->next), &(state->distbits), state->work); + if (ret) { + strm->msg = (char *)"invalid distances set"; + state->mode = BAD; + break; + } + Tracev((stderr, "inflate: codes ok\n")); + state->mode = LEN_; + if (flush == Z_TREES) goto inf_leave; + case LEN_: + state->mode = LEN; + case LEN: + if (have >= INFLATE_FAST_MIN_INPUT && + left >= INFLATE_FAST_MIN_OUTPUT) { + RESTORE(); + inflate_fast_chunk_(strm, out); + LOAD(); + if (state->mode == TYPE) + state->back = -1; + break; + } + state->back = 0; + for (;;) { + here = state->lencode[BITS(state->lenbits)]; + if ((unsigned)(here.bits) <= bits) break; + PULLBYTE(); + } + if (here.op && (here.op & 0xf0) == 0) { + last = here; + for (;;) { + here = state->lencode[last.val + + (BITS(last.bits + last.op) >> last.bits)]; + if ((unsigned)(last.bits + here.bits) <= bits) break; + PULLBYTE(); + } + DROPBITS(last.bits); + state->back += last.bits; + } + DROPBITS(here.bits); + state->back += here.bits; + state->length = (unsigned)here.val; + if ((int)(here.op) == 0) { + Tracevv((stderr, here.val >= 0x20 && here.val < 0x7f ? + "inflate: literal '%c'\n" : + "inflate: literal 0x%02x\n", here.val)); + state->mode = LIT; + break; + } + if (here.op & 32) { + Tracevv((stderr, "inflate: end of block\n")); + state->back = -1; + state->mode = TYPE; + break; + } + if (here.op & 64) { + strm->msg = (char *)"invalid literal/length code"; + state->mode = BAD; + break; + } + state->extra = (unsigned)(here.op) & 15; + state->mode = LENEXT; + case LENEXT: + if (state->extra) { + NEEDBITS(state->extra); + state->length += BITS(state->extra); + DROPBITS(state->extra); + state->back += state->extra; + } + Tracevv((stderr, "inflate: length %u\n", state->length)); + state->was = state->length; + state->mode = DIST; + case DIST: + for (;;) { + here = state->distcode[BITS(state->distbits)]; + if ((unsigned)(here.bits) <= bits) break; + PULLBYTE(); + } + if ((here.op & 0xf0) == 0) { + last = here; + for (;;) { + here = state->distcode[last.val + + (BITS(last.bits + last.op) >> last.bits)]; + if ((unsigned)(last.bits + here.bits) <= bits) break; + PULLBYTE(); + } + DROPBITS(last.bits); + state->back += last.bits; + } + DROPBITS(here.bits); + state->back += here.bits; + if (here.op & 64) { + strm->msg = (char *)"invalid distance code"; + state->mode = BAD; + break; + } + state->offset = (unsigned)here.val; + state->extra = (unsigned)(here.op) & 15; + state->mode = DISTEXT; + case DISTEXT: + if (state->extra) { + NEEDBITS(state->extra); + state->offset += BITS(state->extra); + DROPBITS(state->extra); + state->back += state->extra; + } +#ifdef INFLATE_STRICT + if (state->offset > state->dmax) { + strm->msg = (char *)"invalid distance too far back"; + state->mode = BAD; + break; + } +#endif + Tracevv((stderr, "inflate: distance %u\n", state->offset)); + state->mode = MATCH; + case MATCH: + if (left == 0) goto inf_leave; + copy = out - left; + if (state->offset > copy) { /* copy from window */ + copy = state->offset - copy; + if (copy > state->whave) { + if (state->sane) { + strm->msg = (char *)"invalid distance too far back"; + state->mode = BAD; + break; + } +#ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR + Trace((stderr, "inflate.c too far\n")); + copy -= state->whave; + if (copy > state->length) copy = state->length; + if (copy > left) copy = left; + left -= copy; + state->length -= copy; + do { + *put++ = 0; + } while (--copy); + if (state->length == 0) state->mode = LEN; + break; +#endif + } + if (copy > state->wnext) { + copy -= state->wnext; + from = state->window + (state->wsize - copy); + } + else + from = state->window + (state->wnext - copy); + if (copy > state->length) copy = state->length; + if (copy > left) copy = left; + put = chunkcopy_safe(put, from, copy, put + left); + } + else { /* copy from output */ + copy = state->length; + if (copy > left) copy = left; + put = chunkcopy_lapped_safe(put, state->offset, copy, put + left); + } + left -= copy; + state->length -= copy; + if (state->length == 0) state->mode = LEN; + break; + case LIT: + if (left == 0) goto inf_leave; + *put++ = (unsigned char)(state->length); + left--; + state->mode = LEN; + break; + case CHECK: + if (state->wrap) { + NEEDBITS(32); + out -= left; + strm->total_out += out; + state->total += out; + if ((state->wrap & 4) && out) + strm->adler = state->check = + UPDATE(state->check, put - out, out); + out = left; + if ((state->wrap & 4) && ( +#ifdef GUNZIP + state->flags ? hold : +#endif + ZSWAP32(hold)) != state->check) { + strm->msg = (char *)"incorrect data check"; + state->mode = BAD; + break; + } + INITBITS(); + Tracev((stderr, "inflate: check matches trailer\n")); + } +#ifdef GUNZIP + state->mode = LENGTH; + case LENGTH: + if (state->wrap && state->flags) { + NEEDBITS(32); + if (hold != (state->total & 0xffffffffUL)) { + strm->msg = (char *)"incorrect length check"; + state->mode = BAD; + break; + } + INITBITS(); + Tracev((stderr, "inflate: length matches trailer\n")); + } +#endif + state->mode = DONE; + case DONE: + ret = Z_STREAM_END; + goto inf_leave; + case BAD: + ret = Z_DATA_ERROR; + goto inf_leave; + case MEM: + return Z_MEM_ERROR; + case SYNC: + default: + return Z_STREAM_ERROR; + } + + /* + Return from inflate(), updating the total counts and the check value. + If there was no progress during the inflate() call, return a buffer + error. Call updatewindow() to create and/or update the window state. + Note: a memory error from inflate() is non-recoverable. + */ + inf_leave: + /* We write a defined value in the unused space to help mark + * where the stream has ended. We don't use zeros as that can + * mislead clients relying on undefined behavior (i.e. assuming + * that the data is over when the buffer has a zero/null value). + */ + if (left >= CHUNKCOPY_CHUNK_SIZE) + memset(put, 0x55, CHUNKCOPY_CHUNK_SIZE); + else + memset(put, 0x55, left); + + RESTORE(); + if (state->wsize || (out != strm->avail_out && state->mode < BAD && + (state->mode < CHECK || flush != Z_FINISH))) + if (updatewindow(strm, strm->next_out, out - strm->avail_out)) { + state->mode = MEM; + return Z_MEM_ERROR; + } + in -= strm->avail_in; + out -= strm->avail_out; + strm->total_in += in; + strm->total_out += out; + state->total += out; + if ((state->wrap & 4) && out) + strm->adler = state->check = + UPDATE(state->check, strm->next_out - out, out); + strm->data_type = (int)state->bits + (state->last ? 64 : 0) + + (state->mode == TYPE ? 128 : 0) + + (state->mode == LEN_ || state->mode == COPY_ ? 256 : 0); + if (((in == 0 && out == 0) || flush == Z_FINISH) && ret == Z_OK) + ret = Z_BUF_ERROR; + return ret; +} + +int ZEXPORT inflateEnd(strm) +z_streamp strm; +{ + struct inflate_state FAR *state; + if (inflateStateCheck(strm)) + return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + if (state->window != Z_NULL) ZFREE(strm, state->window); + ZFREE(strm, strm->state); + strm->state = Z_NULL; + Tracev((stderr, "inflate: end\n")); + return Z_OK; +} + +int ZEXPORT inflateGetDictionary(strm, dictionary, dictLength) +z_streamp strm; +Bytef *dictionary; +uInt *dictLength; +{ + struct inflate_state FAR *state; + + /* check state */ + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + + /* copy dictionary */ + if (state->whave && dictionary != Z_NULL) { + zmemcpy(dictionary, state->window + state->wnext, + state->whave - state->wnext); + zmemcpy(dictionary + state->whave - state->wnext, + state->window, state->wnext); + } + if (dictLength != Z_NULL) + *dictLength = state->whave; + return Z_OK; +} + +int ZEXPORT inflateSetDictionary(strm, dictionary, dictLength) +z_streamp strm; +const Bytef *dictionary; +uInt dictLength; +{ + struct inflate_state FAR *state; + unsigned long dictid; + int ret; + + /* check state */ + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + if (state->wrap != 0 && state->mode != DICT) + return Z_STREAM_ERROR; + + /* check for correct dictionary identifier */ + if (state->mode == DICT) { + dictid = adler32(0L, Z_NULL, 0); + dictid = adler32(dictid, dictionary, dictLength); + if (dictid != state->check) + return Z_DATA_ERROR; + } + + /* copy dictionary to window using updatewindow(), which will amend the + existing dictionary if appropriate */ + ret = updatewindow(strm, dictionary + dictLength, dictLength); + if (ret) { + state->mode = MEM; + return Z_MEM_ERROR; + } + state->havedict = 1; + Tracev((stderr, "inflate: dictionary set\n")); + return Z_OK; +} + +int ZEXPORT inflateGetHeader(strm, head) +z_streamp strm; +gz_headerp head; +{ + struct inflate_state FAR *state; + + /* check state */ + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + if ((state->wrap & 2) == 0) return Z_STREAM_ERROR; + + /* save header structure */ + state->head = head; + head->done = 0; + return Z_OK; +} + +/* + Search buf[0..len-1] for the pattern: 0, 0, 0xff, 0xff. Return when found + or when out of input. When called, *have is the number of pattern bytes + found in order so far, in 0..3. On return *have is updated to the new + state. If on return *have equals four, then the pattern was found and the + return value is how many bytes were read including the last byte of the + pattern. If *have is less than four, then the pattern has not been found + yet and the return value is len. In the latter case, syncsearch() can be + called again with more data and the *have state. *have is initialized to + zero for the first call. + */ +local unsigned syncsearch(have, buf, len) +unsigned FAR *have; +const unsigned char FAR *buf; +unsigned len; +{ + unsigned got; + unsigned next; + + got = *have; + next = 0; + while (next < len && got < 4) { + if ((int)(buf[next]) == (got < 2 ? 0 : 0xff)) + got++; + else if (buf[next]) + got = 0; + else + got = 4 - got; + next++; + } + *have = got; + return next; +} + +int ZEXPORT inflateSync(strm) +z_streamp strm; +{ + unsigned len; /* number of bytes to look at or looked at */ + unsigned long in, out; /* temporary to save total_in and total_out */ + unsigned char buf[4]; /* to restore bit buffer to byte string */ + struct inflate_state FAR *state; + + /* check parameters */ + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + if (strm->avail_in == 0 && state->bits < 8) return Z_BUF_ERROR; + + /* if first time, start search in bit buffer */ + if (state->mode != SYNC) { + state->mode = SYNC; + state->hold <<= state->bits & 7; + state->bits -= state->bits & 7; + len = 0; + while (state->bits >= 8) { + buf[len++] = (unsigned char)(state->hold); + state->hold >>= 8; + state->bits -= 8; + } + state->have = 0; + syncsearch(&(state->have), buf, len); + } + + /* search available input */ + len = syncsearch(&(state->have), strm->next_in, strm->avail_in); + strm->avail_in -= len; + strm->next_in += len; + strm->total_in += len; + + /* return no joy or set up to restart inflate() on a new block */ + if (state->have != 4) return Z_DATA_ERROR; + in = strm->total_in; out = strm->total_out; + inflateReset(strm); + strm->total_in = in; strm->total_out = out; + state->mode = TYPE; + return Z_OK; +} + +/* + Returns true if inflate is currently at the end of a block generated by + Z_SYNC_FLUSH or Z_FULL_FLUSH. This function is used by one PPP + implementation to provide an additional safety check. PPP uses + Z_SYNC_FLUSH but removes the length bytes of the resulting empty stored + block. When decompressing, PPP checks that at the end of input packet, + inflate is waiting for these length bytes. + */ +int ZEXPORT inflateSyncPoint(strm) +z_streamp strm; +{ + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + return state->mode == STORED && state->bits == 0; +} + +int ZEXPORT inflateCopy(dest, source) +z_streamp dest; +z_streamp source; +{ + struct inflate_state FAR *state; + struct inflate_state FAR *copy; + unsigned char FAR *window; + unsigned wsize; + + /* check input */ + if (inflateStateCheck(source) || dest == Z_NULL) + return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)source->state; + + /* allocate space */ + copy = (struct inflate_state FAR *) + ZALLOC(source, 1, sizeof(struct inflate_state)); + if (copy == Z_NULL) return Z_MEM_ERROR; + window = Z_NULL; + if (state->window != Z_NULL) { + window = (unsigned char FAR *) + ZALLOC(source, 1U << state->wbits, sizeof(unsigned char)); + if (window == Z_NULL) { + ZFREE(source, copy); + return Z_MEM_ERROR; + } + } + + /* copy state */ + zmemcpy((voidpf)dest, (voidpf)source, sizeof(z_stream)); + zmemcpy((voidpf)copy, (voidpf)state, sizeof(struct inflate_state)); + copy->strm = dest; + if (state->lencode >= state->codes && + state->lencode <= state->codes + ENOUGH - 1) { + copy->lencode = copy->codes + (state->lencode - state->codes); + copy->distcode = copy->codes + (state->distcode - state->codes); + } + copy->next = copy->codes + (state->next - state->codes); + if (window != Z_NULL) { + wsize = 1U << state->wbits; + zmemcpy(window, state->window, wsize); + } + copy->window = window; + dest->state = (struct internal_state FAR *)copy; + return Z_OK; +} + +int ZEXPORT inflateUndermine(strm, subvert) +z_streamp strm; +int subvert; +{ + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; +#ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR + state->sane = !subvert; + return Z_OK; +#else + (void)subvert; + state->sane = 1; + return Z_DATA_ERROR; +#endif +} + +int ZEXPORT inflateValidate(strm, check) +z_streamp strm; +int check; +{ + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + if (check) + state->wrap |= 4; + else + state->wrap &= ~4; + return Z_OK; +} + +long ZEXPORT inflateMark(strm) +z_streamp strm; +{ + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) + return -(1L << 16); + state = (struct inflate_state FAR *)strm->state; + return (long)(((unsigned long)((long)state->back)) << 16) + + (state->mode == COPY ? state->length : + (state->mode == MATCH ? state->was - state->length : 0)); +} + +unsigned long ZEXPORT inflateCodesUsed(strm) +z_streamp strm; +{ + struct inflate_state FAR *state; + if (inflateStateCheck(strm)) return (unsigned long)-1; + state = (struct inflate_state FAR *)strm->state; + return (unsigned long)(state->next - state->codes); +} diff --git a/third_party/zlib/contrib/optimizations/insert_string.h b/third_party/zlib/contrib/optimizations/insert_string.h new file mode 100644 index 00000000..9f634ae3 --- /dev/null +++ b/third_party/zlib/contrib/optimizations/insert_string.h @@ -0,0 +1,127 @@ +/* insert_string.h + * + * Copyright 2019 The Chromium Authors. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the Chromium source repository LICENSE file. + */ + +#if defined(_MSC_VER) +#define INLINE __inline +#else +#define INLINE inline +#endif + +#include "cpu_features.h" + +// clang-format off +#if defined(CRC32_SIMD_SSE42_PCLMUL) + #include <smmintrin.h> /* Required to make MSVC bot build pass. */ + + #if defined(__clang__) || defined(__GNUC__) + #define TARGET_CPU_WITH_CRC __attribute__((target("sse4.2"))) + #else + #define TARGET_CPU_WITH_CRC + #endif + + #define _cpu_crc32_u32 _mm_crc32_u32 + +#elif defined(CRC32_ARMV8_CRC32) + #if defined(__clang__) + #define __crc32cw __builtin_arm_crc32cw + #elif defined(__GNUC__) + #define __crc32cw __builtin_aarch64_crc32cw + #endif + + #if defined(__aarch64__) && defined(__clang__) + #define TARGET_CPU_WITH_CRC __attribute__((target("crc"))) + #elif defined(__aarch64__) && defined(__GNUC__) + #define TARGET_CPU_WITH_CRC __attribute__((target("+crc"))) + #elif defined(__clang__) // !defined(__aarch64__) + #define TARGET_CPU_WITH_CRC __attribute__((target("armv8-a,crc"))) + #endif // defined(__aarch64__) + + #define _cpu_crc32_u32 __crc32cw + +#endif +// clang-format on + +#if defined(TARGET_CPU_WITH_CRC) + +TARGET_CPU_WITH_CRC +local INLINE Pos insert_string_simd(deflate_state* const s, const Pos str) { + Pos ret; + unsigned *ip, val, h = 0; + + ip = (unsigned*)&s->window[str]; + val = *ip; + + if (s->level >= 6) + val &= 0xFFFFFF; + + /* Unlike the case of data integrity checks for GZIP format where the + * polynomial used is defined (https://tools.ietf.org/html/rfc1952#page-11), + * here it is just a hash function for the hash table used while + * performing compression. + */ + h = _cpu_crc32_u32(h, val); + + ret = s->head[h & s->hash_mask]; + s->head[h & s->hash_mask] = str; + s->prev[str & s->w_mask] = ret; + return ret; +} + +#endif // TARGET_CPU_WITH_CRC + +/* =========================================================================== + * Update a hash value with the given input byte + * IN assertion: all calls to UPDATE_HASH are made with consecutive input + * characters, so that a running hash key can be computed from the previous + * key instead of complete recalculation each time. + */ +#define UPDATE_HASH(s, h, c) (h = (((h) << s->hash_shift) ^ (c)) & s->hash_mask) + +/* =========================================================================== + * Insert string str in the dictionary and set match_head to the previous head + * of the hash chain (the most recent string with same hash key). Return + * the previous length of the hash chain. + * If this file is compiled with -DFASTEST, the compression level is forced + * to 1, and no hash chains are maintained. + * IN assertion: all calls to INSERT_STRING are made with consecutive input + * characters and the first MIN_MATCH bytes of str are valid (except for + * the last MIN_MATCH-1 bytes of the input file). + */ +local INLINE Pos insert_string_c(deflate_state* const s, const Pos str) { + Pos ret; + + UPDATE_HASH(s, s->ins_h, s->window[(str) + (MIN_MATCH - 1)]); +#ifdef FASTEST + ret = s->head[s->ins_h]; +#else + ret = s->prev[str & s->w_mask] = s->head[s->ins_h]; +#endif + s->head[s->ins_h] = str; + + return ret; +} + +local INLINE Pos insert_string(deflate_state* const s, const Pos str) { +/* insert_string_simd string dictionary insertion: this SIMD symbol hashing + * significantly improves data compression speed. + * + * Note: the generated compressed output is a valid DEFLATE stream but will + * differ from vanilla zlib output ... + */ +#if defined(CHROMIUM_ZLIB_NO_CASTAGNOLI) +/* ... so this build-time option can used to disable the SIMD symbol hasher + * if matching vanilla zlib DEFLATE output is required. + */ (;) /* FALLTHOUGH */ +#elif defined(TARGET_CPU_WITH_CRC) && defined(CRC32_SIMD_SSE42_PCLMUL) + if (x86_cpu_enable_simd) + return insert_string_simd(s, str); +#elif defined(TARGET_CPU_WITH_CRC) && defined(CRC32_ARMV8_CRC32) + if (arm_cpu_enable_crc32) + return insert_string_simd(s, str); +#endif + return insert_string_c(s, str); +} diff --git a/third_party/zlib/contrib/optimizations/slide_hash_neon.h b/third_party/zlib/contrib/optimizations/slide_hash_neon.h new file mode 100644 index 00000000..26995d70 --- /dev/null +++ b/third_party/zlib/contrib/optimizations/slide_hash_neon.h @@ -0,0 +1,65 @@ +/* Copyright 2018 The Chromium Authors. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the Chromium source repository LICENSE file. + */ +#ifndef __SLIDE_HASH__NEON__ +#define __SLIDE_HASH__NEON__ + +#include "deflate.h" +#include <arm_neon.h> + +inline static void ZLIB_INTERNAL neon_slide_hash_update(Posf *hash, + const uInt hash_size, + const ush w_size) +{ + /* NEON 'Q' registers allow to store 128 bits, so we can load 8x16-bits + * values. For further details, check: + * ARM DHT 0002A, section 1.3.2 NEON Registers. + */ + const size_t chunk = sizeof(uint16x8_t) / sizeof(uint16_t); + /* Unrolling the operation yielded a compression performance boost in both + * ARMv7 (from 11.7% to 13.4%) and ARMv8 (from 3.7% to 7.5%) for HTML4 + * content. For full benchmarking data, check: http://crbug.com/863257. + */ + const size_t stride = 2*chunk; + const uint16x8_t v = vdupq_n_u16(w_size); + + for (Posf *end = hash + hash_size; hash != end; hash += stride) { + uint16x8_t m_low = vld1q_u16(hash); + uint16x8_t m_high = vld1q_u16(hash + chunk); + + /* The first 'q' in vqsubq_u16 makes these subtracts saturate to zero, + * replacing the ternary operator expression in the original code: + * (m >= wsize ? m - wsize : NIL). + */ + m_low = vqsubq_u16(m_low, v); + m_high = vqsubq_u16(m_high, v); + + vst1q_u16(hash, m_low); + vst1q_u16(hash + chunk, m_high); + } +} + + +inline static void ZLIB_INTERNAL neon_slide_hash(Posf *head, Posf *prev, + const unsigned short w_size, + const uInt hash_size) +{ + /* + * SIMD implementation for hash table rebase assumes: + * 1. hash chain offset (Pos) is 2 bytes. + * 2. hash table size is multiple of 32 bytes. + * #1 should be true as Pos is defined as "ush" + * #2 should be true as hash_bits are greater than 7 + */ + const size_t size = hash_size * sizeof(head[0]); + Assert(sizeof(Pos) == 2, "Wrong Pos size."); + Assert((size % sizeof(uint16x8_t) * 2) == 0, "Hash table size error."); + + neon_slide_hash_update(head, hash_size, w_size); +#ifndef FASTEST + neon_slide_hash_update(prev, w_size, w_size); +#endif +} + +#endif diff --git a/third_party/zlib/cpu_features.c b/third_party/zlib/cpu_features.c new file mode 100644 index 00000000..70f01bee --- /dev/null +++ b/third_party/zlib/cpu_features.c @@ -0,0 +1,167 @@ +/* cpu_features.c -- Processor features detection. + * + * Copyright 2018 The Chromium Authors. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the Chromium source repository LICENSE file. + */ + +#include "cpu_features.h" +#include "zutil.h" + +#include <stdint.h> +#if defined(_MSC_VER) +#include <intrin.h> +#elif defined(ADLER32_SIMD_SSSE3) +#include <cpuid.h> +#endif + +/* TODO(cavalcantii): remove checks for x86_flags on deflate. + */ +#if defined(ARMV8_OS_MACOS) +/* crc32 is a baseline feature in ARMv8.1-A, and macOS running on arm64 is new + * enough that this can be assumed without runtime detection. */ +int ZLIB_INTERNAL arm_cpu_enable_crc32 = 1; +#else +int ZLIB_INTERNAL arm_cpu_enable_crc32 = 0; +#endif +int ZLIB_INTERNAL arm_cpu_enable_pmull = 0; +int ZLIB_INTERNAL x86_cpu_enable_sse2 = 0; +int ZLIB_INTERNAL x86_cpu_enable_ssse3 = 0; +int ZLIB_INTERNAL x86_cpu_enable_simd = 0; + +#ifndef CPU_NO_SIMD + +#if defined(ARMV8_OS_ANDROID) || defined(ARMV8_OS_LINUX) || defined(ARMV8_OS_FUCHSIA) +#include <pthread.h> +#endif + +#if defined(ARMV8_OS_ANDROID) +#include <cpu-features.h> +#elif defined(ARMV8_OS_LINUX) +#include <asm/hwcap.h> +#include <sys/auxv.h> +#elif defined(ARMV8_OS_FUCHSIA) +#include <zircon/features.h> +#include <zircon/syscalls.h> +#include <zircon/types.h> +#elif defined(ARMV8_OS_WINDOWS) || defined(X86_WINDOWS) +#include <windows.h> +#elif !defined(_MSC_VER) +#include <pthread.h> +#else +#error cpu_features.c CPU feature detection in not defined for your platform +#endif + +#if !defined(CPU_NO_SIMD) && !defined(ARMV8_OS_MACOS) && !defined(ARM_OS_IOS) +static void _cpu_check_features(void); +#endif + +#if defined(ARMV8_OS_ANDROID) || defined(ARMV8_OS_LINUX) || defined(ARMV8_OS_MACOS) || defined(ARMV8_OS_FUCHSIA) || defined(X86_NOT_WINDOWS) +#if !defined(ARMV8_OS_MACOS) +// _cpu_check_features() doesn't need to do anything on mac/arm since all +// features are known at build time, so don't call it. +// Do provide cpu_check_features() (with a no-op implementation) so that we +// don't have to make all callers of it check for mac/arm. +static pthread_once_t cpu_check_inited_once = PTHREAD_ONCE_INIT; +#endif +void ZLIB_INTERNAL cpu_check_features(void) +{ +#if !defined(ARMV8_OS_MACOS) + pthread_once(&cpu_check_inited_once, _cpu_check_features); +#endif +} +#elif defined(ARMV8_OS_WINDOWS) || defined(X86_WINDOWS) +static INIT_ONCE cpu_check_inited_once = INIT_ONCE_STATIC_INIT; +static BOOL CALLBACK _cpu_check_features_forwarder(PINIT_ONCE once, PVOID param, PVOID* context) +{ + _cpu_check_features(); + return TRUE; +} +void ZLIB_INTERNAL cpu_check_features(void) +{ + InitOnceExecuteOnce(&cpu_check_inited_once, _cpu_check_features_forwarder, + NULL, NULL); +} +#endif + +#if (defined(__ARM_NEON__) || defined(__ARM_NEON)) +/* + * iOS@ARM is a special case where we always have NEON but don't check + * for crypto extensions. + */ +#if !defined(ARMV8_OS_MACOS) && !defined(ARM_OS_IOS) +/* + * See http://bit.ly/2CcoEsr for run-time detection of ARM features and also + * crbug.com/931275 for android_getCpuFeatures() use in the Android sandbox. + */ +static void _cpu_check_features(void) +{ +#if defined(ARMV8_OS_ANDROID) && defined(__aarch64__) + uint64_t features = android_getCpuFeatures(); + arm_cpu_enable_crc32 = !!(features & ANDROID_CPU_ARM64_FEATURE_CRC32); + arm_cpu_enable_pmull = !!(features & ANDROID_CPU_ARM64_FEATURE_PMULL); +#elif defined(ARMV8_OS_ANDROID) /* aarch32 */ + uint64_t features = android_getCpuFeatures(); + arm_cpu_enable_crc32 = !!(features & ANDROID_CPU_ARM_FEATURE_CRC32); + arm_cpu_enable_pmull = !!(features & ANDROID_CPU_ARM_FEATURE_PMULL); +#elif defined(ARMV8_OS_LINUX) && defined(__aarch64__) + unsigned long features = getauxval(AT_HWCAP); + arm_cpu_enable_crc32 = !!(features & HWCAP_CRC32); + arm_cpu_enable_pmull = !!(features & HWCAP_PMULL); +#elif defined(ARMV8_OS_LINUX) && (defined(__ARM_NEON) || defined(__ARM_NEON__)) + /* Query HWCAP2 for ARMV8-A SoCs running in aarch32 mode */ + unsigned long features = getauxval(AT_HWCAP2); + arm_cpu_enable_crc32 = !!(features & HWCAP2_CRC32); + arm_cpu_enable_pmull = !!(features & HWCAP2_PMULL); +#elif defined(ARMV8_OS_FUCHSIA) + uint32_t features; + zx_status_t rc = zx_system_get_features(ZX_FEATURE_KIND_CPU, &features); + if (rc != ZX_OK || (features & ZX_ARM64_FEATURE_ISA_ASIMD) == 0) + return; /* Report nothing if ASIMD(NEON) is missing */ + arm_cpu_enable_crc32 = !!(features & ZX_ARM64_FEATURE_ISA_CRC32); + arm_cpu_enable_pmull = !!(features & ZX_ARM64_FEATURE_ISA_PMULL); +#elif defined(ARMV8_OS_WINDOWS) + arm_cpu_enable_crc32 = IsProcessorFeaturePresent(PF_ARM_V8_CRC32_INSTRUCTIONS_AVAILABLE); + arm_cpu_enable_pmull = IsProcessorFeaturePresent(PF_ARM_V8_CRYPTO_INSTRUCTIONS_AVAILABLE); +#endif +} +#endif +#elif defined(X86_NOT_WINDOWS) || defined(X86_WINDOWS) +/* + * iOS@x86 (i.e. emulator) is another special case where we disable + * SIMD optimizations. + */ +#ifndef CPU_NO_SIMD +/* On x86 we simply use a instruction to check the CPU features. + * (i.e. CPUID). + */ +static void _cpu_check_features(void) +{ + int x86_cpu_has_sse2; + int x86_cpu_has_ssse3; + int x86_cpu_has_sse42; + int x86_cpu_has_pclmulqdq; + int abcd[4]; + +#ifdef _MSC_VER + __cpuid(abcd, 1); +#else + __cpuid(1, abcd[0], abcd[1], abcd[2], abcd[3]); +#endif + + x86_cpu_has_sse2 = abcd[3] & 0x4000000; + x86_cpu_has_ssse3 = abcd[2] & 0x000200; + x86_cpu_has_sse42 = abcd[2] & 0x100000; + x86_cpu_has_pclmulqdq = abcd[2] & 0x2; + + x86_cpu_enable_sse2 = x86_cpu_has_sse2; + + x86_cpu_enable_ssse3 = x86_cpu_has_ssse3; + + x86_cpu_enable_simd = x86_cpu_has_sse2 && + x86_cpu_has_sse42 && + x86_cpu_has_pclmulqdq; +} +#endif +#endif +#endif diff --git a/third_party/zlib/cpu_features.h b/third_party/zlib/cpu_features.h new file mode 100644 index 00000000..c7b15c55 --- /dev/null +++ b/third_party/zlib/cpu_features.h @@ -0,0 +1,18 @@ +/* cpu_features.h -- Processor features detection. + * + * Copyright 2018 The Chromium Authors. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the Chromium source repository LICENSE file. + */ + +#include "zlib.h" + +/* TODO(cavalcantii): remove checks for x86_flags on deflate. + */ +extern int arm_cpu_enable_crc32; +extern int arm_cpu_enable_pmull; +extern int x86_cpu_enable_sse2; +extern int x86_cpu_enable_ssse3; +extern int x86_cpu_enable_simd; + +void cpu_check_features(void); diff --git a/third_party/zlib/crc32.c b/third_party/zlib/crc32.c new file mode 100644 index 00000000..d4c3248d --- /dev/null +++ b/third_party/zlib/crc32.c @@ -0,0 +1,527 @@ +/* crc32.c -- compute the CRC-32 of a data stream + * Copyright (C) 1995-2006, 2010, 2011, 2012, 2016 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + * + * Thanks to Rodney Brown <rbrown64@csc.com.au> for his contribution of faster + * CRC methods: exclusive-oring 32 bits of data at a time, and pre-computing + * tables for updating the shift register in one step with three exclusive-ors + * instead of four steps with four exclusive-ors. This results in about a + * factor of two increase in speed on a Power PC G4 (PPC7455) using gcc -O3. + */ + +/* @(#) $Id$ */ + +/* + Note on the use of DYNAMIC_CRC_TABLE: there is no mutex or semaphore + protection on the static variables used to control the first-use generation + of the crc tables. Therefore, if you #define DYNAMIC_CRC_TABLE, you should + first call get_crc_table() to initialize the tables before allowing more than + one thread to use crc32(). + + DYNAMIC_CRC_TABLE and MAKECRCH can be #defined to write out crc32.h. + */ + +#ifdef MAKECRCH +# include <stdio.h> +# ifndef DYNAMIC_CRC_TABLE +# define DYNAMIC_CRC_TABLE +# endif /* !DYNAMIC_CRC_TABLE */ +#endif /* MAKECRCH */ + +#include "deflate.h" +#include "cpu_features.h" +#include "zutil.h" /* for STDC and FAR definitions */ + +#if defined(CRC32_SIMD_SSE42_PCLMUL) || defined(CRC32_ARMV8_CRC32) +#include "crc32_simd.h" +#endif + +/* Definitions for doing the crc four data bytes at a time. */ +#if !defined(NOBYFOUR) && defined(Z_U4) +# define BYFOUR +#endif +#ifdef BYFOUR + local unsigned long crc32_little OF((unsigned long, + const unsigned char FAR *, z_size_t)); + local unsigned long crc32_big OF((unsigned long, + const unsigned char FAR *, z_size_t)); +# define TBLS 8 +#else +# define TBLS 1 +#endif /* BYFOUR */ + +/* Local functions for crc concatenation */ +local unsigned long gf2_matrix_times OF((unsigned long *mat, + unsigned long vec)); +local void gf2_matrix_square OF((unsigned long *square, unsigned long *mat)); +local uLong crc32_combine_ OF((uLong crc1, uLong crc2, z_off64_t len2)); + + +#ifdef DYNAMIC_CRC_TABLE + +local volatile int crc_table_empty = 1; +local z_crc_t FAR crc_table[TBLS][256]; +local void make_crc_table OF((void)); +#ifdef MAKECRCH + local void write_table OF((FILE *, const z_crc_t FAR *)); +#endif /* MAKECRCH */ +/* + Generate tables for a byte-wise 32-bit CRC calculation on the polynomial: + x^32+x^26+x^23+x^22+x^16+x^12+x^11+x^10+x^8+x^7+x^5+x^4+x^2+x+1. + + Polynomials over GF(2) are represented in binary, one bit per coefficient, + with the lowest powers in the most significant bit. Then adding polynomials + is just exclusive-or, and multiplying a polynomial by x is a right shift by + one. If we call the above polynomial p, and represent a byte as the + polynomial q, also with the lowest power in the most significant bit (so the + byte 0xb1 is the polynomial x^7+x^3+x+1), then the CRC is (q*x^32) mod p, + where a mod b means the remainder after dividing a by b. + + This calculation is done using the shift-register method of multiplying and + taking the remainder. The register is initialized to zero, and for each + incoming bit, x^32 is added mod p to the register if the bit is a one (where + x^32 mod p is p+x^32 = x^26+...+1), and the register is multiplied mod p by + x (which is shifting right by one and adding x^32 mod p if the bit shifted + out is a one). We start with the highest power (least significant bit) of + q and repeat for all eight bits of q. + + The first table is simply the CRC of all possible eight bit values. This is + all the information needed to generate CRCs on data a byte at a time for all + combinations of CRC register values and incoming bytes. The remaining tables + allow for word-at-a-time CRC calculation for both big-endian and little- + endian machines, where a word is four bytes. +*/ +local void make_crc_table() +{ + z_crc_t c; + int n, k; + z_crc_t poly; /* polynomial exclusive-or pattern */ + /* terms of polynomial defining this crc (except x^32): */ + static volatile int first = 1; /* flag to limit concurrent making */ + static const unsigned char p[] = {0,1,2,4,5,7,8,10,11,12,16,22,23,26}; + + /* See if another task is already doing this (not thread-safe, but better + than nothing -- significantly reduces duration of vulnerability in + case the advice about DYNAMIC_CRC_TABLE is ignored) */ + if (first) { + first = 0; + + /* make exclusive-or pattern from polynomial (0xedb88320UL) */ + poly = 0; + for (n = 0; n < (int)(sizeof(p)/sizeof(unsigned char)); n++) + poly |= (z_crc_t)1 << (31 - p[n]); + + /* generate a crc for every 8-bit value */ + for (n = 0; n < 256; n++) { + c = (z_crc_t)n; + for (k = 0; k < 8; k++) + c = c & 1 ? poly ^ (c >> 1) : c >> 1; + crc_table[0][n] = c; + } + +#ifdef BYFOUR + /* generate crc for each value followed by one, two, and three zeros, + and then the byte reversal of those as well as the first table */ + for (n = 0; n < 256; n++) { + c = crc_table[0][n]; + crc_table[4][n] = ZSWAP32(c); + for (k = 1; k < 4; k++) { + c = crc_table[0][c & 0xff] ^ (c >> 8); + crc_table[k][n] = c; + crc_table[k + 4][n] = ZSWAP32(c); + } + } +#endif /* BYFOUR */ + + crc_table_empty = 0; + } + else { /* not first */ + /* wait for the other guy to finish (not efficient, but rare) */ + while (crc_table_empty) + ; + } + +#ifdef MAKECRCH + /* write out CRC tables to crc32.h */ + { + FILE *out; + + out = fopen("crc32.h", "w"); + if (out == NULL) return; + fprintf(out, "/* crc32.h -- tables for rapid CRC calculation\n"); + fprintf(out, " * Generated automatically by crc32.c\n */\n\n"); + fprintf(out, "local const z_crc_t FAR "); + fprintf(out, "crc_table[TBLS][256] =\n{\n {\n"); + write_table(out, crc_table[0]); +# ifdef BYFOUR + fprintf(out, "#ifdef BYFOUR\n"); + for (k = 1; k < 8; k++) { + fprintf(out, " },\n {\n"); + write_table(out, crc_table[k]); + } + fprintf(out, "#endif\n"); +# endif /* BYFOUR */ + fprintf(out, " }\n};\n"); + fclose(out); + } +#endif /* MAKECRCH */ +} + +#ifdef MAKECRCH +local void write_table(out, table) + FILE *out; + const z_crc_t FAR *table; +{ + int n; + + for (n = 0; n < 256; n++) + fprintf(out, "%s0x%08lxUL%s", n % 5 ? "" : " ", + (unsigned long)(table[n]), + n == 255 ? "\n" : (n % 5 == 4 ? ",\n" : ", ")); +} +#endif /* MAKECRCH */ + +#else /* !DYNAMIC_CRC_TABLE */ +/* ======================================================================== + * Tables of CRC-32s of all single-byte values, made by make_crc_table(). + */ +#include "crc32.h" +#endif /* DYNAMIC_CRC_TABLE */ + +/* ========================================================================= + * This function can be used by asm versions of crc32() + */ +const z_crc_t FAR * ZEXPORT get_crc_table() +{ +#ifdef DYNAMIC_CRC_TABLE + if (crc_table_empty) + make_crc_table(); +#endif /* DYNAMIC_CRC_TABLE */ + return (const z_crc_t FAR *)crc_table; +} + +/* ========================================================================= */ +#define DO1 crc = crc_table[0][((int)crc ^ (*buf++)) & 0xff] ^ (crc >> 8) +#define DO8 DO1; DO1; DO1; DO1; DO1; DO1; DO1; DO1 + +/* ========================================================================= */ +unsigned long ZEXPORT crc32_z(crc, buf, len) + unsigned long crc; + const unsigned char FAR *buf; + z_size_t len; +{ + /* + * zlib convention is to call crc32(0, NULL, 0); before making + * calls to crc32(). So this is a good, early (and infrequent) + * place to cache CPU features if needed for those later, more + * interesting crc32() calls. + */ +#if defined(CRC32_SIMD_SSE42_PCLMUL) + /* + * Use x86 sse4.2+pclmul SIMD to compute the crc32. Since this + * routine can be freely used, check CPU features here. + */ + if (buf == Z_NULL) { + if (!len) /* Assume user is calling crc32(0, NULL, 0); */ + cpu_check_features(); + return 0UL; + } + + if (x86_cpu_enable_simd && len >= Z_CRC32_SSE42_MINIMUM_LENGTH) { + /* crc32 16-byte chunks */ + z_size_t chunk_size = len & ~Z_CRC32_SSE42_CHUNKSIZE_MASK; + crc = ~crc32_sse42_simd_(buf, chunk_size, ~(uint32_t)crc); + /* check remaining data */ + len -= chunk_size; + if (!len) + return crc; + /* Fall into the default crc32 for the remaining data. */ + buf += chunk_size; + } +#else + if (buf == Z_NULL) { + return 0UL; + } +#endif /* CRC32_SIMD_SSE42_PCLMUL */ + +#ifdef DYNAMIC_CRC_TABLE + if (crc_table_empty) + make_crc_table(); +#endif /* DYNAMIC_CRC_TABLE */ + +#ifdef BYFOUR + if (sizeof(void *) == sizeof(ptrdiff_t)) { + z_crc_t endian; + + endian = 1; + if (*((unsigned char *)(&endian))) + return crc32_little(crc, buf, len); + else + return crc32_big(crc, buf, len); + } +#endif /* BYFOUR */ + crc = crc ^ 0xffffffffUL; + while (len >= 8) { + DO8; + len -= 8; + } + if (len) do { + DO1; + } while (--len); + return crc ^ 0xffffffffUL; +} + +/* ========================================================================= */ +unsigned long ZEXPORT crc32(crc, buf, len) + unsigned long crc; + const unsigned char FAR *buf; + uInt len; +{ +#if defined(CRC32_ARMV8_CRC32) + /* We got to verify ARM CPU features, so exploit the common usage pattern + * of calling this function with Z_NULL for an initial valid crc value. + * This allows to cache the result of the feature check and avoid extraneous + * function calls. + * TODO: try to move this to crc32_z if we don't loose performance on ARM. + */ + if (buf == Z_NULL) { + if (!len) /* Assume user is calling crc32(0, NULL, 0); */ + cpu_check_features(); + return 0UL; + } + + if (arm_cpu_enable_crc32) + return armv8_crc32_little(crc, buf, len); +#endif + return crc32_z(crc, buf, len); +} + +#ifdef BYFOUR + +/* + This BYFOUR code accesses the passed unsigned char * buffer with a 32-bit + integer pointer type. This violates the strict aliasing rule, where a + compiler can assume, for optimization purposes, that two pointers to + fundamentally different types won't ever point to the same memory. This can + manifest as a problem only if one of the pointers is written to. This code + only reads from those pointers. So long as this code remains isolated in + this compilation unit, there won't be a problem. For this reason, this code + should not be copied and pasted into a compilation unit in which other code + writes to the buffer that is passed to these routines. + */ + +/* ========================================================================= */ +#define DOLIT4 c ^= *buf4++; \ + c = crc_table[3][c & 0xff] ^ crc_table[2][(c >> 8) & 0xff] ^ \ + crc_table[1][(c >> 16) & 0xff] ^ crc_table[0][c >> 24] +#define DOLIT32 DOLIT4; DOLIT4; DOLIT4; DOLIT4; DOLIT4; DOLIT4; DOLIT4; DOLIT4 + +/* ========================================================================= */ +local unsigned long crc32_little(crc, buf, len) + unsigned long crc; + const unsigned char FAR *buf; + z_size_t len; +{ + register z_crc_t c; + register const z_crc_t FAR *buf4; + + c = (z_crc_t)crc; + c = ~c; + while (len && ((ptrdiff_t)buf & 3)) { + c = crc_table[0][(c ^ *buf++) & 0xff] ^ (c >> 8); + len--; + } + + buf4 = (const z_crc_t FAR *)(const void FAR *)buf; + while (len >= 32) { + DOLIT32; + len -= 32; + } + while (len >= 4) { + DOLIT4; + len -= 4; + } + buf = (const unsigned char FAR *)buf4; + + if (len) do { + c = crc_table[0][(c ^ *buf++) & 0xff] ^ (c >> 8); + } while (--len); + c = ~c; + return (unsigned long)c; +} + +/* ========================================================================= */ +#define DOBIG4 c ^= *buf4++; \ + c = crc_table[4][c & 0xff] ^ crc_table[5][(c >> 8) & 0xff] ^ \ + crc_table[6][(c >> 16) & 0xff] ^ crc_table[7][c >> 24] +#define DOBIG32 DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4; DOBIG4 + +/* ========================================================================= */ +local unsigned long crc32_big(crc, buf, len) + unsigned long crc; + const unsigned char FAR *buf; + z_size_t len; +{ + register z_crc_t c; + register const z_crc_t FAR *buf4; + + c = ZSWAP32((z_crc_t)crc); + c = ~c; + while (len && ((ptrdiff_t)buf & 3)) { + c = crc_table[4][(c >> 24) ^ *buf++] ^ (c << 8); + len--; + } + + buf4 = (const z_crc_t FAR *)(const void FAR *)buf; + while (len >= 32) { + DOBIG32; + len -= 32; + } + while (len >= 4) { + DOBIG4; + len -= 4; + } + buf = (const unsigned char FAR *)buf4; + + if (len) do { + c = crc_table[4][(c >> 24) ^ *buf++] ^ (c << 8); + } while (--len); + c = ~c; + return (unsigned long)(ZSWAP32(c)); +} + +#endif /* BYFOUR */ + +#define GF2_DIM 32 /* dimension of GF(2) vectors (length of CRC) */ + +/* ========================================================================= */ +local unsigned long gf2_matrix_times(mat, vec) + unsigned long *mat; + unsigned long vec; +{ + unsigned long sum; + + sum = 0; + while (vec) { + if (vec & 1) + sum ^= *mat; + vec >>= 1; + mat++; + } + return sum; +} + +/* ========================================================================= */ +local void gf2_matrix_square(square, mat) + unsigned long *square; + unsigned long *mat; +{ + int n; + + for (n = 0; n < GF2_DIM; n++) + square[n] = gf2_matrix_times(mat, mat[n]); +} + +/* ========================================================================= */ +local uLong crc32_combine_(crc1, crc2, len2) + uLong crc1; + uLong crc2; + z_off64_t len2; +{ + int n; + unsigned long row; + unsigned long even[GF2_DIM]; /* even-power-of-two zeros operator */ + unsigned long odd[GF2_DIM]; /* odd-power-of-two zeros operator */ + + /* degenerate case (also disallow negative lengths) */ + if (len2 <= 0) + return crc1; + + /* put operator for one zero bit in odd */ + odd[0] = 0xedb88320UL; /* CRC-32 polynomial */ + row = 1; + for (n = 1; n < GF2_DIM; n++) { + odd[n] = row; + row <<= 1; + } + + /* put operator for two zero bits in even */ + gf2_matrix_square(even, odd); + + /* put operator for four zero bits in odd */ + gf2_matrix_square(odd, even); + + /* apply len2 zeros to crc1 (first square will put the operator for one + zero byte, eight zero bits, in even) */ + do { + /* apply zeros operator for this bit of len2 */ + gf2_matrix_square(even, odd); + if (len2 & 1) + crc1 = gf2_matrix_times(even, crc1); + len2 >>= 1; + + /* if no more bits set, then done */ + if (len2 == 0) + break; + + /* another iteration of the loop with odd and even swapped */ + gf2_matrix_square(odd, even); + if (len2 & 1) + crc1 = gf2_matrix_times(odd, crc1); + len2 >>= 1; + + /* if no more bits set, then done */ + } while (len2 != 0); + + /* return combined crc */ + crc1 ^= crc2; + return crc1; +} + +/* ========================================================================= */ +uLong ZEXPORT crc32_combine(crc1, crc2, len2) + uLong crc1; + uLong crc2; + z_off_t len2; +{ + return crc32_combine_(crc1, crc2, len2); +} + +uLong ZEXPORT crc32_combine64(crc1, crc2, len2) + uLong crc1; + uLong crc2; + z_off64_t len2; +{ + return crc32_combine_(crc1, crc2, len2); +} + +ZLIB_INTERNAL void crc_reset(deflate_state *const s) +{ +#ifdef CRC32_SIMD_SSE42_PCLMUL + if (x86_cpu_enable_simd) { + crc_fold_init(s); + return; + } +#endif + s->strm->adler = crc32(0L, Z_NULL, 0); +} + +ZLIB_INTERNAL void crc_finalize(deflate_state *const s) +{ +#ifdef CRC32_SIMD_SSE42_PCLMUL + if (x86_cpu_enable_simd) + s->strm->adler = crc_fold_512to32(s); +#endif +} + +ZLIB_INTERNAL void copy_with_crc(z_streamp strm, Bytef *dst, long size) +{ +#ifdef CRC32_SIMD_SSE42_PCLMUL + if (x86_cpu_enable_simd) { + crc_fold_copy(strm->state, dst, strm->next_in, size); + return; + } +#endif + zmemcpy(dst, strm->next_in, size); + strm->adler = crc32(strm->adler, dst, size); +} diff --git a/third_party/zlib/crc32.h b/third_party/zlib/crc32.h new file mode 100644 index 00000000..9e0c7781 --- /dev/null +++ b/third_party/zlib/crc32.h @@ -0,0 +1,441 @@ +/* crc32.h -- tables for rapid CRC calculation + * Generated automatically by crc32.c + */ + +local const z_crc_t FAR crc_table[TBLS][256] = +{ + { + 0x00000000UL, 0x77073096UL, 0xee0e612cUL, 0x990951baUL, 0x076dc419UL, + 0x706af48fUL, 0xe963a535UL, 0x9e6495a3UL, 0x0edb8832UL, 0x79dcb8a4UL, + 0xe0d5e91eUL, 0x97d2d988UL, 0x09b64c2bUL, 0x7eb17cbdUL, 0xe7b82d07UL, + 0x90bf1d91UL, 0x1db71064UL, 0x6ab020f2UL, 0xf3b97148UL, 0x84be41deUL, + 0x1adad47dUL, 0x6ddde4ebUL, 0xf4d4b551UL, 0x83d385c7UL, 0x136c9856UL, + 0x646ba8c0UL, 0xfd62f97aUL, 0x8a65c9ecUL, 0x14015c4fUL, 0x63066cd9UL, + 0xfa0f3d63UL, 0x8d080df5UL, 0x3b6e20c8UL, 0x4c69105eUL, 0xd56041e4UL, + 0xa2677172UL, 0x3c03e4d1UL, 0x4b04d447UL, 0xd20d85fdUL, 0xa50ab56bUL, + 0x35b5a8faUL, 0x42b2986cUL, 0xdbbbc9d6UL, 0xacbcf940UL, 0x32d86ce3UL, + 0x45df5c75UL, 0xdcd60dcfUL, 0xabd13d59UL, 0x26d930acUL, 0x51de003aUL, + 0xc8d75180UL, 0xbfd06116UL, 0x21b4f4b5UL, 0x56b3c423UL, 0xcfba9599UL, + 0xb8bda50fUL, 0x2802b89eUL, 0x5f058808UL, 0xc60cd9b2UL, 0xb10be924UL, + 0x2f6f7c87UL, 0x58684c11UL, 0xc1611dabUL, 0xb6662d3dUL, 0x76dc4190UL, + 0x01db7106UL, 0x98d220bcUL, 0xefd5102aUL, 0x71b18589UL, 0x06b6b51fUL, + 0x9fbfe4a5UL, 0xe8b8d433UL, 0x7807c9a2UL, 0x0f00f934UL, 0x9609a88eUL, + 0xe10e9818UL, 0x7f6a0dbbUL, 0x086d3d2dUL, 0x91646c97UL, 0xe6635c01UL, + 0x6b6b51f4UL, 0x1c6c6162UL, 0x856530d8UL, 0xf262004eUL, 0x6c0695edUL, + 0x1b01a57bUL, 0x8208f4c1UL, 0xf50fc457UL, 0x65b0d9c6UL, 0x12b7e950UL, + 0x8bbeb8eaUL, 0xfcb9887cUL, 0x62dd1ddfUL, 0x15da2d49UL, 0x8cd37cf3UL, + 0xfbd44c65UL, 0x4db26158UL, 0x3ab551ceUL, 0xa3bc0074UL, 0xd4bb30e2UL, + 0x4adfa541UL, 0x3dd895d7UL, 0xa4d1c46dUL, 0xd3d6f4fbUL, 0x4369e96aUL, + 0x346ed9fcUL, 0xad678846UL, 0xda60b8d0UL, 0x44042d73UL, 0x33031de5UL, + 0xaa0a4c5fUL, 0xdd0d7cc9UL, 0x5005713cUL, 0x270241aaUL, 0xbe0b1010UL, + 0xc90c2086UL, 0x5768b525UL, 0x206f85b3UL, 0xb966d409UL, 0xce61e49fUL, + 0x5edef90eUL, 0x29d9c998UL, 0xb0d09822UL, 0xc7d7a8b4UL, 0x59b33d17UL, + 0x2eb40d81UL, 0xb7bd5c3bUL, 0xc0ba6cadUL, 0xedb88320UL, 0x9abfb3b6UL, + 0x03b6e20cUL, 0x74b1d29aUL, 0xead54739UL, 0x9dd277afUL, 0x04db2615UL, + 0x73dc1683UL, 0xe3630b12UL, 0x94643b84UL, 0x0d6d6a3eUL, 0x7a6a5aa8UL, + 0xe40ecf0bUL, 0x9309ff9dUL, 0x0a00ae27UL, 0x7d079eb1UL, 0xf00f9344UL, + 0x8708a3d2UL, 0x1e01f268UL, 0x6906c2feUL, 0xf762575dUL, 0x806567cbUL, + 0x196c3671UL, 0x6e6b06e7UL, 0xfed41b76UL, 0x89d32be0UL, 0x10da7a5aUL, + 0x67dd4accUL, 0xf9b9df6fUL, 0x8ebeeff9UL, 0x17b7be43UL, 0x60b08ed5UL, + 0xd6d6a3e8UL, 0xa1d1937eUL, 0x38d8c2c4UL, 0x4fdff252UL, 0xd1bb67f1UL, + 0xa6bc5767UL, 0x3fb506ddUL, 0x48b2364bUL, 0xd80d2bdaUL, 0xaf0a1b4cUL, + 0x36034af6UL, 0x41047a60UL, 0xdf60efc3UL, 0xa867df55UL, 0x316e8eefUL, + 0x4669be79UL, 0xcb61b38cUL, 0xbc66831aUL, 0x256fd2a0UL, 0x5268e236UL, + 0xcc0c7795UL, 0xbb0b4703UL, 0x220216b9UL, 0x5505262fUL, 0xc5ba3bbeUL, + 0xb2bd0b28UL, 0x2bb45a92UL, 0x5cb36a04UL, 0xc2d7ffa7UL, 0xb5d0cf31UL, + 0x2cd99e8bUL, 0x5bdeae1dUL, 0x9b64c2b0UL, 0xec63f226UL, 0x756aa39cUL, + 0x026d930aUL, 0x9c0906a9UL, 0xeb0e363fUL, 0x72076785UL, 0x05005713UL, + 0x95bf4a82UL, 0xe2b87a14UL, 0x7bb12baeUL, 0x0cb61b38UL, 0x92d28e9bUL, + 0xe5d5be0dUL, 0x7cdcefb7UL, 0x0bdbdf21UL, 0x86d3d2d4UL, 0xf1d4e242UL, + 0x68ddb3f8UL, 0x1fda836eUL, 0x81be16cdUL, 0xf6b9265bUL, 0x6fb077e1UL, + 0x18b74777UL, 0x88085ae6UL, 0xff0f6a70UL, 0x66063bcaUL, 0x11010b5cUL, + 0x8f659effUL, 0xf862ae69UL, 0x616bffd3UL, 0x166ccf45UL, 0xa00ae278UL, + 0xd70dd2eeUL, 0x4e048354UL, 0x3903b3c2UL, 0xa7672661UL, 0xd06016f7UL, + 0x4969474dUL, 0x3e6e77dbUL, 0xaed16a4aUL, 0xd9d65adcUL, 0x40df0b66UL, + 0x37d83bf0UL, 0xa9bcae53UL, 0xdebb9ec5UL, 0x47b2cf7fUL, 0x30b5ffe9UL, + 0xbdbdf21cUL, 0xcabac28aUL, 0x53b39330UL, 0x24b4a3a6UL, 0xbad03605UL, + 0xcdd70693UL, 0x54de5729UL, 0x23d967bfUL, 0xb3667a2eUL, 0xc4614ab8UL, + 0x5d681b02UL, 0x2a6f2b94UL, 0xb40bbe37UL, 0xc30c8ea1UL, 0x5a05df1bUL, + 0x2d02ef8dUL +#ifdef BYFOUR + }, + { + 0x00000000UL, 0x191b3141UL, 0x32366282UL, 0x2b2d53c3UL, 0x646cc504UL, + 0x7d77f445UL, 0x565aa786UL, 0x4f4196c7UL, 0xc8d98a08UL, 0xd1c2bb49UL, + 0xfaefe88aUL, 0xe3f4d9cbUL, 0xacb54f0cUL, 0xb5ae7e4dUL, 0x9e832d8eUL, + 0x87981ccfUL, 0x4ac21251UL, 0x53d92310UL, 0x78f470d3UL, 0x61ef4192UL, + 0x2eaed755UL, 0x37b5e614UL, 0x1c98b5d7UL, 0x05838496UL, 0x821b9859UL, + 0x9b00a918UL, 0xb02dfadbUL, 0xa936cb9aUL, 0xe6775d5dUL, 0xff6c6c1cUL, + 0xd4413fdfUL, 0xcd5a0e9eUL, 0x958424a2UL, 0x8c9f15e3UL, 0xa7b24620UL, + 0xbea97761UL, 0xf1e8e1a6UL, 0xe8f3d0e7UL, 0xc3de8324UL, 0xdac5b265UL, + 0x5d5daeaaUL, 0x44469febUL, 0x6f6bcc28UL, 0x7670fd69UL, 0x39316baeUL, + 0x202a5aefUL, 0x0b07092cUL, 0x121c386dUL, 0xdf4636f3UL, 0xc65d07b2UL, + 0xed705471UL, 0xf46b6530UL, 0xbb2af3f7UL, 0xa231c2b6UL, 0x891c9175UL, + 0x9007a034UL, 0x179fbcfbUL, 0x0e848dbaUL, 0x25a9de79UL, 0x3cb2ef38UL, + 0x73f379ffUL, 0x6ae848beUL, 0x41c51b7dUL, 0x58de2a3cUL, 0xf0794f05UL, + 0xe9627e44UL, 0xc24f2d87UL, 0xdb541cc6UL, 0x94158a01UL, 0x8d0ebb40UL, + 0xa623e883UL, 0xbf38d9c2UL, 0x38a0c50dUL, 0x21bbf44cUL, 0x0a96a78fUL, + 0x138d96ceUL, 0x5ccc0009UL, 0x45d73148UL, 0x6efa628bUL, 0x77e153caUL, + 0xbabb5d54UL, 0xa3a06c15UL, 0x888d3fd6UL, 0x91960e97UL, 0xded79850UL, + 0xc7cca911UL, 0xece1fad2UL, 0xf5facb93UL, 0x7262d75cUL, 0x6b79e61dUL, + 0x4054b5deUL, 0x594f849fUL, 0x160e1258UL, 0x0f152319UL, 0x243870daUL, + 0x3d23419bUL, 0x65fd6ba7UL, 0x7ce65ae6UL, 0x57cb0925UL, 0x4ed03864UL, + 0x0191aea3UL, 0x188a9fe2UL, 0x33a7cc21UL, 0x2abcfd60UL, 0xad24e1afUL, + 0xb43fd0eeUL, 0x9f12832dUL, 0x8609b26cUL, 0xc94824abUL, 0xd05315eaUL, + 0xfb7e4629UL, 0xe2657768UL, 0x2f3f79f6UL, 0x362448b7UL, 0x1d091b74UL, + 0x04122a35UL, 0x4b53bcf2UL, 0x52488db3UL, 0x7965de70UL, 0x607eef31UL, + 0xe7e6f3feUL, 0xfefdc2bfUL, 0xd5d0917cUL, 0xcccba03dUL, 0x838a36faUL, + 0x9a9107bbUL, 0xb1bc5478UL, 0xa8a76539UL, 0x3b83984bUL, 0x2298a90aUL, + 0x09b5fac9UL, 0x10aecb88UL, 0x5fef5d4fUL, 0x46f46c0eUL, 0x6dd93fcdUL, + 0x74c20e8cUL, 0xf35a1243UL, 0xea412302UL, 0xc16c70c1UL, 0xd8774180UL, + 0x9736d747UL, 0x8e2de606UL, 0xa500b5c5UL, 0xbc1b8484UL, 0x71418a1aUL, + 0x685abb5bUL, 0x4377e898UL, 0x5a6cd9d9UL, 0x152d4f1eUL, 0x0c367e5fUL, + 0x271b2d9cUL, 0x3e001cddUL, 0xb9980012UL, 0xa0833153UL, 0x8bae6290UL, + 0x92b553d1UL, 0xddf4c516UL, 0xc4eff457UL, 0xefc2a794UL, 0xf6d996d5UL, + 0xae07bce9UL, 0xb71c8da8UL, 0x9c31de6bUL, 0x852aef2aUL, 0xca6b79edUL, + 0xd37048acUL, 0xf85d1b6fUL, 0xe1462a2eUL, 0x66de36e1UL, 0x7fc507a0UL, + 0x54e85463UL, 0x4df36522UL, 0x02b2f3e5UL, 0x1ba9c2a4UL, 0x30849167UL, + 0x299fa026UL, 0xe4c5aeb8UL, 0xfdde9ff9UL, 0xd6f3cc3aUL, 0xcfe8fd7bUL, + 0x80a96bbcUL, 0x99b25afdUL, 0xb29f093eUL, 0xab84387fUL, 0x2c1c24b0UL, + 0x350715f1UL, 0x1e2a4632UL, 0x07317773UL, 0x4870e1b4UL, 0x516bd0f5UL, + 0x7a468336UL, 0x635db277UL, 0xcbfad74eUL, 0xd2e1e60fUL, 0xf9ccb5ccUL, + 0xe0d7848dUL, 0xaf96124aUL, 0xb68d230bUL, 0x9da070c8UL, 0x84bb4189UL, + 0x03235d46UL, 0x1a386c07UL, 0x31153fc4UL, 0x280e0e85UL, 0x674f9842UL, + 0x7e54a903UL, 0x5579fac0UL, 0x4c62cb81UL, 0x8138c51fUL, 0x9823f45eUL, + 0xb30ea79dUL, 0xaa1596dcUL, 0xe554001bUL, 0xfc4f315aUL, 0xd7626299UL, + 0xce7953d8UL, 0x49e14f17UL, 0x50fa7e56UL, 0x7bd72d95UL, 0x62cc1cd4UL, + 0x2d8d8a13UL, 0x3496bb52UL, 0x1fbbe891UL, 0x06a0d9d0UL, 0x5e7ef3ecUL, + 0x4765c2adUL, 0x6c48916eUL, 0x7553a02fUL, 0x3a1236e8UL, 0x230907a9UL, + 0x0824546aUL, 0x113f652bUL, 0x96a779e4UL, 0x8fbc48a5UL, 0xa4911b66UL, + 0xbd8a2a27UL, 0xf2cbbce0UL, 0xebd08da1UL, 0xc0fdde62UL, 0xd9e6ef23UL, + 0x14bce1bdUL, 0x0da7d0fcUL, 0x268a833fUL, 0x3f91b27eUL, 0x70d024b9UL, + 0x69cb15f8UL, 0x42e6463bUL, 0x5bfd777aUL, 0xdc656bb5UL, 0xc57e5af4UL, + 0xee530937UL, 0xf7483876UL, 0xb809aeb1UL, 0xa1129ff0UL, 0x8a3fcc33UL, + 0x9324fd72UL + }, + { + 0x00000000UL, 0x01c26a37UL, 0x0384d46eUL, 0x0246be59UL, 0x0709a8dcUL, + 0x06cbc2ebUL, 0x048d7cb2UL, 0x054f1685UL, 0x0e1351b8UL, 0x0fd13b8fUL, + 0x0d9785d6UL, 0x0c55efe1UL, 0x091af964UL, 0x08d89353UL, 0x0a9e2d0aUL, + 0x0b5c473dUL, 0x1c26a370UL, 0x1de4c947UL, 0x1fa2771eUL, 0x1e601d29UL, + 0x1b2f0bacUL, 0x1aed619bUL, 0x18abdfc2UL, 0x1969b5f5UL, 0x1235f2c8UL, + 0x13f798ffUL, 0x11b126a6UL, 0x10734c91UL, 0x153c5a14UL, 0x14fe3023UL, + 0x16b88e7aUL, 0x177ae44dUL, 0x384d46e0UL, 0x398f2cd7UL, 0x3bc9928eUL, + 0x3a0bf8b9UL, 0x3f44ee3cUL, 0x3e86840bUL, 0x3cc03a52UL, 0x3d025065UL, + 0x365e1758UL, 0x379c7d6fUL, 0x35dac336UL, 0x3418a901UL, 0x3157bf84UL, + 0x3095d5b3UL, 0x32d36beaUL, 0x331101ddUL, 0x246be590UL, 0x25a98fa7UL, + 0x27ef31feUL, 0x262d5bc9UL, 0x23624d4cUL, 0x22a0277bUL, 0x20e69922UL, + 0x2124f315UL, 0x2a78b428UL, 0x2bbade1fUL, 0x29fc6046UL, 0x283e0a71UL, + 0x2d711cf4UL, 0x2cb376c3UL, 0x2ef5c89aUL, 0x2f37a2adUL, 0x709a8dc0UL, + 0x7158e7f7UL, 0x731e59aeUL, 0x72dc3399UL, 0x7793251cUL, 0x76514f2bUL, + 0x7417f172UL, 0x75d59b45UL, 0x7e89dc78UL, 0x7f4bb64fUL, 0x7d0d0816UL, + 0x7ccf6221UL, 0x798074a4UL, 0x78421e93UL, 0x7a04a0caUL, 0x7bc6cafdUL, + 0x6cbc2eb0UL, 0x6d7e4487UL, 0x6f38fadeUL, 0x6efa90e9UL, 0x6bb5866cUL, + 0x6a77ec5bUL, 0x68315202UL, 0x69f33835UL, 0x62af7f08UL, 0x636d153fUL, + 0x612bab66UL, 0x60e9c151UL, 0x65a6d7d4UL, 0x6464bde3UL, 0x662203baUL, + 0x67e0698dUL, 0x48d7cb20UL, 0x4915a117UL, 0x4b531f4eUL, 0x4a917579UL, + 0x4fde63fcUL, 0x4e1c09cbUL, 0x4c5ab792UL, 0x4d98dda5UL, 0x46c49a98UL, + 0x4706f0afUL, 0x45404ef6UL, 0x448224c1UL, 0x41cd3244UL, 0x400f5873UL, + 0x4249e62aUL, 0x438b8c1dUL, 0x54f16850UL, 0x55330267UL, 0x5775bc3eUL, + 0x56b7d609UL, 0x53f8c08cUL, 0x523aaabbUL, 0x507c14e2UL, 0x51be7ed5UL, + 0x5ae239e8UL, 0x5b2053dfUL, 0x5966ed86UL, 0x58a487b1UL, 0x5deb9134UL, + 0x5c29fb03UL, 0x5e6f455aUL, 0x5fad2f6dUL, 0xe1351b80UL, 0xe0f771b7UL, + 0xe2b1cfeeUL, 0xe373a5d9UL, 0xe63cb35cUL, 0xe7fed96bUL, 0xe5b86732UL, + 0xe47a0d05UL, 0xef264a38UL, 0xeee4200fUL, 0xeca29e56UL, 0xed60f461UL, + 0xe82fe2e4UL, 0xe9ed88d3UL, 0xebab368aUL, 0xea695cbdUL, 0xfd13b8f0UL, + 0xfcd1d2c7UL, 0xfe976c9eUL, 0xff5506a9UL, 0xfa1a102cUL, 0xfbd87a1bUL, + 0xf99ec442UL, 0xf85cae75UL, 0xf300e948UL, 0xf2c2837fUL, 0xf0843d26UL, + 0xf1465711UL, 0xf4094194UL, 0xf5cb2ba3UL, 0xf78d95faUL, 0xf64fffcdUL, + 0xd9785d60UL, 0xd8ba3757UL, 0xdafc890eUL, 0xdb3ee339UL, 0xde71f5bcUL, + 0xdfb39f8bUL, 0xddf521d2UL, 0xdc374be5UL, 0xd76b0cd8UL, 0xd6a966efUL, + 0xd4efd8b6UL, 0xd52db281UL, 0xd062a404UL, 0xd1a0ce33UL, 0xd3e6706aUL, + 0xd2241a5dUL, 0xc55efe10UL, 0xc49c9427UL, 0xc6da2a7eUL, 0xc7184049UL, + 0xc25756ccUL, 0xc3953cfbUL, 0xc1d382a2UL, 0xc011e895UL, 0xcb4dafa8UL, + 0xca8fc59fUL, 0xc8c97bc6UL, 0xc90b11f1UL, 0xcc440774UL, 0xcd866d43UL, + 0xcfc0d31aUL, 0xce02b92dUL, 0x91af9640UL, 0x906dfc77UL, 0x922b422eUL, + 0x93e92819UL, 0x96a63e9cUL, 0x976454abUL, 0x9522eaf2UL, 0x94e080c5UL, + 0x9fbcc7f8UL, 0x9e7eadcfUL, 0x9c381396UL, 0x9dfa79a1UL, 0x98b56f24UL, + 0x99770513UL, 0x9b31bb4aUL, 0x9af3d17dUL, 0x8d893530UL, 0x8c4b5f07UL, + 0x8e0de15eUL, 0x8fcf8b69UL, 0x8a809decUL, 0x8b42f7dbUL, 0x89044982UL, + 0x88c623b5UL, 0x839a6488UL, 0x82580ebfUL, 0x801eb0e6UL, 0x81dcdad1UL, + 0x8493cc54UL, 0x8551a663UL, 0x8717183aUL, 0x86d5720dUL, 0xa9e2d0a0UL, + 0xa820ba97UL, 0xaa6604ceUL, 0xaba46ef9UL, 0xaeeb787cUL, 0xaf29124bUL, + 0xad6fac12UL, 0xacadc625UL, 0xa7f18118UL, 0xa633eb2fUL, 0xa4755576UL, + 0xa5b73f41UL, 0xa0f829c4UL, 0xa13a43f3UL, 0xa37cfdaaUL, 0xa2be979dUL, + 0xb5c473d0UL, 0xb40619e7UL, 0xb640a7beUL, 0xb782cd89UL, 0xb2cddb0cUL, + 0xb30fb13bUL, 0xb1490f62UL, 0xb08b6555UL, 0xbbd72268UL, 0xba15485fUL, + 0xb853f606UL, 0xb9919c31UL, 0xbcde8ab4UL, 0xbd1ce083UL, 0xbf5a5edaUL, + 0xbe9834edUL + }, + { + 0x00000000UL, 0xb8bc6765UL, 0xaa09c88bUL, 0x12b5afeeUL, 0x8f629757UL, + 0x37def032UL, 0x256b5fdcUL, 0x9dd738b9UL, 0xc5b428efUL, 0x7d084f8aUL, + 0x6fbde064UL, 0xd7018701UL, 0x4ad6bfb8UL, 0xf26ad8ddUL, 0xe0df7733UL, + 0x58631056UL, 0x5019579fUL, 0xe8a530faUL, 0xfa109f14UL, 0x42acf871UL, + 0xdf7bc0c8UL, 0x67c7a7adUL, 0x75720843UL, 0xcdce6f26UL, 0x95ad7f70UL, + 0x2d111815UL, 0x3fa4b7fbUL, 0x8718d09eUL, 0x1acfe827UL, 0xa2738f42UL, + 0xb0c620acUL, 0x087a47c9UL, 0xa032af3eUL, 0x188ec85bUL, 0x0a3b67b5UL, + 0xb28700d0UL, 0x2f503869UL, 0x97ec5f0cUL, 0x8559f0e2UL, 0x3de59787UL, + 0x658687d1UL, 0xdd3ae0b4UL, 0xcf8f4f5aUL, 0x7733283fUL, 0xeae41086UL, + 0x525877e3UL, 0x40edd80dUL, 0xf851bf68UL, 0xf02bf8a1UL, 0x48979fc4UL, + 0x5a22302aUL, 0xe29e574fUL, 0x7f496ff6UL, 0xc7f50893UL, 0xd540a77dUL, + 0x6dfcc018UL, 0x359fd04eUL, 0x8d23b72bUL, 0x9f9618c5UL, 0x272a7fa0UL, + 0xbafd4719UL, 0x0241207cUL, 0x10f48f92UL, 0xa848e8f7UL, 0x9b14583dUL, + 0x23a83f58UL, 0x311d90b6UL, 0x89a1f7d3UL, 0x1476cf6aUL, 0xaccaa80fUL, + 0xbe7f07e1UL, 0x06c36084UL, 0x5ea070d2UL, 0xe61c17b7UL, 0xf4a9b859UL, + 0x4c15df3cUL, 0xd1c2e785UL, 0x697e80e0UL, 0x7bcb2f0eUL, 0xc377486bUL, + 0xcb0d0fa2UL, 0x73b168c7UL, 0x6104c729UL, 0xd9b8a04cUL, 0x446f98f5UL, + 0xfcd3ff90UL, 0xee66507eUL, 0x56da371bUL, 0x0eb9274dUL, 0xb6054028UL, + 0xa4b0efc6UL, 0x1c0c88a3UL, 0x81dbb01aUL, 0x3967d77fUL, 0x2bd27891UL, + 0x936e1ff4UL, 0x3b26f703UL, 0x839a9066UL, 0x912f3f88UL, 0x299358edUL, + 0xb4446054UL, 0x0cf80731UL, 0x1e4da8dfUL, 0xa6f1cfbaUL, 0xfe92dfecUL, + 0x462eb889UL, 0x549b1767UL, 0xec277002UL, 0x71f048bbUL, 0xc94c2fdeUL, + 0xdbf98030UL, 0x6345e755UL, 0x6b3fa09cUL, 0xd383c7f9UL, 0xc1366817UL, + 0x798a0f72UL, 0xe45d37cbUL, 0x5ce150aeUL, 0x4e54ff40UL, 0xf6e89825UL, + 0xae8b8873UL, 0x1637ef16UL, 0x048240f8UL, 0xbc3e279dUL, 0x21e91f24UL, + 0x99557841UL, 0x8be0d7afUL, 0x335cb0caUL, 0xed59b63bUL, 0x55e5d15eUL, + 0x47507eb0UL, 0xffec19d5UL, 0x623b216cUL, 0xda874609UL, 0xc832e9e7UL, + 0x708e8e82UL, 0x28ed9ed4UL, 0x9051f9b1UL, 0x82e4565fUL, 0x3a58313aUL, + 0xa78f0983UL, 0x1f336ee6UL, 0x0d86c108UL, 0xb53aa66dUL, 0xbd40e1a4UL, + 0x05fc86c1UL, 0x1749292fUL, 0xaff54e4aUL, 0x322276f3UL, 0x8a9e1196UL, + 0x982bbe78UL, 0x2097d91dUL, 0x78f4c94bUL, 0xc048ae2eUL, 0xd2fd01c0UL, + 0x6a4166a5UL, 0xf7965e1cUL, 0x4f2a3979UL, 0x5d9f9697UL, 0xe523f1f2UL, + 0x4d6b1905UL, 0xf5d77e60UL, 0xe762d18eUL, 0x5fdeb6ebUL, 0xc2098e52UL, + 0x7ab5e937UL, 0x680046d9UL, 0xd0bc21bcUL, 0x88df31eaUL, 0x3063568fUL, + 0x22d6f961UL, 0x9a6a9e04UL, 0x07bda6bdUL, 0xbf01c1d8UL, 0xadb46e36UL, + 0x15080953UL, 0x1d724e9aUL, 0xa5ce29ffUL, 0xb77b8611UL, 0x0fc7e174UL, + 0x9210d9cdUL, 0x2aacbea8UL, 0x38191146UL, 0x80a57623UL, 0xd8c66675UL, + 0x607a0110UL, 0x72cfaefeUL, 0xca73c99bUL, 0x57a4f122UL, 0xef189647UL, + 0xfdad39a9UL, 0x45115eccUL, 0x764dee06UL, 0xcef18963UL, 0xdc44268dUL, + 0x64f841e8UL, 0xf92f7951UL, 0x41931e34UL, 0x5326b1daUL, 0xeb9ad6bfUL, + 0xb3f9c6e9UL, 0x0b45a18cUL, 0x19f00e62UL, 0xa14c6907UL, 0x3c9b51beUL, + 0x842736dbUL, 0x96929935UL, 0x2e2efe50UL, 0x2654b999UL, 0x9ee8defcUL, + 0x8c5d7112UL, 0x34e11677UL, 0xa9362eceUL, 0x118a49abUL, 0x033fe645UL, + 0xbb838120UL, 0xe3e09176UL, 0x5b5cf613UL, 0x49e959fdUL, 0xf1553e98UL, + 0x6c820621UL, 0xd43e6144UL, 0xc68bceaaUL, 0x7e37a9cfUL, 0xd67f4138UL, + 0x6ec3265dUL, 0x7c7689b3UL, 0xc4caeed6UL, 0x591dd66fUL, 0xe1a1b10aUL, + 0xf3141ee4UL, 0x4ba87981UL, 0x13cb69d7UL, 0xab770eb2UL, 0xb9c2a15cUL, + 0x017ec639UL, 0x9ca9fe80UL, 0x241599e5UL, 0x36a0360bUL, 0x8e1c516eUL, + 0x866616a7UL, 0x3eda71c2UL, 0x2c6fde2cUL, 0x94d3b949UL, 0x090481f0UL, + 0xb1b8e695UL, 0xa30d497bUL, 0x1bb12e1eUL, 0x43d23e48UL, 0xfb6e592dUL, + 0xe9dbf6c3UL, 0x516791a6UL, 0xccb0a91fUL, 0x740cce7aUL, 0x66b96194UL, + 0xde0506f1UL + }, + { + 0x00000000UL, 0x96300777UL, 0x2c610eeeUL, 0xba510999UL, 0x19c46d07UL, + 0x8ff46a70UL, 0x35a563e9UL, 0xa395649eUL, 0x3288db0eUL, 0xa4b8dc79UL, + 0x1ee9d5e0UL, 0x88d9d297UL, 0x2b4cb609UL, 0xbd7cb17eUL, 0x072db8e7UL, + 0x911dbf90UL, 0x6410b71dUL, 0xf220b06aUL, 0x4871b9f3UL, 0xde41be84UL, + 0x7dd4da1aUL, 0xebe4dd6dUL, 0x51b5d4f4UL, 0xc785d383UL, 0x56986c13UL, + 0xc0a86b64UL, 0x7af962fdUL, 0xecc9658aUL, 0x4f5c0114UL, 0xd96c0663UL, + 0x633d0ffaUL, 0xf50d088dUL, 0xc8206e3bUL, 0x5e10694cUL, 0xe44160d5UL, + 0x727167a2UL, 0xd1e4033cUL, 0x47d4044bUL, 0xfd850dd2UL, 0x6bb50aa5UL, + 0xfaa8b535UL, 0x6c98b242UL, 0xd6c9bbdbUL, 0x40f9bcacUL, 0xe36cd832UL, + 0x755cdf45UL, 0xcf0dd6dcUL, 0x593dd1abUL, 0xac30d926UL, 0x3a00de51UL, + 0x8051d7c8UL, 0x1661d0bfUL, 0xb5f4b421UL, 0x23c4b356UL, 0x9995bacfUL, + 0x0fa5bdb8UL, 0x9eb80228UL, 0x0888055fUL, 0xb2d90cc6UL, 0x24e90bb1UL, + 0x877c6f2fUL, 0x114c6858UL, 0xab1d61c1UL, 0x3d2d66b6UL, 0x9041dc76UL, + 0x0671db01UL, 0xbc20d298UL, 0x2a10d5efUL, 0x8985b171UL, 0x1fb5b606UL, + 0xa5e4bf9fUL, 0x33d4b8e8UL, 0xa2c90778UL, 0x34f9000fUL, 0x8ea80996UL, + 0x18980ee1UL, 0xbb0d6a7fUL, 0x2d3d6d08UL, 0x976c6491UL, 0x015c63e6UL, + 0xf4516b6bUL, 0x62616c1cUL, 0xd8306585UL, 0x4e0062f2UL, 0xed95066cUL, + 0x7ba5011bUL, 0xc1f40882UL, 0x57c40ff5UL, 0xc6d9b065UL, 0x50e9b712UL, + 0xeab8be8bUL, 0x7c88b9fcUL, 0xdf1ddd62UL, 0x492dda15UL, 0xf37cd38cUL, + 0x654cd4fbUL, 0x5861b24dUL, 0xce51b53aUL, 0x7400bca3UL, 0xe230bbd4UL, + 0x41a5df4aUL, 0xd795d83dUL, 0x6dc4d1a4UL, 0xfbf4d6d3UL, 0x6ae96943UL, + 0xfcd96e34UL, 0x468867adUL, 0xd0b860daUL, 0x732d0444UL, 0xe51d0333UL, + 0x5f4c0aaaUL, 0xc97c0dddUL, 0x3c710550UL, 0xaa410227UL, 0x10100bbeUL, + 0x86200cc9UL, 0x25b56857UL, 0xb3856f20UL, 0x09d466b9UL, 0x9fe461ceUL, + 0x0ef9de5eUL, 0x98c9d929UL, 0x2298d0b0UL, 0xb4a8d7c7UL, 0x173db359UL, + 0x810db42eUL, 0x3b5cbdb7UL, 0xad6cbac0UL, 0x2083b8edUL, 0xb6b3bf9aUL, + 0x0ce2b603UL, 0x9ad2b174UL, 0x3947d5eaUL, 0xaf77d29dUL, 0x1526db04UL, + 0x8316dc73UL, 0x120b63e3UL, 0x843b6494UL, 0x3e6a6d0dUL, 0xa85a6a7aUL, + 0x0bcf0ee4UL, 0x9dff0993UL, 0x27ae000aUL, 0xb19e077dUL, 0x44930ff0UL, + 0xd2a30887UL, 0x68f2011eUL, 0xfec20669UL, 0x5d5762f7UL, 0xcb676580UL, + 0x71366c19UL, 0xe7066b6eUL, 0x761bd4feUL, 0xe02bd389UL, 0x5a7ada10UL, + 0xcc4add67UL, 0x6fdfb9f9UL, 0xf9efbe8eUL, 0x43beb717UL, 0xd58eb060UL, + 0xe8a3d6d6UL, 0x7e93d1a1UL, 0xc4c2d838UL, 0x52f2df4fUL, 0xf167bbd1UL, + 0x6757bca6UL, 0xdd06b53fUL, 0x4b36b248UL, 0xda2b0dd8UL, 0x4c1b0aafUL, + 0xf64a0336UL, 0x607a0441UL, 0xc3ef60dfUL, 0x55df67a8UL, 0xef8e6e31UL, + 0x79be6946UL, 0x8cb361cbUL, 0x1a8366bcUL, 0xa0d26f25UL, 0x36e26852UL, + 0x95770cccUL, 0x03470bbbUL, 0xb9160222UL, 0x2f260555UL, 0xbe3bbac5UL, + 0x280bbdb2UL, 0x925ab42bUL, 0x046ab35cUL, 0xa7ffd7c2UL, 0x31cfd0b5UL, + 0x8b9ed92cUL, 0x1daede5bUL, 0xb0c2649bUL, 0x26f263ecUL, 0x9ca36a75UL, + 0x0a936d02UL, 0xa906099cUL, 0x3f360eebUL, 0x85670772UL, 0x13570005UL, + 0x824abf95UL, 0x147ab8e2UL, 0xae2bb17bUL, 0x381bb60cUL, 0x9b8ed292UL, + 0x0dbed5e5UL, 0xb7efdc7cUL, 0x21dfdb0bUL, 0xd4d2d386UL, 0x42e2d4f1UL, + 0xf8b3dd68UL, 0x6e83da1fUL, 0xcd16be81UL, 0x5b26b9f6UL, 0xe177b06fUL, + 0x7747b718UL, 0xe65a0888UL, 0x706a0fffUL, 0xca3b0666UL, 0x5c0b0111UL, + 0xff9e658fUL, 0x69ae62f8UL, 0xd3ff6b61UL, 0x45cf6c16UL, 0x78e20aa0UL, + 0xeed20dd7UL, 0x5483044eUL, 0xc2b30339UL, 0x612667a7UL, 0xf71660d0UL, + 0x4d476949UL, 0xdb776e3eUL, 0x4a6ad1aeUL, 0xdc5ad6d9UL, 0x660bdf40UL, + 0xf03bd837UL, 0x53aebca9UL, 0xc59ebbdeUL, 0x7fcfb247UL, 0xe9ffb530UL, + 0x1cf2bdbdUL, 0x8ac2bacaUL, 0x3093b353UL, 0xa6a3b424UL, 0x0536d0baUL, + 0x9306d7cdUL, 0x2957de54UL, 0xbf67d923UL, 0x2e7a66b3UL, 0xb84a61c4UL, + 0x021b685dUL, 0x942b6f2aUL, 0x37be0bb4UL, 0xa18e0cc3UL, 0x1bdf055aUL, + 0x8def022dUL + }, + { + 0x00000000UL, 0x41311b19UL, 0x82623632UL, 0xc3532d2bUL, 0x04c56c64UL, + 0x45f4777dUL, 0x86a75a56UL, 0xc796414fUL, 0x088ad9c8UL, 0x49bbc2d1UL, + 0x8ae8effaUL, 0xcbd9f4e3UL, 0x0c4fb5acUL, 0x4d7eaeb5UL, 0x8e2d839eUL, + 0xcf1c9887UL, 0x5112c24aUL, 0x1023d953UL, 0xd370f478UL, 0x9241ef61UL, + 0x55d7ae2eUL, 0x14e6b537UL, 0xd7b5981cUL, 0x96848305UL, 0x59981b82UL, + 0x18a9009bUL, 0xdbfa2db0UL, 0x9acb36a9UL, 0x5d5d77e6UL, 0x1c6c6cffUL, + 0xdf3f41d4UL, 0x9e0e5acdUL, 0xa2248495UL, 0xe3159f8cUL, 0x2046b2a7UL, + 0x6177a9beUL, 0xa6e1e8f1UL, 0xe7d0f3e8UL, 0x2483dec3UL, 0x65b2c5daUL, + 0xaaae5d5dUL, 0xeb9f4644UL, 0x28cc6b6fUL, 0x69fd7076UL, 0xae6b3139UL, + 0xef5a2a20UL, 0x2c09070bUL, 0x6d381c12UL, 0xf33646dfUL, 0xb2075dc6UL, + 0x715470edUL, 0x30656bf4UL, 0xf7f32abbUL, 0xb6c231a2UL, 0x75911c89UL, + 0x34a00790UL, 0xfbbc9f17UL, 0xba8d840eUL, 0x79dea925UL, 0x38efb23cUL, + 0xff79f373UL, 0xbe48e86aUL, 0x7d1bc541UL, 0x3c2ade58UL, 0x054f79f0UL, + 0x447e62e9UL, 0x872d4fc2UL, 0xc61c54dbUL, 0x018a1594UL, 0x40bb0e8dUL, + 0x83e823a6UL, 0xc2d938bfUL, 0x0dc5a038UL, 0x4cf4bb21UL, 0x8fa7960aUL, + 0xce968d13UL, 0x0900cc5cUL, 0x4831d745UL, 0x8b62fa6eUL, 0xca53e177UL, + 0x545dbbbaUL, 0x156ca0a3UL, 0xd63f8d88UL, 0x970e9691UL, 0x5098d7deUL, + 0x11a9ccc7UL, 0xd2fae1ecUL, 0x93cbfaf5UL, 0x5cd76272UL, 0x1de6796bUL, + 0xdeb55440UL, 0x9f844f59UL, 0x58120e16UL, 0x1923150fUL, 0xda703824UL, + 0x9b41233dUL, 0xa76bfd65UL, 0xe65ae67cUL, 0x2509cb57UL, 0x6438d04eUL, + 0xa3ae9101UL, 0xe29f8a18UL, 0x21cca733UL, 0x60fdbc2aUL, 0xafe124adUL, + 0xeed03fb4UL, 0x2d83129fUL, 0x6cb20986UL, 0xab2448c9UL, 0xea1553d0UL, + 0x29467efbUL, 0x687765e2UL, 0xf6793f2fUL, 0xb7482436UL, 0x741b091dUL, + 0x352a1204UL, 0xf2bc534bUL, 0xb38d4852UL, 0x70de6579UL, 0x31ef7e60UL, + 0xfef3e6e7UL, 0xbfc2fdfeUL, 0x7c91d0d5UL, 0x3da0cbccUL, 0xfa368a83UL, + 0xbb07919aUL, 0x7854bcb1UL, 0x3965a7a8UL, 0x4b98833bUL, 0x0aa99822UL, + 0xc9fab509UL, 0x88cbae10UL, 0x4f5def5fUL, 0x0e6cf446UL, 0xcd3fd96dUL, + 0x8c0ec274UL, 0x43125af3UL, 0x022341eaUL, 0xc1706cc1UL, 0x804177d8UL, + 0x47d73697UL, 0x06e62d8eUL, 0xc5b500a5UL, 0x84841bbcUL, 0x1a8a4171UL, + 0x5bbb5a68UL, 0x98e87743UL, 0xd9d96c5aUL, 0x1e4f2d15UL, 0x5f7e360cUL, + 0x9c2d1b27UL, 0xdd1c003eUL, 0x120098b9UL, 0x533183a0UL, 0x9062ae8bUL, + 0xd153b592UL, 0x16c5f4ddUL, 0x57f4efc4UL, 0x94a7c2efUL, 0xd596d9f6UL, + 0xe9bc07aeUL, 0xa88d1cb7UL, 0x6bde319cUL, 0x2aef2a85UL, 0xed796bcaUL, + 0xac4870d3UL, 0x6f1b5df8UL, 0x2e2a46e1UL, 0xe136de66UL, 0xa007c57fUL, + 0x6354e854UL, 0x2265f34dUL, 0xe5f3b202UL, 0xa4c2a91bUL, 0x67918430UL, + 0x26a09f29UL, 0xb8aec5e4UL, 0xf99fdefdUL, 0x3accf3d6UL, 0x7bfde8cfUL, + 0xbc6ba980UL, 0xfd5ab299UL, 0x3e099fb2UL, 0x7f3884abUL, 0xb0241c2cUL, + 0xf1150735UL, 0x32462a1eUL, 0x73773107UL, 0xb4e17048UL, 0xf5d06b51UL, + 0x3683467aUL, 0x77b25d63UL, 0x4ed7facbUL, 0x0fe6e1d2UL, 0xccb5ccf9UL, + 0x8d84d7e0UL, 0x4a1296afUL, 0x0b238db6UL, 0xc870a09dUL, 0x8941bb84UL, + 0x465d2303UL, 0x076c381aUL, 0xc43f1531UL, 0x850e0e28UL, 0x42984f67UL, + 0x03a9547eUL, 0xc0fa7955UL, 0x81cb624cUL, 0x1fc53881UL, 0x5ef42398UL, + 0x9da70eb3UL, 0xdc9615aaUL, 0x1b0054e5UL, 0x5a314ffcUL, 0x996262d7UL, + 0xd85379ceUL, 0x174fe149UL, 0x567efa50UL, 0x952dd77bUL, 0xd41ccc62UL, + 0x138a8d2dUL, 0x52bb9634UL, 0x91e8bb1fUL, 0xd0d9a006UL, 0xecf37e5eUL, + 0xadc26547UL, 0x6e91486cUL, 0x2fa05375UL, 0xe836123aUL, 0xa9070923UL, + 0x6a542408UL, 0x2b653f11UL, 0xe479a796UL, 0xa548bc8fUL, 0x661b91a4UL, + 0x272a8abdUL, 0xe0bccbf2UL, 0xa18dd0ebUL, 0x62defdc0UL, 0x23efe6d9UL, + 0xbde1bc14UL, 0xfcd0a70dUL, 0x3f838a26UL, 0x7eb2913fUL, 0xb924d070UL, + 0xf815cb69UL, 0x3b46e642UL, 0x7a77fd5bUL, 0xb56b65dcUL, 0xf45a7ec5UL, + 0x370953eeUL, 0x763848f7UL, 0xb1ae09b8UL, 0xf09f12a1UL, 0x33cc3f8aUL, + 0x72fd2493UL + }, + { + 0x00000000UL, 0x376ac201UL, 0x6ed48403UL, 0x59be4602UL, 0xdca80907UL, + 0xebc2cb06UL, 0xb27c8d04UL, 0x85164f05UL, 0xb851130eUL, 0x8f3bd10fUL, + 0xd685970dUL, 0xe1ef550cUL, 0x64f91a09UL, 0x5393d808UL, 0x0a2d9e0aUL, + 0x3d475c0bUL, 0x70a3261cUL, 0x47c9e41dUL, 0x1e77a21fUL, 0x291d601eUL, + 0xac0b2f1bUL, 0x9b61ed1aUL, 0xc2dfab18UL, 0xf5b56919UL, 0xc8f23512UL, + 0xff98f713UL, 0xa626b111UL, 0x914c7310UL, 0x145a3c15UL, 0x2330fe14UL, + 0x7a8eb816UL, 0x4de47a17UL, 0xe0464d38UL, 0xd72c8f39UL, 0x8e92c93bUL, + 0xb9f80b3aUL, 0x3cee443fUL, 0x0b84863eUL, 0x523ac03cUL, 0x6550023dUL, + 0x58175e36UL, 0x6f7d9c37UL, 0x36c3da35UL, 0x01a91834UL, 0x84bf5731UL, + 0xb3d59530UL, 0xea6bd332UL, 0xdd011133UL, 0x90e56b24UL, 0xa78fa925UL, + 0xfe31ef27UL, 0xc95b2d26UL, 0x4c4d6223UL, 0x7b27a022UL, 0x2299e620UL, + 0x15f32421UL, 0x28b4782aUL, 0x1fdeba2bUL, 0x4660fc29UL, 0x710a3e28UL, + 0xf41c712dUL, 0xc376b32cUL, 0x9ac8f52eUL, 0xada2372fUL, 0xc08d9a70UL, + 0xf7e75871UL, 0xae591e73UL, 0x9933dc72UL, 0x1c259377UL, 0x2b4f5176UL, + 0x72f11774UL, 0x459bd575UL, 0x78dc897eUL, 0x4fb64b7fUL, 0x16080d7dUL, + 0x2162cf7cUL, 0xa4748079UL, 0x931e4278UL, 0xcaa0047aUL, 0xfdcac67bUL, + 0xb02ebc6cUL, 0x87447e6dUL, 0xdefa386fUL, 0xe990fa6eUL, 0x6c86b56bUL, + 0x5bec776aUL, 0x02523168UL, 0x3538f369UL, 0x087faf62UL, 0x3f156d63UL, + 0x66ab2b61UL, 0x51c1e960UL, 0xd4d7a665UL, 0xe3bd6464UL, 0xba032266UL, + 0x8d69e067UL, 0x20cbd748UL, 0x17a11549UL, 0x4e1f534bUL, 0x7975914aUL, + 0xfc63de4fUL, 0xcb091c4eUL, 0x92b75a4cUL, 0xa5dd984dUL, 0x989ac446UL, + 0xaff00647UL, 0xf64e4045UL, 0xc1248244UL, 0x4432cd41UL, 0x73580f40UL, + 0x2ae64942UL, 0x1d8c8b43UL, 0x5068f154UL, 0x67023355UL, 0x3ebc7557UL, + 0x09d6b756UL, 0x8cc0f853UL, 0xbbaa3a52UL, 0xe2147c50UL, 0xd57ebe51UL, + 0xe839e25aUL, 0xdf53205bUL, 0x86ed6659UL, 0xb187a458UL, 0x3491eb5dUL, + 0x03fb295cUL, 0x5a456f5eUL, 0x6d2fad5fUL, 0x801b35e1UL, 0xb771f7e0UL, + 0xeecfb1e2UL, 0xd9a573e3UL, 0x5cb33ce6UL, 0x6bd9fee7UL, 0x3267b8e5UL, + 0x050d7ae4UL, 0x384a26efUL, 0x0f20e4eeUL, 0x569ea2ecUL, 0x61f460edUL, + 0xe4e22fe8UL, 0xd388ede9UL, 0x8a36abebUL, 0xbd5c69eaUL, 0xf0b813fdUL, + 0xc7d2d1fcUL, 0x9e6c97feUL, 0xa90655ffUL, 0x2c101afaUL, 0x1b7ad8fbUL, + 0x42c49ef9UL, 0x75ae5cf8UL, 0x48e900f3UL, 0x7f83c2f2UL, 0x263d84f0UL, + 0x115746f1UL, 0x944109f4UL, 0xa32bcbf5UL, 0xfa958df7UL, 0xcdff4ff6UL, + 0x605d78d9UL, 0x5737bad8UL, 0x0e89fcdaUL, 0x39e33edbUL, 0xbcf571deUL, + 0x8b9fb3dfUL, 0xd221f5ddUL, 0xe54b37dcUL, 0xd80c6bd7UL, 0xef66a9d6UL, + 0xb6d8efd4UL, 0x81b22dd5UL, 0x04a462d0UL, 0x33cea0d1UL, 0x6a70e6d3UL, + 0x5d1a24d2UL, 0x10fe5ec5UL, 0x27949cc4UL, 0x7e2adac6UL, 0x494018c7UL, + 0xcc5657c2UL, 0xfb3c95c3UL, 0xa282d3c1UL, 0x95e811c0UL, 0xa8af4dcbUL, + 0x9fc58fcaUL, 0xc67bc9c8UL, 0xf1110bc9UL, 0x740744ccUL, 0x436d86cdUL, + 0x1ad3c0cfUL, 0x2db902ceUL, 0x4096af91UL, 0x77fc6d90UL, 0x2e422b92UL, + 0x1928e993UL, 0x9c3ea696UL, 0xab546497UL, 0xf2ea2295UL, 0xc580e094UL, + 0xf8c7bc9fUL, 0xcfad7e9eUL, 0x9613389cUL, 0xa179fa9dUL, 0x246fb598UL, + 0x13057799UL, 0x4abb319bUL, 0x7dd1f39aUL, 0x3035898dUL, 0x075f4b8cUL, + 0x5ee10d8eUL, 0x698bcf8fUL, 0xec9d808aUL, 0xdbf7428bUL, 0x82490489UL, + 0xb523c688UL, 0x88649a83UL, 0xbf0e5882UL, 0xe6b01e80UL, 0xd1dadc81UL, + 0x54cc9384UL, 0x63a65185UL, 0x3a181787UL, 0x0d72d586UL, 0xa0d0e2a9UL, + 0x97ba20a8UL, 0xce0466aaUL, 0xf96ea4abUL, 0x7c78ebaeUL, 0x4b1229afUL, + 0x12ac6fadUL, 0x25c6adacUL, 0x1881f1a7UL, 0x2feb33a6UL, 0x765575a4UL, + 0x413fb7a5UL, 0xc429f8a0UL, 0xf3433aa1UL, 0xaafd7ca3UL, 0x9d97bea2UL, + 0xd073c4b5UL, 0xe71906b4UL, 0xbea740b6UL, 0x89cd82b7UL, 0x0cdbcdb2UL, + 0x3bb10fb3UL, 0x620f49b1UL, 0x55658bb0UL, 0x6822d7bbUL, 0x5f4815baUL, + 0x06f653b8UL, 0x319c91b9UL, 0xb48adebcUL, 0x83e01cbdUL, 0xda5e5abfUL, + 0xed3498beUL + }, + { + 0x00000000UL, 0x6567bcb8UL, 0x8bc809aaUL, 0xeeafb512UL, 0x5797628fUL, + 0x32f0de37UL, 0xdc5f6b25UL, 0xb938d79dUL, 0xef28b4c5UL, 0x8a4f087dUL, + 0x64e0bd6fUL, 0x018701d7UL, 0xb8bfd64aUL, 0xddd86af2UL, 0x3377dfe0UL, + 0x56106358UL, 0x9f571950UL, 0xfa30a5e8UL, 0x149f10faUL, 0x71f8ac42UL, + 0xc8c07bdfUL, 0xada7c767UL, 0x43087275UL, 0x266fcecdUL, 0x707fad95UL, + 0x1518112dUL, 0xfbb7a43fUL, 0x9ed01887UL, 0x27e8cf1aUL, 0x428f73a2UL, + 0xac20c6b0UL, 0xc9477a08UL, 0x3eaf32a0UL, 0x5bc88e18UL, 0xb5673b0aUL, + 0xd00087b2UL, 0x6938502fUL, 0x0c5fec97UL, 0xe2f05985UL, 0x8797e53dUL, + 0xd1878665UL, 0xb4e03addUL, 0x5a4f8fcfUL, 0x3f283377UL, 0x8610e4eaUL, + 0xe3775852UL, 0x0dd8ed40UL, 0x68bf51f8UL, 0xa1f82bf0UL, 0xc49f9748UL, + 0x2a30225aUL, 0x4f579ee2UL, 0xf66f497fUL, 0x9308f5c7UL, 0x7da740d5UL, + 0x18c0fc6dUL, 0x4ed09f35UL, 0x2bb7238dUL, 0xc518969fUL, 0xa07f2a27UL, + 0x1947fdbaUL, 0x7c204102UL, 0x928ff410UL, 0xf7e848a8UL, 0x3d58149bUL, + 0x583fa823UL, 0xb6901d31UL, 0xd3f7a189UL, 0x6acf7614UL, 0x0fa8caacUL, + 0xe1077fbeUL, 0x8460c306UL, 0xd270a05eUL, 0xb7171ce6UL, 0x59b8a9f4UL, + 0x3cdf154cUL, 0x85e7c2d1UL, 0xe0807e69UL, 0x0e2fcb7bUL, 0x6b4877c3UL, + 0xa20f0dcbUL, 0xc768b173UL, 0x29c70461UL, 0x4ca0b8d9UL, 0xf5986f44UL, + 0x90ffd3fcUL, 0x7e5066eeUL, 0x1b37da56UL, 0x4d27b90eUL, 0x284005b6UL, + 0xc6efb0a4UL, 0xa3880c1cUL, 0x1ab0db81UL, 0x7fd76739UL, 0x9178d22bUL, + 0xf41f6e93UL, 0x03f7263bUL, 0x66909a83UL, 0x883f2f91UL, 0xed589329UL, + 0x546044b4UL, 0x3107f80cUL, 0xdfa84d1eUL, 0xbacff1a6UL, 0xecdf92feUL, + 0x89b82e46UL, 0x67179b54UL, 0x027027ecUL, 0xbb48f071UL, 0xde2f4cc9UL, + 0x3080f9dbUL, 0x55e74563UL, 0x9ca03f6bUL, 0xf9c783d3UL, 0x176836c1UL, + 0x720f8a79UL, 0xcb375de4UL, 0xae50e15cUL, 0x40ff544eUL, 0x2598e8f6UL, + 0x73888baeUL, 0x16ef3716UL, 0xf8408204UL, 0x9d273ebcUL, 0x241fe921UL, + 0x41785599UL, 0xafd7e08bUL, 0xcab05c33UL, 0x3bb659edUL, 0x5ed1e555UL, + 0xb07e5047UL, 0xd519ecffUL, 0x6c213b62UL, 0x094687daUL, 0xe7e932c8UL, + 0x828e8e70UL, 0xd49eed28UL, 0xb1f95190UL, 0x5f56e482UL, 0x3a31583aUL, + 0x83098fa7UL, 0xe66e331fUL, 0x08c1860dUL, 0x6da63ab5UL, 0xa4e140bdUL, + 0xc186fc05UL, 0x2f294917UL, 0x4a4ef5afUL, 0xf3762232UL, 0x96119e8aUL, + 0x78be2b98UL, 0x1dd99720UL, 0x4bc9f478UL, 0x2eae48c0UL, 0xc001fdd2UL, + 0xa566416aUL, 0x1c5e96f7UL, 0x79392a4fUL, 0x97969f5dUL, 0xf2f123e5UL, + 0x05196b4dUL, 0x607ed7f5UL, 0x8ed162e7UL, 0xebb6de5fUL, 0x528e09c2UL, + 0x37e9b57aUL, 0xd9460068UL, 0xbc21bcd0UL, 0xea31df88UL, 0x8f566330UL, + 0x61f9d622UL, 0x049e6a9aUL, 0xbda6bd07UL, 0xd8c101bfUL, 0x366eb4adUL, + 0x53090815UL, 0x9a4e721dUL, 0xff29cea5UL, 0x11867bb7UL, 0x74e1c70fUL, + 0xcdd91092UL, 0xa8beac2aUL, 0x46111938UL, 0x2376a580UL, 0x7566c6d8UL, + 0x10017a60UL, 0xfeaecf72UL, 0x9bc973caUL, 0x22f1a457UL, 0x479618efUL, + 0xa939adfdUL, 0xcc5e1145UL, 0x06ee4d76UL, 0x6389f1ceUL, 0x8d2644dcUL, + 0xe841f864UL, 0x51792ff9UL, 0x341e9341UL, 0xdab12653UL, 0xbfd69aebUL, + 0xe9c6f9b3UL, 0x8ca1450bUL, 0x620ef019UL, 0x07694ca1UL, 0xbe519b3cUL, + 0xdb362784UL, 0x35999296UL, 0x50fe2e2eUL, 0x99b95426UL, 0xfcdee89eUL, + 0x12715d8cUL, 0x7716e134UL, 0xce2e36a9UL, 0xab498a11UL, 0x45e63f03UL, + 0x208183bbUL, 0x7691e0e3UL, 0x13f65c5bUL, 0xfd59e949UL, 0x983e55f1UL, + 0x2106826cUL, 0x44613ed4UL, 0xaace8bc6UL, 0xcfa9377eUL, 0x38417fd6UL, + 0x5d26c36eUL, 0xb389767cUL, 0xd6eecac4UL, 0x6fd61d59UL, 0x0ab1a1e1UL, + 0xe41e14f3UL, 0x8179a84bUL, 0xd769cb13UL, 0xb20e77abUL, 0x5ca1c2b9UL, + 0x39c67e01UL, 0x80fea99cUL, 0xe5991524UL, 0x0b36a036UL, 0x6e511c8eUL, + 0xa7166686UL, 0xc271da3eUL, 0x2cde6f2cUL, 0x49b9d394UL, 0xf0810409UL, + 0x95e6b8b1UL, 0x7b490da3UL, 0x1e2eb11bUL, 0x483ed243UL, 0x2d596efbUL, + 0xc3f6dbe9UL, 0xa6916751UL, 0x1fa9b0ccUL, 0x7ace0c74UL, 0x9461b966UL, + 0xf10605deUL +#endif + } +}; diff --git a/third_party/zlib/crc32_simd.c b/third_party/zlib/crc32_simd.c new file mode 100644 index 00000000..c8e5592f --- /dev/null +++ b/third_party/zlib/crc32_simd.c @@ -0,0 +1,243 @@ +/* crc32_simd.c + * + * Copyright 2017 The Chromium Authors. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the Chromium source repository LICENSE file. + */ + +#include "crc32_simd.h" + +#if defined(CRC32_SIMD_SSE42_PCLMUL) + +/* + * crc32_sse42_simd_(): compute the crc32 of the buffer, where the buffer + * length must be at least 64, and a multiple of 16. Based on: + * + * "Fast CRC Computation for Generic Polynomials Using PCLMULQDQ Instruction" + * V. Gopal, E. Ozturk, et al., 2009, http://intel.ly/2ySEwL0 + */ + +#include <emmintrin.h> +#include <smmintrin.h> +#include <wmmintrin.h> + +uint32_t ZLIB_INTERNAL crc32_sse42_simd_( /* SSE4.2+PCLMUL */ + const unsigned char *buf, + z_size_t len, + uint32_t crc) +{ + /* + * Definitions of the bit-reflected domain constants k1,k2,k3, etc and + * the CRC32+Barrett polynomials given at the end of the paper. + */ + static const uint64_t zalign(16) k1k2[] = { 0x0154442bd4, 0x01c6e41596 }; + static const uint64_t zalign(16) k3k4[] = { 0x01751997d0, 0x00ccaa009e }; + static const uint64_t zalign(16) k5k0[] = { 0x0163cd6124, 0x0000000000 }; + static const uint64_t zalign(16) poly[] = { 0x01db710641, 0x01f7011641 }; + + __m128i x0, x1, x2, x3, x4, x5, x6, x7, x8, y5, y6, y7, y8; + + /* + * There's at least one block of 64. + */ + x1 = _mm_loadu_si128((__m128i *)(buf + 0x00)); + x2 = _mm_loadu_si128((__m128i *)(buf + 0x10)); + x3 = _mm_loadu_si128((__m128i *)(buf + 0x20)); + x4 = _mm_loadu_si128((__m128i *)(buf + 0x30)); + + x1 = _mm_xor_si128(x1, _mm_cvtsi32_si128(crc)); + + x0 = _mm_load_si128((__m128i *)k1k2); + + buf += 64; + len -= 64; + + /* + * Parallel fold blocks of 64, if any. + */ + while (len >= 64) + { + x5 = _mm_clmulepi64_si128(x1, x0, 0x00); + x6 = _mm_clmulepi64_si128(x2, x0, 0x00); + x7 = _mm_clmulepi64_si128(x3, x0, 0x00); + x8 = _mm_clmulepi64_si128(x4, x0, 0x00); + + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x2 = _mm_clmulepi64_si128(x2, x0, 0x11); + x3 = _mm_clmulepi64_si128(x3, x0, 0x11); + x4 = _mm_clmulepi64_si128(x4, x0, 0x11); + + y5 = _mm_loadu_si128((__m128i *)(buf + 0x00)); + y6 = _mm_loadu_si128((__m128i *)(buf + 0x10)); + y7 = _mm_loadu_si128((__m128i *)(buf + 0x20)); + y8 = _mm_loadu_si128((__m128i *)(buf + 0x30)); + + x1 = _mm_xor_si128(x1, x5); + x2 = _mm_xor_si128(x2, x6); + x3 = _mm_xor_si128(x3, x7); + x4 = _mm_xor_si128(x4, x8); + + x1 = _mm_xor_si128(x1, y5); + x2 = _mm_xor_si128(x2, y6); + x3 = _mm_xor_si128(x3, y7); + x4 = _mm_xor_si128(x4, y8); + + buf += 64; + len -= 64; + } + + /* + * Fold into 128-bits. + */ + x0 = _mm_load_si128((__m128i *)k3k4); + + x5 = _mm_clmulepi64_si128(x1, x0, 0x00); + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x1 = _mm_xor_si128(x1, x2); + x1 = _mm_xor_si128(x1, x5); + + x5 = _mm_clmulepi64_si128(x1, x0, 0x00); + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x1 = _mm_xor_si128(x1, x3); + x1 = _mm_xor_si128(x1, x5); + + x5 = _mm_clmulepi64_si128(x1, x0, 0x00); + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x1 = _mm_xor_si128(x1, x4); + x1 = _mm_xor_si128(x1, x5); + + /* + * Single fold blocks of 16, if any. + */ + while (len >= 16) + { + x2 = _mm_loadu_si128((__m128i *)buf); + + x5 = _mm_clmulepi64_si128(x1, x0, 0x00); + x1 = _mm_clmulepi64_si128(x1, x0, 0x11); + x1 = _mm_xor_si128(x1, x2); + x1 = _mm_xor_si128(x1, x5); + + buf += 16; + len -= 16; + } + + /* + * Fold 128-bits to 64-bits. + */ + x2 = _mm_clmulepi64_si128(x1, x0, 0x10); + x3 = _mm_setr_epi32(~0, 0, ~0, 0); + x1 = _mm_srli_si128(x1, 8); + x1 = _mm_xor_si128(x1, x2); + + x0 = _mm_loadl_epi64((__m128i*)k5k0); + + x2 = _mm_srli_si128(x1, 4); + x1 = _mm_and_si128(x1, x3); + x1 = _mm_clmulepi64_si128(x1, x0, 0x00); + x1 = _mm_xor_si128(x1, x2); + + /* + * Barret reduce to 32-bits. + */ + x0 = _mm_load_si128((__m128i*)poly); + + x2 = _mm_and_si128(x1, x3); + x2 = _mm_clmulepi64_si128(x2, x0, 0x10); + x2 = _mm_and_si128(x2, x3); + x2 = _mm_clmulepi64_si128(x2, x0, 0x00); + x1 = _mm_xor_si128(x1, x2); + + /* + * Return the crc32. + */ + return _mm_extract_epi32(x1, 1); +} + +#elif defined(CRC32_ARMV8_CRC32) + +/* CRC32 checksums using ARMv8-a crypto instructions. + * + * TODO: implement a version using the PMULL instruction. + */ + +#if defined(__clang__) +/* CRC32 intrinsics are #ifdef'ed out of arm_acle.h unless we build with an + * armv8 target, which is incompatible with ThinLTO optimizations on Android. + * (Namely, mixing and matching different module-level targets makes ThinLTO + * warn, and Android defaults to armv7-a. This restriction does not apply to + * function-level `target`s, however.) + * + * Since we only need four crc intrinsics, and since clang's implementation of + * those are just wrappers around compiler builtins, it's simplest to #define + * those builtins directly. If this #define list grows too much (or we depend on + * an intrinsic that isn't a trivial wrapper), we may have to find a better way + * to go about this. + * + * NOTE: clang currently complains that "'+soft-float-abi' is not a recognized + * feature for this target (ignoring feature)." This appears to be a harmless + * bug in clang. + */ +#define __crc32b __builtin_arm_crc32b +#define __crc32d __builtin_arm_crc32d +#define __crc32w __builtin_arm_crc32w +#define __crc32cw __builtin_arm_crc32cw + +#if defined(__aarch64__) +#define TARGET_ARMV8_WITH_CRC __attribute__((target("crc"))) +#else // !defined(__aarch64__) +#define TARGET_ARMV8_WITH_CRC __attribute__((target("armv8-a,crc"))) +#endif // defined(__aarch64__) + +#elif defined(__GNUC__) +/* For GCC, we are setting CRC extensions at module level, so ThinLTO is not + * allowed. We can just include arm_acle.h. + */ +#include <arm_acle.h> +#define TARGET_ARMV8_WITH_CRC +#else // !defined(__GNUC__) && !defined(_aarch64__) +#error ARM CRC32 SIMD extensions only supported for Clang and GCC +#endif + +TARGET_ARMV8_WITH_CRC +uint32_t ZLIB_INTERNAL armv8_crc32_little(unsigned long crc, + const unsigned char *buf, + z_size_t len) +{ + uint32_t c = (uint32_t) ~crc; + + while (len && ((uintptr_t)buf & 7)) { + c = __crc32b(c, *buf++); + --len; + } + + const uint64_t *buf8 = (const uint64_t *)buf; + + while (len >= 64) { + c = __crc32d(c, *buf8++); + c = __crc32d(c, *buf8++); + c = __crc32d(c, *buf8++); + c = __crc32d(c, *buf8++); + + c = __crc32d(c, *buf8++); + c = __crc32d(c, *buf8++); + c = __crc32d(c, *buf8++); + c = __crc32d(c, *buf8++); + len -= 64; + } + + while (len >= 8) { + c = __crc32d(c, *buf8++); + len -= 8; + } + + buf = (const unsigned char *)buf8; + + while (len--) { + c = __crc32b(c, *buf++); + } + + return ~c; +} + +#endif diff --git a/third_party/zlib/crc32_simd.h b/third_party/zlib/crc32_simd.h new file mode 100644 index 00000000..68bc235c --- /dev/null +++ b/third_party/zlib/crc32_simd.h @@ -0,0 +1,36 @@ +/* crc32_simd.h + * + * Copyright 2017 The Chromium Authors. All rights reserved. + * Use of this source code is governed by a BSD-style license that can be + * found in the Chromium source repository LICENSE file. + */ + +#include <stdint.h> + +#include "zconf.h" +#include "zutil.h" +#include "deflate.h" + +/* + * crc32_sse42_simd_(): compute the crc32 of the buffer, where the buffer + * length must be at least 64, and a multiple of 16. + */ +uint32_t ZLIB_INTERNAL crc32_sse42_simd_( + const unsigned char *buf, + z_size_t len, + uint32_t crc); + +/* + * crc32_sse42_simd_ buffer size constraints: see the use in zlib/crc32.c + * for computing the crc32 of an arbitrary length buffer. + */ +#define Z_CRC32_SSE42_MINIMUM_LENGTH 64 +#define Z_CRC32_SSE42_CHUNKSIZE_MASK 15 + +/* + * CRC32 checksums using ARMv8-a crypto instructions. + */ +uint32_t ZLIB_INTERNAL armv8_crc32_little(unsigned long crc, + const unsigned char* buf, + z_size_t len); + diff --git a/third_party/zlib/crc_folding.c b/third_party/zlib/crc_folding.c new file mode 100644 index 00000000..ee31d491 --- /dev/null +++ b/third_party/zlib/crc_folding.c @@ -0,0 +1,497 @@ +/* + * Compute the CRC32 using a parallelized folding approach with the PCLMULQDQ + * instruction. + * + * A white paper describing this algorithm can be found at: + * http://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf + * + * Copyright (C) 2013 Intel Corporation. All rights reserved. + * Authors: + * Wajdi Feghali <wajdi.k.feghali@intel.com> + * Jim Guilford <james.guilford@intel.com> + * Vinodh Gopal <vinodh.gopal@intel.com> + * Erdinc Ozturk <erdinc.ozturk@intel.com> + * Jim Kukunas <james.t.kukunas@linux.intel.com> + * + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +#include "deflate.h" + +#ifdef CRC32_SIMD_SSE42_PCLMUL + +#include <inttypes.h> +#include <emmintrin.h> +#include <immintrin.h> +#include <wmmintrin.h> + +#define CRC_LOAD(s) \ + do { \ + __m128i xmm_crc0 = _mm_loadu_si128((__m128i *)s->crc0 + 0);\ + __m128i xmm_crc1 = _mm_loadu_si128((__m128i *)s->crc0 + 1);\ + __m128i xmm_crc2 = _mm_loadu_si128((__m128i *)s->crc0 + 2);\ + __m128i xmm_crc3 = _mm_loadu_si128((__m128i *)s->crc0 + 3);\ + __m128i xmm_crc_part = _mm_loadu_si128((__m128i *)s->crc0 + 4); + +#define CRC_SAVE(s) \ + _mm_storeu_si128((__m128i *)s->crc0 + 0, xmm_crc0);\ + _mm_storeu_si128((__m128i *)s->crc0 + 1, xmm_crc1);\ + _mm_storeu_si128((__m128i *)s->crc0 + 2, xmm_crc2);\ + _mm_storeu_si128((__m128i *)s->crc0 + 3, xmm_crc3);\ + _mm_storeu_si128((__m128i *)s->crc0 + 4, xmm_crc_part);\ + } while (0); + +ZLIB_INTERNAL void crc_fold_init(deflate_state *const s) +{ + CRC_LOAD(s) + + xmm_crc0 = _mm_cvtsi32_si128(0x9db42487); + xmm_crc1 = _mm_setzero_si128(); + xmm_crc2 = _mm_setzero_si128(); + xmm_crc3 = _mm_setzero_si128(); + + CRC_SAVE(s) + + s->strm->adler = 0; +} + +local void fold_1(deflate_state *const s, + __m128i *xmm_crc0, __m128i *xmm_crc1, + __m128i *xmm_crc2, __m128i *xmm_crc3) +{ + const __m128i xmm_fold4 = _mm_set_epi32( + 0x00000001, 0x54442bd4, + 0x00000001, 0xc6e41596); + + __m128i x_tmp3; + __m128 ps_crc0, ps_crc3, ps_res; + + x_tmp3 = *xmm_crc3; + + *xmm_crc3 = *xmm_crc0; + *xmm_crc0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x01); + *xmm_crc3 = _mm_clmulepi64_si128(*xmm_crc3, xmm_fold4, 0x10); + ps_crc0 = _mm_castsi128_ps(*xmm_crc0); + ps_crc3 = _mm_castsi128_ps(*xmm_crc3); + ps_res = _mm_xor_ps(ps_crc0, ps_crc3); + + *xmm_crc0 = *xmm_crc1; + *xmm_crc1 = *xmm_crc2; + *xmm_crc2 = x_tmp3; + *xmm_crc3 = _mm_castps_si128(ps_res); +} + +local void fold_2(deflate_state *const s, + __m128i *xmm_crc0, __m128i *xmm_crc1, + __m128i *xmm_crc2, __m128i *xmm_crc3) +{ + const __m128i xmm_fold4 = _mm_set_epi32( + 0x00000001, 0x54442bd4, + 0x00000001, 0xc6e41596); + + __m128i x_tmp3, x_tmp2; + __m128 ps_crc0, ps_crc1, ps_crc2, ps_crc3, ps_res31, ps_res20; + + x_tmp3 = *xmm_crc3; + x_tmp2 = *xmm_crc2; + + *xmm_crc3 = *xmm_crc1; + *xmm_crc1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x01); + *xmm_crc3 = _mm_clmulepi64_si128(*xmm_crc3, xmm_fold4, 0x10); + ps_crc3 = _mm_castsi128_ps(*xmm_crc3); + ps_crc1 = _mm_castsi128_ps(*xmm_crc1); + ps_res31= _mm_xor_ps(ps_crc3, ps_crc1); + + *xmm_crc2 = *xmm_crc0; + *xmm_crc0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x01); + *xmm_crc2 = _mm_clmulepi64_si128(*xmm_crc2, xmm_fold4, 0x10); + ps_crc0 = _mm_castsi128_ps(*xmm_crc0); + ps_crc2 = _mm_castsi128_ps(*xmm_crc2); + ps_res20= _mm_xor_ps(ps_crc0, ps_crc2); + + *xmm_crc0 = x_tmp2; + *xmm_crc1 = x_tmp3; + *xmm_crc2 = _mm_castps_si128(ps_res20); + *xmm_crc3 = _mm_castps_si128(ps_res31); +} + +local void fold_3(deflate_state *const s, + __m128i *xmm_crc0, __m128i *xmm_crc1, + __m128i *xmm_crc2, __m128i *xmm_crc3) +{ + const __m128i xmm_fold4 = _mm_set_epi32( + 0x00000001, 0x54442bd4, + 0x00000001, 0xc6e41596); + + __m128i x_tmp3; + __m128 ps_crc0, ps_crc1, ps_crc2, ps_crc3, ps_res32, ps_res21, ps_res10; + + x_tmp3 = *xmm_crc3; + + *xmm_crc3 = *xmm_crc2; + *xmm_crc2 = _mm_clmulepi64_si128(*xmm_crc2, xmm_fold4, 0x01); + *xmm_crc3 = _mm_clmulepi64_si128(*xmm_crc3, xmm_fold4, 0x10); + ps_crc2 = _mm_castsi128_ps(*xmm_crc2); + ps_crc3 = _mm_castsi128_ps(*xmm_crc3); + ps_res32 = _mm_xor_ps(ps_crc2, ps_crc3); + + *xmm_crc2 = *xmm_crc1; + *xmm_crc1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x01); + *xmm_crc2 = _mm_clmulepi64_si128(*xmm_crc2, xmm_fold4, 0x10); + ps_crc1 = _mm_castsi128_ps(*xmm_crc1); + ps_crc2 = _mm_castsi128_ps(*xmm_crc2); + ps_res21= _mm_xor_ps(ps_crc1, ps_crc2); + + *xmm_crc1 = *xmm_crc0; + *xmm_crc0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x01); + *xmm_crc1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x10); + ps_crc0 = _mm_castsi128_ps(*xmm_crc0); + ps_crc1 = _mm_castsi128_ps(*xmm_crc1); + ps_res10= _mm_xor_ps(ps_crc0, ps_crc1); + + *xmm_crc0 = x_tmp3; + *xmm_crc1 = _mm_castps_si128(ps_res10); + *xmm_crc2 = _mm_castps_si128(ps_res21); + *xmm_crc3 = _mm_castps_si128(ps_res32); +} + +local void fold_4(deflate_state *const s, + __m128i *xmm_crc0, __m128i *xmm_crc1, + __m128i *xmm_crc2, __m128i *xmm_crc3) +{ + const __m128i xmm_fold4 = _mm_set_epi32( + 0x00000001, 0x54442bd4, + 0x00000001, 0xc6e41596); + + __m128i x_tmp0, x_tmp1, x_tmp2, x_tmp3; + __m128 ps_crc0, ps_crc1, ps_crc2, ps_crc3; + __m128 ps_t0, ps_t1, ps_t2, ps_t3; + __m128 ps_res0, ps_res1, ps_res2, ps_res3; + + x_tmp0 = *xmm_crc0; + x_tmp1 = *xmm_crc1; + x_tmp2 = *xmm_crc2; + x_tmp3 = *xmm_crc3; + + *xmm_crc0 = _mm_clmulepi64_si128(*xmm_crc0, xmm_fold4, 0x01); + x_tmp0 = _mm_clmulepi64_si128(x_tmp0, xmm_fold4, 0x10); + ps_crc0 = _mm_castsi128_ps(*xmm_crc0); + ps_t0 = _mm_castsi128_ps(x_tmp0); + ps_res0 = _mm_xor_ps(ps_crc0, ps_t0); + + *xmm_crc1 = _mm_clmulepi64_si128(*xmm_crc1, xmm_fold4, 0x01); + x_tmp1 = _mm_clmulepi64_si128(x_tmp1, xmm_fold4, 0x10); + ps_crc1 = _mm_castsi128_ps(*xmm_crc1); + ps_t1 = _mm_castsi128_ps(x_tmp1); + ps_res1 = _mm_xor_ps(ps_crc1, ps_t1); + + *xmm_crc2 = _mm_clmulepi64_si128(*xmm_crc2, xmm_fold4, 0x01); + x_tmp2 = _mm_clmulepi64_si128(x_tmp2, xmm_fold4, 0x10); + ps_crc2 = _mm_castsi128_ps(*xmm_crc2); + ps_t2 = _mm_castsi128_ps(x_tmp2); + ps_res2 = _mm_xor_ps(ps_crc2, ps_t2); + + *xmm_crc3 = _mm_clmulepi64_si128(*xmm_crc3, xmm_fold4, 0x01); + x_tmp3 = _mm_clmulepi64_si128(x_tmp3, xmm_fold4, 0x10); + ps_crc3 = _mm_castsi128_ps(*xmm_crc3); + ps_t3 = _mm_castsi128_ps(x_tmp3); + ps_res3 = _mm_xor_ps(ps_crc3, ps_t3); + + *xmm_crc0 = _mm_castps_si128(ps_res0); + *xmm_crc1 = _mm_castps_si128(ps_res1); + *xmm_crc2 = _mm_castps_si128(ps_res2); + *xmm_crc3 = _mm_castps_si128(ps_res3); +} + +local const unsigned zalign(32) pshufb_shf_table[60] = { + 0x84838281,0x88878685,0x8c8b8a89,0x008f8e8d, /* shl 15 (16 - 1)/shr1 */ + 0x85848382,0x89888786,0x8d8c8b8a,0x01008f8e, /* shl 14 (16 - 3)/shr2 */ + 0x86858483,0x8a898887,0x8e8d8c8b,0x0201008f, /* shl 13 (16 - 4)/shr3 */ + 0x87868584,0x8b8a8988,0x8f8e8d8c,0x03020100, /* shl 12 (16 - 4)/shr4 */ + 0x88878685,0x8c8b8a89,0x008f8e8d,0x04030201, /* shl 11 (16 - 5)/shr5 */ + 0x89888786,0x8d8c8b8a,0x01008f8e,0x05040302, /* shl 10 (16 - 6)/shr6 */ + 0x8a898887,0x8e8d8c8b,0x0201008f,0x06050403, /* shl 9 (16 - 7)/shr7 */ + 0x8b8a8988,0x8f8e8d8c,0x03020100,0x07060504, /* shl 8 (16 - 8)/shr8 */ + 0x8c8b8a89,0x008f8e8d,0x04030201,0x08070605, /* shl 7 (16 - 9)/shr9 */ + 0x8d8c8b8a,0x01008f8e,0x05040302,0x09080706, /* shl 6 (16 -10)/shr10*/ + 0x8e8d8c8b,0x0201008f,0x06050403,0x0a090807, /* shl 5 (16 -11)/shr11*/ + 0x8f8e8d8c,0x03020100,0x07060504,0x0b0a0908, /* shl 4 (16 -12)/shr12*/ + 0x008f8e8d,0x04030201,0x08070605,0x0c0b0a09, /* shl 3 (16 -13)/shr13*/ + 0x01008f8e,0x05040302,0x09080706,0x0d0c0b0a, /* shl 2 (16 -14)/shr14*/ + 0x0201008f,0x06050403,0x0a090807,0x0e0d0c0b /* shl 1 (16 -15)/shr15*/ +}; + +local void partial_fold(deflate_state *const s, const size_t len, + __m128i *xmm_crc0, __m128i *xmm_crc1, + __m128i *xmm_crc2, __m128i *xmm_crc3, + __m128i *xmm_crc_part) +{ + + const __m128i xmm_fold4 = _mm_set_epi32( + 0x00000001, 0x54442bd4, + 0x00000001, 0xc6e41596); + const __m128i xmm_mask3 = _mm_set1_epi32(0x80808080); + + __m128i xmm_shl, xmm_shr, xmm_tmp1, xmm_tmp2, xmm_tmp3; + __m128i xmm_a0_0, xmm_a0_1; + __m128 ps_crc3, psa0_0, psa0_1, ps_res; + + xmm_shl = _mm_load_si128((__m128i *)pshufb_shf_table + (len - 1)); + xmm_shr = xmm_shl; + xmm_shr = _mm_xor_si128(xmm_shr, xmm_mask3); + + xmm_a0_0 = _mm_shuffle_epi8(*xmm_crc0, xmm_shl); + + *xmm_crc0 = _mm_shuffle_epi8(*xmm_crc0, xmm_shr); + xmm_tmp1 = _mm_shuffle_epi8(*xmm_crc1, xmm_shl); + *xmm_crc0 = _mm_or_si128(*xmm_crc0, xmm_tmp1); + + *xmm_crc1 = _mm_shuffle_epi8(*xmm_crc1, xmm_shr); + xmm_tmp2 = _mm_shuffle_epi8(*xmm_crc2, xmm_shl); + *xmm_crc1 = _mm_or_si128(*xmm_crc1, xmm_tmp2); + + *xmm_crc2 = _mm_shuffle_epi8(*xmm_crc2, xmm_shr); + xmm_tmp3 = _mm_shuffle_epi8(*xmm_crc3, xmm_shl); + *xmm_crc2 = _mm_or_si128(*xmm_crc2, xmm_tmp3); + + *xmm_crc3 = _mm_shuffle_epi8(*xmm_crc3, xmm_shr); + *xmm_crc_part = _mm_shuffle_epi8(*xmm_crc_part, xmm_shl); + *xmm_crc3 = _mm_or_si128(*xmm_crc3, *xmm_crc_part); + + xmm_a0_1 = _mm_clmulepi64_si128(xmm_a0_0, xmm_fold4, 0x10); + xmm_a0_0 = _mm_clmulepi64_si128(xmm_a0_0, xmm_fold4, 0x01); + + ps_crc3 = _mm_castsi128_ps(*xmm_crc3); + psa0_0 = _mm_castsi128_ps(xmm_a0_0); + psa0_1 = _mm_castsi128_ps(xmm_a0_1); + + ps_res = _mm_xor_ps(ps_crc3, psa0_0); + ps_res = _mm_xor_ps(ps_res, psa0_1); + + *xmm_crc3 = _mm_castps_si128(ps_res); +} + +ZLIB_INTERNAL void crc_fold_copy(deflate_state *const s, + unsigned char *dst, const unsigned char *src, long len) +{ + unsigned long algn_diff; + __m128i xmm_t0, xmm_t1, xmm_t2, xmm_t3; + + CRC_LOAD(s) + + if (len < 16) { + if (len == 0) + return; + goto partial; + } + + algn_diff = (0 - (uintptr_t)src) & 0xF; + if (algn_diff) { + xmm_crc_part = _mm_loadu_si128((__m128i *)src); + _mm_storeu_si128((__m128i *)dst, xmm_crc_part); + + dst += algn_diff; + src += algn_diff; + len -= algn_diff; + + partial_fold(s, algn_diff, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3, + &xmm_crc_part); + } + + while ((len -= 64) >= 0) { + xmm_t0 = _mm_load_si128((__m128i *)src); + xmm_t1 = _mm_load_si128((__m128i *)src + 1); + xmm_t2 = _mm_load_si128((__m128i *)src + 2); + xmm_t3 = _mm_load_si128((__m128i *)src + 3); + + fold_4(s, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3); + + _mm_storeu_si128((__m128i *)dst, xmm_t0); + _mm_storeu_si128((__m128i *)dst + 1, xmm_t1); + _mm_storeu_si128((__m128i *)dst + 2, xmm_t2); + _mm_storeu_si128((__m128i *)dst + 3, xmm_t3); + + xmm_crc0 = _mm_xor_si128(xmm_crc0, xmm_t0); + xmm_crc1 = _mm_xor_si128(xmm_crc1, xmm_t1); + xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_t2); + xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t3); + + src += 64; + dst += 64; + } + + /* + * len = num bytes left - 64 + */ + if (len + 16 >= 0) { + len += 16; + + xmm_t0 = _mm_load_si128((__m128i *)src); + xmm_t1 = _mm_load_si128((__m128i *)src + 1); + xmm_t2 = _mm_load_si128((__m128i *)src + 2); + + fold_3(s, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3); + + _mm_storeu_si128((__m128i *)dst, xmm_t0); + _mm_storeu_si128((__m128i *)dst + 1, xmm_t1); + _mm_storeu_si128((__m128i *)dst + 2, xmm_t2); + + xmm_crc1 = _mm_xor_si128(xmm_crc1, xmm_t0); + xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_t1); + xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t2); + + if (len == 0) + goto done; + + dst += 48; + src += 48; + } else if (len + 32 >= 0) { + len += 32; + + xmm_t0 = _mm_load_si128((__m128i *)src); + xmm_t1 = _mm_load_si128((__m128i *)src + 1); + + fold_2(s, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3); + + _mm_storeu_si128((__m128i *)dst, xmm_t0); + _mm_storeu_si128((__m128i *)dst + 1, xmm_t1); + + xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_t0); + xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t1); + + if (len == 0) + goto done; + + dst += 32; + src += 32; + } else if (len + 48 >= 0) { + len += 48; + + xmm_t0 = _mm_load_si128((__m128i *)src); + + fold_1(s, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3); + + _mm_storeu_si128((__m128i *)dst, xmm_t0); + + xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_t0); + + if (len == 0) + goto done; + + dst += 16; + src += 16; + } else { + len += 64; + if (len == 0) + goto done; + } + +partial: + +#if defined(_MSC_VER) + /* VS does not permit the use of _mm_set_epi64x in 32-bit builds */ + { + int32_t parts[4] = {0, 0, 0, 0}; + memcpy(&parts, src, len); + xmm_crc_part = _mm_set_epi32(parts[3], parts[2], parts[1], parts[0]); + } +#else + { + int64_t parts[2] = {0, 0}; + memcpy(&parts, src, len); + xmm_crc_part = _mm_set_epi64x(parts[1], parts[0]); + } +#endif + + _mm_storeu_si128((__m128i *)dst, xmm_crc_part); + partial_fold(s, len, &xmm_crc0, &xmm_crc1, &xmm_crc2, &xmm_crc3, + &xmm_crc_part); +done: + CRC_SAVE(s) +} + +local const unsigned zalign(16) crc_k[] = { + 0xccaa009e, 0x00000000, /* rk1 */ + 0x751997d0, 0x00000001, /* rk2 */ + 0xccaa009e, 0x00000000, /* rk5 */ + 0x63cd6124, 0x00000001, /* rk6 */ + 0xf7011640, 0x00000001, /* rk7 */ + 0xdb710640, 0x00000001 /* rk8 */ +}; + +local const unsigned zalign(16) crc_mask[4] = { + 0xFFFFFFFF, 0xFFFFFFFF, 0x00000000, 0x00000000 +}; + +local const unsigned zalign(16) crc_mask2[4] = { + 0x00000000, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF +}; + +unsigned ZLIB_INTERNAL crc_fold_512to32(deflate_state *const s) +{ + const __m128i xmm_mask = _mm_load_si128((__m128i *)crc_mask); + const __m128i xmm_mask2 = _mm_load_si128((__m128i *)crc_mask2); + + unsigned crc; + __m128i x_tmp0, x_tmp1, x_tmp2, crc_fold; + + CRC_LOAD(s) + + /* + * k1 + */ + crc_fold = _mm_load_si128((__m128i *)crc_k); + + x_tmp0 = _mm_clmulepi64_si128(xmm_crc0, crc_fold, 0x10); + xmm_crc0 = _mm_clmulepi64_si128(xmm_crc0, crc_fold, 0x01); + xmm_crc1 = _mm_xor_si128(xmm_crc1, x_tmp0); + xmm_crc1 = _mm_xor_si128(xmm_crc1, xmm_crc0); + + x_tmp1 = _mm_clmulepi64_si128(xmm_crc1, crc_fold, 0x10); + xmm_crc1 = _mm_clmulepi64_si128(xmm_crc1, crc_fold, 0x01); + xmm_crc2 = _mm_xor_si128(xmm_crc2, x_tmp1); + xmm_crc2 = _mm_xor_si128(xmm_crc2, xmm_crc1); + + x_tmp2 = _mm_clmulepi64_si128(xmm_crc2, crc_fold, 0x10); + xmm_crc2 = _mm_clmulepi64_si128(xmm_crc2, crc_fold, 0x01); + xmm_crc3 = _mm_xor_si128(xmm_crc3, x_tmp2); + xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc2); + + /* + * k5 + */ + crc_fold = _mm_load_si128((__m128i *)crc_k + 1); + + xmm_crc0 = xmm_crc3; + xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0); + xmm_crc0 = _mm_srli_si128(xmm_crc0, 8); + xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc0); + + xmm_crc0 = xmm_crc3; + xmm_crc3 = _mm_slli_si128(xmm_crc3, 4); + xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0x10); + xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc0); + xmm_crc3 = _mm_and_si128(xmm_crc3, xmm_mask2); + + /* + * k7 + */ + xmm_crc1 = xmm_crc3; + xmm_crc2 = xmm_crc3; + crc_fold = _mm_load_si128((__m128i *)crc_k + 2); + + xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0); + xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc2); + xmm_crc3 = _mm_and_si128(xmm_crc3, xmm_mask); + + xmm_crc2 = xmm_crc3; + xmm_crc3 = _mm_clmulepi64_si128(xmm_crc3, crc_fold, 0x10); + xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc2); + xmm_crc3 = _mm_xor_si128(xmm_crc3, xmm_crc1); + + crc = _mm_extract_epi32(xmm_crc3, 2); + return ~crc; + CRC_SAVE(s) +} + +#endif /* CRC32_SIMD_SSE42_PCLMUL */ diff --git a/third_party/zlib/deflate.c b/third_party/zlib/deflate.c new file mode 100644 index 00000000..fc7ae459 --- /dev/null +++ b/third_party/zlib/deflate.c @@ -0,0 +1,2245 @@ +/* deflate.c -- compress data using the deflation algorithm + * Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +/* + * ALGORITHM + * + * The "deflation" process depends on being able to identify portions + * of the input text which are identical to earlier input (within a + * sliding window trailing behind the input currently being processed). + * + * The most straightforward technique turns out to be the fastest for + * most input files: try all possible matches and select the longest. + * The key feature of this algorithm is that insertions into the string + * dictionary are very simple and thus fast, and deletions are avoided + * completely. Insertions are performed at each input character, whereas + * string matches are performed only when the previous match ends. So it + * is preferable to spend more time in matches to allow very fast string + * insertions and avoid deletions. The matching algorithm for small + * strings is inspired from that of Rabin & Karp. A brute force approach + * is used to find longer strings when a small match has been found. + * A similar algorithm is used in comic (by Jan-Mark Wams) and freeze + * (by Leonid Broukhis). + * A previous version of this file used a more sophisticated algorithm + * (by Fiala and Greene) which is guaranteed to run in linear amortized + * time, but has a larger average cost, uses more memory and is patented. + * However the F&G algorithm may be faster for some highly redundant + * files if the parameter max_chain_length (described below) is too large. + * + * ACKNOWLEDGEMENTS + * + * The idea of lazy evaluation of matches is due to Jan-Mark Wams, and + * I found it in 'freeze' written by Leonid Broukhis. + * Thanks to many people for bug reports and testing. + * + * REFERENCES + * + * Deutsch, L.P.,"DEFLATE Compressed Data Format Specification". + * Available in http://tools.ietf.org/html/rfc1951 + * + * A description of the Rabin and Karp algorithm is given in the book + * "Algorithms" by R. Sedgewick, Addison-Wesley, p252. + * + * Fiala,E.R., and Greene,D.H. + * Data Compression with Finite Windows, Comm.ACM, 32,4 (1989) 490-595 + * + */ + +/* @(#) $Id$ */ +#include <assert.h> +#include "deflate.h" +#include "cpu_features.h" +#include "contrib/optimizations/insert_string.h" + +#if (defined(__ARM_NEON__) || defined(__ARM_NEON)) +#include "contrib/optimizations/slide_hash_neon.h" +#endif +#if defined(CRC32_ARMV8_CRC32) +#include "crc32_simd.h" +#endif + +#ifdef FASTEST +/* See http://crbug.com/1113596 */ +#error "FASTEST is not supported in Chromium's zlib." +#endif + +const char deflate_copyright[] = + " deflate 1.2.11 Copyright 1995-2017 Jean-loup Gailly and Mark Adler "; +/* + If you use the zlib library in a product, an acknowledgment is welcome + in the documentation of your product. If for some reason you cannot + include such an acknowledgment, I would appreciate that you keep this + copyright string in the executable of your product. + */ + +/* =========================================================================== + * Function prototypes. + */ +typedef enum { + need_more, /* block not completed, need more input or more output */ + block_done, /* block flush performed */ + finish_started, /* finish started, need only more output at next deflate */ + finish_done /* finish done, accept no more input or output */ +} block_state; + +typedef block_state (*compress_func) OF((deflate_state *s, int flush)); +/* Compression function. Returns the block state after the call. */ + +local int deflateStateCheck OF((z_streamp strm)); +local void slide_hash OF((deflate_state *s)); +local void fill_window OF((deflate_state *s)); +local block_state deflate_stored OF((deflate_state *s, int flush)); +local block_state deflate_fast OF((deflate_state *s, int flush)); +#ifndef FASTEST +local block_state deflate_slow OF((deflate_state *s, int flush)); +#endif +local block_state deflate_rle OF((deflate_state *s, int flush)); +local block_state deflate_huff OF((deflate_state *s, int flush)); +local void lm_init OF((deflate_state *s)); +local void putShortMSB OF((deflate_state *s, uInt b)); +local void flush_pending OF((z_streamp strm)); +unsigned ZLIB_INTERNAL deflate_read_buf OF((z_streamp strm, Bytef *buf, unsigned size)); +#ifdef ASMV +# pragma message("Assembler code may have bugs -- use at your own risk") + void match_init OF((void)); /* asm code initialization */ + uInt longest_match OF((deflate_state *s, IPos cur_match)); +#else +local uInt longest_match OF((deflate_state *s, IPos cur_match)); +#endif + +#ifdef ZLIB_DEBUG +local void check_match OF((deflate_state *s, IPos start, IPos match, + int length)); +#endif + +/* From crc32.c */ +extern void ZLIB_INTERNAL crc_reset(deflate_state *const s); +extern void ZLIB_INTERNAL crc_finalize(deflate_state *const s); +extern void ZLIB_INTERNAL copy_with_crc(z_streamp strm, Bytef *dst, long size); + +/* =========================================================================== + * Local data + */ + +#define NIL 0 +/* Tail of hash chains */ + +#ifndef TOO_FAR +# define TOO_FAR 4096 +#endif +/* Matches of length 3 are discarded if their distance exceeds TOO_FAR */ + +/* Values for max_lazy_match, good_match and max_chain_length, depending on + * the desired pack level (0..9). The values given below have been tuned to + * exclude worst case performance for pathological files. Better values may be + * found for specific files. + */ +typedef struct config_s { + ush good_length; /* reduce lazy search above this match length */ + ush max_lazy; /* do not perform lazy search above this match length */ + ush nice_length; /* quit search above this match length */ + ush max_chain; + compress_func func; +} config; + +#ifdef FASTEST +local const config configuration_table[2] = { +/* good lazy nice chain */ +/* 0 */ {0, 0, 0, 0, deflate_stored}, /* store only */ +/* 1 */ {4, 4, 8, 4, deflate_fast}}; /* max speed, no lazy matches */ +#else +local const config configuration_table[10] = { +/* good lazy nice chain */ +/* 0 */ {0, 0, 0, 0, deflate_stored}, /* store only */ +/* 1 */ {4, 4, 8, 4, deflate_fast}, /* max speed, no lazy matches */ +/* 2 */ {4, 5, 16, 8, deflate_fast}, +/* 3 */ {4, 6, 32, 32, deflate_fast}, + +/* 4 */ {4, 4, 16, 16, deflate_slow}, /* lazy matches */ +/* 5 */ {8, 16, 32, 32, deflate_slow}, +/* 6 */ {8, 16, 128, 128, deflate_slow}, +/* 7 */ {8, 32, 128, 256, deflate_slow}, +/* 8 */ {32, 128, 258, 1024, deflate_slow}, +/* 9 */ {32, 258, 258, 4096, deflate_slow}}; /* max compression */ +#endif + +/* Note: the deflate() code requires max_lazy >= MIN_MATCH and max_chain >= 4 + * For deflate_fast() (levels <= 3) good is ignored and lazy has a different + * meaning. + */ + +/* rank Z_BLOCK between Z_NO_FLUSH and Z_PARTIAL_FLUSH */ +#define RANK(f) (((f) * 2) - ((f) > 4 ? 9 : 0)) + +/* =========================================================================== + * Initialize the hash table (avoiding 64K overflow for 16 bit systems). + * prev[] will be initialized on the fly. + */ +#define CLEAR_HASH(s) \ + s->head[s->hash_size-1] = NIL; \ + zmemzero((Bytef *)s->head, (unsigned)(s->hash_size-1)*sizeof(*s->head)); + +/* =========================================================================== + * Slide the hash table when sliding the window down (could be avoided with 32 + * bit values at the expense of memory usage). We slide even when level == 0 to + * keep the hash table consistent if we switch back to level > 0 later. + */ +local void slide_hash(s) + deflate_state *s; +{ +#if (defined(__ARM_NEON__) || defined(__ARM_NEON)) + /* NEON based hash table rebase. */ + return neon_slide_hash(s->head, s->prev, s->w_size, s->hash_size); +#endif + unsigned n, m; + Posf *p; + uInt wsize = s->w_size; + + n = s->hash_size; + p = &s->head[n]; + do { + m = *--p; + *p = (Pos)(m >= wsize ? m - wsize : NIL); + } while (--n); + n = wsize; +#ifndef FASTEST + p = &s->prev[n]; + do { + m = *--p; + *p = (Pos)(m >= wsize ? m - wsize : NIL); + /* If n is not on any hash chain, prev[n] is garbage but + * its value will never be used. + */ + } while (--n); +#endif +} + +/* ========================================================================= */ +int ZEXPORT deflateInit_(strm, level, version, stream_size) + z_streamp strm; + int level; + const char *version; + int stream_size; +{ + return deflateInit2_(strm, level, Z_DEFLATED, MAX_WBITS, DEF_MEM_LEVEL, + Z_DEFAULT_STRATEGY, version, stream_size); + /* To do: ignore strm->next_in if we use it as window */ +} + +/* ========================================================================= */ +int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, + version, stream_size) + z_streamp strm; + int level; + int method; + int windowBits; + int memLevel; + int strategy; + const char *version; + int stream_size; +{ + unsigned window_padding = 8; + deflate_state *s; + int wrap = 1; + static const char my_version[] = ZLIB_VERSION; + + // Needed to activate optimized insert_string() that helps compression + // for all wrapper formats (e.g. RAW, ZLIB, GZIP). + // Feature detection is not triggered while using RAW mode (i.e. we never + // call crc32() with a NULL buffer). +#if defined(CRC32_ARMV8_CRC32) || defined(CRC32_SIMD_SSE42_PCLMUL) + cpu_check_features(); +#endif + + if (version == Z_NULL || version[0] != my_version[0] || + stream_size != sizeof(z_stream)) { + return Z_VERSION_ERROR; + } + if (strm == Z_NULL) return Z_STREAM_ERROR; + + strm->msg = Z_NULL; + if (strm->zalloc == (alloc_func)0) { +#ifdef Z_SOLO + return Z_STREAM_ERROR; +#else + strm->zalloc = zcalloc; + strm->opaque = (voidpf)0; +#endif + } + if (strm->zfree == (free_func)0) +#ifdef Z_SOLO + return Z_STREAM_ERROR; +#else + strm->zfree = zcfree; +#endif + +#ifdef FASTEST + if (level != 0) level = 1; +#else + if (level == Z_DEFAULT_COMPRESSION) level = 6; +#endif + + if (windowBits < 0) { /* suppress zlib wrapper */ + wrap = 0; + windowBits = -windowBits; + } +#ifdef GZIP + else if (windowBits > 15) { + wrap = 2; /* write gzip wrapper instead */ + windowBits -= 16; + } +#endif + if (memLevel < 1 || memLevel > MAX_MEM_LEVEL || method != Z_DEFLATED || + windowBits < 8 || windowBits > 15 || level < 0 || level > 9 || + strategy < 0 || strategy > Z_FIXED || (windowBits == 8 && wrap != 1)) { + return Z_STREAM_ERROR; + } + if (windowBits == 8) windowBits = 9; /* until 256-byte window bug fixed */ + s = (deflate_state *) ZALLOC(strm, 1, sizeof(deflate_state)); + if (s == Z_NULL) return Z_MEM_ERROR; + strm->state = (struct internal_state FAR *)s; + s->strm = strm; + s->status = INIT_STATE; /* to pass state test in deflateReset() */ + + s->wrap = wrap; + s->gzhead = Z_NULL; + s->w_bits = (uInt)windowBits; + s->w_size = 1 << s->w_bits; + s->w_mask = s->w_size - 1; + + s->hash_bits = memLevel + 7; + if ((x86_cpu_enable_simd || arm_cpu_enable_crc32) && s->hash_bits < 15) { + s->hash_bits = 15; + } + + s->hash_size = 1 << s->hash_bits; + s->hash_mask = s->hash_size - 1; + s->hash_shift = ((s->hash_bits+MIN_MATCH-1)/MIN_MATCH); + + s->window = (Bytef *) ZALLOC(strm, + s->w_size + window_padding, + 2*sizeof(Byte)); + /* Avoid use of unitialized values in the window, see crbug.com/1137613 and + * crbug.com/1144420 */ + zmemzero(s->window, (s->w_size + window_padding) * (2 * sizeof(Byte))); + s->prev = (Posf *) ZALLOC(strm, s->w_size, sizeof(Pos)); + /* Avoid use of uninitialized value, see: + * https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=11360 + */ + zmemzero(s->prev, s->w_size * sizeof(Pos)); + s->head = (Posf *) ZALLOC(strm, s->hash_size, sizeof(Pos)); + + s->high_water = 0; /* nothing written to s->window yet */ + + s->lit_bufsize = 1 << (memLevel + 6); /* 16K elements by default */ + + /* We overlay pending_buf and sym_buf. This works since the average size + * for length/distance pairs over any compressed block is assured to be 31 + * bits or less. + * + * Analysis: The longest fixed codes are a length code of 8 bits plus 5 + * extra bits, for lengths 131 to 257. The longest fixed distance codes are + * 5 bits plus 13 extra bits, for distances 16385 to 32768. The longest + * possible fixed-codes length/distance pair is then 31 bits total. + * + * sym_buf starts one-fourth of the way into pending_buf. So there are + * three bytes in sym_buf for every four bytes in pending_buf. Each symbol + * in sym_buf is three bytes -- two for the distance and one for the + * literal/length. As each symbol is consumed, the pointer to the next + * sym_buf value to read moves forward three bytes. From that symbol, up to + * 31 bits are written to pending_buf. The closest the written pending_buf + * bits gets to the next sym_buf symbol to read is just before the last + * code is written. At that time, 31*(n-2) bits have been written, just + * after 24*(n-2) bits have been consumed from sym_buf. sym_buf starts at + * 8*n bits into pending_buf. (Note that the symbol buffer fills when n-1 + * symbols are written.) The closest the writing gets to what is unread is + * then n+14 bits. Here n is lit_bufsize, which is 16384 by default, and + * can range from 128 to 32768. + * + * Therefore, at a minimum, there are 142 bits of space between what is + * written and what is read in the overlain buffers, so the symbols cannot + * be overwritten by the compressed data. That space is actually 139 bits, + * due to the three-bit fixed-code block header. + * + * That covers the case where either Z_FIXED is specified, forcing fixed + * codes, or when the use of fixed codes is chosen, because that choice + * results in a smaller compressed block than dynamic codes. That latter + * condition then assures that the above analysis also covers all dynamic + * blocks. A dynamic-code block will only be chosen to be emitted if it has + * fewer bits than a fixed-code block would for the same set of symbols. + * Therefore its average symbol length is assured to be less than 31. So + * the compressed data for a dynamic block also cannot overwrite the + * symbols from which it is being constructed. + */ + s->pending_buf = (uchf *) ZALLOC(strm, s->lit_bufsize, 4); + s->pending_buf_size = (ulg)s->lit_bufsize * 4; + + if (s->window == Z_NULL || s->prev == Z_NULL || s->head == Z_NULL || + s->pending_buf == Z_NULL) { + s->status = FINISH_STATE; + strm->msg = ERR_MSG(Z_MEM_ERROR); + deflateEnd (strm); + return Z_MEM_ERROR; + } + s->sym_buf = s->pending_buf + s->lit_bufsize; + s->sym_end = (s->lit_bufsize - 1) * 3; + /* We avoid equality with lit_bufsize*3 because of wraparound at 64K + * on 16 bit machines and because stored blocks are restricted to + * 64K-1 bytes. + */ + + s->level = level; + s->strategy = strategy; + s->method = (Byte)method; + + return deflateReset(strm); +} + +/* ========================================================================= + * Check for a valid deflate stream state. Return 0 if ok, 1 if not. + */ +local int deflateStateCheck (strm) + z_streamp strm; +{ + deflate_state *s; + if (strm == Z_NULL || + strm->zalloc == (alloc_func)0 || strm->zfree == (free_func)0) + return 1; + s = strm->state; + if (s == Z_NULL || s->strm != strm || (s->status != INIT_STATE && +#ifdef GZIP + s->status != GZIP_STATE && +#endif + s->status != EXTRA_STATE && + s->status != NAME_STATE && + s->status != COMMENT_STATE && + s->status != HCRC_STATE && + s->status != BUSY_STATE && + s->status != FINISH_STATE)) + return 1; + return 0; +} + +/* ========================================================================= */ +int ZEXPORT deflateSetDictionary (strm, dictionary, dictLength) + z_streamp strm; + const Bytef *dictionary; + uInt dictLength; +{ + deflate_state *s; + uInt str, n; + int wrap; + unsigned avail; + z_const unsigned char *next; + + if (deflateStateCheck(strm) || dictionary == Z_NULL) + return Z_STREAM_ERROR; + s = strm->state; + wrap = s->wrap; + if (wrap == 2 || (wrap == 1 && s->status != INIT_STATE) || s->lookahead) + return Z_STREAM_ERROR; + + /* when using zlib wrappers, compute Adler-32 for provided dictionary */ + if (wrap == 1) + strm->adler = adler32(strm->adler, dictionary, dictLength); + s->wrap = 0; /* avoid computing Adler-32 in deflate_read_buf */ + + /* if dictionary would fill window, just replace the history */ + if (dictLength >= s->w_size) { + if (wrap == 0) { /* already empty otherwise */ + CLEAR_HASH(s); + s->strstart = 0; + s->block_start = 0L; + s->insert = 0; + } + dictionary += dictLength - s->w_size; /* use the tail */ + dictLength = s->w_size; + } + + /* insert dictionary into window and hash */ + avail = strm->avail_in; + next = strm->next_in; + strm->avail_in = dictLength; + strm->next_in = (z_const Bytef *)dictionary; + fill_window(s); + while (s->lookahead >= MIN_MATCH) { + str = s->strstart; + n = s->lookahead - (MIN_MATCH-1); + do { + insert_string(s, str); + str++; + } while (--n); + s->strstart = str; + s->lookahead = MIN_MATCH-1; + fill_window(s); + } + s->strstart += s->lookahead; + s->block_start = (long)s->strstart; + s->insert = s->lookahead; + s->lookahead = 0; + s->match_length = s->prev_length = MIN_MATCH-1; + s->match_available = 0; + strm->next_in = next; + strm->avail_in = avail; + s->wrap = wrap; + return Z_OK; +} + +/* ========================================================================= */ +int ZEXPORT deflateGetDictionary (strm, dictionary, dictLength) + z_streamp strm; + Bytef *dictionary; + uInt *dictLength; +{ + deflate_state *s; + uInt len; + + if (deflateStateCheck(strm)) + return Z_STREAM_ERROR; + s = strm->state; + len = s->strstart + s->lookahead; + if (len > s->w_size) + len = s->w_size; + if (dictionary != Z_NULL && len) + zmemcpy(dictionary, s->window + s->strstart + s->lookahead - len, len); + if (dictLength != Z_NULL) + *dictLength = len; + return Z_OK; +} + +/* ========================================================================= */ +int ZEXPORT deflateResetKeep (strm) + z_streamp strm; +{ + deflate_state *s; + + if (deflateStateCheck(strm)) { + return Z_STREAM_ERROR; + } + + strm->total_in = strm->total_out = 0; + strm->msg = Z_NULL; /* use zfree if we ever allocate msg dynamically */ + strm->data_type = Z_UNKNOWN; + + s = (deflate_state *)strm->state; + s->pending = 0; + s->pending_out = s->pending_buf; + + if (s->wrap < 0) { + s->wrap = -s->wrap; /* was made negative by deflate(..., Z_FINISH); */ + } + s->status = +#ifdef GZIP + s->wrap == 2 ? GZIP_STATE : +#endif + s->wrap ? INIT_STATE : BUSY_STATE; + strm->adler = +#ifdef GZIP + s->wrap == 2 ? crc32(0L, Z_NULL, 0) : +#endif + adler32(0L, Z_NULL, 0); + s->last_flush = Z_NO_FLUSH; + + _tr_init(s); + + return Z_OK; +} + +/* ========================================================================= */ +int ZEXPORT deflateReset (strm) + z_streamp strm; +{ + int ret; + + ret = deflateResetKeep(strm); + if (ret == Z_OK) + lm_init(strm->state); + return ret; +} + +/* ========================================================================= */ +int ZEXPORT deflateSetHeader (strm, head) + z_streamp strm; + gz_headerp head; +{ + if (deflateStateCheck(strm) || strm->state->wrap != 2) + return Z_STREAM_ERROR; + strm->state->gzhead = head; + return Z_OK; +} + +/* ========================================================================= */ +int ZEXPORT deflatePending (strm, pending, bits) + unsigned *pending; + int *bits; + z_streamp strm; +{ + if (deflateStateCheck(strm)) return Z_STREAM_ERROR; + if (pending != Z_NULL) + *pending = strm->state->pending; + if (bits != Z_NULL) + *bits = strm->state->bi_valid; + return Z_OK; +} + +/* ========================================================================= */ +int ZEXPORT deflatePrime (strm, bits, value) + z_streamp strm; + int bits; + int value; +{ + deflate_state *s; + int put; + + if (deflateStateCheck(strm)) return Z_STREAM_ERROR; + s = strm->state; + if (s->sym_buf < s->pending_out + ((Buf_size + 7) >> 3)) + return Z_BUF_ERROR; + do { + put = Buf_size - s->bi_valid; + if (put > bits) + put = bits; + s->bi_buf |= (ush)((value & ((1 << put) - 1)) << s->bi_valid); + s->bi_valid += put; + _tr_flush_bits(s); + value >>= put; + bits -= put; + } while (bits); + return Z_OK; +} + +/* ========================================================================= */ +int ZEXPORT deflateParams(strm, level, strategy) + z_streamp strm; + int level; + int strategy; +{ + deflate_state *s; + compress_func func; + + if (deflateStateCheck(strm)) return Z_STREAM_ERROR; + s = strm->state; + +#ifdef FASTEST + if (level != 0) level = 1; +#else + if (level == Z_DEFAULT_COMPRESSION) level = 6; +#endif + if (level < 0 || level > 9 || strategy < 0 || strategy > Z_FIXED) { + return Z_STREAM_ERROR; + } + func = configuration_table[s->level].func; + + if ((strategy != s->strategy || func != configuration_table[level].func) && + s->high_water) { + /* Flush the last buffer: */ + int err = deflate(strm, Z_BLOCK); + if (err == Z_STREAM_ERROR) + return err; + if (strm->avail_out == 0) + return Z_BUF_ERROR; + } + if (s->level != level) { + if (s->level == 0 && s->matches != 0) { + if (s->matches == 1) + slide_hash(s); + else + CLEAR_HASH(s); + s->matches = 0; + } + s->level = level; + s->max_lazy_match = configuration_table[level].max_lazy; + s->good_match = configuration_table[level].good_length; + s->nice_match = configuration_table[level].nice_length; + s->max_chain_length = configuration_table[level].max_chain; + } + s->strategy = strategy; + return Z_OK; +} + +/* ========================================================================= */ +int ZEXPORT deflateTune(strm, good_length, max_lazy, nice_length, max_chain) + z_streamp strm; + int good_length; + int max_lazy; + int nice_length; + int max_chain; +{ + deflate_state *s; + + if (deflateStateCheck(strm)) return Z_STREAM_ERROR; + s = strm->state; + s->good_match = (uInt)good_length; + s->max_lazy_match = (uInt)max_lazy; + s->nice_match = nice_length; + s->max_chain_length = (uInt)max_chain; + return Z_OK; +} + +/* ========================================================================= + * For the default windowBits of 15 and memLevel of 8, this function returns + * a close to exact, as well as small, upper bound on the compressed size. + * They are coded as constants here for a reason--if the #define's are + * changed, then this function needs to be changed as well. The return + * value for 15 and 8 only works for those exact settings. + * + * For any setting other than those defaults for windowBits and memLevel, + * the value returned is a conservative worst case for the maximum expansion + * resulting from using fixed blocks instead of stored blocks, which deflate + * can emit on compressed data for some combinations of the parameters. + * + * This function could be more sophisticated to provide closer upper bounds for + * every combination of windowBits and memLevel. But even the conservative + * upper bound of about 14% expansion does not seem onerous for output buffer + * allocation. + */ +uLong ZEXPORT deflateBound(strm, sourceLen) + z_streamp strm; + uLong sourceLen; +{ + deflate_state *s; + uLong complen, wraplen; + + /* conservative upper bound for compressed data */ + complen = sourceLen + + ((sourceLen + 7) >> 3) + ((sourceLen + 63) >> 6) + 5; + + /* if can't get parameters, return conservative bound plus zlib wrapper */ + if (deflateStateCheck(strm)) + return complen + 6; + + /* compute wrapper length */ + s = strm->state; + switch (s->wrap) { + case 0: /* raw deflate */ + wraplen = 0; + break; + case 1: /* zlib wrapper */ + wraplen = 6 + (s->strstart ? 4 : 0); + break; +#ifdef GZIP + case 2: /* gzip wrapper */ + wraplen = 18; + if (s->gzhead != Z_NULL) { /* user-supplied gzip header */ + Bytef *str; + if (s->gzhead->extra != Z_NULL) + wraplen += 2 + s->gzhead->extra_len; + str = s->gzhead->name; + if (str != Z_NULL) + do { + wraplen++; + } while (*str++); + str = s->gzhead->comment; + if (str != Z_NULL) + do { + wraplen++; + } while (*str++); + if (s->gzhead->hcrc) + wraplen += 2; + } + break; +#endif + default: /* for compiler happiness */ + wraplen = 6; + } + + /* if not default parameters, return conservative bound */ + if (s->w_bits != 15 || s->hash_bits != 8 + 7) + return complen + wraplen; + + /* default settings: return tight bound for that case */ + return sourceLen + (sourceLen >> 12) + (sourceLen >> 14) + + (sourceLen >> 25) + 13 - 6 + wraplen; +} + +/* ========================================================================= + * Put a short in the pending buffer. The 16-bit value is put in MSB order. + * IN assertion: the stream state is correct and there is enough room in + * pending_buf. + */ +local void putShortMSB (s, b) + deflate_state *s; + uInt b; +{ + put_byte(s, (Byte)(b >> 8)); + put_byte(s, (Byte)(b & 0xff)); +} + +/* ========================================================================= + * Flush as much pending output as possible. All deflate() output, except for + * some deflate_stored() output, goes through this function so some + * applications may wish to modify it to avoid allocating a large + * strm->next_out buffer and copying into it. (See also deflate_read_buf()). + */ +local void flush_pending(strm) + z_streamp strm; +{ + unsigned len; + deflate_state *s = strm->state; + + _tr_flush_bits(s); + len = s->pending; + if (len > strm->avail_out) len = strm->avail_out; + if (len == 0) return; + + zmemcpy(strm->next_out, s->pending_out, len); + strm->next_out += len; + s->pending_out += len; + strm->total_out += len; + strm->avail_out -= len; + s->pending -= len; + if (s->pending == 0) { + s->pending_out = s->pending_buf; + } +} + +/* =========================================================================== + * Update the header CRC with the bytes s->pending_buf[beg..s->pending - 1]. + */ +#define HCRC_UPDATE(beg) \ + do { \ + if (s->gzhead->hcrc && s->pending > (beg)) \ + strm->adler = crc32(strm->adler, s->pending_buf + (beg), \ + s->pending - (beg)); \ + } while (0) + +/* ========================================================================= */ +int ZEXPORT deflate (strm, flush) + z_streamp strm; + int flush; +{ + int old_flush; /* value of flush param for previous deflate call */ + deflate_state *s; + + if (deflateStateCheck(strm) || flush > Z_BLOCK || flush < 0) { + return Z_STREAM_ERROR; + } + s = strm->state; + + if (strm->next_out == Z_NULL || + (strm->avail_in != 0 && strm->next_in == Z_NULL) || + (s->status == FINISH_STATE && flush != Z_FINISH)) { + ERR_RETURN(strm, Z_STREAM_ERROR); + } + if (strm->avail_out == 0) ERR_RETURN(strm, Z_BUF_ERROR); + + old_flush = s->last_flush; + s->last_flush = flush; + + /* Flush as much pending output as possible */ + if (s->pending != 0) { + flush_pending(strm); + if (strm->avail_out == 0) { + /* Since avail_out is 0, deflate will be called again with + * more output space, but possibly with both pending and + * avail_in equal to zero. There won't be anything to do, + * but this is not an error situation so make sure we + * return OK instead of BUF_ERROR at next call of deflate: + */ + s->last_flush = -1; + return Z_OK; + } + + /* Make sure there is something to do and avoid duplicate consecutive + * flushes. For repeated and useless calls with Z_FINISH, we keep + * returning Z_STREAM_END instead of Z_BUF_ERROR. + */ + } else if (strm->avail_in == 0 && RANK(flush) <= RANK(old_flush) && + flush != Z_FINISH) { + ERR_RETURN(strm, Z_BUF_ERROR); + } + + /* User must not provide more input after the first FINISH: */ + if (s->status == FINISH_STATE && strm->avail_in != 0) { + ERR_RETURN(strm, Z_BUF_ERROR); + } + + /* Write the header */ + if (s->status == INIT_STATE) { + /* zlib header */ + uInt header = (Z_DEFLATED + ((s->w_bits-8)<<4)) << 8; + uInt level_flags; + + if (s->strategy >= Z_HUFFMAN_ONLY || s->level < 2) + level_flags = 0; + else if (s->level < 6) + level_flags = 1; + else if (s->level == 6) + level_flags = 2; + else + level_flags = 3; + header |= (level_flags << 6); + if (s->strstart != 0) header |= PRESET_DICT; + header += 31 - (header % 31); + + putShortMSB(s, header); + + /* Save the adler32 of the preset dictionary: */ + if (s->strstart != 0) { + putShortMSB(s, (uInt)(strm->adler >> 16)); + putShortMSB(s, (uInt)(strm->adler & 0xffff)); + } + strm->adler = adler32(0L, Z_NULL, 0); + s->status = BUSY_STATE; + + /* Compression must start with an empty pending buffer */ + flush_pending(strm); + if (s->pending != 0) { + s->last_flush = -1; + return Z_OK; + } + } +#ifdef GZIP + if (s->status == GZIP_STATE) { + /* gzip header */ + crc_reset(s); + put_byte(s, 31); + put_byte(s, 139); + put_byte(s, 8); + if (s->gzhead == Z_NULL) { + put_byte(s, 0); + put_byte(s, 0); + put_byte(s, 0); + put_byte(s, 0); + put_byte(s, 0); + put_byte(s, s->level == 9 ? 2 : + (s->strategy >= Z_HUFFMAN_ONLY || s->level < 2 ? + 4 : 0)); + put_byte(s, OS_CODE); + s->status = BUSY_STATE; + + /* Compression must start with an empty pending buffer */ + flush_pending(strm); + if (s->pending != 0) { + s->last_flush = -1; + return Z_OK; + } + } + else { + put_byte(s, (s->gzhead->text ? 1 : 0) + + (s->gzhead->hcrc ? 2 : 0) + + (s->gzhead->extra == Z_NULL ? 0 : 4) + + (s->gzhead->name == Z_NULL ? 0 : 8) + + (s->gzhead->comment == Z_NULL ? 0 : 16) + ); + put_byte(s, (Byte)(s->gzhead->time & 0xff)); + put_byte(s, (Byte)((s->gzhead->time >> 8) & 0xff)); + put_byte(s, (Byte)((s->gzhead->time >> 16) & 0xff)); + put_byte(s, (Byte)((s->gzhead->time >> 24) & 0xff)); + put_byte(s, s->level == 9 ? 2 : + (s->strategy >= Z_HUFFMAN_ONLY || s->level < 2 ? + 4 : 0)); + put_byte(s, s->gzhead->os & 0xff); + if (s->gzhead->extra != Z_NULL) { + put_byte(s, s->gzhead->extra_len & 0xff); + put_byte(s, (s->gzhead->extra_len >> 8) & 0xff); + } + if (s->gzhead->hcrc) + strm->adler = crc32(strm->adler, s->pending_buf, + s->pending); + s->gzindex = 0; + s->status = EXTRA_STATE; + } + } + if (s->status == EXTRA_STATE) { + if (s->gzhead->extra != Z_NULL) { + ulg beg = s->pending; /* start of bytes to update crc */ + uInt left = (s->gzhead->extra_len & 0xffff) - s->gzindex; + while (s->pending + left > s->pending_buf_size) { + uInt copy = s->pending_buf_size - s->pending; + zmemcpy(s->pending_buf + s->pending, + s->gzhead->extra + s->gzindex, copy); + s->pending = s->pending_buf_size; + HCRC_UPDATE(beg); + s->gzindex += copy; + flush_pending(strm); + if (s->pending != 0) { + s->last_flush = -1; + return Z_OK; + } + beg = 0; + left -= copy; + } + zmemcpy(s->pending_buf + s->pending, + s->gzhead->extra + s->gzindex, left); + s->pending += left; + HCRC_UPDATE(beg); + s->gzindex = 0; + } + s->status = NAME_STATE; + } + if (s->status == NAME_STATE) { + if (s->gzhead->name != Z_NULL) { + ulg beg = s->pending; /* start of bytes to update crc */ + int val; + do { + if (s->pending == s->pending_buf_size) { + HCRC_UPDATE(beg); + flush_pending(strm); + if (s->pending != 0) { + s->last_flush = -1; + return Z_OK; + } + beg = 0; + } + val = s->gzhead->name[s->gzindex++]; + put_byte(s, val); + } while (val != 0); + HCRC_UPDATE(beg); + s->gzindex = 0; + } + s->status = COMMENT_STATE; + } + if (s->status == COMMENT_STATE) { + if (s->gzhead->comment != Z_NULL) { + ulg beg = s->pending; /* start of bytes to update crc */ + int val; + do { + if (s->pending == s->pending_buf_size) { + HCRC_UPDATE(beg); + flush_pending(strm); + if (s->pending != 0) { + s->last_flush = -1; + return Z_OK; + } + beg = 0; + } + val = s->gzhead->comment[s->gzindex++]; + put_byte(s, val); + } while (val != 0); + HCRC_UPDATE(beg); + } + s->status = HCRC_STATE; + } + if (s->status == HCRC_STATE) { + if (s->gzhead->hcrc) { + if (s->pending + 2 > s->pending_buf_size) { + flush_pending(strm); + if (s->pending != 0) { + s->last_flush = -1; + return Z_OK; + } + } + put_byte(s, (Byte)(strm->adler & 0xff)); + put_byte(s, (Byte)((strm->adler >> 8) & 0xff)); + strm->adler = crc32(0L, Z_NULL, 0); + } + s->status = BUSY_STATE; + + /* Compression must start with an empty pending buffer */ + flush_pending(strm); + if (s->pending != 0) { + s->last_flush = -1; + return Z_OK; + } + } +#endif + + /* Start a new block or continue the current one. + */ + if (strm->avail_in != 0 || s->lookahead != 0 || + (flush != Z_NO_FLUSH && s->status != FINISH_STATE)) { + block_state bstate; + + bstate = s->level == 0 ? deflate_stored(s, flush) : + s->strategy == Z_HUFFMAN_ONLY ? deflate_huff(s, flush) : + s->strategy == Z_RLE ? deflate_rle(s, flush) : + (*(configuration_table[s->level].func))(s, flush); + + if (bstate == finish_started || bstate == finish_done) { + s->status = FINISH_STATE; + } + if (bstate == need_more || bstate == finish_started) { + if (strm->avail_out == 0) { + s->last_flush = -1; /* avoid BUF_ERROR next call, see above */ + } + return Z_OK; + /* If flush != Z_NO_FLUSH && avail_out == 0, the next call + * of deflate should use the same flush parameter to make sure + * that the flush is complete. So we don't have to output an + * empty block here, this will be done at next call. This also + * ensures that for a very small output buffer, we emit at most + * one empty block. + */ + } + if (bstate == block_done) { + if (flush == Z_PARTIAL_FLUSH) { + _tr_align(s); + } else if (flush != Z_BLOCK) { /* FULL_FLUSH or SYNC_FLUSH */ + _tr_stored_block(s, (char*)0, 0L, 0); + /* For a full flush, this empty block will be recognized + * as a special marker by inflate_sync(). + */ + if (flush == Z_FULL_FLUSH) { + CLEAR_HASH(s); /* forget history */ + if (s->lookahead == 0) { + s->strstart = 0; + s->block_start = 0L; + s->insert = 0; + } + } + } + flush_pending(strm); + if (strm->avail_out == 0) { + s->last_flush = -1; /* avoid BUF_ERROR at next call, see above */ + return Z_OK; + } + } + } + + if (flush != Z_FINISH) return Z_OK; + if (s->wrap <= 0) return Z_STREAM_END; + + /* Write the trailer */ +#ifdef GZIP + if (s->wrap == 2) { + crc_finalize(s); + put_byte(s, (Byte)(strm->adler & 0xff)); + put_byte(s, (Byte)((strm->adler >> 8) & 0xff)); + put_byte(s, (Byte)((strm->adler >> 16) & 0xff)); + put_byte(s, (Byte)((strm->adler >> 24) & 0xff)); + put_byte(s, (Byte)(strm->total_in & 0xff)); + put_byte(s, (Byte)((strm->total_in >> 8) & 0xff)); + put_byte(s, (Byte)((strm->total_in >> 16) & 0xff)); + put_byte(s, (Byte)((strm->total_in >> 24) & 0xff)); + } + else +#endif + { + putShortMSB(s, (uInt)(strm->adler >> 16)); + putShortMSB(s, (uInt)(strm->adler & 0xffff)); + } + flush_pending(strm); + /* If avail_out is zero, the application will call deflate again + * to flush the rest. + */ + if (s->wrap > 0) s->wrap = -s->wrap; /* write the trailer only once! */ + return s->pending != 0 ? Z_OK : Z_STREAM_END; +} + +/* ========================================================================= */ +int ZEXPORT deflateEnd (strm) + z_streamp strm; +{ + int status; + + if (deflateStateCheck(strm)) return Z_STREAM_ERROR; + + status = strm->state->status; + + /* Deallocate in reverse order of allocations: */ + TRY_FREE(strm, strm->state->pending_buf); + TRY_FREE(strm, strm->state->head); + TRY_FREE(strm, strm->state->prev); + TRY_FREE(strm, strm->state->window); + + ZFREE(strm, strm->state); + strm->state = Z_NULL; + + return status == BUSY_STATE ? Z_DATA_ERROR : Z_OK; +} + +/* ========================================================================= + * Copy the source state to the destination state. + * To simplify the source, this is not supported for 16-bit MSDOS (which + * doesn't have enough memory anyway to duplicate compression states). + */ +int ZEXPORT deflateCopy (dest, source) + z_streamp dest; + z_streamp source; +{ +#ifdef MAXSEG_64K + return Z_STREAM_ERROR; +#else + deflate_state *ds; + deflate_state *ss; + + + if (deflateStateCheck(source) || dest == Z_NULL) { + return Z_STREAM_ERROR; + } + + ss = source->state; + + zmemcpy((voidpf)dest, (voidpf)source, sizeof(z_stream)); + + ds = (deflate_state *) ZALLOC(dest, 1, sizeof(deflate_state)); + if (ds == Z_NULL) return Z_MEM_ERROR; + dest->state = (struct internal_state FAR *) ds; + zmemcpy((voidpf)ds, (voidpf)ss, sizeof(deflate_state)); + ds->strm = dest; + + ds->window = (Bytef *) ZALLOC(dest, ds->w_size, 2*sizeof(Byte)); + ds->prev = (Posf *) ZALLOC(dest, ds->w_size, sizeof(Pos)); + ds->head = (Posf *) ZALLOC(dest, ds->hash_size, sizeof(Pos)); + ds->pending_buf = (uchf *) ZALLOC(dest, ds->lit_bufsize, 4); + + if (ds->window == Z_NULL || ds->prev == Z_NULL || ds->head == Z_NULL || + ds->pending_buf == Z_NULL) { + deflateEnd (dest); + return Z_MEM_ERROR; + } + /* following zmemcpy do not work for 16-bit MSDOS */ + zmemcpy(ds->window, ss->window, ds->w_size * 2 * sizeof(Byte)); + zmemcpy((voidpf)ds->prev, (voidpf)ss->prev, ds->w_size * sizeof(Pos)); + zmemcpy((voidpf)ds->head, (voidpf)ss->head, ds->hash_size * sizeof(Pos)); + zmemcpy(ds->pending_buf, ss->pending_buf, (uInt)ds->pending_buf_size); + + ds->pending_out = ds->pending_buf + (ss->pending_out - ss->pending_buf); + ds->sym_buf = ds->pending_buf + ds->lit_bufsize; + + ds->l_desc.dyn_tree = ds->dyn_ltree; + ds->d_desc.dyn_tree = ds->dyn_dtree; + ds->bl_desc.dyn_tree = ds->bl_tree; + + return Z_OK; +#endif /* MAXSEG_64K */ +} + +/* =========================================================================== + * Read a new buffer from the current input stream, update the adler32 + * and total number of bytes read. All deflate() input goes through + * this function so some applications may wish to modify it to avoid + * allocating a large strm->next_in buffer and copying from it. + * (See also flush_pending()). + */ +ZLIB_INTERNAL unsigned deflate_read_buf(strm, buf, size) + z_streamp strm; + Bytef *buf; + unsigned size; +{ + unsigned len = strm->avail_in; + + if (len > size) len = size; + if (len == 0) return 0; + + strm->avail_in -= len; + +#ifdef GZIP + if (strm->state->wrap == 2) + copy_with_crc(strm, buf, len); + else +#endif + { + zmemcpy(buf, strm->next_in, len); + if (strm->state->wrap == 1) + strm->adler = adler32(strm->adler, buf, len); + } + strm->next_in += len; + strm->total_in += len; + + return len; +} + +/* =========================================================================== + * Initialize the "longest match" routines for a new zlib stream + */ +local void lm_init (s) + deflate_state *s; +{ + s->window_size = (ulg)2L*s->w_size; + + CLEAR_HASH(s); + + /* Set the default configuration parameters: + */ + s->max_lazy_match = configuration_table[s->level].max_lazy; + s->good_match = configuration_table[s->level].good_length; + s->nice_match = configuration_table[s->level].nice_length; + s->max_chain_length = configuration_table[s->level].max_chain; + + s->strstart = 0; + s->block_start = 0L; + s->lookahead = 0; + s->insert = 0; + s->match_length = s->prev_length = MIN_MATCH-1; + s->match_available = 0; + s->ins_h = 0; +#ifndef FASTEST +#ifdef ASMV + match_init(); /* initialize the asm code */ +#endif +#endif +} + +#ifndef FASTEST +/* =========================================================================== + * Set match_start to the longest match starting at the given string and + * return its length. Matches shorter or equal to prev_length are discarded, + * in which case the result is equal to prev_length and match_start is + * garbage. + * IN assertions: cur_match is the head of the hash chain for the current + * string (strstart) and its distance is <= MAX_DIST, and prev_length >= 1 + * OUT assertion: the match length is not greater than s->lookahead. + */ +#ifndef ASMV +/* For 80x86 and 680x0, an optimized version will be provided in match.asm or + * match.S. The code will be functionally equivalent. + */ +local uInt longest_match(s, cur_match) + deflate_state *s; + IPos cur_match; /* current match */ +{ + unsigned chain_length = s->max_chain_length;/* max hash chain length */ + register Bytef *scan = s->window + s->strstart; /* current string */ + register Bytef *match; /* matched string */ + register int len; /* length of current match */ + int best_len = (int)s->prev_length; /* best match length so far */ + int nice_match = s->nice_match; /* stop if match long enough */ + IPos limit = s->strstart > (IPos)MAX_DIST(s) ? + s->strstart - (IPos)MAX_DIST(s) : NIL; + /* Stop when cur_match becomes <= limit. To simplify the code, + * we prevent matches with the string of window index 0. + */ + Posf *prev = s->prev; + uInt wmask = s->w_mask; + +#ifdef UNALIGNED_OK + /* Compare two bytes at a time. Note: this is not always beneficial. + * Try with and without -DUNALIGNED_OK to check. + */ + register Bytef *strend = s->window + s->strstart + MAX_MATCH - 1; + register ush scan_start = *(ushf*)scan; + register ush scan_end = *(ushf*)(scan+best_len-1); +#else + register Bytef *strend = s->window + s->strstart + MAX_MATCH; + register Byte scan_end1 = scan[best_len-1]; + register Byte scan_end = scan[best_len]; +#endif + + /* The code is optimized for HASH_BITS >= 8 and MAX_MATCH-2 multiple of 16. + * It is easy to get rid of this optimization if necessary. + */ + Assert(s->hash_bits >= 8 && MAX_MATCH == 258, "Code too clever"); + + /* Do not waste too much time if we already have a good match: */ + if (s->prev_length >= s->good_match) { + chain_length >>= 2; + } + /* Do not look for matches beyond the end of the input. This is necessary + * to make deflate deterministic. + */ + if ((uInt)nice_match > s->lookahead) nice_match = (int)s->lookahead; + + Assert((ulg)s->strstart <= s->window_size-MIN_LOOKAHEAD, "need lookahead"); + + do { + Assert(cur_match < s->strstart, "no future"); + match = s->window + cur_match; + + /* Skip to next match if the match length cannot increase + * or if the match length is less than 2. Note that the checks below + * for insufficient lookahead only occur occasionally for performance + * reasons. Therefore uninitialized memory will be accessed, and + * conditional jumps will be made that depend on those values. + * However the length of the match is limited to the lookahead, so + * the output of deflate is not affected by the uninitialized values. + */ +#if (defined(UNALIGNED_OK) && MAX_MATCH == 258) + /* This code assumes sizeof(unsigned short) == 2. Do not use + * UNALIGNED_OK if your compiler uses a different size. + */ + if (*(ushf*)(match+best_len-1) != scan_end || + *(ushf*)match != scan_start) continue; + + /* It is not necessary to compare scan[2] and match[2] since they are + * always equal when the other bytes match, given that the hash keys + * are equal and that HASH_BITS >= 8. Compare 2 bytes at a time at + * strstart+3, +5, ... up to strstart+257. We check for insufficient + * lookahead only every 4th comparison; the 128th check will be made + * at strstart+257. If MAX_MATCH-2 is not a multiple of 8, it is + * necessary to put more guard bytes at the end of the window, or + * to check more often for insufficient lookahead. + */ + if (!x86_cpu_enable_simd && !arm_cpu_enable_crc32) { + Assert(scan[2] == match[2], "scan[2]?"); + } else { + /* When using CRC hashing, scan[2] and match[2] may mismatch, but in + * that case at least one of the other hashed bytes will mismatch + * also. Bytes 0 and 1 were already checked above, and we know there + * are at least four bytes to check otherwise the mismatch would have + * been found by the scan_end comparison above, so: */ + Assert(scan[2] == match[2] || scan[3] != match[3], "scan[2]??"); + } + scan++, match++; + do { + } while (*(ushf*)(scan+=2) == *(ushf*)(match+=2) && + *(ushf*)(scan+=2) == *(ushf*)(match+=2) && + *(ushf*)(scan+=2) == *(ushf*)(match+=2) && + *(ushf*)(scan+=2) == *(ushf*)(match+=2) && + scan < strend); + /* The funny "do {}" generates better code on most compilers */ + + /* Here, scan <= window+strstart+257 */ + Assert(scan <= s->window+(unsigned)(s->window_size-1), "wild scan"); + if (*scan == *match) scan++; + + len = (MAX_MATCH - 1) - (int)(strend-scan); + scan = strend - (MAX_MATCH-1); + +#else /* UNALIGNED_OK */ + + if (match[best_len] != scan_end || + match[best_len-1] != scan_end1 || + *match != *scan || + *++match != scan[1]) continue; + + /* The check at best_len-1 can be removed because it will be made + * again later. (This heuristic is not always a win.) + * It is not necessary to compare scan[2] and match[2] since they + * are always equal when the other bytes match, given that + * the hash keys are equal and that HASH_BITS >= 8. + */ + scan += 2, match++; + if (!x86_cpu_enable_simd && !arm_cpu_enable_crc32) { + Assert(*scan == *match, "match[2]?"); + } else { + /* When using CRC hashing, scan[2] and match[2] may mismatch, but in + * that case at least one of the other hashed bytes will mismatch + * also. Bytes 0 and 1 were already checked above, and we know there + * are at least four bytes to check otherwise the mismatch would have + * been found by the scan_end comparison above, so: */ + Assert(*scan == *match || scan[1] != match[1], "match[2]??"); + } + + /* We check for insufficient lookahead only every 8th comparison; + * the 256th check will be made at strstart+258. + */ + do { + } while (*++scan == *++match && *++scan == *++match && + *++scan == *++match && *++scan == *++match && + *++scan == *++match && *++scan == *++match && + *++scan == *++match && *++scan == *++match && + scan < strend); + + Assert(scan <= s->window+(unsigned)(s->window_size-1), "wild scan"); + + len = MAX_MATCH - (int)(strend - scan); + scan = strend - MAX_MATCH; + +#endif /* UNALIGNED_OK */ + + if (len > best_len) { + s->match_start = cur_match; + best_len = len; + if (len >= nice_match) break; +#ifdef UNALIGNED_OK + scan_end = *(ushf*)(scan+best_len-1); +#else + scan_end1 = scan[best_len-1]; + scan_end = scan[best_len]; +#endif + } + } while ((cur_match = prev[cur_match & wmask]) > limit + && --chain_length != 0); + + if ((uInt)best_len <= s->lookahead) return (uInt)best_len; + return s->lookahead; +} +#endif /* ASMV */ + +#else /* FASTEST */ + +/* --------------------------------------------------------------------------- + * Optimized version for FASTEST only + */ +local uInt longest_match(s, cur_match) + deflate_state *s; + IPos cur_match; /* current match */ +{ + register Bytef *scan = s->window + s->strstart; /* current string */ + register Bytef *match; /* matched string */ + register int len; /* length of current match */ + register Bytef *strend = s->window + s->strstart + MAX_MATCH; + + /* The code is optimized for HASH_BITS >= 8 and MAX_MATCH-2 multiple of 16. + * It is easy to get rid of this optimization if necessary. + */ + Assert(s->hash_bits >= 8 && MAX_MATCH == 258, "Code too clever"); + + Assert((ulg)s->strstart <= s->window_size-MIN_LOOKAHEAD, "need lookahead"); + + Assert(cur_match < s->strstart, "no future"); + + match = s->window + cur_match; + + /* Return failure if the match length is less than 2: + */ + if (match[0] != scan[0] || match[1] != scan[1]) return MIN_MATCH-1; + + /* The check at best_len-1 can be removed because it will be made + * again later. (This heuristic is not always a win.) + * It is not necessary to compare scan[2] and match[2] since they + * are always equal when the other bytes match, given that + * the hash keys are equal and that HASH_BITS >= 8. + */ + scan += 2, match += 2; + Assert(*scan == *match, "match[2]?"); + + /* We check for insufficient lookahead only every 8th comparison; + * the 256th check will be made at strstart+258. + */ + do { + } while (*++scan == *++match && *++scan == *++match && + *++scan == *++match && *++scan == *++match && + *++scan == *++match && *++scan == *++match && + *++scan == *++match && *++scan == *++match && + scan < strend); + + Assert(scan <= s->window+(unsigned)(s->window_size-1), "wild scan"); + + len = MAX_MATCH - (int)(strend - scan); + + if (len < MIN_MATCH) return MIN_MATCH - 1; + + s->match_start = cur_match; + return (uInt)len <= s->lookahead ? (uInt)len : s->lookahead; +} + +#endif /* FASTEST */ + +#ifdef ZLIB_DEBUG + +#define EQUAL 0 +/* result of memcmp for equal strings */ + +/* =========================================================================== + * Check that the match at match_start is indeed a match. + */ +local void check_match(s, start, match, length) + deflate_state *s; + IPos start, match; + int length; +{ + /* check that the match is indeed a match */ + if (zmemcmp(s->window + match, + s->window + start, length) != EQUAL) { + fprintf(stderr, " start %u, match %u, length %d\n", + start, match, length); + do { + fprintf(stderr, "%c%c", s->window[match++], s->window[start++]); + } while (--length != 0); + z_error("invalid match"); + } + if (z_verbose > 1) { + fprintf(stderr,"\\[%d,%d]", start-match, length); + do { putc(s->window[start++], stderr); } while (--length != 0); + } +} +#else +# define check_match(s, start, match, length) +#endif /* ZLIB_DEBUG */ + +/* =========================================================================== + * Fill the window when the lookahead becomes insufficient. + * Updates strstart and lookahead. + * + * IN assertion: lookahead < MIN_LOOKAHEAD + * OUT assertions: strstart <= window_size-MIN_LOOKAHEAD + * At least one byte has been read, or avail_in == 0; reads are + * performed for at least two bytes (required for the zip translate_eol + * option -- not supported here). + */ +local void fill_window_c(deflate_state *s); + +local void fill_window(deflate_state *s) +{ +#ifdef DEFLATE_FILL_WINDOW_SSE2 + if (x86_cpu_enable_simd) { + fill_window_sse(s); + return; + } +#endif + fill_window_c(s); +} + +local void fill_window_c(s) + deflate_state *s; +{ + unsigned n; + unsigned more; /* Amount of free space at the end of the window. */ + uInt wsize = s->w_size; + + Assert(s->lookahead < MIN_LOOKAHEAD, "already enough lookahead"); + + do { + more = (unsigned)(s->window_size -(ulg)s->lookahead -(ulg)s->strstart); + + /* Deal with !@#$% 64K limit: */ + if (sizeof(int) <= 2) { + if (more == 0 && s->strstart == 0 && s->lookahead == 0) { + more = wsize; + + } else if (more == (unsigned)(-1)) { + /* Very unlikely, but possible on 16 bit machine if + * strstart == 0 && lookahead == 1 (input done a byte at time) + */ + more--; + } + } + + /* If the window is almost full and there is insufficient lookahead, + * move the upper half to the lower one to make room in the upper half. + */ + if (s->strstart >= wsize+MAX_DIST(s)) { + + zmemcpy(s->window, s->window+wsize, (unsigned)wsize - more); + s->match_start -= wsize; + s->strstart -= wsize; /* we now have strstart >= MAX_DIST */ + s->block_start -= (long) wsize; + slide_hash(s); + more += wsize; + } + if (s->strm->avail_in == 0) break; + + /* If there was no sliding: + * strstart <= WSIZE+MAX_DIST-1 && lookahead <= MIN_LOOKAHEAD - 1 && + * more == window_size - lookahead - strstart + * => more >= window_size - (MIN_LOOKAHEAD-1 + WSIZE + MAX_DIST-1) + * => more >= window_size - 2*WSIZE + 2 + * In the BIG_MEM or MMAP case (not yet supported), + * window_size == input_size + MIN_LOOKAHEAD && + * strstart + s->lookahead <= input_size => more >= MIN_LOOKAHEAD. + * Otherwise, window_size == 2*WSIZE so more >= 2. + * If there was sliding, more >= WSIZE. So in all cases, more >= 2. + */ + Assert(more >= 2, "more < 2"); + + n = deflate_read_buf(s->strm, s->window + s->strstart + s->lookahead, more); + s->lookahead += n; + + /* Initialize the hash value now that we have some input: */ + if (s->lookahead + s->insert >= MIN_MATCH) { + uInt str = s->strstart - s->insert; + s->ins_h = s->window[str]; + UPDATE_HASH(s, s->ins_h, s->window[str + 1]); +#if MIN_MATCH != 3 + Call UPDATE_HASH() MIN_MATCH-3 more times +#endif + while (s->insert) { + UPDATE_HASH(s, s->ins_h, s->window[str + MIN_MATCH-1]); +#ifndef FASTEST + s->prev[str & s->w_mask] = s->head[s->ins_h]; +#endif + s->head[s->ins_h] = (Pos)str; + str++; + s->insert--; + if (s->lookahead + s->insert < MIN_MATCH) + break; + } + } + /* If the whole input has less than MIN_MATCH bytes, ins_h is garbage, + * but this is not important since only literal bytes will be emitted. + */ + + } while (s->lookahead < MIN_LOOKAHEAD && s->strm->avail_in != 0); + + /* If the WIN_INIT bytes after the end of the current data have never been + * written, then zero those bytes in order to avoid memory check reports of + * the use of uninitialized (or uninitialised as Julian writes) bytes by + * the longest match routines. Update the high water mark for the next + * time through here. WIN_INIT is set to MAX_MATCH since the longest match + * routines allow scanning to strstart + MAX_MATCH, ignoring lookahead. + */ + if (s->high_water < s->window_size) { + ulg curr = s->strstart + (ulg)(s->lookahead); + ulg init; + + if (s->high_water < curr) { + /* Previous high water mark below current data -- zero WIN_INIT + * bytes or up to end of window, whichever is less. + */ + init = s->window_size - curr; + if (init > WIN_INIT) + init = WIN_INIT; + zmemzero(s->window + curr, (unsigned)init); + s->high_water = curr + init; + } + else if (s->high_water < (ulg)curr + WIN_INIT) { + /* High water mark at or above current data, but below current data + * plus WIN_INIT -- zero out to current data plus WIN_INIT, or up + * to end of window, whichever is less. + */ + init = (ulg)curr + WIN_INIT - s->high_water; + if (init > s->window_size - s->high_water) + init = s->window_size - s->high_water; + zmemzero(s->window + s->high_water, (unsigned)init); + s->high_water += init; + } + } + + Assert((ulg)s->strstart <= s->window_size - MIN_LOOKAHEAD, + "not enough room for search"); +} + +/* =========================================================================== + * Flush the current block, with given end-of-file flag. + * IN assertion: strstart is set to the end of the current match. + */ +#define FLUSH_BLOCK_ONLY(s, last) { \ + _tr_flush_block(s, (s->block_start >= 0L ? \ + (charf *)&s->window[(unsigned)s->block_start] : \ + (charf *)Z_NULL), \ + (ulg)((long)s->strstart - s->block_start), \ + (last)); \ + s->block_start = s->strstart; \ + flush_pending(s->strm); \ + Tracev((stderr,"[FLUSH]")); \ +} + +/* Same but force premature exit if necessary. */ +#define FLUSH_BLOCK(s, last) { \ + FLUSH_BLOCK_ONLY(s, last); \ + if (s->strm->avail_out == 0) return (last) ? finish_started : need_more; \ +} + +/* Maximum stored block length in deflate format (not including header). */ +#define MAX_STORED 65535 + +/* Minimum of a and b. */ +#define MIN(a, b) ((a) > (b) ? (b) : (a)) + +/* =========================================================================== + * Copy without compression as much as possible from the input stream, return + * the current block state. + * + * In case deflateParams() is used to later switch to a non-zero compression + * level, s->matches (otherwise unused when storing) keeps track of the number + * of hash table slides to perform. If s->matches is 1, then one hash table + * slide will be done when switching. If s->matches is 2, the maximum value + * allowed here, then the hash table will be cleared, since two or more slides + * is the same as a clear. + * + * deflate_stored() is written to minimize the number of times an input byte is + * copied. It is most efficient with large input and output buffers, which + * maximizes the opportunites to have a single copy from next_in to next_out. + */ +local block_state deflate_stored(s, flush) + deflate_state *s; + int flush; +{ + /* Smallest worthy block size when not flushing or finishing. By default + * this is 32K. This can be as small as 507 bytes for memLevel == 1. For + * large input and output buffers, the stored block size will be larger. + */ + unsigned min_block = MIN(s->pending_buf_size - 5, s->w_size); + + /* Copy as many min_block or larger stored blocks directly to next_out as + * possible. If flushing, copy the remaining available input to next_out as + * stored blocks, if there is enough space. + */ + unsigned len, left, have, last = 0; + unsigned used = s->strm->avail_in; + do { + /* Set len to the maximum size block that we can copy directly with the + * available input data and output space. Set left to how much of that + * would be copied from what's left in the window. + */ + len = MAX_STORED; /* maximum deflate stored block length */ + have = (s->bi_valid + 42) >> 3; /* number of header bytes */ + if (s->strm->avail_out < have) /* need room for header */ + break; + /* maximum stored block length that will fit in avail_out: */ + have = s->strm->avail_out - have; + left = s->strstart - s->block_start; /* bytes left in window */ + if (len > (ulg)left + s->strm->avail_in) + len = left + s->strm->avail_in; /* limit len to the input */ + if (len > have) + len = have; /* limit len to the output */ + + /* If the stored block would be less than min_block in length, or if + * unable to copy all of the available input when flushing, then try + * copying to the window and the pending buffer instead. Also don't + * write an empty block when flushing -- deflate() does that. + */ + if (len < min_block && ((len == 0 && flush != Z_FINISH) || + flush == Z_NO_FLUSH || + len != left + s->strm->avail_in)) + break; + + /* Make a dummy stored block in pending to get the header bytes, + * including any pending bits. This also updates the debugging counts. + */ + last = flush == Z_FINISH && len == left + s->strm->avail_in ? 1 : 0; + _tr_stored_block(s, (char *)0, 0L, last); + + /* Replace the lengths in the dummy stored block with len. */ + s->pending_buf[s->pending - 4] = len; + s->pending_buf[s->pending - 3] = len >> 8; + s->pending_buf[s->pending - 2] = ~len; + s->pending_buf[s->pending - 1] = ~len >> 8; + + /* Write the stored block header bytes. */ + flush_pending(s->strm); + +#ifdef ZLIB_DEBUG + /* Update debugging counts for the data about to be copied. */ + s->compressed_len += len << 3; + s->bits_sent += len << 3; +#endif + + /* Copy uncompressed bytes from the window to next_out. */ + if (left) { + if (left > len) + left = len; + zmemcpy(s->strm->next_out, s->window + s->block_start, left); + s->strm->next_out += left; + s->strm->avail_out -= left; + s->strm->total_out += left; + s->block_start += left; + len -= left; + } + + /* Copy uncompressed bytes directly from next_in to next_out, updating + * the check value. + */ + if (len) { + deflate_read_buf(s->strm, s->strm->next_out, len); + s->strm->next_out += len; + s->strm->avail_out -= len; + s->strm->total_out += len; + } + } while (last == 0); + + /* Update the sliding window with the last s->w_size bytes of the copied + * data, or append all of the copied data to the existing window if less + * than s->w_size bytes were copied. Also update the number of bytes to + * insert in the hash tables, in the event that deflateParams() switches to + * a non-zero compression level. + */ + used -= s->strm->avail_in; /* number of input bytes directly copied */ + if (used) { + /* If any input was used, then no unused input remains in the window, + * therefore s->block_start == s->strstart. + */ + if (used >= s->w_size) { /* supplant the previous history */ + s->matches = 2; /* clear hash */ + zmemcpy(s->window, s->strm->next_in - s->w_size, s->w_size); + s->strstart = s->w_size; + } + else { + if (s->window_size - s->strstart <= used) { + /* Slide the window down. */ + s->strstart -= s->w_size; + zmemcpy(s->window, s->window + s->w_size, s->strstart); + if (s->matches < 2) + s->matches++; /* add a pending slide_hash() */ + } + zmemcpy(s->window + s->strstart, s->strm->next_in - used, used); + s->strstart += used; + } + s->block_start = s->strstart; + s->insert += MIN(used, s->w_size - s->insert); + } + if (s->high_water < s->strstart) + s->high_water = s->strstart; + + /* If the last block was written to next_out, then done. */ + if (last) + return finish_done; + + /* If flushing and all input has been consumed, then done. */ + if (flush != Z_NO_FLUSH && flush != Z_FINISH && + s->strm->avail_in == 0 && (long)s->strstart == s->block_start) + return block_done; + + /* Fill the window with any remaining input. */ + have = s->window_size - s->strstart - 1; + if (s->strm->avail_in > have && s->block_start >= (long)s->w_size) { + /* Slide the window down. */ + s->block_start -= s->w_size; + s->strstart -= s->w_size; + zmemcpy(s->window, s->window + s->w_size, s->strstart); + if (s->matches < 2) + s->matches++; /* add a pending slide_hash() */ + have += s->w_size; /* more space now */ + } + if (have > s->strm->avail_in) + have = s->strm->avail_in; + if (have) { + deflate_read_buf(s->strm, s->window + s->strstart, have); + s->strstart += have; + } + if (s->high_water < s->strstart) + s->high_water = s->strstart; + + /* There was not enough avail_out to write a complete worthy or flushed + * stored block to next_out. Write a stored block to pending instead, if we + * have enough input for a worthy block, or if flushing and there is enough + * room for the remaining input as a stored block in the pending buffer. + */ + have = (s->bi_valid + 42) >> 3; /* number of header bytes */ + /* maximum stored block length that will fit in pending: */ + have = MIN(s->pending_buf_size - have, MAX_STORED); + min_block = MIN(have, s->w_size); + left = s->strstart - s->block_start; + if (left >= min_block || + ((left || flush == Z_FINISH) && flush != Z_NO_FLUSH && + s->strm->avail_in == 0 && left <= have)) { + len = MIN(left, have); + last = flush == Z_FINISH && s->strm->avail_in == 0 && + len == left ? 1 : 0; + _tr_stored_block(s, (charf *)s->window + s->block_start, len, last); + s->block_start += len; + flush_pending(s->strm); + } + + /* We've done all we can with the available input and output. */ + return last ? finish_started : need_more; +} + +/* =========================================================================== + * Compress as much as possible from the input stream, return the current + * block state. + * This function does not perform lazy evaluation of matches and inserts + * new strings in the dictionary only for unmatched strings or for short + * matches. It is used only for the fast compression options. + */ +local block_state deflate_fast(s, flush) + deflate_state *s; + int flush; +{ + IPos hash_head; /* head of the hash chain */ + int bflush; /* set if current block must be flushed */ + + for (;;) { + /* Make sure that we always have enough lookahead, except + * at the end of the input file. We need MAX_MATCH bytes + * for the next match, plus MIN_MATCH bytes to insert the + * string following the next match. + */ + if (s->lookahead < MIN_LOOKAHEAD) { + fill_window(s); + if (s->lookahead < MIN_LOOKAHEAD && flush == Z_NO_FLUSH) { + return need_more; + } + if (s->lookahead == 0) break; /* flush the current block */ + } + + /* Insert the string window[strstart .. strstart+2] in the + * dictionary, and set hash_head to the head of the hash chain: + */ + hash_head = NIL; + if (s->lookahead >= MIN_MATCH) { + hash_head = insert_string(s, s->strstart); + } + + /* Find the longest match, discarding those <= prev_length. + * At this point we have always match_length < MIN_MATCH + */ + if (hash_head != NIL && s->strstart - hash_head <= MAX_DIST(s)) { + /* To simplify the code, we prevent matches with the string + * of window index 0 (in particular we have to avoid a match + * of the string with itself at the start of the input file). + */ + s->match_length = longest_match (s, hash_head); + /* longest_match() sets match_start */ + } + if (s->match_length >= MIN_MATCH) { + check_match(s, s->strstart, s->match_start, s->match_length); + + _tr_tally_dist(s, s->strstart - s->match_start, + s->match_length - MIN_MATCH, bflush); + + s->lookahead -= s->match_length; + + /* Insert new strings in the hash table only if the match length + * is not too large. This saves time but degrades compression. + */ +#ifndef FASTEST + if (s->match_length <= s->max_insert_length && + s->lookahead >= MIN_MATCH) { + s->match_length--; /* string at strstart already in table */ + do { + s->strstart++; + hash_head = insert_string(s, s->strstart); + /* strstart never exceeds WSIZE-MAX_MATCH, so there are + * always MIN_MATCH bytes ahead. + */ + } while (--s->match_length != 0); + s->strstart++; + } else +#endif + { + s->strstart += s->match_length; + s->match_length = 0; + s->ins_h = s->window[s->strstart]; + UPDATE_HASH(s, s->ins_h, s->window[s->strstart+1]); +#if MIN_MATCH != 3 + Call UPDATE_HASH() MIN_MATCH-3 more times +#endif + /* If lookahead < MIN_MATCH, ins_h is garbage, but it does not + * matter since it will be recomputed at next deflate call. + */ + } + } else { + /* No match, output a literal byte */ + Tracevv((stderr,"%c", s->window[s->strstart])); + _tr_tally_lit (s, s->window[s->strstart], bflush); + s->lookahead--; + s->strstart++; + } + if (bflush) FLUSH_BLOCK(s, 0); + } + s->insert = s->strstart < MIN_MATCH-1 ? s->strstart : MIN_MATCH-1; + if (flush == Z_FINISH) { + FLUSH_BLOCK(s, 1); + return finish_done; + } + if (s->sym_next) + FLUSH_BLOCK(s, 0); + return block_done; +} + +#ifndef FASTEST +/* =========================================================================== + * Same as above, but achieves better compression. We use a lazy + * evaluation for matches: a match is finally adopted only if there is + * no better match at the next window position. + */ +local block_state deflate_slow(s, flush) + deflate_state *s; + int flush; +{ + IPos hash_head; /* head of hash chain */ + int bflush; /* set if current block must be flushed */ + + /* Process the input block. */ + for (;;) { + /* Make sure that we always have enough lookahead, except + * at the end of the input file. We need MAX_MATCH bytes + * for the next match, plus MIN_MATCH bytes to insert the + * string following the next match. + */ + if (s->lookahead < MIN_LOOKAHEAD) { + fill_window(s); + if (s->lookahead < MIN_LOOKAHEAD && flush == Z_NO_FLUSH) { + return need_more; + } + if (s->lookahead == 0) break; /* flush the current block */ + } + + /* Insert the string window[strstart .. strstart+2] in the + * dictionary, and set hash_head to the head of the hash chain: + */ + hash_head = NIL; + if (s->lookahead >= MIN_MATCH) { + hash_head = insert_string(s, s->strstart); + } + + /* Find the longest match, discarding those <= prev_length. + */ + s->prev_length = s->match_length, s->prev_match = s->match_start; + s->match_length = MIN_MATCH-1; + + if (hash_head != NIL && s->prev_length < s->max_lazy_match && + s->strstart - hash_head <= MAX_DIST(s)) { + /* To simplify the code, we prevent matches with the string + * of window index 0 (in particular we have to avoid a match + * of the string with itself at the start of the input file). + */ + s->match_length = longest_match (s, hash_head); + /* longest_match() sets match_start */ + + if (s->match_length <= 5 && (s->strategy == Z_FILTERED +#if TOO_FAR <= 32767 + || (s->match_length == MIN_MATCH && + s->strstart - s->match_start > TOO_FAR) +#endif + )) { + + /* If prev_match is also MIN_MATCH, match_start is garbage + * but we will ignore the current match anyway. + */ + s->match_length = MIN_MATCH-1; + } + } + /* If there was a match at the previous step and the current + * match is not better, output the previous match: + */ + if (s->prev_length >= MIN_MATCH && s->match_length <= s->prev_length) { + uInt max_insert = s->strstart + s->lookahead - MIN_MATCH; + /* Do not insert strings in hash table beyond this. */ + + if (s->prev_match == -1) { + /* The window has slid one byte past the previous match, + * so the first byte cannot be compared. */ + check_match(s, s->strstart, s->prev_match+1, s->prev_length-1); + } else { + check_match(s, s->strstart-1, s->prev_match, s->prev_length); + } + + _tr_tally_dist(s, s->strstart -1 - s->prev_match, + s->prev_length - MIN_MATCH, bflush); + + /* Insert in hash table all strings up to the end of the match. + * strstart-1 and strstart are already inserted. If there is not + * enough lookahead, the last two strings are not inserted in + * the hash table. + */ + s->lookahead -= s->prev_length-1; + s->prev_length -= 2; + do { + if (++s->strstart <= max_insert) { + hash_head = insert_string(s, s->strstart); + } + } while (--s->prev_length != 0); + s->match_available = 0; + s->match_length = MIN_MATCH-1; + s->strstart++; + + if (bflush) FLUSH_BLOCK(s, 0); + + } else if (s->match_available) { + /* If there was no match at the previous position, output a + * single literal. If there was a match but the current match + * is longer, truncate the previous match to a single literal. + */ + Tracevv((stderr,"%c", s->window[s->strstart-1])); + _tr_tally_lit(s, s->window[s->strstart-1], bflush); + if (bflush) { + FLUSH_BLOCK_ONLY(s, 0); + } + s->strstart++; + s->lookahead--; + if (s->strm->avail_out == 0) return need_more; + } else { + /* There is no previous match to compare with, wait for + * the next step to decide. + */ + s->match_available = 1; + s->strstart++; + s->lookahead--; + } + } + Assert (flush != Z_NO_FLUSH, "no flush?"); + if (s->match_available) { + Tracevv((stderr,"%c", s->window[s->strstart-1])); + _tr_tally_lit(s, s->window[s->strstart-1], bflush); + s->match_available = 0; + } + s->insert = s->strstart < MIN_MATCH-1 ? s->strstart : MIN_MATCH-1; + if (flush == Z_FINISH) { + FLUSH_BLOCK(s, 1); + return finish_done; + } + if (s->sym_next) + FLUSH_BLOCK(s, 0); + return block_done; +} +#endif /* FASTEST */ + +/* =========================================================================== + * For Z_RLE, simply look for runs of bytes, generate matches only of distance + * one. Do not maintain a hash table. (It will be regenerated if this run of + * deflate switches away from Z_RLE.) + */ +local block_state deflate_rle(s, flush) + deflate_state *s; + int flush; +{ + int bflush; /* set if current block must be flushed */ + uInt prev; /* byte at distance one to match */ + Bytef *scan, *strend; /* scan goes up to strend for length of run */ + + for (;;) { + /* Make sure that we always have enough lookahead, except + * at the end of the input file. We need MAX_MATCH bytes + * for the longest run, plus one for the unrolled loop. + */ + if (s->lookahead <= MAX_MATCH) { + fill_window(s); + if (s->lookahead <= MAX_MATCH && flush == Z_NO_FLUSH) { + return need_more; + } + if (s->lookahead == 0) break; /* flush the current block */ + } + + /* See how many times the previous byte repeats */ + s->match_length = 0; + if (s->lookahead >= MIN_MATCH && s->strstart > 0) { + scan = s->window + s->strstart - 1; + prev = *scan; + if (prev == *++scan && prev == *++scan && prev == *++scan) { + strend = s->window + s->strstart + MAX_MATCH; + do { + } while (prev == *++scan && prev == *++scan && + prev == *++scan && prev == *++scan && + prev == *++scan && prev == *++scan && + prev == *++scan && prev == *++scan && + scan < strend); + s->match_length = MAX_MATCH - (uInt)(strend - scan); + if (s->match_length > s->lookahead) + s->match_length = s->lookahead; + } + Assert(scan <= s->window+(uInt)(s->window_size-1), "wild scan"); + } + + /* Emit match if have run of MIN_MATCH or longer, else emit literal */ + if (s->match_length >= MIN_MATCH) { + check_match(s, s->strstart, s->strstart - 1, s->match_length); + + _tr_tally_dist(s, 1, s->match_length - MIN_MATCH, bflush); + + s->lookahead -= s->match_length; + s->strstart += s->match_length; + s->match_length = 0; + } else { + /* No match, output a literal byte */ + Tracevv((stderr,"%c", s->window[s->strstart])); + _tr_tally_lit (s, s->window[s->strstart], bflush); + s->lookahead--; + s->strstart++; + } + if (bflush) FLUSH_BLOCK(s, 0); + } + s->insert = 0; + if (flush == Z_FINISH) { + FLUSH_BLOCK(s, 1); + return finish_done; + } + if (s->sym_next) + FLUSH_BLOCK(s, 0); + return block_done; +} + +/* =========================================================================== + * For Z_HUFFMAN_ONLY, do not look for matches. Do not maintain a hash table. + * (It will be regenerated if this run of deflate switches away from Huffman.) + */ +local block_state deflate_huff(s, flush) + deflate_state *s; + int flush; +{ + int bflush; /* set if current block must be flushed */ + + for (;;) { + /* Make sure that we have a literal to write. */ + if (s->lookahead == 0) { + fill_window(s); + if (s->lookahead == 0) { + if (flush == Z_NO_FLUSH) + return need_more; + break; /* flush the current block */ + } + } + + /* Output a literal byte */ + s->match_length = 0; + Tracevv((stderr,"%c", s->window[s->strstart])); + _tr_tally_lit (s, s->window[s->strstart], bflush); + s->lookahead--; + s->strstart++; + if (bflush) FLUSH_BLOCK(s, 0); + } + s->insert = 0; + if (flush == Z_FINISH) { + FLUSH_BLOCK(s, 1); + return finish_done; + } + if (s->sym_next) + FLUSH_BLOCK(s, 0); + return block_done; +} diff --git a/third_party/zlib/deflate.h b/third_party/zlib/deflate.h new file mode 100644 index 00000000..78ffb00a --- /dev/null +++ b/third_party/zlib/deflate.h @@ -0,0 +1,356 @@ +/* deflate.h -- internal compression state + * Copyright (C) 1995-2016 Jean-loup Gailly + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +/* WARNING: this file should *not* be used by applications. It is + part of the implementation of the compression library and is + subject to change. Applications should only use zlib.h. + */ + +/* @(#) $Id$ */ + +#ifndef DEFLATE_H +#define DEFLATE_H + +#include "zutil.h" + +/* define NO_GZIP when compiling if you want to disable gzip header and + trailer creation by deflate(). NO_GZIP would be used to avoid linking in + the crc code when it is not needed. For shared libraries, gzip encoding + should be left enabled. */ +#ifndef NO_GZIP +# define GZIP +#endif + +/* =========================================================================== + * Internal compression state. + */ + +#define LENGTH_CODES 29 +/* number of length codes, not counting the special END_BLOCK code */ + +#define LITERALS 256 +/* number of literal bytes 0..255 */ + +#define L_CODES (LITERALS+1+LENGTH_CODES) +/* number of Literal or Length codes, including the END_BLOCK code */ + +#define D_CODES 30 +/* number of distance codes */ + +#define BL_CODES 19 +/* number of codes used to transfer the bit lengths */ + +#define HEAP_SIZE (2*L_CODES+1) +/* maximum heap size */ + +#define MAX_BITS 15 +/* All codes must not exceed MAX_BITS bits */ + +#define Buf_size 16 +/* size of bit buffer in bi_buf */ + +#define INIT_STATE 42 /* zlib header -> BUSY_STATE */ +#ifdef GZIP +# define GZIP_STATE 57 /* gzip header -> BUSY_STATE | EXTRA_STATE */ +#endif +#define EXTRA_STATE 69 /* gzip extra block -> NAME_STATE */ +#define NAME_STATE 73 /* gzip file name -> COMMENT_STATE */ +#define COMMENT_STATE 91 /* gzip comment -> HCRC_STATE */ +#define HCRC_STATE 103 /* gzip header CRC -> BUSY_STATE */ +#define BUSY_STATE 113 /* deflate -> FINISH_STATE */ +#define FINISH_STATE 666 /* stream complete */ +/* Stream status */ + + +/* Data structure describing a single value and its code string. */ +typedef struct ct_data_s { + union { + ush freq; /* frequency count */ + ush code; /* bit string */ + } fc; + union { + ush dad; /* father node in Huffman tree */ + ush len; /* length of bit string */ + } dl; +} FAR ct_data; + +#define Freq fc.freq +#define Code fc.code +#define Dad dl.dad +#define Len dl.len + +typedef struct static_tree_desc_s static_tree_desc; + +typedef struct tree_desc_s { + ct_data *dyn_tree; /* the dynamic tree */ + int max_code; /* largest code with non zero frequency */ + const static_tree_desc *stat_desc; /* the corresponding static tree */ +} FAR tree_desc; + +typedef ush Pos; +typedef Pos FAR Posf; +typedef unsigned IPos; + +/* A Pos is an index in the character window. We use short instead of int to + * save space in the various tables. IPos is used only for parameter passing. + */ + +typedef struct internal_state { + z_streamp strm; /* pointer back to this zlib stream */ + int status; /* as the name implies */ + Bytef *pending_buf; /* output still pending */ + ulg pending_buf_size; /* size of pending_buf */ + Bytef *pending_out; /* next pending byte to output to the stream */ + ulg pending; /* nb of bytes in the pending buffer */ + int wrap; /* bit 0 true for zlib, bit 1 true for gzip */ + gz_headerp gzhead; /* gzip header information to write */ + ulg gzindex; /* where in extra, name, or comment */ + Byte method; /* can only be DEFLATED */ + int last_flush; /* value of flush param for previous deflate call */ + unsigned crc0[4 * 5]; + /* used by deflate.c: */ + + uInt w_size; /* LZ77 window size (32K by default) */ + uInt w_bits; /* log2(w_size) (8..16) */ + uInt w_mask; /* w_size - 1 */ + + Bytef *window; + /* Sliding window. Input bytes are read into the second half of the window, + * and move to the first half later to keep a dictionary of at least wSize + * bytes. With this organization, matches are limited to a distance of + * wSize-MAX_MATCH bytes, but this ensures that IO is always + * performed with a length multiple of the block size. Also, it limits + * the window size to 64K, which is quite useful on MSDOS. + * To do: use the user input buffer as sliding window. + */ + + ulg window_size; + /* Actual size of window: 2*wSize, except when the user input buffer + * is directly used as sliding window. + */ + + Posf *prev; + /* Link to older string with same hash index. To limit the size of this + * array to 64K, this link is maintained only for the last 32K strings. + * An index in this array is thus a window index modulo 32K. + */ + + Posf *head; /* Heads of the hash chains or NIL. */ + + uInt ins_h; /* hash index of string to be inserted */ + uInt hash_size; /* number of elements in hash table */ + uInt hash_bits; /* log2(hash_size) */ + uInt hash_mask; /* hash_size-1 */ + + uInt hash_shift; + /* Number of bits by which ins_h must be shifted at each input + * step. It must be such that after MIN_MATCH steps, the oldest + * byte no longer takes part in the hash key, that is: + * hash_shift * MIN_MATCH >= hash_bits + */ + + long block_start; + /* Window position at the beginning of the current output block. Gets + * negative when the window is moved backwards. + */ + + uInt match_length; /* length of best match */ + IPos prev_match; /* previous match */ + int match_available; /* set if previous match exists */ + uInt strstart; /* start of string to insert */ + uInt match_start; /* start of matching string */ + uInt lookahead; /* number of valid bytes ahead in window */ + + uInt prev_length; + /* Length of the best match at previous step. Matches not greater than this + * are discarded. This is used in the lazy match evaluation. + */ + + uInt max_chain_length; + /* To speed up deflation, hash chains are never searched beyond this + * length. A higher limit improves compression ratio but degrades the + * speed. + */ + + uInt max_lazy_match; + /* Attempt to find a better match only when the current match is strictly + * smaller than this value. This mechanism is used only for compression + * levels >= 4. + */ +# define max_insert_length max_lazy_match + /* Insert new strings in the hash table only if the match length is not + * greater than this length. This saves time but degrades compression. + * max_insert_length is used only for compression levels <= 3. + */ + + int level; /* compression level (1..9) */ + int strategy; /* favor or force Huffman coding*/ + + uInt good_match; + /* Use a faster search when the previous match is longer than this */ + + int nice_match; /* Stop searching when current match exceeds this */ + + /* used by trees.c: */ + /* Didn't use ct_data typedef below to suppress compiler warning */ + struct ct_data_s dyn_ltree[HEAP_SIZE]; /* literal and length tree */ + struct ct_data_s dyn_dtree[2*D_CODES+1]; /* distance tree */ + struct ct_data_s bl_tree[2*BL_CODES+1]; /* Huffman tree for bit lengths */ + + struct tree_desc_s l_desc; /* desc. for literal tree */ + struct tree_desc_s d_desc; /* desc. for distance tree */ + struct tree_desc_s bl_desc; /* desc. for bit length tree */ + + ush bl_count[MAX_BITS+1]; + /* number of codes at each bit length for an optimal tree */ + + int heap[2*L_CODES+1]; /* heap used to build the Huffman trees */ + int heap_len; /* number of elements in the heap */ + int heap_max; /* element of largest frequency */ + /* The sons of heap[n] are heap[2*n] and heap[2*n+1]. heap[0] is not used. + * The same heap array is used to build all trees. + */ + + uch depth[2*L_CODES+1]; + /* Depth of each subtree used as tie breaker for trees of equal frequency + */ + + uchf *sym_buf; /* buffer for distances and literals/lengths */ + + uInt lit_bufsize; + /* Size of match buffer for literals/lengths. There are 4 reasons for + * limiting lit_bufsize to 64K: + * - frequencies can be kept in 16 bit counters + * - if compression is not successful for the first block, all input + * data is still in the window so we can still emit a stored block even + * when input comes from standard input. (This can also be done for + * all blocks if lit_bufsize is not greater than 32K.) + * - if compression is not successful for a file smaller than 64K, we can + * even emit a stored file instead of a stored block (saving 5 bytes). + * This is applicable only for zip (not gzip or zlib). + * - creating new Huffman trees less frequently may not provide fast + * adaptation to changes in the input data statistics. (Take for + * example a binary file with poorly compressible code followed by + * a highly compressible string table.) Smaller buffer sizes give + * fast adaptation but have of course the overhead of transmitting + * trees more frequently. + * - I can't count above 4 + */ + + uInt sym_next; /* running index in sym_buf */ + uInt sym_end; /* symbol table full when sym_next reaches this */ + + ulg opt_len; /* bit length of current block with optimal trees */ + ulg static_len; /* bit length of current block with static trees */ + uInt matches; /* number of string matches in current block */ + uInt insert; /* bytes at end of window left to insert */ + +#ifdef ZLIB_DEBUG + ulg compressed_len; /* total bit length of compressed file mod 2^32 */ + ulg bits_sent; /* bit length of compressed data sent mod 2^32 */ +#endif + + ush bi_buf; + /* Output buffer. bits are inserted starting at the bottom (least + * significant bits). + */ + int bi_valid; + /* Number of valid bits in bi_buf. All bits above the last valid bit + * are always zero. + */ + + ulg high_water; + /* High water mark offset in window for initialized bytes -- bytes above + * this are set to zero in order to avoid memory check warnings when + * longest match routines access bytes past the input. This is then + * updated to the new high water mark. + */ + +} FAR deflate_state; + +/* Output a byte on the stream. + * IN assertion: there is enough room in pending_buf. + */ +#define put_byte(s, c) {s->pending_buf[s->pending++] = (Bytef)(c);} + + +#define MIN_LOOKAHEAD (MAX_MATCH+MIN_MATCH+1) +/* Minimum amount of lookahead, except at the end of the input file. + * See deflate.c for comments about the MIN_MATCH+1. + */ + +#define MAX_DIST(s) ((s)->w_size-MIN_LOOKAHEAD) +/* In order to simplify the code, particularly on 16 bit machines, match + * distances are limited to MAX_DIST instead of WSIZE. + */ + +#define WIN_INIT MAX_MATCH +/* Number of bytes after end of data in window to initialize in order to avoid + memory checker errors from longest match routines */ + + /* in trees.c */ +void ZLIB_INTERNAL _tr_init OF((deflate_state *s)); +int ZLIB_INTERNAL _tr_tally OF((deflate_state *s, unsigned dist, unsigned lc)); +void ZLIB_INTERNAL _tr_flush_block OF((deflate_state *s, charf *buf, + ulg stored_len, int last)); +void ZLIB_INTERNAL _tr_flush_bits OF((deflate_state *s)); +void ZLIB_INTERNAL _tr_align OF((deflate_state *s)); +void ZLIB_INTERNAL _tr_stored_block OF((deflate_state *s, charf *buf, + ulg stored_len, int last)); + +#define d_code(dist) \ + ((dist) < 256 ? _dist_code[dist] : _dist_code[256+((dist)>>7)]) +/* Mapping from a distance to a distance code. dist is the distance - 1 and + * must not have side effects. _dist_code[256] and _dist_code[257] are never + * used. + */ + +#ifndef ZLIB_DEBUG +/* Inline versions of _tr_tally for speed: */ + +#if defined(GEN_TREES_H) || !defined(STDC) + extern uch ZLIB_INTERNAL _length_code[]; + extern uch ZLIB_INTERNAL _dist_code[]; +#else + extern const uch ZLIB_INTERNAL _length_code[]; + extern const uch ZLIB_INTERNAL _dist_code[]; +#endif + +# define _tr_tally_lit(s, c, flush) \ + { uch cc = (c); \ + s->sym_buf[s->sym_next++] = 0; \ + s->sym_buf[s->sym_next++] = 0; \ + s->sym_buf[s->sym_next++] = cc; \ + s->dyn_ltree[cc].Freq++; \ + flush = (s->sym_next == s->sym_end); \ + } +# define _tr_tally_dist(s, distance, length, flush) \ + { uch len = (uch)(length); \ + ush dist = (ush)(distance); \ + s->sym_buf[s->sym_next++] = dist; \ + s->sym_buf[s->sym_next++] = dist >> 8; \ + s->sym_buf[s->sym_next++] = len; \ + dist--; \ + s->dyn_ltree[_length_code[len]+LITERALS+1].Freq++; \ + s->dyn_dtree[d_code(dist)].Freq++; \ + flush = (s->sym_next == s->sym_end); \ + } +#else +# define _tr_tally_lit(s, c, flush) flush = _tr_tally(s, 0, c) +# define _tr_tally_dist(s, distance, length, flush) \ + flush = _tr_tally(s, distance, length) +#endif + +/* Functions that are SIMD optimised on x86 */ +void ZLIB_INTERNAL crc_fold_init(deflate_state* const s); +void ZLIB_INTERNAL crc_fold_copy(deflate_state* const s, + unsigned char* dst, + const unsigned char* src, + long len); +unsigned ZLIB_INTERNAL crc_fold_512to32(deflate_state* const s); + +void ZLIB_INTERNAL fill_window_sse(deflate_state* s); + +#endif /* DEFLATE_H */ diff --git a/third_party/zlib/fill_window_sse.c b/third_party/zlib/fill_window_sse.c new file mode 100644 index 00000000..a841c999 --- /dev/null +++ b/third_party/zlib/fill_window_sse.c @@ -0,0 +1,182 @@ +/* + * Fill Window with SSE2-optimized hash shifting + * + * Copyright (C) 2013 Intel Corporation + * Authors: + * Arjan van de Ven <arjan@linux.intel.com> + * Jim Kukunas <james.t.kukunas@linux.intel.com> + * + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +#include "deflate.h" + +#ifdef DEFLATE_FILL_WINDOW_SSE2 + +#define UPDATE_HASH(s,h,i) \ + {\ + if (s->level < 6) { \ + h = (3483 * (s->window[i]) +\ + 23081* (s->window[i+1]) +\ + 6954 * (s->window[i+2]) +\ + 20947* (s->window[i+3])) & s->hash_mask;\ + } else {\ + h = (25881* (s->window[i]) +\ + 24674* (s->window[i+1]) +\ + 25811* (s->window[i+2])) & s->hash_mask;\ + }\ + }\ + +extern int deflate_read_buf OF((z_streamp strm, Bytef *buf, unsigned size)); + +#include <immintrin.h> + +void fill_window_sse(deflate_state *s) +{ + const __m128i xmm_wsize = _mm_set1_epi16(s->w_size); + + register unsigned n; + register Posf *p; + unsigned more; /* Amount of free space at the end of the window. */ + uInt wsize = s->w_size; + + Assert(s->lookahead < MIN_LOOKAHEAD, "already enough lookahead"); + + do { + more = (unsigned)(s->window_size -(ulg)s->lookahead -(ulg)s->strstart); + + /* Deal with !@#$% 64K limit: */ + if (sizeof(int) <= 2) { + if (more == 0 && s->strstart == 0 && s->lookahead == 0) { + more = wsize; + + } else if (more == (unsigned)(-1)) { + /* Very unlikely, but possible on 16 bit machine if + * strstart == 0 && lookahead == 1 (input done a byte at time) + */ + more--; + } + } + + /* If the window is almost full and there is insufficient lookahead, + * move the upper half to the lower one to make room in the upper half. + */ + if (s->strstart >= wsize+MAX_DIST(s)) { + + zmemcpy(s->window, s->window+wsize, (unsigned)wsize); + s->match_start -= wsize; + s->strstart -= wsize; /* we now have strstart >= MAX_DIST */ + s->block_start -= (long) wsize; + + /* Slide the hash table (could be avoided with 32 bit values + at the expense of memory usage). We slide even when level == 0 + to keep the hash table consistent if we switch back to level > 0 + later. (Using level 0 permanently is not an optimal usage of + zlib, so we don't care about this pathological case.) + */ + n = s->hash_size; + p = &s->head[n]; + p -= 8; + do { + __m128i value, result; + + value = _mm_loadu_si128((__m128i *)p); + result = _mm_subs_epu16(value, xmm_wsize); + _mm_storeu_si128((__m128i *)p, result); + + p -= 8; + n -= 8; + } while (n > 0); + + n = wsize; +#ifndef FASTEST + p = &s->prev[n]; + p -= 8; + do { + __m128i value, result; + + value = _mm_loadu_si128((__m128i *)p); + result = _mm_subs_epu16(value, xmm_wsize); + _mm_storeu_si128((__m128i *)p, result); + + p -= 8; + n -= 8; + } while (n > 0); +#endif + more += wsize; + } + if (s->strm->avail_in == 0) break; + + /* If there was no sliding: + * strstart <= WSIZE+MAX_DIST-1 && lookahead <= MIN_LOOKAHEAD - 1 && + * more == window_size - lookahead - strstart + * => more >= window_size - (MIN_LOOKAHEAD-1 + WSIZE + MAX_DIST-1) + * => more >= window_size - 2*WSIZE + 2 + * In the BIG_MEM or MMAP case (not yet supported), + * window_size == input_size + MIN_LOOKAHEAD && + * strstart + s->lookahead <= input_size => more >= MIN_LOOKAHEAD. + * Otherwise, window_size == 2*WSIZE so more >= 2. + * If there was sliding, more >= WSIZE. So in all cases, more >= 2. + */ + Assert(more >= 2, "more < 2"); + + n = deflate_read_buf(s->strm, + s->window + s->strstart + s->lookahead, + more); + s->lookahead += n; + + /* Initialize the hash value now that we have some input: */ + if (s->lookahead >= MIN_MATCH) { + uInt str = s->strstart; + s->ins_h = s->window[str]; + if (str >= 1) + UPDATE_HASH(s, s->ins_h, str + 1 - (MIN_MATCH-1)); +#if MIN_MATCH != 3 + Call UPDATE_HASH() MIN_MATCH-3 more times +#endif + } + /* If the whole input has less than MIN_MATCH bytes, ins_h is garbage, + * but this is not important since only literal bytes will be emitted. + */ + + } while (s->lookahead < MIN_LOOKAHEAD && s->strm->avail_in != 0); + + /* If the WIN_INIT bytes after the end of the current data have never been + * written, then zero those bytes in order to avoid memory check reports of + * the use of uninitialized (or uninitialised as Julian writes) bytes by + * the longest match routines. Update the high water mark for the next + * time through here. WIN_INIT is set to MAX_MATCH since the longest match + * routines allow scanning to strstart + MAX_MATCH, ignoring lookahead. + */ + if (s->high_water < s->window_size) { + ulg curr = s->strstart + (ulg)(s->lookahead); + ulg init; + + if (s->high_water < curr) { + /* Previous high water mark below current data -- zero WIN_INIT + * bytes or up to end of window, whichever is less. + */ + init = s->window_size - curr; + if (init > WIN_INIT) + init = WIN_INIT; + zmemzero(s->window + curr, (unsigned)init); + s->high_water = curr + init; + } + else if (s->high_water < (ulg)curr + WIN_INIT) { + /* High water mark at or above current data, but below current data + * plus WIN_INIT -- zero out to current data plus WIN_INIT, or up + * to end of window, whichever is less. + */ + init = (ulg)curr + WIN_INIT - s->high_water; + if (init > s->window_size - s->high_water) + init = s->window_size - s->high_water; + zmemzero(s->window + s->high_water, (unsigned)init); + s->high_water += init; + } + } + + Assert((ulg)s->strstart <= s->window_size - MIN_LOOKAHEAD, + "not enough room for search"); +} + +#endif /* DEFLATE_FILL_WINDOW_SSE2 */ diff --git a/third_party/zlib/gzclose.c b/third_party/zlib/gzclose.c new file mode 100644 index 00000000..caeb99a3 --- /dev/null +++ b/third_party/zlib/gzclose.c @@ -0,0 +1,25 @@ +/* gzclose.c -- zlib gzclose() function + * Copyright (C) 2004, 2010 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +#include "gzguts.h" + +/* gzclose() is in a separate file so that it is linked in only if it is used. + That way the other gzclose functions can be used instead to avoid linking in + unneeded compression or decompression routines. */ +int ZEXPORT gzclose(file) + gzFile file; +{ +#ifndef NO_GZCOMPRESS + gz_statep state; + + if (file == NULL) + return Z_STREAM_ERROR; + state = (gz_statep)file; + + return state->mode == GZ_READ ? gzclose_r(file) : gzclose_w(file); +#else + return gzclose_r(file); +#endif +} diff --git a/third_party/zlib/gzguts.h b/third_party/zlib/gzguts.h new file mode 100644 index 00000000..990a4d25 --- /dev/null +++ b/third_party/zlib/gzguts.h @@ -0,0 +1,218 @@ +/* gzguts.h -- zlib internal header definitions for gz* operations + * Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013, 2016 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +#ifdef _LARGEFILE64_SOURCE +# ifndef _LARGEFILE_SOURCE +# define _LARGEFILE_SOURCE 1 +# endif +# ifdef _FILE_OFFSET_BITS +# undef _FILE_OFFSET_BITS +# endif +#endif + +#ifdef HAVE_HIDDEN +# define ZLIB_INTERNAL __attribute__((visibility ("hidden"))) +#else +# define ZLIB_INTERNAL +#endif + +#include <stdio.h> +#include "zlib.h" +#ifdef STDC +# include <string.h> +# include <stdlib.h> +# include <limits.h> +#endif + +#ifndef _POSIX_SOURCE +# define _POSIX_SOURCE +#endif +#include <fcntl.h> + +#ifdef _WIN32 +# include <stddef.h> +#endif + +#if defined(__TURBOC__) || defined(_MSC_VER) || defined(_WIN32) +# include <io.h> +#endif + +#if defined(_WIN32) || defined(__CYGWIN__) +# define WIDECHAR +#endif + +#ifdef WINAPI_FAMILY +# define open _open +# define read _read +# define write _write +# define close _close +#endif + +#ifdef NO_DEFLATE /* for compatibility with old definition */ +# define NO_GZCOMPRESS +#endif + +#if defined(STDC99) || (defined(__TURBOC__) && __TURBOC__ >= 0x550) +# ifndef HAVE_VSNPRINTF +# define HAVE_VSNPRINTF +# endif +#endif + +#if defined(__CYGWIN__) +# ifndef HAVE_VSNPRINTF +# define HAVE_VSNPRINTF +# endif +#endif + +#if defined(MSDOS) && defined(__BORLANDC__) && (BORLANDC > 0x410) +# ifndef HAVE_VSNPRINTF +# define HAVE_VSNPRINTF +# endif +#endif + +#ifndef HAVE_VSNPRINTF +# ifdef MSDOS +/* vsnprintf may exist on some MS-DOS compilers (DJGPP?), + but for now we just assume it doesn't. */ +# define NO_vsnprintf +# endif +# ifdef __TURBOC__ +# define NO_vsnprintf +# endif +# ifdef WIN32 +/* In Win32, vsnprintf is available as the "non-ANSI" _vsnprintf. */ +# if !defined(vsnprintf) && !defined(NO_vsnprintf) +# if !defined(_MSC_VER) || ( defined(_MSC_VER) && _MSC_VER < 1500 ) +# define vsnprintf _vsnprintf +# endif +# endif +# endif +# ifdef __SASC +# define NO_vsnprintf +# endif +# ifdef VMS +# define NO_vsnprintf +# endif +# ifdef __OS400__ +# define NO_vsnprintf +# endif +# ifdef __MVS__ +# define NO_vsnprintf +# endif +#endif + +/* unlike snprintf (which is required in C99), _snprintf does not guarantee + null termination of the result -- however this is only used in gzlib.c where + the result is assured to fit in the space provided */ +#if defined(_MSC_VER) && _MSC_VER < 1900 +# define snprintf _snprintf +#endif + +#ifndef local +# define local static +#endif +/* since "static" is used to mean two completely different things in C, we + define "local" for the non-static meaning of "static", for readability + (compile with -Dlocal if your debugger can't find static symbols) */ + +/* gz* functions always use library allocation functions */ +#ifndef STDC + extern voidp malloc OF((uInt size)); + extern void free OF((voidpf ptr)); +#endif + +/* get errno and strerror definition */ +#if defined UNDER_CE +# include <windows.h> +# define zstrerror() gz_strwinerror((DWORD)GetLastError()) +#else +# ifndef NO_STRERROR +# include <errno.h> +# define zstrerror() strerror(errno) +# else +# define zstrerror() "stdio error (consult errno)" +# endif +#endif + +/* provide prototypes for these when building zlib without LFS */ +#if !defined(_LARGEFILE64_SOURCE) || _LFS64_LARGEFILE-0 == 0 + ZEXTERN gzFile ZEXPORT gzopen64 OF((const char *, const char *)); + ZEXTERN z_off64_t ZEXPORT gzseek64 OF((gzFile, z_off64_t, int)); + ZEXTERN z_off64_t ZEXPORT gztell64 OF((gzFile)); + ZEXTERN z_off64_t ZEXPORT gzoffset64 OF((gzFile)); +#endif + +/* default memLevel */ +#if MAX_MEM_LEVEL >= 8 +# define DEF_MEM_LEVEL 8 +#else +# define DEF_MEM_LEVEL MAX_MEM_LEVEL +#endif + +/* default i/o buffer size -- double this for output when reading (this and + twice this must be able to fit in an unsigned type) */ +#define GZBUFSIZE 8192 + +/* gzip modes, also provide a little integrity check on the passed structure */ +#define GZ_NONE 0 +#define GZ_READ 7247 +#define GZ_WRITE 31153 +#define GZ_APPEND 1 /* mode set to GZ_WRITE after the file is opened */ + +/* values for gz_state how */ +#define LOOK 0 /* look for a gzip header */ +#define COPY 1 /* copy input directly */ +#define GZIP 2 /* decompress a gzip stream */ + +/* internal gzip file state data structure */ +typedef struct { + /* exposed contents for gzgetc() macro */ + struct gzFile_s x; /* "x" for exposed */ + /* x.have: number of bytes available at x.next */ + /* x.next: next output data to deliver or write */ + /* x.pos: current position in uncompressed data */ + /* used for both reading and writing */ + int mode; /* see gzip modes above */ + int fd; /* file descriptor */ + char *path; /* path or fd for error messages */ + unsigned size; /* buffer size, zero if not allocated yet */ + unsigned want; /* requested buffer size, default is GZBUFSIZE */ + unsigned char *in; /* input buffer (double-sized when writing) */ + unsigned char *out; /* output buffer (double-sized when reading) */ + int direct; /* 0 if processing gzip, 1 if transparent */ + /* just for reading */ + int how; /* 0: get header, 1: copy, 2: decompress */ + z_off64_t start; /* where the gzip data started, for rewinding */ + int eof; /* true if end of input file reached */ + int past; /* true if read requested past end */ + /* just for writing */ + int level; /* compression level */ + int strategy; /* compression strategy */ + /* seek request */ + z_off64_t skip; /* amount to skip (already rewound if backwards) */ + int seek; /* true if seek request pending */ + /* error information */ + int err; /* error code */ + char *msg; /* error message */ + /* zlib inflate or deflate stream */ + z_stream strm; /* stream structure in-place (not a pointer) */ +} gz_state; +typedef gz_state FAR *gz_statep; + +/* shared functions */ +void ZLIB_INTERNAL gz_error OF((gz_statep, int, const char *)); +#if defined UNDER_CE +char ZLIB_INTERNAL *gz_strwinerror OF((DWORD error)); +#endif + +/* GT_OFF(x), where x is an unsigned value, is true if x > maximum z_off64_t + value -- needed when comparing unsigned to z_off64_t, which is signed + (possible z_off64_t types off_t, off64_t, and long are all signed) */ +#ifdef INT_MAX +# define GT_OFF(x) (sizeof(int) == sizeof(z_off64_t) && (x) > INT_MAX) +#else +unsigned ZLIB_INTERNAL gz_intmax OF((void)); +# define GT_OFF(x) (sizeof(int) == sizeof(z_off64_t) && (x) > gz_intmax()) +#endif diff --git a/third_party/zlib/gzlib.c b/third_party/zlib/gzlib.c new file mode 100644 index 00000000..4105e6af --- /dev/null +++ b/third_party/zlib/gzlib.c @@ -0,0 +1,637 @@ +/* gzlib.c -- zlib functions common to reading and writing gzip files + * Copyright (C) 2004-2017 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +#include "gzguts.h" + +#if defined(_WIN32) && !defined(__BORLANDC__) && !defined(__MINGW32__) +# define LSEEK _lseeki64 +#else +#if defined(_LARGEFILE64_SOURCE) && _LFS64_LARGEFILE-0 +# define LSEEK lseek64 +#else +# define LSEEK lseek +#endif +#endif + +/* Local functions */ +local void gz_reset OF((gz_statep)); +local gzFile gz_open OF((const void *, int, const char *)); + +#if defined UNDER_CE + +/* Map the Windows error number in ERROR to a locale-dependent error message + string and return a pointer to it. Typically, the values for ERROR come + from GetLastError. + + The string pointed to shall not be modified by the application, but may be + overwritten by a subsequent call to gz_strwinerror + + The gz_strwinerror function does not change the current setting of + GetLastError. */ +char ZLIB_INTERNAL *gz_strwinerror (error) + DWORD error; +{ + static char buf[1024]; + + wchar_t *msgbuf; + DWORD lasterr = GetLastError(); + DWORD chars = FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM + | FORMAT_MESSAGE_ALLOCATE_BUFFER, + NULL, + error, + 0, /* Default language */ + (LPVOID)&msgbuf, + 0, + NULL); + if (chars != 0) { + /* If there is an \r\n appended, zap it. */ + if (chars >= 2 + && msgbuf[chars - 2] == '\r' && msgbuf[chars - 1] == '\n') { + chars -= 2; + msgbuf[chars] = 0; + } + + if (chars > sizeof (buf) - 1) { + chars = sizeof (buf) - 1; + msgbuf[chars] = 0; + } + + wcstombs(buf, msgbuf, chars + 1); + LocalFree(msgbuf); + } + else { + sprintf(buf, "unknown win32 error (%ld)", error); + } + + SetLastError(lasterr); + return buf; +} + +#endif /* UNDER_CE */ + +/* Reset gzip file state */ +local void gz_reset(state) + gz_statep state; +{ + state->x.have = 0; /* no output data available */ + if (state->mode == GZ_READ) { /* for reading ... */ + state->eof = 0; /* not at end of file */ + state->past = 0; /* have not read past end yet */ + state->how = LOOK; /* look for gzip header */ + } + state->seek = 0; /* no seek request pending */ + gz_error(state, Z_OK, NULL); /* clear error */ + state->x.pos = 0; /* no uncompressed data yet */ + state->strm.avail_in = 0; /* no input data yet */ +} + +/* Open a gzip file either by name or file descriptor. */ +local gzFile gz_open(path, fd, mode) + const void *path; + int fd; + const char *mode; +{ + gz_statep state; + z_size_t len; + int oflag; +#ifdef O_CLOEXEC + int cloexec = 0; +#endif +#ifdef O_EXCL + int exclusive = 0; +#endif + + /* check input */ + if (path == NULL) + return NULL; + + /* allocate gzFile structure to return */ + state = (gz_statep)malloc(sizeof(gz_state)); + if (state == NULL) + return NULL; + state->size = 0; /* no buffers allocated yet */ + state->want = GZBUFSIZE; /* requested buffer size */ + state->msg = NULL; /* no error message yet */ + + /* interpret mode */ + state->mode = GZ_NONE; + state->level = Z_DEFAULT_COMPRESSION; + state->strategy = Z_DEFAULT_STRATEGY; + state->direct = 0; + while (*mode) { + if (*mode >= '0' && *mode <= '9') + state->level = *mode - '0'; + else + switch (*mode) { + case 'r': + state->mode = GZ_READ; + break; +#ifndef NO_GZCOMPRESS + case 'w': + state->mode = GZ_WRITE; + break; + case 'a': + state->mode = GZ_APPEND; + break; +#endif + case '+': /* can't read and write at the same time */ + free(state); + return NULL; + case 'b': /* ignore -- will request binary anyway */ + break; +#ifdef O_CLOEXEC + case 'e': + cloexec = 1; + break; +#endif +#ifdef O_EXCL + case 'x': + exclusive = 1; + break; +#endif + case 'f': + state->strategy = Z_FILTERED; + break; + case 'h': + state->strategy = Z_HUFFMAN_ONLY; + break; + case 'R': + state->strategy = Z_RLE; + break; + case 'F': + state->strategy = Z_FIXED; + break; + case 'T': + state->direct = 1; + break; + default: /* could consider as an error, but just ignore */ + ; + } + mode++; + } + + /* must provide an "r", "w", or "a" */ + if (state->mode == GZ_NONE) { + free(state); + return NULL; + } + + /* can't force transparent read */ + if (state->mode == GZ_READ) { + if (state->direct) { + free(state); + return NULL; + } + state->direct = 1; /* for empty file */ + } + + /* save the path name for error messages */ +#ifdef WIDECHAR + if (fd == -2) { + len = wcstombs(NULL, path, 0); + if (len == (z_size_t)-1) + len = 0; + } + else +#endif + len = strlen((const char *)path); + state->path = (char *)malloc(len + 1); + if (state->path == NULL) { + free(state); + return NULL; + } +#ifdef WIDECHAR + if (fd == -2) + if (len) + wcstombs(state->path, path, len + 1); + else + *(state->path) = 0; + else +#endif +#if !defined(NO_snprintf) && !defined(NO_vsnprintf) + (void)snprintf(state->path, len + 1, "%s", (const char *)path); +#else + strcpy(state->path, path); +#endif + + /* compute the flags for open() */ + oflag = +#ifdef O_LARGEFILE + O_LARGEFILE | +#endif +#ifdef O_BINARY + O_BINARY | +#endif +#ifdef O_CLOEXEC + (cloexec ? O_CLOEXEC : 0) | +#endif + (state->mode == GZ_READ ? + O_RDONLY : + (O_WRONLY | O_CREAT | +#ifdef O_EXCL + (exclusive ? O_EXCL : 0) | +#endif + (state->mode == GZ_WRITE ? + O_TRUNC : + O_APPEND))); + + /* open the file with the appropriate flags (or just use fd) */ + state->fd = fd > -1 ? fd : ( +#ifdef WIDECHAR + fd == -2 ? _wopen(path, oflag, 0666) : +#endif + open((const char *)path, oflag, 0666)); + if (state->fd == -1) { + free(state->path); + free(state); + return NULL; + } + if (state->mode == GZ_APPEND) { + LSEEK(state->fd, 0, SEEK_END); /* so gzoffset() is correct */ + state->mode = GZ_WRITE; /* simplify later checks */ + } + + /* save the current position for rewinding (only if reading) */ + if (state->mode == GZ_READ) { + state->start = LSEEK(state->fd, 0, SEEK_CUR); + if (state->start == -1) state->start = 0; + } + + /* initialize stream */ + gz_reset(state); + + /* return stream */ + return (gzFile)state; +} + +/* -- see zlib.h -- */ +gzFile ZEXPORT gzopen(path, mode) + const char *path; + const char *mode; +{ + return gz_open(path, -1, mode); +} + +/* -- see zlib.h -- */ +gzFile ZEXPORT gzopen64(path, mode) + const char *path; + const char *mode; +{ + return gz_open(path, -1, mode); +} + +/* -- see zlib.h -- */ +gzFile ZEXPORT gzdopen(fd, mode) + int fd; + const char *mode; +{ + char *path; /* identifier for error messages */ + gzFile gz; + + if (fd == -1 || (path = (char *)malloc(7 + 3 * sizeof(int))) == NULL) + return NULL; +#if !defined(NO_snprintf) && !defined(NO_vsnprintf) + (void)snprintf(path, 7 + 3 * sizeof(int), "<fd:%d>", fd); +#else + sprintf(path, "<fd:%d>", fd); /* for debugging */ +#endif + gz = gz_open(path, fd, mode); + free(path); + return gz; +} + +/* -- see zlib.h -- */ +#ifdef WIDECHAR +gzFile ZEXPORT gzopen_w(path, mode) + const wchar_t *path; + const char *mode; +{ + return gz_open(path, -2, mode); +} +#endif + +/* -- see zlib.h -- */ +int ZEXPORT gzbuffer(file, size) + gzFile file; + unsigned size; +{ + gz_statep state; + + /* get internal structure and check integrity */ + if (file == NULL) + return -1; + state = (gz_statep)file; + if (state->mode != GZ_READ && state->mode != GZ_WRITE) + return -1; + + /* make sure we haven't already allocated memory */ + if (state->size != 0) + return -1; + + /* check and set requested size */ + if ((size << 1) < size) + return -1; /* need to be able to double it */ + if (size < 2) + size = 2; /* need two bytes to check magic header */ + state->want = size; + return 0; +} + +/* -- see zlib.h -- */ +int ZEXPORT gzrewind(file) + gzFile file; +{ + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return -1; + state = (gz_statep)file; + + /* check that we're reading and that there's no error */ + if (state->mode != GZ_READ || + (state->err != Z_OK && state->err != Z_BUF_ERROR)) + return -1; + + /* back up and start over */ + if (LSEEK(state->fd, state->start, SEEK_SET) == -1) + return -1; + gz_reset(state); + return 0; +} + +/* -- see zlib.h -- */ +z_off64_t ZEXPORT gzseek64(file, offset, whence) + gzFile file; + z_off64_t offset; + int whence; +{ + unsigned n; + z_off64_t ret; + gz_statep state; + + /* get internal structure and check integrity */ + if (file == NULL) + return -1; + state = (gz_statep)file; + if (state->mode != GZ_READ && state->mode != GZ_WRITE) + return -1; + + /* check that there's no error */ + if (state->err != Z_OK && state->err != Z_BUF_ERROR) + return -1; + + /* can only seek from start or relative to current position */ + if (whence != SEEK_SET && whence != SEEK_CUR) + return -1; + + /* normalize offset to a SEEK_CUR specification */ + if (whence == SEEK_SET) + offset -= state->x.pos; + else if (state->seek) + offset += state->skip; + state->seek = 0; + + /* if within raw area while reading, just go there */ + if (state->mode == GZ_READ && state->how == COPY && + state->x.pos + offset >= 0) { + ret = LSEEK(state->fd, offset - state->x.have, SEEK_CUR); + if (ret == -1) + return -1; + state->x.have = 0; + state->eof = 0; + state->past = 0; + state->seek = 0; + gz_error(state, Z_OK, NULL); + state->strm.avail_in = 0; + state->x.pos += offset; + return state->x.pos; + } + + /* calculate skip amount, rewinding if needed for back seek when reading */ + if (offset < 0) { + if (state->mode != GZ_READ) /* writing -- can't go backwards */ + return -1; + offset += state->x.pos; + if (offset < 0) /* before start of file! */ + return -1; + if (gzrewind(file) == -1) /* rewind, then skip to offset */ + return -1; + } + + /* if reading, skip what's in output buffer (one less gzgetc() check) */ + if (state->mode == GZ_READ) { + n = GT_OFF(state->x.have) || (z_off64_t)state->x.have > offset ? + (unsigned)offset : state->x.have; + state->x.have -= n; + state->x.next += n; + state->x.pos += n; + offset -= n; + } + + /* request skip (if not zero) */ + if (offset) { + state->seek = 1; + state->skip = offset; + } + return state->x.pos + offset; +} + +/* -- see zlib.h -- */ +z_off_t ZEXPORT gzseek(file, offset, whence) + gzFile file; + z_off_t offset; + int whence; +{ + z_off64_t ret; + + ret = gzseek64(file, (z_off64_t)offset, whence); + return ret == (z_off_t)ret ? (z_off_t)ret : -1; +} + +/* -- see zlib.h -- */ +z_off64_t ZEXPORT gztell64(file) + gzFile file; +{ + gz_statep state; + + /* get internal structure and check integrity */ + if (file == NULL) + return -1; + state = (gz_statep)file; + if (state->mode != GZ_READ && state->mode != GZ_WRITE) + return -1; + + /* return position */ + return state->x.pos + (state->seek ? state->skip : 0); +} + +/* -- see zlib.h -- */ +z_off_t ZEXPORT gztell(file) + gzFile file; +{ + z_off64_t ret; + + ret = gztell64(file); + return ret == (z_off_t)ret ? (z_off_t)ret : -1; +} + +/* -- see zlib.h -- */ +z_off64_t ZEXPORT gzoffset64(file) + gzFile file; +{ + z_off64_t offset; + gz_statep state; + + /* get internal structure and check integrity */ + if (file == NULL) + return -1; + state = (gz_statep)file; + if (state->mode != GZ_READ && state->mode != GZ_WRITE) + return -1; + + /* compute and return effective offset in file */ + offset = LSEEK(state->fd, 0, SEEK_CUR); + if (offset == -1) + return -1; + if (state->mode == GZ_READ) /* reading */ + offset -= state->strm.avail_in; /* don't count buffered input */ + return offset; +} + +/* -- see zlib.h -- */ +z_off_t ZEXPORT gzoffset(file) + gzFile file; +{ + z_off64_t ret; + + ret = gzoffset64(file); + return ret == (z_off_t)ret ? (z_off_t)ret : -1; +} + +/* -- see zlib.h -- */ +int ZEXPORT gzeof(file) + gzFile file; +{ + gz_statep state; + + /* get internal structure and check integrity */ + if (file == NULL) + return 0; + state = (gz_statep)file; + if (state->mode != GZ_READ && state->mode != GZ_WRITE) + return 0; + + /* return end-of-file state */ + return state->mode == GZ_READ ? state->past : 0; +} + +/* -- see zlib.h -- */ +const char * ZEXPORT gzerror(file, errnum) + gzFile file; + int *errnum; +{ + gz_statep state; + + /* get internal structure and check integrity */ + if (file == NULL) + return NULL; + state = (gz_statep)file; + if (state->mode != GZ_READ && state->mode != GZ_WRITE) + return NULL; + + /* return error information */ + if (errnum != NULL) + *errnum = state->err; + return state->err == Z_MEM_ERROR ? "out of memory" : + (state->msg == NULL ? "" : state->msg); +} + +/* -- see zlib.h -- */ +void ZEXPORT gzclearerr(file) + gzFile file; +{ + gz_statep state; + + /* get internal structure and check integrity */ + if (file == NULL) + return; + state = (gz_statep)file; + if (state->mode != GZ_READ && state->mode != GZ_WRITE) + return; + + /* clear error and end-of-file */ + if (state->mode == GZ_READ) { + state->eof = 0; + state->past = 0; + } + gz_error(state, Z_OK, NULL); +} + +/* Create an error message in allocated memory and set state->err and + state->msg accordingly. Free any previous error message already there. Do + not try to free or allocate space if the error is Z_MEM_ERROR (out of + memory). Simply save the error message as a static string. If there is an + allocation failure constructing the error message, then convert the error to + out of memory. */ +void ZLIB_INTERNAL gz_error(state, err, msg) + gz_statep state; + int err; + const char *msg; +{ + /* free previously allocated message and clear */ + if (state->msg != NULL) { + if (state->err != Z_MEM_ERROR) + free(state->msg); + state->msg = NULL; + } + + /* if fatal, set state->x.have to 0 so that the gzgetc() macro fails */ + if (err != Z_OK && err != Z_BUF_ERROR) + state->x.have = 0; + + /* set error code, and if no message, then done */ + state->err = err; + if (msg == NULL) + return; + + /* for an out of memory error, return literal string when requested */ + if (err == Z_MEM_ERROR) + return; + + /* construct error message with path */ + if ((state->msg = (char *)malloc(strlen(state->path) + strlen(msg) + 3)) == + NULL) { + state->err = Z_MEM_ERROR; + return; + } +#if !defined(NO_snprintf) && !defined(NO_vsnprintf) + (void)snprintf(state->msg, strlen(state->path) + strlen(msg) + 3, + "%s%s%s", state->path, ": ", msg); +#else + strcpy(state->msg, state->path); + strcat(state->msg, ": "); + strcat(state->msg, msg); +#endif +} + +#ifndef INT_MAX +/* portably return maximum value for an int (when limits.h presumed not + available) -- we need to do this to cover cases where 2's complement not + used, since C standard permits 1's complement and sign-bit representations, + otherwise we could just use ((unsigned)-1) >> 1 */ +unsigned ZLIB_INTERNAL gz_intmax() +{ + unsigned p, q; + + p = 1; + do { + q = p; + p <<= 1; + p++; + } while (p > q); + return q >> 1; +} +#endif diff --git a/third_party/zlib/gzread.c b/third_party/zlib/gzread.c new file mode 100644 index 00000000..832d3ef9 --- /dev/null +++ b/third_party/zlib/gzread.c @@ -0,0 +1,658 @@ +/* gzread.c -- zlib functions for reading gzip files + * Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013, 2016 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +#include "gzguts.h" + +/* Local functions */ +local int gz_load OF((gz_statep, unsigned char *, unsigned, unsigned *)); +local int gz_avail OF((gz_statep)); +local int gz_look OF((gz_statep)); +local int gz_decomp OF((gz_statep)); +local int gz_fetch OF((gz_statep)); +local int gz_skip OF((gz_statep, z_off64_t)); +local z_size_t gz_read OF((gz_statep, voidp, z_size_t)); + +/* Use read() to load a buffer -- return -1 on error, otherwise 0. Read from + state->fd, and update state->eof, state->err, and state->msg as appropriate. + This function needs to loop on read(), since read() is not guaranteed to + read the number of bytes requested, depending on the type of descriptor. */ +local int gz_load(state, buf, len, have) + gz_statep state; + unsigned char *buf; + unsigned len; + unsigned *have; +{ + int ret; + unsigned get, max = ((unsigned)-1 >> 2) + 1; + + *have = 0; + do { + get = len - *have; + if (get > max) + get = max; + ret = read(state->fd, buf + *have, get); + if (ret <= 0) + break; + *have += (unsigned)ret; + } while (*have < len); + if (ret < 0) { + gz_error(state, Z_ERRNO, zstrerror()); + return -1; + } + if (ret == 0) + state->eof = 1; + return 0; +} + +/* Load up input buffer and set eof flag if last data loaded -- return -1 on + error, 0 otherwise. Note that the eof flag is set when the end of the input + file is reached, even though there may be unused data in the buffer. Once + that data has been used, no more attempts will be made to read the file. + If strm->avail_in != 0, then the current data is moved to the beginning of + the input buffer, and then the remainder of the buffer is loaded with the + available data from the input file. */ +local int gz_avail(state) + gz_statep state; +{ + unsigned got; + z_streamp strm = &(state->strm); + + if (state->err != Z_OK && state->err != Z_BUF_ERROR) + return -1; + if (state->eof == 0) { + if (strm->avail_in) { /* copy what's there to the start */ + unsigned char *p = state->in; + unsigned const char *q = strm->next_in; + unsigned n = strm->avail_in; + do { + *p++ = *q++; + } while (--n); + } + if (gz_load(state, state->in + strm->avail_in, + state->size - strm->avail_in, &got) == -1) + return -1; + strm->avail_in += got; + strm->next_in = state->in; + } + return 0; +} + +/* Look for gzip header, set up for inflate or copy. state->x.have must be 0. + If this is the first time in, allocate required memory. state->how will be + left unchanged if there is no more input data available, will be set to COPY + if there is no gzip header and direct copying will be performed, or it will + be set to GZIP for decompression. If direct copying, then leftover input + data from the input buffer will be copied to the output buffer. In that + case, all further file reads will be directly to either the output buffer or + a user buffer. If decompressing, the inflate state will be initialized. + gz_look() will return 0 on success or -1 on failure. */ +local int gz_look(state) + gz_statep state; +{ + z_streamp strm = &(state->strm); + + /* allocate read buffers and inflate memory */ + if (state->size == 0) { + /* allocate buffers */ + state->in = (unsigned char *)malloc(state->want); + state->out = (unsigned char *)malloc(state->want << 1); + if (state->in == NULL || state->out == NULL) { + free(state->out); + free(state->in); + gz_error(state, Z_MEM_ERROR, "out of memory"); + return -1; + } + state->size = state->want; + + /* allocate inflate memory */ + state->strm.zalloc = Z_NULL; + state->strm.zfree = Z_NULL; + state->strm.opaque = Z_NULL; + state->strm.avail_in = 0; + state->strm.next_in = Z_NULL; + if (inflateInit2(&(state->strm), 15 + 16) != Z_OK) { /* gunzip */ + free(state->out); + free(state->in); + state->size = 0; + gz_error(state, Z_MEM_ERROR, "out of memory"); + return -1; + } + } + + /* get at least the magic bytes in the input buffer */ + if (strm->avail_in < 2) { + if (gz_avail(state) == -1) + return -1; + if (strm->avail_in == 0) + return 0; + } + + /* look for gzip magic bytes -- if there, do gzip decoding (note: there is + a logical dilemma here when considering the case of a partially written + gzip file, to wit, if a single 31 byte is written, then we cannot tell + whether this is a single-byte file, or just a partially written gzip + file -- for here we assume that if a gzip file is being written, then + the header will be written in a single operation, so that reading a + single byte is sufficient indication that it is not a gzip file) */ + if (strm->avail_in > 1 && + strm->next_in[0] == 31 && strm->next_in[1] == 139) { + inflateReset(strm); + state->how = GZIP; + state->direct = 0; + return 0; + } + + /* no gzip header -- if we were decoding gzip before, then this is trailing + garbage. Ignore the trailing garbage and finish. */ + if (state->direct == 0) { + strm->avail_in = 0; + state->eof = 1; + state->x.have = 0; + return 0; + } + + /* doing raw i/o, copy any leftover input to output -- this assumes that + the output buffer is larger than the input buffer, which also assures + space for gzungetc() */ + state->x.next = state->out; + if (strm->avail_in) { + memcpy(state->x.next, strm->next_in, strm->avail_in); + state->x.have = strm->avail_in; + strm->avail_in = 0; + } + state->how = COPY; + state->direct = 1; + return 0; +} + +/* Decompress from input to the provided next_out and avail_out in the state. + On return, state->x.have and state->x.next point to the just decompressed + data. If the gzip stream completes, state->how is reset to LOOK to look for + the next gzip stream or raw data, once state->x.have is depleted. Returns 0 + on success, -1 on failure. */ +local int gz_decomp(state) + gz_statep state; +{ + int ret = Z_OK; + unsigned had; + z_streamp strm = &(state->strm); + + /* fill output buffer up to end of deflate stream */ + had = strm->avail_out; + do { + /* get more input for inflate() */ + if (strm->avail_in == 0 && gz_avail(state) == -1) + return -1; + if (strm->avail_in == 0) { + gz_error(state, Z_BUF_ERROR, "unexpected end of file"); + break; + } + + /* decompress and handle errors */ + ret = inflate(strm, Z_NO_FLUSH); + if (ret == Z_STREAM_ERROR || ret == Z_NEED_DICT) { + gz_error(state, Z_STREAM_ERROR, + "internal error: inflate stream corrupt"); + return -1; + } + if (ret == Z_MEM_ERROR) { + gz_error(state, Z_MEM_ERROR, "out of memory"); + return -1; + } + if (ret == Z_DATA_ERROR) { /* deflate stream invalid */ + gz_error(state, Z_DATA_ERROR, + strm->msg == NULL ? "compressed data error" : strm->msg); + return -1; + } + } while (strm->avail_out && ret != Z_STREAM_END); + + /* update available output */ + state->x.have = had - strm->avail_out; + state->x.next = strm->next_out - state->x.have; + + /* if the gzip stream completed successfully, look for another */ + if (ret == Z_STREAM_END) + state->how = LOOK; + + /* good decompression */ + return 0; +} + +/* Fetch data and put it in the output buffer. Assumes state->x.have is 0. + Data is either copied from the input file or decompressed from the input + file depending on state->how. If state->how is LOOK, then a gzip header is + looked for to determine whether to copy or decompress. Returns -1 on error, + otherwise 0. gz_fetch() will leave state->how as COPY or GZIP unless the + end of the input file has been reached and all data has been processed. */ +local int gz_fetch(state) + gz_statep state; +{ + z_streamp strm = &(state->strm); + + do { + switch(state->how) { + case LOOK: /* -> LOOK, COPY (only if never GZIP), or GZIP */ + if (gz_look(state) == -1) + return -1; + if (state->how == LOOK) + return 0; + break; + case COPY: /* -> COPY */ + if (gz_load(state, state->out, state->size << 1, &(state->x.have)) + == -1) + return -1; + state->x.next = state->out; + return 0; + case GZIP: /* -> GZIP or LOOK (if end of gzip stream) */ + strm->avail_out = state->size << 1; + strm->next_out = state->out; + if (gz_decomp(state) == -1) + return -1; + } + } while (state->x.have == 0 && (!state->eof || strm->avail_in)); + return 0; +} + +/* Skip len uncompressed bytes of output. Return -1 on error, 0 on success. */ +local int gz_skip(state, len) + gz_statep state; + z_off64_t len; +{ + unsigned n; + + /* skip over len bytes or reach end-of-file, whichever comes first */ + while (len) + /* skip over whatever is in output buffer */ + if (state->x.have) { + n = GT_OFF(state->x.have) || (z_off64_t)state->x.have > len ? + (unsigned)len : state->x.have; + state->x.have -= n; + state->x.next += n; + state->x.pos += n; + len -= n; + } + + /* output buffer empty -- return if we're at the end of the input */ + else if (state->eof && state->strm.avail_in == 0) + break; + + /* need more data to skip -- load up output buffer */ + else { + /* get more output, looking for header if required */ + if (gz_fetch(state) == -1) + return -1; + } + return 0; +} + +/* Read len bytes into buf from file, or less than len up to the end of the + input. Return the number of bytes read. If zero is returned, either the + end of file was reached, or there was an error. state->err must be + consulted in that case to determine which. */ +local z_size_t gz_read(state, buf, len) + gz_statep state; + voidp buf; + z_size_t len; +{ + z_size_t got; + unsigned n; + + /* if len is zero, avoid unnecessary operations */ + if (len == 0) + return 0; + + /* process a skip request */ + if (state->seek) { + state->seek = 0; + if (gz_skip(state, state->skip) == -1) + return 0; + } + + /* get len bytes to buf, or less than len if at the end */ + got = 0; + do { + /* set n to the maximum amount of len that fits in an unsigned int */ + n = -1; + if (n > len) + n = len; + + /* first just try copying data from the output buffer */ + if (state->x.have) { + if (state->x.have < n) + n = state->x.have; + memcpy(buf, state->x.next, n); + state->x.next += n; + state->x.have -= n; + } + + /* output buffer empty -- return if we're at the end of the input */ + else if (state->eof && state->strm.avail_in == 0) { + state->past = 1; /* tried to read past end */ + break; + } + + /* need output data -- for small len or new stream load up our output + buffer */ + else if (state->how == LOOK || n < (state->size << 1)) { + /* get more output, looking for header if required */ + if (gz_fetch(state) == -1) + return 0; + continue; /* no progress yet -- go back to copy above */ + /* the copy above assures that we will leave with space in the + output buffer, allowing at least one gzungetc() to succeed */ + } + + /* large len -- read directly into user buffer */ + else if (state->how == COPY) { /* read directly */ + if (gz_load(state, (unsigned char *)buf, n, &n) == -1) + return 0; + } + + /* large len -- decompress directly into user buffer */ + else { /* state->how == GZIP */ + state->strm.avail_out = n; + state->strm.next_out = (unsigned char *)buf; + if (gz_decomp(state) == -1) + return 0; + n = state->x.have; + state->x.have = 0; + } + + /* update progress */ + len -= n; + buf = (char *)buf + n; + got += n; + state->x.pos += n; + } while (len); + + /* return number of bytes read into user buffer */ + return got; +} + +/* -- see zlib.h -- */ +int ZEXPORT gzread(file, buf, len) + gzFile file; + voidp buf; + unsigned len; +{ + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return -1; + state = (gz_statep)file; + + /* check that we're reading and that there's no (serious) error */ + if (state->mode != GZ_READ || + (state->err != Z_OK && state->err != Z_BUF_ERROR)) + return -1; + + /* since an int is returned, make sure len fits in one, otherwise return + with an error (this avoids a flaw in the interface) */ + if ((int)len < 0) { + gz_error(state, Z_STREAM_ERROR, "request does not fit in an int"); + return -1; + } + + /* read len or fewer bytes to buf */ + len = gz_read(state, buf, len); + + /* check for an error */ + if (len == 0 && state->err != Z_OK && state->err != Z_BUF_ERROR) + return -1; + + /* return the number of bytes read (this is assured to fit in an int) */ + return (int)len; +} + +/* -- see zlib.h -- */ +z_size_t ZEXPORT gzfread(buf, size, nitems, file) + voidp buf; + z_size_t size; + z_size_t nitems; + gzFile file; +{ + z_size_t len; + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return 0; + state = (gz_statep)file; + + /* check that we're reading and that there's no (serious) error */ + if (state->mode != GZ_READ || + (state->err != Z_OK && state->err != Z_BUF_ERROR)) + return 0; + + /* compute bytes to read -- error on overflow */ + len = nitems * size; + if (size && len / size != nitems) { + gz_error(state, Z_STREAM_ERROR, "request does not fit in a size_t"); + return 0; + } + + /* read len or fewer bytes to buf, return the number of full items read */ + return len ? gz_read(state, buf, len) / size : 0; +} + +/* -- see zlib.h -- */ +#ifdef Z_PREFIX_SET +# undef z_gzgetc +#else +# undef gzgetc +# ifdef Z_CR_PREFIX_SET +# define gzgetc Cr_z_gzgetc +# endif +#endif + +int ZEXPORT gzgetc(file) + gzFile file; +{ + int ret; + unsigned char buf[1]; + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return -1; + state = (gz_statep)file; + + /* check that we're reading and that there's no (serious) error */ + if (state->mode != GZ_READ || + (state->err != Z_OK && state->err != Z_BUF_ERROR)) + return -1; + + /* try output buffer (no need to check for skip request) */ + if (state->x.have) { + state->x.have--; + state->x.pos++; + return *(state->x.next)++; + } + + /* nothing there -- try gz_read() */ + ret = gz_read(state, buf, 1); + return ret < 1 ? -1 : buf[0]; +} + +int ZEXPORT gzgetc_(file) +gzFile file; +{ + return gzgetc(file); +} + +/* -- see zlib.h -- */ +int ZEXPORT gzungetc(c, file) + int c; + gzFile file; +{ + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return -1; + state = (gz_statep)file; + + /* check that we're reading and that there's no (serious) error */ + if (state->mode != GZ_READ || + (state->err != Z_OK && state->err != Z_BUF_ERROR)) + return -1; + + /* process a skip request */ + if (state->seek) { + state->seek = 0; + if (gz_skip(state, state->skip) == -1) + return -1; + } + + /* can't push EOF */ + if (c < 0) + return -1; + + /* if output buffer empty, put byte at end (allows more pushing) */ + if (state->x.have == 0) { + state->x.have = 1; + state->x.next = state->out + (state->size << 1) - 1; + state->x.next[0] = (unsigned char)c; + state->x.pos--; + state->past = 0; + return c; + } + + /* if no room, give up (must have already done a gzungetc()) */ + if (state->x.have == (state->size << 1)) { + gz_error(state, Z_DATA_ERROR, "out of room to push characters"); + return -1; + } + + /* slide output data if needed and insert byte before existing data */ + if (state->x.next == state->out) { + unsigned char *src = state->out + state->x.have; + unsigned char *dest = state->out + (state->size << 1); + while (src > state->out) + *--dest = *--src; + state->x.next = dest; + } + state->x.have++; + state->x.next--; + state->x.next[0] = (unsigned char)c; + state->x.pos--; + state->past = 0; + return c; +} + +/* -- see zlib.h -- */ +char * ZEXPORT gzgets(file, buf, len) + gzFile file; + char *buf; + int len; +{ + unsigned left, n; + char *str; + unsigned char *eol; + gz_statep state; + + /* check parameters and get internal structure */ + if (file == NULL || buf == NULL || len < 1) + return NULL; + state = (gz_statep)file; + + /* check that we're reading and that there's no (serious) error */ + if (state->mode != GZ_READ || + (state->err != Z_OK && state->err != Z_BUF_ERROR)) + return NULL; + + /* process a skip request */ + if (state->seek) { + state->seek = 0; + if (gz_skip(state, state->skip) == -1) + return NULL; + } + + /* copy output bytes up to new line or len - 1, whichever comes first -- + append a terminating zero to the string (we don't check for a zero in + the contents, let the user worry about that) */ + str = buf; + left = (unsigned)len - 1; + if (left) do { + /* assure that something is in the output buffer */ + if (state->x.have == 0 && gz_fetch(state) == -1) + return NULL; /* error */ + if (state->x.have == 0) { /* end of file */ + state->past = 1; /* read past end */ + break; /* return what we have */ + } + + /* look for end-of-line in current output buffer */ + n = state->x.have > left ? left : state->x.have; + eol = (unsigned char *)memchr(state->x.next, '\n', n); + if (eol != NULL) + n = (unsigned)(eol - state->x.next) + 1; + + /* copy through end-of-line, or remainder if not found */ + memcpy(buf, state->x.next, n); + state->x.have -= n; + state->x.next += n; + state->x.pos += n; + left -= n; + buf += n; + } while (left && eol == NULL); + + /* return terminated string, or if nothing, end of file */ + if (buf == str) + return NULL; + buf[0] = 0; + return str; +} + +/* -- see zlib.h -- */ +int ZEXPORT gzdirect(file) + gzFile file; +{ + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return 0; + state = (gz_statep)file; + + /* if the state is not known, but we can find out, then do so (this is + mainly for right after a gzopen() or gzdopen()) */ + if (state->mode == GZ_READ && state->how == LOOK && state->x.have == 0) + (void)gz_look(state); + + /* return 1 if transparent, 0 if processing a gzip stream */ + return state->direct; +} + +/* -- see zlib.h -- */ +int ZEXPORT gzclose_r(file) + gzFile file; +{ + int ret, err; + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return Z_STREAM_ERROR; + state = (gz_statep)file; + + /* check that we're reading */ + if (state->mode != GZ_READ) + return Z_STREAM_ERROR; + + /* free memory and close file */ + if (state->size) { + inflateEnd(&(state->strm)); + free(state->out); + free(state->in); + } + err = state->err == Z_BUF_ERROR ? Z_BUF_ERROR : Z_OK; + gz_error(state, Z_OK, NULL); + free(state->path); + ret = close(state->fd); + free(state); + return ret ? Z_ERRNO : err; +} diff --git a/third_party/zlib/gzwrite.c b/third_party/zlib/gzwrite.c new file mode 100644 index 00000000..c7b5651d --- /dev/null +++ b/third_party/zlib/gzwrite.c @@ -0,0 +1,665 @@ +/* gzwrite.c -- zlib functions for writing gzip files + * Copyright (C) 2004-2017 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +#include "gzguts.h" + +/* Local functions */ +local int gz_init OF((gz_statep)); +local int gz_comp OF((gz_statep, int)); +local int gz_zero OF((gz_statep, z_off64_t)); +local z_size_t gz_write OF((gz_statep, voidpc, z_size_t)); + +/* Initialize state for writing a gzip file. Mark initialization by setting + state->size to non-zero. Return -1 on a memory allocation failure, or 0 on + success. */ +local int gz_init(state) + gz_statep state; +{ + int ret; + z_streamp strm = &(state->strm); + + /* allocate input buffer (double size for gzprintf) */ + state->in = (unsigned char *)malloc(state->want << 1); + if (state->in == NULL) { + gz_error(state, Z_MEM_ERROR, "out of memory"); + return -1; + } + + /* only need output buffer and deflate state if compressing */ + if (!state->direct) { + /* allocate output buffer */ + state->out = (unsigned char *)malloc(state->want); + if (state->out == NULL) { + free(state->in); + gz_error(state, Z_MEM_ERROR, "out of memory"); + return -1; + } + + /* allocate deflate memory, set up for gzip compression */ + strm->zalloc = Z_NULL; + strm->zfree = Z_NULL; + strm->opaque = Z_NULL; + ret = deflateInit2(strm, state->level, Z_DEFLATED, + MAX_WBITS + 16, DEF_MEM_LEVEL, state->strategy); + if (ret != Z_OK) { + free(state->out); + free(state->in); + gz_error(state, Z_MEM_ERROR, "out of memory"); + return -1; + } + strm->next_in = NULL; + } + + /* mark state as initialized */ + state->size = state->want; + + /* initialize write buffer if compressing */ + if (!state->direct) { + strm->avail_out = state->size; + strm->next_out = state->out; + state->x.next = strm->next_out; + } + return 0; +} + +/* Compress whatever is at avail_in and next_in and write to the output file. + Return -1 if there is an error writing to the output file or if gz_init() + fails to allocate memory, otherwise 0. flush is assumed to be a valid + deflate() flush value. If flush is Z_FINISH, then the deflate() state is + reset to start a new gzip stream. If gz->direct is true, then simply write + to the output file without compressing, and ignore flush. */ +local int gz_comp(state, flush) + gz_statep state; + int flush; +{ + int ret, writ; + unsigned have, put, max = ((unsigned)-1 >> 2) + 1; + z_streamp strm = &(state->strm); + + /* allocate memory if this is the first time through */ + if (state->size == 0 && gz_init(state) == -1) + return -1; + + /* write directly if requested */ + if (state->direct) { + while (strm->avail_in) { + put = strm->avail_in > max ? max : strm->avail_in; + writ = write(state->fd, strm->next_in, put); + if (writ < 0) { + gz_error(state, Z_ERRNO, zstrerror()); + return -1; + } + strm->avail_in -= (unsigned)writ; + strm->next_in += writ; + } + return 0; + } + + /* run deflate() on provided input until it produces no more output */ + ret = Z_OK; + do { + /* write out current buffer contents if full, or if flushing, but if + doing Z_FINISH then don't write until we get to Z_STREAM_END */ + if (strm->avail_out == 0 || (flush != Z_NO_FLUSH && + (flush != Z_FINISH || ret == Z_STREAM_END))) { + while (strm->next_out > state->x.next) { + put = strm->next_out - state->x.next > (int)max ? max : + (unsigned)(strm->next_out - state->x.next); + writ = write(state->fd, state->x.next, put); + if (writ < 0) { + gz_error(state, Z_ERRNO, zstrerror()); + return -1; + } + state->x.next += writ; + } + if (strm->avail_out == 0) { + strm->avail_out = state->size; + strm->next_out = state->out; + state->x.next = state->out; + } + } + + /* compress */ + have = strm->avail_out; + ret = deflate(strm, flush); + if (ret == Z_STREAM_ERROR) { + gz_error(state, Z_STREAM_ERROR, + "internal error: deflate stream corrupt"); + return -1; + } + have -= strm->avail_out; + } while (have); + + /* if that completed a deflate stream, allow another to start */ + if (flush == Z_FINISH) + deflateReset(strm); + + /* all done, no errors */ + return 0; +} + +/* Compress len zeros to output. Return -1 on a write error or memory + allocation failure by gz_comp(), or 0 on success. */ +local int gz_zero(state, len) + gz_statep state; + z_off64_t len; +{ + int first; + unsigned n; + z_streamp strm = &(state->strm); + + /* consume whatever's left in the input buffer */ + if (strm->avail_in && gz_comp(state, Z_NO_FLUSH) == -1) + return -1; + + /* compress len zeros (len guaranteed > 0) */ + first = 1; + while (len) { + n = GT_OFF(state->size) || (z_off64_t)state->size > len ? + (unsigned)len : state->size; + if (first) { + memset(state->in, 0, n); + first = 0; + } + strm->avail_in = n; + strm->next_in = state->in; + state->x.pos += n; + if (gz_comp(state, Z_NO_FLUSH) == -1) + return -1; + len -= n; + } + return 0; +} + +/* Write len bytes from buf to file. Return the number of bytes written. If + the returned value is less than len, then there was an error. */ +local z_size_t gz_write(state, buf, len) + gz_statep state; + voidpc buf; + z_size_t len; +{ + z_size_t put = len; + + /* if len is zero, avoid unnecessary operations */ + if (len == 0) + return 0; + + /* allocate memory if this is the first time through */ + if (state->size == 0 && gz_init(state) == -1) + return 0; + + /* check for seek request */ + if (state->seek) { + state->seek = 0; + if (gz_zero(state, state->skip) == -1) + return 0; + } + + /* for small len, copy to input buffer, otherwise compress directly */ + if (len < state->size) { + /* copy to input buffer, compress when full */ + do { + unsigned have, copy; + + if (state->strm.avail_in == 0) + state->strm.next_in = state->in; + have = (unsigned)((state->strm.next_in + state->strm.avail_in) - + state->in); + copy = state->size - have; + if (copy > len) + copy = len; + memcpy(state->in + have, buf, copy); + state->strm.avail_in += copy; + state->x.pos += copy; + buf = (const char *)buf + copy; + len -= copy; + if (len && gz_comp(state, Z_NO_FLUSH) == -1) + return 0; + } while (len); + } + else { + /* consume whatever's left in the input buffer */ + if (state->strm.avail_in && gz_comp(state, Z_NO_FLUSH) == -1) + return 0; + + /* directly compress user buffer to file */ + state->strm.next_in = (z_const Bytef *)buf; + do { + unsigned n = (unsigned)-1; + if (n > len) + n = len; + state->strm.avail_in = n; + state->x.pos += n; + if (gz_comp(state, Z_NO_FLUSH) == -1) + return 0; + len -= n; + } while (len); + } + + /* input was all buffered or compressed */ + return put; +} + +/* -- see zlib.h -- */ +int ZEXPORT gzwrite(file, buf, len) + gzFile file; + voidpc buf; + unsigned len; +{ + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return 0; + state = (gz_statep)file; + + /* check that we're writing and that there's no error */ + if (state->mode != GZ_WRITE || state->err != Z_OK) + return 0; + + /* since an int is returned, make sure len fits in one, otherwise return + with an error (this avoids a flaw in the interface) */ + if ((int)len < 0) { + gz_error(state, Z_DATA_ERROR, "requested length does not fit in int"); + return 0; + } + + /* write len bytes from buf (the return value will fit in an int) */ + return (int)gz_write(state, buf, len); +} + +/* -- see zlib.h -- */ +z_size_t ZEXPORT gzfwrite(buf, size, nitems, file) + voidpc buf; + z_size_t size; + z_size_t nitems; + gzFile file; +{ + z_size_t len; + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return 0; + state = (gz_statep)file; + + /* check that we're writing and that there's no error */ + if (state->mode != GZ_WRITE || state->err != Z_OK) + return 0; + + /* compute bytes to read -- error on overflow */ + len = nitems * size; + if (size && len / size != nitems) { + gz_error(state, Z_STREAM_ERROR, "request does not fit in a size_t"); + return 0; + } + + /* write len bytes to buf, return the number of full items written */ + return len ? gz_write(state, buf, len) / size : 0; +} + +/* -- see zlib.h -- */ +int ZEXPORT gzputc(file, c) + gzFile file; + int c; +{ + unsigned have; + unsigned char buf[1]; + gz_statep state; + z_streamp strm; + + /* get internal structure */ + if (file == NULL) + return -1; + state = (gz_statep)file; + strm = &(state->strm); + + /* check that we're writing and that there's no error */ + if (state->mode != GZ_WRITE || state->err != Z_OK) + return -1; + + /* check for seek request */ + if (state->seek) { + state->seek = 0; + if (gz_zero(state, state->skip) == -1) + return -1; + } + + /* try writing to input buffer for speed (state->size == 0 if buffer not + initialized) */ + if (state->size) { + if (strm->avail_in == 0) + strm->next_in = state->in; + have = (unsigned)((strm->next_in + strm->avail_in) - state->in); + if (have < state->size) { + state->in[have] = (unsigned char)c; + strm->avail_in++; + state->x.pos++; + return c & 0xff; + } + } + + /* no room in buffer or not initialized, use gz_write() */ + buf[0] = (unsigned char)c; + if (gz_write(state, buf, 1) != 1) + return -1; + return c & 0xff; +} + +/* -- see zlib.h -- */ +int ZEXPORT gzputs(file, str) + gzFile file; + const char *str; +{ + int ret; + z_size_t len; + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return -1; + state = (gz_statep)file; + + /* check that we're writing and that there's no error */ + if (state->mode != GZ_WRITE || state->err != Z_OK) + return -1; + + /* write string */ + len = strlen(str); + ret = gz_write(state, str, len); + return ret == 0 && len != 0 ? -1 : ret; +} + +#if defined(STDC) || defined(Z_HAVE_STDARG_H) +#include <stdarg.h> + +/* -- see zlib.h -- */ +int ZEXPORTVA gzvprintf(gzFile file, const char *format, va_list va) +{ + int len; + unsigned left; + char *next; + gz_statep state; + z_streamp strm; + + /* get internal structure */ + if (file == NULL) + return Z_STREAM_ERROR; + state = (gz_statep)file; + strm = &(state->strm); + + /* check that we're writing and that there's no error */ + if (state->mode != GZ_WRITE || state->err != Z_OK) + return Z_STREAM_ERROR; + + /* make sure we have some buffer space */ + if (state->size == 0 && gz_init(state) == -1) + return state->err; + + /* check for seek request */ + if (state->seek) { + state->seek = 0; + if (gz_zero(state, state->skip) == -1) + return state->err; + } + + /* do the printf() into the input buffer, put length in len -- the input + buffer is double-sized just for this function, so there is guaranteed to + be state->size bytes available after the current contents */ + if (strm->avail_in == 0) + strm->next_in = state->in; + next = (char *)(state->in + (strm->next_in - state->in) + strm->avail_in); + next[state->size - 1] = 0; +#ifdef NO_vsnprintf +# ifdef HAS_vsprintf_void + (void)vsprintf(next, format, va); + for (len = 0; len < state->size; len++) + if (next[len] == 0) break; +# else + len = vsprintf(next, format, va); +# endif +#else +# ifdef HAS_vsnprintf_void + (void)vsnprintf(next, state->size, format, va); + len = strlen(next); +# else + len = vsnprintf(next, state->size, format, va); +# endif +#endif + + /* check that printf() results fit in buffer */ + if (len == 0 || (unsigned)len >= state->size || next[state->size - 1] != 0) + return 0; + + /* update buffer and position, compress first half if past that */ + strm->avail_in += (unsigned)len; + state->x.pos += len; + if (strm->avail_in >= state->size) { + left = strm->avail_in - state->size; + strm->avail_in = state->size; + if (gz_comp(state, Z_NO_FLUSH) == -1) + return state->err; + memcpy(state->in, state->in + state->size, left); + strm->next_in = state->in; + strm->avail_in = left; + } + return len; +} + +int ZEXPORTVA gzprintf(gzFile file, const char *format, ...) +{ + va_list va; + int ret; + + va_start(va, format); + ret = gzvprintf(file, format, va); + va_end(va); + return ret; +} + +#else /* !STDC && !Z_HAVE_STDARG_H */ + +/* -- see zlib.h -- */ +int ZEXPORTVA gzprintf (file, format, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, + a11, a12, a13, a14, a15, a16, a17, a18, a19, a20) + gzFile file; + const char *format; + int a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, + a11, a12, a13, a14, a15, a16, a17, a18, a19, a20; +{ + unsigned len, left; + char *next; + gz_statep state; + z_streamp strm; + + /* get internal structure */ + if (file == NULL) + return Z_STREAM_ERROR; + state = (gz_statep)file; + strm = &(state->strm); + + /* check that can really pass pointer in ints */ + if (sizeof(int) != sizeof(void *)) + return Z_STREAM_ERROR; + + /* check that we're writing and that there's no error */ + if (state->mode != GZ_WRITE || state->err != Z_OK) + return Z_STREAM_ERROR; + + /* make sure we have some buffer space */ + if (state->size == 0 && gz_init(state) == -1) + return state->error; + + /* check for seek request */ + if (state->seek) { + state->seek = 0; + if (gz_zero(state, state->skip) == -1) + return state->error; + } + + /* do the printf() into the input buffer, put length in len -- the input + buffer is double-sized just for this function, so there is guaranteed to + be state->size bytes available after the current contents */ + if (strm->avail_in == 0) + strm->next_in = state->in; + next = (char *)(strm->next_in + strm->avail_in); + next[state->size - 1] = 0; +#ifdef NO_snprintf +# ifdef HAS_sprintf_void + sprintf(next, format, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, + a13, a14, a15, a16, a17, a18, a19, a20); + for (len = 0; len < size; len++) + if (next[len] == 0) + break; +# else + len = sprintf(next, format, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, + a12, a13, a14, a15, a16, a17, a18, a19, a20); +# endif +#else +# ifdef HAS_snprintf_void + snprintf(next, state->size, format, a1, a2, a3, a4, a5, a6, a7, a8, a9, + a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20); + len = strlen(next); +# else + len = snprintf(next, state->size, format, a1, a2, a3, a4, a5, a6, a7, a8, + a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20); +# endif +#endif + + /* check that printf() results fit in buffer */ + if (len == 0 || len >= state->size || next[state->size - 1] != 0) + return 0; + + /* update buffer and position, compress first half if past that */ + strm->avail_in += len; + state->x.pos += len; + if (strm->avail_in >= state->size) { + left = strm->avail_in - state->size; + strm->avail_in = state->size; + if (gz_comp(state, Z_NO_FLUSH) == -1) + return state->err; + memcpy(state->in, state->in + state->size, left); + strm->next_in = state->in; + strm->avail_in = left; + } + return (int)len; +} + +#endif + +/* -- see zlib.h -- */ +int ZEXPORT gzflush(file, flush) + gzFile file; + int flush; +{ + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return Z_STREAM_ERROR; + state = (gz_statep)file; + + /* check that we're writing and that there's no error */ + if (state->mode != GZ_WRITE || state->err != Z_OK) + return Z_STREAM_ERROR; + + /* check flush parameter */ + if (flush < 0 || flush > Z_FINISH) + return Z_STREAM_ERROR; + + /* check for seek request */ + if (state->seek) { + state->seek = 0; + if (gz_zero(state, state->skip) == -1) + return state->err; + } + + /* compress remaining data with requested flush */ + (void)gz_comp(state, flush); + return state->err; +} + +/* -- see zlib.h -- */ +int ZEXPORT gzsetparams(file, level, strategy) + gzFile file; + int level; + int strategy; +{ + gz_statep state; + z_streamp strm; + + /* get internal structure */ + if (file == NULL) + return Z_STREAM_ERROR; + state = (gz_statep)file; + strm = &(state->strm); + + /* check that we're writing and that there's no error */ + if (state->mode != GZ_WRITE || state->err != Z_OK) + return Z_STREAM_ERROR; + + /* if no change is requested, then do nothing */ + if (level == state->level && strategy == state->strategy) + return Z_OK; + + /* check for seek request */ + if (state->seek) { + state->seek = 0; + if (gz_zero(state, state->skip) == -1) + return state->err; + } + + /* change compression parameters for subsequent input */ + if (state->size) { + /* flush previous input with previous parameters before changing */ + if (strm->avail_in && gz_comp(state, Z_BLOCK) == -1) + return state->err; + deflateParams(strm, level, strategy); + } + state->level = level; + state->strategy = strategy; + return Z_OK; +} + +/* -- see zlib.h -- */ +int ZEXPORT gzclose_w(file) + gzFile file; +{ + int ret = Z_OK; + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return Z_STREAM_ERROR; + state = (gz_statep)file; + + /* check that we're writing */ + if (state->mode != GZ_WRITE) + return Z_STREAM_ERROR; + + /* check for seek request */ + if (state->seek) { + state->seek = 0; + if (gz_zero(state, state->skip) == -1) + ret = state->err; + } + + /* flush, free memory, and close file */ + if (gz_comp(state, Z_FINISH) == -1) + ret = state->err; + if (state->size) { + if (!state->direct) { + (void)deflateEnd(&(state->strm)); + free(state->out); + } + free(state->in); + } + gz_error(state, Z_OK, NULL); + free(state->path); + if (close(state->fd) == -1) + ret = Z_ERRNO; + free(state); + return ret; +} diff --git a/third_party/zlib/infback.c b/third_party/zlib/infback.c new file mode 100644 index 00000000..aab14b48 --- /dev/null +++ b/third_party/zlib/infback.c @@ -0,0 +1,641 @@ +/* infback.c -- inflate using a call-back interface + * Copyright (C) 1995-2016 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +/* + This code is largely copied from inflate.c. Normally either infback.o or + inflate.o would be linked into an application--not both. The interface + with inffast.c is retained so that optimized assembler-coded versions of + inflate_fast() can be used with either inflate.c or infback.c. + */ + +#include "zutil.h" +#include "inftrees.h" +#include "inflate.h" +#include "inffast.h" + +/* function prototypes */ +local void fixedtables OF((struct inflate_state FAR *state)); + +/* + strm provides memory allocation functions in zalloc and zfree, or + Z_NULL to use the library memory allocation functions. + + windowBits is in the range 8..15, and window is a user-supplied + window and output buffer that is 2**windowBits bytes. + */ +int ZEXPORT inflateBackInit_(strm, windowBits, window, version, stream_size) +z_streamp strm; +int windowBits; +unsigned char FAR *window; +const char *version; +int stream_size; +{ + struct inflate_state FAR *state; + + if (version == Z_NULL || version[0] != ZLIB_VERSION[0] || + stream_size != (int)(sizeof(z_stream))) + return Z_VERSION_ERROR; + if (strm == Z_NULL || window == Z_NULL || + windowBits < 8 || windowBits > 15) + return Z_STREAM_ERROR; + strm->msg = Z_NULL; /* in case we return an error */ + if (strm->zalloc == (alloc_func)0) { +#ifdef Z_SOLO + return Z_STREAM_ERROR; +#else + strm->zalloc = zcalloc; + strm->opaque = (voidpf)0; +#endif + } + if (strm->zfree == (free_func)0) +#ifdef Z_SOLO + return Z_STREAM_ERROR; +#else + strm->zfree = zcfree; +#endif + state = (struct inflate_state FAR *)ZALLOC(strm, 1, + sizeof(struct inflate_state)); + if (state == Z_NULL) return Z_MEM_ERROR; + Tracev((stderr, "inflate: allocated\n")); + strm->state = (struct internal_state FAR *)state; + state->dmax = 32768U; + state->wbits = (uInt)windowBits; + state->wsize = 1U << windowBits; + state->window = window; + state->wnext = 0; + state->whave = 0; + return Z_OK; +} + +/* + Return state with length and distance decoding tables and index sizes set to + fixed code decoding. Normally this returns fixed tables from inffixed.h. + If BUILDFIXED is defined, then instead this routine builds the tables the + first time it's called, and returns those tables the first time and + thereafter. This reduces the size of the code by about 2K bytes, in + exchange for a little execution time. However, BUILDFIXED should not be + used for threaded applications, since the rewriting of the tables and virgin + may not be thread-safe. + */ +local void fixedtables(state) +struct inflate_state FAR *state; +{ +#ifdef BUILDFIXED + static int virgin = 1; + static code *lenfix, *distfix; + static code fixed[544]; + + /* build fixed huffman tables if first call (may not be thread safe) */ + if (virgin) { + unsigned sym, bits; + static code *next; + + /* literal/length table */ + sym = 0; + while (sym < 144) state->lens[sym++] = 8; + while (sym < 256) state->lens[sym++] = 9; + while (sym < 280) state->lens[sym++] = 7; + while (sym < 288) state->lens[sym++] = 8; + next = fixed; + lenfix = next; + bits = 9; + inflate_table(LENS, state->lens, 288, &(next), &(bits), state->work); + + /* distance table */ + sym = 0; + while (sym < 32) state->lens[sym++] = 5; + distfix = next; + bits = 5; + inflate_table(DISTS, state->lens, 32, &(next), &(bits), state->work); + + /* do this just once */ + virgin = 0; + } +#else /* !BUILDFIXED */ +# include "inffixed.h" +#endif /* BUILDFIXED */ + state->lencode = lenfix; + state->lenbits = 9; + state->distcode = distfix; + state->distbits = 5; +} + +/* Macros for inflateBack(): */ + +/* Load returned state from inflate_fast() */ +#define LOAD() \ + do { \ + put = strm->next_out; \ + left = strm->avail_out; \ + next = strm->next_in; \ + have = strm->avail_in; \ + hold = state->hold; \ + bits = state->bits; \ + } while (0) + +/* Set state from registers for inflate_fast() */ +#define RESTORE() \ + do { \ + strm->next_out = put; \ + strm->avail_out = left; \ + strm->next_in = next; \ + strm->avail_in = have; \ + state->hold = hold; \ + state->bits = bits; \ + } while (0) + +/* Clear the input bit accumulator */ +#define INITBITS() \ + do { \ + hold = 0; \ + bits = 0; \ + } while (0) + +/* Assure that some input is available. If input is requested, but denied, + then return a Z_BUF_ERROR from inflateBack(). */ +#define PULL() \ + do { \ + if (have == 0) { \ + have = in(in_desc, &next); \ + if (have == 0) { \ + next = Z_NULL; \ + ret = Z_BUF_ERROR; \ + goto inf_leave; \ + } \ + } \ + } while (0) + +/* Get a byte of input into the bit accumulator, or return from inflateBack() + with an error if there is no input available. */ +#define PULLBYTE() \ + do { \ + PULL(); \ + have--; \ + hold += (unsigned long)(*next++) << bits; \ + bits += 8; \ + } while (0) + +/* Assure that there are at least n bits in the bit accumulator. If there is + not enough available input to do that, then return from inflateBack() with + an error. */ +#define NEEDBITS(n) \ + do { \ + while (bits < (unsigned)(n)) \ + PULLBYTE(); \ + } while (0) + +/* Return the low n bits of the bit accumulator (n < 16) */ +#define BITS(n) \ + ((unsigned)hold & ((1U << (n)) - 1)) + +/* Remove n bits from the bit accumulator */ +#define DROPBITS(n) \ + do { \ + hold >>= (n); \ + bits -= (unsigned)(n); \ + } while (0) + +/* Remove zero to seven bits as needed to go to a byte boundary */ +#define BYTEBITS() \ + do { \ + hold >>= bits & 7; \ + bits -= bits & 7; \ + } while (0) + +/* Assure that some output space is available, by writing out the window + if it's full. If the write fails, return from inflateBack() with a + Z_BUF_ERROR. */ +#define ROOM() \ + do { \ + if (left == 0) { \ + put = state->window; \ + left = state->wsize; \ + state->whave = left; \ + if (out(out_desc, put, left)) { \ + ret = Z_BUF_ERROR; \ + goto inf_leave; \ + } \ + } \ + } while (0) + +/* + strm provides the memory allocation functions and window buffer on input, + and provides information on the unused input on return. For Z_DATA_ERROR + returns, strm will also provide an error message. + + in() and out() are the call-back input and output functions. When + inflateBack() needs more input, it calls in(). When inflateBack() has + filled the window with output, or when it completes with data in the + window, it calls out() to write out the data. The application must not + change the provided input until in() is called again or inflateBack() + returns. The application must not change the window/output buffer until + inflateBack() returns. + + in() and out() are called with a descriptor parameter provided in the + inflateBack() call. This parameter can be a structure that provides the + information required to do the read or write, as well as accumulated + information on the input and output such as totals and check values. + + in() should return zero on failure. out() should return non-zero on + failure. If either in() or out() fails, than inflateBack() returns a + Z_BUF_ERROR. strm->next_in can be checked for Z_NULL to see whether it + was in() or out() that caused in the error. Otherwise, inflateBack() + returns Z_STREAM_END on success, Z_DATA_ERROR for an deflate format + error, or Z_MEM_ERROR if it could not allocate memory for the state. + inflateBack() can also return Z_STREAM_ERROR if the input parameters + are not correct, i.e. strm is Z_NULL or the state was not initialized. + */ +int ZEXPORT inflateBack(strm, in, in_desc, out, out_desc) +z_streamp strm; +in_func in; +void FAR *in_desc; +out_func out; +void FAR *out_desc; +{ + struct inflate_state FAR *state; + z_const unsigned char FAR *next; /* next input */ + unsigned char FAR *put; /* next output */ + unsigned have, left; /* available input and output */ + unsigned long hold; /* bit buffer */ + unsigned bits; /* bits in bit buffer */ + unsigned copy; /* number of stored or match bytes to copy */ + unsigned char FAR *from; /* where to copy match bytes from */ + code here; /* current decoding table entry */ + code last; /* parent table entry */ + unsigned len; /* length to copy for repeats, bits to drop */ + int ret; /* return code */ + static const unsigned short order[19] = /* permutation of code lengths */ + {16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15}; + + /* Check that the strm exists and that the state was initialized */ + if (strm == Z_NULL || strm->state == Z_NULL) + return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + + /* Reset the state */ + strm->msg = Z_NULL; + state->mode = TYPE; + state->last = 0; + state->whave = 0; + next = strm->next_in; + have = next != Z_NULL ? strm->avail_in : 0; + hold = 0; + bits = 0; + put = state->window; + left = state->wsize; + + /* Inflate until end of block marked as last */ + for (;;) + switch (state->mode) { + case TYPE: + /* determine and dispatch block type */ + if (state->last) { + BYTEBITS(); + state->mode = DONE; + break; + } + NEEDBITS(3); + state->last = BITS(1); + DROPBITS(1); + switch (BITS(2)) { + case 0: /* stored block */ + Tracev((stderr, "inflate: stored block%s\n", + state->last ? " (last)" : "")); + state->mode = STORED; + break; + case 1: /* fixed block */ + fixedtables(state); + Tracev((stderr, "inflate: fixed codes block%s\n", + state->last ? " (last)" : "")); + state->mode = LEN; /* decode codes */ + break; + case 2: /* dynamic block */ + Tracev((stderr, "inflate: dynamic codes block%s\n", + state->last ? " (last)" : "")); + state->mode = TABLE; + break; + case 3: + strm->msg = (char *)"invalid block type"; + state->mode = BAD; + } + DROPBITS(2); + break; + + case STORED: + /* get and verify stored block length */ + BYTEBITS(); /* go to byte boundary */ + NEEDBITS(32); + if ((hold & 0xffff) != ((hold >> 16) ^ 0xffff)) { + strm->msg = (char *)"invalid stored block lengths"; + state->mode = BAD; + break; + } + state->length = (unsigned)hold & 0xffff; + Tracev((stderr, "inflate: stored length %u\n", + state->length)); + INITBITS(); + + /* copy stored block from input to output */ + while (state->length != 0) { + copy = state->length; + PULL(); + ROOM(); + if (copy > have) copy = have; + if (copy > left) copy = left; + zmemcpy(put, next, copy); + have -= copy; + next += copy; + left -= copy; + put += copy; + state->length -= copy; + } + Tracev((stderr, "inflate: stored end\n")); + state->mode = TYPE; + break; + + case TABLE: + /* get dynamic table entries descriptor */ + NEEDBITS(14); + state->nlen = BITS(5) + 257; + DROPBITS(5); + state->ndist = BITS(5) + 1; + DROPBITS(5); + state->ncode = BITS(4) + 4; + DROPBITS(4); +#ifndef PKZIP_BUG_WORKAROUND + if (state->nlen > 286 || state->ndist > 30) { + strm->msg = (char *)"too many length or distance symbols"; + state->mode = BAD; + break; + } +#endif + Tracev((stderr, "inflate: table sizes ok\n")); + + /* get code length code lengths (not a typo) */ + state->have = 0; + while (state->have < state->ncode) { + NEEDBITS(3); + state->lens[order[state->have++]] = (unsigned short)BITS(3); + DROPBITS(3); + } + while (state->have < 19) + state->lens[order[state->have++]] = 0; + state->next = state->codes; + state->lencode = (code const FAR *)(state->next); + state->lenbits = 7; + ret = inflate_table(CODES, state->lens, 19, &(state->next), + &(state->lenbits), state->work); + if (ret) { + strm->msg = (char *)"invalid code lengths set"; + state->mode = BAD; + break; + } + Tracev((stderr, "inflate: code lengths ok\n")); + + /* get length and distance code code lengths */ + state->have = 0; + while (state->have < state->nlen + state->ndist) { + for (;;) { + here = state->lencode[BITS(state->lenbits)]; + if ((unsigned)(here.bits) <= bits) break; + PULLBYTE(); + } + if (here.val < 16) { + DROPBITS(here.bits); + state->lens[state->have++] = here.val; + } + else { + if (here.val == 16) { + NEEDBITS(here.bits + 2); + DROPBITS(here.bits); + if (state->have == 0) { + strm->msg = (char *)"invalid bit length repeat"; + state->mode = BAD; + break; + } + len = (unsigned)(state->lens[state->have - 1]); + copy = 3 + BITS(2); + DROPBITS(2); + } + else if (here.val == 17) { + NEEDBITS(here.bits + 3); + DROPBITS(here.bits); + len = 0; + copy = 3 + BITS(3); + DROPBITS(3); + } + else { + NEEDBITS(here.bits + 7); + DROPBITS(here.bits); + len = 0; + copy = 11 + BITS(7); + DROPBITS(7); + } + if (state->have + copy > state->nlen + state->ndist) { + strm->msg = (char *)"invalid bit length repeat"; + state->mode = BAD; + break; + } + while (copy--) + state->lens[state->have++] = (unsigned short)len; + } + } + + /* handle error breaks in while */ + if (state->mode == BAD) break; + + /* check for end-of-block code (better have one) */ + if (state->lens[256] == 0) { + strm->msg = (char *)"invalid code -- missing end-of-block"; + state->mode = BAD; + break; + } + + /* build code tables -- note: do not change the lenbits or distbits + values here (9 and 6) without reading the comments in inftrees.h + concerning the ENOUGH constants, which depend on those values */ + state->next = state->codes; + state->lencode = (code const FAR *)(state->next); + state->lenbits = 9; + ret = inflate_table(LENS, state->lens, state->nlen, &(state->next), + &(state->lenbits), state->work); + if (ret) { + strm->msg = (char *)"invalid literal/lengths set"; + state->mode = BAD; + break; + } + state->distcode = (code const FAR *)(state->next); + state->distbits = 6; + ret = inflate_table(DISTS, state->lens + state->nlen, state->ndist, + &(state->next), &(state->distbits), state->work); + if (ret) { + strm->msg = (char *)"invalid distances set"; + state->mode = BAD; + break; + } + Tracev((stderr, "inflate: codes ok\n")); + state->mode = LEN; + + case LEN: + /* use inflate_fast() if we have enough input and output */ + if (have >= INFLATE_FAST_MIN_INPUT && + left >= INFLATE_FAST_MIN_OUTPUT) { + RESTORE(); + if (state->whave < state->wsize) + state->whave = state->wsize - left; + inflate_fast(strm, state->wsize); + LOAD(); + break; + } + + /* get a literal, length, or end-of-block code */ + for (;;) { + here = state->lencode[BITS(state->lenbits)]; + if ((unsigned)(here.bits) <= bits) break; + PULLBYTE(); + } + if (here.op && (here.op & 0xf0) == 0) { + last = here; + for (;;) { + here = state->lencode[last.val + + (BITS(last.bits + last.op) >> last.bits)]; + if ((unsigned)(last.bits + here.bits) <= bits) break; + PULLBYTE(); + } + DROPBITS(last.bits); + } + DROPBITS(here.bits); + state->length = (unsigned)here.val; + + /* process literal */ + if (here.op == 0) { + Tracevv((stderr, here.val >= 0x20 && here.val < 0x7f ? + "inflate: literal '%c'\n" : + "inflate: literal 0x%02x\n", here.val)); + ROOM(); + *put++ = (unsigned char)(state->length); + left--; + state->mode = LEN; + break; + } + + /* process end of block */ + if (here.op & 32) { + Tracevv((stderr, "inflate: end of block\n")); + state->mode = TYPE; + break; + } + + /* invalid code */ + if (here.op & 64) { + strm->msg = (char *)"invalid literal/length code"; + state->mode = BAD; + break; + } + + /* length code -- get extra bits, if any */ + state->extra = (unsigned)(here.op) & 15; + if (state->extra != 0) { + NEEDBITS(state->extra); + state->length += BITS(state->extra); + DROPBITS(state->extra); + } + Tracevv((stderr, "inflate: length %u\n", state->length)); + + /* get distance code */ + for (;;) { + here = state->distcode[BITS(state->distbits)]; + if ((unsigned)(here.bits) <= bits) break; + PULLBYTE(); + } + if ((here.op & 0xf0) == 0) { + last = here; + for (;;) { + here = state->distcode[last.val + + (BITS(last.bits + last.op) >> last.bits)]; + if ((unsigned)(last.bits + here.bits) <= bits) break; + PULLBYTE(); + } + DROPBITS(last.bits); + } + DROPBITS(here.bits); + if (here.op & 64) { + strm->msg = (char *)"invalid distance code"; + state->mode = BAD; + break; + } + state->offset = (unsigned)here.val; + + /* get distance extra bits, if any */ + state->extra = (unsigned)(here.op) & 15; + if (state->extra != 0) { + NEEDBITS(state->extra); + state->offset += BITS(state->extra); + DROPBITS(state->extra); + } + if (state->offset > state->wsize - (state->whave < state->wsize ? + left : 0)) { + strm->msg = (char *)"invalid distance too far back"; + state->mode = BAD; + break; + } + Tracevv((stderr, "inflate: distance %u\n", state->offset)); + + /* copy match from window to output */ + do { + ROOM(); + copy = state->wsize - state->offset; + if (copy < left) { + from = put + copy; + copy = left - copy; + } + else { + from = put - state->offset; + copy = left; + } + if (copy > state->length) copy = state->length; + state->length -= copy; + left -= copy; + do { + *put++ = *from++; + } while (--copy); + } while (state->length != 0); + break; + + case DONE: + /* inflate stream terminated properly -- write leftover output */ + ret = Z_STREAM_END; + if (left < state->wsize) { + if (out(out_desc, state->window, state->wsize - left)) + ret = Z_BUF_ERROR; + } + goto inf_leave; + + case BAD: + ret = Z_DATA_ERROR; + goto inf_leave; + + default: /* can't happen, but makes compilers happy */ + ret = Z_STREAM_ERROR; + goto inf_leave; + } + + /* Return unused input */ + inf_leave: + strm->next_in = next; + strm->avail_in = have; + return ret; +} + +int ZEXPORT inflateBackEnd(strm) +z_streamp strm; +{ + if (strm == Z_NULL || strm->state == Z_NULL || strm->zfree == (free_func)0) + return Z_STREAM_ERROR; + ZFREE(strm, strm->state); + strm->state = Z_NULL; + Tracev((stderr, "inflate: end\n")); + return Z_OK; +} diff --git a/third_party/zlib/inffast.c b/third_party/zlib/inffast.c new file mode 100644 index 00000000..2797e8a0 --- /dev/null +++ b/third_party/zlib/inffast.c @@ -0,0 +1,330 @@ +/* inffast.c -- fast decoding + * Copyright (C) 1995-2017 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +#include "zutil.h" +#include "inftrees.h" +#include "inflate.h" +#include "inffast.h" + +#ifdef ASMINF +# pragma message("Assembler code may have bugs -- use at your own risk") +#else + +/* + Decode literal, length, and distance codes and write out the resulting + literal and match bytes until either not enough input or output is + available, an end-of-block is encountered, or a data error is encountered. + When large enough input and output buffers are supplied to inflate(), for + example, a 16K input buffer and a 64K output buffer, more than 95% of the + inflate() execution time is spent in this routine. + + Entry assumptions: + + state->mode == LEN + strm->avail_in >= INFLATE_FAST_MIN_INPUT (6 bytes) + strm->avail_out >= INFLATE_FAST_MIN_OUTPUT (258 bytes) + start >= strm->avail_out + state->bits < 8 + + On return, state->mode is one of: + + LEN -- ran out of enough output space or enough available input + TYPE -- reached end of block code, inflate() to interpret next block + BAD -- error in block data + + Notes: + + INFLATE_FAST_MIN_INPUT: 6 bytes + + - The maximum input bits used by a length/distance pair is 15 bits for the + length code, 5 bits for the length extra, 15 bits for the distance code, + and 13 bits for the distance extra. This totals 48 bits, or six bytes. + Therefore if strm->avail_in >= 6, then there is enough input to avoid + checking for available input while decoding. + + INFLATE_FAST_MIN_OUTPUT: 258 bytes + + - The maximum bytes that a single length/distance pair can output is 258 + bytes, which is the maximum length that can be coded. inflate_fast() + requires strm->avail_out >= 258 for each loop to avoid checking for + available output space while decoding. + */ +void ZLIB_INTERNAL inflate_fast(strm, start) +z_streamp strm; +unsigned start; /* inflate()'s starting value for strm->avail_out */ +{ + struct inflate_state FAR *state; + z_const unsigned char FAR *in; /* local strm->next_in */ + z_const unsigned char FAR *last; /* have enough input while in < last */ + unsigned char FAR *out; /* local strm->next_out */ + unsigned char FAR *beg; /* inflate()'s initial strm->next_out */ + unsigned char FAR *end; /* while out < end, enough space available */ +#ifdef INFLATE_STRICT + unsigned dmax; /* maximum distance from zlib header */ +#endif + unsigned wsize; /* window size or zero if not using window */ + unsigned whave; /* valid bytes in the window */ + unsigned wnext; /* window write index */ + unsigned char FAR *window; /* allocated sliding window, if wsize != 0 */ + unsigned long hold; /* local strm->hold */ + unsigned bits; /* local strm->bits */ + code const FAR *lcode; /* local strm->lencode */ + code const FAR *dcode; /* local strm->distcode */ + unsigned lmask; /* mask for first level of length codes */ + unsigned dmask; /* mask for first level of distance codes */ + code here; /* retrieved table entry */ + unsigned op; /* code bits, operation, extra bits, or */ + /* window position, window bytes to copy */ + unsigned len; /* match length, unused bytes */ + unsigned dist; /* match distance */ + unsigned char FAR *from; /* where to copy match from */ + + /* copy state to local variables */ + state = (struct inflate_state FAR *)strm->state; + in = strm->next_in; + last = in + (strm->avail_in - (INFLATE_FAST_MIN_INPUT - 1)); + out = strm->next_out; + beg = out - (start - strm->avail_out); + end = out + (strm->avail_out - (INFLATE_FAST_MIN_OUTPUT - 1)); +#ifdef INFLATE_STRICT + dmax = state->dmax; +#endif + wsize = state->wsize; + whave = state->whave; + wnext = state->wnext; + window = state->window; + hold = state->hold; + bits = state->bits; + lcode = state->lencode; + dcode = state->distcode; + lmask = (1U << state->lenbits) - 1; + dmask = (1U << state->distbits) - 1; + + /* decode literals and length/distances until end-of-block or not enough + input data or output space */ + do { + if (bits < 15) { + hold += (unsigned long)(*in++) << bits; + bits += 8; + hold += (unsigned long)(*in++) << bits; + bits += 8; + } + here = lcode[hold & lmask]; + dolen: + op = (unsigned)(here.bits); + hold >>= op; + bits -= op; + op = (unsigned)(here.op); + if (op == 0) { /* literal */ + Tracevv((stderr, here.val >= 0x20 && here.val < 0x7f ? + "inflate: literal '%c'\n" : + "inflate: literal 0x%02x\n", here.val)); + *out++ = (unsigned char)(here.val); + } + else if (op & 16) { /* length base */ + len = (unsigned)(here.val); + op &= 15; /* number of extra bits */ + if (op) { + if (bits < op) { + hold += (unsigned long)(*in++) << bits; + bits += 8; + } + len += (unsigned)hold & ((1U << op) - 1); + hold >>= op; + bits -= op; + } + Tracevv((stderr, "inflate: length %u\n", len)); + if (bits < 15) { + hold += (unsigned long)(*in++) << bits; + bits += 8; + hold += (unsigned long)(*in++) << bits; + bits += 8; + } + here = dcode[hold & dmask]; + dodist: + op = (unsigned)(here.bits); + hold >>= op; + bits -= op; + op = (unsigned)(here.op); + if (op & 16) { /* distance base */ + dist = (unsigned)(here.val); + op &= 15; /* number of extra bits */ + if (bits < op) { + hold += (unsigned long)(*in++) << bits; + bits += 8; + if (bits < op) { + hold += (unsigned long)(*in++) << bits; + bits += 8; + } + } + dist += (unsigned)hold & ((1U << op) - 1); +#ifdef INFLATE_STRICT + if (dist > dmax) { + strm->msg = (char *)"invalid distance too far back"; + state->mode = BAD; + break; + } +#endif + hold >>= op; + bits -= op; + Tracevv((stderr, "inflate: distance %u\n", dist)); + op = (unsigned)(out - beg); /* max distance in output */ + if (dist > op) { /* see if copy from window */ + op = dist - op; /* distance back in window */ + if (op > whave) { + if (state->sane) { + strm->msg = + (char *)"invalid distance too far back"; + state->mode = BAD; + break; + } +#ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR + if (len <= op - whave) { + do { + *out++ = 0; + } while (--len); + continue; + } + len -= op - whave; + do { + *out++ = 0; + } while (--op > whave); + if (op == 0) { + from = out - dist; + do { + *out++ = *from++; + } while (--len); + continue; + } +#endif + } + from = window; + if (wnext == 0) { /* very common case */ + from += wsize - op; + if (op < len) { /* some from window */ + len -= op; + do { + *out++ = *from++; + } while (--op); + from = out - dist; /* rest from output */ + } + } + else if (wnext < op) { /* wrap around window */ + from += wsize + wnext - op; + op -= wnext; + if (op < len) { /* some from end of window */ + len -= op; + do { + *out++ = *from++; + } while (--op); + from = window; + if (wnext < len) { /* some from start of window */ + op = wnext; + len -= op; + do { + *out++ = *from++; + } while (--op); + from = out - dist; /* rest from output */ + } + } + } + else { /* contiguous in window */ + from += wnext - op; + if (op < len) { /* some from window */ + len -= op; + do { + *out++ = *from++; + } while (--op); + from = out - dist; /* rest from output */ + } + } + while (len > 2) { + *out++ = *from++; + *out++ = *from++; + *out++ = *from++; + len -= 3; + } + if (len) { + *out++ = *from++; + if (len > 1) + *out++ = *from++; + } + } + else { + from = out - dist; /* copy direct from output */ + do { /* minimum length is three */ + *out++ = *from++; + *out++ = *from++; + *out++ = *from++; + len -= 3; + } while (len > 2); + if (len) { + *out++ = *from++; + if (len > 1) + *out++ = *from++; + } + } + } + else if ((op & 64) == 0) { /* 2nd level distance code */ + here = dcode[here.val + (hold & ((1U << op) - 1))]; + goto dodist; + } + else { + strm->msg = (char *)"invalid distance code"; + state->mode = BAD; + break; + } + } + else if ((op & 64) == 0) { /* 2nd level length code */ + here = lcode[here.val + (hold & ((1U << op) - 1))]; + goto dolen; + } + else if (op & 32) { /* end-of-block */ + Tracevv((stderr, "inflate: end of block\n")); + state->mode = TYPE; + break; + } + else { + strm->msg = (char *)"invalid literal/length code"; + state->mode = BAD; + break; + } + } while (in < last && out < end); + + /* return unused bytes (on entry, bits < 8, so in won't go too far back) */ + len = bits >> 3; + in -= len; + bits -= len << 3; + hold &= (1U << bits) - 1; + + /* update state and return */ + strm->next_in = in; + strm->next_out = out; + strm->avail_in = (unsigned)(in < last ? + (INFLATE_FAST_MIN_INPUT - 1) + (last - in) : + (INFLATE_FAST_MIN_INPUT - 1) - (in - last)); + strm->avail_out = (unsigned)(out < end ? + (INFLATE_FAST_MIN_OUTPUT - 1) + (end - out) : + (INFLATE_FAST_MIN_OUTPUT - 1) - (out - end)); + state->hold = hold; + state->bits = bits; + return; +} + +/* + inflate_fast() speedups that turned out slower (on a PowerPC G3 750CXe): + - Using bit fields for code structure + - Different op definition to avoid & for extra bits (do & for table bits) + - Three separate decoding do-loops for direct, window, and wnext == 0 + - Special case for distance > 1 copies to do overlapped load and store copy + - Explicit branch predictions (based on measured branch probabilities) + - Deferring match copy and interspersed it with decoding subsequent codes + - Swapping literal/length else + - Swapping window/direct else + - Larger unrolled copy loops (three is about right) + - Moving len -= 3 statement into middle of loop + */ + +#endif /* !ASMINF */ diff --git a/third_party/zlib/inffast.h b/third_party/zlib/inffast.h new file mode 100644 index 00000000..c7c1c098 --- /dev/null +++ b/third_party/zlib/inffast.h @@ -0,0 +1,26 @@ +/* inffast.h -- header to use inffast.c + * Copyright (C) 1995-2003, 2010 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +/* WARNING: this file should *not* be used by applications. It is + part of the implementation of the compression library and is + subject to change. Applications should only use zlib.h. + */ + +/* INFLATE_FAST_MIN_INPUT: the minimum number of input bytes needed so that + we can safely call inflate_fast() with only one up-front bounds check. One + length/distance code pair (15 bits for the length code, 5 bits for length + extra, 15 bits for the distance code, 13 bits for distance extra) requires + reading up to 48 input bits (6 bytes). +*/ +#define INFLATE_FAST_MIN_INPUT 6 + +/* INFLATE_FAST_MIN_OUTPUT: the minimum number of output bytes needed so that + we can safely call inflate_fast() with only one up-front bounds check. One + length/distance code pair can output up to 258 bytes, which is the maximum + length that can be coded. + */ +#define INFLATE_FAST_MIN_OUTPUT 258 + +void ZLIB_INTERNAL inflate_fast OF((z_streamp strm, unsigned start)); diff --git a/third_party/zlib/inffixed.h b/third_party/zlib/inffixed.h new file mode 100644 index 00000000..d6283277 --- /dev/null +++ b/third_party/zlib/inffixed.h @@ -0,0 +1,94 @@ + /* inffixed.h -- table for decoding fixed codes + * Generated automatically by makefixed(). + */ + + /* WARNING: this file should *not* be used by applications. + It is part of the implementation of this library and is + subject to change. Applications should only use zlib.h. + */ + + static const code lenfix[512] = { + {96,7,0},{0,8,80},{0,8,16},{20,8,115},{18,7,31},{0,8,112},{0,8,48}, + {0,9,192},{16,7,10},{0,8,96},{0,8,32},{0,9,160},{0,8,0},{0,8,128}, + {0,8,64},{0,9,224},{16,7,6},{0,8,88},{0,8,24},{0,9,144},{19,7,59}, + {0,8,120},{0,8,56},{0,9,208},{17,7,17},{0,8,104},{0,8,40},{0,9,176}, + {0,8,8},{0,8,136},{0,8,72},{0,9,240},{16,7,4},{0,8,84},{0,8,20}, + {21,8,227},{19,7,43},{0,8,116},{0,8,52},{0,9,200},{17,7,13},{0,8,100}, + {0,8,36},{0,9,168},{0,8,4},{0,8,132},{0,8,68},{0,9,232},{16,7,8}, + {0,8,92},{0,8,28},{0,9,152},{20,7,83},{0,8,124},{0,8,60},{0,9,216}, + {18,7,23},{0,8,108},{0,8,44},{0,9,184},{0,8,12},{0,8,140},{0,8,76}, + {0,9,248},{16,7,3},{0,8,82},{0,8,18},{21,8,163},{19,7,35},{0,8,114}, + {0,8,50},{0,9,196},{17,7,11},{0,8,98},{0,8,34},{0,9,164},{0,8,2}, + {0,8,130},{0,8,66},{0,9,228},{16,7,7},{0,8,90},{0,8,26},{0,9,148}, + {20,7,67},{0,8,122},{0,8,58},{0,9,212},{18,7,19},{0,8,106},{0,8,42}, + {0,9,180},{0,8,10},{0,8,138},{0,8,74},{0,9,244},{16,7,5},{0,8,86}, + {0,8,22},{64,8,0},{19,7,51},{0,8,118},{0,8,54},{0,9,204},{17,7,15}, + {0,8,102},{0,8,38},{0,9,172},{0,8,6},{0,8,134},{0,8,70},{0,9,236}, + {16,7,9},{0,8,94},{0,8,30},{0,9,156},{20,7,99},{0,8,126},{0,8,62}, + {0,9,220},{18,7,27},{0,8,110},{0,8,46},{0,9,188},{0,8,14},{0,8,142}, + {0,8,78},{0,9,252},{96,7,0},{0,8,81},{0,8,17},{21,8,131},{18,7,31}, + {0,8,113},{0,8,49},{0,9,194},{16,7,10},{0,8,97},{0,8,33},{0,9,162}, + {0,8,1},{0,8,129},{0,8,65},{0,9,226},{16,7,6},{0,8,89},{0,8,25}, + {0,9,146},{19,7,59},{0,8,121},{0,8,57},{0,9,210},{17,7,17},{0,8,105}, + {0,8,41},{0,9,178},{0,8,9},{0,8,137},{0,8,73},{0,9,242},{16,7,4}, + {0,8,85},{0,8,21},{16,8,258},{19,7,43},{0,8,117},{0,8,53},{0,9,202}, + {17,7,13},{0,8,101},{0,8,37},{0,9,170},{0,8,5},{0,8,133},{0,8,69}, + {0,9,234},{16,7,8},{0,8,93},{0,8,29},{0,9,154},{20,7,83},{0,8,125}, + {0,8,61},{0,9,218},{18,7,23},{0,8,109},{0,8,45},{0,9,186},{0,8,13}, + {0,8,141},{0,8,77},{0,9,250},{16,7,3},{0,8,83},{0,8,19},{21,8,195}, + {19,7,35},{0,8,115},{0,8,51},{0,9,198},{17,7,11},{0,8,99},{0,8,35}, + {0,9,166},{0,8,3},{0,8,131},{0,8,67},{0,9,230},{16,7,7},{0,8,91}, + {0,8,27},{0,9,150},{20,7,67},{0,8,123},{0,8,59},{0,9,214},{18,7,19}, + {0,8,107},{0,8,43},{0,9,182},{0,8,11},{0,8,139},{0,8,75},{0,9,246}, + {16,7,5},{0,8,87},{0,8,23},{64,8,0},{19,7,51},{0,8,119},{0,8,55}, + {0,9,206},{17,7,15},{0,8,103},{0,8,39},{0,9,174},{0,8,7},{0,8,135}, + {0,8,71},{0,9,238},{16,7,9},{0,8,95},{0,8,31},{0,9,158},{20,7,99}, + {0,8,127},{0,8,63},{0,9,222},{18,7,27},{0,8,111},{0,8,47},{0,9,190}, + {0,8,15},{0,8,143},{0,8,79},{0,9,254},{96,7,0},{0,8,80},{0,8,16}, + {20,8,115},{18,7,31},{0,8,112},{0,8,48},{0,9,193},{16,7,10},{0,8,96}, + {0,8,32},{0,9,161},{0,8,0},{0,8,128},{0,8,64},{0,9,225},{16,7,6}, + {0,8,88},{0,8,24},{0,9,145},{19,7,59},{0,8,120},{0,8,56},{0,9,209}, + {17,7,17},{0,8,104},{0,8,40},{0,9,177},{0,8,8},{0,8,136},{0,8,72}, + {0,9,241},{16,7,4},{0,8,84},{0,8,20},{21,8,227},{19,7,43},{0,8,116}, + {0,8,52},{0,9,201},{17,7,13},{0,8,100},{0,8,36},{0,9,169},{0,8,4}, + {0,8,132},{0,8,68},{0,9,233},{16,7,8},{0,8,92},{0,8,28},{0,9,153}, + {20,7,83},{0,8,124},{0,8,60},{0,9,217},{18,7,23},{0,8,108},{0,8,44}, + {0,9,185},{0,8,12},{0,8,140},{0,8,76},{0,9,249},{16,7,3},{0,8,82}, + {0,8,18},{21,8,163},{19,7,35},{0,8,114},{0,8,50},{0,9,197},{17,7,11}, + {0,8,98},{0,8,34},{0,9,165},{0,8,2},{0,8,130},{0,8,66},{0,9,229}, + {16,7,7},{0,8,90},{0,8,26},{0,9,149},{20,7,67},{0,8,122},{0,8,58}, + {0,9,213},{18,7,19},{0,8,106},{0,8,42},{0,9,181},{0,8,10},{0,8,138}, + {0,8,74},{0,9,245},{16,7,5},{0,8,86},{0,8,22},{64,8,0},{19,7,51}, + {0,8,118},{0,8,54},{0,9,205},{17,7,15},{0,8,102},{0,8,38},{0,9,173}, + {0,8,6},{0,8,134},{0,8,70},{0,9,237},{16,7,9},{0,8,94},{0,8,30}, + {0,9,157},{20,7,99},{0,8,126},{0,8,62},{0,9,221},{18,7,27},{0,8,110}, + {0,8,46},{0,9,189},{0,8,14},{0,8,142},{0,8,78},{0,9,253},{96,7,0}, + {0,8,81},{0,8,17},{21,8,131},{18,7,31},{0,8,113},{0,8,49},{0,9,195}, + {16,7,10},{0,8,97},{0,8,33},{0,9,163},{0,8,1},{0,8,129},{0,8,65}, + {0,9,227},{16,7,6},{0,8,89},{0,8,25},{0,9,147},{19,7,59},{0,8,121}, + {0,8,57},{0,9,211},{17,7,17},{0,8,105},{0,8,41},{0,9,179},{0,8,9}, + {0,8,137},{0,8,73},{0,9,243},{16,7,4},{0,8,85},{0,8,21},{16,8,258}, + {19,7,43},{0,8,117},{0,8,53},{0,9,203},{17,7,13},{0,8,101},{0,8,37}, + {0,9,171},{0,8,5},{0,8,133},{0,8,69},{0,9,235},{16,7,8},{0,8,93}, + {0,8,29},{0,9,155},{20,7,83},{0,8,125},{0,8,61},{0,9,219},{18,7,23}, + {0,8,109},{0,8,45},{0,9,187},{0,8,13},{0,8,141},{0,8,77},{0,9,251}, + {16,7,3},{0,8,83},{0,8,19},{21,8,195},{19,7,35},{0,8,115},{0,8,51}, + {0,9,199},{17,7,11},{0,8,99},{0,8,35},{0,9,167},{0,8,3},{0,8,131}, + {0,8,67},{0,9,231},{16,7,7},{0,8,91},{0,8,27},{0,9,151},{20,7,67}, + {0,8,123},{0,8,59},{0,9,215},{18,7,19},{0,8,107},{0,8,43},{0,9,183}, + {0,8,11},{0,8,139},{0,8,75},{0,9,247},{16,7,5},{0,8,87},{0,8,23}, + {64,8,0},{19,7,51},{0,8,119},{0,8,55},{0,9,207},{17,7,15},{0,8,103}, + {0,8,39},{0,9,175},{0,8,7},{0,8,135},{0,8,71},{0,9,239},{16,7,9}, + {0,8,95},{0,8,31},{0,9,159},{20,7,99},{0,8,127},{0,8,63},{0,9,223}, + {18,7,27},{0,8,111},{0,8,47},{0,9,191},{0,8,15},{0,8,143},{0,8,79}, + {0,9,255} + }; + + static const code distfix[32] = { + {16,5,1},{23,5,257},{19,5,17},{27,5,4097},{17,5,5},{25,5,1025}, + {21,5,65},{29,5,16385},{16,5,3},{24,5,513},{20,5,33},{28,5,8193}, + {18,5,9},{26,5,2049},{22,5,129},{64,5,0},{16,5,2},{23,5,385}, + {19,5,25},{27,5,6145},{17,5,7},{25,5,1537},{21,5,97},{29,5,24577}, + {16,5,4},{24,5,769},{20,5,49},{28,5,12289},{18,5,13},{26,5,3073}, + {22,5,193},{64,5,0} + }; diff --git a/third_party/zlib/inflate.c b/third_party/zlib/inflate.c new file mode 100644 index 00000000..68902e81 --- /dev/null +++ b/third_party/zlib/inflate.c @@ -0,0 +1,1563 @@ +/* inflate.c -- zlib decompression + * Copyright (C) 1995-2016 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +/* + * Change history: + * + * 1.2.beta0 24 Nov 2002 + * - First version -- complete rewrite of inflate to simplify code, avoid + * creation of window when not needed, minimize use of window when it is + * needed, make inffast.c even faster, implement gzip decoding, and to + * improve code readability and style over the previous zlib inflate code + * + * 1.2.beta1 25 Nov 2002 + * - Use pointers for available input and output checking in inffast.c + * - Remove input and output counters in inffast.c + * - Change inffast.c entry and loop from avail_in >= 7 to >= 6 + * - Remove unnecessary second byte pull from length extra in inffast.c + * - Unroll direct copy to three copies per loop in inffast.c + * + * 1.2.beta2 4 Dec 2002 + * - Change external routine names to reduce potential conflicts + * - Correct filename to inffixed.h for fixed tables in inflate.c + * - Make hbuf[] unsigned char to match parameter type in inflate.c + * - Change strm->next_out[-state->offset] to *(strm->next_out - state->offset) + * to avoid negation problem on Alphas (64 bit) in inflate.c + * + * 1.2.beta3 22 Dec 2002 + * - Add comments on state->bits assertion in inffast.c + * - Add comments on op field in inftrees.h + * - Fix bug in reuse of allocated window after inflateReset() + * - Remove bit fields--back to byte structure for speed + * - Remove distance extra == 0 check in inflate_fast()--only helps for lengths + * - Change post-increments to pre-increments in inflate_fast(), PPC biased? + * - Add compile time option, POSTINC, to use post-increments instead (Intel?) + * - Make MATCH copy in inflate() much faster for when inflate_fast() not used + * - Use local copies of stream next and avail values, as well as local bit + * buffer and bit count in inflate()--for speed when inflate_fast() not used + * + * 1.2.beta4 1 Jan 2003 + * - Split ptr - 257 statements in inflate_table() to avoid compiler warnings + * - Move a comment on output buffer sizes from inffast.c to inflate.c + * - Add comments in inffast.c to introduce the inflate_fast() routine + * - Rearrange window copies in inflate_fast() for speed and simplification + * - Unroll last copy for window match in inflate_fast() + * - Use local copies of window variables in inflate_fast() for speed + * - Pull out common wnext == 0 case for speed in inflate_fast() + * - Make op and len in inflate_fast() unsigned for consistency + * - Add FAR to lcode and dcode declarations in inflate_fast() + * - Simplified bad distance check in inflate_fast() + * - Added inflateBackInit(), inflateBack(), and inflateBackEnd() in new + * source file infback.c to provide a call-back interface to inflate for + * programs like gzip and unzip -- uses window as output buffer to avoid + * window copying + * + * 1.2.beta5 1 Jan 2003 + * - Improved inflateBack() interface to allow the caller to provide initial + * input in strm. + * - Fixed stored blocks bug in inflateBack() + * + * 1.2.beta6 4 Jan 2003 + * - Added comments in inffast.c on effectiveness of POSTINC + * - Typecasting all around to reduce compiler warnings + * - Changed loops from while (1) or do {} while (1) to for (;;), again to + * make compilers happy + * - Changed type of window in inflateBackInit() to unsigned char * + * + * 1.2.beta7 27 Jan 2003 + * - Changed many types to unsigned or unsigned short to avoid warnings + * - Added inflateCopy() function + * + * 1.2.0 9 Mar 2003 + * - Changed inflateBack() interface to provide separate opaque descriptors + * for the in() and out() functions + * - Changed inflateBack() argument and in_func typedef to swap the length + * and buffer address return values for the input function + * - Check next_in and next_out for Z_NULL on entry to inflate() + * + * The history for versions after 1.2.0 are in ChangeLog in zlib distribution. + */ + +#include "zutil.h" +#include "inftrees.h" +#include "inflate.h" +#include "inffast.h" + +#ifdef MAKEFIXED +# ifndef BUILDFIXED +# define BUILDFIXED +# endif +#endif + +/* function prototypes */ +local int inflateStateCheck OF((z_streamp strm)); +local void fixedtables OF((struct inflate_state FAR *state)); +local int updatewindow OF((z_streamp strm, const unsigned char FAR *end, + unsigned copy)); +#ifdef BUILDFIXED + void makefixed OF((void)); +#endif +local unsigned syncsearch OF((unsigned FAR *have, const unsigned char FAR *buf, + unsigned len)); + +local int inflateStateCheck(strm) +z_streamp strm; +{ + struct inflate_state FAR *state; + if (strm == Z_NULL || + strm->zalloc == (alloc_func)0 || strm->zfree == (free_func)0) + return 1; + state = (struct inflate_state FAR *)strm->state; + if (state == Z_NULL || state->strm != strm || + state->mode < HEAD || state->mode > SYNC) + return 1; + return 0; +} + +int ZEXPORT inflateResetKeep(strm) +z_streamp strm; +{ + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + strm->total_in = strm->total_out = state->total = 0; + strm->msg = Z_NULL; + if (state->wrap) /* to support ill-conceived Java test suite */ + strm->adler = state->wrap & 1; + state->mode = HEAD; + state->last = 0; + state->havedict = 0; + state->dmax = 32768U; + state->head = Z_NULL; + state->hold = 0; + state->bits = 0; + state->lencode = state->distcode = state->next = state->codes; + state->sane = 1; + state->back = -1; + Tracev((stderr, "inflate: reset\n")); + return Z_OK; +} + +int ZEXPORT inflateReset(strm) +z_streamp strm; +{ + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + state->wsize = 0; + state->whave = 0; + state->wnext = 0; + return inflateResetKeep(strm); +} + +int ZEXPORT inflateReset2(strm, windowBits) +z_streamp strm; +int windowBits; +{ + int wrap; + struct inflate_state FAR *state; + + /* get the state */ + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + + /* extract wrap request from windowBits parameter */ + if (windowBits < 0) { + wrap = 0; + windowBits = -windowBits; + } + else { + wrap = (windowBits >> 4) + 5; +#ifdef GUNZIP + if (windowBits < 48) + windowBits &= 15; +#endif + } + + /* set number of window bits, free window if different */ + if (windowBits && (windowBits < 8 || windowBits > 15)) + return Z_STREAM_ERROR; + if (state->window != Z_NULL && state->wbits != (unsigned)windowBits) { + ZFREE(strm, state->window); + state->window = Z_NULL; + } + + /* update state and reset the rest of it */ + state->wrap = wrap; + state->wbits = (unsigned)windowBits; + return inflateReset(strm); +} + +int ZEXPORT inflateInit2_(strm, windowBits, version, stream_size) +z_streamp strm; +int windowBits; +const char *version; +int stream_size; +{ + int ret; + struct inflate_state FAR *state; + + if (version == Z_NULL || version[0] != ZLIB_VERSION[0] || + stream_size != (int)(sizeof(z_stream))) + return Z_VERSION_ERROR; + if (strm == Z_NULL) return Z_STREAM_ERROR; + strm->msg = Z_NULL; /* in case we return an error */ + if (strm->zalloc == (alloc_func)0) { +#ifdef Z_SOLO + return Z_STREAM_ERROR; +#else + strm->zalloc = zcalloc; + strm->opaque = (voidpf)0; +#endif + } + if (strm->zfree == (free_func)0) +#ifdef Z_SOLO + return Z_STREAM_ERROR; +#else + strm->zfree = zcfree; +#endif + state = (struct inflate_state FAR *) + ZALLOC(strm, 1, sizeof(struct inflate_state)); + if (state == Z_NULL) return Z_MEM_ERROR; + Tracev((stderr, "inflate: allocated\n")); + strm->state = (struct internal_state FAR *)state; + state->strm = strm; + state->window = Z_NULL; + state->mode = HEAD; /* to pass state test in inflateReset2() */ + state->check = 1L; /* 1L is the result of adler32() zero length data */ + ret = inflateReset2(strm, windowBits); + if (ret != Z_OK) { + ZFREE(strm, state); + strm->state = Z_NULL; + } + return ret; +} + +int ZEXPORT inflateInit_(strm, version, stream_size) +z_streamp strm; +const char *version; +int stream_size; +{ + return inflateInit2_(strm, DEF_WBITS, version, stream_size); +} + +int ZEXPORT inflatePrime(strm, bits, value) +z_streamp strm; +int bits; +int value; +{ + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + if (bits < 0) { + state->hold = 0; + state->bits = 0; + return Z_OK; + } + if (bits > 16 || state->bits + (uInt)bits > 32) return Z_STREAM_ERROR; + value &= (1L << bits) - 1; + state->hold += (unsigned)value << state->bits; + state->bits += (uInt)bits; + return Z_OK; +} + +/* + Return state with length and distance decoding tables and index sizes set to + fixed code decoding. Normally this returns fixed tables from inffixed.h. + If BUILDFIXED is defined, then instead this routine builds the tables the + first time it's called, and returns those tables the first time and + thereafter. This reduces the size of the code by about 2K bytes, in + exchange for a little execution time. However, BUILDFIXED should not be + used for threaded applications, since the rewriting of the tables and virgin + may not be thread-safe. + */ +local void fixedtables(state) +struct inflate_state FAR *state; +{ +#ifdef BUILDFIXED + static int virgin = 1; + static code *lenfix, *distfix; + static code fixed[544]; + + /* build fixed huffman tables if first call (may not be thread safe) */ + if (virgin) { + unsigned sym, bits; + static code *next; + + /* literal/length table */ + sym = 0; + while (sym < 144) state->lens[sym++] = 8; + while (sym < 256) state->lens[sym++] = 9; + while (sym < 280) state->lens[sym++] = 7; + while (sym < 288) state->lens[sym++] = 8; + next = fixed; + lenfix = next; + bits = 9; + inflate_table(LENS, state->lens, 288, &(next), &(bits), state->work); + + /* distance table */ + sym = 0; + while (sym < 32) state->lens[sym++] = 5; + distfix = next; + bits = 5; + inflate_table(DISTS, state->lens, 32, &(next), &(bits), state->work); + + /* do this just once */ + virgin = 0; + } +#else /* !BUILDFIXED */ +# include "inffixed.h" +#endif /* BUILDFIXED */ + state->lencode = lenfix; + state->lenbits = 9; + state->distcode = distfix; + state->distbits = 5; +} + +#ifdef MAKEFIXED +#include <stdio.h> + +/* + Write out the inffixed.h that is #include'd above. Defining MAKEFIXED also + defines BUILDFIXED, so the tables are built on the fly. makefixed() writes + those tables to stdout, which would be piped to inffixed.h. A small program + can simply call makefixed to do this: + + void makefixed(void); + + int main(void) + { + makefixed(); + return 0; + } + + Then that can be linked with zlib built with MAKEFIXED defined and run: + + a.out > inffixed.h + */ +void makefixed() +{ + unsigned low, size; + struct inflate_state state; + + fixedtables(&state); + puts(" /* inffixed.h -- table for decoding fixed codes"); + puts(" * Generated automatically by makefixed()."); + puts(" */"); + puts(""); + puts(" /* WARNING: this file should *not* be used by applications."); + puts(" It is part of the implementation of this library and is"); + puts(" subject to change. Applications should only use zlib.h."); + puts(" */"); + puts(""); + size = 1U << 9; + printf(" static const code lenfix[%u] = {", size); + low = 0; + for (;;) { + if ((low % 7) == 0) printf("\n "); + printf("{%u,%u,%d}", (low & 127) == 99 ? 64 : state.lencode[low].op, + state.lencode[low].bits, state.lencode[low].val); + if (++low == size) break; + putchar(','); + } + puts("\n };"); + size = 1U << 5; + printf("\n static const code distfix[%u] = {", size); + low = 0; + for (;;) { + if ((low % 6) == 0) printf("\n "); + printf("{%u,%u,%d}", state.distcode[low].op, state.distcode[low].bits, + state.distcode[low].val); + if (++low == size) break; + putchar(','); + } + puts("\n };"); +} +#endif /* MAKEFIXED */ + +/* + Update the window with the last wsize (normally 32K) bytes written before + returning. If window does not exist yet, create it. This is only called + when a window is already in use, or when output has been written during this + inflate call, but the end of the deflate stream has not been reached yet. + It is also called to create a window for dictionary data when a dictionary + is loaded. + + Providing output buffers larger than 32K to inflate() should provide a speed + advantage, since only the last 32K of output is copied to the sliding window + upon return from inflate(), and since all distances after the first 32K of + output will fall in the output data, making match copies simpler and faster. + The advantage may be dependent on the size of the processor's data caches. + */ +local int updatewindow(strm, end, copy) +z_streamp strm; +const Bytef *end; +unsigned copy; +{ + struct inflate_state FAR *state; + unsigned dist; + + state = (struct inflate_state FAR *)strm->state; + + /* if it hasn't been done already, allocate space for the window */ + if (state->window == Z_NULL) { + state->window = (unsigned char FAR *) + ZALLOC(strm, 1U << state->wbits, + sizeof(unsigned char)); + if (state->window == Z_NULL) return 1; + } + + /* if window not in use yet, initialize */ + if (state->wsize == 0) { + state->wsize = 1U << state->wbits; + state->wnext = 0; + state->whave = 0; + } + + /* copy state->wsize or less output bytes into the circular window */ + if (copy >= state->wsize) { + zmemcpy(state->window, end - state->wsize, state->wsize); + state->wnext = 0; + state->whave = state->wsize; + } + else { + dist = state->wsize - state->wnext; + if (dist > copy) dist = copy; + zmemcpy(state->window + state->wnext, end - copy, dist); + copy -= dist; + if (copy) { + zmemcpy(state->window, end - copy, copy); + state->wnext = copy; + state->whave = state->wsize; + } + else { + state->wnext += dist; + if (state->wnext == state->wsize) state->wnext = 0; + if (state->whave < state->wsize) state->whave += dist; + } + } + return 0; +} + +/* Macros for inflate(): */ + +/* check function to use adler32() for zlib or crc32() for gzip */ +#ifdef GUNZIP +# define UPDATE(check, buf, len) \ + (state->flags ? crc32(check, buf, len) : adler32(check, buf, len)) +#else +# define UPDATE(check, buf, len) adler32(check, buf, len) +#endif + +/* check macros for header crc */ +#ifdef GUNZIP +# define CRC2(check, word) \ + do { \ + hbuf[0] = (unsigned char)(word); \ + hbuf[1] = (unsigned char)((word) >> 8); \ + check = crc32(check, hbuf, 2); \ + } while (0) + +# define CRC4(check, word) \ + do { \ + hbuf[0] = (unsigned char)(word); \ + hbuf[1] = (unsigned char)((word) >> 8); \ + hbuf[2] = (unsigned char)((word) >> 16); \ + hbuf[3] = (unsigned char)((word) >> 24); \ + check = crc32(check, hbuf, 4); \ + } while (0) +#endif + +/* Load registers with state in inflate() for speed */ +#define LOAD() \ + do { \ + put = strm->next_out; \ + left = strm->avail_out; \ + next = strm->next_in; \ + have = strm->avail_in; \ + hold = state->hold; \ + bits = state->bits; \ + } while (0) + +/* Restore state from registers in inflate() */ +#define RESTORE() \ + do { \ + strm->next_out = put; \ + strm->avail_out = left; \ + strm->next_in = next; \ + strm->avail_in = have; \ + state->hold = hold; \ + state->bits = bits; \ + } while (0) + +/* Clear the input bit accumulator */ +#define INITBITS() \ + do { \ + hold = 0; \ + bits = 0; \ + } while (0) + +/* Get a byte of input into the bit accumulator, or return from inflate() + if there is no input available. */ +#define PULLBYTE() \ + do { \ + if (have == 0) goto inf_leave; \ + have--; \ + hold += (unsigned long)(*next++) << bits; \ + bits += 8; \ + } while (0) + +/* Assure that there are at least n bits in the bit accumulator. If there is + not enough available input to do that, then return from inflate(). */ +#define NEEDBITS(n) \ + do { \ + while (bits < (unsigned)(n)) \ + PULLBYTE(); \ + } while (0) + +/* Return the low n bits of the bit accumulator (n < 16) */ +#define BITS(n) \ + ((unsigned)hold & ((1U << (n)) - 1)) + +/* Remove n bits from the bit accumulator */ +#define DROPBITS(n) \ + do { \ + hold >>= (n); \ + bits -= (unsigned)(n); \ + } while (0) + +/* Remove zero to seven bits as needed to go to a byte boundary */ +#define BYTEBITS() \ + do { \ + hold >>= bits & 7; \ + bits -= bits & 7; \ + } while (0) + +/* + inflate() uses a state machine to process as much input data and generate as + much output data as possible before returning. The state machine is + structured roughly as follows: + + for (;;) switch (state) { + ... + case STATEn: + if (not enough input data or output space to make progress) + return; + ... make progress ... + state = STATEm; + break; + ... + } + + so when inflate() is called again, the same case is attempted again, and + if the appropriate resources are provided, the machine proceeds to the + next state. The NEEDBITS() macro is usually the way the state evaluates + whether it can proceed or should return. NEEDBITS() does the return if + the requested bits are not available. The typical use of the BITS macros + is: + + NEEDBITS(n); + ... do something with BITS(n) ... + DROPBITS(n); + + where NEEDBITS(n) either returns from inflate() if there isn't enough + input left to load n bits into the accumulator, or it continues. BITS(n) + gives the low n bits in the accumulator. When done, DROPBITS(n) drops + the low n bits off the accumulator. INITBITS() clears the accumulator + and sets the number of available bits to zero. BYTEBITS() discards just + enough bits to put the accumulator on a byte boundary. After BYTEBITS() + and a NEEDBITS(8), then BITS(8) would return the next byte in the stream. + + NEEDBITS(n) uses PULLBYTE() to get an available byte of input, or to return + if there is no input available. The decoding of variable length codes uses + PULLBYTE() directly in order to pull just enough bytes to decode the next + code, and no more. + + Some states loop until they get enough input, making sure that enough + state information is maintained to continue the loop where it left off + if NEEDBITS() returns in the loop. For example, want, need, and keep + would all have to actually be part of the saved state in case NEEDBITS() + returns: + + case STATEw: + while (want < need) { + NEEDBITS(n); + keep[want++] = BITS(n); + DROPBITS(n); + } + state = STATEx; + case STATEx: + + As shown above, if the next state is also the next case, then the break + is omitted. + + A state may also return if there is not enough output space available to + complete that state. Those states are copying stored data, writing a + literal byte, and copying a matching string. + + When returning, a "goto inf_leave" is used to update the total counters, + update the check value, and determine whether any progress has been made + during that inflate() call in order to return the proper return code. + Progress is defined as a change in either strm->avail_in or strm->avail_out. + When there is a window, goto inf_leave will update the window with the last + output written. If a goto inf_leave occurs in the middle of decompression + and there is no window currently, goto inf_leave will create one and copy + output to the window for the next call of inflate(). + + In this implementation, the flush parameter of inflate() only affects the + return code (per zlib.h). inflate() always writes as much as possible to + strm->next_out, given the space available and the provided input--the effect + documented in zlib.h of Z_SYNC_FLUSH. Furthermore, inflate() always defers + the allocation of and copying into a sliding window until necessary, which + provides the effect documented in zlib.h for Z_FINISH when the entire input + stream available. So the only thing the flush parameter actually does is: + when flush is set to Z_FINISH, inflate() cannot return Z_OK. Instead it + will return Z_BUF_ERROR if it has not reached the end of the stream. + */ + +int ZEXPORT inflate(strm, flush) +z_streamp strm; +int flush; +{ + struct inflate_state FAR *state; + z_const unsigned char FAR *next; /* next input */ + unsigned char FAR *put; /* next output */ + unsigned have, left; /* available input and output */ + unsigned long hold; /* bit buffer */ + unsigned bits; /* bits in bit buffer */ + unsigned in, out; /* save starting available input and output */ + unsigned copy; /* number of stored or match bytes to copy */ + unsigned char FAR *from; /* where to copy match bytes from */ + code here; /* current decoding table entry */ + code last; /* parent table entry */ + unsigned len; /* length to copy for repeats, bits to drop */ + int ret; /* return code */ +#ifdef GUNZIP + unsigned char hbuf[4]; /* buffer for gzip header crc calculation */ +#endif + static const unsigned short order[19] = /* permutation of code lengths */ + {16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15}; + + if (inflateStateCheck(strm) || strm->next_out == Z_NULL || + (strm->next_in == Z_NULL && strm->avail_in != 0)) + return Z_STREAM_ERROR; + + state = (struct inflate_state FAR *)strm->state; + if (state->mode == TYPE) state->mode = TYPEDO; /* skip check */ + LOAD(); + in = have; + out = left; + ret = Z_OK; + for (;;) + switch (state->mode) { + case HEAD: + if (state->wrap == 0) { + state->mode = TYPEDO; + break; + } + NEEDBITS(16); +#ifdef GUNZIP + if ((state->wrap & 2) && hold == 0x8b1f) { /* gzip header */ + if (state->wbits == 0) + state->wbits = 15; + state->check = crc32(0L, Z_NULL, 0); + CRC2(state->check, hold); + INITBITS(); + state->mode = FLAGS; + break; + } + state->flags = 0; /* expect zlib header */ + if (state->head != Z_NULL) + state->head->done = -1; + if (!(state->wrap & 1) || /* check if zlib header allowed */ +#else + if ( +#endif + ((BITS(8) << 8) + (hold >> 8)) % 31) { + strm->msg = (char *)"incorrect header check"; + state->mode = BAD; + break; + } + if (BITS(4) != Z_DEFLATED) { + strm->msg = (char *)"unknown compression method"; + state->mode = BAD; + break; + } + DROPBITS(4); + len = BITS(4) + 8; + if (state->wbits == 0) + state->wbits = len; + if (len > 15 || len > state->wbits) { + strm->msg = (char *)"invalid window size"; + state->mode = BAD; + break; + } + state->dmax = 1U << len; + Tracev((stderr, "inflate: zlib header ok\n")); + strm->adler = state->check = adler32(0L, Z_NULL, 0); + state->mode = hold & 0x200 ? DICTID : TYPE; + INITBITS(); + break; +#ifdef GUNZIP + case FLAGS: + NEEDBITS(16); + state->flags = (int)(hold); + if ((state->flags & 0xff) != Z_DEFLATED) { + strm->msg = (char *)"unknown compression method"; + state->mode = BAD; + break; + } + if (state->flags & 0xe000) { + strm->msg = (char *)"unknown header flags set"; + state->mode = BAD; + break; + } + if (state->head != Z_NULL) + state->head->text = (int)((hold >> 8) & 1); + if ((state->flags & 0x0200) && (state->wrap & 4)) + CRC2(state->check, hold); + INITBITS(); + state->mode = TIME; + case TIME: + NEEDBITS(32); + if (state->head != Z_NULL) + state->head->time = hold; + if ((state->flags & 0x0200) && (state->wrap & 4)) + CRC4(state->check, hold); + INITBITS(); + state->mode = OS; + case OS: + NEEDBITS(16); + if (state->head != Z_NULL) { + state->head->xflags = (int)(hold & 0xff); + state->head->os = (int)(hold >> 8); + } + if ((state->flags & 0x0200) && (state->wrap & 4)) + CRC2(state->check, hold); + INITBITS(); + state->mode = EXLEN; + case EXLEN: + if (state->flags & 0x0400) { + NEEDBITS(16); + state->length = (unsigned)(hold); + if (state->head != Z_NULL) + state->head->extra_len = (unsigned)hold; + if ((state->flags & 0x0200) && (state->wrap & 4)) + CRC2(state->check, hold); + INITBITS(); + } + else if (state->head != Z_NULL) + state->head->extra = Z_NULL; + state->mode = EXTRA; + case EXTRA: + if (state->flags & 0x0400) { + copy = state->length; + if (copy > have) copy = have; + if (copy) { + if (state->head != Z_NULL && + state->head->extra != Z_NULL) { + len = state->head->extra_len - state->length; + zmemcpy(state->head->extra + len, next, + len + copy > state->head->extra_max ? + state->head->extra_max - len : copy); + } + if ((state->flags & 0x0200) && (state->wrap & 4)) + state->check = crc32(state->check, next, copy); + have -= copy; + next += copy; + state->length -= copy; + } + if (state->length) goto inf_leave; + } + state->length = 0; + state->mode = NAME; + case NAME: + if (state->flags & 0x0800) { + if (have == 0) goto inf_leave; + copy = 0; + do { + len = (unsigned)(next[copy++]); + if (state->head != Z_NULL && + state->head->name != Z_NULL && + state->length < state->head->name_max) + state->head->name[state->length++] = (Bytef)len; + } while (len && copy < have); + if ((state->flags & 0x0200) && (state->wrap & 4)) + state->check = crc32(state->check, next, copy); + have -= copy; + next += copy; + if (len) goto inf_leave; + } + else if (state->head != Z_NULL) + state->head->name = Z_NULL; + state->length = 0; + state->mode = COMMENT; + case COMMENT: + if (state->flags & 0x1000) { + if (have == 0) goto inf_leave; + copy = 0; + do { + len = (unsigned)(next[copy++]); + if (state->head != Z_NULL && + state->head->comment != Z_NULL && + state->length < state->head->comm_max) + state->head->comment[state->length++] = (Bytef)len; + } while (len && copy < have); + if ((state->flags & 0x0200) && (state->wrap & 4)) + state->check = crc32(state->check, next, copy); + have -= copy; + next += copy; + if (len) goto inf_leave; + } + else if (state->head != Z_NULL) + state->head->comment = Z_NULL; + state->mode = HCRC; + case HCRC: + if (state->flags & 0x0200) { + NEEDBITS(16); + if ((state->wrap & 4) && hold != (state->check & 0xffff)) { + strm->msg = (char *)"header crc mismatch"; + state->mode = BAD; + break; + } + INITBITS(); + } + if (state->head != Z_NULL) { + state->head->hcrc = (int)((state->flags >> 9) & 1); + state->head->done = 1; + } + strm->adler = state->check = crc32(0L, Z_NULL, 0); + state->mode = TYPE; + break; +#endif + case DICTID: + NEEDBITS(32); + strm->adler = state->check = ZSWAP32(hold); + INITBITS(); + state->mode = DICT; + case DICT: + if (state->havedict == 0) { + RESTORE(); + return Z_NEED_DICT; + } + strm->adler = state->check = adler32(0L, Z_NULL, 0); + state->mode = TYPE; + case TYPE: + if (flush == Z_BLOCK || flush == Z_TREES) goto inf_leave; + case TYPEDO: + if (state->last) { + BYTEBITS(); + state->mode = CHECK; + break; + } + NEEDBITS(3); + state->last = BITS(1); + DROPBITS(1); + switch (BITS(2)) { + case 0: /* stored block */ + Tracev((stderr, "inflate: stored block%s\n", + state->last ? " (last)" : "")); + state->mode = STORED; + break; + case 1: /* fixed block */ + fixedtables(state); + Tracev((stderr, "inflate: fixed codes block%s\n", + state->last ? " (last)" : "")); + state->mode = LEN_; /* decode codes */ + if (flush == Z_TREES) { + DROPBITS(2); + goto inf_leave; + } + break; + case 2: /* dynamic block */ + Tracev((stderr, "inflate: dynamic codes block%s\n", + state->last ? " (last)" : "")); + state->mode = TABLE; + break; + case 3: + strm->msg = (char *)"invalid block type"; + state->mode = BAD; + } + DROPBITS(2); + break; + case STORED: + BYTEBITS(); /* go to byte boundary */ + NEEDBITS(32); + if ((hold & 0xffff) != ((hold >> 16) ^ 0xffff)) { + strm->msg = (char *)"invalid stored block lengths"; + state->mode = BAD; + break; + } + state->length = (unsigned)hold & 0xffff; + Tracev((stderr, "inflate: stored length %u\n", + state->length)); + INITBITS(); + state->mode = COPY_; + if (flush == Z_TREES) goto inf_leave; + case COPY_: + state->mode = COPY; + case COPY: + copy = state->length; + if (copy) { + if (copy > have) copy = have; + if (copy > left) copy = left; + if (copy == 0) goto inf_leave; + zmemcpy(put, next, copy); + have -= copy; + next += copy; + left -= copy; + put += copy; + state->length -= copy; + break; + } + Tracev((stderr, "inflate: stored end\n")); + state->mode = TYPE; + break; + case TABLE: + NEEDBITS(14); + state->nlen = BITS(5) + 257; + DROPBITS(5); + state->ndist = BITS(5) + 1; + DROPBITS(5); + state->ncode = BITS(4) + 4; + DROPBITS(4); +#ifndef PKZIP_BUG_WORKAROUND + if (state->nlen > 286 || state->ndist > 30) { + strm->msg = (char *)"too many length or distance symbols"; + state->mode = BAD; + break; + } +#endif + Tracev((stderr, "inflate: table sizes ok\n")); + state->have = 0; + state->mode = LENLENS; + case LENLENS: + while (state->have < state->ncode) { + NEEDBITS(3); + state->lens[order[state->have++]] = (unsigned short)BITS(3); + DROPBITS(3); + } + while (state->have < 19) + state->lens[order[state->have++]] = 0; + state->next = state->codes; + state->lencode = (const code FAR *)(state->next); + state->lenbits = 7; + ret = inflate_table(CODES, state->lens, 19, &(state->next), + &(state->lenbits), state->work); + if (ret) { + strm->msg = (char *)"invalid code lengths set"; + state->mode = BAD; + break; + } + Tracev((stderr, "inflate: code lengths ok\n")); + state->have = 0; + state->mode = CODELENS; + case CODELENS: + while (state->have < state->nlen + state->ndist) { + for (;;) { + here = state->lencode[BITS(state->lenbits)]; + if ((unsigned)(here.bits) <= bits) break; + PULLBYTE(); + } + if (here.val < 16) { + DROPBITS(here.bits); + state->lens[state->have++] = here.val; + } + else { + if (here.val == 16) { + NEEDBITS(here.bits + 2); + DROPBITS(here.bits); + if (state->have == 0) { + strm->msg = (char *)"invalid bit length repeat"; + state->mode = BAD; + break; + } + len = state->lens[state->have - 1]; + copy = 3 + BITS(2); + DROPBITS(2); + } + else if (here.val == 17) { + NEEDBITS(here.bits + 3); + DROPBITS(here.bits); + len = 0; + copy = 3 + BITS(3); + DROPBITS(3); + } + else { + NEEDBITS(here.bits + 7); + DROPBITS(here.bits); + len = 0; + copy = 11 + BITS(7); + DROPBITS(7); + } + if (state->have + copy > state->nlen + state->ndist) { + strm->msg = (char *)"invalid bit length repeat"; + state->mode = BAD; + break; + } + while (copy--) + state->lens[state->have++] = (unsigned short)len; + } + } + + /* handle error breaks in while */ + if (state->mode == BAD) break; + + /* check for end-of-block code (better have one) */ + if (state->lens[256] == 0) { + strm->msg = (char *)"invalid code -- missing end-of-block"; + state->mode = BAD; + break; + } + + /* build code tables -- note: do not change the lenbits or distbits + values here (9 and 6) without reading the comments in inftrees.h + concerning the ENOUGH constants, which depend on those values */ + state->next = state->codes; + state->lencode = (const code FAR *)(state->next); + state->lenbits = 9; + ret = inflate_table(LENS, state->lens, state->nlen, &(state->next), + &(state->lenbits), state->work); + if (ret) { + strm->msg = (char *)"invalid literal/lengths set"; + state->mode = BAD; + break; + } + state->distcode = (const code FAR *)(state->next); + state->distbits = 6; + ret = inflate_table(DISTS, state->lens + state->nlen, state->ndist, + &(state->next), &(state->distbits), state->work); + if (ret) { + strm->msg = (char *)"invalid distances set"; + state->mode = BAD; + break; + } + Tracev((stderr, "inflate: codes ok\n")); + state->mode = LEN_; + if (flush == Z_TREES) goto inf_leave; + case LEN_: + state->mode = LEN; + case LEN: + if (have >= INFLATE_FAST_MIN_INPUT && + left >= INFLATE_FAST_MIN_OUTPUT) { + RESTORE(); + inflate_fast(strm, out); + LOAD(); + if (state->mode == TYPE) + state->back = -1; + break; + } + state->back = 0; + for (;;) { + here = state->lencode[BITS(state->lenbits)]; + if ((unsigned)(here.bits) <= bits) break; + PULLBYTE(); + } + if (here.op && (here.op & 0xf0) == 0) { + last = here; + for (;;) { + here = state->lencode[last.val + + (BITS(last.bits + last.op) >> last.bits)]; + if ((unsigned)(last.bits + here.bits) <= bits) break; + PULLBYTE(); + } + DROPBITS(last.bits); + state->back += last.bits; + } + DROPBITS(here.bits); + state->back += here.bits; + state->length = (unsigned)here.val; + if ((int)(here.op) == 0) { + Tracevv((stderr, here.val >= 0x20 && here.val < 0x7f ? + "inflate: literal '%c'\n" : + "inflate: literal 0x%02x\n", here.val)); + state->mode = LIT; + break; + } + if (here.op & 32) { + Tracevv((stderr, "inflate: end of block\n")); + state->back = -1; + state->mode = TYPE; + break; + } + if (here.op & 64) { + strm->msg = (char *)"invalid literal/length code"; + state->mode = BAD; + break; + } + state->extra = (unsigned)(here.op) & 15; + state->mode = LENEXT; + case LENEXT: + if (state->extra) { + NEEDBITS(state->extra); + state->length += BITS(state->extra); + DROPBITS(state->extra); + state->back += state->extra; + } + Tracevv((stderr, "inflate: length %u\n", state->length)); + state->was = state->length; + state->mode = DIST; + case DIST: + for (;;) { + here = state->distcode[BITS(state->distbits)]; + if ((unsigned)(here.bits) <= bits) break; + PULLBYTE(); + } + if ((here.op & 0xf0) == 0) { + last = here; + for (;;) { + here = state->distcode[last.val + + (BITS(last.bits + last.op) >> last.bits)]; + if ((unsigned)(last.bits + here.bits) <= bits) break; + PULLBYTE(); + } + DROPBITS(last.bits); + state->back += last.bits; + } + DROPBITS(here.bits); + state->back += here.bits; + if (here.op & 64) { + strm->msg = (char *)"invalid distance code"; + state->mode = BAD; + break; + } + state->offset = (unsigned)here.val; + state->extra = (unsigned)(here.op) & 15; + state->mode = DISTEXT; + case DISTEXT: + if (state->extra) { + NEEDBITS(state->extra); + state->offset += BITS(state->extra); + DROPBITS(state->extra); + state->back += state->extra; + } +#ifdef INFLATE_STRICT + if (state->offset > state->dmax) { + strm->msg = (char *)"invalid distance too far back"; + state->mode = BAD; + break; + } +#endif + Tracevv((stderr, "inflate: distance %u\n", state->offset)); + state->mode = MATCH; + case MATCH: + if (left == 0) goto inf_leave; + copy = out - left; + if (state->offset > copy) { /* copy from window */ + copy = state->offset - copy; + if (copy > state->whave) { + if (state->sane) { + strm->msg = (char *)"invalid distance too far back"; + state->mode = BAD; + break; + } +#ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR + Trace((stderr, "inflate.c too far\n")); + copy -= state->whave; + if (copy > state->length) copy = state->length; + if (copy > left) copy = left; + left -= copy; + state->length -= copy; + do { + *put++ = 0; + } while (--copy); + if (state->length == 0) state->mode = LEN; + break; +#endif + } + if (copy > state->wnext) { + copy -= state->wnext; + from = state->window + (state->wsize - copy); + } + else + from = state->window + (state->wnext - copy); + if (copy > state->length) copy = state->length; + } + else { /* copy from output */ + from = put - state->offset; + copy = state->length; + } + if (copy > left) copy = left; + left -= copy; + state->length -= copy; + do { + *put++ = *from++; + } while (--copy); + if (state->length == 0) state->mode = LEN; + break; + case LIT: + if (left == 0) goto inf_leave; + *put++ = (unsigned char)(state->length); + left--; + state->mode = LEN; + break; + case CHECK: + if (state->wrap) { + NEEDBITS(32); + out -= left; + strm->total_out += out; + state->total += out; + if ((state->wrap & 4) && out) + strm->adler = state->check = + UPDATE(state->check, put - out, out); + out = left; + if ((state->wrap & 4) && ( +#ifdef GUNZIP + state->flags ? hold : +#endif + ZSWAP32(hold)) != state->check) { + strm->msg = (char *)"incorrect data check"; + state->mode = BAD; + break; + } + INITBITS(); + Tracev((stderr, "inflate: check matches trailer\n")); + } +#ifdef GUNZIP + state->mode = LENGTH; + case LENGTH: + if (state->wrap && state->flags) { + NEEDBITS(32); + if (hold != (state->total & 0xffffffffUL)) { + strm->msg = (char *)"incorrect length check"; + state->mode = BAD; + break; + } + INITBITS(); + Tracev((stderr, "inflate: length matches trailer\n")); + } +#endif + state->mode = DONE; + case DONE: + ret = Z_STREAM_END; + goto inf_leave; + case BAD: + ret = Z_DATA_ERROR; + goto inf_leave; + case MEM: + return Z_MEM_ERROR; + case SYNC: + default: + return Z_STREAM_ERROR; + } + + /* + Return from inflate(), updating the total counts and the check value. + If there was no progress during the inflate() call, return a buffer + error. Call updatewindow() to create and/or update the window state. + Note: a memory error from inflate() is non-recoverable. + */ + inf_leave: + RESTORE(); + if (state->wsize || (out != strm->avail_out && state->mode < BAD && + (state->mode < CHECK || flush != Z_FINISH))) + if (updatewindow(strm, strm->next_out, out - strm->avail_out)) { + state->mode = MEM; + return Z_MEM_ERROR; + } + in -= strm->avail_in; + out -= strm->avail_out; + strm->total_in += in; + strm->total_out += out; + state->total += out; + if ((state->wrap & 4) && out) + strm->adler = state->check = + UPDATE(state->check, strm->next_out - out, out); + strm->data_type = (int)state->bits + (state->last ? 64 : 0) + + (state->mode == TYPE ? 128 : 0) + + (state->mode == LEN_ || state->mode == COPY_ ? 256 : 0); + if (((in == 0 && out == 0) || flush == Z_FINISH) && ret == Z_OK) + ret = Z_BUF_ERROR; + return ret; +} + +int ZEXPORT inflateEnd(strm) +z_streamp strm; +{ + struct inflate_state FAR *state; + if (inflateStateCheck(strm)) + return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + if (state->window != Z_NULL) ZFREE(strm, state->window); + ZFREE(strm, strm->state); + strm->state = Z_NULL; + Tracev((stderr, "inflate: end\n")); + return Z_OK; +} + +int ZEXPORT inflateGetDictionary(strm, dictionary, dictLength) +z_streamp strm; +Bytef *dictionary; +uInt *dictLength; +{ + struct inflate_state FAR *state; + + /* check state */ + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + + /* copy dictionary */ + if (state->whave && dictionary != Z_NULL) { + zmemcpy(dictionary, state->window + state->wnext, + state->whave - state->wnext); + zmemcpy(dictionary + state->whave - state->wnext, + state->window, state->wnext); + } + if (dictLength != Z_NULL) + *dictLength = state->whave; + return Z_OK; +} + +int ZEXPORT inflateSetDictionary(strm, dictionary, dictLength) +z_streamp strm; +const Bytef *dictionary; +uInt dictLength; +{ + struct inflate_state FAR *state; + unsigned long dictid; + int ret; + + /* check state */ + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + if (state->wrap != 0 && state->mode != DICT) + return Z_STREAM_ERROR; + + /* check for correct dictionary identifier */ + if (state->mode == DICT) { + dictid = adler32(0L, Z_NULL, 0); + dictid = adler32(dictid, dictionary, dictLength); + if (dictid != state->check) + return Z_DATA_ERROR; + } + + /* copy dictionary to window using updatewindow(), which will amend the + existing dictionary if appropriate */ + ret = updatewindow(strm, dictionary + dictLength, dictLength); + if (ret) { + state->mode = MEM; + return Z_MEM_ERROR; + } + state->havedict = 1; + Tracev((stderr, "inflate: dictionary set\n")); + return Z_OK; +} + +int ZEXPORT inflateGetHeader(strm, head) +z_streamp strm; +gz_headerp head; +{ + struct inflate_state FAR *state; + + /* check state */ + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + if ((state->wrap & 2) == 0) return Z_STREAM_ERROR; + + /* save header structure */ + state->head = head; + head->done = 0; + return Z_OK; +} + +/* + Search buf[0..len-1] for the pattern: 0, 0, 0xff, 0xff. Return when found + or when out of input. When called, *have is the number of pattern bytes + found in order so far, in 0..3. On return *have is updated to the new + state. If on return *have equals four, then the pattern was found and the + return value is how many bytes were read including the last byte of the + pattern. If *have is less than four, then the pattern has not been found + yet and the return value is len. In the latter case, syncsearch() can be + called again with more data and the *have state. *have is initialized to + zero for the first call. + */ +local unsigned syncsearch(have, buf, len) +unsigned FAR *have; +const unsigned char FAR *buf; +unsigned len; +{ + unsigned got; + unsigned next; + + got = *have; + next = 0; + while (next < len && got < 4) { + if ((int)(buf[next]) == (got < 2 ? 0 : 0xff)) + got++; + else if (buf[next]) + got = 0; + else + got = 4 - got; + next++; + } + *have = got; + return next; +} + +int ZEXPORT inflateSync(strm) +z_streamp strm; +{ + unsigned len; /* number of bytes to look at or looked at */ + unsigned long in, out; /* temporary to save total_in and total_out */ + unsigned char buf[4]; /* to restore bit buffer to byte string */ + struct inflate_state FAR *state; + + /* check parameters */ + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + if (strm->avail_in == 0 && state->bits < 8) return Z_BUF_ERROR; + + /* if first time, start search in bit buffer */ + if (state->mode != SYNC) { + state->mode = SYNC; + state->hold <<= state->bits & 7; + state->bits -= state->bits & 7; + len = 0; + while (state->bits >= 8) { + buf[len++] = (unsigned char)(state->hold); + state->hold >>= 8; + state->bits -= 8; + } + state->have = 0; + syncsearch(&(state->have), buf, len); + } + + /* search available input */ + len = syncsearch(&(state->have), strm->next_in, strm->avail_in); + strm->avail_in -= len; + strm->next_in += len; + strm->total_in += len; + + /* return no joy or set up to restart inflate() on a new block */ + if (state->have != 4) return Z_DATA_ERROR; + in = strm->total_in; out = strm->total_out; + inflateReset(strm); + strm->total_in = in; strm->total_out = out; + state->mode = TYPE; + return Z_OK; +} + +/* + Returns true if inflate is currently at the end of a block generated by + Z_SYNC_FLUSH or Z_FULL_FLUSH. This function is used by one PPP + implementation to provide an additional safety check. PPP uses + Z_SYNC_FLUSH but removes the length bytes of the resulting empty stored + block. When decompressing, PPP checks that at the end of input packet, + inflate is waiting for these length bytes. + */ +int ZEXPORT inflateSyncPoint(strm) +z_streamp strm; +{ + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + return state->mode == STORED && state->bits == 0; +} + +int ZEXPORT inflateCopy(dest, source) +z_streamp dest; +z_streamp source; +{ + struct inflate_state FAR *state; + struct inflate_state FAR *copy; + unsigned char FAR *window; + unsigned wsize; + + /* check input */ + if (inflateStateCheck(source) || dest == Z_NULL) + return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)source->state; + + /* allocate space */ + copy = (struct inflate_state FAR *) + ZALLOC(source, 1, sizeof(struct inflate_state)); + if (copy == Z_NULL) return Z_MEM_ERROR; + window = Z_NULL; + if (state->window != Z_NULL) { + window = (unsigned char FAR *) + ZALLOC(source, 1U << state->wbits, sizeof(unsigned char)); + if (window == Z_NULL) { + ZFREE(source, copy); + return Z_MEM_ERROR; + } + } + + /* copy state */ + zmemcpy((voidpf)dest, (voidpf)source, sizeof(z_stream)); + zmemcpy((voidpf)copy, (voidpf)state, sizeof(struct inflate_state)); + copy->strm = dest; + if (state->lencode >= state->codes && + state->lencode <= state->codes + ENOUGH - 1) { + copy->lencode = copy->codes + (state->lencode - state->codes); + copy->distcode = copy->codes + (state->distcode - state->codes); + } + copy->next = copy->codes + (state->next - state->codes); + if (window != Z_NULL) { + wsize = 1U << state->wbits; + zmemcpy(window, state->window, wsize); + } + copy->window = window; + dest->state = (struct internal_state FAR *)copy; + return Z_OK; +} + +int ZEXPORT inflateUndermine(strm, subvert) +z_streamp strm; +int subvert; +{ + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; +#ifdef INFLATE_ALLOW_INVALID_DISTANCE_TOOFAR_ARRR + state->sane = !subvert; + return Z_OK; +#else + (void)subvert; + state->sane = 1; + return Z_DATA_ERROR; +#endif +} + +int ZEXPORT inflateValidate(strm, check) +z_streamp strm; +int check; +{ + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) return Z_STREAM_ERROR; + state = (struct inflate_state FAR *)strm->state; + if (check) + state->wrap |= 4; + else + state->wrap &= ~4; + return Z_OK; +} + +long ZEXPORT inflateMark(strm) +z_streamp strm; +{ + struct inflate_state FAR *state; + + if (inflateStateCheck(strm)) + return -(1L << 16); + state = (struct inflate_state FAR *)strm->state; + return (long)(((unsigned long)((long)state->back)) << 16) + + (state->mode == COPY ? state->length : + (state->mode == MATCH ? state->was - state->length : 0)); +} + +unsigned long ZEXPORT inflateCodesUsed(strm) +z_streamp strm; +{ + struct inflate_state FAR *state; + if (inflateStateCheck(strm)) return (unsigned long)-1; + state = (struct inflate_state FAR *)strm->state; + return (unsigned long)(state->next - state->codes); +} diff --git a/third_party/zlib/inflate.h b/third_party/zlib/inflate.h new file mode 100644 index 00000000..a46cce6b --- /dev/null +++ b/third_party/zlib/inflate.h @@ -0,0 +1,125 @@ +/* inflate.h -- internal inflate state definition + * Copyright (C) 1995-2016 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +/* WARNING: this file should *not* be used by applications. It is + part of the implementation of the compression library and is + subject to change. Applications should only use zlib.h. + */ + +/* define NO_GZIP when compiling if you want to disable gzip header and + trailer decoding by inflate(). NO_GZIP would be used to avoid linking in + the crc code when it is not needed. For shared libraries, gzip decoding + should be left enabled. */ +#ifndef NO_GZIP +# define GUNZIP +#endif + +/* Possible inflate modes between inflate() calls */ +typedef enum { + HEAD = 16180, /* i: waiting for magic header */ + FLAGS, /* i: waiting for method and flags (gzip) */ + TIME, /* i: waiting for modification time (gzip) */ + OS, /* i: waiting for extra flags and operating system (gzip) */ + EXLEN, /* i: waiting for extra length (gzip) */ + EXTRA, /* i: waiting for extra bytes (gzip) */ + NAME, /* i: waiting for end of file name (gzip) */ + COMMENT, /* i: waiting for end of comment (gzip) */ + HCRC, /* i: waiting for header crc (gzip) */ + DICTID, /* i: waiting for dictionary check value */ + DICT, /* waiting for inflateSetDictionary() call */ + TYPE, /* i: waiting for type bits, including last-flag bit */ + TYPEDO, /* i: same, but skip check to exit inflate on new block */ + STORED, /* i: waiting for stored size (length and complement) */ + COPY_, /* i/o: same as COPY below, but only first time in */ + COPY, /* i/o: waiting for input or output to copy stored block */ + TABLE, /* i: waiting for dynamic block table lengths */ + LENLENS, /* i: waiting for code length code lengths */ + CODELENS, /* i: waiting for length/lit and distance code lengths */ + LEN_, /* i: same as LEN below, but only first time in */ + LEN, /* i: waiting for length/lit/eob code */ + LENEXT, /* i: waiting for length extra bits */ + DIST, /* i: waiting for distance code */ + DISTEXT, /* i: waiting for distance extra bits */ + MATCH, /* o: waiting for output space to copy string */ + LIT, /* o: waiting for output space to write literal */ + CHECK, /* i: waiting for 32-bit check value */ + LENGTH, /* i: waiting for 32-bit length (gzip) */ + DONE, /* finished check, done -- remain here until reset */ + BAD, /* got a data error -- remain here until reset */ + MEM, /* got an inflate() memory error -- remain here until reset */ + SYNC /* looking for synchronization bytes to restart inflate() */ +} inflate_mode; + +/* + State transitions between above modes - + + (most modes can go to BAD or MEM on error -- not shown for clarity) + + Process header: + HEAD -> (gzip) or (zlib) or (raw) + (gzip) -> FLAGS -> TIME -> OS -> EXLEN -> EXTRA -> NAME -> COMMENT -> + HCRC -> TYPE + (zlib) -> DICTID or TYPE + DICTID -> DICT -> TYPE + (raw) -> TYPEDO + Read deflate blocks: + TYPE -> TYPEDO -> STORED or TABLE or LEN_ or CHECK + STORED -> COPY_ -> COPY -> TYPE + TABLE -> LENLENS -> CODELENS -> LEN_ + LEN_ -> LEN + Read deflate codes in fixed or dynamic block: + LEN -> LENEXT or LIT or TYPE + LENEXT -> DIST -> DISTEXT -> MATCH -> LEN + LIT -> LEN + Process trailer: + CHECK -> LENGTH -> DONE + */ + +/* State maintained between inflate() calls -- approximately 7K bytes, not + including the allocated sliding window, which is up to 32K bytes. */ +struct inflate_state { + z_streamp strm; /* pointer back to this zlib stream */ + inflate_mode mode; /* current inflate mode */ + int last; /* true if processing last block */ + int wrap; /* bit 0 true for zlib, bit 1 true for gzip, + bit 2 true to validate check value */ + int havedict; /* true if dictionary provided */ + int flags; /* gzip header method and flags (0 if zlib) */ + unsigned dmax; /* zlib header max distance (INFLATE_STRICT) */ + unsigned long check; /* protected copy of check value */ + unsigned long total; /* protected copy of output count */ + gz_headerp head; /* where to save gzip header information */ + /* sliding window */ + unsigned wbits; /* log base 2 of requested window size */ + unsigned wsize; /* window size or zero if not using window */ + unsigned whave; /* valid bytes in the window */ + unsigned wnext; /* window write index */ + unsigned char FAR *window; /* allocated sliding window, if needed */ + /* bit accumulator */ + unsigned long hold; /* input bit accumulator */ + unsigned bits; /* number of bits in "in" */ + /* for string and stored block copying */ + unsigned length; /* literal or length of data to copy */ + unsigned offset; /* distance back to copy string from */ + /* for table and code decoding */ + unsigned extra; /* extra bits needed */ + /* fixed and dynamic code tables */ + code const FAR *lencode; /* starting table for length/literal codes */ + code const FAR *distcode; /* starting table for distance codes */ + unsigned lenbits; /* index bits for lencode */ + unsigned distbits; /* index bits for distcode */ + /* dynamic table building */ + unsigned ncode; /* number of code length code lengths */ + unsigned nlen; /* number of length code lengths */ + unsigned ndist; /* number of distance code lengths */ + unsigned have; /* number of code lengths in lens[] */ + code FAR *next; /* next available space in codes[] */ + unsigned short lens[320]; /* temporary storage for code lengths */ + unsigned short work[288]; /* work area for code table building */ + code codes[ENOUGH]; /* space for code tables */ + int sane; /* if false, allow invalid distance too far */ + int back; /* bits back of last unprocessed length/lit */ + unsigned was; /* initial length of match */ +}; diff --git a/third_party/zlib/inftrees.c b/third_party/zlib/inftrees.c new file mode 100644 index 00000000..2ea08fc1 --- /dev/null +++ b/third_party/zlib/inftrees.c @@ -0,0 +1,304 @@ +/* inftrees.c -- generate Huffman trees for efficient decoding + * Copyright (C) 1995-2017 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +#include "zutil.h" +#include "inftrees.h" + +#define MAXBITS 15 + +const char inflate_copyright[] = + " inflate 1.2.11 Copyright 1995-2017 Mark Adler "; +/* + If you use the zlib library in a product, an acknowledgment is welcome + in the documentation of your product. If for some reason you cannot + include such an acknowledgment, I would appreciate that you keep this + copyright string in the executable of your product. + */ + +/* + Build a set of tables to decode the provided canonical Huffman code. + The code lengths are lens[0..codes-1]. The result starts at *table, + whose indices are 0..2^bits-1. work is a writable array of at least + lens shorts, which is used as a work area. type is the type of code + to be generated, CODES, LENS, or DISTS. On return, zero is success, + -1 is an invalid code, and +1 means that ENOUGH isn't enough. table + on return points to the next available entry's address. bits is the + requested root table index bits, and on return it is the actual root + table index bits. It will differ if the request is greater than the + longest code or if it is less than the shortest code. + */ +int ZLIB_INTERNAL inflate_table(type, lens, codes, table, bits, work) +codetype type; +unsigned short FAR *lens; +unsigned codes; +code FAR * FAR *table; +unsigned FAR *bits; +unsigned short FAR *work; +{ + unsigned len; /* a code's length in bits */ + unsigned sym; /* index of code symbols */ + unsigned min, max; /* minimum and maximum code lengths */ + unsigned root; /* number of index bits for root table */ + unsigned curr; /* number of index bits for current table */ + unsigned drop; /* code bits to drop for sub-table */ + int left; /* number of prefix codes available */ + unsigned used; /* code entries in table used */ + unsigned huff; /* Huffman code */ + unsigned incr; /* for incrementing code, index */ + unsigned fill; /* index for replicating entries */ + unsigned low; /* low bits for current root entry */ + unsigned mask; /* mask for low root bits */ + code here; /* table entry for duplication */ + code FAR *next; /* next available space in table */ + const unsigned short FAR *base; /* base value table to use */ + const unsigned short FAR *extra; /* extra bits table to use */ + unsigned match; /* use base and extra for symbol >= match */ + unsigned short count[MAXBITS+1]; /* number of codes of each length */ + unsigned short offs[MAXBITS+1]; /* offsets in table for each length */ + static const unsigned short lbase[31] = { /* Length codes 257..285 base */ + 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 17, 19, 23, 27, 31, + 35, 43, 51, 59, 67, 83, 99, 115, 131, 163, 195, 227, 258, 0, 0}; + static const unsigned short lext[31] = { /* Length codes 257..285 extra */ + 16, 16, 16, 16, 16, 16, 16, 16, 17, 17, 17, 17, 18, 18, 18, 18, + 19, 19, 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, 16, 77, 202}; + static const unsigned short dbase[32] = { /* Distance codes 0..29 base */ + 1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193, + 257, 385, 513, 769, 1025, 1537, 2049, 3073, 4097, 6145, + 8193, 12289, 16385, 24577, 0, 0}; + static const unsigned short dext[32] = { /* Distance codes 0..29 extra */ + 16, 16, 16, 16, 17, 17, 18, 18, 19, 19, 20, 20, 21, 21, 22, 22, + 23, 23, 24, 24, 25, 25, 26, 26, 27, 27, + 28, 28, 29, 29, 64, 64}; + + /* + Process a set of code lengths to create a canonical Huffman code. The + code lengths are lens[0..codes-1]. Each length corresponds to the + symbols 0..codes-1. The Huffman code is generated by first sorting the + symbols by length from short to long, and retaining the symbol order + for codes with equal lengths. Then the code starts with all zero bits + for the first code of the shortest length, and the codes are integer + increments for the same length, and zeros are appended as the length + increases. For the deflate format, these bits are stored backwards + from their more natural integer increment ordering, and so when the + decoding tables are built in the large loop below, the integer codes + are incremented backwards. + + This routine assumes, but does not check, that all of the entries in + lens[] are in the range 0..MAXBITS. The caller must assure this. + 1..MAXBITS is interpreted as that code length. zero means that that + symbol does not occur in this code. + + The codes are sorted by computing a count of codes for each length, + creating from that a table of starting indices for each length in the + sorted table, and then entering the symbols in order in the sorted + table. The sorted table is work[], with that space being provided by + the caller. + + The length counts are used for other purposes as well, i.e. finding + the minimum and maximum length codes, determining if there are any + codes at all, checking for a valid set of lengths, and looking ahead + at length counts to determine sub-table sizes when building the + decoding tables. + */ + + /* accumulate lengths for codes (assumes lens[] all in 0..MAXBITS) */ + for (len = 0; len <= MAXBITS; len++) + count[len] = 0; + for (sym = 0; sym < codes; sym++) + count[lens[sym]]++; + + /* bound code lengths, force root to be within code lengths */ + root = *bits; + for (max = MAXBITS; max >= 1; max--) + if (count[max] != 0) break; + if (root > max) root = max; + if (max == 0) { /* no symbols to code at all */ + here.op = (unsigned char)64; /* invalid code marker */ + here.bits = (unsigned char)1; + here.val = (unsigned short)0; + *(*table)++ = here; /* make a table to force an error */ + *(*table)++ = here; + *bits = 1; + return 0; /* no symbols, but wait for decoding to report error */ + } + for (min = 1; min < max; min++) + if (count[min] != 0) break; + if (root < min) root = min; + + /* check for an over-subscribed or incomplete set of lengths */ + left = 1; + for (len = 1; len <= MAXBITS; len++) { + left <<= 1; + left -= count[len]; + if (left < 0) return -1; /* over-subscribed */ + } + if (left > 0 && (type == CODES || max != 1)) + return -1; /* incomplete set */ + + /* generate offsets into symbol table for each length for sorting */ + offs[1] = 0; + for (len = 1; len < MAXBITS; len++) + offs[len + 1] = offs[len] + count[len]; + + /* sort symbols by length, by symbol order within each length */ + for (sym = 0; sym < codes; sym++) + if (lens[sym] != 0) work[offs[lens[sym]]++] = (unsigned short)sym; + + /* + Create and fill in decoding tables. In this loop, the table being + filled is at next and has curr index bits. The code being used is huff + with length len. That code is converted to an index by dropping drop + bits off of the bottom. For codes where len is less than drop + curr, + those top drop + curr - len bits are incremented through all values to + fill the table with replicated entries. + + root is the number of index bits for the root table. When len exceeds + root, sub-tables are created pointed to by the root entry with an index + of the low root bits of huff. This is saved in low to check for when a + new sub-table should be started. drop is zero when the root table is + being filled, and drop is root when sub-tables are being filled. + + When a new sub-table is needed, it is necessary to look ahead in the + code lengths to determine what size sub-table is needed. The length + counts are used for this, and so count[] is decremented as codes are + entered in the tables. + + used keeps track of how many table entries have been allocated from the + provided *table space. It is checked for LENS and DIST tables against + the constants ENOUGH_LENS and ENOUGH_DISTS to guard against changes in + the initial root table size constants. See the comments in inftrees.h + for more information. + + sym increments through all symbols, and the loop terminates when + all codes of length max, i.e. all codes, have been processed. This + routine permits incomplete codes, so another loop after this one fills + in the rest of the decoding tables with invalid code markers. + */ + + /* set up for code type */ + switch (type) { + case CODES: + base = extra = work; /* dummy value--not used */ + match = 20; + break; + case LENS: + base = lbase; + extra = lext; + match = 257; + break; + default: /* DISTS */ + base = dbase; + extra = dext; + match = 0; + } + + /* initialize state for loop */ + huff = 0; /* starting code */ + sym = 0; /* starting code symbol */ + len = min; /* starting code length */ + next = *table; /* current table to fill in */ + curr = root; /* current table index bits */ + drop = 0; /* current bits to drop from code for index */ + low = (unsigned)(-1); /* trigger new sub-table when len > root */ + used = 1U << root; /* use root table entries */ + mask = used - 1; /* mask for comparing low */ + + /* check available table space */ + if ((type == LENS && used > ENOUGH_LENS) || + (type == DISTS && used > ENOUGH_DISTS)) + return 1; + + /* process all codes and make table entries */ + for (;;) { + /* create table entry */ + here.bits = (unsigned char)(len - drop); + if (work[sym] + 1U < match) { + here.op = (unsigned char)0; + here.val = work[sym]; + } + else if (work[sym] >= match) { + here.op = (unsigned char)(extra[work[sym] - match]); + here.val = base[work[sym] - match]; + } + else { + here.op = (unsigned char)(32 + 64); /* end of block */ + here.val = 0; + } + + /* replicate for those indices with low len bits equal to huff */ + incr = 1U << (len - drop); + fill = 1U << curr; + min = fill; /* save offset to next table */ + do { + fill -= incr; + next[(huff >> drop) + fill] = here; + } while (fill != 0); + + /* backwards increment the len-bit code huff */ + incr = 1U << (len - 1); + while (huff & incr) + incr >>= 1; + if (incr != 0) { + huff &= incr - 1; + huff += incr; + } + else + huff = 0; + + /* go to next symbol, update count, len */ + sym++; + if (--(count[len]) == 0) { + if (len == max) break; + len = lens[work[sym]]; + } + + /* create new sub-table if needed */ + if (len > root && (huff & mask) != low) { + /* if first time, transition to sub-tables */ + if (drop == 0) + drop = root; + + /* increment past last table */ + next += min; /* here min is 1 << curr */ + + /* determine length of next table */ + curr = len - drop; + left = (int)(1 << curr); + while (curr + drop < max) { + left -= count[curr + drop]; + if (left <= 0) break; + curr++; + left <<= 1; + } + + /* check for enough space */ + used += 1U << curr; + if ((type == LENS && used > ENOUGH_LENS) || + (type == DISTS && used > ENOUGH_DISTS)) + return 1; + + /* point entry in root table to sub-table */ + low = huff & mask; + (*table)[low].op = (unsigned char)curr; + (*table)[low].bits = (unsigned char)root; + (*table)[low].val = (unsigned short)(next - *table); + } + } + + /* fill in remaining table entry if code is incomplete (guaranteed to have + at most one remaining entry, since if the code is incomplete, the + maximum code length that was allowed to get this far is one bit) */ + if (huff != 0) { + here.op = (unsigned char)64; /* invalid code marker */ + here.bits = (unsigned char)(len - drop); + here.val = (unsigned short)0; + next[huff] = here; + } + + /* set return parameters */ + *table += used; + *bits = root; + return 0; +} diff --git a/third_party/zlib/inftrees.h b/third_party/zlib/inftrees.h new file mode 100644 index 00000000..baa53a0b --- /dev/null +++ b/third_party/zlib/inftrees.h @@ -0,0 +1,62 @@ +/* inftrees.h -- header to use inftrees.c + * Copyright (C) 1995-2005, 2010 Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +/* WARNING: this file should *not* be used by applications. It is + part of the implementation of the compression library and is + subject to change. Applications should only use zlib.h. + */ + +/* Structure for decoding tables. Each entry provides either the + information needed to do the operation requested by the code that + indexed that table entry, or it provides a pointer to another + table that indexes more bits of the code. op indicates whether + the entry is a pointer to another table, a literal, a length or + distance, an end-of-block, or an invalid code. For a table + pointer, the low four bits of op is the number of index bits of + that table. For a length or distance, the low four bits of op + is the number of extra bits to get after the code. bits is + the number of bits in this code or part of the code to drop off + of the bit buffer. val is the actual byte to output in the case + of a literal, the base length or distance, or the offset from + the current table to the next table. Each entry is four bytes. */ +typedef struct { + unsigned char op; /* operation, extra bits, table bits */ + unsigned char bits; /* bits in this part of the code */ + unsigned short val; /* offset in table or code value */ +} code; + +/* op values as set by inflate_table(): + 00000000 - literal + 0000tttt - table link, tttt != 0 is the number of table index bits + 0001eeee - length or distance, eeee is the number of extra bits + 01100000 - end of block + 01000000 - invalid code + */ + +/* Maximum size of the dynamic table. The maximum number of code structures is + 1444, which is the sum of 852 for literal/length codes and 592 for distance + codes. These values were found by exhaustive searches using the program + examples/enough.c found in the zlib distribtution. The arguments to that + program are the number of symbols, the initial root table size, and the + maximum bit length of a code. "enough 286 9 15" for literal/length codes + returns returns 852, and "enough 30 6 15" for distance codes returns 592. + The initial root table size (9 or 6) is found in the fifth argument of the + inflate_table() calls in inflate.c and infback.c. If the root table size is + changed, then these maximum sizes would be need to be recalculated and + updated. */ +#define ENOUGH_LENS 852 +#define ENOUGH_DISTS 592 +#define ENOUGH (ENOUGH_LENS+ENOUGH_DISTS) + +/* Type of code to build for inflate_table() */ +typedef enum { + CODES, + LENS, + DISTS +} codetype; + +int ZLIB_INTERNAL inflate_table OF((codetype type, unsigned short FAR *lens, + unsigned codes, code FAR * FAR *table, + unsigned FAR *bits, unsigned short FAR *work)); diff --git a/third_party/zlib/trees.c b/third_party/zlib/trees.c new file mode 100644 index 00000000..5f89d056 --- /dev/null +++ b/third_party/zlib/trees.c @@ -0,0 +1,1181 @@ +/* trees.c -- output deflated data using Huffman coding + * Copyright (C) 1995-2017 Jean-loup Gailly + * detect_data_type() function provided freely by Cosmin Truta, 2006 + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +/* + * ALGORITHM + * + * The "deflation" process uses several Huffman trees. The more + * common source values are represented by shorter bit sequences. + * + * Each code tree is stored in a compressed form which is itself + * a Huffman encoding of the lengths of all the code strings (in + * ascending order by source values). The actual code strings are + * reconstructed from the lengths in the inflate process, as described + * in the deflate specification. + * + * REFERENCES + * + * Deutsch, L.P.,"'Deflate' Compressed Data Format Specification". + * Available in ftp.uu.net:/pub/archiving/zip/doc/deflate-1.1.doc + * + * Storer, James A. + * Data Compression: Methods and Theory, pp. 49-50. + * Computer Science Press, 1988. ISBN 0-7167-8156-5. + * + * Sedgewick, R. + * Algorithms, p290. + * Addison-Wesley, 1983. ISBN 0-201-06672-6. + */ + +/* @(#) $Id$ */ + +/* #define GEN_TREES_H */ + +#include "deflate.h" + +#ifdef ZLIB_DEBUG +# include <ctype.h> +#endif + +/* =========================================================================== + * Constants + */ + +#define MAX_BL_BITS 7 +/* Bit length codes must not exceed MAX_BL_BITS bits */ + +#define END_BLOCK 256 +/* end of block literal code */ + +#define REP_3_6 16 +/* repeat previous bit length 3-6 times (2 bits of repeat count) */ + +#define REPZ_3_10 17 +/* repeat a zero length 3-10 times (3 bits of repeat count) */ + +#define REPZ_11_138 18 +/* repeat a zero length 11-138 times (7 bits of repeat count) */ + +local const int extra_lbits[LENGTH_CODES] /* extra bits for each length code */ + = {0,0,0,0,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,4,5,5,5,5,0}; + +local const int extra_dbits[D_CODES] /* extra bits for each distance code */ + = {0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13}; + +local const int extra_blbits[BL_CODES]/* extra bits for each bit length code */ + = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,3,7}; + +local const uch bl_order[BL_CODES] + = {16,17,18,0,8,7,9,6,10,5,11,4,12,3,13,2,14,1,15}; +/* The lengths of the bit length codes are sent in order of decreasing + * probability, to avoid transmitting the lengths for unused bit length codes. + */ + +/* =========================================================================== + * Local data. These are initialized only once. + */ + +#define DIST_CODE_LEN 512 /* see definition of array dist_code below */ + +#if defined(GEN_TREES_H) || !defined(STDC) +/* non ANSI compilers may not accept trees.h */ + +local ct_data static_ltree[L_CODES+2]; +/* The static literal tree. Since the bit lengths are imposed, there is no + * need for the L_CODES extra codes used during heap construction. However + * The codes 286 and 287 are needed to build a canonical tree (see _tr_init + * below). + */ + +local ct_data static_dtree[D_CODES]; +/* The static distance tree. (Actually a trivial tree since all codes use + * 5 bits.) + */ + +uch _dist_code[DIST_CODE_LEN]; +/* Distance codes. The first 256 values correspond to the distances + * 3 .. 258, the last 256 values correspond to the top 8 bits of + * the 15 bit distances. + */ + +uch _length_code[MAX_MATCH-MIN_MATCH+1]; +/* length code for each normalized match length (0 == MIN_MATCH) */ + +local int base_length[LENGTH_CODES]; +/* First normalized length for each code (0 = MIN_MATCH) */ + +local int base_dist[D_CODES]; +/* First normalized distance for each code (0 = distance of 1) */ + +#else +# include "trees.h" +#endif /* GEN_TREES_H */ + +struct static_tree_desc_s { + const ct_data *static_tree; /* static tree or NULL */ + const intf *extra_bits; /* extra bits for each code or NULL */ + int extra_base; /* base index for extra_bits */ + int elems; /* max number of elements in the tree */ + int max_length; /* max bit length for the codes */ +}; + +local const static_tree_desc static_l_desc = +{static_ltree, extra_lbits, LITERALS+1, L_CODES, MAX_BITS}; + +local const static_tree_desc static_d_desc = +{static_dtree, extra_dbits, 0, D_CODES, MAX_BITS}; + +local const static_tree_desc static_bl_desc = +{(const ct_data *)0, extra_blbits, 0, BL_CODES, MAX_BL_BITS}; + +/* =========================================================================== + * Local (static) routines in this file. + */ + +local void tr_static_init OF((void)); +local void init_block OF((deflate_state *s)); +local void pqdownheap OF((deflate_state *s, ct_data *tree, int k)); +local void gen_bitlen OF((deflate_state *s, tree_desc *desc)); +local void gen_codes OF((ct_data *tree, int max_code, ushf *bl_count)); +local void build_tree OF((deflate_state *s, tree_desc *desc)); +local void scan_tree OF((deflate_state *s, ct_data *tree, int max_code)); +local void send_tree OF((deflate_state *s, ct_data *tree, int max_code)); +local int build_bl_tree OF((deflate_state *s)); +local void send_all_trees OF((deflate_state *s, int lcodes, int dcodes, + int blcodes)); +local void compress_block OF((deflate_state *s, const ct_data *ltree, + const ct_data *dtree)); +local int detect_data_type OF((deflate_state *s)); +local unsigned bi_reverse OF((unsigned value, int length)); +local void bi_windup OF((deflate_state *s)); +local void bi_flush OF((deflate_state *s)); + +#ifdef GEN_TREES_H +local void gen_trees_header OF((void)); +#endif + +#ifndef ZLIB_DEBUG +# define send_code(s, c, tree) send_bits(s, tree[c].Code, tree[c].Len) + /* Send a code of the given tree. c and tree must not have side effects */ + +#else /* !ZLIB_DEBUG */ +# define send_code(s, c, tree) \ + { if (z_verbose>2) fprintf(stderr,"\ncd %3d ",(c)); \ + send_bits(s, tree[c].Code, tree[c].Len); } +#endif + +/* =========================================================================== + * Output a short LSB first on the stream. + * IN assertion: there is enough room in pendingBuf. + */ +#define put_short(s, w) { \ + put_byte(s, (uch)((w) & 0xff)); \ + put_byte(s, (uch)((ush)(w) >> 8)); \ +} + +/* =========================================================================== + * Send a value on a given number of bits. + * IN assertion: length <= 16 and value fits in length bits. + */ +#ifdef ZLIB_DEBUG +local void send_bits OF((deflate_state *s, int value, int length)); + +local void send_bits(s, value, length) + deflate_state *s; + int value; /* value to send */ + int length; /* number of bits */ +{ + Tracevv((stderr," l %2d v %4x ", length, value)); + Assert(length > 0 && length <= 15, "invalid length"); + s->bits_sent += (ulg)length; + + /* If not enough room in bi_buf, use (valid) bits from bi_buf and + * (16 - bi_valid) bits from value, leaving (width - (16-bi_valid)) + * unused bits in value. + */ + if (s->bi_valid > (int)Buf_size - length) { + s->bi_buf |= (ush)value << s->bi_valid; + put_short(s, s->bi_buf); + s->bi_buf = (ush)value >> (Buf_size - s->bi_valid); + s->bi_valid += length - Buf_size; + } else { + s->bi_buf |= (ush)value << s->bi_valid; + s->bi_valid += length; + } +} +#else /* !ZLIB_DEBUG */ + +#define send_bits(s, value, length) \ +{ int len = length;\ + if (s->bi_valid > (int)Buf_size - len) {\ + int val = (int)value;\ + s->bi_buf |= (ush)val << s->bi_valid;\ + put_short(s, s->bi_buf);\ + s->bi_buf = (ush)val >> (Buf_size - s->bi_valid);\ + s->bi_valid += len - Buf_size;\ + } else {\ + s->bi_buf |= (ush)(value) << s->bi_valid;\ + s->bi_valid += len;\ + }\ +} +#endif /* ZLIB_DEBUG */ + + +/* the arguments must not have side effects */ + +/* =========================================================================== + * Initialize the various 'constant' tables. + */ +local void tr_static_init() +{ +#if defined(GEN_TREES_H) || !defined(STDC) + static int static_init_done = 0; + int n; /* iterates over tree elements */ + int bits; /* bit counter */ + int length; /* length value */ + int code; /* code value */ + int dist; /* distance index */ + ush bl_count[MAX_BITS+1]; + /* number of codes at each bit length for an optimal tree */ + + if (static_init_done) return; + + /* For some embedded targets, global variables are not initialized: */ +#ifdef NO_INIT_GLOBAL_POINTERS + static_l_desc.static_tree = static_ltree; + static_l_desc.extra_bits = extra_lbits; + static_d_desc.static_tree = static_dtree; + static_d_desc.extra_bits = extra_dbits; + static_bl_desc.extra_bits = extra_blbits; +#endif + + /* Initialize the mapping length (0..255) -> length code (0..28) */ + length = 0; + for (code = 0; code < LENGTH_CODES-1; code++) { + base_length[code] = length; + for (n = 0; n < (1<<extra_lbits[code]); n++) { + _length_code[length++] = (uch)code; + } + } + Assert (length == 256, "tr_static_init: length != 256"); + /* Note that the length 255 (match length 258) can be represented + * in two different ways: code 284 + 5 bits or code 285, so we + * overwrite length_code[255] to use the best encoding: + */ + _length_code[length-1] = (uch)code; + + /* Initialize the mapping dist (0..32K) -> dist code (0..29) */ + dist = 0; + for (code = 0 ; code < 16; code++) { + base_dist[code] = dist; + for (n = 0; n < (1<<extra_dbits[code]); n++) { + _dist_code[dist++] = (uch)code; + } + } + Assert (dist == 256, "tr_static_init: dist != 256"); + dist >>= 7; /* from now on, all distances are divided by 128 */ + for ( ; code < D_CODES; code++) { + base_dist[code] = dist << 7; + for (n = 0; n < (1<<(extra_dbits[code]-7)); n++) { + _dist_code[256 + dist++] = (uch)code; + } + } + Assert (dist == 256, "tr_static_init: 256+dist != 512"); + + /* Construct the codes of the static literal tree */ + for (bits = 0; bits <= MAX_BITS; bits++) bl_count[bits] = 0; + n = 0; + while (n <= 143) static_ltree[n++].Len = 8, bl_count[8]++; + while (n <= 255) static_ltree[n++].Len = 9, bl_count[9]++; + while (n <= 279) static_ltree[n++].Len = 7, bl_count[7]++; + while (n <= 287) static_ltree[n++].Len = 8, bl_count[8]++; + /* Codes 286 and 287 do not exist, but we must include them in the + * tree construction to get a canonical Huffman tree (longest code + * all ones) + */ + gen_codes((ct_data *)static_ltree, L_CODES+1, bl_count); + + /* The static distance tree is trivial: */ + for (n = 0; n < D_CODES; n++) { + static_dtree[n].Len = 5; + static_dtree[n].Code = bi_reverse((unsigned)n, 5); + } + static_init_done = 1; + +# ifdef GEN_TREES_H + gen_trees_header(); +# endif +#endif /* defined(GEN_TREES_H) || !defined(STDC) */ +} + +/* =========================================================================== + * Genererate the file trees.h describing the static trees. + */ +#ifdef GEN_TREES_H +# ifndef ZLIB_DEBUG +# include <stdio.h> +# endif + +# define SEPARATOR(i, last, width) \ + ((i) == (last)? "\n};\n\n" : \ + ((i) % (width) == (width)-1 ? ",\n" : ", ")) + +void gen_trees_header() +{ + FILE *header = fopen("trees.h", "w"); + int i; + + Assert (header != NULL, "Can't open trees.h"); + fprintf(header, + "/* header created automatically with -DGEN_TREES_H */\n\n"); + + fprintf(header, "local const ct_data static_ltree[L_CODES+2] = {\n"); + for (i = 0; i < L_CODES+2; i++) { + fprintf(header, "{{%3u},{%3u}}%s", static_ltree[i].Code, + static_ltree[i].Len, SEPARATOR(i, L_CODES+1, 5)); + } + + fprintf(header, "local const ct_data static_dtree[D_CODES] = {\n"); + for (i = 0; i < D_CODES; i++) { + fprintf(header, "{{%2u},{%2u}}%s", static_dtree[i].Code, + static_dtree[i].Len, SEPARATOR(i, D_CODES-1, 5)); + } + + fprintf(header, "const uch ZLIB_INTERNAL _dist_code[DIST_CODE_LEN] = {\n"); + for (i = 0; i < DIST_CODE_LEN; i++) { + fprintf(header, "%2u%s", _dist_code[i], + SEPARATOR(i, DIST_CODE_LEN-1, 20)); + } + + fprintf(header, + "const uch ZLIB_INTERNAL _length_code[MAX_MATCH-MIN_MATCH+1]= {\n"); + for (i = 0; i < MAX_MATCH-MIN_MATCH+1; i++) { + fprintf(header, "%2u%s", _length_code[i], + SEPARATOR(i, MAX_MATCH-MIN_MATCH, 20)); + } + + fprintf(header, "local const int base_length[LENGTH_CODES] = {\n"); + for (i = 0; i < LENGTH_CODES; i++) { + fprintf(header, "%1u%s", base_length[i], + SEPARATOR(i, LENGTH_CODES-1, 20)); + } + + fprintf(header, "local const int base_dist[D_CODES] = {\n"); + for (i = 0; i < D_CODES; i++) { + fprintf(header, "%5u%s", base_dist[i], + SEPARATOR(i, D_CODES-1, 10)); + } + + fclose(header); +} +#endif /* GEN_TREES_H */ + +/* =========================================================================== + * Initialize the tree data structures for a new zlib stream. + */ +void ZLIB_INTERNAL _tr_init(s) + deflate_state *s; +{ + tr_static_init(); + + s->l_desc.dyn_tree = s->dyn_ltree; + s->l_desc.stat_desc = &static_l_desc; + + s->d_desc.dyn_tree = s->dyn_dtree; + s->d_desc.stat_desc = &static_d_desc; + + s->bl_desc.dyn_tree = s->bl_tree; + s->bl_desc.stat_desc = &static_bl_desc; + + s->bi_buf = 0; + s->bi_valid = 0; +#ifdef ZLIB_DEBUG + s->compressed_len = 0L; + s->bits_sent = 0L; +#endif + + /* Initialize the first block of the first file: */ + init_block(s); +} + +/* =========================================================================== + * Initialize a new block. + */ +local void init_block(s) + deflate_state *s; +{ + int n; /* iterates over tree elements */ + + /* Initialize the trees. */ + for (n = 0; n < L_CODES; n++) s->dyn_ltree[n].Freq = 0; + for (n = 0; n < D_CODES; n++) s->dyn_dtree[n].Freq = 0; + for (n = 0; n < BL_CODES; n++) s->bl_tree[n].Freq = 0; + + s->dyn_ltree[END_BLOCK].Freq = 1; + s->opt_len = s->static_len = 0L; + s->sym_next = s->matches = 0; +} + +#define SMALLEST 1 +/* Index within the heap array of least frequent node in the Huffman tree */ + + +/* =========================================================================== + * Remove the smallest element from the heap and recreate the heap with + * one less element. Updates heap and heap_len. + */ +#define pqremove(s, tree, top) \ +{\ + top = s->heap[SMALLEST]; \ + s->heap[SMALLEST] = s->heap[s->heap_len--]; \ + pqdownheap(s, tree, SMALLEST); \ +} + +/* =========================================================================== + * Compares to subtrees, using the tree depth as tie breaker when + * the subtrees have equal frequency. This minimizes the worst case length. + */ +#define smaller(tree, n, m, depth) \ + (tree[n].Freq < tree[m].Freq || \ + (tree[n].Freq == tree[m].Freq && depth[n] <= depth[m])) + +/* =========================================================================== + * Restore the heap property by moving down the tree starting at node k, + * exchanging a node with the smallest of its two sons if necessary, stopping + * when the heap property is re-established (each father smaller than its + * two sons). + */ +local void pqdownheap(s, tree, k) + deflate_state *s; + ct_data *tree; /* the tree to restore */ + int k; /* node to move down */ +{ + int v = s->heap[k]; + int j = k << 1; /* left son of k */ + while (j <= s->heap_len) { + /* Set j to the smallest of the two sons: */ + if (j < s->heap_len && + smaller(tree, s->heap[j+1], s->heap[j], s->depth)) { + j++; + } + /* Exit if v is smaller than both sons */ + if (smaller(tree, v, s->heap[j], s->depth)) break; + + /* Exchange v with the smallest son */ + s->heap[k] = s->heap[j]; k = j; + + /* And continue down the tree, setting j to the left son of k */ + j <<= 1; + } + s->heap[k] = v; +} + +/* =========================================================================== + * Compute the optimal bit lengths for a tree and update the total bit length + * for the current block. + * IN assertion: the fields freq and dad are set, heap[heap_max] and + * above are the tree nodes sorted by increasing frequency. + * OUT assertions: the field len is set to the optimal bit length, the + * array bl_count contains the frequencies for each bit length. + * The length opt_len is updated; static_len is also updated if stree is + * not null. + */ +local void gen_bitlen(s, desc) + deflate_state *s; + tree_desc *desc; /* the tree descriptor */ +{ + ct_data *tree = desc->dyn_tree; + int max_code = desc->max_code; + const ct_data *stree = desc->stat_desc->static_tree; + const intf *extra = desc->stat_desc->extra_bits; + int base = desc->stat_desc->extra_base; + int max_length = desc->stat_desc->max_length; + int h; /* heap index */ + int n, m; /* iterate over the tree elements */ + int bits; /* bit length */ + int xbits; /* extra bits */ + ush f; /* frequency */ + int overflow = 0; /* number of elements with bit length too large */ + + for (bits = 0; bits <= MAX_BITS; bits++) s->bl_count[bits] = 0; + + /* In a first pass, compute the optimal bit lengths (which may + * overflow in the case of the bit length tree). + */ + tree[s->heap[s->heap_max]].Len = 0; /* root of the heap */ + + for (h = s->heap_max+1; h < HEAP_SIZE; h++) { + n = s->heap[h]; + bits = tree[tree[n].Dad].Len + 1; + if (bits > max_length) bits = max_length, overflow++; + tree[n].Len = (ush)bits; + /* We overwrite tree[n].Dad which is no longer needed */ + + if (n > max_code) continue; /* not a leaf node */ + + s->bl_count[bits]++; + xbits = 0; + if (n >= base) xbits = extra[n-base]; + f = tree[n].Freq; + s->opt_len += (ulg)f * (unsigned)(bits + xbits); + if (stree) s->static_len += (ulg)f * (unsigned)(stree[n].Len + xbits); + } + if (overflow == 0) return; + + Tracev((stderr,"\nbit length overflow\n")); + /* This happens for example on obj2 and pic of the Calgary corpus */ + + /* Find the first bit length which could increase: */ + do { + bits = max_length-1; + while (s->bl_count[bits] == 0) bits--; + s->bl_count[bits]--; /* move one leaf down the tree */ + s->bl_count[bits+1] += 2; /* move one overflow item as its brother */ + s->bl_count[max_length]--; + /* The brother of the overflow item also moves one step up, + * but this does not affect bl_count[max_length] + */ + overflow -= 2; + } while (overflow > 0); + + /* Now recompute all bit lengths, scanning in increasing frequency. + * h is still equal to HEAP_SIZE. (It is simpler to reconstruct all + * lengths instead of fixing only the wrong ones. This idea is taken + * from 'ar' written by Haruhiko Okumura.) + */ + for (bits = max_length; bits != 0; bits--) { + n = s->bl_count[bits]; + while (n != 0) { + m = s->heap[--h]; + if (m > max_code) continue; + if ((unsigned) tree[m].Len != (unsigned) bits) { + Tracev((stderr,"code %d bits %d->%d\n", m, tree[m].Len, bits)); + s->opt_len += ((ulg)bits - tree[m].Len) * tree[m].Freq; + tree[m].Len = (ush)bits; + } + n--; + } + } +} + +/* =========================================================================== + * Generate the codes for a given tree and bit counts (which need not be + * optimal). + * IN assertion: the array bl_count contains the bit length statistics for + * the given tree and the field len is set for all tree elements. + * OUT assertion: the field code is set for all tree elements of non + * zero code length. + */ +local void gen_codes (tree, max_code, bl_count) + ct_data *tree; /* the tree to decorate */ + int max_code; /* largest code with non zero frequency */ + ushf *bl_count; /* number of codes at each bit length */ +{ + ush next_code[MAX_BITS+1]; /* next code value for each bit length */ + unsigned code = 0; /* running code value */ + int bits; /* bit index */ + int n; /* code index */ + + /* The distribution counts are first used to generate the code values + * without bit reversal. + */ + for (bits = 1; bits <= MAX_BITS; bits++) { + code = (code + bl_count[bits-1]) << 1; + next_code[bits] = (ush)code; + } + /* Check that the bit counts in bl_count are consistent. The last code + * must be all ones. + */ + Assert (code + bl_count[MAX_BITS]-1 == (1<<MAX_BITS)-1, + "inconsistent bit counts"); + Tracev((stderr,"\ngen_codes: max_code %d ", max_code)); + + for (n = 0; n <= max_code; n++) { + int len = tree[n].Len; + if (len == 0) continue; + /* Now reverse the bits */ + tree[n].Code = (ush)bi_reverse(next_code[len]++, len); + + Tracecv(tree != static_ltree, (stderr,"\nn %3d %c l %2d c %4x (%x) ", + n, (isgraph(n) ? n : ' '), len, tree[n].Code, next_code[len]-1)); + } +} + +/* =========================================================================== + * Construct one Huffman tree and assigns the code bit strings and lengths. + * Update the total bit length for the current block. + * IN assertion: the field freq is set for all tree elements. + * OUT assertions: the fields len and code are set to the optimal bit length + * and corresponding code. The length opt_len is updated; static_len is + * also updated if stree is not null. The field max_code is set. + */ +local void build_tree(s, desc) + deflate_state *s; + tree_desc *desc; /* the tree descriptor */ +{ + ct_data *tree = desc->dyn_tree; + const ct_data *stree = desc->stat_desc->static_tree; + int elems = desc->stat_desc->elems; + int n, m; /* iterate over heap elements */ + int max_code = -1; /* largest code with non zero frequency */ + int node; /* new node being created */ + + /* Construct the initial heap, with least frequent element in + * heap[SMALLEST]. The sons of heap[n] are heap[2*n] and heap[2*n+1]. + * heap[0] is not used. + */ + s->heap_len = 0, s->heap_max = HEAP_SIZE; + + for (n = 0; n < elems; n++) { + if (tree[n].Freq != 0) { + s->heap[++(s->heap_len)] = max_code = n; + s->depth[n] = 0; + } else { + tree[n].Len = 0; + } + } + + /* The pkzip format requires that at least one distance code exists, + * and that at least one bit should be sent even if there is only one + * possible code. So to avoid special checks later on we force at least + * two codes of non zero frequency. + */ + while (s->heap_len < 2) { + node = s->heap[++(s->heap_len)] = (max_code < 2 ? ++max_code : 0); + tree[node].Freq = 1; + s->depth[node] = 0; + s->opt_len--; if (stree) s->static_len -= stree[node].Len; + /* node is 0 or 1 so it does not have extra bits */ + } + desc->max_code = max_code; + + /* The elements heap[heap_len/2+1 .. heap_len] are leaves of the tree, + * establish sub-heaps of increasing lengths: + */ + for (n = s->heap_len/2; n >= 1; n--) pqdownheap(s, tree, n); + + /* Construct the Huffman tree by repeatedly combining the least two + * frequent nodes. + */ + node = elems; /* next internal node of the tree */ + do { + pqremove(s, tree, n); /* n = node of least frequency */ + m = s->heap[SMALLEST]; /* m = node of next least frequency */ + + s->heap[--(s->heap_max)] = n; /* keep the nodes sorted by frequency */ + s->heap[--(s->heap_max)] = m; + + /* Create a new node father of n and m */ + tree[node].Freq = tree[n].Freq + tree[m].Freq; + s->depth[node] = (uch)((s->depth[n] >= s->depth[m] ? + s->depth[n] : s->depth[m]) + 1); + tree[n].Dad = tree[m].Dad = (ush)node; +#ifdef DUMP_BL_TREE + if (tree == s->bl_tree) { + fprintf(stderr,"\nnode %d(%d), sons %d(%d) %d(%d)", + node, tree[node].Freq, n, tree[n].Freq, m, tree[m].Freq); + } +#endif + /* and insert the new node in the heap */ + s->heap[SMALLEST] = node++; + pqdownheap(s, tree, SMALLEST); + + } while (s->heap_len >= 2); + + s->heap[--(s->heap_max)] = s->heap[SMALLEST]; + + /* At this point, the fields freq and dad are set. We can now + * generate the bit lengths. + */ + gen_bitlen(s, (tree_desc *)desc); + + /* The field len is now set, we can generate the bit codes */ + gen_codes ((ct_data *)tree, max_code, s->bl_count); +} + +/* =========================================================================== + * Scan a literal or distance tree to determine the frequencies of the codes + * in the bit length tree. + */ +local void scan_tree (s, tree, max_code) + deflate_state *s; + ct_data *tree; /* the tree to be scanned */ + int max_code; /* and its largest code of non zero frequency */ +{ + int n; /* iterates over all tree elements */ + int prevlen = -1; /* last emitted length */ + int curlen; /* length of current code */ + int nextlen = tree[0].Len; /* length of next code */ + int count = 0; /* repeat count of the current code */ + int max_count = 7; /* max repeat count */ + int min_count = 4; /* min repeat count */ + + if (nextlen == 0) max_count = 138, min_count = 3; + tree[max_code+1].Len = (ush)0xffff; /* guard */ + + for (n = 0; n <= max_code; n++) { + curlen = nextlen; nextlen = tree[n+1].Len; + if (++count < max_count && curlen == nextlen) { + continue; + } else if (count < min_count) { + s->bl_tree[curlen].Freq += count; + } else if (curlen != 0) { + if (curlen != prevlen) s->bl_tree[curlen].Freq++; + s->bl_tree[REP_3_6].Freq++; + } else if (count <= 10) { + s->bl_tree[REPZ_3_10].Freq++; + } else { + s->bl_tree[REPZ_11_138].Freq++; + } + count = 0; prevlen = curlen; + if (nextlen == 0) { + max_count = 138, min_count = 3; + } else if (curlen == nextlen) { + max_count = 6, min_count = 3; + } else { + max_count = 7, min_count = 4; + } + } +} + +/* =========================================================================== + * Send a literal or distance tree in compressed form, using the codes in + * bl_tree. + */ +local void send_tree (s, tree, max_code) + deflate_state *s; + ct_data *tree; /* the tree to be scanned */ + int max_code; /* and its largest code of non zero frequency */ +{ + int n; /* iterates over all tree elements */ + int prevlen = -1; /* last emitted length */ + int curlen; /* length of current code */ + int nextlen = tree[0].Len; /* length of next code */ + int count = 0; /* repeat count of the current code */ + int max_count = 7; /* max repeat count */ + int min_count = 4; /* min repeat count */ + + /* tree[max_code+1].Len = -1; */ /* guard already set */ + if (nextlen == 0) max_count = 138, min_count = 3; + + for (n = 0; n <= max_code; n++) { + curlen = nextlen; nextlen = tree[n+1].Len; + if (++count < max_count && curlen == nextlen) { + continue; + } else if (count < min_count) { + do { send_code(s, curlen, s->bl_tree); } while (--count != 0); + + } else if (curlen != 0) { + if (curlen != prevlen) { + send_code(s, curlen, s->bl_tree); count--; + } + Assert(count >= 3 && count <= 6, " 3_6?"); + send_code(s, REP_3_6, s->bl_tree); send_bits(s, count-3, 2); + + } else if (count <= 10) { + send_code(s, REPZ_3_10, s->bl_tree); send_bits(s, count-3, 3); + + } else { + send_code(s, REPZ_11_138, s->bl_tree); send_bits(s, count-11, 7); + } + count = 0; prevlen = curlen; + if (nextlen == 0) { + max_count = 138, min_count = 3; + } else if (curlen == nextlen) { + max_count = 6, min_count = 3; + } else { + max_count = 7, min_count = 4; + } + } +} + +/* =========================================================================== + * Construct the Huffman tree for the bit lengths and return the index in + * bl_order of the last bit length code to send. + */ +local int build_bl_tree(s) + deflate_state *s; +{ + int max_blindex; /* index of last bit length code of non zero freq */ + + /* Determine the bit length frequencies for literal and distance trees */ + scan_tree(s, (ct_data *)s->dyn_ltree, s->l_desc.max_code); + scan_tree(s, (ct_data *)s->dyn_dtree, s->d_desc.max_code); + + /* Build the bit length tree: */ + build_tree(s, (tree_desc *)(&(s->bl_desc))); + /* opt_len now includes the length of the tree representations, except + * the lengths of the bit lengths codes and the 5+5+4 bits for the counts. + */ + + /* Determine the number of bit length codes to send. The pkzip format + * requires that at least 4 bit length codes be sent. (appnote.txt says + * 3 but the actual value used is 4.) + */ + for (max_blindex = BL_CODES-1; max_blindex >= 3; max_blindex--) { + if (s->bl_tree[bl_order[max_blindex]].Len != 0) break; + } + /* Update opt_len to include the bit length tree and counts */ + s->opt_len += 3*((ulg)max_blindex+1) + 5+5+4; + Tracev((stderr, "\ndyn trees: dyn %ld, stat %ld", + s->opt_len, s->static_len)); + + return max_blindex; +} + +/* =========================================================================== + * Send the header for a block using dynamic Huffman trees: the counts, the + * lengths of the bit length codes, the literal tree and the distance tree. + * IN assertion: lcodes >= 257, dcodes >= 1, blcodes >= 4. + */ +local void send_all_trees(s, lcodes, dcodes, blcodes) + deflate_state *s; + int lcodes, dcodes, blcodes; /* number of codes for each tree */ +{ + int rank; /* index in bl_order */ + + Assert (lcodes >= 257 && dcodes >= 1 && blcodes >= 4, "not enough codes"); + Assert (lcodes <= L_CODES && dcodes <= D_CODES && blcodes <= BL_CODES, + "too many codes"); + Tracev((stderr, "\nbl counts: ")); + send_bits(s, lcodes-257, 5); /* not +255 as stated in appnote.txt */ + send_bits(s, dcodes-1, 5); + send_bits(s, blcodes-4, 4); /* not -3 as stated in appnote.txt */ + for (rank = 0; rank < blcodes; rank++) { + Tracev((stderr, "\nbl code %2d ", bl_order[rank])); + send_bits(s, s->bl_tree[bl_order[rank]].Len, 3); + } + Tracev((stderr, "\nbl tree: sent %ld", s->bits_sent)); + + send_tree(s, (ct_data *)s->dyn_ltree, lcodes-1); /* literal tree */ + Tracev((stderr, "\nlit tree: sent %ld", s->bits_sent)); + + send_tree(s, (ct_data *)s->dyn_dtree, dcodes-1); /* distance tree */ + Tracev((stderr, "\ndist tree: sent %ld", s->bits_sent)); +} + +/* =========================================================================== + * Send a stored block + */ +void ZLIB_INTERNAL _tr_stored_block(s, buf, stored_len, last) + deflate_state *s; + charf *buf; /* input block */ + ulg stored_len; /* length of input block */ + int last; /* one if this is the last block for a file */ +{ + send_bits(s, (STORED_BLOCK<<1)+last, 3); /* send block type */ + bi_windup(s); /* align on byte boundary */ + put_short(s, (ush)stored_len); + put_short(s, (ush)~stored_len); + zmemcpy(s->pending_buf + s->pending, (Bytef *)buf, stored_len); + s->pending += stored_len; +#ifdef ZLIB_DEBUG + s->compressed_len = (s->compressed_len + 3 + 7) & (ulg)~7L; + s->compressed_len += (stored_len + 4) << 3; + s->bits_sent += 2*16; + s->bits_sent += stored_len<<3; +#endif +} + +/* =========================================================================== + * Flush the bits in the bit buffer to pending output (leaves at most 7 bits) + */ +void ZLIB_INTERNAL _tr_flush_bits(s) + deflate_state *s; +{ + bi_flush(s); +} + +/* =========================================================================== + * Send one empty static block to give enough lookahead for inflate. + * This takes 10 bits, of which 7 may remain in the bit buffer. + */ +void ZLIB_INTERNAL _tr_align(s) + deflate_state *s; +{ + send_bits(s, STATIC_TREES<<1, 3); + send_code(s, END_BLOCK, static_ltree); +#ifdef ZLIB_DEBUG + s->compressed_len += 10L; /* 3 for block type, 7 for EOB */ +#endif + bi_flush(s); +} + +/* =========================================================================== + * Determine the best encoding for the current block: dynamic trees, static + * trees or store, and write out the encoded block. + */ +void ZLIB_INTERNAL _tr_flush_block(s, buf, stored_len, last) + deflate_state *s; + charf *buf; /* input block, or NULL if too old */ + ulg stored_len; /* length of input block */ + int last; /* one if this is the last block for a file */ +{ + ulg opt_lenb, static_lenb; /* opt_len and static_len in bytes */ + int max_blindex = 0; /* index of last bit length code of non zero freq */ + + /* Build the Huffman trees unless a stored block is forced */ + if (s->level > 0) { + + /* Check if the file is binary or text */ + if (s->strm->data_type == Z_UNKNOWN) + s->strm->data_type = detect_data_type(s); + + /* Construct the literal and distance trees */ + build_tree(s, (tree_desc *)(&(s->l_desc))); + Tracev((stderr, "\nlit data: dyn %ld, stat %ld", s->opt_len, + s->static_len)); + + build_tree(s, (tree_desc *)(&(s->d_desc))); + Tracev((stderr, "\ndist data: dyn %ld, stat %ld", s->opt_len, + s->static_len)); + /* At this point, opt_len and static_len are the total bit lengths of + * the compressed block data, excluding the tree representations. + */ + + /* Build the bit length tree for the above two trees, and get the index + * in bl_order of the last bit length code to send. + */ + max_blindex = build_bl_tree(s); + + /* Determine the best encoding. Compute the block lengths in bytes. */ + opt_lenb = (s->opt_len+3+7)>>3; + static_lenb = (s->static_len+3+7)>>3; + + Tracev((stderr, "\nopt %lu(%lu) stat %lu(%lu) stored %lu lit %u ", + opt_lenb, s->opt_len, static_lenb, s->static_len, stored_len, + s->sym_next / 3)); + + if (static_lenb <= opt_lenb) opt_lenb = static_lenb; + + } else { + Assert(buf != (char*)0, "lost buf"); + opt_lenb = static_lenb = stored_len + 5; /* force a stored block */ + } + +#ifdef FORCE_STORED + if (buf != (char*)0) { /* force stored block */ +#else + if (stored_len+4 <= opt_lenb && buf != (char*)0) { + /* 4: two words for the lengths */ +#endif + /* The test buf != NULL is only necessary if LIT_BUFSIZE > WSIZE. + * Otherwise we can't have processed more than WSIZE input bytes since + * the last block flush, because compression would have been + * successful. If LIT_BUFSIZE <= WSIZE, it is never too late to + * transform a block into a stored block. + */ + _tr_stored_block(s, buf, stored_len, last); + +#ifdef FORCE_STATIC + } else if (static_lenb >= 0) { /* force static trees */ +#else + } else if (s->strategy == Z_FIXED || static_lenb == opt_lenb) { +#endif + send_bits(s, (STATIC_TREES<<1)+last, 3); + compress_block(s, (const ct_data *)static_ltree, + (const ct_data *)static_dtree); +#ifdef ZLIB_DEBUG + s->compressed_len += 3 + s->static_len; +#endif + } else { + send_bits(s, (DYN_TREES<<1)+last, 3); + send_all_trees(s, s->l_desc.max_code+1, s->d_desc.max_code+1, + max_blindex+1); + compress_block(s, (const ct_data *)s->dyn_ltree, + (const ct_data *)s->dyn_dtree); +#ifdef ZLIB_DEBUG + s->compressed_len += 3 + s->opt_len; +#endif + } + Assert (s->compressed_len == s->bits_sent, "bad compressed size"); + /* The above check is made mod 2^32, for files larger than 512 MB + * and uLong implemented on 32 bits. + */ + init_block(s); + + if (last) { + bi_windup(s); +#ifdef ZLIB_DEBUG + s->compressed_len += 7; /* align on byte boundary */ +#endif + } + Tracev((stderr,"\ncomprlen %lu(%lu) ", s->compressed_len>>3, + s->compressed_len-7*last)); +} + +/* =========================================================================== + * Save the match info and tally the frequency counts. Return true if + * the current block must be flushed. + */ +int ZLIB_INTERNAL _tr_tally (s, dist, lc) + deflate_state *s; + unsigned dist; /* distance of matched string */ + unsigned lc; /* match length-MIN_MATCH or unmatched char (if dist==0) */ +{ + s->sym_buf[s->sym_next++] = dist; + s->sym_buf[s->sym_next++] = dist >> 8; + s->sym_buf[s->sym_next++] = lc; + if (dist == 0) { + /* lc is the unmatched char */ + s->dyn_ltree[lc].Freq++; + } else { + s->matches++; + /* Here, lc is the match length - MIN_MATCH */ + dist--; /* dist = match distance - 1 */ + Assert((ush)dist < (ush)MAX_DIST(s) && + (ush)lc <= (ush)(MAX_MATCH-MIN_MATCH) && + (ush)d_code(dist) < (ush)D_CODES, "_tr_tally: bad match"); + + s->dyn_ltree[_length_code[lc]+LITERALS+1].Freq++; + s->dyn_dtree[d_code(dist)].Freq++; + } + return (s->sym_next == s->sym_end); +} + +/* =========================================================================== + * Send the block data compressed using the given Huffman trees + */ +local void compress_block(s, ltree, dtree) + deflate_state *s; + const ct_data *ltree; /* literal tree */ + const ct_data *dtree; /* distance tree */ +{ + unsigned dist; /* distance of matched string */ + int lc; /* match length or unmatched char (if dist == 0) */ + unsigned sx = 0; /* running index in sym_buf */ + unsigned code; /* the code to send */ + int extra; /* number of extra bits to send */ + + if (s->sym_next != 0) do { + dist = s->sym_buf[sx++] & 0xff; + dist += (unsigned)(s->sym_buf[sx++] & 0xff) << 8; + lc = s->sym_buf[sx++]; + if (dist == 0) { + send_code(s, lc, ltree); /* send a literal byte */ + Tracecv(isgraph(lc), (stderr," '%c' ", lc)); + } else { + /* Here, lc is the match length - MIN_MATCH */ + code = _length_code[lc]; + send_code(s, code+LITERALS+1, ltree); /* send the length code */ + extra = extra_lbits[code]; + if (extra != 0) { + lc -= base_length[code]; + send_bits(s, lc, extra); /* send the extra length bits */ + } + dist--; /* dist is now the match distance - 1 */ + code = d_code(dist); + Assert (code < D_CODES, "bad d_code"); + + send_code(s, code, dtree); /* send the distance code */ + extra = extra_dbits[code]; + if (extra != 0) { + dist -= (unsigned)base_dist[code]; + send_bits(s, dist, extra); /* send the extra distance bits */ + } + } /* literal or match pair ? */ + + /* Check that the overlay between pending_buf and sym_buf is ok: */ + Assert(s->pending < s->lit_bufsize + sx, "pendingBuf overflow"); + + } while (sx < s->sym_next); + + send_code(s, END_BLOCK, ltree); +} + +/* =========================================================================== + * Check if the data type is TEXT or BINARY, using the following algorithm: + * - TEXT if the two conditions below are satisfied: + * a) There are no non-portable control characters belonging to the + * "black list" (0..6, 14..25, 28..31). + * b) There is at least one printable character belonging to the + * "white list" (9 {TAB}, 10 {LF}, 13 {CR}, 32..255). + * - BINARY otherwise. + * - The following partially-portable control characters form a + * "gray list" that is ignored in this detection algorithm: + * (7 {BEL}, 8 {BS}, 11 {VT}, 12 {FF}, 26 {SUB}, 27 {ESC}). + * IN assertion: the fields Freq of dyn_ltree are set. + */ +local int detect_data_type(s) + deflate_state *s; +{ + /* black_mask is the bit mask of black-listed bytes + * set bits 0..6, 14..25, and 28..31 + * 0xf3ffc07f = binary 11110011111111111100000001111111 + */ + unsigned long black_mask = 0xf3ffc07fUL; + int n; + + /* Check for non-textual ("black-listed") bytes. */ + for (n = 0; n <= 31; n++, black_mask >>= 1) + if ((black_mask & 1) && (s->dyn_ltree[n].Freq != 0)) + return Z_BINARY; + + /* Check for textual ("white-listed") bytes. */ + if (s->dyn_ltree[9].Freq != 0 || s->dyn_ltree[10].Freq != 0 + || s->dyn_ltree[13].Freq != 0) + return Z_TEXT; + for (n = 32; n < LITERALS; n++) + if (s->dyn_ltree[n].Freq != 0) + return Z_TEXT; + + /* There are no "black-listed" or "white-listed" bytes: + * this stream either is empty or has tolerated ("gray-listed") bytes only. + */ + return Z_BINARY; +} + +/* =========================================================================== + * Reverse the first len bits of a code, using straightforward code (a faster + * method would use a table) + * IN assertion: 1 <= len <= 15 + */ +local unsigned bi_reverse(code, len) + unsigned code; /* the value to invert */ + int len; /* its bit length */ +{ + register unsigned res = 0; + do { + res |= code & 1; + code >>= 1, res <<= 1; + } while (--len > 0); + return res >> 1; +} + +/* =========================================================================== + * Flush the bit buffer, keeping at most 7 bits in it. + */ +local void bi_flush(s) + deflate_state *s; +{ + if (s->bi_valid == 16) { + put_short(s, s->bi_buf); + s->bi_buf = 0; + s->bi_valid = 0; + } else if (s->bi_valid >= 8) { + put_byte(s, (Byte)s->bi_buf); + s->bi_buf >>= 8; + s->bi_valid -= 8; + } +} + +/* =========================================================================== + * Flush the bit buffer and align the output on a byte boundary + */ +local void bi_windup(s) + deflate_state *s; +{ + if (s->bi_valid > 8) { + put_short(s, s->bi_buf); + } else if (s->bi_valid > 0) { + put_byte(s, (Byte)s->bi_buf); + } + s->bi_buf = 0; + s->bi_valid = 0; +#ifdef ZLIB_DEBUG + s->bits_sent = (s->bits_sent+7) & ~7; +#endif +} diff --git a/third_party/zlib/trees.h b/third_party/zlib/trees.h new file mode 100644 index 00000000..d35639d8 --- /dev/null +++ b/third_party/zlib/trees.h @@ -0,0 +1,128 @@ +/* header created automatically with -DGEN_TREES_H */ + +local const ct_data static_ltree[L_CODES+2] = { +{{ 12},{ 8}}, {{140},{ 8}}, {{ 76},{ 8}}, {{204},{ 8}}, {{ 44},{ 8}}, +{{172},{ 8}}, {{108},{ 8}}, {{236},{ 8}}, {{ 28},{ 8}}, {{156},{ 8}}, +{{ 92},{ 8}}, {{220},{ 8}}, {{ 60},{ 8}}, {{188},{ 8}}, {{124},{ 8}}, +{{252},{ 8}}, {{ 2},{ 8}}, {{130},{ 8}}, {{ 66},{ 8}}, {{194},{ 8}}, +{{ 34},{ 8}}, {{162},{ 8}}, {{ 98},{ 8}}, {{226},{ 8}}, {{ 18},{ 8}}, +{{146},{ 8}}, {{ 82},{ 8}}, {{210},{ 8}}, {{ 50},{ 8}}, {{178},{ 8}}, +{{114},{ 8}}, {{242},{ 8}}, {{ 10},{ 8}}, {{138},{ 8}}, {{ 74},{ 8}}, +{{202},{ 8}}, {{ 42},{ 8}}, {{170},{ 8}}, {{106},{ 8}}, {{234},{ 8}}, +{{ 26},{ 8}}, {{154},{ 8}}, {{ 90},{ 8}}, {{218},{ 8}}, {{ 58},{ 8}}, +{{186},{ 8}}, {{122},{ 8}}, {{250},{ 8}}, {{ 6},{ 8}}, {{134},{ 8}}, +{{ 70},{ 8}}, {{198},{ 8}}, {{ 38},{ 8}}, {{166},{ 8}}, {{102},{ 8}}, +{{230},{ 8}}, {{ 22},{ 8}}, {{150},{ 8}}, {{ 86},{ 8}}, {{214},{ 8}}, +{{ 54},{ 8}}, {{182},{ 8}}, {{118},{ 8}}, {{246},{ 8}}, {{ 14},{ 8}}, +{{142},{ 8}}, {{ 78},{ 8}}, {{206},{ 8}}, {{ 46},{ 8}}, {{174},{ 8}}, +{{110},{ 8}}, {{238},{ 8}}, {{ 30},{ 8}}, {{158},{ 8}}, {{ 94},{ 8}}, +{{222},{ 8}}, {{ 62},{ 8}}, {{190},{ 8}}, {{126},{ 8}}, {{254},{ 8}}, +{{ 1},{ 8}}, {{129},{ 8}}, {{ 65},{ 8}}, {{193},{ 8}}, {{ 33},{ 8}}, +{{161},{ 8}}, {{ 97},{ 8}}, {{225},{ 8}}, {{ 17},{ 8}}, {{145},{ 8}}, +{{ 81},{ 8}}, {{209},{ 8}}, {{ 49},{ 8}}, {{177},{ 8}}, {{113},{ 8}}, +{{241},{ 8}}, {{ 9},{ 8}}, {{137},{ 8}}, {{ 73},{ 8}}, {{201},{ 8}}, +{{ 41},{ 8}}, {{169},{ 8}}, {{105},{ 8}}, {{233},{ 8}}, {{ 25},{ 8}}, +{{153},{ 8}}, {{ 89},{ 8}}, {{217},{ 8}}, {{ 57},{ 8}}, {{185},{ 8}}, +{{121},{ 8}}, {{249},{ 8}}, {{ 5},{ 8}}, {{133},{ 8}}, {{ 69},{ 8}}, +{{197},{ 8}}, {{ 37},{ 8}}, {{165},{ 8}}, {{101},{ 8}}, {{229},{ 8}}, +{{ 21},{ 8}}, {{149},{ 8}}, {{ 85},{ 8}}, {{213},{ 8}}, {{ 53},{ 8}}, +{{181},{ 8}}, {{117},{ 8}}, {{245},{ 8}}, {{ 13},{ 8}}, {{141},{ 8}}, +{{ 77},{ 8}}, {{205},{ 8}}, {{ 45},{ 8}}, {{173},{ 8}}, {{109},{ 8}}, +{{237},{ 8}}, {{ 29},{ 8}}, {{157},{ 8}}, {{ 93},{ 8}}, {{221},{ 8}}, +{{ 61},{ 8}}, {{189},{ 8}}, {{125},{ 8}}, {{253},{ 8}}, {{ 19},{ 9}}, +{{275},{ 9}}, {{147},{ 9}}, {{403},{ 9}}, {{ 83},{ 9}}, {{339},{ 9}}, +{{211},{ 9}}, {{467},{ 9}}, {{ 51},{ 9}}, {{307},{ 9}}, {{179},{ 9}}, +{{435},{ 9}}, {{115},{ 9}}, {{371},{ 9}}, {{243},{ 9}}, {{499},{ 9}}, +{{ 11},{ 9}}, {{267},{ 9}}, {{139},{ 9}}, {{395},{ 9}}, {{ 75},{ 9}}, +{{331},{ 9}}, {{203},{ 9}}, {{459},{ 9}}, {{ 43},{ 9}}, {{299},{ 9}}, +{{171},{ 9}}, {{427},{ 9}}, {{107},{ 9}}, {{363},{ 9}}, {{235},{ 9}}, +{{491},{ 9}}, {{ 27},{ 9}}, {{283},{ 9}}, {{155},{ 9}}, {{411},{ 9}}, +{{ 91},{ 9}}, {{347},{ 9}}, {{219},{ 9}}, {{475},{ 9}}, {{ 59},{ 9}}, +{{315},{ 9}}, {{187},{ 9}}, {{443},{ 9}}, {{123},{ 9}}, {{379},{ 9}}, +{{251},{ 9}}, {{507},{ 9}}, {{ 7},{ 9}}, {{263},{ 9}}, {{135},{ 9}}, +{{391},{ 9}}, {{ 71},{ 9}}, {{327},{ 9}}, {{199},{ 9}}, {{455},{ 9}}, +{{ 39},{ 9}}, {{295},{ 9}}, {{167},{ 9}}, {{423},{ 9}}, {{103},{ 9}}, +{{359},{ 9}}, {{231},{ 9}}, {{487},{ 9}}, {{ 23},{ 9}}, {{279},{ 9}}, +{{151},{ 9}}, {{407},{ 9}}, {{ 87},{ 9}}, {{343},{ 9}}, {{215},{ 9}}, +{{471},{ 9}}, {{ 55},{ 9}}, {{311},{ 9}}, {{183},{ 9}}, {{439},{ 9}}, +{{119},{ 9}}, {{375},{ 9}}, {{247},{ 9}}, {{503},{ 9}}, {{ 15},{ 9}}, +{{271},{ 9}}, {{143},{ 9}}, {{399},{ 9}}, {{ 79},{ 9}}, {{335},{ 9}}, +{{207},{ 9}}, {{463},{ 9}}, {{ 47},{ 9}}, {{303},{ 9}}, {{175},{ 9}}, +{{431},{ 9}}, {{111},{ 9}}, {{367},{ 9}}, {{239},{ 9}}, {{495},{ 9}}, +{{ 31},{ 9}}, {{287},{ 9}}, {{159},{ 9}}, {{415},{ 9}}, {{ 95},{ 9}}, +{{351},{ 9}}, {{223},{ 9}}, {{479},{ 9}}, {{ 63},{ 9}}, {{319},{ 9}}, +{{191},{ 9}}, {{447},{ 9}}, {{127},{ 9}}, {{383},{ 9}}, {{255},{ 9}}, +{{511},{ 9}}, {{ 0},{ 7}}, {{ 64},{ 7}}, {{ 32},{ 7}}, {{ 96},{ 7}}, +{{ 16},{ 7}}, {{ 80},{ 7}}, {{ 48},{ 7}}, {{112},{ 7}}, {{ 8},{ 7}}, +{{ 72},{ 7}}, {{ 40},{ 7}}, {{104},{ 7}}, {{ 24},{ 7}}, {{ 88},{ 7}}, +{{ 56},{ 7}}, {{120},{ 7}}, {{ 4},{ 7}}, {{ 68},{ 7}}, {{ 36},{ 7}}, +{{100},{ 7}}, {{ 20},{ 7}}, {{ 84},{ 7}}, {{ 52},{ 7}}, {{116},{ 7}}, +{{ 3},{ 8}}, {{131},{ 8}}, {{ 67},{ 8}}, {{195},{ 8}}, {{ 35},{ 8}}, +{{163},{ 8}}, {{ 99},{ 8}}, {{227},{ 8}} +}; + +local const ct_data static_dtree[D_CODES] = { +{{ 0},{ 5}}, {{16},{ 5}}, {{ 8},{ 5}}, {{24},{ 5}}, {{ 4},{ 5}}, +{{20},{ 5}}, {{12},{ 5}}, {{28},{ 5}}, {{ 2},{ 5}}, {{18},{ 5}}, +{{10},{ 5}}, {{26},{ 5}}, {{ 6},{ 5}}, {{22},{ 5}}, {{14},{ 5}}, +{{30},{ 5}}, {{ 1},{ 5}}, {{17},{ 5}}, {{ 9},{ 5}}, {{25},{ 5}}, +{{ 5},{ 5}}, {{21},{ 5}}, {{13},{ 5}}, {{29},{ 5}}, {{ 3},{ 5}}, +{{19},{ 5}}, {{11},{ 5}}, {{27},{ 5}}, {{ 7},{ 5}}, {{23},{ 5}} +}; + +const uch ZLIB_INTERNAL _dist_code[DIST_CODE_LEN] = { + 0, 1, 2, 3, 4, 4, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8, + 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, +10, 10, 10, 10, 10, 10, 10, 10, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, +11, 11, 11, 11, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, +12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 13, 13, 13, 13, +13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, +13, 13, 13, 13, 13, 13, 13, 13, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, +14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, +14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, +14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 15, 15, 15, 15, 15, 15, 15, 15, +15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, +15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, +15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 0, 0, 16, 17, +18, 18, 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, 22, 22, 22, 22, 22, 22, 22, 22, +23, 23, 23, 23, 23, 23, 23, 23, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, +24, 24, 24, 24, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, +26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, +26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 27, 27, 27, 27, 27, 27, 27, 27, +27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, +27, 27, 27, 27, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, +28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, +28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, +28, 28, 28, 28, 28, 28, 28, 28, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, +29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, +29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, +29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29, 29 +}; + +const uch ZLIB_INTERNAL _length_code[MAX_MATCH-MIN_MATCH+1]= { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 12, 12, +13, 13, 13, 13, 14, 14, 14, 14, 15, 15, 15, 15, 16, 16, 16, 16, 16, 16, 16, 16, +17, 17, 17, 17, 17, 17, 17, 17, 18, 18, 18, 18, 18, 18, 18, 18, 19, 19, 19, 19, +19, 19, 19, 19, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, +21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 21, 22, 22, 22, 22, +22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 23, 23, 23, 23, 23, 23, 23, 23, +23, 23, 23, 23, 23, 23, 23, 23, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, +24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, +25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, +25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 26, 26, 26, 26, 26, 26, 26, 26, +26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, 26, +26, 26, 26, 26, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, +27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 28 +}; + +local const int base_length[LENGTH_CODES] = { +0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16, 20, 24, 28, 32, 40, 48, 56, +64, 80, 96, 112, 128, 160, 192, 224, 0 +}; + +local const int base_dist[D_CODES] = { + 0, 1, 2, 3, 4, 6, 8, 12, 16, 24, + 32, 48, 64, 96, 128, 192, 256, 384, 512, 768, + 1024, 1536, 2048, 3072, 4096, 6144, 8192, 12288, 16384, 24576 +}; + diff --git a/third_party/zlib/uncompr.c b/third_party/zlib/uncompr.c new file mode 100644 index 00000000..f03a1a86 --- /dev/null +++ b/third_party/zlib/uncompr.c @@ -0,0 +1,93 @@ +/* uncompr.c -- decompress a memory buffer + * Copyright (C) 1995-2003, 2010, 2014, 2016 Jean-loup Gailly, Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +/* @(#) $Id$ */ + +#define ZLIB_INTERNAL +#include "zlib.h" + +/* =========================================================================== + Decompresses the source buffer into the destination buffer. *sourceLen is + the byte length of the source buffer. Upon entry, *destLen is the total size + of the destination buffer, which must be large enough to hold the entire + uncompressed data. (The size of the uncompressed data must have been saved + previously by the compressor and transmitted to the decompressor by some + mechanism outside the scope of this compression library.) Upon exit, + *destLen is the size of the decompressed data and *sourceLen is the number + of source bytes consumed. Upon return, source + *sourceLen points to the + first unused input byte. + + uncompress returns Z_OK if success, Z_MEM_ERROR if there was not enough + memory, Z_BUF_ERROR if there was not enough room in the output buffer, or + Z_DATA_ERROR if the input data was corrupted, including if the input data is + an incomplete zlib stream. +*/ +int ZEXPORT uncompress2 (dest, destLen, source, sourceLen) + Bytef *dest; + uLongf *destLen; + const Bytef *source; + uLong *sourceLen; +{ + z_stream stream; + int err; + const uInt max = (uInt)-1; + uLong len, left; + Byte buf[1]; /* for detection of incomplete stream when *destLen == 0 */ + + len = *sourceLen; + if (*destLen) { + left = *destLen; + *destLen = 0; + } + else { + left = 1; + dest = buf; + } + + stream.next_in = (z_const Bytef *)source; + stream.avail_in = 0; + stream.zalloc = (alloc_func)0; + stream.zfree = (free_func)0; + stream.opaque = (voidpf)0; + + err = inflateInit(&stream); + if (err != Z_OK) return err; + + stream.next_out = dest; + stream.avail_out = 0; + + do { + if (stream.avail_out == 0) { + stream.avail_out = left > (uLong)max ? max : (uInt)left; + left -= stream.avail_out; + } + if (stream.avail_in == 0) { + stream.avail_in = len > (uLong)max ? max : (uInt)len; + len -= stream.avail_in; + } + err = inflate(&stream, Z_NO_FLUSH); + } while (err == Z_OK); + + *sourceLen -= len + stream.avail_in; + if (dest != buf) + *destLen = stream.total_out; + else if (stream.total_out && err == Z_BUF_ERROR) + left = 1; + + inflateEnd(&stream); + return err == Z_STREAM_END ? Z_OK : + err == Z_NEED_DICT ? Z_DATA_ERROR : + err == Z_BUF_ERROR && left + stream.avail_out ? Z_DATA_ERROR : + err; +} + +int ZEXPORT uncompress (dest, destLen, source, sourceLen) + Bytef *dest; + uLongf *destLen; + const Bytef *source; + uLong sourceLen; +{ + return uncompress2(dest, destLen, source, &sourceLen); +} diff --git a/third_party/zlib/zconf.h b/third_party/zlib/zconf.h new file mode 100644 index 00000000..7b3f8da4 --- /dev/null +++ b/third_party/zlib/zconf.h @@ -0,0 +1,553 @@ +/* zconf.h -- configuration of the zlib compression library + * Copyright (C) 1995-2016 Jean-loup Gailly, Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +/* @(#) $Id$ */ + +#ifndef ZCONF_H +#define ZCONF_H + +// ANDROID CHNAGE: Upstream chromium renames all the functions; we don't want +// that. +#define CHROMIUM_ZLIB_NO_CHROMECONF + +/* + * This library is also built as a part of AOSP, which does not need to include + * chromeconf.h. This config does not want chromeconf.h, so it can set this + * macro to opt out. While this works today, there's no guarantee that building + * zlib outside of Chromium keeps working in the future. + */ +#if !defined(CHROMIUM_ZLIB_NO_CHROMECONF) +/* This include does prefixing as below, but with an updated set of names. Also + * sets up export macros in component builds. */ +#include "chromeconf.h" +#endif + +/* + * If you *really* need a unique prefix for all types and library functions, + * compile with -DZ_PREFIX. The "standard" zlib should be compiled without it. + * Even better than compiling with -DZ_PREFIX would be to use configure to set + * this permanently in zconf.h using "./configure --zprefix". + */ +#ifdef Z_PREFIX /* may be set to #if 1 by ./configure */ +# define Z_PREFIX_SET + +/* all linked symbols and init macros */ +# define _dist_code z__dist_code +# define _length_code z__length_code +# define _tr_align z__tr_align +# define _tr_flush_bits z__tr_flush_bits +# define _tr_flush_block z__tr_flush_block +# define _tr_init z__tr_init +# define _tr_stored_block z__tr_stored_block +# define _tr_tally z__tr_tally +# define adler32 z_adler32 +# define adler32_combine z_adler32_combine +# define adler32_combine64 z_adler32_combine64 +# define adler32_z z_adler32_z +# ifndef Z_SOLO +# define compress z_compress +# define compress2 z_compress2 +# define compressBound z_compressBound +# endif +# define crc32 z_crc32 +# define crc32_combine z_crc32_combine +# define crc32_combine64 z_crc32_combine64 +# define crc32_z z_crc32_z +# define deflate z_deflate +# define deflateBound z_deflateBound +# define deflateCopy z_deflateCopy +# define deflateEnd z_deflateEnd +# define deflateGetDictionary z_deflateGetDictionary +# define deflateInit z_deflateInit +# define deflateInit2 z_deflateInit2 +# define deflateInit2_ z_deflateInit2_ +# define deflateInit_ z_deflateInit_ +# define deflateParams z_deflateParams +# define deflatePending z_deflatePending +# define deflatePrime z_deflatePrime +# define deflateReset z_deflateReset +# define deflateResetKeep z_deflateResetKeep +# define deflateSetDictionary z_deflateSetDictionary +# define deflateSetHeader z_deflateSetHeader +# define deflateTune z_deflateTune +# define deflate_copyright z_deflate_copyright +# define get_crc_table z_get_crc_table +# ifndef Z_SOLO +# define gz_error z_gz_error +# define gz_intmax z_gz_intmax +# define gz_strwinerror z_gz_strwinerror +# define gzbuffer z_gzbuffer +# define gzclearerr z_gzclearerr +# define gzclose z_gzclose +# define gzclose_r z_gzclose_r +# define gzclose_w z_gzclose_w +# define gzdirect z_gzdirect +# define gzdopen z_gzdopen +# define gzeof z_gzeof +# define gzerror z_gzerror +# define gzflush z_gzflush +# define gzfread z_gzfread +# define gzfwrite z_gzfwrite +# define gzgetc z_gzgetc +# define gzgetc_ z_gzgetc_ +# define gzgets z_gzgets +# define gzoffset z_gzoffset +# define gzoffset64 z_gzoffset64 +# define gzopen z_gzopen +# define gzopen64 z_gzopen64 +# ifdef _WIN32 +# define gzopen_w z_gzopen_w +# endif +# define gzprintf z_gzprintf +# define gzputc z_gzputc +# define gzputs z_gzputs +# define gzread z_gzread +# define gzrewind z_gzrewind +# define gzseek z_gzseek +# define gzseek64 z_gzseek64 +# define gzsetparams z_gzsetparams +# define gztell z_gztell +# define gztell64 z_gztell64 +# define gzungetc z_gzungetc +# define gzvprintf z_gzvprintf +# define gzwrite z_gzwrite +# endif +# define inflate z_inflate +# define inflateBack z_inflateBack +# define inflateBackEnd z_inflateBackEnd +# define inflateBackInit z_inflateBackInit +# define inflateBackInit_ z_inflateBackInit_ +# define inflateCodesUsed z_inflateCodesUsed +# define inflateCopy z_inflateCopy +# define inflateEnd z_inflateEnd +# define inflateGetDictionary z_inflateGetDictionary +# define inflateGetHeader z_inflateGetHeader +# define inflateInit z_inflateInit +# define inflateInit2 z_inflateInit2 +# define inflateInit2_ z_inflateInit2_ +# define inflateInit_ z_inflateInit_ +# define inflateMark z_inflateMark +# define inflatePrime z_inflatePrime +# define inflateReset z_inflateReset +# define inflateReset2 z_inflateReset2 +# define inflateResetKeep z_inflateResetKeep +# define inflateSetDictionary z_inflateSetDictionary +# define inflateSync z_inflateSync +# define inflateSyncPoint z_inflateSyncPoint +# define inflateUndermine z_inflateUndermine +# define inflateValidate z_inflateValidate +# define inflate_copyright z_inflate_copyright +# define inflate_fast z_inflate_fast +# define inflate_table z_inflate_table +# ifndef Z_SOLO +# define uncompress z_uncompress +# define uncompress2 z_uncompress2 +# endif +# define zError z_zError +# ifndef Z_SOLO +# define zcalloc z_zcalloc +# define zcfree z_zcfree +# endif +# define zlibCompileFlags z_zlibCompileFlags +# define zlibVersion z_zlibVersion + +/* all zlib typedefs in zlib.h and zconf.h */ +# define Byte z_Byte +# define Bytef z_Bytef +# define alloc_func z_alloc_func +# define charf z_charf +# define free_func z_free_func +# ifndef Z_SOLO +# define gzFile z_gzFile +# endif +# define gz_header z_gz_header +# define gz_headerp z_gz_headerp +# define in_func z_in_func +# define intf z_intf +# define out_func z_out_func +# define uInt z_uInt +# define uIntf z_uIntf +# define uLong z_uLong +# define uLongf z_uLongf +# define voidp z_voidp +# define voidpc z_voidpc +# define voidpf z_voidpf + +/* all zlib structs in zlib.h and zconf.h */ +# define gz_header_s z_gz_header_s +# define internal_state z_internal_state + +#endif + +#if defined(__MSDOS__) && !defined(MSDOS) +# define MSDOS +#endif +#if (defined(OS_2) || defined(__OS2__)) && !defined(OS2) +# define OS2 +#endif +#if defined(_WINDOWS) && !defined(WINDOWS) +# define WINDOWS +#endif +#if defined(_WIN32) || defined(_WIN32_WCE) || defined(__WIN32__) +# ifndef WIN32 +# define WIN32 +# endif +#endif +#if (defined(MSDOS) || defined(OS2) || defined(WINDOWS)) && !defined(WIN32) +# if !defined(__GNUC__) && !defined(__FLAT__) && !defined(__386__) +# ifndef SYS16BIT +# define SYS16BIT +# endif +# endif +#endif + +/* + * Compile with -DMAXSEG_64K if the alloc function cannot allocate more + * than 64k bytes at a time (needed on systems with 16-bit int). + */ +#ifdef SYS16BIT +# define MAXSEG_64K +#endif +#ifdef MSDOS +# define UNALIGNED_OK +#endif + +#ifdef __STDC_VERSION__ +# ifndef STDC +# define STDC +# endif +# if __STDC_VERSION__ >= 199901L +# ifndef STDC99 +# define STDC99 +# endif +# endif +#endif +#if !defined(STDC) && (defined(__STDC__) || defined(__cplusplus)) +# define STDC +#endif +#if !defined(STDC) && (defined(__GNUC__) || defined(__BORLANDC__)) +# define STDC +#endif +#if !defined(STDC) && (defined(MSDOS) || defined(WINDOWS) || defined(WIN32)) +# define STDC +#endif +#if !defined(STDC) && (defined(OS2) || defined(__HOS_AIX__)) +# define STDC +#endif + +#if defined(__OS400__) && !defined(STDC) /* iSeries (formerly AS/400). */ +# define STDC +#endif + +#ifndef STDC +# ifndef const /* cannot use !defined(STDC) && !defined(const) on Mac */ +# define const /* note: need a more gentle solution here */ +# endif +#endif + +#if defined(ZLIB_CONST) && !defined(z_const) +# define z_const const +#else +# define z_const +#endif + +#ifdef Z_SOLO + typedef unsigned long z_size_t; +#else +# define z_longlong long long +# if defined(NO_SIZE_T) + typedef unsigned NO_SIZE_T z_size_t; +# elif defined(STDC) +# include <stddef.h> + typedef size_t z_size_t; +# else + typedef unsigned long z_size_t; +# endif +# undef z_longlong +#endif + +/* Maximum value for memLevel in deflateInit2 */ +#ifndef MAX_MEM_LEVEL +# ifdef MAXSEG_64K +# define MAX_MEM_LEVEL 8 +# else +# define MAX_MEM_LEVEL 9 +# endif +#endif + +/* Maximum value for windowBits in deflateInit2 and inflateInit2. + * WARNING: reducing MAX_WBITS makes minigzip unable to extract .gz files + * created by gzip. (Files created by minigzip can still be extracted by + * gzip.) + */ +#ifndef MAX_WBITS +# define MAX_WBITS 15 /* 32K LZ77 window */ +#endif + +/* The memory requirements for deflate are (in bytes): + (1 << (windowBits+2)) + (1 << (memLevel+9)) + that is: 128K for windowBits=15 + 128K for memLevel = 8 (default values) + plus a few kilobytes for small objects. For example, if you want to reduce + the default memory requirements from 256K to 128K, compile with + make CFLAGS="-O -DMAX_WBITS=14 -DMAX_MEM_LEVEL=7" + Of course this will generally degrade compression (there's no free lunch). + + The memory requirements for inflate are (in bytes) 1 << windowBits + that is, 32K for windowBits=15 (default value) plus about 7 kilobytes + for small objects. +*/ + + /* Type declarations */ + +#ifndef OF /* function prototypes */ +# ifdef STDC +# define OF(args) args +# else +# define OF(args) () +# endif +#endif + +#ifndef Z_ARG /* function prototypes for stdarg */ +# if defined(STDC) || defined(Z_HAVE_STDARG_H) +# define Z_ARG(args) args +# else +# define Z_ARG(args) () +# endif +#endif + +/* The following definitions for FAR are needed only for MSDOS mixed + * model programming (small or medium model with some far allocations). + * This was tested only with MSC; for other MSDOS compilers you may have + * to define NO_MEMCPY in zutil.h. If you don't need the mixed model, + * just define FAR to be empty. + */ +#ifdef SYS16BIT +# if defined(M_I86SM) || defined(M_I86MM) + /* MSC small or medium model */ +# define SMALL_MEDIUM +# ifdef _MSC_VER +# define FAR _far +# else +# define FAR far +# endif +# endif +# if (defined(__SMALL__) || defined(__MEDIUM__)) + /* Turbo C small or medium model */ +# define SMALL_MEDIUM +# ifdef __BORLANDC__ +# define FAR _far +# else +# define FAR far +# endif +# endif +#endif + +#if defined(WINDOWS) || defined(WIN32) + /* If building or using zlib as a DLL, define ZLIB_DLL. + * This is not mandatory, but it offers a little performance increase. + */ +# ifdef ZLIB_DLL +# if defined(WIN32) && (!defined(__BORLANDC__) || (__BORLANDC__ >= 0x500)) +# ifdef ZLIB_INTERNAL +# define ZEXTERN extern __declspec(dllexport) +# else +# define ZEXTERN extern __declspec(dllimport) +# endif +# endif +# endif /* ZLIB_DLL */ + /* If building or using zlib with the WINAPI/WINAPIV calling convention, + * define ZLIB_WINAPI. + * Caution: the standard ZLIB1.DLL is NOT compiled using ZLIB_WINAPI. + */ +# ifdef ZLIB_WINAPI +# ifdef FAR +# undef FAR +# endif +# include <windows.h> + /* No need for _export, use ZLIB.DEF instead. */ + /* For complete Windows compatibility, use WINAPI, not __stdcall. */ +# define ZEXPORT WINAPI +# ifdef WIN32 +# define ZEXPORTVA WINAPIV +# else +# define ZEXPORTVA FAR CDECL +# endif +# endif +#endif + +#if defined (__BEOS__) +# ifdef ZLIB_DLL +# ifdef ZLIB_INTERNAL +# define ZEXPORT __declspec(dllexport) +# define ZEXPORTVA __declspec(dllexport) +# else +# define ZEXPORT __declspec(dllimport) +# define ZEXPORTVA __declspec(dllimport) +# endif +# endif +#endif + +#ifndef ZEXTERN +# define ZEXTERN extern +#endif +#ifndef ZEXPORT +# define ZEXPORT +#endif +#ifndef ZEXPORTVA +# define ZEXPORTVA +#endif + +#ifndef FAR +# define FAR +#endif +#ifndef far +# define far +#endif + +#if !defined(__MACTYPES__) +typedef unsigned char Byte; /* 8 bits */ +#endif +typedef unsigned int uInt; /* 16 bits or more */ +typedef unsigned long uLong; /* 32 bits or more */ + +#ifdef SMALL_MEDIUM + /* Borland C/C++ and some old MSC versions ignore FAR inside typedef */ +# define Bytef Byte FAR +#else + typedef Byte FAR Bytef; +#endif +typedef char FAR charf; +typedef int FAR intf; +typedef uInt FAR uIntf; +typedef uLong FAR uLongf; + +#ifdef STDC + typedef void const *voidpc; + typedef void FAR *voidpf; + typedef void *voidp; +#else + typedef Byte const *voidpc; + typedef Byte FAR *voidpf; + typedef Byte *voidp; +#endif + +#if !defined(Z_U4) && !defined(Z_SOLO) && defined(STDC) +# include <limits.h> +# if (UINT_MAX == 0xffffffffUL) +# define Z_U4 unsigned +# elif (ULONG_MAX == 0xffffffffUL) +# define Z_U4 unsigned long +# elif (USHRT_MAX == 0xffffffffUL) +# define Z_U4 unsigned short +# endif +#endif + +#ifdef Z_U4 + typedef Z_U4 z_crc_t; +#else + typedef unsigned long z_crc_t; +#endif + +#if !defined(_WIN32) +# define Z_HAVE_UNISTD_H +#endif + +#ifdef HAVE_STDARG_H /* may be set to #if 1 by ./configure */ +# define Z_HAVE_STDARG_H +#endif + +#ifdef STDC +# ifndef Z_SOLO +# include <sys/types.h> /* for off_t */ +# endif +#endif + +#if defined(STDC) || defined(Z_HAVE_STDARG_H) +# ifndef Z_SOLO +# include <stdarg.h> /* for va_list */ +# endif +#endif + +#ifdef _WIN32 +# ifndef Z_SOLO +# include <stddef.h> /* for wchar_t */ +# endif +#endif + +/* a little trick to accommodate both "#define _LARGEFILE64_SOURCE" and + * "#define _LARGEFILE64_SOURCE 1" as requesting 64-bit operations, (even + * though the former does not conform to the LFS document), but considering + * both "#undef _LARGEFILE64_SOURCE" and "#define _LARGEFILE64_SOURCE 0" as + * equivalently requesting no 64-bit operations + */ +#if defined(_LARGEFILE64_SOURCE) && -_LARGEFILE64_SOURCE - -1 == 1 +# undef _LARGEFILE64_SOURCE +#endif + +#if defined(__WATCOMC__) && !defined(Z_HAVE_UNISTD_H) +# define Z_HAVE_UNISTD_H +#endif +#ifndef Z_SOLO +# if defined(Z_HAVE_UNISTD_H) || defined(_LARGEFILE64_SOURCE) +# include <unistd.h> /* for SEEK_*, off_t, and _LFS64_LARGEFILE */ +# ifdef VMS +# include <unixio.h> /* for off_t */ +# endif +# ifndef z_off_t +# define z_off_t off_t +# endif +# endif +#endif + +#if defined(_LFS64_LARGEFILE) && _LFS64_LARGEFILE-0 +# define Z_LFS64 +#endif + +#if defined(_LARGEFILE64_SOURCE) && defined(Z_LFS64) +# define Z_LARGE64 +#endif + +#if defined(_FILE_OFFSET_BITS) && _FILE_OFFSET_BITS-0 == 64 && defined(Z_LFS64) +# define Z_WANT64 +#endif + +#if !defined(SEEK_SET) && !defined(Z_SOLO) +# define SEEK_SET 0 /* Seek from beginning of file. */ +# define SEEK_CUR 1 /* Seek from current position. */ +# define SEEK_END 2 /* Set file pointer to EOF plus "offset" */ +#endif + +#ifndef z_off_t +# define z_off_t long +#endif + +#if !defined(_WIN32) && defined(Z_LARGE64) +# define z_off64_t off64_t +#else +# if defined(_WIN32) && !defined(__GNUC__) && !defined(Z_SOLO) +# define z_off64_t __int64 +# else +# define z_off64_t z_off_t +# endif +#endif + +/* MVS linker does not support external names larger than 8 bytes */ +#if defined(__MVS__) + #pragma map(deflateInit_,"DEIN") + #pragma map(deflateInit2_,"DEIN2") + #pragma map(deflateEnd,"DEEND") + #pragma map(deflateBound,"DEBND") + #pragma map(inflateInit_,"ININ") + #pragma map(inflateInit2_,"ININ2") + #pragma map(inflateEnd,"INEND") + #pragma map(inflateSync,"INSY") + #pragma map(inflateSetDictionary,"INSEDI") + #pragma map(compressBound,"CMBND") + #pragma map(inflate_table,"INTABL") + #pragma map(inflate_fast,"INFA") + #pragma map(inflate_copyright,"INCOPY") +#endif + +#endif /* ZCONF_H */ diff --git a/third_party/zlib/zlib.h b/third_party/zlib/zlib.h new file mode 100644 index 00000000..d6404513 --- /dev/null +++ b/third_party/zlib/zlib.h @@ -0,0 +1,1998 @@ +/* zlib.h -- interface of the 'zlib' general purpose compression library + version 1.2.11, January 15th, 2017 + + Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + Jean-loup Gailly Mark Adler + jloup@gzip.org madler@alumni.caltech.edu + + + The data format used by the zlib library is described by RFCs (Request for + Comments) 1950 to 1952 in the files http://tools.ietf.org/html/rfc1950 + (zlib format), rfc1951 (deflate format) and rfc1952 (gzip format). +*/ + +#ifndef ZLIB_H +#define ZLIB_H + +#include "zconf.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#define ZLIB_VERSION "1.2.11" +#define ZLIB_VERNUM 0x12b0 +#define ZLIB_VER_MAJOR 1 +#define ZLIB_VER_MINOR 2 +#define ZLIB_VER_REVISION 11 +#define ZLIB_VER_SUBREVISION 0 + +/* + * In Android's NDK we have one zlib.h for all the versions. + * zlib users tend to use ZLIB_VERNUM to check API availability, + * so we need to translate __ANDROID_API__ appropriately. + * + * ZLIB_1.2.7.1 and ZLIB_1.2.9 are the only API changes in the NDK's + * supported range of API levels. + * + * jb-mr2-dev (18): 1.2.7 (but not 1.2.7.1, where the APIs were added!) + * https://android.googlesource.com/platform/external/zlib/+/refs/heads/jb-mr2-dev/src/zlib.h + * kitkat-dev (19): 1.2.8 + * https://android.googlesource.com/platform/external/zlib/+/refs/heads/kitkat-dev/src/zlib.h + * + * oreo-mr1-dev (27): 1.2.8 + * https://android.googlesource.com/platform/external/zlib/+/refs/heads/oreo-mr1-dev/src/zlib.h + * pie-dev (28): 1.2.11 + * https://android.googlesource.com/platform/external/zlib/+/refs/heads/pie-dev/src/zlib.h + * + * So: + * >= 28 --> 1.2.11 + * >= 19 --> 1.2.8 + * < 19 --> 1.2.7 + */ +#if defined(__ANDROID__) +# if __ANDROID_API__ >= 28 + /* Already okay. */ +# elif __ANDROID_API__ >= 19 +# undef ZLIB_VERSION +# define ZLIB_VERSION "1.2.8" +# undef ZLIB_VERNUM +# define ZLIB_VERNUM 0x1280 +# undef ZLIB_VER_REVISION +# define ZLIB_VER_REVISION 8 +# else +# undef ZLIB_VERSION +# define ZLIB_VERSION "1.2.6" +# undef ZLIB_VERNUM +# define ZLIB_VERNUM 0x1260 +# undef ZLIB_VER_REVISION +# define ZLIB_VER_REVISION 6 +# endif +#endif + +/* + The 'zlib' compression library provides in-memory compression and + decompression functions, including integrity checks of the uncompressed data. + This version of the library supports only one compression method (deflation) + but other algorithms will be added later and will have the same stream + interface. + + Compression can be done in a single step if the buffers are large enough, + or can be done by repeated calls of the compression function. In the latter + case, the application must provide more input and/or consume the output + (providing more output space) before each call. + + The compressed data format used by default by the in-memory functions is + the zlib format, which is a zlib wrapper documented in RFC 1950, wrapped + around a deflate stream, which is itself documented in RFC 1951. + + The library also supports reading and writing files in gzip (.gz) format + with an interface similar to that of stdio using the functions that start + with "gz". The gzip format is different from the zlib format. gzip is a + gzip wrapper, documented in RFC 1952, wrapped around a deflate stream. + + This library can optionally read and write gzip and raw deflate streams in + memory as well. + + The zlib format was designed to be compact and fast for use in memory + and on communications channels. The gzip format was designed for single- + file compression on file systems, has a larger header than zlib to maintain + directory information, and uses a different, slower check method than zlib. + + The library does not install any signal handler. The decoder checks + the consistency of the compressed data, so the library should never crash + even in the case of corrupted input. +*/ + +typedef voidpf (*alloc_func) OF((voidpf opaque, uInt items, uInt size)); +typedef void (*free_func) OF((voidpf opaque, voidpf address)); + +struct internal_state; + +typedef struct z_stream_s { + z_const Bytef *next_in; /* next input byte */ + uInt avail_in; /* number of bytes available at next_in */ + uLong total_in; /* total number of input bytes read so far */ + + Bytef *next_out; /* next output byte will go here */ + uInt avail_out; /* remaining free space at next_out */ + uLong total_out; /* total number of bytes output so far */ + + z_const char *msg; /* last error message, NULL if no error */ + struct internal_state FAR *state; /* not visible by applications */ + + alloc_func zalloc; /* used to allocate the internal state */ + free_func zfree; /* used to free the internal state */ + voidpf opaque; /* private data object passed to zalloc and zfree */ + + int data_type; /* best guess about the data type: binary or text + for deflate, or the decoding state for inflate */ + uLong adler; /* Adler-32 or CRC-32 value of the uncompressed data */ + uLong reserved; /* reserved for future use */ +} z_stream; + +typedef z_stream FAR *z_streamp; + +/* + gzip header information passed to and from zlib routines. See RFC 1952 + for more details on the meanings of these fields. +*/ +typedef struct gz_header_s { + int text; /* true if compressed data believed to be text */ + uLong time; /* modification time */ + int xflags; /* extra flags (not used when writing a gzip file) */ + int os; /* operating system */ + Bytef *extra; /* pointer to extra field or Z_NULL if none */ + uInt extra_len; /* extra field length (valid if extra != Z_NULL) */ + uInt extra_max; /* space at extra (only when reading header) */ + Bytef *name; /* pointer to zero-terminated file name or Z_NULL */ + uInt name_max; /* space at name (only when reading header) */ + Bytef *comment; /* pointer to zero-terminated comment or Z_NULL */ + uInt comm_max; /* space at comment (only when reading header) */ + int hcrc; /* true if there was or will be a header crc */ + int done; /* true when done reading gzip header (not used + when writing a gzip file) */ +} gz_header; + +typedef gz_header FAR *gz_headerp; + +/* + The application must update next_in and avail_in when avail_in has dropped + to zero. It must update next_out and avail_out when avail_out has dropped + to zero. The application must initialize zalloc, zfree and opaque before + calling the init function. All other fields are set by the compression + library and must not be updated by the application. + + The opaque value provided by the application will be passed as the first + parameter for calls of zalloc and zfree. This can be useful for custom + memory management. The compression library attaches no meaning to the + opaque value. + + zalloc must return Z_NULL if there is not enough memory for the object. + If zlib is used in a multi-threaded application, zalloc and zfree must be + thread safe. In that case, zlib is thread-safe. When zalloc and zfree are + Z_NULL on entry to the initialization function, they are set to internal + routines that use the standard library functions malloc() and free(). + + On 16-bit systems, the functions zalloc and zfree must be able to allocate + exactly 65536 bytes, but will not be required to allocate more than this if + the symbol MAXSEG_64K is defined (see zconf.h). WARNING: On MSDOS, pointers + returned by zalloc for objects of exactly 65536 bytes *must* have their + offset normalized to zero. The default allocation function provided by this + library ensures this (see zutil.c). To reduce memory requirements and avoid + any allocation of 64K objects, at the expense of compression ratio, compile + the library with -DMAX_WBITS=14 (see zconf.h). + + The fields total_in and total_out can be used for statistics or progress + reports. After compression, total_in holds the total size of the + uncompressed data and may be saved for use by the decompressor (particularly + if the decompressor wants to decompress everything in a single step). +*/ + + /* constants */ + +#define Z_NO_FLUSH 0 +#define Z_PARTIAL_FLUSH 1 +#define Z_SYNC_FLUSH 2 +#define Z_FULL_FLUSH 3 +#define Z_FINISH 4 +#define Z_BLOCK 5 +#define Z_TREES 6 +/* Allowed flush values; see deflate() and inflate() below for details */ + +#define Z_OK 0 +#define Z_STREAM_END 1 +#define Z_NEED_DICT 2 +#define Z_ERRNO (-1) +#define Z_STREAM_ERROR (-2) +#define Z_DATA_ERROR (-3) +#define Z_MEM_ERROR (-4) +#define Z_BUF_ERROR (-5) +#define Z_VERSION_ERROR (-6) +/* Return codes for the compression/decompression functions. Negative values + * are errors, positive values are used for special but normal events. + */ + +#define Z_NO_COMPRESSION 0 +#define Z_BEST_SPEED 1 +#define Z_BEST_COMPRESSION 9 +#define Z_DEFAULT_COMPRESSION (-1) +/* compression levels */ + +#define Z_FILTERED 1 +#define Z_HUFFMAN_ONLY 2 +#define Z_RLE 3 +#define Z_FIXED 4 +#define Z_DEFAULT_STRATEGY 0 +/* compression strategy; see deflateInit2() below for details */ + +#define Z_BINARY 0 +#define Z_TEXT 1 +#define Z_ASCII Z_TEXT /* for compatibility with 1.2.2 and earlier */ +#define Z_UNKNOWN 2 +/* Possible values of the data_type field for deflate() */ + +#define Z_DEFLATED 8 +/* The deflate compression method (the only one supported in this version) */ + +#define Z_NULL 0 /* for initializing zalloc, zfree, opaque */ + +#define zlib_version zlibVersion() +/* for compatibility with versions < 1.0.2 */ + + + /* basic functions */ + +ZEXTERN const char * ZEXPORT zlibVersion OF((void)); +/* The application can compare zlibVersion and ZLIB_VERSION for consistency. + If the first character differs, the library code actually used is not + compatible with the zlib.h header file used by the application. This check + is automatically made by deflateInit and inflateInit. + */ + +/* +ZEXTERN int ZEXPORT deflateInit OF((z_streamp strm, int level)); + + Initializes the internal stream state for compression. The fields + zalloc, zfree and opaque must be initialized before by the caller. If + zalloc and zfree are set to Z_NULL, deflateInit updates them to use default + allocation functions. + + The compression level must be Z_DEFAULT_COMPRESSION, or between 0 and 9: + 1 gives best speed, 9 gives best compression, 0 gives no compression at all + (the input data is simply copied a block at a time). Z_DEFAULT_COMPRESSION + requests a default compromise between speed and compression (currently + equivalent to level 6). + + deflateInit returns Z_OK if success, Z_MEM_ERROR if there was not enough + memory, Z_STREAM_ERROR if level is not a valid compression level, or + Z_VERSION_ERROR if the zlib library version (zlib_version) is incompatible + with the version assumed by the caller (ZLIB_VERSION). msg is set to null + if there is no error message. deflateInit does not perform any compression: + this will be done by deflate(). +*/ + + +ZEXTERN int ZEXPORT deflate OF((z_streamp strm, int flush)); +/* + deflate compresses as much data as possible, and stops when the input + buffer becomes empty or the output buffer becomes full. It may introduce + some output latency (reading input without producing any output) except when + forced to flush. + + The detailed semantics are as follows. deflate performs one or both of the + following actions: + + - Compress more input starting at next_in and update next_in and avail_in + accordingly. If not all input can be processed (because there is not + enough room in the output buffer), next_in and avail_in are updated and + processing will resume at this point for the next call of deflate(). + + - Generate more output starting at next_out and update next_out and avail_out + accordingly. This action is forced if the parameter flush is non zero. + Forcing flush frequently degrades the compression ratio, so this parameter + should be set only when necessary. Some output may be provided even if + flush is zero. + + Before the call of deflate(), the application should ensure that at least + one of the actions is possible, by providing more input and/or consuming more + output, and updating avail_in or avail_out accordingly; avail_out should + never be zero before the call. The application can consume the compressed + output when it wants, for example when the output buffer is full (avail_out + == 0), or after each call of deflate(). If deflate returns Z_OK and with + zero avail_out, it must be called again after making room in the output + buffer because there might be more output pending. See deflatePending(), + which can be used if desired to determine whether or not there is more ouput + in that case. + + Normally the parameter flush is set to Z_NO_FLUSH, which allows deflate to + decide how much data to accumulate before producing output, in order to + maximize compression. + + If the parameter flush is set to Z_SYNC_FLUSH, all pending output is + flushed to the output buffer and the output is aligned on a byte boundary, so + that the decompressor can get all input data available so far. (In + particular avail_in is zero after the call if enough output space has been + provided before the call.) Flushing may degrade compression for some + compression algorithms and so it should be used only when necessary. This + completes the current deflate block and follows it with an empty stored block + that is three bits plus filler bits to the next byte, followed by four bytes + (00 00 ff ff). + + If flush is set to Z_PARTIAL_FLUSH, all pending output is flushed to the + output buffer, but the output is not aligned to a byte boundary. All of the + input data so far will be available to the decompressor, as for Z_SYNC_FLUSH. + This completes the current deflate block and follows it with an empty fixed + codes block that is 10 bits long. This assures that enough bytes are output + in order for the decompressor to finish the block before the empty fixed + codes block. + + If flush is set to Z_BLOCK, a deflate block is completed and emitted, as + for Z_SYNC_FLUSH, but the output is not aligned on a byte boundary, and up to + seven bits of the current block are held to be written as the next byte after + the next deflate block is completed. In this case, the decompressor may not + be provided enough bits at this point in order to complete decompression of + the data provided so far to the compressor. It may need to wait for the next + block to be emitted. This is for advanced applications that need to control + the emission of deflate blocks. + + If flush is set to Z_FULL_FLUSH, all output is flushed as with + Z_SYNC_FLUSH, and the compression state is reset so that decompression can + restart from this point if previous compressed data has been damaged or if + random access is desired. Using Z_FULL_FLUSH too often can seriously degrade + compression. + + If deflate returns with avail_out == 0, this function must be called again + with the same value of the flush parameter and more output space (updated + avail_out), until the flush is complete (deflate returns with non-zero + avail_out). In the case of a Z_FULL_FLUSH or Z_SYNC_FLUSH, make sure that + avail_out is greater than six to avoid repeated flush markers due to + avail_out == 0 on return. + + If the parameter flush is set to Z_FINISH, pending input is processed, + pending output is flushed and deflate returns with Z_STREAM_END if there was + enough output space. If deflate returns with Z_OK or Z_BUF_ERROR, this + function must be called again with Z_FINISH and more output space (updated + avail_out) but no more input data, until it returns with Z_STREAM_END or an + error. After deflate has returned Z_STREAM_END, the only possible operations + on the stream are deflateReset or deflateEnd. + + Z_FINISH can be used in the first deflate call after deflateInit if all the + compression is to be done in a single step. In order to complete in one + call, avail_out must be at least the value returned by deflateBound (see + below). Then deflate is guaranteed to return Z_STREAM_END. If not enough + output space is provided, deflate will not return Z_STREAM_END, and it must + be called again as described above. + + deflate() sets strm->adler to the Adler-32 checksum of all input read + so far (that is, total_in bytes). If a gzip stream is being generated, then + strm->adler will be the CRC-32 checksum of the input read so far. (See + deflateInit2 below.) + + deflate() may update strm->data_type if it can make a good guess about + the input data type (Z_BINARY or Z_TEXT). If in doubt, the data is + considered binary. This field is only for information purposes and does not + affect the compression algorithm in any manner. + + deflate() returns Z_OK if some progress has been made (more input + processed or more output produced), Z_STREAM_END if all input has been + consumed and all output has been produced (only when flush is set to + Z_FINISH), Z_STREAM_ERROR if the stream state was inconsistent (for example + if next_in or next_out was Z_NULL or the state was inadvertently written over + by the application), or Z_BUF_ERROR if no progress is possible (for example + avail_in or avail_out was zero). Note that Z_BUF_ERROR is not fatal, and + deflate() can be called again with more input and more output space to + continue compressing. +*/ + + +ZEXTERN int ZEXPORT deflateEnd OF((z_streamp strm)); +/* + All dynamically allocated data structures for this stream are freed. + This function discards any unprocessed input and does not flush any pending + output. + + deflateEnd returns Z_OK if success, Z_STREAM_ERROR if the + stream state was inconsistent, Z_DATA_ERROR if the stream was freed + prematurely (some input or output was discarded). In the error case, msg + may be set but then points to a static string (which must not be + deallocated). +*/ + + +/* +ZEXTERN int ZEXPORT inflateInit OF((z_streamp strm)); + + Initializes the internal stream state for decompression. The fields + next_in, avail_in, zalloc, zfree and opaque must be initialized before by + the caller. In the current version of inflate, the provided input is not + read or consumed. The allocation of a sliding window will be deferred to + the first call of inflate (if the decompression does not complete on the + first call). If zalloc and zfree are set to Z_NULL, inflateInit updates + them to use default allocation functions. + + inflateInit returns Z_OK if success, Z_MEM_ERROR if there was not enough + memory, Z_VERSION_ERROR if the zlib library version is incompatible with the + version assumed by the caller, or Z_STREAM_ERROR if the parameters are + invalid, such as a null pointer to the structure. msg is set to null if + there is no error message. inflateInit does not perform any decompression. + Actual decompression will be done by inflate(). So next_in, and avail_in, + next_out, and avail_out are unused and unchanged. The current + implementation of inflateInit() does not process any header information -- + that is deferred until inflate() is called. +*/ + + +ZEXTERN int ZEXPORT inflate OF((z_streamp strm, int flush)); +/* + inflate decompresses as much data as possible, and stops when the input + buffer becomes empty or the output buffer becomes full. It may introduce + some output latency (reading input without producing any output) except when + forced to flush. + + The detailed semantics are as follows. inflate performs one or both of the + following actions: + + - Decompress more input starting at next_in and update next_in and avail_in + accordingly. If not all input can be processed (because there is not + enough room in the output buffer), then next_in and avail_in are updated + accordingly, and processing will resume at this point for the next call of + inflate(). + + - Generate more output starting at next_out and update next_out and avail_out + accordingly. inflate() provides as much output as possible, until there is + no more input data or no more space in the output buffer (see below about + the flush parameter). + + Before the call of inflate(), the application should ensure that at least + one of the actions is possible, by providing more input and/or consuming more + output, and updating the next_* and avail_* values accordingly. If the + caller of inflate() does not provide both available input and available + output space, it is possible that there will be no progress made. The + application can consume the uncompressed output when it wants, for example + when the output buffer is full (avail_out == 0), or after each call of + inflate(). If inflate returns Z_OK and with zero avail_out, it must be + called again after making room in the output buffer because there might be + more output pending. + + The flush parameter of inflate() can be Z_NO_FLUSH, Z_SYNC_FLUSH, Z_FINISH, + Z_BLOCK, or Z_TREES. Z_SYNC_FLUSH requests that inflate() flush as much + output as possible to the output buffer. Z_BLOCK requests that inflate() + stop if and when it gets to the next deflate block boundary. When decoding + the zlib or gzip format, this will cause inflate() to return immediately + after the header and before the first block. When doing a raw inflate, + inflate() will go ahead and process the first block, and will return when it + gets to the end of that block, or when it runs out of data. + + The Z_BLOCK option assists in appending to or combining deflate streams. + To assist in this, on return inflate() always sets strm->data_type to the + number of unused bits in the last byte taken from strm->next_in, plus 64 if + inflate() is currently decoding the last block in the deflate stream, plus + 128 if inflate() returned immediately after decoding an end-of-block code or + decoding the complete header up to just before the first byte of the deflate + stream. The end-of-block will not be indicated until all of the uncompressed + data from that block has been written to strm->next_out. The number of + unused bits may in general be greater than seven, except when bit 7 of + data_type is set, in which case the number of unused bits will be less than + eight. data_type is set as noted here every time inflate() returns for all + flush options, and so can be used to determine the amount of currently + consumed input in bits. + + The Z_TREES option behaves as Z_BLOCK does, but it also returns when the + end of each deflate block header is reached, before any actual data in that + block is decoded. This allows the caller to determine the length of the + deflate block header for later use in random access within a deflate block. + 256 is added to the value of strm->data_type when inflate() returns + immediately after reaching the end of the deflate block header. + + inflate() should normally be called until it returns Z_STREAM_END or an + error. However if all decompression is to be performed in a single step (a + single call of inflate), the parameter flush should be set to Z_FINISH. In + this case all pending input is processed and all pending output is flushed; + avail_out must be large enough to hold all of the uncompressed data for the + operation to complete. (The size of the uncompressed data may have been + saved by the compressor for this purpose.) The use of Z_FINISH is not + required to perform an inflation in one step. However it may be used to + inform inflate that a faster approach can be used for the single inflate() + call. Z_FINISH also informs inflate to not maintain a sliding window if the + stream completes, which reduces inflate's memory footprint. If the stream + does not complete, either because not all of the stream is provided or not + enough output space is provided, then a sliding window will be allocated and + inflate() can be called again to continue the operation as if Z_NO_FLUSH had + been used. + + In this implementation, inflate() always flushes as much output as + possible to the output buffer, and always uses the faster approach on the + first call. So the effects of the flush parameter in this implementation are + on the return value of inflate() as noted below, when inflate() returns early + when Z_BLOCK or Z_TREES is used, and when inflate() avoids the allocation of + memory for a sliding window when Z_FINISH is used. + + If a preset dictionary is needed after this call (see inflateSetDictionary + below), inflate sets strm->adler to the Adler-32 checksum of the dictionary + chosen by the compressor and returns Z_NEED_DICT; otherwise it sets + strm->adler to the Adler-32 checksum of all output produced so far (that is, + total_out bytes) and returns Z_OK, Z_STREAM_END or an error code as described + below. At the end of the stream, inflate() checks that its computed Adler-32 + checksum is equal to that saved by the compressor and returns Z_STREAM_END + only if the checksum is correct. + + inflate() can decompress and check either zlib-wrapped or gzip-wrapped + deflate data. The header type is detected automatically, if requested when + initializing with inflateInit2(). Any information contained in the gzip + header is not retained unless inflateGetHeader() is used. When processing + gzip-wrapped deflate data, strm->adler32 is set to the CRC-32 of the output + produced so far. The CRC-32 is checked against the gzip trailer, as is the + uncompressed length, modulo 2^32. + + inflate() returns Z_OK if some progress has been made (more input processed + or more output produced), Z_STREAM_END if the end of the compressed data has + been reached and all uncompressed output has been produced, Z_NEED_DICT if a + preset dictionary is needed at this point, Z_DATA_ERROR if the input data was + corrupted (input stream not conforming to the zlib format or incorrect check + value, in which case strm->msg points to a string with a more specific + error), Z_STREAM_ERROR if the stream structure was inconsistent (for example + next_in or next_out was Z_NULL, or the state was inadvertently written over + by the application), Z_MEM_ERROR if there was not enough memory, Z_BUF_ERROR + if no progress was possible or if there was not enough room in the output + buffer when Z_FINISH is used. Note that Z_BUF_ERROR is not fatal, and + inflate() can be called again with more input and more output space to + continue decompressing. If Z_DATA_ERROR is returned, the application may + then call inflateSync() to look for a good compression block if a partial + recovery of the data is to be attempted. +*/ + + +ZEXTERN int ZEXPORT inflateEnd OF((z_streamp strm)); +/* + All dynamically allocated data structures for this stream are freed. + This function discards any unprocessed input and does not flush any pending + output. + + inflateEnd returns Z_OK if success, or Z_STREAM_ERROR if the stream state + was inconsistent. +*/ + + + /* Advanced functions */ + +/* + The following functions are needed only in some special applications. +*/ + +/* +ZEXTERN int ZEXPORT deflateInit2 OF((z_streamp strm, + int level, + int method, + int windowBits, + int memLevel, + int strategy)); + + This is another version of deflateInit with more compression options. The + fields next_in, zalloc, zfree and opaque must be initialized before by the + caller. + + The method parameter is the compression method. It must be Z_DEFLATED in + this version of the library. + + The windowBits parameter is the base two logarithm of the window size + (the size of the history buffer). It should be in the range 8..15 for this + version of the library. Larger values of this parameter result in better + compression at the expense of memory usage. The default value is 15 if + deflateInit is used instead. + + For the current implementation of deflate(), a windowBits value of 8 (a + window size of 256 bytes) is not supported. As a result, a request for 8 + will result in 9 (a 512-byte window). In that case, providing 8 to + inflateInit2() will result in an error when the zlib header with 9 is + checked against the initialization of inflate(). The remedy is to not use 8 + with deflateInit2() with this initialization, or at least in that case use 9 + with inflateInit2(). + + windowBits can also be -8..-15 for raw deflate. In this case, -windowBits + determines the window size. deflate() will then generate raw deflate data + with no zlib header or trailer, and will not compute a check value. + + windowBits can also be greater than 15 for optional gzip encoding. Add + 16 to windowBits to write a simple gzip header and trailer around the + compressed data instead of a zlib wrapper. The gzip header will have no + file name, no extra data, no comment, no modification time (set to zero), no + header crc, and the operating system will be set to the appropriate value, + if the operating system was determined at compile time. If a gzip stream is + being written, strm->adler is a CRC-32 instead of an Adler-32. + + For raw deflate or gzip encoding, a request for a 256-byte window is + rejected as invalid, since only the zlib header provides a means of + transmitting the window size to the decompressor. + + The memLevel parameter specifies how much memory should be allocated + for the internal compression state. memLevel=1 uses minimum memory but is + slow and reduces compression ratio; memLevel=9 uses maximum memory for + optimal speed. The default value is 8. See zconf.h for total memory usage + as a function of windowBits and memLevel. + + The strategy parameter is used to tune the compression algorithm. Use the + value Z_DEFAULT_STRATEGY for normal data, Z_FILTERED for data produced by a + filter (or predictor), Z_HUFFMAN_ONLY to force Huffman encoding only (no + string match), or Z_RLE to limit match distances to one (run-length + encoding). Filtered data consists mostly of small values with a somewhat + random distribution. In this case, the compression algorithm is tuned to + compress them better. The effect of Z_FILTERED is to force more Huffman + coding and less string matching; it is somewhat intermediate between + Z_DEFAULT_STRATEGY and Z_HUFFMAN_ONLY. Z_RLE is designed to be almost as + fast as Z_HUFFMAN_ONLY, but give better compression for PNG image data. The + strategy parameter only affects the compression ratio but not the + correctness of the compressed output even if it is not set appropriately. + Z_FIXED prevents the use of dynamic Huffman codes, allowing for a simpler + decoder for special applications. + + deflateInit2 returns Z_OK if success, Z_MEM_ERROR if there was not enough + memory, Z_STREAM_ERROR if any parameter is invalid (such as an invalid + method), or Z_VERSION_ERROR if the zlib library version (zlib_version) is + incompatible with the version assumed by the caller (ZLIB_VERSION). msg is + set to null if there is no error message. deflateInit2 does not perform any + compression: this will be done by deflate(). +*/ + +ZEXTERN int ZEXPORT deflateSetDictionary OF((z_streamp strm, + const Bytef *dictionary, + uInt dictLength)); +/* + Initializes the compression dictionary from the given byte sequence + without producing any compressed output. When using the zlib format, this + function must be called immediately after deflateInit, deflateInit2 or + deflateReset, and before any call of deflate. When doing raw deflate, this + function must be called either before any call of deflate, or immediately + after the completion of a deflate block, i.e. after all input has been + consumed and all output has been delivered when using any of the flush + options Z_BLOCK, Z_PARTIAL_FLUSH, Z_SYNC_FLUSH, or Z_FULL_FLUSH. The + compressor and decompressor must use exactly the same dictionary (see + inflateSetDictionary). + + The dictionary should consist of strings (byte sequences) that are likely + to be encountered later in the data to be compressed, with the most commonly + used strings preferably put towards the end of the dictionary. Using a + dictionary is most useful when the data to be compressed is short and can be + predicted with good accuracy; the data can then be compressed better than + with the default empty dictionary. + + Depending on the size of the compression data structures selected by + deflateInit or deflateInit2, a part of the dictionary may in effect be + discarded, for example if the dictionary is larger than the window size + provided in deflateInit or deflateInit2. Thus the strings most likely to be + useful should be put at the end of the dictionary, not at the front. In + addition, the current implementation of deflate will use at most the window + size minus 262 bytes of the provided dictionary. + + Upon return of this function, strm->adler is set to the Adler-32 value + of the dictionary; the decompressor may later use this value to determine + which dictionary has been used by the compressor. (The Adler-32 value + applies to the whole dictionary even if only a subset of the dictionary is + actually used by the compressor.) If a raw deflate was requested, then the + Adler-32 value is not computed and strm->adler is not set. + + deflateSetDictionary returns Z_OK if success, or Z_STREAM_ERROR if a + parameter is invalid (e.g. dictionary being Z_NULL) or the stream state is + inconsistent (for example if deflate has already been called for this stream + or if not at a block boundary for raw deflate). deflateSetDictionary does + not perform any compression: this will be done by deflate(). +*/ + +#if !defined(__ANDROID__) || __ANDROID_API__ >= 28 +ZEXTERN int ZEXPORT deflateGetDictionary OF((z_streamp strm, + Bytef *dictionary, + uInt *dictLength)); +#endif +/* + Returns the sliding dictionary being maintained by deflate. dictLength is + set to the number of bytes in the dictionary, and that many bytes are copied + to dictionary. dictionary must have enough space, where 32768 bytes is + always enough. If deflateGetDictionary() is called with dictionary equal to + Z_NULL, then only the dictionary length is returned, and nothing is copied. + Similary, if dictLength is Z_NULL, then it is not set. + + deflateGetDictionary() may return a length less than the window size, even + when more than the window size in input has been provided. It may return up + to 258 bytes less in that case, due to how zlib's implementation of deflate + manages the sliding window and lookahead for matches, where matches can be + up to 258 bytes long. If the application needs the last window-size bytes of + input, then that would need to be saved by the application outside of zlib. + + deflateGetDictionary returns Z_OK on success, or Z_STREAM_ERROR if the + stream state is inconsistent. +*/ + +ZEXTERN int ZEXPORT deflateCopy OF((z_streamp dest, + z_streamp source)); +/* + Sets the destination stream as a complete copy of the source stream. + + This function can be useful when several compression strategies will be + tried, for example when there are several ways of pre-processing the input + data with a filter. The streams that will be discarded should then be freed + by calling deflateEnd. Note that deflateCopy duplicates the internal + compression state which can be quite large, so this strategy is slow and can + consume lots of memory. + + deflateCopy returns Z_OK if success, Z_MEM_ERROR if there was not + enough memory, Z_STREAM_ERROR if the source stream state was inconsistent + (such as zalloc being Z_NULL). msg is left unchanged in both source and + destination. +*/ + +ZEXTERN int ZEXPORT deflateReset OF((z_streamp strm)); +/* + This function is equivalent to deflateEnd followed by deflateInit, but + does not free and reallocate the internal compression state. The stream + will leave the compression level and any other attributes that may have been + set unchanged. + + deflateReset returns Z_OK if success, or Z_STREAM_ERROR if the source + stream state was inconsistent (such as zalloc or state being Z_NULL). +*/ + +ZEXTERN int ZEXPORT deflateParams OF((z_streamp strm, + int level, + int strategy)); +/* + Dynamically update the compression level and compression strategy. The + interpretation of level and strategy is as in deflateInit2(). This can be + used to switch between compression and straight copy of the input data, or + to switch to a different kind of input data requiring a different strategy. + If the compression approach (which is a function of the level) or the + strategy is changed, and if any input has been consumed in a previous + deflate() call, then the input available so far is compressed with the old + level and strategy using deflate(strm, Z_BLOCK). There are three approaches + for the compression levels 0, 1..3, and 4..9 respectively. The new level + and strategy will take effect at the next call of deflate(). + + If a deflate(strm, Z_BLOCK) is performed by deflateParams(), and it does + not have enough output space to complete, then the parameter change will not + take effect. In this case, deflateParams() can be called again with the + same parameters and more output space to try again. + + In order to assure a change in the parameters on the first try, the + deflate stream should be flushed using deflate() with Z_BLOCK or other flush + request until strm.avail_out is not zero, before calling deflateParams(). + Then no more input data should be provided before the deflateParams() call. + If this is done, the old level and strategy will be applied to the data + compressed before deflateParams(), and the new level and strategy will be + applied to the the data compressed after deflateParams(). + + deflateParams returns Z_OK on success, Z_STREAM_ERROR if the source stream + state was inconsistent or if a parameter was invalid, or Z_BUF_ERROR if + there was not enough output space to complete the compression of the + available input data before a change in the strategy or approach. Note that + in the case of a Z_BUF_ERROR, the parameters are not changed. A return + value of Z_BUF_ERROR is not fatal, in which case deflateParams() can be + retried with more output space. +*/ + +ZEXTERN int ZEXPORT deflateTune OF((z_streamp strm, + int good_length, + int max_lazy, + int nice_length, + int max_chain)); +/* + Fine tune deflate's internal compression parameters. This should only be + used by someone who understands the algorithm used by zlib's deflate for + searching for the best matching string, and even then only by the most + fanatic optimizer trying to squeeze out the last compressed bit for their + specific input data. Read the deflate.c source code for the meaning of the + max_lazy, good_length, nice_length, and max_chain parameters. + + deflateTune() can be called after deflateInit() or deflateInit2(), and + returns Z_OK on success, or Z_STREAM_ERROR for an invalid deflate stream. + */ + +ZEXTERN uLong ZEXPORT deflateBound OF((z_streamp strm, + uLong sourceLen)); +/* + deflateBound() returns an upper bound on the compressed size after + deflation of sourceLen bytes. It must be called after deflateInit() or + deflateInit2(), and after deflateSetHeader(), if used. This would be used + to allocate an output buffer for deflation in a single pass, and so would be + called before deflate(). If that first deflate() call is provided the + sourceLen input bytes, an output buffer allocated to the size returned by + deflateBound(), and the flush value Z_FINISH, then deflate() is guaranteed + to return Z_STREAM_END. Note that it is possible for the compressed size to + be larger than the value returned by deflateBound() if flush options other + than Z_FINISH or Z_NO_FLUSH are used. +*/ + +ZEXTERN int ZEXPORT deflatePending OF((z_streamp strm, + unsigned *pending, + int *bits)); +/* + deflatePending() returns the number of bytes and bits of output that have + been generated, but not yet provided in the available output. The bytes not + provided would be due to the available output space having being consumed. + The number of bits of output not provided are between 0 and 7, where they + await more bits to join them in order to fill out a full byte. If pending + or bits are Z_NULL, then those values are not set. + + deflatePending returns Z_OK if success, or Z_STREAM_ERROR if the source + stream state was inconsistent. + */ + +ZEXTERN int ZEXPORT deflatePrime OF((z_streamp strm, + int bits, + int value)); +/* + deflatePrime() inserts bits in the deflate output stream. The intent + is that this function is used to start off the deflate output with the bits + leftover from a previous deflate stream when appending to it. As such, this + function can only be used for raw deflate, and must be used before the first + deflate() call after a deflateInit2() or deflateReset(). bits must be less + than or equal to 16, and that many of the least significant bits of value + will be inserted in the output. + + deflatePrime returns Z_OK if success, Z_BUF_ERROR if there was not enough + room in the internal buffer to insert the bits, or Z_STREAM_ERROR if the + source stream state was inconsistent. +*/ + +ZEXTERN int ZEXPORT deflateSetHeader OF((z_streamp strm, + gz_headerp head)); +/* + deflateSetHeader() provides gzip header information for when a gzip + stream is requested by deflateInit2(). deflateSetHeader() may be called + after deflateInit2() or deflateReset() and before the first call of + deflate(). The text, time, os, extra field, name, and comment information + in the provided gz_header structure are written to the gzip header (xflag is + ignored -- the extra flags are set according to the compression level). The + caller must assure that, if not Z_NULL, name and comment are terminated with + a zero byte, and that if extra is not Z_NULL, that extra_len bytes are + available there. If hcrc is true, a gzip header crc is included. Note that + the current versions of the command-line version of gzip (up through version + 1.3.x) do not support header crc's, and will report that it is a "multi-part + gzip file" and give up. + + If deflateSetHeader is not used, the default gzip header has text false, + the time set to zero, and os set to 255, with no extra, name, or comment + fields. The gzip header is returned to the default state by deflateReset(). + + deflateSetHeader returns Z_OK if success, or Z_STREAM_ERROR if the source + stream state was inconsistent. +*/ + +/* +ZEXTERN int ZEXPORT inflateInit2 OF((z_streamp strm, + int windowBits)); + + This is another version of inflateInit with an extra parameter. The + fields next_in, avail_in, zalloc, zfree and opaque must be initialized + before by the caller. + + The windowBits parameter is the base two logarithm of the maximum window + size (the size of the history buffer). It should be in the range 8..15 for + this version of the library. The default value is 15 if inflateInit is used + instead. windowBits must be greater than or equal to the windowBits value + provided to deflateInit2() while compressing, or it must be equal to 15 if + deflateInit2() was not used. If a compressed stream with a larger window + size is given as input, inflate() will return with the error code + Z_DATA_ERROR instead of trying to allocate a larger window. + + windowBits can also be zero to request that inflate use the window size in + the zlib header of the compressed stream. + + windowBits can also be -8..-15 for raw inflate. In this case, -windowBits + determines the window size. inflate() will then process raw deflate data, + not looking for a zlib or gzip header, not generating a check value, and not + looking for any check values for comparison at the end of the stream. This + is for use with other formats that use the deflate compressed data format + such as zip. Those formats provide their own check values. If a custom + format is developed using the raw deflate format for compressed data, it is + recommended that a check value such as an Adler-32 or a CRC-32 be applied to + the uncompressed data as is done in the zlib, gzip, and zip formats. For + most applications, the zlib format should be used as is. Note that comments + above on the use in deflateInit2() applies to the magnitude of windowBits. + + windowBits can also be greater than 15 for optional gzip decoding. Add + 32 to windowBits to enable zlib and gzip decoding with automatic header + detection, or add 16 to decode only the gzip format (the zlib format will + return a Z_DATA_ERROR). If a gzip stream is being decoded, strm->adler is a + CRC-32 instead of an Adler-32. Unlike the gunzip utility and gzread() (see + below), inflate() will not automatically decode concatenated gzip streams. + inflate() will return Z_STREAM_END at the end of the gzip stream. The state + would need to be reset to continue decoding a subsequent gzip stream. + + inflateInit2 returns Z_OK if success, Z_MEM_ERROR if there was not enough + memory, Z_VERSION_ERROR if the zlib library version is incompatible with the + version assumed by the caller, or Z_STREAM_ERROR if the parameters are + invalid, such as a null pointer to the structure. msg is set to null if + there is no error message. inflateInit2 does not perform any decompression + apart from possibly reading the zlib header if present: actual decompression + will be done by inflate(). (So next_in and avail_in may be modified, but + next_out and avail_out are unused and unchanged.) The current implementation + of inflateInit2() does not process any header information -- that is + deferred until inflate() is called. +*/ + +ZEXTERN int ZEXPORT inflateSetDictionary OF((z_streamp strm, + const Bytef *dictionary, + uInt dictLength)); +/* + Initializes the decompression dictionary from the given uncompressed byte + sequence. This function must be called immediately after a call of inflate, + if that call returned Z_NEED_DICT. The dictionary chosen by the compressor + can be determined from the Adler-32 value returned by that call of inflate. + The compressor and decompressor must use exactly the same dictionary (see + deflateSetDictionary). For raw inflate, this function can be called at any + time to set the dictionary. If the provided dictionary is smaller than the + window and there is already data in the window, then the provided dictionary + will amend what's there. The application must insure that the dictionary + that was used for compression is provided. + + inflateSetDictionary returns Z_OK if success, Z_STREAM_ERROR if a + parameter is invalid (e.g. dictionary being Z_NULL) or the stream state is + inconsistent, Z_DATA_ERROR if the given dictionary doesn't match the + expected one (incorrect Adler-32 value). inflateSetDictionary does not + perform any decompression: this will be done by subsequent calls of + inflate(). +*/ + +#if !defined(__ANDROID__) || __ANDROID_API__ >= 19 +ZEXTERN int ZEXPORT inflateGetDictionary OF((z_streamp strm, + Bytef *dictionary, + uInt *dictLength)); +#endif +/* + Returns the sliding dictionary being maintained by inflate. dictLength is + set to the number of bytes in the dictionary, and that many bytes are copied + to dictionary. dictionary must have enough space, where 32768 bytes is + always enough. If inflateGetDictionary() is called with dictionary equal to + Z_NULL, then only the dictionary length is returned, and nothing is copied. + Similary, if dictLength is Z_NULL, then it is not set. + + inflateGetDictionary returns Z_OK on success, or Z_STREAM_ERROR if the + stream state is inconsistent. +*/ + +ZEXTERN int ZEXPORT inflateSync OF((z_streamp strm)); +/* + Skips invalid compressed data until a possible full flush point (see above + for the description of deflate with Z_FULL_FLUSH) can be found, or until all + available input is skipped. No output is provided. + + inflateSync searches for a 00 00 FF FF pattern in the compressed data. + All full flush points have this pattern, but not all occurrences of this + pattern are full flush points. + + inflateSync returns Z_OK if a possible full flush point has been found, + Z_BUF_ERROR if no more input was provided, Z_DATA_ERROR if no flush point + has been found, or Z_STREAM_ERROR if the stream structure was inconsistent. + In the success case, the application may save the current current value of + total_in which indicates where valid compressed data was found. In the + error case, the application may repeatedly call inflateSync, providing more + input each time, until success or end of the input data. +*/ + +ZEXTERN int ZEXPORT inflateCopy OF((z_streamp dest, + z_streamp source)); +/* + Sets the destination stream as a complete copy of the source stream. + + This function can be useful when randomly accessing a large stream. The + first pass through the stream can periodically record the inflate state, + allowing restarting inflate at those points when randomly accessing the + stream. + + inflateCopy returns Z_OK if success, Z_MEM_ERROR if there was not + enough memory, Z_STREAM_ERROR if the source stream state was inconsistent + (such as zalloc being Z_NULL). msg is left unchanged in both source and + destination. +*/ + +ZEXTERN int ZEXPORT inflateReset OF((z_streamp strm)); +/* + This function is equivalent to inflateEnd followed by inflateInit, + but does not free and reallocate the internal decompression state. The + stream will keep attributes that may have been set by inflateInit2. + + inflateReset returns Z_OK if success, or Z_STREAM_ERROR if the source + stream state was inconsistent (such as zalloc or state being Z_NULL). +*/ + +ZEXTERN int ZEXPORT inflateReset2 OF((z_streamp strm, + int windowBits)); +/* + This function is the same as inflateReset, but it also permits changing + the wrap and window size requests. The windowBits parameter is interpreted + the same as it is for inflateInit2. If the window size is changed, then the + memory allocated for the window is freed, and the window will be reallocated + by inflate() if needed. + + inflateReset2 returns Z_OK if success, or Z_STREAM_ERROR if the source + stream state was inconsistent (such as zalloc or state being Z_NULL), or if + the windowBits parameter is invalid. +*/ + +ZEXTERN int ZEXPORT inflatePrime OF((z_streamp strm, + int bits, + int value)); +/* + This function inserts bits in the inflate input stream. The intent is + that this function is used to start inflating at a bit position in the + middle of a byte. The provided bits will be used before any bytes are used + from next_in. This function should only be used with raw inflate, and + should be used before the first inflate() call after inflateInit2() or + inflateReset(). bits must be less than or equal to 16, and that many of the + least significant bits of value will be inserted in the input. + + If bits is negative, then the input stream bit buffer is emptied. Then + inflatePrime() can be called again to put bits in the buffer. This is used + to clear out bits leftover after feeding inflate a block description prior + to feeding inflate codes. + + inflatePrime returns Z_OK if success, or Z_STREAM_ERROR if the source + stream state was inconsistent. +*/ + +ZEXTERN long ZEXPORT inflateMark OF((z_streamp strm)); +/* + This function returns two values, one in the lower 16 bits of the return + value, and the other in the remaining upper bits, obtained by shifting the + return value down 16 bits. If the upper value is -1 and the lower value is + zero, then inflate() is currently decoding information outside of a block. + If the upper value is -1 and the lower value is non-zero, then inflate is in + the middle of a stored block, with the lower value equaling the number of + bytes from the input remaining to copy. If the upper value is not -1, then + it is the number of bits back from the current bit position in the input of + the code (literal or length/distance pair) currently being processed. In + that case the lower value is the number of bytes already emitted for that + code. + + A code is being processed if inflate is waiting for more input to complete + decoding of the code, or if it has completed decoding but is waiting for + more output space to write the literal or match data. + + inflateMark() is used to mark locations in the input data for random + access, which may be at bit positions, and to note those cases where the + output of a code may span boundaries of random access blocks. The current + location in the input stream can be determined from avail_in and data_type + as noted in the description for the Z_BLOCK flush parameter for inflate. + + inflateMark returns the value noted above, or -65536 if the provided + source stream state was inconsistent. +*/ + +ZEXTERN int ZEXPORT inflateGetHeader OF((z_streamp strm, + gz_headerp head)); +/* + inflateGetHeader() requests that gzip header information be stored in the + provided gz_header structure. inflateGetHeader() may be called after + inflateInit2() or inflateReset(), and before the first call of inflate(). + As inflate() processes the gzip stream, head->done is zero until the header + is completed, at which time head->done is set to one. If a zlib stream is + being decoded, then head->done is set to -1 to indicate that there will be + no gzip header information forthcoming. Note that Z_BLOCK or Z_TREES can be + used to force inflate() to return immediately after header processing is + complete and before any actual data is decompressed. + + The text, time, xflags, and os fields are filled in with the gzip header + contents. hcrc is set to true if there is a header CRC. (The header CRC + was valid if done is set to one.) If extra is not Z_NULL, then extra_max + contains the maximum number of bytes to write to extra. Once done is true, + extra_len contains the actual extra field length, and extra contains the + extra field, or that field truncated if extra_max is less than extra_len. + If name is not Z_NULL, then up to name_max characters are written there, + terminated with a zero unless the length is greater than name_max. If + comment is not Z_NULL, then up to comm_max characters are written there, + terminated with a zero unless the length is greater than comm_max. When any + of extra, name, or comment are not Z_NULL and the respective field is not + present in the header, then that field is set to Z_NULL to signal its + absence. This allows the use of deflateSetHeader() with the returned + structure to duplicate the header. However if those fields are set to + allocated memory, then the application will need to save those pointers + elsewhere so that they can be eventually freed. + + If inflateGetHeader is not used, then the header information is simply + discarded. The header is always checked for validity, including the header + CRC if present. inflateReset() will reset the process to discard the header + information. The application would need to call inflateGetHeader() again to + retrieve the header from the next gzip stream. + + inflateGetHeader returns Z_OK if success, or Z_STREAM_ERROR if the source + stream state was inconsistent. +*/ + +/* +ZEXTERN int ZEXPORT inflateBackInit OF((z_streamp strm, int windowBits, + unsigned char FAR *window)); + + Initialize the internal stream state for decompression using inflateBack() + calls. The fields zalloc, zfree and opaque in strm must be initialized + before the call. If zalloc and zfree are Z_NULL, then the default library- + derived memory allocation routines are used. windowBits is the base two + logarithm of the window size, in the range 8..15. window is a caller + supplied buffer of that size. Except for special applications where it is + assured that deflate was used with small window sizes, windowBits must be 15 + and a 32K byte window must be supplied to be able to decompress general + deflate streams. + + See inflateBack() for the usage of these routines. + + inflateBackInit will return Z_OK on success, Z_STREAM_ERROR if any of + the parameters are invalid, Z_MEM_ERROR if the internal state could not be + allocated, or Z_VERSION_ERROR if the version of the library does not match + the version of the header file. +*/ + +typedef unsigned (*in_func) OF((void FAR *, + z_const unsigned char FAR * FAR *)); +typedef int (*out_func) OF((void FAR *, unsigned char FAR *, unsigned)); + +ZEXTERN int ZEXPORT inflateBack OF((z_streamp strm, + in_func in, void FAR *in_desc, + out_func out, void FAR *out_desc)); +/* + inflateBack() does a raw inflate with a single call using a call-back + interface for input and output. This is potentially more efficient than + inflate() for file i/o applications, in that it avoids copying between the + output and the sliding window by simply making the window itself the output + buffer. inflate() can be faster on modern CPUs when used with large + buffers. inflateBack() trusts the application to not change the output + buffer passed by the output function, at least until inflateBack() returns. + + inflateBackInit() must be called first to allocate the internal state + and to initialize the state with the user-provided window buffer. + inflateBack() may then be used multiple times to inflate a complete, raw + deflate stream with each call. inflateBackEnd() is then called to free the + allocated state. + + A raw deflate stream is one with no zlib or gzip header or trailer. + This routine would normally be used in a utility that reads zip or gzip + files and writes out uncompressed files. The utility would decode the + header and process the trailer on its own, hence this routine expects only + the raw deflate stream to decompress. This is different from the default + behavior of inflate(), which expects a zlib header and trailer around the + deflate stream. + + inflateBack() uses two subroutines supplied by the caller that are then + called by inflateBack() for input and output. inflateBack() calls those + routines until it reads a complete deflate stream and writes out all of the + uncompressed data, or until it encounters an error. The function's + parameters and return types are defined above in the in_func and out_func + typedefs. inflateBack() will call in(in_desc, &buf) which should return the + number of bytes of provided input, and a pointer to that input in buf. If + there is no input available, in() must return zero -- buf is ignored in that + case -- and inflateBack() will return a buffer error. inflateBack() will + call out(out_desc, buf, len) to write the uncompressed data buf[0..len-1]. + out() should return zero on success, or non-zero on failure. If out() + returns non-zero, inflateBack() will return with an error. Neither in() nor + out() are permitted to change the contents of the window provided to + inflateBackInit(), which is also the buffer that out() uses to write from. + The length written by out() will be at most the window size. Any non-zero + amount of input may be provided by in(). + + For convenience, inflateBack() can be provided input on the first call by + setting strm->next_in and strm->avail_in. If that input is exhausted, then + in() will be called. Therefore strm->next_in must be initialized before + calling inflateBack(). If strm->next_in is Z_NULL, then in() will be called + immediately for input. If strm->next_in is not Z_NULL, then strm->avail_in + must also be initialized, and then if strm->avail_in is not zero, input will + initially be taken from strm->next_in[0 .. strm->avail_in - 1]. + + The in_desc and out_desc parameters of inflateBack() is passed as the + first parameter of in() and out() respectively when they are called. These + descriptors can be optionally used to pass any information that the caller- + supplied in() and out() functions need to do their job. + + On return, inflateBack() will set strm->next_in and strm->avail_in to + pass back any unused input that was provided by the last in() call. The + return values of inflateBack() can be Z_STREAM_END on success, Z_BUF_ERROR + if in() or out() returned an error, Z_DATA_ERROR if there was a format error + in the deflate stream (in which case strm->msg is set to indicate the nature + of the error), or Z_STREAM_ERROR if the stream was not properly initialized. + In the case of Z_BUF_ERROR, an input or output error can be distinguished + using strm->next_in which will be Z_NULL only if in() returned an error. If + strm->next_in is not Z_NULL, then the Z_BUF_ERROR was due to out() returning + non-zero. (in() will always be called before out(), so strm->next_in is + assured to be defined if out() returns non-zero.) Note that inflateBack() + cannot return Z_OK. +*/ + +ZEXTERN int ZEXPORT inflateBackEnd OF((z_streamp strm)); +/* + All memory allocated by inflateBackInit() is freed. + + inflateBackEnd() returns Z_OK on success, or Z_STREAM_ERROR if the stream + state was inconsistent. +*/ + +ZEXTERN uLong ZEXPORT zlibCompileFlags OF((void)); +/* Return flags indicating compile-time options. + + Type sizes, two bits each, 00 = 16 bits, 01 = 32, 10 = 64, 11 = other: + 1.0: size of uInt + 3.2: size of uLong + 5.4: size of voidpf (pointer) + 7.6: size of z_off_t + + Compiler, assembler, and debug options: + 8: ZLIB_DEBUG + 9: ASMV or ASMINF -- use ASM code + 10: ZLIB_WINAPI -- exported functions use the WINAPI calling convention + 11: 0 (reserved) + + One-time table building (smaller code, but not thread-safe if true): + 12: BUILDFIXED -- build static block decoding tables when needed + 13: DYNAMIC_CRC_TABLE -- build CRC calculation tables when needed + 14,15: 0 (reserved) + + Library content (indicates missing functionality): + 16: NO_GZCOMPRESS -- gz* functions cannot compress (to avoid linking + deflate code when not needed) + 17: NO_GZIP -- deflate can't write gzip streams, and inflate can't detect + and decode gzip streams (to avoid linking crc code) + 18-19: 0 (reserved) + + Operation variations (changes in library functionality): + 20: PKZIP_BUG_WORKAROUND -- slightly more permissive inflate + 21: FASTEST -- deflate algorithm with only one, lowest compression level + 22,23: 0 (reserved) + + The sprintf variant used by gzprintf (zero is best): + 24: 0 = vs*, 1 = s* -- 1 means limited to 20 arguments after the format + 25: 0 = *nprintf, 1 = *printf -- 1 means gzprintf() not secure! + 26: 0 = returns value, 1 = void -- 1 means inferred string length returned + + Remainder: + 27-31: 0 (reserved) + */ + +#ifndef Z_SOLO + + /* utility functions */ + +/* + The following utility functions are implemented on top of the basic + stream-oriented functions. To simplify the interface, some default options + are assumed (compression level and memory usage, standard memory allocation + functions). The source code of these utility functions can be modified if + you need special options. +*/ + +ZEXTERN int ZEXPORT compress OF((Bytef *dest, uLongf *destLen, + const Bytef *source, uLong sourceLen)); +/* + Compresses the source buffer into the destination buffer. sourceLen is + the byte length of the source buffer. Upon entry, destLen is the total size + of the destination buffer, which must be at least the value returned by + compressBound(sourceLen). Upon exit, destLen is the actual size of the + compressed data. compress() is equivalent to compress2() with a level + parameter of Z_DEFAULT_COMPRESSION. + + compress returns Z_OK if success, Z_MEM_ERROR if there was not + enough memory, Z_BUF_ERROR if there was not enough room in the output + buffer. +*/ + +ZEXTERN int ZEXPORT compress2 OF((Bytef *dest, uLongf *destLen, + const Bytef *source, uLong sourceLen, + int level)); +/* + Compresses the source buffer into the destination buffer. The level + parameter has the same meaning as in deflateInit. sourceLen is the byte + length of the source buffer. Upon entry, destLen is the total size of the + destination buffer, which must be at least the value returned by + compressBound(sourceLen). Upon exit, destLen is the actual size of the + compressed data. + + compress2 returns Z_OK if success, Z_MEM_ERROR if there was not enough + memory, Z_BUF_ERROR if there was not enough room in the output buffer, + Z_STREAM_ERROR if the level parameter is invalid. +*/ + +ZEXTERN uLong ZEXPORT compressBound OF((uLong sourceLen)); +/* + compressBound() returns an upper bound on the compressed size after + compress() or compress2() on sourceLen bytes. It would be used before a + compress() or compress2() call to allocate the destination buffer. +*/ + +ZEXTERN int ZEXPORT uncompress OF((Bytef *dest, uLongf *destLen, + const Bytef *source, uLong sourceLen)); +/* + Decompresses the source buffer into the destination buffer. sourceLen is + the byte length of the source buffer. Upon entry, destLen is the total size + of the destination buffer, which must be large enough to hold the entire + uncompressed data. (The size of the uncompressed data must have been saved + previously by the compressor and transmitted to the decompressor by some + mechanism outside the scope of this compression library.) Upon exit, destLen + is the actual size of the uncompressed data. + + uncompress returns Z_OK if success, Z_MEM_ERROR if there was not + enough memory, Z_BUF_ERROR if there was not enough room in the output + buffer, or Z_DATA_ERROR if the input data was corrupted or incomplete. In + the case where there is not enough room, uncompress() will fill the output + buffer with the uncompressed data up to that point. +*/ + +#if !defined(__ANDROID__) || __ANDROID_API__ >= 28 +ZEXTERN int ZEXPORT uncompress2 OF((Bytef *dest, uLongf *destLen, + const Bytef *source, uLong *sourceLen)); +#endif +/* + Same as uncompress, except that sourceLen is a pointer, where the + length of the source is *sourceLen. On return, *sourceLen is the number of + source bytes consumed. +*/ + + /* gzip file access functions */ + +/* + This library supports reading and writing files in gzip (.gz) format with + an interface similar to that of stdio, using the functions that start with + "gz". The gzip format is different from the zlib format. gzip is a gzip + wrapper, documented in RFC 1952, wrapped around a deflate stream. +*/ + +typedef struct gzFile_s *gzFile; /* semi-opaque gzip file descriptor */ + +/* +ZEXTERN gzFile ZEXPORT gzopen OF((const char *path, const char *mode)); + + Opens a gzip (.gz) file for reading or writing. The mode parameter is as + in fopen ("rb" or "wb") but can also include a compression level ("wb9") or + a strategy: 'f' for filtered data as in "wb6f", 'h' for Huffman-only + compression as in "wb1h", 'R' for run-length encoding as in "wb1R", or 'F' + for fixed code compression as in "wb9F". (See the description of + deflateInit2 for more information about the strategy parameter.) 'T' will + request transparent writing or appending with no compression and not using + the gzip format. + + "a" can be used instead of "w" to request that the gzip stream that will + be written be appended to the file. "+" will result in an error, since + reading and writing to the same gzip file is not supported. The addition of + "x" when writing will create the file exclusively, which fails if the file + already exists. On systems that support it, the addition of "e" when + reading or writing will set the flag to close the file on an execve() call. + + These functions, as well as gzip, will read and decode a sequence of gzip + streams in a file. The append function of gzopen() can be used to create + such a file. (Also see gzflush() for another way to do this.) When + appending, gzopen does not test whether the file begins with a gzip stream, + nor does it look for the end of the gzip streams to begin appending. gzopen + will simply append a gzip stream to the existing file. + + gzopen can be used to read a file which is not in gzip format; in this + case gzread will directly read from the file without decompression. When + reading, this will be detected automatically by looking for the magic two- + byte gzip header. + + gzopen returns NULL if the file could not be opened, if there was + insufficient memory to allocate the gzFile state, or if an invalid mode was + specified (an 'r', 'w', or 'a' was not provided, or '+' was provided). + errno can be checked to determine if the reason gzopen failed was that the + file could not be opened. +*/ + +ZEXTERN gzFile ZEXPORT gzdopen OF((int fd, const char *mode)); +/* + gzdopen associates a gzFile with the file descriptor fd. File descriptors + are obtained from calls like open, dup, creat, pipe or fileno (if the file + has been previously opened with fopen). The mode parameter is as in gzopen. + + The next call of gzclose on the returned gzFile will also close the file + descriptor fd, just like fclose(fdopen(fd, mode)) closes the file descriptor + fd. If you want to keep fd open, use fd = dup(fd_keep); gz = gzdopen(fd, + mode);. The duplicated descriptor should be saved to avoid a leak, since + gzdopen does not close fd if it fails. If you are using fileno() to get the + file descriptor from a FILE *, then you will have to use dup() to avoid + double-close()ing the file descriptor. Both gzclose() and fclose() will + close the associated file descriptor, so they need to have different file + descriptors. + + gzdopen returns NULL if there was insufficient memory to allocate the + gzFile state, if an invalid mode was specified (an 'r', 'w', or 'a' was not + provided, or '+' was provided), or if fd is -1. The file descriptor is not + used until the next gz* read, write, seek, or close operation, so gzdopen + will not detect if fd is invalid (unless fd is -1). +*/ + +ZEXTERN int ZEXPORT gzbuffer OF((gzFile file, unsigned size)); +/* + Set the internal buffer size used by this library's functions. The + default buffer size is 8192 bytes. This function must be called after + gzopen() or gzdopen(), and before any other calls that read or write the + file. The buffer memory allocation is always deferred to the first read or + write. Three times that size in buffer space is allocated. A larger buffer + size of, for example, 64K or 128K bytes will noticeably increase the speed + of decompression (reading). + + The new buffer size also affects the maximum length for gzprintf(). + + gzbuffer() returns 0 on success, or -1 on failure, such as being called + too late. +*/ + +ZEXTERN int ZEXPORT gzsetparams OF((gzFile file, int level, int strategy)); +/* + Dynamically update the compression level or strategy. See the description + of deflateInit2 for the meaning of these parameters. Previously provided + data is flushed before the parameter change. + + gzsetparams returns Z_OK if success, Z_STREAM_ERROR if the file was not + opened for writing, Z_ERRNO if there is an error writing the flushed data, + or Z_MEM_ERROR if there is a memory allocation error. +*/ + +ZEXTERN int ZEXPORT gzread OF((gzFile file, voidp buf, unsigned len)); +/* + Reads the given number of uncompressed bytes from the compressed file. If + the input file is not in gzip format, gzread copies the given number of + bytes into the buffer directly from the file. + + After reaching the end of a gzip stream in the input, gzread will continue + to read, looking for another gzip stream. Any number of gzip streams may be + concatenated in the input file, and will all be decompressed by gzread(). + If something other than a gzip stream is encountered after a gzip stream, + that remaining trailing garbage is ignored (and no error is returned). + + gzread can be used to read a gzip file that is being concurrently written. + Upon reaching the end of the input, gzread will return with the available + data. If the error code returned by gzerror is Z_OK or Z_BUF_ERROR, then + gzclearerr can be used to clear the end of file indicator in order to permit + gzread to be tried again. Z_OK indicates that a gzip stream was completed + on the last gzread. Z_BUF_ERROR indicates that the input file ended in the + middle of a gzip stream. Note that gzread does not return -1 in the event + of an incomplete gzip stream. This error is deferred until gzclose(), which + will return Z_BUF_ERROR if the last gzread ended in the middle of a gzip + stream. Alternatively, gzerror can be used before gzclose to detect this + case. + + gzread returns the number of uncompressed bytes actually read, less than + len for end of file, or -1 for error. If len is too large to fit in an int, + then nothing is read, -1 is returned, and the error state is set to + Z_STREAM_ERROR. +*/ + +#if !defined(__ANDROID__) || __ANDROID_API__ >= 28 +ZEXTERN z_size_t ZEXPORT gzfread OF((voidp buf, z_size_t size, z_size_t nitems, + gzFile file)); +#endif +/* + Read up to nitems items of size size from file to buf, otherwise operating + as gzread() does. This duplicates the interface of stdio's fread(), with + size_t request and return types. If the library defines size_t, then + z_size_t is identical to size_t. If not, then z_size_t is an unsigned + integer type that can contain a pointer. + + gzfread() returns the number of full items read of size size, or zero if + the end of the file was reached and a full item could not be read, or if + there was an error. gzerror() must be consulted if zero is returned in + order to determine if there was an error. If the multiplication of size and + nitems overflows, i.e. the product does not fit in a z_size_t, then nothing + is read, zero is returned, and the error state is set to Z_STREAM_ERROR. + + In the event that the end of file is reached and only a partial item is + available at the end, i.e. the remaining uncompressed data length is not a + multiple of size, then the final partial item is nevetheless read into buf + and the end-of-file flag is set. The length of the partial item read is not + provided, but could be inferred from the result of gztell(). This behavior + is the same as the behavior of fread() implementations in common libraries, + but it prevents the direct use of gzfread() to read a concurrently written + file, reseting and retrying on end-of-file, when size is not 1. +*/ + +ZEXTERN int ZEXPORT gzwrite OF((gzFile file, + voidpc buf, unsigned len)); +/* + Writes the given number of uncompressed bytes into the compressed file. + gzwrite returns the number of uncompressed bytes written or 0 in case of + error. +*/ + +#if !defined(__ANDROID__) || __ANDROID_API__ >= 28 +ZEXTERN z_size_t ZEXPORT gzfwrite OF((voidpc buf, z_size_t size, + z_size_t nitems, gzFile file)); +#endif +/* + gzfwrite() writes nitems items of size size from buf to file, duplicating + the interface of stdio's fwrite(), with size_t request and return types. If + the library defines size_t, then z_size_t is identical to size_t. If not, + then z_size_t is an unsigned integer type that can contain a pointer. + + gzfwrite() returns the number of full items written of size size, or zero + if there was an error. If the multiplication of size and nitems overflows, + i.e. the product does not fit in a z_size_t, then nothing is written, zero + is returned, and the error state is set to Z_STREAM_ERROR. +*/ + +ZEXTERN int ZEXPORTVA gzprintf Z_ARG((gzFile file, const char *format, ...)); +/* + Converts, formats, and writes the arguments to the compressed file under + control of the format string, as in fprintf. gzprintf returns the number of + uncompressed bytes actually written, or a negative zlib error code in case + of error. The number of uncompressed bytes written is limited to 8191, or + one less than the buffer size given to gzbuffer(). The caller should assure + that this limit is not exceeded. If it is exceeded, then gzprintf() will + return an error (0) with nothing written. In this case, there may also be a + buffer overflow with unpredictable consequences, which is possible only if + zlib was compiled with the insecure functions sprintf() or vsprintf() + because the secure snprintf() or vsnprintf() functions were not available. + This can be determined using zlibCompileFlags(). +*/ + +ZEXTERN int ZEXPORT gzputs OF((gzFile file, const char *s)); +/* + Writes the given null-terminated string to the compressed file, excluding + the terminating null character. + + gzputs returns the number of characters written, or -1 in case of error. +*/ + +ZEXTERN char * ZEXPORT gzgets OF((gzFile file, char *buf, int len)); +/* + Reads bytes from the compressed file until len-1 characters are read, or a + newline character is read and transferred to buf, or an end-of-file + condition is encountered. If any characters are read or if len == 1, the + string is terminated with a null character. If no characters are read due + to an end-of-file or len < 1, then the buffer is left untouched. + + gzgets returns buf which is a null-terminated string, or it returns NULL + for end-of-file or in case of error. If there was an error, the contents at + buf are indeterminate. +*/ + +ZEXTERN int ZEXPORT gzputc OF((gzFile file, int c)); +/* + Writes c, converted to an unsigned char, into the compressed file. gzputc + returns the value that was written, or -1 in case of error. +*/ + +ZEXTERN int ZEXPORT gzgetc OF((gzFile file)); +/* + Reads one byte from the compressed file. gzgetc returns this byte or -1 + in case of end of file or error. This is implemented as a macro for speed. + As such, it does not do all of the checking the other functions do. I.e. + it does not check to see if file is NULL, nor whether the structure file + points to has been clobbered or not. +*/ + +ZEXTERN int ZEXPORT gzungetc OF((int c, gzFile file)); +/* + Push one character back onto the stream to be read as the first character + on the next read. At least one character of push-back is allowed. + gzungetc() returns the character pushed, or -1 on failure. gzungetc() will + fail if c is -1, and may fail if a character has been pushed but not read + yet. If gzungetc is used immediately after gzopen or gzdopen, at least the + output buffer size of pushed characters is allowed. (See gzbuffer above.) + The pushed character will be discarded if the stream is repositioned with + gzseek() or gzrewind(). +*/ + +ZEXTERN int ZEXPORT gzflush OF((gzFile file, int flush)); +/* + Flushes all pending output into the compressed file. The parameter flush + is as in the deflate() function. The return value is the zlib error number + (see function gzerror below). gzflush is only permitted when writing. + + If the flush parameter is Z_FINISH, the remaining data is written and the + gzip stream is completed in the output. If gzwrite() is called again, a new + gzip stream will be started in the output. gzread() is able to read such + concatenated gzip streams. + + gzflush should be called only when strictly necessary because it will + degrade compression if called too often. +*/ + +/* +ZEXTERN z_off_t ZEXPORT gzseek OF((gzFile file, + z_off_t offset, int whence)); + + Sets the starting position for the next gzread or gzwrite on the given + compressed file. The offset represents a number of bytes in the + uncompressed data stream. The whence parameter is defined as in lseek(2); + the value SEEK_END is not supported. + + If the file is opened for reading, this function is emulated but can be + extremely slow. If the file is opened for writing, only forward seeks are + supported; gzseek then compresses a sequence of zeroes up to the new + starting position. + + gzseek returns the resulting offset location as measured in bytes from + the beginning of the uncompressed stream, or -1 in case of error, in + particular if the file is opened for writing and the new starting position + would be before the current position. +*/ + +ZEXTERN int ZEXPORT gzrewind OF((gzFile file)); +/* + Rewinds the given file. This function is supported only for reading. + + gzrewind(file) is equivalent to (int)gzseek(file, 0L, SEEK_SET) +*/ + +/* +ZEXTERN z_off_t ZEXPORT gztell OF((gzFile file)); + + Returns the starting position for the next gzread or gzwrite on the given + compressed file. This position represents a number of bytes in the + uncompressed data stream, and is zero when starting, even if appending or + reading a gzip stream from the middle of a file using gzdopen(). + + gztell(file) is equivalent to gzseek(file, 0L, SEEK_CUR) +*/ + +/* +ZEXTERN z_off_t ZEXPORT gzoffset OF((gzFile file)); + + Returns the current offset in the file being read or written. This offset + includes the count of bytes that precede the gzip stream, for example when + appending or when using gzdopen() for reading. When reading, the offset + does not include as yet unused buffered input. This information can be used + for a progress indicator. On error, gzoffset() returns -1. +*/ + +ZEXTERN int ZEXPORT gzeof OF((gzFile file)); +/* + Returns true (1) if the end-of-file indicator has been set while reading, + false (0) otherwise. Note that the end-of-file indicator is set only if the + read tried to go past the end of the input, but came up short. Therefore, + just like feof(), gzeof() may return false even if there is no more data to + read, in the event that the last read request was for the exact number of + bytes remaining in the input file. This will happen if the input file size + is an exact multiple of the buffer size. + + If gzeof() returns true, then the read functions will return no more data, + unless the end-of-file indicator is reset by gzclearerr() and the input file + has grown since the previous end of file was detected. +*/ + +ZEXTERN int ZEXPORT gzdirect OF((gzFile file)); +/* + Returns true (1) if file is being copied directly while reading, or false + (0) if file is a gzip stream being decompressed. + + If the input file is empty, gzdirect() will return true, since the input + does not contain a gzip stream. + + If gzdirect() is used immediately after gzopen() or gzdopen() it will + cause buffers to be allocated to allow reading the file to determine if it + is a gzip file. Therefore if gzbuffer() is used, it should be called before + gzdirect(). + + When writing, gzdirect() returns true (1) if transparent writing was + requested ("wT" for the gzopen() mode), or false (0) otherwise. (Note: + gzdirect() is not needed when writing. Transparent writing must be + explicitly requested, so the application already knows the answer. When + linking statically, using gzdirect() will include all of the zlib code for + gzip file reading and decompression, which may not be desired.) +*/ + +ZEXTERN int ZEXPORT gzclose OF((gzFile file)); +/* + Flushes all pending output if necessary, closes the compressed file and + deallocates the (de)compression state. Note that once file is closed, you + cannot call gzerror with file, since its structures have been deallocated. + gzclose must not be called more than once on the same file, just as free + must not be called more than once on the same allocation. + + gzclose will return Z_STREAM_ERROR if file is not valid, Z_ERRNO on a + file operation error, Z_MEM_ERROR if out of memory, Z_BUF_ERROR if the + last read ended in the middle of a gzip stream, or Z_OK on success. +*/ + +ZEXTERN int ZEXPORT gzclose_r OF((gzFile file)); +ZEXTERN int ZEXPORT gzclose_w OF((gzFile file)); +/* + Same as gzclose(), but gzclose_r() is only for use when reading, and + gzclose_w() is only for use when writing or appending. The advantage to + using these instead of gzclose() is that they avoid linking in zlib + compression or decompression code that is not used when only reading or only + writing respectively. If gzclose() is used, then both compression and + decompression code will be included the application when linking to a static + zlib library. +*/ + +ZEXTERN const char * ZEXPORT gzerror OF((gzFile file, int *errnum)); +/* + Returns the error message for the last error which occurred on the given + compressed file. errnum is set to zlib error number. If an error occurred + in the file system and not in the compression library, errnum is set to + Z_ERRNO and the application may consult errno to get the exact error code. + + The application must not modify the returned string. Future calls to + this function may invalidate the previously returned string. If file is + closed, then the string previously returned by gzerror will no longer be + available. + + gzerror() should be used to distinguish errors from end-of-file for those + functions above that do not distinguish those cases in their return values. +*/ + +ZEXTERN void ZEXPORT gzclearerr OF((gzFile file)); +/* + Clears the error and end-of-file flags for file. This is analogous to the + clearerr() function in stdio. This is useful for continuing to read a gzip + file that is being written concurrently. +*/ + +#endif /* !Z_SOLO */ + + /* checksum functions */ + +/* + These functions are not related to compression but are exported + anyway because they might be useful in applications using the compression + library. +*/ + +ZEXTERN uLong ZEXPORT adler32 OF((uLong adler, const Bytef *buf, uInt len)); +/* + Update a running Adler-32 checksum with the bytes buf[0..len-1] and + return the updated checksum. If buf is Z_NULL, this function returns the + required initial value for the checksum. + + An Adler-32 checksum is almost as reliable as a CRC-32 but can be computed + much faster. + + Usage example: + + uLong adler = adler32(0L, Z_NULL, 0); + + while (read_buffer(buffer, length) != EOF) { + adler = adler32(adler, buffer, length); + } + if (adler != original_adler) error(); +*/ + +#if !defined(__ANDROID__) || __ANDROID_API__ >= 28 +ZEXTERN uLong ZEXPORT adler32_z OF((uLong adler, const Bytef *buf, + z_size_t len)); +#endif +/* + Same as adler32(), but with a size_t length. +*/ + +/* +ZEXTERN uLong ZEXPORT adler32_combine OF((uLong adler1, uLong adler2, + z_off_t len2)); + + Combine two Adler-32 checksums into one. For two sequences of bytes, seq1 + and seq2 with lengths len1 and len2, Adler-32 checksums were calculated for + each, adler1 and adler2. adler32_combine() returns the Adler-32 checksum of + seq1 and seq2 concatenated, requiring only adler1, adler2, and len2. Note + that the z_off_t type (like off_t) is a signed integer. If len2 is + negative, the result has no meaning or utility. +*/ + +ZEXTERN uLong ZEXPORT crc32 OF((uLong crc, const Bytef *buf, uInt len)); +/* + Update a running CRC-32 with the bytes buf[0..len-1] and return the + updated CRC-32. If buf is Z_NULL, this function returns the required + initial value for the crc. Pre- and post-conditioning (one's complement) is + performed within this function so it shouldn't be done by the application. + + Usage example: + + uLong crc = crc32(0L, Z_NULL, 0); + + while (read_buffer(buffer, length) != EOF) { + crc = crc32(crc, buffer, length); + } + if (crc != original_crc) error(); +*/ + +#if !defined(__ANDROID__) || __ANDROID_API__ >= 28 +ZEXTERN uLong ZEXPORT crc32_z OF((uLong adler, const Bytef *buf, + z_size_t len)); +#endif +/* + Same as crc32(), but with a size_t length. +*/ + +/* +ZEXTERN uLong ZEXPORT crc32_combine OF((uLong crc1, uLong crc2, z_off_t len2)); + + Combine two CRC-32 check values into one. For two sequences of bytes, + seq1 and seq2 with lengths len1 and len2, CRC-32 check values were + calculated for each, crc1 and crc2. crc32_combine() returns the CRC-32 + check value of seq1 and seq2 concatenated, requiring only crc1, crc2, and + len2. +*/ + + + /* various hacks, don't look :) */ + +/* deflateInit and inflateInit are macros to allow checking the zlib version + * and the compiler's view of z_stream: + */ +ZEXTERN int ZEXPORT deflateInit_ OF((z_streamp strm, int level, + const char *version, int stream_size)); +ZEXTERN int ZEXPORT inflateInit_ OF((z_streamp strm, + const char *version, int stream_size)); +ZEXTERN int ZEXPORT deflateInit2_ OF((z_streamp strm, int level, int method, + int windowBits, int memLevel, + int strategy, const char *version, + int stream_size)); +ZEXTERN int ZEXPORT inflateInit2_ OF((z_streamp strm, int windowBits, + const char *version, int stream_size)); +ZEXTERN int ZEXPORT inflateBackInit_ OF((z_streamp strm, int windowBits, + unsigned char FAR *window, + const char *version, + int stream_size)); +#ifdef Z_PREFIX_SET +# define z_deflateInit(strm, level) \ + deflateInit_((strm), (level), ZLIB_VERSION, (int)sizeof(z_stream)) +# define z_inflateInit(strm) \ + inflateInit_((strm), ZLIB_VERSION, (int)sizeof(z_stream)) +# define z_deflateInit2(strm, level, method, windowBits, memLevel, strategy) \ + deflateInit2_((strm),(level),(method),(windowBits),(memLevel),\ + (strategy), ZLIB_VERSION, (int)sizeof(z_stream)) +# define z_inflateInit2(strm, windowBits) \ + inflateInit2_((strm), (windowBits), ZLIB_VERSION, \ + (int)sizeof(z_stream)) +# define z_inflateBackInit(strm, windowBits, window) \ + inflateBackInit_((strm), (windowBits), (window), \ + ZLIB_VERSION, (int)sizeof(z_stream)) +#else +# define deflateInit(strm, level) \ + deflateInit_((strm), (level), ZLIB_VERSION, (int)sizeof(z_stream)) +# define inflateInit(strm) \ + inflateInit_((strm), ZLIB_VERSION, (int)sizeof(z_stream)) +# define deflateInit2(strm, level, method, windowBits, memLevel, strategy) \ + deflateInit2_((strm),(level),(method),(windowBits),(memLevel),\ + (strategy), ZLIB_VERSION, (int)sizeof(z_stream)) +# define inflateInit2(strm, windowBits) \ + inflateInit2_((strm), (windowBits), ZLIB_VERSION, \ + (int)sizeof(z_stream)) +# define inflateBackInit(strm, windowBits, window) \ + inflateBackInit_((strm), (windowBits), (window), \ + ZLIB_VERSION, (int)sizeof(z_stream)) +#endif + +#ifndef Z_SOLO + +/* gzgetc() macro and its supporting function and exposed data structure. Note + * that the real internal state is much larger than the exposed structure. + * This abbreviated structure exposes just enough for the gzgetc() macro. The + * user should not mess with these exposed elements, since their names or + * behavior could change in the future, perhaps even capriciously. They can + * only be used by the gzgetc() macro. You have been warned. + */ +struct gzFile_s { + unsigned have; + unsigned char *next; + z_off64_t pos; +}; +ZEXTERN int ZEXPORT gzgetc_ OF((gzFile file)); /* backward compatibility */ +#ifdef Z_PREFIX_SET +# undef z_gzgetc +# define z_gzgetc(g) \ + ((g)->have ? ((g)->have--, (g)->pos++, *((g)->next)++) : (gzgetc)(g)) +#elif defined(Z_CR_PREFIX_SET) +# undef gzgetc +# define gzgetc(g) \ + ((g)->have ? ((g)->have--, (g)->pos++, *((g)->next)++) \ + : (Cr_z_gzgetc)(g)) +#else +# define gzgetc(g) \ + ((g)->have ? ((g)->have--, (g)->pos++, *((g)->next)++) : (gzgetc)(g)) +#endif + +/* provide 64-bit offset functions if _LARGEFILE64_SOURCE defined, and/or + * change the regular functions to 64 bits if _FILE_OFFSET_BITS is 64 (if + * both are true, the application gets the *64 functions, and the regular + * functions are changed to 64 bits) -- in case these are set on systems + * without large file support, _LFS64_LARGEFILE must also be true + */ +#ifdef Z_LARGE64 + ZEXTERN gzFile ZEXPORT gzopen64 OF((const char *, const char *)); + ZEXTERN z_off64_t ZEXPORT gzseek64 OF((gzFile, z_off64_t, int)); + ZEXTERN z_off64_t ZEXPORT gztell64 OF((gzFile)); + ZEXTERN z_off64_t ZEXPORT gzoffset64 OF((gzFile)); + ZEXTERN uLong ZEXPORT adler32_combine64 OF((uLong, uLong, z_off64_t)); + ZEXTERN uLong ZEXPORT crc32_combine64 OF((uLong, uLong, z_off64_t)); +#endif + +#if !defined(ZLIB_INTERNAL) && defined(Z_WANT64) +# ifdef Z_PREFIX_SET +# define z_gzopen z_gzopen64 +# define z_gzseek z_gzseek64 +# define z_gztell z_gztell64 +# define z_gzoffset z_gzoffset64 +# define z_adler32_combine z_adler32_combine64 +# define z_crc32_combine z_crc32_combine64 +# else +# ifdef gzopen +# undef gzopen +# endif +# define gzopen gzopen64 +# ifdef gzseek +# undef gzseek +# endif +# define gzseek gzseek64 +# ifdef gztell +# undef gztell +# endif +# define gztell gztell64 +# ifdef gzoffset +# undef gzoffset +# endif +# define gzoffset gzoffset64 +# ifdef adler32_combine +# undef adler32_combine +# endif +# define adler32_combine adler32_combine64 +# ifdef crc32_combine +# undef crc32_combine +# endif +# define crc32_combine crc32_combine64 +# endif +# ifndef Z_LARGE64 + ZEXTERN gzFile ZEXPORT gzopen64 OF((const char *, const char *)); + ZEXTERN z_off_t ZEXPORT gzseek64 OF((gzFile, z_off_t, int)); + ZEXTERN z_off_t ZEXPORT gztell64 OF((gzFile)); + ZEXTERN z_off_t ZEXPORT gzoffset64 OF((gzFile)); + ZEXTERN uLong ZEXPORT adler32_combine64 OF((uLong, uLong, z_off_t)); + ZEXTERN uLong ZEXPORT crc32_combine64 OF((uLong, uLong, z_off_t)); +# endif +#else + ZEXTERN gzFile ZEXPORT gzopen OF((const char *, const char *)); + ZEXTERN z_off_t ZEXPORT gzseek OF((gzFile, z_off_t, int)); + ZEXTERN z_off_t ZEXPORT gztell OF((gzFile)); + ZEXTERN z_off_t ZEXPORT gzoffset OF((gzFile)); + ZEXTERN uLong ZEXPORT adler32_combine OF((uLong, uLong, z_off_t)); + ZEXTERN uLong ZEXPORT crc32_combine OF((uLong, uLong, z_off_t)); +#endif + +#else /* Z_SOLO */ + + ZEXTERN uLong ZEXPORT adler32_combine OF((uLong, uLong, z_off_t)); + ZEXTERN uLong ZEXPORT crc32_combine OF((uLong, uLong, z_off_t)); + +#endif /* !Z_SOLO */ + +/* undocumented functions */ +ZEXTERN const char * ZEXPORT zError OF((int)); +ZEXTERN int ZEXPORT inflateSyncPoint OF((z_streamp)); +ZEXTERN const z_crc_t FAR * ZEXPORT get_crc_table OF((void)); +ZEXTERN int ZEXPORT inflateUndermine OF((z_streamp, int)); +#if !defined(__ANDROID__) || __ANDROID_API__ >= 28 +ZEXTERN int ZEXPORT inflateValidate OF((z_streamp, int)); +#endif +#if !defined(__ANDROID__) || __ANDROID_API__ >= 28 +ZEXTERN unsigned long ZEXPORT inflateCodesUsed OF ((z_streamp)); +#endif +ZEXTERN int ZEXPORT inflateResetKeep OF((z_streamp)); +ZEXTERN int ZEXPORT deflateResetKeep OF((z_streamp)); +#if (defined(_WIN32) || defined(__CYGWIN__)) && !defined(Z_SOLO) +ZEXTERN gzFile ZEXPORT gzopen_w OF((const wchar_t *path, + const char *mode)); +#endif +#if defined(STDC) || defined(Z_HAVE_STDARG_H) +# ifndef Z_SOLO +# if !defined(__ANDROID__) || __ANDROID_API__ >= 19 +ZEXTERN int ZEXPORTVA gzvprintf Z_ARG((gzFile file, + const char *format, + va_list va)); +# endif +# endif +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* ZLIB_H */ diff --git a/third_party/zlib/zutil.c b/third_party/zlib/zutil.c new file mode 100644 index 00000000..a76c6b0c --- /dev/null +++ b/third_party/zlib/zutil.c @@ -0,0 +1,325 @@ +/* zutil.c -- target dependent utility functions for the compression library + * Copyright (C) 1995-2017 Jean-loup Gailly + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +/* @(#) $Id$ */ + +#include "zutil.h" +#ifndef Z_SOLO +# include "gzguts.h" +#endif + +z_const char * const z_errmsg[10] = { + (z_const char *)"need dictionary", /* Z_NEED_DICT 2 */ + (z_const char *)"stream end", /* Z_STREAM_END 1 */ + (z_const char *)"", /* Z_OK 0 */ + (z_const char *)"file error", /* Z_ERRNO (-1) */ + (z_const char *)"stream error", /* Z_STREAM_ERROR (-2) */ + (z_const char *)"data error", /* Z_DATA_ERROR (-3) */ + (z_const char *)"insufficient memory", /* Z_MEM_ERROR (-4) */ + (z_const char *)"buffer error", /* Z_BUF_ERROR (-5) */ + (z_const char *)"incompatible version",/* Z_VERSION_ERROR (-6) */ + (z_const char *)"" +}; + + +const char * ZEXPORT zlibVersion() +{ + return ZLIB_VERSION; +} + +uLong ZEXPORT zlibCompileFlags() +{ + uLong flags; + + flags = 0; + switch ((int)(sizeof(uInt))) { + case 2: break; + case 4: flags += 1; break; + case 8: flags += 2; break; + default: flags += 3; + } + switch ((int)(sizeof(uLong))) { + case 2: break; + case 4: flags += 1 << 2; break; + case 8: flags += 2 << 2; break; + default: flags += 3 << 2; + } + switch ((int)(sizeof(voidpf))) { + case 2: break; + case 4: flags += 1 << 4; break; + case 8: flags += 2 << 4; break; + default: flags += 3 << 4; + } + switch ((int)(sizeof(z_off_t))) { + case 2: break; + case 4: flags += 1 << 6; break; + case 8: flags += 2 << 6; break; + default: flags += 3 << 6; + } +#ifdef ZLIB_DEBUG + flags += 1 << 8; +#endif +#if defined(ASMV) || defined(ASMINF) + flags += 1 << 9; +#endif +#ifdef ZLIB_WINAPI + flags += 1 << 10; +#endif +#ifdef BUILDFIXED + flags += 1 << 12; +#endif +#ifdef DYNAMIC_CRC_TABLE + flags += 1 << 13; +#endif +#ifdef NO_GZCOMPRESS + flags += 1L << 16; +#endif +#ifdef NO_GZIP + flags += 1L << 17; +#endif +#ifdef PKZIP_BUG_WORKAROUND + flags += 1L << 20; +#endif +#ifdef FASTEST + flags += 1L << 21; +#endif +#if defined(STDC) || defined(Z_HAVE_STDARG_H) +# ifdef NO_vsnprintf + flags += 1L << 25; +# ifdef HAS_vsprintf_void + flags += 1L << 26; +# endif +# else +# ifdef HAS_vsnprintf_void + flags += 1L << 26; +# endif +# endif +#else + flags += 1L << 24; +# ifdef NO_snprintf + flags += 1L << 25; +# ifdef HAS_sprintf_void + flags += 1L << 26; +# endif +# else +# ifdef HAS_snprintf_void + flags += 1L << 26; +# endif +# endif +#endif + return flags; +} + +#ifdef ZLIB_DEBUG +#include <stdlib.h> +# ifndef verbose +# define verbose 0 +# endif +int ZLIB_INTERNAL z_verbose = verbose; + +void ZLIB_INTERNAL z_error (m) + char *m; +{ + fprintf(stderr, "%s\n", m); + exit(1); +} +#endif + +/* exported to allow conversion of error code to string for compress() and + * uncompress() + */ +const char * ZEXPORT zError(err) + int err; +{ + return ERR_MSG(err); +} + +#if defined(_WIN32_WCE) + /* The Microsoft C Run-Time Library for Windows CE doesn't have + * errno. We define it as a global variable to simplify porting. + * Its value is always 0 and should not be used. + */ + int errno = 0; +#endif + +#ifndef HAVE_MEMCPY + +void ZLIB_INTERNAL zmemcpy(dest, source, len) + Bytef* dest; + const Bytef* source; + uInt len; +{ + if (len == 0) return; + do { + *dest++ = *source++; /* ??? to be unrolled */ + } while (--len != 0); +} + +int ZLIB_INTERNAL zmemcmp(s1, s2, len) + const Bytef* s1; + const Bytef* s2; + uInt len; +{ + uInt j; + + for (j = 0; j < len; j++) { + if (s1[j] != s2[j]) return 2*(s1[j] > s2[j])-1; + } + return 0; +} + +void ZLIB_INTERNAL zmemzero(dest, len) + Bytef* dest; + uInt len; +{ + if (len == 0) return; + do { + *dest++ = 0; /* ??? to be unrolled */ + } while (--len != 0); +} +#endif + +#ifndef Z_SOLO + +#ifdef SYS16BIT + +#ifdef __TURBOC__ +/* Turbo C in 16-bit mode */ + +# define MY_ZCALLOC + +/* Turbo C malloc() does not allow dynamic allocation of 64K bytes + * and farmalloc(64K) returns a pointer with an offset of 8, so we + * must fix the pointer. Warning: the pointer must be put back to its + * original form in order to free it, use zcfree(). + */ + +#define MAX_PTR 10 +/* 10*64K = 640K */ + +local int next_ptr = 0; + +typedef struct ptr_table_s { + voidpf org_ptr; + voidpf new_ptr; +} ptr_table; + +local ptr_table table[MAX_PTR]; +/* This table is used to remember the original form of pointers + * to large buffers (64K). Such pointers are normalized with a zero offset. + * Since MSDOS is not a preemptive multitasking OS, this table is not + * protected from concurrent access. This hack doesn't work anyway on + * a protected system like OS/2. Use Microsoft C instead. + */ + +voidpf ZLIB_INTERNAL zcalloc (voidpf opaque, unsigned items, unsigned size) +{ + voidpf buf; + ulg bsize = (ulg)items*size; + + (void)opaque; + + /* If we allocate less than 65520 bytes, we assume that farmalloc + * will return a usable pointer which doesn't have to be normalized. + */ + if (bsize < 65520L) { + buf = farmalloc(bsize); + if (*(ush*)&buf != 0) return buf; + } else { + buf = farmalloc(bsize + 16L); + } + if (buf == NULL || next_ptr >= MAX_PTR) return NULL; + table[next_ptr].org_ptr = buf; + + /* Normalize the pointer to seg:0 */ + *((ush*)&buf+1) += ((ush)((uch*)buf-0) + 15) >> 4; + *(ush*)&buf = 0; + table[next_ptr++].new_ptr = buf; + return buf; +} + +void ZLIB_INTERNAL zcfree (voidpf opaque, voidpf ptr) +{ + int n; + + (void)opaque; + + if (*(ush*)&ptr != 0) { /* object < 64K */ + farfree(ptr); + return; + } + /* Find the original pointer */ + for (n = 0; n < next_ptr; n++) { + if (ptr != table[n].new_ptr) continue; + + farfree(table[n].org_ptr); + while (++n < next_ptr) { + table[n-1] = table[n]; + } + next_ptr--; + return; + } + Assert(0, "zcfree: ptr not found"); +} + +#endif /* __TURBOC__ */ + + +#ifdef M_I86 +/* Microsoft C in 16-bit mode */ + +# define MY_ZCALLOC + +#if (!defined(_MSC_VER) || (_MSC_VER <= 600)) +# define _halloc halloc +# define _hfree hfree +#endif + +voidpf ZLIB_INTERNAL zcalloc (voidpf opaque, uInt items, uInt size) +{ + (void)opaque; + return _halloc((long)items, size); +} + +void ZLIB_INTERNAL zcfree (voidpf opaque, voidpf ptr) +{ + (void)opaque; + _hfree(ptr); +} + +#endif /* M_I86 */ + +#endif /* SYS16BIT */ + + +#ifndef MY_ZCALLOC /* Any system without a special alloc function */ + +#ifndef STDC +extern voidp malloc OF((uInt size)); +extern voidp calloc OF((uInt items, uInt size)); +extern void free OF((voidpf ptr)); +#endif + +voidpf ZLIB_INTERNAL zcalloc (opaque, items, size) + voidpf opaque; + unsigned items; + unsigned size; +{ + (void)opaque; + return sizeof(uInt) > 2 ? (voidpf)malloc(items * size) : + (voidpf)calloc(items, size); +} + +void ZLIB_INTERNAL zcfree (opaque, ptr) + voidpf opaque; + voidpf ptr; +{ + (void)opaque; + free(ptr); +} + +#endif /* MY_ZCALLOC */ + +#endif /* !Z_SOLO */ diff --git a/third_party/zlib/zutil.h b/third_party/zlib/zutil.h new file mode 100644 index 00000000..4425bcf7 --- /dev/null +++ b/third_party/zlib/zutil.h @@ -0,0 +1,292 @@ +/* zutil.h -- internal interface and configuration of the compression library + * Copyright (C) 1995-2016 Jean-loup Gailly, Mark Adler + * For conditions of distribution and use, see copyright notice in zlib.h + */ + +/* WARNING: this file should *not* be used by applications. It is + part of the implementation of the compression library and is + subject to change. Applications should only use zlib.h. + */ + +/* @(#) $Id$ */ + +#ifndef ZUTIL_H +#define ZUTIL_H + +#ifdef HAVE_HIDDEN +# define ZLIB_INTERNAL __attribute__((visibility ("hidden"))) +#else +# define ZLIB_INTERNAL +#endif + +#include "zlib.h" + +#if defined(STDC) && !defined(Z_SOLO) +# if !(defined(_WIN32_WCE) && defined(_MSC_VER)) +# include <stddef.h> +# endif +# include <string.h> +# include <stdlib.h> +#endif +#ifdef NO_ERRNO_H +# ifdef _WIN32_WCE + /* The Microsoft C Run-Time Library for Windows CE doesn't have + * errno. We define it as a global variable to simplify porting. + * Its value is always 0 and should not be used. We rename it to + * avoid conflict with other libraries that use the same workaround. + */ +# define errno z_errno +# endif + extern int errno; +#else +# ifndef _WIN32_WCE +# include <errno.h> +# endif +#endif + +#ifdef Z_SOLO + typedef long ptrdiff_t; /* guess -- will be caught if guess is wrong */ +#endif + +#ifndef local +# define local static +#endif +/* since "static" is used to mean two completely different things in C, we + define "local" for the non-static meaning of "static", for readability + (compile with -Dlocal if your debugger can't find static symbols) */ + +typedef unsigned char uch; +typedef uch FAR uchf; +typedef unsigned short ush; +typedef ush FAR ushf; +typedef unsigned long ulg; + +extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ +/* (size given to avoid silly warnings with Visual C++) */ + +#define ERR_MSG(err) z_errmsg[Z_NEED_DICT-(err)] + +#define ERR_RETURN(strm,err) \ + return (strm->msg = ERR_MSG(err), (err)) +/* To be used only when the state is known to be valid */ + + /* common constants */ + +#ifndef DEF_WBITS +# define DEF_WBITS MAX_WBITS +#endif +/* default windowBits for decompression. MAX_WBITS is for compression only */ + +#if MAX_MEM_LEVEL >= 8 +# define DEF_MEM_LEVEL 8 +#else +# define DEF_MEM_LEVEL MAX_MEM_LEVEL +#endif +/* default memLevel */ + +#define STORED_BLOCK 0 +#define STATIC_TREES 1 +#define DYN_TREES 2 +/* The three kinds of block type */ + +#define MIN_MATCH 3 +#define MAX_MATCH 258 +/* The minimum and maximum match lengths */ + +#define PRESET_DICT 0x20 /* preset dictionary flag in zlib header */ + + /* target dependencies */ + +#if defined(MSDOS) || (defined(WINDOWS) && !defined(WIN32)) +# define OS_CODE 0x00 +# ifndef Z_SOLO +# if defined(__TURBOC__) || defined(__BORLANDC__) +# if (__STDC__ == 1) && (defined(__LARGE__) || defined(__COMPACT__)) + /* Allow compilation with ANSI keywords only enabled */ + void _Cdecl farfree( void *block ); + void *_Cdecl farmalloc( unsigned long nbytes ); +# else +# include <alloc.h> +# endif +# else /* MSC or DJGPP */ +# include <malloc.h> +# endif +# endif +#endif + +#ifdef AMIGA +# define OS_CODE 1 +#endif + +#if defined(VAXC) || defined(VMS) +# define OS_CODE 2 +# define F_OPEN(name, mode) \ + fopen((name), (mode), "mbc=60", "ctx=stm", "rfm=fix", "mrs=512") +#endif + +#ifdef __370__ +# if __TARGET_LIB__ < 0x20000000 +# define OS_CODE 4 +# elif __TARGET_LIB__ < 0x40000000 +# define OS_CODE 11 +# else +# define OS_CODE 8 +# endif +#endif + +#if defined(ATARI) || defined(atarist) +# define OS_CODE 5 +#endif + +#ifdef OS2 +# define OS_CODE 6 +# if defined(M_I86) && !defined(Z_SOLO) +# include <malloc.h> +# endif +#endif + +#if defined(MACOS) || defined(TARGET_OS_MAC) +# define OS_CODE 7 +# ifndef Z_SOLO +# if defined(__MWERKS__) && __dest_os != __be_os && __dest_os != __win32_os +# include <unix.h> /* for fdopen */ +# else +# ifndef fdopen +# define fdopen(fd,mode) NULL /* No fdopen() */ +# endif +# endif +# endif +#endif + +#ifdef __acorn +# define OS_CODE 13 +#endif + +#if defined(WIN32) && !defined(__CYGWIN__) +# define OS_CODE 10 +#endif + +#ifdef _BEOS_ +# define OS_CODE 16 +#endif + +#ifdef __TOS_OS400__ +# define OS_CODE 18 +#endif + +#ifdef __APPLE__ +# define OS_CODE 19 +#endif + +#if defined(_BEOS_) || defined(RISCOS) +# define fdopen(fd,mode) NULL /* No fdopen() */ +#endif + +#if (defined(_MSC_VER) && (_MSC_VER > 600)) && !defined __INTERIX +# if defined(_WIN32_WCE) +# define fdopen(fd,mode) NULL /* No fdopen() */ +# ifndef _PTRDIFF_T_DEFINED + typedef int ptrdiff_t; +# define _PTRDIFF_T_DEFINED +# endif +# else +# define fdopen(fd,type) _fdopen(fd,type) +# endif +#endif + +#if defined(__BORLANDC__) && !defined(MSDOS) + #pragma warn -8004 + #pragma warn -8008 + #pragma warn -8066 +#endif + +/* provide prototypes for these when building zlib without LFS */ +#if !defined(_WIN32) && \ + (!defined(_LARGEFILE64_SOURCE) || _LFS64_LARGEFILE-0 == 0) + ZEXTERN uLong ZEXPORT adler32_combine64 OF((uLong, uLong, z_off_t)); + ZEXTERN uLong ZEXPORT crc32_combine64 OF((uLong, uLong, z_off_t)); +#endif + + /* common defaults */ + +#ifndef OS_CODE +# define OS_CODE 3 /* assume Unix */ +#endif + +#ifndef F_OPEN +# define F_OPEN(name, mode) fopen((name), (mode)) +#endif + + /* functions */ + +#if defined(pyr) || defined(Z_SOLO) +# define NO_MEMCPY +#endif +#if defined(SMALL_MEDIUM) && !defined(_MSC_VER) && !defined(__SC__) + /* Use our own functions for small and medium model with MSC <= 5.0. + * You may have to use the same strategy for Borland C (untested). + * The __SC__ check is for Symantec. + */ +# define NO_MEMCPY +#endif +#if defined(STDC) && !defined(HAVE_MEMCPY) && !defined(NO_MEMCPY) +# define HAVE_MEMCPY +#endif +#ifdef HAVE_MEMCPY +# ifdef SMALL_MEDIUM /* MSDOS small or medium model */ +# define zmemcpy _fmemcpy +# define zmemcmp _fmemcmp +# define zmemzero(dest, len) _fmemset(dest, 0, len) +# else +# define zmemcpy memcpy +# define zmemcmp memcmp +# define zmemzero(dest, len) memset(dest, 0, len) +# endif +#else + void ZLIB_INTERNAL zmemcpy OF((Bytef* dest, const Bytef* source, uInt len)); + int ZLIB_INTERNAL zmemcmp OF((const Bytef* s1, const Bytef* s2, uInt len)); + void ZLIB_INTERNAL zmemzero OF((Bytef* dest, uInt len)); +#endif + +/* Diagnostic functions */ +#ifdef ZLIB_DEBUG +# include <stdio.h> + extern int ZLIB_INTERNAL z_verbose; + extern void ZLIB_INTERNAL z_error OF((char *m)); +# define Assert(cond,msg) {if(!(cond)) z_error(msg);} +# define Trace(x) {if (z_verbose>=0) fprintf x ;} +# define Tracev(x) {if (z_verbose>0) fprintf x ;} +# define Tracevv(x) {if (z_verbose>1) fprintf x ;} +# define Tracec(c,x) {if (z_verbose>0 && (c)) fprintf x ;} +# define Tracecv(c,x) {if (z_verbose>1 && (c)) fprintf x ;} +#else +# define Assert(cond,msg) +# define Trace(x) +# define Tracev(x) +# define Tracevv(x) +# define Tracec(c,x) +# define Tracecv(c,x) +#endif + +#ifndef Z_SOLO + voidpf ZLIB_INTERNAL zcalloc OF((voidpf opaque, unsigned items, + unsigned size)); + void ZLIB_INTERNAL zcfree OF((voidpf opaque, voidpf ptr)); +#endif + +#define ZALLOC(strm, items, size) \ + (*((strm)->zalloc))((strm)->opaque, (items), (size)) +#define ZFREE(strm, addr) (*((strm)->zfree))((strm)->opaque, (voidpf)(addr)) +#define TRY_FREE(s, p) {if (p) ZFREE(s, p);} + +/* Reverse the bytes in a 32-bit value */ +#define ZSWAP32(q) ((((q) >> 24) & 0xff) + (((q) >> 8) & 0xff00) + \ + (((q) & 0xff00) << 8) + (((q) & 0xff) << 24)) + +#ifdef _MSC_VER +#define zalign(x) __declspec(align(x)) +#else +#define zalign(x) __attribute__((aligned((x)))) +#endif + +#endif /* ZUTIL_H */ |