diff options
Diffstat (limited to 'src/main/java/com/android/tools')
63 files changed, 11519 insertions, 0 deletions
diff --git a/src/main/java/com/android/tools/build/apkzlib/sign/DigestAlgorithm.java b/src/main/java/com/android/tools/build/apkzlib/sign/DigestAlgorithm.java new file mode 100644 index 0000000..29ccdc8 --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/sign/DigestAlgorithm.java @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.sign; + +import javax.annotation.Nonnull; + +/** + * Message digest algorithms. + */ +public enum DigestAlgorithm { + /** + * SHA-1 digest. + * <p> + * Android 2.3 (API Level 9) to 4.2 (API Level 17) (inclusive) do not support SHA-2 + * JAR signatures. + * <p> + * Moreover, platforms prior to API Level 18, without the additional + * Digest-Algorithms attribute, only support SHA or SHA1 algorithm names in .SF and + * MANIFEST.MF attributes. + */ + SHA1("SHA1", "SHA-1"), + + /** + * SHA-256 digest. + */ + SHA256("SHA-256", "SHA-256"); + + /** + * API level which supports {@link #SHA256} with {@link SignatureAlgorithm#RSA} and + * {@link SignatureAlgorithm#ECDSA}. + */ + public static final int API_SHA_256_RSA_AND_ECDSA = 18; + + /** + * API level which supports {@link #SHA256} for all {@link SignatureAlgorithm}s. + * + * <p>Before that, SHA256 can only be used with RSA and ECDSA. + */ + public static final int API_SHA_256_ALL_ALGORITHMS = 21; + + /** + * Name of algorithm for message digest. + */ + @Nonnull + public final String messageDigestName; + + /** + * Name of attribute in signature file with the manifest digest. + */ + @Nonnull + public final String manifestAttributeName; + + /** + * Name of attribute in entry (both manifest and signature file) with the entry's digest. + */ + @Nonnull + public final String entryAttributeName; + + /** + * Creates a digest algorithm. + * + * @param attributeName attribute name in the signature file + * @param messageDigestName name of algorithm for message digest + */ + DigestAlgorithm(@Nonnull String attributeName, @Nonnull String messageDigestName) { + this.messageDigestName = messageDigestName; + this.entryAttributeName = attributeName + "-Digest"; + this.manifestAttributeName = attributeName + "-Digest-Manifest"; + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/sign/ManifestGenerationExtension.java b/src/main/java/com/android/tools/build/apkzlib/sign/ManifestGenerationExtension.java new file mode 100644 index 0000000..b8df2a9 --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/sign/ManifestGenerationExtension.java @@ -0,0 +1,244 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.sign; + +import com.android.tools.build.apkzlib.utils.CachedSupplier; +import com.android.tools.build.apkzlib.utils.IOExceptionRunnable; +import com.android.tools.build.apkzlib.zfile.ManifestAttributes; +import com.android.tools.build.apkzlib.zip.StoredEntry; +import com.android.tools.build.apkzlib.zip.ZFile; +import com.android.tools.build.apkzlib.zip.ZFileExtension; +import com.google.common.base.Preconditions; +import com.google.common.base.Verify; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.jar.Attributes; +import java.util.jar.Manifest; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * Extension to {@link ZFile} that will generate a manifest. The extension will register + * automatically with the {@link ZFile}. + * + * <p>Creating this extension will ensure a manifest for the zip exists. + * This extension will generate a manifest if one does not exist and will update an existing + * manifest, if one does exist. The extension will also provide access to the manifest so that + * others may update the manifest. + * + * <p>Apart from standard manifest elements, this extension does not handle any particular manifest + * features such as signing or adding custom attributes. It simply generates a plain manifest and + * provides infrastructure so that other extensions can add data in the manifest. + * + * <p>The manifest itself will only be written when the {@link ZFileExtension#beforeUpdate()} + * notification is received, meaning all manifest manipulation is done in-memory. + */ +public class ManifestGenerationExtension { + + /** + * Name of META-INF directory. + */ + private static final String META_INF_DIR = "META-INF"; + + /** + * Name of the manifest file. + */ + static final String MANIFEST_NAME = META_INF_DIR + "/MANIFEST.MF"; + + /** + * Who should be reported as the manifest builder. + */ + @Nonnull + private final String builtBy; + + /** + * Who should be reported as the manifest creator. + */ + @Nonnull + private final String createdBy; + + /** + * The file this extension is attached to. {@code null} if not yet registered. + */ + @Nullable + private ZFile zFile; + + /** + * The zip file's manifest. + */ + @Nonnull + private final Manifest manifest; + + /** + * Byte representation of the manifest. There is no guarantee that two writes of the java's + * {@code Manifest} object will yield the same byte array (there is no guaranteed order + * of entries in the manifest). + * + * <p>Because we need the byte representation of the manifest to be stable if there are + * no changes to the manifest, we cannot rely on {@code Manifest} to generate the byte + * representation every time we need the byte representation. + * + * <p>This cache will ensure that we will request one byte generation from the {@code Manifest} + * and will cache it. All further requests of the manifest's byte representation will + * receive the same byte array. + */ + @Nonnull + private CachedSupplier<byte[]> manifestBytes; + + /** + * Has the current manifest been changed and not yet flushed? If {@link #dirty} is + * {@code true}, then {@link #manifestBytes} should not be valid. This means that + * marking the manifest as dirty should also invalidate {@link #manifestBytes}. To avoid + * breaking the invariant, instead of setting {@link #dirty}, {@link #markDirty()} should + * be called. + */ + private boolean dirty; + + /** + * The extension to register with the {@link ZFile}. {@code null} if not registered. + */ + @Nullable + private ZFileExtension extension; + + /** + * Creates a new extension. This will not register the extension with the provided + * {@link ZFile}. Until {@link #register(ZFile)} is invoked, this extension is not used. + * + * @param builtBy who built the manifest? + * @param createdBy who created the manifest? + */ + public ManifestGenerationExtension(@Nonnull String builtBy, @Nonnull String createdBy) { + this.builtBy = builtBy; + this.createdBy = createdBy; + manifest = new Manifest(); + dirty = false; + manifestBytes = new CachedSupplier<>(() -> { + ByteArrayOutputStream outBytes = new ByteArrayOutputStream(); + try { + manifest.write(outBytes); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + + return outBytes.toByteArray(); + }); + } + + /** + * Marks the manifest as being dirty, <i>i.e.</i>, its data has changed since it was last + * read and/or written. + */ + private void markDirty() { + dirty = true; + manifestBytes.reset(); + } + + /** + * Registers the extension with the {@link ZFile} provided in the constructor. + * + * @param zFile the zip file to add the extension to + * @throws IOException failed to analyze the zip + */ + public void register(@Nonnull ZFile zFile) throws IOException { + Preconditions.checkState(extension == null, "register() has already been invoked."); + this.zFile = zFile; + + rebuildManifest(); + + extension = new ZFileExtension() { + @Nullable + @Override + public IOExceptionRunnable beforeUpdate() { + return ManifestGenerationExtension.this::updateManifest; + } + }; + + this.zFile.addZFileExtension(extension); + } + + /** + * Rebuilds the zip file's manifest, if it needs changes. + */ + private void rebuildManifest() throws IOException { + Verify.verifyNotNull(zFile, "zFile == null"); + + StoredEntry manifestEntry = zFile.get(MANIFEST_NAME); + + if (manifestEntry != null) { + /* + * Read the manifest entry in the zip file. Make sure we store these byte sequence + * because writing the manifest may not generate the same byte sequence, which may + * trigger an unnecessary re-sign of the jar. + */ + manifest.clear(); + byte[] manifestBytes = manifestEntry.read(); + manifest.read(new ByteArrayInputStream(manifestBytes)); + this.manifestBytes.precomputed(manifestBytes); + } + + Attributes mainAttributes = manifest.getMainAttributes(); + String currentVersion = mainAttributes.getValue(ManifestAttributes.MANIFEST_VERSION); + if (currentVersion == null) { + setMainAttribute( + ManifestAttributes.MANIFEST_VERSION, + ManifestAttributes.CURRENT_MANIFEST_VERSION); + } else { + if (!currentVersion.equals(ManifestAttributes.CURRENT_MANIFEST_VERSION)) { + throw new IOException("Unsupported manifest version: " + currentVersion + "."); + } + } + + /* + * We "blindly" override all other main attributes. + */ + setMainAttribute(ManifestAttributes.BUILT_BY, builtBy); + setMainAttribute(ManifestAttributes.CREATED_BY, createdBy); + } + + /** + * Sets the value of a main attribute. + * + * @param attribute the attribute + * @param value the value + */ + private void setMainAttribute(@Nonnull String attribute, @Nonnull String value) { + Attributes mainAttributes = manifest.getMainAttributes(); + String current = mainAttributes.getValue(attribute); + if (!value.equals(current)) { + mainAttributes.putValue(attribute, value); + markDirty(); + } + } + + /** + * Updates the manifest in the zip file, if it has been changed. + * + * @throws IOException failed to update the manifest + */ + private void updateManifest() throws IOException { + Verify.verifyNotNull(zFile, "zFile == null"); + + if (!dirty) { + return; + } + + zFile.add(MANIFEST_NAME, new ByteArrayInputStream(manifestBytes.get())); + dirty = false; + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/sign/SignatureAlgorithm.java b/src/main/java/com/android/tools/build/apkzlib/sign/SignatureAlgorithm.java new file mode 100644 index 0000000..ef7c71d --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/sign/SignatureAlgorithm.java @@ -0,0 +1,103 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.sign; + +import java.security.NoSuchAlgorithmException; +import javax.annotation.Nonnull; + +/** + * Signature algorithm. + */ +public enum SignatureAlgorithm { + /** RSA algorithm. */ + RSA("RSA", 1, "withRSA"), + + /** ECDSA algorithm. */ + ECDSA("EC", 18, "withECDSA"), + + /** DSA algorithm. */ + DSA("DSA", 1, "withDSA"); + + /** Name of the private key as reported by {@code PrivateKey}. */ + @Nonnull public final String keyAlgorithm; + + /** + * Minimum SDK version that allows this signature. + */ + public final int minSdkVersion; + + /** + * Suffix appended to digest algorithm to obtain signature algorithm. + */ + @Nonnull + public final String signatureAlgorithmSuffix; + + /** + * Creates a new signature algorithm. + * + * @param keyAlgorithm the name as reported by {@code PrivateKey} + * @param minSdkVersion minimum SDK version that allows this signature + * @param signatureAlgorithmSuffix suffix for signature name with used with a digest + */ + SignatureAlgorithm( + @Nonnull String keyAlgorithm, int minSdkVersion, @Nonnull String signatureAlgorithmSuffix) { + this.keyAlgorithm = keyAlgorithm; + this.minSdkVersion = minSdkVersion; + this.signatureAlgorithmSuffix = signatureAlgorithmSuffix; + } + + /** + * Obtains the signature algorithm that corresponds to a private key name applicable to a + * SDK version. + * + * @param keyAlgorithm the named referred in the {@code PrivateKey} + * @param minSdkVersion minimum SDK version to run + * @return the algorithm that has {@link #keyAlgorithm} equal to {@code keyAlgorithm} + * @throws NoSuchAlgorithmException if no algorithm was found for the given private key; an + * algorithm was found but is not applicable to the given SDK version + */ + @Nonnull + public static SignatureAlgorithm fromKeyAlgorithm(@Nonnull String keyAlgorithm, + int minSdkVersion) throws NoSuchAlgorithmException { + for (SignatureAlgorithm alg : values()) { + if (alg.keyAlgorithm.equalsIgnoreCase(keyAlgorithm)) { + if (alg.minSdkVersion > minSdkVersion) { + throw new NoSuchAlgorithmException("Signatures with " + keyAlgorithm + + " keys are not supported on minSdkVersion " + minSdkVersion + + ". They are supported only for minSdkVersion >= " + + alg.minSdkVersion); + } + + return alg; + } + } + + throw new NoSuchAlgorithmException("Signing with " + keyAlgorithm + + " keys is not supported"); + } + + /** + * Obtains the name of the signature algorithm when used with a digest algorithm. + * + * @param digestAlgorithm the digest algorithm to use + * @return the name of the signature algorithm + */ + @Nonnull + public String signatureAlgorithmName(@Nonnull DigestAlgorithm digestAlgorithm) { + return digestAlgorithm.messageDigestName.replace("-", "") + signatureAlgorithmSuffix; + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/sign/SigningExtension.java b/src/main/java/com/android/tools/build/apkzlib/sign/SigningExtension.java new file mode 100644 index 0000000..e72de69 --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/sign/SigningExtension.java @@ -0,0 +1,392 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.android.tools.build.apkzlib.sign; + +import com.android.apksig.ApkSignerEngine; +import com.android.apksig.ApkVerifier; +import com.android.apksig.DefaultApkSignerEngine; +import com.android.apksig.apk.ApkFormatException; +import com.android.apksig.util.DataSource; +import com.android.apksig.util.DataSources; +import com.android.tools.build.apkzlib.utils.IOExceptionRunnable; +import com.android.tools.build.apkzlib.zip.StoredEntry; +import com.android.tools.build.apkzlib.zip.ZFile; +import com.android.tools.build.apkzlib.zip.ZFileExtension; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.security.PrivateKey; +import java.security.SignatureException; +import java.security.cert.CertificateEncodingException; +import java.security.cert.X509Certificate; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * {@link ZFile} extension which signs the APK. + * + * <p> + * This extension is capable of signing the APK using JAR signing (aka v1 scheme) and APK Signature + * Scheme v2 (aka v2 scheme). Which schemes are actually used is specified by parameters to this + * extension's constructor. + */ +public class SigningExtension { + // IMPLEMENTATION NOTE: Most of the heavy lifting is performed by the ApkSignerEngine primitive + // from apksig library. This class is an adapter between ZFile extension and ApkSignerEngine. + // This class takes care of invoking the right methods on ApkSignerEngine in response to ZFile + // extension events/callbacks. + // + // The main issue leading to additional complexity in this class is that the current build + // pipeline does not reuse ApkSignerEngine instances (or ZFile extension instances for that + // matter) for incremental builds. Thus: + // * ZFile extension receives no events for JAR entries already in the APK whereas + // ApkSignerEngine needs to know about all JAR entries to be covered by signature. Thus, this + // class, during "beforeUpdate" ZFile event, notifies ApkSignerEngine about JAR entries + // already in the APK which ApkSignerEngine hasn't yet been told about -- these are the JAR + // entries which the incremental build session did not touch. + // * The build pipeline expects the APK not to change if no JAR entry was added to it or removed + // from it whereas ApkSignerEngine produces no output only if it has already produced a signed + // APK and no changes have since been made to it. This class addresses this issue by checking + // in its "register" method whether the APK is correctly signed and, only if that's the case, + // doesn't modify the APK unless a JAR entry is added to it or removed from it after + // "register". + + /** + * Minimum API Level on which this APK is supposed to run. + */ + private final int minSdkVersion; + + /** + * Whether JAR signing (aka v1 signing) is enabled. + */ + private final boolean v1SigningEnabled; + + /** + * Whether APK Signature Scheme v2 sining (aka v2 signing) is enabled. + */ + private final boolean v2SigningEnabled; + + /** + * Certificate of the signer, to be embedded into the APK's signature. + */ + @Nonnull + private final X509Certificate certificate; + + /** + * APK signer which performs most of the heavy lifting. + */ + @Nonnull + private final ApkSignerEngine signer; + + /** + * Names of APK entries which have been processed by {@link #signer}. + */ + private final Set<String> signerProcessedOutputEntryNames = new HashSet<>(); + + /** + * Cached contents of the most recently output APK Signing Block or {@code null} if the block + * hasn't yet been output. + */ + @Nullable + private byte[] cachedApkSigningBlock; + + /** + * {@code true} if signatures may need to be output, {@code false} if there's no need to output + * signatures. This is used in an optimization where we don't modify the APK if it's already + * signed and if no JAR entries have been added to or removed from the file. + */ + private boolean dirty; + + /** + * The extension registered with the {@link ZFile}. {@code null} if not registered. + */ + @Nullable + private ZFileExtension extension; + + /** + * The file this extension is attached to. {@code null} if not yet registered. + */ + @Nullable + private ZFile zFile; + + public SigningExtension( + int minSdkVersion, + @Nonnull X509Certificate certificate, + @Nonnull PrivateKey privateKey, + boolean v1SigningEnabled, + boolean v2SigningEnabled) throws InvalidKeyException { + DefaultApkSignerEngine.SignerConfig signerConfig = + new DefaultApkSignerEngine.SignerConfig.Builder( + "CERT", privateKey, ImmutableList.of(certificate)).build(); + signer = + new DefaultApkSignerEngine.Builder(ImmutableList.of(signerConfig), minSdkVersion) + .setOtherSignersSignaturesPreserved(false) + .setV1SigningEnabled(v1SigningEnabled) + .setV2SigningEnabled(v2SigningEnabled) + .setCreatedBy("1.0 (Android)") + .build(); + this.minSdkVersion = minSdkVersion; + this.v1SigningEnabled = v1SigningEnabled; + this.v2SigningEnabled = v2SigningEnabled; + this.certificate = certificate; + } + + public void register(@Nonnull ZFile zFile) throws NoSuchAlgorithmException, IOException { + Preconditions.checkState(extension == null, "register() already invoked"); + this.zFile = zFile; + dirty = !isCurrentSignatureAsRequested(); + extension = new ZFileExtension() { + @Override + public IOExceptionRunnable added( + @Nonnull StoredEntry entry, @Nullable StoredEntry replaced) { + return () -> onZipEntryOutput(entry); + } + + @Override + public IOExceptionRunnable removed(@Nonnull StoredEntry entry) { + String entryName = entry.getCentralDirectoryHeader().getName(); + return () -> onZipEntryRemovedFromOutput(entryName); + } + + @Override + public IOExceptionRunnable beforeUpdate() throws IOException { + return () -> onOutputZipReadyForUpdate(); + } + + @Override + public void entriesWritten() throws IOException { + onOutputZipEntriesWritten(); + } + + @Override + public void closed() { + onOutputClosed(); + } + }; + this.zFile.addZFileExtension(extension); + } + + /** + * Returns {@code true} if the APK's signatures are as requested by parameters to this signing + * extension. + */ + private boolean isCurrentSignatureAsRequested() throws IOException, NoSuchAlgorithmException { + ApkVerifier.Result result; + try { + result = + new ApkVerifier.Builder(new ZFileDataSource(zFile)) + .setMinCheckedPlatformVersion(minSdkVersion) + .build() + .verify(); + } catch (ApkFormatException e) { + // Malformed APK + return false; + } + + if (!result.isVerified()) { + // Signature(s) did not verify + return false; + } + + if ((result.isVerifiedUsingV1Scheme() != v1SigningEnabled) + || (result.isVerifiedUsingV2Scheme() != v2SigningEnabled)) { + // APK isn't signed with exactly the schemes we want it to be signed + return false; + } + + List<X509Certificate> verifiedSignerCerts = result.getSignerCertificates(); + if (verifiedSignerCerts.size() != 1) { + // APK is not signed by exactly one signer + return false; + } + + byte[] expectedEncodedCert; + byte[] actualEncodedCert; + try { + expectedEncodedCert = certificate.getEncoded(); + actualEncodedCert = verifiedSignerCerts.get(0).getEncoded(); + } catch (CertificateEncodingException e) { + // Failed to encode signing certificates + return false; + } + + if (!Arrays.equals(expectedEncodedCert, actualEncodedCert)) { + // APK is signed by a wrong signer + return false; + } + + // APK is signed the way we want it to be signed + return true; + } + + private void onZipEntryOutput(@Nonnull StoredEntry entry) throws IOException { + setDirty(); + String entryName = entry.getCentralDirectoryHeader().getName(); + // This event may arrive after the entry has already been deleted. In that case, we don't + // report the addition of the entry to ApkSignerEngine. + if (entry.isDeleted()) { + return; + } + ApkSignerEngine.InspectJarEntryRequest inspectEntryRequest = + signer.outputJarEntry(entryName); + signerProcessedOutputEntryNames.add(entryName); + if (inspectEntryRequest != null) { + byte[] entryContents = entry.read(); + inspectEntryRequest.getDataSink().consume(entryContents, 0, entryContents.length); + inspectEntryRequest.done(); + } + } + + private void onZipEntryRemovedFromOutput(@Nonnull String entryName) { + setDirty(); + signer.outputJarEntryRemoved(entryName); + signerProcessedOutputEntryNames.remove(entryName); + } + + private void onOutputZipReadyForUpdate() throws IOException { + if (!dirty) { + return; + } + + // Notify signer engine about ZIP entries that have appeared in the output without the + // engine knowing. Also identify ZIP entries which disappeared from the output without the + // engine knowing. + Set<String> unprocessedRemovedEntryNames = new HashSet<>(signerProcessedOutputEntryNames); + for (StoredEntry entry : zFile.entries()) { + String entryName = entry.getCentralDirectoryHeader().getName(); + unprocessedRemovedEntryNames.remove(entryName); + if (!signerProcessedOutputEntryNames.contains(entryName)) { + // Signer engine is not yet aware that this entry is in the output + onZipEntryOutput(entry); + } + } + + // Notify signer engine about entries which disappeared from the output without the engine + // knowing + for (String entryName : unprocessedRemovedEntryNames) { + onZipEntryRemovedFromOutput(entryName); + } + + // Check whether we need to output additional JAR entries which comprise the v1 signature + ApkSignerEngine.OutputJarSignatureRequest addV1SignatureRequest; + try { + addV1SignatureRequest = signer.outputJarEntries(); + } catch (Exception e) { + throw new IOException("Failed to generate v1 signature", e); + } + if (addV1SignatureRequest == null) { + return; + } + + // We need to output additional JAR entries which comprise the v1 signature + List<ApkSignerEngine.OutputJarSignatureRequest.JarEntry> v1SignatureEntries = + new ArrayList<>(addV1SignatureRequest.getAdditionalJarEntries()); + + // Reorder the JAR entries comprising the v1 signature so that MANIFEST.MF is the first + // entry. This ensures that it cleanly overwrites the existing MANIFEST.MF output by + // ManifestGenerationExtension. + for (int i = 0; i < v1SignatureEntries.size(); i++) { + ApkSignerEngine.OutputJarSignatureRequest.JarEntry entry = v1SignatureEntries.get(i); + String name = entry.getName(); + if (!ManifestGenerationExtension.MANIFEST_NAME.equals(name)) { + continue; + } + if (i != 0) { + v1SignatureEntries.remove(i); + v1SignatureEntries.add(0, entry); + } + break; + } + + // Output the JAR entries comprising the v1 signature + for (ApkSignerEngine.OutputJarSignatureRequest.JarEntry entry : v1SignatureEntries) { + String name = entry.getName(); + byte[] data = entry.getData(); + zFile.add(name, new ByteArrayInputStream(data)); + } + + addV1SignatureRequest.done(); + } + + private void onOutputZipEntriesWritten() throws IOException { + if (!dirty) { + return; + } + + // Check whether we should output an APK Signing Block which contains v2 signatures + byte[] apkSigningBlock; + byte[] centralDirBytes = zFile.getCentralDirectoryBytes(); + byte[] eocdBytes = zFile.getEocdBytes(); + ApkSignerEngine.OutputApkSigningBlockRequest addV2SignatureRequest; + // This event may arrive a second time -- after we write out the APK Signing Block. Thus, we + // cache the block to speed things up. The cached block is invalidated by any changes to the + // file (as reported to this extension). + if (cachedApkSigningBlock != null) { + apkSigningBlock = cachedApkSigningBlock; + addV2SignatureRequest = null; + } else { + DataSource centralDir = DataSources.asDataSource(ByteBuffer.wrap(centralDirBytes)); + DataSource eocd = DataSources.asDataSource(ByteBuffer.wrap(eocdBytes)); + long zipEntriesSizeBytes = + zFile.getCentralDirectoryOffset() - zFile.getExtraDirectoryOffset(); + DataSource zipEntries = new ZFileDataSource(zFile, 0, zipEntriesSizeBytes); + try { + addV2SignatureRequest = signer.outputZipSections(zipEntries, centralDir, eocd); + } catch (NoSuchAlgorithmException | InvalidKeyException | SignatureException + | ApkFormatException | IOException e) { + throw new IOException("Failed to generate v2 signature", e); + } + apkSigningBlock = + (addV2SignatureRequest != null) + ? addV2SignatureRequest.getApkSigningBlock() : new byte[0]; + cachedApkSigningBlock = apkSigningBlock; + } + + // Insert the APK Signing Block into the output right before the ZIP Central Directory and + // accordingly update the start offset of ZIP Central Directory in ZIP End of Central + // Directory. + zFile.directWrite( + zFile.getCentralDirectoryOffset() - zFile.getExtraDirectoryOffset(), + apkSigningBlock); + zFile.setExtraDirectoryOffset(apkSigningBlock.length); + + if (addV2SignatureRequest != null) { + addV2SignatureRequest.done(); + } + } + + private void onOutputClosed() { + if (!dirty) { + return; + } + signer.outputDone(); + dirty = false; + } + + private void setDirty() { + dirty = true; + cachedApkSigningBlock = null; + } +}
\ No newline at end of file diff --git a/src/main/java/com/android/tools/build/apkzlib/sign/ZFileDataSource.java b/src/main/java/com/android/tools/build/apkzlib/sign/ZFileDataSource.java new file mode 100644 index 0000000..3a1fc3c --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/sign/ZFileDataSource.java @@ -0,0 +1,157 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.sign; + +import com.android.apksig.util.DataSink; +import com.android.apksig.util.DataSource; +import com.android.tools.build.apkzlib.zip.ZFile; +import com.google.common.base.Preconditions; +import java.io.EOFException; +import java.io.IOException; +import java.nio.ByteBuffer; +import javax.annotation.Nonnull; + +/** + * {@link DataSource} backed by contents of {@link ZFile}. + */ +class ZFileDataSource implements DataSource { + + private static final int MAX_READ_CHUNK_SIZE = 65536; + + @Nonnull + private final ZFile file; + + /** + * Offset (in bytes) relative to the start of file where the region visible in this data source + * starts. + */ + private final long offset; + + /** + * Size (in bytes) of the file region visible in this data source or {@code -1} if the whole + * file is visible in this data source and thus its size may change if the file's size changes. + */ + private final long size; + + /** + * Constructs a new {@code ZFileDataSource} based on the data contained in the file. Changes to + * the contents of the file, including the size of the file, will be visible in this data + * source. + */ + public ZFileDataSource(@Nonnull ZFile file) { + this.file = file; + offset = 0; + size = -1; + } + + /** + * Constructs a new {@code ZFileDataSource} based on the data contained in the specified region + * of the provided file. Changes to the contents of this region of the file will be visible in + * this data source. + */ + public ZFileDataSource(@Nonnull ZFile file, long offset, long size) { + Preconditions.checkArgument(offset >= 0, "offset < 0"); + Preconditions.checkArgument(size >= 0, "size < 0"); + this.file = file; + this.offset = offset; + this.size = size; + } + + @Override + public long size() { + if (size == -1) { + // Data source size is the current size of the file + try { + return file.directSize(); + } catch (IOException e) { + return 0; + } + } else { + // Data source size is fixed + return size; + } + } + + @Override + public DataSource slice(long offset, long size) { + long sourceSize = size(); + checkChunkValid(offset, size, sourceSize); + if ((offset == 0) && (size == sourceSize)) { + return this; + } + + return new ZFileDataSource(file, this.offset + offset, size); + } + + @Override + public void feed(long offset, long size, @Nonnull DataSink sink) throws IOException { + long sourceSize = size(); + checkChunkValid(offset, size, sourceSize); + if (size == 0) { + return; + } + + long chunkOffsetInFile = this.offset + offset; + long remaining = size; + byte[] buf = new byte[(int) Math.min(remaining, MAX_READ_CHUNK_SIZE)]; + while (remaining > 0) { + int chunkSize = (int) Math.min(remaining, buf.length); + int readSize = file.directRead(chunkOffsetInFile, buf, 0, chunkSize); + if (readSize == -1) { + throw new EOFException("Premature EOF"); + } + if (readSize > 0) { + sink.consume(buf, 0, readSize); + chunkOffsetInFile += readSize; + remaining -= readSize; + } + } + } + + @Override + public void copyTo(long offset, int size, @Nonnull ByteBuffer dest) throws IOException { + long sourceSize = size(); + checkChunkValid(offset, size, sourceSize); + if (size == 0) { + return; + } + + int prevLimit = dest.limit(); + try { + file.directFullyRead(this.offset + offset, dest); + } finally { + dest.limit(prevLimit); + } + } + + @Override + public ByteBuffer getByteBuffer(long offset, int size) throws IOException { + ByteBuffer result = ByteBuffer.allocate(size); + copyTo(offset, size, result); + result.flip(); + return result; + } + + private static void checkChunkValid(long offset, long size, long sourceSize) { + Preconditions.checkArgument(offset >= 0, "offset < 0"); + Preconditions.checkArgument(size >= 0, "size < 0"); + Preconditions.checkArgument(offset <= sourceSize, "offset > sourceSize"); + long endOffset = offset + size; + Preconditions.checkArgument(offset <= endOffset, "offset > endOffset"); + Preconditions.checkArgument(endOffset <= sourceSize, "endOffset > sourceSize"); + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/sign/package-info.java b/src/main/java/com/android/tools/build/apkzlib/sign/package-info.java new file mode 100644 index 0000000..c1b0829 --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/sign/package-info.java @@ -0,0 +1,153 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + +The {@code sign} package provides extensions for the {@code zip} package that allow: +<ul> + <li>Adding a {@code MANIFEST.MF} file to a zip making a jar.</li> + <li>Signing a jar.</li> + <li>Fully signing a jar using v2 apk signature.</li> +</ul> +<p> +Because the {@code zip} package is completely independent of the {@code sign} package, the +actual coordination between the two is complex. The {@code sign} package works by registering +extensions with the {@code zip} package. These extensions are notified in changes made in the zip +and will change the zip file itself. +<p> +The {@link com.android.apkzlib.sign.ManifestGenerationExtension} extension will +ensure the zip has a manifest file and is, therefore, a valid jar. +The {@link com.android.apkzlib.sign.SigningExtension} extension will +ensure the jar is signed. +<p> +The extension mechanism used is the one provided in the {@code zip} package (see +{@link com.android.apkzlib.zip.ZFile} +and {@link com.android.apkzlib.zip.ZFileExtension}. Building the zip and then +operating the extensions is not done sequentially, as we don't want to build a zip and then sign it. +We want to build a zip that is automatically signed. Extension are basically observers that +register on the zip and are notified when things happen in the zip. They will then modify the zip +accordingly. +<p> +The zip file notifies extensions in 4 critical moments: when a file is added or removed from the +zip, when the zip is about to be flushed to disk and when the zip's entries have been flushed but +the central directory not. At these moments, the extensions can act to update the zip in any way +they need. +<p> +To see how this works, consider the manifest generation extension: when the extension is created, +it checks the zip file to see if there is a manifest. If a manifest exists and does not need +updating, it does not change anything, otherwise it generates a new manifest for the zip file. At +this point, the extension could write the manifest to the zip, but we opted not to. It would be +irrelevant anyway as the zip will only be written when flushed. +<p> +Now, when the {@code ZFile} notifies the extension that it is about to start writing the zip file, +the manifest extension, if it has noted that the manifest needs to be rewritten, will -- before the +{@code ZFile} actually writes anything -- modify the zip and add or replace the existing manifest +file. So, process-wise, the zip is written only once with the correct manifest. The flow is as +follows (if only the manifest generation extension was added to the {@code ZFile}): +<ol> + <li>{@code ZFile.update()} is called.</li> + <li>{@code ZFile} calls {@code beforeUpdate()} for all {@code ZFileExtensions} registered, in + this case, only the instance of the anonymous inner class generated in the + {@code ManifestGenerationExtension} constructor is invoked.</li> + <li>{@code ManifestGenerationExtension.updateManifest()} is called.</li> + <li>If the manifest does not need to be updated, {@code updateManifest()} returns + immediately.</li> + <li>If the manifest needs updating, {@code ZFile.add()} is invoked to add or replace the + manifest.</li> + <li>{@code ManifestGenerationExtension.updateManifest()} returns.</li> + <li>{@code ZFile.update()} continues and writes the zip file, containing the manifest.</li> + <li>The zip is finally written with an updated manifest.</li> +</ol> +<p> +To generate a signed apk, we need to add a second extension, the {@code SigningExtension}. +This extension will also register listeners with the {@code ZFile}. +<p> +In this case the flow would be (starting a bit earlier for clarity and assuming a package task +in the build process): +<ol> + <li>Package task creates a {@code ZFile} on the target apk (or non-existing file, if there is + no target apk in the output directory).</li> + <li>Package task configures the {@code ZFile} with alignment rules.</li> + <li>Package task creates a {@code ManifestGenerationExtension}.</li> + <li>Package task registers the {@code ManifestGenerationExtension} with the {@code ZFile}.</li> + <li>The {@code ManifestGenerationExtension} looks at the {@code ZFile} to see if there is valid + manifest. No changes are done to the {@code ZFile}.</li> + <li>Package task creates a {@code SigningExtension}.</li> + <li>Package task registers the {@code SigningExtension} with the {@code ZFile}.</li> + <li>The {@code SigningExtension} registers a {@code ZFileExtension} with the {@code ZFile} + and look at the {@code ZFile} to see if there is a valid signature file.</li> + <li>If there are changes to the digital signature file needed, these are marked internally in + the extension. If there are changes needed to the digests, the manifest is updated (by calling + {@code ManifestGenerationExtension}.<br> + <em>(note that this point, the apk file, if any existed, has not been touched, the manifest is + only updated in memory and the digests of all files in the apk, if any, have been computed and + stored in memory only; the digital signature of the {@code SF} file has not been computed.) + </em></li> + <li>The Package task now adds all files to the {@code ZFile}.</li> + <li>For each file that is added (*), {@code ZFile} calls the added {@code ZFileExtension.added} + method of all registered extensions.</li> + <li>The {@code ManifestGenerationExtension} ignores added invocations.</li> + <li>The {@code SigningExtension} computes the digest for the added file and stores them in + the manifest.<br> + <em>(when all files are added to the apk, all digests are computed and the manifest is updated + but only in memory; the apk file has not been touched; also note that {@code ZFile} has not + actually written anything to disk at this point, all files added are kept in memory).</em></li> + <li>Package task calls {@code ZFile.update()} to update the apk.</li> + <li>{@code ZFile} calls {@code before()} for all {@code ZFileExtensions} registered. This is + done before anything is written. In this case both the {@code ManifestGenerationExtension} and + {@code SigningExtension} are invoked.</li> + <li>The {@code ManifestGenerationExtension} will update the {@code ZFile} with the new manifest, + unless nothing has changed, in which case it does nothing.</li> + <li>The {@code SigningExtension} will add the SF file (unless nothing has changed), will + compute the digital signature of the SF file and write it to the {@code ZFile}.<br> + <em>(note that the order by which the {@code ManifestGenerationExtension} and + {@code SigningExtension} are called is non-deterministic; however, this is not a problem + because the manifest is already computed by the {@code ManifestGenerationExtension} at this + time and the {@code SigningExtension} will obtain the manifest data from the + {@code ManifestGenerationExtension} and not from the {@code ZFile}; this means that the + {@code SF} file may be added to the {@code ZFile} before the {@code MF} file, but that is + irrelevant.)</em></li> + <li>Once both extensions have finished doing the {@code beforeUpdate()} method, the + {@code ZFile.update()} method continues.</li> + <li>{@code ZFile.update()} writes all changes and new entries to the zip file.</li> + <li>{@code ZFile.update()} calls {@code ZFileExtension.entriesWritten()} for all + registered extensions. {@code SigningExtension} will kick in at this point, if v2 signature + has changed.</li> + <li>{@code ZFile} writes the central directory and EOCD.</li> + <li>{@code ZFile.update()} returns control to the package task.</li> + <li>The package task finishes.</li> +</ol> +<em>(*) There is a number of optimizations if we're adding files from another {@code ZFile}, which +is the case when we add the output of aapt to the apk. In particular, files from the aapt are +ignored if they are already in the apk (same name, same CRC32) and also files copied from +the aapt's output are not recompressed (the binary compressed data is directly copied to the +zip).</em> +<p> +If there are no changes to the {@code ZFile} made by the package task and the file's manifest and v1 +signatures are correct, neither the {@code ManifestGenerationExtension} nor the +{@code SigningExtension} will not do anything on the {@code beforeUpdate()} and the +{@code ZFile} won't even be open for writing. +<p> +This implementation provides perfect incremental updates. +<p> +Additionally, by adding/removing extensions we can configure what type of apk we want: +<ul> + <li>No SigningExtension ⇒ Aligned, unsigned apk.</li> + <li>SigningExtension ⇒ Aligned, signed apk. +</ul> +So, by configuring which extensions to add, the package task can decide what type of apk we want. +*/ +package com.android.apkzlib.sign; diff --git a/src/main/java/com/android/tools/build/apkzlib/utils/ApkZLibPair.java b/src/main/java/com/android/tools/build/apkzlib/utils/ApkZLibPair.java new file mode 100644 index 0000000..f9f4177 --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/utils/ApkZLibPair.java @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.utils; + +/** + * Pair implementation to use with the {@code apkzlib} library. + */ +public class ApkZLibPair<T1, T2> { + + /** + * First value. + */ + public T1 v1; + + /** + * Second value. + */ + public T2 v2; + + /** + * Creates a new pair. + * + * @param v1 the first value + * @param v2 the second value + */ + public ApkZLibPair(T1 v1, T2 v2) { + this.v1 = v1; + this.v2 = v2; + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/utils/CachedFileContents.java b/src/main/java/com/android/tools/build/apkzlib/utils/CachedFileContents.java new file mode 100644 index 0000000..3600800 --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/utils/CachedFileContents.java @@ -0,0 +1,176 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.utils; + +import com.google.common.base.Objects; +import com.google.common.hash.HashCode; +import com.google.common.hash.Hashing; +import com.google.common.io.Files; +import java.io.File; +import java.io.IOException; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * A cache for file contents. The cache allows closing a file and saving in memory its contents (or + * some related information). It can then be used to check if the contents are still valid at some + * later time. Typical usage flow is: + * + * <p> + * + * <pre>{@code + * Object fileRepresentation = // ... + * File toWrite = // ... + * // Write file contents and update in memory representation + * CachedFileContents<Object> contents = new CachedFileContents<Object>(toWrite); + * contents.closed(fileRepresentation); + * + * // Later, when data is needed: + * if (contents.isValid()) { + * fileRepresentation = contents.getCache(); + * } else { + * // Re-read the file and recreate the file representation + * } + * }</pre> + * + * @param <T> the type of cached contents + */ +public class CachedFileContents<T> { + + /** + * The file. + */ + @Nonnull + private File file; + + /** + * Time when last closed (time when {@link #closed(Object)} was invoked). + */ + private long lastClosed; + + /** + * Size of the file when last closed. + */ + private long size; + + /** + * Hash of the file when closed. {@code null} if hashing failed for some reason. + */ + @Nullable + private HashCode hash; + + /** + * Cached data associated with the file. + */ + @Nullable + private T cache; + + /** + * Creates a new contents. When the file is written, {@link #closed(Object)} should be invoked + * to set the cache. + * + * @param file the file + */ + public CachedFileContents(@Nonnull File file) { + this.file = file; + } + + /** + * Should be called when the file's contents are set and the file closed. This will save the + * cache and register the file's timestamp to later detect if it has been modified. + * <p> + * This method can be called as many times as the file has been written. + * + * @param cache an optional cache to save + */ + public void closed(@Nullable T cache) { + this.cache = cache; + lastClosed = file.lastModified(); + size = file.length(); + hash = hashFile(); + } + + /** + * Are the cached contents still valid? If this method determines that the file has been + * modified since the last time {@link #closed(Object)} was invoked. + * + * @return are the cached contents still valid? If this method returns {@code false}, the + * cache is cleared + */ + public boolean isValid() { + boolean valid = true; + + if (!file.exists()) { + valid = false; + } + + if (valid && file.lastModified() != lastClosed) { + valid = false; + } + + if (valid && file.length() != size) { + valid = false; + } + + if (valid && !Objects.equal(hash, hashFile())) { + valid = false; + } + + if (!valid) { + cache = null; + } + + return valid; + } + + /** + * Obtains the cached data set with {@link #closed(Object)} if the file has not been modified + * since {@link #closed(Object)} was invoked. + * + * @return the last cached data or {@code null} if the file has been modified since + * {@link #closed(Object)} has been invoked + */ + @Nullable + public T getCache() { + return cache; + } + + /** + * Computes the hashcode of the cached file. + * + * @return the hash code + */ + @Nullable + private HashCode hashFile() { + try { + return Files.hash(file, Hashing.crc32()); + } catch (IOException e) { + return null; + } + } + + /** + * Obtains the file used for caching. + * + * @return the file; this file always exists and contains the old (cached) contents of the + * file + */ + @Nonnull + public File getFile() { + return file; + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/utils/CachedSupplier.java b/src/main/java/com/android/tools/build/apkzlib/utils/CachedSupplier.java new file mode 100644 index 0000000..84505bc --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/utils/CachedSupplier.java @@ -0,0 +1,118 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.utils; + +import java.util.function.Supplier; +import javax.annotation.Nonnull; + +/** + * Supplier that will cache a computed value and always supply the same value. It can be used to + * lazily compute data. For example: + * + * <pre>{@code + * CachedSupplier<Integer> value = new CachedSupplier<>(() -> { + * Integer result; + * // Do some expensive computation. + * return result; + * }); + * + * if (a) { + * // We need the result of the expensive computation. + * Integer r = value.get(); + * } + * + * if (b) { + * // We also need the result of the expensive computation. + * Integer r = value.get(); + * } + * + * // If neither a nor b are true, we avoid doing the computation at all. + * }</pre> + */ +public class CachedSupplier<T> { + + /** + * The cached data, {@code null} if computation resulted in {@code null}. It is also + * {@code null} if the cached data has not yet been computed. + */ + private T cached; + + /** + * Is the current data in {@link #cached} valid? + */ + private boolean valid; + + /** + * Actual supplier of data, if computation is needed. + */ + @Nonnull + private final Supplier<T> supplier; + + /** + * Creates a new supplier. + */ + public CachedSupplier(@Nonnull Supplier<T> supplier) { + valid = false; + this.supplier = supplier; + } + + + /** + * Obtains the value. + * + * @return the value, either cached (if one exists) or computed + */ + public synchronized T get() { + if (!valid) { + cached = supplier.get(); + valid = true; + } + + return cached; + } + + /** + * Resets the cache forcing a {@code get()} on the supplier next time {@link #get()} is invoked. + */ + public synchronized void reset() { + cached = null; + valid = false; + } + + /** + * In some cases, we may be able to precompute the cache value (or load it from somewhere we + * had previously stored it). This method allows the cache value to be loaded. + * + * <p>If this method is invoked, then an invocation of {@link #get()} will not trigger an + * invocation of the supplier provided in the constructor. + * + * @param t the new cache contents; will replace any currently cache content, if one exists + */ + public synchronized void precomputed(T t) { + cached = t; + valid = true; + } + + /** + * Checks if the contents of the cache are valid. + * + * @return are there valid contents in the cache? + */ + public synchronized boolean isValid() { + return valid; + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/utils/IOExceptionConsumer.java b/src/main/java/com/android/tools/build/apkzlib/utils/IOExceptionConsumer.java new file mode 100644 index 0000000..98aefe7 --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/utils/IOExceptionConsumer.java @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.utils; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.function.Consumer; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * Consumer that can throw an {@link IOException}. + */ +@FunctionalInterface +public interface IOExceptionConsumer<T> { + + /** + * Performs an operation on the given input. + * + * @param input the input + */ + void accept(@Nullable T input) throws IOException; + + /** + * Wraps a consumer that may throw an IO Exception throwing an {@code UncheckedIOException}. + * + * @param c the consumer + */ + @Nonnull + static <T> Consumer<T> asConsumer(@Nonnull IOExceptionConsumer<T> c) { + return i -> { + try { + c.accept(i); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }; + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/utils/IOExceptionFunction.java b/src/main/java/com/android/tools/build/apkzlib/utils/IOExceptionFunction.java new file mode 100644 index 0000000..4ccce5f --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/utils/IOExceptionFunction.java @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.utils; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.function.Function; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * Function that can throw an I/O Exception + */ +@FunctionalInterface +public interface IOExceptionFunction<F, T> { + + /** + * Applies the function to the given input. + * @param input the input + * @return the function result + */ + @Nullable T apply(@Nullable F input) throws IOException; + + /** + * Wraps a function that may throw an IO Exception throwing an {@code UncheckedIOException}. + * + * @param f the function + */ + @Nonnull + static <F, T> Function<F, T> asFunction(@Nonnull IOExceptionFunction<F, T> f) { + return i -> { + try { + return f.apply(i); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }; + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/utils/IOExceptionRunnable.java b/src/main/java/com/android/tools/build/apkzlib/utils/IOExceptionRunnable.java new file mode 100644 index 0000000..40de80b --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/utils/IOExceptionRunnable.java @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.utils; + +import java.io.IOException; +import java.io.UncheckedIOException; +import javax.annotation.Nonnull; + +/** + * Runnable that can throw I/O exceptions. + */ +@FunctionalInterface +public interface IOExceptionRunnable { + + /** + * Runs the runnable. + * + * @throws IOException failed to run + */ + void run() throws IOException; + + /** + * Wraps a runnable that may throw an IO Exception throwing an {@code UncheckedIOException}. + * + * @param r the runnable + */ + @Nonnull + public static Runnable asRunnable(@Nonnull IOExceptionRunnable r) { + return () -> { + try { + r.run(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }; + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/utils/IOExceptionWrapper.java b/src/main/java/com/android/tools/build/apkzlib/utils/IOExceptionWrapper.java new file mode 100644 index 0000000..d6f4d8b --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/utils/IOExceptionWrapper.java @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.utils; + +import java.io.IOException; +import javax.annotation.Nonnull; + +/** + * Runtime exception used to encapsulate an IO Exception. This is used to allow throwing I/O + * exceptions in functional interfaces that do not allow it and catching the exception afterwards. + */ +public class IOExceptionWrapper extends RuntimeException { + + /** + * Creates a new exception. + * + * @param e the I/O exception to encapsulate + */ + public IOExceptionWrapper(@Nonnull IOException e) { + super(e); + } + + @Override + @Nonnull + public IOException getCause() { + return (IOException) super.getCause(); + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/utils/package-info.java b/src/main/java/com/android/tools/build/apkzlib/utils/package-info.java new file mode 100644 index 0000000..64d8a62 --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/utils/package-info.java @@ -0,0 +1,20 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Utilities to work with {@code apkzlib}. + */ +package com.android.tools.build.apkzlib.utils; diff --git a/src/main/java/com/android/tools/build/apkzlib/zfile/ApkCreator.java b/src/main/java/com/android/tools/build/apkzlib/zfile/ApkCreator.java new file mode 100644 index 0000000..e47b04e --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zfile/ApkCreator.java @@ -0,0 +1,71 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zfile; + +import java.io.Closeable; +import java.io.File; +import java.io.IOException; +import java.util.function.Function; +import java.util.function.Predicate; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * Creates or updates APKs based on provided entries. + */ +public interface ApkCreator extends Closeable { + + /** + * Copies the content of a Jar/Zip archive into the receiver archive. + * + * <p>An optional predicate allows to selectively choose which files to copy over and an + * option function allows renaming the files as they are copied. + * + * @param zip the zip to copy data from + * @param transform an optional transform to apply to file names before copying them + * @param isIgnored an optional filter or {@code null} to mark which out files should not be + * added, even through they are on the zip; if {@code transform} is specified, then this + * predicate applies after transformation + * @throws IOException I/O error + */ + void writeZip( + @Nonnull File zip, + @Nullable Function<String, String> transform, + @Nullable Predicate<String> isIgnored) + throws IOException; + + /** + * Writes a new {@link File} into the archive. If a file already existed with the given + * path, it should be replaced. + * + * @param inputFile the {@link File} to write. + * @param apkPath the filepath inside the archive. + * @throws IOException I/O error + */ + void writeFile(@Nonnull File inputFile, @Nonnull String apkPath) throws IOException; + + /** + * Deletes a file in a given path. + * + * @param apkPath the path to remove + * @throws IOException failed to remove the entry + */ + void deleteFile(@Nonnull String apkPath) throws IOException; + + /** Returns true if the APK will be rewritten on close. */ + boolean hasPendingChangesWithWait() throws IOException; +} diff --git a/src/main/java/com/android/tools/build/apkzlib/zfile/ApkCreatorFactory.java b/src/main/java/com/android/tools/build/apkzlib/zfile/ApkCreatorFactory.java new file mode 100644 index 0000000..e782206 --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zfile/ApkCreatorFactory.java @@ -0,0 +1,246 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zfile; + +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.common.base.Preconditions; +import java.io.File; +import java.security.PrivateKey; +import java.security.cert.X509Certificate; +import java.util.function.Predicate; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * Factory that creates instances of {@link ApkCreator}. + */ +public interface ApkCreatorFactory { + + /** + * Creates an {@link ApkCreator} with a given output location, and signing information. + * + * @param creationData the information to create the APK + */ + ApkCreator make(@Nonnull CreationData creationData); + + /** + * Data structure with the required information to initiate the creation of an APK. See + * {@link ApkCreatorFactory#make(CreationData)}. + */ + class CreationData { + + /** + * The path where the APK should be located. May already exist or not (if it does, then + * the APK may be updated instead of created). + */ + @Nonnull + private final File apkPath; + + /** + * Key used to sign the APK. May be {@code null}. + */ + @Nullable + private final PrivateKey key; + + /** + * Certificate used to sign the APK. Is {@code null} if and only if {@link #key} is + * {@code null}. + */ + @Nullable + private final X509Certificate certificate; + + /** + * Whether signing the APK with JAR Signing Scheme (aka v1 signing) is enabled. + */ + private final boolean v1SigningEnabled; + + /** + * Whether signing the APK with APK Signature Scheme v2 (aka v2 signing) is enabled. + */ + private final boolean v2SigningEnabled; + + /** + * Built-by information for the APK, if any. + */ + @Nullable + private final String builtBy; + + /** + * Created-by information for the APK, if any. + */ + @Nullable + private final String createdBy; + + /** + * Minimum SDk version that will run the APK. + */ + private final int minSdkVersion; + + /** + * How should native libraries be packaged? + */ + @Nonnull + private final NativeLibrariesPackagingMode nativeLibrariesPackagingMode; + + /** + * Predicate identifying paths that should not be compressed. + */ + @Nonnull + private final Predicate<String> noCompressPredicate; + + /** + * + * @param apkPath the path where the APK should be located. May already exist or not (if it + * does, then the APK may be updated instead of created) + * @param key key used to sign the APK. May be {@code null} + * @param certificate certificate used to sign the APK. Is {@code null} if and only if + * {@code key} is {@code null} + * @param v1SigningEnabled {@code true} if this APK should be signed with JAR Signature + * Scheme (aka v1 scheme). + * @param v2SigningEnabled {@code true} if this APK should be signed with APK Signature + * Scheme v2 (aka v2 scheme). + * @param builtBy built-by information for the APK, if any; if {@code null} then the default + * should be used + * @param createdBy created-by information for the APK, if any; if {@code null} then the + * default should be used + * @param minSdkVersion minimum SDK version that will run the APK + * @param nativeLibrariesPackagingMode packaging mode for native libraries + * @param noCompressPredicate predicate to decide which file paths should be uncompressed; + * returns {@code true} for files that should not be compressed + */ + public CreationData( + @Nonnull File apkPath, + @Nullable PrivateKey key, + @Nullable X509Certificate certificate, + boolean v1SigningEnabled, + boolean v2SigningEnabled, + @Nullable String builtBy, + @Nullable String createdBy, + int minSdkVersion, + @Nonnull NativeLibrariesPackagingMode nativeLibrariesPackagingMode, + @Nonnull Predicate<String> noCompressPredicate) { + Preconditions.checkArgument((key == null) == (certificate == null), + "(key == null) != (certificate == null)"); + Preconditions.checkArgument(minSdkVersion >= 0, "minSdkVersion < 0"); + + this.apkPath = apkPath; + this.key = key; + this.certificate = certificate; + this.v1SigningEnabled = v1SigningEnabled; + this.v2SigningEnabled = v2SigningEnabled; + this.builtBy = builtBy; + this.createdBy = createdBy; + this.minSdkVersion = minSdkVersion; + this.nativeLibrariesPackagingMode = checkNotNull(nativeLibrariesPackagingMode); + this.noCompressPredicate = checkNotNull(noCompressPredicate); + } + + /** + * Obtains the path where the APK should be located. If the path already exists, then the + * APK may be updated instead of re-created. + * + * @return the path that may already exist or not + */ + @Nonnull + public File getApkPath() { + return apkPath; + } + + /** + * Obtains the private key used to sign the APK. + * + * @return the private key or {@code null} if the APK should not be signed + */ + @Nullable + public PrivateKey getPrivateKey() { + return key; + } + + /** + * Obtains the certificate used to sign the APK. + * + * @return the certificate or {@code null} if the APK should not be signed; this will return + * {@code null} if and only if {@link #getPrivateKey()} returns {@code null} + */ + @Nullable + public X509Certificate getCertificate() { + return certificate; + } + + /** + * Returns {@code true} if this APK should be signed with JAR Signature Scheme (aka v1 + * scheme). + */ + public boolean isV1SigningEnabled() { + return v1SigningEnabled; + } + + /** + * Returns {@code true} if this APK should be signed with APK Signature Scheme v2 (aka v2 + * scheme). + */ + public boolean isV2SigningEnabled() { + return v2SigningEnabled; + } + + /** + * Obtains the "built-by" text for the APK. + * + * @return the text or {@code null} if the default should be used + */ + @Nullable + public String getBuiltBy() { + return builtBy; + } + + /** + * Obtains the "created-by" text for the APK. + * + * @return the text or {@code null} if the default should be used + */ + @Nullable + public String getCreatedBy() { + return createdBy; + } + + /** + * Obtains the minimum SDK version to run the APK. + * + * @return the minimum SDK version + */ + public int getMinSdkVersion() { + return minSdkVersion; + } + + /** + * Returns the packaging policy that the {@link ApkCreator} should use for native libraries. + */ + @Nonnull + public NativeLibrariesPackagingMode getNativeLibrariesPackagingMode() { + return nativeLibrariesPackagingMode; + } + + /** + * Returns the predicate to decide which file paths should be uncompressed. + */ + @Nonnull + public Predicate<String> getNoCompressPredicate() { + return noCompressPredicate; + } + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/zfile/ApkZFileCreator.java b/src/main/java/com/android/tools/build/apkzlib/zfile/ApkZFileCreator.java new file mode 100644 index 0000000..0a0b90d --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zfile/ApkZFileCreator.java @@ -0,0 +1,192 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zfile; + +import com.android.tools.build.apkzlib.zip.AlignmentRule; +import com.android.tools.build.apkzlib.zip.AlignmentRules; +import com.android.tools.build.apkzlib.zip.StoredEntry; +import com.android.tools.build.apkzlib.zip.ZFile; +import com.android.tools.build.apkzlib.zip.ZFileOptions; +import com.google.common.base.Preconditions; +import com.google.common.io.Closer; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.function.Function; +import java.util.function.Predicate; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * {@link ApkCreator} that uses {@link ZFileOptions} to generate the APK. + */ +class ApkZFileCreator implements ApkCreator { + + /** + * Suffix for native libraries. + */ + private static final String NATIVE_LIBRARIES_SUFFIX = ".so"; + + /** + * Shared libraries are alignment at 4096 boundaries. + */ + private static final AlignmentRule SO_RULE = + AlignmentRules.constantForSuffix(NATIVE_LIBRARIES_SUFFIX, 4096); + + /** + * The zip file. + */ + @Nonnull + private final ZFile zip; + + /** + * Has the zip file been closed? + */ + private boolean closed; + + /** + * Predicate defining which files should not be compressed. + */ + @Nonnull + private final Predicate<String> noCompressPredicate; + + /** + * Creates a new creator. + * + * @param creationData the data needed to create the APK + * @param options zip file options + * @throws IOException failed to create the zip + */ + ApkZFileCreator( + @Nonnull ApkCreatorFactory.CreationData creationData, + @Nonnull ZFileOptions options) + throws IOException { + + switch (creationData.getNativeLibrariesPackagingMode()) { + case COMPRESSED: + noCompressPredicate = creationData.getNoCompressPredicate(); + break; + case UNCOMPRESSED_AND_ALIGNED: + noCompressPredicate = + creationData.getNoCompressPredicate().or( + name -> name.endsWith(NATIVE_LIBRARIES_SUFFIX)); + options.setAlignmentRule( + AlignmentRules.compose(SO_RULE, options.getAlignmentRule())); + break; + default: + throw new AssertionError(); + } + + zip = ZFiles.apk( + creationData.getApkPath(), + options, + creationData.getPrivateKey(), + creationData.getCertificate(), + creationData.isV1SigningEnabled(), + creationData.isV2SigningEnabled(), + creationData.getBuiltBy(), + creationData.getCreatedBy(), + creationData.getMinSdkVersion()); + closed = false; + } + + @Override + public void writeZip(@Nonnull File zip, @Nullable Function<String, String> transform, + @Nullable Predicate<String> isIgnored) throws IOException { + Preconditions.checkState(!closed, "closed == true"); + Preconditions.checkArgument(zip.isFile(), "!zip.isFile()"); + + Closer closer = Closer.create(); + try { + ZFile toMerge = closer.register(new ZFile(zip)); + + Predicate<String> ignorePredicate; + if (isIgnored == null) { + ignorePredicate = s -> false; + } else { + ignorePredicate = isIgnored; + } + + // Files that *must* be uncompressed in the result should not be merged and should be + // added after. This is just very slightly less efficient than ignoring just the ones + // that were compressed and must be uncompressed, but it is a lot simpler :) + Predicate<String> noMergePredicate = ignorePredicate.or(noCompressPredicate); + + this.zip.mergeFrom(toMerge, noMergePredicate); + + for (StoredEntry toMergeEntry : toMerge.entries()) { + String path = toMergeEntry.getCentralDirectoryHeader().getName(); + if (noCompressPredicate.test(path) && !ignorePredicate.test(path)) { + // This entry *must* be uncompressed so it was ignored in the merge and should + // now be added to the apk. + try (InputStream ignoredData = toMergeEntry.open()) { + this.zip.add(path, ignoredData, false); + } + } + } + } catch (Throwable t) { + throw closer.rethrow(t); + } finally { + closer.close(); + } + } + + @Override + public void writeFile(@Nonnull File inputFile, @Nonnull String apkPath) throws IOException { + Preconditions.checkState(!closed, "closed == true"); + + boolean mayCompress = !noCompressPredicate.test(apkPath); + + Closer closer = Closer.create(); + try { + FileInputStream inputFileStream = closer.register(new FileInputStream(inputFile)); + zip.add(apkPath, inputFileStream, mayCompress); + } catch (IOException e) { + throw closer.rethrow(e, IOException.class); + } catch (Throwable t) { + throw closer.rethrow(t); + } finally { + closer.close(); + } + } + + @Override + public void deleteFile(@Nonnull String apkPath) throws IOException { + Preconditions.checkState(!closed, "closed == true"); + + StoredEntry entry = zip.get(apkPath); + if (entry != null) { + entry.delete(); + } + } + + @Override + public boolean hasPendingChangesWithWait() throws IOException { + return zip.hasPendingChangesWithWait(); + } + + @Override + public void close() throws IOException { + if (closed) { + return; + } + + zip.close(); + closed = true; + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/zfile/ApkZFileCreatorFactory.java b/src/main/java/com/android/tools/build/apkzlib/zfile/ApkZFileCreatorFactory.java new file mode 100644 index 0000000..b19885d --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zfile/ApkZFileCreatorFactory.java @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zfile; + +import com.android.tools.build.apkzlib.zip.ZFileOptions; +import java.io.IOException; +import java.io.UncheckedIOException; +import javax.annotation.Nonnull; + +/** + * Creates instances of {@link ApkZFileCreator}. + */ +public class ApkZFileCreatorFactory implements ApkCreatorFactory { + + /** + * Options for the {@link ZFileOptions} to use in all APKs. + */ + @Nonnull + private final ZFileOptions options; + + /** + * Creates a new factory. + * + * @param options the options to use for all instances created + */ + public ApkZFileCreatorFactory(@Nonnull ZFileOptions options) { + this.options = options; + } + + + @Override + @Nonnull + public ApkCreator make(@Nonnull CreationData creationData) { + try { + return new ApkZFileCreator(creationData, options); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/zfile/ManifestAttributes.java b/src/main/java/com/android/tools/build/apkzlib/zfile/ManifestAttributes.java new file mode 100644 index 0000000..e8e6c2d --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zfile/ManifestAttributes.java @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zfile; + +/** + * Java manifest attributes and some default values. + */ +public interface ManifestAttributes { + /** + * Manifest attribute with the built by information. + */ + String BUILT_BY = "Built-By"; + + /** + * Manifest attribute with the created by information. + */ + String CREATED_BY = "Created-By"; + + /** + * Manifest attribute with the manifest version. + */ + String MANIFEST_VERSION = "Manifest-Version"; + + /** + * Manifest attribute value with the manifest version. + */ + String CURRENT_MANIFEST_VERSION = "1.0"; + +} diff --git a/src/main/java/com/android/tools/build/apkzlib/zfile/NativeLibrariesPackagingMode.java b/src/main/java/com/android/tools/build/apkzlib/zfile/NativeLibrariesPackagingMode.java new file mode 100644 index 0000000..0dab9b1 --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zfile/NativeLibrariesPackagingMode.java @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zfile; + +/** + * Describes how native libs should be packaged. + */ +public enum NativeLibrariesPackagingMode { + /** + * Native libs are packaged as any other file. + */ + COMPRESSED, + + /** + * Native libs are packaged uncompressed and page-aligned, so they can be mapped into memory + * at runtime. + * + * <p>Support for this mode was added in Android 23, it only works if the + * {@code extractNativeLibs} attribute is set in the manifest. + */ + UNCOMPRESSED_AND_ALIGNED; +} diff --git a/src/main/java/com/android/tools/build/apkzlib/zfile/ZFiles.java b/src/main/java/com/android/tools/build/apkzlib/zfile/ZFiles.java new file mode 100644 index 0000000..a54c50d --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zfile/ZFiles.java @@ -0,0 +1,132 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zfile; + +import com.android.tools.build.apkzlib.sign.ManifestGenerationExtension; +import com.android.tools.build.apkzlib.sign.SigningExtension; +import com.android.tools.build.apkzlib.zip.AlignmentRule; +import com.android.tools.build.apkzlib.zip.AlignmentRules; +import com.android.tools.build.apkzlib.zip.ZFile; +import com.android.tools.build.apkzlib.zip.ZFileOptions; +import java.io.File; +import java.io.IOException; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.security.PrivateKey; +import java.security.cert.X509Certificate; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * Factory for {@link ZFile}s that are specifically configured to be APKs, AARs, ... + */ +public class ZFiles { + + /** + * By default all non-compressed files are alignment at 4 byte boundaries.. + */ + private static final AlignmentRule APK_DEFAULT_RULE = AlignmentRules.constant(4); + + /** + * Default build by string. + */ + private static final String DEFAULT_BUILD_BY = "Generated-by-ADT"; + + /** + * Default created by string. + */ + private static final String DEFAULT_CREATED_BY = "Generated-by-ADT"; + + /** + * Creates a new zip file configured as an apk, based on a given file. + * + * @param f the file, if this path does not represent an existing path, will create a + * {@link ZFile} based on an non-existing path (a zip will be created when + * {@link ZFile#close()} is invoked) + * @param options the options to create the {@link ZFile} + * @return the zip file + * @throws IOException failed to create the zip file + */ + @Nonnull + public static ZFile apk(@Nonnull File f, @Nonnull ZFileOptions options) throws IOException { + options.setAlignmentRule( + AlignmentRules.compose(options.getAlignmentRule(), APK_DEFAULT_RULE)); + return new ZFile(f, options); + } + + /** + * Creates a new zip file configured as an apk, based on a given file. + * + * @param f the file, if this path does not represent an existing path, will create a + * {@link ZFile} based on an non-existing path (a zip will be created when + * {@link ZFile#close()} is invoked) + * @param options the options to create the {@link ZFile} + * @param key the {@link PrivateKey} used to sign the archive, or {@code null}. + * @param certificate the {@link X509Certificate} used to sign the archive, or + * {@code null}. + * @param v1SigningEnabled whether signing with JAR Signature Scheme (aka v1 signing) is + * enabled. + * @param v2SigningEnabled whether signing with APK Signature Scheme v2 (aka v2 signing) is + * enabled. + * @param builtBy who to mark as builder in the manifest + * @param createdBy who to mark as creator in the manifest + * @param minSdkVersion minimum SDK version supported + * @return the zip file + * @throws IOException failed to create the zip file + */ + @Nonnull + public static ZFile apk( + @Nonnull File f, + @Nonnull ZFileOptions options, + @Nullable PrivateKey key, + @Nullable X509Certificate certificate, + boolean v1SigningEnabled, + boolean v2SigningEnabled, + @Nullable String builtBy, + @Nullable String createdBy, + int minSdkVersion) + throws IOException { + ZFile zfile = apk(f, options); + + if (builtBy == null) { + builtBy = DEFAULT_BUILD_BY; + } + + if (createdBy == null) { + createdBy = DEFAULT_CREATED_BY; + } + + ManifestGenerationExtension manifestExt = new ManifestGenerationExtension(builtBy, + createdBy); + manifestExt.register(zfile); + + if (key != null && certificate != null) { + try { + new SigningExtension( + minSdkVersion, + certificate, + key, + v1SigningEnabled, + v2SigningEnabled).register(zfile); + } catch (NoSuchAlgorithmException | InvalidKeyException e) { + throw new IOException("Failed to create signature extensions", e); + } + } + + return zfile; + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/zfile/package-info.java b/src/main/java/com/android/tools/build/apkzlib/zfile/package-info.java new file mode 100644 index 0000000..703b209 --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zfile/package-info.java @@ -0,0 +1,18 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** The {@code zfile} package contains */ +package com.android.tools.build.apkzlib.zfile; diff --git a/src/main/java/com/android/tools/build/apkzlib/zip/AlignmentRule.java b/src/main/java/com/android/tools/build/apkzlib/zip/AlignmentRule.java new file mode 100644 index 0000000..d599a03 --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zip/AlignmentRule.java @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zip; + +import javax.annotation.Nonnull; + +/** + * An alignment rule defines how to a file should be aligned in a zip, based on its name. + */ +public interface AlignmentRule { + + /** + * Alignment value of files that do not require alignment. + */ + int NO_ALIGNMENT = 1; + + /** + * Obtains the alignment this rule computes for a given path. + * + * @param path the path in the zip file + * @return the alignment value, always greater than {@code 0}; if this rule places no + * restrictions on the provided path, then {@link AlignmentRule#NO_ALIGNMENT} is returned + */ + int alignment(@Nonnull String path); +} diff --git a/src/main/java/com/android/tools/build/apkzlib/zip/AlignmentRules.java b/src/main/java/com/android/tools/build/apkzlib/zip/AlignmentRules.java new file mode 100644 index 0000000..f654708 --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zip/AlignmentRules.java @@ -0,0 +1,77 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zip; + +import com.google.common.base.Preconditions; +import javax.annotation.Nonnull; + +/** + * Factory for instances of {@link AlignmentRule}. + */ +public final class AlignmentRules { + + private AlignmentRules() {} + + /** + * A rule that defines a constant alignment for all files. + * + * @param alignment the alignment + * @return the rule + */ + public static AlignmentRule constant(int alignment) { + Preconditions.checkArgument(alignment > 0, "alignment <= 0"); + + return (String path) -> alignment; + } + + /** + * A rule that defines constant alignment for all files with a certain suffix, placing no + * restrictions on other files. + * + * @param suffix the suffix + * @param alignment the alignment for paths that match the provided suffix + * @return the rule + */ + public static AlignmentRule constantForSuffix(@Nonnull String suffix, int alignment) { + Preconditions.checkArgument(!suffix.isEmpty(), "suffix.isEmpty()"); + Preconditions.checkArgument(alignment > 0, "alignment <= 0"); + + return (String path) -> path.endsWith(suffix) ? alignment : AlignmentRule.NO_ALIGNMENT; + } + + /** + * A rule that applies other rules in order. + * + * @param rules all rules to be tried; the first rule that does not return + * {@link AlignmentRule#NO_ALIGNMENT} will define the alignment for a path; if there are no + * rules that return a value different from {@link AlignmentRule#NO_ALIGNMENT}, then + * {@link AlignmentRule#NO_ALIGNMENT} is returned + * @return the composition rule + */ + public static AlignmentRule compose(@Nonnull AlignmentRule... rules) { + return (String path) -> { + for (AlignmentRule r : rules) { + int align = r.alignment(path); + if (align != AlignmentRule.NO_ALIGNMENT) { + return align; + } + } + + return AlignmentRule.NO_ALIGNMENT; + }; + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/zip/CentralDirectory.java b/src/main/java/com/android/tools/build/apkzlib/zip/CentralDirectory.java new file mode 100644 index 0000000..909f2d0 --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zip/CentralDirectory.java @@ -0,0 +1,489 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zip; + +import com.android.tools.build.apkzlib.utils.CachedSupplier; +import com.android.tools.build.apkzlib.zip.utils.MsDosDateTimeUtils; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.primitives.Ints; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; +import java.util.Set; +import javax.annotation.Nonnull; + +/** + * Representation of the central directory of a zip archive. + */ +class CentralDirectory { + + /** + * Field in the central directory with the central directory signature. + */ + private static final ZipField.F4 F_SIGNATURE = new ZipField.F4(0, 0x02014b50, "Signature"); + + /** + * Field in the central directory with the "made by" code. + */ + private static final ZipField.F2 F_MADE_BY = new ZipField.F2(F_SIGNATURE.endOffset(), + "Made by", new ZipFieldInvariantNonNegative()); + + /** + * Field in the central directory with the minimum version required to extract the entry. + */ + @VisibleForTesting + static final ZipField.F2 F_VERSION_EXTRACT = new ZipField.F2(F_MADE_BY.endOffset(), + "Version to extract", new ZipFieldInvariantNonNegative()); + + /** + * Field in the central directory with the GP bit flag. + */ + private static final ZipField.F2 F_GP_BIT = new ZipField.F2(F_VERSION_EXTRACT.endOffset(), + "GP bit"); + + /** + * Field in the central directory with the code of the compression method. See + * {@link CompressionMethod#fromCode(long)}. + */ + private static final ZipField.F2 F_METHOD = new ZipField.F2(F_GP_BIT.endOffset(), "Method"); + + /** + * Field in the central directory with the last modification time in MS-DOS format (see + * {@link MsDosDateTimeUtils#packTime(long)}). + */ + private static final ZipField.F2 F_LAST_MOD_TIME = new ZipField.F2(F_METHOD.endOffset(), + "Last modification time"); + + /** + * Field in the central directory with the last modification date in MS-DOS format. See + * {@link MsDosDateTimeUtils#packDate(long)}. + */ + private static final ZipField.F2 F_LAST_MOD_DATE = new ZipField.F2(F_LAST_MOD_TIME.endOffset(), + "Last modification date"); + + /** + * Field in the central directory with the CRC32 checksum of the entry. This will be zero for + * directories and files with no content. + */ + private static final ZipField.F4 F_CRC32 = new ZipField.F4(F_LAST_MOD_DATE.endOffset(), + "CRC32"); + + /** + * Field in the central directory with the entry's compressed size, <em>i.e.</em>, the file on + * the archive. This will be the same as the uncompressed size if the method is + * {@link CompressionMethod#STORE}. + */ + private static final ZipField.F4 F_COMPRESSED_SIZE = new ZipField.F4(F_CRC32.endOffset(), + "Compressed size", new ZipFieldInvariantNonNegative()); + + /** + * Field in the central directory with the entry's uncompressed size, <em>i.e.</em>, the size + * the file will have when extracted from the zip. This will be zero for directories and empty + * files and will be the same as the compressed size if the method is + * {@link CompressionMethod#STORE}. + */ + private static final ZipField.F4 F_UNCOMPRESSED_SIZE = new ZipField.F4( + F_COMPRESSED_SIZE.endOffset(), "Uncompressed size", new ZipFieldInvariantNonNegative()); + + /** + * Field in the central directory with the length of the file name. The file name is stored + * after the offset field ({@link #F_OFFSET}). The number of characters in the file name are + * stored in this field. + */ + private static final ZipField.F2 F_FILE_NAME_LENGTH = new ZipField.F2( + F_UNCOMPRESSED_SIZE.endOffset(), "File name length", + new ZipFieldInvariantNonNegative()); + + /** + * Field in the central directory with the length of the extra field. The extra field is + * stored after the file name ({@link #F_FILE_NAME_LENGTH}). The contents of this field are + * partially defined in the zip specification but we do not parse it. + */ + private static final ZipField.F2 F_EXTRA_FIELD_LENGTH = new ZipField.F2( + F_FILE_NAME_LENGTH.endOffset(), "Extra field length", + new ZipFieldInvariantNonNegative()); + + /** + * Field in the central directory with the length of the comment. The comment is stored after + * the extra field ({@link #F_EXTRA_FIELD_LENGTH}). We do not parse the comment. + */ + private static final ZipField.F2 F_COMMENT_LENGTH = new ZipField.F2( + F_EXTRA_FIELD_LENGTH.endOffset(), "Comment length", new ZipFieldInvariantNonNegative()); + + /** + * Number of the disk where the central directory starts. Because we do not support multi-file + * archives, this field has to have value {@code 0}. + */ + private static final ZipField.F2 F_DISK_NUMBER_START = new ZipField.F2( + F_COMMENT_LENGTH.endOffset(), 0, "Disk start"); + + /** + * Internal attributes. This field can only contain one bit set, the {@link #ASCII_BIT}. + */ + private static final ZipField.F2 F_INTERNAL_ATTRIBUTES = new ZipField.F2( + F_DISK_NUMBER_START.endOffset(), "Int attributes"); + + /** + * External attributes. This field is ignored. + */ + private static final ZipField.F4 F_EXTERNAL_ATTRIBUTES = new ZipField.F4( + F_INTERNAL_ATTRIBUTES.endOffset(), "Ext attributes"); + + /** + * Offset into the archive where the entry starts. This is the offset to the local header + * (see {@link StoredEntry} for information on the local header), not to the file data itself. + * The file data, if there is any, will be stored after the local header. + */ + private static final ZipField.F4 F_OFFSET = new ZipField.F4(F_EXTERNAL_ATTRIBUTES.endOffset(), + "Offset", new ZipFieldInvariantNonNegative()); + + /** + * Maximum supported version to extract. + */ + private static final int MAX_VERSION_TO_EXTRACT = 20; + + /** + * Bit that can be set on the internal attributes stating that the file is an ASCII file. We + * don't do anything with this information, but we check that nothing unexpected appears in the + * internal attributes. + */ + private static final int ASCII_BIT = 1; + + /** + * Contains all entries in the directory mapped from their names. + */ + @Nonnull + private final Map<String, StoredEntry> entries; + + /** + * The file where this directory belongs to. + */ + @Nonnull + private final ZFile file; + + /** + * Supplier that provides a byte representation of the central directory. + */ + @Nonnull + private final CachedSupplier<byte[]> bytesSupplier; + + /** + * Verify log for the central directory. + */ + @Nonnull + private final VerifyLog verifyLog; + + /** + * Creates a new, empty, central directory, for a given zip file. + * + * @param file the file + */ + CentralDirectory(@Nonnull ZFile file) { + entries = Maps.newHashMap(); + this.file = file; + bytesSupplier = new CachedSupplier<>(this::computeByteRepresentation); + verifyLog = file.getVerifyLog(); + } + + /** + * Reads the central directory data from a zip file, parses it, and creates the in-memory + * structure representing the directory. + * + * @param bytes the data of the central directory; the directory is read from the buffer's + * current position; when this method terminates, the buffer's position is the first byte + * after the directory + * @param count the number of entries expected in the central directory (usually read from the + * {@link Eocd}). + * @param file the zip file this central directory belongs to + * @return the central directory + * @throws IOException failed to read data from the zip, or the central directory is corrupted + * or has unsupported features + */ + static CentralDirectory makeFromData(@Nonnull ByteBuffer bytes, int count, @Nonnull ZFile file) + throws IOException { + Preconditions.checkNotNull(bytes, "bytes == null"); + Preconditions.checkArgument(count >= 0, "count < 0"); + + CentralDirectory directory = new CentralDirectory(file); + + for (int i = 0; i < count; i++) { + try { + directory.readEntry(bytes); + } catch (IOException e) { + throw new IOException( + "Failed to read directory entry index " + + i + + " (total " + + "directory bytes read: " + + bytes.position() + + ").", + e); + } + } + + return directory; + } + + /** + * Creates a new central directory from the entries. This is used to build a new central + * directory from entries in the zip file. + * + * @param entries the entries in the zip file + * @param file the zip file itself + * @return the created central directory + */ + static CentralDirectory makeFromEntries( + @Nonnull Set<StoredEntry> entries, + @Nonnull ZFile file) { + CentralDirectory directory = new CentralDirectory(file); + for (StoredEntry entry : entries) { + CentralDirectoryHeader cdr = entry.getCentralDirectoryHeader(); + Preconditions.checkArgument( + !directory.entries.containsKey(cdr.getName()), + "Duplicate filename"); + directory.entries.put(cdr.getName(), entry); + } + + return directory; + } + + /** + * Reads the next entry from the central directory and adds it to {@link #entries}. + * + * @param bytes the central directory's data, positioned starting at the beginning of the next + * entry to read; when finished, the buffer's position will be at the first byte after the + * entry + * @throws IOException failed to read the directory entry, either because of an I/O error, + * because it is corrupt or contains unsupported features + */ + private void readEntry(@Nonnull ByteBuffer bytes) throws IOException { + F_SIGNATURE.verify(bytes); + long madeBy = F_MADE_BY.read(bytes); + + long versionNeededToExtract = F_VERSION_EXTRACT.read(bytes); + verifyLog.verify( + versionNeededToExtract <= MAX_VERSION_TO_EXTRACT, + "Ignored unknown version needed to extract in zip directory entry: %s.", + versionNeededToExtract); + + long gpBit = F_GP_BIT.read(bytes); + GPFlags flags = GPFlags.from(gpBit); + + long methodCode = F_METHOD.read(bytes); + CompressionMethod method = CompressionMethod.fromCode(methodCode); + verifyLog.verify(method != null, "Unknown method in zip directory entry: %s.", methodCode); + + long lastModTime; + long lastModDate; + if (file.areTimestampsIgnored()) { + lastModTime = 0; + lastModDate = 0; + F_LAST_MOD_TIME.skip(bytes); + F_LAST_MOD_DATE.skip(bytes); + } else { + lastModTime = F_LAST_MOD_TIME.read(bytes); + lastModDate = F_LAST_MOD_DATE.read(bytes); + } + + long crc32 = F_CRC32.read(bytes); + long compressedSize = F_COMPRESSED_SIZE.read(bytes); + long uncompressedSize = F_UNCOMPRESSED_SIZE.read(bytes); + int fileNameLength = Ints.checkedCast(F_FILE_NAME_LENGTH.read(bytes)); + int extraFieldLength = Ints.checkedCast(F_EXTRA_FIELD_LENGTH.read(bytes)); + int fileCommentLength = Ints.checkedCast(F_COMMENT_LENGTH.read(bytes)); + + F_DISK_NUMBER_START.verify(bytes, verifyLog); + long internalAttributes = F_INTERNAL_ATTRIBUTES.read(bytes); + verifyLog.verify( + (internalAttributes & ~ASCII_BIT) == 0, + "Ignored invalid internal attributes: %s.", + internalAttributes); + + long externalAttributes = F_EXTERNAL_ATTRIBUTES.read(bytes); + long entryOffset = F_OFFSET.read(bytes); + + long remainingSize = fileNameLength + extraFieldLength + fileCommentLength; + + if (bytes.remaining() < fileNameLength + extraFieldLength + fileCommentLength) { + throw new IOException( + "Directory entry should have " + + remainingSize + + " bytes remaining (name = " + + fileNameLength + + ", extra = " + + extraFieldLength + + ", comment = " + + fileCommentLength + + "), but it has " + + bytes.remaining() + + "."); + } + + byte[] encodedFileName = new byte[fileNameLength]; + bytes.get(encodedFileName); + String fileName = EncodeUtils.decode(encodedFileName, flags); + + byte[] extraField = new byte[extraFieldLength]; + bytes.get(extraField); + + byte[] fileCommentField = new byte[fileCommentLength]; + bytes.get(fileCommentField); + + /* + * Tricky: to create a CentralDirectoryHeader we need the future that will hold the result + * of the compress information. But, to actually create the result of the compress + * information we need the CentralDirectoryHeader + */ + ListenableFuture<CentralDirectoryHeaderCompressInfo> compressInfo = + Futures.immediateFuture( + new CentralDirectoryHeaderCompressInfo( + method, + compressedSize, + versionNeededToExtract)); + CentralDirectoryHeader centralDirectoryHeader = + new CentralDirectoryHeader( + fileName, encodedFileName, uncompressedSize, compressInfo, flags, file); + centralDirectoryHeader.setMadeBy(madeBy); + centralDirectoryHeader.setLastModTime(lastModTime); + centralDirectoryHeader.setLastModDate(lastModDate); + centralDirectoryHeader.setCrc32(crc32); + centralDirectoryHeader.setInternalAttributes(internalAttributes); + centralDirectoryHeader.setExternalAttributes(externalAttributes); + centralDirectoryHeader.setOffset(entryOffset); + centralDirectoryHeader.setExtraFieldNoNotify(new ExtraField(extraField)); + centralDirectoryHeader.setComment(fileCommentField); + + StoredEntry entry; + + try { + entry = new StoredEntry(centralDirectoryHeader, file, null); + } catch (IOException e) { + throw new IOException("Failed to read stored entry '" + fileName + "'.", e); + } + + if (entries.containsKey(fileName)) { + verifyLog.log("File file contains duplicate file '" + fileName + "'."); + } + + entries.put(fileName, entry); + } + + /** + * Obtains all the entries in the central directory. + * + * @return all entries on a non-modifiable map + */ + @Nonnull + Map<String, StoredEntry> getEntries() { + return ImmutableMap.copyOf(entries); + } + + /** + * Obtains the byte representation of the central directory. + * + * @return a byte array containing the whole central directory + * @throws IOException failed to write the byte array + */ + byte[] toBytes() throws IOException { + return bytesSupplier.get(); + } + + /** + * Computes the byte representation of the central directory. + * + * @return a byte array containing the whole central directory + * @throws UncheckedIOException failed to write the byte array + */ + private byte[] computeByteRepresentation() { + + List<StoredEntry> sorted = Lists.newArrayList(entries.values()); + sorted.sort(StoredEntry.COMPARE_BY_NAME); + + CentralDirectoryHeader[] cdhs = new CentralDirectoryHeader[entries.size()]; + CentralDirectoryHeaderCompressInfo[] compressInfos = + new CentralDirectoryHeaderCompressInfo[entries.size()]; + byte[][] encodedFileNames = new byte[entries.size()][]; + byte[][] extraFields = new byte[entries.size()][]; + byte[][] comments = new byte[entries.size()][]; + + try { + /* + * First collect all the data and compute the total size of the central directory. + */ + int idx = 0; + int total = 0; + for (StoredEntry entry : sorted) { + cdhs[idx] = entry.getCentralDirectoryHeader(); + compressInfos[idx] = cdhs[idx].getCompressionInfoWithWait(); + encodedFileNames[idx] = cdhs[idx].getEncodedFileName(); + extraFields[idx] = new byte[cdhs[idx].getExtraField().size()]; + cdhs[idx].getExtraField().write(ByteBuffer.wrap(extraFields[idx])); + comments[idx] = cdhs[idx].getComment(); + + total += F_OFFSET.endOffset() + encodedFileNames[idx].length + + extraFields[idx].length + comments[idx].length; + idx++; + } + + ByteBuffer out = ByteBuffer.allocate(total); + + for (idx = 0; idx < entries.size(); idx++) { + F_SIGNATURE.write(out); + F_MADE_BY.write(out, cdhs[idx].getMadeBy()); + F_VERSION_EXTRACT.write(out, compressInfos[idx].getVersionExtract()); + F_GP_BIT.write(out, cdhs[idx].getGpBit().getValue()); + F_METHOD.write(out, compressInfos[idx].getMethod().methodCode); + + if (file.areTimestampsIgnored()) { + F_LAST_MOD_TIME.write(out, 0); + F_LAST_MOD_DATE.write(out, 0); + } else { + F_LAST_MOD_TIME.write(out, cdhs[idx].getLastModTime()); + F_LAST_MOD_DATE.write(out, cdhs[idx].getLastModDate()); + } + + F_CRC32.write(out, cdhs[idx].getCrc32()); + F_COMPRESSED_SIZE.write(out, compressInfos[idx].getCompressedSize()); + F_UNCOMPRESSED_SIZE.write(out, cdhs[idx].getUncompressedSize()); + + F_FILE_NAME_LENGTH.write(out, cdhs[idx].getEncodedFileName().length); + F_EXTRA_FIELD_LENGTH.write(out, cdhs[idx].getExtraField().size()); + F_COMMENT_LENGTH.write(out, cdhs[idx].getComment().length); + F_DISK_NUMBER_START.write(out); + F_INTERNAL_ATTRIBUTES.write(out, cdhs[idx].getInternalAttributes()); + F_EXTERNAL_ATTRIBUTES.write(out, cdhs[idx].getExternalAttributes()); + F_OFFSET.write(out, cdhs[idx].getOffset()); + + out.put(encodedFileNames[idx]); + out.put(extraFields[idx]); + out.put(comments[idx]); + } + + return out.array(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/zip/CentralDirectoryHeader.java b/src/main/java/com/android/tools/build/apkzlib/zip/CentralDirectoryHeader.java new file mode 100644 index 0000000..353ed3d --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zip/CentralDirectoryHeader.java @@ -0,0 +1,434 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zip; + +import com.android.tools.build.apkzlib.zip.utils.MsDosDateTimeUtils; +import com.google.common.base.Verify; +import java.io.IOException; +import java.util.Arrays; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import javax.annotation.Nonnull; + +/** + * The Central Directory Header contains information about files stored in the zip. Instances of + * this class contain information for files that already are in the zip and, for which the data was + * read from the Central Directory. But some instances of this class are used for new files. + * Because instances of this class can refer to files not yet on the zip, some of the fields may + * not be filled in, or may be filled in with default values. + * <p> + * Because compression decision is done lazily, some data is stored with futures. + */ +public class CentralDirectoryHeader implements Cloneable { + + /** + * Default "version made by" field: upper byte needs to be 0 to set to MS-DOS compatibility. + * Lower byte can be anything, really. We use 18 because aapt uses 17 :) + */ + private static final int DEFAULT_VERSION_MADE_BY = 0x0018; + + /** + * Name of the file. + */ + @Nonnull + private String name; + + /** + * CRC32 of the data. 0 if not yet computed. + */ + private long crc32; + + /** + * Size of the file uncompressed. 0 if the file has no data. + */ + private long uncompressedSize; + + /** + * Code of the program that made the zip. We actually don't care about this. + */ + private long madeBy; + + /** + * General-purpose bit flag. + */ + @Nonnull + private GPFlags gpBit; + + /** + * Last modification time in MS-DOS format (see {@link MsDosDateTimeUtils#packTime(long)}). + */ + private long lastModTime; + + /** + * Last modification time in MS-DOS format (see {@link MsDosDateTimeUtils#packDate(long)}). + */ + private long lastModDate; + + /** + * Extra data field contents. This field follows a specific structure according to the + * specification. + */ + @Nonnull + private ExtraField extraField; + + /** + * File comment. + */ + @Nonnull + private byte[] comment; + + /** + * File internal attributes. + */ + private long internalAttributes; + + /** + * File external attributes. + */ + private long externalAttributes; + + /** + * Offset in the file where the data is located. This will be -1 if the header corresponds to + * a new file that is not yet written in the zip and, therefore, has no written data. + */ + private long offset; + + /** + * Encoded file name. + */ + private byte[] encodedFileName; + + /** + * Compress information that may not have been computed yet due to lazy compression. + */ + @Nonnull + private Future<CentralDirectoryHeaderCompressInfo> compressInfo; + + /** + * The file this header belongs to. + */ + @Nonnull + private final ZFile file; + + /** + * Creates data for a file. + * + * @param name the file name + * @param encodedFileName the encoded file name, this array will be owned by the header + * @param uncompressedSize the uncompressed file size + * @param compressInfo computation that defines the compression information + * @param flags flags used in the entry + * @param zFile the file this header belongs to + */ + CentralDirectoryHeader( + @Nonnull String name, + @Nonnull byte[] encodedFileName, + long uncompressedSize, + @Nonnull Future<CentralDirectoryHeaderCompressInfo> compressInfo, + @Nonnull GPFlags flags, + @Nonnull ZFile zFile) { + this.name = name; + this.uncompressedSize = uncompressedSize; + crc32 = 0; + + /* + * Set sensible defaults for the rest. + */ + madeBy = DEFAULT_VERSION_MADE_BY; + + gpBit = flags; + lastModTime = MsDosDateTimeUtils.packCurrentTime(); + lastModDate = MsDosDateTimeUtils.packCurrentDate(); + extraField = new ExtraField(); + comment = new byte[0]; + internalAttributes = 0; + externalAttributes = 0; + offset = -1; + this.encodedFileName = encodedFileName; + this.compressInfo = compressInfo; + file = zFile; + } + + /** + * Obtains the name of the file. + * + * @return the name + */ + @Nonnull + public String getName() { + return name; + } + + /** + * Obtains the size of the uncompressed file. + * + * @return the size of the file + */ + public long getUncompressedSize() { + return uncompressedSize; + } + + /** + * Obtains the CRC32 of the data. + * + * @return the CRC32, 0 if not yet computed + */ + public long getCrc32() { + return crc32; + } + + /** + * Sets the CRC32 of the data. + * + * @param crc32 the CRC 32 + */ + void setCrc32(long crc32) { + this.crc32 = crc32; + } + + /** + * Obtains the code of the program that made the zip. + * + * @return the code + */ + public long getMadeBy() { + return madeBy; + } + + /** + * Sets the code of the progtram that made the zip. + * + * @param madeBy the code + */ + void setMadeBy(long madeBy) { + this.madeBy = madeBy; + } + + /** + * Obtains the general-purpose bit flag. + * + * @return the bit flag + */ + @Nonnull + public GPFlags getGpBit() { + return gpBit; + } + + /** + * Obtains the last modification time of the entry. + * + * @return the last modification time in MS-DOS format (see + * {@link MsDosDateTimeUtils#packTime(long)}) + */ + public long getLastModTime() { + return lastModTime; + } + + /** + * Sets the last modification time of the entry. + * + * @param lastModTime the last modification time in MS-DOS format (see + * {@link MsDosDateTimeUtils#packTime(long)}) + */ + void setLastModTime(long lastModTime) { + this.lastModTime = lastModTime; + } + + /** + * Obtains the last modification date of the entry. + * + * @return the last modification date in MS-DOS format (see + * {@link MsDosDateTimeUtils#packDate(long)}) + */ + public long getLastModDate() { + return lastModDate; + } + + /** + * Sets the last modification date of the entry. + * + * @param lastModDate the last modification date in MS-DOS format (see + * {@link MsDosDateTimeUtils#packDate(long)}) + */ + void setLastModDate(long lastModDate) { + this.lastModDate = lastModDate; + } + + /** + * Obtains the data in the extra field. + * + * @return the data (returns an empty array if there is none) + */ + @Nonnull + public ExtraField getExtraField() { + return extraField; + } + + /** + * Sets the data in the extra field. + * + * @param extraField the data to set + */ + public void setExtraField(@Nonnull ExtraField extraField) { + setExtraFieldNoNotify(extraField); + file.centralDirectoryChanged(); + } + + /** + * Sets the data in the extra field, but does not notify {@link ZFile}. This method is invoked + * when the {@link ZFile} knows the extra field is being set. + * + * @param extraField the data to set + */ + void setExtraFieldNoNotify(@Nonnull ExtraField extraField) { + this.extraField = extraField; + } + + /** + * Obtains the entry's comment. + * + * @return the comment (returns an empty array if there is no comment) + */ + @Nonnull + public byte[] getComment() { + return comment; + } + + /** + * Sets the entry's comment. + * + * @param comment the comment + */ + void setComment(@Nonnull byte[] comment) { + this.comment = comment; + } + + /** + * Obtains the entry's internal attributes. + * + * @return the entry's internal attributes + */ + public long getInternalAttributes() { + return internalAttributes; + } + + /** + * Sets the entry's internal attributes. + * + * @param internalAttributes the entry's internal attributes + */ + void setInternalAttributes(long internalAttributes) { + this.internalAttributes = internalAttributes; + } + + /** + * Obtains the entry's external attributes. + * + * @return the entry's external attributes + */ + public long getExternalAttributes() { + return externalAttributes; + } + + /** + * Sets the entry's external attributes. + * + * @param externalAttributes the entry's external attributes + */ + void setExternalAttributes(long externalAttributes) { + this.externalAttributes = externalAttributes; + } + + /** + * Obtains the offset in the zip file where this entry's data is. + * + * @return the offset or {@code -1} if the file has no data in the zip and, therefore, data + * is stored in memory + */ + public long getOffset() { + return offset; + } + + /** + * Sets the offset in the zip file where this entry's data is. + * + * @param offset the offset or {@code -1} if the file is new and has no data in the zip yet + */ + void setOffset(long offset) { + this.offset = offset; + } + + /** + * Obtains the encoded file name. + * + * @return the encoded file name + */ + public byte[] getEncodedFileName() { + return encodedFileName; + } + + /** + * Resets the deferred CRC flag in the GP flags. + */ + void resetDeferredCrc() { + /* + * We actually create a new set of flags. Since the only information we care about is the + * UTF-8 encoding, we'll just create a brand new object. + */ + gpBit = GPFlags.make(gpBit.isUtf8FileName()); + } + + @Override + protected CentralDirectoryHeader clone() throws CloneNotSupportedException { + CentralDirectoryHeader cdr = (CentralDirectoryHeader) super.clone(); + cdr.extraField = extraField; + cdr.comment = Arrays.copyOf(comment, comment.length); + cdr.encodedFileName = Arrays.copyOf(encodedFileName, encodedFileName.length); + return cdr; + } + + /** + * Obtains the future with the compression information. + * + * @return the information + */ + @Nonnull + public Future<CentralDirectoryHeaderCompressInfo> getCompressionInfo() { + return compressInfo; + } + + /** + * Equivalent to {@code getCompressionInfo().get()} but masking the possible exceptions and + * guaranteeing non-{@code null} return. + * + * @return the result of the future + * @throws IOException failed to get the information + */ + @Nonnull + public CentralDirectoryHeaderCompressInfo getCompressionInfoWithWait() + throws IOException { + try { + CentralDirectoryHeaderCompressInfo info = getCompressionInfo().get(); + Verify.verifyNotNull(info, "info == null"); + return info; + } catch (InterruptedException e) { + throw new IOException("Interrupted while waiting for compression information.", e); + } catch (ExecutionException e) { + throw new IOException("Execution of compression failed.", e); + } + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/zip/CentralDirectoryHeaderCompressInfo.java b/src/main/java/com/android/tools/build/apkzlib/zip/CentralDirectoryHeaderCompressInfo.java new file mode 100644 index 0000000..f6fd749 --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zip/CentralDirectoryHeaderCompressInfo.java @@ -0,0 +1,119 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zip; + +import javax.annotation.Nonnull; + +/** + * Information stored in the {@link CentralDirectoryHeader} that is related to compression and may + * need to be computed lazily. + */ +public class CentralDirectoryHeaderCompressInfo { + + /** + * Version of zip file that only supports stored files. + */ + public static final long VERSION_WITH_STORE_FILES_ONLY = 10L; + + /** + * Version of zip file that only supports directories and deflated files. + */ + public static final long VERSION_WITH_DIRECTORIES_AND_DEFLATE = 20L; + + /** + * The compression method. + */ + @Nonnull + private final CompressionMethod mMethod; + + /** + * Size of the file compressed. 0 if the file has no data. + */ + private final long compressedSize; + + /** + * Version needed to extract the zip. + */ + private final long versionExtract; + + /** + * Creates new compression information for the central directory header. + * + * @param method the compression method + * @param compressedSize the compressed size + * @param versionToExtract minimum version to extract (typically + * {@link #VERSION_WITH_STORE_FILES_ONLY} or {@link #VERSION_WITH_DIRECTORIES_AND_DEFLATE}) + */ + public CentralDirectoryHeaderCompressInfo( + @Nonnull CompressionMethod method, + long compressedSize, + long versionToExtract) { + mMethod = method; + this.compressedSize = compressedSize; + versionExtract = versionToExtract; + } + + /** + * Creates new compression information for the central directory header. + * + * @param header the header this information relates to + * @param method the compression method + * @param compressedSize the compressed size + */ + public CentralDirectoryHeaderCompressInfo(@Nonnull CentralDirectoryHeader header, + @Nonnull CompressionMethod method, long compressedSize) { + mMethod = method; + this.compressedSize = compressedSize; + + if (header.getName().endsWith("/") || method == CompressionMethod.DEFLATE) { + /* + * Directories and compressed files only in version 2.0. + */ + versionExtract = VERSION_WITH_DIRECTORIES_AND_DEFLATE; + } else { + versionExtract = VERSION_WITH_STORE_FILES_ONLY; + } + } + + /** + * Obtains the compression data size. + * + * @return the compressed data size + */ + public long getCompressedSize() { + return compressedSize; + } + + /** + * Obtains the compression method. + * + * @return the compression method + */ + @Nonnull + public CompressionMethod getMethod() { + return mMethod; + } + + /** + * Obtains the minimum version for extract. + * + * @return the minimum version + */ + public long getVersionExtract() { + return versionExtract; + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/zip/CompressionMethod.java b/src/main/java/com/android/tools/build/apkzlib/zip/CompressionMethod.java new file mode 100644 index 0000000..82f374b --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zip/CompressionMethod.java @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zip; + +import javax.annotation.Nullable; + +/** + * Enumeration with all known compression methods. + */ +public enum CompressionMethod { + /** + * STORE method: data is stored without any compression. + */ + STORE(0), + + /** + * DEFLATE method: data is stored compressed using the DEFLATE algorithm. + */ + DEFLATE(8); + + /** + * Code, within the zip file, that identifies this compression method. + */ + int methodCode; + + /** + * Creates a new compression method. + * + * @param methodCode the code used in the zip file that identifies the compression method + */ + CompressionMethod(int methodCode) { + this.methodCode = methodCode; + } + + /** + * Obtains the compression method that corresponds to the provided code. + * + * @param code the code + * @return the method or {@code null} if no method has the provided code + */ + @Nullable + static CompressionMethod fromCode(long code) { + for (CompressionMethod method : values()) { + if (method.methodCode == code) { + return method; + } + } + + return null; + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/zip/CompressionResult.java b/src/main/java/com/android/tools/build/apkzlib/zip/CompressionResult.java new file mode 100644 index 0000000..1688248 --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zip/CompressionResult.java @@ -0,0 +1,86 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zip; + +import com.android.tools.build.apkzlib.zip.utils.CloseableByteSource; +import javax.annotation.Nonnull; + +/** + * Result of compressing data. + */ +public class CompressionResult { + + /** + * The compression method used. + */ + @Nonnull + private final CompressionMethod compressionMethod; + + /** + * The resulting data. + */ + @Nonnull + private final CloseableByteSource source; + + /** + * Size of the compressed source. Kept because {@code source.size()} can throw + * {@code IOException}. + */ + private final long mSize; + + /** + * Creates a new compression result. + * + * @param source the data source + * @param method the compression method + */ + public CompressionResult(@Nonnull CloseableByteSource source, @Nonnull CompressionMethod method, + long size) { + compressionMethod = method; + this.source = source; + mSize = size; + } + + /** + * Obtains the compression method. + * + * @return the compression method + */ + @Nonnull + public CompressionMethod getCompressionMethod() { + return compressionMethod; + } + + /** + * Obtains the compressed data. + * + * @return the data, the resulting array should not be modified + */ + @Nonnull + public CloseableByteSource getSource() { + return source; + } + + /** + * Obtains the size of the compression result. + * + * @return the size + */ + public long getSize() { + return mSize; + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/zip/Compressor.java b/src/main/java/com/android/tools/build/apkzlib/zip/Compressor.java new file mode 100644 index 0000000..9c70129 --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zip/Compressor.java @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zip; + +import com.android.tools.build.apkzlib.zip.utils.CloseableByteSource; +import com.google.common.util.concurrent.ListenableFuture; +import javax.annotation.Nonnull; + +/** + * A compressor is capable of, well, compressing data. Data is read from an {@code ByteSource}. + * Compressors are asynchronous: compressing results in a {@code ListenableFuture} that will contain + * the compression result. + */ +public interface Compressor { + + /** + * Compresses an entry source. + * + * @param source the source to compress + * @return a future that will eventually contain the compression result + */ + @Nonnull + ListenableFuture<CompressionResult> compress(@Nonnull CloseableByteSource source); +} diff --git a/src/main/java/com/android/tools/build/apkzlib/zip/DataDescriptorType.java b/src/main/java/com/android/tools/build/apkzlib/zip/DataDescriptorType.java new file mode 100644 index 0000000..d7d9086 --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zip/DataDescriptorType.java @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zip; + +/** + * Type of data descriptor that an entry has. Data descriptors are used if the CRC and sizing data + * is not known when the data is being written and cannot be placed in the file's local header. + * In those cases, after the file data itself, a data descriptor is placed after the entry's + * contents. + * <p> + * While the zip specification says the data descriptor should be used but it is optional. We + * record also whether the data descriptor contained the 4-byte signature at the start of the + * block or not. + */ +public enum DataDescriptorType { + /** + * The entry has no data descriptor. + */ + NO_DATA_DESCRIPTOR(0), + + /** + * The entry has a data descriptor that does not contain a signature. + */ + DATA_DESCRIPTOR_WITHOUT_SIGNATURE(12), + + /** + * The entry has a data descriptor that contains a signature. + */ + DATA_DESCRIPTOR_WITH_SIGNATURE(16); + + /** + * The number of bytes the data descriptor spans. + */ + public int size; + + /** + * Creates a new data descriptor. + * + * @param size the number of bytes the data descriptor spans + */ + DataDescriptorType(int size) { + this.size = size; + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/zip/EncodeUtils.java b/src/main/java/com/android/tools/build/apkzlib/zip/EncodeUtils.java new file mode 100644 index 0000000..34111e6 --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zip/EncodeUtils.java @@ -0,0 +1,139 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zip; + +import com.google.common.base.Charsets; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.Charset; +import java.nio.charset.CodingErrorAction; +import javax.annotation.Nonnull; + +/** + * Utilities to encode and decode file names in zips. + */ +public class EncodeUtils { + + /** + * Utility class: no constructor. + */ + private EncodeUtils() { + /* + * Nothing to do. + */ + } + + /** + * Decodes a file name. + * + * @param bytes the raw data buffer to read from + * @param length the number of bytes in the raw data buffer containing the string to decode + * @param flags the zip entry flags + * @return the decode file name + */ + @Nonnull + public static String decode(@Nonnull ByteBuffer bytes, int length, @Nonnull GPFlags flags) + throws IOException { + if (bytes.remaining() < length) { + throw new IOException("Only " + bytes.remaining() + " bytes exist in the buffer, but " + + "length is " + length + "."); + } + + byte[] stringBytes = new byte[length]; + bytes.get(stringBytes); + return decode(stringBytes, flags); + } + + /** + * Decodes a file name. + * + * @param data the raw data + * @param flags the zip entry flags + * @return the decode file name + */ + @Nonnull + public static String decode(@Nonnull byte[] data, @Nonnull GPFlags flags) { + return decode(data, flagsCharset(flags)); + } + + /** + * Decodes a file name. + * + * @param data the raw data + * @param charset the charset to use + * @return the decode file name + */ + @Nonnull + private static String decode(@Nonnull byte[] data, @Nonnull Charset charset) { + try { + return charset.newDecoder() + .onMalformedInput(CodingErrorAction.REPORT) + .decode(ByteBuffer.wrap(data)) + .toString(); + } catch (CharacterCodingException e) { + // If we're trying to decode ASCII, try UTF-8. Otherwise, revert to the default + // behavior (usually replacing invalid characters). + if (charset.equals(Charsets.US_ASCII)) { + return decode(data, Charsets.UTF_8); + } else { + return charset.decode(ByteBuffer.wrap(data)).toString(); + } + } + } + + /** + * Encodes a file name. + * + * @param name the name to encode + * @param flags the zip entry flags + * @return the encoded file name + */ + @Nonnull + public static byte[] encode(@Nonnull String name, @Nonnull GPFlags flags) { + Charset charset = flagsCharset(flags); + ByteBuffer bytes = charset.encode(name); + byte[] result = new byte[bytes.remaining()]; + bytes.get(result); + return result; + } + + /** + * Obtains the charset to encode and decode zip entries, given a set of flags. + * + * @param flags the flags + * @return the charset to use + */ + @Nonnull + private static Charset flagsCharset(@Nonnull GPFlags flags) { + if (flags.isUtf8FileName()) { + return Charsets.UTF_8; + } else { + return Charsets.US_ASCII; + } + } + + /** + * Checks if some text may be encoded using ASCII. + * + * @param text the text to check + * @return can it be encoded using ASCII? + */ + public static boolean canAsciiEncode(String text) { + return Charsets.US_ASCII.newEncoder().canEncode(text); + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/zip/Eocd.java b/src/main/java/com/android/tools/build/apkzlib/zip/Eocd.java new file mode 100644 index 0000000..a9da14b --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zip/Eocd.java @@ -0,0 +1,271 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zip; + +import com.android.tools.build.apkzlib.utils.CachedSupplier; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.base.Verify; +import com.google.common.primitives.Ints; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.ByteBuffer; +import javax.annotation.Nonnull; + +/** + * End Of Central Directory record in a zip file. + */ +class Eocd { + /** + * Field in the record: the record signature, fixed at this value by the specification. + */ + private static final ZipField.F4 F_SIGNATURE = new ZipField.F4(0, 0x06054b50, "EOCD signature"); + + /** + * Field in the record: the number of the disk where the EOCD is located. It has to be zero + * because we do not support multi-file archives. + */ + private static final ZipField.F2 F_NUMBER_OF_DISK = new ZipField.F2(F_SIGNATURE.endOffset(), 0, + "Number of this disk"); + + /** + * Field in the record: the number of the disk where the Central Directory starts. Has to be + * zero because we do not support multi-file archives. + */ + private static final ZipField.F2 F_DISK_CD_START = new ZipField.F2(F_NUMBER_OF_DISK.endOffset(), + 0, "Disk where CD starts"); + + /** + * Field in the record: the number of entries in the Central Directory on this disk. Because + * we do not support multi-file archives, this is the same as {@link #F_RECORDS_TOTAL}. + */ + private static final ZipField.F2 F_RECORDS_DISK = new ZipField.F2(F_DISK_CD_START.endOffset(), + "Record on disk count", new ZipFieldInvariantNonNegative()); + + /** + * Field in the record: the total number of entries in the Central Directory. + */ + private static final ZipField.F2 F_RECORDS_TOTAL = new ZipField.F2(F_RECORDS_DISK.endOffset(), + "Total records", new ZipFieldInvariantNonNegative(), + new ZipFieldInvariantMaxValue(Integer.MAX_VALUE)); + + /** + * Field in the record: number of bytes of the Central Directory. + * This is not private because it is required in unit tests. + */ + @VisibleForTesting + static final ZipField.F4 F_CD_SIZE = new ZipField.F4(F_RECORDS_TOTAL.endOffset(), + "Directory size", new ZipFieldInvariantNonNegative()); + + /** + * Field in the record: offset, from the archive start, where the Central Directory starts. + * This is not private because it is required in unit tests. + */ + @VisibleForTesting + static final ZipField.F4 F_CD_OFFSET = new ZipField.F4(F_CD_SIZE.endOffset(), + "Directory offset", new ZipFieldInvariantNonNegative()); + + /** + * Field in the record: number of bytes of the file comment (located at the end of the EOCD + * record). + */ + private static final ZipField.F2 F_COMMENT_SIZE = new ZipField.F2(F_CD_OFFSET.endOffset(), + "File comment size", new ZipFieldInvariantNonNegative()); + + /** + * Number of entries in the central directory. + */ + private final int totalRecords; + + /** + * Offset from the beginning of the archive where the Central Directory is located. + */ + private final long directoryOffset; + + /** + * Number of bytes of the Central Directory. + */ + private final long directorySize; + + /** + * Contents of the EOCD comment. + */ + @Nonnull + private final byte[] comment; + + /** + * Supplier of the byte representation of the EOCD. + */ + @Nonnull + private final CachedSupplier<byte[]> byteSupplier; + + /** + * Creates a new EOCD, reading it from a byte source. This method will parse the byte source + * and obtain the EOCD. It will check that the byte source starts with the EOCD signature. + * + * @param bytes the byte buffer with the EOCD data; when this method finishes, the byte + * buffer's position will have moved to the end of the EOCD + * @throws IOException failed to read information or the EOCD data is corrupt or invalid + */ + Eocd(@Nonnull ByteBuffer bytes) throws IOException { + + /* + * Read the EOCD record. + */ + F_SIGNATURE.verify(bytes); + F_NUMBER_OF_DISK.verify(bytes); + F_DISK_CD_START.verify(bytes); + long totalRecords1 = F_RECORDS_DISK.read(bytes); + long totalRecords2 = F_RECORDS_TOTAL.read(bytes); + long directorySize = F_CD_SIZE.read(bytes); + long directoryOffset = F_CD_OFFSET.read(bytes); + int commentSize = Ints.checkedCast(F_COMMENT_SIZE.read(bytes)); + + /* + * Some sanity checks. + */ + if (totalRecords1 != totalRecords2) { + throw new IOException("Zip states records split in multiple disks, which is not " + + "supported."); + } + + Verify.verify(totalRecords1 <= Integer.MAX_VALUE); + + totalRecords = Ints.checkedCast(totalRecords1); + this.directorySize = directorySize; + this.directoryOffset = directoryOffset; + + if (bytes.remaining() < commentSize) { + throw new IOException("Corrupt EOCD record: not enough data for comment (comment " + + "size is " + commentSize + ")."); + } + + comment = new byte[commentSize]; + bytes.get(comment); + byteSupplier = new CachedSupplier<>(this::computeByteRepresentation); + } + + /** + * Creates a new EOCD. This is used when generating an EOCD for an Central Directory that has + * just been generated. The EOCD will be generated without any comment. + * + * @param totalRecords total number of records in the directory + * @param directoryOffset offset, since beginning of archive, where the Central Directory is + * located + * @param directorySize number of bytes of the Central Directory + * @param comment the EOCD comment + */ + Eocd(int totalRecords, long directoryOffset, long directorySize, @Nonnull byte[] comment) { + Preconditions.checkArgument(totalRecords >= 0, "totalRecords < 0"); + Preconditions.checkArgument(directoryOffset >= 0, "directoryOffset < 0"); + Preconditions.checkArgument(directorySize >= 0, "directorySize < 0"); + + this.totalRecords = totalRecords; + this.directoryOffset = directoryOffset; + this.directorySize = directorySize; + this.comment = comment; + byteSupplier = new CachedSupplier<>(this::computeByteRepresentation); + } + + /** + * Obtains the number of records in the Central Directory. + * + * @return the number of records + */ + int getTotalRecords() { + return totalRecords; + } + + /** + * Obtains the offset since the beginning of the zip archive where the Central Directory is + * located. + * + * @return the offset where the Central Directory is located + */ + long getDirectoryOffset() { + return directoryOffset; + } + + /** + * Obtains the size of the Central Directory. + * + * @return the number of bytes that make up the Central Directory + */ + long getDirectorySize() { + return directorySize; + } + + /** + * Obtains the size of the EOCD. + * + * @return the size, in bytes, of the EOCD + */ + long getEocdSize() { + return (long) F_COMMENT_SIZE.endOffset() + comment.length; + } + + /** + * Generates the EOCD data. + * + * @return a byte representation of the EOCD that has exactly {@link #getEocdSize()} bytes + * @throws IOException failed to generate the EOCD data + */ + @Nonnull + byte[] toBytes() throws IOException { + return byteSupplier.get(); + } + + /* + * Obtains the comment in the EOCD. + * + * @return the comment exactly as it is represented in the file (no encoding conversion is + * done) + */ + @Nonnull + byte[] getComment() { + byte[] commentCopy = new byte[comment.length]; + System.arraycopy(comment, 0, commentCopy, 0, comment.length); + return commentCopy; + } + + /** + * Computes the byte representation of the EOCD. + * + * @return a byte representation of the EOCD that has exactly {@link #getEocdSize()} bytes + * @throws UncheckedIOException failed to generate the EOCD data + */ + @Nonnull + private byte[] computeByteRepresentation() { + ByteBuffer out = ByteBuffer.allocate(F_COMMENT_SIZE.endOffset() + comment.length); + + try { + F_SIGNATURE.write(out); + F_NUMBER_OF_DISK.write(out); + F_DISK_CD_START.write(out); + F_RECORDS_DISK.write(out, totalRecords); + F_RECORDS_TOTAL.write(out, totalRecords); + F_CD_SIZE.write(out, directorySize); + F_CD_OFFSET.write(out, directoryOffset); + F_COMMENT_SIZE.write(out, comment.length); + out.put(comment); + + return out.array(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/zip/ExtraField.java b/src/main/java/com/android/tools/build/apkzlib/zip/ExtraField.java new file mode 100644 index 0000000..aa41491 --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zip/ExtraField.java @@ -0,0 +1,406 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zip; + +import com.android.tools.build.apkzlib.zip.utils.LittleEndianUtils; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * Contains an extra field. + * + * <p>According to the zip specification, the extra field is composed of a sequence of fields. + * This class provides a way to access, parse and modify that information. + * + * <p>The zip specification calls fields to the fields inside the extra field. Because this + * terminology is confusing, we use <i>segment</i> to refer to a part of the extra field. Each + * segment is represented by an instance of {@link Segment} and contains a header ID and data. + * + * <p>Each instance of {@link ExtraField} is immutable. The extra field of a particular entry can + * be changed by creating a new instanceof {@link ExtraField} and pass it to + * {@link StoredEntry#setLocalExtra(ExtraField)}. + * + * <p>Instances of {@link ExtraField} can be created directly from the list of segments in it + * or from the raw byte data. If created from the raw byte data, the data will only be parsed + * on demand. So, if neither {@link #getSegments()} nor {@link #getSingleSegment(int)} is + * invoked, the extra field will not be parsed. This guarantees low performance impact of the + * using the extra field unless its contents are needed. + */ +public class ExtraField { + + /** + * Header ID for field with zip alignment. + */ + static final int ALIGNMENT_ZIP_EXTRA_DATA_FIELD_HEADER_ID = 0xd935; + + /** + * The field's raw data, if it is known. Either this variable or {@link #segments} must be + * non-{@code null}. + */ + @Nullable + private final byte[] rawData; + + /** + * The list of field's segments. Will be populated if the extra field is created based on a + * list of segments; will also be populated after parsing if the extra field is created based + * on the raw bytes. + */ + @Nullable + private ImmutableList<Segment> segments; + + /** + * Creates an extra field based on existing raw data. + * + * @param rawData the raw data; will not be parsed unless needed + */ + public ExtraField(@Nonnull byte[] rawData) { + this.rawData = rawData; + segments = null; + } + + /** + * Creates a new extra field with no segments. + */ + public ExtraField() { + rawData = null; + segments = ImmutableList.of(); + } + + /** + * Creates a new extra field with the given segments. + * + * @param segments the segments + */ + public ExtraField(@Nonnull ImmutableList<Segment> segments) { + rawData = null; + this.segments = segments; + } + + /** + * Obtains all segments in the extra field. + * + * @return all segments + * @throws IOException failed to parse the extra field + */ + public ImmutableList<Segment> getSegments() throws IOException { + if (segments == null) { + parseSegments(); + } + + Preconditions.checkNotNull(segments); + return segments; + } + + /** + * Obtains the only segment with the provided header ID. + * + * @param headerId the header ID + * @return the segment found or {@code null} if no segment contains the provided header ID + * @throws IOException there is more than one header with the provided header ID + */ + @Nullable + public Segment getSingleSegment(int headerId) throws IOException { + List<Segment> found = + getSegments().stream() + .filter(s -> s.getHeaderId() == headerId) + .collect(Collectors.toList()); + if (found.isEmpty()) { + return null; + } else if (found.size() == 1) { + return found.get(0); + } else { + throw new IOException(found.size() + " segments with header ID " + headerId + "found"); + } + } + + /** + * Parses the raw data and generates all segments in {@link #segments}. + * + * @throws IOException failed to parse the data + */ + private void parseSegments() throws IOException { + Preconditions.checkNotNull(rawData); + Preconditions.checkState(segments == null); + + List<Segment> segments = new ArrayList<>(); + ByteBuffer buffer = ByteBuffer.wrap(rawData); + + while (buffer.remaining() > 0) { + int headerId = LittleEndianUtils.readUnsigned2Le(buffer); + int dataSize = LittleEndianUtils.readUnsigned2Le(buffer); + if (dataSize < 0) { + throw new IOException( + "Invalid data size for extra field segment with header ID " + + headerId + + ": " + + dataSize); + } + + byte[] data = new byte[dataSize]; + if (buffer.remaining() < dataSize) { + throw new IOException( + "Invalid data size for extra field segment with header ID " + + headerId + + ": " + + dataSize + + " (only " + + buffer.remaining() + + " bytes are available)"); + } + buffer.get(data); + + SegmentFactory factory = identifySegmentFactory(headerId); + Segment seg = factory.make(headerId, data); + segments.add(seg); + } + + this.segments = ImmutableList.copyOf(segments); + } + + /** + * Obtains the size of the extra field. + * + * @return the size + */ + public int size() { + if (rawData != null) { + return rawData.length; + } else { + Preconditions.checkNotNull(segments); + int sz = 0; + for (Segment s : segments) { + sz += s.size(); + } + + return sz; + } + } + + /** + * Writes the extra field to the given output buffer. + * + * @param out the output buffer to write the field; exactly {@link #size()} bytes will be + * written + * @throws IOException failed to write the extra fields + */ + public void write(@Nonnull ByteBuffer out) throws IOException { + if (rawData != null) { + out.put(rawData); + } else { + Preconditions.checkNotNull(segments); + for (Segment s : segments) { + s.write(out); + } + } + } + + /** + * Identifies the factory to create the segment with the provided header ID. + * + * @param headerId the header ID + * @return the segmnet factory that creates segments with the given header + */ + @Nonnull + private static SegmentFactory identifySegmentFactory(int headerId) { + if (headerId == ALIGNMENT_ZIP_EXTRA_DATA_FIELD_HEADER_ID) { + return AlignmentSegment::new; + } + + return RawDataSegment::new; + } + + /** + * Field inside the extra field. A segment contains a header ID and data. Specific types of + * segments implement this interface. + */ + public interface Segment { + + /** + * Obtains the segment's header ID. + * + * @return the segment's header ID + */ + int getHeaderId(); + + /** + * Obtains the size of the segment including the header ID. + * + * @return the number of bytes needed to write the segment + */ + int size(); + + /** + * Writes the segment to a buffer. + * + * @param out the buffer where to write the segment to; exactly {@link #size()} bytes will + * be written + * @throws IOException failed to write segment data + */ + void write(@Nonnull ByteBuffer out) throws IOException; + } + + /** + * Factory that creates a segment. + */ + @FunctionalInterface + interface SegmentFactory { + + /** + * Creates a new segment. + * + * @param headerId the header ID + * @param data the segment's data + * @return the created segment + * @throws IOException failed to create the segment from the data + */ + @Nonnull + Segment make(int headerId, @Nonnull byte[] data) throws IOException; + } + + /** + * Segment of raw data: this class represents a general segment containing an array of bytes + * as data. + */ + public static class RawDataSegment implements Segment { + + /** + * Header ID. + */ + private final int headerId; + + /** + * Data in the segment. + */ + @Nonnull + private final byte[] data; + + /** + * Creates a new raw data segment. + * + * @param headerId the header ID + * @param data the segment data + */ + RawDataSegment(int headerId, @Nonnull byte[] data) { + this.headerId = headerId; + this.data = data; + } + + @Override + public int getHeaderId() { + return headerId; + } + + @Override + public void write(@Nonnull ByteBuffer out) throws IOException { + LittleEndianUtils.writeUnsigned2Le(out, headerId); + LittleEndianUtils.writeUnsigned2Le(out, data.length); + out.put(data); + } + + @Override + public int size() { + return 4 + data.length; + } + } + + /** + * Segment with information on an alignment: this segment contains information on how an entry + * should be aligned and contains zero-filled data to force alignment. + * + * <p>An alignment segment contains the header ID, the size of the data, the alignment value + * and zero bytes to pad + */ + public static class AlignmentSegment implements Segment { + + /** + * Minimum size for an alignment segment. + */ + public static final int MINIMUM_SIZE = 6; + + /** + * The alignment value. + */ + private int alignment; + + /** + * How many bytes of padding are in this segment? + */ + private int padding; + + /** + * Creates a new alignment segment. + * + * @param alignment the alignment value + * @param totalSize how many bytes should this segment take? + */ + public AlignmentSegment(int alignment, int totalSize) { + Preconditions.checkArgument(alignment > 0, "alignment <= 0"); + Preconditions.checkArgument(totalSize >= MINIMUM_SIZE, "totalSize < MINIMUM_SIZE"); + + /* + * We have 6 bytes of fixed data: header ID (2 bytes), data size (2 bytes), alignment + * value (2 bytes). + */ + this.alignment = alignment; + padding = totalSize - MINIMUM_SIZE; + } + + /** + * Creates a new alignment segment from extra data. + * + * @param headerId the header ID + * @param data the segment data + * @throws IOException failed to create the segment from the data + */ + public AlignmentSegment(int headerId, @Nonnull byte[] data) throws IOException { + Preconditions.checkArgument(headerId == ALIGNMENT_ZIP_EXTRA_DATA_FIELD_HEADER_ID); + + ByteBuffer dataBuffer = ByteBuffer.wrap(data); + alignment = LittleEndianUtils.readUnsigned2Le(dataBuffer); + if (alignment <= 0) { + throw new IOException("Invalid alignment in alignment field: " + alignment); + } + + padding = data.length - 2; + } + + @Override + public void write(@Nonnull ByteBuffer out) throws IOException { + LittleEndianUtils.writeUnsigned2Le(out, ALIGNMENT_ZIP_EXTRA_DATA_FIELD_HEADER_ID); + LittleEndianUtils.writeUnsigned2Le(out, padding + 2); + LittleEndianUtils.writeUnsigned2Le(out, alignment); + out.put(new byte[padding]); + } + + @Override + public int size() { + return padding + 6; + } + + @Override + public int getHeaderId() { + return ALIGNMENT_ZIP_EXTRA_DATA_FIELD_HEADER_ID; + } + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/zip/FileUseMap.java b/src/main/java/com/android/tools/build/apkzlib/zip/FileUseMap.java new file mode 100644 index 0000000..7e8e9d9 --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zip/FileUseMap.java @@ -0,0 +1,601 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zip; + +import com.google.common.base.Preconditions; +import com.google.common.base.Verify; +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; +import com.google.common.primitives.Ints; +import java.util.List; +import java.util.Set; +import java.util.SortedSet; +import java.util.StringJoiner; +import java.util.TreeSet; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * The file use map keeps track of which parts of the zip file are used which parts are not. + * It essentially maintains an ordered set of entries ({@link FileUseMapEntry}). Each entry either has + * some data (an entry, the Central Directory, the EOCD) or is a free entry. + * + * <p>For example: [0-95, "foo/"][95-260, "xpto"][260-310, free][310-360, Central Directory] + * [360-390,EOCD] + * + * <p>There are a few invariants in this structure: + * <ul> + * <li>there are no gaps between map entries; + * <li>the map is fully covered up to its size; + * <li>there are no two free entries next to each other; this is guaranteed by coalescing the + * entries upon removal (see {@link #coalesce(FileUseMapEntry)}); + * <li>all free entries have a minimum size defined in the constructor, with the possible exception + * of the last one + * </ul> + */ +class FileUseMap { + /** + * Size of the file according to the map. This should always match the last entry in + * {@code #map}. + */ + private long size; + + /** + * Tree with all intervals ordered by position. Contains coverage from 0 up to {@link #size}. + * If {@link #size} is zero then this set is empty. This is the only situation in which the map + * will be empty. + */ + @Nonnull + private TreeSet<FileUseMapEntry<?>> map; + + /** + * Tree with all free blocks ordered by size. This is essentially a view over {@link #map} + * containing only the free blocks, but in a different order. + */ + @Nonnull + private TreeSet<FileUseMapEntry<?>> free; + + /** + * If defined, defines the minimum size for a free entry. + */ + private int mMinFreeSize; + + /** + * Creates a new, empty file map. + * + * @param size the size of the file + * @param minFreeSize minimum size of a free entry + */ + FileUseMap(long size, int minFreeSize) { + Preconditions.checkArgument(size >= 0, "size < 0"); + Preconditions.checkArgument(minFreeSize >= 0, "minFreeSize < 0"); + + this.size = size; + map = new TreeSet<>(FileUseMapEntry.COMPARE_BY_START); + free = new TreeSet<>(FileUseMapEntry.COMPARE_BY_SIZE); + mMinFreeSize = minFreeSize; + + if (size > 0) { + internalAdd(FileUseMapEntry.makeFree(0, size)); + } + } + + /** + * Adds an entry to the internal structures. + * + * @param entry the entry to add + */ + private void internalAdd(@Nonnull FileUseMapEntry<?> entry) { + map.add(entry); + + if (entry.isFree()) { + free.add(entry); + } + } + + /** + * Removes an entry from the internal structures. + * + * @param entry the entry to remove + */ + private void internalRemove(@Nonnull FileUseMapEntry<?> entry) { + boolean wasRemoved = map.remove(entry); + Preconditions.checkState(wasRemoved, "entry not in map"); + + if (entry.isFree()) { + free.remove(entry); + } + } + + /** + * Adds a new file to the map. The interval specified by {@code entry} must fit inside an + * empty entry in the map. That entry will be replaced by entry and additional free entries + * will be added before and after if needed to make sure no spaces exist on the map. + * + * @param entry the entry to add + */ + private void add(@Nonnull FileUseMapEntry<?> entry) { + Preconditions.checkArgument(entry.getStart() < size, "entry.getStart() >= size"); + Preconditions.checkArgument(entry.getEnd() <= size, "entry.getEnd() > size"); + Preconditions.checkArgument(!entry.isFree(), "entry.isFree()"); + + FileUseMapEntry<?> container = findContainer(entry); + Verify.verify(container.isFree(), "!container.isFree()"); + + Set<FileUseMapEntry<?>> replacements = split(container, entry); + internalRemove(container); + for (FileUseMapEntry<?> r : replacements) { + internalAdd(r); + } + } + + /** + * Removes a file from the map, replacing it with an empty one that is then coalesced with + * neighbors (if the neighbors are free). + * + * @param entry the entry + */ + void remove(@Nonnull FileUseMapEntry<?> entry) { + Preconditions.checkState(map.contains(entry), "!map.contains(entry)"); + Preconditions.checkArgument(!entry.isFree(), "entry.isFree()"); + + internalRemove(entry); + + FileUseMapEntry<?> replacement = FileUseMapEntry.makeFree(entry.getStart(), entry.getEnd()); + internalAdd(replacement); + coalesce(replacement); + } + + /** + * Adds a new file to the map. The interval specified by ({@code start}, {@code end}) must fit + * inside an empty entry in the map. That entry will be replaced by entry and additional free + * entries will be added before and after if needed to make sure no spaces exist on the map. + * + * <p>The entry cannot extend beyong the end of the map. If necessary, extend the map using + * {@link #extend(long)}. + * + * @param start the start of this entry + * @param end the end of the entry + * @param store extra data to store with the entry + * @param <T> the type of data to store in the entry + * @return the new entry + */ + <T> FileUseMapEntry<T> add(long start, long end, @Nonnull T store) { + Preconditions.checkArgument(start >= 0, "start < 0"); + Preconditions.checkArgument(end > start, "end < start"); + + FileUseMapEntry<T> entry = FileUseMapEntry.makeUsed(start, end, store); + add(entry); + return entry; + } + + /** + * Finds the entry that fully contains the given one. It is assumed that one exists. + * + * @param entry the entry whose container we're looking for + * @return the container + */ + @Nonnull + private FileUseMapEntry<?> findContainer(@Nonnull FileUseMapEntry<?> entry) { + FileUseMapEntry container = map.floor(entry); + Verify.verifyNotNull(container); + Verify.verify(container.getStart() <= entry.getStart()); + Verify.verify(container.getEnd() >= entry.getEnd()); + + return container; + } + + /** + * Splits a container to add an entry, adding new free entries before and after the provided + * entry if needed. + * + * @param container the container entry, a free entry that is in {@link #map} that that + * encloses {@code entry} + * @param entry the entry that will be used to split {@code container} + * @return a set of non-overlapping entries that completely covers {@code container} and that + * includes {@code entry} + */ + @Nonnull + private static Set<FileUseMapEntry<?>> split(@Nonnull FileUseMapEntry<?> container, + @Nonnull FileUseMapEntry<?> entry) { + Preconditions.checkArgument(container.isFree(), "!container.isFree()"); + + long farStart = container.getStart(); + long start = entry.getStart(); + long end = entry.getEnd(); + long farEnd = container.getEnd(); + + Verify.verify(farStart <= start, "farStart > start"); + Verify.verify(start < end, "start >= end"); + Verify.verify(farEnd >= end, "farEnd < end"); + + Set<FileUseMapEntry<?>> result = Sets.newHashSet(); + if (farStart < start) { + result.add(FileUseMapEntry.makeFree(farStart, start)); + } + + result.add(entry); + + if (end < farEnd) { + result.add(FileUseMapEntry.makeFree(end, farEnd)); + } + + return result; + } + + /** + * Coalesces a free entry replacing it and neighboring free entries with a single, larger + * entry. This method does nothing if {@code entry} does not have free neighbors. + * + * @param entry the free entry to coalesce with neighbors + */ + private void coalesce(@Nonnull FileUseMapEntry<?> entry) { + Preconditions.checkArgument(entry.isFree(), "!entry.isFree()"); + + FileUseMapEntry<?> prevToMerge = null; + long start = entry.getStart(); + if (start > 0) { + /* + * See if we have a previous entry to merge with this one. + */ + prevToMerge = map.floor(FileUseMapEntry.makeFree(start - 1, start)); + Verify.verifyNotNull(prevToMerge); + if (!prevToMerge.isFree()) { + prevToMerge = null; + } + } + + FileUseMapEntry<?> nextToMerge = null; + long end = entry.getEnd(); + if (end < size) { + /* + * See if we have a next entry to merge with this one. + */ + nextToMerge = map.ceiling(FileUseMapEntry.makeFree(end, end + 1)); + Verify.verifyNotNull(nextToMerge); + if (!nextToMerge.isFree()) { + nextToMerge = null; + } + } + + if (prevToMerge == null && nextToMerge == null) { + return; + } + + long newStart = start; + if (prevToMerge != null) { + newStart = prevToMerge.getStart(); + internalRemove(prevToMerge); + } + + long newEnd = end; + if (nextToMerge != null) { + newEnd = nextToMerge.getEnd(); + internalRemove(nextToMerge); + } + + internalRemove(entry); + internalAdd(FileUseMapEntry.makeFree(newStart, newEnd)); + } + + /** + * Truncates map removing the top entry if it is free and reducing the map's size. + */ + void truncate() { + if (size == 0) { + return; + } + + /* + * Find the last entry. + */ + FileUseMapEntry<?> last = map.last(); + Verify.verifyNotNull(last, "last == null"); + if (last.isFree()) { + internalRemove(last); + size = last.getStart(); + } + } + + /** + * Obtains the size of the map. + * + * @return the size + */ + long size() { + return size; + } + + /** + * Obtains the largest used offset in the map. This will be size of the map after truncation. + * + * @return the size of the file discounting the last block if it is empty + */ + long usedSize() { + if (size == 0) { + return 0; + } + + /* + * Find the last entry to see if it is an empty entry. If it is, we need to remove its size + * from the returned value. + */ + FileUseMapEntry<?> last = map.last(); + Verify.verifyNotNull(last, "last == null"); + if (last.isFree()) { + return last.getStart(); + } else { + Verify.verify(last.getEnd() == size); + return size; + } + } + + /** + * Extends the map to guarantee it has at least {@code size} bytes. If the current size is + * as large as {@code size}, this method does nothing. + * + * @param size the new size of the map that cannot be smaller that the current size + */ + void extend(long size) { + Preconditions.checkArgument(size >= this.size, "size < size"); + + if (this.size == size) { + return; + } + + FileUseMapEntry<?> newBlock = FileUseMapEntry.makeFree(this.size, size); + internalAdd(newBlock); + + this.size = size; + + coalesce(newBlock); + } + + /** + * Locates a free area in the map with at least {@code size} bytes such that + * {@code ((start + alignOffset) % align == 0} and such that the free space before {@code start} + * is not smaller than the minimum free entry size. This method will follow the algorithm + * specified by {@code alg}. + * + * <p>If no free contiguous block exists in the map that can hold the provided + * size then the first free index at the end of the map is provided. This means that the map + * may need to be extended before data can be added. + * + * @param size the size of the contiguous area requested + * @param alignOffset an offset to which alignment needs to be computed (see method description) + * @param align alignment at the offset (see method description) + * @param alg which algorithm to use + * @return the location of the contiguous area; this may be located at the end of the map + */ + long locateFree(long size, long alignOffset, long align, @Nonnull PositionAlgorithm alg) { + Preconditions.checkArgument(size > 0, "size <= 0"); + + FileUseMapEntry<?> minimumSizedEntry = FileUseMapEntry.makeFree(0, size); + SortedSet<FileUseMapEntry<?>> matches; + + switch (alg) { + case BEST_FIT: + matches = free.tailSet(minimumSizedEntry); + break; + case FIRST_FIT: + matches = map; + break; + default: + throw new AssertionError(); + } + + FileUseMapEntry<?> best = null; + long bestExtraSize = 0; + for (FileUseMapEntry<?> curr : matches) { + /* + * We don't care about blocks that aren't free. + */ + if (!curr.isFree()) { + continue; + } + + /* + * Compute any extra size we need in this block to make sure we verify the alignment. + * There must be a better to do this... + */ + long extraSize; + if (align == 0) { + extraSize = 0; + } else { + extraSize = (align - ((curr.getStart() + alignOffset) % align)) % align; + } + + /* + * We can't leave than mMinFreeSize before. So if the extraSize is less than + * mMinFreeSize, we have to increase it by 'align' as many times as needed. For + * example, if mMinFreeSize is 20, align 4 and extraSize is 5. We need to increase it + * to 21 (5 + 4 * 4) + */ + if (extraSize > 0 && extraSize < mMinFreeSize) { + int addAlignBlocks = + Ints.checkedCast((mMinFreeSize - extraSize + align - 1) / align); + extraSize += addAlignBlocks * align; + } + + /* + * We don't care about blocks where we don't fit in. + */ + if (curr.getSize() < (size + extraSize)) { + continue; + } + + /* + * We don't care about blocks that leave less than the minimum size after. There are + * two exceptions: (1) this is the last block and (2) the next block is free in which + * case, after coalescing, the free block with have at least the minimum size. + */ + long emptySpaceLeft = curr.getSize() - (size + extraSize); + if (emptySpaceLeft > 0 && emptySpaceLeft < mMinFreeSize) { + FileUseMapEntry<?> next = map.higher(curr); + if (next != null && !next.isFree()) { + continue; + } + } + + /* + * We don't care about blocks that are bigger than the best so far (otherwise this + * wouldn't be a best-fit algorithm). + */ + if (best != null && best.getSize() < curr.getSize()) { + continue; + } + + best = curr; + bestExtraSize = extraSize; + + /* + * If we're doing first fit, we don't want to search for a better one :) + */ + if (alg == PositionAlgorithm.FIRST_FIT) { + break; + } + } + + /* + * If no entry that could hold size is found, get the first free byte. + */ + long firstFree = this.size; + if (best == null && !map.isEmpty()) { + FileUseMapEntry<?> last = map.last(); + if (last.isFree()) { + firstFree = last.getStart(); + } + } + + /* + * We're done: either we found something or we didn't, in which the new entry needs to + * be added to the end of the map. + */ + if (best == null) { + long extra = (align - ((firstFree + alignOffset) % align)) % align; + + /* + * If adding this entry at the end would create a space smaller than the minimum, + * push it for 'align' bytes forward. + */ + if (extra > 0) { + if (extra < mMinFreeSize) { + extra += align * (((mMinFreeSize - extra) + (align - 1)) / align); + } + } + + return firstFree + extra; + } else { + return best.getStart() + bestExtraSize; + } + } + + /** + * Obtains all free areas of the map, excluding any trailing free area. + * + * @return all free areas, an empty set if there are no free areas; the areas are returned + * in file order, that is, if area {@code x} starts before area {@code y}, then area {@code x} + * will be stored before area {@code y} in the list + */ + @Nonnull + List<FileUseMapEntry<?>> getFreeAreas() { + List<FileUseMapEntry<?>> freeAreas = Lists.newArrayList(); + + for (FileUseMapEntry<?> area : map) { + if (area.isFree() && area.getEnd() != size) { + freeAreas.add(area); + } + } + + return freeAreas; + } + + /** + * Obtains the entry that is located before the one provided. + * + * @param entry the map entry to get the previous one for; must belong to the map + * @return the entry before the provided one, {@code null} if {@code entry} is the first entry + * in the map + */ + @Nullable + FileUseMapEntry<?> before(@Nonnull FileUseMapEntry<?> entry) { + Preconditions.checkNotNull(entry, "entry == null"); + + return map.lower(entry); + } + + /** + * Obtains the entry that is located after the one provided. + * + * @param entry the map entry to get the next one for; must belong to the map + * @return the entry after the provided one, {@code null} if {@code entry} is the last entry in + * the map + */ + @Nullable + FileUseMapEntry<?> after(@Nonnull FileUseMapEntry<?> entry) { + Preconditions.checkNotNull(entry, "entry == null"); + + return map.higher(entry); + } + + /** + * Obtains the entry at the given offset. + * + * @param offset the offset to look for + * @return the entry found or {@code null} if there is no entry (not even a free one) at the + * given offset + */ + @Nullable + FileUseMapEntry<?> at(long offset) { + Preconditions.checkArgument(offset >= 0, "offset < 0"); + Preconditions.checkArgument(offset < size, "offset >= size"); + + FileUseMapEntry<?> entry = map.floor(FileUseMapEntry.makeFree(offset, offset + 1)); + if (entry == null) { + return null; + } + + Verify.verify(entry.getStart() <= offset); + Verify.verify(entry.getEnd() > offset); + + return entry; + } + + @Override + public String toString() { + StringJoiner j = new StringJoiner(", "); + map.stream() + .map(e -> e.getStart() + " - " + e.getEnd() + ": " + e.getStore()) + .forEach(j::add); + return "FileUseMap[" + j.toString() + "]"; + } + + /** + * Algorithms used to position entries in blocks. + */ + public enum PositionAlgorithm { + /** + * Best fit: finds the smallest free block that can receive the entry. + */ + BEST_FIT, + + /** + * First fit: finds the first free block that can receive the entry. + */ + FIRST_FIT + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/zip/FileUseMapEntry.java b/src/main/java/com/android/tools/build/apkzlib/zip/FileUseMapEntry.java new file mode 100644 index 0000000..01cbbfc --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zip/FileUseMapEntry.java @@ -0,0 +1,162 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zip; + +import com.google.common.base.MoreObjects; +import com.google.common.base.Preconditions; +import com.google.common.primitives.Ints; +import java.util.Comparator; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * Represents an entry in the {@link FileUseMap}. Each entry contains an interval of bytes. The + * end of the interval is exclusive. + * <p> + * Entries can either be free or used. Used entries <em>must</em> store an object. Free entries + * do not store anything. + * <p> + * File map entries are used to keep track of which parts of a file map are used and not. + * @param <T> the type of data stored + */ +class FileUseMapEntry<T> { + + /** + * Comparator that compares entries by their start date. + */ + public static final Comparator<FileUseMapEntry<?>> COMPARE_BY_START = + (o1, o2) -> Ints.saturatedCast(o1.getStart() - o2.getStart()); + + /** + * Comparator that compares entries by their size. + */ + public static final Comparator<FileUseMapEntry<?>> COMPARE_BY_SIZE = + (o1, o2) -> Ints.saturatedCast(o1.getSize() - o2.getSize()); + + /** + * The first byte in the entry. + */ + private final long start; + + /** + * The first byte no longer in the entry. + */ + private final long end; + + /** + * The stored data. If {@code null} then this entry represents a free entry. + */ + @Nullable + private final T store; + + /** + * Creates a new map entry. + * + * @param start the start of the entry + * @param end the end of the entry (first byte no longer in the entry) + * @param store the data to store in the entry or {@code null} if this is a free entry + */ + private FileUseMapEntry(long start, long end, @Nullable T store) { + Preconditions.checkArgument(start >= 0, "start < 0"); + Preconditions.checkArgument(end > start, "end <= start"); + + this.start = start; + this.end = end; + this.store = store; + } + + /** + * Creates a new free entry. + * + * @param start the start of the entry + * @param end the end of the entry (first byte no longer in the entry) + * @return the entry + */ + public static FileUseMapEntry<Object> makeFree(long start, long end) { + return new FileUseMapEntry<>(start, end, null); + } + + /** + * Creates a new used entry. + * + * @param start the start of the entry + * @param end the end of the entry (first byte no longer in the entry) + * @param store the data to store in the entry + * @param <T> the type of data to store in the entry + * @return the entry + */ + public static <T> FileUseMapEntry<T> makeUsed(long start, long end, @Nonnull T store) { + Preconditions.checkNotNull(store, "store == null"); + return new FileUseMapEntry<>(start, end, store); + } + + /** + * Obtains the first byte in the entry. + * + * @return the first byte in the entry (if the same value as {@link #getEnd()} then the entry + * is empty and contains no data) + */ + long getStart() { + return start; + } + + /** + * Obtains the first byte no longer in the entry. + * + * @return the first byte no longer in the entry + */ + long getEnd() { + return end; + } + + /** + * Obtains the size of the entry. + * + * @return the number of bytes contained in the entry + */ + long getSize() { + return end - start; + } + + /** + * Determines if this is a free entry. + * + * @return is this entry free? + */ + boolean isFree() { + return store == null; + } + + /** + * Obtains the data stored in the entry. + * + * @return the data stored or {@code null} if this entry is a free entry + */ + @Nullable + T getStore() { + return store; + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("start", start) + .add("end", end) + .add("store", store) + .toString(); + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/zip/GPFlags.java b/src/main/java/com/android/tools/build/apkzlib/zip/GPFlags.java new file mode 100644 index 0000000..96062ca --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zip/GPFlags.java @@ -0,0 +1,179 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zip; + +import java.io.IOException; +import javax.annotation.Nonnull; + +/** + * General purpose bit flags. Contains the encoding of the zip's general purpose bits. + * + * <p>We don't really care about the method bit(s). These are bits 1 and 2. Here are the values: + * <ul> + * <li>0 (00): Normal (-en) compression option was used. + * <li>1 (01): Maximum (-exx/-ex) compression option was used. + * <li>2 (10): Fast (-ef) compression option was used. + * <li>3 (11): Super Fast (-es) compression option was used. + * </ul> + */ +class GPFlags { + + /** + * Is the entry encrypted? + */ + private static final int BIT_ENCRYPTION = 1; + + /** + * Has CRC computation been deferred and, therefore, does a data description block exist? + */ + private static final int BIT_DEFERRED_CRC = (1 << 3); + + /** + * Is enhanced deflating used? + */ + private static final int BIT_ENHANCED_DEFLATING = (1 << 4); + + /** + * Does the entry contain patched data? + */ + private static final int BIT_PATCHED_DATA = (1 << 5); + + /** + * Is strong encryption used? + */ + private static final int BIT_STRONG_ENCRYPTION = (1 << 6) | (1 << 13); + + /** + * If this bit is set the filename and comment fields for this file must be encoded using UTF-8. + */ + private static final int BIT_EFS = (1 << 11); + + /** + * Unused bits. + */ + private static final int BIT_UNUSED = (1 << 7) | (1 << 8) | (1 << 9) | (1 << 10) + | (1 << 14) | (1 << 15); + + /** + * Bit flag value. + */ + private final long value; + + /** + * Has the CRC computation beeen deferred? + */ + private boolean deferredCrc; + + /** + * Is the file name encoded in UTF-8? + */ + private boolean utf8FileName; + + /** + * Creates a new flags object. + * + * @param value the value of the bit mask + */ + private GPFlags(long value) { + this.value = value; + + deferredCrc = ((value & BIT_DEFERRED_CRC) != 0); + utf8FileName = ((value & BIT_EFS) != 0); + } + + /** + * Obtains the flags value. + * + * @return the value of the bit mask + */ + public long getValue() { + return value; + } + + /** + * Is the CRC computation deferred? + * + * @return is the CRC computation deferred? + */ + public boolean isDeferredCrc() { + return deferredCrc; + } + + /** + * Is the file name encoded in UTF-8? + * + * @return is the file name encoded in UTF-8? + */ + public boolean isUtf8FileName() { + return utf8FileName; + } + + /** + * Creates a new bit mask. + * + * @param utf8Encoding should UTF-8 encoding be used? + * @return the new bit mask + */ + @Nonnull + static GPFlags make(boolean utf8Encoding) { + long flags = 0; + + if (utf8Encoding) { + flags |= BIT_EFS; + } + + return new GPFlags(flags); + } + + /** + * Creates the flag information from a byte. This method will also validate that only + * supported options are defined in the flag. + * + * @param bits the bit mask + * @return the created flag information + * @throws IOException unsupported options are used in the bit mask + */ + @Nonnull + static GPFlags from(long bits) throws IOException { + if ((bits & BIT_ENCRYPTION) != 0) { + throw new IOException("Zip files with encrypted of entries not supported."); + } + + if ((bits & BIT_ENHANCED_DEFLATING) != 0) { + throw new IOException("Enhanced deflating not supported."); + } + + if ((bits & BIT_PATCHED_DATA) != 0) { + throw new IOException("Compressed patched data not supported."); + } + + if ((bits & BIT_STRONG_ENCRYPTION) != 0) { + throw new IOException("Strong encryption not supported."); + } + + if ((bits & BIT_UNUSED) != 0) { + throw new IOException("Unused bits set in directory entry. Weird. I don't know what's " + + "going on."); + } + + if ((bits & 0xffffffff00000000L) != 0) { + throw new IOException("Unsupported bits after 32."); + } + + return new GPFlags(bits); + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/zip/InflaterByteSource.java b/src/main/java/com/android/tools/build/apkzlib/zip/InflaterByteSource.java new file mode 100644 index 0000000..6efe2c7 --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zip/InflaterByteSource.java @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zip; + +import com.android.tools.build.apkzlib.zip.utils.CloseableByteSource; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.SequenceInputStream; +import java.util.zip.Inflater; +import java.util.zip.InflaterInputStream; +import javax.annotation.Nonnull; + +/** + * Byte source that inflates another byte source. It assumed the inner byte source has deflated + * data. + */ +public class InflaterByteSource extends CloseableByteSource { + + /** + * The stream factory for the deflated data. + */ + @Nonnull + private final CloseableByteSource deflatedSource; + + /** + * Creates a new source. + * @param byteSource the factory for deflated data + */ + public InflaterByteSource(@Nonnull CloseableByteSource byteSource) { + deflatedSource = byteSource; + } + + @Override + public InputStream openStream() throws IOException { + /* + * The extra byte is a dummy byte required by the inflater. Weirdo. + * (see the java.util.Inflater documentation). Looks like a hack... + * "Oh, I need an extra dummy byte to allow for some... err... optimizations..." + */ + ByteArrayInputStream hackByte = new ByteArrayInputStream(new byte[] { 0 }); + return new InflaterInputStream(new SequenceInputStream(deflatedSource.openStream(), + hackByte), new Inflater(true)); + } + + @Override + public void innerClose() throws IOException { + deflatedSource.close(); + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/zip/LazyDelegateByteSource.java b/src/main/java/com/android/tools/build/apkzlib/zip/LazyDelegateByteSource.java new file mode 100644 index 0000000..b755bae --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zip/LazyDelegateByteSource.java @@ -0,0 +1,156 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zip; + +import com.android.tools.build.apkzlib.zip.utils.CloseableByteSource; +import com.google.common.hash.HashCode; +import com.google.common.hash.HashFunction; +import com.google.common.io.ByteProcessor; +import com.google.common.io.ByteSink; +import com.google.common.io.ByteSource; +import com.google.common.io.CharSource; +import com.google.common.util.concurrent.ListenableFuture; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.Charset; +import java.util.concurrent.ExecutionException; +import javax.annotation.Nonnull; + +/** + * {@code ByteSource} that delegates all operations to another {@code ByteSource}. The other + * byte source, the <em>delegate</em>, may be computed lazily. + */ +public class LazyDelegateByteSource extends CloseableByteSource { + + /** + * Byte source where we delegate operations to. + */ + @Nonnull + private final ListenableFuture<CloseableByteSource> delegate; + + /** + * Creates a new byte source that delegates operations to the provided source. + * @param delegate the source that will receive all operations + */ + public LazyDelegateByteSource(@Nonnull ListenableFuture<CloseableByteSource> delegate) { + this.delegate = delegate; + } + + /** + * Obtains the delegate future. + * @return the delegate future, that may be computed or not + */ + @Nonnull + public ListenableFuture<CloseableByteSource> getDelegate() { + return delegate; + } + + /** + * Obtains the byte source, waiting for the future to be computed. + * @return the byte source + * @throws IOException failed to compute the future :) + */ + @Nonnull + private CloseableByteSource get() throws IOException { + try { + CloseableByteSource r = delegate.get(); + if (r == null) { + throw new IOException("Delegate byte source computation resulted in null."); + } + + return r; + } catch (InterruptedException e) { + throw new IOException("Interrupted while waiting for byte source computation.", e); + } catch (ExecutionException e) { + throw new IOException("Failed to compute byte source.", e); + } + } + + @Override + public CharSource asCharSource(Charset charset) { + try { + return get().asCharSource(charset); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public InputStream openBufferedStream() throws IOException { + return get().openBufferedStream(); + } + + @Override + public ByteSource slice(long offset, long length) { + try { + return get().slice(offset, length); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public boolean isEmpty() throws IOException { + return get().isEmpty(); + } + + @Override + public long size() throws IOException { + return get().size(); + } + + @Override + public long copyTo(@Nonnull OutputStream output) throws IOException { + return get().copyTo(output); + } + + @Override + public long copyTo(@Nonnull ByteSink sink) throws IOException { + return get().copyTo(sink); + } + + @Override + public byte[] read() throws IOException { + return get().read(); + } + + @Override + public <T> T read(@Nonnull ByteProcessor<T> processor) throws IOException { + return get().read(processor); + } + + @Override + public HashCode hash(HashFunction hashFunction) throws IOException { + return get().hash(hashFunction); + } + + @Override + public boolean contentEquals(@Nonnull ByteSource other) throws IOException { + return get().contentEquals(other); + } + + @Override + public InputStream openStream() throws IOException { + return get().openStream(); + } + + @Override + public void innerClose() throws IOException { + get().close(); + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/zip/ProcessedAndRawByteSources.java b/src/main/java/com/android/tools/build/apkzlib/zip/ProcessedAndRawByteSources.java new file mode 100644 index 0000000..8308406 --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zip/ProcessedAndRawByteSources.java @@ -0,0 +1,86 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zip; + +import com.android.tools.build.apkzlib.zip.utils.CloseableByteSource; +import com.google.common.io.Closer; +import java.io.Closeable; +import java.io.IOException; +import javax.annotation.Nonnull; + +/** + * Container that has two bytes sources: one representing raw data and another processed data. + * In case of compression, the raw data is the compressed data and the processed data is the + * uncompressed data. It is valid for a RaP ("Raw-and-Processed") to contain the same byte sources + * for both processed and raw data. + */ +public class ProcessedAndRawByteSources implements Closeable { + + /** + * The processed byte source. + */ + @Nonnull + private final CloseableByteSource processedSource; + + /** + * The processed raw source. + */ + @Nonnull + private final CloseableByteSource rawSource; + + /** + * Creates a new container. + * + * @param processedSource the processed source + * @param rawSource the raw source + */ + public ProcessedAndRawByteSources(@Nonnull CloseableByteSource processedSource, + @Nonnull CloseableByteSource rawSource) { + this.processedSource = processedSource; + this.rawSource = rawSource; + } + + /** + * Obtains a byte source that read the processed contents of the entry. + * + * @return a byte source + */ + @Nonnull + public CloseableByteSource getProcessedByteSource() { + return processedSource; + } + + /** + * Obtains a byte source that reads the raw contents of an entry. This is the data that is + * ultimately stored in the file and, in the case of compressed files, is the same data in the + * source returned by {@link #getProcessedByteSource()}. + * + * @return a byte source + */ + @Nonnull + public CloseableByteSource getRawByteSource() { + return rawSource; + } + + @Override + public void close() throws IOException { + Closer closer = Closer.create(); + closer.register(processedSource); + closer.register(rawSource); + closer.close(); + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/zip/StoredEntry.java b/src/main/java/com/android/tools/build/apkzlib/zip/StoredEntry.java new file mode 100644 index 0000000..16f505e --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zip/StoredEntry.java @@ -0,0 +1,818 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zip; + +import com.android.tools.build.apkzlib.zip.utils.CloseableByteSource; +import com.android.tools.build.apkzlib.zip.utils.CloseableDelegateByteSource; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.base.Verify; +import com.google.common.io.ByteSource; +import com.google.common.io.ByteStreams; +import com.google.common.primitives.Ints; +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.util.Comparator; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * A stored entry represents a file in the zip. The entry may or may not be written to the zip + * file. + * + * <p>Stored entries provide the operations that are related to the files themselves, not to the + * zip. It is through the {@code StoredEntry} class that entries can be deleted ({@link #delete()}, + * open ({@link #open()}) or realigned ({@link #realign()}). + * + * <p>Entries are not created directly. They are created using + * {@link ZFile#add(String, InputStream, boolean)} and obtained from the zip file + * using {@link ZFile#get(String)} or {@link ZFile#entries()}. + * + * <p>Most of the data in the an entry is in the Central Directory Header. This includes the name, + * compression method, file compressed and uncompressed sizes, CRC32 checksum, etc. The CDH can + * be obtained using the {@link #getCentralDirectoryHeader()} method. + */ +public class StoredEntry { + + /** + * Comparator that compares instances of {@link StoredEntry} by their names. + */ + static final Comparator<StoredEntry> COMPARE_BY_NAME = + (o1, o2) -> { + if (o1 == null && o2 == null) { + return 0; + } + + if (o1 == null) { + return -1; + } + + if (o2 == null) { + return 1; + } + + String name1 = o1.getCentralDirectoryHeader().getName(); + String name2 = o2.getCentralDirectoryHeader().getName(); + return name1.compareTo(name2); + }; + + /** + * Signature of the data descriptor. + */ + private static final int DATA_DESC_SIGNATURE = 0x08074b50; + + /** + * Local header field: signature. + */ + private static final ZipField.F4 F_LOCAL_SIGNATURE = new ZipField.F4(0, 0x04034b50, + "Signature"); + + /** + * Local header field: version to extract, should match the CDH's. + */ + @VisibleForTesting + static final ZipField.F2 F_VERSION_EXTRACT = new ZipField.F2( + F_LOCAL_SIGNATURE.endOffset(), "Version to extract", + new ZipFieldInvariantNonNegative()); + + /** + * Local header field: GP bit flag, should match the CDH's. + */ + private static final ZipField.F2 F_GP_BIT = new ZipField.F2(F_VERSION_EXTRACT.endOffset(), + "GP bit flag"); + + /** + * Local header field: compression method, should match the CDH's. + */ + private static final ZipField.F2 F_METHOD = new ZipField.F2(F_GP_BIT.endOffset(), + "Compression method", new ZipFieldInvariantNonNegative()); + + /** + * Local header field: last modification time, should match the CDH's. + */ + private static final ZipField.F2 F_LAST_MOD_TIME = new ZipField.F2(F_METHOD.endOffset(), + "Last modification time"); + + /** + * Local header field: last modification time, should match the CDH's. + */ + private static final ZipField.F2 F_LAST_MOD_DATE = new ZipField.F2(F_LAST_MOD_TIME.endOffset(), + "Last modification date"); + + /** + * Local header field: CRC32 checksum, should match the CDH's. 0 if there is no data. + */ + private static final ZipField.F4 F_CRC32 = new ZipField.F4(F_LAST_MOD_DATE.endOffset(), + "CRC32"); + + /** + * Local header field: compressed size, size the data takes in the zip file. + */ + private static final ZipField.F4 F_COMPRESSED_SIZE = new ZipField.F4(F_CRC32.endOffset(), + "Compressed size", new ZipFieldInvariantNonNegative()); + + /** + * Local header field: uncompressed size, size the data takes after extraction. + */ + private static final ZipField.F4 F_UNCOMPRESSED_SIZE = new ZipField.F4( + F_COMPRESSED_SIZE.endOffset(), "Uncompressed size", new ZipFieldInvariantNonNegative()); + + /** + * Local header field: length of the file name. + */ + private static final ZipField.F2 F_FILE_NAME_LENGTH = new ZipField.F2( + F_UNCOMPRESSED_SIZE.endOffset(), "@File name length", + new ZipFieldInvariantNonNegative()); + + /** + * Local header filed: length of the extra field. + */ + private static final ZipField.F2 F_EXTRA_LENGTH = new ZipField.F2( + F_FILE_NAME_LENGTH.endOffset(), "Extra length", new ZipFieldInvariantNonNegative()); + + /** + * Local header size (fixed part, not counting file name or extra field). + */ + static final int FIXED_LOCAL_FILE_HEADER_SIZE = F_EXTRA_LENGTH.endOffset(); + + /** + * Type of entry. + */ + @Nonnull + private StoredEntryType type; + + /** + * The central directory header with information about the file. + */ + @Nonnull + private CentralDirectoryHeader cdh; + + /** + * The file this entry is associated with + */ + @Nonnull + private ZFile file; + + /** + * Has this entry been deleted? + */ + private boolean deleted; + + /** + * Extra field specified in the local directory. + */ + @Nonnull + private ExtraField localExtra; + + /** + * Type of data descriptor associated with the entry. + */ + @Nonnull + private DataDescriptorType dataDescriptorType; + + /** + * Source for this entry's data. If this entry is a directory, this source has to have zero + * size. + */ + @Nonnull + private ProcessedAndRawByteSources source; + + /** + * Verify log for the entry. + */ + @Nonnull + private final VerifyLog verifyLog; + + /** + * Creates a new stored entry. + * + * @param header the header with the entry information; if the header does not contain an + * offset it means that this entry is not yet written in the zip file + * @param file the zip file containing the entry + * @param source the entry's data source; it can be {@code null} only if the source can be + * read from the zip file, that is, if {@code header.getOffset()} is non-negative + * @throws IOException failed to create the entry + */ + StoredEntry( + @Nonnull CentralDirectoryHeader header, + @Nonnull ZFile file, + @Nullable ProcessedAndRawByteSources source) + throws IOException { + cdh = header; + this.file = file; + deleted = false; + verifyLog = file.makeVerifyLog(); + + if (header.getOffset() >= 0) { + /* + * This will be overwritten during readLocalHeader. However, IJ complains if we don't + * assign a value to localExtra because of the @Nonnull annotation. + */ + localExtra = new ExtraField(); + + readLocalHeader(); + + Preconditions.checkArgument( + source == null, + "Source was defined but contents already exist on file."); + + /* + * Since the file is already in the zip, dynamically create a source that will read + * the file from the zip when needed. The assignment is not really needed, but we + * would get a warning because of the @NotNull otherwise. + */ + this.source = createSourceFromZip(cdh.getOffset()); + } else { + /* + * There is no local extra data for new files. + */ + localExtra = new ExtraField(); + + Preconditions.checkNotNull( + source, + "Source was not defined, but contents are not on file."); + this.source = source; + } + + /* + * It seems that zip utilities store directories as names ending with "/". + * This seems to be respected by all zip utilities although I could not find there anywhere + * in the specification. + */ + if (cdh.getName().endsWith(Character.toString(ZFile.SEPARATOR))) { + type = StoredEntryType.DIRECTORY; + verifyLog.verify( + this.source.getProcessedByteSource().isEmpty(), + "Directory source is not empty."); + verifyLog.verify(cdh.getCrc32() == 0, "Directory has CRC32 = %s.", cdh.getCrc32()); + verifyLog.verify( + cdh.getUncompressedSize() == 0, + "Directory has uncompressed size = %s.", + cdh.getUncompressedSize()); + + /* + * Some clever (OMG!) tools, like jar will actually try to compress the directory + * contents and generate a 2 byte compressed data. Of course, the uncompressed size is + * zero and we're just wasting space. + */ + long compressedSize = cdh.getCompressionInfoWithWait().getCompressedSize(); + verifyLog.verify( + compressedSize == 0 || compressedSize == 2, + "Directory has compressed size = %s.", compressedSize); + } else { + type = StoredEntryType.FILE; + } + + /* + * By default we assume there is no data descriptor unless the CRC is marked as deferred + * in the header's GP Bit. + */ + dataDescriptorType = DataDescriptorType.NO_DATA_DESCRIPTOR; + if (header.getGpBit().isDeferredCrc()) { + /* + * If the deferred CRC bit exists, then we have an extra descriptor field. This extra + * field may have a signature. + */ + Verify.verify(header.getOffset() >= 0, "Files that are not on disk cannot have the " + + "deferred CRC bit set."); + + try { + readDataDescriptorRecord(); + } catch (IOException e) { + throw new IOException("Failed to read data descriptor record.", e); + } + } + } + + /** + * Obtains the size of the local header of this entry. + * + * @return the local header size in bytes + */ + public int getLocalHeaderSize() { + Preconditions.checkState(!deleted, "deleted"); + return FIXED_LOCAL_FILE_HEADER_SIZE + cdh.getEncodedFileName().length + localExtra.size(); + } + + /** + * Obtains the size of the whole entry on disk, including local header and data descriptor. + * This method will wait until compression information is complete, if needed. + * + * @return the number of bytes + * @throws IOException failed to get compression information + */ + long getInFileSize() throws IOException { + Preconditions.checkState(!deleted, "deleted"); + return cdh.getCompressionInfoWithWait().getCompressedSize() + getLocalHeaderSize() + + dataDescriptorType.size; + } + + /** + * Obtains a stream that allows reading from the entry. + * + * @return a stream that will return as many bytes as the uncompressed entry size + * @throws IOException failed to open the stream + */ + @Nonnull + public InputStream open() throws IOException { + return source.getProcessedByteSource().openStream(); + } + + /** + * Obtains the contents of the file. + * + * @return a byte array with the contents of the file (uncompressed if the file was compressed) + * @throws IOException failed to read the file + */ + @Nonnull + public byte[] read() throws IOException { + try (InputStream is = open()) { + return ByteStreams.toByteArray(is); + } + } + + /** + * Obtains the contents of the file in an existing buffer. + * + * @param bytes buffer to read the file contents in. + * @return the number of bytes read + * @throws IOException failed to read the file. + */ + public int read(byte[] bytes) throws IOException { + if (bytes.length < getCentralDirectoryHeader().getUncompressedSize()) { + throw new RuntimeException( + "Buffer to small while reading {}" + getCentralDirectoryHeader().getName()); + } + try (InputStream is = new BufferedInputStream(open())) { + return ByteStreams.read(is, bytes, 0, bytes.length); + } + } + + /** + * Obtains the type of entry. + * + * @return the type of entry + */ + @Nonnull + public StoredEntryType getType() { + Preconditions.checkState(!deleted, "deleted"); + return type; + } + + /** + * Deletes this entry from the zip file. Invoking this method doesn't update the zip itself. + * To eventually write updates to disk, {@link ZFile#update()} must be called. + * + * @throws IOException failed to delete the entry + * @throws IllegalStateException if the zip file was open in read-only mode + */ + public void delete() throws IOException { + delete(true); + } + + /** + * Deletes this entry from the zip file. Invoking this method doesn't update the zip itself. + * To eventually write updates to disk, {@link ZFile#update()} must be called. + * + * @param notify should listeners be notified of the deletion? This will only be + * {@code false} if the entry is being removed as part of a replacement + * @throws IOException failed to delete the entry + * @throws IllegalStateException if the zip file was open in read-only mode + */ + void delete(boolean notify) throws IOException { + Preconditions.checkState(!deleted, "deleted"); + file.delete(this, notify); + deleted = true; + source.close(); + } + + /** + * Returns {@code true} if this entry has been deleted/replaced. + */ + public boolean isDeleted() { + return deleted; + } + + /** + * Obtains the CDH associated with this entry. + * + * @return the CDH + */ + @Nonnull + public CentralDirectoryHeader getCentralDirectoryHeader() { + return cdh; + } + + /** + * Reads the file's local header and verifies that it matches the Central Directory + * Header provided in the constructor. This method should only be called if the entry already + * exists on disk; new entries do not have local headers. + * <p> + * This method will define the {@link #localExtra} field that is only defined in the + * local descriptor. + * + * @throws IOException failed to read the local header + */ + private void readLocalHeader() throws IOException { + byte[] localHeader = new byte[FIXED_LOCAL_FILE_HEADER_SIZE]; + file.directFullyRead(cdh.getOffset(), localHeader); + + CentralDirectoryHeaderCompressInfo compressInfo = cdh.getCompressionInfoWithWait(); + + ByteBuffer bytes = ByteBuffer.wrap(localHeader); + F_LOCAL_SIGNATURE.verify(bytes); + F_VERSION_EXTRACT.verify(bytes, compressInfo.getVersionExtract(), verifyLog); + F_GP_BIT.verify(bytes, cdh.getGpBit().getValue(), verifyLog); + F_METHOD.verify(bytes, compressInfo.getMethod().methodCode, verifyLog); + + if (file.areTimestampsIgnored()) { + F_LAST_MOD_TIME.skip(bytes); + F_LAST_MOD_DATE.skip(bytes); + } else { + F_LAST_MOD_TIME.verify(bytes, cdh.getLastModTime(), verifyLog); + F_LAST_MOD_DATE.verify(bytes, cdh.getLastModDate(), verifyLog); + } + + /* + * If CRC-32, compressed size and uncompressed size are deferred, their values in Local + * File Header must be ignored and their actual values must be read from the Data + * Descriptor following the contents of this entry. See readDataDescriptorRecord(). + */ + if (cdh.getGpBit().isDeferredCrc()) { + F_CRC32.skip(bytes); + F_COMPRESSED_SIZE.skip(bytes); + F_UNCOMPRESSED_SIZE.skip(bytes); + } else { + F_CRC32.verify(bytes, cdh.getCrc32(), verifyLog); + F_COMPRESSED_SIZE.verify(bytes, compressInfo.getCompressedSize(), verifyLog); + F_UNCOMPRESSED_SIZE.verify(bytes, cdh.getUncompressedSize(), verifyLog); + } + + F_FILE_NAME_LENGTH.verify(bytes, cdh.getEncodedFileName().length); + long extraLength = F_EXTRA_LENGTH.read(bytes); + long fileNameStart = cdh.getOffset() + F_EXTRA_LENGTH.endOffset(); + byte[] fileNameData = new byte[cdh.getEncodedFileName().length]; + file.directFullyRead(fileNameStart, fileNameData); + + String fileName = EncodeUtils.decode(fileNameData, cdh.getGpBit()); + if (!fileName.equals(cdh.getName())) { + verifyLog.log( + String.format( + "Central directory reports file as being named '%s' but local header" + + "reports file being named '%s'.", + cdh.getName(), + fileName)); + } + + long localExtraStart = fileNameStart + cdh.getEncodedFileName().length; + byte[] localExtraRaw = new byte[Ints.checkedCast(extraLength)]; + file.directFullyRead(localExtraStart, localExtraRaw); + localExtra = new ExtraField(localExtraRaw); + } + + /** + * Reads the data descriptor record. This method can only be invoked once it is established + * that a data descriptor does exist. It will read the data descriptor and check that the data + * described there matches the data provided in the Central Directory. + * <p> + * This method will set the {@link #dataDescriptorType} field to the appropriate type of + * data descriptor record. + * + * @throws IOException failed to read the data descriptor record + */ + private void readDataDescriptorRecord() throws IOException { + CentralDirectoryHeaderCompressInfo compressInfo = cdh.getCompressionInfoWithWait(); + + long ddStart = cdh.getOffset() + FIXED_LOCAL_FILE_HEADER_SIZE + + cdh.getName().length() + localExtra.size() + compressInfo.getCompressedSize(); + byte[] ddData = new byte[DataDescriptorType.DATA_DESCRIPTOR_WITH_SIGNATURE.size]; + file.directFullyRead(ddStart, ddData); + + ByteBuffer ddBytes = ByteBuffer.wrap(ddData); + + ZipField.F4 signatureField = new ZipField.F4(0, "Data descriptor signature"); + int cpos = ddBytes.position(); + long sig = signatureField.read(ddBytes); + if (sig == DATA_DESC_SIGNATURE) { + dataDescriptorType = DataDescriptorType.DATA_DESCRIPTOR_WITH_SIGNATURE; + } else { + dataDescriptorType = DataDescriptorType.DATA_DESCRIPTOR_WITHOUT_SIGNATURE; + ddBytes.position(cpos); + } + + ZipField.F4 crc32Field = new ZipField.F4(0, "CRC32"); + ZipField.F4 compressedField = new ZipField.F4(crc32Field.endOffset(), "Compressed size"); + ZipField.F4 uncompressedField = new ZipField.F4(compressedField.endOffset(), + "Uncompressed size"); + + crc32Field.verify(ddBytes, cdh.getCrc32(), verifyLog); + compressedField.verify(ddBytes, compressInfo.getCompressedSize(), verifyLog); + uncompressedField.verify(ddBytes, cdh.getUncompressedSize(), verifyLog); + } + + /** + * Creates a new source that reads data from the zip. + * + * @param zipOffset the offset into the zip file where the data is, must be non-negative + * @throws IOException failed to close the old source + * @return the created source + */ + @Nonnull + private ProcessedAndRawByteSources createSourceFromZip(final long zipOffset) + throws IOException { + Preconditions.checkArgument(zipOffset >= 0, "zipOffset < 0"); + + final CentralDirectoryHeaderCompressInfo compressInfo; + try { + compressInfo = cdh.getCompressionInfoWithWait(); + } catch (IOException e) { + throw new RuntimeException("IOException should never occur here because compression " + + "information should be immediately available if reading from zip.", e); + } + + /* + * Create a source that will return whatever is on the zip file. + */ + CloseableByteSource rawContents = new CloseableByteSource() { + @Override + public long size() throws IOException { + return compressInfo.getCompressedSize(); + } + + @Nonnull + @Override + public InputStream openStream() throws IOException { + Preconditions.checkState(!deleted, "deleted"); + + long dataStart = zipOffset + getLocalHeaderSize(); + long dataEnd = dataStart + compressInfo.getCompressedSize(); + + file.openReadOnly(); + return file.directOpen(dataStart, dataEnd); + } + + @Override + protected void innerClose() throws IOException { + /* + * Nothing to do here. + */ + } + }; + + return createSourcesFromRawContents(rawContents); + } + + /** + * Creates a {@link ProcessedAndRawByteSources} from the raw data source . The processed source + * will either inflate or do nothing depending on the compression information that, at this + * point, should already be available + * + * @param rawContents the raw data to create the source from + * @return the sources for this entry + */ + @Nonnull + private ProcessedAndRawByteSources createSourcesFromRawContents( + @Nonnull CloseableByteSource rawContents) { + CentralDirectoryHeaderCompressInfo compressInfo; + try { + compressInfo = cdh.getCompressionInfoWithWait(); + } catch (IOException e) { + throw new RuntimeException("IOException should never occur here because compression " + + "information should be immediately available if creating from raw " + + "contents.", e); + } + + CloseableByteSource contents; + + /* + * If the contents are deflated, wrap that source in an inflater source so we get the + * uncompressed data. + */ + if (compressInfo.getMethod() == CompressionMethod.DEFLATE) { + contents = new InflaterByteSource(rawContents); + } else { + contents = rawContents; + } + + return new ProcessedAndRawByteSources(contents, rawContents); + } + + /** + * Replaces {@link #source} with one that reads file data from the zip file. + * + * @param zipFileOffset the offset in the zip file where data is written; must be non-negative + * @throws IOException failed to replace the source + */ + void replaceSourceFromZip(long zipFileOffset) throws IOException { + Preconditions.checkArgument(zipFileOffset >= 0, "zipFileOffset < 0"); + + ProcessedAndRawByteSources oldSource = source; + source = createSourceFromZip(zipFileOffset); + cdh.setOffset(zipFileOffset); + oldSource.close(); + } + + /** + * Loads all data in memory and replaces {@link #source} with one that contains all the data + * in memory. + * + * <p>If the entry's contents are already in memory, this call does nothing. + * + * @throws IOException failed to replace the source + */ + void loadSourceIntoMemory() throws IOException { + if (cdh.getOffset() == -1) { + /* + * No offset in the CDR means data has not been written to disk which, in turn, + * means data is already loaded into memory. + */ + return; + } + + ProcessedAndRawByteSources oldSource = source; + byte[] rawContents = oldSource.getRawByteSource().read(); + source = createSourcesFromRawContents(new CloseableDelegateByteSource( + ByteSource.wrap(rawContents), rawContents.length)); + cdh.setOffset(-1); + oldSource.close(); + } + + /** + * Obtains the source data for this entry. This method can only be called for files, it + * cannot be called for directories. + * + * @return the entry source + */ + @Nonnull + ProcessedAndRawByteSources getSource() { + return source; + } + + /** + * Obtains the type of data descriptor used in the entry. + * + * @return the type of data descriptor + */ + @Nonnull + public DataDescriptorType getDataDescriptorType() { + return dataDescriptorType; + } + + /** + * Removes the data descriptor, if it has one and resets the data descriptor bit in the + * central directory header. + * + * @return was the data descriptor remove? + */ + boolean removeDataDescriptor() { + if (dataDescriptorType == DataDescriptorType.NO_DATA_DESCRIPTOR) { + return false; + } + + dataDescriptorType = DataDescriptorType.NO_DATA_DESCRIPTOR; + cdh.resetDeferredCrc(); + return true; + } + + /** + * Obtains the local header data. + * + * @return the header data + * @throws IOException failed to get header byte data + */ + @Nonnull + byte[] toHeaderData() throws IOException { + + byte[] encodedFileName = cdh.getEncodedFileName(); + + ByteBuffer out = + ByteBuffer.allocate( + F_EXTRA_LENGTH.endOffset() + encodedFileName.length + localExtra.size()); + + CentralDirectoryHeaderCompressInfo compressInfo = cdh.getCompressionInfoWithWait(); + + F_LOCAL_SIGNATURE.write(out); + F_VERSION_EXTRACT.write(out, compressInfo.getVersionExtract()); + F_GP_BIT.write(out, cdh.getGpBit().getValue()); + F_METHOD.write(out, compressInfo.getMethod().methodCode); + + if (file.areTimestampsIgnored()) { + F_LAST_MOD_TIME.write(out, 0); + F_LAST_MOD_DATE.write(out, 0); + } else { + F_LAST_MOD_TIME.write(out, cdh.getLastModTime()); + F_LAST_MOD_DATE.write(out, cdh.getLastModDate()); + } + + F_CRC32.write(out, cdh.getCrc32()); + F_COMPRESSED_SIZE.write(out, compressInfo.getCompressedSize()); + F_UNCOMPRESSED_SIZE.write(out, cdh.getUncompressedSize()); + F_FILE_NAME_LENGTH.write(out, cdh.getEncodedFileName().length); + F_EXTRA_LENGTH.write(out, localExtra.size()); + + out.put(cdh.getEncodedFileName()); + localExtra.write(out); + + return out.array(); + } + + /** + * Requests that this entry be realigned. If this entry is already aligned according to the + * rules in {@link ZFile} then this method does nothing. Otherwise it will move the file's data + * into memory and place it in a different area of the zip. + * + * @return has this file been changed? Note that if the entry has not yet been written on the + * file, realignment does not count as a change as nothing needs to be updated in the file; + * also, if the entry has been changed, this object may have been marked as deleted and a new + * stored entry may need to be fetched from the file + * @throws IOException failed to realign the entry; the entry may no longer exist in the zip + * file + */ + public boolean realign() throws IOException { + Preconditions.checkState(!deleted, "Entry has been deleted."); + + return file.realign(this); + } + + /** + * Obtains the contents of the local extra field. + * + * @return the contents of the local extra field + */ + @Nonnull + public ExtraField getLocalExtra() { + return localExtra; + } + + /** + * Sets the contents of the local extra field. + * + * @param localExtra the contents of the local extra field + * @throws IOException failed to update the zip file + */ + public void setLocalExtra(@Nonnull ExtraField localExtra) throws IOException { + boolean resized = setLocalExtraNoNotify(localExtra); + file.localHeaderChanged(this, resized); + } + + /** + * Sets the contents of the local extra field, does not notify the {@link ZFile} of the change. + * This is used internally when the {@link ZFile} itself wants to change the local extra and + * doesn't need the callback. + * + * @param localExtra the contents of the local extra field + * @return has the local header size changed? + * @throws IOException failed to load the file + */ + boolean setLocalExtraNoNotify(@Nonnull ExtraField localExtra) throws IOException { + boolean sizeChanged; + + /* + * Make sure we load into memory. + * + * If we change the size of the local header, the actual start of the file changes + * according to our in-memory structures so, if we don't read the file now, we won't be + * able to load it later :) + * + * But, even if the size doesn't change, we need to read it force the entry to be + * rewritten otherwise the changes in the local header aren't written. Of course this case + * may be optimized with some extra complexity added :) + */ + loadSourceIntoMemory(); + + if (this.localExtra.size() != localExtra.size()) { + sizeChanged = true; + } else { + sizeChanged = false; + } + + this.localExtra = localExtra; + return sizeChanged; + } + + /** + * Obtains the verify log for the entry. + * + * @return the verify log + */ + @Nonnull + public VerifyLog getVerifyLog() { + return verifyLog; + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/zip/StoredEntryType.java b/src/main/java/com/android/tools/build/apkzlib/zip/StoredEntryType.java new file mode 100644 index 0000000..736a813 --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zip/StoredEntryType.java @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zip; + +/** + * Type of stored entry. + */ +public enum StoredEntryType { + /** + * Entry is a file. + */ + FILE, + + /** + * Entry is a directory. + */ + DIRECTORY +} diff --git a/src/main/java/com/android/tools/build/apkzlib/zip/VerifyLog.java b/src/main/java/com/android/tools/build/apkzlib/zip/VerifyLog.java new file mode 100644 index 0000000..4d8ebf8 --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zip/VerifyLog.java @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zip; + +import com.google.common.collect.ImmutableList; +import javax.annotation.Nonnull; + +/** + * The verify log contains verification messages. It is used to capture validation issues with a + * zip file or with parts of a zip file. + */ +public interface VerifyLog { + + /** + * Logs a message. + * + * @param message the message to verify + */ + void log(@Nonnull String message); + + /** + * Obtains all save logged messages. + * + * @return the logged messages + */ + @Nonnull + ImmutableList<String> getLogs(); + + /** + * Performs verification of a non-critical condition, logging a message if the condition is + * not verified. + * + * @param condition the condition + * @param message the message to write if {@code condition} is {@code false}. + * @param args arguments for formatting {@code message} using {@code String.format} + */ + default void verify(boolean condition, @Nonnull String message, @Nonnull Object... args) { + if (!condition) { + log(String.format(message, args)); + } + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/zip/VerifyLogs.java b/src/main/java/com/android/tools/build/apkzlib/zip/VerifyLogs.java new file mode 100644 index 0000000..d1bea7a --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zip/VerifyLogs.java @@ -0,0 +1,77 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zip; + +import com.google.common.collect.ImmutableList; +import java.util.ArrayList; +import java.util.List; +import javax.annotation.Nonnull; + +/** + * Factory for verification logs. + */ +final class VerifyLogs { + + private VerifyLogs() {} + + /** + * Creates a {@link VerifyLog} that ignores all messages logged. + * + * @return the log + */ + @Nonnull + static VerifyLog devNull() { + return new VerifyLog() { + @Override + public void log(@Nonnull String message) {} + + @Nonnull + @Override + public ImmutableList<String> getLogs() { + return ImmutableList.of(); + } + }; + } + + /** + * Creates a {@link VerifyLog} that stores all log messages. + * + * @return the log + */ + @Nonnull + static VerifyLog unlimited() { + return new VerifyLog() { + + /** + * All saved messages. + */ + @Nonnull + private final List<String> messages = new ArrayList<>(); + + @Override + public void log(@Nonnull String message) { + messages.add(message); + } + + @Nonnull + @Override + public ImmutableList<String> getLogs() { + return ImmutableList.copyOf(messages); + } + }; + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/zip/ZFile.java b/src/main/java/com/android/tools/build/apkzlib/zip/ZFile.java new file mode 100644 index 0000000..b7949b5 --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zip/ZFile.java @@ -0,0 +1,2764 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zip; + +import com.android.tools.build.apkzlib.utils.CachedFileContents; +import com.android.tools.build.apkzlib.utils.IOExceptionFunction; +import com.android.tools.build.apkzlib.utils.IOExceptionRunnable; +import com.android.tools.build.apkzlib.zip.compress.Zip64NotSupportedException; +import com.android.tools.build.apkzlib.zip.utils.ByteTracker; +import com.android.tools.build.apkzlib.zip.utils.CloseableByteSource; +import com.android.tools.build.apkzlib.zip.utils.LittleEndianUtils; +import com.google.common.base.Preconditions; +import com.google.common.base.Verify; +import com.google.common.base.VerifyException; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.google.common.hash.Hashing; +import com.google.common.io.ByteSource; +import com.google.common.io.Closer; +import com.google.common.io.Files; +import com.google.common.primitives.Ints; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.common.util.concurrent.SettableFuture; +import java.io.ByteArrayInputStream; +import java.io.Closeable; +import java.io.EOFException; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.function.Supplier; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * The {@code ZFile} provides the main interface for interacting with zip files. A {@code ZFile} + * can be created on a new file or in an existing file. Once created, files can be added or removed + * from the zip file. + * + * <p>Changes in the zip file are always deferred. Any change requested is made in memory and + * written to disk only when {@link #update()} or {@link #close()} is invoked. + * + * <p>Zip files are open initially in read-only mode and will switch to read-write when needed. This + * is done automatically. Because modifications to the file are done in-memory, the zip file can + * be manipulated when closed. When invoking {@link #update()} or {@link #close()} the zip file + * will be reopen and changes will be written. However, the zip file cannot be modified outside + * the control of {@code ZFile}. So, if a {@code ZFile} is closed, modified outside and then a file + * is added or removed from the zip file, when reopening the zip file, {@link ZFile} will detect + * the outside modification and will fail. + * + * <p>In memory manipulation means that files added to the zip file are kept in memory until written + * to disk. This provides much faster operation and allows better zip file allocation (see below). + * It may, however, increase the memory footprint of the application. When adding large files, if + * memory consumption is a concern, a call to {@link #update()} will actually write the file to + * disk and discard the memory buffer. Information about allocation can be obtained from a + * {@link ByteTracker} that can be given to the file on creation. + * + * <p>{@code ZFile} keeps track of allocation inside of the zip file. If a file is deleted, its + * space is marked as freed and will be reused for an added file if it fits in the space. + * Allocation of files to empty areas is done using a <em>best fit</em> algorithm. When adding a + * file, if it doesn't fit in any free area, the zip file will be extended. + * + * <p>{@code ZFile} provides a fast way to merge data from another zip file + * (see {@link #mergeFrom(ZFile, Predicate)}) avoiding recompression and copying of equal files. + * When merging, patterns of files may be provided that are ignored. This allows handling special + * files in the merging process, such as files in {@code META-INF}. + * + * <p>When adding files to the zip file, unless files are explicitly required to be stored, files + * will be deflated. However, deflating will not occur if the deflated file is larger then the + * stored file, <em>e.g.</em> if compression would yield a bigger file. See {@link Compressor} for + * details on how compression works. + * + * <p>Because {@code ZFile} was designed to be used in a build system and not as general-purpose + * zip utility, it is very strict (and unforgiving) about the zip format and unsupported features. + * + * <p>{@code ZFile} supports <em>alignment</em>. Alignment means that file data (not entries -- the + * local header must be discounted) must start at offsets that are multiple of a number -- the + * alignment. Alignment is defined by an alignment rules ({@link AlignmentRule} in the + * {@link ZFileOptions} object used to create the {@link ZFile}. + * + * <p>When a file is added to the zip, the alignment rules will be checked and alignment will be + * honored when positioning the file in the zip. This means that unused spaces in the zip may + * be generated as a result. However, alignment of existing entries will not be changed. + * + * <p>Entries can be realigned individually (see {@link StoredEntry#realign()} or the full zip file + * may be realigned (see {@link #realign()}). When realigning the full zip entries that are already + * aligned will not be affected. + * + * <p>Because realignment may cause files to move in the zip, realignment is done in-memory meaning + * that files that need to change location will moved to memory and will only be flushed when + * either {@link #update()} or {@link #close()} are called. + * + * <p>Alignment only applies to filed that are forced to be uncompressed. This is because alignment + * is used to allow mapping files in the archive directly into memory and compressing defeats the + * purpose of alignment. + * + * <p>Manipulating zip files with {@link ZFile} may yield zip files with empty spaces between files. + * This happens in two situations: (1) if alignment is required, files may be shifted to conform to + * the request alignment leaving an empty space before the previous file, and (2) if a file is + * removed or replaced with a file that does not fit the space it was in. By default, {@link ZFile} + * does not do any special processing in these situations. Files are indexed by their offsets from + * the central directory and empty spaces can exist in the zip file. + * + * <p>However, it is possible to tell {@link ZFile} to use the extra field in the local header + * to do cover the empty spaces. This is done by setting + * {@link ZFileOptions#setCoverEmptySpaceUsingExtraField(boolean)} to {@code true}. This has the + * advantage of leaving no gaps between entries in the zip, as required by some tools like Oracle's + * {code jar} tool. However, setting this option will destroy the contents of the file's extra + * field. + * + * <p>Activating {@link ZFileOptions#setCoverEmptySpaceUsingExtraField(boolean)} may lead to + * <i>virtual files</i> being added to the zip file. Since extra field is limited to 64k, it is not + * possible to cover any space bigger than that using the extra field. In those cases, <i>virtual + * files</i> are added to the file. A virtual file is a file that exists in the actual zip data, + * but is not referenced from the central directory. A zip-compliant utility should ignore these + * files. However, zip utilities that expect the zip to be a stream, such as Oracle's jar, will + * find these files instead of considering the zip to be corrupt. + * + * <p>{@code ZFile} support sorting zip files. Sorting (done through the {@link #sortZipContents()} + * method) is a process by which all files are re-read into memory, if not already in memory, + * removed from the zip and re-added in alphabetical order, respecting alignment rules. So, in + * general, file {@code b} will come after file {@code a} unless file {@code a} is subject to + * alignment that forces an empty space before that can be occupied by {@code b}. Sorting can be + * used to minimize the changes between two zips. + * + * <p>Sorting in {@code ZFile} can be done manually or automatically. Manual sorting is done by + * invoking {@link #sortZipContents()}. Automatic sorting is done by setting the + * {@link ZFileOptions#getAutoSortFiles()} option when creating the {@code ZFile}. Automatic + * sorting invokes {@link #sortZipContents()} immediately when doing an {@link #update()} after + * all extensions have processed the {@link ZFileExtension#beforeUpdate()}. This has the guarantee + * that files added by extensions will be sorted, something that does not happen if the invocation + * is sequential, <i>i.e.</i>, {@link #sortZipContents()} called before {@link #update()}. The + * drawback of automatic sorting is that sorting will happen every time {@link #update()} is + * called and the file is dirty having a possible penalty in performance. + * + * <p>To allow whole-apk signing, the {@code ZFile} allows the central directory location to be + * offset by a fixed amount. This amount can be set using the {@link #setExtraDirectoryOffset(long)} + * method. Setting a non-zero value will add extra (unused) space in the zip file before the + * central directory. This value can be changed at any time and it will force the central directory + * rewritten when the file is updated or closed. + * + * <p>{@code ZFile} provides an extension mechanism to allow objects to register with the file + * and be notified when changes to the file happen. This should be used + * to add extra features to the zip file while providing strong decoupling. See + * {@link ZFileExtension}, {@link ZFile#addZFileExtension(ZFileExtension)} and + * {@link ZFile#removeZFileExtension(ZFileExtension)}. + * + * <p>This class is <strong>not</strong> thread-safe. Neither are any of the classes associated with + * it in this package, except when otherwise noticed. + */ +public class ZFile implements Closeable { + + /** + * The file separator in paths in the zip file. This is fixed by the zip specification + * (section 4.4.17). + */ + public static final char SEPARATOR = '/'; + + /** + * Minimum size the EOCD can have. + */ + private static final int MIN_EOCD_SIZE = 22; + + /** + * Number of bytes of the Zip64 EOCD locator record. + */ + private static final int ZIP64_EOCD_LOCATOR_SIZE = 20; + + /** + * Maximum size for the EOCD. + */ + private static final int MAX_EOCD_COMMENT_SIZE = 65535; + + /** + * How many bytes to look back from the end of the file to look for the EOCD signature. + */ + private static final int LAST_BYTES_TO_READ = MIN_EOCD_SIZE + MAX_EOCD_COMMENT_SIZE; + + /** + * Signature of the Zip64 EOCD locator record. + */ + private static final int ZIP64_EOCD_LOCATOR_SIGNATURE = 0x07064b50; + + /** + * Signature of the EOCD record. + */ + private static final byte[] EOCD_SIGNATURE = new byte[] { 0x06, 0x05, 0x4b, 0x50 }; + + /** + * Size of buffer for I/O operations. + */ + private static final int IO_BUFFER_SIZE = 1024 * 1024; + + /** + * When extensions request re-runs, we do maximum number of cycles until we decide to stop and + * flag a infinite recursion problem. + */ + private static final int MAXIMUM_EXTENSION_CYCLE_COUNT = 10; + + /** + * Minimum size for the extra field when we have to add one. We rely on the alignment segment + * to do that so the minimum size for the extra field is the minimum size of an alignment + * segment. + */ + private static final int MINIMUM_EXTRA_FIELD_SIZE = ExtraField.AlignmentSegment.MINIMUM_SIZE; + + /** + * Maximum size of the extra field. + * + * <p>Theoretically, this is (1 << 16) - 1 = 65535 and not (1 < 15) -1 = 32767. However, due to + * http://b.android.com/221703, we need to keep this limited. + */ + private static final int MAX_LOCAL_EXTRA_FIELD_CONTENTS_SIZE = (1 << 15) - 1; + + /** + * File zip file. + */ + @Nonnull + private final File file; + + /** + * The random access file used to access the zip file. This will be {@code null} if and only + * if {@link #state} is {@link ZipFileState#CLOSED}. + */ + @Nullable + private RandomAccessFile raf; + + /** + * The map containing the in-memory contents of the zip file. It keeps track of which parts of + * the zip file are used and which are not. + */ + @Nonnull + private final FileUseMap map; + + /** + * The EOCD entry. Will be {@code null} if there is no EOCD (because the zip is new) or the + * one that exists on disk is no longer valid (because the zip has been changed). + * + * <p>If the EOCD is deleted because the zip has been changed and the old EOCD was no longer + * valid, then {@link #eocdComment} will contain the comment saved from the EOCD. + */ + @Nullable + private FileUseMapEntry<Eocd> eocdEntry; + + /** + * The Central Directory entry. Will be {@code null} if there is no Central Directory (because + * the zip is new) or because the one that exists on disk is no longer valid (because the zip + * has been changed). + */ + @Nullable + private FileUseMapEntry<CentralDirectory> directoryEntry; + + /** + * All entries in the zip file. It includes in-memory changes and may not reflect what is + * written on disk. Only entries that have been compressed are in this list. + */ + @Nonnull + private final Map<String, FileUseMapEntry<StoredEntry>> entries; + + /** + * Entries added to the zip file, but that are not yet compressed. When compression is done, + * these entries are eventually moved to {@link #entries}. uncompressedEntries is a list + * because entries need to be kept in the order by which they were added. It allows adding + * multiple files with the same name and getting the right notifications on which files replaced + * which. + * + * <p>Files are placed in this list in {@link #add(StoredEntry)} method. This method will + * keep files here temporarily and move then to {@link #entries} when the data is + * available. + * + * <p>Moving files out of this list to {@link #entries} is done by + * {@link #processAllReadyEntries()}. + */ + @Nonnull + private final List<StoredEntry> uncompressedEntries; + + /** + * Current state of the zip file. + */ + @Nonnull + private ZipFileState state; + + /** + * Are the in-memory changes that have not been written to the zip file? + * + * <p>This might be false, but will become true after {@link #processAllReadyEntriesWithWait()} + * is called if there are {@link #uncompressedEntries} compressing in the background. + */ + private boolean dirty; + + /** + * Non-{@code null} only if the file is currently closed. Used to detect if the zip is + * modified outside this object's control. If the file has never been written, this will + * be {@code null} even if it is closed. + */ + @Nullable + private CachedFileContents<Object> closedControl; + + /** + * The alignment rule. + */ + @Nonnull + private final AlignmentRule alignmentRule; + + /** + * Extensions registered with the file. + */ + @Nonnull + private final List<ZFileExtension> extensions; + + /** + * When notifying extensions, extensions may request that some runnables are executed. This + * list collects all runnables by the order they were requested. Together with + * {@link #isNotifying}, it is used to avoid reordering notifications. + */ + @Nonnull + private final List<IOExceptionRunnable> toRun; + + /** + * {@code true} when {@link #notify(com.android.tools.build.apkzlib.utils.IOExceptionFunction)} + * is notifying extensions. Used to avoid reordering notifications. + */ + private boolean isNotifying; + + /** + * An extra offset for the central directory location. {@code 0} if the central directory + * should be written in its standard location. + */ + private long extraDirectoryOffset; + + /** + * Should all timestamps be zeroed when reading / writing the zip? + */ + private boolean noTimestamps; + + /** + * Compressor to use. + */ + @Nonnull + private Compressor compressor; + + /** + * Byte tracker to use. + */ + @Nonnull + private final ByteTracker tracker; + + /** + * Use the zip entry's "extra field" field to cover empty space in the zip file? + */ + private boolean coverEmptySpaceUsingExtraField; + + /** + * Should files be automatically sorted when updating? + */ + private boolean autoSortFiles; + + /** + * Verify log factory to use. + */ + @Nonnull + private final Supplier<VerifyLog> verifyLogFactory; + + /** + * Verify log to use. + */ + @Nonnull + private final VerifyLog verifyLog; + + /** + * This field contains the comment in the zip's EOCD if there is no in-memory EOCD structure. + * This may happen, for example, if the zip has been changed and the Central Directory and + * EOCD have been deleted (in-memory). In that case, this field will save the comment to place + * on the EOCD once it is created. + * + * <p>This field will only be non-{@code null} if there is no in-memory EOCD structure + * (<i>i.e.</i>, {@link #eocdEntry} is {@code null}). If there is an {@link #eocdEntry}, then + * the comment will be there instead of being in this field. + */ + @Nullable + private byte[] eocdComment; + + /** + * Is the file in read-only mode? In read-only mode no changes are allowed. + */ + private boolean readOnly; + + + /** + * Creates a new zip file. If the zip file does not exist, then no file is created at this + * point and {@code ZFile} will contain an empty structure. However, an (empty) zip file will + * be created if either {@link #update()} or {@link #close()} are used. If a zip file exists, + * it will be parsed and read. + * + * @param file the zip file + * @throws IOException some file exists but could not be read + */ + public ZFile(@Nonnull File file) throws IOException { + this(file, new ZFileOptions()); + } + + /** + * Creates a new zip file. If the zip file does not exist, then no file is created at this + * point and {@code ZFile} will contain an empty structure. However, an (empty) zip file will + * be created if either {@link #update()} or {@link #close()} are used. If a zip file exists, + * it will be parsed and read. + * + * @param file the zip file + * @param options configuration options + * @throws IOException some file exists but could not be read + */ + public ZFile(@Nonnull File file, @Nonnull ZFileOptions options) throws IOException { + this(file, options, false); + } + + /** + * Creates a new zip file. If the zip file does not exist, then no file is created at this + * point and {@code ZFile} will contain an empty structure. However, an (empty) zip file will + * be created if either {@link #update()} or {@link #close()} are used. If a zip file exists, + * it will be parsed and read. + * + * @param file the zip file + * @param options configuration options + * @param readOnly should the file be open in read-only mode? If {@code true} then the file must + * exist and no methods can be invoked that could potentially change the file + * @throws IOException some file exists but could not be read + */ + public ZFile(@Nonnull File file, @Nonnull ZFileOptions options, boolean readOnly) + throws IOException { + this.file = file; + map = new FileUseMap( + 0, + options.getCoverEmptySpaceUsingExtraField() + ? MINIMUM_EXTRA_FIELD_SIZE + : 0); + this.readOnly = readOnly; + dirty = false; + closedControl = null; + alignmentRule = options.getAlignmentRule(); + extensions = Lists.newArrayList(); + toRun = Lists.newArrayList(); + noTimestamps = options.getNoTimestamps(); + tracker = options.getTracker(); + compressor = options.getCompressor(); + coverEmptySpaceUsingExtraField = options.getCoverEmptySpaceUsingExtraField(); + autoSortFiles = options.getAutoSortFiles(); + verifyLogFactory = options.getVerifyLogFactory(); + verifyLog = verifyLogFactory.get(); + + /* + * These two values will be overwritten by openReadOnly() below if the file exists. + */ + state = ZipFileState.CLOSED; + raf = null; + + if (file.exists()) { + openReadOnly(); + } else if (readOnly) { + throw new IOException("File does not exist but read-only mode requested"); + } else { + dirty = true; + } + + entries = Maps.newHashMap(); + uncompressedEntries = Lists.newArrayList(); + extraDirectoryOffset = 0; + + try { + if (state != ZipFileState.CLOSED) { + long rafSize = raf.length(); + if (rafSize > Integer.MAX_VALUE) { + throw new IOException("File exceeds size limit of " + Integer.MAX_VALUE + "."); + } + + map.extend(Ints.checkedCast(rafSize)); + readData(); + } + + // If we don't have an EOCD entry, set the comment to empty. + if (eocdEntry == null) { + eocdComment = new byte[0]; + } + + // Notify the extensions if the zip file has been open. + if (state != ZipFileState.CLOSED) { + notify(ZFileExtension::open); + } + } catch (Zip64NotSupportedException e) { + throw e; + } catch (IOException e) { + throw new IOException("Failed to read zip file '" + file.getAbsolutePath() + "'.", e); + } catch (IllegalStateException | IllegalArgumentException | VerifyException e) { + throw new RuntimeException( + "Internal error when trying to read zip file '" + file.getAbsolutePath() + "'.", + e); + } + } + + /** + * Obtains all entries in the file. Entries themselves may be or not written in disk. However, + * all of them can be open for reading. + * + * @return all entries in the zip + */ + @Nonnull + public Set<StoredEntry> entries() { + Map<String, StoredEntry> entries = Maps.newHashMap(); + + for (FileUseMapEntry<StoredEntry> mapEntry : this.entries.values()) { + StoredEntry entry = mapEntry.getStore(); + assert entry != null; + entries.put(entry.getCentralDirectoryHeader().getName(), entry); + } + + /* + * mUncompressed may override mEntriesReady as we may not have yet processed all + * entries. + */ + for (StoredEntry uncompressed : uncompressedEntries) { + entries.put(uncompressed.getCentralDirectoryHeader().getName(), uncompressed); + } + + return Sets.newHashSet(entries.values()); + } + + /** + * Obtains an entry at a given path in the zip. + * + * @param path the path + * @return the entry at the path or {@code null} if none exists + */ + @Nullable + public StoredEntry get(@Nonnull String path) { + /* + * The latest entries are the last ones in uncompressed and they may eventually override + * files in entries. + */ + for (StoredEntry stillUncompressed : Lists.reverse(uncompressedEntries)) { + if (stillUncompressed.getCentralDirectoryHeader().getName().equals(path)) { + return stillUncompressed; + } + } + + FileUseMapEntry<StoredEntry> found = entries.get(path); + if (found == null) { + return null; + } + + return found.getStore(); + } + + /** + * Reads all the data in the zip file, except the contents of the entries themselves. This + * method will populate the directory and maps in the instance variables. + * + * @throws IOException failed to read the zip file + */ + private void readData() throws IOException { + Preconditions.checkState(state != ZipFileState.CLOSED, "state == ZipFileState.CLOSED"); + Preconditions.checkState(raf != null, "raf == null"); + + readEocd(); + readCentralDirectory(); + + /* + * Go over all files and create the usage map, verifying there is no overlap in the files. + */ + long entryEndOffset; + long directoryStartOffset; + + if (directoryEntry != null) { + CentralDirectory directory = directoryEntry.getStore(); + assert directory != null; + + entryEndOffset = 0; + + for (StoredEntry entry : directory.getEntries().values()) { + long start = entry.getCentralDirectoryHeader().getOffset(); + long end = start + entry.getInFileSize(); + + /* + * If isExtraAlignmentBlock(entry.getLocalExtra()) is true, we know the entry + * has an extra field that is solely used for alignment. This means the + * actual entry could start at start + extra.length and leave space before. + * + * But, if we did this here, we would be modifying the zip file and that is + * weird because we're just opening it for reading. + * + * The downside is that we will never reuse that space. Maybe one day ZFile + * can be clever enough to remove the local extra when we start modifying the zip + * file. + */ + + Verify.verify(start >= 0, "start < 0"); + Verify.verify(end < map.size(), "end >= map.size()"); + + FileUseMapEntry<?> found = map.at(start); + Verify.verifyNotNull(found); + + // We've got a problem if the found entry is not free or is a free entry but + // doesn't cover the whole file. + if (!found.isFree() || found.getEnd() < end) { + if (found.isFree()) { + found = map.after(found); + Verify.verify(found != null && !found.isFree()); + } + + Object foundEntry = found.getStore(); + Verify.verify(foundEntry != null); + + // Obtains a custom description of an entry. + IOExceptionFunction<StoredEntry, String> describe = + e -> + String.format( + "'%s' (offset: %d, size: %d)", + e.getCentralDirectoryHeader().getName(), + e.getCentralDirectoryHeader().getOffset(), + e.getInFileSize()); + + String overlappingEntryDescription; + if (foundEntry instanceof StoredEntry) { + StoredEntry foundStored = (StoredEntry) foundEntry; + overlappingEntryDescription = describe.apply((StoredEntry) foundEntry); + } else { + overlappingEntryDescription = + "Central Directory / EOCD: " + + found.getStart() + + " - " + + found.getEnd(); + } + + throw new IOException( + "Cannot read entry " + + describe.apply(entry) + + " because it overlaps with " + + overlappingEntryDescription); + } + + FileUseMapEntry<StoredEntry> mapEntry = map.add(start, end, entry); + entries.put(entry.getCentralDirectoryHeader().getName(), mapEntry); + + if (end > entryEndOffset) { + entryEndOffset = end; + } + } + + directoryStartOffset = directoryEntry.getStart(); + } else { + /* + * No directory means an empty zip file. Use the start of the EOCD to compute + * an existing offset. + */ + Verify.verifyNotNull(eocdEntry); + assert eocdEntry != null; + directoryStartOffset = eocdEntry.getStart(); + entryEndOffset = 0; + } + + /* + * Check if there is an extra central directory offset. If there is, save it. Note that + * we can't call extraDirectoryOffset() because that would mark the file as dirty. + */ + long extraOffset = directoryStartOffset - entryEndOffset; + Verify.verify(extraOffset >= 0, "extraOffset (%s) < 0", extraOffset); + extraDirectoryOffset = extraOffset; + } + + /** + * Finds the EOCD marker and reads it. It will populate the {@link #eocdEntry} variable. + * + * @throws IOException failed to read the EOCD + */ + private void readEocd() throws IOException { + Preconditions.checkState(state != ZipFileState.CLOSED, "state == ZipFileState.CLOSED"); + Preconditions.checkState(raf != null, "raf == null"); + + /* + * Read the last part of the zip into memory. If we don't find the EOCD signature by then, + * the file is corrupt. + */ + int lastToRead = LAST_BYTES_TO_READ; + if (lastToRead > raf.length()) { + lastToRead = Ints.checkedCast(raf.length()); + } + + byte[] last = new byte[lastToRead]; + directFullyRead(raf.length() - lastToRead, last); + + + /* + * Start endIdx at the first possible location where the signature can be located and then + * move backwards. Because the EOCD must have at least MIN_EOCD size, the first byte of the + * signature (and first byte of the EOCD) must be located at last.length - MIN_EOCD_SIZE. + * + * Because the EOCD signature may exist in the file comment, when we find a signature we + * will try to read the Eocd. If we fail, we continue searching for the signature. However, + * we will keep the last exception in case we don't find any signature. + */ + Eocd eocd = null; + int foundEocdSignature = -1; + IOException errorFindingSignature = null; + int eocdStart = -1; + + for (int endIdx = last.length - MIN_EOCD_SIZE; endIdx >= 0 && foundEocdSignature == -1; + endIdx--) { + /* + * Remember: little endian... + */ + if (last[endIdx] == EOCD_SIGNATURE[3] + && last[endIdx + 1] == EOCD_SIGNATURE[2] + && last[endIdx + 2] == EOCD_SIGNATURE[1] + && last[endIdx + 3] == EOCD_SIGNATURE[0]) { + + /* + * We found a signature. Try to read the EOCD record. + */ + + foundEocdSignature = endIdx; + ByteBuffer eocdBytes = + ByteBuffer.wrap(last, foundEocdSignature, last.length - foundEocdSignature); + + try { + eocd = new Eocd(eocdBytes); + eocdStart = Ints.checkedCast(raf.length() - lastToRead + foundEocdSignature); + + /* + * Make sure the EOCD takes the whole file up to the end. Log an error if it + * doesn't. + */ + if (eocdStart + eocd.getEocdSize() != raf.length()) { + verifyLog.log("EOCD starts at " + + eocdStart + + " and has " + + eocd.getEocdSize() + + " bytes, but file ends at " + + raf.length() + + "."); + } + } catch (IOException e) { + if (errorFindingSignature != null) { + e.addSuppressed(errorFindingSignature); + } + + errorFindingSignature = e; + foundEocdSignature = -1; + eocd = null; + } + } + } + + if (foundEocdSignature == -1) { + throw new IOException("EOCD signature not found in the last " + + lastToRead + " bytes of the file.", errorFindingSignature); + } + + Verify.verify(eocdStart >= 0); + + /* + * Look for the Zip64 central directory locator. If we find it, then this file is a Zip64 + * file and we do not support it. + */ + int zip64LocatorStart = eocdStart - ZIP64_EOCD_LOCATOR_SIZE; + if (zip64LocatorStart >= 0) { + byte[] possibleZip64Locator = new byte[4]; + directFullyRead(zip64LocatorStart, possibleZip64Locator); + if (LittleEndianUtils.readUnsigned4Le(ByteBuffer.wrap(possibleZip64Locator)) == + ZIP64_EOCD_LOCATOR_SIGNATURE) { + throw new Zip64NotSupportedException( + "Zip64 EOCD locator found but Zip64 format is not supported."); + } + } + + eocdEntry = map.add(eocdStart, eocdStart + eocd.getEocdSize(), eocd); + } + + /** + * Reads the zip's central directory and populates the {@link #directoryEntry} variable. This + * method can only be called after the EOCD has been read. If the central directory is empty + * (if there are no files on the zip archive), then {@link #directoryEntry} will be set to + * {@code null}. + * + * @throws IOException failed to read the central directory + */ + private void readCentralDirectory() throws IOException { + Preconditions.checkNotNull(eocdEntry, "eocdEntry == null"); + Preconditions.checkNotNull(eocdEntry.getStore(), "eocdEntry.getStore() == null"); + Preconditions.checkState(state != ZipFileState.CLOSED, "state == ZipFileState.CLOSED"); + Preconditions.checkState(raf != null, "raf == null"); + Preconditions.checkState(directoryEntry == null, "directoryEntry != null"); + + Eocd eocd = eocdEntry.getStore(); + + long dirSize = eocd.getDirectorySize(); + if (dirSize > Integer.MAX_VALUE) { + throw new IOException("Cannot read central directory with size " + dirSize + "."); + } + + long centralDirectoryEnd = eocd.getDirectoryOffset() + dirSize; + if (centralDirectoryEnd != eocdEntry.getStart()) { + String msg = "Central directory is stored in [" + + eocd.getDirectoryOffset() + + " - " + + (centralDirectoryEnd - 1) + + "] and EOCD starts at " + + eocdEntry.getStart() + + "."; + + /* + * If there is an empty space between the central directory and the EOCD, we proceed + * logging an error. If the central directory ends after the start of the EOCD (and + * therefore, they overlap), throw an exception. + */ + if (centralDirectoryEnd > eocdEntry.getSize()) { + throw new IOException(msg); + } else { + verifyLog.log(msg); + } + } + + byte[] directoryData = new byte[Ints.checkedCast(dirSize)]; + directFullyRead(eocd.getDirectoryOffset(), directoryData); + + CentralDirectory directory = + CentralDirectory.makeFromData( + ByteBuffer.wrap(directoryData), + eocd.getTotalRecords(), + this); + if (eocd.getDirectorySize() > 0) { + directoryEntry = map.add( + eocd.getDirectoryOffset(), + eocd.getDirectoryOffset() + eocd.getDirectorySize(), + directory); + } + } + + /** + * Opens a portion of the zip for reading. The zip must be open for this method to be invoked. + * Note that if the zip has not been updated, the individual zip entries may not have been + * written yet. + * + * @param start the index within the zip file to start reading + * @param end the index within the zip file to end reading (the actual byte pointed by + * <em>end</em> will not be read) + * @return a stream that will read the portion of the file; no decompression is done, data is + * returned <em>as is</em> + * @throws IOException failed to open the zip file + */ + @Nonnull + public InputStream directOpen(final long start, final long end) throws IOException { + Preconditions.checkState(state != ZipFileState.CLOSED, "state == ZipFileState.CLOSED"); + Preconditions.checkState(raf != null, "raf == null"); + Preconditions.checkArgument(start >= 0, "start < 0"); + Preconditions.checkArgument(end >= start, "end < start"); + Preconditions.checkArgument(end <= raf.length(), "end > raf.length()"); + + return new InputStream() { + private long mCurr = start; + + @Override + public int read() throws IOException { + if (mCurr == end) { + return -1; + } + + byte[] b = new byte[1]; + int r = directRead(mCurr, b); + if (r > 0) { + mCurr++; + return b[0]; + } else { + return -1; + } + } + + @Override + public int read(@Nonnull byte[] b, int off, int len) throws IOException { + Preconditions.checkNotNull(b, "b == null"); + Preconditions.checkArgument(off >= 0, "off < 0"); + Preconditions.checkArgument(off <= b.length, "off > b.length"); + Preconditions.checkArgument(len >= 0, "len < 0"); + Preconditions.checkArgument(off + len <= b.length, "off + len > b.length"); + + long availableToRead = end - mCurr; + long toRead = Math.min(len, availableToRead); + + if (toRead == 0) { + return -1; + } + + if (toRead > Integer.MAX_VALUE) { + throw new IOException("Cannot read " + toRead + " bytes."); + } + + int r = directRead(mCurr, b, off, Ints.checkedCast(toRead)); + if (r > 0) { + mCurr += r; + } + + return r; + } + }; + } + + /** + * Deletes an entry from the zip. This method does not actually delete anything on disk. It + * just changes in-memory structures. Use {@link #update()} to update the contents on disk. + * + * @param entry the entry to delete + * @param notify should listeners be notified of the deletion? This will only be + * {@code false} if the entry is being removed as part of a replacement + * @throws IOException failed to delete the entry + * @throws IllegalStateException if open in read-only mode + */ + void delete(@Nonnull final StoredEntry entry, boolean notify) throws IOException { + checkNotInReadOnlyMode(); + + String path = entry.getCentralDirectoryHeader().getName(); + FileUseMapEntry<StoredEntry> mapEntry = entries.get(path); + Preconditions.checkNotNull(mapEntry, "mapEntry == null"); + Preconditions.checkArgument(entry == mapEntry.getStore(), "entry != mapEntry.getStore()"); + + dirty = true; + + map.remove(mapEntry); + entries.remove(path); + + if (notify) { + notify(ext -> ext.removed(entry)); + } + } + + /** + * Checks that the file is not in read-only mode. + * + * @throws IllegalStateException if the file is in read-only mode + */ + private void checkNotInReadOnlyMode() { + if (readOnly) { + throw new IllegalStateException("Illegal operation in read only model"); + } + } + + /** + * Updates the file writing new entries and removing deleted entries. This will force + * reopening the file as read/write if the file wasn't open in read/write mode. + * + * @throws IOException failed to update the file; this exception may have been thrown by + * the compressor but only reported here + */ + public void update() throws IOException { + checkNotInReadOnlyMode(); + + /* + * Process all background stuff before calling in the extensions. + */ + processAllReadyEntriesWithWait(); + + notify(ZFileExtension::beforeUpdate); + + /* + * Process all background stuff that may be leftover by the extensions. + */ + processAllReadyEntriesWithWait(); + + + if (!dirty) { + return; + } + + reopenRw(); + + /* + * At this point, no more files can be added. We may need to repack to remove extra + * empty spaces or sort. If we sort, we don't need to repack as sorting forces the + * zip file to be as compact as possible. + */ + if (autoSortFiles) { + sortZipContents(); + } else { + packIfNecessary(); + } + + /* + * We're going to change the file so delete the central directory and the EOCD as they + * will have to be rewritten. + */ + deleteDirectoryAndEocd(); + map.truncate(); + + /* + * If we need to use the extra field to cover empty spaces, we do the processing here. + */ + if (coverEmptySpaceUsingExtraField) { + + /* We will go over all files in the zip and check whether there is empty space before + * them. If there is, then we will move the entry to the beginning of the empty space + * (covering it) and extend the extra field with the size of the empty space. + */ + for (FileUseMapEntry<StoredEntry> entry : new HashSet<>(entries.values())) { + StoredEntry storedEntry = entry.getStore(); + assert storedEntry != null; + + FileUseMapEntry<?> before = map.before(entry); + if (before == null || !before.isFree()) { + continue; + } + + /* + * We have free space before the current entry. However, we do know that it can + * be covered by the extra field, because both sortZipContents() and + * packIfNecessary() guarantee it. + */ + int localExtraSize = + storedEntry.getLocalExtra().size() + Ints.checkedCast(before.getSize()); + Verify.verify(localExtraSize <= MAX_LOCAL_EXTRA_FIELD_CONTENTS_SIZE); + + /* + * Move file back in the zip. + */ + storedEntry.loadSourceIntoMemory(); + + long newStart = before.getStart(); + long newSize = entry.getSize() + before.getSize(); + + /* + * Remove the entry. + */ + String name = storedEntry.getCentralDirectoryHeader().getName(); + map.remove(entry); + Verify.verify(entry == entries.remove(name)); + + /* + * Make a list will all existing segments in the entry's extra field, but remove + * the alignment field, if it exists. Also, sum the size of all kept extra field + * segments. + */ + ImmutableList<ExtraField.Segment> currentSegments; + try { + currentSegments = storedEntry.getLocalExtra().getSegments(); + } catch (IOException e) { + /* + * Parsing current segments has failed. This means the contents of the extra + * field are not valid. We'll continue discarding the existing segments. + */ + currentSegments = ImmutableList.of(); + } + + List<ExtraField.Segment> extraFieldSegments = new ArrayList<>(); + int newExtraFieldSize = currentSegments.stream() + .filter(s -> s.getHeaderId() + != ExtraField.ALIGNMENT_ZIP_EXTRA_DATA_FIELD_HEADER_ID) + .peek(extraFieldSegments::add) + .map(ExtraField.Segment::size) + .reduce(0, Integer::sum); + + int spaceToFill = + Ints.checkedCast( + before.getSize() + + storedEntry.getLocalExtra().size() + - newExtraFieldSize); + + extraFieldSegments.add( + new ExtraField.AlignmentSegment(chooseAlignment(storedEntry),spaceToFill)); + + storedEntry.setLocalExtraNoNotify( + new ExtraField(ImmutableList.copyOf(extraFieldSegments))); + entries.put(name, map.add(newStart, newStart + newSize, storedEntry)); + + /* + * Reset the offset to force the file to be rewritten. + */ + storedEntry.getCentralDirectoryHeader().setOffset(-1); + } + } + + /* + * Write new files in the zip. We identify new files because they don't have an offset + * in the zip where they are written although we already know, by their location in the + * file map, where they will be written to. + * + * Before writing the files, we sort them in the order they are written in the file so that + * writes are made in order on disk. + * This is, however, unlikely to optimize anything relevant given the way the Operating + * System does caching, but it certainly won't hurt :) + */ + TreeMap<FileUseMapEntry<?>, StoredEntry> toWriteToStore = + new TreeMap<>(FileUseMapEntry.COMPARE_BY_START); + + for (FileUseMapEntry<StoredEntry> entry : entries.values()) { + StoredEntry entryStore = entry.getStore(); + assert entryStore != null; + if (entryStore.getCentralDirectoryHeader().getOffset() == -1) { + toWriteToStore.put(entry, entryStore); + } + } + + /* + * Add all free entries to the set. + */ + for(FileUseMapEntry<?> freeArea : map.getFreeAreas()) { + toWriteToStore.put(freeArea, null); + } + + /* + * Write everything to file. + */ + for (FileUseMapEntry<?> fileUseMapEntry : toWriteToStore.keySet()) { + StoredEntry entry = toWriteToStore.get(fileUseMapEntry); + if (entry == null) { + int size = Ints.checkedCast(fileUseMapEntry.getSize()); + directWrite(fileUseMapEntry.getStart(), new byte[size]); + } else { + writeEntry(entry, fileUseMapEntry.getStart()); + } + } + + boolean hasCentralDirectory; + int extensionBugDetector = MAXIMUM_EXTENSION_CYCLE_COUNT; + do { + computeCentralDirectory(); + computeEocd(); + + hasCentralDirectory = (directoryEntry != null); + + notify(ext -> { + ext.entriesWritten(); + return null; + }); + + if ((--extensionBugDetector) == 0) { + throw new IOException("Extensions keep resetting the central directory. This is " + + "probably a bug."); + } + } while (hasCentralDirectory && directoryEntry == null); + + appendCentralDirectory(); + appendEocd(); + + Verify.verifyNotNull(raf); + raf.setLength(map.size()); + + dirty = false; + + notify(ext -> { + ext.updated(); + return null; + }); + } + + /** + * Reorganizes the zip so that there are no gaps between files bigger than + * {@link #MAX_LOCAL_EXTRA_FIELD_CONTENTS_SIZE} if {@link #coverEmptySpaceUsingExtraField} + * is set to {@code true}. + * + * <p>Essentially, this makes sure we can cover any empty space with the extra field, given + * that the local extra field is limited to {@link #MAX_LOCAL_EXTRA_FIELD_CONTENTS_SIZE}. If + * an entry is too far from the previous one, it is removed and re-added. + * + * @throws IOException failed to repack + */ + private void packIfNecessary() throws IOException { + if (!coverEmptySpaceUsingExtraField) { + return; + } + + SortedSet<FileUseMapEntry<StoredEntry>> entriesByLocation = + new TreeSet<>(FileUseMapEntry.COMPARE_BY_START); + entriesByLocation.addAll(entries.values()); + + for (FileUseMapEntry<StoredEntry> entry : entriesByLocation) { + StoredEntry storedEntry = entry.getStore(); + assert storedEntry != null; + + FileUseMapEntry<?> before = map.before(entry); + if (before == null || !before.isFree()) { + continue; + } + + int localExtraSize = + storedEntry.getLocalExtra().size() + Ints.checkedCast(before.getSize()); + if (localExtraSize > MAX_LOCAL_EXTRA_FIELD_CONTENTS_SIZE) { + /* + * This entry is too far from the previous one. Remove it and re-add it to the + * zip file. + */ + reAdd(storedEntry, PositionHint.LOWEST_OFFSET); + } + } + } + + /** + * Removes a stored entry from the zip and adds it back again. This will force the entry to be + * loaded into memory and repositioned in the zip file. It will also mark the archive as + * being dirty. + * + * @param entry the entry + * @param positionHint hint to where the file should be positioned when re-adding + * @throws IOException failed to load the entry into memory + */ + private void reAdd(@Nonnull StoredEntry entry, @Nonnull PositionHint positionHint) + throws IOException { + String name = entry.getCentralDirectoryHeader().getName(); + FileUseMapEntry<StoredEntry> mapEntry = entries.get(name); + Preconditions.checkNotNull(mapEntry); + Preconditions.checkState(mapEntry.getStore() == entry); + + entry.loadSourceIntoMemory(); + + map.remove(mapEntry); + entries.remove(name); + FileUseMapEntry<StoredEntry> positioned = positionInFile(entry, positionHint); + entries.put(name, positioned); + dirty = true; + } + + /** + * Invoked from {@link StoredEntry} when entry has changed in a way that forces the local + * header to be rewritten + * + * @param entry the entry that changed + * @param resized was the local header resized? + * @throws IOException failed to load the entry into memory + */ + void localHeaderChanged(@Nonnull StoredEntry entry, boolean resized) throws IOException { + dirty = true; + + if (resized) { + reAdd(entry, PositionHint.ANYWHERE); + } + } + + /** + * Invoked when the central directory has changed and needs to be rewritten. + */ + void centralDirectoryChanged() { + dirty = true; + deleteDirectoryAndEocd(); + } + + /** + * Updates the file and closes it. + */ + @Override + public void close() throws IOException { + // We need to make sure to release raf, otherwise we end up locking the file on + // Windows. Use try-with-resources to handle exception suppressing. + try (Closeable ignored = this::innerClose) { + if (!readOnly) { + update(); + } + } + + notify(ext -> { + ext.closed(); + return null; + }); + } + + /** + * Removes the Central Directory and EOCD from the file. This will free space for new entries + * as well as allowing the zip file to be truncated if files have been removed. + * + * <p>This method does not mark the zip as dirty. + */ + private void deleteDirectoryAndEocd() { + if (directoryEntry != null) { + map.remove(directoryEntry); + directoryEntry = null; + } + + if (eocdEntry != null) { + map.remove(eocdEntry); + + Eocd eocd = eocdEntry.getStore(); + Verify.verify(eocd != null); + eocdComment = eocd.getComment(); + eocdEntry = null; + } + } + + /** + * Writes an entry's data in the zip file. This includes everything: the local header and + * the data itself. After writing, the entry is updated with the offset and its source replaced + * with a source that reads from the zip file. + * + * @param entry the entry to write + * @param offset the offset at which the entry should be written + * @throws IOException failed to write the entry + */ + private void writeEntry(@Nonnull StoredEntry entry, long offset) throws IOException { + Preconditions.checkArgument(entry.getDataDescriptorType() + == DataDescriptorType. NO_DATA_DESCRIPTOR, "Cannot write entries with a data " + + "descriptor."); + Preconditions.checkNotNull(raf, "raf == null"); + Preconditions.checkState(state == ZipFileState.OPEN_RW, "state != ZipFileState.OPEN_RW"); + + /* + * Place the cursor and write the local header. + */ + byte[] headerData = entry.toHeaderData(); + directWrite(offset, headerData); + + /* + * Get the raw source data to write. + */ + ProcessedAndRawByteSources source = entry.getSource(); + ByteSource rawContents = source.getRawByteSource(); + + /* + * Write the source data. + */ + byte[] chunk = new byte[IO_BUFFER_SIZE]; + int r; + long writeOffset = offset + headerData.length; + InputStream is = rawContents.openStream(); + while ((r = is.read(chunk)) >= 0) { + directWrite(writeOffset, chunk, 0, r); + writeOffset += r; + } + + is.close(); + + /* + * Set the entry's offset and create the entry source. + */ + entry.replaceSourceFromZip(offset); + } + + /** + * Computes the central directory. The central directory must not have been computed yet. When + * this method finishes, the central directory has been computed {@link #directoryEntry}, + * unless the directory is empty in which case {@link #directoryEntry} + * is left as {@code null}. Nothing is written to disk as a result of this method's invocation. + * + * @throws IOException failed to append the central directory + */ + private void computeCentralDirectory() throws IOException { + Preconditions.checkState(state == ZipFileState.OPEN_RW, "state != ZipFileState.OPEN_RW"); + Preconditions.checkNotNull(raf, "raf == null"); + Preconditions.checkState(directoryEntry == null, "directoryEntry == null"); + + Set<StoredEntry> newStored = Sets.newHashSet(); + for (FileUseMapEntry<StoredEntry> mapEntry : entries.values()) { + newStored.add(mapEntry.getStore()); + } + + /* + * Make sure we truncate the map before computing the central directory's location since + * the central directory is the last part of the file. + */ + map.truncate(); + + CentralDirectory newDirectory = CentralDirectory.makeFromEntries(newStored, this); + byte[] newDirectoryBytes = newDirectory.toBytes(); + long directoryOffset = map.size() + extraDirectoryOffset; + + map.extend(directoryOffset + newDirectoryBytes.length); + + if (newDirectoryBytes.length > 0) { + directoryEntry = map.add(directoryOffset, directoryOffset + newDirectoryBytes.length, + newDirectory); + } + } + + /** + * Writes the central directory to the end of the zip file. {@link #directoryEntry} may be + * {@code null} only if there are no files in the archive. + * + * @throws IOException failed to append the central directory + */ + private void appendCentralDirectory() throws IOException { + Preconditions.checkState(state == ZipFileState.OPEN_RW, "state != ZipFileState.OPEN_RW"); + Preconditions.checkNotNull(raf, "raf == null"); + + if (entries.isEmpty()) { + Preconditions.checkState(directoryEntry == null, "directoryEntry != null"); + return; + } + + Preconditions.checkNotNull(directoryEntry, "directoryEntry != null"); + + CentralDirectory newDirectory = directoryEntry.getStore(); + Preconditions.checkNotNull(newDirectory, "newDirectory != null"); + + byte[] newDirectoryBytes = newDirectory.toBytes(); + long directoryOffset = directoryEntry.getStart(); + + /* + * It is fine to seek beyond the end of file. Seeking beyond the end of file will not extend + * the file. Even if we do not have any directory data to write, the extend() call below + * will force the file to be extended leaving exactly extraDirectoryOffset bytes empty at + * the beginning. + */ + directWrite(directoryOffset, newDirectoryBytes); + } + + /** + * Obtains the byte array representation of the central directory. The central directory must + * have been already computed. If there are no entries in the zip, the central directory will be + * empty. + * + * @return the byte representation, or an empty array if there are no entries in the zip + * @throws IOException failed to compute the central directory byte representation + */ + @Nonnull + public byte[] getCentralDirectoryBytes() throws IOException { + if (entries.isEmpty()) { + Preconditions.checkState(directoryEntry == null, "directoryEntry != null"); + return new byte[0]; + } + + Preconditions.checkNotNull(directoryEntry, "directoryEntry == null"); + + CentralDirectory cd = directoryEntry.getStore(); + Preconditions.checkNotNull(cd, "cd == null"); + return cd.toBytes(); + } + + /** + * Computes the EOCD. This creates a new {@link #eocdEntry}. The + * central directory must already be written. If {@link #directoryEntry} is {@code null}, then + * the zip file must not have any entries. + * + * @throws IOException failed to write the EOCD + */ + private void computeEocd() throws IOException { + Preconditions.checkState(state == ZipFileState.OPEN_RW, "state != ZipFileState.OPEN_RW"); + Preconditions.checkNotNull(raf, "raf == null"); + if (directoryEntry == null) { + Preconditions.checkState(entries.isEmpty(), + "directoryEntry == null && !entries.isEmpty()"); + } + + long dirStart; + long dirSize = 0; + + if (directoryEntry != null) { + CentralDirectory directory = directoryEntry.getStore(); + assert directory != null; + + dirStart = directoryEntry.getStart(); + dirSize = directoryEntry.getSize(); + Verify.verify(directory.getEntries().size() == entries.size()); + } else { + /* + * If we do not have a directory, then we must leave any requested offset empty. + */ + dirStart = extraDirectoryOffset; + } + + Verify.verify(eocdComment != null); + Eocd eocd = new Eocd(entries.size(), dirStart, dirSize, eocdComment); + eocdComment = null; + + byte[] eocdBytes = eocd.toBytes(); + long eocdOffset = map.size(); + + map.extend(eocdOffset + eocdBytes.length); + + eocdEntry = map.add(eocdOffset, eocdOffset + eocdBytes.length, eocd); + } + + /** + * Writes the EOCD to the end of the zip file. This creates a new {@link #eocdEntry}. The + * central directory must already be written. If {@link #directoryEntry} is {@code null}, then + * the zip file must not have any entries. + * + * @throws IOException failed to write the EOCD + */ + private void appendEocd() throws IOException { + Preconditions.checkState(state == ZipFileState.OPEN_RW, "state != ZipFileState.OPEN_RW"); + Preconditions.checkNotNull(raf, "raf == null"); + Preconditions.checkNotNull(eocdEntry, "eocdEntry == null"); + + Eocd eocd = eocdEntry.getStore(); + Preconditions.checkNotNull(eocd, "eocd == null"); + + byte[] eocdBytes = eocd.toBytes(); + long eocdOffset = eocdEntry.getStart(); + + directWrite(eocdOffset, eocdBytes); + } + + /** + * Obtains the byte array representation of the EOCD. The EOCD must have already been computed + * for this method to be invoked. + * + * @return the byte representation of the EOCD + * @throws IOException failed to obtain the byte representation of the EOCD + */ + @Nonnull + public byte[] getEocdBytes() throws IOException { + Preconditions.checkNotNull(eocdEntry, "eocdEntry == null"); + + Eocd eocd = eocdEntry.getStore(); + Preconditions.checkNotNull(eocd, "eocd == null"); + return eocd.toBytes(); + } + + /** + * Closes the file, if it is open. + * + * @throws IOException failed to close the file + */ + private void innerClose() throws IOException { + if (state == ZipFileState.CLOSED) { + return; + } + + Verify.verifyNotNull(raf, "raf == null"); + + raf.close(); + raf = null; + state = ZipFileState.CLOSED; + if (closedControl == null) { + closedControl = new CachedFileContents<>(file); + } + + closedControl.closed(null); + } + + /** + * If the zip file is closed, opens it in read-only mode. If it is already open, does nothing. + * In general, it is not necessary to directly invoke this method. However, if directly + * reading the zip file using, for example {@link #directRead(long, byte[])}, then this + * method needs to be called. + * @throws IOException failed to open the file + */ + public void openReadOnly() throws IOException { + if (state != ZipFileState.CLOSED) { + return; + } + + state = ZipFileState.OPEN_RO; + raf = new RandomAccessFile(file, "r"); + } + + /** + * Opens (or reopens) the zip file as read-write. This method will ensure that + * {@link #raf} is not null and open for writing. + * + * @throws IOException failed to open the file, failed to close it or the file was closed and + * has been modified outside the control of this object + */ + private void reopenRw() throws IOException { + // We an never open a file RW in read-only mode. We should never get this far, though. + Verify.verify(!readOnly); + + if (state == ZipFileState.OPEN_RW) { + return; + } + + boolean wasClosed; + if (state == ZipFileState.OPEN_RO) { + /* + * ReadAccessFile does not have a way to reopen as RW so we have to close it and + * open it again. + */ + innerClose(); + wasClosed = false; + } else { + wasClosed = true; + } + + Verify.verify(state == ZipFileState.CLOSED, "state != ZpiFileState.CLOSED"); + Verify.verify(raf == null, "raf != null"); + + if (closedControl != null && !closedControl.isValid()) { + throw new IOException("File '" + file.getAbsolutePath() + "' has been modified " + + "by an external application."); + } + + raf = new RandomAccessFile(file, "rw"); + state = ZipFileState.OPEN_RW; + + /* + * Now that we've open the zip and are ready to write, clear out any data descriptors + * in the zip since we don't need them and they take space in the archive. + */ + for (StoredEntry entry : entries()) { + dirty |= entry.removeDataDescriptor(); + } + + if (wasClosed) { + notify(ZFileExtension::open); + } + } + + /** + * Equivalent to call {@link #add(String, InputStream, boolean)} using + * {@code true} as {@code mayCompress}. + * + * @param name the file name (<em>i.e.</em>, path); paths should be defined using slashes + * and the name should not end in slash + * @param stream the source for the file's data + * @throws IOException failed to read the source data + * @throws IllegalStateException if the file is in read-only mode + */ + public void add(@Nonnull String name, @Nonnull InputStream stream) throws IOException { + checkNotInReadOnlyMode(); + add(name, stream, true); + } + + /** + * Creates a stored entry. This does not add the entry to the zip file, it just creates the + * {@link StoredEntry} object. + * + * @param name the name of the entry + * @param stream the input stream with the entry's data + * @param mayCompress can the entry be compressed? + * @return the created entry + * @throws IOException failed to create the entry + */ + @Nonnull + private StoredEntry makeStoredEntry( + @Nonnull String name, + @Nonnull InputStream stream, + boolean mayCompress) + throws IOException { + CloseableByteSource source = tracker.fromStream(stream); + long crc32 = source.hash(Hashing.crc32()).padToLong(); + + boolean encodeWithUtf8 = !EncodeUtils.canAsciiEncode(name); + + SettableFuture<CentralDirectoryHeaderCompressInfo> compressInfo = + SettableFuture.create(); + GPFlags flags = GPFlags.make(encodeWithUtf8); + CentralDirectoryHeader newFileData = + new CentralDirectoryHeader( + name, + EncodeUtils.encode(name, flags), + source.size(), + compressInfo, + flags, + this); + newFileData.setCrc32(crc32); + + /* + * Create the new entry and sets its data source. Offset should be set to -1 automatically + * because this is a new file. With offset set to -1, StoredEntry does not try to verify the + * local header. Since this is a new file, there is no local header and not checking it is + * what we want to happen. + */ + Verify.verify(newFileData.getOffset() == -1); + return new StoredEntry( + newFileData, + this, + createSources(mayCompress, source, compressInfo, newFileData)); + } + + /** + * Creates the processed and raw sources for an entry. + * + * @param mayCompress can the entry be compressed? + * @param source the entry's data (uncompressed) + * @param compressInfo the compression info future that will be set when the raw entry is + * created and the {@link CentralDirectoryHeaderCompressInfo} object can be created + * @param newFileData the central directory header for the new file + * @return the sources whose data may or may not be already defined + * @throws IOException failed to create the raw sources + */ + @Nonnull + private ProcessedAndRawByteSources createSources( + boolean mayCompress, + @Nonnull CloseableByteSource source, + @Nonnull SettableFuture<CentralDirectoryHeaderCompressInfo> compressInfo, + @Nonnull CentralDirectoryHeader newFileData) + throws IOException { + if (mayCompress) { + ListenableFuture<CompressionResult> result = compressor.compress(source); + Futures.addCallback( + result, + new FutureCallback<CompressionResult>() { + @Override + public void onSuccess(CompressionResult result) { + compressInfo.set( + new CentralDirectoryHeaderCompressInfo( + newFileData, + result.getCompressionMethod(), + result.getSize())); + } + + @Override + public void onFailure(@Nonnull Throwable t) { + compressInfo.setException(t); + } + }, + MoreExecutors.directExecutor()); + + ListenableFuture<CloseableByteSource> compressedByteSourceFuture = + Futures.transform( + result, CompressionResult::getSource, MoreExecutors.directExecutor()); + LazyDelegateByteSource compressedByteSource = new LazyDelegateByteSource( + compressedByteSourceFuture); + return new ProcessedAndRawByteSources(source, compressedByteSource); + } else { + compressInfo.set(new CentralDirectoryHeaderCompressInfo(newFileData, + CompressionMethod.STORE, source.size())); + return new ProcessedAndRawByteSources(source, source); + } + } + + /** + * Adds a file to the archive. + * + * <p>Adding the file will not update the archive immediately. Updating will only happen + * when the {@link #update()} method is invoked. + * + * <p>Adding a file with the same name as an existing file will replace that file in the + * archive. + * + * @param name the file name (<em>i.e.</em>, path); paths should be defined using slashes + * and the name should not end in slash + * @param stream the source for the file's data + * @param mayCompress can the file be compressed? This flag will be ignored if the alignment + * rules force the file to be aligned, in which case the file will not be compressed. + * @throws IOException failed to read the source data + * @throws IllegalStateException if the file is in read-only mode + */ + public void add(@Nonnull String name, @Nonnull InputStream stream, boolean mayCompress) + throws IOException { + checkNotInReadOnlyMode(); + + /* + * Clean pending background work, if needed. + */ + processAllReadyEntries(); + + add(makeStoredEntry(name, stream, mayCompress)); + } + + /** + * Adds a {@link StoredEntry} to the zip. The entry is not immediately added to + * {@link #entries} because data may not yet be available. Instead, it is placed under + * {@link #uncompressedEntries} and later moved to {@link #processAllReadyEntries()} when + * done. + * + * <p>This method invokes {@link #processAllReadyEntries()} to move the entry if it has already + * been computed so, if there is no delay in compression, and no more files are in waiting + * queue, then the entry is added to {@link #entries} immediately. + * + * @param newEntry the entry to add + * @throws IOException failed to process this entry (or a previous one whose future only + * completed now) + */ + private void add(@Nonnull final StoredEntry newEntry) throws IOException { + uncompressedEntries.add(newEntry); + processAllReadyEntries(); + } + + /** + * Moves all ready entries from {@link #uncompressedEntries} to {@link #entries}. It will + * stop as soon as entry whose future has not been completed is found. + * + * @throws IOException the exception reported in the future computation, if any, or failed + * to add a file to the archive + */ + private void processAllReadyEntries() throws IOException { + /* + * Many things can happen during addToEntries(). Because addToEntries() fires + * notifications to extensions, other files can be added, removed, etc. Ee are *not* + * guaranteed that new stuff does not get into uncompressedEntries: add() will still work + * and will add new entries in there. + * + * However -- important -- processReadyEntries() may be invoked during addToEntries() + * because of the extension mechanism. This means that stuff *can* be removed from + * uncompressedEntries and moved to entries during addToEntries(). + */ + while (!uncompressedEntries.isEmpty()) { + StoredEntry next = uncompressedEntries.get(0); + CentralDirectoryHeader cdh = next.getCentralDirectoryHeader(); + Future<CentralDirectoryHeaderCompressInfo> compressionInfo = cdh.getCompressionInfo(); + if (!compressionInfo.isDone()) { + /* + * First entry in queue is not yet complete. We can't do anything else. + */ + return; + } + + uncompressedEntries.remove(0); + + try { + compressionInfo.get(); + } catch (InterruptedException e) { + throw new IOException("Impossible I/O exception: get for already computed " + + "future throws InterruptedException", e); + } catch (ExecutionException e) { + throw new IOException("Failed to obtain compression information for entry", e); + } + + addToEntries(next); + } + } + + /** + * Waits until {@link #uncompressedEntries} is empty. + * + * @throws IOException the exception reported in the future computation, if any, or failed + * to add a file to the archive + */ + private void processAllReadyEntriesWithWait() throws IOException { + processAllReadyEntries(); + while (!uncompressedEntries.isEmpty()) { + /* + * Wait for the first future to complete and then try again. Keep looping until we're + * done. + */ + StoredEntry first = uncompressedEntries.get(0); + CentralDirectoryHeader cdh = first.getCentralDirectoryHeader(); + cdh.getCompressionInfoWithWait(); + + processAllReadyEntries(); + } + } + + /** + * Adds a new file to {@link #entries}. This is actually added to the zip and its space + * allocated in the {@link #map}. + * + * @param newEntry the new entry to add + * @throws IOException failed to add the file + */ + private void addToEntries(@Nonnull final StoredEntry newEntry) throws IOException { + Preconditions.checkArgument(newEntry.getDataDescriptorType() == + DataDescriptorType.NO_DATA_DESCRIPTOR, "newEntry has data descriptor"); + + /* + * If there is a file with the same name in the archive, remove it. We remove it by + * calling delete() on the entry (this is the public API to remove a file from the archive). + * StoredEntry.delete() will call {@link ZFile#delete(StoredEntry, boolean)} to perform + * data structure cleanup. + */ + FileUseMapEntry<StoredEntry> toReplace = entries.get( + newEntry.getCentralDirectoryHeader().getName()); + final StoredEntry replaceStore; + if (toReplace != null) { + replaceStore = toReplace.getStore(); + assert replaceStore != null; + replaceStore.delete(false); + } else { + replaceStore = null; + } + + FileUseMapEntry<StoredEntry> fileUseMapEntry = + positionInFile(newEntry, PositionHint.ANYWHERE); + entries.put(newEntry.getCentralDirectoryHeader().getName(), fileUseMapEntry); + + dirty = true; + + notify(ext -> ext.added(newEntry, replaceStore)); + } + + /** + * Finds a location in the zip where this entry will be added to and create the map entry. + * This method cannot be called if there is already a map entry for the given entry (if you + * do that, then you're doing something wrong somewhere). + * + * <p>This may delete the central directory and EOCD (if it deletes one, it deletes the other) + * if there is no space before the central directory. Otherwise, the file would be added + * after the central directory. This would force a new central directory to be written + * when updating the file and would create a hole in the zip. Me no like holes. Holes are evil. + * + * @param entry the entry to place in the zip + * @param positionHint hint to where the file should be positioned + * @return the position in the file where the entry should be placed + */ + @Nonnull + private FileUseMapEntry<StoredEntry> positionInFile( + @Nonnull StoredEntry entry, + @Nonnull PositionHint positionHint) + throws IOException { + deleteDirectoryAndEocd(); + long size = entry.getInFileSize(); + int localHeaderSize = entry.getLocalHeaderSize(); + int alignment = chooseAlignment(entry); + + FileUseMap.PositionAlgorithm algorithm; + + switch (positionHint) { + case LOWEST_OFFSET: + algorithm = FileUseMap.PositionAlgorithm.FIRST_FIT; + break; + case ANYWHERE: + algorithm = FileUseMap.PositionAlgorithm.BEST_FIT; + break; + default: + throw new AssertionError(); + } + + long newOffset = map.locateFree(size, localHeaderSize, alignment, algorithm); + long newEnd = newOffset + entry.getInFileSize(); + if (newEnd > map.size()) { + map.extend(newEnd); + } + + return map.add(newOffset, newEnd, entry); + } + + /** + * Determines what is the alignment value of an entry. + * + * @param entry the entry + * @return the alignment value, {@link AlignmentRule#NO_ALIGNMENT} if there is no alignment + * required for the entry + * @throws IOException failed to determine the alignment + */ + private int chooseAlignment(@Nonnull StoredEntry entry) throws IOException { + CentralDirectoryHeader cdh = entry.getCentralDirectoryHeader(); + CentralDirectoryHeaderCompressInfo compressionInfo = cdh.getCompressionInfoWithWait(); + + boolean isCompressed = compressionInfo.getMethod() != CompressionMethod.STORE; + if (isCompressed) { + return AlignmentRule.NO_ALIGNMENT; + } else { + return alignmentRule.alignment(cdh.getName()); + } + } + + /** + * Adds all files from another zip file, maintaining their compression. Files specified in + * <em>src</em> that are already on this file will replace the ones in this file. However, if + * their sizes and checksums are equal, they will be ignored. + * + * <p> This method will not perform any changes in itself, it will only update in-memory data + * structures. To actually write the zip file, invoke either {@link #update()} or + * {@link #close()}. + * + * @param src the source archive + * @param ignoreFilter predicate that, if {@code true}, identifies files in <em>src</em> that + * should be ignored by merging; merging will behave as if these files were not there + * @throws IOException failed to read from <em>src</em> or write on the output + * @throws IllegalStateException if the file is in read-only mode + */ + public void mergeFrom(@Nonnull ZFile src, @Nonnull Predicate<String> ignoreFilter) + throws IOException { + checkNotInReadOnlyMode(); + + for (StoredEntry fromEntry : src.entries()) { + if (ignoreFilter.test(fromEntry.getCentralDirectoryHeader().getName())) { + continue; + } + + boolean replaceCurrent = true; + String path = fromEntry.getCentralDirectoryHeader().getName(); + FileUseMapEntry<StoredEntry> currentEntry = entries.get(path); + + if (currentEntry != null) { + long fromSize = fromEntry.getCentralDirectoryHeader().getUncompressedSize(); + long fromCrc = fromEntry.getCentralDirectoryHeader().getCrc32(); + + StoredEntry currentStore = currentEntry.getStore(); + assert currentStore != null; + + long currentSize = currentStore.getCentralDirectoryHeader().getUncompressedSize(); + long currentCrc = currentStore.getCentralDirectoryHeader().getCrc32(); + + if (fromSize == currentSize && fromCrc == currentCrc) { + replaceCurrent = false; + } + } + + if (replaceCurrent) { + CentralDirectoryHeader fromCdr = fromEntry.getCentralDirectoryHeader(); + CentralDirectoryHeaderCompressInfo fromCompressInfo = + fromCdr.getCompressionInfoWithWait(); + CentralDirectoryHeader newFileData; + try { + /* + * We make two changes in the central directory from the file to merge: + * we reset the offset to force the entry to be written and we reset the + * deferred CRC bit as we don't need the extra stuff after the file. It takes + * space and is totally useless. + */ + newFileData = fromCdr.clone(); + newFileData.setOffset(-1); + newFileData.resetDeferredCrc(); + } catch (CloneNotSupportedException e) { + throw new IOException("Failed to clone CDR.", e); + } + + /* + * Read the data (read directly the compressed source if there is one). + */ + ProcessedAndRawByteSources fromSource = fromEntry.getSource(); + InputStream fromInput = fromSource.getRawByteSource().openStream(); + long sourceSize = fromSource.getRawByteSource().size(); + if (sourceSize > Integer.MAX_VALUE) { + throw new IOException("Cannot read source with " + sourceSize + " bytes."); + } + + byte[] data = new byte[Ints.checkedCast(sourceSize)]; + int read = 0; + while (read < data.length) { + int r = fromInput.read(data, read, data.length - read); + Verify.verify(r >= 0, "There should be at least 'size' bytes in the stream."); + read += r; + } + + /* + * Build the new source and wrap it around an inflater source if data came from + * a compressed source. + */ + CloseableByteSource rawContents = tracker.fromSource(fromSource.getRawByteSource()); + CloseableByteSource processedContents; + if (fromCompressInfo.getMethod() == CompressionMethod.DEFLATE) { + //noinspection IOResourceOpenedButNotSafelyClosed + processedContents = new InflaterByteSource(rawContents); + } else { + processedContents = rawContents; + } + + ProcessedAndRawByteSources newSource = new ProcessedAndRawByteSources( + processedContents, rawContents); + + /* + * Add will replace any current entry with the same name. + */ + StoredEntry newEntry = new StoredEntry(newFileData, this, newSource); + add(newEntry); + } + } + } + + /** + * Forcibly marks this zip file as touched, forcing it to be updated when {@link #update()} + * or {@link #close()} are invoked. + * + * @throws IllegalStateException if the file is in read-only mode + */ + public void touch() { + checkNotInReadOnlyMode(); + dirty = true; + } + + /** + * Wait for any background tasks to finish and report any errors. In general this method does + * not need to be invoked directly as errors from background tasks are reported during + * {@link #add(String, InputStream, boolean)}, {@link #update()} and {@link #close()}. + * However, if required for some purposes, <em>e.g.</em>, ensuring all notifications have been + * done to extensions, then this method may be called. It will wait for all background tasks + * to complete. + * @throws IOException some background work failed + */ + public void finishAllBackgroundTasks() throws IOException { + processAllReadyEntriesWithWait(); + } + + /** + * Realigns all entries in the zip. This is equivalent to call {@link StoredEntry#realign()} + * for all entries in the zip file. + * + * @return has any entry been changed? Note that for entries that have not yet been written on + * the file, realignment does not count as a change as nothing needs to be updated in the file; + * entries that have been updated may have been recreated and the existing references outside + * of {@code ZFile} may refer to {@link StoredEntry}s that are no longer valid + * @throws IOException failed to realign the zip; some entries in the zip may have been lost + * due to the I/O error + * @throws IllegalStateException if the file is in read-only mode + */ + public boolean realign() throws IOException { + checkNotInReadOnlyMode(); + + boolean anyChanges = false; + for (StoredEntry entry : entries()) { + anyChanges |= entry.realign(); + } + + if (anyChanges) { + dirty = true; + } + + return anyChanges; + } + + /** + * Realigns a stored entry, if necessary. Realignment is done by removing and re-adding the file + * if it was not aligned. + * + * @param entry the entry to realign + * @return has the entry been changed? Note that if the entry has not yet been written on the + * file, realignment does not count as a change as nothing needs to be updated in the file + * @throws IOException failed to read/write an entry; the entry may no longer exist in the + * file + */ + boolean realign(@Nonnull StoredEntry entry) throws IOException { + FileUseMapEntry<StoredEntry> mapEntry = + entries.get(entry.getCentralDirectoryHeader().getName()); + Verify.verify(entry == mapEntry.getStore()); + long currentDataOffset = mapEntry.getStart() + entry.getLocalHeaderSize(); + + int expectedAlignment = chooseAlignment(entry); + long misalignment = currentDataOffset % expectedAlignment; + if (misalignment == 0) { + /* + * Good. File is aligned properly. + */ + return false; + } + + if (entry.getCentralDirectoryHeader().getOffset() == -1) { + /* + * File is not aligned but it is not written. We do not really need to do much other + * than find another place in the map. + */ + map.remove(mapEntry); + long newStart = + map.locateFree( + mapEntry.getSize(), + entry.getLocalHeaderSize(), + expectedAlignment, + FileUseMap.PositionAlgorithm.BEST_FIT); + mapEntry = map.add(newStart, newStart + entry.getInFileSize(), entry); + entries.put(entry.getCentralDirectoryHeader().getName(), mapEntry); + + /* + * Just for safety. We're modifying the in-memory structures but the file should + * already be marked as dirty. + */ + Verify.verify(dirty); + + return false; + + } + + /* + * Get the entry data source, but check if we have a compressed one (we don't want to + * inflate and deflate). + */ + CentralDirectoryHeaderCompressInfo compressInfo = + entry.getCentralDirectoryHeader().getCompressionInfoWithWait(); + + ProcessedAndRawByteSources source = entry.getSource(); + + CentralDirectoryHeader clonedCdh; + try { + clonedCdh = entry.getCentralDirectoryHeader().clone(); + } catch (CloneNotSupportedException e) { + Verify.verify(false); + return false; + } + + /* + * We make two changes in the central directory when realigning: + * we reset the offset to force the entry to be written and we reset the + * deferred CRC bit as we don't need the extra stuff after the file. It takes + * space and is totally useless and we may need the extra space to realign the entry... + */ + clonedCdh.setOffset(-1); + clonedCdh.resetDeferredCrc(); + + CloseableByteSource rawContents = tracker.fromSource(source.getRawByteSource()); + CloseableByteSource processedContents; + + if (compressInfo.getMethod() == CompressionMethod.DEFLATE) { + //noinspection IOResourceOpenedButNotSafelyClosed + processedContents = new InflaterByteSource(rawContents); + } else { + processedContents = rawContents; + } + + ProcessedAndRawByteSources newSource = new ProcessedAndRawByteSources(processedContents, + rawContents); + + /* + * Add the new file. This will replace the existing one. + */ + StoredEntry newEntry = new StoredEntry(clonedCdh, this, newSource); + add(newEntry); + return true; + } + + /** + * Adds an extension to this zip file. + * + * @param extension the listener to add + * @throws IllegalStateException if the file is in read-only mode + */ + public void addZFileExtension(@Nonnull ZFileExtension extension) { + checkNotInReadOnlyMode(); + extensions.add(extension); + } + + /** + * Removes an extension from this zip file. + * + * @param extension the listener to remove + * @throws IllegalStateException if the file is in read-only mode + */ + public void removeZFileExtension(@Nonnull ZFileExtension extension) { + checkNotInReadOnlyMode(); + extensions.remove(extension); + } + + /** + * Notifies all extensions, collecting their execution requests and running them. + * + * @param function the function to apply to all listeners, it will generally invoke the + * notification method on the listener and return the result of that invocation + * @throws IOException failed to process some extensions + */ + private void notify(@Nonnull IOExceptionFunction<ZFileExtension, IOExceptionRunnable> function) + throws IOException { + for (ZFileExtension fl : Lists.newArrayList(extensions)) { + IOExceptionRunnable r = function.apply(fl); + if (r != null) { + toRun.add(r); + } + } + + if (!isNotifying) { + isNotifying = true; + + try { + while (!toRun.isEmpty()) { + IOExceptionRunnable r = toRun.remove(0); + r.run(); + } + } finally { + isNotifying = false; + } + } + } + + /** + * Directly writes data in the zip file. <strong>Incorrect use of this method may corrupt the + * zip file</strong>. Invoking this method may force the zip to be reopened in read/write + * mode. + * + * @param offset the offset at which data should be written + * @param data the data to write, may be an empty array + * @param start start offset in {@code data} where data to write is located + * @param count number of bytes of data to write + * @throws IOException failed to write the data + * @throws IllegalStateException if the file is in read-only mode + */ + public void directWrite(long offset, @Nonnull byte[] data, int start, int count) + throws IOException { + checkNotInReadOnlyMode(); + + Preconditions.checkArgument(offset >= 0, "offset < 0"); + Preconditions.checkArgument(start >= 0, "start >= 0"); + Preconditions.checkArgument(count >= 0, "count >= 0"); + + if (data.length == 0) { + return; + } + + Preconditions.checkArgument(start <= data.length, "start > data.length"); + Preconditions.checkArgument(start + count <= data.length, "start + count > data.length"); + + reopenRw(); + assert raf != null; + + raf.seek(offset); + raf.write(data, start, count); + } + + /** + * Same as {@code directWrite(offset, data, 0, data.length)}. + * + * @param offset the offset at which data should be written + * @param data the data to write, may be an empty array + * @throws IOException failed to write the data + * @throws IllegalStateException if the file is in read-only mode + */ + public void directWrite(long offset, @Nonnull byte[] data) throws IOException { + checkNotInReadOnlyMode(); + directWrite(offset, data, 0, data.length); + } + + /** + * Returns the current size (in bytes) of the underlying file. + * + * @throws IOException if an I/O error occurs + */ + public long directSize() throws IOException { + /* + * Only force a reopen if the file is closed. + */ + if (raf == null) { + reopenRw(); + assert raf != null; + } + return raf.length(); + } + + /** + * Directly reads data from the zip file. Invoking this method may force the zip to be reopened + * in read/write mode. + * + * @param offset the offset at which data should be written + * @param data the array where read data should be stored + * @param start start position in the array where to write data to + * @param count how many bytes of data can be written + * @return how many bytes of data have been written or {@code -1} if there are no more bytes + * to be read + * @throws IOException failed to write the data + */ + public int directRead(long offset, @Nonnull byte[] data, int start, int count) + throws IOException { + Preconditions.checkArgument(start >= 0, "start >= 0"); + Preconditions.checkArgument(count >= 0, "count >= 0"); + Preconditions.checkArgument(start <= data.length, "start > data.length"); + Preconditions.checkArgument(start + count <= data.length, "start + count > data.length"); + return directRead(offset, ByteBuffer.wrap(data, start, count)); + } + + /** + * Directly reads data from the zip file. Invoking this method may force the zip to be reopened + * in read/write mode. + * + * @param offset the offset from which data should be read + * @param dest the output buffer to fill with data from the {@code offset}. + * @return how many bytes of data have been written or {@code -1} if there are no more bytes + * to be read + * @throws IOException failed to write the data + */ + public int directRead(long offset, @Nonnull ByteBuffer dest) throws IOException { + Preconditions.checkArgument(offset >= 0, "offset < 0"); + + if (!dest.hasRemaining()) { + return 0; + } + + /* + * Only force a reopen if the file is closed. + */ + if (raf == null) { + reopenRw(); + assert raf != null; + } + + raf.seek(offset); + return raf.getChannel().read(dest); + } + + /** + * Same as {@code directRead(offset, data, 0, data.length)}. + * + * @param offset the offset at which data should be read + * @param data receives the read data, may be an empty array + * @throws IOException failed to read the data + */ + public int directRead(long offset, @Nonnull byte[] data) throws IOException { + return directRead(offset, data, 0, data.length); + } + + /** + * Reads exactly {@code data.length} bytes of data, failing if it was not possible to read all + * the requested data. + * + * @param offset the offset at which to start reading + * @param data the array that receives the data read + * @throws IOException failed to read some data or there is not enough data to read + */ + public void directFullyRead(long offset, @Nonnull byte[] data) throws IOException { + directFullyRead(offset, ByteBuffer.wrap(data)); + } + + /** + * Reads exactly {@code dest.remaining()} bytes of data, failing if it was not possible to read + * all the requested data. + * + * @param offset the offset at which to start reading + * @param dest the output buffer to fill with data + * @throws IOException failed to read some data or there is not enough data to read + */ + public void directFullyRead(long offset, @Nonnull ByteBuffer dest) throws IOException { + Preconditions.checkArgument(offset >= 0, "offset < 0"); + + if (!dest.hasRemaining()) { + return; + } + + /* + * Only force a reopen if the file is closed. + */ + if (raf == null) { + reopenRw(); + assert raf != null; + } + + FileChannel fileChannel = raf.getChannel(); + while (dest.hasRemaining()) { + fileChannel.position(offset); + int chunkSize = fileChannel.read(dest); + if (chunkSize == -1) { + throw new EOFException( + "Failed to read " + dest.remaining() + " more bytes: premature EOF"); + } + offset += chunkSize; + } + } + + /** + * Adds all files and directories recursively. + * <p> + * Equivalent to calling {@link #addAllRecursively(File, Function)} using a function that + * always returns {@code true} + * + * @param file a file or directory; if it is a directory, all files and directories will be + * added recursively + * @throws IOException failed to some (or all ) of the files + * @throws IllegalStateException if the file is in read-only mode + */ + public void addAllRecursively(@Nonnull File file) throws IOException { + checkNotInReadOnlyMode(); + addAllRecursively(file, f -> true); + } + + /** + * Adds all files and directories recursively. + * + * @param file a file or directory; if it is a directory, all files and directories will be + * added recursively + * @param mayCompress a function that decides whether files may be compressed + * @throws IOException failed to some (or all ) of the files + * @throws IllegalStateException if the file is in read-only mode + */ + public void addAllRecursively( + @Nonnull File file, + @Nonnull Function<? super File, Boolean> mayCompress) throws IOException { + checkNotInReadOnlyMode(); + + /* + * The case of file.isFile() is different because if file.isFile() we will add it to the + * zip in the root. However, if file.isDirectory() we won't add it and add its children. + */ + if (file.isFile()) { + boolean mayCompressFile = Verify.verifyNotNull(mayCompress.apply(file), + "mayCompress.apply() returned null"); + + try (Closer closer = Closer.create()) { + FileInputStream fileInput = closer.register(new FileInputStream(file)); + add(file.getName(), fileInput, mayCompressFile); + } + + return; + } + + for (File f : Files.fileTreeTraverser().preOrderTraversal(file).skip(1)) { + String path = file.toURI().relativize(f.toURI()).getPath(); + + InputStream stream; + try (Closer closer = Closer.create()) { + boolean mayCompressFile; + if (f.isDirectory()) { + stream = closer.register(new ByteArrayInputStream(new byte[0])); + mayCompressFile = false; + } else { + stream = closer.register(new FileInputStream(f)); + mayCompressFile = Verify.verifyNotNull(mayCompress.apply(f), + "mayCompress.apply() returned null"); + } + + add(path, stream, mayCompressFile); + } + } + } + + /** + * Obtains the offset at which the central directory exists, or at which it will be written + * if the zip file were to be flushed immediately. + * + * @return the offset, in bytes, where the central directory is or will be written; this value + * includes any extra offset for the central directory + */ + public long getCentralDirectoryOffset() { + if (directoryEntry != null) { + return directoryEntry.getStart(); + } + + /* + * If there are no entries, the central directory is written at the start of the file. + */ + if (entries.isEmpty()) { + return extraDirectoryOffset; + } + + /* + * The Central Directory is written after all entries. This will be at the end of the file + * if the + */ + return map.usedSize() + extraDirectoryOffset; + } + + /** + * Obtains the size of the central directory, if the central directory is written in the zip + * file. + * + * @return the size of the central directory or {@code -1} if the central directory has not + * been computed + */ + public long getCentralDirectorySize() { + if (directoryEntry != null) { + return directoryEntry.getSize(); + } + + if (entries.isEmpty()) { + return 0; + } + + return 1; + } + + /** + * Obtains the offset of the EOCD record, if the EOCD has been written to the file. + * + * @return the offset of the EOCD or {@code -1} if none exists yet + */ + public long getEocdOffset() { + if (eocdEntry == null) { + return -1; + } + + return eocdEntry.getStart(); + } + + /** + * Obtains the size of the EOCD record, if the EOCD has been written to the file. + * + * @return the size of the EOCD of {@code -1} it none exists yet + */ + public long getEocdSize() { + if (eocdEntry == null) { + return -1; + } + + return eocdEntry.getSize(); + } + + /** + * Obtains the comment in the EOCD. + * + * @return the comment exactly as it was encoded in the EOCD, no encoding conversion is done + */ + @Nonnull + public byte[] getEocdComment() { + if (eocdEntry == null) { + Verify.verify(eocdComment != null); + byte[] eocdCommentCopy = new byte[eocdComment.length]; + System.arraycopy(eocdComment, 0, eocdCommentCopy, 0, eocdComment.length); + return eocdCommentCopy; + } + + Eocd eocd = eocdEntry.getStore(); + Verify.verify(eocd != null); + return eocd.getComment(); + } + + /** + * Sets the comment in the EOCD. + * + * @param comment the new comment; no conversion is done, these exact bytes will be placed in + * the EOCD comment + * @throws IllegalStateException if file is in read-only mode + */ + public void setEocdComment(@Nonnull byte[] comment) { + checkNotInReadOnlyMode(); + + if (comment.length > MAX_EOCD_COMMENT_SIZE) { + throw new IllegalArgumentException( + "EOCD comment size (" + + comment.length + + ") is larger than the maximum allowed (" + + MAX_EOCD_COMMENT_SIZE + + ")"); + } + + // Check if the EOCD signature appears anywhere in the comment we need to check if it + // is valid. + for (int i = 0; i < comment.length - MIN_EOCD_SIZE; i++) { + // Remember: little endian... + if (comment[i] == EOCD_SIGNATURE[3] + && comment[i + 1] == EOCD_SIGNATURE[2] + && comment[i + 2] == EOCD_SIGNATURE[1] + && comment[i + 3] == EOCD_SIGNATURE[0]) { + // We found a possible EOCD signature at position i. Try to read it. + ByteBuffer bytes = ByteBuffer.wrap(comment, i, comment.length - i); + try { + new Eocd(bytes); + throw new IllegalArgumentException( + "Position " + + i + + " of the comment contains a valid EOCD record."); + } catch (IOException e) { + // Fine, this is an invalid record. Move along... + } + } + } + + deleteDirectoryAndEocd(); + eocdComment = new byte[comment.length]; + System.arraycopy(comment, 0, eocdComment, 0, comment.length); + dirty = true; + } + + /** + * Sets an extra offset for the central directory. See class description for details. Changing + * this value will mark the file as dirty and force a rewrite of the central directory when + * updated. + * + * @param offset the offset or {@code 0} to write the central directory at its current location + * @throws IllegalStateException if file is in read-only mode + */ + public void setExtraDirectoryOffset(long offset) { + checkNotInReadOnlyMode(); + Preconditions.checkArgument(offset >= 0, "offset < 0"); + + if (extraDirectoryOffset != offset) { + extraDirectoryOffset = offset; + deleteDirectoryAndEocd(); + dirty = true; + } + } + + /** + * Obtains the extra offset for the central directory. See class description for details. + * + * @return the offset or {@code 0} if no offset is set + */ + public long getExtraDirectoryOffset() { + return extraDirectoryOffset; + } + + /** + * Obtains whether this {@code ZFile} is ignoring timestamps. + * + * @return are the timestamps being ignored? + */ + public boolean areTimestampsIgnored() { + return noTimestamps; + } + + /** + * Sorts all files in the zip. This will force all files to be loaded and will wait for all + * background tasks to complete. Sorting files is never done implicitly and will operate in + * memory only (maybe reading files from the zip disk into memory, if needed). It will leave + * the zip in dirty state, requiring a call to {@link #update()} to force the entries to be + * written to disk. + * + * @throws IOException failed to load or move a file in the zip + * @throws IllegalStateException if file is in read-only mode + */ + public void sortZipContents() throws IOException { + checkNotInReadOnlyMode(); + reopenRw(); + + processAllReadyEntriesWithWait(); + + Verify.verify(uncompressedEntries.isEmpty()); + + SortedSet<StoredEntry> sortedEntries = Sets.newTreeSet(StoredEntry.COMPARE_BY_NAME); + for (FileUseMapEntry<StoredEntry> fmEntry : entries.values()) { + StoredEntry entry = fmEntry.getStore(); + Preconditions.checkNotNull(entry); + sortedEntries.add(entry); + entry.loadSourceIntoMemory(); + + map.remove(fmEntry); + } + + entries.clear(); + for (StoredEntry entry : sortedEntries) { + String name = entry.getCentralDirectoryHeader().getName(); + FileUseMapEntry<StoredEntry> positioned = + positionInFile(entry, PositionHint.LOWEST_OFFSET); + + entries.put(name, positioned); + } + + dirty = true; + } + + /** + * Obtains the filesystem path to the zip file. + * + * @return the file that may or may not exist (depending on whether something existed there + * before the zip was created and on whether the zip has been updated or not) + */ + @Nonnull + public File getFile() { + return file; + } + + /** + * Creates a new verify log. + * + * @return the new verify log + */ + @Nonnull + VerifyLog makeVerifyLog() { + VerifyLog log = verifyLogFactory.get(); + assert log != null; + return log; + } + + /** + * Obtains the zip file's verify log. + * + * @return the verify log + */ + @Nonnull + VerifyLog getVerifyLog() { + return verifyLog; + } + + /** + * Are there in-memory changes that have not been written to the zip file? + * + * <p>Waits for all pending processing which may make changes. + */ + public boolean hasPendingChangesWithWait() throws IOException { + processAllReadyEntriesWithWait(); + return dirty; + } + + /** Hint to where files should be positioned. */ + enum PositionHint { + /** + * File may be positioned anywhere, caller doesn't care. + */ + ANYWHERE, + + /** + * File should be positioned at the lowest offset possible. + */ + LOWEST_OFFSET + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/zip/ZFileExtension.java b/src/main/java/com/android/tools/build/apkzlib/zip/ZFileExtension.java new file mode 100644 index 0000000..2723a61 --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zip/ZFileExtension.java @@ -0,0 +1,146 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zip; + +import com.android.tools.build.apkzlib.utils.IOExceptionRunnable; +import java.io.IOException; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * An extension of a {@link ZFile}. Extensions are notified when files are open, updated, closed and + * when files are added or removed from the zip. These notifications are received after the zip + * has been updated in memory for open, when files are added or removed and when the zip has been + * updated on disk or closed. + * <p> + * An extension is also notified before the file is updated, allowing it to modify the file before + * the update happens. If it does, then all extensions are notified of the changes on the zip file. + * Because the order of the notifications is preserved, all extensions are notified in the same + * order. For example, if two extensions E1 and E2 are registered and they both add a file at + * update time, this would be the flow: + * <ul> + * <li>E1 receives {@code beforeUpdate} notification.</li> + * <li>E1 adds file F1 to the zip (notifying the addition is suspended because another + * notification is in progress).</li> + * <li>E2 receives {@code beforeUpdate} notification.</li> + * <li>E2 adds file F2 to the zip (notifying the addition is suspended because another + * notification is in progress).</li> + * <li>E1 is notified F1 was added.</li> + * <li>E2 is notified F1 was added.</li> + * <li>E1 is notified F2 was added.</li> + * <li>E2 is notified F2 was added.</li> + * <li>(zip file is updated on disk)</li> + * <li>E1 is notified the zip was updated.</li> + * <li>E2 is notified the zip was updated.</li> + * </ul> + * <p> + * An extension should not modify the zip file when notified of changes. If allowed, this would + * break event notification order in case multiple extensions are registered with the zip file. + * To allow performing changes to the zip file, all notification method return a + * {@code IOExceptionRunnable} that is invoked when {@link ZFile} has finished notifying all + * extensions. + */ +public abstract class ZFileExtension { + + /** + * The zip file has been open and the zip's contents have been read. The default implementation + * does nothing and returns {@code null}. + * + * @return an optional runnable to run when notification of all listeners has ended + * @throws IOException failed to process the event + */ + @Nullable + public IOExceptionRunnable open() throws IOException { + return null; + } + + /** + * The zip will be updated. This method allows the extension to register changes to the zip + * file before the file is written. The default implementation does nothing and returns + * {@code null}. + * <p> + * After this notification is received, the extension will receive further + * {@link #added(StoredEntry, StoredEntry)} and {@link #removed(StoredEntry)} notifications if + * it or other extensions add or remove files before update. + * <p> + * When no more files are updated, the {@link #entriesWritten()} notification is sent. + * + * @return an optional runnable to run when notification of all listeners has ended + * @throws IOException failed to process the event + */ + @Nullable + public IOExceptionRunnable beforeUpdate() throws IOException { + return null; + } + + /** + * This notification is sent when all entries have been written in the file but the central + * directory and the EOCD have not yet been written. No entries should be added, removed or + * updated during this notification. If this method forces an update of either the central + * directory or EOCD, then this method will be invoked again for all extensions with the new + * central directory and EOCD. + * <p> + * After this notification, {@link #updated()} is sent. + * + * @throws IOException failed to process the event + */ + public void entriesWritten() throws IOException { + } + + /** + * The zip file has been updated on disk. The default implementation does nothing. + * + * @throws IOException failed to perform update tasks + */ + public void updated() throws IOException { + } + + /** + * The zip file has been closed. Note that if {@link ZFile#close()} requires that the zip file + * be updated (because it had in-memory changes), {@link #updated()} will be called before + * this method. The default implementation does nothing. + */ + public void closed() { + } + + /** + * A new entry has been added to the zip, possibly replacing an entry in there. The + * default implementation does nothing and returns {@code null}. + * + * @param entry the entry that was added + * @param replaced the entry that was replaced, if any + * @return an optional runnable to run when notification of all listeners has ended + */ + @Nullable + public IOExceptionRunnable added(@Nonnull StoredEntry entry, @Nullable StoredEntry replaced) { + return null; + } + + /** + * An entry has been removed from the zip. This method is not invoked for entries that have + * been replaced. Those entries are notified using <em>replaced</em> in + * {@link #added(StoredEntry, StoredEntry)}. The default implementation does nothing and + * returns {@code null}. + * + * @param entry the entry that was deleted + * @return an optional runnable to run when notification of all listeners has ended + */ + @Nullable + public IOExceptionRunnable removed(@Nonnull StoredEntry entry) { + return null; + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/zip/ZFileOptions.java b/src/main/java/com/android/tools/build/apkzlib/zip/ZFileOptions.java new file mode 100644 index 0000000..e260455 --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zip/ZFileOptions.java @@ -0,0 +1,214 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zip; + +import com.android.tools.build.apkzlib.zip.compress.DeflateExecutionCompressor; +import com.android.tools.build.apkzlib.zip.utils.ByteTracker; +import java.util.function.Supplier; +import java.util.zip.Deflater; +import javax.annotation.Nonnull; + +/** + * Options to create a {@link ZFile}. + */ +public class ZFileOptions { + + /** + * The byte tracker. + */ + @Nonnull + private ByteTracker tracker; + + /** + * The compressor to use. + */ + @Nonnull + private Compressor compressor; + + /** + * Should timestamps be zeroed? + */ + private boolean noTimestamps; + + /** + * The alignment rule to use. + */ + @Nonnull + private AlignmentRule alignmentRule; + + /** + * Should the extra field be used to cover empty space? + */ + private boolean coverEmptySpaceUsingExtraField; + + /** + * Should files be automatically sorted before update? + */ + private boolean autoSortFiles; + + /** + * Factory creating verification logs to use. + */ + @Nonnull + private Supplier<VerifyLog> verifyLogFactory; + + /** + * Creates a new options object. All options are set to their defaults. + */ + public ZFileOptions() { + tracker = new ByteTracker(); + compressor = + new DeflateExecutionCompressor( + Runnable::run, + tracker, + Deflater.DEFAULT_COMPRESSION); + alignmentRule = AlignmentRules.compose(); + verifyLogFactory = VerifyLogs::devNull; + } + + /** + * Obtains the ZFile's byte tracker. + * + * @return the byte tracker + */ + @Nonnull + public ByteTracker getTracker() { + return tracker; + } + + /** + * Obtains the compressor to use. + * + * @return the compressor + */ + @Nonnull + public Compressor getCompressor() { + return compressor; + } + + /** + * Sets the compressor to use. + * + * @param compressor the compressor + */ + public ZFileOptions setCompressor(@Nonnull Compressor compressor) { + this.compressor = compressor; + return this; + } + + /** + * Obtains whether timestamps should be zeroed. + * + * @return should timestamps be zeroed? + */ + public boolean getNoTimestamps() { + return noTimestamps; + } + + /** + * Sets whether timestamps should be zeroed. + * + * @param noTimestamps should timestamps be zeroed? + */ + public ZFileOptions setNoTimestamps(boolean noTimestamps) { + this.noTimestamps = noTimestamps; + return this; + } + + /** + * Obtains the alignment rule. + * + * @return the alignment rule + */ + @Nonnull + public AlignmentRule getAlignmentRule() { + return alignmentRule; + } + + /** + * Sets the alignment rule. + * + * @param alignmentRule the alignment rule + */ + public ZFileOptions setAlignmentRule(@Nonnull AlignmentRule alignmentRule) { + this.alignmentRule = alignmentRule; + return this; + } + + /** + * Obtains whether the extra field should be used to cover empty spaces. See {@link ZFile} for + * an explanation on using the extra field for covering empty spaces. + * + * @return should the extra field be used to cover empty spaces? + */ + public boolean getCoverEmptySpaceUsingExtraField() { + return coverEmptySpaceUsingExtraField; + } + + /** + * Sets whether the extra field should be used to cover empty spaces. See {@link ZFile} for an + * explanation on using the extra field for covering empty spaces. + * + * @param coverEmptySpaceUsingExtraField should the extra field be used to cover empty spaces? + */ + public ZFileOptions setCoverEmptySpaceUsingExtraField(boolean coverEmptySpaceUsingExtraField) { + this.coverEmptySpaceUsingExtraField = coverEmptySpaceUsingExtraField; + return this; + } + + /** + * Obtains whether files should be automatically sorted before updating the zip file. See + * {@link ZFile} for an explanation on automatic sorting. + * + * @return should the file be automatically sorted? + */ + public boolean getAutoSortFiles() { + return autoSortFiles; + } + + /** + * Sets whether files should be automatically sorted before updating the zip file. See {@link + * ZFile} for an explanation on automatic sorting. + * + * @param autoSortFiles should the file be automatically sorted? + */ + public ZFileOptions setAutoSortFiles(boolean autoSortFiles) { + this.autoSortFiles = autoSortFiles; + return this; + } + + /** + * Sets the verification log factory. + * + * @param verifyLogFactory verification log factory + */ + public ZFileOptions setVerifyLogFactory(@Nonnull Supplier<VerifyLog> verifyLogFactory) { + this.verifyLogFactory = verifyLogFactory; + return this; + } + + /** + * Obtains the verification log factory. By default, the verification log doesn't store + * anything and will always return an empty log. + * + * @return the verification log factory + */ + @Nonnull + public Supplier<VerifyLog> getVerifyLogFactory() { + return verifyLogFactory; + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/zip/ZipField.java b/src/main/java/com/android/tools/build/apkzlib/zip/ZipField.java new file mode 100644 index 0000000..8e8d27d --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zip/ZipField.java @@ -0,0 +1,364 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zip; + +import com.android.tools.build.apkzlib.zip.utils.LittleEndianUtils; +import com.google.common.base.Preconditions; +import com.google.common.base.Verify; +import com.google.common.collect.Sets; +import com.google.common.primitives.Ints; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.Set; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * The ZipField class represents a field in a record in a zip file. Zip files are made with records + * that have fields. This class makes it easy to read, write and verify field values. + * <p> + * There are two main types of fields: 2-byte fields and 4-byte fields. We represent each one as + * a subclass of {@code ZipField}, {@code F2} for the 2-byte field and {@code F4} for the 4-byte + * field. Because Java's {@code int} data type is guaranteed to be 4-byte, all methods use Java's + * native {@link int} as data type. + * <p> + * For each field we can either read, write or verify. Verification is used for fields whose value + * we know. Some fields, <em>e.g.</em> signature fields, have fixed value. Other fields have + * variable values, but in some situations we know which value they have. For example, the last + * modification time of a file's local header will have to match the value of the file's + * modification time as stored in the central directory. + * <p> + * Because records are compact, <em>i.e.</em> fields are stored sequentially with no empty spaces, + * fields are generally created in the sequence they exist and the end offset of a field is used + * as the offset of the next one. The end of a field can be obtained by invoking + * {@link #endOffset()}. This allows creating fields in sequence without doing offset computation: + * <pre> + * ZipField.F2 firstField = new ZipField.F2(0, "First field"); + * ZipField.F4 secondField = new ZipField(firstField.endOffset(), "Second field"); + * </pre> + */ +abstract class ZipField { + + /** + * Field name. Used for providing (more) useful error messages. + */ + @Nonnull + private final String name; + + /** + * Offset of the file in the record. + */ + protected final int offset; + + /** + * Size of the field. Only 2 or 4 allowed. + */ + private final int size; + + /** + * If a fixed value exists for the field, then this attribute will contain that value. + */ + @Nullable + private final Long expected; + + /** + * All invariants that this field must verify. + */ + @Nonnull + private Set<ZipFieldInvariant> invariants; + + /** + * Creates a new field that does not contain a fixed value. + * + * @param offset the field's offset in the record + * @param size the field size + * @param name the field's name + * @param invariants the invariants that must be verified by the field + */ + ZipField(int offset, int size, @Nonnull String name, ZipFieldInvariant... invariants) { + Preconditions.checkArgument(offset >= 0, "offset >= 0"); + Preconditions.checkArgument(size == 2 || size == 4, "size != 2 && size != 4"); + + this.name = name; + this.offset = offset; + this.size = size; + expected = null; + this.invariants = Sets.newHashSet(invariants); + } + + /** + * Creates a new field that contains a fixed value. + * + * @param offset the field's offset in the record + * @param size the field size + * @param expected the expected field value + * @param name the field's name + */ + ZipField(int offset, int size, long expected, @Nonnull String name) { + Preconditions.checkArgument(offset >= 0, "offset >= 0"); + Preconditions.checkArgument(size == 2 || size == 4, "size != 2 && size != 4"); + + this.name = name; + this.offset = offset; + this.size = size; + this.expected = expected; + invariants = Sets.newHashSet(); + } + + /** + * Checks whether a value verifies the field's invariants. Nothing happens if the value verifies + * the invariants. + * + * @param value the value + * @throws IOException the invariants are not verified + */ + private void checkVerifiesInvariants(long value) throws IOException { + for (ZipFieldInvariant invariant : invariants) { + if (!invariant.isValid(value)) { + throw new IOException("Value " + value + " of field " + name + " is invalid " + + "(fails '" + invariant.getName() + "')."); + } + } + } + + /** + * Advances the position in the provided byte buffer by the size of this field. + * + * @param bytes the byte buffer; at the end of the method its position will be greater by + * the size of this field + * @throws IOException failed to advance the buffer + */ + void skip(@Nonnull ByteBuffer bytes) throws IOException { + if (bytes.remaining() < size) { + throw new IOException("Cannot skip field " + name + " because only " + + bytes.remaining() + " remain in the buffer."); + } + + bytes.position(bytes.position() + size); + } + + /** + * Reads a field value. + * + * @param bytes the byte buffer with the record data; after this method finishes, the buffer + * will be positioned at the first byte after the field + * @return the value of the field + * @throws IOException failed to read the field + */ + long read(@Nonnull ByteBuffer bytes) throws IOException { + if (bytes.remaining() < size) { + throw new IOException("Cannot skip field " + name + " because only " + + bytes.remaining() + " remain in the buffer."); + } + + bytes.order(ByteOrder.LITTLE_ENDIAN); + + long r; + if (size == 2) { + r = LittleEndianUtils.readUnsigned2Le(bytes); + } else { + r = LittleEndianUtils.readUnsigned4Le(bytes); + } + + checkVerifiesInvariants(r); + return r; + } + + /** + * Verifies that the field at the current buffer position has the expected value. The field + * must have been created with the constructor that defines the expected value. + * + * @param bytes the byte buffer with the record data; after this method finishes, the buffer + * will be positioned at the first byte after the field + * @throws IOException failed to read the field or the field does not have the expected value + */ + void verify(@Nonnull ByteBuffer bytes) throws IOException { + verify(bytes, null); + } + + /** + * Verifies that the field at the current buffer position has the expected value. The field + * must have been created with the constructor that defines the expected value. + * + * @param bytes the byte buffer with the record data; after this method finishes, the buffer + * will be positioned at the first byte after the field + * @param verifyLog if non-{@code null}, will log the verification error + * @throws IOException failed to read the data or the field does not have the expected value; + * only thrown if {@code verifyLog} is {@code null} + */ + void verify(@Nonnull ByteBuffer bytes, @Nullable VerifyLog verifyLog) throws IOException { + Preconditions.checkState(expected != null, "expected == null"); + verify(bytes, expected, verifyLog); + } + + /** + * Verifies that the field has an expected value. + * + * @param bytes the byte buffer with the record data; after this method finishes, the buffer + * will be positioned at the first byte after the field + * @param expected the value we expect the field to have; if this field has invariants, the + * value must verify them + * @throws IOException failed to read the data or the field does not have the expected value + */ + void verify(@Nonnull ByteBuffer bytes, long expected) throws IOException { + verify(bytes, expected, null); + } + + /** + * Verifies that the field has an expected value. + * + * @param bytes the byte buffer with the record data; after this method finishes, the buffer + * will be positioned at the first byte after the field + * @param expected the value we expect the field to have; if this field has invariants, the + * value must verify them + * @param verifyLog if non-{@code null}, will log the verification error + * @throws IOException failed to read the data or the field does not have the expected value; + * only thrown if {@code verifyLog} is {@code null} + */ + void verify( + @Nonnull ByteBuffer bytes, + long expected, + @Nullable VerifyLog verifyLog) throws IOException { + checkVerifiesInvariants(expected); + long r = read(bytes); + if (r != expected) { + String error = + String.format( + "Incorrect value for field '%s': value is %s but %s expected.", + name, + r, + expected); + + if (verifyLog == null) { + throw new IOException(error); + } else { + verifyLog.log(error); + } + } + } + + /** + * Writes the value of the field. + * + * @param output where to write the field; the field will be written at the current position + * of the buffer + * @param value the value to write + * @throws IOException failed to write the value in the stream + */ + void write(@Nonnull ByteBuffer output, long value) throws IOException { + checkVerifiesInvariants(value); + + Preconditions.checkArgument(value >= 0, "value (%s) < 0", value); + + if (size == 2) { + Preconditions.checkArgument(value <= 0x0000ffff, "value (%s) > 0x0000ffff", value); + LittleEndianUtils.writeUnsigned2Le(output, Ints.checkedCast(value)); + } else { + Verify.verify(size == 4); + Preconditions.checkArgument(value <= 0x00000000ffffffffL, + "value (%s) > 0x00000000ffffffffL", value); + LittleEndianUtils.writeUnsigned4Le(output, value); + } + } + + /** + * Writes the value of the field. The field must have an expected value set in the constructor. + * + * @param output where to write the field; the field will be written at the current position + * of the buffer + * @throws IOException failed to write the value in the stream + */ + void write(@Nonnull ByteBuffer output) throws IOException { + Preconditions.checkState(expected != null, "expected == null"); + write(output, expected); + } + + /** + * Obtains the offset at which the field starts. + * + * @return the start offset + */ + int offset() { + return offset; + } + + /** + * Obtains the offset at which the field ends. This is the exact offset at which the next + * field starts. + * + * @return the end offset + */ + int endOffset() { + return offset + size; + } + + /** + * Concrete implementation of {@link ZipField} that represents a 2-byte field. + */ + static class F2 extends ZipField { + + /** + * Creates a new field. + * + * @param offset the field's offset in the record + * @param name the field's name + * @param invariants the invariants that must be verified by the field + */ + F2(int offset, @Nonnull String name, ZipFieldInvariant... invariants) { + super(offset, 2, name, invariants); + } + + /** + * Creates a new field that contains a fixed value. + * + * @param offset the field's offset in the record + * @param expected the expected field value + * @param name the field's name + */ + F2(int offset, long expected, @Nonnull String name) { + super(offset, 2, expected, name); + } + } + + /** + * Concrete implementation of {@link ZipField} that represents a 4-byte field. + */ + static class F4 extends ZipField { + /** + * Creates a new field. + * + * @param offset the field's offset in the record + * @param name the field's name + * @param invariants the invariants that must be verified by the field + */ + F4(int offset, @Nonnull String name, ZipFieldInvariant... invariants) { + super(offset, 4, name, invariants); + } + + /** + * Creates a new field that contains a fixed value. + * + * @param offset the field's offset in the record + * @param expected the expected field value + * @param name the field's name + */ + F4(int offset, long expected, @Nonnull String name) { + super(offset, 4, expected, name); + } + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/zip/ZipFieldInvariant.java b/src/main/java/com/android/tools/build/apkzlib/zip/ZipFieldInvariant.java new file mode 100644 index 0000000..7207080 --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zip/ZipFieldInvariant.java @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zip; + +/** + * A field rule defines an invariant (<em>i.e.</em>, a constraint) that has to be verified by a + * field value. + */ +interface ZipFieldInvariant { + + /** + * Evalutes the invariant against a value. + * + * @param value the value to check the invariant + * @return is the invariant valid? + */ + boolean isValid(long value); + + /** + * Obtains the name of the invariant. Used for information purposes. + * + * @return the name of the invariant + */ + String getName(); +} diff --git a/src/main/java/com/android/tools/build/apkzlib/zip/ZipFieldInvariantMaxValue.java b/src/main/java/com/android/tools/build/apkzlib/zip/ZipFieldInvariantMaxValue.java new file mode 100644 index 0000000..65c4c5b --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zip/ZipFieldInvariantMaxValue.java @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zip; + +/** + * Invariant checking a zip field does not exceed a threshold. + */ +class ZipFieldInvariantMaxValue implements ZipFieldInvariant { + + /** + * The maximum value allowed. + */ + private long max; + + /** + * Creates a new invariant. + * + * @param max the maximum value allowed for the field + */ + ZipFieldInvariantMaxValue(int max) { + this.max = max; + } + + @Override + public boolean isValid(long value) { + return value <= max; + } + + @Override + public String getName() { + return "Maximum value " + max; + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/zip/ZipFieldInvariantNonNegative.java b/src/main/java/com/android/tools/build/apkzlib/zip/ZipFieldInvariantNonNegative.java new file mode 100644 index 0000000..76c2fb2 --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zip/ZipFieldInvariantNonNegative.java @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zip; + +/** + * Invariant that verifies a field's value is not negative. + */ +class ZipFieldInvariantNonNegative implements ZipFieldInvariant { + + @Override + public boolean isValid(long value) { + return value >= 0; + } + + @Override + public String getName() { + return "Is positive"; + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/zip/ZipFileState.java b/src/main/java/com/android/tools/build/apkzlib/zip/ZipFileState.java new file mode 100644 index 0000000..6d72816 --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zip/ZipFileState.java @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zip; + +/** + * The {@code ZipFileState} enumeration holds the state of a {@link ZFile}. + */ +enum ZipFileState { + /** + * Zip file is closed. + */ + CLOSED, + + /** + * File file is open in read-only mode. + */ + OPEN_RO, + + /** + * File file is open in read-write mode. + */ + OPEN_RW +} diff --git a/src/main/java/com/android/tools/build/apkzlib/zip/compress/BestAndDefaultDeflateExecutorCompressor.java b/src/main/java/com/android/tools/build/apkzlib/zip/compress/BestAndDefaultDeflateExecutorCompressor.java new file mode 100644 index 0000000..0e646cf --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zip/compress/BestAndDefaultDeflateExecutorCompressor.java @@ -0,0 +1,89 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zip.compress; + +import com.android.tools.build.apkzlib.zip.CompressionResult; +import com.android.tools.build.apkzlib.zip.utils.ByteTracker; +import com.android.tools.build.apkzlib.zip.utils.CloseableByteSource; +import com.google.common.base.Preconditions; +import java.util.concurrent.Executor; +import java.util.zip.Deflater; +import javax.annotation.Nonnull; + +/** + * Compressor that tries both the best and default compression algorithms and picks the default + * unless the best is at least a given percentage smaller. + */ +public class BestAndDefaultDeflateExecutorCompressor extends ExecutorCompressor { + + /** + * Deflater using the default compression level. + */ + @Nonnull + private final DeflateExecutionCompressor defaultDeflater; + + /** + * Deflater using the best compression level. + */ + @Nonnull + private final DeflateExecutionCompressor bestDeflater; + + /** + * Minimum best compression size / default compression size ratio needed to pick the default + * compression size. + */ + private final double minRatio; + + /** + * Creates a new compressor. + * + * @param executor the executor used to perform compression activities. + * @param tracker the byte tracker to keep track of allocated bytes + * @param minRatio the minimum best compression size / default compression size needed to pick + * the default compression size; if {@code 0.0} then the default compression is always picked, + * if {@code 1.0} then the best compression is always picked unless it produces the exact same + * size as the default compression. + */ + public BestAndDefaultDeflateExecutorCompressor(@Nonnull Executor executor, + @Nonnull ByteTracker tracker, double minRatio) { + super(executor); + + Preconditions.checkArgument(minRatio >= 0.0, "minRatio < 0.0"); + Preconditions.checkArgument(minRatio <= 1.0, "minRatio > 1.0"); + + defaultDeflater = + new DeflateExecutionCompressor(executor, tracker, Deflater.DEFAULT_COMPRESSION); + bestDeflater = + new DeflateExecutionCompressor(executor, tracker, Deflater.BEST_COMPRESSION); + this.minRatio = minRatio; + } + + @Nonnull + @Override + protected CompressionResult immediateCompress(@Nonnull CloseableByteSource source) + throws Exception { + CompressionResult defaultResult = defaultDeflater.immediateCompress(source); + CompressionResult bestResult = bestDeflater.immediateCompress(source); + + double sizeRatio = bestResult.getSize() / (double) defaultResult.getSize(); + if (sizeRatio >= minRatio) { + return defaultResult; + } else { + return bestResult; + } + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/zip/compress/DeflateExecutionCompressor.java b/src/main/java/com/android/tools/build/apkzlib/zip/compress/DeflateExecutionCompressor.java new file mode 100644 index 0000000..dbaeff0 --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zip/compress/DeflateExecutionCompressor.java @@ -0,0 +1,81 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zip.compress; + +import com.android.tools.build.apkzlib.zip.CompressionMethod; +import com.android.tools.build.apkzlib.zip.CompressionResult; +import com.android.tools.build.apkzlib.zip.utils.ByteTracker; +import com.android.tools.build.apkzlib.zip.utils.CloseableByteSource; +import java.io.ByteArrayOutputStream; +import java.util.concurrent.Executor; +import java.util.zip.Deflater; +import java.util.zip.DeflaterOutputStream; +import javax.annotation.Nonnull; + +/** + * Compressor that uses deflate with an executor. + */ +public class DeflateExecutionCompressor extends ExecutorCompressor { + + + /** + * Deflate compression level. + */ + private final int level; + + /** + * Byte tracker to use to create byte sources. + */ + @Nonnull + private final ByteTracker tracker; + + /** + * Creates a new compressor. + * + * @param executor the executor to run deflation tasks + * @param tracker the byte tracker to use to keep track of memory usage + * @param level the compression level + */ + public DeflateExecutionCompressor( + @Nonnull Executor executor, + @Nonnull ByteTracker tracker, + int level) { + super(executor); + + this.level = level; + this.tracker = tracker; + } + + @Nonnull + @Override + protected CompressionResult immediateCompress(@Nonnull CloseableByteSource source) + throws Exception { + ByteArrayOutputStream output = new ByteArrayOutputStream(); + Deflater deflater = new Deflater(level, true); + + try (DeflaterOutputStream dos = new DeflaterOutputStream(output, deflater)) { + dos.write(source.read()); + } + + CloseableByteSource result = tracker.fromStream(output); + if (result.size() >= source.size()) { + return new CompressionResult(source, CompressionMethod.STORE, source.size()); + } else { + return new CompressionResult(result, CompressionMethod.DEFLATE, result.size()); + } + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/zip/compress/ExecutorCompressor.java b/src/main/java/com/android/tools/build/apkzlib/zip/compress/ExecutorCompressor.java new file mode 100644 index 0000000..6a19907 --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zip/compress/ExecutorCompressor.java @@ -0,0 +1,72 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zip.compress; + +import com.android.tools.build.apkzlib.zip.CompressionResult; +import com.android.tools.build.apkzlib.zip.Compressor; +import com.android.tools.build.apkzlib.zip.utils.CloseableByteSource; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; +import java.util.concurrent.Executor; +import javax.annotation.Nonnull; + +/** + * A synchronous compressor is a compressor that computes the result of compression immediately + * and never returns an uncomputed future object. + */ +public abstract class ExecutorCompressor implements Compressor { + + /** + * The executor that does the work. + */ + @Nonnull + private final Executor executor; + + /** + * Compressor that delegates execution into the given executor. + * @param executor the executor that will do the compress + */ + public ExecutorCompressor(@Nonnull Executor executor) { + this.executor = executor; + } + + @Nonnull + @Override + public ListenableFuture<CompressionResult> compress( + @Nonnull final CloseableByteSource source) { + final SettableFuture<CompressionResult> future = SettableFuture.create(); + executor.execute(() -> { + try { + future.set(immediateCompress(source)); + } catch (Throwable e) { + future.setException(e); + } + }); + + return future; + } + + /** + * Immediately compresses a source. + * @param source the source to compress + * @return the result of compression + * @throws Exception failed to compress + */ + @Nonnull + protected abstract CompressionResult immediateCompress(@Nonnull CloseableByteSource source) + throws Exception; +} diff --git a/src/main/java/com/android/tools/build/apkzlib/zip/compress/Zip64NotSupportedException.java b/src/main/java/com/android/tools/build/apkzlib/zip/compress/Zip64NotSupportedException.java new file mode 100644 index 0000000..bf77e34 --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zip/compress/Zip64NotSupportedException.java @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zip.compress; + +import java.io.IOException; + +/** Exception raised by ZFile when encountering unsupported Zip64 format jar files. */ +public class Zip64NotSupportedException extends IOException { + + public Zip64NotSupportedException(String message) { + super(message); + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/zip/compress/package-info.java b/src/main/java/com/android/tools/build/apkzlib/zip/compress/package-info.java new file mode 100644 index 0000000..cdc85f8 --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zip/compress/package-info.java @@ -0,0 +1,20 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Compressors to use with the {@code zip} package. + */ +package com.android.tools.build.apkzlib.zip.compress; diff --git a/src/main/java/com/android/tools/build/apkzlib/zip/utils/ByteTracker.java b/src/main/java/com/android/tools/build/apkzlib/zip/utils/ByteTracker.java new file mode 100644 index 0000000..d956dd2 --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zip/utils/ByteTracker.java @@ -0,0 +1,120 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zip.utils; + +import com.google.common.io.ByteSource; +import com.google.common.io.ByteStreams; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import javax.annotation.Nonnull; + +/** + * Keeps track of used bytes allowing gauging memory usage. + */ +public class ByteTracker { + + /** + * Number of bytes currently in use. + */ + private long bytesUsed; + + /** + * Maximum number of bytes used. + */ + private long maxBytesUsed; + + /** + * Creates a new byte source by fully reading an input stream. + * + * @param stream the input stream + * @return a byte source containing the cached data from the given stream + * @throws IOException failed to read the stream + */ + public CloseableDelegateByteSource fromStream(@Nonnull InputStream stream) throws IOException { + byte[] data = ByteStreams.toByteArray(stream); + updateUsage(data.length); + return new CloseableDelegateByteSource(ByteSource.wrap(data), data.length) { + @Override + public synchronized void innerClose() throws IOException { + super.innerClose(); + updateUsage(-sizeNoException()); + } + }; + } + + /** + * Creates a new byte source by snapshotting the provided stream. + * + * @param stream the stream with the data + * @return a byte source containing the cached data from the given stream + * @throws IOException failed to read the stream + */ + public CloseableDelegateByteSource fromStream(@Nonnull ByteArrayOutputStream stream) + throws IOException { + byte[] data = stream.toByteArray(); + updateUsage(data.length); + return new CloseableDelegateByteSource(ByteSource.wrap(data), data.length) { + @Override + public synchronized void innerClose() throws IOException { + super.innerClose(); + updateUsage(-sizeNoException()); + } + }; + } + + /** + * Creates a new byte source from another byte source. + * + * @param source the byte source to copy data from + * @return the tracked byte source + * @throws IOException failed to read data from the byte source + */ + public CloseableDelegateByteSource fromSource(@Nonnull ByteSource source) throws IOException { + return fromStream(source.openStream()); + } + + /** + * Updates the memory used by this tracker. + * + * @param delta the number of bytes to add or remove, if negative + */ + private synchronized void updateUsage(long delta) { + bytesUsed += delta; + if (maxBytesUsed < bytesUsed) { + maxBytesUsed = bytesUsed; + } + } + + /** + * Obtains the number of bytes currently used. + * + * @return the number of bytes + */ + public synchronized long getBytesUsed() { + return bytesUsed; + } + + /** + * Obtains the maximum number of bytes ever used by this tracker. + * + * @return the number of bytes + */ + public synchronized long getMaxBytesUsed() { + return maxBytesUsed; + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/zip/utils/CloseableByteSource.java b/src/main/java/com/android/tools/build/apkzlib/zip/utils/CloseableByteSource.java new file mode 100644 index 0000000..9af9671 --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zip/utils/CloseableByteSource.java @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zip.utils; + +import com.google.common.io.ByteSource; +import java.io.Closeable; +import java.io.IOException; + +/** + * Byte source that can be closed. Closing a byte source allows releasing any resources associated + * with it. This should not be confused with closing streams. For example, {@link ByteTracker} uses + * {@code CloseableByteSources} to know when the data associated with the byte source can be + * released. + */ +public abstract class CloseableByteSource extends ByteSource implements Closeable { + + /** + * Has the source been closed? + */ + private boolean closed; + + /** + * Creates a new byte source. + */ + public CloseableByteSource() { + closed = false; + } + + @Override + public final synchronized void close() throws IOException { + if (closed) { + return; + } + + try { + innerClose(); + } finally { + closed = true; + } + } + + /** + * Closes the by source. This method is only invoked once, even if {@link #close()} is + * called multiple times. + * + * @throws IOException failed to close + */ + protected abstract void innerClose() throws IOException; +} diff --git a/src/main/java/com/android/tools/build/apkzlib/zip/utils/CloseableDelegateByteSource.java b/src/main/java/com/android/tools/build/apkzlib/zip/utils/CloseableDelegateByteSource.java new file mode 100644 index 0000000..df084d4 --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zip/utils/CloseableDelegateByteSource.java @@ -0,0 +1,171 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zip.utils; + +import com.google.common.hash.HashCode; +import com.google.common.hash.HashFunction; +import com.google.common.io.ByteProcessor; +import com.google.common.io.ByteSink; +import com.google.common.io.ByteSource; +import com.google.common.io.CharSource; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.Charset; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * Closeable byte source that delegates to another byte source. + */ +public class CloseableDelegateByteSource extends CloseableByteSource { + + /** + * The byte source we delegate all operations to. {@code null} if disposed. + */ + @Nullable + private ByteSource inner; + + /** + * Size of the byte source. This is the same as {@code inner.size()} (when {@code inner} + * is not {@code null}), but we keep it separate to avoid calling {@code inner.size()} + * because it might throw {@code IOException}. + */ + private final long mSize; + + /** + * Creates a new byte source. + * + * @param inner the inner byte source + * @param size the size of the source + */ + public CloseableDelegateByteSource(@Nonnull ByteSource inner, long size) { + this.inner = inner; + mSize = size; + } + + /** + * Obtains the inner byte source. Will throw an exception if the inner by byte source has + * been disposed of. + * + * @return the inner byte source + */ + @Nonnull + private synchronized ByteSource get() { + if (inner == null) { + throw new ByteSourceDisposedException(); + } + + return inner; + } + + /** + * Mark the byte source as disposed. + */ + @Override + protected synchronized void innerClose() throws IOException { + if (inner == null) { + return; + } + + inner = null; + } + + /** + * Obtains the size of this byte source. Equivalent to {@link #size()} but not throwing + * {@code IOException}. + * + * @return the size of the byte source + */ + public long sizeNoException() { + return mSize; + } + + @Override + public CharSource asCharSource(Charset charset) { + return get().asCharSource(charset); + } + + @Override + public InputStream openBufferedStream() throws IOException { + return get().openBufferedStream(); + } + + @Override + public ByteSource slice(long offset, long length) { + return get().slice(offset, length); + } + + @Override + public boolean isEmpty() throws IOException { + return get().isEmpty(); + } + + @Override + public long size() throws IOException { + return get().size(); + } + + @Override + public long copyTo(@Nonnull OutputStream output) throws IOException { + return get().copyTo(output); + } + + @Override + public long copyTo(@Nonnull ByteSink sink) throws IOException { + return get().copyTo(sink); + } + + @Override + public byte[] read() throws IOException { + return get().read(); + } + + @Override + public <T> T read(@Nonnull ByteProcessor<T> processor) throws IOException { + return get().read(processor); + } + + @Override + public HashCode hash(HashFunction hashFunction) throws IOException { + return get().hash(hashFunction); + } + + @Override + public boolean contentEquals(@Nonnull ByteSource other) throws IOException { + return get().contentEquals(other); + } + + @Override + public InputStream openStream() throws IOException { + return get().openStream(); + } + + /** + * Exception thrown when trying to use a byte source that has been disposed. + */ + private static class ByteSourceDisposedException extends RuntimeException { + + /** + * Creates a new exception. + */ + private ByteSourceDisposedException() { + super("Byte source was created by a ByteTracker and is now disposed. If you see " + + "this message, then there is a bug."); + } + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/zip/utils/LittleEndianUtils.java b/src/main/java/com/android/tools/build/apkzlib/zip/utils/LittleEndianUtils.java new file mode 100644 index 0000000..1fd056a --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zip/utils/LittleEndianUtils.java @@ -0,0 +1,129 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zip.utils; + +import com.google.common.base.Preconditions; +import com.google.common.base.Verify; +import java.io.EOFException; +import java.io.IOException; +import java.nio.ByteBuffer; +import javax.annotation.Nonnull; + +/** + * Utilities to read and write 16 and 32 bit integers with support for little-endian + * encoding, as used in zip files. Zip files actually use unsigned data types. We use Java's native + * (signed) data types but will use long (64 bit) to ensure we can fit the whole range. + */ +public class LittleEndianUtils { + /** + * Utility class, no constructor. + */ + private LittleEndianUtils() { + } + + /** + * Reads 4 bytes in little-endian format and converts them into a 32-bit value. + * + * @param bytes from where should the bytes be read; the first 4 bytes of the source will be + * read + * @return the 32-bit value + * @throws IOException failed to read the value + */ + public static long readUnsigned4Le(@Nonnull ByteBuffer bytes) throws IOException { + Preconditions.checkNotNull(bytes, "bytes == null"); + + if (bytes.remaining() < 4) { + throw new EOFException("Not enough data: 4 bytes expected, " + bytes.remaining() + + " available."); + } + + byte b0 = bytes.get(); + byte b1 = bytes.get(); + byte b2 = bytes.get(); + byte b3 = bytes.get(); + long r = (b0 & 0xff) | ((b1 & 0xff) << 8) | ((b2 & 0xff) << 16) | ((b3 & 0xffL) << 24); + Verify.verify(r >= 0); + Verify.verify(r <= 0x00000000ffffffffL); + return r; + } + + /** + * Reads 2 bytes in little-endian format and converts them into a 16-bit value. + * + * @param bytes from where should the bytes be read; the first 2 bytes of the source will be + * read + * @return the 16-bit value + * @throws IOException failed to read the value + */ + public static int readUnsigned2Le(@Nonnull ByteBuffer bytes) throws IOException { + Preconditions.checkNotNull(bytes, "bytes == null"); + + if (bytes.remaining() < 2) { + throw new EOFException( + "Not enough data: 2 bytes expected, " + + bytes.remaining() + + " available."); + } + + byte b0 = bytes.get(); + byte b1 = bytes.get(); + int r = (b0 & 0xff) | ((b1 & 0xff) << 8); + + Verify.verify(r >= 0); + Verify.verify(r <= 0x0000ffff); + return r; + } + + /** + * Writes 4 bytes in little-endian format, converting them from a 32-bit value. + * + * @param output the output stream where the bytes will be written + * @param value the 32-bit value to convert + * @throws IOException failed to write the value data + */ + public static void writeUnsigned4Le(@Nonnull ByteBuffer output, long value) + throws IOException { + Preconditions.checkNotNull(output, "output == null"); + Preconditions.checkArgument(value >= 0, "value (%s) < 0", value); + Preconditions.checkArgument( + value <= 0x00000000ffffffffL, + "value (%s) > 0x00000000ffffffffL", + value); + + output.put((byte) (value & 0xff)); + output.put((byte) ((value >> 8) & 0xff)); + output.put((byte) ((value >> 16) & 0xff)); + output.put((byte) ((value >> 24) & 0xff)); + } + + /** + * Writes 2 bytes in little-endian format, converting them from a 16-bit value. + * + * @param output the output stream where the bytes will be written + * @param value the 16-bit value to convert + * @throws IOException failed to write the value data + */ + public static void writeUnsigned2Le(@Nonnull ByteBuffer output, int value) + throws IOException { + Preconditions.checkNotNull(output, "output == null"); + Preconditions.checkArgument(value >= 0, "value (%s) < 0", value); + Preconditions.checkArgument(value <= 0x0000ffff, "value (%s) > 0x0000ffff", value); + + output.put((byte) (value & 0xff)); + output.put((byte) ((value >> 8) & 0xff)); + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/zip/utils/MsDosDateTimeUtils.java b/src/main/java/com/android/tools/build/apkzlib/zip/utils/MsDosDateTimeUtils.java new file mode 100644 index 0000000..cc16cb4 --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zip/utils/MsDosDateTimeUtils.java @@ -0,0 +1,110 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zip.utils; + +import com.google.common.base.Verify; +import java.util.Calendar; +import java.util.Date; + +/** + * Yes. This actually refers to MS-DOS in 2015. That's all I have to say about legacy stuff. + */ +public class MsDosDateTimeUtils { + /** + * Utility class: no constructor. + */ + private MsDosDateTimeUtils() { + } + + /** + * Packs java time value into an MS-DOS time value. + * + * @param time the time value + * @return the MS-DOS packed time + */ + public static int packTime(long time) { + Calendar c = Calendar.getInstance(); + c.setTime(new Date(time)); + + int seconds = c.get(Calendar.SECOND); + int minutes = c.get(Calendar.MINUTE); + int hours = c.get(Calendar.HOUR_OF_DAY); + + /* + * Here is how MS-DOS packs a time value: + * 0-4: seconds (divided by 2 because we only have 5 bits = 32 different numbers) + * 5-10: minutes (6 bits = 64 possible values) + * 11-15: hours (5 bits = 32 possible values) + * + * source: https://msdn.microsoft.com/en-us/library/windows/desktop/ms724247(v=vs.85).aspx + */ + return (hours << 11) | (minutes << 5) | (seconds / 2); + } + + /** + * Packs the current time value into an MS-DOS time value. + * + * @return the MS-DOS packed time + */ + public static int packCurrentTime() { + return packTime(new Date().getTime()); + } + + /** + * Packs java time value into an MS-DOS date value. + * + * @param time the time value + * @return the MS-DOS packed date + */ + public static int packDate(long time) { + Calendar c = Calendar.getInstance(); + c.setTime(new Date(time)); + + /* + * Even MS-DOS used 1 for January. Someone wasn't really thinking when they decided on Java + * it would start at 0... + */ + int day = c.get(Calendar.DAY_OF_MONTH); + int month = c.get(Calendar.MONTH) + 1; + + /* + * MS-DOS counts years starting from 1980. Since its launch date was in 81, it was obviously + * not necessary to talk about dates earlier than that. + */ + int year = c.get(Calendar.YEAR) - 1980; + Verify.verify(year >= 0 && year < 128); + + /* + * Here is how MS-DOS packs a date value: + * 0-4: day (5 bits = 32 values) + * 5-8: month (4 bits = 16 values) + * 9-15: year (7 bits = 128 values) + * + * source: https://msdn.microsoft.com/en-us/library/windows/desktop/ms724247(v=vs.85).aspx + */ + return (year << 9) | (month << 5) | day; + } + + /** + * Packs the current time value into an MS-DOS date value. + * + * @return the MS-DOS packed date + */ + public static int packCurrentDate() { + return packDate(new Date().getTime()); + } +} diff --git a/src/main/java/com/android/tools/build/apkzlib/zip/utils/RandomAccessFileUtils.java b/src/main/java/com/android/tools/build/apkzlib/zip/utils/RandomAccessFileUtils.java new file mode 100644 index 0000000..17c2d6c --- /dev/null +++ b/src/main/java/com/android/tools/build/apkzlib/zip/utils/RandomAccessFileUtils.java @@ -0,0 +1,59 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tools.build.apkzlib.zip.utils; + +import java.io.IOException; +import java.io.RandomAccessFile; +import javax.annotation.Nonnull; + +/** + * Utility class with utility methods for random access files. + */ +public final class RandomAccessFileUtils { + + private RandomAccessFileUtils() {} + + /** + * Reads from an random access file until the provided array is filled. Data is read from the + * current position in the file. + * + * @param raf the file to read data from + * @param data the array that will receive the data + * @throws IOException failed to read the data + */ + public static void fullyRead(@Nonnull RandomAccessFile raf, @Nonnull byte[] data) + throws IOException { + int r; + int p = 0; + + while ((r = raf.read(data, p, data.length - p)) > 0) { + p += r; + if (p == data.length) { + break; + } + } + + if (p < data.length) { + throw new IOException( + "Failed to read " + + data.length + + " bytes from file. Only " + + p + + " bytes could be read."); + } + } +} |