diff options
Diffstat (limited to 'src/main/java/com')
27 files changed, 3740 insertions, 0 deletions
diff --git a/src/main/java/com/google/android/downloader/AndroidConnectivityHandler.java b/src/main/java/com/google/android/downloader/AndroidConnectivityHandler.java new file mode 100644 index 0000000..adac6e9 --- /dev/null +++ b/src/main/java/com/google/android/downloader/AndroidConnectivityHandler.java @@ -0,0 +1,273 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import static android.Manifest.permission.ACCESS_NETWORK_STATE; +import static android.net.ConnectivityManager.CONNECTIVITY_ACTION; +import static androidx.core.content.ContextCompat.checkSelfPermission; +import static androidx.core.content.ContextCompat.getSystemService; +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.util.concurrent.MoreExecutors.directExecutor; +import static java.util.concurrent.TimeUnit.MILLISECONDS; + +import android.annotation.SuppressLint; +import android.annotation.TargetApi; +import android.content.BroadcastReceiver; +import android.content.Context; +import android.content.Intent; +import android.content.IntentFilter; +import android.content.pm.PackageManager; +import android.net.ConnectivityManager; +import android.net.Network; +import android.net.NetworkCapabilities; +import android.net.NetworkInfo; +import android.os.Build.VERSION; +import android.os.Build.VERSION_CODES; +import androidx.annotation.RequiresPermission; +import androidx.core.net.ConnectivityManagerCompat; +import com.google.android.downloader.DownloadConstraints.NetworkType; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.flogger.GoogleLogger; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListenableFutureTask; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeoutException; +import javax.annotation.Nullable; + +/** + * Default implementation of {@link ConnectivityHandler}, relying on Android's {@link + * ConnectivityManager}. + */ +public class AndroidConnectivityHandler implements ConnectivityHandler { + private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); + + private final Context context; + private final ScheduledExecutorService scheduledExecutorService; + private final ConnectivityManager connectivityManager; + private final long timeoutMillis; + + /** + * Creates a new AndroidConnectivityHandler to handle connectivity checks for the Downloader. + * + * @param context the context to use for perform Android API checks. Will be retained, so should + * not be a UI context. + * @param scheduledExecutorService a scheduled executor used to timeout operations waiting for + * connectivity. Beware that there are problems with this, see go/executors-timing for + * details. + * @param timeoutMillis how long to wait before timing out a connectivity check. If more than this + * amount of time elapses, the connectivity check will timeout, and the {@link + * ListenableFuture} returned by {@link #checkConnectivity} will resolve with a {@link + * TimeoutException}. + */ + public AndroidConnectivityHandler( + Context context, ScheduledExecutorService scheduledExecutorService, long timeoutMillis) { + if (PackageManager.PERMISSION_GRANTED != checkSelfPermission(context, ACCESS_NETWORK_STATE)) { + throw new IllegalStateException( + "AndroidConnectivityHandler requires the ACCESS_NETWORK_STATE permission."); + } + + this.context = context; + this.scheduledExecutorService = scheduledExecutorService; + this.connectivityManager = checkNotNull(getSystemService(context, ConnectivityManager.class)); + this.timeoutMillis = timeoutMillis; + } + + @Override + @RequiresPermission(ACCESS_NETWORK_STATE) + public ListenableFuture<Void> checkConnectivity(DownloadConstraints constraints) { + if (connectivitySatisfied(constraints)) { + return Futures.immediateVoidFuture(); + } + + ListenableFutureTask<Void> futureTask = ListenableFutureTask.create(() -> null); + // TODO: Using a receiver here isn't great. Ideally we'd use + // ConnectivityManager.requestNetwork(request, callback, timeout), but that's only available + // on SDK 26+, so we'd still need a fallback on older versions of Android. + NetworkBroadcastReceiver receiver = new NetworkBroadcastReceiver(constraints, futureTask); + context.registerReceiver(receiver, new IntentFilter(CONNECTIVITY_ACTION)); + futureTask.addListener(() -> context.unregisterReceiver(receiver), directExecutor()); + return Futures.withTimeout(futureTask, timeoutMillis, MILLISECONDS, scheduledExecutorService); + } + + @RequiresPermission(ACCESS_NETWORK_STATE) + private boolean connectivitySatisfied(DownloadConstraints downloadConstraints) { + // Special case the NONE value - if that is specified then skip all further checks. + if (downloadConstraints == DownloadConstraints.NONE) { + return true; + } + + NetworkType networkType; + + if (VERSION.SDK_INT >= VERSION_CODES.M) { + Network network = connectivityManager.getActiveNetwork(); + if (network == null) { + logger.atFine().log("No current network, connectivity cannot be satisfied."); + return false; + } + + NetworkCapabilities networkCapabilities = connectivityManager.getNetworkCapabilities(network); + if (networkCapabilities == null) { + logger.atFine().log( + "Can't determine network capabilities, connectivity cannot be satisfied"); + return false; + } + + if (!networkCapabilities.hasCapability(NetworkCapabilities.NET_CAPABILITY_INTERNET)) { + logger.atFine().log( + "Network does not have internet capabilities, connectivity cannot be satisfied."); + return false; + } + + if (downloadConstraints.requireUnmeteredNetwork() + && ConnectivityManagerCompat.isActiveNetworkMetered(connectivityManager)) { + logger.atFine().log("Network is metered, connectivity cannot be satisfied."); + return false; + } + + if (downloadConstraints.requiredNetworkTypes().contains(NetworkType.ANY)) { + // If the request doesn't care about the network type (by way of having NetworkType.ANY in + // its set of allowed network types), then stop checking now. + return true; + } + + networkType = computeNetworkType(networkCapabilities); + } else { + @SuppressLint("MissingPermission") // We just checked the permission above. + NetworkInfo networkInfo = connectivityManager.getActiveNetworkInfo(); + if (networkInfo == null) { + logger.atFine().log("No current network, connectivity cannot be satisfied."); + return false; + } + + if (!networkInfo.isConnected()) { + // Regardless of which type of network we have right now, if it's not connected then all + // downloads will fail, so just queue up all downloads in this case. + logger.atFine().log("Network disconnected, connectivity cannot be satisfied."); + return false; + } + + if (downloadConstraints.requireUnmeteredNetwork() + && ConnectivityManagerCompat.isActiveNetworkMetered(connectivityManager)) { + logger.atFine().log("Network is metered, connectivity cannot be satisfied."); + return false; + } + + if (downloadConstraints.requiredNetworkTypes().contains(NetworkType.ANY)) { + // If the request doesn't care about the network type (by way of having NetworkType.ANY in + // its set of allowed network types), then stop checking now. + return true; + } + + networkType = computeNetworkType(networkInfo.getType()); + } + + if (networkType == null) { + // If for some reason we couldn't determine the network type from Android (unexpected value?), + // then we can't validate it against the set of constraints, so fail the check. + return false; + } + + // Otherwise, just make sure that the current network type is allowed by this request. + return downloadConstraints.requiredNetworkTypes().contains(networkType); + } + + @Nullable + private static NetworkType computeNetworkType(int networkType) { + if (VERSION.SDK_INT >= VERSION_CODES.HONEYCOMB_MR2 + && networkType == ConnectivityManager.TYPE_BLUETOOTH) { + return NetworkType.BLUETOOTH; + } else if (VERSION.SDK_INT >= VERSION_CODES.HONEYCOMB_MR2 + && networkType == ConnectivityManager.TYPE_ETHERNET) { + return NetworkType.ETHERNET; + } else if (networkType == ConnectivityManager.TYPE_MOBILE + || networkType == ConnectivityManager.TYPE_MOBILE_MMS + || networkType == ConnectivityManager.TYPE_MOBILE_SUPL + || networkType == ConnectivityManager.TYPE_MOBILE_DUN + || networkType == ConnectivityManager.TYPE_MOBILE_HIPRI) { + return NetworkType.CELLULAR; + } else if (VERSION.SDK_INT >= VERSION_CODES.LOLLIPOP + && networkType == ConnectivityManager.TYPE_VPN) { + // There's no way to determine the underlying transport used by a VPN, so it's best to + // be conservative and treat it is a cellular network. + return NetworkType.CELLULAR; + } else if (networkType == ConnectivityManager.TYPE_WIFI) { + return NetworkType.WIFI; + } else if (networkType == ConnectivityManager.TYPE_WIMAX) { + // WiMAX and Cellular aren't really the same thing, but in practice they can be treated + // the same, as they are both typically available over long distances and are often metered. + return NetworkType.CELLULAR; + } + + return null; + } + + @Nullable + @TargetApi(VERSION_CODES.LOLLIPOP) + private static NetworkType computeNetworkType(NetworkCapabilities networkCapabilities) { + if (networkCapabilities.hasTransport(NetworkCapabilities.TRANSPORT_CELLULAR)) { + return NetworkType.CELLULAR; + } else if (networkCapabilities.hasTransport(NetworkCapabilities.TRANSPORT_WIFI)) { + return NetworkType.WIFI; + } else if (networkCapabilities.hasTransport(NetworkCapabilities.TRANSPORT_BLUETOOTH)) { + return NetworkType.BLUETOOTH; + } else if (networkCapabilities.hasTransport(NetworkCapabilities.TRANSPORT_ETHERNET)) { + return NetworkType.ETHERNET; + } else if (networkCapabilities.hasTransport(NetworkCapabilities.TRANSPORT_VPN)) { + // There's no way to determine the underlying transport used by a VPN, so it's best to + // be conservative and treat it is a cellular network. + return NetworkType.CELLULAR; + } + return null; + } + + @VisibleForTesting + class NetworkBroadcastReceiver extends BroadcastReceiver { + private final DownloadConstraints constraints; + private final Runnable completionRunnable; + + public NetworkBroadcastReceiver(DownloadConstraints constraints, Runnable completionRunnable) { + this.constraints = constraints; + this.completionRunnable = completionRunnable; + } + + @Override + @RequiresPermission(ACCESS_NETWORK_STATE) + public void onReceive(Context context, Intent intent) { + if (!CONNECTIVITY_ACTION.equals(intent.getAction())) { + logger.atSevere().log( + "NetworkBroadcastReceiver received an unexpected intent action: %s", + intent.getAction()); + return; + } + + if (intent.getBooleanExtra(ConnectivityManager.EXTRA_NO_CONNECTIVITY, false)) { + logger.atInfo().log("NetworkBroadcastReceiver updated but NO_CONNECTIVITY extra set"); + return; + } + + logger.atInfo().log( + "NetworkBroadcastReceiver received intent: %s %s", + intent.getAction(), intent.getExtras()); + + if (connectivitySatisfied(constraints)) { + logger.atInfo().log("Connectivity satisfied in BroadcastReceiver, running completion"); + completionRunnable.run(); + } else { + logger.atInfo().log("Connectivity NOT satisfied in BroadcastReceiver"); + } + } + } +} diff --git a/src/main/java/com/google/android/downloader/AndroidDownloaderLogger.java b/src/main/java/com/google/android/downloader/AndroidDownloaderLogger.java new file mode 100644 index 0000000..88fd157 --- /dev/null +++ b/src/main/java/com/google/android/downloader/AndroidDownloaderLogger.java @@ -0,0 +1,64 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import android.util.Log; +import com.google.errorprone.annotations.FormatMethod; +import com.google.errorprone.annotations.FormatString; +import javax.annotation.Nullable; + +/** + * Implementation of {@link DownloaderLogger} backed by Android's {@link Log} system. Useful for + * contexts (e.g. AOSP) that can't take the Flogger dependency. + */ +public class AndroidDownloaderLogger implements DownloaderLogger { + private static final String TAG = "downloader2"; + + @Override + @FormatMethod + public void logFine(@FormatString String message, Object... args) { + Log.d(TAG, String.format(message, args)); + } + + @Override + @FormatMethod + public void logInfo(@FormatString String message, Object... args) { + Log.i(TAG, String.format(message, args)); + } + + @Override + @FormatMethod + public void logWarning(@FormatString String message, Object... args) { + Log.w(TAG, String.format(message, args)); + } + + @Override + @FormatMethod + public void logWarning(@Nullable Throwable cause, @FormatString String message, Object... args) { + Log.w(TAG, String.format(message, args), cause); + } + + @Override + @FormatMethod + public void logError(@FormatString String message, Object... args) { + Log.e(TAG, String.format(message, args)); + } + + @Override + @FormatMethod + public void logError(@Nullable Throwable cause, @FormatString String message, Object... args) { + Log.e(TAG, String.format(message, args), cause); + } +} diff --git a/src/main/java/com/google/android/downloader/ConnectivityHandler.java b/src/main/java/com/google/android/downloader/ConnectivityHandler.java new file mode 100644 index 0000000..ecd8ef5 --- /dev/null +++ b/src/main/java/com/google/android/downloader/ConnectivityHandler.java @@ -0,0 +1,33 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import com.google.common.util.concurrent.ListenableFuture; + +/** Interface for dealing with connectivity checking and handling in the downloader. */ +public interface ConnectivityHandler { + /** + * Handles the checking connectivity for the given {@link DownloadConstraints}. This method must + * return a {@link ListenableFuture} that resolves when the constraints are satisfied. + * + * <p>The returned future may be resolved in failure state with a {@link + * java.util.concurrent.TimeoutException} if too much time without sufficient connectivity has + * been observed. The future may also be cancelled (resulting in a {@link + * java.util.concurrent.CancellationException}) if observing connectivity changes is no longer + * necessary. Either state will be given special treatment to retry downloads. Any other failure + * state will be considered fatal and will fail ongoing downloads. + */ + ListenableFuture<Void> checkConnectivity(DownloadConstraints constraints); +} diff --git a/src/main/java/com/google/android/downloader/CronetUrlEngine.java b/src/main/java/com/google/android/downloader/CronetUrlEngine.java new file mode 100644 index 0000000..f4f41a1 --- /dev/null +++ b/src/main/java/com/google/android/downloader/CronetUrlEngine.java @@ -0,0 +1,334 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.util.concurrent.MoreExecutors.directExecutor; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableSet; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; +import java.net.HttpURLConnection; +import java.nio.ByteBuffer; +import java.nio.channels.WritableByteChannel; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Executor; +import javax.annotation.Nullable; +import org.chromium.net.CallbackException; +import org.chromium.net.CronetEngine; +import org.chromium.net.CronetException; +import org.chromium.net.NetworkException; +import org.chromium.net.UrlResponseInfo; + +/** + * {@link UrlEngine} implementation that uses Cronet for network connectivity. + * + * <p>Note: Internally this implementation allocates a 128kb direct byte buffer per request to + * transfer bytes around. If memory use is sensitive, then the number of concurrent requests should + * be limited. + */ +public final class CronetUrlEngine implements UrlEngine { + private static final ImmutableSet<String> SCHEMES = ImmutableSet.of("http", "https"); + @VisibleForTesting static final int BUFFER_SIZE_BYTES = 128 * 1024; // 128kb + + private final CronetEngine cronetEngine; + private final Executor callbackExecutor; + + /** + * Creates a new Cronet-based {@link UrlEngine}. + * + * @param cronetEngine The pre-configured {@link CronetEngine} that will be used to implement HTTP + * connections. + * @param callbackExecutor The {@link Executor} on which Cronet's callbacks will be executed. Note + * that this request factory implementation will perform I/O in the callbacks, so make sure + * the threads backing the executor can block safely (i.e. do not run on the UI thread!) + */ + public CronetUrlEngine(CronetEngine cronetEngine, Executor callbackExecutor) { + this.cronetEngine = cronetEngine; + this.callbackExecutor = callbackExecutor; + } + + @Override + public UrlRequest.Builder createRequest(String url) { + SettableFuture<UrlResponse> responseFuture = SettableFuture.create(); + CronetCallback callback = new CronetCallback(responseFuture); + org.chromium.net.UrlRequest.Builder builder = + cronetEngine.newUrlRequestBuilder(url, callback, callbackExecutor); + return new CronetUrlRequestBuilder(builder, responseFuture); + } + + @Override + public Set<String> supportedSchemes() { + return SCHEMES; + } + + /** Cronet-specific implementation of {@link UrlRequest} */ + static class CronetUrlRequest implements UrlRequest { + private final org.chromium.net.UrlRequest urlRequest; + private final ListenableFuture<UrlResponse> responseFuture; + + CronetUrlRequest(CronetUrlRequestBuilder builder) { + urlRequest = builder.requestBuilder.build(); + responseFuture = builder.responseFuture; + + responseFuture.addListener( + () -> { + if (responseFuture.isCancelled()) { + urlRequest.cancel(); + } + }, + directExecutor()); + } + + @Override + public ListenableFuture<UrlResponse> send() { + urlRequest.start(); + return responseFuture; + } + } + + /** Cronet-specific implementation of {@link UrlRequest.Builder} */ + static class CronetUrlRequestBuilder implements UrlRequest.Builder { + private final org.chromium.net.UrlRequest.Builder requestBuilder; + private final ListenableFuture<UrlResponse> responseFuture; + + CronetUrlRequestBuilder( + org.chromium.net.UrlRequest.Builder requestBuilder, + ListenableFuture<UrlResponse> responseFuture) { + this.requestBuilder = requestBuilder; + this.responseFuture = responseFuture; + } + + @Override + public UrlRequest.Builder addHeader(String key, String value) { + requestBuilder.addHeader(key, value); + return this; + } + + @Override + public UrlRequest build() { + return new CronetUrlRequest(this); + } + } + + /** + * Cronet-specific implementation of {@link UrlResponse}. Implements its functionality by using + * Cronet's {@link org.chromium.net.UrlRequest} and {@link UrlResponseInfo} objects. + */ + static class CronetResponse implements UrlResponse { + private final org.chromium.net.UrlRequest urlRequest; + private final UrlResponseInfo urlResponseInfo; + private final SettableFuture<Long> completionFuture; + private final CronetCallback callback; + + CronetResponse( + org.chromium.net.UrlRequest urlRequest, + UrlResponseInfo urlResponseInfo, + SettableFuture<Long> completionFuture, + CronetCallback callback) { + this.urlRequest = urlRequest; + this.urlResponseInfo = urlResponseInfo; + this.completionFuture = completionFuture; + this.callback = callback; + } + + @Override + public int getResponseCode() { + return urlResponseInfo.getHttpStatusCode(); + } + + @Override + public Map<String, List<String>> getResponseHeaders() { + return urlResponseInfo.getAllHeaders(); + } + + @Override + public ListenableFuture<Long> readResponseBody(WritableByteChannel destinationChannel) { + IOUtil.validateChannel(destinationChannel); + callback.destinationChannel = destinationChannel; + urlRequest.read(ByteBuffer.allocateDirect(BUFFER_SIZE_BYTES)); + return completionFuture; + } + + @Override + public void close() { + urlRequest.cancel(); + } + } + + /** + * Implementation of {@link org.chromium.net.UrlRequest.Callback} to handle the lifecycle of a + * Cronet url request. The operations of handling the response metadata returned by the server as + * well as actually reading the response body happen here. + */ + static class CronetCallback extends org.chromium.net.UrlRequest.Callback { + private final SettableFuture<UrlResponse> responseFuture; + private final SettableFuture<Long> completionFuture = SettableFuture.create(); + + @Nullable private CronetResponse cronetResponse; + @Nullable private WritableByteChannel destinationChannel; + private long numBytesWritten; + + CronetCallback(SettableFuture<UrlResponse> responseFuture) { + this.responseFuture = responseFuture; + } + + @Override + public void onRedirectReceived( + org.chromium.net.UrlRequest urlRequest, + UrlResponseInfo urlResponseInfo, + String newLocationUrl) { + // Just blindly follow redirects; that's pretty much always what you want to do. + urlRequest.followRedirect(); + } + + @Override + public void onResponseStarted( + org.chromium.net.UrlRequest urlRequest, UrlResponseInfo urlResponseInfo) { + // We've received the response metadata from the server, so we have a status code and + // response headers to examine. At this point we can create a response object and complete + // the response future. If necessary, the body itself will be downloaded via a subsequent + // call urlRequest.read inside the CronetResponse.writeResponseBody, which will trigger the + // other lifecycle callbacks. + int httpCode = urlResponseInfo.getHttpStatusCode(); + if (httpCode >= HttpURLConnection.HTTP_BAD_REQUEST) { + responseFuture.setException( + new RequestException( + ErrorDetails.createFromHttpErrorResponse( + httpCode, + urlResponseInfo.getAllHeaders(), + urlResponseInfo.getHttpStatusText()))); + urlRequest.cancel(); + } else { + cronetResponse = new CronetResponse(urlRequest, urlResponseInfo, completionFuture, this); + responseFuture.set(cronetResponse); + } + } + + @Override + public void onReadCompleted( + org.chromium.net.UrlRequest urlRequest, + UrlResponseInfo urlResponseInfo, + ByteBuffer byteBuffer) + throws Exception { + // If we're already done, just bail out. + if (urlRequest.isDone()) { + return; + } + + // If the underlying future has been cancelled, cancel the request and abort. + if (completionFuture.isCancelled()) { + urlRequest.cancel(); + return; + } + + // Flip the buffer to prepare for reading from it. + byteBuffer.flip(); + + // Write however many bytes are in our buffer to the underlying channel. + numBytesWritten += IOUtil.blockingWrite(byteBuffer, checkNotNull(destinationChannel)); + + // Reset the buffer to be reused on the next iteration. + byteBuffer.clear(); + + // Finally, request more bytes. This is necessary per the Cronet API. + urlRequest.read(byteBuffer); + } + + @Override + public void onSucceeded( + org.chromium.net.UrlRequest urlRequest, UrlResponseInfo urlResponseInfo) { + // The body has been successfully streamed. Close the underlying response object to free + // up resources it holds, and resolve the pending future with the number of bytes written. + closeResponse(); + completionFuture.set(numBytesWritten); + } + + @Override + public void onFailed( + org.chromium.net.UrlRequest urlRequest, + UrlResponseInfo urlResponseInfo, + CronetException exception) { + // There was some sort of error with the connection. Clean up and resolve the pending future + // with the exception we encountered. + closeResponse(); + + ErrorDetails errorDetails; + if (urlResponseInfo != null + && urlResponseInfo.getHttpStatusCode() >= HttpURLConnection.HTTP_BAD_REQUEST) { + errorDetails = + ErrorDetails.createFromHttpErrorResponse( + urlResponseInfo.getHttpStatusCode(), + urlResponseInfo.getAllHeaders(), + urlResponseInfo.getHttpStatusText()); + } else if (exception instanceof NetworkException) { + NetworkException networkException = (NetworkException) exception; + errorDetails = + ErrorDetails.builder() + .setInternalErrorCode(networkException.getCronetInternalErrorCode()) + .setErrorMessage(Strings.nullToEmpty(networkException.getMessage())) + .build(); + } else { + errorDetails = + ErrorDetails.builder() + .setErrorMessage(Strings.nullToEmpty(exception.getMessage())) + .build(); + } + + RequestException requestException = + new RequestException(errorDetails, unwrapException(exception)); + + if (!responseFuture.isDone()) { + responseFuture.setException(requestException); + } else { + // N.B: The completion future is available iff the response future is resolved, so + // we don't need to resolve it with an exception here unless the response future is done. + completionFuture.setException(requestException); + } + } + + @Override + public void onCanceled( + org.chromium.net.UrlRequest urlRequest, UrlResponseInfo urlResponseInfo) { + // The request was cancelled. This only occurs when UrlRequest.cancel is called, which + // in turn only happens when UrlResponse.close is called. Clean up internal state + // and resolve the future with an error. + closeResponse(); + completionFuture.setException(new RequestException("UrlRequest cancelled")); + } + + /** Safely closes the current response object, if any. */ + private void closeResponse() { + CronetResponse cronetResponse = this.cronetResponse; + if (cronetResponse == null) { + return; + } + cronetResponse.close(); + } + } + + private static Throwable unwrapException(CronetException exception) { + // CallbackExceptions aren't interesting, so unwrap them. + if (exception instanceof CallbackException) { + Throwable cause = exception.getCause(); + return cause == null ? exception : cause; + } + return exception; + } +} diff --git a/src/main/java/com/google/android/downloader/DataUrl.java b/src/main/java/com/google/android/downloader/DataUrl.java new file mode 100644 index 0000000..7e83018 --- /dev/null +++ b/src/main/java/com/google/android/downloader/DataUrl.java @@ -0,0 +1,104 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import com.google.common.base.Splitter; +import com.google.common.io.BaseEncoding; +import java.util.List; + +/** + * Utility for handling data URLs. See the MDN for documentation of the format: + * https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URIs + * + * <p>See http://www.ietf.org/rfc/rfc2397.txt for the formal specification of the data URL scheme. + */ +class DataUrl { + private static final BaseEncoding BASE64_URL = BaseEncoding.base64Url(); + private static final BaseEncoding BASE64 = BaseEncoding.base64(); + private static final Splitter SEMICOLON_SPLITTER = Splitter.on(';'); + + private final byte[] data; + private final String mimeType; + + /** Thrown to indicate that the input URL could not be parsed as an RFC2397 data URL. */ + static class DataUrlException extends IllegalArgumentException { + DataUrlException(String message) { + super(message); + } + + DataUrlException(String message, Throwable cause) { + super(message, cause); + } + } + + private DataUrl(byte[] data, String mimeType) { + this.data = data; + this.mimeType = mimeType; + } + + byte[] data() { + return data; + } + + String mimeType() { + return mimeType; + } + + /** + * Decodes a data url, returning the payload as a {@link DataUrl} instance. + * + * <p>The syntax of a valid data URL is as follows: data:[<mediatype>][;base64],<data> + * This method will assert that the provided URL is conformant, and throws an {@link + * DataUrlException} if the URL is invalid or can't be parsed. + * + * <p>Note that this method requires the data URL to be encoded in base64, and assumes the data is + * binary. Data URLs may be text/plain data, but that is not supported by this method. + * + * @param url the data url to decode + */ + static DataUrl parseFromString(String url) throws DataUrlException { + if (!url.startsWith("data:")) { + throw new DataUrlException("URL doesn't have the data scheme"); + } + + int commaPos = url.indexOf(','); + if (commaPos == -1) { + throw new DataUrlException("Comma not found in data URL"); + } + String data = url.substring(commaPos + 1); + List<String> options = SEMICOLON_SPLITTER.splitToList(url.substring(5, commaPos)); + if (options.size() < 2) { + throw new DataUrlException("Insufficient number of options for data URL"); + } + String mimeType = options.get(0); + String encoding = options.get(1); + if (!encoding.equals("base64")) { + throw new DataUrlException("Invalid encoding: " + encoding + ", only base64 is supported"); + } + + try { + return new DataUrl(BASE64_URL.decode(data), mimeType); + } catch (IllegalArgumentException unused) { + // If the URL-safe base64 encoding doesn't work, try the vanilla base64 encoding. Ideally + // users would only use web-safe data URIs, but in many environments regular base64 payloads + // can be used without issue. + try { + return new DataUrl(BASE64.decode(data), mimeType); + } catch (IllegalArgumentException e) { + throw new DataUrlException("Invalid base64 payload in data URL", e); + } + } + } +} diff --git a/src/main/java/com/google/android/downloader/DataUrlEngine.java b/src/main/java/com/google/android/downloader/DataUrlEngine.java new file mode 100644 index 0000000..fec55aa --- /dev/null +++ b/src/main/java/com/google/android/downloader/DataUrlEngine.java @@ -0,0 +1,134 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import com.google.android.downloader.DataUrl.DataUrlException; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.net.HttpHeaders; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.nio.ByteBuffer; +import java.nio.channels.WritableByteChannel; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** {@link UrlEngine} implementation for handling data URLs. */ +public final class DataUrlEngine implements UrlEngine { + private static final ImmutableSet<String> DATA_SCHEME = ImmutableSet.of("data"); + + private final ListeningExecutorService transferExecutorService; + + public DataUrlEngine(ListeningExecutorService transferExecutorService) { + this.transferExecutorService = transferExecutorService; + } + + @Override + public UrlRequest.Builder createRequest(String url) { + return new DataUrlRequestBuilder(url); + } + + @Override + public Set<String> supportedSchemes() { + return DATA_SCHEME; + } + + /** Implementation of {@link UrlRequest.Builder} for Data URLs. */ + class DataUrlRequestBuilder implements UrlRequest.Builder { + private final String url; + + DataUrlRequestBuilder(String url) { + this.url = url; + } + + @Override + public UrlRequest build() { + return new DataUrlRequest(url); + } + } + + /** + * Implementation of {@link UrlRequest} for data URLs. Note that this is pretty trivial because no + * network connection has to happen. There's also no meaningful way to cancel a data URL request, + * so cancelling the future returned by {@link #send} has no effect on processing the data URL. + */ + class DataUrlRequest implements UrlRequest { + private final String url; + + DataUrlRequest(String url) { + this.url = url; + } + + @Override + public ListenableFuture<UrlResponse> send() { + return transferExecutorService.submit( + () -> { + try { + return new DataUrlResponse(DataUrl.parseFromString(url)); + } catch (DataUrlException e) { + throw new RequestException(e); + } + }); + } + } + + /** + * Implementation of {@link DataUrlResponse} for data URLs. Emulates network-specific APIs for + * data URLs, which means that this always return HTTP_OK/200 as a response code, and an empty set + * of response headers. + * + * <p>The response body is written by decoding the data URL string and writing the decoded bytes + * to the destination channel. + */ + class DataUrlResponse implements UrlResponse { + private final DataUrl dataUrl; + + DataUrlResponse(DataUrl dataUrl) { + this.dataUrl = dataUrl; + } + + @Override + public int getResponseCode() { + return HttpURLConnection.HTTP_OK; + } + + @Override + public Map<String, List<String>> getResponseHeaders() { + return ImmutableMap.of(HttpHeaders.CONTENT_TYPE, ImmutableList.of(dataUrl.mimeType())); + } + + @Override + public ListenableFuture<Long> readResponseBody(WritableByteChannel destinationChannel) { + IOUtil.validateChannel(destinationChannel); + return transferExecutorService.submit( + () -> { + try { + return IOUtil.blockingWrite(ByteBuffer.wrap(dataUrl.data()), destinationChannel); + } catch (IOException e) { + throw new RequestException(e); + } + }); + } + + @Override + public void close() throws IOException { + // Nothing to close for a data URL response. + } + } +} diff --git a/src/main/java/com/google/android/downloader/DownloadConstraints.java b/src/main/java/com/google/android/downloader/DownloadConstraints.java new file mode 100644 index 0000000..74b1a72 --- /dev/null +++ b/src/main/java/com/google/android/downloader/DownloadConstraints.java @@ -0,0 +1,122 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableSet; +import java.util.Set; + +/** + * Possible constraints that a download request requires. Primarily used by {@link + * ConnectivityHandler} to determine if sufficient connectivity is available. + */ +@AutoValue +public abstract class DownloadConstraints { + /** + * Special value of {@code DownloadConstraints}. If this value is specified, then the {@link + * ConnectivityHandler} short-circuits and no constraint checks are performed, even if no network + * is present at all! + */ + public static final DownloadConstraints NONE = + DownloadConstraints.builder() + // Note: Can't use EnumSet because it relies on reflection over the enum class, which + // breaks unless the right proguard configuration is applied. + .setRequiredNetworkTypes(ImmutableSet.of()) + .setRequireUnmeteredNetwork(false) + .build(); + + /** + * Common value to indicate that the active network must be unmetered. This value permits any + * network type as long as it doesn't indicate it is metered in some way. + */ + public static final DownloadConstraints NETWORK_UNMETERED = + DownloadConstraints.builder() + // Note: Can't use EnumSet because it relies on reflection over the enum class, which + // breaks unless the right proguard configuration is applied. + .setRequiredNetworkTypes(ImmutableSet.of(NetworkType.ANY)) + .setRequireUnmeteredNetwork(true) + .build(); + + /** + * Common value to indicate that the required network must simply be connected in some way, and + * otherwise doesn't have any restrictions. Any network type is allowed. + * + * <p>This is the default value for download requests. + */ + public static final DownloadConstraints NETWORK_CONNECTED = + DownloadConstraints.builder() + // Note: Can't use EnumSet because it relies on reflection over the enum class, which + // breaks unless the right proguard configuration is applied. + .setRequiredNetworkTypes(ImmutableSet.of(NetworkType.ANY)) + .setRequireUnmeteredNetwork(false) + .build(); + + /** + * The type of network that is required. This is a subset of the network types enumerated by + * {@link android.net.ConnectivityManager}. + */ + public enum NetworkType { + /** Special network type to allow any type of network, even if it not one of the known types. */ + ANY, + /** Equivalent to {@link android.net.NetworkCapabilities#TRANSPORT_BLUETOOTH} */ + BLUETOOTH, + /** Equivalent to {@link android.net.NetworkCapabilities#TRANSPORT_ETHERNET} */ + ETHERNET, + /** Equivalent to {@link android.net.NetworkCapabilities#TRANSPORT_CELLULAR} */ + CELLULAR, + /** Equivalent to {@link android.net.NetworkCapabilities#TRANSPORT_WIFI} */ + WIFI, + } + + /** + * Whether the connection must be unmetered to pass connectivity checks. See {@link + * androidx.core.net.ConnectivityManagerCompat#isActiveNetworkMetered} for more details on this + * variable. + * + * <p>False by default. + */ + public abstract boolean requireUnmeteredNetwork(); + + /** + * The types of networks that are allowed for the request to pass connectivity checks. The + * currently active network type must be one of the values in this set. This set may not be empty. + */ + public abstract ImmutableSet<NetworkType> requiredNetworkTypes(); + + /** Creates a {@code DownloadConstraints.Builder} instance. */ + public static Builder builder() { + return new AutoValue_DownloadConstraints.Builder().setRequireUnmeteredNetwork(false); + } + + /** Converts this instance to a builder for modifications. */ + public abstract Builder toBuilder(); + + /** Builder for creating instances of {@link DownloadConstraints}. */ + @AutoValue.Builder + public abstract static class Builder { + public abstract Builder setRequiredNetworkTypes(Set<NetworkType> networkTypes); + + abstract ImmutableSet.Builder<NetworkType> requiredNetworkTypesBuilder(); + + public Builder addRequiredNetworkType(NetworkType networkType) { + requiredNetworkTypesBuilder().add(networkType); + return this; + } + + public abstract Builder setRequireUnmeteredNetwork(boolean requireUnmeteredNetwork); + + public abstract DownloadConstraints build(); + } +} diff --git a/src/main/java/com/google/android/downloader/DownloadDestination.java b/src/main/java/com/google/android/downloader/DownloadDestination.java new file mode 100644 index 0000000..462b8c7 --- /dev/null +++ b/src/main/java/com/google/android/downloader/DownloadDestination.java @@ -0,0 +1,71 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import java.io.IOException; +import java.nio.channels.WritableByteChannel; + +/** + * An abstract "sink" for the stream of bytes produced by a download. A common implementation is + * {@link ProtoFileDownloadDestination}, which streams the downloaded bytes to a writeable file on + * disk, and serializes {@link DownloadMetadata} as a protocol buffer. + * + * <p>Persistent destinations may support partial download resumption by implementing {@link + * #numExistingBytes} to return a non-zero value. + */ +public interface DownloadDestination { + /** + * How many bytes are already present for the destination. Used to avoid re-downloading data by + * using HTTP range downloads. + */ + long numExistingBytes() throws IOException; + + /** + * Reads metadata for the destination. The metadata is used to set request headers as appropriate + * to support range downloads and content-aware requests. + * + * <p>Note that this operation likely needs to read persistent state from disk to retrieve the + * metadata (e.g. a database read). An {@link IOException} may be thrown if an error is + * encountered. This method will always be called on the {@link java.util.concurrent.Executor} + * that is supplied to the downloader via {@link Downloader.Builder#withIOExecutor}. + */ + DownloadMetadata readMetadata() throws IOException; + + /** + * Opens the byte channel to write the download data to, with server-provided metadata to control + * how the destination should be initialized and stored. The downloader will close the channel + * when the download is complete. + * + * <p>The returned {@link WritableByteChannel} must have its {@code position} set to {@code + * byteOffset}. The underlying storage mechanism should also persist the metadata message, so that + * future calls to {@link #readMetadata} represent those values. Failure to do so may result in + * data corruption. + * + * @param byteOffset initial byte position for the returned channel. + * @param metadata metadata to configure the destination with + * @throws IllegalArgumentException if {@code byteOffset} is outside of range [0, {@link + * #numExistingBytes}] + */ + WritableByteChannel openByteChannel(long byteOffset, DownloadMetadata metadata) + throws IOException; + + /** + * Erases any existing bytes stored at this destination. + * + * <p>Implementations can either invalidate any channels previously returned by {@link + * #openByteChannel} or have them silently drop subsequent writes. + */ + void clear() throws IOException; +} diff --git a/src/main/java/com/google/android/downloader/DownloadException.java b/src/main/java/com/google/android/downloader/DownloadException.java new file mode 100644 index 0000000..f24172e --- /dev/null +++ b/src/main/java/com/google/android/downloader/DownloadException.java @@ -0,0 +1,36 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import javax.annotation.Nullable; + +/** Exception type for errors that can be returned by the downloader library. */ +public class DownloadException extends Exception { + public DownloadException() { + super(); + } + + public DownloadException(String message) { + super(message); + } + + public DownloadException(String message, @Nullable Throwable cause) { + super(message, cause); + } + + public DownloadException(@Nullable Throwable cause) { + super(cause); + } +} diff --git a/src/main/java/com/google/android/downloader/DownloadMetadata.java b/src/main/java/com/google/android/downloader/DownloadMetadata.java new file mode 100644 index 0000000..1837d57 --- /dev/null +++ b/src/main/java/com/google/android/downloader/DownloadMetadata.java @@ -0,0 +1,48 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import com.google.auto.value.AutoValue; + +/** + * Value interface for representing download metadata in-memory. Note that this is generalized so + * that the exact serialization mechanism used by a {@link DownloadDestination} implementation can + * be hidden from the downloader, as there are contexts where for example using a protocol buffer + * isn't desirable for dependency purposes. + */ +@AutoValue +public abstract class DownloadMetadata { + /** The HTTP content tag of the download, or the empty string if none was returned. */ + abstract String getContentTag(); + + /** + * The last modification timestamp of the download, in seconds since the UNIX epoch, or 0 if none + * was returned. + */ + abstract long getLastModifiedTimeSeconds(); + + /** Creates an empty instance of {@link DownloadMetadata}. */ + public static DownloadMetadata create() { + return new AutoValue_DownloadMetadata("", 0L); + } + + /** + * Creates an instance of {@link DownloadMetadata} with the provided content tag and modified + * timestamp. + */ + public static DownloadMetadata create(String contentTag, long lastModifiedTimeSeconds) { + return new AutoValue_DownloadMetadata(contentTag, lastModifiedTimeSeconds); + } +} diff --git a/src/main/java/com/google/android/downloader/DownloadRequest.java b/src/main/java/com/google/android/downloader/DownloadRequest.java new file mode 100644 index 0000000..d225b0d --- /dev/null +++ b/src/main/java/com/google/android/downloader/DownloadRequest.java @@ -0,0 +1,110 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import com.google.auto.value.AutoValue; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableMultimap; +import java.net.URI; +import javax.annotation.Nullable; + +/** + * Buildable value class used to construct and represent an individual download request. Instances + * are created via {@link Builder}, which in turn can be obtained via {@link + * Downloader2#newRequestBuilder}. Requests are executed via {@link Downloader2#execute}. + */ +@AutoValue +public abstract class DownloadRequest { + public abstract URI uri(); + + public abstract ImmutableMultimap<String, String> headers(); + + public abstract DownloadConstraints downloadConstraints(); + + public abstract @Nullable OAuthTokenProvider oAuthTokenProvider(); + + public abstract DownloadDestination destination(); + + @VisibleForTesting // Package-private outside of testing use + public static Builder newBuilder() { + return new AutoValue_DownloadRequest.Builder(); + } + + /** Builder facility for constructing instances of an {@link DownloadRequest}. */ + @AutoValue.Builder + public abstract static class Builder { + /** Sets the {@link DownloadDestination} that the download result should be sent to. */ + public abstract Builder setDestination(DownloadDestination destination); + + /** + * Sets the {@link URI} to download from. The URI scheme returned by {@link URI#getScheme} must + * be supported by one of the {@link com.google.android.libraries.net.urlengine.UrlEngine} + * instances attached to the downloader. + */ + public abstract Builder setUri(URI uri); + + /** + * Sets the URL to download from. This must be a valid URL string per RFC 2396; invalid strings + * will result in an {@link IllegalArgumentException} to be thrown from this method. + * + * <p>Internally this has the same effect as calling {@code setUri(URI.create(string))}, and + * exists here as a convenience method. + */ + public Builder setUrl(String url) { + return setUri(URI.create(url)); + } + + abstract ImmutableMultimap.Builder<String, String> headersBuilder(); + + /** + * Adds a request header to be sent as part of this request. Request headers are only used in + * {@link com.google.android.libraries.net.urlengine.UrlEngine} instances where they make sense + * (e.g. headers are attached to HTTP requests but ignored for Data URIs). + */ + public Builder addHeader(String key, String value) { + headersBuilder().put(key, value); + return this; + } + + /** + * Specifies the {@link com.google.android.libraries.net.downloader2.DownloadConstraints} for + * this request. + * + * <p>Note that the checking the download constraints for a request only occurs right before the + * request is executed, including cases where the request is retried due to a network error. + * + * <p>The ability to detect changes in connectivity depends on the underlying implementation of + * the {@link com.google.android.libraries.net.urlengine.UrlEngine} that processes this request. + * Some network stacks may be able to switch network types without interrupting the connection + * (e.g. seamless hand-off between WiFI and Cellular). The downloader relies on an interrupted + * connection to detect network changes, so in those cases the download may silently continue on + * the changed network. + * + * <p>The default value is {@link + * com.google.android.libraries.net.downloader2.DownloadConstraints#NETWORK_CONNECTED}. + */ + public abstract Builder setDownloadConstraints(DownloadConstraints downloadConstraints); + + /** + * Sets an optional {@link OAuthTokenProvider} on the request. When set, the downloader will + * consult the OAuthTokenProvider on this request and use it to attach "Authorization" headers + * to outgoing HTTP requests. + */ + public abstract Builder setOAuthTokenProvider( + @Nullable OAuthTokenProvider oAuthTokenProvider); + + public abstract DownloadRequest build(); + } +} diff --git a/src/main/java/com/google/android/downloader/DownloadResult.java b/src/main/java/com/google/android/downloader/DownloadResult.java new file mode 100644 index 0000000..67bed12 --- /dev/null +++ b/src/main/java/com/google/android/downloader/DownloadResult.java @@ -0,0 +1,38 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import com.google.auto.value.AutoValue; +import com.google.common.annotations.VisibleForTesting; + +/** + * Value representing the result of a {@link DownloadRequest}. Contains information about the + * completed download. + * + * <p>Note that this is only used to represent a successful download. A failed download will be + * represented as a failed {@link java.util.concurrent.Future} as returned by {@link + * Downloader#execute}, with the exception containing more information about the failure. + */ +@AutoValue +public abstract class DownloadResult { + + /** Returns the number of bytes written to this result. */ + public abstract long bytesWritten(); + + @VisibleForTesting // Package-private outside of testing use + public static DownloadResult create(long bytesWritten) { + return new AutoValue_DownloadResult(bytesWritten); + } +} diff --git a/src/main/java/com/google/android/downloader/Downloader.java b/src/main/java/com/google/android/downloader/Downloader.java new file mode 100644 index 0000000..c94c9ef --- /dev/null +++ b/src/main/java/com/google/android/downloader/Downloader.java @@ -0,0 +1,885 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.base.Predicates.instanceOf; +import static com.google.common.base.Throwables.getCausalChain; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; + +import com.google.auto.value.AutoValue; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.net.HttpHeaders; +import com.google.common.util.concurrent.ClosingFuture; +import com.google.common.util.concurrent.FluentFuture; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListenableFutureTask; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.errorprone.annotations.CheckReturnValue; +import com.google.errorprone.annotations.FormatMethod; +import com.google.errorprone.annotations.FormatString; +import com.google.errorprone.annotations.concurrent.GuardedBy; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URI; +import java.nio.channels.WritableByteChannel; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Queue; +import java.util.TimeZone; +import java.util.concurrent.CancellationException; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeoutException; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import javax.annotation.Nullable; + +/** + * In-process download system. Provides a light-weight mechanism to download resources over the + * network. Downloader includes the following features: + * + * <ul> + * <li>Configurable choice of network stack, with several defaults available out of the box. + * <li>Fully asynchronous behavior, allowing downloads to progress and complete without blocking + * (assuming the underlying network stack can avoid blocking). + * <li>Support for HTTP range requests, allowing partial downloads to resume mid-way through, and + * avoiding redownloading of fully-downloaded requests. + * <li>Detection of network interruptions, and a configurable way to retry requests that have + * failed after losing connectivity. + * </ul> + * + * <p>Note that because this library performs downloads in-process, it is subject to having + * downloads stall or abort when the app is suspended or killed. Download requests only live in + * memory, and thus are lost when the process ends. Library users should persist the set of + * downloads to be executed in persistent storage, such as a SQLite database, and run the downloads + * in the context of a persistent operation (either a foreground service or via Android's {@link + * android.app.job.JobScheduler} mechanism). + * + * <p>This is intended as a functional (but not drop-in) replacement for Android's {@link + * android.app.DownloadManager}, which has a number of issues: + * + * <ul> + * <li>It relies on the network stack built into Android, and thus cannot be patched to receive + * updates. For older versions of Android, that means the DownloadManager is vulnerable to + * issues in the network stack, such as b/18432707, which can result in MITM attacks. + * <li>When downloading to external storage on older versions of Android (via {@link + * android.app.DownloadManager.Request#setDestinationInExternalFilesDir} or {@link + * android.app.DownloadManager.Request#setDestinationInExternalPublicDir}), downloads may + * exceed the maximum size of the download directory (see b/22605830). + * <li>Downloads may pause and never resume again without external interference (b/18110151) + * <li>The DownloadManager may lose track of some files (b/18265497) + * </ul> + * + * <p>This library mitigates these issues by performing downloads in-process over the app-provided + * network stack, thus ensuring that an up-to-date network stack is used, and handing off storage + * management directly to the app without any additional restrictions. + */ +public class Downloader { + @VisibleForTesting static final int HTTP_RANGE_NOT_SATISFIABLE = 416; + @VisibleForTesting static final int HTTP_PARTIAL_CONTENT = 206; + + private static final ImmutableSet<String> SCHEMES_REQUIRING_CONNECTIVITY = + ImmutableSet.of("http", "https"); + + private static final Pattern CONTENT_RANGE_HEADER_PATTERN = + Pattern.compile("bytes (\\d+)-(\\d+)/(\\d+|\\*)"); + + @GuardedBy("SIMPLE_DATE_FORMAT_LOCK") + private static final SimpleDateFormat RFC_1123_FORMATTER; + + private static final Object SIMPLE_DATE_FORMAT_LOCK = new Object(); + + static { + synchronized (SIMPLE_DATE_FORMAT_LOCK) { + RFC_1123_FORMATTER = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss 'GMT'", Locale.US); + RFC_1123_FORMATTER.setTimeZone(TimeZone.getTimeZone("UTC")); + } + } + + /** Builder for configuring and constructing an instance of the Downloader. */ + public static class Builder { + private final Map<String, UrlEngine> urlEngineMap = new HashMap<>(); + private Executor ioExecutor; + private DownloaderLogger logger; + private ConnectivityHandler connectivityHandler; + private int maxConcurrentDownloads = 3; + + /** + * Creates a downloader builder. + * + * <p>Note that all parameters are required except for {@link #withMaxConcurrentDownloads}. + */ + public Builder() {} + + /** + * Specifies the executor to use internally for I/O. + * + * <p>I/O operations can block so don't use a direct executor or one that runs on the main + * thread. + */ + public Builder withIOExecutor(Executor ioExecutor) { + this.ioExecutor = ioExecutor; + return this; + } + + /** + * Sets the {@link ConnectivityHandler} to use in order to determine if connectivity is + * sufficient, and if not, how to handle it. + */ + public Builder withConnectivityHandler(ConnectivityHandler connectivityHandler) { + this.connectivityHandler = connectivityHandler; + return this; + } + + /** + * Limits the number of downloads in flight at a time. If a download request arrives that would + * exceed this limit, it will be queued until one already in flight completes. Beware that a + * download that is waiting for connectivity requirements is still considered to be in flight, + * so it is possible to saturate the downloader with requests waiting on connectivity + * requirements if the number of concurrent downloads isn't set high enough. + * + * <p>Note that other factors might restrict download concurrency even further, for instance the + * number of threads on the I/O executor when using a blocking engine. + */ + public Builder withMaxConcurrentDownloads(int maxConcurrentDownloads) { + checkArgument(maxConcurrentDownloads > 0); + this.maxConcurrentDownloads = maxConcurrentDownloads; + return this; + } + + /** + * Adds an {@link UrlEngine} to handle network requests for the given URL scheme. Note that the + * engine must support the provided scheme, and only one engine may ever be registered for a + * specific URL scheme. An {@link IllegalArgumentException} will be thrown if the engine doesn't + * support the scheme or if an engine is already registered for the scheme. + */ + public Builder addUrlEngine(String scheme, UrlEngine urlEngine) { + checkArgument( + urlEngine.supportedSchemes().contains(scheme), + "Provided UrlEngine must support URL scheme: %s", + scheme); + checkArgument( + !urlEngineMap.containsKey(scheme), + "Requested scheme already has a UrlEngine registered: %s", + scheme); + urlEngineMap.put(scheme, urlEngine); + return this; + } + + /** + * Adds an {@link UrlEngine} to handle network requests for the given URL schemes. Note that the + * engine must support the provided schemes, and only one engine may ever be registered for a + * specific URL scheme. An error will be thrown if the engine doesn't support the schemes or if + * an engine is already registered for the schemes. + */ + public Builder addUrlEngine(Iterable<String> schemes, UrlEngine urlEngine) { + for (String scheme : schemes) { + addUrlEngine(scheme, urlEngine); + } + return this; + } + + /** Sets the {@link DownloaderLogger} to use for logging in this downloader instance. */ + public Builder withLogger(DownloaderLogger logger) { + this.logger = logger; + return this; + } + + public Downloader build() { + return new Downloader(this); + } + } + + /** + * Value class for capturing a state snapshot of the downloader, for use in the state callbacks + * that can be registered via {@link #registerStateChangeCallback}. + */ + @AutoValue + public abstract static class State { + /** + * Returns the current number of downloads currently in flight, i.e. the number of downloads + * that are concurrently executing, or attempting to make progress on their underlying network + * stack. Note that this number should be in the range of [0, maxConcurrentDownloads), as + * configured by {@link Builder#withMaxConcurrentDownloads}. + */ + public abstract int getNumDownloadsInFlight(); + + /** + * Returns the number of downloads that have been requested via {@link #execute} but not yet + * started. They may not have been started due to internal asynchronous code, due to waiting on + * connectivity requirements, or due to the limit enforced by {@code maxConcurrentDownloads}. + */ + public abstract int getNumQueuedDownloads(); + + /** + * Returns the current number of downloads that are waiting for sufficient connectivity + * conditions to be met. These downloads will start running once the device network conditions + * change (e.g. connects to WiFi), if there is enough space as determined by {@code + * maxConcurrentDownloads}. + * + * <p>Note that this number should be in the range of [0, numQueuedDownloads], as a download + * pending connectivity is necessarily queued. However, a download may also be queued due to + * {@code maxConcurrentDownloads}. + */ + public abstract int getNumDownloadsPendingConnectivity(); + + /** Creates an instance of the state object. Internal-only. */ + @VisibleForTesting + public static State create( + int numDownloadsInFlight, int numQueuedDownloads, int numDownloadsPendingConnectivity) { + return new AutoValue_Downloader_State( + numDownloadsInFlight, numQueuedDownloads, numDownloadsPendingConnectivity); + } + } + + /** + * Functional callback interface for observing changes to Downloader {@link State}. Used with + * {@link #registerStateChangeCallback} and {@link #unregisterStateChangeCallback}. + * + * <p>Note that the callbacks could just use {@code java.util.function.Consumer} in contexts that + * support the Java 8 SDK, and {@code com.google.common.base.Receiver} in contexts that don't but + * have access to Guava APIs that aren't made public. + */ + public interface StateChangeCallback { + void onStateChange(State state); + } + + private final ImmutableMap<String, UrlEngine> urlEngineMap; + private final Executor ioExecutor; + private final DownloaderLogger logger; + private final ConnectivityHandler connectivityHandler; + private final int maxConcurrentDownloads; + + @GuardedBy("lock") + private final IdentityHashMap<StateChangeCallback, Executor> stateCallbackMap = + new IdentityHashMap<>(); + + // TODO: Consider using a PriorityQueue here instead, so that queued downloads can + // retain the order in the queue as they get added/removed due to waiting on connectivity. + @GuardedBy("lock") + private final Queue<QueuedDownload> queuedDownloads = new ArrayDeque<>(); + + @GuardedBy("lock") + private final List<FluentFuture<DownloadResult>> unresolvedFutures = new ArrayList<>(); + + private final Object lock = new Object(); + + @GuardedBy("lock") + private int numDownloadsInFlight = 0; + + @GuardedBy("lock") + private int numDownloadsPendingConnectivity = 0; + + private Downloader(Builder builder) { + ImmutableMap<String, UrlEngine> urlEngineMap = ImmutableMap.copyOf(builder.urlEngineMap); + checkArgument(!urlEngineMap.isEmpty(), "Must have at least one UrlEngine"); + checkArgument(builder.ioExecutor != null, "Must set a callback executor"); + checkArgument(builder.logger != null, "Must set a logger"); + checkArgument(builder.connectivityHandler != null, "Must set a connectivity handler"); + + this.urlEngineMap = urlEngineMap; + this.ioExecutor = builder.ioExecutor; + this.logger = builder.logger; + this.connectivityHandler = builder.connectivityHandler; + this.maxConcurrentDownloads = builder.maxConcurrentDownloads; + } + + /** + * Creates a new {@link DownloadRequest.Builder} for the given {@link URI} and {@link + * DownloadDestination}. The builder may be used to further customize the request. To execute the + * request, pass the built request to {@link #execute}. + */ + @CheckReturnValue + public DownloadRequest.Builder newRequestBuilder(URI uri, DownloadDestination destination) { + return DownloadRequest.newBuilder() + .setDestination(destination) + .setUri(uri) + .setDownloadConstraints(DownloadConstraints.NETWORK_CONNECTED); + } + + /** + * Executes the provided request. The request will be handled by the underlying {@link UrlEngine} + * and the result is streamed to the {@link DownloadDestination} created for the request. The + * download result is provided asynchronously as a {@link FluentFuture} that resolves when the + * download is complete. + * + * <p>The download can be cancelled by calling {@link FluentFuture#cancel} on the future instance + * returned by this method. Cancellation is best-effort and does not guarantee that the download + * will stop immediately, as it is impossible to stop a thread that is in the middle of reading + * bytes off the network. + * + * <p>Note that this method is not idempotent! The downloader does not attempt to merge/de-dupe + * incoming requests, even if the same exact request is executed twice. The calling code needs to + * be careful to manage its downloads in such a way that duplicated downloads don't occur. + * + * <p>TODO: Document what types of exceptions can be set on the returned future, and how clients + * are expected to handle them. + */ + @CheckReturnValue + public FluentFuture<DownloadResult> execute(DownloadRequest request) { + FluentFuture<DownloadResult> resultFuture; + + synchronized (lock) { + ClosingFuture<DownloadResult> closingFuture = enqueueRequest(request); + closingFuture.statusFuture().addListener(() -> onDownloadComplete(request), ioExecutor); + + resultFuture = closingFuture.finishToFuture(); + unresolvedFutures.add(resultFuture); + resultFuture.addListener( + () -> { + synchronized (lock) { + unresolvedFutures.remove(resultFuture); + } + }, + MoreExecutors.directExecutor()); + } + + logger.logFine("New request enqueued, running queue: %s", request.uri()); + maybeRunQueuedDownloads(); + return resultFuture; + } + + /** + * Cancels <strong>all</strong> ongoing downloads. This will result in all unresolved {@link + * FluentFuture} instances returned by {@link #execute} to be cancelled immediately and have their + * error callbacks invoked. + * + * <p>Because implementations of {@link FluentFuture} allow callbacks to be garbage collected + * after the future is resolved, calling {@code cancelAll} is an effective way to avoid having the + * Downloader leak memory after it is logically no longer needed, e.g. if it is only used from an + * Android {@link android.app.Activity}, and that Activity is destroyed. + * + * <p>However, in general the Downloader should be run from a process-level context, e.g. in an + * Android {@link android.app.Service}, so that the downloader doesn't implicitly hold on to + * UI-scoped objects. + */ + public void cancelAll() { + List<FluentFuture<DownloadResult>> unresolvedFuturesCopy; + + synchronized (lock) { + // Copy the set of unresolved futures to a local variable to avoid hitting + // ConcurrentModificationExceptions, which could happen since canceling the future may + // trigger the future callback in execute() that removes the future from the set of + // unresolved futures. + unresolvedFuturesCopy = new ArrayList<>(unresolvedFutures); + unresolvedFutures.clear(); + } + + for (FluentFuture<DownloadResult> unresolvedFuture : unresolvedFuturesCopy) { + unresolvedFuture.cancel(true); + } + } + + /** + * Registers an {@link StateChangeCallback} with this downloader instance, to be run when various + * state changes occur. The callback will be executed on the provided {@link Executor}. Use {@link + * #unregisterStateChangeCallback} to unregister a previously registered callback. The callback is + * invoked when the following state transitions occur: + * + * <ul> + * <li>A download was requested via a call to {@link #execute} + * <li>A download completed + * <li>A download started waiting for its connectivity constraints to be satisfied + * <li>A download stopped waiting for its connectivity constraints to be satisfied + * </ul> + * + * <p>A newly registered callback instance will only called for state changes that are triggered + * after the registration; there is no mechanism to replay previous state changes on the callback. + * + * <p>Registering the same callback multiple times has no effect, except that it will overwrite + * the {@link Executor} that was used in a previous registration call. + * + * <p>Invocation of the callbacks will be internally serialized to avoid concurrent invocations of + * the callback with possibly conflicting state. A new invocation of the callback will not start + * running until the previous one has completed. If multiple callbacks are being registered that + * must be synchronized with each other, then the caller must take care to coordinate this + * externally, such as locking in the callbacks or using an executor that guarantees + * serialization, such as {@link MoreExecutors#newSequentialExecutor}. + * + * <p>Warning: Do not use {@link MoreExecutors#directExecutor} or similar executor mechanisms, as + * doing so can easily lead to a deadlock, since the internal downloader lock is held while + * scheduling the callbacks on the provided executor. + */ + public void registerStateChangeCallback(StateChangeCallback callback, Executor executor) { + synchronized (lock) { + stateCallbackMap.put(callback, MoreExecutors.newSequentialExecutor(executor)); + } + } + + /** + * Unregisters a previously registered {@link StateChangeCallback} with this downloader instance. + * + * <p>Attempting to unregister a callback that was never registered is a no-op. + */ + public void unregisterStateChangeCallback(StateChangeCallback callback) { + synchronized (lock) { + stateCallbackMap.remove(callback); + } + } + + private void onDownloadComplete(DownloadRequest request) { + synchronized (lock) { + // The number of downloads should be well-balanced and this case should never + // trigger, so this is just a defensive check. + checkState( + numDownloadsInFlight >= 0, "Encountered < 0 downloads in flight, this shouldn't happen"); + } + + logger.logFine("Download complete, running queued downloads: %s", request.uri()); + maybeRunQueuedDownloads(); + } + + private void maybeRunQueuedDownloads() { + // Loop until we run out of download slots or out of queued downloads. It should be impossible + // for this to loop forever. Also, note that the synchronized block is inside the loop, since + // the outer loop conditions don't touch shared mutable state. + while (true) { + synchronized (lock) { + if (numDownloadsInFlight >= maxConcurrentDownloads) { + logger.logInfo("Exceeded max concurrent downloads, not running another queued request"); + return; + } + + QueuedDownload queuedDownload = queuedDownloads.poll(); + if (queuedDownload == null) { + return; + } + + DownloadRequest request = queuedDownload.request(); + ListenableFuture<Void> connectivityFuture = checkConnectivity(request); + if (connectivityFuture.isDone()) { + logger.logFine("Connectivity satisfied; running request. uri=%s", request.uri()); + numDownloadsInFlight++; + ListenableFuture<?> statusFuture = queuedDownload.resultFuture().statusFuture(); + statusFuture.addListener( + () -> { + synchronized (lock) { + numDownloadsInFlight--; + + // One less download in flight, let the state change listeners know. + runStateCallbacks(); + } + + // A download slot was just freed up, run queued downloads again. + logger.logFine( + "Queued download completed, running queued downloads: %s", request.uri()); + maybeRunQueuedDownloads(); + }, + ioExecutor); + + // A new download is about to be in flight, let state callbacks know about the + // state change. + runStateCallbacks(); + queuedDownload.task().run(); + } else { + logger.logInfo("Waiting on connectivity for request: uri=%s", request.uri()); + handlePendingConnectivity(connectivityFuture, queuedDownload); + } + } + } + } + + @GuardedBy("lock") + private void handlePendingConnectivity( + ListenableFuture<Void> connectivityFuture, QueuedDownload queuedDownload) { + // Keep track of the number of requests waiting. + numDownloadsPendingConnectivity++; + connectivityFuture.addListener( + () -> { + synchronized (lock) { + numDownloadsPendingConnectivity--; + + // Let the listeners know we are no longer waiting. + runStateCallbacks(); + } + }, + MoreExecutors.directExecutor()); + + // Let the listeners know we're waiting. + runStateCallbacks(); + + Futures.addCallback( + connectivityFuture, + new FutureCallback<Void>() { + @Override + public void onSuccess(Void result1) { + logger.logInfo("Connectivity changed, running queued requests"); + requeue(queuedDownload); + } + + @Override + public void onFailure(Throwable t) { + if (t instanceof TimeoutException) { + logger.logInfo("Timed out waiting for connectivity change"); + requeue(queuedDownload); + } else if (t instanceof CancellationException) { + logger.logFine("Connectivity future cancelled, running queued downloads"); + maybeRunQueuedDownloads(); + } else { + logger.logError(t, "Error observing connectivity changes"); + cancelAll(); + } + } + }, + ioExecutor); + + // Run state callbacks and cancel the connectivity future when the result task resolves. + // It doesn't matter if it succeeded or failed, either way it means we no longer need to wait + // for connectivity. + queuedDownload + .task() + .addListener( + () -> { + synchronized (lock) { + logger.logInfo("Queued task completed, cancelling connectivity check"); + runStateCallbacks(); + connectivityFuture.cancel(false); + } + }, + MoreExecutors.directExecutor()); + } + + private void requeue(QueuedDownload queuedDownload) { + synchronized (lock) { + addToQueue(queuedDownload); + } + + logger.logInfo( + "Requeuing download after connectivity change: %s", queuedDownload.request().uri()); + maybeRunQueuedDownloads(); + } + + private ClosingFuture<DownloadResult> enqueueRequest(DownloadRequest request) { + synchronized (lock) { + ListenableFutureTask<Void> task = ListenableFutureTask.create(() -> null); + // When the task runs (i.e. is taken off the queue and is explicitly run), all pre-flight + // checks should have been made, so at that point the request is send to the underlying + // network stack for execution. + ClosingFuture<DownloadResult> resultFuture = + ClosingFuture.from(task) + .transformAsync((closer, result) -> runRequest(request), ioExecutor); + addToQueue(QueuedDownload.create(request, task, resultFuture)); + return resultFuture; + } + } + + @GuardedBy("lock") + private void addToQueue(QueuedDownload queuedDownload) { + queuedDownloads.add(queuedDownload); + + // Make sure that when the task completes, the queued download is definitely removed + // from the queue. This is necessary to be robust in the face of cancellation, as + // canceled tasks may not get removed from the queue otherwise. + queuedDownload + .task() + .addListener( + () -> { + synchronized (lock) { + if (queuedDownloads.remove(queuedDownload)) { + // If the queued download was actually removed, update the state callbacks + // to reflect the state change. + runStateCallbacks(); + } + } + }, + MoreExecutors.directExecutor()); + + // Now that a new request is on the queue, run the state callbacks. + runStateCallbacks(); + } + + private ClosingFuture<DownloadResult> runRequest(DownloadRequest request) throws IOException { + URI uri = request.uri(); + UrlEngine urlEngine = checkNotNull(urlEngineMap.get(uri.getScheme())); + UrlRequest.Builder urlRequestBuilder = urlEngine.createRequest(uri.toString()); + for (Map.Entry<String, String> entry : request.headers().entries()) { + urlRequestBuilder.addHeader(entry.getKey(), entry.getValue()); + } + + long numExistingBytes = request.destination().numExistingBytes(); + if (numExistingBytes > 0) { + logger.logInfo( + "Existing bytes found. numExistingBytes=%d, uri=%s", numExistingBytes, request.uri()); + urlRequestBuilder.addHeader(HttpHeaders.RANGE, "bytes=" + numExistingBytes + "-"); + + DownloadMetadata destinationMetadata = request.destination().readMetadata(); + String contentTag = destinationMetadata.getContentTag(); + long lastModifiedTimeSeconds = destinationMetadata.getLastModifiedTimeSeconds(); + if (!contentTag.isEmpty()) { + urlRequestBuilder.addHeader(HttpHeaders.IF_RANGE, contentTag); + } else if (lastModifiedTimeSeconds > 0) { + urlRequestBuilder.addHeader( + HttpHeaders.IF_RANGE, formatRfc1123Date(lastModifiedTimeSeconds)); + } else { + // TODO: This should probably just clear the destination and remove the Range + // header so there's no chance of data corruption. Leaving this as-is for now to + // keep supporting range requests for offline maps. + logger.logWarning( + "Sending range request without If-Range header, due to missing destination " + + "metadata. Data corruption is possible."); + } + } + + ListenableFuture<UrlRequest> urlRequestFuture; + OAuthTokenProvider oAuthTokenProvider = request.oAuthTokenProvider(); + if (oAuthTokenProvider != null) { + urlRequestFuture = + Futures.transform( + oAuthTokenProvider.provideOAuthToken(uri), + authToken -> { + if (authToken != null) { + urlRequestBuilder.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + authToken); + } + return urlRequestBuilder.build(); + }, + ioExecutor); + } else { + urlRequestFuture = Futures.immediateFuture(urlRequestBuilder.build()); + } + + return ClosingFuture.from(urlRequestFuture) + .transform((closer, urlRequest) -> checkNotNull(urlRequest).send(), ioExecutor) + .transformAsync( + (closer, responseFuture) -> completeRequest(request, checkNotNull(responseFuture)), + ioExecutor); + } + + /** + * @param request the original request that triggered this call + * @param responseFuture the {@link UrlResponse}, provided asynchronously via a {@link + * ListenableFuture} + */ + private ClosingFuture<DownloadResult> completeRequest( + DownloadRequest request, ListenableFuture<UrlResponse> responseFuture) { + return ClosingFuture.from(responseFuture) + .transformAsync( + (closer, urlResponse) -> { + checkNotNull(urlResponse); + // We want to close the response regardless of whether we succeed or fail. + closer.eventuallyClose(urlResponse, ioExecutor); + logger.logFine( + "Got URL response, starting to read response body. uri=%s", request.uri()); + DownloadDestination destination = request.destination(); + if (request.headers().containsKey(HttpHeaders.RANGE) + && checkNotNull(urlResponse).getResponseCode() != HTTP_PARTIAL_CONTENT) { + logger.logFine("Clearing %s as our range request wasn't honored", destination); + destination.clear(); + } + WritableByteChannel byteChannel = + destination.openByteChannel( + parseResponseStartOffset(urlResponse), parseResponseMetadata(urlResponse)); + closer.eventuallyClose(byteChannel, ioExecutor); + return ClosingFuture.from(checkNotNull(urlResponse).readResponseBody(byteChannel)); + }, + ioExecutor) + .catching( + RequestException.class, + (closer, requestException) -> { + if (checkNotNull(requestException).getErrorDetails().getHttpStatusCode() + == HTTP_RANGE_NOT_SATISFIABLE) { + // This is a bit of a special edge case. Encountering this error means the server + // rejected our HTTP range request because it was outside the range of available + // bytes. This may well mean that the request was malformed (e.g. data on disk was + // corrupted and the existing file size ended up larger than what exists on the + // server). But the more common cause for this error is that the entire file was + // in fact already downloaded, so the requested range would cover 0 bytes, which + // the server interprets as not satisfiable. To mitigate that case we simply return + // a success state here by indicating 0 bytes were downloaded. + + // This isn't exactly ideal, as it means that a potential class of errors will + // go unnoticed. A better solution might find a way to distinguish between the + // common, file-already-downloaded case and the file corrupted case. This solution + // is put in place mainly to retain parity with the older downloader implementation. + return 0L; + } else { + throw new DownloadException(requestException); + } + }, + ioExecutor) + .transform( + (closer, bytesWritten) -> { + logger.logFine( + "Response body written. bytesWritten=%d, uri=%s", + checkNotNull(bytesWritten), request.uri()); + return DownloadResult.create(checkNotNull(bytesWritten)); + }, + ioExecutor) + .catchingAsync( + Exception.class, + (closer, exception) -> { + ClosingFuture<DownloadResult> result; + synchronized (lock) { + logger.logWarning( + exception, "Error reading download result. uri=%s", request.uri()); + RequestException requestException = getRequestException(exception); + if (requestException != null + && requestException.getErrorDetails().isRetryableAsIs()) { + // Retry the request by just re-enqueueing it. Note that we also need to + // call maybeRunQueuedRequest to keep the queue moving. Also, in this particular + // case we need to decrement the in-flight downloads count, as we are taking a + // previously in-flight download and putting it back in the queue without + // resolving the underlying result future. + numDownloadsInFlight--; + result = enqueueRequest(request); + } else { + throw new DownloadException(exception); + } + } + + logger.logInfo("Running queued downloads after handling request exception"); + maybeRunQueuedDownloads(); + return result; + }, + ioExecutor); + } + + @GuardedBy("lock") + private ListenableFuture<Void> checkConnectivity(DownloadRequest request) { + if (!SCHEMES_REQUIRING_CONNECTIVITY.contains(request.uri().getScheme())) { + return Futures.immediateVoidFuture(); + } + + return connectivityHandler.checkConnectivity(request.downloadConstraints()); + } + + @GuardedBy("lock") + private void runStateCallbacks() { + State state = + State.create(numDownloadsInFlight, queuedDownloads.size(), numDownloadsPendingConnectivity); + for (Map.Entry<StateChangeCallback, Executor> callbackEntry : stateCallbackMap.entrySet()) { + callbackEntry.getValue().execute(() -> callbackEntry.getKey().onStateChange(state)); + } + } + + private static String formatRfc1123Date(long unixTimeSeconds) { + synchronized (SIMPLE_DATE_FORMAT_LOCK) { + return RFC_1123_FORMATTER.format(new Date(SECONDS.toMillis(unixTimeSeconds))); + } + } + + private static long parseResponseStartOffset(UrlResponse response) throws DownloadException { + if (response.getResponseCode() != HttpURLConnection.HTTP_PARTIAL) { + return 0; + } + + List<String> contentRangeHeaders = response.getResponseHeaders().get(HttpHeaders.CONTENT_RANGE); + + checkDownloadState( + contentRangeHeaders != null && !contentRangeHeaders.isEmpty(), + "Host returned 206/PARTIAL response code but didn't provide a " + + "'Content-Range' response header"); + String contentRangeHeader = checkNotNull(contentRangeHeaders).get(0); + Matcher matcher = CONTENT_RANGE_HEADER_PATTERN.matcher(contentRangeHeader); + checkDownloadState( + matcher.matches() && matcher.groupCount() > 0, + "Content-Range response header didn't match expected pattern. " + "Was '%s', expected '%s'", + contentRangeHeader, + CONTENT_RANGE_HEADER_PATTERN.pattern()); + return Long.parseLong(checkNotNull(matcher.group(1))); + } + + private static DownloadMetadata parseResponseMetadata(UrlResponse response) + throws DownloadException { + String contentTag = parseResponseContentTag(response); + long lastModifiedTimeSeconds = parseResponseModifiedTime(response); + return DownloadMetadata.create(contentTag, lastModifiedTimeSeconds); + } + + private static long parseResponseModifiedTime(UrlResponse response) throws DownloadException { + List<String> lastModifiedHeaders = response.getResponseHeaders().get(HttpHeaders.LAST_MODIFIED); + if (lastModifiedHeaders == null || lastModifiedHeaders.isEmpty()) { + return 0L; + } + + String lastModifiedHeader = lastModifiedHeaders.get(0); + Date date; + + try { + synchronized (SIMPLE_DATE_FORMAT_LOCK) { + date = RFC_1123_FORMATTER.parse(lastModifiedHeader); + } + if (date == null) { + throw new DownloadException("Invalid Last-Modified header: " + lastModifiedHeader); + } + } catch (ParseException e) { + throw new DownloadException("Invalid Last-Modified header: " + lastModifiedHeader, e); + } + + return MILLISECONDS.toSeconds(date.getTime()); + } + + private static String parseResponseContentTag(UrlResponse response) { + List<String> contentTagHeaders = response.getResponseHeaders().get(HttpHeaders.ETAG); + if (contentTagHeaders == null || contentTagHeaders.isEmpty()) { + return ""; + } + + return contentTagHeaders.get(0); + } + + @AutoValue + abstract static class QueuedDownload { + abstract DownloadRequest request(); + + abstract ListenableFutureTask<?> task(); + + abstract ClosingFuture<DownloadResult> resultFuture(); + + static QueuedDownload create( + DownloadRequest request, + ListenableFutureTask<?> task, + ClosingFuture<DownloadResult> resultFuture) { + return new AutoValue_Downloader_QueuedDownload(request, task, resultFuture); + } + } + + @VisibleForTesting + @Nullable + static RequestException getRequestException(@Nullable Throwable throwable) { + if (throwable == null) { + return null; + } else { + return (RequestException) + Iterables.find( + getCausalChain(throwable), + instanceOf(RequestException.class), + /* defaultValue= */ null); + } + } + + @FormatMethod + private static void checkDownloadState( + boolean state, @FormatString String message, Object... args) throws DownloadException { + if (!state) { + throw new DownloadException(String.format(message, args)); + } + } +} diff --git a/src/main/java/com/google/android/downloader/DownloaderLogger.java b/src/main/java/com/google/android/downloader/DownloaderLogger.java new file mode 100644 index 0000000..cfd6f88 --- /dev/null +++ b/src/main/java/com/google/android/downloader/DownloaderLogger.java @@ -0,0 +1,43 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import com.google.errorprone.annotations.FormatMethod; +import com.google.errorprone.annotations.FormatString; +import javax.annotation.Nullable; + +/** + * Interface for a generic logging system, as used by the downloader. Allows the caller to configure + * the logging implementation as needed, and avoid extra dependencies in the downloader. + */ +public interface DownloaderLogger { + @FormatMethod + void logFine(@FormatString String message, Object... args); + + @FormatMethod + void logInfo(@FormatString String message, Object... args); + + @FormatMethod + void logWarning(@FormatString String message, Object... args); + + @FormatMethod + void logWarning(@Nullable Throwable cause, @FormatString String message, Object... args); + + @FormatMethod + void logError(@FormatString String message, Object... args); + + @FormatMethod + void logError(@Nullable Throwable cause, @FormatString String message, Object... args); +} diff --git a/src/main/java/com/google/android/downloader/ErrorDetails.java b/src/main/java/com/google/android/downloader/ErrorDetails.java new file mode 100644 index 0000000..3991b27 --- /dev/null +++ b/src/main/java/com/google/android/downloader/ErrorDetails.java @@ -0,0 +1,169 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import com.google.auto.value.AutoValue; +import com.google.common.base.Strings; +import com.google.common.net.HttpHeaders; +import java.net.HttpURLConnection; +import java.util.List; +import java.util.Map; +import javax.annotation.Nullable; + +/** Simple container object that contains information about a request error. */ +@AutoValue +public abstract class ErrorDetails { + // Additional HTTP status codes not listed in HttpURLConnection. + static final int HTTP_TOO_MANY_REQUESTS = 429; + + /** + * Returns the underlying numerical error value associated with this error. The meaning of this + * value depends on the network stack in use. Consult the network stack documentation to determine + * the meaning of this value. By default this returns the value 0. + */ + public abstract int getInternalErrorCode(); + + /** + * Returns the human-readable error message associated with this error. This message is for + * debugging purposes only and should not be parsed programmatically. By default this returns the + * empty string. + */ + public abstract String getErrorMessage(); + + /** + * Returns the HTTP status value associated with this error, if any. If the request succeeded but + * the server returned an error (e.g. server error 500), then this field can help classify the + * problem. If no HTTP status value is associated with this error, then the value -1 will be + * returned instead. + */ + public abstract int getHttpStatusCode(); + + /** + * Returns whether the request that triggered the error is retryable as-is without further changes + * to the request or state of the client. + * + * <p>Whether or not a request can be retried can depend on nuanced details of how an error + * occurred, so individual URL engines may make local decisions on whether an error should be + * retried. For example, TCP's connection reset error is often caused by a crash of the server + * during processing. Resending the exact same request, perhaps after some delay, has a good + * chance of hitting a different, healthy server and so in general this error is retryable. On the + * other hand, it doesn't make sense to spend resources retrying a HTTP_NOT_FOUND/404 since it + * isn't likely that the requested URL will spontaneously appear. A request that fails with + * HTTP_UNAUTHORIZED/401 is also not retryable under this definition. Although it makes sense to + * send an authenticated version of the failed request, this modification must happen at a higher + * layer. + */ + public abstract boolean isRetryableAsIs(); + + /** Creates a new builder instance for constructing request errors. */ + public static Builder builder() { + return new AutoValue_ErrorDetails.Builder() + .setInternalErrorCode(0) + .setErrorMessage("") + .setHttpStatusCode(-1) + .setRetryableAsIs(false); + } + + /** + * Create a new error instance for the given value and message. A convenience factory function for + * the most common use case. + */ + public static ErrorDetails create(@Nullable String message) { + return builder().setErrorMessage(Strings.nullToEmpty(message)).build(); + } + + /** + * Create a new error instance for an HTTP error response, as represented by the provided {@code + * httpResponseCode} and {@code httpResponseHeaders}. The canonical error code and retryability + * bit is computed based on the values of the response. + */ + public static ErrorDetails createFromHttpErrorResponse( + int httpResponseCode, + Map<String, List<String>> httpResponseHeaders, + @Nullable String message) { + return builder() + .setErrorMessage(Strings.nullToEmpty(message)) + .setRetryableAsIs(isRetryableHttpError(httpResponseCode, httpResponseHeaders)) + .setHttpStatusCode(httpResponseCode) + .setInternalErrorCode(httpResponseCode) + .build(); + } + + /** + * Obtains a connection error from a {@link Throwable}. If the throwable is an instance of {@link + * RequestException}, then it returns the error instance associated with that exception. + * Otherwise, a new connection error is constructed with default values and an error message set + * to the value returned by {@link Throwable#getMessage}. + */ + public static ErrorDetails fromThrowable(Throwable throwable) { + if (throwable instanceof RequestException) { + RequestException requestException = (RequestException) throwable; + return requestException.getErrorDetails(); + } else { + return builder().setErrorMessage(Strings.nullToEmpty(throwable.getMessage())).build(); + } + } + + /** + * Determine if a given HTTP error, as represented by an HTTP response code and response headers, + * is retryable. See the comment on {@link #isRetryableAsIs} for a longer explanation on how this + * related to the canonical error code. + */ + private static boolean isRetryableHttpError( + int httpCode, Map<String, List<String>> responseHeaders) { + switch (httpCode) { + case HttpURLConnection.HTTP_CLIENT_TIMEOUT: + // Client timeout means some client-side timeout was encountered. Retrying is safe. + return true; + case HttpURLConnection.HTTP_ENTITY_TOO_LARGE: + // Entity too large means the request was too large for the server to process. Retrying is + // safe if the server provided the retry-after header. + return responseHeaders.containsKey(HttpHeaders.RETRY_AFTER); + case HTTP_TOO_MANY_REQUESTS: + // Too many requests means the server is overloaded and is rejecting requests to temporarily + // reduce load. See go/rfc/6585. Retrying is safe. + return true; + case HttpURLConnection.HTTP_UNAVAILABLE: + // Unavailable means the server is currently unable to service the request. Retrying is + // safe if the server provided the retry-after header. + return responseHeaders.containsKey(HttpHeaders.RETRY_AFTER); + case HttpURLConnection.HTTP_GATEWAY_TIMEOUT: + // Gateway timeout means there was a server timeout somewhere. Retrying is safe. + return true; + default: + // By default, assume any other HTTP error is not retryable. + return false; + } + } + + /** Builder for creating instances of {@link ErrorDetails}. */ + @AutoValue.Builder + public abstract static class Builder { + /** Sets the error value. */ + public abstract Builder setInternalErrorCode(int internalErrorCode); + + /** Sets the error message. */ + public abstract Builder setErrorMessage(String errorMessage); + + /** Sets the http status value. */ + public abstract Builder setHttpStatusCode(int httpStatusCode); + + /** Sets whether the error is retryable as-is. */ + public abstract Builder setRetryableAsIs(boolean retryable); + + /** Builds the request error instance. */ + public abstract ErrorDetails build(); + } +} diff --git a/src/main/java/com/google/android/downloader/FloggerDownloaderLogger.java b/src/main/java/com/google/android/downloader/FloggerDownloaderLogger.java new file mode 100644 index 0000000..eeb0c4e --- /dev/null +++ b/src/main/java/com/google/android/downloader/FloggerDownloaderLogger.java @@ -0,0 +1,64 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import com.google.common.flogger.GoogleLogger; +import com.google.errorprone.annotations.FormatMethod; +import com.google.errorprone.annotations.FormatString; +import javax.annotation.Nullable; + +/** + * Implementation of {@link DownloaderLogger} backed by {@link GoogleLogger}. Google-internal code + * should always use this logger. + */ +public class FloggerDownloaderLogger implements DownloaderLogger { + private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); + + @Override + @FormatMethod + public void logFine(@FormatString String message, Object... args) { + logger.atFine().logVarargs(message, args); + } + + @Override + @FormatMethod + public void logInfo(@FormatString String message, Object... args) { + logger.atInfo().logVarargs(message, args); + } + + @Override + @FormatMethod + public void logWarning(@FormatString String message, Object... args) { + logger.atWarning().logVarargs(message, args); + } + + @Override + @FormatMethod + public void logWarning(@Nullable Throwable cause, @FormatString String message, Object... args) { + logger.atWarning().withCause(cause).logVarargs(message, args); + } + + @Override + @FormatMethod + public void logError(@FormatString String message, Object... args) { + logger.atSevere().logVarargs(message, args); + } + + @Override + @FormatMethod + public void logError(@Nullable Throwable cause, @FormatString String message, Object... args) { + logger.atSevere().withCause(cause).logVarargs(message, args); + } +} diff --git a/src/main/java/com/google/android/downloader/IOUtil.java b/src/main/java/com/google/android/downloader/IOUtil.java new file mode 100644 index 0000000..1461dd5 --- /dev/null +++ b/src/main/java/com/google/android/downloader/IOUtil.java @@ -0,0 +1,62 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import static com.google.common.base.Preconditions.checkState; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.SelectableChannel; +import java.nio.channels.WritableByteChannel; + +/** Package-local utilities for facilitating common I/O operations across engine implementations. */ +final class IOUtil { + + private IOUtil() {} + + /** + * Validates the given channel as a valid byte sink for the UrlEngine library. Specifically, it + * checks that the given channel is open, and if it is a {@link SelectableChannel}, then it must + * be in blocking mode. + * + * <p>This is to guard against having a sink that refuses write operations in a non-blocking + * manner by returning from {@link WritableByteChannel#write} immediately with a return value of + * 0, indicating no bytes were written. That would be problematic for the UrlEngine library + * because then threads will spin constantly trying to stream bytes from the source to the sink, + * saturating the CPU in the process. + * + * <p>TODO: Figure out a more robust way to handle this. Maybe writing implement proper support + * for selectable channels? + */ + static void validateChannel(WritableByteChannel channel) { + if (channel instanceof SelectableChannel) { + SelectableChannel selectableChannel = (SelectableChannel) channel; + checkState( + selectableChannel.isBlocking(), + "Target channels used by UrlEngine must be in blocking mode to ensure " + + "writes happen correctly; call SelectableChannel#configureBlocking(true)."); + } + + checkState(channel.isOpen()); + } + + static long blockingWrite(ByteBuffer source, WritableByteChannel target) throws IOException { + long written = 0; + while (source.hasRemaining()) { + written += target.write(source); + } + return written; + } +} diff --git a/src/main/java/com/google/android/downloader/OAuthTokenProvider.java b/src/main/java/com/google/android/downloader/OAuthTokenProvider.java new file mode 100644 index 0000000..dc0582c --- /dev/null +++ b/src/main/java/com/google/android/downloader/OAuthTokenProvider.java @@ -0,0 +1,28 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import com.google.common.util.concurrent.ListenableFuture; +import java.net.URI; + +/** + * Provider interface for supplying OAuth tokens for a request. The tokens generated by this method + * are added to requests as http headers. For more details, see the official spec for this + * authorization mechanism: https://tools.ietf.org/html/rfc6750#page-5 + */ +public interface OAuthTokenProvider { + /** Provides an OAuth bearer token for the given URI. */ + ListenableFuture<String> provideOAuthToken(URI uri); +} diff --git a/src/main/java/com/google/android/downloader/OkHttp2UrlEngine.java b/src/main/java/com/google/android/downloader/OkHttp2UrlEngine.java new file mode 100644 index 0000000..efbdb2e --- /dev/null +++ b/src/main/java/com/google/android/downloader/OkHttp2UrlEngine.java @@ -0,0 +1,216 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.util.concurrent.MoreExecutors.directExecutor; + +import com.google.common.collect.ImmutableMultimap; +import com.google.common.collect.ImmutableSet; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.SettableFuture; +import com.squareup.okhttp.Call; +import com.squareup.okhttp.Callback; +import com.squareup.okhttp.OkHttpClient; +import com.squareup.okhttp.Request; +import com.squareup.okhttp.Response; +import com.squareup.okhttp.ResponseBody; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.channels.Channels; +import java.nio.channels.WritableByteChannel; +import java.util.List; +import java.util.Map; +import java.util.Set; +import okio.Okio; +import okio.Sink; + +/** {@link UrlEngine} implementation that uses OkHttp3 for network connectivity. */ +public class OkHttp2UrlEngine implements UrlEngine { + private static final ImmutableSet<String> HTTP_SCHEMES = ImmutableSet.of("http", "https"); + + private final OkHttpClient client; + private final ListeningExecutorService transferExecutorService; + + /** + * Constructs an instance of the OkHttp URL engine, for the given OkHttpClient instance. + * + * <p>Note that due to how OkHttp is implemented, reads from the network are blocking operations, + * and thus threads in the provided {@link ListeningExecutorService} can be tied up for long + * periods of time waiting on network responses. To mitigate, set {@link + * OkHttpClient#setReadTimeout(long, java.util.concurrent.TimeUnit)} to a value that is reasonable + * for your use case. + * + * @param transferExecutorService Executor on which the requests are synchronously executed. + */ + public OkHttp2UrlEngine(OkHttpClient client, ListeningExecutorService transferExecutorService) { + checkNotNull(client.getDispatcher()); + this.client = client; + this.transferExecutorService = transferExecutorService; + } + + @Override + public UrlRequest.Builder createRequest(String url) { + return new OkHttpUrlRequestBuilder(url); + } + + @Override + public Set<String> supportedSchemes() { + return HTTP_SCHEMES; + } + + class OkHttpUrlRequestBuilder implements UrlRequest.Builder { + private final String url; + private final ImmutableMultimap.Builder<String, String> headers = ImmutableMultimap.builder(); + + OkHttpUrlRequestBuilder(String url) { + this.url = url; + } + + @Override + public UrlRequest.Builder addHeader(String key, String value) { + headers.put(key, value); + return this; + } + + @Override + public UrlRequest build() { + return new OkHttpUrlRequest(url, headers.build()); + } + } + + /** + * Implementation of {@link UrlRequest} for OkHttp. Wraps OkHttp's {@link Call} to make network + * requests. + */ + class OkHttpUrlRequest implements UrlRequest { + private final String url; + private final ImmutableMultimap<String, String> headers; + + OkHttpUrlRequest(String url, ImmutableMultimap<String, String> headers) { + this.url = url; + this.headers = headers; + } + + @Override + public ListenableFuture<UrlResponse> send() { + Request.Builder requestBuilder = new Request.Builder(); + + try { + requestBuilder.url(url); + } catch (IllegalArgumentException e) { + return Futures.immediateFailedFuture(new RequestException(e)); + } + + for (String key : headers.keys()) { + for (String value : headers.get(key)) { + requestBuilder.header(key, value); + } + } + + SettableFuture<UrlResponse> responseFuture = SettableFuture.create(); + Call call = client.newCall(requestBuilder.build()); + call.enqueue( + new Callback() { + @Override + public void onResponse(Response response) { + if (response.isSuccessful()) { + responseFuture.set(new OkHttpUrlResponse(response)); + } else { + responseFuture.setException( + new RequestException( + ErrorDetails.createFromHttpErrorResponse( + response.code(), response.headers().toMultimap(), response.message()))); + try { + response.body().close(); + } catch (IOException e) { + // Ignore, an exception on the future was already set. + } + } + } + + @Override + public void onFailure(Request request, IOException exception) { + responseFuture.setException(new RequestException(exception)); + } + }); + responseFuture.addListener( + () -> { + if (responseFuture.isCancelled()) { + call.cancel(); + } + }, + directExecutor()); + return responseFuture; + } + } + + /** + * Implementation of {@link UrlResponse} for OkHttp. Wraps OkHttp's {@link okhttp3.Response} to + * complete its operations. + */ + class OkHttpUrlResponse implements UrlResponse { + private final Response response; + + OkHttpUrlResponse(Response response) { + this.response = response; + } + + @Override + public int getResponseCode() { + return response.code(); + } + + @Override + public Map<String, List<String>> getResponseHeaders() { + return response.headers().toMultimap(); + } + + @Override + public ListenableFuture<Long> readResponseBody(WritableByteChannel destinationChannel) { + IOUtil.validateChannel(destinationChannel); + return transferExecutorService.submit( + () -> { + try (ResponseBody body = checkNotNull(response.body())) { + // Transfer the response body to the destination channel via OkHttp's Okio API. + // Sadly this needs to operate on OutputStream instead of Channels, but at least + // Okio manages buffers efficiently internally. + OutputStream outputStream = Channels.newOutputStream(destinationChannel); + Sink sink = Okio.sink(outputStream); + return body.source().readAll(sink); + } catch (IllegalStateException e) { + // OkHttp throws an IllegalStateException if the stream is closed while + // trying to write. Catch and rethrow. + throw new RequestException(e); + } catch (IOException e) { + if (e instanceof RequestException) { + throw e; + } else { + throw new RequestException(e); + } + } finally { + response.body().close(); + } + }); + } + + @Override + public void close() throws IOException { + response.body().close(); + } + } +} diff --git a/src/main/java/com/google/android/downloader/OkHttp3UrlEngine.java b/src/main/java/com/google/android/downloader/OkHttp3UrlEngine.java new file mode 100644 index 0000000..b51282e --- /dev/null +++ b/src/main/java/com/google/android/downloader/OkHttp3UrlEngine.java @@ -0,0 +1,212 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.util.concurrent.MoreExecutors.directExecutor; + +import com.google.common.collect.ImmutableMultimap; +import com.google.common.collect.ImmutableSet; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.SettableFuture; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.channels.Channels; +import java.nio.channels.WritableByteChannel; +import java.util.List; +import java.util.Map; +import java.util.Set; +import okhttp3.Call; +import okhttp3.Callback; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; +import okhttp3.ResponseBody; +import okio.Okio; +import okio.Sink; + +/** {@link UrlEngine} implementation that uses OkHttp3 for network connectivity. */ +public class OkHttp3UrlEngine implements UrlEngine { + private static final ImmutableSet<String> HTTP_SCHEMES = ImmutableSet.of("http", "https"); + + private final OkHttpClient client; + private final ListeningExecutorService transferExecutorService; + + /** + * Constructs an instance of the OkHttp URL engine, for the given OkHttpClient instance. + * + * <p>Note that due to how OkHttp is implemented, reads from the network are blocking operations, + * and thus threads in the provided {@link ListeningExecutorService} can be tied up for long + * periods of time waiting on network responses. To mitigate, set {@link + * OkHttpClient.Builder#readTimeout(long, java.util.concurrent.TimeUnit)} to a value that is + * reasonable for your use case. + * + * @param transferExecutorService Executor on which the requests are synchronously executed. + */ + public OkHttp3UrlEngine(OkHttpClient client, ListeningExecutorService transferExecutorService) { + checkNotNull(client.dispatcher()); + this.client = client; + this.transferExecutorService = transferExecutorService; + } + + @Override + public UrlRequest.Builder createRequest(String url) { + return new OkHttpUrlRequestBuilder(url); + } + + @Override + public Set<String> supportedSchemes() { + return HTTP_SCHEMES; + } + + class OkHttpUrlRequestBuilder implements UrlRequest.Builder { + private final String url; + private final ImmutableMultimap.Builder<String, String> headers = ImmutableMultimap.builder(); + + OkHttpUrlRequestBuilder(String url) { + this.url = url; + } + + @Override + public UrlRequest.Builder addHeader(String key, String value) { + headers.put(key, value); + return this; + } + + @Override + public UrlRequest build() { + return new OkHttpUrlRequest(url, headers.build()); + } + } + + /** + * Implementation of {@link UrlRequest} for OkHttp. Wraps OkHttp's {@link Call} to make network + * requests. + */ + class OkHttpUrlRequest implements UrlRequest { + private final String url; + private final ImmutableMultimap<String, String> headers; + + OkHttpUrlRequest(String url, ImmutableMultimap<String, String> headers) { + this.url = url; + this.headers = headers; + } + + @Override + public ListenableFuture<UrlResponse> send() { + Request.Builder requestBuilder = new Request.Builder(); + + try { + requestBuilder.url(url); + } catch (IllegalArgumentException e) { + return Futures.immediateFailedFuture(new RequestException(e)); + } + + for (String key : headers.keys()) { + for (String value : headers.get(key)) { + requestBuilder.header(key, value); + } + } + + SettableFuture<UrlResponse> responseFuture = SettableFuture.create(); + Call call = client.newCall(requestBuilder.build()); + call.enqueue( + new Callback() { + @Override + public void onResponse(Call call, Response response) { + if (response.isSuccessful()) { + responseFuture.set(new OkHttpUrlResponse(response)); + } else { + responseFuture.setException( + new RequestException( + ErrorDetails.createFromHttpErrorResponse( + response.code(), response.headers().toMultimap(), response.message()))); + response.close(); + } + } + + @Override + public void onFailure(Call call, IOException exception) { + responseFuture.setException(new RequestException(exception)); + } + }); + responseFuture.addListener( + () -> { + if (responseFuture.isCancelled()) { + call.cancel(); + } + }, + directExecutor()); + return responseFuture; + } + } + + /** + * Implementation of {@link UrlResponse} for OkHttp. Wraps OkHttp's {@link Response} to complete + * its operations. + */ + class OkHttpUrlResponse implements UrlResponse { + private final Response response; + + OkHttpUrlResponse(Response response) { + this.response = response; + } + + @Override + public int getResponseCode() { + return response.code(); + } + + @Override + public Map<String, List<String>> getResponseHeaders() { + return response.headers().toMultimap(); + } + + @Override + public ListenableFuture<Long> readResponseBody(WritableByteChannel destinationChannel) { + IOUtil.validateChannel(destinationChannel); + return transferExecutorService.submit( + () -> { + try (ResponseBody body = checkNotNull(response.body())) { + // Transfer the response body to the destination channel via OkHttp's Okio API. + // Sadly this needs to operate on OutputStream instead of Channels, but at least + // Okio manages buffers efficiently internally. + OutputStream outputStream = Channels.newOutputStream(destinationChannel); + Sink sink = Okio.sink(outputStream); + return body.source().readAll(sink); + } catch (IllegalStateException e) { + // OkHttp throws an IllegalStateException if the stream is closed while + // trying to write. Catch and rethrow. + throw new RequestException(e); + } catch (IOException e) { + if (e instanceof RequestException) { + throw e; + } else { + throw new RequestException(e); + } + } finally { + response.close(); + } + }); + } + + @Override + public void close() { + response.close(); + } + } +} diff --git a/src/main/java/com/google/android/downloader/PlatformUrlEngine.java b/src/main/java/com/google/android/downloader/PlatformUrlEngine.java new file mode 100644 index 0000000..b9357be --- /dev/null +++ b/src/main/java/com/google/android/downloader/PlatformUrlEngine.java @@ -0,0 +1,292 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableMultimap; +import com.google.common.collect.ImmutableSet; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLConnection; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.nio.channels.ReadableByteChannel; +import java.nio.channels.WritableByteChannel; +import java.util.List; +import java.util.Map; +import java.util.Set; +import javax.annotation.Nullable; +import javax.annotation.concurrent.GuardedBy; + +/** + * {@link UrlEngine} implementation that uses Java's built-in network stack (i.e. {@link + * HttpURLConnection}). + * + * <p>Note that internally this engine allocates a 512kb direct byte buffer per request to transfer + * bytes around. If memory usage is a concern, then the number of concurrent requests should be + * limited. + */ +public final class PlatformUrlEngine implements UrlEngine { + @VisibleForTesting static final int BUFFER_SIZE_BYTES = 512 * 1024; + private static final ImmutableSet<String> SCHEMES = ImmutableSet.of("http", "https", "file"); + + private final ListeningExecutorService requestExecutorService; + private final int connectTimeoutsMs; + private final int readTimeoutsMs; + + public PlatformUrlEngine( + ListeningExecutorService requestExecutorService, int connectTimeoutMs, int readTimeoutMs) { + this.requestExecutorService = requestExecutorService; + this.connectTimeoutsMs = connectTimeoutMs; + this.readTimeoutsMs = readTimeoutMs; + } + + @Override + public UrlRequest.Builder createRequest(String url) { + return new PlatformUrlRequestBuilder(url); + } + + @Override + public Set<String> supportedSchemes() { + return SCHEMES; + } + + /** Implementation of {@link UrlRequest.Builder} for the built-in network stack. */ + class PlatformUrlRequestBuilder implements UrlRequest.Builder { + private final String url; + private final ImmutableMultimap.Builder<String, String> headers = + new ImmutableMultimap.Builder<>(); + + PlatformUrlRequestBuilder(String url) { + this.url = url; + } + + @Override + public UrlRequest.Builder addHeader(String key, String value) { + headers.put(key, value); + return this; + } + + @Override + public UrlRequest build() { + return new PlatformUrlRequest(url, headers.build()); + } + } + + /** + * Implementation of {@link UrlRequest} for the platform network stack. + * + * <p>The design of this class has some nuance in its design. Because HttpURLConnection isn't + * thread-safe, this implementation holds on to a single thread for the entire duration of an + * {@link HttpURLConnection} lifecycle - from connect until disconnect. + */ + class PlatformUrlRequest implements UrlRequest { + private final String url; + private final ImmutableMultimap<String, String> headers; + + PlatformUrlRequest(String url, ImmutableMultimap<String, String> headers) { + this.url = url; + this.headers = headers; + } + + @Override + public ListenableFuture<UrlResponse> send() { + return requestExecutorService.submit( + () -> { + URL url; + + try { + url = new URL(this.url); + } catch (MalformedURLException e) { + throw new RequestException(e); + } + + throwIfCancelled(); + + URLConnection urlConnection = null; + + try { + urlConnection = url.openConnection(); + urlConnection.setConnectTimeout(connectTimeoutsMs); + urlConnection.setReadTimeout(readTimeoutsMs); + + for (String key : headers.keySet()) { + for (String value : headers.get(key)) { + urlConnection.addRequestProperty(key, value); + } + } + + throwIfCancelled(); + + urlConnection.connect(); + + throwIfCancelled(); + + int httpResponseCode = getResponseCode(urlConnection); + + throwIfCancelled(); + + // We've successfully connected, so resolve the request to let client code decide + // what to do next. + PlatformUrlResponse response = + new PlatformUrlResponse(urlConnection, httpResponseCode); + urlConnection = null; + return response; + } catch (RequestException e) { + throw e; + } catch (IOException e) { + throw new RequestException(e); + } finally { + maybeDisconnect(urlConnection); + } + }); + } + } + + /** Implementation of {@link UrlResponse} for the platform network stack. */ + class PlatformUrlResponse implements UrlResponse { + @GuardedBy("this") + @Nullable + private URLConnection urlConnection; + + private final int httpResponseCode; + private final Map<String, List<String>> responseHeaders; + + PlatformUrlResponse(URLConnection urlConnection, int httpResponseCode) { + this.urlConnection = urlConnection; + this.httpResponseCode = httpResponseCode; + this.responseHeaders = urlConnection.getHeaderFields(); + } + + @Nullable + private synchronized URLConnection consumeConnection() { + URLConnection urlConnection = this.urlConnection; + this.urlConnection = null; + return urlConnection; + } + + @Override + public int getResponseCode() { + return httpResponseCode; + } + + @Override + public Map<String, List<String>> getResponseHeaders() { + return responseHeaders; + } + + @Override + public ListenableFuture<Long> readResponseBody(WritableByteChannel destinationChannel) { + IOUtil.validateChannel(destinationChannel); + return requestExecutorService.submit( + () -> { + URLConnection urlConnection = consumeConnection(); + if (urlConnection == null) { + throw new RequestException("URLConnection already closed"); + } + + try (ReadableByteChannel sourceChannel = + Channels.newChannel(urlConnection.getInputStream())) { + ByteBuffer buffer = ByteBuffer.allocateDirect(BUFFER_SIZE_BYTES); + long written = 0; + + while (sourceChannel.read(buffer) != -1) { + throwIfCancelled(); + buffer.flip(); + written += IOUtil.blockingWrite(buffer, destinationChannel); + buffer.clear(); + throwIfCancelled(); + } + + return written; + } catch (IOException e) { + throw new RequestException(e); + } finally { + maybeDisconnect(urlConnection); + } + }); + } + + @Override + public void close() throws IOException { + URLConnection urlConnection = consumeConnection(); + if (urlConnection != null) { + // At this point, both HttpURLConnection.getResponseCode and URLConnection.getHeaderFields + // have been called, so the InputStream has already been implicitly created, and the call + // to URLConnection.getInputStream will be cheap. Normally calling it can be pretty heavy- + // weight and thus likely shouldn't happen in the close() method. + urlConnection.getInputStream().close(); + maybeDisconnect(urlConnection); + } + } + } + + private static void throwIfCancelled() throws RequestException { + // ListeningExecutorService turns Future.cancel() into Thread.interrupt() + if (Thread.interrupted()) { + throw new RequestException("Request canceled"); + } + } + + private static void maybeDisconnect(@Nullable URLConnection urlConnection) { + if (urlConnection == null) { + return; + } + + if (urlConnection instanceof HttpURLConnection) { + HttpURLConnection httpUrlConnection = (HttpURLConnection) urlConnection; + httpUrlConnection.disconnect(); + } + } + + private static int getResponseCode(URLConnection urlConnection) throws IOException { + if (urlConnection instanceof HttpURLConnection) { + HttpURLConnection httpUrlConnection = (HttpURLConnection) urlConnection; + InputStream inputStream = getInputStream(httpUrlConnection); + int httpResponseCode = httpUrlConnection.getResponseCode(); + if (httpResponseCode >= HttpURLConnection.HTTP_BAD_REQUEST) { + String responseMessage = httpUrlConnection.getResponseMessage(); + Map<String, List<String>> responseHeaders = httpUrlConnection.getHeaderFields(); + if (inputStream != null) { + inputStream.close(); + } + throw new RequestException( + ErrorDetails.createFromHttpErrorResponse( + httpResponseCode, responseHeaders, responseMessage)); + } + return httpResponseCode; + } else { + // Note: This happens for URLConnections that aren't over HTTP, e.g. to + // file URLs instead (e.g. sun.net.www.protocol.file.FileURLConnection). The + // code doesn't directly check for these classes because they aren't officially + // part of the JDK. + return HttpURLConnection.HTTP_OK; + } + } + + @Nullable + private static InputStream getInputStream(HttpURLConnection httpURLConnection) { + try { + return httpURLConnection.getInputStream(); + } catch (IOException e) { + return null; + } + } +} diff --git a/src/main/java/com/google/android/downloader/ProtoFileDownloadDestination.java b/src/main/java/com/google/android/downloader/ProtoFileDownloadDestination.java new file mode 100644 index 0000000..43b8ed0 --- /dev/null +++ b/src/main/java/com/google/android/downloader/ProtoFileDownloadDestination.java @@ -0,0 +1,97 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import static com.google.common.base.Preconditions.checkState; + +import com.google.common.io.Files; +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.channels.FileChannel; +import java.nio.channels.WritableByteChannel; + +/** + * Basic implementation of {@link DownloadDestination} which streams the download to a {@link File}. + * + * <p>This implementation also keeps track of metadata in a separate file, encoded as a protocol + * buffer message. Note that this implementation isn't especially robust - concurrent reads and + * writes could result in errors or in data corruption, and invalid/corrupt metadata will result in + * persistent errors. + */ +public class ProtoFileDownloadDestination implements DownloadDestination { + private final File targetFile; + private final File metadataFile; + + public ProtoFileDownloadDestination(File targetFile, File metadataFile) { + this.targetFile = targetFile; + this.metadataFile = metadataFile; + } + + @Override + public long numExistingBytes() throws IOException { + return targetFile.length(); + } + + @Override + public DownloadMetadata readMetadata() throws IOException { + return metadataFile.exists() + ? readMetadataFromBytes(Files.toByteArray(metadataFile)) + : DownloadMetadata.create(); + } + + @Override + public WritableByteChannel openByteChannel(long offsetBytes, DownloadMetadata metadata) + throws IOException { + checkState( + offsetBytes <= targetFile.length(), + "Opening byte channel with offset past known end of file"); + Files.write(writeMetadataToBytes(metadata), metadataFile); + FileChannel fileChannel = new RandomAccessFile(targetFile, "rw").getChannel(); + // Seek to the requested offset, so we can append data rather than overwrite data. + fileChannel.position(offsetBytes); + return fileChannel; + } + + @Override + public void clear() throws IOException { + if (targetFile.exists() && !targetFile.delete()) { + throw new IOException("Failed to delete()"); + } + } + + @Override + public String toString() { + return targetFile.toString(); + } + + private static DownloadMetadata readMetadataFromBytes(byte[] bytes) throws IOException { + DownloadMetadataProto proto = DownloadMetadataProto.parseFrom(bytes); + return DownloadMetadata.create(proto.getContentTag(), proto.getLastModifiedTimeSeconds()); + } + + private static byte[] writeMetadataToBytes(DownloadMetadata metadata) { + DownloadMetadataProto.Builder builder = DownloadMetadataProto.newBuilder(); + + if (!metadata.getContentTag().isEmpty()) { + builder.setContentTag(metadata.getContentTag()); + } + if (metadata.getLastModifiedTimeSeconds() > 0) { + builder.setLastModifiedTimeSeconds(metadata.getLastModifiedTimeSeconds()); + } + + return builder.build().toByteArray(); + } +} diff --git a/src/main/java/com/google/android/downloader/RequestException.java b/src/main/java/com/google/android/downloader/RequestException.java new file mode 100644 index 0000000..529e1c7 --- /dev/null +++ b/src/main/java/com/google/android/downloader/RequestException.java @@ -0,0 +1,53 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import java.io.IOException; + +/** + * An exception that occurred during creation or execution of a request; contains {@link + * ErrorDetails} for more detail on the error. + */ +public final class RequestException extends IOException { + private final ErrorDetails errorDetails; + + public RequestException(ErrorDetails errorDetails) { + this.errorDetails = errorDetails; + } + + public RequestException(ErrorDetails errorDetails, Throwable cause) { + super(cause); + this.errorDetails = errorDetails; + } + + public RequestException(Throwable cause) { + super(cause); + this.errorDetails = ErrorDetails.create(cause.getMessage()); + } + + public RequestException(String message) { + super(message); + this.errorDetails = ErrorDetails.create(message); + } + + public ErrorDetails getErrorDetails() { + return errorDetails; + } + + @Override + public String getMessage() { + return super.getMessage() + "; " + errorDetails; + } +} diff --git a/src/main/java/com/google/android/downloader/SimpleFileDownloadDestination.java b/src/main/java/com/google/android/downloader/SimpleFileDownloadDestination.java new file mode 100644 index 0000000..ddc5169 --- /dev/null +++ b/src/main/java/com/google/android/downloader/SimpleFileDownloadDestination.java @@ -0,0 +1,103 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import static com.google.common.base.Preconditions.checkState; + +import com.google.common.io.Files; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.channels.FileChannel; +import java.nio.channels.WritableByteChannel; + +/** + * Basic implementation of {@link DownloadDestination} which streams the download to a {@link File}. + * + * <p>This implementation also keeps track of metadata in a separate file, encoded using + * hand-written serialization. Note that this implementation isn't especially robust - concurrent + * reads and writes could result in errors or in data corruption, and invalid/corrupt metadata will + * result in persistent errors. + */ +public class SimpleFileDownloadDestination implements DownloadDestination { + private final File targetFile; + private final File metadataFile; + + public SimpleFileDownloadDestination(File targetFile, File metadataFile) { + this.targetFile = targetFile; + this.metadataFile = metadataFile; + } + + @Override + public long numExistingBytes() throws IOException { + return targetFile.length(); + } + + @Override + public DownloadMetadata readMetadata() throws IOException { + return metadataFile.exists() + ? readMetadataFromBytes(Files.toByteArray(metadataFile)) + : DownloadMetadata.create(); + } + + @Override + public WritableByteChannel openByteChannel(long offsetBytes, DownloadMetadata metadata) + throws IOException { + checkState( + offsetBytes <= targetFile.length(), + "Opening byte channel with offset past known end of file"); + Files.write(writeMetadataToBytes(metadata), metadataFile); + FileChannel fileChannel = new RandomAccessFile(targetFile, "rw").getChannel(); + // Seek to the requested offset, so we can append data rather than overwrite data. + fileChannel.position(offsetBytes); + return fileChannel; + } + + @Override + public void clear() throws IOException { + if (targetFile.exists() && !targetFile.delete()) { + throw new IOException("Failed to delete()"); + } + } + + @Override + public String toString() { + return targetFile.toString(); + } + + private static DownloadMetadata readMetadataFromBytes(byte[] bytes) throws IOException { + try (DataInputStream inputStream = new DataInputStream(new ByteArrayInputStream(bytes))) { + String etag = inputStream.readUTF(); + long lastModifiedTimeSeconds = inputStream.readLong(); + checkState(lastModifiedTimeSeconds >= 0); + return DownloadMetadata.create(etag, lastModifiedTimeSeconds); + } + } + + private static byte[] writeMetadataToBytes(DownloadMetadata metadata) throws IOException { + try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + DataOutputStream dataOutputStream = new DataOutputStream(byteArrayOutputStream)) { + dataOutputStream.writeUTF(metadata.getContentTag()); + dataOutputStream.writeLong(metadata.getLastModifiedTimeSeconds()); + dataOutputStream.flush(); + byteArrayOutputStream.flush(); + return byteArrayOutputStream.toByteArray(); + } + } +} diff --git a/src/main/java/com/google/android/downloader/UrlEngine.java b/src/main/java/com/google/android/downloader/UrlEngine.java new file mode 100644 index 0000000..7d35245 --- /dev/null +++ b/src/main/java/com/google/android/downloader/UrlEngine.java @@ -0,0 +1,37 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import java.util.Set; + +/** Abstraction of a request engine, which is a high-level interface into a network stack. */ +public interface UrlEngine { + /** + * Creates a request for the given URL. The request is returned as a {@link UrlRequest.Builder}, + * so that callers may further customize the request. Callers must call {@link UrlRequest#send} to + * execute the request. + * + * @param url the URL to connect to. Internal code will assume this is a valid, well-formed URL, + * so client code must have already verified the URL. + */ + UrlRequest.Builder createRequest(String url); + + /** + * Returns the set of URL schemes supported by this url request factory instance. Network stacks + * that connect over HTTP will likely return the set {"http", "https"}, but there may be many + * other types of schemes available (e.g. "file" or "data"). + */ + Set<String> supportedSchemes(); +} diff --git a/src/main/java/com/google/android/downloader/UrlRequest.java b/src/main/java/com/google/android/downloader/UrlRequest.java new file mode 100644 index 0000000..43f5285 --- /dev/null +++ b/src/main/java/com/google/android/downloader/UrlRequest.java @@ -0,0 +1,61 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import com.google.common.util.concurrent.ListenableFuture; + +/** + * Interface representing a request to a URL. Intended for use with a {@link UrlEngine}. + * + * <p>Note: Unless explicitly noted, instances of this object are designed to be thread-safe, + * meaning that methods can be called from any thread. It is however not advisable to use instances + * of this object as a lock, as internal logic may synchronize on the request instance. Also, + * methods should not be called from the Android main/UI thread, as some I/O may occur. + */ +public interface UrlRequest { + /** Builder interface for constructing URL requests. */ + interface Builder { + /** Adds an individual HTTP header for this request. */ + default Builder addHeader(String key, String value) { + return this; + } + + /** Builds the URL request. */ + UrlRequest build(); + } + + /** + * Executes this request by connecting to the URL represented by this request. Returns + * immediately, with the {@link UrlResponse} becoming available asynchronously as a {@link + * ListenableFuture}. May only be called once; multiple calls result in undefined behavior. + * + * <p>If the request fails due to I/O errors or due to an error response from the server (e.g. + * http status codes in the 400s or 500s), then the future will resolve with either an {@link + * RequestException} or an {@link java.io.IOException}. Any other type of exception (e.g. {@link + * IllegalStateException}) indicates an unexpected error occurred, and such errors should likely + * be reported at a severe level and possibly even crash the app. + * + * <p>To cancel an executed requested, call {@link ListenableFuture#cancel} on the future returned + * by this method. Cancellation is opportunistic and best-effort; calling will not guarantee that + * no more network activity will occur, as it is not possible to immediately stop a thread that is + * transferring bytes from the network. Cancellation may fail internally; observe the resolution + * of the future to capture such failures. + * + * <p>Note that although this method returns its result asynchronously and doesn't block on the + * request, the operation may still involve performing some I/O (e.g. verifying the existence of a + * file). Do not call this on the UI thread! + */ + ListenableFuture<UrlResponse> send(); +} diff --git a/src/main/java/com/google/android/downloader/UrlResponse.java b/src/main/java/com/google/android/downloader/UrlResponse.java new file mode 100644 index 0000000..c1a355c --- /dev/null +++ b/src/main/java/com/google/android/downloader/UrlResponse.java @@ -0,0 +1,51 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import com.google.common.util.concurrent.ListenableFuture; +import java.io.Closeable; +import java.nio.channels.WritableByteChannel; +import java.util.List; +import java.util.Map; + +/** + * Interface representing a response to a URL request. + * + * <p>Note that this extends from {@link Closeable}. Callers that successfully obtain an instance of + * the response (via the future returned by {@link UrlRequest#send}) must take care to close the + * response once done using it. + */ +public interface UrlResponse extends Closeable { + /** + * The HTTP status code returned by the server. Returns -1 if the response code can't be discerned + * or doesn't make sense for this response implementation, as mentioned in the javadocs for {@link + * java.net.HttpURLConnection#getResponseCode} + */ + int getResponseCode(); + + /** The multi-valued HTTP response headers returned by the server. */ + Map<String, List<String>> getResponseHeaders(); + + /** + * Writes the response body to the provided {@link WritableByteChannel}. The channel must be open + * prior to calling this method. + * + * <p>This method may only be called once for a given response! Attempting to call this method + * multiple times results in undefined behavior. + * + * @return future that resolves to the number of bytes written to the channel. + */ + ListenableFuture<Long> readResponseBody(WritableByteChannel destinationChannel); +} |