diff options
Diffstat (limited to 'src')
53 files changed, 7577 insertions, 0 deletions
diff --git a/src/example/AndroidManifest.xml b/src/example/AndroidManifest.xml new file mode 100644 index 0000000..b7b64bb --- /dev/null +++ b/src/example/AndroidManifest.xml @@ -0,0 +1,37 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- + Copyright 2021 The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<manifest xmlns:android="http://schemas.android.com/apk/res/android" + package="com.google.android.downloader.example"> + <uses-sdk android:minSdkVersion="22" android:targetSdkVersion="26"/> + <uses-permission android:name="android.permission.INTERNET"/> + <uses-permission android:name="android.permission.ACCESS_NETWORK_STATE"/> + <application + android:label="Downloader Test App" + android:name=".DownloaderTestApplication"> + <activity + android:name=".DownloaderTestActivity" + android:theme="@style/Theme.AppCompat" + android:label="Downloader Test App"> + <intent-filter> + <action android:name="android.intent.action.MAIN" /> + <category android:name="android.intent.category.DEFAULT" /> + <category android:name="android.intent.category.LAUNCHER" /> + </intent-filter> + </activity> + </application> +</manifest>
\ No newline at end of file diff --git a/src/example/java/com/google/android/downloader/example/DownloaderTestActivity.java b/src/example/java/com/google/android/downloader/example/DownloaderTestActivity.java new file mode 100644 index 0000000..99d87bd --- /dev/null +++ b/src/example/java/com/google/android/downloader/example/DownloaderTestActivity.java @@ -0,0 +1,261 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader.example; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.concurrent.TimeUnit.SECONDS; + +import android.os.Bundle; +import android.support.v7.app.AppCompatActivity; +import android.widget.ArrayAdapter; +import android.widget.Button; +import android.widget.Spinner; +import android.widget.TextView; +import com.google.android.downloader.AndroidConnectivityHandler; +import com.google.android.downloader.CronetUrlEngine; +import com.google.android.downloader.DownloadRequest; +import com.google.android.downloader.DownloadResult; +import com.google.android.downloader.Downloader; +import com.google.android.downloader.FloggerDownloaderLogger; +import com.google.android.downloader.OkHttp2UrlEngine; +import com.google.android.downloader.OkHttp3UrlEngine; +import com.google.android.downloader.PlatformUrlEngine; +import com.google.android.downloader.ProtoFileDownloadDestination; +import com.google.android.downloader.UrlEngine; +import com.google.common.collect.ImmutableSet; +import com.google.common.hash.Hashing; +import com.google.common.util.concurrent.FluentFuture; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import com.squareup.okhttp.OkHttpClient; +import java.io.File; +import java.net.URI; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.chromium.net.CronetEngine; + +/** {@link android.app.Activity} instance for the downloader test app. */ +public class DownloaderTestActivity extends AppCompatActivity { + private static final ImmutableSet<String> URLS = + ImmutableSet.of( + "https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb", + "https://dl.google.com/dl/googletv-eureka/beta-channel/" + + "ota.103344.tz_beta-channel.joplin-b1.5f2ce3bb4bf4d25ebfdcee97fcce42e32419b06f.zip", + "https://ssl.gstatic.com/allo/stickers/pack-8/v1/xxxhdpi/all.zip", + "https://ssl.gstatic.com/playatoms/apk_dna/testdata/20171026/shared_libs.zip"); + + @Nullable private Downloader downloader = null; + private ListeningExecutorService transportExecutorService; + private ExecutorService callbackExecutor; + private Executor uiExecutor; + private ScheduledExecutorService scheduledExecutorService; + private File targetDirectory; + private TextView textView; + private Spinner networkSelector; + private final Map<String, UrlEngine> connectionFactoryMap = new HashMap<>(); + + @Override + protected void onCreate(@Nullable Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + setContentView(R.layout.test_app_main_layout); + + targetDirectory = getFilesDir(); + transportExecutorService = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(5)); + callbackExecutor = Executors.newSingleThreadExecutor(); + scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); + + connectionFactoryMap.put("platform", new PlatformUrlEngine(transportExecutorService, 500, 500)); + + okhttp3.OkHttpClient okHttp3Client = + new okhttp3.OkHttpClient.Builder() + .connectTimeout(30, SECONDS) + .readTimeout(30, SECONDS) + .writeTimeout(30, SECONDS) + .build(); + connectionFactoryMap.put( + "okhttp3", new OkHttp3UrlEngine(okHttp3Client, transportExecutorService)); + + OkHttpClient okHttp2Client = new OkHttpClient(); + okHttp2Client.setConnectTimeout(30, SECONDS); + okHttp2Client.setReadTimeout(30, SECONDS); + okHttp2Client.setWriteTimeout(30, SECONDS); + connectionFactoryMap.put( + "okhttp2", new OkHttp2UrlEngine(okHttp2Client, transportExecutorService)); + + CronetEngine cronetEngine = new CronetEngine.Builder(getApplicationContext()).build(); + connectionFactoryMap.put("cronet", new CronetUrlEngine(cronetEngine, transportExecutorService)); + + uiExecutor = this::runOnUiThread; + } + + private Downloader.Builder createDownloaderBuilder() { + return new Downloader.Builder() + .withIOExecutor(callbackExecutor) + .withLogger(new FloggerDownloaderLogger()) + .withConnectivityHandler( + new AndroidConnectivityHandler( + /* context= */ this, scheduledExecutorService, /* timeoutMillis=*/ 10000L)); + } + + @Override + protected void onStart() { + super.onStart(); + + Button startButton = findViewById(R.id.start_button); + startButton.setOnClickListener(view -> startDownloads()); + + Button cancelButton = findViewById(R.id.cancel_button); + cancelButton.setOnClickListener(view -> cancelDownloads()); + + Button clearButton = findViewById(R.id.clear_button); + clearButton.setOnClickListener(view -> clearDownloads()); + + textView = findViewById(R.id.log_text); + networkSelector = findViewById(R.id.network_select); + + ArrayAdapter<String> adapter = + new ArrayAdapter<>( + this, + android.R.layout.simple_list_item_1, + connectionFactoryMap.keySet().toArray(new String[] {})); + networkSelector.setAdapter(adapter); + } + + @Override + protected void onDestroy() { + cancelDownloads(); + transportExecutorService.shutdown(); + callbackExecutor.shutdown(); + scheduledExecutorService.shutdown(); + super.onDestroy(); + } + + private void startDownloads() { + Object selectedItem = networkSelector.getSelectedItem(); + if (selectedItem == null) { + appendLog("No network selected"); + return; + } + + String networkKey = selectedItem.toString(); + Downloader downloader = + createDownloaderBuilder() + .addUrlEngine(ImmutableSet.of("http", "https"), connectionFactoryMap.get(networkKey)) + .build(); + this.downloader = downloader; + + downloader.registerStateChangeCallback( + state -> + appendLog( + String.format( + Locale.ROOT, + "State callback, inFlight=%d, pendingConnectivity=%d, queued=%d", + state.getNumDownloadsInFlight(), + state.getNumDownloadsPendingConnectivity(), + state.getNumQueuedDownloads())), + uiExecutor); + + appendLog("Downloading with network: " + networkKey); + + List<ListenableFuture<DownloadResult>> downloads = new ArrayList<>(); + for (String url : URLS) { + appendLog("Downloading url: " + url); + String fileName = "file" + Hashing.murmur3_128().hashString(url, UTF_8); + URI uri = URI.create(url); + DownloadRequest request = + downloader + .newRequestBuilder( + uri, + new ProtoFileDownloadDestination( + new File(targetDirectory, fileName), + new File(targetDirectory, fileName + ".metadata"))) + .build(); + FluentFuture<DownloadResult> resultFuture = downloader.execute(request); + + // !!! WARNING WARNING WARNING !!! + // This code will leak the activity context into the downloader's innards and thus cause the + // activity to outlive is intended lifecycle. This code should live in its own app-level class + // that avoids references to any UI contexts (e.g. activities, fragments, views). + + // TODO: Refactor the test app to avoid this caveat. + + downloads.add( + resultFuture + .transform( + result -> { + appendLog("Downloaded url. Bytes written: " + result.bytesWritten()); + return result; + }, + uiExecutor) + .catching( + Throwable.class, + throwable -> { + appendLog( + "Failed to download, uri: " + + request.uri() + + " error: " + + throwable.getMessage()); + return null; + }, + uiExecutor)); + } + + Futures.addCallback( + Futures.whenAllComplete(downloads).call(() -> null, uiExecutor), + new FutureCallback<Void>() { + @Override + public void onSuccess(Void result) { + appendLog("Download success"); + } + + @Override + public void onFailure(Throwable t) { + appendLog("Error downloading"); + } + }, + uiExecutor); + } + + private void cancelDownloads() { + Downloader downloader = this.downloader; + if (downloader != null) { + downloader.cancelAll(); + } + } + + private void clearDownloads() { + for (File file : targetDirectory.listFiles()) { + file.delete(); + } + textView.setText(""); + } + + private void appendLog(String msg) { + String existingText = textView.getText().toString(); + existingText = msg + "\n" + existingText; + textView.setText(existingText); + } +} diff --git a/src/example/java/com/google/android/downloader/example/DownloaderTestApplication.java b/src/example/java/com/google/android/downloader/example/DownloaderTestApplication.java new file mode 100644 index 0000000..439681a --- /dev/null +++ b/src/example/java/com/google/android/downloader/example/DownloaderTestApplication.java @@ -0,0 +1,31 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader.example; + +import android.app.Application; +import com.google.common.flogger.backend.android.AndroidLoggerConfig; +import com.google.common.flogger.backend.android.AndroidLoggerConfig.CustomConfig; +import com.google.common.flogger.backend.android.SimpleAndroidLoggerBackend; + +/** {@link Application} instance for the downloader test app. */ +public class DownloaderTestApplication extends Application { + @Override + public void onCreate() { + super.onCreate(); + AndroidLoggerConfig.useCustomConfig( + CustomConfig.newCustomConfig() + .withBackendFactory(new SimpleAndroidLoggerBackend.Factory())); + } +} diff --git a/src/example/res/layout/test_app_main_layout.xml b/src/example/res/layout/test_app_main_layout.xml new file mode 100644 index 0000000..419c6bc --- /dev/null +++ b/src/example/res/layout/test_app_main_layout.xml @@ -0,0 +1,49 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- + Copyright 2021 The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android" + android:orientation="vertical" + android:layout_width="match_parent" + android:layout_height="match_parent"> + <LinearLayout + android:layout_width="match_parent" + android:layout_height="wrap_content"> + <Button + android:id="@+id/start_button" + android:layout_width="wrap_content" + android:layout_height="wrap_content" + android:text="Start"/> + <Button + android:id="@+id/cancel_button" + android:layout_width="wrap_content" + android:layout_height="wrap_content" + android:text="Cancel"/> + <Button + android:id="@+id/clear_button" + android:layout_width="wrap_content" + android:layout_height="wrap_content" + android:text="Clear"/> + <Spinner + android:id="@+id/network_select" + android:layout_width="wrap_content" + android:layout_height="wrap_content"/> + </LinearLayout> + <TextView + android:id="@+id/log_text" + android:layout_width="match_parent" + android:layout_height="match_parent"/> +</LinearLayout>
\ No newline at end of file diff --git a/src/main/Android.bp b/src/main/Android.bp new file mode 100644 index 0000000..f400f06 --- /dev/null +++ b/src/main/Android.bp @@ -0,0 +1,47 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +android_library { + name: "android_downloader_lib", + srcs: ["java/**/*.java"], + exclude_srcs: [ + "java/com/google/android/downloader/CronetUrlEngine.java", + "java/com/google/android/downloader/FloggerDownloaderLogger.java", + "java/com/google/android/downloader/OkHttp2UrlEngine.java", + "java/com/google/android/downloader/OkHttp3UrlEngine.java", + "java/com/google/android/downloader/ProtoFileDownloadDestination.java", + "java/com/google/android/downloader/AndroidConnectivityHandler.java", + ], + static_libs: [ + "androidx.core_core", + "androidx.annotation_annotation", + "error_prone_annotations", + "guava", + ], + libs: [ + "auto_value_annotations", + ], + plugins: [ + "auto_value_plugin", + ], + sdk_version: "current", + min_sdk_version: "30", + apex_available: [ + "//apex_available:platform", + "com.android.extservices", + ], + visibility: [ + "//external/libtextclassifier:__subpackages__", + ], +} diff --git a/src/main/AndroidManifest.xml b/src/main/AndroidManifest.xml new file mode 100644 index 0000000..d002f10 --- /dev/null +++ b/src/main/AndroidManifest.xml @@ -0,0 +1,20 @@ +<!-- + Copyright 2021 The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<manifest xmlns:android="http://schemas.android.com/apk/res/android" + package="com.google.android.downloader"> + <uses-sdk android:minSdkVersion="14"/> +</manifest>
\ No newline at end of file diff --git a/src/main/AndroidTestAppManifest.xml b/src/main/AndroidTestAppManifest.xml new file mode 100644 index 0000000..be6c76a --- /dev/null +++ b/src/main/AndroidTestAppManifest.xml @@ -0,0 +1,37 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- + Copyright 2021 The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<manifest xmlns:android="http://schemas.android.com/apk/res/android" + package="com.google.android.downloader.testapp"> + <uses-sdk android:minSdkVersion="22" android:targetSdkVersion="26"/> + <uses-permission android:name="android.permission.INTERNET"/> + <uses-permission android:name="android.permission.ACCESS_NETWORK_STATE"/> + <application + android:label="Downloader Test App" + android:name=".DownloaderTestApplication"> + <activity + android:name=".DownloaderTestActivity" + android:theme="@style/Theme.AppCompat" + android:label="Downloader Test App"> + <intent-filter> + <action android:name="android.intent.action.MAIN" /> + <category android:name="android.intent.category.DEFAULT" /> + <category android:name="android.intent.category.LAUNCHER" /> + </intent-filter> + </activity> + </application> +</manifest>
\ No newline at end of file diff --git a/src/main/LICENSE b/src/main/LICENSE new file mode 100644 index 0000000..7a4a3ea --- /dev/null +++ b/src/main/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License.
\ No newline at end of file diff --git a/src/main/java/com/google/android/downloader/AndroidConnectivityHandler.java b/src/main/java/com/google/android/downloader/AndroidConnectivityHandler.java new file mode 100644 index 0000000..adac6e9 --- /dev/null +++ b/src/main/java/com/google/android/downloader/AndroidConnectivityHandler.java @@ -0,0 +1,273 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import static android.Manifest.permission.ACCESS_NETWORK_STATE; +import static android.net.ConnectivityManager.CONNECTIVITY_ACTION; +import static androidx.core.content.ContextCompat.checkSelfPermission; +import static androidx.core.content.ContextCompat.getSystemService; +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.util.concurrent.MoreExecutors.directExecutor; +import static java.util.concurrent.TimeUnit.MILLISECONDS; + +import android.annotation.SuppressLint; +import android.annotation.TargetApi; +import android.content.BroadcastReceiver; +import android.content.Context; +import android.content.Intent; +import android.content.IntentFilter; +import android.content.pm.PackageManager; +import android.net.ConnectivityManager; +import android.net.Network; +import android.net.NetworkCapabilities; +import android.net.NetworkInfo; +import android.os.Build.VERSION; +import android.os.Build.VERSION_CODES; +import androidx.annotation.RequiresPermission; +import androidx.core.net.ConnectivityManagerCompat; +import com.google.android.downloader.DownloadConstraints.NetworkType; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.flogger.GoogleLogger; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListenableFutureTask; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeoutException; +import javax.annotation.Nullable; + +/** + * Default implementation of {@link ConnectivityHandler}, relying on Android's {@link + * ConnectivityManager}. + */ +public class AndroidConnectivityHandler implements ConnectivityHandler { + private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); + + private final Context context; + private final ScheduledExecutorService scheduledExecutorService; + private final ConnectivityManager connectivityManager; + private final long timeoutMillis; + + /** + * Creates a new AndroidConnectivityHandler to handle connectivity checks for the Downloader. + * + * @param context the context to use for perform Android API checks. Will be retained, so should + * not be a UI context. + * @param scheduledExecutorService a scheduled executor used to timeout operations waiting for + * connectivity. Beware that there are problems with this, see go/executors-timing for + * details. + * @param timeoutMillis how long to wait before timing out a connectivity check. If more than this + * amount of time elapses, the connectivity check will timeout, and the {@link + * ListenableFuture} returned by {@link #checkConnectivity} will resolve with a {@link + * TimeoutException}. + */ + public AndroidConnectivityHandler( + Context context, ScheduledExecutorService scheduledExecutorService, long timeoutMillis) { + if (PackageManager.PERMISSION_GRANTED != checkSelfPermission(context, ACCESS_NETWORK_STATE)) { + throw new IllegalStateException( + "AndroidConnectivityHandler requires the ACCESS_NETWORK_STATE permission."); + } + + this.context = context; + this.scheduledExecutorService = scheduledExecutorService; + this.connectivityManager = checkNotNull(getSystemService(context, ConnectivityManager.class)); + this.timeoutMillis = timeoutMillis; + } + + @Override + @RequiresPermission(ACCESS_NETWORK_STATE) + public ListenableFuture<Void> checkConnectivity(DownloadConstraints constraints) { + if (connectivitySatisfied(constraints)) { + return Futures.immediateVoidFuture(); + } + + ListenableFutureTask<Void> futureTask = ListenableFutureTask.create(() -> null); + // TODO: Using a receiver here isn't great. Ideally we'd use + // ConnectivityManager.requestNetwork(request, callback, timeout), but that's only available + // on SDK 26+, so we'd still need a fallback on older versions of Android. + NetworkBroadcastReceiver receiver = new NetworkBroadcastReceiver(constraints, futureTask); + context.registerReceiver(receiver, new IntentFilter(CONNECTIVITY_ACTION)); + futureTask.addListener(() -> context.unregisterReceiver(receiver), directExecutor()); + return Futures.withTimeout(futureTask, timeoutMillis, MILLISECONDS, scheduledExecutorService); + } + + @RequiresPermission(ACCESS_NETWORK_STATE) + private boolean connectivitySatisfied(DownloadConstraints downloadConstraints) { + // Special case the NONE value - if that is specified then skip all further checks. + if (downloadConstraints == DownloadConstraints.NONE) { + return true; + } + + NetworkType networkType; + + if (VERSION.SDK_INT >= VERSION_CODES.M) { + Network network = connectivityManager.getActiveNetwork(); + if (network == null) { + logger.atFine().log("No current network, connectivity cannot be satisfied."); + return false; + } + + NetworkCapabilities networkCapabilities = connectivityManager.getNetworkCapabilities(network); + if (networkCapabilities == null) { + logger.atFine().log( + "Can't determine network capabilities, connectivity cannot be satisfied"); + return false; + } + + if (!networkCapabilities.hasCapability(NetworkCapabilities.NET_CAPABILITY_INTERNET)) { + logger.atFine().log( + "Network does not have internet capabilities, connectivity cannot be satisfied."); + return false; + } + + if (downloadConstraints.requireUnmeteredNetwork() + && ConnectivityManagerCompat.isActiveNetworkMetered(connectivityManager)) { + logger.atFine().log("Network is metered, connectivity cannot be satisfied."); + return false; + } + + if (downloadConstraints.requiredNetworkTypes().contains(NetworkType.ANY)) { + // If the request doesn't care about the network type (by way of having NetworkType.ANY in + // its set of allowed network types), then stop checking now. + return true; + } + + networkType = computeNetworkType(networkCapabilities); + } else { + @SuppressLint("MissingPermission") // We just checked the permission above. + NetworkInfo networkInfo = connectivityManager.getActiveNetworkInfo(); + if (networkInfo == null) { + logger.atFine().log("No current network, connectivity cannot be satisfied."); + return false; + } + + if (!networkInfo.isConnected()) { + // Regardless of which type of network we have right now, if it's not connected then all + // downloads will fail, so just queue up all downloads in this case. + logger.atFine().log("Network disconnected, connectivity cannot be satisfied."); + return false; + } + + if (downloadConstraints.requireUnmeteredNetwork() + && ConnectivityManagerCompat.isActiveNetworkMetered(connectivityManager)) { + logger.atFine().log("Network is metered, connectivity cannot be satisfied."); + return false; + } + + if (downloadConstraints.requiredNetworkTypes().contains(NetworkType.ANY)) { + // If the request doesn't care about the network type (by way of having NetworkType.ANY in + // its set of allowed network types), then stop checking now. + return true; + } + + networkType = computeNetworkType(networkInfo.getType()); + } + + if (networkType == null) { + // If for some reason we couldn't determine the network type from Android (unexpected value?), + // then we can't validate it against the set of constraints, so fail the check. + return false; + } + + // Otherwise, just make sure that the current network type is allowed by this request. + return downloadConstraints.requiredNetworkTypes().contains(networkType); + } + + @Nullable + private static NetworkType computeNetworkType(int networkType) { + if (VERSION.SDK_INT >= VERSION_CODES.HONEYCOMB_MR2 + && networkType == ConnectivityManager.TYPE_BLUETOOTH) { + return NetworkType.BLUETOOTH; + } else if (VERSION.SDK_INT >= VERSION_CODES.HONEYCOMB_MR2 + && networkType == ConnectivityManager.TYPE_ETHERNET) { + return NetworkType.ETHERNET; + } else if (networkType == ConnectivityManager.TYPE_MOBILE + || networkType == ConnectivityManager.TYPE_MOBILE_MMS + || networkType == ConnectivityManager.TYPE_MOBILE_SUPL + || networkType == ConnectivityManager.TYPE_MOBILE_DUN + || networkType == ConnectivityManager.TYPE_MOBILE_HIPRI) { + return NetworkType.CELLULAR; + } else if (VERSION.SDK_INT >= VERSION_CODES.LOLLIPOP + && networkType == ConnectivityManager.TYPE_VPN) { + // There's no way to determine the underlying transport used by a VPN, so it's best to + // be conservative and treat it is a cellular network. + return NetworkType.CELLULAR; + } else if (networkType == ConnectivityManager.TYPE_WIFI) { + return NetworkType.WIFI; + } else if (networkType == ConnectivityManager.TYPE_WIMAX) { + // WiMAX and Cellular aren't really the same thing, but in practice they can be treated + // the same, as they are both typically available over long distances and are often metered. + return NetworkType.CELLULAR; + } + + return null; + } + + @Nullable + @TargetApi(VERSION_CODES.LOLLIPOP) + private static NetworkType computeNetworkType(NetworkCapabilities networkCapabilities) { + if (networkCapabilities.hasTransport(NetworkCapabilities.TRANSPORT_CELLULAR)) { + return NetworkType.CELLULAR; + } else if (networkCapabilities.hasTransport(NetworkCapabilities.TRANSPORT_WIFI)) { + return NetworkType.WIFI; + } else if (networkCapabilities.hasTransport(NetworkCapabilities.TRANSPORT_BLUETOOTH)) { + return NetworkType.BLUETOOTH; + } else if (networkCapabilities.hasTransport(NetworkCapabilities.TRANSPORT_ETHERNET)) { + return NetworkType.ETHERNET; + } else if (networkCapabilities.hasTransport(NetworkCapabilities.TRANSPORT_VPN)) { + // There's no way to determine the underlying transport used by a VPN, so it's best to + // be conservative and treat it is a cellular network. + return NetworkType.CELLULAR; + } + return null; + } + + @VisibleForTesting + class NetworkBroadcastReceiver extends BroadcastReceiver { + private final DownloadConstraints constraints; + private final Runnable completionRunnable; + + public NetworkBroadcastReceiver(DownloadConstraints constraints, Runnable completionRunnable) { + this.constraints = constraints; + this.completionRunnable = completionRunnable; + } + + @Override + @RequiresPermission(ACCESS_NETWORK_STATE) + public void onReceive(Context context, Intent intent) { + if (!CONNECTIVITY_ACTION.equals(intent.getAction())) { + logger.atSevere().log( + "NetworkBroadcastReceiver received an unexpected intent action: %s", + intent.getAction()); + return; + } + + if (intent.getBooleanExtra(ConnectivityManager.EXTRA_NO_CONNECTIVITY, false)) { + logger.atInfo().log("NetworkBroadcastReceiver updated but NO_CONNECTIVITY extra set"); + return; + } + + logger.atInfo().log( + "NetworkBroadcastReceiver received intent: %s %s", + intent.getAction(), intent.getExtras()); + + if (connectivitySatisfied(constraints)) { + logger.atInfo().log("Connectivity satisfied in BroadcastReceiver, running completion"); + completionRunnable.run(); + } else { + logger.atInfo().log("Connectivity NOT satisfied in BroadcastReceiver"); + } + } + } +} diff --git a/src/main/java/com/google/android/downloader/AndroidDownloaderLogger.java b/src/main/java/com/google/android/downloader/AndroidDownloaderLogger.java new file mode 100644 index 0000000..88fd157 --- /dev/null +++ b/src/main/java/com/google/android/downloader/AndroidDownloaderLogger.java @@ -0,0 +1,64 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import android.util.Log; +import com.google.errorprone.annotations.FormatMethod; +import com.google.errorprone.annotations.FormatString; +import javax.annotation.Nullable; + +/** + * Implementation of {@link DownloaderLogger} backed by Android's {@link Log} system. Useful for + * contexts (e.g. AOSP) that can't take the Flogger dependency. + */ +public class AndroidDownloaderLogger implements DownloaderLogger { + private static final String TAG = "downloader2"; + + @Override + @FormatMethod + public void logFine(@FormatString String message, Object... args) { + Log.d(TAG, String.format(message, args)); + } + + @Override + @FormatMethod + public void logInfo(@FormatString String message, Object... args) { + Log.i(TAG, String.format(message, args)); + } + + @Override + @FormatMethod + public void logWarning(@FormatString String message, Object... args) { + Log.w(TAG, String.format(message, args)); + } + + @Override + @FormatMethod + public void logWarning(@Nullable Throwable cause, @FormatString String message, Object... args) { + Log.w(TAG, String.format(message, args), cause); + } + + @Override + @FormatMethod + public void logError(@FormatString String message, Object... args) { + Log.e(TAG, String.format(message, args)); + } + + @Override + @FormatMethod + public void logError(@Nullable Throwable cause, @FormatString String message, Object... args) { + Log.e(TAG, String.format(message, args), cause); + } +} diff --git a/src/main/java/com/google/android/downloader/ConnectivityHandler.java b/src/main/java/com/google/android/downloader/ConnectivityHandler.java new file mode 100644 index 0000000..ecd8ef5 --- /dev/null +++ b/src/main/java/com/google/android/downloader/ConnectivityHandler.java @@ -0,0 +1,33 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import com.google.common.util.concurrent.ListenableFuture; + +/** Interface for dealing with connectivity checking and handling in the downloader. */ +public interface ConnectivityHandler { + /** + * Handles the checking connectivity for the given {@link DownloadConstraints}. This method must + * return a {@link ListenableFuture} that resolves when the constraints are satisfied. + * + * <p>The returned future may be resolved in failure state with a {@link + * java.util.concurrent.TimeoutException} if too much time without sufficient connectivity has + * been observed. The future may also be cancelled (resulting in a {@link + * java.util.concurrent.CancellationException}) if observing connectivity changes is no longer + * necessary. Either state will be given special treatment to retry downloads. Any other failure + * state will be considered fatal and will fail ongoing downloads. + */ + ListenableFuture<Void> checkConnectivity(DownloadConstraints constraints); +} diff --git a/src/main/java/com/google/android/downloader/CronetUrlEngine.java b/src/main/java/com/google/android/downloader/CronetUrlEngine.java new file mode 100644 index 0000000..f4f41a1 --- /dev/null +++ b/src/main/java/com/google/android/downloader/CronetUrlEngine.java @@ -0,0 +1,334 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.util.concurrent.MoreExecutors.directExecutor; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableSet; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; +import java.net.HttpURLConnection; +import java.nio.ByteBuffer; +import java.nio.channels.WritableByteChannel; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Executor; +import javax.annotation.Nullable; +import org.chromium.net.CallbackException; +import org.chromium.net.CronetEngine; +import org.chromium.net.CronetException; +import org.chromium.net.NetworkException; +import org.chromium.net.UrlResponseInfo; + +/** + * {@link UrlEngine} implementation that uses Cronet for network connectivity. + * + * <p>Note: Internally this implementation allocates a 128kb direct byte buffer per request to + * transfer bytes around. If memory use is sensitive, then the number of concurrent requests should + * be limited. + */ +public final class CronetUrlEngine implements UrlEngine { + private static final ImmutableSet<String> SCHEMES = ImmutableSet.of("http", "https"); + @VisibleForTesting static final int BUFFER_SIZE_BYTES = 128 * 1024; // 128kb + + private final CronetEngine cronetEngine; + private final Executor callbackExecutor; + + /** + * Creates a new Cronet-based {@link UrlEngine}. + * + * @param cronetEngine The pre-configured {@link CronetEngine} that will be used to implement HTTP + * connections. + * @param callbackExecutor The {@link Executor} on which Cronet's callbacks will be executed. Note + * that this request factory implementation will perform I/O in the callbacks, so make sure + * the threads backing the executor can block safely (i.e. do not run on the UI thread!) + */ + public CronetUrlEngine(CronetEngine cronetEngine, Executor callbackExecutor) { + this.cronetEngine = cronetEngine; + this.callbackExecutor = callbackExecutor; + } + + @Override + public UrlRequest.Builder createRequest(String url) { + SettableFuture<UrlResponse> responseFuture = SettableFuture.create(); + CronetCallback callback = new CronetCallback(responseFuture); + org.chromium.net.UrlRequest.Builder builder = + cronetEngine.newUrlRequestBuilder(url, callback, callbackExecutor); + return new CronetUrlRequestBuilder(builder, responseFuture); + } + + @Override + public Set<String> supportedSchemes() { + return SCHEMES; + } + + /** Cronet-specific implementation of {@link UrlRequest} */ + static class CronetUrlRequest implements UrlRequest { + private final org.chromium.net.UrlRequest urlRequest; + private final ListenableFuture<UrlResponse> responseFuture; + + CronetUrlRequest(CronetUrlRequestBuilder builder) { + urlRequest = builder.requestBuilder.build(); + responseFuture = builder.responseFuture; + + responseFuture.addListener( + () -> { + if (responseFuture.isCancelled()) { + urlRequest.cancel(); + } + }, + directExecutor()); + } + + @Override + public ListenableFuture<UrlResponse> send() { + urlRequest.start(); + return responseFuture; + } + } + + /** Cronet-specific implementation of {@link UrlRequest.Builder} */ + static class CronetUrlRequestBuilder implements UrlRequest.Builder { + private final org.chromium.net.UrlRequest.Builder requestBuilder; + private final ListenableFuture<UrlResponse> responseFuture; + + CronetUrlRequestBuilder( + org.chromium.net.UrlRequest.Builder requestBuilder, + ListenableFuture<UrlResponse> responseFuture) { + this.requestBuilder = requestBuilder; + this.responseFuture = responseFuture; + } + + @Override + public UrlRequest.Builder addHeader(String key, String value) { + requestBuilder.addHeader(key, value); + return this; + } + + @Override + public UrlRequest build() { + return new CronetUrlRequest(this); + } + } + + /** + * Cronet-specific implementation of {@link UrlResponse}. Implements its functionality by using + * Cronet's {@link org.chromium.net.UrlRequest} and {@link UrlResponseInfo} objects. + */ + static class CronetResponse implements UrlResponse { + private final org.chromium.net.UrlRequest urlRequest; + private final UrlResponseInfo urlResponseInfo; + private final SettableFuture<Long> completionFuture; + private final CronetCallback callback; + + CronetResponse( + org.chromium.net.UrlRequest urlRequest, + UrlResponseInfo urlResponseInfo, + SettableFuture<Long> completionFuture, + CronetCallback callback) { + this.urlRequest = urlRequest; + this.urlResponseInfo = urlResponseInfo; + this.completionFuture = completionFuture; + this.callback = callback; + } + + @Override + public int getResponseCode() { + return urlResponseInfo.getHttpStatusCode(); + } + + @Override + public Map<String, List<String>> getResponseHeaders() { + return urlResponseInfo.getAllHeaders(); + } + + @Override + public ListenableFuture<Long> readResponseBody(WritableByteChannel destinationChannel) { + IOUtil.validateChannel(destinationChannel); + callback.destinationChannel = destinationChannel; + urlRequest.read(ByteBuffer.allocateDirect(BUFFER_SIZE_BYTES)); + return completionFuture; + } + + @Override + public void close() { + urlRequest.cancel(); + } + } + + /** + * Implementation of {@link org.chromium.net.UrlRequest.Callback} to handle the lifecycle of a + * Cronet url request. The operations of handling the response metadata returned by the server as + * well as actually reading the response body happen here. + */ + static class CronetCallback extends org.chromium.net.UrlRequest.Callback { + private final SettableFuture<UrlResponse> responseFuture; + private final SettableFuture<Long> completionFuture = SettableFuture.create(); + + @Nullable private CronetResponse cronetResponse; + @Nullable private WritableByteChannel destinationChannel; + private long numBytesWritten; + + CronetCallback(SettableFuture<UrlResponse> responseFuture) { + this.responseFuture = responseFuture; + } + + @Override + public void onRedirectReceived( + org.chromium.net.UrlRequest urlRequest, + UrlResponseInfo urlResponseInfo, + String newLocationUrl) { + // Just blindly follow redirects; that's pretty much always what you want to do. + urlRequest.followRedirect(); + } + + @Override + public void onResponseStarted( + org.chromium.net.UrlRequest urlRequest, UrlResponseInfo urlResponseInfo) { + // We've received the response metadata from the server, so we have a status code and + // response headers to examine. At this point we can create a response object and complete + // the response future. If necessary, the body itself will be downloaded via a subsequent + // call urlRequest.read inside the CronetResponse.writeResponseBody, which will trigger the + // other lifecycle callbacks. + int httpCode = urlResponseInfo.getHttpStatusCode(); + if (httpCode >= HttpURLConnection.HTTP_BAD_REQUEST) { + responseFuture.setException( + new RequestException( + ErrorDetails.createFromHttpErrorResponse( + httpCode, + urlResponseInfo.getAllHeaders(), + urlResponseInfo.getHttpStatusText()))); + urlRequest.cancel(); + } else { + cronetResponse = new CronetResponse(urlRequest, urlResponseInfo, completionFuture, this); + responseFuture.set(cronetResponse); + } + } + + @Override + public void onReadCompleted( + org.chromium.net.UrlRequest urlRequest, + UrlResponseInfo urlResponseInfo, + ByteBuffer byteBuffer) + throws Exception { + // If we're already done, just bail out. + if (urlRequest.isDone()) { + return; + } + + // If the underlying future has been cancelled, cancel the request and abort. + if (completionFuture.isCancelled()) { + urlRequest.cancel(); + return; + } + + // Flip the buffer to prepare for reading from it. + byteBuffer.flip(); + + // Write however many bytes are in our buffer to the underlying channel. + numBytesWritten += IOUtil.blockingWrite(byteBuffer, checkNotNull(destinationChannel)); + + // Reset the buffer to be reused on the next iteration. + byteBuffer.clear(); + + // Finally, request more bytes. This is necessary per the Cronet API. + urlRequest.read(byteBuffer); + } + + @Override + public void onSucceeded( + org.chromium.net.UrlRequest urlRequest, UrlResponseInfo urlResponseInfo) { + // The body has been successfully streamed. Close the underlying response object to free + // up resources it holds, and resolve the pending future with the number of bytes written. + closeResponse(); + completionFuture.set(numBytesWritten); + } + + @Override + public void onFailed( + org.chromium.net.UrlRequest urlRequest, + UrlResponseInfo urlResponseInfo, + CronetException exception) { + // There was some sort of error with the connection. Clean up and resolve the pending future + // with the exception we encountered. + closeResponse(); + + ErrorDetails errorDetails; + if (urlResponseInfo != null + && urlResponseInfo.getHttpStatusCode() >= HttpURLConnection.HTTP_BAD_REQUEST) { + errorDetails = + ErrorDetails.createFromHttpErrorResponse( + urlResponseInfo.getHttpStatusCode(), + urlResponseInfo.getAllHeaders(), + urlResponseInfo.getHttpStatusText()); + } else if (exception instanceof NetworkException) { + NetworkException networkException = (NetworkException) exception; + errorDetails = + ErrorDetails.builder() + .setInternalErrorCode(networkException.getCronetInternalErrorCode()) + .setErrorMessage(Strings.nullToEmpty(networkException.getMessage())) + .build(); + } else { + errorDetails = + ErrorDetails.builder() + .setErrorMessage(Strings.nullToEmpty(exception.getMessage())) + .build(); + } + + RequestException requestException = + new RequestException(errorDetails, unwrapException(exception)); + + if (!responseFuture.isDone()) { + responseFuture.setException(requestException); + } else { + // N.B: The completion future is available iff the response future is resolved, so + // we don't need to resolve it with an exception here unless the response future is done. + completionFuture.setException(requestException); + } + } + + @Override + public void onCanceled( + org.chromium.net.UrlRequest urlRequest, UrlResponseInfo urlResponseInfo) { + // The request was cancelled. This only occurs when UrlRequest.cancel is called, which + // in turn only happens when UrlResponse.close is called. Clean up internal state + // and resolve the future with an error. + closeResponse(); + completionFuture.setException(new RequestException("UrlRequest cancelled")); + } + + /** Safely closes the current response object, if any. */ + private void closeResponse() { + CronetResponse cronetResponse = this.cronetResponse; + if (cronetResponse == null) { + return; + } + cronetResponse.close(); + } + } + + private static Throwable unwrapException(CronetException exception) { + // CallbackExceptions aren't interesting, so unwrap them. + if (exception instanceof CallbackException) { + Throwable cause = exception.getCause(); + return cause == null ? exception : cause; + } + return exception; + } +} diff --git a/src/main/java/com/google/android/downloader/DataUrl.java b/src/main/java/com/google/android/downloader/DataUrl.java new file mode 100644 index 0000000..7e83018 --- /dev/null +++ b/src/main/java/com/google/android/downloader/DataUrl.java @@ -0,0 +1,104 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import com.google.common.base.Splitter; +import com.google.common.io.BaseEncoding; +import java.util.List; + +/** + * Utility for handling data URLs. See the MDN for documentation of the format: + * https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URIs + * + * <p>See http://www.ietf.org/rfc/rfc2397.txt for the formal specification of the data URL scheme. + */ +class DataUrl { + private static final BaseEncoding BASE64_URL = BaseEncoding.base64Url(); + private static final BaseEncoding BASE64 = BaseEncoding.base64(); + private static final Splitter SEMICOLON_SPLITTER = Splitter.on(';'); + + private final byte[] data; + private final String mimeType; + + /** Thrown to indicate that the input URL could not be parsed as an RFC2397 data URL. */ + static class DataUrlException extends IllegalArgumentException { + DataUrlException(String message) { + super(message); + } + + DataUrlException(String message, Throwable cause) { + super(message, cause); + } + } + + private DataUrl(byte[] data, String mimeType) { + this.data = data; + this.mimeType = mimeType; + } + + byte[] data() { + return data; + } + + String mimeType() { + return mimeType; + } + + /** + * Decodes a data url, returning the payload as a {@link DataUrl} instance. + * + * <p>The syntax of a valid data URL is as follows: data:[<mediatype>][;base64],<data> + * This method will assert that the provided URL is conformant, and throws an {@link + * DataUrlException} if the URL is invalid or can't be parsed. + * + * <p>Note that this method requires the data URL to be encoded in base64, and assumes the data is + * binary. Data URLs may be text/plain data, but that is not supported by this method. + * + * @param url the data url to decode + */ + static DataUrl parseFromString(String url) throws DataUrlException { + if (!url.startsWith("data:")) { + throw new DataUrlException("URL doesn't have the data scheme"); + } + + int commaPos = url.indexOf(','); + if (commaPos == -1) { + throw new DataUrlException("Comma not found in data URL"); + } + String data = url.substring(commaPos + 1); + List<String> options = SEMICOLON_SPLITTER.splitToList(url.substring(5, commaPos)); + if (options.size() < 2) { + throw new DataUrlException("Insufficient number of options for data URL"); + } + String mimeType = options.get(0); + String encoding = options.get(1); + if (!encoding.equals("base64")) { + throw new DataUrlException("Invalid encoding: " + encoding + ", only base64 is supported"); + } + + try { + return new DataUrl(BASE64_URL.decode(data), mimeType); + } catch (IllegalArgumentException unused) { + // If the URL-safe base64 encoding doesn't work, try the vanilla base64 encoding. Ideally + // users would only use web-safe data URIs, but in many environments regular base64 payloads + // can be used without issue. + try { + return new DataUrl(BASE64.decode(data), mimeType); + } catch (IllegalArgumentException e) { + throw new DataUrlException("Invalid base64 payload in data URL", e); + } + } + } +} diff --git a/src/main/java/com/google/android/downloader/DataUrlEngine.java b/src/main/java/com/google/android/downloader/DataUrlEngine.java new file mode 100644 index 0000000..fec55aa --- /dev/null +++ b/src/main/java/com/google/android/downloader/DataUrlEngine.java @@ -0,0 +1,134 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import com.google.android.downloader.DataUrl.DataUrlException; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.net.HttpHeaders; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.nio.ByteBuffer; +import java.nio.channels.WritableByteChannel; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** {@link UrlEngine} implementation for handling data URLs. */ +public final class DataUrlEngine implements UrlEngine { + private static final ImmutableSet<String> DATA_SCHEME = ImmutableSet.of("data"); + + private final ListeningExecutorService transferExecutorService; + + public DataUrlEngine(ListeningExecutorService transferExecutorService) { + this.transferExecutorService = transferExecutorService; + } + + @Override + public UrlRequest.Builder createRequest(String url) { + return new DataUrlRequestBuilder(url); + } + + @Override + public Set<String> supportedSchemes() { + return DATA_SCHEME; + } + + /** Implementation of {@link UrlRequest.Builder} for Data URLs. */ + class DataUrlRequestBuilder implements UrlRequest.Builder { + private final String url; + + DataUrlRequestBuilder(String url) { + this.url = url; + } + + @Override + public UrlRequest build() { + return new DataUrlRequest(url); + } + } + + /** + * Implementation of {@link UrlRequest} for data URLs. Note that this is pretty trivial because no + * network connection has to happen. There's also no meaningful way to cancel a data URL request, + * so cancelling the future returned by {@link #send} has no effect on processing the data URL. + */ + class DataUrlRequest implements UrlRequest { + private final String url; + + DataUrlRequest(String url) { + this.url = url; + } + + @Override + public ListenableFuture<UrlResponse> send() { + return transferExecutorService.submit( + () -> { + try { + return new DataUrlResponse(DataUrl.parseFromString(url)); + } catch (DataUrlException e) { + throw new RequestException(e); + } + }); + } + } + + /** + * Implementation of {@link DataUrlResponse} for data URLs. Emulates network-specific APIs for + * data URLs, which means that this always return HTTP_OK/200 as a response code, and an empty set + * of response headers. + * + * <p>The response body is written by decoding the data URL string and writing the decoded bytes + * to the destination channel. + */ + class DataUrlResponse implements UrlResponse { + private final DataUrl dataUrl; + + DataUrlResponse(DataUrl dataUrl) { + this.dataUrl = dataUrl; + } + + @Override + public int getResponseCode() { + return HttpURLConnection.HTTP_OK; + } + + @Override + public Map<String, List<String>> getResponseHeaders() { + return ImmutableMap.of(HttpHeaders.CONTENT_TYPE, ImmutableList.of(dataUrl.mimeType())); + } + + @Override + public ListenableFuture<Long> readResponseBody(WritableByteChannel destinationChannel) { + IOUtil.validateChannel(destinationChannel); + return transferExecutorService.submit( + () -> { + try { + return IOUtil.blockingWrite(ByteBuffer.wrap(dataUrl.data()), destinationChannel); + } catch (IOException e) { + throw new RequestException(e); + } + }); + } + + @Override + public void close() throws IOException { + // Nothing to close for a data URL response. + } + } +} diff --git a/src/main/java/com/google/android/downloader/DownloadConstraints.java b/src/main/java/com/google/android/downloader/DownloadConstraints.java new file mode 100644 index 0000000..74b1a72 --- /dev/null +++ b/src/main/java/com/google/android/downloader/DownloadConstraints.java @@ -0,0 +1,122 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableSet; +import java.util.Set; + +/** + * Possible constraints that a download request requires. Primarily used by {@link + * ConnectivityHandler} to determine if sufficient connectivity is available. + */ +@AutoValue +public abstract class DownloadConstraints { + /** + * Special value of {@code DownloadConstraints}. If this value is specified, then the {@link + * ConnectivityHandler} short-circuits and no constraint checks are performed, even if no network + * is present at all! + */ + public static final DownloadConstraints NONE = + DownloadConstraints.builder() + // Note: Can't use EnumSet because it relies on reflection over the enum class, which + // breaks unless the right proguard configuration is applied. + .setRequiredNetworkTypes(ImmutableSet.of()) + .setRequireUnmeteredNetwork(false) + .build(); + + /** + * Common value to indicate that the active network must be unmetered. This value permits any + * network type as long as it doesn't indicate it is metered in some way. + */ + public static final DownloadConstraints NETWORK_UNMETERED = + DownloadConstraints.builder() + // Note: Can't use EnumSet because it relies on reflection over the enum class, which + // breaks unless the right proguard configuration is applied. + .setRequiredNetworkTypes(ImmutableSet.of(NetworkType.ANY)) + .setRequireUnmeteredNetwork(true) + .build(); + + /** + * Common value to indicate that the required network must simply be connected in some way, and + * otherwise doesn't have any restrictions. Any network type is allowed. + * + * <p>This is the default value for download requests. + */ + public static final DownloadConstraints NETWORK_CONNECTED = + DownloadConstraints.builder() + // Note: Can't use EnumSet because it relies on reflection over the enum class, which + // breaks unless the right proguard configuration is applied. + .setRequiredNetworkTypes(ImmutableSet.of(NetworkType.ANY)) + .setRequireUnmeteredNetwork(false) + .build(); + + /** + * The type of network that is required. This is a subset of the network types enumerated by + * {@link android.net.ConnectivityManager}. + */ + public enum NetworkType { + /** Special network type to allow any type of network, even if it not one of the known types. */ + ANY, + /** Equivalent to {@link android.net.NetworkCapabilities#TRANSPORT_BLUETOOTH} */ + BLUETOOTH, + /** Equivalent to {@link android.net.NetworkCapabilities#TRANSPORT_ETHERNET} */ + ETHERNET, + /** Equivalent to {@link android.net.NetworkCapabilities#TRANSPORT_CELLULAR} */ + CELLULAR, + /** Equivalent to {@link android.net.NetworkCapabilities#TRANSPORT_WIFI} */ + WIFI, + } + + /** + * Whether the connection must be unmetered to pass connectivity checks. See {@link + * androidx.core.net.ConnectivityManagerCompat#isActiveNetworkMetered} for more details on this + * variable. + * + * <p>False by default. + */ + public abstract boolean requireUnmeteredNetwork(); + + /** + * The types of networks that are allowed for the request to pass connectivity checks. The + * currently active network type must be one of the values in this set. This set may not be empty. + */ + public abstract ImmutableSet<NetworkType> requiredNetworkTypes(); + + /** Creates a {@code DownloadConstraints.Builder} instance. */ + public static Builder builder() { + return new AutoValue_DownloadConstraints.Builder().setRequireUnmeteredNetwork(false); + } + + /** Converts this instance to a builder for modifications. */ + public abstract Builder toBuilder(); + + /** Builder for creating instances of {@link DownloadConstraints}. */ + @AutoValue.Builder + public abstract static class Builder { + public abstract Builder setRequiredNetworkTypes(Set<NetworkType> networkTypes); + + abstract ImmutableSet.Builder<NetworkType> requiredNetworkTypesBuilder(); + + public Builder addRequiredNetworkType(NetworkType networkType) { + requiredNetworkTypesBuilder().add(networkType); + return this; + } + + public abstract Builder setRequireUnmeteredNetwork(boolean requireUnmeteredNetwork); + + public abstract DownloadConstraints build(); + } +} diff --git a/src/main/java/com/google/android/downloader/DownloadDestination.java b/src/main/java/com/google/android/downloader/DownloadDestination.java new file mode 100644 index 0000000..462b8c7 --- /dev/null +++ b/src/main/java/com/google/android/downloader/DownloadDestination.java @@ -0,0 +1,71 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import java.io.IOException; +import java.nio.channels.WritableByteChannel; + +/** + * An abstract "sink" for the stream of bytes produced by a download. A common implementation is + * {@link ProtoFileDownloadDestination}, which streams the downloaded bytes to a writeable file on + * disk, and serializes {@link DownloadMetadata} as a protocol buffer. + * + * <p>Persistent destinations may support partial download resumption by implementing {@link + * #numExistingBytes} to return a non-zero value. + */ +public interface DownloadDestination { + /** + * How many bytes are already present for the destination. Used to avoid re-downloading data by + * using HTTP range downloads. + */ + long numExistingBytes() throws IOException; + + /** + * Reads metadata for the destination. The metadata is used to set request headers as appropriate + * to support range downloads and content-aware requests. + * + * <p>Note that this operation likely needs to read persistent state from disk to retrieve the + * metadata (e.g. a database read). An {@link IOException} may be thrown if an error is + * encountered. This method will always be called on the {@link java.util.concurrent.Executor} + * that is supplied to the downloader via {@link Downloader.Builder#withIOExecutor}. + */ + DownloadMetadata readMetadata() throws IOException; + + /** + * Opens the byte channel to write the download data to, with server-provided metadata to control + * how the destination should be initialized and stored. The downloader will close the channel + * when the download is complete. + * + * <p>The returned {@link WritableByteChannel} must have its {@code position} set to {@code + * byteOffset}. The underlying storage mechanism should also persist the metadata message, so that + * future calls to {@link #readMetadata} represent those values. Failure to do so may result in + * data corruption. + * + * @param byteOffset initial byte position for the returned channel. + * @param metadata metadata to configure the destination with + * @throws IllegalArgumentException if {@code byteOffset} is outside of range [0, {@link + * #numExistingBytes}] + */ + WritableByteChannel openByteChannel(long byteOffset, DownloadMetadata metadata) + throws IOException; + + /** + * Erases any existing bytes stored at this destination. + * + * <p>Implementations can either invalidate any channels previously returned by {@link + * #openByteChannel} or have them silently drop subsequent writes. + */ + void clear() throws IOException; +} diff --git a/src/main/java/com/google/android/downloader/DownloadException.java b/src/main/java/com/google/android/downloader/DownloadException.java new file mode 100644 index 0000000..f24172e --- /dev/null +++ b/src/main/java/com/google/android/downloader/DownloadException.java @@ -0,0 +1,36 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import javax.annotation.Nullable; + +/** Exception type for errors that can be returned by the downloader library. */ +public class DownloadException extends Exception { + public DownloadException() { + super(); + } + + public DownloadException(String message) { + super(message); + } + + public DownloadException(String message, @Nullable Throwable cause) { + super(message, cause); + } + + public DownloadException(@Nullable Throwable cause) { + super(cause); + } +} diff --git a/src/main/java/com/google/android/downloader/DownloadMetadata.java b/src/main/java/com/google/android/downloader/DownloadMetadata.java new file mode 100644 index 0000000..1837d57 --- /dev/null +++ b/src/main/java/com/google/android/downloader/DownloadMetadata.java @@ -0,0 +1,48 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import com.google.auto.value.AutoValue; + +/** + * Value interface for representing download metadata in-memory. Note that this is generalized so + * that the exact serialization mechanism used by a {@link DownloadDestination} implementation can + * be hidden from the downloader, as there are contexts where for example using a protocol buffer + * isn't desirable for dependency purposes. + */ +@AutoValue +public abstract class DownloadMetadata { + /** The HTTP content tag of the download, or the empty string if none was returned. */ + abstract String getContentTag(); + + /** + * The last modification timestamp of the download, in seconds since the UNIX epoch, or 0 if none + * was returned. + */ + abstract long getLastModifiedTimeSeconds(); + + /** Creates an empty instance of {@link DownloadMetadata}. */ + public static DownloadMetadata create() { + return new AutoValue_DownloadMetadata("", 0L); + } + + /** + * Creates an instance of {@link DownloadMetadata} with the provided content tag and modified + * timestamp. + */ + public static DownloadMetadata create(String contentTag, long lastModifiedTimeSeconds) { + return new AutoValue_DownloadMetadata(contentTag, lastModifiedTimeSeconds); + } +} diff --git a/src/main/java/com/google/android/downloader/DownloadRequest.java b/src/main/java/com/google/android/downloader/DownloadRequest.java new file mode 100644 index 0000000..d225b0d --- /dev/null +++ b/src/main/java/com/google/android/downloader/DownloadRequest.java @@ -0,0 +1,110 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import com.google.auto.value.AutoValue; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableMultimap; +import java.net.URI; +import javax.annotation.Nullable; + +/** + * Buildable value class used to construct and represent an individual download request. Instances + * are created via {@link Builder}, which in turn can be obtained via {@link + * Downloader2#newRequestBuilder}. Requests are executed via {@link Downloader2#execute}. + */ +@AutoValue +public abstract class DownloadRequest { + public abstract URI uri(); + + public abstract ImmutableMultimap<String, String> headers(); + + public abstract DownloadConstraints downloadConstraints(); + + public abstract @Nullable OAuthTokenProvider oAuthTokenProvider(); + + public abstract DownloadDestination destination(); + + @VisibleForTesting // Package-private outside of testing use + public static Builder newBuilder() { + return new AutoValue_DownloadRequest.Builder(); + } + + /** Builder facility for constructing instances of an {@link DownloadRequest}. */ + @AutoValue.Builder + public abstract static class Builder { + /** Sets the {@link DownloadDestination} that the download result should be sent to. */ + public abstract Builder setDestination(DownloadDestination destination); + + /** + * Sets the {@link URI} to download from. The URI scheme returned by {@link URI#getScheme} must + * be supported by one of the {@link com.google.android.libraries.net.urlengine.UrlEngine} + * instances attached to the downloader. + */ + public abstract Builder setUri(URI uri); + + /** + * Sets the URL to download from. This must be a valid URL string per RFC 2396; invalid strings + * will result in an {@link IllegalArgumentException} to be thrown from this method. + * + * <p>Internally this has the same effect as calling {@code setUri(URI.create(string))}, and + * exists here as a convenience method. + */ + public Builder setUrl(String url) { + return setUri(URI.create(url)); + } + + abstract ImmutableMultimap.Builder<String, String> headersBuilder(); + + /** + * Adds a request header to be sent as part of this request. Request headers are only used in + * {@link com.google.android.libraries.net.urlengine.UrlEngine} instances where they make sense + * (e.g. headers are attached to HTTP requests but ignored for Data URIs). + */ + public Builder addHeader(String key, String value) { + headersBuilder().put(key, value); + return this; + } + + /** + * Specifies the {@link com.google.android.libraries.net.downloader2.DownloadConstraints} for + * this request. + * + * <p>Note that the checking the download constraints for a request only occurs right before the + * request is executed, including cases where the request is retried due to a network error. + * + * <p>The ability to detect changes in connectivity depends on the underlying implementation of + * the {@link com.google.android.libraries.net.urlengine.UrlEngine} that processes this request. + * Some network stacks may be able to switch network types without interrupting the connection + * (e.g. seamless hand-off between WiFI and Cellular). The downloader relies on an interrupted + * connection to detect network changes, so in those cases the download may silently continue on + * the changed network. + * + * <p>The default value is {@link + * com.google.android.libraries.net.downloader2.DownloadConstraints#NETWORK_CONNECTED}. + */ + public abstract Builder setDownloadConstraints(DownloadConstraints downloadConstraints); + + /** + * Sets an optional {@link OAuthTokenProvider} on the request. When set, the downloader will + * consult the OAuthTokenProvider on this request and use it to attach "Authorization" headers + * to outgoing HTTP requests. + */ + public abstract Builder setOAuthTokenProvider( + @Nullable OAuthTokenProvider oAuthTokenProvider); + + public abstract DownloadRequest build(); + } +} diff --git a/src/main/java/com/google/android/downloader/DownloadResult.java b/src/main/java/com/google/android/downloader/DownloadResult.java new file mode 100644 index 0000000..67bed12 --- /dev/null +++ b/src/main/java/com/google/android/downloader/DownloadResult.java @@ -0,0 +1,38 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import com.google.auto.value.AutoValue; +import com.google.common.annotations.VisibleForTesting; + +/** + * Value representing the result of a {@link DownloadRequest}. Contains information about the + * completed download. + * + * <p>Note that this is only used to represent a successful download. A failed download will be + * represented as a failed {@link java.util.concurrent.Future} as returned by {@link + * Downloader#execute}, with the exception containing more information about the failure. + */ +@AutoValue +public abstract class DownloadResult { + + /** Returns the number of bytes written to this result. */ + public abstract long bytesWritten(); + + @VisibleForTesting // Package-private outside of testing use + public static DownloadResult create(long bytesWritten) { + return new AutoValue_DownloadResult(bytesWritten); + } +} diff --git a/src/main/java/com/google/android/downloader/Downloader.java b/src/main/java/com/google/android/downloader/Downloader.java new file mode 100644 index 0000000..c94c9ef --- /dev/null +++ b/src/main/java/com/google/android/downloader/Downloader.java @@ -0,0 +1,885 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.base.Predicates.instanceOf; +import static com.google.common.base.Throwables.getCausalChain; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; + +import com.google.auto.value.AutoValue; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import com.google.common.net.HttpHeaders; +import com.google.common.util.concurrent.ClosingFuture; +import com.google.common.util.concurrent.FluentFuture; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListenableFutureTask; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.errorprone.annotations.CheckReturnValue; +import com.google.errorprone.annotations.FormatMethod; +import com.google.errorprone.annotations.FormatString; +import com.google.errorprone.annotations.concurrent.GuardedBy; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URI; +import java.nio.channels.WritableByteChannel; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Queue; +import java.util.TimeZone; +import java.util.concurrent.CancellationException; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeoutException; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import javax.annotation.Nullable; + +/** + * In-process download system. Provides a light-weight mechanism to download resources over the + * network. Downloader includes the following features: + * + * <ul> + * <li>Configurable choice of network stack, with several defaults available out of the box. + * <li>Fully asynchronous behavior, allowing downloads to progress and complete without blocking + * (assuming the underlying network stack can avoid blocking). + * <li>Support for HTTP range requests, allowing partial downloads to resume mid-way through, and + * avoiding redownloading of fully-downloaded requests. + * <li>Detection of network interruptions, and a configurable way to retry requests that have + * failed after losing connectivity. + * </ul> + * + * <p>Note that because this library performs downloads in-process, it is subject to having + * downloads stall or abort when the app is suspended or killed. Download requests only live in + * memory, and thus are lost when the process ends. Library users should persist the set of + * downloads to be executed in persistent storage, such as a SQLite database, and run the downloads + * in the context of a persistent operation (either a foreground service or via Android's {@link + * android.app.job.JobScheduler} mechanism). + * + * <p>This is intended as a functional (but not drop-in) replacement for Android's {@link + * android.app.DownloadManager}, which has a number of issues: + * + * <ul> + * <li>It relies on the network stack built into Android, and thus cannot be patched to receive + * updates. For older versions of Android, that means the DownloadManager is vulnerable to + * issues in the network stack, such as b/18432707, which can result in MITM attacks. + * <li>When downloading to external storage on older versions of Android (via {@link + * android.app.DownloadManager.Request#setDestinationInExternalFilesDir} or {@link + * android.app.DownloadManager.Request#setDestinationInExternalPublicDir}), downloads may + * exceed the maximum size of the download directory (see b/22605830). + * <li>Downloads may pause and never resume again without external interference (b/18110151) + * <li>The DownloadManager may lose track of some files (b/18265497) + * </ul> + * + * <p>This library mitigates these issues by performing downloads in-process over the app-provided + * network stack, thus ensuring that an up-to-date network stack is used, and handing off storage + * management directly to the app without any additional restrictions. + */ +public class Downloader { + @VisibleForTesting static final int HTTP_RANGE_NOT_SATISFIABLE = 416; + @VisibleForTesting static final int HTTP_PARTIAL_CONTENT = 206; + + private static final ImmutableSet<String> SCHEMES_REQUIRING_CONNECTIVITY = + ImmutableSet.of("http", "https"); + + private static final Pattern CONTENT_RANGE_HEADER_PATTERN = + Pattern.compile("bytes (\\d+)-(\\d+)/(\\d+|\\*)"); + + @GuardedBy("SIMPLE_DATE_FORMAT_LOCK") + private static final SimpleDateFormat RFC_1123_FORMATTER; + + private static final Object SIMPLE_DATE_FORMAT_LOCK = new Object(); + + static { + synchronized (SIMPLE_DATE_FORMAT_LOCK) { + RFC_1123_FORMATTER = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss 'GMT'", Locale.US); + RFC_1123_FORMATTER.setTimeZone(TimeZone.getTimeZone("UTC")); + } + } + + /** Builder for configuring and constructing an instance of the Downloader. */ + public static class Builder { + private final Map<String, UrlEngine> urlEngineMap = new HashMap<>(); + private Executor ioExecutor; + private DownloaderLogger logger; + private ConnectivityHandler connectivityHandler; + private int maxConcurrentDownloads = 3; + + /** + * Creates a downloader builder. + * + * <p>Note that all parameters are required except for {@link #withMaxConcurrentDownloads}. + */ + public Builder() {} + + /** + * Specifies the executor to use internally for I/O. + * + * <p>I/O operations can block so don't use a direct executor or one that runs on the main + * thread. + */ + public Builder withIOExecutor(Executor ioExecutor) { + this.ioExecutor = ioExecutor; + return this; + } + + /** + * Sets the {@link ConnectivityHandler} to use in order to determine if connectivity is + * sufficient, and if not, how to handle it. + */ + public Builder withConnectivityHandler(ConnectivityHandler connectivityHandler) { + this.connectivityHandler = connectivityHandler; + return this; + } + + /** + * Limits the number of downloads in flight at a time. If a download request arrives that would + * exceed this limit, it will be queued until one already in flight completes. Beware that a + * download that is waiting for connectivity requirements is still considered to be in flight, + * so it is possible to saturate the downloader with requests waiting on connectivity + * requirements if the number of concurrent downloads isn't set high enough. + * + * <p>Note that other factors might restrict download concurrency even further, for instance the + * number of threads on the I/O executor when using a blocking engine. + */ + public Builder withMaxConcurrentDownloads(int maxConcurrentDownloads) { + checkArgument(maxConcurrentDownloads > 0); + this.maxConcurrentDownloads = maxConcurrentDownloads; + return this; + } + + /** + * Adds an {@link UrlEngine} to handle network requests for the given URL scheme. Note that the + * engine must support the provided scheme, and only one engine may ever be registered for a + * specific URL scheme. An {@link IllegalArgumentException} will be thrown if the engine doesn't + * support the scheme or if an engine is already registered for the scheme. + */ + public Builder addUrlEngine(String scheme, UrlEngine urlEngine) { + checkArgument( + urlEngine.supportedSchemes().contains(scheme), + "Provided UrlEngine must support URL scheme: %s", + scheme); + checkArgument( + !urlEngineMap.containsKey(scheme), + "Requested scheme already has a UrlEngine registered: %s", + scheme); + urlEngineMap.put(scheme, urlEngine); + return this; + } + + /** + * Adds an {@link UrlEngine} to handle network requests for the given URL schemes. Note that the + * engine must support the provided schemes, and only one engine may ever be registered for a + * specific URL scheme. An error will be thrown if the engine doesn't support the schemes or if + * an engine is already registered for the schemes. + */ + public Builder addUrlEngine(Iterable<String> schemes, UrlEngine urlEngine) { + for (String scheme : schemes) { + addUrlEngine(scheme, urlEngine); + } + return this; + } + + /** Sets the {@link DownloaderLogger} to use for logging in this downloader instance. */ + public Builder withLogger(DownloaderLogger logger) { + this.logger = logger; + return this; + } + + public Downloader build() { + return new Downloader(this); + } + } + + /** + * Value class for capturing a state snapshot of the downloader, for use in the state callbacks + * that can be registered via {@link #registerStateChangeCallback}. + */ + @AutoValue + public abstract static class State { + /** + * Returns the current number of downloads currently in flight, i.e. the number of downloads + * that are concurrently executing, or attempting to make progress on their underlying network + * stack. Note that this number should be in the range of [0, maxConcurrentDownloads), as + * configured by {@link Builder#withMaxConcurrentDownloads}. + */ + public abstract int getNumDownloadsInFlight(); + + /** + * Returns the number of downloads that have been requested via {@link #execute} but not yet + * started. They may not have been started due to internal asynchronous code, due to waiting on + * connectivity requirements, or due to the limit enforced by {@code maxConcurrentDownloads}. + */ + public abstract int getNumQueuedDownloads(); + + /** + * Returns the current number of downloads that are waiting for sufficient connectivity + * conditions to be met. These downloads will start running once the device network conditions + * change (e.g. connects to WiFi), if there is enough space as determined by {@code + * maxConcurrentDownloads}. + * + * <p>Note that this number should be in the range of [0, numQueuedDownloads], as a download + * pending connectivity is necessarily queued. However, a download may also be queued due to + * {@code maxConcurrentDownloads}. + */ + public abstract int getNumDownloadsPendingConnectivity(); + + /** Creates an instance of the state object. Internal-only. */ + @VisibleForTesting + public static State create( + int numDownloadsInFlight, int numQueuedDownloads, int numDownloadsPendingConnectivity) { + return new AutoValue_Downloader_State( + numDownloadsInFlight, numQueuedDownloads, numDownloadsPendingConnectivity); + } + } + + /** + * Functional callback interface for observing changes to Downloader {@link State}. Used with + * {@link #registerStateChangeCallback} and {@link #unregisterStateChangeCallback}. + * + * <p>Note that the callbacks could just use {@code java.util.function.Consumer} in contexts that + * support the Java 8 SDK, and {@code com.google.common.base.Receiver} in contexts that don't but + * have access to Guava APIs that aren't made public. + */ + public interface StateChangeCallback { + void onStateChange(State state); + } + + private final ImmutableMap<String, UrlEngine> urlEngineMap; + private final Executor ioExecutor; + private final DownloaderLogger logger; + private final ConnectivityHandler connectivityHandler; + private final int maxConcurrentDownloads; + + @GuardedBy("lock") + private final IdentityHashMap<StateChangeCallback, Executor> stateCallbackMap = + new IdentityHashMap<>(); + + // TODO: Consider using a PriorityQueue here instead, so that queued downloads can + // retain the order in the queue as they get added/removed due to waiting on connectivity. + @GuardedBy("lock") + private final Queue<QueuedDownload> queuedDownloads = new ArrayDeque<>(); + + @GuardedBy("lock") + private final List<FluentFuture<DownloadResult>> unresolvedFutures = new ArrayList<>(); + + private final Object lock = new Object(); + + @GuardedBy("lock") + private int numDownloadsInFlight = 0; + + @GuardedBy("lock") + private int numDownloadsPendingConnectivity = 0; + + private Downloader(Builder builder) { + ImmutableMap<String, UrlEngine> urlEngineMap = ImmutableMap.copyOf(builder.urlEngineMap); + checkArgument(!urlEngineMap.isEmpty(), "Must have at least one UrlEngine"); + checkArgument(builder.ioExecutor != null, "Must set a callback executor"); + checkArgument(builder.logger != null, "Must set a logger"); + checkArgument(builder.connectivityHandler != null, "Must set a connectivity handler"); + + this.urlEngineMap = urlEngineMap; + this.ioExecutor = builder.ioExecutor; + this.logger = builder.logger; + this.connectivityHandler = builder.connectivityHandler; + this.maxConcurrentDownloads = builder.maxConcurrentDownloads; + } + + /** + * Creates a new {@link DownloadRequest.Builder} for the given {@link URI} and {@link + * DownloadDestination}. The builder may be used to further customize the request. To execute the + * request, pass the built request to {@link #execute}. + */ + @CheckReturnValue + public DownloadRequest.Builder newRequestBuilder(URI uri, DownloadDestination destination) { + return DownloadRequest.newBuilder() + .setDestination(destination) + .setUri(uri) + .setDownloadConstraints(DownloadConstraints.NETWORK_CONNECTED); + } + + /** + * Executes the provided request. The request will be handled by the underlying {@link UrlEngine} + * and the result is streamed to the {@link DownloadDestination} created for the request. The + * download result is provided asynchronously as a {@link FluentFuture} that resolves when the + * download is complete. + * + * <p>The download can be cancelled by calling {@link FluentFuture#cancel} on the future instance + * returned by this method. Cancellation is best-effort and does not guarantee that the download + * will stop immediately, as it is impossible to stop a thread that is in the middle of reading + * bytes off the network. + * + * <p>Note that this method is not idempotent! The downloader does not attempt to merge/de-dupe + * incoming requests, even if the same exact request is executed twice. The calling code needs to + * be careful to manage its downloads in such a way that duplicated downloads don't occur. + * + * <p>TODO: Document what types of exceptions can be set on the returned future, and how clients + * are expected to handle them. + */ + @CheckReturnValue + public FluentFuture<DownloadResult> execute(DownloadRequest request) { + FluentFuture<DownloadResult> resultFuture; + + synchronized (lock) { + ClosingFuture<DownloadResult> closingFuture = enqueueRequest(request); + closingFuture.statusFuture().addListener(() -> onDownloadComplete(request), ioExecutor); + + resultFuture = closingFuture.finishToFuture(); + unresolvedFutures.add(resultFuture); + resultFuture.addListener( + () -> { + synchronized (lock) { + unresolvedFutures.remove(resultFuture); + } + }, + MoreExecutors.directExecutor()); + } + + logger.logFine("New request enqueued, running queue: %s", request.uri()); + maybeRunQueuedDownloads(); + return resultFuture; + } + + /** + * Cancels <strong>all</strong> ongoing downloads. This will result in all unresolved {@link + * FluentFuture} instances returned by {@link #execute} to be cancelled immediately and have their + * error callbacks invoked. + * + * <p>Because implementations of {@link FluentFuture} allow callbacks to be garbage collected + * after the future is resolved, calling {@code cancelAll} is an effective way to avoid having the + * Downloader leak memory after it is logically no longer needed, e.g. if it is only used from an + * Android {@link android.app.Activity}, and that Activity is destroyed. + * + * <p>However, in general the Downloader should be run from a process-level context, e.g. in an + * Android {@link android.app.Service}, so that the downloader doesn't implicitly hold on to + * UI-scoped objects. + */ + public void cancelAll() { + List<FluentFuture<DownloadResult>> unresolvedFuturesCopy; + + synchronized (lock) { + // Copy the set of unresolved futures to a local variable to avoid hitting + // ConcurrentModificationExceptions, which could happen since canceling the future may + // trigger the future callback in execute() that removes the future from the set of + // unresolved futures. + unresolvedFuturesCopy = new ArrayList<>(unresolvedFutures); + unresolvedFutures.clear(); + } + + for (FluentFuture<DownloadResult> unresolvedFuture : unresolvedFuturesCopy) { + unresolvedFuture.cancel(true); + } + } + + /** + * Registers an {@link StateChangeCallback} with this downloader instance, to be run when various + * state changes occur. The callback will be executed on the provided {@link Executor}. Use {@link + * #unregisterStateChangeCallback} to unregister a previously registered callback. The callback is + * invoked when the following state transitions occur: + * + * <ul> + * <li>A download was requested via a call to {@link #execute} + * <li>A download completed + * <li>A download started waiting for its connectivity constraints to be satisfied + * <li>A download stopped waiting for its connectivity constraints to be satisfied + * </ul> + * + * <p>A newly registered callback instance will only called for state changes that are triggered + * after the registration; there is no mechanism to replay previous state changes on the callback. + * + * <p>Registering the same callback multiple times has no effect, except that it will overwrite + * the {@link Executor} that was used in a previous registration call. + * + * <p>Invocation of the callbacks will be internally serialized to avoid concurrent invocations of + * the callback with possibly conflicting state. A new invocation of the callback will not start + * running until the previous one has completed. If multiple callbacks are being registered that + * must be synchronized with each other, then the caller must take care to coordinate this + * externally, such as locking in the callbacks or using an executor that guarantees + * serialization, such as {@link MoreExecutors#newSequentialExecutor}. + * + * <p>Warning: Do not use {@link MoreExecutors#directExecutor} or similar executor mechanisms, as + * doing so can easily lead to a deadlock, since the internal downloader lock is held while + * scheduling the callbacks on the provided executor. + */ + public void registerStateChangeCallback(StateChangeCallback callback, Executor executor) { + synchronized (lock) { + stateCallbackMap.put(callback, MoreExecutors.newSequentialExecutor(executor)); + } + } + + /** + * Unregisters a previously registered {@link StateChangeCallback} with this downloader instance. + * + * <p>Attempting to unregister a callback that was never registered is a no-op. + */ + public void unregisterStateChangeCallback(StateChangeCallback callback) { + synchronized (lock) { + stateCallbackMap.remove(callback); + } + } + + private void onDownloadComplete(DownloadRequest request) { + synchronized (lock) { + // The number of downloads should be well-balanced and this case should never + // trigger, so this is just a defensive check. + checkState( + numDownloadsInFlight >= 0, "Encountered < 0 downloads in flight, this shouldn't happen"); + } + + logger.logFine("Download complete, running queued downloads: %s", request.uri()); + maybeRunQueuedDownloads(); + } + + private void maybeRunQueuedDownloads() { + // Loop until we run out of download slots or out of queued downloads. It should be impossible + // for this to loop forever. Also, note that the synchronized block is inside the loop, since + // the outer loop conditions don't touch shared mutable state. + while (true) { + synchronized (lock) { + if (numDownloadsInFlight >= maxConcurrentDownloads) { + logger.logInfo("Exceeded max concurrent downloads, not running another queued request"); + return; + } + + QueuedDownload queuedDownload = queuedDownloads.poll(); + if (queuedDownload == null) { + return; + } + + DownloadRequest request = queuedDownload.request(); + ListenableFuture<Void> connectivityFuture = checkConnectivity(request); + if (connectivityFuture.isDone()) { + logger.logFine("Connectivity satisfied; running request. uri=%s", request.uri()); + numDownloadsInFlight++; + ListenableFuture<?> statusFuture = queuedDownload.resultFuture().statusFuture(); + statusFuture.addListener( + () -> { + synchronized (lock) { + numDownloadsInFlight--; + + // One less download in flight, let the state change listeners know. + runStateCallbacks(); + } + + // A download slot was just freed up, run queued downloads again. + logger.logFine( + "Queued download completed, running queued downloads: %s", request.uri()); + maybeRunQueuedDownloads(); + }, + ioExecutor); + + // A new download is about to be in flight, let state callbacks know about the + // state change. + runStateCallbacks(); + queuedDownload.task().run(); + } else { + logger.logInfo("Waiting on connectivity for request: uri=%s", request.uri()); + handlePendingConnectivity(connectivityFuture, queuedDownload); + } + } + } + } + + @GuardedBy("lock") + private void handlePendingConnectivity( + ListenableFuture<Void> connectivityFuture, QueuedDownload queuedDownload) { + // Keep track of the number of requests waiting. + numDownloadsPendingConnectivity++; + connectivityFuture.addListener( + () -> { + synchronized (lock) { + numDownloadsPendingConnectivity--; + + // Let the listeners know we are no longer waiting. + runStateCallbacks(); + } + }, + MoreExecutors.directExecutor()); + + // Let the listeners know we're waiting. + runStateCallbacks(); + + Futures.addCallback( + connectivityFuture, + new FutureCallback<Void>() { + @Override + public void onSuccess(Void result1) { + logger.logInfo("Connectivity changed, running queued requests"); + requeue(queuedDownload); + } + + @Override + public void onFailure(Throwable t) { + if (t instanceof TimeoutException) { + logger.logInfo("Timed out waiting for connectivity change"); + requeue(queuedDownload); + } else if (t instanceof CancellationException) { + logger.logFine("Connectivity future cancelled, running queued downloads"); + maybeRunQueuedDownloads(); + } else { + logger.logError(t, "Error observing connectivity changes"); + cancelAll(); + } + } + }, + ioExecutor); + + // Run state callbacks and cancel the connectivity future when the result task resolves. + // It doesn't matter if it succeeded or failed, either way it means we no longer need to wait + // for connectivity. + queuedDownload + .task() + .addListener( + () -> { + synchronized (lock) { + logger.logInfo("Queued task completed, cancelling connectivity check"); + runStateCallbacks(); + connectivityFuture.cancel(false); + } + }, + MoreExecutors.directExecutor()); + } + + private void requeue(QueuedDownload queuedDownload) { + synchronized (lock) { + addToQueue(queuedDownload); + } + + logger.logInfo( + "Requeuing download after connectivity change: %s", queuedDownload.request().uri()); + maybeRunQueuedDownloads(); + } + + private ClosingFuture<DownloadResult> enqueueRequest(DownloadRequest request) { + synchronized (lock) { + ListenableFutureTask<Void> task = ListenableFutureTask.create(() -> null); + // When the task runs (i.e. is taken off the queue and is explicitly run), all pre-flight + // checks should have been made, so at that point the request is send to the underlying + // network stack for execution. + ClosingFuture<DownloadResult> resultFuture = + ClosingFuture.from(task) + .transformAsync((closer, result) -> runRequest(request), ioExecutor); + addToQueue(QueuedDownload.create(request, task, resultFuture)); + return resultFuture; + } + } + + @GuardedBy("lock") + private void addToQueue(QueuedDownload queuedDownload) { + queuedDownloads.add(queuedDownload); + + // Make sure that when the task completes, the queued download is definitely removed + // from the queue. This is necessary to be robust in the face of cancellation, as + // canceled tasks may not get removed from the queue otherwise. + queuedDownload + .task() + .addListener( + () -> { + synchronized (lock) { + if (queuedDownloads.remove(queuedDownload)) { + // If the queued download was actually removed, update the state callbacks + // to reflect the state change. + runStateCallbacks(); + } + } + }, + MoreExecutors.directExecutor()); + + // Now that a new request is on the queue, run the state callbacks. + runStateCallbacks(); + } + + private ClosingFuture<DownloadResult> runRequest(DownloadRequest request) throws IOException { + URI uri = request.uri(); + UrlEngine urlEngine = checkNotNull(urlEngineMap.get(uri.getScheme())); + UrlRequest.Builder urlRequestBuilder = urlEngine.createRequest(uri.toString()); + for (Map.Entry<String, String> entry : request.headers().entries()) { + urlRequestBuilder.addHeader(entry.getKey(), entry.getValue()); + } + + long numExistingBytes = request.destination().numExistingBytes(); + if (numExistingBytes > 0) { + logger.logInfo( + "Existing bytes found. numExistingBytes=%d, uri=%s", numExistingBytes, request.uri()); + urlRequestBuilder.addHeader(HttpHeaders.RANGE, "bytes=" + numExistingBytes + "-"); + + DownloadMetadata destinationMetadata = request.destination().readMetadata(); + String contentTag = destinationMetadata.getContentTag(); + long lastModifiedTimeSeconds = destinationMetadata.getLastModifiedTimeSeconds(); + if (!contentTag.isEmpty()) { + urlRequestBuilder.addHeader(HttpHeaders.IF_RANGE, contentTag); + } else if (lastModifiedTimeSeconds > 0) { + urlRequestBuilder.addHeader( + HttpHeaders.IF_RANGE, formatRfc1123Date(lastModifiedTimeSeconds)); + } else { + // TODO: This should probably just clear the destination and remove the Range + // header so there's no chance of data corruption. Leaving this as-is for now to + // keep supporting range requests for offline maps. + logger.logWarning( + "Sending range request without If-Range header, due to missing destination " + + "metadata. Data corruption is possible."); + } + } + + ListenableFuture<UrlRequest> urlRequestFuture; + OAuthTokenProvider oAuthTokenProvider = request.oAuthTokenProvider(); + if (oAuthTokenProvider != null) { + urlRequestFuture = + Futures.transform( + oAuthTokenProvider.provideOAuthToken(uri), + authToken -> { + if (authToken != null) { + urlRequestBuilder.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + authToken); + } + return urlRequestBuilder.build(); + }, + ioExecutor); + } else { + urlRequestFuture = Futures.immediateFuture(urlRequestBuilder.build()); + } + + return ClosingFuture.from(urlRequestFuture) + .transform((closer, urlRequest) -> checkNotNull(urlRequest).send(), ioExecutor) + .transformAsync( + (closer, responseFuture) -> completeRequest(request, checkNotNull(responseFuture)), + ioExecutor); + } + + /** + * @param request the original request that triggered this call + * @param responseFuture the {@link UrlResponse}, provided asynchronously via a {@link + * ListenableFuture} + */ + private ClosingFuture<DownloadResult> completeRequest( + DownloadRequest request, ListenableFuture<UrlResponse> responseFuture) { + return ClosingFuture.from(responseFuture) + .transformAsync( + (closer, urlResponse) -> { + checkNotNull(urlResponse); + // We want to close the response regardless of whether we succeed or fail. + closer.eventuallyClose(urlResponse, ioExecutor); + logger.logFine( + "Got URL response, starting to read response body. uri=%s", request.uri()); + DownloadDestination destination = request.destination(); + if (request.headers().containsKey(HttpHeaders.RANGE) + && checkNotNull(urlResponse).getResponseCode() != HTTP_PARTIAL_CONTENT) { + logger.logFine("Clearing %s as our range request wasn't honored", destination); + destination.clear(); + } + WritableByteChannel byteChannel = + destination.openByteChannel( + parseResponseStartOffset(urlResponse), parseResponseMetadata(urlResponse)); + closer.eventuallyClose(byteChannel, ioExecutor); + return ClosingFuture.from(checkNotNull(urlResponse).readResponseBody(byteChannel)); + }, + ioExecutor) + .catching( + RequestException.class, + (closer, requestException) -> { + if (checkNotNull(requestException).getErrorDetails().getHttpStatusCode() + == HTTP_RANGE_NOT_SATISFIABLE) { + // This is a bit of a special edge case. Encountering this error means the server + // rejected our HTTP range request because it was outside the range of available + // bytes. This may well mean that the request was malformed (e.g. data on disk was + // corrupted and the existing file size ended up larger than what exists on the + // server). But the more common cause for this error is that the entire file was + // in fact already downloaded, so the requested range would cover 0 bytes, which + // the server interprets as not satisfiable. To mitigate that case we simply return + // a success state here by indicating 0 bytes were downloaded. + + // This isn't exactly ideal, as it means that a potential class of errors will + // go unnoticed. A better solution might find a way to distinguish between the + // common, file-already-downloaded case and the file corrupted case. This solution + // is put in place mainly to retain parity with the older downloader implementation. + return 0L; + } else { + throw new DownloadException(requestException); + } + }, + ioExecutor) + .transform( + (closer, bytesWritten) -> { + logger.logFine( + "Response body written. bytesWritten=%d, uri=%s", + checkNotNull(bytesWritten), request.uri()); + return DownloadResult.create(checkNotNull(bytesWritten)); + }, + ioExecutor) + .catchingAsync( + Exception.class, + (closer, exception) -> { + ClosingFuture<DownloadResult> result; + synchronized (lock) { + logger.logWarning( + exception, "Error reading download result. uri=%s", request.uri()); + RequestException requestException = getRequestException(exception); + if (requestException != null + && requestException.getErrorDetails().isRetryableAsIs()) { + // Retry the request by just re-enqueueing it. Note that we also need to + // call maybeRunQueuedRequest to keep the queue moving. Also, in this particular + // case we need to decrement the in-flight downloads count, as we are taking a + // previously in-flight download and putting it back in the queue without + // resolving the underlying result future. + numDownloadsInFlight--; + result = enqueueRequest(request); + } else { + throw new DownloadException(exception); + } + } + + logger.logInfo("Running queued downloads after handling request exception"); + maybeRunQueuedDownloads(); + return result; + }, + ioExecutor); + } + + @GuardedBy("lock") + private ListenableFuture<Void> checkConnectivity(DownloadRequest request) { + if (!SCHEMES_REQUIRING_CONNECTIVITY.contains(request.uri().getScheme())) { + return Futures.immediateVoidFuture(); + } + + return connectivityHandler.checkConnectivity(request.downloadConstraints()); + } + + @GuardedBy("lock") + private void runStateCallbacks() { + State state = + State.create(numDownloadsInFlight, queuedDownloads.size(), numDownloadsPendingConnectivity); + for (Map.Entry<StateChangeCallback, Executor> callbackEntry : stateCallbackMap.entrySet()) { + callbackEntry.getValue().execute(() -> callbackEntry.getKey().onStateChange(state)); + } + } + + private static String formatRfc1123Date(long unixTimeSeconds) { + synchronized (SIMPLE_DATE_FORMAT_LOCK) { + return RFC_1123_FORMATTER.format(new Date(SECONDS.toMillis(unixTimeSeconds))); + } + } + + private static long parseResponseStartOffset(UrlResponse response) throws DownloadException { + if (response.getResponseCode() != HttpURLConnection.HTTP_PARTIAL) { + return 0; + } + + List<String> contentRangeHeaders = response.getResponseHeaders().get(HttpHeaders.CONTENT_RANGE); + + checkDownloadState( + contentRangeHeaders != null && !contentRangeHeaders.isEmpty(), + "Host returned 206/PARTIAL response code but didn't provide a " + + "'Content-Range' response header"); + String contentRangeHeader = checkNotNull(contentRangeHeaders).get(0); + Matcher matcher = CONTENT_RANGE_HEADER_PATTERN.matcher(contentRangeHeader); + checkDownloadState( + matcher.matches() && matcher.groupCount() > 0, + "Content-Range response header didn't match expected pattern. " + "Was '%s', expected '%s'", + contentRangeHeader, + CONTENT_RANGE_HEADER_PATTERN.pattern()); + return Long.parseLong(checkNotNull(matcher.group(1))); + } + + private static DownloadMetadata parseResponseMetadata(UrlResponse response) + throws DownloadException { + String contentTag = parseResponseContentTag(response); + long lastModifiedTimeSeconds = parseResponseModifiedTime(response); + return DownloadMetadata.create(contentTag, lastModifiedTimeSeconds); + } + + private static long parseResponseModifiedTime(UrlResponse response) throws DownloadException { + List<String> lastModifiedHeaders = response.getResponseHeaders().get(HttpHeaders.LAST_MODIFIED); + if (lastModifiedHeaders == null || lastModifiedHeaders.isEmpty()) { + return 0L; + } + + String lastModifiedHeader = lastModifiedHeaders.get(0); + Date date; + + try { + synchronized (SIMPLE_DATE_FORMAT_LOCK) { + date = RFC_1123_FORMATTER.parse(lastModifiedHeader); + } + if (date == null) { + throw new DownloadException("Invalid Last-Modified header: " + lastModifiedHeader); + } + } catch (ParseException e) { + throw new DownloadException("Invalid Last-Modified header: " + lastModifiedHeader, e); + } + + return MILLISECONDS.toSeconds(date.getTime()); + } + + private static String parseResponseContentTag(UrlResponse response) { + List<String> contentTagHeaders = response.getResponseHeaders().get(HttpHeaders.ETAG); + if (contentTagHeaders == null || contentTagHeaders.isEmpty()) { + return ""; + } + + return contentTagHeaders.get(0); + } + + @AutoValue + abstract static class QueuedDownload { + abstract DownloadRequest request(); + + abstract ListenableFutureTask<?> task(); + + abstract ClosingFuture<DownloadResult> resultFuture(); + + static QueuedDownload create( + DownloadRequest request, + ListenableFutureTask<?> task, + ClosingFuture<DownloadResult> resultFuture) { + return new AutoValue_Downloader_QueuedDownload(request, task, resultFuture); + } + } + + @VisibleForTesting + @Nullable + static RequestException getRequestException(@Nullable Throwable throwable) { + if (throwable == null) { + return null; + } else { + return (RequestException) + Iterables.find( + getCausalChain(throwable), + instanceOf(RequestException.class), + /* defaultValue= */ null); + } + } + + @FormatMethod + private static void checkDownloadState( + boolean state, @FormatString String message, Object... args) throws DownloadException { + if (!state) { + throw new DownloadException(String.format(message, args)); + } + } +} diff --git a/src/main/java/com/google/android/downloader/DownloaderLogger.java b/src/main/java/com/google/android/downloader/DownloaderLogger.java new file mode 100644 index 0000000..cfd6f88 --- /dev/null +++ b/src/main/java/com/google/android/downloader/DownloaderLogger.java @@ -0,0 +1,43 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import com.google.errorprone.annotations.FormatMethod; +import com.google.errorprone.annotations.FormatString; +import javax.annotation.Nullable; + +/** + * Interface for a generic logging system, as used by the downloader. Allows the caller to configure + * the logging implementation as needed, and avoid extra dependencies in the downloader. + */ +public interface DownloaderLogger { + @FormatMethod + void logFine(@FormatString String message, Object... args); + + @FormatMethod + void logInfo(@FormatString String message, Object... args); + + @FormatMethod + void logWarning(@FormatString String message, Object... args); + + @FormatMethod + void logWarning(@Nullable Throwable cause, @FormatString String message, Object... args); + + @FormatMethod + void logError(@FormatString String message, Object... args); + + @FormatMethod + void logError(@Nullable Throwable cause, @FormatString String message, Object... args); +} diff --git a/src/main/java/com/google/android/downloader/ErrorDetails.java b/src/main/java/com/google/android/downloader/ErrorDetails.java new file mode 100644 index 0000000..3991b27 --- /dev/null +++ b/src/main/java/com/google/android/downloader/ErrorDetails.java @@ -0,0 +1,169 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import com.google.auto.value.AutoValue; +import com.google.common.base.Strings; +import com.google.common.net.HttpHeaders; +import java.net.HttpURLConnection; +import java.util.List; +import java.util.Map; +import javax.annotation.Nullable; + +/** Simple container object that contains information about a request error. */ +@AutoValue +public abstract class ErrorDetails { + // Additional HTTP status codes not listed in HttpURLConnection. + static final int HTTP_TOO_MANY_REQUESTS = 429; + + /** + * Returns the underlying numerical error value associated with this error. The meaning of this + * value depends on the network stack in use. Consult the network stack documentation to determine + * the meaning of this value. By default this returns the value 0. + */ + public abstract int getInternalErrorCode(); + + /** + * Returns the human-readable error message associated with this error. This message is for + * debugging purposes only and should not be parsed programmatically. By default this returns the + * empty string. + */ + public abstract String getErrorMessage(); + + /** + * Returns the HTTP status value associated with this error, if any. If the request succeeded but + * the server returned an error (e.g. server error 500), then this field can help classify the + * problem. If no HTTP status value is associated with this error, then the value -1 will be + * returned instead. + */ + public abstract int getHttpStatusCode(); + + /** + * Returns whether the request that triggered the error is retryable as-is without further changes + * to the request or state of the client. + * + * <p>Whether or not a request can be retried can depend on nuanced details of how an error + * occurred, so individual URL engines may make local decisions on whether an error should be + * retried. For example, TCP's connection reset error is often caused by a crash of the server + * during processing. Resending the exact same request, perhaps after some delay, has a good + * chance of hitting a different, healthy server and so in general this error is retryable. On the + * other hand, it doesn't make sense to spend resources retrying a HTTP_NOT_FOUND/404 since it + * isn't likely that the requested URL will spontaneously appear. A request that fails with + * HTTP_UNAUTHORIZED/401 is also not retryable under this definition. Although it makes sense to + * send an authenticated version of the failed request, this modification must happen at a higher + * layer. + */ + public abstract boolean isRetryableAsIs(); + + /** Creates a new builder instance for constructing request errors. */ + public static Builder builder() { + return new AutoValue_ErrorDetails.Builder() + .setInternalErrorCode(0) + .setErrorMessage("") + .setHttpStatusCode(-1) + .setRetryableAsIs(false); + } + + /** + * Create a new error instance for the given value and message. A convenience factory function for + * the most common use case. + */ + public static ErrorDetails create(@Nullable String message) { + return builder().setErrorMessage(Strings.nullToEmpty(message)).build(); + } + + /** + * Create a new error instance for an HTTP error response, as represented by the provided {@code + * httpResponseCode} and {@code httpResponseHeaders}. The canonical error code and retryability + * bit is computed based on the values of the response. + */ + public static ErrorDetails createFromHttpErrorResponse( + int httpResponseCode, + Map<String, List<String>> httpResponseHeaders, + @Nullable String message) { + return builder() + .setErrorMessage(Strings.nullToEmpty(message)) + .setRetryableAsIs(isRetryableHttpError(httpResponseCode, httpResponseHeaders)) + .setHttpStatusCode(httpResponseCode) + .setInternalErrorCode(httpResponseCode) + .build(); + } + + /** + * Obtains a connection error from a {@link Throwable}. If the throwable is an instance of {@link + * RequestException}, then it returns the error instance associated with that exception. + * Otherwise, a new connection error is constructed with default values and an error message set + * to the value returned by {@link Throwable#getMessage}. + */ + public static ErrorDetails fromThrowable(Throwable throwable) { + if (throwable instanceof RequestException) { + RequestException requestException = (RequestException) throwable; + return requestException.getErrorDetails(); + } else { + return builder().setErrorMessage(Strings.nullToEmpty(throwable.getMessage())).build(); + } + } + + /** + * Determine if a given HTTP error, as represented by an HTTP response code and response headers, + * is retryable. See the comment on {@link #isRetryableAsIs} for a longer explanation on how this + * related to the canonical error code. + */ + private static boolean isRetryableHttpError( + int httpCode, Map<String, List<String>> responseHeaders) { + switch (httpCode) { + case HttpURLConnection.HTTP_CLIENT_TIMEOUT: + // Client timeout means some client-side timeout was encountered. Retrying is safe. + return true; + case HttpURLConnection.HTTP_ENTITY_TOO_LARGE: + // Entity too large means the request was too large for the server to process. Retrying is + // safe if the server provided the retry-after header. + return responseHeaders.containsKey(HttpHeaders.RETRY_AFTER); + case HTTP_TOO_MANY_REQUESTS: + // Too many requests means the server is overloaded and is rejecting requests to temporarily + // reduce load. See go/rfc/6585. Retrying is safe. + return true; + case HttpURLConnection.HTTP_UNAVAILABLE: + // Unavailable means the server is currently unable to service the request. Retrying is + // safe if the server provided the retry-after header. + return responseHeaders.containsKey(HttpHeaders.RETRY_AFTER); + case HttpURLConnection.HTTP_GATEWAY_TIMEOUT: + // Gateway timeout means there was a server timeout somewhere. Retrying is safe. + return true; + default: + // By default, assume any other HTTP error is not retryable. + return false; + } + } + + /** Builder for creating instances of {@link ErrorDetails}. */ + @AutoValue.Builder + public abstract static class Builder { + /** Sets the error value. */ + public abstract Builder setInternalErrorCode(int internalErrorCode); + + /** Sets the error message. */ + public abstract Builder setErrorMessage(String errorMessage); + + /** Sets the http status value. */ + public abstract Builder setHttpStatusCode(int httpStatusCode); + + /** Sets whether the error is retryable as-is. */ + public abstract Builder setRetryableAsIs(boolean retryable); + + /** Builds the request error instance. */ + public abstract ErrorDetails build(); + } +} diff --git a/src/main/java/com/google/android/downloader/FloggerDownloaderLogger.java b/src/main/java/com/google/android/downloader/FloggerDownloaderLogger.java new file mode 100644 index 0000000..eeb0c4e --- /dev/null +++ b/src/main/java/com/google/android/downloader/FloggerDownloaderLogger.java @@ -0,0 +1,64 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import com.google.common.flogger.GoogleLogger; +import com.google.errorprone.annotations.FormatMethod; +import com.google.errorprone.annotations.FormatString; +import javax.annotation.Nullable; + +/** + * Implementation of {@link DownloaderLogger} backed by {@link GoogleLogger}. Google-internal code + * should always use this logger. + */ +public class FloggerDownloaderLogger implements DownloaderLogger { + private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); + + @Override + @FormatMethod + public void logFine(@FormatString String message, Object... args) { + logger.atFine().logVarargs(message, args); + } + + @Override + @FormatMethod + public void logInfo(@FormatString String message, Object... args) { + logger.atInfo().logVarargs(message, args); + } + + @Override + @FormatMethod + public void logWarning(@FormatString String message, Object... args) { + logger.atWarning().logVarargs(message, args); + } + + @Override + @FormatMethod + public void logWarning(@Nullable Throwable cause, @FormatString String message, Object... args) { + logger.atWarning().withCause(cause).logVarargs(message, args); + } + + @Override + @FormatMethod + public void logError(@FormatString String message, Object... args) { + logger.atSevere().logVarargs(message, args); + } + + @Override + @FormatMethod + public void logError(@Nullable Throwable cause, @FormatString String message, Object... args) { + logger.atSevere().withCause(cause).logVarargs(message, args); + } +} diff --git a/src/main/java/com/google/android/downloader/IOUtil.java b/src/main/java/com/google/android/downloader/IOUtil.java new file mode 100644 index 0000000..1461dd5 --- /dev/null +++ b/src/main/java/com/google/android/downloader/IOUtil.java @@ -0,0 +1,62 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import static com.google.common.base.Preconditions.checkState; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.SelectableChannel; +import java.nio.channels.WritableByteChannel; + +/** Package-local utilities for facilitating common I/O operations across engine implementations. */ +final class IOUtil { + + private IOUtil() {} + + /** + * Validates the given channel as a valid byte sink for the UrlEngine library. Specifically, it + * checks that the given channel is open, and if it is a {@link SelectableChannel}, then it must + * be in blocking mode. + * + * <p>This is to guard against having a sink that refuses write operations in a non-blocking + * manner by returning from {@link WritableByteChannel#write} immediately with a return value of + * 0, indicating no bytes were written. That would be problematic for the UrlEngine library + * because then threads will spin constantly trying to stream bytes from the source to the sink, + * saturating the CPU in the process. + * + * <p>TODO: Figure out a more robust way to handle this. Maybe writing implement proper support + * for selectable channels? + */ + static void validateChannel(WritableByteChannel channel) { + if (channel instanceof SelectableChannel) { + SelectableChannel selectableChannel = (SelectableChannel) channel; + checkState( + selectableChannel.isBlocking(), + "Target channels used by UrlEngine must be in blocking mode to ensure " + + "writes happen correctly; call SelectableChannel#configureBlocking(true)."); + } + + checkState(channel.isOpen()); + } + + static long blockingWrite(ByteBuffer source, WritableByteChannel target) throws IOException { + long written = 0; + while (source.hasRemaining()) { + written += target.write(source); + } + return written; + } +} diff --git a/src/main/java/com/google/android/downloader/OAuthTokenProvider.java b/src/main/java/com/google/android/downloader/OAuthTokenProvider.java new file mode 100644 index 0000000..dc0582c --- /dev/null +++ b/src/main/java/com/google/android/downloader/OAuthTokenProvider.java @@ -0,0 +1,28 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import com.google.common.util.concurrent.ListenableFuture; +import java.net.URI; + +/** + * Provider interface for supplying OAuth tokens for a request. The tokens generated by this method + * are added to requests as http headers. For more details, see the official spec for this + * authorization mechanism: https://tools.ietf.org/html/rfc6750#page-5 + */ +public interface OAuthTokenProvider { + /** Provides an OAuth bearer token for the given URI. */ + ListenableFuture<String> provideOAuthToken(URI uri); +} diff --git a/src/main/java/com/google/android/downloader/OkHttp2UrlEngine.java b/src/main/java/com/google/android/downloader/OkHttp2UrlEngine.java new file mode 100644 index 0000000..efbdb2e --- /dev/null +++ b/src/main/java/com/google/android/downloader/OkHttp2UrlEngine.java @@ -0,0 +1,216 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.util.concurrent.MoreExecutors.directExecutor; + +import com.google.common.collect.ImmutableMultimap; +import com.google.common.collect.ImmutableSet; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.SettableFuture; +import com.squareup.okhttp.Call; +import com.squareup.okhttp.Callback; +import com.squareup.okhttp.OkHttpClient; +import com.squareup.okhttp.Request; +import com.squareup.okhttp.Response; +import com.squareup.okhttp.ResponseBody; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.channels.Channels; +import java.nio.channels.WritableByteChannel; +import java.util.List; +import java.util.Map; +import java.util.Set; +import okio.Okio; +import okio.Sink; + +/** {@link UrlEngine} implementation that uses OkHttp3 for network connectivity. */ +public class OkHttp2UrlEngine implements UrlEngine { + private static final ImmutableSet<String> HTTP_SCHEMES = ImmutableSet.of("http", "https"); + + private final OkHttpClient client; + private final ListeningExecutorService transferExecutorService; + + /** + * Constructs an instance of the OkHttp URL engine, for the given OkHttpClient instance. + * + * <p>Note that due to how OkHttp is implemented, reads from the network are blocking operations, + * and thus threads in the provided {@link ListeningExecutorService} can be tied up for long + * periods of time waiting on network responses. To mitigate, set {@link + * OkHttpClient#setReadTimeout(long, java.util.concurrent.TimeUnit)} to a value that is reasonable + * for your use case. + * + * @param transferExecutorService Executor on which the requests are synchronously executed. + */ + public OkHttp2UrlEngine(OkHttpClient client, ListeningExecutorService transferExecutorService) { + checkNotNull(client.getDispatcher()); + this.client = client; + this.transferExecutorService = transferExecutorService; + } + + @Override + public UrlRequest.Builder createRequest(String url) { + return new OkHttpUrlRequestBuilder(url); + } + + @Override + public Set<String> supportedSchemes() { + return HTTP_SCHEMES; + } + + class OkHttpUrlRequestBuilder implements UrlRequest.Builder { + private final String url; + private final ImmutableMultimap.Builder<String, String> headers = ImmutableMultimap.builder(); + + OkHttpUrlRequestBuilder(String url) { + this.url = url; + } + + @Override + public UrlRequest.Builder addHeader(String key, String value) { + headers.put(key, value); + return this; + } + + @Override + public UrlRequest build() { + return new OkHttpUrlRequest(url, headers.build()); + } + } + + /** + * Implementation of {@link UrlRequest} for OkHttp. Wraps OkHttp's {@link Call} to make network + * requests. + */ + class OkHttpUrlRequest implements UrlRequest { + private final String url; + private final ImmutableMultimap<String, String> headers; + + OkHttpUrlRequest(String url, ImmutableMultimap<String, String> headers) { + this.url = url; + this.headers = headers; + } + + @Override + public ListenableFuture<UrlResponse> send() { + Request.Builder requestBuilder = new Request.Builder(); + + try { + requestBuilder.url(url); + } catch (IllegalArgumentException e) { + return Futures.immediateFailedFuture(new RequestException(e)); + } + + for (String key : headers.keys()) { + for (String value : headers.get(key)) { + requestBuilder.header(key, value); + } + } + + SettableFuture<UrlResponse> responseFuture = SettableFuture.create(); + Call call = client.newCall(requestBuilder.build()); + call.enqueue( + new Callback() { + @Override + public void onResponse(Response response) { + if (response.isSuccessful()) { + responseFuture.set(new OkHttpUrlResponse(response)); + } else { + responseFuture.setException( + new RequestException( + ErrorDetails.createFromHttpErrorResponse( + response.code(), response.headers().toMultimap(), response.message()))); + try { + response.body().close(); + } catch (IOException e) { + // Ignore, an exception on the future was already set. + } + } + } + + @Override + public void onFailure(Request request, IOException exception) { + responseFuture.setException(new RequestException(exception)); + } + }); + responseFuture.addListener( + () -> { + if (responseFuture.isCancelled()) { + call.cancel(); + } + }, + directExecutor()); + return responseFuture; + } + } + + /** + * Implementation of {@link UrlResponse} for OkHttp. Wraps OkHttp's {@link okhttp3.Response} to + * complete its operations. + */ + class OkHttpUrlResponse implements UrlResponse { + private final Response response; + + OkHttpUrlResponse(Response response) { + this.response = response; + } + + @Override + public int getResponseCode() { + return response.code(); + } + + @Override + public Map<String, List<String>> getResponseHeaders() { + return response.headers().toMultimap(); + } + + @Override + public ListenableFuture<Long> readResponseBody(WritableByteChannel destinationChannel) { + IOUtil.validateChannel(destinationChannel); + return transferExecutorService.submit( + () -> { + try (ResponseBody body = checkNotNull(response.body())) { + // Transfer the response body to the destination channel via OkHttp's Okio API. + // Sadly this needs to operate on OutputStream instead of Channels, but at least + // Okio manages buffers efficiently internally. + OutputStream outputStream = Channels.newOutputStream(destinationChannel); + Sink sink = Okio.sink(outputStream); + return body.source().readAll(sink); + } catch (IllegalStateException e) { + // OkHttp throws an IllegalStateException if the stream is closed while + // trying to write. Catch and rethrow. + throw new RequestException(e); + } catch (IOException e) { + if (e instanceof RequestException) { + throw e; + } else { + throw new RequestException(e); + } + } finally { + response.body().close(); + } + }); + } + + @Override + public void close() throws IOException { + response.body().close(); + } + } +} diff --git a/src/main/java/com/google/android/downloader/OkHttp3UrlEngine.java b/src/main/java/com/google/android/downloader/OkHttp3UrlEngine.java new file mode 100644 index 0000000..b51282e --- /dev/null +++ b/src/main/java/com/google/android/downloader/OkHttp3UrlEngine.java @@ -0,0 +1,212 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.util.concurrent.MoreExecutors.directExecutor; + +import com.google.common.collect.ImmutableMultimap; +import com.google.common.collect.ImmutableSet; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.SettableFuture; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.channels.Channels; +import java.nio.channels.WritableByteChannel; +import java.util.List; +import java.util.Map; +import java.util.Set; +import okhttp3.Call; +import okhttp3.Callback; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; +import okhttp3.ResponseBody; +import okio.Okio; +import okio.Sink; + +/** {@link UrlEngine} implementation that uses OkHttp3 for network connectivity. */ +public class OkHttp3UrlEngine implements UrlEngine { + private static final ImmutableSet<String> HTTP_SCHEMES = ImmutableSet.of("http", "https"); + + private final OkHttpClient client; + private final ListeningExecutorService transferExecutorService; + + /** + * Constructs an instance of the OkHttp URL engine, for the given OkHttpClient instance. + * + * <p>Note that due to how OkHttp is implemented, reads from the network are blocking operations, + * and thus threads in the provided {@link ListeningExecutorService} can be tied up for long + * periods of time waiting on network responses. To mitigate, set {@link + * OkHttpClient.Builder#readTimeout(long, java.util.concurrent.TimeUnit)} to a value that is + * reasonable for your use case. + * + * @param transferExecutorService Executor on which the requests are synchronously executed. + */ + public OkHttp3UrlEngine(OkHttpClient client, ListeningExecutorService transferExecutorService) { + checkNotNull(client.dispatcher()); + this.client = client; + this.transferExecutorService = transferExecutorService; + } + + @Override + public UrlRequest.Builder createRequest(String url) { + return new OkHttpUrlRequestBuilder(url); + } + + @Override + public Set<String> supportedSchemes() { + return HTTP_SCHEMES; + } + + class OkHttpUrlRequestBuilder implements UrlRequest.Builder { + private final String url; + private final ImmutableMultimap.Builder<String, String> headers = ImmutableMultimap.builder(); + + OkHttpUrlRequestBuilder(String url) { + this.url = url; + } + + @Override + public UrlRequest.Builder addHeader(String key, String value) { + headers.put(key, value); + return this; + } + + @Override + public UrlRequest build() { + return new OkHttpUrlRequest(url, headers.build()); + } + } + + /** + * Implementation of {@link UrlRequest} for OkHttp. Wraps OkHttp's {@link Call} to make network + * requests. + */ + class OkHttpUrlRequest implements UrlRequest { + private final String url; + private final ImmutableMultimap<String, String> headers; + + OkHttpUrlRequest(String url, ImmutableMultimap<String, String> headers) { + this.url = url; + this.headers = headers; + } + + @Override + public ListenableFuture<UrlResponse> send() { + Request.Builder requestBuilder = new Request.Builder(); + + try { + requestBuilder.url(url); + } catch (IllegalArgumentException e) { + return Futures.immediateFailedFuture(new RequestException(e)); + } + + for (String key : headers.keys()) { + for (String value : headers.get(key)) { + requestBuilder.header(key, value); + } + } + + SettableFuture<UrlResponse> responseFuture = SettableFuture.create(); + Call call = client.newCall(requestBuilder.build()); + call.enqueue( + new Callback() { + @Override + public void onResponse(Call call, Response response) { + if (response.isSuccessful()) { + responseFuture.set(new OkHttpUrlResponse(response)); + } else { + responseFuture.setException( + new RequestException( + ErrorDetails.createFromHttpErrorResponse( + response.code(), response.headers().toMultimap(), response.message()))); + response.close(); + } + } + + @Override + public void onFailure(Call call, IOException exception) { + responseFuture.setException(new RequestException(exception)); + } + }); + responseFuture.addListener( + () -> { + if (responseFuture.isCancelled()) { + call.cancel(); + } + }, + directExecutor()); + return responseFuture; + } + } + + /** + * Implementation of {@link UrlResponse} for OkHttp. Wraps OkHttp's {@link Response} to complete + * its operations. + */ + class OkHttpUrlResponse implements UrlResponse { + private final Response response; + + OkHttpUrlResponse(Response response) { + this.response = response; + } + + @Override + public int getResponseCode() { + return response.code(); + } + + @Override + public Map<String, List<String>> getResponseHeaders() { + return response.headers().toMultimap(); + } + + @Override + public ListenableFuture<Long> readResponseBody(WritableByteChannel destinationChannel) { + IOUtil.validateChannel(destinationChannel); + return transferExecutorService.submit( + () -> { + try (ResponseBody body = checkNotNull(response.body())) { + // Transfer the response body to the destination channel via OkHttp's Okio API. + // Sadly this needs to operate on OutputStream instead of Channels, but at least + // Okio manages buffers efficiently internally. + OutputStream outputStream = Channels.newOutputStream(destinationChannel); + Sink sink = Okio.sink(outputStream); + return body.source().readAll(sink); + } catch (IllegalStateException e) { + // OkHttp throws an IllegalStateException if the stream is closed while + // trying to write. Catch and rethrow. + throw new RequestException(e); + } catch (IOException e) { + if (e instanceof RequestException) { + throw e; + } else { + throw new RequestException(e); + } + } finally { + response.close(); + } + }); + } + + @Override + public void close() { + response.close(); + } + } +} diff --git a/src/main/java/com/google/android/downloader/PlatformUrlEngine.java b/src/main/java/com/google/android/downloader/PlatformUrlEngine.java new file mode 100644 index 0000000..b9357be --- /dev/null +++ b/src/main/java/com/google/android/downloader/PlatformUrlEngine.java @@ -0,0 +1,292 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableMultimap; +import com.google.common.collect.ImmutableSet; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLConnection; +import java.nio.ByteBuffer; +import java.nio.channels.Channels; +import java.nio.channels.ReadableByteChannel; +import java.nio.channels.WritableByteChannel; +import java.util.List; +import java.util.Map; +import java.util.Set; +import javax.annotation.Nullable; +import javax.annotation.concurrent.GuardedBy; + +/** + * {@link UrlEngine} implementation that uses Java's built-in network stack (i.e. {@link + * HttpURLConnection}). + * + * <p>Note that internally this engine allocates a 512kb direct byte buffer per request to transfer + * bytes around. If memory usage is a concern, then the number of concurrent requests should be + * limited. + */ +public final class PlatformUrlEngine implements UrlEngine { + @VisibleForTesting static final int BUFFER_SIZE_BYTES = 512 * 1024; + private static final ImmutableSet<String> SCHEMES = ImmutableSet.of("http", "https", "file"); + + private final ListeningExecutorService requestExecutorService; + private final int connectTimeoutsMs; + private final int readTimeoutsMs; + + public PlatformUrlEngine( + ListeningExecutorService requestExecutorService, int connectTimeoutMs, int readTimeoutMs) { + this.requestExecutorService = requestExecutorService; + this.connectTimeoutsMs = connectTimeoutMs; + this.readTimeoutsMs = readTimeoutMs; + } + + @Override + public UrlRequest.Builder createRequest(String url) { + return new PlatformUrlRequestBuilder(url); + } + + @Override + public Set<String> supportedSchemes() { + return SCHEMES; + } + + /** Implementation of {@link UrlRequest.Builder} for the built-in network stack. */ + class PlatformUrlRequestBuilder implements UrlRequest.Builder { + private final String url; + private final ImmutableMultimap.Builder<String, String> headers = + new ImmutableMultimap.Builder<>(); + + PlatformUrlRequestBuilder(String url) { + this.url = url; + } + + @Override + public UrlRequest.Builder addHeader(String key, String value) { + headers.put(key, value); + return this; + } + + @Override + public UrlRequest build() { + return new PlatformUrlRequest(url, headers.build()); + } + } + + /** + * Implementation of {@link UrlRequest} for the platform network stack. + * + * <p>The design of this class has some nuance in its design. Because HttpURLConnection isn't + * thread-safe, this implementation holds on to a single thread for the entire duration of an + * {@link HttpURLConnection} lifecycle - from connect until disconnect. + */ + class PlatformUrlRequest implements UrlRequest { + private final String url; + private final ImmutableMultimap<String, String> headers; + + PlatformUrlRequest(String url, ImmutableMultimap<String, String> headers) { + this.url = url; + this.headers = headers; + } + + @Override + public ListenableFuture<UrlResponse> send() { + return requestExecutorService.submit( + () -> { + URL url; + + try { + url = new URL(this.url); + } catch (MalformedURLException e) { + throw new RequestException(e); + } + + throwIfCancelled(); + + URLConnection urlConnection = null; + + try { + urlConnection = url.openConnection(); + urlConnection.setConnectTimeout(connectTimeoutsMs); + urlConnection.setReadTimeout(readTimeoutsMs); + + for (String key : headers.keySet()) { + for (String value : headers.get(key)) { + urlConnection.addRequestProperty(key, value); + } + } + + throwIfCancelled(); + + urlConnection.connect(); + + throwIfCancelled(); + + int httpResponseCode = getResponseCode(urlConnection); + + throwIfCancelled(); + + // We've successfully connected, so resolve the request to let client code decide + // what to do next. + PlatformUrlResponse response = + new PlatformUrlResponse(urlConnection, httpResponseCode); + urlConnection = null; + return response; + } catch (RequestException e) { + throw e; + } catch (IOException e) { + throw new RequestException(e); + } finally { + maybeDisconnect(urlConnection); + } + }); + } + } + + /** Implementation of {@link UrlResponse} for the platform network stack. */ + class PlatformUrlResponse implements UrlResponse { + @GuardedBy("this") + @Nullable + private URLConnection urlConnection; + + private final int httpResponseCode; + private final Map<String, List<String>> responseHeaders; + + PlatformUrlResponse(URLConnection urlConnection, int httpResponseCode) { + this.urlConnection = urlConnection; + this.httpResponseCode = httpResponseCode; + this.responseHeaders = urlConnection.getHeaderFields(); + } + + @Nullable + private synchronized URLConnection consumeConnection() { + URLConnection urlConnection = this.urlConnection; + this.urlConnection = null; + return urlConnection; + } + + @Override + public int getResponseCode() { + return httpResponseCode; + } + + @Override + public Map<String, List<String>> getResponseHeaders() { + return responseHeaders; + } + + @Override + public ListenableFuture<Long> readResponseBody(WritableByteChannel destinationChannel) { + IOUtil.validateChannel(destinationChannel); + return requestExecutorService.submit( + () -> { + URLConnection urlConnection = consumeConnection(); + if (urlConnection == null) { + throw new RequestException("URLConnection already closed"); + } + + try (ReadableByteChannel sourceChannel = + Channels.newChannel(urlConnection.getInputStream())) { + ByteBuffer buffer = ByteBuffer.allocateDirect(BUFFER_SIZE_BYTES); + long written = 0; + + while (sourceChannel.read(buffer) != -1) { + throwIfCancelled(); + buffer.flip(); + written += IOUtil.blockingWrite(buffer, destinationChannel); + buffer.clear(); + throwIfCancelled(); + } + + return written; + } catch (IOException e) { + throw new RequestException(e); + } finally { + maybeDisconnect(urlConnection); + } + }); + } + + @Override + public void close() throws IOException { + URLConnection urlConnection = consumeConnection(); + if (urlConnection != null) { + // At this point, both HttpURLConnection.getResponseCode and URLConnection.getHeaderFields + // have been called, so the InputStream has already been implicitly created, and the call + // to URLConnection.getInputStream will be cheap. Normally calling it can be pretty heavy- + // weight and thus likely shouldn't happen in the close() method. + urlConnection.getInputStream().close(); + maybeDisconnect(urlConnection); + } + } + } + + private static void throwIfCancelled() throws RequestException { + // ListeningExecutorService turns Future.cancel() into Thread.interrupt() + if (Thread.interrupted()) { + throw new RequestException("Request canceled"); + } + } + + private static void maybeDisconnect(@Nullable URLConnection urlConnection) { + if (urlConnection == null) { + return; + } + + if (urlConnection instanceof HttpURLConnection) { + HttpURLConnection httpUrlConnection = (HttpURLConnection) urlConnection; + httpUrlConnection.disconnect(); + } + } + + private static int getResponseCode(URLConnection urlConnection) throws IOException { + if (urlConnection instanceof HttpURLConnection) { + HttpURLConnection httpUrlConnection = (HttpURLConnection) urlConnection; + InputStream inputStream = getInputStream(httpUrlConnection); + int httpResponseCode = httpUrlConnection.getResponseCode(); + if (httpResponseCode >= HttpURLConnection.HTTP_BAD_REQUEST) { + String responseMessage = httpUrlConnection.getResponseMessage(); + Map<String, List<String>> responseHeaders = httpUrlConnection.getHeaderFields(); + if (inputStream != null) { + inputStream.close(); + } + throw new RequestException( + ErrorDetails.createFromHttpErrorResponse( + httpResponseCode, responseHeaders, responseMessage)); + } + return httpResponseCode; + } else { + // Note: This happens for URLConnections that aren't over HTTP, e.g. to + // file URLs instead (e.g. sun.net.www.protocol.file.FileURLConnection). The + // code doesn't directly check for these classes because they aren't officially + // part of the JDK. + return HttpURLConnection.HTTP_OK; + } + } + + @Nullable + private static InputStream getInputStream(HttpURLConnection httpURLConnection) { + try { + return httpURLConnection.getInputStream(); + } catch (IOException e) { + return null; + } + } +} diff --git a/src/main/java/com/google/android/downloader/ProtoFileDownloadDestination.java b/src/main/java/com/google/android/downloader/ProtoFileDownloadDestination.java new file mode 100644 index 0000000..43b8ed0 --- /dev/null +++ b/src/main/java/com/google/android/downloader/ProtoFileDownloadDestination.java @@ -0,0 +1,97 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import static com.google.common.base.Preconditions.checkState; + +import com.google.common.io.Files; +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.channels.FileChannel; +import java.nio.channels.WritableByteChannel; + +/** + * Basic implementation of {@link DownloadDestination} which streams the download to a {@link File}. + * + * <p>This implementation also keeps track of metadata in a separate file, encoded as a protocol + * buffer message. Note that this implementation isn't especially robust - concurrent reads and + * writes could result in errors or in data corruption, and invalid/corrupt metadata will result in + * persistent errors. + */ +public class ProtoFileDownloadDestination implements DownloadDestination { + private final File targetFile; + private final File metadataFile; + + public ProtoFileDownloadDestination(File targetFile, File metadataFile) { + this.targetFile = targetFile; + this.metadataFile = metadataFile; + } + + @Override + public long numExistingBytes() throws IOException { + return targetFile.length(); + } + + @Override + public DownloadMetadata readMetadata() throws IOException { + return metadataFile.exists() + ? readMetadataFromBytes(Files.toByteArray(metadataFile)) + : DownloadMetadata.create(); + } + + @Override + public WritableByteChannel openByteChannel(long offsetBytes, DownloadMetadata metadata) + throws IOException { + checkState( + offsetBytes <= targetFile.length(), + "Opening byte channel with offset past known end of file"); + Files.write(writeMetadataToBytes(metadata), metadataFile); + FileChannel fileChannel = new RandomAccessFile(targetFile, "rw").getChannel(); + // Seek to the requested offset, so we can append data rather than overwrite data. + fileChannel.position(offsetBytes); + return fileChannel; + } + + @Override + public void clear() throws IOException { + if (targetFile.exists() && !targetFile.delete()) { + throw new IOException("Failed to delete()"); + } + } + + @Override + public String toString() { + return targetFile.toString(); + } + + private static DownloadMetadata readMetadataFromBytes(byte[] bytes) throws IOException { + DownloadMetadataProto proto = DownloadMetadataProto.parseFrom(bytes); + return DownloadMetadata.create(proto.getContentTag(), proto.getLastModifiedTimeSeconds()); + } + + private static byte[] writeMetadataToBytes(DownloadMetadata metadata) { + DownloadMetadataProto.Builder builder = DownloadMetadataProto.newBuilder(); + + if (!metadata.getContentTag().isEmpty()) { + builder.setContentTag(metadata.getContentTag()); + } + if (metadata.getLastModifiedTimeSeconds() > 0) { + builder.setLastModifiedTimeSeconds(metadata.getLastModifiedTimeSeconds()); + } + + return builder.build().toByteArray(); + } +} diff --git a/src/main/java/com/google/android/downloader/RequestException.java b/src/main/java/com/google/android/downloader/RequestException.java new file mode 100644 index 0000000..529e1c7 --- /dev/null +++ b/src/main/java/com/google/android/downloader/RequestException.java @@ -0,0 +1,53 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import java.io.IOException; + +/** + * An exception that occurred during creation or execution of a request; contains {@link + * ErrorDetails} for more detail on the error. + */ +public final class RequestException extends IOException { + private final ErrorDetails errorDetails; + + public RequestException(ErrorDetails errorDetails) { + this.errorDetails = errorDetails; + } + + public RequestException(ErrorDetails errorDetails, Throwable cause) { + super(cause); + this.errorDetails = errorDetails; + } + + public RequestException(Throwable cause) { + super(cause); + this.errorDetails = ErrorDetails.create(cause.getMessage()); + } + + public RequestException(String message) { + super(message); + this.errorDetails = ErrorDetails.create(message); + } + + public ErrorDetails getErrorDetails() { + return errorDetails; + } + + @Override + public String getMessage() { + return super.getMessage() + "; " + errorDetails; + } +} diff --git a/src/main/java/com/google/android/downloader/SimpleFileDownloadDestination.java b/src/main/java/com/google/android/downloader/SimpleFileDownloadDestination.java new file mode 100644 index 0000000..ddc5169 --- /dev/null +++ b/src/main/java/com/google/android/downloader/SimpleFileDownloadDestination.java @@ -0,0 +1,103 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import static com.google.common.base.Preconditions.checkState; + +import com.google.common.io.Files; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.channels.FileChannel; +import java.nio.channels.WritableByteChannel; + +/** + * Basic implementation of {@link DownloadDestination} which streams the download to a {@link File}. + * + * <p>This implementation also keeps track of metadata in a separate file, encoded using + * hand-written serialization. Note that this implementation isn't especially robust - concurrent + * reads and writes could result in errors or in data corruption, and invalid/corrupt metadata will + * result in persistent errors. + */ +public class SimpleFileDownloadDestination implements DownloadDestination { + private final File targetFile; + private final File metadataFile; + + public SimpleFileDownloadDestination(File targetFile, File metadataFile) { + this.targetFile = targetFile; + this.metadataFile = metadataFile; + } + + @Override + public long numExistingBytes() throws IOException { + return targetFile.length(); + } + + @Override + public DownloadMetadata readMetadata() throws IOException { + return metadataFile.exists() + ? readMetadataFromBytes(Files.toByteArray(metadataFile)) + : DownloadMetadata.create(); + } + + @Override + public WritableByteChannel openByteChannel(long offsetBytes, DownloadMetadata metadata) + throws IOException { + checkState( + offsetBytes <= targetFile.length(), + "Opening byte channel with offset past known end of file"); + Files.write(writeMetadataToBytes(metadata), metadataFile); + FileChannel fileChannel = new RandomAccessFile(targetFile, "rw").getChannel(); + // Seek to the requested offset, so we can append data rather than overwrite data. + fileChannel.position(offsetBytes); + return fileChannel; + } + + @Override + public void clear() throws IOException { + if (targetFile.exists() && !targetFile.delete()) { + throw new IOException("Failed to delete()"); + } + } + + @Override + public String toString() { + return targetFile.toString(); + } + + private static DownloadMetadata readMetadataFromBytes(byte[] bytes) throws IOException { + try (DataInputStream inputStream = new DataInputStream(new ByteArrayInputStream(bytes))) { + String etag = inputStream.readUTF(); + long lastModifiedTimeSeconds = inputStream.readLong(); + checkState(lastModifiedTimeSeconds >= 0); + return DownloadMetadata.create(etag, lastModifiedTimeSeconds); + } + } + + private static byte[] writeMetadataToBytes(DownloadMetadata metadata) throws IOException { + try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + DataOutputStream dataOutputStream = new DataOutputStream(byteArrayOutputStream)) { + dataOutputStream.writeUTF(metadata.getContentTag()); + dataOutputStream.writeLong(metadata.getLastModifiedTimeSeconds()); + dataOutputStream.flush(); + byteArrayOutputStream.flush(); + return byteArrayOutputStream.toByteArray(); + } + } +} diff --git a/src/main/java/com/google/android/downloader/UrlEngine.java b/src/main/java/com/google/android/downloader/UrlEngine.java new file mode 100644 index 0000000..7d35245 --- /dev/null +++ b/src/main/java/com/google/android/downloader/UrlEngine.java @@ -0,0 +1,37 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import java.util.Set; + +/** Abstraction of a request engine, which is a high-level interface into a network stack. */ +public interface UrlEngine { + /** + * Creates a request for the given URL. The request is returned as a {@link UrlRequest.Builder}, + * so that callers may further customize the request. Callers must call {@link UrlRequest#send} to + * execute the request. + * + * @param url the URL to connect to. Internal code will assume this is a valid, well-formed URL, + * so client code must have already verified the URL. + */ + UrlRequest.Builder createRequest(String url); + + /** + * Returns the set of URL schemes supported by this url request factory instance. Network stacks + * that connect over HTTP will likely return the set {"http", "https"}, but there may be many + * other types of schemes available (e.g. "file" or "data"). + */ + Set<String> supportedSchemes(); +} diff --git a/src/main/java/com/google/android/downloader/UrlRequest.java b/src/main/java/com/google/android/downloader/UrlRequest.java new file mode 100644 index 0000000..43f5285 --- /dev/null +++ b/src/main/java/com/google/android/downloader/UrlRequest.java @@ -0,0 +1,61 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import com.google.common.util.concurrent.ListenableFuture; + +/** + * Interface representing a request to a URL. Intended for use with a {@link UrlEngine}. + * + * <p>Note: Unless explicitly noted, instances of this object are designed to be thread-safe, + * meaning that methods can be called from any thread. It is however not advisable to use instances + * of this object as a lock, as internal logic may synchronize on the request instance. Also, + * methods should not be called from the Android main/UI thread, as some I/O may occur. + */ +public interface UrlRequest { + /** Builder interface for constructing URL requests. */ + interface Builder { + /** Adds an individual HTTP header for this request. */ + default Builder addHeader(String key, String value) { + return this; + } + + /** Builds the URL request. */ + UrlRequest build(); + } + + /** + * Executes this request by connecting to the URL represented by this request. Returns + * immediately, with the {@link UrlResponse} becoming available asynchronously as a {@link + * ListenableFuture}. May only be called once; multiple calls result in undefined behavior. + * + * <p>If the request fails due to I/O errors or due to an error response from the server (e.g. + * http status codes in the 400s or 500s), then the future will resolve with either an {@link + * RequestException} or an {@link java.io.IOException}. Any other type of exception (e.g. {@link + * IllegalStateException}) indicates an unexpected error occurred, and such errors should likely + * be reported at a severe level and possibly even crash the app. + * + * <p>To cancel an executed requested, call {@link ListenableFuture#cancel} on the future returned + * by this method. Cancellation is opportunistic and best-effort; calling will not guarantee that + * no more network activity will occur, as it is not possible to immediately stop a thread that is + * transferring bytes from the network. Cancellation may fail internally; observe the resolution + * of the future to capture such failures. + * + * <p>Note that although this method returns its result asynchronously and doesn't block on the + * request, the operation may still involve performing some I/O (e.g. verifying the existence of a + * file). Do not call this on the UI thread! + */ + ListenableFuture<UrlResponse> send(); +} diff --git a/src/main/java/com/google/android/downloader/UrlResponse.java b/src/main/java/com/google/android/downloader/UrlResponse.java new file mode 100644 index 0000000..c1a355c --- /dev/null +++ b/src/main/java/com/google/android/downloader/UrlResponse.java @@ -0,0 +1,51 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import com.google.common.util.concurrent.ListenableFuture; +import java.io.Closeable; +import java.nio.channels.WritableByteChannel; +import java.util.List; +import java.util.Map; + +/** + * Interface representing a response to a URL request. + * + * <p>Note that this extends from {@link Closeable}. Callers that successfully obtain an instance of + * the response (via the future returned by {@link UrlRequest#send}) must take care to close the + * response once done using it. + */ +public interface UrlResponse extends Closeable { + /** + * The HTTP status code returned by the server. Returns -1 if the response code can't be discerned + * or doesn't make sense for this response implementation, as mentioned in the javadocs for {@link + * java.net.HttpURLConnection#getResponseCode} + */ + int getResponseCode(); + + /** The multi-valued HTTP response headers returned by the server. */ + Map<String, List<String>> getResponseHeaders(); + + /** + * Writes the response body to the provided {@link WritableByteChannel}. The channel must be open + * prior to calling this method. + * + * <p>This method may only be called once for a given response! Attempting to call this method + * multiple times results in undefined behavior. + * + * @return future that resolves to the number of bytes written to the channel. + */ + ListenableFuture<Long> readResponseBody(WritableByteChannel destinationChannel); +} diff --git a/src/main/proto/download_metadata.proto b/src/main/proto/download_metadata.proto new file mode 100644 index 0000000..3a1ae32 --- /dev/null +++ b/src/main/proto/download_metadata.proto @@ -0,0 +1,39 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto2"; + +package android.downloader; + +option java_multiple_files = true; +option java_package = "com.google.android.downloader"; +option java_outer_classname = "DownloadMetadataProtoOuter"; + +// Message encoding download metadata. May be persisted by implementations of +// DownloadDestination (e.g. ProtoFileDownloadDestination) so that interrupted +// downloads can be safely resumed. +message DownloadMetadataProto { + // The opaque content tag associated with the download. Commonly this + // is a checksum or hash of the actual data. However, the client should never + // attempt to infer any properties about this string, and should instead + // treat it as a token to be persisted and returned unaltered. This tag + // is used to ensure the underlying HTTP resource did not change across + // requests. + optional string content_tag = 1; + + // The UNIX epoch timestamp, in seconds, for when the resource the download + // targets was last modified. This timestamp is used to determine if the + // underlying HTTP resource changed across requests. + optional int64 last_modified_time_seconds = 2; +}
\ No newline at end of file diff --git a/src/test/AndroidManifest.xml b/src/test/AndroidManifest.xml new file mode 100644 index 0000000..c7ea9bd --- /dev/null +++ b/src/test/AndroidManifest.xml @@ -0,0 +1,34 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- + Copyright 2021 The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<manifest xmlns:android="http://schemas.android.com/apk/res/android" + package="com.google.android.downloader" + android:versionCode="1" + android:versionName="1.0" > + + <uses-permission android:name="android.permission.INTERNET"/> + <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" /> + <uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" /> + + <uses-sdk + android:minSdkVersion="19" + android:targetSdkVersion="24" /> + + <application> + <activity android:name=".CronetUrlEngineTestActivity" /> + </application> +</manifest>
\ No newline at end of file diff --git a/src/test/AndroidTestManifest.xml b/src/test/AndroidTestManifest.xml new file mode 100644 index 0000000..720ee51 --- /dev/null +++ b/src/test/AndroidTestManifest.xml @@ -0,0 +1,38 @@ +<?xml version="1.0" encoding="utf-8"?> +<!-- + Copyright 2021 The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<manifest xmlns:android="http://schemas.android.com/apk/res/android" + package="com.google.android.downloader.test" + android:versionCode="1" + android:versionName="1.0" > + + <uses-sdk + android:minSdkVersion="19" + android:targetSdkVersion="24" /> + + <application> + <uses-library android:name="android.test.runner" /> + </application> + + <!-- + Needed for coverage in google3 to work. In open source code, this should be + androidx.test.runner.AndroidJUnitRunner instead. + --> + <instrumentation + android:name="com.google.android.apps.common.testing.testrunner.Google3InstrumentationTestRunner" + android:targetPackage="com.google.android.downloader" /> +</manifest> diff --git a/src/test/java/com/google/android/downloader/AndroidConnectivityHandlerTest.java b/src/test/java/com/google/android/downloader/AndroidConnectivityHandlerTest.java new file mode 100644 index 0000000..247080b --- /dev/null +++ b/src/test/java/com/google/android/downloader/AndroidConnectivityHandlerTest.java @@ -0,0 +1,331 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import static android.net.ConnectivityManager.CONNECTIVITY_ACTION; +import static android.os.Looper.getMainLooper; +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; +import static org.robolectric.Shadows.shadowOf; + +import android.Manifest.permission; +import android.app.Application; +import android.content.Context; +import android.content.Intent; +import android.net.ConnectivityManager; +import android.net.Network; +import android.net.NetworkCapabilities; +import android.net.NetworkInfo.DetailedState; +import android.net.NetworkInfo.State; +import android.os.Build.VERSION_CODES; +import androidx.test.core.app.ApplicationProvider; +import com.google.android.downloader.DownloadConstraints.NetworkType; +import com.google.common.collect.ImmutableSet; +import com.google.common.util.concurrent.ListenableFuture; +import java.time.Duration; +import java.util.EnumSet; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeoutException; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.robolectric.RobolectricTestRunner; +import org.robolectric.annotation.Config; +import org.robolectric.shadows.ShadowApplication; +import org.robolectric.shadows.ShadowConnectivityManager; +import org.robolectric.shadows.ShadowNetwork; +import org.robolectric.shadows.ShadowNetworkCapabilities; +import org.robolectric.shadows.ShadowNetworkInfo; + +/** Unit tests for AndroidConnectivityHandler. */ +@RunWith(RobolectricTestRunner.class) +@Config(sdk = VERSION_CODES.KITKAT) +public final class AndroidConnectivityHandlerTest { + private static final long DEFAULT_TIMEOUT_MILLIS = 1000; + + @Rule + public TestExecutorRule testExecutorRule = + new TestExecutorRule(Duration.ofMillis(2 * DEFAULT_TIMEOUT_MILLIS)); + + private Application application; + private ConnectivityManager connectivityManager; + private ScheduledExecutorService scheduledExecutorService; + + @Before + public void setUp() { + application = ApplicationProvider.getApplicationContext(); + scheduledExecutorService = testExecutorRule.newSingleThreadScheduledExecutor(); + connectivityManager = + (ConnectivityManager) application.getSystemService(Context.CONNECTIVITY_SERVICE); + } + + @Test + public void checkConnectivity_permissionNotGranted() { + assertThrows(IllegalStateException.class, this::createConnectivityHandler); + } + + @Test + public void checkConnectivity_noConnectivityRequired() throws Exception { + shadowOf(application).grantPermissions(permission.ACCESS_NETWORK_STATE); + + AndroidConnectivityHandler connectivityHandler = createConnectivityHandler(); + ListenableFuture<Void> result = connectivityHandler.checkConnectivity(DownloadConstraints.NONE); + + assertThat(result.isDone()).isTrue(); + assertThat(result.get()).isNull(); + } + + @Test + public void checkConnectivity_noNetwork() { + shadowOf(application).grantPermissions(permission.ACCESS_NETWORK_STATE); + + AndroidConnectivityHandler connectivityHandler = createConnectivityHandler(); + + shadowOf(connectivityManager).setActiveNetworkInfo(null); + + ListenableFuture<Void> result = + connectivityHandler.checkConnectivity(DownloadConstraints.NETWORK_CONNECTED); + + assertThat(result.isDone()).isFalse(); + } + + @Test + public void checkConnectivity_networkNotConnected() { + shadowOf(application).grantPermissions(permission.ACCESS_NETWORK_STATE); + + AndroidConnectivityHandler connectivityHandler = createConnectivityHandler(); + + shadowOf(connectivityManager) + .setActiveNetworkInfo( + ShadowNetworkInfo.newInstance( + DetailedState.DISCONNECTED, + ConnectivityManager.TYPE_WIFI, + 0, + false, + State.DISCONNECTED)); + + ListenableFuture<Void> result = + connectivityHandler.checkConnectivity(DownloadConstraints.NETWORK_CONNECTED); + + assertThat(result.isDone()).isFalse(); + } + + @Test + public void checkConnectivity_wrongNetworkType() { + shadowOf(application).grantPermissions(permission.ACCESS_NETWORK_STATE); + + AndroidConnectivityHandler connectivityHandler = createConnectivityHandler(); + + shadowOf(connectivityManager) + .setActiveNetworkInfo( + ShadowNetworkInfo.newInstance( + DetailedState.CONNECTED, + ConnectivityManager.TYPE_MOBILE, + 0, + true, + State.CONNECTED)); + + ListenableFuture<Void> result = + connectivityHandler.checkConnectivity(DownloadConstraints.NETWORK_UNMETERED); + + assertThat(result.isDone()).isFalse(); + } + + @Test + public void checkConnectivity_anyNetworkType() { + shadowOf(application).grantPermissions(permission.ACCESS_NETWORK_STATE); + + AndroidConnectivityHandler connectivityHandler = createConnectivityHandler(); + + shadowOf(connectivityManager) + .setActiveNetworkInfo( + ShadowNetworkInfo.newInstance( + DetailedState.CONNECTED, + ConnectivityManager.TYPE_MOBILE, + 0, + true, + State.CONNECTED)); + + ListenableFuture<Void> result = + connectivityHandler.checkConnectivity( + DownloadConstraints.builder() + .setRequireUnmeteredNetwork(false) + .setRequiredNetworkTypes(ImmutableSet.of(NetworkType.ANY)) + .build()); + assertThat(result.isDone()).isTrue(); + } + + @Test + public void checkConnectivity_unknownNetworkType() { + shadowOf(application).grantPermissions(permission.ACCESS_NETWORK_STATE); + + AndroidConnectivityHandler connectivityHandler = createConnectivityHandler(); + + shadowOf(connectivityManager) + .setActiveNetworkInfo( + ShadowNetworkInfo.newInstance( + DetailedState.CONNECTED, + 100, // Invalid network type + 0, + true, + State.CONNECTED)); + + ListenableFuture<Void> result = + connectivityHandler.checkConnectivity( + DownloadConstraints.builder() + .setRequiredNetworkTypes(EnumSet.of(NetworkType.WIFI)) + .setRequireUnmeteredNetwork(false) + .build()); + + assertThat(result.isDone()).isFalse(); + } + + @Test + public void checkConnectivity_requiredNetworkConnected_wifiOnly() { + shadowOf(application).grantPermissions(permission.ACCESS_NETWORK_STATE); + + AndroidConnectivityHandler connectivityHandler = createConnectivityHandler(); + + shadowOf(connectivityManager) + .setActiveNetworkInfo( + ShadowNetworkInfo.newInstance( + DetailedState.CONNECTED, ConnectivityManager.TYPE_WIFI, 0, true, State.CONNECTED)); + + ListenableFuture<Void> result = + connectivityHandler.checkConnectivity(DownloadConstraints.NETWORK_UNMETERED); + + assertThat(result.isDone()).isTrue(); + } + + @Test + public void checkConnectivity_requiredNetworkConnected_wifiOrCellular() { + shadowOf(application).grantPermissions(permission.ACCESS_NETWORK_STATE); + + AndroidConnectivityHandler connectivityHandler = createConnectivityHandler(); + + shadowOf(connectivityManager) + .setActiveNetworkInfo( + ShadowNetworkInfo.newInstance( + DetailedState.CONNECTED, + ConnectivityManager.TYPE_MOBILE, + 0, + true, + State.CONNECTED)); + + ListenableFuture<Void> result = + connectivityHandler.checkConnectivity(DownloadConstraints.NETWORK_CONNECTED); + + assertThat(result.isDone()).isTrue(); + } + + @Config(sdk = VERSION_CODES.M) + @Test + public void checkConnectivity_requiredNetworkConnected_sdk23() { + shadowOf(application).grantPermissions(permission.ACCESS_NETWORK_STATE); + + AndroidConnectivityHandler connectivityHandler = createConnectivityHandler(); + + ShadowConnectivityManager shadowConnectivityManager = shadowOf(connectivityManager); + Network network = ShadowNetwork.newInstance(0); + shadowConnectivityManager.addNetwork( + network, + ShadowNetworkInfo.newInstance( + DetailedState.CONNECTED, ConnectivityManager.TYPE_WIFI, 0, true, State.CONNECTED)); + NetworkCapabilities networkCapabilities = ShadowNetworkCapabilities.newInstance(); + shadowOf(networkCapabilities).addCapability(NetworkCapabilities.NET_CAPABILITY_INTERNET); + shadowConnectivityManager.setNetworkCapabilities(network, networkCapabilities); + shadowConnectivityManager.setDefaultNetworkActive(true); + + ListenableFuture<Void> result = + connectivityHandler.checkConnectivity(DownloadConstraints.NETWORK_CONNECTED); + + assertThat(result.isDone()).isTrue(); + } + + @Test + public void checkConnectivity_networkChange() throws Exception { + ShadowApplication shadowApplication = shadowOf(application); + shadowApplication.grantPermissions(permission.ACCESS_NETWORK_STATE); + + AndroidConnectivityHandler connectivityHandler = createConnectivityHandler(); + + shadowOf(connectivityManager) + .setActiveNetworkInfo( + ShadowNetworkInfo.newInstance( + DetailedState.DISCONNECTED, + ConnectivityManager.TYPE_WIFI, + 0, + false, + State.DISCONNECTED)); + + ListenableFuture<Void> result = + connectivityHandler.checkConnectivity(DownloadConstraints.NETWORK_CONNECTED); + assertThat(result.isDone()).isFalse(); + assertThat(shadowApplication.getRegisteredReceivers()).hasSize(1); + assertThat(shadowApplication.getRegisteredReceivers().get(0).getBroadcastReceiver()) + .isInstanceOf(AndroidConnectivityHandler.NetworkBroadcastReceiver.class); + assertThat( + shadowApplication + .getRegisteredReceivers() + .get(0) + .getIntentFilter() + .hasAction(CONNECTIVITY_ACTION)) + .isTrue(); + + // Change state to be available. + shadowOf(connectivityManager) + .setActiveNetworkInfo( + ShadowNetworkInfo.newInstance( + DetailedState.CONNECTED, ConnectivityManager.TYPE_WIFI, 0, true, State.CONNECTED)); + + application.sendBroadcast(new Intent(CONNECTIVITY_ACTION)); + + shadowOf(getMainLooper()).idle(); + + assertThat(result.isDone()).isTrue(); + assertThat(result.get()).isNull(); + assertThat(shadowApplication.getRegisteredReceivers()).isEmpty(); + } + + @Test + public void checkConnectivity_timeout() { + shadowOf(application).grantPermissions(permission.ACCESS_NETWORK_STATE); + + AndroidConnectivityHandler connectivityHandler = createConnectivityHandler(); + + shadowOf(connectivityManager) + .setActiveNetworkInfo( + ShadowNetworkInfo.newInstance( + DetailedState.DISCONNECTED, + ConnectivityManager.TYPE_WIFI, + 0, + false, + State.DISCONNECTED)); + + ListenableFuture<Void> result = + connectivityHandler.checkConnectivity(DownloadConstraints.NETWORK_CONNECTED); + assertThat(result.isDone()).isFalse(); + + ExecutionException exception = assertThrows(ExecutionException.class, result::get); + assertThat(exception).hasCauseThat().isInstanceOf(TimeoutException.class); + } + + private AndroidConnectivityHandler createConnectivityHandler() { + return new AndroidConnectivityHandler( + application, scheduledExecutorService, DEFAULT_TIMEOUT_MILLIS); + } +} diff --git a/src/test/java/com/google/android/downloader/CronetUrlEngineTest.java b/src/test/java/com/google/android/downloader/CronetUrlEngineTest.java new file mode 100644 index 0000000..1a2a257 --- /dev/null +++ b/src/test/java/com/google/android/downloader/CronetUrlEngineTest.java @@ -0,0 +1,129 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import static androidx.test.InstrumentationRegistry.getInstrumentation; +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; + +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.common.base.Charsets; +import com.google.common.io.Files; +import com.google.common.util.concurrent.ListenableFuture; +import java.io.File; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import org.chromium.net.CronetEngine; +import org.chromium.net.NetworkException; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; + +/** Unit tests for CronetUrlEngine. */ +@RunWith(AndroidJUnit4.class) +public class CronetUrlEngineTest { + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + private MockWebServerUrlEngineTestHelper testHelper; + private CronetUrlEngine engine; + + @Before + public void setUp() { + ExecutorService executorService = Executors.newSingleThreadExecutor(); + TestingExecutorService testingExecutorService = new TestingExecutorService(executorService); + testHelper = new MockWebServerUrlEngineTestHelper(temporaryFolder, testingExecutorService); + CronetEngine cronetEngine = + new CronetEngine.Builder(getInstrumentation().getTargetContext()).build(); + engine = new CronetUrlEngine(cronetEngine, executorService); + } + + @After + public void tearDown() throws Exception { + testHelper.tearDown(); + } + + @Test + public void executeRequest_normalResponse_succeeds() throws Exception { + testHelper.executeRequest_normalResponse_succeeds(engine); + } + + @Test + public void executeRequest_responseThrottled_succeeds() throws Exception { + testHelper.executeRequest_responseThrottled_succeeds(engine); + } + + @Test + public void executeRequest_largeResponse_succeeds() throws Exception { + testHelper.executeRequest_largeResponse_succeeds(engine, CronetUrlEngine.BUFFER_SIZE_BYTES * 3); + } + + @Test + public void executeRequest_closeBeforeWrite_failsAborted() throws Exception { + testHelper.executeRequest_closeBeforeWrite_failsAborted(engine); + } + + @Test + public void executeRequest_serverError_failsInternalError() throws Exception { + testHelper.executeRequest_serverError_failsInternalError(engine); + } + + @Test + public void executeRequest_networkError_failsInternalError() throws Exception { + testHelper.executeRequest_networkError_failsInternalError(engine); + } + + @Test + public void executeRequest_writeError_failsInternalError() throws Exception { + testHelper.executeRequest_writeError_failsInternalError(engine); + } + + @Test + public void executeRequest_requestCanceled_requestNeverSent() throws Exception { + testHelper.executeRequest_requestCanceled_requestNeverSent(engine); + } + + @Test + public void executeRequest_invalidUrl_failsInvalidArgument() throws Exception { + testHelper.executeRequest_invalidUrl_failsInvalidArgument(engine); + } + + @Test + public void executeRequest_fileUrl_requestFails() throws Exception { + // Note: File url testing doesn't exist in MockWebServerUrlEngineTestHelper because + // it doesn't involve the MockWebServer and doesn't apply to all UrlEngine implementations. + String message = "foobar"; + File sourceFile = temporaryFolder.newFile(); + Files.asCharSink(sourceFile, Charsets.UTF_8).write(message); + + String url = sourceFile.toURI().toURL().toString(); + UrlRequest request = engine.createRequest(url).build(); + ListenableFuture<? extends UrlResponse> responseFuture = request.send(); + + // Cronet doesn't support file URLs. Verify that the request fails with an underlying + // error that communicates this problem. + ExecutionException exception = assertThrows(ExecutionException.class, responseFuture::get); + assertThat(exception).hasCauseThat().isInstanceOf(RequestException.class); + assertThat(exception).hasCauseThat().hasCauseThat().isInstanceOf(NetworkException.class); + assertThat(exception) + .hasCauseThat() + .hasCauseThat() + .hasMessageThat() + .contains("UNKNOWN_URL_SCHEME"); + } +} diff --git a/src/test/java/com/google/android/downloader/CronetUrlEngineTestActivity.java b/src/test/java/com/google/android/downloader/CronetUrlEngineTestActivity.java new file mode 100644 index 0000000..15decd5 --- /dev/null +++ b/src/test/java/com/google/android/downloader/CronetUrlEngineTestActivity.java @@ -0,0 +1,22 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import android.app.Activity; + +/** + * Dummy activity for the Cronet instrumentation test setup to ensure we don't have an empty APK. + */ +public class CronetUrlEngineTestActivity extends Activity {} diff --git a/src/test/java/com/google/android/downloader/DataUrlEngineTest.java b/src/test/java/com/google/android/downloader/DataUrlEngineTest.java new file mode 100644 index 0000000..37a1ab0 --- /dev/null +++ b/src/test/java/com/google/android/downloader/DataUrlEngineTest.java @@ -0,0 +1,105 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import static com.google.common.truth.Truth.assertThat; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +import com.google.common.collect.ImmutableList; +import com.google.common.io.Files; +import com.google.common.net.HttpHeaders; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.testing.mockito.Mocks; +import java.io.File; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.channels.WritableByteChannel; +import java.nio.file.StandardOpenOption; +import java.util.concurrent.ExecutionException; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Mock; + +/** Unit tests for DataUrlEngine. */ +@RunWith(JUnit4.class) +public class DataUrlEngineTest { + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Rule public Mocks mocks = new Mocks(this); + + @Mock WritableByteChannel mockByteChannel; + + private DataUrlEngine engine; + + @Before + public void setUp() { + engine = new DataUrlEngine(MoreExecutors.newDirectExecutorService()); + } + + @Test + public void executeRequest() throws Exception { + File file = temporaryFolder.newFile(); + FileChannel channel = FileChannel.open(file.toPath(), StandardOpenOption.WRITE); + + UrlRequest request = engine.createRequest("data:text/plain;base64,Zm9vYmFy").build(); + UrlResponse response = request.send().get(); + long bytesWritten = response.readResponseBody(channel).get(); + response.close(); + channel.close(); + + assertThat(bytesWritten).isGreaterThan(0L); + assertThat(Files.asCharSource(file, UTF_8).read()).isEqualTo("foobar"); + assertThat(response.getResponseCode()).isEqualTo(HttpURLConnection.HTTP_OK); + assertThat(response.getResponseHeaders()) + .containsExactly(HttpHeaders.CONTENT_TYPE, ImmutableList.of("text/plain")); + } + + @Test + public void executeRequest_invalidDataUrl() throws Exception { + UrlRequest request = engine.createRequest("data:text/plain;base64,foobar*").build(); + ListenableFuture<? extends UrlResponse> responseFuture = request.send(); + + ExecutionException exception = assertThrows(ExecutionException.class, responseFuture::get); + assertThat(exception).hasCauseThat().isInstanceOf(RequestException.class); + } + + @Test + public void executeRequest_writeError() throws Exception { + when(mockByteChannel.isOpen()).thenReturn(true); + when(mockByteChannel.write(any(ByteBuffer.class))).thenThrow(new IOException()); + + UrlRequest request = engine.createRequest("data:text/plain;base64,Zm9vYmFy").build(); + UrlResponse response = request.send().get(); + ListenableFuture<Long> writeFuture = response.readResponseBody(mockByteChannel); + + ExecutionException exception = assertThrows(ExecutionException.class, writeFuture::get); + assertThat(exception).hasCauseThat().isInstanceOf(RequestException.class); + + response.close(); + + assertThat(response.getResponseCode()).isEqualTo(HttpURLConnection.HTTP_OK); + assertThat(response.getResponseHeaders()) + .containsExactly(HttpHeaders.CONTENT_TYPE, ImmutableList.of("text/plain")); + } +} diff --git a/src/test/java/com/google/android/downloader/DataUrlTest.java b/src/test/java/com/google/android/downloader/DataUrlTest.java new file mode 100644 index 0000000..e0673a8 --- /dev/null +++ b/src/test/java/com/google/android/downloader/DataUrlTest.java @@ -0,0 +1,74 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import static com.google.common.truth.Truth.assertThat; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.assertThrows; + +import com.google.android.downloader.DataUrl.DataUrlException; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for DataUrl parsing. */ +@RunWith(JUnit4.class) +public class DataUrlTest { + @Test + public void invalidScheme() { + assertThrows(DataUrlException.class, () -> DataUrl.parseFromString("http://example.com")); + } + + @Test + public void invalidSyntax() { + assertThrows(DataUrlException.class, () -> DataUrl.parseFromString("data:foobar")); + } + + @Test + public void missingEncoding() { + assertThrows(DataUrlException.class, () -> DataUrl.parseFromString("data:text/plain,foobar")); + } + + @Test + public void invalidEncoding() { + assertThrows( + DataUrlException.class, () -> DataUrl.parseFromString("data:text/plain;base32,foobar")); + } + + @Test + public void invalidBase64Data() { + // Note that '*' is not a valid character in base64. + assertThrows( + DataUrlException.class, () -> DataUrl.parseFromString("data:text/plain;base64,foobar*")); + } + + @Test + public void validData() { + // Note: 'Zm9vYmFy' is the base64-encoding of 'foobar'. + DataUrl dataUrl = DataUrl.parseFromString("data:text/plain;base64,Zm9vYmFy"); + assertThat(dataUrl.data()).isEqualTo("foobar".getBytes(UTF_8)); + assertThat(dataUrl.mimeType()).isEqualTo("text/plain"); + } + + @Test + public void validData_notUrl() { + // Note: 'Zm9vLiw/YmFy' is the base64-encoding of 'foo.,?bar'. Because there's a slash in the + // base64 payload, this isn't a url-safe base64 encoding, so this test will verify we fall + // back to regular base64 encoding instead of just url-safe encoding. + DataUrl dataUrl = DataUrl.parseFromString("data:text/plain;base64,Zm9vLiw/YmFy"); + assertThat(dataUrl.data()).isEqualTo("foo.,?bar".getBytes(UTF_8)); + assertThat(dataUrl.mimeType()).isEqualTo("text/plain"); + } +} diff --git a/src/test/java/com/google/android/downloader/DownloaderTest.java b/src/test/java/com/google/android/downloader/DownloaderTest.java new file mode 100644 index 0000000..1bbcaea --- /dev/null +++ b/src/test/java/com/google/android/downloader/DownloaderTest.java @@ -0,0 +1,1053 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import static com.google.common.truth.StreamSubject.streams; +import static com.google.common.truth.Truth.assertAbout; +import static com.google.common.truth.Truth.assertThat; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.time.format.DateTimeFormatter.RFC_1123_DATE_TIME; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.junit.Assert.assertThrows; +import static org.mockito.AdditionalMatchers.or; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import com.google.android.downloader.Downloader.State; +import com.google.common.base.Utf8; +import com.google.common.collect.ImmutableSet; +import com.google.common.io.Files; +import com.google.common.net.HttpHeaders; +import com.google.common.util.concurrent.ClosingFuture; +import com.google.common.util.concurrent.FluentFuture; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.common.util.concurrent.SettableFuture; +import com.google.testing.mockito.Mocks; +import java.io.File; +import java.net.HttpURLConnection; +import java.net.URI; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.concurrent.CancellationException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.function.Supplier; +import java.util.logging.Handler; +import java.util.logging.Level; +import java.util.logging.LogRecord; +import java.util.logging.Logger; +import okhttp3.mockwebserver.Dispatcher; +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; +import okhttp3.mockwebserver.QueueDispatcher; +import okhttp3.mockwebserver.RecordedRequest; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Mock; + +/** Unit tests for Downloader. */ +@RunWith(JUnit4.class) +public final class DownloaderTest { + private static final int CONNECT_TIMEOUT_MS = 500; + private static final int READ_TIMEOUT_MS = 300; + private static final int MAX_CONCURRENT_DOWNLOADS = 1; + + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Rule public Mocks mocks = new Mocks(this); + @Rule public TestExecutorRule testExecutorRule = new TestExecutorRule(Duration.ofSeconds(2)); + + @Mock Downloader.StateChangeCallback mockCallback; + + private ListeningExecutorService urlEngineExecutor; + private ExecutorService ioExecutor; + private MockWebServer mockWebServer; + private DispatcherImpl dispatcher; + + @Before + public void setUp() { + mockWebServer = new MockWebServer(); + dispatcher = new DispatcherImpl(); + mockWebServer.setDispatcher(dispatcher); + urlEngineExecutor = + MoreExecutors.listeningDecorator(testExecutorRule.newSingleThreadExecutor()); + ioExecutor = testExecutorRule.newSingleThreadExecutor(); + } + + private Downloader.Builder buildDownloader() { + return new Downloader.Builder() + .addUrlEngine( + ImmutableSet.of("http", "https", "file"), + new PlatformUrlEngine(urlEngineExecutor, CONNECT_TIMEOUT_MS, READ_TIMEOUT_MS)) + .withConnectivityHandler(new AlwaysConnected()) + .withIOExecutor(ioExecutor) + .withLogger(new FloggerDownloaderLogger()) + .withMaxConcurrentDownloads(MAX_CONCURRENT_DOWNLOADS); + } + + @After + public void tearDown() throws Exception { + mockWebServer.shutdown(); + } + + @Test + public void downloadOneFile() throws Exception { + File destinationFile = temporaryFolder.newFile(); + File metadataFile = new File(destinationFile.getParent(), destinationFile.getName() + ".meta"); + SimpleFileDownloadDestination destination = + new SimpleFileDownloadDestination(destinationFile, metadataFile); + + String contentTag = "content_tag_abc"; + long lastModifiedTimeSeconds = 123456789; + dispatcher.responseForPath( + "/foo", + () -> + new MockResponse() + .setResponseCode(HttpURLConnection.HTTP_OK) + .setBody("hello world") + .setHeader(HttpHeaders.ETAG, contentTag) + .setHeader( + HttpHeaders.LAST_MODIFIED, + RFC_1123_DATE_TIME + .withLocale(Locale.US) + .withZone(ZoneId.of("UTC")) + .format(Instant.ofEpochSecond(lastModifiedTimeSeconds)))); + mockWebServer.start(); + + Downloader downloader = buildDownloader().build(); + DownloadRequest request = + downloader.newRequestBuilder(mockWebServer.url("/foo").uri(), destination).build(); + FluentFuture<DownloadResult> resultFuture = downloader.execute(request); + assertThat(resultFuture.isDone()).isFalse(); + + DownloadResult result = resultFuture.get(); + assertThat(result.bytesWritten()).isEqualTo(11); // == "hello world".length + assertThat(Files.asCharSource(destinationFile, UTF_8).read()).isEqualTo("hello world"); + + assertThat(destination.numExistingBytes()).isEqualTo(11); + DownloadMetadata metadata = destination.readMetadata(); + assertThat(metadata).isEqualTo(DownloadMetadata.create(contentTag, lastModifiedTimeSeconds)); + } + + @Test + public void downloadOneFile_existingContent_partialContent() throws Exception { + File destinationFile = temporaryFolder.newFile(); + File metadataFile = new File(destinationFile.getParent(), destinationFile.getName() + ".meta"); + SimpleFileDownloadDestination destination = + new SimpleFileDownloadDestination(destinationFile, metadataFile); + Files.asCharSink(destinationFile, UTF_8).write("hello world"); + + dispatcher.responseForPath( + "/foo", + () -> + new MockResponse() + .setResponseCode(HttpURLConnection.HTTP_PARTIAL) + .setBody("goodbye world") + // 24 == ("hello world" + "goodbye world").length + .setHeader(HttpHeaders.CONTENT_RANGE, "bytes 11-24/24")); + mockWebServer.start(); + + Downloader downloader = buildDownloader().build(); + DownloadRequest request = + downloader.newRequestBuilder(mockWebServer.url("/foo").uri(), destination).build(); + FluentFuture<DownloadResult> resultFuture = downloader.execute(request); + assertThat(resultFuture.isDone()).isFalse(); + + DownloadResult result = resultFuture.get(); + assertThat(result.bytesWritten()).isEqualTo(13); // == "goodbye world".length + assertThat(Files.asCharSource(destinationFile, UTF_8).read()) + .isEqualTo("hello worldgoodbye world"); + assertThat(mockWebServer.takeRequest().getHeader(HttpHeaders.RANGE)) + .isEqualTo("bytes=11-"); // == "hello world".length + } + + @Test + public void downloadOneFile_existingContent_noMetadata() throws Exception { + File destinationFile = temporaryFolder.newFile(); + File metadataFile = new File(destinationFile.getParent(), destinationFile.getName() + ".meta"); + SimpleFileDownloadDestination destination = + new SimpleFileDownloadDestination(destinationFile, metadataFile); + Files.asCharSink(destinationFile, UTF_8).write("hello world"); + + dispatcher.responseForPath( + "/foo", + () -> + new MockResponse().setResponseCode(HttpURLConnection.HTTP_OK).setBody("goodbye world")); + mockWebServer.start(); + + Downloader downloader = buildDownloader().build(); + DownloadRequest request = + downloader.newRequestBuilder(mockWebServer.url("/foo").uri(), destination).build(); + downloader.execute(request).get(); + + RecordedRequest recordedRequest = mockWebServer.takeRequest(); + assertThat(recordedRequest.getHeader(HttpHeaders.RANGE)) + .isEqualTo("bytes=11-"); // == "hello world".length + assertThat(recordedRequest.getHeader(HttpHeaders.IF_RANGE)).isNull(); + } + + @Test + public void downloadOneFile_existingContent_withEtag() throws Exception { + File destinationFile = temporaryFolder.newFile(); + File metadataFile = new File(destinationFile.getParent(), destinationFile.getName() + ".meta"); + SimpleFileDownloadDestination destination = + new SimpleFileDownloadDestination(destinationFile, metadataFile); + String contentTag = "content_tag_abc"; + destination.openByteChannel(0L, DownloadMetadata.create(contentTag, 0L)).close(); + Files.asCharSink(destinationFile, UTF_8).write("hello world"); + + dispatcher.responseForPath( + "/foo", + () -> + new MockResponse().setResponseCode(HttpURLConnection.HTTP_OK).setBody("goodbye world")); + mockWebServer.start(); + + Downloader downloader = buildDownloader().build(); + DownloadRequest request = + downloader.newRequestBuilder(mockWebServer.url("/foo").uri(), destination).build(); + downloader.execute(request).get(); + + RecordedRequest recordedRequest = mockWebServer.takeRequest(); + assertThat(recordedRequest.getHeader(HttpHeaders.RANGE)) + .isEqualTo("bytes=11-"); // == "hello world".length + assertThat(recordedRequest.getHeader(HttpHeaders.IF_RANGE)).isEqualTo(contentTag); + } + + @Test + public void downloadOneFile_existingContent_withLastModified() throws Exception { + File destinationFile = temporaryFolder.newFile(); + File metadataFile = new File(destinationFile.getParent(), destinationFile.getName() + ".meta"); + SimpleFileDownloadDestination destination = + new SimpleFileDownloadDestination(destinationFile, metadataFile); + long lastModifiedTimestampSeconds = 123456789; + destination + .openByteChannel(0L, DownloadMetadata.create("", lastModifiedTimestampSeconds)) + .close(); + Files.asCharSink(destinationFile, UTF_8).write("hello world"); + + dispatcher.responseForPath( + "/foo", + () -> + new MockResponse().setResponseCode(HttpURLConnection.HTTP_OK).setBody("goodbye world")); + mockWebServer.start(); + + Downloader downloader = buildDownloader().build(); + DownloadRequest request = + downloader.newRequestBuilder(mockWebServer.url("/foo").uri(), destination).build(); + downloader.execute(request).get(); + + RecordedRequest recordedRequest = mockWebServer.takeRequest(); + assertThat(recordedRequest.getHeader(HttpHeaders.RANGE)) + .isEqualTo("bytes=11-"); // == "hello world".length + assertThat(recordedRequest.getHeader(HttpHeaders.IF_RANGE)) + .isEqualTo( + RFC_1123_DATE_TIME + .withLocale(Locale.US) + .withZone(ZoneId.of("UTC")) + .format(Instant.ofEpochSecond(lastModifiedTimestampSeconds))); + } + + @Test + public void downloadOneFile_existingContent_partialContent_overwritesExistingContent() + throws Exception { + File destinationFile = temporaryFolder.newFile(); + File metadataFile = new File(destinationFile.getParent(), destinationFile.getName() + ".meta"); + SimpleFileDownloadDestination destination = + new SimpleFileDownloadDestination(destinationFile, metadataFile); + Files.asCharSink(destinationFile, UTF_8).write("hello world"); + + dispatcher.responseForPath( + "/foo", + () -> + new MockResponse() + .setResponseCode(HttpURLConnection.HTTP_PARTIAL) + .setBody("goodbye world") + // 6 == ("hello world" - "world").length + // 19 == ("hello world" - "world" + "goodbye world").length + .setHeader(HttpHeaders.CONTENT_RANGE, "bytes 6-19/19")); + mockWebServer.start(); + + Downloader downloader = buildDownloader().build(); + DownloadRequest request = + downloader.newRequestBuilder(mockWebServer.url("/foo").uri(), destination).build(); + FluentFuture<DownloadResult> resultFuture = downloader.execute(request); + assertThat(resultFuture.isDone()).isFalse(); + + DownloadResult result = resultFuture.get(); + assertThat(result.bytesWritten()).isEqualTo(13); // == "goodbye world".length + assertThat(Files.asCharSource(destinationFile, UTF_8).read()).isEqualTo("hello goodbye world"); + assertThat(mockWebServer.takeRequest().getHeader(HttpHeaders.RANGE)) + .isEqualTo("bytes=11-"); // == "hello world".length + } + + @Test + public void downloadOneFile_existingContent_noServerSupport() throws Exception { + File destinationFile = temporaryFolder.newFile(); + File metadataFile = new File(destinationFile.getParent(), destinationFile.getName() + ".meta"); + SimpleFileDownloadDestination destination = + new SimpleFileDownloadDestination(destinationFile, metadataFile); + Files.asCharSink(destinationFile, UTF_8).write("hello world"); + + // Let the server ignore our Range header and reply with the entire resource. + dispatcher.responseForPath( + "/foo", + () -> + new MockResponse() + .setResponseCode(HttpURLConnection.HTTP_OK) + .setBody("hello worldgoodbye world")); + mockWebServer.start(); + + Downloader downloader = buildDownloader().build(); + DownloadRequest request = + downloader.newRequestBuilder(mockWebServer.url("/foo").uri(), destination).build(); + FluentFuture<DownloadResult> resultFuture = downloader.execute(request); + assertThat(resultFuture.isDone()).isFalse(); + + DownloadResult result = resultFuture.get(); + // 24 == ("hello world" + "goodbye world").length + assertThat(result.bytesWritten()).isEqualTo(24); + assertThat(Files.asCharSource(destinationFile, UTF_8).read()) + .isEqualTo("hello worldgoodbye world"); + assertThat(mockWebServer.takeRequest().getHeader(HttpHeaders.RANGE)) + .isEqualTo("bytes=11-"); // == "hello world".length + } + + @Test + public void downloadOneFile_rangeNotSatisfiable() throws Exception { + File destinationFile = temporaryFolder.newFile(); + File metadataFile = new File(destinationFile.getParent(), destinationFile.getName() + ".meta"); + SimpleFileDownloadDestination destination = + new SimpleFileDownloadDestination(destinationFile, metadataFile); + + dispatcher.responseForPath( + "/foo", () -> new MockResponse().setResponseCode(Downloader.HTTP_RANGE_NOT_SATISFIABLE)); + mockWebServer.start(); + + Downloader downloader = buildDownloader().build(); + DownloadRequest request = + downloader.newRequestBuilder(mockWebServer.url("/foo").uri(), destination).build(); + FluentFuture<DownloadResult> resultFuture = downloader.execute(request); + assertThat(resultFuture.isDone()).isFalse(); + + DownloadResult result = resultFuture.get(); + assertThat(result.bytesWritten()).isEqualTo(0); + assertThat(Files.asCharSource(destinationFile, UTF_8).read()).isEmpty(); + } + + @Test + public void downloadOneFile_oAuthTokenProvider() throws Exception { + File destinationFile = temporaryFolder.newFile(); + File metadataFile = new File(destinationFile.getParent(), destinationFile.getName() + ".meta"); + SimpleFileDownloadDestination destination = + new SimpleFileDownloadDestination(destinationFile, metadataFile); + + dispatcher.responseForPath( + "/foo", + () -> new MockResponse().setResponseCode(HttpURLConnection.HTTP_OK).setBody("hello world")); + mockWebServer.start(); + + Downloader downloader = buildDownloader().build(); + DownloadRequest request = + downloader + .newRequestBuilder(mockWebServer.url("/foo").uri(), destination) + .setOAuthTokenProvider(uri -> Futures.immediateFuture("test_token")) + .build(); + FluentFuture<DownloadResult> resultFuture = downloader.execute(request); + assertThat(resultFuture.isDone()).isFalse(); + + DownloadResult result = resultFuture.get(); + assertThat(result.bytesWritten()).isEqualTo(Utf8.encodedLength("hello world")); + assertThat(Files.asCharSource(destinationFile, UTF_8).read()).isEqualTo("hello world"); + assertThat(mockWebServer.takeRequest().getHeader(HttpHeaders.AUTHORIZATION)) + .isEqualTo("Bearer test_token"); + } + + @Test + public void downloadOneFile_oAuthTokenProvider_nullToken() throws Exception { + File destinationFile = temporaryFolder.newFile(); + File metadataFile = new File(destinationFile.getParent(), destinationFile.getName() + ".meta"); + SimpleFileDownloadDestination destination = + new SimpleFileDownloadDestination(destinationFile, metadataFile); + + dispatcher.responseForPath( + "/foo", + () -> new MockResponse().setResponseCode(HttpURLConnection.HTTP_OK).setBody("hello world")); + mockWebServer.start(); + + Downloader downloader = buildDownloader().build(); + DownloadRequest request = + downloader + .newRequestBuilder(mockWebServer.url("/foo").uri(), destination) + .setOAuthTokenProvider(uri -> Futures.immediateFuture(null)) + .build(); + FluentFuture<DownloadResult> resultFuture = downloader.execute(request); + assertThat(resultFuture.isDone()).isFalse(); + + DownloadResult result = resultFuture.get(); + assertThat(result.bytesWritten()).isEqualTo(Utf8.encodedLength("hello world")); + assertThat(Files.asCharSource(destinationFile, UTF_8).read()).isEqualTo("hello world"); + assertThat(mockWebServer.takeRequest().getHeaders().names()) + .doesNotContain(HttpHeaders.AUTHORIZATION); + } + + @Test + public void downloadOneFile_notFound() throws Exception { + File destinationFile = temporaryFolder.newFile(); + File metadataFile = new File(destinationFile.getParent(), destinationFile.getName() + ".meta"); + SimpleFileDownloadDestination destination = + new SimpleFileDownloadDestination(destinationFile, metadataFile); + + mockWebServer.start(); + + Downloader downloader = buildDownloader().build(); + downloader.registerStateChangeCallback(mockCallback, MoreExecutors.directExecutor()); + DownloadRequest request = + downloader.newRequestBuilder(mockWebServer.url("/foo").uri(), destination).build(); + FluentFuture<DownloadResult> resultFuture = downloader.execute(request); + assertThat(resultFuture.isDone()).isFalse(); + verify(mockCallback, atLeastOnce()).onStateChange(State.create(1, 0, 0)); + + Exception exception = assertThrows(Exception.class, resultFuture::get); + RequestException requestException = Downloader.getRequestException(exception); + assertThat(requestException).isNotNull(); + + // Flush through the I/O executor to make sure internal state has settled. + ioExecutor.submit(() -> {}).get(); + + verify(mockCallback, atLeastOnce()).onStateChange(State.create(0, 0, 0)); + } + + @Test + public void downloadOneFile_customHeader() throws Exception { + File destinationFile = temporaryFolder.newFile(); + File metadataFile = new File(destinationFile.getParent(), destinationFile.getName() + ".meta"); + SimpleFileDownloadDestination destination = + new SimpleFileDownloadDestination(destinationFile, metadataFile); + + dispatcher.responseForPath( + "/foo", + () -> new MockResponse().setResponseCode(HttpURLConnection.HTTP_OK).setBody("hello world")); + mockWebServer.start(); + + String headerKey = "fooHeader"; + String headerValue = "fooHeaderValue"; + Downloader downloader = buildDownloader().build(); + DownloadRequest request = + downloader + .newRequestBuilder(mockWebServer.url("/foo").uri(), destination) + .addHeader(headerKey, headerValue) + .build(); + FluentFuture<DownloadResult> resultFuture = downloader.execute(request); + assertThat(resultFuture.isDone()).isFalse(); + + DownloadResult result = resultFuture.get(); + assertThat(result.bytesWritten()).isEqualTo(Utf8.encodedLength("hello world")); + assertThat(Files.asCharSource(destinationFile, UTF_8).read()).isEqualTo("hello world"); + assertThat(mockWebServer.takeRequest().getHeader(headerKey)).isEqualTo(headerValue); + } + + @Test + public void downloadOneFile_fileSystem() throws Exception { + File sourceFile = temporaryFolder.newFile(); + File destinationFile = temporaryFolder.newFile(); + File metadataFile = new File(destinationFile.getParent(), destinationFile.getName() + ".meta"); + SimpleFileDownloadDestination destination = + new SimpleFileDownloadDestination(destinationFile, metadataFile); + + Files.asCharSink(sourceFile, UTF_8).write("hello world"); + + Downloader downloader = buildDownloader().build(); + DownloadRequest request = downloader.newRequestBuilder(sourceFile.toURI(), destination).build(); + FluentFuture<DownloadResult> resultFuture = downloader.execute(request); + assertThat(resultFuture.isDone()).isFalse(); + + DownloadResult result = resultFuture.get(); + assertThat(result.bytesWritten()).isEqualTo(Utf8.encodedLength("hello world")); + assertThat(Files.asCharSource(destinationFile, UTF_8).read()).isEqualTo("hello world"); + assertThat(mockWebServer.getRequestCount()).isEqualTo(0); + } + + @Test + public void downloadOneFile_fileSystem_existingContent() throws Exception { + File sourceFile = temporaryFolder.newFile(); + File destinationFile = temporaryFolder.newFile(); + File metadataFile = new File(destinationFile.getParent(), destinationFile.getName() + ".meta"); + SimpleFileDownloadDestination destination = + new SimpleFileDownloadDestination(destinationFile, metadataFile); + + Files.asCharSink(sourceFile, UTF_8).write("hello world"); + Files.asCharSink(destinationFile, UTF_8).write("hello"); + + Downloader downloader = buildDownloader().build(); + DownloadRequest request = downloader.newRequestBuilder(sourceFile.toURI(), destination).build(); + FluentFuture<DownloadResult> resultFuture = downloader.execute(request); + assertThat(resultFuture.isDone()).isFalse(); + + DownloadResult result = resultFuture.get(); + assertThat(result.bytesWritten()).isEqualTo(Utf8.encodedLength("hello world")); + assertThat(Files.asCharSource(destinationFile, UTF_8).read()).isEqualTo("hello world"); + } + + @Test + public void downloadOneFile_retryAfterFailure() throws Exception { + mockWebServer.setDispatcher(new QueueDispatcher()); + + File destinationFile = temporaryFolder.newFile(); + File metadataFile = new File(destinationFile.getParent(), destinationFile.getName() + ".meta"); + SimpleFileDownloadDestination destination = + new SimpleFileDownloadDestination(destinationFile, metadataFile); + + mockWebServer.enqueue( + new MockResponse().setResponseCode(HttpURLConnection.HTTP_GATEWAY_TIMEOUT)); + mockWebServer.enqueue( + new MockResponse().setResponseCode(HttpURLConnection.HTTP_OK).setBody("hello world")); + mockWebServer.start(); + + Downloader downloader = buildDownloader().build(); + DownloadRequest request = + downloader.newRequestBuilder(mockWebServer.url("/foo").uri(), destination).build(); + FluentFuture<DownloadResult> resultFuture = downloader.execute(request); + assertThat(resultFuture.isDone()).isFalse(); + + DownloadResult result = resultFuture.get(); + assertThat(result.bytesWritten()).isEqualTo(Utf8.encodedLength("hello world")); + assertThat(Files.asCharSource(destinationFile, UTF_8).read()).isEqualTo("hello world"); + assertThat(mockWebServer.getRequestCount()).isEqualTo(2); + } + + @Test + public void downloadOneFile_waitForConnectivity() throws Exception { + File destinationFile = temporaryFolder.newFile(); + File metadataFile = new File(destinationFile.getParent(), destinationFile.getName() + ".meta"); + SimpleFileDownloadDestination destination = + new SimpleFileDownloadDestination(destinationFile, metadataFile); + String text = "hello world"; + + dispatcher.responseForPath( + "/foo", () -> new MockResponse().setResponseCode(HttpURLConnection.HTTP_OK).setBody(text)); + mockWebServer.start(); + + ManualConnectivity connectivityHandler = new ManualConnectivity(); + + Downloader downloader = buildDownloader().withConnectivityHandler(connectivityHandler).build(); + downloader.registerStateChangeCallback(mockCallback, MoreExecutors.directExecutor()); + DownloadRequest request = + downloader.newRequestBuilder(mockWebServer.url("/foo").uri(), destination).build(); + + FluentFuture<DownloadResult> resultFuture = downloader.execute(request); + assertThat(resultFuture.isDone()).isFalse(); + assertThrows(TimeoutException.class, () -> resultFuture.get(1, SECONDS)); + assertThat(mockWebServer.getRequestCount()).isEqualTo(0); + + verify(mockCallback, atLeastOnce()).onStateChange(State.create(0, 0, 1)); + + connectivityHandler.setConnectivitySatisfied(); + connectivityHandler.succeed(); + + DownloadResult result = resultFuture.get(); + assertThat(result.bytesWritten()).isEqualTo(Utf8.encodedLength(text)); + assertThat(Files.asCharSource(destinationFile, UTF_8).read()).isEqualTo(text); + assertThat(mockWebServer.getRequestCount()).isEqualTo(1); + + // Flush through the I/O executor to make sure internal state has settled. + ioExecutor.submit(() -> {}).get(); + + verify(mockCallback, atLeastOnce()).onStateChange(State.create(0, 0, 0)); + } + + @Test + public void downloadOneFile_waitForConnectivity_canceled() throws Exception { + File destinationFile = temporaryFolder.newFile(); + File metadataFile = new File(destinationFile.getParent(), destinationFile.getName() + ".meta"); + SimpleFileDownloadDestination destination = + new SimpleFileDownloadDestination(destinationFile, metadataFile); + String text = "hello world"; + + dispatcher.responseForPath( + "/foo", () -> new MockResponse().setResponseCode(HttpURLConnection.HTTP_OK).setBody(text)); + mockWebServer.start(); + + ManualConnectivity connectivityHandler = new ManualConnectivity(); + + Downloader downloader = buildDownloader().withConnectivityHandler(connectivityHandler).build(); + downloader.registerStateChangeCallback(mockCallback, MoreExecutors.directExecutor()); + DownloadRequest request = + downloader.newRequestBuilder(mockWebServer.url("/foo").uri(), destination).build(); + + FluentFuture<DownloadResult> resultFuture = downloader.execute(request); + assertThat(resultFuture.isDone()).isFalse(); + assertThat(mockWebServer.getRequestCount()).isEqualTo(0); + + verify(mockCallback, atLeastOnce()).onStateChange(State.create(0, 0, 1)); + + resultFuture.cancel(true); + + connectivityHandler.setConnectivitySatisfied(); + connectivityHandler.succeed(); + + assertThrows(CancellationException.class, resultFuture::get); + assertThat(mockWebServer.getRequestCount()).isEqualTo(0); + + verify(mockCallback, atLeastOnce()).onStateChange(State.create(0, 0, 0)); + } + + @Test + public void downloadOneFile_timeoutConnectivity() throws Exception { + File destinationFile = temporaryFolder.newFile(); + File metadataFile = new File(destinationFile.getParent(), destinationFile.getName() + ".meta"); + SimpleFileDownloadDestination destination = + new SimpleFileDownloadDestination(destinationFile, metadataFile); + String text = "hello world"; + + dispatcher.responseForPath( + "/foo", () -> new MockResponse().setResponseCode(HttpURLConnection.HTTP_OK).setBody(text)); + mockWebServer.start(); + + ManualConnectivity connectivityHandler = new ManualConnectivity(); + + Downloader downloader = buildDownloader().withConnectivityHandler(connectivityHandler).build(); + downloader.registerStateChangeCallback(mockCallback, MoreExecutors.directExecutor()); + DownloadRequest request = + downloader.newRequestBuilder(mockWebServer.url("/foo").uri(), destination).build(); + + FluentFuture<DownloadResult> resultFuture = downloader.execute(request); + assertThat(resultFuture.isDone()).isFalse(); + assertThrows(TimeoutException.class, () -> resultFuture.get(1, SECONDS)); + assertThat(mockWebServer.getRequestCount()).isEqualTo(0); + + verify(mockCallback, atLeastOnce()).onStateChange(State.create(0, 0, 1)); + + connectivityHandler.setConnectivitySatisfied(); + connectivityHandler.fail(new TimeoutException()); + + DownloadResult result = resultFuture.get(); + assertThat(result.bytesWritten()).isEqualTo(Utf8.encodedLength(text)); + assertThat(Files.asCharSource(destinationFile, UTF_8).read()).isEqualTo(text); + assertThat(mockWebServer.getRequestCount()).isEqualTo(1); + + // Flush through the I/O executor to make sure internal state has settled. + ioExecutor.submit(() -> {}).get(); + + verify(mockCallback, atLeastOnce()).onStateChange(State.create(0, 0, 0)); + } + + @Test + public void downloadTwoFiles_sequentially() throws Exception { + File destinationFile1 = temporaryFolder.newFile(); + File destinationFile2 = temporaryFolder.newFile(); + File metadataFile1 = + new File(destinationFile1.getParent(), destinationFile1.getName() + ".meta"); + File metadataFile2 = + new File(destinationFile2.getParent(), destinationFile2.getName() + ".meta"); + SimpleFileDownloadDestination destination1 = + new SimpleFileDownloadDestination(destinationFile1, metadataFile1); + SimpleFileDownloadDestination destination2 = + new SimpleFileDownloadDestination(destinationFile2, metadataFile2); + + dispatcher.responseForPath( + "/foo", + () -> + new MockResponse() + .setResponseCode(HttpURLConnection.HTTP_OK) + .setBodyDelay(50, TimeUnit.MILLISECONDS) + .setBody("hello world one")); + dispatcher.responseForPath( + "/bar", + () -> + new MockResponse() + .setResponseCode(HttpURLConnection.HTTP_OK) + .setBodyDelay(50, TimeUnit.MILLISECONDS) + .setBody("hello world two")); + mockWebServer.start(); + + Downloader downloader = buildDownloader().build(); + downloader.registerStateChangeCallback(mockCallback, MoreExecutors.directExecutor()); + DownloadRequest request1 = + downloader.newRequestBuilder(mockWebServer.url("/foo").uri(), destination1).build(); + DownloadRequest request2 = + downloader.newRequestBuilder(mockWebServer.url("/bar").uri(), destination2).build(); + FluentFuture<DownloadResult> resultFuture1 = downloader.execute(request1); + FluentFuture<DownloadResult> resultFuture2 = downloader.execute(request2); + + verify(mockCallback, atLeastOnce()).onStateChange(State.create(1, 1, 0)); + + DownloadResult result1 = resultFuture1.get(); + DownloadResult result2 = resultFuture2.get(); + assertThat(result1.bytesWritten()).isEqualTo(15); // == "hello world one".length + assertThat(result2.bytesWritten()).isEqualTo(15); // == "hello world two".length + assertThat(Files.asCharSource(destinationFile1, UTF_8).read()).isEqualTo("hello world one"); + assertThat(Files.asCharSource(destinationFile2, UTF_8).read()).isEqualTo("hello world two"); + assertThat(mockWebServer.getRequestCount()).isEqualTo(2); + + // Flush through the I/O executor to make sure internal state has settled. + ioExecutor.submit(() -> {}).get(); + + verify(mockCallback, atLeastOnce()).onStateChange(State.create(0, 0, 0)); + } + + @Test + public void downloadTwoFiles_sequentially_unregisterCallback() throws Exception { + File destinationFile1 = temporaryFolder.newFile(); + File destinationFile2 = temporaryFolder.newFile(); + File metadataFile1 = + new File(destinationFile1.getParent(), destinationFile1.getName() + ".meta"); + File metadataFile2 = + new File(destinationFile2.getParent(), destinationFile2.getName() + ".meta"); + SimpleFileDownloadDestination destination1 = + new SimpleFileDownloadDestination(destinationFile1, metadataFile1); + SimpleFileDownloadDestination destination2 = + new SimpleFileDownloadDestination(destinationFile2, metadataFile2); + + dispatcher.responseForPath( + "/foo", + () -> + new MockResponse() + .setResponseCode(HttpURLConnection.HTTP_OK) + .setBodyDelay(50, TimeUnit.MILLISECONDS) + .setBody("hello world one")); + dispatcher.responseForPath( + "/bar", + () -> + new MockResponse() + .setResponseCode(HttpURLConnection.HTTP_OK) + .setBodyDelay(50, TimeUnit.MILLISECONDS) + .setBody("hello world two")); + mockWebServer.start(); + + Downloader downloader = buildDownloader().build(); + downloader.registerStateChangeCallback(mockCallback, MoreExecutors.directExecutor()); + + DownloadRequest request1 = + downloader.newRequestBuilder(mockWebServer.url("/foo").uri(), destination1).build(); + FluentFuture<DownloadResult> resultFuture1 = downloader.execute(request1); + + downloader.unregisterStateChangeCallback(mockCallback); + + DownloadRequest request2 = + downloader.newRequestBuilder(mockWebServer.url("/bar").uri(), destination2).build(); + FluentFuture<DownloadResult> resultFuture2 = downloader.execute(request2); + + DownloadResult result1 = resultFuture1.get(); + DownloadResult result2 = resultFuture2.get(); + assertThat(result1.bytesWritten()).isEqualTo(15); // == "hello world one".length + assertThat(result2.bytesWritten()).isEqualTo(15); // == "hello world two".length + assertThat(Files.asCharSource(destinationFile1, UTF_8).read()).isEqualTo("hello world one"); + assertThat(Files.asCharSource(destinationFile2, UTF_8).read()).isEqualTo("hello world two"); + assertThat(mockWebServer.getRequestCount()).isEqualTo(2); + + // Only run two times, for the first request: + // - Once for enqueueing the request + // - Once for starting the request + // but no more progress registered otherwise. + verify(mockCallback, times(2)).onStateChange(any()); + } + + @Test + public void downloadTwoFiles_concurrently() throws Exception { + File destinationFile1 = temporaryFolder.newFile(); + File destinationFile2 = temporaryFolder.newFile(); + File metadataFile1 = + new File(destinationFile1.getParent(), destinationFile1.getName() + ".meta"); + File metadataFile2 = + new File(destinationFile2.getParent(), destinationFile2.getName() + ".meta"); + SimpleFileDownloadDestination destination1 = + new SimpleFileDownloadDestination(destinationFile1, metadataFile1); + SimpleFileDownloadDestination destination2 = + new SimpleFileDownloadDestination(destinationFile2, metadataFile2); + + dispatcher.responseForPath( + "/foo", + () -> + new MockResponse() + .setResponseCode(HttpURLConnection.HTTP_OK) + .setBodyDelay(50, TimeUnit.MILLISECONDS) + .setBody("hello world one")); + dispatcher.responseForPath( + "/bar", + () -> + new MockResponse() + .setResponseCode(HttpURLConnection.HTTP_OK) + .setBodyDelay(50, TimeUnit.MILLISECONDS) + .setBody("hello world two")); + mockWebServer.start(); + + Downloader downloader = buildDownloader().withMaxConcurrentDownloads(2).build(); + downloader.registerStateChangeCallback(mockCallback, MoreExecutors.directExecutor()); + DownloadRequest request1 = + downloader.newRequestBuilder(mockWebServer.url("/foo").uri(), destination1).build(); + DownloadRequest request2 = + downloader.newRequestBuilder(mockWebServer.url("/bar").uri(), destination2).build(); + FluentFuture<DownloadResult> resultFuture1 = downloader.execute(request1); + FluentFuture<DownloadResult> resultFuture2 = downloader.execute(request2); + assertThat(resultFuture1.isDone()).isFalse(); + assertThat(resultFuture2.isDone()).isFalse(); + + verify(mockCallback, atLeastOnce()).onStateChange(State.create(2, 0, 0)); + + DownloadResult result1 = resultFuture1.get(); + DownloadResult result2 = resultFuture2.get(); + assertThat(result1.bytesWritten()).isEqualTo(15); // == "hello world one".length + assertThat(result2.bytesWritten()).isEqualTo(15); // == "hello world two".length + assertThat(Files.asCharSource(destinationFile1, UTF_8).read()).isEqualTo("hello world one"); + assertThat(Files.asCharSource(destinationFile2, UTF_8).read()).isEqualTo("hello world two"); + assertThat(mockWebServer.getRequestCount()).isEqualTo(2); + + // Flush through the I/O executor to make sure internal state has settled. + ioExecutor.submit(() -> {}).get(); + + verify(mockCallback, atLeastOnce()).onStateChange(State.create(0, 0, 0)); + } + + @Test + public void downloadThreeFiles_sequentiallyWithCancellation() throws Exception { + File destinationFile1 = temporaryFolder.newFile(); + File destinationFile2 = temporaryFolder.newFile(); + File destinationFile3 = temporaryFolder.newFile(); + File metadataFile1 = + new File(destinationFile1.getParent(), destinationFile1.getName() + ".meta"); + File metadataFile2 = + new File(destinationFile2.getParent(), destinationFile2.getName() + ".meta"); + File metadataFile3 = + new File(destinationFile3.getParent(), destinationFile3.getName() + ".meta"); + SimpleFileDownloadDestination destination1 = + new SimpleFileDownloadDestination(destinationFile1, metadataFile1); + SimpleFileDownloadDestination destination2 = + new SimpleFileDownloadDestination(destinationFile2, metadataFile2); + SimpleFileDownloadDestination destination3 = + new SimpleFileDownloadDestination(destinationFile3, metadataFile3); + + dispatcher.responseForPath( + "/foo", + () -> + new MockResponse() + .setResponseCode(HttpURLConnection.HTTP_OK) + .setBodyDelay(50, TimeUnit.MILLISECONDS) + .setBody("hello world one")); + dispatcher.responseForPath( + "/bar", + () -> + new MockResponse() + .setResponseCode(HttpURLConnection.HTTP_OK) + .setBodyDelay(50, TimeUnit.MILLISECONDS) + .setBody("hello world two")); + dispatcher.responseForPath( + "/baz", + () -> + new MockResponse() + .setResponseCode(HttpURLConnection.HTTP_OK) + .setBodyDelay(50, TimeUnit.MILLISECONDS) + .setBody("hello world three")); + mockWebServer.start(); + + Downloader downloader = buildDownloader().build(); + downloader.registerStateChangeCallback(mockCallback, MoreExecutors.directExecutor()); + DownloadRequest request1 = + downloader.newRequestBuilder(mockWebServer.url("/foo").uri(), destination1).build(); + DownloadRequest request2 = + downloader.newRequestBuilder(mockWebServer.url("/bar").uri(), destination2).build(); + DownloadRequest request3 = + downloader.newRequestBuilder(mockWebServer.url("/baz").uri(), destination3).build(); + FluentFuture<DownloadResult> resultFuture1 = downloader.execute(request1); + FluentFuture<DownloadResult> resultFuture2 = downloader.execute(request2); + FluentFuture<DownloadResult> resultFuture3 = downloader.execute(request3); + + verify(mockCallback, atLeastOnce()).onStateChange(State.create(1, 2, 0)); + + // Cancel the first request. + assertThat(resultFuture1.cancel(true)).isTrue(); + assertThrows(CancellationException.class, resultFuture1::get); + + // Flush through the I/O executor to make sure internal state has settled. + ioExecutor.submit(() -> {}).get(); + + // Cancellation is racy. After it happens, there is either one request in flight and one still + // queued, or both still queued. The third request did not ignore the limit, see b/148559122. + verify(mockCallback, atLeastOnce()) + .onStateChange(or(eq(State.create(1, 1, 0)), eq(State.create(0, 2, 0)))); + + DownloadResult result2 = resultFuture2.get(10, SECONDS); + DownloadResult result3 = resultFuture3.get(10, SECONDS); + assertThat(result2.bytesWritten()).isEqualTo(15L); // == "hello world two".length + assertThat(result3.bytesWritten()).isEqualTo(17L); // == "hello world three".length + assertThat(Files.asCharSource(destinationFile2, UTF_8).read()).isEqualTo("hello world two"); + assertThat(Files.asCharSource(destinationFile3, UTF_8).read()).isEqualTo("hello world three"); + assertThat(mockWebServer.getRequestCount()).isAnyOf(2, 3); + + // Flush through the I/O executor to make sure internal state has settled. + ioExecutor.submit(() -> {}).get(); + + verify(mockCallback, atLeastOnce()).onStateChange(State.create(0, 0, 0)); + } + + @Test + public void closingFutureDoesntLeak() throws Exception { + TestLogHandler logHandler = new TestLogHandler(); + Logger.getLogger(ClosingFuture.class.getName()).addHandler(logHandler); + + File destinationFile1 = temporaryFolder.newFile(); + File destinationFile2 = temporaryFolder.newFile(); + File metadataFile1 = + new File(destinationFile1.getParent(), destinationFile1.getName() + ".meta"); + File metadataFile2 = + new File(destinationFile2.getParent(), destinationFile2.getName() + ".meta"); + SimpleFileDownloadDestination destination1 = + new SimpleFileDownloadDestination(destinationFile1, metadataFile1); + SimpleFileDownloadDestination destination2 = + new SimpleFileDownloadDestination(destinationFile2, metadataFile2); + + dispatcher.responseForPath( + "/foo1", + () -> + new MockResponse() + .setResponseCode(HttpURLConnection.HTTP_OK) + .setBodyDelay(50, TimeUnit.MILLISECONDS) + .setBody("hello world one")); + dispatcher.responseForPath( + "/foo2", + () -> + new MockResponse() + .setResponseCode(HttpURLConnection.HTTP_OK) + .setBodyDelay(50, TimeUnit.MILLISECONDS) + .setBody("hello world two")); + mockWebServer.start(); + + ManualConnectivity connectivityHandler = new ManualConnectivity(); + + Downloader downloader = buildDownloader().withConnectivityHandler(connectivityHandler).build(); + + DownloadRequest request1 = + downloader.newRequestBuilder(mockWebServer.url("/foo1").uri(), destination1).build(); + URI request2Uri = mockWebServer.url("/foo2").uri(); + DownloadRequest request2 = downloader.newRequestBuilder(request2Uri, destination2).build(); + + FluentFuture<DownloadResult> resultFuture1 = downloader.execute(request1); + connectivityHandler.setConnectivitySatisfied(); + connectivityHandler.succeed(); + + FluentFuture<DownloadResult> resultFuture2 = downloader.execute(request2); + + resultFuture1.get(); + resultFuture2.get(); + System.gc(); + + assertAbout(streams()) + .that(logHandler.logRecords.stream().map(LogRecord::getLevel)) + .doesNotContain(Level.SEVERE); + } + + private static class AlwaysConnected implements ConnectivityHandler { + @Override + public ListenableFuture<Void> checkConnectivity(DownloadConstraints constraints) { + return Futures.immediateFuture(null); + } + } + + private static class ManualConnectivity implements ConnectivityHandler { + private final List<SettableFuture<Void>> futures = new ArrayList<>(); + private boolean connectivitySatisfied = false; + + public void succeed() { + for (SettableFuture<Void> future : futures) { + future.set(null); + } + futures.clear(); + } + + public void fail(Throwable throwable) { + for (SettableFuture<Void> future : futures) { + future.setException(throwable); + } + futures.clear(); + } + + void setConnectivitySatisfied() { + this.connectivitySatisfied = true; + } + + @Override + public ListenableFuture<Void> checkConnectivity(DownloadConstraints constraints) { + if (connectivitySatisfied) { + return Futures.immediateFuture(null); + } + SettableFuture<Void> future = SettableFuture.create(); + futures.add(future); + return future; + } + } + + private static class TestLogHandler extends Handler { + List<LogRecord> logRecords = new ArrayList<>(); + + @Override + public void close() {} + + @Override + public void flush() {} + + @Override + public void publish(LogRecord record) { + System.err.println("Handling logrecord: " + record.getMessage()); + logRecords.add(record); + } + } + + private static class DispatcherImpl extends Dispatcher { + private Map<String, Supplier<MockResponse>> responseMap = new HashMap<>(); + + public void responseForPath(String path, Supplier<MockResponse> responseSupplier) { + responseMap.put(path, responseSupplier); + } + + @Override + public MockResponse dispatch(RecordedRequest recordedRequest) throws InterruptedException { + String path = recordedRequest.getPath(); + if (path == null) { + return defaultResponse(); + } + Supplier<MockResponse> responseSupplier = responseMap.get(recordedRequest.getPath()); + if (responseSupplier == null) { + return defaultResponse(); + } + return responseSupplier.get(); + } + + @Override + public void shutdown() { + responseMap.clear(); + super.shutdown(); + } + + private MockResponse defaultResponse() { + return new MockResponse().setResponseCode(HttpURLConnection.HTTP_NOT_FOUND); + } + } +} diff --git a/src/test/java/com/google/android/downloader/IOUtilTest.java b/src/test/java/com/google/android/downloader/IOUtilTest.java new file mode 100644 index 0000000..b0a03bf --- /dev/null +++ b/src/test/java/com/google/android/downloader/IOUtilTest.java @@ -0,0 +1,73 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import static com.google.common.truth.Truth.assertThat; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.assertThrows; + +import com.google.common.io.Files; +import java.io.File; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.channels.SocketChannel; +import java.nio.file.StandardOpenOption; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for IOUtil. */ +@RunWith(JUnit4.class) +public class IOUtilTest { + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void validateChannel_nonBlocking() throws Exception { + SocketChannel channel = SocketChannel.open(); + channel.configureBlocking(false); + assertThrows(IllegalStateException.class, () -> IOUtil.validateChannel(channel)); + } + + @Test + public void validateChannel_notOpen() throws Exception { + File testFile = temporaryFolder.newFile(); + FileChannel channel = FileChannel.open(testFile.toPath()); + channel.close(); + + assertThrows(IllegalStateException.class, () -> IOUtil.validateChannel(channel)); + } + + @Test + public void validateChannel_valid() throws Exception { + File testFile = temporaryFolder.newFile(); + FileChannel channel = FileChannel.open(testFile.toPath()); + IOUtil.validateChannel(channel); + } + + @Test + public void blockingWrite() throws Exception { + String message = "hello world"; + File testFile = temporaryFolder.newFile(); + + FileChannel channel = FileChannel.open(testFile.toPath(), StandardOpenOption.WRITE); + ByteBuffer buffer = ByteBuffer.wrap(message.getBytes(UTF_8)); + IOUtil.blockingWrite(buffer, channel); + channel.close(); + + assertThat(Files.asCharSource(testFile, UTF_8).read()).isEqualTo(message); + } +} diff --git a/src/test/java/com/google/android/downloader/MockWebServerUrlEngineTestHelper.java b/src/test/java/com/google/android/downloader/MockWebServerUrlEngineTestHelper.java new file mode 100644 index 0000000..fa80c4e --- /dev/null +++ b/src/test/java/com/google/android/downloader/MockWebServerUrlEngineTestHelper.java @@ -0,0 +1,338 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import static com.google.common.truth.Truth.assertThat; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.assertThrows; + +import com.google.common.collect.ImmutableList; +import com.google.common.io.Files; +import com.google.common.net.HttpHeaders; +import com.google.common.util.concurrent.ListenableFuture; +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.net.HttpURLConnection; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.channels.WritableByteChannel; +import java.util.concurrent.CancellationException; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import okhttp3.HttpUrl; +import okhttp3.mockwebserver.MockResponse; +import okhttp3.mockwebserver.MockWebServer; +import okhttp3.mockwebserver.RecordedRequest; +import okhttp3.mockwebserver.SocketPolicy; +import okio.Buffer; +import org.junit.rules.TemporaryFolder; + +/** + * Helper class for executing common test behaviors for {@link UrlEngine} instances that interact + * with the {@link MockWebServer}. + */ +class MockWebServerUrlEngineTestHelper { + private final TemporaryFolder temporaryFolder; + private final TestingExecutorService executorService; + private final MockWebServer server; + + MockWebServerUrlEngineTestHelper( + TemporaryFolder temporaryFolder, TestingExecutorService executorService) { + this.temporaryFolder = temporaryFolder; + this.executorService = executorService; + server = new MockWebServer(); + } + + void tearDown() throws Exception { + server.shutdown(); + } + + void executeRequest_normalResponse_succeeds(UrlEngine engine) throws Exception { + String message = "foobar"; + String encoding = "text/plain"; + String path = "/path"; + + server.enqueue( + new MockResponse() + .setBody(message) + .setHeader(HttpHeaders.CONTENT_TYPE, encoding) + .setResponseCode(HttpURLConnection.HTTP_OK)); + server.start(); + + HttpUrl url = server.url("/path"); + + File file = temporaryFolder.newFile(); + RandomAccessFile randomAccessFile = new RandomAccessFile(file, "rw"); + FileChannel channel = randomAccessFile.getChannel(); + + UrlRequest request = + engine + .createRequest(url.toString()) + .addHeader(HttpHeaders.CACHE_CONTROL, "no-cache") + .build(); + ListenableFuture<UrlResponse> responseFuture = request.send(); + UrlResponse response = responseFuture.get(); + ListenableFuture<Long> writeFuture = response.readResponseBody(channel); + long bytesWritten = writeFuture.get(); + response.close(); + channel.close(); + + assertThat(bytesWritten).isEqualTo(message.getBytes(UTF_8).length); + assertThat(Files.asCharSource(file, UTF_8).read()).isEqualTo(message); + assertThat(response.getResponseCode()).isEqualTo(HttpURLConnection.HTTP_OK); + assertThat(response.getResponseHeaders()) + .containsEntry(HttpHeaders.CONTENT_TYPE, ImmutableList.of(encoding)); + assertThat(server.getRequestCount()).isEqualTo(1); + RecordedRequest recordedRequest = server.takeRequest(); + assertThat(recordedRequest.getHeaders().toMultimap()) + .containsEntry(HttpHeaders.CACHE_CONTROL, ImmutableList.of("no-cache")); + assertThat(recordedRequest.getMethod()).isEqualTo("GET"); + assertThat(recordedRequest.getPath()).isEqualTo(path); + } + + void executeRequest_responseThrottled_succeeds(UrlEngine engine) throws Exception { + String message = "foobar"; + String encoding = "text/plain"; + String path = "/path"; + + server.enqueue( + new MockResponse() + .setBody(message) + .throttleBody(2, 500, TimeUnit.MILLISECONDS) + .setHeader(HttpHeaders.CONTENT_TYPE, encoding) + .setResponseCode(HttpURLConnection.HTTP_OK)); + server.start(); + + HttpUrl url = server.url("/path"); + + File file = temporaryFolder.newFile(); + RandomAccessFile randomAccessFile = new RandomAccessFile(file, "rw"); + FileChannel channel = randomAccessFile.getChannel(); + + UrlRequest request = + engine + .createRequest(url.toString()) + .addHeader(HttpHeaders.CACHE_CONTROL, "no-cache") + .build(); + ListenableFuture<UrlResponse> responseFuture = request.send(); + UrlResponse response = responseFuture.get(); + ListenableFuture<Long> writeFuture = response.readResponseBody(channel); + long bytesWritten = writeFuture.get(); + response.close(); + channel.close(); + + assertThat(bytesWritten).isEqualTo(message.getBytes(UTF_8).length); + assertThat(Files.asCharSource(file, UTF_8).read()).isEqualTo(message); + assertThat(response.getResponseCode()).isEqualTo(HttpURLConnection.HTTP_OK); + assertThat(response.getResponseHeaders()) + .containsEntry(HttpHeaders.CONTENT_TYPE, ImmutableList.of(encoding)); + assertThat(server.getRequestCount()).isEqualTo(1); + RecordedRequest recordedRequest = server.takeRequest(); + assertThat(recordedRequest.getHeaders().toMultimap()) + .containsEntry(HttpHeaders.CACHE_CONTROL, ImmutableList.of("no-cache")); + assertThat(recordedRequest.getMethod()).isEqualTo("GET"); + assertThat(recordedRequest.getPath()).isEqualTo(path); + } + + void executeRequest_largeResponse_succeeds(UrlEngine engine, long bufferSizeBytes) + throws Exception { + Buffer bodyBuffer = new Buffer(); + for (long i = 0; i < bufferSizeBytes / 8 /* = Long.BYTES */; i++) { + bodyBuffer.writeLong(i); + } + + String encoding = "text/plain"; + String path = "/path"; + + server.enqueue( + new MockResponse() + .setBody(bodyBuffer.clone()) + .setHeader(HttpHeaders.CONTENT_TYPE, encoding) + .setResponseCode(HttpURLConnection.HTTP_OK)); + server.start(); + + HttpUrl url = server.url("/path"); + + File file = temporaryFolder.newFile(); + RandomAccessFile randomAccessFile = new RandomAccessFile(file, "rw"); + FileChannel channel = randomAccessFile.getChannel(); + + UrlRequest request = + engine + .createRequest(url.toString()) + .addHeader(HttpHeaders.CACHE_CONTROL, "no-cache") + .build(); + ListenableFuture<UrlResponse> responseFuture = request.send(); + UrlResponse response = responseFuture.get(); + ListenableFuture<Long> writeFuture = response.readResponseBody(channel); + long bytesWritten = writeFuture.get(); + response.close(); + channel.close(); + + assertThat(bytesWritten).isEqualTo(bodyBuffer.size()); + assertThat(Files.asByteSource(file).read()).isEqualTo(bodyBuffer.readByteArray()); + assertThat(response.getResponseCode()).isEqualTo(HttpURLConnection.HTTP_OK); + assertThat(response.getResponseHeaders()) + .containsEntry(HttpHeaders.CONTENT_TYPE, ImmutableList.of(encoding)); + assertThat(server.getRequestCount()).isEqualTo(1); + RecordedRequest recordedRequest = server.takeRequest(); + assertThat(recordedRequest.getHeaders().toMultimap()) + .containsEntry(HttpHeaders.CACHE_CONTROL, ImmutableList.of("no-cache")); + assertThat(recordedRequest.getMethod()).isEqualTo("GET"); + assertThat(recordedRequest.getPath()).isEqualTo(path); + } + + void executeRequest_closeBeforeWrite_failsAborted(UrlEngine engine) throws Exception { + String message = "foobar"; + String encoding = "text/plain"; + + server.enqueue( + new MockResponse() + .setBody(message) + .setHeader(HttpHeaders.CONTENT_TYPE, encoding) + .setResponseCode(HttpURLConnection.HTTP_OK)); + server.start(); + + HttpUrl url = server.url("/path"); + + File file = temporaryFolder.newFile(); + RandomAccessFile randomAccessFile = new RandomAccessFile(file, "rw"); + FileChannel channel = randomAccessFile.getChannel(); + + UrlRequest request = engine.createRequest(url.toString()).build(); + ListenableFuture<UrlResponse> responseFuture = request.send(); + UrlResponse response = responseFuture.get(); + response.close(); + + ListenableFuture<Long> writeFuture = response.readResponseBody(channel); + ExecutionException exception = assertThrows(ExecutionException.class, writeFuture::get); + channel.close(); + + assertThat(exception).hasCauseThat().isInstanceOf(RequestException.class); + } + + void executeRequest_serverError_failsInternalError(UrlEngine engine) throws Exception { + server.enqueue(new MockResponse().setResponseCode(HttpURLConnection.HTTP_INTERNAL_ERROR)); + server.start(); + + HttpUrl url = server.url("/path"); + + UrlRequest request = engine.createRequest(url.toString()).build(); + ListenableFuture<UrlResponse> responseFuture = request.send(); + + ExecutionException exception = assertThrows(ExecutionException.class, responseFuture::get); + assertThat(exception).hasCauseThat().isInstanceOf(RequestException.class); + + RequestException requestException = (RequestException) exception.getCause(); + assertThat(requestException.getErrorDetails().getHttpStatusCode()) + .isEqualTo(HttpURLConnection.HTTP_INTERNAL_ERROR); + } + + void executeRequest_networkError_failsInternalError(UrlEngine engine) throws Exception { + server.enqueue( + new MockResponse() + .setBody("foobar") + .setSocketPolicy(SocketPolicy.DISCONNECT_AFTER_REQUEST) + .setResponseCode(HttpURLConnection.HTTP_OK)); + server.start(); + + HttpUrl url = server.url("/path"); + + UrlRequest request = engine.createRequest(url.toString()).build(); + ListenableFuture<UrlResponse> responseFuture = request.send(); + + ExecutionException exception = assertThrows(ExecutionException.class, responseFuture::get); + assertThat(exception).hasCauseThat().isInstanceOf(RequestException.class); + } + + void executeRequest_writeError_failsInternalError(UrlEngine engine) throws Exception { + String message = "foobar"; + String errorMessage = "error message"; + + server.enqueue(new MockResponse().setBody(message).setResponseCode(HttpURLConnection.HTTP_OK)); + server.start(); + + HttpUrl url = server.url("/path"); + + UrlRequest request = engine.createRequest(url.toString()).build(); + ListenableFuture<UrlResponse> responseFuture = request.send(); + UrlResponse response = responseFuture.get(); + ListenableFuture<Long> writeFuture = + response.readResponseBody(new ThrowingWritableByteChannel(errorMessage)); + ExecutionException exception = assertThrows(ExecutionException.class, writeFuture::get); + response.close(); + + assertThat(response.getResponseCode()).isEqualTo(HttpURLConnection.HTTP_OK); + assertThat(server.getRequestCount()).isEqualTo(1); + + assertThat(exception).hasCauseThat().isInstanceOf(RequestException.class); + RequestException requestException = (RequestException) exception.getCause(); + assertThat(requestException).hasCauseThat().isInstanceOf(IOException.class); + assertThat(requestException).hasCauseThat().hasMessageThat().isEqualTo(errorMessage); + } + + void executeRequest_requestCanceled_requestNeverSent(UrlEngine engine) throws Exception { + server.enqueue( + new MockResponse() + .setSocketPolicy(SocketPolicy.NO_RESPONSE) + .setResponseCode(HttpURLConnection.HTTP_OK)); + server.start(); + executorService.pause(); + + HttpUrl url = server.url("/path"); + + UrlRequest request = engine.createRequest(url.toString()).build(); + ListenableFuture<UrlResponse> responseFuture = request.send(); + + responseFuture.cancel(true); + + executorService.resume(); + + assertThrows(CancellationException.class, responseFuture::get); + + // No requests were sent to the server! + assertThat(server.getRequestCount()).isEqualTo(0); + } + + void executeRequest_invalidUrl_failsInvalidArgument(UrlEngine engine) { + UrlRequest request = engine.createRequest("foo:bar").build(); + ListenableFuture<UrlResponse> responseFuture = request.send(); + + ExecutionException exception = assertThrows(ExecutionException.class, responseFuture::get); + assertThat(exception).hasCauseThat().isInstanceOf(RequestException.class); + } + + private static class ThrowingWritableByteChannel implements WritableByteChannel { + private final String message; + + ThrowingWritableByteChannel(String message) { + this.message = message; + } + + @Override + public int write(ByteBuffer src) throws IOException { + throw new IOException(message); + } + + @Override + public boolean isOpen() { + return true; + } + + @Override + public void close() {} + } +} diff --git a/src/test/java/com/google/android/downloader/OkHttp2UrlEngineTest.java b/src/test/java/com/google/android/downloader/OkHttp2UrlEngineTest.java new file mode 100644 index 0000000..a19c7b0 --- /dev/null +++ b/src/test/java/com/google/android/downloader/OkHttp2UrlEngineTest.java @@ -0,0 +1,106 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import static com.google.common.util.concurrent.MoreExecutors.listeningDecorator; +import static java.util.concurrent.Executors.newSingleThreadExecutor; + +import com.google.common.util.concurrent.ListeningExecutorService; +import com.squareup.okhttp.Dispatcher; +import com.squareup.okhttp.OkHttpClient; +import java.util.concurrent.Executors; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for OkHttp2UrlEngine. */ +@RunWith(JUnit4.class) +public class OkHttp2UrlEngineTest { + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + private MockWebServerUrlEngineTestHelper testHelper; + private OkHttp2UrlEngine engine; + private TestingExecutorService dispatchExecutorService; + private ListeningExecutorService transferExecutorService; + + @Before + public void setUp() { + dispatchExecutorService = new TestingExecutorService(Executors.newSingleThreadExecutor()); + testHelper = new MockWebServerUrlEngineTestHelper(temporaryFolder, dispatchExecutorService); + transferExecutorService = listeningDecorator(newSingleThreadExecutor()); + // Note: The OkHttpClient dispatcher uses the TestingExecutorService (which waits to execute + // tasks) in order to ensure that OkHttp requests are executed in an asynchronous manner, so + // that we can properly test request cancellation. + OkHttpClient client = new OkHttpClient(); + client.setDispatcher(new Dispatcher(dispatchExecutorService)); + engine = new OkHttp2UrlEngine(client, transferExecutorService); + } + + @After + public void tearDown() throws Exception { + testHelper.tearDown(); + transferExecutorService.shutdown(); + dispatchExecutorService.shutdown(); + } + + @Test + public void executeRequest_normalResponse_succeeds() throws Exception { + testHelper.executeRequest_normalResponse_succeeds(engine); + } + + @Test + public void executeRequest_responseThrottled_succeeds() throws Exception { + testHelper.executeRequest_responseThrottled_succeeds(engine); + } + + @Test + public void executeRequest_largeResponse_succeeds() throws Exception { + testHelper.executeRequest_largeResponse_succeeds(engine, 1024 * 1024); + } + + @Test + public void executeRequest_closeBeforeWrite_failsAborted() throws Exception { + testHelper.executeRequest_closeBeforeWrite_failsAborted(engine); + } + + @Test + public void executeRequest_serverError_failsInternalError() throws Exception { + testHelper.executeRequest_serverError_failsInternalError(engine); + } + + @Test + public void executeRequest_networkError_failsInternalError() throws Exception { + testHelper.executeRequest_networkError_failsInternalError(engine); + } + + @Test + public void executeRequest_writeError_failsInternalError() throws Exception { + testHelper.executeRequest_writeError_failsInternalError(engine); + } + + @Test + public void executeRequest_requestCanceled_requestNeverSent() throws Exception { + testHelper.executeRequest_requestCanceled_requestNeverSent(engine); + } + + @Test + public void executeRequest_invalidUrl_failsInvalidArgument() { + testHelper.executeRequest_invalidUrl_failsInvalidArgument(engine); + } +} diff --git a/src/test/java/com/google/android/downloader/OkHttp3UrlEngineTest.java b/src/test/java/com/google/android/downloader/OkHttp3UrlEngineTest.java new file mode 100644 index 0000000..88236b0 --- /dev/null +++ b/src/test/java/com/google/android/downloader/OkHttp3UrlEngineTest.java @@ -0,0 +1,106 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import static com.google.common.util.concurrent.MoreExecutors.listeningDecorator; +import static java.util.concurrent.Executors.newSingleThreadExecutor; + +import com.google.common.util.concurrent.ListeningExecutorService; +import java.util.concurrent.Executors; +import okhttp3.Dispatcher; +import okhttp3.OkHttpClient; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for OkHttp3UrlEngine. */ +@RunWith(JUnit4.class) +public class OkHttp3UrlEngineTest { + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + private MockWebServerUrlEngineTestHelper testHelper; + private OkHttp3UrlEngine engine; + private TestingExecutorService dispatchExecutorService; + private ListeningExecutorService transferExecutorService; + + @Before + public void setUp() { + dispatchExecutorService = new TestingExecutorService(Executors.newSingleThreadExecutor()); + testHelper = new MockWebServerUrlEngineTestHelper(temporaryFolder, dispatchExecutorService); + transferExecutorService = listeningDecorator(newSingleThreadExecutor()); + // Note: The OkHttpClient dispatcher uses the TestingExecutorService (which waits to execute + // tasks) in order to ensure that OkHttp requests are executed in an asynchronous manner, so + // that we can properly test request cancellation. + OkHttpClient client = + new OkHttpClient.Builder().dispatcher(new Dispatcher(dispatchExecutorService)).build(); + engine = new OkHttp3UrlEngine(client, transferExecutorService); + } + + @After + public void tearDown() throws Exception { + testHelper.tearDown(); + transferExecutorService.shutdown(); + dispatchExecutorService.shutdown(); + } + + @Test + public void executeRequest_normalResponse_succeeds() throws Exception { + testHelper.executeRequest_normalResponse_succeeds(engine); + } + + @Test + public void executeRequest_responseThrottled_succeeds() throws Exception { + testHelper.executeRequest_responseThrottled_succeeds(engine); + } + + @Test + public void executeRequest_largeResponse_succeeds() throws Exception { + testHelper.executeRequest_largeResponse_succeeds(engine, 1024 * 1024); + } + + @Test + public void executeRequest_closeBeforeWrite_failsAborted() throws Exception { + testHelper.executeRequest_closeBeforeWrite_failsAborted(engine); + } + + @Test + public void executeRequest_serverError_failsInternalError() throws Exception { + testHelper.executeRequest_serverError_failsInternalError(engine); + } + + @Test + public void executeRequest_networkError_failsInternalError() throws Exception { + testHelper.executeRequest_networkError_failsInternalError(engine); + } + + @Test + public void executeRequest_writeError_failsInternalError() throws Exception { + testHelper.executeRequest_writeError_failsInternalError(engine); + } + + @Test + public void executeRequest_requestCanceled_requestNeverSent() throws Exception { + testHelper.executeRequest_requestCanceled_requestNeverSent(engine); + } + + @Test + public void executeRequest_invalidUrl_failsInvalidArgument() { + testHelper.executeRequest_invalidUrl_failsInvalidArgument(engine); + } +} diff --git a/src/test/java/com/google/android/downloader/PlatformUrlEngineTest.java b/src/test/java/com/google/android/downloader/PlatformUrlEngineTest.java new file mode 100644 index 0000000..10ec850 --- /dev/null +++ b/src/test/java/com/google/android/downloader/PlatformUrlEngineTest.java @@ -0,0 +1,133 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.util.concurrent.MoreExecutors.listeningDecorator; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.concurrent.Executors.newSingleThreadExecutor; + +import com.google.common.io.Files; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import java.io.File; +import java.net.HttpURLConnection; +import java.nio.channels.FileChannel; +import java.nio.file.StandardOpenOption; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for PlatformUrlEngine. */ +@RunWith(JUnit4.class) +public class PlatformUrlEngineTest { + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + private MockWebServerUrlEngineTestHelper testHelper; + private PlatformUrlEngine engine; + private ListeningExecutorService executorService; + + @Before + public void setUp() { + executorService = listeningDecorator(newSingleThreadExecutor()); + testHelper = + new MockWebServerUrlEngineTestHelper( + temporaryFolder, new TestingExecutorService(executorService)); + engine = + new PlatformUrlEngine( + executorService, /* connectTimeoutMs= */ 1000, /* readTimeoutMs= */ 1000); + } + + @After + public void tearDown() throws Exception { + testHelper.tearDown(); + executorService.shutdown(); + } + + @Test + public void executeRequest_normalResponse_succeeds() throws Exception { + testHelper.executeRequest_normalResponse_succeeds(engine); + } + + @Test + public void executeRequest_responseThrottled_succeeds() throws Exception { + testHelper.executeRequest_responseThrottled_succeeds(engine); + } + + @Test + public void executeRequest_largeResponse_succeeds() throws Exception { + testHelper.executeRequest_largeResponse_succeeds( + engine, PlatformUrlEngine.BUFFER_SIZE_BYTES * 3); + } + + @Test + public void executeRequest_closeBeforeWrite_failsAborted() throws Exception { + testHelper.executeRequest_closeBeforeWrite_failsAborted(engine); + } + + @Test + public void executeRequest_serverError_failsInternalError() throws Exception { + testHelper.executeRequest_serverError_failsInternalError(engine); + } + + @Test + public void executeRequest_networkError_failsInternalError() throws Exception { + testHelper.executeRequest_networkError_failsInternalError(engine); + } + + @Test + public void executeRequest_writeError_failsInternalError() throws Exception { + testHelper.executeRequest_writeError_failsInternalError(engine); + } + + @Test + public void executeRequest_requestCanceled_requestNeverSent() throws Exception { + testHelper.executeRequest_requestCanceled_requestNeverSent(engine); + } + + @Test + public void executeRequest_invalidUrl_failsInvalidArgument() { + testHelper.executeRequest_invalidUrl_failsInvalidArgument(engine); + } + + @Test + public void executeRequest_fileUrl() throws Exception { + // Note: File url testing doesn't exist in MockWebServerUrlEngineTestHelper because + // it doesn't involve the MockWebServer and doesn't apply to all UrlEngine implementations. + String message = "foobar"; + File sourceFile = temporaryFolder.newFile(); + Files.asCharSink(sourceFile, UTF_8).write(message); + + String url = sourceFile.toURI().toURL().toString(); + UrlRequest request = engine.createRequest(url).build(); + ListenableFuture<? extends UrlResponse> responseFuture = request.send(); + UrlResponse response = responseFuture.get(); + + File targetFile = temporaryFolder.newFile(); + FileChannel channel = FileChannel.open(targetFile.toPath(), StandardOpenOption.WRITE); + ListenableFuture<Long> writeFuture = response.readResponseBody(channel); + long bytesWritten = writeFuture.get(); + response.close(); + channel.close(); + + assertThat(bytesWritten).isGreaterThan(0L); + assertThat(Files.asCharSource(targetFile, UTF_8).read()).isEqualTo(message); + assertThat(response.getResponseCode()).isEqualTo(HttpURLConnection.HTTP_OK); + } +} diff --git a/src/test/java/com/google/android/downloader/ProtoFileDownloadDestinationTest.java b/src/test/java/com/google/android/downloader/ProtoFileDownloadDestinationTest.java new file mode 100644 index 0000000..cc63cc7 --- /dev/null +++ b/src/test/java/com/google/android/downloader/ProtoFileDownloadDestinationTest.java @@ -0,0 +1,186 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import static com.google.common.truth.Truth.assertThat; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.assertThrows; + +import com.google.common.io.CharSource; +import com.google.common.io.Files; +import java.io.File; +import java.io.FileNotFoundException; +import java.nio.channels.Channels; +import java.nio.channels.WritableByteChannel; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for ProtoFileDownloadDestination. */ +@RunWith(JUnit4.class) +public final class ProtoFileDownloadDestinationTest { + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void numExistingBytes_fileDoesNotExist() throws Exception { + File targetFile = new File("/does/not/exist"); + File metadataFile = new File(targetFile.getParent(), targetFile.getName() + ".meta"); + ProtoFileDownloadDestination downloadDestination = + new ProtoFileDownloadDestination(targetFile, metadataFile); + + // Perhaps a bit surprising, but java.io.File.length returns 0 for a file that does not exist + assertThat(downloadDestination.numExistingBytes()).isEqualTo(0); + } + + @Test + public void numExistingBytes_fileEmpty() throws Exception { + File targetFile = temporaryFolder.newFile(); + File metadataFile = new File(targetFile.getParent(), targetFile.getName() + ".meta"); + ProtoFileDownloadDestination downloadDestination = + new ProtoFileDownloadDestination(targetFile, metadataFile); + assertThat(downloadDestination.numExistingBytes()).isEqualTo(0); + } + + @Test + public void numExistingBytes_fileNonEmpty() throws Exception { + File targetFile = temporaryFolder.newFile(); + File metadataFile = new File(targetFile.getParent(), targetFile.getName() + ".meta"); + ProtoFileDownloadDestination downloadDestination = + new ProtoFileDownloadDestination(targetFile, metadataFile); + Files.asCharSink(targetFile, UTF_8).write("Hello world"); + assertThat(downloadDestination.numExistingBytes()).isEqualTo(targetFile.length()); + } + + @Test + public void metadata_fileDoesNotExist() throws Exception { + File targetFile = new File("/does/not/exist"); + File metadataFile = new File(targetFile.getParent(), targetFile.getName() + ".meta"); + ProtoFileDownloadDestination downloadDestination = + new ProtoFileDownloadDestination(targetFile, metadataFile); + + DownloadMetadata metadata = downloadDestination.readMetadata(); + assertThat(metadata).isEqualTo(DownloadMetadata.create()); + } + + @Test + public void metadata_fileEmpty() throws Exception { + File targetFile = temporaryFolder.newFile(); + File metadataFile = new File(targetFile.getParent(), targetFile.getName() + ".meta"); + ProtoFileDownloadDestination downloadDestination = + new ProtoFileDownloadDestination(targetFile, metadataFile); + + DownloadMetadata metadata = downloadDestination.readMetadata(); + assertThat(metadata).isEqualTo(DownloadMetadata.create()); + } + + @Test + public void metadata_fileNonEmpty() throws Exception { + File targetFile = temporaryFolder.newFile(); + File metadataFile = new File(targetFile.getParent(), targetFile.getName() + ".meta"); + ProtoFileDownloadDestination downloadDestination = + new ProtoFileDownloadDestination(targetFile, metadataFile); + Files.asCharSink(targetFile, UTF_8).write("Hello world"); + + DownloadMetadata metadata = downloadDestination.readMetadata(); + assertThat(metadata).isEqualTo(DownloadMetadata.create()); + } + + @Test + public void openByteChannel_fileDoesNotExist() { + File targetFile = new File("/does/not/exist"); + File metadataFile = new File(targetFile.getParent(), targetFile.getName() + ".meta"); + ProtoFileDownloadDestination downloadDestination = + new ProtoFileDownloadDestination(targetFile, metadataFile); + assertThrows( + FileNotFoundException.class, + () -> downloadDestination.openByteChannel(0L, DownloadMetadata.create())); + } + + @Test + public void openByteChannel_fileEmpty() throws Exception { + File targetFile = temporaryFolder.newFile(); + File metadataFile = new File(targetFile.getParent(), targetFile.getName() + ".meta"); + ProtoFileDownloadDestination downloadDestination = + new ProtoFileDownloadDestination(targetFile, metadataFile); + String text = "Hello world"; + + WritableByteChannel channel = + downloadDestination.openByteChannel(0L, DownloadMetadata.create()); + assertThat(channel.isOpen()).isTrue(); + CharSource.wrap(text).asByteSource(UTF_8).copyTo(Channels.newOutputStream(channel)); + channel.close(); + + assertThat(Files.asCharSource(targetFile, UTF_8).read()).isEqualTo(text); + } + + @Test + public void openByteChannel_fileNonEmpty() throws Exception { + File targetFile = temporaryFolder.newFile(); + File metadataFile = new File(targetFile.getParent(), targetFile.getName() + ".meta"); + ProtoFileDownloadDestination downloadDestination = + new ProtoFileDownloadDestination(targetFile, metadataFile); + String text1 = "Hello world"; + Files.asCharSink(targetFile, UTF_8).write(text1); + String text2 = "Bye world"; + + WritableByteChannel channel = + downloadDestination.openByteChannel(targetFile.length(), DownloadMetadata.create()); + assertThat(channel.isOpen()).isTrue(); + CharSource.wrap(text2).asByteSource(UTF_8).copyTo(Channels.newOutputStream(channel)); + channel.close(); + + assertThat(Files.asCharSource(targetFile, UTF_8).read()).isEqualTo(text1 + text2); + } + + @Test + public void openByteChannel_metadata() throws Exception { + File targetFile = temporaryFolder.newFile(); + File metadataFile = new File(targetFile.getParent(), targetFile.getName() + ".meta"); + ProtoFileDownloadDestination downloadDestination = + new ProtoFileDownloadDestination(targetFile, metadataFile); + + String contentTag = "content_tag_abc"; + long lastModifiedTimeSeconds = 12345; + downloadDestination + .openByteChannel(0L, DownloadMetadata.create(contentTag, lastModifiedTimeSeconds)) + .close(); + + DownloadMetadata metadata = downloadDestination.readMetadata(); + assertThat(metadata).isEqualTo(DownloadMetadata.create(contentTag, lastModifiedTimeSeconds)); + } + + @Test + public void clear_fileNonEmpty() throws Exception { + File targetFile = temporaryFolder.newFile(); + File metadataFile = new File(targetFile.getParent(), targetFile.getName() + ".meta"); + ProtoFileDownloadDestination downloadDestination = + new ProtoFileDownloadDestination(targetFile, metadataFile); + Files.asCharSink(targetFile, UTF_8).write("existing"); + + WritableByteChannel channel1 = + downloadDestination.openByteChannel(0L, DownloadMetadata.create()); + downloadDestination.clear(); + WritableByteChannel channel2 = + downloadDestination.openByteChannel(0L, DownloadMetadata.create()); + + CharSource.wrap("swallowed").asByteSource(UTF_8).copyTo(Channels.newOutputStream(channel1)); + CharSource.wrap("replacement").asByteSource(UTF_8).copyTo(Channels.newOutputStream(channel2)); + channel2.close(); + + assertThat(Files.asCharSource(targetFile, UTF_8).read()).isEqualTo("replacement"); + } +} diff --git a/src/test/java/com/google/android/downloader/SimpleFileDownloadDestinationTest.java b/src/test/java/com/google/android/downloader/SimpleFileDownloadDestinationTest.java new file mode 100644 index 0000000..1eb0aaf --- /dev/null +++ b/src/test/java/com/google/android/downloader/SimpleFileDownloadDestinationTest.java @@ -0,0 +1,186 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import static com.google.common.truth.Truth.assertThat; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.Assert.assertThrows; + +import com.google.common.io.CharSource; +import com.google.common.io.Files; +import java.io.File; +import java.io.FileNotFoundException; +import java.nio.channels.Channels; +import java.nio.channels.WritableByteChannel; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Unit tests for SimpleFileDownloadDestination. */ +@RunWith(JUnit4.class) +public final class SimpleFileDownloadDestinationTest { + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void numExistingBytes_fileDoesNotExist() throws Exception { + File targetFile = new File("/does/not/exist"); + File metadataFile = new File(targetFile.getParent(), targetFile.getName() + ".meta"); + SimpleFileDownloadDestination downloadDestination = + new SimpleFileDownloadDestination(targetFile, metadataFile); + + // Perhaps a bit surprising, but java.io.File.length returns 0 for a file that does not exist + assertThat(downloadDestination.numExistingBytes()).isEqualTo(0); + } + + @Test + public void numExistingBytes_fileEmpty() throws Exception { + File targetFile = temporaryFolder.newFile(); + File metadataFile = new File(targetFile.getParent(), targetFile.getName() + ".meta"); + SimpleFileDownloadDestination downloadDestination = + new SimpleFileDownloadDestination(targetFile, metadataFile); + assertThat(downloadDestination.numExistingBytes()).isEqualTo(0); + } + + @Test + public void numExistingBytes_fileNonEmpty() throws Exception { + File targetFile = temporaryFolder.newFile(); + File metadataFile = new File(targetFile.getParent(), targetFile.getName() + ".meta"); + SimpleFileDownloadDestination downloadDestination = + new SimpleFileDownloadDestination(targetFile, metadataFile); + Files.asCharSink(targetFile, UTF_8).write("Hello world"); + assertThat(downloadDestination.numExistingBytes()).isEqualTo(targetFile.length()); + } + + @Test + public void metadata_fileDoesNotExist() throws Exception { + File targetFile = new File("/does/not/exist"); + File metadataFile = new File(targetFile.getParent(), targetFile.getName() + ".meta"); + SimpleFileDownloadDestination downloadDestination = + new SimpleFileDownloadDestination(targetFile, metadataFile); + + DownloadMetadata metadata = downloadDestination.readMetadata(); + assertThat(metadata).isEqualTo(DownloadMetadata.create()); + } + + @Test + public void metadata_fileEmpty() throws Exception { + File targetFile = temporaryFolder.newFile(); + File metadataFile = new File(targetFile.getParent(), targetFile.getName() + ".meta"); + SimpleFileDownloadDestination downloadDestination = + new SimpleFileDownloadDestination(targetFile, metadataFile); + + DownloadMetadata metadata = downloadDestination.readMetadata(); + assertThat(metadata).isEqualTo(DownloadMetadata.create()); + } + + @Test + public void metadata_fileNonEmpty() throws Exception { + File targetFile = temporaryFolder.newFile(); + File metadataFile = new File(targetFile.getParent(), targetFile.getName() + ".meta"); + SimpleFileDownloadDestination downloadDestination = + new SimpleFileDownloadDestination(targetFile, metadataFile); + Files.asCharSink(targetFile, UTF_8).write("Hello world"); + + DownloadMetadata metadata = downloadDestination.readMetadata(); + assertThat(metadata).isEqualTo(DownloadMetadata.create()); + } + + @Test + public void openByteChannel_fileDoesNotExist() { + File targetFile = new File("/does/not/exist"); + File metadataFile = new File(targetFile.getParent(), targetFile.getName() + ".meta"); + SimpleFileDownloadDestination downloadDestination = + new SimpleFileDownloadDestination(targetFile, metadataFile); + assertThrows( + FileNotFoundException.class, + () -> downloadDestination.openByteChannel(0L, DownloadMetadata.create())); + } + + @Test + public void openByteChannel_fileEmpty() throws Exception { + File targetFile = temporaryFolder.newFile(); + File metadataFile = new File(targetFile.getParent(), targetFile.getName() + ".meta"); + SimpleFileDownloadDestination downloadDestination = + new SimpleFileDownloadDestination(targetFile, metadataFile); + String text = "Hello world"; + + WritableByteChannel channel = + downloadDestination.openByteChannel(0L, DownloadMetadata.create()); + assertThat(channel.isOpen()).isTrue(); + CharSource.wrap(text).asByteSource(UTF_8).copyTo(Channels.newOutputStream(channel)); + channel.close(); + + assertThat(Files.asCharSource(targetFile, UTF_8).read()).isEqualTo(text); + } + + @Test + public void openByteChannel_fileNonEmpty() throws Exception { + File targetFile = temporaryFolder.newFile(); + File metadataFile = new File(targetFile.getParent(), targetFile.getName() + ".meta"); + SimpleFileDownloadDestination downloadDestination = + new SimpleFileDownloadDestination(targetFile, metadataFile); + String text1 = "Hello world"; + Files.asCharSink(targetFile, UTF_8).write(text1); + String text2 = "Bye world"; + + WritableByteChannel channel = + downloadDestination.openByteChannel(targetFile.length(), DownloadMetadata.create()); + assertThat(channel.isOpen()).isTrue(); + CharSource.wrap(text2).asByteSource(UTF_8).copyTo(Channels.newOutputStream(channel)); + channel.close(); + + assertThat(Files.asCharSource(targetFile, UTF_8).read()).isEqualTo(text1 + text2); + } + + @Test + public void openByteChannel_metadata() throws Exception { + File targetFile = temporaryFolder.newFile(); + File metadataFile = new File(targetFile.getParent(), targetFile.getName() + ".meta"); + SimpleFileDownloadDestination downloadDestination = + new SimpleFileDownloadDestination(targetFile, metadataFile); + + String contentTag = "content_tag_abc"; + long lastModifiedTimeSeconds = 12345; + downloadDestination + .openByteChannel(0L, DownloadMetadata.create(contentTag, lastModifiedTimeSeconds)) + .close(); + + DownloadMetadata metadata = downloadDestination.readMetadata(); + assertThat(metadata).isEqualTo(DownloadMetadata.create(contentTag, lastModifiedTimeSeconds)); + } + + @Test + public void clear_fileNonEmpty() throws Exception { + File targetFile = temporaryFolder.newFile(); + File metadataFile = new File(targetFile.getParent(), targetFile.getName() + ".meta"); + SimpleFileDownloadDestination downloadDestination = + new SimpleFileDownloadDestination(targetFile, metadataFile); + Files.asCharSink(targetFile, UTF_8).write("existing"); + + WritableByteChannel channel1 = + downloadDestination.openByteChannel(0L, DownloadMetadata.create()); + downloadDestination.clear(); + WritableByteChannel channel2 = + downloadDestination.openByteChannel(0L, DownloadMetadata.create()); + + CharSource.wrap("swallowed").asByteSource(UTF_8).copyTo(Channels.newOutputStream(channel1)); + CharSource.wrap("replacement").asByteSource(UTF_8).copyTo(Channels.newOutputStream(channel2)); + channel2.close(); + + assertThat(Files.asCharSource(targetFile, UTF_8).read()).isEqualTo("replacement"); + } +} diff --git a/src/test/java/com/google/android/downloader/TestExecutorRule.java b/src/test/java/com/google/android/downloader/TestExecutorRule.java new file mode 100644 index 0000000..250abe7 --- /dev/null +++ b/src/test/java/com/google/android/downloader/TestExecutorRule.java @@ -0,0 +1,114 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import static com.google.common.base.Throwables.getStackTraceAsString; +import static com.google.common.truth.Truth.assertThat; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.stream.Collectors.joining; +import static org.junit.Assert.fail; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadFactory; +import org.junit.rules.ExternalResource; + +/** + * A {@link org.junit.rules.TestRule} that manages and provides instances of {@link + * java.util.concurrent.Executor} and its various more specific interfaces. Takes care of shutting + * down any started threads and executors during execution, and also collects uncaught exceptions, + * failing the test and reporting the uncaught exception if any are found during execution. + */ +public class TestExecutorRule extends ExternalResource { + private final Duration timeout; + private final List<Throwable> uncaughtExceptions = new ArrayList<>(); + private final List<ExecutorService> executorServices = new ArrayList<>(); + private final ThreadFactory threadFactory = + runnable -> { + Thread thread = Executors.defaultThreadFactory().newThread(runnable); + // Insert an uncaught exception handler so that that errors happening on a background + // thread can be collected and cause test failures. + thread.setUncaughtExceptionHandler((t, e) -> uncaughtExceptions.add(e)); + return thread; + }; + + /** + * Constructs a new instance of this rule with the provided {@code timeout}. The timeout will be + * used when calling {@link ExecutorService#awaitTermination} on any {@link ExecutorService} + * instances created by this rule. + */ + public TestExecutorRule(Duration timeout) { + this.timeout = timeout; + } + + /** + * Creates a new single-threaded {@link ExecutorService} for use in tests. The executor will + * collect any uncaught exceptions encountered during test execution, and will fail the test with + * a detailed report of exceptions, if any are encountered. The executor will also be shut down + * and will await termination. Failure to shutdown in time (e.g. due to a blocked thread) will + * result in a test failure as well. + */ + public ExecutorService newSingleThreadExecutor() { + ExecutorService executorService = Executors.newSingleThreadExecutor(threadFactory); + executorServices.add(executorService); + return executorService; + } + + /** + * Creates a new single-threaded {@link ScheduledExecutorService} for use in tests. The executor + * will collect any uncaught exceptions encountered during test execution, and will fail the test + * with a detailed report of exceptions, if any are encountered. The executor will also be shut + * down and will await termination. Failure to shutdown in time (e.g. due to a blocked thread) + * will result in a test failure as well. + */ + public ScheduledExecutorService newSingleThreadScheduledExecutor() { + ScheduledExecutorService executorService = + Executors.newSingleThreadScheduledExecutor(threadFactory); + executorServices.add(executorService); + return executorService; + } + + @Override + protected void after() { + try { + for (ExecutorService executorService : executorServices) { + try { + executorService.shutdown(); + assertThat(executorService.awaitTermination(timeout.toMillis(), MILLISECONDS)).isTrue(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + fail("Error shutting down executor service:" + e); + } catch (Exception e) { + fail("Error shutting down executor service:" + e); + } + } + + if (!uncaughtExceptions.isEmpty()) { + String message = + uncaughtExceptions.stream() + .map(e -> "\n\t" + getStackTraceAsString(e).replace("\t", "\t\t")) + .collect(joining("\n")); + fail("Uncaught exceptions found: " + message); + } + } finally { + uncaughtExceptions.clear(); + executorServices.clear(); + } + } +} diff --git a/src/test/java/com/google/android/downloader/TestingExecutorService.java b/src/test/java/com/google/android/downloader/TestingExecutorService.java new file mode 100644 index 0000000..2fa6b40 --- /dev/null +++ b/src/test/java/com/google/android/downloader/TestingExecutorService.java @@ -0,0 +1,86 @@ +// Copyright 2021 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.android.downloader; + +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.common.util.concurrent.AbstractListeningExecutorService; +import java.util.ArrayDeque; +import java.util.List; +import java.util.Queue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; + +/** + * Implementation of {@link com.google.common.util.concurrent.ListeningExecutorService} for testing + * purposes. It artificially delays runnables enqueued via calls to {@link #execute} to allow tests + * to exercise async behavior. + */ +final class TestingExecutorService extends AbstractListeningExecutorService { + private final ExecutorService delegate; + private final Queue<Runnable> taskQueue = new ArrayDeque<>(); + + private boolean paused = false; + + TestingExecutorService(ExecutorService delegate) { + this.delegate = delegate; + } + + @Override + public void shutdown() { + delegate.shutdown(); + } + + @Override + public List<Runnable> shutdownNow() { + return delegate.shutdownNow(); + } + + @Override + public boolean isShutdown() { + return delegate.isShutdown(); + } + + @Override + public boolean isTerminated() { + return delegate.isTerminated(); + } + + @Override + public boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException { + return delegate.awaitTermination(timeout, unit); + } + + public synchronized void pause() { + paused = true; + } + + public synchronized void resume() { + paused = false; + while (!taskQueue.isEmpty()) { + delegate.execute(checkNotNull(taskQueue.poll())); + } + } + + @Override + public void execute(Runnable command) { + if (paused) { + taskQueue.add(command); + return; + } + + delegate.execute(command); + } +} |