diff options
Diffstat (limited to 'java/com')
161 files changed, 6328 insertions, 3115 deletions
diff --git a/java/com/google/android/libraries/mobiledatadownload/AggregateException.java b/java/com/google/android/libraries/mobiledatadownload/AggregateException.java index 94080d9..117918a 100644 --- a/java/com/google/android/libraries/mobiledatadownload/AggregateException.java +++ b/java/com/google/android/libraries/mobiledatadownload/AggregateException.java @@ -175,7 +175,7 @@ public final class AggregateException extends Exception { @VisibleForTesting static String throwableToString(Throwable failure) { - return throwableToString(failure, /*depth=*/ 1); + return throwableToString(failure, /* depth= */ 1); } private static String throwableToString(Throwable failure, int depth) { diff --git a/java/com/google/android/libraries/mobiledatadownload/BUILD b/java/com/google/android/libraries/mobiledatadownload/BUILD index 733d814..ca39a4e 100644 --- a/java/com/google/android/libraries/mobiledatadownload/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/BUILD @@ -13,7 +13,10 @@ # limitations under the License. load("@build_bazel_rules_android//android:rules.bzl", "android_library") +# MDI download (MDD) visibility is restricted to the following set of packages. Any +# new clients must be added to this list in order to grant build visibility. package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], @@ -22,27 +25,22 @@ package( android_library( name = "mobiledatadownload", - srcs = glob( - ["*.java"], - exclude = [ - "AccountSource.java", - "AggregateException.java", - "Configurator.java", - "TimeSource.java", - "Flags.java", - "Constants.java", - "DownloadException.java", - "DownloadListener.java", - "Logger.java", - "MobileDataDownloadBuilder.java", - "SilentFeedback.java", - "UsageEvent.java", - "SingleFileDownloadRequest.java", - "SingleFileDownloadListener.java", - "FileSource.java", - "ExperimentationConfig.java", - ], - ), + srcs = [ + "AddFileGroupRequest.java", + "CustomFileGroupValidator.java", + "DownloadFileGroupRequest.java", + "FileGroupPopulator.java", + "GetFileGroupRequest.java", + "GetFileGroupsByFilterRequest.java", + "ImportFilesRequest.java", + "MobileDataDownload.java", + "MobileDataDownloadImpl.java", + "ReadDataFileGroupRequest.java", + "RemoveFileGroupRequest.java", + "RemoveFileGroupsByFilterRequest.java", + "RemoveFileGroupsByFilterResponse.java", + "TaskScheduler.java", + ], exports = [ ":single_file_interfaces", ], @@ -51,22 +49,32 @@ android_library( ":DownloadListener", ":FileSource", ":Flags", + ":TimeSource", ":UsageEvent", ":single_file_interfaces", "//java/com/google/android/libraries/mobiledatadownload/account:AccountUtil", "//java/com/google/android/libraries/mobiledatadownload/file", + "//java/com/google/android/libraries/mobiledatadownload/foreground:ForegroundDownloadKey", "//java/com/google/android/libraries/mobiledatadownload/foreground:NotificationUtil", + "//java/com/google/android/libraries/mobiledatadownload/internal:DownloadGroupState", + "//java/com/google/android/libraries/mobiledatadownload/internal:ExceptionToMddResultMapper", + "//java/com/google/android/libraries/mobiledatadownload/internal:MddConstants", "//java/com/google/android/libraries/mobiledatadownload/internal:MobileDataDownloadManager", + "//java/com/google/android/libraries/mobiledatadownload/internal/collect", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:EventLogger", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LogUtil", "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", + "//java/com/google/android/libraries/mobiledatadownload/internal/util:DownloadFutureMap", "//java/com/google/android/libraries/mobiledatadownload/internal/util:MddLiteConversionUtil", "//java/com/google/android/libraries/mobiledatadownload/internal/util:ProtoConversionUtil", "//java/com/google/android/libraries/mobiledatadownload/lite", "//java/com/google/android/libraries/mobiledatadownload/monitor:DownloadProgressMonitor", "//java/com/google/android/libraries/mobiledatadownload/tracing", + "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", "//proto:client_config_java_proto_lite", "//proto:download_config_java_proto_lite", + "//proto:log_enums_java_proto_lite", + "//proto:logs_java_proto_lite", "@androidx_annotation_annotation", "@androidx_core_core", "@com_google_auto_value", @@ -86,20 +94,16 @@ android_library( ":AccountSource", ":Configurator", ":Constants", - ":DownloadException", - ":DownloadListener", ":ExperimentationConfig", ":Flags", ":Logger", ":SilentFeedback", ":mobiledatadownload", "//java/com/google/android/libraries/mobiledatadownload/account:AccountManagerAccountSource", - "//java/com/google/android/libraries/mobiledatadownload/account:AccountUtil", "//java/com/google/android/libraries/mobiledatadownload/delta:DeltaDecoder", "//java/com/google/android/libraries/mobiledatadownload/downloader:FileDownloader", "//java/com/google/android/libraries/mobiledatadownload/file", "//java/com/google/android/libraries/mobiledatadownload/foreground:NotificationUtil", - "//java/com/google/android/libraries/mobiledatadownload/internal:MobileDataDownloadManager", "//java/com/google/android/libraries/mobiledatadownload/internal/dagger:ApplicationContextModule", "//java/com/google/android/libraries/mobiledatadownload/internal/dagger:DownloaderModule", "//java/com/google/android/libraries/mobiledatadownload/internal/dagger:ExecutorsModule", @@ -111,19 +115,17 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/internal/logging:MddEventLogger", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:NoOpEventLogger", "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", - "//java/com/google/android/libraries/mobiledatadownload/internal/util:ProtoConversionUtil", "//java/com/google/android/libraries/mobiledatadownload/lite", "//java/com/google/android/libraries/mobiledatadownload/monitor:DownloadProgressMonitor", "//java/com/google/android/libraries/mobiledatadownload/monitor:NetworkUsageMonitor", - "//java/com/google/android/libraries/mobiledatadownload/tracing", + "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", "//proto:client_config_java_proto_lite", "//proto:download_config_java_proto_lite", + "//proto:logs_java_proto_lite", "@androidx_core_core", "@com_google_auto_value", - "@com_google_code_findbugs_jsr305", "@com_google_dagger", "@com_google_guava_guava", - "@com_google_protobuf//:protobuf_lite", ], ) @@ -199,7 +201,10 @@ android_library( android_library( name = "DownloadException", srcs = ["DownloadException.java"], - deps = ["@com_google_guava_guava"], + deps = [ + "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", + "@com_google_guava_guava", + ], ) android_library( @@ -241,6 +246,7 @@ android_library( ], deps = [ "//proto:client_config_java_proto_lite", + "//proto:log_enums_java_proto_lite", "@com_google_auto_value", ], ) diff --git a/java/com/google/android/libraries/mobiledatadownload/Constants.java b/java/com/google/android/libraries/mobiledatadownload/Constants.java index 7c71cd1..7b234b9 100644 --- a/java/com/google/android/libraries/mobiledatadownload/Constants.java +++ b/java/com/google/android/libraries/mobiledatadownload/Constants.java @@ -36,7 +36,7 @@ public final class Constants { /** The version of MDD library. Same as mdi_download module version. */ // TODO(b/122271766): Figure out how to update this automatically. // LINT.IfChange - public static final int MDD_LIB_VERSION = 422883838; + public static final int MDD_LIB_VERSION = 516938429; // LINT.ThenChange(<internal>) // <internal> diff --git a/java/com/google/android/libraries/mobiledatadownload/DownloadException.java b/java/com/google/android/libraries/mobiledatadownload/DownloadException.java index cc9a148..43f8659 100644 --- a/java/com/google/android/libraries/mobiledatadownload/DownloadException.java +++ b/java/com/google/android/libraries/mobiledatadownload/DownloadException.java @@ -17,10 +17,11 @@ package com.google.android.libraries.mobiledatadownload; import static com.google.common.util.concurrent.Futures.immediateFailedFuture; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; import com.google.common.base.Preconditions; -import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; +import com.google.errorprone.annotations.CanIgnoreReturnValue; /** Thrown when there is a download failure. */ public final class DownloadException extends Exception { @@ -171,18 +172,21 @@ public final class DownloadException extends Exception { private Throwable cause; /** Sets the {@link DownloadResultCode}. */ + @CanIgnoreReturnValue public Builder setDownloadResultCode(DownloadResultCode downloadResultCode) { this.downloadResultCode = downloadResultCode; return this; } /** Sets the error message. */ + @CanIgnoreReturnValue public Builder setMessage(String message) { this.message = message; return this; } /** Sets the cause of the exception. */ + @CanIgnoreReturnValue public Builder setCause(Throwable cause) { this.cause = cause; return this; @@ -213,7 +217,7 @@ public final class DownloadException extends Exception { */ public static <T> ListenableFuture<T> wrapIfFailed( ListenableFuture<T> future, DownloadResultCode code, String message) { - return Futures.catchingAsync( + return PropagatedFutures.catchingAsync( future, Throwable.class, (Throwable t) -> immediateFailedFuture(wrap(t, code, message)), diff --git a/java/com/google/android/libraries/mobiledatadownload/DownloadFileGroupRequest.java b/java/com/google/android/libraries/mobiledatadownload/DownloadFileGroupRequest.java index 8b98527..63c337c 100644 --- a/java/com/google/android/libraries/mobiledatadownload/DownloadFileGroupRequest.java +++ b/java/com/google/android/libraries/mobiledatadownload/DownloadFileGroupRequest.java @@ -27,12 +27,10 @@ import javax.annotation.concurrent.Immutable; public abstract class DownloadFileGroupRequest { /** Defines notifiction behavior for foreground download requests. */ - // LINT.IfChange(show_notifications) public enum ShowNotifications { NONE, ALL, } - // LINT.ThenChange(<internal>) DownloadFileGroupRequest() {} @@ -81,11 +79,16 @@ public abstract class DownloadFileGroupRequest { public abstract boolean preserveZipDirectories(); + public abstract boolean verifyIsolatedStructure(); + + public abstract Builder toBuilder(); + public static Builder newBuilder() { return new AutoValue_DownloadFileGroupRequest.Builder() .setGroupSizeBytes(0) .setShowNotifications(ShowNotifications.ALL) - .setPreserveZipDirectories(false); + .setPreserveZipDirectories(false) + .setVerifyIsolatedStructure(true); } /** Builder for {@link DownloadFileGroupRequest}. */ @@ -154,6 +157,21 @@ public abstract class DownloadFileGroupRequest { */ public abstract Builder setPreserveZipDirectories(boolean preserve); + /** + * By default, file groups will isolated structures will have this structure checked for each + * file when returning the file group. If the isolated structure is not correct, MDD will return + * a failure. + * + * <p>Setting this option to false allows clients to bypass this check, reducing the latency for + * critical callpaths. + * + * <p>For groups that do not have an isolated structure, this option is a no-op. + * + * <p>NOTE: All groups with isolated structures are also verified/fixed during MDD's maintenance + * periodic task. + */ + public abstract Builder setVerifyIsolatedStructure(boolean verifyIsolatedStructure); + public abstract DownloadFileGroupRequest build(); } } diff --git a/java/com/google/android/libraries/mobiledatadownload/DownloadListener.java b/java/com/google/android/libraries/mobiledatadownload/DownloadListener.java index 240406d..673bfc7 100644 --- a/java/com/google/android/libraries/mobiledatadownload/DownloadListener.java +++ b/java/com/google/android/libraries/mobiledatadownload/DownloadListener.java @@ -42,8 +42,14 @@ public interface DownloadListener { * * <p>The onComplete is run on MDD Control Executor. If you need to do heavy work, please offload * to a background task. + * + * <p>If using foreground downloads, an exception may be thrown here to tell MDD a failure + * notification should be shown instead of a success notification. <b>NOTE:</b> this is the only + * case where the exception will be taken into account. Throwing an exception here will + * <em>NOT</em> cause the download future returned by MDD to fail. */ - void onComplete(ClientFileGroup clientFileGroup); + // TODO (b/236401280): Switch to async api + void onComplete(ClientFileGroup clientFileGroup) throws Exception; /** This will be called when the download failed. */ default void onFailure(Throwable t) { diff --git a/java/com/google/android/libraries/mobiledatadownload/Flags.java b/java/com/google/android/libraries/mobiledatadownload/Flags.java index 6a5bead..1af1cb2 100644 --- a/java/com/google/android/libraries/mobiledatadownload/Flags.java +++ b/java/com/google/android/libraries/mobiledatadownload/Flags.java @@ -141,6 +141,7 @@ public interface Flags { return true; } + /** Controls whether daily maintenance includes {@link MobileDataDownload#collectGarbage}. */ default boolean mddEnableGarbageCollection() { return true; } @@ -184,10 +185,20 @@ public interface Flags { } default boolean enableRngBasedDeviceStableSampling() { - return false; // TODO(b/144684763): Switch to true after fully rolled out. + return true; + } + + /** + * Controls the key used for file download deduping. + * + * <p>By default, this flag is FALSE, so file download deduping is performed using the destination + * file uri. If this flag is enabled (TRUE), file download deduping will use NewFileKey. + */ + default boolean enableFileDownloadDedupByFileKey() { + return false; } - // PeriodTaskFlags + // PeriodicTaskFlags default long maintenanceGcmTaskPeriod() { return 86400; } diff --git a/java/com/google/android/libraries/mobiledatadownload/GetFileGroupRequest.java b/java/com/google/android/libraries/mobiledatadownload/GetFileGroupRequest.java index bf117d5..05cabf8 100644 --- a/java/com/google/android/libraries/mobiledatadownload/GetFileGroupRequest.java +++ b/java/com/google/android/libraries/mobiledatadownload/GetFileGroupRequest.java @@ -34,8 +34,12 @@ public abstract class GetFileGroupRequest { public abstract boolean preserveZipDirectories(); + public abstract boolean verifyIsolatedStructure(); + public static Builder newBuilder() { - return new AutoValue_GetFileGroupRequest.Builder().setPreserveZipDirectories(false); + return new AutoValue_GetFileGroupRequest.Builder() + .setPreserveZipDirectories(false) + .setVerifyIsolatedStructure(true); } /** Builder for {@link GetFileGroupRequest}. */ @@ -60,6 +64,21 @@ public abstract class GetFileGroupRequest { */ public abstract Builder setPreserveZipDirectories(boolean preserve); + /** + * By default, file groups will isolated structures will have this structure checked for each + * file when returning the file group. If the isolated structure is not correct, MDD will return + * a failure. + * + * <p>Setting this option to false allows clients to bypass this check, reducing the latency for + * critical callpaths. + * + * <p>For groups that do not have an isolated structure, this option is a no-op. + * + * <p>NOTE: All groups with isolated structures are also verified/fixed during MDD's maintenance + * periodic task. + */ + public abstract Builder setVerifyIsolatedStructure(boolean verifyIsolatedStructure); + public abstract GetFileGroupRequest build(); } } diff --git a/java/com/google/android/libraries/mobiledatadownload/GetFileGroupsByFilterRequest.java b/java/com/google/android/libraries/mobiledatadownload/GetFileGroupsByFilterRequest.java index 504ddf7..2901074 100644 --- a/java/com/google/android/libraries/mobiledatadownload/GetFileGroupsByFilterRequest.java +++ b/java/com/google/android/libraries/mobiledatadownload/GetFileGroupsByFilterRequest.java @@ -41,11 +41,14 @@ public abstract class GetFileGroupsByFilterRequest { public abstract boolean preserveZipDirectories(); + public abstract boolean verifyIsolatedStructure(); + public static Builder newBuilder() { return new AutoValue_GetFileGroupsByFilterRequest.Builder() .setIncludeAllGroups(false) .setGroupWithNoAccountOnly(false) - .setPreserveZipDirectories(false); + .setPreserveZipDirectories(false) + .setVerifyIsolatedStructure(true); } /** Builder for {@link GetFileGroupsByFilterRequest}. */ @@ -76,6 +79,21 @@ public abstract class GetFileGroupsByFilterRequest { */ public abstract Builder setPreserveZipDirectories(boolean preserve); + /** + * By default, file groups will isolated structures will have this structure checked for each + * file when returning the file group. If the isolated structure is not correct, MDD will return + * a failure. + * + * <p>Setting this option to false allows clients to bypass this check, reducing the latency for + * critical callpaths. + * + * <p>For groups that do not have an isolated structure, this option is a no-op. + * + * <p>NOTE: All groups with isolated structures are also verified/fixed during MDD's maintenance + * periodic task. + */ + public abstract Builder setVerifyIsolatedStructure(boolean verifyIsolatedStructure); + abstract GetFileGroupsByFilterRequest autoBuild(); public final GetFileGroupsByFilterRequest build() { @@ -84,6 +102,7 @@ public abstract class GetFileGroupsByFilterRequest { if (getFileGroupsByFilterRequest.includeAllGroups()) { checkArgument(!getFileGroupsByFilterRequest.groupNameOptional().isPresent()); checkArgument(!getFileGroupsByFilterRequest.accountOptional().isPresent()); + checkArgument(!getFileGroupsByFilterRequest.groupWithNoAccountOnly()); } else { checkArgument( getFileGroupsByFilterRequest.groupNameOptional().isPresent(), diff --git a/java/com/google/android/libraries/mobiledatadownload/MobileDataDownload.java b/java/com/google/android/libraries/mobiledatadownload/MobileDataDownload.java index 688691e..1894e86 100644 --- a/java/com/google/android/libraries/mobiledatadownload/MobileDataDownload.java +++ b/java/com/google/android/libraries/mobiledatadownload/MobileDataDownload.java @@ -18,10 +18,12 @@ package com.google.android.libraries.mobiledatadownload; import com.google.android.libraries.mobiledatadownload.TaskScheduler.ConstraintOverrides; import com.google.common.base.Optional; import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.errorprone.annotations.CanIgnoreReturnValue; import com.google.errorprone.annotations.CheckReturnValue; import com.google.mobiledatadownload.ClientConfigProto.ClientFileGroup; +import com.google.mobiledatadownload.DownloadConfigProto.DataFileGroup; import java.util.Map; /** The root object and entry point for the MobileDataDownload library. */ @@ -80,6 +82,18 @@ public interface MobileDataDownload { RemoveFileGroupsByFilterRequest removeFileGroupsByFilterRequest); /** + * Gets the file group definition that was added to MDD. This API cannot be used to access files, + * but it can be accessed by populators to manipulate the existing file group state - eg, to + * rename a file group, or otherwise migrate from one format to another. + * + * @return DataFileGroup if downloaded file group is found, otherwise a failing LF. + */ + default ListenableFuture<DataFileGroup> readDataFileGroup( + ReadDataFileGroupRequest readDataFileGroupRequest) { + throw new UnsupportedOperationException(); + } + + /** * Returns the latest downloaded data that we have for the given group name. * * <p>This api takes an instance of {@link GetFileGroupRequest} that contains group name, and it @@ -88,6 +102,10 @@ public interface MobileDataDownload { * <p>This listenable future will return null if no group exists or has been downloaded for the * given group name. * + * <p>Note: getFileGroup returns a snapshot of the latest state, but it's possible for the state + * to change between a getFileGroup call and accessing the files if the ClientFileGroup gets + * cached. Caching the returned ClientFileGroup is therefore discouraged. + * * @param getFileGroupRequest The request to get a single file group. * @return The ListenableFuture of requested client file group for the given request. */ @@ -102,6 +120,10 @@ public interface MobileDataDownload { * filtering, i.e. when no account is specified in the filter, file groups won't be filtered based * on account. * + * <p>Note: getFileGroupsByFilter returns a snapshot of the latest state, but it's possible for + * the state to change between a getFileGroupsByFilter call and accessing the files if the + * ClientFileGroup gets cached. Caching the returned ClientFileGroup is therefore discouraged. + * * @param getFileGroupsByFilterRequest The request to get multiple file groups after filtering. * @return The ListenableFuture that will resolve to a list of the requested client file groups, * including pending and downloaded versions; this ListenableFuture will resolve to all client @@ -227,8 +249,6 @@ public interface MobileDataDownload { * * @param downloadFileGroupRequest The request to download file group. */ - // TODO: Handle the case where a client calls this API for the same group when the - // earlier call has not finished. ListenableFuture<ClientFileGroup> downloadFileGroup( DownloadFileGroupRequest downloadFileGroupRequest); @@ -302,13 +322,15 @@ public interface MobileDataDownload { * <p>Attempts to cancel an on-going foreground download using best effort. If download is unknown * to MDD, this operation is a noop. * - * <p>If the download was started with {@link - * #downloadFileGroupWithForegroundService(DownloadFileGroupRequest)}, the specific {@code - * downloadKey} must be the group name of the file group. + * <p>The key passed here must be created using {@link ForegroundDownloadKey}, and must match the + * properties used from the request. Depending on which API was used to start the download, this + * would be {@link DownloadFileGroupRequest} for {@link SingleFileDownloadRequest}. * - * <p>If the download was started with {@link - * #downloadFileWithForegroundService(SingleFileDownloadRequest)}, the specific {@code - * downloadKey} must be the destination file uri (in string form). + * <p><b>NOTE:</b> In most cases, clients will not need to call this -- it is meant to allow the + * ForegroundDownloadService to cancel a download via the Cancel action registered to a + * notification. + * + * <p>Clients should prefer to cancel the future returned to them from the download call. * * @param downloadKey the key associated with the download */ @@ -328,6 +350,16 @@ public interface MobileDataDownload { ListenableFuture<Void> maintenance(); /** + * Perform garbage collection, which includes removing expired file groups and unreferenced files. + * + * <p>By default, this is run as part of {@link #maintenance} so doesn't need to be invoked + * directly by client code. If you disabled that behavior via {@link + * Flags#mddEnableGarbageCollection} then this method should be periodically called to clean up + * unused files. + */ + ListenableFuture<Void> collectGarbage(); + + /** * Schedule periodic tasks that will download and verify all file groups when the required * conditions are met, using the given {@link TaskScheduler}. * @@ -376,6 +408,18 @@ public interface MobileDataDownload { Optional<Map<String, ConstraintOverrides>> constraintOverridesMap); /** + * Cancels previously-scheduled periodic background tasks using the given {@link TaskScheduler}. + * Cancelling is best-effort and only meant to be used in an emergency; most apps will never need + * to call it. + * + * <p>If the host app doesn't provide a TaskScheduler, calling this API is a no-op. + */ + default ListenableFuture<Void> cancelPeriodicBackgroundTasks() { + // TODO(b/223822302): remove default once all implementations have been updated to include it + return Futures.immediateVoidFuture(); + } + + /** * Handle a task scheduled via a task scheduling service. * * <p>This method should not be called on the main thread, as it does work on the thread it is diff --git a/java/com/google/android/libraries/mobiledatadownload/MobileDataDownloadBuilder.java b/java/com/google/android/libraries/mobiledatadownload/MobileDataDownloadBuilder.java index 5cfb0eb..931dbac 100644 --- a/java/com/google/android/libraries/mobiledatadownload/MobileDataDownloadBuilder.java +++ b/java/com/google/android/libraries/mobiledatadownload/MobileDataDownloadBuilder.java @@ -34,22 +34,25 @@ import com.google.android.libraries.mobiledatadownload.internal.logging.NoOpEven import com.google.android.libraries.mobiledatadownload.lite.Downloader; import com.google.android.libraries.mobiledatadownload.monitor.DownloadProgressMonitor; import com.google.android.libraries.mobiledatadownload.monitor.NetworkUsageMonitor; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; import com.google.common.base.Optional; import com.google.common.base.Preconditions; import com.google.common.base.Supplier; import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.security.SecureRandom; import java.util.ArrayList; import java.util.List; import java.util.concurrent.Executor; /** - * A Builder for the {@link MobileDataDownload}. + * A builder for {@link MobileDataDownload}. + * + * <p> * * <p>WARNING: Only one object should be built. Otherwise, there may be locking errors on the * underlying database and unnecessary memory consumption. @@ -89,6 +92,7 @@ public final class MobileDataDownloadBuilder { componentBuilder = DaggerStandaloneComponent.builder(); } + @CanIgnoreReturnValue public MobileDataDownloadBuilder setContext(Context context) { this.context = context.getApplicationContext(); return this; @@ -103,6 +107,7 @@ public final class MobileDataDownloadBuilder { * directory, and periodic backbround tasks. There is no sharing and no-dedup between instances. * Please talk to <internal>@ before using this. */ + @CanIgnoreReturnValue public MobileDataDownloadBuilder setInstanceIdOptional(Optional<String> instanceIdOptional) { this.instanceIdOptional = instanceIdOptional; return this; @@ -114,6 +119,7 @@ public final class MobileDataDownloadBuilder { * <p>NOTE: Control Executor must not be single thread executor otherwise it could lead to * deadlock or other side effects. */ + @CanIgnoreReturnValue public MobileDataDownloadBuilder setControlExecutor(ListeningExecutorService controlExecutor) { Preconditions.checkNotNull(controlExecutor); // Executor that will execute tasks sequentially. @@ -127,6 +133,7 @@ public final class MobileDataDownloadBuilder { * <p>If this is not set, then the client is responsible for refreshing the list of file groups in * MDD as and when they see fit. */ + @CanIgnoreReturnValue public MobileDataDownloadBuilder addFileGroupPopulator(FileGroupPopulator fileGroupPopulator) { this.fileGroupPopulatorList.add(fileGroupPopulator); return this; @@ -139,6 +146,7 @@ public final class MobileDataDownloadBuilder { * <p>If this is not set, then the client is responsible for refreshing the list of file groups in * MDD as and when they see fit. */ + @CanIgnoreReturnValue public MobileDataDownloadBuilder addFileGroupPopulators( ImmutableList<FileGroupPopulator> fileGroupPopulators) { this.fileGroupPopulatorList.addAll(fileGroupPopulators); @@ -150,24 +158,28 @@ public final class MobileDataDownloadBuilder { * can use GCM, FJD or Work Manager to schedule tasks, and then forward the notification to {@link * MobileDataDownload#handleTask(String)}. */ + @CanIgnoreReturnValue public MobileDataDownloadBuilder setTaskScheduler(Optional<TaskScheduler> taskSchedulerOptional) { this.taskSchedulerOptional = taskSchedulerOptional; return this; } /** Set the optional Configurator which if present will be used by MDD to configure its flags. */ + @CanIgnoreReturnValue public MobileDataDownloadBuilder setConfiguratorOptional(Optional<Configurator> configurator) { this.configurator = configurator; return this; } /** Set the optional Logger which if present will be used by MDD to log events. */ + @CanIgnoreReturnValue public MobileDataDownloadBuilder setLoggerOptional(Optional<Logger> logger) { this.loggerOptional = logger; return this; } /** Set the flags otherwise default values will be used only. */ + @CanIgnoreReturnValue public MobileDataDownloadBuilder setFlagsOptional(Optional<Flags> flags) { this.flagsOptional = flags; return this; @@ -176,6 +188,7 @@ public final class MobileDataDownloadBuilder { /** * Set the optional SilentFeedback which if present will be used by MDD to send silent feedbacks. */ + @CanIgnoreReturnValue public MobileDataDownloadBuilder setSilentFeedbackOptional( Optional<SilentFeedback> silentFeedbackOptional) { this.silentFeedbackOptional = silentFeedbackOptional; @@ -186,6 +199,7 @@ public final class MobileDataDownloadBuilder { * Set the MobStore SynchronousFileStorage. Ideally this should be the same object as the one used * by the client app to read files from MDD */ + @CanIgnoreReturnValue public MobileDataDownloadBuilder setFileStorage(SynchronousFileStorage fileStorage) { this.fileStorage = fileStorage; return this; @@ -195,6 +209,7 @@ public final class MobileDataDownloadBuilder { * Set the NetworkUsageMonitor. This NetworkUsageMonitor instance must be the same instance that * is registered with SynchronousFileStorage. */ + @CanIgnoreReturnValue public MobileDataDownloadBuilder setNetworkUsageMonitor(NetworkUsageMonitor networkUsageMonitor) { this.networkUsageMonitor = networkUsageMonitor; return this; @@ -204,6 +219,7 @@ public final class MobileDataDownloadBuilder { * Set the DownloadProgressMonitor. This DownloadProgressMonitor instance must be the same * instance that is registered with SynchronousFileStorage. */ + @CanIgnoreReturnValue public MobileDataDownloadBuilder setDownloadMonitorOptional( Optional<DownloadProgressMonitor> downloadMonitorOptional) { this.downloadMonitorOptional = downloadMonitorOptional; @@ -214,6 +230,7 @@ public final class MobileDataDownloadBuilder { * Set the FileDownloader Supplier. MDD takes in a Supplier of FileDownload to support lazy * instantiation of the FileDownloader */ + @CanIgnoreReturnValue public MobileDataDownloadBuilder setFileDownloaderSupplier( Supplier<FileDownloader> fileDownloaderSupplier) { this.fileDownloaderSupplier = fileDownloaderSupplier; @@ -221,6 +238,7 @@ public final class MobileDataDownloadBuilder { } /** Set the Delta file decoder. */ + @CanIgnoreReturnValue public MobileDataDownloadBuilder setDeltaDecoderOptional( Optional<DeltaDecoder> deltaDecoderOptional) { this.deltaDecoderOptional = deltaDecoderOptional; @@ -235,6 +253,7 @@ public final class MobileDataDownloadBuilder { * shared as an optimization. Please talk to <internal>@ on how to setup a shared Foreground * Download Service. */ + @CanIgnoreReturnValue public MobileDataDownloadBuilder setForegroundDownloadServiceOptional( Optional<Class<?>> foregroundDownloadServiceClass) { this.foregroundDownloadServiceClassOptional = foregroundDownloadServiceClass; @@ -245,6 +264,7 @@ public final class MobileDataDownloadBuilder { * Sets the AccountSource that's used to wipeout account-related data at maintenance time. If this * method is not called, an account source based on AccountManager will be injected. */ + @CanIgnoreReturnValue public MobileDataDownloadBuilder setAccountSourceOptional( Optional<AccountSource> accountSourceOptional) { this.accountSourceOptional = accountSourceOptional; @@ -252,6 +272,7 @@ public final class MobileDataDownloadBuilder { return this; } + @CanIgnoreReturnValue public MobileDataDownloadBuilder setCustomFileGroupValidatorOptional( Optional<CustomFileGroupValidator> customFileGroupValidatorOptional) { this.customFileGroupValidatorOptional = customFileGroupValidatorOptional; @@ -263,6 +284,7 @@ public final class MobileDataDownloadBuilder { * sources. If this is not called, experiment ids are not propagated. See <internal> for more * details. */ + @CanIgnoreReturnValue public MobileDataDownloadBuilder setExperimentationConfigOptional( Optional<ExperimentationConfig> experimentationConfigOptional) { this.experimentationConfigOptional = experimentationConfigOptional; @@ -271,6 +293,7 @@ public final class MobileDataDownloadBuilder { // We use java.util.concurrent.Executor directly to create default Control Executor and // Download Executor. + public MobileDataDownload build() { Preconditions.checkNotNull(context); Preconditions.checkNotNull(taskSchedulerOptional); @@ -286,10 +309,10 @@ public final class MobileDataDownloadBuilder { // Submit commit task to sequentialControlExecutor to ensure that the commit task finishes // before any other API tasks can run. ListenableFuture<Void> commitFuture = - Futures.submitAsync( + PropagatedFutures.submitAsync( () -> configurator.get().commitToFlagSnapshot(), sequentialControlExecutor); - Futures.addCallback( + PropagatedFutures.addCallback( commitFuture, new FutureCallback<Void>() { @Override @@ -380,6 +403,7 @@ public final class MobileDataDownloadBuilder { foregroundDownloadServiceClassOptional, flags, singleFileDownloader, - customFileGroupValidatorOptional); + customFileGroupValidatorOptional, + component.getTimeSource()); } } diff --git a/java/com/google/android/libraries/mobiledatadownload/MobileDataDownloadImpl.java b/java/com/google/android/libraries/mobiledatadownload/MobileDataDownloadImpl.java index 4201b19..04abda1 100644 --- a/java/com/google/android/libraries/mobiledatadownload/MobileDataDownloadImpl.java +++ b/java/com/google/android/libraries/mobiledatadownload/MobileDataDownloadImpl.java @@ -15,16 +15,19 @@ */ package com.google.android.libraries.mobiledatadownload; -import static com.google.android.libraries.mobiledatadownload.tracing.TracePropagation.propagateAsyncCallable; import static com.google.android.libraries.mobiledatadownload.tracing.TracePropagation.propagateAsyncFunction; -import static com.google.android.libraries.mobiledatadownload.tracing.TracePropagation.propagateCallable; +import static com.google.android.libraries.mobiledatadownload.tracing.TracePropagation.propagateRunnable; +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.util.concurrent.Futures.getDone; +import static com.google.common.util.concurrent.Futures.immediateFailedFuture; +import static com.google.common.util.concurrent.Futures.immediateFuture; +import static com.google.common.util.concurrent.Futures.immediateVoidFuture; +import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import android.accounts.Account; import android.content.Context; import android.net.Uri; import android.text.TextUtils; -import android.util.Pair; -import androidx.annotation.VisibleForTesting; import androidx.core.app.NotificationCompat; import androidx.core.app.NotificationManagerCompat; import com.google.android.libraries.mobiledatadownload.DownloadException.DownloadResultCode; @@ -32,24 +35,33 @@ import com.google.android.libraries.mobiledatadownload.TaskScheduler.ConstraintO import com.google.android.libraries.mobiledatadownload.TaskScheduler.NetworkState; import com.google.android.libraries.mobiledatadownload.account.AccountUtil; import com.google.android.libraries.mobiledatadownload.file.SynchronousFileStorage; +import com.google.android.libraries.mobiledatadownload.foreground.ForegroundDownloadKey; import com.google.android.libraries.mobiledatadownload.foreground.NotificationUtil; +import com.google.android.libraries.mobiledatadownload.internal.DownloadGroupState; +import com.google.android.libraries.mobiledatadownload.internal.ExceptionToMddResultMapper; +import com.google.android.libraries.mobiledatadownload.internal.MddConstants; import com.google.android.libraries.mobiledatadownload.internal.MobileDataDownloadManager; +import com.google.android.libraries.mobiledatadownload.internal.collect.GroupKeyAndGroup; +import com.google.android.libraries.mobiledatadownload.internal.collect.GroupPair; import com.google.android.libraries.mobiledatadownload.internal.logging.EventLogger; import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; +import com.google.android.libraries.mobiledatadownload.internal.util.DownloadFutureMap; import com.google.android.libraries.mobiledatadownload.internal.util.MddLiteConversionUtil; import com.google.android.libraries.mobiledatadownload.internal.util.ProtoConversionUtil; import com.google.android.libraries.mobiledatadownload.lite.Downloader; import com.google.android.libraries.mobiledatadownload.monitor.DownloadProgressMonitor; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedExecutionSequencer; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFluentFuture; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; import com.google.common.base.Optional; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.util.concurrent.AsyncFunction; -import com.google.common.util.concurrent.ExecutionSequencer; -import com.google.common.util.concurrent.FluentFuture; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListenableFutureTask; import com.google.mobiledatadownload.ClientConfigProto.ClientFile; import com.google.mobiledatadownload.ClientConfigProto.ClientFileGroup; import com.google.mobiledatadownload.DownloadConfigProto; @@ -57,15 +69,15 @@ import com.google.mobiledatadownload.DownloadConfigProto.DataFileGroup; import com.google.mobiledatadownload.internal.MetadataProto.DataFile; import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; import com.google.mobiledatadownload.internal.MetadataProto.DownloadConditions; +import com.google.mobiledatadownload.internal.MetadataProto.DownloadConditions.DeviceNetworkPolicy; import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; +import com.google.mobiledatadownload.LogProto.DataDownloadFileGroupStats; import com.google.protobuf.Any; -import com.google.protobuf.GeneratedMessageLite; import com.google.protobuf.InvalidProtocolBufferException; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.PrintWriter; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; @@ -73,11 +85,13 @@ import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import javax.annotation.Nullable; + /** * Default implementation for {@link * com.google.android.libraries.mobiledatadownload.MobileDataDownload}. */ class MobileDataDownloadImpl implements MobileDataDownload { + private static final String TAG = "MobileDataDownload"; private static final long DUMP_DEBUG_INFO_TIMEOUT = 3; @@ -90,6 +104,13 @@ class MobileDataDownloadImpl implements MobileDataDownload { private final Flags flags; private final Downloader singleFileDownloader; + // Track all the on-going foreground downloads. This map is keyed by ForegroundDownloadKey. + private final DownloadFutureMap<ClientFileGroup> foregroundDownloadFutureMap; + + // Track all on-going background download requests started by downloadFileGroup. This map is keyed + // by ForegroundDownloadKey so request can be kept in sync with foregroundDownloadFutureMap. + private final DownloadFutureMap<ClientFileGroup> downloadFutureMap; + // This executor will execute tasks sequentially. private final Executor sequentialControlExecutor; // ExecutionSequencer will execute a ListenableFuture and its Futures.transforms before taking the @@ -97,15 +118,12 @@ class MobileDataDownloadImpl implements MobileDataDownload { // ExecutionSequencer to guarantee Metadata synchronization. Currently only downloadFileGroup and // handleTask APIs do not use ExecutionSequencer since their execution could take long time and // using ExecutionSequencer would block other APIs. - private final ExecutionSequencer futureSerializer = ExecutionSequencer.create(); + private final PropagatedExecutionSequencer futureSerializer = + PropagatedExecutionSequencer.create(); private final Optional<DownloadProgressMonitor> downloadMonitorOptional; private final Optional<Class<?>> foregroundDownloadServiceClassOptional; private final AsyncFunction<DataFileGroupInternal, Boolean> customFileGroupValidator; - - // Synchronization will be done through sequentialControlExecutor - // Keep all the on-going foreground downloads. - @VisibleForTesting - final Map<String, ListenableFuture<ClientFileGroup>> keyToListenableFuture = new HashMap<>(); + private final TimeSource timeSource; MobileDataDownloadImpl( Context context, @@ -119,7 +137,8 @@ class MobileDataDownloadImpl implements MobileDataDownload { Optional<Class<?>> foregroundDownloadServiceClassOptional, Flags flags, Downloader singleFileDownloader, - Optional<CustomFileGroupValidator> customValidatorOptional) { + Optional<CustomFileGroupValidator> customValidatorOptional, + TimeSource timeSource) { this.context = context; this.eventLogger = eventLogger; this.fileGroupPopulatorList = fileGroupPopulatorList; @@ -137,6 +156,12 @@ class MobileDataDownloadImpl implements MobileDataDownload { mobileDataDownloadManager, sequentialControlExecutor, fileStorage); + this.downloadFutureMap = DownloadFutureMap.create(sequentialControlExecutor); + this.foregroundDownloadFutureMap = + DownloadFutureMap.create( + sequentialControlExecutor, + createCallbacksForForegroundService(context, foregroundDownloadServiceClassOptional)); + this.timeSource = timeSource; } // Wraps the custom validator because the validation at a lower level of the stack where @@ -148,16 +173,17 @@ class MobileDataDownloadImpl implements MobileDataDownload { Executor executor, SynchronousFileStorage fileStorage) { if (!validatorOptional.isPresent()) { - return unused -> Futures.immediateFuture(true); + return unused -> immediateFuture(true); } return internalFileGroup -> - Futures.transformAsync( + PropagatedFutures.transformAsync( createClientFileGroup( internalFileGroup, /* account= */ null, ClientFileGroup.Status.PENDING_CUSTOM_VALIDATION, /* preserveZipDirectories= */ false, + /* verifyIsolatedStructure= */ true, mobileDataDownloadManager, executor, fileStorage), @@ -166,63 +192,168 @@ class MobileDataDownloadImpl implements MobileDataDownload { executor); } + /** + * Functional interface used as callback for logging file group stats. Used to create file group + * stats from the result of the future. + * + * @see attachMddApiLogging + */ + private interface StatsFromApiResultCreator<T> { + DataDownloadFileGroupStats create(T result); + } + + /** + * Functional interface used as callback when logging API result. Used to get the API result code + * from the result of the API future if it succeeds. + * + * <p>Note: The need for this is due to {@link addFileGroup} returning false instead of an + * exception if it fails. For other APIs with proper exception handling, it should suffice to + * immediately return the success code. + * + * <p>TODO(b/143572409): Remove once addGroupForDownload is updated to return void. + * + * @see attachMddApiLogging + */ + private interface ResultCodeFromApiResultGetter<T> { + int get(T result); + } + + /** + * Helper function used to log mdd api stats. Adds FutureCallback to the {@code resultFuture} + * which is the result of mdd api call and logs in onSuccess and onFailure functions of callback. + * + * @param apiName Code of the api being logged. + * @param resultFuture Future result of the api call. + * @param startTimeNs start time in ns. + * @param defaultFileGroupStats Initial file group stats. + * @param statsCreator This functional interface is invoked from the onSuccess of FutureCallback + * with the result of the future. File group stats returned here is merged with the initial + * stats and logged. + */ + private <T> void attachMddApiLogging( + int apiName, + ListenableFuture<T> resultFuture, + long startTimeNs, + DataDownloadFileGroupStats defaultFileGroupStats, + StatsFromApiResultCreator<T> statsCreator, + ResultCodeFromApiResultGetter<T> resultCodeGetter) { + // Using listener instead of transform since we need to log even if the future fails. + // Note: Listener is being registered on directexecutor for accurate latency measurement. + resultFuture.addListener( + propagateRunnable( + () -> { + long latencyNs = timeSource.elapsedRealtimeNanos() - startTimeNs; + // Log the stats asynchronously. + // Note: To avoid adding latency to mdd api calls, log asynchronously. + var unused = + PropagatedFutures.submit( + () -> { + int resultCode; + T result = null; + DataDownloadFileGroupStats fileGroupStats = defaultFileGroupStats; + try { + result = Futures.getDone(resultFuture); + resultCode = resultCodeGetter.get(result); + } catch (Throwable t) { + resultCode = ExceptionToMddResultMapper.map(t); + } + + // Merge stats created from result of api with the default stats. + if (result != null) { + fileGroupStats = + fileGroupStats.toBuilder() + .mergeFrom(statsCreator.create(result)) + .build(); + } + + Void resultLog = null; + + eventLogger.logMddLibApiResultLog(resultLog); + }, + sequentialControlExecutor); + }), + directExecutor()); + } + @Override public ListenableFuture<Boolean> addFileGroup(AddFileGroupRequest addFileGroupRequest) { - return futureSerializer.submitAsync( - propagateAsyncCallable( - () -> { - LogUtil.d( - "%s: Adding for download group = '%s', variant = '%s' and associating it with" - + " account = '%s', variant = '%s'", - TAG, - addFileGroupRequest.dataFileGroup().getGroupName(), - addFileGroupRequest.dataFileGroup().getVariantId(), - String.valueOf(addFileGroupRequest.accountOptional().orNull()), - String.valueOf(addFileGroupRequest.variantIdOptional().orNull())); - - DataFileGroup dataFileGroup = addFileGroupRequest.dataFileGroup(); - - // Ensure that the owner package is always set as the host app. - if (!dataFileGroup.hasOwnerPackage()) { - dataFileGroup = - dataFileGroup.toBuilder().setOwnerPackage(context.getPackageName()).build(); - } else if (!context.getPackageName().equals(dataFileGroup.getOwnerPackage())) { - LogUtil.e( - "%s: Added group = '%s' with wrong owner package: '%s' v.s. '%s' ", - TAG, - dataFileGroup.getGroupName(), - context.getPackageName(), - dataFileGroup.getOwnerPackage()); - return Futures.immediateFuture(false); - } + long startTimeNs = timeSource.elapsedRealtimeNanos(); + + ListenableFuture<Boolean> resultFuture = + futureSerializer.submitAsync( + () -> addFileGroupHelper(addFileGroupRequest), sequentialControlExecutor); + + DataDownloadFileGroupStats defaultFileGroupStats = + DataDownloadFileGroupStats.newBuilder() + .setFileGroupName(addFileGroupRequest.dataFileGroup().getGroupName()) + .setBuildId(addFileGroupRequest.dataFileGroup().getBuildId()) + .setVariantId(addFileGroupRequest.dataFileGroup().getVariantId()) + .setHasAccount(addFileGroupRequest.accountOptional().isPresent()) + .setFileGroupVersionNumber( + addFileGroupRequest.dataFileGroup().getFileGroupVersionNumber()) + .setOwnerPackage(addFileGroupRequest.dataFileGroup().getOwnerPackage()) + .setFileCount(addFileGroupRequest.dataFileGroup().getFileCount()) + .build(); + attachMddApiLogging( + 0, + resultFuture, + startTimeNs, + defaultFileGroupStats, + /* statsCreator= */ unused -> defaultFileGroupStats, + /* resultCodeGetter= */ succeeded -> succeeded ? 0 : 0); + + return resultFuture; + } - GroupKey.Builder groupKeyBuilder = - GroupKey.newBuilder() - .setGroupName(dataFileGroup.getGroupName()) - .setOwnerPackage(dataFileGroup.getOwnerPackage()); + private ListenableFuture<Boolean> addFileGroupHelper(AddFileGroupRequest addFileGroupRequest) { + LogUtil.d( + "%s: Adding for download group = '%s', variant = '%s', buildId = '%d' and" + + " associating it with account = '%s', variant = '%s'", + TAG, + addFileGroupRequest.dataFileGroup().getGroupName(), + addFileGroupRequest.dataFileGroup().getVariantId(), + addFileGroupRequest.dataFileGroup().getBuildId(), + String.valueOf(addFileGroupRequest.accountOptional().orNull()), + String.valueOf(addFileGroupRequest.variantIdOptional().orNull())); + + DataFileGroup dataFileGroup = addFileGroupRequest.dataFileGroup(); + + // Ensure that the owner package is always set as the host app. + if (!dataFileGroup.hasOwnerPackage()) { + dataFileGroup = dataFileGroup.toBuilder().setOwnerPackage(context.getPackageName()).build(); + } else if (!context.getPackageName().equals(dataFileGroup.getOwnerPackage())) { + LogUtil.e( + "%s: Added group = '%s' with wrong owner package: '%s' v.s. '%s' ", + TAG, + dataFileGroup.getGroupName(), + context.getPackageName(), + dataFileGroup.getOwnerPackage()); + return immediateFuture(false); + } - if (addFileGroupRequest.accountOptional().isPresent()) { - groupKeyBuilder.setAccount( - AccountUtil.serialize(addFileGroupRequest.accountOptional().get())); - } + GroupKey.Builder groupKeyBuilder = + GroupKey.newBuilder() + .setGroupName(dataFileGroup.getGroupName()) + .setOwnerPackage(dataFileGroup.getOwnerPackage()); - if (addFileGroupRequest.variantIdOptional().isPresent()) { - groupKeyBuilder.setVariantId(addFileGroupRequest.variantIdOptional().get()); - } + if (addFileGroupRequest.accountOptional().isPresent()) { + groupKeyBuilder.setAccount( + AccountUtil.serialize(addFileGroupRequest.accountOptional().get())); + } - try { - DataFileGroupInternal dataFileGroupInternal = - ProtoConversionUtil.convert(dataFileGroup); - return mobileDataDownloadManager.addGroupForDownloadInternal( - groupKeyBuilder.build(), dataFileGroupInternal, customFileGroupValidator); - } catch (InvalidProtocolBufferException e) { - // TODO(b/118137672): Consider rethrow exception instead of returning false. - LogUtil.e( - e, "%s: Unable to convert from DataFileGroup to DataFileGroupInternal.", TAG); - return Futures.immediateFuture(false); - } - }), - sequentialControlExecutor); + if (addFileGroupRequest.variantIdOptional().isPresent()) { + groupKeyBuilder.setVariantId(addFileGroupRequest.variantIdOptional().get()); + } + + try { + DataFileGroupInternal dataFileGroupInternal = ProtoConversionUtil.convert(dataFileGroup); + return mobileDataDownloadManager.addGroupForDownloadInternal( + groupKeyBuilder.build(), dataFileGroupInternal, customFileGroupValidator); + } catch (InvalidProtocolBufferException e) { + // TODO(b/118137672): Consider rethrow exception instead of returning false. + LogUtil.e(e, "%s: Unable to convert from DataFileGroup to DataFileGroupInternal.", TAG); + return immediateFuture(false); + } } // TODO: Change to return ListenableFuture<Void>. @@ -243,7 +374,7 @@ class MobileDataDownloadImpl implements MobileDataDownload { } GroupKey groupKey = groupKeyBuilder.build(); - return Futures.transform( + return PropagatedFutures.transform( mobileDataDownloadManager.removeFileGroup( groupKey, removeFileGroupRequest.pendingOnly()), voidArg -> true, @@ -257,29 +388,28 @@ class MobileDataDownloadImpl implements MobileDataDownload { RemoveFileGroupsByFilterRequest removeFileGroupsByFilterRequest) { return futureSerializer.submitAsync( () -> - FluentFuture.from(mobileDataDownloadManager.getAllFreshGroups()) + PropagatedFluentFuture.from(mobileDataDownloadManager.getAllFreshGroups()) .transformAsync( - allFreshGroups -> { + allFreshGroupKeyAndGroups -> { ImmutableSet.Builder<GroupKey> groupKeysToRemoveBuilder = ImmutableSet.builder(); - for (Pair<GroupKey, DataFileGroupInternal> keyDataFileGroupPair : - allFreshGroups) { + for (GroupKeyAndGroup groupKeyAndGroup : allFreshGroupKeyAndGroups) { if (applyRemoveFileGroupsFilter( - removeFileGroupsByFilterRequest, keyDataFileGroupPair)) { + removeFileGroupsByFilterRequest, groupKeyAndGroup)) { // Remove downloaded status so pending/downloaded versions of the same // group are treated as one. groupKeysToRemoveBuilder.add( - keyDataFileGroupPair.first.toBuilder().clearDownloaded().build()); + groupKeyAndGroup.groupKey().toBuilder().clearDownloaded().build()); } } ImmutableSet<GroupKey> groupKeysToRemove = groupKeysToRemoveBuilder.build(); if (groupKeysToRemove.isEmpty()) { - return Futures.immediateFuture( + return immediateFuture( RemoveFileGroupsByFilterResponse.newBuilder() .setRemovedFileGroupsCount(0) .build()); } - return Futures.transform( + return PropagatedFutures.transform( mobileDataDownloadManager.removeFileGroups(groupKeysToRemove.asList()), unused -> RemoveFileGroupsByFilterResponse.newBuilder() @@ -294,67 +424,135 @@ class MobileDataDownloadImpl implements MobileDataDownload { // Perform filtering using options from RemoveFileGroupsByFilterRequest private static boolean applyRemoveFileGroupsFilter( RemoveFileGroupsByFilterRequest removeFileGroupsByFilterRequest, - Pair<GroupKey, DataFileGroupInternal> keyDataFileGroupPair) { + GroupKeyAndGroup groupKeyAndGroup) { // If request filters by account, ensure account is present and is equal Optional<Account> accountOptional = removeFileGroupsByFilterRequest.accountOptional(); - if (!accountOptional.isPresent() && keyDataFileGroupPair.first.hasAccount()) { + if (!accountOptional.isPresent() && groupKeyAndGroup.groupKey().hasAccount()) { // Account must explicitly be provided in order to remove account associated file groups. return false; } if (accountOptional.isPresent() && !AccountUtil.serialize(accountOptional.get()) - .equals(keyDataFileGroupPair.first.getAccount())) { + .equals(groupKeyAndGroup.groupKey().getAccount())) { return false; } return true; } + /** + * Helper function to create {@link DataDownloadFileGroupStats} object from {@link + * GetFileGroupRequest} for getFileGroup() logging. + * + * <p>Used when the matching file group is not found or a failure occurred. + * file_group_version_number and build_id are set to -1 by default. + */ + private DataDownloadFileGroupStats createFileGroupStatsFromGetFileGroupRequest( + GetFileGroupRequest getFileGroupRequest) { + DataDownloadFileGroupStats.Builder fileGroupStatsBuilder = + DataDownloadFileGroupStats.newBuilder(); + fileGroupStatsBuilder.setFileGroupName(getFileGroupRequest.groupName()); + if (getFileGroupRequest.variantIdOptional().isPresent()) { + fileGroupStatsBuilder.setVariantId(getFileGroupRequest.variantIdOptional().get()); + } + if (getFileGroupRequest.accountOptional().isPresent()) { + fileGroupStatsBuilder.setHasAccount(true); + } else { + fileGroupStatsBuilder.setHasAccount(false); + } + + fileGroupStatsBuilder.setFileGroupVersionNumber( + MddConstants.FILE_GROUP_NOT_FOUND_FILE_GROUP_VERSION_NUMBER); + fileGroupStatsBuilder.setBuildId(MddConstants.FILE_GROUP_NOT_FOUND_BUILD_ID); + + return fileGroupStatsBuilder.build(); + } + // TODO: Futures.immediateFuture(null) uses a different annotation for Nullable. @SuppressWarnings("nullness") @Override public ListenableFuture<ClientFileGroup> getFileGroup(GetFileGroupRequest getFileGroupRequest) { - return futureSerializer.submitAsync( - () -> { - GroupKey.Builder groupKeyBuilder = - GroupKey.newBuilder() - .setGroupName(getFileGroupRequest.groupName()) - .setOwnerPackage(context.getPackageName()); + long startTimeNs = timeSource.elapsedRealtimeNanos(); - if (getFileGroupRequest.accountOptional().isPresent()) { - groupKeyBuilder.setAccount( - AccountUtil.serialize(getFileGroupRequest.accountOptional().get())); - } + ListenableFuture<ClientFileGroup> resultFuture = + futureSerializer.submitAsync( + () -> { + GroupKey groupKey = + createGroupKey( + getFileGroupRequest.groupName(), + getFileGroupRequest.accountOptional(), + getFileGroupRequest.variantIdOptional()); + return PropagatedFutures.transformAsync( + mobileDataDownloadManager.getFileGroup(groupKey, /* downloaded= */ true), + dataFileGroup -> + createClientFileGroupAndLogQueryStats( + groupKey, + dataFileGroup, + /* downloaded= */ true, + getFileGroupRequest.preserveZipDirectories(), + getFileGroupRequest.verifyIsolatedStructure()), + sequentialControlExecutor); + }, + sequentialControlExecutor); - if (getFileGroupRequest.variantIdOptional().isPresent()) { - groupKeyBuilder.setVariantId(getFileGroupRequest.variantIdOptional().get()); - } + attachMddApiLogging( + 0, + resultFuture, + startTimeNs, + createFileGroupStatsFromGetFileGroupRequest(getFileGroupRequest), + /* statsCreator= */ result -> createFileGroupDetails(result), + /* resultCodeGetter= */ unused -> 0); + return resultFuture; + } - GroupKey groupKey = groupKeyBuilder.build(); - return Futures.transformAsync( - mobileDataDownloadManager.getFileGroup(groupKey, /*downloaded=*/ true), - dataFileGroup -> - createClientFileGroupAndLogQueryStats( - groupKey, - dataFileGroup, - /*downloaded=*/ true, - getFileGroupRequest.preserveZipDirectories()), + @SuppressWarnings("nullness") + @Override + public ListenableFuture<DataFileGroup> readDataFileGroup( + ReadDataFileGroupRequest readDataFileGroupRequest) { + return futureSerializer.submitAsync( + () -> { + GroupKey groupKey = + createGroupKey( + readDataFileGroupRequest.groupName(), + readDataFileGroupRequest.accountOptional(), + readDataFileGroupRequest.variantIdOptional()); + return PropagatedFutures.transformAsync( + mobileDataDownloadManager.getFileGroup(groupKey, /* downloaded= */ true), + internalFileGroup -> immediateFuture(ProtoConversionUtil.reverse(internalFileGroup)), sequentialControlExecutor); }, sequentialControlExecutor); } + private GroupKey createGroupKey( + String groupName, Optional<Account> accountOptional, Optional<String> variantOptional) { + GroupKey.Builder groupKeyBuilder = + GroupKey.newBuilder().setGroupName(groupName).setOwnerPackage(context.getPackageName()); + + if (accountOptional.isPresent()) { + groupKeyBuilder.setAccount(AccountUtil.serialize(accountOptional.get())); + } + + if (variantOptional.isPresent()) { + groupKeyBuilder.setVariantId(variantOptional.get()); + } + + return groupKeyBuilder.build(); + } + private ListenableFuture<ClientFileGroup> createClientFileGroupAndLogQueryStats( GroupKey groupKey, @Nullable DataFileGroupInternal dataFileGroup, boolean downloaded, - boolean preserveZipDirectories) { - return Futures.transform( + boolean preserveZipDirectories, + boolean verifyIsolatedStructure) { + return PropagatedFutures.transform( createClientFileGroup( dataFileGroup, groupKey.hasAccount() ? groupKey.getAccount() : null, downloaded ? ClientFileGroup.Status.DOWNLOADED : ClientFileGroup.Status.PENDING, preserveZipDirectories, + verifyIsolatedStructure, mobileDataDownloadManager, sequentialControlExecutor, fileStorage), @@ -373,90 +571,91 @@ class MobileDataDownloadImpl implements MobileDataDownload { @Nullable String account, ClientFileGroup.Status status, boolean preserveZipDirectories, + boolean verifyIsolatedStructure, MobileDataDownloadManager manager, Executor executor, SynchronousFileStorage fileStorage) { if (dataFileGroup == null) { - return Futures.immediateFuture(null); + return immediateFuture(null); } - ClientFileGroup.Builder clientFileGroupBuilderInit = + ClientFileGroup.Builder clientFileGroupBuilder = ClientFileGroup.newBuilder() .setGroupName(dataFileGroup.getGroupName()) .setOwnerPackage(dataFileGroup.getOwnerPackage()) .setVersionNumber(dataFileGroup.getFileGroupVersionNumber()) +// .setCustomProperty(dataFileGroup.getCustomProperty()) .setBuildId(dataFileGroup.getBuildId()) .setVariantId(dataFileGroup.getVariantId()) .setStatus(status) .addAllLocale(dataFileGroup.getLocaleList()); if (account != null) { - clientFileGroupBuilderInit.setAccount(account); + clientFileGroupBuilder.setAccount(account); } if (dataFileGroup.hasCustomMetadata()) { - clientFileGroupBuilderInit.setCustomMetadata(dataFileGroup.getCustomMetadata()); + clientFileGroupBuilder.setCustomMetadata(dataFileGroup.getCustomMetadata()); } - ListenableFuture<ClientFileGroup.Builder> clientFileGroupBuilderFuture = - Futures.immediateFuture(clientFileGroupBuilderInit); - for (DataFile dataFile : dataFileGroup.getFileList()) { - clientFileGroupBuilderFuture = - Futures.transformAsync( - clientFileGroupBuilderFuture, - clientFileGroupBuilder -> { - if (status == ClientFileGroup.Status.DOWNLOADED - || status == ClientFileGroup.Status.PENDING_CUSTOM_VALIDATION) { - return Futures.transformAsync( - manager.getDataFileUri(dataFile, dataFileGroup), - fileUri -> { - if (fileUri == null) { - return Futures.immediateFailedFuture( - DownloadException.builder() - .setDownloadResultCode( - DownloadResultCode.DOWNLOADED_FILE_NOT_FOUND_ERROR) - .setMessage("getDataFileUri() resolved to null") - .build()); - } - try { - if (!preserveZipDirectories && fileStorage.isDirectory(fileUri)) { - String rootPath = fileUri.getPath(); - if (rootPath != null) { - clientFileGroupBuilder.addAllFile( - listAllClientFilesOfDirectory(fileStorage, fileUri, rootPath)); - } - } else { - clientFileGroupBuilder.addFile( - createClientFile( - dataFile.getFileId(), - dataFile.getByteSize(), - dataFile.getDownloadedFileByteSize(), - fileUri.toString(), - dataFile.hasCustomMetadata() - ? dataFile.getCustomMetadata() - : null)); + List<DataFile> dataFiles = dataFileGroup.getFileList(); + ListenableFuture<Void> addOnDeviceUrisFuture = immediateVoidFuture(); + if (status == ClientFileGroup.Status.DOWNLOADED + || status == ClientFileGroup.Status.PENDING_CUSTOM_VALIDATION) { + addOnDeviceUrisFuture = + PropagatedFluentFuture.from( + manager.getDataFileUris(dataFileGroup, verifyIsolatedStructure)) + .transformAsync( + dataFileUriMap -> { + for (DataFile dataFile : dataFiles) { + if (!dataFileUriMap.containsKey(dataFile)) { + return immediateFailedFuture( + DownloadException.builder() + .setDownloadResultCode( + DownloadResultCode.DOWNLOADED_FILE_NOT_FOUND_ERROR) + .setMessage("getDataFileUris() resolved to null") + .build()); + } + Uri uri = dataFileUriMap.get(dataFile); + + try { + if (!preserveZipDirectories && fileStorage.isDirectory(uri)) { + String rootPath = uri.getPath(); + if (rootPath != null) { + clientFileGroupBuilder.addAllFile( + listAllClientFilesOfDirectory(fileStorage, uri, rootPath)); } - } catch (IOException e) { - LogUtil.e(e, "Failed to list files under directory:" + fileUri); + } else { + clientFileGroupBuilder.addFile( + createClientFile( + dataFile.getFileId(), + dataFile.getByteSize(), + dataFile.getDownloadedFileByteSize(), + uri.toString(), + dataFile.hasCustomMetadata() + ? dataFile.getCustomMetadata() + : null)); } - return Futures.immediateFuture(clientFileGroupBuilder); - }, - executor); - } else { - clientFileGroupBuilder.addFile( - createClientFile( - dataFile.getFileId(), - dataFile.getByteSize(), - dataFile.getDownloadedFileByteSize(), - /* uri = */ null, - dataFile.hasCustomMetadata() ? dataFile.getCustomMetadata() : null)); - return Futures.immediateFuture(clientFileGroupBuilder); - } - }, - executor); + } catch (IOException e) { + LogUtil.e(e, "Failed to list files under directory:" + uri); + } + } + return immediateVoidFuture(); + }, + executor); + } else { + for (DataFile dataFile : dataFiles) { + clientFileGroupBuilder.addFile( + createClientFile( + dataFile.getFileId(), + dataFile.getByteSize(), + dataFile.getDownloadedFileByteSize(), + /* uri= */ null, + dataFile.hasCustomMetadata() ? dataFile.getCustomMetadata() : null)); + } } - return FluentFuture.from(clientFileGroupBuilderFuture) - .transform(GeneratedMessageLite.Builder::build, executor) + return PropagatedFluentFuture.from(addOnDeviceUrisFuture) + .transform(unused -> clientFileGroupBuilder.build(), executor) .catching(DownloadException.class, exn -> null, executor); } @@ -510,28 +709,29 @@ class MobileDataDownloadImpl implements MobileDataDownload { GetFileGroupsByFilterRequest getFileGroupsByFilterRequest) { return futureSerializer.submitAsync( () -> - Futures.transformAsync( + PropagatedFutures.transformAsync( mobileDataDownloadManager.getAllFreshGroups(), - allFreshGroups -> { + allFreshGroupKeyAndGroups -> { ListenableFuture<ImmutableList.Builder<ClientFileGroup>> clientFileGroupsBuilderFuture = - Futures.immediateFuture(ImmutableList.<ClientFileGroup>builder()); - for (Pair<GroupKey, DataFileGroupInternal> keyDataFileGroupPair : - allFreshGroups) { + immediateFuture(ImmutableList.<ClientFileGroup>builder()); + for (GroupKeyAndGroup groupKeyAndGroup : allFreshGroupKeyAndGroups) { clientFileGroupsBuilderFuture = - Futures.transformAsync( + PropagatedFutures.transformAsync( clientFileGroupsBuilderFuture, clientFileGroupsBuilder -> { - GroupKey groupKey = keyDataFileGroupPair.first; - DataFileGroupInternal dataFileGroup = keyDataFileGroupPair.second; + GroupKey groupKey = groupKeyAndGroup.groupKey(); + DataFileGroupInternal dataFileGroup = + groupKeyAndGroup.dataFileGroup(); if (applyFilter( getFileGroupsByFilterRequest, groupKey, dataFileGroup)) { - return Futures.transform( + return PropagatedFutures.transform( createClientFileGroupAndLogQueryStats( groupKey, dataFileGroup, groupKey.getDownloaded(), - getFileGroupsByFilterRequest.preserveZipDirectories()), + getFileGroupsByFilterRequest.preserveZipDirectories(), + getFileGroupsByFilterRequest.verifyIsolatedStructure()), clientFileGroup -> { if (clientFileGroup != null) { clientFileGroupsBuilder.add(clientFileGroup); @@ -540,12 +740,12 @@ class MobileDataDownloadImpl implements MobileDataDownload { }, sequentialControlExecutor); } - return Futures.immediateFuture(clientFileGroupsBuilder); + return immediateFuture(clientFileGroupsBuilder); }, sequentialControlExecutor); } - return Futures.transform( + return PropagatedFutures.transform( clientFileGroupsBuilderFuture, ImmutableList.Builder::build, sequentialControlExecutor); @@ -585,11 +785,19 @@ class MobileDataDownloadImpl implements MobileDataDownload { } /** - * Creates {@link IcingDataDownloadFileGroupStats} from {@link ClientFileGroup} for remote logging + * Creates {@link DataDownloadFileGroupStats} from {@link ClientFileGroup} for remote logging * purposes. */ - private static Void createFileGroupDetails(ClientFileGroup clientFileGroup) { - return null; + private static DataDownloadFileGroupStats createFileGroupDetails( + ClientFileGroup clientFileGroup) { + return DataDownloadFileGroupStats.newBuilder() + .setFileGroupName(clientFileGroup.getGroupName()) + .setOwnerPackage(clientFileGroup.getOwnerPackage()) + .setFileGroupVersionNumber(clientFileGroup.getVersionNumber()) + .setFileCount(clientFileGroup.getFileCount()) + .setVariantId(clientFileGroup.getVariantId()) + .setBuildId(clientFileGroup.getBuildId()) + .build(); } @Override @@ -633,6 +841,37 @@ class MobileDataDownloadImpl implements MobileDataDownload { @Override public ListenableFuture<ClientFileGroup> downloadFileGroup( DownloadFileGroupRequest downloadFileGroupRequest) { + // Submit the call to sequentialControlExecutor, but don't use futureSerializer. This will + // ensure that multiple calls are enqueued to the executor in a FIFO order, but these calls + // won't block each other when the download is in progress. + return PropagatedFutures.submitAsync( + () -> + PropagatedFutures.transformAsync( + // Check if requested file group has already been downloaded + getDownloadGroupState(downloadFileGroupRequest), + downloadGroupState -> { + switch (downloadGroupState.getKind()) { + case IN_PROGRESS_FUTURE: + // If the file group download is in progress, return that future immediately + return downloadGroupState.inProgressFuture(); + case DOWNLOADED_GROUP: + // If the file group is already downloaded, return that immediately. + return immediateFuture(downloadGroupState.downloadedGroup()); + case PENDING_GROUP: + return downloadPendingFileGroup(downloadFileGroupRequest); + } + throw new AssertionError( + String.format( + "received unsupported DownloadGroupState kind %s", + downloadGroupState.getKind())); + }, + sequentialControlExecutor), + sequentialControlExecutor); + } + + /** Helper method to download a group after it's determined to be pending. */ + private ListenableFuture<ClientFileGroup> downloadPendingFileGroup( + DownloadFileGroupRequest downloadFileGroupRequest) { String groupName = downloadFileGroupRequest.groupName(); GroupKey.Builder groupKeyBuilder = GroupKey.newBuilder().setGroupName(groupName).setOwnerPackage(context.getPackageName()); @@ -647,74 +886,107 @@ class MobileDataDownloadImpl implements MobileDataDownload { GroupKey groupKey = groupKeyBuilder.build(); - ListenableFuture<ClientFileGroup> downloadFuture = - Futures.submitAsync( - () -> { - if (downloadFileGroupRequest.listenerOptional().isPresent()) { - if (downloadMonitorOptional.isPresent()) { - downloadMonitorOptional - .get() - .addDownloadListener( - groupName, downloadFileGroupRequest.listenerOptional().get()); - } else { - return Futures.immediateFailedFuture( - DownloadException.builder() - .setDownloadResultCode( - DownloadResultCode.DOWNLOAD_MONITOR_NOT_PROVIDED_ERROR) - .setMessage( - "downloadFileGroup: DownloadListener is present but Download Monitor" - + " is not provided!") - .build()); - } - } + if (downloadFileGroupRequest.listenerOptional().isPresent()) { + if (downloadMonitorOptional.isPresent()) { + downloadMonitorOptional + .get() + .addDownloadListener(groupName, downloadFileGroupRequest.listenerOptional().get()); + } else { + return immediateFailedFuture( + DownloadException.builder() + .setDownloadResultCode(DownloadResultCode.DOWNLOAD_MONITOR_NOT_PROVIDED_ERROR) + .setMessage( + "downloadFileGroup: DownloadListener is present but Download Monitor" + + " is not provided!") + .build()); + } + } - Optional<DownloadConditions> downloadConditions = - downloadFileGroupRequest.downloadConditionsOptional().isPresent() - ? Optional.of( - ProtoConversionUtil.convert( - downloadFileGroupRequest.downloadConditionsOptional().get())) - : Optional.absent(); - ListenableFuture<DataFileGroupInternal> downloadFileGroupFuture = - mobileDataDownloadManager.downloadFileGroup( - groupKey, downloadConditions, customFileGroupValidator); - - return Futures.transformAsync( - downloadFileGroupFuture, - dataFileGroup -> { - return Futures.transform( - createClientFileGroup( - dataFileGroup, - downloadFileGroupRequest.accountOptional().isPresent() - ? AccountUtil.serialize( - downloadFileGroupRequest.accountOptional().get()) - : null, - ClientFileGroup.Status.DOWNLOADED, - downloadFileGroupRequest.preserveZipDirectories(), - mobileDataDownloadManager, - sequentialControlExecutor, - fileStorage), - Preconditions::checkNotNull, - sequentialControlExecutor); - }, - sequentialControlExecutor); - }, - sequentialControlExecutor); + Optional<DownloadConditions> downloadConditions; + try { + downloadConditions = + downloadFileGroupRequest.downloadConditionsOptional().isPresent() + ? Optional.of( + ProtoConversionUtil.convert( + downloadFileGroupRequest.downloadConditionsOptional().get())) + : Optional.absent(); + } catch (InvalidProtocolBufferException e) { + return immediateFailedFuture(e); + } + + // Get the key used for the download future map + ForegroundDownloadKey downloadKey = + ForegroundDownloadKey.ofFileGroup( + downloadFileGroupRequest.groupName(), + downloadFileGroupRequest.accountOptional(), + downloadFileGroupRequest.variantIdOptional()); + + // Create a ListenableFutureTask to delay starting the downloadFuture until we can add the + // future to our map. + ListenableFutureTask<Void> startTask = ListenableFutureTask.create(() -> null); + ListenableFuture<ClientFileGroup> downloadFuture = + PropagatedFluentFuture.from(startTask) + .transformAsync( + unused -> + mobileDataDownloadManager.downloadFileGroup( + groupKey, downloadConditions, customFileGroupValidator), + sequentialControlExecutor) + .transformAsync( + dataFileGroup -> + createClientFileGroup( + dataFileGroup, + downloadFileGroupRequest.accountOptional().isPresent() + ? AccountUtil.serialize( + downloadFileGroupRequest.accountOptional().get()) + : null, + ClientFileGroup.Status.DOWNLOADED, + downloadFileGroupRequest.preserveZipDirectories(), + downloadFileGroupRequest.verifyIsolatedStructure(), + mobileDataDownloadManager, + sequentialControlExecutor, + fileStorage), + sequentialControlExecutor) + .transform(Preconditions::checkNotNull, sequentialControlExecutor); + + // Get a handle on the download task so we can get the CFG during transforms + PropagatedFluentFuture<ClientFileGroup> downloadTaskFuture = + PropagatedFluentFuture.from(downloadFutureMap.add(downloadKey.toString(), downloadFuture)) + .transformAsync( + unused -> { + // Now that the download future is added, start the task and return the future + startTask.run(); + return downloadFuture; + }, + sequentialControlExecutor); ListenableFuture<ClientFileGroup> transformFuture = - Futures.transform( - downloadFuture, - clientFileGroup -> { - if (downloadFileGroupRequest.listenerOptional().isPresent()) { - downloadFileGroupRequest.listenerOptional().get().onComplete(clientFileGroup); - if (downloadMonitorOptional.isPresent()) { - downloadMonitorOptional.get().removeDownloadListener(groupName); - } - } - return clientFileGroup; - }, - sequentialControlExecutor); + downloadTaskFuture + .transformAsync( + unused -> downloadFutureMap.remove(downloadKey.toString()), + sequentialControlExecutor) + .transformAsync( + unused -> { + ClientFileGroup clientFileGroup = getDone(downloadTaskFuture); + + if (downloadFileGroupRequest.listenerOptional().isPresent()) { + try { + downloadFileGroupRequest.listenerOptional().get().onComplete(clientFileGroup); + } catch (Exception e) { + LogUtil.w( + e, + "%s: Listener onComplete failed for group %s", + TAG, + clientFileGroup.getGroupName()); + } + if (downloadMonitorOptional.isPresent()) { + downloadMonitorOptional.get().removeDownloadListener(groupName); + } + } + return immediateFuture(clientFileGroup); + }, + sequentialControlExecutor); - Futures.addCallback( + PropagatedFutures.addCallback( transformFuture, new FutureCallback<ClientFileGroup>() { @Override @@ -722,10 +994,16 @@ class MobileDataDownloadImpl implements MobileDataDownload { @Override public void onFailure(Throwable t) { - if (downloadFileGroupRequest.listenerOptional().isPresent() - && downloadMonitorOptional.isPresent()) { - downloadMonitorOptional.get().removeDownloadListener(groupName); + if (downloadFileGroupRequest.listenerOptional().isPresent()) { + downloadFileGroupRequest.listenerOptional().get().onFailure(t); + + if (downloadMonitorOptional.isPresent()) { + downloadMonitorOptional.get().removeDownloadListener(groupName); + } } + + // Remove future from map + ListenableFuture<Void> unused = downloadFutureMap.remove(downloadKey.toString()); } }, sequentialControlExecutor); @@ -745,14 +1023,14 @@ class MobileDataDownloadImpl implements MobileDataDownload { DownloadFileGroupRequest downloadFileGroupRequest) { LogUtil.d("%s: downloadFileGroupWithForegroundService start.", TAG); if (!foregroundDownloadServiceClassOptional.isPresent()) { - return Futures.immediateFailedFuture( + return immediateFailedFuture( new IllegalArgumentException( "downloadFileGroupWithForegroundService: ForegroundDownloadService is not" + " provided!")); } if (!downloadMonitorOptional.isPresent()) { - return Futures.immediateFailedFuture( + return immediateFailedFuture( DownloadException.builder() .setDownloadResultCode(DownloadResultCode.DOWNLOAD_MONITOR_NOT_PROVIDED_ERROR) .setMessage( @@ -760,6 +1038,41 @@ class MobileDataDownloadImpl implements MobileDataDownload { .build()); } + // Submit the call to sequentialControlExecutor, but don't use futureSerializer. This will + // ensure that multiple calls are enqueued to the executor in a FIFO order, but these calls + // won't block each other when the download is in progress. + return PropagatedFutures.submitAsync( + () -> + PropagatedFutures.transformAsync( + // Check if requested file group has already been downloaded + getDownloadGroupState(downloadFileGroupRequest), + downloadGroupState -> { + switch (downloadGroupState.getKind()) { + case IN_PROGRESS_FUTURE: + // If the file group download is in progress, return that future immediately + return downloadGroupState.inProgressFuture(); + case DOWNLOADED_GROUP: + // If the file group is already downloaded, return that immediately + return immediateFuture(downloadGroupState.downloadedGroup()); + case PENDING_GROUP: + return downloadPendingFileGroupWithForegroundService( + downloadFileGroupRequest, downloadGroupState.pendingGroup()); + } + throw new AssertionError( + String.format( + "received unsupported DownloadGroupState kind %s", + downloadGroupState.getKind())); + }, + sequentialControlExecutor), + sequentialControlExecutor); + } + + /** + * Helper method to download a file group in the foreground after it has been confirmed to be + * pending. + */ + private ListenableFuture<ClientFileGroup> downloadPendingFileGroupWithForegroundService( + DownloadFileGroupRequest downloadFileGroupRequest, DataFileGroupInternal pendingGroup) { // It's OK to recreate the NotificationChannel since it can also be used to restore a // deleted channel and to update an existing channel's name, description, group, and/or // importance. @@ -778,106 +1091,109 @@ class MobileDataDownloadImpl implements MobileDataDownload { } GroupKey groupKey = groupKeyBuilder.build(); + ForegroundDownloadKey foregroundDownloadKey = + ForegroundDownloadKey.ofFileGroup( + groupName, + downloadFileGroupRequest.accountOptional(), + downloadFileGroupRequest.variantIdOptional()); + + DownloadListener downloadListenerWithNotification = + createDownloadListenerWithNotification(downloadFileGroupRequest, pendingGroup); + // The downloadMonitor will trigger the DownloadListener. + downloadMonitorOptional + .get() + .addDownloadListener( + downloadFileGroupRequest.groupName(), downloadListenerWithNotification); + + Optional<DownloadConditions> downloadConditions; + try { + downloadConditions = + downloadFileGroupRequest.downloadConditionsOptional().isPresent() + ? Optional.of( + ProtoConversionUtil.convert( + downloadFileGroupRequest.downloadConditionsOptional().get())) + : Optional.absent(); + } catch (InvalidProtocolBufferException e) { + return immediateFailedFuture(e); + } - ListenableFuture<ClientFileGroup> downloadFuture = - Futures.transformAsync( - // Check if requested file group has already been downloaded - tryToGetDownloadedFileGroup(downloadFileGroupRequest), - downloadedFileGroupOptional -> { - // If the file group has already been downloaded, return that one. - if (downloadedFileGroupOptional.isPresent()) { - return Futures.immediateFuture(downloadedFileGroupOptional.get()); - } - - // if there is the same on-going request, return that one. - if (keyToListenableFuture.containsKey(downloadFileGroupRequest.groupName())) { - // keyToListenableFuture.get must return Non-null since we check the containsKey - // above. - // checkNotNull is to suppress false alarm about @Nullable result. - return Preconditions.checkNotNull( - keyToListenableFuture.get(downloadFileGroupRequest.groupName())); - } - - // Only start the foreground download service when this is the first download - // request. - if (keyToListenableFuture.isEmpty()) { - NotificationUtil.startForegroundDownloadService( - context, - foregroundDownloadServiceClassOptional.get(), - downloadFileGroupRequest.groupName()); - } - - DownloadListener downloadListenerWithNotification = - createDownloadListenerWithNotification(downloadFileGroupRequest); - // The downloadMonitor will trigger the DownloadListener. - downloadMonitorOptional - .get() - .addDownloadListener( - downloadFileGroupRequest.groupName(), downloadListenerWithNotification); - - Optional<DownloadConditions> downloadConditions = - downloadFileGroupRequest.downloadConditionsOptional().isPresent() - ? Optional.of( - ProtoConversionUtil.convert( - downloadFileGroupRequest.downloadConditionsOptional().get())) - : Optional.absent(); - ListenableFuture<DataFileGroupInternal> downloadFileGroupFuture = - mobileDataDownloadManager.downloadFileGroup( - groupKey, downloadConditions, customFileGroupValidator); - - ListenableFuture<ClientFileGroup> transformFuture = - Futures.transformAsync( - downloadFileGroupFuture, - dataFileGroup -> { - return Futures.transform( - createClientFileGroup( - dataFileGroup, - downloadFileGroupRequest.accountOptional().isPresent() - ? AccountUtil.serialize( - downloadFileGroupRequest.accountOptional().get()) - : null, - ClientFileGroup.Status.DOWNLOADED, - downloadFileGroupRequest.preserveZipDirectories(), - mobileDataDownloadManager, - sequentialControlExecutor, - fileStorage), - Preconditions::checkNotNull, - sequentialControlExecutor); - }, - sequentialControlExecutor); - - Futures.addCallback( - transformFuture, - new FutureCallback<ClientFileGroup>() { - @Override - public void onSuccess(ClientFileGroup clientFileGroup) { - // Currently the MobStore monitor does not support onSuccess so we have to add - // callback to the download future here. - // TODO(b/148057674): Use the same logic as MDDLite to keep the foreground - // download service alive until the client's onComplete finishes. - downloadListenerWithNotification.onComplete(clientFileGroup); - } - - @Override - public void onFailure(Throwable t) { - // Currently the MobStore monitor does not support onFailure so we have to add - // callback to the download future here. - downloadListenerWithNotification.onFailure(t); - } - }, - sequentialControlExecutor); + // Create a ListenableFutureTask to delay starting the downloadFuture until we can add the + // future to our map. + ListenableFutureTask<Void> startTask = ListenableFutureTask.create(() -> null); + PropagatedFluentFuture<ClientFileGroup> downloadFileGroupFuture = + PropagatedFluentFuture.from(startTask) + .transformAsync( + unused -> + mobileDataDownloadManager.downloadFileGroup( + groupKey, downloadConditions, customFileGroupValidator), + sequentialControlExecutor) + .transformAsync( + dataFileGroup -> + createClientFileGroup( + dataFileGroup, + downloadFileGroupRequest.accountOptional().isPresent() + ? AccountUtil.serialize( + downloadFileGroupRequest.accountOptional().get()) + : null, + ClientFileGroup.Status.DOWNLOADED, + downloadFileGroupRequest.preserveZipDirectories(), + downloadFileGroupRequest.verifyIsolatedStructure(), + mobileDataDownloadManager, + sequentialControlExecutor, + fileStorage), + sequentialControlExecutor) + .transform(Preconditions::checkNotNull, sequentialControlExecutor); - keyToListenableFuture.put(downloadFileGroupRequest.groupName(), transformFuture); - return transformFuture; + ListenableFuture<ClientFileGroup> transformFuture = + PropagatedFutures.transformAsync( + foregroundDownloadFutureMap.add( + foregroundDownloadKey.toString(), downloadFileGroupFuture), + unused -> { + // Now that the download future is added, start the task and return the future + startTask.run(); + return downloadFileGroupFuture; }, sequentialControlExecutor); - return downloadFuture; + PropagatedFutures.addCallback( + transformFuture, + new FutureCallback<ClientFileGroup>() { + @Override + public void onSuccess(ClientFileGroup clientFileGroup) { + // Currently the MobStore monitor does not support onSuccess so we have to add + // callback to the download future here. + try { + downloadListenerWithNotification.onComplete(clientFileGroup); + } catch (Exception e) { + LogUtil.w( + e, + "%s: Listener onComplete failed for group %s", + TAG, + clientFileGroup.getGroupName()); + } + } + + @Override + public void onFailure(Throwable t) { + // Currently the MobStore monitor does not support onFailure so we have to add + // callback to the download future here. + downloadListenerWithNotification.onFailure(t); + } + }, + sequentialControlExecutor); + + return transformFuture; } - /** Helper method to check if file group has been downloaded and return it early. */ - private ListenableFuture<Optional<ClientFileGroup>> tryToGetDownloadedFileGroup( + /** Helper method to return a {@link DownloadGroupState} for the given request. */ + private ListenableFuture<DownloadGroupState> getDownloadGroupState( DownloadFileGroupRequest downloadFileGroupRequest) { + ForegroundDownloadKey foregroundDownloadKey = + ForegroundDownloadKey.ofFileGroup( + downloadFileGroupRequest.groupName(), + downloadFileGroupRequest.accountOptional(), + downloadFileGroupRequest.variantIdOptional()); + String groupName = downloadFileGroupRequest.groupName(); GroupKey.Builder groupKeyBuilder = GroupKey.newBuilder().setGroupName(groupName).setOwnerPackage(context.getPackageName()); @@ -886,101 +1202,164 @@ class MobileDataDownloadImpl implements MobileDataDownload { groupKeyBuilder.setAccount( AccountUtil.serialize(downloadFileGroupRequest.accountOptional().get())); } + + if (downloadFileGroupRequest.variantIdOptional().isPresent()) { + groupKeyBuilder.setVariantId(downloadFileGroupRequest.variantIdOptional().get()); + } + boolean isDownloadListenerPresent = downloadFileGroupRequest.listenerOptional().isPresent(); GroupKey groupKey = groupKeyBuilder.build(); - // Get pending and downloaded versions to tell if we should return downloaded version early - ListenableFuture<Pair<DataFileGroupInternal, DataFileGroupInternal>> fileGroupVersionsFuture = - Futures.transformAsync( - mobileDataDownloadManager.getFileGroup(groupKey, /* downloaded = */ false), - pendingDataFileGroup -> - Futures.transform( - mobileDataDownloadManager.getFileGroup(groupKey, /* downloaded = */ true), - downloadedDataFileGroup -> - Pair.create(pendingDataFileGroup, downloadedDataFileGroup), - sequentialControlExecutor), - sequentialControlExecutor); - - return Futures.transformAsync( - fileGroupVersionsFuture, - fileGroupVersionsPair -> { - // if pending version is not null, return absent - if (fileGroupVersionsPair.first != null) { - return Futures.immediateFuture(Optional.absent()); - } - // If both groups are null, return group not found failure - if (fileGroupVersionsPair.second == null) { - // TODO(b/174808410): Add Logging - // file group is not pending nor downloaded -- return failure. - DownloadException failure = - DownloadException.builder() - .setDownloadResultCode(DownloadResultCode.GROUP_NOT_FOUND_ERROR) - .setMessage("Nothing to download for file group: " + groupKey.getGroupName()) - .build(); - if (isDownloadListenerPresent) { - downloadFileGroupRequest.listenerOptional().get().onFailure(failure); - } - return Futures.immediateFailedFuture(failure); - } + return futureSerializer.submitAsync( + () -> { + ListenableFuture<Optional<ListenableFuture<ClientFileGroup>>> + foregroundDownloadFutureOptional = + foregroundDownloadFutureMap.get(foregroundDownloadKey.toString()); + ListenableFuture<Optional<ListenableFuture<ClientFileGroup>>> + backgroundDownloadFutureOptional = + downloadFutureMap.get(foregroundDownloadKey.toString()); + + return PropagatedFutures.whenAllSucceed( + foregroundDownloadFutureOptional, backgroundDownloadFutureOptional) + .callAsync( + () -> { + if (getDone(foregroundDownloadFutureOptional).isPresent()) { + return immediateFuture( + DownloadGroupState.ofInProgressFuture( + getDone(foregroundDownloadFutureOptional).get())); + } else if (getDone(backgroundDownloadFutureOptional).isPresent()) { + return immediateFuture( + DownloadGroupState.ofInProgressFuture( + getDone(backgroundDownloadFutureOptional).get())); + } - DataFileGroupInternal downloadedDataFileGroup = fileGroupVersionsPair.second; + // Get pending and downloaded versions to tell if we should return downloaded + // version early + ListenableFuture<GroupPair> fileGroupVersionsFuture = + PropagatedFutures.transformAsync( + mobileDataDownloadManager.getFileGroup( + groupKey, /* downloaded= */ false), + pendingDataFileGroup -> + PropagatedFutures.transform( + mobileDataDownloadManager.getFileGroup( + groupKey, /* downloaded= */ true), + downloadedDataFileGroup -> + GroupPair.create( + pendingDataFileGroup, downloadedDataFileGroup), + sequentialControlExecutor), + sequentialControlExecutor); - // Notify download listener (if present) that file group has been downloaded. - if (isDownloadListenerPresent) { - downloadMonitorOptional - .get() - .addDownloadListener( - downloadFileGroupRequest.groupName(), - downloadFileGroupRequest.listenerOptional().get()); - } - FluentFuture<Optional<ClientFileGroup>> transformFuture = - FluentFuture.from( - createClientFileGroup( - downloadedDataFileGroup, - downloadFileGroupRequest.accountOptional().isPresent() - ? AccountUtil.serialize( - downloadFileGroupRequest.accountOptional().get()) - : null, - ClientFileGroup.Status.DOWNLOADED, - downloadFileGroupRequest.preserveZipDirectories(), - mobileDataDownloadManager, - sequentialControlExecutor, - fileStorage)) - .transform(Preconditions::checkNotNull, sequentialControlExecutor) - .transform( - clientFileGroup -> { - if (isDownloadListenerPresent) { - downloadFileGroupRequest - .listenerOptional() - .get() - .onComplete(clientFileGroup); - downloadMonitorOptional.get().removeDownloadListener(groupName); - } - return Optional.of(clientFileGroup); - }, - sequentialControlExecutor); - transformFuture.addCallback( - new FutureCallback<Optional<ClientFileGroup>>() { - @Override - public void onSuccess(Optional<ClientFileGroup> result) {} - - @Override - public void onFailure(Throwable t) { - if (isDownloadListenerPresent) { - downloadMonitorOptional.get().removeDownloadListener(groupName); - } - } - }, - sequentialControlExecutor); + return PropagatedFutures.transformAsync( + fileGroupVersionsFuture, + fileGroupVersionsPair -> { + // if pending version is not null, return pending version + if (fileGroupVersionsPair.pendingGroup() != null) { + return immediateFuture( + DownloadGroupState.ofPendingGroup( + checkNotNull(fileGroupVersionsPair.pendingGroup()))); + } + // If both groups are null, return group not found failure + if (fileGroupVersionsPair.downloadedGroup() == null) { + // TODO(b/174808410): Add Logging + // file group is not pending nor downloaded -- return failure. + DownloadException failure = + DownloadException.builder() + .setDownloadResultCode(DownloadResultCode.GROUP_NOT_FOUND_ERROR) + .setMessage( + "Nothing to download for file group: " + + groupKey.getGroupName()) + .build(); + if (isDownloadListenerPresent) { + downloadFileGroupRequest.listenerOptional().get().onFailure(failure); + } + return immediateFailedFuture(failure); + } - return transformFuture; + DataFileGroupInternal downloadedDataFileGroup = + checkNotNull(fileGroupVersionsPair.downloadedGroup()); + + // Notify download listener (if present) that file group has been + // downloaded. + if (isDownloadListenerPresent) { + downloadMonitorOptional + .get() + .addDownloadListener( + downloadFileGroupRequest.groupName(), + downloadFileGroupRequest.listenerOptional().get()); + } + PropagatedFluentFuture<ClientFileGroup> transformFuture = + PropagatedFluentFuture.from( + createClientFileGroup( + downloadedDataFileGroup, + downloadFileGroupRequest.accountOptional().isPresent() + ? AccountUtil.serialize( + downloadFileGroupRequest.accountOptional().get()) + : null, + ClientFileGroup.Status.DOWNLOADED, + downloadFileGroupRequest.preserveZipDirectories(), + downloadFileGroupRequest.verifyIsolatedStructure(), + mobileDataDownloadManager, + sequentialControlExecutor, + fileStorage)) + .transform(Preconditions::checkNotNull, sequentialControlExecutor) + .transform( + clientFileGroup -> { + if (isDownloadListenerPresent) { + try { + downloadFileGroupRequest + .listenerOptional() + .get() + .onComplete(clientFileGroup); + } catch (Exception e) { + LogUtil.w( + e, + "%s: Listener onComplete failed for group %s", + TAG, + clientFileGroup.getGroupName()); + } + downloadMonitorOptional + .get() + .removeDownloadListener(groupName); + } + return clientFileGroup; + }, + sequentialControlExecutor); + transformFuture.addCallback( + new FutureCallback<ClientFileGroup>() { + @Override + public void onSuccess(ClientFileGroup result) {} + + @Override + public void onFailure(Throwable t) { + if (isDownloadListenerPresent) { + downloadMonitorOptional.get().removeDownloadListener(groupName); + } + } + }, + sequentialControlExecutor); + + // Use directExecutor here since we are performing a trivial operation. + return transformFuture.transform( + DownloadGroupState::ofDownloadedGroup, directExecutor()); + }, + sequentialControlExecutor); + }, + sequentialControlExecutor); }, sequentialControlExecutor); } private DownloadListener createDownloadListenerWithNotification( - DownloadFileGroupRequest downloadRequest) { + DownloadFileGroupRequest downloadRequest, DataFileGroupInternal fileGroup) { + + String networkPausedMessage = getNetworkPausedMessage(downloadRequest, fileGroup); + NotificationManagerCompat notificationManager = NotificationManagerCompat.from(context); + ForegroundDownloadKey foregroundDownloadKey = + ForegroundDownloadKey.ofFileGroup( + downloadRequest.groupName(), + downloadRequest.accountOptional(), + downloadRequest.variantIdOptional()); NotificationCompat.Builder notification = NotificationUtil.createNotificationBuilder( @@ -994,7 +1373,7 @@ class MobileDataDownloadImpl implements MobileDataDownload { NotificationUtil.createCancelAction( context, foregroundDownloadServiceClassOptional.get(), - downloadRequest.groupName(), + foregroundDownloadKey.toString(), notification, notificationKey); @@ -1004,133 +1383,192 @@ class MobileDataDownloadImpl implements MobileDataDownload { return new DownloadListener() { @Override public void onProgress(long currentSize) { - sequentialControlExecutor.execute( - () -> { - // There can be a race condition, where onPausedForConnectivity can be called - // after onComplete or onFailure which removes the future and the notification. - if (keyToListenableFuture.containsKey(downloadRequest.groupName()) - && downloadRequest.showNotifications() - == DownloadFileGroupRequest.ShowNotifications.ALL) { - notification - .setCategory(NotificationCompat.CATEGORY_PROGRESS) - .setSmallIcon(android.R.drawable.stat_sys_download) - .setProgress( - downloadRequest.groupSizeBytes(), - (int) currentSize, - /* indeterminate = */ downloadRequest.groupSizeBytes() <= 0); - notificationManager.notify(notificationKey, notification.build()); - } - if (downloadRequest.listenerOptional().isPresent()) { - downloadRequest.listenerOptional().get().onProgress(currentSize); - } - }); + // TODO(b/229123693): return this future once DownloadListener has an async api. + // There can be a race condition, where onProgress can be called + // after onComplete or onFailure which removes the future and the notification. + // Check foregroundDownloadFutureMap first before updating notification. + ListenableFuture<?> unused = + PropagatedFutures.transformAsync( + foregroundDownloadFutureMap.containsKey(foregroundDownloadKey.toString()), + futureInProgress -> { + if (futureInProgress + && downloadRequest.showNotifications() + == DownloadFileGroupRequest.ShowNotifications.ALL) { + notification + .setCategory(NotificationCompat.CATEGORY_PROGRESS) + .setSmallIcon(android.R.drawable.stat_sys_download) + .setProgress( + downloadRequest.groupSizeBytes(), + (int) currentSize, + /* indeterminate= */ downloadRequest.groupSizeBytes() <= 0); + notificationManager.notify(notificationKey, notification.build()); + } + if (downloadRequest.listenerOptional().isPresent()) { + downloadRequest.listenerOptional().get().onProgress(currentSize); + } + return immediateVoidFuture(); + }, + sequentialControlExecutor); } @Override public void pausedForConnectivity() { - sequentialControlExecutor.execute( - () -> { - // There can be a race condition, where pausedForConnectivity can be called - // after onComplete or onFailure which removes the future and the notification. - if (keyToListenableFuture.containsKey(downloadRequest.groupName()) - && downloadRequest.showNotifications() - == DownloadFileGroupRequest.ShowNotifications.ALL) { - notification - .setCategory(NotificationCompat.CATEGORY_STATUS) - .setContentText(NotificationUtil.getDownloadPausedMessage(context)) - .setSmallIcon(android.R.drawable.stat_sys_download) - .setOngoing(true) - // hide progress bar. - .setProgress(0, 0, false); - notificationManager.notify(notificationKey, notification.build()); - } - - if (downloadRequest.listenerOptional().isPresent()) { - downloadRequest.listenerOptional().get().pausedForConnectivity(); - } - }); + // TODO(b/229123693): return this future once DownloadListener has an async api. + // There can be a race condition, where pausedForConnectivity can be called + // after onComplete or onFailure which removes the future and the notification. + // Check foregroundDownloadFutureMap first before updating notification. + ListenableFuture<?> unused = + PropagatedFutures.transformAsync( + foregroundDownloadFutureMap.containsKey(foregroundDownloadKey.toString()), + futureInProgress -> { + if (futureInProgress + && downloadRequest.showNotifications() + == DownloadFileGroupRequest.ShowNotifications.ALL) { + notification + .setCategory(NotificationCompat.CATEGORY_STATUS) + .setContentText(networkPausedMessage) + .setSmallIcon(android.R.drawable.stat_sys_download) + .setOngoing(true) + // hide progress bar. + .setProgress(0, 0, false); + notificationManager.notify(notificationKey, notification.build()); + } + if (downloadRequest.listenerOptional().isPresent()) { + downloadRequest.listenerOptional().get().pausedForConnectivity(); + } + return immediateVoidFuture(); + }, + sequentialControlExecutor); } @Override public void onComplete(ClientFileGroup clientFileGroup) { - sequentialControlExecutor.execute( - () -> { - // Clear the notification action. - if (downloadRequest.showNotifications() - == DownloadFileGroupRequest.ShowNotifications.ALL) { - notification.mActions.clear(); - - NotificationUtil.cancelNotificationForKey(context, downloadRequest.groupName()); - } + // TODO(b/229123693): return this future once DownloadListener has an async api. + ListenableFuture<?> unused = + PropagatedFutures.submitAsync( + () -> { + boolean onCompleteFailed = false; + if (downloadRequest.listenerOptional().isPresent()) { + try { + downloadRequest.listenerOptional().get().onComplete(clientFileGroup); + } catch (Exception e) { + LogUtil.w( + e, + "%s: Delegate onComplete failed for group %s, showing failure" + + " notification.", + TAG, + clientFileGroup.getGroupName()); + onCompleteFailed = true; + } + } - keyToListenableFuture.remove(downloadRequest.groupName()); - // If there is no other on-going foreground download, shutdown the - // ForegroundDownloadService - if (keyToListenableFuture.isEmpty()) { - NotificationUtil.stopForegroundDownloadService( - context, foregroundDownloadServiceClassOptional.get()); - } + // Clear the notification action. + if (downloadRequest.showNotifications() + == DownloadFileGroupRequest.ShowNotifications.ALL) { + notification.mActions.clear(); + + if (onCompleteFailed) { + // Show download failed in notification. + notification + .setCategory(NotificationCompat.CATEGORY_STATUS) + .setContentText(NotificationUtil.getDownloadFailedMessage(context)) + .setOngoing(false) + .setSmallIcon(android.R.drawable.stat_sys_warning) + // hide progress bar. + .setProgress(0, 0, false); + + notificationManager.notify(notificationKey, notification.build()); + } else { + NotificationUtil.cancelNotificationForKey( + context, downloadRequest.groupName()); + } + } - if (downloadRequest.listenerOptional().isPresent()) { - downloadRequest.listenerOptional().get().onComplete(clientFileGroup); - } + downloadMonitorOptional.get().removeDownloadListener(downloadRequest.groupName()); - downloadMonitorOptional.get().removeDownloadListener(downloadRequest.groupName()); - }); + return foregroundDownloadFutureMap.remove(foregroundDownloadKey.toString()); + }, + sequentialControlExecutor); } @Override public void onFailure(Throwable t) { - sequentialControlExecutor.execute( - () -> { - if (downloadRequest.showNotifications() - == DownloadFileGroupRequest.ShowNotifications.ALL) { - // Clear the notification action. - notification.mActions.clear(); - - // Show download failed in notification. - notification - .setCategory(NotificationCompat.CATEGORY_STATUS) - .setContentText(NotificationUtil.getDownloadFailedMessage(context)) - .setOngoing(false) - .setSmallIcon(android.R.drawable.stat_sys_warning) - // hide progress bar. - .setProgress(0, 0, false); - - notificationManager.notify(notificationKey, notification.build()); - } - keyToListenableFuture.remove(downloadRequest.groupName()); + // TODO(b/229123693): return this future once DownloadListener has an async api. + ListenableFuture<?> unused = + PropagatedFutures.submitAsync( + () -> { + if (downloadRequest.showNotifications() + == DownloadFileGroupRequest.ShowNotifications.ALL) { + // Clear the notification action. + notification.mActions.clear(); + + // Show download failed in notification. + notification + .setCategory(NotificationCompat.CATEGORY_STATUS) + .setContentText(NotificationUtil.getDownloadFailedMessage(context)) + .setOngoing(false) + .setSmallIcon(android.R.drawable.stat_sys_warning) + // hide progress bar. + .setProgress(0, 0, false); + + notificationManager.notify(notificationKey, notification.build()); + } - // If there is no other on-going foreground download, shutdown the - // ForegroundDownloadService - if (keyToListenableFuture.isEmpty()) { - NotificationUtil.stopForegroundDownloadService( - context, foregroundDownloadServiceClassOptional.get()); - } + if (downloadRequest.listenerOptional().isPresent()) { + downloadRequest.listenerOptional().get().onFailure(t); + } + downloadMonitorOptional.get().removeDownloadListener(downloadRequest.groupName()); - if (downloadRequest.listenerOptional().isPresent()) { - downloadRequest.listenerOptional().get().onFailure(t); - } - downloadMonitorOptional.get().removeDownloadListener(downloadRequest.groupName()); - }); + return foregroundDownloadFutureMap.remove(foregroundDownloadKey.toString()); + }, + sequentialControlExecutor); } }; } + // Helper method to get the correct network paused message + private String getNetworkPausedMessage( + DownloadFileGroupRequest downloadRequest, DataFileGroupInternal fileGroup) { + DeviceNetworkPolicy networkPolicyForDownload = + fileGroup.getDownloadConditions().getDeviceNetworkPolicy(); + if (downloadRequest.downloadConditionsOptional().isPresent()) { + try { + networkPolicyForDownload = + ProtoConversionUtil.convert(downloadRequest.downloadConditionsOptional().get()) + .getDeviceNetworkPolicy(); + } catch (InvalidProtocolBufferException unused) { + // Do nothing -- we will rely on the file group's network policy. + } + } + + switch (networkPolicyForDownload) { + case DOWNLOAD_FIRST_ON_WIFI_THEN_ON_ANY_NETWORK: // fallthrough + case DOWNLOAD_ONLY_ON_WIFI: + return NotificationUtil.getDownloadPausedWifiMessage(context); + default: + return NotificationUtil.getDownloadPausedMessage(context); + } + } + @Override public void cancelForegroundDownload(String downloadKey) { LogUtil.d("%s: CancelForegroundDownload for key = %s", TAG, downloadKey); - sequentialControlExecutor.execute( - () -> { - if (keyToListenableFuture.containsKey(downloadKey)) { - keyToListenableFuture.get(downloadKey).cancel(true); - } else { - // downloadKey is not a file group, attempt cancel with internal MDD Lite instance in - // case it's a single file uri (cancel call is a noop if internal MDD Lite doesn't know - // about it). - singleFileDownloader.cancelForegroundDownload(downloadKey); - } - }); + ListenableFuture<?> unused = + PropagatedFutures.transformAsync( + foregroundDownloadFutureMap.get(downloadKey), + downloadFuture -> { + if (downloadFuture.isPresent()) { + LogUtil.v( + "%s: CancelForegroundDownload future found for key = %s, cancelling...", + TAG, downloadKey); + downloadFuture.get().cancel(false); + } + return immediateVoidFuture(); + }, + sequentialControlExecutor); + // Attempt cancel with internal MDD Lite instance in case it's a single file uri (cancel call is + // a noop if internal MDD Lite doesn't know about it). + singleFileDownloader.cancelForegroundDownload(downloadKey); } @Override @@ -1141,11 +1579,10 @@ class MobileDataDownloadImpl implements MobileDataDownload { @Override public ListenableFuture<Void> schedulePeriodicBackgroundTasks() { return futureSerializer.submit( - propagateCallable( - () -> { - schedulePeriodicTasksInternal(/* constraintOverridesMap = */ Optional.absent()); - return null; - }), + () -> { + schedulePeriodicTasksInternal(/* constraintOverridesMap= */ Optional.absent()); + return null; + }, sequentialControlExecutor); } @@ -1153,11 +1590,10 @@ class MobileDataDownloadImpl implements MobileDataDownload { public ListenableFuture<Void> schedulePeriodicBackgroundTasks( Optional<Map<String, ConstraintOverrides>> constraintOverridesMap) { return futureSerializer.submit( - propagateCallable( - () -> { - schedulePeriodicTasksInternal(constraintOverridesMap); - return null; - }), + () -> { + schedulePeriodicTasksInternal(constraintOverridesMap); + return null; + }, sequentialControlExecutor); } @@ -1211,6 +1647,30 @@ class MobileDataDownloadImpl implements MobileDataDownload { } @Override + public ListenableFuture<Void> cancelPeriodicBackgroundTasks() { + return futureSerializer.submit( + () -> { + cancelPeriodicTasksInternal(); + return null; + }, + sequentialControlExecutor); + } + + private void cancelPeriodicTasksInternal() { + if (!taskSchedulerOptional.isPresent()) { + LogUtil.w("%s: Called cancelPeriodicTasksInternal when taskScheduler is not provided.", TAG); + return; + } + + TaskScheduler taskScheduler = taskSchedulerOptional.get(); + + taskScheduler.cancelPeriodicTask(TaskScheduler.CHARGING_PERIODIC_TASK); + taskScheduler.cancelPeriodicTask(TaskScheduler.MAINTENANCE_PERIODIC_TASK); + taskScheduler.cancelPeriodicTask(TaskScheduler.CELLULAR_CHARGING_PERIODIC_TASK); + taskScheduler.cancelPeriodicTask(TaskScheduler.WIFI_CHARGING_PERIODIC_TASK); + } + + @Override public ListenableFuture<Void> handleTask(String tag) { // All work done here that touches metadata (MobileDataDownloadManager) should be serialized // through sequentialControlExecutor. @@ -1221,7 +1681,7 @@ class MobileDataDownloadImpl implements MobileDataDownload { case TaskScheduler.CHARGING_PERIODIC_TASK: ListenableFuture<Void> refreshFileGroupsFuture = refreshFileGroups(); - return Futures.transformAsync( + return PropagatedFutures.transformAsync( refreshFileGroupsFuture, propagateAsyncFunction( v -> mobileDataDownloadManager.verifyAllPendingGroups(customFileGroupValidator)), @@ -1235,7 +1695,7 @@ class MobileDataDownloadImpl implements MobileDataDownload { default: LogUtil.d("%s: gcm task doesn't belong to MDD", TAG); - return Futures.immediateFailedFuture( + return immediateFailedFuture( new IllegalArgumentException("Unknown task tag sent to MDD.handleTask() " + tag)); } } @@ -1243,7 +1703,7 @@ class MobileDataDownloadImpl implements MobileDataDownload { private ListenableFuture<Void> refreshAndDownload(boolean onWifi) { // We will do 2 passes to support 2-step downloads. In each step, we will refresh and then // download. - return FluentFuture.from(refreshFileGroups()) + return PropagatedFluentFuture.from(refreshFileGroups()) .transformAsync( v -> mobileDataDownloadManager.downloadAllPendingGroups( @@ -1263,7 +1723,8 @@ class MobileDataDownloadImpl implements MobileDataDownload { refreshFutures.add(fileGroupPopulator.refreshFileGroups(this)); } - return Futures.whenAllComplete(refreshFutures).call(() -> null, sequentialControlExecutor); + return PropagatedFutures.whenAllComplete(refreshFutures) + .call(() -> null, sequentialControlExecutor); } @Override @@ -1272,6 +1733,12 @@ class MobileDataDownloadImpl implements MobileDataDownload { } @Override + public ListenableFuture<Void> collectGarbage() { + return futureSerializer.submitAsync( + mobileDataDownloadManager::removeExpiredGroupsAndFiles, sequentialControlExecutor); + } + + @Override public ListenableFuture<Void> clear() { return futureSerializer.submitAsync( mobileDataDownloadManager::clear, sequentialControlExecutor); @@ -1307,6 +1774,29 @@ class MobileDataDownloadImpl implements MobileDataDownload { public ListenableFuture<Void> reportUsage(UsageEvent usageEvent) { eventLogger.logMddUsageEvent(createFileGroupDetails(usageEvent.clientFileGroup()), null); - return Futures.immediateVoidFuture(); + return immediateVoidFuture(); + } + + private static DownloadFutureMap.StateChangeCallbacks createCallbacksForForegroundService( + Context context, Optional<Class<?>> foregroundDownloadServiceClassOptional) { + return new DownloadFutureMap.StateChangeCallbacks() { + @Override + public void onAdd(String key, int newSize) { + // Only start foreground service if this is the first future we are adding. + if (newSize == 1 && foregroundDownloadServiceClassOptional.isPresent()) { + NotificationUtil.startForegroundDownloadService( + context, foregroundDownloadServiceClassOptional.get(), key); + } + } + + @Override + public void onRemove(String key, int newSize) { + // Only stop foreground service if there are no more futures remaining. + if (newSize == 0 && foregroundDownloadServiceClassOptional.isPresent()) { + NotificationUtil.stopForegroundDownloadService( + context, foregroundDownloadServiceClassOptional.get(), key); + } + } + }; } } diff --git a/java/com/google/android/libraries/mobiledatadownload/ReadDataFileGroupRequest.java b/java/com/google/android/libraries/mobiledatadownload/ReadDataFileGroupRequest.java new file mode 100644 index 0000000..88fc970 --- /dev/null +++ b/java/com/google/android/libraries/mobiledatadownload/ReadDataFileGroupRequest.java @@ -0,0 +1,54 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.libraries.mobiledatadownload; + +import android.accounts.Account; +import com.google.auto.value.AutoValue; +import com.google.common.base.Optional; +import javax.annotation.concurrent.Immutable; + +/** Request to get a single file group definition. */ +@AutoValue +@Immutable +public abstract class ReadDataFileGroupRequest { + + public abstract String groupName(); + + public abstract Optional<Account> accountOptional(); + + public abstract Optional<String> variantIdOptional(); + + public static Builder newBuilder() { + return new AutoValue_ReadDataFileGroupRequest.Builder(); + } + + /** Builder for {@link ReadDataFileGroupRequest}. */ + @AutoValue.Builder + public abstract static class Builder { + Builder() {} + + /** Sets the data file group, which is required. */ + public abstract Builder setGroupName(String groupName); + + /** Sets the account associated with the group, which is optional. */ + public abstract Builder setAccountOptional(Optional<Account> accountOptional); + + /** Sets the variant id associated with the group, which is optional. */ + public abstract Builder setVariantIdOptional(Optional<String> variantIdOptional); + + public abstract ReadDataFileGroupRequest build(); + } +} diff --git a/java/com/google/android/libraries/mobiledatadownload/TaskScheduler.java b/java/com/google/android/libraries/mobiledatadownload/TaskScheduler.java index c06c22b..dc6147c 100644 --- a/java/com/google/android/libraries/mobiledatadownload/TaskScheduler.java +++ b/java/com/google/android/libraries/mobiledatadownload/TaskScheduler.java @@ -148,4 +148,14 @@ public interface TaskScheduler { // update all clients. schedulePeriodicTask(tag, period, networkState); } + + /** + * Cancel future invocations of a previously-scheduled task. No guarantee is made whether the task + * will be interrupted if it's currently running. + * + * @param tag tag of the scheduled task. + */ + default void cancelPeriodicTask(String tag) { + // TODO(b/223822302): remove default once all implementations have been updated to include it + } } diff --git a/java/com/google/android/libraries/mobiledatadownload/TimeSource.java b/java/com/google/android/libraries/mobiledatadownload/TimeSource.java index d632382..d045568 100644 --- a/java/com/google/android/libraries/mobiledatadownload/TimeSource.java +++ b/java/com/google/android/libraries/mobiledatadownload/TimeSource.java @@ -15,13 +15,11 @@ */ package com.google.android.libraries.mobiledatadownload; -/** - * Interface through which the SystemClock can be read. - * - * <p>This interface is analogous to {@code com.google.common.time.TimeSource#now#toEpochMilli} - * without the dependency on Java8. - */ +/** Interface through which the SystemClock can be read. */ public interface TimeSource { /** Returns the current system time in milliseconds since January 1, 1970 00:00:00 UTC. */ long currentTimeMillis(); + + /** Returns nanoseconds since boot, including time spent in sleep. */ + long elapsedRealtimeNanos(); } diff --git a/java/com/google/android/libraries/mobiledatadownload/account/AccountUtil.java b/java/com/google/android/libraries/mobiledatadownload/account/AccountUtil.java index 012226e..4c1ad8a 100644 --- a/java/com/google/android/libraries/mobiledatadownload/account/AccountUtil.java +++ b/java/com/google/android/libraries/mobiledatadownload/account/AccountUtil.java @@ -41,7 +41,11 @@ public final class AccountUtil { return new Account(name, type); } - /** Serializes an {@link Account} into a string. */ + /** + * Serializes an {@link Account} into a string. + * + * <p>TODO(b/222110940): make this function consistent with deserialize. + */ public static String serialize(Account account) { return account.type + ACCOUNT_DELIMITER + account.name; } @@ -49,10 +53,14 @@ public final class AccountUtil { /** * Deserializes a string into an {@link Account}. * - * @return The account parsed from string. Returns null if there is any error during parse. + * @return The account parsed from string. Returns null if the accountStr is empty or if there is + * any error during parse. */ @Nullable public static Account deserialize(String accountStr) { + if (accountStr.isEmpty()) { + return null; + } int splitIndex = accountStr.indexOf(ACCOUNT_DELIMITER); if (splitIndex < 0) { LogUtil.e("%s: Unable to parse Account with string = '%s'", TAG, accountStr); diff --git a/java/com/google/android/libraries/mobiledatadownload/account/BUILD b/java/com/google/android/libraries/mobiledatadownload/account/BUILD index cd9bd61..23cd484 100644 --- a/java/com/google/android/libraries/mobiledatadownload/account/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/account/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], diff --git a/java/com/google/android/libraries/mobiledatadownload/annotations/BUILD b/java/com/google/android/libraries/mobiledatadownload/annotations/BUILD index 9bc3d32..6066dbe 100644 --- a/java/com/google/android/libraries/mobiledatadownload/annotations/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/annotations/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], diff --git a/java/com/google/android/libraries/mobiledatadownload/delta/BUILD b/java/com/google/android/libraries/mobiledatadownload/delta/BUILD index 50556e2..afd5b5c 100644 --- a/java/com/google/android/libraries/mobiledatadownload/delta/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/delta/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], diff --git a/java/com/google/android/libraries/mobiledatadownload/downloader/BUILD b/java/com/google/android/libraries/mobiledatadownload/downloader/BUILD index 3831c2e..95150b5 100644 --- a/java/com/google/android/libraries/mobiledatadownload/downloader/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/downloader/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], diff --git a/java/com/google/android/libraries/mobiledatadownload/downloader/DownloadConstraints.java b/java/com/google/android/libraries/mobiledatadownload/downloader/DownloadConstraints.java index e6489c9..9801982 100644 --- a/java/com/google/android/libraries/mobiledatadownload/downloader/DownloadConstraints.java +++ b/java/com/google/android/libraries/mobiledatadownload/downloader/DownloadConstraints.java @@ -17,6 +17,7 @@ package com.google.android.libraries.mobiledatadownload.downloader; import com.google.auto.value.AutoValue; import com.google.common.collect.ImmutableSet; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.util.EnumSet; import java.util.Set; @@ -107,6 +108,7 @@ public abstract class DownloadConstraints { abstract ImmutableSet.Builder<NetworkType> requiredNetworkTypesBuilder(); + @CanIgnoreReturnValue public final Builder addRequiredNetworkType(NetworkType networkType) { requiredNetworkTypesBuilder().add(networkType); return this; diff --git a/java/com/google/android/libraries/mobiledatadownload/downloader/MultiSchemeFileDownloader.java b/java/com/google/android/libraries/mobiledatadownload/downloader/MultiSchemeFileDownloader.java index c0b82a3..7dfc5b4 100644 --- a/java/com/google/android/libraries/mobiledatadownload/downloader/MultiSchemeFileDownloader.java +++ b/java/com/google/android/libraries/mobiledatadownload/downloader/MultiSchemeFileDownloader.java @@ -24,6 +24,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import com.google.errorprone.annotations.CheckReturnValue; import java.net.MalformedURLException; import java.util.HashMap; @@ -41,6 +42,7 @@ public final class MultiSchemeFileDownloader implements FileDownloader { private final Map<String, FileDownloader> schemeToDownloader = new HashMap<>(); /** Associates a url scheme (e.g. "http") with a specific {@link FileDownloader} delegate. */ + @CanIgnoreReturnValue public MultiSchemeFileDownloader.Builder addScheme(String scheme, FileDownloader downloader) { schemeToDownloader.put( Preconditions.checkNotNull(scheme), Preconditions.checkNotNull(downloader)); diff --git a/java/com/google/android/libraries/mobiledatadownload/downloader/inline/BUILD b/java/com/google/android/libraries/mobiledatadownload/downloader/inline/BUILD index a09dd65..5c4fa53 100644 --- a/java/com/google/android/libraries/mobiledatadownload/downloader/inline/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/downloader/inline/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], @@ -32,6 +33,7 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/file/openers:stream", "//java/com/google/android/libraries/mobiledatadownload/internal:MddConstants", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LogUtil", + "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", "@com_google_guava_guava", ], ) diff --git a/java/com/google/android/libraries/mobiledatadownload/downloader/inline/InlineFileDownloader.java b/java/com/google/android/libraries/mobiledatadownload/downloader/inline/InlineFileDownloader.java index 8f3d472..9102b56 100644 --- a/java/com/google/android/libraries/mobiledatadownload/downloader/inline/InlineFileDownloader.java +++ b/java/com/google/android/libraries/mobiledatadownload/downloader/inline/InlineFileDownloader.java @@ -16,6 +16,8 @@ package com.google.android.libraries.mobiledatadownload.downloader.inline; import static com.google.android.libraries.mobiledatadownload.internal.MddConstants.INLINE_FILE_URL_SCHEME; +import static com.google.common.util.concurrent.Futures.immediateFailedFuture; +import static com.google.common.util.concurrent.Futures.immediateVoidFuture; import com.google.android.libraries.mobiledatadownload.DownloadException; import com.google.android.libraries.mobiledatadownload.DownloadException.DownloadResultCode; @@ -26,8 +28,8 @@ import com.google.android.libraries.mobiledatadownload.file.SynchronousFileStora import com.google.android.libraries.mobiledatadownload.file.openers.ReadStreamOpener; import com.google.android.libraries.mobiledatadownload.file.openers.WriteStreamOpener; import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; import com.google.common.io.ByteStreams; -import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import java.io.IOException; import java.io.InputStream; @@ -67,7 +69,7 @@ public final class InlineFileDownloader implements FileDownloader { LogUtil.e( "%s: Invalid url given, expected to start with 'inlinefile:', but was %s", TAG, downloadRequest.urlToDownload()); - return Futures.immediateFailedFuture( + return immediateFailedFuture( DownloadException.builder() .setDownloadResultCode(DownloadResultCode.INVALID_INLINE_FILE_URL_SCHEME) .setMessage("InlineFileDownloader only supports copying inlinefile: scheme") @@ -78,7 +80,7 @@ public final class InlineFileDownloader implements FileDownloader { InlineDownloadParams inlineDownloadParams = downloadRequest.inlineDownloadParamsOptional().get(); - return Futures.submitAsync( + return PropagatedFutures.submitAsync( () -> { try (InputStream inlineFileStream = getInputStream(inlineDownloadParams); OutputStream destinationStream = @@ -87,13 +89,13 @@ public final class InlineFileDownloader implements FileDownloader { destinationStream.flush(); } catch (IOException e) { LogUtil.e(e, "%s: Unable to copy file content.", TAG); - return Futures.immediateFailedFuture( + return immediateFailedFuture( DownloadException.builder() .setCause(e) .setDownloadResultCode(DownloadResultCode.INLINE_FILE_IO_ERROR) .build()); } - return Futures.immediateVoidFuture(); + return immediateVoidFuture(); }, downloadExecutor); } diff --git a/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/BUILD b/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/BUILD index 4774127..5d8f6d6 100644 --- a/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], @@ -34,6 +35,7 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/file/integration/downloader:downloader2", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LogUtil", "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", + "@androidx_concurrent_concurrent", "@com_google_code_findbugs_jsr305", "@com_google_guava_guava", "@downloader", @@ -64,6 +66,7 @@ android_library( deps = [ "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LogUtil", "@com_google_code_findbugs_jsr305", + "@com_google_errorprone_error_prone_annotations", "@com_google_guava_guava", ], ) diff --git a/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/ExceptionHandler.java b/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/ExceptionHandler.java index 759c805..b096bd6 100644 --- a/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/ExceptionHandler.java +++ b/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/ExceptionHandler.java @@ -71,7 +71,7 @@ public final class ExceptionHandler { return (DownloadException) throwable; } - DownloadResultCode code = mapExceptionToDownloadResultCode(throwable, /* iteration = */ 0); + DownloadResultCode code = mapExceptionToDownloadResultCode(throwable, /* iteration= */ 0); return DownloadException.builder() .setMessage(message) diff --git a/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/Offroad2FileDownloader.java b/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/Offroad2FileDownloader.java index dc2ce32..b88e005 100644 --- a/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/Offroad2FileDownloader.java +++ b/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/Offroad2FileDownloader.java @@ -17,6 +17,7 @@ package com.google.android.libraries.mobiledatadownload.downloader.offroad; import android.net.Uri; import android.util.Pair; + import com.google.android.downloader.DownloadConstraints; import com.google.android.downloader.DownloadConstraints.NetworkType; import com.google.android.downloader.DownloadDestination; @@ -41,9 +42,11 @@ import com.google.common.base.Strings; import com.google.common.util.concurrent.FluentFuture; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; + import java.io.IOException; import java.net.URI; import java.util.concurrent.Executor; + import javax.annotation.Nullable; /** @@ -51,152 +54,169 @@ import javax.annotation.Nullable; * com.google.android.libraries.mobiledatadownload.downloader.FileDownloader} using <internal> */ public final class Offroad2FileDownloader implements FileDownloader { - private static final String TAG = "Offroad2FileDownloader"; - - private final Downloader downloader; - private final SynchronousFileStorage fileStorage; - private final Executor downloadExecutor; - private final DownloadMetadataStore downloadMetadataStore; - private final ExceptionHandler exceptionHandler; - private final Optional<Integer> defaultTrafficTag; - @Nullable private final OAuthTokenProvider authTokenProvider; - - // TODO(b/208703042): refactor injection to remove dependency on ProtoDataStore - public Offroad2FileDownloader( - Downloader downloader, - SynchronousFileStorage fileStorage, - Executor downloadExecutor, - @Nullable OAuthTokenProvider authTokenProvider, - DownloadMetadataStore downloadMetadataStore, - ExceptionHandler exceptionHandler, - Optional<Integer> defaultTrafficTag) { - this.downloader = downloader; - this.fileStorage = fileStorage; - this.downloadExecutor = downloadExecutor; - this.authTokenProvider = authTokenProvider; - this.downloadMetadataStore = downloadMetadataStore; - this.exceptionHandler = exceptionHandler; - this.defaultTrafficTag = defaultTrafficTag; - } - - @Override - public ListenableFuture<Void> startDownloading( - com.google.android.libraries.mobiledatadownload.downloader.DownloadRequest - fileDownloaderRequest) { - String fileName = Strings.nullToEmpty(fileDownloaderRequest.fileUri().getLastPathSegment()); - - DownloadDestination downloadDestination; - try { - downloadDestination = buildDownloadDestination(fileDownloaderRequest.fileUri()); - } catch (DownloadException e) { - return Futures.immediateFailedFuture(e); + private static final String TAG = "Offroad2FileDownloader"; + + private final Downloader downloader; + private final SynchronousFileStorage fileStorage; + private final Executor downloadExecutor; + private final DownloadMetadataStore downloadMetadataStore; + private final ExceptionHandler exceptionHandler; + // private final Optional<Supplier<CookieJar>> cookieJarSupplierOptional; + private final Optional<Integer> defaultTrafficTag; + @Nullable + private final OAuthTokenProvider authTokenProvider; + + public Offroad2FileDownloader( + Downloader downloader, + SynchronousFileStorage fileStorage, + Executor downloadExecutor, + @Nullable OAuthTokenProvider authTokenProvider, + DownloadMetadataStore downloadMetadataStore, + ExceptionHandler exceptionHandler, +// Optional<Supplier<CookieJar>> cookieJarSupplierOptional, + Optional<Integer> defaultTrafficTag) { + this.downloader = downloader; + this.fileStorage = fileStorage; + this.downloadExecutor = downloadExecutor; + this.authTokenProvider = authTokenProvider; + this.downloadMetadataStore = downloadMetadataStore; + this.exceptionHandler = exceptionHandler; +// this.cookieJarSupplierOptional = cookieJarSupplierOptional; + this.defaultTrafficTag = defaultTrafficTag; + } + + @Override + public ListenableFuture<Void> startDownloading( + com.google.android.libraries.mobiledatadownload.downloader.DownloadRequest + fileDownloaderRequest) { + String fileName = Strings.nullToEmpty(fileDownloaderRequest.fileUri().getLastPathSegment()); + + DownloadDestination downloadDestination; + try { + downloadDestination = buildDownloadDestination(fileDownloaderRequest.fileUri()); + } catch (DownloadException e) { + return Futures.immediateFailedFuture(e); + } + + DownloadRequest offroad2DownloadRequest = + buildDownloadRequest(fileDownloaderRequest, downloadDestination); + + FluentFuture<DownloadResult> resultFuture = downloader.execute(offroad2DownloadRequest); + + LogUtil.d( + "%s: Data download scheduled for file: %s", TAG, + fileDownloaderRequest.urlToDownload()); + + return PropagatedFluentFuture.from(resultFuture) + .catchingAsync( + Exception.class, + cause -> { + LogUtil.d( + cause, + "%s: Failed to download file %s due to: %s", + TAG, + fileName, + Strings.nullToEmpty(cause.getMessage())); + + DownloadException exception = + exceptionHandler.mapToDownloadException("failure in download!", + cause); + + return Futures.immediateFailedFuture(exception); + }, + downloadExecutor) + .transformAsync( + (DownloadResult result) -> { + LogUtil.d( + "%s: Downloaded file %s, bytes written: %d", + TAG, fileName, result.bytesWritten()); + return PropagatedFutures.catchingAsync( + downloadMetadataStore.delete(fileDownloaderRequest.fileUri()), + Exception.class, + e -> { + // Failing to clean up metadata shouldn't cause a failure + // in the future, log and + // return void. + LogUtil.d(e, "%s: Failed to cleanup metadata", TAG); + return Futures.immediateVoidFuture(); + }, + downloadExecutor); + }, + downloadExecutor); } - DownloadRequest offroad2DownloadRequest = - buildDownloadRequest(fileDownloaderRequest, downloadDestination); - - FluentFuture<DownloadResult> resultFuture = downloader.execute(offroad2DownloadRequest); - - LogUtil.d( - "%s: Data download scheduled for file: %s", TAG, fileDownloaderRequest.urlToDownload()); - - return PropagatedFluentFuture.from(resultFuture) - .catchingAsync( - Exception.class, - cause -> { - LogUtil.d( - cause, - "%s: Failed to download file %s due to: %s", - TAG, - fileName, - Strings.nullToEmpty(cause.getMessage())); - - DownloadException exception = - exceptionHandler.mapToDownloadException("failure in download!", cause); - - return Futures.immediateFailedFuture(exception); - }, - downloadExecutor) - .transformAsync( - (DownloadResult result) -> { - LogUtil.d( - "%s: Downloaded file %s, bytes written: %d", - TAG, fileName, result.bytesWritten()); - return PropagatedFutures.catchingAsync( - downloadMetadataStore.delete(fileDownloaderRequest.fileUri()), - Exception.class, - e -> { - // Failing to clean up metadata shouldn't cause a failure in the future, log and - // return void. - LogUtil.d(e, "%s: Failed to cleanup metadata", TAG); - return Futures.immediateVoidFuture(); - }, - downloadExecutor); - }, - downloadExecutor); - } - - @Override - public ListenableFuture<CheckContentChangeResponse> isContentChanged( - CheckContentChangeRequest checkContentChangeRequest) { - return Futures.immediateFailedFuture( - new UnsupportedOperationException( - "Checking for content changes is currently unsupported for Downloader2")); - } - - private DownloadDestination buildDownloadDestination(Uri destinationUri) - throws DownloadException { - try { - // Create DownloadDestination using mobstore - return fileStorage.open( - destinationUri, DownloadDestinationOpener.create(downloadMetadataStore)); - } catch (IOException e) { - if (e instanceof MalformedUriException || e.getCause() instanceof IllegalArgumentException) { - LogUtil.e("%s: The file uri is invalid, uri = %s", TAG, destinationUri); - throw DownloadException.builder() - .setDownloadResultCode(DownloadResultCode.MALFORMED_FILE_URI_ERROR) - .setCause(e) - .build(); - } else { - LogUtil.e(e, "%s: Unable to create DownloadDestination for file %s", TAG, destinationUri); - // TODO: the result code is the most equivalent to downloader1 -- consider - // creating a separate result code that's more appropriate for downloader2. - throw DownloadException.builder() - .setDownloadResultCode( - DownloadResultCode.UNABLE_TO_CREATE_MOBSTORE_RESPONSE_WRITER_ERROR) - .setCause(e) - .build(); - } + @Override + public ListenableFuture<CheckContentChangeResponse> isContentChanged( + CheckContentChangeRequest checkContentChangeRequest) { + return Futures.immediateFailedFuture( + new UnsupportedOperationException( + "Checking for content changes is currently unsupported for Downloader2")); } - } - - private DownloadRequest buildDownloadRequest( - com.google.android.libraries.mobiledatadownload.downloader.DownloadRequest - fileDownloaderRequest, - DownloadDestination downloadDestination) { - DownloadRequest.Builder requestBuilder = - downloader.newRequestBuilder( - URI.create(fileDownloaderRequest.urlToDownload()), downloadDestination); - - requestBuilder.setOAuthTokenProvider(authTokenProvider); - - if (com.google.android.libraries.mobiledatadownload.downloader.DownloadConstraints - .NETWORK_CONNECTED - == fileDownloaderRequest.downloadConstraints()) { - requestBuilder.setDownloadConstraints(DownloadConstraints.NETWORK_CONNECTED); - } else { - // Use all network types except cellular and require unmetered network. - requestBuilder.setDownloadConstraints( - DownloadConstraints.builder() - .addRequiredNetworkType(NetworkType.WIFI) - .addRequiredNetworkType(NetworkType.ETHERNET) - .addRequiredNetworkType(NetworkType.BLUETOOTH) - .setRequireUnmeteredNetwork(true) - .build()); + + private DownloadDestination buildDownloadDestination(Uri destinationUri) + throws DownloadException { + try { + // Create DownloadDestination using mobstore + // NOTE: the use of DirectExecutor here should be fine since all async operations + // of DownloadDestination happen within Downloader2 IOExecutor. Consider replacing + // this with + // lightweight executor. + return fileStorage.open( + destinationUri, + DownloadDestinationOpener.create(downloadMetadataStore)); + } catch (IOException e) { + if (e instanceof MalformedUriException + || e.getCause() instanceof IllegalArgumentException) { + LogUtil.e("%s: The file uri is invalid, uri = %s", TAG, destinationUri); + throw DownloadException.builder() + .setDownloadResultCode(DownloadResultCode.MALFORMED_FILE_URI_ERROR) + .setCause(e) + .build(); + } else { + LogUtil.e(e, "%s: Unable to create DownloadDestination for file %s", TAG, + destinationUri); + // TODO: the result code is the most equivalent to downloader1 -- consider + // creating a separate result code that's more appropriate for downloader2. + throw DownloadException.builder() + .setDownloadResultCode( + DownloadResultCode.UNABLE_TO_CREATE_MOBSTORE_RESPONSE_WRITER_ERROR) + .setCause(e) + .build(); + } + } } - // TODO(b/237653774): Enable traffic tagging. - /* if (fileDownloaderRequest.trafficTag() > 0) { + private DownloadRequest buildDownloadRequest( + com.google.android.libraries.mobiledatadownload.downloader.DownloadRequest + fileDownloaderRequest, + DownloadDestination downloadDestination) { + DownloadRequest.Builder requestBuilder = + downloader.newRequestBuilder( + URI.create(fileDownloaderRequest.urlToDownload()), downloadDestination); + +// if (cookieJarSupplierOptional.isPresent()) { +// requestBuilder.setCookieJar(cookieJarSupplierOptional.get().get()); +// } + + requestBuilder.setOAuthTokenProvider(authTokenProvider); + + if (com.google.android.libraries.mobiledatadownload.downloader.DownloadConstraints + .NETWORK_CONNECTED + == fileDownloaderRequest.downloadConstraints()) { + requestBuilder.setDownloadConstraints(DownloadConstraints.NETWORK_CONNECTED); + } else { + // Use all network types except cellular and require unmetered network. + requestBuilder.setDownloadConstraints( + DownloadConstraints.builder() + .addRequiredNetworkType(NetworkType.WIFI) + .addRequiredNetworkType(NetworkType.ETHERNET) + .addRequiredNetworkType(NetworkType.BLUETOOTH) + .setRequireUnmeteredNetwork(true) + .build()); + } + + // TODO(b/237653774): Enable traffic tagging. + /*if (fileDownloaderRequest.trafficTag() > 0) { // Prefer traffic tag from request. requestBuilder.setTrafficStatsTag(fileDownloaderRequest.trafficTag()); } else if (defaultTrafficTag.isPresent() && defaultTrafficTag.get() > 0) { @@ -204,10 +224,10 @@ public final class Offroad2FileDownloader implements FileDownloader { requestBuilder.setTrafficStatsTag(defaultTrafficTag.get()); }*/ - for (Pair<String, String> header : fileDownloaderRequest.extraHttpHeaders()) { - requestBuilder.addHeader(header.first, header.second); - } + for (Pair<String, String> header : fileDownloaderRequest.extraHttpHeaders()) { + requestBuilder.addHeader(header.first, header.second); + } - return requestBuilder.build(); - } + return requestBuilder.build(); + } } diff --git a/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/ThrottlingExecutor.java b/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/ThrottlingExecutor.java index 9cebf11..bbea425 100644 --- a/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/ThrottlingExecutor.java +++ b/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/ThrottlingExecutor.java @@ -18,10 +18,10 @@ package com.google.android.libraries.mobiledatadownload.downloader.offroad; import static com.google.common.base.Preconditions.checkNotNull; import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; +import com.google.errorprone.annotations.concurrent.GuardedBy; import java.util.ArrayDeque; import java.util.Queue; import java.util.concurrent.Executor; -import javax.annotation.concurrent.GuardedBy; /** * Passes tasks to a delegate {@link Executor} for execution, ensuring that no more than a fixed diff --git a/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/annotations/BUILD b/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/annotations/BUILD new file mode 100644 index 0000000..38f24ec --- /dev/null +++ b/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/annotations/BUILD @@ -0,0 +1,33 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +load("@build_bazel_rules_android//android:rules.bzl", "android_library") + +package( + default_applicable_licenses = ["//:license"], + default_visibility = [ + "//visibility:public", + ], + licenses = ["notice"], +) + +android_library( + name = "downloader2", + srcs = [ + "DownloaderFollowRedirectsImmediately.java", + ], + deps = [ + "@com_google_dagger", + "@javax_inject", + ], +) diff --git a/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/annotations/DownloaderFollowRedirectsImmediately.java b/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/annotations/DownloaderFollowRedirectsImmediately.java new file mode 100644 index 0000000..2f053fb --- /dev/null +++ b/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/annotations/DownloaderFollowRedirectsImmediately.java @@ -0,0 +1,35 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.libraries.mobiledatadownload.downloader.offroad.annotations; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Target; +import javax.inject.Qualifier; + +/** + * A Flag that controls whether the url engines registered to Downloader should follow redirects + * immediately. + * + * <p>In most common cases, this flag should be true, but there are some features which require this + * flag to be false (such as when providing Cookies on redirect requests is required). + * + * <p>NOTE: This flag will be calculated in MDD's {@link BaseFileDownloaderDepsModule} based on + * other client-provided dependencies, so clients do not have to provide a binding for the flag + * itself. + */ +@Target({ElementType.PARAMETER, ElementType.METHOD, ElementType.FIELD}) +@Qualifier +public @interface DownloaderFollowRedirectsImmediately {} diff --git a/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/dagger/BUILD b/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/dagger/BUILD index 60814e8..91b9524 100644 --- a/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/dagger/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/dagger/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], diff --git a/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/dagger/downloader2/BUILD b/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/dagger/downloader2/BUILD index 13f05e0..4e2b70f 100644 --- a/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/dagger/downloader2/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/dagger/downloader2/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], @@ -34,7 +35,6 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/file", "//java/com/google/android/libraries/mobiledatadownload/file/integration/downloader:downloader2", "//java/com/google/android/libraries/mobiledatadownload/monitor:DownloadProgressMonitor", - "@androidx_annotation_annotation", "@com_google_code_findbugs_jsr305", "@com_google_dagger", "@com_google_guava_guava", @@ -47,9 +47,13 @@ android_library( name = "base_deps", srcs = ["BaseFileDownloaderDepsModule.java"], deps = [ + "//java/com/google/android/libraries/mobiledatadownload/annotations", "//java/com/google/android/libraries/mobiledatadownload/downloader/offroad:ExceptionHandler", - "@androidx_annotation_annotation", + "//java/com/google/android/libraries/mobiledatadownload/downloader/offroad/annotations:downloader2", + "//java/com/google/android/libraries/mobiledatadownload/internal/util:DirectoryUtil", "@com_google_dagger", + "@com_google_guava_guava", "@downloader", + "@javax_inject", ], ) diff --git a/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/dagger/downloader2/BaseFileDownloaderDepsModule.java b/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/dagger/downloader2/BaseFileDownloaderDepsModule.java index f98d13f..cac3bf4 100644 --- a/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/dagger/downloader2/BaseFileDownloaderDepsModule.java +++ b/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/dagger/downloader2/BaseFileDownloaderDepsModule.java @@ -15,9 +15,9 @@ */ package com.google.android.libraries.mobiledatadownload.downloader.offroad.dagger.downloader2; -import androidx.annotation.VisibleForTesting; import com.google.android.downloader.UrlEngine; import com.google.android.libraries.mobiledatadownload.downloader.offroad.ExceptionHandler; + import dagger.BindsOptionalOf; import dagger.Module; @@ -30,24 +30,43 @@ import dagger.Module; * used across all FileDownloaders backed by Android Downloader2. */ @Module -@VisibleForTesting public abstract class BaseFileDownloaderDepsModule { - /** - * Platform specific {@link ExceptionHandler}. - * - * <p>If no specific exception handler is available, the default one will be used. - */ - @BindsOptionalOf - abstract ExceptionHandler platformSpecificExceptionHandler(); + /** + * Platform specific {@link ExceptionHandler}. + * + * <p>If no specific exception handler is available, the default one will be used. + */ + @BindsOptionalOf + abstract ExceptionHandler platformSpecificExceptionHandler(); + + /** + * Platform specific {@link UrlEngine}. + * + * <p>If no specific engine is provided, the platform engine will be used. + */ + @BindsOptionalOf + abstract UrlEngine platformSpecificUrlEngine(); - /** - * Platform specific {@link UrlEngine}. - * - * <p>If no specific engine is provided, the platform engine will be used. - */ - @BindsOptionalOf - abstract UrlEngine platformSpecificUrlEngine(); + /** + * Optional {@link CookieJar} which will be supplied to each download request. + * + * <p>If no cookie jar is provided, no cookie handling will be performed. + * + * <p>NOTE: CookieJar support is only available for Cronet at this time. // TODO(b/254955843) + * : Add + * support for platform/okhttp2/okhttp3 engines + */ +// @BindsOptionalOf +// abstract Supplier<CookieJar> requestCookieJarSupplier(); - private BaseFileDownloaderDepsModule() {} + /** Calculate whether or not we should follow redirects immediately. */ +// @Provides +// @DownloaderFollowRedirectsImmediately +// static boolean provideFollowRedirectsImmediatelyFlag( +// Optional<Supplier<CookieJar>> cookieJarSupplier) { +// return !cookieJarSupplier.isPresent(); +// } + private BaseFileDownloaderDepsModule() { + } } diff --git a/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/dagger/downloader2/BaseFileDownloaderModule.java b/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/dagger/downloader2/BaseFileDownloaderModule.java index 425608c..b518574 100644 --- a/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/dagger/downloader2/BaseFileDownloaderModule.java +++ b/java/com/google/android/libraries/mobiledatadownload/downloader/offroad/dagger/downloader2/BaseFileDownloaderModule.java @@ -18,12 +18,11 @@ package com.google.android.libraries.mobiledatadownload.downloader.offroad.dagge import static com.google.common.util.concurrent.Futures.immediateFuture; import android.content.Context; -import androidx.annotation.VisibleForTesting; + import com.google.android.downloader.AndroidConnectivityHandler; import com.google.android.downloader.Downloader; import com.google.android.downloader.Downloader.StateChangeCallback; import com.google.android.downloader.FloggerDownloaderLogger; -import com.google.android.downloader.PlatformAndroidTrafficStatsTagger; import com.google.android.downloader.PlatformUrlEngine; import com.google.android.downloader.UrlEngine; import com.google.android.libraries.mobiledatadownload.Flags; @@ -41,14 +40,17 @@ import com.google.android.libraries.mobiledatadownload.monitor.DownloadProgressM import com.google.common.base.Optional; import com.google.common.base.Supplier; import com.google.common.util.concurrent.ListeningExecutorService; + +import java.util.concurrent.ScheduledExecutorService; + +import javax.annotation.Nullable; +import javax.inject.Singleton; + import dagger.Lazy; import dagger.Module; import dagger.Provides; import dagger.multibindings.IntoMap; import dagger.multibindings.StringKey; -import java.util.concurrent.ScheduledExecutorService; -import javax.annotation.Nullable; -import javax.inject.Singleton; /** * Dagger module for providing FileDownloader that uses Android Downloader2. @@ -58,121 +60,136 @@ import javax.inject.Singleton; * module assumes is available to bind into. */ @Module( - includes = { - BaseOffroadFileDownloaderModule.class, - BaseFileDownloaderDepsModule.class, - }) -@VisibleForTesting + includes = { + BaseOffroadFileDownloaderModule.class, + BaseFileDownloaderDepsModule.class, + }) public abstract class BaseFileDownloaderModule { - @Provides - @Singleton - @IntoMap - @StringKey("https") - static Supplier<FileDownloader> provideFileDownloader( - Context context, - @MddDownloadExecutor ScheduledExecutorService downloadExecutor, - @MddControlExecutor ListeningExecutorService controlExecutor, - SynchronousFileStorage fileStorage, - DownloadMetadataStore downloadMetadataStore, - Optional<DownloadProgressMonitor> downloadProgressMonitor, - Optional<Lazy<UrlEngine>> urlEngineOptional, - Optional<Lazy<ExceptionHandler>> exceptionHandlerOptional, - Optional<Lazy<OAuthTokenProvider>> authTokenProviderOptional, - @SocketTrafficTag Optional<Integer> trafficTag, - Flags flags) { - return () -> - createOffroad2FileDownloader( - context, - downloadExecutor, - controlExecutor, - fileStorage, - downloadMetadataStore, - downloadProgressMonitor, - urlEngineOptional, - exceptionHandlerOptional, - authTokenProviderOptional, - trafficTag, - flags); - } - - @VisibleForTesting - public static Offroad2FileDownloader createOffroad2FileDownloader( - Context context, - ScheduledExecutorService downloadExecutor, - ListeningExecutorService controlExecutor, - SynchronousFileStorage fileStorage, - DownloadMetadataStore downloadMetadataStore, - Optional<DownloadProgressMonitor> downloadProgressMonitor, - Optional<Lazy<UrlEngine>> urlEngineOptional, - Optional<Lazy<ExceptionHandler>> exceptionHandlerOptional, - Optional<Lazy<OAuthTokenProvider>> authTokenProviderOptional, - Optional<Integer> trafficTag, - Flags flags) { - @Nullable - com.google.android.downloader.OAuthTokenProvider authTokenProvider = - authTokenProviderOptional.isPresent() - ? convertToDownloaderAuthTokenProvider(authTokenProviderOptional.get().get()) - : null; - - ExceptionHandler handler = - exceptionHandlerOptional.transform(Lazy::get).or(ExceptionHandler.withDefaultHandling()); - - UrlEngine urlEngine; - if (urlEngineOptional.isPresent()) { - urlEngine = urlEngineOptional.get().get(); - } else { - // Use {@link PlatformUrlEngine} if one was not provided. - urlEngine = - new PlatformUrlEngine( - controlExecutor, - /* connectTimeoutMs = */ flags.timeToWaitForDownloader(), - /* readTimeoutMs = */ flags.timeToWaitForDownloader(), - new PlatformAndroidTrafficStatsTagger()); + @Provides + @Singleton + @IntoMap + @StringKey("https") + static Supplier<FileDownloader> provideFileDownloader( + Context context, + @MddDownloadExecutor ScheduledExecutorService downloadExecutor, + @MddControlExecutor ListeningExecutorService controlExecutor, + SynchronousFileStorage fileStorage, + DownloadMetadataStore downloadMetadataStore, + Optional<DownloadProgressMonitor> downloadProgressMonitor, + Optional<Lazy<UrlEngine>> urlEngineOptional, + Optional<Lazy<ExceptionHandler>> exceptionHandlerOptional, + Optional<Lazy<OAuthTokenProvider>> authTokenProviderOptional, +// Optional<Supplier<CookieJar>> cookieJarSupplierOptional, + @SocketTrafficTag Optional<Integer> trafficTag, + Flags flags) { + return () -> + createOffroad2FileDownloader( + context, + downloadExecutor, + controlExecutor, + fileStorage, + downloadMetadataStore, + downloadProgressMonitor, + urlEngineOptional, + exceptionHandlerOptional, + authTokenProviderOptional, +// cookieJarSupplierOptional, + trafficTag, + flags); } - AndroidConnectivityHandler connectivityHandler = - new AndroidConnectivityHandler( - context, downloadExecutor, /* timeoutMillis = */ flags.timeToWaitForDownloader()); - - FloggerDownloaderLogger logger = new FloggerDownloaderLogger(); - - Downloader downloader = - new Downloader.Builder() - .withIOExecutor(controlExecutor) - .withConnectivityHandler(connectivityHandler) - .withMaxConcurrentDownloads(flags.downloaderMaxThreads()) - .withLogger(logger) - .addUrlEngine("https", urlEngine) - .build(); - - if (downloadProgressMonitor.isPresent()) { - // Wire up downloader's state changes to DownloadProgressMonitor to handle connectivity - // pauses. - StateChangeCallback callback = - state -> { - if (state.getNumDownloadsPendingConnectivity() > 0 - && state.getNumDownloadsInFlight() == 0) { - // Handle network connectivity pauses - downloadProgressMonitor.get().pausedForConnectivity(); - } - }; - downloader.registerStateChangeCallback(callback, controlExecutor); + /** + * Manual provider of Offroad2FileDownloader. + * + * <p>NOTE: This method should only be used when manually wiring up dependencies, such as when + * dagger/hilt are not available. If using dagger/hilt, this method is not needed. By + * registering + * this module in the dagger graph, the above @Provides method will automatically provide this + * dependency. + */ + public static Offroad2FileDownloader createOffroad2FileDownloader( + Context context, + ScheduledExecutorService downloadExecutor, + ListeningExecutorService controlExecutor, + SynchronousFileStorage fileStorage, + DownloadMetadataStore downloadMetadataStore, + Optional<DownloadProgressMonitor> downloadProgressMonitor, + Optional<Lazy<UrlEngine>> urlEngineOptional, + Optional<Lazy<ExceptionHandler>> exceptionHandlerOptional, + Optional<Lazy<OAuthTokenProvider>> authTokenProviderOptional, +// Optional<Supplier<CookieJar>> cookieJarSupplierOptional, + Optional<Integer> trafficTag, + Flags flags) { + @Nullable + com.google.android.downloader.OAuthTokenProvider authTokenProvider = + authTokenProviderOptional.isPresent() + ? convertToDownloaderAuthTokenProvider( + authTokenProviderOptional.get().get()) + : null; + + ExceptionHandler handler = + exceptionHandlerOptional.transform(Lazy::get).or( + ExceptionHandler.withDefaultHandling()); + + UrlEngine urlEngine; + if (urlEngineOptional.isPresent()) { + urlEngine = urlEngineOptional.get().get(); + } else { + // Use {@link PlatformUrlEngine} if one was not provided. + urlEngine = + new PlatformUrlEngine( + controlExecutor, + /* connectTimeoutMs = */ flags.timeToWaitForDownloader(), + /* readTimeoutMs = */ flags.timeToWaitForDownloader() + ); + } + + AndroidConnectivityHandler connectivityHandler = + new AndroidConnectivityHandler( + context, downloadExecutor, /* timeoutMillis= */ + flags.timeToWaitForDownloader()); + + FloggerDownloaderLogger logger = new FloggerDownloaderLogger(); + + Downloader downloader = + new Downloader.Builder() + .withIOExecutor(controlExecutor) + .withConnectivityHandler(connectivityHandler) + .withMaxConcurrentDownloads(flags.downloaderMaxThreads()) + .withLogger(logger) + .addUrlEngine("https", urlEngine) + .build(); + + if (downloadProgressMonitor.isPresent()) { + // Wire up downloader's state changes to DownloadProgressMonitor to handle connectivity + // pauses. + StateChangeCallback callback = + state -> { + if (state.getNumDownloadsPendingConnectivity() > 0 + && state.getNumDownloadsInFlight() == 0) { + // Handle network connectivity pauses + downloadProgressMonitor.get().pausedForConnectivity(); + } + }; + downloader.registerStateChangeCallback(callback, controlExecutor); + } + + return new Offroad2FileDownloader( + downloader, + fileStorage, + downloadExecutor, + authTokenProvider, + downloadMetadataStore, + handler, +// cookieJarSupplierOptional, + trafficTag); } - return new Offroad2FileDownloader( - downloader, - fileStorage, - downloadExecutor, - authTokenProvider, - downloadMetadataStore, - handler, - trafficTag); - } - - private static com.google.android.downloader.OAuthTokenProvider - convertToDownloaderAuthTokenProvider(OAuthTokenProvider authTokenProvider) { - return uri -> immediateFuture(authTokenProvider.provideOAuthToken(uri.toString())); - } - - private BaseFileDownloaderModule() {} + private static com.google.android.downloader.OAuthTokenProvider + convertToDownloaderAuthTokenProvider(OAuthTokenProvider authTokenProvider) { + return uri -> immediateFuture(authTokenProvider.provideOAuthToken(uri.toString())); + } + + private BaseFileDownloaderModule() { + } } diff --git a/java/com/google/android/libraries/mobiledatadownload/file/BUILD b/java/com/google/android/libraries/mobiledatadownload/file/BUILD index 34950b9..d3da9ef 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/file/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], diff --git a/java/com/google/android/libraries/mobiledatadownload/file/OpenContext.java b/java/com/google/android/libraries/mobiledatadownload/file/OpenContext.java index ddcb968..486544d 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/OpenContext.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/OpenContext.java @@ -20,6 +20,7 @@ import com.google.android.libraries.mobiledatadownload.file.spi.Backend; import com.google.android.libraries.mobiledatadownload.file.spi.Monitor; import com.google.android.libraries.mobiledatadownload.file.spi.Transform; import com.google.common.collect.Iterables; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -51,31 +52,37 @@ public final class OpenContext { private Builder() {} + @CanIgnoreReturnValue Builder setStorage(SynchronousFileStorage storage) { this.storage = storage; return this; } + @CanIgnoreReturnValue Builder setBackend(Backend backend) { this.backend = backend; return this; } + @CanIgnoreReturnValue Builder setTransforms(List<Transform> transforms) { this.transforms = transforms; return this; } + @CanIgnoreReturnValue Builder setMonitors(List<Monitor> monitors) { this.monitors = monitors; return this; } + @CanIgnoreReturnValue Builder setEncodedUri(Uri encodedUri) { this.encodedUri = encodedUri; return this; } + @CanIgnoreReturnValue Builder setOriginalUri(Uri originalUri) { this.originalUri = originalUri; return this; diff --git a/java/com/google/android/libraries/mobiledatadownload/file/backends/AndroidFileBackend.java b/java/com/google/android/libraries/mobiledatadownload/file/backends/AndroidFileBackend.java index e1d8b9d..67ebbc8 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/backends/AndroidFileBackend.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/backends/AndroidFileBackend.java @@ -28,12 +28,13 @@ import com.google.android.libraries.mobiledatadownload.file.common.MalformedUriE import com.google.android.libraries.mobiledatadownload.file.common.internal.Preconditions; import com.google.android.libraries.mobiledatadownload.file.spi.Backend; import com.google.android.libraries.mobiledatadownload.file.spi.ForwardingBackend; +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import com.google.errorprone.annotations.concurrent.GuardedBy; import java.io.Closeable; import java.io.File; import java.io.IOException; import java.io.InputStream; import javax.annotation.Nullable; -import javax.annotation.concurrent.GuardedBy; /** A backend that implements "android:" scheme using {@link JavaFileBackend}. */ public final class AndroidFileBackend extends ForwardingBackend { @@ -92,6 +93,7 @@ public final class AndroidFileBackend extends ForwardingBackend { * than your own. The only methods called on {@code remoteBackend} are {@link #openForRead} and * {@link #openForNativeRead}, though this may expand in the future. Defaults to {@code null}. */ + @CanIgnoreReturnValue public Builder setRemoteBackend(Backend remoteBackend) { this.remoteBackend = remoteBackend; return this; @@ -101,6 +103,7 @@ public final class AndroidFileBackend extends ForwardingBackend { * Sets the {@link AccountManager} invoked to resolve "managed" URIs. Defaults to {@code null}, * in which case operations on "managed" URIs will fail. */ + @CanIgnoreReturnValue public Builder setAccountManager(AccountManager accountManager) { this.accountManager = accountManager; return this; @@ -111,6 +114,7 @@ public final class AndroidFileBackend extends ForwardingBackend { * injection is only necessary if there are multiple backend instances in the same process and * there's a risk of them acquiring a lock on the same underlying file. */ + @CanIgnoreReturnValue public Builder setLockScope(LockScope lockScope) { Preconditions.checkArgument( backend == null, diff --git a/java/com/google/android/libraries/mobiledatadownload/file/backends/AndroidUri.java b/java/com/google/android/libraries/mobiledatadownload/file/backends/AndroidUri.java index da6bc2e..26b0107 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/backends/AndroidUri.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/backends/AndroidUri.java @@ -24,6 +24,7 @@ import com.google.android.libraries.mobiledatadownload.file.common.internal.Lite import com.google.android.libraries.mobiledatadownload.file.common.internal.Preconditions; import com.google.android.libraries.mobiledatadownload.file.transforms.TransformProtos; import com.google.common.collect.ImmutableList; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import com.google.mobiledatadownload.TransformProto; import java.io.File; import java.util.Arrays; @@ -149,42 +150,51 @@ public final class AndroidUri { /** * Sets the package to use in the android uri AUTHORITY. Default is context.getPackageName(). */ + @CanIgnoreReturnValue public Builder setPackage(String packageName) { this.packageName = packageName; return this; } + @CanIgnoreReturnValue private Builder setLocation(String location) { AndroidUri.validateLocation(location); this.location = location; return this; } + @CanIgnoreReturnValue public Builder setManagedLocation() { return setLocation(MANAGED_LOCATION); } + @CanIgnoreReturnValue public Builder setExternalLocation() { return setLocation(EXTERNAL_LOCATION); } + @CanIgnoreReturnValue public Builder setDirectBootFilesLocation() { return setLocation(DIRECT_BOOT_FILES_LOCATION); } + @CanIgnoreReturnValue public Builder setDirectBootCacheLocation() { return setLocation(DIRECT_BOOT_CACHE_LOCATION); } /** Internal location, aka "files", is the default location. */ + @CanIgnoreReturnValue public Builder setInternalLocation() { return setLocation(FILES_LOCATION); } + @CanIgnoreReturnValue public Builder setCacheLocation() { return setLocation(CACHE_LOCATION); } + @CanIgnoreReturnValue public Builder setModule(String module) { AndroidUri.validateModule(module); this.module = module; @@ -210,6 +220,7 @@ public final class AndroidUri { * @param account The account to set. * @return The fluent Builder. */ + @CanIgnoreReturnValue public Builder setAccount(Account account) { AccountSerialization.serialize(account); // performs validation internally this.account = account; @@ -220,6 +231,7 @@ public final class AndroidUri { * Sets the component of the path after location, module and account. A single leading slash * will be trimmed if present. */ + @CanIgnoreReturnValue public Builder setRelativePath(String relativePath) { if (relativePath.startsWith("/")) { relativePath = relativePath.substring(1); @@ -233,6 +245,7 @@ public final class AndroidUri { * Updates builder with multiple fields from file param: location, module, account and relative * path. This method will fail on "managed" paths (see {@link fromFile(File, AccountManager)}). */ + @CanIgnoreReturnValue public Builder fromFile(File file) { return fromAbsolutePath(file.getAbsolutePath(), /* accountManager= */ null); } @@ -241,6 +254,7 @@ public final class AndroidUri { * Updates builder with multiple fields from file param: location, module, account and relative * path. A non-null {@code accountManager} is required to handle "managed" paths. */ + @CanIgnoreReturnValue public Builder fromFile(File file, @Nullable AccountManager accountManager) { return fromAbsolutePath(file.getAbsolutePath(), accountManager); } @@ -250,6 +264,7 @@ public final class AndroidUri { * relative path. This method will fail on "managed" paths (see {@link fromAbsolutePath(String, * AccountManager)}). */ + @CanIgnoreReturnValue public Builder fromAbsolutePath(String absolutePath) { return fromAbsolutePath(absolutePath, /* accountManager= */ null); } @@ -259,6 +274,7 @@ public final class AndroidUri { * relative path. A non-null {@code accountManager} is required to handle "managed" paths. */ // TODO(b/129467051): remove requirement for segments after 0th (logical location) + @CanIgnoreReturnValue public Builder fromAbsolutePath(String absolutePath, @Nullable AccountManager accountManager) { // Get the file's path within internal files, /module/account</relativePath> File filesDir = AndroidFileEnvironment.getFilesDirWithPreNWorkaround(context); @@ -341,6 +357,7 @@ public final class AndroidUri { return this; } + @CanIgnoreReturnValue public Builder withTransform(TransformProto.Transform spec) { encodedSpecs.add(TransformProtos.toEncodedSpec(spec)); return this; diff --git a/java/com/google/android/libraries/mobiledatadownload/file/backends/AndroidUriAdapter.java b/java/com/google/android/libraries/mobiledatadownload/file/backends/AndroidUriAdapter.java index 7f42232..c7c8ff1 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/backends/AndroidUriAdapter.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/backends/AndroidUriAdapter.java @@ -28,7 +28,7 @@ import javax.annotation.Nullable; /** * Adapter for converting "android:" URIs into java.io.File. This is considered dangerous since it - * ignores parts of the Uri at the caller's peril, and thus is only available to allowlisted clients + * ignores parts of the Uri at the caller's peril, and thus is only available to whitelisted clients * (mostly internal). */ public final class AndroidUriAdapter implements UriAdapter { diff --git a/java/com/google/android/libraries/mobiledatadownload/file/backends/BUILD b/java/com/google/android/libraries/mobiledatadownload/file/backends/BUILD index 0e919e5..2abed92 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/backends/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/file/backends/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], @@ -52,6 +53,7 @@ android_library( "//proto:transform_java_proto_lite", "@androidx_annotation_annotation", "@com_google_code_findbugs_jsr305", + "@com_google_errorprone_error_prone_annotations", "@com_google_guava_guava", ], ) @@ -63,6 +65,7 @@ android_library( ], deps = [ "//java/com/google/android/libraries/mobiledatadownload/file/common", + "@com_google_errorprone_error_prone_annotations", "@com_google_guava_guava", ], ) @@ -78,6 +81,7 @@ android_library( ":file_descriptor", "//java/com/google/android/libraries/mobiledatadownload/file/common", "//java/com/google/android/libraries/mobiledatadownload/file/spi", + "@com_google_code_findbugs_jsr305", "@com_google_guava_guava", ], ) @@ -92,6 +96,7 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/file/common", "//java/com/google/android/libraries/mobiledatadownload/file/common/internal:preconditions", "//java/com/google/android/libraries/mobiledatadownload/file/spi", + "@com_google_errorprone_error_prone_annotations", ], ) @@ -121,6 +126,7 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/file/spi", "//java/com/google/android/libraries/mobiledatadownload/file/transforms:proto", "//proto:transform_java_proto_lite", + "@com_google_errorprone_error_prone_annotations", "@com_google_guava_guava", ], ) @@ -137,6 +143,7 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/file/spi", "//java/com/google/android/libraries/mobiledatadownload/file/transforms:proto", "//proto:transform_java_proto_lite", + "@com_google_errorprone_error_prone_annotations", "@com_google_guava_guava", ], ) @@ -171,6 +178,7 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/file/transforms:proto", "//proto:transform_java_proto_lite", "@com_google_code_findbugs_jsr305", + "@com_google_errorprone_error_prone_annotations", "@com_google_guava_guava", ], ) @@ -211,6 +219,7 @@ android_library( visibility = ["//:__subpackages__"], deps = [ "//java/com/google/android/libraries/mobiledatadownload/file/common/internal:preconditions", + "@com_google_errorprone_error_prone_annotations", # NOTE: dependency of gmscore client lib <internal> ], ) diff --git a/java/com/google/android/libraries/mobiledatadownload/file/backends/BlobStoreBackend.java b/java/com/google/android/libraries/mobiledatadownload/file/backends/BlobStoreBackend.java index 497efc0..80ae24b 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/backends/BlobStoreBackend.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/backends/BlobStoreBackend.java @@ -34,6 +34,7 @@ import java.io.OutputStream; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; +import javax.annotation.Nullable; /** * Backend for accessing the Android blob Sharing Service. @@ -118,7 +119,7 @@ public final class BlobStoreBackend implements Backend { * @throws IOException when there is an I/O error while writing the blob/lease. */ @Override - public OutputStream openForWrite(Uri uri) throws IOException { + public @Nullable OutputStream openForWrite(Uri uri) throws IOException { BlobUri.validateUri(uri); byte[] checksum = BlobUri.getChecksum(uri.getPath()); try { diff --git a/java/com/google/android/libraries/mobiledatadownload/file/backends/BlobUri.java b/java/com/google/android/libraries/mobiledatadownload/file/backends/BlobUri.java index 28b14e5..29fc1f1 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/backends/BlobUri.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/backends/BlobUri.java @@ -21,6 +21,7 @@ import android.text.TextUtils; import com.google.android.libraries.mobiledatadownload.file.common.MalformedUriException; import com.google.common.base.Splitter; import com.google.common.io.BaseEncoding; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.util.List; /** Helper class for "blobstore" URIs. */ @@ -151,17 +152,20 @@ public final class BlobUri { this.packageName = context.getPackageName(); } + @CanIgnoreReturnValue public Builder setBlobParameters(String checksum) { path = checksum; return this; } + @CanIgnoreReturnValue public Builder setLeaseParameters(String checksum, long expiryDateSecs) { path = checksum + LEASE_URI_SUFFIX; this.expiryDateSecs = expiryDateSecs; return this; } + @CanIgnoreReturnValue public Builder setAllLeasesParameters() { path = ALL_LEASES_PATH; return this; diff --git a/java/com/google/android/libraries/mobiledatadownload/file/backends/ContentResolverBackend.java b/java/com/google/android/libraries/mobiledatadownload/file/backends/ContentResolverBackend.java index 5c31747..d5e8da9 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/backends/ContentResolverBackend.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/backends/ContentResolverBackend.java @@ -22,6 +22,7 @@ import android.util.Pair; import com.google.android.libraries.mobiledatadownload.file.common.MalformedUriException; import com.google.android.libraries.mobiledatadownload.file.common.internal.Preconditions; import com.google.android.libraries.mobiledatadownload.file.spi.Backend; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.io.Closeable; import java.io.IOException; import java.io.InputStream; @@ -39,7 +40,7 @@ import java.io.InputStream; * * <p>NOTE: In most cases, you'll want to use the GmsClientBackend for accessing files from GMS * core. This backend is used to access files from other Apps. Since there are possible security - * concerns with doing so, ContentResolverBackend is restricted to the "content_resolver_allowlist". + * concerns with doing so, ContentResolverBackend is restricted to the "content_resolver_whitelist". * See <internal> for more information. */ public final class ContentResolverBackend implements Backend { @@ -67,6 +68,7 @@ public final class ContentResolverBackend implements Backend { * Tells whether this backend is expected to be embedded in another backend. If so, rewrites the * scheme to "content"; if not, requires that the scheme be "content". */ + @CanIgnoreReturnValue public Builder setEmbedded(boolean isEmbedded) { this.isEmbedded = isEmbedded; return this; diff --git a/java/com/google/android/libraries/mobiledatadownload/file/backends/FileUri.java b/java/com/google/android/libraries/mobiledatadownload/file/backends/FileUri.java index 58f9508..5a00ec9 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/backends/FileUri.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/backends/FileUri.java @@ -19,6 +19,7 @@ import android.net.Uri; import com.google.android.libraries.mobiledatadownload.file.common.internal.LiteTransformFragments; import com.google.android.libraries.mobiledatadownload.file.transforms.TransformProtos; import com.google.common.collect.ImmutableList; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import com.google.mobiledatadownload.TransformProto; import java.io.File; @@ -46,21 +47,25 @@ public final class FileUri { private Builder() {} + @CanIgnoreReturnValue public Builder setPath(String path) { uri.path(path); return this; } + @CanIgnoreReturnValue public Builder fromFile(File file) { uri.path(file.getAbsolutePath()); return this; } + @CanIgnoreReturnValue public Builder appendPath(String segment) { uri.appendPath(segment); return this; } + @CanIgnoreReturnValue public Builder withTransform(TransformProto.Transform spec) { encodedSpecs.add(TransformProtos.toEncodedSpec(spec)); return this; diff --git a/java/com/google/android/libraries/mobiledatadownload/file/backends/FileUriAdapter.java b/java/com/google/android/libraries/mobiledatadownload/file/backends/FileUriAdapter.java index ea73f06..625e1c1 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/backends/FileUriAdapter.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/backends/FileUriAdapter.java @@ -22,7 +22,7 @@ import java.io.File; /** * Adapter for converting "file:" URIs into java.io.File. This is considered dangerous since it - * ignores parts of the Uri at the caller's peril, and thus is only available to allowlisted clients + * ignores parts of the Uri at the caller's peril, and thus is only available to whitelisted clients * (mostly internal). */ public class FileUriAdapter implements UriAdapter { diff --git a/java/com/google/android/libraries/mobiledatadownload/file/backends/GenericUriAdapter.java b/java/com/google/android/libraries/mobiledatadownload/file/backends/GenericUriAdapter.java index 5e244f2..c9d85c6 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/backends/GenericUriAdapter.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/backends/GenericUriAdapter.java @@ -22,7 +22,7 @@ import java.io.File; /** * Adapter for converting "android:" URIs into java.io.File. This is considered dangerous since it - * ignores parts of the Uri at the caller's peril, and thus is only available to allowlisted clients + * ignores parts of the Uri at the caller's peril, and thus is only available to whitelisted clients * (mostly internal). */ public final class GenericUriAdapter implements UriAdapter { diff --git a/java/com/google/android/libraries/mobiledatadownload/file/backends/MemoryUri.java b/java/com/google/android/libraries/mobiledatadownload/file/backends/MemoryUri.java index 2d69d72..7f96b32 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/backends/MemoryUri.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/backends/MemoryUri.java @@ -21,6 +21,7 @@ import com.google.android.libraries.mobiledatadownload.file.common.MalformedUriE import com.google.android.libraries.mobiledatadownload.file.common.internal.LiteTransformFragments; import com.google.android.libraries.mobiledatadownload.file.transforms.TransformProtos; import com.google.common.collect.ImmutableList; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import com.google.mobiledatadownload.TransformProto; /** @@ -45,6 +46,7 @@ public final class MemoryUri { private Builder() {} /** Sets the non-empty key to be used as a file identifier. */ + @CanIgnoreReturnValue public Builder setKey(String key) { this.key = key; return this; @@ -53,6 +55,7 @@ public final class MemoryUri { /** * Appends a transform to the Uri. Calling twice with the same transform replaces the original. */ + @CanIgnoreReturnValue public Builder withTransform(TransformProto.Transform spec) { encodedSpecs.add(TransformProtos.toEncodedSpec(spec)); return this; diff --git a/java/com/google/android/libraries/mobiledatadownload/file/backends/UriAdapter.java b/java/com/google/android/libraries/mobiledatadownload/file/backends/UriAdapter.java index e9a73aa..029893b 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/backends/UriAdapter.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/backends/UriAdapter.java @@ -22,7 +22,7 @@ import java.io.File; /** * Interface for converting certain URI schemes to raw java.io.Files. Implementations of this are * considered dangerous since they ignore parts of the URI incluging the fragment at the caller's - * peril, and thus is only available to allowlisted clients (mostly internal). + * peril, and thus is only available to whitelisted clients (mostly internal). */ interface UriAdapter { /** diff --git a/java/com/google/android/libraries/mobiledatadownload/file/behaviors/BUILD b/java/com/google/android/libraries/mobiledatadownload/file/behaviors/BUILD index 5d2195a..977f913 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/behaviors/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/file/behaviors/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], diff --git a/java/com/google/android/libraries/mobiledatadownload/file/common/BUILD b/java/com/google/android/libraries/mobiledatadownload/file/common/BUILD index c183243..7b1fb08 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/common/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/file/common/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], @@ -39,6 +40,9 @@ android_library( "UnsupportedFileStorageOperation.java", ], deps = [ + "//java/com/google/android/libraries/mobiledatadownload/file/common/internal:exponential_backoff_iterator", + "@com_google_guava_guava", + "@com_google_errorprone_error_prone_annotations", "@com_google_code_findbugs_jsr305", # NOTE: dependency of gmscore client lib <internal> ], @@ -53,6 +57,7 @@ android_library( deps = [ "//java/com/google/android/libraries/mobiledatadownload/file/common/internal:charsets", "//java/com/google/android/libraries/mobiledatadownload/file/common/internal:preconditions", + "@com_google_errorprone_error_prone_annotations", "@com_google_code_findbugs_jsr305", # NOTE: dependency of gmscore client lib <internal> ], diff --git a/java/com/google/android/libraries/mobiledatadownload/file/common/Fragment.java b/java/com/google/android/libraries/mobiledatadownload/file/common/Fragment.java index 5d02885..62e78e3 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/common/Fragment.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/common/Fragment.java @@ -20,6 +20,7 @@ import static com.google.android.libraries.mobiledatadownload.file.common.intern import android.net.Uri; import android.text.TextUtils; import com.google.android.libraries.mobiledatadownload.file.common.internal.Preconditions; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.io.UnsupportedEncodingException; import java.net.URLDecoder; import java.net.URLEncoder; @@ -128,12 +129,14 @@ public final class Fragment { } /** Adds a param. If a param with same key already exists, this replaces it. */ + @CanIgnoreReturnValue public Builder addParam(Param param) { addParam(param.toBuilder()); return this; } /** Adds a param. If a param with the same key already exist, this replaces it. */ + @CanIgnoreReturnValue public Builder addParam(Param.Builder param) { for (int i = 0; i < params.size(); i++) { if (params.get(i).key.equals(param.key)) { @@ -146,6 +149,7 @@ public final class Fragment { } /** Adds a simple param with no value. */ + @CanIgnoreReturnValue public Builder addParam(String key) { return addParam(Param.builder(key)); } @@ -266,6 +270,7 @@ public final class Fragment { * Adds a value to this param. If a value already exists with the same name, this will replace * it. */ + @CanIgnoreReturnValue public Builder addValue(ParamValue value) { addValue(value.toBuilder()); return this; @@ -275,6 +280,7 @@ public final class Fragment { * Adds a value to this param. If a value already exists with the same name, this will replace * it. */ + @CanIgnoreReturnValue public Builder addValue(ParamValue.Builder value) { for (int i = 0; i < values.size(); i++) { if (values.get(i).name.equals(value.name)) { @@ -287,6 +293,7 @@ public final class Fragment { } /** Adds a value that has no subparams. Also replaces existing value if present. */ + @CanIgnoreReturnValue public Builder addValue(String name) { return addValue(new ParamValue.Builder(name, null)); } @@ -434,6 +441,7 @@ public final class Fragment { * @param subparam * @return The subparam or null if not found. */ + @CanIgnoreReturnValue public Builder addSubParam(SubParam subparam) { for (int i = 0; i < subparams.size(); i++) { if (subparams.get(i).key.equals(subparam.key)) { @@ -452,6 +460,7 @@ public final class Fragment { * @param key The subparam key. * @param value The subparam value. */ + @CanIgnoreReturnValue public Builder addSubParam(String key, String value) { return addSubParam(new SubParam(key, value)); } diff --git a/java/com/google/android/libraries/mobiledatadownload/file/common/LockScope.java b/java/com/google/android/libraries/mobiledatadownload/file/common/LockScope.java index 00e68e0..2c68111 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/common/LockScope.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/common/LockScope.java @@ -16,11 +16,15 @@ package com.google.android.libraries.mobiledatadownload.file.common; import android.net.Uri; +import android.os.SystemClock; +import com.google.android.libraries.mobiledatadownload.file.common.internal.ExponentialBackoffIterator; +import com.google.common.base.Optional; import java.io.Closeable; import java.io.IOException; import java.io.InterruptedIOException; import java.nio.channels.FileChannel; import java.nio.channels.FileLock; +import java.util.Iterator; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.Semaphore; @@ -44,6 +48,17 @@ import javax.annotation.Nullable; */ public final class LockScope { + // NOTE(b/254717998): due to the design of Linux file lock, it would throw an IOException with + // "Resource deadlock would occur" as false alarms in some use cases. As the fix, in the case of + // such failures where error message matches with {@link DEADLOCK_ERROR_MESSAGE}, we first do + // exponential backoff to retry to get file lock, and then retry every second until it succeeds. + private static final String DEADLOCK_ERROR_MESSAGE = "Resource deadlock would occur"; + + // Wait for 10 ms if need to retry file locking for the first time + private static final Long INITIAL_WAIT_MILLIS = 10L; + // Wait for 1 minute if need to retry file locking with the upper bound wait time + private static final Long UPPER_BOUND_WAIT_MILLIS = 60_000L; + @Nullable private final ConcurrentMap<String, Semaphore> lockMap; /** @@ -109,8 +124,29 @@ public final class LockScope { /** Acquires a cross-process lock on {@code channel}. This blocks until the lock is obtained. */ public Lock fileLock(FileChannel channel, boolean shared) throws IOException { - FileLock lock = channel.lock(0L /* position */, Long.MAX_VALUE /* size */, shared); - return new FileLockImpl(lock); + Optional<FileLockImpl> fileLock = fileLockAndThrowIfNotDeadlock(channel, shared); + if (fileLock.isPresent()) { + return fileLock.get(); + } + + // if an IOException with "Resource deadlock would occur" is thrown from getting file lock, we + // will keep retrying until it succeeds + Iterator<Long> retryIterator = + ExponentialBackoffIterator.create(INITIAL_WAIT_MILLIS, UPPER_BOUND_WAIT_MILLIS); + // TODO(b/254717998): error after a number of retry attempts if needed. And possibly detect real + // deadlocks in client use cases. + while (retryIterator.hasNext()) { + long waitTime = retryIterator.next(); + SystemClock.sleep(waitTime); + + Optional<FileLockImpl> fileLockImpl = fileLockAndThrowIfNotDeadlock(channel, shared); + if (fileLockImpl.isPresent()) { + return fileLockImpl.get(); + } + } + // should never reach here because ExponentialBackoffIterator guarantees it will always hasNext, + // make builder happy + throw new IllegalStateException("should have gotten file lock and returned"); } /** @@ -136,38 +172,36 @@ public final class LockScope { return lockMap != null; } + /** + * Returns the file lock got from given channel. If gets an IOException with {@link + * DEADLOCK_ERROR_MESSAGE}, returns empty; otherwise throws the error. + */ + private static Optional<FileLockImpl> fileLockAndThrowIfNotDeadlock( + FileChannel channel, boolean shared) throws IOException { + try { + FileLock lock = channel.lock(0L /* position */, Long.MAX_VALUE /* size */, shared); + return Optional.of(new FileLockImpl(lock)); + } catch (IOException ex) { + if (!ex.getMessage().contains(DEADLOCK_ERROR_MESSAGE)) { + throw ex; + } + return Optional.absent(); + } + } + private static class FileLockImpl implements Lock { private FileLock fileLock; - private Semaphore semaphore; public FileLockImpl(FileLock fileLock) { this.fileLock = fileLock; - this.semaphore = null; - } - - /** - * @deprecated Prefer the single-argument {@code FileLockImpl(FileLock)}. - */ - @Deprecated - public FileLockImpl(FileLock fileLock, Semaphore semaphore) { - this.fileLock = fileLock; - this.semaphore = semaphore; } @Override public void release() throws IOException { - // The semaphore guards access to the fileLock, so fileLock *must* be released first. - try { - if (fileLock != null) { - fileLock.release(); - fileLock = null; - } - } finally { - if (semaphore != null) { - semaphore.release(); - semaphore = null; - } + if (fileLock != null) { + fileLock.release(); + fileLock = null; } } diff --git a/java/com/google/android/libraries/mobiledatadownload/file/common/internal/BUILD b/java/com/google/android/libraries/mobiledatadownload/file/common/internal/BUILD index 0ffd400..adc1b24 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/common/internal/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/file/common/internal/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = ["//:__subpackages__"], licenses = ["notice"], ) @@ -77,3 +78,9 @@ android_library( "@com_google_guava_guava", ], ) + +android_library( + name = "exponential_backoff_iterator", + srcs = ["ExponentialBackoffIterator.java"], + deps = ["@com_google_guava_guava"], +) diff --git a/java/com/google/android/libraries/mobiledatadownload/file/common/internal/ExponentialBackoffIterator.java b/java/com/google/android/libraries/mobiledatadownload/file/common/internal/ExponentialBackoffIterator.java new file mode 100644 index 0000000..574ff22 --- /dev/null +++ b/java/com/google/android/libraries/mobiledatadownload/file/common/internal/ExponentialBackoffIterator.java @@ -0,0 +1,67 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.libraries.mobiledatadownload.file.common.internal; + +import static com.google.common.base.Preconditions.checkArgument; + +import java.util.Iterator; + +/** + * Provide an iterator for a infinite sequence of exponential backoffs. The sequence begins with the + * provided initial backoff and doubles up everytime a new backoff is acceessed, after the backoff + * reaches the upper bound, it always returns the upper bound backoff. + */ +public final class ExponentialBackoffIterator implements Iterator<Long> { + + /** + * Create a new instance with positive integers. {@code upperBoundBackoff} should be no less than + * {@code initialBackoff}. + */ + public static ExponentialBackoffIterator create(long initialBackoff, long upperBoundBackoff) { + checkArgument(initialBackoff > 0); + checkArgument(upperBoundBackoff >= initialBackoff); + return new ExponentialBackoffIterator(initialBackoff, upperBoundBackoff); + } + + private long nextBackoff; + private final long upperBoundBackoff; + + private ExponentialBackoffIterator(long initialBackoff, long upperBoundBackoff) { + this.nextBackoff = initialBackoff; + this.upperBoundBackoff = upperBoundBackoff; + } + + /** + * Returns if the iterator has the next delay. It always returns true because the sequence is + * infinite. + */ + @Override + public boolean hasNext() { + return true; + } + + /** Returns the next delay. */ + @Override + public Long next() { + long currentBackoff = nextBackoff; + if (nextBackoff >= upperBoundBackoff / 2) { + nextBackoff = upperBoundBackoff; + } else { + nextBackoff *= 2; + } + return currentBackoff; + } +} diff --git a/java/com/google/android/libraries/mobiledatadownload/file/common/testing/BUILD b/java/com/google/android/libraries/mobiledatadownload/file/common/testing/BUILD index 410729f..9085543 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/common/testing/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/file/common/testing/BUILD @@ -15,6 +15,7 @@ load("//tools/build_defs/testing:bzl_library.bzl", "bzl_library") load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_testonly = 1, default_visibility = ["//:__subpackages__"], licenses = ["notice"], @@ -32,6 +33,7 @@ java_library( ], deps = [ "@android_sdk_linux", + "@com_google_errorprone_error_prone_annotations", "@robolectric", ], ) @@ -78,6 +80,7 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/file/common/internal:charsets", "//java/com/google/android/libraries/mobiledatadownload/file/openers:stream", "//java/com/google/android/libraries/mobiledatadownload/file/spi", + "@com_google_errorprone_error_prone_annotations", "@com_google_guava_guava", "@junit", "@truth", @@ -102,6 +105,7 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/file/common:fragment", "//java/com/google/android/libraries/mobiledatadownload/file/common/internal:forwarding_stream", "//java/com/google/android/libraries/mobiledatadownload/file/spi", + "@com_google_errorprone_error_prone_annotations", "@com_google_guava_guava", "@junit", "@truth", @@ -120,18 +124,20 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/file/common/internal:forwarding_stream", "//java/com/google/android/libraries/mobiledatadownload/file/spi", "@com_google_code_findbugs_jsr305", + "@com_google_errorprone_error_prone_annotations", "@org_checkerframework_qual", ], ) java_lite_proto_library( name = "test_message_java_proto_lite", - deps = [":test_message_proto"], + deps = ["//java/com/google/android/libraries/mobiledatadownload/file/common/testing:test_message_proto"], ) proto_library( name = "test_message_proto", srcs = ["test_message.proto"], + deps = ["@com_google_protobuf//:timestamp_proto"], ) bzl_library( diff --git a/java/com/google/android/libraries/mobiledatadownload/file/common/testing/BackendTestBase.java b/java/com/google/android/libraries/mobiledatadownload/file/common/testing/BackendTestBase.java index ade1277..5e2af9c 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/common/testing/BackendTestBase.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/common/testing/BackendTestBase.java @@ -266,7 +266,7 @@ public abstract class BackendTestBase { try (OutputStream stream = backend().openForAppend(uri)) { assertThat(stream).isInstanceOf(FileConvertible.class); File file = ((FileConvertible) stream).toFile(); - writeFileToSink(new FileOutputStream(file, /* append = */ true), TEST_CONTENT); + writeFileToSink(new FileOutputStream(file, /* append= */ true), TEST_CONTENT); } assertThat(readFileInBytes(storage(), uri)) .isEqualTo(Bytes.concat(OTHER_CONTENT, TEST_CONTENT)); diff --git a/java/com/google/android/libraries/mobiledatadownload/file/common/testing/ExceptionTesting.java b/java/com/google/android/libraries/mobiledatadownload/file/common/testing/ExceptionTesting.java index 77557bb..e09768d 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/common/testing/ExceptionTesting.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/common/testing/ExceptionTesting.java @@ -23,6 +23,7 @@ import java.util.concurrent.Future; /** Common helper utilities for testing exceptions. */ public final class ExceptionTesting { + public static <T extends Throwable> T assertThrowsAsync( Class<T> throwableType, Future<?> future) { ExecutionException executionException = assertThrows(ExecutionException.class, future::get); diff --git a/java/com/google/android/libraries/mobiledatadownload/file/common/testing/FakeFileBackend.java b/java/com/google/android/libraries/mobiledatadownload/file/common/testing/FakeFileBackend.java index 2581c9a..83b883d 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/common/testing/FakeFileBackend.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/common/testing/FakeFileBackend.java @@ -22,6 +22,7 @@ import com.google.android.libraries.mobiledatadownload.file.common.GcParam; import com.google.android.libraries.mobiledatadownload.file.common.LockScope; import com.google.android.libraries.mobiledatadownload.file.common.internal.ForwardingOutputStream; import com.google.android.libraries.mobiledatadownload.file.spi.Backend; +import com.google.errorprone.annotations.concurrent.GuardedBy; import java.io.Closeable; import java.io.File; import java.io.IOException; @@ -30,7 +31,6 @@ import java.io.OutputStream; import java.util.EnumMap; import java.util.Map; import java.util.concurrent.CountDownLatch; -import javax.annotation.concurrent.GuardedBy; import org.checkerframework.checker.nullness.qual.Nullable; /** A Fake Backend for testing. It allows overriding certain behavior. */ @@ -53,6 +53,7 @@ public class FakeFileBackend implements Backend { QUERY, // exists, isDirectory, fileSize, children, getGcParam, toFile MANAGE, // delete, rename, createDirectory, setGcParam WRITE_STREAM, // openForWrite/Append return streams that fail + EXISTS, // exists } /** @@ -212,6 +213,7 @@ public class FakeFileBackend implements Backend { @Override public boolean exists(Uri uri) throws IOException { + throwOrSuspendIf(OperationType.EXISTS); throwOrSuspendIf(OperationType.QUERY); return delegate.exists(uri); } diff --git a/java/com/google/android/libraries/mobiledatadownload/file/common/testing/FileDescriptorLeakChecker.java b/java/com/google/android/libraries/mobiledatadownload/file/common/testing/FileDescriptorLeakChecker.java index 981de70..5bb36b8 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/common/testing/FileDescriptorLeakChecker.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/common/testing/FileDescriptorLeakChecker.java @@ -25,6 +25,7 @@ import android.util.Log; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.io.BufferedReader; import java.io.File; import java.io.IOException; @@ -56,11 +57,13 @@ public class FileDescriptorLeakChecker implements MethodRule { * * @param processesToMonitor The names of the processes to monitor. */ + @CanIgnoreReturnValue public FileDescriptorLeakChecker withProcessesToMonitor(List<String> processesToMonitor) { this.processesToMonitor = processesToMonitor; return this; } + @CanIgnoreReturnValue public FileDescriptorLeakChecker withFilesToMonitor(List<String> filesToMonitor) { this.filesToMonitor = filesToMonitor; return this; @@ -72,6 +75,7 @@ public class FileDescriptorLeakChecker implements MethodRule { * * @param msToWait Milliseconds the FileDescriptorLeakChecker needs to wait before retrying. */ + @CanIgnoreReturnValue public FileDescriptorLeakChecker withWaitIfFails(long msToWait) { this.msToWait = msToWait; return this; diff --git a/java/com/google/android/libraries/mobiledatadownload/file/common/testing/build_defs.bzl b/java/com/google/android/libraries/mobiledatadownload/file/common/testing/build_defs.bzl index eef907a..a628f20 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/common/testing/build_defs.bzl +++ b/java/com/google/android/libraries/mobiledatadownload/file/common/testing/build_defs.bzl @@ -36,9 +36,11 @@ _DEFAULT_TARGET_APIS = [ _GENERIC_DEVICE_FMT = "generic_phone:google_%s_x86_gms_stable" _EMULATOR_DIRECTORY = "//tools/android/emulated_devices/%s" +# TODO: Consider adding the local test for additional target devices def android_test_multi_api( name, target_apis = _DEFAULT_TARGET_APIS, + additional_targets = {}, **kwargs): """Simple definition for running an android_application_test against multiple API levels. @@ -56,6 +58,8 @@ def android_test_multi_api( name: Name of the "default" test target and used to derive subtargets target_apis: List of Android API levels as strings for which a test should be generated. If unspecified, 16-28 excluding 20 are used. + additional_targets: Map of additional target devices other than automatically generated ones, + with keys as target device names and values as emulator directory. **kwargs: Parameters that are passed to the generated android_application_test rule. """ @@ -67,6 +71,7 @@ def android_test_multi_api( android_test_multi_device( name = name, target_devices = target_devices, + additional_targets_map = additional_targets, **kwargs ) @@ -84,6 +89,7 @@ def android_test_multi_api( def android_test_multi_device( name, target_devices, + additional_targets_map, **kwargs): """Simple definition for running an android_application_test against multiple devices. @@ -91,14 +97,35 @@ def android_test_multi_device( name: Name of the test rule; we generate several sub-targets based on API. target_devices: List of emulators as strings for which a test should be generated. + additional_targets_map: Map of additional target devices other than automatically generated + ones, with keys as target device names and values as emulator directory. **kwargs: Parameters that are passed to the generated android_application_test rule. """ for target_device in target_devices: - sanitized_device = target_device.replace(":", "_") # ":" is invalid - test_name = "%s_%s" % (name, sanitized_device) - test_device = _EMULATOR_DIRECTORY % (target_device) - android_application_test( - name = test_name, - target_devices = [test_device], - **kwargs - ) + android_test_single_device(name, target_device, _EMULATOR_DIRECTORY, **kwargs) + for additional_target, emulator_dir in additional_targets_map.items(): + if not emulator_dir.endswith("%s"): + emulator_dir += "%s" + android_test_single_device(name, additional_target, emulator_dir, **kwargs) + +def android_test_single_device( + name, + target_device, + emulator_directory, + **kwargs): + """Simple definition for running an android_application_test against single device. + + Args: + name: Name of the test rule; we generate several sub-targets based on API. + target_device: An emulator as a string for which a test should be generated. + emulator_directory: A string representing the diretory where the emulator locates at. + **kwargs: Parameters that are passed to the generated android_application_test rule. + """ + sanitized_device = target_device.replace(":", "_") # ":" is invalid + test_name = "%s_%s" % (name, sanitized_device) + test_device = emulator_directory % (target_device) + android_application_test( + name = test_name, + target_devices = [test_device], + **kwargs + ) diff --git a/java/com/google/android/libraries/mobiledatadownload/file/common/testing/test_message.proto b/java/com/google/android/libraries/mobiledatadownload/file/common/testing/test_message.proto index 4611f9c..8eeed3e 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/common/testing/test_message.proto +++ b/java/com/google/android/libraries/mobiledatadownload/file/common/testing/test_message.proto @@ -16,6 +16,8 @@ syntax = "proto2"; package google.android.storage.common; +import "google/protobuf/timestamp.proto"; + option java_package = "com.google.mobiledatadownload.testing"; option java_outer_classname = "TestMessageProto"; @@ -24,6 +26,7 @@ message FooProto { optional bool boolean = 2; optional int32 integer = 3; optional bytes bytes = 4; + optional google.protobuf.Timestamp timestamp = 5; } message BarProto { diff --git a/java/com/google/android/libraries/mobiledatadownload/file/integration/downloader/BUILD b/java/com/google/android/libraries/mobiledatadownload/file/integration/downloader/BUILD index 42e34fa..e2d867d 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/integration/downloader/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/file/integration/downloader/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], @@ -30,6 +31,7 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/file", "//java/com/google/android/libraries/mobiledatadownload/file/common", "//java/com/google/android/libraries/mobiledatadownload/file/openers:random_access_file", + "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", "@com_google_guava_guava", "@downloader", ], diff --git a/java/com/google/android/libraries/mobiledatadownload/file/integration/downloader/DownloadDestinationOpener.java b/java/com/google/android/libraries/mobiledatadownload/file/integration/downloader/DownloadDestinationOpener.java index 855ec85..1a3322a 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/integration/downloader/DownloadDestinationOpener.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/integration/downloader/DownloadDestinationOpener.java @@ -72,7 +72,7 @@ public final class DownloadDestinationOpener implements Opener<DownloadDestinati private final SynchronousFileStorage fileStorage; private DownloadDestinationImpl( - Uri onDeviceUri, SynchronousFileStorage fileStorage, DownloadMetadataStore metadataStore) { + Uri onDeviceUri, SynchronousFileStorage fileStorage, DownloadMetadataStore metadataStore) { this.onDeviceUri = onDeviceUri; this.metadataStore = metadataStore; this.fileStorage = fileStorage; @@ -87,7 +87,7 @@ public final class DownloadDestinationOpener implements Opener<DownloadDestinati public DownloadMetadata readMetadata() throws IOException { synchronized (lock) { Optional<DownloadMetadata> existingMetadata = - blockingGet(metadataStore.read(onDeviceUri), "Failed to read metadata."); + blockingGet(metadataStore.read(onDeviceUri), "Failed to read metadata."); // Return existing metadata, or a new instance. return existingMetadata.or(DownloadMetadata::create); @@ -96,16 +96,16 @@ public final class DownloadDestinationOpener implements Opener<DownloadDestinati @Override public WritableByteChannel openByteChannel(long byteOffset, DownloadMetadata metadata) - throws IOException { + throws IOException { // Ensure that metadata is not null checkArgument(metadata != null, "Received null metadata to store"); // Check that offset is in range long fileSize = numExistingBytes(); checkArgument( - byteOffset >= 0 && byteOffset <= fileSize, - "Offset for write (%s) out of range of existing file size (%s bytes)", - byteOffset, - fileSize); + byteOffset >= 0 && byteOffset <= fileSize, + "Offset for write (%s) out of range of existing file size (%s bytes)", + byteOffset, + fileSize); synchronized (lock) { // Update metadata first. @@ -113,8 +113,8 @@ public final class DownloadDestinationOpener implements Opener<DownloadDestinati // Use ReleasableResource to ensure channel is setup properly before returning it. try (ReleasableResource<RandomAccessFile> file = - ReleasableResource.create( - fileStorage.open(onDeviceUri, RandomAccessFileOpener.createForReadWrite()))) { + ReleasableResource.create( + fileStorage.open(onDeviceUri, RandomAccessFileOpener.createForReadWrite()))) { // Get channel and seek to correct offset. FileChannel channel = file.get().getChannel(); channel.position(byteOffset); @@ -143,7 +143,7 @@ public final class DownloadDestinationOpener implements Opener<DownloadDestinati * <p>Exceptions due to an async call failure are handled and wrapped in an IOException. */ private static <V> V blockingGet(ListenableFuture<V> future, String errorMessage) - throws IOException { + throws IOException { try { return future.get(TIMEOUT_MS, MILLISECONDS); } catch (InterruptedException e) { @@ -167,17 +167,17 @@ public final class DownloadDestinationOpener implements Opener<DownloadDestinati public DownloadDestination open(OpenContext openContext) throws IOException { if (openContext.hasTransforms()) { throw new UnsupportedFileStorageOperation( - "Transforms are not supported by this Opener: " + openContext.originalUri()); + "Transforms are not supported by this Opener: " + openContext.originalUri()); } // Check whether or not the file uri is a directory. if (openContext.storage().isDirectory(openContext.originalUri())) { throw new IOException( - new IllegalArgumentException("Requested file download is already a directory.")); + new IllegalArgumentException("Requested file download is already a directory.")); } return new DownloadDestinationImpl( - openContext.originalUri(), openContext.storage(), metadataStore); + openContext.originalUri(), openContext.storage(), metadataStore); } public static DownloadDestinationOpener create(DownloadMetadataStore metadataStore) { diff --git a/java/com/google/android/libraries/mobiledatadownload/file/monitors/BUILD b/java/com/google/android/libraries/mobiledatadownload/file/monitors/BUILD index 11f0cbd..2b65923 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/monitors/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/file/monitors/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], @@ -23,7 +24,5 @@ package( android_library( name = "monitors", srcs = ["ByteCountingOutputMonitor.java"], - deps = [ - "//java/com/google/android/libraries/mobiledatadownload/file/spi", - ], + deps = ["//java/com/google/android/libraries/mobiledatadownload/file/spi"], ) diff --git a/java/com/google/android/libraries/mobiledatadownload/file/openers/AppendStreamOpener.java b/java/com/google/android/libraries/mobiledatadownload/file/openers/AppendStreamOpener.java index bff5543..bfb561d 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/openers/AppendStreamOpener.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/openers/AppendStreamOpener.java @@ -18,6 +18,7 @@ package com.google.android.libraries.mobiledatadownload.file.openers; import com.google.android.libraries.mobiledatadownload.file.Behavior; import com.google.android.libraries.mobiledatadownload.file.OpenContext; import com.google.android.libraries.mobiledatadownload.file.Opener; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.io.IOException; import java.io.OutputStream; import java.util.List; @@ -37,6 +38,7 @@ public final class AppendStreamOpener implements Opener<OutputStream> { * Supports adding options to writes. For example, SyncBehavior will force data to be flushed and * durably persisted. */ + @CanIgnoreReturnValue public AppendStreamOpener withBehaviors(Behavior... behaviors) { this.behaviors = behaviors; return this; diff --git a/java/com/google/android/libraries/mobiledatadownload/file/openers/BUILD b/java/com/google/android/libraries/mobiledatadownload/file/openers/BUILD index 8511d59..186c80c 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/openers/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/file/openers/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], @@ -58,6 +59,7 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/file/common", "@androidx_annotation_annotation", # buildcleaner: keep "@com_google_code_findbugs_jsr305", + "@com_google_errorprone_error_prone_annotations", ], ) @@ -69,6 +71,7 @@ android_library( ":scratch", "//java/com/google/android/libraries/mobiledatadownload/file", "@com_google_code_findbugs_jsr305", + "@com_google_errorprone_error_prone_annotations", ], ) @@ -103,6 +106,7 @@ android_library( ":scratch", ":stream", "//java/com/google/android/libraries/mobiledatadownload/file", + "@com_google_errorprone_error_prone_annotations", "@com_google_protobuf//:protobuf_lite", ], ) @@ -117,6 +121,7 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/file", "//java/com/google/android/libraries/mobiledatadownload/file/common", "@com_google_code_findbugs_jsr305", + "@com_google_errorprone_error_prone_annotations", ], ) @@ -124,8 +129,10 @@ android_library( name = "recursive_delete", srcs = ["RecursiveDeleteOpener.java"], deps = [ + ":file", "//java/com/google/android/libraries/mobiledatadownload/file", "//java/com/google/android/libraries/mobiledatadownload/file/common/internal:exceptions", + "@com_google_errorprone_error_prone_annotations", ], ) @@ -145,7 +152,10 @@ android_library( "ReadStreamOpener.java", "WriteStreamOpener.java", ], - deps = ["//java/com/google/android/libraries/mobiledatadownload/file"], + deps = [ + "//java/com/google/android/libraries/mobiledatadownload/file", + "@com_google_errorprone_error_prone_annotations", + ], ) android_library( @@ -171,6 +181,7 @@ android_library( ":stream", "//java/com/google/android/libraries/mobiledatadownload/file", "@com_google_code_findbugs_jsr305", + "@com_google_errorprone_error_prone_annotations", "@com_google_guava_guava", ], ) @@ -185,6 +196,7 @@ android_library( ":bytes", "//java/com/google/android/libraries/mobiledatadownload/file", "//java/com/google/android/libraries/mobiledatadownload/file/common/internal:charsets", + "@com_google_errorprone_error_prone_annotations", ], ) @@ -198,6 +210,7 @@ android_library( ":stream", "//java/com/google/android/libraries/mobiledatadownload/file", "//java/com/google/android/libraries/mobiledatadownload/file/common", + "@com_google_errorprone_error_prone_annotations", "@com_google_guava_guava", ], ) diff --git a/java/com/google/android/libraries/mobiledatadownload/file/openers/LockFileOpener.java b/java/com/google/android/libraries/mobiledatadownload/file/openers/LockFileOpener.java index c608ec2..9cfcb67 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/openers/LockFileOpener.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/openers/LockFileOpener.java @@ -20,6 +20,7 @@ import com.google.android.libraries.mobiledatadownload.file.OpenContext; import com.google.android.libraries.mobiledatadownload.file.Opener; import com.google.android.libraries.mobiledatadownload.file.common.FileChannelConvertible; import com.google.android.libraries.mobiledatadownload.file.common.ReleasableResource; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.io.Closeable; import java.io.IOException; import java.io.RandomAccessFile; @@ -42,7 +43,7 @@ import javax.annotation.Nullable; */ public final class LockFileOpener implements Opener<Closeable> { - private static final String LOCK_SUFFIX = ".lock"; + public static final String LOCK_SUFFIX = ".lock"; private final boolean shared; private final boolean readOnly; @@ -84,6 +85,7 @@ public final class LockFileOpener implements Opener<Closeable> { * If enabled and the lock cannot be acquired immediately, {@link #open} will return {@code null} * instead of waiting until the lock can be acquired. */ + @CanIgnoreReturnValue public LockFileOpener nonBlocking(boolean isNonBlocking) { this.isNonBlocking = isNonBlocking; return this; diff --git a/java/com/google/android/libraries/mobiledatadownload/file/openers/RandomAccessFileOpener.java b/java/com/google/android/libraries/mobiledatadownload/file/openers/RandomAccessFileOpener.java index 25e1839..6589b07 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/openers/RandomAccessFileOpener.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/openers/RandomAccessFileOpener.java @@ -36,7 +36,7 @@ public final class RandomAccessFileOpener implements Opener<RandomAccessFile> { } public static RandomAccessFileOpener createForRead() { - return new RandomAccessFileOpener(/*writeSupport=*/ false); + return new RandomAccessFileOpener(/* writeSupport= */ false); } /** @@ -44,7 +44,7 @@ public final class RandomAccessFileOpener implements Opener<RandomAccessFile> { * parent directories do not exist, they will be created. */ public static RandomAccessFileOpener createForReadWrite() { - return new RandomAccessFileOpener(/*writeSupport=*/ true); + return new RandomAccessFileOpener(/* writeSupport= */ true); } @Override diff --git a/java/com/google/android/libraries/mobiledatadownload/file/openers/ReadFileOpener.java b/java/com/google/android/libraries/mobiledatadownload/file/openers/ReadFileOpener.java index a02b9bb..9bb70fa 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/openers/ReadFileOpener.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/openers/ReadFileOpener.java @@ -23,6 +23,7 @@ import com.google.android.libraries.mobiledatadownload.file.Opener; import com.google.android.libraries.mobiledatadownload.file.common.FileConvertible; import com.google.android.libraries.mobiledatadownload.file.common.ReleasableResource; import com.google.android.libraries.mobiledatadownload.file.common.UnsupportedFileStorageOperation; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -88,6 +89,7 @@ public final class ReadFileOpener implements Opener<File> { * @param context Android context for the root directory where fifos are stored. * @return This opener. */ + @CanIgnoreReturnValue public ReadFileOpener withFallbackToPipeUsingExecutor(ExecutorService executor, Context context) { this.executor = executor; this.context = context; @@ -99,6 +101,7 @@ public final class ReadFileOpener implements Opener<File> { * there are any transforms enabled. This is like the {@link UriAdapter} interface, but with more * guard rails to make it safe to expose publicly. */ + @CanIgnoreReturnValue public ReadFileOpener withShortCircuit() { this.shortCircuit = true; return this; diff --git a/java/com/google/android/libraries/mobiledatadownload/file/openers/ReadProtoOpener.java b/java/com/google/android/libraries/mobiledatadownload/file/openers/ReadProtoOpener.java index 9762803..cd8530d 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/openers/ReadProtoOpener.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/openers/ReadProtoOpener.java @@ -17,6 +17,7 @@ package com.google.android.libraries.mobiledatadownload.file.openers; import com.google.android.libraries.mobiledatadownload.file.OpenContext; import com.google.android.libraries.mobiledatadownload.file.Opener; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import com.google.protobuf.ExtensionRegistryLite; import com.google.protobuf.MessageLite; import com.google.protobuf.Parser; @@ -60,6 +61,7 @@ public final class ReadProtoOpener<T extends MessageLite> implements Opener<T> { } /** Adds an extension registry used while parsing the proto. */ + @CanIgnoreReturnValue public ReadProtoOpener<T> withExtensionRegistry(ExtensionRegistryLite registry) { this.registry = registry; return this; diff --git a/java/com/google/android/libraries/mobiledatadownload/file/openers/ReadStreamOpener.java b/java/com/google/android/libraries/mobiledatadownload/file/openers/ReadStreamOpener.java index 94848ee..71dfea6 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/openers/ReadStreamOpener.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/openers/ReadStreamOpener.java @@ -18,6 +18,7 @@ package com.google.android.libraries.mobiledatadownload.file.openers; import com.google.android.libraries.mobiledatadownload.file.Behavior; import com.google.android.libraries.mobiledatadownload.file.OpenContext; import com.google.android.libraries.mobiledatadownload.file.Opener; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; @@ -35,6 +36,7 @@ public final class ReadStreamOpener implements Opener<InputStream> { return new ReadStreamOpener(); } + @CanIgnoreReturnValue public ReadStreamOpener withBehaviors(Behavior... behaviors) { this.behaviors = behaviors; return this; @@ -48,6 +50,7 @@ public final class ReadStreamOpener implements Opener<InputStream> { * * <p>Discouraged: protos (already buffered internally). */ + @CanIgnoreReturnValue public ReadStreamOpener withBufferedIo() { this.bufferIo = true; return this; diff --git a/java/com/google/android/libraries/mobiledatadownload/file/openers/ReadStringOpener.java b/java/com/google/android/libraries/mobiledatadownload/file/openers/ReadStringOpener.java index ff434aa..6f75e2c 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/openers/ReadStringOpener.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/openers/ReadStringOpener.java @@ -18,6 +18,7 @@ package com.google.android.libraries.mobiledatadownload.file.openers; import com.google.android.libraries.mobiledatadownload.file.OpenContext; import com.google.android.libraries.mobiledatadownload.file.Opener; import com.google.android.libraries.mobiledatadownload.file.common.internal.Charsets; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.io.IOException; import java.nio.charset.Charset; @@ -31,6 +32,7 @@ public final class ReadStringOpener implements Opener<String> { return new ReadStringOpener(); } + @CanIgnoreReturnValue public ReadStringOpener withCharset(Charset charset) { this.charset = charset; return this; diff --git a/java/com/google/android/libraries/mobiledatadownload/file/openers/RecursiveDeleteOpener.java b/java/com/google/android/libraries/mobiledatadownload/file/openers/RecursiveDeleteOpener.java index 80fe27f..237def1 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/openers/RecursiveDeleteOpener.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/openers/RecursiveDeleteOpener.java @@ -16,10 +16,17 @@ package com.google.android.libraries.mobiledatadownload.file.openers; import android.net.Uri; +import android.os.Build.VERSION; +import android.os.Build.VERSION_CODES; +import android.system.Os; +import android.system.OsConstants; +import android.system.StructStat; import com.google.android.libraries.mobiledatadownload.file.OpenContext; import com.google.android.libraries.mobiledatadownload.file.Opener; import com.google.android.libraries.mobiledatadownload.file.SynchronousFileStorage; import com.google.android.libraries.mobiledatadownload.file.common.internal.Exceptions; +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -37,8 +44,6 @@ import java.util.List; * * <ul> * <li>Directory tree traversal is not an atomic operation - * <li>There are no special considerations for symlinks, meaning the opener could get caught in a - * recursive directory loop (i.e. a directory that contains a symlink to itself) * </ul> * * <p>Usage: <code> @@ -46,12 +51,18 @@ import java.util.List; * </code> */ public final class RecursiveDeleteOpener implements Opener<Void> { - private RecursiveDeleteOpener() {} + private boolean noFollowLinks; public static RecursiveDeleteOpener create() { return new RecursiveDeleteOpener(); } + @CanIgnoreReturnValue + public RecursiveDeleteOpener withNoFollowLinks() { + this.noFollowLinks = true; + return this; + } + @Override public Void open(OpenContext openContext) throws IOException { List<IOException> exceptions = new ArrayList<>(); @@ -63,12 +74,15 @@ public final class RecursiveDeleteOpener implements Opener<Void> { return null; // for Void return type } - private static void deleteRecursively( + private void deleteRecursively( SynchronousFileStorage storage, Uri uri, List<IOException> exceptions) { + ReadFileOpener readFileOpener = ReadFileOpener.create().withShortCircuit(); try { if (storage.isDirectory(uri)) { - for (Uri child : storage.children(uri)) { - deleteRecursively(storage, child, exceptions); + if (!noFollowLinks || !isSymlink(uri, storage, readFileOpener)) { + for (Uri child : storage.children(uri)) { + deleteRecursively(storage, child, exceptions); + } } storage.deleteDirectory(uri); } else { @@ -78,4 +92,23 @@ public final class RecursiveDeleteOpener implements Opener<Void> { exceptions.add(e); } } + + private static boolean isSymlink( + Uri uri, SynchronousFileStorage storage, ReadFileOpener readFileOpener) { + if (VERSION.SDK_INT >= VERSION_CODES.LOLLIPOP) { + try { + File file = storage.open(uri, readFileOpener); + if (file == null || !file.exists()) { + return false; + } + StructStat stat = Os.lstat(file.getAbsolutePath()); + return (stat.st_mode & OsConstants.S_IFLNK) != 0; + } catch (Exception e) { + // NOTE: this should be ErrnoException, but we're forced to catch Exception to avoid + // breaking lower sdk levels (exceptions aren't stripped from dead code blocks). + return false; + } + } + return false; + } } diff --git a/java/com/google/android/libraries/mobiledatadownload/file/openers/StreamMutationOpener.java b/java/com/google/android/libraries/mobiledatadownload/file/openers/StreamMutationOpener.java index d26538f..d00af35 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/openers/StreamMutationOpener.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/openers/StreamMutationOpener.java @@ -19,6 +19,7 @@ import android.net.Uri; import com.google.android.libraries.mobiledatadownload.file.Behavior; import com.google.android.libraries.mobiledatadownload.file.OpenContext; import com.google.android.libraries.mobiledatadownload.file.Opener; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.io.ByteArrayInputStream; import java.io.Closeable; import java.io.FileNotFoundException; @@ -68,12 +69,14 @@ public final class StreamMutationOpener implements Opener<StreamMutationOpener.M * Enable exclusive locking with this opener. This is useful if multiple processes or threads need * to maintain transactional isolation. */ + @CanIgnoreReturnValue public StreamMutationOpener withLocking(LockFileOpener locking) { this.locking = locking; return this; } /** Apply these behaviors while writing only. */ + @CanIgnoreReturnValue public StreamMutationOpener withBehaviors(Behavior... behaviors) { this.behaviors = behaviors; return this; diff --git a/java/com/google/android/libraries/mobiledatadownload/file/openers/SystemLibraryOpener.java b/java/com/google/android/libraries/mobiledatadownload/file/openers/SystemLibraryOpener.java index 0f90b41..4865549 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/openers/SystemLibraryOpener.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/openers/SystemLibraryOpener.java @@ -21,6 +21,7 @@ import android.util.Base64; import com.google.android.libraries.mobiledatadownload.file.OpenContext; import com.google.android.libraries.mobiledatadownload.file.Opener; import com.google.common.io.ByteStreams; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; @@ -52,6 +53,7 @@ public final class SystemLibraryOpener implements Opener<Void> { private SystemLibraryOpener() {} + @CanIgnoreReturnValue public SystemLibraryOpener withCacheDirectory(Uri dir) { this.cacheDirectory = dir; return this; diff --git a/java/com/google/android/libraries/mobiledatadownload/file/openers/WriteByteArrayOpener.java b/java/com/google/android/libraries/mobiledatadownload/file/openers/WriteByteArrayOpener.java index 4676e7e..932530b 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/openers/WriteByteArrayOpener.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/openers/WriteByteArrayOpener.java @@ -18,6 +18,7 @@ package com.google.android.libraries.mobiledatadownload.file.openers; import com.google.android.libraries.mobiledatadownload.file.Behavior; import com.google.android.libraries.mobiledatadownload.file.OpenContext; import com.google.android.libraries.mobiledatadownload.file.Opener; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.io.IOException; import java.io.OutputStream; @@ -37,6 +38,7 @@ public final class WriteByteArrayOpener implements Opener<Void> { this.bytesToWrite = bytesToWrite; } + @CanIgnoreReturnValue public WriteByteArrayOpener withBehaviors(Behavior... behaviors) { this.behaviors = behaviors; return this; diff --git a/java/com/google/android/libraries/mobiledatadownload/file/openers/WriteFileOpener.java b/java/com/google/android/libraries/mobiledatadownload/file/openers/WriteFileOpener.java index c930f11..2b49af4 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/openers/WriteFileOpener.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/openers/WriteFileOpener.java @@ -22,6 +22,7 @@ import com.google.android.libraries.mobiledatadownload.file.Opener; import com.google.android.libraries.mobiledatadownload.file.common.FileConvertible; import com.google.android.libraries.mobiledatadownload.file.common.ReleasableResource; import com.google.android.libraries.mobiledatadownload.file.openers.WriteFileOpener.FileCloser; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.io.Closeable; import java.io.File; import java.io.FileInputStream; @@ -148,6 +149,7 @@ public final class WriteFileOpener implements Opener<FileCloser> { * @param context Android context for the root directory where fifos are stored. * @return This opener. */ + @CanIgnoreReturnValue public WriteFileOpener withFallbackToPipeUsingExecutor( ExecutorService executor, Context context) { this.executor = executor; diff --git a/java/com/google/android/libraries/mobiledatadownload/file/openers/WriteProtoOpener.java b/java/com/google/android/libraries/mobiledatadownload/file/openers/WriteProtoOpener.java index 81f3eb6..4431ffa 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/openers/WriteProtoOpener.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/openers/WriteProtoOpener.java @@ -19,6 +19,7 @@ import android.net.Uri; import com.google.android.libraries.mobiledatadownload.file.Behavior; import com.google.android.libraries.mobiledatadownload.file.OpenContext; import com.google.android.libraries.mobiledatadownload.file.Opener; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import com.google.protobuf.MessageLite; import java.io.FileNotFoundException; import java.io.IOException; @@ -43,6 +44,7 @@ public final class WriteProtoOpener implements Opener<Void> { * Supports adding options to writes. For example, SyncBehavior will force data to be flushed and * durably persisted. */ + @CanIgnoreReturnValue public WriteProtoOpener withBehaviors(Behavior... behaviors) { this.behaviors = behaviors; return this; diff --git a/java/com/google/android/libraries/mobiledatadownload/file/openers/WriteStreamOpener.java b/java/com/google/android/libraries/mobiledatadownload/file/openers/WriteStreamOpener.java index f6e6c37..3eccfdd 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/openers/WriteStreamOpener.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/openers/WriteStreamOpener.java @@ -18,6 +18,7 @@ package com.google.android.libraries.mobiledatadownload.file.openers; import com.google.android.libraries.mobiledatadownload.file.Behavior; import com.google.android.libraries.mobiledatadownload.file.OpenContext; import com.google.android.libraries.mobiledatadownload.file.Opener; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.io.IOException; import java.io.OutputStream; import java.util.List; @@ -35,6 +36,7 @@ public final class WriteStreamOpener implements Opener<OutputStream> { return new WriteStreamOpener(); } + @CanIgnoreReturnValue public WriteStreamOpener withBehaviors(Behavior... behaviors) { this.behaviors = behaviors; return this; diff --git a/java/com/google/android/libraries/mobiledatadownload/file/openers/WriteStringOpener.java b/java/com/google/android/libraries/mobiledatadownload/file/openers/WriteStringOpener.java index 9c2a98c..2c8fd8f 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/openers/WriteStringOpener.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/openers/WriteStringOpener.java @@ -19,6 +19,7 @@ import com.google.android.libraries.mobiledatadownload.file.Behavior; import com.google.android.libraries.mobiledatadownload.file.OpenContext; import com.google.android.libraries.mobiledatadownload.file.Opener; import com.google.android.libraries.mobiledatadownload.file.common.internal.Charsets; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.io.IOException; import java.nio.charset.Charset; @@ -36,11 +37,13 @@ public final class WriteStringOpener implements Opener<Void> { return new WriteStringOpener(string); } + @CanIgnoreReturnValue public WriteStringOpener withCharset(Charset charset) { this.charset = charset; return this; } + @CanIgnoreReturnValue public WriteStringOpener withBehaviors(Behavior... behaviors) { this.behaviors = behaviors; return this; diff --git a/java/com/google/android/libraries/mobiledatadownload/file/samples/BUILD b/java/com/google/android/libraries/mobiledatadownload/file/samples/BUILD index 0d21230..f5a8afb 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/samples/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/file/samples/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = ["//:__subpackages__"], licenses = ["notice"], ) @@ -27,9 +28,11 @@ android_library( deps = [ "//java/com/google/android/libraries/mobiledatadownload/file/backends:file", "//java/com/google/android/libraries/mobiledatadownload/file/common", + "//java/com/google/android/libraries/mobiledatadownload/file/common:fragment", "//java/com/google/android/libraries/mobiledatadownload/file/common/internal:charsets", "//java/com/google/android/libraries/mobiledatadownload/file/spi", "@androidx_appcompat_appcompat", # buildcleaner: keep + "@com_google_code_findbugs_jsr305", "@com_google_guava_guava", ], ) diff --git a/java/com/google/android/libraries/mobiledatadownload/file/samples/CapitalizationTransform.java b/java/com/google/android/libraries/mobiledatadownload/file/samples/CapitalizationTransform.java index 73d99d8..8a55656 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/samples/CapitalizationTransform.java +++ b/java/com/google/android/libraries/mobiledatadownload/file/samples/CapitalizationTransform.java @@ -21,6 +21,7 @@ import com.google.android.libraries.mobiledatadownload.file.spi.Transform; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import javax.annotation.Nullable; /** * This is a toy transform that is useful to illustrate that the invocation order is correct when @@ -59,7 +60,7 @@ public final class CapitalizationTransform implements Transform { } @Override - public Long size() throws IOException { + public @Nullable Long size() throws IOException { if (!(in instanceof Sizable)) { return null; } diff --git a/java/com/google/android/libraries/mobiledatadownload/file/spi/BUILD b/java/com/google/android/libraries/mobiledatadownload/file/spi/BUILD index 23cc319..49458c1 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/spi/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/file/spi/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], @@ -25,6 +26,7 @@ android_library( srcs = glob(["*.java"]), deps = [ "//java/com/google/android/libraries/mobiledatadownload/file/common", + "@com_google_errorprone_error_prone_annotations", "@com_google_code_findbugs_jsr305", # NOTE: dependency of gmscore client lib <internal> ], diff --git a/java/com/google/android/libraries/mobiledatadownload/file/transforms/BUILD b/java/com/google/android/libraries/mobiledatadownload/file/transforms/BUILD index 9f9525d..1933092 100644 --- a/java/com/google/android/libraries/mobiledatadownload/file/transforms/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/file/transforms/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], @@ -65,6 +66,7 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/file/common/internal:charsets", "//java/com/google/android/libraries/mobiledatadownload/file/common/internal:lite_transform_fragments", "//proto:transform_java_proto_lite", + "@com_google_errorprone_error_prone_annotations", "@com_google_guava_guava", ], ) diff --git a/java/com/google/android/libraries/mobiledatadownload/foreground/BUILD b/java/com/google/android/libraries/mobiledatadownload/foreground/BUILD index b98c623..8a120c9 100644 --- a/java/com/google/android/libraries/mobiledatadownload/foreground/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/foreground/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], @@ -27,6 +28,18 @@ filegroup( ]), ) +android_library( + name = "ForegroundDownloadKey", + srcs = ["ForegroundDownloadKey.java"], + deps = [ + "//java/com/google/android/libraries/mobiledatadownload/account:AccountUtil", + "//java/com/google/android/libraries/mobiledatadownload/internal:MddConstants", + "//third_party/java/android_libs/guava_jdk5:hash", + "@com_google_auto_value", + "@com_google_guava_guava", + ], +) + # This includes all translated strings for MDD Notifications. Apps can choose to include subset of the # supported locale resources in their binary using the `resource_configuration_filters` option in # their android_binary rule. For more info, see: <internal> @@ -34,7 +47,8 @@ android_library( name = "NotificationUtil", srcs = ["NotificationUtil.java"], manifest = "AndroidManifest.xml", - resource_files = glob(["res/**"]), + resource_files = glob(["res/**"]) + [ + ], deps = [ "@androidx_annotation_annotation", "@androidx_core_core", diff --git a/java/com/google/android/libraries/mobiledatadownload/foreground/ForegroundDownloadKey.java b/java/com/google/android/libraries/mobiledatadownload/foreground/ForegroundDownloadKey.java new file mode 100644 index 0000000..a4f3ea2 --- /dev/null +++ b/java/com/google/android/libraries/mobiledatadownload/foreground/ForegroundDownloadKey.java @@ -0,0 +1,100 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.libraries.mobiledatadownload.foreground; + +import static com.google.android.libraries.mobiledatadownload.internal.MddConstants.SPLIT_CHAR; + +import android.accounts.Account; +import android.net.Uri; +import com.google.android.libraries.mobiledatadownload.account.AccountUtil; +import com.google.auto.value.AutoValue; +import com.google.common.base.Optional; +import com.google.common.hash.Hasher; +import com.google.common.hash.Hashing; + +/** + * Container class for unique key of a foreground download. + * + * <p>There are two kinds of foreground downloads supported: file group and single files. + * + * <p>Each kind has different requirements to build the unique key that must be provided when + * building a ForegroundDownloadKey. + */ +@AutoValue +public abstract class ForegroundDownloadKey { + + /** + * Kind of {@link ForegroundDownloadKey}. + * + * <p>Only two types of foreground downloads are supported, file groups and single files. + */ + public enum Kind { + FILE_GROUP, + SINGLE_FILE, + } + + public abstract Kind kind(); + + public abstract String key(); + + /** + * Unique Identifier of a File Group used to identify a group during a foreground download. + * + * <p><b>NOTE:</b> Properties set here <em>must</em> match the properties set in {@link + * DownloadFileGroupRequest} when starting a Foreground Download. + * + * @param groupName The name of the group to download (required) + * @param account An associated account of the group, if applicable (optional) + * @param variantId An associated variantId fo the group, if applicable (optional) + */ + public static ForegroundDownloadKey ofFileGroup( + String groupName, Optional<Account> account, Optional<String> variantId) { + Hasher keyHasher = Hashing.sha256().newHasher().putUnencodedChars(groupName); + + if (account.isPresent()) { + keyHasher + .putUnencodedChars(SPLIT_CHAR) + .putUnencodedChars(AccountUtil.serialize(account.get())); + } + + if (variantId.isPresent()) { + keyHasher.putUnencodedChars(SPLIT_CHAR).putUnencodedChars(variantId.get()); + } + return new AutoValue_ForegroundDownloadKey(Kind.FILE_GROUP, keyHasher.hash().toString()); + } + + /** + * Unique Identifier of a File used to identify it during a foreground download. + * + * <p><b>NOTE:</b> Properties set here <em>must</em> match the properties set in {@link + * SingleFileDownloadRequest} or {@link DownloadRequest} when starting a Foreground Download. + * + * @param destinationUri The on-device location where the file will be downloaded (required) + */ + public static ForegroundDownloadKey ofSingleFile(Uri destinationUri) { + Hasher keyHasher = + Hashing.sha256() + .newHasher() + .putUnencodedChars(destinationUri.toString()) + .putUnencodedChars(SPLIT_CHAR); + return new AutoValue_ForegroundDownloadKey(Kind.SINGLE_FILE, keyHasher.hash().toString()); + } + + @Override + public final String toString() { + return key(); + } +} diff --git a/java/com/google/android/libraries/mobiledatadownload/foreground/NotificationUtil.java b/java/com/google/android/libraries/mobiledatadownload/foreground/NotificationUtil.java index 5ba9a90..75ddfc8 100644 --- a/java/com/google/android/libraries/mobiledatadownload/foreground/NotificationUtil.java +++ b/java/com/google/android/libraries/mobiledatadownload/foreground/NotificationUtil.java @@ -22,178 +22,192 @@ import android.content.Context; import android.content.Intent; import android.os.Build.VERSION; import android.os.Build.VERSION_CODES; + import androidx.annotation.RequiresApi; import androidx.core.app.NotificationCompat; -import androidx.core.app.NotificationCompat.BigTextStyle; import androidx.core.app.NotificationManagerCompat; import androidx.core.content.ContextCompat; + import com.google.common.base.Preconditions; + import javax.annotation.Nullable; /** Utilities for creating and managing notifications. */ // TODO(b/148401016): Add UI test for NotificationUtil. public final class NotificationUtil { - public static final String CANCEL_ACTION_EXTRA = "cancel-action"; - public static final String KEY_EXTRA = "key"; - public static final String STOP_SERVICE_EXTRA = "stop-service"; - - private NotificationUtil() {} - - public static final String NOTIFICATION_CHANNEL_ID = "download-notification-channel-id"; - - /** Create the NotificationBuilder for the Foreground Download Service */ - public static NotificationCompat.Builder createForegroundServiceNotificationBuilder( - Context context) { - return getNotificationBuilder(context) - .setContentTitle( - "Downloading") - .setSmallIcon(android.R.drawable.stat_notify_sync_noanim); - } - - /** Create a Notification.Builder. */ - public static NotificationCompat.Builder createNotificationBuilder( - Context context, int size, String contentTitle, String contentText) { - return getNotificationBuilder(context) - .setContentTitle(contentTitle) - .setContentText(contentText) - .setSmallIcon(android.R.drawable.stat_sys_download) - .setOngoing(true) - .setProgress(size, 0, false) - .setStyle(new BigTextStyle().bigText(contentText)); - } - - private static NotificationCompat.Builder getNotificationBuilder(Context context) { - return new NotificationCompat.Builder(context, NOTIFICATION_CHANNEL_ID) - .setCategory(NotificationCompat.CATEGORY_SERVICE) - .setOnlyAlertOnce(true); - } - - /** - * Create a Notification for a key. - * - * @param key Key to identify the download this notification is created for. - */ - public static void cancelNotificationForKey(Context context, String key) { - NotificationManagerCompat notificationManager = NotificationManagerCompat.from(context); - notificationManager.cancel(notificationKeyForKey(key)); - } - - /** Create the Cancel Menu Action which will be attach to the download notification. */ - // FLAG_IMMUTABLE is only for api >= 23, however framework still recommends to use this: - // <internal> - @SuppressLint("InlinedApi") - public static void createCancelAction( - Context context, - Class<?> foregroundDownloadServiceClass, - String key, - NotificationCompat.Builder notification, - int notificationKey) { - SaferIntentUtils intentUtils = new SaferIntentUtils() {}; - - Intent cancelIntent = new Intent(context, foregroundDownloadServiceClass); - cancelIntent.setPackage(context.getPackageName()); - cancelIntent.putExtra(CANCEL_ACTION_EXTRA, notificationKey); - cancelIntent.putExtra(KEY_EXTRA, key); - - // It should be safe since we are using SaferPendingIntent, setting Package and Component, and - // use PendingIntent.FLAG_ONE_SHOT | PendingIntent.FLAG_IMMUTABLE. - PendingIntent pendingCancelIntent; - if (VERSION.SDK_INT >= VERSION_CODES.O) { - pendingCancelIntent = - intentUtils.getForegroundService( - context, - notificationKey, - cancelIntent, - PendingIntent.FLAG_ONE_SHOT | PendingIntent.FLAG_IMMUTABLE); - } else { - pendingCancelIntent = - intentUtils.getService( - context, - notificationKey, - cancelIntent, - PendingIntent.FLAG_ONE_SHOT | PendingIntent.FLAG_IMMUTABLE); + public static final String CANCEL_ACTION_EXTRA = "cancel-action"; + public static final String KEY_EXTRA = "key"; + public static final String STOP_SERVICE_EXTRA = "stop-service"; + + private NotificationUtil() { + } + + public static final String NOTIFICATION_CHANNEL_ID = "download-notification-channel-id"; + + /** Create the NotificationBuilder for the Foreground Download Service */ + public static NotificationCompat.Builder createForegroundServiceNotificationBuilder( + Context context) { + return getNotificationBuilder(context) + .setContentTitle("Downloading") + .setSmallIcon(android.R.drawable.stat_notify_sync_noanim); + } + + /** Create a Notification.Builder. */ + public static NotificationCompat.Builder createNotificationBuilder( + Context context, int size, String contentTitle, String contentText) { + return getNotificationBuilder(context) + .setContentTitle(contentTitle) + .setContentText(contentText) + .setSmallIcon(android.R.drawable.stat_sys_download) + .setOngoing(true) + .setProgress(size, 0, false); + } + + private static NotificationCompat.Builder getNotificationBuilder(Context context) { + return new NotificationCompat.Builder(context, NOTIFICATION_CHANNEL_ID) + .setCategory(NotificationCompat.CATEGORY_SERVICE) + .setOnlyAlertOnce(true); + } + + /** + * Create a Notification for a key. + * + * @param key Key to identify the download this notification is created for. + */ + public static void cancelNotificationForKey(Context context, String key) { + NotificationManagerCompat notificationManager = NotificationManagerCompat.from(context); + notificationManager.cancel(notificationKeyForKey(key)); + } + + /** Create the Cancel Menu Action which will be attach to the download notification. */ + // FLAG_IMMUTABLE is only for api >= 23, however framework still recommends to use this: + // <internal> + @SuppressLint("InlinedApi") + public static void createCancelAction( + Context context, + Class<?> foregroundDownloadServiceClass, + String key, + NotificationCompat.Builder notification, + int notificationKey) { + SaferIntentUtils intentUtils = new SaferIntentUtils() { + }; + + Intent cancelIntent = new Intent(context, foregroundDownloadServiceClass); + cancelIntent.setPackage(context.getPackageName()); + cancelIntent.putExtra(CANCEL_ACTION_EXTRA, notificationKey); + cancelIntent.putExtra(KEY_EXTRA, key); + + // It should be safe since we are using SaferPendingIntent, setting Package and + // Component, and + // use PendingIntent.FLAG_ONE_SHOT | PendingIntent.FLAG_IMMUTABLE. + PendingIntent pendingCancelIntent; + if (VERSION.SDK_INT >= VERSION_CODES.O) { + pendingCancelIntent = + intentUtils.getForegroundService( + context, + notificationKey, + cancelIntent, + PendingIntent.FLAG_ONE_SHOT | PendingIntent.FLAG_IMMUTABLE); + } else { + pendingCancelIntent = + intentUtils.getService( + context, + notificationKey, + cancelIntent, + PendingIntent.FLAG_ONE_SHOT | PendingIntent.FLAG_IMMUTABLE); + } + NotificationCompat.Action action = + new NotificationCompat.Action.Builder( + android.R.drawable.stat_sys_warning, + "Cancel", + Preconditions.checkNotNull(pendingCancelIntent)) + .build(); + notification.addAction(action); } - NotificationCompat.Action action = - new NotificationCompat.Action.Builder( - android.R.drawable.stat_sys_warning, - "Cancel", - Preconditions.checkNotNull(pendingCancelIntent)) - .build(); - notification.addAction(action); - } - - /** Generate the Notification Key for the Key */ - public static int notificationKeyForKey(String key) { - // Consider if we could have collision. - // Heavier alternative is Hashing.goodFastHash(32).hashUnencodedChars(key).asInt(); - return key.hashCode(); - } - - /** Send intent to start the DownloadService in foreground. */ - public static void startForegroundDownloadService( - Context context, Class<?> foregroundDownloadService, String key) { - Intent intent = new Intent(context, foregroundDownloadService); - intent.putExtra(KEY_EXTRA, key); - - // Start ForegroundDownloadService to download in the foreground. - ContextCompat.startForegroundService(context, intent); - } - - /** Sending the intent to stop the foreground download service */ - public static void stopForegroundDownloadService( - Context context, Class<?> foregroundDownloadService) { - Intent intent = new Intent(context, foregroundDownloadService); - intent.putExtra(STOP_SERVICE_EXTRA, true); - - // This will send the intent to stop the service. - ContextCompat.startForegroundService(context, intent); - } - - /** - * Return the String message to display in Notification when the download is paused to wait for - * network connection. - */ - public static String getDownloadPausedMessage(Context context) { - return "Waiting for network connection"; - } - - /** Return the String message to display in Notification when the download is failed. */ - public static String getDownloadFailedMessage(Context context) { - return "Download failed"; - } - - /** Return the String message to display in Notification when the download is success. */ - public static String getDownloadSuccessMessage(Context context) { - return "Downloaded"; - } - - /** Create the Notification Channel for Downloading. */ - public static void createNotificationChannel(Context context) { - if (VERSION.SDK_INT >= VERSION_CODES.O) { - NotificationChannel notificationChannel = - new NotificationChannel( - NOTIFICATION_CHANNEL_ID, - "Data Download Notification Channel", - android.app.NotificationManager.IMPORTANCE_DEFAULT); - - android.app.NotificationManager manager = - context.getSystemService(android.app.NotificationManager.class); - manager.createNotificationChannel(notificationChannel); + + /** Generate the Notification Key for the Key */ + public static int notificationKeyForKey(String key) { + // Consider if we could have collision. + // Heavier alternative is Hashing.goodFastHash(32).hashUnencodedChars(key).asInt(); + return key.hashCode(); } - } - - /** Utilities for safely accessing PendingIntent APIs. */ - private interface SaferIntentUtils { - @Nullable - @RequiresApi(VERSION_CODES.O) // to match PendingIntent.getForegroundService() - default PendingIntent getForegroundService( - Context context, int requestCode, Intent intent, int flags) { - return PendingIntent.getForegroundService(context, requestCode, intent, flags); + + /** Send intent to start the DownloadService in foreground. */ + public static void startForegroundDownloadService( + Context context, Class<?> foregroundDownloadService, String key) { + Intent intent = new Intent(context, foregroundDownloadService); + intent.putExtra(KEY_EXTRA, key); + + // Start ForegroundDownloadService to download in the foreground. + ContextCompat.startForegroundService(context, intent); + } + + /** Sending the intent to stop the foreground download service */ + public static void stopForegroundDownloadService( + Context context, Class<?> foregroundDownloadService, String key) { + Intent intent = new Intent(context, foregroundDownloadService); + intent.putExtra(STOP_SERVICE_EXTRA, true); + intent.putExtra(KEY_EXTRA, key); + + // This will send the intent to stop the service. + ContextCompat.startForegroundService(context, intent); + } + + /** + * Return the String message to display in Notification when the download is paused to wait for + * network connection. + */ + public static String getDownloadPausedMessage(Context context) { + return "Waiting for network connection"; + } + + /** + * Return the String message to display in Notification when the download is paused due to a + * missing wifi connection. + */ + public static String getDownloadPausedWifiMessage(Context context) { + return "Waiting for WiFi connection"; + } + + /** Return the String message to display in Notification when the download is failed. */ + public static String getDownloadFailedMessage(Context context) { + return "Download failed"; + } + + /** Return the String message to display in Notification when the download is success. */ + public static String getDownloadSuccessMessage(Context context) { + return "Downloaded"; + } + + /** Create the Notification Channel for Downloading. */ + public static void createNotificationChannel(Context context) { + if (VERSION.SDK_INT >= VERSION_CODES.O) { + NotificationChannel notificationChannel = + new NotificationChannel( + NOTIFICATION_CHANNEL_ID, + "Data Download Notification Channel", + android.app.NotificationManager.IMPORTANCE_DEFAULT); + + android.app.NotificationManager manager = + context.getSystemService(android.app.NotificationManager.class); + manager.createNotificationChannel(notificationChannel); + } } - @Nullable - default PendingIntent getService(Context context, int requestCode, Intent intent, int flags) { - return PendingIntent.getService(context, requestCode, intent, flags); + /** Utilities for safely accessing PendingIntent APIs. */ + private interface SaferIntentUtils { + + @Nullable + @RequiresApi(VERSION_CODES.O) // to match PendingIntent.getForegroundService() + default PendingIntent getForegroundService( + Context context, int requestCode, Intent intent, int flags) { + return PendingIntent.getForegroundService(context, requestCode, intent, flags); + } + + @Nullable + default PendingIntent getService(Context context, int requestCode, Intent intent, + int flags) { + return PendingIntent.getService(context, requestCode, intent, flags); + } } - } } diff --git a/java/com/google/android/libraries/mobiledatadownload/foreground/res/values/strings.xml b/java/com/google/android/libraries/mobiledatadownload/foreground/res/values/strings.xml index 1896b0c..1b7fb7a 100644 --- a/java/com/google/android/libraries/mobiledatadownload/foreground/res/values/strings.xml +++ b/java/com/google/android/libraries/mobiledatadownload/foreground/res/values/strings.xml @@ -30,11 +30,17 @@ </string> <!-- Notification title that is shown for every file that is currently - downloading but is temporary paused due to network connection. [CHAR_LIMIT=80] --> + downloading but is temporary paused due to missing any network connection. [CHAR_LIMIT=80] --> <string name="mdd_notification_download_paused"> Waiting for network connection </string> + <!-- Notification title that is shown for every file that is currently + downloading but is temporary paused due to missing wifi network connection. [CHAR_LIMIT=80] --> + <string name="mdd_notification_download_paused_wifi"> + Waiting for WiFi connection + </string> + <!-- Notification title that is shown for every file that was successfully downloaded.[CHAR_LIMIT=80] --> <string name="mdd_notification_download_success"> diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/AndroidTimeSource.java b/java/com/google/android/libraries/mobiledatadownload/internal/AndroidTimeSource.java new file mode 100644 index 0000000..71984cb --- /dev/null +++ b/java/com/google/android/libraries/mobiledatadownload/internal/AndroidTimeSource.java @@ -0,0 +1,41 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.libraries.mobiledatadownload.internal; + +import android.os.Build.VERSION_CODES; +import android.os.SystemClock; +import androidx.annotation.RequiresApi; +import com.google.android.libraries.mobiledatadownload.TimeSource; + +/** + * Implementation of {@link com.google.android.libraries.mobiledatadownload.TimeSource} based on + * Android platform APIs. + */ + +// necessary since cgal.clock isn't available in 3P +@RequiresApi(VERSION_CODES.JELLY_BEAN_MR1) // android.os.SystemClock#elapsedRealtimeNanos +public final class AndroidTimeSource implements TimeSource { + + @Override + public long currentTimeMillis() { + return System.currentTimeMillis(); + } + + @Override + public long elapsedRealtimeNanos() { + return SystemClock.elapsedRealtimeNanos(); + } +} diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/BUILD b/java/com/google/android/libraries/mobiledatadownload/internal/BUILD index 68f9139..a0fbf4d 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/internal/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = ["//:__subpackages__"], licenses = ["notice"], ) @@ -37,8 +38,10 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/annotations", "//java/com/google/android/libraries/mobiledatadownload/file/transforms:proto", "//java/com/google/android/libraries/mobiledatadownload/internal/annotations:SequentialControlExecutor", + "//java/com/google/android/libraries/mobiledatadownload/internal/collect", "//java/com/google/android/libraries/mobiledatadownload/internal/downloader:FileValidator", "//java/com/google/android/libraries/mobiledatadownload/internal/experimentation:DownloadStageManager", + "//java/com/google/android/libraries/mobiledatadownload/internal/logging:DownloadStateLogger", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:EventLogger", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:FileGroupStatsLogger", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LogUtil", @@ -49,6 +52,7 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/internal/util:FileGroupUtil", "//java/com/google/android/libraries/mobiledatadownload/internal/util:SharedPreferencesUtil", "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", + "//proto:log_enums_java_proto_lite", "//proto:transform_java_proto_lite", "@androidx_annotation_annotation", "@com_google_code_findbugs_jsr305", @@ -99,6 +103,7 @@ android_library( deps = [ "//java/com/google/android/libraries/mobiledatadownload:SilentFeedback", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LogUtil", + "@com_google_errorprone_error_prone_annotations", ], ) @@ -134,6 +139,7 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/annotations", "//java/com/google/android/libraries/mobiledatadownload/file", "//java/com/google/android/libraries/mobiledatadownload/internal/annotations:SequentialControlExecutor", + "//java/com/google/android/libraries/mobiledatadownload/internal/collect", "//java/com/google/android/libraries/mobiledatadownload/internal/experimentation:DownloadStageManager", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:DownloadStateLogger", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:EventLogger", @@ -145,6 +151,9 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/internal/util:SymlinkUtil", "//java/com/google/android/libraries/mobiledatadownload/tracing", "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", + "//proto:log_enums_java_proto_lite", + "//proto:logs_java_proto_lite", + "//proto:transform_java_proto_lite", "@androidx_annotation_annotation", "@com_google_auto_value", "@com_google_code_findbugs_jsr305", @@ -159,6 +168,7 @@ android_library( name = "FileGroupsMetadata", srcs = ["FileGroupsMetadata.java"], deps = [ + "//java/com/google/android/libraries/mobiledatadownload/internal/collect", "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", "@com_google_guava_guava", "@org_checkerframework_qual", @@ -175,12 +185,14 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload:TimeSource", "//java/com/google/android/libraries/mobiledatadownload/annotations", "//java/com/google/android/libraries/mobiledatadownload/internal/annotations:SequentialControlExecutor", + "//java/com/google/android/libraries/mobiledatadownload/internal/collect", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LogUtil", "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", "//java/com/google/android/libraries/mobiledatadownload/internal/util:FileGroupUtil", "//java/com/google/android/libraries/mobiledatadownload/internal/util:FileGroupsMetadataUtil", "//java/com/google/android/libraries/mobiledatadownload/internal/util:ProtoLiteUtil", "//java/com/google/android/libraries/mobiledatadownload/internal/util:SharedPreferencesUtil", + "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", "@androidx_annotation_annotation", "@com_google_errorprone_error_prone_annotations", "@com_google_guava_guava", @@ -203,12 +215,15 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/annotations", "//java/com/google/android/libraries/mobiledatadownload/file", "//java/com/google/android/libraries/mobiledatadownload/internal/annotations:SequentialControlExecutor", + "//java/com/google/android/libraries/mobiledatadownload/internal/collect", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:EventLogger", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LogUtil", "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", "//java/com/google/android/libraries/mobiledatadownload/internal/util:DirectoryUtil", "//java/com/google/android/libraries/mobiledatadownload/internal/util:FileGroupUtil", "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", + "//proto:log_enums_java_proto_lite", + "//proto:logs_java_proto_lite", "@androidx_annotation_annotation", "@com_google_guava_guava", "@javax_inject", @@ -245,6 +260,9 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/internal/util:DirectoryUtil", "//java/com/google/android/libraries/mobiledatadownload/internal/util:SharedPreferencesUtil", "//java/com/google/android/libraries/mobiledatadownload/monitor:DownloadProgressMonitor", + "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", + "//proto:log_enums_java_proto_lite", + "//proto:logs_java_proto_lite", "@androidx_annotation_annotation", "@com_google_code_findbugs_jsr305", "@com_google_dagger", @@ -283,10 +301,41 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", "//java/com/google/android/libraries/mobiledatadownload/internal/util:SharedFilesMetadataUtil", "//java/com/google/android/libraries/mobiledatadownload/internal/util:SharedPreferencesUtil", + "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", "//proto:transform_java_proto_lite", "@androidx_annotation_annotation", "@com_google_errorprone_error_prone_annotations", "@com_google_guava_guava", "@javax_inject", + "@org_checkerframework_qual", + ], +) + +android_library( + name = "DownloadGroupState", + srcs = ["DownloadGroupState.java"], + deps = [ + "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", + "//proto:client_config_java_proto_lite", + "@com_google_code_findbugs_jsr305", + "@com_google_guava_guava", + ], +) + +android_library( + name = "AndroidTimeSource", + srcs = ["AndroidTimeSource.java"], + deps = [ + "//java/com/google/android/libraries/mobiledatadownload:TimeSource", + "@androidx_annotation_annotation", + ], +) + +android_library( + name = "ExceptionToMddResultMapper", + srcs = ["ExceptionToMddResultMapper.java"], + deps = [ + "//java/com/google/android/libraries/mobiledatadownload:DownloadException", + "//proto:log_enums_java_proto_lite", ], ) diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/DataFileGroupValidator.java b/java/com/google/android/libraries/mobiledatadownload/internal/DataFileGroupValidator.java index 3d49157..f05e831 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/DataFileGroupValidator.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/DataFileGroupValidator.java @@ -20,7 +20,6 @@ import com.google.android.libraries.mobiledatadownload.Flags; import com.google.android.libraries.mobiledatadownload.file.transforms.TransformProtos; import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; import com.google.android.libraries.mobiledatadownload.internal.util.FileGroupUtil; -import com.google.mobiledatadownload.TransformProto.Transforms; import com.google.mobiledatadownload.internal.MetadataProto.DataFile; import com.google.mobiledatadownload.internal.MetadataProto.DataFile.ChecksumType; import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; @@ -28,6 +27,7 @@ import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInterna import com.google.mobiledatadownload.internal.MetadataProto.DeltaFile; import com.google.mobiledatadownload.internal.MetadataProto.DeltaFile.DiffDecoder; import com.google.mobiledatadownload.internal.MetadataProto.DownloadConditions.DeviceNetworkPolicy; +import com.google.mobiledatadownload.TransformProto.Transforms; /** DataFileGroupValidator - validates the passed in DataFileGroup */ public class DataFileGroupValidator { diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/DownloadGroupState.java b/java/com/google/android/libraries/mobiledatadownload/internal/DownloadGroupState.java new file mode 100644 index 0000000..1833f6f --- /dev/null +++ b/java/com/google/android/libraries/mobiledatadownload/internal/DownloadGroupState.java @@ -0,0 +1,183 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.libraries.mobiledatadownload.internal; + +import com.google.common.util.concurrent.ListenableFuture; +import com.google.mobiledatadownload.ClientConfigProto.ClientFileGroup; +import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; +import javax.annotation.Nullable; +import javax.annotation.concurrent.Immutable; + +/** A helper class that includes information about the state of a file group download. */ +@Immutable +public abstract class DownloadGroupState { + /** The kind of {@link DownloadGroupState}. */ + public enum Kind { + /** A pending that hasn't been downloaded yet. */ + PENDING_GROUP, + + /** A pending group whose download has already stated. */ + IN_PROGRESS_FUTURE, + + /** A group that has already been downloaded. */ + DOWNLOADED_GROUP, + } + + public abstract Kind getKind(); + + public abstract DataFileGroupInternal pendingGroup(); + + public abstract ListenableFuture<ClientFileGroup> inProgressFuture(); + + public abstract ClientFileGroup downloadedGroup(); + + public static DownloadGroupState ofPendingGroup(DataFileGroupInternal dataFileGroup) { + return new ImplPendingGroup(dataFileGroup); + } + + public static DownloadGroupState ofInProgressFuture( + ListenableFuture<ClientFileGroup> clientFileGroupFuture) { + return new ImplInProgressFuture(clientFileGroupFuture); + } + + public static DownloadGroupState ofDownloadedGroup(ClientFileGroup clientFileGroup) { + return new ImplDownloadedGroup(clientFileGroup); + } + + private DownloadGroupState() {} + + // Parent class that each implementation will inherit from. + private abstract static class Parent extends DownloadGroupState { + @Override + public DataFileGroupInternal pendingGroup() { + throw new UnsupportedOperationException(getKind().toString()); + } + + @Override + public ListenableFuture<ClientFileGroup> inProgressFuture() { + throw new UnsupportedOperationException(getKind().toString()); + } + + @Override + public ClientFileGroup downloadedGroup() { + throw new UnsupportedOperationException(getKind().toString()); + } + } + + // Implementation when the contained property is "pendingGroup". + private static final class ImplPendingGroup extends Parent { + private final DataFileGroupInternal pendingGroup; + + ImplPendingGroup(DataFileGroupInternal pendingGroup) { + this.pendingGroup = pendingGroup; + } + + @Override + public DataFileGroupInternal pendingGroup() { + return pendingGroup; + } + + @Override + public DownloadGroupState.Kind getKind() { + return DownloadGroupState.Kind.PENDING_GROUP; + } + + @Override + public boolean equals(@Nullable Object x) { + if (x instanceof DownloadGroupState) { + DownloadGroupState that = (DownloadGroupState) x; + return this.getKind() == that.getKind() && this.pendingGroup.equals(that.pendingGroup()); + } else { + return false; + } + } + + @Override + public int hashCode() { + return pendingGroup.hashCode(); + } + } + + // Implementation when the contained property is "inProgressFuture". + private static final class ImplInProgressFuture extends Parent { + private final ListenableFuture<ClientFileGroup> inProgressFuture; + + ImplInProgressFuture(ListenableFuture<ClientFileGroup> inProgressFuture) { + this.inProgressFuture = inProgressFuture; + } + + @Override + public ListenableFuture<ClientFileGroup> inProgressFuture() { + return inProgressFuture; + } + + @Override + public DownloadGroupState.Kind getKind() { + return DownloadGroupState.Kind.IN_PROGRESS_FUTURE; + } + + @Override + public boolean equals(@Nullable Object x) { + if (x instanceof DownloadGroupState) { + DownloadGroupState that = (DownloadGroupState) x; + return this.getKind() == that.getKind() + && this.inProgressFuture.equals(that.inProgressFuture()); + } else { + return false; + } + } + + @Override + public int hashCode() { + return inProgressFuture.hashCode(); + } + } + + // Implementation when the contained property is "downloadedGroup". + private static final class ImplDownloadedGroup extends Parent { + private final ClientFileGroup downloadedGroup; + + ImplDownloadedGroup(ClientFileGroup downloadedGroup) { + this.downloadedGroup = downloadedGroup; + } + + @Override + public ClientFileGroup downloadedGroup() { + return downloadedGroup; + } + + @Override + public DownloadGroupState.Kind getKind() { + return DownloadGroupState.Kind.DOWNLOADED_GROUP; + } + + @Override + public boolean equals(@Nullable Object x) { + if (x instanceof DownloadGroupState) { + DownloadGroupState that = (DownloadGroupState) x; + return this.getKind() == that.getKind() + && this.downloadedGroup.equals(that.downloadedGroup()); + } else { + return false; + } + } + + @Override + public int hashCode() { + return downloadedGroup.hashCode(); + } + } +} diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/ExceptionToMddResultMapper.java b/java/com/google/android/libraries/mobiledatadownload/internal/ExceptionToMddResultMapper.java new file mode 100644 index 0000000..f5fa536 --- /dev/null +++ b/java/com/google/android/libraries/mobiledatadownload/internal/ExceptionToMddResultMapper.java @@ -0,0 +1,65 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.libraries.mobiledatadownload.internal; + +import com.google.android.libraries.mobiledatadownload.DownloadException; +import java.io.IOException; +import java.util.concurrent.CancellationException; +import java.util.concurrent.ExecutionException; + +/** + * Maps exception to MddLibApiResult.Code. Used for logging. + * + * @see wireless.android.icing.proto.MddLibApiResult + */ +public final class ExceptionToMddResultMapper { + + private ExceptionToMddResultMapper() {} + + /** + * Maps Exception to appropriate int for logging. + * + * <p>If t is an ExecutionException, then the cause (t.getCause()) is mapped. + */ + public static int map(Throwable t) { + + Throwable cause; + if (t instanceof ExecutionException) { + cause = t.getCause(); + } else { + cause = t; + } + + if (cause instanceof CancellationException) { + return 0; + } else if (cause instanceof InterruptedException) { + return 0; + } else if (cause instanceof IOException) { + return 0; + } else if (cause instanceof IllegalStateException) { + return 0; + } else if (cause instanceof IllegalArgumentException) { + return 0; + } else if (cause instanceof UnsupportedOperationException) { + return 0; + } else if (cause instanceof DownloadException) { + return 0; + } + + // Capturing all other errors occurred during execution as unknown errors. + return 0; + } +} diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/ExpirationHandler.java b/java/com/google/android/libraries/mobiledatadownload/internal/ExpirationHandler.java index b5efe22..7c0f698 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/ExpirationHandler.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/ExpirationHandler.java @@ -20,7 +20,6 @@ import static java.lang.Math.min; import android.content.Context; import android.net.Uri; -import android.util.Pair; import androidx.annotation.VisibleForTesting; import com.google.android.libraries.mobiledatadownload.Flags; import com.google.android.libraries.mobiledatadownload.SilentFeedback; @@ -28,6 +27,7 @@ import com.google.android.libraries.mobiledatadownload.TimeSource; import com.google.android.libraries.mobiledatadownload.annotations.InstanceId; import com.google.android.libraries.mobiledatadownload.file.SynchronousFileStorage; import com.google.android.libraries.mobiledatadownload.internal.annotations.SequentialControlExecutor; +import com.google.android.libraries.mobiledatadownload.internal.collect.GroupKeyAndGroup; import com.google.android.libraries.mobiledatadownload.internal.logging.EventLogger; import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; import com.google.android.libraries.mobiledatadownload.internal.util.DirectoryUtil; @@ -39,6 +39,7 @@ import com.google.mobiledatadownload.internal.MetadataProto.DataFile; import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; import com.google.mobiledatadownload.internal.MetadataProto.NewFileKey; +import com.google.mobiledatadownload.LogEnumsProto.MddClientEvent; import java.io.IOException; import java.util.ArrayList; import java.util.HashSet; @@ -98,8 +99,7 @@ public class ExpirationHandler { this.flags = flags; } - // TODO(b/124072754): Change to package private once all code is refactored. - public ListenableFuture<Void> updateExpiration() { + ListenableFuture<Void> updateExpiration() { return PropagatedFutures.transformAsync( removeExpiredStaleGroups(), voidArg0 -> @@ -116,16 +116,16 @@ public class ExpirationHandler { fileGroupsMetadata.getAllFreshGroups(), groups -> { List<GroupKey> expiredGroupKeys = new ArrayList<>(); - for (Pair<GroupKey, DataFileGroupInternal> pair : groups) { - GroupKey groupKey = pair.first; - DataFileGroupInternal dataFileGroup = pair.second; + for (GroupKeyAndGroup pair : groups) { + GroupKey groupKey = pair.groupKey(); + DataFileGroupInternal dataFileGroup = pair.dataFileGroup(); Long groupExpirationDateMillis = FileGroupUtil.getExpirationDateMillis(dataFileGroup); LogUtil.d( "%s: Checking group %s with expiration date %s", TAG, dataFileGroup.getGroupName(), groupExpirationDateMillis); if (FileGroupUtil.isExpired(groupExpirationDateMillis, timeSource)) { eventLogger.logEventSampled( - 0, + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, dataFileGroup.getGroupName(), dataFileGroup.getFileGroupVersionNumber(), dataFileGroup.getBuildId(), @@ -147,7 +147,7 @@ public class ExpirationHandler { fileGroupsMetadata.removeAllGroupsWithKeys(expiredGroupKeys), removeSuccess -> { if (!removeSuccess) { - eventLogger.logEventSampled(0); + eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); LogUtil.e("%s: Failed to remove expired groups!", TAG); } return null; @@ -173,7 +173,7 @@ public class ExpirationHandler { // Remove the group from this list if its expired. if (FileGroupUtil.isExpired(actualExpirationDateMillis, timeSource)) { eventLogger.logEventSampled( - 0, + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, staleGroup.getGroupName(), staleGroup.getFileGroupVersionNumber(), staleGroup.getBuildId(), @@ -197,7 +197,7 @@ public class ExpirationHandler { fileGroupsMetadata.writeStaleGroups(nonExpiredStaleGroups), writeSuccess -> { if (!writeSuccess) { - eventLogger.logEventSampled(0); + eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); LogUtil.e("%s: Failed to write back stale groups!", TAG); } return immediateVoidFuture(); @@ -239,7 +239,8 @@ public class ExpirationHandler { if (success) { removedMetadataCount.getAndIncrement(); } else { - eventLogger.logEventSampled(0); + eventLogger.logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); LogUtil.e( "%s: Unsubscribe from file %s failed!", TAG, newFileKey); @@ -325,8 +326,8 @@ public class ExpirationHandler { allGroupsByKey -> { Set<NewFileKey> fileKeysReferencedByAnyGroup = new HashSet<>(); List<DataFileGroupInternal> dataFileGroups = new ArrayList<>(); - for (Pair<GroupKey, DataFileGroupInternal> dataFileGroupPair : allGroupsByKey) { - dataFileGroups.add(dataFileGroupPair.second); + for (GroupKeyAndGroup dataFileGroupPair : allGroupsByKey) { + dataFileGroups.add(dataFileGroupPair.dataFileGroup()); } return PropagatedFutures.transform( fileGroupsMetadata.getAllStaleGroups(), @@ -364,8 +365,8 @@ public class ExpirationHandler { return PropagatedFutures.transform( fileGroupsMetadata.getAllFreshGroups(), groupKeyAndGroupList -> { - for (Pair<GroupKey, DataFileGroupInternal> groupKeyAndGroup : groupKeyAndGroupList) { - DataFileGroupInternal freshGroup = groupKeyAndGroup.second; + for (GroupKeyAndGroup groupKeyAndGroup : groupKeyAndGroupList) { + DataFileGroupInternal freshGroup = groupKeyAndGroup.dataFileGroup(); // Skip any groups that don't support isolated structures if (!FileGroupUtil.isIsolatedStructureAllowed(freshGroup)) { continue; @@ -390,9 +391,9 @@ public class ExpirationHandler { try { fileStorage.deleteFile(sharedFile); releasedFiles += 1; - eventLogger.logEventSampled(0); + eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); } catch (IOException e) { - eventLogger.logEventSampled(0); + eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); LogUtil.e(e, "%s: Failed to release unaccounted file!", TAG); } } @@ -422,13 +423,13 @@ public class ExpirationHandler { } } catch (IOException e) { - eventLogger.logEventSampled(0); + eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); LogUtil.e(e, "%s: Failed to delete unaccounted file!", TAG); } } } catch (IOException e) { - eventLogger.logEventSampled(0); + eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); LogUtil.e(e, "%s: Failed to delete unaccounted file!", TAG); } return unaccountedFileCount; diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/FileGroupManager.java b/java/com/google/android/libraries/mobiledatadownload/internal/FileGroupManager.java index a23531a..7cf3eb6 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/FileGroupManager.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/FileGroupManager.java @@ -17,6 +17,7 @@ package com.google.android.libraries.mobiledatadownload.internal; import static com.google.android.libraries.mobiledatadownload.tracing.TracePropagation.propagateAsyncFunction; import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.util.concurrent.Futures.getDone; import static com.google.common.util.concurrent.Futures.immediateFailedFuture; import static com.google.common.util.concurrent.Futures.immediateFuture; import static com.google.common.util.concurrent.Futures.immediateVoidFuture; @@ -30,7 +31,6 @@ import android.net.Uri; import android.os.Build.VERSION; import android.os.Build.VERSION_CODES; import android.text.TextUtils; -import android.util.Pair; import androidx.annotation.RequiresApi; import com.google.android.libraries.mobiledatadownload.AccountSource; import com.google.android.libraries.mobiledatadownload.AggregateException; @@ -44,8 +44,11 @@ import com.google.android.libraries.mobiledatadownload.account.AccountUtil; import com.google.android.libraries.mobiledatadownload.annotations.InstanceId; import com.google.android.libraries.mobiledatadownload.file.SynchronousFileStorage; import com.google.android.libraries.mobiledatadownload.internal.annotations.SequentialControlExecutor; +import com.google.android.libraries.mobiledatadownload.internal.collect.GroupKeyAndGroup; +import com.google.android.libraries.mobiledatadownload.internal.collect.GroupPair; import com.google.android.libraries.mobiledatadownload.internal.experimentation.DownloadStageManager; import com.google.android.libraries.mobiledatadownload.internal.logging.DownloadStateLogger; +import com.google.android.libraries.mobiledatadownload.internal.logging.DownloadStateLogger.Operation; import com.google.android.libraries.mobiledatadownload.internal.logging.EventLogger; import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; import com.google.android.libraries.mobiledatadownload.internal.util.AndroidSharingUtil; @@ -53,9 +56,9 @@ import com.google.android.libraries.mobiledatadownload.internal.util.AndroidShar import com.google.android.libraries.mobiledatadownload.internal.util.DirectoryUtil; import com.google.android.libraries.mobiledatadownload.internal.util.FileGroupUtil; import com.google.android.libraries.mobiledatadownload.internal.util.SymlinkUtil; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedExecutionSequencer; import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFluentFuture; import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; -import com.google.auto.value.AutoValue; import com.google.common.base.Function; import com.google.common.base.Optional; import com.google.common.base.Preconditions; @@ -63,6 +66,7 @@ import com.google.common.collect.ComparisonChain; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import com.google.common.util.concurrent.AsyncFunction; import com.google.common.util.concurrent.FutureCallback; @@ -82,6 +86,7 @@ import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; import com.google.mobiledatadownload.internal.MetadataProto.GroupKeyProperties; import com.google.mobiledatadownload.internal.MetadataProto.NewFileKey; import com.google.mobiledatadownload.internal.MetadataProto.SharedFile; +import com.google.mobiledatadownload.LogEnumsProto.MddClientEvent; import com.google.mobiledatadownload.LogEnumsProto.MddDownloadResult; import com.google.mobiledatadownload.LogProto.DataDownloadFileGroupStats; import com.google.protobuf.Any; @@ -93,6 +98,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicReference; @@ -119,6 +125,9 @@ public class FileGroupManager { /** The download of at least one file failed. */ FAILED, + + /** The status of the group is unknown. */ + UNKNOWN, } private static final String TAG = "FileGroupManager"; @@ -136,6 +145,10 @@ public class FileGroupManager { private final DownloadStageManager downloadStageManager; private final Flags flags; + // Create an internal ExecutionSequencer to ensure that certain operations remain synced. + private final PropagatedExecutionSequencer futureSerializer = + PropagatedExecutionSequencer.create(); + @Inject public FileGroupManager( @ApplicationContext Context context, @@ -178,18 +191,22 @@ public class FileGroupManager { @SuppressWarnings("nullness") public ListenableFuture<Boolean> addGroupForDownload( GroupKey groupKey, DataFileGroupInternal receivedGroup) - throws ExpiredFileGroupException, IOException, UninstalledAppException, + throws ExpiredFileGroupException, + IOException, + UninstalledAppException, ActivationRequiredForGroupException { if (FileGroupUtil.isActiveGroupExpired(receivedGroup, timeSource)) { LogUtil.e("%s: Trying to add expired group %s.", TAG, groupKey.getGroupName()); - logEventWithDataFileGroup(0, eventLogger, receivedGroup); + logEventWithDataFileGroup( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, eventLogger, receivedGroup); throw new ExpiredFileGroupException(); } if (!isAppInstalled(groupKey.getOwnerPackage())) { LogUtil.e( "%s: Trying to add group %s for uninstalled app %s.", TAG, groupKey.getGroupName(), groupKey.getOwnerPackage()); - logEventWithDataFileGroup(0, eventLogger, receivedGroup); + logEventWithDataFileGroup( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, eventLogger, receivedGroup); throw new UninstalledAppException(); } @@ -212,7 +229,8 @@ public class FileGroupManager { "%s: Trying to add group %s that requires activation %s.", TAG, groupKey.getGroupName(), groupKey.getOwnerPackage()); - logEventWithDataFileGroup(0, eventLogger, receivedGroup); + logEventWithDataFileGroup( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, eventLogger, receivedGroup); throw new ActivationRequiredForGroupException(); } @@ -224,19 +242,34 @@ public class FileGroupManager { .transformAsync( voidArg -> isAddedGroupDuplicate(groupKey, receivedGroup), sequentialControlExecutor) .transformAsync( - isDuplicate -> { - if (isDuplicate) { + newConfigReason -> { + if (!newConfigReason.isPresent()) { + // Absent reason means the config is not new LogUtil.d( "%s: Received duplicate config for group: %s", TAG, groupKey.getGroupName()); return immediateFuture(false); } + + // If supported, set the isolated root before writing to metadata + DataFileGroupInternal receivedGroupWithIsolatedRoot = + FileGroupUtil.maybeSetIsolatedRoot(receivedGroup, groupKey); + return transformSequentialAsync( - maybeSetGroupNewFilesReceivedTimestamp(groupKey, receivedGroup), + maybeSetGroupNewFilesReceivedTimestamp(groupKey, receivedGroupWithIsolatedRoot), receivedGroupCopy -> { LogUtil.d( "%s: Received new config for group: %s", TAG, groupKey.getGroupName()); - logEventWithDataFileGroup(0, eventLogger, receivedGroupCopy); + eventLogger.logNewConfigReceived( + DataDownloadFileGroupStats.newBuilder() + .setFileGroupName(receivedGroupCopy.getGroupName()) + .setOwnerPackage(receivedGroupCopy.getOwnerPackage()) + .setFileGroupVersionNumber( + receivedGroupCopy.getFileGroupVersionNumber()) + .setBuildId(receivedGroupCopy.getBuildId()) + .setVariantId(receivedGroupCopy.getVariantId()) + .build(), + null); return transformSequentialAsync( subscribeGroup(receivedGroupCopy), @@ -278,7 +311,7 @@ public class FileGroupManager { .transformAsync( writeSuccess -> { if (!writeSuccess) { - eventLogger.logEventSampled(0); + eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); return immediateFailedFuture( new IOException("Failed to commit new group metadata to disk.")); } @@ -337,7 +370,8 @@ public class FileGroupManager { "%s: Failed to remove pending version for group: '%s';" + " account: '%s'", TAG, groupKey.getGroupName(), groupKey.getAccount()); - eventLogger.logEventSampled(0); + eventLogger.logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); return immediateFailedFuture( new IOException( "Failed to remove pending group: " @@ -366,7 +400,8 @@ public class FileGroupManager { "%s: Failed to remove the downloaded version for group:" + " '%s'; account: '%s'", TAG, groupKey.getGroupName(), groupKey.getAccount()); - eventLogger.logEventSampled(0); + eventLogger.logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); return immediateFailedFuture( new IOException( "Failed to remove downloaded group: " @@ -381,7 +416,8 @@ public class FileGroupManager { "%s: Failed to add to stale for group: '%s';" + " account: '%s'", TAG, groupKey.getGroupName(), groupKey.getAccount()); - eventLogger.logEventSampled(0); + eventLogger.logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); return immediateFailedFuture( new IOException( "Failed to add downloaded group to stale: " @@ -514,7 +550,8 @@ public class FileGroupManager { "%s: Failed to remove %d pending versions of %d requested" + " groups", TAG, pendingGroupsToRemove.size(), groupKeys.size()); - eventLogger.logEventSampled(0); + eventLogger.logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); return immediateFailedFuture( new IOException( "Failed to remove pending group keys, count = " @@ -567,7 +604,8 @@ public class FileGroupManager { "%s: Failed to remove %d downloaded versions of %d requested" + " groups", TAG, downloadedGroupsToRemove.size(), groupKeys.size()); - eventLogger.logEventSampled(0); + eventLogger.logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); return immediateFailedFuture( new IOException( "Failed to remove downloaded groups, count = " @@ -600,7 +638,7 @@ public class FileGroupManager { LogUtil.e( "%s: Failed to add to stale for group: '%s';", TAG, staleGroup.getGroupName()); - eventLogger.logEventSampled(0); + eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); return immediateFailedFuture( new IOException( "Failed to add downloaded group to stale: " @@ -670,15 +708,7 @@ public class FileGroupManager { public ListenableFuture<@NullableType DataFileGroupInternal> getFileGroup( GroupKey groupKey, boolean downloaded) { GroupKey downloadedKey = groupKey.toBuilder().setDownloaded(downloaded).build(); - return transformSequentialAsync( - fileGroupsMetadata.read(downloadedKey), - dataFileGroup -> - transformSequentialAsync( - // TODO(b/194688687): consider moving this verification to the - // MobileDataDownloadManager level since that is where verification happens for - // getDataFileUri. - maybeVerifyIsolatedStructure(dataFileGroup, downloaded), - result -> immediateFuture(result ? dataFileGroup : null))); + return fileGroupsMetadata.read(downloadedKey); } /** @@ -689,25 +719,24 @@ public class FileGroupManager { * pending/downloded states of a file group, so the downloaded status in the given groupKey is not * considered by this method. * - * <p>If a group is found, state of the file group (downloaded/pending) and file group will be - * returned in a Pair. If a group is not found, null will be returned. The boolean returned will - * be true if the group is downloaded and false if the group is pending. + * <p>If a group is found, a {@link GroupKeyAndGroup} will be returned. If a group is not found, + * null will be returned. The boolean returned will be true if the group is downloaded and false + * if the group is pending. * * @param groupKey The key for the data to be returned. This is should include group name, owner * package and user account * @param buildId The expected buildId of the file group * @param variantId The expected variantId of the file group * @param customPropertyOptional The expected customProperty, if necessary - * @return A ListenableFuture that resolves, if the requested group is found, with a Pair - * containing Boolean value of whether or not the Group is downloaded and the Group itself, or - * null otherwise. + * @return A ListenableFuture that resolves, if the requested group is found, to a {@link + * GroupKeyAndGroup}, or null if no group is found. */ - private ListenableFuture<@NullableType Pair<Boolean, DataFileGroupInternal>> getGroupPairById( + private ListenableFuture<@NullableType GroupKeyAndGroup> getGroupPairById( GroupKey groupKey, long buildId, String variantId, Optional<Any> customPropertyOptional) { return transformSequential( fileGroupsMetadata.getAllFreshGroups(), freshGroupPairList -> { - for (Pair<GroupKey, DataFileGroupInternal> freshGroupPair : freshGroupPairList) { + for (GroupKeyAndGroup freshGroupPair : freshGroupPairList) { if (!verifyGroupPairMatchesIdentifiers( freshGroupPair, groupKey.getAccount(), @@ -719,19 +748,19 @@ public class FileGroupManager { } // Group matches ID, but ensure that it also matches requested group name - if (!groupKey.getGroupName().equals(freshGroupPair.first.getGroupName())) { + if (!groupKey.getGroupName().equals(freshGroupPair.groupKey().getGroupName())) { LogUtil.e( "%s: getGroupPairById: Group %s matches the given buildId = %d and variantId =" + " %s, but does not match the given group name %s", TAG, - freshGroupPair.first.getGroupName(), + freshGroupPair.groupKey().getGroupName(), buildId, variantId, groupKey.getGroupName()); continue; } - return Pair.create(freshGroupPair.first.getDownloaded(), freshGroupPair.second); + return freshGroupPair; } // No compatible group found, return null; @@ -792,7 +821,7 @@ public class FileGroupManager { fileGroupsMetadata.remove(groupKey), removeSuccess -> { if (!removeSuccess) { - eventLogger.logEventSampled(0); + eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); } return immediateVoidFuture(); }); @@ -844,11 +873,11 @@ public class FileGroupManager { DownloadStateLogger downloadStateLogger = DownloadStateLogger.forImport(eventLogger); // Get group that should be updated for import, or return group not found failure - ListenableFuture<Pair<Boolean, DataFileGroupInternal>> groupPairToUpdateFuture = + ListenableFuture<GroupKeyAndGroup> groupKeyAndGroupToUpdateFuture = transformSequentialAsync( getGroupPairById(groupKey, buildId, variantId, customPropertyOptional), - foundGroupPair -> { - if (foundGroupPair == null) { + foundGroupKeyAndGroup -> { + if (foundGroupKeyAndGroup == null) { // Group with identifiers could not be found, return failure. LogUtil.e( "%s: importFiles for group name: %s, buildId: %d, variantId: %s, but no group" @@ -865,16 +894,17 @@ public class FileGroupManager { } // wrap in checkNotNull to ensure type safety. - return immediateFuture(checkNotNull(foundGroupPair)); + return immediateFuture(checkNotNull(foundGroupKeyAndGroup)); }); - return PropagatedFluentFuture.from(groupPairToUpdateFuture) + return PropagatedFluentFuture.from(groupKeyAndGroupToUpdateFuture) .transformAsync( - groupPairToUpdate -> { + groupKeyAndGroupToUpdate -> { // Perform an in-memory merge of updatedDataFileList into the group, so we get the // correct list of files to import. DataFileGroupInternal mergedFileGroup = - mergeFilesIntoFileGroup(updatedDataFileList, groupPairToUpdate.second); + mergeFilesIntoFileGroup( + updatedDataFileList, groupKeyAndGroupToUpdate.dataFileGroup()); // Log the start of the import now that we have the group. downloadStateLogger.logStarted(mergedFileGroup); @@ -900,7 +930,8 @@ public class FileGroupManager { sequentialControlExecutor) .transformAsync( mergedFileGroup -> { - boolean groupIsDownloaded = Futures.getDone(groupPairToUpdateFuture).first; + boolean groupIsDownloaded = + Futures.getDone(groupKeyAndGroupToUpdateFuture).groupKey().getDownloaded(); // If we are updating a pending group and the import is successful, the pending // version should be removed from metadata. @@ -915,12 +946,15 @@ public class FileGroupManager { PropagatedFutures.whenAllComplete(allImportFutures) .callAsync( () -> - verifyGroupDownloaded( - groupKey, - mergedFileGroup, - removePendingVersion, - customFileGroupValidator, - downloadStateLogger), + futureSerializer.submitAsync( + () -> + verifyGroupDownloaded( + groupKey, + mergedFileGroup, + removePendingVersion, + customFileGroupValidator, + downloadStateLogger), + sequentialControlExecutor), sequentialControlExecutor); return transformSequentialAsync( combinedImportFuture, @@ -934,16 +968,16 @@ public class FileGroupManager { // We log other results in verifyGroupDownloaded, so only check for // downloaded here. if (groupDownloadStatus == GroupDownloadStatus.DOWNLOADED) { - eventLogger.logMddDownloadResult( - MddDownloadResult.Code.SUCCESS, - DataDownloadFileGroupStats.newBuilder() - .setFileGroupName(groupKey.getGroupName()) - .setOwnerPackage(groupKey.getOwnerPackage()) - .setFileGroupVersionNumber( - mergedFileGroup.getFileGroupVersionNumber()) - .setBuildId(mergedFileGroup.getBuildId()) - .setVariantId(mergedFileGroup.getVariantId()) - .build()); + eventLogger.logMddDownloadResult( + MddDownloadResult.Code.SUCCESS, + DataDownloadFileGroupStats.newBuilder() + .setFileGroupName(groupKey.getGroupName()) + .setOwnerPackage(groupKey.getOwnerPackage()) + .setFileGroupVersionNumber( + mergedFileGroup.getFileGroupVersionNumber()) + .setBuildId(mergedFileGroup.getBuildId()) + .setVariantId(mergedFileGroup.getVariantId()) + .build()); // group downloaded, so it will be written in verifyGroupDownloaded, return // early. return immediateVoidFuture(); @@ -959,7 +993,7 @@ public class FileGroupManager { mergedFileGroup), writeSuccess -> { if (!writeSuccess) { - eventLogger.logEventSampled(0); + eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); return immediateFailedFuture( DownloadException.builder() .setMessage( @@ -1027,13 +1061,13 @@ public class FileGroupManager { * </ul> */ private static boolean verifyGroupPairMatchesIdentifiers( - Pair<GroupKey, DataFileGroupInternal> groupPair, + GroupKeyAndGroup groupPair, String serializedAccount, long buildId, String variantId, Optional<Any> customPropertyOptional) { - DataFileGroupInternal fileGroup = groupPair.second; - if (!groupPair.first.getAccount().equals(serializedAccount)) { + DataFileGroupInternal fileGroup = groupPair.dataFileGroup(); + if (!groupPair.groupKey().getAccount().equals(serializedAccount)) { LogUtil.v( "%s: verifyGroupPairMatchesIdentifiers failed for group %s due to mismatched account", TAG, fileGroup.getGroupName()); @@ -1222,17 +1256,42 @@ public class FileGroupManager { return PropagatedFutures.whenAllComplete(allFileFutures) .callAsync( () -> - transformSequentialAsync( - verifyPendingGroupDownloaded( - groupKey, - updatedPendingGroup, - customFileGroupValidator), - groupDownloadStatus -> - finalizeDownloadFileFutures( - allFileFutures, - groupDownloadStatus, - updatedPendingGroup, - groupKey)), + futureSerializer.submitAsync( + () -> + transformSequentialAsync( + getGroupPair(groupKey), + groupPair -> { + @NullableType + DataFileGroupInternal groupToVerify = + groupPair.pendingGroup() != null + ? groupPair.pendingGroup() + : groupPair.downloadedGroup(); + if (groupToVerify != null) { + return transformSequentialAsync( + verifyGroupDownloaded( + groupKey, + groupToVerify, + /* removePendingVersion= */ true, + customFileGroupValidator, + DownloadStateLogger.forDownload( + eventLogger)), + groupDownloadStatus -> + finalizeDownloadFileFutures( + allFileFutures, + groupDownloadStatus, + groupToVerify, + groupKey)); + } else { + // No group to verify, which should be + // impossible -- force a failure state so we can + // track any download file failures. + handleDownloadFileFutureFailures( + allFileFutures, groupKey); + return immediateFailedFuture( + new AssertionError("impossible error")); + } + }), + sequentialControlExecutor), sequentialControlExecutor); }, sequentialControlExecutor); @@ -1292,6 +1351,24 @@ public class FileGroupManager { sequentialControlExecutor); } + private ListenableFuture<GroupPair> getGroupPair(GroupKey groupKey) { + return PropagatedFutures.submitAsync( + () -> { + ListenableFuture<@NullableType DataFileGroupInternal> pendingGroupFuture = + getFileGroup(groupKey, /* downloaded= */ false); + ListenableFuture<@NullableType DataFileGroupInternal> downloadedGroupFuture = + getFileGroup(groupKey, /* downloaded= */ true); + return PropagatedFutures.whenAllSucceed(pendingGroupFuture, downloadedGroupFuture) + .callAsync( + () -> + immediateFuture( + GroupPair.create( + getDone(pendingGroupFuture), getDone(downloadedGroupFuture))), + sequentialControlExecutor); + }, + sequentialControlExecutor); + } + private List<ListenableFuture<Void>> startDownloadFutures( @Nullable DownloadConditions downloadConditions, DataFileGroupInternal pendingGroup, @@ -1375,33 +1452,40 @@ public class FileGroupManager { // TODO(b/136112848): When all fileFutures succeed, we don't need to verify them again. However // we still need logic to remove pending and update stale group. if (groupDownloadStatus != GroupDownloadStatus.DOWNLOADED) { - LogUtil.e( - "%s downloadFileGroup %s %s can't finish!", - TAG, groupKey.getGroupName(), groupKey.getOwnerPackage()); - - AggregateException.throwIfFailed( - allFileFutures, "Failed to download file group %s", groupKey.getGroupName()); - - // TODO(b/118137672): Investigate on the unknown error that we've missed. There is a download - // failure that we don't recognize. - LogUtil.e("%s: An unknown error has occurred during" + " download", TAG); - throw DownloadException.builder() - .setDownloadResultCode(DownloadResultCode.UNKNOWN_ERROR) - .build(); + handleDownloadFileFutureFailures(allFileFutures, groupKey); } - eventLogger.logMddDownloadResult( - MddDownloadResult.Code.SUCCESS, - DataDownloadFileGroupStats.newBuilder() - .setFileGroupName(groupKey.getGroupName()) - .setOwnerPackage(groupKey.getOwnerPackage()) - .setFileGroupVersionNumber(pendingGroup.getFileGroupVersionNumber()) - .setBuildId(pendingGroup.getBuildId()) - .setVariantId(pendingGroup.getVariantId()) - .build()); + eventLogger.logMddDownloadResult( + MddDownloadResult.Code.SUCCESS, + DataDownloadFileGroupStats.newBuilder() + .setFileGroupName(groupKey.getGroupName()) + .setOwnerPackage(groupKey.getOwnerPackage()) + .setFileGroupVersionNumber(pendingGroup.getFileGroupVersionNumber()) + .setBuildId(pendingGroup.getBuildId()) + .setVariantId(pendingGroup.getVariantId()) + .build()); return immediateFuture(pendingGroup); } + // Requires that all futures in allFileFutures are completed. + private void handleDownloadFileFutureFailures( + List<ListenableFuture<Void>> allFileFutures, GroupKey groupKey) + throws DownloadException, AggregateException { + LogUtil.e( + "%s downloadFileGroup %s %s can't finish!", + TAG, groupKey.getGroupName(), groupKey.getOwnerPackage()); + + AggregateException.throwIfFailed( + allFileFutures, "Failed to download file group %s", groupKey.getGroupName()); + + // TODO(b/118137672): Investigate on the unknown error that we've missed. There is a download + // failure that we don't recognize. + LogUtil.e("%s: An unknown error has occurred during" + " download", TAG); + throw DownloadException.builder() + .setDownloadResultCode(DownloadResultCode.UNKNOWN_ERROR) + .build(); + } + /** * If the file is available in the shared blob storage, it acquires the lease and updates the * shared file metadata. The {@code FileStatus} will be set to DOWNLOAD_COMPLETE so that the file @@ -1494,7 +1578,7 @@ public class FileGroupManager { fileGroup, dataFile, fileStorage, - /* afterDownload = */ false); + /* afterDownload= */ false); return transformSequentialAsync( maybeUpdateLeaseAndSharedMetadata( fileGroup, @@ -1608,7 +1692,6 @@ public class FileGroupManager { 0), res -> { if (res) { - deleteLocalCopy(downloadFileOnDeviceUri, fileGroup, dataFile); return immediateVoidFuture(); } return updateMaxExpirationDateSecs( @@ -1629,7 +1712,7 @@ public class FileGroupManager { fileGroup, dataFile, fileStorage, - /* afterDownload = */ true); + /* afterDownload= */ true); return transformSequentialAsync( maybeUpdateLeaseAndSharedMetadata( fileGroup, @@ -1641,7 +1724,6 @@ public class FileGroupManager { 0), res -> { if (res) { - deleteLocalCopy(downloadFileOnDeviceUri, fileGroup, dataFile); return immediateVoidFuture(); } return updateMaxExpirationDateSecs( @@ -1759,7 +1841,7 @@ public class FileGroupManager { dataFile.getChecksum(), silentFeedback, instanceId, - /* androidShared = */ false); + /* androidShared= */ false); if (downloadFileOnDeviceUri == null) { LogUtil.e("%s: Failed to get file uri!", TAG); throw new AndroidSharingException(0, "Failed to get local file uri"); @@ -1767,19 +1849,6 @@ public class FileGroupManager { return downloadFileOnDeviceUri; } - private void deleteLocalCopy( - Uri downloadFileOnDeviceUri, DataFileGroupInternal fileGroup, DataFile dataFile) { - try { - fileStorage.deleteFile(downloadFileOnDeviceUri); - } catch (IOException e) { - LogUtil.e( - "%s: Failed to delete the local copy after android-sharing the file" - + " %s, file group %s", - TAG, dataFile.getFileId(), fileGroup.getGroupName()); - logMddAndroidSharingLog(eventLogger, fileGroup, dataFile, 0); - } - } - /** * Download and Verify all files present in any pending groups. * @@ -1861,30 +1930,8 @@ public class FileGroupManager { } /** - * Verifies that the given pending group was downloaded, and updates the metadata if the download - * has completed. - * - * @param groupKey The key of the group to verify for download. - * @param pendingGroup The group to verify for download. - * @return A future that resolves to true if the given group was verify for download, false - * otherwise. - */ - // TODO(b/124072754): Change to package private once all code is refactored. - public ListenableFuture<GroupDownloadStatus> verifyPendingGroupDownloaded( - GroupKey groupKey, - DataFileGroupInternal pendingGroup, - AsyncFunction<DataFileGroupInternal, Boolean> customFileGroupValidator) { - return verifyGroupDownloaded( - groupKey, - pendingGroup, - /* removePendingVersion = */ true, - customFileGroupValidator, - /* downloadStateLogger = */ DownloadStateLogger.forDownload(eventLogger)); - } - - /** - * Verifies that the given pending group was downloaded, and updates the metadata if the download - * has completed. + * Verifies that the given group was downloaded, and updates the metadata if the download has + * completed. * * @param groupKey The key of the group to verify for download. * @param fileGroup The group to verify for download. @@ -1893,7 +1940,7 @@ public class FileGroupManager { * @return A future that resolves to true if the given group was verify for download, false * otherwise. */ - private ListenableFuture<GroupDownloadStatus> verifyGroupDownloaded( + ListenableFuture<GroupDownloadStatus> verifyGroupDownloaded( GroupKey groupKey, DataFileGroupInternal fileGroup, boolean removePendingVersion, @@ -1906,6 +1953,11 @@ public class FileGroupManager { GroupKey downloadedGroupKey = groupKey.toBuilder().setDownloaded(true).build(); GroupKey pendingGroupKey = groupKey.toBuilder().setDownloaded(false).build(); + // It's possible that we are calling verifyGroupDownloaded concurrently, which would lead to + // multiple DOWNLOAD_COMPLETE logs. To prevent this, we check to see if we've already logged the + // timestamp so we can skip logging later. + boolean completeAlreadyLogged = + fileGroup.getBookkeeping().hasGroupDownloadedTimestampInMillis(); DataFileGroupInternal downloadedFileGroupWithTimestamp = FileGroupUtil.setDownloadedTimestampInMillis(fileGroup, timeSource.currentTimeMillis()); @@ -1936,6 +1988,8 @@ public class FileGroupManager { // supported if (FileGroupUtil.isIsolatedStructureAllowed(fileGroup) && VERSION.SDK_INT >= VERSION_CODES.LOLLIPOP) { + // TODO(b/225409326): Prevent race condition where recreation of isolated + // paths happens at the same time as group access. return createIsolatedFilePaths(fileGroup); } return immediateVoidFuture(); @@ -1958,7 +2012,12 @@ public class FileGroupManager { .transformAsync(this::addGroupAsStaleIfPresent, sequentialControlExecutor) .transform( voidArg -> { - downloadStateLogger.logComplete(downloadedFileGroupWithTimestamp); + // Only log complete if we are performing an import operation OR we haven't + // already logged a download complete event. + if (!completeAlreadyLogged + || downloadStateLogger.getOperation() == Operation.IMPORT) { + downloadStateLogger.logComplete(downloadedFileGroupWithTimestamp); + } return GroupDownloadStatus.DOWNLOADED; }, sequentialControlExecutor); @@ -1985,7 +2044,7 @@ public class FileGroupManager { .transformAsync( writeSuccess -> { if (!writeSuccess) { - eventLogger.logEventSampled(0); + eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); return immediateFailedFuture( new IOException( "Failed to write updated group: " + downloadedGroupKey.getGroupName())); @@ -2004,7 +2063,7 @@ public class FileGroupManager { fileGroupsMetadata.remove(pendingGroupKey), removeSuccess -> { if (!removeSuccess) { - eventLogger.logEventSampled(0); + eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); } return toReturn; }); @@ -2038,7 +2097,7 @@ public class FileGroupManager { "%s: Failed to remove pending version for group: '%s';" + " account: '%s'", TAG, pendingGroupKey.getGroupName(), pendingGroupKey.getAccount()); - eventLogger.logEventSampled(0); + eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); return immediateFailedFuture( new IOException( "Failed to remove pending group: " + pendingGroupKey.getGroupName())); @@ -2069,7 +2128,7 @@ public class FileGroupManager { // unaccounted for, and the files will get deleted // in the next daily maintenance, hence not // enforcing its stale lifetime. - eventLogger.logEventSampled(0); + eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); } return immediateVoidFuture(); }); @@ -2106,28 +2165,31 @@ public class FileGroupManager { .setCause(e) .build()); } - List<ListenableFuture<Void>> createSymlinkFutures = - new ArrayList<>(dataFileGroup.getFileCount()); - for (DataFile dataFile : dataFileGroup.getFileList()) { - if (dataFile.getAndroidSharingType() == AndroidSharingType.ANDROID_BLOB_WHEN_AVAILABLE) { - createSymlinkFutures.add( - immediateFailedFuture( - new UnsupportedOperationException( - "Preserve File Paths is invalid with Android Blob Sharing"))); - // break out of loop since we've already hit a failure. - break; - } + List<DataFile> dataFiles = dataFileGroup.getFileList(); - // Get the original path - ListenableFuture<Void> createSymlinkFuture = - transformSequentialAsync( - getOnDeviceUri(dataFile, dataFileGroup), - (Uri originalUri) -> { - Uri symlinkUri = - FileGroupUtil.getIsolatedFileUri(context, instanceId, dataFile, dataFileGroup); + if (Iterables.tryFind( + dataFiles, + dataFile -> + dataFile.getAndroidSharingType() == AndroidSharingType.ANDROID_BLOB_WHEN_AVAILABLE) + .isPresent()) { + // Creating isolated structure is not supported when android sharing is enabled in the group; + // return immediately. + return immediateFailedFuture( + new UnsupportedOperationException( + "Preserve File Paths is invalid with Android Blob Sharing")); + } + ImmutableMap<DataFile, Uri> isolatedFileUriMap = getIsolatedFileUris(dataFileGroup); + ListenableFuture<Void> createIsolatedStructureFuture = + PropagatedFutures.transformAsync( + getOnDeviceUris(dataFileGroup), + onDeviceUriMap -> { + for (DataFile dataFile : dataFiles) { try { + Uri symlinkUri = checkNotNull(isolatedFileUriMap.get(dataFile)); + Uri originalUri = checkNotNull(onDeviceUriMap.get(dataFile)); + // Check/create parent dir of symlink. Uri symlinkParentDir = Uri.parse( @@ -2137,8 +2199,8 @@ public class FileGroupManager { if (!fileStorage.exists(symlinkParentDir)) { fileStorage.createDirectory(symlinkParentDir); } - SymlinkUtil.createSymlink(context, symlinkUri, checkNotNull(originalUri)); - } catch (IOException e) { + SymlinkUtil.createSymlink(context, symlinkUri, originalUri); + } catch (NullPointerException | IOException e) { return immediateFailedFuture( DownloadException.builder() .setDownloadResultCode( @@ -2147,15 +2209,13 @@ public class FileGroupManager { .setCause(e) .build()); } - return immediateVoidFuture(); - }); - createSymlinkFutures.add(createSymlinkFuture); - } - ListenableFuture<Void> combinedFuture = - Futures.whenAllSucceed(createSymlinkFutures).call(() -> null, sequentialControlExecutor); + } + return immediateVoidFuture(); + }, + sequentialControlExecutor); PropagatedFutures.addCallback( - combinedFuture, + createIsolatedStructureFuture, new FutureCallback<Void>() { @Override public void onSuccess(Void unused) {} @@ -2174,29 +2234,7 @@ public class FileGroupManager { }, sequentialControlExecutor); - return combinedFuture; - } - - /** - * Gets the Isolated File Uri and verifies that it exists and points to the given uri. - * - * <p>Throws IOException when verifying the symlink fails. - */ - @RequiresApi(VERSION_CODES.LOLLIPOP) - Uri getAndVerifyIsolatedFileUri( - Uri originalFileUri, DataFile dataFile, DataFileGroupInternal dataFileGroup) - throws IOException { - Uri isolatedFileUri = - FileGroupUtil.getIsolatedFileUri(context, instanceId, dataFile, dataFileGroup); - - Uri targetFileUri = SymlinkUtil.readSymlink(context, isolatedFileUri); - - if (!fileStorage.exists(isolatedFileUri) - || !targetFileUri.toString().equals(originalFileUri.toString())) { - throw new IOException("Isolated file uri does not exist or points to an unexpected target"); - } - - return isolatedFileUri; + return createIsolatedStructureFuture; } /** @@ -2220,7 +2258,7 @@ public class FileGroupManager { * * <p>This method is annotated with @TargetApi(21) since symlink structure methods require API * level 21 or later. The FileGroupUtil.isIsolatedStructureAllowed check will ensure this - * condition is met before calling getAndVerifyIsolatedFileUri and createIsolatedFilePaths. + * condition is met before calling verifyIsolatedFileUris and createIsolatedFilePaths. * * @return Future that resolves to true if the isolated structure is verified, or false if the * structure couldn't be verified @@ -2236,36 +2274,24 @@ public class FileGroupManager { return immediateFuture(true); } - List<ListenableFuture<Void>> verifyIsolatedFileFutures = - new ArrayList<>(dataFileGroup.getFileCount()); - for (DataFile dataFile : dataFileGroup.getFileList()) { - verifyIsolatedFileFutures.add( - transformSequentialAsync( - getOnDeviceUri(dataFile, dataFileGroup), - onDeviceUri -> { - if (onDeviceUri != null) { - Uri unused = getAndVerifyIsolatedFileUri(onDeviceUri, dataFile, dataFileGroup); + return PropagatedFluentFuture.from(getOnDeviceUris(dataFileGroup)) + .transform( + onDeviceUriMap -> { + ImmutableMap<DataFile, Uri> verifiedUriMap = + verifyIsolatedFileUris(getIsolatedFileUris(dataFileGroup), onDeviceUriMap); + for (DataFile dataFile : dataFileGroup.getFileList()) { + if (!verifiedUriMap.containsKey(dataFile)) { + // File is missing from map, so verification failed, log this error and return + // false. + LogUtil.w( + "%s: Detected corruption of isolated structure for group %s %s", + TAG, dataFileGroup.getGroupName(), dataFile.getFileId()); + return false; } - return immediateVoidFuture(); - })); - } - - return PropagatedFutures.catching( - Futures.whenAllSucceed(verifyIsolatedFileFutures) - .call(() -> true, sequentialControlExecutor), - IOException.class, - ex -> { - // TODO(b/194688687): Log these events to clearcut along with their file group info so - // we can understand how often this is happening. - LogUtil.w( - ex, - "%s: Detected corruption of isolated structure for group %s", - TAG, - dataFileGroup.getGroupName()); - - return false; - }, - sequentialControlExecutor); + } + return true; + }, + sequentialControlExecutor); } /** @@ -2291,6 +2317,119 @@ public class FileGroupManager { } /** + * Gets the on-device uri of the given list of {@link DataFile}s. + * + * <p>Checks for sideloading support. If the file is sideloaded and sideloading is enabled, the + * sideloaded uri will be returned immediately. If sideloading is not enabled, returns a faliure. + * + * <p>If file is not sideloaded, delegates to {@link SharedFileManager#getOnDeviceUris()}. + * + * <p>NOTE: The returned map will contain entries for all data files with a known uri. If the uri + * is unable to be calculated, it will not be included in the returned list. + */ + ListenableFuture<ImmutableMap<DataFile, Uri>> getOnDeviceUris( + DataFileGroupInternal dataFileGroup) { + ImmutableMap.Builder<DataFile, Uri> onDeviceUriMap = ImmutableMap.builder(); + ImmutableMap.Builder<DataFile, NewFileKey> nonSideloadedKeyMapBuilder = ImmutableMap.builder(); + for (DataFile dataFile : dataFileGroup.getFileList()) { + if (FileGroupUtil.isSideloadedFile(dataFile)) { + // Sideloaded file -- put in map immediately + onDeviceUriMap.put(dataFile, Uri.parse(dataFile.getUrlToDownload())); + } else { + // Non sideloaded file -- mark for further lookup + nonSideloadedKeyMapBuilder.put( + dataFile, + SharedFilesMetadata.createKeyFromDataFile( + dataFile, dataFileGroup.getAllowedReadersEnum())); + } + } + ImmutableMap<DataFile, NewFileKey> nonSideloadedKeyMap = + nonSideloadedKeyMapBuilder.build(); + + return PropagatedFluentFuture.from( + sharedFileManager.getOnDeviceUris(ImmutableSet.copyOf(nonSideloadedKeyMap.values()))) + .transform( + nonSideloadedUriMap -> { + // Extract the <DataFile, Uri> entries from the two non-sideloaded maps. + // DataFile -> NewFileKey -> Uri now becomes DataFile -> Uri + for (Entry<DataFile, NewFileKey> keyMapEntry : nonSideloadedKeyMap.entrySet()) { + NewFileKey newFileKey = keyMapEntry.getValue(); + if (newFileKey != null && nonSideloadedUriMap.containsKey(newFileKey)) { + onDeviceUriMap.put(keyMapEntry.getKey(), nonSideloadedUriMap.get(newFileKey)); + } + } + return onDeviceUriMap.build(); + }, + sequentialControlExecutor); + } + + /** + * Helper method to get a map of isolated file uris. + * + * <p>This method does not check whether or not isolated uris are allowed to be created/used, but + * simply returns all calculated isolated file uris. The caller is responsible for checking if the + * returned uris can/should be used! + */ + ImmutableMap<DataFile, Uri> getIsolatedFileUris(DataFileGroupInternal dataFileGroup) { + ImmutableMap.Builder<DataFile, Uri> isolatedFileUrisBuilder = ImmutableMap.builder(); + Uri isolatedRootUri = + FileGroupUtil.getIsolatedRootDirectory(context, instanceId, dataFileGroup); + for (DataFile dataFile : dataFileGroup.getFileList()) { + isolatedFileUrisBuilder.put( + dataFile, FileGroupUtil.appendIsolatedFileUri(isolatedRootUri, dataFile)); + } + return isolatedFileUrisBuilder.build(); + } + + /** + * Verify the given isolated uris point to the given on-device uris. + * + * <p>The verification steps include 1) ensuring each isolated uri exists; 2) each isolated uri + * points to the corresponding on-device uri. Isolated uris and on-device uris will be matched by + * their {@link DataFile} keys from the input maps. + * + * <p>Each verified isolated uri is included in the return map. If an isolated uri cannot be + * verified, no entry for the corresponding data file will be included in the return map. + * + * <p>If an entry for a DataFile key is missing from either input map, it is also omitted from the + * return map (i.e. this method returns an INNER JOIN of the two input maps) + * + * @return map of isolated uris which have been verified + */ + @RequiresApi(VERSION_CODES.LOLLIPOP) + ImmutableMap<DataFile, Uri> verifyIsolatedFileUris( + ImmutableMap<DataFile, Uri> isolatedFileUris, ImmutableMap<DataFile, Uri> onDeviceUris) { + ImmutableMap.Builder<DataFile, Uri> verifiedUriMapBuilder = ImmutableMap.builder(); + for (Entry<DataFile, Uri> onDeviceEntry : onDeviceUris.entrySet()) { + // Skip null/missing uris + if (onDeviceEntry.getValue() == null + || !isolatedFileUris.containsKey(onDeviceEntry.getKey())) { + continue; + } + + Uri isolatedUri = isolatedFileUris.get(onDeviceEntry.getKey()); + Uri onDeviceUri = onDeviceEntry.getValue(); + + try { + Uri targetFileUri = SymlinkUtil.readSymlink(context, isolatedUri); + if (fileStorage.exists(isolatedUri) + && targetFileUri.toString().equals(onDeviceUri.toString())) { + verifiedUriMapBuilder.put(onDeviceEntry.getKey(), isolatedUri); + } else { + LogUtil.e( + "%s verifyIsolatedFileUris unable to get isolated file uri! %s %s", + TAG, isolatedUri, onDeviceUri); + } + } catch (IOException e) { + LogUtil.e( + "%s verifyIsolatedFileUris unable to get isolated file uri! %s %s", + TAG, isolatedUri, onDeviceUri); + } + } + return verifiedUriMapBuilder.build(); + } + + /** * Get the current status of the file group. Since the status of the group is not stored in the * file group, this method iterates over all files and re-calculates the current status. * @@ -2300,9 +2439,9 @@ public class FileGroupManager { DataFileGroupInternal dataFileGroup) { return getFileGroupDownloadStatusIter( dataFileGroup, - /* downloadFailed = */ false, - /* downloadPending = */ false, - /* index = */ 0, + /* downloadFailed= */ false, + /* downloadPending= */ false, + /* index= */ 0, dataFileGroup.getFileCount()); } @@ -2354,7 +2493,7 @@ public class FileGroupManager { return getFileGroupDownloadStatusIter( dataFileGroup, downloadFailed, - /* downloadPending = */ true, + /* downloadPending= */ true, index + 1, fileCount); } else { @@ -2363,7 +2502,7 @@ public class FileGroupManager { TAG, dataFile.getFileId(), dataFileGroup.getGroupName()); return getFileGroupDownloadStatusIter( dataFileGroup, - /* downloadFailed = */ true, + /* downloadFailed= */ true, downloadPending, index + 1, fileCount); @@ -2395,9 +2534,6 @@ public class FileGroupManager { verifyAllPendingGroupsDownloaded(groupKeyList, customFileGroupValidator))); } - @SuppressWarnings("nullness") - // Suppress nullness warnings because otherwise static analysis would require us to falsely label - // verifyPendingGroupDownloaded with @NullableType private ListenableFuture<Void> verifyAllPendingGroupsDownloaded( List<GroupKey> groupKeyList, AsyncFunction<DataFileGroupInternal, Boolean> customFileGroupValidator) { @@ -2408,13 +2544,18 @@ public class FileGroupManager { } allFileFutures.add( transformSequentialAsync( - fileGroupsMetadata.read(groupKey), + getFileGroup(groupKey, /* downloaded= */ false), pendingGroup -> { + // If no pending group exists for this group key, skip the verification. if (pendingGroup == null) { - return immediateFuture(null); + return immediateFuture(GroupDownloadStatus.PENDING); } - return verifyPendingGroupDownloaded( - groupKey, pendingGroup, customFileGroupValidator); + return verifyGroupDownloaded( + groupKey, + pendingGroup, + /* removePendingVersion= */ true, + customFileGroupValidator, + DownloadStateLogger.forDownload(eventLogger)); })); } return PropagatedFutures.whenAllComplete(allFileFutures) @@ -2439,12 +2580,13 @@ public class FileGroupManager { LogUtil.d( "%s: Deleting file group %s for uninstalled app %s", TAG, key.getGroupName(), key.getOwnerPackage()); - eventLogger.logEventSampled(0); + eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); return transformSequentialAsync( fileGroupsMetadata.remove(key), removeSuccess -> { if (!removeSuccess) { - eventLogger.logEventSampled(0); + eventLogger.logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); } return immediateVoidFuture(); }); @@ -2493,14 +2635,16 @@ public class FileGroupManager { LogUtil.d( "%s: Deleting file group %s for removed account %s", TAG, key.getGroupName(), key.getOwnerPackage()); - logEventWithDataFileGroup(0, eventLogger, group); + logEventWithDataFileGroup( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, eventLogger, group); // Remove the group from fresh file groups if the account is removed. return transformSequentialAsync( fileGroupsMetadata.remove(key), removeSuccess -> { if (!removeSuccess) { - logEventWithDataFileGroup(0, eventLogger, group); + logEventWithDataFileGroup( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, eventLogger, group); } return immediateVoidFuture(); }); @@ -2542,7 +2686,7 @@ public class FileGroupManager { fileGroupsMetadata.write(pendingGroupKey, pendingGroup), writeSuccess -> { if (!writeSuccess) { - eventLogger.logEventSampled(0); + eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); return immediateFailedFuture(new IOException("Unable to update file group metadata")); } @@ -2562,8 +2706,8 @@ public class FileGroupManager { return transformSequential( fileGroupsMetadata.getAllFreshGroups(), pairs -> { - for (Pair<GroupKey, DataFileGroupInternal> pair : pairs) { - DataFileGroupInternal fileGroup = pair.second; + for (GroupKeyAndGroup pair : pairs) { + DataFileGroupInternal fileGroup = pair.dataFileGroup(); for (DataFile dataFile : fileGroup.getFileList()) { NewFileKey newFileKey = SharedFilesMetadata.createKeyFromDataFile( @@ -2576,31 +2720,33 @@ public class FileGroupManager { } /** Logs download failure remotely via {@code eventLogger}. */ + // incompatible argument for parameter code of logMddDownloadResult. + @SuppressWarnings("nullness:argument.type.incompatible") private ListenableFuture<Void> logDownloadFailure( GroupKey groupKey, DownloadException downloadException, long buildId, String variantId) { - DataDownloadFileGroupStats.Builder groupDetails = - DataDownloadFileGroupStats.newBuilder() - .setFileGroupName(groupKey.getGroupName()) - .setOwnerPackage(groupKey.getOwnerPackage()) - .setBuildId(buildId) - .setVariantId(variantId); + DataDownloadFileGroupStats.Builder groupDetails = + DataDownloadFileGroupStats.newBuilder() + .setFileGroupName(groupKey.getGroupName()) + .setOwnerPackage(groupKey.getOwnerPackage()) + .setBuildId(buildId) + .setVariantId(variantId); return transformSequentialAsync( fileGroupsMetadata.read(groupKey.toBuilder().setDownloaded(false).build()), dataFileGroup -> { - if (dataFileGroup != null) { - groupDetails.setFileGroupVersionNumber(dataFileGroup.getFileGroupVersionNumber()); - } + if (dataFileGroup != null) { + groupDetails.setFileGroupVersionNumber(dataFileGroup.getFileGroupVersionNumber()); + } - eventLogger.logMddDownloadResult( - MddDownloadResult.Code.forNumber(downloadException.getDownloadResultCode().getCode()), - groupDetails.build()); + eventLogger.logMddDownloadResult( + MddDownloadResult.Code.forNumber(downloadException.getDownloadResultCode().getCode()), + groupDetails.build()); return immediateVoidFuture(); }); } private ListenableFuture<Boolean> subscribeGroup(DataFileGroupInternal dataFileGroup) { - return subscribeGroup(dataFileGroup, /* index = */ 0, dataFileGroup.getFileCount()); + return subscribeGroup(dataFileGroup, /* index= */ 0, dataFileGroup.getFileCount()); } // Because the decision to continue iterating or not depends on the result of the asynchronous @@ -2637,7 +2783,7 @@ public class FileGroupManager { } } - private ListenableFuture<Boolean> isAddedGroupDuplicate( + private ListenableFuture<Optional<Integer>> isAddedGroupDuplicate( GroupKey groupKey, DataFileGroupInternal dataFileGroup) { // Search for a non-downloaded version of this group. GroupKey pendingGroupKey = groupKey.toBuilder().setDownloaded(false).build(); @@ -2653,9 +2799,9 @@ public class FileGroupManager { return transformSequentialAsync( fileGroupsMetadata.read(downloadedGroupKey), downloadedGroup -> { - boolean result = + Optional<Integer> result = (downloadedGroup == null) - ? false + ? Optional.of(0) : areSameGroup(dataFileGroup, downloadedGroup); return immediateFuture(result); }); @@ -2668,38 +2814,41 @@ public class FileGroupManager { * * @param newGroup The new config that we received for the client. * @param prevGroup The old config that we already have for the client. - * @return true if the new config contains an upgrade to any file. + * @return absent if the group is the same, otherwise a code for why the new config isn't the same */ - private static boolean areSameGroup( + private static Optional<Integer> areSameGroup( DataFileGroupInternal newGroup, DataFileGroupInternal prevGroup) { // We do not compare the protos directly and check individual fields because proto.equals // also compares extensions (and unknown fields). // TODO: Consider clearing extensions and then comparing protos. if (prevGroup.getBuildId() != newGroup.getBuildId()) { - return false; + return Optional.of(0); } if (!prevGroup.getVariantId().equals(newGroup.getVariantId())) { - return false; + return Optional.of(0); } if (prevGroup.getFileGroupVersionNumber() != newGroup.getFileGroupVersionNumber()) { - return false; + return Optional.of(0); } if (!hasSameFiles(newGroup, prevGroup)) { - return false; + return Optional.of(0); } if (prevGroup.getStaleLifetimeSecs() != newGroup.getStaleLifetimeSecs()) { - return false; + return Optional.of(0); } if (prevGroup.getExpirationDateSecs() != newGroup.getExpirationDateSecs()) { - return false; + return Optional.of(0); } if (!prevGroup.getDownloadConditions().equals(newGroup.getDownloadConditions())) { - return false; + return Optional.of(0); } if (!prevGroup.getAllowedReadersEnum().equals(newGroup.getAllowedReadersEnum())) { - return false; + return Optional.of(0); } - return true; +// if (!prevGroup.getExperimentInfo().equals(newGroup.getExperimentInfo())) { +// return Optional.of(0); +// } + return Optional.absent(); } /** @@ -2774,10 +2923,6 @@ public class FileGroupManager { groupKeyAndGroup -> { DataFileGroupInternal dataFileGroup = groupKeyAndGroup.dataFileGroup(); - if (dataFileGroup == null) { - return immediateVoidFuture(); - } - for (DataFile dataFile : dataFileGroup.getFileList()) { NewFileKey newFileKey = SharedFilesMetadata.createKeyFromDataFile( @@ -2788,7 +2933,8 @@ public class FileGroupManager { SharedFileMissingException.class, e -> { LogUtil.e("%s: Missing file. Logging and deleting file group.", TAG); - logEventWithDataFileGroup(0, eventLogger, dataFileGroup); + logEventWithDataFileGroup( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, eventLogger, dataFileGroup); if (flags.deleteFileGroupsWithFilesMissing()) { return transformSequentialAsync( @@ -2826,7 +2972,7 @@ public class FileGroupManager { } return transformSequentialAsync( - maybeVerifyIsolatedStructure(dataFileGroup, /*isDownloaded=*/ true), + maybeVerifyIsolatedStructure(dataFileGroup, /* isDownloaded= */ true), verified -> { if (!verified) { return PropagatedFluentFuture.from(createIsolatedFilePaths(dataFileGroup)) @@ -2848,19 +2994,6 @@ public class FileGroupManager { }); } - @AutoValue - abstract static class GroupKeyAndGroup { - static GroupKeyAndGroup create( - GroupKey groupKey, @Nullable DataFileGroupInternal dataFileGroup) { - return new AutoValue_FileGroupManager_GroupKeyAndGroup(groupKey, dataFileGroup); - } - - abstract GroupKey groupKey(); - - @Nullable - abstract DataFileGroupInternal dataFileGroup(); - } - private ListenableFuture<Void> iterateOverAllFileGroups( AsyncFunction<GroupKeyAndGroup, Void> processGroup) { @@ -2874,7 +3007,9 @@ public class FileGroupManager { transformSequentialAsync( fileGroupsMetadata.read(groupKey), dataFileGroup -> - processGroup.apply(GroupKeyAndGroup.create(groupKey, dataFileGroup)))); + (dataFileGroup != null) + ? processGroup.apply(GroupKeyAndGroup.create(groupKey, dataFileGroup)) + : immediateVoidFuture())); } return PropagatedFutures.whenAllComplete(allGroupsProcessed) .call(() -> null, sequentialControlExecutor); @@ -2889,22 +3024,21 @@ public class FileGroupManager { transformSequentialAsync( fileGroupsMetadata.getAllFreshGroups(), dataFileGroups -> { - ArrayList<Pair<GroupKey, DataFileGroupInternal>> sortedFileGroups = - new ArrayList<>(dataFileGroups); + ArrayList<GroupKeyAndGroup> sortedFileGroups = new ArrayList<>(dataFileGroups); Collections.sort( sortedFileGroups, (pairA, pairB) -> ComparisonChain.start() - .compare(pairA.first.getGroupName(), pairB.first.getGroupName()) - .compare(pairA.first.getAccount(), pairB.first.getAccount()) + .compare(pairA.groupKey().getGroupName(), pairB.groupKey().getGroupName()) + .compare(pairA.groupKey().getAccount(), pairB.groupKey().getAccount()) .result()); - for (Pair<GroupKey, DataFileGroupInternal> dataFileGroupPair : sortedFileGroups) { + for (GroupKeyAndGroup dataFileGroupPair : sortedFileGroups) { // TODO(b/131166925): MDD dump should not use lite proto toString. writer.format( "GroupName: %s\nAccount: %s\nDataFileGroup:\n %s\n\n", - dataFileGroupPair.first.getGroupName(), - dataFileGroupPair.first.getAccount(), - dataFileGroupPair.second.toString()); + dataFileGroupPair.groupKey().getGroupName(), + dataFileGroupPair.groupKey().getAccount(), + dataFileGroupPair.dataFileGroup().toString()); } return immediateVoidFuture(); }); @@ -2953,7 +3087,7 @@ public class FileGroupManager { } private static void logEventWithDataFileGroup( - int code, EventLogger eventLogger, DataFileGroupInternal fileGroup) { + MddClientEvent.Code code, EventLogger eventLogger, DataFileGroupInternal fileGroup) { eventLogger.logEventSampled( code, fileGroup.getGroupName(), diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/FileGroupsMetadata.java b/java/com/google/android/libraries/mobiledatadownload/internal/FileGroupsMetadata.java index af555b0..469a09c 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/FileGroupsMetadata.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/FileGroupsMetadata.java @@ -15,7 +15,7 @@ */ package com.google.android.libraries.mobiledatadownload.internal; -import android.util.Pair; +import com.google.android.libraries.mobiledatadownload.internal.collect.GroupKeyAndGroup; import com.google.common.util.concurrent.ListenableFuture; import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; @@ -71,7 +71,7 @@ public interface FileGroupsMetadata { * @return A future resolving to a list containing pairs of serialized GroupKeys and the * corresponding DataFileGroups. */ - ListenableFuture<List<Pair<GroupKey, DataFileGroupInternal>>> getAllFreshGroups(); + ListenableFuture<List<GroupKeyAndGroup>> getAllFreshGroups(); /** * Removes all entries with a key in keys from the SharedPreferencesFileGroupsMetadata's storage. diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/MddConstants.java b/java/com/google/android/libraries/mobiledatadownload/internal/MddConstants.java index fae84b2..539ac59 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/MddConstants.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/MddConstants.java @@ -58,4 +58,12 @@ public class MddConstants { public static final String SIDELOAD_FILE_URL_SCHEME = "file"; public static final String EMBEDDED_ASSET_URL_SCHEME = "asset"; + + /** + * Currently used in getFileGroup logging. If a matching file group is not found, build_id and + * file_group_version_number are set to below values for logging. + */ + public static final int FILE_GROUP_NOT_FOUND_BUILD_ID = -1; + + public static final int FILE_GROUP_NOT_FOUND_FILE_GROUP_VERSION_NUMBER = -1; } diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/MobileDataDownloadManager.java b/java/com/google/android/libraries/mobiledatadownload/internal/MobileDataDownloadManager.java index 7285ebe..b9496e2 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/MobileDataDownloadManager.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/MobileDataDownloadManager.java @@ -15,6 +15,9 @@ */ package com.google.android.libraries.mobiledatadownload.internal; +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.util.concurrent.Futures.getDone; +import static com.google.common.util.concurrent.Futures.immediateFailedFuture; import static com.google.common.util.concurrent.Futures.immediateFuture; import static com.google.common.util.concurrent.Futures.immediateVoidFuture; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; @@ -22,9 +25,6 @@ import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import android.content.Context; import android.content.SharedPreferences; import android.net.Uri; -import android.os.Build.VERSION; -import android.os.Build.VERSION_CODES; -import android.util.Pair; import androidx.annotation.VisibleForTesting; import com.google.android.libraries.mobiledatadownload.FileSource; import com.google.android.libraries.mobiledatadownload.Flags; @@ -33,8 +33,10 @@ import com.google.android.libraries.mobiledatadownload.annotations.InstanceId; import com.google.android.libraries.mobiledatadownload.file.transforms.TransformProtos; import com.google.android.libraries.mobiledatadownload.internal.FileGroupManager.GroupDownloadStatus; import com.google.android.libraries.mobiledatadownload.internal.annotations.SequentialControlExecutor; +import com.google.android.libraries.mobiledatadownload.internal.collect.GroupKeyAndGroup; import com.google.android.libraries.mobiledatadownload.internal.downloader.FileValidator; import com.google.android.libraries.mobiledatadownload.internal.experimentation.DownloadStageManager; +import com.google.android.libraries.mobiledatadownload.internal.logging.DownloadStateLogger; import com.google.android.libraries.mobiledatadownload.internal.logging.EventLogger; import com.google.android.libraries.mobiledatadownload.internal.logging.FileGroupStatsLogger; import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; @@ -49,21 +51,21 @@ import com.google.common.base.Optional; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.AsyncFunction; -import com.google.common.util.concurrent.FluentFuture; -import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.errorprone.annotations.CheckReturnValue; -import com.google.mobiledatadownload.TransformProto.Transforms; import com.google.mobiledatadownload.internal.MetadataProto.DataFile; import com.google.mobiledatadownload.internal.MetadataProto.DataFile.ChecksumType; import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; import com.google.mobiledatadownload.internal.MetadataProto.DownloadConditions; import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; +import com.google.mobiledatadownload.LogEnumsProto.MddClientEvent; +import com.google.mobiledatadownload.TransformProto.Transforms; import com.google.protobuf.Any; import java.io.IOException; import java.io.PrintWriter; import java.util.ArrayList; import java.util.List; +import java.util.Map.Entry; import java.util.concurrent.Executor; import javax.annotation.concurrent.NotThreadSafe; import javax.inject.Inject; @@ -167,11 +169,12 @@ public class MobileDataDownloadManager { if (isInitialized) { return immediateVoidFuture(); } - SharedPreferences prefs = - SharedPreferencesUtil.getSharedPreferences(context, MDD_MANAGER_METADATA, instanceId); - return PropagatedFluentFuture.from(Futures.immediateFuture(null)) + return PropagatedFluentFuture.from(immediateVoidFuture()) .transformAsync( voidArg -> { + SharedPreferences prefs = + SharedPreferencesUtil.getSharedPreferences( + context, MDD_MANAGER_METADATA, instanceId); // Offroad downloader migration. Since the migration has been enabled in gms // v18, most devices have migrated. For the remaining, we will clear MDD // storage. @@ -185,7 +188,7 @@ public class MobileDataDownloadManager { }, sequentialControlExecutor); } - return Futures.immediateFuture(null); + return immediateVoidFuture(); }, sequentialControlExecutor) .transformAsync( @@ -195,10 +198,11 @@ public class MobileDataDownloadManager { initSuccess -> { if (!initSuccess) { // This should be init before the shared file metadata. - LogUtil.w("%s Failed to init shared file manager.", TAG); + LogUtil.w( + "%s Clearing MDD since FileManager failed or needs migration.", TAG); return clearForInit(); } - return Futures.immediateVoidFuture(); + return immediateVoidFuture(); }, sequentialControlExecutor), sequentialControlExecutor) @@ -208,10 +212,11 @@ public class MobileDataDownloadManager { sharedFilesMetadata.init(), initSuccess -> { if (!initSuccess) { - LogUtil.w("%s Failed to init shared file metadata.", TAG); + LogUtil.w( + "%s Clearing MDD since FilesMetadata failed or needs migration.", TAG); return clearForInit(); } - return Futures.immediateVoidFuture(); + return immediateVoidFuture(); }, sequentialControlExecutor), sequentialControlExecutor) @@ -243,8 +248,7 @@ public class MobileDataDownloadManager { // instead of boolean for failure public ListenableFuture<Boolean> addGroupForDownload( GroupKey groupKey, DataFileGroupInternal dataFileGroup) { - return addGroupForDownloadInternal( - groupKey, dataFileGroup, unused -> Futures.immediateFuture(true)); + return addGroupForDownloadInternal(groupKey, dataFileGroup, unused -> immediateFuture(true)); } public ListenableFuture<Boolean> addGroupForDownloadInternal( @@ -258,55 +262,93 @@ public class MobileDataDownloadManager { // Check if the group we received is a valid group. if (!DataFileGroupValidator.isValidGroup(dataFileGroup, context, flags)) { eventLogger.logEventSampled( - 0, + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, dataFileGroup.getGroupName(), dataFileGroup.getFileGroupVersionNumber(), dataFileGroup.getBuildId(), dataFileGroup.getVariantId()); - return Futures.immediateFuture(false); + return immediateFuture(false); } DataFileGroupInternal populatedDataFileGroup = mayPopulateChecksum(dataFileGroup); try { - return PropagatedFutures.transformAsync( - fileGroupManager.addGroupForDownload(groupKey, populatedDataFileGroup), - addGroupForDownloadResult -> { - if (addGroupForDownloadResult) { - return PropagatedFutures.transform( - fileGroupManager.verifyPendingGroupDownloaded( - groupKey, populatedDataFileGroup, customFileGroupValidator), - verifyPendingGroupDownloadedResult -> { - if (verifyPendingGroupDownloadedResult - == GroupDownloadStatus.DOWNLOADED) { - eventLogger.logEventSampled( - 0, - populatedDataFileGroup.getGroupName(), - populatedDataFileGroup.getFileGroupVersionNumber(), - populatedDataFileGroup.getBuildId(), - populatedDataFileGroup.getVariantId()); - } - return true; - }, - sequentialControlExecutor); - } - return Futures.immediateFuture(true); - }, - sequentialControlExecutor); + return PropagatedFluentFuture.from( + fileGroupManager.addGroupForDownload(groupKey, populatedDataFileGroup)) + .transformAsync( + addGroupForDownloadResult -> { + if (addGroupForDownloadResult) { + return maybeMarkPendingGroupAsDownloadedImmediately( + groupKey, customFileGroupValidator); + } + return immediateVoidFuture(); + }, + sequentialControlExecutor) + .transform(unused -> true, sequentialControlExecutor); } catch (ExpiredFileGroupException | UninstalledAppException | ActivationRequiredForGroupException e) { LogUtil.w("%s %s", TAG, e.getClass()); - return Futures.immediateFailedFuture(e); + return immediateFailedFuture(e); } catch (IOException e) { LogUtil.e("%s %s", TAG, e.getClass()); silentFeedback.send(e, "Failed to add group to MDD"); - return Futures.immediateFailedFuture(e); + return immediateFailedFuture(e); } }, sequentialControlExecutor); } /** + * Helper method to mark a group as downloaded immediately. + * + * <p>This method checks if a pending group is already downloaded and updates its state in MDD's + * metadata if it is downloaded. Additionally, a download complete immediate event is logged for + * this case. + * + * <p>If no pending version of the group is available, this method is a no-op. + * + * <p>NOTE: This method is only meant to be called during addFileGroup, where it makes sense to + * log the immediate download complete event. + */ + private ListenableFuture<Void> maybeMarkPendingGroupAsDownloadedImmediately( + GroupKey groupKey, AsyncFunction<DataFileGroupInternal, Boolean> customFileGroupValidator) { + ListenableFuture<@NullableType DataFileGroupInternal> pendingGroupFuture = + fileGroupManager.getFileGroup(groupKey, /* downloaded= */ false); + return PropagatedFluentFuture.from(pendingGroupFuture) + .transformAsync( + pendingGroup -> { + if (pendingGroup == null) { + // send pending state to skip logging the event + return immediateFuture(GroupDownloadStatus.PENDING); + } + // Verify the group is downloaded (and commit this to metadata). + return fileGroupManager.verifyGroupDownloaded( + groupKey, + pendingGroup, + /* removePendingVersion= */ true, + customFileGroupValidator, + DownloadStateLogger.forDownload(eventLogger)); + }, + sequentialControlExecutor) + .transformAsync( + verifyPendingGroupDownloadedResult -> { + if (verifyPendingGroupDownloadedResult == GroupDownloadStatus.DOWNLOADED) { + // Use checkNotNull to satisfy nullness checker -- if the group status is + // downloaded, pendingGroup must be non-null. + DataFileGroupInternal group = checkNotNull(getDone(pendingGroupFuture)); + eventLogger.logEventSampled( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + group.getGroupName(), + group.getFileGroupVersionNumber(), + group.getBuildId(), + group.getVariantId()); + } + return immediateVoidFuture(); + }, + sequentialControlExecutor); + } + + /** * Removes the file group from MDD with the given group key. This will cancel any ongoing download * of the file group. * @@ -321,7 +363,7 @@ public class MobileDataDownloadManager { throws SharedFileMissingException, IOException { LogUtil.d("%s removeFileGroup %s", TAG, groupKey.getGroupName()); - return Futures.transformAsync( + return PropagatedFutures.transformAsync( init(), voidArg -> fileGroupManager.removeFileGroup(groupKey, pendingOnly), sequentialControlExecutor); @@ -339,7 +381,7 @@ public class MobileDataDownloadManager { public ListenableFuture<Void> removeFileGroups(List<GroupKey> groupKeys) { LogUtil.d("%s removeFileGroups for %d groups", TAG, groupKeys.size()); - return Futures.transformAsync( + return PropagatedFutures.transformAsync( init(), voidArg -> fileGroupManager.removeFileGroups(groupKeys), sequentialControlExecutor); } @@ -356,65 +398,115 @@ public class MobileDataDownloadManager { GroupKey groupKey, boolean downloaded) { LogUtil.d("%s getFileGroup %s %s", TAG, groupKey.getGroupName(), groupKey.getOwnerPackage()); - return Futures.transformAsync( + return PropagatedFutures.transformAsync( init(), voidArg -> fileGroupManager.getFileGroup(groupKey, downloaded), sequentialControlExecutor); } /** Returns a future resolving to a list of all pending and downloaded groups in MDD. */ - public ListenableFuture<List<Pair<GroupKey, DataFileGroupInternal>>> getAllFreshGroups() { + public ListenableFuture<List<GroupKeyAndGroup>> getAllFreshGroups() { LogUtil.d("%s getAllFreshGroups", TAG); - return Futures.transformAsync( + return PropagatedFutures.transformAsync( init(), voidArg -> fileGroupsMetadata.getAllFreshGroups(), sequentialControlExecutor); } /** - * Returns a future resolving to the URI at which the given data file is located on the disc. - * Returns null if there was error in generating the URI. + * Returns a map of on-device URIs for the requested {@link DataFileGroupInternal}. + * + * <p>If a DataFile does not have an on-device URI (e.g. the download for the file is not + * completed), The returned map will not contain an entry for that DataFile. + * + * <p>If the group supports isolated structures, verification of the isolated structure can be + * controlled. If a file fails the verification (either the symlink is not created, or does not + * point to the correct location), it will be omitted from the map. + * + * <p>NOTE: Verification should only be turned off on critical access paths where latency must be + * minimized. This may lead to an edge case where the isolated structure becomes broken and/or + * corrupted until MDD can fix the structure in its daily maintenance task. */ - public ListenableFuture<@NullableType Uri> getDataFileUri( - DataFile dataFile, DataFileGroupInternal dataFileGroup) { - LogUtil.d("%s getDataFileUri %s %s", TAG, dataFile.getFileId(), dataFileGroup.getGroupName()); - return Futures.transformAsync( - init(), - voidArg -> { - ListenableFuture<@NullableType Uri> onDeviceUriFuture = - fileGroupManager.getOnDeviceUri(dataFile, dataFileGroup); - return Futures.transform( - onDeviceUriFuture, - onDeviceUri -> { - Uri finalOnDeviceUri = onDeviceUri; - // Check if file group should use isolated uri - if (finalOnDeviceUri != null - && FileGroupUtil.isIsolatedStructureAllowed(dataFileGroup) - && VERSION.SDK_INT >= VERSION_CODES.LOLLIPOP) { - try { - finalOnDeviceUri = - fileGroupManager.getAndVerifyIsolatedFileUri( - finalOnDeviceUri, dataFile, dataFileGroup); - } catch (IOException e) { - LogUtil.e( - e, - "%s getDataFileUri %s %s unable to get isolated file uri!", - TAG, - dataFile.getFileId(), - dataFileGroup.getGroupName()); - finalOnDeviceUri = null; - } + public ListenableFuture<ImmutableMap<DataFile, Uri>> getDataFileUris( + DataFileGroupInternal dataFileGroup, boolean verifyIsolatedStructure) { + LogUtil.d("%s: getDataFileUris %s", TAG, dataFileGroup.getGroupName()); + + boolean useIsolatedStructure = FileGroupUtil.isIsolatedStructureAllowed(dataFileGroup); + + // If isolated structure is supported, get the isolated uris (symlinks which point to the + // on-device location). These can be calculated synchronously and before init since they only + // require the file group metadata. + ImmutableMap.Builder<DataFile, Uri> isolatedUriMapBuilder = ImmutableMap.builder(); + if (useIsolatedStructure) { + isolatedUriMapBuilder.putAll(fileGroupManager.getIsolatedFileUris(dataFileGroup)); + } + ImmutableMap<DataFile, Uri> isolatedUriMap = isolatedUriMapBuilder.build(); + + return PropagatedFluentFuture.from(init()) + .transformAsync( + unused -> { + // Lookup on-device uris only if required to reduce latency. On-device lookups happen + // asynchronously since we need to access the latest underlying file metadata. + // 1. The group does not support an isolated structure + // 2. The group supports an isolated structure AND verification of that structure + // should occur. + if (!useIsolatedStructure || verifyIsolatedStructure) { + return fileGroupManager.getOnDeviceUris(dataFileGroup); + } + + // Return an empty map here since we won't be using the on-device uris. + return immediateFuture(ImmutableMap.of()); + }, + sequentialControlExecutor) + .transform( + onDeviceUriMap -> { + if (useIsolatedStructure) { + if (verifyIsolatedStructure) { + // Return verified map of isolated uris. + return fileGroupManager.verifyIsolatedFileUris(isolatedUriMap, onDeviceUriMap); } - if (finalOnDeviceUri != null && dataFile.hasReadTransforms()) { - finalOnDeviceUri = - applyTransformsToFileUri(finalOnDeviceUri, dataFile.getReadTransforms()); + // Verification not required, return isolated uris. + return isolatedUriMap; + } + + // Isolated structure are not in use, return on-device uris. + return onDeviceUriMap; + }, + sequentialControlExecutor) + .transform( + selectedUriMap -> { + // Before returning uri map, apply read transforms if required. + ImmutableMap.Builder<DataFile, Uri> finalUriMapBuilder = ImmutableMap.builder(); + for (Entry<DataFile, Uri> entry : selectedUriMap.entrySet()) { + DataFile dataFile = entry.getKey(); + // Skip entries which have a null uri value. + if (entry.getValue() == null) { + continue; + } + if (dataFile.hasReadTransforms()) { + finalUriMapBuilder.put( + dataFile, + applyTransformsToFileUri(entry.getValue(), dataFile.getReadTransforms())); + } else { + finalUriMapBuilder.put(entry); } + } + return finalUriMapBuilder.build(); + }, + sequentialControlExecutor); + } - return finalOnDeviceUri; - }, - sequentialControlExecutor); - }, - sequentialControlExecutor); + /** + * Convenience method for {@link #getDataFileUris(DataFileGroupInternal, boolean)} when only a + * single data file is required. + */ + public ListenableFuture<@NullableType Uri> getDataFileUri( + DataFile dataFile, DataFileGroupInternal dataFileGroup, boolean verifyIsolatedStructure) { + LogUtil.d("%s getDataFileUri %s %s", TAG, dataFile.getFileId(), dataFileGroup.getGroupName()); + return PropagatedFutures.transform( + getDataFileUris(dataFileGroup, verifyIsolatedStructure), + dataFileUris -> dataFileUris.get(dataFile), + directExecutor()); } private Uri applyTransformsToFileUri(Uri fileUri, Transforms transforms) { @@ -428,7 +520,7 @@ public class MobileDataDownloadManager { } /** - * Import inline files into an exising DataFileGroup and update its metadata accordingly. + * Import inline files into an existing DataFileGroup and update its metadata accordingly. * * @param groupKey The key of file group to update * @param buildId build id to identify the file group to update @@ -448,7 +540,7 @@ public class MobileDataDownloadManager { Optional<Any> customPropertyOptional, AsyncFunction<DataFileGroupInternal, Boolean> customFileGroupValidator) { LogUtil.d("%s: importFiles %s %s", TAG, groupKey.getGroupName(), groupKey.getOwnerPackage()); - return Futures.transformAsync( + return PropagatedFutures.transformAsync( init(), voidArg -> fileGroupManager.importFilesIntoFileGroup( @@ -476,7 +568,7 @@ public class MobileDataDownloadManager { AsyncFunction<DataFileGroupInternal, Boolean> customFileGroupValidator) { LogUtil.d( "%s downloadFileGroup %s %s", TAG, groupKey.getGroupName(), groupKey.getOwnerPackage()); - return Futures.transformAsync( + return PropagatedFutures.transformAsync( init(), voidArg -> fileGroupManager.downloadFileGroup( @@ -494,7 +586,7 @@ public class MobileDataDownloadManager { public ListenableFuture<Boolean> setGroupActivation(GroupKey groupKey, boolean activation) { LogUtil.d( "%s setGroupActivation %s %s", TAG, groupKey.getGroupName(), groupKey.getOwnerPackage()); - return Futures.transformAsync( + return PropagatedFutures.transformAsync( init(), voidArg -> fileGroupManager.setGroupActivation(groupKey, activation), sequentialControlExecutor); @@ -509,11 +601,11 @@ public class MobileDataDownloadManager { public ListenableFuture<Void> downloadAllPendingGroups( boolean onWifi, AsyncFunction<DataFileGroupInternal, Boolean> customFileGroupValidator) { LogUtil.d("%s downloadAllPendingGroups on wifi = %s", TAG, onWifi); - return Futures.transformAsync( + return PropagatedFutures.transformAsync( init(), voidArg -> { if (flags.mddEnableDownloadPendingGroups()) { - eventLogger.logEventSampled(0); + eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); return fileGroupManager.scheduleAllPendingGroupsForDownload( onWifi, customFileGroupValidator); } @@ -529,11 +621,11 @@ public class MobileDataDownloadManager { public ListenableFuture<Void> verifyAllPendingGroups( AsyncFunction<DataFileGroupInternal, Boolean> customFileGroupValidator) { LogUtil.d("%s verifyAllPendingGroups", TAG); - return Futures.transformAsync( + return PropagatedFutures.transformAsync( init(), voidArg -> { if (flags.mddEnableVerifyPendingGroups()) { - eventLogger.logEventSampled(0); + eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); return fileGroupManager.verifyAllPendingGroupsDownloaded(customFileGroupValidator); } return immediateVoidFuture(); @@ -552,7 +644,7 @@ public class MobileDataDownloadManager { public ListenableFuture<Void> maintenance() { LogUtil.d("%s Running maintenance", TAG); - return FluentFuture.from(init()) + return PropagatedFluentFuture.from(init()) .transformAsync(voidArg -> getAndResetDaysSinceLastMaintenance(), directExecutor()) .transformAsync( daysSinceLastLog -> { @@ -582,7 +674,7 @@ public class MobileDataDownloadManager { if (flags.mddEnableGarbageCollection()) { maintenanceFutures.add(expirationHandler.updateExpiration()); - eventLogger.logEventSampled(0); + eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); } // Log daily file group stats. @@ -601,18 +693,27 @@ public class MobileDataDownloadManager { context, MDD_MANAGER_METADATA, instanceId); prefs.edit().remove(MDD_PH_CONFIG_VERSION).remove(MDD_PH_CONFIG_VERSION_TS).commit(); - return Futures.whenAllComplete(maintenanceFutures) + return PropagatedFutures.whenAllComplete(maintenanceFutures) .call(() -> null, sequentialControlExecutor); }, sequentialControlExecutor); } + /** + * Removes expired FileGroups (whether active or stale) and deletes files no longer referenced by + * a FileGroup. + */ + public ListenableFuture<Void> removeExpiredGroupsAndFiles() { + return PropagatedFluentFuture.from(init()) + .transformAsync(voidArg -> expirationHandler.updateExpiration(), sequentialControlExecutor); + } + /** Dumps the current internal state of the MDD manager. */ public ListenableFuture<Void> dump(final PrintWriter writer) { - return Futures.transformAsync( + return PropagatedFutures.transformAsync( init(), voidArg -> - Futures.transformAsync( + PropagatedFutures.transformAsync( fileGroupManager.dump(writer), voidParam -> sharedFileManager.dump(writer), sequentialControlExecutor), @@ -622,7 +723,7 @@ public class MobileDataDownloadManager { /** Checks to see if a flag change requires MDD to clear its data. */ public ListenableFuture<Void> checkResetTrigger() { LogUtil.d("%s checkResetTrigger", TAG); - return Futures.transformAsync( + return PropagatedFutures.transformAsync( init(), voidArg -> { SharedPreferences prefs = @@ -637,7 +738,7 @@ public class MobileDataDownloadManager { if (savedResetValue < currentResetValue) { prefs.edit().putInt(RESET_TRIGGER, currentResetValue).commit(); LogUtil.d("%s Received reset trigger. Clearing all Mdd data.", TAG); - eventLogger.logEventSampled(0); + eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); return clearAllFilesAndMetadata(); } return immediateVoidFuture(); @@ -692,12 +793,12 @@ public class MobileDataDownloadManager { /* Clear all metadata and files, also cancel pending download. */ private ListenableFuture<Void> clearAllFilesAndMetadata() { - return Futures.transformAsync( + return PropagatedFutures.transformAsync( // Need to cancel download after MDD is already initialized. sharedFileManager.cancelDownloadAndClear(), voidArg1 -> // The metadata files should be cleared after the classes have been cleared. - Futures.transformAsync( + PropagatedFutures.transformAsync( sharedFilesMetadata.clear(), voidArg2 -> fileGroupsMetadata.clear(), sequentialControlExecutor), @@ -772,7 +873,7 @@ public class MobileDataDownloadManager { return immediateFuture(DEFAULT_DAYS_SINCE_LAST_MAINTENANCE); } - return FluentFuture.from(loggingStateStore.getAndResetDaysSinceLastMaintenance()) + return PropagatedFluentFuture.from(loggingStateStore.getAndResetDaysSinceLastMaintenance()) .catching( IOException.class, exception -> { diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/SharedFileManager.java b/java/com/google/android/libraries/mobiledatadownload/internal/SharedFileManager.java index c0804b7..2c1775d 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/SharedFileManager.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/SharedFileManager.java @@ -15,7 +15,11 @@ */ package com.google.android.libraries.mobiledatadownload.internal; +import static com.google.common.util.concurrent.Futures.getDone; +import static com.google.common.util.concurrent.Futures.immediateFailedFuture; +import static com.google.common.util.concurrent.Futures.immediateFuture; import static com.google.common.util.concurrent.Futures.immediateVoidFuture; +import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import android.content.Context; import android.content.SharedPreferences; @@ -45,8 +49,11 @@ import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; import com.google.android.libraries.mobiledatadownload.internal.util.DirectoryUtil; import com.google.android.libraries.mobiledatadownload.internal.util.SharedPreferencesUtil; import com.google.android.libraries.mobiledatadownload.monitor.DownloadProgressMonitor; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFluentFuture; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; import com.google.common.base.Optional; -import com.google.common.util.concurrent.FluentFuture; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.errorprone.annotations.CheckReturnValue; @@ -61,6 +68,7 @@ import com.google.mobiledatadownload.internal.MetadataProto.FileStatus; import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; import com.google.mobiledatadownload.internal.MetadataProto.NewFileKey; import com.google.mobiledatadownload.internal.MetadataProto.SharedFile; +import com.google.mobiledatadownload.LogEnumsProto.MddClientEvent; import java.io.IOException; import java.io.PrintWriter; import java.util.ArrayList; @@ -166,7 +174,7 @@ public class SharedFileManager { sharedFileManagerMetadata.edit().remove(PREFS_KEY_MIGRATED_TO_NEW_FILE_KEY).commit(); } - return Futures.immediateFuture(true); + return immediateFuture(true); } /** @@ -178,12 +186,12 @@ public class SharedFileManager { */ // TODO - refactor to throw Exception when write to SharedPreferences fails public ListenableFuture<Boolean> reserveFileEntry(NewFileKey newFileKey) { - return Futures.transformAsync( + return PropagatedFutures.transformAsync( sharedFilesMetadata.read(newFileKey), sharedFile -> { if (sharedFile != null) { // There's already an entry for this file. Nothing to do here. - return Futures.immediateFuture(true); + return immediateFuture(true); } // Set the file name and update the metadata file. SharedPreferences sharedFileManagerMetadata = @@ -198,7 +206,7 @@ public class SharedFileManager { .commit()) { // TODO(b/131166925): MDD dump should not use lite proto toString. LogUtil.e("%s: Unable to update file name %s", TAG, newFileKey); - return Futures.immediateFuture(false); + return immediateFuture(false); } String fileName = FILE_NAME_PREFIX + nextFileName; @@ -207,7 +215,7 @@ public class SharedFileManager { .setFileStatus(FileStatus.SUBSCRIBED) .setFileName(fileName) .build(); - return Futures.transformAsync( + return PropagatedFutures.transformAsync( sharedFilesMetadata.write(newFileKey, sharedFile), writeSuccess -> { if (!writeSuccess) { @@ -215,9 +223,9 @@ public class SharedFileManager { LogUtil.e( "%s: Unable to write back subscription for file entry with %s", TAG, newFileKey); - return Futures.immediateFuture(false); + return immediateFuture(false); } - return Futures.immediateFuture(true); + return immediateFuture(true); }, sequentialControlExecutor); }, @@ -239,13 +247,13 @@ public class SharedFileManager { @Nullable DownloadConditions downloadConditions, FileSource inlineFileSource) { if (!dataFile.getUrlToDownload().startsWith(MddConstants.INLINE_FILE_URL_SCHEME)) { - return Futures.immediateFailedFuture( + return immediateFailedFuture( DownloadException.builder() .setDownloadResultCode(DownloadResultCode.INVALID_INLINE_FILE_URL_SCHEME) .setMessage("Importing an inline file requires inlinefile scheme") .build()); } - return Futures.transformAsync( + return PropagatedFutures.transformAsync( sharedFilesMetadata.read(newFileKey), sharedFile -> { if (sharedFile == null) { @@ -254,7 +262,7 @@ public class SharedFileManager { TAG, dataFile.getFileId()); SharedFileMissingException cause = new SharedFileMissingException(); // TODO(b/167582815): Log to Clearcut - return Futures.immediateFailedFuture( + return immediateFailedFuture( DownloadException.builder() .setDownloadResultCode(DownloadResultCode.SHARED_FILE_NOT_FOUND_ERROR) .setCause(cause) @@ -274,7 +282,7 @@ public class SharedFileManager { sharedFile.getFileName(), dataFile.getDownloadedFileChecksum()) : sharedFile.getFileName(); - return Futures.transformAsync( + return PropagatedFutures.transformAsync( getDataFileGroupOrDefault(groupKey), dataFileGroup -> getImportFuture( @@ -313,7 +321,7 @@ public class SharedFileManager { ListenableFuture<Uri> downloadFileOnDeviceUriFuture = getDownloadFileOnDeviceUri( newFileKey.getAllowedReaders(), downloadFileName, dataFile.getChecksum()); - return FluentFuture.from(downloadFileOnDeviceUriFuture) + return PropagatedFluentFuture.from(downloadFileOnDeviceUriFuture) .transformAsync( unused -> { sharedFileBuilder.setFileStatus(FileStatus.DOWNLOAD_IN_PROGRESS); @@ -327,7 +335,7 @@ public class SharedFileManager { sequentialControlExecutor) .transformAsync( unused -> { - Uri downloadFileOnDeviceUri = Futures.getDone(downloadFileOnDeviceUriFuture); + Uri downloadFileOnDeviceUri = getDone(downloadFileOnDeviceUriFuture); DownloaderCallback downloaderCallback = new DownloaderCallbackImpl( sharedFilesMetadata, @@ -345,6 +353,7 @@ public class SharedFileManager { // progress here. return fileDownloader.startCopying( + newFileKey.getChecksum(), downloadFileOnDeviceUri, dataFile.getUrlToDownload(), dataFile.getByteSize(), @@ -376,7 +385,7 @@ public class SharedFileManager { int trafficTag, List<ExtraHttpHeader> extraHttpHeaders) { if (dataFile.getUrlToDownload().startsWith(MddConstants.INLINE_FILE_URL_SCHEME)) { - return Futures.immediateFailedFuture( + return immediateFailedFuture( DownloadException.builder() .setDownloadResultCode(DownloadResultCode.INVALID_INLINE_FILE_URL_SCHEME) .setMessage( @@ -384,77 +393,134 @@ public class SharedFileManager { + " instead.") .build()); } - return Futures.transformAsync( - sharedFilesMetadata.read(newFileKey), - sharedFile -> { - if (sharedFile == null) { - // TODO(b/131166925): MDD dump should not use lite proto toString. - LogUtil.e( - "%s: Start download called on file that doesn't exists. Key = %s!", - TAG, newFileKey); - SharedFileMissingException cause = new SharedFileMissingException(); - silentFeedback.send(cause, "Shared file not found in downloadFileGroup"); - return Futures.immediateFailedFuture( - DownloadException.builder() - .setDownloadResultCode(DownloadResultCode.SHARED_FILE_NOT_FOUND_ERROR) - .setCause(cause) - .build()); - } - // If we have already downloaded the file, then return. - if (sharedFile.getFileStatus() == FileStatus.DOWNLOAD_COMPLETE) { - if (downloadMonitorOptional.isPresent()) { - // For the downloaded file, we don't need to monitor the file change. We just need to - // inform the monitor about its current size. - downloadMonitorOptional - .get() - .notifyCurrentFileSize(groupKey.getGroupName(), dataFile.getByteSize()); - } - return immediateVoidFuture(); - } + // Start futures in parallel for various calculated properties. + ListenableFuture<SharedFile> sharedFileFuture = getSharedFile(newFileKey); + + ListenableFuture<@NullableType DeltaFile> firstDeltaFileFuture = + findFirstDeltaFileWithBaseFileDownloaded(dataFile, newFileKey.getAllowedReaders()); + + ListenableFuture<String> downloadFileNameFuture = + PropagatedFutures.whenAllSucceed(sharedFileFuture, firstDeltaFileFuture) + .call( + () -> { + String downloadFileName = getDone(sharedFileFuture).getFileName(); + DeltaFile deltaFile = getDone(firstDeltaFileFuture); + if (deltaFile != null) { + downloadFileName = + FileNameUtil.getTempFileNameWithDownloadedFileChecksum( + downloadFileName, deltaFile.getChecksum()); + } else if (dataFile.hasDownloadTransforms()) { + downloadFileName = + FileNameUtil.getTempFileNameWithDownloadedFileChecksum( + downloadFileName, dataFile.getDownloadedFileChecksum()); + } + return downloadFileName; + }, + directExecutor()); + + ListenableFuture<Uri> downloadFileOnDeviceUriFuture = + PropagatedFutures.transformAsync( + downloadFileNameFuture, + downloadFileName -> + getDownloadFileOnDeviceUri( + newFileKey.getAllowedReaders(), downloadFileName, dataFile.getChecksum()), + sequentialControlExecutor); + + ListenableFuture<DataFileGroupInternal> dataFileGroupFuture = + getDataFileGroupOrDefault(groupKey); + + // Combine all futures together so all complete successfully before continuing + ListenableFuture<Void> combinedPropertiesFuture = + PropagatedFutures.whenAllSucceed( + sharedFileFuture, + firstDeltaFileFuture, + downloadFileNameFuture, + downloadFileOnDeviceUriFuture, + dataFileGroupFuture) + .callAsync(Futures::immediateVoidFuture, directExecutor()); + + return PropagatedFluentFuture.from(combinedPropertiesFuture) + .transformAsync( + unused -> { + SharedFile sharedFile = getDone(sharedFileFuture); + DeltaFile deltaFile = getDone(firstDeltaFileFuture); + String downloadFileName = getDone(downloadFileNameFuture); + Uri downloadFileOnDeviceUri = getDone(downloadFileOnDeviceUriFuture); + DataFileGroupInternal dataFileGroup = getDone(dataFileGroupFuture); - return Futures.transformAsync( - findFirstDeltaFileWithBaseFileDownloaded(dataFile, newFileKey.getAllowedReaders()), - deltaFile -> { - SharedFile.Builder sharedFileBuilder = sharedFile.toBuilder(); - String downloadFileName = sharedFile.getFileName(); - if (deltaFile != null) { - downloadFileName = - FileNameUtil.getTempFileNameWithDownloadedFileChecksum( - downloadFileName, deltaFile.getChecksum()); - } else if (dataFile.hasDownloadTransforms()) { - downloadFileName = - FileNameUtil.getTempFileNameWithDownloadedFileChecksum( - downloadFileName, dataFile.getDownloadedFileChecksum()); + // Check if download is complete + if (sharedFile.getFileStatus() == FileStatus.DOWNLOAD_COMPLETE) { + if (downloadMonitorOptional.isPresent()) { + // For the downloaded file, we don't need to monitor the file change. We just need + // to inform the monitor about its current size. + downloadMonitorOptional + .get() + .notifyCurrentFileSize(groupKey.getGroupName(), dataFile.getByteSize()); } + return immediateVoidFuture(); + } - // Variables captured in lambdas must be effectively final. - String downloadFileNameCapture = downloadFileName; - return Futures.transformAsync( - getDataFileGroupOrDefault(groupKey), - dataFileGroup -> - getDownloadFuture( - sharedFileBuilder, - newFileKey, - downloadFileNameCapture, - dataFileGroup.getFileGroupVersionNumber(), - dataFileGroup.getBuildId(), - dataFileGroup.getVariantId(), - groupKey, - dataFile, - deltaFile, - downloadConditions, - trafficTag, - extraHttpHeaders), + // Check if a download is already in progress + if (sharedFile.getFileStatus() == FileStatus.DOWNLOAD_IN_PROGRESS) { + return PropagatedFutures.transformAsync( + fileDownloader.getInProgressFuture( + newFileKey.getChecksum(), downloadFileOnDeviceUri), + inProgressFuture -> { + if (inProgressFuture.isPresent()) { + mayNotifyCurrentSizeOfPartiallyDownloadedFile( + groupKey, downloadFileOnDeviceUri); + return inProgressFuture.get(); + } + return getDownloadFuture( + newFileKey, + downloadFileName, + dataFileGroup.getFileGroupVersionNumber(), + dataFileGroup.getBuildId(), + dataFileGroup.getVariantId(), + groupKey, + dataFile, + deltaFile, + downloadConditions, + trafficTag, + extraHttpHeaders); + }, sequentialControlExecutor); - }, - sequentialControlExecutor); - }, - sequentialControlExecutor); + } + + // Download is not in progress, start it. + return getDownloadFuture( + newFileKey, + downloadFileName, + dataFileGroup.getFileGroupVersionNumber(), + dataFileGroup.getBuildId(), + dataFileGroup.getVariantId(), + groupKey, + dataFile, + deltaFile, + downloadConditions, + trafficTag, + extraHttpHeaders); + }, + sequentialControlExecutor) + .catchingAsync( + SharedFileMissingException.class, + ex -> { + // TODO(b/131166925): MDD dump should not use lite proto toString. + LogUtil.e( + "%s: Start download called on file that doesn't exist. Key = %s!", + TAG, newFileKey); + silentFeedback.send(ex, "Shared file not found in downloadFileGroup"); + return immediateFailedFuture( + DownloadException.builder() + .setDownloadResultCode(DownloadResultCode.SHARED_FILE_NOT_FOUND_ERROR) + .setCause(ex) + .build()); + }, + sequentialControlExecutor); } private ListenableFuture<Void> getDownloadFuture( - SharedFile.Builder sharedFileBuilder, NewFileKey newFileKey, String downloadFileName, int fileGroupVersionNumber, @@ -466,91 +532,114 @@ public class SharedFileManager { @Nullable DownloadConditions downloadConditions, int trafficTag, List<ExtraHttpHeader> extraHttpHeaders) { - ListenableFuture<Uri> downloadFileOnDeviceUriFuture = - getDownloadFileOnDeviceUri( - newFileKey.getAllowedReaders(), downloadFileName, dataFile.getChecksum()); - return FluentFuture.from(downloadFileOnDeviceUriFuture) - .transformAsync( - unused -> { - sharedFileBuilder.setFileStatus(FileStatus.DOWNLOAD_IN_PROGRESS); + // It's possible to hit a race condition where the caller of this method sees the file as not + // downloaded and by the time this method is executed, the file is already downloaded. + // + // Check the shared file status before starting the download to confirm it is not downloaded and + // a download is not already in progress. + return PropagatedFutures.transformAsync( + getSharedFile(newFileKey), + latestSharedFile -> { + if (latestSharedFile.getFileStatus() == FileStatus.DOWNLOAD_COMPLETE) { + return immediateVoidFuture(); + } - // Ignoring failure to write back here, as it will just result in one extra try to - // download the file. - return sharedFilesMetadata.write(newFileKey, sharedFileBuilder.build()); - }, - sequentialControlExecutor) - .transformAsync( - unused -> { - Uri downloadFileOnDeviceUri = Futures.getDone(downloadFileOnDeviceUriFuture); - ListenableFuture<Void> fileDownloadFuture; - if (!deltaDecoderOptional.isPresent() || deltaFile == null) { - // Download full file when delta file is null - DownloaderCallback downloaderCallback = - new DownloaderCallbackImpl( - sharedFilesMetadata, - fileStorage, - dataFile, - newFileKey.getAllowedReaders(), - eventLogger, - groupKey, - fileGroupVersionNumber, - buildId, - variantId, - flags, - sequentialControlExecutor); + // Download is not complete, proceed with starting the future. + SharedFile.Builder sharedFileBuilder = latestSharedFile.toBuilder(); + ListenableFuture<Uri> downloadFileOnDeviceUriFuture = + getDownloadFileOnDeviceUri( + newFileKey.getAllowedReaders(), downloadFileName, dataFile.getChecksum()); + return PropagatedFluentFuture.from(downloadFileOnDeviceUriFuture) + .transformAsync( + unused -> { + sharedFileBuilder.setFileStatus(FileStatus.DOWNLOAD_IN_PROGRESS); - mayNotifyCurrentSizeOfPartiallyDownloadedFile(groupKey, downloadFileOnDeviceUri); + // Ignoring failure to write back here, as it will just result in one + // extra try + // to download the file. + return sharedFilesMetadata.write(newFileKey, sharedFileBuilder.build()); + }, + sequentialControlExecutor) + .transformAsync( + unused -> { + Uri downloadFileOnDeviceUri = getDone(downloadFileOnDeviceUriFuture); + ListenableFuture<Void> fileDownloadFuture; + if (!deltaDecoderOptional.isPresent() || deltaFile == null) { + // Download full file when delta file is null + DownloaderCallback downloaderCallback = + new DownloaderCallbackImpl( + sharedFilesMetadata, + fileStorage, + dataFile, + newFileKey.getAllowedReaders(), + eventLogger, + groupKey, + fileGroupVersionNumber, + buildId, + variantId, + flags, + sequentialControlExecutor); - fileDownloadFuture = - fileDownloader.startDownloading( - groupKey, - fileGroupVersionNumber, - buildId, - downloadFileOnDeviceUri, - dataFile.getUrlToDownload(), - dataFile.getByteSize(), - downloadConditions, - downloaderCallback, - trafficTag, - extraHttpHeaders); - } else { - DownloaderCallback downloaderCallback = - new DeltaFileDownloaderCallbackImpl( - context, - sharedFilesMetadata, - fileStorage, - silentFeedback, - dataFile, - newFileKey.getAllowedReaders(), - deltaDecoderOptional.get(), - deltaFile, - eventLogger, - groupKey, - fileGroupVersionNumber, - buildId, - variantId, - instanceId, - flags, - sequentialControlExecutor); + mayNotifyCurrentSizeOfPartiallyDownloadedFile( + groupKey, downloadFileOnDeviceUri); - mayNotifyCurrentSizeOfPartiallyDownloadedFile(groupKey, downloadFileOnDeviceUri); + fileDownloadFuture = + fileDownloader.startDownloading( + newFileKey.getChecksum(), + groupKey, + fileGroupVersionNumber, + buildId, + variantId, + downloadFileOnDeviceUri, + dataFile.getUrlToDownload(), + dataFile.getByteSize(), + downloadConditions, + downloaderCallback, + trafficTag, + extraHttpHeaders); + } else { + DownloaderCallback downloaderCallback = + new DeltaFileDownloaderCallbackImpl( + context, + sharedFilesMetadata, + fileStorage, + silentFeedback, + dataFile, + newFileKey.getAllowedReaders(), + deltaDecoderOptional.get(), + deltaFile, + eventLogger, + groupKey, + fileGroupVersionNumber, + buildId, + variantId, + instanceId, + flags, + sequentialControlExecutor); - fileDownloadFuture = - fileDownloader.startDownloading( - groupKey, - fileGroupVersionNumber, - buildId, - downloadFileOnDeviceUri, - deltaFile.getUrlToDownload(), - deltaFile.getByteSize(), - downloadConditions, - downloaderCallback, - trafficTag, - extraHttpHeaders); - } - return fileDownloadFuture; - }, - sequentialControlExecutor); + mayNotifyCurrentSizeOfPartiallyDownloadedFile( + groupKey, downloadFileOnDeviceUri); + + fileDownloadFuture = + fileDownloader.startDownloading( + newFileKey.getChecksum(), + groupKey, + fileGroupVersionNumber, + buildId, + variantId, + downloadFileOnDeviceUri, + deltaFile.getUrlToDownload(), + deltaFile.getByteSize(), + downloadConditions, + downloaderCallback, + trafficTag, + extraHttpHeaders); + } + return fileDownloadFuture; + }, + sequentialControlExecutor); + }, + sequentialControlExecutor); } /** @@ -570,15 +659,15 @@ public class SharedFileManager { checksum, silentFeedback, instanceId, - /* androidShared = */ false); + /* androidShared= */ false); if (downloadFileOnDeviceUri == null) { LogUtil.e("%s: Failed to get file uri!", TAG); - return Futures.immediateFailedFuture( + return immediateFailedFuture( DownloadException.builder() .setDownloadResultCode(DownloadResultCode.UNABLE_TO_CREATE_FILE_URI_ERROR) .build()); } - return Futures.immediateFuture(downloadFileOnDeviceUri); + return immediateFuture(downloadFileOnDeviceUri); } private void mayNotifyCurrentSizeOfPartiallyDownloadedFile( @@ -599,10 +688,10 @@ public class SharedFileManager { } private ListenableFuture<DataFileGroupInternal> getDataFileGroupOrDefault(GroupKey groupKey) { - return Futures.transformAsync( + return PropagatedFutures.transformAsync( fileGroupsMetadata.read(groupKey), fileGroup -> - Futures.immediateFuture( + immediateFuture( (fileGroup == null) ? DataFileGroupInternal.getDefaultInstance() : fileGroup), sequentialControlExecutor); } @@ -621,10 +710,10 @@ public class SharedFileManager { < FileKeyVersion.USE_CHECKSUM_ONLY.value || !deltaDecoderOptional.isPresent() || deltaDecoderOptional.get().getDecoderName() == DiffDecoder.UNSPECIFIED) { - return Futures.immediateFuture(null); + return immediateFuture(null); } return findFirstDeltaFileWithBaseFileDownloaded( - dataFile.getDeltaFileList(), /* index = */ 0, allowedReaders); + dataFile.getDeltaFileList(), /* index= */ 0, allowedReaders); } // We must use recursion here since the decision to continue iterating is dependent on the result @@ -632,7 +721,7 @@ public class SharedFileManager { private ListenableFuture<@NullableType DeltaFile> findFirstDeltaFileWithBaseFileDownloaded( List<DeltaFile> deltaFiles, int index, AllowedReaders allowedReaders) { if (index == deltaFiles.size()) { - return Futures.immediateFuture(null); + return immediateFuture(null); } DeltaFile deltaFile = deltaFiles.get(index); if (deltaFile.getDiffDecoder() != deltaDecoderOptional.get().getDecoderName()) { @@ -643,7 +732,7 @@ public class SharedFileManager { .setChecksum(deltaFile.getBaseFile().getChecksum()) .setAllowedReaders(allowedReaders) .build(); - return Futures.transformAsync( + return PropagatedFutures.transformAsync( sharedFilesMetadata.read(baseFileKey), baseFileMetadata -> { if (baseFileMetadata != null @@ -656,9 +745,9 @@ public class SharedFileManager { baseFileKey.getChecksum(), silentFeedback, instanceId, - /* androidShared = */ false); + /* androidShared= */ false); if (baseFileUri != null) { - return Futures.immediateFuture(deltaFile); + return immediateFuture(deltaFile); } } return findFirstDeltaFileWithBaseFileDownloaded(deltaFiles, index + 1, allowedReaders); @@ -673,9 +762,9 @@ public class SharedFileManager { * a SharedFileMissingException if the shared file metadata is missing. */ ListenableFuture<FileStatus> getFileStatus(NewFileKey newFileKey) { - return Futures.transformAsync( + return PropagatedFutures.transformAsync( getSharedFile(newFileKey), - existingSharedFile -> Futures.immediateFuture(existingSharedFile.getFileStatus()), + existingSharedFile -> immediateFuture(existingSharedFile.getFileStatus()), sequentialControlExecutor); } @@ -688,14 +777,14 @@ public class SharedFileManager { * metadata is missing or the on disk file is corrupted. */ ListenableFuture<Void> reVerifyFile(NewFileKey newFileKey, DataFile dataFile) { - return FluentFuture.from(getSharedFile(newFileKey)) + return PropagatedFluentFuture.from(getSharedFile(newFileKey)) .transformAsync( existingSharedFile -> { if (existingSharedFile.getFileStatus() != FileStatus.DOWNLOAD_COMPLETE) { - return Futures.immediateVoidFuture(); + return immediateVoidFuture(); } // Double check that it's really complete, and update status if it's not. - return FluentFuture.from(getOnDeviceUri(newFileKey)) + return PropagatedFluentFuture.from(getOnDeviceUri(newFileKey)) .transformAsync( uri -> { if (uri == null) { @@ -717,7 +806,7 @@ public class SharedFileManager { FileValidator.validateDownloadedFile( fileStorage, dataFile, uri, dataFile.getChecksum()); } - return Futures.immediateVoidFuture(); + return immediateVoidFuture(); }, sequentialControlExecutor) .catchingAsync( @@ -730,7 +819,7 @@ public class SharedFileManager { existingSharedFile.toBuilder() .setFileStatus(FileStatus.CORRUPTED) .build(); - return FluentFuture.from( + return PropagatedFluentFuture.from( sharedFilesMetadata.write(newFileKey, updatedSharedFile)) .transformAsync( ok -> { @@ -755,16 +844,16 @@ public class SharedFileManager { * may throw a SharedFileMissingException if the shared file metadata is missing. */ ListenableFuture<SharedFile> getSharedFile(NewFileKey newFileKey) { - return Futures.transformAsync( + return PropagatedFutures.transformAsync( sharedFilesMetadata.read(newFileKey), existingSharedFile -> { if (existingSharedFile == null) { // TODO(b/131166925): MDD dump should not use lite proto toString. LogUtil.e( - "%s: getSharedFile called on file that doesn't exists! Key = %s", TAG, newFileKey); - return Futures.immediateFailedFuture(new SharedFileMissingException()); + "%s: getSharedFile called on file that doesn't exist! Key = %s", TAG, newFileKey); + return immediateFailedFuture(new SharedFileMissingException()); } - return Futures.immediateFuture(existingSharedFile); + return immediateFuture(existingSharedFile); }, sequentialControlExecutor); } @@ -803,7 +892,7 @@ public class SharedFileManager { */ ListenableFuture<Boolean> updateMaxExpirationDateSecs( NewFileKey newFileKey, long fileExpirationDateSecs) { - return Futures.transformAsync( + return PropagatedFutures.transformAsync( getSharedFile(newFileKey), existingSharedFile -> { if (fileExpirationDateSecs > existingSharedFile.getMaxExpirationDateSecs()) { @@ -813,7 +902,7 @@ public class SharedFileManager { .build(); return sharedFilesMetadata.write(newFileKey, updatedSharedFile); } - return Futures.immediateFuture(true); + return immediateFuture(true); }, sequentialControlExecutor); } @@ -827,29 +916,56 @@ public class SharedFileManager { * is an error populating the uri of the file. */ public ListenableFuture<@NullableType Uri> getOnDeviceUri(NewFileKey newFileKey) { - return Futures.transformAsync( - sharedFilesMetadata.read(newFileKey), - sharedFile -> { - if (sharedFile == null) { - // TODO(b/131166925): MDD dump should not use lite proto toString. - LogUtil.e( - "%s: getOnDeviceUri called on file that doesn't exists. Key = %s!", - TAG, newFileKey); - return Futures.immediateFailedFuture(new SharedFileMissingException()); - } + return PropagatedFutures.transform( + getOnDeviceUris(ImmutableSet.of(newFileKey)), + uris -> uris.get(newFileKey), + directExecutor()); + } - Uri onDeviceUri = - DirectoryUtil.getOnDeviceUri( - context, - newFileKey.getAllowedReaders(), - sharedFile.getFileName(), - sharedFile.getAndroidSharingChecksum(), - silentFeedback, - instanceId, - sharedFile.getAndroidShared()); - return Futures.immediateFuture(onDeviceUri); - }, - sequentialControlExecutor); + /** + * Get the known on-device uris for a given list of {@link NewFileKey}s + * + * <p>The returned map may or may not have an entry for each NewFileKey on the list, depending on + * if it was possible to create the uri (see {@link DirectoryUtil#getOnDeviceUri()} for more + * details). + * + * <p>If any {@link NewFileKey} does not map to a {@link SharedFile}, the returned future will be + * a failure containing {@link SharedFileMissingException}. + */ + ListenableFuture<ImmutableMap<NewFileKey, Uri>> getOnDeviceUris( + ImmutableSet<NewFileKey> newFileKeys) { + return PropagatedFluentFuture.from(sharedFilesMetadata.readAll(newFileKeys)) + .transformAsync( + sharedFileMap -> { + ImmutableMap.Builder<NewFileKey, Uri> uriMapBuilder = ImmutableMap.builder(); + for (NewFileKey newFileKey : newFileKeys) { + // Make sure all SharedFiles exist. + if (!sharedFileMap.containsKey(newFileKey)) { + // TODO(b/131166925): MDD dump should not use lite proto toString. + LogUtil.e( + "%s: getOnDeviceUris called on file that doesn't exist. Key = %s!", + TAG, newFileKey); + return immediateFailedFuture(new SharedFileMissingException()); + } + + SharedFile sharedFile = sharedFileMap.get(newFileKey); + + Uri onDeviceUri = + DirectoryUtil.getOnDeviceUri( + context, + newFileKey.getAllowedReaders(), + sharedFile.getFileName(), + sharedFile.getAndroidSharingChecksum(), + silentFeedback, + instanceId, + sharedFile.getAndroidShared()); + if (onDeviceUri != null) { + uriMapBuilder.put(newFileKey, onDeviceUri); + } + } + return immediateFuture(uriMapBuilder.build()); + }, + sequentialControlExecutor); } /** @@ -861,13 +977,13 @@ public class SharedFileManager { */ // TODO - refactor to throw Exception when write to SharedPreferences fails ListenableFuture<Boolean> removeFileEntry(NewFileKey newFileKey) { - return Futures.transformAsync( + return PropagatedFutures.transformAsync( sharedFilesMetadata.read(newFileKey), sharedFile -> { if (sharedFile == null) { // TODO(b/131166925): MDD dump should not use lite proto toString. LogUtil.e("%s: No file entry with key %s", TAG, newFileKey); - return Futures.immediateFuture(false); + return immediateFuture(false); } Uri onDeviceUri = @@ -878,19 +994,19 @@ public class SharedFileManager { newFileKey.getChecksum(), silentFeedback, instanceId, - /* androidShared = */ false); + /* androidShared= */ false); if (onDeviceUri != null) { - fileDownloader.stopDownloading(onDeviceUri); + fileDownloader.stopDownloading(newFileKey.getChecksum(), onDeviceUri); } - return Futures.transformAsync( + return PropagatedFutures.transformAsync( sharedFilesMetadata.remove(newFileKey), removeSuccess -> { if (!removeSuccess) { // TODO(b/131166925): MDD dump should not use lite proto toString. LogUtil.e("%s: Unable to modify file subscription for key %s", TAG, newFileKey); - return Futures.immediateFuture(false); + return immediateFuture(false); } - return Futures.immediateFuture(true); + return immediateFuture(true); }, sequentialControlExecutor); }, @@ -901,6 +1017,7 @@ public class SharedFileManager { * Clears all storage used by the SharedFileManager and deletes all files that have been * downloaded to MDD's directory. */ + // TODO(b/124072754): Change to package private once all code is refactored. public ListenableFuture<Void> clear() { // If sdk is R+, try release all leases that the MDD Client may have acquired. This @@ -913,14 +1030,14 @@ public class SharedFileManager { } catch (IOException e) { silentFeedback.send(e, "Failure while deleting mdd storage during clear"); } - return Futures.immediateVoidFuture(); + return immediateVoidFuture(); } private void releaseAllAndroidSharedFiles() { try { Uri allLeasesUri = DirectoryUtil.getBlobStoreAllLeasesUri(context); fileStorage.deleteFile(allLeasesUri); - eventLogger.logEventSampled(0); + eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); } catch (UnsupportedFileStorageOperation e) { LogUtil.v( "%s: Failed to release the leases in the android shared storage." @@ -928,12 +1045,12 @@ public class SharedFileManager { TAG); } catch (IOException e) { LogUtil.e(e, "%s: Failed to release the leases in the android shared storage", TAG); - eventLogger.logEventSampled(0); + eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); } } public ListenableFuture<Void> cancelDownloadAndClear() { - return Futures.transformAsync( + return PropagatedFutures.transformAsync( sharedFilesMetadata.getAllFileKeys(), newFileKeyList -> { List<ListenableFuture<Void>> cancelDownloadFutures = new ArrayList<>(); @@ -947,19 +1064,19 @@ public class SharedFileManager { } catch (Exception e) { silentFeedback.send(e, "Failed to cancel all downloads during clear"); } - return Futures.whenAllComplete(cancelDownloadFutures) + return PropagatedFutures.whenAllComplete(cancelDownloadFutures) .callAsync(this::clear, sequentialControlExecutor); }, sequentialControlExecutor); } public ListenableFuture<Void> cancelDownload(NewFileKey newFileKey) { - return Futures.transformAsync( + return PropagatedFutures.transformAsync( sharedFilesMetadata.read(newFileKey), sharedFile -> { if (sharedFile == null) { LogUtil.e("%s: Unable to read sharedFile from shared preferences.", TAG); - return Futures.immediateFailedFuture(new SharedFileMissingException()); + return immediateFailedFuture(new SharedFileMissingException()); } if (sharedFile.getFileStatus() != FileStatus.DOWNLOAD_COMPLETE) { Uri onDeviceUri = @@ -970,9 +1087,19 @@ public class SharedFileManager { newFileKey.getChecksum(), silentFeedback, instanceId, - /* androidShared = */ false); // while downloading androidShared is always false + /* androidShared= */ false); // while downloading androidShared is always false if (onDeviceUri != null) { - fileDownloader.stopDownloading(onDeviceUri); + fileDownloader.stopDownloading(newFileKey.getChecksum(), onDeviceUri); + } + // If the download was in progress, reset it back to subscribed, so it can be properly + // restarted. + if (sharedFile.getFileStatus() == FileStatus.DOWNLOAD_IN_PROGRESS) { + return PropagatedFutures.transformAsync( + sharedFilesMetadata.write( + newFileKey, + sharedFile.toBuilder().setFileStatus(FileStatus.SUBSCRIBED).build()), + unused -> immediateVoidFuture(), + sequentialControlExecutor); } } return immediateVoidFuture(); @@ -983,22 +1110,22 @@ public class SharedFileManager { /** Dumps the current internal state of the SharedFileManager. */ public ListenableFuture<Void> dump(final PrintWriter writer) { writer.println("==== MDD_SHARED_FILES ===="); - return Futures.transformAsync( + return PropagatedFutures.transformAsync( sharedFilesMetadata.getAllFileKeys(), allFileKeys -> { ListenableFuture<Void> writeFilesFuture = immediateVoidFuture(); for (NewFileKey newFileKey : allFileKeys) { writeFilesFuture = - Futures.transformAsync( + PropagatedFutures.transformAsync( writeFilesFuture, voidArg -> - Futures.transformAsync( + PropagatedFutures.transformAsync( sharedFilesMetadata.read(newFileKey), sharedFile -> { if (sharedFile == null) { LogUtil.e( "%s: Unable to read sharedFile from shared preferences.", TAG); - return Futures.immediateVoidFuture(); + return immediateVoidFuture(); } // TODO(b/131166925): MDD dump should not use lite proto toString. writer.format( @@ -1017,14 +1144,14 @@ public class SharedFileManager { newFileKey.getChecksum(), silentFeedback, instanceId, - /* androidShared = */ false); + /* androidShared= */ false); if (serializedUri != null) { writer.format( "Checksum downloaded file: %s\n", FileValidator.computeSha1Digest(fileStorage, serializedUri)); } } - return Futures.immediateVoidFuture(); + return immediateVoidFuture(); }, sequentialControlExecutor), sequentialControlExecutor); diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/SharedFilesMetadata.java b/java/com/google/android/libraries/mobiledatadownload/internal/SharedFilesMetadata.java index c5e6019..df1034e 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/SharedFilesMetadata.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/SharedFilesMetadata.java @@ -18,6 +18,8 @@ package com.google.android.libraries.mobiledatadownload.internal; import android.content.Context; import com.google.android.libraries.mobiledatadownload.SilentFeedback; import com.google.android.libraries.mobiledatadownload.internal.util.FileGroupUtil; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.common.util.concurrent.ListenableFuture; import com.google.mobiledatadownload.internal.MetadataProto.DataFile; import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal.AllowedReaders; @@ -126,6 +128,14 @@ public interface SharedFilesMetadata { public ListenableFuture<SharedFile> read(NewFileKey newFileKey); /** + * Returns all known {@link SharedFile}s for the given set of {@link NewFileKey}s + * + * <p>The map will contain a SharedFile entry if it exists. + */ + public ListenableFuture<ImmutableMap<NewFileKey, SharedFile>> readAll( + ImmutableSet<NewFileKey> newFileKeys); + + /** * Map the key "newFileKey" to the value "sharedFile". Returns a future resolving to true if the * operation succeeds, false if it fails. */ diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/SharedPreferencesFileGroupsMetadata.java b/java/com/google/android/libraries/mobiledatadownload/internal/SharedPreferencesFileGroupsMetadata.java index 9b1ba9a..47c0d3d 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/SharedPreferencesFileGroupsMetadata.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/SharedPreferencesFileGroupsMetadata.java @@ -15,28 +15,31 @@ */ package com.google.android.libraries.mobiledatadownload.internal; +import static com.google.common.util.concurrent.Futures.getDone; +import static com.google.common.util.concurrent.Futures.immediateFuture; +import static com.google.common.util.concurrent.Futures.immediateVoidFuture; + import android.content.Context; import android.content.SharedPreferences; -import android.util.Pair; import androidx.annotation.VisibleForTesting; import com.google.android.libraries.mobiledatadownload.SilentFeedback; import com.google.android.libraries.mobiledatadownload.TimeSource; import com.google.android.libraries.mobiledatadownload.annotations.InstanceId; import com.google.android.libraries.mobiledatadownload.internal.annotations.SequentialControlExecutor; +import com.google.android.libraries.mobiledatadownload.internal.collect.GroupKeyAndGroup; import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; import com.google.android.libraries.mobiledatadownload.internal.util.FileGroupUtil; import com.google.android.libraries.mobiledatadownload.internal.util.FileGroupsMetadataUtil; import com.google.android.libraries.mobiledatadownload.internal.util.FileGroupsMetadataUtil.GroupKeyDeserializationException; import com.google.android.libraries.mobiledatadownload.internal.util.ProtoLiteUtil; import com.google.android.libraries.mobiledatadownload.internal.util.SharedPreferencesUtil; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; import com.google.common.base.Optional; -import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.errorprone.annotations.CheckReturnValue; import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; import com.google.mobiledatadownload.internal.MetadataProto.GroupKeyProperties; - import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; @@ -55,7 +58,7 @@ public final class SharedPreferencesFileGroupsMetadata implements FileGroupsMeta private static final String TAG = "SharedPreferencesFileGroupsMetadata"; private static final String MDD_FILE_GROUPS = FileGroupsMetadataUtil.MDD_FILE_GROUPS; private static final String MDD_FILE_GROUP_KEY_PROPERTIES = - FileGroupsMetadataUtil.MDD_FILE_GROUP_KEY_PROPERTIES; + FileGroupsMetadataUtil.MDD_FILE_GROUP_KEY_PROPERTIES; // TODO(b/144033163): Migrate the Garbage Collector File to PDS. @VisibleForTesting static final String MDD_GARBAGE_COLLECTION_FILE = "gms_icing_mdd_garbage_file"; @@ -68,11 +71,11 @@ public final class SharedPreferencesFileGroupsMetadata implements FileGroupsMeta @Inject SharedPreferencesFileGroupsMetadata( - @ApplicationContext Context context, - TimeSource timeSource, - SilentFeedback silentFeedback, - @InstanceId Optional<String> instanceId, - @SequentialControlExecutor Executor sequentialControlExecutor) { + @ApplicationContext Context context, + TimeSource timeSource, + SilentFeedback silentFeedback, + @InstanceId Optional<String> instanceId, + @SequentialControlExecutor Executor sequentialControlExecutor) { this.context = context; this.timeSource = timeSource; this.silentFeedback = silentFeedback; @@ -82,7 +85,7 @@ public final class SharedPreferencesFileGroupsMetadata implements FileGroupsMeta @Override public ListenableFuture<Void> init() { - return Futures.immediateVoidFuture(); + return immediateVoidFuture(); } @Override @@ -90,11 +93,11 @@ public final class SharedPreferencesFileGroupsMetadata implements FileGroupsMeta String serializedGroupKey = FileGroupsMetadataUtil.getSerializedGroupKey(groupKey); SharedPreferences prefs = - SharedPreferencesUtil.getSharedPreferences(context, MDD_FILE_GROUPS, instanceId); + SharedPreferencesUtil.getSharedPreferences(context, MDD_FILE_GROUPS, instanceId); DataFileGroupInternal fileGroup = - SharedPreferencesUtil.readProto(prefs, serializedGroupKey, DataFileGroupInternal.parser()); + SharedPreferencesUtil.readProto(prefs, serializedGroupKey, DataFileGroupInternal.parser()); - return Futures.immediateFuture(fileGroup); + return immediateFuture(fileGroup); } @Override @@ -102,9 +105,8 @@ public final class SharedPreferencesFileGroupsMetadata implements FileGroupsMeta String serializedGroupKey = FileGroupsMetadataUtil.getSerializedGroupKey(groupKey); SharedPreferences prefs = - SharedPreferencesUtil.getSharedPreferences(context, MDD_FILE_GROUPS, instanceId); - return Futures.immediateFuture( - SharedPreferencesUtil.writeProto(prefs, serializedGroupKey, fileGroup)); + SharedPreferencesUtil.getSharedPreferences(context, MDD_FILE_GROUPS, instanceId); + return immediateFuture(SharedPreferencesUtil.writeProto(prefs, serializedGroupKey, fileGroup)); } @Override @@ -112,41 +114,41 @@ public final class SharedPreferencesFileGroupsMetadata implements FileGroupsMeta String serializedGroupKey = FileGroupsMetadataUtil.getSerializedGroupKey(groupKey); SharedPreferences prefs = - SharedPreferencesUtil.getSharedPreferences(context, MDD_FILE_GROUPS, instanceId); - return Futures.immediateFuture(SharedPreferencesUtil.removeProto(prefs, serializedGroupKey)); + SharedPreferencesUtil.getSharedPreferences(context, MDD_FILE_GROUPS, instanceId); + return immediateFuture(SharedPreferencesUtil.removeProto(prefs, serializedGroupKey)); } @Override public ListenableFuture<@NullableType GroupKeyProperties> readGroupKeyProperties( - GroupKey groupKey) { + GroupKey groupKey) { String serializedGroupKey = FileGroupsMetadataUtil.getSerializedGroupKey(groupKey); SharedPreferences prefs = - SharedPreferencesUtil.getSharedPreferences( - context, MDD_FILE_GROUP_KEY_PROPERTIES, instanceId); + SharedPreferencesUtil.getSharedPreferences( + context, MDD_FILE_GROUP_KEY_PROPERTIES, instanceId); GroupKeyProperties groupKeyProperties = - SharedPreferencesUtil.readProto(prefs, serializedGroupKey, GroupKeyProperties.parser()); + SharedPreferencesUtil.readProto(prefs, serializedGroupKey, GroupKeyProperties.parser()); - return Futures.immediateFuture(groupKeyProperties); + return immediateFuture(groupKeyProperties); } @Override public ListenableFuture<Boolean> writeGroupKeyProperties( - GroupKey groupKey, GroupKeyProperties groupKeyProperties) { + GroupKey groupKey, GroupKeyProperties groupKeyProperties) { String serializedGroupKey = FileGroupsMetadataUtil.getSerializedGroupKey(groupKey); SharedPreferences prefs = - SharedPreferencesUtil.getSharedPreferences( - context, MDD_FILE_GROUP_KEY_PROPERTIES, instanceId); - return Futures.immediateFuture( - SharedPreferencesUtil.writeProto(prefs, serializedGroupKey, groupKeyProperties)); + SharedPreferencesUtil.getSharedPreferences( + context, MDD_FILE_GROUP_KEY_PROPERTIES, instanceId); + return immediateFuture( + SharedPreferencesUtil.writeProto(prefs, serializedGroupKey, groupKeyProperties)); } @Override public ListenableFuture<List<GroupKey>> getAllGroupKeys() { List<GroupKey> groupKeyList = new ArrayList<>(); SharedPreferences prefs = - SharedPreferencesUtil.getSharedPreferences(context, MDD_FILE_GROUPS, instanceId); + SharedPreferencesUtil.getSharedPreferences(context, MDD_FILE_GROUPS, instanceId); SharedPreferences.Editor editor = null; for (String serializedGroupKey : prefs.getAll().keySet()) { try { @@ -170,55 +172,55 @@ public final class SharedPreferencesFileGroupsMetadata implements FileGroupsMeta if (editor != null) { editor.commit(); } - return Futures.immediateFuture(groupKeyList); + return immediateFuture(groupKeyList); } @Override - public ListenableFuture<List<Pair<GroupKey, DataFileGroupInternal>>> getAllFreshGroups() { - return Futures.transformAsync( - getAllGroupKeys(), - groupKeyList -> { - List<ListenableFuture<@NullableType DataFileGroupInternal>> groupReadFutures = - new ArrayList<>(); - for (GroupKey key : groupKeyList) { - groupReadFutures.add(read(key)); - } - return Futures.whenAllComplete(groupReadFutures) - .callAsync( - () -> { - List<Pair<GroupKey, DataFileGroupInternal>> retrievedGroups = new ArrayList<>(); - for (int i = 0; i < groupKeyList.size(); i++) { - GroupKey key = groupKeyList.get(i); - DataFileGroupInternal group = Futures.getDone(groupReadFutures.get(i)); - if (group == null) { - continue; - } - retrievedGroups.add(Pair.create(key, group)); - } - return Futures.immediateFuture(retrievedGroups); - }, - sequentialControlExecutor); - }, - sequentialControlExecutor); + public ListenableFuture<List<GroupKeyAndGroup>> getAllFreshGroups() { + return PropagatedFutures.transformAsync( + getAllGroupKeys(), + groupKeyList -> { + List<ListenableFuture<@NullableType DataFileGroupInternal>> groupReadFutures = + new ArrayList<>(); + for (GroupKey key : groupKeyList) { + groupReadFutures.add(read(key)); + } + return PropagatedFutures.whenAllComplete(groupReadFutures) + .callAsync( + () -> { + List<GroupKeyAndGroup> retrievedGroups = new ArrayList<>(); + for (int i = 0; i < groupKeyList.size(); i++) { + GroupKey key = groupKeyList.get(i); + DataFileGroupInternal group = getDone(groupReadFutures.get(i)); + if (group == null) { + continue; + } + retrievedGroups.add(GroupKeyAndGroup.create(key, group)); + } + return immediateFuture(retrievedGroups); + }, + sequentialControlExecutor); + }, + sequentialControlExecutor); } @Override public ListenableFuture<Boolean> removeAllGroupsWithKeys(List<GroupKey> keys) { SharedPreferences prefs = - SharedPreferencesUtil.getSharedPreferences(context, MDD_FILE_GROUPS, instanceId); + SharedPreferencesUtil.getSharedPreferences(context, MDD_FILE_GROUPS, instanceId); SharedPreferences.Editor editor = prefs.edit(); for (GroupKey key : keys) { LogUtil.d("%s: Removing group %s %s", TAG, key.getGroupName(), key.getOwnerPackage()); SharedPreferencesUtil.removeProto(editor, key); } - return Futures.immediateFuture(editor.commit()); + return immediateFuture(editor.commit()); } @Override public ListenableFuture<List<DataFileGroupInternal>> getAllStaleGroups() { - return Futures.immediateFuture( - FileGroupsMetadataUtil.getAllStaleGroups( - FileGroupsMetadataUtil.getGarbageCollectorFile(context, instanceId))); + return immediateFuture( + FileGroupsMetadataUtil.getAllStaleGroups( + FileGroupsMetadataUtil.getGarbageCollectorFile(context, instanceId))); } @Override @@ -227,8 +229,8 @@ public final class SharedPreferencesFileGroupsMetadata implements FileGroupsMeta long currentTimeSeconds = timeSource.currentTimeMillis() / 1000; fileGroup = - FileGroupUtil.setStaleExpirationDate( - fileGroup, currentTimeSeconds + fileGroup.getStaleLifetimeSecs()); + FileGroupUtil.setStaleExpirationDate( + fileGroup, currentTimeSeconds + fileGroup.getStaleLifetimeSecs()); List<DataFileGroupInternal> fileGroups = new ArrayList<>(); fileGroups.add(fileGroup); @@ -244,7 +246,7 @@ public final class SharedPreferencesFileGroupsMetadata implements FileGroupsMeta outputStream = new FileOutputStream(garbageCollectorFile, /* append */ true); } catch (FileNotFoundException e) { LogUtil.e("File %s not found while writing.", garbageCollectorFile.getAbsolutePath()); - return Futures.immediateFuture(false); + return immediateFuture(false); } try { @@ -256,9 +258,9 @@ public final class SharedPreferencesFileGroupsMetadata implements FileGroupsMeta outputStream.close(); } catch (IOException e) { LogUtil.e("IOException occurred while writing file groups."); - return Futures.immediateFuture(false); + return immediateFuture(false); } - return Futures.immediateFuture(true); + return immediateFuture(true); } @VisibleForTesting @@ -270,18 +272,18 @@ public final class SharedPreferencesFileGroupsMetadata implements FileGroupsMeta @Override public ListenableFuture<Void> removeAllStaleGroups() { getGarbageCollectorFile().delete(); - return Futures.immediateVoidFuture(); + return immediateVoidFuture(); } @Override public ListenableFuture<Void> clear() { SharedPreferences prefs = - SharedPreferencesUtil.getSharedPreferences(context, MDD_FILE_GROUPS, instanceId); + SharedPreferencesUtil.getSharedPreferences(context, MDD_FILE_GROUPS, instanceId); prefs.edit().clear().commit(); SharedPreferences activatedGroupPrefs = - SharedPreferencesUtil.getSharedPreferences( - context, MDD_FILE_GROUP_KEY_PROPERTIES, instanceId); + SharedPreferencesUtil.getSharedPreferences( + context, MDD_FILE_GROUP_KEY_PROPERTIES, instanceId); activatedGroupPrefs.edit().clear().commit(); return removeAllStaleGroups(); diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/SharedPreferencesSharedFilesMetadata.java b/java/com/google/android/libraries/mobiledatadownload/internal/SharedPreferencesSharedFilesMetadata.java index 662ac5b..851225b 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/SharedPreferencesSharedFilesMetadata.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/SharedPreferencesSharedFilesMetadata.java @@ -17,10 +17,13 @@ package com.google.android.libraries.mobiledatadownload.internal; import static com.google.android.libraries.mobiledatadownload.internal.MddConstants.SPLIT_CHAR; import static com.google.android.libraries.mobiledatadownload.internal.util.SharedFilesMetadataUtil.MDD_SHARED_FILES; +import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import android.content.Context; import android.content.SharedPreferences; + import androidx.annotation.VisibleForTesting; + import com.google.android.libraries.mobiledatadownload.Flags; import com.google.android.libraries.mobiledatadownload.SilentFeedback; import com.google.android.libraries.mobiledatadownload.annotations.InstanceId; @@ -29,15 +32,20 @@ import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; import com.google.android.libraries.mobiledatadownload.internal.util.SharedFilesMetadataUtil; import com.google.android.libraries.mobiledatadownload.internal.util.SharedFilesMetadataUtil.FileKeyDeserializationException; import com.google.android.libraries.mobiledatadownload.internal.util.SharedPreferencesUtil; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; import com.google.common.base.Optional; import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.errorprone.annotations.CheckReturnValue; import com.google.mobiledatadownload.internal.MetadataProto.NewFileKey; import com.google.mobiledatadownload.internal.MetadataProto.SharedFile; + import java.util.ArrayList; import java.util.List; + import javax.inject.Inject; /** @@ -49,281 +57,305 @@ import javax.inject.Inject; @CheckReturnValue public final class SharedPreferencesSharedFilesMetadata implements SharedFilesMetadata { - private static final String TAG = "SharedFilesMetadata"; - - @VisibleForTesting static final String PREFS_KEY_NEXT_FILE_NAME_OLD = "next_file_name"; - @VisibleForTesting static final String PREFS_KEY_NEXT_FILE_NAME = "next_file_name_v2"; - - private final Context context; - private final SilentFeedback silentFeedback; - private final Optional<String> instanceId; - private final Flags flags; - - @Inject - public SharedPreferencesSharedFilesMetadata( - @ApplicationContext Context context, - SilentFeedback silentFeedback, - @InstanceId Optional<String> instanceId, - Flags flags) { - this.context = context; - this.silentFeedback = silentFeedback; - this.instanceId = instanceId; - this.flags = flags; - } - - @Override - public ListenableFuture<Boolean> init() { - // Migrate to the new file key. - if (!Migrations.isMigratedToNewFileKey(context)) { - LogUtil.d("%s Device isn't migrated to new file key, clear and set migration.", TAG); - Migrations.setMigratedToNewFileKey(context, true); - Migrations.setCurrentVersion(context, FileKeyVersion.getVersion(flags.fileKeyVersion())); - return Futures.immediateFuture(false); + private static final String TAG = "SharedFilesMetadata"; + + @VisibleForTesting + static final String PREFS_KEY_NEXT_FILE_NAME_OLD = "next_file_name"; + @VisibleForTesting + static final String PREFS_KEY_NEXT_FILE_NAME = "next_file_name_v2"; + + private final Context context; + private final SilentFeedback silentFeedback; + private final Optional<String> instanceId; + private final Flags flags; + + @Inject + public SharedPreferencesSharedFilesMetadata( + @ApplicationContext Context context, + SilentFeedback silentFeedback, + @InstanceId Optional<String> instanceId, + Flags flags) { + this.context = context; + this.silentFeedback = silentFeedback; + this.instanceId = instanceId; + this.flags = flags; } - return Futures.immediateFuture(upgradeToNewVersion()); - } - - /** - * Sequentially upgrade FileKey version to FeatureFlags.fileKeyVersion - * - * @return false if any upgrade fails which will result in clearing of all meta data, true on - * successful upgrade. - */ - private boolean upgradeToNewVersion() { - final FileKeyVersion targetVersion = FileKeyVersion.getVersion(flags.fileKeyVersion()); - final FileKeyVersion currentVersion = Migrations.getCurrentVersion(context, silentFeedback); - - if (targetVersion.value == currentVersion.value) { - return true; + + @Override + public ListenableFuture<Boolean> init() { + // Migrate to the new file key. + if (!Migrations.isMigratedToNewFileKey(context)) { + LogUtil.d("%s Device isn't migrated to new file key, clear and set migration.", TAG); + Migrations.setMigratedToNewFileKey(context, true); + Migrations.setCurrentVersion(context, + FileKeyVersion.getVersion(flags.fileKeyVersion())); + return Futures.immediateFuture(false); + } + return Futures.immediateFuture(upgradeToNewVersion()); } - if (targetVersion.value < currentVersion.value) { - // We don't support downgrading file key version. Clear everything. - LogUtil.e( - "%s Cannot migrate back from value %s to %s. Clear everything!", - TAG, currentVersion, targetVersion); - silentFeedback.send( - new Exception( - "Downgraded file key from " + currentVersion + " to " + targetVersion + "."), - "FileKey migrations unexpected downgrade."); - Migrations.setCurrentVersion(context, targetVersion); - return false; + /** + * Sequentially upgrade FileKey version to FeatureFlags.fileKeyVersion + * + * @return false if any upgrade fails which will result in clearing of all meta data, true on + * successful upgrade. + */ + private boolean upgradeToNewVersion() { + final FileKeyVersion targetVersion = FileKeyVersion.getVersion(flags.fileKeyVersion()); + final FileKeyVersion currentVersion = Migrations.getCurrentVersion(context, silentFeedback); + + if (targetVersion.value == currentVersion.value) { + return true; + } + + if (targetVersion.value < currentVersion.value) { + // We don't support downgrading file key version. Clear everything. + LogUtil.e( + "%s Cannot migrate back from value %s to %s. Clear everything!", + TAG, currentVersion, targetVersion); + silentFeedback.send( + new Exception( + "Downgraded file key from " + currentVersion + " to " + targetVersion + + "."), + "FileKey migrations unexpected downgrade."); + Migrations.setCurrentVersion(context, targetVersion); + return false; + } + + // Migrate one version at a time one by one + try { + for (int nextVersion = currentVersion.value + 1; + nextVersion <= targetVersion.value; + nextVersion++) { + if (upgradeTo(FileKeyVersion.getVersion(nextVersion))) { + Migrations.setCurrentVersion(context, FileKeyVersion.getVersion(nextVersion)); + } else { + // If migration to next version fail, we will clear all data and set the + // currentVersion + // to targetVersion (phFileKeyVersion) + return false; + } + } + } finally { + if (Migrations.getCurrentVersion(context, silentFeedback).value + != targetVersion.value) { + if (!Migrations.setCurrentVersion(context, targetVersion)) { + LogUtil.e( + "Failed to commit migration version to disk. Fail to set target " + + "version to " + + targetVersion + + "."); + silentFeedback.send( + new Exception("Fail to set target version " + targetVersion + "."), + "Failed to commit migration version to disk."); + } + } + } + + return true; } - // Migrate one version at a time one by one - try { - for (int nextVersion = currentVersion.value + 1; - nextVersion <= targetVersion.value; - nextVersion++) { - if (upgradeTo(FileKeyVersion.getVersion(nextVersion))) { - Migrations.setCurrentVersion(context, FileKeyVersion.getVersion(nextVersion)); - } else { - // If migration to next version fail, we will clear all data and set the currentVersion - // to targetVersion (phFileKeyVersion) - return false; + private boolean upgradeTo(FileKeyVersion targetVersion) { + switch (targetVersion) { + case ADD_DOWNLOAD_TRANSFORM: + return migrateToAddDownloadTransform(); + case USE_CHECKSUM_ONLY: + return migrateToDedupOnChecksumOnly(); + default: + throw new UnsupportedOperationException( + "Upgrade to version " + targetVersion.name() + "not supported!"); } - } - } finally { - if (Migrations.getCurrentVersion(context, silentFeedback).value != targetVersion.value) { - if (!Migrations.setCurrentVersion(context, targetVersion)) { - LogUtil.e( - "Failed to commit migration version to disk. Fail to set target version to " - + targetVersion - + "."); - silentFeedback.send( - new Exception("Fail to set target version " + targetVersion + "."), - "Failed to commit migration version to disk."); + } + + /** A one off method that is called when we migrate key to add download transform. */ + private boolean migrateToAddDownloadTransform() { + LogUtil.d("%s: Starting migration to add download transform", TAG); + SharedPreferences prefs = + SharedPreferencesUtil.getSharedPreferences(context, MDD_SHARED_FILES, instanceId); + SharedPreferences.Editor editor = prefs.edit(); + for (String serializedFileKey : prefs.getAll().keySet()) { + + // Remove the data that we are unable to read or parse. + NewFileKey newFileKey; + try { + newFileKey = + SharedFilesMetadataUtil.deserializeNewFileKey( + serializedFileKey, context, silentFeedback); + } catch (FileKeyDeserializationException e) { + LogUtil.e( + "%s Failed to deserialize file key %s, remove and continue.", TAG, + serializedFileKey); + silentFeedback.send(e, "Failed to deserialize file key, remove and continue."); + editor.remove(serializedFileKey); + continue; + } + SharedFile sharedFile = + SharedPreferencesUtil.readProto(prefs, serializedFileKey, SharedFile.parser()); + if (sharedFile == null) { + LogUtil.e("%s: Unable to read sharedFile from shared preferences.", TAG); + editor.remove(serializedFileKey); + continue; + } + + // Remove the old key and write the new one. + SharedPreferencesUtil.removeProto(editor, serializedFileKey); + SharedPreferencesUtil.writeProto( + editor, + SharedFilesMetadataUtil.serializeNewFileKeyWithDownloadTransform(newFileKey), + sharedFile); } - } + + if (!editor.commit()) { + LogUtil.e("Failed to commit migration metadata to disk"); + silentFeedback.send( + new Exception("Migrate to DownloadTransform failed."), + "Failed to commit migration metadata to disk."); + return false; + } + + return true; } - return true; - } - - private boolean upgradeTo(FileKeyVersion targetVersion) { - switch (targetVersion) { - case ADD_DOWNLOAD_TRANSFORM: - return migrateToAddDownloadTransform(); - case USE_CHECKSUM_ONLY: - return migrateToDedupOnChecksumOnly(); - default: - throw new UnsupportedOperationException( - "Upgrade to version " + targetVersion.name() + "not supported!"); + /** A one off method that is called when we migrate key to contain checksum and + * allowedReaders. */ + private boolean migrateToDedupOnChecksumOnly() { + LogUtil.d("%s: Starting migration to dedup on checksum only", TAG); + SharedPreferences prefs = + SharedPreferencesUtil.getSharedPreferences(context, MDD_SHARED_FILES, instanceId); + SharedPreferences.Editor editor = prefs.edit(); + for (String serializedFileKey : prefs.getAll().keySet()) { + + // Remove the data that we are unable to read or parse. + NewFileKey newFileKey; + try { + newFileKey = + SharedFilesMetadataUtil.deserializeNewFileKey( + serializedFileKey, context, silentFeedback); + } catch (FileKeyDeserializationException e) { + LogUtil.e( + "%s Failed to deserialize file key %s, remove and continue.", TAG, + serializedFileKey); + silentFeedback.send(e, "Failed to deserialize file key, remove and continue."); + editor.remove(serializedFileKey); + continue; + } + + SharedFile sharedFile = + SharedPreferencesUtil.readProto(prefs, serializedFileKey, SharedFile.parser()); + if (sharedFile == null) { + LogUtil.e("%s: Unable to read sharedFile from shared preferences.", TAG); + editor.remove(serializedFileKey); + continue; + } + + // Remove the old key and write the new one. + SharedPreferencesUtil.removeProto(editor, serializedFileKey); + SharedPreferencesUtil.writeProto( + editor, + SharedFilesMetadataUtil.serializeNewFileKeyWithChecksumOnly(newFileKey), + sharedFile); + } + + if (!editor.commit()) { + LogUtil.e("Failed to commit migration metadata to disk"); + silentFeedback.send( + new Exception("Migrate to ChecksumOnly failed."), + "Failed to commit migration metadata to disk."); + return false; + } + + return true; } - } - - /** A one off method that is called when we migrate key to add download transform. */ - private boolean migrateToAddDownloadTransform() { - LogUtil.d("%s: Starting migration to add download transform", TAG); - SharedPreferences prefs = - SharedPreferencesUtil.getSharedPreferences(context, MDD_SHARED_FILES, instanceId); - SharedPreferences.Editor editor = prefs.edit(); - for (String serializedFileKey : prefs.getAll().keySet()) { - - // Remove the data that we are unable to read or parse. - NewFileKey newFileKey; - try { - newFileKey = - SharedFilesMetadataUtil.deserializeNewFileKey( - serializedFileKey, context, silentFeedback); - } catch (FileKeyDeserializationException e) { - LogUtil.e( - "%s Failed to deserialize file key %s, remove and continue.", TAG, serializedFileKey); - silentFeedback.send(e, "Failed to deserialize file key, remove and continue."); - editor.remove(serializedFileKey); - continue; - } - SharedFile sharedFile = - SharedPreferencesUtil.readProto(prefs, serializedFileKey, SharedFile.parser()); - if (sharedFile == null) { - LogUtil.e("%s: Unable to read sharedFile from shared preferences.", TAG); - editor.remove(serializedFileKey); - continue; - } - - // Remove the old key and write the new one. - SharedPreferencesUtil.removeProto(editor, serializedFileKey); - SharedPreferencesUtil.writeProto( - editor, - SharedFilesMetadataUtil.serializeNewFileKeyWithDownloadTransform(newFileKey), - sharedFile); + + @SuppressWarnings("nullness") + @Override + public ListenableFuture<SharedFile> read(NewFileKey newFileKey) { + return PropagatedFutures.transform( + readAll(ImmutableSet.of(newFileKey)), + sharedFiles -> sharedFiles.get(newFileKey), + directExecutor()); } - if (!editor.commit()) { - LogUtil.e("Failed to commit migration metadata to disk"); - silentFeedback.send( - new Exception("Migrate to DownloadTransform failed."), - "Failed to commit migration metadata to disk."); - return false; + @Override + public ListenableFuture<ImmutableMap<NewFileKey, SharedFile>> readAll( + ImmutableSet<NewFileKey> newFileKeys) { + SharedPreferences prefs = + SharedPreferencesUtil.getSharedPreferences(context, MDD_SHARED_FILES, instanceId); + ImmutableMap.Builder<NewFileKey, SharedFile> sharedFileMapBuilder = ImmutableMap.builder(); + for (NewFileKey newFileKey : newFileKeys) { + String serializedFileKey = + SharedFilesMetadataUtil.getSerializedFileKey(newFileKey, context, + silentFeedback); + SharedFile sharedFile = + SharedPreferencesUtil.readProto(prefs, serializedFileKey, SharedFile.parser()); + if (sharedFile != null) { + sharedFileMapBuilder.put(newFileKey, sharedFile); + } + } + return Futures.immediateFuture(sharedFileMapBuilder.build()); } - return true; - } - - /** A one off method that is called when we migrate key to contain checksum and allowedReaders. */ - private boolean migrateToDedupOnChecksumOnly() { - LogUtil.d("%s: Starting migration to dedup on checksum only", TAG); - SharedPreferences prefs = - SharedPreferencesUtil.getSharedPreferences(context, MDD_SHARED_FILES, instanceId); - SharedPreferences.Editor editor = prefs.edit(); - for (String serializedFileKey : prefs.getAll().keySet()) { - - // Remove the data that we are unable to read or parse. - NewFileKey newFileKey; - try { - newFileKey = - SharedFilesMetadataUtil.deserializeNewFileKey( - serializedFileKey, context, silentFeedback); - } catch (FileKeyDeserializationException e) { - LogUtil.e( - "%s Failed to deserialize file key %s, remove and continue.", TAG, serializedFileKey); - silentFeedback.send(e, "Failed to deserialize file key, remove and continue."); - editor.remove(serializedFileKey); - continue; - } - - SharedFile sharedFile = - SharedPreferencesUtil.readProto(prefs, serializedFileKey, SharedFile.parser()); - if (sharedFile == null) { - LogUtil.e("%s: Unable to read sharedFile from shared preferences.", TAG); - editor.remove(serializedFileKey); - continue; - } - - // Remove the old key and write the new one. - SharedPreferencesUtil.removeProto(editor, serializedFileKey); - SharedPreferencesUtil.writeProto( - editor, - SharedFilesMetadataUtil.serializeNewFileKeyWithChecksumOnly(newFileKey), - sharedFile); + @Override + public ListenableFuture<Boolean> write(NewFileKey newFileKey, SharedFile sharedFile) { + String serializedFileKey = + SharedFilesMetadataUtil.getSerializedFileKey(newFileKey, context, silentFeedback); + + SharedPreferences prefs = + SharedPreferencesUtil.getSharedPreferences(context, MDD_SHARED_FILES, instanceId); + return Futures.immediateFuture( + SharedPreferencesUtil.writeProto(prefs, serializedFileKey, sharedFile)); } - if (!editor.commit()) { - LogUtil.e("Failed to commit migration metadata to disk"); - silentFeedback.send( - new Exception("Migrate to ChecksumOnly failed."), - "Failed to commit migration metadata to disk."); - return false; + @Override + public ListenableFuture<Boolean> remove(NewFileKey newFileKey) { + String serializedFileKey = + SharedFilesMetadataUtil.getSerializedFileKey(newFileKey, context, silentFeedback); + + SharedPreferences prefs = + SharedPreferencesUtil.getSharedPreferences(context, MDD_SHARED_FILES, instanceId); + return Futures.immediateFuture(SharedPreferencesUtil.removeProto(prefs, serializedFileKey)); } - return true; - } - - @SuppressWarnings("nullness") - @Override - public ListenableFuture<SharedFile> read(NewFileKey newFileKey) { - String serializedFileKey = - SharedFilesMetadataUtil.getSerializedFileKey(newFileKey, context, silentFeedback); - - SharedPreferences prefs = - SharedPreferencesUtil.getSharedPreferences(context, MDD_SHARED_FILES, instanceId); - SharedFile sharedFile = - SharedPreferencesUtil.readProto(prefs, serializedFileKey, SharedFile.parser()); - - return Futures.immediateFuture(sharedFile); - } - - @Override - public ListenableFuture<Boolean> write(NewFileKey newFileKey, SharedFile sharedFile) { - String serializedFileKey = - SharedFilesMetadataUtil.getSerializedFileKey(newFileKey, context, silentFeedback); - - SharedPreferences prefs = - SharedPreferencesUtil.getSharedPreferences(context, MDD_SHARED_FILES, instanceId); - return Futures.immediateFuture( - SharedPreferencesUtil.writeProto(prefs, serializedFileKey, sharedFile)); - } - - @Override - public ListenableFuture<Boolean> remove(NewFileKey newFileKey) { - String serializedFileKey = - SharedFilesMetadataUtil.getSerializedFileKey(newFileKey, context, silentFeedback); - - SharedPreferences prefs = - SharedPreferencesUtil.getSharedPreferences(context, MDD_SHARED_FILES, instanceId); - return Futures.immediateFuture(SharedPreferencesUtil.removeProto(prefs, serializedFileKey)); - } - - @Override - public ListenableFuture<List<NewFileKey>> getAllFileKeys() { - List<NewFileKey> newFileKeyList = new ArrayList<>(); - SharedPreferences prefs = - SharedPreferencesUtil.getSharedPreferences(context, MDD_SHARED_FILES, instanceId); - SharedPreferences.Editor editor = null; - for (String serializedFileKey : prefs.getAll().keySet()) { - try { - NewFileKey newFileKey = - SharedFilesMetadataUtil.deserializeNewFileKey( - serializedFileKey, context, silentFeedback); - newFileKeyList.add(newFileKey); - } catch (FileKeyDeserializationException e) { - LogUtil.e(e, "Failed to deserialize newFileKey:" + serializedFileKey); - silentFeedback.send( - e, - "Failed to deserialize newFileKey, unexpected key size: %d", - Splitter.on(SPLIT_CHAR).splitToList(serializedFileKey).size()); - // TODO(b/128850000): Refactor this code to a single corruption handling task during - // maintenance. - // Remove the corrupted file metadata and the related FileGroup metadata will be deleted - // in next maintenance task. - if (editor == null) { - editor = prefs.edit(); + @Override + public ListenableFuture<List<NewFileKey>> getAllFileKeys() { + List<NewFileKey> newFileKeyList = new ArrayList<>(); + SharedPreferences prefs = + SharedPreferencesUtil.getSharedPreferences(context, MDD_SHARED_FILES, instanceId); + SharedPreferences.Editor editor = null; + for (String serializedFileKey : prefs.getAll().keySet()) { + try { + NewFileKey newFileKey = + SharedFilesMetadataUtil.deserializeNewFileKey( + serializedFileKey, context, silentFeedback); + newFileKeyList.add(newFileKey); + } catch (FileKeyDeserializationException e) { + LogUtil.e(e, "Failed to deserialize newFileKey:" + serializedFileKey); + silentFeedback.send( + e, + "Failed to deserialize newFileKey, unexpected key size: %d", + Splitter.on(SPLIT_CHAR).splitToList(serializedFileKey).size()); + // TODO(b/128850000): Refactor this code to a single corruption handling task during + // maintenance. + // Remove the corrupted file metadata and the related FileGroup metadata will be deleted + // in next maintenance task. + if (editor == null) { + editor = prefs.edit(); + } + editor.remove(serializedFileKey); + continue; + } + } + if (editor != null) { + editor.commit(); } - editor.remove(serializedFileKey); - continue; - } + return Futures.immediateFuture(newFileKeyList); } - if (editor != null) { - editor.commit(); + + @Override + public ListenableFuture<Void> clear() { + SharedPreferences prefs = + SharedPreferencesUtil.getSharedPreferences(context, MDD_SHARED_FILES, instanceId); + prefs.edit().clear().commit(); + return Futures.immediateFuture(null); } - return Futures.immediateFuture(newFileKeyList); - } - - @Override - public ListenableFuture<Void> clear() { - SharedPreferences prefs = - SharedPreferencesUtil.getSharedPreferences(context, MDD_SHARED_FILES, instanceId); - prefs.edit().clear().commit(); - return Futures.immediateFuture(null); - } } diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/annotations/BUILD b/java/com/google/android/libraries/mobiledatadownload/internal/annotations/BUILD index dc959e6..a6b734f 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/annotations/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/internal/annotations/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/collect/BUILD b/java/com/google/android/libraries/mobiledatadownload/internal/collect/BUILD new file mode 100644 index 0000000..c381862 --- /dev/null +++ b/java/com/google/android/libraries/mobiledatadownload/internal/collect/BUILD @@ -0,0 +1,33 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +load("@build_bazel_rules_android//android:rules.bzl", "android_library") + +package( + default_applicable_licenses = ["//:license"], + default_visibility = [ + "//visibility:public", + ], + licenses = ["notice"], +) + +android_library( + name = "collect", + srcs = glob(["*.java"]), + deps = [ + "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", + "@com_google_auto_value", + "@com_google_code_findbugs_jsr305", + "@com_google_errorprone_error_prone_annotations", + ], +) diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/collect/GroupKeyAndGroup.java b/java/com/google/android/libraries/mobiledatadownload/internal/collect/GroupKeyAndGroup.java new file mode 100644 index 0000000..c84a8d6 --- /dev/null +++ b/java/com/google/android/libraries/mobiledatadownload/internal/collect/GroupKeyAndGroup.java @@ -0,0 +1,34 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.libraries.mobiledatadownload.internal.collect; + +import com.google.auto.value.AutoValue; +import com.google.errorprone.annotations.Immutable; +import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; +import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; + +/** Container for associated {@link GroupKey} and {@link DataFileGroupInternal}. */ +@AutoValue +@Immutable +public abstract class GroupKeyAndGroup { + public static GroupKeyAndGroup create(GroupKey groupKey, DataFileGroupInternal dataFileGroup) { + return new AutoValue_GroupKeyAndGroup(groupKey, dataFileGroup); + } + + public abstract GroupKey groupKey(); + + public abstract DataFileGroupInternal dataFileGroup(); +} diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/collect/GroupPair.java b/java/com/google/android/libraries/mobiledatadownload/internal/collect/GroupPair.java new file mode 100644 index 0000000..3a76887 --- /dev/null +++ b/java/com/google/android/libraries/mobiledatadownload/internal/collect/GroupPair.java @@ -0,0 +1,38 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.libraries.mobiledatadownload.internal.collect; + +import com.google.auto.value.AutoValue; +import com.google.errorprone.annotations.Immutable; +import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; +import javax.annotation.Nullable; + +/** Container for associated downloaded and pending versions of the same group. */ +@AutoValue +@Immutable +public abstract class GroupPair { + public static GroupPair create( + @Nullable DataFileGroupInternal pendingGroup, + @Nullable DataFileGroupInternal downloadedGroup) { + return new AutoValue_GroupPair(pendingGroup, downloadedGroup); + } + + @Nullable + public abstract DataFileGroupInternal pendingGroup(); + + @Nullable + public abstract DataFileGroupInternal downloadedGroup(); +} diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/dagger/BUILD b/java/com/google/android/libraries/mobiledatadownload/internal/dagger/BUILD index 065c222..7255a39 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/dagger/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/internal/dagger/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = ["//:__subpackages__"], licenses = ["notice"], ) @@ -60,17 +61,20 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload:TimeSource", "//java/com/google/android/libraries/mobiledatadownload/annotations", "//java/com/google/android/libraries/mobiledatadownload/file", + "//java/com/google/android/libraries/mobiledatadownload/internal:AndroidTimeSource", "//java/com/google/android/libraries/mobiledatadownload/internal:ApplicationContext", "//java/com/google/android/libraries/mobiledatadownload/internal:FileGroupsMetadata", "//java/com/google/android/libraries/mobiledatadownload/internal:SharedFilesMetadata", "//java/com/google/android/libraries/mobiledatadownload/internal:SharedPreferencesFileGroupsMetadata", "//java/com/google/android/libraries/mobiledatadownload/internal:SharedPreferencesSharedFilesMetadata", + "//java/com/google/android/libraries/mobiledatadownload/internal/annotations", "//java/com/google/android/libraries/mobiledatadownload/internal/annotations:SequentialControlExecutor", "//java/com/google/android/libraries/mobiledatadownload/internal/experimentation:DownloadStageManager", "//java/com/google/android/libraries/mobiledatadownload/internal/experimentation:NoOpDownloadStageManager", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:EventLogger", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LoggingStateStore", - "//java/com/google/android/libraries/mobiledatadownload/internal/logging:NoOpLoggingState", + "//java/com/google/android/libraries/mobiledatadownload/internal/logging:SharedPreferencesLoggingState", + "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", "//java/com/google/android/libraries/mobiledatadownload/internal/util:FuturesUtil", "//java/com/google/android/libraries/mobiledatadownload/monitor:DownloadProgressMonitor", "//java/com/google/android/libraries/mobiledatadownload/monitor:NetworkUsageMonitor", @@ -90,6 +94,7 @@ android_library( ":DownloaderModule", ":ExecutorsModule", ":MainMddLibModule", + "//java/com/google/android/libraries/mobiledatadownload:TimeSource", "//java/com/google/android/libraries/mobiledatadownload/internal:MobileDataDownloadManager", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:EventLogger", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LoggingStateStore", diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/dagger/MainMddLibModule.java b/java/com/google/android/libraries/mobiledatadownload/internal/dagger/MainMddLibModule.java index 04ab285..39957f0 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/dagger/MainMddLibModule.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/dagger/MainMddLibModule.java @@ -16,7 +16,6 @@ package com.google.android.libraries.mobiledatadownload.internal.dagger; import android.content.Context; - import com.google.android.libraries.mobiledatadownload.AccountSource; import com.google.android.libraries.mobiledatadownload.ExperimentationConfig; import com.google.android.libraries.mobiledatadownload.Flags; @@ -24,6 +23,7 @@ import com.google.android.libraries.mobiledatadownload.SilentFeedback; import com.google.android.libraries.mobiledatadownload.TimeSource; import com.google.android.libraries.mobiledatadownload.annotations.InstanceId; import com.google.android.libraries.mobiledatadownload.file.SynchronousFileStorage; +import com.google.android.libraries.mobiledatadownload.internal.AndroidTimeSource; import com.google.android.libraries.mobiledatadownload.internal.ApplicationContext; import com.google.android.libraries.mobiledatadownload.internal.FileGroupsMetadata; import com.google.android.libraries.mobiledatadownload.internal.SharedFilesMetadata; @@ -39,159 +39,175 @@ import com.google.android.libraries.mobiledatadownload.internal.util.FuturesUtil import com.google.android.libraries.mobiledatadownload.monitor.DownloadProgressMonitor; import com.google.android.libraries.mobiledatadownload.monitor.NetworkUsageMonitor; import com.google.common.base.Optional; - +import dagger.Module; +import dagger.Provides; import java.security.SecureRandom; import java.util.concurrent.Executor; - import javax.inject.Singleton; -import dagger.Module; -import dagger.Provides; - /** Module for MDD Lib dependencies */ @Module public class MainMddLibModule { - /** The version of MDD library. Same as mdi_download module version. */ - // TODO(b/122271766): Figure out how to update this automatically. - public static final int MDD_LIB_VERSION = 422883838; - - private final SynchronousFileStorage fileStorage; - private final NetworkUsageMonitor networkUsageMonitor; - private final EventLogger eventLogger; - private final Optional<DownloadProgressMonitor> downloadProgressMonitorOptional; - private final Optional<SilentFeedback> silentFeedbackOptional; - private final Optional<String> instanceId; - private final Optional<AccountSource> accountSourceOptional; - private final Flags flags; - private final Optional<ExperimentationConfig> experimentationConfigOptional; - - public MainMddLibModule( - SynchronousFileStorage fileStorage, - NetworkUsageMonitor networkUsageMonitor, - EventLogger eventLogger, - Optional<DownloadProgressMonitor> downloadProgressMonitorOptional, - Optional<SilentFeedback> silentFeedbackOptional, - Optional<String> instanceId, - Optional<AccountSource> accountSourceOptional, - Flags flags, - Optional<ExperimentationConfig> experimentationConfigOptional) { - this.fileStorage = fileStorage; - this.networkUsageMonitor = networkUsageMonitor; - this.eventLogger = eventLogger; - this.downloadProgressMonitorOptional = downloadProgressMonitorOptional; - this.silentFeedbackOptional = silentFeedbackOptional; - this.instanceId = instanceId; - this.accountSourceOptional = accountSourceOptional; - this.flags = flags; - this.experimentationConfigOptional = experimentationConfigOptional; - } - - @Provides - @Singleton - static FileGroupsMetadata provideFileGroupsMetadata( - SharedPreferencesFileGroupsMetadata fileGroupsMetadata) { - return fileGroupsMetadata; - } - - @Provides - @Singleton - static SharedFilesMetadata provideSharedFilesMetadata( - SharedPreferencesSharedFilesMetadata sharedFilesMetadata) { - return sharedFilesMetadata; - } - - @Provides - @Singleton - EventLogger provideEventLogger() { - return eventLogger; - } - - @Provides - @Singleton - SilentFeedback providesSilentFeedback() { - if (this.silentFeedbackOptional.isPresent()) { - return this.silentFeedbackOptional.get(); - } else { - return (throwable, description, args) -> { - // No-op SilentFeedback. - }; - } - } - - @Provides - @Singleton - Optional<AccountSource> provideAccountSourceOptional(@ApplicationContext Context context) { - return this.accountSourceOptional; - } - - @Provides - @Singleton - static TimeSource provideTimeSource() { - return System::currentTimeMillis; - } - - @Provides - @Singleton - @InstanceId - Optional<String> provideInstanceId() { - return this.instanceId; - } - - @Provides - @Singleton - NetworkUsageMonitor provideNetworkUsageMonitor() { - return this.networkUsageMonitor; - } - - @Provides - @Singleton - // TODO(b/243706147): We don't need to have @Singleton here and few other places in this - // class since it comes from the this instance. We should remove this since it could - // increase APK size. - Optional<DownloadProgressMonitor> provideDownloadProgressMonitor() { - return this.downloadProgressMonitorOptional; - } - - @Provides - @Singleton - SynchronousFileStorage provideSynchronousFileStorage() { - return this.fileStorage; - } - - @Provides - @Singleton - Flags provideFlags() { - return this.flags; - } - - @Provides - Optional<ExperimentationConfig> provideExperimentationConfigOptional() { - return this.experimentationConfigOptional; - } - - @Provides - @Singleton - static FuturesUtil provideFuturesUtil(@SequentialControlExecutor Executor sequentialExecutor) { - return new FuturesUtil(sequentialExecutor); - } - - @Provides - @Singleton - static LoggingStateStore provideLoggingStateStore( - @ApplicationContext Context context, - @InstanceId Optional<String> instanceId, - TimeSource timeSource, - @SequentialControlExecutor Executor sequentialExecutor) { - return SharedPreferencesLoggingState.createFromContext( - context, instanceId, timeSource, sequentialExecutor, new SecureRandom()); - } - - @Provides - static DownloadStageManager provideDownloadStageManager( - FileGroupsMetadata fileGroupsMetadata, - Optional<ExperimentationConfig> experimentationConfigOptional, - @SequentialControlExecutor Executor executor, - Flags flags) { - return new NoOpDownloadStageManager(); - } + /** The version of MDD library. Same as mdi_download module version. */ + // TODO(b/122271766): Figure out how to update this automatically. + // LINT.IfChange + public static final int MDD_LIB_VERSION = 516938429; + // LINT.ThenChange(<internal>) + + private final SynchronousFileStorage fileStorage; + + private final NetworkUsageMonitor networkUsageMonitor; + + private final EventLogger eventLogger; + + private final Optional<DownloadProgressMonitor> downloadProgressMonitorOptional; + + private final Optional<SilentFeedback> silentFeedbackOptional; + + private final Optional<String> instanceId; + + private final Optional<AccountSource> accountSourceOptional; + + private final Flags flags; + + private final Optional<ExperimentationConfig> experimentationConfigOptional; + + public MainMddLibModule( + SynchronousFileStorage fileStorage, + NetworkUsageMonitor networkUsageMonitor, + EventLogger eventLogger, + Optional<DownloadProgressMonitor> downloadProgressMonitorOptional, + Optional<SilentFeedback> silentFeedbackOptional, + Optional<String> instanceId, + Optional<AccountSource> accountSourceOptional, + Flags flags, + Optional<ExperimentationConfig> experimentationConfigOptional) { + this.fileStorage = fileStorage; + this.networkUsageMonitor = networkUsageMonitor; + this.eventLogger = eventLogger; + this.downloadProgressMonitorOptional = downloadProgressMonitorOptional; + this.silentFeedbackOptional = silentFeedbackOptional; + this.instanceId = instanceId; + this.accountSourceOptional = accountSourceOptional; + this.flags = flags; + this.experimentationConfigOptional = experimentationConfigOptional; + } + + @Provides + @Singleton + static FileGroupsMetadata provideFileGroupsMetadata( + SharedPreferencesFileGroupsMetadata fileGroupsMetadata) { + return fileGroupsMetadata; + } + + @Provides + @Singleton + static SharedFilesMetadata provideSharedFilesMetadata( + SharedPreferencesSharedFilesMetadata sharedFilesMetadata) { + return sharedFilesMetadata; + } + + @Provides + @Singleton + @SuppressWarnings("Framework.StaticProvides") + EventLogger provideEventLogger() { + return eventLogger; + } + + @Provides + @Singleton + @SuppressWarnings("Framework.StaticProvides") + SilentFeedback providesSilentFeedback() { + if (this.silentFeedbackOptional.isPresent()) { + return this.silentFeedbackOptional.get(); + } else { + return (throwable, description, args) -> { + // No-op SilentFeedback. + }; + } + } + + @Provides + @Singleton + @SuppressWarnings("Framework.StaticProvides") + Optional<AccountSource> provideAccountSourceOptional(@ApplicationContext Context context) { + return this.accountSourceOptional; + } + + @Provides + @Singleton + static TimeSource provideTimeSource() { + return new AndroidTimeSource(); + } + + @Provides + @Singleton + @InstanceId + @SuppressWarnings("Framework.StaticProvides") + Optional<String> provideInstanceId() { + return this.instanceId; + } + + @Provides + @Singleton + @SuppressWarnings("Framework.StaticProvides") + NetworkUsageMonitor provideNetworkUsageMonitor() { + return this.networkUsageMonitor; + } + + @Provides + @Singleton + @SuppressWarnings("Framework.StaticProvides") + // TODO: We don't need to have @Singleton here and few other places in this class + // since it comes from the this instance. We should remove this since it could increase APK size. + Optional<DownloadProgressMonitor> provideDownloadProgressMonitor() { + return this.downloadProgressMonitorOptional; + } + + @Provides + @Singleton + @SuppressWarnings("Framework.StaticProvides") + SynchronousFileStorage provideSynchronousFileStorage() { + return this.fileStorage; + } + + @Provides + @Singleton + @SuppressWarnings("Framework.StaticProvides") + Flags provideFlags() { + return this.flags; + } + + @Provides + @SuppressWarnings("Framework.StaticProvides") + Optional<ExperimentationConfig> provideExperimentationConfigOptional() { + return this.experimentationConfigOptional; + } + + @Provides + @Singleton + static FuturesUtil provideFuturesUtil(@SequentialControlExecutor Executor sequentialExecutor) { + return new FuturesUtil(sequentialExecutor); + } + + @Provides + @Singleton + static LoggingStateStore provideLoggingStateStore( + @ApplicationContext Context context, + @InstanceId Optional<String> instanceId, + TimeSource timeSource, + @SequentialControlExecutor Executor sequentialExecutor) { + return SharedPreferencesLoggingState.createFromContext( + context, instanceId, timeSource, sequentialExecutor, new SecureRandom()); + } + + @Provides + @SuppressWarnings("Framework.StaticProvides") + DownloadStageManager provideDownloadStageManager( + FileGroupsMetadata fileGroupsMetadata, + Optional<ExperimentationConfig> experimentationConfigOptional, + @SequentialControlExecutor Executor executor, + Flags flags) { + return new NoOpDownloadStageManager(); + } } diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/dagger/StandaloneComponent.java b/java/com/google/android/libraries/mobiledatadownload/internal/dagger/StandaloneComponent.java index 48367a4..b4cae41 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/dagger/StandaloneComponent.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/dagger/StandaloneComponent.java @@ -15,6 +15,7 @@ */ package com.google.android.libraries.mobiledatadownload.internal.dagger; +import com.google.android.libraries.mobiledatadownload.TimeSource; import com.google.android.libraries.mobiledatadownload.internal.MobileDataDownloadManager; import com.google.android.libraries.mobiledatadownload.internal.logging.EventLogger; import com.google.android.libraries.mobiledatadownload.internal.logging.LoggingStateStore; @@ -38,4 +39,6 @@ public abstract class StandaloneComponent { // TODO(b/214632773): remove this when event logger can be constructed internally public abstract LoggingStateStore getLoggingStateStore(); + + public abstract TimeSource getTimeSource(); } diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/downloader/BUILD b/java/com/google/android/libraries/mobiledatadownload/internal/downloader/BUILD index 5d239c1..4288856 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/downloader/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/internal/downloader/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = ["//:__subpackages__"], licenses = ["notice"], ) @@ -34,8 +35,11 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LogUtil", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LoggingStateStore", "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", + "//java/com/google/android/libraries/mobiledatadownload/internal/util:DownloadFutureMap", + "//java/com/google/android/libraries/mobiledatadownload/internal/util:FileGroupUtil", "//java/com/google/android/libraries/mobiledatadownload/monitor:DownloadProgressMonitor", "//java/com/google/android/libraries/mobiledatadownload/monitor:NetworkUsageMonitor", + "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", "@androidx_annotation_annotation", "@com_google_code_findbugs_jsr305", "@com_google_dagger", @@ -85,6 +89,8 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", "//java/com/google/android/libraries/mobiledatadownload/internal/util:FileGroupUtil", "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", + "//proto:log_enums_java_proto_lite", + "//proto:logs_java_proto_lite", "@androidx_annotation_annotation", "@com_google_guava_guava", ], @@ -109,6 +115,8 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", "//java/com/google/android/libraries/mobiledatadownload/internal/util:DirectoryUtil", "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", + "//proto:log_enums_java_proto_lite", + "//proto:logs_java_proto_lite", "@com_google_guava_guava", ], ) diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/downloader/DeltaFileDownloaderCallbackImpl.java b/java/com/google/android/libraries/mobiledatadownload/internal/downloader/DeltaFileDownloaderCallbackImpl.java index a84397a..345616d 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/downloader/DeltaFileDownloaderCallbackImpl.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/downloader/DeltaFileDownloaderCallbackImpl.java @@ -44,6 +44,7 @@ import com.google.mobiledatadownload.internal.MetadataProto.DeltaFile; import com.google.mobiledatadownload.internal.MetadataProto.FileStatus; import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; import com.google.mobiledatadownload.internal.MetadataProto.NewFileKey; +import com.google.mobiledatadownload.LogProto.DataDownloadFileGroupStats; import java.io.IOException; import java.util.concurrent.Executor; @@ -210,7 +211,7 @@ public final class DeltaFileDownloaderCallbackImpl implements DownloaderCallback baseFileKey.getChecksum(), silentFeedback, instanceId, - /* androidShared = */ false); + /* androidShared= */ false); } if (baseFileUri == null) { @@ -237,7 +238,14 @@ public final class DeltaFileDownloaderCallbackImpl implements DownloaderCallback .setCause(e) .build()); } - Void fileGroupStats = null; + DataDownloadFileGroupStats fileGroupStats = + DataDownloadFileGroupStats.newBuilder() + .setFileGroupName(groupKey.getGroupName()) + .setFileGroupVersionNumber(fileGroupVersionNumber) + .setOwnerPackage(groupKey.getOwnerPackage()) + .setBuildId(buildId) + .setVariantId(variantId) + .build(); eventLogger.logMddNetworkSavings( fileGroupStats, 0, diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/downloader/DownloaderCallbackImpl.java b/java/com/google/android/libraries/mobiledatadownload/internal/downloader/DownloaderCallbackImpl.java index 897261d..bd784b3 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/downloader/DownloaderCallbackImpl.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/downloader/DownloaderCallbackImpl.java @@ -40,6 +40,8 @@ import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFluentF import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; import com.google.common.io.ByteStreams; import com.google.common.util.concurrent.ListenableFuture; +import com.google.mobiledatadownload.LogEnumsProto.MddClientEvent; +import com.google.mobiledatadownload.LogProto.DataDownloadFileGroupStats; import com.google.mobiledatadownload.internal.MetadataProto.DataFile; import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal.AllowedReaders; import com.google.mobiledatadownload.internal.MetadataProto.FileStatus; @@ -262,14 +264,21 @@ public class DownloaderCallbackImpl implements DownloaderCallback { long fullFileSize = fileStorage.fileSize(target); long downloadedFileSize = fileStorage.fileSize(source); if (fullFileSize > downloadedFileSize) { - Void fileGroupStats = null; + DataDownloadFileGroupStats fileGroupStats = + DataDownloadFileGroupStats.newBuilder() + .setFileGroupName(groupKey.getGroupName()) + .setFileGroupVersionNumber(fileGroupVersionNumber) + .setBuildId(buildId) + .setVariantId(variantId) + .setOwnerPackage(groupKey.getOwnerPackage()) + .build(); eventLogger.logMddNetworkSavings( fileGroupStats, 0, fullFileSize, downloadedFileSize, dataFile.getFileId(), - /* deltaIndex = */ 0); + /* deltaIndex= */ 0); } } fileStorage.deleteFile(source); @@ -303,7 +312,14 @@ public class DownloaderCallbackImpl implements DownloaderCallback { .build(); } try { - Void fileGroupStats = null; + DataDownloadFileGroupStats fileGroupStats = + DataDownloadFileGroupStats.newBuilder() + .setFileGroupName(groupKey.getGroupName()) + .setFileGroupVersionNumber(fileGroupVersionNumber) + .setBuildId(buildId) + .setVariantId(variantId) + .setOwnerPackage(groupKey.getOwnerPackage()) + .build(); eventLogger.logMddNetworkSavings( fileGroupStats, 0, @@ -387,7 +403,7 @@ public class DownloaderCallbackImpl implements DownloaderCallback { "%s: Checksum mismatch detected but the has already reached retry limit!" + " Skipping removal for file %s", TAG, checksum); - eventLogger.logEventSampled(0); + eventLogger.logEventSampled(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED); } else { LogUtil.d( "%s: Removing file and marking as corrupted due to checksum mismatch", TAG); diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/downloader/MddFileDownloader.java b/java/com/google/android/libraries/mobiledatadownload/internal/downloader/MddFileDownloader.java index b1de88b..1c23a97 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/downloader/MddFileDownloader.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/downloader/MddFileDownloader.java @@ -15,6 +15,9 @@ */ package com.google.android.libraries.mobiledatadownload.internal.downloader; +import static com.google.common.util.concurrent.Futures.immediateFailedFuture; +import static com.google.common.util.concurrent.Futures.immediateFuture; +import static com.google.common.util.concurrent.Futures.immediateVoidFuture; import static java.lang.Math.min; import android.content.Context; @@ -35,14 +38,18 @@ import com.google.android.libraries.mobiledatadownload.internal.ApplicationConte import com.google.android.libraries.mobiledatadownload.internal.annotations.SequentialControlExecutor; import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; import com.google.android.libraries.mobiledatadownload.internal.logging.LoggingStateStore; +import com.google.android.libraries.mobiledatadownload.internal.util.DownloadFutureMap; +import com.google.android.libraries.mobiledatadownload.internal.util.FileGroupUtil; import com.google.android.libraries.mobiledatadownload.monitor.DownloadProgressMonitor; import com.google.android.libraries.mobiledatadownload.monitor.NetworkUsageMonitor; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFluentFuture; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; import com.google.common.base.Optional; import com.google.common.base.Supplier; import com.google.common.collect.ImmutableList; -import com.google.common.util.concurrent.FluentFuture; -import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.AsyncFunction; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListenableFutureTask; import com.google.mobiledatadownload.internal.MetadataProto.DownloadConditions; import com.google.mobiledatadownload.internal.MetadataProto.DownloadConditions.DeviceNetworkPolicy; import com.google.mobiledatadownload.internal.MetadataProto.ExtraHttpHeader; @@ -80,8 +87,18 @@ public class MddFileDownloader { private final Flags flags; // Cache for all on-going downloads. This will be used to de-dup download requests. + // NOTE: all operations are internally sequenced through an ExecutionSequencer. + // NOTE: this map and fileUriToDownloadFutureMap are mutually exclusive and the use of + // one or the other is based on an MDD feature flag (enableFileDownloadDedupByFileKey). Once the + // flag is fully rolled out, this map will be used exclusively. + private final DownloadFutureMap<Void> downloadOrCopyFutureMap; + + // Cache for all on-going downloads. This will be used to de-dup download requests. // NOTE: currently we assume that this map will only be accessed through the // SequentialControlExecutor, so we don't need synchronization here. + // NOTE: this map and downloadOrCopyFutureMap are mutually exclusive and the use of + // one or the other is based on an MDD feature flag (enableFileDownloadDedupByFileKey). Once the + // flag is fully rolled out, this map will not be used. @VisibleForTesting final HashMap<Uri, ListenableFuture<Void>> fileUriToDownloadFutureMap = new HashMap<>(); @@ -103,14 +120,17 @@ public class MddFileDownloader { this.loggingStateStore = loggingStateStore; this.sequentialControlExecutor = sequentialControlExecutor; this.flags = flags; + this.downloadOrCopyFutureMap = DownloadFutureMap.create(sequentialControlExecutor); } /** * Start downloading the file. * + * @param fileKey key that identifies the shared file to download. * @param groupKey GroupKey that contains the file to download. * @param fileGroupVersionNumber version number of the group that contains the file to download. * @param buildId build id of the group that contains the file to download. + * @param variantId variant id of the group that contains the file to download. * @param fileUri - the File Uri to download the file at. * @param urlToDownload - The url of the file to download. * @param fileSize - the expected size of the file to download. @@ -121,9 +141,11 @@ public class MddFileDownloader { * @return - ListenableFuture representing the download result of a file. */ public ListenableFuture<Void> startDownloading( + String fileKey, GroupKey groupKey, int fileGroupVersionNumber, long buildId, + String variantId, Uri fileUri, String urlToDownload, int fileSize, @@ -131,77 +153,132 @@ public class MddFileDownloader { DownloaderCallback callback, int trafficTag, List<ExtraHttpHeader> extraHttpHeaders) { - if (fileUriToDownloadFutureMap.containsKey(fileUri)) { - return fileUriToDownloadFutureMap.get(fileUri); - } - return addCallbackAndRegister( - fileUri, - callback, - startDownloadingInternal( - groupKey, - fileGroupVersionNumber, - buildId, - fileUri, - urlToDownload, - fileSize, - downloadConditions, - trafficTag, - extraHttpHeaders)); + return PropagatedFutures.transformAsync( + getInProgressFuture(fileKey, fileUri), + inProgressFuture -> { + if (inProgressFuture.isPresent()) { + return inProgressFuture.get(); + } + return addCallbackAndRegister( + fileKey, + fileUri, + callback, + unused -> + startDownloadingInternal( + groupKey, + fileGroupVersionNumber, + buildId, + variantId, + fileUri, + urlToDownload, + fileSize, + downloadConditions, + trafficTag, + extraHttpHeaders)); + }, + sequentialControlExecutor); } /** * Adds Callback to given Future and Registers future in in-progress cache. * - * <p>Contains shared logic of connecting {@code callback} to {@code downloadOrCopyFuture} and + * <p>Contains shared logic of connecting {@code callback} to {@code downloadOrCopyFunction} and * registers future in the internal in-progress cache. This cache allows similar download/copy * requests to be deduped instead of being performed twice. * * <p>NOTE: this method assumes the cache has already been checked for an in-progress operation * and no in-progress operation exists for {@code fileUri}. * + * @param fileKey key that identifies the shared file. * @param fileUri the destination of the download/copy (used as Key in in-progress cache) * @param callback the callback that should be run after the given download/copy future - * @param downloadOrCopyFuture a ListenableFuture that will perform the download/copy + * @param downloadOrCopyFunction an AsyncFunction that will perform the download/copy * @return a ListenableFuture that calls the correct callback after {@code downloadOrCopyFuture * completes} */ private ListenableFuture<Void> addCallbackAndRegister( - Uri fileUri, DownloaderCallback callback, ListenableFuture<Void> downloadOrCopyFuture) { + String fileKey, + Uri fileUri, + DownloaderCallback callback, + AsyncFunction<Void, Void> downloadOrCopyFunction) { + // Use ListenableFutureTask to create a future without starting it. This ensures we can + // successfully add our future to download/copy before the operation starts. + ListenableFutureTask<Void> startTask = ListenableFutureTask.create(() -> null); + // Use transform & catching to ensure that we correctly chain everything. - FluentFuture<Void> transformedFuture = - FluentFuture.from(downloadOrCopyFuture) + PropagatedFluentFuture<Void> downloadOrCopyFuture = + PropagatedFluentFuture.from(startTask) + .transformAsync(downloadOrCopyFunction, sequentialControlExecutor) .transformAsync( voidArg -> callback.onDownloadComplete(fileUri), sequentialControlExecutor /*Run callbacks on @SequentialControlExecutor*/) .catchingAsync( - DownloadException.class, + Exception.class, e -> - Futures.transformAsync( - callback.onDownloadFailed(e), + // Rethrow exception so the failure is passed back up the future chain. + PropagatedFutures.transformAsync( + callback.onDownloadFailed(asDownloadException(e)), voidArg -> { throw e; }, sequentialControlExecutor), sequentialControlExecutor /*Run callbacks on @SequentialControlExecutor*/); - fileUriToDownloadFutureMap.put(fileUri, transformedFuture); + // Add this future to the future map, then start startTask to unblock download/copy. The order + // ensures that the download/copy happens only if we were able to add the future to the map. + PropagatedFluentFuture<Void> transformedFuture = + PropagatedFluentFuture.from(addFutureToMap(downloadOrCopyFuture, fileKey, fileUri)) + .transformAsync( + unused -> { + startTask.run(); + return downloadOrCopyFuture; + }, + sequentialControlExecutor); - // We want to remove the transformedFuture from the cache when the transformedFuture finishes. + // We want to remove the future from the cache when the transformedFuture finishes. // However there may be a race condition and transformedFuture may finish before we put it into // the cache. // To prevent this race condition, we add a callback to transformedFuture to make sure the // removal happens after the putting it in the map. // A transform would not work since we want to run the removal even when the transform failed. transformedFuture.addListener( - () -> fileUriToDownloadFutureMap.remove(fileUri), sequentialControlExecutor); + () -> { + ListenableFuture<Void> unused = removeFutureFromMap(fileKey, fileUri); + }, + sequentialControlExecutor); return transformedFuture; } + private ListenableFuture<Void> addFutureToMap( + ListenableFuture<Void> downloadOrCopyFuture, String fileKey, Uri fileUri) { + if (!flags.enableFileDownloadDedupByFileKey()) { + fileUriToDownloadFutureMap.put(fileUri, downloadOrCopyFuture); + return immediateVoidFuture(); + } else { + return downloadOrCopyFutureMap.add(fileKey, downloadOrCopyFuture); + } + } + + private ListenableFuture<Void> removeFutureFromMap(String fileKey, Uri fileUri) { + if (!flags.enableFileDownloadDedupByFileKey()) { + // Return the removed future if it exists, otherwise return immediately (Extra check added to + // satisfy nullness checker). + ListenableFuture<Void> removedFuture = fileUriToDownloadFutureMap.remove(fileUri); + if (removedFuture != null) { + return removedFuture; + } + return immediateVoidFuture(); + } else { + return downloadOrCopyFutureMap.remove(fileKey); + } + } + private ListenableFuture<Void> startDownloadingInternal( GroupKey groupKey, int fileGroupVersionNumber, long buildId, + String variantId, Uri fileUri, String urlToDownload, int fileSize, @@ -212,7 +289,7 @@ public class MddFileDownloader { && flags.downloaderEnforceHttps() && !urlToDownload.startsWith("https")) { LogUtil.e("%s: File url = %s is not secure", TAG, urlToDownload); - return Futures.immediateFailedFuture( + return immediateFailedFuture( DownloadException.builder() .setDownloadResultCode(DownloadResultCode.INSECURE_URL_ERROR) .build()); @@ -227,16 +304,17 @@ public class MddFileDownloader { } try { - checkStorageConstraints(context, fileSize - currentFileSize, downloadConditions, flags); + checkStorageConstraints( + context, urlToDownload, fileSize - currentFileSize, downloadConditions, flags); } catch (DownloadException e) { // Wrap exception in future to break future chain. LogUtil.e("%s: Not enough space to download file %s", TAG, urlToDownload); - return Futures.immediateFailedFuture(e); + return immediateFailedFuture(e); } if (flags.logNetworkStats()) { networkUsageMonitor.monitorUri( - fileUri, groupKey, buildId, fileGroupVersionNumber, loggingStateStore); + fileUri, groupKey, buildId, variantId, fileGroupVersionNumber, loggingStateStore); } else { LogUtil.w("%s: NetworkUsageMonitor is disabled", TAG); } @@ -273,8 +351,29 @@ public class MddFileDownloader { } /** + * Gets an in-progress future (if it exists), otherwise returns absent. + * + * <p>This method allows easier deduplication of file downloads/copies, by allowing callers to + * query against the internal download future map. This method is assumed to be called when a + * SharedFile state is DOWNLOAD_IN_PROGRESS. + * + * @param fileKey key that identifies the shared file. + * @param fileUri - the File Uri to download the file at. + * @return - ListenableFuture representing an in-progress download/copy for the given file. + */ + public ListenableFuture<Optional<ListenableFuture<Void>>> getInProgressFuture( + String fileKey, Uri fileUri) { + if (!flags.enableFileDownloadDedupByFileKey()) { + return immediateFuture(Optional.fromNullable(fileUriToDownloadFutureMap.get(fileUri))); + } else { + return downloadOrCopyFutureMap.get(fileKey); + } + } + + /** * Start Copying a file to internal storage * + * @param fileKey key that identifies the shared file to copy. * @param fileUri the File Uri where content should be copied. * @param urlToDownload the url to copy, should be inlinefile: scheme. * @param fileSize the size of the file to copy. @@ -284,20 +383,28 @@ public class MddFileDownloader { * @return ListenableFuture representing the result of a file copy. */ public ListenableFuture<Void> startCopying( + String fileKey, Uri fileUri, String urlToDownload, int fileSize, @Nullable DownloadConditions downloadConditions, DownloaderCallback downloaderCallback, FileSource inlineFileSource) { - if (fileUriToDownloadFutureMap.containsKey(fileUri)) { - return fileUriToDownloadFutureMap.get(fileUri); - } - return addCallbackAndRegister( - fileUri, - downloaderCallback, - startCopyingInternal( - fileUri, urlToDownload, fileSize, downloadConditions, inlineFileSource)); + return PropagatedFutures.transformAsync( + getInProgressFuture(fileKey, fileUri), + inProgressFuture -> { + if (inProgressFuture.isPresent()) { + return inProgressFuture.get(); + } + return addCallbackAndRegister( + fileKey, + fileUri, + downloaderCallback, + unused -> + startCopyingInternal( + fileUri, urlToDownload, fileSize, downloadConditions, inlineFileSource)); + }, + sequentialControlExecutor); } private ListenableFuture<Void> startCopyingInternal( @@ -307,12 +414,24 @@ public class MddFileDownloader { @Nullable DownloadConditions downloadConditions, FileSource inlineFileSource) { + int finalFileSize = fileSize; + if (inlineFileSource.getKind().equals(FileSource.Kind.BYTESTRING)) { + int sourceFileSize = inlineFileSource.byteString().size(); + if (sourceFileSize != fileSize) { + LogUtil.w( + "%s: expected file size (%d) does not match source file size (%d) -- using source file" + + " size for storage check; file: %s", + TAG, fileSize, sourceFileSize, urlToCopy); + finalFileSize = sourceFileSize; + } + } + try { - checkStorageConstraints(context, fileSize, downloadConditions, flags); + checkStorageConstraints(context, urlToCopy, finalFileSize, downloadConditions, flags); } catch (DownloadException e) { // Wrap exception in future to break future chain. LogUtil.e("%s: Not enough space to download file %s", TAG, urlToCopy); - return Futures.immediateFailedFuture(e); + return immediateFailedFuture(e); } // TODO(b/177361344): Only monitor file if download listener is supported @@ -332,17 +451,24 @@ public class MddFileDownloader { /** * Stop downloading the file. * + * @param fileKey - key that identifies the file to stop downloading. * @param fileUri - the File Uri of the file to stop downloading. */ - public void stopDownloading(Uri fileUri) { - ListenableFuture<Void> pendingDownloadFuture = fileUriToDownloadFutureMap.get(fileUri); - if (pendingDownloadFuture != null) { - LogUtil.d("%s: Cancel download file %s", TAG, fileUri); - fileUriToDownloadFutureMap.remove(fileUri); - pendingDownloadFuture.cancel(true); - } else { - LogUtil.w("%s: stopDownloading on non-existent download", TAG); - } + public void stopDownloading(String fileKey, Uri fileUri) { + ListenableFuture<Void> unused = + PropagatedFutures.transformAsync( + getInProgressFuture(fileKey, fileUri), + inProgressFuture -> { + if (inProgressFuture.isPresent()) { + LogUtil.d("%s: Cancel download file %s", TAG, fileUri); + inProgressFuture.get().cancel(/* mayInterruptIfRunning= */ true); + return removeFutureFromMap(fileKey, fileUri); + } else { + LogUtil.w("%s: stopDownloading on non-existent download", TAG); + return immediateVoidFuture(); + } + }, + sequentialControlExecutor); } /** @@ -363,14 +489,15 @@ public class MddFileDownloader { * @throws DownloadException when storing a file with the given size would hit the given storage * thresholds */ - public static void checkStorageConstraints( + private static void checkStorageConstraints( Context context, + String url, long bytesNeeded, @Nullable DownloadConditions downloadConditions, Flags flags) throws DownloadException { if (flags.enforceLowStorageBehavior() - && !shouldDownload(context, bytesNeeded, downloadConditions, flags)) { + && !shouldDownload(context, url, bytesNeeded, downloadConditions, flags)) { throw DownloadException.builder() .setDownloadResultCode(DownloadResultCode.LOW_DISK_ERROR) .build(); @@ -385,9 +512,15 @@ public class MddFileDownloader { */ private static boolean shouldDownload( Context context, + String url, long bytesNeeded, @Nullable DownloadConditions downloadConditions, Flags flags) { + // If we are using a placeholder (inline file + 0 byte size), bypass storage checks. + if (FileGroupUtil.isInlineFile(url) && bytesNeeded == 0L) { + return true; + } + StatFs stats = new StatFs(context.getFilesDir().getAbsolutePath()); long totalBytes = (long) stats.getBlockCount() * stats.getBlockSize(); @@ -421,6 +554,23 @@ public class MddFileDownloader { return remainingBytesAfterDownload > minBytes; } + /** + * Wraps throwable as DownloadException if it isn't one already. + * + * <p>This method doesn't check the incoming throwable besides the type and defaults the download + * result code to UNKNOWN_ERROR. + */ + private static DownloadException asDownloadException(Throwable t) { + if (t instanceof DownloadException) { + return (DownloadException) t; + } + + return DownloadException.builder() + .setCause(t) + .setDownloadResultCode(DownloadResultCode.UNKNOWN_ERROR) + .build(); + } + /** Interface called by the downloader when download either completes or fails. */ public static interface DownloaderCallback { /** Called on download complete. */ diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/experimentation/BUILD b/java/com/google/android/libraries/mobiledatadownload/internal/experimentation/BUILD index ffa6fc9..e6857e1 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/experimentation/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/internal/experimentation/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], @@ -25,6 +26,7 @@ android_library( srcs = ["DownloadStageManager.java"], deps = [ "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", + "@androidx_annotation_annotation", "@com_google_errorprone_error_prone_annotations", "@com_google_guava_guava", ], @@ -36,6 +38,7 @@ android_library( deps = [ ":DownloadStageManager", "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", + "@androidx_annotation_annotation", "@com_google_errorprone_error_prone_annotations", "@com_google_guava_guava", ], diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/logging/BUILD b/java/com/google/android/libraries/mobiledatadownload/internal/logging/BUILD index 8ea9550..6ce86c3 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/logging/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/internal/logging/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = ["//:__subpackages__"], licenses = ["notice"], ) @@ -31,6 +32,8 @@ android_library( name = "EventLogger", srcs = ["EventLogger.java"], deps = [ + "//proto:log_enums_java_proto_lite", + "//proto:logs_java_proto_lite", "@com_google_auto_value", "@com_google_guava_guava", ], @@ -41,6 +44,8 @@ android_library( srcs = ["NoOpEventLogger.java"], deps = [ ":EventLogger", + "//proto:log_enums_java_proto_lite", + "//proto:logs_java_proto_lite", "@com_google_guava_guava", ], ) @@ -53,8 +58,12 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/internal:FileGroupManager", "//java/com/google/android/libraries/mobiledatadownload/internal:FileGroupsMetadata", "//java/com/google/android/libraries/mobiledatadownload/internal/annotations:SequentialControlExecutor", + "//java/com/google/android/libraries/mobiledatadownload/internal/collect", "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", + "//java/com/google/android/libraries/mobiledatadownload/internal/util:FileGroupUtil", "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", + "//proto:log_enums_java_proto_lite", + "//proto:logs_java_proto_lite", "@com_google_guava_guava", "@javax_inject", ], @@ -67,7 +76,10 @@ android_library( ], deps = [ ":EventLogger", + ":LogUtil", "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", + "//proto:log_enums_java_proto_lite", + "//proto:logs_java_proto_lite", "@androidx_annotation_annotation", "@com_google_errorprone_error_prone_annotations", ], @@ -79,13 +91,15 @@ android_library( "MddEventLogger.java", ], deps = [ - ":EventLogger", - ":LogSampler", - ":LogUtil", - ":LoggingStateStore", "//java/com/google/android/libraries/mobiledatadownload:Flags", "//java/com/google/android/libraries/mobiledatadownload:Logger", + "//java/com/google/android/libraries/mobiledatadownload/internal/logging:EventLogger", + "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LogSampler", + "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LogUtil", + "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LoggingStateStore", "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", + "//proto:log_enums_java_proto_lite", + "//proto:logs_java_proto_lite", "@com_google_guava_guava", ], ) @@ -99,6 +113,7 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload:SilentFeedback", "//java/com/google/android/libraries/mobiledatadownload/annotations", "//java/com/google/android/libraries/mobiledatadownload/file", + "//java/com/google/android/libraries/mobiledatadownload/file/openers:recursive_size", "//java/com/google/android/libraries/mobiledatadownload/internal:ApplicationContext", "//java/com/google/android/libraries/mobiledatadownload/internal:FileGroupsMetadata", "//java/com/google/android/libraries/mobiledatadownload/internal:MddConstants", @@ -106,10 +121,12 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/internal:SharedFileManager", "//java/com/google/android/libraries/mobiledatadownload/internal:SharedFilesMetadata", "//java/com/google/android/libraries/mobiledatadownload/internal/annotations:SequentialControlExecutor", + "//java/com/google/android/libraries/mobiledatadownload/internal/collect", "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", + "//java/com/google/android/libraries/mobiledatadownload/internal/util:DirectoryUtil", "//java/com/google/android/libraries/mobiledatadownload/internal/util:FileGroupUtil", "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", - "@com_google_auto_value", + "//proto:logs_java_proto_lite", "@com_google_guava_guava", "@javax_inject", ], @@ -120,12 +137,13 @@ android_library( srcs = ["NetworkLogger.java"], deps = [ ":EventLogger", - ":LoggingStateStore", "//java/com/google/android/libraries/mobiledatadownload:Flags", "//java/com/google/android/libraries/mobiledatadownload/annotations", "//java/com/google/android/libraries/mobiledatadownload/internal:ApplicationContext", + "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LoggingStateStore", "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", + "//proto:logs_java_proto_lite", "@com_google_guava_guava", "@javax_inject", ], @@ -149,7 +167,10 @@ android_library( ":LogUtil", ":LoggingStateStore", "//java/com/google/android/libraries/mobiledatadownload:Flags", + "//java/com/google/android/libraries/mobiledatadownload/tracing", "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", + "//java/com/google/protobuf/util:time_lite", + "//proto:logs_java_proto_lite", "@com_google_errorprone_error_prone_annotations", "@com_google_guava_guava", ], @@ -166,3 +187,23 @@ android_library( "@com_google_guava_guava", ], ) + +android_library( + name = "SharedPreferencesLoggingState", + srcs = [ + "SharedPreferencesLoggingState.java", + ], + deps = [ + ":LoggingStateStore", + "//google/protobuf:timestamp_java_proto_lite", + "//java/com/google/android/libraries/mobiledatadownload:TimeSource", + "//java/com/google/android/libraries/mobiledatadownload/internal:MddConstants", + "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", + "//java/com/google/android/libraries/mobiledatadownload/internal/util:FileGroupsMetadataUtil", + "//java/com/google/android/libraries/mobiledatadownload/internal/util:SharedPreferencesUtil", + "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", + "//java/com/google/protobuf/util:time_lite", + "@androidx_annotation_annotation", + "@com_google_guava_guava", + ], +) diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/logging/DownloadStateLogger.java b/java/com/google/android/libraries/mobiledatadownload/internal/logging/DownloadStateLogger.java index a8f388f..85b13a9 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/logging/DownloadStateLogger.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/logging/DownloadStateLogger.java @@ -15,8 +15,9 @@ */ package com.google.android.libraries.mobiledatadownload.internal.logging; -import androidx.annotation.VisibleForTesting; import com.google.errorprone.annotations.CheckReturnValue; +import com.google.mobiledatadownload.LogEnumsProto.MddClientEvent; +import com.google.mobiledatadownload.LogProto.DataDownloadFileGroupStats; import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupBookkeeping; import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; @@ -25,8 +26,8 @@ import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInterna public final class DownloadStateLogger { private static final String TAG = "FileGroupStatusLogger"; - @VisibleForTesting - enum Operation { + /** The type of operation for which the logger will log events. */ + public enum Operation { DOWNLOAD, IMPORT, }; @@ -47,13 +48,18 @@ public final class DownloadStateLogger { return new DownloadStateLogger(eventLogger, Operation.IMPORT); } + /** Gets the operation associated with this logger. */ + public Operation getOperation() { + return operation; + } + public void logStarted(DataFileGroupInternal fileGroup) { switch (operation) { case DOWNLOAD: - logEventWithDataFileGroup(0, fileGroup); + logEventWithDataFileGroup(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, fileGroup); break; case IMPORT: - logEventWithDataFileGroup(0, fileGroup); + logEventWithDataFileGroup(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, fileGroup); break; } } @@ -61,10 +67,10 @@ public final class DownloadStateLogger { public void logPending(DataFileGroupInternal fileGroup) { switch (operation) { case DOWNLOAD: - logEventWithDataFileGroup(0, fileGroup); + logEventWithDataFileGroup(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, fileGroup); break; case IMPORT: - logEventWithDataFileGroup(0, fileGroup); + logEventWithDataFileGroup(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, fileGroup); break; } } @@ -72,10 +78,10 @@ public final class DownloadStateLogger { public void logFailed(DataFileGroupInternal fileGroup) { switch (operation) { case DOWNLOAD: - logEventWithDataFileGroup(0, fileGroup); + logEventWithDataFileGroup(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, fileGroup); break; case IMPORT: - logEventWithDataFileGroup(0, fileGroup); + logEventWithDataFileGroup(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, fileGroup); break; } } @@ -83,11 +89,11 @@ public final class DownloadStateLogger { public void logComplete(DataFileGroupInternal fileGroup) { switch (operation) { case DOWNLOAD: - logEventWithDataFileGroup(0, fileGroup); + logEventWithDataFileGroup(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, fileGroup); logDownloadLatency(fileGroup); break; case IMPORT: - logEventWithDataFileGroup(0, fileGroup); + logEventWithDataFileGroup(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, fileGroup); break; } } @@ -99,7 +105,15 @@ public final class DownloadStateLogger { return; } - Void fileGroupDetails = null; + DataDownloadFileGroupStats fileGroupDetails = + DataDownloadFileGroupStats.newBuilder() + .setOwnerPackage(fileGroup.getOwnerPackage()) + .setFileGroupName(fileGroup.getGroupName()) + .setFileGroupVersionNumber(fileGroup.getFileGroupVersionNumber()) + .setFileCount(fileGroup.getFileCount()) + .setBuildId(fileGroup.getBuildId()) + .setVariantId(fileGroup.getVariantId()) + .build(); DataFileGroupBookkeeping bookkeeping = fileGroup.getBookkeeping(); long newFilesReceivedTimestamp = bookkeeping.getGroupNewFilesReceivedTimestamp(); @@ -111,7 +125,8 @@ public final class DownloadStateLogger { eventLogger.logMddDownloadLatency(fileGroupDetails, downloadLatency); } - private void logEventWithDataFileGroup(int code, DataFileGroupInternal fileGroup) { + private void logEventWithDataFileGroup( + MddClientEvent.Code code, DataFileGroupInternal fileGroup) { eventLogger.logEventSampled( code, fileGroup.getGroupName(), diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/logging/EventLogger.java b/java/com/google/android/libraries/mobiledatadownload/internal/logging/EventLogger.java index b128ab1..83e8311 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/logging/EventLogger.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/logging/EventLogger.java @@ -18,32 +18,32 @@ package com.google.android.libraries.mobiledatadownload.internal.logging; import com.google.auto.value.AutoValue; import com.google.common.util.concurrent.AsyncCallable; import com.google.common.util.concurrent.ListenableFuture; +import com.google.mobiledatadownload.LogEnumsProto.MddClientEvent; import com.google.mobiledatadownload.LogEnumsProto.MddDownloadResult; import com.google.mobiledatadownload.LogProto.DataDownloadFileGroupStats; import com.google.mobiledatadownload.LogProto.MddFileGroupStatus; import com.google.mobiledatadownload.LogProto.MddStorageStats; - import java.util.List; /** Interface for remote logging. */ public interface EventLogger { /** Log an mdd event */ - void logEventSampled(int eventCode); + void logEventSampled(MddClientEvent.Code eventCode); /** Log an mdd event with an associated file group. */ void logEventSampled( - int eventCode, - String fileGroupName, - int fileGroupVersionNumber, - long buildId, - String variantId); + MddClientEvent.Code eventCode, + String fileGroupName, + int fileGroupVersionNumber, + long buildId, + String variantId); /** * Log an mdd event. This not sampled. Caller should make sure this method is called after * sampling at the passed in value of sample interval. */ - void logEventAfterSample(int eventCode, int sampleInterval); + void logEventAfterSample(MddClientEvent.Code eventCode, int sampleInterval); /** * Log mdd file group stats. The buildFileGroupStats callable is only called if the event is going @@ -55,7 +55,7 @@ public interface EventLogger { * failure if the callable fails or if there is an error when logging. */ ListenableFuture<Void> logMddFileGroupStats( - AsyncCallable<List<FileGroupStatusWithDetails>> buildFileGroupStats); + AsyncCallable<List<FileGroupStatusWithDetails>> buildFileGroupStats); /** Simple wrapper class for MDD file group stats and details. */ @AutoValue @@ -65,20 +65,22 @@ public interface EventLogger { abstract DataDownloadFileGroupStats fileGroupDetails(); static FileGroupStatusWithDetails create( - MddFileGroupStatus fileGroupStatus, DataDownloadFileGroupStats fileGroupDetails) { + MddFileGroupStatus fileGroupStatus, DataDownloadFileGroupStats fileGroupDetails) { return new AutoValue_EventLogger_FileGroupStatusWithDetails( - fileGroupStatus, fileGroupDetails); + fileGroupStatus, fileGroupDetails); } } /** Log mdd api call stats. */ - void logMddApiCallStats(Void fileGroupDetails, Void apiCallStats); + void logMddApiCallStats(DataDownloadFileGroupStats fileGroupDetails, Void apiCallStats); + + void logMddLibApiResultLog(Void mddLibApiResultLog); /** * Log mdd storage stats. The buildMddStorageStats callable is only called if the event is going * to be logged. * - * @param buildMddStorageStats callable which builds the Void to log. + * @param buildMddStorageStats callable which builds the MddStorageStats to log. * @return a future that completes when the logging work is done. The future will complete with a * failure if the callable fails or if there is an error when logging. */ @@ -99,26 +101,30 @@ public interface EventLogger { /** Log the network savings of MDD download features */ void logMddNetworkSavings( - Void fileGroupDetails, - int code, - long fullFileSize, - long downloadedFileSize, - String fileId, - int deltaIndex); + DataDownloadFileGroupStats fileGroupDetails, + int code, + long fullFileSize, + long downloadedFileSize, + String fileId, + int deltaIndex); /** Log mdd download result events. */ void logMddDownloadResult( - MddDownloadResult.Code code, DataDownloadFileGroupStats fileGroupDetails); + MddDownloadResult.Code code, DataDownloadFileGroupStats fileGroupDetails); /** Log stats of mdd {@code getFileGroup} and {@code getFileGroupByFilter} calls. */ - void logMddQueryStats(Void fileGroupDetails); + void logMddQueryStats(DataDownloadFileGroupStats fileGroupDetails); /** Log mdd stats on android sharing events. */ void logMddAndroidSharingLog(Void event); /** Log mdd download latency. */ - void logMddDownloadLatency(Void fileGroupStats, Void downloadLatency); + void logMddDownloadLatency(DataDownloadFileGroupStats fileGroupStats, Void downloadLatency); /** Log mdd usage event. */ - void logMddUsageEvent(Void fileGroupDetails, Void usageEventLog); -}
\ No newline at end of file + void logMddUsageEvent(DataDownloadFileGroupStats fileGroupDetails, Void usageEventLog); + + /** Log new config received event. */ + void logNewConfigReceived( + DataDownloadFileGroupStats fileGroupDetails, Void newConfigReceivedInfo); +} diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/logging/FileGroupStatsLogger.java b/java/com/google/android/libraries/mobiledatadownload/internal/logging/FileGroupStatsLogger.java index 6ef267b..3e10eaa 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/logging/FileGroupStatsLogger.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/logging/FileGroupStatsLogger.java @@ -17,20 +17,20 @@ package com.google.android.libraries.mobiledatadownload.internal.logging; import static com.google.common.util.concurrent.Futures.immediateFuture; -import android.util.Pair; import com.google.android.libraries.mobiledatadownload.internal.FileGroupManager; import com.google.android.libraries.mobiledatadownload.internal.FileGroupManager.GroupDownloadStatus; import com.google.android.libraries.mobiledatadownload.internal.FileGroupsMetadata; import com.google.android.libraries.mobiledatadownload.internal.annotations.SequentialControlExecutor; +import com.google.android.libraries.mobiledatadownload.internal.collect.GroupKeyAndGroup; import com.google.android.libraries.mobiledatadownload.internal.util.FileGroupUtil; import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; -import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; -import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; import com.google.mobiledatadownload.LogEnumsProto.MddFileGroupDownloadStatus; import com.google.mobiledatadownload.LogProto.DataDownloadFileGroupStats; import com.google.mobiledatadownload.LogProto.MddFileGroupStatus; +import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; +import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; import java.util.ArrayList; import java.util.List; import java.util.concurrent.Executor; @@ -50,10 +50,10 @@ public class FileGroupStatsLogger { @Inject public FileGroupStatsLogger( - FileGroupManager fileGroupManager, - FileGroupsMetadata fileGroupsMetadata, - EventLogger eventLogger, - @SequentialControlExecutor Executor sequentialControlExecutor) { + FileGroupManager fileGroupManager, + FileGroupsMetadata fileGroupsMetadata, + EventLogger eventLogger, + @SequentialControlExecutor Executor sequentialControlExecutor) { this.fileGroupManager = fileGroupManager; this.fileGroupsMetadata = fileGroupsMetadata; this.eventLogger = eventLogger; @@ -66,51 +66,51 @@ public class FileGroupStatsLogger { } private ListenableFuture<List<EventLogger.FileGroupStatusWithDetails>> buildFileGroupStatusList( - int daysSinceLastLog) { + int daysSinceLastLog) { return PropagatedFutures.transformAsync( - fileGroupsMetadata.getAllFreshGroups(), - downloadedAndPendingGroups -> { - List<ListenableFuture<EventLogger.FileGroupStatusWithDetails>> futures = - new ArrayList<>(); - for (Pair<GroupKey, DataFileGroupInternal> pair : downloadedAndPendingGroups) { - GroupKey groupKey = pair.first; - DataFileGroupInternal dataFileGroup = pair.second; - if (dataFileGroup == null) { - continue; - } + fileGroupsMetadata.getAllFreshGroups(), + downloadedAndPendingGroups -> { + List<ListenableFuture<EventLogger.FileGroupStatusWithDetails>> futures = + new ArrayList<>(); + for (GroupKeyAndGroup pair : downloadedAndPendingGroups) { + GroupKey groupKey = pair.groupKey(); + DataFileGroupInternal dataFileGroup = pair.dataFileGroup(); + if (dataFileGroup == null) { + continue; + } - DataDownloadFileGroupStats fileGroupDetails = - DataDownloadFileGroupStats.newBuilder() - .setFileGroupName(groupKey.getGroupName()) - .setOwnerPackage(groupKey.getOwnerPackage()) - .setFileGroupVersionNumber(dataFileGroup.getFileGroupVersionNumber()) - .setFileCount(dataFileGroup.getFileCount()) - .setInlineFileCount(FileGroupUtil.getInlineFileCount(dataFileGroup)) - .setHasAccount(!groupKey.getAccount().isEmpty()) - .setBuildId(dataFileGroup.getBuildId()) - .setVariantId(dataFileGroup.getVariantId()) - .build(); + DataDownloadFileGroupStats fileGroupDetails = + DataDownloadFileGroupStats.newBuilder() + .setFileGroupName(groupKey.getGroupName()) + .setOwnerPackage(groupKey.getOwnerPackage()) + .setFileGroupVersionNumber(dataFileGroup.getFileGroupVersionNumber()) + .setFileCount(dataFileGroup.getFileCount()) + .setInlineFileCount(FileGroupUtil.getInlineFileCount(dataFileGroup)) + .setHasAccount(!groupKey.getAccount().isEmpty()) + .setBuildId(dataFileGroup.getBuildId()) + .setVariantId(dataFileGroup.getVariantId()) + .build(); - futures.add( - PropagatedFutures.transform( - buildFileGroupStatus(dataFileGroup, groupKey, daysSinceLastLog), - fileGroupStatus -> - EventLogger.FileGroupStatusWithDetails.create( - fileGroupStatus, fileGroupDetails), - sequentialControlExecutor)); - } - return Futures.allAsList(futures); - }, - sequentialControlExecutor); + futures.add( + PropagatedFutures.transform( + buildFileGroupStatus(dataFileGroup, groupKey, daysSinceLastLog), + fileGroupStatus -> + EventLogger.FileGroupStatusWithDetails.create( + fileGroupStatus, fileGroupDetails), + sequentialControlExecutor)); + } + return Futures.allAsList(futures); + }, + sequentialControlExecutor); } private ListenableFuture<MddFileGroupStatus> buildFileGroupStatus( - DataFileGroupInternal dataFileGroup, GroupKey groupKey, int daysSinceLastLog) { + DataFileGroupInternal dataFileGroup, GroupKey groupKey, int daysSinceLastLog) { MddFileGroupStatus.Builder fileGroupStatus = - MddFileGroupStatus.newBuilder().setDaysSinceLastLog(daysSinceLastLog); + MddFileGroupStatus.newBuilder().setDaysSinceLastLog(daysSinceLastLog); if (dataFileGroup.getBookkeeping().hasGroupNewFilesReceivedTimestamp()) { fileGroupStatus.setGroupAddedTimestampInSeconds( - dataFileGroup.getBookkeeping().getGroupNewFilesReceivedTimestamp() / 1000); + dataFileGroup.getBookkeeping().getGroupNewFilesReceivedTimestamp() / 1000); } else { fileGroupStatus.setGroupAddedTimestampInSeconds(-1); } @@ -119,7 +119,7 @@ public class FileGroupStatsLogger { fileGroupStatus.setFileGroupDownloadStatus(MddFileGroupDownloadStatus.Code.COMPLETE); if (dataFileGroup.getBookkeeping().hasGroupDownloadedTimestampInMillis()) { fileGroupStatus.setGroupDownloadedTimestampInSeconds( - dataFileGroup.getBookkeeping().getGroupDownloadedTimestampInMillis() / 1000); + dataFileGroup.getBookkeeping().getGroupDownloadedTimestampInMillis() / 1000); } else { fileGroupStatus.setGroupDownloadedTimestampInSeconds(-1); } @@ -127,19 +127,19 @@ public class FileGroupStatsLogger { } else { fileGroupStatus.setGroupDownloadedTimestampInSeconds(-1); return PropagatedFutures.transform( - fileGroupManager.getFileGroupDownloadStatus(dataFileGroup), - status -> { - if (status == GroupDownloadStatus.DOWNLOADED || status == GroupDownloadStatus.PENDING) { - // Log pending even if verify returns downloaded, as it will be marked as - // completed in the next periodic task. - fileGroupStatus.setFileGroupDownloadStatus(MddFileGroupDownloadStatus.Code.PENDING); - } else { - // TODO(b/73490689): Log the reason for failure along with this. - fileGroupStatus.setFileGroupDownloadStatus(MddFileGroupDownloadStatus.Code.FAILED); - } - return fileGroupStatus.build(); - }, - sequentialControlExecutor); + fileGroupManager.getFileGroupDownloadStatus(dataFileGroup), + status -> { + if (status == GroupDownloadStatus.DOWNLOADED || status == GroupDownloadStatus.PENDING) { + // Log pending even if verify returns downloaded, as it will be marked as + // completed in the next periodic task. + fileGroupStatus.setFileGroupDownloadStatus(MddFileGroupDownloadStatus.Code.PENDING); + } else { + // TODO(b/73490689): Log the reason for failure along with this. + fileGroupStatus.setFileGroupDownloadStatus(MddFileGroupDownloadStatus.Code.FAILED); + } + return fileGroupStatus.build(); + }, + sequentialControlExecutor); } } -}
\ No newline at end of file +} diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/logging/LogSampler.java b/java/com/google/android/libraries/mobiledatadownload/internal/logging/LogSampler.java index 6e6ac72..b25028c 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/logging/LogSampler.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/logging/LogSampler.java @@ -24,7 +24,6 @@ import com.google.common.base.Optional; import com.google.common.util.concurrent.ListenableFuture; import com.google.errorprone.annotations.CheckReturnValue; import com.google.mobiledatadownload.LogProto.StableSamplingInfo; -import com.google.protobuf.Timestamp; import java.util.Random; @@ -32,121 +31,123 @@ import java.util.Random; @CheckReturnValue public final class LogSampler { - private final Flags flags; - private final Random random; + private final Flags flags; + private final Random random; - /** - * Construct the log sampler. - * - * @param flags used to check whether stable sampling is enabled. - * @param random used to generate random numbers for event based sampling only. - */ - public LogSampler(Flags flags, Random random) { - this.flags = flags; - this.random = random; - } - - /** - * Determines whether the event should be logged. If the event should be logged it returns an - * instance of Void that should be attached to the log events. - * - * <p>If stable sampling is enabled, this is deterministic. If stable sampling is disabled, the - * result can change on each call based on the provided Random instance. - * - * @param sampleInterval the inverse sampling rate to use. This is controlled by flags per - * event-type. For stable sampling it's expected that 100 % sampleInterval == 0. - * @param loggingStateStore used to read persisted random number when stable sampling is enabled. - * If it is absent, stable sampling will not be used. - * @return a future of an optional of StableSamplingInfo. The future will resolve to an absent - * Optional if the event should not be logged. If the event should be logged, the returned - * Void should be attached to the log event. - */ - public ListenableFuture<Optional<StableSamplingInfo>> shouldLog( - long sampleInterval, Optional<LoggingStateStore> loggingStateStore) { - if (sampleInterval == 0L) { - return immediateFuture(Optional.absent()); - } else if (sampleInterval < 0L) { - LogUtil.e("Bad sample interval (negative number): %d", sampleInterval); - return immediateFuture(Optional.absent()); - } else if (flags.enableRngBasedDeviceStableSampling() && loggingStateStore.isPresent()) { - return shouldLogDeviceStable(sampleInterval, loggingStateStore.get()); - } else { - return shouldLogPerEvent(sampleInterval); + /** + * Construct the log sampler. + * + * @param flags used to check whether stable sampling is enabled. + * @param random used to generate random numbers for event based sampling only. + */ + public LogSampler(Flags flags, Random random) { + this.flags = flags; + this.random = random; } - } - /** - * Returns standard random event based sampling. - * - * @return if the event should be sampled, returns the Void with stable_sampling_used = false. - * Otherwise, returns an empty Optional. - */ - private ListenableFuture<Optional<StableSamplingInfo>> shouldLogPerEvent(long sampleInterval) { - if (shouldSamplePerEvent(sampleInterval)) { - return immediateFuture( - Optional.of(StableSamplingInfo.newBuilder().setStableSamplingUsed(false).build())); - } else { - return immediateFuture(Optional.absent()); + /** + * Determines whether the event should be logged. If the event should be logged it returns an + * instance of StableSamplingInfo that should be attached to the log events. + * + * <p>If stable sampling is enabled, this is deterministic. If stable sampling is disabled, the + * result can change on each call based on the provided Random instance. + * + * @param sampleInterval the inverse sampling rate to use. This is controlled by flags per + * event-type. For stable sampling it's expected that 100 % + * sampleInterval == 0. + * @param loggingStateStore used to read persisted random number when stable sampling is + * enabled. + * If it is absent, stable sampling will not be used. + * @return a future of an optional of StableSamplingInfo. The future will resolve to an absent + * Optional if the event should not be logged. If the event should be logged, the returned + * StableSamplingInfo should be attached to the log event. + */ + public ListenableFuture<Optional<StableSamplingInfo>> shouldLog( + long sampleInterval, Optional<LoggingStateStore> loggingStateStore) { + if (sampleInterval == 0L) { + return immediateFuture(Optional.absent()); + } else if (sampleInterval < 0L) { + LogUtil.e("Bad sample interval (negative number): %d", sampleInterval); + return immediateFuture(Optional.absent()); + } else if (flags.enableRngBasedDeviceStableSampling() && loggingStateStore.isPresent()) { + return shouldLogDeviceStable(sampleInterval, loggingStateStore.get()); + } else { + return shouldLogPerEvent(sampleInterval); + } } - } - private boolean shouldSamplePerEvent(long sampleInterval) { - if (sampleInterval == 0L) { - return false; - } else if (sampleInterval < 0L) { - LogUtil.e("Bad sample interval (negative number): %d", sampleInterval); - return false; - } else { - return isPartOfSample(random.nextLong(), sampleInterval); + /** + * Returns standard random event based sampling. + * + * @return if the event should be sampled, returns the StableSamplingInfo with + * stable_sampling_used = false. Otherwise, returns an empty Optional. + */ + private ListenableFuture<Optional<StableSamplingInfo>> shouldLogPerEvent(long sampleInterval) { + if (shouldSamplePerEvent(sampleInterval)) { + return immediateFuture( + Optional.of( + StableSamplingInfo.newBuilder().setStableSamplingUsed(false).build())); + } else { + return immediateFuture(Optional.absent()); + } } - } - /** - * Returns device stable sampling. - * - * @return if the event should be sampled, returns the Void with stable_sampling_used = true and - * all other fields populated. Otherwise, returns an empty Optional. - */ - private ListenableFuture<Optional<StableSamplingInfo>> shouldLogDeviceStable( - long sampleInterval, LoggingStateStore loggingStateStore) { - return PropagatedFluentFuture.from(loggingStateStore.getStableSamplingInfo()) - .transform( - samplingInfo -> { - boolean invalidSamplingRateUsed = ((100 % sampleInterval) != 0); - if (invalidSamplingRateUsed) { - LogUtil.e( - "Bad sample interval (1 percent cohort will not log): %d", sampleInterval); - } + private boolean shouldSamplePerEvent(long sampleInterval) { + if (sampleInterval == 0L) { + return false; + } else if (sampleInterval < 0L) { + LogUtil.e("Bad sample interval (negative number): %d", sampleInterval); + return false; + } else { + return isPartOfSample(random.nextLong(), sampleInterval); + } + } - if (!isPartOfSample(samplingInfo.getStableLogSamplingSalt(), sampleInterval)) { - return Optional.absent(); - } + /** + * Returns device stable sampling. + * + * @return if the event should be sampled, returns the StableSamplingInfo with + * stable_sampling_used = true and all other fields populated. Otherwise, returns an empty + * Optional. + */ + private ListenableFuture<Optional<StableSamplingInfo>> shouldLogDeviceStable( + long sampleInterval, LoggingStateStore loggingStateStore) { + return PropagatedFluentFuture.from(loggingStateStore.getStableSamplingInfo()) + .transform( + samplingInfo -> { + boolean invalidSamplingRateUsed = ((100 % sampleInterval) != 0); + if (invalidSamplingRateUsed) { + LogUtil.e( + "Bad sample interval (1 percent cohort will not log): %d", + sampleInterval); + } - return Optional.of( - StableSamplingInfo.newBuilder() - .setStableSamplingUsed(true) - .setStableSamplingFirstEnabledTimestampMs( - toMillis(samplingInfo.getLogSamplingSaltSetTimestamp())) - .setPartOfAlwaysLoggingGroup( - isPartOfSample( - samplingInfo.getStableLogSamplingSalt(), /*sampleInterval=*/ 100)) - .setInvalidSamplingRateUsed(invalidSamplingRateUsed) - .build()); - }, - directExecutor()); - } + if (!isPartOfSample(samplingInfo.getStableLogSamplingSalt(), + sampleInterval)) { + return Optional.absent(); + } - /** - * Returns whether this device is part of the sample with the given sampling rate and random - * number. - */ - private boolean isPartOfSample(long randomNumber, long sampleInterval) { - return randomNumber % sampleInterval == 0; - } + return Optional.of( + StableSamplingInfo.newBuilder() + .setStableSamplingUsed(true) + .setStableSamplingFirstEnabledTimestampMs( + TimestampsUtil.toMillis( + samplingInfo.getLogSamplingSaltSetTimestamp())) + .setPartOfAlwaysLoggingGroup( + isPartOfSample( + samplingInfo.getStableLogSamplingSalt(), /* sampleInterval= */ + 100)) + .setInvalidSamplingRateUsed(invalidSamplingRateUsed) + .build()); + }, + directExecutor()); + } - // Copy from com.google.protobuf.util.Timestamps - // TODO(b/243397277) Remove toMillis. - private static long toMillis(Timestamp timestamp) { - return timestamp.getSeconds() * 1000L + (long)timestamp.getNanos() / 1000000L; - } -}
\ No newline at end of file + /** + * Returns whether this device is part of the sample with the given sampling rate and random + * number. + */ + private boolean isPartOfSample(long randomNumber, long sampleInterval) { + return randomNumber % sampleInterval == 0; + } +} diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/logging/LogUtil.java b/java/com/google/android/libraries/mobiledatadownload/internal/logging/LogUtil.java index bba7ab3..bc4375e 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/logging/LogUtil.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/logging/LogUtil.java @@ -25,12 +25,12 @@ import java.util.Random; import javax.annotation.Nullable; /** Utility class for logging with the "MDD" tag. */ -@CanIgnoreReturnValue public class LogUtil { public static final String TAG = "MDD"; private static final Random random = new Random(); + @CanIgnoreReturnValue // pushed down from class to method; see <internal> public static int getLogPriority() { int level = Log.ASSERT; while (level > Log.VERBOSE) { @@ -42,6 +42,7 @@ public class LogUtil { return level; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> public static int v(String msg) { if (Log.isLoggable(TAG, Log.VERBOSE)) { return Log.v(TAG, msg); @@ -49,6 +50,7 @@ public class LogUtil { return 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> @FormatMethod public static int v(@FormatString String format, Object obj0) { if (Log.isLoggable(TAG, Log.VERBOSE)) { @@ -58,6 +60,7 @@ public class LogUtil { return 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> @FormatMethod public static int v(@FormatString String format, Object obj0, Object obj1) { if (Log.isLoggable(TAG, Log.VERBOSE)) { @@ -67,6 +70,7 @@ public class LogUtil { return 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> @FormatMethod public static int v(@FormatString String format, Object... params) { if (Log.isLoggable(TAG, Log.VERBOSE)) { @@ -76,6 +80,7 @@ public class LogUtil { return 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> public static int d(String msg) { if (Log.isLoggable(TAG, Log.DEBUG)) { return Log.d(TAG, msg); @@ -83,6 +88,7 @@ public class LogUtil { return 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> @FormatMethod public static int d(@FormatString String format, Object obj0) { if (Log.isLoggable(TAG, Log.DEBUG)) { @@ -92,6 +98,7 @@ public class LogUtil { return 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> @FormatMethod public static int d(@FormatString String format, Object obj0, Object obj1) { if (Log.isLoggable(TAG, Log.DEBUG)) { @@ -101,6 +108,7 @@ public class LogUtil { return 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> @FormatMethod public static int d(@FormatString String format, Object... params) { if (Log.isLoggable(TAG, Log.DEBUG)) { @@ -110,6 +118,7 @@ public class LogUtil { return 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> @FormatMethod public static int d(@Nullable Throwable tr, @FormatString String format, Object... params) { if (Log.isLoggable(TAG, Log.DEBUG)) { @@ -119,6 +128,7 @@ public class LogUtil { return 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> public static int i(String msg) { if (Log.isLoggable(TAG, Log.INFO)) { return Log.i(TAG, msg); @@ -126,6 +136,7 @@ public class LogUtil { return 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> @FormatMethod public static int i(@FormatString String format, Object obj0) { if (Log.isLoggable(TAG, Log.INFO)) { @@ -135,6 +146,7 @@ public class LogUtil { return 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> @FormatMethod public static int i(@FormatString String format, Object obj0, Object obj1) { if (Log.isLoggable(TAG, Log.INFO)) { @@ -144,6 +156,7 @@ public class LogUtil { return 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> @FormatMethod public static int i(@FormatString String format, Object... params) { if (Log.isLoggable(TAG, Log.INFO)) { @@ -153,6 +166,7 @@ public class LogUtil { return 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> public static int e(String msg) { if (Log.isLoggable(TAG, Log.ERROR)) { return Log.e(TAG, msg); @@ -160,6 +174,7 @@ public class LogUtil { return 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> @FormatMethod public static int e(@FormatString String format, Object obj0) { if (Log.isLoggable(TAG, Log.ERROR)) { @@ -169,6 +184,7 @@ public class LogUtil { return 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> @FormatMethod public static int e(@FormatString String format, Object obj0, Object obj1) { if (Log.isLoggable(TAG, Log.ERROR)) { @@ -178,6 +194,7 @@ public class LogUtil { return 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> @FormatMethod public static int e(@FormatString String format, Object... params) { if (Log.isLoggable(TAG, Log.ERROR)) { @@ -187,6 +204,7 @@ public class LogUtil { return 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> @SuppressLint("LogTagMismatch") public static int e(@Nullable Throwable tr, String msg) { if (Log.isLoggable(TAG, Log.ERROR)) { @@ -201,11 +219,13 @@ public class LogUtil { return 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> @FormatMethod public static int e(@Nullable Throwable tr, @FormatString String format, Object... params) { return Log.isLoggable(TAG, Log.ERROR) ? e(tr, format(format, params)) : 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> public static int w(String msg) { if (Log.isLoggable(TAG, Log.WARN)) { return Log.w(TAG, msg); @@ -213,6 +233,7 @@ public class LogUtil { return 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> @FormatMethod public static int w(@FormatString String format, Object obj0) { if (Log.isLoggable(TAG, Log.WARN)) { @@ -222,6 +243,7 @@ public class LogUtil { return 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> @FormatMethod public static int w(@FormatString String format, Object obj0, Object obj1) { if (Log.isLoggable(TAG, Log.WARN)) { @@ -231,6 +253,7 @@ public class LogUtil { return 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> @FormatMethod public static int w(@FormatString String format, Object... params) { if (Log.isLoggable(TAG, Log.WARN)) { @@ -240,6 +263,7 @@ public class LogUtil { return 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> @SuppressLint("LogTagMismatch") @FormatMethod public static int w(@Nullable Throwable tr, @FormatString String format, Object... params) { @@ -256,11 +280,13 @@ public class LogUtil { return 0; } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> @FormatMethod private static String format(@FormatString String format, Object... args) { return String.format(Locale.US, format, args); } + @CanIgnoreReturnValue // pushed down from class to method; see <internal> public static boolean shouldSampleInterval(long sampleInterval) { if (sampleInterval <= 0L) { if (sampleInterval < 0L) { diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/logging/MddEventLogger.java b/java/com/google/android/libraries/mobiledatadownload/internal/logging/MddEventLogger.java index 264a1a9..620421c 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/logging/MddEventLogger.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/logging/MddEventLogger.java @@ -23,14 +23,13 @@ import android.content.Intent; import android.content.IntentFilter; import com.google.android.libraries.mobiledatadownload.Flags; import com.google.android.libraries.mobiledatadownload.Logger; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFluentFuture; import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; import com.google.common.base.Optional; import com.google.common.util.concurrent.AsyncCallable; -import com.google.common.util.concurrent.FluentFuture; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.ListenableFuture; import com.google.mobiledatadownload.LogEnumsProto.MddClientEvent; -import com.google.mobiledatadownload.LogEnumsProto.MddClientEvent.Code; import com.google.mobiledatadownload.LogEnumsProto.MddDownloadResult; import com.google.mobiledatadownload.LogProto.AndroidClientInfo; import com.google.mobiledatadownload.LogProto.DataDownloadFileGroupStats; @@ -60,7 +59,7 @@ public final class MddEventLogger implements EventLogger { private Optional<LoggingStateStore> loggingStateStore = Optional.absent(); public MddEventLogger( - Context context, Logger logger, int moduleVersion, LogSampler logSampler, Flags flags) { + Context context, Logger logger, int moduleVersion, LogSampler logSampler, Flags flags) { this.context = context; this.logger = logger; this.moduleVersion = moduleVersion; @@ -84,135 +83,164 @@ public final class MddEventLogger implements EventLogger { } @Override - public void logEventSampled(int eventCode) {} + public void logEventSampled(MddClientEvent.Code eventCode) { + sampleAndSendLogEvent(eventCode, MddLogData.newBuilder(), flags.mddDefaultSampleInterval()); + } @Override public void logEventSampled( - int eventCode, - String fileGroupName, - int fileGroupVersionNumber, - long buildId, - String variantId) { + MddClientEvent.Code eventCode, + String fileGroupName, + int fileGroupVersionNumber, + long buildId, + String variantId) { + + DataDownloadFileGroupStats dataDownloadFileGroupStats = + DataDownloadFileGroupStats.newBuilder() + .setFileGroupName(fileGroupName) + .setFileGroupVersionNumber(fileGroupVersionNumber) + .setBuildId(buildId) + .setVariantId(variantId) + .build(); - Void dataDownloadFileGroupStats = null; + sampleAndSendLogEvent( + eventCode, + MddLogData.newBuilder().setDataDownloadFileGroupStats(dataDownloadFileGroupStats), + flags.mddDefaultSampleInterval()); } @Override - public void logEventAfterSample(int eventCode, int sampleInterval) { + public void logEventAfterSample(MddClientEvent.Code eventCode, int sampleInterval) { // TODO(b/138392640): delete this method once the pds migration is complete. If it's necessary // for other use cases, we can establish a pattern where this class is still responsible for // sampling. - Void logData = null; + MddLogData.Builder logData = MddLogData.newBuilder(); processAndSendEventWithoutStableSampling(eventCode, logData, sampleInterval); } @Override - public void logMddApiCallStats(Void fileGroupDetails, Void apiCallStats) { + public void logMddApiCallStats(DataDownloadFileGroupStats fileGroupDetails, Void apiCallStats) { // TODO(b/144684763): update this to use stable sampling. Leaving it as is for now since it is // fairly high volume. long sampleInterval = flags.apiLoggingSampleInterval(); if (!LogUtil.shouldSampleInterval(sampleInterval)) { return; } - Void logData = null; - processAndSendEventWithoutStableSampling(0, logData, sampleInterval); + MddLogData.Builder logData = + MddLogData.newBuilder().setDataDownloadFileGroupStats(fileGroupDetails); + processAndSendEventWithoutStableSampling( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, logData, sampleInterval); + } + + @Override + public void logMddLibApiResultLog(Void mddLibApiResultLog) { + MddLogData.Builder logData = MddLogData.newBuilder(); + + sampleAndSendLogEvent( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, logData, flags.apiLoggingSampleInterval()); } @Override public ListenableFuture<Void> logMddFileGroupStats( - AsyncCallable<List<EventLogger.FileGroupStatusWithDetails>> buildFileGroupStats) { + AsyncCallable<List<EventLogger.FileGroupStatusWithDetails>> buildFileGroupStats) { return lazySampleAndSendLogEvent( - Code.DATA_DOWNLOAD_FILE_GROUP_STATUS, - () -> - PropagatedFutures.transform( - buildFileGroupStats.call(), - fileGroupStatusAndDetailsList -> { - List<MddLogData> allMddLogData = new ArrayList<>(); - - for (FileGroupStatusWithDetails fileGroupStatusAndDetails : - fileGroupStatusAndDetailsList) { - allMddLogData.add( - MddLogData.newBuilder() - .setMddFileGroupStatus(fileGroupStatusAndDetails.fileGroupStatus()) - .setDataDownloadFileGroupStats( - fileGroupStatusAndDetails.fileGroupDetails()) - .build()); - } - return allMddLogData; - }, - directExecutor()), - flags.groupStatsLoggingSampleInterval()); + MddClientEvent.Code.DATA_DOWNLOAD_FILE_GROUP_STATUS, + () -> + PropagatedFutures.transform( + buildFileGroupStats.call(), + fileGroupStatusAndDetailsList -> { + List<MddLogData> allMddLogData = new ArrayList<>(); + + for (FileGroupStatusWithDetails fileGroupStatusAndDetails : + fileGroupStatusAndDetailsList) { + allMddLogData.add( + MddLogData.newBuilder() + .setMddFileGroupStatus(fileGroupStatusAndDetails.fileGroupStatus()) + .setDataDownloadFileGroupStats( + fileGroupStatusAndDetails.fileGroupDetails()) + .build()); + } + return allMddLogData; + }, + directExecutor()), + flags.groupStatsLoggingSampleInterval()); } @Override public ListenableFuture<Void> logMddStorageStats( - AsyncCallable<MddStorageStats> buildStorageStats) { + AsyncCallable<MddStorageStats> buildStorageStats) { return lazySampleAndSendLogEvent( - Code.DATA_DOWNLOAD_STORAGE_STATS, - () -> - PropagatedFutures.transform( - buildStorageStats.call(), - storageStats -> - Arrays.asList(MddLogData.newBuilder().setMddStorageStats(storageStats).build()), - directExecutor()), - flags.storageStatsLoggingSampleInterval()); + MddClientEvent.Code.DATA_DOWNLOAD_STORAGE_STATS, + () -> + PropagatedFutures.transform( + buildStorageStats.call(), + storageStats -> + Arrays.asList(MddLogData.newBuilder().setMddStorageStats(storageStats).build()), + directExecutor()), + flags.storageStatsLoggingSampleInterval()); } @Override public ListenableFuture<Void> logMddNetworkStats(AsyncCallable<Void> buildNetworkStats) { return lazySampleAndSendLogEvent( - Code.EVENT_CODE_UNSPECIFIED, - () -> - PropagatedFutures.transform( - buildNetworkStats.call(), networkStats -> Arrays.asList(), directExecutor()), - flags.networkStatsLoggingSampleInterval()); + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, + () -> + PropagatedFutures.transform( + buildNetworkStats.call(), networkStats -> Arrays.asList(), directExecutor()), + flags.networkStatsLoggingSampleInterval()); } @Override public void logMddDataDownloadFileExpirationEvent(int eventCode, int count) { - MddLogData.Builder logData = null; - sampleAndSendLogEvent(Code.EVENT_CODE_UNSPECIFIED, logData, flags.mddDefaultSampleInterval()); + MddLogData.Builder logData = MddLogData.newBuilder(); + sampleAndSendLogEvent( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, logData, flags.mddDefaultSampleInterval()); } @Override public void logMddNetworkSavings( - Void fileGroupDetails, - int code, - long fullFileSize, - long downloadedFileSize, - String fileId, - int deltaIndex) { - MddLogData.Builder logData = null; - - sampleAndSendLogEvent(Code.EVENT_CODE_UNSPECIFIED, logData, flags.mddDefaultSampleInterval()); + DataDownloadFileGroupStats fileGroupDetails, + int code, + long fullFileSize, + long downloadedFileSize, + String fileId, + int deltaIndex) { + MddLogData.Builder logData = MddLogData.newBuilder(); + + sampleAndSendLogEvent( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, logData, flags.mddDefaultSampleInterval()); } @Override - public void logMddQueryStats(Void fileGroupDetails) { - MddLogData.Builder logData = null; + public void logMddQueryStats(DataDownloadFileGroupStats fileGroupDetails) { + MddLogData.Builder logData = MddLogData.newBuilder(); - sampleAndSendLogEvent(Code.EVENT_CODE_UNSPECIFIED, logData, flags.mddDefaultSampleInterval()); + sampleAndSendLogEvent( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, logData, flags.mddDefaultSampleInterval()); } @Override - public void logMddDownloadLatency(Void fileGroupDetails, Void downloadLatency) { - MddLogData.Builder logData = null; + public void logMddDownloadLatency( + DataDownloadFileGroupStats fileGroupDetails, Void downloadLatency) { + MddLogData.Builder logData = + MddLogData.newBuilder().setDataDownloadFileGroupStats(fileGroupDetails); - sampleAndSendLogEvent(Code.EVENT_CODE_UNSPECIFIED, logData, flags.mddDefaultSampleInterval()); + sampleAndSendLogEvent( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, logData, flags.mddDefaultSampleInterval()); } @Override public void logMddDownloadResult( - MddDownloadResult.Code code, DataDownloadFileGroupStats fileGroupDetails) { + MddDownloadResult.Code code, DataDownloadFileGroupStats fileGroupDetails) { MddLogData.Builder logData = - MddLogData.newBuilder() - .setMddDownloadResultLog( - MddDownloadResultLog.newBuilder() - .setResult(code) - .setDataDownloadFileGroupStats(fileGroupDetails)); + MddLogData.newBuilder() + .setMddDownloadResultLog( + MddDownloadResultLog.newBuilder() + .setResult(code) + .setDataDownloadFileGroupStats(fileGroupDetails)); sampleAndSendLogEvent( - Code.DATA_DOWNLOAD_RESULT_LOG, logData, flags.mddDefaultSampleInterval()); + MddClientEvent.Code.DATA_DOWNLOAD_RESULT_LOG, logData, flags.mddDefaultSampleInterval()); } @Override @@ -222,15 +250,27 @@ public final class MddEventLogger implements EventLogger { if (!LogUtil.shouldSampleInterval(sampleInterval)) { return; } - Void logData = null; - processAndSendEventWithoutStableSampling(0, logData, sampleInterval); + MddLogData.Builder logData = MddLogData.newBuilder(); + processAndSendEventWithoutStableSampling( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, logData, sampleInterval); } @Override - public void logMddUsageEvent(Void fileGroupDetails, Void usageEventLog) { - MddLogData.Builder logData = null; + public void logMddUsageEvent(DataDownloadFileGroupStats fileGroupDetails, Void usageEventLog) { + MddLogData.Builder logData = + MddLogData.newBuilder().setDataDownloadFileGroupStats(fileGroupDetails); + + sampleAndSendLogEvent( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, logData, flags.mddDefaultSampleInterval()); + } - sampleAndSendLogEvent(Code.EVENT_CODE_UNSPECIFIED, logData, flags.mddDefaultSampleInterval()); + @Override + public void logNewConfigReceived( + DataDownloadFileGroupStats fileGroupDetails, Void newConfigReceivedInfo) { + MddLogData.Builder logData = + MddLogData.newBuilder().setDataDownloadFileGroupStats(fileGroupDetails); + sampleAndSendLogEvent( + MddClientEvent.Code.EVENT_CODE_UNSPECIFIED, logData, flags.mddDefaultSampleInterval()); } /** @@ -239,82 +279,86 @@ public final class MddEventLogger implements EventLogger { * constructs the log event lazy. This is useful if constructing the log event is expensive. */ private ListenableFuture<Void> lazySampleAndSendLogEvent( - Code eventCode, AsyncCallable<List<MddLogData>> buildStats, int sampleInterval) { + MddClientEvent.Code eventCode, + AsyncCallable<List<MddLogData>> buildStats, + int sampleInterval) { return PropagatedFutures.transformAsync( - logSampler.shouldLog(sampleInterval, loggingStateStore), - samplingInfoOptional -> { - if (!samplingInfoOptional.isPresent()) { - return immediateVoidFuture(); - } - - return FluentFuture.from(buildStats.call()) - .transform( - icingLogDataList -> { - if (icingLogDataList != null) { - for (MddLogData icingLogData : icingLogDataList) { - processAndSendEvent( - eventCode, - icingLogData.toBuilder(), - sampleInterval, - samplingInfoOptional.get()); - } - } - return null; - }, - directExecutor()); - }, - directExecutor()); + logSampler.shouldLog(sampleInterval, loggingStateStore), + samplingInfoOptional -> { + if (!samplingInfoOptional.isPresent()) { + return immediateVoidFuture(); + } + + return PropagatedFluentFuture.from(buildStats.call()) + .transform( + icingLogDataList -> { + if (icingLogDataList != null) { + for (MddLogData icingLogData : icingLogDataList) { + processAndSendEvent( + eventCode, + icingLogData.toBuilder(), + sampleInterval, + samplingInfoOptional.get()); + } + } + return null; + }, + directExecutor()); + }, + directExecutor()); } private void sampleAndSendLogEvent( - MddClientEvent.Code eventCode, MddLogData.Builder logData, long sampleInterval) { + MddClientEvent.Code eventCode, MddLogData.Builder logData, long sampleInterval) { + // NOTE: When using a single-threaded executor, logging may be delayed since other + // work will come before the log sampler check. PropagatedFutures.addCallback( - logSampler.shouldLog(sampleInterval, loggingStateStore), - new FutureCallback<Optional<StableSamplingInfo>>() { - @Override - public void onSuccess(Optional<StableSamplingInfo> stableSamplingInfo) { - if (stableSamplingInfo.isPresent()) { - processAndSendEvent(eventCode, logData, sampleInterval, stableSamplingInfo.get()); - } - } - - @Override - public void onFailure(Throwable t) { - LogUtil.e(t, "%s: failure when sampling log!", TAG); - } - }, - directExecutor()); + logSampler.shouldLog(sampleInterval, loggingStateStore), + new FutureCallback<Optional<StableSamplingInfo>>() { + @Override + public void onSuccess(Optional<StableSamplingInfo> stableSamplingInfo) { + if (stableSamplingInfo.isPresent()) { + processAndSendEvent(eventCode, logData, sampleInterval, stableSamplingInfo.get()); + } + } + + @Override + public void onFailure(Throwable t) { + LogUtil.e(t, "%s: failure when sampling log!", TAG); + } + }, + directExecutor()); } /** Adds all transforms common to all logs and sends the event to Logger. */ private void processAndSendEventWithoutStableSampling( - int eventCode, Void logData, long sampleInterval) { + MddClientEvent.Code eventCode, MddLogData.Builder logData, long sampleInterval) { processAndSendEvent( - Code.EVENT_CODE_UNSPECIFIED, - MddLogData.newBuilder(), - sampleInterval, - StableSamplingInfo.newBuilder().setStableSamplingUsed(false).build()); + eventCode, + logData, + sampleInterval, + StableSamplingInfo.newBuilder().setStableSamplingUsed(false).build()); } /** Adds all transforms common to all logs and sends the event to Logger. */ private void processAndSendEvent( - Code eventCode, - MddLogData.Builder logData, - long sampleInterval, - StableSamplingInfo stableSamplingInfo) { - if (eventCode.equals(Code.EVENT_CODE_UNSPECIFIED)) { + MddClientEvent.Code eventCode, + MddLogData.Builder logData, + long sampleInterval, + StableSamplingInfo stableSamplingInfo) { + if (eventCode.equals(MddClientEvent.Code.EVENT_CODE_UNSPECIFIED)) { LogUtil.e("%s: unspecified code used, skipping event log", TAG); // return early for unspecified codes. return; } logData - .setSamplingInterval(sampleInterval) - .setDeviceInfo(MddDeviceInfo.newBuilder().setDeviceStorageLow(isDeviceStorageLow(context))) - .setAndroidClientInfo( - AndroidClientInfo.newBuilder() - .setHostPackageName(hostPackageName) - .setModuleVersion(moduleVersion)) - .setStableSamplingInfo(stableSamplingInfo); + .setSamplingInterval(sampleInterval) + .setDeviceInfo(MddDeviceInfo.newBuilder().setDeviceStorageLow(isDeviceStorageLow(context))) + .setAndroidClientInfo( + AndroidClientInfo.newBuilder() + .setHostPackageName(hostPackageName) + .setModuleVersion(moduleVersion)) + .setStableSamplingInfo(stableSamplingInfo); logger.log(logData.build(), eventCode.getNumber()); } @@ -322,6 +366,6 @@ public final class MddEventLogger implements EventLogger { private static boolean isDeviceStorageLow(Context context) { // Check if the system says storage is low, by reading the sticky intent. return context.registerReceiver(null, new IntentFilter(Intent.ACTION_DEVICE_STORAGE_LOW)) - != null; + != null; } -}
\ No newline at end of file +} diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/logging/NoOpEventLogger.java b/java/com/google/android/libraries/mobiledatadownload/internal/logging/NoOpEventLogger.java index 46dda2b..2f043f5 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/logging/NoOpEventLogger.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/logging/NoOpEventLogger.java @@ -19,6 +19,7 @@ import static com.google.common.util.concurrent.Futures.immediateVoidFuture; import com.google.common.util.concurrent.AsyncCallable; import com.google.common.util.concurrent.ListenableFuture; +import com.google.mobiledatadownload.LogEnumsProto.MddClientEvent; import com.google.mobiledatadownload.LogEnumsProto.MddDownloadResult; import com.google.mobiledatadownload.LogProto.DataDownloadFileGroupStats; import com.google.mobiledatadownload.LogProto.MddStorageStats; @@ -28,18 +29,18 @@ import java.util.List; public final class NoOpEventLogger implements EventLogger { @Override - public void logEventSampled(int eventCode) {} + public void logEventSampled(MddClientEvent.Code eventCode) {} @Override public void logEventSampled( - int eventCode, + MddClientEvent.Code eventCode, String fileGroupName, int fileGroupVersionNumber, long buildId, String variantId) {} @Override - public void logEventAfterSample(int eventCode, int sampleInterval) {} + public void logEventAfterSample(MddClientEvent.Code eventCode, int sampleInterval) {} @Override public ListenableFuture<Void> logMddFileGroupStats( @@ -48,11 +49,14 @@ public final class NoOpEventLogger implements EventLogger { } @Override - public void logMddApiCallStats(Void fileGroupDetails, Void apiCallStats) {} + public void logMddApiCallStats(DataDownloadFileGroupStats fileGroupDetails, Void apiCallStats) {} + + @Override + public void logMddLibApiResultLog(Void mddLibApiResultLog) {} @Override public ListenableFuture<Void> logMddStorageStats( - AsyncCallable<MddStorageStats> buildStorageStats) { + AsyncCallable<MddStorageStats> buildMddStorageStats) { return immediateVoidFuture(); } @@ -66,7 +70,7 @@ public final class NoOpEventLogger implements EventLogger { @Override public void logMddNetworkSavings( - Void fileGroupDetails, + DataDownloadFileGroupStats fileGroupDetails, int code, long fullFileSize, long downloadedFileSize, @@ -75,17 +79,22 @@ public final class NoOpEventLogger implements EventLogger { @Override public void logMddDownloadResult( - MddDownloadResult.Code code, DataDownloadFileGroupStats fileGroupDetails) {} + MddDownloadResult.Code code, DataDownloadFileGroupStats fileGroupDetails) {} @Override - public void logMddQueryStats(Void fileGroupDetails) {} + public void logMddQueryStats(DataDownloadFileGroupStats fileGroupDetails) {} @Override public void logMddAndroidSharingLog(Void event) {} @Override - public void logMddDownloadLatency(Void fileGroupStats, Void downloadLatency) {} + public void logMddDownloadLatency( + DataDownloadFileGroupStats fileGroupStats, Void downloadLatency) {} + + @Override + public void logMddUsageEvent(DataDownloadFileGroupStats fileGroupDetails, Void usageEventLog) {} @Override - public void logMddUsageEvent(Void fileGroupDetails, Void usageEventLog) {} + public void logNewConfigReceived( + DataDownloadFileGroupStats fileGroupDetails, Void newConfigReceivedInfo) {} } diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/logging/SharedPreferencesLoggingState.java b/java/com/google/android/libraries/mobiledatadownload/internal/logging/SharedPreferencesLoggingState.java index e4debc5..7fd90ef 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/logging/SharedPreferencesLoggingState.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/logging/SharedPreferencesLoggingState.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 The Android Open Source Project + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,15 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package com.google.android.libraries.mobiledatadownload.internal.logging; import static com.google.android.libraries.mobiledatadownload.internal.MddConstants.SPLIT_CHAR; + import static java.util.concurrent.TimeUnit.MILLISECONDS; import android.content.Context; import android.content.SharedPreferences; + import androidx.annotation.VisibleForTesting; + import com.google.android.libraries.mobiledatadownload.TimeSource; import com.google.android.libraries.mobiledatadownload.internal.util.FileGroupsMetadataUtil; import com.google.android.libraries.mobiledatadownload.internal.util.FileGroupsMetadataUtil.GroupKeyDeserializationException; @@ -37,6 +39,7 @@ import com.google.mobiledatadownload.internal.MetadataProto.FileGroupLoggingStat import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; import com.google.mobiledatadownload.internal.MetadataProto.SamplingInfo; import com.google.protobuf.Timestamp; + import java.io.IOException; import java.util.ArrayList; import java.util.Calendar; @@ -55,8 +58,10 @@ public final class SharedPreferencesLoggingState implements LoggingStateStore { private static final String LAST_MAINTENANCE_RUN_SECS_KEY = "last_maintenance_secs"; - @VisibleForTesting static final String SALT_KEY = "stable_log_sampling_salt"; - private static final String SALT_TIMESTAMP_MILLIS_KEY = "log_sampling_salt_set_timestamp_millis"; + @VisibleForTesting + static final String SALT_KEY = "stable_log_sampling_salt"; + private static final String SALT_TIMESTAMP_MILLIS_KEY = + "log_sampling_salt_set_timestamp_millis"; private final Supplier<SharedPreferences> sharedPrefs; private final Executor backgroundExecutor; @@ -71,15 +76,17 @@ public final class SharedPreferencesLoggingState implements LoggingStateStore { * Constructs a new instance. * * @param sharedPrefs may be called multiple times, so memoization is recommended. The returned - * instance must be exclusive to {@link SharedPreferencesLoggingState} since {@link #clear} - * may clear the data at any time. + * instance must be exclusive to {@link SharedPreferencesLoggingState} since + * {@link #clear} + * may clear the data at any time. */ public static SharedPreferencesLoggingState create( Supplier<SharedPreferences> sharedPrefs, TimeSource timeSource, Executor backgroundExecutor, Random random) { - return new SharedPreferencesLoggingState(sharedPrefs, timeSource, backgroundExecutor, random); + return new SharedPreferencesLoggingState(sharedPrefs, timeSource, backgroundExecutor, + random); } /** Constructs a new instance. */ @@ -95,7 +102,8 @@ public final class SharedPreferencesLoggingState implements LoggingStateStore { () -> SharedPreferencesUtil.getSharedPreferences( context, SHARED_PREFS_NAME, instanceIdOptional)); - return new SharedPreferencesLoggingState(sharedPrefs, timeSource, backgroundExecutor, random); + return new SharedPreferencesLoggingState(sharedPrefs, timeSource, backgroundExecutor, + random); } private SharedPreferencesLoggingState( @@ -180,15 +188,18 @@ public final class SharedPreferencesLoggingState implements LoggingStateStore { boolean hasEverDoneMaintenance = sharedPrefs.get().contains(LAST_MAINTENANCE_RUN_SECS_KEY); if (hasEverDoneMaintenance) { - long persistedTimestamp = sharedPrefs.get().getLong(LAST_MAINTENANCE_RUN_SECS_KEY, 0); + long persistedTimestamp = sharedPrefs.get().getLong( + LAST_MAINTENANCE_RUN_SECS_KEY, 0); long currentStartOfDay = truncateTimestampToStartOfDay(currentTimestamp); long previousStartOfDay = truncateTimestampToStartOfDay(persistedTimestamp); - // Note: ignore MillisTo_Days java optional suggestion because Duration is api + // Note: ignore MillisTo_Days java optional suggestion because Duration + // is api // 26+. daysSinceLastMaintenance = Optional.of( Ints.saturatedCast( - MILLISECONDS.toDays(currentStartOfDay - previousStartOfDay))); + MILLISECONDS.toDays( + currentStartOfDay - previousStartOfDay))); } else { daysSinceLastMaintenance = Optional.absent(); } @@ -209,10 +220,12 @@ public final class SharedPreferencesLoggingState implements LoggingStateStore { Entry entry = Entry.fromLoggingState(dataUsageIncrements); long currentCellarUsage = - sharedPrefs.get().getLong(entry.getSharedPrefsKey(Key.CELLULAR_USAGE), 0); + sharedPrefs.get().getLong(entry.getSharedPrefsKey(Key.CELLULAR_USAGE), + 0); long currentWifiUsage = sharedPrefs.get().getLong(entry.getSharedPrefsKey(Key.WIFI_USAGE), 0); - long updatedCellarUsage = currentCellarUsage + dataUsageIncrements.getCellularUsage(); + long updatedCellarUsage = + currentCellarUsage + dataUsageIncrements.getCellularUsage(); long updatedWifiUsage = currentWifiUsage + dataUsageIncrements.getWifiUsage(); SharedPreferences.Editor editor = sharedPrefs.get().edit(); @@ -250,9 +263,12 @@ public final class SharedPreferencesLoggingState implements LoggingStateStore { .setBuildId(entry.buildId) .setFileGroupVersionNumber(entry.fileGroupVersionNumber) .setCellularUsage( - sharedPrefs.get().getLong(entry.getSharedPrefsKey(Key.CELLULAR_USAGE), 0)) + sharedPrefs.get().getLong( + entry.getSharedPrefsKey(Key.CELLULAR_USAGE), + 0)) .setWifiUsage( - sharedPrefs.get().getLong(entry.getSharedPrefsKey(Key.WIFI_USAGE), 0)) + sharedPrefs.get().getLong( + entry.getSharedPrefsKey(Key.WIFI_USAGE), 0)) .build(); allLoggingStates.add(loggingState); @@ -287,7 +303,8 @@ public final class SharedPreferencesLoggingState implements LoggingStateStore { boolean hasCreatedSalt = sharedPrefs.get().contains(SALT_KEY); if (hasCreatedSalt) { salt = sharedPrefs.get().getLong(SALT_KEY, 0); - persistedTimestampMillis = sharedPrefs.get().getLong(SALT_TIMESTAMP_MILLIS_KEY, 0); + persistedTimestampMillis = sharedPrefs.get().getLong( + SALT_TIMESTAMP_MILLIS_KEY, 0); } else { salt = random.nextLong(); persistedTimestampMillis = timeSource.currentTimeMillis(); @@ -298,7 +315,7 @@ public final class SharedPreferencesLoggingState implements LoggingStateStore { commitOrThrow(editor); } - Timestamp timestamp = fromMillis(persistedTimestampMillis); + Timestamp timestamp = TimestampsUtil.fromMillis(persistedTimestampMillis); return SamplingInfo.newBuilder() .setStableLogSamplingSalt(salt) .setLogSamplingSaltSetTimestamp(timestamp) @@ -330,38 +347,4 @@ public final class SharedPreferencesLoggingState implements LoggingStateStore { } return null; } - - // TODO(b/243397277) Remove following methods. - public static Timestamp fromMillis(long milliseconds) { - return normalizedTimestamp(milliseconds / 1000L, (int)(milliseconds % 1000L * 1000000L)); - } - - private static Timestamp normalizedTimestamp(long seconds, int nanos) { - if ((long)nanos <= -1000000000L || (long)nanos >= 1000000000L) { - seconds += (long)nanos / 1000000000L; - nanos = (int)((long)nanos % 1000000000L); - } - - if (nanos < 0) { - nanos = (int)((long)nanos + 1000000000L); - --seconds; - } - - checkValid(seconds, nanos); - return Timestamp.newBuilder().setSeconds(seconds).setNanos(nanos).build(); - } - - private static void checkValid(long seconds, int nanos) { - if (!isValid(seconds, (long)nanos)) { - throw new IllegalArgumentException(String.format("Timestamp is not valid. See proto definition for valid values. Seconds (%s) must be in range [-62,135,596,800, +253,402,300,799].Nanos (%s) must be in range [0, +999,999,999].", seconds, nanos)); - } - } - - private static boolean isValid(long seconds, long nanos) { - if (seconds >= -62135596800L && seconds <= 253402300799L) { - return nanos >= 0L && nanos < 1000000000L; - } else { - return false; - } - } -}
\ No newline at end of file +} diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/logging/StorageLogger.java b/java/com/google/android/libraries/mobiledatadownload/internal/logging/StorageLogger.java index adfe96e..d707f42 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/logging/StorageLogger.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/logging/StorageLogger.java @@ -16,10 +16,10 @@ package com.google.android.libraries.mobiledatadownload.internal.logging; import static com.google.android.libraries.mobiledatadownload.internal.MddConstants.SPLIT_CHAR; +import static com.google.common.util.concurrent.Futures.immediateFuture; import android.content.Context; import android.net.Uri; -import android.util.Pair; import com.google.android.libraries.mobiledatadownload.SilentFeedback; import com.google.android.libraries.mobiledatadownload.annotations.InstanceId; import com.google.android.libraries.mobiledatadownload.file.SynchronousFileStorage; @@ -31,18 +31,14 @@ import com.google.android.libraries.mobiledatadownload.internal.SharedFileManage import com.google.android.libraries.mobiledatadownload.internal.SharedFileMissingException; import com.google.android.libraries.mobiledatadownload.internal.SharedFilesMetadata; import com.google.android.libraries.mobiledatadownload.internal.annotations.SequentialControlExecutor; -import com.google.android.libraries.mobiledatadownload.file.openers.RecursiveSizeOpener; +import com.google.android.libraries.mobiledatadownload.internal.collect.GroupKeyAndGroup; import com.google.android.libraries.mobiledatadownload.internal.util.DirectoryUtil; import com.google.android.libraries.mobiledatadownload.internal.util.FileGroupUtil; import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFluentFuture; import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; -import com.google.auto.value.AutoValue; import com.google.common.base.Optional; import com.google.common.base.Preconditions; import com.google.common.base.Splitter; -import com.google.common.base.Strings; -import com.google.common.util.concurrent.FluentFuture; -import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.mobiledatadownload.LogProto.DataDownloadFileGroupStats; import com.google.mobiledatadownload.LogProto.MddStorageStats; @@ -121,7 +117,7 @@ public class StorageLogger { private static GroupKey createGroupKey(DataFileGroupInternal fileGroup) { GroupKey.Builder groupKey = GroupKey.newBuilder().setGroupName(fileGroup.getGroupName()); - if (Strings.isNullOrEmpty(fileGroup.getOwnerPackage())) { + if (fileGroup.getOwnerPackage().isEmpty()) { groupKey.setOwnerPackage(MddConstants.GMS_PACKAGE); } else { groupKey.setOwnerPackage(fileGroup.getOwnerPackage()); @@ -136,31 +132,29 @@ public class StorageLogger { private ListenableFuture<MddStorageStats> buildStorageStatsLogData(int daysSinceLastLog) { return PropagatedFluentFuture.from(fileGroupsMetadata.getAllFreshGroups()) - .transformAsync( - allGroups -> - PropagatedFutures.transformAsync( - fileGroupsMetadata.getAllStaleGroups(), - staleGroups -> - buildStorageStatsInternal(allGroups, staleGroups, daysSinceLastLog), - sequentialControlExecutor), - sequentialControlExecutor); + .transformAsync( + allGroups -> + PropagatedFutures.transformAsync( + fileGroupsMetadata.getAllStaleGroups(), + staleGroups -> + buildStorageStatsInternal(allGroups, staleGroups, daysSinceLastLog), + sequentialControlExecutor), + sequentialControlExecutor); } private ListenableFuture<MddStorageStats> buildStorageStatsInternal( - List<Pair<GroupKey, DataFileGroupInternal>> allKeysAndGroupPairs, + List<GroupKeyAndGroup> allKeysAndGroupPairs, List<DataFileGroupInternal> staleGroups, int daysSinceLastLog) { - List<GroupKeyAndDataFileGroupInternal> allKeysAndGroups = new ArrayList<>(); - for (Pair<GroupKey, DataFileGroupInternal> groupKeyAndGroup : allKeysAndGroupPairs) { - allKeysAndGroups.add( - GroupKeyAndDataFileGroupInternal.create(groupKeyAndGroup.first, groupKeyAndGroup.second)); + List<GroupKeyAndGroup> allKeysAndGroups = new ArrayList<>(); + for (GroupKeyAndGroup groupKeyAndGroup : allKeysAndGroupPairs) { + allKeysAndGroups.add(groupKeyAndGroup); } // Adding staleGroups to allGroups. for (DataFileGroupInternal fileGroup : staleGroups) { - allKeysAndGroups.add( - GroupKeyAndDataFileGroupInternal.create(createGroupKey(fileGroup), fileGroup)); + allKeysAndGroups.add(GroupKeyAndGroup.create(createGroupKey(fileGroup), fileGroup)); } Map<String, GroupStorage> groupKeyToGroupStorage = new HashMap<>(); @@ -175,7 +169,7 @@ public class StorageLogger { AtomicLong totalMddBytesUsed = new AtomicLong(0L); List<ListenableFuture<Void>> futures = new ArrayList<>(); - for (GroupKeyAndDataFileGroupInternal groupKeyAndGroup : allKeysAndGroups) { + for (GroupKeyAndGroup groupKeyAndGroup : allKeysAndGroups) { Set<NewFileKey> fileKeys = safeGetFileKeys( @@ -194,20 +188,20 @@ public class StorageLogger { getGroupWithOwnerPackageKey(groupKeyAndGroup.groupKey())); downloadedGroupKeyToDataFileGroup.put( getGroupWithOwnerPackageKey(groupKeyAndGroup.groupKey()), - groupKeyAndGroup.dataFileGroupInternal()); + groupKeyAndGroup.dataFileGroup()); } // Variables captured by lambdas must be effectively final. Set<NewFileKey> downloadedFileKeys = downloadedFileKeysInit; - int totalFileCount = groupKeyAndGroup.dataFileGroupInternal().getFileCount(); - for (DataFile dataFile : groupKeyAndGroup.dataFileGroupInternal().getFileList()) { + int totalFileCount = groupKeyAndGroup.dataFileGroup().getFileCount(); + for (DataFile dataFile : groupKeyAndGroup.dataFileGroup().getFileList()) { boolean isInlineFile = FileGroupUtil.isInlineFile(dataFile); NewFileKey fileKey = SharedFilesMetadata.createKeyFromDataFile( - dataFile, groupKeyAndGroup.dataFileGroupInternal().getAllowedReadersEnum()); + dataFile, groupKeyAndGroup.dataFileGroup().getAllowedReadersEnum()); futures.add( - Futures.transform( + PropagatedFutures.transform( computeFileSize(fileKey), fileSize -> { if (!allFileKeys.contains(fileKey)) { @@ -247,32 +241,32 @@ public class StorageLogger { groupStorage.totalFileCount = totalFileCount; } - return Futures.whenAllComplete(futures) + return PropagatedFutures.whenAllComplete(futures) .call( () -> { MddStorageStats.Builder storageStatsBuilder = MddStorageStats.newBuilder(); for (String groupName : groupKeyToGroupStorage.keySet()) { GroupStorage groupStorage = groupKeyToGroupStorage.get(groupName); List<String> groupNameAndOwnerPackage = - Splitter.on(SPLIT_CHAR).splitToList(groupName); + Splitter.on(SPLIT_CHAR).splitToList(groupName); DataDownloadFileGroupStats.Builder fileGroupDetailsBuilder = - DataDownloadFileGroupStats.newBuilder() - .setFileGroupName(groupNameAndOwnerPackage.get(0)) - .setOwnerPackage(groupNameAndOwnerPackage.get(1)) - .setFileCount(groupStorage.totalFileCount) - .setInlineFileCount(groupStorage.totalInlineFileCount); + DataDownloadFileGroupStats.newBuilder() + .setFileGroupName(groupNameAndOwnerPackage.get(0)) + .setOwnerPackage(groupNameAndOwnerPackage.get(1)) + .setFileCount(groupStorage.totalFileCount) + .setInlineFileCount(groupStorage.totalInlineFileCount); DataFileGroupInternal dataFileGroup = - downloadedGroupKeyToDataFileGroup.get(groupName); + downloadedGroupKeyToDataFileGroup.get(groupName); if (dataFileGroup == null) { fileGroupDetailsBuilder.setFileGroupVersionNumber(-1); } else { fileGroupDetailsBuilder - .setFileGroupVersionNumber(dataFileGroup.getFileGroupVersionNumber()) - .setBuildId(dataFileGroup.getBuildId()) - .setVariantId(dataFileGroup.getVariantId()); + .setFileGroupVersionNumber(dataFileGroup.getFileGroupVersionNumber()) + .setBuildId(dataFileGroup.getBuildId()) + .setVariantId(dataFileGroup.getVariantId()); } storageStatsBuilder.addDataDownloadFileGroupStats(fileGroupDetailsBuilder.build()); @@ -280,9 +274,9 @@ public class StorageLogger { storageStatsBuilder.addTotalBytesUsed(groupStorage.totalBytesUsed); storageStatsBuilder.addTotalInlineBytesUsed(groupStorage.totalInlineBytesUsed); storageStatsBuilder.addDownloadedGroupBytesUsed( - groupStorage.downloadedGroupBytesUsed); + groupStorage.downloadedGroupBytesUsed); storageStatsBuilder.addDownloadedGroupInlineBytesUsed( - groupStorage.downloadedGroupInlineBytesUsed); + groupStorage.downloadedGroupInlineBytesUsed); } storageStatsBuilder.setTotalMddBytesUsed(totalMddBytesUsed.get()); @@ -296,14 +290,14 @@ public class StorageLogger { } catch (IOException e) { mddDirectoryBytesUsed = 0; LogUtil.e( - e, "%s: Failed to call Mobstore to compute MDD Directory bytes used!", TAG); + e, "%s: Failed to call Mobstore to compute MDD Directory bytes used!", TAG); silentFeedback.send( - e, "Failed to call Mobstore to compute MDD Directory bytes used!"); + e, "Failed to call Mobstore to compute MDD Directory bytes used!"); } storageStatsBuilder - .setTotalMddDirectoryBytesUsed(mddDirectoryBytesUsed) - .setDaysSinceLastLog(daysSinceLastLog); + .setTotalMddDirectoryBytesUsed(mddDirectoryBytesUsed) + .setDaysSinceLastLog(daysSinceLastLog); return storageStatsBuilder.build(); }, @@ -338,11 +332,9 @@ public class StorageLogger { } private ListenableFuture<Long> computeFileSize(NewFileKey newFileKey) { - return FluentFuture.from(sharedFileManager.getOnDeviceUri(newFileKey)) + return PropagatedFluentFuture.from(sharedFileManager.getOnDeviceUri(newFileKey)) .catchingAsync( - SharedFileMissingException.class, - e -> Futures.immediateFuture(null), - sequentialControlExecutor) + SharedFileMissingException.class, e -> immediateFuture(null), sequentialControlExecutor) .transform( fileUri -> { if (fileUri != null) { @@ -356,17 +348,4 @@ public class StorageLogger { }, sequentialControlExecutor); } - - @AutoValue - abstract static class GroupKeyAndDataFileGroupInternal { - static GroupKeyAndDataFileGroupInternal create( - GroupKey groupKey, DataFileGroupInternal dataFileGroupInternal) { - return new AutoValue_StorageLogger_GroupKeyAndDataFileGroupInternal( - groupKey, dataFileGroupInternal); - } - - abstract GroupKey groupKey(); - - abstract DataFileGroupInternal dataFileGroupInternal(); - } } diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/logging/TimestampsUtil.java b/java/com/google/android/libraries/mobiledatadownload/internal/logging/TimestampsUtil.java new file mode 100644 index 0000000..e0f2205 --- /dev/null +++ b/java/com/google/android/libraries/mobiledatadownload/internal/logging/TimestampsUtil.java @@ -0,0 +1,111 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.libraries.mobiledatadownload.internal.logging; + +import static com.google.common.math.LongMath.checkedAdd; +import static com.google.common.math.LongMath.checkedMultiply; +import static com.google.common.math.LongMath.checkedSubtract; + +import com.google.protobuf.Timestamp; + +/** + * Utilities to help create/manipulate {@code protobuf/timestamp.proto}. + */ +public class TimestampsUtil { + + // Timestamp for "0001-01-01T00:00:00Z" + static final long TIMESTAMP_SECONDS_MIN = -62135596800L; + + // Timestamp for "9999-12-31T23:59:59Z" + static final long TIMESTAMP_SECONDS_MAX = 253402300799L; + + static final int NANOS_PER_SECOND = 1000000000; + static final int NANOS_PER_MILLISECOND = 1000000; + static final int NANOS_PER_MICROSECOND = 1000; + static final int MILLIS_PER_SECOND = 1000; + static final int MICROS_PER_SECOND = 1000000; + + @SuppressWarnings("GoodTime") // this is a legacy conversion API + public static long toMillis(Timestamp timestamp) { + checkValid(timestamp); + return checkedAdd( + checkedMultiply(timestamp.getSeconds(), MILLIS_PER_SECOND), + timestamp.getNanos() / NANOS_PER_MILLISECOND); + } + + + /** Create a Timestamp from the number of milliseconds elapsed from the epoch. */ + @SuppressWarnings("GoodTime") // this is a legacy conversion API + public static Timestamp fromMillis(long milliseconds) { + return normalizedTimestamp( + milliseconds / MILLIS_PER_SECOND, + (int) (milliseconds % MILLIS_PER_SECOND * NANOS_PER_MILLISECOND)); + } + + public static Timestamp checkValid(Timestamp timestamp) { + long seconds = timestamp.getSeconds(); + int nanos = timestamp.getNanos(); + if (!isValid(seconds, nanos)) { + throw new IllegalArgumentException( + String.format( + "Timestamp is not valid. See proto definition for valid values. " + + "Seconds (%s) must be in range [-62,135,596,800, +253,402," + + "300,799]. " + + "Nanos (%s) must be in range [0, +999,999,999].", + seconds, nanos)); + } + return timestamp; + } + + /** + * Returns true if the given number of seconds and nanos is a valid {@link Timestamp}. The + * {@code + * seconds} value must be in the range [-62,135,596,800, +253,402,300,799] (i.e., between + * 0001-01-01T00:00:00Z and 9999-12-31T23:59:59Z). The {@code nanos} value must be in the range + * [0, +999,999,999]. + * + * <p><b>Note:</b> Negative second values with fractional seconds must still have non-negative + * nanos values that count forward in time. + */ + @SuppressWarnings("GoodTime") // this is a legacy conversion API + public static boolean isValid(long seconds, int nanos) { + if (seconds < TIMESTAMP_SECONDS_MIN || seconds > TIMESTAMP_SECONDS_MAX) { + return false; + } + if (nanos < 0 || nanos >= NANOS_PER_SECOND) { + return false; + } + return true; + } + + static Timestamp normalizedTimestamp(long seconds, int nanos) { + if (nanos <= -NANOS_PER_SECOND || nanos >= NANOS_PER_SECOND) { + seconds = checkedAdd(seconds, nanos / NANOS_PER_SECOND); + nanos = (int) (nanos % NANOS_PER_SECOND); + } + if (nanos < 0) { + nanos = + (int) + (nanos + + NANOS_PER_SECOND); // no overflow since nanos is negative + // (and we're adding) + seconds = checkedSubtract(seconds, 1); + } + Timestamp timestamp = Timestamp.newBuilder().setSeconds(seconds).setNanos(nanos).build(); + return checkValid(timestamp); + } +} diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/logging/testing/BUILD b/java/com/google/android/libraries/mobiledatadownload/internal/logging/testing/BUILD index 9bf9510..1a20ff9 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/logging/testing/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/internal/logging/testing/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], @@ -26,6 +27,8 @@ android_library( srcs = ["FakeEventLogger.java"], deps = [ "//java/com/google/android/libraries/mobiledatadownload/internal/logging:EventLogger", + "//proto:log_enums_java_proto_lite", + "//proto:logs_java_proto_lite", "@com_google_guava_guava", ], ) diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/logging/testing/FakeEventLogger.java b/java/com/google/android/libraries/mobiledatadownload/internal/logging/testing/FakeEventLogger.java index 76c1bbe..ea5134c 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/logging/testing/FakeEventLogger.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/logging/testing/FakeEventLogger.java @@ -22,6 +22,7 @@ import com.google.android.libraries.mobiledatadownload.internal.logging.EventLog import com.google.common.collect.ArrayListMultimap; import com.google.common.util.concurrent.AsyncCallable; import com.google.common.util.concurrent.ListenableFuture; +import com.google.mobiledatadownload.LogEnumsProto.MddClientEvent; import com.google.mobiledatadownload.LogEnumsProto.MddDownloadResult; import com.google.mobiledatadownload.LogProto.DataDownloadFileGroupStats; import com.google.mobiledatadownload.LogProto.MddStorageStats; @@ -31,17 +32,22 @@ import java.util.List; /** Fake implementation of {@link EventLogger} for use in tests. */ public final class FakeEventLogger implements EventLogger { - private final ArrayList<Integer> loggedCodes = new ArrayList<>(); - private final ArrayListMultimap<Void, Void> loggedLatencies = ArrayListMultimap.create(); + private final ArrayList<MddClientEvent.Code> loggedCodes = new ArrayList<>(); + private final ArrayListMultimap<DataDownloadFileGroupStats, Void> loggedLatencies = + ArrayListMultimap.create(); + private final ArrayListMultimap<DataDownloadFileGroupStats, Void> loggedNewConfigReceived = + ArrayListMultimap.create(); + private final List<Void> loggedMddLibApiResultLog = new ArrayList<>(); + private final ArrayList<DataDownloadFileGroupStats> loggedMddQueryStats = new ArrayList<>(); @Override - public void logEventSampled(int eventCode) { + public void logEventSampled(MddClientEvent.Code eventCode) { loggedCodes.add(eventCode); } @Override public void logEventSampled( - int eventCode, + MddClientEvent.Code eventCode, String fileGroupName, int fileGroupVersionNumber, long buildId, @@ -50,7 +56,7 @@ public final class FakeEventLogger implements EventLogger { } @Override - public void logEventAfterSample(int eventCode, int sampleInterval) { + public void logEventAfterSample(MddClientEvent.Code eventCode, int sampleInterval) { loggedCodes.add(eventCode); } @@ -62,15 +68,24 @@ public final class FakeEventLogger implements EventLogger { } @Override - public void logMddApiCallStats(Void fileGroupDetails, Void apiCallStats) { + public void logMddApiCallStats(DataDownloadFileGroupStats fileGroupDetails, Void apiCallStats) { throw new UnsupportedOperationException("This method is not implemented in the fake yet."); } @Override + public void logMddLibApiResultLog(Void mddLibApiResultLog) { + loggedMddLibApiResultLog.add(mddLibApiResultLog); + } + + public List<Void> getLoggedMddLibApiResultLogs() { + return loggedMddLibApiResultLog; + } + + @Override public ListenableFuture<Void> logMddStorageStats( - AsyncCallable<MddStorageStats> buildMddStorageStats) { + AsyncCallable<MddStorageStats> buildMddStorageStats) { return immediateFailedFuture( - new UnsupportedOperationException("This method is not implemented in the fake yet.")); + new UnsupportedOperationException("This method is not implemented in the fake yet.")); } @Override @@ -86,7 +101,7 @@ public final class FakeEventLogger implements EventLogger { @Override public void logMddNetworkSavings( - Void fileGroupDetails, + DataDownloadFileGroupStats fileGroupDetails, int code, long fullFileSize, long downloadedFileSize, @@ -97,13 +112,13 @@ public final class FakeEventLogger implements EventLogger { @Override public void logMddDownloadResult( - MddDownloadResult.Code code, DataDownloadFileGroupStats fileGroupDetails) { + MddDownloadResult.Code code, DataDownloadFileGroupStats fileGroupDetails) { throw new UnsupportedOperationException("This method is not implemented in the fake yet."); } @Override - public void logMddQueryStats(Void fileGroupDetails) { - throw new UnsupportedOperationException("This method is not implemented in the fake yet."); + public void logMddQueryStats(DataDownloadFileGroupStats fileGroupDetails) { + loggedMddQueryStats.add(fileGroupDetails); } @Override @@ -112,20 +127,43 @@ public final class FakeEventLogger implements EventLogger { } @Override - public void logMddDownloadLatency(Void fileGroupStats, Void downloadLatency) { + public void logMddDownloadLatency( + DataDownloadFileGroupStats fileGroupStats, Void downloadLatency) { loggedLatencies.put(fileGroupStats, downloadLatency); } @Override - public void logMddUsageEvent(Void fileGroupDetails, Void usageEventLog) { + public void logMddUsageEvent(DataDownloadFileGroupStats fileGroupDetails, Void usageEventLog) { throw new UnsupportedOperationException("This method is not implemented in the fake yet."); } - public List<Integer> getLoggedCodes() { + @Override + public void logNewConfigReceived( + DataDownloadFileGroupStats fileGroupDetails, Void newConfigReceivedInfo) { + loggedNewConfigReceived.put(fileGroupDetails, newConfigReceivedInfo); + } + + public void reset() { + loggedCodes.clear(); + loggedLatencies.clear(); + loggedMddQueryStats.clear(); + loggedNewConfigReceived.clear(); + loggedMddLibApiResultLog.clear(); + } + + public ArrayListMultimap<DataDownloadFileGroupStats, Void> getLoggedNewConfigReceived() { + return loggedNewConfigReceived; + } + + public List<MddClientEvent.Code> getLoggedCodes() { return loggedCodes; } - public ArrayListMultimap<Void, Void> getLoggedLatencies() { + public ArrayListMultimap<DataDownloadFileGroupStats, Void> getLoggedLatencies() { return loggedLatencies; } + + public ArrayList<DataDownloadFileGroupStats> getLoggedMddQueryStats() { + return loggedMddQueryStats; + } } diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/proto/BUILD b/java/com/google/android/libraries/mobiledatadownload/internal/proto/BUILD index d6f0c9d..6be1b57 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/proto/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/internal/proto/BUILD @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. package( + default_applicable_licenses = ["//:license"], default_visibility = ["//:__subpackages__"], licenses = ["notice"], ) diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/proto/metadata.proto b/java/com/google/android/libraries/mobiledatadownload/internal/proto/metadata.proto index cab8a0f..406133c 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/proto/metadata.proto +++ b/java/com/google/android/libraries/mobiledatadownload/internal/proto/metadata.proto @@ -41,7 +41,7 @@ message ExtraHttpHeader { // The tag number of extra fields should start from 1000 to reserve room for // growing DataFileGroup. // -// Next id: 1000 +// Next id: 1001 message DataFileGroupInternal { // Extra information that is kept on disk. // @@ -199,6 +199,15 @@ message DataFileGroupInternal { reserved 28; + // If a group enables preserve_filenames_and_isolate_files + // this property will contain the directory root of the isolated + // structure. Specifically, the property will be a string created from the + // group name and a hash of other identifying properties (account, variantid, + // buildid). + // + // currently only used in aMDD. + optional string isolated_directory_root = 1000; + reserved 4, 5, 7, 8, 9, 15, 18, 22, 24; } @@ -507,8 +516,23 @@ message GroupKey { // Whether or not all files in a fileGroup have been downloaded. optional bool downloaded = 4; - // The variant id of the group. A null or empty value indicates that the group - // does not have an associated variant. + // The variant id of the group for identification purposes. + // + // This is used to ensure that groups with different variants can have + // different entries in MDD metadata, and therefore have different lifecycles. + // + // Note that clients can choose to opt-in to a SINGLE_VARIANT flow where + // different variants replace each other on-device (only single variant can + // exist on a device at a time). In this case, an empty variant_id is set here + // so groups with different variants share the same GroupKey and are subject + // to the same lifecycle, even though the DataFileGroup does have a non-empty + // variant_id. + // + // Because of the SINGLE_VARIANT flow and because groups may still be added + // with no variant_id associated, using this property to tell if the + // associated file group has a variant_id is unreliable. Instead, the + // variant_id set within a DataFileGroup should be used as the source of truth + // about the group (such as when logging). optional string variant_id = 6; reserved 3; @@ -651,11 +675,26 @@ message LoggingState { // This proto is used to store state for logging that is specific to a File // Group. This includes network usage logging and maybe download tiers (for // <internal>). +// +// NEXT TAG: 7 message FileGroupLoggingState { + // GroupKey associated with a file group -- this is used to populate the group + // name and host package name. optional GroupKey group_key = 1; + + // The build_id associated with the file group. optional int64 build_id = 2; + + // The variant_id associated with the file group. + optional string variant_id = 6; + + // The file group version number associated with the file group. optional int32 file_group_version_number = 3; + + // The number of bytes downloaded over a cellular (metered) network. optional int64 cellular_usage = 4; + + // The number of bytes downloaded over a wifi (unmetered) network. optional int64 wifi_usage = 5; } diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/util/BUILD b/java/com/google/android/libraries/mobiledatadownload/internal/util/BUILD index de5e844..1ba22c1 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/util/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/internal/util/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = ["//:__subpackages__"], licenses = ["notice"], ) @@ -36,6 +37,18 @@ android_library( ) android_library( + name = "DownloadFutureMap", + srcs = ["DownloadFutureMap.java"], + deps = [ + "//java/com/google/android/libraries/mobiledatadownload/foreground:NotificationUtil", + "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LogUtil", + "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", + "@androidx_core_core", + "@com_google_guava_guava", + ], +) + +android_library( name = "AndroidSharingUtil", srcs = ["AndroidSharingUtil.java"], deps = [ @@ -45,6 +58,7 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/file/openers:stream", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LogUtil", "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", + "//proto:log_enums_java_proto_lite", "@com_google_guava_guava", ], ) @@ -60,6 +74,7 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/internal:MddConstants", "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", "//proto:transform_java_proto_lite", + "//third_party/java/android_libs/guava_jdk5:hash", "@com_google_code_findbugs_jsr305", "@com_google_guava_guava", ], @@ -83,6 +98,7 @@ android_library( srcs = ["FuturesUtil.java"], deps = [ "//java/com/google/android/libraries/mobiledatadownload/internal/annotations:SequentialControlExecutor", + "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", "@com_google_guava_guava", ], ) diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/util/DirectoryUtil.java b/java/com/google/android/libraries/mobiledatadownload/internal/util/DirectoryUtil.java index 822b421..8ccd20b 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/util/DirectoryUtil.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/util/DirectoryUtil.java @@ -108,6 +108,7 @@ public class DirectoryUtil { * URI, otherwise it returns the "android" scheme URI. */ // TODO(b/118137672): getOnDeviceUri shouldn't return null on error. + @Nullable public static Uri getOnDeviceUri( Context context, diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/util/DownloadFutureMap.java b/java/com/google/android/libraries/mobiledatadownload/internal/util/DownloadFutureMap.java new file mode 100644 index 0000000..81c354f --- /dev/null +++ b/java/com/google/android/libraries/mobiledatadownload/internal/util/DownloadFutureMap.java @@ -0,0 +1,127 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.libraries.mobiledatadownload.internal.util; + +import static com.google.common.util.concurrent.Futures.immediateFailedFuture; +import static com.google.common.util.concurrent.Futures.immediateVoidFuture; + +import androidx.annotation.VisibleForTesting; +import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedExecutionSequencer; +import com.google.common.base.Optional; +import com.google.common.util.concurrent.ListenableFuture; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.Executor; + +/** + * Helper class to maintain the state of MDD download futures. + * + * <p>This follows a limited Map interface and uses {@link ExecutionSequencer} to ensure that all + * operations on the map are synchronized. + * + * <p><b>NOTE:</b> This class is meant to be a container class for download futures and <em>should + * not</em> include any download-specific logic. Its sole purpose is to maintain any in-progress + * download futures in a synchronized manner. Download-specific logic should be implemented outside + * of this class, and can rely on {@link StateChangeCallbacks} to respond to events from this map. + */ +public final class DownloadFutureMap<T> { + private static final String TAG = "DownloadFutureMap"; + + // ExecutionSequencer ensures that enqueued futures are executed sequentially (regardless of the + // executor used). This allows us to keep critical state changes sequential. + private final PropagatedExecutionSequencer futureSerializer = + PropagatedExecutionSequencer.create(); + + private final Executor sequentialControlExecutor; + private final StateChangeCallbacks callbacks; + + // Underlying map to store futures -- synchronization of accesses/updates is handled by + // ExecutionSequencer. + @VisibleForTesting + public final Map<String, ListenableFuture<T>> keyToDownloadFutureMap = new HashMap<>(); + + private DownloadFutureMap(Executor sequentialControlExecutor, StateChangeCallbacks callbacks) { + this.sequentialControlExecutor = sequentialControlExecutor; + this.callbacks = callbacks; + } + + /** Convenience creator when no callbacks should be registered. */ + public static <T> DownloadFutureMap<T> create(Executor sequentialControlExecutor) { + return create(sequentialControlExecutor, new StateChangeCallbacks() {}); + } + + /** Creates a new instance of DownloadFutureMap. */ + public static <T> DownloadFutureMap<T> create( + Executor sequentialControlExecutor, StateChangeCallbacks callbacks) { + return new DownloadFutureMap<T>(sequentialControlExecutor, callbacks); + } + + /** Callback to support custom events based on the state of the map. */ + public static interface StateChangeCallbacks { + /** Respond to the event immediately before a new future is added to the map. */ + default void onAdd(String key, int newSize) throws Exception {} + + /** Respond to the event immediately after a future is removed from the map. */ + default void onRemove(String key, int newSize) throws Exception {} + } + + public ListenableFuture<Void> add(String key, ListenableFuture<T> downloadFuture) { + LogUtil.v("%s: submitting request to add in-progress download future with key: %s", TAG, key); + return futureSerializer.submitAsync( + () -> { + try { + callbacks.onAdd(key, keyToDownloadFutureMap.size() + 1); + keyToDownloadFutureMap.put(key, downloadFuture); + } catch (Exception e) { + LogUtil.e(e, "%s: Failed to add download future (%s) to map", TAG, key); + return immediateFailedFuture(e); + } + return immediateVoidFuture(); + }, + sequentialControlExecutor); + } + + @SuppressWarnings("FutureReturnValueIgnored") + public ListenableFuture<Void> remove(String key) { + LogUtil.v( + "%s: submitting request to remove in-progress download future with key: %s", TAG, key); + return futureSerializer.submitAsync( + () -> { + try { + keyToDownloadFutureMap.remove(key); + callbacks.onRemove(key, keyToDownloadFutureMap.size()); + } catch (Exception e) { + LogUtil.e(e, "%s: Failed to remove download future (%s) from map", TAG, key); + return immediateFailedFuture(e); + } + return immediateVoidFuture(); + }, + sequentialControlExecutor); + } + + public ListenableFuture<Optional<ListenableFuture<T>>> get(String key) { + LogUtil.v("%s: submitting request for in-progress download future with key: %s", TAG, key); + return futureSerializer.submit( + () -> Optional.fromNullable(keyToDownloadFutureMap.get(key)), sequentialControlExecutor); + } + + public ListenableFuture<Boolean> containsKey(String key) { + LogUtil.v("%s: submitting check for in-progress download future with key: %s", TAG, key); + return futureSerializer.submit( + () -> keyToDownloadFutureMap.containsKey(key), sequentialControlExecutor); + } +} diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/util/FileGroupUtil.java b/java/com/google/android/libraries/mobiledatadownload/internal/util/FileGroupUtil.java index eed5da0..63bf9a0 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/util/FileGroupUtil.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/util/FileGroupUtil.java @@ -28,6 +28,8 @@ import com.google.common.base.Optional; import com.google.common.base.Preconditions; import com.google.common.base.Strings; import com.google.common.collect.ImmutableSet; +import com.google.common.hash.Hasher; +import com.google.common.hash.Hashing; import com.google.mobiledatadownload.TransformProto.Transform; import com.google.mobiledatadownload.internal.MetadataProto.DataFile; import com.google.mobiledatadownload.internal.MetadataProto.DataFile.AndroidSharingType; @@ -51,9 +53,7 @@ public class FileGroupUtil { : TimeUnit.SECONDS.toMillis(fileGroup.getExpirationDateSecs()); } - /** - * @return the expiration date of this stale file group in millis - */ + /** Returns the expiration date of this stale file group in millis. */ public static long getStaleExpirationDateMillis(DataFileGroupInternal fileGroup) { return TimeUnit.SECONDS.toMillis(fileGroup.getBookkeeping().getStaleExpirationDate()); } @@ -151,6 +151,29 @@ public class FileGroupUtil { return dataFileGroup; } + /** Sets the isolated root if the file group supports isolated structures. */ + public static DataFileGroupInternal maybeSetIsolatedRoot( + DataFileGroupInternal dataFileGroup, GroupKey groupKey) { + // Check if isolated structure is allowed before adding the root + if (!isIsolatedStructureAllowed(dataFileGroup)) { + return dataFileGroup; + } + + Hasher isolatedRootHasher = + Hashing.sha256() + .newHasher() + .putUnencodedChars(dataFileGroup.getVariantId()) + .putUnencodedChars(MddConstants.SPLIT_CHAR) + .putUnencodedChars(groupKey.getAccount()) + .putUnencodedChars(MddConstants.SPLIT_CHAR) + .putLong(dataFileGroup.getBuildId()); + + String hash = isolatedRootHasher.hash().toString(); + String directoryRoot = String.format("%s_%s", dataFileGroup.getGroupName(), hash); + + return dataFileGroup.toBuilder().setIsolatedDirectoryRoot(directoryRoot).build(); + } + /** Shared method to test whether the given file group supports isolated file structures. */ public static boolean isIsolatedStructureAllowed(DataFileGroupInternal dataFileGroupInternal) { if (VERSION.SDK_INT < VERSION_CODES.LOLLIPOP @@ -174,10 +197,20 @@ public class FileGroupUtil { */ public static Uri getIsolatedRootDirectory( Context context, Optional<String> instanceId, DataFileGroupInternal fileGroupInternal) { + String groupRoot; + if (!fileGroupInternal.getIsolatedDirectoryRoot().isEmpty()) { + groupRoot = fileGroupInternal.getIsolatedDirectoryRoot(); + } else { + // NOTE: Only the group name was used before the isolated directory root field was + // added. To preserve backwards compatibility, fallback to group name if isolated directory + // root is not present. + groupRoot = fileGroupInternal.getGroupName(); + } + return DirectoryUtil.getDownloadSymlinkDirectory( context, fileGroupInternal.getAllowedReadersEnum(), instanceId) .buildUpon() - .appendPath(fileGroupInternal.getGroupName()) + .appendPath(groupRoot) .build(); } @@ -190,8 +223,13 @@ public class FileGroupUtil { Optional<String> instanceId, DataFile dataFile, DataFileGroupInternal parentFileGroup) { - Uri.Builder fileUriBuilder = - getIsolatedRootDirectory(context, instanceId, parentFileGroup).buildUpon(); + Uri rootUri = getIsolatedRootDirectory(context, instanceId, parentFileGroup); + return appendIsolatedFileUri(rootUri, dataFile); + } + + /** Helper method to append isolated file uri to an already known root. */ + public static Uri appendIsolatedFileUri(Uri rootUri, DataFile dataFile) { + Uri.Builder fileUriBuilder = rootUri.buildUpon(); if (dataFile.getRelativeFilePath().isEmpty()) { // If no relative path specified get the last segment from the // urlToDownload. @@ -223,7 +261,8 @@ public class FileGroupUtil { Uri isolatedRootDir = FileGroupUtil.getIsolatedRootDirectory(context, instanceId, dataFileGroup); if (fileStorage.exists(isolatedRootDir)) { - Void unused = fileStorage.open(isolatedRootDir, RecursiveDeleteOpener.create()); + Void unused = + fileStorage.open(isolatedRootDir, RecursiveDeleteOpener.create().withNoFollowLinks()); } } @@ -257,24 +296,29 @@ public class FileGroupUtil { public static boolean isSideloadedFile(DataFile dataFile) { return isFileWithMatchingScheme( - dataFile, + dataFile.getUrlToDownload(), ImmutableSet.of( MddConstants.SIDELOAD_FILE_URL_SCHEME, MddConstants.EMBEDDED_ASSET_URL_SCHEME)); } public static boolean isInlineFile(DataFile dataFile) { - return isFileWithMatchingScheme(dataFile, ImmutableSet.of(MddConstants.INLINE_FILE_URL_SCHEME)); + return isFileWithMatchingScheme( + dataFile.getUrlToDownload(), ImmutableSet.of(MddConstants.INLINE_FILE_URL_SCHEME)); + } + + public static boolean isInlineFile(String url) { + return isFileWithMatchingScheme(url, ImmutableSet.of(MddConstants.INLINE_FILE_URL_SCHEME)); } // Helper method to test whether a DataFile's url scheme is contained in the given scheme set. - private static boolean isFileWithMatchingScheme(DataFile dataFile, ImmutableSet<String> schemes) { - if (!dataFile.hasUrlToDownload()) { + private static boolean isFileWithMatchingScheme(String url, ImmutableSet<String> schemes) { + if (url.isEmpty()) { return false; } - int colon = dataFile.getUrlToDownload().indexOf(':'); + int colon = url.indexOf(':'); // TODO(b/196593240): Ensure this is always handled, or replace with a checked exception - Preconditions.checkState(colon > -1, "Invalid url: %s", dataFile.getUrlToDownload()); - String fileScheme = dataFile.getUrlToDownload().substring(0, colon); + Preconditions.checkState(colon > -1, "Invalid url: %s", url); + String fileScheme = url.substring(0, colon); for (String scheme : schemes) { if (Ascii.equalsIgnoreCase(fileScheme, scheme)) { return true; diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/util/FileGroupsMetadataUtil.java b/java/com/google/android/libraries/mobiledatadownload/internal/util/FileGroupsMetadataUtil.java index 7853bfa..2948df6 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/util/FileGroupsMetadataUtil.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/util/FileGroupsMetadataUtil.java @@ -20,9 +20,9 @@ import android.util.Base64; import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; import com.google.common.base.Optional; import com.google.common.collect.ImmutableList; +import com.google.protobuf.InvalidProtocolBufferException; import com.google.mobiledatadownload.internal.MetadataProto.DataFileGroupInternal; import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; -import com.google.protobuf.InvalidProtocolBufferException; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; @@ -102,7 +102,8 @@ public final class FileGroupsMetadataUtil { /** * Converts a string representing a serialized GroupKey into a GroupKey. * - * @return - groupKey if able to parse stringKey properly. null if parsing fails. + * @return groupKey if able to parse string key properly. + * @throws GroupKeyDeserializationException when unable to parse string key */ // TODO(b/129702287): Move away from proto based deserialization. public static GroupKey deserializeGroupKey(String serializedGroupKey) @@ -110,7 +111,7 @@ public final class FileGroupsMetadataUtil { try { return SharedPreferencesUtil.parseLiteFromEncodedString( serializedGroupKey, GroupKey.parser()); - } catch (InvalidProtocolBufferException e) { + } catch (NullPointerException | InvalidProtocolBufferException e) { throw new GroupKeyDeserializationException( "Failed to deserialize key:" + serializedGroupKey, e); } diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/util/FuturesUtil.java b/java/com/google/android/libraries/mobiledatadownload/internal/util/FuturesUtil.java index cdf1ea3..0e0013c 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/util/FuturesUtil.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/util/FuturesUtil.java @@ -15,10 +15,13 @@ */ package com.google.android.libraries.mobiledatadownload.internal.util; +import static com.google.common.util.concurrent.Futures.immediateFuture; + import com.google.android.libraries.mobiledatadownload.internal.annotations.SequentialControlExecutor; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; import com.google.common.base.Function; -import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.util.ArrayList; import java.util.List; import java.util.concurrent.Executor; @@ -89,18 +92,20 @@ public final class FuturesUtil { this.init = init; } + @CanIgnoreReturnValue public SequentialFutureChain<T> chain(Function<T, T> operation) { operations.add(new DirectFutureChainElement<>(operation)); return this; } + @CanIgnoreReturnValue public SequentialFutureChain<T> chainAsync(Function<T, ListenableFuture<T>> operation) { operations.add(new AsyncFutureChainElement<>(operation)); return this; } public ListenableFuture<T> start() { - ListenableFuture<T> result = Futures.immediateFuture(init); + ListenableFuture<T> result = immediateFuture(init); for (FutureChainElement<T> operation : operations) { result = operation.apply(result); } @@ -121,7 +126,7 @@ public final class FuturesUtil { @Override public ListenableFuture<T> apply(ListenableFuture<T> input) { - return Futures.transform(input, operation::apply, sequentialExecutor); + return PropagatedFutures.transform(input, operation, sequentialExecutor); } } @@ -134,7 +139,7 @@ public final class FuturesUtil { @Override public ListenableFuture<T> apply(ListenableFuture<T> input) { - return Futures.transformAsync(input, operation::apply, sequentialExecutor); + return PropagatedFutures.transformAsync(input, operation::apply, sequentialExecutor); } } } diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/util/ProtoConversionUtil.java b/java/com/google/android/libraries/mobiledatadownload/internal/util/ProtoConversionUtil.java index 04e3446..cdc8a58 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/util/ProtoConversionUtil.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/util/ProtoConversionUtil.java @@ -41,6 +41,13 @@ public final class ProtoConversionUtil { group.toByteArray(), ExtensionRegistryLite.getEmptyRegistry()); } + public static DataFileGroup reverse(DataFileGroupInternal group) + throws InvalidProtocolBufferException { + // Cannot use generated registry here, because it may cause NPE to clients. + // For more detail, see b/140135059. + return DataFileGroup.parseFrom(group.toByteArray(), ExtensionRegistryLite.getEmptyRegistry()); + } + /** * Converts external proto {@link DownloadConditions} into internal proto {@link * MetadataProto.DownloadConditions}. @@ -61,6 +68,10 @@ public final class ProtoConversionUtil { // TODO(b/176103639): Use automated proto converter instead // LINT.IfChange(data_file_convert) public static MetadataProto.DataFile convertDataFile(DataFile dataFile) { + // incompatible argument for parameter value of setChecksumType. + // incompatible argument for parameter value of setAndroidSharingType. + // incompatible argument for parameter value of setAndroidSharingChecksumType. + @SuppressWarnings("nullness:argument.type.incompatible") MetadataProto.DataFile.Builder dataFileBuilder = MetadataProto.DataFile.newBuilder() .setFileId(dataFile.getFileId()) @@ -110,6 +121,8 @@ public final class ProtoConversionUtil { */ // TODO(b/176103639): Use automated proto converter instead // LINT.IfChange(delta_file_convert) + // incompatible argument for parameter value of setDiffDecoder. + @SuppressWarnings("nullness:argument.type.incompatible") public static MetadataProto.DeltaFile convertDeltaFile(DeltaFile deltaFile) { return MetadataProto.DeltaFile.newBuilder() .setUrlToDownload(deltaFile.getUrlToDownload()) diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/util/SharedFilesMetadataUtil.java b/java/com/google/android/libraries/mobiledatadownload/internal/util/SharedFilesMetadataUtil.java index 323819b..7f7cff6 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/util/SharedFilesMetadataUtil.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/util/SharedFilesMetadataUtil.java @@ -93,6 +93,8 @@ public final class SharedFilesMetadataUtil { .toString(); } + // incompatible argument for parameter value of setAllowedReaders. + @SuppressWarnings("nullness:argument.type.incompatible") public static NewFileKey deserializeNewFileKey( String serializedFileKey, Context context, SilentFeedback silentFeedback) throws FileKeyDeserializationException { diff --git a/java/com/google/android/libraries/mobiledatadownload/internal/util/SymlinkUtil.java b/java/com/google/android/libraries/mobiledatadownload/internal/util/SymlinkUtil.java index 9ec91e8..ba4dc3d 100644 --- a/java/com/google/android/libraries/mobiledatadownload/internal/util/SymlinkUtil.java +++ b/java/com/google/android/libraries/mobiledatadownload/internal/util/SymlinkUtil.java @@ -29,6 +29,7 @@ import com.google.android.libraries.mobiledatadownload.file.common.MalformedUriE import java.io.IOException; /** Utility class to create symlinks (if supported). */ +@RequiresApi(VERSION_CODES.LOLLIPOP) public final class SymlinkUtil { private SymlinkUtil() {} diff --git a/java/com/google/android/libraries/mobiledatadownload/lite/BUILD b/java/com/google/android/libraries/mobiledatadownload/lite/BUILD index c1eb8fb..4f8c1f5 100644 --- a/java/com/google/android/libraries/mobiledatadownload/lite/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/lite/BUILD @@ -16,6 +16,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") # MDD Lite visibility is restricted to the following set of packages. Any # new clients must be added to this list in order to grant build visibility. package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], @@ -37,11 +38,15 @@ android_library( ":DownloadProgressMonitor", "//java/com/google/android/libraries/mobiledatadownload:DownloadException", "//java/com/google/android/libraries/mobiledatadownload/downloader:FileDownloader", + "//java/com/google/android/libraries/mobiledatadownload/foreground:ForegroundDownloadKey", "//java/com/google/android/libraries/mobiledatadownload/foreground:NotificationUtil", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LogUtil", + "//java/com/google/android/libraries/mobiledatadownload/internal/util:DownloadFutureMap", + "//java/com/google/android/libraries/mobiledatadownload/tracing", + "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", "@androidx_core_core", "@com_google_auto_value", - "@com_google_dagger", + "@com_google_errorprone_error_prone_annotations", "@com_google_guava_guava", "@org_checkerframework_qual", ], @@ -66,9 +71,11 @@ android_library( ":DownloadListener", "//java/com/google/android/libraries/mobiledatadownload:TimeSource", "//java/com/google/android/libraries/mobiledatadownload/file/spi", + "//java/com/google/android/libraries/mobiledatadownload/internal:AndroidTimeSource", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LogUtil", "@androidx_annotation_annotation", "@com_google_code_findbugs_jsr305", + "@com_google_errorprone_error_prone_annotations", "@com_google_guava_guava", ], ) diff --git a/java/com/google/android/libraries/mobiledatadownload/lite/DownloadProgressMonitor.java b/java/com/google/android/libraries/mobiledatadownload/lite/DownloadProgressMonitor.java index d0fd6fa..4c5fbd6 100644 --- a/java/com/google/android/libraries/mobiledatadownload/lite/DownloadProgressMonitor.java +++ b/java/com/google/android/libraries/mobiledatadownload/lite/DownloadProgressMonitor.java @@ -21,11 +21,11 @@ import com.google.android.libraries.mobiledatadownload.TimeSource; import com.google.android.libraries.mobiledatadownload.file.spi.Monitor; import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; import com.google.common.util.concurrent.MoreExecutors; +import com.google.errorprone.annotations.concurrent.GuardedBy; import java.util.HashMap; import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicLong; import javax.annotation.Nullable; -import javax.annotation.concurrent.GuardedBy; import javax.annotation.concurrent.ThreadSafe; /** A Download Progress Monitor to support {@link DownloadListener}. */ @@ -37,7 +37,6 @@ public class DownloadProgressMonitor implements Monitor, SingleFileDownloadProgr private final TimeSource timeSource; private final Executor sequentialControlExecutor; - // NOTE: GuardRails prohibits multiple public constructors private DownloadProgressMonitor(TimeSource timeSource, Executor controlExecutor) { this.timeSource = timeSource; diff --git a/java/com/google/android/libraries/mobiledatadownload/lite/Downloader.java b/java/com/google/android/libraries/mobiledatadownload/lite/Downloader.java index 208132c..ea4f450 100644 --- a/java/com/google/android/libraries/mobiledatadownload/lite/Downloader.java +++ b/java/com/google/android/libraries/mobiledatadownload/lite/Downloader.java @@ -22,6 +22,7 @@ import com.google.common.base.Preconditions; import com.google.common.base.Supplier; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import com.google.errorprone.annotations.CheckReturnValue; import java.util.concurrent.Executor; @@ -73,8 +74,19 @@ public interface Downloader { @CheckReturnValue ListenableFuture<Void> downloadWithForegroundService(DownloadRequest downloadRequest); - /** Cancel an on-going foreground download. */ - void cancelForegroundDownload(String destinationFileUri); + /** + * Cancel an on-going foreground download. + * + * <p>Use {@link ForegroundDownloadKey} to construct the unique key. + * + * <p><b>NOTE:</b> In most cases, clients will not need to call this -- it is meant to allow the + * ForegroundDownloadService to cancel a download via the Cancel action registered to a + * notification. + * + * <p>Clients should prefer to cancel the future returned to them from {@link + * #downloadWithForegroundService} instead. + */ + void cancelForegroundDownload(String downloadKey); static Downloader.Builder newBuilder() { return new Downloader.Builder(); @@ -91,12 +103,14 @@ public interface Downloader { private Optional<SingleFileDownloadProgressMonitor> downloadMonitorOptional = Optional.absent(); private Optional<Class<?>> foregroundDownloadServiceClassOptional = Optional.absent(); + @CanIgnoreReturnValue public Builder setContext(Context context) { this.context = context.getApplicationContext(); return this; } /** Set the Control Executor which will run MDDLite control flow. */ + @CanIgnoreReturnValue public Builder setControlExecutor(Executor controlExecutor) { Preconditions.checkNotNull(controlExecutor); // Executor that will execute tasks sequentially. @@ -115,6 +129,7 @@ public interface Downloader { * DownloadListener} to {@link Downloader#download}. The DownloadListener's {@code onFailure} * and {@code onComplete} will be invoked regardless of whether this is set. */ + @CanIgnoreReturnValue public Builder setDownloadMonitor(SingleFileDownloadProgressMonitor downloadMonitor) { this.downloadMonitorOptional = Optional.of(downloadMonitor); return this; @@ -127,6 +142,7 @@ public interface Downloader { * <p>This is required to use {@link Downloader#downloadWithForegroundService}. Not providing * this will result in a failed future when calling downloadWithForegroundService. */ + @CanIgnoreReturnValue public Builder setForegroundDownloadService(Class<?> foregroundDownloadServiceClass) { this.foregroundDownloadServiceClassOptional = Optional.of(foregroundDownloadServiceClass); return this; @@ -136,6 +152,7 @@ public interface Downloader { * Set the FileDownloader Supplier. MDDLite takes in a Supplier of FileDownload to support lazy * instantiation of the FileDownloader */ + @CanIgnoreReturnValue public Builder setFileDownloaderSupplier(Supplier<FileDownloader> fileDownloaderSupplier) { this.fileDownloaderSupplier = fileDownloaderSupplier; return this; diff --git a/java/com/google/android/libraries/mobiledatadownload/lite/DownloaderImpl.java b/java/com/google/android/libraries/mobiledatadownload/lite/DownloaderImpl.java index 1c0cb49..8472667 100644 --- a/java/com/google/android/libraries/mobiledatadownload/lite/DownloaderImpl.java +++ b/java/com/google/android/libraries/mobiledatadownload/lite/DownloaderImpl.java @@ -15,6 +15,9 @@ */ package com.google.android.libraries.mobiledatadownload.lite; +import static com.google.common.util.concurrent.Futures.immediateFailedFuture; +import static com.google.common.util.concurrent.Futures.immediateVoidFuture; + import android.content.Context; import androidx.annotation.VisibleForTesting; import androidx.core.app.NotificationCompat; @@ -22,17 +25,18 @@ import androidx.core.app.NotificationManagerCompat; import com.google.android.libraries.mobiledatadownload.DownloadException; import com.google.android.libraries.mobiledatadownload.DownloadException.DownloadResultCode; import com.google.android.libraries.mobiledatadownload.downloader.FileDownloader; +import com.google.android.libraries.mobiledatadownload.foreground.ForegroundDownloadKey; import com.google.android.libraries.mobiledatadownload.foreground.NotificationUtil; import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; +import com.google.android.libraries.mobiledatadownload.internal.util.DownloadFutureMap; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFluentFuture; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; import com.google.common.base.Optional; -import com.google.common.base.Preconditions; import com.google.common.base.Supplier; import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListenableFutureTask; import com.google.common.util.concurrent.MoreExecutors; -import java.util.HashMap; -import java.util.Map; import java.util.concurrent.Executor; import org.checkerframework.checker.nullness.compatqual.NullableDecl; @@ -46,9 +50,8 @@ final class DownloaderImpl implements Downloader { private final Optional<SingleFileDownloadProgressMonitor> downloadMonitorOptional; private final Supplier<FileDownloader> fileDownloaderSupplier; - // Synchronization will be done through sequentialControlExecutor - @VisibleForTesting - final Map<String, ListenableFuture<Void>> keyToListenableFuture = new HashMap<>(); + @VisibleForTesting final DownloadFutureMap<Void> downloadFutureMap; + @VisibleForTesting final DownloadFutureMap<Void> foregroundDownloadFutureMap; DownloaderImpl( Context context, @@ -61,19 +64,25 @@ final class DownloaderImpl implements Downloader { this.foregroundDownloadServiceClassOptional = foregroundDownloadServiceClassOptional; this.downloadMonitorOptional = downloadMonitorOptional; this.fileDownloaderSupplier = fileDownloaderSupplier; + this.downloadFutureMap = DownloadFutureMap.create(sequentialControlExecutor); + this.foregroundDownloadFutureMap = + DownloadFutureMap.create( + sequentialControlExecutor, + createCallbacksForForegroundService(context, foregroundDownloadServiceClassOptional)); } @Override public ListenableFuture<Void> download(DownloadRequest downloadRequest) { LogUtil.d("%s: download for Uri = %s", TAG, downloadRequest.destinationFileUri().toString()); - return Futures.submitAsync( - () -> { + ForegroundDownloadKey foregroundDownloadKey = + ForegroundDownloadKey.ofSingleFile(downloadRequest.destinationFileUri()); + + return PropagatedFutures.transformAsync( + getInProgressDownloadFuture(foregroundDownloadKey.toString()), + (Optional<ListenableFuture<Void>> existingDownloadFuture) -> { // if there is the same on-going request, return that one. - if (keyToListenableFuture.containsKey(downloadRequest.destinationFileUri().toString())) { - // uriToListenableFuture.get must return Non-null since we check the containsKey above. - // checkNotNull is to suppress false alarm about @Nullable result. - return Preconditions.checkNotNull( - keyToListenableFuture.get(downloadRequest.destinationFileUri().toString())); + if (existingDownloadFuture.isPresent()) { + return existingDownloadFuture.get(); } // Register listener with monitor if present @@ -87,13 +96,19 @@ final class DownloaderImpl implements Downloader { } else { LogUtil.w( "%s: download request included DownloadListener, but DownloadMonitor is not" - + " present! DownloadListener will only be invoked for complete/failure."); + + " present! DownloadListener will only be invoked for complete/failure.", + TAG); } } - ListenableFuture<Void> downloadFuture = startDownload(downloadRequest); + // Create a ListenableFutureTask to delay starting the downloadFuture until we can add the + // future to our map. + ListenableFutureTask<Void> startTask = ListenableFutureTask.create(() -> null); + ListenableFuture<Void> downloadFuture = + PropagatedFutures.transformAsync( + startTask, unused -> startDownload(downloadRequest), sequentialControlExecutor); - Futures.addCallback( + PropagatedFutures.addCallback( downloadFuture, new FutureCallback<Void>() { @Override @@ -104,36 +119,36 @@ final class DownloaderImpl implements Downloader { // Remove download listener and remove download future from map after listener // completes if (downloadRequest.listenerOptional().isPresent()) { - Futures.addCallback( + PropagatedFutures.addCallback( downloadRequest.listenerOptional().get().onComplete(), new FutureCallback<Void>() { @Override public void onSuccess(@NullableDecl Void result) { - keyToListenableFuture.remove( - downloadRequest.destinationFileUri().toString()); if (downloadMonitorOptional.isPresent()) { downloadMonitorOptional .get() .removeDownloadListener(downloadRequest.destinationFileUri()); } + ListenableFuture<Void> unused = + downloadFutureMap.remove(foregroundDownloadKey.toString()); } @Override public void onFailure(Throwable t) { LogUtil.e(t, "%s: Failed to run client onComplete", TAG); - keyToListenableFuture.remove( - downloadRequest.destinationFileUri().toString()); if (downloadMonitorOptional.isPresent()) { downloadMonitorOptional .get() .removeDownloadListener(downloadRequest.destinationFileUri()); } + ListenableFuture<Void> unused = + downloadFutureMap.remove(foregroundDownloadKey.toString()); } }, sequentialControlExecutor); } else { - // remove from future map immediately - keyToListenableFuture.remove(downloadRequest.destinationFileUri().toString()); + ListenableFuture<Void> unused = + downloadFutureMap.remove(foregroundDownloadKey.toString()); } } @@ -151,14 +166,20 @@ final class DownloaderImpl implements Downloader { .removeDownloadListener(downloadRequest.destinationFileUri()); } } - keyToListenableFuture.remove(downloadRequest.destinationFileUri().toString()); + ListenableFuture<Void> unused = + downloadFutureMap.remove(foregroundDownloadKey.toString()); } }, MoreExecutors.directExecutor()); - keyToListenableFuture.put( - downloadRequest.destinationFileUri().toString(), downloadFuture); - return downloadFuture; + return PropagatedFutures.transformAsync( + downloadFutureMap.add(foregroundDownloadKey.toString(), downloadFuture), + unused -> { + // Now that the download future is added, start the task and return the future + startTask.run(); + return downloadFuture; + }, + sequentialControlExecutor); }, sequentialControlExecutor); } @@ -178,7 +199,7 @@ final class DownloaderImpl implements Downloader { return fileDownloaderSupplier.get().startDownloading(fileDownloaderRequest); } catch (RuntimeException e) { // Catch any unchecked exceptions that prevented the download from starting. - return Futures.immediateFailedFuture( + return immediateFailedFuture( DownloadException.builder() .setDownloadResultCode(DownloadResultCode.UNKNOWN_ERROR) .setCause(e) @@ -192,23 +213,25 @@ final class DownloaderImpl implements Downloader { "%s: downloadWithForegroundService for Uri = %s", TAG, downloadRequest.destinationFileUri().toString()); if (!downloadMonitorOptional.isPresent()) { - return Futures.immediateFailedFuture( + return immediateFailedFuture( new IllegalStateException( "downloadWithForegroundService: DownloadMonitor is not provided!")); } if (!foregroundDownloadServiceClassOptional.isPresent()) { - return Futures.immediateFailedFuture( + return immediateFailedFuture( new IllegalStateException( "downloadWithForegroundService: ForegroundDownloadService is not provided!")); } - return Futures.submitAsync( - () -> { + + ForegroundDownloadKey foregroundDownloadKey = + ForegroundDownloadKey.ofSingleFile(downloadRequest.destinationFileUri()); + + return PropagatedFutures.transformAsync( + getInProgressDownloadFuture(foregroundDownloadKey.toString()), + (Optional<ListenableFuture<Void>> existingDownloadFuture) -> { // if there is the same on-going request, return that one. - if (keyToListenableFuture.containsKey(downloadRequest.destinationFileUri().toString())) { - // uriToListenableFuture.get must return Non-null since we check the containsKey above. - // checkNotNull is to suppress false alarm about @Nullable result. - return Preconditions.checkNotNull( - keyToListenableFuture.get(downloadRequest.destinationFileUri().toString())); + if (existingDownloadFuture.isPresent()) { + return existingDownloadFuture.get(); } // It's OK to recreate the NotificationChannel since it can also be used to restore a @@ -216,14 +239,6 @@ final class DownloaderImpl implements Downloader { // importance. NotificationUtil.createNotificationChannel(context); - // Only start the foreground download service when there is the first download request. - if (keyToListenableFuture.isEmpty()) { - NotificationUtil.startForegroundDownloadService( - context, - foregroundDownloadServiceClassOptional.get(), - downloadRequest.destinationFileUri().toString()); - } - DownloadListener downloadListenerWithNotification = createDownloadListenerWithNotification(downloadRequest); @@ -233,9 +248,14 @@ final class DownloaderImpl implements Downloader { .addDownloadListener( downloadRequest.destinationFileUri(), downloadListenerWithNotification); - ListenableFuture<Void> downloadFuture = startDownload(downloadRequest); + // Create a ListenableFutureTask to delay starting the downloadFuture until we can add the + // future to our map. + ListenableFutureTask<Void> startTask = ListenableFutureTask.create(() -> null); + ListenableFuture<Void> downloadFuture = + PropagatedFutures.transformAsync( + startTask, unused -> startDownload(downloadRequest), sequentialControlExecutor); - Futures.addCallback( + PropagatedFutures.addCallback( downloadFuture, new FutureCallback<Void>() { @Override @@ -243,7 +263,7 @@ final class DownloaderImpl implements Downloader { // Currently the MobStore monitor does not support onSuccess so we have to add // callback to the download future here. - Futures.addCallback( + PropagatedFutures.addCallback( downloadListenerWithNotification.onComplete(), new FutureCallback<Void>() { @Override @@ -267,15 +287,25 @@ final class DownloaderImpl implements Downloader { }, MoreExecutors.directExecutor()); - keyToListenableFuture.put( - downloadRequest.destinationFileUri().toString(), downloadFuture); - return downloadFuture; + return PropagatedFutures.transformAsync( + foregroundDownloadFutureMap.add(foregroundDownloadKey.toString(), downloadFuture), + unused -> { + // Now that the download future is added, start the task and return the future + startTask.run(); + return downloadFuture; + }, + sequentialControlExecutor); }, sequentialControlExecutor); } // Assertion: foregroundDownloadService and downloadMonitor are present private DownloadListener createDownloadListenerWithNotification(DownloadRequest downloadRequest) { + String networkPausedMessage = + downloadRequest.downloadConstraints().requireUnmeteredNetwork() + ? NotificationUtil.getDownloadPausedWifiMessage(context) + : NotificationUtil.getDownloadPausedMessage(context); + NotificationManagerCompat notificationManager = NotificationManagerCompat.from(context); NotificationCompat.Builder notification = NotificationUtil.createNotificationBuilder( @@ -284,14 +314,16 @@ final class DownloaderImpl implements Downloader { downloadRequest.notificationContentTitle(), downloadRequest.notificationContentTextOptional().or(downloadRequest.urlToDownload())); - int notificationKey = - NotificationUtil.notificationKeyForKey(downloadRequest.destinationFileUri().toString()); + ForegroundDownloadKey foregroundDownloadKey = + ForegroundDownloadKey.ofSingleFile(downloadRequest.destinationFileUri()); + + int notificationKey = NotificationUtil.notificationKeyForKey(foregroundDownloadKey.toString()); // Attach the Cancel action to the notification. NotificationUtil.createCancelAction( context, foregroundDownloadServiceClassOptional.get(), - downloadRequest.destinationFileUri().toString(), + foregroundDownloadKey.toString(), notification, notificationKey); notificationManager.notify(notificationKey, notification.build()); @@ -299,49 +331,56 @@ final class DownloaderImpl implements Downloader { return new DownloadListener() { @Override public void onProgress(long currentSize) { - sequentialControlExecutor.execute( - () -> { - // There can be a race condition, where onPausedForConnectivity can be called - // after onComplete or onFailure which removes the future and the notification. - if (keyToListenableFuture.containsKey( - downloadRequest.destinationFileUri().toString())) { - notification - .setCategory(NotificationCompat.CATEGORY_PROGRESS) - .setSmallIcon(android.R.drawable.stat_sys_download) - .setProgress( - downloadRequest.fileSizeBytes(), - (int) currentSize, - /* indeterminate = */ downloadRequest.fileSizeBytes() <= 0); - notificationManager.notify(notificationKey, notification.build()); - } - if (downloadRequest.listenerOptional().isPresent()) { - downloadRequest.listenerOptional().get().onProgress(currentSize); - } - }); + // TODO(b/229123693): return this future once DownloadListener has an async api. + ListenableFuture<?> unused = + PropagatedFutures.transformAsync( + foregroundDownloadFutureMap.containsKey(foregroundDownloadKey.toString()), + futureInProgress -> { + if (futureInProgress) { + notification + .setCategory(NotificationCompat.CATEGORY_PROGRESS) + .setContentText( + downloadRequest + .notificationContentTextOptional() + .or(downloadRequest.urlToDownload())) + .setSmallIcon(android.R.drawable.stat_sys_download) + .setProgress( + downloadRequest.fileSizeBytes(), + (int) currentSize, + /* indeterminate= */ downloadRequest.fileSizeBytes() <= 0); + notificationManager.notify(notificationKey, notification.build()); + } + if (downloadRequest.listenerOptional().isPresent()) { + downloadRequest.listenerOptional().get().onProgress(currentSize); + } + return immediateVoidFuture(); + }, + sequentialControlExecutor); } @Override public void onPausedForConnectivity() { - sequentialControlExecutor.execute( - () -> { - // There can be a race condition, where onPausedForConnectivity can be called - // after onComplete or onFailure which removes the future and the notification. - if (keyToListenableFuture.containsKey( - downloadRequest.destinationFileUri().toString())) { - notification - .setCategory(NotificationCompat.CATEGORY_STATUS) - .setContentText(NotificationUtil.getDownloadPausedMessage(context)) - .setSmallIcon(android.R.drawable.stat_sys_download) - .setOngoing(true) - // hide progress bar. - .setProgress(0, 0, false); - notificationManager.notify(notificationKey, notification.build()); - } - - if (downloadRequest.listenerOptional().isPresent()) { - downloadRequest.listenerOptional().get().onPausedForConnectivity(); - } - }); + // TODO(b/229123693): return this future once DownloadListener has an async api. + ListenableFuture<?> unused = + PropagatedFutures.transformAsync( + foregroundDownloadFutureMap.containsKey(foregroundDownloadKey.toString()), + futureInProgress -> { + if (futureInProgress) { + notification + .setCategory(NotificationCompat.CATEGORY_STATUS) + .setContentText(networkPausedMessage) + .setSmallIcon(android.R.drawable.stat_sys_download) + .setOngoing(true) + // hide progress bar. + .setProgress(0, 0, false); + notificationManager.notify(notificationKey, notification.build()); + } + if (downloadRequest.listenerOptional().isPresent()) { + downloadRequest.listenerOptional().get().onPausedForConnectivity(); + } + return immediateVoidFuture(); + }, + sequentialControlExecutor); } @Override @@ -350,92 +389,154 @@ final class DownloaderImpl implements Downloader { ListenableFuture<Void> clientOnCompleteFuture = downloadRequest.listenerOptional().isPresent() ? downloadRequest.listenerOptional().get().onComplete() - : Futures.immediateVoidFuture(); + : immediateVoidFuture(); // Logic to shutdown Foreground Download Service after the client's provided onComplete // finished - clientOnCompleteFuture.addListener( - () -> { - // Clear the notification action. - notification.mActions.clear(); - - if (downloadRequest.showDownloadedNotification()) { - notification - .setCategory(NotificationCompat.CATEGORY_STATUS) - .setContentText(NotificationUtil.getDownloadSuccessMessage(context)) - .setOngoing(false) - .setSmallIcon(android.R.drawable.stat_sys_download_done) - // hide progress bar. - .setProgress(0, 0, false); - - notificationManager.notify(notificationKey, notification.build()); - } else { - NotificationUtil.cancelNotificationForKey( - context, downloadRequest.destinationFileUri().toString()); - } - - keyToListenableFuture.remove(downloadRequest.destinationFileUri().toString()); - // If there is no other on-going foreground download, shutdown the - // ForegroundDownloadService - if (keyToListenableFuture.isEmpty()) { - NotificationUtil.stopForegroundDownloadService( - context, foregroundDownloadServiceClassOptional.get()); - } + return PropagatedFluentFuture.from(clientOnCompleteFuture) + .transformAsync( + unused -> { + // onComplete succeeded, show a success message + notification.mActions.clear(); + + if (downloadRequest.showDownloadedNotification()) { + notification + .setCategory(NotificationCompat.CATEGORY_STATUS) + .setContentText(NotificationUtil.getDownloadSuccessMessage(context)) + .setOngoing(false) + .setSmallIcon(android.R.drawable.stat_sys_download_done) + // hide progress bar. + .setProgress(0, 0, false); + + notificationManager.notify(notificationKey, notification.build()); + } else { + NotificationUtil.cancelNotificationForKey( + context, foregroundDownloadKey.toString()); + } + return immediateVoidFuture(); + }, + sequentialControlExecutor) + .catchingAsync( + Exception.class, + e -> { + LogUtil.w( + e, + "%s: Delegate onComplete failed for uri: %s, showing failure notification.", + TAG, + downloadRequest.destinationFileUri()); + notification.mActions.clear(); + + if (downloadRequest.showDownloadedNotification()) { + notification + .setCategory(NotificationCompat.CATEGORY_STATUS) + .setContentText(NotificationUtil.getDownloadFailedMessage(context)) + .setOngoing(false) + .setSmallIcon(android.R.drawable.stat_sys_warning) + // hide progress bar. + .setProgress(0, 0, false); + + notificationManager.notify(notificationKey, notification.build()); + } else { + NotificationUtil.cancelNotificationForKey( + context, downloadRequest.destinationFileUri().toString()); + } - downloadMonitorOptional - .get() - .removeDownloadListener(downloadRequest.destinationFileUri()); - }, - sequentialControlExecutor); - return clientOnCompleteFuture; + return immediateVoidFuture(); + }, + sequentialControlExecutor) + .transformAsync( + unused -> { + // After success or failure notification is shown, clean up + downloadMonitorOptional + .get() + .removeDownloadListener(downloadRequest.destinationFileUri()); + + return foregroundDownloadFutureMap.remove(foregroundDownloadKey.toString()); + }, + sequentialControlExecutor); } @Override public void onFailure(Throwable t) { - sequentialControlExecutor.execute( - () -> { - // Clear the notification action. - notification.mActions.clear(); - - // Show download failed in notification. - notification - .setCategory(NotificationCompat.CATEGORY_STATUS) - .setContentText(NotificationUtil.getDownloadFailedMessage(context)) - .setOngoing(false) - .setSmallIcon(android.R.drawable.stat_sys_warning) - // hide progress bar. - .setProgress(0, 0, false); - - notificationManager.notify(notificationKey, notification.build()); - - keyToListenableFuture.remove(downloadRequest.destinationFileUri().toString()); - - // If there is no other on-going foreground download, shutdown the - // ForegroundDownloadService - if (keyToListenableFuture.isEmpty()) { - NotificationUtil.stopForegroundDownloadService( - context, foregroundDownloadServiceClassOptional.get()); - } + // TODO(b/229123693): return this future once DownloadListener has an async api. + ListenableFuture<?> unused = + PropagatedFutures.submitAsync( + () -> { + // Clear the notification action. + notification.mActions.clear(); + + // Show download failed in notification. + notification + .setCategory(NotificationCompat.CATEGORY_STATUS) + .setContentText(NotificationUtil.getDownloadFailedMessage(context)) + .setOngoing(false) + .setSmallIcon(android.R.drawable.stat_sys_warning) + // hide progress bar. + .setProgress(0, 0, false); + + notificationManager.notify(notificationKey, notification.build()); - if (downloadRequest.listenerOptional().isPresent()) { - downloadRequest.listenerOptional().get().onFailure(t); - } - downloadMonitorOptional - .get() - .removeDownloadListener(downloadRequest.destinationFileUri()); - }); + if (downloadRequest.listenerOptional().isPresent()) { + downloadRequest.listenerOptional().get().onFailure(t); + } + downloadMonitorOptional + .get() + .removeDownloadListener(downloadRequest.destinationFileUri()); + + return foregroundDownloadFutureMap.remove(foregroundDownloadKey.toString()); + }, + sequentialControlExecutor); } }; } @Override - public void cancelForegroundDownload(String destinationFileUri) { - LogUtil.d("%s: CancelForegroundDownload for Uri = %s", TAG, destinationFileUri); - sequentialControlExecutor.execute( - () -> { - if (keyToListenableFuture.containsKey(destinationFileUri)) { - keyToListenableFuture.get(destinationFileUri).cancel(true); - } - }); + public void cancelForegroundDownload(String downloadKey) { + LogUtil.d("%s: CancelForegroundDownload for Uri = %s", TAG, downloadKey); + ListenableFuture<?> unused = + PropagatedFutures.transformAsync( + getInProgressDownloadFuture(downloadKey), + downloadFuture -> { + if (downloadFuture.isPresent()) { + LogUtil.v( + "%s: CancelForegroundDownload future found for key = %s, cancelling...", + TAG, downloadKey); + downloadFuture.get().cancel(false); + } + return immediateVoidFuture(); + }, + sequentialControlExecutor); + } + + private ListenableFuture<Optional<ListenableFuture<Void>>> getInProgressDownloadFuture( + String key) { + return PropagatedFutures.transformAsync( + foregroundDownloadFutureMap.containsKey(key), + isInForeground -> + isInForeground ? foregroundDownloadFutureMap.get(key) : downloadFutureMap.get(key), + sequentialControlExecutor); + } + + private static DownloadFutureMap.StateChangeCallbacks createCallbacksForForegroundService( + Context context, Optional<Class<?>> foregroundDownloadServiceClassOptional) { + return new DownloadFutureMap.StateChangeCallbacks() { + @Override + public void onAdd(String key, int newSize) { + // Only start foreground service if this is the first future we are adding. + if (newSize == 1 && foregroundDownloadServiceClassOptional.isPresent()) { + NotificationUtil.startForegroundDownloadService( + context, foregroundDownloadServiceClassOptional.get(), key); + } + } + + @Override + public void onRemove(String key, int newSize) { + // Only stop foreground service if there are no more futures remaining. + if (newSize == 0 && foregroundDownloadServiceClassOptional.isPresent()) { + NotificationUtil.stopForegroundDownloadService( + context, foregroundDownloadServiceClassOptional.get(), key); + } + } + }; } } diff --git a/java/com/google/android/libraries/mobiledatadownload/lite/annotations/BUILD b/java/com/google/android/libraries/mobiledatadownload/lite/annotations/BUILD index fd00b3b..17ac54b 100644 --- a/java/com/google/android/libraries/mobiledatadownload/lite/annotations/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/lite/annotations/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], diff --git a/java/com/google/android/libraries/mobiledatadownload/logger/BUILD b/java/com/google/android/libraries/mobiledatadownload/logger/BUILD index d8a3560..6395421 100644 --- a/java/com/google/android/libraries/mobiledatadownload/logger/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/logger/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], @@ -27,5 +28,7 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload:Flags", "//java/com/google/android/libraries/mobiledatadownload:Logger", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LogUtil", + "//proto:log_enums_java_proto_lite", + "//proto:logs_java_proto_lite", ], ) diff --git a/java/com/google/android/libraries/mobiledatadownload/logger/FileGroupPopulatorLogger.java b/java/com/google/android/libraries/mobiledatadownload/logger/FileGroupPopulatorLogger.java index 435f3b3..d7597b9 100644 --- a/java/com/google/android/libraries/mobiledatadownload/logger/FileGroupPopulatorLogger.java +++ b/java/com/google/android/libraries/mobiledatadownload/logger/FileGroupPopulatorLogger.java @@ -18,6 +18,7 @@ package com.google.android.libraries.mobiledatadownload.logger; import com.google.android.libraries.mobiledatadownload.Flags; import com.google.android.libraries.mobiledatadownload.Logger; import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; +import com.google.mobiledatadownload.LogEnumsProto.MddDownloadResult; /** The event logger for {@code FileGroupPopulator}'s. */ public final class FileGroupPopulatorLogger { @@ -32,21 +33,22 @@ public final class FileGroupPopulatorLogger { /** Logs the refresh result of {@code ManifestFileGroupPopulator}. */ public void logManifestFileGroupPopulatorRefreshResult( - int code, String manifestId, String ownerPackageName, String manifestFileUrl) { + MddDownloadResult.Code code, + String manifestId, + String ownerPackageName, + String manifestFileUrl) { int sampleInterval = flags.mddDefaultSampleInterval(); if (!LogUtil.shouldSampleInterval(sampleInterval)) { return; } - Void logData = null; } /** Logs the refresh result of {@code GellerFileGroupPopulator}. */ public void logGddFileGroupPopulatorRefreshResult( - int code, String configurationId, String ownerPackageName, String corpus) { + MddDownloadResult.Code code, String configurationId, String ownerPackageName, String corpus) { int sampleInterval = flags.mddDefaultSampleInterval(); if (!LogUtil.shouldSampleInterval(sampleInterval)) { return; } - Void logData = null; } } diff --git a/java/com/google/android/libraries/mobiledatadownload/monitor/BUILD b/java/com/google/android/libraries/mobiledatadownload/monitor/BUILD index caeaa3c..2f77809 100644 --- a/java/com/google/android/libraries/mobiledatadownload/monitor/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/monitor/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], @@ -27,10 +28,11 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload:TimeSource", "//java/com/google/android/libraries/mobiledatadownload/file/monitors", "//java/com/google/android/libraries/mobiledatadownload/file/spi", + "//java/com/google/android/libraries/mobiledatadownload/internal:AndroidTimeSource", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LogUtil", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LoggingStateStore", "//java/com/google/android/libraries/mobiledatadownload/internal/proto:metadata_java_proto_lite", - "//java/com/google/android/libraries/mobiledatadownload/tracing", + "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", "@androidx_annotation_annotation", "@com_google_code_findbugs_jsr305", "@com_google_guava_guava", @@ -45,11 +47,13 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload:TimeSource", "//java/com/google/android/libraries/mobiledatadownload/file/monitors", "//java/com/google/android/libraries/mobiledatadownload/file/spi", + "//java/com/google/android/libraries/mobiledatadownload/internal:AndroidTimeSource", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LogUtil", "//java/com/google/android/libraries/mobiledatadownload/lite:DownloadListener", "//java/com/google/android/libraries/mobiledatadownload/lite:DownloadProgressMonitor", "@androidx_annotation_annotation", "@com_google_code_findbugs_jsr305", + "@com_google_errorprone_error_prone_annotations", "@com_google_guava_guava", ], ) diff --git a/java/com/google/android/libraries/mobiledatadownload/monitor/DownloadProgressMonitor.java b/java/com/google/android/libraries/mobiledatadownload/monitor/DownloadProgressMonitor.java index 5dcbf6a..b8e7307 100644 --- a/java/com/google/android/libraries/mobiledatadownload/monitor/DownloadProgressMonitor.java +++ b/java/com/google/android/libraries/mobiledatadownload/monitor/DownloadProgressMonitor.java @@ -24,12 +24,12 @@ import com.google.android.libraries.mobiledatadownload.file.spi.Monitor; import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; import com.google.android.libraries.mobiledatadownload.lite.SingleFileDownloadProgressMonitor; import com.google.common.util.concurrent.MoreExecutors; +import com.google.errorprone.annotations.concurrent.GuardedBy; import java.util.HashMap; import java.util.concurrent.Executor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import javax.annotation.Nullable; -import javax.annotation.concurrent.GuardedBy; import javax.annotation.concurrent.ThreadSafe; /** diff --git a/java/com/google/android/libraries/mobiledatadownload/monitor/NetworkUsageMonitor.java b/java/com/google/android/libraries/mobiledatadownload/monitor/NetworkUsageMonitor.java index 413a2d1..d41f45c 100644 --- a/java/com/google/android/libraries/mobiledatadownload/monitor/NetworkUsageMonitor.java +++ b/java/com/google/android/libraries/mobiledatadownload/monitor/NetworkUsageMonitor.java @@ -15,7 +15,6 @@ */ package com.google.android.libraries.mobiledatadownload.monitor; -import static com.google.android.libraries.mobiledatadownload.tracing.TracePropagation.propagateFutureCallback; import static com.google.common.util.concurrent.MoreExecutors.directExecutor; import static java.util.concurrent.TimeUnit.SECONDS; @@ -31,8 +30,8 @@ import com.google.android.libraries.mobiledatadownload.file.monitors.ByteCountin import com.google.android.libraries.mobiledatadownload.file.spi.Monitor; import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; import com.google.android.libraries.mobiledatadownload.internal.logging.LoggingStateStore; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.mobiledatadownload.internal.MetadataProto.FileGroupLoggingState; import com.google.mobiledatadownload.internal.MetadataProto.GroupKey; @@ -98,6 +97,7 @@ public class NetworkUsageMonitor implements Monitor { * @param uri The Uri of the data file. * @param groupKey The groupKey part of the file group. * @param buildId The build id of the file group. + * @param variantId The variant id of the file group. * @param versionNumber The version number of the file group. * @param loggingStateStore The storage for the network usage logs */ @@ -105,12 +105,14 @@ public class NetworkUsageMonitor implements Monitor { Uri uri, GroupKey groupKey, long buildId, + String variantId, int versionNumber, LoggingStateStore loggingStateStore) { FileGroupLoggingState fileGroupLoggingStateKey = FileGroupLoggingState.newBuilder() .setGroupKey(groupKey) .setBuildId(buildId) + .setVariantId(variantId) .setFileGroupVersionNumber(versionNumber) .build(); @@ -189,26 +191,25 @@ public class NetworkUsageMonitor implements Monitor { .setWifiUsage(wifiCounter.getAndSet(0)) .build()); - Futures.addCallback( + PropagatedFutures.addCallback( incrementDataUsage, - propagateFutureCallback( - new FutureCallback<Void>() { - @Override - public void onSuccess(Void unused) { - LogUtil.d( - "%s: Successfully incremented LoggingStateStore network usage for %s", - TAG, fileGroupLoggingStateKey.getGroupKey().getGroupName()); - } + new FutureCallback<Void>() { + @Override + public void onSuccess(Void unused) { + LogUtil.d( + "%s: Successfully incremented LoggingStateStore network usage for %s", + TAG, fileGroupLoggingStateKey.getGroupKey().getGroupName()); + } - @Override - public void onFailure(Throwable t) { - LogUtil.e( - t, - "%s: Unable to increment LoggingStateStore network usage for %s", - TAG, - fileGroupLoggingStateKey.getGroupKey().getGroupName()); - } - }), + @Override + public void onFailure(Throwable t) { + LogUtil.e( + t, + "%s: Unable to increment LoggingStateStore network usage for %s", + TAG, + fileGroupLoggingStateKey.getGroupKey().getGroupName()); + } + }, directExecutor()); } } diff --git a/java/com/google/android/libraries/mobiledatadownload/populator/BUILD b/java/com/google/android/libraries/mobiledatadownload/populator/BUILD index 42f7e73..d9d2a7a 100644 --- a/java/com/google/android/libraries/mobiledatadownload/populator/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/populator/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//visibility:public", ], @@ -36,6 +37,7 @@ android_library( ":DataFileGroupOverrider", "//java/com/google/android/libraries/mobiledatadownload", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LogUtil", + "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", "//proto:download_config_java_proto_lite", "@com_google_guava_guava", ], @@ -81,6 +83,8 @@ android_library( deps = [ ":ManifestConfigOverrider", "//java/com/google/android/libraries/mobiledatadownload", + "//java/com/google/android/libraries/mobiledatadownload:AggregateException", + "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LogUtil", "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", "//proto:download_config_java_proto_lite", "@androidx_annotation_annotation", @@ -105,7 +109,9 @@ android_library( ":ManifestConfigOverrider", "//java/com/google/android/libraries/mobiledatadownload", "//java/com/google/android/libraries/mobiledatadownload/internal/logging:LogUtil", + "//java/com/google/android/libraries/mobiledatadownload/populator/proto:metadata_java_proto_lite", "//proto:download_config_java_proto_lite", + "@com_google_errorprone_error_prone_annotations", "@com_google_guava_guava", ], ) @@ -131,6 +137,7 @@ android_library( "//java/com/google/android/libraries/mobiledatadownload/tracing", "//java/com/google/android/libraries/mobiledatadownload/tracing:concurrent", "//proto:download_config_java_proto_lite", + "//proto:log_enums_java_proto_lite", "@androidx_annotation_annotation", "@com_google_code_findbugs_jsr305", "@com_google_guava_guava", @@ -143,8 +150,6 @@ android_library( srcs = [ "ManifestFileMetadataStore.java", ], - # DO NOT ADD VISIBILITY: this isn't an open interface for clients to implement. - visibility = ["//visibility:private"], deps = [ "//java/com/google/android/libraries/mobiledatadownload/populator/proto:metadata_java_proto_lite", "@com_google_guava_guava", diff --git a/java/com/google/android/libraries/mobiledatadownload/populator/LocaleOverrider.java b/java/com/google/android/libraries/mobiledatadownload/populator/LocaleOverrider.java index 1985caa..1556c9a 100644 --- a/java/com/google/android/libraries/mobiledatadownload/populator/LocaleOverrider.java +++ b/java/com/google/android/libraries/mobiledatadownload/populator/LocaleOverrider.java @@ -28,6 +28,7 @@ import com.google.common.base.Supplier; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import com.google.mobiledatadownload.DownloadConfigProto.DataFileGroup; import com.google.mobiledatadownload.DownloadConfigProto.ManifestConfig; import java.util.ArrayList; @@ -68,6 +69,7 @@ public final class LocaleOverrider implements ManifestConfigOverrider { private Executor lightweightExecutor; /** only one of setLocaleSupplier or setLocaleFutureSupplier is required */ + @CanIgnoreReturnValue public Builder setLocaleSupplier(Supplier<Locale> localeSupplier) { this.localeSupplier = () -> Futures.immediateFuture(localeSupplier.get()); this.lightweightExecutor = @@ -75,6 +77,7 @@ public final class LocaleOverrider implements ManifestConfigOverrider { return this; } + @CanIgnoreReturnValue public Builder setLocaleFutureSupplier( Supplier<ListenableFuture<Locale>> localeSupplier, Executor lightweightExecutor) { this.localeSupplier = localeSupplier; @@ -87,6 +90,7 @@ public final class LocaleOverrider implements ManifestConfigOverrider { * the config. The set of Locale should be related to ONE {@code group_name} of {@link * DataFilegroup}. */ + @CanIgnoreReturnValue public Builder setMatchStrategy( BiFunction<Locale, Set<Locale>, Optional<Locale>> matchStrategy) { this.matchStrategy = matchStrategy; diff --git a/java/com/google/android/libraries/mobiledatadownload/populator/ManifestConfigFlagPopulator.java b/java/com/google/android/libraries/mobiledatadownload/populator/ManifestConfigFlagPopulator.java index 37ffbc7..9169d17 100644 --- a/java/com/google/android/libraries/mobiledatadownload/populator/ManifestConfigFlagPopulator.java +++ b/java/com/google/android/libraries/mobiledatadownload/populator/ManifestConfigFlagPopulator.java @@ -23,8 +23,10 @@ import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; import com.google.common.base.Joiner; import com.google.common.base.Optional; import com.google.common.base.Supplier; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.common.util.concurrent.ListenableFuture; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import com.google.mobiledatadownload.DownloadConfigProto.DataFileGroup; import com.google.mobiledatadownload.DownloadConfigProto.ManifestConfig; @@ -56,6 +58,7 @@ public final class ManifestConfigFlagPopulator implements FileGroupPopulator { private Optional<ManifestConfigOverrider> overriderOptional = Optional.absent(); /** Set the ManifestConfig supplier. */ + @CanIgnoreReturnValue public Builder setManifestConfigSupplier(Supplier<ManifestConfig> manifestConfigSupplier) { this.manifestConfigSupplier = manifestConfigSupplier; return this; @@ -65,6 +68,7 @@ public final class ManifestConfigFlagPopulator implements FileGroupPopulator { * Sets the optional Overrider that takes a {@link ManifestConfig} and returns a list of {@link * DataFileGroup} which will be added to MDD. The Overrider will enable the on device targeting. */ + @CanIgnoreReturnValue public Builder setOverriderOptional(Optional<ManifestConfigOverrider> overriderOptional) { this.overriderOptional = overriderOptional; return this; @@ -104,6 +108,10 @@ public final class ManifestConfigFlagPopulator implements FileGroupPopulator { LogUtil.d("%s: Add groups [%s] from ManifestConfig to MDD.", TAG, groups); return ManifestConfigHelper.refreshFromManifestConfig( - mobileDataDownload, manifestConfigSupplier.get(), overriderOptional); + mobileDataDownload, + manifestConfigSupplier.get(), + overriderOptional, + /* accounts= */ ImmutableList.of(), + /* addGroupsWithVariantId= */ false); } } diff --git a/java/com/google/android/libraries/mobiledatadownload/populator/ManifestConfigHelper.java b/java/com/google/android/libraries/mobiledatadownload/populator/ManifestConfigHelper.java index d2c8722..eb74c8a 100644 --- a/java/com/google/android/libraries/mobiledatadownload/populator/ManifestConfigHelper.java +++ b/java/com/google/android/libraries/mobiledatadownload/populator/ManifestConfigHelper.java @@ -15,9 +15,11 @@ */ package com.google.android.libraries.mobiledatadownload.populator; -import android.util.Log; +import android.accounts.Account; import com.google.android.libraries.mobiledatadownload.AddFileGroupRequest; +import com.google.android.libraries.mobiledatadownload.AggregateException; import com.google.android.libraries.mobiledatadownload.MobileDataDownload; +import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFluentFuture; import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; import com.google.common.base.Optional; @@ -40,69 +42,150 @@ public final class ManifestConfigHelper { private final MobileDataDownload mobileDataDownload; private final Optional<ManifestConfigOverrider> overriderOptional; + private final List<Account> accounts; + private final boolean addGroupsWithVariantId; /** Creates a new helper for converting manifest configs into data file groups. */ ManifestConfigHelper( - MobileDataDownload mobileDataDownload, Optional<ManifestConfigOverrider> overriderOptional) { + MobileDataDownload mobileDataDownload, + Optional<ManifestConfigOverrider> overriderOptional, + List<Account> accounts, + boolean addGroupsWithVariantId) { this.mobileDataDownload = mobileDataDownload; this.overriderOptional = overriderOptional; + this.accounts = accounts; + this.addGroupsWithVariantId = addGroupsWithVariantId; } /** * Reads file groups from {@link ManifestConfig} and adds to MDD after applying the {@link - * ManifestConfigOverrider} if it's present. This static method is shared with {@link - * ManifestFileGroupPopulator}. + * ManifestConfigOverrider} if it's present. + * + * <p>This static method encapsulates shared logic between a few populators: + * + * <ul> + * <li>{@link ManifestFileGroupPopulator} + * <li>{@link ManifestConfigFlagPopulator} + * <li>{@link LocalManifestFileGroupPopulator} + * <li>{@link EmbeddedAssetManifestPopulator} + * </ul> * - * @param mobileDataDownload The MDD instance. - * @param manifestConfig The proto that contains configs for file groups and modifiers. + * @param mobileDataDownload The MDD instance + * @param manifestConfig The proto that contains configs for file groups and modifiers * @param overriderOptional An optional overrider that takes manifest config and returns a list of - * file groups to be added to MDD. + * file groups to be added ot MDD + * @param accounts A list of accounts that the parsed file groups should be associated with + * @param addGroupsWithVariantId whether variantId should be included when adding the parsed file + * groups */ static ListenableFuture<Void> refreshFromManifestConfig( MobileDataDownload mobileDataDownload, ManifestConfig manifestConfig, - Optional<ManifestConfigOverrider> overriderOptional) { - ManifestConfigHelper helper = new ManifestConfigHelper(mobileDataDownload, overriderOptional); + Optional<ManifestConfigOverrider> overriderOptional, + List<Account> accounts, + boolean addGroupsWithVariantId) { + ManifestConfigHelper helper = + new ManifestConfigHelper( + mobileDataDownload, overriderOptional, accounts, addGroupsWithVariantId); return PropagatedFluentFuture.from(helper.applyOverrider(manifestConfig)) - .transformAsync(helper::addAllFileGroups, MoreExecutors.directExecutor()); + .transformAsync(helper::addAllFileGroups, MoreExecutors.directExecutor()) + .catchingAsync( + AggregateException.class, + ex -> Futures.immediateVoidFuture(), + MoreExecutors.directExecutor()); } /** Adds the specified list of file groups to MDD. */ ListenableFuture<Void> addAllFileGroups(List<DataFileGroup> fileGroups) { List<ListenableFuture<Boolean>> addFileGroupFutures = new ArrayList<>(); + Optional<String> variantId = Optional.absent(); for (DataFileGroup dataFileGroup : fileGroups) { if (dataFileGroup == null || dataFileGroup.getGroupName().isEmpty()) { continue; } - ListenableFuture<Boolean> addFileGroupFuture = - mobileDataDownload.addFileGroup( - AddFileGroupRequest.newBuilder().setDataFileGroup(dataFileGroup).build()); + // Include variantId if variant is present and helper is configured to do so + if (addGroupsWithVariantId && !dataFileGroup.getVariantId().isEmpty()) { + variantId = Optional.of(dataFileGroup.getVariantId()); + } - PropagatedFutures.addCallback( - addFileGroupFuture, - new FutureCallback<Boolean>() { - @Override - public void onSuccess(Boolean result) { - String groupName = dataFileGroup.getGroupName(); - if (result.booleanValue()) { - Log.d(TAG, "Added file groups " + groupName); - } else { - Log.d(TAG, "Failed to add file group " + groupName); - } - } + AddFileGroupRequest.Builder addFileGroupRequestBuilder = + AddFileGroupRequest.newBuilder() + .setDataFileGroup(dataFileGroup) + .setVariantIdOptional(variantId); - @Override - public void onFailure(Throwable t) { - Log.e(TAG, "Failed to add file group", t); - } - }, - MoreExecutors.directExecutor()); + // Add once without any account + ListenableFuture<Boolean> addFileGroupFuture = + mobileDataDownload.addFileGroup(addFileGroupRequestBuilder.build()); + attachLoggingCallback( + addFileGroupFuture, + dataFileGroup.getGroupName(), + /* account= */ Optional.absent(), + variantId); addFileGroupFutures.add(addFileGroupFuture); + + // Add for each account + for (Account account : accounts) { + ListenableFuture<Boolean> addFileGroupFutureWithAccount = + mobileDataDownload.addFileGroup( + addFileGroupRequestBuilder.setAccountOptional(Optional.of(account)).build()); + attachLoggingCallback( + addFileGroupFutureWithAccount, + dataFileGroup.getGroupName(), + Optional.of(account), + variantId); + addFileGroupFutures.add(addFileGroupFutureWithAccount); + } } return PropagatedFutures.whenAllComplete(addFileGroupFutures) - .call(() -> null, MoreExecutors.directExecutor()); + .call( + () -> { + AggregateException.throwIfFailed(addFileGroupFutures, "Failed to add file groups"); + return null; + }, + MoreExecutors.directExecutor()); + } + + private void attachLoggingCallback( + ListenableFuture<Boolean> addFileGroupFuture, + String groupName, + Optional<Account> account, + Optional<String> variant) { + PropagatedFutures.addCallback( + addFileGroupFuture, + new FutureCallback<Boolean>() { + @Override + public void onSuccess(Boolean result) { + if (result.booleanValue()) { + LogUtil.d( + "%s: Added file group %s with account: %s, variant: %s", + TAG, + groupName, + String.valueOf(account.orNull()), + String.valueOf(variant.orNull())); + } else { + LogUtil.d( + "%s: Failed to add file group %s with account: %s, variant: %s", + TAG, + groupName, + String.valueOf(account.orNull()), + String.valueOf(variant.orNull())); + } + } + + @Override + public void onFailure(Throwable t) { + LogUtil.e( + t, + "%s: Failed to add file group %s with account: %s, variant: %s", + TAG, + groupName, + String.valueOf(account.orNull()), + String.valueOf(variant.orNull())); + } + }, + MoreExecutors.directExecutor()); } /** Applies the overrider to the manifest config to generate a list of file groups for adding. */ diff --git a/java/com/google/android/libraries/mobiledatadownload/populator/ManifestFileGroupPopulator.java b/java/com/google/android/libraries/mobiledatadownload/populator/ManifestFileGroupPopulator.java index 27dc5f3..b8d3551 100644 --- a/java/com/google/android/libraries/mobiledatadownload/populator/ManifestFileGroupPopulator.java +++ b/java/com/google/android/libraries/mobiledatadownload/populator/ManifestFileGroupPopulator.java @@ -22,8 +22,6 @@ import static com.google.common.util.concurrent.Futures.immediateVoidFuture; import android.content.Context; import android.net.Uri; import androidx.annotation.VisibleForTesting; -import com.google.mobiledatadownload.populator.MetadataProto.ManifestFileBookkeeping; -import com.google.mobiledatadownload.populator.MetadataProto.ManifestFileBookkeeping.Status; import com.google.android.libraries.mobiledatadownload.AggregateException; import com.google.android.libraries.mobiledatadownload.DownloadException; import com.google.android.libraries.mobiledatadownload.DownloadException.DownloadResultCode; @@ -40,6 +38,7 @@ import com.google.android.libraries.mobiledatadownload.file.SynchronousFileStora import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; import com.google.android.libraries.mobiledatadownload.internal.util.DirectoryUtil; import com.google.android.libraries.mobiledatadownload.logger.FileGroupPopulatorLogger; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedExecutionSequencer; import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFluentFuture; import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; import com.google.common.base.Optional; @@ -49,9 +48,13 @@ import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.ExecutionSequencer; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.ListenableFuture; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import com.google.mobiledatadownload.DownloadConfigProto.DataFileGroup; import com.google.mobiledatadownload.DownloadConfigProto.ManifestConfig; import com.google.mobiledatadownload.DownloadConfigProto.ManifestFileFlag; +import com.google.mobiledatadownload.LogEnumsProto.MddDownloadResult; +import com.google.mobiledatadownload.populator.MetadataProto.ManifestFileBookkeeping; +import com.google.mobiledatadownload.populator.MetadataProto.ManifestFileBookkeeping.Status; import java.io.IOException; import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicReference; @@ -91,8 +94,7 @@ import javax.inject.Singleton; * hosting service needs to support ETag (e.g. Lorry), otherwise the behavior will be unexpected. * Talk to <internal>@ if you are not sure if the hosting service supports ETag. * - * <p>Note that {@link SynchronousFileStorage} and {@link ProtoDataStoreFactory} passed to builder - * must be @Singleton. + * <p> * * <p>This class is @Singleton, because it provides the guarantee that all the operations are * serialized correctly by {@link ExecutionSequencer}. @@ -118,6 +120,7 @@ public final class ManifestFileGroupPopulator implements FileGroupPopulator { public static final class Builder { private boolean allowsInsecureHttp = false; private boolean dedupDownloadWithEtag = true; + private boolean forceManifestSyncs = true; private Context context; private Supplier<ManifestFileFlag> manifestFileFlagSupplier; private Supplier<FileDownloader> fileDownloader; @@ -137,6 +140,7 @@ public final class ManifestFileGroupPopulator implements FileGroupPopulator { * * <p>For testing only. */ + @CanIgnoreReturnValue @VisibleForTesting Builder setAllowsInsecureHttp(boolean allowsInsecureHttp) { this.allowsInsecureHttp = allowsInsecureHttp; @@ -147,18 +151,41 @@ public final class ManifestFileGroupPopulator implements FileGroupPopulator { * By default, an HTTP HEAD request is made to avoid duplicate downloads of the manifest file. * Setting this to false disables that behavior. */ + @CanIgnoreReturnValue public Builder setDedupDownloadWithEtag(boolean dedup) { this.dedupDownloadWithEtag = dedup; return this; } + /** + * Force manifest syncs when {@link setDedupDownloadWithEtag} is set to false. + * + * <p>When NOT deduping with ETag, it's possible that a downloaded version of a manifest may + * override a potentially newer version of a manifest, preventing new file groups from being + * synced. + * + * <p>This flag controls whether or not the fix (always downloading the manifest) should be + * used. + * + * <p>NOTE: By default, this flag will be set to true -- if clients would rather have a + * controlled rollout of this behavior change, they should include this option in their builder + * and connect this to an experimental rollout system. See b/243926815 for more details. + */ + @CanIgnoreReturnValue + public Builder setForceManifestSyncsWithoutETag(boolean forceManifestSyncs) { + this.forceManifestSyncs = forceManifestSyncs; + return this; + } + /** Sets the context. */ + @CanIgnoreReturnValue public Builder setContext(Context context) { this.context = context.getApplicationContext(); return this; } /** Sets the manifest file flag. */ + @CanIgnoreReturnValue public Builder setManifestFileFlagSupplier( Supplier<ManifestFileFlag> manifestFileFlagSupplier) { this.manifestFileFlagSupplier = manifestFileFlagSupplier; @@ -166,53 +193,66 @@ public final class ManifestFileGroupPopulator implements FileGroupPopulator { } /** Sets the file downloader. */ + @CanIgnoreReturnValue public Builder setFileDownloader(Supplier<FileDownloader> fileDownloader) { this.fileDownloader = fileDownloader; return this; } /** Sets the manifest config parser that takes file uri and returns {@link ManifestConfig}. */ + @CanIgnoreReturnValue public Builder setManifestConfigParser(ManifestConfigParser manifestConfigParser) { this.manifestConfigParser = manifestConfigParser; return this; } /** Sets the mobstore file storage. Mobstore file storage must be singleton. */ + @CanIgnoreReturnValue public Builder setFileStorage(SynchronousFileStorage fileStorage) { this.fileStorage = fileStorage; return this; } /** Sets the background executor that executes populator's tasks sequentially. */ + @CanIgnoreReturnValue public Builder setBackgroundExecutor(Executor backgroundExecutor) { this.backgroundExecutor = backgroundExecutor; return this; } - /** Sets the ManifestFileMetadataStore. */ + /** + * Sets the ManifestFileMetadataStore. + * + * <p> + */ + @CanIgnoreReturnValue public Builder setMetadataStore(ManifestFileMetadataStore manifestFileMetadataStore) { this.manifestFileMetadataStore = manifestFileMetadataStore; return this; } /** Sets the MDD logger. */ + @CanIgnoreReturnValue public Builder setLogger(Logger logger) { this.logger = logger; return this; } /** Sets the optional manifest config overrider. */ + @CanIgnoreReturnValue public Builder setOverriderOptional(Optional<ManifestConfigOverrider> overriderOptional) { this.overriderOptional = overriderOptional; return this; } /** Sets the optional instance ID. */ + @CanIgnoreReturnValue public Builder setInstanceIdOptional(Optional<String> instanceIdOptional) { this.instanceIdOptional = instanceIdOptional; return this; } + @CanIgnoreReturnValue public Builder setFlags(Flags flags) { this.flags = flags; return this; @@ -246,6 +286,7 @@ public final class ManifestFileGroupPopulator implements FileGroupPopulator { private final boolean allowsInsecureHttp; private final boolean dedupDownloadWithEtag; + private final boolean forceManifestSyncs; private final Context context; private final Uri manifestDirectoryUri; private final Supplier<ManifestFileFlag> manifestFileFlagSupplier; @@ -257,9 +298,11 @@ public final class ManifestFileGroupPopulator implements FileGroupPopulator { private final ManifestFileMetadataStore manifestFileMetadataStore; private final FileGroupPopulatorLogger eventLogger; // We use futureSerializer for synchronization. - private final ExecutionSequencer futureSerializer = ExecutionSequencer.create(); + private final PropagatedExecutionSequencer futureSerializer = + PropagatedExecutionSequencer.create(); private final EnabledSupplier enabledSupplier; + /** Returns a Builder for {@link ManifestFileGroupPopulator}. */ public static Builder builder() { return new Builder(); @@ -268,6 +311,7 @@ public final class ManifestFileGroupPopulator implements FileGroupPopulator { private ManifestFileGroupPopulator(Builder builder) { this.allowsInsecureHttp = builder.allowsInsecureHttp; this.dedupDownloadWithEtag = builder.dedupDownloadWithEtag; + this.forceManifestSyncs = builder.forceManifestSyncs; this.context = builder.context; this.manifestDirectoryUri = DirectoryUtil.getManifestDirectory(builder.context, builder.instanceIdOptional); @@ -295,7 +339,8 @@ public final class ManifestFileGroupPopulator implements FileGroupPopulator { if (manifestFileFlag == null || manifestFileFlag.equals(ManifestFileFlag.getDefaultInstance())) { LogUtil.w("%s: The ManifestFileFlag is empty.", TAG); - logRefreshResult(0, ManifestFileFlag.getDefaultInstance()); + logRefreshResult( + MddDownloadResult.Code.SUCCESS, ManifestFileFlag.getDefaultInstance()); return immediateVoidFuture(); } @@ -312,7 +357,9 @@ public final class ManifestFileGroupPopulator implements FileGroupPopulator { } if (!validate(manifestFileFlag)) { - logRefreshResult(0, manifestFileFlag); + logRefreshResult( + MddDownloadResult.Code.MANIFEST_FILE_GROUP_POPULATOR_INVALID_FLAG_ERROR, + manifestFileFlag); LogUtil.e("%s: Invalid manifest config from manifest flag.", TAG); return immediateFailedFuture(new IllegalArgumentException("Invalid manifest flag.")); } @@ -424,7 +471,7 @@ public final class ManifestFileGroupPopulator implements FileGroupPopulator { manifestFileFlag); // If there is any failure, it should have been thrown already. Therefore, we log refresh // success here. - logRefreshResult(0, manifestFileFlag); + logRefreshResult(MddDownloadResult.Code.SUCCESS, manifestFileFlag); return immediateVoidFuture(); }, backgroundExecutor); @@ -452,7 +499,11 @@ public final class ManifestFileGroupPopulator implements FileGroupPopulator { .transformAsync( (final ManifestConfig manifestConfig) -> ManifestConfigHelper.refreshFromManifestConfig( - mobileDataDownload, manifestConfig, overriderOptional), + mobileDataDownload, + manifestConfig, + overriderOptional, + /* accounts= */ ImmutableList.of(), + /* addGroupsWithVariantId= */ false), backgroundExecutor) .transformAsync( voidArg -> { @@ -503,17 +554,7 @@ public final class ManifestFileGroupPopulator implements FileGroupPopulator { LogUtil.d("%s: Prepare for downloading manifest file.", TAG); if (!dedupDownloadWithEtag) { - LogUtil.d( - "%s: Not relying on etag to dedup manifest -- forcing re-download; urlToDownload = %s;" - + " manifestFileUri = %s", - TAG, urlToDownload, manifestFileUri); - try { - deleteManifestFileChecked(manifestFileUri); - } catch (DownloadException e) { - return immediateFailedFuture(e); - } - bookkeepingRef.set(createDefaultManifestFileBookkeeping(urlToDownload)); - return immediateVoidFuture(); + return handleManifestDedupWithoutETag(urlToDownload, manifestFileUri, bookkeepingRef); } ManifestFileBookkeeping bookkeeping = bookkeepingRef.get(); @@ -556,6 +597,41 @@ public final class ManifestFileGroupPopulator implements FileGroupPopulator { backgroundExecutor); } + /** + * Handle Manifest Bookkeeping when ETag check should be bypassed. + * + * <p>If forced syncs are enabled, the existing manifest file will be deleted and the bookkeeping + * reference will be updated to a default value. This forces the manifest to be redownloaded. + * + * <p>If forced syncs are disabled, this is a no-op and existing bookkeeping will be used. This + * reuses a downloaded manifest if one exists, or continues a download of a pending manifest. + */ + private ListenableFuture<Void> handleManifestDedupWithoutETag( + String urlToDownload, + Uri manifestFileUri, + AtomicReference<ManifestFileBookkeeping> bookkeepingRef) { + LogUtil.d( + "%s: Not relying on etag to dedup manifest -- checking if manifest should be force" + + " downloaded", + TAG); + if (forceManifestSyncs) { + LogUtil.d( + "%s: forcing re-download; urlToDownload = %s;" + " manifestFileUri = %s", + TAG, urlToDownload, manifestFileUri); + try { + deleteManifestFileChecked(manifestFileUri); + } catch (DownloadException e) { + return immediateFailedFuture(e); + } + bookkeepingRef.set(createDefaultManifestFileBookkeeping(urlToDownload)); + } else { + LogUtil.d( + "%s: not forcing re-download; urlToDownload = %s;" + " manifestFileUri =%s", + TAG, urlToDownload, manifestFileUri); + } + return immediateVoidFuture(); + } + private ListenableFuture<Void> checkForContentChangeAfterDownload( String urlToDownload, Uri manifestFileUri, @@ -564,9 +640,9 @@ public final class ManifestFileGroupPopulator implements FileGroupPopulator { if (!dedupDownloadWithEtag) { LogUtil.d( - "%s: Not relying on etag to dedup manifest, so the downloaded manifest is" - + " assumed to be the latest; urlToDownload = %s, manifestFileUri = %s", - TAG, urlToDownload, manifestFileUri); + "%s: Not relying on etag to dedup manifest, so the downloaded manifest is" + + " assumed to be the latest; urlToDownload = %s, manifestFileUri = %s", + TAG, urlToDownload, manifestFileUri); return immediateVoidFuture(); } @@ -646,15 +722,17 @@ public final class ManifestFileGroupPopulator implements FileGroupPopulator { } } + // incompatible argument for parameter code of logManifestFileGroupPopulatorRefreshResult. + @SuppressWarnings("nullness:argument.type.incompatible") private void logRefreshResult(DownloadException e, ManifestFileFlag manifestFileFlag) { eventLogger.logManifestFileGroupPopulatorRefreshResult( - 0, + MddDownloadResult.Code.forNumber(e.getDownloadResultCode().getCode()), manifestFileFlag.getManifestId(), context.getPackageName(), manifestFileFlag.getManifestFileUrl()); } - private void logRefreshResult(int code, ManifestFileFlag manifestFileFlag) { + private void logRefreshResult(MddDownloadResult.Code code, ManifestFileFlag manifestFileFlag) { eventLogger.logManifestFileGroupPopulatorRefreshResult( code, manifestFileFlag.getManifestId(), @@ -695,7 +773,7 @@ public final class ManifestFileGroupPopulator implements FileGroupPopulator { private static ManifestFileBookkeeping createDefaultManifestFileBookkeeping( String manifestFileUrl) { return createManifestFileBookkeeping( - manifestFileUrl, Status.PENDING, /* eTagOptional = */ Optional.absent()); + manifestFileUrl, Status.PENDING, /* eTagOptional= */ Optional.absent()); } private static ManifestFileBookkeeping createManifestFileBookkeeping( diff --git a/java/com/google/android/libraries/mobiledatadownload/populator/ManifestFileMetadataStore.java b/java/com/google/android/libraries/mobiledatadownload/populator/ManifestFileMetadataStore.java index 4d80080..874571b 100644 --- a/java/com/google/android/libraries/mobiledatadownload/populator/ManifestFileMetadataStore.java +++ b/java/com/google/android/libraries/mobiledatadownload/populator/ManifestFileMetadataStore.java @@ -15,9 +15,9 @@ */ package com.google.android.libraries.mobiledatadownload.populator; -import com.google.mobiledatadownload.populator.MetadataProto.ManifestFileBookkeeping; import com.google.common.base.Optional; import com.google.common.util.concurrent.ListenableFuture; +import com.google.mobiledatadownload.populator.MetadataProto.ManifestFileBookkeeping; /** Storage mechanism for ManifestFileBookkeeping. */ interface ManifestFileMetadataStore { diff --git a/java/com/google/android/libraries/mobiledatadownload/populator/SharedPreferencesManifestFileMetadata.java b/java/com/google/android/libraries/mobiledatadownload/populator/SharedPreferencesManifestFileMetadata.java index 8656e91..3fb1db2 100644 --- a/java/com/google/android/libraries/mobiledatadownload/populator/SharedPreferencesManifestFileMetadata.java +++ b/java/com/google/android/libraries/mobiledatadownload/populator/SharedPreferencesManifestFileMetadata.java @@ -17,13 +17,13 @@ package com.google.android.libraries.mobiledatadownload.populator; import android.content.Context; import android.content.SharedPreferences; -import com.google.mobiledatadownload.populator.MetadataProto.ManifestFileBookkeeping; import com.google.android.libraries.mobiledatadownload.internal.util.SharedPreferencesUtil; import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; import com.google.common.base.Optional; import com.google.common.base.Supplier; import com.google.common.base.Suppliers; import com.google.common.util.concurrent.ListenableFuture; +import com.google.mobiledatadownload.populator.MetadataProto.ManifestFileBookkeeping; import java.io.IOException; import java.util.concurrent.Executor; diff --git a/java/com/google/android/libraries/mobiledatadownload/populator/SingleDataFileGroupPopulator.java b/java/com/google/android/libraries/mobiledatadownload/populator/SingleDataFileGroupPopulator.java index 10bd8c8..d513168 100644 --- a/java/com/google/android/libraries/mobiledatadownload/populator/SingleDataFileGroupPopulator.java +++ b/java/com/google/android/libraries/mobiledatadownload/populator/SingleDataFileGroupPopulator.java @@ -15,17 +15,20 @@ */ package com.google.android.libraries.mobiledatadownload.populator; +import static com.google.common.util.concurrent.Futures.immediateFuture; + import android.util.Log; import com.google.android.libraries.mobiledatadownload.AddFileGroupRequest; import com.google.android.libraries.mobiledatadownload.FileGroupPopulator; import com.google.android.libraries.mobiledatadownload.MobileDataDownload; import com.google.android.libraries.mobiledatadownload.internal.logging.LogUtil; +import com.google.android.libraries.mobiledatadownload.tracing.PropagatedFutures; import com.google.common.base.Optional; import com.google.common.base.Supplier; import com.google.common.util.concurrent.FutureCallback; -import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import com.google.mobiledatadownload.DownloadConfigProto.DataFileGroup; /** @@ -46,6 +49,7 @@ public final class SingleDataFileGroupPopulator implements FileGroupPopulator { private Supplier<DataFileGroup> dataFileGroupSupplier; private Optional<DataFileGroupOverrider> overriderOptional = Optional.absent(); + @CanIgnoreReturnValue public Builder setDataFileGroupSupplier(Supplier<DataFileGroup> dataFileGroupSupplier) { this.dataFileGroupSupplier = dataFileGroupSupplier; return this; @@ -56,6 +60,7 @@ public final class SingleDataFileGroupPopulator implements FileGroupPopulator { * {@link DataFileGroup} after being overridden. If the overrider returns a null data file * group, nothing will be populated. */ + @CanIgnoreReturnValue public Builder setOverriderOptional(Optional<DataFileGroupOverrider> overriderOptional) { this.overriderOptional = overriderOptional; return this; @@ -86,17 +91,17 @@ public final class SingleDataFileGroupPopulator implements FileGroupPopulator { // Override data file group if the overrider is present. If the overrider returns an absent // data file group, nothing will be populated. ListenableFuture<Optional<DataFileGroup>> dataFileGroupOptionalFuture = - Futures.immediateFuture(Optional.absent()); + immediateFuture(Optional.absent()); if (dataFileGroupSupplier.get() != null && !dataFileGroupSupplier.get().getGroupName().isEmpty()) { dataFileGroupOptionalFuture = overriderOptional.isPresent() ? overriderOptional.get().override(dataFileGroupSupplier.get()) - : Futures.immediateFuture(Optional.of(dataFileGroupSupplier.get())); + : immediateFuture(Optional.of(dataFileGroupSupplier.get())); } ListenableFuture<Boolean> addFileGroupFuture = - Futures.transformAsync( + PropagatedFutures.transformAsync( dataFileGroupOptionalFuture, dataFileGroupOptional -> { if (dataFileGroupOptional.isPresent() @@ -107,11 +112,11 @@ public final class SingleDataFileGroupPopulator implements FileGroupPopulator { .build()); } LogUtil.d("%s: Not adding file group because of overrider.", TAG); - return Futures.immediateFuture(false); + return immediateFuture(false); }, MoreExecutors.directExecutor()); - Futures.addCallback( + PropagatedFutures.addCallback( addFileGroupFuture, new FutureCallback<Boolean>() { @Override @@ -131,7 +136,7 @@ public final class SingleDataFileGroupPopulator implements FileGroupPopulator { }, MoreExecutors.directExecutor()); - return Futures.whenAllComplete(addFileGroupFuture) + return PropagatedFutures.whenAllComplete(addFileGroupFuture) .call(() -> null, MoreExecutors.directExecutor()); } } diff --git a/java/com/google/android/libraries/mobiledatadownload/populator/proto/BUILD b/java/com/google/android/libraries/mobiledatadownload/populator/proto/BUILD index 91be276..637afee 100644 --- a/java/com/google/android/libraries/mobiledatadownload/populator/proto/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/populator/proto/BUILD @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. package( + default_applicable_licenses = ["//:license"], default_visibility = [ "//:__subpackages__", ], diff --git a/java/com/google/android/libraries/mobiledatadownload/testing/BUILD b/java/com/google/android/libraries/mobiledatadownload/testing/BUILD new file mode 100644 index 0000000..80f6902 --- /dev/null +++ b/java/com/google/android/libraries/mobiledatadownload/testing/BUILD @@ -0,0 +1,20 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +package( + default_applicable_licenses = ["//:license"], + default_visibility = [ + "//visibility:public", + ], + licenses = ["notice"], +) diff --git a/java/com/google/android/libraries/mobiledatadownload/tracing/BUILD b/java/com/google/android/libraries/mobiledatadownload/tracing/BUILD index 18ae88e..8629dc4 100644 --- a/java/com/google/android/libraries/mobiledatadownload/tracing/BUILD +++ b/java/com/google/android/libraries/mobiledatadownload/tracing/BUILD @@ -14,6 +14,7 @@ load("@build_bazel_rules_android//android:rules.bzl", "android_library") package( + default_applicable_licenses = ["//:license"], default_visibility = ["//:__subpackages__"], licenses = ["notice"], ) @@ -32,6 +33,7 @@ android_library( android_library( name = "concurrent", srcs = [ + "PropagatedExecutionSequencer.java", "PropagatedFluentFuture.java", "PropagatedFluentFutures.java", "PropagatedFutures.java", diff --git a/java/com/google/android/libraries/mobiledatadownload/tracing/PropagatedExecutionSequencer.java b/java/com/google/android/libraries/mobiledatadownload/tracing/PropagatedExecutionSequencer.java index c2bfec5..0d2073f 100644 --- a/java/com/google/android/libraries/mobiledatadownload/tracing/PropagatedExecutionSequencer.java +++ b/java/com/google/android/libraries/mobiledatadownload/tracing/PropagatedExecutionSequencer.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2022 The Android Open Source Project + * Copyright 2022 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,24 +25,24 @@ import org.checkerframework.checker.nullness.qual.Nullable; /** Wrapper around {@link ExecutionSequencer} with trace propagation. */ public final class PropagatedExecutionSequencer { - private final ExecutionSequencer executionSequencer = ExecutionSequencer.create(); + private final ExecutionSequencer executionSequencer = ExecutionSequencer.create(); - private PropagatedExecutionSequencer() {} + private PropagatedExecutionSequencer() {} - /** Creates a new instance. */ - public static PropagatedExecutionSequencer create() { - return new PropagatedExecutionSequencer(); - } + /** Creates a new instance. */ + public static PropagatedExecutionSequencer create() { + return new PropagatedExecutionSequencer(); + } - /** See {@link ExecutionSequencer#submit(Callable, Executor)}. */ - public <T extends @Nullable Object> ListenableFuture<T> submit( - Callable<T> callable, Executor executor) { - return executionSequencer.submit(callable, executor); - } + /** See {@link ExecutionSequencer#submit(Callable, Executor)}. */ + public <T extends @Nullable Object> ListenableFuture<T> submit( + Callable<T> callable, Executor executor) { + return executionSequencer.submit(callable, executor); + } - /** See {@link ExecutionSequencer#submitAsync(AsyncCallable, Executor)}. */ - public <T extends @Nullable Object> ListenableFuture<T> submitAsync( - AsyncCallable<T> callable, Executor executor) { - return executionSequencer.submitAsync(callable, executor); - } -}
\ No newline at end of file + /** See {@link ExecutionSequencer#submitAsync(AsyncCallable, Executor)}. */ + public <T extends @Nullable Object> ListenableFuture<T> submitAsync( + AsyncCallable<T> callable, Executor executor) { + return executionSequencer.submitAsync(callable, executor); + } +} diff --git a/java/com/google/android/libraries/mobiledatadownload/tracing/TracePropagation.java b/java/com/google/android/libraries/mobiledatadownload/tracing/TracePropagation.java index 7c1ee13..d2c9f79 100644 --- a/java/com/google/android/libraries/mobiledatadownload/tracing/TracePropagation.java +++ b/java/com/google/android/libraries/mobiledatadownload/tracing/TracePropagation.java @@ -61,5 +61,10 @@ public final class TracePropagation { return closingFunction; } + @CheckReturnValue + public static Runnable propagateRunnable(Runnable runnable) { + return runnable; + } + private TracePropagation() {} } |