// Copyright 2022 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // The main purpose of mirroring external protos here is to separate API and // storage protos. The same tag numbers are used for making migration work. syntax = "proto2"; package mdi.download.internal; import "google/protobuf/any.proto"; import "google/protobuf/timestamp.proto"; import "transform.proto"; option java_package = "com.google.mobiledatadownload.internal"; option java_outer_classname = "MetadataProto"; // Mirrors mdi.download.ExtraHttpHeader // // HTTP headers are described in https://tools.ietf.org/html/rfc7230#section-3.2 // as key:value, where the value may have a whitespace on each end. message ExtraHttpHeader { optional string key = 1; optional string value = 2; } // This proto is used to store file group metadata on disk for internal use. It // consists of all fields mirrored from DataFileGroup and an extra field for // bookkeeping. // // The tag number of extra fields should start from 1000 to reserve room for // growing DataFileGroup. // // Next id: 1001 message DataFileGroupInternal { // Extra information that is kept on disk. // // The extension was originally introduced in cl/248813966. We are migrating // away from the extension. However, we still need to read from fields in the // extension. Reuse the same tag number as the extension number to read from // the extension. optional DataFileGroupBookkeeping bookkeeping = 248813966; // Unique name to identify the group. It should be unique per owner package. // In GMSCore, use the module name as the prefix of the group name. // // Ex: A group name in mdisync module could be named: mdisync-profile-photos. // // This shouldn't ideally be something like "config", and // instead should better define the feature it will be used for. // // Ex: "icing-language-detection-model", "smart-action-detection-model" optional string group_name = 1; // The name of the package that owns this group. If this field is left empty, // the owner is assumed to be the package name of the host app. // // The files will only be downloaded onto the device if the owner package is // present on the device. // // Ex: "com.google.android.gms", "com.google.android.apps.bugle" optional string owner_package = 6; // Client set version number used to identify the file group. // // Note that this does not uniquely identify the contents of the file group. // It simply reflects a snapshot of client config changes. // For example: say there's a file group 'language-detector-model' that // downloads a different file per user locale. // data_file_group { // file_group_name = 'language-detector-model' // file_group_version_number = 1 // file { // url = 'en-model' // } // } // data_file_group { // file_group_name = 'language-detector-model' // file_group_version_number = 1 // file { // url = 'es-model' // } // } // Note that even though the actual contents of the file group are different // for each locale, the version is the same because this config was pushed // at the same snapshot. // // Available GMS v18+. optional int32 file_group_version_number = 10; // DEPRECATED // MDD team recommends to use explicit properties instead. optional google.protobuf.Any custom_property = 20 [deprecated = true]; // Custom metadata attached to the file group. // // This allows clients to include specific metadata about the group for their // own processing purposes. The metadata will be stored with the group and // accessible when the file group is retrieved. // // This property should only be used if absolutely necessary. Please consult // with @ if you have questions about this property or a potential // use-case. optional google.protobuf.Any custom_metadata = 27; reserved 21; // Mirrors mdi.download.DataFileGroup.AllowedReaders enum AllowedReaders { ALL_GOOGLE_APPS = 0; ONLY_GOOGLE_PLAY_SERVICES = 1; ALL_APPS = 2; } // Defines who is allowed to read this file group. Currently the options are: // // ALL_GOOGLE_APPS: accessible to all Google 1p Apps. // ONLY_GOOGLE_PLAY_SERVICES: accessible to only GMS Core. // // If this field is not explicitly set it defaults to "ALL_GOOGLE_APPS". // // Available GMS v20+. optional AllowedReaders allowed_readers_enum = 12; // Length of time (in seconds) for which a file group version will live after // since a newer version became fully downloaded. Clients should set this time // to be more than the time in which they call MDD to refresh their data. // NOTE: MDD will delete the file group version within a day of this time. // Ex: 172800 // 2 Days optional int64 stale_lifetime_secs = 3; // The timestamp at which this filegroup should be deleted, even if it is // still active, specified in seconds since epoch. // NOTE: MDD will delete the file group version within a day of this time. optional int64 expiration_date_secs = 11; // Specify the conditions under which the file group should be downloaded. optional DownloadConditions download_conditions = 13; // Setting this flag to true will mean that the downloaded files will appear // to be in a directory by themselves. // The file name/file path of the exposed file will be the filename set in the // file.relative_file_path field, OR if that field is empty, the file name // from the file.url_to_download field. This enables downloaded files to refer // to each other by name. // It's invalid to set this flag to true if two files end up with the same // file path. // Valid on iOS, cMDD, and aMDD. // // NOTE: For aMDD, this feature is not available if Android Blob Sharing is // enabled or if using an API level below 21 (L). If either case is true, this // option will be ignored. optional bool preserve_filenames_and_isolate_files = 14; // List of files in the group. repeated DataFile file = 2; // Tag for the network traffic to download this file group. // Tag space is determined by the host app. // For Gmscore, the tag should come from: // optional int32 traffic_tag = 16; // Extra HTTP headers to apply when downloading all files in the group. repeated ExtraHttpHeader group_extra_http_headers = 17; reserved 19; // Unique identifier of a DataFileGroup config (i.e. a "snapshot") created // when using MDD Ingress API. optional int64 build_id = 23; // A fingerprint allowing clients to identify a DataFileGroup // config based on a given set of properties (i.e. a "partition" of // any file group properties). This can be used by clients as an exact match // for a class of DataFileGroups during targeting or as a compatibility check. optional string variant_id = 26; // The locales compatible with the file group. This can be different from the // device locale. // // Values in this list may be exact locales (e.g. "en-US") or language-only // ("en-*"). // Example 1: locale = ["en-US"]; // compatible with "en-US" only // Example 2: locale = ["en-US", "en-CA"]; // compatible with "en-US" or // // "en-CA" // Example 3: locale = ["en-*"]; // compatible with all "en" locales repeated string locale = 25; reserved 28; // If a group enables preserve_filenames_and_isolate_files // this property will contain the directory root of the isolated // structure. Specifically, the property will be a string created from the // group name and a hash of other identifying properties (account, variantid, // buildid). // // currently only used in aMDD. optional string isolated_directory_root = 1000; reserved 4, 5, 7, 8, 9, 15, 18, 22, 24; } // Mirrors mdi.download.DataFile // // A data file represents all the metadata to download the file and then // manage it on the device. // Next tag: 22 // // This should not contain any fields that are marked internal, as we compare // the protos directly to decide if it is a new version of the file. // LINT.IfChange(data_file) message DataFile { // A unique identifier of the file within the group, that can be used to // get this file from the group. // Ex: "language-detection-model" optional string file_id = 7; // Url from where the file is to be downloaded. // Ex: https://www.gstatic.com/group-name/model_1234.zip optional string url_to_download = 2; // Exact size of the file. This is used to check if there is space available // for the file before scheduling the download. // The byte_size is optional. If not set, MDD will try with best effort to get // the file size using the HTTP HEAD request. optional int32 byte_size = 4; // Enum for checksum types. // NOTE: do not add any new checksum type here, older MDD versions would break // otherwise. enum ChecksumType { // Default checksum is SHA1. DEFAULT = 0; // No checksum is provided. NONE = 1; reserved 2 /* SHA256 */; } optional ChecksumType checksum_type = 15; // SHA1 checksum to verify the file before it can be used. This is also used // to de-duplicate files between different groups. // For most files, this will be the checksum of the file being downloaded. // For files with download_transform, this should contain the transform of // the file after the transforms have been applied. // The checksum is optional. If not set, the checksum_type must be // ChecksumType.NONE. optional string checksum = 5; // The following are transforms to apply to the downloaded files. // Transforms are bi-directional and defined in terms of what they do on // write. Since these transforms are applied while reading, their // directionality is reversed. Eg, you'll see 'compress' to indicate that the // file should be decompressed. // These transforms are applied once by MDD after downloading the file. // Currently only compress is available. // Valid on Android. iOS support is tracked by b/118828045. optional mobstore.proto.Transforms download_transforms = 11; // If DataFile has download_transforms, this field must be provided with the // SHA1 checksum of the file before any transform are applied. The original // checksum would also be checked after the download_transforms are applied. optional string downloaded_file_checksum = 14; // Exact size of the downloaded file. If the DataFile has download transforms // like compress and zip, the downloaded file size would be different than // the final file size on disk. Client could use // this field to track the downloaded file size and calculate the download // progress percentage. This field is not used by MDD currently. optional int32 downloaded_file_byte_size = 16; // These transforms are evaluated by the caller on-the-fly when reading the // data with MobStore. Any transforms installed in the caller's MobStore // instance is available. // Valid on Android. iOS support is tracked by b/118759254. optional mobstore.proto.Transforms read_transforms = 12; // List of delta files that can be encoded and decoded with base files. // If the device has any base file, the delta file which is much // smaller will be downloaded instead of the full file. // For most clients, only one delta file should be enough. If specifying // multiple delta files, they should be in a sequence from the most recent // base file to the oldest. // This is currently only supported on Android. repeated DeltaFile delta_file = 13; enum AndroidSharingType { // The dataFile isn't available for sharing. UNSUPPORTED = 0; // If sharing with the Android Blob Sharing Service isn't available, fall // back to normal behavior, i.e. download locally. ANDROID_BLOB_WHEN_AVAILABLE = 1; } // Defines whether the file should be shared and how. // NOTE: currently this field is only used by aMDD and has no effect on iMDD. optional AndroidSharingType android_sharing_type = 17; // Enum for android sharing checksum types. enum AndroidSharingChecksumType { NOT_SET = 0; // If the file group should be shared through the Android Blob Sharing // Service, the checksum type must be set to SHA256. SHA2_256 = 1; } optional AndroidSharingChecksumType android_sharing_checksum_type = 18; // Checksum used to access files through the Android Blob Sharing Service. optional string android_sharing_checksum = 19; // Relative file path and file name to be preserved within the parent // directory when creating symlinks for the file groups that have // preserve_filenames_and_isolate_files set to true. // This filename should NOT start or end with a '/', and it can not contain // the substring '..'. // Working example: "subDir/FileName.txt". optional string relative_file_path = 20; // Custom metadata attached to the file. // // This allows clients to include specific metadata about the file for their // own processing purposes. The metadata will be stored with the file and // accessible when the file's file group is retrieved. // // This property should only be used if absolutely necessary. Please consult // with @ if you have questions about this property or a potential // use-case. optional google.protobuf.Any custom_metadata = 21; reserved 1, 3, 6, 8, 9; } // LINT.ThenChange( // , // ) // Mirrors mdi.download.DeltaFile // // A delta file represents all the metadata to download for a diff file encoded // based on a base file // LINT.IfChange(delta_file) message DeltaFile { // These fields all mirror the similarly-named fields in DataFile. optional string url_to_download = 1; optional int32 byte_size = 2; optional string checksum = 3; // Enum of all diff decoders supported enum DiffDecoder { // Default to have no diff decoder specified, will thrown unsupported // exception UNSPECIFIED = 0; // VcDIFF decoder // Generic Differencing and Compression Data Format // For more information, please refer to rfc3284 // The VcDiff decoder for GMS service: // VC_DIFF = 1; } // The diff decoder used to generate full file with delta and base file. // For MDD as a GMS service, a VcDiff decoder will be registered and injected // in by default. Using MDD as a library, clients need to register and inject // in a VcDiff decoder, otherwise, an exception will be thrown. optional DiffDecoder diff_decoder = 5; // The base file represents to a full file on device. It should contain the // bare minimum fields of a DataFile to identify a DataFile on device. optional BaseFile base_file = 6; reserved 4; } // LINT.ThenChange( // , // ) // Mirrors mdi.download.BaseFile message BaseFile { // SHA1 checksum of the base file to identify a file on device. It should // match the checksum field of the base file used to generate the delta file. optional string checksum = 1; } // Mirrors mdi.download.DownloadConditions // // Next id: 5 message DownloadConditions { // TODO(b/143548753): The first value in an enum must have a specific prefix. enum DeviceStoragePolicy { // MDD will block download of files in android low storage. Currently MDD // doesn't delete the files in case the device reaches low storage // after the file has been downloaded. BLOCK_DOWNLOAD_IN_LOW_STORAGE = 0; // Block download of files only under a lower threshold defined here // BLOCK_DOWNLOAD_LOWER_THRESHOLD = 1; // Set the storage threshold to an extremely low value when downloading. // IMPORTANT: if the download make the device runs out of disk, this could // render the device unusable. // This should only be used for critical use cases such as privacy // violations. Emergency fix should not belong to this category. Please // talks to @ when you want to use this option. EXTREMELY_LOW_THRESHOLD = 2; } // Specify the device storage under which the files should be downloaded. // By default, the files will only be downloaded if the device is not in // low storage. optional DeviceStoragePolicy device_storage_policy = 1; // TODO(b/143548753): The first value in an enum must have a specific prefix. enum DeviceNetworkPolicy { // Only download files on wifi. DOWNLOAD_ONLY_ON_WIFI = 0; // Allow download on any network including wifi and cellular. DOWNLOAD_ON_ANY_NETWORK = 1; // Allow downloading only on wifi first, then after a configurable time // period set in the field download_first_on_wifi_period_secs below, // allow downloading on any network including wifi and cellular. DOWNLOAD_FIRST_ON_WIFI_THEN_ON_ANY_NETWORK = 2; } // Specify the device network under which the files should be downloaded. // By default, the files will only be downloaded on wifi. // // If your feature targets below v20 and want to download on cellular in // these versions of gms, also set allow_dowload_without_wifi = true; optional DeviceNetworkPolicy device_network_policy = 2; // This field will only be used when the // DeviceNetworkPolicy = DOWNLOAD_FIRST_ON_WIFI_THEN_ON_ANY_NETWORK // MDD will download the file only on wifi for this period of time. If the // download was not finished, MDD will download on any network including // wifi and cellular. // Ex: 604800 // 7 Days optional int64 download_first_on_wifi_period_secs = 4; // TODO(b/143548753): The first value in an enum must have a specific prefix. enum ActivatingCondition { // The download is activated as soon the server side config is received and // the server configured download conditions are satisfied. ALWAYS_ACTIVATED = 0; // The download is activated when both server side activation conditions // are satisfied and the client has activated the download on device. // // Clients can activate this group using the activateFileGroup API. // DEVICE_ACTIVATED = 1; } // Specify how the download is activated. By default, the download is // activated as soon as server configured activating conditions are satisfied. optional ActivatingCondition activating_condition = 3; } // This proto contains extra information about a file group for bookkeeping. // Next tag: 6 message DataFileGroupBookkeeping { // The epoch time (seconds since 1/1/1970) at which this stale file group will // be deleted. optional int64 stale_expiration_date = 1; // The timestamp (epoch time, milliseconds since 1/1/1970) that the file group // was first received. // // If this is an update on an existing group, then the timestamp from the old // group is used if no files were updated. optional int64 group_new_files_received_timestamp = 2; // The timestamp (epoch time, milliseconds since 1/1/1970) at which the group // was first marked as downloaded. optional int64 group_downloaded_timestamp_in_millis = 3; // The timestamp (epoch time, milliseconds since 1/1/1970) that MDD starts // downloading the file group for the first time. optional int64 group_download_started_timestamp_in_millis = 4; // The total count of download periodic tasks needed to fully download the // file group. optional int32 download_started_count = 5; } // Key used by mdd to uniquely identify a client group. message GroupKey { // The name of the group. optional string group_name = 1; // The package name of the group owner. A null value or empty value means // that the group is not associated with any package. optional string owner_package = 2; // The account associated to the file group. optional string account = 5; // Whether or not all files in a fileGroup have been downloaded. optional bool downloaded = 4; // The variant id of the group for identification purposes. // // This is used to ensure that groups with different variants can have // different entries in MDD metadata, and therefore have different lifecycles. // // Note that clients can choose to opt-in to a SINGLE_VARIANT flow where // different variants replace each other on-device (only single variant can // exist on a device at a time). In this case, an empty variant_id is set here // so groups with different variants share the same GroupKey and are subject // to the same lifecycle, even though the DataFileGroup does have a non-empty // variant_id. // // Because of the SINGLE_VARIANT flow and because groups may still be added // with no variant_id associated, using this property to tell if the // associated file group has a variant_id is unreliable. Instead, the // variant_id set within a DataFileGroup should be used as the source of truth // about the group (such as when logging). optional string variant_id = 6; reserved 3; } // Group Key properties that apply to all groups with that key. message GroupKeyProperties { // Whether this group key has been activated on the device. optional bool activated_on_device = 1; } // SharedFile is a internal data structure of the SharedFileManager. message SharedFile { optional string file_name = 4; optional FileStatus file_status = 5; // This field will be used to determine if a file can be retrieved from the // Android Blob Sharing Service. optional bool android_shared = 8; // The maximum expiration date found for a downloaded data file. If // {@code android_shared} is set to true, this field stores the current lease // expiration date. The default value is 0. // See for more details. optional int64 max_expiration_date_secs = 9; // Checksum used to access files through the Android Blob Sharing Service. optional string android_sharing_checksum = 10; // If the file is downloaded successfully but fails checksum matching, we will // attempt to delete the file so it can be redownloaded from scratch. To // prevent unnecessary network bandwidth, we keep track of the number of these // attempts in this field and stop after a certain number. (configurable by a // download flag). optional int32 checksum_mismatch_retry_download_count = 11; reserved 1, 2, 3, 6, 7; } // Metadata used by // com.google.android.libraries.mobiledatadownload.MobileDataDownloadManager message MobileDataDownloadManagerMetadata { optional bool mdd_migrated_to_offroad = 1; optional int32 reset_trigger = 2; } // Metadata used by // com.google.android.libraries.mobiledatadownload.SharedFileManager message SharedFileManagerMetadata { optional bool migrated_to_new_file_key = 1; optional int64 next_file_name = 2; } // Collects all data used by // com.google.android.libraries.mobiledatadownload.internal.Migrations message MigrationsStore { enum FileKeyVersion { NEW_FILE_KEY = 0; ADD_DOWNLOAD_TRANSFORM = 1; USE_CHECKSUM_ONLY = 2; } optional bool is_migrated_to_new_file_key = 1; optional FileKeyVersion current_version = 2; } // Collects all data used by // com.google.android.libraries.mobiledatadownload.internal.FileGroupsMetadata message FileGroupsMetadataStore { // Key must be a serialized GroupKey. map data_file_groups = 1; // Key must be a serialized GroupKey. map group_key_properties = 2; repeated DataFileGroupInternal stale_groups = 3; } // Collects all data used by // com.google.android.libraries.mobiledatadownload.internal.SharedFilesMetadata message SharedFilesMetadataStore { // The key must be a serialized NewFileKey. map shared_files = 1; } enum FileStatus { // The file has never been seen before. NONE = 0; // The file has been subscribed to, but download has not been attempted. SUBSCRIBED = 1; // The file download is currently in progress. DOWNLOAD_IN_PROGRESS = 2; // Downloading the file failed. DOWNLOAD_FAILED = 3; // The file was downloaded completely, and is available for use. DOWNLOAD_COMPLETE = 4; // The file was corrupted or lost after being successfully downloaded. CORRUPTED = 6; // Status returned when their is an error while getting the file status. // This is never saved on disk. INTERNAL_ERROR = 5; } // Key used by the SharedFileManager to identify a shared file. message NewFileKey { // These fields all mirror the similarly-named fields in DataFile. optional string url_to_download = 1 [deprecated = true]; optional int32 byte_size = 2 [deprecated = true]; optional string checksum = 3; optional DataFileGroupInternal.AllowedReaders allowed_readers = 4; optional mobstore.proto.Transforms download_transforms = 5 [deprecated = true]; } // This proto is used to store state for logging. See details at // message LoggingState { // The last time maintenance was run. This should be updated every time // maintenance is run. // Note: the current implementation only uses this to determine the date of // the last log event, but in the future we may want more granular // measurements for this, so we store the timestamp as-is. optional google.protobuf.Timestamp last_maintenance_run_timestamp = 1; // File group specific logging state keyed by GroupKey, build id and version // number. repeated FileGroupLoggingState file_group_logging_state = 2; // Set to true once the shared preferences migration is complete. // Note: this field isn't strictly necessary at the moment - we could just // check that the file_group_logging_state is empty since no one should write // to the network usage monitor shared prefs since the migration will be // installed at the same cl where the code is removed. However, if we were to // add more fields to FileGroupLoggingState, it would be less straightforward // to check for migration state - so having this boolean makes it a bit safer. optional bool shared_preferences_network_usage_monitor_migration_complete = 3; // Info to enable stable sampling. See for more // info. This field will be set by a migration on first access. optional SamplingInfo sampling_info = 4; } // This proto is used to store state for logging that is specific to a File // Group. This includes network usage logging and maybe download tiers (for // ). // // NEXT TAG: 7 message FileGroupLoggingState { // GroupKey associated with a file group -- this is used to populate the group // name and host package name. optional GroupKey group_key = 1; // The build_id associated with the file group. optional int64 build_id = 2; // The variant_id associated with the file group. optional string variant_id = 6; // The file group version number associated with the file group. optional int32 file_group_version_number = 3; // The number of bytes downloaded over a cellular (metered) network. optional int64 cellular_usage = 4; // The number of bytes downloaded over a wifi (unmetered) network. optional int64 wifi_usage = 5; } // Next id: 3 message SamplingInfo { // Random number generated and persisted on device. This number should not // change (unless device storage/mdd is cleared). It is used as a stable // identifier to determine whether MDD should log events. optional int64 stable_log_sampling_salt = 1; // When the stable_log_sampling_salt was first set. This will be used during // roll out to determine which devices have enabled stable sampling for a // sufficient time period. optional google.protobuf.Timestamp log_sampling_salt_set_timestamp = 2; }