path: root/javatests/com/google/android/libraries/mobiledatadownload/MddGarbageCollectionWithAndroidSharingIntegrationTest.java

/*
 * Copyright 2022 Google LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.google.android.libraries.mobiledatadownload;

import static com.google.android.libraries.mobiledatadownload.testing.MddTestDependencies.ExecutorType;
import static com.google.common.truth.Truth.assertThat;
import static java.util.concurrent.TimeUnit.DAYS;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.mockito.Mockito.eq;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;

import android.app.blob.BlobStoreManager;
import android.content.Context;
import android.net.Uri;
import android.util.Log;
import androidx.test.core.app.ApplicationProvider;
import com.google.android.libraries.mobiledatadownload.downloader.FileDownloader;
import com.google.android.libraries.mobiledatadownload.file.SynchronousFileStorage;
import com.google.android.libraries.mobiledatadownload.file.backends.AndroidFileBackend;
import com.google.android.libraries.mobiledatadownload.file.backends.BlobStoreBackend;
import com.google.android.libraries.mobiledatadownload.file.backends.BlobUri;
import com.google.android.libraries.mobiledatadownload.file.backends.JavaFileBackend;
import com.google.android.libraries.mobiledatadownload.file.transforms.CompressTransform;
import com.google.android.libraries.mobiledatadownload.internal.MddTestUtil;
import com.google.android.libraries.mobiledatadownload.monitor.DownloadProgressMonitor;
import com.google.android.libraries.mobiledatadownload.monitor.NetworkUsageMonitor;
import com.google.android.libraries.mobiledatadownload.testing.TestFileDownloader;
import com.google.android.libraries.mobiledatadownload.testing.TestFlags;
import com.google.common.base.Optional;
import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableList;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.mobiledatadownload.ClientConfigProto.ClientFile;
import com.google.mobiledatadownload.ClientConfigProto.ClientFileGroup;
import com.google.mobiledatadownload.DownloadConfigProto.DataFileGroup;
import com.google.mobiledatadownload.DownloadConfigProto.DownloadConditions.DeviceNetworkPolicy;
import com.google.mobiledatadownload.LogProto.DataDownloadFileGroupStats;
import com.google.mobiledatadownload.LogProto.MddLogData;
import com.google.testing.junit.testparameterinjector.TestParameter;
import com.google.testing.junit.testparameterinjector.TestParameterInjector;
import java.util.List;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.junit.MockitoJUnit;
import org.mockito.junit.MockitoRule;

@RunWith(TestParameterInjector.class)
public final class MddGarbageCollectionWithAndroidSharingIntegrationTest {
  private static final String TAG = "MddGarbageCollectionWithAndroidSharingIntegrationTest";
  private static final int MAX_DOWNLOAD_FILE_GROUP_WAIT_TIME_SECS = 300;

  private static final String TEST_DATA_RELATIVE_PATH =
      "third_party/java_src/android_libs/mobiledatadownload/javatests/com/google/android/libraries/mobiledatadownload/testdata/";

  private static final ScheduledExecutorService DOWNLOAD_EXECUTOR =
      Executors.newScheduledThreadPool(2);

  private static final String FILE_GROUP_TO_SHARE_1 = "test-group-1";
  private static final String FILE_ID_1 = "test-file-to-share-1";
  private static final String FILE_CHECKSUM_1 = "fcc96b272633cdf6c4bbd2d77512cca51bfb1dbd"; // SHA_1
  static final String FILE_ANDROID_SHARING_CHECKSUM_1 =
      "225017b5d5ec35732940af813b1ab7be5191e4c52659953e75a1a36a1398c48d"; // SHA_256
  static final int FILE_SIZE_1 = 57;
  static final String FILE_URL_1 = "https://www.gstatic.com/icing/idd/sample_group/step1.txt";

  private static final Context context = ApplicationProvider.getApplicationContext();

  @Mock private TaskScheduler mockTaskScheduler;
  @Mock private NetworkUsageMonitor mockNetworkUsageMonitor;
  @Mock private DownloadProgressMonitor mockDownloadProgressMonitor;
  @Mock private Logger mockLogger;

  private SynchronousFileStorage fileStorage;

  private BlobStoreManager blobStoreManager;
  private MobileDataDownload mobileDataDownload;
  private Supplier<FileDownloader> fileDownloaderSupplier;
  private ListeningExecutorService controlExecutor;

  private final TestFlags flags = new TestFlags();

  @Rule(order = 1)
  public final MockitoRule mocks = MockitoJUnit.rule();

  @TestParameter ExecutorType controlExecutorType;

  @Before
  public void setUp() throws Exception {

    // cl/439051122 created a temporary FALSE override targeted to ASGA devices. This test suite
    // relies on garbage collection being enabled to test the metadata state transistions, but
    // all_on testing doesn't respect diversion criteria in the launch.
    //
    // So we temporarily force it on to bypass the launch so the tests can rely on expected
    // behavior.
    // TODO(b/226551373): remove these overrides once AsgaDisableMddLibGcLaunch is turned down
    flags.mddEnableGarbageCollection = Optional.of(true);

    flags.mddAndroidSharingSampleInterval = Optional.of(1);

    flags.mddDefaultSampleInterval = Optional.of(1);

    BlobStoreBackend blobStoreBackend = new BlobStoreBackend(context);
    blobStoreManager = (BlobStoreManager) context.getSystemService(Context.BLOB_STORE_SERVICE);

    controlExecutor = controlExecutorType.executor();

    fileStorage =
        new SynchronousFileStorage(
            /* backends= */ ImmutableList.of(
                AndroidFileBackend.builder(context).build(),
                blobStoreBackend,
                new JavaFileBackend()),
            /* transforms= */ ImmutableList.of(new CompressTransform()),
            /* monitors= */ ImmutableList.of(mockNetworkUsageMonitor, mockDownloadProgressMonitor));

    fileDownloaderSupplier =
        () ->
            new TestFileDownloader(
                TEST_DATA_RELATIVE_PATH,
                fileStorage,
                MoreExecutors.listeningDecorator(DOWNLOAD_EXECUTOR));
  }

  @After
  public void tearDown() throws Exception {
    mobileDataDownload.clear().get();
    // Commands to clean up the blob storage.
    MddTestUtil.runShellCmd("cmd blob_store clear-all-sessions");
    MddTestUtil.runShellCmd("cmd blob_store clear-all-blobs");
  }

  private void downloadFileGroup(DataFileGroup fileGroup) throws Exception {
    assertThat(
            mobileDataDownload
                .addFileGroup(AddFileGroupRequest.newBuilder().setDataFileGroup(fileGroup).build())
                .get())
        .isTrue();
    mobileDataDownload
        .downloadFileGroup(
            DownloadFileGroupRequest.newBuilder()
                .setGroupName(fileGroup.getGroupName())
                .setListenerOptional(
                    Optional.of(
                        new DownloadListener() {
                          @Override
                          public void onProgress(long currentSize) {
                            Log.i(TAG, "onProgress " + currentSize);
                          }

                          @Override
                          public void onComplete(ClientFileGroup clientFileGroup) {
                            Log.i(TAG, "onComplete " + clientFileGroup.getGroupName());
                          }
                        }))
                .build())
        .get(MAX_DOWNLOAD_FILE_GROUP_WAIT_TIME_SECS, SECONDS);
  }

  private ClientFileGroup verifyDownloadedGroupIsDownloaded(DataFileGroup fileGroup, int fileCount)
      throws Exception {
    ClientFileGroup clientFileGroup =
        mobileDataDownload
            .getFileGroup(
                GetFileGroupRequest.newBuilder().setGroupName(fileGroup.getGroupName()).build())
            .get();

    assertThat(clientFileGroup).isNotNull();
    assertThat(clientFileGroup.getGroupName()).isEqualTo(fileGroup.getGroupName());
    assertThat(clientFileGroup.getFileCount()).isEqualTo(fileCount);
    return clientFileGroup;
  }

  @Test
  public void deletesStaleGroups_staleLifetimeZero() throws Exception {
    mobileDataDownload = builderForTest().build();
    Uri androidUri =
        BlobUri.builder(context).setBlobParameters(FILE_ANDROID_SHARING_CHECKSUM_1).build();
    assertThat(fileStorage.exists(androidUri)).isFalse();

    // Download file group with stale lifetime 0.
    DataFileGroup fileGroup =
        TestFileGroupPopulator.createDataFileGroup(
            FILE_GROUP_TO_SHARE_1,
            context.getPackageName(),
            new String[] {FILE_ID_1},
            new int[] {FILE_SIZE_1},
            new String[] {FILE_CHECKSUM_1},
            new String[] {FILE_ANDROID_SHARING_CHECKSUM_1},
            new String[] {FILE_URL_1},
            DeviceNetworkPolicy.DOWNLOAD_ON_ANY_NETWORK);

    downloadFileGroup(fileGroup);

    ClientFileGroup clientFileGroup = verifyDownloadedGroupIsDownloaded(fileGroup, 1);

    ClientFile clientFile = clientFileGroup.getFileList().get(0);
    assertThat(clientFile.getFileId()).isEqualTo(FILE_ID_1);
    Uri uri = Uri.parse(clientFile.getFileUri());

    // The file is now available in the android shared storage.
    assertThat(uri).isEqualTo(androidUri);
    assertThat(fileStorage.exists(uri)).isTrue();
    assertThat(blobStoreManager.getLeasedBlobs()).hasSize(1);

    // Send an empty group so that the old group is now stale.
    DataFileGroup emptyFileGroup =
        TestFileGroupPopulator.createDataFileGroup(
            FILE_GROUP_TO_SHARE_1,
            context.getPackageName(),
            new String[] {},
            new int[] {},
            new String[] {},
            new String[] {},
            new String[] {},
            DeviceNetworkPolicy.DOWNLOAD_ON_ANY_NETWORK);

    downloadFileGroup(emptyFileGroup);

    // Run maintenance taks.
    mobileDataDownload.maintenance().get(MAX_DOWNLOAD_FILE_GROUP_WAIT_TIME_SECS, SECONDS);

    verifyDownloadedGroupIsDownloaded(emptyFileGroup, 0);

    // Old stale file has been released
    assertThat(blobStoreManager.getLeasedBlobs()).isEmpty();

    // Verify logging events.

    ArgumentCaptor<MddLogData> logDataCaptor = ArgumentCaptor.forClass(MddLogData.class);
    // 1050 is the tag number for MddClientEvent.Code.EVENT_CODE_UNSPECIFIED.
    verify(mockLogger).log(logDataCaptor.capture(), /* eventCode= */ eq(1050));
    List<MddLogData> logData = logDataCaptor.getAllValues();
    assertThat(logData).hasSize(1);

    DataDownloadFileGroupStats dataDownloadFileGroupStats =
        logData.get(0).getDataDownloadFileGroupStats();
    DataDownloadFileGroupStats staleGroupExpired =
        DataDownloadFileGroupStats.newBuilder()
            .setFileGroupName(fileGroup.getGroupName())
            .setFileGroupVersionNumber(fileGroup.getFileGroupVersionNumber())
            .setBuildId(0)
            .setVariantId("")
            .build();
    assertThat(dataDownloadFileGroupStats).isEqualTo(staleGroupExpired);

    logDataCaptor = ArgumentCaptor.forClass(MddLogData.class);
    // 1084 is the tag number for MddClientEvent.Code.EVENT_CODE_UNSPECIFIED;
    // Called once for every released lease.
    verify(mockLogger).log(logDataCaptor.capture(), /* eventCode= */ eq(1084));

    logDataCaptor = ArgumentCaptor.forClass(MddLogData.class);
    // 1051 is the tag number for MddClientEvent.Code.EVENT_CODE_UNSPECIFIED.
    // It's logged once by mobileDataDownload.maintenance() and three times in the
    // ExpirationHandler, once when the file metadata is deleted, once when the lease is released
    // and once when the temporary local copy of the shared file is deleted.
    verify(mockLogger, times(4)).log(logDataCaptor.capture(), /* eventCode= */ eq(1051));
    logData = logDataCaptor.getAllValues();
    assertThat(logData).hasSize(4);

    Void metadafileDeleted = null;
    Void fileReleased = null;
    Void fileDeleted = null;
  }

  @Test
  public void deletesStaleGroups_staleLifetimeTwoDays() throws Exception {
    mobileDataDownload = builderForTest().build();
    Uri androidUri =
        BlobUri.builder(context).setBlobParameters(FILE_ANDROID_SHARING_CHECKSUM_1).build();
    assertThat(fileStorage.exists(androidUri)).isFalse();

    // Download file group with stale lifetime +2 days.
    DataFileGroup fileGroup =
        TestFileGroupPopulator.createDataFileGroup(
            FILE_GROUP_TO_SHARE_1,
            context.getPackageName(),
            new String[] {FILE_ID_1},
            new int[] {FILE_SIZE_1},
            new String[] {FILE_CHECKSUM_1},
            new String[] {FILE_ANDROID_SHARING_CHECKSUM_1},
            new String[] {FILE_URL_1},
            DeviceNetworkPolicy.DOWNLOAD_ON_ANY_NETWORK);
    fileGroup = fileGroup.toBuilder().setStaleLifetimeSecs(DAYS.toSeconds(2)).build();

    downloadFileGroup(fileGroup);

    ClientFileGroup clientFileGroup = verifyDownloadedGroupIsDownloaded(fileGroup, 1);

    ClientFile clientFile = clientFileGroup.getFileList().get(0);
    assertThat(clientFile.getFileId()).isEqualTo(FILE_ID_1);
    Uri uri = Uri.parse(clientFile.getFileUri());

    // The file is now available in the android shared storage.
    assertThat(uri).isEqualTo(androidUri);
    assertThat(fileStorage.exists(uri)).isTrue();
    assertThat(blobStoreManager.getLeasedBlobs()).hasSize(1);

    // Send an empty group so that the old group is now stale.
    DataFileGroup emptyFileGroup =
        TestFileGroupPopulator.createDataFileGroup(
            FILE_GROUP_TO_SHARE_1,
            context.getPackageName(),
            new String[] {},
            new int[] {},
            new String[] {},
            new String[] {},
            new String[] {},
            DeviceNetworkPolicy.DOWNLOAD_ON_ANY_NETWORK);
    downloadFileGroup(emptyFileGroup);

    // Run maintenance taks.
    mobileDataDownload.maintenance().get(MAX_DOWNLOAD_FILE_GROUP_WAIT_TIME_SECS, SECONDS);

    verifyDownloadedGroupIsDownloaded(emptyFileGroup, 0);

    // Old stale file hasn't been released yet
    assertThat(blobStoreManager.getLeasedBlobs()).hasSize(1);

    // Advance time by 2 days, and verify that the lease on the shared file has been
    // released.
    MddTestUtil.timeTravel(context, DAYS.toMillis(2));
    mobileDataDownload.maintenance().get(MAX_DOWNLOAD_FILE_GROUP_WAIT_TIME_SECS, SECONDS);

    verifyDownloadedGroupIsDownloaded(emptyFileGroup, 0);

    assertThat(blobStoreManager.getLeasedBlobs()).isEmpty();

    // Verify logging events.

    ArgumentCaptor<MddLogData> logDataCaptor = ArgumentCaptor.forClass(MddLogData.class);
    // 1050 is the tag number for MddClientEvent.Code.EVENT_CODE_UNSPECIFIED.
    verify(mockLogger).log(logDataCaptor.capture(), /* eventCode= */ eq(1050));
    List<MddLogData> logData = logDataCaptor.getAllValues();
    assertThat(logData).hasSize(1);

    DataDownloadFileGroupStats dataDownloadFileGroupStats =
        logData.get(0).getDataDownloadFileGroupStats();
    DataDownloadFileGroupStats staleGroupExpired =
        DataDownloadFileGroupStats.newBuilder()
            .setFileGroupName(fileGroup.getGroupName())
            .setFileGroupVersionNumber(fileGroup.getFileGroupVersionNumber())
            .setBuildId(0)
            .setVariantId("")
            .build();
    assertThat(dataDownloadFileGroupStats).isEqualTo(staleGroupExpired);

    logDataCaptor = ArgumentCaptor.forClass(MddLogData.class);
    // 1084 is the tag number for MddClientEvent.Code.EVENT_CODE_UNSPECIFIED;
    // Called once for every released lease.
    verify(mockLogger).log(logDataCaptor.capture(), /* eventCode= */ eq(1084));

    logDataCaptor = ArgumentCaptor.forClass(MddLogData.class);
    // 1051 is the tag number for MddClientEvent.Code.EVENT_CODE_UNSPECIFIED.
    // It's logged once every time mobileDataDownload.maintenance() is called and three times in the
    // ExpirationHandler, once when the file metadata is deleted, once when the lease is released
    // and once when the temporary local copy of the shared file is deleted.
    verify(mockLogger, times(5)).log(logDataCaptor.capture(), /* eventCode= */ eq(1051));
    logData = logDataCaptor.getAllValues();
    assertThat(logData).hasSize(5);

    Void metadafileDeleted = null;
    Void fileReleased = null;
    Void fileDeleted = null;
  }

  @Test
  public void deletesExpiredGroups() throws Exception {
    mobileDataDownload = builderForTest().build();
    Uri androidUri =
        BlobUri.builder(context).setBlobParameters(FILE_ANDROID_SHARING_CHECKSUM_1).build();
    assertThat(fileStorage.exists(androidUri)).isFalse();

    // Download file group with stale lifetime +2 days.
    DataFileGroup fileGroup =
        TestFileGroupPopulator.createDataFileGroup(
            FILE_GROUP_TO_SHARE_1,
            context.getPackageName(),
            new String[] {FILE_ID_1},
            new int[] {FILE_SIZE_1},
            new String[] {FILE_CHECKSUM_1},
            new String[] {FILE_ANDROID_SHARING_CHECKSUM_1},
            new String[] {FILE_URL_1},
            DeviceNetworkPolicy.DOWNLOAD_ON_ANY_NETWORK);

    // It expires in two days.
    fileGroup = fileGroup.toBuilder().setExpirationDate(MddTestUtil.daysFromNow(2)).build();

    downloadFileGroup(fileGroup);

    ClientFileGroup clientFileGroup = verifyDownloadedGroupIsDownloaded(fileGroup, 1);

    ClientFile clientFile = clientFileGroup.getFileList().get(0);
    assertThat(clientFile.getFileId()).isEqualTo(FILE_ID_1);
    Uri uri = Uri.parse(clientFile.getFileUri());

    // The file is now available in the android shared storage.
    assertThat(uri).isEqualTo(androidUri);
    assertThat(fileStorage.exists(uri)).isTrue();
    assertThat(blobStoreManager.getLeasedBlobs()).hasSize(1);

    // Run maintenance tasks and verify that we still own the lease om the shared file.
    mobileDataDownload.maintenance().get(MAX_DOWNLOAD_FILE_GROUP_WAIT_TIME_SECS, SECONDS);

    verifyDownloadedGroupIsDownloaded(fileGroup, 1);

    assertThat(blobStoreManager.getLeasedBlobs()).hasSize(1);

    // Advance time by 3 days, and verify that the group and files can no longer be read
    // because they expired.
    MddTestUtil.timeTravel(context, DAYS.toMillis(3));
    mobileDataDownload.maintenance().get(MAX_DOWNLOAD_FILE_GROUP_WAIT_TIME_SECS, SECONDS);
    clientFileGroup =
        mobileDataDownload
            .getFileGroup(
                GetFileGroupRequest.newBuilder().setGroupName(FILE_GROUP_TO_SHARE_1).build())
            .get();
    assertThat(clientFileGroup).isNull();

    assertThat(blobStoreManager.getLeasedBlobs()).isEmpty();

    // Verify logging events.

    ArgumentCaptor<MddLogData> logDataCaptor = ArgumentCaptor.forClass(MddLogData.class);
    // 1049 is the tag number for MddClientEvent.Code.EVENT_CODE_UNSPECIFIED.
    verify(mockLogger).log(logDataCaptor.capture(), /* eventCode= */ eq(1049));

    List<MddLogData> logData = logDataCaptor.getAllValues();
    assertThat(logData).hasSize(1);

    DataDownloadFileGroupStats dataDownloadFileGroupStats =
        logData.get(0).getDataDownloadFileGroupStats();
    DataDownloadFileGroupStats groupExpired =
        DataDownloadFileGroupStats.newBuilder()
            .setFileGroupName(fileGroup.getGroupName())
            .setFileGroupVersionNumber(fileGroup.getFileGroupVersionNumber())
            .setBuildId(0)
            .setVariantId("")
            .build();
    assertThat(dataDownloadFileGroupStats).isEqualTo(groupExpired);

    logDataCaptor = ArgumentCaptor.forClass(MddLogData.class);
    // 1084 is the tag number for MddClientEvent.Code.EVENT_CODE_UNSPECIFIED;
    // Called once for every released lease.
    verify(mockLogger).log(logDataCaptor.capture(), /* eventCode= */ eq(1084));

    logDataCaptor = ArgumentCaptor.forClass(MddLogData.class);
    // 1051 is the tag number for MddClientEvent.Code.EVENT_CODE_UNSPECIFIED.
    // It's logged once every time mobileDataDownload.maintenance() is called and three times in the
    // ExpirationHandler, once when the file metadata is deleted, once when the lease is
    // released and once when the temporary local copy of the shared file is deleted.
    verify(mockLogger, times(5)).log(logDataCaptor.capture(), /* eventCode= */ eq(1051));
    logData = logDataCaptor.getAllValues();
    assertThat(logData).hasSize(5);

    Void metadafileDeleted = null;
    Void fileReleased = null;
    Void fileDeleted = null;
  }

  /**
   * Returns MDD Builder with common dependencies set -- additional dependencies are added in each
   * test as needed.
   */
  private MobileDataDownloadBuilder builderForTest() {
    return MobileDataDownloadBuilder.newBuilder()
        .setContext(context)
        .setControlExecutor(controlExecutor)
        .setFileDownloaderSupplier(fileDownloaderSupplier)
        .setTaskScheduler(Optional.of(mockTaskScheduler))
        .setDeltaDecoderOptional(Optional.absent())
        .setFileStorage(fileStorage)
        .setNetworkUsageMonitor(mockNetworkUsageMonitor)
        .setDownloadMonitorOptional(Optional.of(mockDownloadProgressMonitor))
        .setLoggerOptional(Optional.of(mockLogger))
        .setFlagsOptional(Optional.of(flags));
  }
}