Delete Debug Report Records Over Retry Limit

Expanding MeasurementDao.deleteExpiredRecords to remove records from Debug Report Table where Max Retry attempts were exceeded.

Bug: 296633717
Test: atest AdServicesFrameworkUnitTests AdServicesServiceCoreUnitTests AdServicesManagerServiceTests AdServicesEndToEndTests
Change-Id: I985e1a4533b9b55f59344b7d333c2ad2ab3f853f

6 files changed, 321 insertions, 1 deletions

diff --git a/adservices/service-core/java/com/android/adservices/data/measurement/MeasurementDao.java b/adservices/service-core/java/com/android/adservices/data/measurement/MeasurementDao.java
index 1e6554fedf..ef2ae0179a 100644
--- a/adservices/service-core/java/com/android/adservices/data/measurement/MeasurementDao.java
+++ b/adservices/service-core/java/com/android/adservices/data/measurement/MeasurementDao.java
@@ -1668,6 +1668,36 @@ class MeasurementDao implements IMeasurementDao {
                         + subQuery
                         + ")",
                 new String[] {KeyValueData.DataType.REGISTRATION_REDIRECT_COUNT.toString()});
+
+        // When Limiting Retries, consider Verbose Debug Reports Expired when Exceeds Limit.
+        if (mReportingRetryLimitEnabledSupplier.get()) {
+            db.delete(
+                    MeasurementTables.DebugReportContract.TABLE,
+                    MeasurementTables.DebugReportContract.ID
+                            + " IN ("
+                            + "SELECT "
+                            + MeasurementTables.DebugReportContract.ID
+                            + " FROM "
+                            + MeasurementTables.DebugReportContract.TABLE
+                            + " LEFT JOIN "
+                            + MeasurementTables.KeyValueDataContract.TABLE
+                            + " ON ("
+                            + MeasurementTables.DebugReportContract.ID
+                            + " = "
+                            + MeasurementTables.KeyValueDataContract.KEY
+                            + ") "
+                            + "WHERE CAST("
+                            + MeasurementTables.KeyValueDataContract.VALUE
+                            + " AS INTEGER) >= ? "
+                            + "AND "
+                            + MeasurementTables.KeyValueDataContract.DATA_TYPE
+                            + " = ? "
+                            + ")",
+                    new String[] {
+                        mReportingRetryLimitSupplier.get().toString(),
+                        DataType.DEBUG_REPORT_RETRY_COUNT.toString()
+                    });
+        }
     }
 
     @Override
diff --git a/adservices/service-core/java/com/android/adservices/service/measurement/reporting/DebugReport.java b/adservices/service-core/java/com/android/adservices/service/measurement/reporting/DebugReport.java
index ec1a20aae3..69265789a3 100644
--- a/adservices/service-core/java/com/android/adservices/service/measurement/reporting/DebugReport.java
+++ b/adservices/service-core/java/com/android/adservices/service/measurement/reporting/DebugReport.java
@@ -51,6 +51,7 @@ public final class DebugReport {
 
     @Override
     public boolean equals(Object obj) {
+        // TODO (b/300109438) Investigate DebugReport::equals
         if (!(obj instanceof DebugReport)) {
             return false;
         }
diff --git a/adservices/tests/unittest/service-core/assets/measurement_delete_expired_test.json b/adservices/tests/unittest/service-core/assets/measurement_delete_expired_test.json
index 48c1266b69..17da13e107 100644
--- a/adservices/tests/unittest/service-core/assets/measurement_delete_expired_test.json
+++ b/adservices/tests/unittest/service-core/assets/measurement_delete_expired_test.json
@@ -366,6 +366,52 @@
           }
         ]
       }
+    },
+    {
+      "name": "1 non-expired DebugReport, 1 expired DebugReport",
+      "input": {
+        "key_values": [
+          {
+            "key":"DR-Expired",
+            "dataType": "DEBUG_REPORT_RETRY_COUNT",
+            "value": "1000"
+          }
+        ],
+        "debug_reports": [
+          {
+            "id": "DR-non-Expired",
+            "type": "source-success",
+            "body": "{\"test\":\"body\"}"
+          },
+          {
+            "id": "DR-Expired",
+            "type": "source-success",
+            "body": "{\"test\":\"body\"}"
+          }
+        ]
+      },
+      "output": {
+        "sources": [],
+        "source_destinations": [],
+        "triggers": [],
+        "event_reports": [],
+        "attributions": [],
+        "async_registrations": [],
+        "debug_reports": [
+          {
+            "id": "DR-non-Expired",
+            "type": "source-success",
+            "body": "{\"test\":\"body\"}"
+          }
+        ],
+        "key_values": [
+          {
+            "key":"DR-Expired",
+            "dataType": "DEBUG_REPORT_RETRY_COUNT",
+            "value": "1000"
+          }
+        ]
+      }
     }
   ]
 }
diff --git a/adservices/tests/unittest/service-core/src/com/android/adservices/data/measurement/AbstractDbIntegrationTest.java b/adservices/tests/unittest/service-core/src/com/android/adservices/data/measurement/AbstractDbIntegrationTest.java
index 447ce5ccf1..d23383ee7b 100644
--- a/adservices/tests/unittest/service-core/src/com/android/adservices/data/measurement/AbstractDbIntegrationTest.java
+++ b/adservices/tests/unittest/service-core/src/com/android/adservices/data/measurement/AbstractDbIntegrationTest.java
@@ -28,11 +28,13 @@ import com.android.adservices.data.DbTestUtil;
 import com.android.adservices.service.FlagsFactory;
 import com.android.adservices.service.measurement.Attribution;
 import com.android.adservices.service.measurement.EventReport;
+import com.android.adservices.service.measurement.KeyValueData;
 import com.android.adservices.service.measurement.Source;
 import com.android.adservices.service.measurement.Trigger;
 import com.android.adservices.service.measurement.aggregation.AggregateEncryptionKey;
 import com.android.adservices.service.measurement.aggregation.AggregateReport;
 import com.android.adservices.service.measurement.registration.AsyncRegistration;
+import com.android.adservices.service.measurement.reporting.DebugReport;
 import com.android.adservices.service.measurement.util.UnsignedLong;
 import com.android.dx.mockito.inline.extended.ExtendedMockito;
 
@@ -141,6 +143,36 @@ public abstract class AbstractDbIntegrationTest {
         Assert.assertTrue(
                 "Async Registration mismatch",
                 areEqual(mOutput.mAsyncRegistrationList, dbState.mAsyncRegistrationList));
+        for (int i = 0; i < mOutput.mDebugReportList.size(); i++) {
+            Assert.assertTrue(
+                    "DebugReport Mismatch",
+                    areDebugReportContentsSimilar(
+                            mOutput.mDebugReportList.get(i), dbState.mDebugReportList.get(i)));
+        }
+        for (int i = 0; i < mOutput.mKeyValueDataList.size(); i++) {
+            Assert.assertTrue(
+                    "KeyValueData Mismatch",
+                    areEqualKeyValueData(
+                            mOutput.mKeyValueDataList.get(i), dbState.mKeyValueDataList.get(i)));
+        }
+    }
+
+    private boolean areDebugReportContentsSimilar(
+            DebugReport debugReport, DebugReport debugReport1) {
+        // TODO (b/300109438) Investigate DebugReport::equals
+        return Objects.equals(debugReport.getType(), debugReport1.getType())
+                && Objects.equals(
+                        debugReport.getBody().toString(), debugReport1.getBody().toString())
+                && Objects.equals(debugReport.getEnrollmentId(), debugReport1.getEnrollmentId())
+                && Objects.equals(
+                        debugReport.getRegistrationOrigin(), debugReport1.getRegistrationOrigin())
+                && Objects.equals(debugReport.getReferenceId(), debugReport1.getReferenceId());
+    }
+
+    private boolean areEqualKeyValueData(KeyValueData keyValueData, KeyValueData keyValueData1) {
+        return Objects.equals(keyValueData.getKey(), keyValueData1.getKey())
+                && Objects.equals(keyValueData.getDataType(), keyValueData1.getDataType())
+                && Objects.equals(keyValueData.getValue(), keyValueData1.getValue());
     }
 
     private boolean fuzzyCompareAggregateReport(
@@ -271,6 +303,8 @@ public abstract class AbstractDbIntegrationTest {
         db.delete(MeasurementTables.AggregateReport.TABLE, null, null);
         db.delete(MeasurementTables.AggregateEncryptionKey.TABLE, null, null);
         db.delete(MeasurementTables.AsyncRegistrationContract.TABLE, null, null);
+        db.delete(MeasurementTables.DebugReportContract.TABLE, null, null);
+        db.delete(MeasurementTables.KeyValueDataContract.TABLE, null, null);
     }
 
     /**
@@ -299,10 +333,15 @@ public abstract class AbstractDbIntegrationTest {
         for (AggregateEncryptionKey key : input.mAggregateEncryptionKeyList) {
             insertToDb(key, db);
         }
-
         for (AsyncRegistration registration : input.mAsyncRegistrationList) {
             insertToDb(registration, db);
         }
+        for (DebugReport debugReport : input.mDebugReportList) {
+            insertToDb(debugReport, db);
+        }
+        for (KeyValueData keyValueData : input.mKeyValueDataList) {
+            insertToDb(keyValueData, db);
+        }
     }
 
     /**
@@ -594,6 +633,40 @@ public abstract class AbstractDbIntegrationTest {
         }
     }
 
+    public static void insertToDb(KeyValueData keyValueData, SQLiteDatabase db)
+            throws SQLiteException {
+        ContentValues values = new ContentValues();
+        values.put(MeasurementTables.KeyValueDataContract.KEY, keyValueData.getKey());
+        values.put(MeasurementTables.KeyValueDataContract.VALUE, keyValueData.getValue());
+        values.put(
+                MeasurementTables.KeyValueDataContract.DATA_TYPE,
+                keyValueData.getDataType().toString());
+        long rowId =
+                db.insert(
+                        MeasurementTables.KeyValueDataContract.TABLE,
+                        /* nullColumnHack= */ null,
+                        values);
+        if (rowId == -1) {
+            throw new SQLiteException("KeyValueData insertion failed.");
+        }
+    }
+
+    public static void insertToDb(DebugReport debugReport, SQLiteDatabase db)
+            throws SQLiteException {
+        ContentValues values = new ContentValues();
+        values.put(MeasurementTables.DebugReportContract.ID, debugReport.getId());
+        values.put(MeasurementTables.DebugReportContract.BODY, debugReport.getBody().toString());
+        values.put(MeasurementTables.DebugReportContract.TYPE, debugReport.getType());
+        long rowId =
+                db.insert(
+                        MeasurementTables.DebugReportContract.TABLE,
+                        /* nullColumnHack= */ null,
+                        values);
+        if (rowId == -1) {
+            throw new SQLiteException("DebugReport insertion failed.");
+        }
+    }
+
     private static Long getNullableUnsignedLong(UnsignedLong ulong) {
         return Optional.ofNullable(ulong).map(UnsignedLong::getValue).orElse(null);
     }
diff --git a/adservices/tests/unittest/service-core/src/com/android/adservices/data/measurement/DbState.java b/adservices/tests/unittest/service-core/src/com/android/adservices/data/measurement/DbState.java
index 790e0b2998..729b7ee3fb 100644
--- a/adservices/tests/unittest/service-core/src/com/android/adservices/data/measurement/DbState.java
+++ b/adservices/tests/unittest/service-core/src/com/android/adservices/data/measurement/DbState.java
@@ -22,6 +22,7 @@ import android.net.Uri;
 
 import com.android.adservices.service.measurement.Attribution;
 import com.android.adservices.service.measurement.EventReport;
+import com.android.adservices.service.measurement.KeyValueData;
 import com.android.adservices.service.measurement.Source;
 import com.android.adservices.service.measurement.Trigger;
 import com.android.adservices.service.measurement.WebUtil;
@@ -57,6 +58,7 @@ public class DbState {
     List<AggregateReport> mAggregateReportList;
     List<DebugReport> mDebugReportList;
     List<AsyncRegistration> mAsyncRegistrationList;
+    List<KeyValueData> mKeyValueDataList;
 
     public DbState() {
         mSourceList = new ArrayList<>();
@@ -68,6 +70,7 @@ public class DbState {
         mAggregateReportList = new ArrayList<>();
         mDebugReportList = new ArrayList<>();
         mAsyncRegistrationList = new ArrayList<>();
+        mKeyValueDataList = new ArrayList<>();
     }
 
     public DbState(JSONObject testInput) throws JSONException {
@@ -162,6 +165,16 @@ public class DbState {
                 mAsyncRegistrationList.add(asyncRegistration);
             }
         }
+
+        if (testInput.has("key_values")) {
+            // KeyValues
+            JSONArray keyValues = testInput.getJSONArray("key_values");
+            for (int i = 0; i < keyValues.length(); i++) {
+                JSONObject aJSON = keyValues.getJSONObject(i);
+                KeyValueData keyValueData = getKeyValueDataFrom(aJSON);
+                mKeyValueDataList.add(keyValueData);
+            }
+        }
     }
 
     public DbState(SQLiteDatabase readerDB) {
@@ -271,6 +284,29 @@ public class DbState {
                     SqliteObjectMapper.constructAsyncRegistration(asyncRegistrationCursor));
         }
         asyncRegistrationCursor.close();
+
+        // Read KeyValueData table
+        Cursor keyValueDataCursor =
+                readerDB.query(
+                        MeasurementTables.KeyValueDataContract.TABLE,
+                        new String[] {
+                            MeasurementTables.KeyValueDataContract.KEY,
+                            MeasurementTables.KeyValueDataContract.VALUE,
+                            MeasurementTables.KeyValueDataContract.DATA_TYPE
+                        },
+                        null,
+                        null,
+                        null,
+                        null,
+                        MeasurementTables.KeyValueDataContract.KEY);
+        while (keyValueDataCursor.moveToNext()) {
+            KeyValueData.Builder builder = new KeyValueData.Builder();
+            builder.setKey(keyValueDataCursor.getString(0));
+            builder.setValue(keyValueDataCursor.getString(1));
+            builder.setDataType(KeyValueData.DataType.valueOf(keyValueDataCursor.getString(2)));
+            mKeyValueDataList.add(builder.build());
+        }
+        keyValueDataCursor.close();
     }
 
     public void sortAll() {
@@ -302,6 +338,8 @@ public class DbState {
         mDebugReportList.sort(Comparator.comparing(DebugReport::getId));
 
         mAsyncRegistrationList.sort(Comparator.comparing(AsyncRegistration::getRequestTime));
+
+        mKeyValueDataList.sort(Comparator.comparing(KeyValueData::getKey));
     }
 
     public List<AggregateEncryptionKey> getAggregateEncryptionKeyList() {
@@ -528,6 +566,14 @@ public class DbState {
                 .build();
     }
 
+    private KeyValueData getKeyValueDataFrom(JSONObject kJSON) throws JSONException {
+        return new KeyValueData.Builder()
+                .setKey(kJSON.getString("key"))
+                .setDataType(KeyValueData.DataType.valueOf(kJSON.getString("dataType")))
+                .setValue(kJSON.getString("value"))
+                .build();
+    }
+
     private Uri getRegistrationOrigin(JSONObject json) throws JSONException {
         return Uri.parse(
                 json.isNull("registration_origin")
diff --git a/adservices/tests/unittest/service-core/src/com/android/adservices/data/measurement/MeasurementDaoTest.java b/adservices/tests/unittest/service-core/src/com/android/adservices/data/measurement/MeasurementDaoTest.java
index a07faa4d69..2da11de3d2 100644
--- a/adservices/tests/unittest/service-core/src/com/android/adservices/data/measurement/MeasurementDaoTest.java
+++ b/adservices/tests/unittest/service-core/src/com/android/adservices/data/measurement/MeasurementDaoTest.java
@@ -5473,6 +5473,130 @@ public class MeasurementDaoTest {
     }
 
     @Test
+    public void deleteExpiredRecords_VerboseDebugReportsWhileLimitingRetries() {
+        // Mocking that the flags return a Max Retry of 1
+        Flags mockFlags = Mockito.mock(Flags.class);
+        ExtendedMockito.doReturn(mockFlags).when(FlagsFactory::getFlags);
+        ExtendedMockito.doReturn(1).when(mockFlags).getMeasurementReportingRetryLimit();
+        ExtendedMockito.doReturn(true).when(mockFlags).getMeasurementReportingRetryLimitEnabled();
+
+        SQLiteDatabase db = MeasurementDbHelper.getInstance(sContext).getReadableDatabase();
+
+        DebugReport debugReport1 =
+                new DebugReport.Builder()
+                        .setId("reportId1")
+                        .setType("trigger-event-deduplicated")
+                        .setBody(
+                                " {\n"
+                                        + "      \"attribution_destination\":"
+                                        + " \"https://destination.example\",\n"
+                                        + "      \"source_event_id\": \"45623\"\n"
+                                        + "    }")
+                        .setEnrollmentId("1")
+                        .setRegistrationOrigin(REGISTRATION_ORIGIN)
+                        .build();
+
+        DebugReport debugReport2 =
+                new DebugReport.Builder()
+                        .setId("reportId2")
+                        .setType("trigger-event-deduplicated")
+                        .setBody(
+                                " {\n"
+                                        + "      \"attribution_destination\":"
+                                        + " \"https://destination.example\",\n"
+                                        + "      \"source_event_id\": \"45623\"\n"
+                                        + "    }")
+                        .setEnrollmentId("1")
+                        .setRegistrationOrigin(REGISTRATION_ORIGIN)
+                        .build();
+
+        mDatastoreManager.runInTransaction((dao) -> dao.insertDebugReport(debugReport1));
+        mDatastoreManager.runInTransaction((dao) -> dao.insertDebugReport(debugReport2));
+
+        mDatastoreManager.runInTransaction(dao -> dao.deleteExpiredRecords(0, 0));
+        assertEquals(
+                2,
+                DatabaseUtils.longForQuery(
+                        db,
+                        "SELECT COUNT("
+                                + MeasurementTables.DebugReportContract.ID
+                                + ") FROM "
+                                + MeasurementTables.DebugReportContract.TABLE,
+                        null));
+        // Increment Attempt Record 1
+        mDatastoreManager.runInTransaction(
+                (dao) ->
+                        dao.incrementAndGetReportingRetryCount(
+                                debugReport1.getId(), DataType.DEBUG_REPORT_RETRY_COUNT));
+        // Delete Expired (Record 1)
+        mDatastoreManager.runInTransaction(dao -> dao.deleteExpiredRecords(0, 0));
+
+        // Assert Record 2 remains.
+        assertEquals(
+                1,
+                DatabaseUtils.longForQuery(
+                        db,
+                        "SELECT COUNT("
+                                + MeasurementTables.DebugReportContract.ID
+                                + ") FROM "
+                                + MeasurementTables.DebugReportContract.TABLE
+                                + " WHERE "
+                                + MeasurementTables.DebugReportContract.ID
+                                + " = ?",
+                        new String[] {debugReport2.getId()}));
+
+        // Assert Record 1 Removed
+        assertEquals(
+                0,
+                DatabaseUtils.longForQuery(
+                        db,
+                        "SELECT COUNT("
+                                + MeasurementTables.DebugReportContract.ID
+                                + ") FROM "
+                                + MeasurementTables.DebugReportContract.TABLE
+                                + " WHERE "
+                                + MeasurementTables.DebugReportContract.ID
+                                + " = ?",
+                        new String[] {debugReport1.getId()}));
+    }
+
+    @Test
+    public void deleteExpiredRecords_VerboseDebugReportsWhileNotLimitingRetries() {
+        // Mocking that the retry Limiting Disable, but has limit number,
+        Flags mockFlags = Mockito.mock(Flags.class);
+        ExtendedMockito.doReturn(mockFlags).when(FlagsFactory::getFlags);
+        ExtendedMockito.doReturn(1).when(mockFlags).getMeasurementReportingRetryLimit();
+        ExtendedMockito.doReturn(false).when(mockFlags).getMeasurementReportingRetryLimitEnabled();
+
+        DebugReport debugReport = createDebugReport();
+        // Insert
+        mDatastoreManager.runInTransaction((dao) -> dao.insertDebugReport(debugReport));
+        // Increment Attempt
+        mDatastoreManager.runInTransaction(
+                (dao) ->
+                        dao.incrementAndGetReportingRetryCount(
+                                debugReport.getId(), DataType.DEBUG_REPORT_RETRY_COUNT));
+        // Delete Expired
+        mDatastoreManager.runInTransaction(dao -> dao.deleteExpiredRecords(0, 0));
+        try (Cursor cursor =
+                MeasurementDbHelper.getInstance(sContext)
+                        .getReadableDatabase()
+                        .query(
+                                MeasurementTables.DebugReportContract.TABLE,
+                                null,
+                                null,
+                                null,
+                                null,
+                                null,
+                                null)) {
+            // Assert Record not removed while Limiting Disabled
+            assertTrue(cursor.moveToNext());
+            DebugReport report = SqliteObjectMapper.constructDebugReportFromCursor(cursor);
+            assertNotNull(report);
+        }
+    }
+
+    @Test
     public void getRegistrationRedirectCount_keyMissing() {
         Optional<KeyValueData> optKeyValueData =
                 mDatastoreManager.runInTransactionWithResult(