diff options
Diffstat (limited to 'adservices/service-core/java/com/android/adservices/service/topics')
12 files changed, 751 insertions, 175 deletions
diff --git a/adservices/service-core/java/com/android/adservices/service/topics/AppUpdateManager.java b/adservices/service-core/java/com/android/adservices/service/topics/AppUpdateManager.java index 22eb558dab..2fc29f2914 100644 --- a/adservices/service-core/java/com/android/adservices/service/topics/AppUpdateManager.java +++ b/adservices/service-core/java/com/android/adservices/service/topics/AppUpdateManager.java @@ -20,10 +20,12 @@ import android.annotation.NonNull; import android.content.Context; import android.content.pm.ApplicationInfo; import android.content.pm.PackageManager; +import android.database.sqlite.SQLiteDatabase; import android.net.Uri; import android.util.Pair; import com.android.adservices.LogUtil; +import com.android.adservices.data.DbHelper; import com.android.adservices.data.topics.Topic; import com.android.adservices.data.topics.TopicsDao; import com.android.adservices.data.topics.TopicsTables; @@ -31,9 +33,8 @@ import com.android.adservices.service.Flags; import com.android.adservices.service.FlagsFactory; import com.android.internal.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; - import java.util.ArrayList; +import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -51,8 +52,8 @@ import java.util.stream.Collectors; * * <p>See go/rb-topics-app-update for details. */ -// TODO(b/239553255): Use transaction for methods have both read and write to the database. public class AppUpdateManager { + private static final String EMPTY_SDK = ""; private static AppUpdateManager sSingleton; // Tables that needs to be wiped out for application data @@ -77,11 +78,17 @@ public class AppUpdateManager { TopicsTables.AppUsageHistoryContract.APP) }; + private final DbHelper mDbHelper; private final TopicsDao mTopicsDao; private final Random mRandom; private final Flags mFlags; - AppUpdateManager(@NonNull TopicsDao topicsDao, @NonNull Random random, @NonNull Flags flags) { + AppUpdateManager( + @NonNull DbHelper dbHelper, + @NonNull TopicsDao topicsDao, + @NonNull Random random, + @NonNull Flags flags) { + mDbHelper = dbHelper; mTopicsDao = topicsDao; mRandom = random; mFlags = flags; @@ -99,6 +106,7 @@ public class AppUpdateManager { if (sSingleton == null) { sSingleton = new AppUpdateManager( + DbHelper.getInstance(context), TopicsDao.getInstance(context), new Random(), FlagsFactory.getFlags()); @@ -109,30 +117,77 @@ public class AppUpdateManager { } /** - * Delete application data for a specific application. + * Handle application uninstallation for Topics API. * - * <p>This method allows other usages besides daily maintenance job, such as real-time data - * wiping for an app uninstallation. + * <ul> + * <li>Delete all derived data for an uninstalled app. + * <li>When the feature is enabled, remove a topic if it has the uninstalled app as the only + * contributor in an epoch. + * </ul> * - * @param apps a {@link List} of applications to wipe data for + * @param packageUri The {@link Uri} got from Broadcast Intent + * @param currentEpochId the epoch id of current Epoch */ - public void deleteAppDataFromTableByApps(@NonNull List<String> apps) { - for (Pair<String, String> tableColumnNamePair : TABLE_INFO_TO_ERASE_APP_DATA) { - mTopicsDao.deleteAppFromTable( - tableColumnNamePair.first, tableColumnNamePair.second, apps); + public void handleAppUninstallationInRealTime(@NonNull Uri packageUri, long currentEpochId) { + String packageName = convertUriToAppName(packageUri); + + SQLiteDatabase db = mDbHelper.safeGetWritableDatabase(); + if (db == null) { + LogUtil.e( + "Database is not available, Stop processing app uninstallation for %s!", + packageName); + return; } - LogUtil.v("Have deleted data for application " + apps); + // This cross db and java boundaries multiple times, so we need to have a db transaction. + db.beginTransaction(); + + try { + if (supportsTopicContributorFeature()) { + handleTopTopicsWithoutContributors(currentEpochId, packageName); + } + + deleteAppDataFromTableByApps(List.of(packageName)); + + // Mark the transaction successful. + db.setTransactionSuccessful(); + } finally { + db.endTransaction(); + LogUtil.d("End of processing app uninstallation for %s", packageName); + } } /** - * Delete application data for a specific application. + * Handle application installation for Topics API. + * + * <p>Assign topics to past epochs for the installed app. * * @param packageUri The {@link Uri} got from Broadcast Intent + * @param currentEpochId the epoch id of current Epoch */ - public void deleteAppDataByUri(@NonNull Uri packageUri) { - String appName = convertUriToAppName(packageUri); - deleteAppDataFromTableByApps(List.of(appName)); + public void handleAppInstallationInRealTime(@NonNull Uri packageUri, long currentEpochId) { + String packageName = convertUriToAppName(packageUri); + + SQLiteDatabase db = mDbHelper.safeGetWritableDatabase(); + if (db == null) { + LogUtil.e( + "Database is not available, Stop processing app installation for %s", + packageName); + return; + } + + // This cross db and java boundaries multiple times, so we need to have a db transaction. + db.beginTransaction(); + + try { + assignTopicsToNewlyInstalledApps(packageName, currentEpochId); + + // Mark the transaction successful. + db.setTransactionSuccessful(); + } finally { + db.endTransaction(); + LogUtil.d("End of processing app installation for %s", packageName); + } } /** @@ -149,8 +204,9 @@ public class AppUpdateManager { * </ul> * * @param context the context + * @param currentEpochId epoch ID of current epoch */ - public void reconcileUninstalledApps(@NonNull Context context) { + public void reconcileUninstalledApps(@NonNull Context context, long currentEpochId) { Set<String> currentInstalledApps = getCurrentInstalledApps(context); Set<String> unhandledUninstalledApps = getUnhandledUninstalledApps(currentInstalledApps); if (unhandledUninstalledApps.isEmpty()) { @@ -160,8 +216,25 @@ public class AppUpdateManager { LogUtil.v( "Detect below unhandled mismatched applications: %s", unhandledUninstalledApps.toString()); - handleUninstalledApps(unhandledUninstalledApps); - LogUtil.v("App uninstallation reconciliation is finished!"); + + SQLiteDatabase db = mDbHelper.safeGetWritableDatabase(); + if (db == null) { + LogUtil.e("Database is not available, Stop reconciling app uninstallation in Topics!"); + return; + } + + // This cross db and java boundaries multiple times, so we need to have a db transaction. + db.beginTransaction(); + + try { + handleUninstalledAppsInReconciliation(unhandledUninstalledApps, currentEpochId); + + // Mark the transaction successful. + db.setTransactionSuccessful(); + } finally { + db.endTransaction(); + LogUtil.v("App uninstallation reconciliation in Topics is finished!"); + } } /** @@ -191,25 +264,30 @@ public class AppUpdateManager { LogUtil.v( "Detect below unhandled installed applications: %s", unhandledInstalledApps.toString()); - handleInstalledApps(unhandledInstalledApps, currentEpochId); - LogUtil.v("App installation reconciliation is finished!"); - } - /** - * An overloading method to allow passing in Uri instead of app name in string format. - * - * <p>For newly installed app, to allow it get topics in current epoch, one of top topics in - * past epochs will be assigned to this app. - * - * <p>See more details in go/rb-topics-app-update - * - * @param packageUri the Uri of newly installed application - * @param currentEpochId current epoch id - */ - public void assignTopicsToNewlyInstalledApps(@NonNull Uri packageUri, long currentEpochId) { - assignTopicsToNewlyInstalledApps(convertUriToAppName(packageUri), currentEpochId); + SQLiteDatabase db = mDbHelper.safeGetWritableDatabase(); + if (db == null) { + LogUtil.e("Database is not available, Stop reconciling app installation in Topics!"); + return; + } + + // This cross db and java boundaries multiple times, so we need to have a db transaction. + db.beginTransaction(); + + try { + handleInstalledAppsInReconciliation(unhandledInstalledApps, currentEpochId); + + // Mark the transaction successful. + db.setTransactionSuccessful(); + } finally { + db.endTransaction(); + LogUtil.v("App installation reconciliation in Topics is finished!"); + } } + // TODO(b/256703300): Currently we handled app-sdk topic assignments in serving flow. Move the + // logic back to app installation after we can get all SDKs when an app is + // installed. /** * For a newly installed app, in case SDKs that this app uses are not known when the app is * installed, the returned topic for an SDK can only be assigned when user calls getTopic(). @@ -231,7 +309,7 @@ public class AppUpdateManager { } int numberOfLookBackEpochs = mFlags.getTopicsNumberOfLookBackEpochs(); - Pair<String, String> appOnlyCaller = Pair.create(app, /* sdk */ ""); + Pair<String, String> appOnlyCaller = Pair.create(app, EMPTY_SDK); Pair<String, String> appSdkCaller = Pair.create(app, sdk); // Get ReturnedTopics and CallerCanLearnTopics for past epochs in @@ -286,32 +364,220 @@ public class AppUpdateManager { /** * Generating a random topic from given top topic list * - * @param topTopics a {@link List} of top topics in current epoch - * @param numberOfTopTopics the number of regular top topics - * @param numberOfRandomTopics the number of random top topics + * @param regularTopics a {@link List} of non-random topics in current epoch, excluding those + * which have no contributors + * @param randomTopics a {@link List} of random top topics * @param percentageForRandomTopic the probability to select random object * @return a selected {@link Topic} to be assigned to newly installed app */ + @VisibleForTesting @NonNull - public Topic selectAssignedTopicFromTopTopics( - @NonNull List<Topic> topTopics, - int numberOfTopTopics, - int numberOfRandomTopics, + Topic selectAssignedTopicFromTopTopics( + @NonNull List<Topic> regularTopics, + @NonNull List<Topic> randomTopics, int percentageForRandomTopic) { - // Validate the Top Topics are combined with correct number of topics and random topics - Preconditions.checkArgument(numberOfTopTopics + numberOfRandomTopics == topTopics.size()); - // If random number is in [0, randomPercentage - 1], a random topic will be selected. boolean shouldSelectRandomTopic = mRandom.nextInt(100) < percentageForRandomTopic; - if (shouldSelectRandomTopic) { - // Generate a random number to pick one of random topics. - // Random topics' index starts from numberOfTopTopics - return topTopics.get(numberOfTopTopics + mRandom.nextInt(numberOfRandomTopics)); + return shouldSelectRandomTopic + ? randomTopics.get(mRandom.nextInt(randomTopics.size())) + : regularTopics.get(mRandom.nextInt(regularTopics.size())); + } + + /** + * Delete application data for a specific application. + * + * <p>This method allows other usages besides daily maintenance job, such as real-time data + * wiping for an app uninstallation. + * + * @param apps a {@link List} of applications to wipe data for + */ + @VisibleForTesting + void deleteAppDataFromTableByApps(@NonNull List<String> apps) { + List<Pair<String, String>> tableToEraseData = + Arrays.stream(TABLE_INFO_TO_ERASE_APP_DATA).collect(Collectors.toList()); + if (supportsTopicContributorFeature()) { + tableToEraseData.add( + Pair.create( + TopicsTables.TopicContributorsContract.TABLE, + TopicsTables.TopicContributorsContract.APP)); + } + + mTopicsDao.deleteFromTableByColumn( + /* tableNamesAndColumnNamePairs */ tableToEraseData, /* valuesToDelete */ apps); + + LogUtil.v("Have deleted data for application " + apps); + } + + /** + * Assign a top Topic for the newly installed app. This allows SDKs in the newly installed app + * to get the past 3 epochs' topics if they did observe the topic in the past. + * + * <p>See more details in go/rb-topics-app-update + * + * @param app the app package name of newly installed application + * @param currentEpochId current epoch id + */ + @VisibleForTesting + void assignTopicsToNewlyInstalledApps(@NonNull String app, long currentEpochId) { + Objects.requireNonNull(app); + + final int numberOfEpochsToAssignTopics = mFlags.getTopicsNumberOfLookBackEpochs(); + final int numberOfTopTopics = mFlags.getTopicsNumberOfTopTopics(); + final int topicsPercentageForRandomTopic = mFlags.getTopicsPercentageForRandomTopic(); + + Pair<String, String> appOnlyCaller = Pair.create(app, EMPTY_SDK); + + // For each past epoch, assign a random topic to this newly installed app. + // The assigned topic should align the probability with rule to generate top topics. + for (long epochId = currentEpochId - 1; + epochId >= currentEpochId - numberOfEpochsToAssignTopics && epochId >= 0; + epochId--) { + List<Topic> topTopics = mTopicsDao.retrieveTopTopics(epochId); + + if (topTopics.isEmpty()) { + LogUtil.v( + "Empty top topic list in Epoch %d, do not assign topic to App %s.", + epochId, app); + continue; + } + + // Regular Topics are placed at the beginning of top topic list. + List<Topic> regularTopics = topTopics.subList(0, numberOfTopTopics); + // If enabled, filter out topics without contributors. + if (supportsTopicContributorFeature()) { + regularTopics = filterRegularTopicsWithoutContributors(regularTopics, epochId); + } + List<Topic> randomTopics = topTopics.subList(numberOfTopTopics, topTopics.size()); + + if (regularTopics.isEmpty() && randomTopics.isEmpty()) { + LogUtil.v( + "No topic is available to assign in Epoch %d, do not assign topic to App" + + " %s.", + epochId, app); + continue; + } + + Topic assignedTopic = + selectAssignedTopicFromTopTopics( + regularTopics, randomTopics, topicsPercentageForRandomTopic); + + // Persist this topic to database as returned topic in this epoch + mTopicsDao.persistReturnedAppTopicsMap(epochId, Map.of(appOnlyCaller, assignedTopic)); + + LogUtil.v( + "Topic %s has been assigned to newly installed App %s in Epoch %d", + assignedTopic.getTopic(), app, epochId); } + } - // Regular top topics start from index 0 - return topTopics.get(mRandom.nextInt(numberOfTopTopics)); + /** + * When an app is uninstalled, we need to check whether any of its classified topics has no + * contributors on epoch basis for past epochs to look back. Note in an epoch, an app is a + * contributor to a topic if the app has called Topics API in this epoch and is classified to + * the topic. + * + * <p>If such topic exists, remove this topic from ReturnedTopicsTable in the epoch. This method + * is invoked before {@code deleteAppDataFromTableByApps}, so the uninstalled app will be + * cleared in TopicContributors Table there. + * + * <p>NOTE: We are only interested in the epochs which will be used for getTopics(), i.e. past + * numberOfLookBackEpochs epochs. + * + * @param currentEpochId the id of epoch when the method gets invoked + * @param uninstalledApp the newly uninstalled app + */ + @VisibleForTesting + void handleTopTopicsWithoutContributors(long currentEpochId, @NonNull String uninstalledApp) { + // This check is on epoch basis for past epochs to look back + for (long epochId = currentEpochId - 1; + epochId >= currentEpochId - mFlags.getTopicsNumberOfLookBackEpochs() + && epochId >= 0; + epochId--) { + Map<String, List<Topic>> appClassificationTopics = + mTopicsDao.retrieveAppClassificationTopics(epochId); + List<Topic> topTopics = mTopicsDao.retrieveTopTopics(epochId); + Map<Integer, Set<String>> topTopicsToContributorsMap = + mTopicsDao.retrieveTopicToContributorsMap(epochId); + + List<Topic> classifiedTopics = + appClassificationTopics.getOrDefault(uninstalledApp, new ArrayList<>()); + // Collect all top topics to delete to make only one Db Update + List<String> topTopicsToDelete = + classifiedTopics.stream() + .filter( + classifiedTopic -> + topTopics.contains(classifiedTopic) + && topTopicsToContributorsMap.containsKey( + classifiedTopic.getTopic()) + // Filter out the topic that has ONLY + // the uninstalled app as a contributor + && topTopicsToContributorsMap + .get(classifiedTopic.getTopic()) + .size() + == 1 + && topTopicsToContributorsMap + .get(classifiedTopic.getTopic()) + .contains(uninstalledApp)) + .map(Topic::getTopic) + .map(String::valueOf) + .collect(Collectors.toList()); + + if (!topTopicsToDelete.isEmpty()) { + LogUtil.v( + "Topics %s will not have contributors at epoch %d. Delete them in" + + " epoch %d", + topTopicsToDelete, epochId, epochId); + } + + mTopicsDao.deleteEntriesFromTableByColumnWithEqualCondition( + List.of( + Pair.create( + TopicsTables.ReturnedTopicContract.TABLE, + TopicsTables.ReturnedTopicContract.TOPIC)), + topTopicsToDelete, + TopicsTables.ReturnedTopicContract.EPOCH_ID, + String.valueOf(epochId), + /* isStringEqualConditionColumnValue */ false); + } + } + + /** + * Filter out regular topics without any contributors. Note in an epoch, an app is a contributor + * to a topic if the app has called Topics API in this epoch and is classified to the topic. + * + * <p>For padded Topics (Classifier randomly pads top topics if they are not enough), as we put + * {@link EpochManager#PADDED_TOP_TOPICS_STRING} into TopicContributors Map, padded topics + * actually have "contributor" PADDED_TOP_TOPICS_STRING. Therefore, they won't be filtered out. + * + * @param regularTopics non-random top topics + * @param epochId epochId of current epoch + * @return the filtered regular topics + */ + @NonNull + @VisibleForTesting + List<Topic> filterRegularTopicsWithoutContributors( + @NonNull List<Topic> regularTopics, long epochId) { + Map<Integer, Set<String>> topicToContributorMap = + mTopicsDao.retrieveTopicToContributorsMap(epochId); + return regularTopics.stream() + .filter( + regularTopic -> + topicToContributorMap.containsKey(regularTopic.getTopic()) + && !topicToContributorMap + .get(regularTopic.getTopic()) + .isEmpty()) + .collect(Collectors.toList()); + } + + /** + * Check whether TopContributors Feature is enabled. It's enabled only when TopicContributors + * table is supported and the feature flag is on. + */ + @VisibleForTesting + boolean supportsTopicContributorFeature() { + return mFlags.getEnableTopicContributorsCheck() + && mDbHelper.supportsTopicContributorsTable(); } // An app will be regarded as an unhandled uninstalled app if it has an entry in any epoch of @@ -368,7 +634,8 @@ public class AppUpdateManager { // Get current installed applications from package manager @NonNull - private Set<String> getCurrentInstalledApps(Context context) { + @VisibleForTesting + Set<String> getCurrentInstalledApps(Context context) { PackageManager packageManager = context.getPackageManager(); List<ApplicationInfo> appInfoList = packageManager.getInstalledApplications( @@ -377,73 +644,46 @@ public class AppUpdateManager { return appInfoList.stream().map(appInfo -> appInfo.packageName).collect(Collectors.toSet()); } + /** + * Get App Package Name from a Uri. + * + * <p>Across PPAPI, package Uri is in the form of "package:com.example.adservices.sampleapp". + * "package" is a scheme of Uri and "com.example.adservices.sampleapp" is the app package name. + * Topics API persists app package name into database so this method extracts it from a Uri. + * + * @param packageUri the {@link Uri} of a package + * @return the app package name + */ + @VisibleForTesting + @NonNull + String convertUriToAppName(@NonNull Uri packageUri) { + return packageUri.getSchemeSpecificPart(); + } + // Handle Uninstalled applications that still have derived data in database // - // Currently, simply wipe out these data in the database for an app. i.e. Deleting all - // derived data from all tables that are related to app (has app column) - private void handleUninstalledApps(@NonNull Set<String> newlyUninstalledApps) { - deleteAppDataFromTableByApps(new ArrayList<>(newlyUninstalledApps)); + // 1) Delete all derived data for an uninstalled app. + // 2) Remove a topic if it has the uninstalled app as the only contributor in an epoch. In an + // epoch, an app is a contributor to a topic if the app has called Topics API in this epoch and + // is classified to the topic. + private void handleUninstalledAppsInReconciliation( + @NonNull Set<String> newlyUninstalledApps, long currentEpochId) { + for (String app : newlyUninstalledApps) { + if (supportsTopicContributorFeature()) { + handleTopTopicsWithoutContributors(currentEpochId, app); + } + + deleteAppDataFromTableByApps(List.of(app)); + } } // Handle newly installed applications // // Assign topics as real-time service to the app only, if the app isn't assigned with topics. - private void handleInstalledApps(@NonNull Set<String> newlyInstalledApps, long currentEpochId) { + private void handleInstalledAppsInReconciliation( + @NonNull Set<String> newlyInstalledApps, long currentEpochId) { for (String newlyInstalledApp : newlyInstalledApps) { assignTopicsToNewlyInstalledApps(newlyInstalledApp, currentEpochId); } } - - // - // For newly installed app, to allow it get topics in current epoch, one of top topics in past - // epochs will be assigned to this app. - // - // See more details in go/rb-topics-app-update - private void assignTopicsToNewlyInstalledApps(@NonNull String app, long currentEpochId) { - Objects.requireNonNull(app); - - // Read topics related configurations from Flags - final int numberOfEpochsToAssignTopics = mFlags.getTopicsNumberOfLookBackEpochs(); - final int topicsNumberOfTopTopics = mFlags.getTopicsNumberOfTopTopics(); - final int topicsNumberOfRandomTopics = mFlags.getTopicsNumberOfRandomTopics(); - final int topicsPercentageForRandomTopic = mFlags.getTopicsPercentageForRandomTopic(); - - Pair<String, String> appOnlyCaller = Pair.create(app, /* sdk */ ""); - - // For each past epoch, assign a random topic to this newly installed app. - // The assigned topic should align the probability with rule to generate top topics. - for (long epochId = currentEpochId - 1; - epochId >= currentEpochId - numberOfEpochsToAssignTopics && epochId >= 0; - epochId--) { - List<Topic> topTopics = mTopicsDao.retrieveTopTopics(epochId); - - if (topTopics.isEmpty()) { - LogUtil.v( - "Empty top topic list in Epoch %d, do not assign topic to App %s in Epoch" - + "%d.", - epochId, app, epochId); - continue; - } - - Topic assignedTopic = - selectAssignedTopicFromTopTopics( - topTopics, - topicsNumberOfTopTopics, - topicsNumberOfRandomTopics, - topicsPercentageForRandomTopic); - - // Persist this topic to database as returned topic in this epoch - mTopicsDao.persistReturnedAppTopicsMap(epochId, Map.of(appOnlyCaller, assignedTopic)); - - LogUtil.v( - "Topic %s has been assigned to newly installed App %s in Epoch %d", - assignedTopic.getTopic(), app, epochId); - } - } - - // packageUri.toString() has only app name, without "package:" in the front, i.e. it'll be like - // "com.example.adservices.sampleapp". - private String convertUriToAppName(@NonNull Uri packageUri) { - return packageUri.getSchemeSpecificPart(); - } } diff --git a/adservices/service-core/java/com/android/adservices/service/topics/CacheManager.java b/adservices/service-core/java/com/android/adservices/service/topics/CacheManager.java index 91c9950a75..25358bf528 100644 --- a/adservices/service-core/java/com/android/adservices/service/topics/CacheManager.java +++ b/adservices/service-core/java/com/android/adservices/service/topics/CacheManager.java @@ -26,6 +26,9 @@ import com.android.adservices.data.topics.Topic; import com.android.adservices.data.topics.TopicsDao; import com.android.adservices.service.Flags; import com.android.adservices.service.FlagsFactory; +import com.android.adservices.service.stats.AdServicesLogger; +import com.android.adservices.service.stats.AdServicesLoggerImpl; +import com.android.adservices.service.stats.GetTopicsReportedStats; import com.android.internal.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; @@ -61,12 +64,15 @@ public class CacheManager implements Dumpable { private HashSet<Topic> mCachedBlockedTopics = new HashSet<>(); // HashSet<TopicId> private HashSet<Integer> mCachedBlockedTopicIds = new HashSet<>(); + private final AdServicesLogger mLogger; @VisibleForTesting - CacheManager(EpochManager epochManager, TopicsDao topicsDao, Flags flags) { + CacheManager( + EpochManager epochManager, TopicsDao topicsDao, Flags flags, AdServicesLogger logger) { mEpochManager = epochManager; mTopicsDao = topicsDao; mFlags = flags; + mLogger = logger; } /** Returns an instance of the CacheManager given a context. */ @@ -78,7 +84,8 @@ public class CacheManager implements Dumpable { new CacheManager( EpochManager.getInstance(context), TopicsDao.getInstance(context), - FlagsFactory.getFlags()); + FlagsFactory.getFlags(), + AdServicesLoggerImpl.getInstance()); } return sSingleton; } @@ -139,13 +146,20 @@ public class CacheManager implements Dumpable { Set<Integer> topicsSet = new HashSet<>(); mReadWriteLock.readLock().lock(); + int duplicateTopicCount = 0, blockedTopicCount = 0; try { for (int numEpoch = 0; numEpoch < numberOfLookBackEpochs; numEpoch++) { if (mCachedTopics.containsKey(epochId - numEpoch)) { Topic topic = mCachedTopics.get(epochId - numEpoch).get(Pair.create(app, sdk)); - if (topic != null - && !topicsSet.contains(topic.getTopic()) - && !mCachedBlockedTopicIds.contains(topic.getTopic())) { + if (topic != null) { + if (topicsSet.contains(topic.getTopic())) { + duplicateTopicCount++; + continue; + } + if (mCachedBlockedTopicIds.contains(topic.getTopic())) { + blockedTopicCount++; + continue; + } topics.add(topic); topicsSet.add(topic.getTopic()); } @@ -156,6 +170,19 @@ public class CacheManager implements Dumpable { } Collections.shuffle(topics, random); + + // Log GetTopics stats. + ImmutableList.Builder<Integer> topicIds = ImmutableList.builder(); + for (Topic topic : topics) { + topicIds.add(topic.getTopic()); + } + mLogger.logGetTopicsReportedStats( + GetTopicsReportedStats.builder() + .setDuplicateTopicCount(duplicateTopicCount) + .setFilteredBlockedTopicCount(blockedTopicCount) + .setTopicIdsCount(topics.size()) + .build()); + return topics; } @@ -173,6 +200,42 @@ public class CacheManager implements Dumpable { } /** + * Get cached topics within certain epoch range. This is a helper method to get cached topics + * for an app-sdk caller, without considering other constraints, like UI blocking logic. + * + * @param epochLowerBound the earliest epoch to include cached topics from + * @param epochUpperBound the latest epoch to included cached topics to + * @param app the app + * @param sdk the sdk. In case the app calls the Topics API directly, the sdk == empty string. + * @return {@link List<Topic>} a list of Topics between {@code epochLowerBound} and {@code + * epochUpperBound}. + */ + @NonNull + public List<Topic> getTopicsInEpochRange( + long epochLowerBound, long epochUpperBound, @NonNull String app, @NonNull String sdk) { + List<Topic> topics = new ArrayList<>(); + // To deduplicate returned topics + Set<Integer> topicsSet = new HashSet<>(); + + mReadWriteLock.readLock().lock(); + try { + for (long epochId = epochLowerBound; epochId <= epochUpperBound; epochId++) { + if (mCachedTopics.containsKey(epochId)) { + Topic topic = mCachedTopics.get(epochId).get(Pair.create(app, sdk)); + if (topic != null && !topicsSet.contains(topic.getTopic())) { + topics.add(topic); + topicsSet.add(topic.getTopic()); + } + } + } + } finally { + mReadWriteLock.readLock().unlock(); + } + + return topics; + } + + /** * Gets a list of all topics that could be returned to the user in the last * numberOfLookBackEpochs epochs. Does not include the current epoch, so range is * [currentEpochId - numberOfLookBackEpochs, currentEpochId - 1]. diff --git a/adservices/service-core/java/com/android/adservices/service/topics/EpochJobService.java b/adservices/service-core/java/com/android/adservices/service/topics/EpochJobService.java index 5b28e40cf5..a0aeadf0ed 100644 --- a/adservices/service-core/java/com/android/adservices/service/topics/EpochJobService.java +++ b/adservices/service-core/java/com/android/adservices/service/topics/EpochJobService.java @@ -32,6 +32,7 @@ import com.android.adservices.LogUtil; import com.android.adservices.concurrency.AdServicesExecutors; import com.android.adservices.service.FlagsFactory; +import com.google.common.annotations.VisibleForTesting; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -91,7 +92,8 @@ public final class EpochJobService extends JobService { return false; } - private static void schedule( + @VisibleForTesting + static void schedule( Context context, @NonNull JobScheduler jobScheduler, long epochJobPeriodMs, @@ -101,6 +103,7 @@ public final class EpochJobService extends JobService { TOPICS_EPOCH_JOB_ID, new ComponentName(context, EpochJobService.class)) .setRequiresCharging(true) + .setPersisted(true) .setPeriodic(epochJobPeriodMs, epochJobFlexMs) .build(); diff --git a/adservices/service-core/java/com/android/adservices/service/topics/EpochManager.java b/adservices/service-core/java/com/android/adservices/service/topics/EpochManager.java index 2e02c220f2..e712447808 100644 --- a/adservices/service-core/java/com/android/adservices/service/topics/EpochManager.java +++ b/adservices/service-core/java/com/android/adservices/service/topics/EpochManager.java @@ -50,17 +50,6 @@ import java.util.Set; /** A class to manage Epoch computation. */ public class EpochManager implements Dumpable { - // TODO(b/223915674): make this configurable. - // The Top Topics will have 6 topics. - // The first 5 topics are the Top Topics derived by ML, and the 6th is a random topic from - // taxonomy. - // The index starts from 0. - private static final int RANDOM_TOPIC_INDEX = 5; - - // TODO(b/223916172): make this configurable. - // The number of top Topics not including the random one. - private static final int NUM_TOP_TOPICS_NOT_INCLUDING_RANDOM_ONE = 5; - // The tables to do garbage collection for old epochs // and its corresponding epoch_id column name. // Pair<Table Name, Column Name> @@ -86,6 +75,28 @@ public class EpochManager implements Dumpable { TopicsTables.AppUsageHistoryContract.EPOCH_ID) }; + /** + * The string to annotate that the topic is a padded topic in {@code TopicContributors} table. + * After the computation of {@code TopicContributors} table, if there is a top topic without + * contributors, it must be a padded topic. Persist {@code Entry{Topic, + * PADDED_TOP_TOPICS_STRING}} into {@code TopicContributors} table. + * + * <p>The reason to persist {@code Entry{Topic, PADDED_TOP_TOPICS_STRING}} is because topics + * need to be assigned to newly installed app. Moreover, non-random top topics without + * contributors, due to app uninstallations, are filtered out as candidate topics to assign + * with. Generally, a padded topic should have no contributors, but it should NOT be filtered + * out as a non-random top topics without contributors. Based on these facts, {@code + * Entry{Topic, PADDED_TOP_TOPICS_STRING}} is persisted to annotate that do NOT remove this + * padded topic though it has no contributors. + * + * <p>Put a "!" at last to avoid a spoof app to name itself with {@code + * PADDED_TOP_TOPICS_STRING}. Refer to + * https://developer.android.com/studio/build/configure-app-module, application name can only + * contain [a-zA-Z0-9_]. + */ + @VisibleForTesting + public static final String PADDED_TOP_TOPICS_STRING = "no_contributors_due_to_padding!"; + private static EpochManager sSingleton; private final TopicsDao mTopicsDao; @@ -138,6 +149,7 @@ public class EpochManager implements Dumpable { } // This cross db and java boundaries multiple times, so we need to have a db transaction. + LogUtil.d("Start of Epoch Computation"); db.beginTransaction(); long currentEpochId = getCurrentEpochId(); @@ -201,6 +213,17 @@ public class EpochManager implements Dumpable { // Then save Top Topics into DB mTopicsDao.persistTopTopics(currentEpochId, topTopics); + // Compute TopicToContributors mapping for top topics. In an epoch, an app is a + // contributor to a topic if the app has called Topics API in this epoch and is + // classified to the topic. + // Do this only when feature is enabled. + if (supportsTopicContributorFeature()) { + Map<Integer, Set<String>> topTopicsToContributorsMap = + computeTopTopicsToContributorsMap(appClassificationTopicsMap, topTopics); + // Then save Topic Contributors into DB + mTopicsDao.persistTopicContributors(currentEpochId, topTopicsToContributorsMap); + } + // Step 6: Assign topics to apps and SDK from the global top topics. // Currently, hard-code the taxonomyVersion and the modelVersion. // Return returnedAppSdkTopics = Map<Pair<App, Sdk>, Topic> @@ -218,6 +241,7 @@ public class EpochManager implements Dumpable { db.setTransactionSuccessful(); } finally { db.endTransaction(); + LogUtil.d("End of Epoch Computation"); } } @@ -412,14 +436,16 @@ public class EpochManager implements Dumpable { + mFlags.getTopicsNumberOfRandomTopics()); int random = mRandom.nextInt(100); + // First random topic would be after numberOfTopTopics. + int randomTopicIndex = mFlags.getTopicsNumberOfTopTopics(); // For 5%, get the random topic. if (random < mFlags.getTopicsPercentageForRandomTopic()) { // The random topic is the last one on the list. - return topTopics.get(RANDOM_TOPIC_INDEX); + return topTopics.get(randomTopicIndex); } - // For 95%, pick randomly one out of 5 top topics. - return topTopics.get(random % NUM_TOP_TOPICS_NOT_INCLUDING_RANDOM_ONE); + // For 95%, pick randomly one out of first n top topics. + return topTopics.get(random % randomTopicIndex); } // To garbage collect data for old epochs. @@ -433,6 +459,56 @@ public class EpochManager implements Dumpable { mTopicsDao.deleteDataOfOldEpochs( tableColumnPair.first, tableColumnPair.second, epochToDeleteFrom); } + + // Handle TopicContributors Table if feature flag is ON + if (supportsTopicContributorFeature()) { + mTopicsDao.deleteDataOfOldEpochs( + TopicsTables.TopicContributorsContract.TABLE, + TopicsTables.TopicContributorsContract.EPOCH_ID, + epochToDeleteFrom); + } + } + + // Compute the mapping of topic to its contributor apps. In an epoch, an app is a contributor to + // a topic if the app has called Topics API in this epoch and is classified to the topic. Only + // computed for top topics. + @VisibleForTesting + Map<Integer, Set<String>> computeTopTopicsToContributorsMap( + @NonNull Map<String, List<Topic>> appClassificationTopicsMap, + @NonNull List<Topic> topTopics) { + Map<Integer, Set<String>> topicToContributorMap = new HashMap<>(); + + for (Map.Entry<String, List<Topic>> appTopics : appClassificationTopicsMap.entrySet()) { + String app = appTopics.getKey(); + + for (Topic topic : appTopics.getValue()) { + // Only compute for top topics. + if (topTopics.contains(topic)) { + int topicId = topic.getTopic(); + topicToContributorMap.putIfAbsent(topicId, new HashSet<>()); + topicToContributorMap.get(topicId).add(app); + } + } + } + + // At last, check whether there is any top topics without contributors. If so, annotate it + // with PADDED_TOP_TOPICS_STRING in the map. See PADDED_TOP_TOPICS_STRING for more details. + for (int i = 0; i < mFlags.getTopicsNumberOfTopTopics(); i++) { + Topic topTopic = topTopics.get(i); + topicToContributorMap.putIfAbsent( + topTopic.getTopic(), Set.of(PADDED_TOP_TOPICS_STRING)); + } + + return topicToContributorMap; + } + + /** + * Check whether TopContributors Feature is enabled. It's enabled only when TopicContributors + * table is supported and the feature flag is on. + */ + public boolean supportsTopicContributorFeature() { + return mFlags.getEnableTopicContributorsCheck() + && mTopicsDao.supportsTopicContributorsTable(); } @Override diff --git a/adservices/service-core/java/com/android/adservices/service/topics/TopicsServiceImpl.java b/adservices/service-core/java/com/android/adservices/service/topics/TopicsServiceImpl.java index 393d51fba7..e66206fe97 100644 --- a/adservices/service-core/java/com/android/adservices/service/topics/TopicsServiceImpl.java +++ b/adservices/service-core/java/com/android/adservices/service/topics/TopicsServiceImpl.java @@ -15,7 +15,6 @@ */ package com.android.adservices.service.topics; - import static android.adservices.common.AdServicesStatusUtils.STATUS_BACKGROUND_CALLER; import static android.adservices.common.AdServicesStatusUtils.STATUS_CALLER_NOT_ALLOWED; import static android.adservices.common.AdServicesStatusUtils.STATUS_INTERNAL_ERROR; @@ -27,6 +26,7 @@ import static android.adservices.common.AdServicesStatusUtils.STATUS_USER_CONSEN import static com.android.adservices.service.stats.AdServicesStatsLog.AD_SERVICES_API_CALLED__API_CLASS__TARGETING; import static com.android.adservices.service.stats.AdServicesStatsLog.AD_SERVICES_API_CALLED__API_NAME__GET_TOPICS; +import static com.android.adservices.service.stats.AdServicesStatsLog.AD_SERVICES_API_CALLED__API_NAME__GET_TOPICS_PREVIEW_API; import android.adservices.common.AdServicesStatusUtils; import android.adservices.common.CallerMetadata; @@ -138,8 +138,10 @@ public class TopicsServiceImpl extends ITopicsService.Stub { callback.onResult(mTopicsWorker.getTopics(packageName, sdkName)); - mTopicsWorker.recordUsage( - topicsParam.getAppPackageName(), topicsParam.getSdkName()); + if (topicsParam.shouldRecordObservation()) { + mTopicsWorker.recordUsage( + topicsParam.getAppPackageName(), topicsParam.getSdkName()); + } } catch (RemoteException e) { LogUtil.e(e, "Unable to send result to the callback"); resultCode = STATUS_INTERNAL_ERROR; @@ -154,7 +156,10 @@ public class TopicsServiceImpl extends ITopicsService.Stub { new ApiCallStats.Builder() .setCode(AdServicesStatsLog.AD_SERVICES_API_CALLED) .setApiClass(AD_SERVICES_API_CALLED__API_CLASS__TARGETING) - .setApiName(AD_SERVICES_API_CALLED__API_NAME__GET_TOPICS) + .setApiName( + topicsParam.shouldRecordObservation() + ? AD_SERVICES_API_CALLED__API_NAME__GET_TOPICS + : AD_SERVICES_API_CALLED__API_NAME__GET_TOPICS_PREVIEW_API) .setAppPackageName(packageName) .setSdkPackageName(sdkName) .setLatencyMillisecond(apiLatency) @@ -264,7 +269,7 @@ public class TopicsServiceImpl extends ITopicsService.Stub { return false; } - AdServicesApiConsent userConsent = mConsentManager.getConsent(mContext.getPackageManager()); + AdServicesApiConsent userConsent = mConsentManager.getConsent(); if (!userConsent.isGiven()) { invokeCallbackWithStatus( callback, STATUS_USER_CONSENT_REVOKED, "User consent revoked."); @@ -283,7 +288,9 @@ public class TopicsServiceImpl extends ITopicsService.Stub { mContext, Process.isSdkSandboxUid(callingUid), topicsParam.getAppPackageName(), - enrollmentData.getEnrollmentId()); + enrollmentData.getEnrollmentId()) + && !mFlags.isEnrollmentBlocklisted(enrollmentData.getEnrollmentId()); + if (!permitted) { invokeCallbackWithStatus( callback, STATUS_CALLER_NOT_ALLOWED, "Caller is not authorized."); diff --git a/adservices/service-core/java/com/android/adservices/service/topics/TopicsWorker.java b/adservices/service-core/java/com/android/adservices/service/topics/TopicsWorker.java index b28ffe2845..7277281d9e 100644 --- a/adservices/service-core/java/com/android/adservices/service/topics/TopicsWorker.java +++ b/adservices/service-core/java/com/android/adservices/service/topics/TopicsWorker.java @@ -26,6 +26,7 @@ import android.net.Uri; import com.android.adservices.LogUtil; import com.android.adservices.data.topics.Topic; +import com.android.adservices.data.topics.TopicsTables; import com.android.adservices.service.Flags; import com.android.adservices.service.FlagsFactory; import com.android.internal.annotations.VisibleForTesting; @@ -271,12 +272,16 @@ public class TopicsWorker { /** * Delete all data generated by Topics API, except for tables in the exclusion list. * - * @param tablesToExclude a {@link List} of tables that won't be deleted. + * @param tablesToExclude an {@link ArrayList} of tables that won't be deleted. */ - public void clearAllTopicsData(@NonNull List<String> tablesToExclude) { + public void clearAllTopicsData(@NonNull ArrayList<String> tablesToExclude) { // Here we use Write lock to block Read during that computation time. mReadWriteLock.writeLock().lock(); try { + // Clear data for TopicContributors Table only when feature flag is supported + if (!mEpochManager.supportsTopicContributorFeature()) { + tablesToExclude.add(TopicsTables.TopicContributorsContract.TABLE); + } mCacheManager.clearAllTopicsData(tablesToExclude); loadCache(); @@ -301,24 +306,26 @@ public class TopicsWorker { public void reconcileApplicationUpdate(Context context) { mReadWriteLock.writeLock().lock(); try { - mAppUpdateManager.reconcileUninstalledApps(context); + mAppUpdateManager.reconcileUninstalledApps(context, mEpochManager.getCurrentEpochId()); mAppUpdateManager.reconcileInstalledApps(context, mEpochManager.getCurrentEpochId()); loadCache(); } finally { mReadWriteLock.writeLock().unlock(); + LogUtil.d("App Update Reconciliation is done!"); } } /** - * Delete derived data for a specific app + * Handle application uninstallation for Topics API. * * @param packageUri The {@link Uri} got from Broadcast Intent */ - public void deletePackageData(@NonNull Uri packageUri) { + public void handleAppUninstallation(@NonNull Uri packageUri) { mReadWriteLock.writeLock().lock(); try { - mAppUpdateManager.deleteAppDataByUri(packageUri); + mAppUpdateManager.handleAppUninstallationInRealTime( + packageUri, mEpochManager.getCurrentEpochId()); loadCache(); LogUtil.v("Derived data is cleared for %s", packageUri.toString()); @@ -335,7 +342,7 @@ public class TopicsWorker { public void handleAppInstallation(@NonNull Uri packageUri) { mReadWriteLock.writeLock().lock(); try { - mAppUpdateManager.assignTopicsToNewlyInstalledApps( + mAppUpdateManager.handleAppInstallationInRealTime( packageUri, mEpochManager.getCurrentEpochId()); loadCache(); @@ -349,7 +356,13 @@ public class TopicsWorker { // Handle topic assignment to SDK for newly installed applications. Cached topics need to be // reloaded if any topic assignment happens. - private void handleSdkTopicsAssignment(String app, String sdk) { + private void handleSdkTopicsAssignment(@NonNull String app, @NonNull String sdk) { + // Return if any topic has been assigned to this app-sdk. + List<Topic> existingTopics = getExistingTopicsForAppSdk(app, sdk); + if (!existingTopics.isEmpty()) { + return; + } + mReadWriteLock.writeLock().lock(); try { if (mAppUpdateManager.assignTopicsToSdkForAppInstallation( @@ -364,4 +377,27 @@ public class TopicsWorker { mReadWriteLock.writeLock().unlock(); } } + + // Get all existing topics from cache for a pair of app and sdk. + // The epoch range is [currentEpochId - numberOfLookBackEpochs, currentEpochId]. + @NonNull + private List<Topic> getExistingTopicsForAppSdk(@NonNull String app, @NonNull String sdk) { + List<Topic> existingTopics; + + mReadWriteLock.readLock().lock(); + // Get existing returned topics map for last 3 epochs and current epoch. + try { + long currentEpochId = mEpochManager.getCurrentEpochId(); + existingTopics = + mCacheManager.getTopicsInEpochRange( + currentEpochId - mFlags.getTopicsNumberOfLookBackEpochs(), + currentEpochId, + app, + sdk); + } finally { + mReadWriteLock.readLock().unlock(); + } + + return existingTopics == null ? new ArrayList<>() : existingTopics; + } } diff --git a/adservices/service-core/java/com/android/adservices/service/topics/classifier/ClassifierManager.java b/adservices/service-core/java/com/android/adservices/service/topics/classifier/ClassifierManager.java index 55809cb974..e395db77f4 100644 --- a/adservices/service-core/java/com/android/adservices/service/topics/classifier/ClassifierManager.java +++ b/adservices/service-core/java/com/android/adservices/service/topics/classifier/ClassifierManager.java @@ -26,6 +26,7 @@ import com.android.adservices.data.topics.Topic; import com.android.adservices.service.Flags; import com.android.adservices.service.Flags.ClassifierType; import com.android.adservices.service.FlagsFactory; +import com.android.adservices.service.stats.AdServicesLoggerImpl; import com.android.adservices.service.topics.PackageManagerUtil; import com.android.internal.annotations.VisibleForTesting; @@ -71,11 +72,13 @@ public class ClassifierManager implements Classifier { new Preprocessor(context), new PackageManagerUtil(context), new Random(), - ModelManager.getInstance(context))), + ModelManager.getInstance(context), + AdServicesLoggerImpl.getInstance())), Suppliers.memoize( () -> new PrecomputedClassifier( - ModelManager.getInstance(context)))); + ModelManager.getInstance(context), + AdServicesLoggerImpl.getInstance()))); } } return sSingleton; diff --git a/adservices/service-core/java/com/android/adservices/service/topics/classifier/CommonClassifierHelper.java b/adservices/service-core/java/com/android/adservices/service/topics/classifier/CommonClassifierHelper.java index 13e05eea54..0b59b706a1 100644 --- a/adservices/service-core/java/com/android/adservices/service/topics/classifier/CommonClassifierHelper.java +++ b/adservices/service-core/java/com/android/adservices/service/topics/classifier/CommonClassifierHelper.java @@ -21,6 +21,8 @@ import android.content.res.AssetManager; import com.android.adservices.LogUtil; import com.android.adservices.data.topics.Topic; +import com.android.adservices.service.stats.AdServicesLogger; +import com.android.adservices.service.stats.EpochComputationGetTopTopicsStats; import com.android.internal.util.Preconditions; import java.io.IOException; @@ -74,7 +76,7 @@ class CommonClassifierHelper { // This bytes[] has bytes in decimal format; // Convert it to hexadecimal format - for(int i = 0; i < bytes.length; i++) { + for (int i = 0; i < bytes.length; i++) { assetSha256CheckSum.append( Integer.toString((bytes[i] & 0xff) + 0x100, 16).substring(1)); } @@ -109,7 +111,8 @@ class CommonClassifierHelper { @NonNull List<Integer> labelIds, @NonNull Random random, @NonNull int numberOfTopTopics, - @NonNull int numberOfRandomTopics) { + @NonNull int numberOfRandomTopics, + @NonNull AdServicesLogger logger) { Preconditions.checkArgument( numberOfTopTopics > 0, "numberOfTopTopics should larger than 0"); Preconditions.checkArgument( @@ -126,7 +129,14 @@ class CommonClassifierHelper { // If there are no topic in the appTopics list, an empty topic list will be returned. if (topicsToAppTopicCount.isEmpty()) { LogUtil.w("Unable to retrieve any topics from device."); - + // Log atom for getTopTopics call. + logger.logEpochComputationGetTopTopicsStats( + EpochComputationGetTopTopicsStats.builder() + .setTopTopicCount(0) + .setPaddedRandomTopicsCount(0) + .setAppsConsideredCount(-1) + .setSdksConsideredCount(-1) + .build()); return new ArrayList<>(); } @@ -142,6 +152,16 @@ class CommonClassifierHelper { List<Topic> topTopics = allSortedTopics.subList(0, Math.min(numberOfTopTopics, allSortedTopics.size())); + // Log atom for getTopTopics call. + // TODO(b/256638889): Log apps and sdk considered count. + logger.logEpochComputationGetTopTopicsStats( + EpochComputationGetTopTopicsStats.builder() + .setTopTopicCount(numberOfTopTopics) + .setPaddedRandomTopicsCount(numberOfRandomPaddingTopics) + .setAppsConsideredCount(-1) + .setSdksConsideredCount(-1) + .build()); + // If the size of topTopics smaller than numberOfTopTopics, // the top topics list will be padded by numberOfRandomPaddingTopics random topics. return getRandomTopics( diff --git a/adservices/service-core/java/com/android/adservices/service/topics/classifier/ModelManager.java b/adservices/service-core/java/com/android/adservices/service/topics/classifier/ModelManager.java index f182492880..eadc892e00 100644 --- a/adservices/service-core/java/com/android/adservices/service/topics/classifier/ModelManager.java +++ b/adservices/service-core/java/com/android/adservices/service/topics/classifier/ModelManager.java @@ -91,7 +91,8 @@ public class ModelManager { private static final String ASSET_ELEMENT_NAME = "asset_name"; // The attributions of assets property in classifier_assets_metadata.json private static final Set<String> ASSETS_PROPERTY_ATTRIBUTIONS = - new HashSet(Arrays.asList("taxonomy_type", "taxonomy_version", "updated_date")); + new HashSet( + Arrays.asList("taxonomy_type", "taxonomy_version", "build_id", "updated_date")); // The attributions of assets metadata in classifier_assets_metadata.json private static final Set<String> ASSETS_NORMAL_ATTRIBUTIONS = new HashSet(Arrays.asList("asset_version", "path", "checksum", "updated_date")); @@ -179,9 +180,11 @@ public class ModelManager { return downloadedFiles; } - // Return true if Model Manager should uses downloaded model. Otherwise, use bundled model. + // Return true if Model Manager should use downloaded model. Otherwise, use bundled model. private boolean useDownloadedFiles() { - return mDownloadedFiles != null && mDownloadedFiles.size() > 0; + return mDownloadedFiles != null + && mDownloadedFiles.size() > 0 + && !FlagsFactory.getFlags().getClassifierForceUseBundledFiles(); } /** @@ -205,14 +208,36 @@ public class ModelManager { return buffer; } } else { - // Use bundled files. - AssetFileDescriptor fileDescriptor = mAssetManager.openFd(mModelFilePath); - FileInputStream inputStream = new FileInputStream(fileDescriptor.getFileDescriptor()); - FileChannel fileChannel = inputStream.getChannel(); + try { + // Use bundled files. + AssetFileDescriptor fileDescriptor = mAssetManager.openFd(mModelFilePath); + FileInputStream inputStream = + new FileInputStream(fileDescriptor.getFileDescriptor()); + FileChannel fileChannel = inputStream.getChannel(); + + long startOffset = fileDescriptor.getStartOffset(); + long declaredLength = fileDescriptor.getDeclaredLength(); + return fileChannel.map(FileChannel.MapMode.READ_ONLY, startOffset, declaredLength); + } catch (IOException | NullPointerException e) { + LogUtil.e(e, "Error loading the bundled classifier model"); + return ByteBuffer.allocate(0); + } + } + } - long startOffset = fileDescriptor.getStartOffset(); - long declaredLength = fileDescriptor.getDeclaredLength(); - return fileChannel.map(FileChannel.MapMode.READ_ONLY, startOffset, declaredLength); + /** Returns true if the classifier model is available for classification. */ + public boolean isModelAvailable() { + if (useDownloadedFiles()) { + // Downloaded model is always expected to be available. + return true; + } else { + // Check if the non-zero model file is present in the apk assets. + try { + return mAssetManager.openFd(mModelFilePath).getLength() > 0; + } catch (IOException e) { + LogUtil.e(e, "[ML] No classifier model available."); + return false; + } } } diff --git a/adservices/service-core/java/com/android/adservices/service/topics/classifier/OnDeviceClassifier.java b/adservices/service-core/java/com/android/adservices/service/topics/classifier/OnDeviceClassifier.java index af32594aa4..70c2a135a9 100644 --- a/adservices/service-core/java/com/android/adservices/service/topics/classifier/OnDeviceClassifier.java +++ b/adservices/service-core/java/com/android/adservices/service/topics/classifier/OnDeviceClassifier.java @@ -16,6 +16,10 @@ package com.android.adservices.service.topics.classifier; +import static com.android.adservices.service.stats.AdServicesStatsLog.AD_SERVICES_EPOCH_COMPUTATION_CLASSIFIER_REPORTED__CLASSIFIER_TYPE__ON_DEVICE_CLASSIFIER; +import static com.android.adservices.service.stats.AdServicesStatsLog.AD_SERVICES_EPOCH_COMPUTATION_CLASSIFIER_REPORTED__ON_DEVICE_CLASSIFIER_STATUS__ON_DEVICE_CLASSIFIER_STATUS_FAILURE; +import static com.android.adservices.service.stats.AdServicesStatsLog.AD_SERVICES_EPOCH_COMPUTATION_CLASSIFIER_REPORTED__ON_DEVICE_CLASSIFIER_STATUS__ON_DEVICE_CLASSIFIER_STATUS_SUCCESS; +import static com.android.adservices.service.stats.AdServicesStatsLog.AD_SERVICES_EPOCH_COMPUTATION_CLASSIFIER_REPORTED__PRECOMPUTED_CLASSIFIER_STATUS__PRECOMPUTED_CLASSIFIER_STATUS_NOT_INVOKED; import static com.android.adservices.service.topics.classifier.Preprocessor.limitDescriptionSize; import android.annotation.NonNull; @@ -23,6 +27,8 @@ import android.annotation.NonNull; import com.android.adservices.LogUtil; import com.android.adservices.data.topics.Topic; import com.android.adservices.service.FlagsFactory; +import com.android.adservices.service.stats.AdServicesLogger; +import com.android.adservices.service.stats.EpochComputationClassifierStats; import com.android.adservices.service.topics.AppInfo; import com.android.adservices.service.topics.PackageManagerUtil; @@ -54,6 +60,8 @@ public class OnDeviceClassifier implements Classifier { private static final String MODEL_ASSET_FIELD = "tflite_model"; private static final String LABELS_ASSET_FIELD = "labels_topics"; private static final String ASSET_VERSION_FIELD = "asset_version"; + private static final String VERSION_INFO_FIELD = "version_info"; + private static final String BUILD_ID_FIELD = "build_id"; private static final String NO_VERSION_INFO = "NO_VERSION_INFO"; @@ -66,20 +74,24 @@ public class OnDeviceClassifier implements Classifier { private ImmutableList<Integer> mLabels; private long mModelVersion; private long mLabelsVersion; + private int mBuildId; private boolean mLoaded; private ImmutableMap<String, AppInfo> mAppInfoMap; + private final AdServicesLogger mLogger; OnDeviceClassifier( @NonNull Preprocessor preprocessor, @NonNull PackageManagerUtil packageManagerUtil1, @NonNull Random random, - @NonNull ModelManager modelManager) { + @NonNull ModelManager modelManager, + @NonNull AdServicesLogger logger) { mPreprocessor = preprocessor; mPackageManagerUtil = packageManagerUtil1; mRandom = random; mLoaded = false; mAppInfoMap = ImmutableMap.of(); mModelManager = modelManager; + mLogger = logger; } @Override @@ -89,6 +101,12 @@ public class OnDeviceClassifier implements Classifier { return ImmutableMap.of(); } + if (!mModelManager.isModelAvailable()) { + // Return empty map since no model is available. + LogUtil.d("[ML] No ML model available for classification. Return empty Map."); + return ImmutableMap.of(); + } + // Load the assets if not loaded already. if (!isLoaded()) { mLoaded = load(); @@ -104,6 +122,7 @@ public class OnDeviceClassifier implements Classifier { for (String appPackageName : appPackageNames) { String appDescription = getProcessedAppDescription(appPackageName); List<Topic> appClassificationTopics = getAppClassificationTopics(appDescription); + logEpochComputationClassifierStats(appClassificationTopics); LogUtil.v( "[ML] Top classification for app description \"" + appDescription @@ -115,6 +134,28 @@ public class OnDeviceClassifier implements Classifier { return packageNameToTopics.build(); } + private void logEpochComputationClassifierStats(List<Topic> topics) { + // Log atom for getTopTopics call. + ImmutableList.Builder<Integer> topicIds = ImmutableList.builder(); + for (Topic topic : topics) { + topicIds.add(topic.getTopic()); + } + mLogger.logEpochComputationClassifierStats( + EpochComputationClassifierStats.builder() + .setTopicIds(topicIds.build()) + .setBuildId(mBuildId) + .setAssetVersion(Long.toString(mModelVersion)) + .setClassifierType( + AD_SERVICES_EPOCH_COMPUTATION_CLASSIFIER_REPORTED__CLASSIFIER_TYPE__ON_DEVICE_CLASSIFIER) + .setOnDeviceClassifierStatus( + topics.isEmpty() + ? AD_SERVICES_EPOCH_COMPUTATION_CLASSIFIER_REPORTED__ON_DEVICE_CLASSIFIER_STATUS__ON_DEVICE_CLASSIFIER_STATUS_FAILURE + : AD_SERVICES_EPOCH_COMPUTATION_CLASSIFIER_REPORTED__ON_DEVICE_CLASSIFIER_STATUS__ON_DEVICE_CLASSIFIER_STATUS_SUCCESS) + .setPrecomputedClassifierStatus( + AD_SERVICES_EPOCH_COMPUTATION_CLASSIFIER_REPORTED__PRECOMPUTED_CLASSIFIER_STATUS__PRECOMPUTED_CLASSIFIER_STATUS_NOT_INVOKED) + .build()); + } + @Override @NonNull public List<Topic> getTopTopics( @@ -125,7 +166,7 @@ public class OnDeviceClassifier implements Classifier { } return CommonClassifierHelper.getTopTopics( - appTopics, mLabels, mRandom, numberOfTopTopics, numberOfRandomTopics); + appTopics, mLabels, mRandom, numberOfTopTopics, numberOfRandomTopics, mLogger); } // Uses the BertNLClassifier to fetch the most relevant topic id based on the input app @@ -133,7 +174,15 @@ public class OnDeviceClassifier implements Classifier { private List<Topic> getAppClassificationTopics(@NonNull String appDescription) { // Returns list of labelIds with their corresponding score in Category for the app // description. - List<Category> classifications = mBertNLClassifier.classify(appDescription); + List<Category> classifications = ImmutableList.of(); + try { + classifications = mBertNLClassifier.classify(appDescription); + } catch (Exception e) { + // (TODO:b/242926783): Update to more granular Exception after resolving JNI error + // propagation. + LogUtil.e("[ML] classify call failed for mBertNLClassifier."); + return ImmutableList.of(); + } // Get the highest score first. Sort in decreasing order. classifications.sort(Comparator.comparing(Category::getScore).reversed()); @@ -253,7 +302,7 @@ public class OnDeviceClassifier implements Classifier { // Load Bert model. try { mBertNLClassifier = loadModel(); - } catch (IOException e) { + } catch (Exception e) { LogUtil.e(e, "Loading ML model failed."); return false; } @@ -270,7 +319,15 @@ public class OnDeviceClassifier implements Classifier { mLabelsVersion = Long.parseLong( classifierAssetsMetadata.get(LABELS_ASSET_FIELD).get(ASSET_VERSION_FIELD)); - + try { + mBuildId = + Integer.parseInt( + classifierAssetsMetadata.get(VERSION_INFO_FIELD).get(BUILD_ID_FIELD)); + } catch (NumberFormatException e) { + // No build id is available. + LogUtil.d(e, "Build id is not available"); + mBuildId = -1; + } return true; } diff --git a/adservices/service-core/java/com/android/adservices/service/topics/classifier/PrecomputedClassifier.java b/adservices/service-core/java/com/android/adservices/service/topics/classifier/PrecomputedClassifier.java index 002beef062..b7074d981e 100644 --- a/adservices/service-core/java/com/android/adservices/service/topics/classifier/PrecomputedClassifier.java +++ b/adservices/service-core/java/com/android/adservices/service/topics/classifier/PrecomputedClassifier.java @@ -16,9 +16,17 @@ package com.android.adservices.service.topics.classifier; +import static com.android.adservices.service.stats.AdServicesStatsLog.AD_SERVICES_EPOCH_COMPUTATION_CLASSIFIER_REPORTED__CLASSIFIER_TYPE__PRECOMPUTED_CLASSIFIER; +import static com.android.adservices.service.stats.AdServicesStatsLog.AD_SERVICES_EPOCH_COMPUTATION_CLASSIFIER_REPORTED__ON_DEVICE_CLASSIFIER_STATUS__ON_DEVICE_CLASSIFIER_STATUS_NOT_INVOKED; +import static com.android.adservices.service.stats.AdServicesStatsLog.AD_SERVICES_EPOCH_COMPUTATION_CLASSIFIER_REPORTED__PRECOMPUTED_CLASSIFIER_STATUS__PRECOMPUTED_CLASSIFIER_STATUS_FAILURE; +import static com.android.adservices.service.stats.AdServicesStatsLog.AD_SERVICES_EPOCH_COMPUTATION_CLASSIFIER_REPORTED__PRECOMPUTED_CLASSIFIER_STATUS__PRECOMPUTED_CLASSIFIER_STATUS_SUCCESS; + import android.annotation.NonNull; +import com.android.adservices.LogUtil; import com.android.adservices.data.topics.Topic; +import com.android.adservices.service.stats.AdServicesLogger; +import com.android.adservices.service.stats.EpochComputationClassifierStats; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -48,6 +56,8 @@ public class PrecomputedClassifier implements Classifier { private static final String MODEL_ASSET_FIELD = "tflite_model"; private static final String LABELS_ASSET_FIELD = "labels_topics"; private static final String ASSET_VERSION_FIELD = "asset_version"; + private static final String VERSION_INFO_FIELD = "version_info"; + private static final String BUILD_ID_FIELD = "build_id"; private final ModelManager mModelManager; @@ -58,10 +68,13 @@ public class PrecomputedClassifier implements Classifier { private Map<String, List<Integer>> mAppTopics = new HashMap<>(); private long mModelVersion; private long mLabelsVersion; + private int mBuildId; + private final AdServicesLogger mLogger; - PrecomputedClassifier(@NonNull ModelManager modelManager) { + PrecomputedClassifier(@NonNull ModelManager modelManager, @NonNull AdServicesLogger logger) { mModelManager = modelManager; mLoaded = false; + mLogger = logger; } @NonNull @@ -79,6 +92,22 @@ public class PrecomputedClassifier implements Classifier { List<Topic> topics = topicIds.stream().map(this::createTopic).collect(Collectors.toList()); + // Log atom for getTopTopics call. + mLogger.logEpochComputationClassifierStats( + EpochComputationClassifierStats.builder() + .setTopicIds(ImmutableList.copyOf(topicIds)) + .setBuildId(mBuildId) + .setAssetVersion(Long.toString(mModelVersion)) + .setClassifierType( + AD_SERVICES_EPOCH_COMPUTATION_CLASSIFIER_REPORTED__CLASSIFIER_TYPE__PRECOMPUTED_CLASSIFIER) + .setOnDeviceClassifierStatus( + AD_SERVICES_EPOCH_COMPUTATION_CLASSIFIER_REPORTED__ON_DEVICE_CLASSIFIER_STATUS__ON_DEVICE_CLASSIFIER_STATUS_NOT_INVOKED) + .setPrecomputedClassifierStatus( + topicIds.isEmpty() + ? AD_SERVICES_EPOCH_COMPUTATION_CLASSIFIER_REPORTED__PRECOMPUTED_CLASSIFIER_STATUS__PRECOMPUTED_CLASSIFIER_STATUS_FAILURE + : AD_SERVICES_EPOCH_COMPUTATION_CLASSIFIER_REPORTED__PRECOMPUTED_CLASSIFIER_STATUS__PRECOMPUTED_CLASSIFIER_STATUS_SUCCESS) + .build()); + appsToClassifiedTopics.put(app, topics); } } @@ -97,7 +126,7 @@ public class PrecomputedClassifier implements Classifier { } return CommonClassifierHelper.getTopTopics( - appTopics, mLabels, new Random(), numberOfTopTopics, numberOfRandomTopics); + appTopics, mLabels, new Random(), numberOfTopTopics, numberOfRandomTopics, mLogger); } long getModelVersion() { @@ -136,7 +165,15 @@ public class PrecomputedClassifier implements Classifier { mLabelsVersion = Long.parseLong( classifierAssetsMetadata.get(LABELS_ASSET_FIELD).get(ASSET_VERSION_FIELD)); - + try { + mBuildId = + Integer.parseInt( + classifierAssetsMetadata.get(VERSION_INFO_FIELD).get(BUILD_ID_FIELD)); + } catch (NumberFormatException e) { + // No build id is available. + LogUtil.d(e, "Build id is not available"); + mBuildId = -1; + } mLoaded = true; } diff --git a/adservices/service-core/java/com/android/adservices/service/topics/classifier/Preprocessor.java b/adservices/service-core/java/com/android/adservices/service/topics/classifier/Preprocessor.java index ac7ad5eab1..ef19adba00 100644 --- a/adservices/service-core/java/com/android/adservices/service/topics/classifier/Preprocessor.java +++ b/adservices/service-core/java/com/android/adservices/service/topics/classifier/Preprocessor.java @@ -19,6 +19,7 @@ import static java.util.Objects.requireNonNull; import android.annotation.NonNull; import android.content.Context; +import android.util.Patterns; import com.android.adservices.LogUtil; @@ -36,19 +37,22 @@ import java.util.stream.Collectors; /** Util class for pre-processing input for the classifier. */ public final class Preprocessor { - // This regular expression is to identify URLs like https://google.com - private static final Pattern URL_REGEX = - Pattern.compile("(https?)://[-a-zA-Z0-9+&@#/%?=~_|!:,.;]*[-a-zA-Z0-9+&@#/%=~_|]"); + // This regular expression is to identify URLs like https://google.com or www.youtube.com. + private static final Pattern URL_REGEX = Patterns.WEB_URL; // At mention regular expression. Example @Rubidium1. private static final Pattern MENTIONS_REGEX = Pattern.compile("@[A-Za-z0-9]+"); + // This regular expression is to identify html tags like <span><\span> + private static final Pattern HTML_TAG_REGEX = Pattern.compile("\\<.*?\\>"); + // Regular expression to primarily remove punctuation marks. // Selects out lower case english alphabets and space. private static final Pattern ALPHABET_REGEX = Pattern.compile("[^a-z\\s]|"); private static final Pattern NEW_LINE_REGEX = Pattern.compile("\\n"); private static final Pattern MULTIPLE_SPACES_REGEX = Pattern.compile("\\s+"); + private static final Pattern TAB_REGEX = Pattern.compile("\\t"); private static final String SINGLE_SPACE = " "; private static final String EMPTY_STRING = ""; @@ -114,8 +118,10 @@ public final class Preprocessor { * <li>Converts text to lowercase. * <li>Removes URLs. * <li>Removes @mentions. - * <li>Removes everything other than lower case english alphabets and spaces. - * <li>Convert multiple spaces to a single space. + * <li>Removes html tags + * <li>Removes tabs and newlines + * <li>Converts multiple spaces to a single space. + * <li>Eliminates leading and tailing spaces. * </ul> * * @param description is the string description of the app. @@ -129,11 +135,14 @@ public final class Preprocessor { description = URL_REGEX.matcher(description).replaceAll(EMPTY_STRING); description = MENTIONS_REGEX.matcher(description).replaceAll(EMPTY_STRING); - description = ALPHABET_REGEX.matcher(description).replaceAll(EMPTY_STRING); + description = HTML_TAG_REGEX.matcher(description).replaceAll(EMPTY_STRING); description = NEW_LINE_REGEX.matcher(description).replaceAll(SINGLE_SPACE); + description = TAB_REGEX.matcher(description).replaceAll(SINGLE_SPACE); description = MULTIPLE_SPACES_REGEX.matcher(description).replaceAll(SINGLE_SPACE); + description = description.trim(); + return description; } |