diff options
author | android-build-team Robot <android-build-team-robot@google.com> | 2019-05-02 03:08:55 +0000 |
---|---|---|
committer | android-build-team Robot <android-build-team-robot@google.com> | 2019-05-02 03:08:55 +0000 |
commit | c12846986d3d04047d3b66d4d74e198e731aeae6 (patch) | |
tree | af7f743dc0dae890c1e5a738271e5dc6010809fc | |
parent | ff01380d16c345d3b14ea7102a045055eb25775f (diff) | |
parent | 29129a3d4cc8fd9f7e52d690445d68f2b1af2acd (diff) | |
download | platform_testing-c12846986d3d04047d3b66d4d74e198e731aeae6.tar.gz |
Snap for 5523284 from 29129a3d4cc8fd9f7e52d690445d68f2b1af2acd to qt-release
Change-Id: Ifa8566a8e4c529a525bf6c5a2d9b58c33193fe0c
8 files changed, 956 insertions, 0 deletions
diff --git a/libraries/collectors-helper/jank/Android.bp b/libraries/collectors-helper/jank/Android.bp new file mode 100644 index 000000000..b447f96d3 --- /dev/null +++ b/libraries/collectors-helper/jank/Android.bp @@ -0,0 +1,33 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Used for collecting jank metrics. +java_library { + name: "jank-helper", + defaults: ["tradefed_errorprone_defaults"], + + srcs: [ + "src/**/*.java", + ], + + static_libs: [ + "androidx.test.runner", + "collector-helper-utilities", + "guava", + "ub-uiautomator", + ], + + sdk_version: "current", +} + diff --git a/libraries/collectors-helper/jank/src/com/android/helpers/JankCollectionHelper.java b/libraries/collectors-helper/jank/src/com/android/helpers/JankCollectionHelper.java new file mode 100644 index 000000000..69a22dccb --- /dev/null +++ b/libraries/collectors-helper/jank/src/com/android/helpers/JankCollectionHelper.java @@ -0,0 +1,320 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.helpers; + +import static com.android.helpers.MetricUtility.constructKey; + +import android.support.test.uiautomator.UiDevice; +import android.util.Log; +import androidx.annotation.VisibleForTesting; +import androidx.test.InstrumentationRegistry; + +import com.google.common.base.Verify; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** An {@link ICollectorHelper} for collecting jank metrics for all or a list of processes. */ +public class JankCollectionHelper implements ICollectorHelper<Double> { + + private static final String LOG_TAG = JankCollectionHelper.class.getSimpleName(); + + // Prefix for all output metrics that come from the gfxinfo dump. + @VisibleForTesting static final String GFXINFO_METRICS_PREFIX = "gfxinfo"; + // Shell dump commands to get and reset the tracked gfxinfo metrics. + @VisibleForTesting static final String GFXINFO_COMMAND_GET = "dumpsys gfxinfo %s"; + @VisibleForTesting static final String GFXINFO_COMMAND_RESET = GFXINFO_COMMAND_GET + " --reset"; + // Pattern matchers and enumerators to verify and pull gfxinfo metrics. + // Example: "** Graphics info for pid 853 [com.google.android.leanbacklauncher] **" + private static final String GFXINFO_OUTPUT_HEADER = "Graphics info for pid (\\d+) \\[(%s)\\]"; + // Note: use the [\\s\\S]* multi-line matcher to support String#matches(). Instead of splitting + // the larger sections into more granular lines, we can match across all lines for simplicity. + private static final String MULTILINE_MATCHER = "[\\s\\S]*%s[\\s\\S]*"; + + public enum GfxInfoMetric { + // Example: "Total frames rendered: 20391" + TOTAL_FRAMES( + Pattern.compile(".*Total frames rendered: (\\d+).*", Pattern.DOTALL), + 1, + "total_frames"), + // Example: "Janky frames: 785 (3.85%)" + JANKY_FRAMES_COUNT( + Pattern.compile(".*Janky frames: (\\d+) \\((.+)\\%\\).*", Pattern.DOTALL), + 1, + "janky_frames_count"), + // Example: "Janky frames: 785 (3.85%)" + JANKY_FRAMES_PRCNT( + Pattern.compile(".*Janky frames: (\\d+) \\((.+)\\%\\).*", Pattern.DOTALL), + 2, + "janky_frames_percent"), + // Example: "50th percentile: 9ms" + FRAME_TIME_50TH( + Pattern.compile(".*50th percentile: (\\d+)ms.*", Pattern.DOTALL), + 1, + "jank_percentile_50"), + // Example: "90th percentile: 9ms" + FRAME_TIME_90TH( + Pattern.compile(".*90th percentile: (\\d+)ms.*", Pattern.DOTALL), + 1, + "jank_percentile_90"), + // Example: "95th percentile: 9ms" + FRAME_TIME_95TH( + Pattern.compile(".*95th percentile: (\\d+)ms.*", Pattern.DOTALL), + 1, + "jank_percentile_95"), + // Example: "99th percentile: 9ms" + FRAME_TIME_99TH( + Pattern.compile(".*99th percentile: (\\d+)ms.*", Pattern.DOTALL), + 1, + "jank_percentile_99"), + // Example: "Number Missed Vsync: 0" + NUM_MISSED_VSYNC( + Pattern.compile(".*Number Missed Vsync: (\\d+).*", Pattern.DOTALL), + 1, + "missed_vsync"), + // Example: "Number High input latency: 0" + NUM_HIGH_INPUT_LATENCY( + Pattern.compile(".*Number High input latency: (\\d+).*", Pattern.DOTALL), + 1, + "high_input_latency"), + // Example: "Number Slow UI thread: 0" + NUM_SLOW_UI_THREAD( + Pattern.compile(".*Number Slow UI thread: (\\d+).*", Pattern.DOTALL), + 1, + "slow_ui_thread"), + // Example: "Number Slow bitmap uploads: 0" + NUM_SLOW_BITMAP_UPLOADS( + Pattern.compile(".*Number Slow bitmap uploads: (\\d+).*", Pattern.DOTALL), + 1, + "slow_bmp_upload"), + // Example: "Number Slow issue draw commands: 0" + NUM_SLOW_DRAW( + Pattern.compile(".*Number Slow issue draw commands: (\\d+).*", Pattern.DOTALL), + 1, + "slow_issue_draw_cmds"), + // Example: "Number Frame deadline missed: 0" + NUM_FRAME_DEADLINE_MISSED( + Pattern.compile(".*Number Frame deadline missed: (\\d+).*", Pattern.DOTALL), + 1, + "deadline_missed"); + + private Pattern mPattern; + private int mGroupIndex; + private String mMetricId; + + GfxInfoMetric(Pattern pattern, int groupIndex, String metricId) { + mPattern = pattern; + mGroupIndex = groupIndex; + mMetricId = metricId; + } + + public Double parse(String lines) { + Matcher matcher = mPattern.matcher(lines); + if (matcher.matches()) { + return Double.valueOf(matcher.group(mGroupIndex)); + } else { + return null; + } + } + + public String getMetricId() { + return mMetricId; + } + } + + private Set<String> mTrackedPackages = new HashSet<>(); + private UiDevice mDevice; + + /** Clear existing jank metrics, unless explicitly configured. */ + @Override + public boolean startCollecting() { + if (mTrackedPackages.isEmpty()) { + clearGfxInfo(); + } else { + int exceptionCount = 0; + Exception lastException = null; + for (String pkg : mTrackedPackages) { + try { + clearGfxInfo(pkg); + } catch (Exception e) { + Log.e(LOG_TAG, "Encountered exception resetting gfxinfo.", e); + lastException = e; + exceptionCount++; + } + } + // Throw exceptions after to not quit on a single failure. + if (exceptionCount > 1) { + throw new RuntimeException( + "Multiple exceptions were encountered resetting gfxinfo. Reporting the last" + + " one only; others are visible in logs.", + lastException); + } else if (exceptionCount == 1) { + throw new RuntimeException( + "Encountered exception resetting gfxinfo.", lastException); + } + } + // No exceptions denotes success. + return true; + } + + /** Collect the {@code gfxinfo} metrics for tracked processes (or all, if unspecified). */ + @Override + public Map<String, Double> getMetrics() { + Map<String, Double> result = new HashMap<>(); + if (mTrackedPackages.isEmpty()) { + result.putAll(getGfxInfoMetrics()); + } else { + int exceptionCount = 0; + Exception lastException = null; + for (String pkg : mTrackedPackages) { + try { + result.putAll(getGfxInfoMetrics(pkg)); + } catch (Exception e) { + Log.e(LOG_TAG, "Encountered exception getting gfxinfo.", e); + lastException = e; + exceptionCount++; + } + } + // Throw exceptions after to ensure all failures are reported. The metrics will still + // not be collected at this point, but it will possibly make the issue cause clearer. + if (exceptionCount > 1) { + throw new RuntimeException( + "Multiple exceptions were encountered getting gfxinfo. Reporting the last" + + " one only; others are visible in logs.", + lastException); + } else if (exceptionCount == 1) { + throw new RuntimeException("Encountered exception getting gfxinfo.", lastException); + } + } + return result; + } + + /** Do nothing, because nothing is needed to disable jank. */ + @Override + public boolean stopCollecting() { + return true; + } + + /** Add a package or list of packages to be tracked. */ + public void addTrackedPackages(String... packages) { + Collections.addAll(mTrackedPackages, packages); + } + + /** Clear the {@code gfxinfo} for all packages. */ + @VisibleForTesting + void clearGfxInfo() { + // Not specifying a package will clear everything. + clearGfxInfo(""); + } + + /** Clear the {@code gfxinfo} for the {@code pkg} specified. */ + @VisibleForTesting + void clearGfxInfo(String pkg) { + try { + String command = String.format(GFXINFO_COMMAND_RESET, pkg); + String output = getDevice().executeShellCommand(command); + // Success if the (specified package or any if unspecified) header exists in the output. + verifyMatches(output, getHeaderMatcher(pkg), "Did not find package header in output."); + Log.v(LOG_TAG, String.format("Cleared %s gfxinfo.", pkg.isEmpty() ? "all" : pkg)); + } catch (IOException e) { + throw new RuntimeException("Failed to clear gfxinfo.", e); + } + } + + /** Return a {@code Map<String, Double>} of {@code gfxinfo} metrics for all processes. */ + @VisibleForTesting + Map<String, Double> getGfxInfoMetrics() { + return getGfxInfoMetrics(""); + } + + /** Return a {@code Map<String, Double>} of {@code gfxinfo} metrics for {@code pkg}. */ + @VisibleForTesting + Map<String, Double> getGfxInfoMetrics(String pkg) { + try { + String command = String.format(GFXINFO_COMMAND_GET, pkg); + String output = getDevice().executeShellCommand(command); + verifyMatches(output, getHeaderMatcher(pkg), "Missing package header."); + // Split each new section starting with two asterisks '**', and then query and append + // all metrics. This method supports both single-package and multi-package outputs. + String[] pkgMetricSections = output.split("\n\\*\\*"); + Map<String, Double> result = new HashMap<>(); + // Skip the 1st section, which contains only header information. + for (int i = 1; i < pkgMetricSections.length; i++) { + result.putAll(parseGfxInfoMetrics(pkgMetricSections[i])); + } + return result; + } catch (IOException e) { + throw new RuntimeException("Failed to get gfxinfo.", e); + } + } + + /** Parse the {@code output} of {@code gfxinfo} to a {@code Map<String, Double>} of metrics. */ + private Map<String, Double> parseGfxInfoMetrics(String output) { + Matcher header = Pattern.compile(getHeaderMatcher("")).matcher(output); + if (!header.matches()) { + throw new RuntimeException("Failed to parse package from gfxinfo output."); + } + // Package name is the only required field. + String packageName = header.group(2); + Log.v(LOG_TAG, String.format("Collecting metrics for: %s", packageName)); + // Parse each metric from the results via a common pattern. + Map<String, Double> results = new HashMap<String, Double>(); + for (GfxInfoMetric metric : GfxInfoMetric.values()) { + String metricKey = + constructKey(GFXINFO_METRICS_PREFIX, packageName, metric.getMetricId()); + // Find the metric or log that it's missing. + Double value = metric.parse(output); + if (value == null) { + Log.d(LOG_TAG, String.format("Did not find %s from %s", metricKey, packageName)); + } else { + results.put(metricKey, value); + } + } + return results; + } + + /** + * Returns a matcher {@code String} for {@code pkg}'s {@code gfxinfo} headers. + * + * <p>Note: {@code pkg} may be empty. + */ + private String getHeaderMatcher(String pkg) { + return String.format( + MULTILINE_MATCHER, + String.format(GFXINFO_OUTPUT_HEADER, (pkg.isEmpty() ? ".*" : pkg))); + } + + /** Verify the {@code output} matches {@code match}, or throw if not. */ + private void verifyMatches(String output, String match, String message, Object... args) { + Verify.verify(output.matches(match), message, args); + } + + /** Returns the {@link UiDevice} under test. */ + @VisibleForTesting + protected UiDevice getDevice() { + if (mDevice == null) { + mDevice = UiDevice.getInstance(InstrumentationRegistry.getInstrumentation()); + } + return mDevice; + } +} diff --git a/libraries/collectors-helper/jank/test/Android.bp b/libraries/collectors-helper/jank/test/Android.bp new file mode 100644 index 000000000..73c6f09d4 --- /dev/null +++ b/libraries/collectors-helper/jank/test/Android.bp @@ -0,0 +1,30 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +java_library { + name: "jank-helper-test", + defaults: ["tradefed_errorprone_defaults"], + + srcs: ["src/**/*.java"], + + static_libs: [ + "androidx.test.runner", + "jank-helper", + "junit", + "mockito-target", + "truth-prebuilt", + ], + + sdk_version: "current", +} diff --git a/libraries/collectors-helper/jank/test/src/com/android/helpers/JankCollectionHelperTest.java b/libraries/collectors-helper/jank/test/src/com/android/helpers/JankCollectionHelperTest.java new file mode 100644 index 000000000..9b614fce1 --- /dev/null +++ b/libraries/collectors-helper/jank/test/src/com/android/helpers/JankCollectionHelperTest.java @@ -0,0 +1,420 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.android.helpers; + +import static com.android.helpers.MetricUtility.constructKey; +import static com.android.helpers.JankCollectionHelper.GFXINFO_COMMAND_GET; +import static com.android.helpers.JankCollectionHelper.GFXINFO_COMMAND_RESET; +import static com.android.helpers.JankCollectionHelper.GfxInfoMetric.TOTAL_FRAMES; +import static com.android.helpers.JankCollectionHelper.GfxInfoMetric.JANKY_FRAMES_COUNT; +import static com.android.helpers.JankCollectionHelper.GfxInfoMetric.JANKY_FRAMES_PRCNT; +import static com.android.helpers.JankCollectionHelper.GfxInfoMetric.FRAME_TIME_50TH; +import static com.android.helpers.JankCollectionHelper.GfxInfoMetric.FRAME_TIME_90TH; +import static com.android.helpers.JankCollectionHelper.GfxInfoMetric.FRAME_TIME_95TH; +import static com.android.helpers.JankCollectionHelper.GfxInfoMetric.FRAME_TIME_99TH; +import static com.android.helpers.JankCollectionHelper.GfxInfoMetric.NUM_MISSED_VSYNC; +import static com.android.helpers.JankCollectionHelper.GfxInfoMetric.NUM_HIGH_INPUT_LATENCY; +import static com.android.helpers.JankCollectionHelper.GfxInfoMetric.NUM_SLOW_UI_THREAD; +import static com.android.helpers.JankCollectionHelper.GfxInfoMetric.NUM_SLOW_BITMAP_UPLOADS; +import static com.android.helpers.JankCollectionHelper.GfxInfoMetric.NUM_SLOW_DRAW; +import static com.android.helpers.JankCollectionHelper.GfxInfoMetric.NUM_FRAME_DEADLINE_MISSED; +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth.assertWithMessage; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.when; + +import android.support.test.uiautomator.UiDevice; +import androidx.test.runner.AndroidJUnit4; + +import java.io.IOException; +import java.util.Map; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InOrder; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; + +/** Android Unit tests for {@link JankCollectionHelper}. */ +@RunWith(AndroidJUnit4.class) +public class JankCollectionHelperTest { + private static final String GFXINFO_RESET_FORMAT = + "\n\n** Graphics info for pid 9999 [%s] **" + + "\n" + + "\nTotal frames rendered: 0" + + "\nJanky frames: 0 (00.00%%)" + + "\n50th percentile: 0ms" + + "\n90th percentile: 0ms" + + "\n95th percentile: 0ms" + + "\n99th percentile: 0ms" + + "\nNumber Missed Vsync: 0" + + "\nNumber High input latency: 0" + + "\nNumber Slow UI thread: 0" + + "\nNumber Slow bitmap uploads: 0" + + "\nNumber Slow issue draw commands: 0" + + "\nNumber Frame deadline missed: 0"; + private static final String GFXINFO_GET_FORMAT = + "\n\n** Graphics info for pid 9999 [%s] **" + + "\n" + + "\nTotal frames rendered: 900" + + "\nJanky frames: 300 (33.33%%)" + + "\n50th percentile: 150ms" + + "\n90th percentile: 190ms" + + "\n95th percentile: 195ms" + + "\n99th percentile: 199ms" + + "\nNumber Missed Vsync: 1" + + "\nNumber High input latency: 2" + + "\nNumber Slow UI thread: 3" + + "\nNumber Slow bitmap uploads: 4" + + "\nNumber Slow issue draw commands: 5" + + "\nNumber Frame deadline missed: 6"; + + private @Mock UiDevice mUiDevice; + private JankCollectionHelper mHelper; + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + mHelper = Mockito.spy(new JankCollectionHelper()); + when(mHelper.getDevice()).thenReturn(mUiDevice); + } + + /** Test track a single, valid package. */ + @Test + public void testCollect_valuesMatch() throws Exception { + mockResetCommand("pkg1", String.format(GFXINFO_RESET_FORMAT, "pkg1")); + mockGetCommand("pkg1", String.format(GFXINFO_GET_FORMAT, "pkg1")); + + mHelper.addTrackedPackages("pkg1"); + mHelper.startCollecting(); + Map<String, Double> metrics = mHelper.getMetrics(); + assertThat(metrics.get(buildMetricKey("pkg1", TOTAL_FRAMES.getMetricId()))) + .isEqualTo(900.0); + assertThat(metrics.get(buildMetricKey("pkg1", JANKY_FRAMES_COUNT.getMetricId()))) + .isEqualTo(300.0); + assertThat(metrics.get(buildMetricKey("pkg1", JANKY_FRAMES_PRCNT.getMetricId()))) + .isEqualTo(33.33); + assertThat(metrics.get(buildMetricKey("pkg1", FRAME_TIME_50TH.getMetricId()))) + .isEqualTo(150.0); + assertThat(metrics.get(buildMetricKey("pkg1", FRAME_TIME_90TH.getMetricId()))) + .isEqualTo(190.0); + assertThat(metrics.get(buildMetricKey("pkg1", FRAME_TIME_95TH.getMetricId()))) + .isEqualTo(195.0); + assertThat(metrics.get(buildMetricKey("pkg1", FRAME_TIME_99TH.getMetricId()))) + .isEqualTo(199.0); + assertThat(metrics.get(buildMetricKey("pkg1", NUM_MISSED_VSYNC.getMetricId()))) + .isEqualTo(1.0); + assertThat(metrics.get(buildMetricKey("pkg1", NUM_HIGH_INPUT_LATENCY.getMetricId()))) + .isEqualTo(2.0); + assertThat(metrics.get(buildMetricKey("pkg1", NUM_SLOW_UI_THREAD.getMetricId()))) + .isEqualTo(3.0); + assertThat(metrics.get(buildMetricKey("pkg1", NUM_SLOW_BITMAP_UPLOADS.getMetricId()))) + .isEqualTo(4.0); + assertThat(metrics.get(buildMetricKey("pkg1", NUM_SLOW_DRAW.getMetricId()))).isEqualTo(5.0); + assertThat(metrics.get(buildMetricKey("pkg1", NUM_FRAME_DEADLINE_MISSED.getMetricId()))) + .isEqualTo(6.0); + mHelper.stopCollecting(); + } + + /** Test track a single, valid package. */ + @Test + public void testCollect_singlePackage() throws Exception { + mockResetCommand("pkg1", String.format(GFXINFO_RESET_FORMAT, "pkg1")); + mockGetCommand("pkg1", String.format(GFXINFO_GET_FORMAT, "pkg1")); + + mHelper.addTrackedPackages("pkg1"); + mHelper.startCollecting(); + Map<String, Double> metrics = mHelper.getMetrics(); + for (String key : metrics.keySet()) { + assertWithMessage("All keys must contains the single watched package name.") + .that(key) + .contains("pkg1"); + } + mHelper.stopCollecting(); + } + + /** Test track multiple valid packages. */ + @Test + public void testCollect_multiPackage() throws Exception { + mockResetCommand("pkg1", String.format(GFXINFO_RESET_FORMAT, "pkg1")); + mockGetCommand("pkg1", String.format(GFXINFO_GET_FORMAT, "pkg1")); + mockResetCommand("pkg2", String.format(GFXINFO_RESET_FORMAT, "pkg2")); + mockGetCommand("pkg2", String.format(GFXINFO_GET_FORMAT, "pkg2")); + mockResetCommand("pkg3", String.format(GFXINFO_RESET_FORMAT, "pkg3")); + mockGetCommand("pkg3", String.format(GFXINFO_GET_FORMAT, "pkg3")); + + mHelper.addTrackedPackages("pkg1", "pkg2"); + mHelper.startCollecting(); + Map<String, Double> metrics = mHelper.getMetrics(); + // Assert against all keys that they only match expected packages. + for (String key : metrics.keySet()) { + assertWithMessage("All keys must contains one of the 2 watched package names.") + .that(key) + .containsMatch(".*pkg(1|2).*"); + assertWithMessage("The unwatched package should not be included in metrics.") + .that(key) + .doesNotContain("pkg3"); + } + // Assert that it contains keys for both packages being watched. + assertThat(metrics).containsKey(buildMetricKey("pkg1", TOTAL_FRAMES.getMetricId())); + assertThat(metrics).containsKey(buildMetricKey("pkg2", TOTAL_FRAMES.getMetricId())); + mHelper.stopCollecting(); + } + + /** Test track all packages when unspecified. */ + @Test + public void testCollect_allPackages() throws Exception { + String resetOutput = + String.join( + "\n", + String.format(GFXINFO_RESET_FORMAT, "pkg1"), + String.format(GFXINFO_RESET_FORMAT, "pkg2"), + String.format(GFXINFO_RESET_FORMAT, "pkg3")); + String getOutput = + String.join( + "\n", + String.format(GFXINFO_GET_FORMAT, "pkg1"), + String.format(GFXINFO_GET_FORMAT, "pkg2"), + String.format(GFXINFO_GET_FORMAT, "pkg3")); + mockResetCommand("", resetOutput); + mockGetCommand("", getOutput); + + mHelper.startCollecting(); + Map<String, Double> metrics = mHelper.getMetrics(); + // Assert against all keys that they only match expected packages. + for (String key : metrics.keySet()) { + assertWithMessage("All keys must contains one of the output package names.") + .that(key) + .containsMatch(".*pkg(1|2|3).*"); + } + // Assert that it contains keys for all packages being watched. + assertThat(metrics).containsKey(buildMetricKey("pkg1", TOTAL_FRAMES.getMetricId())); + assertThat(metrics).containsKey(buildMetricKey("pkg2", TOTAL_FRAMES.getMetricId())); + assertThat(metrics).containsKey(buildMetricKey("pkg3", TOTAL_FRAMES.getMetricId())); + mHelper.stopCollecting(); + } + + /** Test that it collects available fields, even if some are missing. */ + @Test + public void testCollect_ignoreMissingFields() throws Exception { + String missingResets = + "\n\n** Graphics info for pid 9999 [pkg1] **" + + "\n" + + "\nTotal frames rendered: 0" + + "\nJanky frames: 0 (00.00%%)" + + "\nNumber Missed Vsync: 0" + + "\nNumber High input latency: 0" + + "\nNumber Slow UI thread: 0" + + "\nNumber Slow bitmap uploads: 0" + + "\nNumber Slow issue draw commands: 0" + + "\nNumber Frame deadline missed: 0"; + String missingGets = + "\n\n** Graphics info for pid 9999 [pkg1] **" + + "\n" + + "\nTotal frames rendered: 900" + + "\nJanky frames: 300 (33.33%)" + + "\nNumber Missed Vsync: 1" + + "\nNumber High input latency: 2" + + "\nNumber Slow UI thread: 3" + + "\nNumber Slow bitmap uploads: 4" + + "\nNumber Slow issue draw commands: 5" + + "\nNumber Frame deadline missed: 6"; + + mockResetCommand("pkg1", missingResets); + mockGetCommand("pkg1", missingGets); + + mHelper.addTrackedPackages("pkg1"); + mHelper.startCollecting(); + Map<String, Double> metrics = mHelper.getMetrics(); + assertThat(metrics.keySet()) + .containsExactly( + buildMetricKey("pkg1", TOTAL_FRAMES.getMetricId()), + buildMetricKey("pkg1", JANKY_FRAMES_COUNT.getMetricId()), + buildMetricKey("pkg1", JANKY_FRAMES_PRCNT.getMetricId()), + buildMetricKey("pkg1", NUM_MISSED_VSYNC.getMetricId()), + buildMetricKey("pkg1", NUM_HIGH_INPUT_LATENCY.getMetricId()), + buildMetricKey("pkg1", NUM_SLOW_UI_THREAD.getMetricId()), + buildMetricKey("pkg1", NUM_SLOW_BITMAP_UPLOADS.getMetricId()), + buildMetricKey("pkg1", NUM_SLOW_DRAW.getMetricId()), + buildMetricKey("pkg1", NUM_FRAME_DEADLINE_MISSED.getMetricId())); + mHelper.stopCollecting(); + } + + /** Test that it collects known fields, even if some are unknown. */ + @Test + public void testCollect_ignoreUnknownField() throws Exception { + String extraFields = + "\nWhatever: 1" + + "\nWhateverClose: 2" + + "\nWhateverNotSo: 3" + + "\nWhateverBlahs: 4"; + mockResetCommand("pkg1", String.format(GFXINFO_RESET_FORMAT + extraFields, "pkg1")); + mockGetCommand("pkg1", String.format(GFXINFO_GET_FORMAT + extraFields, "pkg1")); + + mHelper.addTrackedPackages("pkg1"); + mHelper.startCollecting(); + Map<String, Double> metrics = mHelper.getMetrics(); + assertThat(metrics.keySet()) + .containsExactly( + buildMetricKey("pkg1", TOTAL_FRAMES.getMetricId()), + buildMetricKey("pkg1", JANKY_FRAMES_COUNT.getMetricId()), + buildMetricKey("pkg1", JANKY_FRAMES_PRCNT.getMetricId()), + buildMetricKey("pkg1", FRAME_TIME_50TH.getMetricId()), + buildMetricKey("pkg1", FRAME_TIME_90TH.getMetricId()), + buildMetricKey("pkg1", FRAME_TIME_95TH.getMetricId()), + buildMetricKey("pkg1", FRAME_TIME_99TH.getMetricId()), + buildMetricKey("pkg1", NUM_MISSED_VSYNC.getMetricId()), + buildMetricKey("pkg1", NUM_HIGH_INPUT_LATENCY.getMetricId()), + buildMetricKey("pkg1", NUM_SLOW_UI_THREAD.getMetricId()), + buildMetricKey("pkg1", NUM_SLOW_BITMAP_UPLOADS.getMetricId()), + buildMetricKey("pkg1", NUM_SLOW_DRAW.getMetricId()), + buildMetricKey("pkg1", NUM_FRAME_DEADLINE_MISSED.getMetricId())); + mHelper.stopCollecting(); + } + + /** Test that it continues resetting even if certain packages throw for some reason. */ + @Test + public void testCollect_delayExceptions_onReset() throws Exception { + // Package 1 is problematic to reset, but package 2 and 3 are good. + String cmd = String.format(GFXINFO_COMMAND_RESET, "pkg1"); + when(mUiDevice.executeShellCommand(cmd)).thenThrow(new RuntimeException()); + mockResetCommand("pkg2", String.format(GFXINFO_RESET_FORMAT, "pkg2")); + mockResetCommand("pkg3", String.format(GFXINFO_RESET_FORMAT, "pkg3")); + + mHelper.addTrackedPackages("pkg1", "pkg2", "pkg3"); + try { + mHelper.startCollecting(); + fail("Should have thrown an exception resetting pkg1."); + } catch (Exception e) { + // assert that all of the packages were reset and pass. + InOrder inOrder = inOrder(mUiDevice); + inOrder.verify(mUiDevice) + .executeShellCommand(String.format(GFXINFO_COMMAND_RESET, "pkg1")); + inOrder.verify(mUiDevice) + .executeShellCommand(String.format(GFXINFO_COMMAND_RESET, "pkg2")); + inOrder.verify(mUiDevice) + .executeShellCommand(String.format(GFXINFO_COMMAND_RESET, "pkg3")); + } + } + + /** Test that it continues collecting even if certain packages throw for some reason. */ + @Test + public void testCollect_delayExceptions_onGet() throws Exception { + // Package 1 is problematic to reset, but package 2 and 3 are good. + mockResetCommand("pkg1", String.format(GFXINFO_RESET_FORMAT, "pkg1")); + mockResetCommand("pkg2", String.format(GFXINFO_RESET_FORMAT, "pkg2")); + mockResetCommand("pkg3", String.format(GFXINFO_RESET_FORMAT, "pkg3")); + String cmd = String.format(GFXINFO_COMMAND_GET, "pkg1"); + when(mUiDevice.executeShellCommand(cmd)).thenThrow(new RuntimeException()); + mockGetCommand("pkg2", String.format(GFXINFO_GET_FORMAT, "pkg2")); + mockGetCommand("pkg3", String.format(GFXINFO_GET_FORMAT, "pkg3")); + + mHelper.addTrackedPackages("pkg1", "pkg2", "pkg3"); + try { + mHelper.startCollecting(); + mHelper.getMetrics(); + fail("Should have thrown an exception getting pkg1."); + } catch (Exception e) { + // assert that all of the packages were reset and gotten and pass. + InOrder inOrder = inOrder(mUiDevice); + inOrder.verify(mUiDevice) + .executeShellCommand(String.format(GFXINFO_COMMAND_RESET, "pkg1")); + inOrder.verify(mUiDevice) + .executeShellCommand(String.format(GFXINFO_COMMAND_RESET, "pkg2")); + inOrder.verify(mUiDevice) + .executeShellCommand(String.format(GFXINFO_COMMAND_RESET, "pkg3")); + inOrder.verify(mUiDevice) + .executeShellCommand(String.format(GFXINFO_COMMAND_GET, "pkg1")); + inOrder.verify(mUiDevice) + .executeShellCommand(String.format(GFXINFO_COMMAND_GET, "pkg2")); + inOrder.verify(mUiDevice) + .executeShellCommand(String.format(GFXINFO_COMMAND_GET, "pkg3")); + } + } + + /** Test that it fails if the {@code gfxinfo} metrics cannot be cleared. */ + @Test + public void testFailures_cannotClear() throws Exception { + String cmd = String.format(JankCollectionHelper.GFXINFO_COMMAND_RESET, ""); + when(mUiDevice.executeShellCommand(cmd)).thenReturn(""); + try { + mHelper.startCollecting(); + fail("Should have thrown an exception."); + } catch (RuntimeException e) { + // pass + } + } + + /** Test that it fails when encountering an {@code IOException} on reset. */ + @Test + public void testFailures_ioFailure() throws Exception { + String cmd = String.format(JankCollectionHelper.GFXINFO_COMMAND_RESET, ""); + when(mUiDevice.executeShellCommand(cmd)).thenThrow(new IOException()); + try { + mHelper.startCollecting(); + fail("Should have thrown an exception."); + } catch (RuntimeException e) { + // pass + } + } + + /** Test that it fails when the package does not show up on reset. */ + @Test + public void testFailures_noPackageOnReset() throws Exception { + mockResetCommand("pkg1", String.format(GFXINFO_RESET_FORMAT, "pkg2")); + + mHelper.addTrackedPackages("pkg1"); + try { + mHelper.startCollecting(); + fail("Should have thrown an exception."); + } catch (RuntimeException e) { + // pass + } + } + + /** Test that it fails when the package does not show up on get. */ + @Test + public void testFailures_noPackageOnGet() throws Exception { + mockResetCommand("pkg1", String.format(GFXINFO_RESET_FORMAT, "pkg1")); + mockGetCommand("pkg1", String.format(GFXINFO_GET_FORMAT, "pkg2")); + + mHelper.addTrackedPackages("pkg1"); + try { + mHelper.startCollecting(); + mHelper.getMetrics(); + fail("Should have thrown an exception."); + } catch (RuntimeException e) { + // pass + } + } + + private String buildMetricKey(String pkg, String id) { + return constructKey(JankCollectionHelper.GFXINFO_METRICS_PREFIX, pkg, id); + } + + private void mockResetCommand(String pkg, String output) throws IOException { + String cmd = String.format(GFXINFO_COMMAND_RESET, pkg); + when(mUiDevice.executeShellCommand(cmd)).thenReturn(output); + } + + private void mockGetCommand(String pkg, String output) throws IOException { + String cmd = String.format(GFXINFO_COMMAND_GET, pkg); + when(mUiDevice.executeShellCommand(cmd)).thenReturn(output); + } +} diff --git a/libraries/device-collectors/src/main/Android.bp b/libraries/device-collectors/src/main/Android.bp index 4d513c636..9e144ed3f 100644 --- a/libraries/device-collectors/src/main/Android.bp +++ b/libraries/device-collectors/src/main/Android.bp @@ -20,6 +20,7 @@ java_library { static_libs: [ "androidx.test.runner", + "jank-helper", "junit", "memory-helper", "perfetto-helper", diff --git a/libraries/device-collectors/src/main/java/android/device/collectors/JankListener.java b/libraries/device-collectors/src/main/java/android/device/collectors/JankListener.java new file mode 100644 index 000000000..b5db1f765 --- /dev/null +++ b/libraries/device-collectors/src/main/java/android/device/collectors/JankListener.java @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.device.collectors; + +import android.device.collectors.annotations.OptionClass; +import android.os.Bundle; +import android.util.Log; +import androidx.annotation.VisibleForTesting; + +import com.android.helpers.JankCollectionHelper; + +import java.util.Arrays; + +/** + * A {@link BaseCollectionListener} that captures and records jank metrics for a specific package or + * for all packages if none are specified. + */ +@OptionClass(alias = "jank-listener") +public class JankListener extends BaseCollectionListener<Double> { + private static final String LOG_TAG = JankListener.class.getSimpleName(); + + @VisibleForTesting static final String PACKAGE_SEPARATOR = ","; + @VisibleForTesting static final String PACKAGE_NAMES_KEY = "jank-package-names"; + + public JankListener() { + createHelperInstance(new JankCollectionHelper()); + } + + @VisibleForTesting + public JankListener(Bundle args, JankCollectionHelper helper) { + super(args, helper); + } + + /** Tracks the provided packages if specified, or all packages if not specified. */ + @Override + public void setupAdditionalArgs() { + Bundle args = getArgsBundle(); + String pkgs = args.getString(PACKAGE_NAMES_KEY); + if (pkgs != null) { + Log.v(LOG_TAG, String.format("Adding packages: %s", pkgs)); + // Basic malformed input check: trim packages and remove empty ones. + String[] splitPkgs = + Arrays.stream(pkgs.split(PACKAGE_SEPARATOR)) + .map(String::trim) + .filter(item -> !item.isEmpty()) + .toArray(String[]::new); + ((JankCollectionHelper) mHelper).addTrackedPackages(splitPkgs); + } else { + Log.v(LOG_TAG, "Tracking all packages for jank."); + } + } +} diff --git a/libraries/device-collectors/src/test/java/android/device/collectors/JankListenerTest.java b/libraries/device-collectors/src/test/java/android/device/collectors/JankListenerTest.java new file mode 100644 index 000000000..dd9394fd2 --- /dev/null +++ b/libraries/device-collectors/src/test/java/android/device/collectors/JankListenerTest.java @@ -0,0 +1,85 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package android.device.collectors; + +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import android.app.Instrumentation; +import android.os.Bundle; +import androidx.test.runner.AndroidJUnit4; + +import com.android.helpers.JankCollectionHelper; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.Description; +import org.junit.runner.Result; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +/** Unit tests for {@link JankListener} specific behavior. */ +@RunWith(AndroidJUnit4.class) +public final class JankListenerTest { + + // A {@code Description} to pass when faking a test run start call. + private static final Description RUN_DESCRIPTION = Description.createSuiteDescription("run"); + private static final Description TEST_DESCRIPTION = + Description.createTestDescription("run", "test"); + + @Mock private JankCollectionHelper mHelper; + @Mock private Instrumentation mInstrumentation; + + @Before + public void setUp() throws Exception { + MockitoAnnotations.initMocks(this); + } + + /** Test that packages are specified when set in arguments. */ + @Test + public void testCollect_specificProcess() throws Exception { + Bundle twoProcBundle = new Bundle(); + twoProcBundle.putString( + JankListener.PACKAGE_NAMES_KEY, + String.join(JankListener.PACKAGE_SEPARATOR, "pkg1", "pkg2")); + JankListener collector = new JankListener(twoProcBundle, mHelper); + collector.setInstrumentation(mInstrumentation); + + // Simulate a test run and verify the "specific process collection" behavior. + collector.testRunStarted(RUN_DESCRIPTION); + verify(mHelper, times(1)).addTrackedPackages("pkg1", "pkg2"); + collector.testStarted(TEST_DESCRIPTION); + collector.testFinished(TEST_DESCRIPTION); + collector.testRunFinished(new Result()); + } + + /** Test that no packages are specified when not set in arguments. */ + @Test + public void testCollect_allProcesses() throws Exception { + JankListener collector = new JankListener(new Bundle(), mHelper); + collector.setInstrumentation(mInstrumentation); + + // Simulate a test run and verify the "all process collection" behavior. + collector.testRunStarted(RUN_DESCRIPTION); + verify(mHelper, never()).addTrackedPackages(anyString()); + collector.testStarted(TEST_DESCRIPTION); + collector.testFinished(TEST_DESCRIPTION); + collector.testRunFinished(new Result()); + } +} diff --git a/libraries/launcher-helper/src/android/support/test/launcherhelper/AutoLauncherStrategy.java b/libraries/launcher-helper/src/android/support/test/launcherhelper/AutoLauncherStrategy.java index f84c43aed..27e98855e 100644 --- a/libraries/launcher-helper/src/android/support/test/launcherhelper/AutoLauncherStrategy.java +++ b/libraries/launcher-helper/src/android/support/test/launcherhelper/AutoLauncherStrategy.java @@ -224,6 +224,7 @@ public class AutoLauncherStrategy implements IAutoLauncherStrategy { @SuppressWarnings("unused") @Override public long launch(String appName, String packageName) { + openApp(appName); return 0; } } |