diff options
author | Becky Wen <jinghuanwen@google.com> | 2023-02-24 16:40:19 -0800 |
---|---|---|
committer | Becky Wen <jinghuanwen@google.com> | 2023-03-02 00:35:40 +0000 |
commit | 2280a465d20c39b158fa41f0c0293e08f4078637 (patch) | |
tree | d2d547e53e6678fc4de9d2e13ad9c31493a23b52 | |
parent | b15977f730ddbe8058f97e45801b0c59978a8162 (diff) | |
download | platform_testing-2280a465d20c39b158fa41f0c0293e08f4078637.tar.gz |
Create a generic collector
Test: unit test
Bug: 237110678
Change-Id: I23c046ebc8bccf19658970ef3cef1529ffb2f910
7 files changed, 482 insertions, 0 deletions
diff --git a/libraries/collectors-helper/generic/Android.bp b/libraries/collectors-helper/generic/Android.bp new file mode 100644 index 000000000..07127c026 --- /dev/null +++ b/libraries/collectors-helper/generic/Android.bp @@ -0,0 +1,35 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Used for collecting generic metrics from binary test. +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +java_library { + name: "generic-helper", + defaults: ["tradefed_errorprone_defaults"], + + srcs: [ + "src/**/*.java", + ], + + static_libs: [ + "androidx.test.runner", + "androidx.test.uiautomator_uiautomator", + "collector-helper-utilities", + ], + + sdk_version: "current", +} diff --git a/libraries/collectors-helper/generic/src/com/android/helpers/GenericExecutableCollectorHelper.java b/libraries/collectors-helper/generic/src/com/android/helpers/GenericExecutableCollectorHelper.java new file mode 100644 index 000000000..65069344e --- /dev/null +++ b/libraries/collectors-helper/generic/src/com/android/helpers/GenericExecutableCollectorHelper.java @@ -0,0 +1,160 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.helpers; + +import android.util.Log; + +import androidx.annotation.VisibleForTesting; +import androidx.test.InstrumentationRegistry; +import androidx.test.uiautomator.UiDevice; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * Helper to run the generic collector that runs binary files that output metrics in a fixed format. + * <a href="http://go/generic-collector">(Design doc)</a> + */ +public class GenericExecutableCollectorHelper implements ICollectorHelper<String> { + private static final String TAG = GenericExecutableCollectorHelper.class.getSimpleName(); + private static final String CSV_SEPARATOR = ","; + private static final String METRIC_KEY_SEPARATOR = "_"; + + private Path mExecutableDir; + private UiDevice mUiDevice; + private List<Path> mExecutableFilePaths; + + /** + * Setup + * + * @param executableDir a string of executable directory + */ + public void setUp(String executableDir) { + mExecutableDir = Paths.get(executableDir); + mUiDevice = UiDevice.getInstance(InstrumentationRegistry.getInstrumentation()); + if (mExecutableDir == null || !Files.isDirectory(mExecutableDir)) { + throw new IllegalArgumentException( + "Executable directory was not a directory or was not specified."); + } + mExecutableFilePaths = listFilesInAllSubdirs(mExecutableDir); + Log.i( + TAG, + String.format( + "Found the following files: %s", + mExecutableFilePaths.stream() + .map(Path::toString) + .collect(Collectors.joining(", ")))); + if (mExecutableFilePaths.isEmpty()) { + throw new IllegalArgumentException( + String.format("No test file found in the directory %s", mExecutableDir)); + } + } + + @Override + public boolean startCollecting() { + return true; + } + + @Override + public Map<String, String> getMetrics() { + Map<String, String> results = new HashMap<>(); + mExecutableFilePaths.forEach( + (path) -> { + try { + results.putAll(execAndGetResults(path)); + } catch (IOException e) { + Log.e(TAG, String.format("Failed to execute file: %s", path), e); + } + }); + return results; + } + + @Override + public boolean stopCollecting() { + return true; + } + + /** + * List all files from a directory, including all levels of sub-directories. + * + * @param dir: a path of directory + * @return return: a list of paths of executable files + */ + private List<Path> listFilesInAllSubdirs(Path dir) { + List<Path> result = new ArrayList<>(); + try (Stream<Path> allFilesAndDirs = Files.walk(dir)) { + result = allFilesAndDirs.filter(Files::isRegularFile).collect(Collectors.toList()); + } catch (IOException e) { + Log.e(TAG, String.format("Failed to walk the files under path %s", dir), e); + } + return result; + } + + /** + * Running the binary by shell command and reformatting the output. + * + * <p>Example of output = "name,binder_use,binder_started,count\n" + "DockObserver,0,32,2\n" + + * "SurfaceFlinger,0,5,8\n" + "SurfaceFlingerAIDL,0,5,8\n"; + * + * <p>Example of lines = ["name,binder_use,binder_started,count", "DockObserver,0,32,2", + * "SurfaceFlinger,0,5,8", "SurfaceFlingerAIDL,0,5,8"] + * + * <p>Example of headers = ["name", "binder_use", "binder_started", "count"] + * + * <p>Example of result = { "DockObserver_binder_use" : 0 "DockObserver_binder_started" : 32 + * "DockObserver_count" : 2 } + * + * @param executable: a path of the executable file path + * @return result: a map including the metrics and values from the output + * @throws IOException if the shell command runs into errors + */ + private Map<String, String> execAndGetResults(Path executable) throws IOException { + String prefix = mExecutableDir.relativize(executable).toString(); + Map<String, String> result = new HashMap<>(); + String output = executeShellCommand(executable.toString()); + if (output.length() <= 0) { + return result; + } + String[] lines = output.split(System.lineSeparator()); + String[] headers = lines[0].split(CSV_SEPARATOR); + for (int row = 1; row < lines.length; row++) { + String[] l = lines[row].split(CSV_SEPARATOR); + for (int col = 1; col < l.length; col++) { + result.put(String.join(METRIC_KEY_SEPARATOR, prefix, l[0], headers[col]), l[col]); + } + } + return result; + } + + /** + * Execute a shell command and return its output. + * + * @param command a string of command + */ + @VisibleForTesting + public String executeShellCommand(String command) throws IOException { + return mUiDevice.executeShellCommand(command); + } +} diff --git a/libraries/collectors-helper/generic/test/Android.bp b/libraries/collectors-helper/generic/test/Android.bp new file mode 100644 index 000000000..98b5de113 --- /dev/null +++ b/libraries/collectors-helper/generic/test/Android.bp @@ -0,0 +1,33 @@ +// Copyright (C) 2023 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package { + default_applicable_licenses: ["Android-Apache-2.0"], +} + +java_library { + name: "generic-helper-test", + defaults: ["tradefed_errorprone_defaults"], + + srcs: ["src/**/*.java"], + + static_libs: [ + "androidx.test.runner", + "junit", + "mockito-target", + "generic-helper", + ], + + sdk_version: "current", +} diff --git a/libraries/collectors-helper/generic/test/src/com/android/helpers/tests/GenericExecutableCollectorHelperTest.java b/libraries/collectors-helper/generic/test/src/com/android/helpers/tests/GenericExecutableCollectorHelperTest.java new file mode 100644 index 000000000..15000565c --- /dev/null +++ b/libraries/collectors-helper/generic/test/src/com/android/helpers/tests/GenericExecutableCollectorHelperTest.java @@ -0,0 +1,203 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.helpers.tests; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; + +import androidx.test.runner.AndroidJUnit4; + +import com.android.helpers.GenericExecutableCollectorHelper; + +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.MockitoAnnotations; +import org.mockito.Spy; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.Map; + +/** + * Android unit tests for {@link GenericExecutableCollectorHelper}. + * + * <p>To run: atest CollectorsHelperAospTest:GenericExecutableCollectorHelperTest + */ +@RunWith(AndroidJUnit4.class) +public class GenericExecutableCollectorHelperTest { + private static final String TAG = GenericExecutableCollectorHelperTest.class.getSimpleName(); + private static final String VALID_EMPTY_DIR = "/data"; + private static final String INVALID_INPUT_DIR = "0"; + private static final String TEST_FILE_NAME = "test_file_"; + private static File sTestFile1; + private static File sTestFile2; + private static String sTestFile1NamePrefix; + private static String sTestFile2NamePrefix; + private @Spy GenericExecutableCollectorHelper mGenericExecutableCollectorHelper; + + @BeforeClass + public static void setUpFiles() throws IOException { + sTestFile1 = Files.createTempFile(TEST_FILE_NAME, "1").toFile(); + sTestFile2 = Files.createTempFile(TEST_FILE_NAME, "2").toFile(); + sTestFile1NamePrefix = sTestFile1.getName() + "_"; + sTestFile2NamePrefix = sTestFile2.getName() + "_"; + } + + @Before + public void setUp() { + MockitoAnnotations.initMocks(this); + } + + /** Test invalid input directory and throw an IllegalArgumentException */ + @Test + public void testBadInputDir() { + assertThrows( + "Executable directory was not a directory or was not specified.", + IllegalArgumentException.class, + () -> mGenericExecutableCollectorHelper.setUp(INVALID_INPUT_DIR)); + } + + /** Test valid input directory but empty folder and throw an IllegalArgumentException */ + @Test + public void testEmptyDir() { + assertThrows( + String.format("No test file found in the directory %s", VALID_EMPTY_DIR), + IllegalArgumentException.class, + () -> mGenericExecutableCollectorHelper.setUp(VALID_EMPTY_DIR)); + } + + /** Test valid input directory */ + @Test + public void testGoodDir() throws IOException { + mGenericExecutableCollectorHelper.setUp(sTestFile1.getParent()); + assertTrue(mGenericExecutableCollectorHelper.startCollecting()); + } + + /** Test valid input directory with multiple files */ + @Test + public void testMultipleGoodFiles() throws IOException { + String testOutput1 = + "name,binder_threads_in_use,binder_threads_started,client_count\n" + + "DockObserver,0,32,2\n" + + "SurfaceFlinger,0,5,8\n" + + "SurfaceFlingerAIDL,0,5,8\n"; + String testOutput2 = + "name,binder_threads_in_use,binder_threads_started,client_count\n" + + "camera.provider/internal/0,0,3,3\n" + + "cas.IMediaCasService/default,1,1,2\n" + + "confirmationui.IConfirmationUI/default,0,1,2\n"; + doReturn(testOutput1) + .when(mGenericExecutableCollectorHelper) + .executeShellCommand(sTestFile1.getPath()); + doReturn(testOutput2) + .when(mGenericExecutableCollectorHelper) + .executeShellCommand(sTestFile2.getPath()); + + mGenericExecutableCollectorHelper.setUp(sTestFile1.getParent()); + assertTrue(mGenericExecutableCollectorHelper.startCollecting()); + Map<String, String> metrics = mGenericExecutableCollectorHelper.getMetrics(); + + assertFalse(metrics.isEmpty()); + assertTrue( + metrics.containsKey(sTestFile1NamePrefix + "DockObserver_binder_threads_in_use")); + assertTrue( + metrics.containsKey(sTestFile1NamePrefix + "DockObserver_binder_threads_started")); + assertTrue(metrics.containsKey(sTestFile1NamePrefix + "DockObserver_client_count")); + assertEquals( + metrics.get(sTestFile1NamePrefix + "SurfaceFlinger_binder_threads_in_use"), "0"); + assertEquals( + metrics.get(sTestFile1NamePrefix + "SurfaceFlinger_binder_threads_started"), "5"); + assertEquals(metrics.get(sTestFile1NamePrefix + "SurfaceFlinger_client_count"), "8"); + + assertTrue( + metrics.containsKey( + sTestFile2NamePrefix + + "confirmationui.IConfirmationUI/default_binder_threads_in_use")); + assertTrue( + metrics.containsKey( + sTestFile2NamePrefix + + "confirmationui.IConfirmationUI/default_binder_threads_started")); + assertTrue( + metrics.containsKey( + sTestFile2NamePrefix + + "confirmationui.IConfirmationUI/default_client_count")); + assertEquals( + metrics.get( + sTestFile2NamePrefix + "camera.provider/internal/0_binder_threads_in_use"), + "0"); + assertEquals( + metrics.get( + sTestFile2NamePrefix + "camera.provider/internal/0_binder_threads_started"), + "3"); + assertEquals( + metrics.get(sTestFile2NamePrefix + "camera.provider/internal/0_client_count"), "3"); + } + + /** + * Test valid input directory with multiple files. If there is a bad file, the metrics are still + * collected from other good files. + */ + @Test + public void testBadExectuable_goodExecutableStillCollects() throws IOException { + String testOutput2 = + "name,binder_threads_in_use,binder_threads_started,client_count\n" + + "camera.provider/internal/0,0,3,3\n" + + "cas.IMediaCasService/default,1,1,2\n" + + "confirmationui.IConfirmationUI/default,0,1,2\n"; + doThrow(IOException.class) + .when(mGenericExecutableCollectorHelper) + .executeShellCommand(sTestFile1.getPath()); + doReturn(testOutput2) + .when(mGenericExecutableCollectorHelper) + .executeShellCommand(sTestFile2.getPath()); + + mGenericExecutableCollectorHelper.setUp(sTestFile1.getParent()); + assertTrue(mGenericExecutableCollectorHelper.startCollecting()); + Map<String, String> metrics = mGenericExecutableCollectorHelper.getMetrics(); + + assertFalse(metrics.isEmpty()); + assertTrue( + metrics.containsKey( + sTestFile2NamePrefix + + "confirmationui.IConfirmationUI/default_binder_threads_in_use")); + assertTrue( + metrics.containsKey( + sTestFile2NamePrefix + + "confirmationui.IConfirmationUI/default_binder_threads_started")); + assertTrue( + metrics.containsKey( + sTestFile2NamePrefix + + "confirmationui.IConfirmationUI/default_client_count")); + assertEquals( + metrics.get( + sTestFile2NamePrefix + "camera.provider/internal/0_binder_threads_in_use"), + "0"); + assertEquals( + metrics.get( + sTestFile2NamePrefix + "camera.provider/internal/0_binder_threads_started"), + "3"); + assertEquals( + metrics.get(sTestFile2NamePrefix + "camera.provider/internal/0_client_count"), "3"); + } +} diff --git a/libraries/collectors-helper/tests/Android.bp b/libraries/collectors-helper/tests/Android.bp index 502e9fcf9..06220c640 100644 --- a/libraries/collectors-helper/tests/Android.bp +++ b/libraries/collectors-helper/tests/Android.bp @@ -23,6 +23,7 @@ android_test { static_libs: [ "perfetto-helper-test", "app-collector-helper-test", + "generic-helper-test", "jank-helper-test", "memory-helper-test", "system-helper-test", diff --git a/libraries/device-collectors/src/main/Android.bp b/libraries/device-collectors/src/main/Android.bp index 98f9ec560..a09824504 100644 --- a/libraries/device-collectors/src/main/Android.bp +++ b/libraries/device-collectors/src/main/Android.bp @@ -27,6 +27,7 @@ java_library { "androidx.test.runner", "androidx.test.uiautomator", "app-collector-helper", + "generic-helper", "jank-helper", "junit", "lyric-metric-helper", diff --git a/libraries/device-collectors/src/main/java/android/device/collectors/GenericExecutableCollector.java b/libraries/device-collectors/src/main/java/android/device/collectors/GenericExecutableCollector.java new file mode 100644 index 000000000..9c8648120 --- /dev/null +++ b/libraries/device-collectors/src/main/java/android/device/collectors/GenericExecutableCollector.java @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package android.device.collectors; + +import android.device.collectors.annotations.OptionClass; +import android.os.Bundle; + +import com.android.helpers.GenericExecutableCollectorHelper; + +/** + * A {@link GenericExecutableCollector} a generic metric collector that collects metrics from + * arbitrary executables that output metrics in CSV format to handle all types of executable file + * execution and extract the metrics into a standard format. For more details: go/generic-collector + */ +@OptionClass(alias = "generic-executable-collector") +public class GenericExecutableCollector extends BaseCollectionListener<String> { + private static final String TAG = GenericExecutableCollector.class.getSimpleName(); + + static final String EXECUTABLE_DIR = "executable-dir"; + static final String DEFAULT_EXECUTABLE_DIR = "/data/generic_executable_collector/"; + + private GenericExecutableCollectorHelper mGenericExecutableCollectorHelper = + new GenericExecutableCollectorHelper(); + + public GenericExecutableCollector() { + createHelperInstance(mGenericExecutableCollectorHelper); + } + + @Override + public void setupAdditionalArgs() { + Bundle args = getArgsBundle(); + String executableDir = args.getString(EXECUTABLE_DIR, DEFAULT_EXECUTABLE_DIR); + mGenericExecutableCollectorHelper.setUp(executableDir); + } +} |