diff options
Diffstat (limited to 'src')
13 files changed, 523 insertions, 230 deletions
diff --git a/src/com/android/tradefed/build/OtatoolsBuildInfo.java b/src/com/android/tradefed/build/OtatoolsBuildInfo.java index 9f9d3f163..f9e732916 100644 --- a/src/com/android/tradefed/build/OtatoolsBuildInfo.java +++ b/src/com/android/tradefed/build/OtatoolsBuildInfo.java @@ -29,6 +29,13 @@ public class OtatoolsBuildInfo extends BuildInfo { private static final String RELEASETOOLS_DIR_NAME = "otatools_releasetools"; /** + * Creates a {@link OtatoolsBuildInfo} + */ + public OtatoolsBuildInfo(String buildId, String buildTargetName) { + super(buildId, buildTargetName); + } + + /** * Add /build/target/product/security to this file map */ public void setSecurityDir(File dir, String version) { diff --git a/src/com/android/tradefed/command/Console.java b/src/com/android/tradefed/command/Console.java index 56f35790a..4cd4db7e8 100644 --- a/src/com/android/tradefed/command/Console.java +++ b/src/com/android/tradefed/command/Console.java @@ -873,12 +873,16 @@ public class Console extends Thread { */ @SuppressWarnings("unchecked") void executeCmdRunnable(Runnable command, CaptureList groups) { - if (command instanceof ArgRunnable) { - // FIXME: verify that command implements ArgRunnable<CaptureList> instead - // FIXME: of just ArgRunnable - ((ArgRunnable<CaptureList>)command).run(groups); - } else { - command.run(); + try { + if (command instanceof ArgRunnable) { + // FIXME: verify that command implements ArgRunnable<CaptureList> instead + // FIXME: of just ArgRunnable + ((ArgRunnable<CaptureList>) command).run(groups); + } else { + command.run(); + } + } catch (RuntimeException e) { + e.printStackTrace(); } } diff --git a/src/com/android/tradefed/profiler/AggregatingProfiler.java b/src/com/android/tradefed/profiler/AggregatingProfiler.java index b24f87e4d..52865bc06 100644 --- a/src/com/android/tradefed/profiler/AggregatingProfiler.java +++ b/src/com/android/tradefed/profiler/AggregatingProfiler.java @@ -114,7 +114,8 @@ public class AggregatingProfiler implements IAggregatingTestProfiler { @Override public void reportAllMetrics(ITestInvocationListener listener) { mOutputUtil.addMetrics("aggregate", mContext.getTestTag(), mAggregateMetrics); - listener.testLog(getDescription(), LogDataType.TEXT, mOutputUtil.getFormattedMetrics()); + listener.testLog(getDescription(), LogDataType.MUGSHOT_LOG, + mOutputUtil.getFormattedMetrics()); } /** diff --git a/src/com/android/tradefed/profiler/recorder/MetricType.java b/src/com/android/tradefed/profiler/recorder/MetricType.java index 36d325ec4..65b85bc6b 100644 --- a/src/com/android/tradefed/profiler/recorder/MetricType.java +++ b/src/com/android/tradefed/profiler/recorder/MetricType.java @@ -16,13 +16,13 @@ package com.android.tradefed.profiler.recorder; -/** - * An enum describing different ways that {@link TraceMetric}s can be aggregated. - */ +/** An enum describing different ways that {@link TraceMetric}s can be aggregated. */ public enum MetricType { + AVG, + AVGTIME, COUNT, - SUM, COUNTPOS, - AVG, - AVGTIME + MAX, + MIN, + SUM, } diff --git a/src/com/android/tradefed/profiler/recorder/NumericAggregateFunction.java b/src/com/android/tradefed/profiler/recorder/NumericAggregateFunction.java new file mode 100644 index 000000000..6de96abb5 --- /dev/null +++ b/src/com/android/tradefed/profiler/recorder/NumericAggregateFunction.java @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.android.tradefed.profiler.recorder; + +import java.util.function.BiFunction; + +/** A wrapper of BiFunction that aggregates numeric values. */ +public class NumericAggregateFunction { + + private double mCount = 0; + private BiFunction<Double, Double, Double> f; + + /** Creates an aggregate function for the given {@link MetricType}. */ + public NumericAggregateFunction(MetricType metricType) { + switch (metricType) { + case AVG: + case AVGTIME: + f = (avg, value) -> avg + ((value - avg) / ++mCount); + return; + case COUNT: + f = (count, value) -> count + 1; + return; + case COUNTPOS: + f = (count, value) -> (value > 0 ? count + 1 : count); + return; + case MAX: + f = (max, value) -> Math.max(max, value); + return; + case MIN: + f = (min, value) -> Math.min(min, value); + return; + case SUM: + f = (sum, value) -> sum + value; + return; + default: + throw new IllegalArgumentException("Unknown metric type " + metricType.toString()); + } + } + + /** Returns the stored aggregate function. */ + public BiFunction<Double, Double, Double> getFunction() { + return f; + } +} diff --git a/src/com/android/tradefed/profiler/recorder/NumericMetricsRecorder.java b/src/com/android/tradefed/profiler/recorder/NumericMetricsRecorder.java deleted file mode 100644 index fcc48d17e..000000000 --- a/src/com/android/tradefed/profiler/recorder/NumericMetricsRecorder.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (C) 2016 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.android.tradefed.profiler.recorder; - -import java.util.function.BiFunction; - -/** - * A {@link IMetricsRecorder} that aggregates metrics using some basic numeric functions. This class - * doesn't implement any methods of the {@link IMetricsRecorder} interface, it just provides - * (possibly stateful) numeric functions to its subclasses. - */ -public abstract class NumericMetricsRecorder implements IMetricsRecorder { - - private double mRunningCount = 0; - - /** - * Provides an aggregator function which sums values. - * - * @return a sum function - */ - protected BiFunction<Double, Double, Double> sum() { - return (oldVal, newVal) -> oldVal + newVal; - } - - /** - * Provides an aggregator function which counts values. - * - * @return a count function - */ - protected BiFunction<Double, Double, Double> count() { - return (oldVal, newVal) -> oldVal + 1; - } - - /** - * Provides an aggregator function which counts positive values. - * - * @return a countpos function - */ - protected BiFunction<Double, Double, Double> countpos() { - return (oldVal, newVal) -> (newVal == 0 ? oldVal : oldVal + 1); - } - - /** - * Provides an aggregator function which average values. - * - * @return an average function - */ - protected BiFunction<Double, Double, Double> avg() { - return (prevAvg, newVal) -> prevAvg + ((newVal - prevAvg) / ++mRunningCount); - } -} diff --git a/src/com/android/tradefed/profiler/recorder/TraceMetricsRecorder.java b/src/com/android/tradefed/profiler/recorder/TraceMetricsRecorder.java index 57960ebc4..453ddd0b5 100644 --- a/src/com/android/tradefed/profiler/recorder/TraceMetricsRecorder.java +++ b/src/com/android/tradefed/profiler/recorder/TraceMetricsRecorder.java @@ -37,7 +37,7 @@ import java.util.function.BiFunction; * Metrics to be recorded need to be provided as TraceMetrics. The default descriptor * has the format prefix:funcname:param[=expectedval]:metrictype. */ -public class TraceMetricsRecorder extends NumericMetricsRecorder { +public class TraceMetricsRecorder implements IMetricsRecorder { private static final String TRACE_DIR = "/d/tracing"; private static final String EVENT_DIR = TRACE_DIR + "/events/"; @@ -56,7 +56,8 @@ public class TraceMetricsRecorder extends NumericMetricsRecorder { TraceMetric metric = TraceMetric.parse(descriptor); enableSingleEventTrace(device, metric.getPrefix() + "/" + metric.getFuncName()); mTraceMetrics.put(metric.getFuncName(), metric); - mMergeFunctions.put(metric, getMergeFunctionByMetricType(metric.getMetricType())); + mMergeFunctions.put( + metric, new NumericAggregateFunction(metric.getMetricType()).getFunction()); } } @@ -147,17 +148,6 @@ public class TraceMetricsRecorder extends NumericMetricsRecorder { device.executeShellCommand("echo 1 > " + fullLocation); } - private BiFunction<Double, Double, Double> getMergeFunctionByMetricType(MetricType t) { - switch(t) { - case COUNT: return count(); - case COUNTPOS: return countpos(); - case SUM: return sum(); - case AVG: - case AVGTIME: return avg(); - default: throw new IllegalArgumentException("unknown metric type " + t); - } - } - protected BufferedReader getReaderFromFile(File trace) throws FileNotFoundException { return new BufferedReader(new FileReader(trace)); } diff --git a/src/com/android/tradefed/result/FileMetadataCollector.java b/src/com/android/tradefed/result/FileMetadataCollector.java index 28f8fffb2..15905cfe2 100644 --- a/src/com/android/tradefed/result/FileMetadataCollector.java +++ b/src/com/android/tradefed/result/FileMetadataCollector.java @@ -110,6 +110,8 @@ public class FileMetadataCollector implements ILogSaverListener, ITestInvocation return LogType.COMPACT_MEMINFO; case SERVICES: return LogType.SERVICES; + case MUGSHOT_LOG: + return LogType.MUGSHOT; default: // All others return LogType.UNKNOWN; } diff --git a/src/com/android/tradefed/result/LogDataType.java b/src/com/android/tradefed/result/LogDataType.java index ee19af415..58d79f083 100644 --- a/src/com/android/tradefed/result/LogDataType.java +++ b/src/com/android/tradefed/result/LogDataType.java @@ -37,6 +37,7 @@ public enum LogDataType { LOGCAT("txt", "text/plain", false, true), KERNEL_LOG("txt", "text/plain", false, true), MONKEY_LOG("txt", "text/plain", false, true), + MUGSHOT_LOG("txt", "text/plain", false, true), PROCRANK("txt", "text/plain", false, true), MEM_INFO("txt", "text/plain", false, true), TOP("txt", "text/plain", false, true), diff --git a/src/com/android/tradefed/testtype/CodeCoverageTestBase.java b/src/com/android/tradefed/testtype/CodeCoverageTestBase.java index 8a4ae4ece..932c19306 100644 --- a/src/com/android/tradefed/testtype/CodeCoverageTestBase.java +++ b/src/com/android/tradefed/testtype/CodeCoverageTestBase.java @@ -15,6 +15,8 @@ */ package com.android.tradefed.testtype; +import static com.google.common.base.Preconditions.checkState; + import com.android.ddmlib.testrunner.IRemoteAndroidTestRunner; import com.android.ddmlib.testrunner.RemoteAndroidTestRunner; import com.android.ddmlib.testrunner.TestIdentifier; @@ -35,6 +37,8 @@ import com.android.tradefed.util.FileUtil; import com.android.tradefed.util.ICompressionStrategy; import com.android.tradefed.util.ListInstrumentationParser; import com.android.tradefed.util.ListInstrumentationParser.InstrumentationTarget; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableMap; import java.io.File; import java.io.IOException; @@ -65,9 +69,11 @@ public abstract class CodeCoverageTestBase<T extends CodeCoverageReportFormat> description = "Only run instrumentation targets with the given test runner") private List<String> mRunnerFilter = new ArrayList<>(); - @Option(name = "instrumentation-arg", - description = "Additional instrumentation arguments to provide to the runner") - private Map<String, String> mInstrArgMap = new HashMap<String, String>(); + @Option( + name = "instrumentation-arg", + description = "Additional instrumentation arguments to provide to the runner" + ) + private Map<String, String> mInstrumentationArgs = new HashMap<String, String>(); @Option(name = "max-tests-per-chunk", description = "Maximum number of tests to execute in a single call to 'am instrument'. " @@ -113,14 +119,32 @@ public abstract class CodeCoverageTestBase<T extends CodeCoverageReportFormat> return mPackageFilter; } + /** Sets the package-filter option for testing. */ + @VisibleForTesting + void setPackageFilter(List<String> packageFilter) { + mPackageFilter = packageFilter; + } + /** Returns the runner filter as set by the --runner option(s). */ List<String> getRunnerFilter() { return mRunnerFilter; } + /** Sets the runner-filter option for testing. */ + @VisibleForTesting + void setRunnerFilter(List<String> runnerFilter) { + mRunnerFilter = runnerFilter; + } + /** Returns the instrumentation arguments as set by the --instrumentation-arg option(s). */ Map<String, String> getInstrumentationArgs() { - return mInstrArgMap; + return mInstrumentationArgs; + } + + /** Sets the instrumentation-arg options for testing. */ + @VisibleForTesting + void setInstrumentationArgs(Map<String, String> instrumentationArgs) { + mInstrumentationArgs = ImmutableMap.copyOf(instrumentationArgs); } /** Returns the maximum number of tests to run at once as set by --max-tests-per-chunk. */ @@ -128,6 +152,12 @@ public abstract class CodeCoverageTestBase<T extends CodeCoverageReportFormat> return mMaxTestsPerChunk; } + /** Sets the max-tests-per-chunk option for testing. */ + @VisibleForTesting + void setMaxTestsPerChunk(int maxTestsPerChunk) { + mMaxTestsPerChunk = maxTestsPerChunk; + } + /** Returns the compression strategy that should be used to archive the coverage report. */ ICompressionStrategy getCompressionStrategy() { try { @@ -159,7 +189,7 @@ public abstract class CodeCoverageTestBase<T extends CodeCoverageReportFormat> File reportArchive = null; // Initialize a listener to collect logged coverage files try (CoverageCollectingListener coverageListener = - new CoverageCollectingListener(listener)) { + new CoverageCollectingListener(getDevice(), listener)) { // Make sure there are some installed instrumentation targets Collection<InstrumentationTarget> instrumentationTargets = getInstrumentationTargets(); @@ -195,8 +225,9 @@ public abstract class CodeCoverageTestBase<T extends CodeCoverageReportFormat> } // Generate the coverage report(s) and log it + List<File> measurements = coverageListener.getCoverageFiles(); for (T format : getReportFormat()) { - File report = generateCoverageReport(coverageListener.getCoverageFiles(), format); + File report = generateCoverageReport(measurements, format); try { doLogReport("coverage", format.getLogDataType(), report, listener); } finally { @@ -359,7 +390,7 @@ public abstract class CodeCoverageTestBase<T extends CodeCoverageReportFormat> */ TestRunResult runTest(InstrumentationTarget target, int shardIndex, int numShards, ITestInvocationListener listener) throws DeviceNotAvailableException { - return runTest(createCoverageTest(target, shardIndex, numShards), listener); + return runTest(createTest(target, shardIndex, numShards), listener); } /** @@ -372,11 +403,11 @@ public abstract class CodeCoverageTestBase<T extends CodeCoverageReportFormat> */ TestRunResult runTest(InstrumentationTarget target, TestIdentifier identifier, ITestInvocationListener listener) throws DeviceNotAvailableException { - return runTest(createCoverageTest(target, identifier), listener); + return runTest(createTest(target, identifier), listener); } - /** Runs the given {@link CodeCoverageTest} and returns the {@link TestRunResult}. */ - TestRunResult runTest(CodeCoverageTest test, ITestInvocationListener listener) + /** Runs the given {@link InstrumentationTest} and returns the {@link TestRunResult}. */ + TestRunResult runTest(InstrumentationTest test, ITestInvocationListener listener) throws DeviceNotAvailableException { // Run the test, and return the run results CollectingTestListener results = new CollectingTestListener(); @@ -384,31 +415,35 @@ public abstract class CodeCoverageTestBase<T extends CodeCoverageReportFormat> return results.getCurrentRunResults(); } - /** Returns a new {@link CodeCoverageTest}. Exposed for unit testing. */ - CodeCoverageTest internalCreateCoverageTest() { - return new CodeCoverageTest(); + /** Returns a new {@link InstrumentationTest}. Exposed for unit testing. */ + InstrumentationTest internalCreateTest() { + return new InstrumentationTest(); } - /** Returns a new {@link CodeCoverageTest} for the given target. */ - CodeCoverageTest createCoverageTest(InstrumentationTarget target) { - // Get a new CodeCoverageTest instance - CodeCoverageTest ret = internalCreateCoverageTest(); + /** Returns a new {@link InstrumentationTest} for the given target. */ + InstrumentationTest createTest(InstrumentationTarget target) { + // Get a new InstrumentationTest instance + InstrumentationTest ret = internalCreateTest(); ret.setDevice(getDevice()); ret.setPackageName(target.packageName); ret.setRunnerName(target.runnerName); + // Disable rerun mode, we want to stop the tests as soon as we fail. + ret.setRerunMode(false); + // Add instrumentation arguments for (Map.Entry<String, String> argEntry : getInstrumentationArgs().entrySet()) { ret.addInstrumentationArg(argEntry.getKey(), argEntry.getValue()); } + ret.addInstrumentationArg("coverage", "true"); return ret; } - /** Returns a new {@link CodeCoverageTest} for the identified test on the given target. */ - CodeCoverageTest createCoverageTest(InstrumentationTarget target, TestIdentifier identifier) { - // Get a new CodeCoverageTest instance - CodeCoverageTest ret = createCoverageTest(target); + /** Returns a new {@link InstrumentationTest} for the identified test on the given target. */ + InstrumentationTest createTest(InstrumentationTarget target, TestIdentifier identifier) { + // Get a new InstrumentationTest instance + InstrumentationTest ret = createTest(target); // Set the specific test method to run ret.setClassName(identifier.getClassName()); @@ -417,11 +452,10 @@ public abstract class CodeCoverageTestBase<T extends CodeCoverageReportFormat> return ret; } - /** Returns a new {@link CodeCoverageTest} for a particular shard on the given target. */ - CodeCoverageTest createCoverageTest(InstrumentationTarget target, int shardIndex, - int numShards) { - // Get a new CodeCoverageTest instance - CodeCoverageTest ret = createCoverageTest(target); + /** Returns a new {@link InstrumentationTest} for a particular shard on the given target. */ + InstrumentationTest createTest(InstrumentationTarget target, int shardIndex, int numShards) { + // Get a new InstrumentationTest instance + InstrumentationTest ret = createTest(target); // Add shard options if necessary if (numShards > 1) { @@ -436,22 +470,24 @@ public abstract class CodeCoverageTestBase<T extends CodeCoverageReportFormat> public static class CoverageCollectingListener extends ResultForwarder implements AutoCloseable { + private ITestDevice mDevice; private List<File> mCoverageFiles = new ArrayList<>(); private File mCoverageDir; + private String mCurrentRunName; - public CoverageCollectingListener(ITestInvocationListener... listeners) throws IOException { + public CoverageCollectingListener(ITestDevice device, ITestInvocationListener... listeners) + throws IOException { super(listeners); + mDevice = device; + // Initialize a directory to store the coverage files mCoverageDir = FileUtil.createTempDir("execution_data"); } /** Returns the list of collected coverage files. */ public List<File> getCoverageFiles() { - // It is an error to use this object after it has been closed - if (mCoverageDir == null) { - throw new IllegalStateException("This object is closed"); - } + checkState(mCoverageDir != null, "This object is closed"); return mCoverageFiles; } @@ -460,32 +496,66 @@ public abstract class CodeCoverageTestBase<T extends CodeCoverageReportFormat> */ @Override public void testLog(String dataName, LogDataType dataType, InputStreamSource dataStream) { - // It is an error to use this object after it has been closed - if (mCoverageDir == null) { - throw new IllegalStateException("This object is closed"); - } + super.testLog(dataName, dataType, dataStream); + checkState(mCoverageDir != null, "This object is closed"); // We only care about coverage files - if (!LogDataType.COVERAGE.equals(dataType)) { - super.testLog(dataName, dataType, dataStream); - return; + if (LogDataType.COVERAGE.equals(dataType)) { + // Save coverage data to a temporary location, and don't inform the listeners yet + try { + File coverageFile = + FileUtil.createTempFile(dataName + "_", ".exec", mCoverageDir); + FileUtil.writeToFile(dataStream.createInputStream(), coverageFile); + mCoverageFiles.add(coverageFile); + CLog.d("Got coverage file: %s", coverageFile.getAbsolutePath()); + } catch (IOException e) { + CLog.e("Failed to save coverage file"); + CLog.e(e); + } } + } - // Save coverage data to a temporary location, and don't inform the listeners yet - try { - File coverageFile = FileUtil.createTempFile(dataName + "_", ".exec", mCoverageDir); - FileUtil.writeToFile(dataStream.createInputStream(), coverageFile); - mCoverageFiles.add(coverageFile); - CLog.d("Got coverage file: %s", coverageFile.getAbsolutePath()); - } catch (IOException e) { - CLog.e("Failed to save coverage file"); - CLog.e(e); + /** {@inheritDoc} */ + @Override + public void testRunStarted(String runName, int testCount) { + super.testRunStarted(runName, testCount); + mCurrentRunName = runName; + } + + /** {@inheritDoc} */ + @Override + public void testRunEnded(long elapsedTime, Map<String, String> runMetrics) { + // Look for the coverage file path from the run metrics + String coverageFilePath = runMetrics.get(CodeCoverageTest.COVERAGE_REMOTE_FILE_LABEL); + if (coverageFilePath != null) { + CLog.d("Coverage file at %s", coverageFilePath); + + // Try to pull the coverage measurements off of the device + File coverageFile = null; + try { + coverageFile = mDevice.pullFile(coverageFilePath); + if (coverageFile != null) { + FileInputStreamSource source = new FileInputStreamSource(coverageFile); + testLog( + mCurrentRunName + "_runtime_coverage", + LogDataType.COVERAGE, + source); + source.cancel(); + } else { + CLog.w("Failed to pull coverage file from device: %s", coverageFilePath); + } + } catch (DeviceNotAvailableException e) { + // Nothing we can do, so just log the error. + CLog.w(e); + } finally { + FileUtil.deleteFile(coverageFile); + } } + + super.testRunEnded(elapsedTime, runMetrics); } - /** - * {@inheritDoc} - */ + /** {@inheritDoc} */ @Override public void close() { FileUtil.recursiveDelete(mCoverageDir); diff --git a/src/com/android/tradefed/testtype/GTest.java b/src/com/android/tradefed/testtype/GTest.java index dcff95bfc..23af7bf95 100644 --- a/src/com/android/tradefed/testtype/GTest.java +++ b/src/com/android/tradefed/testtype/GTest.java @@ -116,6 +116,9 @@ public class GTest description = "adb shell command(s) to run after GTest.") private List<String> mAfterTestCmd = new ArrayList<>(); + @Option(name = "run-test-as", description = "User to execute test binary as.") + private String mRunTestAs = null; + @Option(name = "ld-library-path", description = "LD_LIBRARY_PATH value to include in the GTest execution command.") private String mLdLibraryPath = null; @@ -183,10 +186,6 @@ public class GTest return mDevice; } - public void setEnableXmlOutput(boolean b) { - mEnableXmlOutput = b; - } - /** * Set the Android native test module to run. * @@ -649,6 +648,12 @@ public class GTest gTestCmdLine.append(String.format("GTEST_SHARD_INDEX=%s ", mShardIndex)); gTestCmdLine.append(String.format("GTEST_TOTAL_SHARDS=%s ", mShardCount)); } + + // su to requested user + if (mRunTestAs != null) { + gTestCmdLine.append(String.format("su %s ", mRunTestAs)); + } + gTestCmdLine.append(String.format("%s %s", fullPath, flags)); return gTestCmdLine.toString(); } @@ -751,7 +756,4 @@ public class GTest mCollectTestsOnly = shouldCollectTest; } - protected void setLoadFilterFromFile(String loadFilterFromFile) { - mTestFilterKey = loadFilterFromFile; - } } diff --git a/src/com/android/tradefed/testtype/PythonUnitTestResultParser.java b/src/com/android/tradefed/testtype/PythonUnitTestResultParser.java index 7a893e5f8..1cafb3dd6 100644 --- a/src/com/android/tradefed/testtype/PythonUnitTestResultParser.java +++ b/src/com/android/tradefed/testtype/PythonUnitTestResultParser.java @@ -18,13 +18,14 @@ package com.android.tradefed.testtype; import com.android.ddmlib.MultiLineReceiver; import com.android.ddmlib.testrunner.ITestRunListener; import com.android.ddmlib.testrunner.TestIdentifier; -import com.android.tradefed.log.LogUtil.CLog; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * Interprets the output of tests run with Python's unittest framework and translates it into @@ -39,8 +40,7 @@ import java.util.Map.Entry; * Status ::= “OK” | “FAILED (errors=” int “)”. * Traceback ::= string+. * - * Example output: - * (passing) + * Example output (passing): * test_size (test_rangelib.RangeSetTest) ... ok * test_str (test_rangelib.RangeSetTest) ... ok * test_subtract (test_rangelib.RangeSetTest) ... ok @@ -51,7 +51,8 @@ import java.util.Map.Entry; * Ran 5 tests in 0.002s * * OK - * (failed) + * + * Example output (failed) * test_size (test_rangelib.RangeSetTest) ... ERROR * * ====================================================================== @@ -62,8 +63,50 @@ import java.util.Map.Entry; * raise ValueError() * ValueError * ---------------------------------------------------------------------- - * Ran 1 tests in 0.001s + * Ran 1 test in 0.001s * FAILED (errors=1) + * + * Example output with several edge cases (failed): + * testError (foo.testFoo) ... ERROR + * testExpectedFailure (foo.testFoo) ... expected failure + * testFail (foo.testFoo) ... FAIL + * testFailWithDocString (foo.testFoo) + * foo bar ... FAIL + * testOk (foo.testFoo) ... ok + * testOkWithDocString (foo.testFoo) + * foo bar ... ok + * testSkipped (foo.testFoo) ... skipped 'reason foo' + * testUnexpectedSuccess (foo.testFoo) ... unexpected success + * + * ====================================================================== + * ERROR: testError (foo.testFoo) + * ---------------------------------------------------------------------- + * Traceback (most recent call last): + * File "foo.py", line 11, in testError + * self.assertEqual(2+2, 5/0) + * ZeroDivisionError: integer division or modulo by zero + * + * ====================================================================== + * FAIL: testFail (foo.testFoo) + * ---------------------------------------------------------------------- + * Traceback (most recent call last): + * File "foo.py", line 8, in testFail + * self.assertEqual(2+2, 5) + * AssertionError: 4 != 5 + * + * ====================================================================== + * FAIL: testFailWithDocString (foo.testFoo) + * foo bar + * ---------------------------------------------------------------------- + * Traceback (most recent call last): + * File "foo.py", line 31, in testFailWithDocString + * self.assertEqual(2+2, 5) + * AssertionError: 4 != 5 + * + * ---------------------------------------------------------------------- + * Ran 8 tests in 0.001s + * + * FAILED (failures=2, errors=1, skipped=1, expected failures=1, unexpected successes=1) */ public class PythonUnitTestResultParser extends MultiLineReceiver { @@ -80,22 +123,38 @@ public class PythonUnitTestResultParser extends MultiLineReceiver { int mTotalTestCount; // General state - private Map<TestIdentifier, String> mTestResultCache; private int mFailedTestCount; private final Collection<ITestRunListener> mListeners; private final String mRunName; + private Map<TestIdentifier, String> mTestResultCache; + // Use a special entry to mark skipped test in mTestResultCache + static final String SKIPPED_ENTRY = "Skipped"; // Constant tokens that appear in the result grammar. static final String EQLINE = "======================================================================"; static final String LINE = "----------------------------------------------------------------------"; - static final String TRACEBACK_LINE = "Traceback (most recent call last):"; - static final String CASE_OK = "ok"; - static final String CASE_EXPECTED_FAILURE_1 = "expected"; - static final String CASE_EXPECTED_FAILURE_2 = "failure"; - static final String RUN_OK = "OK"; - static final String RUN_FAILED = "FAILED"; + static final String TRACEBACK_LINE = + "Traceback (most recent call last):"; + + static final Pattern PATTERN_TEST_SUCCESS = Pattern.compile("ok|expected failure"); + static final Pattern PATTERN_TEST_FAILURE = Pattern.compile("FAIL|ERROR"); + static final Pattern PATTERN_TEST_SKIPPED = Pattern.compile("skipped '.*"); + static final Pattern PATTERN_TEST_UNEXPECTED_SUCCESS = Pattern.compile("unexpected success"); + + static final Pattern PATTERN_ONE_LINE_RESULT = Pattern.compile( + "(\\S*) \\((\\S*)\\) ... (ok|expected failure|FAIL|ERROR|skipped '.*'|unexpected success)"); + static final Pattern PATTERN_TWO_LINE_RESULT_FIRST = Pattern.compile( + "(\\S*) \\((\\S*)\\)"); + static final Pattern PATTERN_TWO_LINE_RESULT_SECOND = Pattern.compile( + "(.*) ... (ok|expected failure|FAIL|ERROR|skipped '.*'|unexpected success)"); + static final Pattern PATTERN_FAIL_MESSAGE = Pattern.compile( + "(FAIL|ERROR): (\\S*) \\((\\S*)\\)"); + static final Pattern PATTERN_RUN_SUMMARY = Pattern.compile( + "Ran (\\d+) tests? in (\\d+(.\\d*)?)s"); + + static final Pattern PATTERN_RUN_RESULT = Pattern.compile("(OK|FAILED).*"); /** * Keeps track of the state the parser is currently in. @@ -107,20 +166,21 @@ public class PythonUnitTestResultParser extends MultiLineReceiver { * State progression: * * v------, - * TEST_CASE-'->[failed?]-(n)->TEST_SUMMARY-->TEST_STATUS-->COMPLETE + * TEST_CASE-'->[failed?]-(n)-->RUN_SUMMARY-->RUN_RESULT-->COMPLETE * | ^ * (y) '------(n)--, - * | ,-TEST_TRACEBACK->[more?] + * | ,---TRACEBACK---->[more?] * v v ^ | * FAIL_MESSAGE ---' (y) * ^-------------------' */ static enum ParserState { TEST_CASE, - TEST_TRACEBACK, - TEST_SUMMARY, - TEST_STATUS, + TRACEBACK, + RUN_SUMMARY, + RUN_RESULT, FAIL_MESSAGE, + FAIL_MESSAGE_OPTIONAL_DOCSTRING, COMPLETE } @@ -166,92 +226,115 @@ public class PythonUnitTestResultParser extends MultiLineReceiver { void parse() throws PythonUnitTestParseException { switch (mCurrentParseState) { case TEST_CASE: - testResult(); + testCase(); break; - case TEST_TRACEBACK: + case TRACEBACK: traceback(); break; - case TEST_SUMMARY: - summary(); + case RUN_SUMMARY: + runSummary(); break; - case TEST_STATUS: - completeTestRun(); + case RUN_RESULT: + runResult(); break; case FAIL_MESSAGE: failMessage(); break; + case FAIL_MESSAGE_OPTIONAL_DOCSTRING: + failMessageOptionalDocstring(); + break; case COMPLETE: break; } } - void testResult() throws PythonUnitTestParseException { - // we're at the end of the TEST_CASE section + void testCase() throws PythonUnitTestParseException { + // separate line before traceback message if (eqline()) { mCurrentParseState = ParserState.FAIL_MESSAGE; return; } + // separate line before test summary if (line()) { - mCurrentParseState = ParserState.TEST_SUMMARY; + mCurrentParseState = ParserState.RUN_SUMMARY; + return; + } + // empty line preceding the separate line + if (emptyLine()) { + // skip return; } // actually process the test case mCurrentParseState = ParserState.TEST_CASE; - String[] toks = mCurrentLine.split(" "); - try { - String testName = toks[0]; - // strip surrounding parens from class name - String testClass = toks[1].substring(1, toks[1].length() - 1); - mCurrentTestId = new TestIdentifier(testClass, testName); - // 3rd token is just "..." - if (toks.length == 4) { - // one-word status ("ok" | "ERROR") - String status = toks[3]; - if (CASE_OK.equals(status)) { - markTestSuccess(); - } - // if there's an error just do nothing, we can't get the trace - // immediately anyway - } else if (toks.length == 5) { - // two-word status ("expected failure") - String status1 = toks[3]; - String status2 = toks[4]; - if (CASE_EXPECTED_FAILURE_1.equals(status1) - && CASE_EXPECTED_FAILURE_2.equals(status2)) { - markTestSuccess(); - } - } else { - parseError("TestResult"); + String testName = null, testClass = null, status = null; + Matcher m = PATTERN_ONE_LINE_RESULT.matcher(mCurrentLine); + if (m.matches()) { + // one line test result + testName = m.group(1); + testClass = m.group(2); + status = m.group(3); + } else { + // two line test result + Matcher m1 = PATTERN_TWO_LINE_RESULT_FIRST.matcher(mCurrentLine); + if (!m1.matches()) { + parseError("Test case and result"); + } + testName = m1.group(1); + testClass = m1.group(2); + if (!advance()) { + parseError("Second line of test result"); + } + Matcher m2 = PATTERN_TWO_LINE_RESULT_SECOND.matcher(mCurrentLine); + if (!m2.matches()) { + parseError("Second line of test result"); } - } catch (ArrayIndexOutOfBoundsException e) { - CLog.d("Underlying error in testResult: " + e); - throw new PythonUnitTestParseException("FailMessage"); + status = m2.group(2); + } + mCurrentTestId = new TestIdentifier(testClass, testName); + if (PATTERN_TEST_SUCCESS.matcher(status).matches()) { + markTestSuccess(); + } else if (PATTERN_TEST_SKIPPED.matcher(status).matches()) { + markTestSkipped(); + } else if (PATTERN_TEST_UNEXPECTED_SUCCESS.matcher(status).matches()) { + markTestUnexpectedSuccess(); + } else if (PATTERN_TEST_FAILURE.matcher(status).matches()) { + // Do nothing because we can't get the trace immediately + } else { + throw new PythonUnitTestParseException("Unrecognized test status"); } } void failMessage() throws PythonUnitTestParseException { - // traceback is starting - if (line()) { - mCurrentParseState = ParserState.TEST_TRACEBACK; - mCurrentTraceback = new StringBuilder(); - return; + Matcher m = PATTERN_FAIL_MESSAGE.matcher(mCurrentLine); + if (!m.matches()) { + throw new PythonUnitTestParseException("Failed to parse test failure message"); } - String[] toks = mCurrentLine.split(" "); - // 1st token is "ERROR:" - try { - String testName = toks[1]; - String testClass = toks[2].substring(1, toks[2].length() - 1); - mCurrentTestId = new TestIdentifier(testClass, testName); - } catch (ArrayIndexOutOfBoundsException e) { - CLog.d("Underlying error in failMessage: " + e); - throw new PythonUnitTestParseException("FailMessage"); + String testName = m.group(2); + String testClass = m.group(3); + mCurrentTestId = new TestIdentifier(testClass, testName); + mCurrentParseState = ParserState.FAIL_MESSAGE_OPTIONAL_DOCSTRING; + } + + void failMessageOptionalDocstring() throws PythonUnitTestParseException { + // skip the optional docstring line if there is one; do nothing otherwise + if (!line()) { + advance(); + } + preTraceback(); + } + + void preTraceback() throws PythonUnitTestParseException { + if (!line()) { + throw new PythonUnitTestParseException("Failed to parse test failure message"); } + mCurrentParseState = ParserState.TRACEBACK; + mCurrentTraceback = new StringBuilder(); } void traceback() throws PythonUnitTestParseException { // traceback is always terminated with LINE or EQLINE - while (!mCurrentLine.startsWith(LINE) && !mCurrentLine.startsWith(EQLINE)) { + while (!line() && !eqline()) { mCurrentTraceback.append(mCurrentLine); if (!advance()) return; } @@ -260,7 +343,7 @@ public class PythonUnitTestResultParser extends MultiLineReceiver { markTestFailure(); // move on to the next section if (line()) { - mCurrentParseState = ParserState.TEST_SUMMARY; + mCurrentParseState = ParserState.RUN_SUMMARY; } else if (eqline()) { mCurrentParseState = ParserState.FAIL_MESSAGE; @@ -270,24 +353,27 @@ public class PythonUnitTestResultParser extends MultiLineReceiver { } } - void summary() throws PythonUnitTestParseException { - String[] toks = mCurrentLine.split(" "); + void runSummary() throws PythonUnitTestParseException { + Matcher m = PATTERN_RUN_SUMMARY.matcher(mCurrentLine); + if (!m.matches()) { + throw new PythonUnitTestParseException("Failed to parse test summary"); + } double time = 0; try { - mTotalTestCount = Integer.parseInt(toks[1]); + mTotalTestCount = Integer.parseInt(m.group(1)); } catch (NumberFormatException e) { parseError("integer"); } try { - time = Double.parseDouble(toks[4].substring(0, toks[4].length() - 1)); + time = Double.parseDouble(m.group(2)); } catch (NumberFormatException e) { parseError("double"); } mTotalElapsedTime = (long) time * 1000; - mCurrentParseState = ParserState.TEST_STATUS; + mCurrentParseState = ParserState.RUN_RESULT; } - boolean completeTestRun() throws PythonUnitTestParseException { + void runResult() throws PythonUnitTestParseException { String failReason = String.format("Failed %d tests", mFailedTestCount); for (ITestRunListener listener: mListeners) { // do testRunStarted @@ -296,22 +382,25 @@ public class PythonUnitTestResultParser extends MultiLineReceiver { // mark each test passed or failed for (Entry<TestIdentifier, String> test : mTestResultCache.entrySet()) { listener.testStarted(test.getKey()); - if (test.getValue() != null) { + if (SKIPPED_ENTRY.equals(test.getValue())) { + listener.testIgnored(test.getKey()); + } else if (test.getValue() != null) { listener.testFailed(test.getKey(), test.getValue()); } listener.testEnded(test.getKey(), Collections.<String, String>emptyMap()); } // mark the whole run as passed or failed - if (mCurrentLine.startsWith(RUN_FAILED)) { + // do not rely on the final result message, because Python consider "unexpected success" + // passed while we consider it failed + if (!PATTERN_RUN_RESULT.matcher(mCurrentLine).matches()) { + parseError("Status"); + } + if (mFailedTestCount > 0) { listener.testRunFailed(failReason); } listener.testRunEnded(mTotalElapsedTime, Collections.<String, String>emptyMap()); - if (!mCurrentLine.startsWith(RUN_FAILED) && !mCurrentLine.startsWith(RUN_OK)) { - parseError("Status"); - } } - return true; } boolean eqline() { @@ -326,6 +415,10 @@ public class PythonUnitTestResultParser extends MultiLineReceiver { return mCurrentLine.startsWith(TRACEBACK_LINE); } + boolean emptyLine() { + return mCurrentLine.isEmpty(); + } + /** * Advance to the next non-empty line. * @return true if a non-empty line was found, false otherwise. @@ -356,6 +449,18 @@ public class PythonUnitTestResultParser extends MultiLineReceiver { mFailedTestCount++; } + private void markTestSkipped() { + mTestResultCache.put(mCurrentTestId, SKIPPED_ENTRY); + } + + private void markTestUnexpectedSuccess() { + // In Python unittest, "unexpected success" (tests that are marked with + // @unittest.expectedFailure but passed) will not fail the entire test run. + // This behaviour is usually not desired, and such test should be treated as failed. + mTestResultCache.put(mCurrentTestId, "Test unexpected succeeded"); + mFailedTestCount++; + } + @Override public boolean isCancelled() { return false; diff --git a/src/com/android/tradefed/testtype/suite/TestSuiteInfo.java b/src/com/android/tradefed/testtype/suite/TestSuiteInfo.java new file mode 100644 index 000000000..8a6c36efe --- /dev/null +++ b/src/com/android/tradefed/testtype/suite/TestSuiteInfo.java @@ -0,0 +1,118 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.android.tradefed.testtype.suite; + +import com.android.tradefed.log.LogUtil.CLog; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; + +/** + * A class that resolves loading of build related metadata for test suite + * <p> + * To properly expose related info, a test suite must include a + * <code>test-suite-info.properties</code> file in its jar resources + */ +public class TestSuiteInfo { + + /** expected property filename in jar resource */ + private static final String SUITE_INFO_PROPERTY = "/test-suite-info.properties"; + /** suite info keys */ + private static final String BUILD_NUMBER = "build_number"; + private static final String TARGET_ARCH = "target_arch"; + private static final String NAME = "name"; + private static final String FULLNAME = "fullname"; + private static final String VERSION = "version"; + + private static TestSuiteInfo sInstance; + private Properties mTestSuiteInfo; + + private TestSuiteInfo() { + try (InputStream is = TestSuiteInfo.class.getResourceAsStream(SUITE_INFO_PROPERTY)) { + if (is != null) { + mTestSuiteInfo = loadSuiteInfo(is); + } else { + CLog.w("Unable to load suite info from jar resource %s, using stub info instead", + SUITE_INFO_PROPERTY); + mTestSuiteInfo = new Properties(); + mTestSuiteInfo.setProperty(BUILD_NUMBER, "[stub build number]"); + mTestSuiteInfo.setProperty(TARGET_ARCH, "[stub target arch]"); + mTestSuiteInfo.setProperty(NAME, "[stub name]"); + mTestSuiteInfo.setProperty(FULLNAME, "[stub fullname]"); + mTestSuiteInfo.setProperty(VERSION, "[stub version]"); + } + } catch (IOException ioe) { + // rethrow as runtime exception + throw new RuntimeException(String.format( + "error loading jar resource file \"%s\" for test suite info", + SUITE_INFO_PROPERTY)); + } + } + + /** Performs the actual loading of properties */ + protected Properties loadSuiteInfo(InputStream is) throws IOException { + Properties p = new Properties(); + p.load(is); + return p; + } + + /** + * Retrieves the singleton instance, which also triggers loading of the related test suite info + * from embedded resource files + * @return + */ + public static TestSuiteInfo getInstance() { + if (sInstance == null) { + sInstance = new TestSuiteInfo(); + } + return sInstance; + } + + /** Gets the build number of the test suite */ + public String getBuildNumber() { + return mTestSuiteInfo.getProperty(BUILD_NUMBER); + } + + /** Gets the target archs supported by the test suite */ + public String getTargetArch() { + return mTestSuiteInfo.getProperty(TARGET_ARCH); + } + + /** Gets the short name of the test suite */ + public String getName() { + return mTestSuiteInfo.getProperty(NAME); + } + + /** Gets the full name of the test suite */ + public String getFullName() { + return mTestSuiteInfo.getProperty(FULLNAME); + } + + /** Gets the version name of the test suite */ + public String getVersion() { + return mTestSuiteInfo.getProperty(VERSION); + } + + /** + * Retrieves test information keyed with the provided name + * @param name + * @return + */ + public String get(String name) { + return mTestSuiteInfo.getProperty(name); + } +} |