path: root/src/com/android/tradefed/testtype/AndroidJUnitTest.java

/*
 * Copyright (C) 2015 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.android.tradefed.testtype;

import com.android.ddmlib.testrunner.IRemoteAndroidTestRunner;
import com.android.tradefed.config.ConfigurationException;
import com.android.tradefed.config.Option;
import com.android.tradefed.config.OptionCopier;
import com.android.tradefed.device.DeviceNotAvailableException;
import com.android.tradefed.device.ITestDevice;
import com.android.tradefed.log.LogUtil.CLog;
import com.android.tradefed.metrics.proto.MetricMeasurement.Metric;
import com.android.tradefed.result.ITestInvocationListener;
import com.android.tradefed.util.ArrayUtil;
import com.android.tradefed.util.ListInstrumentationParser;

import com.google.common.annotations.VisibleForTesting;

import org.junit.runner.notification.RunListener;

import java.io.File;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Set;

/**
 * A Test that runs an instrumentation test package on given device using the
 * android.support.test.runner.AndroidJUnitRunner.
 */
public class AndroidJUnitTest extends InstrumentationTest implements IRuntimeHintProvider,
        ITestFileFilterReceiver, ITestFilterReceiver, ITestAnnotationFilterReceiver,
        IShardableTest, IStrictShardableTest {

    /** instrumentation test runner argument key used for including a class/test */
    private static final String INCLUDE_CLASS_INST_ARGS_KEY = "class";
    /** instrumentation test runner argument key used for excluding a class/test */
    private static final String EXCLUDE_CLASS_INST_ARGS_KEY = "notClass";
    /** instrumentation test runner argument key used for including a package */
    private static final String INCLUDE_PACKAGE_INST_ARGS_KEY = "package";
    /** instrumentation test runner argument key used for excluding a package */
    private static final String EXCLUDE_PACKAGE_INST_ARGS_KEY = "notPackage";
    /** instrumentation test runner argument key used for adding annotation filter */
    private static final String ANNOTATION_INST_ARGS_KEY = "annotation";
    /** instrumentation test runner argument key used for adding notAnnotation filter */
    private static final String NOT_ANNOTATION_INST_ARGS_KEY = "notAnnotation";
    /** instrumentation test runner argument used for adding testFile filter */
    private static final String TEST_FILE_INST_ARGS_KEY = "testFile";
    /** instrumentation test runner argument used for adding notTestFile filter */
    private static final String NOT_TEST_FILE_INST_ARGS_KEY = "notTestFile";
    /** instrumentation test runner argument used to specify the shardIndex of the test */
    private static final String SHARD_INDEX_INST_ARGS_KEY = "shardIndex";
    /** instrumentation test runner argument used to specify the total number of shards */
    private static final String NUM_SHARD_INST_ARGS_KEY = "numShards";
    /**
     * instrumentation test runner argument used to enable the new {@link RunListener} order on
     * device side.
     */
    public static final String NEW_RUN_LISTENER_ORDER_KEY = "newRunListenerMode";

    private static final String INCLUDE_FILE = "includes.txt";
    private static final String EXCLUDE_FILE = "excludes.txt";

    @Option(name = "runtime-hint",
            isTimeVal=true,
            description="The hint about the test's runtime.")
    private long mRuntimeHint = 60000;// 1 minute

    @Option(name = "include-filter",
            description="The include filters of the test name to run.")
    private List<String> mIncludeFilters = new ArrayList<>();

    @Option(name = "exclude-filter",
            description="The exclude filters of the test name to run.")
    private List<String> mExcludeFilters = new ArrayList<>();

    @Option(name = "include-annotation",
            description="The annotation class name of the test name to run, can be repeated")
    private List<String> mIncludeAnnotation = new ArrayList<>();

    @Option(name = "exclude-annotation",
            description="The notAnnotation class name of the test name to run, can be repeated")
    private List<String> mExcludeAnnotation = new ArrayList<>();

    @Option(name = "test-file-include-filter",
            description="A file containing a list of line separated test classes and optionally"
            + " methods to include")
    private File mIncludeTestFile = null;

    @Option(name = "test-file-exclude-filter",
            description="A file containing a list of line separated test classes and optionally"
            + " methods to exclude")
    private File mExcludeTestFile = null;

    @Option(name = "test-filter-dir",
            description="The device directory path to which the test filtering files are pushed")
    private String mTestFilterDir = "/data/local/tmp/ajur";

    @Option(
        name = "ajur-max-shard",
        description = "The maximum number of shard we want to allow the AJUR test to shard into"
    )
    private Integer mMaxShard = null;

    @Option(
        name = "device-listeners",
        description =
                "Specify a device side instrumentation listener to be added for the run. "
                        + "Can be repeated."
    )
    private List<String> mExtraDeviceListener = new ArrayList<>();

    @Option(
        name = "use-new-run-listener-order",
        description = "Enables the new RunListener Order for AJUR."
    )
    // Default to true as it is harmless if not supported.
    private boolean mNewRunListenerOrderMode = true;

    private String mDeviceIncludeFile = null;
    private String mDeviceExcludeFile = null;
    private int mTotalShards = 0;
    private int mShardIndex = 0;
    // Flag to avoid re-sharding a test that already was.
    private boolean mIsSharded = false;

    public AndroidJUnitTest() {
        super();
        setEnforceFormat(true);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public long getRuntimeHint() {
        return mRuntimeHint;
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void addIncludeFilter(String filter) {
        mIncludeFilters.add(filter);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void addAllIncludeFilters(Set<String> filters) {
        mIncludeFilters.addAll(filters);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void addExcludeFilter(String filter) {
        mExcludeFilters.add(filter);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void addAllExcludeFilters(Set<String> filters) {
        mExcludeFilters.addAll(filters);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void setIncludeTestFile(File testFile) {
        mIncludeTestFile = testFile;
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void setExcludeTestFile(File testFile) {
        mExcludeTestFile = testFile;
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void addIncludeAnnotation(String annotation) {
        mIncludeAnnotation.add(annotation);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void addAllIncludeAnnotation(Set<String> annotations) {
        mIncludeAnnotation.addAll(annotations);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void addExcludeAnnotation(String excludeAnnotation) {
        mExcludeAnnotation.add(excludeAnnotation);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void addAllExcludeAnnotation(Set<String> excludeAnnotations) {
        mExcludeAnnotation.addAll(excludeAnnotations);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void run(final ITestInvocationListener listener) throws DeviceNotAvailableException {
        if (getDevice() == null) {
            throw new IllegalArgumentException("Device has not been set");
        }
        boolean pushedFile = false;
        // if mIncludeTestFile is set, perform filtering with this file
        if (mIncludeTestFile != null && mIncludeTestFile.length() > 0) {
            mDeviceIncludeFile = mTestFilterDir.replaceAll("/$", "") + "/" + INCLUDE_FILE;
            pushTestFile(mIncludeTestFile, mDeviceIncludeFile, listener);
            pushedFile = true;
            // If an explicit include file filter is provided, do not use the package
            setTestPackageName(null);
        }

        // if mExcludeTestFile is set, perform filtering with this file
        if (mExcludeTestFile != null) {
            mDeviceExcludeFile = mTestFilterDir.replaceAll("/$", "") + "/" + EXCLUDE_FILE;
            pushTestFile(mExcludeTestFile, mDeviceExcludeFile, listener);
            pushedFile = true;
        }
        if (mTotalShards > 0 && !isShardable() && mShardIndex != 0) {
            // If not shardable, only first shard can run.
            CLog.i("%s is not shardable.", getRunnerName());
            return;
        }
        super.run(listener);
        if (pushedFile) {
            // Remove the directory where the files where pushed
            removeTestFilterDir();
        }
    }

    /**
     * {@inheritDoc}
     */
    @Override
    protected void setRunnerArgs(IRemoteAndroidTestRunner runner) {
        super.setRunnerArgs(runner);

        // if mIncludeTestFile is set, perform filtering with this file
        if (mDeviceIncludeFile != null) {
            runner.addInstrumentationArg(TEST_FILE_INST_ARGS_KEY, mDeviceIncludeFile);
        }

        // if mExcludeTestFile is set, perform filtering with this file
        if (mDeviceExcludeFile != null) {
            runner.addInstrumentationArg(NOT_TEST_FILE_INST_ARGS_KEY, mDeviceExcludeFile);
        }

        // Split filters into class, notClass, package and notPackage
        List<String> classArg = new ArrayList<String>();
        List<String> notClassArg = new ArrayList<String>();
        List<String> packageArg = new ArrayList<String>();
        List<String> notPackageArg = new ArrayList<String>();
        for (String test : mIncludeFilters) {
            if (isClassOrMethod(test)) {
                classArg.add(test);
            } else {
                packageArg.add(test);
            }
        }
        for (String test : mExcludeFilters) {
            if (isClassOrMethod(test)) {
                notClassArg.add(test);
            } else {
                notPackageArg.add(test);
            }
        }
        if (!classArg.isEmpty()) {
            runner.addInstrumentationArg(INCLUDE_CLASS_INST_ARGS_KEY,
                    ArrayUtil.join(",", classArg));
        }
        if (!notClassArg.isEmpty()) {
            runner.addInstrumentationArg(EXCLUDE_CLASS_INST_ARGS_KEY,
                    ArrayUtil.join(",", notClassArg));
        }
        if (!packageArg.isEmpty()) {
            runner.addInstrumentationArg(INCLUDE_PACKAGE_INST_ARGS_KEY,
                    ArrayUtil.join(",", packageArg));
        }
        if (!notPackageArg.isEmpty()) {
            runner.addInstrumentationArg(EXCLUDE_PACKAGE_INST_ARGS_KEY,
                    ArrayUtil.join(",", notPackageArg));
        }
        if (!mIncludeAnnotation.isEmpty()) {
            runner.addInstrumentationArg(ANNOTATION_INST_ARGS_KEY,
                    ArrayUtil.join(",", mIncludeAnnotation));
        }
        if (!mExcludeAnnotation.isEmpty()) {
            runner.addInstrumentationArg(NOT_ANNOTATION_INST_ARGS_KEY,
                    ArrayUtil.join(",", mExcludeAnnotation));
        }
        if (mTotalShards > 0 && isShardable()) {
            runner.addInstrumentationArg(SHARD_INDEX_INST_ARGS_KEY, Integer.toString(mShardIndex));
            runner.addInstrumentationArg(NUM_SHARD_INST_ARGS_KEY, Integer.toString(mTotalShards));
        }
        if (mNewRunListenerOrderMode) {
            runner.addInstrumentationArg(
                    NEW_RUN_LISTENER_ORDER_KEY, Boolean.toString(mNewRunListenerOrderMode));
        }
        // Add the listeners received from Options
        addDeviceListener(mExtraDeviceListener);
    }

    /**
     * Push the testFile to the requested destination. This should only be called for a non-null
     * testFile
     *
     * @param testFile file to be pushed from the host to the device.
     * @param destination the path on the device to which testFile is pushed
     * @param listener {@link ITestInvocationListener} to report failures.
     */
    private void pushTestFile(File testFile, String destination, ITestInvocationListener listener)
            throws DeviceNotAvailableException {
        if (!testFile.canRead() || !testFile.isFile()) {
            String message = String.format("Cannot read test file %s", testFile.getAbsolutePath());
            reportEarlyFailure(listener, message);
            throw new IllegalArgumentException(message);
        }
        ITestDevice device = getDevice();
        try {
            CLog.d("Attempting to push filters to %s", destination);
            if (!device.pushFile(testFile, destination)) {
                String message =
                        String.format(
                                "Failed to push file %s to %s for %s in pushTestFile",
                                testFile.getAbsolutePath(), destination, device.getSerialNumber());
                reportEarlyFailure(listener, message);
                throw new RuntimeException(message);
            }
            // in case the folder was created as 'root' we make is usable.
            device.executeShellCommand(String.format("chown -R shell:shell %s", mTestFilterDir));
        } catch (DeviceNotAvailableException e) {
            reportEarlyFailure(listener, e.getMessage());
            throw e;
        }
    }

    private void removeTestFilterDir() throws DeviceNotAvailableException {
        getDevice().deleteFile(mTestFilterDir);
    }

    private void reportEarlyFailure(ITestInvocationListener listener, String errorMessage) {
        listener.testRunStarted("AndroidJUnitTest_setupError", 0);
        listener.testRunFailed(errorMessage);
        listener.testRunEnded(0, new HashMap<String, Metric>());
    }

    /**
     * Return if a string is the name of a Class or a Method.
     */
    @VisibleForTesting
    public boolean isClassOrMethod(String filter) {
        if (filter.contains("#")) {
            return true;
        }
        String[] parts = filter.split("\\.");
        if (parts.length > 0) {
            // FIXME Assume java package names starts with lowercase and class names start with
            // uppercase.
            // Return true iff the first character of the last word is uppercase
            // com.android.foobar.Test
            return Character.isUpperCase(parts[parts.length - 1].charAt(0));
        }
        return false;
    }

    /**
     * Helper to return if the runner is one that support sharding.
     */
    private boolean isShardable() {
        // Edge toward shardable if no explicit runner specified. The runner will be determined
        // later and if not shardable only the first shard will run.
        if (getRunnerName() == null) {
            return true;
        }
        return ListInstrumentationParser.SHARDABLE_RUNNERS.contains(getRunnerName());
    }

    /** {@inheritDoc} */
    @Override
    public Collection<IRemoteTest> split(int shardCount) {
        if (!isShardable()) {
            return null;
        }
        if (mMaxShard != null) {
            shardCount = Math.min(shardCount, mMaxShard);
        }
        if (!mIsSharded && shardCount > 1) {
            mIsSharded = true;
            Collection<IRemoteTest> shards = new ArrayList<>(shardCount);
            for (int index = 0; index < shardCount; index++) {
                shards.add(getTestShard(shardCount, index));
            }
            return shards;
        }
        return null;
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public IRemoteTest getTestShard(int shardCount, int shardIndex) {
        AndroidJUnitTest shard;
        // ensure we handle runners that extend AndroidJUnitRunner
        try {
            shard = this.getClass().newInstance();
        } catch (InstantiationException | IllegalAccessException e) {
            throw new RuntimeException(e);
        }
        try {
            OptionCopier.copyOptions(this, shard);
        } catch (ConfigurationException e) {
            CLog.e("Failed to copy instrumentation options: %s", e.getMessage());
        }
        shard.mShardIndex = shardIndex;
        shard.mTotalShards = shardCount;
        shard.mIsSharded = true;
        shard.setAbi(getAbi());
        // We approximate the runtime of each shard to be equal since we can't know.
        shard.mRuntimeHint = mRuntimeHint / shardCount;
        return shard;
    }
}