Enable CTS test

Change-Id: Ib5f96f77981a34120a6064c9b74b936c8eb17c1a

4 files changed, 145 insertions, 125 deletions

diff --git a/emu_test/dotest.py b/emu_test/dotest.py
index 1c3d7153..f691d1e7 100755
--- a/emu_test/dotest.py
+++ b/emu_test/dotest.py
@@ -27,6 +27,8 @@ TIMEOUT_REGEX = re.compile(r"(^\d+)([smhd])?$")
 
 main_logger = logging.getLogger()
 def printResult(emuResult):
+    def getTestName(id):
+        return id.rsplit('.', 1)[-1]
     print
     main_logger.info("Test Summary")
     main_logger.info("Run %d tests (%d fail, %d pass, %d xfail, %d xpass)",
@@ -35,26 +37,26 @@ def printResult(emuResult):
     if len(emuResult.errors) > 0 or len(emuResult.failures) > 0:
         for x in emuResult.errors:
             if x[1].splitlines()[-1] == "TimeoutError":
-                main_logger.info("TIMEOUT: %s", x[0].id())
+                main_logger.info("TIMEOUT: %s", getTestName(x[0].id()))
             else:
-                main_logger.info("FAIL: %s", x[0].id())
+                main_logger.info("FAIL: %s", getTestName(x[0].id()))
         for x in emuResult.failures:
-            main_logger.info("FAIL: %s", x[0].id())
+            main_logger.info("FAIL: %s", getTestName(x[0].id()))
 
     if len(emuResult.passes) > 0:
         main_logger.info('------------------------------------------------------')
     for x in emuResult.passes:
-        main_logger.info("PASS: %s", x.id())
+        main_logger.info("PASS: %s, boot time: %s", getTestName(x.id()), x.boot_time)
 
     if len(emuResult.expectedFailures) > 0:
         main_logger.info('------------------------------------------------------')
     for x in emuResult.expectedFailures:
-        main_logger.info("Expected Failure: %s", x[0].id())
+        main_logger.info("Expected Failure: %s", getTestName(x[0].id()))
 
     if len(emuResult.unexpectedSuccesses) > 0:
         main_logger.info('------------------------------------------------------')
     for x in emuResult.unexpectedSuccesses:
-        main_logger.info("Unexpected Success: %s", x.id())
+        main_logger.info("Unexpected Success: %s", getTestName(x.id()))
 
     main_logger.info('')
     main_logger.info("Test successful - %s", emuResult.wasSuccessful())
diff --git a/emu_test/test_boot/test_boot.py b/emu_test/test_boot/test_boot.py
index d0ca2b22..f4449dc3 100644
--- a/emu_test/test_boot/test_boot.py
+++ b/emu_test/test_boot/test_boot.py
@@ -2,21 +2,19 @@
 
 import unittest
 import os
-import platform
 import time
 import psutil
-import csv
 import shutil
 
 from utils.emu_error import *
 from utils.emu_argparser import emu_args
+import utils.emu_testcase
 from utils.emu_testcase import EmuBaseTestCase, AVDConfig
 
 class BootTestCase(EmuBaseTestCase):
     def __init__(self, *args, **kwargs):
         super(BootTestCase, self).__init__(*args, **kwargs)
         self.avd_config = None
-
     @classmethod
     def setUpClass(cls):
         super(BootTestCase, cls).setUpClass()
@@ -29,8 +27,9 @@ class BootTestCase(EmuBaseTestCase):
                     proc = psutil.Process(x.pid)
                     # mips 64 use qemu-system-mipsel64, others emulator-[arch]
                     if any([x in proc.name() for x in proc_names]):
-                        self.m_logger.info("kill_proc_by_name - %s" % proc.name())
-                        proc.kill()
+                        self.m_logger.info("kill_proc_by_name - %s, %s" % (proc.name(), proc.status()))
+                        if proc.status() != psutil.STATUS_ZOMBIE:
+                            proc.kill()
                 except psutil.NoSuchProcess:
                     pass
 
@@ -54,86 +53,15 @@ class BootTestCase(EmuBaseTestCase):
             pass
 
     def boot_check(self, avd):
-        boot_time = self.launch_emu_and_wait(avd)
-        self.m_logger.info('AVD %s, boot time: %s, expected time: %s', avd, boot_time, emu_args.expected_boot_time)
-        self.assertLessEqual(boot_time, emu_args.expected_boot_time)
+        self.boot_time = self.launch_emu_and_wait(avd)
+        self.m_logger.info('AVD %s, boot time: %s, expected time: %s', avd, self.boot_time, emu_args.expected_boot_time)
+        self.assertLessEqual(self.boot_time, emu_args.expected_boot_time)
 
     def run_boot_test(self, avd_config):
         self.avd_config = avd_config
         self.assertEqual(self.create_avd(avd_config), 0)
         self.boot_check(avd_config)
 
-def get_port():
-    if not hasattr(get_port, '_port'):
-        get_port._port = 5552
-    get_port._port += 2
-    return str(get_port._port)
-
-def create_test_case_from_file():
-    """ Create test case based on test configuration file. """
-
-    def valid_case(avd_config):
-        if emu_args.filter_dict is not None:
-            for key, value in emu_args.filter_dict.iteritems():
-                if getattr(avd_config, key) != value:
-                    return False
-        return True
-
-    def create_test_case(avd_config, op):
-        if op == "S" or op == "" or not valid_case(avd_config):
-            return
-
-        func = lambda self: self.run_boot_test(avd_config)
-        if op == "X":
-            func = unittest.expectedFailure(func)
-        # TODO: handle flakey tests
-        elif op == "F":
-            func = func
-        qemu_str = "_qemu2" if avd_config.ranchu == "yes" else ""
-        setattr(BootTestCase, "test_boot_%s%s" % (str(avd_config), qemu_str), func)
-
-    with open(emu_args.config_file, "rb") as file:
-        reader = csv.reader(file)
-        for row in reader:
-            #skip the first line
-            if reader.line_num == 1:
-                continue
-            if reader.line_num == 2:
-                builder_idx = row.index(emu_args.builder_name)
-            else:
-                if(row[0].strip() != ""):
-                    api = row[0].split("API", 1)[1].strip()
-                if(row[1].strip() != ""):
-                    tag = row[1].strip()
-                if(row[2].strip() != ""):
-                    abi = row[2].strip()
-
-                # P - config should be passing
-                # X - config is expected to fail
-                # S and everything else - Skip this config
-                op = row[builder_idx].strip().upper()
-                if op in ["P", "X", "F"]:
-                    device = row[3]
-                    if row[4] != "":
-                        ram = row[4]
-                    else:
-                        ram = "512" if device == "" else "1536"
-                    if row[5] != "":
-                        gpu = row[5]
-                    else:
-                        gpu = "yes" if api > "15" else "no"
-                    # For 32 bit machine, ram should be less than 768MB
-                    if not platform.machine().endswith('64'):
-                        ram = str(min([int(ram), 768]))
-                    if api < "22" and row[6] == "yes":
-                        raise ConfigError()
-                    tot_image = row[6] if row[6] == "yes" else "no"
-                    avd_config = AVDConfig(api, tag, abi, device, ram, gpu, tot_image, ranchu="no", port=get_port())
-                    create_test_case(avd_config, op)
-                    # for unreleased images, test with qemu2 in addition
-                    if tot_image == "yes":
-                        avd_config = AVDConfig(api, tag, abi, device, ram, gpu, tot_image, ranchu="yes", port=get_port())
-                        create_test_case(avd_config, op)
 
 def create_test_case_for_avds():
     avd_list = emu_args.avd_list
@@ -145,7 +73,7 @@ def create_test_case_for_avds():
 if emu_args.config_file is None:
     create_test_case_for_avds()
 else:
-    create_test_case_from_file()
+    utils.emu_testcase.create_test_case_from_file("boot", BootTestCase, BootTestCase.run_boot_test)
 
 if __name__ == '__main__':
     os.environ["SHELL"] = "/bin/bash"
diff --git a/emu_test/test_cts/test_cts.py b/emu_test/test_cts/test_cts.py
index b1587ad1..ad3b286a 100644
--- a/emu_test/test_cts/test_cts.py
+++ b/emu_test/test_cts/test_cts.py
@@ -5,12 +5,13 @@ import unittest
 import time
 import psutil
 import re
-from subprocess import PIPE
+import threading
+from subprocess import PIPE,STDOUT
 
 from utils.emu_error import *
 from utils.emu_argparser import emu_args
-from utils.emu_testcase import EmuBaseTestCase
-
+from utils.emu_testcase import EmuBaseTestCase, AVDConfig
+import utils.emu_testcase
 api_to_android_version = {"23": "6.0",
                           "22": "5.1",
                           "21": "5.0",
@@ -48,46 +49,59 @@ class CTSTestCase(EmuBaseTestCase):
 
     def get_cts_exec(self, avd):
         home_dir = os.path.expanduser('~')
-        host = platform.system()
-        if host == "Darwin" or host == "Linux":
-            cts_home = os.path.join(home_dir, 'Android/CTS')
-        elif host == "Windows":
-            cts_home = 'C:\\CTS'
-        # expected avd name [arch]-[api]-[CTS]
-        name_pattern = re.compile("^(.*)-(.*)-CTS$")
-        res = re.match(name_pattern, avd)
-        assert res is not None
-        arch = res.group(1)
-        api = res.group(2)
-        cts_dir = "%s-%s" % (api_to_android_version[api], arch)
+        cts_home = os.path.join(home_dir, 'Android', 'CTS')
+        cts_dir = "%s-%s" % (api_to_android_version[avd.api], avd.abi)
         return os.path.join(cts_home, cts_dir, 'android-cts', 'tools', 'cts-tradefed')
 
     def run_cts_plan(self, avd, plan):
         result_re = re.compile("^.*XML test result file generated at .*Passed ([0-9]+), Failed ([0-9]+), Not Executed ([0-9]+)")
         result_line = ""
+        #self.assertEqual(self.create_avd(avd), 0)
         self.launch_emu_and_wait(avd)
+
         exec_path = self.get_cts_exec(avd)
-        cts_proc = psutil.Popen([exec_path, "run", "cts", "--plan", plan, "--disable-reboot"], stdout=PIPE, stderr=PIPE)
-        self.m_logger.debug("CTS process poll %s", cts_proc.poll())
-        while cts_proc.poll() is None:
-            line = cts_proc.stdout.readline()
-            self.simple_logger.info(line)
-            if re.match(result_re, line):
-                result_line = line
-        if result_line is not "":
-            pass_count = re.match(result_re, result_line).group(1)
-            fail_count = re.match(result_re, result_line).group(2)
-            self.assertNotEqual(pass_count, '0')
+        cst_cmd = [exec_path, "run", "cts", "--plan", plan, "--disable-reboot"]
+        def launch_in_thread():
+            self.m_logger.info('executable path: ' + exec_path)
+            cts_proc = psutil.Popen(cst_cmd, stdout=PIPE, stderr=STDOUT)
+            lines_iterator = iter(cts_proc.stdout.readline, b"")
+            for line in lines_iterator:
+                self.simple_logger.info(line)
+                if re.match(result_re, line):
+                    result_line = line
+
+        self.m_logger.info('Launching cts-tradefed, cmd: %s', ' '.join(cst_cmd))
+        t_launch = threading.Thread(target=launch_in_thread)
+        t_launch.start()
+        t_launch.join()
+
+        if result_line != "":
+            pass_count, fail_count, skip_count = re.match(result_re, result_line).groups()
+            self.m_logger.info("Pass: %s, Fail: %s, Not Executed: %s", pass_count, fail_count, skip_count)
             self.assertEqual(fail_count, '0')
-        self.m_logger.debug("CTS process poll %s", cts_proc.poll())
+        else
+            self.assertEqual('NA', '0')
 
 def create_test_case_for_avds():
-    avd_list = emu_args.avd_list
-    for avd in avd_list:
-        def fn(i, plan):
-            return lambda self: self.run_cts_plan(i, plan)
-        if "CTS" in avd:
-            setattr(CTSTestCase, "test_cts_Short_%s" % avd, fn(avd, "Short"))
+    avd_name_re = re.compile("([^-]*)-(.*)-(.*)-(\d+)-gpu_(.*)-api(\d+)-CTS")
+    def create_avd_from_name(avd_str):
+        res = avd_name_re.match(avd_str)
+        assert res is not None
+        tag, abi, device, ram, gpu, api = avd_name_re.match(avd_str).groups()
+        avd_config = AVDConfig(api, tag, abi, device, ram, gpu, "yes", ranchu="no", port="", cts=True)
+        return avd_config
+
+    def fn(avd_name, plan):
+        #return lambda self: self.run_cts_plan(create_avd_from_name(avd_name), plan)
+        return lambda self: self.run_cts_plan(create_avd_from_name(avd_name), plan)
+
+    for avd in emu_args.avd_list:
+        if avd_name_re.match(avd):
+            setattr(CTSTestCase, "test_cts_%s" % avd, fn(avd, "CTS"))
+
+# TODO: create test case based on config file. Since we need to do some pre-work to run CTS, use static AVD at this time for simplicity.
+create_test_case_for_avds()
+#utils.emu_testcase.create_test_case_from_file("cts", CTSTestCase, CTSTestCase.run_cts_plan)
 
 create_test_case_for_avds()
 if __name__ == '__main__':
diff --git a/emu_test/utils/emu_testcase.py b/emu_test/utils/emu_testcase.py
index 631a5edc..1fe8722c 100644
--- a/emu_test/utils/emu_testcase.py
+++ b/emu_test/utils/emu_testcase.py
@@ -9,6 +9,8 @@ import unittest
 import logging
 import time
 import psutil
+import csv
+import platform
 import threading
 from emu_error import *
 from emu_argparser import emu_args
@@ -16,15 +18,15 @@ from subprocess import PIPE, STDOUT
 from collections import namedtuple
 from ConfigParser import ConfigParser
 
-class AVDConfig(namedtuple('AVDConfig', 'api, tag, abi, device, ram, gpu, tot_image, ranchu, port')):
+class AVDConfig(namedtuple('AVDConfig', 'api, tag, abi, device, ram, gpu, tot_image, ranchu, port, cts')):
     __slots__ = ()
     def __str__(self):
         device = self.device if self.device != '' else 'defdev'
         for ch in [' ', '(', ')']:
             device = device.replace(ch, '_')
-        return str("%s-%s-%s-%s-gpu_%s-api%s" % (self.tag, self.abi,
+        return str("%s-%s-%s-%s-gpu_%s-api%s%s" % (self.tag, self.abi,
                                                  device, self.ram, self.gpu,
-                                                 self.api))
+                                                 self.api, "-CTS" if self.cts else ""))
     def name(self):
         return str(self)
 class LoggedTestCase(unittest.TestCase):
@@ -84,6 +86,7 @@ class EmuBaseTestCase(LoggedTestCase):
     """
     def __init__(self, *args, **kwargs):
         super(EmuBaseTestCase, self).__init__(*args, **kwargs)
+        self.boot_time = 0
 
     @classmethod
     def setUpClass(cls):
@@ -183,11 +186,12 @@ class EmuBaseTestCase(LoggedTestCase):
         if completed is not "1":
             self.m_logger.info('command output - %s %s', output, err)
             self.m_logger.error('AVD %s didn\'t boot up within %s seconds', avd, emu_args.timeout_in_seconds)
+            self.boot_time = -1
             raise TimeoutError(avd, emu_args.timeout_in_seconds)
-        boot_time = time.time() - start_time
-        self.m_logger.info('AVD %s, boot time is %s', avd, boot_time)
+        self.boot_time = time.time() - start_time
+        self.m_logger.info('AVD %s, boot time is %s', avd, self.boot_time)
 
-        return boot_time
+        return self.boot_time
 
     def update_config(self, avd_config):
         # avd should be found $HOME/.android/avd/
@@ -317,3 +321,75 @@ class EmuBaseTestCase(LoggedTestCase):
         self.simple_logger.debug(err)
         self.m_logger.debug('return value of update proc: %s', update_proc.poll())
         return update_proc.poll()
+
+def create_test_case_from_file(desc, testcase_class, test_func):
+    """ Create test case based on test configuration file. """
+
+    def get_port():
+        if not hasattr(get_port, '_port'):
+            get_port._port = 5552
+        get_port._port += 2
+        return str(get_port._port)
+
+    def valid_case(avd_config):
+        if emu_args.filter_dict is not None:
+            for key, value in emu_args.filter_dict.iteritems():
+                if getattr(avd_config, key) != value:
+                    return False
+        return True
+
+    def create_test_case(avd_config, op):
+        if op == "S" or op == "" or not valid_case(avd_config):
+            return
+
+        func = lambda self: test_func(self, avd_config)
+        if op == "X":
+            func = unittest.expectedFailure(func)
+        # TODO: handle flakey tests
+        elif op == "F":
+            func = func
+        qemu_str = "_qemu2" if avd_config.ranchu == "yes" else ""
+        setattr(testcase_class, "test_%s_%s%s" % (desc, str(avd_config), qemu_str), func)
+
+    with open(emu_args.config_file, "rb") as file:
+        reader = csv.reader(file)
+        for row in reader:
+            #skip the first line
+            if reader.line_num == 1:
+                continue
+            if reader.line_num == 2:
+                builder_idx = row.index(emu_args.builder_name)
+            else:
+                if(row[0].strip() != ""):
+                    api = row[0].split("API", 1)[1].strip()
+                if(row[1].strip() != ""):
+                    tag = row[1].strip()
+                if(row[2].strip() != ""):
+                    abi = row[2].strip()
+
+                # P - config should be passing
+                # X - config is expected to fail
+                # S and everything else - Skip this config
+                op = row[builder_idx].strip().upper()
+                if op in ["P", "X", "F"]:
+                    device = row[3]
+                    if row[4] != "":
+                        ram = row[4]
+                    else:
+                        ram = "512" if device == "" else "1536"
+                    if row[5] != "":
+                        gpu = row[5]
+                    else:
+                        gpu = "yes" if api > "15" else "no"
+                    # For 32 bit machine, ram should be less than 768MB
+                    if not platform.machine().endswith('64'):
+                        ram = str(min([int(ram), 768]))
+                    if api < "22" and row[6] == "yes":
+                        raise ConfigError()
+                    tot_image = row[6] if row[6] == "yes" else "no"
+                    avd_config = AVDConfig(api, tag, abi, device, ram, gpu, tot_image, ranchu="no", port=get_port(), cts=False)
+                    create_test_case(avd_config, op)
+                    # for unreleased images, test with qemu2 in addition
+                    if tot_image == "yes":
+                        avd_config = AVDConfig(api, tag, abi, device, ram, gpu, tot_image, ranchu="yes", port=get_port(), cts=False)
+                        create_test_case(avd_config, op)