[tast] report skipped tast tests in tast.generic-list wrapper

skipped test results were not being reported when run with tast wrapper,
update so that results along with reason are reported and test that is
skipped by tast will have TEST_NA with the reason reported by tast.
This change adds a flag so that only the tast.generic-list wrapper which
is used by PVS will report skipped results.  This fixes the bug for PVS
without breaking tests and stainless

BUG=b:231928841
TEST=unit test passed

Change-Id: I4c82d0724c503136d69ddfd1648bdf5913681042
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/autotest/+/3665436
Tested-by: Brett Brotherton <bbrotherton@google.com>
Auto-Submit: Brett Brotherton <bbrotherton@google.com>
Reviewed-by: Seewai Fu <seewaifu@google.com>
Commit-Queue: Brett Brotherton <bbrotherton@google.com>

3 files changed, 52 insertions, 10 deletions

diff --git a/server/site_tests/tast/control.generic-list b/server/site_tests/tast/control.generic-list
index 170c718dd8..3ea0a2e699 100644
--- a/server/site_tests/tast/control.generic-list
+++ b/server/site_tests/tast/control.generic-list
@@ -28,7 +28,8 @@ more information.
 
 This test runs arbitary Tast-based tests specified by args given to test_that.
 This test might be useful on debugging to simulate Tast test runs invoked via
-Autotest.
+Autotest.  Tests run with this wrapper will report skipped tests using the 
+autotest TEST NA status.
 
 Examples:
     test_that --args=tast_list=tast.test_1,tast.test_2 ${DUT} tast.generic_list
@@ -50,6 +51,7 @@ def run(machine):
                  ignore_test_failures=True, max_run_sec=3600,
                  command_args=command_args,
                  varslist=varslist,
-                 exclude_missing=True)
+                 exclude_missing=True,
+                 report_skipped=True)
 
 parallel_simple(run, machines)
diff --git a/server/site_tests/tast/tast.py b/server/site_tests/tast/tast.py
index 03ab080f07..7ed7678417 100644
--- a/server/site_tests/tast/tast.py
+++ b/server/site_tests/tast/tast.py
@@ -171,12 +171,14 @@ class tast(test.test):
     _JOB_STATUS_END_GOOD = 'END GOOD'
     _JOB_STATUS_END_FAIL = 'END FAIL'
     _JOB_STATUS_END_NOSTATUS = 'END NOSTATUS'
+    _JOB_STATUS_END_SKIP = 'END TEST_NA'
 
     # In-job TKO event status codes from base_client_job._run_test_base in
     # client/bin/job.py and client/common_lib/error.py.
     _JOB_STATUS_GOOD = 'GOOD'
     _JOB_STATUS_FAIL = 'FAIL'
     _JOB_STATUS_NOSTATUS = 'NOSTATUS'
+    _JOB_STATUS_SKIP = 'TEST_NA'
 
     # Status reason used when an individual Tast test doesn't finish running.
     _TEST_DID_NOT_FINISH_MSG = 'Test did not finish'
@@ -208,7 +210,8 @@ class tast(test.test):
                    ephemeraldevserver=None,
                    is_cft=False,
                    exclude_missing=False,
-                   test_filter_files=[]):
+                   test_filter_files=[],
+                   report_skipped=False):
         """
         @param host: remote.RemoteHost instance representing DUT.
         @param test_exprs: Array of strings describing tests to run.
@@ -257,6 +260,8 @@ class tast(test.test):
         `tast list` command
         @param test_filter_files: This option includes a list of files containing names
         of test to be disabled.
+        @param report_skipped: If true then skipped tests will be reported in
+        the status.log
 
         When the F20 breadcrumb is detected, it is assumed we are running in
             the F20 container, meaning we will force disable SSP (though the
@@ -301,6 +306,7 @@ class tast(test.test):
         self._ephemeraldevserver = ephemeraldevserver
         self._exclude_missing = exclude_missing
         self._test_filter_files = test_filter_files
+        self._report_skipped = report_skipped
 
         # Need to pass in dut_servers for every test in CFT.
         # But only add it if not already in varslist.
@@ -1108,7 +1114,7 @@ class tast(test.test):
         test_finished = end_time > 0
 
         # Avoid reporting tests that were skipped.
-        if test_skipped and not test_reported_errors:
+        if test_skipped and not test_reported_errors and not self._report_skipped:
             return
 
         # Look for magic error _TEST_DID_NOT_RUN_MSG and mark test as not run.
@@ -1123,6 +1129,10 @@ class tast(test.test):
             self._log_test_event(self._JOB_STATUS_NOSTATUS, name, end_time,
                                  test[_KEY_MISSING_REASON])
             end_status = self._JOB_STATUS_END_NOSTATUS
+        elif test_skipped and not test_reported_errors and self._report_skipped:
+            self._log_test_event(self._JOB_STATUS_SKIP, name, end_time,
+                                 test.get(_KEY_SKIP_REASON))
+            end_status = self._JOB_STATUS_END_SKIP
         elif test_finished and not test_reported_errors:
             self._log_test_event(self._JOB_STATUS_GOOD, name, end_time)
             end_status = self._JOB_STATUS_END_GOOD
diff --git a/server/site_tests/tast/tast_unittest.py b/server/site_tests/tast/tast_unittest.py
index d7f3ec244d..b13afe0580 100755
--- a/server/site_tests/tast/tast_unittest.py
+++ b/server/site_tests/tast/tast_unittest.py
@@ -244,7 +244,8 @@ class TastTest(unittest.TestCase):
                   maybemissingvars='',
                   use_camera_box=False,
                   vars_gs_path='',
-                  test_filter_files=[]):
+                  test_filter_files=[],
+                  report_skipped=False):
         """Writes fake_tast.py's configuration and runs the test.
 
         @param ignore_test_failures: Passed as the identically-named arg to
@@ -267,6 +268,7 @@ class TastTest(unittest.TestCase):
         @param maybemissingvars: a regex to pass to tast run command as
             |-maybemissingvars| arguments.
         @param use_camera_box: Whether the test run in CameraBox.
+        @param report_skipped: Whether or not skipped tests should be reported.
         """
         self._test.initialize(self._host,
                               self.TEST_PATTERNS,
@@ -288,7 +290,8 @@ class TastTest(unittest.TestCase):
                               maybemissingvars=maybemissingvars,
                               use_camera_box=use_camera_box,
                               vars_gs_path=vars_gs_path,
-                              test_filter_files=test_filter_files)
+                              test_filter_files=test_filter_files,
+                              report_skipped=report_skipped)
         self._test.set_fake_now_for_testing(
                 (NOW - tast._UNIX_EPOCH).total_seconds())
 
@@ -374,14 +377,32 @@ class TastTest(unittest.TestCase):
 
     def testSkippedTest(self):
         """Tests that skipped tests aren't reported."""
-        tests = [TestInfo('pkg.Normal', 0, 1),
-                 TestInfo('pkg.Skipped', 2, 2, skip_reason='missing deps')]
+        tests = [
+                TestInfo('pkg.Normal', 0, 1),
+                TestInfo('pkg.Skipped', 2, 2, skip_reason='missing deps')
+        ]
         self._init_tast_commands(tests)
         self._run_test()
         self.assertEqual(status_string(get_status_entries_from_tests(tests)),
                          status_string(self._job.status_entries))
         self.assertIs(self._load_job_keyvals(), None)
 
+    def testSkippedTestWithReportSkipped(self):
+        """Tests that skipped tests are reported correctly when report_skipped=True."""
+        tests = [
+                TestInfo('pkg.Normal', 0, 1),
+                TestInfo('pkg.Skipped',
+                         2,
+                         3,
+                         skip_reason='missing deps',
+                         report_skipped=True)
+        ]
+        self._init_tast_commands(tests)
+        self._run_test(report_skipped=True)
+        self.assertEqual(status_string(get_status_entries_from_tests(tests)),
+                         status_string(self._job.status_entries))
+        self.assertIs(self._load_job_keyvals(), None)
+
     def testSkippedTestWithErrors(self):
         """Tests that skipped tests are reported if they also report errors."""
         tests = [TestInfo('pkg.Normal', 0, 1),
@@ -839,7 +860,8 @@ class TestInfo:
                  skip_reason=None,
                  attr=None,
                  timeout_ns=0,
-                 missing_reason=None):
+                 missing_reason=None,
+                 report_skipped=False):
         """
         @param name: Name of the test, e.g. 'ui.ChromeLogin'.
         @param start_offset: Start time as int seconds offset from BASE_TIME,
@@ -855,6 +877,7 @@ class TestInfo:
         @param attr: List of string test attributes assigned to the test, or
             None if no attributes are assigned.
         @param timeout_ns: Test timeout in nanoseconds.
+        @param report_skipped: Decide if skipped tests should be reported
         """
         def from_offset(offset):
             """Returns an offset from BASE_TIME.
@@ -874,6 +897,7 @@ class TestInfo:
         self._skip_reason = skip_reason
         self._attr = list(attr) if attr else []
         self._timeout_ns = timeout_ns
+        self._report_skipped = report_skipped
 
     def name(self):
         # pylint: disable=missing-docstring
@@ -919,7 +943,7 @@ class TestInfo:
         """
         # Deliberately-skipped tests shouldn't have status entries unless errors
         # were also reported.
-        if self._skip_reason and not self._errors:
+        if not self._report_skipped and self._skip_reason and not self._errors:
             return []
 
         def make(status_code, dt, msg=''):
@@ -949,6 +973,12 @@ class TestInfo:
 
             entries.append(make(tast.tast._JOB_STATUS_NOSTATUS, None, reason))
             entries.append(make(tast.tast._JOB_STATUS_END_NOSTATUS, None))
+        elif self._end_time and self._skip_reason and not self._errors:
+            entries.append(
+                    make(tast.tast._JOB_STATUS_SKIP, self._end_time,
+                         self._skip_reason))
+            entries.append(make(tast.tast._JOB_STATUS_END_SKIP,
+                                self._end_time))
         elif self._end_time and not self._errors:
             entries.append(make(tast.tast._JOB_STATUS_GOOD, self._end_time))
             entries.append(make(tast.tast._JOB_STATUS_END_GOOD, self._end_time))