Add analysis-test test framework (#110)

This framework allows for easy creation of unittest-like tests to make assertions
on the provider-values returned by real targets.

3 files changed, 183 insertions, 10 deletions

diff --git a/tests/BUILD b/tests/BUILD
index 5bd929d..d91aeb1 100644
--- a/tests/BUILD
+++ b/tests/BUILD
@@ -11,6 +11,7 @@ load(":sets_tests.bzl", "sets_test_suite")
 load(":shell_tests.bzl", "shell_test_suite")
 load(":structs_tests.bzl", "structs_test_suite")
 load(":types_tests.bzl", "types_test_suite")
+load(":unittest_tests.bzl", "unittest_passing_tests_suite")
 load(":versions_tests.bzl", "versions_test_suite")
 
 licenses(["notice"])
@@ -39,11 +40,14 @@ structs_test_suite()
 
 types_test_suite()
 
+unittest_passing_tests_suite()
+
 versions_test_suite()
 
 bzl_library(
-    name = "unittest_tests",
+    name = "unittest_tests_bzl",
     srcs = ["unittest_tests.bzl"],
+    visibility = ["//visibility:private"],
     deps = ["//lib:unittest"],
 )
 
@@ -52,7 +56,7 @@ sh_test(
     srcs = ["unittest_test.sh"],
     data = [
         ":unittest.bash",
-        ":unittest_tests",
+        ":unittest_tests_bzl",
         "//toolchains/unittest:test_deps",
         "@bazel_tools//tools/bash/runfiles",
     ],
diff --git a/tests/unittest_test.sh b/tests/unittest_test.sh
index 841717c..e5403d4 100755
--- a/tests/unittest_test.sh
+++ b/tests/unittest_test.sh
@@ -69,11 +69,21 @@ EOF
   cat > testdir/BUILD <<EOF
 load("//tests:unittest_tests.bzl",
     "basic_passing_test",
-    "basic_failing_test")
+    "basic_failing_test",
+    "fail_unexpected_passing_test",
+    "fail_unexpected_passing_fake_rule")
 
 basic_passing_test(name = "basic_passing_test")
 
 basic_failing_test(name = "basic_failing_test")
+
+fail_unexpected_passing_test(
+    name = "fail_unexpected_passing_test",
+    target_under_test = ":fail_unexpected_passing_fake_target",
+)
+fail_unexpected_passing_fake_rule(
+    name = "fail_unexpected_passing_fake_target",
+    tags = ["manual"])
 EOF
 }
 
@@ -94,4 +104,11 @@ function test_basic_failing_test() {
   expect_log "In test _basic_failing_test from //tests:unittest_tests.bzl: Expected \"1\", but got \"2\""
 }
 
+function test_fail_unexpected_passing_test() {
+  ! bazel test //testdir:fail_unexpected_passing_test --test_output=all --verbose_failures \
+      >"$TEST_log" 2>&1 || fail "Expected test to fail"
+
+  expect_log "Expected failure of target_under_test, but found success"
+}
+
 run_suite "unittest test suite"
diff --git a/tests/unittest_tests.bzl b/tests/unittest_tests.bzl
index af0e919..9bdecc6 100644
--- a/tests/unittest_tests.bzl
+++ b/tests/unittest_tests.bzl
@@ -1,5 +1,23 @@
-load("//lib:unittest.bzl", "asserts", "unittest")
+"""Unit tests for unittest.bzl."""
 
+load("//lib:unittest.bzl", "analysistest", "asserts", "unittest")
+
+###################################
+####### fail_basic_test ###########
+###################################
+def _basic_failing_test(ctx):
+    """Unit tests for a basic library verification test that fails."""
+    env = unittest.begin(ctx)
+
+    asserts.equals(env, 1, 2)
+
+    return unittest.end(env)
+
+basic_failing_test = unittest.make(_basic_failing_test)
+
+###################################
+####### basic_passing_test ########
+###################################
 def _basic_passing_test(ctx):
     """Unit tests for a basic library verification test."""
     env = unittest.begin(ctx)
@@ -10,12 +28,146 @@ def _basic_passing_test(ctx):
 
 basic_passing_test = unittest.make(_basic_passing_test)
 
-def _basic_failing_test(ctx):
-    """Unit tests for a basic library verification test that fails."""
-    env = unittest.begin(ctx)
+###################################
+####### change_setting_test #######
+###################################
+def _change_setting_test(ctx):
+    """Test to verify that an analysis test may change configuration."""
+    env = analysistest.begin(ctx)
 
-    asserts.equals(env, 1, 2)
+    dep_min_os_version = ctx.attr.target_under_test[0][_ChangeSettingInfo].min_os_version
+    asserts.equals(env, "1234.5678", dep_min_os_version)
 
-    return unittest.end(env)
+    return analysistest.end(env)
 
-basic_failing_test = unittest.make(_basic_failing_test)
+_ChangeSettingInfo = provider()
+
+def _change_setting_fake_rule(ctx):
+    return [_ChangeSettingInfo(min_os_version = ctx.fragments.cpp.minimum_os_version())]
+
+change_setting_fake_rule = rule(
+    implementation = _change_setting_fake_rule,
+    fragments = ["cpp"],
+)
+
+change_setting_test = analysistest.make(
+    _change_setting_test,
+    config_settings = {
+        "//command_line_option:minimum_os_version": "1234.5678",
+    },
+)
+
+####################################
+####### failure_testing_test #######
+####################################
+def _failure_testing_test(ctx):
+    """Test to verify that an analysis test may verify a rule fails with fail()."""
+    env = analysistest.begin(ctx)
+
+    asserts.expect_failure(env, "This rule should never work")
+
+    return analysistest.end(env)
+
+def _failure_testing_fake_rule(ctx):
+    fail("This rule should never work")
+
+failure_testing_fake_rule = rule(
+    implementation = _failure_testing_fake_rule,
+)
+
+failure_testing_test = analysistest.make(
+    _failure_testing_test,
+    expect_failure = True,
+)
+
+############################################
+####### fail_unexpected_passing_test #######
+############################################
+def _fail_unexpected_passing_test(ctx):
+    """Test that fails by expecting an error that never occurs."""
+    env = analysistest.begin(ctx)
+
+    asserts.expect_failure(env, "Oh no, going to fail")
+
+    return analysistest.end(env)
+
+def _fail_unexpected_passing_fake_rule(ctx):
+    return []
+
+fail_unexpected_passing_fake_rule = rule(
+    implementation = _fail_unexpected_passing_fake_rule,
+)
+
+fail_unexpected_passing_test = analysistest.make(
+    _fail_unexpected_passing_test,
+    expect_failure = True,
+)
+
+################################################
+####### change_setting_with_failure_test #######
+################################################
+def _change_setting_with_failure_test(ctx):
+    """Test verifying failure while changing configuration."""
+    env = analysistest.begin(ctx)
+
+    asserts.expect_failure(env, "unexpected minimum_os_version!!!")
+
+    return analysistest.end(env)
+
+def _change_setting_with_failure_fake_rule(ctx):
+    if ctx.fragments.cpp.minimum_os_version() == "error_error":
+        fail("unexpected minimum_os_version!!!")
+    return []
+
+change_setting_with_failure_fake_rule = rule(
+    implementation = _change_setting_with_failure_fake_rule,
+    fragments = ["cpp"],
+)
+
+change_setting_with_failure_test = analysistest.make(
+    _change_setting_with_failure_test,
+    expect_failure = True,
+    config_settings = {
+        "//command_line_option:minimum_os_version": "error_error",
+    },
+)
+
+#########################################
+
+def unittest_passing_tests_suite():
+    """Creates the test targets and test suite for passing unittest.bzl tests.
+
+    Not all tests are included. Some unittest.bzl tests verify a test fails
+    when assertions are not met. Such tests must be run in an e2e shell test.
+    This suite only includes tests which verify success tests."""
+    unittest.suite(
+        "unittest_tests",
+        basic_passing_test,
+    )
+
+    change_setting_test(
+        name = "change_setting_test",
+        target_under_test = ":change_setting_fake_target",
+    )
+    change_setting_fake_rule(
+        name = "change_setting_fake_target",
+        tags = ["manual"],
+    )
+
+    failure_testing_test(
+        name = "failure_testing_test",
+        target_under_test = ":failure_testing_fake_target",
+    )
+    failure_testing_fake_rule(
+        name = "failure_testing_fake_target",
+        tags = ["manual"],
+    )
+
+    change_setting_with_failure_test(
+        name = "change_setting_with_failure_test",
+        target_under_test = ":change_setting_with_failure_fake_target",
+    )
+    change_setting_with_failure_fake_rule(
+        name = "change_setting_with_failure_fake_target",
+        tags = ["manual"],
+    )