Add support for internal error status checking in regres

This CL adds support for checking if the status is InternalError
on deqp when run on regres. This CL also adds support for running
deqp-vk with --deqp-validation=enabled if needed. The status for
validation errors is InternalError.

Bug: b/181718332
Change-Id: I63196f6f6fb244551be8ed03cce7ed79be95ea38
Reviewed-on: https://swiftshader-review.googlesource.com/c/SwiftShader/+/53268
Tested-by: Venni Ihanakangas <venni.ihanakangas@siru.fi>
Reviewed-by: Paul Thomson <paulthomson@google.com>
Reviewed-by: Ben Clayton <bclayton@google.com>
Kokoro-Result: kokoro <noreply+kokoro@google.com>
Commit-Queue: Venni Ihanakangas <venni.ihanakangas@siru.fi>

3 files changed, 31 insertions, 12 deletions

diff --git a/tests/regres/cmd/run_testlist/main.go b/tests/regres/cmd/run_testlist/main.go
index 10eccbef6..867450226 100644
--- a/tests/regres/cmd/run_testlist/main.go
+++ b/tests/regres/cmd/run_testlist/main.go
@@ -53,16 +53,17 @@ func min(a, b int) int {
 }
 
 var (
-	deqpVkBinary  = flag.String("deqp-vk", "deqp-vk", "path to the deqp-vk binary")
-	testList      = flag.String("test-list", "vk-master-PASS.txt", "path to a test list file")
-	numThreads    = flag.Int("num-threads", min(runtime.NumCPU(), 100), "number of parallel test runner processes")
-	maxProcMemory = flag.Uint64("max-proc-mem", shell.MaxProcMemory, "maximum virtual memory per child process")
-	output        = flag.String("output", "results.json", "path to an output JSON results file")
-	filter        = flag.String("filter", "", "filter for test names. Start with a '/' to indicate regex")
-	limit         = flag.Int("limit", 0, "only run a maximum of this number of tests")
-	shuffle       = flag.Bool("shuffle", false, "shuffle tests")
-	noResults     = flag.Bool("no-results", false, "disable generation of results.json file")
-	genCoverage   = flag.Bool("coverage", false, "generate test coverage")
+	deqpVkBinary     = flag.String("deqp-vk", "deqp-vk", "path to the deqp-vk binary")
+	testList         = flag.String("test-list", "vk-master-PASS.txt", "path to a test list file")
+	numThreads       = flag.Int("num-threads", min(runtime.NumCPU(), 100), "number of parallel test runner processes")
+	maxProcMemory    = flag.Uint64("max-proc-mem", shell.MaxProcMemory, "maximum virtual memory per child process")
+	output           = flag.String("output", "results.json", "path to an output JSON results file")
+	filter           = flag.String("filter", "", "filter for test names. Start with a '/' to indicate regex")
+	limit            = flag.Int("limit", 0, "only run a maximum of this number of tests")
+	shuffle          = flag.Bool("shuffle", false, "shuffle tests")
+	noResults        = flag.Bool("no-results", false, "disable generation of results.json file")
+	genCoverage      = flag.Bool("coverage", false, "generate test coverage")
+	enableValidation = flag.Bool("validation", false, "run deqp-vk with Vulkan validation layers")
 )
 
 const testTimeout = time.Minute * 2
@@ -111,6 +112,7 @@ func run() error {
 		NumParallelTests: *numThreads,
 		TestLists:        testlist.Lists{group},
 		TestTimeout:      testTimeout,
+		ValidationLayer:  *enableValidation,
 	}
 
 	if *genCoverage {
diff --git a/tests/regres/deqp/deqp.go b/tests/regres/deqp/deqp.go
index 818a6dc52..138ca17aa 100644
--- a/tests/regres/deqp/deqp.go
+++ b/tests/regres/deqp/deqp.go
@@ -42,7 +42,7 @@ const dataVersion = 1
 
 var (
 	// Regular expression to parse the output of a dEQP test.
-	deqpRE = regexp.MustCompile(`(Fail|Pass|NotSupported|CompatibilityWarning|QualityWarning) \(([^\)]*)\)`)
+	deqpRE = regexp.MustCompile(`(Fail|Pass|NotSupported|CompatibilityWarning|QualityWarning|InternalError) \(([^\)]*)\)`)
 	// Regular expression to parse a test that failed due to UNIMPLEMENTED()
 	unimplementedRE = regexp.MustCompile(`[^\n]*UNIMPLEMENTED:[^\n]*`)
 	// Regular expression to parse a test that failed due to UNSUPPORTED()
@@ -68,6 +68,7 @@ type Config struct {
 	NumParallelTests int
 	CoverageEnv      *cov.Env
 	TestTimeout      time.Duration
+	ValidationLayer  bool
 }
 
 // Results holds the results of tests across all APIs.
@@ -316,7 +317,14 @@ nextTest:
 		// log.Printf("Running test '%s'\n", name)
 
 		start := time.Now()
+		// Set validation layer according to flag.
+		validation := "disable"
+		if c.ValidationLayer {
+			validation = "enable"
+		}
+
 		outRaw, err := shell.Exec(c.TestTimeout, exe, filepath.Dir(exe), env,
+			"--deqp-validation="+validation,
 			"--deqp-surface-type=pbuffer",
 			"--deqp-shadercache=disable",
 			"--deqp-log-images=disable",
@@ -411,6 +419,12 @@ nextTest:
 					err = toks[2]
 				}
 				results <- TestResult{Test: name, Status: testlist.Fail, Err: err, TimeTaken: duration, Coverage: coverage}
+			case "InternalError":
+				var err string
+				if toks[2] != "InternalError" {
+					err = toks[2]
+				}
+				results <- TestResult{Test: name, Status: testlist.InternalError, Err: err, TimeTaken: duration, Coverage: coverage}
 			default:
 				err := fmt.Sprintf("Couldn't parse test output:\n%s", out)
 				log.Println("Warning: ", err)
diff --git a/tests/regres/testlist/testlist.go b/tests/regres/testlist/testlist.go
index 22e72c43f..04c8eee3e 100644
--- a/tests/regres/testlist/testlist.go
+++ b/tests/regres/testlist/testlist.go
@@ -198,6 +198,8 @@ const (
 	CompatibilityWarning = Status("COMPATIBILITY_WARNING")
 	// QualityWarning is the status passing test with a warning.
 	QualityWarning = Status("QUALITY_WARNING")
+	// InternalError is the status of a test that failed on an API usage error.
+	InternalError = Status("INTERNAL_ERROR")
 )
 
 // Statuses is the full list of status types
@@ -214,12 +216,13 @@ var Statuses = []Status{
 	NotSupported,
 	CompatibilityWarning,
 	QualityWarning,
+	InternalError,
 }
 
 // Failing returns true if the task status requires fixing.
 func (s Status) Failing() bool {
 	switch s {
-	case Fail, Timeout, Crash, Unimplemented, Unreachable, Assert, Abort:
+	case Fail, Timeout, Crash, Unimplemented, Unreachable, Assert, Abort, InternalError:
 		return true
 	case Unsupported:
 		// This may seem surprising that this should be a failure, however these