diff options
author | Kalle Raita <kraita@google.com> | 2016-03-25 13:19:09 -0700 |
---|---|---|
committer | Kalle Raita <kraita@google.com> | 2017-05-18 10:25:10 -0700 |
commit | 5284bc1667ddc38b59d452d4c89440d1c301a2fc (patch) | |
tree | 66b60b0803fccda5c1707254f1a7496fb54b6b20 | |
parent | f5edc9544e25d95cfdef5705d3ddfdafcc10fa95 (diff) | |
download | cherry-5284bc1667ddc38b59d452d4c89440d1c301a2fc.tar.gz |
Test set loading and selection without frills
Test: Manual and the new server_test added in the change
Bug: 27303201
Change-Id: I1f89f7b4e5e0e9c949b48becff96098eb80c2fc4
-rw-r--r-- | cherry/config.go | 34 | ||||
-rw-r--r-- | cherry/data.go | 87 | ||||
-rw-r--r-- | cherry/rpc.go | 115 | ||||
-rw-r--r-- | cherry/testrunner.go | 18 | ||||
-rw-r--r-- | cherry/testset.go | 58 | ||||
-rw-r--r-- | client/css/style.css | 14 | ||||
-rw-r--r-- | client/js/controllers.js | 12 | ||||
-rw-r--r-- | client/js/directives.js | 18 | ||||
-rw-r--r-- | client/js/objectOp.js | 22 | ||||
-rw-r--r-- | client/js/testLaunch.js | 253 | ||||
-rw-r--r-- | client/partials/testLaunch.html | 111 | ||||
-rw-r--r-- | server.go | 76 | ||||
-rw-r--r-- | test/server_test.py | 144 |
13 files changed, 814 insertions, 148 deletions
diff --git a/cherry/config.go b/cherry/config.go index 8d4e994..f191be6 100644 --- a/cherry/config.go +++ b/cherry/config.go @@ -19,7 +19,6 @@ package cherry import ( "encoding/xml" "fmt" - "os" "strings" "io" ) @@ -61,21 +60,6 @@ func (tree *TestCaseTree) GetLinearizedList () []string { return list } -// Data types for importing dEQP-*-TestSets.xml - -type TestSet struct { - Name string `xml:"Name,attr"` - Filters string `xml:"TestNameFilters,attr"` -} - -type TestSetList struct { - TestSets []TestSet `xml:"TestSet"` -} - -type CandyConfig struct { - TestSetList TestSetList `xml:"TestSets"` -} - // Import dEQP-*-cases.xml func importTestCaseTree (treeXml io.Reader, packageName string) (*TestCaseTree, error) { @@ -92,24 +76,6 @@ func importTestCaseTree (treeXml io.Reader, packageName string) (*TestCaseTree, return &tree, nil } -// Import dEQP-*-TestSets.xml - -func importTestSets (fileName string) (*TestSetList, error) { - // Open file. - xmlFile, err := os.Open(fileName) - if err != nil { return nil, err } - defer xmlFile.Close() - - // Parse XML. - config := CandyConfig{} - decoder := xml.NewDecoder(xmlFile) - err = decoder.Decode(&config) - if err != nil { return nil, err } - - // Return. - return &config.TestSetList, nil -} - // Tests. func printNode (indent int, node *TestCaseNode) { diff --git a/cherry/data.go b/cherry/data.go index 9497864..6523df9 100644 --- a/cherry/data.go +++ b/cherry/data.go @@ -286,6 +286,9 @@ type BatchExecParams struct { TestBinaryCommandLine string `json:"testBinaryCommandLine"` TestBinaryWorkingDir string `json:"testBinaryWorkingDir"` + // \todo [2017-05-18 kraita]: This field is relevant for 1/3 batch + // launch types. I didn't find any code actually reading the field + // from the DB for anything useful. Should nuke the field altogether. TestNameFilters []string `json:"testNameFilters"` // Filters for test case names (only execute matching). } @@ -526,6 +529,77 @@ func (list *ActiveBatchResultList) Clear () error { return nil } +// TestSetHeader +type TestSetHeader struct { + Id string `json:"id"` + Name string `json:"name"` +} + +func (header *TestSetHeader) PostLoad () { +} + +func (testSet *TestSetHeader) Init (src TestSetHeader) error { + *testSet = src + return nil +} + +// TestSet +type TestSet struct { + Id string `json:"id"` + Name string `json:"name"` + Filters []string `json:"filters"` +} + +func (testSet *TestSet) PostLoad () { + if testSet.Filters == nil { + testSet.Filters = make([]string, 0) + } +} + +func (testSet *TestSet) Init (src TestSet) error { + *testSet = src + return nil +} + +func (device *TestSet) Delete () error { + log.Printf("\\todo [kraita] implement rtdb delete operation!\n") + return nil +} + +// TestSetList +type TestSetList struct { + TestSetHeaders []TestSetHeader `json:"testSetHeaders"` +} + +func (list *TestSetList) PostLoad () { + if list.TestSetHeaders == nil { + list.TestSetHeaders = make([]TestSetHeader, 0) + } +} + +func (list *TestSetList) Init () error { + if list.TestSetHeaders == nil { + list.TestSetHeaders = make([]TestSetHeader, 0) + } + log.Printf("TestSetList.Init(): Loaded %d\n", len(list.TestSetHeaders)) + return nil +} + +func (list *TestSetList) Append (setHeader TestSetHeader) error { + list.TestSetHeaders = append(list.TestSetHeaders, setHeader) + return nil +} + +func (list *TestSetList) Remove (testSetId string) error { + for ndx, header := range list.TestSetHeaders { + if header.Id == testSetId { + list.TestSetHeaders = append(list.TestSetHeaders[:ndx], list.TestSetHeaders[ndx+1:]...) + return nil + } + } + return fmt.Errorf("[data] WARNING: trying to remove unknown test set '%s' from the test set list", testSetId) +} + // DeviceBatchQueue type DeviceBatchQueue struct { @@ -678,6 +752,9 @@ var ( typeDeviceBatchQueueList = reflect.TypeOf((*DeviceBatchQueueList)(nil)).Elem() typeADBDeviceConnectionList = reflect.TypeOf((*ADBDeviceConnectionList)(nil)).Elem() typeCherryObjectSchemaVersion = reflect.TypeOf((*CherryObjectSchemaVersion)(nil)).Elem() + typeTestSetHeader = reflect.TypeOf((*TestSetHeader)(nil)).Elem() + typeTestSet = reflect.TypeOf((*TestSet)(nil)).Elem() + typeTestSetList = reflect.TypeOf((*TestSetList)(nil)).Elem() ) // Return all RTDB object types used by Cherry. @@ -699,6 +776,9 @@ func GetObjectTypes () []reflect.Type { typeDeviceBatchQueueList, typeADBDeviceConnectionList, typeCherryObjectSchemaVersion, + typeTestSetHeader, + typeTestSet, + typeTestSetList, } } @@ -719,9 +799,10 @@ func InitDB (rtdbServer *rtdb.Server) { // Initialize empty active & full batch result lists, as well as device batch queue list. { opSet := rtdb.NewOpSet() - opSet.Call(typeBatchResultList, "batchResultList", "Init") - opSet.Call(typeActiveBatchResultList, "activeBatchResultList", "Init") - opSet.Call(typeDeviceBatchQueueList, "deviceBatchQueueList", "Init") + opSet.Call(typeBatchResultList, "batchResultList", "Init") + opSet.Call(typeActiveBatchResultList, "activeBatchResultList", "Init") + opSet.Call(typeDeviceBatchQueueList, "deviceBatchQueueList", "Init") + opSet.Call(typeTestSetList, "testSetList", "Init") err := rtdbServer.ExecuteOpSet(opSet) if err != nil { panic(err) } } diff --git a/cherry/rpc.go b/cherry/rpc.go index 5c42682..98e9556 100644 --- a/cherry/rpc.go +++ b/cherry/rpc.go @@ -81,52 +81,55 @@ func (handler *RPCHandler) Unsubscribe (client *RPCClient, args UnsubscribeArgs) } // ExecuteTestBatch -// \todo [petri] replace most of this struct with references to relevant places? -type ExecuteTestBatchArgs struct { - // Device config. - TargetAddress string `json:"targetAddress"` - TargetPort int `json:"targetPort"` - SpawnLocalProcess string `json:"spawnLocalProcess"` - DeviceId string `json:"deviceId"` +type DeviceConfigArgs struct { + TargetAddress string `json:"targetAddress"` + TargetPort int `json:"targetPort"` + SpawnLocalProcess string `json:"spawnLocalProcess"` + DeviceId string `json:"deviceId"` +} - // Test package config. - TestBinaryName string `json:"testBinaryName"` - TestBinaryCommandLine string `json:"testBinaryCommandLine"` - TestBinaryWorkingDir string `json:"testBinaryWorkingDir"` +type TestPackageConfigArgs struct { + TestBinaryName string `json:"testBinaryName"` + TestBinaryCommandLine string `json:"testBinaryCommandLine"` + TestBinaryWorkingDir string `json:"testBinaryWorkingDir"` +} - // Test set config. - TestNameFilters string `json:"testNameFilters"` +// \todo [2017-03-15 kraita] Embedded structs could be replaced by object references? +type ExecuteTestBatchArgs struct { + DeviceConfig DeviceConfigArgs `json:"deviceConfig"` + TestPackageConfig TestPackageConfigArgs `json:"testPackageConfig"` + TestNameFilters string `json:"testNameFilters"` +} + +func getDefaultBatchName () string { + return time.Now().Format(defaultHumanReadableTimeFormat) } func (handler *RPCHandler) ExecuteTestBatch (client *RPCClient, args ExecuteTestBatchArgs) (string, error) { log.Println("[rpc] ExecuteTestBatch requested") // Check that the specified device exists. - err := handler.rtdbServer.GetObject(args.DeviceId, &DeviceConfig{}) + err := handler.rtdbServer.GetObject(args.DeviceConfig.DeviceId, &DeviceConfig{}) if err != nil { return "", err } // Generate time-based id for batch result. startTime := time.Now() - batchResultName := startTime.Format(defaultHumanReadableTimeFormat) // Execute tests in background. execParams := BatchExecParams { - // device - TargetAddress: args.TargetAddress, - TargetPort: args.TargetPort, - SpawnLocalProcess: args.SpawnLocalProcess, - DeviceId: args.DeviceId, - - // test package - TestBinaryName: args.TestBinaryName, - TestBinaryCommandLine: args.TestBinaryCommandLine, - TestBinaryWorkingDir: args.TestBinaryWorkingDir, - - // tests + TargetAddress: args.DeviceConfig.TargetAddress, + TargetPort: args.DeviceConfig.TargetPort, + SpawnLocalProcess: args.DeviceConfig.SpawnLocalProcess, + DeviceId: args.DeviceConfig.DeviceId, + + TestBinaryName: args.TestPackageConfig.TestBinaryName, + TestBinaryCommandLine: args.TestPackageConfig.TestBinaryCommandLine, + TestBinaryWorkingDir: args.TestPackageConfig.TestBinaryWorkingDir, + TestNameFilters: strings.Split(args.TestNameFilters, ";"), } - batchResultId, err := handler.testRunner.ExecuteTestBatch(batchResultName, execParams, startTime) + batchResultId, err := handler.testRunner.ExecuteTestBatch(getDefaultBatchName(), execParams, startTime) if err != nil { return "", err } return batchResultId, nil @@ -151,18 +154,56 @@ func (handler *RPCHandler) ExecuteSubTestBatch (client *RPCClient, args ExecuteS err = handler.rtdbServer.GetObject(args.OriginalBatchResultId, &originalCaseList) if err != nil { return "", err } - // Generate time-based id for batch result. startTime := time.Now() - batchResultName := startTime.Format(defaultHumanReadableTimeFormat) testCasePaths := filterTestCaseNames(originalCaseList.Paths, strings.Split(args.TestNameFilters, ";")) - batchResultId, err := handler.testRunner.ExecuteTestBatchWithCaseList(batchResultName, original.ExecParams, startTime, testCasePaths) + batchResultId, err := handler.testRunner.ExecuteTestBatchWithCaseList(getDefaultBatchName(), original.ExecParams, startTime, testCasePaths) if err != nil { return "", err } return batchResultId, nil } +type ExecuteTestSetArgs struct { + DeviceConfig DeviceConfigArgs `json:"deviceConfig"` + TestPackageConfig TestPackageConfigArgs `json:"testPackageConfig"` + TestSetId string `json:"testSetId"` +} + +func (handler *RPCHandler) ExecuteTestSet (client *RPCClient, args ExecuteTestSetArgs) (string, error) { + log.Println("[rpc] ExecuteTestSet requested") + + // Check that the specified device exists. + err := handler.rtdbServer.GetObject(args.DeviceConfig.DeviceId, &DeviceConfig{}) + if err != nil { return "", err } + + startTime := time.Now() + + // Execute tests in background. + execParams := BatchExecParams { + TargetAddress: args.DeviceConfig.TargetAddress, + TargetPort: args.DeviceConfig.TargetPort, + SpawnLocalProcess: args.DeviceConfig.SpawnLocalProcess, + DeviceId: args.DeviceConfig.DeviceId, + + TestBinaryName: args.TestPackageConfig.TestBinaryName, + TestBinaryCommandLine: args.TestPackageConfig.TestBinaryCommandLine, + TestBinaryWorkingDir: args.TestPackageConfig.TestBinaryWorkingDir, + + TestNameFilters: make([]string, 0), + } + + // Test sets can have some tens of MBs data. Should not load the + // full test set in the server thread. Passing the test set ID to + // the runner instead. + log.Printf("Requesting execution of test set: %s", args.TestSetId) + batchResultId, err := handler.testRunner.ExecuteTestBatchWithTestSet(getDefaultBatchName(), execParams, startTime, args.TestSetId) + if err != nil { return "", err } + + return batchResultId, nil +} + + // WouldQueueWithOnlyDifferentDevices type WouldQueueWithOnlyDifferentDevicesArgs struct { @@ -348,5 +389,15 @@ func (handler *RPCHandler) GetVersionViewedObjects (client *RPCClient, args GetV // GetTestCaseList func (handler *RPCHandler) GetTestCaseList (client *RPCClient, args struct{}) ([]string, error) { - return handler.testRunner.fullTestCaseList, nil + return handler.testRunner.fullTestCaseList, nil +} + +// Delete TestSet + +func (handler *RPCHandler) DeleteTestSet (client *RPCClient, testSetId string) (bool, error) { + opSet := rtdb.NewOpSet() + opSet.Delete(typeTestSet, testSetId) + opSet.Call(typeTestSetList, "testSetList", "Remove", testSetId) + err := handler.rtdbServer.ExecuteOpSet(opSet) + return err == nil, err } diff --git a/cherry/testrunner.go b/cherry/testrunner.go index 42079b4..ec75de3 100644 --- a/cherry/testrunner.go +++ b/cherry/testrunner.go @@ -69,7 +69,6 @@ type TestRunner struct { // \todo [petri] these should be dynamic, not loaded at init time! testPackages map[string]TestPackageInfo fullTestCaseList []string - // Control channel for batch executions, read in runner.handleQueue(). queueControl chan<- batchExecQueueControl @@ -697,6 +696,23 @@ func (runner *TestRunner) ExecuteTestBatch (batchName string, batchParams BatchE return runner.ExecuteTestBatchWithCaseList(batchName, batchParams, timestamp, testCasePaths) } +func (runner *TestRunner) ExecuteTestBatchWithTestSet (batchName string, batchParams BatchExecParams, timestamp time.Time, testSetId string) (string, error) { + var testSet TestSet + err := runner.rtdbServer.GetObject(testSetId, &testSet) + log.Printf("Start test set '%s' with %d filters.\n", testSetId, len(testSet.Filters)) + if err != nil { + panic(err) + } + for ndx, filter := range(testSet.Filters) { + if ndx > 5 { + log.Printf("...") + break + } + log.Printf(" Filter: '%s'", filter) + } + return runner.ExecuteTestBatchWithCaseList(batchName, batchParams, timestamp, testSet.Filters) +} + // Create a new batch, with a specific case list and no regard to batchParams.TestNameFilters, and start executing asynchronously. func (runner *TestRunner) ExecuteTestBatchWithCaseList (batchName string, batchParams BatchExecParams, timestamp time.Time, testCasePaths []string) (string, error) { batchResultId := runner.rtdbServer.MakeUniqueID() diff --git a/cherry/testset.go b/cherry/testset.go new file mode 100644 index 0000000..e4ba15d --- /dev/null +++ b/cherry/testset.go @@ -0,0 +1,58 @@ +/* + * Copyright 2017 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cherry + +import ( + "fmt" + "regexp" + "../rtdb" +) + +func checkTestSetFilters (setFilters []string) error { + filterChecker := regexp.MustCompile(`^dEQP-[a-zA-Z0-9-_.*]+$`) + for _, filter := range setFilters { + if !filterChecker.MatchString(filter) { + return fmt.Errorf("'%s' is not a valid path filter", filter) + } + } + return nil +} + +func AddTestSet (rtdbServer *rtdb.Server, setName string, setFilters []string) error { + err := checkTestSetFilters(setFilters) + if err != nil { + return err + } + + opSet := rtdb.NewOpSet() + testSetId := rtdbServer.MakeUniqueID() + + testSet := TestSet { + Id: testSetId, + Name: setName, + Filters: setFilters, + } + testSetHeader := TestSetHeader { + Id: testSet.Id, + Name: testSet.Name, + } + opSet.Call(typeTestSet, testSetId, "Init", testSet) + opSet.Call(typeTestSetList, "testSetList", "Append", testSetHeader) + err = rtdbServer.ExecuteOpSet(opSet) + return err +} + diff --git a/client/css/style.css b/client/css/style.css index 7cb6f74..d74d75b 100644 --- a/client/css/style.css +++ b/client/css/style.css @@ -272,3 +272,17 @@ ul.nav-sidebar > li.active { box-shadow: 0 2px 2px rgba(0, 0, 0, 0.075) inset, 0 0 8px rgba(102, 233, 125, 0.6); outline: 0 none; } + +span.selected-test-set { + font-weight: bold; +} + +.test-set-panel { + padding-top: 8px; + padding-bottom: 8px; +} + +.btn-test-set { + margin-top: 8px; + margin-bottom: 0px; +} diff --git a/client/js/controllers.js b/client/js/controllers.js index afd969d..d241f71 100644 --- a/client/js/controllers.js +++ b/client/js/controllers.js @@ -14,6 +14,17 @@ 'use strict'; +// Function to load a test set. +function loadTestSet(rpc, testSetToLoad) +{ + var testSet = testSetToLoad; // Note: Closure for the then call. + rpc.call('rtdb.GetTestSetFilters', {testSetName:testSet.name}) + .then(function(paths) + { + testSet.paths = paths; + }); +} + // Controllers angular.module('cherry.controllers', []) @@ -26,6 +37,7 @@ angular.module('cherry.controllers', []) rtdb.prefetch('DeviceConfigList', 'deviceConfigList', $scope); rtdb.prefetch('BatchResultList', 'batchResultList', $scope); rtdb.prefetch('ActiveBatchResultList', 'activeBatchResultList', $scope); + rtdb.prefetch('TestSetList', 'testSetList', $scope); var numActiveUploads = 0; angular.extend($scope, diff --git a/client/js/directives.js b/client/js/directives.js index e2ce122..ffb3e1e 100644 --- a/client/js/directives.js +++ b/client/js/directives.js @@ -149,6 +149,24 @@ angular.module('cherry.directives', []) } }) +.directive('fileModel', function($compile) +{ + return { + restrict: 'A', + link: function(scope, elem, attrs) + { + elem.attr('onchange', 'angular.element(this).scope().' + attrs.fileModel + ' = this.files;'); + scope.$watch(attrs.fileModel, function(file) { + if (typeof file === "undefined" || file.length == 0) + { + // File input can only be cleared programmatically. + elem.val(""); + } + }); + } + } +}) + .directive('onOffButtonModel', function($compile) { return { diff --git a/client/js/objectOp.js b/client/js/objectOp.js index 58f9314..9e850b5 100644 --- a/client/js/objectOp.js +++ b/client/js/objectOp.js @@ -371,5 +371,27 @@ function copyProperties(dst, src) value.connections.push(connection); } }, + + // TestSetList + TestSetList: + { + Append: function(value, setHeader) + { + value.testSetHeaders.push(setHeader) + }, + + Remove: function(value, testSetId) + { + for (var ndx = 0; ndx < value.testSetHeaders.length; ndx++) + { + var set = value.testSetHeaders[ndx]; + if (set.id == testSetId) + { + value.testSetHeaders.splice(ndx, 1); + return; + } + } + } + }, }; })(typeof exports === 'undefined' ? this['objectop'] = {} : exports); diff --git a/client/js/testLaunch.js b/client/js/testLaunch.js index 070fe16..69eaf42 100644 --- a/client/js/testLaunch.js +++ b/client/js/testLaunch.js @@ -34,75 +34,125 @@ angular.module('cherry.testLaunch', []) deviceSettings: {}, // child scopes will fill this with { deviceId:$scope.value } deviceError: {}, // { deviceId:error } if the specific device has an error prohibiting test execution deviceIsADB: {}, + activeSelectorTab: 'tree', + selectedTestSetId: undefined, selectDevice: function(deviceId) { $scope.selectedDeviceId = deviceId; }, - executeTestBatch: function() + setSelectorTab: function(tab) { - // Combine final test filters string from test case tree selections and manually specified field. - var testFilters = []; - if (testTreeAccess.nodeSelected) - { - var str = genTestCaseTreeSubsetFilter($scope.fullTestCaseTree, testTreeAccess.nodeSelected); - if (str !== '') - testFilters.push(str); - } - if ($scope.testNameFilters) - testFilters = testFilters.concat(_.filter($scope.testNameFilters.split(";"), function(f) { return f.length !== 0; })); - testFilters = testFilters.join(";"); + $scope.activeSelectorTab = tab + }, + canExecuteTestBatch: function() + { + var setsValid = $scope.activeSelectorTab === 'sets' && !!$scope.selectedTestSetId; + return !!$scope.selectedDeviceId && ($scope.activeSelectorTab === 'tree' || setsValid); + }, + + getDeviceConfig: function(deviceId) + { // Use settings specified in the child element (device config). - var deviceConfig = $scope.deviceSettings[$scope.selectedDeviceId]; - console.log('[exec] config "' + $scope.selectedDeviceId + '":'); + var deviceConfig = $scope.deviceSettings[deviceId]; + console.log('[exec] config "' + deviceId + '":'); _.map(deviceConfig, function(value, key) { console.log(' ' + key + ': ' + JSON.stringify(value)); }); var targetPort = parseInt(deviceConfig.targetPort) || 0; // \todo [petri] better error checking/handling - if ($scope.deviceError.hasOwnProperty($scope.selectedDeviceId)) - alert($scope.deviceError[$scope.selectedDeviceId]); + // \todo [2017-03-15 kraita] Also, split sanity checks into another function. + if ($scope.deviceError.hasOwnProperty(deviceId)) + alert($scope.deviceError[deviceId]); else if (!deviceConfig.targetAddress) alert('Invalid target address: "' + (deviceConfig.targetAddress || '') + '"'); else if (targetPort <= 0) alert('Invalid target port: "' + targetPort + '"'); - else if (!testFilters) - alert('No tests selected'); + + var config = { + targetAddress: deviceConfig.targetAddress, + targetPort: targetPort, + spawnLocalProcess: deviceConfig.localProcessPath, + deviceId: deviceId, + } + + return config; + }, + + getTestPackageConfig: function(deviceId) + { + var deviceConfig = $scope.deviceSettings[deviceId]; + var config = { + testBinaryName: deviceConfig.binaryPath, + testBinaryCommandLine: deviceConfig.commandLine || '', + testBinaryWorkingDir: deviceConfig.workingDir, + }; + return config; + }, + + executeTestBatch: function() + { + var params = { + deviceConfig: $scope.getDeviceConfig($scope.selectedDeviceId), + testPackageConfig: $scope.getTestPackageConfig($scope.selectedDeviceId), + }; + + var executeMethod = undefined; + + if ($scope.activeSelectorTab === 'sets') + { + params.testSetId = $scope.selectedTestSetId; + if (!params.testSetId) { + alert('No test set selected'); + return; + } + executeMethod = 'rtdb.ExecuteTestSet'; + console.log("[exec] Execute test set: " + params.testSetId); + } else { - var params = { - targetAddress: deviceConfig.targetAddress, - targetPort: targetPort, - spawnLocalProcess: deviceConfig.localProcessPath, - deviceId: $scope.selectedDeviceId, - testBinaryName: deviceConfig.binaryPath, - testBinaryCommandLine: deviceConfig.commandLine || '', - testBinaryWorkingDir: deviceConfig.workingDir, - testNameFilters: testFilters, - }; - - // Check if the queue that this execution would go to contains different devices, and not this one. - // It's unlikely that one would like to queue (i.e. use same port for) different devices. - rpc.call('rtdb.WouldQueueWithOnlyDifferentDevices', { - targetAddress: deviceConfig.targetAddress, - targetPort: targetPort, - deviceId: $scope.selectedDeviceId, - }) - .then(function(wouldQueueWithOnlyDifferentDevices) + // Combine final test filters string from test case tree selections and manually specified field. + var testFilters = []; + if (testTreeAccess.nodeSelected) { - if (!wouldQueueWithOnlyDifferentDevices || confirm("Different device is using the same address and port - really queue?")) - { - rpc.call('rtdb.ExecuteTestBatch', params) - .then(function(batchResultId) - { - console.log('Executing ' + batchResultId); - $location.url('/results/batch/' + batchResultId); - }); - } - }); + var str = genTestCaseTreeSubsetFilter($scope.fullTestCaseTree, testTreeAccess.nodeSelected); + if (str !== '') + testFilters.push(str); + } + if ($scope.testNameFilters) + testFilters = testFilters.concat(_.filter($scope.testNameFilters.split(";"), function(f) { return f.length !== 0; })); + testFilters = testFilters.join(";"); + + if (!testFilters) { + alert('No tests selected'); + return; + } + + params.testNameFilters = testFilters; + executeMethod = 'rtdb.ExecuteTestBatch' } + + // Check if the queue that this execution would go to contains different devices, and not this one. + // It's unlikely that one would like to queue (i.e. use same port for) different devices. + rpc.call('rtdb.WouldQueueWithOnlyDifferentDevices', { + targetAddress: params.deviceConfig.targetAddress, + targetPort: params.deviceConfig.targetPort, + deviceId: params.deviceId, + }) + .then(function(wouldQueueWithOnlyDifferentDevices) + { + if (!wouldQueueWithOnlyDifferentDevices || confirm("Different device is using the same address and port - really queue?")) + { + rpc.call(executeMethod, params) + .then(function(batchResultId) + { + console.log('Executing ' + batchResultId); + $location.url('/results/batch/' + batchResultId); + }); + } + }); }, setTestTreeAccess: function(access) @@ -110,6 +160,11 @@ angular.module('cherry.testLaunch', []) testTreeAccess = access; }, + getTestTreeAccess: function() + { + return testTreeAccess; + }, + testTreeSelectionType: function(event) { if (event.shiftKey) @@ -238,4 +293,108 @@ angular.module('cherry.testLaunch', []) }); }]) +.controller('TestSetSelectCtrl', ['$scope', 'rtdb', 'rpc', function($scope, rtdb, rpc) +{ + rtdb.bind('TestSetList', 'testSetList', $scope, { valueName: 'testSets' }); + + angular.extend($scope, + { + selectedSetHeader: undefined, + editableTestSetName: undefined, + uploadFilterFile: [], + + selectTestSet: function(testSet) + { + if (!testSet || (!!$scope.selectedSetHeader && testSet.id === $scope.selectedSetHeader.id)) + { + $scope.selectedSetHeader = undefined; + $scope.$parent.selectedTestSetId = undefined; + $scope.editableTestSetName = undefined; + } + else + { + $scope.selectedSetHeader = testSet; + $scope.$parent.selectedTestSetId = testSet.id; + $scope.editableTestSetName = $scope.selectedSetHeader.name; + } + }, + + clearTestSetUploadFields: function() + { + $scope.editableTestSetName = undefined; + $scope.uploadFilterFile = []; + }, + + makeUploadFailureNotification: function(setName) + { + var name = setName; + return function() { + alert("Uploading test set '" + name + "' failed"); + $scope.uploadFinished(); + }; + }, + + uploadTestSet: function() + { + if (!$scope.editableTestSetName || $scope.uploadFilterFile[0] == null) + return; + + console.log("Uploading test set: " + $scope.editableTestSetName); + + var formData = new FormData(); + var setName = $scope.editableTestSetName; + var filterFile = $scope.uploadFilterFile[0]; + + formData.append("set-name", setName); + formData.append("set-filters", filterFile); + + $.ajax({ + type: 'POST', + url: '/importTestSet/', + beforeSend: $scope.uploadStarted, + success: $scope.uploadFinished, + error: $scope.makeUploadFailureNotification(setName), + data: formData, + cache: false, + contentType: false, + processData: false, + }); + + $scope.clearTestSetUploadFields(); + }, + + deleteTestSet: function(testSet) + { + if (confirm('Really delete test set "' + $scope.selectedSetHeader.name + '"?')) + { + var setId = $scope.selectedSetHeader.id + rpc.call('rtdb.DeleteTestSet', setId) + .then(function() + { + console.log('deleted test set: ' + setId); + }, + function() + { + alert("Test set deletion failed!"); + }); + + // Immediately de-select test set to discourage interacting with it before delete is complete. + $scope.selectTestSet(undefined); + } + }, + + initTestSetCtrl: function() + { + }, + // Test set todos: + // - Button text 'Add' vs. 'Update' when existing selected, i.e., change text based on action + // - Rename choose files + // - Visualization of filters + // - Wildcards (expand against local tree or executable tree?) + // - Enable uploading new set of filters? + // - Interactive editing of filters? + // - What to do with the 'Additional tests' field? Move under the tree select tab? Concatenate with the test set filter list? + }); +}]) + ; diff --git a/client/partials/testLaunch.html b/client/partials/testLaunch.html index d084c39..ab2902c 100644 --- a/client/partials/testLaunch.html +++ b/client/partials/testLaunch.html @@ -200,44 +200,93 @@ limitations under the License. <form id="testCaseSelection" class="form-horizontal"> <div class="row form-group"> - <div class="col-md-offset-2 col-md-8"> - <div class="panel panel-default"> - <div class="panel-heading"> - <span class="test-path-filter-input-container"> - <input id="testCasePathFilter" type="text" placeholder="Path filter" class="form-control input-sm test-path-filter-input" ng-model="testCasePathFilter" /> - </span> - </div> - <div class="panel-body test-launch-tree-container" style="overflow-y:scroll;"> - <div class="tree-border"> - <!-- \todo [nuutti 14-08-2014] Shift-click selection annoyingly hilights the text. - Do something so that doesn't happen. Funnily, doesn't seem to happen - with <a> elements with href (on most browsers at least). --> - <treecontrol class="tree-light" tree-model="fullTestCaseTree" tree-selectable="{selectionType:testTreeSelectionType, setAccess:setTestTreeAccess}"> - {{ node.label }} - </treecontrol> + <div class="container"> + <div class="row form-group"> + <div class="col-md-8 col-md-offset-1"> + <ul class="nav nav-tabs"> + <li class="clickable" ng-class="{active:activeSelectorTab==='tree'}"><a ng-click="setSelectorTab('tree')">Select</a></li> + <li class="clickable" ng-class="{active:activeSelectorTab==='sets'}"><a ng-click="setSelectorTab('sets')">Test sets</a></li> + </ul> + <div class="tab-content"> + <div id="select" class="tab-pane" ng-class="{active:activeSelectorTab==='tree'}"> + <div class="panel panel-default"> + <div class="panel-heading"> + <span class="test-path-filter-input-container"> + <input id="testCasePathFilter" type="text" placeholder="Path filter" class="form-control input-sm test-path-filter-input" ng-model="testCasePathFilter" /> + </span> + </div> + <div class="panel-body test-launch-tree-container" style="overflow-y:scroll;"> + <div class="tree-border"> + <!-- \todo [nuutti 14-08-2014] Shift-click selection annoyingly hilights the text. + Do something so that doesn't happen. Funnily, doesn't seem to happen + with <a> elements with href (on most browsers at least). --> + <treecontrol class="tree-light" tree-model="fullTestCaseTree" tree-selectable="{selectionType:testTreeSelectionType, setAccess:setTestTreeAccess}"> + {{ node.label }} + </treecontrol> + </div> + </div> + </div> + </div> + <div id="testsets" class="tab-pane" ng-class="{active:activeSelectorTab==='sets'}" ng-controller="TestSetSelectCtrl" ng-init="initTestSetCtrl()"> + <div style="margin-top: 8px;"> + <span class="clickable" id="expandButton" ng-click="isOpen = !isOpen"> + Manage sets + <i class="glyphicon" ng-class="{'glyphicon-chevron-down': isOpen, 'glyphicon-chevron-right': !isOpen}"></i> + </span> + </div> + <div class="panel-collapse" collapse="!isOpen"> + <div class="panel-body test-set-panel"> + <form enctype="multipart/form-data"> + <div class="row controller-row"> + <div class="col-md-6"> + <input id="editableTestSetName" type="text" placeholder="Test set name" class="form-control input" ng-model="editableTestSetName" /> <!-- Separate editable name from selected set name --> + </div> + <div class="col-md-6"> + <input id="selectedFilterFile" file-model="uploadFilterFile" type="file" multiple> + </div> + </div> + <div class="row"> + <div class="col-md-2"> + <button id="addTestSetButton" type="button" class="btn btn-primary btn-block btn-test-set" ng-click="uploadTestSet()" ng-disabled="!editableTestSetName">Add</button> + </div> + <div class="col-md-2"> + <button id="deleteTestSetButton" type="button" class="btn btn-primary btn-block btn-test-set" ng-click="deleteTestSet(selectedTestSet)" ng-disabled="!selectedSetHeader">Delete</button> + </div> + </div> + </form> + </div> + </div> + <div class="panel panel-default"> + <div class="panel-body test-launch-tree-container" style="overflow-y:scroll;"> + <div ng-repeat="testSet in testSets.testSetHeaders" class="clickable tree-label" ng-click="selectTestSet(testSet)" > + <span ng-class="{'selected-test-set':selectedSetHeader.id===testSet.id}">{{ testSet.name }} </span> + </div> + </div> + </div> + </div> </div> </div> </div> - </div> - </div> - - <div class="row form-group"> - <label class="col-md-2 control-label" for="testNameFilters">Additional Tests:</label> - <div class="col-md-8"> - <input id="testNameFilters" type="text" class="form-control input-md" ng-model="testNameFilters" /> + <div class="row form-group"> + <label class="col-md-2 control-label" for="testNameFilters">Additional Tests:</label> + </div> + <div class="row form-group"> + <div class="col-md-7 col-md-offset-1"> + <input id="testNameFilters" type="text" class="form-control input-md" ng-model="testNameFilters" /> + </div> + </div> + <!-- Execute button --> + <div class="pull-right clearfix"> + <!-- \todo [petri] disable button selected device if not properly configured? --> + <button id="executeButton" class="btn btn-lg btn-success" ng-disabled="!canExecuteTestBatch()" ng-click="executeTestBatch()">Execute Tests!</button> + </div> + <!-- \todo [petri] kludge to add a bit of margin below button --> + <br /> </div> </div> </form> - - <!-- Execute button --> - <div class="pull-right clearfix"> - <!-- \todo [petri] disable button selected device if not properly configured? --> - <button id="executeButton" class="btn btn-lg btn-success" ng-disabled="selectedDeviceId===undefined" ng-click="executeTestBatch()">Execute Tests!</button> - </div> - - <!-- \todo [petri] kludge to add a bit of margin below button --> - <br /> </div> </div> +</div> </div> @@ -32,6 +32,8 @@ import ( "time" "strconv" "io" + "io/ioutil" + "strings" ) var rtdbServer *rtdb.Server @@ -295,6 +297,79 @@ func importHandler (response http.ResponseWriter, request *http.Request) { } } +// Garbage in, strings split by newlines and comments removed out. +func splitAndTrimTestSetFilters (setFilters string) []string { + cleanFilters := make([]string, 0) + commentEraser := regexp.MustCompile(`#.*$`) + + for _, filter := range strings.Split(strings.Replace(setFilters,"\r\n","\n", -1), "\n") { + noComments := commentEraser.ReplaceAllString(filter, "") + clean := strings.TrimSpace(noComments) + if len(clean) > 0 { + cleanFilters = append(cleanFilters, clean) + } + } + return cleanFilters +} + +func importTestSetHandler (response http.ResponseWriter, request *http.Request) { + if request.Method != "POST" || request.RequestURI != "/importTestSet/" { + http.Error(response, "Invalid request", 400) + return + } + + parts, err := request.MultipartReader() + if err != nil { + http.Error(response, "Expected a multipart upload", 400) + return + } + + log.Printf("[test set import] Received request with Content-Length %d\n", request.ContentLength) + + anyImportFailed := false + + var setName string + var setFilters string + + for { + part, err := parts.NextPart(); + if err != nil { + break + } + + data, err := ioutil.ReadAll(part) + if err != nil { + log.Printf("[test set import] Reading upload failed with error %v\n", err) + anyImportFailed = true + } else { + switch part.FormName() { + case "set-name": + setName = string(data) + case "set-filters": + setFilters = string(data) + } + } + part.Close() + } + + request.Body.Close() + + filterList := splitAndTrimTestSetFilters(setFilters) + if !anyImportFailed && len(setName) > 0 && len(filterList) > 0 { + err = cherry.AddTestSet(rtdbServer, setName, filterList) + if err != nil { + log.Printf("[test set import] Import failed with error %v\n", err) + http.Error(response, "Import failed", 500) + } else { + log.Printf("[test set import] Imported test set %s with %d filters\n", setName, len(filterList)) + response.WriteHeader(http.StatusOK) + } + } else { + log.Printf("[test set import] Tried to import empty filter set or empty set name or other failure.\n") + http.Error(response, "Import failed", 500) + } +} + // Mapping of third party locations to desired server locations // This allows us to remap the locations for release packages or when versions change func setUpFileMappings() { @@ -375,6 +450,7 @@ func main () { http.HandleFunc("/executionLog/", executionLogExportHandler) http.HandleFunc("/export/", exportHandler) http.HandleFunc("/import/", importHandler) + http.HandleFunc("/importTestSet/", importTestSetHandler) // Fire up http server! addr := fmt.Sprintf("127.0.0.1:%d", *port) diff --git a/test/server_test.py b/test/server_test.py new file mode 100644 index 0000000..dac4063 --- /dev/null +++ b/test/server_test.py @@ -0,0 +1,144 @@ +# Copyright 2017 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import sys +import time +import shutil +import subprocess +import unittest +import argparse +import socket +import requests +import textwrap + +server_port = 0 + +def getBaseUrl(): + return 'http://localhost:%d' % server_port + + +class TestSetUploadTest(unittest.TestCase): + def testSimpleUpload(self): + files = { + "set-name": ("Foo", "RealSetName"), + "set-filters": ("filterList.txt", + textwrap.dedent("""dEQP-GLES2.functional.fragment_ops.blend.rgb_func_alpha_func.dst.one_minus_src_alpha_src_color + dEQP-GLES2.functional.fragment_ops.blend.rgb_func_alpha_func.dst.one_minus_src_alpha_one_minus_src_color""")) + } + r = requests.post(getBaseUrl()+"/importTestSet/", files=files) + self.assertEqual(r.status_code, requests.codes.ok) + + # \todo [2017-05-11 kraita]: Use python websockets to verify DB changes? + # \todo [2017-05-11 kraita]: Re-do in Go and use existing Go code? + + def testEmptyFilterFile(self): + files = { + "set-name": ("Foo", "RealSetName"), + "set-filters": ("filterList.txt", "") + } + r = requests.post(getBaseUrl()+"/importTestSet/", files=files) + self.assertEqual(r.status_code, requests.codes.server_error) + + def testEmptyName(self): + files = { + "set-name": ("Foo", ""), + "set-filters": ("filterList.txt", + textwrap.dedent("""dEQP-GLES2.functional.fragment_ops.blend.rgb_func_alpha_func.dst.one_minus_src_alpha_src_color + dEQP-GLES2.functional.fragment_ops.blend.rgb_func_alpha_func.dst.one_minus_src_alpha_one_minus_src_color""")) + } + r = requests.post(getBaseUrl()+"/importTestSet/", files=files) + self.assertEqual(r.status_code, requests.codes.server_error) + + def testInvalidFilter(self): + files = { + "set-name": ("Foo", "Broken filter"), + "set-filters": ("filterList.txt", + textwrap.dedent("""dEQP-GLES2.functional.fragment_ops.blend.rgb_func_alpha_func.dst.one_minus_src_alpha_src_color + dEQP-!%^@!^@^.6t=-34. + dEQP-GLES2.functional.fragment_ops.blend.rgb_func_alpha_func.dst.one_minus_src_alpha_one_minus_src_color""")) + } + r = requests.post(getBaseUrl()+"/importTestSet/", files=files) + self.assertEqual(r.status_code, requests.codes.server_error) + + def testPatternFilter(self): + files = { + "set-name": ("Foo", "PatternFilter"), + "set-filters": ("filterList.txt", + textwrap.dedent("""dEQP-GLES2.functional.fragment_ops.blend.rgb_func_alpha_func.dst.one_minus_src_alpha_src_color + dEQP-EGL.functional.* + dEQP-GLES2.functional.fragment_ops.blend.rgb_func_alpha_func.dst.one_minus_src_alpha_one_minus_src_color""")) + } + r = requests.post(getBaseUrl()+"/importTestSet/", files=files) + self.assertEqual(r.status_code, requests.codes.ok) + + def testComments(self): + files = { + "set-name": ("Foo", "FilterWithComments"), + "set-filters": ("filterList.txt", + textwrap.dedent("""dEQP-GLES2.functional.fragment_ops.blend.rgb_func_alpha_func.dst.one_minus_src_alpha_src_color + #This is a foobar comment 1()*&*^(_) + dEQP-EGL.foo.bar # Comment at the end of line )(*&^%$^&*()*#$&*(":}:{:{}[] + dEQP-GLES2.functional.fragment_ops.blend.rgb_func_alpha_func.dst.one_minus_src_alpha_one_minus_src_color""")) + } + r = requests.post(getBaseUrl()+"/importTestSet/", files=files) + self.assertEqual(r.status_code, requests.codes.ok) + +if __name__ == '__main__': + # Make sure we're in cherry/test directory + os.chdir(os.path.dirname(__file__)) + + # Parse command-line arguments. + parser = argparse.ArgumentParser(description='Cherry Server Tests') + parser.add_argument('-p', '--port', help='port used for ExecServer communication', default='8086') + args = parser.parse_args() + + server_port = int(args.port) + serverAddress = ('localhost', server_port) + + # Take a temp copy of Cherry database file to execute tests with. + # \note Database contains a pre-generated data so that test cases can be built to assume existence of such data. + tmpDBName = '_tmp.db' + shutil.copy2('cherry-test.db', tmpDBName) + + # Spawn Cherry server. + server = subprocess.Popen(['./server', '--db=test/'+tmpDBName, '--port=%d' % server_port], cwd='../', stdin=None, stdout=sys.stdout, stderr=subprocess.STDOUT) + + try: + # Wait for server to come up + connected = False + while not connected: + try: + dummyConnection = socket.create_connection(serverAddress, 60.0) # Wait max a minute for connection + dummyConnection.close() + connected = True + except Exception as e: + time.sleep(0.2) + sys.stdout.flush() + + unittest.main(verbosity=2) + sys.stdout.flush() + time.sleep(2) # HACK: Wait for server stdout to clear. + finally: + try: + print "Killing child process." + server.terminate() + server.communicate() + server.wait() + # Remove tmp database. + os.remove(tmpDBName) + except Exception as e: + print "WARNING: unable to kill process: %s" % e + |