diff options
34 files changed, 1656 insertions, 1478 deletions
@@ -57,7 +57,6 @@ declare_args() { skia_generate_workarounds = false skia_lex = false - skia_skqp_enable_driver_correctness_workarounds = false skia_skqp_global_error_tolerance = 0 skia_llvm_path = "" @@ -1965,16 +1964,13 @@ if (skia_enable_tools) { if (!is_win) { test_lib("skqp_lib") { - public_include_dirs = [ "tools/skqp" ] + public_include_dirs = [ "tools/skqp/src" ] defines = [ "SK_SKQP_GLOBAL_ERROR_TOLERANCE=$skia_skqp_global_error_tolerance" ] - if (skia_skqp_enable_driver_correctness_workarounds) { - defines += [ "SK_SKQP_ENABLE_DRIVER_CORRECTNESS_WORKAROUNDS" ] - } sources = [ "dm/DMGpuTestProcs.cpp", - "tools/skqp/gm_knowledge.cpp", - "tools/skqp/gm_runner.cpp", + "tools/skqp/src/skqp.cpp", + "tools/skqp/src/skqp_model.cpp", ] deps = [ ":gm", @@ -1986,13 +1982,22 @@ if (skia_enable_tools) { } test_app("skqp") { sources = [ - "tools/skqp/skqp.cpp", + "tools/skqp/src/skqp_main.cpp", ] deps = [ ":skia", ":skqp_lib", ":tool_utils", - "//third_party/googletest", + ] + } + test_app("jitter_gms") { + sources = [ + "tools/skqp/jitter_gms.cpp", + ] + deps = [ + ":gm", + ":skia", + ":skqp_lib", ] } } @@ -2000,7 +2005,7 @@ if (skia_enable_tools) { test_app("skqp_app") { is_shared_library = true sources = [ - "tools/skqp/jni/org_skia_skqp_SkQPRunner.cpp", + "tools/skqp/src/jni_skqp.cpp", ] deps = [ ":skia", @@ -12,7 +12,6 @@ deps = { "third_party/externals/egl-registry" : "https://skia.googlesource.com/external/github.com/KhronosGroup/EGL-Registry@a0bca08de07c7d7651047bedc0b653cfaaa4f2ae", "third_party/externals/expat" : "https://android.googlesource.com/platform/external/expat.git@android-6.0.1_r55", "third_party/externals/freetype" : "https://skia.googlesource.com/third_party/freetype2.git@7edc937fe679d14d66f55cf6f7fa607925d38f3c", - "third_party/externals/googletest" : "https://android.googlesource.com/platform/external/googletest@dd43b9998e9a44a579a7aba6c1309407d1a5ed95", "third_party/externals/harfbuzz" : "https://skia.googlesource.com/third_party/harfbuzz.git@8be74d85534534dbdd39a0a6f496e26e9f3e661d", "third_party/externals/icu" : "https://chromium.googlesource.com/chromium/deps/icu.git@ec9c1133693148470ffe2e5e53576998e3650c1d", "third_party/externals/imgui" : "https://skia.googlesource.com/external/github.com/ocornut/imgui.git@bc6ac8b2aee0614debd940e45bc9cd0d9b355c86", diff --git a/platform_tools/android/apps/skqp/src/main/java/org/skia/skqp/SkQP.java b/platform_tools/android/apps/skqp/src/main/java/org/skia/skqp/SkQP.java index 58e09ec982..2f1381b0ab 100644 --- a/platform_tools/android/apps/skqp/src/main/java/org/skia/skqp/SkQP.java +++ b/platform_tools/android/apps/skqp/src/main/java/org/skia/skqp/SkQP.java @@ -14,8 +14,8 @@ import java.io.File; import java.io.IOException; public class SkQP { - protected native void nInit(AssetManager assetManager, String dataDir, boolean experimentalMode); - protected native float nExecuteGM(int gm, int backend) throws SkQPException; + protected native void nInit(AssetManager assetManager, String dataDir); + protected native long nExecuteGM(int gm, int backend) throws SkQPException; protected native String[] nExecuteUnitTest(int test); protected native void nMakeReport(); @@ -42,13 +42,13 @@ public class SkQP { // Note: nInit will initialize the mGMs, mBackends and mUnitTests fields. AssetManager assetManager = context.getResources().getAssets(); - this.nInit(assetManager, outputDirPath, true); + this.nInit(assetManager, outputDirPath); for (int backend = 0; backend < mBackends.length; backend++) { String classname = kSkiaGM + mBackends[backend]; for (int gm = 0; gm < mGMs.length; gm++) { String testName = kSkiaGM + mBackends[backend] + "_" +mGMs[gm]; - float value = java.lang.Float.MAX_VALUE; + long value = java.lang.Long.MAX_VALUE; String error = null; Log.w(LOG_PREFIX, "Running: " + testName); try { diff --git a/platform_tools/android/apps/skqp/src/main/java/org/skia/skqp/SkQPRunner.java b/platform_tools/android/apps/skqp/src/main/java/org/skia/skqp/SkQPRunner.java index 11a8e2bd6c..7827f3e88c 100644 --- a/platform_tools/android/apps/skqp/src/main/java/org/skia/skqp/SkQPRunner.java +++ b/platform_tools/android/apps/skqp/src/main/java/org/skia/skqp/SkQPRunner.java @@ -14,6 +14,8 @@ import android.support.test.InstrumentationRegistry; import android.util.Log; import java.io.File; import java.io.IOException; +import java.text.SimpleDateFormat; +import java.util.Date; import org.junit.runner.Description; import org.junit.runner.RunWith; import org.junit.runner.Runner; @@ -28,48 +30,40 @@ public class SkQPRunner extends Runner implements Filterable { private int mShouldRunTestCount; private Description[] mTests; private boolean[] mShouldSkipTest; - private SkQP impl; + private String mOutputDirectory; + private SkQP mImpl; private static final String TAG = SkQP.LOG_PREFIX; private static void Fail(Description desc, RunNotifier notifier, String failure) { notifier.fireTestFailure(new Failure(desc, new SkQPFailure(failure))); } - private static File GetOutputDir() { - Context c = InstrumentationRegistry.getTargetContext(); - // File f = c.getFilesDir(); - File f = c.getExternalFilesDir(null); - return new File(f, "output"); - } - //////////////////////////////////////////////////////////////////////////// public SkQPRunner(Class testClass) { - impl = new SkQP(); - File filesDir = SkQPRunner.GetOutputDir(); - try { - SkQP.ensureEmtpyDirectory(filesDir); - } catch (IOException e) { - Log.w(TAG, "ensureEmtpyDirectory: " + e.getMessage()); - } - Log.i(TAG, String.format("output written to \"%s\"", filesDir.getAbsolutePath())); + mImpl = new SkQP(); + Context context = InstrumentationRegistry.getTargetContext(); + String now = (new SimpleDateFormat("yyyy-MM-dd'T'HHmmss")).format(new Date()); + File reportPath = new File(context.getExternalFilesDir(null), "skqp_report_" + now); + reportPath.mkdirs(); + mOutputDirectory = reportPath.getAbsolutePath(); + Log.i(TAG, String.format("output written to \"%s\"", mOutputDirectory)); - Resources resources = InstrumentationRegistry.getTargetContext().getResources(); - AssetManager mAssetManager = resources.getAssets(); - impl.nInit(mAssetManager, filesDir.getAbsolutePath(), false); + AssetManager assetManager = context.getResources().getAssets(); + mImpl.nInit(assetManager, mOutputDirectory); mTests = new Description[this.testCount()]; mShouldSkipTest = new boolean[mTests.length]; // = {false, false, ....}; int index = 0; - for (int backend = 0; backend < impl.mBackends.length; backend++) { - for (int gm = 0; gm < impl.mGMs.length; gm++) { + for (int backend = 0; backend < mImpl.mBackends.length; backend++) { + for (int gm = 0; gm < mImpl.mGMs.length; gm++) { mTests[index++] = Description.createTestDescription(SkQPRunner.class, - impl.mBackends[backend] + "_" + impl.mGMs[gm]); + mImpl.mBackends[backend] + "_" + mImpl.mGMs[gm]); } } - for (int unitTest = 0; unitTest < impl.mUnitTests.length; unitTest++) { + for (int unitTest = 0; unitTest < mImpl.mUnitTests.length; unitTest++) { mTests[index++] = Description.createTestDescription(SkQPRunner.class, - "unitTest_" + impl.mUnitTests[unitTest]); + "unitTest_" + mImpl.mUnitTests[unitTest]); } assert(index == mTests.length); mShouldRunTestCount = mTests.length; @@ -101,15 +95,15 @@ public class SkQPRunner extends Runner implements Filterable { @Override public int testCount() { - return impl.mUnitTests.length + impl.mGMs.length * impl.mBackends.length; + return mImpl.mUnitTests.length + mImpl.mGMs.length * mImpl.mBackends.length; } @Override public void run(RunNotifier notifier) { int testNumber = 0; // out of number of actually run tests. int testIndex = 0; // out of potential tests. - for (int backend = 0; backend < impl.mBackends.length; backend++) { - for (int gm = 0; gm < impl.mGMs.length; gm++, testIndex++) { + for (int backend = 0; backend < mImpl.mBackends.length; backend++) { + for (int gm = 0; gm < mImpl.mGMs.length; gm++, testIndex++) { Description desc = mTests[testIndex]; String name = desc.getMethodName(); if (mShouldSkipTest[testIndex]) { @@ -117,10 +111,10 @@ public class SkQPRunner extends Runner implements Filterable { } ++testNumber; notifier.fireTestStarted(desc); - float value = java.lang.Float.MAX_VALUE; + long value = java.lang.Long.MAX_VALUE; String error = null; try { - value = impl.nExecuteGM(gm, backend); + value = mImpl.nExecuteGM(gm, backend); } catch (SkQPException exept) { error = exept.getMessage(); } @@ -131,8 +125,8 @@ public class SkQPRunner extends Runner implements Filterable { result = "ERROR"; } else if (value != 0) { SkQPRunner.Fail(desc, notifier, String.format( - "Image mismatch: max channel diff = %f", value)); - Log.w(TAG, String.format("[FAIL] '%s': %f > 0", name, value)); + "Image mismatch: max channel diff = %d", value)); + Log.w(TAG, String.format("[FAIL] '%s': %d > 0", name, value)); result = "FAIL"; } notifier.fireTestFinished(desc); @@ -140,7 +134,7 @@ public class SkQPRunner extends Runner implements Filterable { name, testNumber, mShouldRunTestCount, result)); } } - for (int unitTest = 0; unitTest < impl.mUnitTests.length; unitTest++, testIndex++) { + for (int unitTest = 0; unitTest < mImpl.mUnitTests.length; unitTest++, testIndex++) { Description desc = mTests[testIndex]; String name = desc.getMethodName(); if (mShouldSkipTest[testIndex]) { @@ -148,7 +142,7 @@ public class SkQPRunner extends Runner implements Filterable { } ++testNumber; notifier.fireTestStarted(desc); - String[] errors = impl.nExecuteUnitTest(unitTest); + String[] errors = mImpl.nExecuteUnitTest(unitTest); String result = "pass"; if (errors != null && errors.length > 0) { Log.w(TAG, String.format("[FAIL] Test '%s' had %d failures.", name, errors.length)); @@ -162,7 +156,7 @@ public class SkQPRunner extends Runner implements Filterable { Log.i(TAG, String.format("Test '%s' complete (%d/%d). [%s]", name, testNumber, mShouldRunTestCount, result)); } - impl.nMakeReport(); - Log.i(TAG, String.format("output written to \"%s\"", GetOutputDir().getAbsolutePath())); + mImpl.nMakeReport(); + Log.i(TAG, String.format("output written to \"%s\"", mOutputDirectory)); } } diff --git a/third_party/googletest/BUILD.gn b/third_party/googletest/BUILD.gn deleted file mode 100644 index 518360bdbe..0000000000 --- a/third_party/googletest/BUILD.gn +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright 2017 Google Inc. -# -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -import("../third_party.gni") - -if (!is_win) { - third_party("googletest") { - public_include_dirs = [ "../externals/googletest/googletest/include" ] - include_dirs = [ "../externals/googletest/googletest" ] - sources = [ - "../externals/googletest/googletest/src/gtest-all.cc", - ] - } -} diff --git a/tools/skqp/README_ALGORITHM.md b/tools/skqp/README_ALGORITHM.md new file mode 100644 index 0000000000..c1eb23bcf9 --- /dev/null +++ b/tools/skqp/README_ALGORITHM.md @@ -0,0 +1,96 @@ +SkQP Render Test Algorithm +========================== + +The following is a description of the render test validation algorithm that +will be used by the version of SkQP that will be released for Android Q-release. + +There is a global macro constant: `SK_SKQP_GLOBAL_ERROR_TOLERANCE`, which +reflects the `gn` variable `skia_skqp_global_error_tolerance`. This is usually +set to 8. + +First, look for a file named `skqp/rendertests.txt` in the +`platform_tools/android/apps/skqp/src/main/assets` directory. The format of +this file is: each line contains one render test name, followed by a comma, +followed by an integer. The integer is the `passing_threshold` for that test. + +For each test, we have a `max_image` and a `min_image`. These are PNG-encoded +images stored in SkQP's APK's asset directory (in the paths `gmkb/${TEST}/min.png` +and `gmkb/${TEST}/max.png`). + +The test input is a rendered image. This will be produced by running one of +the render tests against the either the `vk` (Vulkan) or `gles` (OpenGL ES) +Skia backend. + +Here is psuedocode for the error calculation: + + function calculate_pixel_error(pixel_value, pixel_max, pixel_min): + pixel_error = 0 + + for color_channel in { red, green, blue, alpha }: + value = get_color(pixel_value, color_channel) + v_max = get_color(pixel_max, color_channel) + v_min = get_color(pixel_min, color_channel) + + if value > v_max: + channel_error = value - v_max + elif value < v_min: + channel_error = v_min - value + else: + channel_error = 0 + pixel_error = max(pixel_error, channel_error) + + return max(0, pixel_error - SK_SKQP_GLOBAL_ERROR_TOLERANCE); + + function get_error(rendered_image, max_image, min_image): + assert(dimensions(rendered_image) == dimensions(max_image)) + assert(dimensions(rendered_image) == dimensions(min_image)) + + max_error = 0 + bad_pixels = 0 + total_error = 0 + + error_image = allocate_bitmap(dimensions(rendered_image)) + + for xy in list_all_pixel_coordinates(rendered_image): + pixel_error = calculate_pixel_error(rendered_image(xy), + max_image(xy), + min_image(xy)) + if pixel_error > 0: + for neighboring_xy in find_neighbors(xy): + if not inside(neighboring_xy, dimensions(rendered_image)): + continue + pixel_error = min(pixel_error, + calculate_pixel_error(rendered_image(xy), + max_image(neighboring_xy), + min_image(neighboring_xy))) + + if pixel_error > 0: + max_error = max(max_error, pixel_error) + bad_pixels += 1 + total_error += pixel_error + + error_image(xy) = linear_interpolation(black, red, pixel_error) + else: + error_image(xy) = white + + return ((total_error, max_error, bad_pixels), error_image) + +For each render test, there is a threshold value for `total_error`, : +`passing_threshold`. + +If `passing_threshold >= 0 && total_error > passing_threshold`, then the test +is a failure and is included in the report. if `passing_threshold == -1`, then +the test always passes, but we do execute the test to verify that the driver +does not crash. + +We generate a report with the following information for each test: + + backend_name,render_test_name,max_error,bad_pixels,total_error + +in CSV format in the file `out.csv`. A HTML report of just the failing tests +is written to the file `report.html`. This version includes four images for +each test: `rendered_image`, `max_image`, `min_image`, and `error_image`, as +well as the three metrics: `max_error`, `bad_pixels`, and `total_error`. + + + diff --git a/tools/skqp/README_GENERATING_MODELS.md b/tools/skqp/README_GENERATING_MODELS.md new file mode 100644 index 0000000000..67be5d316b --- /dev/null +++ b/tools/skqp/README_GENERATING_MODELS.md @@ -0,0 +1,71 @@ +How SkQP Generates Render Test Models +===================================== + +We will, at regular intervals, generate new models from the [master branch of +Skia][1]. Here is how that process works: + +1. Get the positively triaged results from Gold: + + Go to [Skia Gold's search][2] and search for results that are + + * Positive + * config=gles OR config=vk + * Span as many commits in the past as possible. + + Then go to "Actions" → "Export" and save the resulting `meta.json` file. + +2. From a checkout of Skia's master branch, execute: + + origin https://skia.googlesource.com/skia.git + git checkout origin/master + tools/skqp/cut_release META_JSON_FILE + + This will create the following files: + + platform_tools/android/apps/skqp/src/main/assets/files.checksum + platform_tools/android/apps/skqp/src/main/assets/skqp/rendertests.txt + platform_tools/android/apps/skqp/src/main/assets/skqp/unittests.txt + + These three files can be commited to Skia to create a new commit. Make + `origin/skqp/dev` a parent of this commit (without merging it in), and + push this new commit to `origin/skqp/dev`: + + git merge -s ours origin/skqp/dev -m "Cut SkQP $(date +%Y-%m-%d)" + git add \ + platform_tools/android/apps/skqp/src/main/assets/files.checksum \ + platform_tools/android/apps/skqp/src/main/assets/skqp/rendertests.txt \ + platform_tools/android/apps/skqp/src/main/assets/skqp/unittests.txt + git commit --amend --reuse-message=HEAD + git push origin HEAD:refs/for/skqp/dev + +`tools/skqp/cut_release` +------------------------ + +This tool will call `make_gmkb.go` to generate the `m{ax,in}.png` files for +each render test. Additionaly, a `models.txt` file enumerates all of the +models. + +Then it calls `jitter_gms` to see which render tests pass the jitter test. +`jitter_gms` respects the `bad_gms.txt` file by ignoring the render tests +enumerated in that file. Tests which pass the jitter test are enumerated in +the file `good.txt`, those that fail in the `bad.txt` file. + +Next, the `skqp/rendertests.txt` file is created. This file lists the render +tests that will be executed by SkQP. These are the union of the tests +enumerated in the `good.txt` and `bad.txt` files. If the render test is found +in the `models.txt` file and the `good.txt` file, its per-test threshold is set +to 0 (a later CL can manually change this, if needed). Otherwise, the +threshold is set to -1; this indicated that the rendertest will be executed (to +verify that the driver will not crash), but the output will not be compared +against the model. Unnecessary models will be removed. + +Next, all of the files that represent the models are uploaded to cloud storage. +A single checksum hash is kept in the `files.checksum` file. This is enough +to re-download those files later, but we don't have to fill the git repository +with a lot of binary data. + +Finally, a list of the current gpu unit tests is created and stored in +`skqp/unittests.txt`. + +[1]: https://skia.googlesource.com/skia/+log/master "Skia Master Branch" +[2]: https://gold.skia.org/search "Skia Gold Search" diff --git a/tools/skqp/bad_gms.txt b/tools/skqp/bad_gms.txt new file mode 100644 index 0000000000..dc570541be --- /dev/null +++ b/tools/skqp/bad_gms.txt @@ -0,0 +1,4 @@ +drawbitmaprect +drawbitmaprect-imagerect +p3 +skbug1719 diff --git a/tools/skqp/cut_release b/tools/skqp/cut_release new file mode 100755 index 0000000000..cc920720a9 --- /dev/null +++ b/tools/skqp/cut_release @@ -0,0 +1,31 @@ +#! /bin/sh +# Copyright 2018 Google LLC. +# Use of this source code is governed by a BSD-style license that can be found in the LICENSE file. + +if [ -z "$1" ]; then + echo "Usage: $0 META.JSON" >&2 + exit 1 +fi + +set -x +set -e +META_JSON="$1" +cd "$(dirname "$0")/../.." + +if [ -z "$SKQP_SKIP_INFRA_UPDATE" ]; then + go get -u go.skia.org/infra/golden/go/search +fi +go run tools/skqp/make_gmkb.go \ + "$META_JSON" \ + platform_tools/android/apps/skqp/src/main/assets/gmkb +env GIT_SYNC_DEPS_QUIET=1 python tools/git-sync-deps +O='out/ndebug' +mkdir -p $O +bin/gn gen $O --args='cc="clang" cxx="clang++" is_debug=false' +ninja -C $O jitter_gms list_gpu_unit_tests +$O/jitter_gms tools/skqp/bad_gms.txt +python tools/skqp/make_rendertests_list.py +rm 'bad.txt' 'good.txt' +sh tools/skqp/upload_model +$O/list_gpu_unit_tests \ + > platform_tools/android/apps/skqp/src/main/assets/skqp/unittests.txt diff --git a/tools/skqp/generate_gn_args b/tools/skqp/generate_gn_args index 60d9de10e9..1eb98fcf10 100755 --- a/tools/skqp/generate_gn_args +++ b/tools/skqp/generate_gn_args @@ -15,15 +15,15 @@ ndk = "{android_ndk_dir}" ndk_api = {api_level} skia_enable_fontmgr_empty = true skia_enable_pdf = false -skia_skqp_global_error_tolerance = 4 +skia_skqp_global_error_tolerance = 8 skia_use_dng_sdk = false skia_use_expat = false skia_use_icu = false skia_use_libheif = false skia_use_lua = false skia_use_piex = false -skia_skqp_enable_driver_correctness_workarounds = {enable_workarounds} skia_tools_require_resources = true +extra_cflags = [ "-DSK_ENABLE_DUMP_GPU" ] ''' def parse_args(): diff --git a/tools/skqp/gm_knowledge.cpp b/tools/skqp/gm_knowledge.cpp deleted file mode 100644 index 72dba2b4d4..0000000000 --- a/tools/skqp/gm_knowledge.cpp +++ /dev/null @@ -1,410 +0,0 @@ -/* - * Copyright 2017 Google Inc. - * - * Use of this source code is governed by a BSD-style license that can be - * found in the LICENSE file. - */ - -#include "gm_knowledge.h" - -#include <cfloat> -#include <cstdlib> -#include <fstream> -#include <mutex> -#include <sstream> -#include <string> -#include <vector> - -#include "../../src/core/SkStreamPriv.h" -#include "../../src/core/SkTSort.h" -#include "SkBitmap.h" -#include "SkCodec.h" -#include "SkOSFile.h" -#include "SkOSPath.h" -#include "SkPngEncoder.h" -#include "SkStream.h" - -#include "skqp_asset_manager.h" - -#define IMAGES_DIRECTORY_PATH "images" -#define PATH_MAX_PNG "max.png" -#define PATH_MIN_PNG "min.png" -#define PATH_IMG_PNG "image.png" -#define PATH_ERR_PNG "errors.png" -#define PATH_REPORT "report.html" -#define PATH_CSV "out.csv" - -#ifndef SK_SKQP_GLOBAL_ERROR_TOLERANCE -#define SK_SKQP_GLOBAL_ERROR_TOLERANCE 0 -#endif - -//////////////////////////////////////////////////////////////////////////////// - -static int get_error(uint32_t value, uint32_t value_max, uint32_t value_min) { - int error = 0; - for (int j : {0, 8, 16, 24}) { - uint8_t v = (value >> j) & 0xFF, - vmin = (value_min >> j) & 0xFF, - vmax = (value_max >> j) & 0xFF; - if (v > vmax) { - error = std::max(v - vmax, error); - } else if (v < vmin) { - error = std::max(vmin - v, error); - } - } - return std::max(0, error - SK_SKQP_GLOBAL_ERROR_TOLERANCE); -} - -static int get_error_with_nearby(int x, int y, const SkPixmap& pm, - const SkPixmap& pm_max, const SkPixmap& pm_min) { - struct NearbyPixels { - const int x, y, w, h; - struct Iter { - const int x, y, w, h; - int8_t curr; - SkIPoint operator*() const { return this->get(); } - SkIPoint get() const { - switch (curr) { - case 0: return {x - 1, y - 1}; - case 1: return {x , y - 1}; - case 2: return {x + 1, y - 1}; - case 3: return {x - 1, y }; - case 4: return {x + 1, y }; - case 5: return {x - 1, y + 1}; - case 6: return {x , y + 1}; - case 7: return {x + 1, y + 1}; - default: SkASSERT(false); return {0, 0}; - } - } - void skipBad() { - while (curr < 8) { - SkIPoint p = this->get(); - if (p.x() >= 0 && p.y() >= 0 && p.x() < w && p.y() < h) { - return; - } - ++curr; - } - curr = -1; - } - void operator++() { - if (-1 == curr) { return; } - ++curr; - this->skipBad(); - } - bool operator!=(const Iter& other) const { return curr != other.curr; } - }; - Iter begin() const { Iter i{x, y, w, h, 0}; i.skipBad(); return i; } - Iter end() const { return Iter{x, y, w, h, -1}; } - }; - - uint32_t c = *pm.addr32(x, y); - int error = get_error(c, *pm_max.addr32(x, y), *pm_min.addr32(x, y)); - for (SkIPoint p : NearbyPixels{x, y, pm.width(), pm.height()}) { - if (error == 0) { - return 0; - } - error = SkTMin(error, get_error( - c, *pm_max.addr32(p.x(), p.y()), *pm_min.addr32(p.x(), p.y()))); - } - return error; -} - -static float set_error_code(gmkb::Error* error_out, gmkb::Error error) { - SkASSERT(error != gmkb::Error::kNone); - if (error_out) { - *error_out = error; - } - return FLT_MAX; -} - -static bool WritePixmapToFile(const SkPixmap& pixmap, const char* path) { - SkFILEWStream wStream(path); - return wStream.isValid() && SkPngEncoder::Encode(&wStream, pixmap, SkPngEncoder::Options()); -} - -constexpr SkColorType kColorType = kRGBA_8888_SkColorType; -constexpr SkAlphaType kAlphaType = kUnpremul_SkAlphaType; - -static SkPixmap rgba8888_to_pixmap(const uint32_t* pixels, int width, int height) { - SkImageInfo info = SkImageInfo::Make(width, height, kColorType, kAlphaType); - return SkPixmap(info, pixels, width * sizeof(uint32_t)); -} - -static bool copy(skqp::AssetManager* mgr, const char* path, const char* dst) { - if (mgr) { - if (auto stream = mgr->open(path)) { - SkFILEWStream wStream(dst); - return wStream.isValid() && SkStreamCopy(&wStream, stream.get()); - } - } - return false; -} - -static SkBitmap ReadPngRgba8888FromFile(skqp::AssetManager* assetManager, const char* path) { - SkBitmap bitmap; - if (auto codec = SkCodec::MakeFromStream(assetManager->open(path))) { - SkISize size = codec->getInfo().dimensions(); - SkASSERT(!size.isEmpty()); - SkImageInfo info = SkImageInfo::Make(size.width(), size.height(), kColorType, kAlphaType); - bitmap.allocPixels(info); - SkASSERT(bitmap.rowBytes() == (unsigned)bitmap.width() * sizeof(uint32_t)); - if (SkCodec::kSuccess != codec->getPixels(bitmap.pixmap())) { - bitmap.reset(); - } - } - return bitmap; -} - -namespace { -struct Run { - SkString fBackend; - SkString fGM; - int fMaxerror; - int fBadpixels; -}; -} // namespace - -static std::vector<Run> gErrors; -static std::mutex gMutex; - -static SkString make_path(const SkString& images_directory, - const char* backend, - const char* gm_name, - const char* thing) { - auto path = SkStringPrintf("%s_%s_%s", backend, gm_name, thing); - return SkOSPath::Join(images_directory.c_str(), path.c_str()); -} - - -namespace gmkb { -float Check(const uint32_t* pixels, - int width, - int height, - const char* name, - const char* backend, - skqp::AssetManager* assetManager, - const char* report_directory_path, - Error* error_out) { - if (report_directory_path && report_directory_path[0]) { - SkASSERT_RELEASE(sk_isdir(report_directory_path)); - } - if (width <= 0 || height <= 0) { - return set_error_code(error_out, Error::kBadInput); - } - constexpr char PATH_ROOT[] = "gmkb"; - SkString img_path = SkOSPath::Join(PATH_ROOT, name); - SkString max_path = SkOSPath::Join(img_path.c_str(), PATH_MAX_PNG); - SkString min_path = SkOSPath::Join(img_path.c_str(), PATH_MIN_PNG); - SkBitmap max_image = ReadPngRgba8888FromFile(assetManager, max_path.c_str()); - SkBitmap min_image = ReadPngRgba8888FromFile(assetManager, min_path.c_str()); - if (max_image.isNull() || min_image.isNull()) { - // No data. - if (error_out) { - *error_out = Error::kNone; - } - return 0; - } - if (max_image.width() != min_image.width() || - max_image.height() != min_image.height()) - { - return set_error_code(error_out, Error::kBadData); - } - if (max_image.width() != width || max_image.height() != height) { - return set_error_code(error_out, Error::kBadInput); - } - - int badness = 0; - int badPixelCount = 0; - SkPixmap pm(SkImageInfo::Make(width, height, kColorType, kAlphaType), - pixels, width * sizeof(uint32_t)); - SkPixmap pm_max = max_image.pixmap(); - SkPixmap pm_min = min_image.pixmap(); - for (int y = 0; y < pm.height(); ++y) { - for (int x = 0; x < pm.width(); ++x) { - int error = get_error_with_nearby(x, y, pm, pm_max, pm_min) ; - if (error > 0) { - badness = SkTMax(error, badness); - ++badPixelCount; - } - } - } - - if (badness == 0) { - std::lock_guard<std::mutex> lock(gMutex); - gErrors.push_back(Run{SkString(backend), SkString(name), 0, 0}); - } - if (report_directory_path && badness > 0 && report_directory_path[0] != '\0') { - if (!backend) { - backend = "skia"; - } - SkString images_directory = SkOSPath::Join(report_directory_path, IMAGES_DIRECTORY_PATH); - sk_mkdir(images_directory.c_str()); - - SkString image_path = make_path(images_directory, backend, name, PATH_IMG_PNG); - SkString error_path = make_path(images_directory, backend, name, PATH_ERR_PNG); - SkString max_path_out = make_path(images_directory, backend, name, PATH_MAX_PNG); - SkString min_path_out = make_path(images_directory, backend, name, PATH_MIN_PNG); - - SkAssertResult(WritePixmapToFile(rgba8888_to_pixmap(pixels, width, height), - image_path.c_str())); - - SkBitmap errorBitmap; - errorBitmap.allocPixels(SkImageInfo::Make(width, height, kColorType, kAlphaType)); - for (int y = 0; y < pm.height(); ++y) { - for (int x = 0; x < pm.width(); ++x) { - int error = get_error_with_nearby(x, y, pm, pm_max, pm_min); - *errorBitmap.getAddr32(x, y) = - error > 0 ? 0xFF000000 + (unsigned)error : 0xFFFFFFFF; - } - } - SkAssertResult(WritePixmapToFile(errorBitmap.pixmap(), error_path.c_str())); - - (void)copy(assetManager, max_path.c_str(), max_path_out.c_str()); - (void)copy(assetManager, min_path.c_str(), min_path_out.c_str()); - - std::lock_guard<std::mutex> lock(gMutex); - gErrors.push_back(Run{SkString(backend), SkString(name), badness, badPixelCount}); - } - if (error_out) { - *error_out = Error::kNone; - } - return (float)badness; -} - -static constexpr char kDocHead[] = - "<!doctype html>\n" - "<html lang=\"en\">\n" - "<head>\n" - "<meta charset=\"UTF-8\">\n" - "<title>SkQP Report</title>\n" - "<style>\n" - "img { max-width:48%; border:1px green solid;\n" - " image-rendering: pixelated;\n" - " background-image:url('data:image/png;base64,iVBORw0KGgoA" - "AAANSUhEUgAAABAAAAAQCAAAAAA6mKC9AAAAAXNSR0IArs4c6QAAAAJiS0dEAP+H" - "j8y/AAAACXBIWXMAAA7DAAAOwwHHb6hkAAAAB3RJTUUH3gUBEi4DGRAQYgAAAB1J" - "REFUGNNjfMoAAVJQmokBDdBHgPE/lPFsYN0BABdaAwN6tehMAAAAAElFTkSuQmCC" - "'); }\n" - "</style>\n" - "<script>\n" - "function ce(t) { return document.createElement(t); }\n" - "function ct(n) { return document.createTextNode(n); }\n" - "function ac(u,v) { return u.appendChild(v); }\n" - "function br(u) { ac(u, ce(\"br\")); }\n" - "function ma(s, c) { var a = ce(\"a\"); a.href = s; ac(a, c); return a; }\n" - "function f(backend, gm, e1, e2) {\n" - " var b = ce(\"div\");\n" - " var x = ce(\"h2\");\n" - " var t = backend + \"_\" + gm;\n" - " ac(x, ct(t));\n" - " ac(b, x);\n" - " ac(b, ct(\"backend: \" + backend));\n" - " br(b);\n" - " ac(b, ct(\"gm name: \" + gm));\n" - " br(b);\n" - " ac(b, ct(\"maximum error: \" + e1));\n" - " br(b);\n" - " ac(b, ct(\"bad pixel counts: \" + e2));\n" - " br(b);\n" - " var q = \"" IMAGES_DIRECTORY_PATH "/\" + backend + \"_\" + gm + \"_\";\n" - " var i = ce(\"img\");\n" - " i.src = q + \"" PATH_IMG_PNG "\";\n" - " i.alt = \"img\";\n" - " ac(b, ma(i.src, i));\n" - " i = ce(\"img\");\n" - " i.src = q + \"" PATH_ERR_PNG "\";\n" - " i.alt = \"err\";\n" - " ac(b, ma(i.src, i));\n" - " br(b);\n" - " ac(b, ct(\"Expectation: \"));\n" - " ac(b, ma(q + \"" PATH_MAX_PNG "\", ct(\"max\")));\n" - " ac(b, ct(\" | \"));\n" - " ac(b, ma(q + \"" PATH_MIN_PNG "\", ct(\"min\")));\n" - " ac(b, ce(\"hr\"));\n" - " b.id = backend + \":\" + gm;\n" - " ac(document.body, b);\n" - " l = ce(\"li\");\n" - " ac(l, ct(\"[\" + e1 + \"] \"));\n" - " ac(l, ma(\"#\" + backend +\":\"+ gm , ct(t)));\n" - " ac(document.getElementById(\"toc\"), l);\n" - "}\n" - "function main() {\n"; - -static constexpr char kDocMiddle[] = - "}\n" - "</script>\n" - "</head>\n" - "<body onload=\"main()\">\n" - "<h1>SkQP Report</h1>\n"; - -static constexpr char kDocTail[] = - "<ul id=\"toc\"></ul>\n" - "<hr>\n" - "<p>Left image: test result<br>\n" - "Right image: errors (white = no error, black = smallest error, red = biggest error)</p>\n" - "<hr>\n" - "</body>\n" - "</html>\n"; - -static void write(SkWStream* wStream, const SkString& text) { - wStream->write(text.c_str(), text.size()); -} - -enum class Backend { - kUnknown, - kGLES, - kVulkan, -}; - -static Backend get_backend(const SkString& s) { - if (s.equals("gles")) { - return Backend::kGLES; - } else if (s.equals("vk")) { - return Backend::kVulkan; - } - return Backend::kUnknown; -} - - -bool MakeReport(const char* report_directory_path) { - int glesErrorCount = 0, vkErrorCount = 0, gles = 0, vk = 0; - - SkASSERT_RELEASE(sk_isdir(report_directory_path)); - std::lock_guard<std::mutex> lock(gMutex); - SkFILEWStream csvOut(SkOSPath::Join(report_directory_path, PATH_CSV).c_str()); - SkFILEWStream htmOut(SkOSPath::Join(report_directory_path, PATH_REPORT).c_str()); - SkASSERT_RELEASE(csvOut.isValid()); - if (!csvOut.isValid() || !htmOut.isValid()) { - return false; - } - htmOut.writeText(kDocHead); - for (const Run& run : gErrors) { - auto backend = get_backend(run.fBackend); - switch (backend) { - case Backend::kGLES: ++gles; break; - case Backend::kVulkan: ++vk; break; - default: break; - } - write(&csvOut, SkStringPrintf("\"%s\",\"%s\",%d,%d\n", - run.fBackend.c_str(), run.fGM.c_str(), - run.fMaxerror, run.fBadpixels)); - if (run.fMaxerror == 0 && run.fBadpixels == 0) { - continue; - } - write(&htmOut, SkStringPrintf(" f(\"%s\", \"%s\", %d, %d);\n", - run.fBackend.c_str(), run.fGM.c_str(), - run.fMaxerror, run.fBadpixels)); - switch (backend) { - case Backend::kGLES: ++glesErrorCount; break; - case Backend::kVulkan: ++vkErrorCount; break; - default: break; - } - } - htmOut.writeText(kDocMiddle); - write(&htmOut, SkStringPrintf("<p>gles errors: %d (of %d)</br>\n" - "vk errors: %d (of %d)</p>\n", - glesErrorCount, gles, vkErrorCount, vk)); - htmOut.writeText(kDocTail); - return true; -} -} // namespace gmkb diff --git a/tools/skqp/gm_knowledge.h b/tools/skqp/gm_knowledge.h deleted file mode 100644 index 6a19a87034..0000000000 --- a/tools/skqp/gm_knowledge.h +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright 2017 Google Inc. - * - * Use of this source code is governed by a BSD-style license that can be - * found in the LICENSE file. - */ -#ifndef gm_knowledge_DEFINED -#define gm_knowledge_DEFINED - -#include <cstdint> - -namespace skqp { -class AssetManager; -} - -namespace gmkb { - -enum class Error { - kNone, /**< No error. */ - kBadInput, /**< Error with the given image data. */ - kBadData, /**< Error with the given gmkb data directory. */ -}; - -/** -Check if the given test image matches the expected results. - -Each pixel is an un-pre-multiplied RGBA color: - uint32_t make_color(uint8_t r, uint8_t g, uint8_t b, uint8_t a) { - return (r << 0) | (g << 8) | (b << 16) | (a << 24); - } - -The image's rowBytes is width*sizeof(uint32_t): - uint32_t* get_pixel_addr(uint32_t* pixels, int width, int height, int x, int y) { - assert(x >= 0 && x < width); - assert(y >= 0 && y < height); - return &pixels[x + (width * y)]; - } - -@param pixels, width, height the image -@param gm_name the name of the rendering test that produced the image -@param backend (optional) name of the backend -@param asset_manager GM KnowledgeBase data files -@param report_directory_path (optional) locatation to write report to. -@param error_out (optional) error return code. - -@return 0 if the test passes, otherwise a positive number representing how - badly it failed. Return FLT_MAX on error. - */ - -float Check(const uint32_t* pixels, - int width, - int height, - const char* name, - const char* backend, - skqp::AssetManager* asset_manager, - const char* report_directory_path, - Error* error_out); - -/** -Call this after running all checks. - -@param report_directory_path locatation to write report to. -*/ -bool MakeReport(const char* report_directory_path); -} // namespace gmkb - -#endif // gm_knowledge_DEFINED diff --git a/tools/skqp/gm_runner.cpp b/tools/skqp/gm_runner.cpp deleted file mode 100644 index 0cd031c26c..0000000000 --- a/tools/skqp/gm_runner.cpp +++ /dev/null @@ -1,280 +0,0 @@ -/* - * Copyright 2017 Google Inc. - * - * Use of this source code is governed by a BSD-style license that can be - * found in the LICENSE file. - */ - -#include "gm_runner.h" - -#include <algorithm> - -#include "../tools/fonts/SkTestFontMgr.h" -#include "GrContext.h" -#include "GrContextOptions.h" -#include "SkFontMgrPriv.h" -#include "SkFontStyle.h" -#include "SkGraphics.h" -#include "SkImageInfoPriv.h" -#include "SkSurface.h" -#include "Test.h" -#include "gl/GLTestContext.h" -#include "gm.h" -#include "gm_knowledge.h" -#include "vk/VkTestContext.h" - -static SkTHashSet<SkString> gDoNotScoreInCompatibilityTestMode; -static SkTHashSet<SkString> gDoNotExecuteInExperimentalMode; -static SkTHashSet<SkString> gKnownGpuUnitTests; -static SkTHashSet<SkString> gKnownGMs; -static gm_runner::Mode gMode = gm_runner::Mode::kCompatibilityTestMode; - -static bool is_empty(const SkTHashSet<SkString>& set) { - return 0 == set.count(); -} -static bool in_set(const char* s, const SkTHashSet<SkString>& set) { - return !is_empty(set) && nullptr != set.find(SkString(s)); -} - -static void readlist(skqp::AssetManager* mgr, const char* path, SkTHashSet<SkString>* dst) { - auto asset = mgr->open(path); - if (!asset || asset->getLength() == 0) { - return; // missing file same as empty file. - } - std::vector<char> buffer(asset->getLength() + 1); - asset->read(buffer.data(), buffer.size()); - buffer.back() = '\0'; - const char* ptr = buffer.data(); - const char* end = &buffer.back(); - SkASSERT(ptr < end); - while (true) { - while (*ptr == '\n' && ptr < end) { - ++ptr; - } - if (ptr == end) { - return; - } - const char* find = strchr(ptr, '\n'); - if (!find) { - find = end; - } - SkASSERT(find > ptr); - dst->add(SkString(ptr, find - ptr)); - ptr = find; - } -} - -namespace gm_runner { - -const char* GetErrorString(Error e) { - switch (e) { - case Error::None: return ""; - case Error::BadSkiaOutput: return "Bad Skia Output"; - case Error::BadGMKBData: return "Bad GMKB Data"; - case Error::SkiaFailure: return "Skia Failure"; - default: SkASSERT(false); - return "unknown"; - } -} - -std::vector<std::string> ExecuteTest(UnitTest test) { - struct : public skiatest::Reporter { - std::vector<std::string> fErrors; - void reportFailed(const skiatest::Failure& failure) override { - SkString desc = failure.toString(); - fErrors.push_back(std::string(desc.c_str(), desc.size())); - } - } r; - GrContextOptions options; - // options.fDisableDriverCorrectnessWorkarounds = true; - if (test->fContextOptionsProc) { - test->fContextOptionsProc(&options); - } - test->proc(&r, options); - return std::move(r.fErrors); -} - -const char* GetUnitTestName(UnitTest test) { return test->name; } - -std::vector<UnitTest> GetUnitTests() { - std::vector<UnitTest> tests; - for (const skiatest::Test& test : skiatest::TestRegistry::Range()) { - if ((is_empty(gKnownGpuUnitTests) || in_set(test.name, gKnownGpuUnitTests)) - && test.needsGpu) { - tests.push_back(&test); - } - } - struct { - bool operator()(UnitTest u, UnitTest v) const { return strcmp(u->name, v->name) < 0; } - } less; - std::sort(tests.begin(), tests.end(), less); - return tests; -} - -const char* GetBackendName(SkiaBackend backend) { - switch (backend) { - case SkiaBackend::kGL: return "gl"; - case SkiaBackend::kGLES: return "gles"; - case SkiaBackend::kVulkan: return "vk"; - default: SkASSERT(false); - return "error"; - } -} - -static std::unique_ptr<sk_gpu_test::TestContext> make_test_context(SkiaBackend backend) { - using U = std::unique_ptr<sk_gpu_test::TestContext>; - switch (backend) { - case SkiaBackend::kGL: - return U(sk_gpu_test::CreatePlatformGLTestContext(kGL_GrGLStandard, nullptr)); - case SkiaBackend::kGLES: - return U(sk_gpu_test::CreatePlatformGLTestContext(kGLES_GrGLStandard, nullptr)); -#ifdef SK_VULKAN - case SkiaBackend::kVulkan: - return U(sk_gpu_test::CreatePlatformVkTestContext(nullptr)); -#endif - default: - return nullptr; - } -} - -static GrContextOptions context_options(skiagm::GM* gm = nullptr) { - GrContextOptions grContextOptions; - grContextOptions.fAllowPathMaskCaching = true; - grContextOptions.fSuppressPathRendering = true; - #ifndef SK_SKQP_ENABLE_DRIVER_CORRECTNESS_WORKAROUNDS - grContextOptions.fDisableDriverCorrectnessWorkarounds = true; - #endif - if (gm) { - gm->modifyGrContextOptions(&grContextOptions); - } - return grContextOptions; -} - -std::vector<SkiaBackend> GetSupportedBackends() { - std::vector<SkiaBackend> result; - SkiaBackend backends[] = { - #ifndef SK_BUILD_FOR_ANDROID - SkiaBackend::kGL, // Used for testing on desktop machines. - #endif - SkiaBackend::kGLES, - SkiaBackend::kVulkan, - }; - for (SkiaBackend backend : backends) { - std::unique_ptr<sk_gpu_test::TestContext> testCtx = make_test_context(backend); - if (testCtx) { - testCtx->makeCurrent(); - if (nullptr != testCtx->makeGrContext(context_options())) { - result.push_back(backend); - } - } - } - SkASSERT_RELEASE(result.size() > 0); - return result; -} - -static bool evaluate_gm(SkiaBackend backend, - skiagm::GM* gm, - int* width, - int* height, - std::vector<uint32_t>* storage) { - constexpr SkColorType ct = kRGBA_8888_SkColorType; - SkASSERT(storage); - SkASSERT(gm); - SkASSERT(width); - SkASSERT(height); - SkISize size = gm->getISize(); - int w = size.width(), - h = size.height(); - *width = w; - *height = h; - SkImageInfo info = SkImageInfo::Make(w, h, ct, kPremul_SkAlphaType, nullptr); - SkSurfaceProps props(0, SkSurfaceProps::kLegacyFontHost_InitType); - - std::unique_ptr<sk_gpu_test::TestContext> testCtx = make_test_context(backend); - if (!testCtx) { - return false; - } - testCtx->makeCurrent(); - sk_sp<SkSurface> surf = SkSurface::MakeRenderTarget( - testCtx->makeGrContext(context_options(gm)).get(), SkBudgeted::kNo, info, 0, &props); - if (!surf) { - return false; - } - gm->draw(surf->getCanvas()); - - storage->resize(w * h); - uint32_t* pix = storage->data(); - size_t rb = w * sizeof(uint32_t); - SkASSERT(SkColorTypeBytesPerPixel(ct) == sizeof(uint32_t)); - if (!surf->readPixels(SkImageInfo::Make(w, h, ct, kUnpremul_SkAlphaType), pix, rb, 0, 0)) { - storage->resize(0); - return false; - } - return true; -} - -std::tuple<float, Error> EvaluateGM(SkiaBackend backend, - GMFactory gmFact, - skqp::AssetManager* assetManager, - const char* reportDirectoryPath) { - std::vector<uint32_t> pixels; - SkASSERT(gmFact); - std::unique_ptr<skiagm::GM> gm(gmFact(nullptr)); - SkASSERT(gm); - const char* name = gm->getName(); - int width = 0, height = 0; - if (!evaluate_gm(backend, gm.get(), &width, &height, &pixels)) { - return std::make_tuple(FLT_MAX, Error::SkiaFailure); - } - if (Mode::kCompatibilityTestMode == gMode && in_set(name, gDoNotScoreInCompatibilityTestMode)) { - return std::make_tuple(0, Error::None); - } - - gmkb::Error e; - float value = gmkb::Check(pixels.data(), width, height, - name, GetBackendName(backend), assetManager, - reportDirectoryPath, &e); - Error error = gmkb::Error::kBadInput == e ? Error::BadSkiaOutput - : gmkb::Error::kBadData == e ? Error::BadGMKBData - : Error::None; - return std::make_tuple(value, error); -} - -void InitSkia(Mode mode, skqp::AssetManager* mgr) { - SkGraphics::Init(); - gSkFontMgr_DefaultFactory = &sk_tool_utils::MakePortableFontMgr; - - gMode = mode; - readlist(mgr, "skqp/DoNotScoreInCompatibilityTestMode.txt", - &gDoNotScoreInCompatibilityTestMode); - readlist(mgr, "skqp/DoNotExecuteInExperimentalMode.txt", &gDoNotExecuteInExperimentalMode); - readlist(mgr, "skqp/KnownGpuUnitTests.txt", &gKnownGpuUnitTests); - readlist(mgr, "skqp/KnownGMs.txt", &gKnownGMs); -} - -std::vector<GMFactory> GetGMFactories(skqp::AssetManager* assetManager) { - std::vector<GMFactory> result; - for (const GMFactory& f : skiagm::GMRegistry::Range()) { - SkASSERT(f); - auto name = GetGMName(f); - if ((is_empty(gKnownGMs) || in_set(name.c_str(), gKnownGMs)) && - !(Mode::kExperimentalMode == gMode && - in_set(name.c_str(), gDoNotExecuteInExperimentalMode))) - { - result.push_back(f); - } - } - struct { - bool operator()(GMFactory u, GMFactory v) const { return GetGMName(u) < GetGMName(v); } - } less; - std::sort(result.begin(), result.end(), less); - return result; -} - -std::string GetGMName(GMFactory gmFactory) { - SkASSERT(gmFactory); - std::unique_ptr<skiagm::GM> gm(gmFactory(nullptr)); - SkASSERT(gm); - return std::string(gm->getName()); -} -} // namespace gm_runner diff --git a/tools/skqp/gm_runner.h b/tools/skqp/gm_runner.h deleted file mode 100644 index 2707966c9c..0000000000 --- a/tools/skqp/gm_runner.h +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright 2017 Google Inc. - * - * Use of this source code is governed by a BSD-style license that can be - * found in the LICENSE file. - */ -#ifndef gm_runner_DEFINED -#define gm_runner_DEFINED - -#include <memory> -#include <string> -#include <tuple> -#include <vector> - -#include "skqp_asset_manager.h" - -/** -A Skia GM is a single rendering test that can be executed on any Skia backend Canvas. -*/ -namespace skiagm { -class GM; -} - -namespace skiatest { -struct Test; -} - -namespace gm_runner { - -using GMFactory = skiagm::GM* (*)(void*); - -using UnitTest = const skiatest::Test*; - -enum class SkiaBackend { - kGL, - kGLES, - kVulkan, -}; - -enum class Mode { - /** This mode is set when used by Android CTS. All known tests are executed. */ - kCompatibilityTestMode, - /** This mode is set when used in the test lab. Some tests are skipped, if - they are known to cause crashes in older devices. All GMs are evaluated - with stricter requirements. */ - kExperimentalMode, - -}; - -/** -Initialize Skia -*/ -void InitSkia(Mode, skqp::AssetManager*); - -std::vector<SkiaBackend> GetSupportedBackends(); - -/** -@return a list of all Skia GMs in lexicographic order. -*/ -std::vector<GMFactory> GetGMFactories(skqp::AssetManager*); - -/** -@return a list of all Skia GPU unit tests in lexicographic order. -*/ -std::vector<UnitTest> GetUnitTests(); - -/** -@return a descriptive name for the GM. -*/ -std::string GetGMName(GMFactory); - -/** -@return a descriptive name for the unit test. -*/ -const char* GetUnitTestName(UnitTest); - -/** -@return a descriptive name for the backend. -*/ -const char* GetBackendName(SkiaBackend); - -enum class Error { - None = 0, - BadSkiaOutput = 1, - BadGMKBData = 2, - SkiaFailure = 3, -}; - -const char* GetErrorString(Error); - -/** -@return A non-negative float representing how badly the GM failed (or zero for - success). Any error running or evaluating the GM will result in a non-zero - error code. -*/ -std::tuple<float, Error> EvaluateGM(SkiaBackend backend, - GMFactory gmFact, - skqp::AssetManager* assetManager, - const char* reportDirectoryPath); - -/** -@return a (hopefully empty) list of errors produced by this unit test. -*/ -std::vector<std::string> ExecuteTest(UnitTest); - -} // namespace gm_runner - -#endif // gm_runner_DEFINED diff --git a/tools/skqp/gn_to_bp.py b/tools/skqp/gn_to_bp.py index 530d2b5512..02bd638c4b 100644 --- a/tools/skqp/gn_to_bp.py +++ b/tools/skqp/gn_to_bp.py @@ -121,7 +121,7 @@ gn_args = { # setup skqp 'is_debug': 'false', 'ndk_api': '26', - 'skia_skqp_global_error_tolerance': '4', + 'skia_skqp_global_error_tolerance': '8', # setup vulkan 'skia_use_vulkan': 'true', diff --git a/tools/skqp/inflate.py b/tools/skqp/inflate.py deleted file mode 100755 index 97cc3a405d..0000000000 --- a/tools/skqp/inflate.py +++ /dev/null @@ -1,8 +0,0 @@ -#! /usr/bin/env python2 -# Copyright 2017 Google Inc. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. -import sys -import zlib -sys.stdout.write(zlib.decompress(sys.stdin.read())) - diff --git a/tools/skqp/jitter_gms.cpp b/tools/skqp/jitter_gms.cpp new file mode 100644 index 0000000000..96804712fe --- /dev/null +++ b/tools/skqp/jitter_gms.cpp @@ -0,0 +1,145 @@ +// Copyright 2018 Google LLC. +// Use of this source code is governed by a BSD-style license that can be found in the LICENSE file. + +// Jitter GMs +// +// Re-execute rendering tests with slight translational changes and see if +// there is a significant change. Print `1` if the named test has no +// significant change, `0` otherwise + +#include "gm.h" +#include "SkGraphics.h" +#include "SkExecutor.h" +#include "SkSemaphore.h" + +#include "skqp_model.h" + +#include <algorithm> +#include <cstdio> +#include <fstream> +#include <iostream> +#include <mutex> +#include <string> +#include <vector> + +// Error tolerance distance in 8888 color space with Manhattan metric on color channel. +static constexpr uint8_t kSkiaSkqpGlobalErrorTolerance = 8; + +// Number of times to jitter the canvas. +static constexpr int kNumberOfJitters = 7; + +// Distance to translate the canvas in each jitter (direction will be different each time). +static constexpr float kJitterMagnitude = 0.03125f; + +// The `kNumberOfJitters` different runs will each go in a different direction. +// this is the angle (in radians) for the first one. +static constexpr float kPhase = 0.3f; + +static void do_gm(SkBitmap* bm, skiagm::GM* gm, SkPoint jitter) { + SkASSERT(bm); + SkASSERT(gm); + SkASSERT(bm->dimensions() == gm->getISize()); + SkCanvas canvas(*bm); + SkAutoCanvasRestore autoCanvasRestore(&canvas, true); + canvas.clear(SK_ColorWHITE); + canvas.translate(jitter.x(), jitter.y()); + gm->draw(&canvas); + canvas.flush(); +} + +// Return true if passes jitter test. +static bool test_jitter(skiagm::GM* gm) { + SkASSERT(gm); + SkISize size = gm->getISize(); + SkBitmap control, experimental; + control.allocN32Pixels(size.width(), size.height()); + experimental.allocN32Pixels(size.width(), size.height()); + do_gm(&control, gm, {0, 0}); + for (int i = 0; i < kNumberOfJitters; ++i) { + float angle = i * (6.2831853f / kNumberOfJitters) + kPhase; + do_gm(&experimental, gm, SkPoint{kJitterMagnitude * cosf(angle), + kJitterMagnitude * sinf(angle)}); + SkQP::RenderOutcome result = skqp::Check( + control.pixmap(), control.pixmap(), experimental.pixmap(), + kSkiaSkqpGlobalErrorTolerance, nullptr); + if (result.fTotalError > 0) { + return false; + } + } + return true; +} + +static bool do_this_test(const char* name, + const std::vector<std::string>& doNotRun, + const std::vector<std::string>& testOnlyThese) { + for (const std::string& bad : doNotRun) { + if (bad == name) { + return false; + } + } + for (const std::string& good : testOnlyThese) { + if (good == name) { + return true; + } + } + return testOnlyThese.empty(); +} + + +int main(int argc, char** argv) { + std::vector<std::string> doNotRun; + std::vector<std::string> testOnlyThese; + if (argc > 1) { + std::ifstream ifs(argv[1]); + if (ifs.is_open()) { + std::string str; + while (std::getline(ifs, str)) { + doNotRun.push_back(str); + } + } + } + if (argc > 2) { + for (int i = 2; i < argc; ++i) { + testOnlyThese.emplace_back(argv[i]); + } + } + SkGraphics::Init(); + std::mutex mutex; + std::vector<std::string> goodResults; + std::vector<std::string> badResults; + + int total = 0; + SkSemaphore semaphore; + auto executor = SkExecutor::MakeFIFOThreadPool(); + for (skiagm::GMFactory factory : skiagm::GMRegistry::Range()) { + ++total; + executor->add([factory, &mutex, &goodResults, &badResults, + &semaphore, &doNotRun, &testOnlyThese](){ + std::unique_ptr<skiagm::GM> gm(factory(nullptr)); + const char* name = gm->getName(); + if (do_this_test(name, doNotRun, testOnlyThese)) { + bool success = test_jitter(gm.get()); + std::lock_guard<std::mutex> lock(mutex); + if (success) { + goodResults.emplace_back(name); + } else { + badResults.emplace_back(name); + } + fputc('.', stderr); + fflush(stderr); + } + semaphore.signal(); + }); + } + while (total-- > 0) { semaphore.wait(); } + fputc('\n', stderr); + fflush(stderr); + std::sort(goodResults.begin(), goodResults.end()); + std::sort(badResults.begin(), badResults.end()); + std::ofstream good("good.txt"); + std::ofstream bad("bad.txt"); + for (const std::string& s : goodResults) { good << s << '\n'; } + for (const std::string& s : badResults) { bad << s << '\n'; } + fprintf(stderr, "good = %u\nbad = %u\n\n", + (unsigned)goodResults.size(), (unsigned)badResults.size()); +} diff --git a/tools/skqp/jni/org_skia_skqp_SkQPRunner.cpp b/tools/skqp/jni/org_skia_skqp_SkQPRunner.cpp deleted file mode 100644 index 4eb01d4fac..0000000000 --- a/tools/skqp/jni/org_skia_skqp_SkQPRunner.cpp +++ /dev/null @@ -1,242 +0,0 @@ -/* - * Copyright 2017 Google Inc. - * - * Use of this source code is governed by a BSD-style license that can be - * found in the LICENSE file. - */ - -#include <mutex> -#include <vector> - -#include <android/asset_manager.h> -#include <android/asset_manager_jni.h> -#include <jni.h> -#include <sys/stat.h> - -#include "ResourceFactory.h" -#include "SkOSPath.h" -#include "SkStream.h" -#include "SkTo.h" -#include "gm_knowledge.h" -#include "gm_runner.h" -#include "skqp_asset_manager.h" - -//////////////////////////////////////////////////////////////////////////////// -extern "C" { -JNIEXPORT void JNICALL Java_org_skia_skqp_SkQP_nInit(JNIEnv*, jobject, jobject, jstring, jboolean); -JNIEXPORT jfloat JNICALL Java_org_skia_skqp_SkQP_nExecuteGM(JNIEnv*, jobject, jint, jint); -JNIEXPORT jobjectArray JNICALL Java_org_skia_skqp_SkQP_nExecuteUnitTest(JNIEnv*, jobject, - jint); -JNIEXPORT void JNICALL Java_org_skia_skqp_SkQP_nMakeReport(JNIEnv*, jobject); -} // extern "C" -//////////////////////////////////////////////////////////////////////////////// - -namespace { -struct AndroidAssetManager : public skqp::AssetManager { - AAssetManager* fMgr = nullptr; - std::unique_ptr<SkStreamAsset> open(const char* path) override { - struct AAStrm : public SkStreamAsset { - AAssetManager* fMgr; - std::string fPath; - AAsset* fAsset; - AAStrm(AAssetManager* m, std::string p, AAsset* a) - : fMgr(m), fPath(std::move(p)), fAsset(a) {} - ~AAStrm() override { AAsset_close(fAsset); } - size_t read(void* buffer, size_t size) override { - size_t r = SkTMin(size, SkToSizeT(AAsset_getRemainingLength(fAsset))); - if (buffer) { - return SkToSizeT(AAsset_read(fAsset, buffer, r)); - } else { - this->move(SkTo<long>(r)); - return r; - } - } - size_t getLength() const override { return SkToSizeT(AAsset_getLength(fAsset)); } - size_t peek(void* buffer, size_t size) const override { - size_t r = const_cast<AAStrm*>(this)->read(buffer, size); - const_cast<AAStrm*>(this)->move(-(long)r); - return r; - } - bool isAtEnd() const override { return 0 == AAsset_getRemainingLength(fAsset); } - bool rewind() override { return this->seek(0); } - size_t getPosition() const override { - return SkToSizeT(AAsset_seek(fAsset, 0, SEEK_CUR)); - } - bool seek(size_t position) override { - return -1 != AAsset_seek(fAsset, SkTo<off_t>(position), SEEK_SET); - } - bool move(long offset) override { - return -1 != AAsset_seek(fAsset, SkTo<off_t>(offset), SEEK_CUR); - } - SkStreamAsset* onDuplicate() const override { - AAsset* dupAsset = AndroidAssetManager::OpenAsset(fMgr, fPath.c_str()); - return dupAsset ? new AAStrm(fMgr, fPath, dupAsset) : nullptr; - } - SkStreamAsset* onFork() const override { - SkStreamAsset* dup = this->onDuplicate(); - if (dup) { (void)dup->seek(this->getPosition()); } - return dup; - } - }; - // SkDebugf("AndroidAssetManager::open(\"%s\");", path); - AAsset* asset = AndroidAssetManager::OpenAsset(fMgr, path); - return asset ? std::unique_ptr<SkStreamAsset>(new AAStrm(fMgr, std::string(path), asset)) - : nullptr; - } - static AAsset* OpenAsset(AAssetManager* mgr, const char* path) { - return mgr ? AAssetManager_open(mgr, path, AASSET_MODE_STREAMING) : nullptr; - } -}; -} - -static void set_string_array_element(JNIEnv* env, jobjectArray a, const char* s, unsigned i) { - jstring jstr = env->NewStringUTF(s); - env->SetObjectArrayElement(a, (jsize)i, jstr); - env->DeleteLocalRef(jstr); -} - -#define jassert(env, cond) do { if (!(cond)) { \ - (env)->ThrowNew((env)->FindClass("java/lang/Exception"), \ - __FILE__ ": assert(" #cond ") failed."); } } while (0) - -//////////////////////////////////////////////////////////////////////////////// - -static std::mutex gMutex; -static std::vector<gm_runner::SkiaBackend> gBackends; -static std::vector<gm_runner::GMFactory> gGMs; -static std::vector<gm_runner::UnitTest> gUnitTests; -static AndroidAssetManager gAssetManager; -static std::string gReportDirectory; -static jclass gStringClass = nullptr; - -//////////////////////////////////////////////////////////////////////////////// - -sk_sp<SkData> get_resource(const char* resource) { - AAssetManager* mgr = gAssetManager.fMgr; - if (!mgr) { - return nullptr; - } - SkString path = SkOSPath::Join("resources", resource); - AAsset* asset = AAssetManager_open(mgr, path.c_str(), AASSET_MODE_STREAMING); - if (!asset) { - return nullptr; - } - size_t size = SkToSizeT(AAsset_getLength(asset)); - sk_sp<SkData> data = SkData::MakeUninitialized(size); - (void)AAsset_read(asset, data->writable_data(), size); - AAsset_close(asset); - return data; -} - -//////////////////////////////////////////////////////////////////////////////// - -template <typename T, typename F> -jobjectArray to_java_string_array(JNIEnv* env, - const std::vector<T>& array, - F toString) { - jobjectArray jarray = env->NewObjectArray((jint)array.size(), gStringClass, nullptr); - for (unsigned i = 0; i < array.size(); ++i) { - set_string_array_element(env, jarray, std::string(toString(array[i])).c_str(), i); - } - return jarray; -} - -void Java_org_skia_skqp_SkQP_nInit(JNIEnv* env, jobject object, jobject assetManager, - jstring dataDir, jboolean experimentalMode) { - jclass clazz = env->GetObjectClass(object); - jassert(env, assetManager); - - std::lock_guard<std::mutex> lock(gMutex); - gAssetManager.fMgr = AAssetManager_fromJava(env, assetManager); - jassert(env, gAssetManager.fMgr); - - gm_runner::InitSkia(experimentalMode ? gm_runner::Mode::kExperimentalMode - : gm_runner::Mode::kCompatibilityTestMode, - &gAssetManager); - gResourceFactory = &get_resource; - - const char* dataDirString = env->GetStringUTFChars(dataDir, nullptr); - jassert(env, dataDirString && dataDirString[0]); - gReportDirectory = std::string(dataDirString) + "/skqp_report"; - int mkdirRetval = mkdir(gReportDirectory.c_str(), 0777); - SkASSERT_RELEASE(0 == mkdirRetval); - - env->ReleaseStringUTFChars(dataDir, dataDirString); - - gBackends = gm_runner::GetSupportedBackends(); - jassert(env, gBackends.size() > 0); - gGMs = gm_runner::GetGMFactories(&gAssetManager); - jassert(env, gGMs.size() > 0); - gUnitTests = gm_runner::GetUnitTests(); - jassert(env, gUnitTests.size() > 0); - gStringClass = env->FindClass("java/lang/String"); - jassert(env, gStringClass); - - constexpr char stringArrayType[] = "[Ljava/lang/String;"; - env->SetObjectField(object, env->GetFieldID(clazz, "mBackends", stringArrayType), - to_java_string_array(env, gBackends, gm_runner::GetBackendName)); - env->SetObjectField(object, env->GetFieldID(clazz, "mUnitTests", stringArrayType), - to_java_string_array(env, gUnitTests, gm_runner::GetUnitTestName)); - env->SetObjectField(object, env->GetFieldID(clazz, "mGMs", stringArrayType), - to_java_string_array(env, gGMs, gm_runner::GetGMName)); -} - -jfloat Java_org_skia_skqp_SkQP_nExecuteGM(JNIEnv* env, - jobject object, - jint gmIndex, - jint backendIndex) { - jassert(env, gmIndex < (jint)gGMs.size()); - jassert(env, backendIndex < (jint)gBackends.size()); - gm_runner::GMFactory gm; - gm_runner::SkiaBackend backend; - std::string reportDirectoryPath; - { - std::lock_guard<std::mutex> lock(gMutex); - backend = gBackends[backendIndex]; - gm = gGMs[gmIndex]; - reportDirectoryPath = gReportDirectory; - } - float result; - gm_runner::Error error; - std::tie(result, error) = gm_runner::EvaluateGM(backend, gm, &gAssetManager, - reportDirectoryPath.c_str()); - if (error != gm_runner::Error::None) { - (void)env->ThrowNew(env->FindClass("org/skia/skqp/SkQPException"), - gm_runner::GetErrorString(error)); - } - return result; -} - -jobjectArray Java_org_skia_skqp_SkQP_nExecuteUnitTest(JNIEnv* env, - jobject object, - jint index) { - jassert(env, index < (jint)gUnitTests.size()); - gm_runner::UnitTest test; - { - std::lock_guard<std::mutex> lock(gMutex); - test = gUnitTests[index]; - } - std::vector<std::string> errors = gm_runner::ExecuteTest(test); - if (errors.size() == 0) { - return nullptr; - } - jclass stringClass = env->FindClass("java/lang/String"); - jassert(env, stringClass); - jobjectArray array = env->NewObjectArray(errors.size(), stringClass, nullptr); - for (unsigned i = 0; i < errors.size(); ++i) { - set_string_array_element(env, array, errors[i].c_str(), i); - } - return (jobjectArray)env->NewGlobalRef(array); -} - -void Java_org_skia_skqp_SkQP_nMakeReport(JNIEnv*, jobject) { - std::string reportDirectoryPath; - { - std::lock_guard<std::mutex> lock(gMutex); - reportDirectoryPath = gReportDirectory; - } - (void)gmkb::MakeReport(reportDirectoryPath.c_str()); -} - -//////////////////////////////////////////////////////////////////////////////// - diff --git a/tools/skqp/make_gmkb.go b/tools/skqp/make_gmkb.go index a445f6df80..2b5dda542b 100644 --- a/tools/skqp/make_gmkb.go +++ b/tools/skqp/make_gmkb.go @@ -53,9 +53,9 @@ func clampU8(v int) uint8 { return uint8(v) } -func processTest(testName string, imgUrls []string, output string) error { +func processTest(testName string, imgUrls []string, output string) (bool, error) { if strings.ContainsRune(testName, '/') { - return nil + return false, nil } output_directory := path.Join(output, testName) var img_max image.NRGBA @@ -63,12 +63,12 @@ func processTest(testName string, imgUrls []string, output string) error { for _, url := range imgUrls { resp, err := http.Get(url) if err != nil { - return err + return false, err } img, err := png.Decode(resp.Body) resp.Body.Close() if err != nil { - return err + return false, err } if img_max.Rect.Max.X == 0 { // N.B. img_max.Pix may alias img.Pix (if they're already NRGBA). @@ -79,7 +79,7 @@ func processTest(testName string, imgUrls []string, output string) error { w := img.Bounds().Max.X - img.Bounds().Min.X h := img.Bounds().Max.Y - img.Bounds().Min.Y if img_max.Rect.Max.X != w || img_max.Rect.Max.Y != h { - return errors.New("size mismatch") + return false, errors.New("size mismatch") } img_nrgba := toNrgba(img) for i, value := range img_nrgba.Pix { @@ -91,22 +91,33 @@ func processTest(testName string, imgUrls []string, output string) error { } } if img_max.Rect.Max.X == 0 { - return nil + return false, nil } if err := os.Mkdir(output_directory, os.ModePerm); err != nil && !os.IsExist(err) { - return err + return false, err } if err := writePngToFile(path.Join(output_directory, min_png), &img_min); err != nil { - return err + return false, err } if err := writePngToFile(path.Join(output_directory, max_png), &img_max); err != nil { - return err + return false, err } - return nil + return true, nil +} + +type LockedStringList struct { + List []string + mux sync.Mutex +} +func (l *LockedStringList) add(v string) { + l.mux.Lock() + defer l.mux.Unlock() + l.List = append(l.List, v) } + func readMetaJsonFile(filename string) ([]search.ExportTestRecord, error) { file, err := os.Open(filename) if err != nil { @@ -165,6 +176,7 @@ func main() { } sort.Sort(ExportTestRecordArray(records)) + var results LockedStringList var wg sync.WaitGroup for _, record := range records { var goodUrls []string @@ -176,14 +188,26 @@ func main() { } } wg.Add(1) - go func(testName string, imgUrls []string, output string) { + go func(testName string, imgUrls []string, output string, results* LockedStringList) { defer wg.Done() - if err := processTest(testName, imgUrls, output); err != nil { + success, err := processTest(testName, imgUrls, output) + if err != nil { log.Fatal(err) } + if success { + results.add(testName) + } fmt.Printf("\r%-60s", testName) - }(record.TestName, goodUrls, output) + }(record.TestName, goodUrls, output, &results) } wg.Wait() fmt.Printf("\r%60s\n", "") + sort.Strings(results.List) + modelFile, err := os.Create(path.Join(output, "models.txt")) + if err != nil { + log.Fatal(err) + } + for _, v := range results.List { + fmt.Fprintln(modelFile, v) + } } diff --git a/tools/skqp/make_known_tests.sh b/tools/skqp/make_known_tests.sh deleted file mode 100755 index 7242db625d..0000000000 --- a/tools/skqp/make_known_tests.sh +++ /dev/null @@ -1,26 +0,0 @@ -#! /bin/sh - -# Copyright 2018 Google Inc. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -set -e -x - -cd "$(dirname "$0")/../.." - -BUILD=out/default - -python tools/git-sync-deps - -bin/gn gen $BUILD - -ninja -C $BUILD list_gms list_gpu_unit_tests - -DIR=platform_tools/android/apps/skqp/src/main/assets/skqp - -mkdir -p $DIR - -$BUILD/list_gms > $DIR/KnownGMs.txt - -$BUILD/list_gpu_unit_tests > $DIR/KnownGpuUnitTests.txt - diff --git a/tools/skqp/make_model.sh b/tools/skqp/make_model.sh deleted file mode 100755 index 3946dd9e85..0000000000 --- a/tools/skqp/make_model.sh +++ /dev/null @@ -1,20 +0,0 @@ -#! /bin/sh - -# Copyright 2018 Google Inc. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -if ! [ -f "$1" ]; then - printf 'Usage:\n %s META_JSON_FILE_PATH\n\n' "$0" >&2 - exit 1 -fi - -set -e -x - -SKIA="$(dirname "$0")/../.." - -go get -u go.skia.org/infra/golden/go/search - -go run "${SKIA}/tools/skqp/make_gmkb.go" "$1" \ - "${SKIA}/platform_tools/android/apps/skqp/src/main/assets/gmkb" - diff --git a/tools/skqp/make_rendertests_list.py b/tools/skqp/make_rendertests_list.py new file mode 100755 index 0000000000..54d9203fe1 --- /dev/null +++ b/tools/skqp/make_rendertests_list.py @@ -0,0 +1,49 @@ +#! /usr/bin/env python + +# Copyright 2018 Google LLC. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import csv +import os +import shutil +import sys + +def gset(path): + s = set() + if os.path.isfile(path): + with open(path, 'r') as f: + for line in f: + s.add(line.strip()) + return s + +def main(): + assert '/' in [os.sep, os.altsep] + assets = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, + 'platform_tools/android/apps/skqp/src/main/assets') + models = gset(assets + '/gmkb/models.txt') + good = gset('good.txt') + bad = gset('bad.txt') + assert good.isdisjoint(bad) + do_score = good & models + no_score = bad | (good - models) + to_delete = models & bad + for d in to_delete: + path = assets + '/gmkb/' + d + if os.path.isdir(path): + shutil.rmtree(path) + results = dict() + for n in do_score: + results[n] = 0 + for n in no_score: + results[n] = -1 + skqp = assets + '/skqp' + if not os.path.isdir(skqp): + os.mkdir(skqp) + with open(skqp + '/rendertests.txt', 'w') as o: + for n in sorted(results): + o.write('%s,%d\n' % (n, results[n])) + +if __name__ == '__main__': + main() + diff --git a/tools/skqp/make_universal_apk.py b/tools/skqp/make_universal_apk.py index 029238f8de..1578ecb668 100755 --- a/tools/skqp/make_universal_apk.py +++ b/tools/skqp/make_universal_apk.py @@ -121,8 +121,6 @@ def make_apk(architectures, if os.path.exists(apps_dir + '/skqp/src/main/assets/files.checksum'): check_call([sys.executable, 'tools/skqp/download_model']) - if os.environ.get('SKQP_EXTRA_MODELS' ,''): - check_call([sys.executable, 'tools/skqp/remove_unneeded_assets']) else: sys.stderr.write( '\n* * *\n\nNote: SkQP models are missing!!!!\n\n* * *\n\n') diff --git a/tools/skqp/remove_unneeded_assets b/tools/skqp/remove_unneeded_assets deleted file mode 100755 index 67bf9cff03..0000000000 --- a/tools/skqp/remove_unneeded_assets +++ /dev/null @@ -1,34 +0,0 @@ -#! /usr/bin/env python - -# Copyright 2018 Google Inc. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -import os -import shutil -import sys - -def gset(path): - s = set() - if os.path.isfile(path): - with open(path, 'r') as f: - for line in f: - s.add(line.strip()) - return s - -def main(): - assets = os.path.join('platform_tools', 'android', 'apps', 'skqp', 'src', 'main', 'assets') - os.chdir(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, assets)) - known = gset('skqp/KnownGMs.txt') - nope = gset('skqp/DoNotScoreInCompatibilityTestMode.txt') - present = set(os.listdir('gmkb')) - to_delete = present & nope - if (known): - to_delete |= (present - known) - for x in to_delete: - shutil.rmtree(os.path.join('gmkb', x)) - sys.stdout.write('%s: %d of %d models removed\n' %(sys.argv[0], len(to_delete), len(present))) - -if __name__ == '__main__': - main() - diff --git a/tools/skqp/skqp.cpp b/tools/skqp/skqp.cpp deleted file mode 100644 index cba52de6be..0000000000 --- a/tools/skqp/skqp.cpp +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Copyright 2017 Google Inc. - * - * Use of this source code is governed by a BSD-style license that can be - * found in the LICENSE file. - */ - -#include <sys/stat.h> - -#include "gm_knowledge.h" -#include "gm_runner.h" - -#ifdef __clang__ -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wused-but-marked-unused" -#endif - -#include "gtest/gtest.h" - -#ifdef __clang__ -#pragma clang diagnostic pop -#endif - -#include "Resources.h" -#include "SkStream.h" -#include "SkString.h" - -//////////////////////////////////////////////////////////////////////////////// - -static std::string gReportDirectoryPath; -static std::unique_ptr<skqp::AssetManager> gAssetMgr; - -//////////////////////////////////////////////////////////////////////////////// - -struct GMTestCase { - gm_runner::GMFactory fGMFactory; - gm_runner::SkiaBackend fBackend; -}; - -struct GMTest : public testing::Test { - GMTestCase fTest; - GMTest(GMTestCase t) : fTest(t) {} - void TestBody() override { - float result; - gm_runner::Error error; - std::tie(result, error) = - gm_runner::EvaluateGM(fTest.fBackend, fTest.fGMFactory, - gAssetMgr.get(), gReportDirectoryPath.c_str()); - EXPECT_EQ(error, gm_runner::Error::None); - if (gm_runner::Error::None == error) { - EXPECT_EQ(result, 0); - } - } -}; - -struct GMTestFactory : public testing::internal::TestFactoryBase { - GMTestCase fTest; - GMTestFactory(GMTestCase t) : fTest(t) {} - testing::Test* CreateTest() override { return new GMTest(fTest); } -}; - -//////////////////////////////////////////////////////////////////////////////// - -struct UnitTestTest : public testing::Test { - gm_runner::UnitTest fTest; - UnitTestTest(gm_runner::UnitTest test) : fTest(test) {} - void TestBody() override { - std::vector<std::string> errors = gm_runner::ExecuteTest(fTest); - for (const std::string& error : errors) { - GTEST_NONFATAL_FAILURE_(error.c_str()); - } - } -}; - -struct UnitTestFactory : public testing::internal::TestFactoryBase { - gm_runner::UnitTest fTest; - UnitTestFactory(gm_runner::UnitTest test) : fTest(test) {} - testing::Test* CreateTest() override { return new UnitTestTest(fTest); } -}; - -//////////////////////////////////////////////////////////////////////////////// - -static void reg_test(const char* test, const char* testCase, - testing::internal::TestFactoryBase* fact) { - testing::internal::MakeAndRegisterTestInfo(test, - testCase, - nullptr, - nullptr, - testing::internal::CodeLocation(__FILE__, __LINE__), - testing::internal::GetTestTypeId(), - testing::Test::SetUpTestCase, - testing::Test::TearDownTestCase, - fact); -} - - -void register_skia_tests() { - gm_runner::InitSkia(gm_runner::Mode::kCompatibilityTestMode, gAssetMgr.get()); - - // Rendering Tests - std::vector<gm_runner::SkiaBackend> backends = gm_runner::GetSupportedBackends(); - std::vector<gm_runner::GMFactory> gms = gm_runner::GetGMFactories(gAssetMgr.get()); - for (auto backend : backends) { - const char* backendName = GetBackendName(backend); - std::string test = std::string("SkiaGM_") + backendName; - for (auto gmFactory : gms) { - std::string gmName = gm_runner::GetGMName(gmFactory); - reg_test(test.c_str(), gmName.c_str(), - new GMTestFactory(GMTestCase{gmFactory, backend})); - } - } - - for (gm_runner::UnitTest test : gm_runner::GetUnitTests()) { - reg_test("Skia_Unit_Tests", gm_runner::GetUnitTestName(test), new UnitTestFactory{test}); - } -} - -namespace { -struct StdAssetManager : public skqp::AssetManager { - SkString fPrefix; - StdAssetManager(const char* p) : fPrefix(p) {} - std::unique_ptr<SkStreamAsset> open(const char* path) override { - SkString fullPath = fPrefix.isEmpty() - ? SkString(path) - : SkStringPrintf("%s/%s", fPrefix.c_str(), path); - return SkStream::MakeFromFile(fullPath.c_str()); - } -}; -} - -int main(int argc, char** argv) { - testing::InitGoogleTest(&argc, argv); - if (argc < 2) { - std::cerr << "Usage:\n " << argv[0] - << " [GTEST_ARGUMENTS] GMKB_DIRECTORY_PATH GMKB_REPORT_PATH\n\n"; - return 1; - } - SetResourcePath((std::string(argv[1]) + "/resources").c_str()); - gAssetMgr.reset(new StdAssetManager(argv[1])); - if (argc > 2) { - gReportDirectoryPath = argv[2]; - (void)mkdir(gReportDirectoryPath.c_str(), 0777); - } - register_skia_tests(); - int ret = RUN_ALL_TESTS(); - (void)gmkb::MakeReport(gReportDirectoryPath.c_str()); - return ret; -} diff --git a/tools/skqp/skqp_asset_manager.h b/tools/skqp/skqp_asset_manager.h deleted file mode 100644 index b9bd4d6c81..0000000000 --- a/tools/skqp/skqp_asset_manager.h +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright 2017 Google Inc. - * - * Use of this source code is governed by a BSD-style license that can be - * found in the LICENSE file. - */ -#ifndef skqp_asset_manager_DEFINED -#define skqp_asset_manager_DEFINED - -#include <memory> - -class SkStreamAsset; - -namespace skqp { -class AssetManager { -public: - virtual ~AssetManager() {} - virtual std::unique_ptr<SkStreamAsset> open(const char* path) = 0; -}; -} // namespace skqp -#endif // skqp_asset_manager_DEFINED diff --git a/tools/skqp/src/jni_skqp.cpp b/tools/skqp/src/jni_skqp.cpp new file mode 100644 index 0000000000..1d45ffd747 --- /dev/null +++ b/tools/skqp/src/jni_skqp.cpp @@ -0,0 +1,184 @@ +/* + * Copyright 2017 Google Inc. + * + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +#include <mutex> + +#include <android/asset_manager.h> +#include <android/asset_manager_jni.h> +#include <jni.h> +#include <sys/stat.h> + +#include "ResourceFactory.h" +#include "SkStream.h" +#include "SkTo.h" + +#include "skqp.h" + +//////////////////////////////////////////////////////////////////////////////// +extern "C" { +JNIEXPORT void JNICALL Java_org_skia_skqp_SkQP_nInit(JNIEnv*, jobject, jobject, jstring); +JNIEXPORT jlong JNICALL Java_org_skia_skqp_SkQP_nExecuteGM(JNIEnv*, jobject, jint, jint); +JNIEXPORT jobjectArray JNICALL Java_org_skia_skqp_SkQP_nExecuteUnitTest(JNIEnv*, jobject, jint); +JNIEXPORT void JNICALL Java_org_skia_skqp_SkQP_nMakeReport(JNIEnv*, jobject); +} // extern "C" +//////////////////////////////////////////////////////////////////////////////// + +static AAssetManager* gAAssetManager = nullptr; + +static sk_sp<SkData> open_asset_data(const char* path) { + sk_sp<SkData> data; + if (gAAssetManager) { + if (AAsset* asset = AAssetManager_open(gAAssetManager, path, AASSET_MODE_STREAMING)) { + if (size_t size = SkToSizeT(AAsset_getLength(asset))) { + data = SkData::MakeUninitialized(size); + int ret = AAsset_read(asset, data->writable_data(), size); + if (ret != SkToInt(size)) { + SkDebugf("ERROR: AAsset_read != AAsset_getLength (%s)\n", path); + } + } + AAsset_close(asset); + } + } + return data; +} + +namespace { +struct AndroidAssetManager : public SkQPAssetManager { + sk_sp<SkData> open(const char* path) override { return open_asset_data(path); } +}; +} + +// TODO(halcanary): Should not have global variables; SkQP Java object should +// own pointers and manage concurency. +static AndroidAssetManager gAndroidAssetManager; +static std::mutex gMutex; +static SkQP gSkQP; + +#define jassert(env, cond, ret) do { if (!(cond)) { \ + (env)->ThrowNew((env)->FindClass("java/lang/Exception"), \ + __FILE__ ": assert(" #cond ") failed."); \ + return ret; } } while (0) + +static void set_string_array_element(JNIEnv* env, jobjectArray a, const char* s, unsigned i) { + jstring jstr = env->NewStringUTF(s); + jassert(env, jstr != nullptr,); + env->SetObjectArrayElement(a, (jsize)i, jstr); + env->DeleteLocalRef(jstr); +} + +//////////////////////////////////////////////////////////////////////////////// + +sk_sp<SkData> get_resource(const char* resource) { + return open_asset_data((std::string("resources/") + resource).c_str()); +} + +//////////////////////////////////////////////////////////////////////////////// + +template <typename T, typename F> +jobjectArray to_java_string_array(JNIEnv* env, + const std::vector<T>& array, + F toString) { + jclass stringClass = env->FindClass("java/lang/String"); + jassert(env, stringClass, nullptr); + jobjectArray jarray = env->NewObjectArray((jint)array.size(), stringClass, nullptr); + jassert(env, jarray != nullptr, nullptr); + for (unsigned i = 0; i < array.size(); ++i) { + set_string_array_element(env, jarray, std::string(toString(array[i])).c_str(), i); + } + return jarray; +} + +static std::string to_string(JNIEnv* env, jstring jString) { + const char* utf8String = env->GetStringUTFChars(jString, nullptr); + jassert(env, utf8String && utf8String[0], ""); + std::string sString(utf8String); + env->ReleaseStringUTFChars(jString, utf8String); + return sString; +} + +void Java_org_skia_skqp_SkQP_nInit(JNIEnv* env, jobject object, jobject assetManager, + jstring dataDir) { + jclass SkQP_class = env->GetObjectClass(object); + + // tools/Resources + gResourceFactory = &get_resource; + + std::string reportDirectory = to_string(env, dataDir); + + jassert(env, assetManager,); + // This global must be set before using AndroidAssetManager + gAAssetManager = AAssetManager_fromJava(env, assetManager); + jassert(env, gAAssetManager,); + + std::lock_guard<std::mutex> lock(gMutex); + gSkQP.init(&gAndroidAssetManager, reportDirectory.c_str()); + + auto backends = gSkQP.getSupportedBackends(); + jassert(env, backends.size() > 0,); + auto gms = gSkQP.getGMs(); + jassert(env, gms.size() > 0,); + auto unitTests = gSkQP.getUnitTests(); + jassert(env, unitTests.size() > 0,); + + constexpr char kStringArrayType[] = "[Ljava/lang/String;"; + env->SetObjectField(object, env->GetFieldID(SkQP_class, "mBackends", kStringArrayType), + to_java_string_array(env, backends, SkQP::GetBackendName)); + env->SetObjectField(object, env->GetFieldID(SkQP_class, "mUnitTests", kStringArrayType), + to_java_string_array(env, unitTests, SkQP::GetUnitTestName)); + env->SetObjectField(object, env->GetFieldID(SkQP_class, "mGMs", kStringArrayType), + to_java_string_array(env, gms, SkQP::GetGMName)); +} + +jlong Java_org_skia_skqp_SkQP_nExecuteGM(JNIEnv* env, + jobject object, + jint gmIndex, + jint backendIndex) { + SkQP::RenderOutcome outcome; + std::string except; + { + std::lock_guard<std::mutex> lock(gMutex); + jassert(env, backendIndex < (jint)gSkQP.getSupportedBackends().size(), -1); + jassert(env, gmIndex < (jint)gSkQP.getGMs().size(), -1); + SkQP::SkiaBackend backend = gSkQP.getSupportedBackends()[backendIndex]; + SkQP::GMFactory gm = gSkQP.getGMs()[gmIndex]; + std::tie(outcome, except) = gSkQP.evaluateGM(backend, gm); + } + + if (!except.empty()) { + (void)env->ThrowNew(env->FindClass("org/skia/skqp/SkQPException"), except.c_str()); + } + return (jlong)outcome.fTotalError; +} + +jobjectArray Java_org_skia_skqp_SkQP_nExecuteUnitTest(JNIEnv* env, + jobject object, + jint index) { + std::vector<std::string> errors; + { + jassert(env, index < (jint)gSkQP.getUnitTests().size(), nullptr); + std::lock_guard<std::mutex> lock(gMutex); + errors = gSkQP.executeTest(gSkQP.getUnitTests()[index]); + } + if (errors.size() == 0) { + return nullptr; + } + jclass stringClass = env->FindClass("java/lang/String"); + jassert(env, stringClass, nullptr); + jobjectArray array = env->NewObjectArray(errors.size(), stringClass, nullptr); + for (unsigned i = 0; i < errors.size(); ++i) { + set_string_array_element(env, array, errors[i].c_str(), i); + } + return (jobjectArray)env->NewGlobalRef(array); +} + +void Java_org_skia_skqp_SkQP_nMakeReport(JNIEnv*, jobject) { + std::lock_guard<std::mutex> lock(gMutex); + gSkQP.makeReport(); +} + +//////////////////////////////////////////////////////////////////////////////// + diff --git a/tools/skqp/src/skqp.cpp b/tools/skqp/src/skqp.cpp new file mode 100644 index 0000000000..7c41a99bee --- /dev/null +++ b/tools/skqp/src/skqp.cpp @@ -0,0 +1,494 @@ +/* + * Copyright 2018 Google Inc. + * + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +#include "skqp.h" + +#include "../../../src/core/SkStreamPriv.h" +#include "../../tools/fonts/SkTestFontMgr.h" +#include "GrContext.h" +#include "GrContextOptions.h" +#include "GrContextPriv.h" +#include "SkFontMgrPriv.h" +#include "SkFontStyle.h" +#include "SkGraphics.h" +#include "SkImageInfoPriv.h" +#include "SkOSFile.h" +#include "SkOSPath.h" +#include "SkPngEncoder.h" +#include "SkStream.h" +#include "SkSurface.h" +#include "Test.h" +#include "gl/GLTestContext.h" +#include "gm.h" +#include "vk/VkTestContext.h" + +#include <algorithm> +#include <cinttypes> +#include <sstream> + +#include "skqp_model.h" + +#define IMAGES_DIRECTORY_PATH "images" +#define PATH_MAX_PNG "max.png" +#define PATH_MIN_PNG "min.png" +#define PATH_IMG_PNG "image.png" +#define PATH_ERR_PNG "errors.png" +#define PATH_MODEL "model" + +static constexpr char kRenderTestCSVReport[] = "out.csv"; +static constexpr char kRenderTestReportPath[] = "report.html"; +static constexpr char kRenderTestsPath[] = "skqp/rendertests.txt"; +static constexpr char kUnitTestReportPath[] = "unit_tests.txt"; +static constexpr char kUnitTestsPath[] = "skqp/unittests.txt"; + +// Kind of like Python's readlines(), but without any allocation. +// Calls f() on each line. +// F is [](const char*, size_t) -> void +template <typename F> +static void readlines(const void* data, size_t size, F f) { + const char* start = (const char*)data; + const char* end = start + size; + const char* ptr = start; + while (ptr < end) { + while (*ptr++ != '\n' && ptr < end) {} + size_t len = ptr - start; + f(start, len); + start = ptr; + } +} + +static void get_unit_tests(SkQPAssetManager* mgr, std::vector<SkQP::UnitTest>* unitTests) { + std::unordered_set<std::string> testset; + auto insert = [&testset](const char* s, size_t l) { + SkASSERT(l > 1) ; + if (l > 0 && s[l - 1] == '\n') { // strip line endings. + --l; + } + if (l > 0) { // only add non-empty strings. + testset.insert(std::string(s, l)); + } + }; + if (sk_sp<SkData> dat = mgr->open(kUnitTestsPath)) { + readlines(dat->data(), dat->size(), insert); + } + for (const skiatest::Test& test : skiatest::TestRegistry::Range()) { + if ((testset.empty() || testset.count(std::string(test.name)) > 0) && test.needsGpu) { + unitTests->push_back(&test); + } + } + auto lt = [](SkQP::UnitTest u, SkQP::UnitTest v) { return strcmp(u->name, v->name) < 0; }; + std::sort(unitTests->begin(), unitTests->end(), lt); +} + +static void get_render_tests(SkQPAssetManager* mgr, + std::vector<SkQP::GMFactory>* gmlist, + std::unordered_map<std::string, int64_t>* gmThresholds) { + auto insert = [gmThresholds](const char* s, size_t l) { + SkASSERT(l > 1) ; + if (l > 0 && s[l - 1] == '\n') { // strip line endings. + --l; + } + if (l == 0) { + return; + } + const char* end = s + l; + const char* ptr = s; + constexpr char kDelimeter = ','; + while (ptr < end && *ptr != kDelimeter) { ++ptr; } + if (ptr + 1 >= end) { + SkASSERT(false); // missing delimeter + return; + } + std::string key(s, ptr - s); + ++ptr; // skip delimeter + std::string number(ptr, end - ptr); // null-terminated copy. + int64_t value = 0; + if (1 != sscanf(number.c_str(), "%" SCNd64 , &value)) { + SkASSERT(false); // Not a number + return; + } + gmThresholds->insert({std::move(key), value}); // (*gmThresholds)[s] = value; + }; + if (sk_sp<SkData> dat = mgr->open(kRenderTestsPath)) { + readlines(dat->data(), dat->size(), insert); + } + using GmAndName = std::pair<SkQP::GMFactory, std::string>; + std::vector<GmAndName> gmsWithNames; + for (skiagm::GMFactory f : skiagm::GMRegistry::Range()) { + std::string name = SkQP::GetGMName(f); + if ((gmThresholds->empty() || gmThresholds->count(name) > 0)) { + gmsWithNames.push_back(std::make_pair(f, std::move(name))); + } + } + std::sort(gmsWithNames.begin(), gmsWithNames.end(), + [](GmAndName u, GmAndName v) { return u.second < v.second; }); + gmlist->reserve(gmsWithNames.size()); + for (const GmAndName& gmn : gmsWithNames) { + gmlist->push_back(gmn.first); + } +} + +static std::unique_ptr<sk_gpu_test::TestContext> make_test_context(SkQP::SkiaBackend backend) { + using U = std::unique_ptr<sk_gpu_test::TestContext>; + switch (backend) { + case SkQP::SkiaBackend::kGL: + return U(sk_gpu_test::CreatePlatformGLTestContext(kGL_GrGLStandard, nullptr)); + case SkQP::SkiaBackend::kGLES: + return U(sk_gpu_test::CreatePlatformGLTestContext(kGLES_GrGLStandard, nullptr)); +#ifdef SK_VULKAN + case SkQP::SkiaBackend::kVulkan: + return U(sk_gpu_test::CreatePlatformVkTestContext(nullptr)); +#endif + default: + return nullptr; + } +} + +static GrContextOptions context_options(skiagm::GM* gm = nullptr) { + GrContextOptions grContextOptions; + grContextOptions.fAllowPathMaskCaching = true; + grContextOptions.fSuppressPathRendering = true; + grContextOptions.fDisableDriverCorrectnessWorkarounds = true; + if (gm) { + gm->modifyGrContextOptions(&grContextOptions); + } + return grContextOptions; +} + +static std::vector<SkQP::SkiaBackend> get_backends() { + std::vector<SkQP::SkiaBackend> result; + SkQP::SkiaBackend backends[] = { + #ifndef SK_BUILD_FOR_ANDROID + SkQP::SkiaBackend::kGL, // Used for testing on desktop machines. + #endif + SkQP::SkiaBackend::kGLES, + #ifdef SK_VULKAN + SkQP::SkiaBackend::kVulkan, + #endif + }; + for (SkQP::SkiaBackend backend : backends) { + std::unique_ptr<sk_gpu_test::TestContext> testCtx = make_test_context(backend); + if (testCtx) { + testCtx->makeCurrent(); + if (nullptr != testCtx->makeGrContext(context_options())) { + result.push_back(backend); + } + } + } + SkASSERT_RELEASE(result.size() > 0); + return result; +} + +static void print_backend_info(const char* dstPath, + const std::vector<SkQP::SkiaBackend>& backends) { +#ifdef SK_ENABLE_DUMP_GPU + SkFILEWStream out(dstPath); + out.writeText("[\n"); + for (SkQP::SkiaBackend backend : backends) { + if (std::unique_ptr<sk_gpu_test::TestContext> testCtx = make_test_context(backend)) { + testCtx->makeCurrent(); + if (sk_sp<GrContext> ctx = testCtx->makeGrContext(context_options())) { + SkString info = ctx->contextPriv().dump(); + // remove null + out.write(info.c_str(), info.size()); + out.writeText(",\n"); + } + } + } + out.writeText("]\n"); +#endif +} + +static void encode_png(const SkBitmap& src, const std::string& dst) { + SkFILEWStream wStream(dst.c_str()); + SkPngEncoder::Options options; + bool success = wStream.isValid() && SkPngEncoder::Encode(&wStream, src.pixmap(), options); + SkASSERT_RELEASE(success); +} + +static void write_to_file(const sk_sp<SkData>& src, const std::string& dst) { + SkFILEWStream wStream(dst.c_str()); + bool success = wStream.isValid() && wStream.write(src->data(), src->size()); + SkASSERT_RELEASE(success); +} + +//////////////////////////////////////////////////////////////////////////////// + +const char* SkQP::GetBackendName(SkQP::SkiaBackend b) { + switch (b) { + case SkQP::SkiaBackend::kGL: return "gl"; + case SkQP::SkiaBackend::kGLES: return "gles"; + case SkQP::SkiaBackend::kVulkan: return "vk"; + } + return ""; +} + +std::string SkQP::GetGMName(SkQP::GMFactory f) { + std::unique_ptr<skiagm::GM> gm(f ? f(nullptr) : nullptr); + return std::string(gm ? gm->getName() : ""); +} + +const char* SkQP::GetUnitTestName(SkQP::UnitTest t) { return t->name; } + +SkQP::SkQP() {} + +SkQP::~SkQP() {} + +void SkQP::init(SkQPAssetManager* am, const char* reportDirectory) { + SkASSERT_RELEASE(!fAssetManager); + SkASSERT_RELEASE(am); + fAssetManager = am; + fReportDirectory = reportDirectory; + + SkGraphics::Init(); + gSkFontMgr_DefaultFactory = &sk_tool_utils::MakePortableFontMgr; + + /* If the file "skqp/rendertests.txt" does not exist or is empty, run all + render tests. Otherwise only run tests mentioned in that file. */ + get_render_tests(fAssetManager, &fGMs, &fGMThresholds); + /* If the file "skqp/unittests.txt" does not exist or is empty, run all gpu + unit tests. Otherwise only run tests mentioned in that file. */ + get_unit_tests(fAssetManager, &fUnitTests); + fSupportedBackends = get_backends(); + + print_backend_info((fReportDirectory + "/grdump.txt").c_str(), fSupportedBackends); +} + +std::tuple<SkQP::RenderOutcome, std::string> SkQP::evaluateGM(SkQP::SkiaBackend backend, + SkQP::GMFactory gmFact) { + SkASSERT_RELEASE(fAssetManager); + static constexpr SkQP::RenderOutcome kError = {INT_MAX, INT_MAX, INT64_MAX}; + static constexpr SkQP::RenderOutcome kPass = {0, 0, 0}; + + SkASSERT(gmFact); + std::unique_ptr<skiagm::GM> gm(gmFact(nullptr)); + SkASSERT(gm); + const char* const name = gm->getName(); + const SkISize size = gm->getISize(); + const int w = size.width(); + const int h = size.height(); + const SkImageInfo info = + SkImageInfo::Make(w, h, skqp::kColorType, kPremul_SkAlphaType, nullptr); + const SkSurfaceProps props(0, SkSurfaceProps::kLegacyFontHost_InitType); + + std::unique_ptr<sk_gpu_test::TestContext> testCtx = make_test_context(backend); + if (!testCtx) { + return std::make_tuple(kError, "Skia Failure: test context"); + } + testCtx->makeCurrent(); + sk_sp<SkSurface> surf = SkSurface::MakeRenderTarget( + testCtx->makeGrContext(context_options(gm.get())).get(), + SkBudgeted::kNo, info, 0, &props); + if (!surf) { + return std::make_tuple(kError, "Skia Failure: gr-context"); + } + gm->draw(surf->getCanvas()); + + SkBitmap image; + image.allocPixels(SkImageInfo::Make(w, h, skqp::kColorType, skqp::kAlphaType)); + + // SkColorTypeBytesPerPixel should be constexpr, but is not. + SkASSERT(SkColorTypeBytesPerPixel(skqp::kColorType) == sizeof(uint32_t)); + // Call readPixels because we need to compare pixels. + if (!surf->readPixels(image.pixmap(), 0, 0)) { + return std::make_tuple(kError, "Skia Failure: read pixels"); + } + int64_t passingThreshold = fGMThresholds.empty() ? -1 : fGMThresholds[std::string(name)]; + + if (-1 == passingThreshold) { + return std::make_tuple(kPass, ""); + } + skqp::ModelResult modelResult = + skqp::CheckAgainstModel(name, image.pixmap(), fAssetManager); + + if (!modelResult.fErrorString.empty()) { + return std::make_tuple(kError, std::move(modelResult.fErrorString)); + } + fRenderResults.push_back(SkQP::RenderResult{backend, gmFact, modelResult.fOutcome}); + if (modelResult.fOutcome.fMaxError <= passingThreshold) { + return std::make_tuple(kPass, ""); + } + std::string imagesDirectory = fReportDirectory + "/" IMAGES_DIRECTORY_PATH; + if (!sk_mkdir(imagesDirectory.c_str())) { + SkDebugf("ERROR: sk_mkdir('%s');\n", imagesDirectory.c_str()); + return std::make_tuple(modelResult.fOutcome, ""); + } + std::ostringstream tmp; + tmp << imagesDirectory << '/' << SkQP::GetBackendName(backend) << '_' << name << '_'; + std::string imagesPathPrefix1 = tmp.str(); + tmp = std::ostringstream(); + tmp << imagesDirectory << '/' << PATH_MODEL << '_' << name << '_'; + std::string imagesPathPrefix2 = tmp.str(); + encode_png(image, imagesPathPrefix1 + PATH_IMG_PNG); + encode_png(modelResult.fErrors, imagesPathPrefix1 + PATH_ERR_PNG); + write_to_file(modelResult.fMaxPng, imagesPathPrefix2 + PATH_MAX_PNG); + write_to_file(modelResult.fMinPng, imagesPathPrefix2 + PATH_MIN_PNG); + return std::make_tuple(modelResult.fOutcome, ""); +} + +std::vector<std::string> SkQP::executeTest(SkQP::UnitTest test) { + SkASSERT_RELEASE(fAssetManager); + struct : public skiatest::Reporter { + std::vector<std::string> fErrors; + void reportFailed(const skiatest::Failure& failure) override { + SkString desc = failure.toString(); + fErrors.push_back(std::string(desc.c_str(), desc.size())); + } + } r; + GrContextOptions options; + options.fDisableDriverCorrectnessWorkarounds = true; + if (test->fContextOptionsProc) { + test->fContextOptionsProc(&options); + } + test->proc(&r, options); + fUnitTestResults.push_back(UnitTestResult{test, r.fErrors}); + return r.fErrors; +} + +//////////////////////////////////////////////////////////////////////////////// + +static constexpr char kDocHead[] = + "<!doctype html>\n" + "<html lang=\"en\">\n" + "<head>\n" + "<meta charset=\"UTF-8\">\n" + "<title>SkQP Report</title>\n" + "<style>\n" + "img { max-width:48%; border:1px green solid;\n" + " image-rendering: pixelated;\n" + " background-image:url('data:image/png;base64,iVBORw0KGgoA" + "AAANSUhEUgAAABAAAAAQCAAAAAA6mKC9AAAAAXNSR0IArs4c6QAAAAJiS0dEAP+H" + "j8y/AAAACXBIWXMAAA7DAAAOwwHHb6hkAAAAB3RJTUUH3gUBEi4DGRAQYgAAAB1J" + "REFUGNNjfMoAAVJQmokBDdBHgPE/lPFsYN0BABdaAwN6tehMAAAAAElFTkSuQmCC" + "'); }\n" + "</style>\n" + "<script>\n" + "function ce(t) { return document.createElement(t); }\n" + "function ct(n) { return document.createTextNode(n); }\n" + "function ac(u,v) { return u.appendChild(v); }\n" + "function br(u) { ac(u, ce(\"br\")); }\n" + "function ma(s, c) { var a = ce(\"a\"); a.href = s; ac(a, c); return a; }\n" + "function f(backend, gm, e1, e2, e3) {\n" + " var b = ce(\"div\");\n" + " var x = ce(\"h2\");\n" + " var t = backend + \"_\" + gm;\n" + " ac(x, ct(t));\n" + " ac(b, x);\n" + " ac(b, ct(\"backend: \" + backend));\n" + " br(b);\n" + " ac(b, ct(\"gm name: \" + gm));\n" + " br(b);\n" + " ac(b, ct(\"maximum error: \" + e1));\n" + " br(b);\n" + " ac(b, ct(\"bad pixel counts: \" + e2));\n" + " br(b);\n" + " ac(b, ct(\"total error: \" + e3));\n" + " br(b);\n" + " var q = \"" IMAGES_DIRECTORY_PATH "/\" + backend + \"_\" + gm + \"_\";\n" + " var p = \"" IMAGES_DIRECTORY_PATH "/" PATH_MODEL "_\" + gm + \"_\";\n" + " var i = ce(\"img\");\n" + " i.src = q + \"" PATH_IMG_PNG "\";\n" + " i.alt = \"img\";\n" + " ac(b, ma(i.src, i));\n" + " i = ce(\"img\");\n" + " i.src = q + \"" PATH_ERR_PNG "\";\n" + " i.alt = \"err\";\n" + " ac(b, ma(i.src, i));\n" + " br(b);\n" + " ac(b, ct(\"Expectation: \"));\n" + " ac(b, ma(p + \"" PATH_MAX_PNG "\", ct(\"max\")));\n" + " ac(b, ct(\" | \"));\n" + " ac(b, ma(p + \"" PATH_MIN_PNG "\", ct(\"min\")));\n" + " ac(b, ce(\"hr\"));\n" + " b.id = backend + \":\" + gm;\n" + " ac(document.body, b);\n" + " l = ce(\"li\");\n" + " ac(l, ct(\"[\" + e3 + \"] \"));\n" + " ac(l, ma(\"#\" + backend +\":\"+ gm , ct(t)));\n" + " ac(document.getElementById(\"toc\"), l);\n" + "}\n" + "function main() {\n"; + +static constexpr char kDocMiddle[] = + "}\n" + "</script>\n" + "</head>\n" + "<body onload=\"main()\">\n" + "<h1>SkQP Report</h1>\n"; + +static constexpr char kDocTail[] = + "<ul id=\"toc\"></ul>\n" + "<hr>\n" + "<p>Left image: test result<br>\n" + "Right image: errors (white = no error, black = smallest error, red = biggest error; " + "other errors are a color between black and red.)</p>\n" + "<hr>\n" + "</body>\n" + "</html>\n"; + +template <typename T> +inline void write(SkWStream* wStream, const T& text) { + wStream->write(text.c_str(), text.size()); +} + +void SkQP::makeReport() { + SkASSERT_RELEASE(fAssetManager); + int glesErrorCount = 0, vkErrorCount = 0, gles = 0, vk = 0; + + if (!sk_isdir(fReportDirectory.c_str())) { + SkDebugf("Report destination does not exist: '%s'\n", fReportDirectory.c_str()); + return; + } + SkFILEWStream csvOut(SkOSPath::Join(fReportDirectory.c_str(), kRenderTestCSVReport).c_str()); + SkFILEWStream htmOut(SkOSPath::Join(fReportDirectory.c_str(), kRenderTestReportPath).c_str()); + SkASSERT_RELEASE(csvOut.isValid() && htmOut.isValid()); + htmOut.writeText(kDocHead); + for (const SkQP::RenderResult& run : fRenderResults) { + switch (run.fBackend) { + case SkQP::SkiaBackend::kGLES: ++gles; break; + case SkQP::SkiaBackend::kVulkan: ++vk; break; + default: break; + } + const char* backendName = SkQP::GetBackendName(run.fBackend); + std::string gmName = SkQP::GetGMName(run.fGM); + SkQP::RenderOutcome outcome; + auto str = SkStringPrintf("\"%s\",\"%s\",%d,%d,%" PRId64, backendName, gmName.c_str(), + outcome.fMaxError, outcome.fBadPixelCount, outcome.fTotalError); + write(&csvOut, SkStringPrintf("%s\n", str.c_str())); + + int64_t passingThreshold = fGMThresholds.empty() ? 0 : fGMThresholds[gmName]; + if (passingThreshold == -1 || outcome.fMaxError <= passingThreshold) { + continue; + } + write(&htmOut, SkStringPrintf(" f(%s);\n", str.c_str())); + switch (run.fBackend) { + case SkQP::SkiaBackend::kGLES: ++glesErrorCount; break; + case SkQP::SkiaBackend::kVulkan: ++vkErrorCount; break; + default: break; + } + } + htmOut.writeText(kDocMiddle); + write(&htmOut, SkStringPrintf("<p>gles errors: %d (of %d)</br>\n" + "vk errors: %d (of %d)</p>\n", + glesErrorCount, gles, vkErrorCount, vk)); + htmOut.writeText(kDocTail); + SkFILEWStream unitOut(SkOSPath::Join(fReportDirectory.c_str(), kUnitTestReportPath).c_str()); + SkASSERT_RELEASE(unitOut.isValid()); + for (const SkQP::UnitTestResult& result : fUnitTestResults) { + unitOut.writeText(GetUnitTestName(result.fUnitTest)); + if (result.fErrors.empty()) { + unitOut.writeText(" PASSED\n* * *\n"); + } else { + write(&unitOut, SkStringPrintf(" FAILED (%u errors)\n", result.fErrors.size())); + for (const std::string& err : result.fErrors) { + write(&unitOut, err); + unitOut.newline(); + } + unitOut.writeText("* * *\n"); + } + } +} diff --git a/tools/skqp/src/skqp.h b/tools/skqp/src/skqp.h new file mode 100644 index 0000000000..c26928ea84 --- /dev/null +++ b/tools/skqp/src/skqp.h @@ -0,0 +1,122 @@ +/* + * Copyright 2018 Google Inc. + * + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +#ifndef skqp_DEFINED +#define skqp_DEFINED + +#include <cstdint> +#include <memory> +#include <string> +#include <tuple> +#include <unordered_set> +#include <unordered_map> +#include <vector> + +class SkData; +template <typename T> class sk_sp; + +namespace skiagm { +class GM; +} + +namespace skiatest { +struct Test; +} + +class SkStreamAsset; + +//////////////////////////////////////////////////////////////////////////////// +class SkQPAssetManager { +public: + SkQPAssetManager() {} + virtual ~SkQPAssetManager() {} + virtual sk_sp<SkData> open(const char* path) = 0; +private: + SkQPAssetManager(const SkQPAssetManager&) = delete; + SkQPAssetManager& operator=(const SkQPAssetManager&) = delete; +}; + +class SkQP { +public: + enum class SkiaBackend { + kGL, + kGLES, + kVulkan, + }; + using GMFactory = skiagm::GM* (*)(void*); + using UnitTest = const skiatest::Test*; + + //////////////////////////////////////////////////////////////////////////// + + /** These functions provide a descriptive name for the given value.*/ + static std::string GetGMName(GMFactory); + static const char* GetUnitTestName(UnitTest); + static const char* GetBackendName(SkiaBackend); + + SkQP(); + ~SkQP(); + + /** + Initialize Skia and the SkQP. Should be executed only once. + + @param assetManager - provides assets for the models. Does not take ownership. + @param reportDirectory - where to write out report. + */ + void init(SkQPAssetManager* assetManager, const char* reportDirectory); + + struct RenderOutcome { + // All three values will be 0 if the test passes. + int fMaxError = 0; // maximum error of all pixel. + int fBadPixelCount = 0; // number of pixels with non-zero error. + int64_t fTotalError = 0; // sum of error for all bad pixels. + }; + + /** + @return render outcome and error string. Only errors running or + evaluating the GM will result in a non-empty error string. + */ + std::tuple<RenderOutcome, std::string> evaluateGM(SkiaBackend, GMFactory); + + /** @return a (hopefully empty) list of errors produced by this unit test. */ + std::vector<std::string> executeTest(UnitTest); + + /** Call this after running all checks to write a report into the given + report directory. */ + void makeReport(); + + /** @return a list of backends that this version of SkQP supports. */ + const std::vector<SkiaBackend>& getSupportedBackends() const { return fSupportedBackends; } + /** @return a list of all Skia GMs in lexicographic order. */ + const std::vector<GMFactory>& getGMs() const { return fGMs; } + /** @return a list of all Skia GPU unit tests in lexicographic order. */ + const std::vector<UnitTest>& getUnitTests() const { return fUnitTests; } + //////////////////////////////////////////////////////////////////////////// + +private: + struct RenderResult { + SkiaBackend fBackend; + GMFactory fGM; + RenderOutcome fOutcome; + }; + struct UnitTestResult { + UnitTest fUnitTest; + std::vector<std::string> fErrors; + }; + std::vector<RenderResult> fRenderResults; + std::vector<UnitTestResult> fUnitTestResults; + std::vector<SkiaBackend> fSupportedBackends; + SkQPAssetManager* fAssetManager = nullptr; + std::string fReportDirectory; + std::vector<UnitTest> fUnitTests; + std::vector<GMFactory> fGMs; + std::unordered_map<std::string, int64_t> fGMThresholds; + + SkQP(const SkQP&) = delete; + SkQP& operator=(const SkQP&) = delete; +}; +#endif // skqp_DEFINED + diff --git a/tools/skqp/src/skqp_main.cpp b/tools/skqp/src/skqp_main.cpp new file mode 100644 index 0000000000..2737a9ac75 --- /dev/null +++ b/tools/skqp/src/skqp_main.cpp @@ -0,0 +1,145 @@ +/* + * Copyright 2017 Google Inc. + * + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +#include <iostream> +#include <sys/stat.h> + +#include "skqp.h" + +#include "Resources.h" +#include "SkData.h" +#include "SkOSFile.h" + +//////////////////////////////////////////////////////////////////////////////// + +namespace { +class StdAssetManager : public SkQPAssetManager { +public: + StdAssetManager(const char* p) : fPrefix(p) { + SkASSERT(!fPrefix.empty()); + //TODO(halcanary): does this need to be changed if I run SkQP in Windows? + fPrefix += "/"; + } + sk_sp<SkData> open(const char* path) override { + return SkData::MakeFromFileName((fPrefix + path).c_str()); + } +private: + std::string fPrefix; +}; +} + +static constexpr char kSkipUsage[] = + " TEST_MATCH_RULES:" + " [~][^]substring[$] [...] of name to run.\n" + " Multiple matches may be separated by spaces.\n" + " ~ causes a matching name to always be skipped\n" + " ^ requires the start of the name to match\n" + " $ requires the end of the name to match\n" + " ^ and $ requires an exact match\n" + " If a name does not match any list entry,\n" + " it is skipped unless some list entry starts with ~\n"; + +static bool should_skip(const char* const* rules, size_t count, const char* name) { + size_t testLen = strlen(name); + bool anyExclude = count == 0; + for (size_t i = 0; i < count; ++i) { + const char* matchName = rules[i]; + size_t matchLen = strlen(matchName); + bool matchExclude, matchStart, matchEnd; + if ((matchExclude = matchName[0] == '~')) { + anyExclude = true; + matchName++; + matchLen--; + } + if ((matchStart = matchName[0] == '^')) { + matchName++; + matchLen--; + } + if ((matchEnd = matchName[matchLen - 1] == '$')) { + matchLen--; + } + if (matchStart ? (!matchEnd || matchLen == testLen) + && strncmp(name, matchName, matchLen) == 0 + : matchEnd ? matchLen <= testLen + && strncmp(name + testLen - matchLen, matchName, matchLen) == 0 + : strstr(name, matchName) != nullptr) { + return matchExclude; + } + } + return !anyExclude; +} + +int main(int argc, char** argv) { + if (argc < 3) { + std::cerr << "Usage:\n " << argv[0] + << " ASSET_DIRECTORY_PATH SKQP_REPORT_PATH [TEST_MATCH_RULES]\n" + << kSkipUsage << '\n'; + return 1; + } + SetResourcePath((std::string(argv[1]) + "/resources").c_str()); + if (!sk_mkdir(argv[2])) { + std::cerr << "sk_mkdir(" << argv[2] << ") failed.\n"; + return 2; + } + StdAssetManager mgr(argv[1]); + SkQP skqp; + skqp.init(&mgr, argv[2]); + int ret = 0; + + const char* const* matchRules = &argv[3]; + size_t matchRulesCount = (size_t)(argc - 3); + + // Rendering Tests + std::ostream& out = std::cout; + for (auto backend : skqp.getSupportedBackends()) { + auto testPrefix = std::string(SkQP::GetBackendName(backend)) + "_"; + for (auto gmFactory : skqp.getGMs()) { + auto testName = testPrefix + SkQP::GetGMName(gmFactory); + if (should_skip(matchRules, matchRulesCount, testName.c_str())) { + continue; + } + out << "Starting: " << testName << std::endl; + SkQP::RenderOutcome outcome; + std::string except; + + std::tie(outcome, except) = skqp.evaluateGM(backend, gmFactory); + if (!except.empty()) { + out << "ERROR: " << testName << " (" << except << ")\n"; + ret = 1; + } else if (outcome.fMaxError != 0) { + out << "FAILED: " << testName << " (" << outcome.fMaxError << ")\n"; + ret = 1; + } else { + out << "Passed: " << testName << "\n"; + } + out.flush(); + } + } + + // Unit Tests + for (auto test : skqp.getUnitTests()) { + auto testName = std::string("unitTest_") + SkQP::GetUnitTestName(test); + if (should_skip(matchRules, matchRulesCount, testName.c_str())) { + continue; + } + out << "Starting test: " << testName << std::endl; + std::vector<std::string> errors = skqp.executeTest(test); + if (!errors.empty()) { + out << "TEST FAILED (" << errors.size() << "): " << testName << "\n"; + for (const std::string& error : errors) { + out << error << "\n"; + } + ret = 1; + } else { + out << "Test passed: " << testName << "\n"; + } + out.flush(); + } + skqp.makeReport(); + + return ret; +} diff --git a/tools/skqp/src/skqp_model.cpp b/tools/skqp/src/skqp_model.cpp new file mode 100644 index 0000000000..79fd2dc196 --- /dev/null +++ b/tools/skqp/src/skqp_model.cpp @@ -0,0 +1,144 @@ +/* + * Copyright 2018 Google Inc. + * + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ + +#include "skqp_model.h" +#include "skqp.h" + +#include "SkBitmap.h" +#include "SkCodec.h" +#include "SkOSPath.h" +#include "SkStream.h" + +#ifndef SK_SKQP_GLOBAL_ERROR_TOLERANCE +#define SK_SKQP_GLOBAL_ERROR_TOLERANCE 0 +#endif + +//////////////////////////////////////////////////////////////////////////////// + +static inline uint32_t color(const SkPixmap& pm, SkIPoint p) { + return *pm.addr32(p.x(), p.y()); +} + +static inline bool inside(SkIPoint point, SkISize dimensions) { + return (unsigned)point.x() < (unsigned)dimensions.width() && + (unsigned)point.y() < (unsigned)dimensions.height(); +} + +SkQP::RenderOutcome skqp::Check(const SkPixmap& minImg, + const SkPixmap& maxImg, + const SkPixmap& img, + unsigned tolerance, + SkBitmap* errorOut) { + SkQP::RenderOutcome result; + SkISize dim = img.info().dimensions(); + SkASSERT(minImg.info().dimensions() == dim); + SkASSERT(maxImg.info().dimensions() == dim); + static const SkIPoint kNeighborhood[9] = { + { 0, 0}, // ordered by closest pixels first. + {-1, 0}, { 1, 0}, { 0, -1}, { 0, 1}, + {-1, -1}, { 1, -1}, {-1, 1}, { 1, 1}, + }; + for (int y = 0; y < dim.height(); ++y) { + for (int x = 0; x < dim.width(); ++x) { + const SkIPoint xy{x, y}; + const uint32_t c = color(img, xy); + int error = INT_MAX; + // loop over neighborhood (halo); + for (SkIPoint delta : kNeighborhood) { + SkIPoint point = xy + delta; + if (inside(point, dim)) { // skip out of pixmap bounds. + int err = 0; + // loop over four color channels. + // Return Manhattan distance in channel-space. + for (int component : {0, 8, 16, 24}) { + uint8_t v = (c >> component) & 0xFF, + vmin = (color(minImg, point) >> component) & 0xFF, + vmax = (color(maxImg, point) >> component) & 0xFF; + err = SkMax32(err, SkMax32((int)v - (int)vmax, (int)vmin - (int)v)); + } + error = SkMin32(error, err); + } + } + if (error > (int)tolerance) { + ++result.fBadPixelCount; + result.fTotalError += error; + result.fMaxError = SkMax32(error, result.fMaxError); + if (errorOut) { + if (!errorOut->getPixels()) { + errorOut->allocPixels(SkImageInfo::Make( + dim.width(), dim.height(), + kBGRA_8888_SkColorType, + kOpaque_SkAlphaType)); + errorOut->eraseColor(SK_ColorWHITE); + } + SkASSERT((unsigned)error < 256); + *(errorOut->getAddr32(x, y)) = SkColorSetARGB(0xFF, (uint8_t)error, 0, 0); + } + } + } + } + return result; +} + +static SkBitmap decode(sk_sp<SkData> data) { + SkBitmap bitmap; + if (auto codec = SkCodec::MakeFromData(std::move(data))) { + SkISize size = codec->getInfo().dimensions(); + SkASSERT(!size.isEmpty()); + SkImageInfo info = SkImageInfo::Make(size.width(), size.height(), + skqp::kColorType, skqp::kAlphaType); + bitmap.allocPixels(info); + if (SkCodec::kSuccess != codec->getPixels(bitmap.pixmap())) { + bitmap.reset(); + } + } + return bitmap; +} + +skqp::ModelResult skqp::CheckAgainstModel(const char* name, + const SkPixmap& pm, + SkQPAssetManager* mgr) { + skqp::ModelResult result; + if (pm.colorType() != kColorType || pm.alphaType() != kAlphaType) { + result.fErrorString = "Model failed: source image format."; + return result; + } + if (pm.info().isEmpty()) { + result.fErrorString = "Model failed: empty source image"; + return result; + } + constexpr char PATH_ROOT[] = "gmkb"; + SkString img_path = SkOSPath::Join(PATH_ROOT, name); + SkString max_path = SkOSPath::Join(img_path.c_str(), kMaxPngPath); + SkString min_path = SkOSPath::Join(img_path.c_str(), kMinPngPath); + + result.fMaxPng = mgr->open(max_path.c_str()); + result.fMinPng = mgr->open(min_path.c_str()); + + SkBitmap max_image = decode(result.fMaxPng); + SkBitmap min_image = decode(result.fMinPng); + + if (max_image.isNull() || min_image.isNull()) { + result.fErrorString = "Model missing"; + return result; + } + if (max_image.info().dimensions() != min_image.info().dimensions()) { + result.fErrorString = "Model has mismatched data."; + return result; + } + + if (max_image.info().dimensions() != pm.info().dimensions()) { + result.fErrorString = "Model data does not match source size."; + return result; + } + result.fOutcome = Check(min_image.pixmap(), + max_image.pixmap(), + pm, + SK_SKQP_GLOBAL_ERROR_TOLERANCE, + &result.fErrors); + return result; +} diff --git a/tools/skqp/src/skqp_model.h b/tools/skqp/src/skqp_model.h new file mode 100644 index 0000000000..d807cdeaba --- /dev/null +++ b/tools/skqp/src/skqp_model.h @@ -0,0 +1,58 @@ +/* + * Copyright 2018 Google Inc. + * + * Use of this source code is governed by a BSD-style license that can be + * found in the LICENSE file. + */ +#ifndef skqp_model_DEFINED +#define skqp_model_DEFINED + +#include <cstdint> +#include <string> + +#include "SkBitmap.h" + +#include "skqp.h" + +class SkQPAssetManager; +class SkStreamAsset; + +namespace skqp { + +/** Prefered colortype for comparing test outcomes. */ +constexpr SkColorType kColorType = kRGBA_8888_SkColorType; + +/** Prefered alphatype for comparing test outcomes. */ +constexpr SkAlphaType kAlphaType = kUnpremul_SkAlphaType; + +/** Where to find the maximum and minimum of the model. */ +constexpr char kMaxPngPath[] = "max.png"; +constexpr char kMinPngPath[] = "min.png"; + +struct ModelResult { + SkBitmap fErrors; // Correct pixels are white, failing pixels scale from black + // (1 value off) to red (255 off in some channel). + sk_sp<SkData> fMinPng; // original model data, PNG encoded image. + sk_sp<SkData> fMaxPng; // original model data, PNG encoded image. + SkQP::RenderOutcome fOutcome; + std::string fErrorString; // if non-empty, an error occured. +}; + +SkQP::RenderOutcome Check(const SkPixmap& minImg, + const SkPixmap& maxImg, + const SkPixmap& img, + unsigned tolerance, + SkBitmap* errorOut); + +/** Check if the given test image matches the expected results. + + @param name the name of the rendering test that produced the image + @param image the image to be tested. Should be kRGBA_8888_SkColorType + and kUnpremul_SkAlphaType. + @param assetManager provides model data files +*/ + +ModelResult CheckAgainstModel(const char* name, const SkPixmap& image, + SkQPAssetManager* assetManager); +} +#endif // skqp_model_DEFINED diff --git a/tools/skqp/test_apk.sh b/tools/skqp/test_apk.sh index 039f529cfa..1938f8816d 100755 --- a/tools/skqp/test_apk.sh +++ b/tools/skqp/test_apk.sh @@ -24,7 +24,7 @@ if ! [ -f "$APK" ]; then fi ARGS='' -if [ "$#" -gt 1 ]; then +if [ "$#" -gt 0 ]; then ARGS="-e class org.skia.skqp.SkQPRunner#${1}" shift for arg; do @@ -34,26 +34,16 @@ fi TDIR="$(mktemp -d "${TMPDIR:-/tmp}/skqp_report.XXXXXXXXXX")" -filter() { - local re='^.*org\.skia\.skqp: output written to "\([^"]*\)".*$' - while IFS='' read -r line ; do - if printf '%s\n' "$line" | grep -q "$re"; then - D="$(printf '%s\n' "$line" | sed -n "s/${re}/\1/p")" - echo "$D" > "${TDIR}/loc" - fi - printf '%s\n' "$line" | sed 's/^[0-9-]\+ [0-9.:]\+ [0-9]\+ [0-9]\+//' - done -} - adb install -r "$APK" || exit 2 adb logcat -c -adb logcat TestRunner org.skia.skqp skia DEBUG "*:S" | tee "${TDIR}/logcat.txt" | filter & +adb logcat TestRunner org.skia.skqp skia DEBUG "*:S" | tee "${TDIR}/logcat.txt" & LOGCAT_PID=$! ADBSHELL_PID='' trap 'kill $LOGCAT_PID; kill $ADBSHELL_PID' INT +printf '\n%s\n\n' "adb shell am instrument $ARGS -w org.skia.skqp" adb shell am instrument $ARGS -w org.skia.skqp \ > "${TDIR}/stdout.txt" \ 2> "${TDIR}/stderr.txt" & @@ -65,26 +55,31 @@ kill $LOGCAT_PID printf '\nTEST OUTPUT IS IN: "%s"\n\n' "$TDIR" -if ! [ -f "${TDIR}/loc" ]; then exit 2; fi - -ODIR="$(cat "${TDIR}/loc")/skqp_report" +SED_CMD='s/^.* org.skia.skqp: output written to "\([^"]*\)".*$/\1/p' +ODIR="$(sed -n "$SED_CMD" "${TDIR}/logcat.txt" | head -1)" if ! adb shell "test -d '$ODIR'" ; then echo 'missing output :(' exit 3 fi -adb pull "${ODIR}/out.csv" "${ODIR}/report.html" "${ODIR}/images" "${TDIR}/" -REPORT="$TDIR/report.html" -grep 'f(.*;' "$REPORT" -echo "$REPORT" -case "$(uname)" in - Linux) - [ "$DISPLAY" ] && xdg-open "$REPORT" & - sleep 1 - ;; - Darwin) - open "$REPORT" & - ;; -esac +odir_basename="$(basename "$ODIR")" + +adb pull "${ODIR}" "${TDIR}/${odir_basename}" +REPORT="${TDIR}/${odir_basename}/report.html" + +open_file() { + case "$(uname)" in + Linux) [ "$DISPLAY" ] && xdg-open "$@" > /dev/null 2>&1 & ;; + Darwin) open "$@" & ;; + esac +} + +if [ -f "$REPORT" ]; then + grep 'f(.*;' "$REPORT" + echo "$REPORT" + open_file "$REPORT" +else + echo "$TDIR" +fi diff --git a/tools/skqp/upload_model b/tools/skqp/upload_model index cd9554c671..b1058e6247 100755 --- a/tools/skqp/upload_model +++ b/tools/skqp/upload_model @@ -7,7 +7,7 @@ set -e BASE_DIR='platform_tools/android/apps/skqp/src/main/assets' -PATH_LIST='gmkb skqp' +PATH_LIST='gmkb' BUCKET=skia-skqp-assets EXTANT="$(mktemp "${TMPDIR:-/tmp}/extant.XXXXXXXXXX")" |