1 files changed, 405 insertions, 0 deletions
diff --git a/projects/SelfTest/IntrospectiveTests/InternalBenchmark.tests.cpp b/projects/SelfTest/IntrospectiveTests/InternalBenchmark.tests.cpp
new file mode 100644
index 00000000..d17998d8
--- /dev/null
+++ b/projects/SelfTest/IntrospectiveTests/InternalBenchmark.tests.cpp
@@ -0,0 +1,405 @@
+/*
+ *  Created by Joachim on 16/04/2019.
+ *  Adapted from donated nonius code.
+ *
+ *  Distributed under the Boost Software License, Version 1.0. (See accompanying
+ *  file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
+ */
+
+#include "catch.hpp"
+#if defined(CATCH_CONFIG_ENABLE_BENCHMARKING)
+namespace {
+    struct manual_clock {
+    public:
+        using duration = std::chrono::nanoseconds;
+        using time_point = std::chrono::time_point<manual_clock, duration>;
+        using rep = duration::rep;
+        using period = duration::period;
+        enum { is_steady = true };
+
+        static time_point now() {
+            return time_point(duration(tick()));
+        }
+
+        static void advance(int ticks = 1) {
+            tick() += ticks;
+        }
+
+    private:
+        static rep& tick() {
+            static rep the_tick = 0;
+            return the_tick;
+        }
+    };
+
+    struct counting_clock {
+    public:
+        using duration = std::chrono::nanoseconds;
+        using time_point = std::chrono::time_point<counting_clock, duration>;
+        using rep = duration::rep;
+        using period = duration::period;
+        enum { is_steady = true };
+
+        static time_point now() {
+            static rep ticks = 0;
+            return time_point(duration(ticks += rate()));
+        }
+
+        static void set_rate(rep new_rate) { rate() = new_rate; }
+
+    private:
+        static rep& rate() {
+            static rep the_rate = 1;
+            return the_rate;
+        }
+    };
+
+    struct TestChronometerModel : Catch::Benchmark::Detail::ChronometerConcept {
+        int started = 0;
+        int finished = 0;
+
+        void start() override { ++started; }
+        void finish() override { ++finished; }
+    };
+} // namespace
+
+TEST_CASE("warmup", "[benchmark]") {
+    auto rate = 1000;
+    counting_clock::set_rate(rate);
+
+    auto start = counting_clock::now();
+    auto iterations = Catch::Benchmark::Detail::warmup<counting_clock>();
+    auto end = counting_clock::now();
+
+    REQUIRE((iterations * rate) > Catch::Benchmark::Detail::warmup_time.count());
+    REQUIRE((end - start) > Catch::Benchmark::Detail::warmup_time);
+}
+
+TEST_CASE("resolution", "[benchmark]") {
+    auto rate = 1000;
+    counting_clock::set_rate(rate);
+
+    size_t count = 10;
+    auto res = Catch::Benchmark::Detail::resolution<counting_clock>(static_cast<int>(count));
+
+    REQUIRE(res.size() == count);
+
+    for (size_t i = 1; i < count; ++i) {
+        REQUIRE(res[i] == rate);
+    }
+}
+
+TEST_CASE("estimate_clock_resolution", "[benchmark]") {
+    auto rate = 1000;
+    counting_clock::set_rate(rate);
+
+    int iters = 160000;
+    auto res = Catch::Benchmark::Detail::estimate_clock_resolution<counting_clock>(iters);
+
+    REQUIRE(res.mean.count() == rate);
+    REQUIRE(res.outliers.total() == 0);
+}
+
+TEST_CASE("benchmark function call", "[benchmark]") {
+    SECTION("without chronometer") {
+        auto called = 0;
+        auto model = TestChronometerModel{};
+        auto meter = Catch::Benchmark::Chronometer{ model, 1 };
+        auto fn = Catch::Benchmark::Detail::BenchmarkFunction{ [&] {
+                CHECK(model.started == 1);
+                CHECK(model.finished == 0);
+                ++called;
+            } };
+
+        fn(meter);
+
+        CHECK(model.started == 1);
+        CHECK(model.finished == 1);
+        CHECK(called == 1);
+    }
+
+    SECTION("with chronometer") {
+        auto called = 0;
+        auto model = TestChronometerModel{};
+        auto meter = Catch::Benchmark::Chronometer{ model, 1 };
+        auto fn = Catch::Benchmark::Detail::BenchmarkFunction{ [&](Catch::Benchmark::Chronometer) {
+                CHECK(model.started == 0);
+                CHECK(model.finished == 0);
+                ++called;
+            } };
+
+        fn(meter);
+
+        CHECK(model.started == 0);
+        CHECK(model.finished == 0);
+        CHECK(called == 1);
+    }
+}
+
+TEST_CASE("uniform samples", "[benchmark]") {
+    std::vector<double> samples(100);
+    std::fill(samples.begin(), samples.end(), 23);
+
+    using it = std::vector<double>::iterator;
+    auto e = Catch::Benchmark::Detail::bootstrap(0.95, samples.begin(), samples.end(), samples, [](it a, it b) {
+        auto sum = std::accumulate(a, b, 0.);
+        return sum / (b - a);
+    });
+    CHECK(e.point == 23);
+    CHECK(e.upper_bound == 23);
+    CHECK(e.lower_bound == 23);
+    CHECK(e.confidence_interval == 0.95);
+}
+
+
+TEST_CASE("normal_cdf", "[benchmark]") {
+    using Catch::Benchmark::Detail::normal_cdf;
+    CHECK(normal_cdf(0.000000) == Approx(0.50000000000000000));
+    CHECK(normal_cdf(1.000000) == Approx(0.84134474606854293));
+    CHECK(normal_cdf(-1.000000) == Approx(0.15865525393145705));
+    CHECK(normal_cdf(2.809729) == Approx(0.99752083845315409));
+    CHECK(normal_cdf(-1.352570) == Approx(0.08809652095066035));
+}
+
+TEST_CASE("erfc_inv", "[benchmark]") {
+    using Catch::Benchmark::Detail::erfc_inv;
+    CHECK(erfc_inv(1.103560) == Approx(-0.09203687623843015));
+    CHECK(erfc_inv(1.067400) == Approx(-0.05980291115763361));
+    CHECK(erfc_inv(0.050000) == Approx(1.38590382434967796));
+}
+
+TEST_CASE("normal_quantile", "[benchmark]") {
+    using Catch::Benchmark::Detail::normal_quantile;
+    CHECK(normal_quantile(0.551780) == Approx(0.13015979861484198));
+    CHECK(normal_quantile(0.533700) == Approx(0.08457408802851875));
+    CHECK(normal_quantile(0.025000) == Approx(-1.95996398454005449));
+}
+
+
+TEST_CASE("mean", "[benchmark]") {
+    std::vector<double> x{ 10., 20., 14., 16., 30., 24. };
+
+    auto m = Catch::Benchmark::Detail::mean(x.begin(), x.end());
+
+    REQUIRE(m == 19.);
+}
+
+TEST_CASE("weighted_average_quantile", "[benchmark]") {
+    std::vector<double> x{ 10., 20., 14., 16., 30., 24. };
+
+    auto q1 = Catch::Benchmark::Detail::weighted_average_quantile(1, 4, x.begin(), x.end());
+    auto med = Catch::Benchmark::Detail::weighted_average_quantile(1, 2, x.begin(), x.end());
+    auto q3 = Catch::Benchmark::Detail::weighted_average_quantile(3, 4, x.begin(), x.end());
+
+    REQUIRE(q1 == 14.5);
+    REQUIRE(med == 18.);
+    REQUIRE(q3 == 23.);
+}
+
+TEST_CASE("classify_outliers", "[benchmark]") {
+    auto require_outliers = [](Catch::Benchmark::OutlierClassification o, int los, int lom, int him, int his) {
+        REQUIRE(o.low_severe == los);
+        REQUIRE(o.low_mild == lom);
+        REQUIRE(o.high_mild == him);
+        REQUIRE(o.high_severe == his);
+        REQUIRE(o.total() == los + lom + him + his);
+    };
+
+    SECTION("none") {
+        std::vector<double> x{ 10., 20., 14., 16., 30., 24. };
+
+        auto o = Catch::Benchmark::Detail::classify_outliers(x.begin(), x.end());
+
+        REQUIRE(o.samples_seen == static_cast<int>(x.size()));
+        require_outliers(o, 0, 0, 0, 0);
+    }
+    SECTION("low severe") {
+        std::vector<double> x{ -12., 20., 14., 16., 30., 24. };
+
+        auto o = Catch::Benchmark::Detail::classify_outliers(x.begin(), x.end());
+
+        REQUIRE(o.samples_seen == static_cast<int>(x.size()));
+        require_outliers(o, 1, 0, 0, 0);
+    }
+    SECTION("low mild") {
+        std::vector<double> x{ 1., 20., 14., 16., 30., 24. };
+
+        auto o = Catch::Benchmark::Detail::classify_outliers(x.begin(), x.end());
+
+        REQUIRE(o.samples_seen == static_cast<int>(x.size()));
+        require_outliers(o, 0, 1, 0, 0);
+    }
+    SECTION("high mild") {
+        std::vector<double> x{ 10., 20., 14., 16., 36., 24. };
+
+        auto o = Catch::Benchmark::Detail::classify_outliers(x.begin(), x.end());
+
+        REQUIRE(o.samples_seen == static_cast<int>(x.size()));
+        require_outliers(o, 0, 0, 1, 0);
+    }
+    SECTION("high severe") {
+        std::vector<double> x{ 10., 20., 14., 16., 49., 24. };
+
+        auto o = Catch::Benchmark::Detail::classify_outliers(x.begin(), x.end());
+
+        REQUIRE(o.samples_seen == static_cast<int>(x.size()));
+        require_outliers(o, 0, 0, 0, 1);
+    }
+    SECTION("mixed") {
+        std::vector<double> x{ -20., 20., 14., 16., 39., 24. };
+
+        auto o = Catch::Benchmark::Detail::classify_outliers(x.begin(), x.end());
+
+        REQUIRE(o.samples_seen == static_cast<int>(x.size()));
+        require_outliers(o, 1, 0, 1, 0);
+    }
+}
+
+TEST_CASE("analyse", "[benchmark]") {
+    Catch::ConfigData data{};
+    data.benchmarkConfidenceInterval = 0.95;
+    data.benchmarkNoAnalysis = false;
+    data.benchmarkResamples = 1000;
+    data.benchmarkSamples = 99;
+    Catch::Config config{data};
+
+    using Duration = Catch::Benchmark::FloatDuration<Catch::Benchmark::default_clock>;
+
+    Catch::Benchmark::Environment<Duration> env;
+    std::vector<Duration> samples(99);
+    for (size_t i = 0; i < samples.size(); ++i) {
+        samples[i] = Duration(23 + (i % 3 - 1));
+    }
+
+    auto analysis = Catch::Benchmark::Detail::analyse(config, env, samples.begin(), samples.end());
+    CHECK(analysis.mean.point.count() == 23);
+    CHECK(analysis.mean.lower_bound.count() < 23);
+    CHECK(analysis.mean.lower_bound.count() > 22);
+    CHECK(analysis.mean.upper_bound.count() > 23);
+    CHECK(analysis.mean.upper_bound.count() < 24);
+
+    CHECK(analysis.standard_deviation.point.count() > 0.5);
+    CHECK(analysis.standard_deviation.point.count() < 1);
+    CHECK(analysis.standard_deviation.lower_bound.count() > 0.5);
+    CHECK(analysis.standard_deviation.lower_bound.count() < 1);
+    CHECK(analysis.standard_deviation.upper_bound.count() > 0.5);
+    CHECK(analysis.standard_deviation.upper_bound.count() < 1);
+
+    CHECK(analysis.outliers.total() == 0);
+    CHECK(analysis.outliers.low_mild == 0);
+    CHECK(analysis.outliers.low_severe == 0);
+    CHECK(analysis.outliers.high_mild == 0);
+    CHECK(analysis.outliers.high_severe == 0);
+    CHECK(analysis.outliers.samples_seen == samples.size());
+
+    CHECK(analysis.outlier_variance < 0.5);
+    CHECK(analysis.outlier_variance > 0);
+}
+
+TEST_CASE("analyse no analysis", "[benchmark]") {
+    Catch::ConfigData data{};
+    data.benchmarkConfidenceInterval = 0.95;
+    data.benchmarkNoAnalysis = true;
+    data.benchmarkResamples = 1000;
+    data.benchmarkSamples = 99;
+    Catch::Config config{ data };
+
+    using Duration = Catch::Benchmark::FloatDuration<Catch::Benchmark::default_clock>;
+
+    Catch::Benchmark::Environment<Duration> env;
+    std::vector<Duration> samples(99);
+    for (size_t i = 0; i < samples.size(); ++i) {
+        samples[i] = Duration(23 + (i % 3 - 1));
+    }
+
+    auto analysis = Catch::Benchmark::Detail::analyse(config, env, samples.begin(), samples.end());
+    CHECK(analysis.mean.point.count() == 23);
+    CHECK(analysis.mean.lower_bound.count() == 23);
+    CHECK(analysis.mean.upper_bound.count() == 23);
+
+    CHECK(analysis.standard_deviation.point.count() == 0);
+    CHECK(analysis.standard_deviation.lower_bound.count() == 0);
+    CHECK(analysis.standard_deviation.upper_bound.count() == 0);
+
+    CHECK(analysis.outliers.total() == 0);
+    CHECK(analysis.outliers.low_mild == 0);
+    CHECK(analysis.outliers.low_severe == 0);
+    CHECK(analysis.outliers.high_mild == 0);
+    CHECK(analysis.outliers.high_severe == 0);
+    CHECK(analysis.outliers.samples_seen == 0);
+
+    CHECK(analysis.outlier_variance == 0);
+}
+
+TEST_CASE("run_for_at_least, int", "[benchmark]") {
+    manual_clock::duration time(100);
+
+    int old_x = 1;
+    auto Timing = Catch::Benchmark::Detail::run_for_at_least<manual_clock>(time, 1, [&old_x](int x) -> int {
+        CHECK(x >= old_x);
+        manual_clock::advance(x);
+        old_x = x;
+        return x + 17;
+    });
+
+    REQUIRE(Timing.elapsed >= time);
+    REQUIRE(Timing.result == Timing.iterations + 17);
+    REQUIRE(Timing.iterations >= time.count());
+}
+
+TEST_CASE("run_for_at_least, chronometer", "[benchmark]") {
+    manual_clock::duration time(100);
+
+    int old_runs = 1;
+    auto Timing = Catch::Benchmark::Detail::run_for_at_least<manual_clock>(time, 1, [&old_runs](Catch::Benchmark::Chronometer meter) -> int {
+        CHECK(meter.runs() >= old_runs);
+        manual_clock::advance(100);
+        meter.measure([] {
+            manual_clock::advance(1);
+        });
+        old_runs = meter.runs();
+        return meter.runs() + 17;
+    });
+
+    REQUIRE(Timing.elapsed >= time);
+    REQUIRE(Timing.result == Timing.iterations + 17);
+    REQUIRE(Timing.iterations >= time.count());
+}
+
+
+TEST_CASE("measure", "[benchmark]") {
+    auto r = Catch::Benchmark::Detail::measure<manual_clock>([](int x) -> int {
+        CHECK(x == 17);
+        manual_clock::advance(42);
+        return 23;
+    }, 17);
+    auto s = Catch::Benchmark::Detail::measure<manual_clock>([](int x) -> int {
+        CHECK(x == 23);
+        manual_clock::advance(69);
+        return 17;
+    }, 23);
+
+    CHECK(r.elapsed.count() == 42);
+    CHECK(r.result == 23);
+    CHECK(r.iterations == 1);
+
+    CHECK(s.elapsed.count() == 69);
+    CHECK(s.result == 17);
+    CHECK(s.iterations == 1);
+}
+
+TEST_CASE("run benchmark", "[benchmark]") {
+    counting_clock::set_rate(1000);
+    auto start = counting_clock::now();
+    
+    Catch::Benchmark::Benchmark bench{ "Test Benchmark", [](Catch::Benchmark::Chronometer meter) {
+        counting_clock::set_rate(100000);
+        meter.measure([] { return counting_clock::now(); });
+    } };
+
+    bench.run<counting_clock>();
+    auto end = counting_clock::now();
+
+    CHECK((end - start).count() == 2867251000);
+}
+#endif // CATCH_CONFIG_ENABLE_BENCHMARKING