diff options
Diffstat (limited to 'src/test/java/com/code_intelligence/jazzer')
96 files changed, 12982 insertions, 0 deletions
diff --git a/src/test/java/com/code_intelligence/jazzer/api/AutofuzzTest.java b/src/test/java/com/code_intelligence/jazzer/api/AutofuzzTest.java new file mode 100644 index 00000000..ee192d46 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/api/AutofuzzTest.java @@ -0,0 +1,107 @@ +// Copyright 2021 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.api; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.fail; + +import java.util.Arrays; +import java.util.Collections; +import org.junit.Test; + +public class AutofuzzTest { + public interface UnimplementedInterface {} + + public interface ImplementedInterface {} + public static class ImplementingClass implements ImplementedInterface {} + + private static boolean implIsNotNull(ImplementedInterface impl) { + return impl != null; + } + + private static boolean implIsNotNull(UnimplementedInterface impl) { + return impl != null; + } + + private static void checkAllTheArguments( + String arg1, int arg2, byte arg3, ImplementedInterface arg4) { + if (!arg1.equals("foobar") || arg2 != 42 || arg3 != 5 || arg4 == null) { + throw new IllegalArgumentException(); + } + } + + @Test + public void testConsume() { + FuzzedDataProvider data = CannedFuzzedDataProvider.create( + Arrays.asList((byte) 1 /* do not return null */, 0 /* first class on the classpath */, + (byte) 1 /* do not return null */, 0 /* first constructor */)); + ImplementedInterface result = Autofuzz.consume(data, ImplementedInterface.class); + assertNotNull(result); + } + + @Test + public void testConsumeFailsWithoutException() { + FuzzedDataProvider data = CannedFuzzedDataProvider.create(Collections.singletonList( + (byte) 1 /* do not return null without searching for implementing classes */)); + assertNull(Autofuzz.consume(data, UnimplementedInterface.class)); + } + + @Test + public void testAutofuzz() { + FuzzedDataProvider data = CannedFuzzedDataProvider.create( + Arrays.asList((byte) 1 /* do not return null */, 0 /* first class on the classpath */, + (byte) 1 /* do not return null */, 0 /* first constructor */)); + assertEquals(Boolean.TRUE, + Autofuzz.autofuzz(data, (Function1<ImplementedInterface, ?>) AutofuzzTest::implIsNotNull)); + } + + @Test + public void testAutofuzzFailsWithException() { + FuzzedDataProvider data = CannedFuzzedDataProvider.create( + Collections.singletonList((byte) 1 /* do not return null */)); + try { + Autofuzz.autofuzz(data, (Function1<UnimplementedInterface, ?>) AutofuzzTest::implIsNotNull); + } catch (AutofuzzConstructionException e) { + // Pass. + return; + } + fail("should have thrown an AutofuzzConstructionException"); + } + + @Test + public void testAutofuzzConsumer() { + FuzzedDataProvider data = CannedFuzzedDataProvider.create( + Arrays.asList((byte) 1 /* do not return null */, 6 /* string length */, "foobar", 42, + (byte) 5, (byte) 1 /* do not return null */, 0 /* first class on the classpath */, + (byte) 1 /* do not return null */, 0 /* first constructor */)); + Autofuzz.autofuzz(data, AutofuzzTest::checkAllTheArguments); + } + + @Test + public void testAutofuzzConsumerThrowsException() { + FuzzedDataProvider data = + CannedFuzzedDataProvider.create(Arrays.asList((byte) 1 /* do not return null */, + 6 /* string length */, "foobar", 42, (byte) 5, (byte) 0 /* *do* return null */)); + try { + Autofuzz.autofuzz(data, AutofuzzTest::checkAllTheArguments); + } catch (IllegalArgumentException e) { + // Pass. + return; + } + fail("should have thrown an IllegalArgumentException"); + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/api/BUILD.bazel b/src/test/java/com/code_intelligence/jazzer/api/BUILD.bazel new file mode 100644 index 00000000..86014c7b --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/api/BUILD.bazel @@ -0,0 +1,22 @@ +java_test( + name = "AutofuzzTest", + size = "small", + srcs = [ + "AutofuzzTest.java", + ], + env = { + # Also consider implementing classes from com.code_intelligence.jazzer.*. + "JAZZER_AUTOFUZZ_TESTING": "1", + }, + test_class = "com.code_intelligence.jazzer.api.AutofuzzTest", + runtime_deps = [ + "//src/main/java/com/code_intelligence/jazzer/autofuzz", + # Needed for JazzerInternal. + "//src/main/java/com/code_intelligence/jazzer/runtime", + ], + deps = [ + "//src/main/java/com/code_intelligence/jazzer/api", + "//src/main/native/com/code_intelligence/jazzer/driver:jazzer_driver", + "@maven//:junit_junit", + ], +) diff --git a/src/test/java/com/code_intelligence/jazzer/autofuzz/AutofuzzCodegenVisitorTest.java b/src/test/java/com/code_intelligence/jazzer/autofuzz/AutofuzzCodegenVisitorTest.java new file mode 100644 index 00000000..814292e6 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/autofuzz/AutofuzzCodegenVisitorTest.java @@ -0,0 +1,43 @@ +/* + * Copyright 2022 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.code_intelligence.jazzer.autofuzz; + +import static com.code_intelligence.jazzer.autofuzz.AutofuzzCodegenVisitor.escapeForLiteral; +import static org.junit.Assert.assertEquals; + +import org.junit.Test; + +public class AutofuzzCodegenVisitorTest { + @Test + public void escapeForLiteralTest() { + assertEquals("\\t", escapeForLiteral("\t")); + assertEquals("\\\\\\t", escapeForLiteral("\\\t")); + assertEquals("\\b", escapeForLiteral("\b")); + assertEquals("\\\\\\b", escapeForLiteral("\\\b")); + assertEquals("\\n", escapeForLiteral("\n")); + assertEquals("\\\\\\n", escapeForLiteral("\\\n")); + assertEquals("\\r", escapeForLiteral("\r")); + assertEquals("\\\\\\r", escapeForLiteral("\\\r")); + assertEquals("\\f", escapeForLiteral("\f")); + assertEquals("\\\\\\f", escapeForLiteral("\\\f")); + assertEquals("\\'", escapeForLiteral("'")); + assertEquals("\\\\\\'", escapeForLiteral("\\'")); + assertEquals("\\\"", escapeForLiteral("\"")); + assertEquals("\\\\\\\"", escapeForLiteral("\\\"")); + assertEquals("\\\\", escapeForLiteral("\\")); + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/autofuzz/BUILD.bazel b/src/test/java/com/code_intelligence/jazzer/autofuzz/BUILD.bazel new file mode 100644 index 00000000..a5ee59b1 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/autofuzz/BUILD.bazel @@ -0,0 +1,93 @@ +java_test( + name = "MetaTest", + size = "small", + srcs = [ + "MetaTest.java", + ], + env = { + "JAZZER_AUTOFUZZ_DEBUG": "1", + # Also consider implementing classes from com.code_intelligence.jazzer.*. + "JAZZER_AUTOFUZZ_TESTING": "1", + }, + test_class = "com.code_intelligence.jazzer.autofuzz.MetaTest", + deps = [ + ":test_helpers", + "//src/main/java/com/code_intelligence/jazzer/api", + "//src/main/java/com/code_intelligence/jazzer/autofuzz", + "@maven//:com_mikesamuel_json_sanitizer", + "@maven//:junit_junit", + ], +) + +java_test( + name = "InterfaceCreationTest", + size = "small", + srcs = [ + "InterfaceCreationTest.java", + ], + env = { + "JAZZER_AUTOFUZZ_DEBUG": "1", + # Also consider implementing classes from com.code_intelligence.jazzer.*. + "JAZZER_AUTOFUZZ_TESTING": "1", + }, + test_class = "com.code_intelligence.jazzer.autofuzz.InterfaceCreationTest", + deps = [ + ":test_helpers", + "@maven//:junit_junit", + ], +) + +java_test( + name = "BuilderPatternTest", + size = "small", + srcs = [ + "BuilderPatternTest.java", + ], + env = { + "JAZZER_AUTOFUZZ_DEBUG": "1", + }, + test_class = "com.code_intelligence.jazzer.autofuzz.BuilderPatternTest", + deps = [ + ":test_helpers", + "@maven//:junit_junit", + ], +) + +java_test( + name = "SettersTest", + size = "small", + srcs = [ + "SettersTest.java", + ], + env = { + "JAZZER_AUTOFUZZ_DEBUG": "1", + }, + test_class = "com.code_intelligence.jazzer.autofuzz.SettersTest", + deps = [ + ":test_helpers", + "//src/test/java/com/code_intelligence/jazzer/autofuzz/testdata:test_data", + "@maven//:junit_junit", + ], +) + +java_test( + name = "AutofuzzCodegenVisitorTest", + srcs = [ + "AutofuzzCodegenVisitorTest.java", + ], + test_class = "com.code_intelligence.jazzer.autofuzz.AutofuzzCodegenVisitorTest", + deps = [ + "//src/main/java/com/code_intelligence/jazzer/autofuzz", + "@maven//:junit_junit", + ], +) + +java_library( + name = "test_helpers", + srcs = ["TestHelpers.java"], + deps = [ + "//src/main/java/com/code_intelligence/jazzer/api", + "//src/main/java/com/code_intelligence/jazzer/autofuzz", + "@maven//:junit_junit", + ], +) diff --git a/src/test/java/com/code_intelligence/jazzer/autofuzz/BuilderPatternTest.java b/src/test/java/com/code_intelligence/jazzer/autofuzz/BuilderPatternTest.java new file mode 100644 index 00000000..a602d712 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/autofuzz/BuilderPatternTest.java @@ -0,0 +1,103 @@ +// Copyright 2021 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.autofuzz; + +import static com.code_intelligence.jazzer.autofuzz.TestHelpers.consumeTestCase; + +import java.util.Arrays; +import java.util.Objects; +import org.junit.Test; + +class Employee { + private final String firstName; + private final String lastName; + private final String jobTitle; + private final int age; + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + Employee hero = (Employee) o; + return age == hero.age && Objects.equals(firstName, hero.firstName) + && Objects.equals(lastName, hero.lastName) && Objects.equals(jobTitle, hero.jobTitle); + } + + @Override + public int hashCode() { + return Objects.hash(firstName, lastName, jobTitle, age); + } + + private Employee(Builder builder) { + this.jobTitle = builder.jobTitle; + this.firstName = builder.firstName; + this.lastName = builder.lastName; + this.age = builder.age; + } + + public static class Builder { + private final String firstName; + private final String lastName; + private String jobTitle; + private int age; + + public Builder(String firstName, String lastName) { + this.firstName = firstName; + this.lastName = lastName; + } + + public Builder withAge(int age) { + this.age = age; + return this; + } + + public Builder withJobTitle(String jobTitle) { + this.jobTitle = jobTitle; + return this; + } + + public Employee build() { + return new Employee(this); + } + } +} + +public class BuilderPatternTest { + @Test + public void testBuilderPattern() { + consumeTestCase(new Employee.Builder("foo", "bar").withAge(20).withJobTitle("baz").build(), + "new com.code_intelligence.jazzer.autofuzz.Employee.Builder(\"foo\", \"bar\").withAge(20).withJobTitle(\"baz\").build()", + Arrays.asList((byte) 1, // do not return null + 0, // Select the first Builder + 2, // Select two Builder methods returning a builder object (fluent design) + 0, // Select the first build method + 0, // pick the first remaining builder method (withAge) + 0, // pick the first remaining builder method (withJobTitle) + 0, // pick the first build method + (byte) 1, // do not return null + 6, // remaining bytes + "foo", // firstName + (byte) 1, // do not return null + 6, // remaining bytes + "bar", // lastName + 20, // age + (byte) 1, // do not return null + 6, // remaining bytes + "baz" // jobTitle + )); + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/autofuzz/InterfaceCreationTest.java b/src/test/java/com/code_intelligence/jazzer/autofuzz/InterfaceCreationTest.java new file mode 100644 index 00000000..3fecb973 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/autofuzz/InterfaceCreationTest.java @@ -0,0 +1,110 @@ +// Copyright 2021 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.autofuzz; + +import static com.code_intelligence.jazzer.autofuzz.TestHelpers.consumeTestCase; + +import java.util.Arrays; +import java.util.Objects; +import org.junit.Test; + +public class InterfaceCreationTest { + public interface InterfaceA { + void foo(); + + void bar(); + } + + public static abstract class ClassA1 implements InterfaceA { + @Override + public void foo() {} + } + + public static class ClassB1 extends ClassA1 { + int n; + + public ClassB1(int _n) { + n = _n; + } + + @Override + public void bar() {} + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + ClassB1 classB1 = (ClassB1) o; + return n == classB1.n; + } + + @Override + public int hashCode() { + return Objects.hash(n); + } + } + + public static class ClassB2 implements InterfaceA { + String s; + + public ClassB2(String _s) { + s = _s; + } + + @Override + public void foo() {} + + @Override + public void bar() {} + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + ClassB2 classB2 = (ClassB2) o; + return Objects.equals(s, classB2.s); + } + + @Override + public int hashCode() { + return Objects.hash(s); + } + } + @Test + public void testConsumeInterface() { + consumeTestCase(InterfaceA.class, new ClassB1(5), + "(com.code_intelligence.jazzer.autofuzz.InterfaceCreationTest.InterfaceA) new com.code_intelligence.jazzer.autofuzz.InterfaceCreationTest.ClassB1(5)", + Arrays.asList((byte) 1, // do not return null + 0, // pick ClassB1 + (byte) 1, // do not return null + 0, // pick first constructor + 5 // arg for ClassB1 constructor + )); + consumeTestCase(InterfaceA.class, new ClassB2("test"), + "(com.code_intelligence.jazzer.autofuzz.InterfaceCreationTest.InterfaceA) new com.code_intelligence.jazzer.autofuzz.InterfaceCreationTest.ClassB2(\"test\")", + Arrays.asList((byte) 1, // do not return null + 1, // pick ClassB2 + (byte) 1, // do not return null + 0, // pick first constructor + (byte) 1, // do not return null + 8, // remaining bytes + "test" // arg for ClassB2 constructor + )); + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/autofuzz/MetaTest.java b/src/test/java/com/code_intelligence/jazzer/autofuzz/MetaTest.java new file mode 100644 index 00000000..d2fec3ae --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/autofuzz/MetaTest.java @@ -0,0 +1,219 @@ +// Copyright 2021 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.autofuzz; + +import static com.code_intelligence.jazzer.autofuzz.TestHelpers.autofuzzTestCase; +import static com.code_intelligence.jazzer.autofuzz.TestHelpers.consumeTestCase; +import static org.junit.Assert.assertEquals; + +import com.code_intelligence.jazzer.api.CannedFuzzedDataProvider; +import com.code_intelligence.jazzer.api.FuzzedDataProvider; +import com.google.json.JsonSanitizer; +import java.io.ByteArrayInputStream; +import java.lang.reflect.Type; +import java.util.Arrays; +import java.util.Collections; +import java.util.Map; +import org.junit.Test; + +public class MetaTest { + public enum TestEnum { + FOO, + BAR, + BAZ, + } + + @Test + public void testConsume() throws NoSuchMethodException { + consumeTestCase(5, "5", Collections.singletonList(5)); + consumeTestCase((short) 5, "(short) 5", Collections.singletonList((short) 5)); + consumeTestCase(5L, "5L", Collections.singletonList(5L)); + consumeTestCase(5.0F, "5.0F", Collections.singletonList(5.0F)); + consumeTestCase('\n', "'\\n'", Collections.singletonList('\n')); + consumeTestCase('\'', "'\\''", Collections.singletonList('\'')); + consumeTestCase('\\', "'\\\\'", Collections.singletonList('\\')); + + String testString = "foo\n\t\\\"bar"; + // The expected string is obtained from testString by escaping and wrapping into escaped quotes. + consumeTestCase(testString, "\"foo\\n\\t\\\\\\\"bar\"", + Arrays.asList((byte) 1, // do not return null + testString.length(), testString)); + + consumeTestCase(null, "null", Collections.singletonList((byte) 0)); + + boolean[] testBooleans = new boolean[] {true, false, true}; + consumeTestCase(testBooleans, "new boolean[]{true, false, true}", + Arrays.asList((byte) 1, // do not return null for the array + 2 * 3, testBooleans)); + + char[] testChars = new char[] {'a', '\n', '\''}; + consumeTestCase(testChars, "new char[]{'a', '\\n', '\\''}", + Arrays.asList((byte) 1, // do not return null for the array + 2 * 3 * Character.BYTES + Character.BYTES, testChars[0], 2 * 3 * Character.BYTES, + 2 * 3 * Character.BYTES, // remaining bytes, 2 times what is needed for 3 chars + testChars[1], testChars[2])); + + char[] testNoChars = new char[] {}; + consumeTestCase(testNoChars, "new char[]{}", + Arrays.asList((byte) 1, // do not return null for the array + 0, 'a', 0, 0)); + + short[] testShorts = new short[] {(short) 1, (short) 2, (short) 3}; + consumeTestCase(testShorts, "new short[]{(short) 1, (short) 2, (short) 3}", + Arrays.asList((byte) 1, // do not return null for the array + 2 * 3 * Short.BYTES, // remaining bytes + testShorts)); + + long[] testLongs = new long[] {1L, 2L, 3L}; + consumeTestCase(testLongs, "new long[]{1L, 2L, 3L}", + Arrays.asList((byte) 1, // do not return null for the array + 2 * 3 * Long.BYTES, // remaining bytes + testLongs)); + + consumeTestCase(new String[] {"foo", "bar", "foo\nbar"}, + "new java.lang.String[]{\"foo\", \"bar\", \"foo\\nbar\"}", + Arrays.asList((byte) 1, // do not return null for the array + 32, // remaining bytes + (byte) 1, // do not return null for the string + 31, // remaining bytes + "foo", + 28, // remaining bytes + 28, // array length + (byte) 1, // do not return null for the string + 27, // remaining bytes + "bar", + (byte) 1, // do not return null for the string + 23, // remaining bytes + "foo\nbar")); + + byte[] testInputStreamBytes = new byte[] {(byte) 1, (byte) 2, (byte) 3}; + consumeTestCase(new ByteArrayInputStream(testInputStreamBytes), + "new java.io.ByteArrayInputStream(new byte[]{(byte) 1, (byte) 2, (byte) 3})", + Arrays.asList((byte) 1, // do not return null for the InputStream + 2 * 3, // remaining bytes (twice the desired length) + testInputStreamBytes)); + + consumeTestCase(TestEnum.BAR, + String.format("%s.%s", TestEnum.class.getName(), TestEnum.BAR.name()), + Arrays.asList((byte) 1, // do not return null for the enum value + 1 /* second value */ + )); + + consumeTestCase(YourAverageJavaClass.class, + "com.code_intelligence.jazzer.autofuzz.YourAverageJavaClass.class", + Collections.singletonList((byte) 1)); + + Type stringStringMapType = + MetaTest.class.getDeclaredMethod("returnsStringStringMap").getGenericReturnType(); + Map<String, String> expectedMap = + java.util.stream.Stream + .of(new java.util.AbstractMap.SimpleEntry<>("key0", "value0"), + new java.util.AbstractMap.SimpleEntry<>("key1", "value1"), + new java.util.AbstractMap.SimpleEntry<>("key2", (java.lang.String) null)) + .collect(java.util.HashMap::new, + (map, e) -> map.put(e.getKey(), e.getValue()), java.util.HashMap::putAll); + consumeTestCase(stringStringMapType, expectedMap, + "java.util.stream.Stream.<java.util.AbstractMap.SimpleEntry<java.lang.String, java.lang.String>>of(new java.util.AbstractMap.SimpleEntry<>(\"key0\", \"value0\"), new java.util.AbstractMap.SimpleEntry<>(\"key1\", \"value1\"), new java.util.AbstractMap.SimpleEntry<>(\"key2\", (java.lang.String) null)).collect(java.util.HashMap::new, (map, e) -> map.put(e.getKey(), e.getValue()), java.util.HashMap::putAll)", + Arrays.asList((byte) 1, // do not return null for the map + 32, // remaining bytes + (byte) 1, // do not return null for the string + 31, // remaining bytes + "key0", + (byte) 1, // do not return null for the string + 28, // remaining bytes + "value0", + 28, // remaining bytes + 28, // consumeArrayLength + (byte) 1, // do not return null for the string + 27, // remaining bytes + "key1", + (byte) 1, // do not return null for the string + 23, // remaining bytes + "value1", + (byte) 1, // do not return null for the string + 27, // remaining bytes + "key2", + (byte) 0 // *do* return null for the string + )); + } + + private Map<String, String> returnsStringStringMap() { + throw new IllegalStateException( + "Should not be called, only exists to construct its generic return type"); + } + + public static boolean isFive(int arg) { + return arg == 5; + } + + public static boolean intEquals(int arg1, int arg2) { + return arg1 == arg2; + } + + @Test + public void testAutofuzz() throws NoSuchMethodException { + autofuzzTestCase(true, "com.code_intelligence.jazzer.autofuzz.MetaTest.isFive(5)", + MetaTest.class.getMethod("isFive", int.class), Collections.singletonList(5)); + autofuzzTestCase(false, "com.code_intelligence.jazzer.autofuzz.MetaTest.intEquals(5, 4)", + MetaTest.class.getMethod("intEquals", int.class, int.class), Arrays.asList(5, 4)); + autofuzzTestCase("foobar", "(\"foo\").concat(\"bar\")", + String.class.getMethod("concat", String.class), + Arrays.asList((byte) 1, 6, "foo", (byte) 1, 6, "bar")); + autofuzzTestCase("jazzer", "new java.lang.String(\"jazzer\")", + String.class.getConstructor(String.class), Arrays.asList((byte) 1, 12, "jazzer")); + autofuzzTestCase("\"jazzer\"", "com.google.json.JsonSanitizer.sanitize(\"jazzer\")", + JsonSanitizer.class.getMethod("sanitize", String.class), + Arrays.asList((byte) 1, 12, "jazzer")); + + FuzzedDataProvider data = + CannedFuzzedDataProvider.create(Arrays.asList((byte) 1, // do not return null + 8, // remainingBytes + "buzz")); + assertEquals("fizzbuzz", new Meta(null).autofuzz(data, "fizz" ::concat)); + } + + // Regression test for https://github.com/CodeIntelligenceTesting/jazzer/issues/465. + @Test + public void testPrivateInterface() { + autofuzzTestCase(null, + "com.code_intelligence.jazzer.autofuzz.OpinionatedClass.doStuffWithPrivateInterface(((java.util.function.Supplier<com.code_intelligence.jazzer.autofuzz.OpinionatedClass.PublicImplementation>) (() -> {com.code_intelligence.jazzer.autofuzz.OpinionatedClass.PublicImplementation autofuzzVariable0 = new com.code_intelligence.jazzer.autofuzz.OpinionatedClass.PublicImplementation(); return autofuzzVariable0;})).get())", + OpinionatedClass.class.getDeclaredMethods()[0], + Arrays.asList((byte) 1, // do not return null + 0, // first (and only) class on the classpath + (byte) 1, // do not return null + 0 /* first (and only) constructor*/)); + } + + Class<?>[] returnsClassArray() { + throw new IllegalStateException( + "Should not be called, only exists to construct its generic return type"); + } + + @Test + public void testGetRawType() throws NoSuchMethodException { + Type classArrayType = + MetaTest.class.getDeclaredMethod("returnsClassArray").getGenericReturnType(); + assertEquals(Class[].class, Meta.getRawType(classArrayType)); + } +} + +class OpinionatedClass { + public static void doStuffWithPrivateInterface( + @SuppressWarnings("unused") PrivateInterface thing) {} + + private interface PrivateInterface {} + + public static class PublicImplementation implements PrivateInterface {} +} diff --git a/src/test/java/com/code_intelligence/jazzer/autofuzz/SettersTest.java b/src/test/java/com/code_intelligence/jazzer/autofuzz/SettersTest.java new file mode 100644 index 00000000..7c869531 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/autofuzz/SettersTest.java @@ -0,0 +1,43 @@ +// Copyright 2021 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.autofuzz; + +import static com.code_intelligence.jazzer.autofuzz.TestHelpers.consumeTestCase; + +import com.code_intelligence.jazzer.autofuzz.testdata.EmployeeWithSetters; +import java.util.Arrays; +import org.junit.Test; + +public class SettersTest { + @Test + public void testEmptyConstructorWithSetters() { + EmployeeWithSetters employee = new EmployeeWithSetters(); + employee.setFirstName("foo"); + employee.setAge(26); + + consumeTestCase(employee, + "((java.util.function.Supplier<com.code_intelligence.jazzer.autofuzz.testdata.EmployeeWithSetters>) (() -> {com.code_intelligence.jazzer.autofuzz.testdata.EmployeeWithSetters autofuzzVariable0 = new com.code_intelligence.jazzer.autofuzz.testdata.EmployeeWithSetters(); autofuzzVariable0.setFirstName(\"foo\"); autofuzzVariable0.setAge(26); return autofuzzVariable0;})).get()", + Arrays.asList((byte) 1, // do not return null for EmployeeWithSetters + 0, // pick first constructor + 2, // pick two setters + 1, // pick second setter + 0, // pick first setter + (byte) 1, // do not return null for String + 6, // remaining bytes + "foo", // setFirstName + 26 // setAge + )); + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/autofuzz/TestHelpers.java b/src/test/java/com/code_intelligence/jazzer/autofuzz/TestHelpers.java new file mode 100644 index 00000000..89f9c968 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/autofuzz/TestHelpers.java @@ -0,0 +1,85 @@ +// Copyright 2021 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.autofuzz; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + +import com.code_intelligence.jazzer.api.CannedFuzzedDataProvider; +import com.code_intelligence.jazzer.api.FuzzedDataProvider; +import java.io.ByteArrayInputStream; +import java.lang.reflect.Constructor; +import java.lang.reflect.Executable; +import java.lang.reflect.Method; +import java.lang.reflect.Type; +import java.util.List; + +class TestHelpers { + static void assertGeneralEquals(Object expected, Object actual) { + Class<?> type = expected != null ? expected.getClass() : Object.class; + if (type.isArray()) { + if (type.getComponentType() == boolean.class) { + assertArrayEquals((boolean[]) expected, (boolean[]) actual); + } else if (type.getComponentType() == char.class) { + assertArrayEquals((char[]) expected, (char[]) actual); + } else if (type.getComponentType() == short.class) { + assertArrayEquals((short[]) expected, (short[]) actual); + } else if (type.getComponentType() == long.class) { + assertArrayEquals((long[]) expected, (long[]) actual); + } else { + assertArrayEquals((Object[]) expected, (Object[]) actual); + } + } else if (type == ByteArrayInputStream.class) { + ByteArrayInputStream expectedStream = (ByteArrayInputStream) expected; + ByteArrayInputStream actualStream = (ByteArrayInputStream) actual; + assertArrayEquals(readAllBytes(expectedStream), readAllBytes(actualStream)); + } else { + assertEquals(expected, actual); + } + } + + static void consumeTestCase( + Object expectedResult, String expectedResultString, List<Object> cannedData) { + Class<?> type = expectedResult != null ? expectedResult.getClass() : Object.class; + consumeTestCase(type, expectedResult, expectedResultString, cannedData); + } + + static void consumeTestCase( + Type type, Object expectedResult, String expectedResultString, List<Object> cannedData) { + AutofuzzCodegenVisitor visitor = new AutofuzzCodegenVisitor(); + FuzzedDataProvider data = CannedFuzzedDataProvider.create(cannedData); + assertGeneralEquals(expectedResult, new Meta(null).consume(data, type, visitor)); + assertEquals(expectedResultString, visitor.generate()); + } + + static void autofuzzTestCase(Object expectedResult, String expectedResultString, Executable func, + List<Object> cannedData) { + AutofuzzCodegenVisitor visitor = new AutofuzzCodegenVisitor(); + FuzzedDataProvider data = CannedFuzzedDataProvider.create(cannedData); + if (func instanceof Method) { + assertGeneralEquals(expectedResult, new Meta(null).autofuzz(data, (Method) func, visitor)); + } else { + assertGeneralEquals( + expectedResult, new Meta(null).autofuzz(data, (Constructor<?>) func, visitor)); + } + assertEquals(expectedResultString, visitor.generate()); + } + + private static byte[] readAllBytes(ByteArrayInputStream in) { + byte[] result = new byte[in.available()]; + in.read(result, 0, in.available()); + return result; + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/autofuzz/testdata/BUILD.bazel b/src/test/java/com/code_intelligence/jazzer/autofuzz/testdata/BUILD.bazel new file mode 100644 index 00000000..c2c68803 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/autofuzz/testdata/BUILD.bazel @@ -0,0 +1,5 @@ +java_library( + name = "test_data", + srcs = glob(["*.java"]), + visibility = ["//visibility:public"], +) diff --git a/src/test/java/com/code_intelligence/jazzer/autofuzz/testdata/EmployeeWithSetters.java b/src/test/java/com/code_intelligence/jazzer/autofuzz/testdata/EmployeeWithSetters.java new file mode 100644 index 00000000..2c76a61f --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/autofuzz/testdata/EmployeeWithSetters.java @@ -0,0 +1,56 @@ +// Copyright 2021 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.autofuzz.testdata; + +import java.util.Objects; + +public class EmployeeWithSetters { + private String firstName; + private String lastName; + private String jobTitle; + private int age; + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + EmployeeWithSetters hero = (EmployeeWithSetters) o; + return age == hero.age && Objects.equals(firstName, hero.firstName) + && Objects.equals(lastName, hero.lastName) && Objects.equals(jobTitle, hero.jobTitle); + } + + @Override + public int hashCode() { + return Objects.hash(firstName, lastName, jobTitle, age); + } + + public void setFirstName(String firstName) { + this.firstName = firstName; + } + + public void setLastName(String lastName) { + this.lastName = lastName; + } + + public void setJobTitle(String jobTitle) { + this.jobTitle = jobTitle; + } + + public void setAge(int age) { + this.age = age; + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/driver/BUILD.bazel b/src/test/java/com/code_intelligence/jazzer/driver/BUILD.bazel new file mode 100644 index 00000000..678ed2ba --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/driver/BUILD.bazel @@ -0,0 +1,48 @@ +java_test( + name = "FuzzTargetRunnerTest", + srcs = ["FuzzTargetRunnerTest.java"], + jvm_flags = ["-ea"], + use_testrunner = False, + deps = [ + "//src/main/java/com/code_intelligence/jazzer/agent:agent_installer", + "//src/main/java/com/code_intelligence/jazzer/api", + "//src/main/java/com/code_intelligence/jazzer/api:hooks", + "//src/main/java/com/code_intelligence/jazzer/driver:fuzz_target_finder", + "//src/main/java/com/code_intelligence/jazzer/driver:fuzz_target_holder", + "//src/main/java/com/code_intelligence/jazzer/driver:fuzz_target_runner", + "//src/main/java/com/code_intelligence/jazzer/runtime:coverage_map", + "//src/main/java/com/code_intelligence/jazzer/utils:unsafe_provider", + ], +) + +java_test( + name = "FuzzedDataProviderImplTest", + srcs = ["FuzzedDataProviderImplTest.java"], + use_testrunner = False, + deps = [ + "//src/main/java/com/code_intelligence/jazzer/api", + "//src/main/java/com/code_intelligence/jazzer/driver:fuzzed_data_provider_impl", + ], +) + +java_test( + name = "OptTest", + srcs = ["OptTest.java"], + deps = [ + "//src/main/java/com/code_intelligence/jazzer/driver:opt", + "@maven//:junit_junit", + ], +) + +java_test( + name = "RecordingFuzzedDataProviderTest", + srcs = [ + "RecordingFuzzedDataProviderTest.java", + ], + deps = [ + "//src/main/java/com/code_intelligence/jazzer/api", + "//src/main/java/com/code_intelligence/jazzer/driver:fuzzed_data_provider_impl", + "//src/main/java/com/code_intelligence/jazzer/driver:recording_fuzzed_data_provider", + "@maven//:junit_junit", + ], +) diff --git a/src/test/java/com/code_intelligence/jazzer/driver/FuzzTargetRunnerTest.java b/src/test/java/com/code_intelligence/jazzer/driver/FuzzTargetRunnerTest.java new file mode 100644 index 00000000..e0ff3131 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/driver/FuzzTargetRunnerTest.java @@ -0,0 +1,231 @@ +/* + * Copyright 2022 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.code_intelligence.jazzer.driver; + +import com.code_intelligence.jazzer.agent.AgentInstaller; +import com.code_intelligence.jazzer.api.Jazzer; +import com.code_intelligence.jazzer.runtime.CoverageMap; +import com.code_intelligence.jazzer.utils.UnsafeProvider; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import sun.misc.Unsafe; + +public class FuzzTargetRunnerTest { + private static final Pattern DEDUP_TOKEN_PATTERN = + Pattern.compile("(?m)^DEDUP_TOKEN: ([0-9a-f]{16})(?:\r\n|\r|\n)"); + private static final Unsafe UNSAFE = UnsafeProvider.getUnsafe(); + private static final ByteArrayOutputStream recordedErr = new ByteArrayOutputStream(); + private static final ByteArrayOutputStream recordedOut = new ByteArrayOutputStream(); + private static boolean fuzzerInitializeRan = false; + private static boolean finishedAllNonCrashingRuns = false; + + public static void fuzzerInitialize() { + fuzzerInitializeRan = true; + } + + public static void fuzzerTestOneInput(byte[] data) { + switch (new String(data, StandardCharsets.UTF_8)) { + case "no crash": + CoverageMap.recordCoverage(0); + return; + case "first finding": + CoverageMap.recordCoverage(1); + throw new IllegalArgumentException("first finding"); + case "second finding": + CoverageMap.recordCoverage(2); + Jazzer.reportFindingFromHook(new StackOverflowError("second finding")); + throw new IllegalArgumentException("not reported"); + case "crash": + CoverageMap.recordCoverage(3); + throw new RuntimeException("crash"); + } + } + + public static void fuzzerTearDown() { + try { + String errOutput = new String(recordedErr.toByteArray(), StandardCharsets.UTF_8); + assert errOutput.contains("== Java Exception: java.lang.RuntimeException: crash"); + String outOutput = new String(recordedOut.toByteArray(), StandardCharsets.UTF_8); + assert DEDUP_TOKEN_PATTERN.matcher(outOutput).find(); + + assert finishedAllNonCrashingRuns : "Did not finish all expected runs before crashing"; + assert CoverageMap.getCoveredIds().equals(Stream.of(0, 1, 2, 3).collect(Collectors.toSet())); + assert UNSAFE.getByte(CoverageMap.countersAddress) == 2; + assert UNSAFE.getByte(CoverageMap.countersAddress + 1) == 2; + assert UNSAFE.getByte(CoverageMap.countersAddress + 2) == 2; + assert UNSAFE.getByte(CoverageMap.countersAddress + 3) == 1; + } catch (AssertionError e) { + e.printStackTrace(); + Runtime.getRuntime().halt(1); + } + // FuzzTargetRunner calls _Exit after this function, so the test would fail unless this line is + // executed. Use halt rather than exit to get around FuzzTargetRunner's shutdown hook calling + // fuzzerTearDown, which would otherwise result in a shutdown hook loop. + Runtime.getRuntime().halt(0); + } + + public static void main(String[] args) { + PrintStream recordingErr = new TeeOutputStream(new PrintStream(recordedErr, true), System.err); + System.setErr(recordingErr); + PrintStream recordingOut = new TeeOutputStream(new PrintStream(recordedOut, true), System.out); + System.setOut(recordingOut); + + // Do not instrument any classes. + System.setProperty("jazzer.instrumentation_excludes", "**"); + System.setProperty("jazzer.custom_hook_excludes", "**"); + System.setProperty("jazzer.target_class", FuzzTargetRunnerTest.class.getName()); + // Keep going past all "no crash", "first finding" and "second finding" runs, then crash. + System.setProperty("jazzer.keep_going", "3"); + + AgentInstaller.install(true); + FuzzTargetHolder.fuzzTarget = + FuzzTargetFinder.findFuzzTarget(FuzzTargetRunnerTest.class.getName()); + + // Use a loop to simulate two findings with the same stack trace and thus verify that keep_going + // works as advertised. + for (int i = 1; i < 3; i++) { + int result = FuzzTargetRunner.runOne("no crash".getBytes(StandardCharsets.UTF_8)); + if (i == 1) { + // Initializing FuzzTargetRunner, which happens implicitly on the first call to runOne, + // starts the Jazzer agent, which prints out some info messages to stdout. Ignore them. + recordedOut.reset(); + } + + assert result == 0; + assert fuzzerInitializeRan; + assert CoverageMap.getCoveredIds().equals(Stream.of(0).collect(Collectors.toSet())); + assert UNSAFE.getByte(CoverageMap.countersAddress) == i; + assert UNSAFE.getByte(CoverageMap.countersAddress + 1) == 0; + assert UNSAFE.getByte(CoverageMap.countersAddress + 2) == 0; + assert UNSAFE.getByte(CoverageMap.countersAddress + 3) == 0; + + String errOutput = new String(recordedErr.toByteArray(), StandardCharsets.UTF_8); + List<String> unexpectedLines = Arrays.stream(errOutput.split("\n")) + .filter(line -> !line.startsWith("INFO: ")) + .collect(Collectors.toList()); + assert unexpectedLines.isEmpty() + : "Unexpected output on System.err: '" + + String.join("\n", unexpectedLines) + "'"; + String outOutput = new String(recordedOut.toByteArray(), StandardCharsets.UTF_8); + assert outOutput.isEmpty() : "Non-empty System.out: '" + outOutput + "'"; + } + + String firstDedupToken = null; + for (int i = 1; i < 3; i++) { + int result = FuzzTargetRunner.runOne("first finding".getBytes(StandardCharsets.UTF_8)); + + assert result == 0; + assert CoverageMap.getCoveredIds().equals(Stream.of(0, 1).collect(Collectors.toSet())); + assert UNSAFE.getByte(CoverageMap.countersAddress) == 2; + assert UNSAFE.getByte(CoverageMap.countersAddress + 1) == i; + assert UNSAFE.getByte(CoverageMap.countersAddress + 2) == 0; + assert UNSAFE.getByte(CoverageMap.countersAddress + 3) == 0; + + String errOutput = new String(recordedErr.toByteArray(), StandardCharsets.UTF_8); + String outOutput = new String(recordedOut.toByteArray(), StandardCharsets.UTF_8); + if (i == 1) { + assert errOutput.contains( + "== Java Exception: java.lang.IllegalArgumentException: first finding"); + Matcher dedupTokenMatcher = DEDUP_TOKEN_PATTERN.matcher(outOutput); + assert dedupTokenMatcher.matches() : "Unexpected output on System.out: '" + outOutput + "'"; + firstDedupToken = dedupTokenMatcher.group(); + recordedErr.reset(); + recordedOut.reset(); + } else { + assert errOutput.isEmpty(); + assert outOutput.isEmpty(); + } + } + + for (int i = 1; i < 3; i++) { + int result = FuzzTargetRunner.runOne("second finding".getBytes(StandardCharsets.UTF_8)); + + assert result == 0; + assert CoverageMap.getCoveredIds().equals(Stream.of(0, 1, 2).collect(Collectors.toSet())); + assert UNSAFE.getByte(CoverageMap.countersAddress) == 2; + assert UNSAFE.getByte(CoverageMap.countersAddress + 1) == 2; + assert UNSAFE.getByte(CoverageMap.countersAddress + 2) == i; + assert UNSAFE.getByte(CoverageMap.countersAddress + 3) == 0; + + String errOutput = new String(recordedErr.toByteArray(), StandardCharsets.UTF_8); + String outOutput = new String(recordedOut.toByteArray(), StandardCharsets.UTF_8); + if (i == 1) { + // Verify that the StackOverflowError is wrapped in security issue and contains reproducer + // information. + assert errOutput.contains( + "== Java Exception: com.code_intelligence.jazzer.api.FuzzerSecurityIssueLow: Stack overflow (use "); + assert !errOutput.contains("not reported"); + Matcher dedupTokenMatcher = DEDUP_TOKEN_PATTERN.matcher(outOutput); + assert dedupTokenMatcher.matches() : "Unexpected output on System.out: '" + outOutput + "'"; + assert !firstDedupToken.equals(dedupTokenMatcher.group()); + recordedErr.reset(); + recordedOut.reset(); + } else { + assert errOutput.isEmpty(); + assert outOutput.isEmpty(); + } + } + + finishedAllNonCrashingRuns = true; + + FuzzTargetRunner.runOne("crash".getBytes(StandardCharsets.UTF_8)); + + throw new IllegalStateException("Expected FuzzTargetRunner to call fuzzerTearDown"); + } + + /** + * An OutputStream that prints to two OutputStreams simultaneously. + */ + private static class TeeOutputStream extends PrintStream { + private final PrintStream otherOut; + public TeeOutputStream(PrintStream out1, PrintStream out2) { + super(out1, true); + this.otherOut = out2; + } + + @Override + public void flush() { + super.flush(); + otherOut.flush(); + } + + @Override + public void close() { + super.close(); + otherOut.close(); + } + + @Override + public void write(int b) { + super.write(b); + otherOut.write(b); + } + + @Override + public void write(byte[] buf, int off, int len) { + super.write(buf, off, len); + otherOut.write(buf, off, len); + } + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/driver/FuzzedDataProviderImplTest.java b/src/test/java/com/code_intelligence/jazzer/driver/FuzzedDataProviderImplTest.java new file mode 100644 index 00000000..26ebc0df --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/driver/FuzzedDataProviderImplTest.java @@ -0,0 +1,238 @@ +// Copyright 2021 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.driver; + +import com.code_intelligence.jazzer.api.FuzzedDataProvider; +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.util.Arrays; +import java.util.stream.Collectors; + +public class FuzzedDataProviderImplTest { + public static void main(String[] args) { + try (FuzzedDataProviderImpl fuzzedDataProvider = + FuzzedDataProviderImpl.withJavaData(INPUT_BYTES)) { + verifyFuzzedDataProvider(fuzzedDataProvider); + } + } + + private strictfp static void verifyFuzzedDataProvider(FuzzedDataProvider data) { + assertEqual(true, data.consumeBoolean()); + + assertEqual((byte) 0x7F, data.consumeByte()); + assertEqual((byte) 0x14, data.consumeByte((byte) 0x12, (byte) 0x22)); + + assertEqual(0x12345678, data.consumeInt()); + assertEqual(-0x12345600, data.consumeInt(-0x12345678, -0x12345600)); + assertEqual(0x12345679, data.consumeInt(0x12345678, 0x12345679)); + + assertEqual(true, Arrays.equals(new byte[] {0x01, 0x02}, data.consumeBytes(2))); + + assertEqual("jazzer", data.consumeString(6)); + assertEqual("ja\u0000zer", data.consumeString(6)); + assertEqual("€ß", data.consumeString(2)); + + assertEqual("jazzer", data.consumeAsciiString(6)); + assertEqual("ja\u0000zer", data.consumeAsciiString(6)); + assertEqual("\u0062\u0002\u002C\u0043\u001F", data.consumeAsciiString(5)); + + assertEqual(true, + Arrays.equals(new boolean[] {false, false, true, false, true}, data.consumeBooleans(5))); + assertEqual(true, + Arrays.equals(new long[] {0x0123456789abdcefL, 0xfedcba9876543210L}, data.consumeLongs(2))); + + assertAtLeastAsPrecise((float) 0.28969181, data.consumeProbabilityFloat()); + assertAtLeastAsPrecise(0.086814121166605432, data.consumeProbabilityDouble()); + assertAtLeastAsPrecise((float) 0.30104411, data.consumeProbabilityFloat()); + assertAtLeastAsPrecise(0.96218831486039413, data.consumeProbabilityDouble()); + + assertAtLeastAsPrecise((float) -2.8546307e+38, data.consumeRegularFloat()); + assertAtLeastAsPrecise(8.0940194040236032e+307, data.consumeRegularDouble()); + assertAtLeastAsPrecise( + (float) 271.49084, data.consumeRegularFloat((float) 123.0, (float) 777.0)); + assertAtLeastAsPrecise(30.859126145478349, data.consumeRegularDouble(13.37, 31.337)); + + assertEqual((float) 0.0, data.consumeFloat()); + assertEqual((float) -0.0, data.consumeFloat()); + assertEqual(Float.POSITIVE_INFINITY, data.consumeFloat()); + assertEqual(Float.NEGATIVE_INFINITY, data.consumeFloat()); + assertEqual(true, Float.isNaN(data.consumeFloat())); + assertEqual(Float.MIN_VALUE, data.consumeFloat()); + assertEqual(-Float.MIN_VALUE, data.consumeFloat()); + assertEqual(Float.MIN_NORMAL, data.consumeFloat()); + assertEqual(-Float.MIN_NORMAL, data.consumeFloat()); + assertEqual(Float.MAX_VALUE, data.consumeFloat()); + assertEqual(-Float.MAX_VALUE, data.consumeFloat()); + + assertEqual(0.0, data.consumeDouble()); + assertEqual(-0.0, data.consumeDouble()); + assertEqual(Double.POSITIVE_INFINITY, data.consumeDouble()); + assertEqual(Double.NEGATIVE_INFINITY, data.consumeDouble()); + assertEqual(true, Double.isNaN(data.consumeDouble())); + assertEqual(Double.MIN_VALUE, data.consumeDouble()); + assertEqual(-Double.MIN_VALUE, data.consumeDouble()); + assertEqual(Double.MIN_NORMAL, data.consumeDouble()); + assertEqual(-Double.MIN_NORMAL, data.consumeDouble()); + assertEqual(Double.MAX_VALUE, data.consumeDouble()); + assertEqual(-Double.MAX_VALUE, data.consumeDouble()); + + int[] array = {0, 1, 2, 3, 4}; + assertEqual(4, data.pickValue(array)); + assertEqual(2, (int) data.pickValue(Arrays.stream(array).boxed().toArray())); + assertEqual(3, data.pickValue(Arrays.stream(array).boxed().collect(Collectors.toList()))); + assertEqual(2, data.pickValue(Arrays.stream(array).boxed().collect(Collectors.toSet()))); + + // Buffer is almost depleted at this point. + assertEqual(7, data.remainingBytes()); + assertEqual(true, Arrays.equals(new long[0], data.consumeLongs(3))); + assertEqual(7, data.remainingBytes()); + assertEqual(true, Arrays.equals(new int[] {0x12345678}, data.consumeInts(3))); + assertEqual(3, data.remainingBytes()); + assertEqual(0x123456L, data.consumeLong()); + + // Buffer has been fully consumed at this point + assertEqual(0, data.remainingBytes()); + assertEqual(0, data.consumeInt()); + assertEqual(0.0, data.consumeDouble()); + assertEqual(-13.37, data.consumeRegularDouble(-13.37, 31.337)); + assertEqual(true, Arrays.equals(new byte[0], data.consumeBytes(4))); + assertEqual(true, Arrays.equals(new long[0], data.consumeLongs(4))); + assertEqual("", data.consumeRemainingAsAsciiString()); + assertEqual("", data.consumeRemainingAsString()); + assertEqual("", data.consumeAsciiString(100)); + assertEqual("", data.consumeString(100)); + } + + private static void assertAtLeastAsPrecise(double expected, double actual) { + BigDecimal exactExpected = BigDecimal.valueOf(expected); + BigDecimal roundedActual = + BigDecimal.valueOf(actual).setScale(exactExpected.scale(), RoundingMode.HALF_UP); + if (!exactExpected.equals(roundedActual)) { + throw new IllegalArgumentException( + String.format("Expected: %s, got: %s (rounded: %s)", expected, actual, roundedActual)); + } + } + + private static <T extends Comparable<T>> void assertEqual(T a, T b) { + if (a.compareTo(b) != 0) { + throw new IllegalArgumentException("Expected: " + a + ", got: " + b); + } + } + + private static final byte[] INPUT_BYTES = new byte[] { + // Bytes read from the start + 0x01, 0x02, // consumeBytes(2): {0x01, 0x02} + + 'j', 'a', 'z', 'z', 'e', 'r', // consumeString(6): "jazzer" + 'j', 'a', 0x00, 'z', 'e', 'r', // consumeString(6): "ja\u0000zer" + (byte) 0xE2, (byte) 0x82, (byte) 0xAC, (byte) 0xC3, (byte) 0x9F, // consumeString(2): "€ẞ" + + 'j', 'a', 'z', 'z', 'e', 'r', // consumeAsciiString(6): "jazzer" + 'j', 'a', 0x00, 'z', 'e', 'r', // consumeAsciiString(6): "ja\u0000zer" + (byte) 0xE2, (byte) 0x82, (byte) 0xAC, (byte) 0xC3, + (byte) 0x9F, // consumeAsciiString(5): "\u0062\u0002\u002C\u0043\u001F" + + 0, 0, 1, 0, 1, // consumeBooleans(5): { false, false, true, false, true } + (byte) 0xEF, (byte) 0xDC, (byte) 0xAB, (byte) 0x89, 0x67, 0x45, 0x23, 0x01, 0x10, 0x32, 0x54, + 0x76, (byte) 0x98, (byte) 0xBA, (byte) 0xDC, (byte) 0xFE, + // consumeLongs(2): { 0x0123456789ABCDEF, 0xFEDCBA9876543210 } + + 0x78, 0x56, 0x34, 0x12, // consumeInts(3): { 0x12345678 } + 0x56, 0x34, 0x12, // consumeLong(): + + // Bytes read from the end + 0x02, 0x03, 0x02, 0x04, // 4x pickValue in array with five elements + + 0x12, 0x34, 0x56, 0x78, (byte) 0x90, 0x12, 0x34, 0x56, + 0x78, // consumed but unused by consumeDouble() + 10, // -max for next consumeDouble + 0x12, 0x34, 0x56, 0x78, (byte) 0x90, 0x12, 0x34, 0x56, + 0x78, // consumed but unused by consumeDouble() + 9, // max for next consumeDouble + 0x12, 0x34, 0x56, 0x78, (byte) 0x90, 0x12, 0x34, 0x56, + 0x78, // consumed but unused by consumeDouble() + 8, // -min for next consumeDouble + 0x12, 0x34, 0x56, 0x78, (byte) 0x90, 0x12, 0x34, 0x56, + 0x78, // consumed but unused by consumeDouble() + 7, // min for next consumeDouble + 0x12, 0x34, 0x56, 0x78, (byte) 0x90, 0x12, 0x34, 0x56, + 0x78, // consumed but unused by consumeDouble() + 6, // -denorm_min for next consumeDouble + 0x12, 0x34, 0x56, 0x78, (byte) 0x90, 0x12, 0x34, 0x56, + 0x78, // consumed but unused by consumeDouble() + 5, // denorm_min for next consumeDouble + 0x12, 0x34, 0x56, 0x78, (byte) 0x90, 0x12, 0x34, 0x56, + 0x78, // consumed but unused by consumeDouble() + 4, // NaN for next consumeDouble + 0x12, 0x34, 0x56, 0x78, (byte) 0x90, 0x12, 0x34, 0x56, + 0x78, // consumed but unused by consumeDouble() + 3, // -infinity for next consumeDouble + 0x12, 0x34, 0x56, 0x78, (byte) 0x90, 0x12, 0x34, 0x56, + 0x78, // consumed but unused by consumeDouble() + 2, // infinity for next consumeDouble + 0x12, 0x34, 0x56, 0x78, (byte) 0x90, 0x12, 0x34, 0x56, + 0x78, // consumed but unused by consumeDouble() + 1, // -0.0 for next consumeDouble + 0x12, 0x34, 0x56, 0x78, (byte) 0x90, 0x12, 0x34, 0x56, + 0x78, // consumed but unused by consumeDouble() + 0, // 0.0 for next consumeDouble + + 0x12, 0x34, 0x56, 0x78, (byte) 0x90, // consumed but unused by consumeFloat() + 10, // -max for next consumeFloat + 0x12, 0x34, 0x56, 0x78, (byte) 0x90, // consumed but unused by consumeFloat() + 9, // max for next consumeFloat + 0x12, 0x34, 0x56, 0x78, (byte) 0x90, // consumed but unused by consumeFloat() + 8, // -min for next consumeFloat + 0x12, 0x34, 0x56, 0x78, (byte) 0x90, // consumed but unused by consumeFloat() + 7, // min for next consumeFloat + 0x12, 0x34, 0x56, 0x78, (byte) 0x90, // consumed but unused by consumeFloat() + 6, // -denorm_min for next consumeFloat + 0x12, 0x34, 0x56, 0x78, (byte) 0x90, // consumed but unused by consumeFloat() + 5, // denorm_min for next consumeFloat + 0x12, 0x34, 0x56, 0x78, (byte) 0x90, // consumed but unused by consumeFloat() + 4, // NaN for next consumeFloat + 0x12, 0x34, 0x56, 0x78, (byte) 0x90, // consumed but unused by consumeFloat() + 3, // -infinity for next consumeFloat + 0x12, 0x34, 0x56, 0x78, (byte) 0x90, // consumed but unused by consumeFloat() + 2, // infinity for next consumeFloat + 0x12, 0x34, 0x56, 0x78, (byte) 0x90, // consumed but unused by consumeFloat() + 1, // -0.0 for next consumeFloat + 0x12, 0x34, 0x56, 0x78, (byte) 0x90, // consumed but unused by consumeFloat() + 0, // 0.0 for next consumeFloat + + (byte) 0x88, (byte) 0xAB, 0x61, (byte) 0xCB, 0x32, (byte) 0xEB, 0x30, (byte) 0xF9, + // consumeDouble(13.37, 31.337): 30.859126145478349 (small range) + 0x51, (byte) 0xF6, 0x1F, 0x3A, // consumeFloat(123.0, 777.0): 271.49084 (small range) + 0x11, 0x4D, (byte) 0xFD, 0x54, (byte) 0xD6, 0x3D, 0x43, 0x73, 0x39, + // consumeRegularDouble(): 8.0940194040236032e+307 + 0x16, (byte) 0xCF, 0x3D, 0x29, 0x4A, // consumeRegularFloat(): -2.8546307e+38 + + 0x61, (byte) 0xCB, 0x32, (byte) 0xEB, 0x30, (byte) 0xF9, 0x51, (byte) 0xF6, + // consumeProbabilityDouble(): 0.96218831486039413 + 0x1F, 0x3A, 0x11, 0x4D, // consumeProbabilityFloat(): 0.30104411 + (byte) 0xFD, 0x54, (byte) 0xD6, 0x3D, 0x43, 0x73, 0x39, 0x16, + // consumeProbabilityDouble(): 0.086814121166605432 + (byte) 0xCF, 0x3D, 0x29, 0x4A, // consumeProbabilityFloat(): 0.28969181 + + 0x01, // consumeInt(0x12345678, 0x12345679): 0x12345679 + 0x78, // consumeInt(-0x12345678, -0x12345600): -0x12345600 + 0x78, 0x56, 0x34, 0x12, // consumeInt(): 0x12345678 + + 0x02, // consumeByte(0x12, 0x22): 0x14 + 0x7F, // consumeByte(): 0x7F + + 0x01, // consumeBool(): true + }; +} diff --git a/src/test/java/com/code_intelligence/jazzer/driver/OptTest.java b/src/test/java/com/code_intelligence/jazzer/driver/OptTest.java new file mode 100644 index 00000000..6f9f03c8 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/driver/OptTest.java @@ -0,0 +1,44 @@ +/* + * Copyright 2022 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.code_intelligence.jazzer.driver; + +import static org.junit.Assert.assertEquals; + +import java.util.Arrays; +import java.util.stream.Collectors; +import org.junit.Test; + +public class OptTest { + @Test + public void splitString() { + assertStringSplit("", ','); + assertStringSplit(",,,,,", ','); + assertStringSplit("fir\\\\st se\\ cond third", ' ', "fir\\st", "se cond", "third"); + assertStringSplit("first ", ' ', "first"); + assertStringSplit("first\\", ' ', "first"); + } + + @Test(expected = IllegalArgumentException.class) + public void splitString_noBackslashAsSeparator() { + assertStringSplit("foo", '\\'); + } + + public void assertStringSplit(String str, char sep, String... tokens) { + assertEquals(Arrays.stream(tokens).collect(Collectors.toList()), + OptParser.splitOnUnescapedSeparator(str, sep)); + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/driver/RecordingFuzzedDataProviderTest.java b/src/test/java/com/code_intelligence/jazzer/driver/RecordingFuzzedDataProviderTest.java new file mode 100644 index 00000000..de8e3a41 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/driver/RecordingFuzzedDataProviderTest.java @@ -0,0 +1,215 @@ +// Copyright 2021 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.driver; + +import com.code_intelligence.jazzer.api.CannedFuzzedDataProvider; +import com.code_intelligence.jazzer.api.FuzzedDataProvider; +import com.code_intelligence.jazzer.driver.RecordingFuzzedDataProvider; +import java.io.IOException; +import java.util.Arrays; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.LongStream; +import org.junit.Assert; +import org.junit.Test; + +public class RecordingFuzzedDataProviderTest { + @Test + public void testRecordingFuzzedDataProvider() throws IOException { + FuzzedDataProvider mockData = new MockFuzzedDataProvider(); + String referenceResult = sampleFuzzTarget(mockData); + + FuzzedDataProvider recordingMockData = + RecordingFuzzedDataProvider.makeFuzzedDataProviderProxy(mockData); + Assert.assertEquals(referenceResult, sampleFuzzTarget(recordingMockData)); + + String cannedMockDataString = + RecordingFuzzedDataProvider.serializeFuzzedDataProviderProxy(recordingMockData); + FuzzedDataProvider cannedMockData = new CannedFuzzedDataProvider(cannedMockDataString); + Assert.assertEquals(referenceResult, sampleFuzzTarget(cannedMockData)); + } + + private String sampleFuzzTarget(FuzzedDataProvider data) { + StringBuilder result = new StringBuilder(); + result.append(data.consumeString(10)); + int[] ints = data.consumeInts(5); + result.append(Arrays.stream(ints).mapToObj(Integer::toString).collect(Collectors.joining(","))); + result.append(data.pickValue(ints)); + result.append(data.consumeString(20)); + result.append(data.pickValues(Arrays.stream(ints).boxed().collect(Collectors.toSet()), 5) + .stream() + .map(Integer::toHexString) + .collect(Collectors.joining(","))); + result.append(data.remainingBytes()); + return result.toString(); + } + + private static final class MockFuzzedDataProvider implements FuzzedDataProvider { + @Override + public boolean consumeBoolean() { + return true; + } + + @Override + public boolean[] consumeBooleans(int maxLength) { + return new boolean[] {false, true}; + } + + @Override + public byte consumeByte() { + return 2; + } + + @Override + public byte consumeByte(byte min, byte max) { + return max; + } + + @Override + public short consumeShort() { + return 2; + } + + @Override + public short consumeShort(short min, short max) { + return min; + } + + @Override + public short[] consumeShorts(int maxLength) { + return new short[] {2, 4, 7}; + } + + @Override + public int consumeInt() { + return 5; + } + + @Override + public int consumeInt(int min, int max) { + return max; + } + + @Override + public int[] consumeInts(int maxLength) { + return IntStream.range(0, maxLength).toArray(); + } + + @Override + public long consumeLong() { + return 42; + } + + @Override + public long consumeLong(long min, long max) { + return min; + } + + @Override + public long[] consumeLongs(int maxLength) { + return LongStream.range(0, maxLength).toArray(); + } + + @Override + public float consumeFloat() { + return Float.NaN; + } + + @Override + public float consumeRegularFloat() { + return 0.3f; + } + + @Override + public float consumeRegularFloat(float min, float max) { + return min; + } + + @Override + public float consumeProbabilityFloat() { + return 0.2f; + } + + @Override + public double consumeDouble() { + return Double.NaN; + } + + @Override + public double consumeRegularDouble(double min, double max) { + return max; + } + + @Override + public double consumeRegularDouble() { + return Math.PI; + } + + @Override + public double consumeProbabilityDouble() { + return 0.5; + } + + @Override + public char consumeChar() { + return 'C'; + } + + @Override + public char consumeChar(char min, char max) { + return min; + } + + @Override + public char consumeCharNoSurrogates() { + return 'C'; + } + + @Override + public String consumeAsciiString(int maxLength) { + return "foobar"; + } + + @Override + public String consumeString(int maxLength) { + return "foo€ä"; + } + + @Override + public String consumeRemainingAsAsciiString() { + return "foobar"; + } + + @Override + public String consumeRemainingAsString() { + return "foobar"; + } + + @Override + public byte[] consumeBytes(int maxLength) { + return new byte[maxLength]; + } + + @Override + public byte[] consumeRemainingAsBytes() { + return new byte[] {1}; + } + + @Override + public int remainingBytes() { + return 1; + } + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/instrumentor/AfterHooks.java b/src/test/java/com/code_intelligence/jazzer/instrumentor/AfterHooks.java new file mode 100644 index 00000000..f8d6782c --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/instrumentor/AfterHooks.java @@ -0,0 +1,87 @@ +// Copyright 2021 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.instrumentor; + +import com.code_intelligence.jazzer.api.HookType; +import com.code_intelligence.jazzer.api.MethodHook; +import java.lang.invoke.MethodHandle; + +public class AfterHooks { + static AfterHooksTargetContract instance; + + @MethodHook(type = HookType.AFTER, + targetClassName = "com.code_intelligence.jazzer.instrumentor.AfterHooksTarget", + targetMethod = "func1") + public static void + patchFunc1( + MethodHandle method, Object thisObject, Object[] arguments, int hookId, Object returnValue) { + instance = (AfterHooksTargetContract) thisObject; + ((AfterHooksTargetContract) thisObject).registerHasFunc1BeenCalled(); + } + + @MethodHook(type = HookType.AFTER, + targetClassName = "com.code_intelligence.jazzer.instrumentor.AfterHooksTarget", + targetMethod = "registerTimesCalled", targetMethodDescriptor = "()V") + public static void + patchRegisterTimesCalled(MethodHandle method, Object thisObject, Object[] arguments, int hookId, + Object returnValue) throws Throwable { + // Invoke registerTimesCalled() again to pass the test. + method.invoke(); + } + + @MethodHook(type = HookType.AFTER, + targetClassName = "com.code_intelligence.jazzer.instrumentor.AfterHooksTarget", + targetMethod = "getFirstSecret", targetMethodDescriptor = "()Ljava/lang/String;") + public static void + patchGetFirstSecret( + MethodHandle method, Object thisObject, Object[] arguments, int hookId, String returnValue) { + // Use the returned secret to pass the test. + ((AfterHooksTargetContract) thisObject).verifyFirstSecret(returnValue); + } + + @MethodHook(type = HookType.AFTER, + targetClassName = "com.code_intelligence.jazzer.instrumentor.AfterHooksTarget", + targetMethod = "getSecondSecret") + public static void + patchGetSecondSecret( + MethodHandle method, Object thisObject, Object[] arguments, int hookId, Object returnValue) { + // Use the returned secret to pass the test. + ((AfterHooksTargetContract) thisObject).verifySecondSecret((String) returnValue); + } + + // Verify the interaction of a BEFORE and an AFTER hook. The BEFORE hook modifies the argument of + // the StringBuilder constructor. + @MethodHook( + type = HookType.BEFORE, targetClassName = "java.lang.StringBuilder", targetMethod = "<init>") + public static void + patchStringBuilderBeforeInit( + MethodHandle method, Object thisObject, Object[] arguments, int hookId) { + arguments[0] = "hunter3"; + } + + @MethodHook( + type = HookType.AFTER, targetClassName = "java.lang.StringBuilder", targetMethod = "<init>") + public static void + patchStringBuilderInit( + MethodHandle method, Object thisObject, Object[] arguments, int hookId, Object returnValue) { + String secret = ((StringBuilder) thisObject).toString(); + // Verify that the argument passed to this AFTER hook agrees with the argument passed to the + // StringBuilder constructor, which has been modified by the BEFORE hook. + if (secret.equals(arguments[0])) { + // Verify that the argument has been modified to the correct value "hunter3". + instance.verifyThirdSecret(secret); + } + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/instrumentor/AfterHooksPatchTest.kt b/src/test/java/com/code_intelligence/jazzer/instrumentor/AfterHooksPatchTest.kt new file mode 100644 index 00000000..55263786 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/instrumentor/AfterHooksPatchTest.kt @@ -0,0 +1,89 @@ +// Copyright 2021 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.instrumentor + +import com.code_intelligence.jazzer.instrumentor.PatchTestUtils.bytecodeToClass +import com.code_intelligence.jazzer.instrumentor.PatchTestUtils.classToBytecode +import org.junit.Test +import java.io.File + +private fun getOriginalAfterHooksTargetInstance(): AfterHooksTargetContract { + return AfterHooksTarget() +} + +private fun getNoHooksAfterHooksTargetInstance(): AfterHooksTargetContract { + val originalBytecode = classToBytecode(AfterHooksTarget::class.java) + // Let the bytecode pass through the hooking logic, but don't apply any hooks. + val patchedBytecode = HookInstrumentor(emptyList(), false, null).instrument( + AfterHooksTarget::class.java.name.replace('.', '/'), + originalBytecode, + ) + val patchedClass = bytecodeToClass(AfterHooksTarget::class.java.name, patchedBytecode) + return patchedClass.getDeclaredConstructor().newInstance() as AfterHooksTargetContract +} + +private fun getPatchedAfterHooksTargetInstance(classWithHooksEnabledField: Class<*>?): AfterHooksTargetContract { + val originalBytecode = classToBytecode(AfterHooksTarget::class.java) + val hooks = Hooks.loadHooks(emptyList(), setOf(AfterHooks::class.java.name)).first().hooks + val patchedBytecode = HookInstrumentor( + hooks, + false, + classWithHooksEnabledField = classWithHooksEnabledField?.name?.replace('.', '/'), + ).instrument(AfterHooksTarget::class.java.name.replace('.', '/'), originalBytecode) + // Make the patched class available in bazel-testlogs/.../test.outputs for manual inspection. + val outDir = System.getenv("TEST_UNDECLARED_OUTPUTS_DIR") + File("$outDir/${AfterHooksTarget::class.java.simpleName}.class").writeBytes(originalBytecode) + File("$outDir/${AfterHooksTarget::class.java.simpleName}.patched.class").writeBytes(patchedBytecode) + val patchedClass = bytecodeToClass(AfterHooksTarget::class.java.name, patchedBytecode) + return patchedClass.getDeclaredConstructor().newInstance() as AfterHooksTargetContract +} + +class AfterHooksPatchTest { + + @Test + fun testOriginal() { + assertSelfCheck(getOriginalAfterHooksTargetInstance(), false) + } + + @Test + fun testPatchedWithoutHooks() { + assertSelfCheck(getNoHooksAfterHooksTargetInstance(), false) + } + + @Test + fun testPatched() { + assertSelfCheck(getPatchedAfterHooksTargetInstance(null), true) + } + + object HooksEnabled { + @Suppress("unused") + const val hooksEnabled = true + } + + object HooksDisabled { + @Suppress("unused") + const val hooksEnabled = false + } + + @Test + fun testPatchedWithConditionalHooksEnabled() { + assertSelfCheck(getPatchedAfterHooksTargetInstance(HooksEnabled::class.java), true) + } + + @Test + fun testPatchedWithConditionalHooksDisabled() { + assertSelfCheck(getPatchedAfterHooksTargetInstance(HooksDisabled::class.java), false) + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/instrumentor/AfterHooksTarget.java b/src/test/java/com/code_intelligence/jazzer/instrumentor/AfterHooksTarget.java new file mode 100644 index 00000000..a47b03a5 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/instrumentor/AfterHooksTarget.java @@ -0,0 +1,85 @@ +// Copyright 2021 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.instrumentor; + +import java.util.HashMap; +import java.util.Map; + +// selfCheck() only passes with the hooks in AfterHooks.java applied. +public class AfterHooksTarget implements AfterHooksTargetContract { + static Map<String, Boolean> results = new HashMap<>(); + static int timesCalled = 0; + Boolean func1Called = false; + + public static void registerTimesCalled() { + timesCalled++; + results.put("hasBeenCalledTwice", timesCalled == 2); + } + + public Map<String, Boolean> selfCheck() { + results = new HashMap<>(); + + if (results.isEmpty()) { + registerHasFunc1BeenCalled(); + func1(); + } + + timesCalled = 0; + registerTimesCalled(); + + verifyFirstSecret("not_secret"); + getFirstSecret(); + + verifySecondSecret("not_secret_at_all"); + getSecondSecret(); + + verifyThirdSecret("not_the_secret"); + new StringBuilder("not_hunter3"); + + return results; + } + + public void func1() { + func1Called = true; + } + + public void registerHasFunc1BeenCalled() { + results.put("hasFunc1BeenCalled", func1Called); + } + + @SuppressWarnings("UnusedReturnValue") + String getFirstSecret() { + return "hunter2"; + } + + @SuppressWarnings("SameParameterValue") + public void verifyFirstSecret(String secret) { + results.put("verifyFirstSecret", secret.equals("hunter2")); + } + + @SuppressWarnings("UnusedReturnValue") + String getSecondSecret() { + return "hunter2!"; + } + + @SuppressWarnings("SameParameterValue") + public void verifySecondSecret(String secret) { + results.put("verifySecondSecret", secret.equals("hunter2!")); + } + + public void verifyThirdSecret(String secret) { + results.put("verifyThirdSecret", secret.equals("hunter3")); + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/instrumentor/AfterHooksTargetContract.java b/src/test/java/com/code_intelligence/jazzer/instrumentor/AfterHooksTargetContract.java new file mode 100644 index 00000000..cb12b148 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/instrumentor/AfterHooksTargetContract.java @@ -0,0 +1,29 @@ +// Copyright 2021 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.instrumentor; + +/** + * Helper interface used to call methods on instances of AfterHooksTarget classes loaded via + * different class loaders. + */ +public interface AfterHooksTargetContract extends DynamicTestContract { + void registerHasFunc1BeenCalled(); + + void verifyFirstSecret(String secret); + + void verifySecondSecret(String secret); + + void verifyThirdSecret(String secret); +} diff --git a/src/test/java/com/code_intelligence/jazzer/instrumentor/BUILD.bazel b/src/test/java/com/code_intelligence/jazzer/instrumentor/BUILD.bazel new file mode 100644 index 00000000..4fdad567 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/instrumentor/BUILD.bazel @@ -0,0 +1,152 @@ +load("//bazel:kotlin.bzl", "ktlint", "wrapped_kt_jvm_test") +load("@io_bazel_rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") + +kt_jvm_library( + name = "patch_test_utils", + srcs = [ + "DynamicTestContract.java", + "PatchTestUtils.kt", + ], + visibility = ["//visibility:public"], +) + +wrapped_kt_jvm_test( + name = "trace_data_flow_instrumentation_test", + size = "small", + srcs = [ + "MockTraceDataFlowCallbacks.java", + "TraceDataFlowInstrumentationTarget.java", + "TraceDataFlowInstrumentationTest.kt", + ], + associates = [ + "//src/main/java/com/code_intelligence/jazzer/instrumentor:instrumentor", + ], + test_class = "com.code_intelligence.jazzer.instrumentor.TraceDataFlowInstrumentationTest", + deps = [ + ":patch_test_utils", + "@com_github_jetbrains_kotlin//:kotlin-test", + "@maven//:junit_junit", + ], +) + +wrapped_kt_jvm_test( + name = "coverage_instrumentation_test", + size = "small", + srcs = [ + "CoverageInstrumentationSpecialCasesTarget.java", + "CoverageInstrumentationTarget.java", + "CoverageInstrumentationTest.kt", + "MockCoverageMap.java", + ], + associates = [ + "//src/main/java/com/code_intelligence/jazzer/instrumentor:instrumentor", + ], + test_class = "com.code_intelligence.jazzer.instrumentor.CoverageInstrumentationTest", + deps = [ + ":patch_test_utils", + "//src/main/java/com/code_intelligence/jazzer/runtime:coverage_map", + "@com_github_jetbrains_kotlin//:kotlin-test", + "@maven//:junit_junit", + ], +) + +wrapped_kt_jvm_test( + name = "descriptor_utils_test", + size = "small", + srcs = [ + "DescriptorUtilsTest.kt", + ], + associates = [ + "//src/main/java/com/code_intelligence/jazzer/instrumentor:instrumentor", + ], + test_class = "com.code_intelligence.jazzer.instrumentor.DescriptorUtilsTest", + deps = [ + "@com_github_jetbrains_kotlin//:kotlin-test", + "@maven//:junit_junit", + ], +) + +wrapped_kt_jvm_test( + name = "hook_validation_test", + size = "small", + srcs = [ + "HookValidationTest.kt", + "InvalidHookMocks.java", + "ValidHookMocks.java", + ], + associates = [ + "//src/main/java/com/code_intelligence/jazzer/instrumentor:instrumentor", + ], + test_class = "com.code_intelligence.jazzer.instrumentor.HookValidationTest", + deps = [ + "//src/main/java/com/code_intelligence/jazzer/api", + "@com_github_jetbrains_kotlin//:kotlin-test", + "@maven//:junit_junit", + ], +) + +wrapped_kt_jvm_test( + name = "after_hooks_patch_test", + size = "small", + srcs = [ + "AfterHooks.java", + "AfterHooksPatchTest.kt", + "AfterHooksTarget.java", + "AfterHooksTargetContract.java", + ], + associates = [ + "//src/main/java/com/code_intelligence/jazzer/instrumentor:instrumentor", + ], + test_class = "com.code_intelligence.jazzer.instrumentor.AfterHooksPatchTest", + deps = [ + ":patch_test_utils", + "//src/main/java/com/code_intelligence/jazzer/api", + "@com_github_jetbrains_kotlin//:kotlin-test", + "@maven//:junit_junit", + ], +) + +wrapped_kt_jvm_test( + name = "before_hooks_patch_test", + size = "small", + srcs = [ + "BeforeHooks.java", + "BeforeHooksPatchTest.kt", + "BeforeHooksTarget.java", + "BeforeHooksTargetContract.java", + ], + associates = [ + "//src/main/java/com/code_intelligence/jazzer/instrumentor:instrumentor", + ], + test_class = "com.code_intelligence.jazzer.instrumentor.BeforeHooksPatchTest", + deps = [ + ":patch_test_utils", + "//src/main/java/com/code_intelligence/jazzer/api", + "@com_github_jetbrains_kotlin//:kotlin-test", + "@maven//:junit_junit", + ], +) + +wrapped_kt_jvm_test( + name = "replace_hooks_patch_test", + size = "small", + srcs = [ + "ReplaceHooks.java", + "ReplaceHooksInit.java", + "ReplaceHooksPatchTest.kt", + "ReplaceHooksTarget.java", + "ReplaceHooksTargetContract.java", + ], + associates = [ + "//src/main/java/com/code_intelligence/jazzer/instrumentor:instrumentor", + ], + test_class = "com.code_intelligence.jazzer.instrumentor.ReplaceHooksPatchTest", + deps = [ + ":patch_test_utils", + "//src/main/java/com/code_intelligence/jazzer/api", + "@com_github_jetbrains_kotlin//:kotlin-test", + "@maven//:junit_junit", + ], +) + +ktlint() diff --git a/src/test/java/com/code_intelligence/jazzer/instrumentor/BeforeHooks.java b/src/test/java/com/code_intelligence/jazzer/instrumentor/BeforeHooks.java new file mode 100644 index 00000000..31577dad --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/instrumentor/BeforeHooks.java @@ -0,0 +1,53 @@ +// Copyright 2021 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.instrumentor; + +import com.code_intelligence.jazzer.api.HookType; +import com.code_intelligence.jazzer.api.MethodHook; +import java.lang.invoke.MethodHandle; + +public class BeforeHooks { + @MethodHook(type = HookType.BEFORE, + targetClassName = "com.code_intelligence.jazzer.instrumentor.BeforeHooksTarget", + targetMethod = "hasFunc1BeenCalled", targetMethodDescriptor = "()Z") + public static void + patchHasFunc1BeenCalled(MethodHandle method, Object thisObject, Object[] arguments, int hookId) { + ((BeforeHooksTargetContract) thisObject).func1(); + } + + @MethodHook(type = HookType.BEFORE, + targetClassName = "com.code_intelligence.jazzer.instrumentor.BeforeHooksTarget", + targetMethod = "getTimesCalled", targetMethodDescriptor = "()Ljava/lang/Integer;") + public static void + patchHasBeenCalled(MethodHandle method, Object thisObject, Object[] arguments, int hookId) + throws Throwable { + // Invoke static method getTimesCalled() again to pass the test. + method.invoke(); + } + + @MethodHook(type = HookType.BEFORE, + targetClassName = "com.code_intelligence.jazzer.instrumentor.BeforeHooksTarget", + targetMethod = "hasFuncWithArgsBeenCalled") + public static void + patchHasFuncWithArgsBeenCalled( + MethodHandle method, Object thisObject, Object[] arguments, int hookId) { + if (arguments.length == 2 && arguments[0] instanceof Boolean + && arguments[1] instanceof String) { + // only if the arguments passed to the hook match the expected argument types and count invoke + // the method to pass the test + ((BeforeHooksTargetContract) thisObject).setFuncWithArgsCalled((Boolean) arguments[0]); + } + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/instrumentor/BeforeHooksPatchTest.kt b/src/test/java/com/code_intelligence/jazzer/instrumentor/BeforeHooksPatchTest.kt new file mode 100644 index 00000000..aae469c7 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/instrumentor/BeforeHooksPatchTest.kt @@ -0,0 +1,89 @@ +// Copyright 2021 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.instrumentor + +import com.code_intelligence.jazzer.instrumentor.PatchTestUtils.bytecodeToClass +import com.code_intelligence.jazzer.instrumentor.PatchTestUtils.classToBytecode +import org.junit.Test +import java.io.File + +private fun getOriginalBeforeHooksTargetInstance(): BeforeHooksTargetContract { + return BeforeHooksTarget() +} + +private fun getNoHooksBeforeHooksTargetInstance(): BeforeHooksTargetContract { + val originalBytecode = classToBytecode(BeforeHooksTarget::class.java) + // Let the bytecode pass through the hooking logic, but don't apply any hooks. + val patchedBytecode = HookInstrumentor(emptyList(), false, null).instrument( + BeforeHooksTarget::class.java.name.replace('.', '/'), + originalBytecode, + ) + val patchedClass = bytecodeToClass(BeforeHooksTarget::class.java.name, patchedBytecode) + return patchedClass.getDeclaredConstructor().newInstance() as BeforeHooksTargetContract +} + +private fun getPatchedBeforeHooksTargetInstance(classWithHooksEnabledField: Class<*>?): BeforeHooksTargetContract { + val originalBytecode = classToBytecode(BeforeHooksTarget::class.java) + val hooks = Hooks.loadHooks(emptyList(), setOf(BeforeHooks::class.java.name)).first().hooks + val patchedBytecode = HookInstrumentor( + hooks, + false, + classWithHooksEnabledField = classWithHooksEnabledField?.name?.replace('.', '/'), + ).instrument(BeforeHooksTarget::class.java.name.replace('.', '/'), originalBytecode) + // Make the patched class available in bazel-testlogs/.../test.outputs for manual inspection. + val outDir = System.getenv("TEST_UNDECLARED_OUTPUTS_DIR") + File("$outDir/${BeforeHooksTarget::class.java.simpleName}.class").writeBytes(originalBytecode) + File("$outDir/${BeforeHooksTarget::class.java.simpleName}.patched.class").writeBytes(patchedBytecode) + val patchedClass = bytecodeToClass(BeforeHooksTarget::class.java.name, patchedBytecode) + return patchedClass.getDeclaredConstructor().newInstance() as BeforeHooksTargetContract +} + +class BeforeHooksPatchTest { + + @Test + fun testOriginal() { + assertSelfCheck(getOriginalBeforeHooksTargetInstance(), false) + } + + @Test + fun testPatchedWithoutHooks() { + assertSelfCheck(getNoHooksBeforeHooksTargetInstance(), false) + } + + @Test + fun testPatched() { + assertSelfCheck(getPatchedBeforeHooksTargetInstance(null), true) + } + + object HooksEnabled { + @Suppress("unused") + const val hooksEnabled = true + } + + object HooksDisabled { + @Suppress("unused") + const val hooksEnabled = false + } + + @Test + fun testPatchedWithConditionalHooksEnabled() { + assertSelfCheck(getPatchedBeforeHooksTargetInstance(HooksEnabled::class.java), true) + } + + @Test + fun testPatchedWithConditionalHooksDisabled() { + assertSelfCheck(getPatchedBeforeHooksTargetInstance(HooksDisabled::class.java), false) + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/instrumentor/BeforeHooksTarget.java b/src/test/java/com/code_intelligence/jazzer/instrumentor/BeforeHooksTarget.java new file mode 100644 index 00000000..869e04bf --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/instrumentor/BeforeHooksTarget.java @@ -0,0 +1,61 @@ +// Copyright 2021 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.instrumentor; + +import java.util.HashMap; +import java.util.Map; + +// selfCheck() only passes with the hooks in BeforeHooks.java applied. +public class BeforeHooksTarget implements BeforeHooksTargetContract { + static private int timesCalled = 0; + Map<String, Boolean> results = new HashMap<>(); + Boolean func1Called = false; + Boolean funcWithArgsCalled = false; + + static Integer getTimesCalled() { + return ++timesCalled; + } + + public Map<String, Boolean> selfCheck() { + results = new HashMap<>(); + + results.put("hasFunc1BeenCalled", hasFunc1BeenCalled()); + + timesCalled = 0; + results.put("hasBeenCalledTwice", getTimesCalled() == 2); + + if (!results.containsKey("hasBeenCalledWithArgs")) { + results.put("hasBeenCalledWithArgs", hasFuncWithArgsBeenCalled(true, "foo")); + } + + return results; + } + + public void func1() { + func1Called = true; + } + + private boolean hasFunc1BeenCalled() { + return func1Called; + } + + public void setFuncWithArgsCalled(Boolean val) { + funcWithArgsCalled = val; + } + + private boolean hasFuncWithArgsBeenCalled(Boolean boolArgument, String stringArgument) { + return funcWithArgsCalled; + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/instrumentor/BeforeHooksTargetContract.java b/src/test/java/com/code_intelligence/jazzer/instrumentor/BeforeHooksTargetContract.java new file mode 100644 index 00000000..61f79dcc --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/instrumentor/BeforeHooksTargetContract.java @@ -0,0 +1,25 @@ +// Copyright 2021 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.instrumentor; + +/** + * Helper interface used to call methods on instances of BeforeHooksTarget classes loaded via + * different class loaders. + */ +public interface BeforeHooksTargetContract extends DynamicTestContract { + void func1(); + + void setFuncWithArgsCalled(Boolean val); +} diff --git a/src/test/java/com/code_intelligence/jazzer/instrumentor/CoverageInstrumentationSpecialCasesTarget.java b/src/test/java/com/code_intelligence/jazzer/instrumentor/CoverageInstrumentationSpecialCasesTarget.java new file mode 100644 index 00000000..cb811803 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/instrumentor/CoverageInstrumentationSpecialCasesTarget.java @@ -0,0 +1,41 @@ +// Copyright 2021 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.instrumentor; + +import java.util.Random; + +public class CoverageInstrumentationSpecialCasesTarget { + public ReturnClass newAfterJump() { + if (new Random().nextBoolean()) { + throw new RuntimeException(""); + } + return new ReturnClass(new Random().nextBoolean() ? "foo" : "bar"); + } + + public int newAndTryCatch() { + new Random(); + try { + new Random(); + return 2; + } catch (RuntimeException e) { + new Random(); + return 1; + } + } + + public static class ReturnClass { + public ReturnClass(String content) {} + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/instrumentor/CoverageInstrumentationTarget.java b/src/test/java/com/code_intelligence/jazzer/instrumentor/CoverageInstrumentationTarget.java new file mode 100644 index 00000000..7502481d --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/instrumentor/CoverageInstrumentationTarget.java @@ -0,0 +1,67 @@ +// Copyright 2021 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.instrumentor; + +import java.util.HashMap; +import java.util.Map; + +public class CoverageInstrumentationTarget implements DynamicTestContract { + volatile int int1 = 3; + volatile int int2 = 213234; + + @Override + public Map<String, Boolean> selfCheck() { + HashMap<String, Boolean> results = new HashMap<>(); + + results.put("for0", false); + results.put("for1", false); + results.put("for2", false); + results.put("for3", false); + results.put("for4", false); + results.put("foobar", false); + results.put("baz", true); + + if (int1 < int2) { + results.put("block1", true); + } else { + results.put("block2", false); + } + + for (int i = 0; i < 2; i++) { + for (int j = 0; j < 5; j++) { + results.put("for" + j, i != 0); + } + } + + foo(results); + + return results; + } + + private void foo(HashMap<String, Boolean> results) { + bar(results); + } + + // The use of Map instead of HashMap is deliberate here: Since Map#put can throw exceptions, the + // invocation should be instrumented for coverage. + private void bar(Map<String, Boolean> results) { + results.put("foobar", true); + } + + @SuppressWarnings("unused") + private void baz(HashMap<String, Boolean> results) { + results.put("baz", false); + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/instrumentor/CoverageInstrumentationTest.kt b/src/test/java/com/code_intelligence/jazzer/instrumentor/CoverageInstrumentationTest.kt new file mode 100644 index 00000000..5a3c355a --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/instrumentor/CoverageInstrumentationTest.kt @@ -0,0 +1,176 @@ +// Copyright 2021 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.instrumentor + +import com.code_intelligence.jazzer.instrumentor.PatchTestUtils.bytecodeToClass +import com.code_intelligence.jazzer.instrumentor.PatchTestUtils.classToBytecode +import org.junit.Test +import org.objectweb.asm.MethodVisitor +import org.objectweb.asm.Opcodes +import java.io.File +import kotlin.test.assertEquals + +/** + * Amends the instrumentation performed by [strategy] to call the map's public static void method + * updated() after every update to coverage counters. + */ +private fun makeTestable(strategy: EdgeCoverageStrategy): EdgeCoverageStrategy = + object : EdgeCoverageStrategy by strategy { + override fun instrumentControlFlowEdge( + mv: MethodVisitor, + edgeId: Int, + variable: Int, + coverageMapInternalClassName: String, + ) { + strategy.instrumentControlFlowEdge(mv, edgeId, variable, coverageMapInternalClassName) + mv.visitMethodInsn(Opcodes.INVOKESTATIC, coverageMapInternalClassName, "updated", "()V", false) + } + } + +private fun getOriginalInstrumentationTargetInstance(): DynamicTestContract { + return CoverageInstrumentationTarget() +} + +private fun getInstrumentedInstrumentationTargetInstance(): DynamicTestContract { + val originalBytecode = classToBytecode(CoverageInstrumentationTarget::class.java) + val patchedBytecode = EdgeCoverageInstrumentor( + makeTestable(ClassInstrumentor.defaultEdgeCoverageStrategy), + MockCoverageMap::class.java, + 0, + ).instrument(CoverageInstrumentationTarget::class.java.name.replace('.', '/'), originalBytecode) + // Make the patched class available in bazel-testlogs/.../test.outputs for manual inspection. + val outDir = System.getenv("TEST_UNDECLARED_OUTPUTS_DIR") + File("$outDir/${CoverageInstrumentationTarget::class.java.simpleName}.class").writeBytes(originalBytecode) + File("$outDir/${CoverageInstrumentationTarget::class.java.simpleName}.patched.class").writeBytes(patchedBytecode) + val patchedClass = bytecodeToClass(CoverageInstrumentationTarget::class.java.name, patchedBytecode) + return patchedClass.getDeclaredConstructor().newInstance() as DynamicTestContract +} + +private fun assertControlFlow(expectedLocations: List<Int>) { + assertEquals(expectedLocations, MockCoverageMap.locations.toList()) +} + +@Suppress("unused") +class CoverageInstrumentationTest { + + private val constructorReturn = 0 + + private val mapConstructor = 1 + private val addFor0 = 2 + private val addFor1 = 3 + private val addFor2 = 4 + private val addFor3 = 5 + private val addFor4 = 6 + private val addFoobar = 7 + + private val ifTrueBranch = 8 + private val addBlock1 = 9 + private val ifFalseBranch = 10 + private val ifEnd = 11 + + private val outerForCondition = 12 + private val innerForCondition = 13 + private val innerForBodyIfTrueBranch = 14 + private val innerForBodyIfFalseBranch = 15 + private val innerForBodyPutInvocation = 16 + private val outerForIncrementCounter = 17 + + private val afterFooInvocation = 18 + private val fooAfterBarInvocation = 19 + private val barAfterPutInvocation = 20 + + @Test + fun testOriginal() { + assertSelfCheck(getOriginalInstrumentationTargetInstance()) + } + + @Test + fun testInstrumented() { + MockCoverageMap.clear() + assertSelfCheck(getInstrumentedInstrumentationTargetInstance()) + + val mapControlFlow = listOf(mapConstructor, addFor0, addFor1, addFor2, addFor3, addFor4, addFoobar) + val ifControlFlow = listOf(ifTrueBranch, addBlock1, ifEnd) + val forFirstRunControlFlow = mutableListOf<Int>().apply { + add(outerForCondition) + repeat(5) { + addAll(listOf(innerForCondition, innerForBodyIfFalseBranch, innerForBodyPutInvocation)) + } + add(outerForIncrementCounter) + }.toList() + val forSecondRunControlFlow = mutableListOf<Int>().apply { + add(outerForCondition) + repeat(5) { + addAll(listOf(innerForCondition, innerForBodyIfTrueBranch, innerForBodyPutInvocation)) + } + add(outerForIncrementCounter) + }.toList() + val forControlFlow = forFirstRunControlFlow + forSecondRunControlFlow + val fooCallControlFlow = listOf( + barAfterPutInvocation, + fooAfterBarInvocation, + afterFooInvocation, + ) + assertControlFlow( + listOf(constructorReturn) + + mapControlFlow + + ifControlFlow + + forControlFlow + + fooCallControlFlow, + ) + } + + @Test + fun testCounters() { + MockCoverageMap.clear() + + val target = getInstrumentedInstrumentationTargetInstance() + // The constructor of the target is run only once. + val takenOnceEdge = constructorReturn + // Control flows through the first if branch once per run. + val takenOnEveryRunEdge = ifTrueBranch + + var lastCounter = 0.toUByte() + for (i in 1..600) { + assertSelfCheck(target) + assertEquals(1, MockCoverageMap.counters[takenOnceEdge]) + // Verify that the counter increments, but is never zero. + val expectedCounter = (lastCounter + 1U).toUByte().takeUnless { it == 0.toUByte() } + ?: (lastCounter + 2U).toUByte() + lastCounter = expectedCounter + val actualCounter = MockCoverageMap.counters[takenOnEveryRunEdge].toUByte() + assertEquals(expectedCounter, actualCounter, "After $i runs:") + } + } + + @Test + fun testSpecialCases() { + val originalBytecode = classToBytecode(CoverageInstrumentationSpecialCasesTarget::class.java) + val patchedBytecode = EdgeCoverageInstrumentor( + makeTestable(ClassInstrumentor.defaultEdgeCoverageStrategy), + MockCoverageMap::class.java, + 0, + ).instrument(CoverageInstrumentationSpecialCasesTarget::class.java.name.replace('.', '/'), originalBytecode) + // Make the patched class available in bazel-testlogs/.../test.outputs for manual inspection. + val outDir = System.getenv("TEST_UNDECLARED_OUTPUTS_DIR") + File("$outDir/${CoverageInstrumentationSpecialCasesTarget::class.simpleName}.class").writeBytes(originalBytecode) + File("$outDir/${CoverageInstrumentationSpecialCasesTarget::class.simpleName}.patched.class").writeBytes( + patchedBytecode, + ) + val patchedClass = bytecodeToClass(CoverageInstrumentationSpecialCasesTarget::class.java.name, patchedBytecode) + // Trigger a class load + patchedClass.declaredMethods + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/instrumentor/DescriptorUtilsTest.kt b/src/test/java/com/code_intelligence/jazzer/instrumentor/DescriptorUtilsTest.kt new file mode 100644 index 00000000..c1a3584e --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/instrumentor/DescriptorUtilsTest.kt @@ -0,0 +1,72 @@ +// Copyright 2021 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.instrumentor + +import org.junit.Test +import kotlin.test.assertEquals + +class DescriptorUtilsTest { + + @Test + fun testClassDescriptor() { + assertEquals("V", java.lang.Void::class.javaPrimitiveType?.descriptor) + assertEquals("J", java.lang.Long::class.javaPrimitiveType?.descriptor) + assertEquals("[[[Z", Array<Array<BooleanArray>>::class.java.descriptor) + assertEquals("[Ljava/lang/String;", Array<String>::class.java.descriptor) + } + + @Test + fun testExtractInternalClassName() { + assertEquals("java/lang/String", extractInternalClassName("Ljava/lang/String;")) + assertEquals("[Ljava/lang/String;", extractInternalClassName("[Ljava/lang/String;")) + assertEquals("B", extractInternalClassName("B")) + } + + @Test + fun testExtractTypeDescriptors() { + val testCases = listOf( + Triple( + String::class.java.getMethod("equals", Object::class.java), + listOf("Ljava/lang/Object;"), + "Z", + ), + Triple( + String::class.java.getMethod("regionMatches", Boolean::class.javaPrimitiveType, Int::class.javaPrimitiveType, String::class.java, Int::class.javaPrimitiveType, Integer::class.javaPrimitiveType), + listOf("Z", "I", "Ljava/lang/String;", "I", "I"), + "Z", + ), + Triple( + String::class.java.getMethod("getChars", Integer::class.javaPrimitiveType, Int::class.javaPrimitiveType, CharArray::class.java, Int::class.javaPrimitiveType), + listOf("I", "I", "[C", "I"), + "V", + ), + Triple( + String::class.java.getMethod("subSequence", Integer::class.javaPrimitiveType, Integer::class.javaPrimitiveType), + listOf("I", "I"), + "Ljava/lang/CharSequence;", + ), + Triple( + String::class.java.getConstructor(), + emptyList(), + "V", + ), + ) + for ((executable, parameterDescriptors, returnTypeDescriptor) in testCases) { + val descriptor = executable.descriptor + assertEquals(extractParameterTypeDescriptors(descriptor), parameterDescriptors) + assertEquals(extractReturnTypeDescriptor(descriptor), returnTypeDescriptor) + } + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/instrumentor/DynamicTestContract.java b/src/test/java/com/code_intelligence/jazzer/instrumentor/DynamicTestContract.java new file mode 100644 index 00000000..163b226a --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/instrumentor/DynamicTestContract.java @@ -0,0 +1,21 @@ +// Copyright 2021 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.instrumentor; + +import java.util.Map; + +public interface DynamicTestContract { + Map<String, Boolean> selfCheck(); +} diff --git a/src/test/java/com/code_intelligence/jazzer/instrumentor/HookValidationTest.kt b/src/test/java/com/code_intelligence/jazzer/instrumentor/HookValidationTest.kt new file mode 100644 index 00000000..bf02da72 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/instrumentor/HookValidationTest.kt @@ -0,0 +1,40 @@ +// Copyright 2021 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.instrumentor + +import com.code_intelligence.jazzer.api.MethodHook +import org.junit.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith + +class HookValidationTest { + @Test + fun testValidHooks() { + val hooks = Hooks.loadHooks(emptyList(), setOf(ValidHookMocks::class.java.name)).first().hooks + assertEquals(5, hooks.size) + } + + @Test + fun testInvalidHooks() { + for (method in InvalidHookMocks::class.java.methods) { + if (method.isAnnotationPresent(MethodHook::class.java)) { + assertFailsWith<IllegalArgumentException>("Expected ${method.name} to be an invalid hook") { + val methodHook = method.declaredAnnotations.first() as MethodHook + Hook.createAndVerifyHook(method, methodHook, methodHook.targetClassName) + } + } + } + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/instrumentor/InvalidHookMocks.java b/src/test/java/com/code_intelligence/jazzer/instrumentor/InvalidHookMocks.java new file mode 100644 index 00000000..0df349ca --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/instrumentor/InvalidHookMocks.java @@ -0,0 +1,87 @@ +// Copyright 2021 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.instrumentor; + +import com.code_intelligence.jazzer.api.HookType; +import com.code_intelligence.jazzer.api.MethodHook; +import java.lang.invoke.MethodHandle; + +@SuppressWarnings({"unused", "RedundantThrows"}) +class InvalidHookMocks { + @MethodHook(type = HookType.BEFORE, targetClassName = "java.lang.String", targetMethod = "equals") + public static void incorrectHookIdType( + MethodHandle method, String thisObject, Object[] arguments, long hookId) {} + + @MethodHook(type = HookType.AFTER, targetClassName = "java.lang.String", targetMethod = "equals") + private static void invalidAfterHook(MethodHandle method, String thisObject, Object[] arguments, + int hookId, Boolean returnValue) {} + + @MethodHook(type = HookType.AFTER, targetClassName = "java.lang.String", targetMethod = "equals") + public void invalidAfterHook2(MethodHandle method, String thisObject, Object[] arguments, + int hookId, boolean returnValue) {} + + @MethodHook(type = HookType.REPLACE, targetClassName = "java.lang.String", + targetMethod = "equals", targetMethodDescriptor = "(Ljava/lang/Object;)Z") + public static String + incorrectReturnType(MethodHandle method, String thisObject, Object[] arguments, int hookId) { + return "foo"; + } + + @MethodHook( + type = HookType.REPLACE, targetClassName = "java.lang.String", targetMethod = "equals") + public static boolean + invalidReplaceHook2(MethodHandle method, Integer thisObject, Object[] arguments, int hookId) { + return true; + } + + @MethodHook(type = HookType.REPLACE, targetClassName = "java.lang.System", targetMethod = "gc", + targetMethodDescriptor = "()V") + public static Object + invalidReplaceVoidMethod(MethodHandle method, Object thisObject, Object[] arguments, int hookId) { + return null; + } + + @MethodHook(type = HookType.BEFORE, targetClassName = "java.lang.StringBuilder", + targetMethod = "<init>", targetMethodDescriptor = "(Ljava/lang/String;)V") + public static Object + invalidReturnType(MethodHandle method, Object thisObject, Object[] arguments, int hookId) + throws Throwable { + return null; + } + + @MethodHook(type = HookType.AFTER, targetClassName = "java.lang.String", + targetMethod = "startsWith", targetMethodDescriptor = "(Ljava/lang/String;)Z") + public static void + primitiveReturnValueMustBeWrapped(MethodHandle method, String thisObject, Object[] arguments, + int hookId, boolean returnValue) {} + + @MethodHook(type = HookType.REPLACE, targetClassName = "java.lang.StringBuilder", + targetMethod = "<init>", targetMethodDescriptor = "(Ljava/lang/String;)V") + public static void + replaceOnInitWithoutReturnType( + MethodHandle method, Object thisObject, Object[] arguments, int hookId) throws Throwable {} + + @MethodHook(type = HookType.REPLACE, targetClassName = "java.lang.StringBuilder", + targetMethod = "<init>", targetMethodDescriptor = "(Ljava/lang/String;)V") + public static Object + replaceOnInitWithIncompatibleType( + MethodHandle method, Object thisObject, Object[] arguments, int hookId) throws Throwable { + return new Object(); + } + + @MethodHook(type = HookType.AFTER, targetClassName = "java.lang.String", targetMethod = "equals") + public static void primitiveReturnType(MethodHandle method, String thisObject, Object[] arguments, + int hookId, boolean returnValue) {} +} diff --git a/src/test/java/com/code_intelligence/jazzer/instrumentor/MockCoverageMap.java b/src/test/java/com/code_intelligence/jazzer/instrumentor/MockCoverageMap.java new file mode 100644 index 00000000..3ea33d19 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/instrumentor/MockCoverageMap.java @@ -0,0 +1,53 @@ +// Copyright 2021 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.instrumentor; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; + +public class MockCoverageMap { + public static final int SIZE = 65536; + public static final ByteBuffer counters = ByteBuffer.allocate(SIZE); + + private static final ByteBuffer previous_mem = ByteBuffer.allocate(SIZE); + public static ArrayList<Integer> locations = new ArrayList<>(); + + public static void updated() { + int updated_pos = -1; + for (int i = 0; i < SIZE; i++) { + if (previous_mem.get(i) != counters.get(i)) { + updated_pos = i; + } + } + locations.add(updated_pos); + System.arraycopy(counters.array(), 0, previous_mem.array(), 0, SIZE); + } + + public static void enlargeIfNeeded(int nextId) { + // This mock coverage map is statically sized. + } + + public static void recordCoverage(int id) { + byte counter = counters.get(id); + counters.put(id, (byte) (counter == -1 ? 1 : counter + 1)); + } + + public static void clear() { + Arrays.fill(counters.array(), (byte) 0); + Arrays.fill(previous_mem.array(), (byte) 0); + locations.clear(); + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/instrumentor/MockTraceDataFlowCallbacks.java b/src/test/java/com/code_intelligence/jazzer/instrumentor/MockTraceDataFlowCallbacks.java new file mode 100644 index 00000000..ad659da0 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/instrumentor/MockTraceDataFlowCallbacks.java @@ -0,0 +1,106 @@ +// Copyright 2021 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.instrumentor; + +import java.util.ArrayList; +import java.util.List; + +@SuppressWarnings("unused") +public class MockTraceDataFlowCallbacks { + private static List<String> hookCalls; + private static int assertedCalls; + + public static void init() { + hookCalls = new ArrayList<>(); + assertedCalls = 0; + } + + public static boolean hookCall(String expectedCall) { + if (assertedCalls >= hookCalls.size()) { + System.err.println("Not seen (" + hookCalls.size() + " calls, but " + (assertedCalls + 1) + + " expected): " + expectedCall); + return false; + } + + if (!hookCalls.get(assertedCalls).equals(expectedCall)) { + System.err.println("Call " + expectedCall + " not seen, got " + hookCalls.get(assertedCalls)); + return false; + } + + assertedCalls++; + return true; + } + + public static boolean finish() { + if (assertedCalls == hookCalls.size()) + return true; + System.err.println("The following calls were not asserted:"); + for (int i = assertedCalls; i < hookCalls.size(); i++) { + System.err.println(hookCalls.get(i)); + } + + return false; + } + + public static void traceCmpLong(long arg1, long arg2, int pc) { + hookCalls.add("LCMP: " + Math.min(arg1, arg2) + ", " + Math.max(arg1, arg2)); + } + + public static void traceCmpInt(int arg1, int arg2, int pc) { + hookCalls.add("ICMP: " + Math.min(arg1, arg2) + ", " + Math.max(arg1, arg2)); + } + + public static void traceConstCmpInt(int arg1, int arg2, int pc) { + hookCalls.add("CICMP: " + arg1 + ", " + arg2); + } + + public static void traceDivInt(int val, int pc) { + hookCalls.add("IDIV: " + val); + } + + public static void traceDivLong(long val, int pc) { + hookCalls.add("LDIV: " + val); + } + + public static void traceGep(long idx, int pc) { + hookCalls.add("GEP: " + idx); + } + + public static void traceSwitch(long switchValue, long[] libfuzzerCaseValues, int pc) { + if (libfuzzerCaseValues.length < 3 + // number of case values must match length + || libfuzzerCaseValues[0] != libfuzzerCaseValues.length - 2 + // bit size of case values is always 32 (int) + || libfuzzerCaseValues[1] != 32) { + hookCalls.add("INVALID_SWITCH"); + return; + } + + StringBuilder builder = new StringBuilder("SWITCH: " + switchValue + ", ("); + for (int i = 2; i < libfuzzerCaseValues.length; i++) { + builder.append(libfuzzerCaseValues[i]); + builder.append(", "); + } + builder.append(")"); + hookCalls.add(builder.toString()); + } + + public static int traceCmpLongWrapper(long value1, long value2, int pc) { + traceCmpLong(value1, value2, pc); + // Long.compare serves as a substitute for the lcmp opcode here + // (behaviour is the same) + return Long.compare(value1, value2); + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/instrumentor/PatchTestUtils.kt b/src/test/java/com/code_intelligence/jazzer/instrumentor/PatchTestUtils.kt new file mode 100644 index 00000000..de2cc187 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/instrumentor/PatchTestUtils.kt @@ -0,0 +1,64 @@ +// Copyright 2021 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.instrumentor + +import java.io.FileOutputStream + +object PatchTestUtils { + @JvmStatic + fun classToBytecode(targetClass: Class<*>): ByteArray { + return ClassLoader + .getSystemClassLoader() + .getResourceAsStream("${targetClass.name.replace('.', '/')}.class")!! + .use { + it.readBytes() + } + } + + @JvmStatic + fun bytecodeToClass(name: String, bytecode: ByteArray): Class<*> { + return BytecodeClassLoader(name, bytecode).loadClass(name) + } + + @JvmStatic + fun dumpBytecode(outDir: String, name: String, originalBytecode: ByteArray) { + FileOutputStream("$outDir/$name.class").use { fos -> fos.write(originalBytecode) } + } + + /** + * A ClassLoader that dynamically loads a single specified class from byte code and delegates all other class loads to + * its own ClassLoader. + */ + class BytecodeClassLoader(val className: String, private val classBytecode: ByteArray) : + ClassLoader(BytecodeClassLoader::class.java.classLoader) { + override fun loadClass(name: String): Class<*> { + if (name != className) { + return super.loadClass(name) + } + return defineClass(className, classBytecode, 0, classBytecode.size) + } + } +} + +fun assertSelfCheck(target: DynamicTestContract, shouldPass: Boolean = true) { + val results = target.selfCheck() + for ((test, passed) in results) { + if (shouldPass) { + assert(passed) { "$test should pass" } + } else { + assert(!passed) { "$test should not pass" } + } + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/instrumentor/ReplaceHooks.java b/src/test/java/com/code_intelligence/jazzer/instrumentor/ReplaceHooks.java new file mode 100644 index 00000000..7e31b77b --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/instrumentor/ReplaceHooks.java @@ -0,0 +1,136 @@ +// Copyright 2021 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.instrumentor; + +import com.code_intelligence.jazzer.api.HookType; +import com.code_intelligence.jazzer.api.MethodHook; +import java.lang.invoke.MethodHandle; + +@SuppressWarnings("unused") +public class ReplaceHooks { + @MethodHook(type = HookType.REPLACE, + targetClassName = "com.code_intelligence.jazzer.instrumentor.ReplaceHooksTarget", + targetMethod = "shouldReturnTrue1") + public static boolean + patchShouldReturnTrue1(MethodHandle method, Object thisObject, Object[] arguments, int hookId) { + return true; + } + + @MethodHook(type = HookType.REPLACE, + targetClassName = "com.code_intelligence.jazzer.instrumentor.ReplaceHooksTarget", + targetMethod = "shouldReturnTrue2") + public static Boolean + patchShouldReturnTrue2(MethodHandle method, Object thisObject, Object[] arguments, int hookId) { + return true; + } + + @MethodHook(type = HookType.REPLACE, + targetClassName = "com.code_intelligence.jazzer.instrumentor.ReplaceHooksTarget", + targetMethod = "shouldReturnTrue3") + public static Object + patchShouldReturnTrue3(MethodHandle method, Object thisObject, Object[] arguments, int hookId) { + return true; + } + + @MethodHook(type = HookType.REPLACE, + targetClassName = "com.code_intelligence.jazzer.instrumentor.ReplaceHooksTarget", + targetMethod = "shouldReturnFalse1") + public static Boolean + patchShouldReturnFalse1(MethodHandle method, Object thisObject, Object[] arguments, int hookId) { + return false; + } + + @MethodHook(type = HookType.REPLACE, + targetClassName = "com.code_intelligence.jazzer.instrumentor.ReplaceHooksTarget", + targetMethod = "shouldReturnFalse2") + @MethodHook(type = HookType.REPLACE, + targetClassName = "com.code_intelligence.jazzer.instrumentor.ReplaceHooksTarget", + targetMethod = "shouldReturnFalse3") + public static Object + patchShouldReturnFalse2(MethodHandle method, Object thisObject, Object[] arguments, int hookId) { + return false; + } + + @MethodHook(type = HookType.REPLACE, + targetClassName = "com.code_intelligence.jazzer.instrumentor.ReplaceHooksTarget", + targetMethod = "shouldReturnReversed", + targetMethodDescriptor = "(Ljava/lang/String;)Ljava/lang/String;") + public static String + patchShouldReturnReversed( + MethodHandle method, Object thisObject, Object[] arguments, int hookId) { + return new StringBuilder((String) arguments[0]).reverse().toString(); + } + + @MethodHook(type = HookType.REPLACE, + targetClassName = "com.code_intelligence.jazzer.instrumentor.ReplaceHooksTarget", + targetMethod = "shouldIncrement") + public static int + patchShouldIncrement(MethodHandle method, Object thisObject, Object[] arguments, int hookId) { + return ((int) arguments[0]) + 1; + } + + @MethodHook(type = HookType.REPLACE, + targetClassName = "com.code_intelligence.jazzer.instrumentor.ReplaceHooksTarget", + targetMethod = "shouldCallPass") + public static void + patchShouldCallPass(MethodHandle method, Object thisObject, Object[] arguments, int hookId) { + ((ReplaceHooksTargetContract) thisObject).pass("shouldCallPass"); + } + + @MethodHook(type = HookType.REPLACE, + targetClassName = "com.code_intelligence.jazzer.instrumentor.ReplaceHooksTarget", + targetMethod = "idempotent", targetMethodDescriptor = "(I)I") + public static int + patchIdempotent(MethodHandle method, Object thisObject, Object[] arguments, int hookId) + throws Throwable { + // Iterate the function twice to pass the test. + int input = (int) arguments[0]; + int temp = (int) method.invokeWithArguments(thisObject, input); + return (int) method.invokeWithArguments(thisObject, temp); + } + + @MethodHook(type = HookType.REPLACE, targetClassName = "java.util.AbstractList", + targetMethod = "get", targetMethodDescriptor = "(I)Ljava/lang/Object;") + public static Object + patchAbstractListGet(MethodHandle method, Object thisObject, Object[] arguments, int hookId) { + return true; + } + + @MethodHook(type = HookType.REPLACE, targetClassName = "java.util.Set", targetMethod = "contains", + targetMethodDescriptor = "(Ljava/lang/Object;)Z") + public static boolean + patchSetGet(MethodHandle method, Object thisObject, Object[] arguments, int hookId) { + return true; + } + + @MethodHook(type = HookType.REPLACE, + targetClassName = "com.code_intelligence.jazzer.instrumentor.ReplaceHooksInit", + targetMethod = "<init>", targetMethodDescriptor = "()V") + public static ReplaceHooksInit + patchInit(MethodHandle method, Object thisObject, Object[] arguments, int hookId) { + // Test with subclass + return new ReplaceHooksInit() { + { initialized = true; } + }; + } + + @MethodHook(type = HookType.REPLACE, + targetClassName = "com.code_intelligence.jazzer.instrumentor.ReplaceHooksInit", + targetMethod = "<init>", targetMethodDescriptor = "(ZLjava/lang/String;)V") + public static ReplaceHooksInit + patchInitWithParams(MethodHandle method, Object thisObject, Object[] arguments, int hookId) { + return new ReplaceHooksInit(true, ""); + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/instrumentor/ReplaceHooksInit.java b/src/test/java/com/code_intelligence/jazzer/instrumentor/ReplaceHooksInit.java new file mode 100644 index 00000000..da77be81 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/instrumentor/ReplaceHooksInit.java @@ -0,0 +1,26 @@ +// Copyright 2021 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.instrumentor; + +public class ReplaceHooksInit { + public boolean initialized; + + public ReplaceHooksInit() {} + + @SuppressWarnings("unused") + public ReplaceHooksInit(boolean initialized, String ignored) { + this.initialized = initialized; + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/instrumentor/ReplaceHooksPatchTest.kt b/src/test/java/com/code_intelligence/jazzer/instrumentor/ReplaceHooksPatchTest.kt new file mode 100644 index 00000000..275c43f9 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/instrumentor/ReplaceHooksPatchTest.kt @@ -0,0 +1,89 @@ +// Copyright 2021 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.instrumentor + +import com.code_intelligence.jazzer.instrumentor.PatchTestUtils.bytecodeToClass +import com.code_intelligence.jazzer.instrumentor.PatchTestUtils.classToBytecode +import org.junit.Test +import java.io.File + +private fun getOriginalReplaceHooksTargetInstance(): ReplaceHooksTargetContract { + return ReplaceHooksTarget() +} + +private fun getNoHooksReplaceHooksTargetInstance(): ReplaceHooksTargetContract { + val originalBytecode = classToBytecode(ReplaceHooksTarget::class.java) + // Let the bytecode pass through the hooking logic, but don't apply any hooks. + val patchedBytecode = HookInstrumentor(emptyList(), false, null).instrument( + ReplaceHooksTarget::class.java.name.replace('.', '/'), + originalBytecode, + ) + val patchedClass = bytecodeToClass(ReplaceHooksTarget::class.java.name, patchedBytecode) + return patchedClass.getDeclaredConstructor().newInstance() as ReplaceHooksTargetContract +} + +private fun getPatchedReplaceHooksTargetInstance(classWithHooksEnabledField: Class<*>?): ReplaceHooksTargetContract { + val originalBytecode = classToBytecode(ReplaceHooksTarget::class.java) + val hooks = Hooks.loadHooks(emptyList(), setOf(ReplaceHooks::class.java.name)).first().hooks + val patchedBytecode = HookInstrumentor( + hooks, + false, + classWithHooksEnabledField = classWithHooksEnabledField?.name?.replace('.', '/'), + ).instrument(ReplaceHooksTarget::class.java.name.replace('.', '/'), originalBytecode) + // Make the patched class available in bazel-testlogs/.../test.outputs for manual inspection. + val outDir = System.getenv("TEST_UNDECLARED_OUTPUTS_DIR") + File("$outDir/${ReplaceHooksTarget::class.java.simpleName}.class").writeBytes(originalBytecode) + File("$outDir/${ReplaceHooksTarget::class.java.simpleName}.patched.class").writeBytes(patchedBytecode) + val patchedClass = bytecodeToClass(ReplaceHooksTarget::class.java.name, patchedBytecode) + return patchedClass.getDeclaredConstructor().newInstance() as ReplaceHooksTargetContract +} + +class ReplaceHooksPatchTest { + + @Test + fun testOriginal() { + assertSelfCheck(getOriginalReplaceHooksTargetInstance(), false) + } + + @Test + fun testPatchedWithoutHooks() { + assertSelfCheck(getNoHooksReplaceHooksTargetInstance(), false) + } + + @Test + fun testPatched() { + assertSelfCheck(getPatchedReplaceHooksTargetInstance(null), true) + } + + object HooksEnabled { + @Suppress("unused") + const val hooksEnabled = true + } + + object HooksDisabled { + @Suppress("unused") + const val hooksEnabled = false + } + + @Test + fun testPatchedWithConditionalHooksEnabled() { + assertSelfCheck(getPatchedReplaceHooksTargetInstance(HooksEnabled::class.java), true) + } + + @Test + fun testPatchedWithConditionalHooksDisabled() { + assertSelfCheck(getPatchedReplaceHooksTargetInstance(HooksDisabled::class.java), false) + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/instrumentor/ReplaceHooksTarget.java b/src/test/java/com/code_intelligence/jazzer/instrumentor/ReplaceHooksTarget.java new file mode 100644 index 00000000..fadbdf80 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/instrumentor/ReplaceHooksTarget.java @@ -0,0 +1,126 @@ +// Copyright 2021 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.instrumentor; + +import java.security.SecureRandom; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; + +// selfCheck() only passes with the hooks in ReplaceHooks.java applied. +public class ReplaceHooksTarget implements ReplaceHooksTargetContract { + Map<String, Boolean> results = new HashMap<>(); + + public static boolean shouldReturnTrue3() { + // return true; + return false; + } + + public Map<String, Boolean> selfCheck() { + results = new HashMap<>(); + + results.put("shouldReturnTrue1", shouldReturnTrue1()); + results.put("shouldReturnTrue2", shouldReturnTrue2()); + results.put("shouldReturnTrue3", shouldReturnTrue3()); + try { + boolean notTrue = false; + results.put("shouldReturnFalse1", notTrue); + if (!results.get("shouldReturnFalse1")) + results.put("shouldReturnFalse1", !shouldReturnFalse1()); + boolean notFalse = true; + results.put("shouldReturnFalse2", !shouldReturnFalse2() && notFalse); + results.put("shouldReturnFalse3", !shouldReturnFalse3()); + } catch (Exception e) { + boolean notTrue = false; + results.put("shouldNotBeExecuted", notTrue); + } + results.put("shouldReturnReversed", shouldReturnReversed("foo").equals("oof")); + results.put("shouldIncrement", shouldIncrement(5) == 6); + results.put("verifyIdentity", verifyIdentity()); + + results.put("shouldCallPass", false); + if (!results.get("shouldCallPass")) { + shouldCallPass(); + } + + ArrayList<Boolean> boolList = new ArrayList<>(); + boolList.add(false); + results.put("arrayListGet", boolList.get(0)); + + HashSet<Boolean> boolSet = new HashSet<>(); + results.put("stringSetGet", boolSet.contains(Boolean.TRUE)); + + results.put("shouldInitialize", new ReplaceHooksInit().initialized); + results.put("shouldInitializeWithParams", new ReplaceHooksInit(false, "foo").initialized); + + return results; + } + + public boolean shouldReturnTrue1() { + // return true; + return false; + } + + public boolean shouldReturnTrue2() { + // return true; + return false; + } + + protected Boolean shouldReturnFalse1() { + // return false; + return true; + } + + Boolean shouldReturnFalse2() { + // return false; + return true; + } + + public Boolean shouldReturnFalse3() { + // return false; + return true; + } + + public String shouldReturnReversed(String input) { + // return new StringBuilder(input).reverse().toString(); + return input; + } + + public int shouldIncrement(int input) { + // return input + 1; + return input; + } + + private void shouldCallPass() { + // pass("shouldCallPass"); + } + + private boolean verifyIdentity() { + SecureRandom rand = new SecureRandom(); + int input = rand.nextInt(); + // return idempotent(idempotent(input)) == input; + return idempotent(input) == input; + } + + private int idempotent(int input) { + int secret = 0x12345678; + return input ^ secret; + } + + public void pass(String test) { + results.put(test, true); + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/instrumentor/ReplaceHooksTargetContract.java b/src/test/java/com/code_intelligence/jazzer/instrumentor/ReplaceHooksTargetContract.java new file mode 100644 index 00000000..e3dff93e --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/instrumentor/ReplaceHooksTargetContract.java @@ -0,0 +1,23 @@ +// Copyright 2021 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.instrumentor; + +/** + * Helper interface used to call methods on instances of ReplaceHooksTarget classes loaded via + * different class loaders. + */ +public interface ReplaceHooksTargetContract extends DynamicTestContract { + void pass(String test); +} diff --git a/src/test/java/com/code_intelligence/jazzer/instrumentor/TraceDataFlowInstrumentationTarget.java b/src/test/java/com/code_intelligence/jazzer/instrumentor/TraceDataFlowInstrumentationTarget.java new file mode 100644 index 00000000..d8e28881 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/instrumentor/TraceDataFlowInstrumentationTarget.java @@ -0,0 +1,153 @@ +// Copyright 2021 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.instrumentor; + +import java.nio.ByteBuffer; +import java.util.AbstractList; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Stack; +import java.util.Vector; + +public class TraceDataFlowInstrumentationTarget implements DynamicTestContract { + volatile long long1 = 1; + volatile long long2 = 1; + volatile long long3 = 2; + volatile long long4 = 3; + + volatile int int1 = 4; + volatile int int2 = 4; + volatile int int3 = 6; + volatile int int4 = 5; + + volatile int switchValue = 1200; + + @SuppressWarnings("ReturnValueIgnored") + @Override + public Map<String, Boolean> selfCheck() { + Map<String, Boolean> results = new HashMap<>(); + + results.put("longCompareEq", long1 == long2); + results.put("longCompareNe", long3 != long4); + + results.put("intCompareEq", int1 == int2); + results.put("intCompareNe", int3 != int4); + results.put("intCompareLt", int4 < int3); + results.put("intCompareLe", int4 <= int3); + results.put("intCompareGt", int3 > int4); + results.put("intCompareGe", int3 >= int4); + + // Not instrumented since all case values are non-negative and < 256. + switch (switchValue) { + case 119: + case 120: + case 121: + results.put("tableSwitchUninstrumented", false); + break; + default: + results.put("tableSwitchUninstrumented", true); + } + + // Not instrumented since all case values are non-negative and < 256. + switch (switchValue) { + case 1: + case 200: + results.put("lookupSwitchUninstrumented", false); + break; + default: + results.put("lookupSwitchUninstrumented", true); + } + + results.put("emptySwitchUninstrumented", false); + switch (switchValue) { + default: + results.put("emptySwitchUninstrumented", true); + } + + switch (switchValue) { + case 1000: + case 1001: + // case 1002: The tableswitch instruction will contain a gap case for 1002. + case 1003: + results.put("tableSwitch", false); + break; + default: + results.put("tableSwitch", true); + } + + switch (-switchValue) { + case -1200: + results.put("lookupSwitch", true); + break; + case -1: + case -10: + case -1000: + case 200: + default: + results.put("lookupSwitch", false); + } + + results.put("intDiv", (int3 / 2) == 3); + + results.put("longDiv", (long4 / 2) == 1); + + String[] referenceArray = {"foo", "foo", "bar"}; + boolean[] boolArray = {false, false, true}; + byte[] byteArray = {0, 0, 2}; + char[] charArray = {0, 0, 0, 3}; + double[] doubleArray = {0, 0, 0, 0, 4}; + float[] floatArray = {0, 0, 0, 0, 0, 5}; + int[] intArray = {0, 0, 0, 0, 0, 0, 6}; + long[] longArray = {0, 0, 0, 0, 0, 0, 0, 7}; + short[] shortArray = {0, 0, 0, 0, 0, 0, 0, 0, 8}; + + results.put("referenceArrayGep", referenceArray[2].equals("bar")); + results.put("boolArrayGep", boolArray[2]); + results.put("byteArrayGep", byteArray[2] == 2); + results.put("charArrayGep", charArray[3] == 3); + results.put("doubleArrayGep", doubleArray[4] == 4); + results.put("floatArrayGep", floatArray[5] == 5); + results.put("intArrayGep", intArray[6] == 6); + results.put("longArrayGep", longArray[7] == 7); + results.put("shortArrayGep", shortArray[8] == 8); + + ByteBuffer buffer = ByteBuffer.allocate(100); + buffer.get(2); + buffer.getChar(3); + buffer.getDouble(4); + buffer.getFloat(5); + buffer.getInt(6); + buffer.getLong(7); + buffer.getShort(8); + + "foobarbazbat".charAt(9); + "foobarbazbat".codePointAt(10); + new StringBuilder("foobarbazbat").charAt(11); + + (new Vector<>(Collections.nCopies(20, "foo"))).get(12); + (new ArrayList<>(Collections.nCopies(20, "foo"))).get(13); + Stack<String> stack = new Stack<>(); + for (int i = 0; i < 20; i++) stack.push("foo"); + stack.get(14); + stack.get(15); + ((AbstractList<String>) stack).get(16); + ((List<String>) stack).get(17); + + return results; + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/instrumentor/TraceDataFlowInstrumentationTest.kt b/src/test/java/com/code_intelligence/jazzer/instrumentor/TraceDataFlowInstrumentationTest.kt new file mode 100644 index 00000000..b7383f1f --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/instrumentor/TraceDataFlowInstrumentationTest.kt @@ -0,0 +1,143 @@ +// Copyright 2021 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.instrumentor + +import com.code_intelligence.jazzer.instrumentor.PatchTestUtils.bytecodeToClass +import com.code_intelligence.jazzer.instrumentor.PatchTestUtils.classToBytecode +import org.junit.Test +import java.io.File + +private fun getOriginalInstrumentationTargetInstance(): DynamicTestContract { + return TraceDataFlowInstrumentationTarget() +} + +private fun getInstrumentedInstrumentationTargetInstance(): DynamicTestContract { + val originalBytecode = classToBytecode(TraceDataFlowInstrumentationTarget::class.java) + val patchedBytecode = TraceDataFlowInstrumentor( + setOf( + InstrumentationType.CMP, + InstrumentationType.DIV, + InstrumentationType.GEP, + ), + MockTraceDataFlowCallbacks::class.java.name.replace('.', '/'), + ).instrument(TraceDataFlowInstrumentationTarget::class.java.name.replace('.', '/'), originalBytecode) + // Make the patched class available in bazel-testlogs/.../test.outputs for manual inspection. + val outDir = System.getenv("TEST_UNDECLARED_OUTPUTS_DIR") + File("$outDir/${TraceDataFlowInstrumentationTarget::class.simpleName}.class").writeBytes(originalBytecode) + File("$outDir/${TraceDataFlowInstrumentationTarget::class.simpleName}.patched.class").writeBytes(patchedBytecode) + val patchedClass = bytecodeToClass(TraceDataFlowInstrumentationTarget::class.java.name, patchedBytecode) + return patchedClass.getDeclaredConstructor().newInstance() as DynamicTestContract +} + +class TraceDataFlowInstrumentationTest { + + @Test + fun testOriginal() { + MockTraceDataFlowCallbacks.init() + assertSelfCheck(getOriginalInstrumentationTargetInstance()) + assert(MockTraceDataFlowCallbacks.finish()) + } + + @Test + fun testInstrumented() { + MockTraceDataFlowCallbacks.init() + assertSelfCheck(getInstrumentedInstrumentationTargetInstance()) + listOf( + // long compares + "LCMP: 1, 1", + "LCMP: 2, 3", + // int compares + "ICMP: 4, 4", + "ICMP: 5, 6", + "ICMP: 5, 6", + "ICMP: 5, 6", + "ICMP: 5, 6", + "ICMP: 5, 6", + // tableswitch with gap + "SWITCH: 1200, (1000, 1001, 1003, )", + // lookupswitch + "SWITCH: -1200, (200, -1200, -1000, -10, -1, )", + // (6 / 2) == 3 + "IDIV: 2", + "ICMP: 3, 3", + // (3 / 2) == 1 + "LDIV: 2", + "LCMP: 1, 1", + // referenceArray[2] + "GEP: 2", + // boolArray[2] + "GEP: 2", + // byteArray[2] == 2 + "GEP: 2", + "ICMP: 2, 2", + // charArray[3] == 3 + "GEP: 3", + "ICMP: 3, 3", + // doubleArray[4] == 4 + "GEP: 4", + // floatArray[5] == 5 + "GEP: 5", + "CICMP: 0, 0", + // intArray[6] == 6 + "GEP: 6", + "ICMP: 6, 6", + // longArray[7] == 7 + "GEP: 7", + "LCMP: 7, 7", + // shortArray[8] == 8 + "GEP: 8", + "ICMP: 8, 8", + + "GEP: 2", + "GEP: 3", + "GEP: 4", + "GEP: 5", + "GEP: 6", + "GEP: 7", + "GEP: 8", + "GEP: 9", + "GEP: 10", + "GEP: 11", + "GEP: 12", + "GEP: 13", + "ICMP: 0, 20", + "ICMP: 1, 20", + "ICMP: 2, 20", + "ICMP: 3, 20", + "ICMP: 4, 20", + "ICMP: 5, 20", + "ICMP: 6, 20", + "ICMP: 7, 20", + "ICMP: 8, 20", + "ICMP: 9, 20", + "ICMP: 10, 20", + "ICMP: 11, 20", + "ICMP: 12, 20", + "ICMP: 13, 20", + "ICMP: 14, 20", + "ICMP: 15, 20", + "ICMP: 16, 20", + "ICMP: 17, 20", + "ICMP: 18, 20", + "ICMP: 19, 20", + "ICMP: 20, 20", + "GEP: 14", + "GEP: 15", + "GEP: 16", + "GEP: 17", + ).forEach { assert(MockTraceDataFlowCallbacks.hookCall(it)) } + assert(MockTraceDataFlowCallbacks.finish()) + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/instrumentor/ValidHookMocks.java b/src/test/java/com/code_intelligence/jazzer/instrumentor/ValidHookMocks.java new file mode 100644 index 00000000..a919242b --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/instrumentor/ValidHookMocks.java @@ -0,0 +1,45 @@ +// Copyright 2021 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.instrumentor; + +import com.code_intelligence.jazzer.api.HookType; +import com.code_intelligence.jazzer.api.MethodHook; +import java.lang.invoke.MethodHandle; + +class ValidHookMocks { + @MethodHook(type = HookType.BEFORE, targetClassName = "java.lang.String", targetMethod = "equals") + public static void validBeforeHook( + MethodHandle method, String thisObject, Object[] arguments, int hookId) {} + + @MethodHook(type = HookType.AFTER, targetClassName = "java.lang.String", targetMethod = "equals") + public static void validAfterHook(MethodHandle method, String thisObject, Object[] arguments, + int hookId, Boolean returnValue) {} + + @MethodHook(type = HookType.REPLACE, targetClassName = "java.lang.String", + targetMethod = "equals", targetMethodDescriptor = "(Ljava/lang/Object;)Z") + public static Boolean + validReplaceHook(MethodHandle method, String thisObject, Object[] arguments, int hookId) { + return true; + } + + @MethodHook( + type = HookType.REPLACE, targetClassName = "java.lang.String", targetMethod = "equals") + @MethodHook(type = HookType.REPLACE, targetClassName = "java.lang.String", + targetMethod = "equalsIgnoreCase") + public static boolean + validReplaceHook2(MethodHandle method, String thisObject, Object[] arguments, int hookId) { + return true; + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/junit/AutofuzzTest.java b/src/test/java/com/code_intelligence/jazzer/junit/AutofuzzTest.java new file mode 100644 index 00000000..b9abd3fe --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/junit/AutofuzzTest.java @@ -0,0 +1,162 @@ +// Copyright 2022 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.junit; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth.assertWithMessage; +import static com.google.common.truth.Truth8.assertThat; +import static org.junit.Assume.assumeFalse; +import static org.junit.Assume.assumeTrue; +import static org.junit.platform.engine.discovery.DiscoverySelectors.selectMethod; +import static org.junit.platform.testkit.engine.EventConditions.abortedWithReason; +import static org.junit.platform.testkit.engine.EventConditions.container; +import static org.junit.platform.testkit.engine.EventConditions.displayName; +import static org.junit.platform.testkit.engine.EventConditions.event; +import static org.junit.platform.testkit.engine.EventConditions.finishedSuccessfully; +import static org.junit.platform.testkit.engine.EventConditions.finishedWithFailure; +import static org.junit.platform.testkit.engine.EventConditions.test; +import static org.junit.platform.testkit.engine.EventConditions.type; +import static org.junit.platform.testkit.engine.EventConditions.uniqueIdSubstrings; +import static org.junit.platform.testkit.engine.EventType.DYNAMIC_TEST_REGISTERED; +import static org.junit.platform.testkit.engine.EventType.FINISHED; +import static org.junit.platform.testkit.engine.EventType.REPORTING_ENTRY_PUBLISHED; +import static org.junit.platform.testkit.engine.EventType.STARTED; +import static org.junit.platform.testkit.engine.TestExecutionResultConditions.instanceOf; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.junit.Rule; +import org.junit.Test; +import org.junit.platform.testkit.engine.EngineExecutionResults; +import org.junit.platform.testkit.engine.EngineTestKit; +import org.junit.rules.TemporaryFolder; +import org.opentest4j.TestAbortedException; + +public class AutofuzzTest { + @Rule public TemporaryFolder temp = new TemporaryFolder(); + + @Test + public void fuzzingEnabled() throws IOException { + assumeFalse(System.getenv("JAZZER_FUZZ").isEmpty()); + + Path baseDir = temp.getRoot().toPath(); + // Create a fake test resource directory structure to verify that Jazzer uses it and emits a + // crash file into it. + Path testResourceDir = baseDir.resolve("src").resolve("test").resolve("resources"); + Files.createDirectories(testResourceDir); + Path inputsDirectory = testResourceDir.resolve("com") + .resolve("example") + .resolve("AutofuzzFuzzTestInputs") + .resolve("autofuzz"); + + EngineExecutionResults results = + EngineTestKit.engine("junit-jupiter") + .selectors(selectMethod( + "com.example.AutofuzzFuzzTest#autofuzz(java.lang.String,com.example.AutofuzzFuzzTest$IntHolder)")) + .configurationParameter("jazzer.internal.basedir", baseDir.toAbsolutePath().toString()) + .execute(); + + final String engine = "engine:junit-jupiter"; + final String clazz = "class:com.example.AutofuzzFuzzTest"; + final String autofuzz = + "test-template:autofuzz(java.lang.String, com.example.AutofuzzFuzzTest$IntHolder)"; + final String invocation = "test-template-invocation:#"; + + results.containerEvents().assertEventsMatchExactly(event(type(STARTED), container(engine)), + event(type(STARTED), container(uniqueIdSubstrings(engine, clazz))), + event(type(STARTED), container(uniqueIdSubstrings(engine, clazz, autofuzz))), + event(type(FINISHED), container(uniqueIdSubstrings(engine, clazz, autofuzz)), + finishedSuccessfully()), + event(type(FINISHED), container(uniqueIdSubstrings(engine, clazz)), finishedSuccessfully()), + event(type(FINISHED), container(engine), finishedSuccessfully())); + + results.testEvents().assertEventsMatchExactly(event(type(DYNAMIC_TEST_REGISTERED)), + event(type(STARTED)), + event(test(uniqueIdSubstrings(engine, clazz, autofuzz, invocation + 1)), + displayName("<empty input>"), + abortedWithReason(instanceOf(TestAbortedException.class))), + event(type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(engine, clazz, autofuzz))), + event(type(STARTED), test(uniqueIdSubstrings(engine, clazz, autofuzz, invocation + 2)), + displayName("Fuzzing...")), + event(type(FINISHED), test(uniqueIdSubstrings(engine, clazz, autofuzz, invocation + 2)), + displayName("Fuzzing..."), finishedWithFailure(instanceOf(RuntimeException.class)))); + + // Should crash on an input that contains "jazzer", with the crash emitted into the + // automatically created inputs directory. + Path crashingInput; + try (Stream<Path> crashFiles = + Files.list(inputsDirectory) + .filter(path -> path.getFileName().toString().startsWith("crash-"))) { + List<Path> crashFilesList = crashFiles.collect(Collectors.toList()); + assertWithMessage("Expected crashing input in " + baseDir).that(crashFilesList).hasSize(1); + crashingInput = crashFilesList.get(0); + } + assertThat(new String(Files.readAllBytes(crashingInput), StandardCharsets.UTF_8)) + .contains("jazzer"); + + try (Stream<Path> seeds = Files.list(baseDir).filter(Files::isRegularFile)) { + assertThat(seeds).isEmpty(); + } + + // Verify that the engine created the generated corpus directory. Since the crash was not found + // on a seed, it should not be empty. + Path generatedCorpus = + baseDir.resolve(".cifuzz-corpus").resolve("com.example.AutofuzzFuzzTest"); + assertThat(Files.isDirectory(generatedCorpus)).isTrue(); + try (Stream<Path> entries = Files.list(generatedCorpus)) { + assertThat(entries).isNotEmpty(); + } + } + + @Test + public void fuzzingDisabled() { + assumeTrue(System.getenv("JAZZER_FUZZ").isEmpty()); + + EngineExecutionResults results = + EngineTestKit.engine("junit-jupiter") + .selectors(selectMethod( + "com.example.AutofuzzWithCorpusFuzzTest#autofuzzWithCorpus(java.lang.String,int)")) + .execute(); + + final String engine = "engine:junit-jupiter"; + final String clazz = "class:com.example.AutofuzzWithCorpusFuzzTest"; + final String autofuzzWithCorpus = "test-template:autofuzzWithCorpus(java.lang.String, int)"; + + results.containerEvents().assertEventsMatchExactly(event(type(STARTED), container(engine)), + event(type(STARTED), container(uniqueIdSubstrings(engine, clazz))), + event(type(STARTED), container(uniqueIdSubstrings(engine, clazz, autofuzzWithCorpus))), + // "No fuzzing has been performed..." + event(type(REPORTING_ENTRY_PUBLISHED), + container(uniqueIdSubstrings(engine, clazz, autofuzzWithCorpus))), + event(type(FINISHED), container(uniqueIdSubstrings(engine, clazz, autofuzzWithCorpus)), + finishedSuccessfully()), + event(type(FINISHED), container(uniqueIdSubstrings(engine, clazz)), finishedSuccessfully()), + event(type(FINISHED), container(engine), finishedSuccessfully())); + + results.testEvents().assertEventsMatchExactly(event(type(DYNAMIC_TEST_REGISTERED)), + event(type(STARTED)), + event(test("autofuzzWithCorpus", "<empty input>"), finishedSuccessfully()), + event(type(DYNAMIC_TEST_REGISTERED)), event(type(STARTED)), + event(test("autofuzzWithCorpus", "crashing_input"), + finishedWithFailure(instanceOf(RuntimeException.class)))); + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/junit/BUILD.bazel b/src/test/java/com/code_intelligence/jazzer/junit/BUILD.bazel new file mode 100644 index 00000000..a9a5e2ea --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/junit/BUILD.bazel @@ -0,0 +1,321 @@ +load("@contrib_rules_jvm//java:defs.bzl", "JUNIT5_DEPS", "java_junit5_test") + +java_library( + name = "test-method", + srcs = ["TestMethod.java"], + visibility = ["//src/test/java/com/code_intelligence/jazzer/junit:__pkg__"], + deps = [ + "@maven//:org_junit_platform_junit_platform_engine", + ], +) + +java_junit5_test( + name = "UtilsTest", + size = "small", + srcs = ["UtilsTest.java"], + deps = JUNIT5_DEPS + [ + "//src/main/java/com/code_intelligence/jazzer/junit:utils", + "@maven//:com_google_truth_extensions_truth_java8_extension", + "@maven//:com_google_truth_truth", + "@maven//:org_junit_jupiter_junit_jupiter_api", + "@maven//:org_junit_jupiter_junit_jupiter_params", + ], +) + +java_test( + name = "RegressionTestTest", + srcs = ["RegressionTestTest.java"], + test_class = "com.code_intelligence.jazzer.junit.RegressionTestTest", + runtime_deps = [ + "//examples/junit/src/test/java/com/example:ExampleFuzzTests_deploy.jar", + "@maven//:org_junit_jupiter_junit_jupiter_engine", + ], + deps = [ + "//src/main/java/com/code_intelligence/jazzer/api:hooks", + "@maven//:com_google_truth_extensions_truth_java8_extension", + "@maven//:com_google_truth_truth", + "@maven//:junit_junit", + "@maven//:org_junit_jupiter_junit_jupiter_api", + "@maven//:org_junit_platform_junit_platform_engine", + "@maven//:org_junit_platform_junit_platform_testkit", + "@maven//:org_opentest4j_opentest4j", + ], +) + +[ + java_test( + name = "FuzzingWithCrashTest" + JAZZER_FUZZ, + srcs = ["FuzzingWithCrashTest.java"], + env = { + "JAZZER_FUZZ": JAZZER_FUZZ, + }, + test_class = "com.code_intelligence.jazzer.junit.FuzzingWithCrashTest", + runtime_deps = [ + "//examples/junit/src/test/java/com/example:ExampleFuzzTests_deploy.jar", + "@maven//:org_junit_jupiter_junit_jupiter_engine", + ], + deps = [ + "//src/main/java/com/code_intelligence/jazzer/api:hooks", + "//src/test/java/com/code_intelligence/jazzer/junit:test-method", + "@maven//:com_google_truth_extensions_truth_java8_extension", + "@maven//:com_google_truth_truth", + "@maven//:junit_junit", + "@maven//:org_assertj_assertj_core", + "@maven//:org_junit_platform_junit_platform_engine", + "@maven//:org_junit_platform_junit_platform_launcher", + "@maven//:org_junit_platform_junit_platform_testkit", + "@maven//:org_opentest4j_opentest4j", + ], + ) + for JAZZER_FUZZ in [ + "", + "_fuzzing", + ] +] + +[ + java_test( + name = "FuzzingWithoutCrashTest" + JAZZER_FUZZ, + srcs = ["FuzzingWithoutCrashTest.java"], + env = { + "JAZZER_FUZZ": JAZZER_FUZZ, + }, + test_class = "com.code_intelligence.jazzer.junit.FuzzingWithoutCrashTest", + runtime_deps = [ + "//examples/junit/src/test/java/com/example:ExampleFuzzTests_deploy.jar", + "@maven//:org_junit_jupiter_junit_jupiter_engine", + ], + deps = [ + "//src/main/java/com/code_intelligence/jazzer/api:hooks", + "@maven//:com_google_truth_extensions_truth_java8_extension", + "@maven//:com_google_truth_truth", + "@maven//:junit_junit", + "@maven//:org_assertj_assertj_core", + "@maven//:org_junit_platform_junit_platform_engine", + "@maven//:org_junit_platform_junit_platform_testkit", + "@maven//:org_opentest4j_opentest4j", + ], + ) + for JAZZER_FUZZ in [ + "", + "_fuzzing", + ] +] + +[ + java_test( + name = "ValueProfileTest_" + str(JAZZER_VALUE_PROFILE), + srcs = ["ValueProfileTest.java"], + env = { + "JAZZER_FUZZ": "true", + "JAZZER_VALUE_PROFILE": str(JAZZER_VALUE_PROFILE), + }, + # The test is both CPU-intensive and sensitive to timing, which causes it to be flaky on + # slow runners (particularly macOS on GitHub Actions). Since we need to distinguish the two + # test variants by whether they find a finding, we can't just increase the timeout without + # the risk to make the other variant flaky. + tags = ["exclusive"] if JAZZER_VALUE_PROFILE else [], + test_class = "com.code_intelligence.jazzer.junit.ValueProfileTest", + runtime_deps = [ + "//examples/junit/src/test/java/com/example:ExampleFuzzTests_deploy.jar", + "@maven//:org_junit_jupiter_junit_jupiter_engine", + ], + deps = [ + "//src/main/java/com/code_intelligence/jazzer/api:hooks", + "@maven//:com_google_truth_extensions_truth_java8_extension", + "@maven//:com_google_truth_truth", + "@maven//:junit_junit", + "@maven//:org_junit_platform_junit_platform_engine", + "@maven//:org_junit_platform_junit_platform_testkit", + ], + ) + for JAZZER_VALUE_PROFILE in [ + True, + False, + ] +] + +[ + java_test( + name = "DirectoryInputsTest" + JAZZER_FUZZ, + srcs = ["DirectoryInputsTest.java"], + args = [ + # Add a test resource root containing the seed corpus directory in a Maven layout to + # the classpath rather than seeds in a resource directory packaged in a JAR, as + # would happen if we added the directory to java_test's resources. + "--main_advice_classpath=$(rootpath test_resources_root)", + ], + data = ["test_resources_root"], + env = { + "JAZZER_FUZZ": JAZZER_FUZZ, + }, + test_class = "com.code_intelligence.jazzer.junit.DirectoryInputsTest", + runtime_deps = [ + "//examples/junit/src/test/java/com/example:ExampleFuzzTests_deploy.jar", + "@maven//:org_junit_jupiter_junit_jupiter_engine", + ], + deps = [ + "//src/main/java/com/code_intelligence/jazzer/api:hooks", + "@maven//:com_google_truth_extensions_truth_java8_extension", + "@maven//:com_google_truth_truth", + "@maven//:junit_junit", + "@maven//:org_junit_platform_junit_platform_engine", + "@maven//:org_junit_platform_junit_platform_testkit", + ], + ) + for JAZZER_FUZZ in [ + "", + "_fuzzing", + ] +] + +[ + java_test( + name = "CorpusDirectoryTest" + JAZZER_FUZZ, + srcs = ["CorpusDirectoryTest.java"], + args = [ + # Add a test resource root containing the seed corpus directory in a Maven layout to + # the classpath rather than seeds in a resource directory packaged in a JAR, as + # would happen if we added the directory to java_test's resources. + "--main_advice_classpath=$(rootpath test_resources_root)", + ], + data = ["test_resources_root"], + env = { + "JAZZER_FUZZ": JAZZER_FUZZ, + }, + test_class = "com.code_intelligence.jazzer.junit.CorpusDirectoryTest", + runtime_deps = [ + "//examples/junit/src/test/java/com/example:ExampleFuzzTests_deploy.jar", + "@maven//:org_junit_jupiter_junit_jupiter_engine", + ], + deps = [ + "//src/main/java/com/code_intelligence/jazzer/api:hooks", + "@maven//:com_google_truth_extensions_truth_java8_extension", + "@maven//:com_google_truth_truth", + "@maven//:junit_junit", + "@maven//:org_junit_platform_junit_platform_engine", + "@maven//:org_junit_platform_junit_platform_testkit", + ], + ) + for JAZZER_FUZZ in [ + "", + "_fuzzing", + ] +] + +[ + java_test( + name = "AutofuzzTest" + JAZZER_FUZZ, + srcs = ["AutofuzzTest.java"], + env = { + "JAZZER_FUZZ": JAZZER_FUZZ, + }, + test_class = "com.code_intelligence.jazzer.junit.AutofuzzTest", + runtime_deps = [ + "//examples/junit/src/test/java/com/example:ExampleFuzzTests_deploy.jar", + "@maven//:org_junit_jupiter_junit_jupiter_engine", + ], + deps = [ + "//src/main/java/com/code_intelligence/jazzer/api:hooks", + "@maven//:com_google_truth_extensions_truth_java8_extension", + "@maven//:com_google_truth_truth", + "@maven//:junit_junit", + "@maven//:org_junit_platform_junit_platform_engine", + "@maven//:org_junit_platform_junit_platform_testkit", + "@maven//:org_opentest4j_opentest4j", + ], + ) + for JAZZER_FUZZ in [ + "", + "_fuzzing", + ] +] + +[ + java_test( + name = "LifecycleTest" + JAZZER_FUZZ, + srcs = ["LifecycleTest.java"], + env = { + "JAZZER_FUZZ": JAZZER_FUZZ, + }, + test_class = "com.code_intelligence.jazzer.junit.LifecycleTest", + runtime_deps = [ + "//examples/junit/src/test/java/com/example:ExampleFuzzTests_deploy.jar", + "@maven//:org_junit_jupiter_junit_jupiter_engine", + ], + deps = [ + "//src/main/java/com/code_intelligence/jazzer/api:hooks", + "@maven//:junit_junit", + "@maven//:org_junit_platform_junit_platform_engine", + "@maven//:org_junit_platform_junit_platform_testkit", + ], + ) + for JAZZER_FUZZ in [ + "", + "_fuzzing", + ] +] + +java_test( + name = "HermeticInstrumentationTest", + srcs = ["HermeticInstrumentationTest.java"], + test_class = "com.code_intelligence.jazzer.junit.HermeticInstrumentationTest", + runtime_deps = [ + "//examples/junit/src/test/java/com/example:ExampleFuzzTests_deploy.jar", + "@maven//:org_junit_jupiter_junit_jupiter_engine", + ], + deps = [ + "//src/main/java/com/code_intelligence/jazzer/api:hooks", + "@maven//:com_google_truth_truth", + "@maven//:junit_junit", + "@maven//:org_junit_platform_junit_platform_engine", + "@maven//:org_junit_platform_junit_platform_testkit", + ], +) + +java_test( + name = "FindingsBaseDirTest", + srcs = ["FindingsBaseDirTest.java"], + env = { + "JAZZER_FUZZ": "1", + }, + test_class = "com.code_intelligence.jazzer.junit.FindingsBaseDirTest", + runtime_deps = [ + "//examples/junit/src/test/java/com/example:ExampleFuzzTests_deploy.jar", + "@maven//:org_junit_jupiter_junit_jupiter_engine", + ], + deps = [ + "//src/main/java/com/code_intelligence/jazzer/api:hooks", + "@maven//:com_google_truth_extensions_truth_java8_extension", + "@maven//:com_google_truth_truth", + "@maven//:junit_junit", + "@maven//:org_junit_platform_junit_platform_engine", + "@maven//:org_junit_platform_junit_platform_testkit", + ], +) + +[ + java_test( + name = "MutatorTest" + JAZZER_FUZZ, + srcs = ["MutatorTest.java"], + env = { + "JAZZER_FUZZ": JAZZER_FUZZ, + }, + test_class = "com.code_intelligence.jazzer.junit.MutatorTest", + runtime_deps = [ + "//examples/junit/src/test/java/com/example:ExampleFuzzTests_deploy.jar", + "@maven//:org_junit_jupiter_junit_jupiter_engine", + ], + deps = [ + "//src/main/java/com/code_intelligence/jazzer/api:hooks", + "@maven//:junit_junit", + "@maven//:org_assertj_assertj_core", + "@maven//:org_junit_platform_junit_platform_engine", + "@maven//:org_junit_platform_junit_platform_testkit", + ], + ) + for JAZZER_FUZZ in [ + "", + "_fuzzing", + ] +] diff --git a/src/test/java/com/code_intelligence/jazzer/junit/CorpusDirectoryTest.java b/src/test/java/com/code_intelligence/jazzer/junit/CorpusDirectoryTest.java new file mode 100644 index 00000000..372718ef --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/junit/CorpusDirectoryTest.java @@ -0,0 +1,183 @@ +// Copyright 2023 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.junit; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth8.assertThat; +import static org.junit.Assume.assumeFalse; +import static org.junit.Assume.assumeTrue; +import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass; +import static org.junit.platform.testkit.engine.EventConditions.container; +import static org.junit.platform.testkit.engine.EventConditions.displayName; +import static org.junit.platform.testkit.engine.EventConditions.event; +import static org.junit.platform.testkit.engine.EventConditions.finishedSuccessfully; +import static org.junit.platform.testkit.engine.EventConditions.finishedWithFailure; +import static org.junit.platform.testkit.engine.EventConditions.test; +import static org.junit.platform.testkit.engine.EventConditions.type; +import static org.junit.platform.testkit.engine.EventConditions.uniqueIdSubstrings; +import static org.junit.platform.testkit.engine.EventType.DYNAMIC_TEST_REGISTERED; +import static org.junit.platform.testkit.engine.EventType.FINISHED; +import static org.junit.platform.testkit.engine.EventType.REPORTING_ENTRY_PUBLISHED; +import static org.junit.platform.testkit.engine.EventType.STARTED; +import static org.junit.platform.testkit.engine.TestExecutionResultConditions.instanceOf; + +import com.code_intelligence.jazzer.api.FuzzerSecurityIssueMedium; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.stream.Stream; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.platform.testkit.engine.EngineExecutionResults; +import org.junit.platform.testkit.engine.EngineTestKit; +import org.junit.rules.TemporaryFolder; + +public class CorpusDirectoryTest { + private static final String ENGINE = "engine:junit-jupiter"; + private static final String CLAZZ = "class:com.example.CorpusDirectoryFuzzTest"; + private static final String INPUTS_FUZZ = + "test-template:corpusDirectoryFuzz(com.code_intelligence.jazzer.api.FuzzedDataProvider)"; + private static final String INVOCATION = "test-template-invocation:#"; + + @Rule public TemporaryFolder temp = new TemporaryFolder(); + Path baseDir; + + @Before + public void setup() { + baseDir = temp.getRoot().toPath(); + } + + @Test + public void fuzzingEnabled() throws IOException { + assumeFalse(System.getenv("JAZZER_FUZZ").isEmpty()); + + // Create a fake test resource directory structure with an inputs directory to verify that + // Jazzer uses it and emits a crash file into it. + Path artifactsDirectory = baseDir.resolve(Paths.get("src", "test", "resources", "com", + "example", "CorpusDirectoryFuzzTestInputs", "corpusDirectoryFuzz")); + Files.createDirectories(artifactsDirectory); + + // An explicitly stated corpus directory should be used to save new corpus entries. + Path explicitGeneratedCorpus = baseDir.resolve(Paths.get("corpus")); + Files.createDirectories(explicitGeneratedCorpus); + + // The default generated corpus directory should only be used if no explicit corpus directory + // is given. + Path defaultGeneratedCorpus = baseDir.resolve( + Paths.get(".cifuzz-corpus", "com.example.CorpusDirectoryFuzzTest", "corpusDirectoryFuzz")); + + EngineExecutionResults results = + EngineTestKit.engine("junit-jupiter") + .selectors(selectClass("com.example.CorpusDirectoryFuzzTest")) + .configurationParameter("jazzer.internal.basedir", baseDir.toAbsolutePath().toString()) + // Add corpus directory as initial libFuzzer parameter. + .configurationParameter("jazzer.internal.arg.0", "fake_test_argv0") + .configurationParameter( + "jazzer.internal.arg.1", explicitGeneratedCorpus.toAbsolutePath().toString()) + .execute(); + + results.containerEvents().assertEventsMatchExactly(event(type(STARTED), container(ENGINE)), + event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ))), + event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))), + event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ)), + finishedSuccessfully()), + event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ)), finishedSuccessfully()), + event(type(FINISHED), container(ENGINE), finishedSuccessfully())); + + results.testEvents().assertEventsMatchExactly( + event(type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))), + event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 1))), + event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 1)), + displayName("<empty input>"), finishedSuccessfully()), + event(type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))), + event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 2))), + event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 2)), + displayName("seed"), finishedSuccessfully()), + event(type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))), + event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 3)), + displayName("Fuzzing...")), + event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 3)), + displayName("Fuzzing..."), + finishedWithFailure(instanceOf(FuzzerSecurityIssueMedium.class)))); + + // Crash file should be emitted into the artifacts directory and not into corpus directory. + assertCrashFileExistsIn(artifactsDirectory); + assertNoCrashFileExistsIn(baseDir); + assertNoCrashFileExistsIn(explicitGeneratedCorpus); + assertNoCrashFileExistsIn(defaultGeneratedCorpus); + + // Verify that corpus files are written to given corpus directory and not generated one. + assertThat(Files.list(explicitGeneratedCorpus)).isNotEmpty(); + assertThat(Files.list(defaultGeneratedCorpus)).isEmpty(); + } + + @Test + public void fuzzingDisabled() throws IOException { + assumeTrue(System.getenv("JAZZER_FUZZ").isEmpty()); + + Path corpusDirectory = baseDir.resolve(Paths.get("corpus")); + Files.createDirectories(corpusDirectory); + Files.createFile(corpusDirectory.resolve("corpus_entry")); + + EngineExecutionResults results = + EngineTestKit.engine("junit-jupiter") + .selectors(selectClass("com.example.CorpusDirectoryFuzzTest")) + .configurationParameter("jazzer.internal.basedir", baseDir.toAbsolutePath().toString()) + // Add corpus directory as initial libFuzzer parameter. + .configurationParameter("jazzer.internal.arg.0", "fake_test_argv0") + .configurationParameter( + "jazzer.internal.arg.1", corpusDirectory.toAbsolutePath().toString()) + .execute(); + + results.containerEvents().assertEventsMatchExactly(event(type(STARTED), container(ENGINE)), + event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ))), + event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))), + event(type(REPORTING_ENTRY_PUBLISHED), + container(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))), + event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))), + event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ)), finishedSuccessfully()), + event(type(FINISHED), container(ENGINE), finishedSuccessfully())); + + // Verify that corpus_entry is not picked up and corpus directory is ignored in regression mode. + results.testEvents().assertEventsMatchExactly( + event(type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))), + event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 1))), + event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 1)), + displayName("<empty input>"), finishedSuccessfully()), + event(type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))), + event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 2))), + event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 2)), + displayName("seed"), finishedSuccessfully())); + } + + private static void assertCrashFileExistsIn(Path artifactsDirectory) throws IOException { + try (Stream<Path> crashFiles = + Files.list(artifactsDirectory) + .filter(path -> path.getFileName().toString().startsWith("crash-"))) { + assertThat(crashFiles).isNotEmpty(); + } + } + + private static void assertNoCrashFileExistsIn(Path generatedCorpus) throws IOException { + try (Stream<Path> crashFiles = + Files.list(generatedCorpus) + .filter(path -> path.getFileName().toString().startsWith("crash-"))) { + assertThat(crashFiles).isEmpty(); + } + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/junit/DirectoryInputsTest.java b/src/test/java/com/code_intelligence/jazzer/junit/DirectoryInputsTest.java new file mode 100644 index 00000000..7ef27a37 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/junit/DirectoryInputsTest.java @@ -0,0 +1,162 @@ +// Copyright 2022 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.junit; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth8.assertThat; +import static org.junit.Assume.assumeFalse; +import static org.junit.Assume.assumeTrue; +import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass; +import static org.junit.platform.testkit.engine.EventConditions.container; +import static org.junit.platform.testkit.engine.EventConditions.displayName; +import static org.junit.platform.testkit.engine.EventConditions.event; +import static org.junit.platform.testkit.engine.EventConditions.finishedSuccessfully; +import static org.junit.platform.testkit.engine.EventConditions.finishedWithFailure; +import static org.junit.platform.testkit.engine.EventConditions.test; +import static org.junit.platform.testkit.engine.EventConditions.type; +import static org.junit.platform.testkit.engine.EventConditions.uniqueIdSubstrings; +import static org.junit.platform.testkit.engine.EventType.DYNAMIC_TEST_REGISTERED; +import static org.junit.platform.testkit.engine.EventType.FINISHED; +import static org.junit.platform.testkit.engine.EventType.REPORTING_ENTRY_PUBLISHED; +import static org.junit.platform.testkit.engine.EventType.STARTED; +import static org.junit.platform.testkit.engine.TestExecutionResultConditions.instanceOf; + +import com.code_intelligence.jazzer.api.FuzzerSecurityIssueMedium; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.stream.Stream; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.platform.testkit.engine.EngineExecutionResults; +import org.junit.platform.testkit.engine.EngineTestKit; +import org.junit.rules.TemporaryFolder; + +public class DirectoryInputsTest { + private static final String ENGINE = "engine:junit-jupiter"; + private static final String CLAZZ = "class:com.example.DirectoryInputsFuzzTest"; + private static final String INPUTS_FUZZ = + "test-template:inputsFuzz(com.code_intelligence.jazzer.api.FuzzedDataProvider)"; + private static final String INVOCATION = "test-template-invocation:#"; + + @Rule public TemporaryFolder temp = new TemporaryFolder(); + Path baseDir; + + @Before + public void setup() { + baseDir = temp.getRoot().toPath(); + } + + @Test + public void fuzzingEnabled() throws IOException { + assumeFalse(System.getenv("JAZZER_FUZZ").isEmpty()); + + // Create a fake test resource directory structure with an inputs directory to verify that + // Jazzer uses it and emits a crash file into it. + Path inputsDirectory = baseDir.resolve(Paths.get("src", "test", "resources", "com", "example", + "DirectoryInputsFuzzTestInputs", "inputsFuzz")); + Files.createDirectories(inputsDirectory); + + EngineExecutionResults results = + EngineTestKit.engine("junit-jupiter") + .selectors(selectClass("com.example.DirectoryInputsFuzzTest")) + .configurationParameter("jazzer.internal.basedir", baseDir.toAbsolutePath().toString()) + .execute(); + + results.containerEvents().assertEventsMatchExactly(event(type(STARTED), container(ENGINE)), + event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ))), + event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))), + event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ)), + finishedSuccessfully()), + event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ)), finishedSuccessfully()), + event(type(FINISHED), container(ENGINE), finishedSuccessfully())); + + results.testEvents().assertEventsMatchExactly( + event(type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))), + event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 1))), + event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 1)), + displayName("<empty input>"), finishedSuccessfully()), + event(type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))), + event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 2))), + event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 2)), + displayName("seed"), finishedWithFailure(instanceOf(FuzzerSecurityIssueMedium.class))), + event(type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))), + event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 3)), + displayName("Fuzzing...")), + event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 3)), + displayName("Fuzzing..."), + finishedWithFailure(instanceOf(FuzzerSecurityIssueMedium.class)))); + + // Should crash on the exact input "directory" as provided by the seed, with the crash emitted + // into the seed corpus. + try (Stream<Path> crashFiles = Files.list(baseDir).filter( + path -> path.getFileName().toString().startsWith("crash-"))) { + assertThat(crashFiles).isEmpty(); + } + try (Stream<Path> seeds = Files.list(inputsDirectory)) { + assertThat(seeds).containsExactly( + inputsDirectory.resolve("crash-8d392f56d616a516ceabb82ed8906418bce4647d")); + } + assertThat(Files.readAllBytes( + inputsDirectory.resolve("crash-8d392f56d616a516ceabb82ed8906418bce4647d"))) + .isEqualTo("directory".getBytes(StandardCharsets.UTF_8)); + + // Verify that the engine created the generated corpus directory. Since the crash was found on a + // seed, it should be empty. + Path generatedCorpus = baseDir.resolve( + Paths.get(".cifuzz-corpus", "com.example.DirectoryInputsFuzzTest", "inputsFuzz")); + assertThat(Files.isDirectory(generatedCorpus)).isTrue(); + try (Stream<Path> entries = Files.list(generatedCorpus)) { + assertThat(entries).isEmpty(); + } + } + + @Test + public void fuzzingDisabled() { + assumeTrue(System.getenv("JAZZER_FUZZ").isEmpty()); + + EngineExecutionResults results = + EngineTestKit.engine("junit-jupiter") + .selectors(selectClass("com.example.DirectoryInputsFuzzTest")) + .execute(); + + results.containerEvents().assertEventsMatchExactly(event(type(STARTED), container(ENGINE)), + event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ))), + event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))), + event(type(REPORTING_ENTRY_PUBLISHED), + container(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))), + event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))), + event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ)), finishedSuccessfully()), + event(type(FINISHED), container(ENGINE), finishedSuccessfully())); + + results.testEvents().assertEventsMatchExactly( + event(type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))), + event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 1))), + event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 1)), + displayName("<empty input>"), finishedSuccessfully()), + event(type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))), + event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 2))), + event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ, INVOCATION + 2)), + displayName("seed"), finishedWithFailure(instanceOf(FuzzerSecurityIssueMedium.class)))); + + // Verify that the generated corpus directory hasn't been created. + Path generatedCorpus = + baseDir.resolve(Paths.get(".cifuzz-corpus", "com.example.DirectoryInputsFuzzTest")); + assertThat(Files.notExists(generatedCorpus)).isTrue(); + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/junit/FindingsBaseDirTest.java b/src/test/java/com/code_intelligence/jazzer/junit/FindingsBaseDirTest.java new file mode 100644 index 00000000..b3100140 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/junit/FindingsBaseDirTest.java @@ -0,0 +1,83 @@ +// Copyright 2023 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.junit; + +import static com.google.common.truth.Truth8.assertThat; +import static org.junit.Assume.assumeFalse; +import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass; +import static org.junit.platform.testkit.engine.EventConditions.container; +import static org.junit.platform.testkit.engine.EventConditions.event; +import static org.junit.platform.testkit.engine.EventConditions.finishedSuccessfully; +import static org.junit.platform.testkit.engine.EventConditions.type; +import static org.junit.platform.testkit.engine.EventConditions.uniqueIdSubstrings; +import static org.junit.platform.testkit.engine.EventType.FINISHED; +import static org.junit.platform.testkit.engine.EventType.REPORTING_ENTRY_PUBLISHED; +import static org.junit.platform.testkit.engine.EventType.STARTED; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.stream.Stream; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.platform.testkit.engine.EngineExecutionResults; +import org.junit.platform.testkit.engine.EngineTestKit; +import org.junit.rules.TemporaryFolder; + +public class FindingsBaseDirTest { + private static final String ENGINE = "engine:junit-jupiter"; + private static final String CLAZZ = "class:com.example.ThrowingFuzzTest"; + private static final String INPUTS_FUZZ = + "test-template:throwingFuzz(com.code_intelligence.jazzer.api.FuzzedDataProvider)"; + + @Rule public TemporaryFolder temp = new TemporaryFolder(); + + private Path baseDir; + + @Before + public void setup() { + baseDir = temp.getRoot().toPath(); + } + + @Test + public void fuzzingEnabledNoFindingsDir() throws IOException { + assumeFalse(System.getenv("JAZZER_FUZZ").isEmpty()); + + EngineExecutionResults results = + EngineTestKit.engine("junit-jupiter") + .selectors(selectClass("com.example.ThrowingFuzzTest")) + .configurationParameter("jazzer.internal.basedir", baseDir.toAbsolutePath().toString()) + .execute(); + + results.containerEvents().assertEventsMatchExactly(event(type(STARTED), container(ENGINE)), + event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ))), + event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))), + // Warning because the inputs directory hasn't been found in the source tree. + event(type(REPORTING_ENTRY_PUBLISHED), + container(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ))), + event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ, INPUTS_FUZZ)), + finishedSuccessfully()), + event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ)), finishedSuccessfully()), + event(type(FINISHED), container(ENGINE), finishedSuccessfully())); + + // Crash should be emitted into the base directory, as no findings dir available. + try (Stream<Path> baseDirFiles = Files.list(baseDir)) { + Stream<Path> crashFiles = + baseDirFiles.filter(f -> f.getFileName().toString().startsWith("crash-")); + assertThat(crashFiles).hasSize(1); + } + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/junit/FuzzingWithCrashTest.java b/src/test/java/com/code_intelligence/jazzer/junit/FuzzingWithCrashTest.java new file mode 100644 index 00000000..5cc2d1c4 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/junit/FuzzingWithCrashTest.java @@ -0,0 +1,206 @@ +// Copyright 2022 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.junit; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth8.assertThat; +import static org.junit.Assume.assumeFalse; +import static org.junit.Assume.assumeTrue; +import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass; +import static org.junit.platform.testkit.engine.EventConditions.container; +import static org.junit.platform.testkit.engine.EventConditions.displayName; +import static org.junit.platform.testkit.engine.EventConditions.event; +import static org.junit.platform.testkit.engine.EventConditions.finishedSuccessfully; +import static org.junit.platform.testkit.engine.EventConditions.finishedWithFailure; +import static org.junit.platform.testkit.engine.EventConditions.test; +import static org.junit.platform.testkit.engine.EventConditions.type; +import static org.junit.platform.testkit.engine.EventConditions.uniqueIdSubstrings; +import static org.junit.platform.testkit.engine.EventType.DYNAMIC_TEST_REGISTERED; +import static org.junit.platform.testkit.engine.EventType.FINISHED; +import static org.junit.platform.testkit.engine.EventType.REPORTING_ENTRY_PUBLISHED; +import static org.junit.platform.testkit.engine.EventType.SKIPPED; +import static org.junit.platform.testkit.engine.EventType.STARTED; +import static org.junit.platform.testkit.engine.TestExecutionResultConditions.instanceOf; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.stream.Stream; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.platform.launcher.TagFilter; +import org.junit.platform.testkit.engine.EngineExecutionResults; +import org.junit.platform.testkit.engine.EngineTestKit; +import org.junit.rules.TemporaryFolder; +import org.opentest4j.AssertionFailedError; + +public class FuzzingWithCrashTest { + private static final String CRASHING_SEED_NAME = "crashing_seed"; + // Crashes ByteFuzzTest since 'b' % 2 == 0. + private static final byte[] CRASHING_SEED_CONTENT = new byte[] {'b', 'a', 'c'}; + private static final String CRASHING_SEED_DIGEST = "5e4dec23c9afa48bd5bee3daa2a0ab66e147012b"; + private static final String ENGINE = "engine:junit-jupiter"; + private static final String INVOCATION = "test-template-invocation:#"; + + private static final String CLAZZ_NAME = "com.example.ValidFuzzTests"; + + private static final String CLAZZ = "class:" + CLAZZ_NAME; + private static final TestMethod BYTE_FUZZ = new TestMethod(CLAZZ_NAME, "byteFuzz([B)"); + private static final TestMethod NO_CRASH_FUZZ = new TestMethod(CLAZZ_NAME, "noCrashFuzz([B)"); + private static final TestMethod DATA_FUZZ = + new TestMethod(CLAZZ_NAME, "dataFuzz(com.code_intelligence.jazzer.api.FuzzedDataProvider)"); + + @Rule public TemporaryFolder temp = new TemporaryFolder(); + Path baseDir; + Path inputsDirectory; + + @Before + public void setup() throws IOException { + baseDir = temp.getRoot().toPath(); + // Create a fake test resource directory structure with an inputs directory to verify that + // Jazzer uses it and emits a crash file into it. + inputsDirectory = baseDir.resolve( + Paths.get("src", "test", "resources", "com", "example", "ValidFuzzTestsInputs")); + // populate the same seed in all test directories + for (String method : + Arrays.asList(BYTE_FUZZ.getName(), NO_CRASH_FUZZ.getName(), DATA_FUZZ.getName())) { + Path methodInputsDirectory = inputsDirectory.resolve(method); + Files.createDirectories(methodInputsDirectory); + Files.write(methodInputsDirectory.resolve(CRASHING_SEED_NAME), CRASHING_SEED_CONTENT); + } + } + + private EngineExecutionResults executeTests() { + return EngineTestKit.engine("junit-jupiter") + .selectors(selectClass("com.example.ValidFuzzTests")) + .filters(TagFilter.includeTags("jazzer")) + .configurationParameter( + "jazzer.instrument", "com.other.package.**,com.example.**,com.yet.another.package.*") + .configurationParameter("jazzer.internal.basedir", baseDir.toAbsolutePath().toString()) + .execute(); + } + + @Test + public void fuzzingEnabled() throws IOException { + assumeFalse(System.getenv("JAZZER_FUZZ").isEmpty()); + + EngineExecutionResults results = executeTests(); + + results.containerEvents().assertEventsMatchExactly(event(type(STARTED), container(ENGINE)), + event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ))), + event(type(STARTED), + container(uniqueIdSubstrings(ENGINE, CLAZZ, BYTE_FUZZ.getDescriptorId()))), + event(type(FINISHED), + container(uniqueIdSubstrings(ENGINE, CLAZZ, BYTE_FUZZ.getDescriptorId())), + finishedSuccessfully()), + event(type(SKIPPED), + container(uniqueIdSubstrings(ENGINE, CLAZZ, NO_CRASH_FUZZ.getDescriptorId()))), + event(type(SKIPPED), + container(uniqueIdSubstrings(ENGINE, CLAZZ, DATA_FUZZ.getDescriptorId()))), + event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ)), finishedSuccessfully()), + event(type(FINISHED), container(ENGINE), finishedSuccessfully())); + + results.testEvents().assertEventsMatchLooselyInOrder( + event(type(DYNAMIC_TEST_REGISTERED), + test(uniqueIdSubstrings(ENGINE, CLAZZ, BYTE_FUZZ.getDescriptorId()))), + event(type(STARTED), + test(uniqueIdSubstrings(ENGINE, CLAZZ, BYTE_FUZZ.getDescriptorId(), INVOCATION)), + displayName("Fuzzing...")), + event(type(FINISHED), + test(uniqueIdSubstrings(ENGINE, CLAZZ, BYTE_FUZZ.getDescriptorId(), INVOCATION)), + displayName("Fuzzing..."), + finishedWithFailure(instanceOf(AssertionFailedError.class)))); + + // Jazzer first tries the empty input, which doesn't crash the ByteFuzzTest. The second input is + // the seed we planted, which is crashing, so verify that a crash file with the same content is + // created in our fake seed corpus, but not in the current working directory. + try (Stream<Path> crashFiles = Files.list(baseDir).filter( + path -> path.getFileName().toString().startsWith("crash-"))) { + assertThat(crashFiles).isEmpty(); + } + + // the crashing input will be created in the directory for the fuzzed test, in this case + // byteFuzz and will not exist in the directories of the other tests + Path byteFuzzInputDirectory = inputsDirectory.resolve(BYTE_FUZZ.getName()); + try (Stream<Path> seeds = Files.list(byteFuzzInputDirectory)) { + assertThat(seeds).containsExactly( + byteFuzzInputDirectory.resolve("crash-" + CRASHING_SEED_DIGEST), + byteFuzzInputDirectory.resolve(CRASHING_SEED_NAME)); + } + assertThat(Files.readAllBytes(byteFuzzInputDirectory.resolve("crash-" + CRASHING_SEED_DIGEST))) + .isEqualTo(CRASHING_SEED_CONTENT); + + // check that the others only include 1 file + for (String method : Arrays.asList(NO_CRASH_FUZZ.getName(), DATA_FUZZ.getName())) { + Path methodInputsDirectory = inputsDirectory.resolve(method); + try (Stream<Path> seeds = Files.list(methodInputsDirectory)) { + assertThat(seeds).containsExactly(methodInputsDirectory.resolve(CRASHING_SEED_NAME)); + } + } + + // Verify that the engine created the generated corpus directory. As a seed produced the crash, + // it should be empty. + Path generatedCorpus = + baseDir.resolve(Paths.get(".cifuzz-corpus", CLAZZ_NAME, BYTE_FUZZ.getName())); + assertThat(Files.isDirectory(generatedCorpus)).isTrue(); + try (Stream<Path> entries = Files.list(generatedCorpus)) { + assertThat(entries).isEmpty(); + } + } + + @Test + public void fuzzingDisabled() throws IOException { + assumeTrue(System.getenv("JAZZER_FUZZ").isEmpty()); + + EngineExecutionResults results = executeTests(); + + results.containerEvents().assertEventsMatchExactly(event(type(STARTED), container(ENGINE)), + event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ))), + event(type(STARTED), + container(uniqueIdSubstrings(ENGINE, CLAZZ, BYTE_FUZZ.getDescriptorId()))), + event(type(REPORTING_ENTRY_PUBLISHED), + container(uniqueIdSubstrings(ENGINE, CLAZZ, BYTE_FUZZ.getDescriptorId()))), + event(type(FINISHED), + container(uniqueIdSubstrings(ENGINE, CLAZZ, BYTE_FUZZ.getDescriptorId())), + finishedSuccessfully()), + event(type(STARTED), + container(uniqueIdSubstrings(ENGINE, CLAZZ, NO_CRASH_FUZZ.getDescriptorId()))), + event(type(REPORTING_ENTRY_PUBLISHED), + container(uniqueIdSubstrings(ENGINE, CLAZZ, NO_CRASH_FUZZ.getDescriptorId()))), + event(type(FINISHED), + container(uniqueIdSubstrings(ENGINE, CLAZZ, NO_CRASH_FUZZ.getDescriptorId()))), + event(type(STARTED), + container(uniqueIdSubstrings(ENGINE, CLAZZ, DATA_FUZZ.getDescriptorId()))), + event(type(REPORTING_ENTRY_PUBLISHED), + container(uniqueIdSubstrings(ENGINE, CLAZZ, DATA_FUZZ.getDescriptorId()))), + event(type(FINISHED), + container(uniqueIdSubstrings(ENGINE, CLAZZ, DATA_FUZZ.getDescriptorId()))), + event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ)), finishedSuccessfully()), + event(type(FINISHED), container(ENGINE), finishedSuccessfully())); + + // No fuzzing means no crashes means no new seeds. + // Check against all methods' input directories + for (String method : + Arrays.asList(BYTE_FUZZ.getName(), NO_CRASH_FUZZ.getName(), DATA_FUZZ.getName())) { + Path methodInputsDirectory = inputsDirectory.resolve(method); + try (Stream<Path> seeds = Files.list(methodInputsDirectory)) { + assertThat(seeds).containsExactly(methodInputsDirectory.resolve(CRASHING_SEED_NAME)); + } + } + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/junit/FuzzingWithoutCrashTest.java b/src/test/java/com/code_intelligence/jazzer/junit/FuzzingWithoutCrashTest.java new file mode 100644 index 00000000..01fe6252 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/junit/FuzzingWithoutCrashTest.java @@ -0,0 +1,149 @@ +// Copyright 2022 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.junit; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth8.assertThat; +import static org.junit.Assume.assumeFalse; +import static org.junit.Assume.assumeTrue; +import static org.junit.platform.engine.discovery.DiscoverySelectors.selectMethod; +import static org.junit.platform.testkit.engine.EventConditions.container; +import static org.junit.platform.testkit.engine.EventConditions.displayName; +import static org.junit.platform.testkit.engine.EventConditions.event; +import static org.junit.platform.testkit.engine.EventConditions.finishedSuccessfully; +import static org.junit.platform.testkit.engine.EventConditions.finishedWithFailure; +import static org.junit.platform.testkit.engine.EventConditions.test; +import static org.junit.platform.testkit.engine.EventConditions.type; +import static org.junit.platform.testkit.engine.EventConditions.uniqueIdSubstrings; +import static org.junit.platform.testkit.engine.EventType.DYNAMIC_TEST_REGISTERED; +import static org.junit.platform.testkit.engine.EventType.FINISHED; +import static org.junit.platform.testkit.engine.EventType.REPORTING_ENTRY_PUBLISHED; +import static org.junit.platform.testkit.engine.EventType.SKIPPED; +import static org.junit.platform.testkit.engine.EventType.STARTED; +import static org.junit.platform.testkit.engine.TestExecutionResultConditions.instanceOf; + +import com.google.common.truth.Truth8; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.stream.IntStream; +import java.util.stream.Stream; +import org.assertj.core.api.Condition; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.platform.testkit.engine.EngineExecutionResults; +import org.junit.platform.testkit.engine.EngineTestKit; +import org.junit.platform.testkit.engine.Event; +import org.junit.platform.testkit.engine.EventType; +import org.junit.platform.testkit.engine.Events; +import org.junit.rules.TemporaryFolder; +import org.opentest4j.AssertionFailedError; + +public class FuzzingWithoutCrashTest { + private static final String ENGINE = "engine:junit-jupiter"; + private static final String CLAZZ = "class:com.example.ValidFuzzTests"; + private static final String NO_CRASH_FUZZ = "test-template:noCrashFuzz([B)"; + private static final String INVOCATION = "test-template-invocation:#"; + @Rule public TemporaryFolder temp = new TemporaryFolder(); + Path baseDir; + + @Before + public void setup() { + baseDir = temp.getRoot().toPath(); + } + + private EngineExecutionResults executeTests() { + return EngineTestKit.engine("junit-jupiter") + .selectors(selectMethod("com.example.ValidFuzzTests#noCrashFuzz(byte[])")) + .configurationParameter( + "jazzer.instrument", "com.other.package.**,com.example.**,com.yet.another.package.*") + .configurationParameter("jazzer.internal.basedir", baseDir.toAbsolutePath().toString()) + .execute(); + } + + @Test + public void fuzzingEnabled() throws IOException { + assumeFalse(System.getenv("JAZZER_FUZZ").isEmpty()); + + EngineExecutionResults results = executeTests(); + + results.containerEvents().assertEventsMatchExactly(event(type(STARTED), container(ENGINE)), + event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ))), + event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ, NO_CRASH_FUZZ))), + // Warning because the inputs directory hasn't been found in the source tree. + event(type(REPORTING_ENTRY_PUBLISHED), + container(uniqueIdSubstrings(ENGINE, CLAZZ, NO_CRASH_FUZZ))), + // Warning because the inputs directory has been found on the classpath, but only in a JAR. + event(type(REPORTING_ENTRY_PUBLISHED), + container(uniqueIdSubstrings(ENGINE, CLAZZ, NO_CRASH_FUZZ))), + event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ, NO_CRASH_FUZZ)), + finishedSuccessfully()), + event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ)), finishedSuccessfully()), + event(type(FINISHED), container(ENGINE), finishedSuccessfully())); + + results.testEvents().assertEventsMatchLooselyInOrder( + event( + type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, NO_CRASH_FUZZ))), + event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, NO_CRASH_FUZZ, INVOCATION)), + displayName("Fuzzing...")), + event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, NO_CRASH_FUZZ, INVOCATION)), + displayName("Fuzzing..."), finishedSuccessfully())); + + // Verify that the engine created the generated corpus directory. As the fuzz test produces + // coverage (but no crash), it should not be empty. + Path generatedCorpus = + baseDir.resolve(Paths.get(".cifuzz-corpus", "com.example.ValidFuzzTests", "noCrashFuzz")); + assertThat(Files.isDirectory(generatedCorpus)).isTrue(); + try (Stream<Path> entries = Files.list(generatedCorpus)) { + assertThat(entries).isNotEmpty(); + } + } + + @Test + public void fuzzingDisabled() { + assumeTrue(System.getenv("JAZZER_FUZZ").isEmpty()); + + EngineExecutionResults results = executeTests(); + + results.containerEvents().assertEventsMatchExactly(event(type(STARTED), container(ENGINE)), + event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ))), + event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ, NO_CRASH_FUZZ))), + event(type(REPORTING_ENTRY_PUBLISHED), + container(uniqueIdSubstrings(ENGINE, CLAZZ, NO_CRASH_FUZZ))), + event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ, NO_CRASH_FUZZ))), + event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ)), finishedSuccessfully()), + event(type(FINISHED), container(ENGINE), finishedSuccessfully())); + + results.testEvents().assertEventsMatchExactly( + IntStream.rangeClosed(1, 6) + .boxed() + .flatMap(i + -> Stream.of(event(type(DYNAMIC_TEST_REGISTERED), + test(uniqueIdSubstrings(ENGINE, CLAZZ, NO_CRASH_FUZZ))), + event(type(STARTED), + test(uniqueIdSubstrings(ENGINE, CLAZZ, NO_CRASH_FUZZ, INVOCATION + i))), + event(type(FINISHED), + test(uniqueIdSubstrings(ENGINE, CLAZZ, NO_CRASH_FUZZ, INVOCATION + i)), + finishedSuccessfully()))) + .toArray(Condition[] ::new)); + + // Verify that the generated corpus directory hasn't been created. + Path generatedCorpus = + baseDir.resolve(Paths.get(".cifuzz-corpus", "com.example.ValidFuzzTests", "noCrashFuzz")); + assertThat(Files.notExists(generatedCorpus)).isTrue(); + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/junit/HermeticInstrumentationTest.java b/src/test/java/com/code_intelligence/jazzer/junit/HermeticInstrumentationTest.java new file mode 100644 index 00000000..dabbf352 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/junit/HermeticInstrumentationTest.java @@ -0,0 +1,108 @@ +// Copyright 2022 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.junit; + +import static org.junit.Assume.assumeTrue; +import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass; +import static org.junit.platform.testkit.engine.EventConditions.container; +import static org.junit.platform.testkit.engine.EventConditions.displayName; +import static org.junit.platform.testkit.engine.EventConditions.event; +import static org.junit.platform.testkit.engine.EventConditions.finishedSuccessfully; +import static org.junit.platform.testkit.engine.EventConditions.finishedWithFailure; +import static org.junit.platform.testkit.engine.EventConditions.test; +import static org.junit.platform.testkit.engine.EventConditions.type; +import static org.junit.platform.testkit.engine.EventConditions.uniqueIdSubstrings; +import static org.junit.platform.testkit.engine.EventType.DYNAMIC_TEST_REGISTERED; +import static org.junit.platform.testkit.engine.EventType.FINISHED; +import static org.junit.platform.testkit.engine.EventType.REPORTING_ENTRY_PUBLISHED; +import static org.junit.platform.testkit.engine.EventType.STARTED; +import static org.junit.platform.testkit.engine.TestExecutionResultConditions.instanceOf; + +import com.code_intelligence.jazzer.api.FuzzerSecurityIssueLow; +import java.nio.file.Path; +import java.util.regex.PatternSyntaxException; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.platform.testkit.engine.EngineExecutionResults; +import org.junit.platform.testkit.engine.EngineTestKit; +import org.junit.rules.TemporaryFolder; + +public class HermeticInstrumentationTest { + private static final String ENGINE = "engine:junit-jupiter"; + private static final String CLAZZ = "class:com.example.HermeticInstrumentationFuzzTest"; + private static final String FUZZ_TEST_1 = "test-template:fuzzTest1([B)"; + private static final String FUZZ_TEST_2 = "test-template:fuzzTest2([B)"; + private static final String UNIT_TEST_1 = "method:unitTest1()"; + private static final String UNIT_TEST_2 = "method:unitTest2()"; + private static final String INVOCATION = "test-template-invocation:#1"; + @Rule public TemporaryFolder temp = new TemporaryFolder(); + Path baseDir; + + @Before + public void setup() { + baseDir = temp.getRoot().toPath(); + } + + private EngineExecutionResults executeTests() { + return EngineTestKit.engine("junit-jupiter") + .selectors(selectClass("com.example.HermeticInstrumentationFuzzTest")) + .configurationParameter( + "jazzer.instrument", "com.other.package.**,com.example.**,com.yet.another.package.*") + .configurationParameter("jazzer.internal.basedir", baseDir.toAbsolutePath().toString()) + .configurationParameter("junit.jupiter.execution.parallel.enabled", "true") + .execute(); + } + + @Test + public void fuzzingDisabled() { + assumeTrue(System.getenv("JAZZER_FUZZ") == null); + + EngineExecutionResults results = executeTests(); + + results.containerEvents().assertEventsMatchLoosely(event(type(STARTED), container(ENGINE)), + event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ))), + event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ, FUZZ_TEST_1))), + event(type(REPORTING_ENTRY_PUBLISHED), + container(uniqueIdSubstrings(ENGINE, CLAZZ, FUZZ_TEST_1))), + event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ, FUZZ_TEST_1))), + event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ, FUZZ_TEST_2))), + event(type(REPORTING_ENTRY_PUBLISHED), + container(uniqueIdSubstrings(ENGINE, CLAZZ, FUZZ_TEST_2))), + event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ, FUZZ_TEST_2))), + event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ)), finishedSuccessfully()), + event(type(FINISHED), container(ENGINE), finishedSuccessfully())); + + results.testEvents().assertEventsMatchLoosely( + event(type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, FUZZ_TEST_1))), + event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, FUZZ_TEST_1, INVOCATION)), + displayName("<empty input>")), + event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, FUZZ_TEST_1, INVOCATION)), + displayName("<empty input>"), + finishedWithFailure(instanceOf(FuzzerSecurityIssueLow.class))), + event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, UNIT_TEST_1))), + event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, UNIT_TEST_1)), + finishedWithFailure(instanceOf(PatternSyntaxException.class))), + event(type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, FUZZ_TEST_2))), + event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, FUZZ_TEST_2, INVOCATION)), + displayName("<empty input>")), + event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, FUZZ_TEST_2, INVOCATION)), + displayName("<empty input>"), + finishedWithFailure(instanceOf(FuzzerSecurityIssueLow.class))), + event(type(STARTED), test(uniqueIdSubstrings(ENGINE, CLAZZ, UNIT_TEST_2))), + event(type(FINISHED), test(uniqueIdSubstrings(ENGINE, CLAZZ, UNIT_TEST_2)), + finishedWithFailure(instanceOf(PatternSyntaxException.class)))); + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/junit/LifecycleTest.java b/src/test/java/com/code_intelligence/jazzer/junit/LifecycleTest.java new file mode 100644 index 00000000..29dfc664 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/junit/LifecycleTest.java @@ -0,0 +1,132 @@ +// Copyright 2022 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.junit; + +import static org.junit.Assume.assumeFalse; +import static org.junit.Assume.assumeTrue; +import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass; +import static org.junit.platform.testkit.engine.EventConditions.container; +import static org.junit.platform.testkit.engine.EventConditions.displayName; +import static org.junit.platform.testkit.engine.EventConditions.event; +import static org.junit.platform.testkit.engine.EventConditions.finishedSuccessfully; +import static org.junit.platform.testkit.engine.EventConditions.finishedWithFailure; +import static org.junit.platform.testkit.engine.EventConditions.test; +import static org.junit.platform.testkit.engine.EventConditions.type; +import static org.junit.platform.testkit.engine.EventConditions.uniqueIdSubstrings; +import static org.junit.platform.testkit.engine.EventType.DYNAMIC_TEST_REGISTERED; +import static org.junit.platform.testkit.engine.EventType.FINISHED; +import static org.junit.platform.testkit.engine.EventType.REPORTING_ENTRY_PUBLISHED; +import static org.junit.platform.testkit.engine.EventType.SKIPPED; +import static org.junit.platform.testkit.engine.EventType.STARTED; +import static org.junit.platform.testkit.engine.TestExecutionResultConditions.instanceOf; + +import java.io.IOException; +import java.nio.file.Path; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.platform.testkit.engine.EngineExecutionResults; +import org.junit.platform.testkit.engine.EngineTestKit; +import org.junit.rules.TemporaryFolder; + +public class LifecycleTest { + private static final String ENGINE = "engine:junit-jupiter"; + private static final String CLAZZ = "class:com.example.LifecycleFuzzTest"; + private static final String DISABLED_FUZZ = "test-template:disabledFuzz([B)"; + private static final String LIFECYCLE_FUZZ = "test-template:lifecycleFuzz([B)"; + private static final String INVOCATION = "test-template-invocation:#"; + @Rule public TemporaryFolder temp = new TemporaryFolder(); + Path baseDir; + + @Before + public void setup() { + baseDir = temp.getRoot().toPath(); + } + + private EngineExecutionResults executeTests() { + return EngineTestKit.engine("junit-jupiter") + .selectors(selectClass("com.example.LifecycleFuzzTest")) + .configurationParameter( + "jazzer.instrument", "com.other.package.**,com.example.**,com.yet.another.package.*") + .configurationParameter("jazzer.internal.basedir", baseDir.toAbsolutePath().toString()) + .execute(); + } + + @Test + public void fuzzingEnabled() { + assumeFalse(System.getenv("JAZZER_FUZZ").isEmpty()); + + EngineExecutionResults results = executeTests(); + + results.containerEvents().assertEventsMatchExactly(event(type(STARTED), container(ENGINE)), + event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ))), + event(type(SKIPPED), container(uniqueIdSubstrings(ENGINE, CLAZZ, DISABLED_FUZZ))), + event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ))), + // Warning because the seed corpus directory hasn't been found. + event(type(REPORTING_ENTRY_PUBLISHED), + container(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ))), + event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ)), + finishedSuccessfully()), + event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ)), + finishedWithFailure(instanceOf(IOException.class))), + event(type(FINISHED), container(ENGINE), finishedSuccessfully())); + + results.testEvents().assertEventsMatchExactly( + event( + type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ))), + event(type(STARTED), + test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 1)), + displayName("<empty input>")), + event(type(FINISHED), + test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 1)), + displayName("<empty input>"), finishedSuccessfully()), + event( + type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ))), + event(type(STARTED), + test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 2)), + displayName("Fuzzing...")), + event(type(FINISHED), + test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 2)), + displayName("Fuzzing..."), finishedSuccessfully())); + } + + @Test + public void fuzzingDisabled() { + assumeTrue(System.getenv("JAZZER_FUZZ").isEmpty()); + + EngineExecutionResults results = executeTests(); + + results.containerEvents().assertEventsMatchExactly(event(type(STARTED), container(ENGINE)), + event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ))), + event(type(SKIPPED), container(uniqueIdSubstrings(ENGINE, CLAZZ, DISABLED_FUZZ))), + event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ))), + event(type(REPORTING_ENTRY_PUBLISHED), + container(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ))), + event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ))), + event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ)), + finishedWithFailure(instanceOf(IOException.class))), + event(type(FINISHED), container(ENGINE), finishedSuccessfully())); + + results.testEvents().assertEventsMatchExactly( + event( + type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ))), + event(type(STARTED), + test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 1)), + displayName("<empty input>")), + event(type(FINISHED), + test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 1)), + displayName("<empty input>"), finishedSuccessfully())); + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/junit/MutatorTest.java b/src/test/java/com/code_intelligence/jazzer/junit/MutatorTest.java new file mode 100644 index 00000000..3fcc163b --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/junit/MutatorTest.java @@ -0,0 +1,165 @@ +// Copyright 2022 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.junit; + +import static org.junit.Assume.assumeFalse; +import static org.junit.Assume.assumeTrue; +import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass; +import static org.junit.platform.testkit.engine.EventConditions.container; +import static org.junit.platform.testkit.engine.EventConditions.displayName; +import static org.junit.platform.testkit.engine.EventConditions.event; +import static org.junit.platform.testkit.engine.EventConditions.finishedSuccessfully; +import static org.junit.platform.testkit.engine.EventConditions.finishedWithFailure; +import static org.junit.platform.testkit.engine.EventConditions.reportEntry; +import static org.junit.platform.testkit.engine.EventConditions.test; +import static org.junit.platform.testkit.engine.EventConditions.type; +import static org.junit.platform.testkit.engine.EventConditions.uniqueIdSubstrings; +import static org.junit.platform.testkit.engine.EventType.DYNAMIC_TEST_REGISTERED; +import static org.junit.platform.testkit.engine.EventType.FINISHED; +import static org.junit.platform.testkit.engine.EventType.REPORTING_ENTRY_PUBLISHED; +import static org.junit.platform.testkit.engine.EventType.STARTED; +import static org.junit.platform.testkit.engine.TestExecutionResultConditions.instanceOf; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import org.assertj.core.api.Condition; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.platform.engine.reporting.ReportEntry; +import org.junit.platform.testkit.engine.EngineExecutionResults; +import org.junit.platform.testkit.engine.EngineTestKit; +import org.junit.platform.testkit.engine.Event; +import org.junit.rules.TemporaryFolder; + +public class MutatorTest { + private static final String ENGINE = "engine:junit-jupiter"; + private static final String CLASS_NAME = "com.example.MutatorFuzzTest"; + private static final String CLAZZ = "class:" + CLASS_NAME; + private static final String LIFECYCLE_FUZZ = "test-template:mutatorFuzz(java.util.List)"; + private static final String INVOCATION = "test-template-invocation:#"; + private static final String INVALID_SIGNATURE_ENTRY = + "Some files in the seed corpus do not match the fuzz target signature.\n" + + "This indicates that they were generated with a different signature and may cause issues reproducing previous findings."; + + @Rule public TemporaryFolder temp = new TemporaryFolder(); + private Path baseDir; + + @Before + public void setup() throws IOException { + baseDir = temp.getRoot().toPath(); + Path inputsDirectory = baseDir.resolve(Paths.get( + "src", "test", "resources", "com", "example", "MutatorFuzzTestInputs", "mutatorFuzz")); + Files.createDirectories(inputsDirectory); + Files.write(inputsDirectory.resolve("invalid"), "invalid input".getBytes()); + } + + private EngineExecutionResults executeTests() { + System.setProperty("jazzer.experimental_mutator", "true"); + return EngineTestKit.engine("junit-jupiter") + .selectors(selectClass(CLASS_NAME)) + .configurationParameter("jazzer.instrument", "com.example.**") + .configurationParameter("jazzer.internal.basedir", baseDir.toAbsolutePath().toString()) + .execute(); + } + + @Test + public void fuzzingEnabled() { + assumeFalse(System.getenv("JAZZER_FUZZ").isEmpty()); + + EngineExecutionResults results = executeTests(); + + results.containerEvents().assertEventsMatchExactly(event(type(STARTED), container(ENGINE)), + event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ))), + event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ))), + // Invalid corpus input warning + event(type(REPORTING_ENTRY_PUBLISHED), + container(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ)), + new Condition<>( + Event.byPayload(ReportEntry.class, + (it) -> it.getKeyValuePairs().values().contains(INVALID_SIGNATURE_ENTRY)), + "has invalid signature entry reporting entry")), + event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ)), + finishedSuccessfully()), + event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ)), finishedSuccessfully()), + event(type(FINISHED), container(ENGINE), finishedSuccessfully())); + + results.testEvents().assertEventsMatchExactly( + event(type(DYNAMIC_TEST_REGISTERED), + test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 1))), + event(type(STARTED), + test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 1)), + displayName("<empty input>")), + event(type(FINISHED), + test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 1)), + displayName("<empty input>"), finishedSuccessfully()), + event(type(DYNAMIC_TEST_REGISTERED), + test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 2))), + event(type(STARTED), + test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 2)), + displayName("invalid")), + event(type(FINISHED), + test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 2)), + displayName("invalid"), finishedSuccessfully()), + event( + type(DYNAMIC_TEST_REGISTERED), test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ))), + event(type(STARTED), + test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 3)), + displayName("Fuzzing...")), + event(type(FINISHED), + test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 3)), + displayName("Fuzzing..."), finishedWithFailure(instanceOf(AssertionError.class)))); + } + + @Test + public void fuzzingDisabled() { + assumeTrue(System.getenv("JAZZER_FUZZ").isEmpty()); + + EngineExecutionResults results = executeTests(); + + results.containerEvents().assertEventsMatchExactly(event(type(STARTED), container(ENGINE)), + event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ))), + event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ))), + // Deactivated fuzzing warning + event(type(REPORTING_ENTRY_PUBLISHED), + container(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ))), + // Invalid corpus input warning + event(type(REPORTING_ENTRY_PUBLISHED), + container(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ))), + event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ))), + event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ)), finishedSuccessfully()), + event(type(FINISHED), container(ENGINE), finishedSuccessfully())); + + results.testEvents().assertEventsMatchExactly( + event(type(DYNAMIC_TEST_REGISTERED), + test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 1))), + event(type(STARTED), + test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 1)), + displayName("<empty input>")), + event(type(FINISHED), + test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 1)), + displayName("<empty input>"), finishedSuccessfully()), + event(type(DYNAMIC_TEST_REGISTERED), + test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 2))), + event(type(STARTED), + test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 2)), + displayName("invalid")), + event(type(FINISHED), + test(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ, INVOCATION + 2)), + displayName("invalid"), finishedSuccessfully())); + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/junit/RegressionTestTest.java b/src/test/java/com/code_intelligence/jazzer/junit/RegressionTestTest.java new file mode 100644 index 00000000..008b8a4d --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/junit/RegressionTestTest.java @@ -0,0 +1,254 @@ +// Copyright 2022 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.junit; + +import static com.google.common.truth.Truth8.assertThat; +import static org.junit.Assume.assumeTrue; +import static org.junit.platform.engine.discovery.DiscoverySelectors.selectPackage; +import static org.junit.platform.testkit.engine.EventConditions.container; +import static org.junit.platform.testkit.engine.EventConditions.displayName; +import static org.junit.platform.testkit.engine.EventConditions.event; +import static org.junit.platform.testkit.engine.EventConditions.finishedSuccessfully; +import static org.junit.platform.testkit.engine.EventConditions.finishedWithFailure; +import static org.junit.platform.testkit.engine.EventConditions.test; +import static org.junit.platform.testkit.engine.EventConditions.type; +import static org.junit.platform.testkit.engine.EventConditions.uniqueIdSubstrings; +import static org.junit.platform.testkit.engine.EventType.DYNAMIC_TEST_REGISTERED; +import static org.junit.platform.testkit.engine.EventType.FINISHED; +import static org.junit.platform.testkit.engine.EventType.REPORTING_ENTRY_PUBLISHED; +import static org.junit.platform.testkit.engine.EventType.STARTED; +import static org.junit.platform.testkit.engine.TestExecutionResultConditions.instanceOf; +import static org.junit.platform.testkit.engine.TestExecutionResultConditions.message; + +import com.code_intelligence.jazzer.api.FuzzerSecurityIssueCritical; +import com.code_intelligence.jazzer.api.FuzzerSecurityIssueHigh; +import com.code_intelligence.jazzer.api.FuzzerSecurityIssueLow; +import com.code_intelligence.jazzer.api.FuzzerSecurityIssueMedium; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import org.junit.Test; +import org.junit.platform.testkit.engine.EngineExecutionResults; +import org.junit.platform.testkit.engine.EngineTestKit; +import org.opentest4j.AssertionFailedError; + +public class RegressionTestTest { + private static final String ENGINE = "engine:junit-jupiter"; + private static final String BYTE_FUZZ_TEST = "class:com.example.ByteFuzzTest"; + private static final String VALID_FUZZ_TESTS = "class:com.example.ValidFuzzTests"; + private static final String INVALID_FUZZ_TESTS = "class:com.example.InvalidFuzzTests"; + private static final String AUTOFUZZ_WITH_CORPUS_FUZZ_TEST = + "class:com.example.AutofuzzWithCorpusFuzzTest"; + private static final String BYTE_FUZZ = "test-template:byteFuzz([B)"; + private static final String NO_CRASH_FUZZ = "test-template:noCrashFuzz([B)"; + private static final String DATA_FUZZ = + "test-template:dataFuzz(com.code_intelligence.jazzer.api.FuzzedDataProvider)"; + private static final String INVALID_PARAMETER_COUNT_FUZZ = + "test-template:invalidParameterCountFuzz()"; + private static final String AUTOFUZZ_WITH_CORPUS = + "test-template:autofuzzWithCorpus(java.lang.String, int)"; + private static final String INVOCATION = "test-template-invocation:#"; + + private static EngineExecutionResults executeTests() { + return EngineTestKit.engine("junit-jupiter") + .selectors(selectPackage("com.example")) + .configurationParameter( + "jazzer.instrument", "com.other.package.**,com.example.**,com.yet.another.package.*") + .execute(); + } + + @Test + public void regressionTestEnabled() { + assumeTrue(System.getenv("JAZZER_FUZZ") == null); + + // Record Jazzer's stderr. + PrintStream stderr = System.err; + ByteArrayOutputStream recordedStderr = new ByteArrayOutputStream(); + System.setErr(new PrintStream(recordedStderr)); + + EngineExecutionResults results = executeTests(); + System.setErr(stderr); + + // Verify that Jazzer doesn't print any warning or errors. + String[] stderrLines = + new String(recordedStderr.toByteArray(), StandardCharsets.UTF_8).split("\n"); + for (String line : stderrLines) { + System.err.println(line); + } + assertThat(Arrays.stream(stderrLines) + .filter(line -> line.startsWith("WARN:") || line.startsWith("ERROR:"))) + .isEmpty(); + + results.containerEvents().debug().assertEventsMatchLoosely( + event(type(STARTED), container(ENGINE)), + event( + type(STARTED), container(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, NO_CRASH_FUZZ))), + event(type(REPORTING_ENTRY_PUBLISHED), + container(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, NO_CRASH_FUZZ))), + event(type(FINISHED), + container(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, NO_CRASH_FUZZ)), + finishedSuccessfully()), + event(type(STARTED), container(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, DATA_FUZZ))), + event(type(REPORTING_ENTRY_PUBLISHED), + container(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, DATA_FUZZ))), + event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, DATA_FUZZ)), + finishedSuccessfully()), + event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS)), + finishedSuccessfully()), + event(type(STARTED), container(uniqueIdSubstrings(ENGINE, AUTOFUZZ_WITH_CORPUS_FUZZ_TEST))), + event(type(STARTED), + container( + uniqueIdSubstrings(ENGINE, AUTOFUZZ_WITH_CORPUS_FUZZ_TEST, AUTOFUZZ_WITH_CORPUS))), + event(type(REPORTING_ENTRY_PUBLISHED), + container( + uniqueIdSubstrings(ENGINE, AUTOFUZZ_WITH_CORPUS_FUZZ_TEST, AUTOFUZZ_WITH_CORPUS))), + event(type(FINISHED), + container( + uniqueIdSubstrings(ENGINE, AUTOFUZZ_WITH_CORPUS_FUZZ_TEST, AUTOFUZZ_WITH_CORPUS)), + finishedSuccessfully()), + event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, AUTOFUZZ_WITH_CORPUS_FUZZ_TEST)), + finishedSuccessfully()), + event(type(STARTED), container(uniqueIdSubstrings(ENGINE, BYTE_FUZZ_TEST))), + event(type(STARTED), container(uniqueIdSubstrings(ENGINE, BYTE_FUZZ_TEST, BYTE_FUZZ))), + event(type(REPORTING_ENTRY_PUBLISHED), + container(uniqueIdSubstrings(ENGINE, BYTE_FUZZ_TEST, BYTE_FUZZ))), + event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, BYTE_FUZZ_TEST, BYTE_FUZZ)), + finishedSuccessfully()), + event(type(STARTED), container(uniqueIdSubstrings(ENGINE, INVALID_FUZZ_TESTS))), + event(type(STARTED), + container( + uniqueIdSubstrings(ENGINE, INVALID_FUZZ_TESTS, INVALID_PARAMETER_COUNT_FUZZ))), + event(type(FINISHED), + container(uniqueIdSubstrings(ENGINE, INVALID_FUZZ_TESTS, INVALID_PARAMETER_COUNT_FUZZ)), + finishedWithFailure(instanceOf(IllegalArgumentException.class), + message("Methods annotated with @FuzzTest must take at least one parameter"))), + event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, INVALID_FUZZ_TESTS)), + finishedSuccessfully()), + event(type(FINISHED), container(ENGINE), finishedSuccessfully())); + + results.testEvents().debug().assertEventsMatchLoosely( + event(type(DYNAMIC_TEST_REGISTERED), + test(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, DATA_FUZZ, INVOCATION)), + displayName("<empty input>")), + event(type(STARTED), + test(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, DATA_FUZZ, INVOCATION)), + displayName("<empty input>")), + event(type(FINISHED), + test(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, DATA_FUZZ, INVOCATION)), + displayName("<empty input>"), + finishedWithFailure(instanceOf(FuzzerSecurityIssueMedium.class))), + event(type(DYNAMIC_TEST_REGISTERED), + test(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, DATA_FUZZ, INVOCATION)), + displayName("no_crash")), + event(type(STARTED), + test(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, DATA_FUZZ, INVOCATION)), + displayName("no_crash")), + event(type(FINISHED), + test(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, DATA_FUZZ, INVOCATION)), + displayName("no_crash"), finishedSuccessfully()), + event(type(DYNAMIC_TEST_REGISTERED), + test(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, DATA_FUZZ, INVOCATION)), + displayName("assert")), + event(type(STARTED), + test(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, DATA_FUZZ, INVOCATION)), + displayName("assert")), + event(type(FINISHED), + test(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, DATA_FUZZ, INVOCATION)), + displayName("assert"), finishedWithFailure(instanceOf(AssertionFailedError.class))), + event(type(DYNAMIC_TEST_REGISTERED), + test(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, DATA_FUZZ, INVOCATION)), + displayName("honeypot")), + event(type(STARTED), + test(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, DATA_FUZZ, INVOCATION)), + displayName("honeypot")), + event(type(FINISHED), + test(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, DATA_FUZZ, INVOCATION)), + displayName("honeypot"), + finishedWithFailure(instanceOf(FuzzerSecurityIssueHigh.class))), + event(type(DYNAMIC_TEST_REGISTERED), + test(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, DATA_FUZZ, INVOCATION)), + displayName("sanitizer_internal_class")), + event(type(STARTED), + test(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, DATA_FUZZ, INVOCATION)), + displayName("sanitizer_internal_class")), + event(type(FINISHED), + test(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, DATA_FUZZ, INVOCATION)), + displayName("sanitizer_internal_class"), + finishedWithFailure(instanceOf(FuzzerSecurityIssueCritical.class))), + event(type(DYNAMIC_TEST_REGISTERED), + test(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, DATA_FUZZ, INVOCATION)), + displayName("sanitizer_user_class")), + event(type(STARTED), + test(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, DATA_FUZZ, INVOCATION)), + displayName("sanitizer_user_class")), + event(type(FINISHED), + test(uniqueIdSubstrings(ENGINE, VALID_FUZZ_TESTS, DATA_FUZZ, INVOCATION)), + displayName("sanitizer_user_class"), + finishedWithFailure(instanceOf(FuzzerSecurityIssueLow.class))), + event(type(DYNAMIC_TEST_REGISTERED), + test(uniqueIdSubstrings(ENGINE, BYTE_FUZZ_TEST, BYTE_FUZZ, INVOCATION)), + displayName("<empty input>")), + event(type(STARTED), + test(uniqueIdSubstrings(ENGINE, BYTE_FUZZ_TEST, BYTE_FUZZ, INVOCATION)), + displayName("<empty input>")), + event(type(FINISHED), + test(uniqueIdSubstrings(ENGINE, BYTE_FUZZ_TEST, BYTE_FUZZ, INVOCATION)), + displayName("<empty input>"), finishedSuccessfully()), + event(type(DYNAMIC_TEST_REGISTERED), + test(uniqueIdSubstrings(ENGINE, BYTE_FUZZ_TEST, BYTE_FUZZ, INVOCATION)), + displayName("succeeds")), + event(type(STARTED), + test(uniqueIdSubstrings(ENGINE, BYTE_FUZZ_TEST, BYTE_FUZZ, INVOCATION)), + displayName("succeeds")), + event(type(FINISHED), + test(uniqueIdSubstrings(ENGINE, BYTE_FUZZ_TEST, BYTE_FUZZ, INVOCATION)), + displayName("succeeds"), finishedSuccessfully()), + event(type(DYNAMIC_TEST_REGISTERED), + test(uniqueIdSubstrings(ENGINE, BYTE_FUZZ_TEST, BYTE_FUZZ, INVOCATION)), + displayName("fails")), + event(type(STARTED), + test(uniqueIdSubstrings(ENGINE, BYTE_FUZZ_TEST, BYTE_FUZZ, INVOCATION)), + displayName("fails")), + event(type(FINISHED), + test(uniqueIdSubstrings(ENGINE, BYTE_FUZZ_TEST, BYTE_FUZZ, INVOCATION)), + displayName("fails"), finishedWithFailure(instanceOf(AssertionFailedError.class))), + event(type(DYNAMIC_TEST_REGISTERED), + test(uniqueIdSubstrings( + ENGINE, AUTOFUZZ_WITH_CORPUS_FUZZ_TEST, AUTOFUZZ_WITH_CORPUS, INVOCATION)), + displayName("<empty input>")), + event(type(STARTED), + test(uniqueIdSubstrings( + ENGINE, AUTOFUZZ_WITH_CORPUS_FUZZ_TEST, AUTOFUZZ_WITH_CORPUS, INVOCATION)), + displayName("<empty input>")), + event(type(FINISHED), + test(uniqueIdSubstrings( + ENGINE, AUTOFUZZ_WITH_CORPUS_FUZZ_TEST, AUTOFUZZ_WITH_CORPUS, INVOCATION)), + displayName("<empty input>"), finishedSuccessfully()), + event(type(DYNAMIC_TEST_REGISTERED), + test(uniqueIdSubstrings( + ENGINE, AUTOFUZZ_WITH_CORPUS_FUZZ_TEST, AUTOFUZZ_WITH_CORPUS, INVOCATION)), + displayName("crashing_input")), + event(type(STARTED), + test(uniqueIdSubstrings( + ENGINE, AUTOFUZZ_WITH_CORPUS_FUZZ_TEST, AUTOFUZZ_WITH_CORPUS, INVOCATION)), + displayName("crashing_input")), + event(type(FINISHED), + test(uniqueIdSubstrings( + ENGINE, AUTOFUZZ_WITH_CORPUS_FUZZ_TEST, AUTOFUZZ_WITH_CORPUS, INVOCATION)), + displayName("crashing_input"), + finishedWithFailure(instanceOf(RuntimeException.class)))); + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/junit/TestMethod.java b/src/test/java/com/code_intelligence/jazzer/junit/TestMethod.java new file mode 100644 index 00000000..bb542ccf --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/junit/TestMethod.java @@ -0,0 +1,52 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.code_intelligence.jazzer.junit; + +import static org.junit.platform.engine.discovery.DiscoverySelectors.selectMethod; + +import java.lang.reflect.Method; + +/** + * Small class that allows us to capture the methods that we're using as test data. We need similar + * but slightly different data at various points: + * 1. the method name with parameters for finding the method initially and for referring to it in + * JUnit + * 2. the method name without parameters for the findings directories + */ +public class TestMethod { + Method method; + String nameWithParams; + + TestMethod(String className, String methodName) { + nameWithParams = methodName; + method = selectMethod(className + "#" + methodName).getJavaMethod(); + } + + /** + * Returns the {@link org.junit.platform.engine.TestDescriptor} ID for this method + */ + String getDescriptorId() { + return "test-template:" + nameWithParams; + } + + /** + * Returns just the name of the method without parameters + */ + String getName() { + return method.getName(); + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/junit/UtilsTest.java b/src/test/java/com/code_intelligence/jazzer/junit/UtilsTest.java new file mode 100644 index 00000000..da4a7345 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/junit/UtilsTest.java @@ -0,0 +1,151 @@ +// Copyright 2022 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.junit; + +import static com.code_intelligence.jazzer.junit.Utils.durationStringToSeconds; +import static com.code_intelligence.jazzer.junit.Utils.getMarkedArguments; +import static com.code_intelligence.jazzer.junit.Utils.getMarkedInstance; +import static com.code_intelligence.jazzer.junit.Utils.isMarkedInstance; +import static com.code_intelligence.jazzer.junit.Utils.isMarkedInvocation; +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth8.assertThat; +import static java.nio.file.Files.createDirectories; +import static java.nio.file.Files.createFile; +import static java.util.Arrays.stream; +import static java.util.Collections.singletonList; +import static java.util.stream.Collectors.joining; +import static org.junit.jupiter.params.provider.Arguments.arguments; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Method; +import java.nio.file.Path; +import java.util.AbstractList; +import java.util.AbstractMap; +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.InvocationInterceptor; +import org.junit.jupiter.api.extension.ReflectiveInvocationContext; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; + +public class UtilsTest implements InvocationInterceptor { + @TempDir Path temp; + + @Test + void testDurationStringToSeconds() { + assertThat(durationStringToSeconds("1m")).isEqualTo(60); + assertThat(durationStringToSeconds("1min")).isEqualTo(60); + assertThat(durationStringToSeconds("1h")).isEqualTo(60 * 60); + assertThat(durationStringToSeconds("1h 2m 30s")).isEqualTo(60 * 60 + 2 * 60 + 30); + assertThat(durationStringToSeconds("1hr2min30sec")).isEqualTo(60 * 60 + 2 * 60 + 30); + assertThat(durationStringToSeconds("1h2m30s")).isEqualTo(60 * 60 + 2 * 60 + 30); + } + + @ValueSource(classes = {int.class, Class.class, Object.class, String.class, HashMap.class, + Map.class, int[].class, int[][].class, AbstractMap.class, AbstractList.class}) + @ParameterizedTest + void + testMarkedInstances(Class<?> clazz) { + Object instance = getMarkedInstance(clazz); + if (clazz == int.class) { + assertThat(instance).isInstanceOf(Integer.class); + } else { + assertThat(instance).isInstanceOf(clazz); + } + assertThat(isMarkedInstance(instance)).isTrue(); + assertThat(getMarkedInstance(clazz)).isSameInstanceAs(instance); + } + + static Stream<Arguments> testWithMarkedNamedParametersSource() { + Method testMethod = + stream(UtilsTest.class.getDeclaredMethods()) + .filter(method -> method.getName().equals("testWithMarkedNamedParameters")) + .findFirst() + .get(); + return Stream.of( + arguments("foo", 0, new HashMap<>(), singletonList(5), UtilsTest.class, new int[] {1}), + getMarkedArguments(testMethod, "some name"), + arguments("baz", 1, new LinkedHashMap<>(), Arrays.asList(5, 7), String.class, new int[0]), + getMarkedArguments(testMethod, "some other name")); + } + + @MethodSource("testWithMarkedNamedParametersSource") + @ExtendWith(UtilsTest.class) + @ParameterizedTest + void testWithMarkedNamedParameters(String str, int num, AbstractMap<String, String> map, + List<Integer> list, Class<?> clazz, int[] array) {} + + boolean argumentsExpectedToBeMarked = false; + + @Override + public void interceptTestTemplateMethod(Invocation<Void> invocation, + ReflectiveInvocationContext<Method> invocationContext, ExtensionContext extensionContext) + throws Throwable { + assertThat(isMarkedInvocation(invocationContext)).isEqualTo(argumentsExpectedToBeMarked); + argumentsExpectedToBeMarked = !argumentsExpectedToBeMarked; + invocation.proceed(); + } + + @Test + public void testGetClassPathBasedInstrumentationFilter() throws IOException { + Path firstDir = createDirectories(temp.resolve("first_dir")); + Path orgExample = createDirectories(firstDir.resolve("org").resolve("example")); + createFile(orgExample.resolve("Application.class")); + + Path nonExistentDir = temp.resolve("does not exist"); + + Path secondDir = createDirectories(temp.resolve("second").resolve("dir")); + createFile(secondDir.resolve("Root.class")); + Path comExampleProject = + createDirectories(secondDir.resolve("com").resolve("example").resolve("project")); + createFile(comExampleProject.resolve("Main.class")); + Path comExampleOtherProject = + createDirectories(secondDir.resolve("com").resolve("example").resolve("other_project")); + createFile(comExampleOtherProject.resolve("Lib.class")); + + Path emptyDir = createDirectories(temp.resolve("some").resolve("empty").resolve("dir")); + + Path firstJar = createFile(temp.resolve("first.jar")); + Path secondJar = createFile(temp.resolve("second.jar")); + + assertThat(Utils.getClassPathBasedInstrumentationFilter(makeClassPath( + firstDir, firstJar, nonExistentDir, secondDir, secondJar, emptyDir))) + .hasValue("*,com.example.other_project.**,com.example.project.**,org.example.**"); + } + + @Test + public void testGetClassPathBasedInstrumentationFilter_noDirs() throws IOException { + Path firstJar = createFile(temp.resolve("first.jar")); + Path secondJar = createFile(temp.resolve("second.jar")); + + assertThat(Utils.getClassPathBasedInstrumentationFilter(makeClassPath(firstJar, secondJar))) + .isEmpty(); + } + + private static String makeClassPath(Path... paths) { + return Arrays.stream(paths).map(Path::toString).collect(joining(File.pathSeparator)); + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/junit/ValueProfileTest.java b/src/test/java/com/code_intelligence/jazzer/junit/ValueProfileTest.java new file mode 100644 index 00000000..a1cc21cf --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/junit/ValueProfileTest.java @@ -0,0 +1,204 @@ +// Copyright 2022 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.code_intelligence.jazzer.junit; + +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth8.assertThat; +import static org.junit.Assume.assumeFalse; +import static org.junit.Assume.assumeTrue; +import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass; +import static org.junit.platform.testkit.engine.EventConditions.container; +import static org.junit.platform.testkit.engine.EventConditions.displayName; +import static org.junit.platform.testkit.engine.EventConditions.event; +import static org.junit.platform.testkit.engine.EventConditions.finishedSuccessfully; +import static org.junit.platform.testkit.engine.EventConditions.finishedWithFailure; +import static org.junit.platform.testkit.engine.EventConditions.test; +import static org.junit.platform.testkit.engine.EventConditions.type; +import static org.junit.platform.testkit.engine.EventConditions.uniqueIdSubstrings; +import static org.junit.platform.testkit.engine.EventType.DYNAMIC_TEST_REGISTERED; +import static org.junit.platform.testkit.engine.EventType.FINISHED; +import static org.junit.platform.testkit.engine.EventType.SKIPPED; +import static org.junit.platform.testkit.engine.EventType.STARTED; +import static org.junit.platform.testkit.engine.TestExecutionResultConditions.instanceOf; + +import com.code_intelligence.jazzer.api.FuzzerSecurityIssueMedium; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.stream.Stream; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.platform.testkit.engine.EngineExecutionResults; +import org.junit.platform.testkit.engine.EngineTestKit; +import org.junit.platform.testkit.engine.EventType; +import org.junit.rules.TemporaryFolder; + +public class ValueProfileTest { + private static final boolean VALUE_PROFILE_ENABLED = + "True".equals(System.getenv("JAZZER_VALUE_PROFILE")); + + private static final String ENGINE = "engine:junit-jupiter"; + private static final String CLAZZ = "class:com.example.ValueProfileFuzzTest"; + private static final String VALUE_PROFILE_FUZZ = "test-template:valueProfileFuzz([B)"; + private static final String INVOCATION = "test-template-invocation:#"; + + @Rule public TemporaryFolder temp = new TemporaryFolder(); + Path baseDir; + Path inputsDirectories; + + @Before + public void setup() throws IOException { + baseDir = temp.getRoot().toPath(); + // Create a fake test resource directory structure with an input directory to verify that + // Jazzer uses it and emits a crash file into it. + inputsDirectories = baseDir.resolve(Paths.get("src", "test", "resources", "com", "example", + "ValueProfileFuzzTestInputs", "valueProfileFuzz")); + Files.createDirectories(inputsDirectories); + } + + private EngineExecutionResults executeTests() { + return EngineTestKit.engine("junit-jupiter") + .selectors(selectClass("com.example.ValueProfileFuzzTest")) + .configurationParameter( + "jazzer.instrument", "com.other.package.**,com.example.**,com.yet.another.package.*") + .configurationParameter("jazzer.valueprofile", System.getenv("JAZZER_VALUE_PROFILE")) + .configurationParameter("jazzer.internal.basedir", baseDir.toAbsolutePath().toString()) + .execute(); + } + + @Test + public void valueProfileEnabled() throws IOException { + assumeTrue(VALUE_PROFILE_ENABLED); + + EngineExecutionResults results = executeTests(); + + results.containerEvents().assertEventsMatchExactly(event(type(STARTED), container(ENGINE)), + event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ))), + event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ))), + event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ)), + finishedSuccessfully()), + event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ)), finishedSuccessfully()), + event(type(FINISHED), container(ENGINE), finishedSuccessfully())); + + results.testEvents().assertEventsMatchExactly( + event(type(DYNAMIC_TEST_REGISTERED), + test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ))), + event(type(STARTED), + test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ, INVOCATION + 1)), + displayName("<empty input>")), + event(type(FINISHED), + test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ, INVOCATION + 1)), + displayName("<empty input>"), finishedSuccessfully()), + event(type(DYNAMIC_TEST_REGISTERED), + test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ))), + event(type(STARTED), + test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ, INVOCATION + 2)), + displayName("empty_seed")), + event(type(FINISHED), + test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ, INVOCATION + 2)), + displayName("empty_seed"), finishedSuccessfully()), + event(type(DYNAMIC_TEST_REGISTERED), + test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ))), + event(type(STARTED), + test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ, INVOCATION + 3)), + displayName("Fuzzing...")), + event(type(FINISHED), + test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ, INVOCATION + 3)), + displayName("Fuzzing..."), + finishedWithFailure(instanceOf(FuzzerSecurityIssueMedium.class)))); + + // Should crash on the exact input "Jazzer", with the crash emitted into the seed corpus. + try (Stream<Path> crashFiles = Files.list(baseDir).filter( + path -> path.getFileName().toString().startsWith("crash-"))) { + assertThat(crashFiles).isEmpty(); + } + try (Stream<Path> seeds = Files.list(inputsDirectories)) { + assertThat(seeds).containsExactly( + inputsDirectories.resolve("crash-131db69c7fadc408fe5031079dad3a441df09aff")); + } + assertThat(Files.readAllBytes( + inputsDirectories.resolve("crash-131db69c7fadc408fe5031079dad3a441df09aff"))) + .isEqualTo("Jazzer".getBytes(StandardCharsets.UTF_8)); + + // Verify that the engine created the generated corpus directory and emitted inputs into it. + Path generatedCorpus = + baseDir.resolve(Paths.get(".cifuzz-corpus", "com.example.ValueProfileFuzzTest")); + assertThat(Files.isDirectory(generatedCorpus)).isTrue(); + try (Stream<Path> entries = Files.list(generatedCorpus)) { + assertThat(entries).isNotEmpty(); + } + } + + @Test + public void valueProfileDisabled() throws IOException { + assumeFalse(VALUE_PROFILE_ENABLED); + + EngineExecutionResults results = executeTests(); + + results.containerEvents().assertEventsMatchExactly(event(type(STARTED), container(ENGINE)), + event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ))), + event(type(STARTED), container(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ))), + event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ)), + finishedSuccessfully()), + event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ)), finishedSuccessfully()), + event(type(FINISHED), container(ENGINE), finishedSuccessfully())); + + results.testEvents().assertEventsMatchExactly( + event(type(DYNAMIC_TEST_REGISTERED), + test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ))), + event(type(STARTED), + test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ, INVOCATION + 1)), + displayName("<empty input>")), + event(type(FINISHED), + test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ, INVOCATION + 1)), + displayName("<empty input>"), finishedSuccessfully()), + event(type(DYNAMIC_TEST_REGISTERED), + test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ))), + event(type(STARTED), + test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ, INVOCATION + 2)), + displayName("empty_seed")), + event(type(FINISHED), + test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ, INVOCATION + 2)), + displayName("empty_seed"), finishedSuccessfully()), + event(type(DYNAMIC_TEST_REGISTERED), + test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ))), + event(type(STARTED), + test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ, INVOCATION + 3)), + displayName("Fuzzing...")), + event(type(FINISHED), + test(uniqueIdSubstrings(ENGINE, CLAZZ, VALUE_PROFILE_FUZZ, INVOCATION + 3)), + displayName("Fuzzing..."), finishedSuccessfully())); + + // No crash means no crashing input is emitted anywhere. + try (Stream<Path> crashFiles = Files.list(baseDir).filter( + path -> path.getFileName().toString().startsWith("crash-"))) { + assertThat(crashFiles).isEmpty(); + } + try (Stream<Path> seeds = Files.list(inputsDirectories)) { + assertThat(seeds).isEmpty(); + } + + // Verify that the engine created the generated corpus directory and emitted inputs into it. + Path generatedCorpus = + baseDir.resolve(Paths.get(".cifuzz-corpus", "com.example.ValueProfileFuzzTest")); + assertThat(Files.isDirectory(generatedCorpus)).isTrue(); + try (Stream<Path> entries = Files.list(generatedCorpus)) { + assertThat(entries).isNotEmpty(); + } + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/junit/test_resources_root/com/example/CorpusDirectoryFuzzTestInputs/corpusDirectoryFuzz/seed b/src/test/java/com/code_intelligence/jazzer/junit/test_resources_root/com/example/CorpusDirectoryFuzzTestInputs/corpusDirectoryFuzz/seed new file mode 100644 index 00000000..e31de1f3 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/junit/test_resources_root/com/example/CorpusDirectoryFuzzTestInputs/corpusDirectoryFuzz/seed @@ -0,0 +1 @@ +seed diff --git a/src/test/java/com/code_intelligence/jazzer/junit/test_resources_root/com/example/DirectoryInputsFuzzTestInputs/inputsFuzz/seed b/src/test/java/com/code_intelligence/jazzer/junit/test_resources_root/com/example/DirectoryInputsFuzzTestInputs/inputsFuzz/seed new file mode 100644 index 00000000..6d0450cc --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/junit/test_resources_root/com/example/DirectoryInputsFuzzTestInputs/inputsFuzz/seed @@ -0,0 +1 @@ +directory
\ No newline at end of file diff --git a/src/test/java/com/code_intelligence/jazzer/junit/test_resources_root/com/example/DirectoryInputsFuzzTestInputs/nested_dir/seed b/src/test/java/com/code_intelligence/jazzer/junit/test_resources_root/com/example/DirectoryInputsFuzzTestInputs/nested_dir/seed new file mode 100644 index 00000000..6d0450cc --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/junit/test_resources_root/com/example/DirectoryInputsFuzzTestInputs/nested_dir/seed @@ -0,0 +1 @@ +directory
\ No newline at end of file diff --git a/src/test/java/com/code_intelligence/jazzer/mutation/ArgumentsMutatorTest.java b/src/test/java/com/code_intelligence/jazzer/mutation/ArgumentsMutatorTest.java new file mode 100644 index 00000000..9a5bafd8 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/mutation/ArgumentsMutatorTest.java @@ -0,0 +1,298 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.code_intelligence.jazzer.mutation; + +import static com.code_intelligence.jazzer.mutation.support.TestSupport.mockPseudoRandom; +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth8.assertThat; +import static java.util.Collections.singletonList; + +import com.code_intelligence.jazzer.mutation.annotation.NotNull; +import com.code_intelligence.jazzer.mutation.mutator.Mutators; +import com.code_intelligence.jazzer.mutation.support.TestSupport.MockPseudoRandom; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.lang.reflect.Method; +import java.util.List; +import java.util.Optional; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.ResourceLock; + +@SuppressWarnings("OptionalGetWithoutIsPresent") +class ArgumentsMutatorTest { + private static List<List<Boolean>> fuzzThisFunctionArgument1; + private static List<Boolean> fuzzThisFunctionArgument2; + + public static void fuzzThisFunction(List<List<@NotNull Boolean>> list, List<Boolean> otherList) { + fuzzThisFunctionArgument1 = list; + fuzzThisFunctionArgument2 = otherList; + } + + @Test + @ResourceLock(value = "fuzzThisFunction") + void testStaticMethod() throws Throwable { + Method method = + ArgumentsMutatorTest.class.getMethod("fuzzThisFunction", List.class, List.class); + Optional<ArgumentsMutator> maybeMutator = + ArgumentsMutator.forStaticMethod(Mutators.newFactory(), method); + assertThat(maybeMutator).isPresent(); + ArgumentsMutator mutator = maybeMutator.get(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // outer list not null + false, + // outer list size 1 + 1, + // inner list not null + false, + // inner list size 1 + 1, + // boolean + true, + // outer list not null + false, + // outer list size 1 + 1, + // Boolean not null + false, + // boolean + false)) { + mutator.init(prng); + } + + fuzzThisFunctionArgument1 = null; + fuzzThisFunctionArgument2 = null; + mutator.invoke(true); + assertThat(fuzzThisFunctionArgument1).containsExactly(singletonList(true)); + assertThat(fuzzThisFunctionArgument2).containsExactly(false); + + try (MockPseudoRandom prng = mockPseudoRandom( + // mutate first argument + 0, + // Nullable mutator + false, + // Action mutate in outer list + 2, + // Mutate one element, + 1, + // index to get to inner list + 0, + // Nullable mutator + false, + // Action mutate inner list + 2, + // Mutate one element, + 1, + // index to get boolean value + 0)) { + mutator.mutate(prng); + } + + fuzzThisFunctionArgument1 = null; + fuzzThisFunctionArgument2 = null; + mutator.invoke(true); + assertThat(fuzzThisFunctionArgument1).containsExactly(singletonList(false)); + assertThat(fuzzThisFunctionArgument2).containsExactly(false); + + // Modify the arguments passed to the function. + fuzzThisFunctionArgument1.get(0).clear(); + fuzzThisFunctionArgument2.clear(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // mutate first argument + 0, + // Nullable mutator + false, + // Action mutate in outer list + 2, + // Mutate one element, + 1, + // index to get to inner list + 0, + // Nullable mutator + false, + // Action mutate inner list + 2, + // Mutate one element, + 1, + // index to get boolean value + 0)) { + mutator.mutate(prng); + } + + fuzzThisFunctionArgument1 = null; + fuzzThisFunctionArgument2 = null; + mutator.invoke(false); + assertThat(fuzzThisFunctionArgument1).containsExactly(singletonList(true)); + assertThat(fuzzThisFunctionArgument2).containsExactly(false); + } + + private List<List<Boolean>> mutableFuzzThisFunctionArgument1; + private List<Boolean> mutableFuzzThisFunctionArgument2; + + public void mutableFuzzThisFunction(List<List<@NotNull Boolean>> list, List<Boolean> otherList) { + mutableFuzzThisFunctionArgument1 = list; + mutableFuzzThisFunctionArgument2 = otherList; + } + + @Test + void testInstanceMethod() throws Throwable { + Method method = + ArgumentsMutatorTest.class.getMethod("mutableFuzzThisFunction", List.class, List.class); + Optional<ArgumentsMutator> maybeMutator = + ArgumentsMutator.forInstanceMethod(Mutators.newFactory(), this, method); + assertThat(maybeMutator).isPresent(); + ArgumentsMutator mutator = maybeMutator.get(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // outer list not null + false, + // outer list size 1 + 1, + // inner list not null + false, + // inner list size 1 + 1, + // boolean + true, + // outer list not null + false, + // outer list size 1 + 1, + // Boolean not null + false, + // boolean + false)) { + mutator.init(prng); + } + + mutableFuzzThisFunctionArgument1 = null; + mutableFuzzThisFunctionArgument2 = null; + mutator.invoke(true); + assertThat(mutableFuzzThisFunctionArgument1).containsExactly(singletonList(true)); + assertThat(mutableFuzzThisFunctionArgument2).containsExactly(false); + + try (MockPseudoRandom prng = mockPseudoRandom( + // mutate first argument + 0, + // Nullable mutator + false, + // Action mutate in outer list + 2, + // Mutate one element, + 1, + // index to get to inner list + 0, + // Nullable mutator + false, + // Action mutate inner list + 2, + // Mutate one element, + 1, + // index to get boolean value + 0)) { + mutator.mutate(prng); + } + + mutableFuzzThisFunctionArgument1 = null; + mutableFuzzThisFunctionArgument2 = null; + mutator.invoke(true); + assertThat(mutableFuzzThisFunctionArgument1).containsExactly(singletonList(false)); + assertThat(mutableFuzzThisFunctionArgument2).containsExactly(false); + + // Modify the arguments passed to the function. + mutableFuzzThisFunctionArgument1.get(0).clear(); + mutableFuzzThisFunctionArgument2.clear(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // mutate first argument + 0, + // Nullable mutator + false, + // Action mutate in outer list + 2, + // Mutate one element, + 1, + // index to get to inner list + 0, + // Nullable mutator + false, + // Action mutate inner list + 2, + // Mutate one element, + 1, + // index to get boolean value + 0)) { + mutator.mutate(prng); + } + + mutableFuzzThisFunctionArgument1 = null; + mutableFuzzThisFunctionArgument2 = null; + mutator.invoke(false); + assertThat(mutableFuzzThisFunctionArgument1).containsExactly(singletonList(true)); + assertThat(mutableFuzzThisFunctionArgument2).containsExactly(false); + } + + @SuppressWarnings("unused") + public void crossOverFunction(List<Boolean> list) {} + + @Test + @SuppressWarnings("unchecked") + void testCrossOver() throws Throwable { + Method method = ArgumentsMutatorTest.class.getMethod("crossOverFunction", List.class); + Optional<ArgumentsMutator> maybeMutator = + ArgumentsMutator.forInstanceMethod(Mutators.newFactory(), this, method); + assertThat(maybeMutator).isPresent(); + ArgumentsMutator mutator = maybeMutator.get(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // list not null + false, + // list size 1 + 1, + // not null, + false, + // boolean + true)) { + mutator.init(prng); + } + ByteArrayOutputStream baos1 = new ByteArrayOutputStream(); + mutator.write(baos1); + byte[] out1 = baos1.toByteArray(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // list not null + false, + // list size 1 + 1, + // not null + false, + // boolean + false)) { + mutator.init(prng); + } + ByteArrayOutputStream baos2 = new ByteArrayOutputStream(); + mutator.write(baos2); + byte[] out2 = baos1.toByteArray(); + + mutator.crossOver(new ByteArrayInputStream(out1), new ByteArrayInputStream(out2), 12345); + Object[] arguments = mutator.getArguments(); + + assertThat(arguments).isNotEmpty(); + assertThat((List<Boolean>) arguments[0]).isNotEmpty(); + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/mutation/BUILD.bazel b/src/test/java/com/code_intelligence/jazzer/mutation/BUILD.bazel new file mode 100644 index 00000000..9d397570 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/mutation/BUILD.bazel @@ -0,0 +1,15 @@ +load("@contrib_rules_jvm//java:defs.bzl", "java_test_suite") + +java_test_suite( + name = "MutationTests", + size = "small", + srcs = glob(["*Test.java"]), + runner = "junit5", + deps = [ + "//src/main/java/com/code_intelligence/jazzer/mutation", + "//src/main/java/com/code_intelligence/jazzer/mutation/annotation", + "//src/main/java/com/code_intelligence/jazzer/mutation/api", + "//src/main/java/com/code_intelligence/jazzer/mutation/mutator", + "//src/test/java/com/code_intelligence/jazzer/mutation/support:test_support", + ], +) diff --git a/src/test/java/com/code_intelligence/jazzer/mutation/combinator/BUILD.bazel b/src/test/java/com/code_intelligence/jazzer/mutation/combinator/BUILD.bazel new file mode 100644 index 00000000..033c03b6 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/mutation/combinator/BUILD.bazel @@ -0,0 +1,14 @@ +load("@contrib_rules_jvm//java:defs.bzl", "java_test_suite") + +java_test_suite( + name = "CompositeTests", + size = "small", + srcs = glob(["*.java"]), + runner = "junit5", + deps = [ + "//src/main/java/com/code_intelligence/jazzer/mutation/api", + "//src/main/java/com/code_intelligence/jazzer/mutation/combinator", + "//src/main/java/com/code_intelligence/jazzer/mutation/support", + "//src/test/java/com/code_intelligence/jazzer/mutation/support:test_support", + ], +) diff --git a/src/test/java/com/code_intelligence/jazzer/mutation/combinator/MutatorCombinatorsTest.java b/src/test/java/com/code_intelligence/jazzer/mutation/combinator/MutatorCombinatorsTest.java new file mode 100644 index 00000000..d0d06f22 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/mutation/combinator/MutatorCombinatorsTest.java @@ -0,0 +1,526 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.code_intelligence.jazzer.mutation.combinator; + +import static com.code_intelligence.jazzer.mutation.combinator.MutatorCombinators.assemble; +import static com.code_intelligence.jazzer.mutation.combinator.MutatorCombinators.combine; +import static com.code_intelligence.jazzer.mutation.combinator.MutatorCombinators.mutateProduct; +import static com.code_intelligence.jazzer.mutation.combinator.MutatorCombinators.mutateProperty; +import static com.code_intelligence.jazzer.mutation.combinator.MutatorCombinators.mutateSumInPlace; +import static com.code_intelligence.jazzer.mutation.combinator.MutatorCombinators.mutateThenMapToImmutable; +import static com.code_intelligence.jazzer.mutation.combinator.MutatorCombinators.mutateViaView; +import static com.code_intelligence.jazzer.mutation.support.InputStreamSupport.infiniteZeros; +import static com.code_intelligence.jazzer.mutation.support.TestSupport.mockCrossOver; +import static com.code_intelligence.jazzer.mutation.support.TestSupport.mockCrossOverInPlace; +import static com.code_intelligence.jazzer.mutation.support.TestSupport.mockInitInPlace; +import static com.code_intelligence.jazzer.mutation.support.TestSupport.mockInitializer; +import static com.code_intelligence.jazzer.mutation.support.TestSupport.mockMutator; +import static com.code_intelligence.jazzer.mutation.support.TestSupport.mockPseudoRandom; +import static com.code_intelligence.jazzer.mutation.support.TestSupport.nullDataOutputStream; +import static com.google.common.truth.Truth.assertThat; +import static java.util.Collections.singletonList; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.code_intelligence.jazzer.mutation.api.Debuggable; +import com.code_intelligence.jazzer.mutation.api.InPlaceMutator; +import com.code_intelligence.jazzer.mutation.api.PseudoRandom; +import com.code_intelligence.jazzer.mutation.api.Serializer; +import com.code_intelligence.jazzer.mutation.api.SerializingInPlaceMutator; +import com.code_intelligence.jazzer.mutation.api.SerializingMutator; +import com.code_intelligence.jazzer.mutation.support.TestSupport.MockPseudoRandom; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Predicate; +import java.util.function.ToIntFunction; +import org.junit.jupiter.api.Test; + +class MutatorCombinatorsTest { + @Test + void testMutateProperty() { + InPlaceMutator<Foo> mutator = + mutateProperty(Foo::getValue, mockMutator(21, value -> 2 * value), Foo::setValue); + + assertThat(mutator.toString()).isEqualTo("Foo.Integer"); + + Foo foo = new Foo(0, singletonList(13)); + + try (MockPseudoRandom prng = mockPseudoRandom()) { + mutator.initInPlace(foo, prng); + } + assertThat(foo.getValue()).isEqualTo(21); + assertThat(foo.getList()).containsExactly(13); + + try (MockPseudoRandom prng = mockPseudoRandom()) { + mutator.mutateInPlace(foo, prng); + } + + assertThat(foo.getValue()).isEqualTo(42); + assertThat(foo.getList()).containsExactly(13); + } + + @Test + void testCrossOverProperty() { + InPlaceMutator<Foo> mutator = + mutateProperty(Foo::getValue, mockCrossOver((a, b) -> 42), Foo::setValue); + Foo foo = new Foo(0); + Foo otherFoo = new Foo(1); + try (MockPseudoRandom prng = mockPseudoRandom( + // use foo value + 0)) { + mutator.crossOverInPlace(foo, otherFoo, prng); + assertThat(foo.getValue()).isEqualTo(0); + } + try (MockPseudoRandom prng = mockPseudoRandom( + // use otherFoo value + 1)) { + mutator.crossOverInPlace(foo, otherFoo, prng); + assertThat(foo.getValue()).isEqualTo(1); + } + try (MockPseudoRandom prng = mockPseudoRandom( + // use property type cross over + 2)) { + mutator.crossOverInPlace(foo, otherFoo, prng); + assertThat(foo.getValue()).isEqualTo(42); + } + } + + @Test + void testMutateViaView() { + InPlaceMutator<Foo> mutator = mutateViaView(Foo::getList, new InPlaceMutator<List<Integer>>() { + @Override + public void initInPlace(List<Integer> reference, PseudoRandom prng) { + reference.clear(); + reference.add(21); + } + + @Override + public void mutateInPlace(List<Integer> reference, PseudoRandom prng) { + reference.add(reference.get(reference.size() - 1) + 1); + } + + @Override + public void crossOverInPlace( + List<Integer> reference, List<Integer> otherReference, PseudoRandom prng) {} + + @Override + public String toDebugString(Predicate<Debuggable> isInCycle) { + return "List<Integer>"; + } + }); + + assertThat(mutator.toString()).isEqualTo("Foo via List<Integer>"); + + Foo foo = new Foo(13, singletonList(13)); + + try (MockPseudoRandom prng = mockPseudoRandom()) { + mutator.initInPlace(foo, prng); + } + assertThat(foo.getValue()).isEqualTo(13); + assertThat(foo.getList()).containsExactly(21); + + try (MockPseudoRandom prng = mockPseudoRandom()) { + mutator.mutateInPlace(foo, prng); + } + + assertThat(foo.getValue()).isEqualTo(13); + assertThat(foo.getList()).containsExactly(21, 22); + } + + @Test + void testCrossOverViaView() { + InPlaceMutator<Foo> mutator = mutateViaView(Foo::getList, mockCrossOverInPlace((a, b) -> { + a.clear(); + a.add(42); + })); + + Foo foo = new Foo(0, singletonList(0)); + Foo otherFoo = new Foo(0, singletonList(1)); + try (MockPseudoRandom prng = mockPseudoRandom()) { + mutator.crossOverInPlace(foo, otherFoo, prng); + assertThat(foo.getList()).containsExactly(42); + } + } + + @Test + void testMutateCombine() { + InPlaceMutator<Foo> valueMutator = + mutateProperty(Foo::getValue, mockMutator(21, value -> 2 * value), Foo::setValue); + + InPlaceMutator<Foo> listMutator = + mutateViaView(Foo::getList, new InPlaceMutator<List<Integer>>() { + @Override + public void initInPlace(List<Integer> reference, PseudoRandom prng) { + reference.clear(); + reference.add(21); + } + + @Override + public void mutateInPlace(List<Integer> reference, PseudoRandom prng) { + reference.add(reference.get(reference.size() - 1) + 1); + } + + @Override + public void crossOverInPlace( + List<Integer> reference, List<Integer> otherReference, PseudoRandom prng) {} + + @Override + public String toDebugString(Predicate<Debuggable> isInCycle) { + return "List<Integer>"; + } + }); + InPlaceMutator<Foo> mutator = combine(valueMutator, listMutator); + + assertThat(mutator.toString()).isEqualTo("{Foo.Integer, Foo via List<Integer>}"); + + Foo foo = new Foo(13, singletonList(13)); + + try (MockPseudoRandom prng = mockPseudoRandom()) { + mutator.initInPlace(foo, prng); + } + assertThat(foo.getValue()).isEqualTo(21); + assertThat(foo.getList()).containsExactly(21); + + try (MockPseudoRandom prng = mockPseudoRandom(/* use valueMutator */ 0)) { + mutator.mutateInPlace(foo, prng); + } + assertThat(foo.getValue()).isEqualTo(42); + assertThat(foo.getList()).containsExactly(21); + + try (MockPseudoRandom prng = mockPseudoRandom(/* use listMutator */ 1)) { + mutator.mutateInPlace(foo, prng); + } + assertThat(foo.getValue()).isEqualTo(42); + assertThat(foo.getList()).containsExactly(21, 22); + } + + @Test + void testCrossOverCombine() { + InPlaceMutator<Foo> valueMutator = + mutateProperty(Foo::getValue, mockCrossOver((a, b) -> 42), Foo::setValue); + InPlaceMutator<Foo> listMutator = mutateViaView(Foo::getList, mockCrossOverInPlace((a, b) -> { + a.clear(); + a.add(42); + })); + InPlaceMutator<Foo> mutator = combine(valueMutator, listMutator); + + Foo foo = new Foo(0, singletonList(0)); + Foo fooOther = new Foo(1, singletonList(1)); + + try (MockPseudoRandom prng = mockPseudoRandom( + // call cross over in property mutator + 2)) { + mutator.crossOverInPlace(foo, fooOther, prng); + } + assertThat(foo.getValue()).isEqualTo(42); + assertThat(foo.getList()).containsExactly(42); + } + + @Test + void testCrossOverEmptyCombine() { + Foo foo = new Foo(0, singletonList(0)); + Foo fooOther = new Foo(1, singletonList(1)); + InPlaceMutator<Foo> emptyCombineMutator = combine(); + try (MockPseudoRandom prng = mockPseudoRandom()) { + emptyCombineMutator.crossOverInPlace(foo, fooOther, prng); + } + assertThat(foo.getValue()).isEqualTo(0); + assertThat(foo.getList()).containsExactly(0); + } + + @Test + void testMutateAssemble() { + InPlaceMutator<Foo> valueMutator = + mutateProperty(Foo::getValue, mockMutator(21, value -> 2 * value), Foo::setValue); + + InPlaceMutator<Foo> listMutator = + mutateViaView(Foo::getList, new InPlaceMutator<List<Integer>>() { + @Override + public void initInPlace(List<Integer> reference, PseudoRandom prng) { + reference.clear(); + reference.add(21); + } + + @Override + public void mutateInPlace(List<Integer> reference, PseudoRandom prng) { + reference.add(reference.get(reference.size() - 1) + 1); + } + + @Override + public void crossOverInPlace( + List<Integer> reference, List<Integer> otherReference, PseudoRandom prng) {} + + @Override + public String toDebugString(Predicate<Debuggable> isInCycle) { + return "List<Integer>"; + } + }); + + SerializingInPlaceMutator<Foo> mutator = + assemble((m) -> {}, () -> new Foo(0, singletonList(0)), new Serializer<Foo>() { + @Override + public Foo read(DataInputStream in) { + return null; + } + + @Override + public void write(Foo value, DataOutputStream out) {} + + @Override + public Foo detach(Foo value) { + return null; + } + }, () -> combine(valueMutator, listMutator)); + + assertThat(mutator.toString()).isEqualTo("{Foo.Integer, Foo via List<Integer>}"); + + Foo foo = new Foo(13, singletonList(13)); + + try (MockPseudoRandom prng = mockPseudoRandom()) { + mutator.initInPlace(foo, prng); + } + assertThat(foo.getValue()).isEqualTo(21); + assertThat(foo.getList()).containsExactly(21); + + try (MockPseudoRandom prng = mockPseudoRandom(/* use valueMutator */ 0)) { + mutator.mutateInPlace(foo, prng); + } + assertThat(foo.getValue()).isEqualTo(42); + assertThat(foo.getList()).containsExactly(21); + + try (MockPseudoRandom prng = mockPseudoRandom(/* use listMutator */ 1)) { + mutator.mutateInPlace(foo, prng); + } + assertThat(foo.getValue()).isEqualTo(42); + assertThat(foo.getList()).containsExactly(21, 22); + } + + @Test + void testCrossOverAssemble() { + InPlaceMutator<Foo> valueMutator = + mutateProperty(Foo::getValue, mockCrossOver((a, b) -> 42), Foo::setValue); + + InPlaceMutator<Foo> listMutator = mutateViaView(Foo::getList, mockCrossOverInPlace((a, b) -> { + a.clear(); + a.add(42); + })); + + SerializingInPlaceMutator<Foo> mutator = + assemble((m) -> {}, () -> new Foo(0, singletonList(0)), new Serializer<Foo>() { + @Override + public Foo read(DataInputStream in) { + return null; + } + + @Override + public void write(Foo value, DataOutputStream out) {} + + @Override + public Foo detach(Foo value) { + return null; + } + }, () -> combine(valueMutator, listMutator)); + + Foo foo = new Foo(0, singletonList(0)); + Foo fooOther = new Foo(1, singletonList(1)); + + try (MockPseudoRandom prng = mockPseudoRandom( + // cross over in property mutator + 2)) { + mutator.crossOverInPlace(foo, fooOther, prng); + } + assertThat(foo.getValue()).isEqualTo(42); + assertThat(foo.getList()).containsExactly(42); + } + + @Test + void testMutateThenMapToImmutable() throws IOException { + SerializingMutator<char[]> charMutator = + mockMutator(new char[] {'H', 'e', 'l', 'l', 'o'}, chars -> { + for (int i = 0; i < chars.length; i++) { + chars[i] ^= (1 << 5); + } + chars[chars.length - 1]++; + return chars; + }); + SerializingMutator<String> mutator = + mutateThenMapToImmutable(charMutator, String::new, String::toCharArray); + + assertThat(mutator.toString()).isEqualTo("char[] -> String"); + + String value = mutator.read(new DataInputStream(infiniteZeros())); + assertThat(value).isEqualTo("Hello"); + + try (MockPseudoRandom prng = mockPseudoRandom()) { + value = mutator.mutate(value, prng); + } + assertThat(value).isEqualTo("hELLP"); + + try (MockPseudoRandom prng = mockPseudoRandom()) { + value = mutator.mutate(value, prng); + } + assertThat(value).isEqualTo("Hellq"); + + try (MockPseudoRandom prng = mockPseudoRandom()) { + value = mutator.init(prng); + } + assertThat(value).isEqualTo("Hello"); + + try (MockPseudoRandom prng = mockPseudoRandom()) { + value = mutator.mutate(value, prng); + } + assertThat(value).isEqualTo("hELLP"); + + final String capturedValue = value; + assertThrows(UnsupportedOperationException.class, + () -> mutator.write(capturedValue, nullDataOutputStream())); + } + + @Test + void testCrossOverThenMapToImmutable() { + SerializingMutator<char[]> charMutator = mockCrossOver((a, b) -> { + assertThat(a).isEqualTo(new char[] {'H', 'e', 'l', 'l', 'o'}); + assertThat(b).isEqualTo(new char[] {'W', 'o', 'r', 'l', 'd'}); + return new char[] {'T', 'e', 's', 't', 'e', 'd'}; + }); + SerializingMutator<String> mutator = + mutateThenMapToImmutable(charMutator, String::new, String::toCharArray); + + String crossedOver; + try (MockPseudoRandom prng = mockPseudoRandom()) { + crossedOver = mutator.crossOver("Hello", "World", prng); + } + assertThat(crossedOver).isEqualTo("Tested"); + } + + @Test + void testCrossOverProduct() { + SerializingMutator<Boolean> mutator1 = mockCrossOver((a, b) -> true); + SerializingMutator<Integer> mutator2 = mockCrossOver((a, b) -> 42); + ProductMutator mutator = mutateProduct(mutator1, mutator2); + + try (MockPseudoRandom prng = mockPseudoRandom( + // use first value in mutator1 + 0, + // use second value in mutator2 + 0)) { + Object[] crossedOver = + mutator.crossOver(new Object[] {false, 0}, new Object[] {true, 1}, prng); + assertThat(crossedOver).isEqualTo(new Object[] {false, 0}); + } + + try (MockPseudoRandom prng = mockPseudoRandom( + // use first value in mutator1 + 1, + // use second value in mutator2 + 1)) { + Object[] crossedOver = + mutator.crossOver(new Object[] {false, 0}, new Object[] {true, 1}, prng); + assertThat(crossedOver).isEqualTo(new Object[] {true, 1}); + } + + try (MockPseudoRandom prng = mockPseudoRandom( + // use cross over in mutator1 + 2, + // use cross over in mutator2 + 2)) { + Object[] crossedOver = + mutator.crossOver(new Object[] {false, 0}, new Object[] {true, 2}, prng); + assertThat(crossedOver).isEqualTo(new Object[] {true, 42}); + } + } + + @Test + void testCrossOverSumInPlaceSameType() { + ToIntFunction<List<Integer>> mutotarIndexFromValue = (r) -> 0; + InPlaceMutator<List<Integer>> mutator1 = mockCrossOverInPlace((a, b) -> { a.add(42); }); + InPlaceMutator<List<Integer>> mutator2 = mockCrossOverInPlace((a, b) -> {}); + InPlaceMutator<List<Integer>> mutator = + mutateSumInPlace(mutotarIndexFromValue, mutator1, mutator2); + + List<Integer> a = new ArrayList<>(); + List<Integer> b = new ArrayList<>(); + + try (MockPseudoRandom prng = mockPseudoRandom()) { + mutator.crossOverInPlace(a, b, prng); + } + assertThat(a).containsExactly(42); + } + + @Test + void testCrossOverSumInPlaceIndeterminate() { + InPlaceMutator<List<?>> mutator1 = mockCrossOverInPlace((a, b) -> {}); + InPlaceMutator<List<?>> mutator2 = mockCrossOverInPlace((a, b) -> {}); + ToIntFunction<List<?>> bothIndeterminate = (r) -> - 1; + + InPlaceMutator<List<?>> mutator = mutateSumInPlace(bothIndeterminate, mutator1, mutator2); + + List<Integer> a = new ArrayList<>(); + a.add(42); + List<Integer> b = new ArrayList<>(); + + try (MockPseudoRandom prng = mockPseudoRandom()) { + mutator.crossOverInPlace(a, b, prng); + assertThat(a).containsExactly(42); + } + } + + @Test + void testCrossOverSumInPlaceFirstIndeterminate() { + List<Integer> reference = new ArrayList<>(); + List<Integer> otherReference = new ArrayList<>(); + + InPlaceMutator<List<Integer>> mutator1 = mockCrossOverInPlace((a, b) -> {}); + InPlaceMutator<List<Integer>> mutator2 = mockInitInPlace((l) -> { l.add(42); }); + ToIntFunction<List<Integer>> firstIndeterminate = (r) -> r == reference ? -1 : 1; + + InPlaceMutator<List<Integer>> mutator = + mutateSumInPlace(firstIndeterminate, mutator1, mutator2); + + try (MockPseudoRandom prng = mockPseudoRandom()) { + mutator.crossOverInPlace(reference, otherReference, prng); + assertThat(reference).containsExactly(42); + } + } + + static class Foo { + private int value; + private final List<Integer> list; + + public Foo(int value) { + this(value, new ArrayList<>()); + } + public Foo(int value, List<Integer> list) { + this.value = value; + this.list = new ArrayList<>(list); + } + + public List<Integer> getList() { + return list; + } + + public int getValue() { + return value; + } + + public void setValue(int value) { + this.value = value; + } + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/mutation/engine/BUILD.bazel b/src/test/java/com/code_intelligence/jazzer/mutation/engine/BUILD.bazel new file mode 100644 index 00000000..9cb59dee --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/mutation/engine/BUILD.bazel @@ -0,0 +1,13 @@ +load("@contrib_rules_jvm//java:defs.bzl", "java_test_suite") + +java_test_suite( + name = "EngineTests", + size = "small", + srcs = glob(["*.java"]), + runner = "junit5", + deps = [ + "//src/main/java/com/code_intelligence/jazzer/mutation/engine", + "//src/main/java/com/code_intelligence/jazzer/mutation/support", + "//src/test/java/com/code_intelligence/jazzer/mutation/support:test_support", + ], +) diff --git a/src/test/java/com/code_intelligence/jazzer/mutation/engine/SeededPseudoRandomTest.java b/src/test/java/com/code_intelligence/jazzer/mutation/engine/SeededPseudoRandomTest.java new file mode 100644 index 00000000..38ab2eb2 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/mutation/engine/SeededPseudoRandomTest.java @@ -0,0 +1,143 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.code_intelligence.jazzer.mutation.engine; + +import static com.google.common.truth.Truth.assertThat; +import static java.util.stream.Collectors.collectingAndThen; +import static java.util.stream.Collectors.counting; +import static java.util.stream.Collectors.groupingBy; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.params.provider.Arguments.arguments; + +import com.google.common.truth.Correspondence; +import java.util.Map; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +public class SeededPseudoRandomTest { + static Stream<Arguments> doubleClosedRange() { + return Stream.of(arguments(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, false), + arguments(Double.MAX_VALUE, Double.POSITIVE_INFINITY, false), + arguments(Double.NEGATIVE_INFINITY, -Double.MAX_VALUE, false), + arguments(-Double.MAX_VALUE, Double.MAX_VALUE, false), + arguments(-Double.MAX_VALUE, -Double.MAX_VALUE, false), + arguments(-Double.MAX_VALUE * 0.5, Double.MAX_VALUE * 0.5, false), + arguments(-Double.MAX_VALUE * 0.5, Math.nextUp(Double.MAX_VALUE * 0.5), false), + arguments(Double.MAX_VALUE, Double.MAX_VALUE, false), + arguments(-Double.MIN_VALUE, Double.MIN_VALUE, false), + arguments(-Double.MIN_VALUE, 0, false), arguments(0, Double.MIN_VALUE, false), + arguments(-Double.MAX_VALUE, 0, false), arguments(0, Double.MAX_VALUE, false), + arguments(1000.0, Double.MAX_VALUE, false), arguments(0, Double.POSITIVE_INFINITY, false), + arguments(1e200, Double.POSITIVE_INFINITY, false), + arguments(Double.NEGATIVE_INFINITY, -1e200, false), arguments(0.0, 1.0, false), + arguments(-1.0, 1.0, false), arguments(-1e300, 1e300, false), + arguments(0.0, 0.0 + Double.MIN_VALUE, false), + arguments(-Double.MAX_VALUE, -Double.MAX_VALUE + 1e292, false), + arguments(-Double.NaN, 0.0, true), arguments(0.0, Double.NaN, true), + arguments(Double.NaN, Double.NaN, true)); + } + + static Stream<Arguments> floatClosedRange() { + return Stream.of(arguments(Float.NEGATIVE_INFINITY, Float.POSITIVE_INFINITY, false), + arguments(Float.MAX_VALUE, Float.POSITIVE_INFINITY, false), + arguments(Float.NEGATIVE_INFINITY, -Float.MAX_VALUE, false), + arguments(-Float.MAX_VALUE, Float.MAX_VALUE, false), + arguments(-Float.MAX_VALUE, -Float.MAX_VALUE, false), + arguments(Float.MAX_VALUE, Float.MAX_VALUE, false), + arguments(-Float.MAX_VALUE / 2f, Float.MAX_VALUE / 2f, false), + arguments(-Float.MIN_VALUE, Float.MIN_VALUE, false), arguments(-Float.MIN_VALUE, 0f, false), + arguments(0f, Float.MIN_VALUE, false), arguments(-Float.MAX_VALUE, 0f, false), + arguments(0f, Float.MAX_VALUE, false), arguments(-Float.MAX_VALUE, -0f, false), + arguments(-0f, Float.MAX_VALUE, false), arguments(1000f, Float.MAX_VALUE, false), + arguments(0f, Float.POSITIVE_INFINITY, false), + arguments(1e38f, Float.POSITIVE_INFINITY, false), + arguments(Float.NEGATIVE_INFINITY, -1e38f, false), arguments(0f, 1f, false), + arguments(-1f, 1f, false), arguments(-1e38f, 1e38f, false), + arguments(0f, 0f + Float.MIN_VALUE, false), + arguments(-Float.MAX_VALUE, -Float.MAX_VALUE + 1e32f, false), + arguments(-Float.NaN, 0f, true), arguments(0f, Float.NaN, true), + arguments(Float.NaN, Float.NaN, true)); + } + + @ParameterizedTest + @MethodSource("doubleClosedRange") + void testDoubleForceInRange(double minValue, double maxValue, boolean throwsException) { + SeededPseudoRandom seededPseudoRandom = new SeededPseudoRandom(1337); + for (int i = 0; i < 1000; i++) { + if (throwsException) { + assertThrows(IllegalArgumentException.class, + () + -> seededPseudoRandom.closedRange(minValue, maxValue), + "minValue: " + minValue + ", maxValue: " + maxValue); + } else { + double inClosedRange = seededPseudoRandom.closedRange(minValue, maxValue); + assertThat(inClosedRange).isAtLeast(minValue); + assertThat(inClosedRange).isAtMost(maxValue); + assertThat(inClosedRange).isFinite(); + } + } + } + + @ParameterizedTest + @MethodSource("floatClosedRange") + void testFloatForceInRange(float minValue, float maxValue, boolean throwsException) { + SeededPseudoRandom seededPseudoRandom = new SeededPseudoRandom(1337); + for (int i = 0; i < 1000; i++) { + if (throwsException) { + assertThrows(IllegalArgumentException.class, + () + -> seededPseudoRandom.closedRange(minValue, maxValue), + "minValue: " + minValue + ", maxValue: " + maxValue); + } else { + float inClosedRange = seededPseudoRandom.closedRange(minValue, maxValue); + assertThat(inClosedRange).isAtLeast(minValue); + assertThat(inClosedRange).isAtMost(maxValue); + assertThat(inClosedRange).isFinite(); + } + } + } + + @Test + void testClosedRangeBiasedTowardsSmall() { + SeededPseudoRandom prng = new SeededPseudoRandom(1337133371337L); + + assertThrows(IllegalArgumentException.class, () -> prng.closedRangeBiasedTowardsSmall(-1)); + assertThrows(IllegalArgumentException.class, () -> prng.closedRangeBiasedTowardsSmall(2, 1)); + assertThat(prng.closedRangeBiasedTowardsSmall(0)).isEqualTo(0); + assertThat(prng.closedRangeBiasedTowardsSmall(5, 5)).isEqualTo(5); + } + + @Test + void testClosedRangeBiasedTowardsSmall_distribution() { + int num = 5000000; + SeededPseudoRandom prng = new SeededPseudoRandom(1337133371337L); + Map<Integer, Double> frequencies = + Stream.generate(() -> prng.closedRangeBiasedTowardsSmall(9)) + .limit(num) + .collect( + groupingBy(i -> i, collectingAndThen(counting(), count -> ((double) count) / num))); + // Reference values obtained from + // https://www.wolframalpha.com/input?i=N%5BTable%5BPDF%5BZipfDistribution%5B10%2C+1%5D%2C+i%5D%2C+%7Bi%2C+1%2C+10%7D%5D%5D + assertThat(frequencies) + .comparingValuesUsing(Correspondence.tolerance(0.0005)) + .containsExactly(0, 0.645, 1, 0.161, 2, 0.072, 3, 0.040, 4, 0.026, 5, 0.018, 6, 0.013, 7, + 0.01, 8, 0.008, 9, 0.006); + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/mutation/mutator/BUILD.bazel b/src/test/java/com/code_intelligence/jazzer/mutation/mutator/BUILD.bazel new file mode 100644 index 00000000..26943353 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/mutation/mutator/BUILD.bazel @@ -0,0 +1,28 @@ +load("@contrib_rules_jvm//java:defs.bzl", "java_junit5_test") + +TEST_PARALLELISM = 4 + +java_junit5_test( + name = "StressTest", + size = "large", + srcs = ["StressTest.java"], + env = {"JAZZER_MOCK_LIBFUZZER_MUTATOR": "true"}, + jvm_flags = [ + "-Djunit.jupiter.execution.parallel.enabled=true", + "-Djunit.jupiter.execution.parallel.mode.default=concurrent", + "-Djunit.jupiter.execution.parallel.config.strategy=fixed", + "-Djunit.jupiter.execution.parallel.config.fixed.parallelism=" + str(TEST_PARALLELISM), + ], + tags = ["cpu:" + str(TEST_PARALLELISM)], + deps = [ + "//src/main/java/com/code_intelligence/jazzer/mutation/annotation", + "//src/main/java/com/code_intelligence/jazzer/mutation/annotation/proto", + "//src/main/java/com/code_intelligence/jazzer/mutation/api", + "//src/main/java/com/code_intelligence/jazzer/mutation/mutator", + "//src/main/java/com/code_intelligence/jazzer/mutation/support", + "//src/test/java/com/code_intelligence/jazzer/mutation/mutator/proto:proto2_java_proto", + "//src/test/java/com/code_intelligence/jazzer/mutation/mutator/proto:proto3_java_proto", + "//src/test/java/com/code_intelligence/jazzer/mutation/support:test_support", + "@com_google_protobuf_protobuf_java//jar", + ], +) diff --git a/src/test/java/com/code_intelligence/jazzer/mutation/mutator/StressTest.java b/src/test/java/com/code_intelligence/jazzer/mutation/mutator/StressTest.java new file mode 100644 index 00000000..3bf880a4 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/mutation/mutator/StressTest.java @@ -0,0 +1,588 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.code_intelligence.jazzer.mutation.mutator; + +import static com.code_intelligence.jazzer.mutation.mutator.Mutators.validateAnnotationUsage; +import static com.code_intelligence.jazzer.mutation.support.InputStreamSupport.extendWithZeros; +import static com.code_intelligence.jazzer.mutation.support.Preconditions.require; +import static com.code_intelligence.jazzer.mutation.support.TestSupport.anyPseudoRandom; +import static com.code_intelligence.jazzer.mutation.support.TypeSupport.asAnnotatedType; +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth.assertWithMessage; +import static java.lang.Math.floor; +import static java.lang.Math.pow; +import static java.lang.Math.sqrt; +import static java.util.Collections.emptyList; +import static java.util.Collections.singletonList; +import static java.util.stream.IntStream.rangeClosed; +import static org.junit.jupiter.params.provider.Arguments.arguments; + +import com.code_intelligence.jazzer.mutation.annotation.DoubleInRange; +import com.code_intelligence.jazzer.mutation.annotation.FloatInRange; +import com.code_intelligence.jazzer.mutation.annotation.InRange; +import com.code_intelligence.jazzer.mutation.annotation.NotNull; +import com.code_intelligence.jazzer.mutation.annotation.WithSize; +import com.code_intelligence.jazzer.mutation.annotation.proto.AnySource; +import com.code_intelligence.jazzer.mutation.annotation.proto.WithDefaultInstance; +import com.code_intelligence.jazzer.mutation.api.PseudoRandom; +import com.code_intelligence.jazzer.mutation.api.Serializer; +import com.code_intelligence.jazzer.mutation.api.SerializingMutator; +import com.code_intelligence.jazzer.mutation.support.TypeHolder; +import com.code_intelligence.jazzer.protobuf.Proto2.TestProtobuf; +import com.code_intelligence.jazzer.protobuf.Proto3.AnyField3; +import com.code_intelligence.jazzer.protobuf.Proto3.BytesField3; +import com.code_intelligence.jazzer.protobuf.Proto3.DoubleField3; +import com.code_intelligence.jazzer.protobuf.Proto3.EnumField3; +import com.code_intelligence.jazzer.protobuf.Proto3.EnumField3.TestEnum; +import com.code_intelligence.jazzer.protobuf.Proto3.EnumFieldRepeated3; +import com.code_intelligence.jazzer.protobuf.Proto3.EnumFieldRepeated3.TestEnumRepeated; +import com.code_intelligence.jazzer.protobuf.Proto3.FloatField3; +import com.code_intelligence.jazzer.protobuf.Proto3.IntegralField3; +import com.code_intelligence.jazzer.protobuf.Proto3.MapField3; +import com.code_intelligence.jazzer.protobuf.Proto3.MessageField3; +import com.code_intelligence.jazzer.protobuf.Proto3.MessageMapField3; +import com.code_intelligence.jazzer.protobuf.Proto3.OptionalPrimitiveField3; +import com.code_intelligence.jazzer.protobuf.Proto3.PrimitiveField3; +import com.code_intelligence.jazzer.protobuf.Proto3.RepeatedDoubleField3; +import com.code_intelligence.jazzer.protobuf.Proto3.RepeatedFloatField3; +import com.code_intelligence.jazzer.protobuf.Proto3.RepeatedIntegralField3; +import com.code_intelligence.jazzer.protobuf.Proto3.RepeatedRecursiveMessageField3; +import com.code_intelligence.jazzer.protobuf.Proto3.SingleOptionOneOfField3; +import com.code_intelligence.jazzer.protobuf.Proto3.StringField3; +import com.google.protobuf.Any; +import com.google.protobuf.Descriptors.Descriptor; +import com.google.protobuf.Descriptors.FieldDescriptor; +import com.google.protobuf.Descriptors.FieldDescriptor.JavaType; +import com.google.protobuf.DynamicMessage; +import com.google.protobuf.Message; +import com.google.protobuf.Message.Builder; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.lang.reflect.AnnotatedType; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.stream.Stream; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +public class StressTest { + private static final int NUM_INITS = 500; + private static final int NUM_MUTATE_PER_INIT = 100; + private static final double MANY_DISTINCT_ELEMENTS_RATIO = 0.5; + + private enum TestEnumTwo { A, B } + + private enum TestEnumThree { A, B, C } + + @SuppressWarnings("unused") + static Message getTestProtobufDefaultInstance() { + return TestProtobuf.getDefaultInstance(); + } + + public static Stream<Arguments> stressTestCases() { + return Stream.of(arguments(asAnnotatedType(boolean.class), "Boolean", exactly(false, true), + exactly(false, true)), + arguments(new TypeHolder<@NotNull Boolean>() {}.annotatedType(), "Boolean", + exactly(false, true), exactly(false, true)), + arguments(new TypeHolder<Boolean>() {}.annotatedType(), "Nullable<Boolean>", + exactly(null, false, true), exactly(null, false, true)), + arguments(new TypeHolder<@NotNull List<@NotNull Boolean>>() {}.annotatedType(), + "List<Boolean>", exactly(emptyList(), singletonList(false), singletonList(true)), + manyDistinctElements()), + arguments(new TypeHolder<@NotNull List<Boolean>>() {}.annotatedType(), + "List<Nullable<Boolean>>", + exactly(emptyList(), singletonList(null), singletonList(false), singletonList(true)), + manyDistinctElements()), + arguments(new TypeHolder<List<@NotNull Boolean>>() {}.annotatedType(), + "Nullable<List<Boolean>>", + exactly(null, emptyList(), singletonList(false), singletonList(true)), + distinctElementsRatio(0.30)), + arguments(new TypeHolder<List<Boolean>>() {}.annotatedType(), + "Nullable<List<Nullable<Boolean>>>", + exactly( + null, emptyList(), singletonList(null), singletonList(false), singletonList(true)), + distinctElementsRatio(0.30)), + arguments( + new TypeHolder<@NotNull Map<@NotNull String, @NotNull String>>() {}.annotatedType(), + "Map<String,String>", distinctElementsRatio(0.45), distinctElementsRatio(0.45)), + arguments(new TypeHolder<Map<@NotNull String, @NotNull String>>() {}.annotatedType(), + "Nullable<Map<String,String>>", distinctElementsRatio(0.46), + distinctElementsRatio(0.48)), + arguments( + new TypeHolder<@WithSize(max = 3) @NotNull Map<@NotNull Integer, @NotNull Integer>>() { + }.annotatedType(), + "Map<Integer,Integer>", + // Half of all maps are empty, the other half is heavily biased towards special values. + all(mapSizeInClosedRange(0, 3), distinctElementsRatio(0.2)), + all(mapSizeInClosedRange(0, 3), manyDistinctElements())), + arguments( + new TypeHolder<@NotNull Map<@NotNull Boolean, @NotNull Boolean>>() {}.annotatedType(), + "Map<Boolean,Boolean>", + // 1 0-element map, 4 1-element maps + distinctElements(1 + 4), + // 1 0-element map, 4 1-element maps, 4 2-element maps + distinctElements(1 + 4 + 4)), + arguments(asAnnotatedType(byte.class), "Byte", + // init is heavily biased towards special values and only returns a uniformly random + // value in 1 out of 5 calls. + all(expectedNumberOfDistinctElements(1 << Byte.SIZE, boundHits(NUM_INITS, 0.2)), + contains((byte) 0, (byte) 1, Byte.MIN_VALUE, Byte.MAX_VALUE)), + // With mutations, we expect to reach all possible bytes. + exactly(rangeClosed(Byte.MIN_VALUE, Byte.MAX_VALUE).mapToObj(i -> (byte) i).toArray())), + arguments(asAnnotatedType(short.class), "Short", + // init is heavily biased towards special values and only returns a uniformly random + // value in 1 out of 5 calls. + all(expectedNumberOfDistinctElements(1 << Short.SIZE, boundHits(NUM_INITS, 0.2)), + contains((short) 0, (short) 1, Short.MIN_VALUE, Short.MAX_VALUE)), + // The integral type mutator does not always return uniformly random values and the + // random walk it uses is more likely to produce non-distinct elements, hence the test + // only passes with ~90% of the optimal parameters. + expectedNumberOfDistinctElements( + 1 << Short.SIZE, NUM_INITS * NUM_MUTATE_PER_INIT * 9 / 10)), + arguments(asAnnotatedType(int.class), "Integer", + // init is heavily biased towards special values and only returns a uniformly random + // value in 1 out of 5 calls. + all(expectedNumberOfDistinctElements(1L << Integer.SIZE, boundHits(NUM_INITS, 0.2)), + contains(0, 1, Integer.MIN_VALUE, Integer.MAX_VALUE)), + // See "Short" case. + expectedNumberOfDistinctElements( + 1L << Integer.SIZE, NUM_INITS * NUM_MUTATE_PER_INIT * 9 / 10)), + arguments(new TypeHolder<@NotNull @InRange(min = 0) Long>() {}.annotatedType(), "Long", + // init is heavily biased towards special values and only returns a uniformly random + // value in 1 out of 5 calls. + all(expectedNumberOfDistinctElements(1L << Long.SIZE - 1, boundHits(NUM_INITS, 0.2)), + contains(0L, 1L, Long.MAX_VALUE)), + // See "Short" case. + expectedNumberOfDistinctElements( + 1L << Integer.SIZE - 1, NUM_INITS * NUM_MUTATE_PER_INIT * 9 / 10)), + arguments( + new TypeHolder<@NotNull @InRange(max = Integer.MIN_VALUE + 5) Integer>() { + }.annotatedType(), + "Integer", + exactly(rangeClosed(Integer.MIN_VALUE, Integer.MIN_VALUE + 5).boxed().toArray()), + exactly(rangeClosed(Integer.MIN_VALUE, Integer.MIN_VALUE + 5).boxed().toArray())), + arguments(asAnnotatedType(TestEnumTwo.class), "Nullable<Enum<TestEnumTwo>>", + exactly(null, TestEnumTwo.A, TestEnumTwo.B), + exactly(null, TestEnumTwo.A, TestEnumTwo.B)), + arguments(asAnnotatedType(TestEnumThree.class), "Nullable<Enum<TestEnumThree>>", + exactly(null, TestEnumThree.A, TestEnumThree.B, TestEnumThree.C), + exactly(null, TestEnumThree.A, TestEnumThree.B, TestEnumThree.C)), + arguments(new TypeHolder<@NotNull @FloatInRange(min = 0f) Float>() {}.annotatedType(), + "Float", + all(distinctElementsRatio(0.45), + doesNotContain(Float.NEGATIVE_INFINITY, -Float.MAX_VALUE, -Float.MIN_VALUE), + contains(Float.NaN, Float.POSITIVE_INFINITY, Float.MAX_VALUE, Float.MIN_VALUE, 0.0f, + -0.0f)), + all(distinctElementsRatio(0.75), + doesNotContain(Float.NEGATIVE_INFINITY, -Float.MAX_VALUE, -Float.MIN_VALUE))), + arguments(new TypeHolder<@NotNull Float>() {}.annotatedType(), "Float", + all(distinctElementsRatio(0.45), + contains(Float.NaN, Float.NEGATIVE_INFINITY, Float.POSITIVE_INFINITY, + -Float.MAX_VALUE, Float.MAX_VALUE, -Float.MIN_VALUE, Float.MIN_VALUE, 0.0f, + -0.0f)), + distinctElementsRatio(0.76)), + arguments( + new TypeHolder<@NotNull @FloatInRange( + min = -1.0f, max = 1.0f, allowNaN = false) Float>() { + }.annotatedType(), + "Float", + all(distinctElementsRatio(0.45), + doesNotContain(Float.NaN, -Float.MAX_VALUE, Float.MAX_VALUE, + Float.NEGATIVE_INFINITY, Float.POSITIVE_INFINITY), + contains(-Float.MIN_VALUE, Float.MIN_VALUE, 0.0f, -0.0f)), + all(distinctElementsRatio(0.525), + doesNotContain(Float.NaN, -Float.MAX_VALUE, Float.MAX_VALUE, + Float.NEGATIVE_INFINITY, Float.POSITIVE_INFINITY), + contains(-Float.MIN_VALUE, Float.MIN_VALUE, 0.0f, -0.0f))), + arguments(new TypeHolder<@NotNull Double>() {}.annotatedType(), "Double", + all(distinctElementsRatio(0.45), + contains(Double.NaN, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY)), + distinctElementsRatio(0.75)), + arguments( + new TypeHolder<@NotNull @DoubleInRange( + min = -1.0, max = 1.0, allowNaN = false) Double>() { + }.annotatedType(), + "Double", all(distinctElementsRatio(0.45), doesNotContain(Double.NaN)), + all(distinctElementsRatio(0.55), doesNotContain(Double.NaN)))); + } + + public static Stream<Arguments> protoStressTestCases() { + return Stream.of( + arguments(new TypeHolder<@NotNull OptionalPrimitiveField3>() {}.annotatedType(), + "{Builder.Nullable<Boolean>} -> Message", + exactly(OptionalPrimitiveField3.newBuilder().build(), + OptionalPrimitiveField3.newBuilder().setSomeField(false).build(), + OptionalPrimitiveField3.newBuilder().setSomeField(true).build()), + exactly(OptionalPrimitiveField3.newBuilder().build(), + OptionalPrimitiveField3.newBuilder().setSomeField(false).build(), + OptionalPrimitiveField3.newBuilder().setSomeField(true).build())), + arguments(new TypeHolder<@NotNull RepeatedRecursiveMessageField3>() {}.annotatedType(), + "{Builder.Boolean, WithoutInit(Builder via List<(cycle) -> Message>)} -> Message", + // The message field is recursive and thus not initialized. + exactly(RepeatedRecursiveMessageField3.getDefaultInstance(), + RepeatedRecursiveMessageField3.newBuilder().setSomeField(true).build()), + manyDistinctElements()), + arguments(new TypeHolder<@NotNull IntegralField3>() {}.annotatedType(), + "{Builder.Integer} -> Message", + // init is heavily biased towards special values and only returns a uniformly random + // value in 1 out of 5 calls. + all(expectedNumberOfDistinctElements(1L << Integer.SIZE, boundHits(NUM_INITS, 0.2)), + contains(IntegralField3.newBuilder().build(), + IntegralField3.newBuilder().setSomeField(1).build(), + IntegralField3.newBuilder().setSomeField(Integer.MIN_VALUE).build(), + IntegralField3.newBuilder().setSomeField(Integer.MAX_VALUE).build())), + // Our mutations return uniformly random elements in ~3/8 of all cases. + expectedNumberOfDistinctElements( + 1L << Integer.SIZE, NUM_INITS * NUM_MUTATE_PER_INIT * 3 / 8)), + arguments(new TypeHolder<@NotNull RepeatedIntegralField3>() {}.annotatedType(), + "{Builder via List<Integer>} -> Message", + contains(RepeatedIntegralField3.getDefaultInstance(), + RepeatedIntegralField3.newBuilder().addSomeField(0).build(), + RepeatedIntegralField3.newBuilder().addSomeField(1).build(), + RepeatedIntegralField3.newBuilder().addSomeField(Integer.MAX_VALUE).build(), + RepeatedIntegralField3.newBuilder().addSomeField(Integer.MIN_VALUE).build()), + // TODO: This ratio is on the lower end, most likely because of the strong bias towards + // special values combined with the small initial size of the list. When we improve the + // list mutator, this may be increased. + distinctElementsRatio(0.25)), + arguments(new TypeHolder<@NotNull BytesField3>() {}.annotatedType(), + "{Builder.byte[] -> ByteString} -> Message", manyDistinctElements(), + manyDistinctElements()), + arguments(new TypeHolder<@NotNull StringField3>() {}.annotatedType(), + "{Builder.String} -> Message", manyDistinctElements(), manyDistinctElements()), + arguments(new TypeHolder<@NotNull EnumField3>() {}.annotatedType(), + "{Builder.Enum<TestEnum>} -> Message", + exactly(EnumField3.getDefaultInstance(), + EnumField3.newBuilder().setSomeField(TestEnum.VAL2).build()), + exactly(EnumField3.getDefaultInstance(), + EnumField3.newBuilder().setSomeField(TestEnum.VAL2).build())), + arguments(new TypeHolder<@NotNull EnumFieldRepeated3>() {}.annotatedType(), + "{Builder via List<Enum<TestEnumRepeated>>} -> Message", + exactly(EnumFieldRepeated3.getDefaultInstance(), + EnumFieldRepeated3.newBuilder().addSomeField(TestEnumRepeated.UNASSIGNED).build(), + EnumFieldRepeated3.newBuilder().addSomeField(TestEnumRepeated.VAL1).build(), + EnumFieldRepeated3.newBuilder().addSomeField(TestEnumRepeated.VAL2).build()), + manyDistinctElements()), + arguments(new TypeHolder<@NotNull MapField3>() {}.annotatedType(), + "{Builder.Map<Integer,String>} -> Message", distinctElementsRatio(0.47), + manyDistinctElements()), + arguments(new TypeHolder<@NotNull MessageMapField3>() {}.annotatedType(), + "{Builder.Map<String,{Builder.Map<Integer,String>} -> Message>} -> Message", + distinctElementsRatio(0.45), distinctElementsRatio(0.45)), + arguments(new TypeHolder<@NotNull DoubleField3>() {}.annotatedType(), + "{Builder.Double} -> Message", distinctElementsRatio(0.45), distinctElementsRatio(0.7)), + arguments(new TypeHolder<@NotNull RepeatedDoubleField3>() {}.annotatedType(), + "{Builder via List<Double>} -> Message", distinctElementsRatio(0.2), + distinctElementsRatio(0.9)), + arguments(new TypeHolder<@NotNull FloatField3>() {}.annotatedType(), + "{Builder.Float} -> Message", distinctElementsRatio(0.45), distinctElementsRatio(0.7)), + arguments(new TypeHolder<@NotNull RepeatedFloatField3>() {}.annotatedType(), + "{Builder via List<Float>} -> Message", distinctElementsRatio(0.20), + distinctElementsRatio(0.9), emptyList()), + arguments(new TypeHolder<@NotNull TestProtobuf>() {}.annotatedType(), + "{Builder.Nullable<Boolean>, Builder.Nullable<Integer>, Builder.Nullable<Integer>, Builder.Nullable<Long>, Builder.Nullable<Long>, Builder.Nullable<Float>, Builder.Nullable<Double>, Builder.Nullable<String>, Builder.Nullable<Enum<Enum>>, WithoutInit(Builder.Nullable<{Builder.Nullable<Integer>, Builder via List<Integer>, WithoutInit(Builder.Nullable<(cycle) -> Message>)} -> Message>), Builder via List<Boolean>, Builder via List<Integer>, Builder via List<Integer>, Builder via List<Long>, Builder via List<Long>, Builder via List<Float>, Builder via List<Double>, Builder via List<String>, Builder via List<Enum<Enum>>, WithoutInit(Builder via List<(cycle) -> Message>), Builder.Map<Integer,Integer>, Builder.Nullable<FixedValue(OnlyLabel)>, Builder.Nullable<{<empty>} -> Message>, Builder.Nullable<Integer> | Builder.Nullable<Long> | Builder.Nullable<Integer>} -> Message", + manyDistinctElements(), manyDistinctElements()), + arguments( + new TypeHolder<@NotNull @WithDefaultInstance( + "com.code_intelligence.jazzer.mutation.mutator.StressTest#getTestProtobufDefaultInstance") + Message>() { + }.annotatedType(), + "{Builder.Nullable<Boolean>, Builder.Nullable<Integer>, Builder.Nullable<Integer>, Builder.Nullable<Long>, Builder.Nullable<Long>, Builder.Nullable<Float>, Builder.Nullable<Double>, Builder.Nullable<String>, Builder.Nullable<Enum<Enum>>, WithoutInit(Builder.Nullable<{Builder.Nullable<Integer>, Builder via List<Integer>, WithoutInit(Builder.Nullable<(cycle) -> Message>)} -> Message>), Builder via List<Boolean>, Builder via List<Integer>, Builder via List<Integer>, Builder via List<Long>, Builder via List<Long>, Builder via List<Float>, Builder via List<Double>, Builder via List<String>, Builder via List<Enum<Enum>>, WithoutInit(Builder via List<(cycle) -> Message>), Builder.Map<Integer,Integer>, Builder.Nullable<FixedValue(OnlyLabel)>, Builder.Nullable<{<empty>} -> Message>, Builder.Nullable<Integer> | Builder.Nullable<Long> | Builder.Nullable<Integer>} -> Message", + manyDistinctElements(), manyDistinctElements()), + arguments( + new TypeHolder<@NotNull @AnySource( + {PrimitiveField3.class, MessageField3.class}) AnyField3>() { + }.annotatedType(), + "{Builder.Nullable<Builder.{Builder.Boolean} -> Message | Builder.{Builder.Nullable<(cycle) -> Message>} -> Message -> Message>} -> Message", + exactly(AnyField3.getDefaultInstance(), + AnyField3.newBuilder() + .setSomeField(Any.pack(PrimitiveField3.getDefaultInstance())) + .build(), + AnyField3.newBuilder() + .setSomeField(Any.pack(PrimitiveField3.newBuilder().setSomeField(true).build())) + .build(), + AnyField3.newBuilder() + .setSomeField(Any.pack(MessageField3.getDefaultInstance())) + .build(), + AnyField3.newBuilder() + .setSomeField( + Any.pack(MessageField3.newBuilder() + .setMessageField(PrimitiveField3.getDefaultInstance()) + .build())) + .build(), + AnyField3.newBuilder() + .setSomeField(Any.pack( + MessageField3.newBuilder() + .setMessageField(PrimitiveField3.newBuilder().setSomeField(true)) + .build())) + .build()), + exactly(AnyField3.getDefaultInstance(), + AnyField3.newBuilder() + .setSomeField(Any.pack(PrimitiveField3.getDefaultInstance())) + .build(), + AnyField3.newBuilder() + .setSomeField(Any.pack(PrimitiveField3.newBuilder().setSomeField(true).build())) + .build(), + AnyField3.newBuilder() + .setSomeField(Any.pack(MessageField3.getDefaultInstance())) + .build(), + AnyField3.newBuilder() + .setSomeField( + Any.pack(MessageField3.newBuilder() + .setMessageField(PrimitiveField3.getDefaultInstance()) + .build())) + .build(), + AnyField3.newBuilder() + .setSomeField(Any.pack( + MessageField3.newBuilder() + .setMessageField(PrimitiveField3.newBuilder().setSomeField(true)) + .build())) + .build())), + arguments(new TypeHolder<@NotNull SingleOptionOneOfField3>() {}.annotatedType(), + "{Builder.Nullable<Boolean>} -> Message", + exactly(SingleOptionOneOfField3.getDefaultInstance(), + SingleOptionOneOfField3.newBuilder().setBoolField(false).build(), + SingleOptionOneOfField3.newBuilder().setBoolField(true).build()), + exactly(SingleOptionOneOfField3.getDefaultInstance(), + SingleOptionOneOfField3.newBuilder().setBoolField(false).build(), + SingleOptionOneOfField3.newBuilder().setBoolField(true).build()))); + } + + @SafeVarargs + private static Consumer<List<Object>> all(Consumer<List<Object>>... checks) { + return list -> { + for (Consumer<List<Object>> check : checks) { + check.accept(list); + } + }; + } + + private static Consumer<List<Object>> distinctElements(int num) { + return list -> assertThat(new HashSet<>(list).size()).isAtLeast(num); + } + + private static Consumer<List<Object>> manyDistinctElements() { + return distinctElementsRatio(MANY_DISTINCT_ELEMENTS_RATIO); + } + + /** + * Returns a lower bound on the expected number of hits when sampling from a domain of a given + * size with the given probability. + */ + private static int boundHits(long domainSize, double probability) { + // Binomial distribution. + double expectedValue = domainSize * probability; + double variance = domainSize * probability * (1 - probability); + double standardDeviation = sqrt(variance); + // Allow missing the expected value by two standard deviations. For a normal distribution, + // this would correspond to 95% of all cases. + int almostCertainLowerBound = (int) floor(expectedValue - 2 * standardDeviation); + return almostCertainLowerBound; + } + + /** + * Asserts that a given list contains at least as many distinct elements as can be expected when + * picking {@code picks} out of {@code domainSize} elements uniformly at random. + */ + private static Consumer<List<Object>> expectedNumberOfDistinctElements( + long domainSize, int picks) { + // https://www.randomservices.org/random/urn/Birthday.html#mom2 + double expectedValue = domainSize * (1 - pow(1 - 1.0 / domainSize, picks)); + double variance = domainSize * (domainSize - 1) * pow(1 - 2.0 / domainSize, picks) + + domainSize * pow(1 - 1.0 / domainSize, picks) + - domainSize * domainSize * pow(1 - 1.0 / domainSize, 2 * picks); + double standardDeviation = sqrt(variance); + // Allow missing the expected value by two standard deviations. For a normal distribution, + // this would correspond to 95% of all cases. + int almostCertainLowerBound = (int) floor(expectedValue - 2 * standardDeviation); + return list + -> assertWithMessage("V=distinct elements among %s picked out of %s\nE[V]=%s\nσ[V]=%s", + picks, domainSize, expectedValue, standardDeviation) + .that(new HashSet<>(list).size()) + .isAtLeast(almostCertainLowerBound); + } + + private static Consumer<List<Object>> distinctElementsRatio(double ratio) { + require(ratio > 0); + require(ratio <= 1); + return list -> assertThat(new HashSet<>(list).size() / (double) list.size()).isAtLeast(ratio); + } + + private static Consumer<List<Object>> exactly(Object... expected) { + return list -> assertThat(new HashSet<>(list)).containsExactly(expected); + } + + private static Consumer<List<Object>> contains(Object... expected) { + return list -> assertThat(new HashSet<>(list)).containsAtLeastElementsIn(expected); + } + + private static Consumer<List<Object>> doesNotContain(Object... expected) { + return list -> assertThat(new HashSet<>(list)).containsNoneIn(expected); + } + + private static Consumer<List<Object>> mapSizeInClosedRange(int min, int max) { + return list -> { + list.forEach(map -> { + if (map instanceof Map) { + assertThat(((Map) map).size()).isAtLeast(min); + assertThat(((Map) map).size()).isAtMost(max); + } else { + throw new IllegalArgumentException( + "Expected a list of maps, got list of" + map.getClass().getName()); + } + }); + }; + } + + @ParameterizedTest(name = "{index} {0}, {1}") + @MethodSource({"stressTestCases", "protoStressTestCases"}) + void genericMutatorStressTest(AnnotatedType type, String mutatorTree, + Consumer<List<Object>> expectedInitValues, Consumer<List<Object>> expectedMutatedValues) + throws IOException { + validateAnnotationUsage(type); + SerializingMutator mutator = Mutators.newFactory().createOrThrow(type); + assertThat(mutator.toString()).isEqualTo(mutatorTree); + + // Even with a fallback to mutating map values when no new key can be constructed, the map + // {false: true, true: false} will not change its equality class when the fallback picks both + // values to mutate. + boolean mayPerformNoopMutations = + mutatorTree.contains("FixedValue(") || mutatorTree.contains("Map<Boolean,Boolean>"); + + PseudoRandom rng = anyPseudoRandom(); + + List<Object> initValues = new ArrayList<>(); + List<Object> mutatedValues = new ArrayList<>(); + for (int i = 0; i < NUM_INITS; i++) { + Object value = mutator.init(rng); + + // For proto messages, each float field with value -0.0f, and double field with value -0.0 + // will be converted to 0.0f and 0.0, respectively. + Object fixedValue = fixFloatingPointsForProtos(value); + testReadWriteRoundtrip(mutator, fixedValue); + testReadWriteExclusiveRoundtrip(mutator, fixedValue); + + initValues.add(mutator.detach(value)); + value = fixFloatingPointsForProtos(value); + + for (int mutation = 0; mutation < NUM_MUTATE_PER_INIT; mutation++) { + Object detachedOldValue = mutator.detach(value); + value = mutator.mutate(value, rng); + if (!mayPerformNoopMutations) { + if (value instanceof Double) { + assertThat(Double.compare((Double) value, (Double) detachedOldValue)).isNotEqualTo(0); + } else if (value instanceof Float) { + assertThat(Float.compare((Float) value, (Float) detachedOldValue)).isNotEqualTo(0); + } else { + assertThat(detachedOldValue).isNotEqualTo(value); + } + } + + mutatedValues.add(mutator.detach(value)); + + // For proto messages, each float field with value -0.0f, and double field with value -0.0 + // will be converted to 0.0f and 0.0, respectively. This is because the values -0f and 0f + // and their double counterparts are serialized as default values (0f, and 0.0), which is + // relevant for mutation and the round trip tests. This means that the protos with float or + // double fields that equal to negative zero, will start mutation from positive zeros, and + // cause the assertion above to fail from time to time. To avoid this, we convert all + // negative zeros to positive zeros for float and double proto fields. + value = fixFloatingPointsForProtos(value); + testReadWriteRoundtrip(mutator, fixedValue); + testReadWriteExclusiveRoundtrip(mutator, fixedValue); + } + } + + expectedInitValues.accept(initValues); + expectedMutatedValues.accept(mutatedValues); + } + + private static <T> void testReadWriteExclusiveRoundtrip(Serializer<T> serializer, T value) + throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + serializer.writeExclusive(value, out); + T newValue = serializer.readExclusive(new ByteArrayInputStream(out.toByteArray())); + assertThat(newValue).isEqualTo(value); + } + + private static <T> void testReadWriteRoundtrip(Serializer<T> serializer, T value) + throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + serializer.write(value, new DataOutputStream(out)); + T newValue = serializer.read( + new DataInputStream(extendWithZeros(new ByteArrayInputStream(out.toByteArray())))); + assertThat(newValue).isEqualTo(value); + } + + // Filter out floating point values -0.0f and -0.0 and replace them + // by 0.0f and 0.0 respectively. + // This is a workaround for a bug in the protobuf library that causes + // our "...RoundTrip" tests to fail for negative zero in floats and doubles. + private static <T> T fixFloatingPointsForProtos(T value) { + if (!(value instanceof Message)) { + return value; + } + Message.Builder builder = ((Message) value).toBuilder(); + walkFields(builder, oldValue -> { + if (Objects.equals(oldValue, -0.0)) { + return 0.0; + } else if (Objects.equals(oldValue, -0.0f)) { + return 0.0f; + } else { + return oldValue; + } + }); + return (T) builder.build(); + } + + private static void walkFields(Builder builder, Function<Object, Object> transform) { + for (FieldDescriptor field : builder.getDescriptorForType().getFields()) { + if (field.isRepeated()) { + int bound = builder.getRepeatedFieldCount(field); + for (int i = 0; i < bound; i++) { + if (field.getJavaType() == JavaType.MESSAGE) { + Builder repeatedFieldBuilder = + ((Message) builder.getRepeatedField(field, i)).toBuilder(); + walkFields(repeatedFieldBuilder, transform); + builder.setRepeatedField(field, i, repeatedFieldBuilder.build()); + } else { + builder.setRepeatedField(field, i, transform.apply(builder.getRepeatedField(field, i))); + } + } + } else if (field.getJavaType() == JavaType.MESSAGE) { + // Break up unbounded recursion. + if (!builder.hasField(field)) { + continue; + } + Builder fieldBuilder = ((Message) builder.getField(field)).toBuilder(); + walkFields(fieldBuilder, transform); + builder.setField(field, fieldBuilder.build()); + } else { + builder.setField(field, transform.apply(builder.getField(field))); + } + } + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/mutation/mutator/collection/BUILD.bazel b/src/test/java/com/code_intelligence/jazzer/mutation/mutator/collection/BUILD.bazel new file mode 100644 index 00000000..2e60b9d5 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/mutation/mutator/collection/BUILD.bazel @@ -0,0 +1,17 @@ +load("@contrib_rules_jvm//java:defs.bzl", "java_test_suite") + +java_test_suite( + name = "CollectionTests", + size = "small", + srcs = glob(["*.java"]), + env = {"JAZZER_MOCK_LIBFUZZER_MUTATOR": "true"}, + runner = "junit5", + deps = [ + "//src/main/java/com/code_intelligence/jazzer/mutation/annotation", + "//src/main/java/com/code_intelligence/jazzer/mutation/api", + "//src/main/java/com/code_intelligence/jazzer/mutation/mutator/collection", + "//src/main/java/com/code_intelligence/jazzer/mutation/mutator/lang", + "//src/main/java/com/code_intelligence/jazzer/mutation/support", + "//src/test/java/com/code_intelligence/jazzer/mutation/support:test_support", + ], +) diff --git a/src/test/java/com/code_intelligence/jazzer/mutation/mutator/collection/ChunkMutationsTest.java b/src/test/java/com/code_intelligence/jazzer/mutation/mutator/collection/ChunkMutationsTest.java new file mode 100644 index 00000000..2fa0c1cf --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/mutation/mutator/collection/ChunkMutationsTest.java @@ -0,0 +1,237 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.code_intelligence.jazzer.mutation.mutator.collection; + +import static com.code_intelligence.jazzer.mutation.support.TestSupport.asMap; +import static com.code_intelligence.jazzer.mutation.support.TestSupport.asMutableList; +import static com.code_intelligence.jazzer.mutation.support.TestSupport.mockInitializer; +import static com.code_intelligence.jazzer.mutation.support.TestSupport.mockMutator; +import static com.code_intelligence.jazzer.mutation.support.TestSupport.mockPseudoRandom; +import static com.google.common.truth.Truth.assertThat; +import static java.util.stream.Collectors.toCollection; +import static java.util.stream.Collectors.toList; + +import com.code_intelligence.jazzer.mutation.api.SerializingMutator; +import com.code_intelligence.jazzer.mutation.support.TestSupport.MockPseudoRandom; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.Set; +import java.util.stream.IntStream; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; + +class ChunkMutationsTest { + @Test + void testDeleteRandomChunk() { + List<Integer> list = Stream.of(1, 2, 3, 4, 5, 6).collect(toList()); + + try (MockPseudoRandom prng = mockPseudoRandom(2, 3)) { + ChunkMutations.deleteRandomChunk(list, 2, prng); + } + assertThat(list).containsExactly(1, 2, 3, 6).inOrder(); + } + + @Test + void testInsertRandomChunk() { + List<String> list = Stream.of("1", "2", "3", "4", "5", "6").collect(toList()); + + try (MockPseudoRandom prng = mockPseudoRandom(2, 3)) { + ChunkMutations.insertRandomChunk(list, 10, mockInitializer(() -> "7", String::new), prng); + } + assertThat(list).containsExactly("1", "2", "3", "7", "7", "4", "5", "6").inOrder(); + String firstNewValue = list.get(3); + String secondNewValue = list.get(4); + assertThat(firstNewValue).isEqualTo(secondNewValue); + // Verify that the individual new elements were detached. + assertThat(firstNewValue).isNotSameInstanceAs(secondNewValue); + } + + @Test + void testInsertRandomChunkSet() { + Set<Integer> set = Stream.of(1, 2, 3, 4, 5, 6).collect(toCollection(LinkedHashSet::new)); + + Queue<Integer> initReturnValues = + Stream.of(7, 7, 7, 8, 9, 9).collect(toCollection(ArrayDeque::new)); + boolean result; + try (MockPseudoRandom prng = mockPseudoRandom(3)) { + result = ChunkMutations.insertRandomChunk( + set, set::add, 10, mockInitializer(initReturnValues::remove, v -> v), prng); + } + assertThat(result).isTrue(); + assertThat(set).containsExactly(1, 2, 3, 4, 5, 6, 7, 8, 9).inOrder(); + } + + @Test + void testInsertRandomChunkSet_largeChunk() { + Set<Integer> set = Stream.of(1, 2, 3, 4, 5, 6).collect(toCollection(LinkedHashSet::new)); + + Queue<Integer> initReturnValues = + IntStream.rangeClosed(1, 10000).boxed().collect(toCollection(ArrayDeque::new)); + boolean result; + try (MockPseudoRandom prng = mockPseudoRandom(9994)) { + result = ChunkMutations.insertRandomChunk( + set, set::add, 10000, mockInitializer(initReturnValues::remove, v -> v), prng); + } + assertThat(result).isTrue(); + assertThat(set) + .containsExactlyElementsIn(IntStream.rangeClosed(1, 10000).boxed().toArray()) + .inOrder(); + } + + @Test + void testInsertRandomChunkSet_failsToConstructDistinctValues() { + Set<Integer> set = Stream.of(1, 2, 3, 4, 5, 6).collect(toCollection(LinkedHashSet::new)); + + Queue<Integer> initReturnValues = + Stream.concat(Stream.of(7, 7, 7, 8), Stream.generate(() -> 7).limit(1000)) + .collect(toCollection(ArrayDeque::new)); + boolean result; + try (MockPseudoRandom prng = mockPseudoRandom(3)) { + result = ChunkMutations.insertRandomChunk( + set, set::add, 10, mockInitializer(initReturnValues::remove, v -> v), prng); + } + assertThat(result).isFalse(); + assertThat(set).containsExactly(1, 2, 3, 4, 5, 6, 7, 8).inOrder(); + } + + @Test + void testMutateChunk() { + List<Integer> list = Stream.of(1, 2, 3, 4, 5, 6).collect(toList()); + + try (MockPseudoRandom prng = mockPseudoRandom(2, 3)) { + ChunkMutations.mutateRandomChunk(list, mockMutator(1, i -> 2 * i), prng); + } + assertThat(list).containsExactly(1, 2, 3, 8, 10, 6).inOrder(); + } + + @Test + void testMutateRandomValuesChunk() { + Map<Integer, Integer> map = asMap(1, 10, 2, 20, 3, 30, 4, 40, 5, 50, 6, 60); + + try (MockPseudoRandom prng = mockPseudoRandom(2, 3)) { + ChunkMutations.mutateRandomValuesChunk(map, mockMutator(1, i -> 2 * i), prng); + } + assertThat(map).containsExactly(1, 10, 2, 20, 3, 30, 4, 80, 5, 100, 6, 60).inOrder(); + } + + @Test + void testMutateRandomKeysChunk() { + Map<List<Integer>, Integer> map = asMap(asMutableList(1), 10, asMutableList(2), 20, + asMutableList(3), 30, asMutableList(4), 40, asMutableList(5), 50, asMutableList(6), 60); + SerializingMutator<List<Integer>> keyMutator = mockMutator(null, list -> { + List<Integer> newList = list.stream().map(i -> i + 1).collect(toList()); + list.clear(); + return newList; + }, ArrayList::new); + + try (MockPseudoRandom prng = mockPseudoRandom(2, 3)) { + boolean result = ChunkMutations.mutateRandomKeysChunk(map, keyMutator, prng); + assertThat(result).isTrue(); + } + assertThat(map) + .containsExactly(asMutableList(1), 10, asMutableList(2), 20, asMutableList(3), 30, + asMutableList(6), 60, asMutableList(7), 40, asMutableList(8), 50) + .inOrder(); + } + + @Test + void testMutateRandomKeysChunk_failsToConstructSomeDistinctKeys() { + Map<List<Integer>, Integer> map = asMap(asMutableList(1), 10, asMutableList(2), 20, + asMutableList(3), 30, asMutableList(4), 40, asMutableList(5), 50, asMutableList(6), 60); + SerializingMutator<List<Integer>> keyMutator = mockMutator(null, list -> { + list.clear(); + List<Integer> newList = new ArrayList<>(); + newList.add(7); + return newList; + }, ArrayList::new); + + try (MockPseudoRandom prng = mockPseudoRandom(2, 3)) { + boolean result = ChunkMutations.mutateRandomKeysChunk(map, keyMutator, prng); + assertThat(result).isTrue(); + } + assertThat(map) + .containsExactly(asMutableList(1), 10, asMutableList(2), 20, asMutableList(3), 30, + asMutableList(5), 50, asMutableList(6), 60, asMutableList(7), 40) + .inOrder(); + } + + @Test + void testMutateRandomKeysChunk_failsToConstructAnyDistinctKeys() { + Map<List<Integer>, Integer> map = asMap(asMutableList(1), 10, asMutableList(2), 20, + asMutableList(3), 30, asMutableList(4), 40, asMutableList(5), 50, asMutableList(6), 60); + SerializingMutator<List<Integer>> keyMutator = mockMutator(null, list -> { + list.clear(); + List<Integer> newList = new ArrayList<>(); + newList.add(1); + return newList; + }, ArrayList::new); + + try (MockPseudoRandom prng = mockPseudoRandom(2, 3)) { + boolean result = ChunkMutations.mutateRandomKeysChunk(map, keyMutator, prng); + assertThat(result).isFalse(); + } + assertThat(map) + .containsExactly(asMutableList(1), 10, asMutableList(2), 20, asMutableList(3), 30, + asMutableList(4), 40, asMutableList(5), 50, asMutableList(6), 60) + .inOrder(); + } + + @Test + void testMutateRandomKeysChunk_nullKeyAndValue() { + Map<List<Integer>, Integer> map = asMap(asMutableList(1), 10, asMutableList(2), 20, + asMutableList(3), 30, asMutableList(4), null, null, 50, asMutableList(6), 60); + SerializingMutator<List<Integer>> keyMutator = mockMutator(null, list -> { + if (list != null) { + List<Integer> newList = list.stream().map(i -> i + 1).collect(toList()); + list.clear(); + return newList; + } else { + return asMutableList(10); + } + }, list -> list != null ? new ArrayList<>(list) : null); + + try (MockPseudoRandom prng = mockPseudoRandom(2, 3)) { + boolean result = ChunkMutations.mutateRandomKeysChunk(map, keyMutator, prng); + assertThat(result).isTrue(); + } + assertThat(map) + .containsExactly(asMutableList(1), 10, asMutableList(2), 20, asMutableList(3), 30, + asMutableList(6), 60, asMutableList(5), null, asMutableList(10), 50) + .inOrder(); + } + + @Test + void testMutateRandomKeysChunk_mutateKeyToNull() { + Map<List<Integer>, Integer> map = asMap(asMutableList(1), 10, asMutableList(2), 20, + asMutableList(3), 30, asMutableList(4), 40, asMutableList(5), 50, asMutableList(6), 60); + SerializingMutator<List<Integer>> keyMutator = + mockMutator(null, list -> null, list -> list != null ? new ArrayList<>(list) : null); + + try (MockPseudoRandom prng = mockPseudoRandom(1, 3)) { + boolean result = ChunkMutations.mutateRandomKeysChunk(map, keyMutator, prng); + assertThat(result).isTrue(); + } + assertThat(map) + .containsExactly(asMutableList(1), 10, asMutableList(2), 20, asMutableList(3), 30, + asMutableList(5), 50, asMutableList(6), 60, null, 40) + .inOrder(); + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/mutation/mutator/collection/ListMutatorTest.java b/src/test/java/com/code_intelligence/jazzer/mutation/mutator/collection/ListMutatorTest.java new file mode 100644 index 00000000..24299f48 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/mutation/mutator/collection/ListMutatorTest.java @@ -0,0 +1,280 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.code_intelligence.jazzer.mutation.mutator.collection; + +import static com.code_intelligence.jazzer.mutation.support.TestSupport.mockPseudoRandom; +import static com.google.common.truth.Truth.assertThat; +import static java.util.Collections.emptyList; + +import com.code_intelligence.jazzer.mutation.annotation.NotNull; +import com.code_intelligence.jazzer.mutation.annotation.WithSize; +import com.code_intelligence.jazzer.mutation.api.ChainedMutatorFactory; +import com.code_intelligence.jazzer.mutation.api.MutatorFactory; +import com.code_intelligence.jazzer.mutation.api.SerializingMutator; +import com.code_intelligence.jazzer.mutation.mutator.lang.LangMutators; +import com.code_intelligence.jazzer.mutation.support.TestSupport.MockPseudoRandom; +import com.code_intelligence.jazzer.mutation.support.TypeHolder; +import java.lang.reflect.AnnotatedType; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import org.junit.jupiter.api.Test; + +@SuppressWarnings("unchecked") +public class ListMutatorTest { + public static final MutatorFactory FACTORY = + new ChainedMutatorFactory(LangMutators.newFactory(), CollectionMutators.newFactory()); + + private static SerializingMutator<@NotNull List<@NotNull Integer>> defaultListMutator() { + AnnotatedType type = new TypeHolder<@NotNull List<@NotNull Integer>>() {}.annotatedType(); + return (SerializingMutator<@NotNull List<@NotNull Integer>>) FACTORY.createOrThrow(type); + } + + @Test + void testInit() { + SerializingMutator<@NotNull List<@NotNull Integer>> mutator = defaultListMutator(); + assertThat(mutator.toString()).isEqualTo("List<Integer>"); + + List<Integer> list; + try (MockPseudoRandom prng = mockPseudoRandom( + // targetSize + 1, + // elementMutator.init + 1)) { + list = mutator.init(prng); + } + assertThat(list).containsExactly(0); + } + + @Test + void testInitMaxSize() { + AnnotatedType type = + new TypeHolder<@NotNull @WithSize(min = 2, max = 3) List<@NotNull Integer>>(){} + .annotatedType(); + + SerializingMutator<@NotNull List<@NotNull Integer>> mutator = + (SerializingMutator<@NotNull List<@NotNull Integer>>) FACTORY.createOrThrow(type); + + assertThat(mutator.toString()).isEqualTo("List<Integer>"); + List<Integer> list; + try (MockPseudoRandom prng = mockPseudoRandom(2, 4, 42L, 4, 43L)) { + list = mutator.init(prng); + } + + assertThat(list).containsExactly(42, 43).inOrder(); + } + + @Test + void testRemoveSingleElement() { + SerializingMutator<@NotNull List<@NotNull Integer>> mutator = defaultListMutator(); + + List<Integer> list = new ArrayList<>(Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8, 9)); + try (MockPseudoRandom prng = mockPseudoRandom( + // action + 0, + // number of elements to remove + 1, + // index to remove + 2)) { + list = mutator.mutate(list, prng); + } + assertThat(list).containsExactly(1, 2, 4, 5, 6, 7, 8, 9).inOrder(); + } + + @Test + void testRemoveChunk() { + SerializingMutator<@NotNull List<@NotNull Integer>> mutator = defaultListMutator(); + + List<Integer> list = new ArrayList<>(Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8, 9)); + try (MockPseudoRandom prng = mockPseudoRandom( + // action + 0, + // chunk size + 2, + // chunk offset + 3)) { + list = mutator.mutate(list, prng); + } + assertThat(list).containsExactly(1, 2, 3, 6, 7, 8, 9).inOrder(); + } + + @Test + void testAddSingleElement() { + SerializingMutator<@NotNull List<@NotNull Integer>> mutator = defaultListMutator(); + + List<Integer> list = new ArrayList<>(Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8, 9)); + try (MockPseudoRandom prng = mockPseudoRandom( + // action + 1, + // add single element, + 1, + // offset, + 9, + // Integral initImpl sentinel value + 4, + // value + 42L)) { + list = mutator.mutate(list, prng); + } + assertThat(list).containsExactly(1, 2, 3, 4, 5, 6, 7, 8, 9, 42).inOrder(); + } + + @Test + void testAddChunk() { + SerializingMutator<@NotNull List<@NotNull Integer>> mutator = defaultListMutator(); + + List<Integer> list = new ArrayList<>(Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8, 9)); + try (MockPseudoRandom prng = mockPseudoRandom( + // action + 1, + // chunkSize + 2, + // chunkOffset + 3, + // Integral initImpl + 4, + // val + 42L)) { + list = mutator.mutate(list, prng); + } + assertThat(list).containsExactly(1, 2, 3, 42, 42, 4, 5, 6, 7, 8, 9).inOrder(); + } + + @Test + void testChangeSingleElement() { + SerializingMutator<@NotNull List<@NotNull Integer>> mutator = defaultListMutator(); + + List<Integer> list = new ArrayList<>(Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8, 9)); + try (MockPseudoRandom prng = mockPseudoRandom( + // action + 2, + // number of elements to mutate + 1, + // first index to mutate at + 2, + // mutation choice based on `IntegralMutatorFactory` + // 2 == closedRange + 2, + // value + 55L)) { + list = mutator.mutate(list, prng); + } + assertThat(list).containsExactly(1, 2, 55, 4, 5, 6, 7, 8, 9).inOrder(); + } + + @Test + void testChangeChunk() { + SerializingMutator<@NotNull List<@NotNull Integer>> mutator = defaultListMutator(); + + List<Integer> list = new ArrayList<>(Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11)); + try (MockPseudoRandom prng = mockPseudoRandom( + // action + 2, + // number of elements to mutate + 2, + // first index to mutate at + 5, + // mutation: 0 == bitflip + 0, + // shift constant + 13, + // and again + 0, 12)) { + list = mutator.mutate(list, prng); + } + assertThat(list).containsExactly(1, 2, 3, 4, 5, 8198, 4103, 8, 9, 10, 11).inOrder(); + } + + @Test + void testCrossOverEmptyLists() { + SerializingMutator<@NotNull List<@NotNull Integer>> mutator = defaultListMutator(); + + try (MockPseudoRandom prng = mockPseudoRandom()) { + List<Integer> list = mutator.crossOver(emptyList(), emptyList(), prng); + assertThat(list).isEmpty(); + } + } + + @Test + void testCrossOverInsertChunk() { + SerializingMutator<@NotNull List<@NotNull Integer>> mutator = defaultListMutator(); + + List<Integer> list = new ArrayList<>(Arrays.asList(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)); + List<Integer> otherList = + new ArrayList<>(Arrays.asList(10, 11, 12, 13, 14, 15, 16, 17, 18, 19)); + try (MockPseudoRandom prng = mockPseudoRandom( + // insert action + 0, + // chunk size + 3, + // fromPos + 2, + // toPos + 5)) { + list = mutator.crossOver(list, otherList, prng); + } + assertThat(list).containsExactly(0, 1, 2, 3, 4, 12, 13, 14, 5, 6, 7, 8, 9).inOrder(); + } + + @Test + void testCrossOverOverwriteChunk() { + SerializingMutator<@NotNull List<@NotNull Integer>> mutator = defaultListMutator(); + + List<Integer> list = new ArrayList<>(Arrays.asList(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)); + List<Integer> otherList = + new ArrayList<>(Arrays.asList(10, 11, 12, 13, 14, 15, 16, 17, 18, 19)); + try (MockPseudoRandom prng = mockPseudoRandom( + // overwrite action + 1, + // chunk size + 3, + // fromPos + 2, + // toPos + 5)) { + list = mutator.crossOver(list, otherList, prng); + } + assertThat(list).containsExactly(0, 1, 2, 3, 4, 12, 13, 14, 8, 9).inOrder(); + } + + @Test + void testCrossOverCrossOverChunk() { + SerializingMutator<@NotNull List<@NotNull Integer>> mutator = defaultListMutator(); + + List<Integer> list = new ArrayList<>(Arrays.asList(0, 1, 2, 3, 4, 5, 6, 7, 8, 9)); + List<Integer> otherList = + new ArrayList<>(Arrays.asList(10, 11, 12, 13, 14, 15, 16, 17, 18, 19)); + try (MockPseudoRandom prng = mockPseudoRandom( + // overwrite action + 2, + // chunk size + 3, + // fromPos + 2, + // toPos + 2, + // mean value in sub cross over + 0, + // mean value in sub cross over + 0, + // mean value in sub cross over + 0)) { + list = mutator.crossOver(list, otherList, prng); + } + assertThat(list).containsExactly(0, 1, 7, 8, 9, 5, 6, 7, 8, 9).inOrder(); + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/mutation/mutator/collection/MapMutatorTest.java b/src/test/java/com/code_intelligence/jazzer/mutation/mutator/collection/MapMutatorTest.java new file mode 100644 index 00000000..4c2c14f9 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/mutation/mutator/collection/MapMutatorTest.java @@ -0,0 +1,355 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.code_intelligence.jazzer.mutation.mutator.collection; + +import static com.code_intelligence.jazzer.mutation.support.TestSupport.asMap; +import static com.code_intelligence.jazzer.mutation.support.TestSupport.asMutableList; +import static com.code_intelligence.jazzer.mutation.support.TestSupport.mockPseudoRandom; +import static com.google.common.truth.Truth.assertThat; +import static java.util.Collections.emptyMap; + +import com.code_intelligence.jazzer.mutation.annotation.NotNull; +import com.code_intelligence.jazzer.mutation.annotation.WithSize; +import com.code_intelligence.jazzer.mutation.api.ChainedMutatorFactory; +import com.code_intelligence.jazzer.mutation.api.MutatorFactory; +import com.code_intelligence.jazzer.mutation.api.SerializingMutator; +import com.code_intelligence.jazzer.mutation.mutator.lang.LangMutators; +import com.code_intelligence.jazzer.mutation.support.TestSupport.MockPseudoRandom; +import com.code_intelligence.jazzer.mutation.support.TypeHolder; +import java.lang.reflect.AnnotatedType; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; + +@SuppressWarnings("unchecked") +class MapMutatorTest { + public static final MutatorFactory FACTORY = + new ChainedMutatorFactory(LangMutators.newFactory(), CollectionMutators.newFactory()); + + private static SerializingMutator<Map<Integer, Integer>> defaultTestMapMutator() { + AnnotatedType type = + new TypeHolder<@NotNull Map<@NotNull Integer, @NotNull Integer>>() {}.annotatedType(); + return (SerializingMutator<Map<Integer, Integer>>) FACTORY.createOrThrow(type); + } + + @Test + void mapInitInsert() { + AnnotatedType type = + new TypeHolder<@NotNull @WithSize(max = 3) Map<@NotNull String, @NotNull String>>(){} + .annotatedType(); + SerializingMutator<Map<String, String>> mutator = + (SerializingMutator<Map<String, String>>) FACTORY.createOrThrow(type); + assertThat(mutator.toString()).isEqualTo("Map<String,String>"); + + // Initialize new map + Map<String, String> map; + try (MockPseudoRandom prng = mockPseudoRandom( + // Initial map size + 1, + // Key 1 size + 4, + // Key 1 value + "Key1".getBytes(), + // Value size + 6, + // Value value + "Value1".getBytes())) { + map = mutator.init(prng); + } + assertThat(map).containsExactly("Key1", "Value1"); + + // Add 2 new entries + try (MockPseudoRandom prng = mockPseudoRandom( + // grow chunk + 1, + // ChunkSize + 2, + // Key 2 size + 4, + // Key 2 value + "Key2".getBytes(), + // Value size + 6, + // Value value + "Value2".getBytes(), + // Key 3 size + 4, + // Key 3 value + "Key3".getBytes(), + // Value size + 6, + // Value value + "Value3".getBytes())) { + map = mutator.mutate(map, prng); + } + assertThat(map).containsExactly("Key1", "Value1", "Key2", "Value2", "Key3", "Value3").inOrder(); + } + + @Test + void mapDelete() { + AnnotatedType type = + new TypeHolder<@NotNull Map<@NotNull Integer, @NotNull Integer>>() {}.annotatedType(); + SerializingMutator<Map<Integer, Integer>> mutator = + (SerializingMutator<Map<Integer, Integer>>) FACTORY.createOrThrow(type); + assertThat(mutator.toString()).isEqualTo("Map<Integer,Integer>"); + + Map<Integer, Integer> map = asMap(1, 10, 2, 20, 3, 30, 4, 40, 5, 50, 6, 60); + + try (MockPseudoRandom prng = mockPseudoRandom( + // delete chunk + 0, + // chunk size + 2, + // chunk position + 3)) { + map = mutator.mutate(map, prng); + } + assertThat(map).containsExactly(1, 10, 2, 20, 3, 30, 6, 60).inOrder(); + } + + @Test + void mapMutateValues() { + AnnotatedType type = + new TypeHolder<@NotNull Map<@NotNull Integer, @NotNull Integer>>() {}.annotatedType(); + SerializingMutator<Map<Integer, Integer>> mutator = + (SerializingMutator<Map<Integer, Integer>>) FACTORY.createOrThrow(type); + assertThat(mutator.toString()).isEqualTo("Map<Integer,Integer>"); + + Map<Integer, Integer> map = asMap(1, 10, 2, 20, 3, 30, 4, 40, 5, 50, 6, 60); + + try (MockPseudoRandom prng = mockPseudoRandom( + // change chunk + 2, + // mutate values, + true, + // chunk size + 2, + // chunk position + 3, + // uniform pick + 2, + // random integer + 41L, + // uniform pick + 2, + // random integer + 51L)) { + map = mutator.mutate(map, prng); + } + assertThat(map).containsExactly(1, 10, 2, 20, 3, 30, 4, 41, 5, 51, 6, 60).inOrder(); + } + + @Test + void mapMutateKeys() { + AnnotatedType type = + new TypeHolder<@NotNull Map<@NotNull Integer, @NotNull Integer>>() {}.annotatedType(); + SerializingMutator<Map<Integer, Integer>> mutator = + (SerializingMutator<Map<Integer, Integer>>) FACTORY.createOrThrow(type); + assertThat(mutator.toString()).isEqualTo("Map<Integer,Integer>"); + + Map<Integer, Integer> map = asMap(1, 10, 2, 20, 3, 30, 4, 40, 5, 50, 6, 60); + + try (MockPseudoRandom prng = mockPseudoRandom( + // change chunk + 2, + // mutate keys, + false, + // chunk size + 2, + // chunk position + 3, + // uniform pick + 2, + // integer + 7L, + // uniform pick + 2, + // random integer + 8L)) { + map = mutator.mutate(map, prng); + } + assertThat(map).containsExactly(1, 10, 2, 20, 3, 30, 6, 60, 7, 40, 8, 50).inOrder(); + } + + @Test + void mapMutateKeysFallbackToValues() { + AnnotatedType type = + new TypeHolder<@NotNull Map<@NotNull Boolean, @NotNull Boolean>>() {}.annotatedType(); + SerializingMutator<Map<Boolean, Boolean>> mutator = + (SerializingMutator<Map<Boolean, Boolean>>) FACTORY.createOrThrow(type); + assertThat(mutator.toString()).isEqualTo("Map<Boolean,Boolean>"); + + // No new keys can be generated for this map. + Map<Boolean, Boolean> map = asMap(false, false, true, false); + + try (MockPseudoRandom prng = mockPseudoRandom( + // change chunk + 2, + // mutate keys, + false, + // chunk size + 1, + // chunk position + 0, + // chunk size for fallback to mutate values + 2, + // chunk position for fallback + 0)) { + map = mutator.mutate(map, prng); + } + assertThat(map).containsExactly(false, true, true, true).inOrder(); + } + + @Test + void testCrossOverEmptyMaps() { + SerializingMutator<@NotNull Map<@NotNull Integer, @NotNull Integer>> mutator = + defaultTestMapMutator(); + + try (MockPseudoRandom prng = mockPseudoRandom()) { + Map<Integer, Integer> map = mutator.crossOver(emptyMap(), emptyMap(), prng); + assertThat(map).isEmpty(); + } + } + + @Test + void testCrossOverInsertChunk() { + SerializingMutator<@NotNull Map<@NotNull Integer, @NotNull Integer>> mutator = + defaultTestMapMutator(); + + Map<Integer, Integer> map = asMap(1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6); + Map<Integer, Integer> otherMap = asMap(1, 1, 2, 2, 3, 3, 40, 40, 50, 50, 60, 60); + + try (MockPseudoRandom prng = mockPseudoRandom( + // insert action + 0, + // chunk size + 3, + // from chunk offset, will skip first element of chunk as it is already present in map + 3, + // to chunk offset, unused + 0)) { + map = mutator.crossOver(map, otherMap, prng); + assertThat(map) + .containsExactly(1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 40, 40, 50, 50, 60, 60) + .inOrder(); + } + } + + @Test + void testCrossOverOverwriteChunk() { + SerializingMutator<@NotNull Map<@NotNull Integer, @NotNull Integer>> mutator = + defaultTestMapMutator(); + + Map<Integer, Integer> map = asMap(1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6); + Map<Integer, Integer> otherMap = asMap(1, 1, 2, 2, 3, 3, 40, 40, 50, 50, 60, 60); + + try (MockPseudoRandom prng = mockPseudoRandom( + // overwrite action + 1, + // chunk size + 3, + // from chunk offset + 2, + // to chunk offset, will not change first element as values are equal + 2)) { + map = mutator.crossOver(map, otherMap, prng); + assertThat(map).containsExactly(1, 1, 2, 2, 3, 3, 4, 40, 5, 50, 6, 6).inOrder(); + } + } + + @Test + void testCrossOverCrossOverChunkKeys() { + AnnotatedType type = + new TypeHolder<@NotNull Map<@NotNull List<@NotNull Integer>, @NotNull Integer>>() { + }.annotatedType(); + SerializingMutator<@NotNull Map<@NotNull List<@NotNull Integer>, @NotNull Integer>> mutator = + (SerializingMutator<@NotNull Map<@NotNull List<@NotNull Integer>, @NotNull Integer>>) + FACTORY.createOrThrow(type); + + Map<List<Integer>, Integer> map = asMap(asMutableList(1), 1, asMutableList(2), 2, + asMutableList(3), 3, asMutableList(4), 4, asMutableList(5), 5, asMutableList(6), 6); + Map<List<Integer>, Integer> otherMap = asMap(asMutableList(1), 1, asMutableList(2), 2, + asMutableList(3), 3, asMutableList(40), 4, asMutableList(50), 5, asMutableList(60), 6); + + try (MockPseudoRandom prng = mockPseudoRandom( + // cross over action + 2, + // keys + true, + // chunk size + 3, + // from chunk offset + 2, + // to chunk offset, + // first keys ("3") are equal and will be overwritten + 2, + // first key, delegate to list cross over, overwrite 1 entry at offset 0 from offset 0 + 1, 1, 0, 0, + // second key, delegate to list cross over, overwrite 1 entry at offset 0 from offset 0 + 1, 1, 0, 0, + // third key, delegate to list cross over, overwrite 1 entry at offset 0 from offset 0 + 1, 1, 0, 0)) { + map = mutator.crossOver(map, otherMap, prng); + assertThat(map) + .containsExactly(asMutableList(1), 1, asMutableList(2), 2, asMutableList(6), 6, + // Overwritten keys after here + asMutableList(3), 3, asMutableList(40), 4, asMutableList(50), 5) + .inOrder(); + } + } + + @Test + void testCrossOverCrossOverChunkValues() { + AnnotatedType type = + new TypeHolder<@NotNull Map<@NotNull Integer, @NotNull List<@NotNull Integer>>>() { + }.annotatedType(); + SerializingMutator<@NotNull Map<@NotNull Integer, @NotNull List<@NotNull Integer>>> mutator = + (SerializingMutator<@NotNull Map<@NotNull Integer, @NotNull List<@NotNull Integer>>>) + FACTORY.createOrThrow(type); + + Map<Integer, List<Integer>> map = asMap(1, asMutableList(1), 2, asMutableList(2), 3, + asMutableList(3), 4, asMutableList(4), 5, asMutableList(5), 6, asMutableList(6)); + Map<Integer, List<Integer>> otherMap = asMap(1, asMutableList(1), 2, asMutableList(2), 3, + asMutableList(30), 40, asMutableList(40), 50, asMutableList(50), 60, asMutableList(60)); + + try ( + MockPseudoRandom prng = mockPseudoRandom( + // cross over action + 2, + // values + false, + // chunk size + 3, + // from chunk offset + 2, + // to chunk offset, + 2, + // first value, delegate to list cross over, overwrite 1 entry at offset 0 from offset 0 + 1, 1, 0, 0, + // second value, delegate to list cross over, overwrite 1 entry at offset 0 from offset + // 0 + 1, 1, 0, 0, + // third value, delegate to list cross over, overwrite 1 entry at offset 0 from offset 0 + 1, 1, 0, 0)) { + map = mutator.crossOver(map, otherMap, prng); + assertThat(map) + .containsExactly(1, asMutableList(1), 2, asMutableList(2), 3, asMutableList(30), 4, + asMutableList(40), 5, asMutableList(50), 6, asMutableList(6)) + .inOrder(); + } + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/mutation/mutator/lang/BUILD.bazel b/src/test/java/com/code_intelligence/jazzer/mutation/mutator/lang/BUILD.bazel new file mode 100644 index 00000000..05e1d720 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/mutation/mutator/lang/BUILD.bazel @@ -0,0 +1,18 @@ +load("@contrib_rules_jvm//java:defs.bzl", "java_test_suite") + +java_test_suite( + name = "PrimitiveTests", + size = "small", + srcs = glob(["*.java"]), + env = {"JAZZER_MOCK_LIBFUZZER_MUTATOR": "true"}, + runner = "junit5", + deps = [ + "//src/main/java/com/code_intelligence/jazzer/mutation/annotation", + "//src/main/java/com/code_intelligence/jazzer/mutation/api", + "//src/main/java/com/code_intelligence/jazzer/mutation/mutator/lang", + "//src/main/java/com/code_intelligence/jazzer/mutation/mutator/libfuzzer", + "//src/main/java/com/code_intelligence/jazzer/mutation/support", + "//src/test/java/com/code_intelligence/jazzer/mutation/support:test_support", + "@com_google_protobuf//java/core", + ], +) diff --git a/src/test/java/com/code_intelligence/jazzer/mutation/mutator/lang/BooleanMutatorTest.java b/src/test/java/com/code_intelligence/jazzer/mutation/mutator/lang/BooleanMutatorTest.java new file mode 100644 index 00000000..3bf55bcf --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/mutation/mutator/lang/BooleanMutatorTest.java @@ -0,0 +1,74 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.code_intelligence.jazzer.mutation.mutator.lang; + +import static com.code_intelligence.jazzer.mutation.support.TestSupport.mockPseudoRandom; +import static com.google.common.truth.Truth.assertThat; + +import com.code_intelligence.jazzer.mutation.annotation.NotNull; +import com.code_intelligence.jazzer.mutation.api.SerializingMutator; +import com.code_intelligence.jazzer.mutation.support.TestSupport.MockPseudoRandom; +import com.code_intelligence.jazzer.mutation.support.TypeHolder; +import org.junit.jupiter.api.Test; + +@SuppressWarnings("unchecked") +class BooleanMutatorTest { + @Test + void testPrimitive() { + SerializingMutator<Boolean> mutator = LangMutators.newFactory().createOrThrow(boolean.class); + assertThat(mutator.toString()).isEqualTo("Boolean"); + + boolean bool; + try (MockPseudoRandom prng = mockPseudoRandom(true)) { + bool = mutator.init(prng); + } + assertThat(bool).isTrue(); + + try (MockPseudoRandom prng = mockPseudoRandom()) { + bool = mutator.mutate(bool, prng); + } + assertThat(bool).isFalse(); + } + + @Test + void testBoxed() { + SerializingMutator<Boolean> mutator = + (SerializingMutator<Boolean>) LangMutators.newFactory().createOrThrow( + new TypeHolder<@NotNull Boolean>() {}.annotatedType()); + assertThat(mutator.toString()).isEqualTo("Boolean"); + + Boolean bool; + try (MockPseudoRandom prng = mockPseudoRandom(false)) { + bool = mutator.init(prng); + } + assertThat(bool).isFalse(); + + try (MockPseudoRandom prng = mockPseudoRandom()) { + bool = mutator.mutate(bool, prng); + } + assertThat(bool).isTrue(); + } + + @Test + void testCrossOver() { + SerializingMutator<Boolean> mutator = LangMutators.newFactory().createOrThrow(boolean.class); + try (MockPseudoRandom prng = mockPseudoRandom(true, false)) { + assertThat(mutator.crossOver(true, false, prng)).isTrue(); + assertThat(mutator.crossOver(true, false, prng)).isFalse(); + } + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/mutation/mutator/lang/ByteArrayMutatorTest.java b/src/test/java/com/code_intelligence/jazzer/mutation/mutator/lang/ByteArrayMutatorTest.java new file mode 100644 index 00000000..1592b17d --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/mutation/mutator/lang/ByteArrayMutatorTest.java @@ -0,0 +1,189 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.code_intelligence.jazzer.mutation.mutator.lang; + +import static com.code_intelligence.jazzer.mutation.support.TestSupport.mockPseudoRandom; +import static com.google.common.truth.Truth.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.code_intelligence.jazzer.mutation.annotation.NotNull; +import com.code_intelligence.jazzer.mutation.annotation.WithLength; +import com.code_intelligence.jazzer.mutation.api.SerializingMutator; +import com.code_intelligence.jazzer.mutation.mutator.libfuzzer.LibFuzzerMutator; +import com.code_intelligence.jazzer.mutation.support.TestSupport.MockPseudoRandom; +import com.code_intelligence.jazzer.mutation.support.TypeHolder; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +@SuppressWarnings({"unchecked", "ResultOfMethodCallIgnored"}) +public class ByteArrayMutatorTest { + /** + * Some tests may set {@link LibFuzzerMutator#MOCK_SIZE_KEY} which can interfere with other tests + * unless cleared. + */ + @AfterEach + void cleanMockSize() { + System.clearProperty(LibFuzzerMutator.MOCK_SIZE_KEY); + } + + @Test + void testBasicFunction() { + SerializingMutator<byte[]> mutator = + (SerializingMutator<byte[]>) LangMutators.newFactory().createOrThrow( + new TypeHolder<byte[]>() {}.annotatedType()); + assertThat(mutator.toString()).isEqualTo("Nullable<byte[]>"); + + byte[] arr; + try (MockPseudoRandom prng = mockPseudoRandom(false, 5, new byte[] {1, 2, 3, 4, 5})) { + arr = mutator.init(prng); + } + assertThat(arr).isEqualTo(new byte[] {1, 2, 3, 4, 5}); + + System.setProperty(LibFuzzerMutator.MOCK_SIZE_KEY, "10"); + try (MockPseudoRandom prng = mockPseudoRandom(false)) { + arr = mutator.mutate(arr, prng); + } + assertThat(arr).isEqualTo(new byte[] {2, 4, 6, 8, 10, 6, 7, 8, 9, 10}); + } + + @Test + void testMaxLength() { + SerializingMutator<byte[]> mutator = + (SerializingMutator<byte[]>) LangMutators.newFactory().createOrThrow( + new TypeHolder<byte @NotNull @WithLength(max = 10)[]>() {}.annotatedType()); + assertThat(mutator.toString()).isEqualTo("byte[]"); + + byte[] arr; + try (MockPseudoRandom prng = mockPseudoRandom(8, new byte[] {1, 2, 3, 4, 5, 6, 7, 8})) { + arr = mutator.init(prng); + } + assertThat(arr).isEqualTo(new byte[] {1, 2, 3, 4, 5, 6, 7, 8}); + + System.setProperty(LibFuzzerMutator.MOCK_SIZE_KEY, "11"); + try (MockPseudoRandom prng = mockPseudoRandom()) { + // the ByteArrayMutator will limit the maximum size of the data requested from libfuzzer to + // WithLength::max so setting the mock mutator to make it bigger will cause an exception + assertThrows(ArrayIndexOutOfBoundsException.class, () -> { mutator.mutate(arr, prng); }); + } + } + + @Test + void testMaxLengthInitClamp() { + SerializingMutator<byte[]> mutator = + (SerializingMutator<byte[]>) LangMutators.newFactory().createOrThrow( + new TypeHolder<byte @NotNull @WithLength(max = 5)[]>() {}.annotatedType()); + assertThat(mutator.toString()).isEqualTo("byte[]"); + + try (MockPseudoRandom prng = mockPseudoRandom(10)) { + // init will call closedRange(min, max) and the mock prng will assert that the given value + // above is between those values which we want to fail here to show that we're properly + // clamping the range + assertThrows(AssertionError.class, () -> { mutator.init(prng); }); + } + } + + @Test + void testMinLengthInitClamp() { + SerializingMutator<byte[]> mutator = + (SerializingMutator<byte[]>) LangMutators.newFactory().createOrThrow( + new TypeHolder<byte @NotNull @WithLength(min = 5)[]>() {}.annotatedType()); + assertThat(mutator.toString()).isEqualTo("byte[]"); + + try (MockPseudoRandom prng = mockPseudoRandom(3)) { + // init will call closedrange(min, max) and the mock prng will assert that the given value + // above is between those values which we want to fail here to show that we're properly + // clamping the range + assertThrows(AssertionError.class, () -> { mutator.init(prng); }); + } + } + + @Test + void testMinLength() { + SerializingMutator<byte[]> mutator = + (SerializingMutator<byte[]>) LangMutators.newFactory().createOrThrow( + new TypeHolder<byte @NotNull @WithLength(min = 5)[]>() {}.annotatedType()); + assertThat(mutator.toString()).isEqualTo("byte[]"); + + byte[] arr; + try (MockPseudoRandom prng = mockPseudoRandom(10, new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 10})) { + arr = mutator.init(prng); + } + assertThat(arr).hasLength(10); + + System.setProperty(LibFuzzerMutator.MOCK_SIZE_KEY, "3"); + + try (MockPseudoRandom prng = mockPseudoRandom()) { + arr = mutator.mutate(arr, prng); + } + assertThat(arr).hasLength(5); + assertThat(arr).isEqualTo(new byte[] {2, 4, 6, 0, 0}); + } + + @Test + void testCrossOver() { + SerializingMutator<byte[]> mutator = + (SerializingMutator<byte[]>) LangMutators.newFactory().createOrThrow( + new TypeHolder<byte @NotNull[]>() {}.annotatedType()); + assertThat(mutator.toString()).isEqualTo("byte[]"); + + byte[] value = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + byte[] otherValue = {10, 11, 12, 13, 14, 15, 16, 17, 18, 19}; + + byte[] crossedOver; + try (MockPseudoRandom prng = mockPseudoRandom( + // intersect arrays + 0, + // out length + 8, + // copy 3 from first + 3, + // copy 1 from second + 1, + // copy 1 from first, + 1, + // copy 3 from second + 3)) { + crossedOver = mutator.crossOver(value, otherValue, prng); + assertThat(crossedOver).isEqualTo(new byte[] {0, 1, 2, 10, 3, 11, 12, 13}); + } + + try (MockPseudoRandom prng = mockPseudoRandom( + // insert into action + 1, + // copy size + 3, + // from position + 5, + // to position + 2)) { + crossedOver = mutator.crossOver(value, otherValue, prng); + assertThat(crossedOver).isEqualTo(new byte[] {0, 1, 15, 16, 17, 2, 3, 4, 5, 6, 7, 8, 9}); + } + + try (MockPseudoRandom prng = mockPseudoRandom( + // overwrite action + 2, + // to position + 3, + // copy size + 3, + // from position + 4)) { + crossedOver = mutator.crossOver(value, otherValue, prng); + assertThat(crossedOver).isEqualTo(new byte[] {0, 1, 2, 14, 15, 16, 6, 7, 8, 9}); + } + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/mutation/mutator/lang/EnumMutatorTest.java b/src/test/java/com/code_intelligence/jazzer/mutation/mutator/lang/EnumMutatorTest.java new file mode 100644 index 00000000..d2c61397 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/mutation/mutator/lang/EnumMutatorTest.java @@ -0,0 +1,103 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.code_intelligence.jazzer.mutation.mutator.lang; + +import static com.code_intelligence.jazzer.mutation.support.TestSupport.mockPseudoRandom; +import static com.google.common.truth.Truth.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.code_intelligence.jazzer.mutation.annotation.NotNull; +import com.code_intelligence.jazzer.mutation.api.SerializingMutator; +import com.code_intelligence.jazzer.mutation.support.TestSupport.MockPseudoRandom; +import com.code_intelligence.jazzer.mutation.support.TypeHolder; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import org.junit.jupiter.api.Test; + +class EnumMutatorTest { + enum TestEnumOne { A } + + enum TestEnum { A, B, C } + + @Test + void testBoxed() { + SerializingMutator<TestEnum> mutator = + (SerializingMutator<TestEnum>) LangMutators.newFactory().createOrThrow( + new TypeHolder<@NotNull TestEnum>() {}.annotatedType()); + assertThat(mutator.toString()).isEqualTo("Enum<TestEnum>"); + TestEnum cl; + try (MockPseudoRandom prng = mockPseudoRandom(0)) { + cl = mutator.init(prng); + } + assertThat(cl).isEqualTo(TestEnum.A); + + try (MockPseudoRandom prng = mockPseudoRandom(1)) { + cl = mutator.mutate(cl, prng); + } + assertThat(cl).isEqualTo(TestEnum.B); + + try (MockPseudoRandom prng = mockPseudoRandom(0)) { + cl = mutator.mutate(cl, prng); + } + assertThat(cl).isEqualTo(TestEnum.A); + + try (MockPseudoRandom prng = mockPseudoRandom(2)) { + cl = mutator.mutate(cl, prng); + } + assertThat(cl).isEqualTo(TestEnum.C); + + try (MockPseudoRandom prng = mockPseudoRandom(1)) { + cl = mutator.mutate(cl, prng); + } + assertThat(cl).isEqualTo(TestEnum.B); + } + + @Test + void testEnumWithOneElementShouldThrow() { + assertThrows(IllegalArgumentException.class, () -> { + LangMutators.newFactory().createOrThrow( + new TypeHolder<@NotNull TestEnumOne>() {}.annotatedType()); + }, "When trying to build mutators for Enum with one value, an Exception should be thrown."); + } + + @Test + void testEnumBasedOnInvalidInput() throws IOException { + SerializingMutator<TestEnum> mutator = + (SerializingMutator<TestEnum>) LangMutators.newFactory().createOrThrow( + new TypeHolder<@NotNull TestEnum>() {}.annotatedType()); + ByteArrayOutputStream bo = new ByteArrayOutputStream(); + DataOutputStream os = new DataOutputStream(bo); + // Valid values + os.writeInt(0); + os.writeInt(1); + os.writeInt(2); + // Too high indices wrap around + os.writeInt(3); + // Abs. value is used to calculate the index + os.writeInt(-3); + + DataInputStream is = new DataInputStream(new ByteArrayInputStream(bo.toByteArray())); + assertThat(mutator.read(is)).isEqualTo(TestEnum.A); + assertThat(mutator.read(is)).isEqualTo(TestEnum.B); + assertThat(mutator.read(is)).isEqualTo(TestEnum.C); + assertThat(mutator.read(is)).isEqualTo(TestEnum.A); + assertThat(mutator.read(is)).isEqualTo(TestEnum.A); + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/mutation/mutator/lang/FloatingPointMutatorTest.java b/src/test/java/com/code_intelligence/jazzer/mutation/mutator/lang/FloatingPointMutatorTest.java new file mode 100644 index 00000000..9c03b467 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/mutation/mutator/lang/FloatingPointMutatorTest.java @@ -0,0 +1,785 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.code_intelligence.jazzer.mutation.mutator.lang; + +import static com.code_intelligence.jazzer.mutation.mutator.lang.FloatingPointMutatorFactory.DoubleMutator; +import static com.code_intelligence.jazzer.mutation.mutator.lang.FloatingPointMutatorFactory.FloatMutator; +import static com.code_intelligence.jazzer.mutation.support.TestSupport.mockPseudoRandom; +import static com.google.common.truth.Truth.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.params.provider.Arguments.arguments; + +import com.code_intelligence.jazzer.mutation.annotation.DoubleInRange; +import com.code_intelligence.jazzer.mutation.annotation.FloatInRange; +import com.code_intelligence.jazzer.mutation.annotation.NotNull; +import com.code_intelligence.jazzer.mutation.api.SerializingMutator; +import com.code_intelligence.jazzer.mutation.support.TestSupport; +import com.code_intelligence.jazzer.mutation.support.TypeHolder; +import java.util.function.Supplier; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +class FloatingPointMutatorTest { + static final Float UNUSED_FLOAT = 0.0f; + static final Double UNUSED_DOUBLE = 0.0; + + static Stream<Arguments> floatForceInRangeCases() { + float NaN1 = Float.intBitsToFloat(0x7f800001); + float NaN2 = Float.intBitsToFloat(0x7f800002); + float NaN3 = Float.intBitsToFloat(0x7f800003); + assertThat(Float.isNaN(NaN1) && Float.isNaN(NaN2) && Float.isNaN(NaN3)).isTrue(); + + return Stream.of( + // value is already in range: it should stay in range + arguments(0.0f, 0.0f, 1.0f, true), arguments(0.0f, 1.0f, 1.0f, true), + arguments(Float.NEGATIVE_INFINITY, Float.NEGATIVE_INFINITY, 1.0f, true), + arguments(Float.POSITIVE_INFINITY, Float.NEGATIVE_INFINITY, Float.POSITIVE_INFINITY, true), + arguments(Float.NaN, 0.0f, 1.0f, true), + arguments(1e30f, -Float.MAX_VALUE, Float.MAX_VALUE, true), + arguments(-1e30f, -Float.MAX_VALUE, Float.MAX_VALUE, true), + arguments(0.0f, Float.NEGATIVE_INFINITY, Float.MAX_VALUE, true), + arguments(0.0f, Float.NEGATIVE_INFINITY, Float.POSITIVE_INFINITY, true), + arguments(-Float.MAX_VALUE, -Float.MAX_VALUE, Float.MAX_VALUE, true), + arguments(Float.MAX_VALUE, -Float.MAX_VALUE, Float.MAX_VALUE, true), + arguments(-Float.MAX_VALUE, Float.MAX_VALUE - 3.4e30f, Float.MAX_VALUE, false), + arguments(Float.MAX_VALUE, -100.0f, Float.MAX_VALUE, true), + arguments(0.0f, -Float.MIN_VALUE, Float.MIN_VALUE, true), + // Special values and diff/ranges outside the range + arguments(Float.NEGATIVE_INFINITY, -1.0f, 1.0f, true), + arguments(Float.POSITIVE_INFINITY, -1.0f, 1.0f, true), + arguments(Float.POSITIVE_INFINITY, -Float.MAX_VALUE, Float.MAX_VALUE, true), + arguments(Float.POSITIVE_INFINITY, Float.NEGATIVE_INFINITY, Float.MAX_VALUE, true), + arguments(Float.POSITIVE_INFINITY, Float.NEGATIVE_INFINITY, -Float.MAX_VALUE, true), + arguments(Float.NEGATIVE_INFINITY, -Float.MAX_VALUE, Float.MAX_VALUE, true), + arguments(Float.NEGATIVE_INFINITY, -Float.MAX_VALUE, Float.POSITIVE_INFINITY, true), + arguments(Float.NEGATIVE_INFINITY, Float.MAX_VALUE, Float.POSITIVE_INFINITY, true), + // Values outside the range + arguments(-2e30f, -100000.0f, 100000.0f, true), + arguments(2e30f, Float.NEGATIVE_INFINITY, -Float.MAX_VALUE, true), + arguments(-1.0f, 0.0f, 1.0f, false), arguments(5.0f, 0.0f, 1.0f, false), + arguments(-Float.MAX_VALUE, -Float.MAX_VALUE, 100.0f, true), + // NaN not allowed + arguments(Float.NaN, 0.0f, 1.0f, false), + arguments(Float.NaN, -Float.MAX_VALUE, 1.0f, false), + arguments(Float.NaN, Float.NEGATIVE_INFINITY, 1.0f, false), + arguments(Float.NaN, Float.NEGATIVE_INFINITY, Float.POSITIVE_INFINITY, false), + arguments(Float.NaN, 0f, Float.POSITIVE_INFINITY, false), + arguments(Float.NaN, 0f, Float.MAX_VALUE, false), + arguments(Float.NaN, -Float.MAX_VALUE, Float.MAX_VALUE, false), + arguments(Float.NaN, -Float.MIN_VALUE, 0.0f, false), + arguments(Float.NaN, -Float.MIN_VALUE, Float.MIN_VALUE, false), + arguments(Float.NaN, 0.0f, Float.MIN_VALUE, false), + // There are many possible NaN values, test a few of them that are different from Float.NaN + // (0x7fc00000) + arguments(NaN1, 0.0f, 1.0f, false), arguments(NaN2, 0.0f, 1.0f, false), + arguments(NaN3, 0.0f, 1.0f, false)); + } + + static Stream<Arguments> doubleForceInRangeCases() { + double NaN1 = Double.longBitsToDouble(0x7ff0000000000001L); + double NaN2 = Double.longBitsToDouble(0x7ff0000000000002L); + double NaN3 = Double.longBitsToDouble(0x7ff0000000000003L); + double NaNdeadbeef = Double.longBitsToDouble(0x7ff00000deadbeefL); + assertThat( + Double.isNaN(NaN1) && Double.isNaN(NaN2) && Double.isNaN(NaN3) && Double.isNaN(NaNdeadbeef)) + .isTrue(); + + return Stream.of( + // value is already in range: it should stay in range + arguments(0.0, 0.0, 1.0, true), arguments(0.0, 1.0, 1.0, true), + arguments(Double.NEGATIVE_INFINITY, Double.NEGATIVE_INFINITY, 1.0, true), + arguments( + Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, true), + arguments(Double.NaN, 0.0, 1.0, true), + arguments(1e30, -Double.MAX_VALUE, Double.MAX_VALUE, true), + arguments(-1e30, -Double.MAX_VALUE, Double.MAX_VALUE, true), + arguments(0.0, Double.NEGATIVE_INFINITY, Double.MAX_VALUE, true), + arguments(0.0, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, true), + arguments(-Double.MAX_VALUE, -Double.MAX_VALUE, Double.MAX_VALUE, true), + arguments(Double.MAX_VALUE, -Double.MAX_VALUE, Double.MAX_VALUE, true), + arguments(-Double.MAX_VALUE, Double.MAX_VALUE - 3.4e30, Double.MAX_VALUE, false), + arguments(Double.MAX_VALUE, -100.0, Double.MAX_VALUE, true), + arguments(0.0, -Double.MIN_VALUE, Double.MIN_VALUE, true), + // Special values and diff/ranges outside the range + arguments(Double.NEGATIVE_INFINITY, -1.0, 1.0, true), + arguments(Double.POSITIVE_INFINITY, -1.0, 1.0, true), + arguments(Double.POSITIVE_INFINITY, -Double.MAX_VALUE, Double.MAX_VALUE, true), + arguments(Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY, Double.MAX_VALUE, true), + arguments(Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY, -Double.MAX_VALUE, true), + arguments(Double.NEGATIVE_INFINITY, -Double.MAX_VALUE, Double.MAX_VALUE, true), + arguments(Double.NEGATIVE_INFINITY, -Double.MAX_VALUE, Double.POSITIVE_INFINITY, true), + arguments(Double.NEGATIVE_INFINITY, Double.MAX_VALUE, Double.POSITIVE_INFINITY, true), + // Values outside the range + arguments(-2e30, -100000.0, 100000.0, true), + arguments(2e30, Double.NEGATIVE_INFINITY, -Double.MAX_VALUE, true), + arguments(-1.0, 0.0, 1.0, false), arguments(5.0, 0.0, 1.0, false), + arguments(-Double.MAX_VALUE, -Double.MAX_VALUE, 100.0, true), + arguments( + Math.nextDown(Double.MAX_VALUE), -Double.MAX_VALUE * 0.5, Double.MAX_VALUE * 0.5, true), + arguments(Math.nextDown(Double.MAX_VALUE), -Double.MAX_VALUE * 0.5, + Math.nextUp(Double.MAX_VALUE * 0.5), true), + // NaN not allowed + arguments(Double.NaN, 0.0, 1.0, false), + arguments(Double.NaN, -Double.MAX_VALUE, 1.0, false), + arguments(Double.NaN, Double.NEGATIVE_INFINITY, 1.0, false), + arguments(Double.NaN, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, false), + arguments(Double.NaN, 0, Double.POSITIVE_INFINITY, false), + arguments(Double.NaN, 0, Double.MAX_VALUE, false), + arguments(Double.NaN, -Double.MAX_VALUE, Double.MAX_VALUE, false), + arguments(Double.NaN, -Double.MIN_VALUE, 0.0, false), + arguments(Double.NaN, -Double.MIN_VALUE, Double.MIN_VALUE, false), + arguments(Double.NaN, 0.0, Double.MIN_VALUE, false), + // There are many possible NaN values, test a few of them that are different from Double.NaN + // (0x7ff8000000000000L) + arguments(NaN1, 0.0, 1.0, false), arguments(NaN2, 0.0, 1.0, false), + arguments(NaN3, 0.0, 1.0, false), + arguments(NaNdeadbeef, Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, false)); + } + + @ParameterizedTest + @MethodSource("floatForceInRangeCases") + void testFloatForceInRange(float value, float minValue, float maxValue, boolean allowNaN) { + float inRange = FloatMutator.forceInRange(value, minValue, maxValue, allowNaN); + + // inRange can become NaN only if allowNaN is true and value was NaN already + if (Float.isNaN(inRange)) { + if (allowNaN) { + assertThat(Float.isNaN(value)).isTrue(); + return; // NaN is not in range of anything + } else { + throw new AssertionError("NaN is not allowed but was returned"); + } + } + + assertThat(inRange).isAtLeast(minValue); + assertThat(inRange).isAtMost(maxValue); + if (value >= minValue && value <= maxValue) { + assertThat(inRange).isEqualTo(value); + } + } + + @ParameterizedTest + @MethodSource("doubleForceInRangeCases") + void testDoubleForceInRange(double value, double minValue, double maxValue, boolean allowNaN) { + double inRange = DoubleMutator.forceInRange(value, minValue, maxValue, allowNaN); + + // inRange can become NaN only if allowNaN is true and value was NaN already + if (Double.isNaN(inRange)) { + if (allowNaN) { + assertThat(Double.isNaN(value)).isTrue(); + return; // NaN is not in range of anything + } else { + throw new AssertionError("NaN is not allowed but was returned"); + } + } + + assertThat(inRange).isAtLeast(minValue); + assertThat(inRange).isAtMost(maxValue); + if (value >= minValue && value <= maxValue) { + assertThat(inRange).isEqualTo(value); + } + } + + // Tests of mutators' special values after initialization use mocked PRNG to test one special + // value after another. This counter enables adding new special values and testcases for them + // without modifying all the other test cases. + static Supplier<Integer> makeCounter() { + return new Supplier<Integer>() { + private int counter = 0; + + @Override + public Integer get() { + return counter++; + } + }; + } + + static Stream<Arguments> floatInitCasesFullRange() { + SerializingMutator<Float> mutator = + (SerializingMutator<Float>) LangMutators.newFactory().createOrThrow( + new TypeHolder<@NotNull Float>() {}.annotatedType()); + Supplier<Integer> ctr = makeCounter(); + return Stream.of(arguments(mutator, Stream.of(true, ctr.get()), Float.NEGATIVE_INFINITY, true), + arguments(mutator, Stream.of(true, ctr.get()), -Float.MAX_VALUE, true), + arguments(mutator, Stream.of(true, ctr.get()), -Float.MIN_VALUE, true), + arguments(mutator, Stream.of(true, ctr.get()), -0.0f, true), + arguments(mutator, Stream.of(true, ctr.get()), 0.0f, true), + arguments(mutator, Stream.of(true, ctr.get()), Float.MIN_VALUE, true), + arguments(mutator, Stream.of(true, ctr.get()), Float.MAX_VALUE, true), + arguments(mutator, Stream.of(true, ctr.get()), Float.POSITIVE_INFINITY, true), + arguments(mutator, Stream.of(true, ctr.get()), Float.NaN, true), + arguments(mutator, Stream.of(true, ctr.get()), UNUSED_FLOAT, false)); + } + + static Stream<Arguments> floatInitCasesMinusOneToOne() { + SerializingMutator<Float> mutator = + (SerializingMutator<Float>) LangMutators.newFactory().createOrThrow( + new TypeHolder<@NotNull @FloatInRange(min = -1.0f, max = 1.0f) Float>() { + }.annotatedType()); + Supplier<Integer> ctr = makeCounter(); + return Stream.of(arguments(mutator, Stream.of(true, ctr.get()), -1.0f, true), + arguments(mutator, Stream.of(true, ctr.get()), -Float.MIN_VALUE, true), + arguments(mutator, Stream.of(true, ctr.get()), -0.0f, true), + arguments(mutator, Stream.of(true, ctr.get()), 0.0f, true), + arguments(mutator, Stream.of(true, ctr.get()), Float.MIN_VALUE, true), + arguments(mutator, Stream.of(true, ctr.get()), 1.0f, true), + arguments(mutator, Stream.of(true, ctr.get()), Float.NaN, true), + arguments(mutator, Stream.of(true, ctr.get()), UNUSED_FLOAT, false)); + } + + static Stream<Arguments> floatInitCasesMinusMinToMin() { + SerializingMutator<Float> mutator = + (SerializingMutator<Float>) LangMutators.newFactory().createOrThrow( + new TypeHolder<@NotNull @FloatInRange( + min = -Float.MIN_VALUE, max = Float.MIN_VALUE) Float>() { + }.annotatedType()); + Supplier<Integer> ctr = makeCounter(); + return Stream.of(arguments(mutator, Stream.of(true, ctr.get()), -Float.MIN_VALUE, true), + arguments(mutator, Stream.of(true, ctr.get()), -0.0f, true), + arguments(mutator, Stream.of(true, ctr.get()), 0.0f, true), + arguments(mutator, Stream.of(true, ctr.get()), Float.MIN_VALUE, true), + arguments(mutator, Stream.of(true, ctr.get()), Float.NaN, true), + arguments(mutator, Stream.of(true, ctr.get()), UNUSED_FLOAT, false)); + } + + static Stream<Arguments> floatInitCasesMaxToInf() { + SerializingMutator<Float> mutator = + (SerializingMutator<Float>) LangMutators.newFactory().createOrThrow( + new TypeHolder<@NotNull @FloatInRange( + min = Float.MAX_VALUE, max = Float.POSITIVE_INFINITY) Float>() { + }.annotatedType()); + Supplier<Integer> ctr = makeCounter(); + return Stream.of(arguments(mutator, Stream.of(true, ctr.get()), Float.MAX_VALUE, true), + arguments(mutator, Stream.of(true, ctr.get()), Float.POSITIVE_INFINITY, true), + arguments(mutator, Stream.of(true, ctr.get()), Float.NaN, true), + arguments(mutator, Stream.of(true, ctr.get()), UNUSED_FLOAT, false)); + } + + static Stream<Arguments> floatInitCasesMinusInfToMinusMax() { + SerializingMutator<Float> mutator = + (SerializingMutator<Float>) LangMutators.newFactory().createOrThrow( + new TypeHolder<@NotNull @FloatInRange( + min = Float.NEGATIVE_INFINITY, max = -Float.MAX_VALUE) Float>() { + }.annotatedType()); + Supplier<Integer> ctr = makeCounter(); + return Stream.of(arguments(mutator, Stream.of(true, ctr.get()), Float.NEGATIVE_INFINITY, true), + arguments(mutator, Stream.of(true, ctr.get()), -Float.MAX_VALUE, true), + arguments(mutator, Stream.of(true, ctr.get()), Float.NaN, true), + arguments(mutator, Stream.of(true, ctr.get()), UNUSED_FLOAT, false)); + } + + static Stream<Arguments> floatInitCasesFullRangeWithoutNaN() { + SerializingMutator<Float> mutator = + (SerializingMutator<Float>) LangMutators.newFactory().createOrThrow( + new TypeHolder<@NotNull @FloatInRange(min = Float.NEGATIVE_INFINITY, + max = Float.POSITIVE_INFINITY, allowNaN = true) Float>() { + }.annotatedType()); + Supplier<Integer> ctr = makeCounter(); + return Stream.of(arguments(mutator, Stream.of(true, ctr.get()), Float.NEGATIVE_INFINITY, true), + arguments(mutator, Stream.of(true, ctr.get()), -Float.MAX_VALUE, true), + arguments(mutator, Stream.of(true, ctr.get()), -Float.MIN_VALUE, true), + arguments(mutator, Stream.of(true, ctr.get()), -0.0f, true), + arguments(mutator, Stream.of(true, ctr.get()), 0.0f, true), + arguments(mutator, Stream.of(true, ctr.get()), Float.MIN_VALUE, true), + arguments(mutator, Stream.of(true, ctr.get()), Float.MAX_VALUE, true), + arguments(mutator, Stream.of(true, ctr.get()), Float.POSITIVE_INFINITY, true), + arguments(mutator, Stream.of(true, ctr.get()), Float.NaN, true), + arguments(mutator, Stream.of(true, ctr.get()), UNUSED_FLOAT, false)); + } + + @ParameterizedTest + @MethodSource({"floatInitCasesMinusOneToOne", "floatInitCasesFullRange", + "floatInitCasesMinusMinToMin", "floatInitCasesMaxToInf", "floatInitCasesMinusInfToMinusMax", + "floatInitCasesFullRangeWithoutNaN"}) + void + testFloatInitCases(SerializingMutator<Float> mutator, Stream<Object> prngValues, float expected, + boolean specialValueIndexExists) { + assertThat(mutator.toString()).isEqualTo("Float"); + if (specialValueIndexExists) { + Float n = null; + try (TestSupport.MockPseudoRandom prng = mockPseudoRandom(prngValues.toArray())) { + n = mutator.init(prng); + } + assertThat(n).isEqualTo(expected); + } else { // should throw + assertThrows(AssertionError.class, () -> { + try (TestSupport.MockPseudoRandom prng = mockPseudoRandom(prngValues.toArray())) { + mutator.init(prng); + } + }); + } + } + + static Stream<Arguments> floatMutateSanityChecksFullRangeCases() { + SerializingMutator<Float> mutator = + (SerializingMutator<Float>) LangMutators.newFactory().createOrThrow( + new TypeHolder<@NotNull @FloatInRange(min = Float.NEGATIVE_INFINITY, + max = Float.POSITIVE_INFINITY, allowNaN = true) Float>() { + }.annotatedType()); + // Init value can be set to desired one by giving this to the init method: (false, <desired + // value>) + return Stream.of( + // Bit flips + arguments(mutator, Stream.of(false, 0f), Stream.of(false, 0, 0), 1.4e-45f, true), + arguments(mutator, Stream.of(false, 0f), Stream.of(false, 0, 30), 2.0f, true), + arguments(mutator, Stream.of(false, 2f), Stream.of(false, 0, 31), -2.0f, true), + arguments(mutator, Stream.of(false, -2f), Stream.of(false, 0, 22), -3.0f, true), + // mutateExponent + arguments(mutator, Stream.of(false, 0f), Stream.of(false, 1, 0B01111100), 0.125f, true), + arguments(mutator, Stream.of(false, 0f), Stream.of(false, 1, 0B01111110), 0.5f, true), + arguments(mutator, Stream.of(false, 0f), Stream.of(false, 1, 0B01111111), 1.0f, true), + // mutateMantissa + arguments(mutator, Stream.of(false, 0f), Stream.of(false, 2, 0, 100), 1.4e-43f, true), + arguments(mutator, Stream.of(false, Float.intBitsToFloat(1)), Stream.of(false, 2, 0, -1), 0, + true), + // mutateWithMathematicalFn + arguments( + mutator, Stream.of(false, 10.1f), Stream.of(false, 3, 4), 11f, true), // Math::ceil + arguments( + mutator, Stream.of(false, 1000f), Stream.of(false, 3, 11), 3f, true), // Math::log10 + // skip libfuzzer + // random in range + arguments(mutator, Stream.of(false, 0f), Stream.of(false, 5, 10f), 10f, true), + // unknown mutation case exception + arguments(mutator, Stream.of(false, 0f), Stream.of(false, 6), UNUSED_FLOAT, false)); + } + + static Stream<Arguments> floatMutateLimitedRangeCases() { + SerializingMutator<Float> mutator = + (SerializingMutator<Float>) LangMutators.newFactory().createOrThrow( + new TypeHolder<@NotNull @FloatInRange(min = -1f, max = 1f, allowNaN = false) Float>() { + }.annotatedType()); + // Init value can be set to desired one by giving this to the init method: (false, <desired + // value>) + return Stream.of( + // Bit flip; forceInRange(); result equals previous value; adjust value + arguments(mutator, Stream.of(false, 0f), Stream.of(false, 0, 30, true), + 0f - Float.MIN_VALUE, true), + arguments(mutator, Stream.of(false, 1f), Stream.of(false, 0, 30), Math.nextDown(1f), true), + arguments(mutator, Stream.of(false, -1f), Stream.of(false, 0, 30), Math.nextUp(-1f), true), + // NaN after mutateWithMathematicalFn with NaN not allowed; forceInRange will return + // (min+max)/2 + arguments(mutator, Stream.of(false, -1f), Stream.of(false, 3, 16), 0.0f, true)); + } + + static Stream<Arguments> floatMutateLimitedRangeCasesWithNaN() { + SerializingMutator<Float> mutator = + (SerializingMutator<Float>) LangMutators.newFactory().createOrThrow( + new TypeHolder<@NotNull @FloatInRange(min = -1f, max = 1f, allowNaN = true) Float>() { + }.annotatedType()); + // Init value can be set to desired one by giving this to the init method: (false, <desired + // value>) + return Stream.of( + // NaN after mutation and forceInRange(); all good! + arguments(mutator, Stream.of(false, -1f), Stream.of(false, 3, 16), Float.NaN, true), + // NaN (with a set bit #8) after init, mutation, and forceInRange(); need to change NaN to + // something else + arguments(mutator, Stream.of(true, 6), Stream.of(false, 0, 8, 0.3f), 0.3f, true)); + } + + @ParameterizedTest + @MethodSource({"floatMutateSanityChecksFullRangeCases", "floatMutateLimitedRangeCases", + "floatMutateLimitedRangeCasesWithNaN"}) + void + testFloatMutateCases(SerializingMutator<Float> mutator, Stream<Object> initValues, + Stream<Object> mutationValues, float expected, boolean knownMutatorSwitchCase) { + assertThat(mutator.toString()).isEqualTo("Float"); + Float n; + + // Init + try (TestSupport.MockPseudoRandom prng = mockPseudoRandom(initValues.toArray())) { + n = mutator.init(prng); + } + + // Mutate + if (knownMutatorSwitchCase) { + try (TestSupport.MockPseudoRandom prng = mockPseudoRandom(mutationValues.toArray())) { + n = mutator.mutate(n, prng); + } + assertThat(n).isEqualTo(expected); + + if (!((FloatMutator) mutator).allowNaN) { + assertThat(n).isNotEqualTo(Float.NaN); + } + + if (!Float.isNaN(n)) { + assertThat(n).isAtLeast(((FloatMutator) mutator).minValue); + assertThat(n).isAtMost(((FloatMutator) mutator).maxValue); + } + } else { // Invalid mutation because a case is not handled + assertThrows(AssertionError.class, () -> { + try (TestSupport.MockPseudoRandom prng = mockPseudoRandom(mutationValues.toArray())) { + mutator.mutate(UNUSED_FLOAT, prng); + } + }); + } + } + + @Test + void testFloatCrossOverMean() { + SerializingMutator<Float> mutator = + (SerializingMutator<Float>) LangMutators.newFactory().createOrThrow( + new TypeHolder<@NotNull Float>() {}.annotatedType()); + try (TestSupport.MockPseudoRandom prng = + mockPseudoRandom(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)) { + assertThat(mutator.crossOver(0f, 0f, prng)).isWithin(0).of(0f); + assertThat(mutator.crossOver(-0f, 0f, prng)).isWithin(0).of(0f); + assertThat(mutator.crossOver(0f, 2f, prng)).isWithin(1e-10f).of(1.0f); + assertThat(mutator.crossOver(1f, 2f, prng)).isWithin(1e-10f).of(1.5f); + assertThat(mutator.crossOver(1f, 3f, prng)).isWithin(1e-10f).of(2f); + assertThat(mutator.crossOver(Float.MAX_VALUE, Float.MAX_VALUE, prng)) + .isWithin(1e-10f) + .of(Float.MAX_VALUE); + + assertThat(mutator.crossOver(0f, -2f, prng)).isWithin(1e-10f).of(-1.0f); + assertThat(mutator.crossOver(-1f, -2f, prng)).isWithin(1e-10f).of(-1.5f); + assertThat(mutator.crossOver(-1f, -3f, prng)).isWithin(1e-10f).of(-2f); + assertThat(mutator.crossOver(-Float.MAX_VALUE, -Float.MAX_VALUE, prng)) + .isWithin(1e-10f) + .of(-Float.MAX_VALUE); + + assertThat(mutator.crossOver(-100f, 200f, prng)).isWithin(1e-10f).of(50.0f); + assertThat(mutator.crossOver(100f, -200f, prng)).isWithin(1e-10f).of(-50f); + assertThat(mutator.crossOver(-Float.MAX_VALUE, Float.MAX_VALUE, prng)) + .isWithin(1e-10f) + .of(0f); + + assertThat(mutator.crossOver(Float.NEGATIVE_INFINITY, Float.POSITIVE_INFINITY, prng)).isNaN(); + assertThat(mutator.crossOver(Float.POSITIVE_INFINITY, 0f, prng)).isPositiveInfinity(); + assertThat(mutator.crossOver(0f, Float.POSITIVE_INFINITY, prng)).isPositiveInfinity(); + assertThat(mutator.crossOver(Float.NEGATIVE_INFINITY, 0f, prng)).isNegativeInfinity(); + assertThat(mutator.crossOver(0f, Float.NEGATIVE_INFINITY, prng)).isNegativeInfinity(); + assertThat(mutator.crossOver(Float.NaN, 0f, prng)).isNaN(); + assertThat(mutator.crossOver(0f, Float.NaN, prng)).isNaN(); + } + } + + @Test + void testFloatCrossOverExponent() { + SerializingMutator<Float> mutator = + (SerializingMutator<Float>) LangMutators.newFactory().createOrThrow( + new TypeHolder<@NotNull Float>() {}.annotatedType()); + try (TestSupport.MockPseudoRandom prng = mockPseudoRandom(1, 1, 1)) { + assertThat(mutator.crossOver(2.0f, -1.5f, prng)).isWithin(1e-10f).of(1.0f); + assertThat(mutator.crossOver(2.0f, Float.POSITIVE_INFINITY, prng)).isPositiveInfinity(); + assertThat(mutator.crossOver(-1.5f, Float.NEGATIVE_INFINITY, prng)).isNaN(); + } + } + + @Test + void testFloatCrossOverMantissa() { + SerializingMutator<Float> mutator = + (SerializingMutator<Float>) LangMutators.newFactory().createOrThrow( + new TypeHolder<@NotNull Float>() {}.annotatedType()); + try (TestSupport.MockPseudoRandom prng = mockPseudoRandom(2, 2, 2)) { + assertThat(mutator.crossOver(4.0f, 3.5f, prng)).isWithin(1e-10f).of(7.0f); + assertThat(mutator.crossOver(Float.POSITIVE_INFINITY, 3.0f, prng)).isNaN(); + assertThat(mutator.crossOver(Float.MAX_VALUE, 0.0f, prng)).isWithin(1e-10f).of(1.7014118e38f); + } + } + + static Stream<Arguments> doubleInitCasesFullRange() { + SerializingMutator<Double> mutator = + (SerializingMutator<Double>) LangMutators.newFactory().createOrThrow( + new TypeHolder<@NotNull Double>() {}.annotatedType()); + Supplier<Integer> ctr = makeCounter(); + return Stream.of(arguments(mutator, Stream.of(true, ctr.get()), Double.NEGATIVE_INFINITY, true), + arguments(mutator, Stream.of(true, ctr.get()), -Double.MAX_VALUE, true), + arguments(mutator, Stream.of(true, ctr.get()), -Double.MIN_VALUE, true), + arguments(mutator, Stream.of(true, ctr.get()), -0.0, true), + arguments(mutator, Stream.of(true, ctr.get()), 0.0, true), + arguments(mutator, Stream.of(true, ctr.get()), Double.MIN_VALUE, true), + arguments(mutator, Stream.of(true, ctr.get()), Double.MAX_VALUE, true), + arguments(mutator, Stream.of(true, ctr.get()), Double.POSITIVE_INFINITY, true), + arguments(mutator, Stream.of(true, ctr.get()), Double.NaN, true), + arguments(mutator, Stream.of(true, ctr.get()), UNUSED_DOUBLE, false)); + } + + static Stream<Arguments> doubleInitCasesMinusOneToOne() { + SerializingMutator<Double> mutator = + (SerializingMutator<Double>) LangMutators.newFactory().createOrThrow( + new TypeHolder<@NotNull @DoubleInRange(min = -1.0, max = 1.0) Double>() { + }.annotatedType()); + Supplier<Integer> ctr = makeCounter(); + return Stream.of(arguments(mutator, Stream.of(true, ctr.get()), -1.0, true), + arguments(mutator, Stream.of(true, ctr.get()), -Double.MIN_VALUE, true), + arguments(mutator, Stream.of(true, ctr.get()), -0.0, true), + arguments(mutator, Stream.of(true, ctr.get()), 0.0, true), + arguments(mutator, Stream.of(true, ctr.get()), Double.MIN_VALUE, true), + arguments(mutator, Stream.of(true, ctr.get()), 1.0, true), + arguments(mutator, Stream.of(true, ctr.get()), Double.NaN, true), + arguments(mutator, Stream.of(true, ctr.get()), UNUSED_DOUBLE, false)); + } + + static Stream<Arguments> doubleInitCasesMinusMinToMin() { + SerializingMutator<Double> mutator = + (SerializingMutator<Double>) LangMutators.newFactory().createOrThrow( + new TypeHolder<@NotNull @DoubleInRange( + min = -Double.MIN_VALUE, max = Double.MIN_VALUE) Double>() { + }.annotatedType()); + Supplier<Integer> ctr = makeCounter(); + return Stream.of(arguments(mutator, Stream.of(true, ctr.get()), -Double.MIN_VALUE, true), + arguments(mutator, Stream.of(true, ctr.get()), -0.0, true), + arguments(mutator, Stream.of(true, ctr.get()), 0.0, true), + arguments(mutator, Stream.of(true, ctr.get()), Double.MIN_VALUE, true), + arguments(mutator, Stream.of(true, ctr.get()), Double.NaN, true), + arguments(mutator, Stream.of(true, ctr.get()), UNUSED_DOUBLE, false)); + } + + static Stream<Arguments> doubleInitCasesMaxToInf() { + SerializingMutator<Double> mutator = + (SerializingMutator<Double>) LangMutators.newFactory().createOrThrow( + new TypeHolder<@NotNull @DoubleInRange( + min = Double.MAX_VALUE, max = Double.POSITIVE_INFINITY) Double>() { + }.annotatedType()); + Supplier<Integer> ctr = makeCounter(); + return Stream.of(arguments(mutator, Stream.of(true, ctr.get()), Double.MAX_VALUE, true), + arguments(mutator, Stream.of(true, ctr.get()), Double.POSITIVE_INFINITY, true), + arguments(mutator, Stream.of(true, ctr.get()), Double.NaN, true), + arguments(mutator, Stream.of(true, ctr.get()), UNUSED_DOUBLE, false)); + } + + static Stream<Arguments> doubleInitCasesMinusInfToMinusMax() { + SerializingMutator<Double> mutator = + (SerializingMutator<Double>) LangMutators.newFactory().createOrThrow( + new TypeHolder<@NotNull @DoubleInRange( + min = Double.NEGATIVE_INFINITY, max = -Double.MAX_VALUE) Double>() { + }.annotatedType()); + Supplier<Integer> ctr = makeCounter(); + return Stream.of(arguments(mutator, Stream.of(true, ctr.get()), Double.NEGATIVE_INFINITY, true), + arguments(mutator, Stream.of(true, ctr.get()), -Double.MAX_VALUE, true), + arguments(mutator, Stream.of(true, ctr.get()), Double.NaN, true), + arguments(mutator, Stream.of(true, ctr.get()), UNUSED_DOUBLE, false)); + } + + static Stream<Arguments> doubleInitCasesFullRangeWithoutNaN() { + SerializingMutator<Double> mutator = + (SerializingMutator<Double>) LangMutators.newFactory().createOrThrow( + new TypeHolder<@NotNull @DoubleInRange(min = Double.NEGATIVE_INFINITY, + max = Double.POSITIVE_INFINITY, allowNaN = true) Double>() { + }.annotatedType()); + Supplier<Integer> ctr = makeCounter(); + return Stream.of(arguments(mutator, Stream.of(true, ctr.get()), Double.NEGATIVE_INFINITY, true), + arguments(mutator, Stream.of(true, ctr.get()), -Double.MAX_VALUE, true), + arguments(mutator, Stream.of(true, ctr.get()), -Double.MIN_VALUE, true), + arguments(mutator, Stream.of(true, ctr.get()), -0.0, true), + arguments(mutator, Stream.of(true, ctr.get()), 0.0, true), + arguments(mutator, Stream.of(true, ctr.get()), Double.MIN_VALUE, true), + arguments(mutator, Stream.of(true, ctr.get()), Double.MAX_VALUE, true), + arguments(mutator, Stream.of(true, ctr.get()), Double.POSITIVE_INFINITY, true), + arguments(mutator, Stream.of(true, ctr.get()), Double.NaN, true), + arguments(mutator, Stream.of(true, ctr.get()), UNUSED_DOUBLE, false)); + } + + @ParameterizedTest + @MethodSource({"doubleInitCasesMinusOneToOne", "doubleInitCasesFullRange", + "doubleInitCasesMinusMinToMin", "doubleInitCasesMaxToInf", + "doubleInitCasesMinusInfToMinusMax", "doubleInitCasesFullRangeWithoutNaN"}) + void + testDoubleInitCases(SerializingMutator<Double> mutator, Stream<Object> prngValues, + double expected, boolean knownSwitchCase) { + assertThat(mutator.toString()).isEqualTo("Double"); + if (knownSwitchCase) { + Double n = null; + try (TestSupport.MockPseudoRandom prng = mockPseudoRandom(prngValues.toArray())) { + n = mutator.init(prng); + } + assertThat(n).isEqualTo(expected); + } else { + assertThrows(AssertionError.class, () -> { + try (TestSupport.MockPseudoRandom prng = mockPseudoRandom(prngValues.toArray())) { + mutator.init(prng); + } + }); + } + } + + static Stream<Arguments> doubleMutateSanityChecksFullRangeCases() { + SerializingMutator<Double> mutator = + (SerializingMutator<Double>) LangMutators.newFactory().createOrThrow( + new TypeHolder<@NotNull @DoubleInRange(min = Double.NEGATIVE_INFINITY, + max = Double.POSITIVE_INFINITY, allowNaN = true) Double>() { + }.annotatedType()); + // Init value can be set to desired one by giving this to the init method: (false, <desired + // value>) + return Stream.of( + // Bit flips + arguments(mutator, Stream.of(false, 0.0), Stream.of(false, 0, 0), Double.MIN_VALUE, true), + arguments(mutator, Stream.of(false, 0.0), Stream.of(false, 0, 62), 2.0, true), + arguments(mutator, Stream.of(false, 2.0), Stream.of(false, 0, 63), -2.0, true), + arguments(mutator, Stream.of(false, -2.0), Stream.of(false, 0, 51), -3.0, true), + // mutateExponent + arguments(mutator, Stream.of(false, 0.0), Stream.of(false, 1, 0B1111111100), 0.125, true), + arguments(mutator, Stream.of(false, 0.0), Stream.of(false, 1, 0B1111111110), 0.5, true), + arguments(mutator, Stream.of(false, 0.0), Stream.of(false, 1, 0B1111111111), 1.0, true), + // mutateMantissa + arguments(mutator, Stream.of(false, 0.0), Stream.of(false, 2, 0, 100L), 4.94e-322, true), + arguments(mutator, Stream.of(false, Double.longBitsToDouble(1)), + Stream.of(false, 2, 0, -1L), 0, true), + // mutateWithMathematicalFn + arguments(mutator, Stream.of(false, 10.1), Stream.of(false, 3, 4), 11, true), // Math::ceil + arguments( + mutator, Stream.of(false, 1000.0), Stream.of(false, 3, 11), 3, true), // Math::log10 + // skip libfuzzer + // random in range + arguments(mutator, Stream.of(false, 0.0), Stream.of(false, 5, 10.0), 10, true), + // unknown mutation case exception + arguments(mutator, Stream.of(false, 0.0), Stream.of(false, 6), UNUSED_DOUBLE, false)); + } + + static Stream<Arguments> doubleMutateLimitedRangeCases() { + SerializingMutator<Double> mutator = + (SerializingMutator<Double>) LangMutators.newFactory().createOrThrow( + new TypeHolder<@NotNull @DoubleInRange(min = -1, max = 1, allowNaN = false) Double>() { + }.annotatedType()); + // Init value can be set to desired one by giving this to the init method: (false, <desired + // value>) + return Stream.of( + // Bit flip; forceInRange(); result equals previous value; adjust value + arguments(mutator, Stream.of(false, 0.0), Stream.of(false, 0, 62, true), + 0.0 - Double.MIN_VALUE, true), + arguments( + mutator, Stream.of(false, 1.0), Stream.of(false, 0, 62), Math.nextDown(1.0), true), + arguments( + mutator, Stream.of(false, -1.0), Stream.of(false, 0, 62), Math.nextUp(-1.0), true), + // NaN after mutateWithMathematicalFn: sqrt(-1.0); NaN not allowed; forceInRange will return + // (min+max)/2 + arguments(mutator, Stream.of(false, -1.0), Stream.of(false, 3, 16), 0.0, true)); + } + + static Stream<Arguments> doubleMutateLimitedRangeCasesWithNaN() { + SerializingMutator<Double> mutator = + (SerializingMutator<Double>) LangMutators.newFactory().createOrThrow( + new TypeHolder<@NotNull @DoubleInRange(min = -1, max = 1, allowNaN = true) Double>() { + }.annotatedType()); + // Init value can be set to desired one by giving this to the init method: (false, <desired + // value>) + return Stream.of( + // NaN after mutation and forceInRange(); all good! + arguments(mutator, Stream.of(false, -1.0), Stream.of(false, 3, 16), Double.NaN, true), + // NaN (with a set bit #8) after init, mutation, and forceInRange(); need to change NaN to + // something else + arguments(mutator, Stream.of(true, 6), Stream.of(false, 0, 8, 0.3), 0.3, true)); + } + + @ParameterizedTest + @MethodSource({"doubleMutateSanityChecksFullRangeCases", "doubleMutateLimitedRangeCases", + "doubleMutateLimitedRangeCasesWithNaN"}) + void + testDoubleMutateCases(SerializingMutator<Double> mutator, Stream<Object> initValues, + Stream<Object> mutationValues, double expected, boolean knownSwitchCase) { + assertThat(mutator.toString()).isEqualTo("Double"); + Double n; + + // Init + try (TestSupport.MockPseudoRandom prng = mockPseudoRandom(initValues.toArray())) { + n = mutator.init(prng); + } + + // Mutate + if (knownSwitchCase) { + try (TestSupport.MockPseudoRandom prng = mockPseudoRandom(mutationValues.toArray())) { + n = mutator.mutate(n, prng); + } + assertThat(n).isEqualTo(expected); + + if (!((DoubleMutator) mutator).allowNaN) { + assertThat(n).isNotEqualTo(Double.NaN); + } + + if (!Double.isNaN(n)) { + assertThat(n).isAtLeast(((DoubleMutator) mutator).minValue); + assertThat(n).isAtMost(((DoubleMutator) mutator).maxValue); + } + } else { // Invalid mutation because a case is not handled + assertThrows(AssertionError.class, () -> { + try (TestSupport.MockPseudoRandom prng = mockPseudoRandom(mutationValues.toArray())) { + mutator.mutate(UNUSED_DOUBLE, prng); + } + }); + } + } + + @Test + void testDoubleCrossOverMean() { + SerializingMutator<Double> mutator = + (SerializingMutator<Double>) LangMutators.newFactory().createOrThrow( + new TypeHolder<@NotNull Double>() {}.annotatedType()); + try (TestSupport.MockPseudoRandom prng = + mockPseudoRandom(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)) { + assertThat(mutator.crossOver(0.0, 0.0, prng)).isWithin(0).of(0f); + assertThat(mutator.crossOver(-0.0, 0.0, prng)).isWithin(0).of(0f); + assertThat(mutator.crossOver(0.0, 2.0, prng)).isWithin(1e-10f).of(1.0f); + assertThat(mutator.crossOver(1.0, 2.0, prng)).isWithin(1e-10f).of(1.5f); + assertThat(mutator.crossOver(1.0, 3.0, prng)).isWithin(1e-10f).of(2f); + assertThat(mutator.crossOver(Double.MAX_VALUE, Double.MAX_VALUE, prng)) + .isWithin(1e-10f) + .of(Double.MAX_VALUE); + + assertThat(mutator.crossOver(0.0, -2.0, prng)).isWithin(1e-10f).of(-1.0f); + assertThat(mutator.crossOver(-1.0, -2.0, prng)).isWithin(1e-10f).of(-1.5f); + assertThat(mutator.crossOver(-1.0, -3.0, prng)).isWithin(1e-10f).of(-2f); + assertThat(mutator.crossOver(-Double.MAX_VALUE, -Double.MAX_VALUE, prng)) + .isWithin(1e-10f) + .of(-Double.MAX_VALUE); + + assertThat(mutator.crossOver(-100.0, 200.0, prng)).isWithin(1e-10f).of(50.0f); + assertThat(mutator.crossOver(100.0, -200.0, prng)).isWithin(1e-10f).of(-50f); + assertThat(mutator.crossOver(-Double.MAX_VALUE, Double.MAX_VALUE, prng)) + .isWithin(1e-10f) + .of(0f); + + assertThat(mutator.crossOver(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY, prng)) + .isNaN(); + assertThat(mutator.crossOver(Double.POSITIVE_INFINITY, 0.0, prng)).isPositiveInfinity(); + assertThat(mutator.crossOver(0.0, Double.POSITIVE_INFINITY, prng)).isPositiveInfinity(); + assertThat(mutator.crossOver(Double.NEGATIVE_INFINITY, 0.0, prng)).isNegativeInfinity(); + assertThat(mutator.crossOver(0.0, Double.NEGATIVE_INFINITY, prng)).isNegativeInfinity(); + assertThat(mutator.crossOver(Double.NaN, 0.0, prng)).isNaN(); + assertThat(mutator.crossOver(0.0, Double.NaN, prng)).isNaN(); + } + } + + @Test + void testDoubleCrossOverExponent() { + SerializingMutator<Double> mutator = + (SerializingMutator<Double>) LangMutators.newFactory().createOrThrow( + new TypeHolder<@NotNull Double>() {}.annotatedType()); + try (TestSupport.MockPseudoRandom prng = mockPseudoRandom(1, 1, 1)) { + assertThat(mutator.crossOver(2.0, -1.5, prng)).isWithin(1e-10f).of(1.0f); + assertThat(mutator.crossOver(2.0, Double.POSITIVE_INFINITY, prng)).isPositiveInfinity(); + assertThat(mutator.crossOver(-1.5, Double.NEGATIVE_INFINITY, prng)).isNaN(); + } + } + + @Test + void testDoubleCrossOverMantissa() { + SerializingMutator<Double> mutator = + (SerializingMutator<Double>) LangMutators.newFactory().createOrThrow( + new TypeHolder<@NotNull Double>() {}.annotatedType()); + try (TestSupport.MockPseudoRandom prng = mockPseudoRandom(2, 2, 2)) { + assertThat(mutator.crossOver(4.0, 3.5, prng)).isWithin(1e-10f).of(7.0f); + assertThat(mutator.crossOver(Double.POSITIVE_INFINITY, 3.0, prng)).isNaN(); + assertThat(mutator.crossOver(Double.MAX_VALUE, 0.0, prng)) + .isWithin(1e-10f) + .of(8.98846567431158e307); + } + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/mutation/mutator/lang/IntegralMutatorTest.java b/src/test/java/com/code_intelligence/jazzer/mutation/mutator/lang/IntegralMutatorTest.java new file mode 100644 index 00000000..dda8cfed --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/mutation/mutator/lang/IntegralMutatorTest.java @@ -0,0 +1,85 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.code_intelligence.jazzer.mutation.mutator.lang; + +import static com.code_intelligence.jazzer.mutation.mutator.lang.IntegralMutatorFactory.AbstractIntegralMutator.forceInRange; +import static com.code_intelligence.jazzer.mutation.support.TestSupport.mockPseudoRandom; +import static com.google.common.truth.Truth.assertThat; +import static org.junit.jupiter.params.provider.Arguments.arguments; + +import com.code_intelligence.jazzer.mutation.annotation.NotNull; +import com.code_intelligence.jazzer.mutation.api.SerializingMutator; +import com.code_intelligence.jazzer.mutation.support.TestSupport.MockPseudoRandom; +import com.code_intelligence.jazzer.mutation.support.TypeHolder; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +@SuppressWarnings("unchecked") +class IntegralMutatorTest { + static Stream<Arguments> forceInRangeCases() { + return Stream.of(arguments(0, 0, 1), arguments(5, 0, 1), arguments(-5, -10, -1), + arguments(-200, -10, -1), arguments(10, 0, 3), arguments(-5, 0, 3), arguments(10, -7, 7), + arguments(Long.MIN_VALUE, Long.MIN_VALUE, Long.MAX_VALUE), + arguments(Long.MIN_VALUE, Long.MIN_VALUE, 100), + arguments(Long.MIN_VALUE + 100, Long.MIN_VALUE, 100), + arguments(Long.MAX_VALUE, -100, Long.MAX_VALUE), + arguments(Long.MAX_VALUE - 100, -100, Long.MAX_VALUE), + arguments(Long.MAX_VALUE, Long.MIN_VALUE, Long.MAX_VALUE), + arguments(Long.MIN_VALUE, Long.MIN_VALUE + 1, Long.MAX_VALUE), + arguments(Long.MAX_VALUE, Long.MIN_VALUE, Long.MAX_VALUE - 1), + arguments(Long.MIN_VALUE, Long.MAX_VALUE - 5, Long.MAX_VALUE), + arguments(Long.MAX_VALUE, Long.MIN_VALUE, Long.MIN_VALUE + 5)); + } + + @ParameterizedTest + @MethodSource("forceInRangeCases") + void testForceInRange(long value, long minValue, long maxValue) { + long inRange = forceInRange(value, minValue, maxValue); + assertThat(inRange).isAtLeast(minValue); + assertThat(inRange).isAtMost(maxValue); + if (value >= minValue && value <= maxValue) { + assertThat(inRange).isEqualTo(value); + } + } + + @Test + void testCrossOver() { + SerializingMutator<Long> mutator = + (SerializingMutator<Long>) LangMutators.newFactory().createOrThrow( + new TypeHolder<@NotNull Long>() {}.annotatedType()); + // cross over mean values + try (MockPseudoRandom prng = mockPseudoRandom(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)) { + assertThat(mutator.crossOver(0L, 0L, prng)).isEqualTo(0); + assertThat(mutator.crossOver(0L, 2L, prng)).isEqualTo(1); + assertThat(mutator.crossOver(1L, 2L, prng)).isEqualTo(1); + assertThat(mutator.crossOver(1L, 3L, prng)).isEqualTo(2); + assertThat(mutator.crossOver(Long.MAX_VALUE, Long.MAX_VALUE, prng)).isEqualTo(Long.MAX_VALUE); + + assertThat(mutator.crossOver(0L, -2L, prng)).isEqualTo(-1); + assertThat(mutator.crossOver(-1L, -2L, prng)).isEqualTo(-1); + assertThat(mutator.crossOver(-1L, -3L, prng)).isEqualTo(-2); + assertThat(mutator.crossOver(Long.MIN_VALUE, Long.MIN_VALUE, prng)).isEqualTo(Long.MIN_VALUE); + + assertThat(mutator.crossOver(-100L, 200L, prng)).isEqualTo(50); + assertThat(mutator.crossOver(100L, -200L, prng)).isEqualTo(-50); + assertThat(mutator.crossOver(Long.MIN_VALUE, Long.MAX_VALUE, prng)).isEqualTo(0); + } + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/mutation/mutator/lang/NullableMutatorTest.java b/src/test/java/com/code_intelligence/jazzer/mutation/mutator/lang/NullableMutatorTest.java new file mode 100644 index 00000000..bc9a65b2 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/mutation/mutator/lang/NullableMutatorTest.java @@ -0,0 +1,101 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.code_intelligence.jazzer.mutation.mutator.lang; + +import static com.code_intelligence.jazzer.mutation.support.TestSupport.mockPseudoRandom; +import static com.google.common.truth.Truth.assertThat; + +import com.code_intelligence.jazzer.mutation.annotation.NotNull; +import com.code_intelligence.jazzer.mutation.api.ChainedMutatorFactory; +import com.code_intelligence.jazzer.mutation.api.SerializingMutator; +import com.code_intelligence.jazzer.mutation.support.TestSupport.MockPseudoRandom; +import com.code_intelligence.jazzer.mutation.support.TypeHolder; +import java.lang.reflect.AnnotatedType; +import org.junit.jupiter.api.Test; + +@SuppressWarnings("unchecked") +class NullableMutatorTest { + @Test + void testNullable() { + SerializingMutator<Boolean> mutator = + new ChainedMutatorFactory(new NullableMutatorFactory(), new BooleanMutatorFactory()) + .createOrThrow(Boolean.class); + assertThat(mutator.toString()).isEqualTo("Nullable<Boolean>"); + + Boolean bool; + try (MockPseudoRandom prng = mockPseudoRandom(/* init to null */ true)) { + bool = mutator.init(prng); + } + assertThat(bool).isNull(); + + try (MockPseudoRandom prng = mockPseudoRandom(/* init for non-null Boolean */ false)) { + bool = mutator.mutate(bool, prng); + } + assertThat(bool).isFalse(); + + try (MockPseudoRandom prng = mockPseudoRandom(/* mutate to non-null Boolean */ false)) { + bool = mutator.mutate(bool, prng); + } + assertThat(bool).isTrue(); + + try (MockPseudoRandom prng = mockPseudoRandom(/* mutate to null */ true)) { + bool = mutator.mutate(bool, prng); + } + assertThat(bool).isNull(); + } + + @Test + void testNotNull() { + ChainedMutatorFactory factory = + new ChainedMutatorFactory(new NullableMutatorFactory(), new BooleanMutatorFactory()); + AnnotatedType notNullBoolean = new TypeHolder<@NotNull Boolean>() {}.annotatedType(); + SerializingMutator<Boolean> mutator = + (SerializingMutator<Boolean>) factory.createOrThrow(notNullBoolean); + assertThat(mutator.toString()).isEqualTo("Boolean"); + } + + @Test + void testPrimitive() { + ChainedMutatorFactory factory = + new ChainedMutatorFactory(new NullableMutatorFactory(), new BooleanMutatorFactory()); + SerializingMutator<Boolean> mutator = factory.createOrThrow(boolean.class); + assertThat(mutator.toString()).isEqualTo("Boolean"); + } + + @Test + void testCrossOver() { + SerializingMutator<Boolean> mutator = + new ChainedMutatorFactory(new NullableMutatorFactory(), new BooleanMutatorFactory()) + .createOrThrow(Boolean.class); + try (MockPseudoRandom prng = mockPseudoRandom(true)) { + Boolean valueCrossedOver = mutator.crossOver(Boolean.TRUE, Boolean.TRUE, prng); + assertThat(valueCrossedOver).isNotNull(); + } + try (MockPseudoRandom prng = mockPseudoRandom()) { + Boolean bothNull = mutator.crossOver(null, null, prng); + assertThat(bothNull).isNull(); + } + try (MockPseudoRandom prng = mockPseudoRandom(false)) { + Boolean oneNotNull = mutator.crossOver(null, Boolean.TRUE, prng); + assertThat(oneNotNull).isNotNull(); + } + try (MockPseudoRandom prng = mockPseudoRandom(true)) { + Boolean nullFrequency = mutator.crossOver(null, Boolean.TRUE, prng); + assertThat(nullFrequency).isNull(); + } + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/mutation/mutator/lang/StringMutatorTest.java b/src/test/java/com/code_intelligence/jazzer/mutation/mutator/lang/StringMutatorTest.java new file mode 100644 index 00000000..23060359 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/mutation/mutator/lang/StringMutatorTest.java @@ -0,0 +1,224 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.code_intelligence.jazzer.mutation.mutator.lang; + +import static com.code_intelligence.jazzer.mutation.mutator.lang.StringMutatorFactory.fixUpAscii; +import static com.code_intelligence.jazzer.mutation.mutator.lang.StringMutatorFactory.fixUpUtf8; +import static com.code_intelligence.jazzer.mutation.support.TestSupport.mockPseudoRandom; +import static com.google.common.truth.Truth.assertThat; + +import com.code_intelligence.jazzer.mutation.annotation.NotNull; +import com.code_intelligence.jazzer.mutation.annotation.WithUtf8Length; +import com.code_intelligence.jazzer.mutation.api.SerializingMutator; +import com.code_intelligence.jazzer.mutation.mutator.libfuzzer.LibFuzzerMutator; +import com.code_intelligence.jazzer.mutation.support.RandomSupport; +import com.code_intelligence.jazzer.mutation.support.TestSupport.MockPseudoRandom; +import com.code_intelligence.jazzer.mutation.support.TypeHolder; +import com.google.protobuf.ByteString; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.SplittableRandom; +import org.junit.jupiter.api.*; + +class StringMutatorTest { + /** + * Some tests may set {@link LibFuzzerMutator#MOCK_SIZE_KEY} which can interfere with other tests + * unless cleared. + */ + @AfterEach + void cleanMockSize() { + System.clearProperty(LibFuzzerMutator.MOCK_SIZE_KEY); + } + + @RepeatedTest(10) + void testFixAscii_randomInputFixed(RepetitionInfo info) { + SplittableRandom random = new SplittableRandom( + (long) "testFixAscii_randomInputFixed".hashCode() * info.getCurrentRepetition()); + + for (int length = 0; length < 1000; length++) { + byte[] randomBytes = generateRandomBytes(random, length); + byte[] copy = Arrays.copyOf(randomBytes, randomBytes.length); + fixUpAscii(copy); + if (isValidAscii(randomBytes)) { + assertThat(copy).isEqualTo(randomBytes); + } else { + assertThat(isValidAscii(copy)).isTrue(); + } + } + } + + @RepeatedTest(10) + void testFixAscii_validInputNotChanged(RepetitionInfo info) { + SplittableRandom random = new SplittableRandom( + (long) "testFixAscii_validInputNotChanged".hashCode() * info.getCurrentRepetition()); + + for (int codePoints = 0; codePoints < 1000; codePoints++) { + byte[] validAscii = generateValidAsciiBytes(random, codePoints); + byte[] copy = Arrays.copyOf(validAscii, validAscii.length); + fixUpAscii(copy); + assertThat(copy).isEqualTo(validAscii); + } + } + + @RepeatedTest(20) + void testFixUtf8_randomInputFixed(RepetitionInfo info) { + SplittableRandom random = new SplittableRandom( + (long) "testFixUtf8_randomInputFixed".hashCode() * info.getCurrentRepetition()); + + for (int length = 0; length < 1000; length++) { + byte[] randomBytes = generateRandomBytes(random, length); + byte[] copy = Arrays.copyOf(randomBytes, randomBytes.length); + fixUpUtf8(copy); + if (isValidUtf8(randomBytes)) { + assertThat(copy).isEqualTo(randomBytes); + } else { + assertThat(isValidUtf8(copy)).isTrue(); + } + } + } + + @RepeatedTest(20) + void testFixUtf8_validInputNotChanged(RepetitionInfo info) { + SplittableRandom random = new SplittableRandom( + (long) "testFixUtf8_validInputNotChanged".hashCode() * info.getCurrentRepetition()); + + for (int codePoints = 0; codePoints < 1000; codePoints++) { + byte[] validUtf8 = generateValidUtf8Bytes(random, codePoints); + byte[] copy = Arrays.copyOf(validUtf8, validUtf8.length); + fixUpUtf8(copy); + assertThat(copy).isEqualTo(validUtf8); + } + } + + @Test + void testMinLengthInit() { + SerializingMutator<String> mutator = + (SerializingMutator<String>) LangMutators.newFactory().createOrThrow( + new TypeHolder<@NotNull @WithUtf8Length(min = 10) String>() {}.annotatedType()); + assertThat(mutator.toString()).isEqualTo("String"); + + try (MockPseudoRandom prng = mockPseudoRandom(5)) { + // mock prng should throw an assert error when given a lower value than min + Assertions.assertThrows(AssertionError.class, () -> { String s = mutator.init(prng); }); + } + } + + @Test + void testMaxLengthInit() { + SerializingMutator<String> mutator = + (SerializingMutator<String>) LangMutators.newFactory().createOrThrow( + new TypeHolder<@NotNull @WithUtf8Length(max = 50) String>() {}.annotatedType()); + assertThat(mutator.toString()).isEqualTo("String"); + + try (MockPseudoRandom prng = mockPseudoRandom(60)) { + // mock prng should throw an assert error when given a value higher than max + Assertions.assertThrows(AssertionError.class, () -> { String s = mutator.init(prng); }); + } + } + + @Test + void testMinLengthMutate() { + SerializingMutator<String> mutator = + (SerializingMutator<String>) LangMutators.newFactory().createOrThrow( + new TypeHolder<@NotNull @WithUtf8Length(min = 10) String>() {}.annotatedType()); + assertThat(mutator.toString()).isEqualTo("String"); + + String s; + try (MockPseudoRandom prng = mockPseudoRandom(10, "foobarbazf".getBytes())) { + s = mutator.init(prng); + } + assertThat(s).isEqualTo("foobarbazf"); + + System.setProperty(LibFuzzerMutator.MOCK_SIZE_KEY, "5"); + try (MockPseudoRandom prng = mockPseudoRandom()) { + s = mutator.mutate(s, prng); + } + assertThat(s).isEqualTo("gqrff\0\0\0\0\0"); + } + + @Test + void testMaxLengthMutate() { + SerializingMutator<String> mutator = + (SerializingMutator<String>) LangMutators.newFactory().createOrThrow( + new TypeHolder<@NotNull @WithUtf8Length(max = 15) String>() {}.annotatedType()); + assertThat(mutator.toString()).isEqualTo("String"); + + String s; + try (MockPseudoRandom prng = mockPseudoRandom(10, "foobarbazf".getBytes())) { + s = mutator.init(prng); + } + assertThat(s).isEqualTo("foobarbazf"); + + System.setProperty(LibFuzzerMutator.MOCK_SIZE_KEY, "20"); + try (MockPseudoRandom prng = mockPseudoRandom()) { + Assertions.assertThrows( + ArrayIndexOutOfBoundsException.class, () -> { String s2 = mutator.mutate(s, prng); }); + } + } + + @Test + void testMultibyteCharacters() { + SerializingMutator<String> mutator = + (SerializingMutator<String>) LangMutators.newFactory().createOrThrow( + new TypeHolder<@NotNull @WithUtf8Length(min = 10) String>() {}.annotatedType()); + assertThat(mutator.toString()).isEqualTo("String"); + + String s; + try ( + MockPseudoRandom prng = mockPseudoRandom(10, "foobarÖÖ".getBytes(StandardCharsets.UTF_8))) { + s = mutator.init(prng); + } + assertThat(s).hasLength(8); + assertThat(s).isEqualTo("foobarÖÖ"); + } + + private static boolean isValidUtf8(byte[] data) { + return ByteString.copyFrom(data).isValidUtf8(); + } + + private static boolean isValidAscii(byte[] data) { + for (byte b : data) { + if ((b & 0xFF) > 0x7F) { + return false; + } + } + return true; + } + + private static byte[] generateRandomBytes(SplittableRandom random, int length) { + byte[] bytes = new byte[length]; + RandomSupport.nextBytes(random, bytes); + return bytes; + } + + private static byte[] generateValidAsciiBytes(SplittableRandom random, int length) { + return random.ints(0, 0x7F) + .limit(length) + .collect(StringBuilder::new, StringBuilder::appendCodePoint, StringBuilder::append) + .toString() + .getBytes(StandardCharsets.UTF_8); + } + + private static byte[] generateValidUtf8Bytes(SplittableRandom random, long codePoints) { + return random.ints(0, Character.MAX_CODE_POINT + 1) + .filter(code -> code < Character.MIN_SURROGATE || code > Character.MAX_SURROGATE) + .limit(codePoints) + .collect(StringBuilder::new, StringBuilder::appendCodePoint, StringBuilder::append) + .toString() + .getBytes(StandardCharsets.UTF_8); + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/mutation/mutator/proto/BUILD.bazel b/src/test/java/com/code_intelligence/jazzer/mutation/mutator/proto/BUILD.bazel new file mode 100644 index 00000000..bf8b551d --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/mutation/mutator/proto/BUILD.bazel @@ -0,0 +1,60 @@ +load("@contrib_rules_jvm//java:defs.bzl", "java_test_suite") + +proto_library( + name = "proto3_proto", + srcs = ["proto3.proto"], + deps = [ + "@com_google_protobuf//:any_proto", + ], +) + +java_proto_library( + name = "proto3_java_proto", + testonly = True, + visibility = ["//src/test/java/com/code_intelligence/jazzer/mutation/mutator:__pkg__"], + deps = [":proto3_proto"], +) + +proto_library( + name = "proto2_proto", + srcs = ["proto2.proto"], +) + +java_proto_library( + name = "proto2_java_proto", + testonly = True, + visibility = [ + "//src/test/java/com/code_intelligence/jazzer/mutation/mutator:__pkg__", + "//tests:__pkg__", + ], + deps = [":proto2_proto"], +) + +cc_proto_library( + name = "proto2_cc_proto", + testonly = True, + visibility = [ + "//tests:__pkg__", + ], + deps = [":proto2_proto"], +) + +java_test_suite( + name = "ProtoTests", + size = "small", + srcs = glob(["*.java"]), + runner = "junit5", + deps = [ + ":proto2_java_proto", + ":proto3_java_proto", + "//src/main/java/com/code_intelligence/jazzer/mutation/annotation", + "//src/main/java/com/code_intelligence/jazzer/mutation/annotation/proto", + "//src/main/java/com/code_intelligence/jazzer/mutation/api", + "//src/main/java/com/code_intelligence/jazzer/mutation/mutator/collection", + "//src/main/java/com/code_intelligence/jazzer/mutation/mutator/lang", + "//src/main/java/com/code_intelligence/jazzer/mutation/mutator/proto", + "//src/main/java/com/code_intelligence/jazzer/mutation/support", + "//src/test/java/com/code_intelligence/jazzer/mutation/support:test_support", + "@com_google_protobuf//java/core", + ], +) diff --git a/src/test/java/com/code_intelligence/jazzer/mutation/mutator/proto/BuilderAdaptersTest.java b/src/test/java/com/code_intelligence/jazzer/mutation/mutator/proto/BuilderAdaptersTest.java new file mode 100644 index 00000000..7722a6ad --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/mutation/mutator/proto/BuilderAdaptersTest.java @@ -0,0 +1,87 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.code_intelligence.jazzer.mutation.mutator.proto; + +import static com.code_intelligence.jazzer.mutation.mutator.proto.BuilderAdapters.makeMutableRepeatedFieldView; +import static com.google.common.truth.Truth.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.code_intelligence.jazzer.protobuf.Proto3.RepeatedIntegralField3; +import com.google.protobuf.Descriptors.FieldDescriptor; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import org.junit.jupiter.api.Test; + +class BuilderAdaptersTest { + @Test + void testMakeMutableRepeatedFieldView() { + RepeatedIntegralField3.Builder builder = RepeatedIntegralField3.newBuilder(); + FieldDescriptor someField = builder.getDescriptorForType().findFieldByNumber(1); + assertThat(someField).isNotNull(); + + List<Integer> view = makeMutableRepeatedFieldView(builder, someField); + assertThat(builder.build().getSomeFieldList()).isEmpty(); + + assertThat(view.add(1)).isTrue(); + assertThat(view.get(0)).isEqualTo(1); + assertThat(view).hasSize(1); + assertThat(builder.build().getSomeFieldList()).containsExactly(1).inOrder(); + assertThrows(IndexOutOfBoundsException.class, () -> view.get(1)); + + assertThat(view.add(2)).isTrue(); + assertThat(view.add(3)).isTrue(); + assertThat(view).hasSize(3); + assertThat(builder.build().getSomeFieldList()).containsExactly(1, 2, 3).inOrder(); + assertThrows(IndexOutOfBoundsException.class, () -> view.get(3)); + + assertThat(view.set(1, 4)).isEqualTo(2); + assertThat(view).hasSize(3); + assertThat(builder.build().getSomeFieldList()).containsExactly(1, 4, 3).inOrder(); + + assertThat(view.set(1, 5)).isEqualTo(4); + assertThat(view).hasSize(3); + assertThat(builder.build().getSomeFieldList()).containsExactly(1, 5, 3).inOrder(); + + assertThat(view.remove(1)).isEqualTo(5); + assertThat(view).hasSize(2); + assertThat(builder.build().getSomeFieldList()).containsExactly(1, 3).inOrder(); + + assertThrows(IndexOutOfBoundsException.class, () -> view.remove(-1)); + assertThrows(IndexOutOfBoundsException.class, () -> view.remove(2)); + + assertThat(view.addAll(1, Collections.emptyList())).isFalse(); + assertThat(view).hasSize(2); + assertThat(builder.build().getSomeFieldList()).containsExactly(1, 3).inOrder(); + + assertThat(view.addAll(1, Arrays.asList(6, 7, 8))).isTrue(); + assertThat(view).hasSize(5); + assertThat(builder.build().getSomeFieldList()).containsExactly(1, 6, 7, 8, 3).inOrder(); + + view.subList(2, 4).clear(); + assertThat(view).hasSize(3); + assertThat(builder.build().getSomeFieldList()).containsExactly(1, 6, 3).inOrder(); + + assertThat(view.addAll(3, Arrays.asList(9, 10))).isTrue(); + assertThat(view).hasSize(5); + assertThat(builder.build().getSomeFieldList()).containsExactly(1, 6, 3, 9, 10).inOrder(); + + view.clear(); + assertThat(view).hasSize(0); + assertThat(builder.build().getSomeFieldList()).isEmpty(); + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/mutation/mutator/proto/BuilderMutatorProto2Test.java b/src/test/java/com/code_intelligence/jazzer/mutation/mutator/proto/BuilderMutatorProto2Test.java new file mode 100644 index 00000000..9492bcec --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/mutation/mutator/proto/BuilderMutatorProto2Test.java @@ -0,0 +1,447 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.code_intelligence.jazzer.mutation.mutator.proto; + +import static com.code_intelligence.jazzer.mutation.support.TestSupport.mockPseudoRandom; +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.extensions.proto.ProtoTruth.assertThat; + +import com.code_intelligence.jazzer.mutation.annotation.NotNull; +import com.code_intelligence.jazzer.mutation.api.ChainedMutatorFactory; +import com.code_intelligence.jazzer.mutation.api.InPlaceMutator; +import com.code_intelligence.jazzer.mutation.api.MutatorFactory; +import com.code_intelligence.jazzer.mutation.mutator.collection.CollectionMutators; +import com.code_intelligence.jazzer.mutation.mutator.lang.LangMutators; +import com.code_intelligence.jazzer.mutation.support.TestSupport.MockPseudoRandom; +import com.code_intelligence.jazzer.mutation.support.TypeHolder; +import com.code_intelligence.jazzer.protobuf.Proto2.MessageField2; +import com.code_intelligence.jazzer.protobuf.Proto2.OneOfField2; +import com.code_intelligence.jazzer.protobuf.Proto2.PrimitiveField2; +import com.code_intelligence.jazzer.protobuf.Proto2.RecursiveMessageField2; +import com.code_intelligence.jazzer.protobuf.Proto2.RepeatedMessageField2; +import com.code_intelligence.jazzer.protobuf.Proto2.RepeatedOptionalMessageField2; +import com.code_intelligence.jazzer.protobuf.Proto2.RepeatedPrimitiveField2; +import com.code_intelligence.jazzer.protobuf.Proto2.RequiredPrimitiveField2; +import org.junit.jupiter.api.Test; + +class BuilderMutatorProto2Test { + private static final MutatorFactory FACTORY = new ChainedMutatorFactory( + LangMutators.newFactory(), CollectionMutators.newFactory(), ProtoMutators.newFactory()); + + @Test + void testPrimitiveField() { + InPlaceMutator<PrimitiveField2.Builder> mutator = + (InPlaceMutator<PrimitiveField2.Builder>) FACTORY.createInPlaceOrThrow( + new TypeHolder<PrimitiveField2.@NotNull Builder>() {}.annotatedType()); + assertThat(mutator.toString()).isEqualTo("{Builder.Nullable<Boolean>}"); + + PrimitiveField2.Builder builder = PrimitiveField2.newBuilder(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // present + false, + // boolean + false)) { + mutator.initInPlace(builder, prng); + } + assertThat(builder.hasSomeField()).isTrue(); + assertThat(builder.getSomeField()).isFalse(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // present + false, + // boolean + true)) { + mutator.initInPlace(builder, prng); + } + assertThat(builder.hasSomeField()).isTrue(); + assertThat(builder.getSomeField()).isTrue(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // mutate first field + 0, + // mutate as non-null Boolean + false)) { + mutator.mutateInPlace(builder, prng); + } + assertThat(builder.hasSomeField()).isTrue(); + assertThat(builder.getSomeField()).isFalse(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // not present + true)) { + mutator.initInPlace(builder, prng); + } + assertThat(builder.hasSomeField()).isFalse(); + assertThat(builder.getSomeField()).isFalse(); + } + + @Test + void testRequiredPrimitiveField() { + InPlaceMutator<RequiredPrimitiveField2.Builder> mutator = + (InPlaceMutator<RequiredPrimitiveField2.Builder>) FACTORY.createInPlaceOrThrow( + new TypeHolder<RequiredPrimitiveField2.@NotNull Builder>() {}.annotatedType()); + assertThat(mutator.toString()).isEqualTo("{Builder.Boolean}"); + + RequiredPrimitiveField2.Builder builder = RequiredPrimitiveField2.newBuilder(); + + try (MockPseudoRandom prng = mockPseudoRandom(true)) { + mutator.initInPlace(builder, prng); + } + assertThat(builder.getSomeField()).isTrue(); + + try (MockPseudoRandom prng = mockPseudoRandom(/* mutate first field */ 0)) { + mutator.mutateInPlace(builder, prng); + } + assertThat(builder.getSomeField()).isFalse(); + } + + @Test + void testRepeatedPrimitiveField() { + InPlaceMutator<RepeatedPrimitiveField2.Builder> mutator = + (InPlaceMutator<RepeatedPrimitiveField2.Builder>) FACTORY.createInPlaceOrThrow( + new TypeHolder<RepeatedPrimitiveField2.@NotNull Builder>() {}.annotatedType()); + assertThat(mutator.toString()).isEqualTo("{Builder via List<Boolean>}"); + + RepeatedPrimitiveField2.Builder builder = RepeatedPrimitiveField2.newBuilder(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // list size 1 + 1, + // boolean, + true)) { + mutator.initInPlace(builder, prng); + } + assertThat(builder.getSomeFieldList()).containsExactly(true).inOrder(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // mutate first field + 0, + // mutate the list itself by adding an entry + 1, + // add a single element + 1, + // add the element at the end + 1, + // value to add + true)) { + mutator.mutateInPlace(builder, prng); + } + assertThat(builder.getSomeFieldList()).containsExactly(true, true).inOrder(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // mutate first field + 0, + // mutate the list itself by changing an entry + 2, + // mutate a single element + 1, + // mutate the second element + 1)) { + mutator.mutateInPlace(builder, prng); + } + assertThat(builder.getSomeFieldList()).containsExactly(true, false).inOrder(); + } + + @Test + void testMessageField() { + InPlaceMutator<MessageField2.Builder> mutator = + (InPlaceMutator<MessageField2.Builder>) FACTORY.createInPlaceOrThrow( + new TypeHolder<MessageField2.@NotNull Builder>() {}.annotatedType()); + assertThat(mutator.toString()).isEqualTo("{Builder.Nullable<{Builder.Boolean} -> Message>}"); + + MessageField2.Builder builder = MessageField2.newBuilder(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // init submessage + false, + // boolean submessage field + true)) { + mutator.initInPlace(builder, prng); + } + + assertThat(builder.getMessageField()) + .isEqualTo(RequiredPrimitiveField2.newBuilder().setSomeField(true).build()); + assertThat(builder.hasMessageField()).isTrue(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // mutate first field + 0, + // mutate submessage as non-null + false, + // mutate first field + 0)) { + mutator.mutateInPlace(builder, prng); + } + assertThat(builder.getMessageField()) + .isEqualTo(RequiredPrimitiveField2.newBuilder().setSomeField(false).build()); + assertThat(builder.hasMessageField()).isTrue(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // mutate first field + 0, + // mutate submessage to null + true)) { + mutator.mutateInPlace(builder, prng); + } + assertThat(builder.hasMessageField()).isFalse(); + } + + @Test + void testRepeatedOptionalMessageField() { + InPlaceMutator<RepeatedOptionalMessageField2.Builder> mutator = + (InPlaceMutator<RepeatedOptionalMessageField2.Builder>) FACTORY.createInPlaceOrThrow( + new TypeHolder<RepeatedOptionalMessageField2.@NotNull Builder>() {}.annotatedType()); + assertThat(mutator.toString()) + .isEqualTo("{Builder via List<{Builder.Nullable<Boolean>} -> Message>}"); + + RepeatedOptionalMessageField2.Builder builder = RepeatedOptionalMessageField2.newBuilder(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // list size 1 + 1, + // boolean + true)) { + mutator.initInPlace(builder, prng); + } + assertThat(builder.getMessageFieldList().toString()).isEqualTo("[]"); + + try (MockPseudoRandom prng = mockPseudoRandom( + // mutate first field + 0, + // mutate the list itself by adding an entry + 1, + // add a single element + 1, + // add the element at the end + 1, + // Nullable mutator init + false, + // duplicate entry + true)) { + mutator.mutateInPlace(builder, prng); + } + assertThat(builder.getMessageFieldList().size()).isEqualTo(2); + } + + @Test + void testRepeatedRequiredMessageField() { + InPlaceMutator<RepeatedMessageField2.Builder> mutator = + (InPlaceMutator<RepeatedMessageField2.Builder>) FACTORY.createInPlaceOrThrow( + new TypeHolder<RepeatedMessageField2.@NotNull Builder>() {}.annotatedType()); + assertThat(mutator.toString()).isEqualTo("{Builder via List<{Builder.Boolean} -> Message>}"); + + RepeatedMessageField2.Builder builder = RepeatedMessageField2.newBuilder(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // list size 1 + 1, + // boolean + true)) { + mutator.initInPlace(builder, prng); + } + assertThat(builder.getMessageFieldList()) + .containsExactly(RequiredPrimitiveField2.newBuilder().setSomeField(true).build()) + .inOrder(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // mutate first field + 0, + // mutate the list itself by adding an entry + 1, + // add a single element + 1, + // add the element at the end + 1, + // value to add + true)) { + mutator.mutateInPlace(builder, prng); + } + assertThat(builder.getMessageFieldList()) + .containsExactly(RequiredPrimitiveField2.newBuilder().setSomeField(true).build(), + RequiredPrimitiveField2.newBuilder().setSomeField(true).build()) + .inOrder(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // mutate first field + 0, + // change an entry + 2, + // mutate a single element + 1, + // mutate the second element, + 1, + // mutate the first element + 0)) { + mutator.mutateInPlace(builder, prng); + } + assertThat(builder.getMessageFieldList()) + .containsExactly(RequiredPrimitiveField2.newBuilder().setSomeField(true).build(), + RequiredPrimitiveField2.newBuilder().setSomeField(false).build()) + .inOrder(); + } + + @Test + void testRecursiveMessageField() { + InPlaceMutator<RecursiveMessageField2.Builder> mutator = + (InPlaceMutator<RecursiveMessageField2.Builder>) FACTORY.createInPlaceOrThrow( + new TypeHolder<RecursiveMessageField2.@NotNull Builder>() {}.annotatedType()); + assertThat(mutator.toString()) + .isEqualTo("{Builder.Boolean, WithoutInit(Builder.Nullable<(cycle) -> Message>)}"); + RecursiveMessageField2.Builder builder = RecursiveMessageField2.newBuilder(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // boolean + true)) { + mutator.initInPlace(builder, prng); + } + + assertThat(builder.build()) + .isEqualTo(RecursiveMessageField2.newBuilder().setSomeField(true).build()); + assertThat(builder.hasMessageField()).isFalse(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // mutate message field (causes init to non-null) + 1, + // bool field in message field + false)) { + mutator.mutateInPlace(builder, prng); + } + // Nested message field *is* set explicitly and implicitly equal to the default + // instance. + assertThat(builder.build()) + .isEqualTo(RecursiveMessageField2.newBuilder() + .setSomeField(true) + .setMessageField(RecursiveMessageField2.newBuilder().setSomeField(false)) + .build()); + assertThat(builder.hasMessageField()).isTrue(); + assertThat(builder.getMessageField().hasMessageField()).isFalse(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // mutate message field + 1, + // message field as not null + false, + // mutate message field + 1, + // nested boolean, + true)) { + mutator.mutateInPlace(builder, prng); + } + assertThat(builder.build()) + .isEqualTo(RecursiveMessageField2.newBuilder() + .setSomeField(true) + .setMessageField( + RecursiveMessageField2.newBuilder().setSomeField(false).setMessageField( + RecursiveMessageField2.newBuilder().setSomeField(true))) + .build()); + assertThat(builder.hasMessageField()).isTrue(); + assertThat(builder.getMessageField().hasMessageField()).isTrue(); + assertThat(builder.getMessageField().getMessageField().hasMessageField()).isFalse(); + } + + @Test + void testOneOfField2() { + InPlaceMutator<OneOfField2.Builder> mutator = + (InPlaceMutator<OneOfField2.Builder>) FACTORY.createInPlaceOrThrow( + new TypeHolder<OneOfField2.@NotNull Builder>() {}.annotatedType()); + assertThat(mutator.toString()) + .isEqualTo( + "{Builder.Boolean, Builder.Nullable<Boolean>, Builder.Nullable<Boolean> | Builder.Nullable<{Builder.Boolean} -> Message>}"); + OneOfField2.Builder builder = OneOfField2.newBuilder(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // other_field + true, + // yet_another_field + true, + // oneof: first field + 0, + // bool_field present + false, + // bool_field + true)) { + mutator.initInPlace(builder, prng); + } + assertThat(builder.build()) + .isEqualTo(OneOfField2.newBuilder().setOtherField(true).setBoolField(true).build()); + assertThat(builder.build().hasBoolField()).isTrue(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // mutate oneof + 2, + // preserve oneof state + false, + // mutate bool_field as non-null + false)) { + mutator.mutateInPlace(builder, prng); + } + assertThat(builder.build()) + .isEqualTo(OneOfField2.newBuilder().setOtherField(true).setBoolField(false).build()); + assertThat(builder.build().hasBoolField()).isTrue(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // mutate oneof + 2, + // switch oneof state + true, + // new oneof state + 1, + // init message_field as non-null + false, + // init some_field as true + true)) { + mutator.mutateInPlace(builder, prng); + } + assertThat(builder.build()) + .isEqualTo(OneOfField2.newBuilder() + .setOtherField(true) + .setMessageField(RequiredPrimitiveField2.newBuilder().setSomeField(true)) + .build()); + assertThat(builder.build().hasMessageField()).isTrue(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // mutate oneof + 2, + // preserve oneof state + false, + // mutate message_field as non-null + false, + // mutate some_field + 0)) { + mutator.mutateInPlace(builder, prng); + } + assertThat(builder.build()) + .isEqualTo(OneOfField2.newBuilder() + .setOtherField(true) + .setMessageField(RequiredPrimitiveField2.newBuilder().setSomeField(false)) + .build()); + assertThat(builder.build().hasMessageField()).isTrue(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // mutate oneof + 2, + // preserve oneof state + false, + // mutate message_field to null (and thus oneof state to indeterminate) + true)) { + mutator.mutateInPlace(builder, prng); + } + assertThat(builder.build()).isEqualTo(OneOfField2.newBuilder().setOtherField(true).build()); + assertThat(builder.build().hasMessageField()).isFalse(); + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/mutation/mutator/proto/BuilderMutatorProto3Test.java b/src/test/java/com/code_intelligence/jazzer/mutation/mutator/proto/BuilderMutatorProto3Test.java new file mode 100644 index 00000000..ff298540 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/mutation/mutator/proto/BuilderMutatorProto3Test.java @@ -0,0 +1,603 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.code_intelligence.jazzer.mutation.mutator.proto; + +import static com.code_intelligence.jazzer.mutation.support.TestSupport.mockPseudoRandom; +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.extensions.proto.ProtoTruth.assertThat; + +import com.code_intelligence.jazzer.mutation.annotation.NotNull; +import com.code_intelligence.jazzer.mutation.annotation.proto.AnySource; +import com.code_intelligence.jazzer.mutation.api.ChainedMutatorFactory; +import com.code_intelligence.jazzer.mutation.api.InPlaceMutator; +import com.code_intelligence.jazzer.mutation.api.MutatorFactory; +import com.code_intelligence.jazzer.mutation.mutator.collection.CollectionMutators; +import com.code_intelligence.jazzer.mutation.mutator.lang.LangMutators; +import com.code_intelligence.jazzer.mutation.support.TestSupport.MockPseudoRandom; +import com.code_intelligence.jazzer.mutation.support.TypeHolder; +import com.code_intelligence.jazzer.protobuf.Proto3.AnyField3; +import com.code_intelligence.jazzer.protobuf.Proto3.AnyField3.Builder; +import com.code_intelligence.jazzer.protobuf.Proto3.EmptyMessage3; +import com.code_intelligence.jazzer.protobuf.Proto3.EnumField3; +import com.code_intelligence.jazzer.protobuf.Proto3.EnumField3.TestEnum; +import com.code_intelligence.jazzer.protobuf.Proto3.EnumFieldOne3; +import com.code_intelligence.jazzer.protobuf.Proto3.EnumFieldOne3.TestEnumOne; +import com.code_intelligence.jazzer.protobuf.Proto3.EnumFieldOutside3; +import com.code_intelligence.jazzer.protobuf.Proto3.EnumFieldRepeated3; +import com.code_intelligence.jazzer.protobuf.Proto3.EnumFieldRepeated3.TestEnumRepeated; +import com.code_intelligence.jazzer.protobuf.Proto3.MessageField3; +import com.code_intelligence.jazzer.protobuf.Proto3.OneOfField3; +import com.code_intelligence.jazzer.protobuf.Proto3.OptionalPrimitiveField3; +import com.code_intelligence.jazzer.protobuf.Proto3.PrimitiveField3; +import com.code_intelligence.jazzer.protobuf.Proto3.RecursiveMessageField3; +import com.code_intelligence.jazzer.protobuf.Proto3.RepeatedMessageField3; +import com.code_intelligence.jazzer.protobuf.Proto3.RepeatedPrimitiveField3; +import com.code_intelligence.jazzer.protobuf.Proto3.TestEnumOutside3; +import com.google.protobuf.InvalidProtocolBufferException; +import java.util.Arrays; +import org.junit.jupiter.api.Test; + +class BuilderMutatorProto3Test { + private static final MutatorFactory FACTORY = new ChainedMutatorFactory( + LangMutators.newFactory(), CollectionMutators.newFactory(), ProtoMutators.newFactory()); + + @Test + void testPrimitiveField() { + InPlaceMutator<PrimitiveField3.Builder> mutator = + (InPlaceMutator<PrimitiveField3.Builder>) FACTORY.createInPlaceOrThrow( + new TypeHolder<PrimitiveField3.@NotNull Builder>() {}.annotatedType()); + assertThat(mutator.toString()).isEqualTo("{Builder.Boolean}"); + + PrimitiveField3.Builder builder = PrimitiveField3.newBuilder(); + + try (MockPseudoRandom prng = mockPseudoRandom(true)) { + mutator.initInPlace(builder, prng); + } + assertThat(builder.getSomeField()).isTrue(); + + try (MockPseudoRandom prng = mockPseudoRandom(/* mutate first field */ 0)) { + mutator.mutateInPlace(builder, prng); + } + assertThat(builder.getSomeField()).isFalse(); + } + + @Test + void testEnumField() { + InPlaceMutator<EnumField3.Builder> mutator = + (InPlaceMutator<EnumField3.Builder>) FACTORY.createInPlaceOrThrow( + new TypeHolder<EnumField3.@NotNull Builder>() {}.annotatedType()); + assertThat(mutator.toString()).isEqualTo("{Builder.Enum<TestEnum>}"); + EnumField3.Builder builder = EnumField3.newBuilder(); + try (MockPseudoRandom prng = mockPseudoRandom(0)) { + mutator.initInPlace(builder, prng); + } + assertThat(builder.getSomeField()).isEqualTo(TestEnum.VAL1); + try (MockPseudoRandom prng = mockPseudoRandom(0, 1)) { + mutator.mutateInPlace(builder, prng); + } + assertThat(builder.getSomeField()).isEqualTo(TestEnum.VAL2); + } + + @Test + void testEnumFieldOutside() { + InPlaceMutator<EnumFieldOutside3.Builder> mutator = + (InPlaceMutator<EnumFieldOutside3.Builder>) FACTORY.createInPlaceOrThrow( + new TypeHolder<EnumFieldOutside3.@NotNull Builder>() {}.annotatedType()); + assertThat(mutator.toString()).isEqualTo("{Builder.Enum<TestEnumOutside3>}"); + EnumFieldOutside3.Builder builder = EnumFieldOutside3.newBuilder(); + try (MockPseudoRandom prng = mockPseudoRandom(0)) { + mutator.initInPlace(builder, prng); + } + assertThat(builder.getSomeField()).isEqualTo(TestEnumOutside3.VAL1); + try (MockPseudoRandom prng = mockPseudoRandom(0, 2)) { + mutator.mutateInPlace(builder, prng); + } + assertThat(builder.getSomeField()).isEqualTo(TestEnumOutside3.VAL3); + } + + @Test + void testEnumFieldWithOneValue() { + InPlaceMutator<EnumFieldOne3.Builder> mutator = + (InPlaceMutator<EnumFieldOne3.Builder>) FACTORY.createInPlaceOrThrow( + new TypeHolder<EnumFieldOne3.@NotNull Builder>() {}.annotatedType()); + assertThat(mutator.toString()).isEqualTo("{Builder.FixedValue(ONE)}"); + EnumFieldOne3.Builder builder = EnumFieldOne3.newBuilder(); + try (MockPseudoRandom prng = mockPseudoRandom()) { + mutator.initInPlace(builder, prng); + } + assertThat(builder.getSomeField()).isEqualTo(TestEnumOne.ONE); + try (MockPseudoRandom prng = mockPseudoRandom(0)) { + mutator.mutateInPlace(builder, prng); + } + assertThat(builder.getSomeField()).isEqualTo(TestEnumOne.ONE); + } + + @Test + void testRepeatedEnumField() { + InPlaceMutator<EnumFieldRepeated3.Builder> mutator = + (InPlaceMutator<EnumFieldRepeated3.Builder>) FACTORY.createInPlaceOrThrow( + new TypeHolder<EnumFieldRepeated3.@NotNull Builder>() {}.annotatedType()); + assertThat(mutator.toString()).isEqualTo("{Builder via List<Enum<TestEnumRepeated>>}"); + EnumFieldRepeated3.Builder builder = EnumFieldRepeated3.newBuilder(); + try (MockPseudoRandom prng = mockPseudoRandom( + // list size + 1, // Only possible start value + // enum values + 2)) { + mutator.initInPlace(builder, prng); + } + assertThat(builder.getSomeFieldList()).isEqualTo(Arrays.asList(TestEnumRepeated.VAL2)); + + try (MockPseudoRandom prng = mockPseudoRandom( + // mutate first field + 0, + // change an entry + 2, + // mutate a single element + 1, + // mutate to first enum field + 0, + // mutate to first enum value + 1)) { + mutator.mutateInPlace(builder, prng); + } + assertThat(builder.getSomeFieldList()).isEqualTo(Arrays.asList(TestEnumRepeated.VAL1)); + } + + @Test + void testOptionalPrimitiveField() { + InPlaceMutator<OptionalPrimitiveField3.Builder> mutator = + (InPlaceMutator<OptionalPrimitiveField3.Builder>) FACTORY.createInPlaceOrThrow( + new TypeHolder<OptionalPrimitiveField3.@NotNull Builder>() {}.annotatedType()); + assertThat(mutator.toString()).isEqualTo("{Builder.Nullable<Boolean>}"); + + OptionalPrimitiveField3.Builder builder = OptionalPrimitiveField3.newBuilder(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // present + false, + // boolean + false)) { + mutator.initInPlace(builder, prng); + } + assertThat(builder.hasSomeField()).isTrue(); + assertThat(builder.getSomeField()).isFalse(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // present + false, + // boolean + true)) { + mutator.initInPlace(builder, prng); + } + assertThat(builder.hasSomeField()).isTrue(); + assertThat(builder.getSomeField()).isTrue(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // mutate first field + 0, + // mutate as non-null Boolean + false)) { + mutator.mutateInPlace(builder, prng); + } + assertThat(builder.hasSomeField()).isTrue(); + assertThat(builder.getSomeField()).isFalse(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // not present + true)) { + mutator.initInPlace(builder, prng); + } + assertThat(builder.hasSomeField()).isFalse(); + assertThat(builder.getSomeField()).isFalse(); + } + + @Test + void testRepeatedPrimitiveField() { + InPlaceMutator<RepeatedPrimitiveField3.Builder> mutator = + (InPlaceMutator<RepeatedPrimitiveField3.Builder>) FACTORY.createInPlaceOrThrow( + new TypeHolder<RepeatedPrimitiveField3.@NotNull Builder>() {}.annotatedType()); + assertThat(mutator.toString()).isEqualTo("{Builder via List<Boolean>}"); + + RepeatedPrimitiveField3.Builder builder = RepeatedPrimitiveField3.newBuilder(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // list size 1 + 1, + // boolean, + true)) { + mutator.initInPlace(builder, prng); + } + assertThat(builder.getSomeFieldList()).containsExactly(true).inOrder(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // mutate first field + 0, + // mutate the list itself by adding an entry + 1, + // add a single element + 1, + // add the element at the end + 1, + // value to add + true)) { + mutator.mutateInPlace(builder, prng); + } + assertThat(builder.getSomeFieldList()).containsExactly(true, true).inOrder(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // mutate first field + 0, + // mutate the list itself by changing an entry + 2, + // mutate a single element + 1, + // mutate the second element + 1)) { + mutator.mutateInPlace(builder, prng); + } + assertThat(builder.getSomeFieldList()).containsExactly(true, false).inOrder(); + } + + @Test + void testMessageField() { + InPlaceMutator<MessageField3.Builder> mutator = + (InPlaceMutator<MessageField3.Builder>) FACTORY.createInPlaceOrThrow( + new TypeHolder<MessageField3.@NotNull Builder>() {}.annotatedType()); + assertThat(mutator.toString()).isEqualTo("{Builder.Nullable<{Builder.Boolean} -> Message>}"); + + MessageField3.Builder builder = MessageField3.newBuilder(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // init submessage + false, + // boolean submessage field + true)) { + mutator.initInPlace(builder, prng); + } + assertThat(builder.getMessageField()) + .isEqualTo(PrimitiveField3.newBuilder().setSomeField(true).build()); + assertThat(builder.hasMessageField()).isTrue(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // mutate first field + 0, + // mutate submessage as non-null + false, + // mutate first field + 0)) { + mutator.mutateInPlace(builder, prng); + } + assertThat(builder.getMessageField()) + .isEqualTo(PrimitiveField3.newBuilder().setSomeField(false).build()); + assertThat(builder.hasMessageField()).isTrue(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // mutate first field + 0, + // mutate submessage to null + true)) { + mutator.mutateInPlace(builder, prng); + } + assertThat(builder.hasMessageField()).isFalse(); + } + + @Test + void testRepeatedMessageField() { + InPlaceMutator<RepeatedMessageField3.Builder> mutator = + (InPlaceMutator<RepeatedMessageField3.Builder>) FACTORY.createInPlaceOrThrow( + new TypeHolder<RepeatedMessageField3.@NotNull Builder>() {}.annotatedType()); + assertThat(mutator.toString()).isEqualTo("{Builder via List<{Builder.Boolean} -> Message>}"); + + RepeatedMessageField3.Builder builder = RepeatedMessageField3.newBuilder(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // list size 1 + 1, + // boolean + true)) { + mutator.initInPlace(builder, prng); + } + assertThat(builder.getMessageFieldList()) + .containsExactly(PrimitiveField3.newBuilder().setSomeField(true).build()) + .inOrder(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // mutate first field + 0, + // mutate the list itself by adding an entry + 1, + // add a single element + 1, + // add the element at the end + 1, + // value to add + true)) { + mutator.mutateInPlace(builder, prng); + } + assertThat(builder.getMessageFieldList()) + .containsExactly(PrimitiveField3.newBuilder().setSomeField(true).build(), + PrimitiveField3.newBuilder().setSomeField(true).build()) + .inOrder(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // mutate first field + 0, + // change an entry + 2, + // mutate a single element + 1, + // mutate the second element, + 1, + // mutate the first element + 0)) { + mutator.mutateInPlace(builder, prng); + } + assertThat(builder.getMessageFieldList()) + .containsExactly(PrimitiveField3.newBuilder().setSomeField(true).build(), + PrimitiveField3.newBuilder().setSomeField(false).build()) + .inOrder(); + } + + @Test + void testRecursiveMessageField() { + InPlaceMutator<RecursiveMessageField3.Builder> mutator = + (InPlaceMutator<RecursiveMessageField3.Builder>) FACTORY.createInPlaceOrThrow( + new TypeHolder<RecursiveMessageField3.@NotNull Builder>() {}.annotatedType()); + assertThat(mutator.toString()) + .isEqualTo("{Builder.Boolean, WithoutInit(Builder.Nullable<(cycle) -> Message>)}"); + RecursiveMessageField3.Builder builder = RecursiveMessageField3.newBuilder(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // boolean + true)) { + mutator.initInPlace(builder, prng); + } + + assertThat(builder.build()) + .isEqualTo(RecursiveMessageField3.newBuilder().setSomeField(true).build()); + assertThat(builder.hasMessageField()).isFalse(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // mutate message field (causes init to non-null) + 1, + // bool field in message field + false)) { + mutator.mutateInPlace(builder, prng); + } + // Nested message field *is* set explicitly and implicitly equal to the default + // instance. + assertThat(builder.build()) + .isEqualTo(RecursiveMessageField3.newBuilder() + .setSomeField(true) + .setMessageField(RecursiveMessageField3.newBuilder().setSomeField(false)) + .build()); + assertThat(builder.hasMessageField()).isTrue(); + assertThat(builder.getMessageField().hasMessageField()).isFalse(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // mutate message field + 1, + // message field as not null + false, + // mutate message field + 1, + // nested boolean, + true)) { + mutator.mutateInPlace(builder, prng); + } + assertThat(builder.build()) + .isEqualTo(RecursiveMessageField3.newBuilder() + .setSomeField(true) + .setMessageField( + RecursiveMessageField3.newBuilder().setSomeField(false).setMessageField( + RecursiveMessageField3.newBuilder().setSomeField(true))) + .build()); + assertThat(builder.hasMessageField()).isTrue(); + assertThat(builder.getMessageField().hasMessageField()).isTrue(); + assertThat(builder.getMessageField().getMessageField().hasMessageField()).isFalse(); + } + + @Test + void testOneOfField3() { + InPlaceMutator<OneOfField3.Builder> mutator = + (InPlaceMutator<OneOfField3.Builder>) FACTORY.createInPlaceOrThrow( + new TypeHolder<OneOfField3.@NotNull Builder>() {}.annotatedType()); + assertThat(mutator.toString()) + .isEqualTo( + "{Builder.Boolean, Builder.Boolean, Builder.Nullable<Boolean> | Builder.Nullable<{Builder.Boolean} -> Message>}"); + OneOfField3.Builder builder = OneOfField3.newBuilder(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // other_field + true, + // yet_another_field + true, + // oneof: first field + 0, + // bool_field present + false, + // bool_field + true)) { + mutator.initInPlace(builder, prng); + } + assertThat(builder.build()) + .isEqualTo(OneOfField3.newBuilder() + .setOtherField(true) + .setBoolField(true) + .setYetAnotherField(true) + .build()); + assertThat(builder.build().hasBoolField()).isTrue(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // mutate oneof + 2, + // preserve oneof state + false, + // mutate bool_field as non-null + false)) { + mutator.mutateInPlace(builder, prng); + } + assertThat(builder.build()) + .isEqualTo(OneOfField3.newBuilder() + .setOtherField(true) + .setBoolField(false) + .setYetAnotherField(true) + .build()); + assertThat(builder.build().hasBoolField()).isTrue(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // mutate oneof + 2, + // switch oneof state + true, + // new oneof state + 1, + // init message_field as non-null + false, + // init some_field as true + true)) { + mutator.mutateInPlace(builder, prng); + } + assertThat(builder.build()) + .isEqualTo(OneOfField3.newBuilder() + .setOtherField(true) + .setMessageField(PrimitiveField3.newBuilder().setSomeField(true)) + .setYetAnotherField(true) + .build()); + assertThat(builder.build().hasMessageField()).isTrue(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // mutate oneof + 2, + // preserve oneof state + false, + // mutate message_field as non-null + false, + // mutate some_field + 0)) { + mutator.mutateInPlace(builder, prng); + } + assertThat(builder.build()) + .isEqualTo(OneOfField3.newBuilder() + .setOtherField(true) + .setMessageField(PrimitiveField3.newBuilder().setSomeField(false)) + .setYetAnotherField(true) + .build()); + assertThat(builder.build().hasMessageField()).isTrue(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // mutate oneof + 2, + // preserve oneof state + false, + // mutate message_field to null (and thus oneof state to indeterminate) + true)) { + mutator.mutateInPlace(builder, prng); + } + assertThat(builder.build()) + .isEqualTo(OneOfField3.newBuilder().setOtherField(true).setYetAnotherField(true).build()); + assertThat(builder.build().hasMessageField()).isFalse(); + } + + @Test + void testEmptyMessage3() { + InPlaceMutator<EmptyMessage3.Builder> mutator = + (InPlaceMutator<EmptyMessage3.Builder>) FACTORY.createInPlaceOrThrow( + new TypeHolder<EmptyMessage3.@NotNull Builder>() {}.annotatedType()); + assertThat(mutator.toString()).isEqualTo("{<empty>}"); + EmptyMessage3.Builder builder = EmptyMessage3.newBuilder(); + + try (MockPseudoRandom prng = mockPseudoRandom()) { + mutator.initInPlace(builder, prng); + } + assertThat(builder.build()).isEqualTo(EmptyMessage3.getDefaultInstance()); + + try (MockPseudoRandom prng = mockPseudoRandom()) { + mutator.mutateInPlace(builder, prng); + } + assertThat(builder.build()).isEqualTo(EmptyMessage3.getDefaultInstance()); + } + + @Test + void testAnyField3() throws InvalidProtocolBufferException { + InPlaceMutator<AnyField3.Builder> mutator = + (InPlaceMutator<AnyField3.Builder>) FACTORY.createInPlaceOrThrow( + new TypeHolder<@NotNull @AnySource( + {PrimitiveField3.class, MessageField3.class}) Builder>() { + }.annotatedType()); + assertThat(mutator.toString()) + .isEqualTo( + "{Builder.Nullable<Builder.{Builder.Boolean} -> Message | Builder.{Builder.Nullable<(cycle) -> Message>} -> Message -> Message>}"); + AnyField3.Builder builder = AnyField3.newBuilder(); + + try (MockPseudoRandom prng = mockPseudoRandom( + // initialize message field + false, + // PrimitiveField3 + 0, + // boolean field + true)) { + mutator.initInPlace(builder, prng); + } + assertThat(builder.build().getSomeField().unpack(PrimitiveField3.class)) + .isEqualTo(PrimitiveField3.newBuilder().setSomeField(true).build()); + + try (MockPseudoRandom prng = mockPseudoRandom( + // mutate Any field + 0, + // keep non-null message field + false, + // keep Any state, + false, + // mutate boolean field + 0)) { + mutator.mutateInPlace(builder, prng); + } + assertThat(builder.build().getSomeField().unpack(PrimitiveField3.class)) + .isEqualTo(PrimitiveField3.newBuilder().setSomeField(false).build()); + + try (MockPseudoRandom prng = mockPseudoRandom( + // mutate Any field + 0, + // keep non-null message field + false, + // switch Any state + true, + // new Any state + 1, + // non-null message + false, + // boolean field, + true)) { + mutator.mutateInPlace(builder, prng); + } + assertThat(builder.build().getSomeField().unpack(MessageField3.class)) + .isEqualTo(MessageField3.newBuilder() + .setMessageField(PrimitiveField3.newBuilder().setSomeField(true)) + .build()); + } + + @Test + void testAnyField3WithoutAnySourceDoesNotCrash() throws InvalidProtocolBufferException { + InPlaceMutator<AnyField3.Builder> mutator = + (InPlaceMutator<AnyField3.Builder>) FACTORY.createInPlaceOrThrow( + new TypeHolder<@NotNull Builder>() {}.annotatedType()); + assertThat(mutator.toString()) + .isEqualTo("{Builder.Nullable<{Builder.String, Builder.byte[] -> ByteString} -> Message>}"); + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/mutation/mutator/proto/MessageMutatorTest.java b/src/test/java/com/code_intelligence/jazzer/mutation/mutator/proto/MessageMutatorTest.java new file mode 100644 index 00000000..b804c7fb --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/mutation/mutator/proto/MessageMutatorTest.java @@ -0,0 +1,92 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.code_intelligence.jazzer.mutation.mutator.proto; + +import static com.code_intelligence.jazzer.mutation.support.TestSupport.mockPseudoRandom; +import static com.google.common.truth.Truth.assertThat; + +import com.code_intelligence.jazzer.mutation.annotation.NotNull; +import com.code_intelligence.jazzer.mutation.api.ChainedMutatorFactory; +import com.code_intelligence.jazzer.mutation.api.MutatorFactory; +import com.code_intelligence.jazzer.mutation.api.SerializingMutator; +import com.code_intelligence.jazzer.mutation.mutator.collection.CollectionMutators; +import com.code_intelligence.jazzer.mutation.mutator.lang.LangMutators; +import com.code_intelligence.jazzer.mutation.support.TestSupport.MockPseudoRandom; +import com.code_intelligence.jazzer.mutation.support.TypeHolder; +import com.code_intelligence.jazzer.protobuf.Proto2.ExtendedMessage2; +import com.code_intelligence.jazzer.protobuf.Proto2.ExtendedSubmessage2; +import com.code_intelligence.jazzer.protobuf.Proto2.OriginalMessage2; +import com.code_intelligence.jazzer.protobuf.Proto2.OriginalSubmessage2; +import com.code_intelligence.jazzer.protobuf.Proto3.PrimitiveField3; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import org.junit.jupiter.api.Test; + +class MessageMutatorTest { + private static final MutatorFactory FACTORY = new ChainedMutatorFactory( + LangMutators.newFactory(), CollectionMutators.newFactory(), ProtoMutators.newFactory()); + + @Test + void testSimpleMessage() { + SerializingMutator<PrimitiveField3> mutator = FACTORY.createOrThrow(PrimitiveField3.class); + + PrimitiveField3 msg; + + try (MockPseudoRandom prng = mockPseudoRandom( + // not null + false, + // boolean + false)) { + msg = mutator.init(prng); + assertThat(msg).isEqualTo(PrimitiveField3.getDefaultInstance()); + } + + try (MockPseudoRandom prng = mockPseudoRandom( + // not null, + false, + // mutate first field + 0)) { + msg = mutator.mutate(msg, prng); + assertThat(msg).isNotEqualTo(PrimitiveField3.getDefaultInstance()); + } + } + + @Test + void testIncompleteMessageWithRequiredFields() throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + OriginalMessage2.newBuilder() + .setMessageField(OriginalSubmessage2.newBuilder().setNumericField(42).build()) + .setBoolField(true) + .build() + .writeTo(out); + byte[] bytes = out.toByteArray(); + + SerializingMutator<ExtendedMessage2> mutator = + (SerializingMutator<ExtendedMessage2>) FACTORY.createOrThrow( + new TypeHolder<@NotNull ExtendedMessage2>() {}.annotatedType()); + ExtendedMessage2 extendedMessage = mutator.readExclusive(new ByteArrayInputStream(bytes)); + assertThat(extendedMessage) + .isEqualTo(ExtendedMessage2.newBuilder() + .setMessageField( + ExtendedSubmessage2.newBuilder().setNumericField(42).setMessageField( + OriginalSubmessage2.newBuilder().setNumericField(0).build())) + .setBoolField(true) + .setFloatField(0) + .build()); + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/mutation/mutator/proto/proto2.proto b/src/test/java/com/code_intelligence/jazzer/mutation/mutator/proto/proto2.proto new file mode 100644 index 00000000..ea7c9999 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/mutation/mutator/proto/proto2.proto @@ -0,0 +1,161 @@ +// Copyright 2023 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto2"; + +package com.code_intelligence.jazzer.protobuf; +option java_package = "com.code_intelligence.jazzer.protobuf"; + +message PrimitiveField2 { +optional bool some_field = 1; +} + +message RequiredPrimitiveField2 { +required bool some_field = 1; +} + +message RepeatedPrimitiveField2 { +repeated bool some_field = 1; +} + +message MessageField2 { +optional RequiredPrimitiveField2 message_field = 1; +} + +message RepeatedMessageField2 { +repeated RequiredPrimitiveField2 message_field = 1; +} + +message RepeatedOptionalMessageField2 { +repeated PrimitiveField2 message_field = 1; +} + +message RecursiveMessageField2 { +required bool some_field = 1; +optional RecursiveMessageField2 message_field = 2; +} + +message RepeatedRecursiveMessageField2 { +optional bool some_field = 1; +repeated RepeatedRecursiveMessageField2 message_field = 2; +} + +message OneOfField2 { +required bool other_field = 4; +oneof oneof_field { + bool bool_field = 7; + RequiredPrimitiveField2 message_field = 2; +} +optional bool yet_another_field = 1; +} + +message IntegralField2 { +optional uint32 some_field = 1; +} + +message RepeatedIntegralField2 { +repeated uint32 some_field = 1; +} + +message BytesField2 { +optional bytes some_field = 1; +} + +message StringField2 { +optional string some_field = 1; +} + +message Parent { + optional Child child = 1; +} + +message Child { + optional Parent parent = 1; +} + +// Taken from +// https://github.com/google/fuzztest/blob/c5fde4baee6134c84d4f2b618def9f60c7505151/fuzztest/internal/test_protobuf.proto#L24 +message TestSubProtobuf { + optional int32 subproto_i32 = 1; + repeated int32 subproto_rep_i32 = 2 [packed = true]; + optional TestProtobuf parent = 3; +} + +message TestProtobuf { + enum Enum { + Label1 = 0; + Label2 = 1; + Label3 = 2; + Label4 = 3; + Label5 = 4; + } + + optional bool b = 1; + optional int32 i32 = 2; + optional uint32 u32 = 3; + optional int64 i64 = 4; + optional uint64 u64 = 5; + optional float f = 6; + optional double d = 7; + optional string str = 8; + optional Enum e = 9; + optional TestSubProtobuf subproto = 10; + + repeated bool rep_b = 11; + repeated int32 rep_i32 = 12; + repeated uint32 rep_u32 = 13; + repeated int64 rep_i64 = 14; + repeated uint64 rep_u64 = 15; + repeated float rep_f = 16; + repeated double rep_d = 17; + repeated string rep_str = 18; + repeated Enum rep_e = 19; + repeated TestSubProtobuf rep_subproto = 20; + + oneof oneof_field { + int32 oneof_i32 = 21; + int64 oneof_i64 = 22; + uint32 oneof_u32 = 24; + } + + map<int32, int32> map_field = 25; + + // Special cases + enum EnumOneLabel { + OnlyLabel = 17; + } + optional EnumOneLabel enum_one_label = 100; + message EmptyMessage {} + optional EmptyMessage empty_message = 101; +} + +message OriginalSubmessage2 { + required int32 numeric_field = 1; +} + +message OriginalMessage2 { + required OriginalSubmessage2 message_field = 1; + required bool bool_field = 2; +} + +message ExtendedSubmessage2 { + required int32 numeric_field = 1; + required OriginalSubmessage2 message_field = 2; +} + +message ExtendedMessage2 { + required ExtendedSubmessage2 message_field = 1; + required bool bool_field = 2; + required float float_field = 3; +} diff --git a/src/test/java/com/code_intelligence/jazzer/mutation/mutator/proto/proto3.proto b/src/test/java/com/code_intelligence/jazzer/mutation/mutator/proto/proto3.proto new file mode 100644 index 00000000..7bd6ffeb --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/mutation/mutator/proto/proto3.proto @@ -0,0 +1,144 @@ +// Copyright 2023 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +import "google/protobuf/any.proto"; + +option java_package = "com.code_intelligence.jazzer.protobuf"; + +message PrimitiveField3 { + bool some_field = 1; +} + +message OptionalPrimitiveField3 { + optional bool some_field = 1; +} + +message RepeatedPrimitiveField3 { + repeated bool some_field = 1; +} + +message MessageField3 { + PrimitiveField3 message_field = 1; +} + +message RepeatedMessageField3 { + repeated PrimitiveField3 message_field = 1; +} + +message RecursiveMessageField3 { + bool some_field = 1; + RecursiveMessageField3 message_field = 2; +} + +message RepeatedRecursiveMessageField3 { + bool some_field = 1; + repeated RepeatedRecursiveMessageField3 message_field = 2; +} + +message OneOfField3 { + bool other_field = 4; + oneof oneof_field { + bool bool_field = 7; + PrimitiveField3 message_field = 2; + } + bool yet_another_field = 1; +} + +message IntegralField3 { + uint32 some_field = 1; +} + +message RepeatedIntegralField3 { + repeated uint32 some_field = 1; +} + +message BytesField3 { + bytes some_field = 1; +} + +message StringField3 { + string some_field = 1; +} + +message EnumField3 { + enum TestEnum { + VAL1 = 0; + VAL2 = 1; + } + TestEnum some_field = 1; +} + +enum TestEnumOutside3 { + VAL1 = 0; + VAL2 = 1; + VAL3 = 3; +} + +message EnumFieldOutside3 { + TestEnumOutside3 some_field = 1; +} + +message EnumFieldOne3 { + enum TestEnumOne { + ONE = 0; + } + TestEnumOne some_field = 1; +} + +message EnumFieldRepeated3 { + enum TestEnumRepeated { + UNASSIGNED = 0; + VAL1 = 1; + VAL2 = 2; + } + repeated TestEnumRepeated some_field = 1; +} + +message MapField3 { + map<int32, string> some_field = 1; +} + +message MessageMapField3 { + map<string, MapField3> some_field = 1; +} + +message FloatField3 { + float some_field = 1; +} + +message RepeatedFloatField3 { + repeated float some_field = 1; +} + +message DoubleField3 { + double some_field = 1; +} + +message RepeatedDoubleField3 { + repeated double some_field = 1; +} + +message EmptyMessage3 {} + +message AnyField3 { + google.protobuf.Any some_field = 1; +} + +message SingleOptionOneOfField3 { + oneof oneof_field { + bool bool_field = 1; + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/mutation/support/BUILD.bazel b/src/test/java/com/code_intelligence/jazzer/mutation/support/BUILD.bazel new file mode 100644 index 00000000..bcde8ba9 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/mutation/support/BUILD.bazel @@ -0,0 +1,35 @@ +load("@contrib_rules_jvm//java:defs.bzl", "JUNIT5_DEPS", "java_test_suite") + +java_library( + name = "test_support", + testonly = True, + srcs = ["TestSupport.java"], + visibility = ["//src/test/java/com/code_intelligence/jazzer/mutation:__subpackages__"], + exports = JUNIT5_DEPS + [ + # keep sorted + "@maven//:com_google_truth_extensions_truth_java8_extension", + "@maven//:com_google_truth_extensions_truth_proto_extension", + "@maven//:com_google_truth_truth", + "@maven//:org_junit_jupiter_junit_jupiter_api", + "@maven//:org_junit_jupiter_junit_jupiter_params", + ], + deps = [ + "//src/main/java/com/code_intelligence/jazzer/mutation/api", + "//src/main/java/com/code_intelligence/jazzer/mutation/engine", + "//src/main/java/com/code_intelligence/jazzer/mutation/support", + "@com_google_errorprone_error_prone_annotations//jar", + "@maven//:com_google_truth_truth", + ], +) + +java_test_suite( + name = "SupportTests", + size = "small", + srcs = glob(["*Test.java"]), + runner = "junit5", + deps = [ + ":test_support", + "//src/main/java/com/code_intelligence/jazzer/mutation/annotation", + "//src/main/java/com/code_intelligence/jazzer/mutation/support", + ], +) diff --git a/src/test/java/com/code_intelligence/jazzer/mutation/support/ExceptionSupportTest.java b/src/test/java/com/code_intelligence/jazzer/mutation/support/ExceptionSupportTest.java new file mode 100644 index 00000000..630b7cdf --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/mutation/support/ExceptionSupportTest.java @@ -0,0 +1,43 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.code_intelligence.jazzer.mutation.support; + +import static com.code_intelligence.jazzer.mutation.support.ExceptionSupport.asUnchecked; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.io.IOException; +import org.junit.jupiter.api.Test; + +class ExceptionSupportTest { + @Test + void testAsUnchecked_withUncheckedException() { + assertThrows(IllegalStateException.class, () -> { + // noinspection TrivialFunctionalExpressionUsage + ((Runnable) () -> { throw asUnchecked(new IllegalStateException()); }).run(); + }); + } + + @Test + void testAsUnchecked_withCheckedException() { + assertThrows(IOException.class, () -> { + // Verify that asUnchecked can be used to throw a checked exception in a function that doesn't + // declare it as being thrown. + // noinspection TrivialFunctionalExpressionUsage + ((Runnable) () -> { throw asUnchecked(new IOException()); }).run(); + }); + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/mutation/support/HolderTest.java b/src/test/java/com/code_intelligence/jazzer/mutation/support/HolderTest.java new file mode 100644 index 00000000..97450e57 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/mutation/support/HolderTest.java @@ -0,0 +1,114 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.code_intelligence.jazzer.mutation.support; + +import static com.google.common.truth.Truth.assertThat; + +import java.lang.annotation.Annotation; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import java.lang.reflect.AnnotatedParameterizedType; +import java.lang.reflect.AnnotatedType; +import java.lang.reflect.ParameterizedType; +import java.lang.reflect.Type; +import java.util.List; +import org.junit.jupiter.api.Test; + +class HolderTest { + @Test + void testTypeHolder_rawType() { + Type type = new TypeHolder<List<String>>() {}.type(); + assertThat(type).isInstanceOf(ParameterizedType.class); + + ParameterizedType parameterizedType = (ParameterizedType) type; + assertThat(parameterizedType.getRawType()).isEqualTo(List.class); + assertThat(parameterizedType.getActualTypeArguments()).asList().containsExactly(String.class); + } + + @Test + void testTypeHolder_annotatedType() { + AnnotatedType type = new TypeHolder<@Foo List<@Bar String>>() {}.annotatedType(); + assertThat(type).isInstanceOf(AnnotatedParameterizedType.class); + + AnnotatedParameterizedType listType = (AnnotatedParameterizedType) type; + assertThat(listType.getType()).isInstanceOf(ParameterizedType.class); + assertThat(((ParameterizedType) listType.getType()).getRawType()).isEqualTo(List.class); + assertThat(listType.getAnnotations()).hasLength(1); + assertThat(listType.getAnnotations()[0]).isInstanceOf(Foo.class); + assertThat(listType.getAnnotatedActualTypeArguments()).hasLength(1); + + AnnotatedType stringType = listType.getAnnotatedActualTypeArguments()[0]; + assertThat(stringType.getType()).isEqualTo(String.class); + assertThat(stringType.getAnnotations()).hasLength(1); + assertThat(stringType.getAnnotations()[0]).isInstanceOf(Bar.class); + } + + @Test + void testParameterHolder_rawType() { + Type type = new ParameterHolder() { + void foo(List<String> parameter) {} + }.type(); + assertThat(type).isInstanceOf(ParameterizedType.class); + + ParameterizedType parameterizedType = (ParameterizedType) type; + assertThat(parameterizedType.getRawType()).isEqualTo(List.class); + assertThat(parameterizedType.getActualTypeArguments()).asList().containsExactly(String.class); + } + + @Test + void testParameterHolder_annotatedType() { + AnnotatedType type = new ParameterHolder() { + void foo(@ParameterAnnotation @Foo List<@Bar String> parameter) {} + }.annotatedType(); + assertThat(type).isInstanceOf(AnnotatedParameterizedType.class); + + AnnotatedParameterizedType listType = (AnnotatedParameterizedType) type; + assertThat(listType.getType()).isInstanceOf(ParameterizedType.class); + assertThat(((ParameterizedType) listType.getType()).getRawType()).isEqualTo(List.class); + assertThat(listType.getAnnotations()).hasLength(1); + assertThat(listType.getAnnotations()[0]).isInstanceOf(Foo.class); + assertThat(listType.getAnnotatedActualTypeArguments()).hasLength(1); + + AnnotatedType stringType = listType.getAnnotatedActualTypeArguments()[0]; + assertThat(stringType.getType()).isEqualTo(String.class); + assertThat(stringType.getAnnotations()).hasLength(1); + assertThat(stringType.getAnnotations()[0]).isInstanceOf(Bar.class); + } + + @Test + void testParameterHolder_parameterAnnotations() { + Annotation[] annotations = new ParameterHolder() { + void foo(@ParameterAnnotation @Foo List<@Bar String> parameter) {} + }.parameterAnnotations(); + assertThat(annotations).hasLength(1); + assertThat(annotations[0]).isInstanceOf(ParameterAnnotation.class); + } + + @Target(ElementType.TYPE_USE) + @Retention(RetentionPolicy.RUNTIME) + private @interface Foo {} + + @Target(ElementType.TYPE_USE) + @Retention(RetentionPolicy.RUNTIME) + private @interface Bar {} + + @Target(ElementType.PARAMETER) + @Retention(RetentionPolicy.RUNTIME) + private @interface ParameterAnnotation {} +} diff --git a/src/test/java/com/code_intelligence/jazzer/mutation/support/InputStreamSupportTest.java b/src/test/java/com/code_intelligence/jazzer/mutation/support/InputStreamSupportTest.java new file mode 100644 index 00000000..29963f4f --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/mutation/support/InputStreamSupportTest.java @@ -0,0 +1,146 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.code_intelligence.jazzer.mutation.support; + +import static com.code_intelligence.jazzer.mutation.support.InputStreamSupport.cap; +import static com.code_intelligence.jazzer.mutation.support.InputStreamSupport.extendWithReadExactly; +import static com.code_intelligence.jazzer.mutation.support.InputStreamSupport.extendWithZeros; +import static com.code_intelligence.jazzer.mutation.support.InputStreamSupport.infiniteZeros; +import static com.code_intelligence.jazzer.mutation.support.InputStreamSupport.readAllBytes; +import static com.google.common.truth.Truth.assertThat; + +import com.code_intelligence.jazzer.mutation.support.InputStreamSupport.ReadExactlyInputStream; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +class InputStreamSupportTest { + @Test + void testInfiniteZeros() throws IOException { + InputStream input = infiniteZeros(); + + assertThat(input.available()).isEqualTo(Integer.MAX_VALUE); + assertThat(input.read()).isEqualTo(0); + + input.close(); + + assertThat(input.available()).isEqualTo(Integer.MAX_VALUE); + assertThat(input.read()).isEqualTo(0); + } + + @Test + void testExtendWithNullInputStream_empty() throws IOException { + InputStream input = extendWithZeros(new ByteArrayInputStream(new byte[0])); + assertThat(input.skip(5)).isEqualTo(5); + assertThat(input.read()).isEqualTo(0); + byte[] bytes = new byte[] {9, 9, 9, 9, 9}; + assertThat(input.read(bytes)).isEqualTo(5); + assertThat(bytes).asList().containsExactly((byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0); + } + + @Test + void testExtendWithNullInputStream_emptyAfterRead() throws IOException { + InputStream input = extendWithZeros(new ByteArrayInputStream(new byte[] {1})); + assertThat(input.read()).isEqualTo(1); + assertThat(input.read()).isEqualTo(0); + assertThat(input.read()).isEqualTo(0); + byte[] bytes = new byte[] {9, 9, 9, 9, 9}; + assertThat(input.read(bytes)).isEqualTo(5); + assertThat(bytes).asList().containsExactly((byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0); + } + + @Test + void testExtendWithNullInputStream_emptyWithinRead() throws IOException { + InputStream input = extendWithZeros(new ByteArrayInputStream(new byte[] {1, 2, 3})); + byte[] bytes = new byte[] {9, 9, 9, 9, 9}; + assertThat(input.read(bytes)).isEqualTo(5); + assertThat(bytes).asList().containsExactly((byte) 1, (byte) 2, (byte) 3, (byte) 0, (byte) 0); + } + + @Test + void testExtendWithNullInputStream_emptyWithinSkip() throws IOException { + InputStream input = extendWithZeros(new ByteArrayInputStream(new byte[] {1, 2, 3})); + assertThat(input.skip(5)).isEqualTo(5); + byte[] bytes = new byte[] {9, 9, 9, 9, 9}; + assertThat(input.read(bytes)).isEqualTo(5); + assertThat(bytes).asList().containsExactly((byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0); + } + + @Test + void testCap_reachedAfterRead() throws IOException { + InputStream input = cap(new ByteArrayInputStream(new byte[] {1, 2, 3, 4, 5}), 3); + assertThat(input.available()).isEqualTo(3); + assertThat(input.read()).isEqualTo(1); + assertThat(input.available()).isEqualTo(2); + assertThat(input.read()).isEqualTo(2); + assertThat(input.available()).isEqualTo(1); + assertThat(input.read()).isEqualTo(3); + assertThat(input.available()).isEqualTo(0); + assertThat(input.read()).isEqualTo(-1); + assertThat(input.read(new byte[5], 0, 5)).isEqualTo(-1); + } + + @Test + void testCap_reachedWithinRead() throws IOException { + InputStream input = cap(new ByteArrayInputStream(new byte[] {1, 2, 3, 4, 5}), 3); + byte[] bytes = new byte[5]; + assertThat(input.available()).isEqualTo(3); + assertThat(input.read(bytes, 0, 5)).isEqualTo(3); + assertThat(bytes).asList().containsExactly((byte) 1, (byte) 2, (byte) 3, (byte) 0, (byte) 0); + } + + @ParameterizedTest + // 8192 is the internal buffer size. + @ValueSource(ints = {0, 1, 3, 500, 8192, 8192 + 17, 8192 * 8192 + 17}) + void testReadAllBytes(int length) throws IOException { + byte[] bytes = new byte[length]; + for (int i = 0; i < bytes.length; i++) { + bytes[i] = (byte) i; + } + InputStream input = new ByteArrayInputStream(bytes); + + assertThat(readAllBytes(input)).isEqualTo(bytes); + } + + @Test + @SuppressWarnings("ResultOfMethodCallIgnored") + void testReadExactly() throws IOException { + ReadExactlyInputStream ce = extendWithReadExactly(new ByteArrayInputStream(new byte[] {0, 1})); + assertThat(ce.isConsumedExactly()).isFalse(); + ce.read(); + assertThat(ce.isConsumedExactly()).isFalse(); + ce.read(); + assertThat(ce.isConsumedExactly()).isTrue(); + ce.read(); + assertThat(ce.isConsumedExactly()).isFalse(); + } + + @Test + @SuppressWarnings("ResultOfMethodCallIgnored") + void testReadExactly_readBytes() throws IOException { + ReadExactlyInputStream ce = + extendWithReadExactly(new ByteArrayInputStream(new byte[] {0, 1, 2})); + assertThat(ce.isConsumedExactly()).isFalse(); + ce.read(new byte[3]); + assertThat(ce.isConsumedExactly()).isTrue(); + ce.read(new byte[1]); + assertThat(ce.isConsumedExactly()).isFalse(); + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/mutation/support/TestSupport.java b/src/test/java/com/code_intelligence/jazzer/mutation/support/TestSupport.java new file mode 100644 index 00000000..8035ef86 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/mutation/support/TestSupport.java @@ -0,0 +1,425 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.code_intelligence.jazzer.mutation.support; + +import static com.code_intelligence.jazzer.mutation.support.Preconditions.requireNonNullElements; +import static com.google.common.truth.Truth.assertThat; +import static java.util.Arrays.stream; +import static java.util.stream.Collectors.toCollection; + +import com.code_intelligence.jazzer.mutation.api.Debuggable; +import com.code_intelligence.jazzer.mutation.api.InPlaceMutator; +import com.code_intelligence.jazzer.mutation.api.PseudoRandom; +import com.code_intelligence.jazzer.mutation.api.SerializingMutator; +import com.code_intelligence.jazzer.mutation.engine.SeededPseudoRandom; +import com.google.errorprone.annotations.CheckReturnValue; +import com.google.errorprone.annotations.MustBeClosed; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.OutputStream; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Queue; +import java.util.function.BiConsumer; +import java.util.function.BiFunction; +import java.util.function.Consumer; +import java.util.function.Predicate; +import java.util.function.Supplier; +import java.util.function.UnaryOperator; + +public final class TestSupport { + private static final DataOutputStream nullDataOutputStream = + new DataOutputStream(new OutputStream() { + @Override + public void write(int i) {} + }); + + private TestSupport() {} + + public static DataOutputStream nullDataOutputStream() { + return nullDataOutputStream; + } + + /** + * Deterministically creates a new instance of {@link PseudoRandom} whose exact behavior is + * intentionally unspecified. + */ + // TODO: Turn usages of this function into fuzz tests. + public static PseudoRandom anyPseudoRandom() { + // Change this seed from time to time to shake out tests relying on hardcoded behavior. + return new SeededPseudoRandom(8853461259049838337L); + } + + /** + * Creates a {@link PseudoRandom} whose methods return the given values in order. + */ + @MustBeClosed + public static MockPseudoRandom mockPseudoRandom(Object... returnValues) { + return new MockPseudoRandom(returnValues); + } + + @CheckReturnValue + public static <T> SerializingMutator<T> mockMutator(T initialValue, UnaryOperator<T> mutate) { + return mockMutator(initialValue, mutate, value -> value); + } + + @CheckReturnValue + public static <T> SerializingMutator<T> mockMutator( + T initialValue, UnaryOperator<T> mutate, UnaryOperator<T> detach) { + return new AbstractMockMutator<T>() { + @Override + protected T nextInitialValue() { + return initialValue; + } + + @Override + public T mutate(T value, PseudoRandom prng) { + return mutate.apply(value); + } + + @Override + public T detach(T value) { + return detach.apply(value); + } + }; + } + + @CheckReturnValue + public static <T> SerializingMutator<T> mockInitializer( + Supplier<T> getInitialValues, UnaryOperator<T> detach) { + return new AbstractMockMutator<T>() { + @Override + protected T nextInitialValue() { + return getInitialValues.get(); + } + + @Override + public T mutate(T value, PseudoRandom prng) { + throw new UnsupportedOperationException(); + } + + @Override + public T detach(T value) { + return detach.apply(value); + } + }; + } + + @CheckReturnValue + public static <T> SerializingMutator<T> mockCrossOver(BiFunction<T, T, T> getCrossOverValue) { + return new AbstractMockMutator<T>() { + @Override + protected T nextInitialValue() { + throw new UnsupportedOperationException(); + } + + @Override + public T mutate(T value, PseudoRandom prng) { + throw new UnsupportedOperationException(); + } + + @Override + public T crossOver(T value, T otherValue, PseudoRandom prng) { + return getCrossOverValue.apply(value, otherValue); + } + + @Override + public T detach(T value) { + return value; + } + }; + } + + @CheckReturnValue + public static <T> InPlaceMutator<T> mockCrossOverInPlace(BiConsumer<T, T> crossOverInPlace) { + return new AbstractMockInPlaceMutator<T>() { + @Override + public void crossOverInPlace(T reference, T otherReference, PseudoRandom prng) { + crossOverInPlace.accept(reference, otherReference); + } + + @Override + public String toDebugString(Predicate<Debuggable> isInCycle) { + return "CrossOverInPlaceMockMutator"; + } + }; + } + + @CheckReturnValue + public static <T> InPlaceMutator<T> mockInitInPlace(Consumer<T> setInitialValues) { + return new AbstractMockInPlaceMutator<T>() { + @Override + public void initInPlace(T reference, PseudoRandom prng) { + setInitialValues.accept(reference); + } + + @Override + public String toDebugString(Predicate<Debuggable> isInCycle) { + return "InitInPlaceMockMutator"; + } + }; + } + + private static abstract class AbstractMockInPlaceMutator<T> implements InPlaceMutator<T> { + @Override + public void initInPlace(T reference, PseudoRandom prng) { + throw new UnsupportedOperationException(); + } + + @Override + public void mutateInPlace(T reference, PseudoRandom prng) { + throw new UnsupportedOperationException(); + } + + @Override + public void crossOverInPlace(T reference, T otherReference, PseudoRandom prng) { + throw new UnsupportedOperationException(); + } + } + + private static abstract class AbstractMockMutator<T> extends SerializingMutator<T> { + abstract protected T nextInitialValue(); + + @Override + public T read(DataInputStream in) { + return nextInitialValue(); + } + + @Override + public void write(T value, DataOutputStream out) { + throw new UnsupportedOperationException("mockMutator does not support write"); + } + + @Override + public T init(PseudoRandom prng) { + return nextInitialValue(); + } + + @Override + public T crossOver(T value, T otherValue, PseudoRandom prng) { + return value; + } + + @Override + public String toDebugString(Predicate<Debuggable> isInCycle) { + T initialValue = nextInitialValue(); + if (initialValue == null) { + return "null"; + } + return initialValue.getClass().getSimpleName(); + } + + @Override + public T detach(T value) { + return value; + } + } + + public static final class MockPseudoRandom implements PseudoRandom, AutoCloseable { + private final Queue<Object> elements; + + private MockPseudoRandom(Object... objects) { + requireNonNullElements(objects); + this.elements = stream(objects).collect(toCollection(ArrayDeque::new)); + } + + @Override + public boolean choice() { + assertThat(elements).isNotEmpty(); + return (boolean) elements.poll(); + } + + @Override + public boolean trueInOneOutOf(int inverseFrequencyTrue) { + assertThat(inverseFrequencyTrue).isAtLeast(2); + + assertThat(elements).isNotEmpty(); + return (boolean) elements.poll(); + } + + @Override + public <T> T pickIn(T[] array) { + assertThat(array).isNotEmpty(); + + assertThat(elements).isNotEmpty(); + return array[(int) elements.poll()]; + } + + @Override + public <T> T pickIn(List<T> list) { + assertThat(list).isNotEmpty(); + + assertThat(elements).isNotEmpty(); + return list.get((int) elements.poll()); + } + + @Override + public <T> int indexIn(T[] array) { + assertThat(array).isNotEmpty(); + + assertThat(elements).isNotEmpty(); + return (int) elements.poll(); + } + + @Override + public <T> int indexIn(List<T> list) { + assertThat(list).isNotEmpty(); + + assertThat(elements).isNotEmpty(); + return (int) elements.poll(); + } + + @Override + public int indexIn(int range) { + assertThat(range).isAtLeast(1); + + assertThat(elements).isNotEmpty(); + return (int) elements.poll(); + } + + @Override + public <T> int otherIndexIn(T[] array, int currentIndex) { + return otherIndexIn(array.length, currentIndex); + } + + @Override + public int otherIndexIn(int range, int currentValue) { + assertThat(range).isAtLeast(2); + assertThat(elements).isNotEmpty(); + int result = (int) elements.poll(); + assertThat(result).isAtLeast(0); + assertThat(result).isAtMost(range - 1); + assertThat(result).isNotEqualTo(currentValue); + return result; + } + + @Override + public int closedRange(int lowerInclusive, int upperInclusive) { + assertThat(lowerInclusive).isAtMost(upperInclusive); + + assertThat(elements).isNotEmpty(); + int result = (int) elements.poll(); + assertThat(result).isAtLeast(lowerInclusive); + assertThat(result).isAtMost(upperInclusive); + return result; + } + + @Override + public long closedRange(long lowerInclusive, long upperInclusive) { + assertThat(lowerInclusive).isAtMost(upperInclusive); + + assertThat(elements).isNotEmpty(); + long result = (long) elements.poll(); + assertThat(result).isAtLeast(lowerInclusive); + assertThat(result).isAtMost(upperInclusive); + return result; + } + + @Override + public float closedRange(float lowerInclusive, float upperInclusive) { + assertThat(lowerInclusive).isLessThan(upperInclusive); + assertThat(elements).isNotEmpty(); + float result = (float) elements.poll(); + assertThat(result).isAtLeast(lowerInclusive); + assertThat(result).isAtMost(upperInclusive); + return result; + } + + @Override + public double closedRange(double lowerInclusive, double upperInclusive) { + assertThat(lowerInclusive).isLessThan(upperInclusive); + assertThat(elements).isNotEmpty(); + double result = (double) elements.poll(); + assertThat(result).isAtLeast(lowerInclusive); + assertThat(result).isAtMost(upperInclusive); + return result; + } + + @Override + public int closedRangeBiasedTowardsSmall(int upperInclusive) { + assertThat(upperInclusive).isAtLeast(0); + + assertThat(elements).isNotEmpty(); + int result = (int) elements.poll(); + assertThat(result).isAtLeast(0); + assertThat(result).isAtMost(upperInclusive); + return result; + } + + @Override + public int closedRangeBiasedTowardsSmall(int lowerInclusive, int upperInclusive) { + assertThat(lowerInclusive).isAtMost(upperInclusive); + + assertThat(elements).isNotEmpty(); + int result = (int) elements.poll(); + assertThat(result).isAtLeast(lowerInclusive); + assertThat(result).isAtMost(upperInclusive); + return result; + } + + @Override + public void bytes(byte[] bytes) { + assertThat(elements).isNotEmpty(); + byte[] result = (byte[]) elements.poll(); + assertThat(result).hasLength(bytes.length); + System.arraycopy(result, 0, bytes, 0, bytes.length); + } + + @Override + public <T> T pickValue( + T value, T otherValue, Supplier<T> supplier, int inverseSupplierFrequency) { + assertThat(elements).isNotEmpty(); + switch ((int) elements.poll()) { + case 0: + return value; + case 1: + return otherValue; + case 2: + return supplier.get(); + default: + throw new AssertionError("Invalid pickValue element"); + } + } + + @Override + public long nextLong() { + assertThat(elements).isNotEmpty(); + return (long) elements.poll(); + } + + @Override + public void close() { + assertThat(elements).isEmpty(); + } + } + + @SuppressWarnings("unchecked") + public static <K, V> LinkedHashMap<K, V> asMap(Object... objs) { + LinkedHashMap<K, V> map = new LinkedHashMap<>(); + for (int i = 0; i < objs.length; i += 2) { + map.put((K) objs[i], (V) objs[i + 1]); + } + return map; + } + + @SafeVarargs + public static <T> ArrayList<T> asMutableList(T... objs) { + return stream(objs).collect(toCollection(ArrayList::new)); + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/mutation/support/TypeSupportTest.java b/src/test/java/com/code_intelligence/jazzer/mutation/support/TypeSupportTest.java new file mode 100644 index 00000000..bbf4a7e6 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/mutation/support/TypeSupportTest.java @@ -0,0 +1,269 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.code_intelligence.jazzer.mutation.support; + +import static com.code_intelligence.jazzer.mutation.support.TypeSupport.asAnnotatedType; +import static com.code_intelligence.jazzer.mutation.support.TypeSupport.asSubclassOrEmpty; +import static com.code_intelligence.jazzer.mutation.support.TypeSupport.containedInDirectedCycle; +import static com.code_intelligence.jazzer.mutation.support.TypeSupport.visitAnnotatedType; +import static com.code_intelligence.jazzer.mutation.support.TypeSupport.withTypeArguments; +import static com.google.common.truth.Truth.assertThat; +import static com.google.common.truth.Truth8.assertThat; +import static java.util.Arrays.stream; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.code_intelligence.jazzer.mutation.annotation.NotNull; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import java.lang.reflect.AnnotatedParameterizedType; +import java.lang.reflect.AnnotatedType; +import java.lang.reflect.ParameterizedType; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Stream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledForJreRange; +import org.junit.jupiter.api.condition.JRE; + +class TypeSupportTest { + @Test + void testFillTypeVariablesRawType_oneVariable() { + AnnotatedParameterizedType actual = + withTypeArguments(new TypeHolder<@NotNull List>() {}.annotatedType(), + new TypeHolder<@NotNull String>() {}.annotatedType()); + AnnotatedParameterizedType expected = + (AnnotatedParameterizedType) new TypeHolder<@NotNull List<@NotNull String>>() { + }.annotatedType(); + + // Test both equals implementations as we implement them ourselves. + assertThat(actual.getType()).isEqualTo(expected.getType()); + assertThat(expected.getType()).isEqualTo(actual.getType()); + + assertThat(actual.getAnnotations()).isEqualTo(expected.getAnnotations()); + assertThat(expected.getAnnotations()).isEqualTo(actual.getAnnotations()); + + assertThat(((ParameterizedType) actual.getType()).getActualTypeArguments()) + .isEqualTo(((ParameterizedType) expected.getType()).getActualTypeArguments()); + assertThat(((ParameterizedType) expected.getType()).getActualTypeArguments()) + .isEqualTo(((ParameterizedType) actual.getType()).getActualTypeArguments()); + } + + @Test + // Java <= 11 does not implement AnnotatedType#equals. + // https://github.com/openjdk/jdk/commit/ab0128ca51de59aaaa674654ca8d4e16b3b79965 + @EnabledForJreRange(min = JRE.JAVA_12) + void testFillTypeVariablesAnnotatedType_oneVariable() { + AnnotatedParameterizedType actual = + withTypeArguments(new TypeHolder<@NotNull List>() {}.annotatedType(), + new TypeHolder<@NotNull String>() {}.annotatedType()); + AnnotatedParameterizedType expected = + (AnnotatedParameterizedType) new TypeHolder<@NotNull List<@NotNull String>>() { + }.annotatedType(); + + // Test both equals implementations as we implement them ourselves. + assertThat(actual).isEqualTo(expected); + assertThat(expected).isEqualTo(actual); + + assertThat(actual.getType()).isEqualTo(expected.getType()); + assertThat(expected.getType()).isEqualTo(actual.getType()); + + assertThat(actual.getAnnotations()).isEqualTo(expected.getAnnotations()); + assertThat(expected.getAnnotations()).isEqualTo(actual.getAnnotations()); + + assertThat(actual.getAnnotatedActualTypeArguments()) + .isEqualTo(expected.getAnnotatedActualTypeArguments()); + assertThat(expected.getAnnotatedActualTypeArguments()) + .isEqualTo(actual.getAnnotatedActualTypeArguments()); + } + + @Test + void testFillTypeVariablesRawType_oneVariable_differentType() { + AnnotatedParameterizedType actual = + withTypeArguments(new TypeHolder<@NotNull List>() {}.annotatedType(), + new TypeHolder<@NotNull String>() {}.annotatedType()); + AnnotatedParameterizedType differentParameterAnnotation = + (AnnotatedParameterizedType) new TypeHolder<@NotNull List<@NotNull Boolean>>() { + }.annotatedType(); + + // Test both equals implementations as we implement them ourselves. + assertThat(actual.getType()).isNotEqualTo(differentParameterAnnotation.getType()); + assertThat(differentParameterAnnotation.getType()).isNotEqualTo(actual.getType()); + + assertThat(actual.getAnnotations()).isEqualTo(differentParameterAnnotation.getAnnotations()); + assertThat(differentParameterAnnotation.getAnnotations()).isEqualTo(actual.getAnnotations()); + + assertThat(((ParameterizedType) actual.getType()).getActualTypeArguments()) + .isNotEqualTo( + ((ParameterizedType) differentParameterAnnotation.getType()).getActualTypeArguments()); + assertThat( + ((ParameterizedType) differentParameterAnnotation.getType()).getActualTypeArguments()) + .isNotEqualTo(((ParameterizedType) actual.getType()).getActualTypeArguments()); + } + + @Test + // Java <= 11 does not implement AnnotatedType#equals. + // https://github.com/openjdk/jdk/commit/ab0128ca51de59aaaa674654ca8d4e16b3b79965 + @EnabledForJreRange(min = JRE.JAVA_12) + void testFillTypeVariablesAnnotatedType_oneVariable_differentAnnotations() { + AnnotatedParameterizedType actual = + withTypeArguments(new TypeHolder<@NotNull List>() {}.annotatedType(), + new TypeHolder<@NotNull String>() {}.annotatedType()); + AnnotatedParameterizedType differentParameterAnnotation = + (AnnotatedParameterizedType) new TypeHolder<@NotNull List<String>>() {}.annotatedType(); + + // Test both equals implementations as we implement them ourselves. + assertThat(actual).isNotEqualTo(differentParameterAnnotation); + assertThat(differentParameterAnnotation).isNotEqualTo(actual); + + assertThat(actual.getType()).isEqualTo(differentParameterAnnotation.getType()); + assertThat(differentParameterAnnotation.getType()).isEqualTo(actual.getType()); + + assertThat(actual.getAnnotations()).isEqualTo(differentParameterAnnotation.getAnnotations()); + assertThat(differentParameterAnnotation.getAnnotations()).isEqualTo(actual.getAnnotations()); + + assertThat(actual.getAnnotatedActualTypeArguments()) + .isNotEqualTo(differentParameterAnnotation.getAnnotatedActualTypeArguments()); + assertThat(differentParameterAnnotation.getAnnotatedActualTypeArguments()) + .isNotEqualTo(actual.getAnnotatedActualTypeArguments()); + } + + @Test + void testFillTypeVariablesRawType_twoVariables() { + AnnotatedParameterizedType actual = + withTypeArguments(new TypeHolder<@NotNull Map>() {}.annotatedType(), + new TypeHolder<@NotNull String>() {}.annotatedType(), + new TypeHolder<byte[]>() {}.annotatedType()); + AnnotatedParameterizedType expected = + (AnnotatedParameterizedType) new TypeHolder<@NotNull Map<@NotNull String, byte[]>>() { + }.annotatedType(); + + // Test both equals implementations as we implement them ourselves. + assertThat(actual.getType()).isEqualTo(expected.getType()); + assertThat(expected.getType()).isEqualTo(actual.getType()); + + assertThat(actual.getAnnotations()).isEqualTo(expected.getAnnotations()); + assertThat(expected.getAnnotations()).isEqualTo(actual.getAnnotations()); + + assertThat(((ParameterizedType) actual.getType()).getActualTypeArguments()) + .isEqualTo(((ParameterizedType) expected.getType()).getActualTypeArguments()); + assertThat(((ParameterizedType) expected.getType()).getActualTypeArguments()) + .isEqualTo(((ParameterizedType) actual.getType()).getActualTypeArguments()); + } + + @Test + // Java <= 11 does not implement AnnotatedType#equals. + // https://github.com/openjdk/jdk/commit/ab0128ca51de59aaaa674654ca8d4e16b3b79965 + @EnabledForJreRange(min = JRE.JAVA_12) + void testFillTypeVariablesAnnotatedType_twoVariables() { + AnnotatedParameterizedType actual = + withTypeArguments(new TypeHolder<@NotNull Map>() {}.annotatedType(), + new TypeHolder<@NotNull String>() {}.annotatedType(), + new TypeHolder<byte[]>() {}.annotatedType()); + AnnotatedParameterizedType expected = + (AnnotatedParameterizedType) new TypeHolder<@NotNull Map<@NotNull String, byte[]>>() { + }.annotatedType(); + + // Test both equals implementations as we implement them ourselves. + assertThat(actual).isEqualTo(expected); + assertThat(expected).isEqualTo(actual); + + assertThat(actual.getType()).isEqualTo(expected.getType()); + assertThat(expected.getType()).isEqualTo(actual.getType()); + + assertThat(actual.getAnnotations()).isEqualTo(expected.getAnnotations()); + assertThat(expected.getAnnotations()).isEqualTo(actual.getAnnotations()); + + assertThat(actual.getAnnotatedActualTypeArguments()) + .isEqualTo(expected.getAnnotatedActualTypeArguments()); + assertThat(expected.getAnnotatedActualTypeArguments()) + .isEqualTo(actual.getAnnotatedActualTypeArguments()); + } + + @Test + void testFillTypeVariables_failures() { + assertThrows(IllegalArgumentException.class, + () -> withTypeArguments(new TypeHolder<List>() {}.annotatedType())); + assertThrows(IllegalArgumentException.class, () -> withTypeArguments(new TypeHolder<List<?>>() { + }.annotatedType(), asAnnotatedType(String.class))); + } + + @Test + void testAsSubclassOrEmpty() { + assertThat(asSubclassOrEmpty(asAnnotatedType(String.class), String.class)) + .hasValue(String.class); + assertThat(asSubclassOrEmpty(asAnnotatedType(String.class), CharSequence.class)) + .hasValue(String.class); + assertThat(asSubclassOrEmpty(asAnnotatedType(CharSequence.class), String.class)).isEmpty(); + assertThat(asSubclassOrEmpty(new TypeHolder<List<String>>() { + }.annotatedType(), List.class)).isEmpty(); + } + + @Target(ElementType.TYPE_USE) + @Retention(RetentionPolicy.RUNTIME) + private @interface A { + int value(); + } + + @Test + void testVisitAnnotatedType() { + Map<Integer, Class<?>> visited = new LinkedHashMap<>(); + AnnotatedType type = new TypeHolder<@A( + 1) List<@A(2) Map<@A(3) byte @A(4)[] @A(5)[], @A(6) Byte> @A(7)[] @A(8)[]>>(){} + .annotatedType(); + + visitAnnotatedType(type, + (clazz, annotations) + -> stream(annotations) + .map(annotation -> ((A) annotation).value()) + .forEach(value -> visited.put(value, clazz))); + + assertThat(visited) + .containsExactly(1, List.class, 7, Map[][].class, 8, Map[].class, 2, Map.class, 4, + byte[][].class, 5, byte[].class, 3, byte.class, 6, Byte.class) + .inOrder(); + } + + @Test + void testContainedInDirectedCycle() { + Function<Integer, Stream<Integer>> successors = integer -> { + switch (integer) { + case 1: + return Stream.of(2); + case 2: + return Stream.of(3); + case 3: + return Stream.of(4, 5); + case 4: + return Stream.of(2); + case 5: + return Stream.empty(); + default: + throw new IllegalStateException(); + } + }; + + assertThat(containedInDirectedCycle(1, successors)).isFalse(); + assertThat(containedInDirectedCycle(2, successors)).isTrue(); + assertThat(containedInDirectedCycle(3, successors)).isTrue(); + assertThat(containedInDirectedCycle(4, successors)).isTrue(); + assertThat(containedInDirectedCycle(5, successors)).isFalse(); + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/mutation/support/WeakIdentityHashMapTest.java b/src/test/java/com/code_intelligence/jazzer/mutation/support/WeakIdentityHashMapTest.java new file mode 100644 index 00000000..5406ef88 --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/mutation/support/WeakIdentityHashMapTest.java @@ -0,0 +1,61 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.code_intelligence.jazzer.mutation.support; + +import static com.google.common.truth.Truth.assertThat; + +import java.util.Arrays; +import java.util.List; +import org.junit.jupiter.api.Test; + +class WeakIdentityHashMapTest { + private static void reachabilityFence(Object o) { + // Polyfill for JDK 9+ Reference.reachabilityFence: + // https://mail.openjdk.org/pipermail/core-libs-dev/2018-February/051312.html + } + + @Test + void testWeakIdentityHashMap_hasIdentitySemantics() { + WeakIdentityHashMap<List<Integer>, String> map = new WeakIdentityHashMap<>(); + + List<Integer> list = Arrays.asList(1, 2); + map.put(list, "value"); + assertThat(map.containsKey(list)).isTrue(); + + List<Integer> equalList = Arrays.asList(1, 2); + assertThat(map.containsKey(equalList)).isFalse(); + + reachabilityFence(list); + } + + @Test + void testWeakIdentityHashMap_hasWeakSemantics() { + WeakIdentityHashMap<List<Integer>, String> map = new WeakIdentityHashMap<>(); + + List<Integer> list = Arrays.asList(1, 2); + map.put(list, "value"); + assertThat(map.containsKey(list)).isTrue(); + assertThat(map.size()).isEqualTo(1); + assertThat(map.isEmpty()).isFalse(); + + reachabilityFence(list); + map.collectKeysForTesting(); + + assertThat(map.size()).isEqualTo(0); + assertThat(map.isEmpty()).isTrue(); + } +} diff --git a/src/test/java/com/code_intelligence/jazzer/runtime/BUILD.bazel b/src/test/java/com/code_intelligence/jazzer/runtime/BUILD.bazel new file mode 100644 index 00000000..db8e507a --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/runtime/BUILD.bazel @@ -0,0 +1,14 @@ +load("//bazel:compat.bzl", "SKIP_ON_WINDOWS") + +java_test( + name = "TraceCmpHooksTest", + srcs = [ + "TraceCmpHooksTest.java", + ], + target_compatible_with = SKIP_ON_WINDOWS, + deps = [ + "//src/main/java/com/code_intelligence/jazzer/runtime", + "//src/main/native/com/code_intelligence/jazzer/driver:jazzer_driver", + "@maven//:junit_junit", + ], +) diff --git a/src/test/java/com/code_intelligence/jazzer/runtime/TraceCmpHooksTest.java b/src/test/java/com/code_intelligence/jazzer/runtime/TraceCmpHooksTest.java new file mode 100644 index 00000000..a1ef86ff --- /dev/null +++ b/src/test/java/com/code_intelligence/jazzer/runtime/TraceCmpHooksTest.java @@ -0,0 +1,53 @@ +/* + * Copyright 2022 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.code_intelligence.jazzer.runtime; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import org.junit.Test; + +public class TraceCmpHooksTest { + private static final ExecutorService ES = Executors.newFixedThreadPool(5); + + @Test + public void cmpHookShouldHandleConcurrentModifications() throws InterruptedException { + String arg = "test"; + Map<String, Object> map = new HashMap<>(); + map.put(arg, arg); + + // Add elements to map asynchronously + Function<Integer, Runnable> put = (final Integer num) -> () -> { + map.put(String.valueOf(num), num); + }; + for (int i = 0; i < 1_000_000; i++) { + ES.submit(put.apply(i)); + } + + // Call hook + for (int i = 0; i < 1_000; i++) { + TraceCmpHooks.mapGet(null, map, new Object[] {arg}, 1, null); + } + + ES.shutdown(); + // noinspection ResultOfMethodCallIgnored + ES.awaitTermination(5, TimeUnit.SECONDS); + } +} |