split-comparison llvm pass refactor for smaller compilation times (and a small bug fix) (#964)

* Refactored split compare pass to be more efficient in LTO usage and allow splitting to other minimum bitwidths.

Efficiency: avoid looping over the whole llvm module N times, when once is also enough.

Bitwidth: Previously, due to fallthrough in switch-case, all comparisons were split to 8-bit, which might not be desirable e.g., 16 or 32 bit might be enough. So now all comparison are split until they are smaller or equal to the target bitwidth, which is controlled through the `AFL_LLVM_LAF_SPLIT_COMPARES_BITW` environment variable.

* fixed miscompilation due to incorrectly trying to split a signed comparison operator

* minor formatting updates and use IRBuilder when inserting multiple instructions

* added @hexcoder-'s test-int_cases.c to make test

* Avoid recursion; switch to smallvector in splitAndSimplify; use switch case for icmp type;

* Fixed issue when splitting < where the inverse comparison was not further split

* some cleanup

2 files changed, 38 insertions, 15 deletions

diff --git a/test/test-llvm.sh b/test/test-llvm.sh
index f902ffc5..8090e176 100755
--- a/test/test-llvm.sh
+++ b/test/test-llvm.sh
@@ -186,6 +186,29 @@ test -e ../afl-clang-fast -a -e ../split-switches-pass.so && {
   }
   rm -f test-instr.plain
 
+  $ECHO "$GREY[*] llvm_mode laf-intel/compcov testing splitting integer types (this might take some time)"
+  for testcase in ./test-int_cases.c ./test-uint_cases.c; do
+    for I in char short int long "long long"; do
+      for BITS in 8 16 32 64; do
+        bin="$testcase-split-$I-$BITS.compcov" 
+        AFL_LLVM_INSTRUMENT=AFL AFL_DEBUG=1 AFL_LLVM_LAF_SPLIT_COMPARES_BITW=$BITS AFL_LLVM_LAF_SPLIT_COMPARES=1 ../afl-clang-fast -DINT_TYPE="$I" -o "$bin" "$testcase" > test.out 2>&1;
+        if ! test -e "$bin"; then
+            cat test.out
+            $ECHO "$RED[!] llvm_mode laf-intel/compcov integer splitting failed! ($testcase with type $I split to $BITS)!";
+            CODE=1
+            break
+        fi
+        if ! "$bin"; then
+            $ECHO "$RED[!] llvm_mode laf-intel/compcov integer splitting resulted in miscompilation (type $I split to $BITS)!";
+            CODE=1
+            break
+        fi
+        rm -f "$bin" test.out || true
+      done
+    done
+  done
+  rm -f test-int-split*.compcov test.out
+
   AFL_LLVM_INSTRUMENT=AFL AFL_DEBUG=1 AFL_LLVM_LAF_SPLIT_SWITCHES=1 AFL_LLVM_LAF_TRANSFORM_COMPARES=1 AFL_LLVM_LAF_SPLIT_COMPARES=1 ../afl-clang-fast -o test-compcov.compcov test-compcov.c > test.out 2>&1
   test -e test-compcov.compcov && test_compcov_binary_functionality ./test-compcov.compcov && {
     grep --binary-files=text -Eq " [ 123][0-9][0-9] location| [3-9][0-9] location" test.out && {
diff --git a/test/test-uint_cases.c b/test/test-uint_cases.c
index 8496cffe..a277e28a 100644
--- a/test/test-uint_cases.c
+++ b/test/test-uint_cases.c
@@ -1,16 +1,16 @@
 /*
- * compile with -DUINT_TYPE="unsigned char"
- *          or  -DUINT_TYPE="unsigned short"
- *          or  -DUINT_TYPE="unsigned int"
- *          or  -DUINT_TYPE="unsigned long"
- *          or  -DUINT_TYPE="unsigned long long"
+ * compile with -DINT_TYPE="char"
+ *          or  -DINT_TYPE="short"
+ *          or  -DINT_TYPE="int"
+ *          or  -DINT_TYPE="long"
+ *          or  -DINT_TYPE="long long"
  */
 
 #include <assert.h>
 
 int main() {
 
-  volatile UINT_TYPE a, b;
+  volatile unsigned INT_TYPE a, b;
 
   a = 1;
   b = 8;
@@ -21,7 +21,7 @@ int main() {
   assert((a != b));
   assert(!(a == b));
 
-  if ((UINT_TYPE)(~0) > 255) {
+  if ((INT_TYPE)(~0) > 255) {
     volatile unsigned short a, b;
     a = 256+2;
     b = 256+21;
@@ -41,7 +41,7 @@ int main() {
     assert((a != b));
     assert(!(a == b));
 
-    if ((UINT_TYPE)(~0) > 65535) {
+    if ((INT_TYPE)(~0) > 65535) {
       volatile unsigned int a, b;
       a = 65536+2;
       b = 65536+21;
@@ -62,7 +62,7 @@ int main() {
       assert(!(a == b));
     }
 
-    if ((UINT_TYPE)(~0) > 4294967295) {
+    if ((INT_TYPE)(~0) > 4294967295) {
       volatile unsigned long a, b;
       a = 4294967296+2;
       b = 4294967296+21;
@@ -93,7 +93,7 @@ int main() {
   assert((a != b));
   assert(!(a == b));
 
-  if ((UINT_TYPE)(~0) > 255) {
+  if ((INT_TYPE)(~0) > 255) {
     volatile unsigned short a, b;
     a = 256+2;
     b = 256+1;
@@ -113,7 +113,7 @@ int main() {
     assert((a != b));
     assert(!(a == b));
 
-    if ((UINT_TYPE)(~0) > 65535) {
+    if ((INT_TYPE)(~0) > 65535) {
       volatile unsigned int a, b;
       a = 65536+2;
       b = 65536+1;
@@ -133,7 +133,7 @@ int main() {
       assert((a != b));
       assert(!(a == b));
 
-      if ((UINT_TYPE)(~0) > 4294967295) {
+      if ((INT_TYPE)(~0) > 4294967295) {
         volatile unsigned long a, b;
         a = 4294967296+2;
         b = 4294967296+1;
@@ -176,7 +176,7 @@ int main() {
   assert(!(a != b));
   assert((a == b));
 
-  if ((UINT_TYPE)(~0) > 255) {
+  if ((INT_TYPE)(~0) > 255) {
     volatile unsigned short a, b;
     a = 256+5;
     b = 256+5;
@@ -187,7 +187,7 @@ int main() {
     assert(!(a != b));
     assert((a == b));
 
-    if ((UINT_TYPE)(~0) > 65535) {
+    if ((INT_TYPE)(~0) > 65535) {
       volatile unsigned int a, b;
       a = 65536+5;
       b = 65536+5;
@@ -198,7 +198,7 @@ int main() {
       assert(!(a != b));
       assert((a == b));
 
-      if ((UINT_TYPE)(~0) > 4294967295) {
+      if ((INT_TYPE)(~0) > 4294967295) {
         volatile unsigned long a, b;
         a = 4294967296+5;
         b = 4294967296+5;