Merge "dx: Test enhancements"

5 files changed, 120 insertions, 60 deletions

diff --git a/dx/tests/127-merge-stress/com/android/dx/merge/MergeTest.java b/dx/tests/127-merge-stress/com/android/dx/merge/MergeTest.java
index cb0814c17..875cfadb3 100644
--- a/dx/tests/127-merge-stress/com/android/dx/merge/MergeTest.java
+++ b/dx/tests/127-merge-stress/com/android/dx/merge/MergeTest.java
@@ -21,8 +21,15 @@ import com.android.dex.DexIndexOverflowException;
 import com.android.dx.command.dexer.DxContext;
 
 import java.io.File;
+import java.io.IOException;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
 import java.util.Arrays;
 import java.util.Random;
+import java.util.HashSet;
 
 /**
  * This test tries to merge given dex files at random, a first pass at 2 by 2, followed by
@@ -32,31 +39,67 @@ public class MergeTest {
 
   private static final int NUMBER_OF_TRIES = 1000;
 
+  private static final int WORKER_THREADS = 4;
+
+  private static final ExecutorService executor = Executors.newFixedThreadPool(WORKER_THREADS);
+
+  // Helper task to concurrently run merge tests.
+  static class MergeTask implements Runnable {
+    private final DexMerger dexMerger;
+    private final String[] dexFiles;
+
+    MergeTask(String[] dexFiles, Dex[] dexesToMerge) throws IOException {
+      this.dexMerger = new DexMerger(dexesToMerge, CollisionPolicy.KEEP_FIRST, new DxContext());
+      this.dexFiles = dexFiles;
+    }
+
+    public void run() {
+      try {
+        dexMerger.merge();
+      } catch (DexIndexOverflowException e) {
+        // ignore index overflow
+      } catch (Throwable t) {
+        System.err.println("Exception processing DEX files: " + t);
+        System.err.println("Problem merging those dexes: " + Arrays.toString(dexFiles));
+        System.exit(1);
+      }
+    }
+  }
+
   public static void main(String[] args) throws Throwable {
     Random random = new Random();
+    HashSet<Integer> seen = new HashSet<>();
     for (int pass = 0; pass < 2; pass++) {
       for (int i = 0; i < NUMBER_OF_TRIES; i++) {
         // On the first pass only do 2-way merges, then do from 3 to 10 way merges
         // but not more to avoid dex index overflow.
         int numDex = pass == 0 ? 2 : random.nextInt(8) + 3;
 
-        String[] fileNames = new String[numDex]; // only for the error message
-        try {
-          Dex[] dexesToMerge = new Dex[numDex];
-          for (int j = 0; j < numDex; j++) {
-            String fileName = args[random.nextInt(args.length)];
-            fileNames[j] = fileName;
-            dexesToMerge[j] = new Dex(new File(fileName));
+        String[] fileNames = new String[numDex];
+        for (int j = 0; j < numDex; ++j) {
+          int fileIndex = random.nextInt(args.length);
+          fileNames[j] = args[fileIndex];
+        }
+
+        if (!seen.add(fileNames.hashCode())) {
+          // Skip, already seen set of file names with the same hash.
+          continue;
+        }
+
+        Dex[] dexesToMerge = new Dex[numDex];
+        for (int j = 0; j < numDex; ++j) {
+          try {
+            dexesToMerge[j] = new Dex(new File(fileNames[j]));
+          } catch (IOException e) {
+            System.err.println("Error opening " + fileNames[j]);
+            System.err.println(e);
+            System.exit(1);
           }
-          new DexMerger(dexesToMerge, CollisionPolicy.KEEP_FIRST, new DxContext()).merge();
-        } catch (DexIndexOverflowException e) {
-          // ignore index overflow
-        } catch (Throwable t) {
-          System.err.println(
-                  "Problem merging those dexes: " + Arrays.toString(fileNames));
-          throw t;
         }
+        executor.execute(new MergeTask(fileNames, dexesToMerge));
       }
     }
+    executor.shutdown();
+    executor.awaitTermination(8, TimeUnit.HOURS);
   }
 }
diff --git a/dx/tests/127-merge-stress/run b/dx/tests/127-merge-stress/run
index f2df4a507..0bdbd96e1 100755
--- a/dx/tests/127-merge-stress/run
+++ b/dx/tests/127-merge-stress/run
@@ -29,30 +29,47 @@ for i in "$dxjar"; do
     fi
 done
 
-# Find dex files in the tree
-dexes=`find $ANDROID_BUILD_TOP/out -name '*.dex' -printf '%p '`
-dexesinjars=`find $ANDROID_BUILD_TOP/libcore $ANDROID_BUILD_TOP/out -name '*.jar' -exec sh -c 'unzip -l "{}" 2>/dev/null | grep -q classes.dex ' \; -printf '%p '`
-dexesinapks=`find $ANDROID_BUILD_TOP/libcore $ANDROID_BUILD_TOP/out -name '*.apk' -exec sh -c 'unzip -l "{}" 2>/dev/null | grep -q classes.dex ' \; -printf '%p '`
-
-# Select only the valid dex files
-validdexes=""
-for dex in $dexes $dexesinjars $dexesinapks; do
+# Compile and run java merging test
+$JAVAC -Xlint:-options -source 1.7 -target 1.7 -cp $dxjar -d . com/android/dx/merge/MergeTest.java || exit 1
+
+# Find all files that could contain dex bytecode.
+candidates=`find "$ANDROID_BUILD_TOP/out" -name '*.dex' -o -name '*.jar' -o -name '*.apk'`
+validdexes=()
+checksums=()
+for dex in $candidates; do
   if [[ "$dex" = *"core"* ]]; then
     # Ignoring core files as hit limits of merging capability.
     continue
   fi
-  dexdump2 -c $dex >/dev/null
+
+  # Filter out jar files and apks that do not unzip or contain a
+  # classes.dex file. DexMerger assumes file contents based on the
+  # file extension. Android build names some DEX files with a .jar
+  # extension and this raises an exception in DexMerger as it fails
+  # to unzip them.
+  if [[ "$dex" == *"jar" || "$dex" == *"apk" ]]; then
+    unzip -l "$dex" 2>&1 | grep -q -m 1 classes.dex
+    if [[ $? != 0 ]]; then
+      continue
+    fi
+  fi
+
+  # Skip duplicate files
+  checksum=`shasum "$dex" | sed -e 's/ .*//' -e 's/^/_/'`
+  if [[ "$checksums[$checksum]" == "$checksum" ]]; then
+    continue
+  fi
+  checksums[$checksum]=$checksum
+
+  dexdump2 -c $dex >/dev/null 2>&1
   if [ $? -eq 0 ]; then
-    validdexes="$validdexes $dex"
+    validdexes+=("$dex")
   fi
 done
 
-if [ "$validdexes" = "" ]; then
+if [ ${#validdexes[@]} -eq 0 ]; then
   echo No valid DEX files found.
   exit 1
 fi
 
-# Compile and run java merging test
-$JAVAC -Xlint:-options -source 1.7 -target 1.7 -cp $dxjar -d . com/android/dx/merge/MergeTest.java
-java -cp .:$dxjar com.android.dx.merge.MergeTest $validdexes >/dev/null
-
+java -cp .:$dxjar -ea -esa com.android.dx.merge.MergeTest "${validdexes[@]}" > /dev/null
diff --git a/dx/tests/133-source-debug-extension/HelloKt.class b/dx/tests/133-source-debug-extension/HelloKt.class
new file mode 100644
index 000000000..c4e8a67dd
--- /dev/null
+++ b/dx/tests/133-source-debug-extension/HelloKt.class
diff --git a/dx/tests/139-lambda-metafactory/expected.txt b/dx/tests/139-lambda-metafactory/expected.txt
index aabe59ddf..fc21cb47f 100644
--- a/dx/tests/139-lambda-metafactory/expected.txt
+++ b/dx/tests/139-lambda-metafactory/expected.txt
@@ -1,9 +1,9 @@
 Note: Main.java uses unchecked or unsafe operations.
 Note: Recompile with -Xlint:unchecked for details.
 000000: 6465 780a 3033 3800     |magic: "dex\n038\0"
-000008: 0b09 8ceb               |checksum
-00000c: 58f3 aefc 1fe5 e7b3 cb99|signature
-000016: 31ee 6fcf 484a 1019 6918|
+000008: 7e08 861f               |checksum
+00000c: e48a 8e6c 24e0 84f7 c591|signature
+000016: a728 6015 6a2c 6639 54a9|
 000020: 3c07 0000               |file_size:       0000073c
 000024: 7000 0000               |header_size:     00000070
 000028: 7856 3412               |endian_tag:      12345678
@@ -315,10 +315,10 @@ Note: Recompile with -Xlint:unchecked for details.
 0002d0: 7010 0900 0000          |  0000: invoke-direct {v0}, java.lang.Object.<init>:()V // method@0009
 0002d6: 0e00                    |  0003: return-void
                                 |  debug info
-                                |    line_start: 5
+                                |    line_start: 21
                                 |    parameters_size: 0000
                                 |    0000: prologue end
-                                |    0000: line 5
+                                |    0000: line 21
                                 |    end sequence
                                 |
                                 |[2d8] Foo.lambda$bar$0:(ILjava/lang/Object;)V
@@ -338,12 +338,12 @@ Note: Recompile with -Xlint:unchecked for details.
 000300: 6e20 0600 1000          |  000c: invoke-virtual {v0, v1}, java.io.PrintStream.println:(I)V // method@0006
 000306: 0e00                    |  000f: return-void
                                 |  debug info
-                                |    line_start: 8
+                                |    line_start: 24
                                 |    parameters_size: 0002
                                 |    parameter <unnamed> v4
                                 |    parameter k v5
                                 |    0000: prologue end
-                                |    0000: line 8
+                                |    0000: line 24
                                 |    0007: advance pc
                                 |    0007: -local v5 k java.lang.Object
                                 |    end sequence
@@ -363,14 +363,14 @@ Note: Recompile with -Xlint:unchecked for details.
 00032a: 7220 0000 1000          |  0009: invoke-interface {v0, v1}, Consumer.accept:(Ljava/lang/Object;)V // method@0000
 000330: 0e00                    |  000c: return-void
                                 |  debug info
-                                |    line_start: 8
+                                |    line_start: 24
                                 |    parameters_size: 0001
                                 |    parameter j v3
                                 |    0000: prologue end
-                                |    0000: line 8
-                                |    0004: line 9
+                                |    0000: line 24
+                                |    0004: line 25
                                 |    0004: +local v0 consumer Consumer
-                                |    000c: line 10
+                                |    000c: line 26
                                 |    end sequence
                                 |
 000332: 0000                    |
@@ -384,10 +384,10 @@ Note: Recompile with -Xlint:unchecked for details.
 000344: 7010 0900 0000          |  0000: invoke-direct {v0}, java.lang.Object.<init>:()V // method@0009
 00034a: 0e00                    |  0003: return-void
                                 |  debug info
-                                |    line_start: 13
+                                |    line_start: 29
                                 |    parameters_size: 0000
                                 |    0000: prologue end
-                                |    0000: line 13
+                                |    0000: line 29
                                 |    end sequence
                                 |
                                 |[34c] Main.main:([Ljava/lang/String;)V
@@ -403,12 +403,12 @@ Note: Recompile with -Xlint:unchecked for details.
 000368: 6e20 0200 1000          |  0006: invoke-virtual {v0, v1}, Foo.bar:(I)V // method@0002
 00036e: 0e00                    |  0009: return-void
                                 |  debug info
-                                |    line_start: 15
+                                |    line_start: 31
                                 |    parameters_size: 0001
                                 |    parameter args v2
                                 |    0000: prologue end
-                                |    0000: line 15
-                                |    0009: line 16
+                                |    0000: line 31
+                                |    0009: line 32
                                 |    end sequence
                                 |
                                 |[370] annotations directory
@@ -614,48 +614,48 @@ Note: Recompile with -Xlint:unchecked for details.
                                 |
                                 |byte_data:
                                 |[5de] debug info
-0005de: 05                      |line_start: 5
+0005de: 15                      |line_start: 21
 0005df: 00                      |parameters_size: 0000
 0005e0: 07                      |0000: prologue end
-0005e1: 0e                      |0000: line 5
+0005e1: 0e                      |0000: line 21
 0005e2: 00                      |end sequence
                                 |
                                 |[5e3] debug info
-0005e3: 08                      |line_start: 8
+0005e3: 18                      |line_start: 24
 0005e4: 02                      |parameters_size: 0002
 0005e5: 00                      |parameter <unnamed> v4
 0005e6: 24                      |parameter k v5
 0005e7: 07                      |0000: prologue end
-0005e8: 0e                      |0000: line 8
+0005e8: 0e                      |0000: line 24
 0005e9: 0107                    |0007: advance pc
 0005eb: 0505                    |0007: -local v5 k java.lang.Object
 0005ed: 00                      |end sequence
                                 |
                                 |[5ee] debug info
-0005ee: 08                      |line_start: 8
+0005ee: 18                      |line_start: 24
 0005ef: 01                      |parameters_size: 0001
 0005f0: 23                      |parameter j v3
 0005f1: 07                      |0000: prologue end
-0005f2: 0e                      |0000: line 8
-0005f3: 4b                      |0004: line 9
+0005f2: 0e                      |0000: line 24
+0005f3: 4b                      |0004: line 25
 0005f4: 0300 2002               |0004: +local v0 consumer Consumer
-0005f8: 87                      |000c: line 10
+0005f8: 87                      |000c: line 26
 0005f9: 00                      |end sequence
                                 |
                                 |[5fa] debug info
-0005fa: 0d                      |line_start: 13
+0005fa: 1d                      |line_start: 29
 0005fb: 00                      |parameters_size: 0000
 0005fc: 07                      |0000: prologue end
-0005fd: 0e                      |0000: line 13
+0005fd: 0e                      |0000: line 29
 0005fe: 00                      |end sequence
                                 |
                                 |[5ff] debug info
-0005ff: 0f                      |line_start: 15
+0005ff: 1f                      |line_start: 31
 000600: 01                      |parameters_size: 0001
 000601: 1e                      |parameter args v2
 000602: 07                      |0000: prologue end
-000603: 0e                      |0000: line 15
-000604: 96                      |0009: line 16
+000603: 0e                      |0000: line 31
+000604: 96                      |0009: line 32
 000605: 00                      |end sequence
                                 |
                                 |[606] annotation
diff --git a/dx/tests/run-all-tests b/dx/tests/run-all-tests
index dce490c91..1c6be0273 100755
--- a/dx/tests/run-all-tests
+++ b/dx/tests/run-all-tests
@@ -34,7 +34,7 @@ progdir=`pwd`
 prog="${progdir}"/`basename "${prog}"`
 
 # Command-line options
-skip_tests="127-merge-stress"
+skip_tests=""
 sequential="no"
 usage="no"
 while [[ "$1" == "-"* ]]; do
@@ -67,7 +67,7 @@ failed=()
 skipped=()
 
 # Tests failing and require attention (e.g. 115-merge)
-known_bad="100-local-mismatch 115-merge 119-merge-conflict 133-source-debug-extension"
+known_bad="100-local-mismatch 115-merge 119-merge-conflict"
 
 function display_results {
   printf    "\n\nTest Results\n"