From cb523bf186baa02b7478c81a7a64d2a2ee3d6b5f Mon Sep 17 00:00:00 2001 From: Oliver Nguyen Date: Fri, 10 Jul 2020 10:46:29 -0700 Subject: Implement memory-mapped ExecutionData class. This allows apps to write coverage to disk during execution and not require any flushing mechanism in order to collect coverage. The results will be left in /data/misc/trace/jacoco-*.mm.ec and can be collected later for coverage analysis. These .mm.ec files conform to the current exec file format and can be loaded with the command-line tool. Test: m -j EMMA_INSTRUMENT=true EMMA_INSTRUMENT_FRAMEWORK=true and generate coverage report from /data/misc/trace Bug: 147904124 Change-Id: If08492405b86a063ae714c259a2fb954bc5aa946 --- .../src/org/jacoco/agent/rt/internal/Offline.java | 76 ++++- .../org/jacoco/core/data/MappedExecutionData.java | 324 +++++++++++++++++++++ 2 files changed, 399 insertions(+), 1 deletion(-) create mode 100644 org.jacoco.core/src/org/jacoco/core/data/MappedExecutionData.java diff --git a/org.jacoco.agent.rt/src/org/jacoco/agent/rt/internal/Offline.java b/org.jacoco.agent.rt/src/org/jacoco/agent/rt/internal/Offline.java index ee9d40b1..445722af 100644 --- a/org.jacoco.agent.rt/src/org/jacoco/agent/rt/internal/Offline.java +++ b/org.jacoco.agent.rt/src/org/jacoco/agent/rt/internal/Offline.java @@ -11,6 +11,8 @@ *******************************************************************************/ package org.jacoco.agent.rt.internal; +import java.io.File; +import java.io.IOException; import java.util.HashMap; import java.util.Map; import java.util.Properties; @@ -18,6 +20,7 @@ import java.util.Properties; import org.jacoco.core.data.ExecutionData; import org.jacoco.core.data.IExecutionData; import org.jacoco.core.data.ExecutionDataStore; +import org.jacoco.core.data.MappedExecutionData; import org.jacoco.core.runtime.AgentOptions; import org.jacoco.core.runtime.RuntimeData; @@ -30,6 +33,7 @@ public final class Offline { // BEGIN android-change // private static final RuntimeData DATA; private static final Map DATA = new HashMap(); + private static int PID = -1; // END android-change private static final String CONFIG_RESOURCE = "/jacoco-agent.properties"; @@ -64,7 +68,49 @@ public final class Offline { synchronized (DATA) { IExecutionData entry = DATA.get(classid); if (entry == null) { - entry = new ExecutionData(classid, classname, probecount); + // BEGIN android-change + // The list below are not allowed to use the memory-mapped + // implementation due either: + // 1) They are loaded at VM initialization time. + // a) android.* + // b) dalvik.* + // c) libcore.* + // 2) They are used by the memory-mapped execution data + // implementation, which would cause a stack overflow due + // to the circular dependency. + // a) com.android.i18n.* + // b) com.android.icu.* + // c) java.* + // d) org.apache.* + // e) sun.* + // These classes can still have coverage collected through the + // normal execution data process, but requires a flush to be done + // instead of simply being available on disk at all times. + if (classname.startsWith("android/") + || classname.startsWith("com/android/i18n/") + || classname.startsWith("com/android/icu/") + || classname.startsWith("dalvik/") + || classname.startsWith("java/") + || classname.startsWith("libcore/") + || classname.startsWith("org/apache/") + || classname.startsWith("sun/")) { + entry = new ExecutionData(classid, classname, probecount); + } else { + try { + int pid = getPid(); + if (PID != pid) { + PID = pid; + rebuildExecutionData(pid); + } + entry = new MappedExecutionData( + classid, classname, probecount); + } catch (IOException e) { + // Fall back to non-memory-mapped execution data. + entry = new ExecutionData( + classid, classname, probecount); + } + } + // END android-change DATA.put(classid, entry); } else { entry.assertCompatibility(classid, classname, probecount); @@ -72,6 +118,34 @@ public final class Offline { return entry; } } + + private static void rebuildExecutionData(int pid) throws IOException { + MappedExecutionData.prepareFile(pid); + synchronized (DATA) { + for (IExecutionData execData : DATA.values()) { + if (execData instanceof MappedExecutionData) { + // Create new instances of MappedExecutionData using the + // new file, but don't copy the old data. Old data will + // remain in its existing file and can be merged during + // post-processing. + DATA.put( + execData.getId(), + new MappedExecutionData( + execData.getId(), + execData.getName(), + execData.getProbeCount())); + } + } + } + } + + /** + * Helper function to determine the pid of this process. + */ + private static int getPid() throws IOException { + // Read /proc/self and resolve it to obtain its pid. + return Integer.parseInt(new File("/proc/self").getCanonicalFile().getName()); + } // END android-change /** diff --git a/org.jacoco.core/src/org/jacoco/core/data/MappedExecutionData.java b/org.jacoco.core/src/org/jacoco/core/data/MappedExecutionData.java new file mode 100644 index 00000000..39b90fb6 --- /dev/null +++ b/org.jacoco.core/src/org/jacoco/core/data/MappedExecutionData.java @@ -0,0 +1,324 @@ +// BEGIN android-change +package org.jacoco.core.data; + +import static java.lang.String.format; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.charset.Charset; +import java.nio.ByteBuffer; +import java.nio.MappedByteBuffer; + +import org.jacoco.core.data.ExecutionDataWriter; + +/** + * Memory-mapped execution data implementation. + * + * Stores the probe data in a section of a memory-mapped file, so that it will be available without + * requiring a flush of the coverage data. + */ +public final class MappedExecutionData implements IExecutionData { + + private static FileChannel CHANNEL; + + private final long id; + private final String name; + private final int probeCount; + + // Stores the memory-mapped byte buffer containing the probe data, packed as bits. + private MappedByteBuffer probeBuffer; + + /** + * Creates the mapped execution data. + */ + public MappedExecutionData(final long id, final String name, final int probeCount) + throws IOException { + this.id = id; + this.name = name; + this.probeCount = probeCount; + + createMemoryMappedProbeArray(id, name, probeCount); + } + + /** + * Maps a section of the already-opened file for the probe data. + * + * @param id the class id + * @param name the VM class name + * @param probeCount the number of probes for this class + */ + private void createMemoryMappedProbeArray( + final long id, final String name, final int probeCount) throws IOException { + // TODO: Figure out a more portable way to do this. Currently needed to avoid + // cross-VM race conditions when writing/mapping the file. Since most processes + // are forked from zygote, they will initially share the same output file, but + // without synchronization across VMs, this can cause simultaneous maps/writes from + // different VMs to interrupt each other, resulting in malformed execution data. + // Adding file locks did not resolve this issue. Note that only new classes that + // were loaded after the fork will end up in the new file. Classes that were + // already loaded will end up in the original output file for the pre-forked + // process, which is OK. + if (CHANNEL == null) { + throw new IOException("Memory mapped file was not initialized."); + } + + synchronized (CHANNEL) { + int byteCount = (probeCount + 7) / 8; + + // Write the ExecutionData block info. + ByteBuffer execDataBuffer = ByteBuffer.allocate(11); + execDataBuffer.put(ExecutionDataWriter.BLOCK_EXECUTIONDATA); + execDataBuffer.putLong(id); + execDataBuffer.putShort((short) name.length()); + execDataBuffer.flip(); + CHANNEL.write(execDataBuffer); + CHANNEL.write(ByteBuffer.wrap(name.getBytes(Charset.forName("UTF-8")))); + + // Write the probe info and map part of this file for the probe data. + CHANNEL.write(toVarIntByteBuffer(probeCount)); + probeBuffer = CHANNEL.map(FileChannel.MapMode.READ_WRITE, CHANNEL.position(), byteCount); + CHANNEL.position(CHANNEL.position() + byteCount); + } + } + + /** + * Creates the output file that will be mapped for probe data. + */ + public static void prepareFile(int pid) throws IOException { + // Write header information to the file. + ByteBuffer headerBuffer = ByteBuffer.allocate(5); + headerBuffer.put(ExecutionDataWriter.BLOCK_HEADER); + headerBuffer.putChar(ExecutionDataWriter.MAGIC_NUMBER); + headerBuffer.putChar(ExecutionDataWriter.FORMAT_VERSION); + headerBuffer.flip(); + + // If this file already exists (due to pid re-usage), the previous coverage data + // will be lost when the file is overwritten. + File outputFile = new File("/data/misc/trace/jacoco-" + pid + ".mm.ec"); + CHANNEL = new RandomAccessFile(outputFile, "rw").getChannel(); + synchronized (CHANNEL) { + CHANNEL.write(headerBuffer); + } + } + + /** + * Writes a variable-length int to a {@link ByteBuffer}. + * + * @param value the value to write + * @return a ByteBuffer that can be used to write to a FileChannel containing the value + */ + private ByteBuffer toVarIntByteBuffer(int value) { + ByteBuffer buffer = ByteBuffer.allocate(5); + if (value == 0) { + buffer.put((byte) 0); + } else { + while (value > 0) { + if ((value & 0xFFFFFF80) == 0) { + buffer.put((byte) value); + } else { + buffer.put((byte) (0x80 | (value & 0x7F))); + } + value >>>= 7; + } + } + buffer.flip(); + return buffer; + } + + /** + * Return the unique identifier for this class. The identifier is the CRC64 + * checksum of the raw class file definition. + * + * @return class identifier + */ + public long getId() { + return id; + } + + /** + * The VM name of the class. + * + * @return VM name + */ + public String getName() { + return name; + } + + /** + * The number of instrumentation probes for this class. + * + * @return number of probes + */ + public int getProbeCount() { + return probeCount; + } + + /** + * Returns a copy of the probe data as a boolean array. + * + * Changes to the returned array will not be reflected in the execution data. + * + * @return copy of the probe data + */ + public boolean[] getProbesCopy() { + final int bytes = (probeCount + 7) / 8; + boolean[] probes = new boolean[probeCount]; + for (int index = 0; index < probeCount; index += 8) { + byte byteProbe = probeBuffer.get(index / 8); + for (int bit = 0; (bit < 8) && ((index + bit) < probeCount); bit++) { + probes[index + bit] = ((byteProbe & 0x1) > 0); + byteProbe >>>= 1; + } + } + return probes; + } + + /** + * Sets all probes to false. + */ + public void reset() { + final int bytes = (probeCount + 7) / 8; + synchronized (probeBuffer) { + for (int i = 0; i < bytes; i++) { + probeBuffer.put(i, (byte) 0); + } + } + } + + /** + * Checks whether any probe has been hit. + * + * @return true, if at least one probe has been hit + */ + public boolean hasHits() { + final int bytes = (probeCount + 7) / 8; + synchronized (probeBuffer) { + for (int i = 0; i < bytes; i++) { + if (probeBuffer.get(i) > 0) { + return true; + } + } + } + return false; + } + + /** + * Merges the given execution data into the probe data of this object. I.e. + * a probe entry in this object is marked as executed (true) if + * this probe or the corresponding other probe was executed. So the result + * is + * + * 
+	 * A or B
+	 *
+ * + * The probe array of the other object is not modified. + * + * @param other + * execution data to merge + */ + public void merge(final IExecutionData other) { + merge(other, true); + } + + /** + * Merges the given execution data into the probe data of this object. A + * probe in this object is set to the value of flag if the + * corresponding other probe was executed. For flag==true this + * corresponds to + * + * 
+	 * A or B
+	 *
+ * + * For flag==false this can be considered as a subtraction + * + * 
+	 * A and not B
+	 *
+ * + * The probe array of the other object is not modified. + * + * @param other + * execution data to merge + * @param flag + * merge mode + */ + public void merge(final IExecutionData other, final boolean flag) { + synchronized (probeBuffer) { + for (int i = 0; i < probeCount; i++) { + if (other.getProbe(i)) { + setProbe(i); + } + } + } + } + + /** + * Asserts that this execution data object is compatible with the given + * parameters. The purpose of this check is to detect a very unlikely class + * id collision. + * + * @param id + * other class id, must be the same + * @param name + * other name, must be equal to this name + * @param probecount + * probe data length, must be the same as for this data + * @throws IllegalStateException + * if the given parameters do not match this instance + */ + public void assertCompatibility(final long id, final String name, final int probeCount) + throws IllegalStateException { + if (this.id != id) { + throw new IllegalStateException(format( + "Different ids (%016x and %016x).", Long.valueOf(this.id), + Long.valueOf(id))); + } + if (!this.name.equals(name)) { + throw new IllegalStateException(format( + "Different class names %s and %s for id %016x.", this.name, + name, Long.valueOf(id))); + } + if (this.probeCount != probeCount) { + throw new IllegalStateException(format( + "Incompatible execution data for class %s with id %016x.", + name, Long.valueOf(id))); + } + } + + /** + * Returns the execution data probe for a given index. A value of + * true indicates that the corresponding probe was + * executed. + * + * @param index the probe's index to look up + * + * @return probe data + */ + public boolean getProbe(final int index) { + int offset = index / 8; + int bit = 1 << (index % 8); + return (probeBuffer.get(offset) & bit) != 0; + } + + /** + * Sets the execution data probe at the given index to true. + * + * @param index the probe's index to set + */ + public void setProbe(final int index) { + int offset = index / 8; + int bit = 1 << (index % 8); + byte currentValue = probeBuffer.get(offset); + if ((currentValue & bit) == 0) { + synchronized (probeBuffer) { + probeBuffer.put(offset, (byte) (probeBuffer.get(offset) | bit)); + } + } + } +} +// END android-change -- cgit v1.2.3