diff options
Diffstat (limited to 'platform/lang-impl/src/com/intellij/util/indexing/MapReduceIndex.java')
| -rw-r--r-- | platform/lang-impl/src/com/intellij/util/indexing/MapReduceIndex.java | 153 |
1 files changed, 135 insertions, 18 deletions
diff --git a/platform/lang-impl/src/com/intellij/util/indexing/MapReduceIndex.java b/platform/lang-impl/src/com/intellij/util/indexing/MapReduceIndex.java index 3ad505baf207..ebb7911e82be 100644 --- a/platform/lang-impl/src/com/intellij/util/indexing/MapReduceIndex.java +++ b/platform/lang-impl/src/com/intellij/util/indexing/MapReduceIndex.java @@ -22,18 +22,16 @@ import com.intellij.openapi.util.*; import com.intellij.openapi.util.io.BufferExposingByteArrayOutputStream; import com.intellij.openapi.util.io.ByteSequence; import com.intellij.psi.search.GlobalSearchScope; -import com.intellij.util.Processor; -import com.intellij.util.SmartList; -import com.intellij.util.SystemProperties; +import com.intellij.util.*; import com.intellij.util.io.*; +import com.intellij.util.io.DataOutputStream; import gnu.trove.THashMap; import gnu.trove.TObjectObjectProcedure; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import java.io.DataInputStream; -import java.io.File; -import java.io.IOException; +import java.io.*; +import java.nio.charset.Charset; import java.util.*; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -56,6 +54,7 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val private PersistentHashMap<Integer, Collection<Key>> myInputsIndex; private PersistentHashMap<Integer, ByteSequence> myContents; private PersistentHashMap<Integer, Integer> myInputsSnapshotMapping; + private PersistentHashMap<Integer, String> myIndexingTrace; private final ReentrantReadWriteLock myLock = new ReentrantReadWriteLock(); @@ -127,6 +126,10 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val cleanMapping(myInputsSnapshotMapping); myInputsSnapshotMapping = createInputSnapshotMapping(); } + if (myIndexingTrace != null) { + cleanMapping(myIndexingTrace); + myIndexingTrace = createIndexingTrace(); + } if (myContents != null) { cleanMapping(myContents); myContents = createContentsIndex(); @@ -161,6 +164,31 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val } } + private PersistentHashMap<Integer, String> createIndexingTrace() throws IOException { + assert myIndexId != null; + final File mapFile = new File(IndexInfrastructure.getIndexRootDir(myIndexId), "indextrace"); + try { + return new PersistentHashMap<Integer, String>(mapFile, EnumeratorIntegerDescriptor.INSTANCE, + new DataExternalizer<String>() { + @Override + public void save(@NotNull DataOutput out, String value) throws IOException { + out.write((byte[])CompressionUtil.compressCharSequence(value, Charset.defaultCharset())); + } + + @Override + public String read(@NotNull DataInput in) throws IOException { + byte[] b = new byte[((InputStream)in).available()]; + in.readFully(b); + return (String)CompressionUtil.uncompressCharSequence(b, Charset.defaultCharset()); + } + }, 4096); + } + catch (IOException ex) { + IOUtil.deleteAllFilesStartingWith(mapFile); + throw ex; + } + } + private static void cleanMapping(@NotNull PersistentHashMap<?, ?> index) { final File baseFile = index.getBaseFile(); try { @@ -178,6 +206,7 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val getReadLock().lock(); doForce(myInputsIndex); doForce(myInputsSnapshotMapping); + doForce(myIndexingTrace); doForce(myContents); myStorage.flush(); } @@ -216,6 +245,7 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val finally { doClose(myInputsIndex); doClose(myInputsSnapshotMapping); + doClose(myIndexingTrace); doClose(myContents); } } @@ -283,6 +313,9 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val myInputsSnapshotMapping = createInputSnapshotMapping(); } myInputsIndex = createInputsIndex(); + if (DebugAssertions.EXTRA_SANITY_CHECKS && myIndexId != null) { + myIndexingTrace = createIndexingTrace(); + } } @Nullable @@ -318,7 +351,8 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val if (myContents != null && weProcessPhysicalContent && content != null) { try { - hashId = getHashOfContent((FileContent)content); + FileContent fileContent = (FileContent)content; + hashId = getHashOfContent(fileContent); if (doReadSavedPersistentData) { if (!myContents.isBusyReading()) { // avoid blocking read, we can calculate index value ByteSequence bytes = myContents.get(hashId); @@ -326,7 +360,19 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val data = deserializeSavedPersistentData(bytes); havePersistentData = true; if (DebugAssertions.EXTRA_SANITY_CHECKS) { - DebugAssertions.assertTrue(myIndexer.map(content).equals(data)); + Map<Key, Value> contentData = myIndexer.map(content); + boolean sameValueForSavedIndexedResultAndCurrentOne = contentData.equals(data); + if (!sameValueForSavedIndexedResultAndCurrentOne) { + DebugAssertions.error( + "Unexpected difference in indexing of %s by index %s, file type %s, charset %s\ndiff %s\nprevious indexed info %s", + fileContent.getFile(), + myIndexId, + fileContent.getFileType(), + ((FileContentImpl)fileContent).getCharset(), + buildDiff(data, contentData), + myIndexingTrace.get(hashId) + ); + } } } } else { @@ -344,16 +390,27 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val if (data == null) data = content != null ? myIndexer.map(content) : Collections.<Key, Value>emptyMap(); if (hashId != null && !havePersistentData) { - savePersistentData(data, hashId, skippedReadingPersistentDataButMayHaveIt); + boolean saved = savePersistentData(data, hashId, skippedReadingPersistentDataButMayHaveIt); + if (DebugAssertions.EXTRA_SANITY_CHECKS) { + if (saved) { + + FileContent fileContent = (FileContent)content; + try { + myIndexingTrace.put(hashId, ((FileContentImpl)fileContent).getCharset() + "," + fileContent.getFileType()+"," + fileContent.getFile().getPath() + "," + + ExceptionUtil.getThrowableText(new Throwable())); + } catch (IOException ex) { + LOG.error(ex); + } + } + } } ProgressManager.checkCanceled(); final NotNullComputable<Collection<Key>> oldKeysGetter; final int savedInputId; - if (myHasSnapshotMapping && weProcessPhysicalContent) { + if (myHasSnapshotMapping) { try { - - oldKeysGetter = new NotNullComputable<Collection<Key>>() { + final NotNullComputable<Collection<Key>> keysForGivenInputId = new NotNullComputable<Collection<Key>>() { @NotNull @Override public Collection<Key> compute() { @@ -369,15 +426,38 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val } return currentKeys; - } catch (IOException e) { + } + catch (IOException e) { throw new RuntimeException(e); } } }; - if (content instanceof FileContent) { - savedInputId = getHashOfContent((FileContent)content); + if (weProcessPhysicalContent) { + if (content instanceof FileContent) { + savedInputId = getHashOfContent((FileContent)content); + } + else { + savedInputId = NULL_MAPPING; + } + oldKeysGetter = keysForGivenInputId; } else { + oldKeysGetter = new NotNullComputable<Collection<Key>>() { + @NotNull + @Override + public Collection<Key> compute() { + try { + Collection<Key> oldKeys = myInputsIndex.get(inputId); + if (oldKeys == null) { + return keysForGivenInputId.compute(); + } + return oldKeys; + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + }; savedInputId = NULL_MAPPING; } } catch (IOException ex) { @@ -429,6 +509,41 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val }; } + private StringBuilder buildDiff(Map<Key, Value> data, Map<Key, Value> contentData) { + StringBuilder moreInfo = new StringBuilder(); + if (contentData.size() != data.size()) { + moreInfo.append("Indexer has different number of elements, previously ").append(data.size()).append(" after ") + .append(contentData.size()).append("\n"); + } else { + moreInfo.append("total ").append(contentData.size()).append(" entries\n"); + } + + for(Map.Entry<Key, Value> keyValueEntry:contentData.entrySet()) { + if (!data.containsKey(keyValueEntry.getKey())) { + moreInfo.append("Previous data doesn't contain:").append(keyValueEntry.getKey()).append( " with value ").append(keyValueEntry.getValue()).append("\n"); + } + else { + Value value = data.get(keyValueEntry.getKey()); + if (!Comparing.equal(keyValueEntry.getValue(), value)) { + moreInfo.append("Previous data has different value for key:").append(keyValueEntry.getKey()).append( ", new value ").append(keyValueEntry.getValue()).append( ", oldValue:").append(value).append("\n"); + } + } + } + + for(Map.Entry<Key, Value> keyValueEntry:data.entrySet()) { + if (!contentData.containsKey(keyValueEntry.getKey())) { + moreInfo.append("New data doesn't contain:").append(keyValueEntry.getKey()).append( " with value ").append(keyValueEntry.getValue()).append("\n"); + } + else { + Value value = contentData.get(keyValueEntry.getKey()); + if (!Comparing.equal(keyValueEntry.getValue(), value)) { + moreInfo.append("New data has different value for key:").append(keyValueEntry.getKey()).append( " new value ").append(value).append( ", oldValue:").append(keyValueEntry.getValue()).append("\n"); + } + } + } + return moreInfo; + } + private Map<Key, Value> deserializeSavedPersistentData(ByteSequence bytes) throws IOException { DataInputStream stream = new DataInputStream(new UnsyncByteArrayInputStream(bytes.getBytes(), bytes.getOffset(), bytes.getLength())); int pairs = DataInputOutputUtil.readINT(stream); @@ -447,8 +562,9 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val if (previouslyCalculatedContentHashId == null) { byte[] hash = content instanceof FileContentImpl ? ((FileContentImpl)content).getHash():null; if (hash == null) { + Charset charset = content instanceof FileContentImpl ? ((FileContentImpl)content).getCharset() : null; previouslyCalculatedContentHashId = ContentHashesSupport - .calcContentHashIdWithFileType(content.getContent(), content.getFileType()); + .calcContentHashIdWithFileType(content.getContent(), charset, content.getFileType()); } else { previouslyCalculatedContentHashId = ContentHashesSupport.enumerateHash(hash); } @@ -459,9 +575,9 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val private static final ThreadLocalCachedByteArray ourSpareByteArray = new ThreadLocalCachedByteArray(); - private void savePersistentData(Map<Key, Value> data, int id, boolean delayedReading) { + private boolean savePersistentData(Map<Key, Value> data, int id, boolean delayedReading) { try { - if (delayedReading && myContents.containsMapping(id)) return; + if (delayedReading && myContents.containsMapping(id)) return false; BufferExposingByteArrayOutputStream out = new BufferExposingByteArrayOutputStream(ourSpareByteArray.getBuffer(4 * data.size())); DataOutputStream stream = new DataOutputStream(out); int size = data.size(); @@ -496,6 +612,7 @@ public class MapReduceIndex<Key, Value, Input> implements UpdatableIndex<Key,Val } catch (IOException ex) { throw new RuntimeException(ex); } + return true; } private static final com.intellij.openapi.util.Key<Integer> ourSavedContentHashIdKey = com.intellij.openapi.util.Key.create("saved.content.hash.id"); |