path: root/platform/util/src/com/intellij/util/io/PersistentEnumeratorBase.java

/*
 * Copyright 2000-2013 JetBrains s.r.o.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.intellij.util.io;

import com.intellij.openapi.Forceable;
import com.intellij.openapi.diagnostic.Logger;
import com.intellij.openapi.util.io.FileUtil;
import com.intellij.util.CommonProcessors;
import com.intellij.util.IncorrectOperationException;
import com.intellij.util.Processor;
import com.intellij.util.containers.SLRUMap;
import com.intellij.util.containers.ShareableKey;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.jetbrains.annotations.TestOnly;

import java.io.Closeable;
import java.io.File;
import java.io.Flushable;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;

/**
 * @author max
 * @author jeka
 */
@SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
public abstract class PersistentEnumeratorBase<Data> implements Forceable, Closeable {
  protected static final Logger LOG = Logger.getInstance("#com.intellij.util.io.PersistentEnumerator");
  protected static final int NULL_ID = 0;

  private static final int META_DATA_OFFSET = 4;
  protected static final int DATA_START = META_DATA_OFFSET + 16;

  protected final ResizeableMappedFile myStorage;
  private final boolean myAssumeDifferentSerializedBytesMeansObjectsInequality;
  private final AppendableStorageBackedByResizableMappedFile myKeyStorage;

  private boolean myClosed = false;
  private boolean myDirty = false;
  protected final KeyDescriptor<Data> myDataDescriptor;

  private static final CacheKey ourFlyweight = new FlyweightKey();

  protected final File myFile;
  private boolean myCorrupted = false;
  private final int myInitialSize;
  private final Version myVersion;
  private RecordBufferHandler<PersistentEnumeratorBase> myRecordHandler;
  private volatile boolean myDirtyStatusUpdateInProgress;
  private Flushable myMarkCleanCallback;
  private final boolean myDoCaching;

  public static class Version {
    private final int correctlyClosedMagic;
    private final int dirtyMagic;

    public Version(int _correctlyClosedMagic, int _dirtyMagic) {
      correctlyClosedMagic = _correctlyClosedMagic;
      dirtyMagic = _dirtyMagic;
      assert correctlyClosedMagic != dirtyMagic;
    }
  }

  public abstract static class RecordBufferHandler<T extends PersistentEnumeratorBase> {
    abstract int recordWriteOffset(T enumerator, byte[] buf);
    @NotNull
    abstract byte[] getRecordBuffer(T enumerator);
    abstract void setupRecord(T enumerator, int hashCode, final int dataOffset, final byte[] buf);
  }

  private static class CacheKey implements ShareableKey {
    public PersistentEnumeratorBase owner;
    public Object key;

    private CacheKey(Object key, PersistentEnumeratorBase owner) {
      this.key = key;
      this.owner = owner;
    }

    @Override
    public ShareableKey getStableCopy() {
      return this;
    }

    public boolean equals(final Object o) {
      if (this == o) return true;
      if (!(o instanceof CacheKey)) return false;

      final CacheKey cacheKey = (CacheKey)o;

      if (!key.equals(cacheKey.key)) return false;
      if (!owner.equals(cacheKey.owner)) return false;

      return true;
    }

    public int hashCode() {
      return key.hashCode();
    }
  }

  private static CacheKey sharedKey(Object key, PersistentEnumeratorBase owner) {
    ourFlyweight.key = key;
    ourFlyweight.owner = owner;
    return ourFlyweight;
  }

  private static final int ENUMERATION_CACHE_SIZE;
  static {
    String property = System.getProperty("idea.enumerationCacheSize");
    ENUMERATION_CACHE_SIZE = property == null ? 8192 : Integer.valueOf(property);
  }

  private static final SLRUMap<Object, Integer> ourEnumerationCache = new SLRUMap<Object, Integer>(ENUMERATION_CACHE_SIZE, ENUMERATION_CACHE_SIZE);

  @TestOnly
  public static void clearCacheForTests() {
    ourEnumerationCache.clear();
  }

  public static class CorruptedException extends IOException {
    @SuppressWarnings({"HardCodedStringLiteral"})
    public CorruptedException(File file) {
      this("PersistentEnumerator storage corrupted " + file.getPath());
    }

    protected CorruptedException(String message) {
      super(message);
    }
  }

  public static class VersionUpdatedException extends CorruptedException {
    @SuppressWarnings({"HardCodedStringLiteral"})
    public VersionUpdatedException(File file) {
      super("PersistentEnumerator storage corrupted " + file.getPath());
    }
  }

  public PersistentEnumeratorBase(@NotNull File file,
                                  @NotNull ResizeableMappedFile storage,
                                  @NotNull KeyDescriptor<Data> dataDescriptor,
                                  int initialSize,
                                  @NotNull Version version,
                                  @NotNull RecordBufferHandler<? extends PersistentEnumeratorBase> recordBufferHandler,
                                  boolean doCaching) throws IOException {
    myDataDescriptor = dataDescriptor;
    myFile = file;
    myInitialSize = initialSize;
    myVersion = version;
    myRecordHandler = (RecordBufferHandler<PersistentEnumeratorBase>)recordBufferHandler;
    myDoCaching = doCaching;

    if (!file.exists()) {
      FileUtil.delete(keystreamFile());
      if (!FileUtil.createIfDoesntExist(file)) {
        throw new IOException("Cannot create empty file: " + file);
      }
    }

    myStorage = storage;

    lockStorage();
    try {
      if (myStorage.length() == 0) {
        try {
          markDirty(true);
          putMetaData(0);
          putMetaData2(0);
          setupEmptyFile();
        }
        catch (RuntimeException e) {
          LOG.info(e);
          myStorage.close();
          if (e.getCause() instanceof IOException) {
            throw (IOException)e.getCause();
          }
          throw e;
        }
        catch (IOException e) {
          LOG.info(e);
          myStorage.close();
          throw e;
        }
        catch (Exception e) {
          LOG.info(e);
          myStorage.close();
          throw new CorruptedException(file);
        }
      }
      else {
        int sign;
        try {
          sign = myStorage.getInt(0);
        }
        catch(Exception e) {
          LOG.info(e);
          sign = myVersion.dirtyMagic;
        }
        if (sign != myVersion.correctlyClosedMagic) {
          myStorage.close();
          if (sign != myVersion.dirtyMagic) throw new VersionUpdatedException(file);
          throw new CorruptedException(file);
        }
      }
    }
    finally {
      unlockStorage();
    }

    if (myDataDescriptor instanceof InlineKeyDescriptor) {
      myKeyStorage = null;
    }
    else {
      try {
        myKeyStorage = new AppendableStorageBackedByResizableMappedFile(keystreamFile(), initialSize, myStorage.getPagedFileStorage().getStorageLockContext(), PagedFileStorage.MB, false);
      }
      catch (IOException e) {
        myStorage.close();
        throw e;
      }
      catch (Throwable e) {
        LOG.info(e);
        myStorage.close();
        throw new CorruptedException(file);
      }
    }
    myAssumeDifferentSerializedBytesMeansObjectsInequality = myDataDescriptor instanceof DifferentSerializableBytesImplyNonEqualityPolicy;
  }

  public void lockStorage() {
    myStorage.getPagedFileStorage().lock();
  }

  public void unlockStorage() {
    myStorage.getPagedFileStorage().unlock();
  }

  protected abstract void setupEmptyFile() throws IOException;

  @NotNull
  public final RecordBufferHandler<PersistentEnumeratorBase> getRecordHandler() {
    return myRecordHandler;
  }

  public void setRecordHandler(@NotNull RecordBufferHandler<PersistentEnumeratorBase> recordHandler) {
    myRecordHandler = recordHandler;
  }

  public void setMarkCleanCallback(Flushable markCleanCallback) {
    myMarkCleanCallback = markCleanCallback;
  }

  public Data getValue(int keyId, int processingKey) throws IOException {
    return valueOf(keyId);
  }

  protected int tryEnumerate(Data value) throws IOException {
    return doEnumerate(value, true, false);
  }

  private int doEnumerate(Data value, boolean onlyCheckForExisting, boolean saveNewValue) throws IOException {
    if (myDoCaching && !saveNewValue) {
      synchronized (ourEnumerationCache) {
        final Integer cachedId = ourEnumerationCache.get(sharedKey(value, this));
        if (cachedId != null) return cachedId.intValue();
      }
    }

    final int id;
    try {
      id = enumerateImpl(value, onlyCheckForExisting, saveNewValue);
    }
    catch (IOException io) {
      markCorrupted();
      throw io;
    }
    catch (Throwable e) {
      markCorrupted();
      LOG.info(e);
      throw new IOException(e);
    }

    if (myDoCaching && id != NULL_ID) {
      synchronized (ourEnumerationCache) {
        ourEnumerationCache.put(new CacheKey(value, this), id);
      }
    }

    return id;
  }

  public int enumerate(Data value) throws IOException {
    return doEnumerate(value, false, false);
  }

  public interface DataFilter {
    boolean accept(int id);
  }

  protected void putMetaData(long data) throws IOException {
    lockStorage();
    try {
      if (myStorage.length() < META_DATA_OFFSET + 8 || getMetaData() != data) myStorage.putLong(META_DATA_OFFSET, data);
    }
    finally {
      unlockStorage();
    }
  }

  protected long getMetaData() throws IOException {
    lockStorage();
    try {
      return myStorage.getLong(META_DATA_OFFSET);
    }
    finally {
      unlockStorage();
    }
  }

  protected void putMetaData2(long data) throws IOException {
    lockStorage();
    try {
      if (myStorage.length() < META_DATA_OFFSET + 16 || getMetaData2() != data) myStorage.putLong(META_DATA_OFFSET + 8, data);
    }
    finally {
      unlockStorage();
    }
  }

  protected long getMetaData2() throws IOException {
    lockStorage();
    try {
      return myStorage.getLong(META_DATA_OFFSET + 8);
    }
    finally {
      unlockStorage();
    }
  }

  public boolean processAllDataObject(final Processor<Data> processor, @Nullable final DataFilter filter) throws IOException {
    return traverseAllRecords(new RecordsProcessor() {
      @Override
      public boolean process(final int record) throws IOException {
        if (filter == null || filter.accept(record)) {
          return processor.process(valueOf(record));
        }
        return true;
      }
    });

  }

  public Collection<Data> getAllDataObjects(@Nullable final DataFilter filter) throws IOException {
    final List<Data> values = new ArrayList<Data>();
    processAllDataObject(new CommonProcessors.CollectProcessor<Data>(values), filter);
    return values;
  }

  public abstract static class RecordsProcessor {
    private int myKey;

    public abstract boolean process(int record) throws IOException;
    void setCurrentKey(int key) {
      myKey = key;
    }
    int getCurrentKey() {
      return myKey;
    }
  }

  public abstract boolean traverseAllRecords(RecordsProcessor p) throws IOException;

  protected abstract int enumerateImpl(final Data value, final boolean onlyCheckForExisting, boolean saveNewValue) throws IOException;

  protected boolean isKeyAtIndex(final Data value, final int idx) throws IOException {
    if (myKeyStorage == null) return false;

    // check if previous serialized state is the same as for value
    // this is much faster than myDataDescriptor.isEqualTo(valueOf(idx), value) for identical objects
    // TODO: key storage lock
    final int addr = indexToAddr(idx);

    if (myKeyStorage.checkBytesAreTheSame(addr, value, myDataDescriptor)) return true;
    if (myAssumeDifferentSerializedBytesMeansObjectsInequality) return false;
    return myDataDescriptor.isEqual(valueOf(idx), value);
  }

  protected int writeData(final Data value, int hashCode) {
    try {
      markDirty(true);

      final int dataOff = doWriteData(value);

      return setupValueId(hashCode, dataOff);
    }
    catch (IOException e) {
      throw new RuntimeException(e);
    }
  }

  public int getLargestId() {
    assert myKeyStorage != null;
    return myKeyStorage.getCurrentLength();
  }

  protected int doWriteData(Data value) throws IOException {
    if (myKeyStorage != null) {
      return myKeyStorage.append(value, myDataDescriptor);

    }
    return ((InlineKeyDescriptor<Data>)myDataDescriptor).toInt(value);
  }

  protected int setupValueId(int hashCode, int dataOff) {
    final byte[] buf = myRecordHandler.getRecordBuffer(this);
    myRecordHandler.setupRecord(this, hashCode, dataOff, buf);
    final int pos = myRecordHandler.recordWriteOffset(this, buf);
    myStorage.put(pos, buf, 0, buf.length);

    return pos;
  }

  public boolean iterateData(final Processor<Data> processor) throws IOException {
    if (myKeyStorage == null) {
      throw new UnsupportedOperationException("Iteration over InlineIntegerKeyDescriptors is not supported");
    }

    lockStorage(); // todo locking in key storage
    try {
      return myKeyStorage.processAll(processor, myDataDescriptor);
    }
    finally {
      unlockStorage();
    }
  }

  private File keystreamFile() {
    return new File(myFile.getPath() + ".keystream");
  }

  public Data valueOf(int idx) throws IOException {
    lockStorage();
    try {
      int addr = indexToAddr(idx);

      if (myKeyStorage == null) return ((InlineKeyDescriptor<Data>)myDataDescriptor).fromInt(addr);
      return myKeyStorage.read(addr, myDataDescriptor);
    }
    catch (IOException io) {
      markCorrupted();
      throw io;
    }
    catch (Throwable e) {
      markCorrupted();
      throw new RuntimeException(e);
    }
    finally {
      unlockStorage();
    }
  }

  int reenumerate(Data key) throws IOException {
    if (!canReEnumerate()) throw new IncorrectOperationException();
    return doEnumerate(key, false, true);
  }

  boolean canReEnumerate() {
    return false;
  }

  protected abstract int indexToAddr(int idx);

  @Override
  public synchronized void close() throws IOException {
    lockStorage();
    try {
      if (!myClosed) {
        myClosed = true;
        doClose();
      }
    }
    finally {
      unlockStorage();
    }
  }

  protected void doClose() throws IOException {
    try {
      if (myKeyStorage != null) {
        myKeyStorage.close();
      }
      flush();
    }
    finally {
      myStorage.close();
    }
  }

  public synchronized boolean isClosed() {
    return myClosed;
  }

  @Override
  public synchronized boolean isDirty() {
    return myDirty;
  }

  private synchronized void flush() throws IOException {
    lockStorage();
    try {
      if (myStorage.isDirty() || isDirty()) {
        doFlush();
      }
    }
    finally {
      unlockStorage();
    }
  }

  protected void doFlush() throws IOException {
    markDirty(false);
    myStorage.force();
  }

  @Override
  public synchronized void force() {
    lockStorage();

    try {
      if (myKeyStorage != null) {
        myKeyStorage.force();
      }
      flush();
    }
    catch (IOException e) {
      throw new RuntimeException(e);
    }
    finally {
      unlockStorage();
    }
  }

  protected final void markDirty(boolean dirty) throws IOException {
    //assert Thread.holdsLock(this) || Thread.holdsLock(ourLock); // we hold one lock or another so can access myDirty
    if (dirty && myDirty && !myDirtyStatusUpdateInProgress) return;
    lockStorage();
    try {
      if (myDirty) {
        if (!dirty) {
          myDirtyStatusUpdateInProgress = true;
          if (myMarkCleanCallback != null) myMarkCleanCallback.flush();
          if (!myCorrupted) {
            myStorage.putInt(0, myVersion.correctlyClosedMagic);
            myDirty = false;
          }
          myDirtyStatusUpdateInProgress = false;
        }
      }
      else {
        if (dirty) {
          myDirtyStatusUpdateInProgress = true;
          myStorage.putInt(0, myVersion.dirtyMagic);
          myDirtyStatusUpdateInProgress = false;
          myDirty = true;
        }
      }
    }
    finally {
      unlockStorage();
    }
  }

  protected synchronized void markCorrupted() {
    if (!myCorrupted) {
      myCorrupted = true;
      try {
        markDirty(true);
        force();
      }
      catch (IOException e) {
        // ignore...
      }
    }
  }

  private static class FlyweightKey extends CacheKey {
    public FlyweightKey() {
      super(null, null);
    }

    @Override
    public ShareableKey getStableCopy() {
      return new CacheKey(key, owner);
    }
  }
}