diff options
author | Colin Decker <cgdecker@google.com> | 2013-12-05 17:03:29 -0500 |
---|---|---|
committer | Colin Decker <cgdecker@google.com> | 2013-12-05 17:03:29 -0500 |
commit | c5a0e39b247ad86283a54d56003d11d98f03a8e2 (patch) | |
tree | 1fd5f8db32a95481b45cc80f8b4b0a0e108dae7b /jimfs/src/main | |
parent | ee1a751c13a7ef8283d7f0dc722c4a745cc68424 (diff) | |
download | jimfs-c5a0e39b247ad86283a54d56003d11d98f03a8e2.tar.gz |
Add the ability to limit disk size and to limit the maximum amount of unused space to keep cached.
Also, since I feel pretty confident that using direct ByteBuffers for storage is no better than
using heap arrays for most situations and that a disk-based approach (even with no actual caching)
makes the most sense, consolidate down to only one file content implementation for the file system:
- Delete DirectMemoryDisk
- Roll HeapMemoryDisk implementation up in to MemoryDisk and make it final rather than abstract
- Rename MemoryDisk to HeapDisk
- Roll MemoryDiskByteStore up in to ByteStore and make ByteStore final rather than abstract
Diffstat (limited to 'jimfs/src/main')
13 files changed, 712 insertions, 878 deletions
diff --git a/jimfs/src/main/java/com/google/jimfs/internal/BlockList.java b/jimfs/src/main/java/com/google/jimfs/internal/BlockList.java new file mode 100644 index 0000000..338d0ea --- /dev/null +++ b/jimfs/src/main/java/com/google/jimfs/internal/BlockList.java @@ -0,0 +1,110 @@ +/* + * Copyright 2013 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.jimfs.internal; + +import static com.google.jimfs.internal.Util.clear; +import static com.google.jimfs.internal.Util.nextPowerOf2; + +import java.util.Arrays; + +/** + * Simple list of byte array blocks. Blocks can only be added and removed at the end of the list. + * + * @author Colin Decker + */ +final class BlockList { + + private byte[][] blocks; + private int head; + + /** + * Creates a new list with a default initial capacity. + */ + public BlockList() { + this(32); + } + + /** + * Creates a new list with the given initial capacity. + */ + public BlockList(int initialCapacity) { + this.blocks = new byte[initialCapacity][]; + } + + private void expandIfNecessary(int minSize) { + if (minSize > blocks.length) { + this.blocks = Arrays.copyOf(blocks, nextPowerOf2(minSize)); + } + } + + /** + * Returns true if size is 0. + */ + public boolean isEmpty() { + return head == 0; + } + + /** + * Returns the number of blocks this list contains. + */ + public int size() { + return head; + } + + /** + * Copies the last {@code count} blocks from this list to the end of the given target list. + */ + public void copyTo(BlockList target, int count) { + int start = head - count; + int targetEnd = target.head + count; + target.expandIfNecessary(targetEnd); + + System.arraycopy(this.blocks, start, target.blocks, target.head, count); + target.head = targetEnd; + } + + /** + * Transfers the last {@code count} blocks from this list to the end of the given target list. + */ + public void transferTo(BlockList target, int count) { + copyTo(target, count); + truncate(head - count); + } + + /** + * Truncates this list to the given size. + */ + public void truncate(int size) { + clear(blocks, size, head - size); + head = size; + } + + /** + * Adds the given block to the end of this list. + */ + public void add(byte[] block) { + expandIfNecessary(head + 1); + blocks[head++] = block; + } + + /** + * Gets the block at the given index in this list. + */ + public byte[] get(int index) { + return blocks[index]; + } +} diff --git a/jimfs/src/main/java/com/google/jimfs/internal/ByteStore.java b/jimfs/src/main/java/com/google/jimfs/internal/ByteStore.java index 07dcb20..ab114ba 100644 --- a/jimfs/src/main/java/com/google/jimfs/internal/ByteStore.java +++ b/jimfs/src/main/java/com/google/jimfs/internal/ByteStore.java @@ -16,8 +16,11 @@ package com.google.jimfs.internal; +import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; +import com.google.common.primitives.UnsignedBytes; + import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; @@ -32,24 +35,40 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; * * @author Colin Decker */ -abstract class ByteStore implements FileContent { +final class ByteStore implements FileContent { private final ReadWriteLock lock = new ReentrantReadWriteLock(); + private final HeapDisk disk; + private final BlockList blocks; + private long size; + + public ByteStore(HeapDisk disk) { + this(disk, new BlockList(32), 0); + } + + private ByteStore(HeapDisk disk, BlockList blocks, long size) { + this.disk = checkNotNull(disk); + this.blocks = checkNotNull(blocks); + + checkArgument(size >= 0); + this.size = size; + } + private int openCount = 0; private boolean deleted = false; /** * Returns the read lock for this store. */ - protected final Lock readLock() { + public final Lock readLock() { return lock.readLock(); } /** * Returns the write lock for this store. */ - protected final Lock writeLock() { + public final Lock writeLock() { return lock.writeLock(); } @@ -57,27 +76,41 @@ abstract class ByteStore implements FileContent { * Gets the current size of this store in bytes. Does not do locking, so should only be called * when holding a lock. */ - public abstract long currentSize(); + public long currentSize() { + return size; + } /** * Creates a copy of this byte store. + * + * @throws IOException if the disk cannot allocate enough new blocks to create a copy */ - protected abstract ByteStore createCopy(); + protected ByteStore createCopy() throws IOException { + BlockList copyBlocks = new BlockList(Math.max(blocks.size() * 2, 32)); + disk.allocate(copyBlocks, blocks.size()); + + for (int i = 0; i < blocks.size(); i++) { + byte[] block = blocks.get(i); + byte[] copy = copyBlocks.get(i); + System.arraycopy(block, 0, copy, 0, block.length); + } + return new ByteStore(disk, copyBlocks, size); + } // need to lock in these methods since they're defined by an interface @Override - public final long size() { + public long size() { readLock().lock(); try { - return currentSize(); + return size; } finally { readLock().unlock(); } } @Override - public final ByteStore copy() { + public ByteStore copy() throws IOException { readLock().lock(); try { return createCopy(); @@ -91,7 +124,7 @@ abstract class ByteStore implements FileContent { /** * Called when a stream or channel to this store is opened. */ - public final synchronized void opened() { + public synchronized void opened() { openCount++; } @@ -99,7 +132,7 @@ abstract class ByteStore implements FileContent { * Called when a stream or channel to this store is closed. If there are no more streams or * channels open to the store and it has been deleted, its contents may be deleted. */ - public final synchronized void closed() { + public synchronized void closed() { if (--openCount == 0 && deleted) { deleteContents(); } @@ -119,7 +152,7 @@ abstract class ByteStore implements FileContent { * contents are deleted if necessary. */ @Override - public final synchronized void deleted() { + public synchronized void deleted() { deleted = true; if (openCount == 0) { deleteContents(); @@ -130,7 +163,10 @@ abstract class ByteStore implements FileContent { * Deletes the contents of this store. Called when the file that contains this store has been * deleted and all open streams and channels to the file have been closed. */ - protected abstract void deleteContents(); + protected void deleteContents() { + disk.free(blocks); + size = 0; + } /** * Truncates this store to the given {@code size}. If the given size is less than the current size @@ -139,38 +175,167 @@ abstract class ByteStore implements FileContent { * does nothing. Returns {@code true} if this store was modified by the call (its size changed) * and {@code false} otherwise. */ - public abstract boolean truncate(long size); + public boolean truncate(long size) { + if (size >= this.size) { + return false; + } + + long lastPosition = size - 1; + this.size = size; + + int newBlockCount = blockIndex(lastPosition) + 1; + int blocksToRemove = blocks.size() - newBlockCount; + if (blocksToRemove > 0) { + disk.free(blocks, blocksToRemove); + } + + return true; + } + + /** + * Prepares for a write of len bytes starting at position pos. + */ + private void prepareForWrite(long pos, long len) throws IOException { + long end = pos + len; + + // allocate any additional blocks needed + int lastBlockIndex = blocks.size() - 1; + int endBlockIndex = blockIndex(end - 1); + + if (endBlockIndex > lastBlockIndex) { + int additionalBlocksNeeded = endBlockIndex - lastBlockIndex; + disk.allocate(blocks, additionalBlocksNeeded); + } + + // zero bytes between current size and pos + if (pos > size) { + long remaining = pos - size; + + int blockIndex = blockIndex(size); + byte[] block = blocks.get(blockIndex); + int off = offsetInBlock(size); + + remaining -= zero(block, off, length(off, remaining)); + + while (remaining > 0) { + block = blocks.get(++blockIndex); + + remaining -= zero(block, 0, length(remaining)); + } + + size = pos; + } + } /** * Writes the given byte to this store at position {@code pos}. {@code pos} may be greater than * the current size of this store, in which case this store is resized and all bytes between the * current size and {@code pos} are set to 0. Returns the number of bytes written. + * + * @throws IOException if the store needs more blocks but the disk is full */ - public abstract int write(long pos, byte b); + public int write(long pos, byte b) throws IOException { + prepareForWrite(pos, 1); + + byte[] block = blocks.get(blockIndex(pos)); + int off = offsetInBlock(pos); + block[off] = b; + + if (pos >= size) { + size = pos + 1; + } + + return 1; + } /** * Writes {@code len} bytes starting at offset {@code off} in the given byte array to this store * starting at position {@code pos}. {@code pos} may be greater than the current size of this * store, in which case this store is resized and all bytes between the current size and {@code * pos} are set to 0. Returns the number of bytes written. + * + * @throws IOException if the store needs more blocks but the disk is full */ - public abstract int write(long pos, byte[] b, int off, int len); + public int write(long pos, byte[] b, int off, int len) throws IOException { + prepareForWrite(pos, len); + + if (len == 0) { + return 0; + } + + int remaining = len; + + int blockIndex = blockIndex(pos); + byte[] block = blockForWrite(blockIndex); + int offInBlock = offsetInBlock(pos); + + int written = put(block, offInBlock, b, off, length(offInBlock, remaining)); + remaining -= written; + off += written; + + while (remaining > 0) { + block = blocks.get(++blockIndex); + + written = put(block, 0, b, off, length(remaining)); + remaining -= written; + off += written; + } + + long newPos = pos + len; + if (newPos > size) { + size = newPos; + } + + return len; + } /** * Writes all available bytes from buffer {@code buf} to this store starting at position {@code * pos}. {@code pos} may be greater than the current size of this store, in which case this store * is resized and all bytes between the current size and {@code pos} are set to 0. Returns the * number of bytes written. + * + * @throws IOException if the store needs more blocks but the disk is full */ - public abstract int write(long pos, ByteBuffer buf); + public int write(long pos, ByteBuffer buf) throws IOException { + int len = buf.remaining(); + + prepareForWrite(pos, len); + + if (len == 0) { + return 0; + } + + int bytesToWrite = buf.remaining(); + + int blockIndex = blockIndex(pos); + byte[] block = blockForWrite(blockIndex); + int off = offsetInBlock(pos); + + put(block, off, buf); + + while (buf.hasRemaining()) { + block = blocks.get(++blockIndex); + + put(block, 0, buf); + } + + if (pos + bytesToWrite > size) { + size = pos + bytesToWrite; + } + + return bytesToWrite; + } /** * Writes all available bytes from each buffer in {@code bufs}, in order, to this store starting * at position {@code pos}. {@code pos} may be greater than the current size of this store, in * which case this store is resized and all bytes between the current size and {@code pos} are set * to 0. Returns the number of bytes written. + * + * @throws IOException if the store needs more blocks but the disk is full */ - public long write(long pos, Iterable<ByteBuffer> bufs) { + public long write(long pos, Iterable<ByteBuffer> bufs) throws IOException { long start = pos; for (ByteBuffer buf : bufs) { pos += write(pos, buf); @@ -182,29 +347,135 @@ abstract class ByteStore implements FileContent { * Transfers up to {@code count} bytes from the given channel to this store starting at position * {@code pos}. Returns the number of bytes transferred. If {@code pos} is greater than the * current size of this store, the store is truncated up to size {@code pos} before writing. + * + * @throws IOException if the store needs more blocks but the disk is full or if reading from src + * throws an exception */ - public abstract long transferFrom( - ReadableByteChannel src, long pos, long count) throws IOException; + public long transferFrom( + ReadableByteChannel src, long pos, long count) throws IOException { + prepareForWrite(pos, 0); + + if (count == 0) { + return 0; + } + + long remaining = count; + + int blockIndex = blockIndex(pos); + byte[] block = blockForWrite(blockIndex); + int off = offsetInBlock(pos); + + ByteBuffer buf = ByteBuffer.wrap(block, off, length(off, remaining)); + + int read = 0; + while (buf.hasRemaining()) { + read = src.read(buf); + if (read == -1) { + break; + } + + remaining -= read; + } + + if (read != -1) { + outer: while (remaining > 0) { + block = blockForWrite(++blockIndex); + + buf = ByteBuffer.wrap(block, 0, length(remaining)); + while (buf.hasRemaining()) { + read = src.read(buf); + if (read == -1) { + break outer; + } + + remaining -= read; + } + } + } + + long written = count - remaining; + long newPos = pos + written; + if (newPos > size) { + size = newPos; + } + + return written; + } /** * Reads the byte at position {@code pos} in this store as an unsigned integer in the range 0-255. * If {@code pos} is greater than or equal to the size of this store, returns -1 instead. */ - public abstract int read(long pos); + public int read(long pos) { + if (pos >= size) { + return -1; + } + + byte[] block = blocks.get(blockIndex(pos)); + int off = offsetInBlock(pos); + return UnsignedBytes.toInt(block[off]); + } /** * Reads up to {@code len} bytes starting at position {@code pos} in this store to the given byte * array starting at offset {@code off}. Returns the number of bytes actually read or -1 if {@code * pos} is greater than or equal to the size of this store. */ - public abstract int read(long pos, byte[] b, int off, int len); + public int read(long pos, byte[] b, int off, int len) { + // since max is len (an int), result is guaranteed to be an int + int bytesToRead = (int) bytesToRead(pos, len); + + if (bytesToRead > 0) { + int remaining = bytesToRead; + + int blockIndex = blockIndex(pos); + byte[] block = blocks.get(blockIndex); + int offsetInBlock = offsetInBlock(pos); + + int read = get(block, offsetInBlock, b, off, length(offsetInBlock, remaining)); + remaining -= read; + off += read; + + while (remaining > 0) { + int index = ++blockIndex; + block = blocks.get(index); + + read = get(block, 0, b, off, length(remaining)); + remaining -= read; + off += read; + } + } + + return bytesToRead; + } /** * Reads up to {@code buf.remaining()} bytes starting at position {@code pos} in this store to the * given buffer. Returns the number of bytes read or -1 if {@code pos} is greater than or equal to * the size of this store. */ - public abstract int read(long pos, ByteBuffer buf); + public int read(long pos, ByteBuffer buf) { + // since max is buf.remaining() (an int), result is guaranteed to be an int + int bytesToRead = (int) bytesToRead(pos, buf.remaining()); + + if (bytesToRead > 0) { + int remaining = bytesToRead; + + int blockIndex = blockIndex(pos); + byte[] block = blocks.get(blockIndex); + int off = offsetInBlock(pos); + + remaining -= get(block, off, buf, length(off, remaining)); + + while (remaining > 0) { + int index = ++blockIndex; + block = blocks.get(index); + remaining -= get(block, 0, buf, length(remaining)); + } + } + + return bytesToRead; + } /** * Reads up to the total {@code remaining()} number of bytes in each of {@code bufs} starting at @@ -236,6 +507,118 @@ abstract class ByteStore implements FileContent { * equal to the current size. This for consistency with {@link FileChannel#transferTo}, which * this method is primarily intended as an implementation of. */ - public abstract long transferTo( - long pos, long count, WritableByteChannel dest) throws IOException; + public long transferTo( + long pos, long count, WritableByteChannel dest) throws IOException { + long bytesToRead = bytesToRead(pos, count); + + if (bytesToRead > 0) { + long remaining = bytesToRead; + + int blockIndex = blockIndex(pos); + byte[] block = blocks.get(blockIndex); + int off = offsetInBlock(pos); + + ByteBuffer buf = ByteBuffer.wrap(block, off, length(off, remaining)); + while (buf.hasRemaining()) { + remaining -= dest.write(buf); + } + buf.clear(); + + while (remaining > 0) { + int index = ++blockIndex; + block = blocks.get(index); + + buf = ByteBuffer.wrap(block, 0, length(remaining)); + while (buf.hasRemaining()) { + remaining -= dest.write(buf); + } + buf.clear(); + } + } + + return Math.max(bytesToRead, 0); // don't return -1 for this method + } + + /** + * Gets the block at the given index, expanding to create the block if necessary. + */ + private byte[] blockForWrite(int index) throws IOException { + int blockCount = blocks.size(); + if (index >= blockCount) { + int additionalBlocksNeeded = index - blockCount + 1; + disk.allocate(blocks, additionalBlocksNeeded); + } + + return blocks.get(index); + } + + private int blockIndex(long position) { + return (int) (position / disk.blockSize()); + } + + private int offsetInBlock(long position) { + return (int) (position % disk.blockSize()); + } + + private int length(long max) { + return (int) Math.min(disk.blockSize(), max); + } + + private int length(int off, long max) { + return (int) Math.min(disk.blockSize() - off, max); + } + + /** + * Returns the number of bytes that can be read starting at position {@code pos} (up to a maximum + * of {@code max}) or -1 if {@code pos} is greater than or equal to the current size. + */ + private long bytesToRead(long pos, long max) { + long available = size - pos; + if (available <= 0) { + return -1; + } + return Math.min(available, max); + } + + /** + * Zeroes len bytes in the given block starting at the given offset. Returns len. + */ + private static int zero(byte[] block, int offset, int len) { + Util.zero(block, offset, len); + return len; + } + + /** + * Puts the given slice of the given array at the given offset in the given block. + */ + private static int put(byte[] block, int offset, byte[] b, int off, int len) { + System.arraycopy(b, off, block, offset, len); + return len; + } + + /** + * Puts the contents of the given byte buffer at the given offset in the given block. + */ + private static int put(byte[] block, int offset, ByteBuffer buf) { + int len = Math.min(block.length - offset, buf.remaining()); + buf.get(block, offset, len); + return len; + } + + /** + * Reads len bytes starting at the given offset in the given block into the given slice of the + * given byte array. + */ + private static int get(byte[] block, int offset, byte[] b, int off, int len) { + System.arraycopy(block, offset, b, off, len); + return len; + } + + /** + * Reads len bytes starting at the given offset in the given block into the given byte buffer. + */ + private static int get(byte[] block, int offset, ByteBuffer buf, int len) { + buf.put(block, offset, len); + return len; + } } diff --git a/jimfs/src/main/java/com/google/jimfs/internal/DirectMemoryDisk.java b/jimfs/src/main/java/com/google/jimfs/internal/DirectMemoryDisk.java deleted file mode 100644 index 84c4491..0000000 --- a/jimfs/src/main/java/com/google/jimfs/internal/DirectMemoryDisk.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright 2013 Google Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.jimfs.internal; - -import static com.google.jimfs.internal.Util.nextPowerOf2; - -import java.nio.ByteBuffer; -import java.util.Arrays; - -/** - * {@link MemoryDisk} using {@linkplain ByteBuffer#allocateDirect(int) direct byte buffers} for - * blocks. - * - * @author Colin Decker - */ -final class DirectMemoryDisk extends MemoryDisk { - - private ByteBuffer[] blocks = new ByteBuffer[4096]; - - DirectMemoryDisk() { - this(DEFAULT_BLOCK_SIZE); - } - - /** - * Creates a disk with the given block size and max cache size. - */ - public DirectMemoryDisk(int blockSize) { - super(blockSize); - } - - @Override - protected int allocateMoreBlocks(int count) { - int newBlockCount = blockCount() + count; - if (newBlockCount > blocks.length) { - // using nextPowerOf2 instead of multiplying blocks.length * 2 until it's >= newBlockCount - blocks = Arrays.copyOf(blocks, nextPowerOf2(newBlockCount)); - } - - for (int i = blockCount(); i < newBlockCount; i++) { - blocks[i] = ByteBuffer.allocateDirect(blockSize); - free.add(i); - } - - return count; - } - - @Override - public int zero(int block, int offset, int len) { - ByteBuffer buffer = blockForRead(block); - int limit = offset + len; - for (int i = offset; i < limit; i++) { - buffer.put(i, (byte) 0); - } - return len; - } - - @Override - public void copy(int block, int copy) { - ByteBuffer blockBuf = blockForRead(block); - ByteBuffer copyBuf = blocks[copy]; - copyBuf.put(blockBuf); - copyBuf.clear(); - } - - @Override - public void put(int block, int offset, byte b) { - blocks[block].put(offset, b); - } - - @Override - public int put(int block, int offset, byte[] b, int off, int len) { - ByteBuffer blockBuf = blocks[block]; - blockBuf.position(offset); - blockBuf.put(b, off, len); - blockBuf.clear(); - return len; - } - - @Override - public int put(int block, int offset, ByteBuffer buf) { - ByteBuffer blockBuf = blocks[block]; - blockBuf.position(offset); - int len = Math.min(blockSize - offset, buf.remaining()); - blockBuf.put(buf); - blockBuf.clear(); - return len; - } - - @Override - public byte get(int block, int offset) { - return blocks[block].get(offset); - } - - @Override - public int get(int block, int offset, byte[] b, int off, int len) { - ByteBuffer blockBuf = blockForRead(block); - blockBuf.position(offset); - blockBuf.get(b, off, len); - return len; - } - - @Override - public int get(int block, int offset, ByteBuffer buf, int len) { - ByteBuffer blockBuf = blockForRead(block); - blockBuf.position(offset); - blockBuf.limit(offset + len); - buf.put(blockBuf); - return len; - } - - @Override - public ByteBuffer asByteBuffer(int block, int offset, int len) { - ByteBuffer result = blockForRead(block); - result.position(offset); - result.limit(offset + len); - return result; - } - - /** - * Must duplicate the block because we can't have multiple reading threads manipulating the state - * of the same buffer at the same time. Why oh why couldn't some kind of public buffer type - * without positional state have been created? ...and a better buffer API than ByteBuffer created - * on top of it, while I'm wishing. - */ - private ByteBuffer blockForRead(int block) { - return blocks[block].duplicate(); - } - -} diff --git a/jimfs/src/main/java/com/google/jimfs/internal/FileContent.java b/jimfs/src/main/java/com/google/jimfs/internal/FileContent.java index 70347f5..6fb013e 100644 --- a/jimfs/src/main/java/com/google/jimfs/internal/FileContent.java +++ b/jimfs/src/main/java/com/google/jimfs/internal/FileContent.java @@ -16,6 +16,7 @@ package com.google.jimfs.internal; +import java.io.IOException; import java.nio.file.CopyOption; import java.nio.file.Files; import java.nio.file.Path; @@ -33,7 +34,7 @@ interface FileContent { * the original; for example, a copy of a directory is an empty directory regardless of what the * original directory contains. */ - FileContent copy(); + FileContent copy() throws IOException; /** * Returns the size, in bytes, of this content. This may be 0 when there is no logical size we diff --git a/jimfs/src/main/java/com/google/jimfs/internal/FileFactory.java b/jimfs/src/main/java/com/google/jimfs/internal/FileFactory.java index 2a41503..a62a14e 100644 --- a/jimfs/src/main/java/com/google/jimfs/internal/FileFactory.java +++ b/jimfs/src/main/java/com/google/jimfs/internal/FileFactory.java @@ -21,6 +21,7 @@ import static com.google.common.base.Preconditions.checkNotNull; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Supplier; +import java.io.IOException; import java.util.concurrent.atomic.AtomicInteger; /** @@ -32,12 +33,12 @@ final class FileFactory { private final AtomicInteger idGenerator = new AtomicInteger(); - private final MemoryDisk disk; + private final HeapDisk disk; /** * Creates a new file factory using the given disk for regular files. */ - public FileFactory(MemoryDisk disk) { + public FileFactory(HeapDisk disk) { this.disk = checkNotNull(disk); } @@ -71,7 +72,7 @@ final class FileFactory { /** * Creates and returns a copy of the given file. */ - public File copy(File file) { + public File copy(File file) throws IOException { return new File(nextFileId(), file.content().copy()); } diff --git a/jimfs/src/main/java/com/google/jimfs/internal/HeapDisk.java b/jimfs/src/main/java/com/google/jimfs/internal/HeapDisk.java new file mode 100644 index 0000000..1a97ed5 --- /dev/null +++ b/jimfs/src/main/java/com/google/jimfs/internal/HeapDisk.java @@ -0,0 +1,154 @@ +/* + * Copyright 2013 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.jimfs.internal; + +import static com.google.common.base.Preconditions.checkArgument; + +import com.google.common.annotations.VisibleForTesting; + +import java.io.IOException; + +/** + * A resizable pseudo-disk acting as a shared space for storing file data. A disk allocates fixed + * size blocks of bytes to files as needed and may cache blocks that have been freed for reuse. A + * memory disk has a fixed maximum number of blocks it will allocate at a time (which sets the + * total "size" of the disk) and a maximum number of unused blocks it will cache for reuse at a + * time (which sets the minimum amount of space the disk will use once + * + * @author Colin Decker + */ +final class HeapDisk { + + /** 8 KB blocks. */ + public static final int DEFAULT_BLOCK_SIZE = 8192; + + /** 16 GB of space with 8 KB blocks. */ + public static final int DEFAULT_MAX_BLOCK_COUNT = + (int) ((16L * 1024 * 1024 * 1024) / DEFAULT_BLOCK_SIZE); + + /** Fixed size of each block for this disk. */ + private final int blockSize; + + /** Maximum total number of blocks that the disk may contain at any time. */ + private final int maxBlockCount; + + /** Maximum total number of unused blocks that may be cached for reuse at any time. */ + private final int maxCachedBlockCount; + + /** Cache of free blocks to be allocated to files. */ + @VisibleForTesting final BlockList blockCache; + + /** The current total number of blocks that are currently allocated to files. */ + private int allocatedBlockCount; + + /** + * Creates a new heap disk with 8 KB blocks that can store up to 16 GB of data and caches all + * blocks that are freed. + */ + public HeapDisk() { + this(DEFAULT_BLOCK_SIZE, DEFAULT_MAX_BLOCK_COUNT, DEFAULT_MAX_BLOCK_COUNT); + } + + /** + * Creates a new disk with the given {@code blockSize}, {@code maxBlockCount} and + * {@code maxCachedBlockCount}. + */ + public HeapDisk(int blockSize, int maxBlockCount, int maxCachedBlockCount) { + checkArgument(blockSize > 0, "blockSize (%s) must be positive", blockSize); + checkArgument(blockSize % 2 == 0, "blockSize (%s) must be a multiple of 2", blockSize); + checkArgument(maxBlockCount > 0, "maxBlockCount (%s) must be positive", maxBlockCount); + checkArgument(maxCachedBlockCount >= 0, + "maxCachedBlockCount must be non-negative", maxCachedBlockCount); + this.blockSize = blockSize; + this.maxBlockCount = maxBlockCount; + this.maxCachedBlockCount = maxCachedBlockCount; + this.blockCache = new BlockList(Math.min(maxCachedBlockCount, 8192)); + } + + /** + * Creates a new, empty byte store. + */ + public ByteStore createByteStore() { + return new ByteStore(this); + } + + /** + * Returns the size of blocks created by this disk. + */ + public int blockSize() { + return blockSize; + } + + /** + * Returns the total size of this disk. This is the maximum size of the disk and does not reflect + * the amount of data currently allocated or cached. + */ + public synchronized long getTotalSpace() { + return maxBlockCount * (long) blockSize; + } + + /** + * Returns the current number of unallocated bytes on this disk. This is the maximum number of + * additional bytes that could be allocated and does not reflect the number of bytes currently + * actually cached in the disk. + */ + public synchronized long getUnallocatedSpace() { + return (maxBlockCount - allocatedBlockCount) * (long) blockSize; + } + + /** + * Allocates the given number of blocks and adds their identifiers to the given list. + */ + public synchronized void allocate(BlockList blocks, int count) throws IOException { + int newAllocatedBlockCount = allocatedBlockCount + count; + if (newAllocatedBlockCount > maxBlockCount) { + throw new IOException("out of disk space"); + } + + int newBlocksNeeded = Math.max(count - blockCache.size(), 0); + + for (int i = 0; i < newBlocksNeeded; i++) { + blocks.add(new byte[blockSize]); + } + + if (newBlocksNeeded != count) { + blockCache.transferTo(blocks, count - newBlocksNeeded); + } + + allocatedBlockCount = newAllocatedBlockCount; + } + + /** + * Frees all blocks in the given list. + */ + public void free(BlockList blocks) { + free(blocks, blocks.size()); + } + + /** + * Frees the last count blocks from the given list. + */ + public synchronized void free(BlockList blocks, int count) { + int remainingCacheSpace = maxCachedBlockCount - blocks.size(); + if (remainingCacheSpace != 0) { + blocks.copyTo(blockCache, Math.min(count, remainingCacheSpace)); + } + blocks.truncate(blocks.size() - count); + + allocatedBlockCount -= count; + } +} diff --git a/jimfs/src/main/java/com/google/jimfs/internal/HeapMemoryDisk.java b/jimfs/src/main/java/com/google/jimfs/internal/HeapMemoryDisk.java deleted file mode 100644 index 802a4ea..0000000 --- a/jimfs/src/main/java/com/google/jimfs/internal/HeapMemoryDisk.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright 2013 Google Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.jimfs.internal; - -import static com.google.jimfs.internal.Util.nextPowerOf2; - -import java.nio.ByteBuffer; -import java.util.Arrays; - -/** - * {@link MemoryDisk} using byte arrays for blocks. - * - * @author Colin Decker - */ -final class HeapMemoryDisk extends MemoryDisk { - - private byte[][] blocks = new byte[4096][]; - - HeapMemoryDisk() { - this(DEFAULT_BLOCK_SIZE); - } - - /** - * Creates a disk with the given block size. - */ - public HeapMemoryDisk(int blockSize) { - super(blockSize); - } - - @Override - protected int allocateMoreBlocks(int count) { - int newBlockCount = blockCount() + count; - if (newBlockCount > blocks.length) { - // using nextPowerOf2 instead of multiplying blocks.length * 2 until it's >= newBlockCount - blocks = Arrays.copyOf(blocks, nextPowerOf2(newBlockCount)); - } - - for (int i = blockCount(); i < newBlockCount; i++) { - blocks[i] = new byte[blockSize]; - free.add(i); - } - - return count; - } - - @Override - public int zero(int block, int offset, int len) { - Util.zero(blocks[block], offset, len); - return len; - } - - @Override - public void copy(int block, int copy) { - System.arraycopy(blocks[block], 0, blocks[copy], 0, blockSize); - } - - @Override - public void put(int block, int offset, byte b) { - blocks[block][offset] = b; - } - - @Override - public int put(int block, int offset, byte[] b, int off, int len) { - System.arraycopy(b, off, blocks[block], offset, len); - return len; - } - - @Override - public int put(int block, int offset, ByteBuffer buf) { - int len = Math.min(blockSize - offset, buf.remaining()); - buf.get(blocks[block], offset, len); - return len; - } - - @Override - public byte get(int block, int offset) { - return blocks[block][offset]; - } - - @Override - public int get(int block, int offset, byte[] b, int off, int len) { - System.arraycopy(blocks[block], offset, b, off, len); - return len; - } - - @Override - public int get(int block, int offset, ByteBuffer buf, int len) { - buf.put(blocks[block], offset, len); - return len; - } - - @Override - public ByteBuffer asByteBuffer(int block, int offset, int len) { - return ByteBuffer.wrap(blocks[block], offset, len); - } -} diff --git a/jimfs/src/main/java/com/google/jimfs/internal/JimfsFileStore.java b/jimfs/src/main/java/com/google/jimfs/internal/JimfsFileStore.java index e36c67f..8ba8800 100644 --- a/jimfs/src/main/java/com/google/jimfs/internal/JimfsFileStore.java +++ b/jimfs/src/main/java/com/google/jimfs/internal/JimfsFileStore.java @@ -53,14 +53,14 @@ import javax.annotation.Nullable; final class JimfsFileStore extends FileStore { private final FileTree tree; - private final MemoryDisk disk; + private final HeapDisk disk; private final AttributeService attributes; private final FileFactory factory; private final Lock readLock; private final Lock writeLock; - public JimfsFileStore(FileTree tree, FileFactory factory, MemoryDisk disk, + public JimfsFileStore(FileTree tree, FileFactory factory, HeapDisk disk, AttributeService attributes) { this.tree = checkNotNull(tree); this.factory = checkNotNull(factory); @@ -143,7 +143,7 @@ final class JimfsFileStore extends FileStore { * Creates a copy of the given file, copying its attributes as well if copy attributes is true. * Returns the copy. */ - File copy(File file, boolean copyAttributes) { + File copy(File file, boolean copyAttributes) throws IOException { File copy = factory.copy(file); setInitialAttributes(copy); if (copyAttributes) { diff --git a/jimfs/src/main/java/com/google/jimfs/internal/JimfsFileSystems.java b/jimfs/src/main/java/com/google/jimfs/internal/JimfsFileSystems.java index ba887f4..21e751f 100644 --- a/jimfs/src/main/java/com/google/jimfs/internal/JimfsFileSystems.java +++ b/jimfs/src/main/java/com/google/jimfs/internal/JimfsFileSystems.java @@ -56,7 +56,9 @@ final class JimfsFileSystems { private static JimfsFileStore createFileStore( Configuration config, PathService pathService) { AttributeService attributeService = new AttributeService(config); - MemoryDisk disk = new HeapMemoryDisk(); + + // TODO(cgdecker): Make disk values configurable + HeapDisk disk = new HeapDisk(); FileFactory fileFactory = new FileFactory(disk); Map<Name, File> roots = new HashMap<>(); diff --git a/jimfs/src/main/java/com/google/jimfs/internal/MemoryDisk.java b/jimfs/src/main/java/com/google/jimfs/internal/MemoryDisk.java deleted file mode 100644 index a882e83..0000000 --- a/jimfs/src/main/java/com/google/jimfs/internal/MemoryDisk.java +++ /dev/null @@ -1,178 +0,0 @@ -/* - * Copyright 2013 Google Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.jimfs.internal; - -import static com.google.common.base.Preconditions.checkArgument; - -import java.nio.ByteBuffer; - -/** - * A resizable pseudo-disk acting as a shared space for storing file data. A disk allocates fixed - * size blocks of bytes to files as needed and caches blocks that have been freed for reuse. Each - * block is represented by an integer which is used to locate the block for read and write - * operations implemented by the disk. - * - * <p>Currently, once a memory disk has allocated a block it will cache that block indefinitely - * once freed. This means that the total size of the disk is always the maximum number of bytes - * that have been allocated to files at one time so far. - * - * @author Colin Decker - */ -abstract class MemoryDisk { - - /** - * 8K blocks. - */ - protected static final int DEFAULT_BLOCK_SIZE = 8192; - - /** - * Fixed size of each block for this disk. - */ - protected final int blockSize; - - /** - * The current total number of blocks this disk contains, including both free blocks and blocks - * that are allocated to files. - */ - private int blockCount; - - /** - * Queue of free blocks to be allocated to files. - */ - protected final IntList free = new IntList(1024); - - protected MemoryDisk(int blockSize) { - checkArgument(blockSize > 0, "blockSize (%s) must be positive", blockSize); - checkArgument(blockSize % 2 == 0, "blockSize (%s) must be a multiple of 2", blockSize); - this.blockSize = blockSize; - } - - /** - * Creates a new, empty byte store. - */ - public final ByteStore createByteStore() { - return new MemoryDiskByteStore(this); - } - - /** - * Allocates at least {@code minBlocks} more blocks if possible. The {@code free} list - * should have blocks in it when this returns if an exception is not thrown. Returns the number - * of new blocks that were allocated. - */ - protected abstract int allocateMoreBlocks(int count); - - /** - * Returns the size of blocks created by this disk. - */ - public final int blockSize() { - return blockSize; - } - - /** - * Returns the current total number of blocks this disk contains. - */ - protected final int blockCount() { - return blockCount; - } - - /** - * Returns the current total size of this disk. - */ - public final synchronized long getTotalSpace() { - return blockCount * blockSize; - } - - /** - * Returns the current number of unallocated bytes on this disk. - */ - public final synchronized long getUnallocatedSpace() { - return free.size() * blockSize; - } - - /** - * Allocates the given number of blocks and adds their identifiers to the given list. - */ - public final synchronized void allocate(IntList blocks, int count) { - int additionalBlocksNeeded = count - free.size(); - if (additionalBlocksNeeded > 0) { - blockCount += allocateMoreBlocks(additionalBlocksNeeded); - } - - free.transferTo(blocks, count); - } - - /** - * Frees all blocks in the given list. - */ - public final void free(IntList blocks) { - free(blocks, blocks.size()); - } - - /** - * Frees the last count blocks from the given list. - */ - public final synchronized void free(IntList blocks, int count) { - blocks.transferTo(free, count); - } - - /** - * Zeroes len bytes in the given block starting at the given offset. Returns len. - */ - public abstract int zero(int block, int offset, int len); - - /** - * Copies the given block and returns the copy. - */ - public abstract void copy(int block, int copy); - - /** - * Puts the given byte at the given offset in the given block. - */ - public abstract void put(int block, int offset, byte b); - - /** - * Puts the given slice of the given array at the given offset in the given block. - */ - public abstract int put(int block, int offset, byte[] b, int off, int len); - - /** - * Puts the contents of the given byte buffer at the given offset in the given block. - */ - public abstract int put(int block, int offset, ByteBuffer buf); - - /** - * Returns the byte at the given offset in the given block. - */ - public abstract byte get(int block, int offset); - - /** - * Reads len bytes starting at the given offset in the given block into the given slice of the - * given byte array. - */ - public abstract int get(int block, int offset, byte[] b, int off, int len); - - /** - * Reads len bytes starting at the given offset in the given block into the given byte buffer. - */ - public abstract int get(int block, int offset, ByteBuffer buf, int len); - - /** - * Returns a ByteBuffer view of the slice of the given block starting at the given offset and - * having the given length. - */ - public abstract ByteBuffer asByteBuffer(int block, int offset, int len); -} diff --git a/jimfs/src/main/java/com/google/jimfs/internal/MemoryDiskByteStore.java b/jimfs/src/main/java/com/google/jimfs/internal/MemoryDiskByteStore.java deleted file mode 100644 index f79fb10..0000000 --- a/jimfs/src/main/java/com/google/jimfs/internal/MemoryDiskByteStore.java +++ /dev/null @@ -1,396 +0,0 @@ -/* - * Copyright 2013 Google Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.jimfs.internal; - -import static com.google.common.base.Preconditions.checkArgument; -import static com.google.common.base.Preconditions.checkNotNull; - -import com.google.common.primitives.UnsignedBytes; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.channels.ReadableByteChannel; -import java.nio.channels.WritableByteChannel; - -/** - * Byte store backed by a {@link MemoryDisk}. - * - * @author Colin Decker - */ -final class MemoryDiskByteStore extends ByteStore { - - private final MemoryDisk disk; - private final IntList blocks; - private long size; - - public MemoryDiskByteStore(MemoryDisk disk) { - this(disk, new IntList(32), 0); - } - - private MemoryDiskByteStore(MemoryDisk disk, IntList blocks, long size) { - this.disk = checkNotNull(disk); - this.blocks = checkNotNull(blocks); - - checkArgument(size >= 0); - this.size = size; - } - - @Override - public long currentSize() { - return size; - } - - @Override - protected ByteStore createCopy() { - IntList copyBlocks = new IntList(Math.max(blocks.size() * 2, 32)); - disk.allocate(copyBlocks, blocks.size()); - - for (int i = 0; i < blocks.size(); i++) { - int block = blocks.get(i); - int copy = copyBlocks.get(i); - disk.copy(block, copy); - } - return new MemoryDiskByteStore(disk, copyBlocks, size); - } - - @Override - protected final void deleteContents() { - disk.free(blocks); - size = 0; - } - - @Override - public boolean truncate(long size) { - if (size >= this.size) { - return false; - } - - long lastPosition = size - 1; - this.size = size; - - int newBlockCount = blockIndex(lastPosition) + 1; - int blocksToRemove = blocks.size() - newBlockCount; - if (blocksToRemove > 0) { - disk.free(blocks, blocksToRemove); - } - - return true; - } - - /** - * Prepares for a write of len bytes starting at position pos. - */ - private void prepareForWrite(long pos, long len) { - long end = pos + len; - - // allocate any additional blocks needed - int lastBlockIndex = blocks.size() - 1; - int endBlockIndex = blockIndex(end - 1); - - if (endBlockIndex > lastBlockIndex) { - int additionalBlocksNeeded = endBlockIndex - lastBlockIndex; - disk.allocate(blocks, additionalBlocksNeeded); - } - - // zero bytes between current size and pos - if (pos > size) { - long remaining = pos - size; - - int blockIndex = blockIndex(size); - int block = blocks.get(blockIndex); - int off = offsetInBlock(size); - - remaining -= disk.zero(block, off, length(off, remaining)); - - while (remaining > 0) { - block = blocks.get(++blockIndex); - - remaining -= disk.zero(block, 0, length(remaining)); - } - - size = pos; - } - } - - @Override - public int write(long pos, byte b) { - prepareForWrite(pos, 1); - - int block = blocks.get(blockIndex(pos)); - int off = offsetInBlock(pos); - disk.put(block, off, b); - - if (pos >= size) { - size = pos + 1; - } - - return 1; - } - - @Override - public int write(long pos, byte[] b, int off, int len) { - prepareForWrite(pos, len); - - if (len == 0) { - return 0; - } - - int remaining = len; - - int blockIndex = blockIndex(pos); - int block = blockForWrite(blockIndex); - int offInBlock = offsetInBlock(pos); - - int written = disk.put(block, offInBlock, b, off, length(offInBlock, remaining)); - remaining -= written; - off += written; - - while (remaining > 0) { - block = blocks.get(++blockIndex); - - written = disk.put(block, 0, b, off, length(remaining)); - remaining -= written; - off += written; - } - - long newPos = pos + len; - if (newPos > size) { - size = newPos; - } - - return len; - } - - @Override - public int write(long pos, ByteBuffer buf) { - int len = buf.remaining(); - - prepareForWrite(pos, len); - - if (len == 0) { - return 0; - } - - int bytesToWrite = buf.remaining(); - - int blockIndex = blockIndex(pos); - int block = blockForWrite(blockIndex); - int off = offsetInBlock(pos); - - disk.put(block, off, buf); - - while (buf.hasRemaining()) { - block = blocks.get(++blockIndex); - - disk.put(block, 0, buf); - } - - if (pos + bytesToWrite > size) { - size = pos + bytesToWrite; - } - - return bytesToWrite; - } - - @Override - public long transferFrom(ReadableByteChannel src, long pos, long count) throws IOException { - prepareForWrite(pos, 0); - - if (count == 0) { - return 0; - } - - long remaining = count; - - int blockIndex = blockIndex(pos); - int block = blockForWrite(blockIndex); - int off = offsetInBlock(pos); - - ByteBuffer buf = disk.asByteBuffer(block, off, length(off, remaining)); - - int read = 0; - while (buf.hasRemaining()) { - read = src.read(buf); - if (read == -1) { - break; - } - - remaining -= read; - } - - if (read != -1) { - outer: while (remaining > 0) { - block = blockForWrite(++blockIndex); - - buf = disk.asByteBuffer(block, 0, length(remaining)); - while (buf.hasRemaining()) { - read = src.read(buf); - if (read == -1) { - break outer; - } - - remaining -= read; - } - } - } - - long written = count - remaining; - long newPos = pos + written; - if (newPos > size) { - size = newPos; - } - - return written; - } - - @Override - public int read(long pos) { - if (pos >= size) { - return -1; - } - - int block = blocks.get(blockIndex(pos)); - int off = offsetInBlock(pos); - return UnsignedBytes.toInt(disk.get(block, off)); - } - - @Override - public int read(long pos, byte[] b, int off, int len) { - // since max is len (an int), result is guaranteed to be an int - int bytesToRead = (int) bytesToRead(pos, len); - - if (bytesToRead > 0) { - int remaining = bytesToRead; - - int blockIndex = blockIndex(pos); - int block = blocks.get(blockIndex); - int offsetInBlock = offsetInBlock(pos); - - int read = disk.get(block, offsetInBlock, b, off, length(offsetInBlock, remaining)); - remaining -= read; - off += read; - - while (remaining > 0) { - int index = ++blockIndex; - block = blocks.get(index); - - read = disk.get(block, 0, b, off, length(remaining)); - remaining -= read; - off += read; - } - } - - return bytesToRead; - } - - @Override - public int read(long pos, ByteBuffer buf) { - // since max is buf.remaining() (an int), result is guaranteed to be an int - int bytesToRead = (int) bytesToRead(pos, buf.remaining()); - - if (bytesToRead > 0) { - int remaining = bytesToRead; - - int blockIndex = blockIndex(pos); - int block = blocks.get(blockIndex); - int off = offsetInBlock(pos); - - remaining -= disk.get(block, off, buf, length(off, remaining)); - - while (remaining > 0) { - int index = ++blockIndex; - block = blocks.get(index); - remaining -= disk.get(block, 0, buf, length(remaining)); - } - } - - return bytesToRead; - } - - @Override - public long transferTo(long pos, long count, WritableByteChannel dest) throws IOException { - long bytesToRead = bytesToRead(pos, count); - - if (bytesToRead > 0) { - long remaining = bytesToRead; - - int blockIndex = blockIndex(pos); - int block = blocks.get(blockIndex); - int off = offsetInBlock(pos); - - ByteBuffer buf = disk.asByteBuffer(block, off, length(off, remaining)); - while (buf.hasRemaining()) { - remaining -= dest.write(buf); - } - buf.clear(); - - while (remaining > 0) { - int index = ++blockIndex; - block = blocks.get(index); - - buf = disk.asByteBuffer(block, 0, length(remaining)); - while (buf.hasRemaining()) { - remaining -= dest.write(buf); - } - buf.clear(); - } - } - - return Math.max(bytesToRead, 0); // don't return -1 for this method - } - - /** - * Gets the block at the given index, expanding to create the block if necessary. - */ - private int blockForWrite(int index) { - int blockCount = blocks.size(); - if (index >= blockCount) { - int additionalBlocksNeeded = index - blockCount + 1; - disk.allocate(blocks, additionalBlocksNeeded); - } - - return blocks.get(index); - } - - private int blockIndex(long position) { - return (int) (position / disk.blockSize()); - } - - private int offsetInBlock(long position) { - return (int) (position % disk.blockSize()); - } - - private int length(long max) { - return (int) Math.min(disk.blockSize(), max); - } - - private int length(int off, long max) { - return (int) Math.min(disk.blockSize() - off, max); - } - - /** - * Returns the number of bytes that can be read starting at position {@code pos} (up to a maximum - * of {@code max}) or -1 if {@code pos} is greater than or equal to the current size. - */ - private long bytesToRead(long pos, long max) { - long available = size - pos; - if (available <= 0) { - return -1; - } - return Math.min(available, max); - } -} diff --git a/jimfs/src/main/java/com/google/jimfs/internal/Util.java b/jimfs/src/main/java/com/google/jimfs/internal/Util.java index d73dacd..7b5943e 100644 --- a/jimfs/src/main/java/com/google/jimfs/internal/Util.java +++ b/jimfs/src/main/java/com/google/jimfs/internal/Util.java @@ -76,30 +76,42 @@ final class Util { return C2 * Integer.rotateLeft(hashCode * C1, 15); } - private static final int ZERO_ARRAY_LEN = 8192; - private static final byte[] ZERO_ARRAY = new byte[ZERO_ARRAY_LEN]; + private static final int ARRAY_LEN = 8192; + private static final byte[] ZERO_ARRAY = new byte[ARRAY_LEN]; + private static final byte[][] NULL_ARRAY = new byte[ARRAY_LEN][]; /** - * Zeroes all bytes of the given array. + * Zeroes all bytes between off (inclusive) and off + len (exclusive) in the given array. */ - static void zero(byte[] bytes) { - zero(bytes, 0, bytes.length); + static void zero(byte[] bytes, int off, int len) { + // this significantly faster than looping or Arrays.fill (which loops), particularly when the + // length of the slice to be zeroed is <= to ARRAY_LEN (in that case, it's faster by a + // factor of 2) + int remaining = len; + while (remaining >= ARRAY_LEN) { + System.arraycopy(ZERO_ARRAY, 0, bytes, off, ARRAY_LEN); + off += ARRAY_LEN; + remaining -= ARRAY_LEN; + } + + System.arraycopy(ZERO_ARRAY, 0, bytes, off, remaining); } /** - * Zeroes all bytes between off (inclusive) and off + len (exclusive) in the given array. + * Clears (sets to null) all blocks between off (inclusive) and off + len (exclusive) in the + * given array. */ - static void zero(byte[] bytes, int off, int len) { + static void clear(byte[][] blocks, int off, int len) { // this significantly faster than looping or Arrays.fill (which loops), particularly when the - // length of the slice to be zeroed is <= to ZERO_ARRAY_LEN (in that case, it's faster by a + // length of the slice to be cleared is <= to ARRAY_LEN (in that case, it's faster by a // factor of 2) int remaining = len; - while (remaining >= ZERO_ARRAY_LEN) { - System.arraycopy(ZERO_ARRAY, 0, bytes, off, ZERO_ARRAY_LEN); - off += ZERO_ARRAY_LEN; - remaining -= ZERO_ARRAY_LEN; + while (remaining >= ARRAY_LEN) { + System.arraycopy(NULL_ARRAY, 0, blocks, off, ARRAY_LEN); + off += ARRAY_LEN; + remaining -= ARRAY_LEN; } - System.arraycopy(ZERO_ARRAY, 0, bytes, off, remaining); + System.arraycopy(NULL_ARRAY, 0, blocks, off, remaining); } } diff --git a/jimfs/src/main/java/com/google/jimfs/internal/package-info.java b/jimfs/src/main/java/com/google/jimfs/internal/package-info.java index 10a8174..7cef362 100644 --- a/jimfs/src/main/java/com/google/jimfs/internal/package-info.java +++ b/jimfs/src/main/java/com/google/jimfs/internal/package-info.java @@ -48,7 +48,7 @@ * <ul> * <li>{@link com.google.jimfs.internal.FileFactory FileFactory} handles creation of new file * objects.</li> - * <li>{@link com.google.jimfs.internal.MemoryDisk MemoryDisk} handles creation and storage of + * <li>{@link com.google.jimfs.internal.HeapDisk HeapDisk} handles creation and storage of * {@link com.google.jimfs.internal.ByteStore ByteStore} instances, which act as the content of * regular files.</li> * <li>{@link com.google.jimfs.internal.FileTree FileTree} stores the root of the file hierarchy @@ -121,12 +121,10 @@ * <h3>Regular files</h3> * * Currently, the only implementation for regular file content is - * {@link com.google.jimfs.internal.MemoryDiskByteStore MemoryDiskByteStore}, which makes use of a - * singleton {@link com.google.jimfs.internal.MemoryDisk MemoryDisk}. A disk (which may either use - * {@linkplain com.google.jimfs.internal.HeapMemoryDisk heap} or - * {@linkplain com.google.jimfs.internal.DirectMemoryDisk direct} memory) is a resizable cache for + * {@link com.google.jimfs.internal.ByteStore ByteStore}, which makes use of a singleton + * {@link com.google.jimfs.internal.HeapDisk HeapDisk}. A disk is a resizable factory and cache for * fixed size blocks of memory. These blocks are allocated to files as needed and returned to the - * disk when a file is deleted or truncated. When existing free blocks are available, those blocks + * disk when a file is deleted or truncated. When cached free blocks are available, those blocks * are allocated to files first. If more blocks are needed, they are created. * * <h3>Linking</h3> |