From 9ab72b95735747ddbb512c61894ae6548c364a0a Mon Sep 17 00:00:00 2001 From: Colin Decker Date: Thu, 5 Dec 2013 18:14:16 -0500 Subject: Make HeapDisk parameters (block size, max size, max cache size) configurable in Configuration. --- .../main/java/com/google/jimfs/Configuration.java | 112 +++++++++++++++++++++ .../java/com/google/jimfs/internal/HeapDisk.java | 24 ++++- .../google/jimfs/internal/JimfsFileSystems.java | 2 +- 3 files changed, 133 insertions(+), 5 deletions(-) (limited to 'jimfs/src/main') diff --git a/jimfs/src/main/java/com/google/jimfs/Configuration.java b/jimfs/src/main/java/com/google/jimfs/Configuration.java index 3636dd4..e23288b 100644 --- a/jimfs/src/main/java/com/google/jimfs/Configuration.java +++ b/jimfs/src/main/java/com/google/jimfs/Configuration.java @@ -184,6 +184,11 @@ public final class Configuration { private final ImmutableSet nameCanonicalNormalization; private final boolean pathEqualityUsesCanonicalForm; + // Disk configuration + private final int blockSize; + private final long maxSize; + private final long maxCacheSize; + // Attribute configuration private final ImmutableSet attributeViews; private final ImmutableSet attributeProviders; @@ -201,6 +206,9 @@ public final class Configuration { this.nameDisplayNormalization = builder.nameDisplayNormalization; this.nameCanonicalNormalization = builder.nameCanonicalNormalization; this.pathEqualityUsesCanonicalForm = builder.pathEqualityUsesCanonicalForm; + this.blockSize = builder.blockSize; + this.maxSize = builder.maxSize; + this.maxCacheSize = builder.maxCacheSize; this.attributeViews = builder.attributeViews; this.attributeProviders = builder.attributeProviders == null ? ImmutableSet.of() @@ -257,6 +265,51 @@ public final class Configuration { return pathEqualityUsesCanonicalForm; } + /** + * Returns the block size (in bytes) for the file system to use. All regular files will be + * allocated blocks of the given size, so this is the minimum granularity for file size. + * + *

The default is 8192 bytes (8 KB). + */ + public int blockSize() { + return blockSize; + } + + /** + * Returns the maximum size (in bytes) for the file system's in-memory file storage. This maximum + * size determines the maximum number of blocks that can be allocated to regular files, so it + * should generally be a multiple of the {@linkplain #blockSize() block size}. The actual + * maximum size will be the nearest multiple of the block size that is less than or equal to this + * size. + * + *

Note: The in-memory file storage will not be eagerly initialized to this size, so + * it won't use more memory than is needed for the files you create. Also note that in addition + * to this limit, you will of course be limited by the amount of heap space available to the + * JVM and the amount of heap used by other objects, both in the file system and elsewhere. + * + *

The default is 4 GB. + */ + public long maxSize() { + return maxSize; + } + + /** + * Returns the maximum amount of unused space (in bytes) in the file system's in-memory file + * storage that should be cached for reuse. By default, this will be equal to the + * {@linkplain #maxSize() max size} of the storage, meaning that all space that is freed + * when files are truncated or deleted is cached for reuse. This helps to avoid lots of garbage + * collection when creating and deleting many files quickly. This can be set to 0 to disable + * caching entirely (all freed blocks become available for garbage collection) or to some other + * number to put an upper bound on the maximum amount of unused space the file system will keep + * around. + * + *

Like the max size, the actual value will be the closest multiple of the block size that + * is less than or equal to this size. + */ + public long maxCacheSize() { + return maxCacheSize; + } + /** * Returns the set of file attribute views the file system supports. */ @@ -298,6 +351,11 @@ public final class Configuration { private ImmutableSet nameCanonicalNormalization = ImmutableSet.of(); private boolean pathEqualityUsesCanonicalForm; + // Disk configuration + private int blockSize = 8192; // 8 KB + private long maxSize = 4L * 1024 * 1024 * 1024; // 4 GB + private long maxCacheSize = -1; // same as maxSize + // Attribute configuration private ImmutableSet attributeViews = ImmutableSet.of(); private Set attributeProviders = null; @@ -316,6 +374,9 @@ public final class Configuration { this.nameDisplayNormalization = configuration.nameDisplayNormalization; this.nameCanonicalNormalization = configuration.nameCanonicalNormalization; this.pathEqualityUsesCanonicalForm = configuration.pathEqualityUsesCanonicalForm; + this.blockSize = configuration.blockSize; + this.maxSize = configuration.maxSize; + this.maxCacheSize = configuration.maxCacheSize; this.attributeViews = configuration.attributeViews; this.attributeProviders = configuration.attributeProviders.isEmpty() ? null @@ -406,6 +467,57 @@ public final class Configuration { return this; } + /** + * Sets the block size (in bytes) for the file system to use. All regular files will be + * allocated blocks of the given size, so this is the minimum granularity for file size. + * + *

The default is 8192 bytes (8 KB). + */ + public Builder setBlockSize(int blockSize) { + checkArgument(blockSize > 0, "blockSize (%s) must be positive", blockSize); + this.blockSize = blockSize; + return this; + } + + /** + * Sets the maximum size (in bytes) for the file system's in-memory file storage. This maximum + * size determines the maximum number of blocks that can be allocated to regular files, so it + * should generally be a multiple of the {@linkplain #setBlockSize(int) block size}. The actual + * maximum size will be the nearest multiple of the block size that is less than or equal to + * the given size. + * + *

Note: The in-memory file storage will not be eagerly initialized to this size, so + * it won't use more memory than is needed for the files you create. Also note that in addition + * to this limit, you will of course be limited by the amount of heap space available to the + * JVM and the amount of heap used by other objects, both in the file system and elsewhere. + * + *

The default is 4 GB. + */ + public Builder setMaxSize(long maxSize) { + checkArgument(maxSize > 0, "maxSize (%s) must be positive", maxSize); + this.maxSize = maxSize; + return this; + } + + /** + * Sets the maximum amount of unused space (in bytes) in the file system's in-memory file + * storage that should be cached for reuse. By default, this will be equal to the + * {@linkplain #setMaxSize(long) max size} of the storage, meaning that all space that is freed + * when files are truncated or deleted is cached for reuse. This helps to avoid lots of garbage + * collection when creating and deleting many files quickly. This can be set to 0 to disable + * caching entirely (all freed blocks become available for garbage collection) or to some other + * number to put an upper bound on the maximum amount of unused space the file system will keep + * around. + * + *

Like the max size, the actual value will be the closest multiple of the block size that + * is less than or equal to the given size. + */ + public Builder setMaxCacheSize(long maxCacheSize) { + checkArgument(maxCacheSize >= 0, "maxCacheSize (%s) may not be negative", maxCacheSize); + this.maxCacheSize = maxCacheSize; + return this; + } + /** * Adds an attribute provider for a custom view for the file system to support. */ diff --git a/jimfs/src/main/java/com/google/jimfs/internal/HeapDisk.java b/jimfs/src/main/java/com/google/jimfs/internal/HeapDisk.java index 1a97ed5..08c4f9b 100644 --- a/jimfs/src/main/java/com/google/jimfs/internal/HeapDisk.java +++ b/jimfs/src/main/java/com/google/jimfs/internal/HeapDisk.java @@ -19,8 +19,11 @@ package com.google.jimfs.internal; import static com.google.common.base.Preconditions.checkArgument; import com.google.common.annotations.VisibleForTesting; +import com.google.common.math.LongMath; +import com.google.jimfs.Configuration; import java.io.IOException; +import java.math.RoundingMode; /** * A resizable pseudo-disk acting as a shared space for storing file data. A disk allocates fixed @@ -36,9 +39,9 @@ final class HeapDisk { /** 8 KB blocks. */ public static final int DEFAULT_BLOCK_SIZE = 8192; - /** 16 GB of space with 8 KB blocks. */ + /** 4 GB of space with 8 KB blocks. */ public static final int DEFAULT_MAX_BLOCK_COUNT = - (int) ((16L * 1024 * 1024 * 1024) / DEFAULT_BLOCK_SIZE); + (int) ((4L * 1024 * 1024 * 1024) / DEFAULT_BLOCK_SIZE); /** Fixed size of each block for this disk. */ private final int blockSize; @@ -56,20 +59,33 @@ final class HeapDisk { private int allocatedBlockCount; /** - * Creates a new heap disk with 8 KB blocks that can store up to 16 GB of data and caches all + * Creates a new heap disk with 8 KB blocks that can store up to 4 GB of data and caches all * blocks that are freed. */ public HeapDisk() { this(DEFAULT_BLOCK_SIZE, DEFAULT_MAX_BLOCK_COUNT, DEFAULT_MAX_BLOCK_COUNT); } + public HeapDisk(Configuration config) { + this.blockSize = config.blockSize(); + this.maxBlockCount = toBlockCount(config.maxSize(), blockSize); + this.maxCachedBlockCount = config.maxCacheSize() == -1 + ? maxBlockCount + : toBlockCount(config.maxCacheSize(), blockSize); + this.blockCache = new BlockList(Math.min(maxCachedBlockCount, 8192)); + } + + /** Returns the nearest multiple of {@code blockSize} that is <= {@code size}. */ + private static int toBlockCount(long size, int blockSize) { + return (int) LongMath.divide(size, blockSize, RoundingMode.FLOOR); + } + /** * Creates a new disk with the given {@code blockSize}, {@code maxBlockCount} and * {@code maxCachedBlockCount}. */ public HeapDisk(int blockSize, int maxBlockCount, int maxCachedBlockCount) { checkArgument(blockSize > 0, "blockSize (%s) must be positive", blockSize); - checkArgument(blockSize % 2 == 0, "blockSize (%s) must be a multiple of 2", blockSize); checkArgument(maxBlockCount > 0, "maxBlockCount (%s) must be positive", maxBlockCount); checkArgument(maxCachedBlockCount >= 0, "maxCachedBlockCount must be non-negative", maxCachedBlockCount); diff --git a/jimfs/src/main/java/com/google/jimfs/internal/JimfsFileSystems.java b/jimfs/src/main/java/com/google/jimfs/internal/JimfsFileSystems.java index 21e751f..5bfece6 100644 --- a/jimfs/src/main/java/com/google/jimfs/internal/JimfsFileSystems.java +++ b/jimfs/src/main/java/com/google/jimfs/internal/JimfsFileSystems.java @@ -58,7 +58,7 @@ final class JimfsFileSystems { AttributeService attributeService = new AttributeService(config); // TODO(cgdecker): Make disk values configurable - HeapDisk disk = new HeapDisk(); + HeapDisk disk = new HeapDisk(config); FileFactory fileFactory = new FileFactory(disk); Map roots = new HashMap<>(); -- cgit v1.2.3