summaryrefslogtreecommitdiff
path: root/standalone/release.h
diff options
context:
space:
mode:
Diffstat (limited to 'standalone/release.h')
-rw-r--r--standalone/release.h113
1 files changed, 66 insertions, 47 deletions
diff --git a/standalone/release.h b/standalone/release.h
index 323bf9db6dc..b50f36fa0c0 100644
--- a/standalone/release.h
+++ b/standalone/release.h
@@ -49,7 +49,10 @@ private:
// incremented past MaxValue.
class PackedCounterArray {
public:
- PackedCounterArray(uptr NumCounters, uptr MaxValue) : N(NumCounters) {
+ PackedCounterArray(uptr NumberOfRegions, uptr CountersPerRegion,
+ uptr MaxValue)
+ : Regions(NumberOfRegions), NumCounters(CountersPerRegion) {
+ CHECK_GT(Regions, 0);
CHECK_GT(NumCounters, 0);
CHECK_GT(MaxValue, 0);
constexpr uptr MaxCounterBits = sizeof(*Buffer) * 8UL;
@@ -66,10 +69,12 @@ public:
PackingRatioLog = getLog2(PackingRatio);
BitOffsetMask = PackingRatio - 1;
- BufferSize = (roundUpTo(N, static_cast<uptr>(1U) << PackingRatioLog) >>
- PackingRatioLog) *
- sizeof(*Buffer);
- if (BufferSize <= StaticBufferSize && Mutex.tryLock()) {
+ SizePerRegion =
+ roundUpTo(NumCounters, static_cast<uptr>(1U) << PackingRatioLog) >>
+ PackingRatioLog;
+ BufferSize = SizePerRegion * sizeof(*Buffer) * Regions;
+ if (BufferSize <= (StaticBufferCount * sizeof(Buffer[0])) &&
+ Mutex.tryLock()) {
Buffer = &StaticBuffer[0];
memset(Buffer, 0, BufferSize);
} else {
@@ -88,45 +93,51 @@ public:
bool isAllocated() const { return !!Buffer; }
- uptr getCount() const { return N; }
- uptr get(uptr I) const {
- DCHECK_LT(I, N);
+ uptr getCount() const { return NumCounters; }
+
+ uptr get(uptr Region, uptr I) const {
+ DCHECK_LT(Region, Regions);
+ DCHECK_LT(I, NumCounters);
const uptr Index = I >> PackingRatioLog;
const uptr BitOffset = (I & BitOffsetMask) << CounterSizeBitsLog;
- return (Buffer[Index] >> BitOffset) & CounterMask;
+ return (Buffer[Region * SizePerRegion + Index] >> BitOffset) & CounterMask;
}
- void inc(uptr I) const {
- DCHECK_LT(get(I), CounterMask);
+ void inc(uptr Region, uptr I) const {
+ DCHECK_LT(get(Region, I), CounterMask);
const uptr Index = I >> PackingRatioLog;
const uptr BitOffset = (I & BitOffsetMask) << CounterSizeBitsLog;
DCHECK_LT(BitOffset, SCUDO_WORDSIZE);
- Buffer[Index] += static_cast<uptr>(1U) << BitOffset;
+ Buffer[Region * SizePerRegion + Index] += static_cast<uptr>(1U)
+ << BitOffset;
}
- void incRange(uptr From, uptr To) const {
+ void incRange(uptr Region, uptr From, uptr To) const {
DCHECK_LE(From, To);
- const uptr Top = Min(To + 1, N);
+ const uptr Top = Min(To + 1, NumCounters);
for (uptr I = From; I < Top; I++)
- inc(I);
+ inc(Region, I);
}
uptr getBufferSize() const { return BufferSize; }
+ static const uptr StaticBufferCount = 2048U;
+
private:
- const uptr N;
+ const uptr Regions;
+ const uptr NumCounters;
uptr CounterSizeBitsLog;
uptr CounterMask;
uptr PackingRatioLog;
uptr BitOffsetMask;
+ uptr SizePerRegion;
uptr BufferSize;
uptr *Buffer;
static HybridMutex Mutex;
- static const uptr StaticBufferSize = 1024U;
- static uptr StaticBuffer[StaticBufferSize];
+ static uptr StaticBuffer[StaticBufferCount];
};
template <class ReleaseRecorderT> class FreePagesRangeTracker {
@@ -167,7 +178,8 @@ private:
template <class TransferBatchT, class ReleaseRecorderT>
NOINLINE void
releaseFreeMemoryToOS(const IntrusiveList<TransferBatchT> &FreeList, uptr Base,
- uptr Size, uptr BlockSize, ReleaseRecorderT *Recorder) {
+ uptr RegionSize, uptr NumberOfRegions, uptr BlockSize,
+ ReleaseRecorderT *Recorder) {
const uptr PageSize = getPageSizeCached();
// Figure out the number of chunks per page and whether we can take a fast
@@ -204,13 +216,14 @@ releaseFreeMemoryToOS(const IntrusiveList<TransferBatchT> &FreeList, uptr Base,
}
}
- const uptr PagesCount = roundUpTo(Size, PageSize) / PageSize;
- PackedCounterArray Counters(PagesCount, FullPagesBlockCountMax);
+ const uptr PagesCount = roundUpTo(RegionSize, PageSize) / PageSize;
+ PackedCounterArray Counters(NumberOfRegions, PagesCount,
+ FullPagesBlockCountMax);
if (!Counters.isAllocated())
return;
const uptr PageSizeLog = getLog2(PageSize);
- const uptr RoundedSize = PagesCount << PageSizeLog;
+ const uptr RoundedSize = NumberOfRegions * (PagesCount << PageSizeLog);
// Iterate over free chunks and count how many free chunks affect each
// allocated page.
@@ -226,12 +239,13 @@ releaseFreeMemoryToOS(const IntrusiveList<TransferBatchT> &FreeList, uptr Base,
for (u32 I = IsTransferBatch ? 1 : 0; I < It.getCount(); I++) {
const uptr P = reinterpret_cast<uptr>(It.get(I)) - Base;
// This takes care of P < Base and P >= Base + RoundedSize.
- if (P < RoundedSize)
- Counters.inc(P >> PageSizeLog);
+ if (P < RoundedSize) {
+ const uptr RegionIndex = NumberOfRegions == 1U ? 0 : P / RegionSize;
+ const uptr PInRegion = P - RegionIndex * RegionSize;
+ Counters.inc(RegionIndex, PInRegion >> PageSizeLog);
+ }
}
}
- for (uptr P = Size; P < RoundedSize; P += BlockSize)
- Counters.inc(P >> PageSizeLog);
} else {
// In all other cases chunks might affect more than one page.
for (const auto &It : FreeList) {
@@ -242,13 +256,14 @@ releaseFreeMemoryToOS(const IntrusiveList<TransferBatchT> &FreeList, uptr Base,
for (u32 I = IsTransferBatch ? 1 : 0; I < It.getCount(); I++) {
const uptr P = reinterpret_cast<uptr>(It.get(I)) - Base;
// This takes care of P < Base and P >= Base + RoundedSize.
- if (P < RoundedSize)
- Counters.incRange(P >> PageSizeLog,
- (P + BlockSize - 1) >> PageSizeLog);
+ if (P < RoundedSize) {
+ const uptr RegionIndex = NumberOfRegions == 1U ? 0 : P / RegionSize;
+ const uptr PInRegion = P - RegionIndex * RegionSize;
+ Counters.incRange(RegionIndex, PInRegion >> PageSizeLog,
+ (PInRegion + BlockSize - 1) >> PageSizeLog);
+ }
}
}
- for (uptr P = Size; P < RoundedSize; P += BlockSize)
- Counters.incRange(P >> PageSizeLog, (P + BlockSize - 1) >> PageSizeLog);
}
// Iterate over pages detecting ranges of pages with chunk Counters equal
@@ -256,8 +271,10 @@ releaseFreeMemoryToOS(const IntrusiveList<TransferBatchT> &FreeList, uptr Base,
FreePagesRangeTracker<ReleaseRecorderT> RangeTracker(Recorder);
if (SameBlockCountPerPage) {
// Fast path, every page has the same number of chunks affecting it.
- for (uptr I = 0; I < Counters.getCount(); I++)
- RangeTracker.processNextPage(Counters.get(I) == FullPagesBlockCountMax);
+ for (uptr I = 0; I < NumberOfRegions; I++)
+ for (uptr J = 0; J < PagesCount; J++)
+ RangeTracker.processNextPage(Counters.get(I, J) ==
+ FullPagesBlockCountMax);
} else {
// Slow path, go through the pages keeping count how many chunks affect
// each page.
@@ -268,23 +285,25 @@ releaseFreeMemoryToOS(const IntrusiveList<TransferBatchT> &FreeList, uptr Base,
// except the first and the last one) and then the last chunk size, adding
// up the number of chunks on the current page and checking on every step
// whether the page boundary was crossed.
- uptr PrevPageBoundary = 0;
- uptr CurrentBoundary = 0;
- for (uptr I = 0; I < Counters.getCount(); I++) {
- const uptr PageBoundary = PrevPageBoundary + PageSize;
- uptr BlocksPerPage = Pn;
- if (CurrentBoundary < PageBoundary) {
- if (CurrentBoundary > PrevPageBoundary)
- BlocksPerPage++;
- CurrentBoundary += Pnc;
+ for (uptr I = 0; I < NumberOfRegions; I++) {
+ uptr PrevPageBoundary = 0;
+ uptr CurrentBoundary = 0;
+ for (uptr J = 0; J < PagesCount; J++) {
+ const uptr PageBoundary = PrevPageBoundary + PageSize;
+ uptr BlocksPerPage = Pn;
if (CurrentBoundary < PageBoundary) {
- BlocksPerPage++;
- CurrentBoundary += BlockSize;
+ if (CurrentBoundary > PrevPageBoundary)
+ BlocksPerPage++;
+ CurrentBoundary += Pnc;
+ if (CurrentBoundary < PageBoundary) {
+ BlocksPerPage++;
+ CurrentBoundary += BlockSize;
+ }
}
- }
- PrevPageBoundary = PageBoundary;
+ PrevPageBoundary = PageBoundary;
- RangeTracker.processNextPage(Counters.get(I) == BlocksPerPage);
+ RangeTracker.processNextPage(Counters.get(I, J) == BlocksPerPage);
+ }
}
}
RangeTracker.finish();
generated by cgit v1.2.3 (git 2.25.1) at 2024-06-01 22:13:23 +0000