diff options
Diffstat (limited to 'standalone/release.h')
-rw-r--r-- | standalone/release.h | 113 |
1 files changed, 66 insertions, 47 deletions
diff --git a/standalone/release.h b/standalone/release.h index 323bf9db6dc..b50f36fa0c0 100644 --- a/standalone/release.h +++ b/standalone/release.h @@ -49,7 +49,10 @@ private: // incremented past MaxValue. class PackedCounterArray { public: - PackedCounterArray(uptr NumCounters, uptr MaxValue) : N(NumCounters) { + PackedCounterArray(uptr NumberOfRegions, uptr CountersPerRegion, + uptr MaxValue) + : Regions(NumberOfRegions), NumCounters(CountersPerRegion) { + CHECK_GT(Regions, 0); CHECK_GT(NumCounters, 0); CHECK_GT(MaxValue, 0); constexpr uptr MaxCounterBits = sizeof(*Buffer) * 8UL; @@ -66,10 +69,12 @@ public: PackingRatioLog = getLog2(PackingRatio); BitOffsetMask = PackingRatio - 1; - BufferSize = (roundUpTo(N, static_cast<uptr>(1U) << PackingRatioLog) >> - PackingRatioLog) * - sizeof(*Buffer); - if (BufferSize <= StaticBufferSize && Mutex.tryLock()) { + SizePerRegion = + roundUpTo(NumCounters, static_cast<uptr>(1U) << PackingRatioLog) >> + PackingRatioLog; + BufferSize = SizePerRegion * sizeof(*Buffer) * Regions; + if (BufferSize <= (StaticBufferCount * sizeof(Buffer[0])) && + Mutex.tryLock()) { Buffer = &StaticBuffer[0]; memset(Buffer, 0, BufferSize); } else { @@ -88,45 +93,51 @@ public: bool isAllocated() const { return !!Buffer; } - uptr getCount() const { return N; } - uptr get(uptr I) const { - DCHECK_LT(I, N); + uptr getCount() const { return NumCounters; } + + uptr get(uptr Region, uptr I) const { + DCHECK_LT(Region, Regions); + DCHECK_LT(I, NumCounters); const uptr Index = I >> PackingRatioLog; const uptr BitOffset = (I & BitOffsetMask) << CounterSizeBitsLog; - return (Buffer[Index] >> BitOffset) & CounterMask; + return (Buffer[Region * SizePerRegion + Index] >> BitOffset) & CounterMask; } - void inc(uptr I) const { - DCHECK_LT(get(I), CounterMask); + void inc(uptr Region, uptr I) const { + DCHECK_LT(get(Region, I), CounterMask); const uptr Index = I >> PackingRatioLog; const uptr BitOffset = (I & BitOffsetMask) << CounterSizeBitsLog; DCHECK_LT(BitOffset, SCUDO_WORDSIZE); - Buffer[Index] += static_cast<uptr>(1U) << BitOffset; + Buffer[Region * SizePerRegion + Index] += static_cast<uptr>(1U) + << BitOffset; } - void incRange(uptr From, uptr To) const { + void incRange(uptr Region, uptr From, uptr To) const { DCHECK_LE(From, To); - const uptr Top = Min(To + 1, N); + const uptr Top = Min(To + 1, NumCounters); for (uptr I = From; I < Top; I++) - inc(I); + inc(Region, I); } uptr getBufferSize() const { return BufferSize; } + static const uptr StaticBufferCount = 2048U; + private: - const uptr N; + const uptr Regions; + const uptr NumCounters; uptr CounterSizeBitsLog; uptr CounterMask; uptr PackingRatioLog; uptr BitOffsetMask; + uptr SizePerRegion; uptr BufferSize; uptr *Buffer; static HybridMutex Mutex; - static const uptr StaticBufferSize = 1024U; - static uptr StaticBuffer[StaticBufferSize]; + static uptr StaticBuffer[StaticBufferCount]; }; template <class ReleaseRecorderT> class FreePagesRangeTracker { @@ -167,7 +178,8 @@ private: template <class TransferBatchT, class ReleaseRecorderT> NOINLINE void releaseFreeMemoryToOS(const IntrusiveList<TransferBatchT> &FreeList, uptr Base, - uptr Size, uptr BlockSize, ReleaseRecorderT *Recorder) { + uptr RegionSize, uptr NumberOfRegions, uptr BlockSize, + ReleaseRecorderT *Recorder) { const uptr PageSize = getPageSizeCached(); // Figure out the number of chunks per page and whether we can take a fast @@ -204,13 +216,14 @@ releaseFreeMemoryToOS(const IntrusiveList<TransferBatchT> &FreeList, uptr Base, } } - const uptr PagesCount = roundUpTo(Size, PageSize) / PageSize; - PackedCounterArray Counters(PagesCount, FullPagesBlockCountMax); + const uptr PagesCount = roundUpTo(RegionSize, PageSize) / PageSize; + PackedCounterArray Counters(NumberOfRegions, PagesCount, + FullPagesBlockCountMax); if (!Counters.isAllocated()) return; const uptr PageSizeLog = getLog2(PageSize); - const uptr RoundedSize = PagesCount << PageSizeLog; + const uptr RoundedSize = NumberOfRegions * (PagesCount << PageSizeLog); // Iterate over free chunks and count how many free chunks affect each // allocated page. @@ -226,12 +239,13 @@ releaseFreeMemoryToOS(const IntrusiveList<TransferBatchT> &FreeList, uptr Base, for (u32 I = IsTransferBatch ? 1 : 0; I < It.getCount(); I++) { const uptr P = reinterpret_cast<uptr>(It.get(I)) - Base; // This takes care of P < Base and P >= Base + RoundedSize. - if (P < RoundedSize) - Counters.inc(P >> PageSizeLog); + if (P < RoundedSize) { + const uptr RegionIndex = NumberOfRegions == 1U ? 0 : P / RegionSize; + const uptr PInRegion = P - RegionIndex * RegionSize; + Counters.inc(RegionIndex, PInRegion >> PageSizeLog); + } } } - for (uptr P = Size; P < RoundedSize; P += BlockSize) - Counters.inc(P >> PageSizeLog); } else { // In all other cases chunks might affect more than one page. for (const auto &It : FreeList) { @@ -242,13 +256,14 @@ releaseFreeMemoryToOS(const IntrusiveList<TransferBatchT> &FreeList, uptr Base, for (u32 I = IsTransferBatch ? 1 : 0; I < It.getCount(); I++) { const uptr P = reinterpret_cast<uptr>(It.get(I)) - Base; // This takes care of P < Base and P >= Base + RoundedSize. - if (P < RoundedSize) - Counters.incRange(P >> PageSizeLog, - (P + BlockSize - 1) >> PageSizeLog); + if (P < RoundedSize) { + const uptr RegionIndex = NumberOfRegions == 1U ? 0 : P / RegionSize; + const uptr PInRegion = P - RegionIndex * RegionSize; + Counters.incRange(RegionIndex, PInRegion >> PageSizeLog, + (PInRegion + BlockSize - 1) >> PageSizeLog); + } } } - for (uptr P = Size; P < RoundedSize; P += BlockSize) - Counters.incRange(P >> PageSizeLog, (P + BlockSize - 1) >> PageSizeLog); } // Iterate over pages detecting ranges of pages with chunk Counters equal @@ -256,8 +271,10 @@ releaseFreeMemoryToOS(const IntrusiveList<TransferBatchT> &FreeList, uptr Base, FreePagesRangeTracker<ReleaseRecorderT> RangeTracker(Recorder); if (SameBlockCountPerPage) { // Fast path, every page has the same number of chunks affecting it. - for (uptr I = 0; I < Counters.getCount(); I++) - RangeTracker.processNextPage(Counters.get(I) == FullPagesBlockCountMax); + for (uptr I = 0; I < NumberOfRegions; I++) + for (uptr J = 0; J < PagesCount; J++) + RangeTracker.processNextPage(Counters.get(I, J) == + FullPagesBlockCountMax); } else { // Slow path, go through the pages keeping count how many chunks affect // each page. @@ -268,23 +285,25 @@ releaseFreeMemoryToOS(const IntrusiveList<TransferBatchT> &FreeList, uptr Base, // except the first and the last one) and then the last chunk size, adding // up the number of chunks on the current page and checking on every step // whether the page boundary was crossed. - uptr PrevPageBoundary = 0; - uptr CurrentBoundary = 0; - for (uptr I = 0; I < Counters.getCount(); I++) { - const uptr PageBoundary = PrevPageBoundary + PageSize; - uptr BlocksPerPage = Pn; - if (CurrentBoundary < PageBoundary) { - if (CurrentBoundary > PrevPageBoundary) - BlocksPerPage++; - CurrentBoundary += Pnc; + for (uptr I = 0; I < NumberOfRegions; I++) { + uptr PrevPageBoundary = 0; + uptr CurrentBoundary = 0; + for (uptr J = 0; J < PagesCount; J++) { + const uptr PageBoundary = PrevPageBoundary + PageSize; + uptr BlocksPerPage = Pn; if (CurrentBoundary < PageBoundary) { - BlocksPerPage++; - CurrentBoundary += BlockSize; + if (CurrentBoundary > PrevPageBoundary) + BlocksPerPage++; + CurrentBoundary += Pnc; + if (CurrentBoundary < PageBoundary) { + BlocksPerPage++; + CurrentBoundary += BlockSize; + } } - } - PrevPageBoundary = PageBoundary; + PrevPageBoundary = PageBoundary; - RangeTracker.processNextPage(Counters.get(I) == BlocksPerPage); + RangeTracker.processNextPage(Counters.get(I, J) == BlocksPerPage); + } } } RangeTracker.finish(); |