diff --git a/sdk/lib/rtl/heap.c b/sdk/lib/rtl/heap.c index b38f4c93e3b..890d0a8c56a 100644 --- a/sdk/lib/rtl/heap.c +++ b/sdk/lib/rtl/heap.c @@ -212,13 +212,22 @@ RtlpInitializeHeap(OUT PHEAP Heap, for (Index = 0; Index < HEAP_SEGMENTS; ++Index) Heap->Segments[Index] = NULL; - /* Initialise the free entry lists. */ - InitializeListHead(&Heap->FreeLists); - RtlInitializeBitMap(&Heap->FreeHintBitmap, - (PULONG)&Heap->FreeHints[DeCommitFreeBlockThreshold], - DeCommitFreeBlockThreshold); - RtlClearAllBits(&Heap->FreeHintBitmap); - RtlZeroMemory(&Heap->FreeHints[0], sizeof(Heap->FreeHints[0]) * DeCommitFreeBlockThreshold); + /* Initialise the Heap Free Heap Entry lists */ + if (Flags == HEAP_CLASS_1) // Private Heap Not GROWABLE + { + for (Index = 0; Index < HEAP_FREELISTS; ++Index) + InitializeListHead(&Heap->FreeList[Index]); + } + else + { + /* Initialise the free entry lists. */ + InitializeListHead(&Heap->FreeLists); + RtlInitializeBitMap(&Heap->FreeHintBitmap, + (PULONG)&Heap->FreeHints[DeCommitFreeBlockThreshold], + DeCommitFreeBlockThreshold); + RtlClearAllBits(&Heap->FreeHintBitmap); + RtlZeroMemory(&Heap->FreeHints[0], sizeof(Heap->FreeHints[0]) * DeCommitFreeBlockThreshold); + } /* Initialise the Heap Virtual Allocated Blocks list */ InitializeListHead(&Heap->VirtualAllocdBlocks); @@ -234,6 +243,47 @@ RtlpInitializeHeap(OUT PHEAP Heap, return STATUS_SUCCESS; } +FORCEINLINE +VOID +RtlpSetFreeListsBit(PHEAP Heap, + PHEAP_FREE_ENTRY FreeEntry) +{ + ULONG Index, Bit; + + ASSERT(FreeEntry->Size < HEAP_FREELISTS); + + /* Calculate offset in the free list bitmap */ + Index = FreeEntry->Size >> 3; /* = FreeEntry->Size / (sizeof(UCHAR) * 8)*/ + Bit = 1 << (FreeEntry->Size & 7); + + /* Assure it's not already set */ + ASSERT((Heap->u.FreeListsInUseBytes[Index] & Bit) == 0); + + /* Set it */ + Heap->u.FreeListsInUseBytes[Index] |= Bit; +} + +FORCEINLINE +VOID +RtlpClearFreeListsBit(PHEAP Heap, + PHEAP_FREE_ENTRY FreeEntry) +{ + ULONG Index, Bit; + + ASSERT(FreeEntry->Size < HEAP_FREELISTS); + + /* Calculate offset in the free list bitmap */ + Index = FreeEntry->Size >> 3; /* = FreeEntry->Size / (sizeof(UCHAR) * 8)*/ + Bit = 1 << (FreeEntry->Size & 7); + + /* Assure it was set and the corresponding free list is empty */ + ASSERT(Heap->u.FreeListsInUseBytes[Index] & Bit); + ASSERT(IsListEmpty(&Heap->FreeList[FreeEntry->Size])); + + /* Clear it */ + Heap->u.FreeListsInUseBytes[Index] ^= Bit; +} + VOID NTAPI RtlpInsertFreeBlockHelper(PHEAP Heap, PHEAP_FREE_ENTRY FreeEntry, @@ -241,6 +291,9 @@ RtlpInsertFreeBlockHelper(PHEAP Heap, BOOLEAN NoFill) { ULONG HintIndex, NextHintIndex; + PLIST_ENTRY FreeListHead, Current; + PHEAP_FREE_ENTRY CurrentEntry; + ULONG ClassFlags = Heap->ClassFlags & HEAP_CLASS_MASK; ASSERT(FreeEntry->Size == BlockSize); @@ -266,93 +319,199 @@ RtlpInsertFreeBlockHelper(PHEAP Heap, FreeEntry->Flags &= HEAP_ENTRY_LAST_ENTRY; } - /* See if this should go to the dedicated list */ - if (BlockSize > Heap->DeCommitFreeBlockThreshold) + if (ClassFlags == HEAP_CLASS_1) // Private Heap Not GROWABLE { - PLIST_ENTRY ListEntry = Heap->FreeHints[0]; - - /* Check if we have a hint there */ - if (ListEntry == NULL) + /* Insert it either into dedicated or non-dedicated list */ + if (BlockSize < HEAP_FREELISTS) { - ASSERT(!RtlTestBit(&Heap->FreeHintBitmap, 0)); - Heap->FreeHints[0] = &FreeEntry->FreeList; - RtlSetBit(&Heap->FreeHintBitmap, 0); - InsertTailList(&Heap->FreeLists, &FreeEntry->FreeList); - return; + /* Dedicated list */ + FreeListHead = &Heap->FreeList[BlockSize]; + + if (IsListEmpty(FreeListHead)) + { + RtlpSetFreeListsBit(Heap, FreeEntry); + } } + else + { + /* Non-dedicated one */ + FreeListHead = &Heap->FreeList[0]; + Current = FreeListHead->Flink; + + /* Find a position where to insert it to (the list must be sorted) */ + while (FreeListHead != Current) + { + CurrentEntry = CONTAINING_RECORD(Current, HEAP_FREE_ENTRY, FreeList); - ASSERT(RtlTestBit(&Heap->FreeHintBitmap, 0)); + if (BlockSize <= CurrentEntry->Size) + break; + + Current = Current->Flink; + } - while (ListEntry != &Heap->FreeLists) + FreeListHead = Current; + } + + /* Actually insert it into the list */ + InsertTailList(FreeListHead, &FreeEntry->FreeList); + } + else + { + /* See if this should go to the dedicated list */ + if (BlockSize > Heap->DeCommitFreeBlockThreshold) { - PHEAP_FREE_ENTRY PreviousEntry = CONTAINING_RECORD(ListEntry, - HEAP_FREE_ENTRY, - FreeList); - if (PreviousEntry->Size >= BlockSize) + PLIST_ENTRY ListEntry = Heap->FreeHints[0]; + + /* Check if we have a hint there */ + if (ListEntry == NULL) { - DPRINT("Inserting size %lu before %lu.\n", BlockSize, PreviousEntry->Size); - break; + ASSERT(!RtlTestBit(&Heap->FreeHintBitmap, 0)); + Heap->FreeHints[0] = &FreeEntry->FreeList; + RtlSetBit(&Heap->FreeHintBitmap, 0); + InsertTailList(&Heap->FreeLists, &FreeEntry->FreeList); + return; } - ListEntry = ListEntry->Flink; - } + ASSERT(RtlTestBit(&Heap->FreeHintBitmap, 0)); + + while (ListEntry != &Heap->FreeLists) + { + PHEAP_FREE_ENTRY PreviousEntry = CONTAINING_RECORD(ListEntry, + HEAP_FREE_ENTRY, + FreeList); + if (PreviousEntry->Size >= BlockSize) + { + DPRINT("Inserting size %lu before %lu.\n", BlockSize, PreviousEntry->Size); + break; + } - InsertTailList(ListEntry, &FreeEntry->FreeList); + ListEntry = ListEntry->Flink; + } - /* Update our hint if needed */ - if (Heap->FreeHints[0] == ListEntry) - Heap->FreeHints[0] = &FreeEntry->FreeList; + InsertTailList(ListEntry, &FreeEntry->FreeList); - return; - } + /* Update our hint if needed */ + if (Heap->FreeHints[0] == ListEntry) + Heap->FreeHints[0] = &FreeEntry->FreeList; - ASSERT(BlockSize >= 2); - HintIndex = BlockSize - 1; + return; + } - if (Heap->FreeHints[HintIndex] != NULL) - { - ASSERT(RtlTestBit(&Heap->FreeHintBitmap, HintIndex)); + ASSERT(BlockSize >= 2); + HintIndex = BlockSize - 1; - /* Insert it after our hint. */ - InsertHeadList(Heap->FreeHints[HintIndex], &FreeEntry->FreeList); + if (Heap->FreeHints[HintIndex] != NULL) + { + ASSERT(RtlTestBit(&Heap->FreeHintBitmap, HintIndex)); - return; - } + /* Insert it after our hint. */ + InsertHeadList(Heap->FreeHints[HintIndex], &FreeEntry->FreeList); - /* This is the first time we insert such an entry in the list. */ - ASSERT(!RtlTestBit(&Heap->FreeHintBitmap, HintIndex)); - if (IsListEmpty(&Heap->FreeLists)) - { - /* First entry inserted in this list ever */ - InsertHeadList(&Heap->FreeLists, &FreeEntry->FreeList); + return; + } + + /* This is the first time we insert such an entry in the list. */ + ASSERT(!RtlTestBit(&Heap->FreeHintBitmap, HintIndex)); + if (IsListEmpty(&Heap->FreeLists)) + { + /* First entry inserted in this list ever */ + InsertHeadList(&Heap->FreeLists, &FreeEntry->FreeList); + RtlSetBit(&Heap->FreeHintBitmap, HintIndex); + Heap->FreeHints[HintIndex] = &FreeEntry->FreeList; + return; + } + + /* Find the closest one */ + NextHintIndex = RtlFindSetBits(&Heap->FreeHintBitmap, 1, HintIndex); + ASSERT(NextHintIndex != 0xFFFFFFFF); + if ((NextHintIndex == 0) || (NextHintIndex > HintIndex)) + { + /* + * We found a larger entry. Insert this one before. + * It is guaranteed to be our successor in the list. + */ + InsertTailList(Heap->FreeHints[NextHintIndex], &FreeEntry->FreeList); + } + else + { + /* We only found an entry smaller than us. Then we will be the largest one. */ + ASSERT(CONTAINING_RECORD(Heap->FreeLists.Blink, HEAP_FREE_ENTRY, FreeList)->Size < BlockSize); + InsertTailList(&Heap->FreeLists, &FreeEntry->FreeList); + } + + /* Setup our hint */ RtlSetBit(&Heap->FreeHintBitmap, HintIndex); Heap->FreeHints[HintIndex] = &FreeEntry->FreeList; - return; } +} - /* Find the closest one */ - NextHintIndex = RtlFindSetBits(&Heap->FreeHintBitmap, 1, HintIndex); - ASSERT(NextHintIndex != 0xFFFFFFFF); - if ((NextHintIndex == 0) || (NextHintIndex > HintIndex)) - { - /* - * We found a larger entry. Insert this one before. - * It is guaranteed to be our successor in the list. - */ - InsertTailList(Heap->FreeHints[NextHintIndex], &FreeEntry->FreeList); - } - else + +VOID NTAPI +RtlpInsertFreeBlock1(PHEAP Heap, + PHEAP_FREE_ENTRY FreeEntry, + SIZE_T BlockSize) +{ + USHORT Size, PreviousSize; + UCHAR SegmentOffset, Flags; + PHEAP_SEGMENT Segment; + + DPRINT1("RtlpInsertFreeBlock1(%p %p %x)\n", Heap, FreeEntry, BlockSize); + + /* Increase the free size counter */ + Heap->TotalFreeSize += BlockSize; + + /* Remember certain values */ + Flags = FreeEntry->Flags; + PreviousSize = FreeEntry->PreviousSize; + SegmentOffset = FreeEntry->SegmentOffset; + Segment = Heap->Segments[SegmentOffset]; + + /* Process it */ + while (BlockSize) { - /* We only found an entry smaller than us. Then we will be the largest one. */ - ASSERT(CONTAINING_RECORD(Heap->FreeLists.Blink, HEAP_FREE_ENTRY, FreeList)->Size < BlockSize); - InsertTailList(&Heap->FreeLists, &FreeEntry->FreeList); + /* Check for the max size */ + if (BlockSize > HEAP_MAX_BLOCK_SIZE) + { + Size = HEAP_MAX_BLOCK_SIZE; + + /* Special compensation if it goes above limit just by 1 */ + if (BlockSize == (HEAP_MAX_BLOCK_SIZE + 1)) + Size -= 16; + + FreeEntry->Flags = 0; + } + else + { + Size = (USHORT)BlockSize; + FreeEntry->Flags = Flags; + } + + /* Change its size and insert it into a free list */ + FreeEntry->Size = Size; + FreeEntry->PreviousSize = PreviousSize; + FreeEntry->SegmentOffset = SegmentOffset; + + /* Call a helper to actually insert the block */ + RtlpInsertFreeBlockHelper(Heap, FreeEntry, Size, FALSE); + + /* Update sizes */ + PreviousSize = Size; + BlockSize -= Size; + + /* Go to the next entry */ + FreeEntry = (PHEAP_FREE_ENTRY)((PHEAP_ENTRY)FreeEntry + Size); + + /* Check if that's all */ + if ((PHEAP_ENTRY)FreeEntry >= Segment->LastValidEntry) return; } - /* Setup our hint */ - RtlSetBit(&Heap->FreeHintBitmap, HintIndex); - Heap->FreeHints[HintIndex] = &FreeEntry->FreeList; + /* Update previous size if needed */ + if (!(Flags & HEAP_ENTRY_LAST_ENTRY)) + FreeEntry->PreviousSize = PreviousSize; } + + PHEAP_FREE_ENTRY NTAPI RtlpInsertFreeBlock(PHEAP Heap, @@ -426,6 +585,48 @@ RtlpInsertFreeBlock(PHEAP Heap, return LastEntry; } +VOID NTAPI +RtlpRemoveFreeBlock1(PHEAP Heap, + PHEAP_FREE_ENTRY FreeEntry, + BOOLEAN Dedicated, + BOOLEAN NoFill) +{ + SIZE_T Result, RealSize; + + /* Remove the free block and update the freelists bitmap */ + if (RemoveEntryList(&FreeEntry->FreeList) && + (Dedicated || (!Dedicated && FreeEntry->Size < HEAP_FREELISTS))) + { + RtlpClearFreeListsBit(Heap, FreeEntry); + } + + /* Fill with pattern if necessary */ + if (!NoFill && + (FreeEntry->Flags & HEAP_ENTRY_FILL_PATTERN)) + { + RealSize = (FreeEntry->Size << HEAP_ENTRY_SHIFT) - sizeof(*FreeEntry); + + /* Deduct extra stuff from block's real size */ + if (FreeEntry->Flags & HEAP_ENTRY_EXTRA_PRESENT && + RealSize > sizeof(HEAP_FREE_ENTRY_EXTRA)) + { + RealSize -= sizeof(HEAP_FREE_ENTRY_EXTRA); + } + + /* Check if the free filler is intact */ + Result = RtlCompareMemoryUlong((PCHAR)(FreeEntry + 1), + RealSize, + ARENA_FREE_FILLER); + + if (Result != RealSize) + { + DPRINT1("Free heap block %p modified at %p after it was freed\n", + FreeEntry, + (PCHAR)(FreeEntry + 1) + Result); + } + } +} + static VOID RtlpRemoveFreeBlock(PHEAP Heap, @@ -698,6 +899,142 @@ RtlpInsertUnCommittedPages(PHEAP_SEGMENT Segment, Segment->NumberOfUnCommittedRanges++; } + +PHEAP_FREE_ENTRY NTAPI +RtlpFindAndCommitPages1(PHEAP Heap, + PHEAP_SEGMENT Segment, + PSIZE_T Size, + PVOID AddressRequested) +{ + PLIST_ENTRY Current; + ULONG_PTR Address = 0; + PHEAP_UCR_DESCRIPTOR UcrDescriptor, PreviousUcr = NULL; + PHEAP_ENTRY FirstEntry, LastEntry; + NTSTATUS Status; + + DPRINT("RtlpFindAndCommitPages1(%p %p %Ix %08Ix)\n", Heap, Segment, *Size, Address); + + /* Go through UCRs in a segment */ + Current = Segment->UCRSegmentList.Flink; + while (Current != &Segment->UCRSegmentList) + { + UcrDescriptor = CONTAINING_RECORD(Current, HEAP_UCR_DESCRIPTOR, SegmentEntry); + + /* Check if we can use that one right away */ + if (UcrDescriptor->Size >= *Size && + (UcrDescriptor->Address == AddressRequested || !AddressRequested)) + { + /* Get the address */ + Address = (ULONG_PTR)UcrDescriptor->Address; + + /* Commit it */ + if (Heap->CommitRoutine) + { + Status = Heap->CommitRoutine(Heap, (PVOID *)&Address, Size); + } + else + { + Status = ZwAllocateVirtualMemory(NtCurrentProcess(), + (PVOID *)&Address, + 0, + Size, + MEM_COMMIT, + PAGE_READWRITE); + } + + DPRINT("Committed %Iu bytes at base %08Ix, UCR size is %lu\n", *Size, Address, UcrDescriptor->Size); + + /* Fail in unsuccessful case */ + if (!NT_SUCCESS(Status)) + { + DPRINT1("Committing page failed with status 0x%08X\n", Status); + return NULL; + } + + /* Update tracking numbers */ + Segment->NumberOfUnCommittedPages -= (ULONG)(*Size / PAGE_SIZE); + + /* Calculate first and last entries */ + FirstEntry = (PHEAP_ENTRY)Address; + + LastEntry = Segment->LastEntryInSegment; + if (!(LastEntry->Flags & HEAP_ENTRY_LAST_ENTRY) || + LastEntry + LastEntry->Size != FirstEntry) + { + /* Go through the entries to find the last one */ + if (PreviousUcr) + LastEntry = (PHEAP_ENTRY)((ULONG_PTR)PreviousUcr->Address + PreviousUcr->Size); + else + LastEntry = &Segment->Entry; + + while (!(LastEntry->Flags & HEAP_ENTRY_LAST_ENTRY)) + { + ASSERT(LastEntry->Size != 0); + LastEntry += LastEntry->Size; + } + } + ASSERT((LastEntry + LastEntry->Size) == FirstEntry); + + /* Unmark it as a last entry */ + LastEntry->Flags &= ~HEAP_ENTRY_LAST_ENTRY; + + /* Update UCR descriptor */ + UcrDescriptor->Address = (PVOID)((ULONG_PTR)UcrDescriptor->Address + *Size); + UcrDescriptor->Size -= *Size; + + DPRINT("Updating UcrDescriptor %p, new Address %p, size %lu\n", + UcrDescriptor, UcrDescriptor->Address, UcrDescriptor->Size); + + /* Set various first entry fields */ + FirstEntry->SegmentOffset = LastEntry->SegmentOffset; + FirstEntry->Size = (USHORT)(*Size >> HEAP_ENTRY_SHIFT); + FirstEntry->PreviousSize = LastEntry->Size; + + /* Check if anything left in this UCR */ + if (UcrDescriptor->Size == 0) + { + /* It's fully exhausted */ + + /* Check if this is the end of the segment */ + if(UcrDescriptor->Address == Segment->LastValidEntry) + { + FirstEntry->Flags = HEAP_ENTRY_LAST_ENTRY; + Segment->LastEntryInSegment = FirstEntry; + } + else + { + FirstEntry->Flags = 0; + Segment->LastEntryInSegment = Segment->FirstEntry; + /* Update field of next entry */ + ASSERT((FirstEntry + FirstEntry->Size)->PreviousSize == 0); + (FirstEntry + FirstEntry->Size)->PreviousSize = FirstEntry->Size; + } + + /* This UCR needs to be removed because it became useless */ + RemoveEntryList(&UcrDescriptor->SegmentEntry); + + RtlpDestroyUnCommittedRange(Segment, UcrDescriptor); + Segment->NumberOfUnCommittedRanges--; + } + else + { + FirstEntry->Flags = HEAP_ENTRY_LAST_ENTRY; + Segment->LastEntryInSegment = FirstEntry; + } + + /* We're done */ + return (PHEAP_FREE_ENTRY)FirstEntry; + } + + /* Advance to the next descriptor */ + PreviousUcr = UcrDescriptor; + Current = Current->Flink; + } + + return NULL; +} + + static PHEAP_FREE_ENTRY RtlpFindAndCommitPages(PHEAP Heap, @@ -786,60 +1123,206 @@ RtlpFindAndCommitPages(PHEAP Heap, DPRINT("Updating UcrDescriptor %p, new Address %p, size %lu\n", UcrDescriptor, UcrDescriptor->Address, UcrDescriptor->Size); - /* Check if anything left in this UCR */ - if (UcrDescriptor->Size == 0) - { - /* It's fully exhausted. Take the guard entry for us */ - FreeEntry->Size++; - *Size += HEAP_ENTRY_SIZE; + /* Check if anything left in this UCR */ + if (UcrDescriptor->Size == 0) + { + /* It's fully exhausted. Take the guard entry for us */ + FreeEntry->Size++; + *Size += HEAP_ENTRY_SIZE; + + ASSERT((FreeEntry + FreeEntry->Size) == UcrDescriptor->Address); + + /* Check if this is the end of the segment */ + if(UcrDescriptor->Address == Segment->LastValidEntry) + { + FreeEntry->Flags = HEAP_ENTRY_LAST_ENTRY; + } + else + { + PHEAP_ENTRY NextEntry = UcrDescriptor->Address; + + /* We should not have a UCR right behind us */ + ASSERT((UcrDescriptor->SegmentEntry.Flink == &Segment->UCRSegmentList) + || (CONTAINING_RECORD(UcrDescriptor->SegmentEntry.Flink, HEAP_UCR_DESCRIPTOR, SegmentEntry)->Address > UcrDescriptor->Address)); + + ASSERT(NextEntry->PreviousSize == 0); + ASSERT(NextEntry == FreeEntry + FreeEntry->Size); + NextEntry->PreviousSize = FreeEntry->Size; + } + + /* This UCR needs to be removed because it became useless */ + RemoveEntryList(&UcrDescriptor->SegmentEntry); + + RtlpDestroyUnCommittedRange(Segment, UcrDescriptor); + Segment->NumberOfUnCommittedRanges--; + } + else + { + /* Setup a guard entry */ + GuardEntry = (PHEAP_ENTRY)UcrDescriptor->Address - 1; + ASSERT(GuardEntry == FreeEntry + FreeEntry->Size); + GuardEntry->Flags = HEAP_ENTRY_LAST_ENTRY | HEAP_ENTRY_BUSY; + GuardEntry->Size = 1; + GuardEntry->PreviousSize = FreeEntry->Size; + GuardEntry->SegmentOffset = FreeEntry->SegmentOffset; + DPRINT("Setting %p as UCR guard entry.\n", GuardEntry); + } + + /* We're done */ + return (PHEAP_FREE_ENTRY)FreeEntry; + } + + /* Advance to the next descriptor */ + Current = Current->Flink; + } + + return NULL; +} + + +VOID NTAPI +RtlpDeCommitFreeBlock1(PHEAP Heap, + PHEAP_FREE_ENTRY FreeEntry, + SIZE_T Size) +{ + PHEAP_SEGMENT Segment; + PHEAP_ENTRY PrecedingInUseEntry = NULL, NextInUseEntry = NULL; + PHEAP_FREE_ENTRY NextFreeEntry; + PHEAP_UCR_DESCRIPTOR UcrDescriptor; + SIZE_T PrecedingSize, NextSize, DecommitSize; + ULONG_PTR DecommitBase; + NTSTATUS Status; + + DPRINT("Decommitting %p %p %x\n", Heap, FreeEntry, Size); + + /* We can't decommit if there is a commit routine! */ + if (Heap->CommitRoutine) + { + /* Just add it back the usual way */ + RtlpInsertFreeBlock(Heap, FreeEntry, Size); + return; + } + + /* Get the segment */ + Segment = Heap->Segments[FreeEntry->SegmentOffset]; + + /* Get the preceding entry */ + DecommitBase = ROUND_UP(FreeEntry, PAGE_SIZE); + PrecedingSize = (PHEAP_ENTRY)DecommitBase - (PHEAP_ENTRY)FreeEntry; + + if (PrecedingSize == 1) + { + /* Just 1 heap entry, increase the base/size */ + DecommitBase += PAGE_SIZE; + PrecedingSize += PAGE_SIZE >> HEAP_ENTRY_SHIFT; + } + else if (FreeEntry->PreviousSize && + (DecommitBase == (ULONG_PTR)FreeEntry)) + { + PrecedingInUseEntry = (PHEAP_ENTRY)FreeEntry - FreeEntry->PreviousSize; + } + + /* Get the next entry */ + NextFreeEntry = (PHEAP_FREE_ENTRY)((PHEAP_ENTRY)FreeEntry + Size); + DecommitSize = ROUND_DOWN(NextFreeEntry, PAGE_SIZE); + NextSize = (PHEAP_ENTRY)NextFreeEntry - (PHEAP_ENTRY)DecommitSize; + + if (NextSize == 1) + { + /* Just 1 heap entry, increase the size */ + DecommitSize -= PAGE_SIZE; + NextSize += PAGE_SIZE >> HEAP_ENTRY_SHIFT; + } + else if (NextSize == 0 && + !(FreeEntry->Flags & HEAP_ENTRY_LAST_ENTRY)) + { + NextInUseEntry = (PHEAP_ENTRY)NextFreeEntry; + } + + NextFreeEntry = (PHEAP_FREE_ENTRY)((PHEAP_ENTRY)NextFreeEntry - NextSize); + + /* Calculate real decommit size */ + if (DecommitSize > DecommitBase) + { + DecommitSize -= DecommitBase; + } + else + { + /* Nothing to decommit */ + RtlpInsertFreeBlock(Heap, FreeEntry, Size); + return; + } + + /* A decommit is necessary. Create a UCR descriptor */ + UcrDescriptor = RtlpCreateUnCommittedRange(Segment); + if (!UcrDescriptor) + { + DPRINT1("HEAP: Failed to create UCR descriptor\n"); + RtlpInsertFreeBlock(Heap, FreeEntry, PrecedingSize); + return; + } + + /* Decommit the memory */ + Status = ZwFreeVirtualMemory(NtCurrentProcess(), + (PVOID *)&DecommitBase, + &DecommitSize, + MEM_DECOMMIT); - ASSERT((FreeEntry + FreeEntry->Size) == UcrDescriptor->Address); + /* Delete that UCR. This is needed to assure there is an unused UCR entry in the list */ + RtlpDestroyUnCommittedRange(Segment, UcrDescriptor); - /* Check if this is the end of the segment */ - if(UcrDescriptor->Address == Segment->LastValidEntry) - { - FreeEntry->Flags = HEAP_ENTRY_LAST_ENTRY; - } - else - { - PHEAP_ENTRY NextEntry = UcrDescriptor->Address; + if (!NT_SUCCESS(Status)) + { + RtlpInsertFreeBlock(Heap, FreeEntry, Size); + return; + } - /* We should not have a UCR right behind us */ - ASSERT((UcrDescriptor->SegmentEntry.Flink == &Segment->UCRSegmentList) - || (CONTAINING_RECORD(UcrDescriptor->SegmentEntry.Flink, HEAP_UCR_DESCRIPTOR, SegmentEntry)->Address > UcrDescriptor->Address)); + /* Insert uncommitted pages */ + RtlpInsertUnCommittedPages(Segment, DecommitBase, DecommitSize); + Segment->NumberOfUnCommittedPages += (ULONG)(DecommitSize / PAGE_SIZE); - ASSERT(NextEntry->PreviousSize == 0); - ASSERT(NextEntry == FreeEntry + FreeEntry->Size); - NextEntry->PreviousSize = FreeEntry->Size; - } + if (PrecedingSize) + { + /* Adjust size of this free entry and insert it */ + FreeEntry->Flags = HEAP_ENTRY_LAST_ENTRY; + FreeEntry->Size = (USHORT)PrecedingSize; + Heap->TotalFreeSize += PrecedingSize; + Segment->LastEntryInSegment = FreeEntry; - /* This UCR needs to be removed because it became useless */ - RemoveEntryList(&UcrDescriptor->SegmentEntry); + /* Insert it into the free list */ + RtlpInsertFreeBlockHelper(Heap, FreeEntry, PrecedingSize, FALSE); + } + else if (PrecedingInUseEntry) + { + /* Adjust preceding in use entry */ + PrecedingInUseEntry->Flags |= HEAP_ENTRY_LAST_ENTRY; + Segment->LastEntryInSegment = PrecedingInUseEntry; + } + else if ((ULONG_PTR)Segment->LastEntryInSegment >= DecommitBase && + (ULONG_PTR)Segment->LastEntryInSegment < DecommitBase + DecommitSize) + { + /* Invalidate last entry */ + Segment->LastEntryInSegment = Segment->FirstEntry; + } - RtlpDestroyUnCommittedRange(Segment, UcrDescriptor); - Segment->NumberOfUnCommittedRanges--; - } - else - { - /* Setup a guard entry */ - GuardEntry = (PHEAP_ENTRY)UcrDescriptor->Address - 1; - ASSERT(GuardEntry == FreeEntry + FreeEntry->Size); - GuardEntry->Flags = HEAP_ENTRY_LAST_ENTRY | HEAP_ENTRY_BUSY; - GuardEntry->Size = 1; - GuardEntry->PreviousSize = FreeEntry->Size; - GuardEntry->SegmentOffset = FreeEntry->SegmentOffset; - DPRINT("Setting %p as UCR guard entry.\n", GuardEntry); - } + /* Now the next one */ + if (NextSize) + { + /* Adjust size of this free entry and insert it */ + NextFreeEntry->Flags = 0; + NextFreeEntry->PreviousSize = 0; + NextFreeEntry->SegmentOffset = Segment->Entry.SegmentOffset; + NextFreeEntry->Size = (USHORT)NextSize; - /* We're done */ - return (PHEAP_FREE_ENTRY)FreeEntry; - } + ((PHEAP_FREE_ENTRY)((PHEAP_ENTRY)NextFreeEntry + NextSize))->PreviousSize = (USHORT)NextSize; - /* Advance to the next descriptor */ - Current = Current->Flink; + Heap->TotalFreeSize += NextSize; + RtlpInsertFreeBlockHelper(Heap, NextFreeEntry, NextSize, FALSE); + } + else if (NextInUseEntry) + { + NextInUseEntry->PreviousSize = 0; } - - return NULL; } static @@ -1010,6 +1493,83 @@ RtlpDeCommitFreeBlock(PHEAP Heap, } } +NTSTATUS +NTAPI +RtlpInitializeHeapSegment1(IN OUT PHEAP Heap, + OUT PHEAP_SEGMENT Segment, + IN UCHAR SegmentIndex, + IN ULONG SegmentFlags, + IN SIZE_T SegmentReserve, + IN SIZE_T SegmentCommit) +{ + PHEAP_ENTRY HeapEntry; + + /* Preconditions */ + ASSERT(Heap != NULL); + ASSERT(Segment != NULL); + ASSERT(SegmentCommit >= PAGE_SIZE); + ASSERT(ROUND_DOWN(SegmentCommit, PAGE_SIZE) == SegmentCommit); + ASSERT(SegmentReserve >= SegmentCommit); + ASSERT(ROUND_DOWN(SegmentReserve, PAGE_SIZE) == SegmentReserve); + + DPRINT1("RtlpInitializeHeapSegment1(%p %p %x %x %lx %lx)\n", Heap, Segment, SegmentIndex, SegmentFlags, SegmentReserve, SegmentCommit); + + /* Initialise the Heap Entry header if this is not the first Heap Segment */ + if ((PHEAP_SEGMENT) (Heap) != Segment) + { + Segment->Entry.Size = ROUND_UP(sizeof(HEAP_SEGMENT), sizeof(HEAP_ENTRY)) >> HEAP_ENTRY_SHIFT; + Segment->Entry.Flags = HEAP_ENTRY_BUSY; + Segment->Entry.SmallTagIndex = LOBYTE(Segment->Entry.Size) ^ HIBYTE(Segment->Entry.Size) ^ Segment->Entry.Flags; + Segment->Entry.PreviousSize = 0; + Segment->Entry.SegmentOffset = SegmentIndex; + Segment->Entry.UnusedBytes = 0; + } + + /* Sanity check */ + ASSERT((Segment->Entry.Size << HEAP_ENTRY_SHIFT) <= PAGE_SIZE); + + /* Initialise the Heap Segment header */ + Segment->SegmentSignature = HEAP_SEGMENT_SIGNATURE; + Segment->SegmentFlags = SegmentFlags; + Segment->Heap = Heap; + Heap->Segments[SegmentIndex] = Segment; + + /* Initialise the Heap Segment location information */ + Segment->BaseAddress = Segment; + Segment->NumberOfPages = (ULONG)(SegmentReserve >> PAGE_SHIFT); + + /* Initialise the Heap Entries contained within the Heap Segment */ + Segment->FirstEntry = &Segment->Entry + Segment->Entry.Size; + Segment->LastValidEntry = (PHEAP_ENTRY)((ULONG_PTR)Segment + SegmentReserve); + + if (((SIZE_T)Segment->Entry.Size << HEAP_ENTRY_SHIFT) < SegmentCommit) + { + HeapEntry = Segment->FirstEntry; + + /* Prepare a Free Heap Entry header */ + HeapEntry->Flags = HEAP_ENTRY_LAST_ENTRY; + HeapEntry->PreviousSize = Segment->Entry.Size; + HeapEntry->SegmentOffset = SegmentIndex; + + /* Register the Free Heap Entry */ + RtlpInsertFreeBlock1(Heap, (PHEAP_FREE_ENTRY) HeapEntry, (SegmentCommit >> HEAP_ENTRY_SHIFT) - Segment->Entry.Size); + } + + /* Always point to a valid entry */ + Segment->LastEntryInSegment = Segment->FirstEntry; + + /* Initialise the Heap Segment UnCommitted Range information */ + Segment->NumberOfUnCommittedPages = (ULONG)((SegmentReserve - SegmentCommit) >> PAGE_SHIFT); + Segment->NumberOfUnCommittedRanges = 0; + InitializeListHead(&Segment->UCRSegmentList); + + /* Register the UnCommitted Range of the Heap Segment */ + if (Segment->NumberOfUnCommittedPages != 0) + RtlpInsertUnCommittedPages(Segment, (ULONG_PTR) (Segment) + SegmentCommit, SegmentReserve - SegmentCommit); + + return STATUS_SUCCESS; +} + NTSTATUS NTAPI RtlpInitializeHeapSegment(IN OUT PHEAP Heap, @@ -1173,6 +1733,7 @@ RtlpCoalesceFreeBlocks (PHEAP Heap, { PHEAP_FREE_ENTRY CurrentEntry, NextEntry; UCHAR SegmentOffset; + ULONG ClassFlags = Heap->ClassFlags & HEAP_CLASS_MASK; /* Get the previous entry */ CurrentEntry = (PHEAP_FREE_ENTRY)((PHEAP_ENTRY)FreeEntry - FreeEntry->PreviousSize); @@ -1187,7 +1748,10 @@ RtlpCoalesceFreeBlocks (PHEAP Heap, /* Remove it if asked for */ if (Remove) { - RtlpRemoveFreeBlock(Heap, FreeEntry, FALSE); + if (ClassFlags & HEAP_CLASS_1) // Private Heap + RtlpRemoveFreeBlock1(Heap, FreeEntry, FALSE, FALSE); + else + RtlpRemoveFreeBlock(Heap, FreeEntry, FALSE); Heap->TotalFreeSize -= FreeEntry->Size; /* Remove it only once! */ @@ -1195,7 +1759,10 @@ RtlpCoalesceFreeBlocks (PHEAP Heap, } /* Remove previous entry too */ - RtlpRemoveFreeBlock(Heap, CurrentEntry, FALSE); + if (ClassFlags & HEAP_CLASS_1) // Private Heap + RtlpRemoveFreeBlock1(Heap, CurrentEntry, FALSE, FALSE); + else + RtlpRemoveFreeBlock(Heap, CurrentEntry, FALSE); /* Copy flags */ CurrentEntry->Flags = FreeEntry->Flags & HEAP_ENTRY_LAST_ENTRY; @@ -1215,6 +1782,8 @@ RtlpCoalesceFreeBlocks (PHEAP Heap, { SegmentOffset = FreeEntry->SegmentOffset; ASSERT(SegmentOffset < HEAP_SEGMENTS); + if (ClassFlags & HEAP_CLASS_1) // Private Heap + Heap->Segments[SegmentOffset]->LastEntryInSegment = FreeEntry; } } @@ -1231,7 +1800,10 @@ RtlpCoalesceFreeBlocks (PHEAP Heap, /* Remove it if asked for */ if (Remove) { - RtlpRemoveFreeBlock(Heap, FreeEntry, FALSE); + if (ClassFlags & HEAP_CLASS_1) // Private Heap + RtlpRemoveFreeBlock1(Heap, FreeEntry, FALSE, FALSE); + else + RtlpRemoveFreeBlock(Heap, FreeEntry, FALSE); Heap->TotalFreeSize -= FreeEntry->Size; } @@ -1239,7 +1811,10 @@ RtlpCoalesceFreeBlocks (PHEAP Heap, FreeEntry->Flags = NextEntry->Flags & HEAP_ENTRY_LAST_ENTRY; /* Remove next entry now */ - RtlpRemoveFreeBlock(Heap, NextEntry, FALSE); + if (ClassFlags & HEAP_CLASS_1) // Private Heap + RtlpRemoveFreeBlock1(Heap, NextEntry, FALSE, FALSE); + else + RtlpRemoveFreeBlock(Heap, NextEntry, FALSE); /* Update sizes */ *FreeSize = *FreeSize + NextEntry->Size; @@ -1255,6 +1830,8 @@ RtlpCoalesceFreeBlocks (PHEAP Heap, { SegmentOffset = FreeEntry->SegmentOffset; ASSERT(SegmentOffset < HEAP_SEGMENTS); + if (ClassFlags & HEAP_CLASS_1) // Private Heap + Heap->Segments[SegmentOffset]->LastEntryInSegment = FreeEntry; } } } @@ -1272,6 +1849,7 @@ RtlpExtendHeap(PHEAP Heap, PHEAP_SEGMENT Segment; PHEAP_FREE_ENTRY FreeEntry; NTSTATUS Status; + ULONG ClassFlags = Heap->ClassFlags & HEAP_CLASS_MASK; DPRINT("RtlpExtendHeap(%p %x)\n", Heap, Size); @@ -1295,7 +1873,10 @@ RtlpExtendHeap(PHEAP Heap, DPRINT("This segment is suitable\n"); /* Commit needed amount */ - FreeEntry = RtlpFindAndCommitPages(Heap, Segment, &FreeSize, NULL); + if (ClassFlags & HEAP_CLASS_1) // Private Heap + FreeEntry = RtlpFindAndCommitPages1(Heap, Segment, &FreeSize, NULL); + else + FreeEntry = RtlpFindAndCommitPages(Heap, Segment, &FreeSize, NULL); /* Coalesce it with adjacent entries */ if (FreeEntry) @@ -1527,11 +2108,14 @@ RtlCreateHeap(ULONG Flags, Parameters->VirtualMemoryThreshold = MaxBlockSize; } - if (Parameters->DeCommitFreeBlockThreshold != PAGE_SIZE) + if (Flags == HEAP_CLASS_1) // Private Heap Not GROWABLE { - DPRINT1("WARNING: Ignoring DeCommitFreeBlockThreshold %lx, setting it to PAGE_SIZE.\n", - Parameters->DeCommitFreeBlockThreshold); - Parameters->DeCommitFreeBlockThreshold = PAGE_SIZE; + if (Parameters->DeCommitFreeBlockThreshold != PAGE_SIZE) + { + DPRINT1("WARNING: Ignoring DeCommitFreeBlockThreshold %lx, setting it to PAGE_SIZE.\n", + Parameters->DeCommitFreeBlockThreshold); + Parameters->DeCommitFreeBlockThreshold = PAGE_SIZE; + } } /* Check reserve/commit sizes and set default values */ @@ -1711,8 +2295,17 @@ RtlCreateHeap(ULONG Flags, return NULL; } + /* Heap->ClassFlags must be set before calling RtlpInitializeHeapSegment's */ + if (Flags == HEAP_CLASS_1) // Private Heap Not GROWABLE + Heap->ClassFlags = HEAP_CLASS_1; + else + Heap->ClassFlags = 0; + /* Initialize heap's first segment */ - Status = RtlpInitializeHeapSegment(Heap, (PHEAP_SEGMENT) (Heap), 0, HeapSegmentFlags, TotalSize, CommitSize); + if (Flags == HEAP_CLASS_1) // Private Heap Not GROWABLE + Status = RtlpInitializeHeapSegment1(Heap, (PHEAP_SEGMENT) (Heap), 0, HeapSegmentFlags, TotalSize, CommitSize); + else + Status = RtlpInitializeHeapSegment(Heap, (PHEAP_SEGMENT) (Heap), 0, HeapSegmentFlags, TotalSize, CommitSize); if (!NT_SUCCESS(Status)) { DPRINT1("Failed to initialize heap segment (%x)\n", Status); @@ -1923,7 +2516,10 @@ RtlpSplitEntry(PHEAP Heap, SplitBlock->Flags = SplitBlock2->Flags; /* Remove that next entry */ - RtlpRemoveFreeBlock(Heap, SplitBlock2, FALSE); + if (Heap->ClassFlags == HEAP_CLASS_1) // Private Heap Not GROWABLE + RtlpRemoveFreeBlock1(Heap, SplitBlock2, FALSE, FALSE); + else + RtlpRemoveFreeBlock(Heap, SplitBlock2, FALSE); /* Update sizes */ FreeSize += SplitBlock2->Size; @@ -1960,6 +2556,8 @@ RtlpSplitEntry(PHEAP Heap, { SegmentOffset = SplitBlock->SegmentOffset; ASSERT(SegmentOffset < HEAP_SEGMENTS); + if (Heap->ClassFlags == HEAP_CLASS_1) // Private Heap Not GROWABLE + Heap->Segments[SegmentOffset]->LastEntryInSegment = SplitBlock; } } } @@ -1980,75 +2578,146 @@ RtlpAllocateNonDedicated(PHEAP Heap, SIZE_T Index, BOOLEAN HeapLocked) { + PLIST_ENTRY FreeListHead, Next; PHEAP_FREE_ENTRY FreeBlock; + PHEAP_ENTRY InUseEntry; + PHEAP_ENTRY_EXTRA Extra; - /* The entries in the list must be too small for us */ - ASSERT(IsListEmpty(&Heap->FreeLists) || - (CONTAINING_RECORD(Heap->FreeLists.Blink, HEAP_FREE_ENTRY, FreeList)->Size < Index)); + if (Heap->ClassFlags == HEAP_CLASS_1) // Private Heap Not GROWABLE + { + /* Go through the zero list to find a place where to insert the new entry */ + FreeListHead = &Heap->FreeList[0]; - /* Extend the heap */ - FreeBlock = RtlpExtendHeap(Heap, AllocationSize); + /* Start from the largest block to reduce time */ + Next = FreeListHead->Blink; + if (FreeListHead != Next) + { + FreeBlock = CONTAINING_RECORD(Next, HEAP_FREE_ENTRY, FreeList); + if (FreeBlock->Size >= Index) + { + /* Our request is smaller than the largest entry in the zero list */ - /* Use the new biggest entry we've got */ - if (FreeBlock) - { - PHEAP_ENTRY InUseEntry; - PHEAP_ENTRY_EXTRA Extra; + /* Go through the list to find insertion place */ + Next = FreeListHead->Flink; + while (FreeListHead != Next) + { + FreeBlock = CONTAINING_RECORD(Next, HEAP_FREE_ENTRY, FreeList); + if (FreeBlock->Size >= Index) + { + /* Found minimally fitting entry. Proceed to either using it as it is + or splitting it to two entries */ + RemoveEntryList(&FreeBlock->FreeList); - RtlpRemoveFreeBlock(Heap, FreeBlock, TRUE); + /* Split it */ + InUseEntry = RtlpSplitEntry(Heap, Flags, FreeBlock, AllocationSize, Index, Size); - /* Split it */ - InUseEntry = RtlpSplitEntry(Heap, Flags, FreeBlock, AllocationSize, Index, Size); + /* Release the lock */ + if (HeapLocked) RtlLeaveHeapLock(Heap->LockVariable); - /* Release the lock */ - if (HeapLocked) RtlLeaveHeapLock(Heap->LockVariable); + /* Zero memory if that was requested */ + if (Flags & HEAP_ZERO_MEMORY) + RtlZeroMemory(InUseEntry + 1, Size); + else if (Heap->Flags & HEAP_FREE_CHECKING_ENABLED) + { + /* Fill this block with a special pattern */ + RtlFillMemoryUlong(InUseEntry + 1, Size & ~0x3, ARENA_INUSE_FILLER); + } - /* Zero memory if that was requested */ - if (Flags & HEAP_ZERO_MEMORY) - RtlZeroMemory(InUseEntry + 1, Size); - else if (Heap->Flags & HEAP_FREE_CHECKING_ENABLED) - { - /* Fill this block with a special pattern */ - RtlFillMemoryUlong(InUseEntry + 1, Size & ~0x3, ARENA_INUSE_FILLER); - } + /* Fill tail of the block with a special pattern too if requested */ + if (Heap->Flags & HEAP_TAIL_CHECKING_ENABLED) + { + RtlFillMemory((PCHAR)(InUseEntry + 1) + Size, sizeof(HEAP_ENTRY), HEAP_TAIL_FILL); + InUseEntry->Flags |= HEAP_ENTRY_FILL_PATTERN; + } - /* Fill tail of the block with a special pattern too if requested */ - if (Heap->Flags & HEAP_TAIL_CHECKING_ENABLED) - { - RtlFillMemory((PCHAR)(InUseEntry + 1) + Size, sizeof(HEAP_ENTRY), HEAP_TAIL_FILL); - InUseEntry->Flags |= HEAP_ENTRY_FILL_PATTERN; + /* Prepare extra if it's present */ + if (InUseEntry->Flags & HEAP_ENTRY_EXTRA_PRESENT) + { + Extra = RtlpGetExtraStuffPointer(InUseEntry); + RtlZeroMemory(Extra, sizeof(HEAP_ENTRY_EXTRA)); + + // TODO: Tagging + } + + /* Return pointer to the free entry */ + return InUseEntry + 1; + } + + /* Advance to the next entry */ + Next = Next->Flink; + } + } } + } + else + { + /* The entries in the list must be too small for us */ + ASSERT(IsListEmpty(&Heap->FreeLists) || + (CONTAINING_RECORD(Heap->FreeLists.Blink, HEAP_FREE_ENTRY, FreeList)->Size < Index)); - /* Prepare extra if it's present */ - if (InUseEntry->Flags & HEAP_ENTRY_EXTRA_PRESENT) + /* Extend the heap */ + FreeBlock = RtlpExtendHeap(Heap, AllocationSize); + + /* Use the new biggest entry we've got */ + if (FreeBlock) { - Extra = RtlpGetExtraStuffPointer(InUseEntry); - RtlZeroMemory(Extra, sizeof(HEAP_ENTRY_EXTRA)); + PHEAP_ENTRY InUseEntry; + PHEAP_ENTRY_EXTRA Extra; - // TODO: Tagging - } + RtlpRemoveFreeBlock(Heap, FreeBlock, TRUE); - /* Return pointer to the */ - return InUseEntry + 1; - } + /* Split it */ + InUseEntry = RtlpSplitEntry(Heap, Flags, FreeBlock, AllocationSize, Index, Size); - /* Really unfortunate, out of memory condition */ - RtlSetLastWin32ErrorAndNtStatusFromNtStatus(STATUS_NO_MEMORY); + /* Release the lock */ + if (HeapLocked) RtlLeaveHeapLock(Heap->LockVariable); - /* Generate an exception */ - if (Flags & HEAP_GENERATE_EXCEPTIONS) - { - EXCEPTION_RECORD ExceptionRecord; + /* Zero memory if that was requested */ + if (Flags & HEAP_ZERO_MEMORY) + RtlZeroMemory(InUseEntry + 1, Size); + else if (Heap->Flags & HEAP_FREE_CHECKING_ENABLED) + { + /* Fill this block with a special pattern */ + RtlFillMemoryUlong(InUseEntry + 1, Size & ~0x3, ARENA_INUSE_FILLER); + } - ExceptionRecord.ExceptionCode = STATUS_NO_MEMORY; - ExceptionRecord.ExceptionRecord = NULL; - ExceptionRecord.NumberParameters = 1; - ExceptionRecord.ExceptionFlags = 0; - ExceptionRecord.ExceptionInformation[0] = AllocationSize; + /* Fill tail of the block with a special pattern too if requested */ + if (Heap->Flags & HEAP_TAIL_CHECKING_ENABLED) + { + RtlFillMemory((PCHAR)(InUseEntry + 1) + Size, sizeof(HEAP_ENTRY), HEAP_TAIL_FILL); + InUseEntry->Flags |= HEAP_ENTRY_FILL_PATTERN; + } - RtlRaiseException(&ExceptionRecord); - } + /* Prepare extra if it's present */ + if (InUseEntry->Flags & HEAP_ENTRY_EXTRA_PRESENT) + { + Extra = RtlpGetExtraStuffPointer(InUseEntry); + RtlZeroMemory(Extra, sizeof(HEAP_ENTRY_EXTRA)); + + // TODO: Tagging + } + + /* Return pointer to the free entry */ + return InUseEntry + 1; + } + + /* Really unfortunate, out of memory condition */ + RtlSetLastWin32ErrorAndNtStatusFromNtStatus(STATUS_NO_MEMORY); + + /* Generate an exception */ + if (Flags & HEAP_GENERATE_EXCEPTIONS) + { + EXCEPTION_RECORD ExceptionRecord; + + ExceptionRecord.ExceptionCode = STATUS_NO_MEMORY; + ExceptionRecord.ExceptionRecord = NULL; + ExceptionRecord.NumberParameters = 1; + ExceptionRecord.ExceptionFlags = 0; + ExceptionRecord.ExceptionInformation[0] = AllocationSize; + RtlRaiseException(&ExceptionRecord); + } + } /* Release the lock */ if (HeapLocked) RtlLeaveHeapLock(Heap->LockVariable); DPRINT1("HEAP: Allocation failed!\n"); @@ -2070,9 +2739,15 @@ RtlAllocateHeap(IN PVOID HeapPtr, IN SIZE_T Size) { PHEAP Heap = (PHEAP)HeapPtr; + PULONG FreeListsInUse; + ULONG FreeListsInUseUlong; SIZE_T AllocationSize; - SIZE_T Index; - UCHAR EntryFlags = HEAP_ENTRY_BUSY; + SIZE_T Index, InUseIndex, i; + PLIST_ENTRY FreeListHead; + PHEAP_ENTRY InUseEntry; + PHEAP_FREE_ENTRY FreeBlock; + UCHAR FreeFlags, EntryFlags = HEAP_ENTRY_BUSY; + EXCEPTION_RECORD ExceptionRecord; BOOLEAN HeapLocked = FALSE; PHEAP_VIRTUAL_ALLOC_ENTRY VirtualBlock = NULL; PHEAP_ENTRY_EXTRA Extra; @@ -2132,142 +2807,306 @@ RtlAllocateHeap(IN PVOID HeapPtr, HeapLocked = TRUE; } - /* Depending on the size, the allocation is going to be done from dedicated, - non-dedicated lists or a virtual block of memory */ - if (Index <= Heap->VirtualMemoryThreshold) - { - PHEAP_ENTRY InUseEntry; - PHEAP_FREE_ENTRY FreeEntry; + if (Heap->ClassFlags == HEAP_CLASS_1) // Private Heap Not GROWABLE + { + if (Index < HEAP_FREELISTS) + { + FreeListHead = &Heap->FreeList[Index]; + + if (!IsListEmpty(FreeListHead)) + { + /* There is a free entry in this list */ + FreeBlock = CONTAINING_RECORD(FreeListHead->Blink, + HEAP_FREE_ENTRY, + FreeList); + + /* Save flags and remove the free entry */ + FreeFlags = FreeBlock->Flags; + RtlpRemoveFreeBlock1(Heap, FreeBlock, TRUE, FALSE); + + /* Update the total free size of the heap */ + Heap->TotalFreeSize -= Index; + + /* Initialize this block */ + InUseEntry = (PHEAP_ENTRY)FreeBlock; + InUseEntry->Flags = EntryFlags | (FreeFlags & HEAP_ENTRY_LAST_ENTRY); + InUseEntry->UnusedBytes = (UCHAR)(AllocationSize - Size); + InUseEntry->SmallTagIndex = 0; + } + else + { + /* Find smallest free block which this request could fit in */ + InUseIndex = Index >> 5; + FreeListsInUse = &Heap->u.FreeListsInUseUlong[InUseIndex]; + + /* This bit magic disables all sizes which are less than the requested allocation size */ + FreeListsInUseUlong = *FreeListsInUse++ & ~((1 << ((ULONG)Index & 0x1f)) - 1); + + /* If size is definitily more than our lists - go directly to the non-dedicated one */ + if (InUseIndex > 3) + return RtlpAllocateNonDedicated(Heap, Flags, Size, AllocationSize, Index, HeapLocked); + + /* Go through the list */ + for (i = InUseIndex; i < 4; i++) + { + if (FreeListsInUseUlong) + { + FreeListHead = &Heap->FreeList[i * 32]; + break; + } + + if (i < 3) FreeListsInUseUlong = *FreeListsInUse++; + } + + /* Nothing found, search in the non-dedicated list */ + if (i == 4) + return RtlpAllocateNonDedicated(Heap, Flags, Size, AllocationSize, Index, HeapLocked); + + /* That list is found, now calculate exact block */ + FreeListHead += RtlpFindLeastSetBit(FreeListsInUseUlong); + + /* Take this entry and remove it from the list of free blocks */ + FreeBlock = CONTAINING_RECORD(FreeListHead->Blink, + HEAP_FREE_ENTRY, + FreeList); + RtlpRemoveFreeBlock1(Heap, FreeBlock, TRUE, FALSE); + + /* Split it */ + InUseEntry = RtlpSplitEntry(Heap, Flags, FreeBlock, AllocationSize, Index, Size); + } + + /* Release the lock */ + if (HeapLocked) RtlLeaveHeapLock(Heap->LockVariable); + + /* Zero memory if that was requested */ + if (Flags & HEAP_ZERO_MEMORY) + RtlZeroMemory(InUseEntry + 1, Size); + else if (Heap->Flags & HEAP_FREE_CHECKING_ENABLED) + { + /* Fill this block with a special pattern */ + RtlFillMemoryUlong(InUseEntry + 1, Size & ~0x3, ARENA_INUSE_FILLER); + } + + /* Fill tail of the block with a special pattern too if requested */ + if (Heap->Flags & HEAP_TAIL_CHECKING_ENABLED) + { + RtlFillMemory((PCHAR)(InUseEntry + 1) + Size, sizeof(HEAP_ENTRY), HEAP_TAIL_FILL); + InUseEntry->Flags |= HEAP_ENTRY_FILL_PATTERN; + } - /* First quick check: Anybody here ? */ - if (IsListEmpty(&Heap->FreeLists)) - return RtlpAllocateNonDedicated(Heap, Flags, Size, AllocationSize, Index, HeapLocked); + /* Prepare extra if it's present */ + if (InUseEntry->Flags & HEAP_ENTRY_EXTRA_PRESENT) + { + Extra = RtlpGetExtraStuffPointer(InUseEntry); + RtlZeroMemory(Extra, sizeof(HEAP_ENTRY_EXTRA)); + + // TODO: Tagging + } - /* Second quick check: Is there someone for us ? */ - FreeEntry = CONTAINING_RECORD(Heap->FreeLists.Blink, HEAP_FREE_ENTRY, FreeList); - if (FreeEntry->Size < Index) + /* User data starts right after the entry's header */ + return InUseEntry + 1; + } + else if (Index <= Heap->VirtualMemoryThreshold) { - /* Largest entry in the list doesnt fit. */ + /* The block is too large for dedicated lists, but fine for a non-dedicated one */ return RtlpAllocateNonDedicated(Heap, Flags, Size, AllocationSize, Index, HeapLocked); } - - if (Index > Heap->DeCommitFreeBlockThreshold) + else if (Heap->Flags & HEAP_GROWABLE) { - /* Find an entry from the non dedicated list */ - FreeEntry = CONTAINING_RECORD(Heap->FreeHints[0], - HEAP_FREE_ENTRY, - FreeList); + /* We've got a very big allocation request, satisfy it by directly allocating virtual memory */ + AllocationSize += sizeof(HEAP_VIRTUAL_ALLOC_ENTRY) - sizeof(HEAP_ENTRY); + + Status = ZwAllocateVirtualMemory(NtCurrentProcess(), + (PVOID *)&VirtualBlock, + 0, + &AllocationSize, + MEM_COMMIT, + PAGE_READWRITE); - while (FreeEntry->Size < Index) + if (!NT_SUCCESS(Status)) { - /* We made sure we had the right size available */ - ASSERT(FreeEntry->FreeList.Flink != &Heap->FreeLists); - FreeEntry = CONTAINING_RECORD(FreeEntry->FreeList.Flink, - HEAP_FREE_ENTRY, - FreeList); + // Set STATUS! + /* Release the lock */ + if (HeapLocked) RtlLeaveHeapLock(Heap->LockVariable); + DPRINT1("HEAP: Allocation failed!\n"); + return NULL; } + + /* Initialize the newly allocated block */ + VirtualBlock->BusyBlock.Size = (USHORT)(AllocationSize - Size); + ASSERT(VirtualBlock->BusyBlock.Size >= sizeof(HEAP_VIRTUAL_ALLOC_ENTRY)); + VirtualBlock->BusyBlock.Flags = EntryFlags | HEAP_ENTRY_VIRTUAL_ALLOC | HEAP_ENTRY_EXTRA_PRESENT; + VirtualBlock->CommitSize = AllocationSize; + VirtualBlock->ReserveSize = AllocationSize; + + /* Insert it into the list of virtual allocations */ + InsertTailList(&Heap->VirtualAllocdBlocks, &VirtualBlock->Entry); + + /* Release the lock */ + if (HeapLocked) RtlLeaveHeapLock(Heap->LockVariable); + + /* Return pointer to user data */ + return VirtualBlock + 1; } - else + + /* Generate an exception */ + if (Flags & HEAP_GENERATE_EXCEPTIONS) { - /* Get the free entry from the hint */ - ULONG HintIndex = RtlFindSetBits(&Heap->FreeHintBitmap, 1, Index - 1); - ASSERT(HintIndex != 0xFFFFFFFF); - ASSERT((HintIndex >= (Index - 1)) || (HintIndex == 0)); - FreeEntry = CONTAINING_RECORD(Heap->FreeHints[HintIndex], - HEAP_FREE_ENTRY, - FreeList); + ExceptionRecord.ExceptionCode = STATUS_NO_MEMORY; + ExceptionRecord.ExceptionRecord = NULL; + ExceptionRecord.NumberParameters = 1; + ExceptionRecord.ExceptionFlags = 0; + ExceptionRecord.ExceptionInformation[0] = AllocationSize; + + RtlRaiseException(&ExceptionRecord); } - /* Remove the free block, split, profit. */ - RtlpRemoveFreeBlock(Heap, FreeEntry, FALSE); - InUseEntry = RtlpSplitEntry(Heap, Flags, FreeEntry, AllocationSize, Index, Size); + RtlSetLastWin32ErrorAndNtStatusFromNtStatus(STATUS_BUFFER_TOO_SMALL); /* Release the lock */ if (HeapLocked) RtlLeaveHeapLock(Heap->LockVariable); + DPRINT1("HEAP: Allocation failed!\n"); + return NULL; - /* Zero memory if that was requested */ - if (Flags & HEAP_ZERO_MEMORY) - RtlZeroMemory(InUseEntry + 1, Size); - else if (Heap->Flags & HEAP_FREE_CHECKING_ENABLED) - { - /* Fill this block with a special pattern */ - RtlFillMemoryUlong(InUseEntry + 1, Size & ~0x3, ARENA_INUSE_FILLER); - } - - /* Fill tail of the block with a special pattern too if requested */ - if (Heap->Flags & HEAP_TAIL_CHECKING_ENABLED) + } + else + { + /* Depending on the size, the allocation is going to be done from dedicated, + non-dedicated lists or a virtual block of memory */ + if (Index <= Heap->VirtualMemoryThreshold) { - RtlFillMemory((PCHAR)(InUseEntry + 1) + Size, sizeof(HEAP_ENTRY), HEAP_TAIL_FILL); - InUseEntry->Flags |= HEAP_ENTRY_FILL_PATTERN; - } + PHEAP_ENTRY InUseEntry; + PHEAP_FREE_ENTRY FreeEntry; - /* Prepare extra if it's present */ - if (InUseEntry->Flags & HEAP_ENTRY_EXTRA_PRESENT) - { - Extra = RtlpGetExtraStuffPointer(InUseEntry); - RtlZeroMemory(Extra, sizeof(HEAP_ENTRY_EXTRA)); + /* First quick check: Anybody here ? */ + if (IsListEmpty(&Heap->FreeLists)) + return RtlpAllocateNonDedicated(Heap, Flags, Size, AllocationSize, Index, HeapLocked); - // TODO: Tagging - } + /* Second quick check: Is there someone for us ? */ + FreeEntry = CONTAINING_RECORD(Heap->FreeLists.Blink, HEAP_FREE_ENTRY, FreeList); + if (FreeEntry->Size < Index) + { + /* Largest entry in the list doesnt fit. */ + return RtlpAllocateNonDedicated(Heap, Flags, Size, AllocationSize, Index, HeapLocked); + } - /* User data starts right after the entry's header */ - return InUseEntry + 1; - } + if (Index > Heap->DeCommitFreeBlockThreshold) + { + /* Find an entry from the non dedicated list */ + FreeEntry = CONTAINING_RECORD(Heap->FreeHints[0], + HEAP_FREE_ENTRY, + FreeList); - if (Heap->Flags & HEAP_GROWABLE) - { - /* We've got a very big allocation request, satisfy it by directly allocating virtual memory */ - AllocationSize += sizeof(HEAP_VIRTUAL_ALLOC_ENTRY) - sizeof(HEAP_ENTRY); + while (FreeEntry->Size < Index) + { + /* We made sure we had the right size available */ + ASSERT(FreeEntry->FreeList.Flink != &Heap->FreeLists); + FreeEntry = CONTAINING_RECORD(FreeEntry->FreeList.Flink, + HEAP_FREE_ENTRY, + FreeList); + } + } + else + { + /* Get the free entry from the hint */ + ULONG HintIndex = RtlFindSetBits(&Heap->FreeHintBitmap, 1, Index - 1); + ASSERT(HintIndex != 0xFFFFFFFF); + ASSERT((HintIndex >= (Index - 1)) || (HintIndex == 0)); + FreeEntry = CONTAINING_RECORD(Heap->FreeHints[HintIndex], + HEAP_FREE_ENTRY, + FreeList); + } - Status = ZwAllocateVirtualMemory(NtCurrentProcess(), - (PVOID *)&VirtualBlock, - 0, - &AllocationSize, - MEM_COMMIT, - PAGE_READWRITE); + /* Remove the free block, split, profit. */ + RtlpRemoveFreeBlock(Heap, FreeEntry, FALSE); + InUseEntry = RtlpSplitEntry(Heap, Flags, FreeEntry, AllocationSize, Index, Size); - if (!NT_SUCCESS(Status)) - { - // Set STATUS! /* Release the lock */ if (HeapLocked) RtlLeaveHeapLock(Heap->LockVariable); - DPRINT1("HEAP: Allocation failed!\n"); - return NULL; + + /* Zero memory if that was requested */ + if (Flags & HEAP_ZERO_MEMORY) + RtlZeroMemory(InUseEntry + 1, Size); + else if (Heap->Flags & HEAP_FREE_CHECKING_ENABLED) + { + /* Fill this block with a special pattern */ + RtlFillMemoryUlong(InUseEntry + 1, Size & ~0x3, ARENA_INUSE_FILLER); + } + + /* Fill tail of the block with a special pattern too if requested */ + if (Heap->Flags & HEAP_TAIL_CHECKING_ENABLED) + { + RtlFillMemory((PCHAR)(InUseEntry + 1) + Size, sizeof(HEAP_ENTRY), HEAP_TAIL_FILL); + InUseEntry->Flags |= HEAP_ENTRY_FILL_PATTERN; + } + + /* Prepare extra if it's present */ + if (InUseEntry->Flags & HEAP_ENTRY_EXTRA_PRESENT) + { + Extra = RtlpGetExtraStuffPointer(InUseEntry); + RtlZeroMemory(Extra, sizeof(HEAP_ENTRY_EXTRA)); + + // TODO: Tagging + } + + /* User data starts right after the entry's header */ + return InUseEntry + 1; } - /* Initialize the newly allocated block */ - VirtualBlock->BusyBlock.Size = (USHORT)(AllocationSize - Size); - ASSERT(VirtualBlock->BusyBlock.Size >= sizeof(HEAP_VIRTUAL_ALLOC_ENTRY)); - VirtualBlock->BusyBlock.Flags = EntryFlags | HEAP_ENTRY_VIRTUAL_ALLOC | HEAP_ENTRY_EXTRA_PRESENT; - VirtualBlock->CommitSize = AllocationSize; - VirtualBlock->ReserveSize = AllocationSize; + if (Heap->Flags & HEAP_GROWABLE) + { + /* We've got a very big allocation request, satisfy it by directly allocating virtual memory */ + AllocationSize += sizeof(HEAP_VIRTUAL_ALLOC_ENTRY) - sizeof(HEAP_ENTRY); - /* Insert it into the list of virtual allocations */ - InsertTailList(&Heap->VirtualAllocdBlocks, &VirtualBlock->Entry); + Status = ZwAllocateVirtualMemory(NtCurrentProcess(), + (PVOID *)&VirtualBlock, + 0, + &AllocationSize, + MEM_COMMIT, + PAGE_READWRITE); - /* Release the lock */ - if (HeapLocked) RtlLeaveHeapLock(Heap->LockVariable); + if (!NT_SUCCESS(Status)) + { + // Set STATUS! + /* Release the lock */ + if (HeapLocked) RtlLeaveHeapLock(Heap->LockVariable); + DPRINT1("HEAP: Allocation failed!\n"); + return NULL; + } - /* Return pointer to user data */ - return VirtualBlock + 1; - } + /* Initialize the newly allocated block */ + VirtualBlock->BusyBlock.Size = (USHORT)(AllocationSize - Size); + ASSERT(VirtualBlock->BusyBlock.Size >= sizeof(HEAP_VIRTUAL_ALLOC_ENTRY)); + VirtualBlock->BusyBlock.Flags = EntryFlags | HEAP_ENTRY_VIRTUAL_ALLOC | HEAP_ENTRY_EXTRA_PRESENT; + VirtualBlock->CommitSize = AllocationSize; + VirtualBlock->ReserveSize = AllocationSize; - /* Generate an exception */ - if (Flags & HEAP_GENERATE_EXCEPTIONS) - { - EXCEPTION_RECORD ExceptionRecord; + /* Insert it into the list of virtual allocations */ + InsertTailList(&Heap->VirtualAllocdBlocks, &VirtualBlock->Entry); - ExceptionRecord.ExceptionCode = STATUS_NO_MEMORY; - ExceptionRecord.ExceptionRecord = NULL; - ExceptionRecord.NumberParameters = 1; - ExceptionRecord.ExceptionFlags = 0; - ExceptionRecord.ExceptionInformation[0] = AllocationSize; + /* Release the lock */ + if (HeapLocked) RtlLeaveHeapLock(Heap->LockVariable); - RtlRaiseException(&ExceptionRecord); - } + /* Return pointer to user data */ + return VirtualBlock + 1; + } + + /* Generate an exception */ + if (Flags & HEAP_GENERATE_EXCEPTIONS) + { + EXCEPTION_RECORD ExceptionRecord; - RtlSetLastWin32ErrorAndNtStatusFromNtStatus(STATUS_BUFFER_TOO_SMALL); + ExceptionRecord.ExceptionCode = STATUS_NO_MEMORY; + ExceptionRecord.ExceptionRecord = NULL; + ExceptionRecord.NumberParameters = 1; + ExceptionRecord.ExceptionFlags = 0; + ExceptionRecord.ExceptionInformation[0] = AllocationSize; + RtlRaiseException(&ExceptionRecord); + } + + RtlSetLastWin32ErrorAndNtStatusFromNtStatus(STATUS_BUFFER_TOO_SMALL); + } /* Release the lock */ if (HeapLocked) RtlLeaveHeapLock(Heap->LockVariable); DPRINT1("HEAP: Allocation failed!\n"); @@ -2366,37 +3205,101 @@ BOOLEAN NTAPI RtlFreeHeap( } else { - /* Normal allocation */ - BlockSize = HeapEntry->Size; - - // TODO: Tagging + ULONG ClassFlags = Heap->ClassFlags & HEAP_CLASS_MASK; - /* Coalesce in kernel mode, and in usermode if it's not disabled */ - if (RtlpGetMode() == KernelMode || - (RtlpGetMode() == UserMode && !(Heap->Flags & HEAP_DISABLE_COALESCE_ON_FREE))) + if (ClassFlags & HEAP_CLASS_1) { - HeapEntry = (PHEAP_ENTRY)RtlpCoalesceFreeBlocks(Heap, - (PHEAP_FREE_ENTRY)HeapEntry, - &BlockSize, - FALSE); - } + /* Normal allocation */ + BlockSize = HeapEntry->Size; - /* See if we should decommit this block */ - if ((BlockSize >= Heap->DeCommitFreeBlockThreshold) || - (Heap->TotalFreeSize + BlockSize >= Heap->DeCommitTotalFreeThreshold)) - { - RtlpDeCommitFreeBlock(Heap, (PHEAP_FREE_ENTRY)HeapEntry, BlockSize); + // TODO: Tagging + + /* Coalesce in kernel mode, and in usermode if it's not disabled */ + if (RtlpGetMode() == KernelMode || + (RtlpGetMode() == UserMode && !(Heap->Flags & HEAP_DISABLE_COALESCE_ON_FREE))) + { + HeapEntry = (PHEAP_ENTRY)RtlpCoalesceFreeBlocks(Heap, + (PHEAP_FREE_ENTRY)HeapEntry, + &BlockSize, + FALSE); + } + + /* If there is no need to decommit the block - put it into a free list */ + if (BlockSize < Heap->DeCommitFreeBlockThreshold || + (Heap->TotalFreeSize + BlockSize < Heap->DeCommitTotalFreeThreshold)) + { + /* Check if it needs to go to a 0 list */ + if (BlockSize > HEAP_MAX_BLOCK_SIZE) + { + /* General-purpose 0 list */ + RtlpInsertFreeBlock1(Heap, (PHEAP_FREE_ENTRY)HeapEntry, BlockSize); + } + else + { + /* Usual free list */ + RtlpInsertFreeBlockHelper(Heap, (PHEAP_FREE_ENTRY)HeapEntry, BlockSize, FALSE); + + /* Assert sizes are consistent */ + if (!(HeapEntry->Flags & HEAP_ENTRY_LAST_ENTRY)) + { + ASSERT((HeapEntry + BlockSize)->PreviousSize == BlockSize); + } + + /* Increase the free size */ + Heap->TotalFreeSize += BlockSize; + } + + + if (RtlpGetMode() == UserMode && + TagIndex != 0) + { + // FIXME: Tagging + UNIMPLEMENTED; + } + } + else + { + /* Decommit this block */ + RtlpDeCommitFreeBlock1(Heap, (PHEAP_FREE_ENTRY)HeapEntry, BlockSize); + } } else { - /* Insert into the free list */ - RtlpInsertFreeBlock(Heap, (PHEAP_FREE_ENTRY)HeapEntry, BlockSize); + /* Normal allocation */ + BlockSize = HeapEntry->Size; + + // TODO: Tagging + + /* Coalesce in kernel mode, and in usermode if it's not disabled */ + if (RtlpGetMode() == KernelMode || + (RtlpGetMode() == UserMode && !(Heap->Flags & HEAP_DISABLE_COALESCE_ON_FREE))) + { + HeapEntry = (PHEAP_ENTRY)RtlpCoalesceFreeBlocks(Heap, + (PHEAP_FREE_ENTRY)HeapEntry, + &BlockSize, + FALSE); + } - if (RtlpGetMode() == UserMode && - TagIndex != 0) + /* See if we should decommit this block */ + if ((BlockSize >= Heap->DeCommitFreeBlockThreshold) || + (Heap->TotalFreeSize + BlockSize >= Heap->DeCommitTotalFreeThreshold)) + { + if (ClassFlags & HEAP_CLASS_1) + RtlpDeCommitFreeBlock1(Heap, (PHEAP_FREE_ENTRY)HeapEntry, BlockSize); + else + RtlpDeCommitFreeBlock(Heap, (PHEAP_FREE_ENTRY)HeapEntry, BlockSize); + } + else { - // FIXME: Tagging - UNIMPLEMENTED; + /* Insert into the free list */ + RtlpInsertFreeBlock(Heap, (PHEAP_FREE_ENTRY)HeapEntry, BlockSize); + + if (RtlpGetMode() == UserMode && + TagIndex != 0) + { + // FIXME: Tagging + UNIMPLEMENTED; + } } } } @@ -2419,6 +3322,8 @@ RtlpGrowBlockInPlace (IN PHEAP Heap, SIZE_T FreeSize, PrevSize, TailPart, AddedSize = 0; PHEAP_ENTRY_EXTRA OldExtra, NewExtra; UCHAR SegmentOffset; + BOOLEAN EntryLast; + ULONG ClassFlags = Heap->ClassFlags & HEAP_CLASS_MASK; /* We can't grow beyond specified threshold */ if (Index > Heap->VirtualMemoryThreshold) @@ -2427,17 +3332,34 @@ RtlpGrowBlockInPlace (IN PHEAP Heap, /* Get entry flags */ EntryFlags = InUseEntry->Flags; - if (RtlpIsLastCommittedEntry(InUseEntry)) + if (ClassFlags & HEAP_CLASS_1) + { + /* Get the next free entry */ + FreeEntry = (PHEAP_FREE_ENTRY)(InUseEntry + InUseEntry->Size); + EntryLast = EntryFlags & HEAP_ENTRY_LAST_ENTRY; + } + else + { + EntryLast = RtlpIsLastCommittedEntry(InUseEntry); + } + + if (EntryLast) { /* There is no next block, just uncommitted space. Calculate how much is needed */ FreeSize = (Index - InUseEntry->Size) << HEAP_ENTRY_SHIFT; FreeSize = ROUND_UP(FreeSize, PAGE_SIZE); /* Find and commit those pages */ - FreeEntry = RtlpFindAndCommitPages(Heap, - Heap->Segments[InUseEntry->SegmentOffset], - &FreeSize, - (PVOID)PAGE_ROUND_UP(InUseEntry + InUseEntry->Size)); + if (ClassFlags & HEAP_CLASS_1) + FreeEntry = RtlpFindAndCommitPages1(Heap, + Heap->Segments[InUseEntry->SegmentOffset], + &FreeSize, + FreeEntry); + else + FreeEntry = RtlpFindAndCommitPages(Heap, + Heap->Segments[InUseEntry->SegmentOffset], + &FreeSize, + (PVOID)PAGE_ROUND_UP(InUseEntry + InUseEntry->Size)); /* Fail if it failed... */ if (!FreeEntry) return FALSE; @@ -2463,6 +3385,7 @@ RtlpGrowBlockInPlace (IN PHEAP Heap, } else { + if (!(ClassFlags & HEAP_CLASS_1)) FreeEntry = (PHEAP_FREE_ENTRY)(InUseEntry + InUseEntry->Size); /* The next block indeed exists. Check if it's free or in use */ @@ -2476,7 +3399,11 @@ RtlpGrowBlockInPlace (IN PHEAP Heap, RememberFlags = FreeEntry->Flags; /* Remove this block from the free list */ - RtlpRemoveFreeBlock(Heap, FreeEntry, FALSE); + if (ClassFlags & HEAP_CLASS_1) + RtlpRemoveFreeBlock1(Heap, FreeEntry, FALSE, FALSE); + else + RtlpRemoveFreeBlock(Heap, FreeEntry, FALSE); + Heap->TotalFreeSize -= FreeEntry->Size; } @@ -2523,6 +3450,8 @@ RtlpGrowBlockInPlace (IN PHEAP Heap, { SegmentOffset = InUseEntry->SegmentOffset; ASSERT(SegmentOffset < HEAP_SEGMENTS); + if (ClassFlags & HEAP_CLASS_1) + Heap->Segments[SegmentOffset]->LastEntryInSegment = InUseEntry; } } else @@ -2538,6 +3467,8 @@ RtlpGrowBlockInPlace (IN PHEAP Heap, { SegmentOffset = UnusedEntry->SegmentOffset; ASSERT(SegmentOffset < HEAP_SEGMENTS); + if (ClassFlags & HEAP_CLASS_1) + Heap->Segments[SegmentOffset]->LastEntryInSegment = UnusedEntry; /* Set flags and size */ UnusedEntry->Flags = RememberFlags; @@ -2571,7 +3502,11 @@ RtlpGrowBlockInPlace (IN PHEAP Heap, RememberFlags = FollowingEntry->Flags; /* Remove it */ - RtlpRemoveFreeBlock(Heap, FollowingEntry, FALSE); + if (ClassFlags & HEAP_CLASS_1) + RtlpRemoveFreeBlock1(Heap, FollowingEntry, FALSE, FALSE); + else + RtlpRemoveFreeBlock(Heap, FollowingEntry, FALSE); + Heap->TotalFreeSize -= FollowingEntry->Size; /* And make up a new combined block */ @@ -2592,6 +3527,8 @@ RtlpGrowBlockInPlace (IN PHEAP Heap, { SegmentOffset = UnusedEntry->SegmentOffset; ASSERT(SegmentOffset < HEAP_SEGMENTS); + if (ClassFlags & HEAP_CLASS_1) + Heap->Segments[SegmentOffset]->LastEntryInSegment = UnusedEntry; } /* Insert it back and update total size */ @@ -2915,6 +3852,8 @@ RtlReAllocateHeap(HANDLE HeapPtr, { SegmentOffset = SplitBlock->SegmentOffset; ASSERT(SegmentOffset < HEAP_SEGMENTS); + if (Heap->ClassFlags & HEAP_CLASS_1) + Heap->Segments[SegmentOffset]->LastEntryInSegment = SplitBlock; /* Set its size and insert it to the list */ SplitBlock->Size = (USHORT)FreeSize; @@ -2948,7 +3887,11 @@ RtlReAllocateHeap(HANDLE HeapPtr, SplitBlock->Flags = SplitBlock2->Flags; /* Remove it, update total size */ - RtlpRemoveFreeBlock(Heap, SplitBlock2, FALSE); + if (Heap->ClassFlags & HEAP_CLASS_1) + RtlpRemoveFreeBlock1(Heap, SplitBlock2, FALSE, FALSE); + else + RtlpRemoveFreeBlock(Heap, SplitBlock2, FALSE); + Heap->TotalFreeSize -= SplitBlock2->Size; /* Calculate total free size */ @@ -2967,6 +3910,8 @@ RtlReAllocateHeap(HANDLE HeapPtr, { SegmentOffset = SplitBlock->SegmentOffset; ASSERT(SegmentOffset < HEAP_SEGMENTS); + if (Heap->ClassFlags & HEAP_CLASS_1) + Heap->Segments[SegmentOffset]->LastEntryInSegment = SplitBlock; } /* Insert the new one back and update total size */ @@ -3532,125 +4477,214 @@ BOOLEAN NTAPI RtlpValidateHeap(PHEAP Heap, BOOLEAN ForceValidation) { + BOOLEAN EmptyList; UCHAR SegmentOffset; - SIZE_T TotalFreeSize; + SIZE_T Size, TotalFreeSize; + ULONG PreviousSize; PLIST_ENTRY ListHead, NextEntry; + PHEAP_FREE_ENTRY FreeEntry; ULONG FreeBlocksCount, FreeListEntriesCount; ULONG HintIndex; - /* Check headers */ - if (!RtlpValidateHeapHeaders(Heap, FALSE)) - return FALSE; + if (Heap->ClassFlags & HEAP_CLASS_MASK) + { + /* Check headers */ + if (!RtlpValidateHeapHeaders(Heap, FALSE)) + return FALSE; - /* Skip validation if it's not needed */ - if (!ForceValidation && !(Heap->Flags & HEAP_VALIDATE_ALL_ENABLED)) - return TRUE; + /* Skip validation if it's not needed */ + if (!ForceValidation && !(Heap->Flags & HEAP_VALIDATE_ALL_ENABLED)) + return TRUE; - /* Check free list */ - FreeListEntriesCount = 0; - ListHead = &Heap->FreeLists; - NextEntry = ListHead->Flink; + /* Check free lists bitmaps */ + FreeListEntriesCount = 0; + ListHead = &Heap->FreeList[0]; - while (NextEntry != ListHead) - { - PHEAP_FREE_ENTRY FreeEntry = CONTAINING_RECORD(NextEntry, HEAP_FREE_ENTRY, FreeList); + for (Size = 0; Size < HEAP_FREELISTS; Size++) + { + if (Size) + { + /* This is a dedicated list. Check if it's empty */ + EmptyList = IsListEmpty(ListHead); - NextEntry = NextEntry->Flink; + if (Heap->u.FreeListsInUseBytes[Size >> 3] & (1 << (Size & 7))) + { + if (EmptyList) + { + DPRINT1("HEAP: Empty %x-free list marked as non-empty\n", Size); + return FALSE; + } + } + else + { + if (!EmptyList) + { + DPRINT1("HEAP: Non-empty %x-free list marked as empty\n", Size); + return FALSE; + } + } + } - if (NextEntry != ListHead) - { - PHEAP_FREE_ENTRY NextFreeEntry = CONTAINING_RECORD(NextEntry, HEAP_FREE_ENTRY, FreeList); - /* Free entries must be sorted */ - if (FreeEntry->Size > NextFreeEntry->Size) + /* Now check this list entries */ + NextEntry = ListHead->Flink; + PreviousSize = 0; + + while (ListHead != NextEntry) { - DPRINT1("Dedicated free entry %p of size %ld is not put in order.\n", FreeEntry, FreeEntry->Size); + FreeEntry = CONTAINING_RECORD(NextEntry, HEAP_FREE_ENTRY, FreeList); + NextEntry = NextEntry->Flink; + + /* If there is an in-use entry in a free list - that's quite a big problem */ + if (FreeEntry->Flags & HEAP_ENTRY_BUSY) + { + DPRINT1("HEAP: %Ix-dedicated list free element %p is marked in-use\n", Size, FreeEntry); + return FALSE; + } + + /* Check sizes according to that specific list's size */ + if ((Size == 0) && (FreeEntry->Size < HEAP_FREELISTS)) + { + DPRINT1("HEAP: Non dedicated list free element %p has size %x which would fit a dedicated list\n", FreeEntry, FreeEntry->Size); + return FALSE; + } + else if (Size && (FreeEntry->Size != Size)) + { + DPRINT1("HEAP: %Ix-dedicated list free element %p has incorrect size %x\n", Size, FreeEntry, FreeEntry->Size); + return FALSE; + } + else if ((Size == 0) && (FreeEntry->Size < PreviousSize)) + { + DPRINT1("HEAP: Non dedicated list free element %p is not put in order\n", FreeEntry); + return FALSE; + } + + /* Remember previous size*/ + PreviousSize = FreeEntry->Size; + + /* Add up to the total amount of free entries */ + FreeListEntriesCount++; } + + /* Go to the head of the next free list */ + ListHead++; } + } + else + { + /* Check headers */ + if (!RtlpValidateHeapHeaders(Heap, FALSE)) + return FALSE; + + /* Skip validation if it's not needed */ + if (!ForceValidation && !(Heap->Flags & HEAP_VALIDATE_ALL_ENABLED)) + return TRUE; + + /* Check free list */ + FreeListEntriesCount = 0; + ListHead = &Heap->FreeLists; + NextEntry = ListHead->Flink; - /* Check that the hint is there */ - if (FreeEntry->Size > Heap->DeCommitFreeBlockThreshold) + while (NextEntry != ListHead) { - if (Heap->FreeHints[0] == NULL) + PHEAP_FREE_ENTRY FreeEntry = CONTAINING_RECORD(NextEntry, HEAP_FREE_ENTRY, FreeList); + + NextEntry = NextEntry->Flink; + + if (NextEntry != ListHead) { - DPRINT1("No hint pointing to the non-dedicated list although there is a free entry %p of size %ld.\n", - FreeEntry, FreeEntry->Size); + PHEAP_FREE_ENTRY NextFreeEntry = CONTAINING_RECORD(NextEntry, HEAP_FREE_ENTRY, FreeList); + /* Free entries must be sorted */ + if (FreeEntry->Size > NextFreeEntry->Size) + { + DPRINT1("Dedicated free entry %p of size %ld is not put in order.\n", FreeEntry, FreeEntry->Size); + } } - if (!RtlTestBit(&Heap->FreeHintBitmap, 0)) + + /* Check that the hint is there */ + if (FreeEntry->Size > Heap->DeCommitFreeBlockThreshold) { - DPRINT1("Hint bit 0 is not set although there is a free entry %p of size %ld.\n", - FreeEntry, FreeEntry->Size); + if (Heap->FreeHints[0] == NULL) + { + DPRINT1("No hint pointing to the non-dedicated list although there is a free entry %p of size %ld.\n", + FreeEntry, FreeEntry->Size); + } + if (!RtlTestBit(&Heap->FreeHintBitmap, 0)) + { + DPRINT1("Hint bit 0 is not set although there is a free entry %p of size %ld.\n", + FreeEntry, FreeEntry->Size); + } } - } - else - { - if (Heap->FreeHints[FreeEntry->Size - 1] == NULL) + else { - DPRINT1("No hint pointing to the dedicated list although there is a free entry %p of size %ld.\n", - FreeEntry, FreeEntry->Size); + if (Heap->FreeHints[FreeEntry->Size - 1] == NULL) + { + DPRINT1("No hint pointing to the dedicated list although there is a free entry %p of size %ld.\n", + FreeEntry, FreeEntry->Size); + } + if (!RtlTestBit(&Heap->FreeHintBitmap, FreeEntry->Size - 1)) + { + DPRINT1("Hint bit 0 is not set although there is a free entry %p of size %ld.\n", + FreeEntry, FreeEntry->Size); + } } - if (!RtlTestBit(&Heap->FreeHintBitmap, FreeEntry->Size - 1)) + + /* If there is an in-use entry in a free list - that's quite a big problem */ + if (FreeEntry->Flags & HEAP_ENTRY_BUSY) { - DPRINT1("Hint bit 0 is not set although there is a free entry %p of size %ld.\n", - FreeEntry, FreeEntry->Size); + DPRINT1("HEAP: Free element %p is marked in-use\n", FreeEntry); + return FALSE; } - } - /* If there is an in-use entry in a free list - that's quite a big problem */ - if (FreeEntry->Flags & HEAP_ENTRY_BUSY) - { - DPRINT1("HEAP: Free element %p is marked in-use\n", FreeEntry); - return FALSE; + /* Add up to the total amount of free entries */ + FreeListEntriesCount++; } - /* Add up to the total amount of free entries */ - FreeListEntriesCount++; - } - - /* Check free list hints */ - for (HintIndex = 0; HintIndex < Heap->DeCommitFreeBlockThreshold; HintIndex++) - { - if (Heap->FreeHints[HintIndex] != NULL) + /* Check free list hints */ + for (HintIndex = 0; HintIndex < Heap->DeCommitFreeBlockThreshold; HintIndex++) { - PHEAP_FREE_ENTRY FreeEntry = CONTAINING_RECORD(Heap->FreeHints[HintIndex], HEAP_FREE_ENTRY, FreeList); - - if (!RtlTestBit(&Heap->FreeHintBitmap, HintIndex)) + if (Heap->FreeHints[HintIndex] != NULL) { - DPRINT1("Hint bitmap bit at %u is not set, but there is a hint entry.\n", HintIndex); - } + PHEAP_FREE_ENTRY FreeEntry = CONTAINING_RECORD(Heap->FreeHints[HintIndex], HEAP_FREE_ENTRY, FreeList); - if (HintIndex == 0) - { - if (FreeEntry->Size <= Heap->DeCommitFreeBlockThreshold) - { - DPRINT1("There is an entry %p of size %lu, smaller than the decommit threshold %lu in the non-dedicated free list hint.\n", - FreeEntry, FreeEntry->Size, Heap->DeCommitFreeBlockThreshold); - } - } - else - { - if (HintIndex != FreeEntry->Size - 1) + if (!RtlTestBit(&Heap->FreeHintBitmap, HintIndex)) { - DPRINT1("There is an entry %p of size %lu at the position %u in the free entry hint array.\n", - FreeEntry, FreeEntry->Size, HintIndex); + DPRINT1("Hint bitmap bit at %u is not set, but there is a hint entry.\n", HintIndex); } - if (FreeEntry->FreeList.Blink != &Heap->FreeLists) + if (HintIndex == 0) { - /* The entry right before the hint must be smaller. */ - PHEAP_FREE_ENTRY PreviousFreeEntry = CONTAINING_RECORD(FreeEntry->FreeList.Blink, - HEAP_FREE_ENTRY, - FreeList); - if (PreviousFreeEntry->Size >= FreeEntry->Size) + if (FreeEntry->Size <= Heap->DeCommitFreeBlockThreshold) + { + DPRINT1("There is an entry %p of size %lu, smaller than the decommit threshold %lu in the non-dedicated free list hint.\n", + FreeEntry, FreeEntry->Size, Heap->DeCommitFreeBlockThreshold); + } + } + else + { + if (HintIndex != FreeEntry->Size - 1) + { + DPRINT1("There is an entry %p of size %lu at the position %u in the free entry hint array.\n", + FreeEntry, FreeEntry->Size, HintIndex); + } + + if (FreeEntry->FreeList.Blink != &Heap->FreeLists) { - DPRINT1("Free entry hint %p of size %lu is larger than the entry before it %p, which is of size %lu.\n", - FreeEntry, FreeEntry->Size, PreviousFreeEntry, PreviousFreeEntry->Size); + /* The entry right before the hint must be smaller. */ + PHEAP_FREE_ENTRY PreviousFreeEntry = CONTAINING_RECORD(FreeEntry->FreeList.Blink, + HEAP_FREE_ENTRY, + FreeList); + if (PreviousFreeEntry->Size >= FreeEntry->Size) + { + DPRINT1("Free entry hint %p of size %lu is larger than the entry before it %p, which is of size %lu.\n", + FreeEntry, FreeEntry->Size, PreviousFreeEntry, PreviousFreeEntry->Size); + } } } } - } - else if (RtlTestBit(&Heap->FreeHintBitmap, HintIndex)) - { - DPRINT1("Hint bitmap bit at %u is set, but there is no hint entry.\n", HintIndex); + else if (RtlTestBit(&Heap->FreeHintBitmap, HintIndex)) + { + DPRINT1("Hint bitmap bit at %u is set, but there is no hint entry.\n", HintIndex); + } } } diff --git a/sdk/lib/rtl/heap.h b/sdk/lib/rtl/heap.h index 1f0b5436011..10d0bb74b68 100644 --- a/sdk/lib/rtl/heap.h +++ b/sdk/lib/rtl/heap.h @@ -12,6 +12,7 @@ #define RTL_HEAP_H /* Core heap definitions */ +#define HEAP_FREELISTS 128 #define HEAP_SEGMENTS 64 #define HEAP_ENTRY_SIZE ((ULONG)sizeof(HEAP_ENTRY)) @@ -217,13 +218,15 @@ typedef struct _HEAP_LIST_LOOKUP ULONG NumberOfUnCommittedRanges; \ USHORT SegmentAllocatorBackTraceIndex; \ USHORT Reserved; \ - LIST_ENTRY UCRSegmentList + LIST_ENTRY UCRSegmentList; \ + PVOID LastEntryInSegment //FIXME: non-Vista typedef struct _HEAP { HEAP_SEGMENT_MEMBERS; ULONG Flags; + ULONG ClassFlags; ULONG ForceFlags; ULONG CompatibilityFlags; ULONG EncodeFlagMask; @@ -256,7 +259,12 @@ typedef struct _HEAP PVOID BlocksIndex; // HEAP_LIST_LOOKUP PVOID UCRIndex; PHEAP_PSEUDO_TAG_ENTRY PseudoTagEntries; - LIST_ENTRY FreeLists; + LIST_ENTRY FreeLists; // NEEDS Uncommented for BIG PATCH + union + { + ULONG FreeListsInUseUlong[HEAP_FREELISTS / (sizeof(ULONG) * 8)]; //FIXME: non-Vista + UCHAR FreeListsInUseBytes[HEAP_FREELISTS / (sizeof(UCHAR) * 8)]; //FIXME: non-Vista + } u; PHEAP_LOCK LockVariable; PRTL_HEAP_COMMIT_ROUTINE CommitRoutine; PVOID FrontEndHeap; @@ -266,6 +274,7 @@ typedef struct _HEAP HEAP_TUNING_PARAMETERS TuningParameters; RTL_BITMAP FreeHintBitmap; // FIXME: non-Vista PLIST_ENTRY FreeHints[ANYSIZE_ARRAY]; // FIXME: non-Vista + LIST_ENTRY FreeList[HEAP_FREELISTS]; // Need Lists->List for BIG PATCH } HEAP, *PHEAP; typedef struct _HEAP_SEGMENT