Index: ntoskrnl/mm/section.c =================================================================== --- ntoskrnl/mm/section.c (revision 55851) +++ ntoskrnl/mm/section.c (working copy) @@ -182,7 +182,92 @@ /* FUNCTIONS *****************************************************************/ +NTSTATUS NTAPI +MiCreateVirtualMappingAndUpdatePdeCount +(PEPROCESS Process, + PVOID Address, + ULONG Protect, + PPFN_NUMBER Pages, + ULONG Count, + BOOLEAN Unsafe) +{ + NTSTATUS Status = (Unsafe ? &MmCreateVirtualMappingUnsafe : &MmCreateVirtualMapping) + (Process, Address, Protect, Pages, Count); + /* Update pde ref counts -- MmCreateVirtualMapping requires the old PTE to be zero. */ +#if (_MI_PAGING_LEVELS == 2) + /* Reference Page Directory Entry */ + while (Count--) + { + if (Address < MmSystemRangeStart) + { + MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)]++; + ASSERT(MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)] <= PTE_COUNT); + } + Address = (PVOID)((PCHAR)Address + PAGE_SIZE); + } +#endif + return Status; +} +VOID NTAPI +MiDeleteVirtualMappingAndUpdatePdeCount +(PEPROCESS Process, + PVOID Address, + BOOLEAN FreePage, + PBOOLEAN WasDirty, + PPFN_NUMBER OldPage) +{ + MmDeleteVirtualMapping + (Process, Address, FreePage, WasDirty, OldPage); + /* By definition, the page entry will not be used after this */ +#if (_MI_PAGING_LEVELS == 2) + if (Address < MmSystemRangeStart) + { + MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)]--; + ASSERT(MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)] <= PTE_COUNT); + } +#endif +} + +NTSTATUS NTAPI +MiCreatePageFileMappingAndUpdatePdeCount +(PEPROCESS Process, + PVOID Address, + SWAPENTRY SwapEntry) +{ + NTSTATUS Status = MmCreatePageFileMapping(Process, Address, SwapEntry); + /* Update pde ref counts -- MmCreatePageFileMapping requires the old PTE to be zero. */ +#if (_MI_PAGING_LEVELS == 2) + if (Address < MmSystemRangeStart) + { + MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)]++; + ASSERT(MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)] <= PTE_COUNT); + } +#endif + return Status; +} + +VOID NTAPI +MiDeletePageFileMappingAndUpdatePdeCount +(PEPROCESS Process, + PVOID Address, + SWAPENTRY *SwapEntry) +{ + MmDeletePageFileMapping(Process, Address, SwapEntry); +#if (_MI_PAGING_LEVELS == 2) + if (*SwapEntry && Address < MmSystemRangeStart) + { + MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)]--; + ASSERT(MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)] <= PTE_COUNT); + } +#endif +} + +#define MmCreateVirtualMapping __Please_use_MiCreateVirtualMappingAndUpdatePdeCount_instead +#define MmDeleteVirtualMapping __Please_use_MiDeleteVirtualMappingAndUpdatePdeCount_instead +#define MmCreatePageFileMapping __Please_use_MiCreatePageFileMappingAndUpdatePdeCount_instead +#define MmDeletePageFileMapping __Please_use_MiDeletePageFileMappingAndUpdatePdeCount_instead + /* References: [1] Microsoft Corporation, "Microsoft Portable Executable and Common Object @@ -1369,23 +1454,16 @@ MmSharePageEntrySectionSegment(Segment, &Offset); - #if (_MI_PAGING_LEVELS == 2) - /* Reference Page Directory Entry */ - if(Address < MmSystemRangeStart) - { - MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)]++; - ASSERT(MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)] <= PTE_COUNT); - } -#endif - /* FIXME: Should we call MmCreateVirtualMappingUnsafe if * (Section->AllocationAttributes & SEC_PHYSICALMEMORY) is true? */ - Status = MmCreateVirtualMapping(Process, - PAddress, - Attributes, - &Page, - 1); + Status = MiCreateVirtualMappingAndUpdatePdeCount + (Process, + PAddress, + Attributes, + &Page, + 1, + !!(Section->AllocationAttributes & SEC_PHYSICALMEMORY)); if (!NT_SUCCESS(Status)) { DPRINT1("Unable to create virtual mapping\n"); @@ -1418,7 +1496,7 @@ } MmUnlockSectionSegment(Segment); - MmDeletePageFileMapping(Process, Address, &SwapEntry); + MiDeletePageFileMappingAndUpdatePdeCount(Process, Address, &SwapEntry); MmUnlockAddressSpace(AddressSpace); MI_SET_USAGE(MI_USAGE_SECTION); @@ -1437,11 +1515,13 @@ KeBugCheck(MEMORY_MANAGEMENT); } MmLockAddressSpace(AddressSpace); - Status = MmCreateVirtualMapping(Process, - Address, - Region->Protect, - &Page, - 1); + Status = MiCreateVirtualMappingAndUpdatePdeCount + (Process, + Address, + Region->Protect, + &Page, + 1, + FALSE); if (!NT_SUCCESS(Status)) { DPRINT("MmCreateVirtualMapping failed, not out of memory\n"); @@ -1467,15 +1547,6 @@ return(STATUS_SUCCESS); } -#if (_MI_PAGING_LEVELS == 2) - /* Reference Page Directory Entry */ - if(Address < MmSystemRangeStart) - { - MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)]++; - ASSERT(MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)] <= PTE_COUNT); - } -#endif - /* * Satisfying a page fault on a map of /Device/PhysicalMemory is easy */ @@ -1486,11 +1557,13 @@ * Just map the desired physical page */ Page = Offset.QuadPart >> PAGE_SHIFT; - Status = MmCreateVirtualMappingUnsafe(Process, - PAddress, - Region->Protect, - &Page, - 1); + Status = MiCreateVirtualMappingAndUpdatePdeCount + (Process, + PAddress, + Region->Protect, + &Page, + 1, + TRUE); if (!NT_SUCCESS(Status)) { DPRINT("MmCreateVirtualMappingUnsafe failed, not out of memory\n"); @@ -1527,11 +1600,13 @@ { KeBugCheck(MEMORY_MANAGEMENT); } - Status = MmCreateVirtualMapping(Process, - PAddress, - Region->Protect, - &Page, - 1); + Status = MiCreateVirtualMappingAndUpdatePdeCount + (Process, + PAddress, + Region->Protect, + &Page, + 1, + FALSE); if (!NT_SUCCESS(Status)) { DPRINT("MmCreateVirtualMapping failed, not out of memory\n"); @@ -1596,12 +1671,6 @@ /* * Cleanup and release locks */ -#if (_MI_PAGING_LEVELS == 2) - if(Address < MmSystemRangeStart) - { - MmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)]--; - } -#endif MmLockAddressSpace(AddressSpace); PageOp->Status = Status; MmspCompleteAndReleasePageOp(PageOp); @@ -1633,11 +1702,13 @@ MmSetPageEntrySectionSegment(Segment, &Offset, Entry); MmUnlockSectionSegment(Segment); - Status = MmCreateVirtualMapping(Process, - PAddress, - Attributes, - &Page, - 1); + Status = MiCreateVirtualMappingAndUpdatePdeCount + (Process, + PAddress, + Attributes, + &Page, + 1, + FALSE); if (!NT_SUCCESS(Status)) { DPRINT1("Unable to create virtual mapping\n"); @@ -1706,11 +1777,13 @@ * Save the swap entry. */ MmSetSavedSwapEntryPage(Page, SwapEntry); - Status = MmCreateVirtualMapping(Process, - Address, - Region->Protect, - &Page, - 1); + Status = MiCreateVirtualMappingAndUpdatePdeCount + (Process, + Address, + Region->Protect, + &Page, + 1, + FALSE); if (!NT_SUCCESS(Status)) { DPRINT1("Unable to create virtual mapping\n"); @@ -1734,11 +1807,13 @@ MmSharePageEntrySectionSegment(Segment, &Offset); MmUnlockSectionSegment(Segment); - Status = MmCreateVirtualMapping(Process, - PAddress, - Attributes, - &Page, - 1); + Status = MiCreateVirtualMappingAndUpdatePdeCount + (Process, + PAddress, + Attributes, + &Page, + 1, + FALSE); if (!NT_SUCCESS(Status)) { DPRINT1("Unable to create virtual mapping\n"); @@ -1892,16 +1967,18 @@ /* * Delete the old entry. */ - MmDeleteVirtualMapping(Process, Address, FALSE, NULL, NULL); + MiDeleteVirtualMappingAndUpdatePdeCount(Process, Address, FALSE, NULL, NULL); /* * Set the PTE to point to the new page */ - Status = MmCreateVirtualMapping(Process, - PAddress, - Region->Protect, - &NewPage, - 1); + Status = MiCreateVirtualMappingAndUpdatePdeCount + (Process, + PAddress, + Region->Protect, + &NewPage, + 1, + FALSE); if (!NT_SUCCESS(Status)) { DPRINT("MmCreateVirtualMapping failed, not out of memory\n"); @@ -1942,11 +2019,12 @@ MmLockAddressSpace(&Process->Vm); } - MmDeleteVirtualMapping(Process, - Address, - FALSE, - &WasDirty, - &Page); + MiDeleteVirtualMappingAndUpdatePdeCount + (Process, + Address, + FALSE, + &WasDirty, + &Page); if (WasDirty) { PageOutContext->WasDirty = TRUE; @@ -1971,15 +2049,6 @@ MmReleasePageMemoryConsumer(MC_USER, Page); } -#if (_MI_PAGING_LEVELS == 2) - /* If this is for the calling process, we take care of te reference in the main function */ - if((Address < MmSystemRangeStart) && (Process != PageOutContext->CallingProcess)) - { - Process->Vm.VmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)]--; - ASSERT(Process->Vm.VmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)] < PTE_COUNT); - } -#endif - DPRINT("PhysicalAddress %x, Address %x\n", Page << PAGE_SHIFT, Address); } @@ -2151,14 +2220,6 @@ } if (!Context.WasDirty && SwapEntry != 0) { -#if (_MI_PAGING_LEVELS == 2) - /* We keep the pagefile index global to the segment, not in the PTE */ - if(Address < MmSystemRangeStart) - { - Process->Vm.VmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)]--; - ASSERT(Process->Vm.VmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)] < PTE_COUNT); - } -#endif MmSetSavedSwapEntryPage(Page, 0); MmLockSectionSegment(Context.Segment); MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, MAKE_SWAP_SSE(SwapEntry)); @@ -2179,14 +2240,6 @@ } if (!Context.WasDirty || SwapEntry != 0) { -#if (_MI_PAGING_LEVELS == 2) - /* We keep the pagefile index global to the segment, not in the PTE */ - if(Address < MmSystemRangeStart) - { - Process->Vm.VmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)]--; - ASSERT(Process->Vm.VmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)] < PTE_COUNT); - } -#endif MmSetSavedSwapEntryPage(Page, 0); if (SwapEntry != 0) { @@ -2202,14 +2255,6 @@ } else if (!Context.Private && DirectMapped) { -#if (_MI_PAGING_LEVELS == 2) - /* Read only page, no need for a pagefile entry -> PDE-- */ - if(Address < MmSystemRangeStart) - { - Process->Vm.VmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)]--; - ASSERT(Process->Vm.VmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)] < PTE_COUNT); - } -#endif if (SwapEntry != 0) { DPRINT1("Found a swapentry for a non private and direct mapped page (address %x)\n", @@ -2240,14 +2285,6 @@ Address); KeBugCheckEx(MEMORY_MANAGEMENT, SwapEntry, Page, (ULONG_PTR)Process, (ULONG_PTR)Address); } -#if (_MI_PAGING_LEVELS == 2) - /* Non dirty, non private, non direct-mapped -> PDE-- */ - if(Address < MmSystemRangeStart) - { - Process->Vm.VmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)]--; - ASSERT(Process->Vm.VmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)] < PTE_COUNT); - } -#endif MmReleasePageMemoryConsumer(MC_USER, Page); PageOp->Status = STATUS_SUCCESS; MmspCompleteAndReleasePageOp(PageOp); @@ -2258,9 +2295,10 @@ DPRINT1("Not dirty and private and not swapped (%p:%p)\n", Process, Address); MmSetSavedSwapEntryPage(Page, 0); MmLockAddressSpace(AddressSpace); - Status = MmCreatePageFileMapping(Process, - Address, - SwapEntry); + Status = MiCreatePageFileMappingAndUpdatePdeCount + (Process, + Address, + SwapEntry); MmUnlockAddressSpace(AddressSpace); if (!NT_SUCCESS(Status)) { @@ -2288,11 +2326,13 @@ */ if (Context.Private) { - Status = MmCreateVirtualMapping(Process, - Address, - MemoryArea->Protect, - &Page, - 1); + Status = MiCreateVirtualMappingAndUpdatePdeCount + (Process, + Address, + MemoryArea->Protect, + &Page, + 1, + FALSE); MmSetDirtyPage(Process, Address); MmInsertRmap(Page, Process, @@ -2306,11 +2346,13 @@ * set it back into the section segment entry so we don't loose * our copy. Otherwise it will be handled by the cache manager. */ - Status = MmCreateVirtualMapping(Process, - Address, - MemoryArea->Protect, - &Page, - 1); + Status = MiCreateVirtualMappingAndUpdatePdeCount + (Process, + Address, + MemoryArea->Protect, + &Page, + 1, + FALSE); MmSetDirtyPage(Process, Address); MmInsertRmap(Page, Process, @@ -2345,11 +2387,13 @@ MmLockAddressSpace(AddressSpace); if (Context.Private) { - Status = MmCreateVirtualMapping(Process, - Address, - MemoryArea->Protect, - &Page, - 1); + Status = MiCreateVirtualMappingAndUpdatePdeCount + (Process, + Address, + MemoryArea->Protect, + &Page, + 1, + FALSE); MmSetDirtyPage(Process, Address); MmInsertRmap(Page, Process, @@ -2357,11 +2401,13 @@ } else { - Status = MmCreateVirtualMapping(Process, - Address, - MemoryArea->Protect, - &Page, - 1); + Status = MiCreateVirtualMappingAndUpdatePdeCount + (Process, + Address, + MemoryArea->Protect, + &Page, + 1, + FALSE); MmSetDirtyPage(Process, Address); MmInsertRmap(Page, Process, @@ -2394,9 +2440,10 @@ if (Context.Private) { MmLockAddressSpace(AddressSpace); - Status = MmCreatePageFileMapping(Process, - Address, - SwapEntry); + Status = MiCreatePageFileMappingAndUpdatePdeCount + (Process, + Address, + SwapEntry); MmUnlockAddressSpace(AddressSpace); if (!NT_SUCCESS(Status)) { @@ -2406,14 +2453,6 @@ } else { -#if (_MI_PAGING_LEVELS == 2) - /* We keep the pagefile index global to the segment, not in the PTE */ - if(Address < MmSystemRangeStart) - { - Process->Vm.VmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)]--; - ASSERT(Process->Vm.VmWorkingSetList->UsedPageTableEntries[MiGetPdeOffset(Address)] < PTE_COUNT); - } -#endif Entry = MAKE_SWAP_SSE(SwapEntry); MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, Entry); }