Index: ntoskrnl/include/internal/mm.h =================================================================== --- ntoskrnl/include/internal/mm.h (revision 63977) +++ ntoskrnl/include/internal/mm.h (working copy) @@ -373,6 +373,7 @@ extern MMPFNLIST MmZeroedPageListHead; extern MMPFNLIST MmFreePageListHead; extern MMPFNLIST MmStandbyPageListHead; extern MMPFNLIST MmModifiedPageListHead; +extern MMPFNLIST MmModifiedPageListByColor[1]; extern MMPFNLIST MmModifiedNoWritePageListHead; typedef struct _MM_MEMORY_CONSUMER @@ -652,8 +653,27 @@ NTSTATUS NTAPI MiReadPageFile( _In_ PFN_NUMBER Page, - _In_ ULONG PageFileIndex, - _In_ ULONG_PTR PageFileOffset); + _In_ const MMPTE* PointerPte +); + +NTSTATUS +NTAPI +MiWritePageFile( + _In_ PFN_NUMBER Page, + _In_ const MMPTE* PointerPte +); + +VOID +NTAPI +MiFreePageFileEntry( + _In_ PMMPTE PointerPte +); + +NTSTATUS +NTAPI +MiReservePageFileEntry( + _Out_ PMMPTE PointerPte +); /* process.c ****************************************************************/ Index: ntoskrnl/mm/ARM3/miarm.h =================================================================== --- ntoskrnl/mm/ARM3/miarm.h (revision 63977) +++ ntoskrnl/mm/ARM3/miarm.h (working copy) @@ -687,6 +687,7 @@ extern PKEVENT MiLowPagedPoolEvent; extern PKEVENT MiHighPagedPoolEvent; extern PKEVENT MiLowNonPagedPoolEvent; extern PKEVENT MiHighNonPagedPoolEvent; +extern KEVENT MpwThreadEvent; extern PFN_NUMBER MmLowMemoryThreshold; extern PFN_NUMBER MmHighMemoryThreshold; extern PFN_NUMBER MiLowPagedPoolThreshold; @@ -695,6 +696,7 @@ extern PFN_NUMBER MiLowNonPagedPoolThreshold; extern PFN_NUMBER MiHighNonPagedPoolThreshold; extern PFN_NUMBER MmMinimumFreePages; extern PFN_NUMBER MmPlentyFreePages; +extern PFN_NUMBER MmTotalPagesForPagingFile; extern SIZE_T MmMinimumStackCommitInBytes; extern PFN_COUNT MiExpansionPoolPagesInitialCharge; extern PFN_NUMBER MmResidentAvailablePages; Index: ntoskrnl/mm/ARM3/pagfault.c =================================================================== --- ntoskrnl/mm/ARM3/pagfault.c (revision 63977) +++ ntoskrnl/mm/ARM3/pagfault.c (working copy) @@ -583,7 +583,7 @@ MiResolveDemandZeroFault(IN PVOID Address, ASSERT(PointerPte->u.Hard.Valid == 0); /* Assert we have enough pages */ - ASSERT(MmAvailablePages >= 32); + ASSERT(MmAvailablePages != 0); #if MI_TRACE_PFNS if (UserPdeFault) MI_SET_USAGE(MI_USAGE_PAGE_TABLE); @@ -688,7 +688,8 @@ MiCompleteProtoPteFault(IN BOOLEAN StoreInstruction, IN PMMPTE PointerPte, IN PMMPTE PointerProtoPte, IN KIRQL OldIrql, - IN PMMPFN* LockedProtoPfn) + IN PMMPFN* LockedProtoPfn, + PVOID InPageBlock) { MMPTE TempPte; PMMPTE OriginalPte, PageTablePte; @@ -800,6 +801,10 @@ MiCompleteProtoPteFault(IN BOOLEAN StoreInstruction, /* Reset the protection if needed */ if (OriginalProtection) Protection = MM_ZERO_ACCESS; + /* Let waiters go ahead */ + if (InPageBlock) + KeSetEvent(InPageBlock, IO_NO_INCREMENT, FALSE); + /* Return success */ ASSERT(PointerPte == MiAddressToPte(Address)); return STATUS_SUCCESS; @@ -816,26 +821,30 @@ MiResolvePageFileFault(_In_ BOOLEAN StoreInstruction, ULONG Color; PFN_NUMBER Page; NTSTATUS Status; - MMPTE TempPte = *PointerPte; - KEVENT Event; + MMPTE PageFilePte = *PointerPte, TempPte; PMMPFN Pfn1; - ULONG PageFileIndex = TempPte.u.Soft.PageFileLow; - ULONG_PTR PageFileOffset = TempPte.u.Soft.PageFileHigh; + + DPRINT1("Serving page fault for PTE %p (%x).\n", PointerPte, PageFilePte.u.Long); /* Things we don't support yet */ - ASSERT(CurrentProcess > HYDRA_PROCESS); ASSERT(*OldIrql != MM_NOIRQL); /* We must hold the PFN lock */ ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL); /* Some sanity checks */ - ASSERT(TempPte.u.Hard.Valid == 0); - ASSERT(TempPte.u.Soft.PageFileHigh != 0); - ASSERT(TempPte.u.Soft.PageFileHigh != MI_PTE_LOOKUP_NEEDED); + ASSERT(PageFilePte.u.Hard.Valid == 0); + ASSERT(PageFilePte.u.Soft.Transition == 0); + ASSERT(PageFilePte.u.Soft.Prototype == 0); + ASSERT(PageFilePte.u.Soft.PageFileHigh != 0); + ASSERT(PageFilePte.u.Soft.PageFileHigh != MI_PTE_LOOKUP_NEEDED); /* Get any page, it will be overwritten */ - Color = MI_GET_NEXT_PROCESS_COLOR(CurrentProcess); + if (CurrentProcess > HYDRA_PROCESS) + Color = MI_GET_NEXT_PROCESS_COLOR(CurrentProcess); + else + Color = MI_GET_NEXT_COLOR(); + Page = MiRemoveAnyPage(Color); /* Initialize this PFN */ @@ -847,14 +856,13 @@ MiResolvePageFileFault(_In_ BOOLEAN StoreInstruction, ASSERT(Pfn1->u3.e1.ReadInProgress == 0); ASSERT(Pfn1->u3.e1.WriteInProgress == 0); - KeInitializeEvent(&Event, NotificationEvent, FALSE); - Pfn1->u1.Event = &Event; + /* Set NULL event */ + Pfn1->u1.Event = NULL; Pfn1->u3.e1.ReadInProgress = 1; /* We must write the PTE now as the PFN lock will be released while performing the IO operation */ + TempPte.u.Long = 0; TempPte.u.Soft.Transition = 1; - TempPte.u.Soft.PageFileLow = 0; - TempPte.u.Soft.Prototype = 0; TempPte.u.Trans.PageFrameNumber = Page; MI_WRITE_INVALID_PTE(PointerPte, TempPte); @@ -863,54 +871,88 @@ MiResolvePageFileFault(_In_ BOOLEAN StoreInstruction, KeReleaseQueuedSpinLock(LockQueuePfnLock, *OldIrql); /* Do the paging IO */ - Status = MiReadPageFile(Page, PageFileIndex, PageFileOffset); + Status = MiReadPageFile(Page, &PageFilePte); /* Lock the PFN database again */ *OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); /* Nobody should have changed that while we were not looking */ - ASSERT(Pfn1->u1.Event == &Event); ASSERT(Pfn1->u3.e1.ReadInProgress == 1); ASSERT(Pfn1->u3.e1.WriteInProgress == 0); + /* But maybe someone began waiting on us */ + if (Pfn1->u1.Event != NULL) + { + DPRINT1("The page got a waiter!\n"); + /* Waiters gonna wait */ + KeSetEvent(Pfn1->u1.Event, IO_NO_INCREMENT, FALSE); + Pfn1->u1.Event = NULL; + } + + DPRINT1("Pagefile read finished. Status: 0x%08x.\n", Status); + if (!NT_SUCCESS(Status)) { /* Malheur! */ ASSERT(FALSE); Pfn1->u4.InPageError = 1; Pfn1->u1.ReadStatus = Status; + Status = STATUS_IN_PAGE_ERROR; + } + else + { + /* This is a success status */ + Status = STATUS_PAGE_FAULT_PAGING_FILE; } /* This is now a nice and normal PFN */ - Pfn1->u1.Event = NULL; Pfn1->u3.e1.ReadInProgress = 0; - /* And the PTE can finally be valid */ - MI_MAKE_HARDWARE_PTE(&TempPte, PointerPte, TempPte.u.Trans.Protection, Page); - MI_WRITE_VALID_PTE(PointerPte, TempPte); + /* Check if this is a kernel or user address */ + if (PointerPte <= MiHighestUserPte) + { + /* Build the user PTE */ + MI_MAKE_HARDWARE_PTE_USER(&TempPte, + PointerPte, + PageFilePte.u.Soft.Protection, + Page); + } + else + { + /* Build the kernel PTE */ + MI_MAKE_HARDWARE_PTE(&TempPte, + PointerPte, + PageFilePte.u.Soft.Protection, + Page); + } - /* Waiters gonna wait */ - KeSetEvent(&Event, IO_NO_INCREMENT, FALSE); + /* Write it */ + MI_WRITE_VALID_PTE(PointerPte, TempPte); return Status; } NTSTATUS NTAPI -MiResolveTransitionFault(IN PVOID FaultingAddress, - IN PMMPTE PointerPte, - IN PEPROCESS CurrentProcess, - IN KIRQL OldIrql, - OUT PVOID *InPageBlock) +MiResolveTransitionFault( + _In_ BOOLEAN StoreInstruction, + _In_ PVOID FaultingAddress, + _In_ PMMPTE PointerPte, + _In_ PEPROCESS CurrentProcess, + _In_ KIRQL OldIrql, + _In_ PVOID *InPageBlock) { PFN_NUMBER PageFrameIndex; PMMPFN Pfn1; MMPTE TempPte; PMMPTE PointerToPteForProtoPage; - DPRINT1("Transition fault on 0x%p with PTE 0x%p in process %s\n", - FaultingAddress, PointerPte, CurrentProcess->ImageFileName); + BOOLEAN MustWait = FALSE; + KEVENT IoEvent; - /* Windowss does this check */ + DPRINT("Transition fault on 0x%p with PTE 0x%p in process %s\n", + FaultingAddress, PointerPte, CurrentProcess ? CurrentProcess->ImageFileName : "Kernel"); + + /* Windows does this check */ ASSERT(*InPageBlock == NULL); /* ARM3 doesn't support this path */ @@ -936,24 +978,67 @@ MiResolveTransitionFault(IN PVOID FaultingAddress, /* See if we should wait before terminating the fault */ if (Pfn1->u3.e1.ReadInProgress == 1) { - DPRINT1("The page is currently being read!\n"); - ASSERT(Pfn1->u1.Event != NULL); - *InPageBlock = Pfn1->u1.Event; - if (PointerPte == Pfn1->PteAddress) - { - DPRINT1("And this if for this particular PTE.\n"); - /* The PTE will be made valid by the thread serving the fault */ - return STATUS_SUCCESS; // FIXME: Maybe something more descriptive - } + DPRINT1("The page is currently being read!\n"); + + ASSERT(Pfn1->u3.e1.WriteInProgress == 0); + + KeInitializeEvent(&IoEvent, NotificationEvent, FALSE); + *InPageBlock = Pfn1->u1.Event; + Pfn1->u1.Event = &IoEvent; + + MustWait = TRUE; + + if (PointerPte == Pfn1->PteAddress) + { + DPRINT1("And this if for this particular PTE.\n"); + /* The PTE will be made valid by the thread serving the page-in operation */ + goto Quickie; + } + } + + if (Pfn1->u3.e1.WriteInProgress == 1) + { + DPRINT1("The page is currently being written to!\n"); + + ASSERT(Pfn1->u3.e1.ReadInProgress == 0); + + KeInitializeEvent(&IoEvent, NotificationEvent, FALSE); + *InPageBlock = Pfn1->u1.Event; + Pfn1->u1.Event = &IoEvent; + + /* + * Mark it as unmodified. + * The page-out thread will put it back in the standby list + * and we will be able to start anew for this PTE + */ + Pfn1->u3.e1.Modified = 0; + + /* Release the lock so the writer can finish */ + KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); + + /* Wait fot it to really finish */ + KeWaitForSingleObject(&IoEvent, WrPageIn, KernelMode, FALSE, NULL); + + /* Get it again */ + OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); + + /* Read the PTE again */ + TempPte = *PointerPte; + if (TempPte.u.Hard.Valid == 1) + { + /* Someone did it while we were not looking. */ + return STATUS_PAGE_FAULT_TRANSITION; + } + /* Otherwise the state of this PTE must still be the same */ + ASSERT(TempPte.u.Soft.Transition == 1); + ASSERT(TempPte.u.Soft.Prototype == 0); + ASSERT(TempPte.u.Trans.PageFrameNumber == PageFrameIndex); } /* Windows checks there's some free pages and this isn't an in-page error */ ASSERT(MmAvailablePages > 0); ASSERT(Pfn1->u4.InPageError == 0); - /* ReactOS checks for this */ - ASSERT(MmAvailablePages > 32); - /* Was this a transition page in the valid list, or free/zero list? */ if (Pfn1->u3.e1.PageLocation == ActiveAndValid) { @@ -1020,6 +1105,18 @@ MiResolveTransitionFault(IN PVOID FaultingAddress, /* Write the valid PTE */ MI_WRITE_VALID_PTE(PointerPte, TempPte); +Quickie: + if (MustWait) + { + /* Release the lock so the reader can finish */ + KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); + + /* Do the wait */ + KeWaitForSingleObject(&IoEvent, WrPageIn, KernelMode, FALSE, NULL); + + /* Callers expect the lock to still be held */ + OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); + } /* Return success */ return STATUS_PAGE_FAULT_TRANSITION; } @@ -1066,7 +1163,8 @@ MiResolveProtoPteFault(IN BOOLEAN StoreInstruction, PointerPte, PointerProtoPte, OldIrql, - OutPfn); + OutPfn, + NULL); } /* Make sure there's some protection mask */ @@ -1125,18 +1223,25 @@ MiResolveProtoPteFault(IN BOOLEAN StoreInstruction, { /* Resolve the transition fault */ ASSERT(OldIrql != MM_NOIRQL); - Status = MiResolveTransitionFault(Address, + Status = MiResolveTransitionFault(StoreInstruction, + Address, PointerProtoPte, Process, OldIrql, &InPageBlock); ASSERT(NT_SUCCESS(Status)); } + else if (TempPte.u.Soft.PageFileHigh != 0) + { + Status = MiResolvePageFileFault(StoreInstruction, + Address, + PointerProtoPte, + Process, + &OldIrql); + ASSERT(NT_SUCCESS(Status)); + } else { - /* We also don't support paged out pages */ - ASSERT(TempPte.u.Soft.PageFileHigh == 0); - /* Resolve the demand zero fault */ Status = MiResolveDemandZeroFault(Address, PointerProtoPte, @@ -1152,7 +1257,8 @@ MiResolveProtoPteFault(IN BOOLEAN StoreInstruction, PointerPte, PointerProtoPte, OldIrql, - OutPfn); + OutPfn, + InPageBlock); } NTSTATUS @@ -1173,9 +1279,8 @@ MiDispatchFault(IN BOOLEAN StoreInstruction, PMMPFN Pfn1, OutPfn = NULL; PFN_NUMBER PageFrameIndex; PFN_COUNT PteCount, ProcessedPtes; - DPRINT("ARM3 Page Fault Dispatcher for address: %p in process: %p\n", - Address, - Process); + DPRINT("ARM3 Page Fault Dispatcher for address: %p (PTE %p) in process: %p\n", + Address, PointerPte, Process); /* Make sure the addresses are ok */ ASSERT(PointerPte == MiAddressToPte(Address)); @@ -1337,7 +1442,8 @@ MiDispatchFault(IN BOOLEAN StoreInstruction, PointerPte, PointerProtoPte, LockIrql, - &OutPfn); + &OutPfn, + NULL); /* THIS RELEASES THE PFN LOCK! */ break; @@ -1390,14 +1496,14 @@ MiDispatchFault(IN BOOLEAN StoreInstruction, { /* We had a locked PFN, so acquire the PFN lock to dereference it */ ASSERT(PointerProtoPte != NULL); - OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); + LockIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); /* Dereference the locked PFN */ MiDereferencePfnAndDropLockCount(OutPfn); ASSERT(OutPfn->u3.e2.ReferenceCount >= 1); /* And now release the lock */ - KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); + KeReleaseQueuedSpinLock(LockQueuePfnLock, LockIrql); } /* Complete this as a transition fault */ @@ -1416,7 +1522,12 @@ MiDispatchFault(IN BOOLEAN StoreInstruction, LockIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); /* Resolve */ - Status = MiResolveTransitionFault(Address, PointerPte, Process, LockIrql, &InPageBlock); + Status = MiResolveTransitionFault(StoreInstruction, + Address, + PointerPte, + Process, + LockIrql, + &InPageBlock); NT_ASSERT(NT_SUCCESS(Status)); @@ -1425,8 +1536,8 @@ MiDispatchFault(IN BOOLEAN StoreInstruction, if (InPageBlock != NULL) { - /* The page is being paged in by another process */ - KeWaitForSingleObject(InPageBlock, WrPageIn, KernelMode, FALSE, NULL); + /* We must wake the blocked threads */ + KeSetEvent(InPageBlock, IO_NO_INCREMENT, FALSE); } ASSERT(OldIrql == KeGetCurrentIrql()); @@ -1821,9 +1932,6 @@ _WARN("Session space stuff is not implemented yet!") } else { - /* We don't implement transition PTEs */ - ASSERT(TempPte.u.Soft.Transition == 0); - /* Check for no-access PTE */ if (TempPte.u.Soft.Protection == MM_NOACCESS) { @@ -2090,7 +2198,7 @@ UserFault: OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); /* Make sure we have enough pages */ - ASSERT(MmAvailablePages >= 32); + ASSERT(MmAvailablePages != 0); /* Try to get a zero page */ MI_SET_USAGE(MI_USAGE_PEB_TEB); Index: ntoskrnl/mm/ARM3/pfnlist.c =================================================================== --- ntoskrnl/mm/ARM3/pfnlist.c (revision 63977) +++ ntoskrnl/mm/ARM3/pfnlist.c (working copy) @@ -36,7 +36,7 @@ BOOLEAN MmMirroring; ULONG MmSystemPageColor; ULONG MmTransitionSharedPages; -ULONG MmTotalPagesForPagingFile; +PFN_NUMBER MmTotalPagesForPagingFile; MMPFNLIST MmZeroedPageListHead = {0, ZeroedPageList, LIST_HEAD, LIST_HEAD}; MMPFNLIST MmFreePageListHead = {0, FreePageList, LIST_HEAD, LIST_HEAD}; @@ -266,7 +266,6 @@ MiUnlinkPageFromList(IN PMMPFN Pfn) } /* Decrease transition page counter */ - ASSERT(Pfn->u3.e1.PrototypePte == 1); /* Only supported ARM3 case */ MmTransitionSharedPages--; /* One less page */ @@ -288,6 +287,9 @@ MiUnlinkPageFromList(IN PMMPFN Pfn) ListHead->Total--; MmTotalPagesForPagingFile--; + if (MmTotalPagesForPagingFile == 0) + KeClearEvent(&MpwThreadEvent); + /* Pick the correct colored list */ ListHead = &MmModifiedPageListByColor[0]; @@ -477,12 +479,89 @@ MiRemovePageByColor(IN PFN_NUMBER PageIndex, return PageIndex; } +static +PFN_NUMBER +NTAPI +MiPopPageFromStandbyList(void) +{ + unsigned i; + PFN_NUMBER PageFrameIndex; + PMMPFN Pfn1; + PMMPTE PointerPte, SysPte, PteTable; + MMPTE PdePte; + + /* Make sure PFN lock is held and we have pages */ + ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL); + ASSERT(MmAvailablePages != 0); + + /* This should be called in last resort if there are no more free pages */ + ASSERT(MmZeroedPageListHead.Total == 0); + ASSERT(MmFreePageListHead.Total == 0); + + /* Go by priority */ + for (i = 0; i < 8; i++) + { + /* Get the oldest one */ + PageFrameIndex = MmStandbyPageListByPriority[i].Blink; + if (PageFrameIndex != LIST_HEAD) + break; + } + + /* There are no free pages but still available ones, so this must be here */ + ASSERT(PageFrameIndex != LIST_HEAD); + + /* Get the PFN */ + Pfn1 = MI_PFN_ELEMENT(PageFrameIndex); + + DPRINT1("Popping page %x from standbylist (PTE %p).\n", + PageFrameIndex, Pfn1->PteAddress); + + /* Sanity checks */ + ASSERT(Pfn1->u3.e1.PageLocation == StandbyPageList); + ASSERT(Pfn1->u3.e1.Modified == 0); + + /* Get a PTE to map the page directory */ + SysPte = MiReserveSystemPtes(1, SystemPteSpace); + ASSERT(SysPte != NULL); + + /* Build it */ + MI_MAKE_HARDWARE_PTE_KERNEL(&PdePte, + SysPte, + MM_READWRITE, + Pfn1->u4.PteFrame); + /* And map it */ + MI_WRITE_VALID_PTE(SysPte, PdePte); + PteTable = MiPteToAddress(SysPte); + + /* Finally get a pointer to the PTE this page represents */ + PointerPte = &PteTable[MiAddressToPteOffset(MiPteToAddress(Pfn1->PteAddress))]; + + /* Other sanity checks */ + ASSERT(PointerPte->u.Hard.Valid == 0); + ASSERT(PointerPte->u.Soft.Transition == 1); + + /* Restore the PTE to its original value */ + *PointerPte = Pfn1->OriginalPte; + + /* We can get rid of this */ + MiReleaseSystemPtes(SysPte, 1, SystemPteSpace); + + /* And dereference the page table frame */ + MiDecrementShareCount(MI_PFN_ELEMENT(Pfn1->u4.PteFrame), Pfn1->u4.PteFrame); + + /* Unlink */ + MiUnlinkPageFromList(Pfn1); + + return PageFrameIndex; +} + PFN_NUMBER NTAPI MiRemoveAnyPage(IN ULONG Color) { PFN_NUMBER PageIndex; PMMPFN Pfn1; + BOOLEAN FromStandbyList = FALSE; /* Make sure PFN lock is held and we have pages */ ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL); @@ -507,23 +586,27 @@ MiRemoveAnyPage(IN ULONG Color) ASSERT_LIST_INVARIANT(&MmZeroedPageListHead); PageIndex = MmZeroedPageListHead.Flink; Color = PageIndex & MmSecondaryColorMask; - ASSERT(PageIndex != LIST_HEAD); if (PageIndex == LIST_HEAD) { - /* FIXME: Should check the standby list */ ASSERT(MmZeroedPageListHead.Total == 0); + /* Pop one page from the standby list and try again */ + PageIndex = MiPopPageFromStandbyList(); + FromStandbyList = TRUE; + ASSERT(PageIndex != LIST_HEAD); } } } } /* Remove the page from its list */ - PageIndex = MiRemovePageByColor(PageIndex, Color); + if (!FromStandbyList) + PageIndex = MiRemovePageByColor(PageIndex, Color); /* Sanity checks */ Pfn1 = MI_PFN_ELEMENT(PageIndex); ASSERT((Pfn1->u3.e1.PageLocation == FreePageList) || - (Pfn1->u3.e1.PageLocation == ZeroedPageList)); + (Pfn1->u3.e1.PageLocation == ZeroedPageList) || + (Pfn1->u3.e1.PageLocation == StandbyPageList)); ASSERT(Pfn1->u3.e2.ReferenceCount == 0); ASSERT(Pfn1->u2.ShareCount == 0); ASSERT_LIST_INVARIANT(&MmFreePageListHead); @@ -540,6 +623,7 @@ MiRemoveZeroPage(IN ULONG Color) PFN_NUMBER PageIndex; PMMPFN Pfn1; BOOLEAN Zero = FALSE; + BOOLEAN FromStandbyList = FALSE; /* Make sure PFN lock is held and we have pages */ ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL); @@ -567,11 +651,13 @@ MiRemoveZeroPage(IN ULONG Color) ASSERT_LIST_INVARIANT(&MmFreePageListHead); PageIndex = MmFreePageListHead.Flink; Color = PageIndex & MmSecondaryColorMask; - ASSERT(PageIndex != LIST_HEAD); if (PageIndex == LIST_HEAD) { - /* FIXME: Should check the standby list */ ASSERT(MmZeroedPageListHead.Total == 0); + /* Pop one page from the standby list and try again */ + PageIndex = MiPopPageFromStandbyList(); + FromStandbyList = TRUE; + ASSERT(PageIndex != LIST_HEAD); } } } @@ -584,10 +670,12 @@ MiRemoveZeroPage(IN ULONG Color) /* Sanity checks */ Pfn1 = MI_PFN_ELEMENT(PageIndex); ASSERT((Pfn1->u3.e1.PageLocation == FreePageList) || - (Pfn1->u3.e1.PageLocation == ZeroedPageList)); + (Pfn1->u3.e1.PageLocation == ZeroedPageList)|| + (Pfn1->u3.e1.PageLocation == StandbyPageList)); /* Remove the page from its list */ - PageIndex = MiRemovePageByColor(PageIndex, Color); + if (!FromStandbyList) + PageIndex = MiRemovePageByColor(PageIndex, Color); ASSERT(Pfn1 == MI_PFN_ELEMENT(PageIndex)); /* Zero it, if needed */ @@ -744,7 +832,6 @@ MiInsertStandbyListAtFront(IN PFN_NUMBER PageFrameIndex) Pfn1 = MI_PFN_ELEMENT(PageFrameIndex); ASSERT(Pfn1->u4.MustBeCached == 0); ASSERT(Pfn1->u3.e2.ReferenceCount == 0); - ASSERT(Pfn1->u3.e1.PrototypePte == 1); ASSERT(Pfn1->u3.e1.Rom != 1); /* One more transition page on a list */ @@ -979,9 +1066,8 @@ MiInsertPageInList(IN PMMPFNLIST ListHead, } else if (ListName == ModifiedPageList) { - /* In ARM3, page must be destined for page file, and not yet written out */ + /* In ARM3, page must be destined for page file */ ASSERT(Pfn1->OriginalPte.u.Soft.Prototype == 0); - ASSERT(Pfn1->OriginalPte.u.Soft.PageFileHigh == 0); /* One more transition page */ MmTransitionSharedPages++; @@ -989,7 +1075,8 @@ MiInsertPageInList(IN PMMPFNLIST ListHead, /* Increment the number of per-process modified pages */ PsGetCurrentProcess()->ModifiedPageCount++; - /* FIXME: Wake up modified page writer if there are not enough free pages */ + /* Wake the page writer thread */ + KeSetEvent(&MpwThreadEvent, IO_NO_INCREMENT, FALSE); } else if (ListName == ModifiedNoWritePageList) { @@ -1060,6 +1147,9 @@ MiInitializePfn(IN PFN_NUMBER PageFrameIndex, ASSERT(PageFrameIndex != 0); Pfn1->u4.PteFrame = PageFrameIndex; + /* HACK until we get working sets rolling */ + Pfn1->Wsle.u1.e1.Age = 0; + /* Increase its share count so we don't get rid of it */ Pfn1 = MI_PFN_ELEMENT(PageFrameIndex); Pfn1->u2.ShareCount++; @@ -1270,32 +1360,8 @@ MiDecrementShareCount(IN PMMPFN Pfn1, /* PFN lock must be held */ ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL); - if (Pfn1->u3.e2.ReferenceCount == 1) - { - /* Is there still a PFN for this page? */ - if (MI_IS_PFN_DELETED(Pfn1) == TRUE) - { - /* Clear the last reference */ - Pfn1->u3.e2.ReferenceCount = 0; - ASSERT(Pfn1->OriginalPte.u.Soft.Prototype == 0); - - /* Mark the page temporarily as valid, we're going to make it free soon */ - Pfn1->u3.e1.PageLocation = ActiveAndValid; - - /* Bring it back into the free list */ - MiInsertPageInFreeList(PageFrameIndex); - } - else - { - /* PFN not yet deleted, drop a ref count */ - MiDecrementReferenceCount(Pfn1, PageFrameIndex); - } - } - else - { - /* Otherwise, just drop the reference count */ - InterlockedDecrement16((PSHORT)&Pfn1->u3.e2.ReferenceCount); - } + /* Drop a ref count */ + MiDecrementReferenceCount(Pfn1, PageFrameIndex); } } @@ -1338,6 +1404,13 @@ MiDecrementReferenceCount(IN PMMPFN Pfn1, /* Did someone set the delete flag? */ if (MI_IS_PFN_DELETED(Pfn1)) { + /* Did we have a Pagefile entry for it */ + if ((Pfn1->OriginalPte.u.Soft.Prototype == 0) + && (Pfn1->OriginalPte.u.Soft.PageFileHigh != 0)) + { + /* Yes. Set it free */ + MiFreePageFileEntry(&Pfn1->OriginalPte); + } /* Insert it into the free list, there's nothing left to do */ MiInsertPageInFreeList(PageFrameIndex); return; Index: ntoskrnl/mm/ARM3/section.c =================================================================== --- ntoskrnl/mm/ARM3/section.c (revision 63977) +++ ntoskrnl/mm/ARM3/section.c (working copy) @@ -680,25 +680,53 @@ MiSegmentDelete(IN PSEGMENT Segment) PageFrameIndex = PFN_FROM_PTE(&TempPte); Pfn1 = MI_PFN_ELEMENT(PageFrameIndex); - /* As this is a paged-backed section, nobody should reference it anymore (no cache or whatever) */ - ASSERT(Pfn1->u3.ReferenceCount == 0); + if (Pfn1->u3.e1.WriteInProgress == 1) + { + KEVENT WriteEvent; + + DPRINT1("Setting the page free while it is paged out!\n"); + + /* Only the page writer thread should hold a reference to it */ + ASSERT(Pfn1->u3.ReferenceCount == 1); - /* And it should be in standby or modified list */ - ASSERT((Pfn1->u3.e1.PageLocation == ModifiedPageList) || (Pfn1->u3.e1.PageLocation == StandbyPageList)); + /* Set the page event */ + KeInitializeEvent(&WriteEvent, NotificationEvent, FALSE); + /* We must be the only ones waiting */ + ASSERT(Pfn1->u1.Event == NULL); + Pfn1->u1.Event = &WriteEvent; - /* Unlink it and put it back in free list */ - MiUnlinkPageFromList(Pfn1); + /* Tell the page-out thread to abort */ + Pfn1->u3.e1.WriteInProgress = 0; + /* This will make the page-out thread to mark the page free */ + MI_SET_PFN_DELETED(Pfn1); - /* Temporarily mark this as active and make it free again */ - Pfn1->u3.e1.PageLocation = ActiveAndValid; - MI_SET_PFN_DELETED(Pfn1); + /* Let the writer thread finish and go along */ + KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); + KeWaitForSingleObject(&WriteEvent, FreePage, KernelMode, FALSE, NULL); + OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); + } + else + { + /* As this is a paged-backed section, nobody should reference it anymore (no cache or whatever) */ + ASSERT(Pfn1->u3.ReferenceCount == 0); - MiInsertPageInFreeList(PageFrameIndex); + /* And it should be in standby or modified list */ + ASSERT((Pfn1->u3.e1.PageLocation == ModifiedPageList) || + (Pfn1->u3.e1.PageLocation == StandbyPageList)); + + /* Unlink it and temporarily take a reference */ + MiUnlinkPageFromList(Pfn1); + Pfn1->u3.e2.ReferenceCount++; + + /* This will put it back in free list and clean properly up */ + MI_SET_PFN_DELETED(Pfn1); + MiDecrementReferenceCount(Pfn1, PageFrameIndex); + } } else if (TempPte.u.Soft.PageFileHigh != 0) { - /* Should not happen for now */ - ASSERT(FALSE); + /* There isn't much to do */ + MiFreePageFileEntry(&TempPte); } } else Index: ntoskrnl/mm/ARM3/virtual.c =================================================================== --- ntoskrnl/mm/ARM3/virtual.c (revision 63977) +++ ntoskrnl/mm/ARM3/virtual.c (working copy) @@ -407,9 +407,8 @@ MiDeletePte(IN PMMPTE PointerPte, /* See if the PTE is valid */ if (TempPte.u.Hard.Valid == 0) { - /* Prototype and paged out PTEs not supported yet */ + /* Prototype PTEs not supported yet */ ASSERT(TempPte.u.Soft.Prototype == 0); - ASSERT((TempPte.u.Soft.PageFileHigh == 0) || (TempPte.u.Soft.Transition == 1)); if (TempPte.u.Soft.Transition) { @@ -435,10 +434,9 @@ MiDeletePte(IN PMMPTE PointerPte, /* And it should be in standby or modified list */ ASSERT((Pfn1->u3.e1.PageLocation == ModifiedPageList) || (Pfn1->u3.e1.PageLocation == StandbyPageList)); - /* Unlink it and temporarily mark it as active */ + /* Unlink it and temporarily take a reference */ MiUnlinkPageFromList(Pfn1); Pfn1->u3.e2.ReferenceCount++; - Pfn1->u3.e1.PageLocation = ActiveAndValid; /* This will put it back in free list and clean properly up */ MI_SET_PFN_DELETED(Pfn1); @@ -446,6 +444,16 @@ MiDeletePte(IN PMMPTE PointerPte, } return; } + + if (TempPte.u.Soft.PageFileHigh != 0) + { + /* Release the page file entry */ + MiFreePageFileEntry(&TempPte); + + /* And that's pretty much it */ + MI_ERASE_PTE(PointerPte); + return; + } } /* Get the PFN entry */ @@ -475,7 +483,6 @@ MiDeletePte(IN PMMPTE PointerPte, } #endif /* Drop the share count on the page table */ - PointerPde = MiPteToPde(PointerPte); MiDecrementShareCount(MiGetPfnEntry(PointerPde->u.Hard.PageFrameNumber), PointerPde->u.Hard.PageFrameNumber); Index: ntoskrnl/mm/balance.c =================================================================== --- ntoskrnl/mm/balance.c (revision 63977) +++ ntoskrnl/mm/balance.c (working copy) @@ -22,22 +22,14 @@ #pragma alloc_text(INIT, MiInitBalancerThread) #endif +#define TEST_PAGING + -/* TYPES ********************************************************************/ -typedef struct _MM_ALLOCATION_REQUEST -{ - PFN_NUMBER Page; - LIST_ENTRY ListEntry; - KEVENT Event; -} -MM_ALLOCATION_REQUEST, *PMM_ALLOCATION_REQUEST; /* GLOBALS ******************************************************************/ MM_MEMORY_CONSUMER MiMemoryConsumers[MC_MAXIMUM]; static ULONG MiMinimumAvailablePages; static ULONG MiNrTotalPages; -static LIST_ENTRY AllocationListHead; -static KSPIN_LOCK AllocationListLock; static ULONG MiMinimumPagesPerRun; static CLIENT_ID MiBalancerThreadId; @@ -53,8 +45,6 @@ NTAPI MmInitializeBalancer(ULONG NrAvailablePages, ULONG NrSystemPages) { memset(MiMemoryConsumers, 0, sizeof(MiMemoryConsumers)); - InitializeListHead(&AllocationListHead); - KeInitializeSpinLock(&AllocationListLock); MiNrTotalPages = NrAvailablePages; @@ -86,18 +76,10 @@ MmInitializeMemoryConsumer(ULONG Consumer, MiMemoryConsumers[Consumer].Trim = Trim; } -VOID -NTAPI -MiZeroPhysicalPage( - IN PFN_NUMBER PageFrameIndex -); - NTSTATUS NTAPI MmReleasePageMemoryConsumer(ULONG Consumer, PFN_NUMBER Page) { - PMM_ALLOCATION_REQUEST Request; - PLIST_ENTRY Entry; KIRQL OldIrql; if (Page == 0) @@ -110,27 +92,12 @@ MmReleasePageMemoryConsumer(ULONG Consumer, PFN_NUMBER Page) { if(Consumer == MC_USER) MmRemoveLRUUserPage(Page); (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed); - if ((Entry = ExInterlockedRemoveHeadList(&AllocationListHead, &AllocationListLock)) == NULL) - { - OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); - MmDereferencePage(Page); - KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); - } - else - { - Request = CONTAINING_RECORD(Entry, MM_ALLOCATION_REQUEST, ListEntry); - MiZeroPhysicalPage(Page); - Request->Page = Page; - KeSetEvent(&Request->Event, IO_NO_INCREMENT, FALSE); - } - } - else - { - OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); - MmDereferencePage(Page); - KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); } + OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); + MmDereferencePage(Page); + KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); + return(STATUS_SUCCESS); } @@ -149,10 +116,13 @@ MiTrimMemoryConsumer(ULONG Consumer, ULONG InitialTarget) return InitialTarget; } - if (MiMemoryConsumers[Consumer].PagesUsed > MiMemoryConsumers[Consumer].PagesTarget) + // if (MiFreeSwapPages == 0) { - /* Consumer page limit exceeded */ - Target = max(Target, MiMemoryConsumers[Consumer].PagesUsed - MiMemoryConsumers[Consumer].PagesTarget); + if (MiMemoryConsumers[Consumer].PagesUsed > MiMemoryConsumers[Consumer].PagesTarget) + { + /* Consumer page limit exceeded */ + Target = max(Target, MiMemoryConsumers[Consumer].PagesUsed - MiMemoryConsumers[Consumer].PagesTarget); + } } if (MmAvailablePages < MiMinimumAvailablePages) { @@ -162,13 +132,6 @@ MiTrimMemoryConsumer(ULONG Consumer, ULONG InitialTarget) if (Target) { - if (!InitialTarget) - { - /* If there was no initial target, - * swap at least MiMinimumPagesPerRun */ - Target = max(Target, MiMinimumPagesPerRun); - } - /* Now swap the pages out */ Status = MiMemoryConsumers[Consumer].Trim(Target, 0, &NrFreedPages); @@ -265,8 +228,9 @@ MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait, * Make sure we don't exceed our individual target. */ PagesUsed = InterlockedIncrementUL(&MiMemoryConsumers[Consumer].PagesUsed); - if (PagesUsed > MiMemoryConsumers[Consumer].PagesTarget && - !MiIsBalancerThread()) + if ((PagesUsed > MiMemoryConsumers[Consumer].PagesTarget) && + !MiIsBalancerThread() && + (MmNumberOfPagingFiles == 0)) { MmRebalanceMemoryConsumers(); } @@ -285,52 +249,6 @@ MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait, } if (Consumer == MC_USER) MmInsertLRULastUserPage(Page); *AllocatedPage = Page; - if (MmAvailablePages < MiMinimumAvailablePages) - MmRebalanceMemoryConsumers(); - return(STATUS_SUCCESS); - } - - /* - * Make sure we don't exceed global targets. - */ - if (MmAvailablePages < MiMinimumAvailablePages) - { - MM_ALLOCATION_REQUEST Request; - - if (!CanWait) - { - (void)InterlockedDecrementUL(&MiMemoryConsumers[Consumer].PagesUsed); - MmRebalanceMemoryConsumers(); - return(STATUS_NO_MEMORY); - } - - /* Insert an allocation request. */ - Request.Page = 0; - KeInitializeEvent(&Request.Event, NotificationEvent, FALSE); - - ExInterlockedInsertTailList(&AllocationListHead, &Request.ListEntry, &AllocationListLock); - MmRebalanceMemoryConsumers(); - - KeWaitForSingleObject(&Request.Event, - 0, - KernelMode, - FALSE, - NULL); - - Page = Request.Page; - if (Page == 0) - { - KeBugCheck(NO_PAGES_AVAILABLE); - } - - if(Consumer == MC_USER) MmInsertLRULastUserPage(Page); - *AllocatedPage = Page; - - if (MmAvailablePages < MiMinimumAvailablePages) - { - MmRebalanceMemoryConsumers(); - } - return(STATUS_SUCCESS); } @@ -347,40 +265,189 @@ MmRequestPageMemoryConsumer(ULONG Consumer, BOOLEAN CanWait, if(Consumer == MC_USER) MmInsertLRULastUserPage(Page); *AllocatedPage = Page; - if (MmAvailablePages < MiMinimumAvailablePages) - { - MmRebalanceMemoryConsumers(); - } - return(STATUS_SUCCESS); } - VOID NTAPI MiBalancerThread(PVOID Unused) { - PVOID WaitObjects[2]; - NTSTATUS Status; - ULONG i; + PVOID WaitObjects[2]; + NTSTATUS Status; + ULONG i; + PVOID LastPagedPoolAddress = MmPagedPoolStart; +#ifdef TEST_PAGING + PULONG PagingTestBuffer; + BOOLEAN TestPage = FALSE; +#endif - WaitObjects[0] = &MiBalancerEvent; - WaitObjects[1] = &MiBalancerTimer; + WaitObjects[0] = &MiBalancerTimer; + WaitObjects[1] = &MiBalancerEvent; - while (1) - { - Status = KeWaitForMultipleObjects(2, - WaitObjects, - WaitAny, - Executive, - KernelMode, - FALSE, - NULL, - NULL); - - if (Status == STATUS_WAIT_0 || Status == STATUS_WAIT_1) +#ifdef TEST_PAGING + PagingTestBuffer = ExAllocatePoolWithTag(PagedPool, PAGE_SIZE, 'tseT'); + DPRINT1("PagingTestBuffer: %p, PTE %p\n", PagingTestBuffer, MiAddressToPte(PagingTestBuffer)); + for (i = 0; i < (PAGE_SIZE / sizeof(ULONG)); i++) + PagingTestBuffer[i] = i; +#endif + + while (1) + { + Status = KeWaitForMultipleObjects( + 2, + WaitObjects, + WaitAny, + Executive, + KernelMode, + FALSE, + NULL, + NULL); + if (NT_SUCCESS(Status)) + { + /* For now, we only age the paged pool, with 1000 pages per run. */ + LONG Count = 100; + PVOID Address = LastPagedPoolAddress; + KIRQL OldIrql; + PMMPTE PointerPte; + PMMPDE PointerPde; + ULONG PdeIndex; + MMPTE TempPte; + PMMPFN Pfn1; + BOOLEAN Flush; + + DPRINT1("MM Balancer: Starting to loop.\n"); + + while (Count-- > 0) + { + Address = (PVOID)((ULONG_PTR) Address + PAGE_SIZE); + + /* Acquire PFN lock while we are cooking */ + OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); + + if (Address > MmPagedPoolEnd) + { + Address = MmPagedPoolStart; + } + + PointerPde = MiAddressToPde(Address); + PointerPte = MiAddressToPte(Address); + + PdeIndex = ((ULONG_PTR)PointerPde & (SYSTEM_PD_SIZE - 1)) / sizeof(MMPTE); + + if (MmSystemPagePtes[PdeIndex].u.Hard.Valid == 0) + TempPte.u.Long = 0; + else + TempPte = *PointerPte; + + if (TempPte.u.Hard.Valid == 0) + { + /* Bad luck, not a paged-in address */ +#ifdef TEST_PAGING + if (PointerPte == MiAddressToPte(PagingTestBuffer)) + { + /* Of course it should not magically become a prototype pte */ + ASSERT(TempPte.u.Soft.Prototype == 0); + if (TempPte.u.Soft.Transition == 0) + { + /* It was paged out! */ + ASSERT(TempPte.u.Soft.PageFileHigh != 0); + TestPage = TRUE; + } + } +#endif + KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); + continue; + } + + Flush = FALSE; + + /* Get the Pfn */ + Pfn1 = MI_PFN_ELEMENT(PFN_FROM_PTE(&TempPte)); + ASSERT(Pfn1->PteAddress == PointerPte); + ASSERT(Pfn1->u3.e1.PrototypePte == 0); + + /* First check if it was written to */ + if (TempPte.u.Hard.Dirty) + Pfn1->u3.e1.Modified = 1; + + /* See if it was accessed since the last time we looked */ + if (TempPte.u.Hard.Accessed) + { + /* Yes! Mark the page as young and fresh again */ + Pfn1->Wsle.u1.e1.Age = 0; + /* Tell the CPU we want to know for the next time */ + TempPte.u.Hard.Accessed = 0; + MI_UPDATE_VALID_PTE(PointerPte, TempPte); + Flush = TRUE; + } + else if (Pfn1->Wsle.u1.e1.Age == 3) + { + /* Page is getting old: Mark the PTE as transition */ + DPRINT1("MM Balancer: putting %p (page %x) as transition.\n", PointerPte, + PFN_FROM_PTE(&TempPte)); + TempPte.u.Hard.Valid = 0; + TempPte.u.Soft.Transition = 1; + TempPte.u.Trans.Protection = MM_READWRITE; + MiDecrementShareCount(Pfn1, PFN_FROM_PTE(&TempPte)); + MI_WRITE_INVALID_PTE(PointerPte, TempPte); + Flush = TRUE; + } + else + { + /* The page is just getting older */ + Pfn1->Wsle.u1.e1.Age++; + } + + /* Flush the TLB entry if we have to */ + if (Flush) + { +#ifdef CONFIG_SMP + // FIXME: Should invalidate entry in every CPU TLB + ASSERT(FALSE); +#endif + KeInvalidateTlbEntry(Address); + } + + /* Unlock, we're done for this address */ + KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); + } + + DPRINT1("MM Balancer: End of the loop.\n"); + +#ifdef TEST_PAGING + if (TestPage) + { + BOOLEAN Passed = TRUE; + DPRINT1("Verifying data integrity of paged out data!\n"); + for (i = 0; i < (PAGE_SIZE / sizeof(ULONG)); i++) + { + if (PagingTestBuffer[i] != i) + Passed = FALSE; + } + if (Passed) + DPRINT1("PASSED! \\o/\n"); + else + DPRINT1("FAILED /o\\\n"); + TestPage = FALSE; + } +#endif + + /* Remember this for the next run */ + LastPagedPoolAddress = Address; + + if (Status == STATUS_WAIT_1) { + /* Balancer thread was woken up by legacy MM */ ULONG InitialTarget = 0; + /* This means we are really starving. Let's see if this still a problem */ + if ((MmAvailablePages >= MmMinimumFreePages) && (MmNumberOfPagingFiles != 0)) + { + /* Yay! No need to trust the old balancer */ + continue; + } + + DPRINT1("Legacy MM balancer in action!\n"); + #if (_MI_PAGING_LEVELS == 2) if (!MiIsBalancerThread()) { @@ -426,11 +493,12 @@ MiBalancerThread(PVOID Unused) } } while (InitialTarget != 0); } - else - { - DPRINT1("KeWaitForMultipleObjects failed, status = %x\n", Status); - KeBugCheck(MEMORY_MANAGEMENT); - } + } + else + { + DPRINT1("KeWaitForMultipleObjects failed, status = %x\n", Status); + KeBugCheck(MEMORY_MANAGEMENT); + } } } Index: ntoskrnl/mm/mminit.c =================================================================== --- ntoskrnl/mm/mminit.c (revision 63977) +++ ntoskrnl/mm/mminit.c (working copy) @@ -299,36 +299,184 @@ VOID NTAPI MmMpwThreadMain(PVOID Parameter) { - NTSTATUS Status; - ULONG PagesWritten; - LARGE_INTEGER Timeout; - - UNREFERENCED_PARAMETER(Parameter); - - Timeout.QuadPart = -50000000; - - for(;;) - { - Status = KeWaitForSingleObject(&MpwThreadEvent, - 0, - KernelMode, - FALSE, - &Timeout); - if (!NT_SUCCESS(Status)) - { - DbgPrint("MpwThread: Wait failed\n"); - KeBugCheck(MEMORY_MANAGEMENT); - return; - } - - PagesWritten = 0; - -#ifndef NEWCC - // XXX arty -- we flush when evicting pages or destorying cache - // sections. - CcRosFlushDirtyPages(128, &PagesWritten, FALSE); -#endif - } + NTSTATUS Status; + PMMPTE SysPte, PteTable; + + UNREFERENCED_PARAMETER(Parameter); + + /* Reserve a PTE for the page table */ + SysPte = MiReserveSystemPtes(1, SystemPteSpace); + ASSERT(SysPte != NULL); + PteTable = MiPteToAddress(SysPte); + + for(;;) + { + /* + * To start working, we wait for two conditions: + * - there are pages to be paged out. + * - We are in a low memory situation. + */ + Status = KeWaitForSingleObject( + &MpwThreadEvent, + WrPageOut, + KernelMode, + FALSE, + NULL); + + DPRINT("THE KRAKEN WAS RELEASED AND WILL PAGE YOUR ASS OUT!\n"); + + if (NT_SUCCESS(Status)) + { + KIRQL OldIrql; + PFN_NUMBER PageFrameIndex; + PMMPFN Pfn; + MMPTE TempPte, PdePte; + PMMPTE PointerPte; + BOOLEAN PageFileEntryFromPage = FALSE; + + /* Lock the PFN database */ + OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); + + /* The true main loop */ + while ((MmTotalPagesForPagingFile != 0) && (MiFreeSwapPages != 0)) + { + /* Get the first page from the list */ + PageFrameIndex = MmModifiedPageListByColor[0].Flink; + + /* Get The Pfn */ + Pfn = MI_PFN_ELEMENT(PageFrameIndex); + + /* Some things which must always hold */ + ASSERT(Pfn->OriginalPte.u.Soft.Transition == 0); + ASSERT(Pfn->u3.ReferenceCount == 0); + + /* And some that are not yet supported */ + ASSERT(Pfn->OriginalPte.u.Soft.Prototype == 0); + + TempPte.u.Long = 0; + + /* Maybe this is not the first time */ + if ((Pfn->OriginalPte.u.Soft.Prototype == 0) && + (Pfn->OriginalPte.u.Soft.PageFileHigh != 0)) + { + /* Use that */ + TempPte = Pfn->OriginalPte; + PageFileEntryFromPage = TRUE; + } + else + { + MiReservePageFileEntry(&TempPte); + } + + /* Get it out of the list and reference it */ + MiUnlinkPageFromList(Pfn); + MiReferenceUnusedPageAndBumpLockCount(Pfn); + + ASSERT(Pfn->u3.e1.PageLocation == ModifiedPageList); + + /* Mark it as write in progress */ + Pfn->u3.e1.WriteInProgress = 1; + Pfn->u1.Event = NULL; + + /* Release the PFN lock while we are writing */ + KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); + + /* Do the actual write */ + Status = MiWritePageFile(PageFrameIndex, &TempPte); + + /* Get the lock again */ + OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); + + /* Get a mapping to the page table */ + MI_MAKE_HARDWARE_PTE_KERNEL(&PdePte, + SysPte, + MM_READWRITE, + Pfn->u4.PteFrame); + MI_WRITE_VALID_PTE(SysPte, PdePte); + + /* Finally get a pointer to the PTE this page represents */ + PointerPte = &PteTable[MiAddressToPteOffset(MiPteToAddress(Pfn->PteAddress))]; + + /* Get relevant values from the original PTE */ + TempPte.u.Soft.Protection = PointerPte->u.Soft.Protection; + + /* Maybe someone aborted the operation */ + if (Pfn->u3.e1.WriteInProgress == 0) + { + DPRINT1("Someone aborted the page-out operation!\n"); + /* Just set the event and let the waiter go along */ + ASSERT(Pfn->u1.Event != NULL); + KeSetEvent(Pfn->u1.Event, IO_NO_INCREMENT, FALSE); + Pfn->u1.Event = NULL; + /* This is now useless */ + if (!PageFileEntryFromPage) + MiFreePageFileEntry(&TempPte); + } + /* Maybe someone tried to access the page while we were not looking */ + else if (Pfn->u1.Event != NULL) + { + DPRINT1("Page fault occured while we were paging out!\n"); + + /* So the page fault handler marked the page as unmodified */ + ASSERT(Pfn->u3.e1.Modified == 0); + + /* Save the pagefile entry for this page */ + Pfn->OriginalPte = TempPte; + KeSetEvent(Pfn->u1.Event, IO_NO_INCREMENT, FALSE); + Pfn->u1.Event = NULL; + } + else if (!NT_SUCCESS(Status)) + { + DPRINT1("Failed to write page to pagefile!\n"); + /* This is now useless */ + if (!PageFileEntryFromPage) + MiFreePageFileEntry(&TempPte); + /* MiDereferencePfnAndDropLockCount will put it back on the tail of the list */ + } + else + { + DPRINT1("Page %x successfully paged out. PTE pointer %p (-> %x)\n", + PageFrameIndex, Pfn->PteAddress, TempPte.u.Long); + + /* Of course it must already be invalid */ + ASSERT(PointerPte->u.Hard.Valid == 0); + + /* And be in transition */ + ASSERT(PointerPte->u.Soft.Transition == 1); + + /* So the PTE is now officially paged out */ + MI_WRITE_INVALID_PTE(PointerPte, TempPte); + + /* And the pagefile entry belongs to the PTE, not to the page! */ + Pfn->OriginalPte.u.Long = 0; + + /* And dereference the page table frame */ + MiDecrementShareCount(MI_PFN_ELEMENT(Pfn->u4.PteFrame), Pfn->u4.PteFrame); + /* We can finally make it available for real */ + MI_SET_PFN_DELETED(Pfn); + } + + /* We're done with this */ + Pfn->u3.e1.WriteInProgress = 0; + + /* This will put it back in the free list */ + MiDereferencePfnAndDropLockCount(Pfn); + + /* Unmap the Page Table */ + MI_ERASE_PTE(SysPte); + KeInvalidateTlbEntry(PteTable); + } + + /* We're done for this run */ + KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); + } + else + { + DPRINT1("MpwThread: Wait failed\n"); + KeBugCheck(MEMORY_MANAGEMENT); + return; + } + } } NTSTATUS Index: ntoskrnl/mm/pagefile.c =================================================================== --- ntoskrnl/mm/pagefile.c (revision 63977) +++ ntoskrnl/mm/pagefile.c (working copy) @@ -73,9 +73,6 @@ RETRIEVEL_DESCRIPTOR_LIST, *PRETRIEVEL_DESCRIPTOR_LIST; /* List of paging files, both used and free */ static PPAGINGFILE PagingFileList[MAX_PAGING_FILES]; -/* Lock for examining the list of paging files */ -static KSPIN_LOCK PagingFileListLock; - /* Number of paging files */ ULONG MmNumberOfPagingFiles; @@ -221,57 +218,12 @@ NTSTATUS NTAPI MmWriteToSwapPage(SWAPENTRY SwapEntry, PFN_NUMBER Page) { - ULONG i; - ULONG_PTR offset; - LARGE_INTEGER file_offset; - IO_STATUS_BLOCK Iosb; - NTSTATUS Status; - KEVENT Event; - UCHAR MdlBase[sizeof(MDL) + sizeof(ULONG)]; - PMDL Mdl = (PMDL)MdlBase; + MMPTE TempPte; - DPRINT("MmWriteToSwapPage\n"); - - if (SwapEntry == 0) - { - KeBugCheck(MEMORY_MANAGEMENT); - return(STATUS_UNSUCCESSFUL); - } - - i = FILE_FROM_ENTRY(SwapEntry); - offset = OFFSET_FROM_ENTRY(SwapEntry); - - if (PagingFileList[i]->FileObject == NULL || - PagingFileList[i]->FileObject->DeviceObject == NULL) - { - DPRINT1("Bad paging file 0x%.8X\n", SwapEntry); - KeBugCheck(MEMORY_MANAGEMENT); - } - - MmInitializeMdl(Mdl, NULL, PAGE_SIZE); - MmBuildMdlFromPages(Mdl, &Page); - Mdl->MdlFlags |= MDL_PAGES_LOCKED; - - file_offset.QuadPart = offset * PAGE_SIZE; - file_offset = MmGetOffsetPageFile(PagingFileList[i]->RetrievalPointers, file_offset); - - KeInitializeEvent(&Event, NotificationEvent, FALSE); - Status = IoSynchronousPageWrite(PagingFileList[i]->FileObject, - Mdl, - &file_offset, - &Event, - &Iosb); - if (Status == STATUS_PENDING) - { - KeWaitForSingleObject(&Event, Executive, KernelMode, FALSE, NULL); - Status = Iosb.Status; - } - - if (Mdl->MdlFlags & MDL_MAPPED_TO_SYSTEM_VA) - { - MmUnmapLockedPages (Mdl->MappedSystemVa, Mdl); - } - return(Status); + TempPte.u.Long = 0; + TempPte.u.Soft.PageFileHigh = OFFSET_FROM_ENTRY(SwapEntry); + TempPte.u.Soft.PageFileLow = FILE_FROM_ENTRY(SwapEntry); + return MiWritePageFile(Page, &TempPte); } @@ -279,18 +231,25 @@ NTSTATUS NTAPI MmReadFromSwapPage(SWAPENTRY SwapEntry, PFN_NUMBER Page) { - return MiReadPageFile(Page, FILE_FROM_ENTRY(SwapEntry), OFFSET_FROM_ENTRY(SwapEntry)); + MMPTE TempPte; + + TempPte.u.Long = 0; + TempPte.u.Soft.PageFileHigh = OFFSET_FROM_ENTRY(SwapEntry); + TempPte.u.Soft.PageFileLow = FILE_FROM_ENTRY(SwapEntry); + return MiReadPageFile(Page, &TempPte); } NTSTATUS NTAPI MiReadPageFile( _In_ PFN_NUMBER Page, - _In_ ULONG PageFileIndex, - _In_ ULONG_PTR PageFileOffset) + _In_ const MMPTE* PointerPte +) { - LARGE_INTEGER file_offset; + LARGE_INTEGER FileOffset; IO_STATUS_BLOCK Iosb; + ULONG PageFileIndex = PointerPte->u.Soft.PageFileLow; + ULONG_PTR PageFileOffset = PointerPte->u.Soft.PageFileHigh; NTSTATUS Status; KEVENT Event; UCHAR MdlBase[sizeof(MDL) + sizeof(ULONG)]; @@ -305,6 +264,9 @@ MiReadPageFile( return(STATUS_UNSUCCESSFUL); } + /* Normalize it */ + PageFileOffset--; + ASSERT(PageFileIndex < MAX_PAGING_FILES); PagingFile = PagingFileList[PageFileIndex]; @@ -319,13 +281,13 @@ MiReadPageFile( MmBuildMdlFromPages(Mdl, &Page); Mdl->MdlFlags |= MDL_PAGES_LOCKED; - file_offset.QuadPart = PageFileOffset * PAGE_SIZE; - file_offset = MmGetOffsetPageFile(PagingFile->RetrievalPointers, file_offset); + FileOffset.QuadPart = PageFileOffset * PAGE_SIZE; + FileOffset = MmGetOffsetPageFile(PagingFile->RetrievalPointers, FileOffset); KeInitializeEvent(&Event, NotificationEvent, FALSE); Status = IoPageRead(PagingFile->FileObject, Mdl, - &file_offset, + &FileOffset, &Event, &Iosb); if (Status == STATUS_PENDING) @@ -340,124 +302,247 @@ MiReadPageFile( return(Status); } +NTSTATUS +NTAPI +MiWritePageFile( + _In_ PFN_NUMBER Page, + _In_ const MMPTE* PointerPte +) +{ + ULONG PageFileIndex = PointerPte->u.Soft.PageFileLow; + ULONG_PTR PageFileOffset = PointerPte->u.Soft.PageFileHigh; + LARGE_INTEGER FileOffset; + IO_STATUS_BLOCK Iosb; + NTSTATUS Status; + KEVENT Event; + UCHAR MdlBase[sizeof(MDL) + sizeof(ULONG)]; + PMDL Mdl = (PMDL) MdlBase; + PPAGINGFILE PagingFile; + + /* The PTE must already be setup to point to a pagefile entry */ + ASSERT(PointerPte->u.Hard.Valid == 0); + ASSERT(PointerPte->u.Soft.Transition == 0); + ASSERT(PointerPte->u.Soft.Prototype == 0); + + DPRINT("MiWriteSwapFile\n"); + + if (PageFileOffset == 0) + { + KeBugCheck(MEMORY_MANAGEMENT); + return (STATUS_UNSUCCESSFUL); + } + + /* Normalize it */ + PageFileOffset--; + + ASSERT(PageFileIndex < MAX_PAGING_FILES); + + PagingFile = PagingFileList[PageFileIndex]; + + if (PagingFile->FileObject == NULL || PagingFile->FileObject->DeviceObject == NULL) + { + DPRINT1("Bad paging file %u\n", PageFileIndex); + KeBugCheck(MEMORY_MANAGEMENT); + } + + MmInitializeMdl(Mdl, NULL, PAGE_SIZE); + MmBuildMdlFromPages(Mdl, &Page); + Mdl->MdlFlags |= MDL_PAGES_LOCKED; + + FileOffset.QuadPart = PageFileOffset * PAGE_SIZE; + FileOffset = MmGetOffsetPageFile(PagingFile->RetrievalPointers, FileOffset); + + KeInitializeEvent(&Event, NotificationEvent, FALSE); + Status = IoSynchronousPageWrite(PagingFile->FileObject, + Mdl, + &FileOffset, + &Event, + &Iosb); + if (Status == STATUS_PENDING) + { + KeWaitForSingleObject(&Event, Executive, KernelMode, FALSE, NULL); + Status = Iosb.Status; + } + if (Mdl->MdlFlags & MDL_MAPPED_TO_SYSTEM_VA) + { + MmUnmapLockedPages(Mdl->MappedSystemVa, Mdl); + } + return Status; +} + VOID -INIT_FUNCTION NTAPI -MmInitPagingFile(VOID) +MiFreePageFileEntry( + _In_ PMMPTE PointerPte +) { - ULONG i; + ULONG PageFileIndex; + ULONG_PTR PageFileOffset, MapOffset; + PPAGINGFILE PageFile; + ULONG BitSet; - KeInitializeSpinLock(&PagingFileListLock); + /* Some sanity checks */ + ASSERT(PointerPte->u.Hard.Valid == 0); + ASSERT(PointerPte->u.Soft.Transition == 0); + ASSERT(PointerPte->u.Soft.Prototype == 0); - MiFreeSwapPages = 0; - MiUsedSwapPages = 0; - MiReservedSwapPages = 0; + /* We must hold the PFN lock */ + ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL); - for (i = 0; i < MAX_PAGING_FILES; i++) - { - PagingFileList[i] = NULL; - } - MmNumberOfPagingFiles = 0; + PageFileIndex = PointerPte->u.Soft.PageFileLow; + PageFileOffset = PointerPte->u.Soft.PageFileHigh - 1; + MapOffset = PageFileOffset >> 5; + BitSet = 1 << (PageFileOffset & 0x1F); + + DPRINT1("Releasing page file entry %u, %lu\n", PageFileIndex, PageFileOffset); + + ASSERT(PageFileIndex < MAX_PAGING_FILES); + + PageFile = PagingFileList[PageFileIndex]; + + /* And also lock this pagefile */ + KeAcquireSpinLockAtDpcLevel(&PageFile->AllocMapLock); + + /* It must be already in use */ + ASSERT(PageFile->AllocMap[MapOffset] & BitSet); + PageFile->AllocMap[MapOffset] ^= BitSet; + + /* One more */ + PageFile->FreePages++; + PageFile->UsedPages--; + + MiFreeSwapPages++; + MiUsedSwapPages--; + + /* Done */ + KeReleaseSpinLockFromDpcLevel(&PageFile->AllocMapLock); } -static ULONG -MiAllocPageFromPagingFile(PPAGINGFILE PagingFile) +NTSTATUS +NTAPI +MiReservePageFileEntry( + _Out_ PMMPTE PointerPte +) { - KIRQL oldIrql; - ULONG i, j; + ULONG PageFileIndex; + ULONG_PTR PageFileOffset = ~1, MapOffset; + PPAGINGFILE PageFile; - KeAcquireSpinLock(&PagingFile->AllocMapLock, &oldIrql); + /* We must hold the PFN lock */ + ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL); - for (i = 0; i < PagingFile->AllocMapSize; i++) - { - for (j = 0; j < 32; j++) - { - if (!(PagingFile->AllocMap[i] & (1 << j))) - { - PagingFile->AllocMap[i] |= (1 << j); - PagingFile->UsedPages++; - PagingFile->FreePages--; - KeReleaseSpinLock(&PagingFile->AllocMapLock, oldIrql); - return((i * 32) + j); - } - } - } + for (PageFileIndex = 0; PageFileIndex < MAX_PAGING_FILES; PageFileIndex++) + { + if (PagingFileList[PageFileIndex] == NULL) + continue; + if (PagingFileList[PageFileIndex]->FreePages != 0) + break; + } + + if (PageFileIndex == MAX_PAGING_FILES) + { + DPRINT1("Pagefiles are full!\n"); + return STATUS_PAGEFILE_QUOTA; + } - KeReleaseSpinLock(&PagingFile->AllocMapLock, oldIrql); - return(0xFFFFFFFF); + PageFile = PagingFileList[PageFileIndex]; + + /* And also lock this page */ + KeAcquireSpinLockAtDpcLevel(&PageFile->AllocMapLock); + + /* Find a free entry in this page file */ + // FIXME: Use a bitmap... + for (MapOffset = 0; MapOffset < PageFile->AllocMapSize; MapOffset++) + { + unsigned j; + PULONG BitSet = &PageFile->AllocMap[MapOffset]; + for (j = 0; j < 32; j++) + { + if ((*BitSet & (1 << j)) == 0) + { + /* Mark it as being in use */ + *BitSet |= (1 << j); + break; + } + } + + if (j != 32) + { + PageFileOffset = (MapOffset << 5) | j; + break; + } + } + + /* We must have found something as the file was marked as having free entries */ + ASSERT(PageFileOffset != ~1); + + DPRINT1("Reserving page file entry %u, %lu\n", PageFileIndex, PageFileOffset); + + /* Done */ + KeReleaseSpinLockFromDpcLevel(&PageFile->AllocMapLock); + + /* We rely on PageFileHigh being != 0 */ + PointerPte->u.Soft.PageFileHigh = PageFileOffset + 1; + PointerPte->u.Soft.PageFileLow = PageFileIndex; + + return STATUS_SUCCESS; } VOID +INIT_FUNCTION NTAPI -MmFreeSwapPage(SWAPENTRY Entry) +MmInitPagingFile(VOID) { ULONG i; - ULONG_PTR off; - KIRQL oldIrql; - i = FILE_FROM_ENTRY(Entry); - off = OFFSET_FROM_ENTRY(Entry); + MiFreeSwapPages = 0; + MiUsedSwapPages = 0; + MiReservedSwapPages = 0; - KeAcquireSpinLock(&PagingFileListLock, &oldIrql); - if (PagingFileList[i] == NULL) + for (i = 0; i < MAX_PAGING_FILES; i++) { - KeBugCheck(MEMORY_MANAGEMENT); + PagingFileList[i] = NULL; } - KeAcquireSpinLockAtDpcLevel(&PagingFileList[i]->AllocMapLock); - - PagingFileList[i]->AllocMap[off >> 5] &= (~(1 << (off % 32))); + MmNumberOfPagingFiles = 0; +} - PagingFileList[i]->FreePages++; - PagingFileList[i]->UsedPages--; +VOID +NTAPI +MmFreeSwapPage(SWAPENTRY Entry) +{ + MMPTE TempPte; + KIRQL OldIrql; - MiFreeSwapPages++; - MiUsedSwapPages--; + TempPte.u.Long = 0; + TempPte.u.Soft.PageFileLow = FILE_FROM_ENTRY(Entry); + TempPte.u.Soft.PageFileHigh = OFFSET_FROM_ENTRY(Entry); - KeReleaseSpinLockFromDpcLevel(&PagingFileList[i]->AllocMapLock); - KeReleaseSpinLock(&PagingFileListLock, oldIrql); + OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); + MiFreePageFileEntry(&TempPte); + KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); } SWAPENTRY NTAPI MmAllocSwapPage(VOID) { - KIRQL oldIrql; - ULONG i; - ULONG off; - SWAPENTRY entry; + MMPTE TempPte; + SWAPENTRY entry; + NTSTATUS Status; + KIRQL OldIrql; - KeAcquireSpinLock(&PagingFileListLock, &oldIrql); + TempPte.u.Long = 0; - if (MiFreeSwapPages == 0) - { - KeReleaseSpinLock(&PagingFileListLock, oldIrql); - return(0); - } + OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); + Status = MiReservePageFileEntry(&TempPte); + KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); - for (i = 0; i < MAX_PAGING_FILES; i++) - { - if (PagingFileList[i] != NULL && - PagingFileList[i]->FreePages >= 1) - { - off = MiAllocPageFromPagingFile(PagingFileList[i]); - if (off == 0xFFFFFFFF) - { - KeBugCheck(MEMORY_MANAGEMENT); - KeReleaseSpinLock(&PagingFileListLock, oldIrql); - return(STATUS_UNSUCCESSFUL); - } - MiUsedSwapPages++; - MiFreeSwapPages--; - KeReleaseSpinLock(&PagingFileListLock, oldIrql); + if (!NT_SUCCESS(Status)) + return 0; - entry = ENTRY_FROM_FILE_OFFSET(i, off); - return(entry); - } - } + entry = ENTRY_FROM_FILE_OFFSET(TempPte.u.Soft.PageFileLow, TempPte.u.Soft.PageFileHigh); - KeReleaseSpinLock(&PagingFileListLock, oldIrql); - KeBugCheck(MEMORY_MANAGEMENT); - return(0); + return entry; } static PRETRIEVEL_DESCRIPTOR_LIST FASTCALL @@ -488,7 +573,7 @@ NtCreatePagingFile(IN PUNICODE_STRING FileName, IO_STATUS_BLOCK IoStatus; PFILE_OBJECT FileObject; PPAGINGFILE PagingFile; - KIRQL oldIrql; + KIRQL OldIrql; ULONG AllocMapSize; FILE_FS_SIZE_INFORMATION FsSizeInformation; PRETRIEVEL_DESCRIPTOR_LIST RetDescList; @@ -799,7 +884,7 @@ NtCreatePagingFile(IN PUNICODE_STRING FileName, PagingFile->RetrievalPointers->Extents[i].NextVcn.QuadPart *= BytesPerAllocationUnit; } - KeAcquireSpinLock(&PagingFileListLock, &oldIrql); + OldIrql = KeAcquireQueuedSpinLock(LockQueuePfnLock); for (i = 0; i < MAX_PAGING_FILES; i++) { if (PagingFileList[i] == NULL) @@ -810,7 +895,7 @@ NtCreatePagingFile(IN PUNICODE_STRING FileName, } MiFreeSwapPages = MiFreeSwapPages + PagingFile->FreePages; MmNumberOfPagingFiles++; - KeReleaseSpinLock(&PagingFileListLock, oldIrql); + KeReleaseQueuedSpinLock(LockQueuePfnLock, OldIrql); ZwClose(FileHandle); Index: ntoskrnl/mm/rmap.c =================================================================== --- ntoskrnl/mm/rmap.c (revision 63977) +++ ntoskrnl/mm/rmap.c (working copy) @@ -146,7 +146,7 @@ MmPageOutPhysicalAddress(PFN_NUMBER Page) */ Entry = MmGetPageEntrySectionSegment(MemoryArea->Data.SectionData.Segment, (PLARGE_INTEGER)&Offset); - if (Entry && IS_SWAP_FROM_SSE(Entry) && SWAPENTRY_FROM_SSE(Entry) == MM_WAIT_ENTRY) + if (/*Entry && */IS_SWAP_FROM_SSE(Entry)/* && SWAPENTRY_FROM_SSE(Entry) == MM_WAIT_ENTRY*/) { MmUnlockSectionSegment(MemoryArea->Data.SectionData.Segment); MmUnlockAddressSpace(AddressSpace); @@ -158,6 +158,7 @@ MmPageOutPhysicalAddress(PFN_NUMBER Page) return(STATUS_UNSUCCESSFUL); } + /* Put a wait entry if this page is not private to the address space */ MmSetPageEntrySectionSegment(MemoryArea->Data.SectionData.Segment, (PLARGE_INTEGER)&Offset, MAKE_SWAP_SSE(MM_WAIT_ENTRY)); /* Index: ntoskrnl/mm/section.c =================================================================== --- ntoskrnl/mm/section.c (revision 63977) +++ ntoskrnl/mm/section.c (working copy) @@ -1859,6 +1859,7 @@ MmPageOutDeleteMapping(PVOID Context, PEPROCESS Process, PVOID Address) if (PageOutContext->Private) { + DPRINT1("Releasing private page %x.\n", Page); MmReleasePageMemoryConsumer(MC_USER, Page); } } @@ -1955,6 +1956,7 @@ MmPageOutSectionView(PMMSUPPORT AddressSpace, { DPRINT("Cannot page out locked section page: 0x%lu (RefCount: %lu)\n", Page, MmGetReferenceCountPage(Page)); + ASSERT(PFN_FROM_SSE(Entry) == Page); MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, Entry); MmUnlockSectionSegment(Context.Segment); return STATUS_UNSUCCESSFUL; @@ -2242,8 +2244,6 @@ MmPageOutSectionView(PMMSUPPORT AddressSpace, Status = MmCreatePageFileMapping(Process, Address, SwapEntry); - /* We had placed a wait entry upon entry ... replace it before leaving */ - MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, Entry); MmUnlockSectionSegment(Context.Segment); MmUnlockAddressSpace(AddressSpace); if (!NT_SUCCESS(Status))