Index: ntoskrnl/cache/cachesub.c =================================================================== --- ntoskrnl/cache/cachesub.c (revision 55684) +++ ntoskrnl/cache/cachesub.c (working copy) @@ -12,7 +12,7 @@ #include #include "newcc.h" #include "section/newmm.h" -#define NDEBUG +//#define NDEBUG #include /* STRUCTURES *****************************************************************/ @@ -143,10 +143,12 @@ PNOCC_BCB Bcb = NULL; LARGE_INTEGER LowerBound, UpperBound; PLIST_ENTRY ListEntry; - IO_STATUS_BLOCK IOSB = {0}; + IO_STATUS_BLOCK IOSB; - DPRINT1("CcFlushCache (while file) (%s:%d)\n", File, Line); + RtlZeroMemory(&IOSB, sizeof(IO_STATUS_BLOCK)); + DPRINT("CcFlushCache (while file) (%s:%d)\n", File, Line); + if (FileOffset && Length) { LowerBound.QuadPart = FileOffset->QuadPart; @@ -288,7 +290,8 @@ NTAPI CcShutdownSystem() { - ULONG i; + ULONG i, Result; + NTSTATUS Status; DPRINT1("CC: Shutdown\n"); @@ -308,7 +311,10 @@ } } - DPRINT1("Done\n"); + // Evict all section pages + Status = MiRosTrimCache(~0, 0, &Result); + + DPRINT1("Done (Evicted %d, Status %x)\n", Result, Status); } Index: ntoskrnl/cache/copysup.c =================================================================== --- ntoskrnl/cache/copysup.c (revision 55684) +++ ntoskrnl/cache/copysup.c (working copy) @@ -11,7 +11,7 @@ #include #include "newcc.h" #include "section/newmm.h" -#define NDEBUG +//#define NDEBUG #include /* GLOBALS ********************************************************************/ @@ -80,7 +80,7 @@ return FALSE; } - DPRINT1("Copying %d bytes at %08x%08x\n", ReadLen, CacheOffset.HighPart, CacheOffset.LowPart); + DPRINT("Copying %d bytes at %08x%08x\n", ReadLen, CacheOffset.HighPart, CacheOffset.LowPart); RtlCopyMemory (BufferTarget, ReadBuffer, @@ -159,7 +159,7 @@ DPRINT1("CcPreparePinWrite Failed?\n"); if (Wait) RtlRaiseStatus(STATUS_NOT_MAPPED_DATA); else return FALSE; } - DPRINT1("Copying actual memory to BCB#%x (@%x) (from buffer at %x)\n", Bcb - CcCacheSections, WriteBuf, Bcb->BaseAddress); + DPRINT("Copying actual memory to BCB#%x (@%x) (from buffer at %x)\n", Bcb - CcCacheSections, WriteBuf, Bcb->BaseAddress); //MiZeroFillSection(WriteBuf, &CurrentOffset, WriteLen); RtlCopyMemory(WriteBuf, ((PCHAR)Buffer) + Count, WriteLen); Index: ntoskrnl/cache/fssup.c =================================================================== --- ntoskrnl/cache/fssup.c (revision 55684) +++ ntoskrnl/cache/fssup.c (working copy) @@ -12,7 +12,7 @@ #include #include "newcc.h" #include "section/newmm.h" -#define NDEBUG +//#define NDEBUG #include /* GLOBALS ********************************************************************/ @@ -38,44 +38,6 @@ /* FUNCTIONS ******************************************************************/ -// Interact with legacy balance manager for now -// This can fall away when our section implementation supports -// demand paging properly -NTSTATUS -CcRosTrimCache(ULONG Target, ULONG Priority, PULONG NrFreed) -{ - ULONG i, Freed, BcbHead; - - *NrFreed = 0; - - for (i = 0; i < CACHE_NUM_SECTIONS; i++) { - BcbHead = (i+CcCacheClockHand) % CACHE_NUM_SECTIONS; - - // Reference a cache stripe so it won't go away - CcpLock(); - if (CcCacheSections[BcbHead].BaseAddress) { - CcpReferenceCache(BcbHead); - CcpUnlock(); - } else { - CcpUnlock(); - continue; - } - - // Defer to MM to try recovering pages from it - Freed = MiCacheEvictPages - (CcCacheSections[BcbHead].BaseAddress, Target); - - Target -= Freed; - *NrFreed += Freed; - - CcpLock(); - CcpUnpinData(&CcCacheSections[BcbHead], TRUE); - CcpUnlock(); - } - - return STATUS_SUCCESS; -} - BOOLEAN NTAPI CcInitializeCacheManager(VOID) @@ -99,15 +61,7 @@ CcCacheBitmap->SizeOfBitMap = ROUND_UP(CACHE_NUM_SECTIONS, 32); DPRINT("Cache has %d entries\n", CcCacheBitmap->SizeOfBitMap); ExInitializeFastMutex(&CcMutex); - ExInitializeFastMutex(&GlobalPageOperation); - // MM stub - KeInitializeEvent(&MmWaitPageEvent, SynchronizationEvent, FALSE); - - // Until we're fully demand paged, we can do things the old way through - // the balance manager - MmInitializeMemoryConsumer(MC_CACHE, CcRosTrimCache); - return TRUE; } @@ -268,9 +222,9 @@ RemoveEntryList(&PrivateCacheMap->ListEntry); if (IsListEmpty(&PrivateCacheMap->Map->PrivateCacheMaps)) { - while (!IsListEmpty(&Map->AssociatedBcb)) + while (!IsListEmpty(&PrivateCacheMap->Map->AssociatedBcb)) { - PNOCC_BCB Bcb = CONTAINING_RECORD(Map->AssociatedBcb.Flink, NOCC_BCB, ThisFileList); + PNOCC_BCB Bcb = CONTAINING_RECORD(PrivateCacheMap->Map->AssociatedBcb.Flink, NOCC_BCB, ThisFileList); DPRINT("Evicting cache stripe #%x\n", Bcb - CcCacheSections); Bcb->RefCount = 1; CcpDereferenceCache(Bcb - CcCacheSections, TRUE); @@ -383,7 +337,7 @@ { ToWrite = MIN(UpperBound.QuadPart - LowerBound.QuadPart, (PAGE_SIZE - LowerBound.QuadPart) & (PAGE_SIZE - 1)); DPRINT("Zero last half %08x%08x %x\n", Target.u.HighPart, Target.u.LowPart, ToWrite); - Status = MiSimpleRead(FileObject, &Target, ZeroBuf, PAGE_SIZE, &IOSB); + Status = MiSimpleRead(FileObject, &Target, ZeroBuf, PAGE_SIZE, TRUE, &IOSB); if (!NT_SUCCESS(Status)) { ExFreePool(ZeroBuf); @@ -419,7 +373,7 @@ { ToWrite = UpperBound.QuadPart - Target.QuadPart; DPRINT("Zero first half %08x%08x %x\n", Target.u.HighPart, Target.u.LowPart, ToWrite); - Status = MiSimpleRead(FileObject, &Target, ZeroBuf, PAGE_SIZE, &IOSB); + Status = MiSimpleRead(FileObject, &Target, ZeroBuf, PAGE_SIZE, TRUE, &IOSB); if (!NT_SUCCESS(Status)) { ExFreePool(ZeroBuf); Index: ntoskrnl/cache/lazyrite.c =================================================================== --- ntoskrnl/cache/lazyrite.c (revision 55684) +++ ntoskrnl/cache/lazyrite.c (working copy) @@ -10,7 +10,7 @@ #include #include "newcc.h" -#define NDEBUG +//#define NDEBUG #include /* GLOBALS ********************************************************************/ Index: ntoskrnl/cache/logsup.c =================================================================== --- ntoskrnl/cache/logsup.c (revision 55684) +++ ntoskrnl/cache/logsup.c (working copy) @@ -10,7 +10,7 @@ #include #include "newcc.h" -#define NDEBUG +//#define NDEBUG #include /* GLOBALS ********************************************************************/ Index: ntoskrnl/cache/mdlsup.c =================================================================== --- ntoskrnl/cache/mdlsup.c (revision 55684) +++ ntoskrnl/cache/mdlsup.c (working copy) @@ -10,7 +10,7 @@ #include #include "newcc.h" -#define NDEBUG +//#define NDEBUG #include /* GLOBALS ********************************************************************/ Index: ntoskrnl/cache/newcc.h =================================================================== --- ntoskrnl/cache/newcc.h (revision 55684) +++ ntoskrnl/cache/newcc.h (working copy) @@ -1,14 +1,12 @@ #pragma once -struct _MM_CACHE_SECTION_SEGMENT; - typedef struct _NOCC_BCB { /* Public part */ PUBLIC_BCB Bcb; struct _NOCC_CACHE_MAP *Map; - struct _MM_CACHE_SECTION_SEGMENT *SectionObject; + PROS_SECTION_OBJECT SectionObject; LARGE_INTEGER FileOffset; ULONG Length; PVOID BaseAddress; Index: ntoskrnl/cache/pinsup.c =================================================================== --- ntoskrnl/cache/pinsup.c (revision 55684) +++ ntoskrnl/cache/pinsup.c (working copy) @@ -59,7 +59,7 @@ (PFILE_OBJECT FileObject, ULONG Length, ULONG Protect, - PMM_CACHE_SECTION_SEGMENT *Result) + PROS_SECTION_OBJECT *Result) { NTSTATUS Status; LARGE_INTEGER MaxSize; @@ -68,13 +68,14 @@ DPRINT("Making Section for File %x\n", FileObject); DPRINT("File name %wZ\n", &FileObject->FileName); - Status = MmCreateCacheSection - (Result, + Status = MmCreateSection + ((PVOID*)Result, STANDARD_RIGHTS_REQUIRED, NULL, &MaxSize, Protect, SEC_RESERVE | SEC_CACHE, + NULL, FileObject); return Status; @@ -86,7 +87,7 @@ PVOID ToUnmap; LARGE_INTEGER FileOffset; LARGE_INTEGER MapSize; - PMM_CACHE_SECTION_SEGMENT ToDeref; + PROS_SECTION_OBJECT ToDeref; PACQUIRE_FOR_LAZY_WRITE AcquireForLazyWrite; PRELEASE_FROM_LAZY_WRITE ReleaseFromLazyWrite; PVOID LazyContext; @@ -98,11 +99,8 @@ { PWORK_QUEUE_WITH_CONTEXT WorkItem = (PWORK_QUEUE_WITH_CONTEXT)Context; DPRINT("Unmapping (finally) %x\n", WorkItem->ToUnmap); - WorkItem->AcquireForLazyWrite(WorkItem->LazyContext, TRUE); - MiFlushMappedSection(WorkItem->ToUnmap, &WorkItem->FileOffset, &WorkItem->MapSize, WorkItem->Dirty); - WorkItem->ReleaseFromLazyWrite(WorkItem->LazyContext); MmUnmapCacheViewInSystemSpace(WorkItem->ToUnmap); - MmFinalizeSegment(WorkItem->ToDeref); + ObDereferenceObject(WorkItem->ToDeref); ExFreePool(WorkItem); DPRINT("Done\n"); } @@ -134,9 +132,7 @@ if (Immediate) { - PMM_CACHE_SECTION_SEGMENT ToDeref = Bcb->SectionObject; - BOOLEAN Dirty = Bcb->Dirty; - + PROS_SECTION_OBJECT ToDeref = Bcb->SectionObject; Bcb->Map = NULL; Bcb->SectionObject = NULL; Bcb->BaseAddress = NULL; @@ -147,9 +143,10 @@ RemoveEntryList(&Bcb->ThisFileList); CcpUnlock(); - MiFlushMappedSection(ToUnmap, &BaseOffset, &MappedSize, Dirty); + if (Dirty) + MiFlushMappedSection(ToUnmap, &BaseOffset, &MappedSize, Dirty); MmUnmapCacheViewInSystemSpace(ToUnmap); - MmFinalizeSegment(ToDeref); + ObDereferenceObject(ToDeref); CcpLock(); } else @@ -186,7 +183,7 @@ /* Needs mutex */ ULONG CcpAllocateCacheSections (PFILE_OBJECT FileObject, - PMM_CACHE_SECTION_SEGMENT SectionObject) + PROS_SECTION_OBJECT SectionObject) { ULONG i = INVALID_CACHE; PNOCC_CACHE_MAP Map; @@ -313,13 +310,12 @@ /* Note: windows 2000 drivers treat this as a bool */ //BOOLEAN Wait = (Flags & MAP_WAIT) || (Flags == TRUE); LARGE_INTEGER Target, EndInterval; - ULONG BcbHead; + ULONG BcbHead, SectionSize, ViewSize; PNOCC_BCB Bcb = NULL; - PMM_CACHE_SECTION_SEGMENT SectionObject = NULL; + PROS_SECTION_OBJECT SectionObject = NULL; NTSTATUS Status; PNOCC_CACHE_MAP Map = (PNOCC_CACHE_MAP)FileObject->SectionObjectPointer->SharedCacheMap; - ULONG SectionSize; - ULONG ViewSize = CACHE_STRIPE; + ViewSize = CACHE_STRIPE; if (!Map) { @@ -424,10 +420,11 @@ } DPRINT("Selected BCB #%x\n", BcbHead); + ViewSize = CACHE_STRIPE; Bcb = &CcCacheSections[BcbHead]; Status = MmMapCacheViewInSystemSpaceAtOffset - (SectionObject, + (SectionObject->Segment, &Bcb->BaseAddress, &Target, &ViewSize); @@ -436,7 +433,7 @@ { *BcbResult = NULL; *Buffer = NULL; - MmFinalizeSegment(SectionObject); + ObDereferenceObject(SectionObject); RemoveEntryList(&Bcb->ThisFileList); RtlZeroMemory(Bcb, sizeof(*Bcb)); RtlClearBit(CcCacheBitmap, BcbHead); @@ -481,17 +478,16 @@ { // Fault in the pages. This forces reads to happen now. ULONG i; - CHAR Dummy; PCHAR FaultIn = Bcb->BaseAddress; - DPRINT1 + DPRINT ("Faulting in pages at this point: file %wZ %08x%08x:%x\n", &FileObject->FileName, Bcb->FileOffset.HighPart, Bcb->FileOffset.LowPart, Bcb->Length); - for (i = 0; i < Bcb->Length; i++) + for (i = 0; i < Bcb->Length; i += PAGE_SIZE) { - Dummy = FaultIn[i]; + FaultIn[i] ^= 0; } } ASSERT(Bcb >= CcCacheSections && Bcb < (CcCacheSections + CACHE_NUM_SECTIONS)); @@ -657,7 +653,7 @@ ULONG OldProtect; #endif - DPRINT1("CcPreparePinWrite(%x:%x)\n", Buffer, Length); + DPRINT("CcPreparePinWrite(%x:%x)\n", Buffer, Length); Result = CcPinRead (FileObject, Index: ntoskrnl/cache/section/data.c =================================================================== --- ntoskrnl/cache/section/data.c (revision 55684) +++ ntoskrnl/cache/section/data.c (working copy) @@ -52,6 +52,8 @@ #define DPRINTC DPRINT +LIST_ENTRY MiSegmentList; + extern KEVENT MpwThreadEvent; extern KSPIN_LOCK MiSectionPageTableLock; @@ -59,15 +61,6 @@ ULONG_PTR MmSubsectionBase; -NTSTATUS -NTAPI -MiSimpleRead -(PFILE_OBJECT FileObject, - PLARGE_INTEGER FileOffset, - PVOID Buffer, - ULONG Length, - PIO_STATUS_BLOCK ReadStatus); - static const INFORMATION_CLASS_INFO ExSectionInfoClass[] = { ICI_SQ_SAME( sizeof(SECTION_BASIC_INFORMATION), sizeof(ULONG), ICIF_QUERY ), /* SectionBasicInformation */ @@ -80,18 +73,21 @@ VOID NTAPI -_MmLockCacheSectionSegment(PMM_CACHE_SECTION_SEGMENT Segment, const char *file, int line) +_MmLockSectionSegment(PMM_SECTION_SEGMENT Segment, const char *file, int line) { - DPRINT("MmLockSectionSegment(%p,%s:%d)\n", Segment, file, line); + //DPRINT("MmLockSectionSegment(%p,%s:%d)\n", Segment, file, line); ExAcquireFastMutex(&Segment->Lock); + Segment->Locked = TRUE; } VOID NTAPI -_MmUnlockCacheSectionSegment(PMM_CACHE_SECTION_SEGMENT Segment, const char *file, int line) +_MmUnlockSectionSegment(PMM_SECTION_SEGMENT Segment, const char *file, int line) { + ASSERT(Segment->Locked); + Segment->Locked = FALSE; ExReleaseFastMutex(&Segment->Lock); - DPRINT("MmUnlockSectionSegment(%p,%s:%d)\n", Segment, file, line); + //DPRINT("MmUnlockSectionSegment(%p,%s:%d)\n", Segment, file, line); } NTSTATUS @@ -104,7 +100,7 @@ PFN_NUMBER Page; PMMSUPPORT AddressSpace; PMEMORY_AREA MemoryArea; - PMM_CACHE_SECTION_SEGMENT Segment; + PMM_SECTION_SEGMENT Segment; LARGE_INTEGER FileOffset = *FileOffsetPtr, End, FirstMapped; DPRINT("MiZeroFillSection(Address %x,Offset %x,Length %x)\n", Address, FileOffset.LowPart, Length); AddressSpace = MmGetKernelAddressSpace(); @@ -116,11 +112,11 @@ return STATUS_NOT_MAPPED_DATA; } - Segment = MemoryArea->Data.CacheData.Segment; + Segment = MemoryArea->Data.SectionData.Segment; End.QuadPart = FileOffset.QuadPart + Length; End.LowPart = PAGE_ROUND_DOWN(End.LowPart); FileOffset.LowPart = PAGE_ROUND_UP(FileOffset.LowPart); - FirstMapped.QuadPart = MemoryArea->Data.CacheData.ViewOffset.QuadPart; + FirstMapped.QuadPart = MemoryArea->Data.SectionData.ViewOffset.QuadPart; DPRINT ("Pulling zero pages for %08x%08x-%08x%08x\n", FileOffset.u.HighPart, FileOffset.u.LowPart, @@ -134,12 +130,12 @@ break; MmLockAddressSpace(AddressSpace); - MmLockCacheSectionSegment(Segment); + MmLockSectionSegment(Segment); - Entry = MiGetPageEntryCacheSectionSegment(Segment, &FileOffset); + Entry = MmGetPageEntrySectionSegment(Segment, &FileOffset); if (Entry == 0) { - MiSetPageEntryCacheSectionSegment(Segment, &FileOffset, MAKE_PFN_SSE(Page)); + MmSetPageEntrySectionSegment(Segment, &FileOffset, MAKE_PFN_SSE(Page)); Address = ((PCHAR)MemoryArea->StartingAddress) + FileOffset.QuadPart - FirstMapped.QuadPart; MmReferencePage(Page); MmCreateVirtualMapping(NULL, Address, PAGE_READWRITE, &Page, 1); @@ -148,7 +144,7 @@ else MmReleasePageMemoryConsumer(MC_CACHE, Page); - MmUnlockCacheSectionSegment(Segment); + MmUnlockSectionSegment(Segment); MmUnlockAddressSpace(AddressSpace); FileOffset.QuadPart += PAGE_SIZE; @@ -170,7 +166,7 @@ ULONG_PTR PageAddress; PMMSUPPORT AddressSpace = MmGetKernelAddressSpace(); PMEMORY_AREA MemoryArea; - PMM_CACHE_SECTION_SEGMENT Segment; + PMM_SECTION_SEGMENT Segment; ULONG_PTR BeginningAddress, EndingAddress; LARGE_INTEGER ViewOffset; LARGE_INTEGER FileOffset; @@ -189,12 +185,12 @@ } BeginningAddress = PAGE_ROUND_DOWN((ULONG_PTR)MemoryArea->StartingAddress); EndingAddress = PAGE_ROUND_UP((ULONG_PTR)MemoryArea->EndingAddress); - Segment = MemoryArea->Data.CacheData.Segment; - ViewOffset.QuadPart = MemoryArea->Data.CacheData.ViewOffset.QuadPart; + Segment = MemoryArea->Data.SectionData.Segment; + ViewOffset.QuadPart = MemoryArea->Data.SectionData.ViewOffset.QuadPart; ASSERT(ViewOffset.QuadPart == BaseOffset->QuadPart); - MmLockCacheSectionSegment(Segment); + MmLockSectionSegment(Segment); Pages = ExAllocatePool (NonPagedPool, @@ -206,7 +202,7 @@ ASSERT(FALSE); } - DPRINT("Getting pages in range %08x-%08x\n", BeginningAddress, EndingAddress); + //DPRINT("Getting pages in range %08x-%08x\n", BeginningAddress, EndingAddress); for (PageAddress = BeginningAddress; PageAddress < EndingAddress; @@ -215,34 +211,36 @@ ULONG Entry; FileOffset.QuadPart = ViewOffset.QuadPart + PageAddress - BeginningAddress; Entry = - MiGetPageEntryCacheSectionSegment - (MemoryArea->Data.CacheData.Segment, + MmGetPageEntrySectionSegment + (MemoryArea->Data.SectionData.Segment, &FileOffset); Page = PFN_FROM_SSE(Entry); if (Entry != 0 && !IS_SWAP_FROM_SSE(Entry) && (MmIsDirtyPageRmap(Page) || IS_DIRTY_SSE(Entry)) && FileOffset.QuadPart < FileSize->QuadPart) { - Pages[(PageAddress - BeginningAddress) >> PAGE_SHIFT] = Page; + MmReferencePage(Page); + Pages[(PageAddress - BeginningAddress) >> PAGE_SHIFT] = Entry; } else Pages[(PageAddress - BeginningAddress) >> PAGE_SHIFT] = 0; } - MmUnlockCacheSectionSegment(Segment); + MmUnlockSectionSegment(Segment); MmUnlockAddressSpace(AddressSpace); for (PageAddress = BeginningAddress; PageAddress < EndingAddress; PageAddress += PAGE_SIZE) { + ULONG Entry; FileOffset.QuadPart = ViewOffset.QuadPart + PageAddress - BeginningAddress; - Page = Pages[(PageAddress - BeginningAddress) >> PAGE_SHIFT]; + Entry = Pages[(PageAddress - BeginningAddress) >> PAGE_SHIFT]; + Page = PFN_FROM_SSE(Entry); if (Page) { - ULONG Entry; if (WriteData) { - DPRINT("MiWriteBackPage(%wZ,addr %x,%08x%08x)\n", &Segment->FileObject->FileName, PageAddress, FileOffset.u.HighPart, FileOffset.u.LowPart); + //DPRINT("MiWriteBackPage(%wZ,addr %x,%08x%08x)\n", &Segment->FileObject->FileName, PageAddress, FileOffset.u.HighPart, FileOffset.u.LowPart); Status = MiWriteBackPage(Segment->FileObject, &FileOffset, PAGE_SIZE, Page); } else Status = STATUS_SUCCESS; @@ -250,11 +248,11 @@ if (NT_SUCCESS(Status)) { MmLockAddressSpace(AddressSpace); MmSetCleanAllRmaps(Page); - MmLockCacheSectionSegment(Segment); - Entry = MiGetPageEntryCacheSectionSegment(Segment, &FileOffset); + MmLockSectionSegment(Segment); + Entry = MmGetPageEntrySectionSegment(Segment, &FileOffset); if (Entry && !IS_SWAP_FROM_SSE(Entry) && PFN_FROM_SSE(Entry) == Page) - MiSetPageEntryCacheSectionSegment(Segment, &FileOffset, CLEAN_SSE(Entry)); - MmUnlockCacheSectionSegment(Segment); + MmSetPageEntrySectionSegment(Segment, &FileOffset, CLEAN_SSE(Entry)); + MmUnlockSectionSegment(Segment); MmUnlockAddressSpace(AddressSpace); } else { DPRINT @@ -268,6 +266,7 @@ &Segment->FileObject->FileName, Status); } + MmDereferencePage(Page); } } @@ -278,77 +277,92 @@ VOID NTAPI -MmFinalizeSegment(PMM_CACHE_SECTION_SEGMENT Segment) +MmFinalizeSegment(PMM_SECTION_SEGMENT Segment) { KIRQL OldIrql = 0; - MmLockCacheSectionSegment(Segment); + DPRINT("Finalize segment %p\n", Segment); + + MmLockSectionSegment(Segment); + RemoveEntryList(&Segment->ListOfSegments); if (Segment->Flags & MM_DATAFILE_SEGMENT) { KeAcquireSpinLock(&Segment->FileObject->IrpListLock, &OldIrql); if (Segment->Flags & MM_SEGMENT_FINALIZE) { KeReleaseSpinLock(&Segment->FileObject->IrpListLock, OldIrql); - MmUnlockCacheSectionSegment(Segment); + MmUnlockSectionSegment(Segment); return; - } else { - Segment->Flags |= MM_SEGMENT_FINALIZE; } - } - DPRINTC("Finalizing segment %x\n", Segment); - if (Segment->Flags & MM_DATAFILE_SEGMENT) - { - //Segment->FileObject->SectionObjectPointer->DataSectionObject = NULL; + Segment->Flags |= MM_SEGMENT_FINALIZE; + DPRINTC("Finalizing data file segment %p\n", Segment); + + Segment->FileObject->SectionObjectPointer->DataSectionObject = NULL; KeReleaseSpinLock(&Segment->FileObject->IrpListLock, OldIrql); - MiFreePageTablesSectionSegment(Segment, MiFreeSegmentPage); - MmUnlockCacheSectionSegment(Segment); + MmFreePageTablesSectionSegment(Segment, MiFreeSegmentPage); + MmUnlockSectionSegment(Segment); + DPRINT("Dereference file object %wZ\n", &Segment->FileObject->FileName); ObDereferenceObject(Segment->FileObject); + DPRINT("Done with %wZ\n", &Segment->FileObject->FileName); + Segment->FileObject = NULL; } else { - MiFreePageTablesSectionSegment(Segment, MiFreeSegmentPage); - MmUnlockCacheSectionSegment(Segment); + DPRINTC("Finalizing segment %p\n", Segment); + MmFreePageTablesSectionSegment(Segment, MiFreeSegmentPage); + MmUnlockSectionSegment(Segment); } - DPRINTC("Segment %x destroy\n", Segment); + DPRINTC("Segment %p destroy\n", Segment); ExFreePoolWithTag(Segment, TAG_MM_SECTION_SEGMENT); } NTSTATUS NTAPI -MmCreateCacheSection -(PMM_CACHE_SECTION_SEGMENT *SegmentObject, - ACCESS_MASK DesiredAccess, - POBJECT_ATTRIBUTES ObjectAttributes, - PLARGE_INTEGER UMaximumSize, - ULONG SectionPageProtection, - ULONG AllocationAttributes, - PFILE_OBJECT FileObject) +MmCreateCacheSection(PROS_SECTION_OBJECT *SectionObject, + ACCESS_MASK DesiredAccess, + POBJECT_ATTRIBUTES ObjectAttributes, + PLARGE_INTEGER UMaximumSize, + ULONG SectionPageProtection, + ULONG AllocationAttributes, + PFILE_OBJECT FileObject) /* * Create a section backed by a data file */ { + PROS_SECTION_OBJECT Section; NTSTATUS Status; ULARGE_INTEGER MaximumSize; - PMM_CACHE_SECTION_SEGMENT Segment; - ULONG FileAccess; + PMM_SECTION_SEGMENT Segment; IO_STATUS_BLOCK Iosb; CC_FILE_SIZES FileSizes; FILE_STANDARD_INFORMATION FileInfo; + KIRQL OldIrql; + DPRINT("MmCreateDataFileSection\n"); + /* - * Check file access required + * Create the section */ - if (SectionPageProtection & PAGE_READWRITE || - SectionPageProtection & PAGE_EXECUTE_READWRITE) + Status = ObCreateObject(ExGetPreviousMode(), + MmSectionObjectType, + ObjectAttributes, + ExGetPreviousMode(), + NULL, + sizeof(ROS_SECTION_OBJECT), + 0, + 0, + (PVOID*)(PVOID)&Section); + if (!NT_SUCCESS(Status)) { - FileAccess = FILE_READ_DATA | FILE_WRITE_DATA; + DPRINT("Failed: %x\n", Status); + return(Status); } - else - { - FileAccess = FILE_READ_DATA; - } - /* - * Reference the file handle + * Initialize it */ - ObReferenceObject(FileObject); + RtlZeroMemory(Section, sizeof(ROS_SECTION_OBJECT)); + Section->SectionPageProtection = SectionPageProtection; + Section->AllocationAttributes = AllocationAttributes; + Section->Segment = NULL; + Section->FileObject = FileObject; + DPRINT("Getting original file size\n"); /* A hack: If we're cached, we can overcome deadlocking with the upper * layer filesystem call by retriving the object sizes from the cache @@ -363,18 +377,21 @@ * (as in case of the EXT2FS driver by Manoj Paul Joseph where the * standard file information is filled on first request). */ + DPRINT("Querying info\n"); Status = IoQueryFileInformation (FileObject, FileStandardInformation, sizeof(FILE_STANDARD_INFORMATION), &FileInfo, &Iosb.Information); + DPRINT("Query => %x\n", Status); if (!NT_SUCCESS(Status)) { + DPRINT("Status %x\n", Status); + ObDereferenceObject(Section); return Status; } - ASSERT(Status != STATUS_PENDING); FileSizes.ValidDataLength = FileInfo.EndOfFile; @@ -385,9 +402,12 @@ /* * FIXME: Revise this once a locking order for file size changes is * decided + * + * We're handed down a maximum size in every case. Should we still check at all? */ - if (UMaximumSize != NULL) + if (UMaximumSize != NULL && UMaximumSize->QuadPart) { + DPRINT("Taking maximum %x\n", UMaximumSize->LowPart); MaximumSize.QuadPart = UMaximumSize->QuadPart; } else @@ -396,42 +416,99 @@ MaximumSize.QuadPart = FileSizes.FileSize.QuadPart; } - Segment = ExAllocatePoolWithTag(NonPagedPool, sizeof(MM_CACHE_SECTION_SEGMENT), + /* Mapping zero-sized files isn't allowed. */ + if (MaximumSize.QuadPart == 0) + { + DPRINT("Zero size file\n"); + ObDereferenceObject(Section); + return STATUS_FILE_INVALID; + } + + Segment = ExAllocatePoolWithTag(NonPagedPool, sizeof(MM_SECTION_SEGMENT), TAG_MM_SECTION_SEGMENT); if (Segment == NULL) { + DPRINT("Failed: STATUS_NO_MEMORY\n"); + ObDereferenceObject(Section); return(STATUS_NO_MEMORY); } + DPRINT("Zeroing %x\n", Segment); + RtlZeroMemory(Segment, sizeof(MM_SECTION_SEGMENT)); ExInitializeFastMutex(&Segment->Lock); Segment->ReferenceCount = 1; + Segment->Locked = TRUE; + RtlZeroMemory(&Segment->Image, sizeof(Segment->Image)); + Section->Segment = Segment; + KeAcquireSpinLock(&FileObject->IrpListLock, &OldIrql); /* - * Set the lock before assigning the segment to the file object - */ - ExAcquireFastMutex(&Segment->Lock); - - DPRINT("Filling out Segment info (No previous data section)\n"); - ObReferenceObject(FileObject); - Segment->FileObject = FileObject; - Segment->Protection = SectionPageProtection; - Segment->Flags = MM_DATAFILE_SEGMENT; - memset(&Segment->Image, 0, sizeof(Segment->Image)); - Segment->WriteCopy = FALSE; - if (AllocationAttributes & SEC_RESERVE) + * If this file hasn't been mapped as a data file before then allocate a + * section segment to describe the data file mapping + */ + if (FileObject->SectionObjectPointer->DataSectionObject == NULL) { - Segment->Length.QuadPart = Segment->RawLength.QuadPart = 0; + FileObject->SectionObjectPointer->DataSectionObject = (PVOID)Segment; + KeReleaseSpinLock(&FileObject->IrpListLock, OldIrql); + + /* + * Set the lock before assigning the segment to the file object + */ + ExAcquireFastMutex(&Segment->Lock); + + DPRINT("Filling out Segment info (No previous data section)\n"); + ObReferenceObject(FileObject); + Segment->FileObject = FileObject; + Segment->Protection = SectionPageProtection; + Segment->Flags = MM_DATAFILE_SEGMENT; + memset(&Segment->Image, 0, sizeof(Segment->Image)); + Segment->WriteCopy = FALSE; + + if (AllocationAttributes & SEC_RESERVE) + { + Segment->Length.QuadPart = Segment->RawLength.QuadPart = 0; + } + else + { + Segment->RawLength.QuadPart = MaximumSize.QuadPart; + Segment->Length.QuadPart = PAGE_ROUND_UP(Segment->RawLength.QuadPart); + } + MiInitializeSectionPageTable(Segment); + InsertHeadList(&MiSegmentList, &Segment->ListOfSegments); } else { - Segment->RawLength = MaximumSize; - Segment->Length.QuadPart = PAGE_ROUND_UP(Segment->RawLength.QuadPart); + KeReleaseSpinLock(&FileObject->IrpListLock, OldIrql); + DPRINTC("Free Segment %x\n", Segment); + ExFreePool(Segment); + + DPRINT("Filling out Segment info (previous data section)\n"); + + /* + * If the file is already mapped as a data file then we may need + * to extend it + */ + Segment = + (PMM_SECTION_SEGMENT)FileObject->SectionObjectPointer-> + DataSectionObject; + Section->Segment = Segment; + (void)InterlockedIncrementUL(&Segment->ReferenceCount); + + MmLockSectionSegment(Segment); + + if (MaximumSize.QuadPart > Segment->RawLength.QuadPart && + !(AllocationAttributes & SEC_RESERVE)) + { + Segment->RawLength.QuadPart = MaximumSize.QuadPart; + Segment->Length.QuadPart = PAGE_ROUND_UP(Segment->RawLength.QuadPart); + } } - MiInitializeSectionPageTable(Segment); - MmUnlockCacheSectionSegment(Segment); + MmUnlockSectionSegment(Segment); + Section->MaximumSize.QuadPart = MaximumSize.QuadPart; + /* Extend file if section is longer */ DPRINT("MaximumSize %08x%08x ValidDataLength %08x%08x\n", MaximumSize.u.HighPart, MaximumSize.u.LowPart, @@ -444,14 +521,14 @@ if (!NT_SUCCESS(Status)) { DPRINT("Could not expand section\n"); + ObDereferenceObject(Section); return Status; } } DPRINTC("Segment %x created (%x)\n", Segment, Segment->Flags); - *SegmentObject = Segment; - + *SectionObject = Section; return(STATUS_SUCCESS); } @@ -459,7 +536,7 @@ NTAPI _MiMapViewOfSegment (PMMSUPPORT AddressSpace, - PMM_CACHE_SECTION_SEGMENT Segment, + PMM_SECTION_SEGMENT Segment, PVOID* BaseAddress, SIZE_T ViewSize, ULONG Protect, @@ -494,11 +571,11 @@ DPRINTC("MiMapViewOfSegment %x %x %x %x %x %wZ %s:%d\n", MmGetAddressSpaceOwner(AddressSpace), *BaseAddress, Segment, ViewOffset ? ViewOffset->LowPart : 0, ViewSize, Segment->FileObject ? &Segment->FileObject->FileName : NULL, file, line); - MArea->Data.CacheData.Segment = Segment; + MArea->Data.SectionData.Segment = Segment; if (ViewOffset) - MArea->Data.CacheData.ViewOffset = *ViewOffset; + MArea->Data.SectionData.ViewOffset = *ViewOffset; else - MArea->Data.CacheData.ViewOffset.QuadPart = 0; + MArea->Data.SectionData.ViewOffset.QuadPart = 0; #if 0 MArea->NotPresent = MmNotPresentFaultPageFile; @@ -506,6 +583,9 @@ MArea->PageOut = MmPageOutPageFileView; #endif + MmInitializeRegion(&MArea->Data.SectionData.RegionListHead, + ViewSize, 0, Protect); + DPRINTC ("MiMapViewOfSegment(P %x, A %x, T %x)\n", MmGetAddressSpaceOwner(AddressSpace), *BaseAddress, MArea->Type); @@ -516,13 +596,13 @@ VOID NTAPI MiFreeSegmentPage -(PMM_CACHE_SECTION_SEGMENT Segment, +(PMM_SECTION_SEGMENT Segment, PLARGE_INTEGER FileOffset) { ULONG Entry; PFILE_OBJECT FileObject = Segment->FileObject; - Entry = MiGetPageEntryCacheSectionSegment(Segment, FileOffset); + Entry = MmGetPageEntrySectionSegment(Segment, FileOffset); DPRINTC("MiFreeSegmentPage(%x:%08x%08x -> Entry %x\n", Segment, FileOffset->HighPart, FileOffset->LowPart, Entry); @@ -537,7 +617,7 @@ } DPRINTC("Free page %x (off %x from %x) (ref ct %d, ent %x, dirty? %s)\n", OldPage, FileOffset->LowPart, Segment, MmGetReferenceCountPage(OldPage), Entry, IS_DIRTY_SSE(Entry) ? "true" : "false"); - MiSetPageEntryCacheSectionSegment(Segment, FileOffset, 0); + MmSetPageEntrySectionSegment(Segment, FileOffset, 0); MmReleasePageMemoryConsumer(MC_CACHE, OldPage); } else if (IS_SWAP_FROM_SSE(Entry)) @@ -558,7 +638,7 @@ PVOID *ContextData = Context; PMMSUPPORT AddressSpace; PEPROCESS Process; - PMM_CACHE_SECTION_SEGMENT Segment; + PMM_SECTION_SEGMENT Segment; LARGE_INTEGER Offset; DPRINT("MmFreeSectionPage(%x,%x,%x,%x,%d)\n", MmGetAddressSpaceOwner(ContextData[0]), Address, Page, SwapEntry, Dirty); @@ -568,9 +648,9 @@ Address = (PVOID)PAGE_ROUND_DOWN(Address); Segment = ContextData[1]; Offset.QuadPart = (ULONG_PTR)Address - (ULONG_PTR)MemoryArea->StartingAddress + - MemoryArea->Data.CacheData.ViewOffset.QuadPart; + MemoryArea->Data.SectionData.ViewOffset.QuadPart; - Entry = MiGetPageEntryCacheSectionSegment(Segment, &Offset); + Entry = MmGetPageEntrySectionSegment(Segment, &Offset); if (Page) { @@ -583,7 +663,7 @@ if (Page != 0 && PFN_FROM_SSE(Entry) == Page && Dirty) { DPRINT("Freeing section page %x:%x -> %x\n", Segment, Offset.LowPart, Entry); - MiSetPageEntryCacheSectionSegment(Segment, &Offset, DIRTY_SSE(Entry)); + MmSetPageEntrySectionSegment(Segment, &Offset, DIRTY_SSE(Entry)); } else if (SwapEntry != 0) { @@ -599,7 +679,7 @@ { PVOID Context[2]; PMEMORY_AREA MemoryArea; - PMM_CACHE_SECTION_SEGMENT Segment; + PMM_SECTION_SEGMENT Segment; MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace, BaseAddress); if (MemoryArea == NULL) @@ -609,17 +689,17 @@ } MemoryArea->DeleteInProgress = TRUE; - Segment = MemoryArea->Data.CacheData.Segment; - MemoryArea->Data.CacheData.Segment = NULL; + Segment = MemoryArea->Data.SectionData.Segment; + MemoryArea->Data.SectionData.Segment = NULL; - MmLockCacheSectionSegment(Segment); + MmLockSectionSegment(Segment); Context[0] = AddressSpace; Context[1] = Segment; DPRINT("MmFreeMemoryArea(%x,%x)\n", MmGetAddressSpaceOwner(AddressSpace), MemoryArea->StartingAddress); MmFreeMemoryArea(AddressSpace, MemoryArea, MmFreeCacheSectionPage, Context); - MmUnlockCacheSectionSegment(Segment); + MmUnlockSectionSegment(Segment); DPRINTC("MiUnmapViewOfSegment %x %x %x\n", MmGetAddressSpaceOwner(AddressSpace), BaseAddress, Segment); @@ -629,16 +709,17 @@ NTSTATUS NTAPI MmExtendCacheSection -(PMM_CACHE_SECTION_SEGMENT Segment, +(PROS_SECTION_OBJECT Section, PLARGE_INTEGER NewSize, BOOLEAN ExtendFile) { LARGE_INTEGER OldSize; + PMM_SECTION_SEGMENT Segment = Section->Segment; DPRINT("Extend Segment %x\n", Segment); - MmLockCacheSectionSegment(Segment); + MmLockSectionSegment(Segment); OldSize.QuadPart = Segment->RawLength.QuadPart; - MmUnlockCacheSectionSegment(Segment); + MmUnlockSectionSegment(Segment); DPRINT("OldSize %08x%08x NewSize %08x%08x\n", OldSize.u.HighPart, OldSize.u.LowPart, @@ -651,17 +732,17 @@ if (!NT_SUCCESS(Status)) return Status; } - MmLockCacheSectionSegment(Segment); + MmLockSectionSegment(Segment); Segment->RawLength.QuadPart = NewSize->QuadPart; Segment->Length.QuadPart = MAX(Segment->Length.QuadPart, PAGE_ROUND_UP(Segment->RawLength.LowPart)); - MmUnlockCacheSectionSegment(Segment); + MmUnlockSectionSegment(Segment); return STATUS_SUCCESS; } NTSTATUS NTAPI MmMapCacheViewInSystemSpaceAtOffset -(IN PMM_CACHE_SECTION_SEGMENT Segment, +(IN PMM_SECTION_SEGMENT Segment, OUT PVOID *MappedBase, PLARGE_INTEGER FileOffset, IN OUT PULONG ViewSize) @@ -674,7 +755,7 @@ AddressSpace = MmGetKernelAddressSpace(); MmLockAddressSpace(AddressSpace); - MmLockCacheSectionSegment(Segment); + MmLockSectionSegment(Segment); Status = MiMapViewOfSegment (AddressSpace, @@ -685,7 +766,7 @@ FileOffset, 0); - MmUnlockCacheSectionSegment(Segment); + MmUnlockSectionSegment(Segment); MmUnlockAddressSpace(AddressSpace); return Status; Index: ntoskrnl/cache/section/fault.c =================================================================== --- ntoskrnl/cache/section/fault.c (revision 55684) +++ ntoskrnl/cache/section/fault.c (working copy) @@ -66,7 +66,7 @@ NTSTATUS Status; PVOID PAddress; ULONG Consumer; - PMM_CACHE_SECTION_SEGMENT Segment; + PMM_SECTION_SEGMENT Segment; LARGE_INTEGER FileOffset, TotalOffset; ULONG Entry; ULONG Attributes; @@ -88,9 +88,9 @@ PAddress = MM_ROUND_DOWN(Address, PAGE_SIZE); TotalOffset.QuadPart = (ULONG_PTR)PAddress - (ULONG_PTR)MemoryArea->StartingAddress; - Segment = MemoryArea->Data.CacheData.Segment; + Segment = MemoryArea->Data.SectionData.Segment; - TotalOffset.QuadPart += MemoryArea->Data.CacheData.ViewOffset.QuadPart; + TotalOffset.QuadPart += MemoryArea->Data.SectionData.ViewOffset.QuadPart; FileOffset = TotalOffset; //Consumer = (Segment->Flags & MM_DATAFILE_SEGMENT) ? MC_CACHE : MC_USER; @@ -106,12 +106,12 @@ /* * Lock the segment */ - MmLockCacheSectionSegment(Segment); + MmLockSectionSegment(Segment); /* * Get the entry corresponding to the offset within the section */ - Entry = MiGetPageEntryCacheSectionSegment(Segment, &TotalOffset); + Entry = MmGetPageEntrySectionSegment(Segment, &TotalOffset); Attributes = PAGE_READONLY; @@ -125,13 +125,13 @@ if (Required->State & 2) { DPRINT("Set in section @ %x\n", TotalOffset.LowPart); - Status = MiSetPageEntryCacheSectionSegment + Status = MmSetPageEntrySectionSegment (Segment, &TotalOffset, Entry = MAKE_PFN_SSE(Required->Page[0])); if (!NT_SUCCESS(Status)) { MmReleasePageMemoryConsumer(MC_CACHE, Required->Page[0]); } - MmUnlockCacheSectionSegment(Segment); + MmUnlockSectionSegment(Segment); MiSetPageEvent(Process, Address); DPRINT("Status %x\n", Status); return STATUS_MM_RESTART_OPERATION; @@ -149,7 +149,7 @@ // Drop the reference for our address space ... MmReleasePageMemoryConsumer(MC_CACHE, Required->Page[0]); } - MmUnlockCacheSectionSegment(Segment); + MmUnlockSectionSegment(Segment); DPRINTC("XXX Set Event %x\n", Status); MiSetPageEvent(Process, Address); DPRINT("Status %x\n", Status); @@ -170,7 +170,7 @@ } DPRINT("XXX Set Event %x\n", Status); MiSetPageEvent(Process, Address); - MmUnlockCacheSectionSegment(Segment); + MmUnlockSectionSegment(Segment); DPRINT("Status %x\n", Status); return Status; } @@ -188,8 +188,8 @@ Required->FileOffset = FileOffset; Required->Amount = PAGE_SIZE; Required->DoAcquisition = MiReadFilePage; - MiSetPageEntryCacheSectionSegment(Segment, &TotalOffset, MAKE_SWAP_SSE(MM_WAIT_ENTRY)); - MmUnlockCacheSectionSegment(Segment); + MmSetPageEntrySectionSegment(Segment, &TotalOffset, MAKE_SWAP_SSE(MM_WAIT_ENTRY)); + MmUnlockSectionSegment(Segment); return STATUS_MORE_PROCESSING_REQUIRED; } ASSERT(FALSE); @@ -232,36 +232,34 @@ BOOLEAN Locked, PMM_REQUIRED_RESOURCES Required) { - PMM_CACHE_SECTION_SEGMENT Segment; + PMM_SECTION_SEGMENT Segment; PFN_NUMBER NewPage, OldPage; NTSTATUS Status; PVOID PAddress; LARGE_INTEGER Offset; PEPROCESS Process = MmGetAddressSpaceOwner(AddressSpace); - + DPRINT("MmAccessFaultSectionView(%x, %x, %x, %x)\n", AddressSpace, MemoryArea, Address, Locked); - Segment = MemoryArea->Data.CacheData.Segment; + Segment = MemoryArea->Data.SectionData.Segment; /* * Lock the segment */ - MmLockCacheSectionSegment(Segment); + MmLockSectionSegment(Segment); /* * Find the offset of the page */ PAddress = MM_ROUND_DOWN(Address, PAGE_SIZE); Offset.QuadPart = (ULONG_PTR)PAddress - (ULONG_PTR)MemoryArea->StartingAddress + - MemoryArea->Data.CacheData.ViewOffset.QuadPart; + MemoryArea->Data.SectionData.ViewOffset.QuadPart; -#if 0 // XXX Cache sections are not CoW. For now, treat all access violations this way. - if ((!Segment->WriteCopy && - !MemoryArea->Data.CacheData.WriteCopyView) || + if (!Segment->WriteCopy /*&& + !MemoryArea->Data.SectionData.WriteCopyView*/ || Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED) -#endif { -#if 0 // XXX Cache sections don't have regions at present, which streamlines things +#if 0 if (Region->Protect == PAGE_READWRITE || Region->Protect == PAGE_EXECUTE_READWRITE) #endif @@ -272,15 +270,15 @@ { DPRINTC("file %wZ\n", &Segment->FileObject->FileName); } - Entry = MiGetPageEntryCacheSectionSegment(Segment, &Offset); + Entry = MmGetPageEntrySectionSegment(Segment, &Offset); DPRINT("Entry %x\n", Entry); if (Entry && !IS_SWAP_FROM_SSE(Entry) && PFN_FROM_SSE(Entry) == MmGetPfnForProcess(Process, Address)) { - MiSetPageEntryCacheSectionSegment(Segment, &Offset, DIRTY_SSE(Entry)); + MmSetPageEntrySectionSegment(Segment, &Offset, DIRTY_SSE(Entry)); } MmSetPageProtect(Process, PAddress, PAGE_READWRITE); - MmUnlockCacheSectionSegment(Segment); + MmUnlockSectionSegment(Segment); DPRINT("Done\n"); return STATUS_SUCCESS; } @@ -288,7 +286,7 @@ else { DPRINT("Not supposed to be writable\n"); - MmUnlockCacheSectionSegment(Segment); + MmUnlockSectionSegment(Segment); return STATUS_ACCESS_VIOLATION; } #endif @@ -300,7 +298,7 @@ if (MmIsPageSwapEntry(Process, Address)) { MmGetPageFileMapping(Process, Address, &SwapEntry); - MmUnlockCacheSectionSegment(Segment); + MmUnlockSectionSegment(Segment); if (SwapEntry == MM_WAIT_ENTRY) return STATUS_SUCCESS + 1; // Wait ... somebody else is getting it right now else @@ -313,7 +311,7 @@ Required->Line = __LINE__; Required->DoAcquisition = MiGetOnePage; MmCreatePageFileMapping(Process, Address, MM_WAIT_ENTRY); - MmUnlockCacheSectionSegment(Segment); + MmUnlockSectionSegment(Segment); return STATUS_MORE_PROCESSING_REQUIRED; } @@ -343,13 +341,13 @@ { DPRINT1("MmCreateVirtualMapping failed, not out of memory\n"); ASSERT(FALSE); - MmUnlockCacheSectionSegment(Segment); + MmUnlockSectionSegment(Segment); return(Status); } MmInsertRmap(NewPage, Process, PAddress); MmReleasePageMemoryConsumer(MC_CACHE, OldPage); - MmUnlockCacheSectionSegment(Segment); + MmUnlockSectionSegment(Segment); DPRINT("Address 0x%.8X\n", Address); return(STATUS_SUCCESS); @@ -396,7 +394,10 @@ NTSTATUS Status; BOOLEAN Locked = FromMdl; MM_REQUIRED_RESOURCES Resources = { 0 }; + WORK_QUEUE_WITH_CONTEXT Context; + RtlZeroMemory(&Context, sizeof(WORK_QUEUE_WITH_CONTEXT)); + DPRINT("MmAccessFault(Mode %d, Address %x)\n", Mode, Address); if (KeGetCurrentIrql() >= DISPATCH_LEVEL) @@ -479,7 +480,6 @@ { if (Thread->ActiveFaultCount > 0) { - WORK_QUEUE_WITH_CONTEXT Context = {0}; DPRINT("Already fault handling ... going to work item (%x)\n", Address); Context.AddressSpace = AddressSpace; Context.MemoryArea = MemoryArea; @@ -589,8 +589,11 @@ BOOLEAN Locked = FromMdl; PMEMORY_AREA MemoryArea; MM_REQUIRED_RESOURCES Resources = { 0 }; + WORK_QUEUE_WITH_CONTEXT Context; NTSTATUS Status = STATUS_SUCCESS; + RtlZeroMemory(&Context, sizeof(WORK_QUEUE_WITH_CONTEXT)); + if (!FromMdl) { MmLockAddressSpace(AddressSpace); @@ -658,9 +661,8 @@ } else if (Status == STATUS_MORE_PROCESSING_REQUIRED) { - if (Thread->ActiveFaultCount > 1) + if (Thread->ActiveFaultCount > 2) { - WORK_QUEUE_WITH_CONTEXT Context = {0}; DPRINTC("Already fault handling ... going to work item (%x)\n", Address); Context.AddressSpace = AddressSpace; Context.MemoryArea = MemoryArea; Index: ntoskrnl/cache/section/io.c =================================================================== --- ntoskrnl/cache/section/io.c (revision 55684) +++ ntoskrnl/cache/section/io.c (working copy) @@ -64,6 +64,9 @@ return IoGetRelatedDeviceObject(FileObject); } +// Note: +// This completion function is really required. Paging io completion does almost +// nothing, including freeing the mdls. NTSTATUS NTAPI MiSimpleReadComplete @@ -100,9 +103,7 @@ PLARGE_INTEGER FileOffset, PVOID Buffer, ULONG Length, -#ifdef __ROS_DWARF__ BOOLEAN Paging, -#endif PIO_STATUS_BLOCK ReadStatus) { NTSTATUS Status; @@ -145,11 +146,7 @@ return STATUS_NO_MEMORY; } -#ifndef __ROS_DWARF__ - Irp->Flags |= IRP_PAGING_IO | IRP_SYNCHRONOUS_PAGING_IO | IRP_NOCACHE | IRP_SYNCHRONOUS_API; -#else Irp->Flags |= (Paging ? IRP_PAGING_IO | IRP_SYNCHRONOUS_PAGING_IO | IRP_NOCACHE : 0) | IRP_SYNCHRONOUS_API; -#endif Irp->UserEvent = &ReadWait; Irp->Tail.Overlay.OriginalFileObject = FileObject; @@ -159,14 +156,14 @@ IrpSp->FileObject = FileObject; IrpSp->CompletionRoutine = MiSimpleReadComplete; -#ifdef __ROS_DWARF__ - ObReferenceObject(FileObject); -#endif + // Non paging case, the FileObject will be dereferenced at completion + if (!Paging) + ObReferenceObject(FileObject); Status = IoCallDriver(DeviceObject, Irp); if (Status == STATUS_PENDING) { - DPRINT1("KeWaitForSingleObject(&ReadWait)\n"); + DPRINT("KeWaitForSingleObject(&ReadWait)\n"); if (!NT_SUCCESS (KeWaitForSingleObject (&ReadWait, @@ -210,13 +207,13 @@ ASSERT(Buffer); ASSERT(ReadStatus); - ObReferenceObject(FileObject); DeviceObject = MmGetDeviceObjectForFile(FileObject); ASSERT(DeviceObject); DPRINT - ("PAGING WRITE: FileObject %p Offset %x Length %d (%s:%d)\n", + ("PAGING WRITE: FileObject %x <%wZ> Offset %x Length %d (%s:%d)\n", FileObject, + &FileObject->FileName, FileOffset->LowPart, Length, File, @@ -234,7 +231,6 @@ if (!Irp) { - ObDereferenceObject(FileObject); return STATUS_NO_MEMORY; } @@ -252,11 +248,9 @@ Status = IoCallDriver(DeviceObject, Irp); DPRINT("Status %x\n", Status); - ObDereferenceObject(FileObject); - if (Status == STATUS_PENDING) { - DPRINT1("KeWaitForSingleObject(&ReadWait)\n"); + DPRINT("KeWaitForSingleObject(&ReadWait)\n"); if (!NT_SUCCESS (KeWaitForSingleObject (&ReadWait, @@ -302,7 +296,7 @@ MmDeleteHyperspaceMapping(Hyperspace); KeLowerIrql(OldIrql); - DPRINT1("MiWriteBackPage(%wZ,%08x%08x,%s:%d)\n", &FileObject->FileName, FileOffset->u.HighPart, FileOffset->u.LowPart, File, Line); + DPRINT("MiWriteBackPage(%wZ,%08x%08x,%s:%d)\n", &FileObject->FileName, FileOffset->u.HighPart, FileOffset->u.LowPart, File, Line); Status = MiSimpleWrite (FileObject, FileOffset, Index: ntoskrnl/cache/section/newmm.h =================================================================== --- ntoskrnl/cache/section/newmm.h (revision 55684) +++ ntoskrnl/cache/section/newmm.h (working copy) @@ -11,12 +11,11 @@ (IS_SWAP_FROM_SSE(E) && SWAPENTRY_FROM_SSE(E) == MM_WAIT_ENTRY) #define MAKE_PFN_SSE(P) ((P) << PAGE_SHIFT) #define SWAPENTRY_FROM_SSE(E) ((E) >> 1) -#define MAKE_SWAP_SSE(S) (((S) << 1) | 0x1) +#define MAKE_SWAP_SSE(S) (((ULONG)(S) << 1) | 0x1) #define DIRTY_SSE(E) ((E) | 2) #define CLEAN_SSE(E) ((E) & ~2) #define IS_DIRTY_SSE(E) ((E) & 2) -#define MEMORY_AREA_CACHE (2) #define MM_SEGMENT_FINALIZE (0x40000000) #define RMAP_SEGMENT_MASK ~((ULONG_PTR)0xff) @@ -31,7 +30,7 @@ (((Consumer) == MC_USER) || \ ((Consumer) == MC_CACHE)) -#define SEC_CACHE (0x40000000) +#define SEC_CACHE (0x20000000) #define MiWaitForPageEvent(Process,Address) do { \ DPRINT("MiWaitForPageEvent %x:%x #\n", Process, Address); \ @@ -47,32 +46,11 @@ #define ENTRIES_PER_ELEMENT 256 extern KEVENT MmWaitPageEvent; - -typedef struct _MM_CACHE_SECTION_SEGMENT -{ - FAST_MUTEX Lock; /* lock which protects the page directory */ - PFILE_OBJECT FileObject; - ULARGE_INTEGER RawLength; /* length of the segment which is part of the mapped file */ - ULARGE_INTEGER Length; /* absolute length of the segment */ - ULONG ReferenceCount; - ULONG Protection; - ULONG Flags; - BOOLEAN WriteCopy; - struct - { - LONG FileOffset; /* start offset into the file for image sections */ - ULONG_PTR VirtualAddress; /* dtart offset into the address range for image sections */ - ULONG Characteristics; - } Image; - - RTL_GENERIC_TABLE PageTable; -} MM_CACHE_SECTION_SEGMENT, *PMM_CACHE_SECTION_SEGMENT; - typedef struct _CACHE_SECTION_PAGE_TABLE { LARGE_INTEGER FileOffset; - PMM_CACHE_SECTION_SEGMENT Segment; + PMM_SECTION_SEGMENT Segment; ULONG Refcount; ULONG PageEntries[ENTRIES_PER_ELEMENT]; } CACHE_SECTION_PAGE_TABLE, *PCACHE_SECTION_PAGE_TABLE; @@ -111,55 +89,66 @@ int Line; } MM_REQUIRED_RESOURCES, *PMM_REQUIRED_RESOURCES; +NTSTATUS +NTAPI +MmCreateCacheSection +(PROS_SECTION_OBJECT *SectionObject, + ACCESS_MASK DesiredAccess, + POBJECT_ATTRIBUTES ObjectAttributes, + PLARGE_INTEGER UMaximumSize, + ULONG SectionPageProtection, + ULONG AllocationAttributes, + PFILE_OBJECT FileObject); + PFN_NUMBER NTAPI MmWithdrawSectionPage -(PMM_CACHE_SECTION_SEGMENT Segment, PLARGE_INTEGER FileOffset, BOOLEAN *Dirty); +(PMM_SECTION_SEGMENT Segment, PLARGE_INTEGER FileOffset, BOOLEAN *Dirty); NTSTATUS NTAPI MmFinalizeSectionPageOut -(PMM_CACHE_SECTION_SEGMENT Segment, PLARGE_INTEGER FileOffset, PFN_NUMBER Page, +(PMM_SECTION_SEGMENT Segment, PLARGE_INTEGER FileOffset, PFN_NUMBER Page, BOOLEAN Dirty); /* sptab.c *******************************************************************/ VOID NTAPI -MiInitializeSectionPageTable(PMM_CACHE_SECTION_SEGMENT Segment); +MiInitializeSectionPageTable(PMM_SECTION_SEGMENT Segment); NTSTATUS NTAPI -_MiSetPageEntryCacheSectionSegment -(PMM_CACHE_SECTION_SEGMENT Segment, +_MmSetPageEntrySectionSegment +(PMM_SECTION_SEGMENT Segment, PLARGE_INTEGER Offset, ULONG Entry, const char *file, int line); ULONG NTAPI -_MiGetPageEntryCacheSectionSegment -(PMM_CACHE_SECTION_SEGMENT Segment, +_MmGetPageEntrySectionSegment +(PMM_SECTION_SEGMENT Segment, PLARGE_INTEGER Offset, const char *file, int line); -#define MiSetPageEntryCacheSectionSegment(S,O,E) _MiSetPageEntryCacheSectionSegment(S,O,E,__FILE__,__LINE__) -#define MiGetPageEntryCacheSectionSegment(S,O) _MiGetPageEntryCacheSectionSegment(S,O,__FILE__,__LINE__) +#define MmSetPageEntrySectionSegment(S,O,E) _MmSetPageEntrySectionSegment(S,O,E,__FILE__,__LINE__) +#define MmGetPageEntrySectionSegment(S,O) _MmGetPageEntrySectionSegment(S,O,__FILE__,__LINE__) typedef VOID (NTAPI *FREE_SECTION_PAGE_FUN) - (PMM_CACHE_SECTION_SEGMENT Segment, + (PMM_SECTION_SEGMENT Segment, PLARGE_INTEGER Offset); VOID NTAPI -MiFreePageTablesSectionSegment(PMM_CACHE_SECTION_SEGMENT Segment, FREE_SECTION_PAGE_FUN FreePage); +MmFreePageTablesSectionSegment(PMM_SECTION_SEGMENT Segment, FREE_SECTION_PAGE_FUN FreePage); /* Yields a lock */ -PMM_CACHE_SECTION_SEGMENT +PMM_SECTION_SEGMENT NTAPI MmGetSectionAssociation(PFN_NUMBER Page, PLARGE_INTEGER Offset); NTSTATUS NTAPI -MmSetSectionAssociation(PFN_NUMBER Page, PMM_CACHE_SECTION_SEGMENT Segment, PLARGE_INTEGER Offset); +MmSetSectionAssociation(PFN_NUMBER Page, PMM_SECTION_SEGMENT Segment, PLARGE_INTEGER Offset); VOID NTAPI @@ -181,9 +170,7 @@ PLARGE_INTEGER FileOffset, PVOID Buffer, ULONG Length, -#ifdef __ROS_DWARF__ BOOLEAN Paging, -#endif PIO_STATUS_BLOCK ReadStatus); NTSTATUS @@ -262,7 +249,7 @@ VOID NTAPI MiFreeSegmentPage -(PMM_CACHE_SECTION_SEGMENT Segment, +(PMM_SECTION_SEGMENT Segment, PLARGE_INTEGER FileOffset); NTSTATUS @@ -283,15 +270,15 @@ VOID NTAPI -_MmLockCacheSectionSegment(PMM_CACHE_SECTION_SEGMENT Segment, const char *file, int line); +_MmLockSectionSegment(PMM_SECTION_SEGMENT Segment, const char *file, int line); -#define MmLockCacheSectionSegment(x) _MmLockCacheSectionSegment(x,__FILE__,__LINE__) +#define MmLockSectionSegment(x) _MmLockSectionSegment(x,__FILE__,__LINE__) VOID NTAPI -_MmUnlockCacheSectionSegment(PMM_CACHE_SECTION_SEGMENT Segment, const char *file, int line); +_MmUnlockSectionSegment(PMM_SECTION_SEGMENT Segment, const char *file, int line); -#define MmUnlockCacheSectionSegment(x) _MmUnlockCacheSectionSegment(x,__FILE__,__LINE__) +#define MmUnlockSectionSegment(x) _MmUnlockSectionSegment(x,__FILE__,__LINE__) VOID MmFreeCacheSectionPage @@ -306,7 +293,7 @@ VOID NTAPI -MmFinalizeSegment(PMM_CACHE_SECTION_SEGMENT Segment); +MmFinalizeSegment(PMM_SECTION_SEGMENT Segment); VOID NTAPI @@ -314,7 +301,7 @@ NTSTATUS NTAPI MmMapCacheViewInSystemSpaceAtOffset -(IN PMM_CACHE_SECTION_SEGMENT Segment, +(IN PMM_SECTION_SEGMENT Segment, OUT PVOID * MappedBase, IN PLARGE_INTEGER ViewOffset, IN OUT PULONG ViewSize); @@ -323,7 +310,7 @@ NTAPI _MiMapViewOfSegment (PMMSUPPORT AddressSpace, - PMM_CACHE_SECTION_SEGMENT Segment, + PMM_SECTION_SEGMENT Segment, PVOID* BaseAddress, SIZE_T ViewSize, ULONG Protect, @@ -388,7 +375,7 @@ NTSTATUS NTAPI -MmExtendCacheSection(PMM_CACHE_SECTION_SEGMENT Section, PLARGE_INTEGER NewSize, BOOLEAN ExtendFile); +MmExtendCacheSection(PROS_SECTION_OBJECT Section, PLARGE_INTEGER NewSize, BOOLEAN ExtendFile); NTSTATUS NTAPI @@ -396,17 +383,6 @@ #define MiFlushMappedSection(A,O,S,D) _MiFlushMappedSection(A,O,S,D,__FILE__,__LINE__) -NTSTATUS -NTAPI -MmCreateCacheSection -(PMM_CACHE_SECTION_SEGMENT *SegmentObject, - ACCESS_MASK DesiredAccess, - POBJECT_ATTRIBUTES ObjectAttributes, - PLARGE_INTEGER UMaximumSize, - ULONG SectionPageProtection, - ULONG AllocationAttributes, - PFILE_OBJECT FileObject); - PVOID NTAPI MmGetSegmentRmap(PFN_NUMBER Page, PULONG RawOffset); @@ -420,4 +396,7 @@ ULONG NTAPI -MiCacheEvictPages(PVOID BaseAddress, ULONG Target); +MiCacheEvictPages(PMM_SECTION_SEGMENT Segment, ULONG Target); + +NTSTATUS +MiRosTrimCache(ULONG Target, ULONG Priority, PULONG NrFreed); Index: ntoskrnl/cache/section/reqtools.c =================================================================== --- ntoskrnl/cache/section/reqtools.c (revision 55684) +++ ntoskrnl/cache/section/reqtools.c (working copy) @@ -91,9 +91,11 @@ PLARGE_INTEGER FileOffset = &RequiredResources->FileOffset; NTSTATUS Status; PVOID PageBuf = NULL; - PMEMORY_AREA TmpArea; IO_STATUS_BLOCK IOSB; PHYSICAL_ADDRESS BoundaryAddressMultiple; + PPFN_NUMBER Pages; + PMDL Mdl; + PVOID HyperMap; BoundaryAddressMultiple.QuadPart = 0; @@ -110,44 +112,27 @@ return Status; } - MmLockAddressSpace(MmGetKernelAddressSpace()); - Status = MmCreateMemoryArea - (MmGetKernelAddressSpace(), - MEMORY_AREA_VIRTUAL_MEMORY, - &PageBuf, - PAGE_SIZE, - PAGE_READWRITE, - &TmpArea, - FALSE, - MEM_TOP_DOWN, - BoundaryAddressMultiple); - - DPRINT("Status %x, PageBuf %x\n", Status, PageBuf); - if (!NT_SUCCESS(Status)) - { - DPRINT1("STATUS_NO_MEMORY: %x\n", Status); - MmUnlockAddressSpace(MmGetKernelAddressSpace()); + HyperMap = MmCreateHyperspaceMapping(*Page); + + Mdl = IoAllocateMdl(HyperMap, PAGE_SIZE, FALSE, FALSE, NULL); + if (!Mdl) { MmReleasePageMemoryConsumer(RequiredResources->Consumer, *Page); return STATUS_NO_MEMORY; } - - Status = MmCreateVirtualMapping(NULL, PageBuf, PAGE_READWRITE, Page, 1); - if (!NT_SUCCESS(Status)) - { - MmFreeMemoryArea(MmGetKernelAddressSpace(), TmpArea, NULL, NULL); - MmUnlockAddressSpace(MmGetKernelAddressSpace()); - MmReleasePageMemoryConsumer(RequiredResources->Consumer, *Page); - DPRINT1("Status: %x\n", Status); - return Status; - } - - MmUnlockAddressSpace(MmGetKernelAddressSpace()); + MmInitializeMdl(Mdl, HyperMap, PAGE_SIZE); + Pages = (PPFN_NUMBER)(Mdl + 1); + Pages[0] = *Page; + MmProbeAndLockPages(Mdl, KernelMode, IoModifyAccess); + PageBuf = MmGetSystemAddressForMdlSafe(Mdl, NormalPagePriority); + MmDeleteHyperspaceMapping(HyperMap); + Status = MiSimpleRead (FileObject, FileOffset, PageBuf, RequiredResources->Amount, + TRUE, &IOSB); RtlZeroMemory ((PCHAR)PageBuf+RequiredResources->Amount, @@ -155,9 +140,8 @@ DPRINT("Read Status %x (Page %x)\n", Status, *Page); - MmLockAddressSpace(MmGetKernelAddressSpace()); - MmFreeMemoryArea(MmGetKernelAddressSpace(), TmpArea, NULL, NULL); - MmUnlockAddressSpace(MmGetKernelAddressSpace()); + MmUnlockPages(Mdl); + IoFreeMdl(Mdl); if (!NT_SUCCESS(Status)) { Index: ntoskrnl/cache/section/sptab.c =================================================================== --- ntoskrnl/cache/section/sptab.c (revision 55684) +++ ntoskrnl/cache/section/sptab.c (working copy) @@ -43,7 +43,7 @@ { PVOID Result; Result = ExAllocatePoolWithTag(NonPagedPool, Bytes, 'MmPt'); - DPRINT("MiSectionPageTableAllocate(%d) => %p\n", Bytes, Result); + //DPRINT("MiSectionPageTableAllocate(%d) => %p\n", Bytes, Result); return Result; } @@ -52,7 +52,7 @@ NTAPI MiSectionPageTableFree(PRTL_GENERIC_TABLE Table, PVOID Data) { - DPRINT("MiSectionPageTableFree(%p)\n", Data); + //DPRINT("MiSectionPageTableFree(%p)\n", Data); ExFreePoolWithTag(Data, 'MmPt'); } @@ -65,6 +65,7 @@ BOOLEAN Result = (A->QuadPart < B->QuadPart) ? GenericLessThan : (A->QuadPart == B->QuadPart) ? GenericEqual : GenericGreaterThan; +#if 0 DPRINT ("Compare: %08x%08x vs %08x%08x => %s\n", A->u.HighPart, A->u.LowPart, @@ -72,6 +73,7 @@ Result == GenericLessThan ? "GenericLessThan" : Result == GenericGreaterThan ? "GenericGreaterThan" : "GenericEqual"); +#endif return Result; } @@ -103,11 +105,13 @@ PLARGE_INTEGER FileOffset) { LARGE_INTEGER SearchFileOffset; + CACHE_SECTION_PAGE_TABLE SectionZeroPageTable; PCACHE_SECTION_PAGE_TABLE PageTableSlice = MiSectionPageTableGet(Table, FileOffset); + // Please zero memory when taking away zero initialization. + RtlZeroMemory(&SectionZeroPageTable, sizeof(CACHE_SECTION_PAGE_TABLE)); if (!PageTableSlice) { - CACHE_SECTION_PAGE_TABLE SectionZeroPageTable = {0}; SearchFileOffset.QuadPart = ROUND_DOWN(FileOffset->QuadPart, ENTRIES_PER_ELEMENT * PAGE_SIZE); SectionZeroPageTable.FileOffset = SearchFileOffset; SectionZeroPageTable.Refcount = 1; @@ -125,7 +129,7 @@ VOID NTAPI -MiInitializeSectionPageTable(PMM_CACHE_SECTION_SEGMENT Segment) +MiInitializeSectionPageTable(PMM_SECTION_SEGMENT Segment) { RtlInitializeGenericTable (&Segment->PageTable, @@ -138,8 +142,8 @@ NTSTATUS NTAPI -_MiSetPageEntryCacheSectionSegment -(PMM_CACHE_SECTION_SEGMENT Segment, +_MmSetPageEntrySectionSegment +(PMM_SECTION_SEGMENT Segment, PLARGE_INTEGER Offset, ULONG Entry, const char *file, @@ -147,6 +151,9 @@ { ULONG PageIndex, OldEntry; PCACHE_SECTION_PAGE_TABLE PageTable; + ASSERT(Segment->Locked); + if (Entry && !IS_SWAP_FROM_SSE(Entry)) + MmGetRmapListHeadPage(PFN_FROM_SSE(Entry)); PageTable = MiSectionPageTableGetOrAllocate(&Segment->PageTable, Offset); if (!PageTable) return STATUS_NO_MEMORY; @@ -155,23 +162,36 @@ PageIndex = (Offset->QuadPart - PageTable->FileOffset.QuadPart) / PAGE_SIZE; OldEntry = PageTable->PageEntries[PageIndex]; - if (OldEntry && !IS_SWAP_FROM_SSE(OldEntry)) { + DPRINT("MiSetPageEntrySectionSegment(%p,%08x%08x,%x=>%x)\n", + Segment, Offset->u.HighPart, Offset->u.LowPart, OldEntry, Entry); + if (PFN_FROM_SSE(Entry) == PFN_FROM_SSE(OldEntry)) { + // Nothing + } else if (Entry && !IS_SWAP_FROM_SSE(Entry)) { + ASSERT(!OldEntry || IS_SWAP_FROM_SSE(OldEntry)); + MmSetSectionAssociation(PFN_FROM_SSE(Entry), Segment, Offset); + } else if (OldEntry && !IS_SWAP_FROM_SSE(OldEntry)) { + ASSERT(!Entry || IS_SWAP_FROM_SSE(Entry)); MmDeleteSectionAssociation(PFN_FROM_SSE(OldEntry)); + } else if (IS_SWAP_FROM_SSE(Entry)) { + ASSERT(!IS_SWAP_FROM_SSE(OldEntry)); + if (OldEntry) + MmDeleteSectionAssociation(PFN_FROM_SSE(OldEntry)); + } else if (IS_SWAP_FROM_SSE(OldEntry)) { + ASSERT(!IS_SWAP_FROM_SSE(Entry)); + if (Entry) + MmSetSectionAssociation(PFN_FROM_SSE(OldEntry), Segment, Offset); + } else { + // We should not be replacing a page like this + ASSERT(FALSE); } - if (Entry && !IS_SWAP_FROM_SSE(Entry)) { - MmSetSectionAssociation(PFN_FROM_SSE(Entry), Segment, Offset); - } PageTable->PageEntries[PageIndex] = Entry; - DPRINT - ("MiSetPageEntrySectionSegment(%p,%08x%08x,%x) %s:%d\n", - Segment, Offset->u.HighPart, Offset->u.LowPart, Entry, file, line); return STATUS_SUCCESS; } ULONG NTAPI -_MiGetPageEntryCacheSectionSegment -(PMM_CACHE_SECTION_SEGMENT Segment, +_MmGetPageEntrySectionSegment +(PMM_SECTION_SEGMENT Segment, PLARGE_INTEGER Offset, const char *file, int line) @@ -180,6 +200,7 @@ ULONG PageIndex, Result; PCACHE_SECTION_PAGE_TABLE PageTable; + ASSERT(Segment->Locked); FileOffset.QuadPart = ROUND_DOWN(Offset->QuadPart, ENTRIES_PER_ELEMENT * PAGE_SIZE); PageTable = MiSectionPageTableGet(&Segment->PageTable, &FileOffset); @@ -201,14 +222,15 @@ VOID NTAPI -MiFreePageTablesSectionSegment -(PMM_CACHE_SECTION_SEGMENT Segment, FREE_SECTION_PAGE_FUN FreePage) +MmFreePageTablesSectionSegment +(PMM_SECTION_SEGMENT Segment, FREE_SECTION_PAGE_FUN FreePage) { PCACHE_SECTION_PAGE_TABLE Element; DPRINT("MiFreePageTablesSectionSegment(%p)\n", &Segment->PageTable); while ((Element = RtlGetElementGenericTable(&Segment->PageTable, 0))) { DPRINT - ("Delete table for %x -> %08x%08x\n", + ("Delete table for <%wZ> %x -> %08x%08x\n", + Segment->FileObject ? &Segment->FileObject->FileName : NULL, Segment, Element->FileOffset.u.HighPart, Element->FileOffset.u.LowPart); @@ -223,7 +245,7 @@ Entry = Element->PageEntries[i]; if (Entry && !IS_SWAP_FROM_SSE(Entry)) { - DPRINTC("Freeing page %x:%x @ %x\n", Segment, Entry, Offset.LowPart); + DPRINT("Freeing page %x:%x @ %x\n", Segment, Entry, Offset.LowPart); FreePage(Segment, &Offset); } } @@ -234,12 +256,12 @@ DPRINT("Done\n"); } -PMM_CACHE_SECTION_SEGMENT +PMM_SECTION_SEGMENT NTAPI MmGetSectionAssociation(PFN_NUMBER Page, PLARGE_INTEGER Offset) { ULONG RawOffset; - PMM_CACHE_SECTION_SEGMENT Segment = NULL; + PMM_SECTION_SEGMENT Segment = NULL; PCACHE_SECTION_PAGE_TABLE PageTable; PageTable = (PCACHE_SECTION_PAGE_TABLE)MmGetSegmentRmap(Page, &RawOffset); @@ -254,7 +276,7 @@ NTSTATUS NTAPI -MmSetSectionAssociation(PFN_NUMBER Page, PMM_CACHE_SECTION_SEGMENT Segment, PLARGE_INTEGER Offset) +MmSetSectionAssociation(PFN_NUMBER Page, PMM_SECTION_SEGMENT Segment, PLARGE_INTEGER Offset) { PCACHE_SECTION_PAGE_TABLE PageTable; ULONG ActualOffset; Index: ntoskrnl/cache/section/swapout.c =================================================================== --- ntoskrnl/cache/section/swapout.c (revision 55684) +++ ntoskrnl/cache/section/swapout.c (working copy) @@ -55,19 +55,19 @@ extern KEVENT MmWaitPageEvent; extern FAST_MUTEX RmapListLock; -FAST_MUTEX GlobalPageOperation; +FAST_MUTEX MiGlobalPageOperation; PFN_NUMBER NTAPI MmWithdrawSectionPage -(PMM_CACHE_SECTION_SEGMENT Segment, PLARGE_INTEGER FileOffset, BOOLEAN *Dirty) +(PMM_SECTION_SEGMENT Segment, PLARGE_INTEGER FileOffset, BOOLEAN *Dirty) { ULONG Entry; DPRINT("MmWithdrawSectionPage(%x,%08x%08x,%x)\n", Segment, FileOffset->HighPart, FileOffset->LowPart, Dirty); - MmLockCacheSectionSegment(Segment); - Entry = MiGetPageEntryCacheSectionSegment(Segment, FileOffset); + MmLockSectionSegment(Segment); + Entry = MmGetPageEntrySectionSegment(Segment, FileOffset); *Dirty = !!IS_DIRTY_SSE(Entry); @@ -76,28 +76,28 @@ if (!Entry) { DPRINT("Stoeled!\n"); - MmUnlockCacheSectionSegment(Segment); + MmUnlockSectionSegment(Segment); return 0; } else if (MM_IS_WAIT_PTE(Entry)) { DPRINT("WAIT\n"); - MmUnlockCacheSectionSegment(Segment); + MmUnlockSectionSegment(Segment); return MM_WAIT_ENTRY; } else if (Entry && !IS_SWAP_FROM_SSE(Entry)) { DPRINT("Page %x\n", PFN_FROM_SSE(Entry)); *Dirty |= (Entry & 2); - MiSetPageEntryCacheSectionSegment(Segment, FileOffset, MAKE_SWAP_SSE(MM_WAIT_ENTRY)); - MmUnlockCacheSectionSegment(Segment); + MmSetPageEntrySectionSegment(Segment, FileOffset, MAKE_SWAP_SSE(MM_WAIT_ENTRY)); + MmUnlockSectionSegment(Segment); return PFN_FROM_SSE(Entry); } else { DPRINT1("SWAP ENTRY?! (%x:%08x%08x)\n", Segment, FileOffset->HighPart, FileOffset->LowPart); ASSERT(FALSE); - MmUnlockCacheSectionSegment(Segment); + MmUnlockSectionSegment(Segment); return 0; } } @@ -105,14 +105,14 @@ NTSTATUS NTAPI MmFinalizeSectionPageOut -(PMM_CACHE_SECTION_SEGMENT Segment, PLARGE_INTEGER FileOffset, PFN_NUMBER Page, +(PMM_SECTION_SEGMENT Segment, PLARGE_INTEGER FileOffset, PFN_NUMBER Page, BOOLEAN Dirty) { NTSTATUS Status = STATUS_SUCCESS; BOOLEAN WriteZero = FALSE, WritePage = FALSE; SWAPENTRY Swap = MmGetSavedSwapEntryPage(Page); - MmLockCacheSectionSegment(Segment); + MmLockSectionSegment(Segment); (void)InterlockedIncrementUL(&Segment->ReferenceCount); if (Dirty) @@ -131,7 +131,7 @@ DPRINT("Status %x\n", Status); - MmUnlockCacheSectionSegment(Segment); + MmUnlockSectionSegment(Segment); if (WritePage) { @@ -139,17 +139,17 @@ Status = MiWriteBackPage(Segment->FileObject, FileOffset, PAGE_SIZE, Page); } - MmLockCacheSectionSegment(Segment); + MmLockSectionSegment(Segment); if (WriteZero && NT_SUCCESS(Status)) { DPRINT("Setting page entry in segment %x:%x to swap %x\n", Segment, FileOffset->LowPart, Swap); - MiSetPageEntryCacheSectionSegment(Segment, FileOffset, Swap ? MAKE_SWAP_SSE(Swap) : 0); + MmSetPageEntrySectionSegment(Segment, FileOffset, Swap ? MAKE_SWAP_SSE(Swap) : 0); } else { DPRINT("Setting page entry in segment %x:%x to page %x\n", Segment, FileOffset->LowPart, Page); - MiSetPageEntryCacheSectionSegment + MmSetPageEntrySectionSegment (Segment, FileOffset, Page ? (Dirty ? DIRTY_SSE(MAKE_PFN_SSE(Page)) : MAKE_PFN_SSE(Page)) : 0); } @@ -164,7 +164,7 @@ MmDereferencePage(Page); } - MmUnlockCacheSectionSegment(Segment); + MmUnlockSectionSegment(Segment); if (InterlockedDecrementUL(&Segment->ReferenceCount) == 0) { @@ -191,24 +191,24 @@ BOOLEAN Dirty = FALSE; PEPROCESS Process = MmGetAddressSpaceOwner(AddressSpace); LARGE_INTEGER TotalOffset; - PMM_CACHE_SECTION_SEGMENT Segment; + PMM_SECTION_SEGMENT Segment; PVOID PAddress = MM_ROUND_DOWN(Address, PAGE_SIZE); TotalOffset.QuadPart = (ULONG_PTR)PAddress - (ULONG_PTR)MemoryArea->StartingAddress + - MemoryArea->Data.CacheData.ViewOffset.QuadPart; + MemoryArea->Data.SectionData.ViewOffset.QuadPart; - Segment = MemoryArea->Data.CacheData.Segment; + Segment = MemoryArea->Data.SectionData.Segment; - MmLockCacheSectionSegment(Segment); + MmLockSectionSegment(Segment); ASSERT(KeGetCurrentIrql() <= APC_LEVEL); Dirty = MmIsDirtyPageRmap(Required->Page[0]); - Entry = MiGetPageEntryCacheSectionSegment(Segment, &TotalOffset); + Entry = MmGetPageEntrySectionSegment(Segment, &TotalOffset); if (Dirty) { PFN_NUMBER OurPage; - MiSetPageEntryCacheSectionSegment(Segment, &TotalOffset, DIRTY_SSE(Entry)); + MmSetPageEntrySectionSegment(Segment, &TotalOffset, DIRTY_SSE(Entry)); MmDeleteRmap(Required->Page[0], Process, Address); MmDeleteVirtualMapping(Process, Address, FALSE, &Dirty, &OurPage); ASSERT(OurPage == Required->Page[0]); @@ -226,7 +226,7 @@ MmDereferencePage(Required->Page[0]); } - MmUnlockCacheSectionSegment(Segment); + MmUnlockSectionSegment(Segment); MiSetPageEvent(Process, Address); return Status; } @@ -238,7 +238,7 @@ BOOLEAN ProcRef = FALSE; PFN_NUMBER SectionPage = 0; PMM_RMAP_ENTRY entry; - PMM_CACHE_SECTION_SEGMENT Segment = NULL; + PMM_SECTION_SEGMENT Segment = NULL; LARGE_INTEGER FileOffset; PMEMORY_AREA MemoryArea; PMMSUPPORT AddressSpace = MmGetKernelAddressSpace(); @@ -250,17 +250,17 @@ DPRINTC("Page out %x (ref ct %x)\n", Page, MmGetReferenceCountPage(Page)); - ExAcquireFastMutex(&GlobalPageOperation); + ExAcquireFastMutex(&MiGlobalPageOperation); if ((Segment = MmGetSectionAssociation(Page, &FileOffset))) { - DPRINT1("Withdrawing page (%x) %x:%x\n", Page, Segment, FileOffset.LowPart); + DPRINTC("Withdrawing page (%x) %x:%x\n", Page, Segment, FileOffset.LowPart); SectionPage = MmWithdrawSectionPage(Segment, &FileOffset, &Dirty); DPRINTC("SectionPage %x\n", SectionPage); if (SectionPage == MM_WAIT_ENTRY || SectionPage == 0) { DPRINT1("In progress page out %x\n", SectionPage); - ExReleaseFastMutex(&GlobalPageOperation); + ExReleaseFastMutex(&MiGlobalPageOperation); return STATUS_UNSUCCESSFUL; } else @@ -434,9 +434,9 @@ DPRINTC ("Failed to page out %x, replacing %x at %x in segment %x\n", SectionPage, FileOffset.LowPart, Segment); - MmLockCacheSectionSegment(Segment); - MiSetPageEntryCacheSectionSegment(Segment, &FileOffset, Dirty ? MAKE_PFN_SSE(Page) : DIRTY_SSE(MAKE_PFN_SSE(Page))); - MmUnlockCacheSectionSegment(Segment); + MmLockSectionSegment(Segment); + MmSetPageEntrySectionSegment(Segment, &FileOffset, Dirty ? MAKE_PFN_SSE(Page) : DIRTY_SSE(MAKE_PFN_SSE(Page))); + MmUnlockSectionSegment(Segment); } // Alas, we had the last reference @@ -450,60 +450,71 @@ ObDereferenceObject(Process); } - ExReleaseFastMutex(&GlobalPageOperation); + ExReleaseFastMutex(&MiGlobalPageOperation); DPRINTC("%s %x %x\n", NT_SUCCESS(Status) ? "Evicted" : "Spared", Page, Status); return NT_SUCCESS(Status) ? STATUS_SUCCESS : STATUS_UNSUCCESSFUL; } ULONG NTAPI -MiCacheEvictPages(PVOID BaseAddress, ULONG Target) +MiCacheEvictPages(PMM_SECTION_SEGMENT Segment, ULONG Target) { - ULONG i, Entry, Result = 0; + ULONG Entry, Result = 0, i, j; NTSTATUS Status; PFN_NUMBER Page; - PMEMORY_AREA MemoryArea; LARGE_INTEGER Offset; - PMM_CACHE_SECTION_SEGMENT Segment; - MmLockAddressSpace(MmGetKernelAddressSpace()); - MemoryArea = MmLocateMemoryAreaByAddress - (MmGetKernelAddressSpace(), - BaseAddress); + MmLockSectionSegment(Segment); - ASSERT(MemoryArea); - ASSERT(MemoryArea->Type == MEMORY_AREA_CACHE); + for (i = 0; i < RtlNumberGenericTableElements(&Segment->PageTable); i++) { + PCACHE_SECTION_PAGE_TABLE Element = RtlGetElementGenericTable(&Segment->PageTable, i); + ASSERT(Element); + Offset = Element->FileOffset; + for (j = 0; j < ENTRIES_PER_ELEMENT; j++, Offset.QuadPart += PAGE_SIZE) { + Entry = MmGetPageEntrySectionSegment(Segment, &Offset); + if (Entry && !IS_SWAP_FROM_SSE(Entry)) { + Page = PFN_FROM_SSE(Entry); + MmReferencePage(Page); + MmUnlockSectionSegment(Segment); + Status = MmpPageOutPhysicalAddress(Page); + if (NT_SUCCESS(Status)) + Result++; + MmLockSectionSegment(Segment); + MmReleasePageMemoryConsumer(MC_CACHE, Page); + } + } + } - Segment = MemoryArea->Data.CacheData.Segment; + MmUnlockSectionSegment(Segment); - ASSERT(Segment); + return Result; +} - MmLockCacheSectionSegment(Segment); +extern LIST_ENTRY MiSegmentList; - for (i = 0; - i < ((ULONG_PTR)MemoryArea->EndingAddress) - - ((ULONG_PTR)MemoryArea->StartingAddress) && - Result < Target; - i += PAGE_SIZE) { - Offset.QuadPart = MemoryArea->Data.CacheData.ViewOffset.QuadPart + i; - Entry = MiGetPageEntryCacheSectionSegment(Segment, &Offset); - if (Entry && !IS_SWAP_FROM_SSE(Entry)) { - Page = PFN_FROM_SSE(Entry); - MmReferencePage(Page); - MmUnlockCacheSectionSegment(Segment); - MmUnlockAddressSpace(MmGetKernelAddressSpace()); - Status = MmpPageOutPhysicalAddress(Page); - if (NT_SUCCESS(Status)) - Result++; - MmLockCacheSectionSegment(Segment); - MmLockAddressSpace(MmGetKernelAddressSpace()); - MmReleasePageMemoryConsumer(MC_CACHE, Page); - } - } +// Interact with legacy balance manager for now +// This can fall away when our section implementation supports +// demand paging properly +NTSTATUS +MiRosTrimCache(ULONG Target, ULONG Priority, PULONG NrFreed) +{ + ULONG Freed; + PLIST_ENTRY Entry; + PMM_SECTION_SEGMENT Segment; + *NrFreed = 0; - MmUnlockCacheSectionSegment(Segment); - MmUnlockAddressSpace(MmGetKernelAddressSpace()); + for (Entry = MiSegmentList.Flink; *NrFreed < Target && Entry != &MiSegmentList; Entry = Entry->Flink) { + Segment = CONTAINING_RECORD(Entry, MM_SECTION_SEGMENT, ListOfSegments); + // Defer to MM to try recovering pages from it + Freed = MiCacheEvictPages(Segment, Target); + *NrFreed += Freed; + } - return Result; + if (!IsListEmpty(&MiSegmentList)) { + Entry = MiSegmentList.Flink; + RemoveEntryList(Entry); + InsertTailList(&MiSegmentList, Entry); + } + + return STATUS_SUCCESS; } - Index: ntoskrnl/cc/fs.c =================================================================== --- ntoskrnl/cc/fs.c (revision 55684) +++ ntoskrnl/cc/fs.c (working copy) @@ -248,3 +248,21 @@ return NT_SUCCESS(CcRosReleaseFileCache(FileObject)); #endif } + +BOOLEAN +NTAPI +CcGetFileSizes +(IN PFILE_OBJECT FileObject, + IN PCC_FILE_SIZES FileSizes) +{ + PBCB Bcb; + + Bcb = FileObject->SectionObjectPointer->SharedCacheMap; + + if (!Bcb) + return FALSE; + + FileSizes->AllocationSize = Bcb->AllocationSize; + FileSizes->FileSize = FileSizes->ValidDataLength = Bcb->FileSize; + return TRUE; +} Index: ntoskrnl/CMakeLists.txt =================================================================== --- ntoskrnl/CMakeLists.txt (revision 55684) +++ ntoskrnl/CMakeLists.txt (working copy) @@ -18,7 +18,9 @@ set_rc_compiler() -set(NEWCC FALSE) +if(NOT DEFINED NEWCC) + set(NEWCC FALSE) +endif(NOT DEFINED NEWCC) if(NEWCC) add_definitions(-DNEWCC) @@ -29,12 +31,7 @@ cache/lazyrite.c cache/logsup.c cache/mdlsup.c - cache/pinsup.c - cache/section/data.c - cache/section/fault.c - cache/section/reqtools.c - cache/section/sptab.c - cache/section/swapout.c) + cache/pinsup.c) else() list(APPEND SOURCE cc/cacheman.c @@ -47,6 +44,11 @@ list(APPEND SOURCE cache/section/io.c + cache/section/data.c + cache/section/fault.c + cache/section/reqtools.c + cache/section/sptab.c + cache/section/swapout.c config/cmalloc.c config/cmapi.c config/cmboot.c Index: ntoskrnl/include/internal/mm.h =================================================================== --- ntoskrnl/include/internal/mm.h (revision 55684) +++ ntoskrnl/include/internal/mm.h (working copy) @@ -74,6 +74,7 @@ #endif #define MEMORY_AREA_SECTION_VIEW (1) +#define MEMORY_AREA_CACHE (2) #define MEMORY_AREA_VIRTUAL_MEMORY (8) #define MEMORY_AREA_OWNED_BY_ARM3 (15) #define MEMORY_AREA_STATIC (0x80000000) @@ -193,29 +194,28 @@ #define InterlockedExchangePte(PointerPte, Value) \ InterlockedExchange((PLONG)(PointerPte), Value) -typedef struct -{ - ULONG Entry[NR_SECTION_PAGE_ENTRIES]; -} SECTION_PAGE_TABLE, *PSECTION_PAGE_TABLE; - -typedef struct -{ - PSECTION_PAGE_TABLE PageTables[NR_SECTION_PAGE_TABLES]; -} SECTION_PAGE_DIRECTORY, *PSECTION_PAGE_DIRECTORY; - typedef struct _MM_SECTION_SEGMENT { - LONG FileOffset; /* start offset into the file for image sections */ - ULONG_PTR VirtualAddress; /* dtart offset into the address range for image sections */ - ULONG RawLength; /* length of the segment which is part of the mapped file */ - SIZE_T Length; /* absolute length of the segment */ - ULONG Protection; FAST_MUTEX Lock; /* lock which protects the page directory */ + PFILE_OBJECT FileObject; + LARGE_INTEGER RawLength; /* length of the segment which is part of the mapped file */ + LARGE_INTEGER Length; /* absolute length of the segment */ ULONG ReferenceCount; - SECTION_PAGE_DIRECTORY PageDirectory; + ULONG CacheCount; + ULONG Protection; ULONG Flags; - ULONG Characteristics; BOOLEAN WriteCopy; + BOOLEAN Locked; + + struct + { + LONG FileOffset; /* start offset into the file for image sections */ + ULONG_PTR VirtualAddress; /* dtart offset into the address range for image sections */ + ULONG Characteristics; + } Image; + + LIST_ENTRY ListOfSegments; + RTL_GENERIC_TABLE PageTable; } MM_SECTION_SEGMENT, *PMM_SECTION_SEGMENT; typedef struct _MM_IMAGE_SECTION_OBJECT @@ -250,8 +250,6 @@ }; } ROS_SECTION_OBJECT, *PROS_SECTION_OBJECT; -struct _MM_CACHE_SECTION_SEGMENT; - typedef struct _MEMORY_AREA { PVOID StartingAddress; @@ -270,15 +268,10 @@ struct { ROS_SECTION_OBJECT* Section; - ULONG ViewOffset; + LARGE_INTEGER ViewOffset; PMM_SECTION_SEGMENT Segment; LIST_ENTRY RegionListHead; } SectionData; - struct - { - LARGE_INTEGER ViewOffset; - struct _MM_CACHE_SECTION_SEGMENT *Segment; - } CacheData; struct { LIST_ENTRY RegionListHead; @@ -1647,7 +1640,8 @@ MmNotPresentFaultSectionView( PMMSUPPORT AddressSpace, MEMORY_AREA* MemoryArea, - PVOID Address + PVOID Address, + BOOLEAN Locked ); NTSTATUS Index: ntoskrnl/mm/amd64/page.c =================================================================== --- ntoskrnl/mm/amd64/page.c (revision 55684) +++ ntoskrnl/mm/amd64/page.c (working copy) @@ -617,6 +617,8 @@ ULONG i; MMPTE TmplPte, *Pte; + ASSERT((ULONG_PTR)Address % PAGE_SIZE == 0); + /* Check if the range is valid */ if ((Process == NULL && Address < MmSystemRangeStart) || (Process != NULL && Address > MmHighestUserAddress)) @@ -667,6 +669,8 @@ { ULONG i; + ASSERT((ULONG_PTR)Address % PAGE_SIZE == 0); + for (i = 0; i < PageCount; i++) { if (!MmIsPageInUse(Pages[i])) Index: ntoskrnl/mm/i386/page.c =================================================================== --- ntoskrnl/mm/i386/page.c (revision 55684) +++ ntoskrnl/mm/i386/page.c (working copy) @@ -841,6 +841,7 @@ { ULONG i; + ASSERT((ULONG_PTR)Address % PAGE_SIZE == 0); for (i = 0; i < PageCount; i++) { if (!MmIsPageInUse(Pages[i])) Index: ntoskrnl/mm/marea.c =================================================================== --- ntoskrnl/mm/marea.c (revision 55684) +++ ntoskrnl/mm/marea.c (working copy) @@ -43,6 +43,7 @@ #include #define NDEBUG +#include "../cache/section/newmm.h" #include MEMORY_AREA MiStaticMemoryAreas[MI_STATIC_MEMORY_AREAS]; @@ -376,7 +377,7 @@ { PMMVAD Vad; - ASSERT(marea->Type == MEMORY_AREA_VIRTUAL_MEMORY || marea->Type == MEMORY_AREA_SECTION_VIEW); + ASSERT(marea->Type == MEMORY_AREA_VIRTUAL_MEMORY || marea->Type == MEMORY_AREA_SECTION_VIEW || marea->Type == MEMORY_AREA_CACHE); Vad = ExAllocatePoolWithTag(NonPagedPool, sizeof(MMVAD), TAG_MVAD); ASSERT(Vad); RtlZeroMemory(Vad, sizeof(MMVAD)); @@ -744,7 +745,7 @@ if (MemoryArea->Vad) { ASSERT(MemoryArea->EndingAddress < MmSystemRangeStart); - ASSERT(MemoryArea->Type == MEMORY_AREA_VIRTUAL_MEMORY || MemoryArea->Type == MEMORY_AREA_SECTION_VIEW); + ASSERT(MemoryArea->Type == MEMORY_AREA_VIRTUAL_MEMORY || MemoryArea->Type == MEMORY_AREA_SECTION_VIEW || MemoryArea->Type == MEMORY_AREA_CACHE); /* MmCleanProcessAddressSpace might have removed it (and this would be MmDeleteProcessAdressSpace) */ ASSERT(((PMMVAD)MemoryArea->Vad)->u.VadFlags.Spare != 0); @@ -1024,6 +1025,13 @@ MmLockAddressSpace(&Process->Vm); break; + case MEMORY_AREA_CACHE: + Address = (PVOID)MemoryArea->StartingAddress; + MmUnlockAddressSpace(&Process->Vm); + MmUnmapViewOfCacheSegment(&Process->Vm, Address); + MmLockAddressSpace(&Process->Vm); + break; + case MEMORY_AREA_VIRTUAL_MEMORY: MmFreeVirtualMemory(Process, MemoryArea); break; Index: ntoskrnl/mm/mmfault.c =================================================================== --- ntoskrnl/mm/mmfault.c (revision 55684) +++ ntoskrnl/mm/mmfault.c (working copy) @@ -9,9 +9,7 @@ /* INCLUDES *******************************************************************/ #include -#ifdef NEWCC #include "../cache/section/newmm.h" -#endif #define NDEBUG #include @@ -86,17 +84,15 @@ Status = STATUS_ACCESS_VIOLATION; break; -#ifdef NEWCC case MEMORY_AREA_CACHE: // This code locks for itself to keep from having to break a lock // passed in. if (!FromMdl) MmUnlockAddressSpace(AddressSpace); - Status = MmAccessFaultCacheSection(Mode, Address, Locked); + Status = MmAccessFaultCacheSection(Mode, Address, FromMdl); if (!FromMdl) MmLockAddressSpace(AddressSpace); break; -#endif default: Status = STATUS_ACCESS_VIOLATION; @@ -176,7 +172,8 @@ case MEMORY_AREA_SECTION_VIEW: Status = MmNotPresentFaultSectionView(AddressSpace, MemoryArea, - (PVOID)Address); + (PVOID)Address, + FromMdl); break; case MEMORY_AREA_VIRTUAL_MEMORY: @@ -185,17 +182,15 @@ (PVOID)Address); break; -#ifdef NEWCC case MEMORY_AREA_CACHE: // This code locks for itself to keep from having to break a lock // passed in. if (!FromMdl) MmUnlockAddressSpace(AddressSpace); - Status = MmNotPresentFaultCacheSection(Mode, Address, Locked); + Status = MmNotPresentFaultCacheSection(Mode, Address, FromMdl); if (!FromMdl) MmLockAddressSpace(AddressSpace); break; -#endif default: Status = STATUS_ACCESS_VIOLATION; Index: ntoskrnl/mm/mminit.c =================================================================== --- ntoskrnl/mm/mminit.c (revision 55684) +++ ntoskrnl/mm/mminit.c (working copy) @@ -31,6 +31,11 @@ PMMPTE MmSharedUserDataPte; PMMSUPPORT MmKernelAddressSpace; +extern KEVENT MmWaitPageEvent; +extern FAST_MUTEX MiGlobalPageOperation; +extern LIST_ENTRY MiSegmentList; +extern NTSTATUS MiRosTrimCache(ULONG Target, ULONG Priority, PULONG NrFreed); + /* PRIVATE FUNCTIONS *********************************************************/ VOID @@ -394,6 +399,14 @@ /* Initialize the kernel address space */ ASSERT(Phase == 1); + + InitializeListHead(&MiSegmentList); + ExInitializeFastMutex(&MiGlobalPageOperation); + KeInitializeEvent(&MmWaitPageEvent, SynchronizationEvent, FALSE); + // Until we're fully demand paged, we can do things the old way through + // the balance manager + MmInitializeMemoryConsumer(MC_CACHE, MiRosTrimCache); + KeInitializeGuardedMutex(&PsIdleProcess->AddressCreationLock); MmKernelAddressSpace = &PsIdleProcess->Vm; Index: ntoskrnl/mm/rmap.c =================================================================== --- ntoskrnl/mm/rmap.c (revision 55684) +++ ntoskrnl/mm/rmap.c (working copy) @@ -10,9 +10,7 @@ /* INCLUDES *****************************************************************/ #include -#ifdef NEWCC #include "../cache/section/newmm.h" -#endif #define NDEBUG #include @@ -112,8 +110,8 @@ Type = MemoryArea->Type; if (Type == MEMORY_AREA_SECTION_VIEW) { - Offset = (ULONG)((ULONG_PTR)Address - (ULONG_PTR)MemoryArea->StartingAddress - + MemoryArea->Data.SectionData.ViewOffset); + Offset = (ULONG_PTR)Address - (ULONG_PTR)MemoryArea->StartingAddress + + MemoryArea->Data.SectionData.ViewOffset.QuadPart; /* * Get or create a pageop @@ -197,9 +195,7 @@ } while (current_entry != NULL) { -#ifdef NEWCC if (!RMAP_IS_SEGMENT(current_entry->Address)) -#endif MmSetCleanPage(current_entry->Process, current_entry->Address); current_entry = current_entry->Next; } @@ -221,9 +217,7 @@ } while (current_entry != NULL) { -#ifdef NEWCC if (!RMAP_IS_SEGMENT(current_entry->Address)) -#endif MmSetDirtyPage(current_entry->Process, current_entry->Address); current_entry = current_entry->Next; } @@ -246,9 +240,7 @@ while (current_entry != NULL) { if ( -#ifdef NEWCC !RMAP_IS_SEGMENT(current_entry->Address) && -#endif MmIsDirtyPage(current_entry->Process, current_entry->Address)) { ExReleaseFastMutex(&RmapListLock); @@ -268,9 +260,7 @@ PMM_RMAP_ENTRY current_entry; PMM_RMAP_ENTRY new_entry; ULONG PrevSize; -#ifdef NEWCC if (!RMAP_IS_SEGMENT(Address)) -#endif Address = (PVOID)PAGE_ROUND_DOWN(Address); new_entry = ExAllocateFromNPagedLookasideList(&RmapLookasideList); @@ -289,13 +279,12 @@ #endif if ( -#ifdef NEWCC !RMAP_IS_SEGMENT(Address) && -#endif MmGetPfnForProcess(Process, Address) != Page) { DPRINT1("Insert rmap (%d, 0x%.8X) 0x%.8X which doesn't match physical " - "address 0x%.8X\n", Process->UniqueProcessId, Address, + "address 0x%.8X\n", Process ? Process->UniqueProcessId : 0, + Address, MmGetPfnForProcess(Process, Address) << PAGE_SHIFT, Page << PAGE_SHIFT); KeBugCheck(MEMORY_MANAGEMENT); @@ -322,9 +311,7 @@ #endif MmSetRmapListHeadPage(Page, new_entry); ExReleaseFastMutex(&RmapListLock); -#ifdef NEWCC if (!RMAP_IS_SEGMENT(Address)) -#endif { if (Process == NULL) { @@ -360,13 +347,12 @@ } MmSetRmapListHeadPage(Page, NULL); ExReleaseFastMutex(&RmapListLock); + while (current_entry != NULL) { previous_entry = current_entry; current_entry = current_entry->Next; -#ifdef NEWCC if (!RMAP_IS_SEGMENT(previous_entry->Address)) -#endif { if (DeleteMapping) { @@ -384,12 +370,10 @@ (void)InterlockedExchangeAddUL(&Process->Vm.WorkingSetSize, -PAGE_SIZE); } } -#ifdef NEWCC else { ExFreeToNPagedLookasideList(&RmapLookasideList, previous_entry); } -#endif } } @@ -419,9 +403,7 @@ } ExReleaseFastMutex(&RmapListLock); ExFreeToNPagedLookasideList(&RmapLookasideList, current_entry); -#ifdef NEWCC if (!RMAP_IS_SEGMENT(Address)) -#endif { if (Process == NULL) { @@ -440,7 +422,6 @@ KeBugCheck(MEMORY_MANAGEMENT); } -#ifdef NEWCC PVOID NTAPI MmGetSegmentRmap(PFN_NUMBER Page, PULONG RawOffset) @@ -498,4 +479,3 @@ } ExReleaseFastMutex(&RmapListLock); } -#endif Index: ntoskrnl/mm/section.c =================================================================== --- ntoskrnl/mm/section.c (revision 55684) +++ ntoskrnl/mm/section.c (working copy) @@ -45,9 +45,8 @@ /* INCLUDES *****************************************************************/ #include -#ifdef NEWCC +#include "../cache/newcc.h" #include "../cache/section/newmm.h" -#endif #define NDEBUG #include #include @@ -57,6 +56,12 @@ #pragma alloc_text(INIT, MmInitSectionImplementation) #endif +#undef MmSetPageEntrySectionSegment +#define MmSetPageEntrySectionSegment(S,O,E) do { \ + DPRINT("SetPageEntrySectionSegment(old,%x,%x,%x)\n", S,(O)->LowPart,E); \ + _MmSetPageEntrySectionSegment(S,O,E,__FILE__,__LINE__); \ + } while (0) + NTSTATUS NTAPI MiMapViewInSystemSpace(IN PVOID Section, @@ -114,7 +119,7 @@ { PROS_SECTION_OBJECT Section; PMM_SECTION_SEGMENT Segment; - ULONG Offset; + LARGE_INTEGER Offset; BOOLEAN WasDirty; BOOLEAN Private; } @@ -159,10 +164,10 @@ #define PAGE_FROM_SSE(E) ((E) & 0xFFFFF000) #define PFN_FROM_SSE(E) ((E) >> PAGE_SHIFT) -#define SHARE_COUNT_FROM_SSE(E) (((E) & 0x00000FFE) >> 1) +#define SHARE_COUNT_FROM_SSE(E) (((E) & 0x00000FFC) >> 2) #define IS_SWAP_FROM_SSE(E) ((E) & 0x00000001) -#define MAX_SHARE_COUNT 0x7FF -#define MAKE_SSE(P, C) ((ULONG)(P) | ((C) << 1)) +#define MAX_SHARE_COUNT 0x3FF +#define MAKE_SSE(P, C) ((P) | ((C) << 2)) #define SWAPENTRY_FROM_SSE(E) ((E) >> 1) #define MAKE_SWAP_SSE(S) (((ULONG)(S) << 1) | 0x1) @@ -597,12 +602,12 @@ if (nPrevVirtualEndOfSegment < cbHeadersSize) DIE(("Cannot align the size of the section headers\n")); - pssSegments[0].FileOffset = 0; + pssSegments[0].Image.FileOffset = 0; pssSegments[0].Protection = PAGE_READONLY; - pssSegments[0].Length = nPrevVirtualEndOfSegment; - pssSegments[0].RawLength = nFileSizeOfHeaders; - pssSegments[0].VirtualAddress = 0; - pssSegments[0].Characteristics = IMAGE_SCN_CNT_INITIALIZED_DATA; + pssSegments[0].Length.QuadPart = nPrevVirtualEndOfSegment; + pssSegments[0].RawLength.QuadPart = nFileSizeOfHeaders; + pssSegments[0].Image.VirtualAddress = 0; + pssSegments[0].Image.Characteristics = IMAGE_SCN_CNT_INITIALIZED_DATA; pssSegments[0].WriteCopy = TRUE; /* skip the headers segment */ @@ -617,7 +622,7 @@ /* validate the alignment */ if(!IsAligned(pishSectionHeaders[i].VirtualAddress, nSectionAlignment)) - DIE(("VirtualAddress[%u] is not aligned\n", i)); + DIE(("Image.VirtualAddress[%u] is not aligned\n", i)); /* sections must be contiguous, ordered by base address and non-overlapping */ if(pishSectionHeaders[i].VirtualAddress != nPrevVirtualEndOfSegment) @@ -639,16 +644,16 @@ // DIE(("PointerToRawData[%u] is not aligned\n", i)); /* conversion */ - pssSegments[i].FileOffset = pishSectionHeaders[i].PointerToRawData; - pssSegments[i].RawLength = pishSectionHeaders[i].SizeOfRawData; + pssSegments[i].Image.FileOffset = pishSectionHeaders[i].PointerToRawData; + pssSegments[i].RawLength.QuadPart = pishSectionHeaders[i].SizeOfRawData; } else { - ASSERT(pssSegments[i].FileOffset == 0); - ASSERT(pssSegments[i].RawLength == 0); + ASSERT(pssSegments[i].Image.FileOffset == 0); + ASSERT(pssSegments[i].RawLength.QuadPart == 0); } - ASSERT(Intsafe_CanAddLong64(pssSegments[i].FileOffset, pssSegments[i].RawLength)); + ASSERT(Intsafe_CanAddLong64(pssSegments[i].Image.FileOffset, pssSegments[i].RawLength.QuadPart)); nCharacteristics = pishSectionHeaders[i].Characteristics; @@ -670,23 +675,23 @@ pssSegments[i].WriteCopy = !(nCharacteristics & IMAGE_SCN_MEM_SHARED); if(pishSectionHeaders[i].Misc.VirtualSize == 0 || pishSectionHeaders[i].Misc.VirtualSize < pishSectionHeaders[i].SizeOfRawData) - pssSegments[i].Length = pishSectionHeaders[i].SizeOfRawData; + pssSegments[i].Length.QuadPart = pishSectionHeaders[i].SizeOfRawData; else - pssSegments[i].Length = pishSectionHeaders[i].Misc.VirtualSize; + pssSegments[i].Length.QuadPart = pishSectionHeaders[i].Misc.VirtualSize; - pssSegments[i].Length = ALIGN_UP_BY(pssSegments[i].Length, nSectionAlignment); - if (pssSegments[i].Length < pssSegments[i].Length) + pssSegments[i].Length.LowPart = ALIGN_UP_BY(pssSegments[i].Length.LowPart, nSectionAlignment); + if (pssSegments[i].Length.QuadPart < pssSegments[i].Length.QuadPart) DIE(("Cannot align the virtual size of section %u\n", i)); - if(pssSegments[i].Length == 0) + if(pssSegments[i].Length.QuadPart == 0) DIE(("Virtual size of section %u is null\n", i)); - pssSegments[i].VirtualAddress = pishSectionHeaders[i].VirtualAddress; - pssSegments[i].Characteristics = pishSectionHeaders[i].Characteristics; + pssSegments[i].Image.VirtualAddress = pishSectionHeaders[i].VirtualAddress; + pssSegments[i].Image.Characteristics = pishSectionHeaders[i].Characteristics; /* ensure the memory image is no larger than 4GB */ - nPrevVirtualEndOfSegment = pssSegments[i].VirtualAddress + pssSegments[i].Length; - if (nPrevVirtualEndOfSegment < pssSegments[i].VirtualAddress) + nPrevVirtualEndOfSegment = pssSegments[i].Image.VirtualAddress + pssSegments[i].Length.QuadPart; + if (nPrevVirtualEndOfSegment < pssSegments[i].Image.VirtualAddress) DIE(("The image is too large\n")); } @@ -752,24 +757,7 @@ //return KeWaitForSingleObject(&File->Lock, 0, KernelMode, FALSE, NULL); } - VOID -MmFreePageTablesSectionSegment(PMM_SECTION_SEGMENT Segment) -{ - ULONG i; - if (Segment->Length > NR_SECTION_PAGE_TABLES * PAGE_SIZE) - { - for (i = 0; i < NR_SECTION_PAGE_TABLES; i++) - { - if (Segment->PageDirectory.PageTables[i] != NULL) - { - ExFreePool(Segment->PageDirectory.PageTables[i]); - } - } - } -} - -VOID NTAPI MmFreeSectionSegments(PFILE_OBJECT FileObject) { @@ -791,7 +779,7 @@ SectionSegments[i].ReferenceCount); KeBugCheck(MEMORY_MANAGEMENT); } - MmFreePageTablesSectionSegment(&SectionSegments[i]); + MmFreePageTablesSectionSegment(&SectionSegments[i], NULL); } ExFreePool(ImageSectionObject->Segments); ExFreePool(ImageSectionObject); @@ -809,7 +797,7 @@ DPRINT1("Data segment still referenced\n"); KeBugCheck(MEMORY_MANAGEMENT); } - MmFreePageTablesSectionSegment(Segment); + MmFreePageTablesSectionSegment(Segment, NULL); ExFreePool(Segment); FileObject->SectionObjectPointer->DataSectionObject = NULL; } @@ -817,90 +805,8 @@ VOID NTAPI -MmLockSectionSegment(PMM_SECTION_SEGMENT Segment) -{ - ExAcquireFastMutex(&Segment->Lock); -} - -VOID -NTAPI -MmUnlockSectionSegment(PMM_SECTION_SEGMENT Segment) -{ - ExReleaseFastMutex(&Segment->Lock); -} - -VOID -NTAPI -MmSetPageEntrySectionSegment(PMM_SECTION_SEGMENT Segment, - ULONG_PTR Offset, - ULONG_PTR Entry) -{ - PSECTION_PAGE_TABLE Table; - ULONG_PTR DirectoryOffset; - ULONG_PTR TableOffset; - - if (Segment->Length <= NR_SECTION_PAGE_TABLES * PAGE_SIZE) - { - Table = (PSECTION_PAGE_TABLE)&Segment->PageDirectory; - } - else - { - DirectoryOffset = PAGE_TO_SECTION_PAGE_DIRECTORY_OFFSET(Offset); - Table = Segment->PageDirectory.PageTables[DirectoryOffset]; - if (Table == NULL) - { - Table = - Segment->PageDirectory.PageTables[DirectoryOffset] = - ExAllocatePoolWithTag(NonPagedPool, sizeof(SECTION_PAGE_TABLE), - TAG_SECTION_PAGE_TABLE); - if (Table == NULL) - { - KeBugCheck(MEMORY_MANAGEMENT); - } - memset(Table, 0, sizeof(SECTION_PAGE_TABLE)); - DPRINT("Table %x\n", Table); - } - } - TableOffset = PAGE_TO_SECTION_PAGE_TABLE_OFFSET(Offset); - Table->Entry[TableOffset] = (ULONG)Entry; -} - - -ULONG -NTAPI -MmGetPageEntrySectionSegment(PMM_SECTION_SEGMENT Segment, - ULONG_PTR Offset) -{ - PSECTION_PAGE_TABLE Table; - ULONG Entry; - ULONG_PTR DirectoryOffset; - ULONG_PTR TableOffset; - - DPRINT("MmGetPageEntrySection(Segment %x, Offset %x)\n", Segment, Offset); - - if (Segment->Length <= NR_SECTION_PAGE_TABLES * PAGE_SIZE) - { - Table = (PSECTION_PAGE_TABLE)&Segment->PageDirectory; - } - else - { - DirectoryOffset = PAGE_TO_SECTION_PAGE_DIRECTORY_OFFSET(Offset); - Table = Segment->PageDirectory.PageTables[DirectoryOffset]; - DPRINT("Table %x\n", Table); - if (Table == NULL) - { - return(0); - } - } - TableOffset = PAGE_TO_SECTION_PAGE_TABLE_OFFSET(Offset); - Entry = Table->Entry[TableOffset]; - return(Entry); -} - -VOID -NTAPI MmSharePageEntrySectionSegment(PMM_SECTION_SEGMENT Segment, - ULONG_PTR Offset) + PLARGE_INTEGER Offset) { ULONG Entry; @@ -927,7 +833,7 @@ NTAPI MmUnsharePageEntrySectionSegment(PROS_SECTION_OBJECT Section, PMM_SECTION_SEGMENT Segment, - ULONG Offset, + PLARGE_INTEGER Offset, BOOLEAN Dirty, BOOLEAN PageOut) { @@ -957,39 +863,43 @@ if (SHARE_COUNT_FROM_SSE(Entry) == 0) { PFILE_OBJECT FileObject; +#ifndef NEWCC PBCB Bcb; +#endif SWAPENTRY SavedSwapEntry; PFN_NUMBER Page; BOOLEAN IsImageSection; - ULONG FileOffset; + LARGE_INTEGER FileOffset; - FileOffset = Offset + Segment->FileOffset; + FileOffset.QuadPart = Offset->QuadPart + Segment->Image.FileOffset; IsImageSection = Section->AllocationAttributes & SEC_IMAGE ? TRUE : FALSE; Page = PFN_FROM_SSE(Entry); FileObject = Section->FileObject; if (FileObject != NULL && - !(Segment->Characteristics & IMAGE_SCN_MEM_SHARED)) + !(Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) { - if ((FileOffset % PAGE_SIZE) == 0 && - (Offset + PAGE_SIZE <= Segment->RawLength || !IsImageSection)) +#ifndef NEWCC + if ((FileOffset.QuadPart % PAGE_SIZE) == 0 && + (Offset->QuadPart + PAGE_SIZE <= Segment->RawLength.QuadPart || !IsImageSection)) { NTSTATUS Status; Bcb = FileObject->SectionObjectPointer->SharedCacheMap; IsDirectMapped = TRUE; #ifndef NEWCC - Status = CcRosUnmapCacheSegment(Bcb, FileOffset, Dirty); + Status = CcRosUnmapCacheSegment(Bcb, FileOffset.LowPart, Dirty); #else Status = STATUS_SUCCESS; #endif if (!NT_SUCCESS(Status)) { DPRINT1("CcRosUnmapCacheSegment failed, status = %x\n", Status); - KeBugCheck(MEMORY_MANAGEMENT); + KeBugCheck(MEMORY_MANAGEMENT); } } +#endif } SavedSwapEntry = MmGetSavedSwapEntryPage(Page); @@ -997,7 +907,7 @@ { if (!PageOut && ((Segment->Flags & MM_PAGEFILE_SEGMENT) || - (Segment->Characteristics & IMAGE_SCN_MEM_SHARED))) + (Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED))) { /* * FIXME: @@ -1020,7 +930,7 @@ else { if ((Segment->Flags & MM_PAGEFILE_SEGMENT) || - (Segment->Characteristics & IMAGE_SCN_MEM_SHARED)) + (Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) { if (!PageOut) { @@ -1062,12 +972,12 @@ ULONG SegOffset) { #ifndef NEWCC - if (!(MemoryArea->Data.SectionData.Segment->Characteristics & IMAGE_SCN_MEM_SHARED)) + if (!(MemoryArea->Data.SectionData.Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) { PBCB Bcb; PCACHE_SEGMENT CacheSeg; Bcb = MemoryArea->Data.SectionData.Section->FileObject->SectionObjectPointer->SharedCacheMap; - CacheSeg = CcRosLookupCacheSegment(Bcb, SegOffset + MemoryArea->Data.SectionData.Segment->FileOffset); + CacheSeg = CcRosLookupCacheSegment(Bcb, SegOffset + MemoryArea->Data.SectionData.Segment->Image.FileOffset); if (CacheSeg) { CcRosReleaseCacheSegment(Bcb, CacheSeg, CacheSeg->Valid, FALSE, TRUE); @@ -1086,13 +996,15 @@ KIRQL Irql; PVOID TempAddress; + ASSERT((ULONG_PTR)SourceAddress % PAGE_SIZE == 0); Process = PsGetCurrentProcess(); TempAddress = MiMapPageInHyperSpace(Process, DestPage, &Irql); if (TempAddress == NULL) { return(STATUS_NO_MEMORY); } - memcpy(TempAddress, SourceAddress, PAGE_SIZE); + ASSERT((ULONG_PTR)TempAddress % PAGE_SIZE == 0); + RtlCopyMemory(TempAddress, SourceAddress, PAGE_SIZE); MiUnmapPageInHyperSpace(Process, TempAddress, Irql); return(STATUS_SUCCESS); } @@ -1125,8 +1037,8 @@ FileObject = MemoryArea->Data.SectionData.Section->FileObject; Bcb = FileObject->SectionObjectPointer->SharedCacheMap; - RawLength = MemoryArea->Data.SectionData.Segment->RawLength; - FileOffset = SegOffset + MemoryArea->Data.SectionData.Segment->FileOffset; + RawLength = MemoryArea->Data.SectionData.Segment->RawLength.QuadPart; + FileOffset = SegOffset + MemoryArea->Data.SectionData.Segment->Image.FileOffset; IsImageSection = MemoryArea->Data.SectionData.Section->AllocationAttributes & SEC_IMAGE ? TRUE : FALSE; ASSERT(Bcb); @@ -1140,7 +1052,7 @@ */ if ((FileOffset % PAGE_SIZE) == 0 && (SegOffset + PAGE_SIZE <= RawLength || !IsImageSection) && - !(MemoryArea->Data.SectionData.Segment->Characteristics & IMAGE_SCN_MEM_SHARED)) + !(MemoryArea->Data.SectionData.Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) { /* @@ -1290,18 +1202,20 @@ * Page - Variable that receives a page contains the read data. */ { - MM_REQUIRED_RESOURCES Resources = {0}; + MM_REQUIRED_RESOURCES Resources; NTSTATUS Status; + RtlZeroMemory(&Resources, sizeof(MM_REQUIRED_RESOURCES)); + Resources.Context = MemoryArea->Data.SectionData.Section->FileObject; Resources.FileOffset.QuadPart = SegOffset + - MemoryArea->Data.SectionData.Segment->FileOffset; + MemoryArea->Data.SectionData.Segment->Image.FileOffset; Resources.Consumer = MC_USER; Resources.Amount = PAGE_SIZE; DPRINT1("%S, offset %x, len %d, page %x\n", ((PFILE_OBJECT)Resources.Context)->FileName.Buffer, Resources.FileOffset.LowPart, Resources.Amount, Resources.Page[0]); - Status = MiReadFilePage(NULL, NULL, &Resources); + Status = MiReadFilePage(MmGetKernelAddressSpace(), MemoryArea, &Resources); *Page = Resources.Page[0]; return Status; } @@ -1311,9 +1225,10 @@ NTAPI MmNotPresentFaultSectionView(PMMSUPPORT AddressSpace, MEMORY_AREA* MemoryArea, - PVOID Address) + PVOID Address, + BOOLEAN Locked) { - ULONG Offset; + LARGE_INTEGER Offset; PFN_NUMBER Page; NTSTATUS Status; PROS_SECTION_OBJECT Section; @@ -1324,6 +1239,7 @@ PMM_PAGEOP PageOp; PMM_REGION Region; BOOLEAN HasSwapEntry; + PVOID PAddress; PEPROCESS Process = MmGetAddressSpaceOwner(AddressSpace); /* @@ -1344,15 +1260,16 @@ return(STATUS_UNSUCCESSFUL); } - Address = MM_ROUND_DOWN(Address, PAGE_SIZE); - Offset = (ULONG)((ULONG_PTR)Address - (ULONG_PTR)MemoryArea->StartingAddress - + MemoryArea->Data.SectionData.ViewOffset); + PAddress = MM_ROUND_DOWN(Address, PAGE_SIZE); + Offset.QuadPart = (ULONG_PTR)PAddress - (ULONG_PTR)MemoryArea->StartingAddress + + MemoryArea->Data.SectionData.ViewOffset.QuadPart; Segment = MemoryArea->Data.SectionData.Segment; Section = MemoryArea->Data.SectionData.Section; Region = MmFindRegion(MemoryArea->StartingAddress, &MemoryArea->Data.SectionData.RegionListHead, Address, NULL); + ASSERT(Region != NULL); /* * Lock the segment */ @@ -1375,7 +1292,7 @@ /* * Get or create a page operation descriptor */ - PageOp = MmGetPageOp(MemoryArea, NULL, 0, Segment, Offset, MM_PAGEOP_PAGEIN, FALSE); + PageOp = MmGetPageOp(MemoryArea, NULL, 0, Segment, Offset.LowPart, MM_PAGEOP_PAGEIN, FALSE); if (PageOp == NULL) { DPRINT1("MmGetPageOp failed\n"); @@ -1432,8 +1349,8 @@ */ if (!MmIsPagePresent(Process, Address)) { - Entry = MmGetPageEntrySectionSegment(Segment, Offset); - HasSwapEntry = MmIsPageSwapEntry(Process, (PVOID)Address); + Entry = MmGetPageEntrySectionSegment(Segment, &Offset); + HasSwapEntry = MmIsPageSwapEntry(Process, (PVOID)PAddress); if (PAGE_FROM_SSE(Entry) == 0 || HasSwapEntry) { @@ -1447,13 +1364,13 @@ Page = PFN_FROM_SSE(Entry); - MmSharePageEntrySectionSegment(Segment, Offset); + MmSharePageEntrySectionSegment(Segment, &Offset); /* FIXME: Should we call MmCreateVirtualMappingUnsafe if * (Section->AllocationAttributes & SEC_PHYSICALMEMORY) is true? */ Status = MmCreateVirtualMapping(Process, - Address, + PAddress, Attributes, &Page, 1); @@ -1547,9 +1464,9 @@ /* * Just map the desired physical page */ - Page = Offset >> PAGE_SHIFT; + Page = Offset.QuadPart >> PAGE_SHIFT; Status = MmCreateVirtualMappingUnsafe(Process, - Address, + PAddress, Region->Protect, &Page, 1); @@ -1572,7 +1489,7 @@ /* * Map anonymous memory for BSS sections */ - if (Segment->Characteristics & IMAGE_SCN_CNT_UNINITIALIZED_DATA) + if (Segment->Image.Characteristics & IMAGE_SCN_CNT_UNINITIALIZED_DATA) { MmUnlockSectionSegment(Segment); MI_SET_USAGE(MI_USAGE_SECTION); @@ -1590,7 +1507,7 @@ KeBugCheck(MEMORY_MANAGEMENT); } Status = MmCreateVirtualMapping(Process, - Address, + PAddress, Region->Protect, &Page, 1); @@ -1614,7 +1531,7 @@ /* * Get the entry corresponding to the offset within the section */ - Entry = MmGetPageEntrySectionSegment(Segment, Offset); + Entry = MmGetPageEntrySectionSegment(Segment, &Offset); if (Entry == 0) { @@ -1630,7 +1547,7 @@ MmUnlockAddressSpace(AddressSpace); if ((Segment->Flags & MM_PAGEFILE_SEGMENT) || - (Offset >= PAGE_ROUND_UP(Segment->RawLength) && Section->AllocationAttributes & SEC_IMAGE)) + (Offset.QuadPart >= PAGE_ROUND_UP(Segment->RawLength.QuadPart) && Section->AllocationAttributes & SEC_IMAGE)) { MI_SET_USAGE(MI_USAGE_SECTION); if (Process) MI_SET_PROCESS2(Process->ImageFileName); @@ -1644,7 +1561,7 @@ } else { - Status = MiReadPage(MemoryArea, Offset, &Page); + Status = MiReadPage(MemoryArea, Offset.QuadPart, &Page); if (!NT_SUCCESS(Status)) { DPRINT1("MiReadPage failed (Status %x)\n", Status); @@ -1674,7 +1591,7 @@ * Check the entry. No one should change the status of a page * that has a pending page-in. */ - Entry1 = MmGetPageEntrySectionSegment(Segment, Offset); + Entry1 = MmGetPageEntrySectionSegment(Segment, &Offset); if (Entry != Entry1) { DPRINT1("Someone changed ppte entry while we slept\n"); @@ -1686,11 +1603,11 @@ * data */ Entry = MAKE_SSE(Page << PAGE_SHIFT, 1); - MmSetPageEntrySectionSegment(Segment, Offset, Entry); + MmSetPageEntrySectionSegment(Segment, &Offset, Entry); MmUnlockSectionSegment(Segment); Status = MmCreateVirtualMapping(Process, - Address, + PAddress, Attributes, &Page, 1); @@ -1743,7 +1660,7 @@ * Check the entry. No one should change the status of a page * that has a pending page-in. */ - Entry1 = MmGetPageEntrySectionSegment(Segment, Offset); + Entry1 = MmGetPageEntrySectionSegment(Segment, &Offset); if (Entry != Entry1) { DPRINT1("Someone changed ppte entry while we slept\n"); @@ -1755,7 +1672,7 @@ * data */ Entry = MAKE_SSE(Page << PAGE_SHIFT, 1); - MmSetPageEntrySectionSegment(Segment, Offset, Entry); + MmSetPageEntrySectionSegment(Segment, &Offset, Entry); MmUnlockSectionSegment(Segment); /* @@ -1787,11 +1704,11 @@ Page = PFN_FROM_SSE(Entry); - MmSharePageEntrySectionSegment(Segment, Offset); + MmSharePageEntrySectionSegment(Segment, &Offset); MmUnlockSectionSegment(Segment); Status = MmCreateVirtualMapping(Process, - Address, + PAddress, Attributes, &Page, 1); @@ -1819,7 +1736,8 @@ PFN_NUMBER OldPage; PFN_NUMBER NewPage; NTSTATUS Status; - ULONG Offset; + PVOID PAddress; + LARGE_INTEGER Offset; PMM_PAGEOP PageOp; PMM_REGION Region; ULONG Entry; @@ -1839,22 +1757,23 @@ /* * Find the offset of the page */ - Address = MM_ROUND_DOWN(Address, PAGE_SIZE); - Offset = (ULONG)((ULONG_PTR)Address - (ULONG_PTR)MemoryArea->StartingAddress - + MemoryArea->Data.SectionData.ViewOffset); + PAddress = MM_ROUND_DOWN(Address, PAGE_SIZE); + Offset.QuadPart = (ULONG_PTR)PAddress - (ULONG_PTR)MemoryArea->StartingAddress + + MemoryArea->Data.SectionData.ViewOffset.QuadPart; Segment = MemoryArea->Data.SectionData.Segment; Section = MemoryArea->Data.SectionData.Section; Region = MmFindRegion(MemoryArea->StartingAddress, &MemoryArea->Data.SectionData.RegionListHead, Address, NULL); + ASSERT(Region != NULL); /* * Lock the segment */ MmLockSectionSegment(Segment); OldPage = MmGetPfnForProcess(Process, Address); - Entry = MmGetPageEntrySectionSegment(Segment, Offset); + Entry = MmGetPageEntrySectionSegment(Segment, &Offset); MmUnlockSectionSegment(Segment); @@ -1880,7 +1799,7 @@ /* * Get or create a pageop */ - PageOp = MmGetPageOp(MemoryArea, NULL, 0, Segment, Offset, + PageOp = MmGetPageOp(MemoryArea, NULL, 0, Segment, Offset.LowPart, MM_PAGEOP_ACCESSFAULT, FALSE); if (PageOp == NULL) { @@ -1937,7 +1856,7 @@ /* * Copy the old page */ - MiCopyFromUserPage(NewPage, Address); + MiCopyFromUserPage(NewPage, PAddress); MmLockAddressSpace(AddressSpace); /* @@ -1949,7 +1868,7 @@ * Set the PTE to point to the new page */ Status = MmCreateVirtualMapping(Process, - Address, + PAddress, Region->Protect, &NewPage, 1); @@ -1968,10 +1887,10 @@ /* * Unshare the old page. */ - MmDeleteRmap(OldPage, Process, Address); - MmInsertRmap(NewPage, Process, Address); + MmDeleteRmap(OldPage, Process, PAddress); + MmInsertRmap(NewPage, Process, PAddress); MmLockSectionSegment(Segment); - MmUnsharePageEntrySectionSegment(Section, Segment, Offset, FALSE, FALSE); + MmUnsharePageEntrySectionSegment(Section, Segment, &Offset, FALSE, FALSE); MmUnlockSectionSegment(Segment); PageOp->Status = STATUS_SUCCESS; @@ -2007,7 +1926,7 @@ MmLockSectionSegment(PageOutContext->Segment); MmUnsharePageEntrySectionSegment((PROS_SECTION_OBJECT)PageOutContext->Section, PageOutContext->Segment, - PageOutContext->Offset, + &PageOutContext->Offset, PageOutContext->WasDirty, TRUE); MmUnlockSectionSegment(PageOutContext->Segment); @@ -2035,7 +1954,6 @@ PFN_NUMBER Page; MM_SECTION_PAGEOUT_CONTEXT Context; SWAPENTRY SwapEntry; - ULONG Entry; ULONG FileOffset; NTSTATUS Status; PFILE_OBJECT FileObject; @@ -2046,6 +1964,7 @@ BOOLEAN IsImageSection; PEPROCESS Process = MmGetAddressSpaceOwner(AddressSpace); KIRQL OldIrql; + ULONG Entry; Address = (PVOID)PAGE_ROUND_DOWN(Address); @@ -2055,17 +1974,18 @@ Context.Segment = MemoryArea->Data.SectionData.Segment; Context.Section = MemoryArea->Data.SectionData.Section; - Context.Offset = (ULONG)((ULONG_PTR)Address - (ULONG_PTR)MemoryArea->StartingAddress - + MemoryArea->Data.SectionData.ViewOffset); - FileOffset = Context.Offset + Context.Segment->FileOffset; + Context.Offset.QuadPart = (ULONG_PTR)Address - (ULONG_PTR)MemoryArea->StartingAddress + + MemoryArea->Data.SectionData.ViewOffset.QuadPart; + FileOffset = Context.Offset.QuadPart + Context.Segment->Image.FileOffset; IsImageSection = Context.Section->AllocationAttributes & SEC_IMAGE ? TRUE : FALSE; FileObject = Context.Section->FileObject; DirectMapped = FALSE; + #ifndef NEWCC if (FileObject != NULL && - !(Context.Segment->Characteristics & IMAGE_SCN_MEM_SHARED)) + !(Context.Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) { Bcb = FileObject->SectionObjectPointer->SharedCacheMap; @@ -2075,7 +1995,7 @@ * then note this is a direct mapped page. */ if ((FileOffset % PAGE_SIZE) == 0 && - (Context.Offset + PAGE_SIZE <= Context.Segment->RawLength || !IsImageSection)) + (Context.Offset.QuadPart + PAGE_SIZE <= Context.Segment->RawLength.QuadPart || !IsImageSection)) { DirectMapped = TRUE; } @@ -2098,7 +2018,6 @@ /* * Get the section segment entry and the physical address. */ - Entry = MmGetPageEntrySectionSegment(Context.Segment, Context.Offset); if (!MmIsPagePresent(Process, Address)) { DPRINT1("Trying to page out not-present page at (%d,0x%.8X).\n", @@ -2123,8 +2042,11 @@ /* * Prepare the context structure for the rmap delete call. */ + MmLockSectionSegment(Context.Segment); + Entry = MmGetPageEntrySectionSegment(Context.Segment, &Context.Offset); + MmUnlockSectionSegment(Context.Segment); Context.WasDirty = FALSE; - if (Context.Segment->Characteristics & IMAGE_SCN_CNT_UNINITIALIZED_DATA || + if (Context.Segment->Image.Characteristics & IMAGE_SCN_CNT_UNINITIALIZED_DATA || IS_SWAP_FROM_SSE(Entry) || PFN_FROM_SSE(Entry) != Page) { @@ -2140,7 +2062,7 @@ */ if (DirectMapped && !Context.Private) { - if(!MiIsPageFromCache(MemoryArea, Context.Offset)) + if(!MiIsPageFromCache(MemoryArea, Context.Offset.LowPart)) { DPRINT1("Direct mapped non private page is not associated with the cache.\n"); KeBugCheck(MEMORY_MANAGEMENT); @@ -2154,17 +2076,20 @@ } MmDeleteAllRmaps(Page, (PVOID)&Context, MmPageOutDeleteMapping); + MmLockSectionSegment(Context.Segment); + Entry = MmGetPageEntrySectionSegment(Context.Segment, &Context.Offset); + MmUnlockSectionSegment(Context.Segment); /* * If this wasn't a private page then we should have reduced the entry to * zero by deleting all the rmaps. */ - if (!Context.Private && MmGetPageEntrySectionSegment(Context.Segment, Context.Offset) != 0) + if (!Context.Private && Entry != 0) { if (!(Context.Segment->Flags & MM_PAGEFILE_SEGMENT) && - !(Context.Segment->Characteristics & IMAGE_SCN_MEM_SHARED)) + !(Context.Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) { - KeBugCheck(MEMORY_MANAGEMENT); + KeBugCheckEx(MEMORY_MANAGEMENT, Entry, (ULONG_PTR)Process, (ULONG_PTR)Address, 0); } } @@ -2182,32 +2107,36 @@ { DPRINT1("Found a %s private page (address %x) in a pagefile segment.\n", Context.WasDirty ? "dirty" : "clean", Address); - KeBugCheck(MEMORY_MANAGEMENT); + KeBugCheckEx(MEMORY_MANAGEMENT, SwapEntry, (ULONG_PTR)Process, (ULONG_PTR)Address, 0); } if (!Context.WasDirty && SwapEntry != 0) { MmSetSavedSwapEntryPage(Page, 0); - MmSetPageEntrySectionSegment(Context.Segment, Context.Offset, MAKE_SWAP_SSE(SwapEntry)); + MmLockSectionSegment(Context.Segment); + MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, MAKE_SWAP_SSE(SwapEntry)); + MmUnlockSectionSegment(Context.Segment); MmReleasePageMemoryConsumer(MC_USER, Page); PageOp->Status = STATUS_SUCCESS; MmspCompleteAndReleasePageOp(PageOp); return(STATUS_SUCCESS); } } - else if (Context.Segment->Characteristics & IMAGE_SCN_MEM_SHARED) + else if (Context.Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED) { if (Context.Private) { DPRINT1("Found a %s private page (address %x) in a shared section segment.\n", Context.WasDirty ? "dirty" : "clean", Address); - KeBugCheck(MEMORY_MANAGEMENT); + KeBugCheckEx(MEMORY_MANAGEMENT, Page, (ULONG_PTR)Process, (ULONG_PTR)Address, 0); } if (!Context.WasDirty || SwapEntry != 0) { MmSetSavedSwapEntryPage(Page, 0); if (SwapEntry != 0) { - MmSetPageEntrySectionSegment(Context.Segment, Context.Offset, MAKE_SWAP_SSE(SwapEntry)); + MmLockSectionSegment(Context.Segment); + MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, MAKE_SWAP_SSE(SwapEntry)); + MmUnlockSectionSegment(Context.Segment); } MmReleasePageMemoryConsumer(MC_USER, Page); PageOp->Status = STATUS_SUCCESS; @@ -2221,18 +2150,20 @@ { DPRINT1("Found a swapentry for a non private and direct mapped page (address %x)\n", Address); - KeBugCheck(MEMORY_MANAGEMENT); + KeBugCheckEx(MEMORY_MANAGEMENT, STATUS_UNSUCCESSFUL, SwapEntry, (ULONG_PTR)Process, (ULONG_PTR)Address); } #ifndef NEWCC Status = CcRosUnmapCacheSegment(Bcb, FileOffset, FALSE); #else Status = STATUS_SUCCESS; #endif +#ifndef NEWCC if (!NT_SUCCESS(Status)) { DPRINT1("CCRosUnmapCacheSegment failed, status = %x\n", Status); - KeBugCheck(MEMORY_MANAGEMENT); + KeBugCheckEx(MEMORY_MANAGEMENT, Status, (ULONG_PTR)Bcb, FileOffset, (ULONG_PTR)Address); } +#endif PageOp->Status = STATUS_SUCCESS; MmspCompleteAndReleasePageOp(PageOp); return(STATUS_SUCCESS); @@ -2243,7 +2174,7 @@ { DPRINT1("Found a swap entry for a non dirty, non private and not direct mapped page (address %x)\n", Address); - KeBugCheck(MEMORY_MANAGEMENT); + KeBugCheckEx(MEMORY_MANAGEMENT, SwapEntry, Page, (ULONG_PTR)Process, (ULONG_PTR)Address); } MmReleasePageMemoryConsumer(MC_USER, Page); PageOp->Status = STATUS_SUCCESS; @@ -2252,6 +2183,7 @@ } else if (!Context.WasDirty && Context.Private && SwapEntry != 0) { + DPRINT1("Not dirty and private and not swapped (%p:%p)\n", Process, Address); MmSetSavedSwapEntryPage(Page, 0); MmLockAddressSpace(AddressSpace); Status = MmCreatePageFileMapping(Process, @@ -2260,7 +2192,8 @@ MmUnlockAddressSpace(AddressSpace); if (!NT_SUCCESS(Status)) { - KeBugCheck(MEMORY_MANAGEMENT); + DPRINT1("Status %x Swapping out %p:%p\n", Status, Process, Address); + KeBugCheckEx(MEMORY_MANAGEMENT, Status, (ULONG_PTR)Process, (ULONG_PTR)Address, SwapEntry); } MmReleasePageMemoryConsumer(MC_USER, Page); PageOp->Status = STATUS_SUCCESS; @@ -2310,7 +2243,7 @@ Process, Address); Entry = MAKE_SSE(Page << PAGE_SHIFT, 1); - MmSetPageEntrySectionSegment(Context.Segment, Context.Offset, Entry); + MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, Entry); } MmUnlockAddressSpace(AddressSpace); PageOp->Status = STATUS_UNSUCCESSFUL; @@ -2356,7 +2289,7 @@ Process, Address); Entry = MAKE_SSE(Page << PAGE_SHIFT, 1); - MmSetPageEntrySectionSegment(Context.Segment, Context.Offset, Entry); + MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, Entry); } MmUnlockAddressSpace(AddressSpace); PageOp->Status = STATUS_UNSUCCESSFUL; @@ -2368,11 +2301,12 @@ * Otherwise we have succeeded. */ DPRINT("MM: Wrote section page 0x%.8X to swap!\n", Page << PAGE_SHIFT); + MmLockSectionSegment(Context.Segment); MmSetSavedSwapEntryPage(Page, 0); if (Context.Segment->Flags & MM_PAGEFILE_SEGMENT || - Context.Segment->Characteristics & IMAGE_SCN_MEM_SHARED) + Context.Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED) { - MmSetPageEntrySectionSegment(Context.Segment, Context.Offset, MAKE_SWAP_SSE(SwapEntry)); + MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, MAKE_SWAP_SSE(SwapEntry)); } else { @@ -2388,15 +2322,17 @@ MmUnlockAddressSpace(AddressSpace); if (!NT_SUCCESS(Status)) { - KeBugCheck(MEMORY_MANAGEMENT); + DPRINT1("Status %x Creating page file mapping for %p:%p\n", Status, Process, Address); + KeBugCheckEx(MEMORY_MANAGEMENT, Status, (ULONG_PTR)Process, (ULONG_PTR)Address, SwapEntry); } } else { Entry = MAKE_SWAP_SSE(SwapEntry); - MmSetPageEntrySectionSegment(Context.Segment, Context.Offset, Entry); + MmSetPageEntrySectionSegment(Context.Segment, &Context.Offset, Entry); } + MmUnlockSectionSegment(Context.Segment); PageOp->Status = STATUS_SUCCESS; MmspCompleteAndReleasePageOp(PageOp); return(STATUS_SUCCESS); @@ -2409,7 +2345,7 @@ PVOID Address, PMM_PAGEOP PageOp) { - ULONG_PTR Offset; + LARGE_INTEGER Offset; PROS_SECTION_OBJECT Section; PMM_SECTION_SEGMENT Segment; PFN_NUMBER Page; @@ -2425,8 +2361,8 @@ Address = (PVOID)PAGE_ROUND_DOWN(Address); - Offset = (ULONG_PTR)Address - (ULONG_PTR)MemoryArea->StartingAddress - + MemoryArea->Data.SectionData.ViewOffset; + Offset.QuadPart = (ULONG_PTR)Address - (ULONG_PTR)MemoryArea->StartingAddress + + MemoryArea->Data.SectionData.ViewOffset.QuadPart; /* * Get the segment and section. @@ -2438,7 +2374,7 @@ FileObject = Section->FileObject; DirectMapped = FALSE; if (FileObject != NULL && - !(Segment->Characteristics & IMAGE_SCN_MEM_SHARED)) + !(Segment->Image.Characteristics & IMAGE_SCN_MEM_SHARED)) { Bcb = FileObject->SectionObjectPointer->SharedCacheMap; @@ -2447,8 +2383,8 @@ * memory area was mapped at an offset in the file which is page aligned * then note this is a direct mapped page. */ - if (((Offset + Segment->FileOffset) % PAGE_SIZE) == 0 && - (Offset + PAGE_SIZE <= Segment->RawLength || !IsImageSection)) + if (((Offset.QuadPart + Segment->Image.FileOffset) % PAGE_SIZE) == 0 && + (Offset.QuadPart + PAGE_SIZE <= Segment->RawLength.QuadPart || !IsImageSection)) { DirectMapped = TRUE; } @@ -2469,7 +2405,7 @@ /* * Get the section segment entry and the physical address. */ - Entry = MmGetPageEntrySectionSegment(Segment, Offset); + Entry = MmGetPageEntrySectionSegment(Segment, &Offset); if (!MmIsPagePresent(Process, Address)) { DPRINT1("Trying to page out not-present page at (%d,0x%.8X).\n", @@ -2482,7 +2418,7 @@ /* * Check for a private (COWed) page. */ - if (Segment->Characteristics & IMAGE_SCN_CNT_UNINITIALIZED_DATA || + if (Segment->Image.Characteristics & IMAGE_SCN_CNT_UNINITIALIZED_DATA || IS_SWAP_FROM_SSE(Entry) || PFN_FROM_SSE(Entry) != Page) { @@ -2506,7 +2442,7 @@ { ASSERT(SwapEntry == 0); #ifndef NEWCC - CcRosMarkDirtyCacheSegment(Bcb, (ULONG)Offset + Segment->FileOffset); + CcRosMarkDirtyCacheSegment(Bcb, Offset.LowPart + Segment->Image.FileOffset); #endif PageOp->Status = STATUS_SUCCESS; MmspCompleteAndReleasePageOp(PageOp); @@ -2568,8 +2504,9 @@ PEPROCESS Process = MmGetAddressSpaceOwner(AddressSpace); MemoryArea = MmLocateMemoryAreaByAddress(AddressSpace, BaseAddress); - ASSERT(MemoryArea); + ASSERT(MemoryArea != NULL); Segment = MemoryArea->Data.SectionData.Segment; + MmLockSectionSegment(Segment); if ((Segment->WriteCopy) && (NewProtect == PAGE_READWRITE || NewProtect == PAGE_EXECUTE_READWRITE)) @@ -2590,17 +2527,17 @@ */ if (DoCOW && MmIsPagePresent(Process, Address)) { - ULONG_PTR Offset; + LARGE_INTEGER Offset; ULONG Entry; PFN_NUMBER Page; - Offset = (ULONG_PTR)Address - (ULONG_PTR)MemoryArea->StartingAddress - + MemoryArea->Data.SectionData.ViewOffset; - Entry = MmGetPageEntrySectionSegment(Segment, Offset); + Offset.QuadPart = (ULONG_PTR)Address - (ULONG_PTR)MemoryArea->StartingAddress + + MemoryArea->Data.SectionData.ViewOffset.QuadPart; + Entry = MmGetPageEntrySectionSegment(Segment, &Offset); Page = MmGetPfnForProcess(Process, Address); Protect = PAGE_READONLY; - if (Segment->Characteristics & IMAGE_SCN_CNT_UNINITIALIZED_DATA || + if (Segment->Image.Characteristics & IMAGE_SCN_CNT_UNINITIALIZED_DATA || IS_SWAP_FROM_SSE(Entry) || PFN_FROM_SSE(Entry) != Page) { @@ -2615,6 +2552,8 @@ } } } + + MmUnlockSectionSegment(Segment); } NTSTATUS @@ -2637,6 +2576,8 @@ Region = MmFindRegion(MemoryArea->StartingAddress, &MemoryArea->Data.SectionData.RegionListHead, BaseAddress, NULL); + ASSERT(Region != NULL); + if ((MemoryArea->Flags & SEC_NO_CHANGE) && Region->Protect != Protect) { @@ -2675,7 +2616,7 @@ if (Section->AllocationAttributes & SEC_IMAGE) { Segment = MemoryArea->Data.SectionData.Segment; - Info->AllocationBase = (PUCHAR)MemoryArea->StartingAddress - Segment->VirtualAddress; + Info->AllocationBase = (PUCHAR)MemoryArea->StartingAddress - Segment->Image.VirtualAddress; Info->Type = MEM_IMAGE; } else @@ -2698,19 +2639,22 @@ MmpFreePageFileSegment(PMM_SECTION_SEGMENT Segment) { ULONG Length; - ULONG Offset; + LARGE_INTEGER Offset; ULONG Entry; SWAPENTRY SavedSwapEntry; PFN_NUMBER Page; Page = 0; - Length = PAGE_ROUND_UP(Segment->Length); - for (Offset = 0; Offset < Length; Offset += PAGE_SIZE) + MmLockSectionSegment(Segment); + + Length = PAGE_ROUND_UP(Segment->Length.QuadPart); + for (Offset.QuadPart = 0; Offset.QuadPart < Length; Offset.QuadPart += PAGE_SIZE) { - Entry = MmGetPageEntrySectionSegment(Segment, Offset); + Entry = MmGetPageEntrySectionSegment(Segment, &Offset); if (Entry) { + MmSetPageEntrySectionSegment(Segment, &Offset, 0); if (IS_SWAP_FROM_SSE(Entry)) { MmFreeSwapPage(SWAPENTRY_FROM_SSE(Entry)); @@ -2726,9 +2670,10 @@ } MmReleasePageMemoryConsumer(MC_USER, Page); } - MmSetPageEntrySectionSegment(Segment, Offset, 0); } } + + MmUnlockSectionSegment(Segment); } VOID NTAPI @@ -2736,7 +2681,7 @@ { PROS_SECTION_OBJECT Section = (PROS_SECTION_OBJECT)ObjectBody; - DPRINT("MmpDeleteSection(ObjectBody %x)\n", ObjectBody); + DPRINT("MmpDeleteSection(ObjectBody %p)\n", ObjectBody); if (Section->AllocationAttributes & SEC_IMAGE) { ULONG i; @@ -2758,21 +2703,40 @@ for (i = 0; i < NrSegments; i++) { - if (SectionSegments[i].Characteristics & IMAGE_SCN_MEM_SHARED) + if (SectionSegments[i].Image.Characteristics & IMAGE_SCN_MEM_SHARED) { MmLockSectionSegment(&SectionSegments[i]); } RefCount = InterlockedDecrementUL(&SectionSegments[i].ReferenceCount); - if (SectionSegments[i].Characteristics & IMAGE_SCN_MEM_SHARED) + if (SectionSegments[i].Image.Characteristics & IMAGE_SCN_MEM_SHARED) { + MmUnlockSectionSegment(&SectionSegments[i]); if (RefCount == 0) { MmpFreePageFileSegment(&SectionSegments[i]); } - MmUnlockSectionSegment(&SectionSegments[i]); } } } +#ifdef NEWCC + else if (Section->Segment && Section->Segment->Flags & MM_DATAFILE_SEGMENT) + { + ULONG RefCount = 0; + PMM_SECTION_SEGMENT Segment = Section->Segment; + + if (Segment && + (RefCount = InterlockedDecrementUL(&Segment->ReferenceCount)) == 0) + { + DPRINT("Freeing section segment\n"); + Section->Segment = NULL; + MmFinalizeSegment(Segment); + } + else + { + DPRINT("RefCount %d\n", RefCount); + } + } +#endif else { /* @@ -2785,7 +2749,7 @@ if (Section->Segment->Flags & MM_PAGEFILE_SEGMENT) { MmpFreePageFileSegment(Section->Segment); - MmFreePageTablesSectionSegment(Section->Segment); + MmFreePageTablesSectionSegment(Section->Segment, NULL); ExFreePool(Section->Segment); Section->Segment = NULL; } @@ -2948,23 +2912,23 @@ ObDereferenceObject(Section); return(STATUS_NO_MEMORY); } + RtlZeroMemory(Segment, sizeof(MM_SECTION_SEGMENT)); Section->Segment = Segment; Segment->ReferenceCount = 1; ExInitializeFastMutex(&Segment->Lock); - Segment->FileOffset = 0; + Segment->Image.FileOffset = 0; Segment->Protection = SectionPageProtection; - Segment->RawLength = MaximumSize.u.LowPart; - Segment->Length = PAGE_ROUND_UP(MaximumSize.u.LowPart); + Segment->RawLength.QuadPart = MaximumSize.u.LowPart; + Segment->Length.QuadPart = PAGE_ROUND_UP(MaximumSize.u.LowPart); Segment->Flags = MM_PAGEFILE_SEGMENT; Segment->WriteCopy = FALSE; - RtlZeroMemory(&Segment->PageDirectory, sizeof(SECTION_PAGE_DIRECTORY)); - Segment->VirtualAddress = 0; - Segment->Characteristics = 0; + Segment->Image.VirtualAddress = 0; + Segment->Image.Characteristics = 0; *SectionObject = Section; + MiInitializeSectionPageTable(Segment); return(STATUS_SUCCESS); } - NTSTATUS NTAPI MmCreateDataFileSection(PROS_SECTION_OBJECT *SectionObject, @@ -3001,7 +2965,7 @@ sizeof(ROS_SECTION_OBJECT), 0, 0, - (PVOID*)(PVOID)&Section); + (PVOID*)&Section); if (!NT_SUCCESS(Status)) { return(Status); @@ -3151,22 +3115,23 @@ ExAcquireFastMutex(&Segment->Lock); FileObject->SectionObjectPointer->DataSectionObject = (PVOID)Segment; - Segment->FileOffset = 0; + Segment->Image.FileOffset = 0; Segment->Protection = SectionPageProtection; Segment->Flags = MM_DATAFILE_SEGMENT; - Segment->Characteristics = 0; + Segment->Image.Characteristics = 0; Segment->WriteCopy = (SectionPageProtection & (PAGE_WRITECOPY | PAGE_EXECUTE_WRITECOPY)); if (AllocationAttributes & SEC_RESERVE) { - Segment->Length = Segment->RawLength = 0; + Segment->Length.QuadPart = Segment->RawLength.QuadPart = 0; } else { - Segment->RawLength = MaximumSize.u.LowPart; - Segment->Length = PAGE_ROUND_UP(Segment->RawLength); + Segment->RawLength.QuadPart = MaximumSize.QuadPart; + Segment->Length.QuadPart = PAGE_ROUND_UP(Segment->RawLength.QuadPart); } - Segment->VirtualAddress = 0; - RtlZeroMemory(&Segment->PageDirectory, sizeof(SECTION_PAGE_DIRECTORY)); + Segment->Image.VirtualAddress = 0; + Segment->Locked = TRUE; + MiInitializeSectionPageTable(Segment); } else { @@ -3181,11 +3146,11 @@ (void)InterlockedIncrementUL(&Segment->ReferenceCount); MmLockSectionSegment(Segment); - if (MaximumSize.u.LowPart > Segment->RawLength && + if (MaximumSize.QuadPart > Segment->RawLength.QuadPart && !(AllocationAttributes & SEC_RESERVE)) { - Segment->RawLength = MaximumSize.u.LowPart; - Segment->Length = PAGE_ROUND_UP(Segment->RawLength); + Segment->RawLength.QuadPart = MaximumSize.QuadPart; + Segment->Length.QuadPart = PAGE_ROUND_UP(Segment->RawLength.QuadPart); } } MmUnlockSectionSegment(Segment); @@ -3276,6 +3241,8 @@ ULONG BufferSize; ULONG UsedSize; PVOID Buffer; + PFILE_OBJECT FileObject = File; + IO_STATUS_BLOCK Iosb; ASSERT_IRQL_LESS(DISPATCH_LEVEL); @@ -3315,39 +3282,10 @@ UsedSize = 0; -#if 0 - Status = MmspPageRead(File, - Buffer, - BufferSize, - &FileOffset, - &UsedSize); -#else -/* - * FIXME: if we don't use ZwReadFile, caching is not enabled for the file and - * nothing will work. But using ZwReadFile is wrong, and using its side effects - * to initialize internal state is even worse. Our cache manager is in need of - * professional help - */ - { - IO_STATUS_BLOCK Iosb; + Status = MiSimpleRead(FileObject, &FileOffset, Buffer, BufferSize, TRUE, &Iosb); - Status = ZwReadFile(File, - NULL, - NULL, - NULL, - &Iosb, - Buffer, - BufferSize, - &FileOffset, - NULL); + UsedSize = Iosb.Information; - if(NT_SUCCESS(Status)) - { - UsedSize = (ULONG)Iosb.Information; - } - } -#endif - if(NT_SUCCESS(Status) && UsedSize < OffsetAdjustment) { Status = STATUS_IN_PAGE_ERROR; @@ -3382,8 +3320,8 @@ for( i = 1; i < ImageSectionObject->NrSegments; ++ i ) { - ASSERT(ImageSectionObject->Segments[i].VirtualAddress >= - ImageSectionObject->Segments[i - 1].VirtualAddress); + ASSERT(ImageSectionObject->Segments[i].Image.VirtualAddress >= + ImageSectionObject->Segments[i - 1].Image.VirtualAddress); } } @@ -3398,13 +3336,13 @@ for( i = 0; i < ImageSectionObject->NrSegments; ++ i ) { - ASSERT(ImageSectionObject->Segments[i].Length > 0); + ASSERT(ImageSectionObject->Segments[i].Length.QuadPart > 0); if(i > 0) { - ASSERT(ImageSectionObject->Segments[i].VirtualAddress >= - (ImageSectionObject->Segments[i - 1].VirtualAddress + - ImageSectionObject->Segments[i - 1].Length)); + ASSERT(ImageSectionObject->Segments[i].Image.VirtualAddress >= + (ImageSectionObject->Segments[i - 1].Image.VirtualAddress + + ImageSectionObject->Segments[i - 1].Length.QuadPart)); } } } @@ -3418,8 +3356,8 @@ for( i = 0; i < ImageSectionObject->NrSegments; ++ i ) { - ASSERT((ImageSectionObject->Segments[i].VirtualAddress % PAGE_SIZE) == 0); - ASSERT((ImageSectionObject->Segments[i].Length % PAGE_SIZE) == 0); + ASSERT((ImageSectionObject->Segments[i].Image.VirtualAddress % PAGE_SIZE) == 0); + ASSERT((ImageSectionObject->Segments[i].Length.QuadPart % PAGE_SIZE) == 0); } } #endif @@ -3434,7 +3372,7 @@ const MM_SECTION_SEGMENT *Segment2 = (const MM_SECTION_SEGMENT *)y; return - (Segment1->VirtualAddress - Segment2->VirtualAddress) >> + (Segment1->Image.VirtualAddress - Segment2->Image.VirtualAddress) >> ((sizeof(ULONG_PTR) - sizeof(int)) * 8); } @@ -3487,7 +3425,7 @@ for ( i = 0; i < ImageSectionObject->NrSegments; ++ i ) { - if(ImageSectionObject->Segments[i].Length == 0) + if(ImageSectionObject->Segments[i].Length.QuadPart == 0) { return FALSE; } @@ -3501,9 +3439,9 @@ * (NtMapViewOfSection could then refuse to map them, and they could * e.g. only be allowed as parameters to NtCreateProcess, like on UNIX) */ - if ((ImageSectionObject->Segments[i - 1].VirtualAddress + - ImageSectionObject->Segments[i - 1].Length) != - ImageSectionObject->Segments[i].VirtualAddress) + if ((ImageSectionObject->Segments[i - 1].Image.VirtualAddress + + ImageSectionObject->Segments[i - 1].Length.QuadPart) != + ImageSectionObject->Segments[i].Image.VirtualAddress) { return FALSE; } @@ -3549,19 +3487,19 @@ ULONG_PTR VirtualAddress; ULONG_PTR VirtualOffset; - VirtualAddress = EffectiveSegment->VirtualAddress; + VirtualAddress = EffectiveSegment->Image.VirtualAddress; /* Round down the virtual address to the nearest page */ - EffectiveSegment->VirtualAddress = PAGE_ROUND_DOWN(VirtualAddress); + EffectiveSegment->Image.VirtualAddress = PAGE_ROUND_DOWN(VirtualAddress); /* Round up the virtual size to the nearest page */ - EffectiveSegment->Length = (ULONG)(PAGE_ROUND_UP(VirtualAddress + EffectiveSegment->Length) - - EffectiveSegment->VirtualAddress); + EffectiveSegment->Length.QuadPart = PAGE_ROUND_UP(VirtualAddress + EffectiveSegment->Length.QuadPart) - + EffectiveSegment->Image.VirtualAddress; /* Adjust the raw address and size */ - VirtualOffset = VirtualAddress - EffectiveSegment->VirtualAddress; + VirtualOffset = VirtualAddress - EffectiveSegment->Image.VirtualAddress; - if (EffectiveSegment->FileOffset < (LONG_PTR)VirtualOffset) + if (EffectiveSegment->Image.FileOffset < VirtualOffset) { return FALSE; } @@ -3571,22 +3509,22 @@ * offset point in curious and odd places, but that's what we were * asked for */ - EffectiveSegment->FileOffset -= (ULONG)VirtualOffset; - EffectiveSegment->RawLength += (ULONG)VirtualOffset; + EffectiveSegment->Image.FileOffset -= VirtualOffset; + EffectiveSegment->RawLength.QuadPart += VirtualOffset; } else { PMM_SECTION_SEGMENT Segment = &ImageSectionObject->Segments[i]; ULONG_PTR EndOfEffectiveSegment; - EndOfEffectiveSegment = EffectiveSegment->VirtualAddress + EffectiveSegment->Length; + EndOfEffectiveSegment = EffectiveSegment->Image.VirtualAddress + EffectiveSegment->Length.QuadPart; ASSERT((EndOfEffectiveSegment % PAGE_SIZE) == 0); /* * The current segment begins exactly where the current effective * segment ended, therefore beginning a new effective segment */ - if (EndOfEffectiveSegment == Segment->VirtualAddress) + if (EndOfEffectiveSegment == Segment->Image.VirtualAddress) { LastSegment ++; ASSERT(LastSegment <= i); @@ -3607,14 +3545,14 @@ * Page-align the virtual size. We know for sure the virtual address * already is */ - ASSERT((EffectiveSegment->VirtualAddress % PAGE_SIZE) == 0); - EffectiveSegment->Length = PAGE_ROUND_UP(EffectiveSegment->Length); + ASSERT((EffectiveSegment->Image.VirtualAddress % PAGE_SIZE) == 0); + EffectiveSegment->Length.QuadPart = PAGE_ROUND_UP(EffectiveSegment->Length.QuadPart); } /* * The current segment is still part of the current effective segment: * extend the effective segment to reflect this */ - else if (EndOfEffectiveSegment > Segment->VirtualAddress) + else if (EndOfEffectiveSegment > Segment->Image.VirtualAddress) { static const ULONG FlagsToProtection[16] = { @@ -3643,21 +3581,21 @@ */ /* Unaligned segments must be contiguous within the file */ - if (Segment->FileOffset != (EffectiveSegment->FileOffset + - EffectiveSegment->RawLength)) + if (Segment->Image.FileOffset != (EffectiveSegment->Image.FileOffset + + EffectiveSegment->RawLength.QuadPart)) { return FALSE; } - EffectiveSegment->RawLength += Segment->RawLength; + EffectiveSegment->RawLength.QuadPart += Segment->RawLength.QuadPart; /* * Extend the virtual size */ - ASSERT(PAGE_ROUND_UP(Segment->VirtualAddress + Segment->Length) >= EndOfEffectiveSegment); + ASSERT(PAGE_ROUND_UP(Segment->Image.VirtualAddress + Segment->Length.QuadPart) >= EndOfEffectiveSegment); - EffectiveSegment->Length = (ULONG)(PAGE_ROUND_UP(Segment->VirtualAddress + Segment->Length) - - EffectiveSegment->VirtualAddress); + EffectiveSegment->Length.QuadPart = PAGE_ROUND_UP(Segment->Image.VirtualAddress + Segment->Length.QuadPart) - + EffectiveSegment->Image.VirtualAddress; /* * Merge the protection @@ -3845,9 +3783,7 @@ { ExInitializeFastMutex(&ImageSectionObject->Segments[i].Lock); ImageSectionObject->Segments[i].ReferenceCount = 1; - - RtlZeroMemory(&ImageSectionObject->Segments[i].PageDirectory, - sizeof(ImageSectionObject->Segments[i].PageDirectory)); + MiInitializeSectionPageTable(&ImageSectionObject->Segments[i]); } ASSERT(NT_SUCCESS(Status)); @@ -3861,32 +3797,17 @@ PLARGE_INTEGER UMaximumSize, ULONG SectionPageProtection, ULONG AllocationAttributes, - HANDLE FileHandle) + PFILE_OBJECT FileObject) { PROS_SECTION_OBJECT Section; NTSTATUS Status; - PFILE_OBJECT FileObject; PMM_SECTION_SEGMENT SectionSegments; PMM_IMAGE_SECTION_OBJECT ImageSectionObject; ULONG i; - ULONG FileAccess = 0; - /* - * Reference the file handle - */ - FileAccess = MiArm3GetCorrectFileAccessMask(SectionPageProtection); - Status = ObReferenceObjectByHandle(FileHandle, - FileAccess, - IoFileObjectType, - ExGetPreviousMode(), - (PVOID*)(PVOID)&FileObject, - NULL); + if (FileObject == NULL) + return STATUS_INVALID_FILE_FOR_SECTION; - if (!NT_SUCCESS(Status)) - { - return Status; - } - /* * Create the section */ @@ -3936,7 +3857,7 @@ RtlZeroMemory(ImageSectionObject, sizeof(MM_IMAGE_SECTION_OBJECT)); - StatusExeFmt = ExeFmtpCreateImageSection(FileHandle, ImageSectionObject); + StatusExeFmt = ExeFmtpCreateImageSection(FileObject, ImageSectionObject); if (!NT_SUCCESS(StatusExeFmt)) { @@ -4056,6 +3977,14 @@ BoundaryAddressMultiple.QuadPart = 0; +#ifdef NEWCC + if (Segment->Flags & MM_DATAFILE_SEGMENT) { + LARGE_INTEGER FileOffset; + FileOffset.QuadPart = ViewOffset; + ObReferenceObject(Section); + return _MiMapViewOfSegment(AddressSpace, Segment, BaseAddress, ViewSize, Protect, &FileOffset, AllocationType, __FILE__, __LINE__); + } +#endif Status = MmCreateMemoryArea(AddressSpace, MEMORY_AREA_SECTION_VIEW, BaseAddress, @@ -4076,7 +4005,7 @@ MArea->Data.SectionData.Segment = Segment; MArea->Data.SectionData.Section = Section; - MArea->Data.SectionData.ViewOffset = ViewOffset; + MArea->Data.SectionData.ViewOffset.QuadPart = ViewOffset; MmInitializeRegion(&MArea->Data.SectionData.RegionListHead, ViewSize, 0, Protect); @@ -4084,7 +4013,6 @@ } - static VOID MmFreeSectionPage(PVOID Context, MEMORY_AREA* MemoryArea, PVOID Address, PFN_NUMBER Page, SWAPENTRY SwapEntry, BOOLEAN Dirty) @@ -4092,7 +4020,7 @@ ULONG Entry; PFILE_OBJECT FileObject; PBCB Bcb; - ULONG Offset; + LARGE_INTEGER Offset; SWAPENTRY SavedSwapEntry; PMM_PAGEOP PageOp; NTSTATUS Status; @@ -4106,13 +4034,13 @@ Address = (PVOID)PAGE_ROUND_DOWN(Address); - Offset = (ULONG)(((ULONG_PTR)Address - (ULONG_PTR)MemoryArea->StartingAddress) + - MemoryArea->Data.SectionData.ViewOffset); + Offset.QuadPart = ((ULONG_PTR)Address - (ULONG_PTR)MemoryArea->StartingAddress) + + MemoryArea->Data.SectionData.ViewOffset.QuadPart; Section = MemoryArea->Data.SectionData.Section; Segment = MemoryArea->Data.SectionData.Segment; - PageOp = MmCheckForPageOp(MemoryArea, NULL, NULL, Segment, Offset); + PageOp = MmCheckForPageOp(MemoryArea, NULL, NULL, Segment, Offset.LowPart); while (PageOp) { @@ -4129,10 +4057,10 @@ MmLockAddressSpace(AddressSpace); MmLockSectionSegment(Segment); MmspCompleteAndReleasePageOp(PageOp); - PageOp = MmCheckForPageOp(MemoryArea, NULL, NULL, Segment, Offset); + PageOp = MmCheckForPageOp(MemoryArea, NULL, NULL, Segment, Offset.LowPart); } - Entry = MmGetPageEntrySectionSegment(Segment, Offset); + Entry = MmGetPageEntrySectionSegment(Segment, &Offset); /* * For a dirty, datafile, non-private page mark it as dirty in the @@ -4145,7 +4073,7 @@ FileObject = MemoryArea->Data.SectionData.Section->FileObject; Bcb = FileObject->SectionObjectPointer->SharedCacheMap; #ifndef NEWCC - CcRosMarkDirtyCacheSegment(Bcb, Offset + Segment->FileOffset); + CcRosMarkDirtyCacheSegment(Bcb, Offset.QuadPart + Segment->Image.FileOffset); #endif ASSERT(SwapEntry == 0); } @@ -4191,7 +4119,7 @@ else { MmDeleteRmap(Page, Process, Address); - MmUnsharePageEntrySectionSegment(Section, Segment, Offset, Dirty, FALSE); + MmUnsharePageEntrySectionSegment(Section, Segment, &Offset, Dirty, FALSE); } } } @@ -4219,6 +4147,11 @@ Section = MemoryArea->Data.SectionData.Section; Segment = MemoryArea->Data.SectionData.Segment; +#ifdef NEWCC + if (Segment->Flags & MM_DATAFILE_SEGMENT) + return MmUnmapViewOfCacheSegment(AddressSpace, BaseAddress); +#endif + MmLockSectionSegment(Segment); RegionListHead = &MemoryArea->Data.SectionData.RegionListHead; @@ -4292,7 +4225,7 @@ Offset -= PAGE_SIZE; PageOp = MmCheckForPageOp(MemoryArea, NULL, NULL, MemoryArea->Data.SectionData.Segment, - (ULONG)Offset + MemoryArea->Data.SectionData.ViewOffset); + Offset + MemoryArea->Data.SectionData.ViewOffset.QuadPart); if (PageOp) { MmUnlockAddressSpace(AddressSpace); @@ -4335,11 +4268,14 @@ * and calculate the image base address */ for (i = 0; i < NrSegments; i++) { - if (Segment == &SectionSegments[i]) - { - ImageBaseAddress = (char*)BaseAddress - (ULONG_PTR)SectionSegments[i].VirtualAddress; - break; - } + if (!(SectionSegments[i].Image.Characteristics & IMAGE_SCN_TYPE_NOLOAD)) + { + if (Segment == &SectionSegments[i]) + { + ImageBaseAddress = (char*)BaseAddress - (ULONG_PTR)SectionSegments[i].Image.VirtualAddress; + break; + } + } } if (i >= NrSegments) { @@ -4348,10 +4284,13 @@ for (i = 0; i < NrSegments; i++) { - PVOID SBaseAddress = (PVOID) - ((char*)ImageBaseAddress + (ULONG_PTR)SectionSegments[i].VirtualAddress); + if (!(SectionSegments[i].Image.Characteristics & IMAGE_SCN_TYPE_NOLOAD)) + { + PVOID SBaseAddress = (PVOID) + ((char*)ImageBaseAddress + (ULONG_PTR)SectionSegments[i].Image.VirtualAddress); - Status = MmUnmapViewOfSegment(AddressSpace, SBaseAddress); + Status = MmUnmapViewOfSegment(AddressSpace, SBaseAddress); + } } } else @@ -4445,8 +4384,8 @@ } else { - Sbi->BaseAddress = (PVOID)Section->Segment->VirtualAddress; - Sbi->Size.QuadPart = Section->Segment->Length; + Sbi->BaseAddress = (PVOID)Section->Segment->Image.VirtualAddress; + Sbi->Size.QuadPart = Section->Segment->Length.QuadPart; } if (ResultLength != NULL) @@ -4628,10 +4567,13 @@ ImageSize = 0; for (i = 0; i < NrSegments; i++) { - ULONG_PTR MaxExtent; - MaxExtent = (ULONG_PTR)SectionSegments[i].VirtualAddress + - SectionSegments[i].Length; - ImageSize = max(ImageSize, MaxExtent); + if (!(SectionSegments[i].Image.Characteristics & IMAGE_SCN_TYPE_NOLOAD)) + { + ULONG_PTR MaxExtent; + MaxExtent = (ULONG_PTR)SectionSegments[i].Image.VirtualAddress + + SectionSegments[i].Length.QuadPart; + ImageSize = max(ImageSize, MaxExtent); + } } ImageSectionObject->ImageSize = (ULONG)ImageSize; @@ -4665,23 +4607,26 @@ for (i = 0; i < NrSegments; i++) { - PVOID SBaseAddress = (PVOID) - ((char*)ImageBase + (ULONG_PTR)SectionSegments[i].VirtualAddress); - MmLockSectionSegment(&SectionSegments[i]); - Status = MmMapViewOfSegment(AddressSpace, - Section, - &SectionSegments[i], - &SBaseAddress, - SectionSegments[i].Length, - SectionSegments[i].Protection, - 0, - 0); - MmUnlockSectionSegment(&SectionSegments[i]); - if (!NT_SUCCESS(Status)) - { - MmUnlockAddressSpace(AddressSpace); - return(Status); - } + if (!(SectionSegments[i].Image.Characteristics & IMAGE_SCN_TYPE_NOLOAD)) + { + PVOID SBaseAddress = (PVOID) + ((char*)ImageBase + (ULONG_PTR)SectionSegments[i].Image.VirtualAddress); + MmLockSectionSegment(&SectionSegments[i]); + Status = MmMapViewOfSegment(AddressSpace, + Section, + &SectionSegments[i], + &SBaseAddress, + SectionSegments[i].Length.LowPart, + SectionSegments[i].Protection, + 0, + 0); + MmUnlockSectionSegment(&SectionSegments[i]); + if (!NT_SUCCESS(Status)) + { + MmUnlockAddressSpace(AddressSpace); + return(Status); + } + } } *BaseAddress = (PVOID)ImageBase; @@ -4794,6 +4739,29 @@ if (Segment->ReferenceCount != 0) { +#ifdef NEWCC + CC_FILE_SIZES FileSizes; + CcpLock(); + if (SectionObjectPointer->SharedCacheMap && (Segment->ReferenceCount > CcpCountCacheSections((PNOCC_CACHE_MAP)SectionObjectPointer->SharedCacheMap))) + { + CcpUnlock(); + /* Check size of file */ + if (SectionObjectPointer->SharedCacheMap) + { + if (!CcGetFileSizes(Segment->FileObject, &FileSizes)) + { + return FALSE; + } + + if (NewFileSize->QuadPart <= FileSizes.FileSize.QuadPart) + { + return FALSE; + } + } + } + else + CcpUnlock(); +#else /* Check size of file */ if (SectionObjectPointer->SharedCacheMap) { @@ -4803,6 +4771,7 @@ return FALSE; } } +#endif } else { @@ -4828,6 +4797,11 @@ MmFlushImageSection (IN PSECTION_OBJECT_POINTERS SectionObjectPointer, IN MMFLUSH_TYPE FlushType) { + BOOLEAN Result = TRUE; +#ifdef NEWCC + PMM_SECTION_SEGMENT Segment; +#endif + switch(FlushType) { case MmFlushForDelete: @@ -4841,7 +4815,25 @@ #endif return TRUE; case MmFlushForWrite: - break; + { + DPRINT("MmFlushImageSection(%d)\n", FlushType); +#ifdef NEWCC + Segment = (PMM_SECTION_SEGMENT)SectionObjectPointer->DataSectionObject; +#endif + + if (SectionObjectPointer->ImageSectionObject) { + DPRINT1("SectionObject has ImageSection\n"); + return FALSE; + } + +#ifdef NEWCC + CcpLock(); + Result = !SectionObjectPointer->SharedCacheMap || (Segment->ReferenceCount == CcpCountCacheSections((PNOCC_CACHE_MAP)SectionObjectPointer->SharedCacheMap)); + CcpUnlock(); + DPRINT("Result %d\n", Result); +#endif + return Result; + } } return FALSE; } @@ -4989,10 +4981,11 @@ IN ULONG SectionPageProtection, IN ULONG AllocationAttributes, IN HANDLE FileHandle OPTIONAL, - IN PFILE_OBJECT File OPTIONAL) + IN PFILE_OBJECT FileObject OPTIONAL) { - ULONG Protection; - PROS_SECTION_OBJECT *SectionObject = (PROS_SECTION_OBJECT *)Section; + NTSTATUS Status; + ULONG Protection, FileAccess; + PROS_SECTION_OBJECT *SectionObject = (PROS_SECTION_OBJECT *)Section; /* Check if an ARM3 section is being created instead */ if (AllocationAttributes & 1) @@ -5005,54 +4998,129 @@ SectionPageProtection, AllocationAttributes &~ 1, FileHandle, - File); + FileObject); } - /* - * Check the protection - */ - Protection = SectionPageProtection & ~(PAGE_GUARD|PAGE_NOCACHE); - if (Protection != PAGE_READONLY && - Protection != PAGE_READWRITE && - Protection != PAGE_WRITECOPY && - Protection != PAGE_EXECUTE && - Protection != PAGE_EXECUTE_READ && - Protection != PAGE_EXECUTE_READWRITE && - Protection != PAGE_EXECUTE_WRITECOPY) - { - return STATUS_INVALID_PAGE_PROTECTION; - } + /* + * Check the protection + */ + Protection = SectionPageProtection & ~(PAGE_GUARD | PAGE_NOCACHE); + if (Protection != PAGE_READONLY && + Protection != PAGE_READWRITE && + Protection != PAGE_WRITECOPY && + Protection != PAGE_EXECUTE && + Protection != PAGE_EXECUTE_READ && + Protection != PAGE_EXECUTE_READWRITE && + Protection != PAGE_EXECUTE_WRITECOPY) + { + return STATUS_INVALID_PAGE_PROTECTION; + } - if (AllocationAttributes & SEC_IMAGE) - { - return(MmCreateImageSection(SectionObject, - DesiredAccess, - ObjectAttributes, - MaximumSize, - SectionPageProtection, - AllocationAttributes, - FileHandle)); - } + if ((DesiredAccess & SECTION_MAP_WRITE) && + (Protection == PAGE_READWRITE || + Protection == PAGE_EXECUTE_READWRITE) && + !(AllocationAttributes & SEC_IMAGE)) + { + DPRINT("Creating a section with WRITE access\n"); + FileAccess = FILE_READ_DATA | FILE_WRITE_DATA | SYNCHRONIZE; + } + else + { + DPRINT("Creating a section with READ access\n"); + FileAccess = FILE_READ_DATA | SYNCHRONIZE; + } - if (FileHandle != NULL) - { - return(MmCreateDataFileSection(SectionObject, - DesiredAccess, - ObjectAttributes, - MaximumSize, - SectionPageProtection, - AllocationAttributes, - FileHandle)); - } + /* FIXME: somehow combine this with the above checks */ + if (AllocationAttributes & SEC_IMAGE) + FileAccess = MiArm3GetCorrectFileAccessMask(SectionPageProtection); - return(MmCreatePageFileSection(SectionObject, - DesiredAccess, - ObjectAttributes, - MaximumSize, - SectionPageProtection, - AllocationAttributes)); -} + if (!FileObject && FileHandle) + { + Status = ObReferenceObjectByHandle(FileHandle, + FileAccess, + IoFileObjectType, + ExGetPreviousMode(), + (PVOID *)&FileObject, + NULL); + if (!NT_SUCCESS(Status)) + { + DPRINT("Failed: 0x%08lx\n", Status); + return Status; + } + } + else if (FileObject) + ObReferenceObject(FileObject); +#ifndef NEWCC // A hack for initializing caching. + // This is needed only in the old case. + if (FileHandle) + { + IO_STATUS_BLOCK Iosb; + NTSTATUS Status; + CHAR Buffer; + LARGE_INTEGER ByteOffset; + ByteOffset.QuadPart = 0; + Status = ZwReadFile(FileHandle, + NULL, + NULL, + NULL, + &Iosb, + &Buffer, + sizeof(Buffer), + &ByteOffset, + NULL); + if (!NT_SUCCESS(Status) && Status != STATUS_END_OF_FILE) + return Status; + // Caching is initialized... + } +#endif + if (AllocationAttributes & SEC_IMAGE) + { + Status = MmCreateImageSection(SectionObject, + DesiredAccess, + ObjectAttributes, + MaximumSize, + SectionPageProtection, + AllocationAttributes, + FileObject); + } +#ifndef NEWCC + else if (FileHandle != NULL) + { + Status = MmCreateDataFileSection(SectionObject, + DesiredAccess, + ObjectAttributes, + MaximumSize, + SectionPageProtection, + AllocationAttributes, + FileHandle); + if (FileObject) + ObDereferenceObject(FileObject); + } +#else + else if (FileHandle != NULL || FileObject != NULL) + { + Status = MmCreateCacheSection(SectionObject, + DesiredAccess, + ObjectAttributes, + MaximumSize, + SectionPageProtection, + AllocationAttributes, + FileObject); + } +#endif + else + { + Status = MmCreatePageFileSection(SectionObject, + DesiredAccess, + ObjectAttributes, + MaximumSize, + SectionPageProtection, + AllocationAttributes); + } + return Status; +} + /* EOF */ Index: ntoskrnl/ntoskrnl-generic.rbuild =================================================================== --- ntoskrnl/ntoskrnl-generic.rbuild (revision 55684) +++ ntoskrnl/ntoskrnl-generic.rbuild (working copy) @@ -140,8 +140,8 @@ view.c - - + + cachesub.c copysup.c fssup.c @@ -149,16 +149,16 @@ logsup.c mdlsup.c pinsup.c - - data.c - fault.c - io.c - reqtools.c - sptab.c - swapout.c - + + + data.c + fault.c + io.c + reqtools.c + sptab.c + swapout.c - + Index: ntoskrnl/po/poshtdwn.c =================================================================== --- ntoskrnl/po/poshtdwn.c (revision 55684) +++ ntoskrnl/po/poshtdwn.c (working copy) @@ -138,7 +138,7 @@ /* Get the next process */ Process = PsGetNextProcess(Process); } - + /* First, the HAL handles any "end of boot" special functionality */ DPRINT1("HAL shutting down\n"); HalEndOfBoot(); @@ -152,16 +152,17 @@ CmShutdownSystem(); /* Note that modified pages should be written here (MiShutdownSystem) */ +#ifdef NEWCC + /* Flush all user files before we start shutting down IO */ + /* This is where modified pages are written back by the IO manager */ + CcShutdownSystem(); +#endif /* In this step, the I/O manager does last-chance shutdown notification */ DPRINT1("I/O manager shutting down in phase 1\n"); IoShutdownSystem(1); CcWaitForCurrentLazyWriterActivity(); -#ifdef NEWCC - CcShutdownSystem(); -#endif - /* Note that here, we should broadcast the power IRP to devices */ /* In this step, the HAL disables any wake timers */