sys: FspFileNodeFlushAndPurgeCache

src/sys/driver.c

@@ -7,10 +7,12 @@
 #include <sys/driver.h>
 
 DRIVER_INITIALIZE DriverEntry;
+static VOID FspDriverMultiVersionInitialize(VOID);
 DRIVER_UNLOAD FspUnload;
 
 #ifdef ALLOC_PRAGMA
 #pragma alloc_text(INIT, DriverEntry)
+#pragma alloc_text(INIT, FspDriverMultiVersionInitialize)
 #pragma alloc_text(PAGE, FspUnload)
 #endif
 
@@ -19,6 +21,8 @@ NTSTATUS DriverEntry(
 {
     FSP_ENTER();
 
+    FspDriverMultiVersionInitialize();
+
     FspDriverObject = DriverObject;
 
     /* create the file system control device objects */
@@ -143,6 +147,18 @@ NTSTATUS DriverEntry(
         &DriverObject->DriverName, RegistryPath);
 }
 
+static VOID FspDriverMultiVersionInitialize(VOID)
+{
+    UNICODE_STRING Name;
+
+    if (RtlIsNtDdiVersionAvailable(NTDDI_WIN7))
+    {
+        RtlInitUnicodeString(&Name, L"CcCoherencyFlushAndPurgeCache");
+        FspMvCcCoherencyFlushAndPurgeCache =
+            (FSP_MV_CcCoherencyFlushAndPurgeCache *)(UINT_PTR)MmGetSystemRoutineAddress(&Name);
+    }
+}
+
 VOID FspUnload(
     PDRIVER_OBJECT DriverObject)
 {
@@ -164,3 +180,5 @@ PDEVICE_OBJECT FspFsctlDiskDeviceObject;
 PDEVICE_OBJECT FspFsctlNetDeviceObject;
 FAST_IO_DISPATCH FspFastIoDispatch;
 CACHE_MANAGER_CALLBACKS FspCacheManagerCallbacks;
+
+FSP_MV_CcCoherencyFlushAndPurgeCache *FspMvCcCoherencyFlushAndPurgeCache;

src/sys/driver.h

@@ -835,6 +835,7 @@ VOID FspFileNodeAcquireSharedF(FSP_FILE_NODE *FileNode, ULONG Flags);
 BOOLEAN FspFileNodeTryAcquireSharedF(FSP_FILE_NODE *FileNode, ULONG Flags, BOOLEAN Wait);
 VOID FspFileNodeAcquireExclusiveF(FSP_FILE_NODE *FileNode, ULONG Flags);
 BOOLEAN FspFileNodeTryAcquireExclusiveF(FSP_FILE_NODE *FileNode, ULONG Flags, BOOLEAN Wait);
+VOID FspFileNodeConvertExclusiveToSharedF(FSP_FILE_NODE *FileNode, ULONG Flags);
 VOID FspFileNodeSetOwnerF(FSP_FILE_NODE *FileNode, ULONG Flags, PVOID Owner);
 VOID FspFileNodeReleaseF(FSP_FILE_NODE *FileNode, ULONG Flags);
 VOID FspFileNodeReleaseOwnerF(FSP_FILE_NODE *FileNode, ULONG Flags, PVOID Owner);
@@ -844,6 +845,8 @@ VOID FspFileNodeCleanup(FSP_FILE_NODE *FileNode, PFILE_OBJECT FileObject,
     PBOOLEAN PDeletePending);
 VOID FspFileNodeCleanupComplete(FSP_FILE_NODE *FileNode, PFILE_OBJECT FileObject);
 VOID FspFileNodeClose(FSP_FILE_NODE *FileNode, PFILE_OBJECT FileObject);
+NTSTATUS FspFileNodeFlushAndPurgeCache(FSP_FILE_NODE *FileNode,
+    UINT64 FlushOffset64, ULONG FlushLength, BOOLEAN FlushAndPurge);
 VOID FspFileNodeRename(FSP_FILE_NODE *FileNode, PUNICODE_STRING NewFileName);
 BOOLEAN FspFileNodeHasOpenHandles(PDEVICE_OBJECT FsvolDeviceObject,
     PUNICODE_STRING FileName, BOOLEAN SubpathOnly);
@@ -863,6 +866,7 @@ VOID FspFileDescDelete(FSP_FILE_DESC *FileDesc);
 #define FspFileNodeTryAcquireShared(N,F)    FspFileNodeTryAcquireSharedF(N, FspFileNodeAcquire ## F, FALSE)
 #define FspFileNodeAcquireExclusive(N,F)    FspFileNodeAcquireExclusiveF(N, FspFileNodeAcquire ## F)
 #define FspFileNodeTryAcquireExclusive(N,F) FspFileNodeTryAcquireExclusiveF(N, FspFileNodeAcquire ## F, FALSE)
+#define FspFileNodeConvertExclusiveToShared(N,F)    FspFileNodeConvertExclusiveToSharedF(N, FspFileNodeAcquire ## F)
 #define FspFileNodeSetOwner(N,F,P)      FspFileNodeSetOwnerF(N, FspFileNodeAcquire ## F, P)
 #define FspFileNodeRelease(N,F)         FspFileNodeReleaseF(N, FspFileNodeAcquire ## F)
 #define FspFileNodeReleaseOwner(N,F,P)  FspFileNodeReleaseOwnerF(N, FspFileNodeAcquire ## F, P)
@@ -904,4 +908,17 @@ extern CACHE_MANAGER_CALLBACKS FspCacheManagerCallbacks;
 extern FSP_IOPREP_DISPATCH *FspIopPrepareFunction[];
 extern FSP_IOCMPL_DISPATCH *FspIopCompleteFunction[];
 
+/* multiversion support */
+typedef
+NTKERNELAPI
+VOID
+FSP_MV_CcCoherencyFlushAndPurgeCache(
+    _In_ PSECTION_OBJECT_POINTERS SectionObjectPointer,
+    _In_opt_ PLARGE_INTEGER FileOffset,
+    _In_ ULONG Length,
+    _Out_ PIO_STATUS_BLOCK IoStatus,
+    _In_opt_ ULONG Flags
+    );
+extern FSP_MV_CcCoherencyFlushAndPurgeCache *FspMvCcCoherencyFlushAndPurgeCache;
+
 #endif

src/sys/file.c

@@ -13,6 +13,7 @@ VOID FspFileNodeAcquireSharedF(FSP_FILE_NODE *FileNode, ULONG Flags);
 BOOLEAN FspFileNodeTryAcquireSharedF(FSP_FILE_NODE *FileNode, ULONG Flags, BOOLEAN Wait);
 VOID FspFileNodeAcquireExclusiveF(FSP_FILE_NODE *FileNode, ULONG Flags);
 BOOLEAN FspFileNodeTryAcquireExclusiveF(FSP_FILE_NODE *FileNode, ULONG Flags, BOOLEAN Wait);
+VOID FspFileNodeConvertExclusiveToSharedF(FSP_FILE_NODE *FileNode, ULONG Flags);
 VOID FspFileNodeSetOwnerF(FSP_FILE_NODE *FileNode, ULONG Flags, PVOID Owner);
 VOID FspFileNodeReleaseF(FSP_FILE_NODE *FileNode, ULONG Flags);
 VOID FspFileNodeReleaseOwnerF(FSP_FILE_NODE *FileNode, ULONG Flags, PVOID Owner);
@@ -22,6 +23,8 @@ VOID FspFileNodeCleanup(FSP_FILE_NODE *FileNode, PFILE_OBJECT FileObject,
     PBOOLEAN PDeletePending);
 VOID FspFileNodeCleanupComplete(FSP_FILE_NODE *FileNode, PFILE_OBJECT FileObject);
 VOID FspFileNodeClose(FSP_FILE_NODE *FileNode, PFILE_OBJECT FileObject);
+NTSTATUS FspFileNodeFlushAndPurgeCache(FSP_FILE_NODE *FileNode,
+    UINT64 FlushOffset64, ULONG FlushLength, BOOLEAN FlushAndPurge);
 VOID FspFileNodeRename(FSP_FILE_NODE *FileNode, PUNICODE_STRING NewFileName);
 BOOLEAN FspFileNodeHasOpenHandles(PDEVICE_OBJECT FsvolDeviceObject,
     PUNICODE_STRING FileName, BOOLEAN SubpathOnly);
@@ -45,6 +48,7 @@ VOID FspFileDescDelete(FSP_FILE_DESC *FileDesc);
 #pragma alloc_text(PAGE, FspFileNodeTryAcquireSharedF)
 #pragma alloc_text(PAGE, FspFileNodeAcquireExclusiveF)
 #pragma alloc_text(PAGE, FspFileNodeTryAcquireExclusiveF)
+#pragma alloc_text(PAGE, FspFileNodeConvertExclusiveToSharedF)
 #pragma alloc_text(PAGE, FspFileNodeSetOwnerF)
 #pragma alloc_text(PAGE, FspFileNodeReleaseF)
 #pragma alloc_text(PAGE, FspFileNodeReleaseOwnerF)
@@ -52,6 +56,7 @@ VOID FspFileDescDelete(FSP_FILE_DESC *FileDesc);
 #pragma alloc_text(PAGE, FspFileNodeCleanup)
 #pragma alloc_text(PAGE, FspFileNodeCleanupComplete)
 #pragma alloc_text(PAGE, FspFileNodeClose)
+#pragma alloc_text(PAGE, FspFileNodeFlushAndPurgeCache)
 #pragma alloc_text(PAGE, FspFileNodeRename)
 #pragma alloc_text(PAGE, FspFileNodeHasOpenHandles)
 #pragma alloc_text(PAGE, FspFileNodeGetFileInfo)
@@ -246,6 +251,19 @@ BOOLEAN FspFileNodeTryAcquireExclusiveF(FSP_FILE_NODE *FileNode, ULONG Flags, BO
     return Result;
 }
 
+VOID FspFileNodeConvertExclusiveToSharedF(FSP_FILE_NODE *FileNode, ULONG Flags)
+{
+    PAGED_CODE();
+
+    FSP_FILE_NODE_GET_FLAGS();
+
+    if (Flags & FspFileNodeAcquirePgio)
+        ExConvertExclusiveToSharedLite(FileNode->Header.PagingIoResource);
+
+    if (Flags & FspFileNodeAcquireMain)
+        ExConvertExclusiveToSharedLite(FileNode->Header.Resource);
+}
+
 VOID FspFileNodeSetOwnerF(FSP_FILE_NODE *FileNode, ULONG Flags, PVOID Owner)
 {
     PAGED_CODE();
@@ -515,6 +533,54 @@ VOID FspFileNodeClose(FSP_FILE_NODE *FileNode, PFILE_OBJECT FileObject)
         FspFileNodeDereference(FileNode);
 }
 
+NTSTATUS FspFileNodeFlushAndPurgeCache(FSP_FILE_NODE *FileNode,
+    UINT64 FlushOffset64, ULONG FlushLength, BOOLEAN FlushAndPurge)
+{
+    /*
+     * The FileNode must be acquired exclusive (Full) when calling this function.
+     */
+
+    PAGED_CODE();
+
+    LARGE_INTEGER FlushOffset;
+    PLARGE_INTEGER PFlushOffset = &FlushOffset;
+    FSP_FSCTL_FILE_INFO FileInfo;
+    IO_STATUS_BLOCK IoStatus = { 0 };
+
+    FlushOffset.QuadPart = FlushOffset64;
+    if (FILE_WRITE_TO_END_OF_FILE == FlushOffset.LowPart && -1L == FlushOffset.HighPart)
+    {
+        if (FspFileNodeTryGetFileInfo(FileNode, &FileInfo))
+            FlushOffset.QuadPart = FileInfo.FileSize;
+        else
+            PFlushOffset = 0; /* we don't know how big the file is, so flush it all! */
+    }
+
+    if (0 != FspMvCcCoherencyFlushAndPurgeCache)
+    {
+        /* if we are on Win7+ use CcCoherencyFlushAndPurgeCache */
+        FspMvCcCoherencyFlushAndPurgeCache(
+            &FileNode->NonPaged->SectionObjectPointers, PFlushOffset, FlushLength, &IoStatus,
+            FlushAndPurge ? 0 : CC_FLUSH_AND_PURGE_NO_PURGE);
+
+        return STATUS_CACHE_PAGE_LOCKED == IoStatus.Status ?
+            STATUS_SUCCESS/* liar! */:
+            IoStatus.Status;
+    }
+    else
+    {
+        /* do it the old-fashioned way; non-cached and mmap'ed I/O are non-coherent */
+        CcFlushCache(&FileNode->NonPaged->SectionObjectPointers, PFlushOffset, FlushLength, &IoStatus);
+        if (!NT_SUCCESS(IoStatus.Status))
+            return IoStatus.Status;
+
+        if (FlushAndPurge)
+            CcPurgeCacheSection(&FileNode->NonPaged->SectionObjectPointers, PFlushOffset, FlushLength, FALSE);
+
+        return STATUS_SUCCESS;
+    }
+}
+
 VOID FspFileNodeRename(FSP_FILE_NODE *FileNode, PUNICODE_STRING NewFileName)
 {
     PAGED_CODE();

src/sys/read.c

@@ -252,11 +252,19 @@ NTSTATUS FspFsvolReadPrepare(
     PIO_STACK_LOCATION IrpSp = IoGetCurrentIrpStackLocation(Irp);
     PFILE_OBJECT FileObject = IrpSp->FileObject;
     FSP_FILE_NODE *FileNode = FileObject->FsContext;
+    BOOLEAN PagingIo = BooleanFlagOn(Irp->Flags, IRP_PAGING_IO);
     FSP_SAFE_MDL *SafeMdl = 0;
     PVOID Address;
     PEPROCESS Process;
+    BOOLEAN FlushCache;
     BOOLEAN Success;
 
+    FlushCache = !PagingIo && 0 != FileObject->SectionObjectPointer->DataSectionObject;
+        /* !!!: DataSectionObject accessed outside a lock. Hmmm! */
+
+    if (FlushCache)
+        Success = DEBUGTEST(90, TRUE) && FspFileNodeTryAcquireExclusive(FileNode, Full);
+    else
         Success = DEBUGTEST(90, TRUE) && FspFileNodeTryAcquireShared(FileNode, Full);
     if (!Success)
     {
@@ -264,6 +272,22 @@ NTSTATUS FspFsvolReadPrepare(
         return Result;
     }
 
+    /* if this is a non-cached transfer on a cached file then flush the file */
+    if (FlushCache)
+    {
+        Result = FspFileNodeFlushAndPurgeCache(FileNode,
+            IrpSp->Parameters.Read.ByteOffset.QuadPart,
+            IrpSp->Parameters.Read.Length,
+            FALSE);
+        if (!NT_SUCCESS(Result))
+        {
+            FspFileNodeRelease(FileNode, Full);
+            return Result;
+        }
+
+        FspFileNodeConvertExclusiveToShared(FileNode, Full);
+    }
+
     /* create a "safe" MDL if necessary */
     if (!FspSafeMdlCheck(Irp->MdlAddress))
     {

src/sys/write.c

@@ -335,28 +335,15 @@ NTSTATUS FspFsvolWritePrepare(
     /* if this is a non-cached transfer on a cached file then flush and purge the file */
     if (!PagingIo && 0 != FileObject->SectionObjectPointer->DataSectionObject)
     {
-        LARGE_INTEGER FlushOffset = IrpSp->Parameters.Write.ByteOffset;
+        Result = FspFileNodeFlushAndPurgeCache(FileNode,
-        PLARGE_INTEGER PFlushOffset = &FlushOffset;
+            IrpSp->Parameters.Write.ByteOffset.QuadPart,
-        ULONG FlushLength = IrpSp->Parameters.Write.Length;
+            IrpSp->Parameters.Write.Length,
-        FSP_FSCTL_FILE_INFO FileInfo;
+            TRUE);
-        IO_STATUS_BLOCK IoStatus = { 0 };
+        if (!NT_SUCCESS(Result))
-
-        if (FILE_WRITE_TO_END_OF_FILE == FlushOffset.LowPart && -1L == FlushOffset.HighPart)
-        {
-            if (FspFileNodeTryGetFileInfo(FileNode, &FileInfo))
-                FlushOffset.QuadPart = FileInfo.FileSize;
-            else
-                PFlushOffset = 0; /* we don't know how big the file is, so flush it all! */
-        }
-
-        CcFlushCache(FileObject->SectionObjectPointer, PFlushOffset, FlushLength, &IoStatus);
-        if (!NT_SUCCESS(IoStatus.Status))
         {
             FspFileNodeRelease(FileNode, Full);
-            return IoStatus.Status;
+            return Result;
         }
-
-        CcPurgeCacheSection(FileObject->SectionObjectPointer, PFlushOffset, FlushLength, FALSE);
     }
 
     /* create a "safe" MDL if necessary */

tst/winfsp-tests/rdwr-test.c

@@ -2,6 +2,7 @@
 #include <tlib/testsuite.h>
 #include <strsafe.h>
 #include <time.h>
+#include <VersionHelpers.h>
 #include "memfs.h"
 
 void *memfs_start_ex(ULONG Flags, ULONG FileInfoTimeout);
@@ -710,16 +711,27 @@ void rdwr_mmap_test(void)
     {
         /*
          * WinFsp does not currently provide coherency between mmap'ed I/O and ReadFile/WriteFile
-         * in the following circumstances:
+         * before Windows 7 in the following cases:
          *   - FileInfoTimeout != INFINITE
          *   - CreateFlags & FILE_FLAG_NO_BUFFERING
+         *
+         * In Windows 7 and above the new DDI CcCoherencyFlushAndPurgeCache allows us to provide
+         * coherency in those cases.
          */
 
+        if (IsWindows7OrGreater())
+        {
+            rdwr_mmap_dotest(MemfsDisk, 0, 0, 1000, 0, FALSE);
+            rdwr_mmap_dotest(MemfsDisk, 0, 0, 1000, FILE_FLAG_NO_BUFFERING, FALSE);
+            rdwr_mmap_dotest(MemfsDisk, 0, 0, 1000, FILE_FLAG_WRITE_THROUGH, FALSE);
+        }
         rdwr_mmap_dotest(MemfsDisk, 0, 0, 1000, 0, TRUE);
         rdwr_mmap_dotest(MemfsDisk, 0, 0, 1000, FILE_FLAG_NO_BUFFERING, TRUE);
         rdwr_mmap_dotest(MemfsDisk, 0, 0, 1000, FILE_FLAG_WRITE_THROUGH, TRUE);
 
         rdwr_mmap_dotest(MemfsDisk, 0, 0, INFINITE, 0, FALSE);
+        if (IsWindows7OrGreater())
+            rdwr_mmap_dotest(MemfsDisk, 0, 0, INFINITE, FILE_FLAG_NO_BUFFERING, FALSE);
         rdwr_mmap_dotest(MemfsDisk, 0, 0, INFINITE, FILE_FLAG_WRITE_THROUGH, FALSE);
         rdwr_mmap_dotest(MemfsDisk, 0, 0, INFINITE, 0, TRUE);
         rdwr_mmap_dotest(MemfsDisk, 0, 0, INFINITE, FILE_FLAG_NO_BUFFERING, TRUE);
@@ -729,16 +741,27 @@ void rdwr_mmap_test(void)
     {
         /*
          * WinFsp does not currently provide coherency between mmap'ed I/O and ReadFile/WriteFile
-         * in the following circumstances:
+         * before Windows 7 in the following cases:
          *   - FileInfoTimeout != INFINITE
          *   - CreateFlags & FILE_FLAG_NO_BUFFERING
+         *
+         * In Windows 7 and above the new DDI CcCoherencyFlushAndPurgeCache allows us to provide
+         * coherency in those cases.
          */
 
+        if (IsWindows7OrGreater())
+        {
+            rdwr_mmap_dotest(MemfsNet, L"\\\\memfs\\share", L"\\\\memfs\\share", 1000, 0, FALSE);
+            rdwr_mmap_dotest(MemfsNet, L"\\\\memfs\\share", L"\\\\memfs\\share", 1000, FILE_FLAG_NO_BUFFERING, FALSE);
+            rdwr_mmap_dotest(MemfsNet, L"\\\\memfs\\share", L"\\\\memfs\\share", 1000, FILE_FLAG_WRITE_THROUGH, FALSE);
+        }
         rdwr_mmap_dotest(MemfsNet, L"\\\\memfs\\share", L"\\\\memfs\\share", 1000, 0, TRUE);
         rdwr_mmap_dotest(MemfsNet, L"\\\\memfs\\share", L"\\\\memfs\\share", 1000, FILE_FLAG_NO_BUFFERING, TRUE);
         rdwr_mmap_dotest(MemfsNet, L"\\\\memfs\\share", L"\\\\memfs\\share", 1000, FILE_FLAG_WRITE_THROUGH, TRUE);
 
         rdwr_mmap_dotest(MemfsNet, L"\\\\memfs\\share", L"\\\\memfs\\share", INFINITE, 0, FALSE);
+        if (IsWindows7OrGreater())
+            rdwr_mmap_dotest(MemfsNet, L"\\\\memfs\\share", L"\\\\memfs\\share", INFINITE, FILE_FLAG_NO_BUFFERING, FALSE);
         rdwr_mmap_dotest(MemfsNet, L"\\\\memfs\\share", L"\\\\memfs\\share", INFINITE, FILE_FLAG_WRITE_THROUGH, FALSE);
         rdwr_mmap_dotest(MemfsNet, L"\\\\memfs\\share", L"\\\\memfs\\share", INFINITE, 0, TRUE);
         rdwr_mmap_dotest(MemfsNet, L"\\\\memfs\\share", L"\\\\memfs\\share", INFINITE, FILE_FLAG_NO_BUFFERING, TRUE);