sys: FastIo: read/write implementation

src/sys/callbacks.c

@@ -66,11 +66,17 @@ BOOLEAN FspFastIoCheckIfPossible(
     PIO_STATUS_BLOCK IoStatus,
     PDEVICE_OBJECT DeviceObject)
 {
+#if 1
+    ASSERT(FALSE);
+    return FALSE;
+
+#else
     FSP_ENTER_BOOL(PAGED_CODE());
 
     Result = FALSE;
 
     FSP_LEAVE_BOOL("FileObject=%p", FileObject);
+#endif
 }
 
 VOID FspAcquireFileForNtCreateSection(
@@ -299,9 +305,16 @@ VOID FspPropagateTopFlags(PIRP Irp, PIRP TopLevelIrp)
 
     if ((PIRP)FSRTL_MAX_TOP_LEVEL_IRP_FLAG >= TopLevelIrp)
     {
+        /*
+         * FAST I/O only acquires the Main lock.
+         * Other (non-IRP) top levels acquire the Full lock.
+         */
         DEBUGBREAK_EX(iorecu);
 
-        FspIrpSetTopFlags(Irp, FspFileNodeAcquireFull);
+        FspIrpSetTopFlags(Irp,
+            (PIRP)FSRTL_FAST_IO_TOP_LEVEL_IRP == TopLevelIrp ?
+            FspFileNodeAcquireMain :
+            FspFileNodeAcquireFull);
     }
     else if ((PIRP)MM_SYSTEM_RANGE_START <= TopLevelIrp && IO_TYPE_IRP == TopLevelIrp->Type)
     {

src/sys/driver.c

@@ -89,8 +89,8 @@ NTSTATUS DriverEntry(
     /* setup fast I/O and resource acquisition */
     FspFastIoDispatch.SizeOfFastIoDispatch = sizeof FspFastIoDispatch;
     FspFastIoDispatch.FastIoCheckIfPossible = FspFastIoCheckIfPossible;
-    //FspFastIoDispatch.FastIoRead = 0;
+    FspFastIoDispatch.FastIoRead = FspFastIoRead;
-    //FspFastIoDispatch.FastIoWrite = 0;
+    FspFastIoDispatch.FastIoWrite = FspFastIoWrite;
     FspFastIoDispatch.FastIoQueryBasicInfo = FspFastIoQueryBasicInfo;
     FspFastIoDispatch.FastIoQueryStandardInfo = FspFastIoQueryStandardInfo;
     //FspFastIoDispatch.FastIoLock = 0;

src/sys/driver.h

@@ -304,6 +304,43 @@ VOID FspTraceNtStatus(const char *file, int line, const char *func, NTSTATUS Sta
         }                               \
     );                                  \
     return Result
+#define FSP_ENTER_FIO(...)              \
+    PDEVICE_OBJECT FsvolDeviceObject;   \
+    if (FspFsmupDeviceExtensionKind == FspDeviceExtension(DeviceObject)->Kind)\
+    {                                   \
+        FsvolDeviceObject = FspMupGetFsvolDeviceObject(FileObject);\
+        if (0 == FsvolDeviceObject)     \
+        {                               \
+            IoStatus->Status = STATUS_INVALID_DEVICE_REQUEST;\
+            IoStatus->Information = 0;  \
+            return TRUE;                \
+        }                               \
+    }                                   \
+    else                                \
+        FsvolDeviceObject = DeviceObject;\
+    BOOLEAN Result = TRUE;              \
+    BOOLEAN fsp_device_deref = FALSE;   \
+    FSP_ENTER_(ioentr, __VA_ARGS__);    \
+    do                                  \
+    {                                   \
+        ASSERT(0 == IoGetTopLevelIrp()); \
+        IoSetTopLevelIrp((PIRP)FSRTL_FAST_IO_TOP_LEVEL_IRP); \
+        if (!FspDeviceReference(FsvolDeviceObject))\
+        {                               \
+            IoStatus->Status = STATUS_CANCELLED;\
+            IoStatus->Information = 0;  \
+            goto fsp_leave_label;       \
+        }                               \
+        fsp_device_deref = TRUE;        \
+    } while (0,0)
+#define FSP_LEAVE_FIO(fmt, ...)         \
+    FSP_LEAVE_(                         \
+        FSP_DEBUGLOG_(fmt, " = %s", __VA_ARGS__, Result ? "TRUE" : "FALSE");\
+        if (fsp_device_deref)           \
+            FspDeviceDereference(FsvolDeviceObject);\
+        IoSetTopLevelIrp(0);            \
+    );                                  \
+    return Result
 #define FSP_ENTER_BOOL(...)             \
     BOOLEAN Result = TRUE; FSP_ENTER_(iocall, __VA_ARGS__)
 #define FSP_LEAVE_BOOL(fmt, ...)        \
@@ -383,6 +420,8 @@ FSP_IOPREP_DISPATCH FspFsvolWritePrepare;
 FSP_IOCMPL_DISPATCH FspFsvolWriteComplete;
 
 /* fast I/O and resource acquisition callbacks */
+FAST_IO_READ FspFastIoRead;
+FAST_IO_WRITE FspFastIoWrite;
 FAST_IO_QUERY_BASIC_INFO FspFastIoQueryBasicInfo;
 FAST_IO_QUERY_STANDARD_INFO FspFastIoQueryStandardInfo;
 FAST_IO_QUERY_NETWORK_OPEN_INFO FspFastIoQueryNetworkOpenInfo;
@@ -1360,6 +1399,8 @@ NTSTATUS FspMupRegister(
     PDEVICE_OBJECT FsmupDeviceObject, PDEVICE_OBJECT FsvolDeviceObject);
 VOID FspMupUnregister(
     PDEVICE_OBJECT FsmupDeviceObject, PDEVICE_OBJECT FsvolDeviceObject);
+PDEVICE_OBJECT FspMupGetFsvolDeviceObject(
+    PFILE_OBJECT FileObject);
 NTSTATUS FspMupHandleIrp(
     PDEVICE_OBJECT FsmupDeviceObject, PIRP Irp);
 

src/sys/file.c

@@ -342,7 +342,7 @@ NTSTATUS FspFileNodeCreate(PDEVICE_OBJECT DeviceObject,
     RtlZeroMemory(FileNode, sizeof *FileNode + ExtraSize);
     FileNode->Header.NodeTypeCode = FspFileNodeFileKind;
     FileNode->Header.NodeByteSize = sizeof *FileNode;
-    FileNode->Header.IsFastIoPossible = FastIoIsNotPossible;
+    FileNode->Header.IsFastIoPossible = FastIoIsQuestionable;
     FileNode->Header.Resource = &NonPaged->Resource;
     FileNode->Header.PagingIoResource = &NonPaged->PagingIoResource;
     FileNode->Header.ValidDataLength.QuadPart = MAXLONGLONG;

src/sys/mup.c

@@ -43,6 +43,8 @@ NTSTATUS FspMupRegister(
     PDEVICE_OBJECT FsmupDeviceObject, PDEVICE_OBJECT FsvolDeviceObject);
 VOID FspMupUnregister(
     PDEVICE_OBJECT FsmupDeviceObject, PDEVICE_OBJECT FsvolDeviceObject);
+PDEVICE_OBJECT FspMupGetFsvolDeviceObject(
+    PFILE_OBJECT FileObject);
 NTSTATUS FspMupHandleIrp(
     PDEVICE_OBJECT FsmupDeviceObject, PIRP Irp);
 static NTSTATUS FspMupRedirQueryPathEx(
@@ -52,6 +54,7 @@ static NTSTATUS FspMupRedirQueryPathEx(
 #pragma alloc_text(PAGE, FspMupGetClassName)
 #pragma alloc_text(PAGE, FspMupRegister)
 #pragma alloc_text(PAGE, FspMupUnregister)
+#pragma alloc_text(PAGE, FspMupGetFsvolDeviceObject)
 #pragma alloc_text(PAGE, FspMupHandleIrp)
 #pragma alloc_text(PAGE, FspMupRedirQueryPathEx)
 #endif
@@ -191,6 +194,27 @@ VOID FspMupUnregister(
     ExReleaseResourceLite(&FsmupDeviceExtension->PrefixTableResource);
 }
 
+PDEVICE_OBJECT FspMupGetFsvolDeviceObject(
+    PFILE_OBJECT FileObject)
+{
+    PAGED_CODE();
+
+    PDEVICE_OBJECT FsvolDeviceObject = 0;
+
+    ASSERT(0 != FileObject);
+
+    if (FspFileNodeIsValid(FileObject->FsContext))
+        FsvolDeviceObject = ((FSP_FILE_NODE*)FileObject->FsContext)->FsvolDeviceObject;
+    else if (0 != FileObject->FsContext2 &&
+#pragma prefast(disable:28175, "We are a filesystem: ok to access DeviceObject->Type")
+        IO_TYPE_DEVICE == ((PDEVICE_OBJECT)FileObject->FsContext2)->Type &&
+        0 != ((PDEVICE_OBJECT)FileObject->FsContext2)->DeviceExtension &&
+        FspFsvolDeviceExtensionKind == FspDeviceExtension((PDEVICE_OBJECT)FileObject->FsContext2)->Kind)
+        FsvolDeviceObject = (PDEVICE_OBJECT)FileObject->FsContext2;
+
+    return FsvolDeviceObject;
+}
+
 NTSTATUS FspMupHandleIrp(
     PDEVICE_OBJECT FsmupDeviceObject, PIRP Irp)
 {
@@ -276,7 +300,7 @@ NTSTATUS FspMupHandleIrp(
                 FsvolDeviceObject = ((FSP_FILE_NODE *)FileObject->FsContext)->FsvolDeviceObject;
             else if (0 != FileObject->FsContext2 &&
 #pragma prefast(disable:28175, "We are a filesystem: ok to access DeviceObject->Type")
-                3 == ((PDEVICE_OBJECT)FileObject->FsContext2)->Type &&
+                IO_TYPE_DEVICE == ((PDEVICE_OBJECT)FileObject->FsContext2)->Type &&
                 0 != ((PDEVICE_OBJECT)FileObject->FsContext2)->DeviceExtension &&
                 FspFsvolDeviceExtensionKind == FspDeviceExtension((PDEVICE_OBJECT)FileObject->FsContext2)->Kind)
                 FsvolDeviceObject = (PDEVICE_OBJECT)FileObject->FsContext2;

src/sys/read.c

@@ -21,6 +21,7 @@
 
 #include <sys/driver.h>
 
+FAST_IO_READ FspFastIoRead;
 static NTSTATUS FspFsvolRead(
     PDEVICE_OBJECT FsvolDeviceObject, PIRP Irp, PIO_STACK_LOCATION IrpSp);
 static NTSTATUS FspFsvolReadCached(
@@ -35,6 +36,7 @@ static FSP_IOP_REQUEST_FINI FspFsvolReadNonCachedRequestFini;
 FSP_DRIVER_DISPATCH FspRead;
 
 #ifdef ALLOC_PRAGMA
+#pragma alloc_text(PAGE, FspFastIoRead)
 #pragma alloc_text(PAGE, FspFsvolRead)
 #pragma alloc_text(PAGE, FspFsvolReadCached)
 #pragma alloc_text(PAGE, FspFsvolReadNonCached)
@@ -55,6 +57,107 @@ enum
 };
 FSP_FSCTL_STATIC_ASSERT(RequestCookie == RequestSafeMdl, "");
 
+BOOLEAN FspFastIoRead(
+    PFILE_OBJECT FileObject,
+    PLARGE_INTEGER ByteOffset,
+    ULONG Length,
+    BOOLEAN CanWait,
+    ULONG Key,
+    PVOID UserBuffer,
+    PIO_STATUS_BLOCK IoStatus,
+    PDEVICE_OBJECT DeviceObject)
+{
+    FSP_ENTER_FIO(PAGED_CODE());
+
+    IoStatus->Status = STATUS_SUCCESS;
+    IoStatus->Information = 0;
+
+    if (!FspFileNodeIsValid(FileObject->FsContext))
+    {
+        IoStatus->Status = STATUS_INVALID_DEVICE_REQUEST;
+        FSP_RETURN(Result = TRUE);
+    }
+
+    FSP_FILE_NODE* FileNode = FileObject->FsContext;
+    LARGE_INTEGER ReadOffset = *ByteOffset;
+    ULONG ReadLength = Length;
+    FSP_FSCTL_FILE_INFO FileInfo;
+
+    /* only regular files can be read */
+    if (FileNode->IsDirectory)
+    {
+        IoStatus->Status = STATUS_INVALID_PARAMETER;
+        FSP_RETURN(Result = TRUE);
+    }
+
+    /* do we have anything to read? */
+    if (0 == ReadLength)
+        FSP_RETURN(Result = TRUE);
+
+    /* does the file support caching? */
+    if (!FlagOn(FileObject->Flags, FO_CACHE_SUPPORTED))
+        FSP_RETURN(Result = FALSE);
+
+    /* try to acquire the FileNode Main shared */
+    Result = DEBUGTEST(90) &&
+        FspFileNodeTryAcquireSharedF(FileNode, FspFileNodeAcquireMain, CanWait);
+    if (!Result)
+        FSP_RETURN(Result = FALSE);
+
+    /* is the file actually cached? */
+    if (0 == FileObject->PrivateCacheMap)
+    {
+        FspFileNodeRelease(FileNode, Main);
+        FSP_RETURN(Result = FALSE);
+    }
+
+    /* does the file have oplocks or file locks? */
+    Result =
+        FsRtlOplockIsFastIoPossible(FspFileNodeAddrOfOplock(FileNode)) &&
+        !FsRtlAreThereCurrentFileLocks(&FileNode->FileLock);
+    if (!Result)
+    {
+        FspFileNodeRelease(FileNode, Main);
+        FSP_RETURN(Result = FALSE);
+    }
+
+    /* trim ReadLength; the cache manager does not tolerate reads beyond file size */
+    ASSERT(FspTimeoutInfinity32 ==
+        FspFsvolDeviceExtension(FsvolDeviceObject)->VolumeParams.FileInfoTimeout);
+    FspFileNodeGetFileInfo(FileNode, &FileInfo);
+    if ((UINT64)ReadOffset.QuadPart >= FileInfo.FileSize)
+    {
+        FspFileNodeRelease(FileNode, Main);
+        IoStatus->Status = STATUS_END_OF_FILE;
+        FSP_RETURN(Result = TRUE);
+    }
+    if ((UINT64)ReadLength > FileInfo.FileSize - ReadOffset.QuadPart)
+        ReadLength = (ULONG)(FileInfo.FileSize - ReadOffset.QuadPart);
+
+    NTSTATUS Result0 =
+        FspCcCopyRead(FileObject, &ReadOffset, ReadLength, CanWait, UserBuffer, IoStatus);
+    if (!NT_SUCCESS(Result0))
+    {
+        IoStatus->Status = Result0;
+        IoStatus->Information = 0;
+        FspFileNodeRelease(FileNode, Main);
+        FSP_RETURN(Result = TRUE);
+    }
+    Result = STATUS_SUCCESS == Result0;
+
+    SetFlag(FileObject->Flags, FO_FILE_FAST_IO_READ);
+    if (Result)
+        FileObject->CurrentByteOffset.QuadPart = ReadOffset.QuadPart + ReadLength;
+
+    FspFileNodeRelease(FileNode, Main);
+
+    FSP_LEAVE_FIO(
+        "FileObject=%p, UserBuffer=%p, CanWait=%d, "
+        "Key=%#lx, ByteOffset=%#lx:%#lx, Length=%ld",
+        FileObject, UserBuffer, CanWait,
+        Key, ByteOffset->HighPart, ByteOffset->LowPart, Length);
+}
+
 static NTSTATUS FspFsvolRead(
     PDEVICE_OBJECT FsvolDeviceObject, PIRP Irp, PIO_STACK_LOCATION IrpSp)
 {

src/sys/write.c

@@ -21,6 +21,7 @@
 
 #include <sys/driver.h>
 
+FAST_IO_WRITE FspFastIoWrite;
 static NTSTATUS FspFsvolWrite(
     PDEVICE_OBJECT FsvolDeviceObject, PIRP Irp, PIO_STACK_LOCATION IrpSp);
 static NTSTATUS FspFsvolWriteCached(
@@ -36,6 +37,7 @@ static FSP_IOP_REQUEST_FINI FspFsvolWriteNonCachedRequestFini;
 FSP_DRIVER_DISPATCH FspWrite;
 
 #ifdef ALLOC_PRAGMA
+#pragma alloc_text(PAGE, FspFastIoWrite)
 #pragma alloc_text(PAGE, FspFsvolWrite)
 #pragma alloc_text(PAGE, FspFsvolWriteCached)
 #pragma alloc_text(PAGE, FspFsvolWriteNonCached)
@@ -56,6 +58,129 @@ enum
 };
 FSP_FSCTL_STATIC_ASSERT(RequestCookie == RequestSafeMdl, "");
 
+BOOLEAN FspFastIoWrite(
+    PFILE_OBJECT FileObject,
+    PLARGE_INTEGER ByteOffset,
+    ULONG Length,
+    BOOLEAN CanWait,
+    ULONG Key,
+    PVOID UserBuffer,
+    PIO_STATUS_BLOCK IoStatus,
+    PDEVICE_OBJECT DeviceObject)
+{
+    FSP_ENTER_FIO(PAGED_CODE());
+
+    IoStatus->Status = STATUS_SUCCESS;
+    IoStatus->Information = 0;
+
+    if (!FspFileNodeIsValid(FileObject->FsContext))
+    {
+        IoStatus->Status = STATUS_INVALID_DEVICE_REQUEST;
+        FSP_RETURN(Result = TRUE);
+    }
+
+    FSP_FILE_NODE* FileNode = FileObject->FsContext;
+    LARGE_INTEGER WriteOffset = *ByteOffset;
+    ULONG WriteLength = Length;
+    BOOLEAN WriteToEndOfFile =
+        FILE_WRITE_TO_END_OF_FILE == WriteOffset.LowPart && -1L == WriteOffset.HighPart;
+    FSP_FSCTL_FILE_INFO FileInfo;
+    UINT64 WriteEndOffset;
+    BOOLEAN ExtendingFile;
+
+    /* only regular files can be written */
+    if (FileNode->IsDirectory)
+    {
+        IoStatus->Status = STATUS_INVALID_PARAMETER;
+        FSP_RETURN(Result = TRUE);
+    }
+
+    /* do we have anything to write? */
+    if (0 == WriteLength)
+        FSP_RETURN(Result = TRUE);
+
+    /* WinFsp cannot do Fast I/O when extending file */
+    if (WriteToEndOfFile)
+        FSP_RETURN(Result = FALSE);
+
+    /* does the file support caching? is it write-through? */
+    if (!FlagOn(FileObject->Flags, FO_CACHE_SUPPORTED) ||
+        FlagOn(FileObject->Flags, FO_WRITE_THROUGH))
+        FSP_RETURN(Result = FALSE);
+
+    /* can we write without flushing? */
+    Result = DEBUGTEST(90) &&
+        CcCanIWrite(FileObject, WriteLength, CanWait, FALSE) &&
+        CcCopyWriteWontFlush(FileObject, &WriteOffset, WriteLength);
+    if (!Result)
+        FSP_RETURN(Result = FALSE);
+
+    /* try to acquire the FileNode Main exclusive */
+    Result = DEBUGTEST(90) &&
+        FspFileNodeTryAcquireExclusiveF(FileNode, FspFileNodeAcquireMain, CanWait);
+    if (!Result)
+        FSP_RETURN(Result = FALSE);
+
+    /* is the file actually cached? */
+    if (0 == FileObject->PrivateCacheMap)
+    {
+        FspFileNodeRelease(FileNode, Main);
+        FSP_RETURN(Result = FALSE);
+    }
+
+    /* does the file have oplocks or file locks? */
+    Result =
+        FsRtlOplockIsFastIoPossible(FspFileNodeAddrOfOplock(FileNode)) &&
+        !FsRtlAreThereCurrentFileLocks(&FileNode->FileLock);
+    if (!Result)
+    {
+        FspFileNodeRelease(FileNode, Main);
+        FSP_RETURN(Result = FALSE);
+    }
+
+    /* compute new file size */
+    ASSERT(FspTimeoutInfinity32 ==
+        FspFsvolDeviceExtension(FsvolDeviceObject)->VolumeParams.FileInfoTimeout);
+    FspFileNodeGetFileInfo(FileNode, &FileInfo);
+    if (WriteToEndOfFile)
+        WriteOffset.QuadPart = FileInfo.FileSize;
+    WriteEndOffset = WriteOffset.QuadPart + WriteLength;
+    ExtendingFile = FileInfo.FileSize < WriteEndOffset;
+
+    /* WinFsp cannot do Fast I/O when extending file */
+    if (ExtendingFile)
+    {
+        FspFileNodeRelease(FileNode, Main);
+        FSP_RETURN(Result = FALSE);
+    }
+
+    NTSTATUS Result0 =
+        FspCcCopyWrite(FileObject, &WriteOffset, WriteLength, CanWait, UserBuffer);
+    if (!NT_SUCCESS(Result0))
+    {
+        IoStatus->Status = Result0;
+        IoStatus->Information = 0;
+        FspFileNodeRelease(FileNode, Main);
+        FSP_RETURN(Result = FALSE);
+    }
+    Result = STATUS_SUCCESS == Result0;
+
+    if (Result)
+    {
+        SetFlag(FileObject->Flags, FO_FILE_MODIFIED);
+        FileObject->CurrentByteOffset.QuadPart = WriteEndOffset;
+        IoStatus->Information = WriteLength;
+    }
+
+    FspFileNodeRelease(FileNode, Main);
+
+    FSP_LEAVE_FIO(
+        "FileObject=%p, UserBuffer=%p, CanWait=%d, "
+        "Key=%#lx, ByteOffset=%#lx:%#lx, Length=%ld",
+        FileObject, UserBuffer, CanWait,
+        Key, ByteOffset->HighPart, ByteOffset->LowPart, Length);
+}
+
 static NTSTATUS FspFsvolWrite(
     PDEVICE_OBJECT FsvolDeviceObject, PIRP Irp, PIO_STACK_LOCATION IrpSp)
 {