From 20680fa5b5a90526cb5f38058b489ed1acc50d53 Mon Sep 17 00:00:00 2001 From: Bill Zissimopoulos Date: Sun, 20 Mar 2022 20:31:54 +0000 Subject: [PATCH] sys: FastIo: read/write implementation --- src/sys/callbacks.c | 15 +++++- src/sys/driver.c | 4 +- src/sys/driver.h | 41 +++++++++++++++ src/sys/file.c | 2 +- src/sys/mup.c | 26 ++++++++- src/sys/read.c | 103 ++++++++++++++++++++++++++++++++++++ src/sys/write.c | 125 ++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 311 insertions(+), 5 deletions(-) diff --git a/src/sys/callbacks.c b/src/sys/callbacks.c index 0129b21f..56e23d81 100644 --- a/src/sys/callbacks.c +++ b/src/sys/callbacks.c @@ -66,11 +66,17 @@ BOOLEAN FspFastIoCheckIfPossible( PIO_STATUS_BLOCK IoStatus, PDEVICE_OBJECT DeviceObject) { +#if 1 + ASSERT(FALSE); + return FALSE; + +#else FSP_ENTER_BOOL(PAGED_CODE()); Result = FALSE; FSP_LEAVE_BOOL("FileObject=%p", FileObject); +#endif } VOID FspAcquireFileForNtCreateSection( @@ -299,9 +305,16 @@ VOID FspPropagateTopFlags(PIRP Irp, PIRP TopLevelIrp) if ((PIRP)FSRTL_MAX_TOP_LEVEL_IRP_FLAG >= TopLevelIrp) { + /* + * FAST I/O only acquires the Main lock. + * Other (non-IRP) top levels acquire the Full lock. + */ DEBUGBREAK_EX(iorecu); - FspIrpSetTopFlags(Irp, FspFileNodeAcquireFull); + FspIrpSetTopFlags(Irp, + (PIRP)FSRTL_FAST_IO_TOP_LEVEL_IRP == TopLevelIrp ? + FspFileNodeAcquireMain : + FspFileNodeAcquireFull); } else if ((PIRP)MM_SYSTEM_RANGE_START <= TopLevelIrp && IO_TYPE_IRP == TopLevelIrp->Type) { diff --git a/src/sys/driver.c b/src/sys/driver.c index c633487f..cb7d2bfd 100644 --- a/src/sys/driver.c +++ b/src/sys/driver.c @@ -89,8 +89,8 @@ NTSTATUS DriverEntry( /* setup fast I/O and resource acquisition */ FspFastIoDispatch.SizeOfFastIoDispatch = sizeof FspFastIoDispatch; FspFastIoDispatch.FastIoCheckIfPossible = FspFastIoCheckIfPossible; - //FspFastIoDispatch.FastIoRead = 0; - //FspFastIoDispatch.FastIoWrite = 0; + FspFastIoDispatch.FastIoRead = FspFastIoRead; + FspFastIoDispatch.FastIoWrite = FspFastIoWrite; FspFastIoDispatch.FastIoQueryBasicInfo = FspFastIoQueryBasicInfo; FspFastIoDispatch.FastIoQueryStandardInfo = FspFastIoQueryStandardInfo; //FspFastIoDispatch.FastIoLock = 0; diff --git a/src/sys/driver.h b/src/sys/driver.h index 0a882d72..ebe29371 100644 --- a/src/sys/driver.h +++ b/src/sys/driver.h @@ -304,6 +304,43 @@ VOID FspTraceNtStatus(const char *file, int line, const char *func, NTSTATUS Sta } \ ); \ return Result +#define FSP_ENTER_FIO(...) \ + PDEVICE_OBJECT FsvolDeviceObject; \ + if (FspFsmupDeviceExtensionKind == FspDeviceExtension(DeviceObject)->Kind)\ + { \ + FsvolDeviceObject = FspMupGetFsvolDeviceObject(FileObject);\ + if (0 == FsvolDeviceObject) \ + { \ + IoStatus->Status = STATUS_INVALID_DEVICE_REQUEST;\ + IoStatus->Information = 0; \ + return TRUE; \ + } \ + } \ + else \ + FsvolDeviceObject = DeviceObject;\ + BOOLEAN Result = TRUE; \ + BOOLEAN fsp_device_deref = FALSE; \ + FSP_ENTER_(ioentr, __VA_ARGS__); \ + do \ + { \ + ASSERT(0 == IoGetTopLevelIrp()); \ + IoSetTopLevelIrp((PIRP)FSRTL_FAST_IO_TOP_LEVEL_IRP); \ + if (!FspDeviceReference(FsvolDeviceObject))\ + { \ + IoStatus->Status = STATUS_CANCELLED;\ + IoStatus->Information = 0; \ + goto fsp_leave_label; \ + } \ + fsp_device_deref = TRUE; \ + } while (0,0) +#define FSP_LEAVE_FIO(fmt, ...) \ + FSP_LEAVE_( \ + FSP_DEBUGLOG_(fmt, " = %s", __VA_ARGS__, Result ? "TRUE" : "FALSE");\ + if (fsp_device_deref) \ + FspDeviceDereference(FsvolDeviceObject);\ + IoSetTopLevelIrp(0); \ + ); \ + return Result #define FSP_ENTER_BOOL(...) \ BOOLEAN Result = TRUE; FSP_ENTER_(iocall, __VA_ARGS__) #define FSP_LEAVE_BOOL(fmt, ...) \ @@ -383,6 +420,8 @@ FSP_IOPREP_DISPATCH FspFsvolWritePrepare; FSP_IOCMPL_DISPATCH FspFsvolWriteComplete; /* fast I/O and resource acquisition callbacks */ +FAST_IO_READ FspFastIoRead; +FAST_IO_WRITE FspFastIoWrite; FAST_IO_QUERY_BASIC_INFO FspFastIoQueryBasicInfo; FAST_IO_QUERY_STANDARD_INFO FspFastIoQueryStandardInfo; FAST_IO_QUERY_NETWORK_OPEN_INFO FspFastIoQueryNetworkOpenInfo; @@ -1360,6 +1399,8 @@ NTSTATUS FspMupRegister( PDEVICE_OBJECT FsmupDeviceObject, PDEVICE_OBJECT FsvolDeviceObject); VOID FspMupUnregister( PDEVICE_OBJECT FsmupDeviceObject, PDEVICE_OBJECT FsvolDeviceObject); +PDEVICE_OBJECT FspMupGetFsvolDeviceObject( + PFILE_OBJECT FileObject); NTSTATUS FspMupHandleIrp( PDEVICE_OBJECT FsmupDeviceObject, PIRP Irp); diff --git a/src/sys/file.c b/src/sys/file.c index 23bdfda6..1c07c7c6 100644 --- a/src/sys/file.c +++ b/src/sys/file.c @@ -342,7 +342,7 @@ NTSTATUS FspFileNodeCreate(PDEVICE_OBJECT DeviceObject, RtlZeroMemory(FileNode, sizeof *FileNode + ExtraSize); FileNode->Header.NodeTypeCode = FspFileNodeFileKind; FileNode->Header.NodeByteSize = sizeof *FileNode; - FileNode->Header.IsFastIoPossible = FastIoIsNotPossible; + FileNode->Header.IsFastIoPossible = FastIoIsQuestionable; FileNode->Header.Resource = &NonPaged->Resource; FileNode->Header.PagingIoResource = &NonPaged->PagingIoResource; FileNode->Header.ValidDataLength.QuadPart = MAXLONGLONG; diff --git a/src/sys/mup.c b/src/sys/mup.c index 49d70d23..30fb5655 100644 --- a/src/sys/mup.c +++ b/src/sys/mup.c @@ -43,6 +43,8 @@ NTSTATUS FspMupRegister( PDEVICE_OBJECT FsmupDeviceObject, PDEVICE_OBJECT FsvolDeviceObject); VOID FspMupUnregister( PDEVICE_OBJECT FsmupDeviceObject, PDEVICE_OBJECT FsvolDeviceObject); +PDEVICE_OBJECT FspMupGetFsvolDeviceObject( + PFILE_OBJECT FileObject); NTSTATUS FspMupHandleIrp( PDEVICE_OBJECT FsmupDeviceObject, PIRP Irp); static NTSTATUS FspMupRedirQueryPathEx( @@ -52,6 +54,7 @@ static NTSTATUS FspMupRedirQueryPathEx( #pragma alloc_text(PAGE, FspMupGetClassName) #pragma alloc_text(PAGE, FspMupRegister) #pragma alloc_text(PAGE, FspMupUnregister) +#pragma alloc_text(PAGE, FspMupGetFsvolDeviceObject) #pragma alloc_text(PAGE, FspMupHandleIrp) #pragma alloc_text(PAGE, FspMupRedirQueryPathEx) #endif @@ -191,6 +194,27 @@ VOID FspMupUnregister( ExReleaseResourceLite(&FsmupDeviceExtension->PrefixTableResource); } +PDEVICE_OBJECT FspMupGetFsvolDeviceObject( + PFILE_OBJECT FileObject) +{ + PAGED_CODE(); + + PDEVICE_OBJECT FsvolDeviceObject = 0; + + ASSERT(0 != FileObject); + + if (FspFileNodeIsValid(FileObject->FsContext)) + FsvolDeviceObject = ((FSP_FILE_NODE*)FileObject->FsContext)->FsvolDeviceObject; + else if (0 != FileObject->FsContext2 && +#pragma prefast(disable:28175, "We are a filesystem: ok to access DeviceObject->Type") + IO_TYPE_DEVICE == ((PDEVICE_OBJECT)FileObject->FsContext2)->Type && + 0 != ((PDEVICE_OBJECT)FileObject->FsContext2)->DeviceExtension && + FspFsvolDeviceExtensionKind == FspDeviceExtension((PDEVICE_OBJECT)FileObject->FsContext2)->Kind) + FsvolDeviceObject = (PDEVICE_OBJECT)FileObject->FsContext2; + + return FsvolDeviceObject; +} + NTSTATUS FspMupHandleIrp( PDEVICE_OBJECT FsmupDeviceObject, PIRP Irp) { @@ -276,7 +300,7 @@ NTSTATUS FspMupHandleIrp( FsvolDeviceObject = ((FSP_FILE_NODE *)FileObject->FsContext)->FsvolDeviceObject; else if (0 != FileObject->FsContext2 && #pragma prefast(disable:28175, "We are a filesystem: ok to access DeviceObject->Type") - 3 == ((PDEVICE_OBJECT)FileObject->FsContext2)->Type && + IO_TYPE_DEVICE == ((PDEVICE_OBJECT)FileObject->FsContext2)->Type && 0 != ((PDEVICE_OBJECT)FileObject->FsContext2)->DeviceExtension && FspFsvolDeviceExtensionKind == FspDeviceExtension((PDEVICE_OBJECT)FileObject->FsContext2)->Kind) FsvolDeviceObject = (PDEVICE_OBJECT)FileObject->FsContext2; diff --git a/src/sys/read.c b/src/sys/read.c index 25dfbca1..38ade58d 100644 --- a/src/sys/read.c +++ b/src/sys/read.c @@ -21,6 +21,7 @@ #include +FAST_IO_READ FspFastIoRead; static NTSTATUS FspFsvolRead( PDEVICE_OBJECT FsvolDeviceObject, PIRP Irp, PIO_STACK_LOCATION IrpSp); static NTSTATUS FspFsvolReadCached( @@ -35,6 +36,7 @@ static FSP_IOP_REQUEST_FINI FspFsvolReadNonCachedRequestFini; FSP_DRIVER_DISPATCH FspRead; #ifdef ALLOC_PRAGMA +#pragma alloc_text(PAGE, FspFastIoRead) #pragma alloc_text(PAGE, FspFsvolRead) #pragma alloc_text(PAGE, FspFsvolReadCached) #pragma alloc_text(PAGE, FspFsvolReadNonCached) @@ -55,6 +57,107 @@ enum }; FSP_FSCTL_STATIC_ASSERT(RequestCookie == RequestSafeMdl, ""); +BOOLEAN FspFastIoRead( + PFILE_OBJECT FileObject, + PLARGE_INTEGER ByteOffset, + ULONG Length, + BOOLEAN CanWait, + ULONG Key, + PVOID UserBuffer, + PIO_STATUS_BLOCK IoStatus, + PDEVICE_OBJECT DeviceObject) +{ + FSP_ENTER_FIO(PAGED_CODE()); + + IoStatus->Status = STATUS_SUCCESS; + IoStatus->Information = 0; + + if (!FspFileNodeIsValid(FileObject->FsContext)) + { + IoStatus->Status = STATUS_INVALID_DEVICE_REQUEST; + FSP_RETURN(Result = TRUE); + } + + FSP_FILE_NODE* FileNode = FileObject->FsContext; + LARGE_INTEGER ReadOffset = *ByteOffset; + ULONG ReadLength = Length; + FSP_FSCTL_FILE_INFO FileInfo; + + /* only regular files can be read */ + if (FileNode->IsDirectory) + { + IoStatus->Status = STATUS_INVALID_PARAMETER; + FSP_RETURN(Result = TRUE); + } + + /* do we have anything to read? */ + if (0 == ReadLength) + FSP_RETURN(Result = TRUE); + + /* does the file support caching? */ + if (!FlagOn(FileObject->Flags, FO_CACHE_SUPPORTED)) + FSP_RETURN(Result = FALSE); + + /* try to acquire the FileNode Main shared */ + Result = DEBUGTEST(90) && + FspFileNodeTryAcquireSharedF(FileNode, FspFileNodeAcquireMain, CanWait); + if (!Result) + FSP_RETURN(Result = FALSE); + + /* is the file actually cached? */ + if (0 == FileObject->PrivateCacheMap) + { + FspFileNodeRelease(FileNode, Main); + FSP_RETURN(Result = FALSE); + } + + /* does the file have oplocks or file locks? */ + Result = + FsRtlOplockIsFastIoPossible(FspFileNodeAddrOfOplock(FileNode)) && + !FsRtlAreThereCurrentFileLocks(&FileNode->FileLock); + if (!Result) + { + FspFileNodeRelease(FileNode, Main); + FSP_RETURN(Result = FALSE); + } + + /* trim ReadLength; the cache manager does not tolerate reads beyond file size */ + ASSERT(FspTimeoutInfinity32 == + FspFsvolDeviceExtension(FsvolDeviceObject)->VolumeParams.FileInfoTimeout); + FspFileNodeGetFileInfo(FileNode, &FileInfo); + if ((UINT64)ReadOffset.QuadPart >= FileInfo.FileSize) + { + FspFileNodeRelease(FileNode, Main); + IoStatus->Status = STATUS_END_OF_FILE; + FSP_RETURN(Result = TRUE); + } + if ((UINT64)ReadLength > FileInfo.FileSize - ReadOffset.QuadPart) + ReadLength = (ULONG)(FileInfo.FileSize - ReadOffset.QuadPart); + + NTSTATUS Result0 = + FspCcCopyRead(FileObject, &ReadOffset, ReadLength, CanWait, UserBuffer, IoStatus); + if (!NT_SUCCESS(Result0)) + { + IoStatus->Status = Result0; + IoStatus->Information = 0; + FspFileNodeRelease(FileNode, Main); + FSP_RETURN(Result = TRUE); + } + Result = STATUS_SUCCESS == Result0; + + SetFlag(FileObject->Flags, FO_FILE_FAST_IO_READ); + if (Result) + FileObject->CurrentByteOffset.QuadPart = ReadOffset.QuadPart + ReadLength; + + FspFileNodeRelease(FileNode, Main); + + FSP_LEAVE_FIO( + "FileObject=%p, UserBuffer=%p, CanWait=%d, " + "Key=%#lx, ByteOffset=%#lx:%#lx, Length=%ld", + FileObject, UserBuffer, CanWait, + Key, ByteOffset->HighPart, ByteOffset->LowPart, Length); +} + static NTSTATUS FspFsvolRead( PDEVICE_OBJECT FsvolDeviceObject, PIRP Irp, PIO_STACK_LOCATION IrpSp) { diff --git a/src/sys/write.c b/src/sys/write.c index e98bdfdf..ccba56a1 100644 --- a/src/sys/write.c +++ b/src/sys/write.c @@ -21,6 +21,7 @@ #include +FAST_IO_WRITE FspFastIoWrite; static NTSTATUS FspFsvolWrite( PDEVICE_OBJECT FsvolDeviceObject, PIRP Irp, PIO_STACK_LOCATION IrpSp); static NTSTATUS FspFsvolWriteCached( @@ -36,6 +37,7 @@ static FSP_IOP_REQUEST_FINI FspFsvolWriteNonCachedRequestFini; FSP_DRIVER_DISPATCH FspWrite; #ifdef ALLOC_PRAGMA +#pragma alloc_text(PAGE, FspFastIoWrite) #pragma alloc_text(PAGE, FspFsvolWrite) #pragma alloc_text(PAGE, FspFsvolWriteCached) #pragma alloc_text(PAGE, FspFsvolWriteNonCached) @@ -56,6 +58,129 @@ enum }; FSP_FSCTL_STATIC_ASSERT(RequestCookie == RequestSafeMdl, ""); +BOOLEAN FspFastIoWrite( + PFILE_OBJECT FileObject, + PLARGE_INTEGER ByteOffset, + ULONG Length, + BOOLEAN CanWait, + ULONG Key, + PVOID UserBuffer, + PIO_STATUS_BLOCK IoStatus, + PDEVICE_OBJECT DeviceObject) +{ + FSP_ENTER_FIO(PAGED_CODE()); + + IoStatus->Status = STATUS_SUCCESS; + IoStatus->Information = 0; + + if (!FspFileNodeIsValid(FileObject->FsContext)) + { + IoStatus->Status = STATUS_INVALID_DEVICE_REQUEST; + FSP_RETURN(Result = TRUE); + } + + FSP_FILE_NODE* FileNode = FileObject->FsContext; + LARGE_INTEGER WriteOffset = *ByteOffset; + ULONG WriteLength = Length; + BOOLEAN WriteToEndOfFile = + FILE_WRITE_TO_END_OF_FILE == WriteOffset.LowPart && -1L == WriteOffset.HighPart; + FSP_FSCTL_FILE_INFO FileInfo; + UINT64 WriteEndOffset; + BOOLEAN ExtendingFile; + + /* only regular files can be written */ + if (FileNode->IsDirectory) + { + IoStatus->Status = STATUS_INVALID_PARAMETER; + FSP_RETURN(Result = TRUE); + } + + /* do we have anything to write? */ + if (0 == WriteLength) + FSP_RETURN(Result = TRUE); + + /* WinFsp cannot do Fast I/O when extending file */ + if (WriteToEndOfFile) + FSP_RETURN(Result = FALSE); + + /* does the file support caching? is it write-through? */ + if (!FlagOn(FileObject->Flags, FO_CACHE_SUPPORTED) || + FlagOn(FileObject->Flags, FO_WRITE_THROUGH)) + FSP_RETURN(Result = FALSE); + + /* can we write without flushing? */ + Result = DEBUGTEST(90) && + CcCanIWrite(FileObject, WriteLength, CanWait, FALSE) && + CcCopyWriteWontFlush(FileObject, &WriteOffset, WriteLength); + if (!Result) + FSP_RETURN(Result = FALSE); + + /* try to acquire the FileNode Main exclusive */ + Result = DEBUGTEST(90) && + FspFileNodeTryAcquireExclusiveF(FileNode, FspFileNodeAcquireMain, CanWait); + if (!Result) + FSP_RETURN(Result = FALSE); + + /* is the file actually cached? */ + if (0 == FileObject->PrivateCacheMap) + { + FspFileNodeRelease(FileNode, Main); + FSP_RETURN(Result = FALSE); + } + + /* does the file have oplocks or file locks? */ + Result = + FsRtlOplockIsFastIoPossible(FspFileNodeAddrOfOplock(FileNode)) && + !FsRtlAreThereCurrentFileLocks(&FileNode->FileLock); + if (!Result) + { + FspFileNodeRelease(FileNode, Main); + FSP_RETURN(Result = FALSE); + } + + /* compute new file size */ + ASSERT(FspTimeoutInfinity32 == + FspFsvolDeviceExtension(FsvolDeviceObject)->VolumeParams.FileInfoTimeout); + FspFileNodeGetFileInfo(FileNode, &FileInfo); + if (WriteToEndOfFile) + WriteOffset.QuadPart = FileInfo.FileSize; + WriteEndOffset = WriteOffset.QuadPart + WriteLength; + ExtendingFile = FileInfo.FileSize < WriteEndOffset; + + /* WinFsp cannot do Fast I/O when extending file */ + if (ExtendingFile) + { + FspFileNodeRelease(FileNode, Main); + FSP_RETURN(Result = FALSE); + } + + NTSTATUS Result0 = + FspCcCopyWrite(FileObject, &WriteOffset, WriteLength, CanWait, UserBuffer); + if (!NT_SUCCESS(Result0)) + { + IoStatus->Status = Result0; + IoStatus->Information = 0; + FspFileNodeRelease(FileNode, Main); + FSP_RETURN(Result = FALSE); + } + Result = STATUS_SUCCESS == Result0; + + if (Result) + { + SetFlag(FileObject->Flags, FO_FILE_MODIFIED); + FileObject->CurrentByteOffset.QuadPart = WriteEndOffset; + IoStatus->Information = WriteLength; + } + + FspFileNodeRelease(FileNode, Main); + + FSP_LEAVE_FIO( + "FileObject=%p, UserBuffer=%p, CanWait=%d, " + "Key=%#lx, ByteOffset=%#lx:%#lx, Length=%ld", + FileObject, UserBuffer, CanWait, + Key, ByteOffset->HighPart, ByteOffset->LowPart, Length); +} + static NTSTATUS FspFsvolWrite( PDEVICE_OBJECT FsvolDeviceObject, PIRP Irp, PIO_STACK_LOCATION IrpSp) {