sys: ioq: now includes a dictionary for fast translation of IrpHint to Irp

build/VStudio/winfsp_sys.vcxproj

@@ -152,6 +152,7 @@
     <ClCompile Include="..\..\src\sys\debug.c" />
     <ClCompile Include="..\..\src\sys\devctl.c" />
     <ClCompile Include="..\..\src\sys\device.c" />
+    <ClCompile Include="..\..\src\sys\dict.c" />
     <ClCompile Include="..\..\src\sys\dirctl.c" />
     <ClCompile Include="..\..\src\sys\driver.c" />
     <ClCompile Include="..\..\src\sys\ea.c" />

build/VStudio/winfsp_sys.vcxproj.filters

@@ -86,6 +86,9 @@
     <ClCompile Include="..\..\src\sys\filectx.c">
       <Filter>Source</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\src\sys\dict.c">
+      <Filter>Source</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\..\src\sys\driver.h">

src/sys/device.c

@@ -13,7 +13,7 @@ NTSTATUS FspDeviceCreateSecure(UINT32 Kind, ULONG ExtraSize,
 NTSTATUS FspDeviceCreate(UINT32 Kind, ULONG ExtraSize,
     DEVICE_TYPE DeviceType,
     PDEVICE_OBJECT *PDeviceObject);
-VOID FspDeviceInitComplete(PDEVICE_OBJECT DeviceObject);
+NTSTATUS FspDeviceInitialize(PDEVICE_OBJECT DeviceObject);
 VOID FspDeviceDelete(PDEVICE_OBJECT DeviceObject);
 BOOLEAN FspDeviceRetain(PDEVICE_OBJECT DeviceObject);
 VOID FspDeviceRelease(PDEVICE_OBJECT DeviceObject);
@@ -22,7 +22,6 @@ static BOOLEAN FspDeviceRetainAtDpcLevel(PDEVICE_OBJECT DeviceObject);
 _IRQL_requires_(DISPATCH_LEVEL)
 static VOID FspDeviceReleaseFromDpcLevel(PDEVICE_OBJECT DeviceObject);
 static NTSTATUS FspFsvolDeviceInit(PDEVICE_OBJECT DeviceObject);
-static VOID FspFsvolDeviceInitComplete(PDEVICE_OBJECT DeviceObject);
 static VOID FspFsvolDeviceFini(PDEVICE_OBJECT DeviceObject);
 static IO_TIMER_ROUTINE FspFsvolDeviceTimerRoutine;
 static WORKER_THREAD_ROUTINE FspFsvolDeviceExpirationRoutine;
@@ -49,10 +48,9 @@ VOID FspDeviceDeleteAll(VOID);
 #ifdef ALLOC_PRAGMA
 #pragma alloc_text(PAGE, FspDeviceCreateSecure)
 #pragma alloc_text(PAGE, FspDeviceCreate)
-#pragma alloc_text(PAGE, FspDeviceInitComplete)
+#pragma alloc_text(PAGE, FspDeviceInitialize)
 #pragma alloc_text(PAGE, FspDeviceDelete)
 #pragma alloc_text(PAGE, FspFsvolDeviceInit)
-#pragma alloc_text(PAGE, FspFsvolDeviceInitComplete)
 #pragma alloc_text(PAGE, FspFsvolDeviceFini)
 #pragma alloc_text(PAGE, FspFsvolDeviceLockContext)
 #pragma alloc_text(PAGE, FspFsvolDeviceUnlockContext)
@@ -79,6 +77,8 @@ NTSTATUS FspDeviceCreateSecure(UINT32 Kind, ULONG ExtraSize,
     PDEVICE_OBJECT DeviceObject;
     FSP_DEVICE_EXTENSION *DeviceExtension;
 
+    *PDeviceObject = 0;
+
     switch (Kind)
     {
     case FspFsvolDeviceExtensionKind:
@@ -112,20 +112,7 @@ NTSTATUS FspDeviceCreateSecure(UINT32 Kind, ULONG ExtraSize,
     DeviceExtension->RefCount = 1;
     DeviceExtension->Kind = Kind;
 
-    switch (Kind)
+    *PDeviceObject = DeviceObject;
-    {
-    case FspFsvolDeviceExtensionKind:
-        Result = FspFsvolDeviceInit(DeviceObject);
-        break;
-    case FspFsvrtDeviceExtensionKind:
-    case FspFsctlDeviceExtensionKind:
-        break;
-    }
-
-    if (!NT_SUCCESS(Result))
-        IoDeleteDevice(DeviceObject);
-    else
-        *PDeviceObject = DeviceObject;
 
     return Result;
 }
@@ -139,26 +126,31 @@ NTSTATUS FspDeviceCreate(UINT32 Kind, ULONG ExtraSize,
     return FspDeviceCreateSecure(Kind, ExtraSize, 0, DeviceType, 0, 0, PDeviceObject);
 }
 
-VOID FspDeviceInitComplete(PDEVICE_OBJECT DeviceObject)
+NTSTATUS FspDeviceInitialize(PDEVICE_OBJECT DeviceObject)
 {
     PAGED_CODE();
 
+    NTSTATUS Result;
     FSP_DEVICE_EXTENSION *DeviceExtension = FspDeviceExtension(DeviceObject);
 
     switch (DeviceExtension->Kind)
     {
     case FspFsvolDeviceExtensionKind:
-        FspFsvolDeviceInitComplete(DeviceObject);
+        Result = FspFsvolDeviceInit(DeviceObject);
         break;
     case FspFsvrtDeviceExtensionKind:
     case FspFsctlDeviceExtensionKind:
+        Result = STATUS_SUCCESS;
         break;
     default:
         ASSERT(0);
-        return;
+        return STATUS_INVALID_PARAMETER;
     }
 
-    ClearFlag(DeviceObject->Flags, DO_DEVICE_INITIALIZING);
+    if (NT_SUCCESS(Result))
+        ClearFlag(DeviceObject->Flags, DO_DEVICE_INITIALIZING);
+
+    return Result;
 }
 
 VOID FspDeviceDelete(PDEVICE_OBJECT DeviceObject)
@@ -266,64 +258,64 @@ static NTSTATUS FspFsvolDeviceInit(PDEVICE_OBJECT DeviceObject)
 
     NTSTATUS Result;
     FSP_FSVOL_DEVICE_EXTENSION *FsvolDeviceExtension = FspFsvolDeviceExtension(DeviceObject);
+    LARGE_INTEGER IrpTimeout;
 
-    /* initialize our timer routine */
+    /*
-#pragma prefast(suppress:28133, "We are a filesystem: we do not have AddDevice")
+     * Volume device initialization is a mess, because of the different ways of
-    Result = IoInitializeTimer(DeviceObject, FspFsvolDeviceTimerRoutine, 0);
+     * creating/initializing different resources. So we will use some bits just
-    if (!NT_SUCCESS(Result))
+     * to track what has been initialized!
-        return Result;
+     */
 
-    /* allocate a spare VPB in case we are mounted on a virtual disk */
+    /* is there a virtual disk? */
-    if (FILE_DEVICE_DISK_FILE_SYSTEM == DeviceObject->DeviceType)
+    if (0 != FsvolDeviceExtension->FsvrtDeviceObject)
     {
+        /* allocate a spare VPB so that we can be mounted on the virtual disk */
         FsvolDeviceExtension->SwapVpb = FspAllocNonPagedExternal(sizeof *FsvolDeviceExtension->SwapVpb);
         if (0 == FsvolDeviceExtension->SwapVpb)
             return STATUS_INSUFFICIENT_RESOURCES;
         RtlZeroMemory(FsvolDeviceExtension->SwapVpb, sizeof *FsvolDeviceExtension->SwapVpb);
+
+        /* reference the virtual disk device so that it will not go away while we are using it */
+        ObReferenceObject(FsvolDeviceExtension->FsvrtDeviceObject);
+        FsvolDeviceExtension->InitDoneFsvrt = 1;
     }
 
     /* initialize our delete lock */
     ExInitializeResourceLite(&FsvolDeviceExtension->DeleteResource);
+    FsvolDeviceExtension->InitDoneDelRsc = 1;
+
+    /* setup our Ioq */
+    IrpTimeout.QuadPart = FsvolDeviceExtension->VolumeParams.IrpTimeout * 10000;
+        /* convert millis to nanos */
+    Result = FspIoqCreate(
+        FsvolDeviceExtension->VolumeParams.IrpCapacity, &IrpTimeout, FspIopCompleteCanceledIrp,
+        &FsvolDeviceExtension->Ioq);
+    if (!NT_SUCCESS(Result))
+        return Result;
+    FsvolDeviceExtension->InitDoneIoq = 1;
 
     /* initialize our generic table */
     ExInitializeFastMutex(&FsvolDeviceExtension->GenericTableFastMutex);
     RtlInitializeGenericTableAvl(&FsvolDeviceExtension->GenericTable,
         FspFsvolDeviceCompareElement, FspFsvolDeviceAllocateElement, FspFsvolDeviceFreeElement, 0);
+    FsvolDeviceExtension->InitDoneGenTab = 1;
 
     /* initialize the volume name buffer */
     RtlInitEmptyUnicodeString(&FsvolDeviceExtension->VolumeName,
         FsvolDeviceExtension->VolumeNameBuf, sizeof FsvolDeviceExtension->VolumeNameBuf);
 
-    return STATUS_SUCCESS;
+    /* initialize our timer routine and start our expiration timer */
-}
+#pragma prefast(suppress:28133, "We are a filesystem: we do not have AddDevice")
-
+    Result = IoInitializeTimer(DeviceObject, FspFsvolDeviceTimerRoutine, 0);
-static VOID FspFsvolDeviceInitComplete(PDEVICE_OBJECT DeviceObject)
+    if (!NT_SUCCESS(Result))
-{
+        return Result;
-    PAGED_CODE();
-
-    FSP_FSVOL_DEVICE_EXTENSION *FsvolDeviceExtension = FspFsvolDeviceExtension(DeviceObject);
-    LARGE_INTEGER IrpTimeout;
-
-    /*
-     * Setup our Ioq and expiration fields.
-     * We must do this in InitComplete because Ioq initialization depends on VolumeParams.
-     */
-    IrpTimeout.QuadPart = FsvolDeviceExtension->VolumeParams.IrpTimeout * 10000;
-        /* convert millis to nanos */
-    FspIoqInitialize(&FsvolDeviceExtension->Ioq,
-        &IrpTimeout, FsvolDeviceExtension->VolumeParams.IrpCapacity, FspIopCompleteCanceledIrp);
     KeInitializeSpinLock(&FsvolDeviceExtension->ExpirationLock);
     ExInitializeWorkItem(&FsvolDeviceExtension->ExpirationWorkItem,
         FspFsvolDeviceExpirationRoutine, DeviceObject);
-
-    /*
-     * Reference the virtual volume device so that it will not go away while we are using it.
-     */
-    if (0 != FsvolDeviceExtension->FsvrtDeviceObject)
-        ObReferenceObject(FsvolDeviceExtension->FsvrtDeviceObject);
-
-    /* start our expiration timer */
     IoStartTimer(DeviceObject);
+    FsvolDeviceExtension->InitDoneTimer = 1;
+
+    return STATUS_SUCCESS;
 }
 
 static VOID FspFsvolDeviceFini(PDEVICE_OBJECT DeviceObject)
@@ -339,7 +331,8 @@ static VOID FspFsvolDeviceFini(PDEVICE_OBJECT DeviceObject)
      * However a work item may be in flight. For this reason our IoTimer routine
      * retains our DeviceObject before queueing work items.
      */
-    IoStopTimer(DeviceObject);
+    if (FsvolDeviceExtension->InitDoneTimer)
+        IoStopTimer(DeviceObject);
 
 #if 0
     /* FspDeviceFreeElement is now a no-op, so this is no longer necessary */
@@ -347,23 +340,31 @@ static VOID FspFsvolDeviceFini(PDEVICE_OBJECT DeviceObject)
      * Enumerate and delete all entries in the GenericTable.
      * There is no need to protect accesses to the table as we are in the device destructor.
      */
-    FSP_DEVICE_GENERIC_TABLE_ELEMENT_DATA *Element;
+    if (FsvolDeviceExtension->InitDoneGenTab)
-    while (0 != (Element = RtlGetElementGenericTableAvl(&FsvolDeviceExtension->GenericTable, 0)))
+    {
-        RtlDeleteElementGenericTableAvl(&FsvolDeviceExtension->GenericTable, &Element->Identifier);
+        FSP_DEVICE_GENERIC_TABLE_ELEMENT_DATA *Element;
+        while (0 != (Element = RtlGetElementGenericTableAvl(&FsvolDeviceExtension->GenericTable, 0)))
+            RtlDeleteElementGenericTableAvl(&FsvolDeviceExtension->GenericTable, &Element->Identifier);
+    }
 #endif
 
-    /*
+    if (FsvolDeviceExtension->InitDoneIoq)
-     * Dereference the virtual volume device so that it can now go away.
+        FspIoqDelete(FsvolDeviceExtension->Ioq);
-     */
-    if (0 != FsvolDeviceExtension->FsvrtDeviceObject)
-        ObDereferenceObject(FsvolDeviceExtension->FsvrtDeviceObject);
 
     /* finalize our delete lock */
-    ExDeleteResourceLite(&FsvolDeviceExtension->DeleteResource);
+    if (FsvolDeviceExtension->InitDoneDelRsc)
+        ExDeleteResourceLite(&FsvolDeviceExtension->DeleteResource);
 
-    /* free the spare VPB if we still have it */
+    /* is there a virtual disk? */
-    if (0 != FsvolDeviceExtension->SwapVpb)
+    if (FsvolDeviceExtension->InitDoneFsvrt)
-        FspFreeExternal(FsvolDeviceExtension->SwapVpb);
+    {
+        /* dereference the virtual volume device so that it can now go away */
+        ObDereferenceObject(FsvolDeviceExtension->FsvrtDeviceObject);
+
+        /* free the spare VPB if we still have it */
+        if (0 != FsvolDeviceExtension->SwapVpb)
+            FspFreeExternal(FsvolDeviceExtension->SwapVpb);
+    }
 }
 
 static VOID FspFsvolDeviceTimerRoutine(PDEVICE_OBJECT DeviceObject, PVOID Context)
@@ -405,7 +406,7 @@ static VOID FspFsvolDeviceExpirationRoutine(PVOID Context)
     FSP_FSVOL_DEVICE_EXTENSION *FsvolDeviceExtension = FspFsvolDeviceExtension(DeviceObject);
     KIRQL Irql;
 
-    FspIoqRemoveExpired(&FsvolDeviceExtension->Ioq);
+    FspIoqRemoveExpired(FsvolDeviceExtension->Ioq);
 
     KeAcquireSpinLock(&FsvolDeviceExtension->ExpirationLock, &Irql);
     FsvolDeviceExtension->ExpirationInProgress = FALSE;

src/sys/dict.c

@@ -0,0 +1,47 @@
+/**
+ * @file sys/dict.c
+ *
+ * @copyright 2015 Bill Zissimopoulos
+ */
+
+#include <sys/driver.h>
+
+VOID FspDictInitialize(FSP_DICT *Dict,
+    FSP_DICT_EQUAL_FUNCTION *EqualFunction, FSP_DICT_HASH_FUNCTION *HashFunction,
+    FSP_DICT_ENTRY **Buckets, ULONG BucketCount)
+{
+    RtlZeroMemory(Buckets, BucketCount * sizeof Buckets[0]);
+    Dict->EqualFunction = EqualFunction;
+    Dict->HashFunction = HashFunction;
+    Dict->Buckets = Buckets;
+    Dict->BucketCount = BucketCount;
+}
+
+FSP_DICT_ENTRY *FspDictGetEntry(FSP_DICT *Dict, PVOID Key)
+{
+    ULONG Index = Dict->HashFunction(Key) % Dict->BucketCount;
+    for (FSP_DICT_ENTRY *Entry = Dict->Buckets[Index]; Entry; Entry = Entry->Next)
+        if (Dict->EqualFunction(Key, Entry->Key))
+            return Entry;
+    return 0;
+}
+
+VOID FspDictSetEntry(FSP_DICT *Dict, FSP_DICT_ENTRY *Entry)
+{
+    ULONG Index = Dict->HashFunction(Entry->Key) % Dict->BucketCount;
+    Entry->Next = Dict->Buckets[Index];
+    Dict->Buckets[Index] = Entry;
+}
+
+FSP_DICT_ENTRY *FspDictRemoveEntry(FSP_DICT *Dict, PVOID Key)
+{
+    ULONG Index = Dict->HashFunction(Key) % Dict->BucketCount;
+    for (FSP_DICT_ENTRY **PEntry = &Dict->Buckets[Index]; *PEntry; PEntry = &(*PEntry)->Next)
+        if (Dict->EqualFunction(Key, (*PEntry)->Key))
+        {
+            FSP_DICT_ENTRY *Entry = *PEntry;
+            *PEntry = Entry->Next;
+            return Entry;
+        }
+    return 0;
+}

src/sys/driver.c

@@ -39,8 +39,10 @@ NTSTATUS DriverEntry(
         &FspFsctlNetDeviceObject);
     if (!NT_SUCCESS(Result))
         FSP_RETURN(FspDeviceDelete(FspFsctlDiskDeviceObject));
-    FspDeviceInitComplete(FspFsctlDiskDeviceObject);
+    Result = FspDeviceInitialize(FspFsctlDiskDeviceObject);
-    FspDeviceInitComplete(FspFsctlNetDeviceObject);
+    ASSERT(STATUS_SUCCESS == Result);
+    Result = FspDeviceInitialize(FspFsctlNetDeviceObject);
+    ASSERT(STATUS_SUCCESS == Result);
 
     /* setup the driver object */
     DriverObject->DriverUnload = FspUnload;

src/sys/driver.h

@@ -127,7 +127,7 @@ extern __declspec(selectany) int bpglobal = 1;
                 /* if the IRP has not been marked pending already */\
                 FSP_FSVOL_DEVICE_EXTENSION *FsvolDeviceExtension =\
                     FspFsvolDeviceExtension(DeviceObject);\
-                if (!FspIoqPostIrp(&FsvolDeviceExtension->Ioq, Irp, &Result))\
+                if (!FspIoqPostIrp(FsvolDeviceExtension->Ioq, Irp, &Result))\
                     FspIopCompleteIrp(Irp, Result);\
             }                           \
         }                               \
@@ -291,12 +291,38 @@ VOID FspInitializeDelayedWorkItem(FSP_DELAYED_WORK_ITEM *DelayedWorkItem,
     PWORKER_THREAD_ROUTINE Routine, PVOID Context);
 VOID FspQueueDelayedWorkItem(FSP_DELAYED_WORK_ITEM *DelayedWorkItem, LARGE_INTEGER Delay);
 
+/* dictionary */
+typedef BOOLEAN FSP_DICT_EQUAL_FUNCTION(PVOID Key1, PVOID Key2);
+typedef ULONG FSP_DICT_HASH_FUNCTION(PVOID Key);
+typedef struct _FSP_DICT_ENTRY
+{
+    PVOID Key, Value;
+    struct _FSP_DICT_ENTRY *Next;
+} FSP_DICT_ENTRY;
+typedef struct _FSP_DICT
+{
+    FSP_DICT_EQUAL_FUNCTION *EqualFunction;
+    FSP_DICT_HASH_FUNCTION *HashFunction;
+    FSP_DICT_ENTRY **Buckets;
+    ULONG BucketCount;
+} FSP_DICT;
+VOID FspDictInitialize(FSP_DICT *Dict,
+    FSP_DICT_EQUAL_FUNCTION *EqualFunction, FSP_DICT_HASH_FUNCTION *HashFunction,
+    FSP_DICT_ENTRY **Buckets, ULONG BucketCount);
+FSP_DICT_ENTRY *FspDictGetEntry(FSP_DICT *Dict, PVOID Key);
+VOID FspDictSetEntry(FSP_DICT *Dict, FSP_DICT_ENTRY *Entry);
+FSP_DICT_ENTRY *FspDictRemoveEntry(FSP_DICT *Dict, PVOID Key);
+
 /* IRP context */
-#define FspIrpTimestamp(Irp)            \
+typedef struct
-    (*(ULONGLONG *)&(Irp)->Tail.Overlay.DriverContext[0])
+{
-    /* FspIrpTimestamp() uses up DriverContext[0] and [1] in 32-bit builds */
+    IO_CSQ_IRP_CONTEXT IoCsqIrpContext;
+    ULONGLONG ExpirationTime;
+} FSP_IRP_CONTEXT;
+#define FspIrpContext(Irp)              \
+    (*(FSP_IRP_CONTEXT **)&(Irp)->Tail.Overlay.DriverContext[0])
 #define FspIrpRequest(Irp)              \
-    (*(FSP_FSCTL_TRANSACT_REQ **)&(Irp)->Tail.Overlay.DriverContext[2])
+    (*(FSP_FSCTL_TRANSACT_REQ **)&(Irp)->Tail.Overlay.DriverContext[0])
 
 /* I/O queue */
 #define FspIoqTimeout                   ((PIRP)1)
@@ -307,12 +333,16 @@ typedef struct
     KEVENT PendingIrpEvent;
     LIST_ENTRY PendingIrpList, ProcessIrpList;
     IO_CSQ PendingIoCsq, ProcessIoCsq;
+    FSP_DICT ProcessIrpDict;
     LARGE_INTEGER IrpTimeout;
     ULONG PendingIrpCapacity, PendingIrpCount;
     VOID (*CompleteCanceledIrp)(PIRP Irp);
+    FSP_DICT_ENTRY *ProcessIrpDictBuckets[];
 } FSP_IOQ;
-VOID FspIoqInitialize(FSP_IOQ *Ioq,
+NTSTATUS FspIoqCreate(
-    PLARGE_INTEGER IrpTimeout, ULONG IrpCapacity, VOID (*CompleteCanceledIrp)(PIRP Irp));
+    ULONG IrpCapacity, PLARGE_INTEGER IrpTimeout, VOID (*CompleteCanceledIrp)(PIRP Irp),
+    FSP_IOQ **PIoq);
+VOID FspIoqDelete(FSP_IOQ *Ioq);
 VOID FspIoqStop(FSP_IOQ *Ioq);
 BOOLEAN FspIoqStopped(FSP_IOQ *Ioq);
 VOID FspIoqRemoveExpired(FSP_IOQ *Ioq);
@@ -370,6 +400,7 @@ typedef struct
 typedef struct
 {
     FSP_DEVICE_EXTENSION Base;
+    UINT32 InitDoneFsvrt:1, InitDoneDelRsc:1, InitDoneIoq:1, InitDoneGenTab:1, InitDoneTimer:1;
     PDEVICE_OBJECT FsctlDeviceObject;
     PDEVICE_OBJECT FsvrtDeviceObject;
     HANDLE MupHandle;
@@ -377,7 +408,7 @@ typedef struct
     FSP_DELAYED_WORK_ITEM DeleteVolumeDelayedWorkItem;
     ERESOURCE DeleteResource;
     FSP_FSCTL_VOLUME_PARAMS VolumeParams;
-    FSP_IOQ Ioq;
+    FSP_IOQ *Ioq;
     KSPIN_LOCK ExpirationLock;
     WORK_QUEUE_ITEM ExpirationWorkItem;
     BOOLEAN ExpirationInProgress;
@@ -405,7 +436,7 @@ NTSTATUS FspDeviceCreateSecure(UINT32 Kind, ULONG ExtraSize,
 NTSTATUS FspDeviceCreate(UINT32 Kind, ULONG ExtraSize,
     DEVICE_TYPE DeviceType,
     PDEVICE_OBJECT *PDeviceObject);
-VOID FspDeviceInitComplete(PDEVICE_OBJECT DeviceObject);
+NTSTATUS FspDeviceInitialize(PDEVICE_OBJECT DeviceObject);
 VOID FspDeviceDelete(PDEVICE_OBJECT DeviceObject);
 BOOLEAN FspDeviceRetain(PDEVICE_OBJECT DeviceObject);
 VOID FspDeviceRelease(PDEVICE_OBJECT DeviceObject);

src/sys/iop.c

@@ -29,6 +29,7 @@ VOID FspIopDispatchComplete(PIRP Irp, const FSP_FSCTL_TRANSACT_RSP *Response);
 
 typedef struct
 {
+    FSP_IRP_CONTEXT IrpContext;
     FSP_IOP_REQUEST_FINI *RequestFini;
     PVOID Context[3];
     __declspec(align(MEMORY_ALLOCATION_ALIGNMENT)) UINT8 RequestBuf[];

src/sys/ioq.c

@@ -91,21 +91,19 @@ static PIRP FspIoqPendingPeekNextIrp(PIO_CSQ IoCsq, PIRP Irp, PVOID PeekContext)
         return 0;
     PLIST_ENTRY Head = &Ioq->PendingIrpList;
     PLIST_ENTRY Entry = 0 == Irp ? Head->Flink : Irp->Tail.Overlay.ListEntry.Flink;
+    if (Head == Entry)
+        return 0;
     if (!PeekContext)
-        return Head != Entry ? CONTAINING_RECORD(Entry, IRP, Tail.Overlay.ListEntry) : 0;
+        return CONTAINING_RECORD(Entry, IRP, Tail.Overlay.ListEntry);
     PVOID IrpHint = ((FSP_IOQ_PEEK_CONTEXT *)PeekContext)->IrpHint;
     if (0 == IrpHint)
     {
-        if (Head != Entry)
+        ULONGLONG ExpirationTime = ((FSP_IOQ_PEEK_CONTEXT *)PeekContext)->ExpirationTime;
-        {
+        return FspIrpContext(Irp)->ExpirationTime <= ExpirationTime ?
-            Irp = CONTAINING_RECORD(Entry, IRP, Tail.Overlay.ListEntry);
+            CONTAINING_RECORD(Entry, IRP, Tail.Overlay.ListEntry) : 0;
-            if (FspIrpTimestamp(Irp) <= ((FSP_IOQ_PEEK_CONTEXT *)PeekContext)->ExpirationTime)
-                /* IRP has expired; return it */
-                return Irp;
-        }
-        return 0;
     }
-    return Head != Entry ? CONTAINING_RECORD(Entry, IRP, Tail.Overlay.ListEntry) : 0;
+    else
+        return CONTAINING_RECORD(Entry, IRP, Tail.Overlay.ListEntry);
 }
 
 _IRQL_raises_(DISPATCH_LEVEL)
@@ -134,11 +132,19 @@ static NTSTATUS FspIoqProcessInsertIrpEx(PIO_CSQ IoCsq, PIRP Irp, PVOID InsertCo
     if (Ioq->Stopped)
         return STATUS_CANCELLED;
     InsertTailList(&Ioq->ProcessIrpList, &Irp->Tail.Overlay.ListEntry);
+    ASSERT(0 == FspDictGetEntry(&Ioq->ProcessIrpDict, Irp));
+    FSP_DICT_ENTRY DictEntry = { 0 };
+    DictEntry.Key = Irp;
+    FspDictSetEntry(&Ioq->ProcessIrpDict, &DictEntry);
     return STATUS_SUCCESS;
 }
 
 static VOID FspIoqProcessRemoveIrp(PIO_CSQ IoCsq, PIRP Irp)
 {
+    FSP_IOQ *Ioq = CONTAINING_RECORD(IoCsq, FSP_IOQ, ProcessIoCsq);
+    FSP_DICT_ENTRY *DictEntry; (VOID)DictEntry;
+    DictEntry = FspDictRemoveEntry(&Ioq->ProcessIrpDict, Irp);
+    ASSERT(0 == DictEntry);
     RemoveEntryList(&Irp->Tail.Overlay.ListEntry);
 }
 
@@ -149,27 +155,22 @@ static PIRP FspIoqProcessPeekNextIrp(PIO_CSQ IoCsq, PIRP Irp, PVOID PeekContext)
         return 0;
     PLIST_ENTRY Head = &Ioq->ProcessIrpList;
     PLIST_ENTRY Entry = 0 == Irp ? Head->Flink : Irp->Tail.Overlay.ListEntry.Flink;
+    if (Head == Entry)
+        return 0;
     if (!PeekContext)
-        return Head != Entry ? CONTAINING_RECORD(Entry, IRP, Tail.Overlay.ListEntry) : 0;
+        return CONTAINING_RECORD(Entry, IRP, Tail.Overlay.ListEntry);
     PVOID IrpHint = ((FSP_IOQ_PEEK_CONTEXT *)PeekContext)->IrpHint;
     if (0 == IrpHint)
     {
-        if (Head != Entry)
+        ULONGLONG ExpirationTime = ((FSP_IOQ_PEEK_CONTEXT *)PeekContext)->ExpirationTime;
-        {
+        return FspIrpContext(Irp)->ExpirationTime <= ExpirationTime ?
-            Irp = CONTAINING_RECORD(Entry, IRP, Tail.Overlay.ListEntry);
+            CONTAINING_RECORD(Entry, IRP, Tail.Overlay.ListEntry) : 0;
-            if (FspIrpTimestamp(Irp) <= ((FSP_IOQ_PEEK_CONTEXT *)PeekContext)->ExpirationTime)
-                /* IRP has expired; return it */
-                return Irp;
-        }
-        return 0;
     }
-    for (; Head != Entry; Entry = Entry->Flink)
+    else
     {
-        Irp = CONTAINING_RECORD(Entry, IRP, Tail.Overlay.ListEntry);
+        FSP_DICT_ENTRY *DictEntry = FspDictGetEntry(&Ioq->ProcessIrpDict, IrpHint);
-        if (Irp == IrpHint)
+        return 0 != DictEntry ? (PIRP)IrpHint : 0;
-            return Irp;
     }
-    return 0;
 }
 
 _IRQL_raises_(DISPATCH_LEVEL)
@@ -192,11 +193,28 @@ static VOID FspIoqProcessCompleteCanceledIrp(PIO_CSQ IoCsq, PIRP Irp)
     Ioq->CompleteCanceledIrp(Irp);
 }
 
-VOID FspIoqInitialize(FSP_IOQ *Ioq,
+BOOLEAN FspIoqIrpEquals(PVOID Irp1, PVOID Irp2)
-    PLARGE_INTEGER IrpTimeout, ULONG IrpCapacity, VOID (*CompleteCanceledIrp)(PIRP Irp))
+{
+    return Irp1 == Irp2;
+}
+
+ULONG FspIoqIrpHash(PVOID Irp)
+{
+    return (ULONG)(UINT_PTR)Irp;
+}
+
+NTSTATUS FspIoqCreate(
+    ULONG IrpCapacity, PLARGE_INTEGER IrpTimeout, VOID (*CompleteCanceledIrp)(PIRP Irp),
+    FSP_IOQ **PIoq)
 {
     ASSERT(0 != CompleteCanceledIrp);
 
+    ULONG BucketCount = IrpCapacity / 2;
+    FSP_IOQ *Ioq = FspAllocNonPaged(
+        sizeof *Ioq + sizeof Ioq->ProcessIrpDictBuckets[0] * BucketCount);
+    if (0 == Ioq)
+        return STATUS_INSUFFICIENT_RESOURCES;
+
     RtlZeroMemory(Ioq, sizeof *Ioq);
     KeInitializeSpinLock(&Ioq->SpinLock);
     KeInitializeEvent(&Ioq->PendingIrpEvent, NotificationEvent, FALSE);
@@ -216,9 +234,19 @@ VOID FspIoqInitialize(FSP_IOQ *Ioq,
         FspIoqProcessAcquireLock,
         FspIoqProcessReleaseLock,
         FspIoqProcessCompleteCanceledIrp);
+    FspDictInitialize(&Ioq->ProcessIrpDict,
+        FspIoqIrpEquals, FspIoqIrpHash, Ioq->ProcessIrpDictBuckets, BucketCount);
     Ioq->IrpTimeout = *IrpTimeout;
     Ioq->PendingIrpCapacity = IrpCapacity;
     Ioq->CompleteCanceledIrp = CompleteCanceledIrp;
+
+    return STATUS_SUCCESS;
+}
+
+VOID FspIoqDelete(FSP_IOQ *Ioq)
+{
+    FspIoqStop(Ioq);
+    FspFree(Ioq);
 }
 
 VOID FspIoqStop(FSP_IOQ *Ioq)
@@ -261,7 +289,7 @@ VOID FspIoqRemoveExpired(FSP_IOQ *Ioq)
 BOOLEAN FspIoqPostIrp(FSP_IOQ *Ioq, PIRP Irp, NTSTATUS *PResult)
 {
     NTSTATUS Result;
-    FspIrpTimestamp(Irp) = KeQueryInterruptTime() + Ioq->IrpTimeout.QuadPart;
+    FspIrpContext(Irp)->ExpirationTime = KeQueryInterruptTime() + Ioq->IrpTimeout.QuadPart;
     Result = IoCsqInsertIrpEx(&Ioq->PendingIoCsq, Irp, 0, (PVOID)1);
     if (NT_SUCCESS(Result))
         return TRUE;

src/sys/volume.c

@@ -158,9 +158,18 @@ NTSTATUS FspVolumeCreate(
     FsvolDeviceExtension->FsvrtDeviceObject = FsvrtDeviceObject;
     FsvolDeviceExtension->VolumeParams = VolumeParams;
     RtlCopyUnicodeString(&FsvolDeviceExtension->VolumeName, &VolumeName);
-    FspDeviceInitComplete(FsvolDeviceObject);
+    Result = FspDeviceInitialize(FsvolDeviceObject);
-    if (0 != FsvrtDeviceObject)
+    if (NT_SUCCESS(Result))
-        FspDeviceInitComplete(FsvrtDeviceObject);
+    {
+        if (0 != FsvrtDeviceObject)
+            Result = FspDeviceInitialize(FsvrtDeviceObject);
+    }
+    if (!NT_SUCCESS(Result))
+    {
+        if (0 != FsvrtDeviceObject)
+            FspDeviceRelease(FsvrtDeviceObject);
+        FspDeviceRelease(FsvolDeviceObject);
+    }
 
     /* do we need to register with MUP? */
     if (0 == FsvrtDeviceObject)
@@ -220,7 +229,7 @@ VOID FspVolumeDelete(
     ExAcquireResourceExclusiveLite(&FsvolDeviceExtension->DeleteResource, TRUE);
 
     /* stop the I/O queue */
-    FspIoqStop(&FsvolDeviceExtension->Ioq);
+    FspIoqStop(FsvolDeviceExtension->Ioq);
 
     /* do we have a virtual disk device or a MUP handle? */
     if (0 != FsvolDeviceExtension->FsvrtDeviceObject)
@@ -356,7 +365,7 @@ NTSTATUS FspVolumeMount(
                     if (FsvolDeviceExtension->FsvrtDeviceObject == FsvrtDeviceObject)
                     {
                         ExAcquireResourceExclusiveLite(&FsvolDeviceExtension->DeleteResource, TRUE);
-                        if (!FspIoqStopped(&FsvolDeviceExtension->Ioq))
+                        if (!FspIoqStopped(FsvolDeviceExtension->Ioq))
                         {
                             Result = STATUS_SUCCESS;
                             /* break out of the loop without FspDeviceRelease or DeleteResource release! */
@@ -423,7 +432,7 @@ NTSTATUS FspVolumeRedirQueryPathEx(
     ExAcquireResourceExclusiveLite(&FsvolDeviceExtension->DeleteResource, TRUE);
 
     Result = STATUS_BAD_NETWORK_PATH;
-    if (!FspIoqStopped(&FsvolDeviceExtension->Ioq))
+    if (!FspIoqStopped(FsvolDeviceExtension->Ioq))
     {
         RtlInitUnicodeString(&Prefix, FsvolDeviceExtension->VolumeParams.Prefix);
         if (Prefix.Length <= QueryPathRequest->PathName.Length &&
@@ -515,7 +524,7 @@ NTSTATUS FspVolumeTransact(
         if (0 == NextResponse)
             break;
 
-        ProcessIrp = FspIoqEndProcessingIrp(&FsvolDeviceExtension->Ioq, (UINT_PTR)Response->Hint);
+        ProcessIrp = FspIoqEndProcessingIrp(FsvolDeviceExtension->Ioq, (UINT_PTR)Response->Hint);
         if (0 == ProcessIrp)
             /* either IRP was canceled or a bogus Hint was provided */
             continue;
@@ -538,9 +547,9 @@ NTSTATUS FspVolumeTransact(
     KeQuerySystemTime(&Timeout);
     Timeout.QuadPart += FsvolDeviceExtension->VolumeParams.TransactTimeout * 10000;
         /* convert millis to nanos and add to absolute time */
-    while (0 == (PendingIrp = FspIoqNextPendingIrp(&FsvolDeviceExtension->Ioq, &Timeout)))
+    while (0 == (PendingIrp = FspIoqNextPendingIrp(FsvolDeviceExtension->Ioq, &Timeout)))
     {
-        if (FspIoqStopped(&FsvolDeviceExtension->Ioq))
+        if (FspIoqStopped(FsvolDeviceExtension->Ioq))
             return STATUS_CANCELLED;
     }
     if (FspIoqTimeout == PendingIrp)
@@ -565,7 +574,7 @@ NTSTATUS FspVolumeTransact(
             RtlCopyMemory(Request, PendingIrpRequest, PendingIrpRequest->Size);
             Request = FspFsctlTransactProduceRequest(Request, PendingIrpRequest->Size);
 
-            if (!FspIoqStartProcessingIrp(&FsvolDeviceExtension->Ioq, PendingIrp))
+            if (!FspIoqStartProcessingIrp(FsvolDeviceExtension->Ioq, PendingIrp))
             {
                 /*
                  * This can only happen if the Ioq was stopped. Abandon everything
@@ -573,7 +582,7 @@ NTSTATUS FspVolumeTransact(
                  * queues of the Ioq will be cancelled during FspIoqStop(). We must
                  * also cancel the PendingIrp we have in our hands.
                  */
-                ASSERT(FspIoqStopped(&FsvolDeviceExtension->Ioq));
+                ASSERT(FspIoqStopped(FsvolDeviceExtension->Ioq));
                 FspIopCompleteIrp(PendingIrp, STATUS_CANCELLED);
                 return STATUS_CANCELLED;
             }
@@ -583,7 +592,7 @@ NTSTATUS FspVolumeTransact(
                 break;
         }
 
-        PendingIrp = FspIoqNextPendingIrp(&FsvolDeviceExtension->Ioq, 0);
+        PendingIrp = FspIoqNextPendingIrp(FsvolDeviceExtension->Ioq, 0);
         if (0 == PendingIrp)
             break;
     }
@@ -619,7 +628,7 @@ NTSTATUS FspVolumeWork(
      * so that we can disassociate the Request on failure and release ownership
      * back to the caller.
      */
-    if (!FspIoqPostIrp(&FsvolDeviceExtension->Ioq, Irp, &Result))
+    if (!FspIoqPostIrp(FsvolDeviceExtension->Ioq, Irp, &Result))
     {
         Request->Hint = 0;
         FspIrpRequest(Irp) = 0;