sys: ioq

build/VStudio/winfsp_sys.vcxproj

@@ -152,7 +152,6 @@
     <ClCompile Include="..\..\src\sys\debug.c" />
     <ClCompile Include="..\..\src\sys\devctl.c" />
     <ClCompile Include="..\..\src\sys\device.c" />
-    <ClCompile Include="..\..\src\sys\dict.c" />
     <ClCompile Include="..\..\src\sys\dirctl.c" />
     <ClCompile Include="..\..\src\sys\driver.c" />
     <ClCompile Include="..\..\src\sys\ea.c" />

build/VStudio/winfsp_sys.vcxproj.filters

@@ -86,9 +86,6 @@
     <ClCompile Include="..\..\src\sys\filectx.c">
       <Filter>Source</Filter>
     </ClCompile>
-    <ClCompile Include="..\..\src\sys\dict.c">
-      <Filter>Source</Filter>
-    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\..\src\sys\driver.h">

src/sys/dict.c

@@ -1,47 +0,0 @@
-/**
- * @file sys/dict.c
- *
- * @copyright 2015 Bill Zissimopoulos
- */
-
-#include <sys/driver.h>
-
-VOID FspDictInitialize(FSP_DICT *Dict,
-    FSP_DICT_EQUAL_FUNCTION *EqualFunction, FSP_DICT_HASH_FUNCTION *HashFunction,
-    FSP_DICT_ENTRY **Buckets, ULONG BucketCount)
-{
-    RtlZeroMemory(Buckets, BucketCount * sizeof Buckets[0]);
-    Dict->EqualFunction = EqualFunction;
-    Dict->HashFunction = HashFunction;
-    Dict->Buckets = Buckets;
-    Dict->BucketCount = BucketCount;
-}
-
-FSP_DICT_ENTRY *FspDictGetEntry(FSP_DICT *Dict, PVOID Key)
-{
-    ULONG Index = Dict->HashFunction(Key) % Dict->BucketCount;
-    for (FSP_DICT_ENTRY *Entry = Dict->Buckets[Index]; Entry; Entry = Entry->Next)
-        if (Dict->EqualFunction(Key, Entry->Key))
-            return Entry;
-    return 0;
-}
-
-VOID FspDictSetEntry(FSP_DICT *Dict, FSP_DICT_ENTRY *Entry)
-{
-    ULONG Index = Dict->HashFunction(Entry->Key) % Dict->BucketCount;
-    Entry->Next = Dict->Buckets[Index];
-    Dict->Buckets[Index] = Entry;
-}
-
-FSP_DICT_ENTRY *FspDictRemoveEntry(FSP_DICT *Dict, PVOID Key)
-{
-    ULONG Index = Dict->HashFunction(Key) % Dict->BucketCount;
-    for (FSP_DICT_ENTRY **PEntry = &Dict->Buckets[Index]; *PEntry; PEntry = &(*PEntry)->Next)
-        if (Dict->EqualFunction(Key, (*PEntry)->Key))
-        {
-            FSP_DICT_ENTRY *Entry = *PEntry;
-            *PEntry = Entry->Next;
-            return Entry;
-        }
-    return 0;
-}

src/sys/driver.h

@@ -265,6 +265,40 @@ VOID FspFreeExternal(PVOID Pointer)
     ExFreePool(Pointer);
 }
 
+/* hash mix */
+/* Based on the MurmurHash3 fmix32/fmix64 function:
+ * See: https://code.google.com/p/smhasher/source/browse/trunk/MurmurHash3.cpp?r=152#68
+ */
+static inline
+UINT32 FspHashMix32(UINT32 h)
+{
+    h ^= h >> 16;
+    h *= 0x85ebca6b;
+    h ^= h >> 13;
+    h *= 0xc2b2ae35;
+    h ^= h >> 16;
+    return h;
+}
+static inline
+UINT64 FspHashMix64(UINT64 k)
+{
+    k ^= k >> 33;
+    k *= 0xff51afd7ed558ccdULL;
+    k ^= k >> 33;
+    k *= 0xc4ceb9fe1a85ec53ULL;
+    k ^= k >> 33;
+    return k;
+}
+static inline
+ULONG FspHashMixPointer(PVOID Pointer)
+{
+#if _WIN64
+    return (ULONG)FspHashMix64((UINT64)Pointer);
+#else
+    return (ULONG)FspHashMix32((UINT32)Pointer);
+#endif
+}
+
 /* utility: GUIDs */
 NTSTATUS FspCreateGuid(GUID *Guid);
 
@@ -291,38 +325,13 @@ VOID FspInitializeDelayedWorkItem(FSP_DELAYED_WORK_ITEM *DelayedWorkItem,
     PWORKER_THREAD_ROUTINE Routine, PVOID Context);
 VOID FspQueueDelayedWorkItem(FSP_DELAYED_WORK_ITEM *DelayedWorkItem, LARGE_INTEGER Delay);
 
-/* dictionary */
-typedef BOOLEAN FSP_DICT_EQUAL_FUNCTION(PVOID Key1, PVOID Key2);
-typedef ULONG FSP_DICT_HASH_FUNCTION(PVOID Key);
-typedef struct _FSP_DICT_ENTRY
-{
-    PVOID Key, Value;
-    struct _FSP_DICT_ENTRY *Next;
-} FSP_DICT_ENTRY;
-typedef struct _FSP_DICT
-{
-    FSP_DICT_EQUAL_FUNCTION *EqualFunction;
-    FSP_DICT_HASH_FUNCTION *HashFunction;
-    FSP_DICT_ENTRY **Buckets;
-    ULONG BucketCount;
-} FSP_DICT;
-VOID FspDictInitialize(FSP_DICT *Dict,
-    FSP_DICT_EQUAL_FUNCTION *EqualFunction, FSP_DICT_HASH_FUNCTION *HashFunction,
-    FSP_DICT_ENTRY **Buckets, ULONG BucketCount);
-FSP_DICT_ENTRY *FspDictGetEntry(FSP_DICT *Dict, PVOID Key);
-VOID FspDictSetEntry(FSP_DICT *Dict, FSP_DICT_ENTRY *Entry);
-FSP_DICT_ENTRY *FspDictRemoveEntry(FSP_DICT *Dict, PVOID Key);
-
 /* IRP context */
-typedef struct
+#define FspIrpTimestamp(Irp)            \
-{
+    (*(ULONG *)&(Irp)->Tail.Overlay.DriverContext[0])
-    IO_CSQ_IRP_CONTEXT IoCsqIrpContext;
+#define FspIrpDictNext(Irp)             \
-    ULONGLONG ExpirationTime;
+    (*(PIRP *)&(Irp)->Tail.Overlay.DriverContext[1])
-} FSP_IRP_CONTEXT;
-#define FspIrpContext(Irp)              \
-    (*(FSP_IRP_CONTEXT **)&(Irp)->Tail.Overlay.DriverContext[0])
 #define FspIrpRequest(Irp)              \
-    (*(FSP_FSCTL_TRANSACT_REQ **)&(Irp)->Tail.Overlay.DriverContext[0])
+    (*(FSP_FSCTL_TRANSACT_REQ **)&(Irp)->Tail.Overlay.DriverContext[2])
 
 /* I/O queue */
 #define FspIoqTimeout                   ((PIRP)1)
@@ -333,11 +342,11 @@ typedef struct
     KEVENT PendingIrpEvent;
     LIST_ENTRY PendingIrpList, ProcessIrpList;
     IO_CSQ PendingIoCsq, ProcessIoCsq;
-    FSP_DICT ProcessIrpDict;
+    ULONG IrpTimeout;
-    LARGE_INTEGER IrpTimeout;
     ULONG PendingIrpCapacity, PendingIrpCount;
     VOID (*CompleteCanceledIrp)(PIRP Irp);
-    FSP_DICT_ENTRY *ProcessIrpDictBuckets[];
+    ULONG ProcessIrpBucketCount;
+    PVOID ProcessIrpBuckets[];
 } FSP_IOQ;
 NTSTATUS FspIoqCreate(
     ULONG IrpCapacity, PLARGE_INTEGER IrpTimeout, VOID (*CompleteCanceledIrp)(PIRP Irp),

src/sys/iop.c

@@ -29,7 +29,6 @@ VOID FspIopDispatchComplete(PIRP Irp, const FSP_FSCTL_TRANSACT_RSP *Response);
 
 typedef struct
 {
-    FSP_IRP_CONTEXT IrpContext;
     FSP_IOP_REQUEST_FINI *RequestFini;
     PVOID Context[3];
     __declspec(align(MEMORY_ALLOCATION_ALIGNMENT)) UINT8 RequestBuf[];

src/sys/ioq.c

@@ -55,10 +55,14 @@
  * queue is not empty.
  */
 
+#define InterruptTimeToSecFactor        10000000
+#define ConvertInterruptTimeToSec(Time) ((ULONG)((Time) / InterruptTimeToSecFactor))
+#define QueryInterruptTimeInSec()       ConvertInterruptTimeToSec(KeQueryInterruptTime())
+
 typedef struct
 {
     PVOID IrpHint;
-    ULONGLONG ExpirationTime;
+    ULONG ExpirationTime;
 } FSP_IOQ_PEEK_CONTEXT;
 
 static NTSTATUS FspIoqPendingInsertIrpEx(PIO_CSQ IoCsq, PIRP Irp, PVOID InsertContext)
@@ -99,8 +103,8 @@ static PIRP FspIoqPendingPeekNextIrp(PIO_CSQ IoCsq, PIRP Irp, PVOID PeekContext)
     PVOID IrpHint = ((FSP_IOQ_PEEK_CONTEXT *)PeekContext)->IrpHint;
     if (0 == IrpHint)
     {
-        ULONGLONG ExpirationTime = ((FSP_IOQ_PEEK_CONTEXT *)PeekContext)->ExpirationTime;
+        ULONG ExpirationTime = ((FSP_IOQ_PEEK_CONTEXT *)PeekContext)->ExpirationTime;
-        return FspIrpContext(Irp)->ExpirationTime <= ExpirationTime ? Irp : 0;
+        return FspIrpTimestamp(Irp) <= ExpirationTime ? Irp : 0;
     }
     else
         return Irp;
@@ -132,19 +136,29 @@ static NTSTATUS FspIoqProcessInsertIrpEx(PIO_CSQ IoCsq, PIRP Irp, PVOID InsertCo
     if (Ioq->Stopped)
         return STATUS_CANCELLED;
     InsertTailList(&Ioq->ProcessIrpList, &Irp->Tail.Overlay.ListEntry);
-    ASSERT(0 == FspDictGetEntry(&Ioq->ProcessIrpDict, Irp));
+    ULONG Index = FspHashMixPointer(Irp) % Ioq->ProcessIrpBucketCount;
-    FSP_DICT_ENTRY DictEntry = { 0 };
+#if DBG
-    DictEntry.Key = Irp;
+    for (PIRP IrpX = Ioq->ProcessIrpBuckets[Index]; IrpX; IrpX = FspIrpDictNext(IrpX))
-    FspDictSetEntry(&Ioq->ProcessIrpDict, &DictEntry);
+        ASSERT(IrpX != Irp);
+#endif
+    FspIrpDictNext(Irp) = Ioq->ProcessIrpBuckets[Index];
+    Ioq->ProcessIrpBuckets[Index] = Irp;
     return STATUS_SUCCESS;
 }
 
 static VOID FspIoqProcessRemoveIrp(PIO_CSQ IoCsq, PIRP Irp)
 {
     FSP_IOQ *Ioq = CONTAINING_RECORD(IoCsq, FSP_IOQ, ProcessIoCsq);
-    FSP_DICT_ENTRY *DictEntry; (VOID)DictEntry;
+    ULONG Index = FspHashMixPointer(Irp) % Ioq->ProcessIrpBucketCount;
-    DictEntry = FspDictRemoveEntry(&Ioq->ProcessIrpDict, Irp);
+    for (PIRP *PIrp = (PIRP *)&Ioq->ProcessIrpBuckets[Index];; PIrp = &FspIrpDictNext(*PIrp))
-    ASSERT(0 == DictEntry);
+    {
+        ASSERT(0 != *PIrp);
+        if (*PIrp == Irp)
+        {
+            *PIrp = FspIrpDictNext(*PIrp);
+            break;
+        }
+    }
     RemoveEntryList(&Irp->Tail.Overlay.ListEntry);
 }
 
@@ -163,13 +177,16 @@ static PIRP FspIoqProcessPeekNextIrp(PIO_CSQ IoCsq, PIRP Irp, PVOID PeekContext)
     PVOID IrpHint = ((FSP_IOQ_PEEK_CONTEXT *)PeekContext)->IrpHint;
     if (0 == IrpHint)
     {
-        ULONGLONG ExpirationTime = ((FSP_IOQ_PEEK_CONTEXT *)PeekContext)->ExpirationTime;
+        ULONG ExpirationTime = ((FSP_IOQ_PEEK_CONTEXT *)PeekContext)->ExpirationTime;
-        return FspIrpContext(Irp)->ExpirationTime <= ExpirationTime ? Irp : 0;
+        return FspIrpTimestamp(Irp) <= ExpirationTime ? Irp : 0;
     }
     else
     {
-        FSP_DICT_ENTRY *DictEntry = FspDictGetEntry(&Ioq->ProcessIrpDict, IrpHint);
+        ULONG Index = FspHashMixPointer(IrpHint) % Ioq->ProcessIrpBucketCount;
-        return 0 != DictEntry ? (PIRP)IrpHint : 0;
+        for (Irp = Ioq->ProcessIrpBuckets[Index]; Irp; Irp = FspIrpDictNext(Irp))
+            if (Irp == IrpHint)
+                return Irp;
+        return 0;
     }
 }
 
@@ -193,16 +210,6 @@ static VOID FspIoqProcessCompleteCanceledIrp(PIO_CSQ IoCsq, PIRP Irp)
     Ioq->CompleteCanceledIrp(Irp);
 }
 
-BOOLEAN FspIoqIrpEquals(PVOID Irp1, PVOID Irp2)
-{
-    return Irp1 == Irp2;
-}
-
-ULONG FspIoqIrpHash(PVOID Irp)
-{
-    return (ULONG)(UINT_PTR)Irp;
-}
-
 NTSTATUS FspIoqCreate(
     ULONG IrpCapacity, PLARGE_INTEGER IrpTimeout, VOID (*CompleteCanceledIrp)(PIRP Irp),
     FSP_IOQ **PIoq)
@@ -212,12 +219,12 @@ NTSTATUS FspIoqCreate(
     *PIoq = 0;
 
     FSP_IOQ *Ioq;
-    ULONG BucketCount = (PAGE_SIZE - sizeof *Ioq) / sizeof Ioq->ProcessIrpDictBuckets[0];
+    ULONG BucketCount = (PAGE_SIZE - sizeof *Ioq) / sizeof Ioq->ProcessIrpBuckets[0];
     Ioq = FspAllocNonPaged(PAGE_SIZE);
     if (0 == Ioq)
         return STATUS_INSUFFICIENT_RESOURCES;
+    RtlZeroMemory(Ioq, PAGE_SIZE);
 
-    RtlZeroMemory(Ioq, sizeof *Ioq);
     KeInitializeSpinLock(&Ioq->SpinLock);
     KeInitializeEvent(&Ioq->PendingIrpEvent, NotificationEvent, FALSE);
     InitializeListHead(&Ioq->PendingIrpList);
@@ -236,11 +243,11 @@ NTSTATUS FspIoqCreate(
         FspIoqProcessAcquireLock,
         FspIoqProcessReleaseLock,
         FspIoqProcessCompleteCanceledIrp);
-    FspDictInitialize(&Ioq->ProcessIrpDict,
+    Ioq->IrpTimeout = ConvertInterruptTimeToSec(IrpTimeout->QuadPart + InterruptTimeToSecFactor - 1);
-        FspIoqIrpEquals, FspIoqIrpHash, Ioq->ProcessIrpDictBuckets, BucketCount);
+        /* convert to seconds (and round up) */
-    Ioq->IrpTimeout = *IrpTimeout;
     Ioq->PendingIrpCapacity = IrpCapacity;
     Ioq->CompleteCanceledIrp = CompleteCanceledIrp;
+    Ioq->ProcessIrpBucketCount = BucketCount;
 
     *PIoq = Ioq;
 
@@ -282,7 +289,7 @@ VOID FspIoqRemoveExpired(FSP_IOQ *Ioq)
 {
     FSP_IOQ_PEEK_CONTEXT PeekContext;
     PeekContext.IrpHint = 0;
-    PeekContext.ExpirationTime = KeQueryInterruptTime();
+    PeekContext.ExpirationTime = QueryInterruptTimeInSec();
     PIRP Irp;
     while (0 != (Irp = IoCsqRemoveNextIrp(&Ioq->PendingIoCsq, &PeekContext)))
         Ioq->CompleteCanceledIrp(Irp);
@@ -293,7 +300,7 @@ VOID FspIoqRemoveExpired(FSP_IOQ *Ioq)
 BOOLEAN FspIoqPostIrp(FSP_IOQ *Ioq, PIRP Irp, NTSTATUS *PResult)
 {
     NTSTATUS Result;
-    FspIrpContext(Irp)->ExpirationTime = KeQueryInterruptTime() + Ioq->IrpTimeout.QuadPart;
+    FspIrpTimestamp(Irp) = QueryInterruptTimeInSec() + Ioq->IrpTimeout;
     Result = IoCsqInsertIrpEx(&Ioq->PendingIoCsq, Irp, 0, (PVOID)1);
     if (NT_SUCCESS(Result))
         return TRUE;
@@ -323,6 +330,7 @@ PIRP FspIoqNextPendingIrp(FSP_IOQ *Ioq, PLARGE_INTEGER Timeout)
 BOOLEAN FspIoqStartProcessingIrp(FSP_IOQ *Ioq, PIRP Irp)
 {
     NTSTATUS Result;
+    FspIrpTimestamp(Irp) = QueryInterruptTimeInSec() + Ioq->IrpTimeout;
     Result = IoCsqInsertIrpEx(&Ioq->ProcessIoCsq, Irp, 0, 0);
     return NT_SUCCESS(Result);
 }