From 72fdea5e780802957f53e890e97eec514d65ba22 Mon Sep 17 00:00:00 2001 From: Bill Zissimopoulos Date: Mon, 28 Dec 2015 11:17:37 -0800 Subject: [PATCH] sys: ioq --- build/VStudio/winfsp_sys.vcxproj | 1 - build/VStudio/winfsp_sys.vcxproj.filters | 3 - src/sys/dict.c | 47 --------------- src/sys/driver.h | 75 +++++++++++++----------- src/sys/iop.c | 1 - src/sys/ioq.c | 70 ++++++++++++---------- 6 files changed, 81 insertions(+), 116 deletions(-) delete mode 100644 src/sys/dict.c diff --git a/build/VStudio/winfsp_sys.vcxproj b/build/VStudio/winfsp_sys.vcxproj index 1f10ba45..4d50bb29 100644 --- a/build/VStudio/winfsp_sys.vcxproj +++ b/build/VStudio/winfsp_sys.vcxproj @@ -152,7 +152,6 @@ - diff --git a/build/VStudio/winfsp_sys.vcxproj.filters b/build/VStudio/winfsp_sys.vcxproj.filters index ff94b395..29d159a0 100644 --- a/build/VStudio/winfsp_sys.vcxproj.filters +++ b/build/VStudio/winfsp_sys.vcxproj.filters @@ -86,9 +86,6 @@ Source - - Source - diff --git a/src/sys/dict.c b/src/sys/dict.c deleted file mode 100644 index 57226f37..00000000 --- a/src/sys/dict.c +++ /dev/null @@ -1,47 +0,0 @@ -/** - * @file sys/dict.c - * - * @copyright 2015 Bill Zissimopoulos - */ - -#include - -VOID FspDictInitialize(FSP_DICT *Dict, - FSP_DICT_EQUAL_FUNCTION *EqualFunction, FSP_DICT_HASH_FUNCTION *HashFunction, - FSP_DICT_ENTRY **Buckets, ULONG BucketCount) -{ - RtlZeroMemory(Buckets, BucketCount * sizeof Buckets[0]); - Dict->EqualFunction = EqualFunction; - Dict->HashFunction = HashFunction; - Dict->Buckets = Buckets; - Dict->BucketCount = BucketCount; -} - -FSP_DICT_ENTRY *FspDictGetEntry(FSP_DICT *Dict, PVOID Key) -{ - ULONG Index = Dict->HashFunction(Key) % Dict->BucketCount; - for (FSP_DICT_ENTRY *Entry = Dict->Buckets[Index]; Entry; Entry = Entry->Next) - if (Dict->EqualFunction(Key, Entry->Key)) - return Entry; - return 0; -} - -VOID FspDictSetEntry(FSP_DICT *Dict, FSP_DICT_ENTRY *Entry) -{ - ULONG Index = Dict->HashFunction(Entry->Key) % Dict->BucketCount; - Entry->Next = Dict->Buckets[Index]; - Dict->Buckets[Index] = Entry; -} - -FSP_DICT_ENTRY *FspDictRemoveEntry(FSP_DICT *Dict, PVOID Key) -{ - ULONG Index = Dict->HashFunction(Key) % Dict->BucketCount; - for (FSP_DICT_ENTRY **PEntry = &Dict->Buckets[Index]; *PEntry; PEntry = &(*PEntry)->Next) - if (Dict->EqualFunction(Key, (*PEntry)->Key)) - { - FSP_DICT_ENTRY *Entry = *PEntry; - *PEntry = Entry->Next; - return Entry; - } - return 0; -} diff --git a/src/sys/driver.h b/src/sys/driver.h index b03e59b8..824dbefb 100644 --- a/src/sys/driver.h +++ b/src/sys/driver.h @@ -265,6 +265,40 @@ VOID FspFreeExternal(PVOID Pointer) ExFreePool(Pointer); } +/* hash mix */ +/* Based on the MurmurHash3 fmix32/fmix64 function: + * See: https://code.google.com/p/smhasher/source/browse/trunk/MurmurHash3.cpp?r=152#68 + */ +static inline +UINT32 FspHashMix32(UINT32 h) +{ + h ^= h >> 16; + h *= 0x85ebca6b; + h ^= h >> 13; + h *= 0xc2b2ae35; + h ^= h >> 16; + return h; +} +static inline +UINT64 FspHashMix64(UINT64 k) +{ + k ^= k >> 33; + k *= 0xff51afd7ed558ccdULL; + k ^= k >> 33; + k *= 0xc4ceb9fe1a85ec53ULL; + k ^= k >> 33; + return k; +} +static inline +ULONG FspHashMixPointer(PVOID Pointer) +{ +#if _WIN64 + return (ULONG)FspHashMix64((UINT64)Pointer); +#else + return (ULONG)FspHashMix32((UINT32)Pointer); +#endif +} + /* utility: GUIDs */ NTSTATUS FspCreateGuid(GUID *Guid); @@ -291,38 +325,13 @@ VOID FspInitializeDelayedWorkItem(FSP_DELAYED_WORK_ITEM *DelayedWorkItem, PWORKER_THREAD_ROUTINE Routine, PVOID Context); VOID FspQueueDelayedWorkItem(FSP_DELAYED_WORK_ITEM *DelayedWorkItem, LARGE_INTEGER Delay); -/* dictionary */ -typedef BOOLEAN FSP_DICT_EQUAL_FUNCTION(PVOID Key1, PVOID Key2); -typedef ULONG FSP_DICT_HASH_FUNCTION(PVOID Key); -typedef struct _FSP_DICT_ENTRY -{ - PVOID Key, Value; - struct _FSP_DICT_ENTRY *Next; -} FSP_DICT_ENTRY; -typedef struct _FSP_DICT -{ - FSP_DICT_EQUAL_FUNCTION *EqualFunction; - FSP_DICT_HASH_FUNCTION *HashFunction; - FSP_DICT_ENTRY **Buckets; - ULONG BucketCount; -} FSP_DICT; -VOID FspDictInitialize(FSP_DICT *Dict, - FSP_DICT_EQUAL_FUNCTION *EqualFunction, FSP_DICT_HASH_FUNCTION *HashFunction, - FSP_DICT_ENTRY **Buckets, ULONG BucketCount); -FSP_DICT_ENTRY *FspDictGetEntry(FSP_DICT *Dict, PVOID Key); -VOID FspDictSetEntry(FSP_DICT *Dict, FSP_DICT_ENTRY *Entry); -FSP_DICT_ENTRY *FspDictRemoveEntry(FSP_DICT *Dict, PVOID Key); - /* IRP context */ -typedef struct -{ - IO_CSQ_IRP_CONTEXT IoCsqIrpContext; - ULONGLONG ExpirationTime; -} FSP_IRP_CONTEXT; -#define FspIrpContext(Irp) \ - (*(FSP_IRP_CONTEXT **)&(Irp)->Tail.Overlay.DriverContext[0]) +#define FspIrpTimestamp(Irp) \ + (*(ULONG *)&(Irp)->Tail.Overlay.DriverContext[0]) +#define FspIrpDictNext(Irp) \ + (*(PIRP *)&(Irp)->Tail.Overlay.DriverContext[1]) #define FspIrpRequest(Irp) \ - (*(FSP_FSCTL_TRANSACT_REQ **)&(Irp)->Tail.Overlay.DriverContext[0]) + (*(FSP_FSCTL_TRANSACT_REQ **)&(Irp)->Tail.Overlay.DriverContext[2]) /* I/O queue */ #define FspIoqTimeout ((PIRP)1) @@ -333,11 +342,11 @@ typedef struct KEVENT PendingIrpEvent; LIST_ENTRY PendingIrpList, ProcessIrpList; IO_CSQ PendingIoCsq, ProcessIoCsq; - FSP_DICT ProcessIrpDict; - LARGE_INTEGER IrpTimeout; + ULONG IrpTimeout; ULONG PendingIrpCapacity, PendingIrpCount; VOID (*CompleteCanceledIrp)(PIRP Irp); - FSP_DICT_ENTRY *ProcessIrpDictBuckets[]; + ULONG ProcessIrpBucketCount; + PVOID ProcessIrpBuckets[]; } FSP_IOQ; NTSTATUS FspIoqCreate( ULONG IrpCapacity, PLARGE_INTEGER IrpTimeout, VOID (*CompleteCanceledIrp)(PIRP Irp), diff --git a/src/sys/iop.c b/src/sys/iop.c index 60c78131..3776e27b 100644 --- a/src/sys/iop.c +++ b/src/sys/iop.c @@ -29,7 +29,6 @@ VOID FspIopDispatchComplete(PIRP Irp, const FSP_FSCTL_TRANSACT_RSP *Response); typedef struct { - FSP_IRP_CONTEXT IrpContext; FSP_IOP_REQUEST_FINI *RequestFini; PVOID Context[3]; __declspec(align(MEMORY_ALLOCATION_ALIGNMENT)) UINT8 RequestBuf[]; diff --git a/src/sys/ioq.c b/src/sys/ioq.c index 0aa9ffce..8bb7d57c 100644 --- a/src/sys/ioq.c +++ b/src/sys/ioq.c @@ -55,10 +55,14 @@ * queue is not empty. */ +#define InterruptTimeToSecFactor 10000000 +#define ConvertInterruptTimeToSec(Time) ((ULONG)((Time) / InterruptTimeToSecFactor)) +#define QueryInterruptTimeInSec() ConvertInterruptTimeToSec(KeQueryInterruptTime()) + typedef struct { PVOID IrpHint; - ULONGLONG ExpirationTime; + ULONG ExpirationTime; } FSP_IOQ_PEEK_CONTEXT; static NTSTATUS FspIoqPendingInsertIrpEx(PIO_CSQ IoCsq, PIRP Irp, PVOID InsertContext) @@ -99,8 +103,8 @@ static PIRP FspIoqPendingPeekNextIrp(PIO_CSQ IoCsq, PIRP Irp, PVOID PeekContext) PVOID IrpHint = ((FSP_IOQ_PEEK_CONTEXT *)PeekContext)->IrpHint; if (0 == IrpHint) { - ULONGLONG ExpirationTime = ((FSP_IOQ_PEEK_CONTEXT *)PeekContext)->ExpirationTime; - return FspIrpContext(Irp)->ExpirationTime <= ExpirationTime ? Irp : 0; + ULONG ExpirationTime = ((FSP_IOQ_PEEK_CONTEXT *)PeekContext)->ExpirationTime; + return FspIrpTimestamp(Irp) <= ExpirationTime ? Irp : 0; } else return Irp; @@ -132,19 +136,29 @@ static NTSTATUS FspIoqProcessInsertIrpEx(PIO_CSQ IoCsq, PIRP Irp, PVOID InsertCo if (Ioq->Stopped) return STATUS_CANCELLED; InsertTailList(&Ioq->ProcessIrpList, &Irp->Tail.Overlay.ListEntry); - ASSERT(0 == FspDictGetEntry(&Ioq->ProcessIrpDict, Irp)); - FSP_DICT_ENTRY DictEntry = { 0 }; - DictEntry.Key = Irp; - FspDictSetEntry(&Ioq->ProcessIrpDict, &DictEntry); + ULONG Index = FspHashMixPointer(Irp) % Ioq->ProcessIrpBucketCount; +#if DBG + for (PIRP IrpX = Ioq->ProcessIrpBuckets[Index]; IrpX; IrpX = FspIrpDictNext(IrpX)) + ASSERT(IrpX != Irp); +#endif + FspIrpDictNext(Irp) = Ioq->ProcessIrpBuckets[Index]; + Ioq->ProcessIrpBuckets[Index] = Irp; return STATUS_SUCCESS; } static VOID FspIoqProcessRemoveIrp(PIO_CSQ IoCsq, PIRP Irp) { FSP_IOQ *Ioq = CONTAINING_RECORD(IoCsq, FSP_IOQ, ProcessIoCsq); - FSP_DICT_ENTRY *DictEntry; (VOID)DictEntry; - DictEntry = FspDictRemoveEntry(&Ioq->ProcessIrpDict, Irp); - ASSERT(0 == DictEntry); + ULONG Index = FspHashMixPointer(Irp) % Ioq->ProcessIrpBucketCount; + for (PIRP *PIrp = (PIRP *)&Ioq->ProcessIrpBuckets[Index];; PIrp = &FspIrpDictNext(*PIrp)) + { + ASSERT(0 != *PIrp); + if (*PIrp == Irp) + { + *PIrp = FspIrpDictNext(*PIrp); + break; + } + } RemoveEntryList(&Irp->Tail.Overlay.ListEntry); } @@ -163,13 +177,16 @@ static PIRP FspIoqProcessPeekNextIrp(PIO_CSQ IoCsq, PIRP Irp, PVOID PeekContext) PVOID IrpHint = ((FSP_IOQ_PEEK_CONTEXT *)PeekContext)->IrpHint; if (0 == IrpHint) { - ULONGLONG ExpirationTime = ((FSP_IOQ_PEEK_CONTEXT *)PeekContext)->ExpirationTime; - return FspIrpContext(Irp)->ExpirationTime <= ExpirationTime ? Irp : 0; + ULONG ExpirationTime = ((FSP_IOQ_PEEK_CONTEXT *)PeekContext)->ExpirationTime; + return FspIrpTimestamp(Irp) <= ExpirationTime ? Irp : 0; } else { - FSP_DICT_ENTRY *DictEntry = FspDictGetEntry(&Ioq->ProcessIrpDict, IrpHint); - return 0 != DictEntry ? (PIRP)IrpHint : 0; + ULONG Index = FspHashMixPointer(IrpHint) % Ioq->ProcessIrpBucketCount; + for (Irp = Ioq->ProcessIrpBuckets[Index]; Irp; Irp = FspIrpDictNext(Irp)) + if (Irp == IrpHint) + return Irp; + return 0; } } @@ -193,16 +210,6 @@ static VOID FspIoqProcessCompleteCanceledIrp(PIO_CSQ IoCsq, PIRP Irp) Ioq->CompleteCanceledIrp(Irp); } -BOOLEAN FspIoqIrpEquals(PVOID Irp1, PVOID Irp2) -{ - return Irp1 == Irp2; -} - -ULONG FspIoqIrpHash(PVOID Irp) -{ - return (ULONG)(UINT_PTR)Irp; -} - NTSTATUS FspIoqCreate( ULONG IrpCapacity, PLARGE_INTEGER IrpTimeout, VOID (*CompleteCanceledIrp)(PIRP Irp), FSP_IOQ **PIoq) @@ -212,12 +219,12 @@ NTSTATUS FspIoqCreate( *PIoq = 0; FSP_IOQ *Ioq; - ULONG BucketCount = (PAGE_SIZE - sizeof *Ioq) / sizeof Ioq->ProcessIrpDictBuckets[0]; + ULONG BucketCount = (PAGE_SIZE - sizeof *Ioq) / sizeof Ioq->ProcessIrpBuckets[0]; Ioq = FspAllocNonPaged(PAGE_SIZE); if (0 == Ioq) return STATUS_INSUFFICIENT_RESOURCES; + RtlZeroMemory(Ioq, PAGE_SIZE); - RtlZeroMemory(Ioq, sizeof *Ioq); KeInitializeSpinLock(&Ioq->SpinLock); KeInitializeEvent(&Ioq->PendingIrpEvent, NotificationEvent, FALSE); InitializeListHead(&Ioq->PendingIrpList); @@ -236,11 +243,11 @@ NTSTATUS FspIoqCreate( FspIoqProcessAcquireLock, FspIoqProcessReleaseLock, FspIoqProcessCompleteCanceledIrp); - FspDictInitialize(&Ioq->ProcessIrpDict, - FspIoqIrpEquals, FspIoqIrpHash, Ioq->ProcessIrpDictBuckets, BucketCount); - Ioq->IrpTimeout = *IrpTimeout; + Ioq->IrpTimeout = ConvertInterruptTimeToSec(IrpTimeout->QuadPart + InterruptTimeToSecFactor - 1); + /* convert to seconds (and round up) */ Ioq->PendingIrpCapacity = IrpCapacity; Ioq->CompleteCanceledIrp = CompleteCanceledIrp; + Ioq->ProcessIrpBucketCount = BucketCount; *PIoq = Ioq; @@ -282,7 +289,7 @@ VOID FspIoqRemoveExpired(FSP_IOQ *Ioq) { FSP_IOQ_PEEK_CONTEXT PeekContext; PeekContext.IrpHint = 0; - PeekContext.ExpirationTime = KeQueryInterruptTime(); + PeekContext.ExpirationTime = QueryInterruptTimeInSec(); PIRP Irp; while (0 != (Irp = IoCsqRemoveNextIrp(&Ioq->PendingIoCsq, &PeekContext))) Ioq->CompleteCanceledIrp(Irp); @@ -293,7 +300,7 @@ VOID FspIoqRemoveExpired(FSP_IOQ *Ioq) BOOLEAN FspIoqPostIrp(FSP_IOQ *Ioq, PIRP Irp, NTSTATUS *PResult) { NTSTATUS Result; - FspIrpContext(Irp)->ExpirationTime = KeQueryInterruptTime() + Ioq->IrpTimeout.QuadPart; + FspIrpTimestamp(Irp) = QueryInterruptTimeInSec() + Ioq->IrpTimeout; Result = IoCsqInsertIrpEx(&Ioq->PendingIoCsq, Irp, 0, (PVOID)1); if (NT_SUCCESS(Result)) return TRUE; @@ -323,6 +330,7 @@ PIRP FspIoqNextPendingIrp(FSP_IOQ *Ioq, PLARGE_INTEGER Timeout) BOOLEAN FspIoqStartProcessingIrp(FSP_IOQ *Ioq, PIRP Irp) { NTSTATUS Result; + FspIrpTimestamp(Irp) = QueryInterruptTimeInSec() + Ioq->IrpTimeout; Result = IoCsqInsertIrpEx(&Ioq->ProcessIoCsq, Irp, 0, 0); return NT_SUCCESS(Result); }