diff --git a/src/sys/driver.h b/src/sys/driver.h index abdb979f..51dc963f 100644 --- a/src/sys/driver.h +++ b/src/sys/driver.h @@ -639,7 +639,151 @@ VOID FspIrpSetTopFlags(PIRP Irp, ULONG Flags) Irp->Tail.Overlay.DriverContext[2] = (PVOID)((UINT_PTR)Request | (Flags << 2)); } +/* + * Queued Events + * + * Queued Events are an implementation of SynchronizationEvent's using + * a KQUEUE. The reason we do this is because a KQUEUE has some desirable + * properties: + * + * - It has a LIFO wait discipline, which is advantageous in many situations. + * - It can limit the numbers of threads that can be satisfied concurrently. + * + * Queued Events must always be allocated in non-paged storage. + * + * Here is how Queued Events work. A queued event consists of a KQUEUE and a + * spin lock. There is also a LIST_ENTRY which is used as a dummy item to + * place in the KQUEUE. + * + * The KQUEUE is guaranteed to contain either 0 or 1 items. When the KQUEUE + * contains 0 items the queued event is considered non-signaled. When the + * KQUEUE contains 1 items the queued event is considered signaled. + * + * To transition from the non-signaled to the signaled state, we acquire the + * spin lock and then insert the dummy item in the KQUEUE using KeInsertQueue. + * To transition from the signaled to the non-signaled state, we simply (wait + * and) remove the dummy item from the KQUEUE using KeRemoveQueue (without + * the use of the spin lock). + * + * EventSet: + * AcquireSpinLock + * if (0 == KeReadState()) // if KQUEUE is empty + * KeInsertQueue(DUMMY); + * ReleaseSpinLock + * + * EventWait: + * KeRemoveQueue(); // (wait and) remove item + * + * First notice that EventSet is serialized by the use of the spin lock. This + * guarantees that the dummy item can be only inserted ONCE in the KQUEUE + * and that the only possible signaled state transitions for EventSet are 0->1 + * and 1->1. This is how KeSetEvent works for a SynchronizationEvent. + * + * Second notice that EventWait is not protected by the spin lock, which means + * that it can happen at any time including concurrently with EventSet or + * another EventWait. Notice also that for EventWait the only possible + * transitions are 1->0 or 0->0 (0->block->0). This is how + * KeWaitForSingleObject works for a SynchronizationEvent. + * + * We now have to consider what happens when we have one EventSet concurrently + * with one or more EventWait's: + * + * 1. The EventWait(s) happen before KeReadState. If the KQUEUE has an + * item one EventWait gets satisfied, otherwise it blocks. In this case + * KeReadState will read the KQUEUE's state as 0 and KeInsertQueue will + * insert the dummy item, which will unblock the EventWait. + * + * 2. The EventWait(s) happen after KeReadState, but before KeInsertQueue. + * If the dummy item was already in the KQUEUE the KeReadState test will + * fail and KeInsertQueue will not be executed, but EventWait will be + * satisfied immediately. If the dummy item was not in the KQUEUE the + * KeReadState will succeed and EventWait will momentarily block until + * KeInsertQueue releases it. + * + * 3. The EventWait(s) happen after KeInsertQueue. In this case the dummy + * item in is the KQUEUE already and one EventWait will be satisfied + * immediately. + * + * A final note: Queued Events cannot cleanly support an EventClear operation. + * The obvious choice of using KeRemoveQueue with a 0 timeout is insufficient + * because it would associate the current thread with the KQUEUE and that is + * not desirable. KeRundownQueue cannot be used either because it + * disassociates all threads from the KQUEUE. + */ +typedef struct +{ + KQUEUE Queue; + LIST_ENTRY DummyEntry; + KSPIN_LOCK SpinLock; +} FSP_QEVENT; +static inline +VOID FspQeventInitialize(FSP_QEVENT *Qevent, ULONG ThreadCount) +{ + KeInitializeQueue(&Qevent->Queue, ThreadCount); + RtlZeroMemory(&Qevent->DummyEntry, sizeof Qevent->DummyEntry); + KeInitializeSpinLock(&Qevent->SpinLock); +} +static inline +VOID FspQeventFinalize(FSP_QEVENT *Qevent) +{ + KeRundownQueue(&Qevent->Queue); +} +static inline +VOID FspQeventSetNoLock(FSP_QEVENT *Qevent) +{ + ASSERT(KeGetCurrentIrql() == DISPATCH_LEVEL); + if (0 == KeReadStateQueue(&Qevent->Queue)) + KeInsertQueue(&Qevent->Queue, &Qevent->DummyEntry); +} +static inline +VOID FspQeventSet(FSP_QEVENT *Qevent) +{ + KIRQL Irql; + KeAcquireSpinLock(&Qevent->SpinLock, &Irql); + FspQeventSetNoLock(Qevent); + KeReleaseSpinLock(&Qevent->SpinLock, Irql); +} +static inline +NTSTATUS FspQeventWait(FSP_QEVENT *Qevent, + KPROCESSOR_MODE WaitMode, BOOLEAN Alertable, PLARGE_INTEGER PTimeout) +{ + PLIST_ENTRY ListEntry; + KeRemoveQueueEx(&Qevent->Queue, WaitMode, Alertable, PTimeout, &ListEntry, 1); + if (ListEntry == &Qevent->DummyEntry) + return STATUS_SUCCESS; + return (NTSTATUS)(UINT_PTR)ListEntry; +} +static inline +NTSTATUS FspQeventCancellableWait(FSP_QEVENT *Qevent, + PLARGE_INTEGER PTimeout, PIRP Irp) +{ + NTSTATUS Result; + UINT64 ExpirationTime = 0, InterruptTime; + if (0 != PTimeout && 0 > PTimeout->QuadPart) + ExpirationTime = KeQueryInterruptTime() - PTimeout->QuadPart; +retry: + Result = FspQeventWait(Qevent, KernelMode, TRUE, PTimeout); + if (STATUS_ALERTED == Result) + { + if (PsIsThreadTerminating(PsGetCurrentThread())) + return STATUS_THREAD_IS_TERMINATING; + if (0 != Irp && Irp->Cancel) + return STATUS_CANCELLED; + if (0 != ExpirationTime) + { + InterruptTime = KeQueryInterruptTime(); + if (ExpirationTime <= InterruptTime) + return STATUS_TIMEOUT; + PTimeout->QuadPart = (INT64)InterruptTime - (INT64)ExpirationTime; + } + goto retry; + } + return Result; +} + /* I/O queue */ +#define FSP_IOQ_USE_QEVENT +#define FSP_IOQ_PROCESS_NO_CANCEL #define FspIoqTimeout ((PIRP)1) #define FspIoqCancelled ((PIRP)2) #define FspIoqPostIrp(Q, I, R) FspIoqPostIrpEx(Q, I, FALSE, R) @@ -648,7 +792,11 @@ typedef struct { KSPIN_LOCK SpinLock; BOOLEAN Stopped; +#if defined(FSP_IOQ_USE_QEVENT) + FSP_QEVENT PendingIrpEvent; +#else KEVENT PendingIrpEvent; +#endif LIST_ENTRY PendingIrpList, ProcessIrpList, RetriedIrpList; IO_CSQ PendingIoCsq, ProcessIoCsq, RetriedIoCsq; ULONG IrpTimeout; diff --git a/src/sys/ioq.c b/src/sys/ioq.c index 2efa8f91..0723338d 100644 --- a/src/sys/ioq.c +++ b/src/sys/ioq.c @@ -22,7 +22,9 @@ * * [NOTE: this comment no longer describes accurately an FSP_IOQ. The main * difference is that an FSP_IOQ now has a third queue which is used to - * retry IRP completions. However the main ideas below are still valid, so + * retry IRP completions. Another difference is that the FSP_IOQ can now + * use Queued Events (which are implemented on top of KQUEUE) instead of + * SynchronizationEvent's. However the main ideas below are still valid, so * I am leaving the rest of the comment intact.] * * An FSP_IOQ encapsulates the main FSP mechanism for handling IRP's. @@ -123,8 +125,31 @@ * To deal with the second problem we simply call FspIoqPendingResetSynch after * a WaitForSingleObject call if the IRP dequeueing fails; this ensures that the * event is in the correst state. + * + * UPDATE: We can now use a Queued Event which behaves like a SynchronizationEvent, + * but has better performance. Unfortunately Queued Events cannot cleanly implement + * an EventClear operation. However the EventClear operation is not strictly needed. */ +/* + * FSP_IOQ_USE_QEVENT + * + * Define this macro to use Queued Events instead of simple SynchronizationEvent's. + */ +#if defined(FSP_IOQ_USE_QEVENT) +#define FspIoqEventInitialize(E) FspQeventInitialize(E, 0) +#define FspIoqEventFinalize(E) FspQeventFinalize(E) +#define FspIoqEventSet(E) FspQeventSetNoLock(E) +#define FspIoqEventCancellableWait(E,T,I) FspQeventCancellableWait(E,T,I) +#define FspIoqEventClear(E) ((VOID)0) +#else +#define FspIoqEventInitialize(E) KeInitializeEvent(E, SynchronizationEvent, FALSE) +#define FspIoqEventFinalize(E) ((VOID)0) +#define FspIoqEventSet(E) KeSetEvent(E, 1, FALSE) +#define FspIoqEventCancellableWait(E,T,I) FsRtlCancellableWaitForSingleObject(E,T,I) +#define FspIoqEventClear(E) KeClearEvent(E) +#endif + /* * FSP_IOQ_PROCESS_NO_CANCEL * @@ -136,8 +161,6 @@ * inform it of whether the operation was successful or not. We can only do this reliably * if we do not allow cancelation after an operation has been started. */ -#define FSP_IOQ_PROCESS_NO_CANCEL - #if defined(FSP_IOQ_PROCESS_NO_CANCEL) static NTSTATUS FspCsqInsertIrpEx(PIO_CSQ Csq, PIRP Irp, PIO_CSQ_IRP_CONTEXT Context, PVOID InsertContext) { @@ -199,10 +222,11 @@ static inline VOID FspIoqPendingResetSynch(FSP_IOQ *Ioq) */ if (0 != Ioq->PendingIrpCount || Ioq->Stopped) /* list is not empty or is stopped; wake up a waiter */ - KeSetEvent(&Ioq->PendingIrpEvent, 1, FALSE); + FspIoqEventSet(&Ioq->PendingIrpEvent); else /* list is empty and not stopped; future threads should go to sleep */ - KeClearEvent(&Ioq->PendingIrpEvent); + /* NOTE: this is not stricly necessary! */ + FspIoqEventClear(&Ioq->PendingIrpEvent); } static NTSTATUS FspIoqPendingInsertIrpEx(PIO_CSQ IoCsq, PIRP Irp, PVOID InsertContext) @@ -214,7 +238,7 @@ static NTSTATUS FspIoqPendingInsertIrpEx(PIO_CSQ IoCsq, PIRP Irp, PVOID InsertCo return STATUS_INSUFFICIENT_RESOURCES; Ioq->PendingIrpCount++; InsertTailList(&Ioq->PendingIrpList, &Irp->Tail.Overlay.ListEntry); - KeSetEvent(&Ioq->PendingIrpEvent, 1, FALSE); + FspIoqEventSet(&Ioq->PendingIrpEvent); /* equivalent to FspIoqPendingResetSynch(Ioq) */ return STATUS_SUCCESS; } @@ -460,7 +484,7 @@ NTSTATUS FspIoqCreate( RtlZeroMemory(Ioq, PAGE_SIZE); KeInitializeSpinLock(&Ioq->SpinLock); - KeInitializeEvent(&Ioq->PendingIrpEvent, SynchronizationEvent, FALSE); + FspIoqEventInitialize(&Ioq->PendingIrpEvent); InitializeListHead(&Ioq->PendingIrpList); InitializeListHead(&Ioq->ProcessIrpList); InitializeListHead(&Ioq->RetriedIrpList); @@ -499,6 +523,7 @@ NTSTATUS FspIoqCreate( VOID FspIoqDelete(FSP_IOQ *Ioq) { FspIoqStop(Ioq); + FspIoqEventFinalize(&Ioq->PendingIrpEvent); FspFree(Ioq); } @@ -508,7 +533,7 @@ VOID FspIoqStop(FSP_IOQ *Ioq) KeAcquireSpinLock(&Ioq->SpinLock, &Irql); Ioq->Stopped = TRUE; /* we are being stopped, permanently wake up waiters */ - KeSetEvent(&Ioq->PendingIrpEvent, 1, FALSE); + FspIoqEventSet(&Ioq->PendingIrpEvent); /* equivalent to FspIoqPendingResetSynch(Ioq) */ KeReleaseSpinLock(&Ioq->SpinLock, Irql); PIRP Irp; @@ -577,7 +602,7 @@ PIRP FspIoqNextPendingIrp(FSP_IOQ *Ioq, PIRP BoundaryIrp, PLARGE_INTEGER Timeout if (0 != Timeout) { NTSTATUS Result; - Result = FsRtlCancellableWaitForSingleObject(&Ioq->PendingIrpEvent, Timeout, + Result = FspIoqEventCancellableWait(&Ioq->PendingIrpEvent, Timeout, CancellableIrp); if (STATUS_TIMEOUT == Result) return FspIoqTimeout;