sys,dll: FSP_FSCTL_UNLOAD

build/VStudio/testing/winfsp-tests.vcxproj

@@ -284,6 +284,7 @@
     <ClCompile Include="..\..\..\tst\winfsp-tests\security-test.c" />
     <ClCompile Include="..\..\..\tst\winfsp-tests\stream-tests.c" />
     <ClCompile Include="..\..\..\tst\winfsp-tests\timeout-test.c" />
+    <ClCompile Include="..\..\..\tst\winfsp-tests\loadun-test.c" />
     <ClCompile Include="..\..\..\tst\winfsp-tests\uuid5-test.c" />
     <ClCompile Include="..\..\..\tst\winfsp-tests\version-test.c" />
     <ClCompile Include="..\..\..\tst\winfsp-tests\volpath-test.c" />

build/VStudio/testing/winfsp-tests.vcxproj.filters

@@ -112,6 +112,9 @@
     <ClCompile Include="..\..\..\tst\winfsp-tests\notify-test.c">
       <Filter>Source</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\..\tst\winfsp-tests\loadun-test.c">
+      <Filter>Source</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\..\..\ext\tlib\testsuite.h">

inc/winfsp/fsctl.h

@@ -111,6 +111,8 @@ extern const __declspec(selectany) GUID FspFsvrtDeviceClassGuid =
     CTL_CODE(FILE_DEVICE_FILE_SYSTEM, 0x800 + 's', METHOD_BUFFERED, FILE_ANY_ACCESS)
 #define FSP_FSCTL_NOTIFY                \
     CTL_CODE(FILE_DEVICE_FILE_SYSTEM, 0x800 + 'n', METHOD_NEITHER, FILE_ANY_ACCESS)
+#define FSP_FSCTL_UNLOAD                \
+    CTL_CODE(FILE_DEVICE_FILE_SYSTEM, 0x800 + 'U', METHOD_NEITHER, FILE_ANY_ACCESS)
 
 /* fsctl internal device codes (usable only in-kernel) */
 #define FSP_FSCTL_TRANSACT_INTERNAL     \
@@ -694,6 +696,8 @@ FSP_API NTSTATUS FspFsctlNotify(HANDLE VolumeHandle,
 FSP_API NTSTATUS FspFsctlGetVolumeList(PWSTR DevicePath,
     PWCHAR VolumeListBuf, PSIZE_T PVolumeListSize);
 FSP_API NTSTATUS FspFsctlPreflight(PWSTR DevicePath);
+FSP_API NTSTATUS FspFsctlStartService(VOID);
+FSP_API NTSTATUS FspFsctlStopService(VOID);
 
 typedef struct
 {

inc/winfsp/winfsp.h

@@ -1047,11 +1047,14 @@ typedef struct _FSP_FILE_SYSTEM_INTERFACE
 
     NTSTATUS (*Obsolete0)(VOID);
 
+    VOID (*DispatcherStopped)(FSP_FILE_SYSTEM *FileSystem,
+        BOOLEAN Normally);
+
     /*
      * This ensures that this interface will always contain 64 function pointers.
      * Please update when changing the interface as it is important for future compatibility.
      */
-    NTSTATUS (*Reserved[32])();
+    NTSTATUS (*Reserved[31])();
 } FSP_FILE_SYSTEM_INTERFACE;
 FSP_FSCTL_STATIC_ASSERT(sizeof(FSP_FILE_SYSTEM_INTERFACE) == 64 * sizeof(NTSTATUS (*)()),
     "FSP_FILE_SYSTEM_INTERFACE must have 64 entries.");
@@ -1074,7 +1077,8 @@ typedef struct _FSP_FILE_SYSTEM
     SRWLOCK OpGuardLock;
     BOOLEAN UmFileContextIsUserContext2, UmFileContextIsFullContext;
     UINT16 UmNoReparsePointsDirCheck:1;
-    UINT16 UmReservedFlags:15;
+    UINT16 UmReservedFlags:14;
+    UINT16 DispatcherStopping:1;
 } FSP_FILE_SYSTEM;
 FSP_FSCTL_STATIC_ASSERT(
     (4 == sizeof(PVOID) && 660 == sizeof(FSP_FILE_SYSTEM)) ||

src/dll/fs.c

@@ -358,6 +358,21 @@ exit:
         CloseHandle(DispatcherThread);
     }
 
+    if (GetCurrentThreadId() == GetThreadId(FileSystem->DispatcherThread))
+    {
+        if (0 != FileSystem->Interface->DispatcherStopped)
+        {
+            /* Normally = !!FileSystem->DispatcherStopping */
+            BOOLEAN Normally = !!(
+                _InterlockedOr16(
+                    (PVOID)((PUINT8)&FileSystem->UmFileContextIsFullContext +
+                        sizeof(FileSystem->UmFileContextIsFullContext)),
+                    0) &
+                0x8000);
+            FileSystem->Interface->DispatcherStopped(FileSystem, Normally);
+        }
+    }
+
     return Result;
 }
 
@@ -399,6 +414,12 @@ FSP_API VOID FspFileSystemStopDispatcher(FSP_FILE_SYSTEM *FileSystem)
     if (0 == FileSystem->DispatcherThread)
         return;
 
+    /* FileSystem->DispatcherStopping = 1 */
+    _InterlockedOr16(
+        (PVOID)((PUINT8)&FileSystem->UmFileContextIsFullContext +
+            sizeof(FileSystem->UmFileContextIsFullContext)),
+        0x8000);
+
     FspFsctlStop0(FileSystem->VolumeHandle);
 
     WaitForSingleObject(FileSystem->DispatcherThread, INFINITE);
@@ -406,6 +427,12 @@ FSP_API VOID FspFileSystemStopDispatcher(FSP_FILE_SYSTEM *FileSystem)
     FileSystem->DispatcherThread = 0;
 
     FspFsctlStop(FileSystem->VolumeHandle);
+
+    /* FileSystem->DispatcherStopping = 0 */
+    _InterlockedAnd16(
+        (PVOID)((PUINT8)&FileSystem->UmFileContextIsFullContext +
+            sizeof(FileSystem->UmFileContextIsFullContext)),
+        0x7fff);
 }
 
 FSP_API VOID FspFileSystemSendResponse(FSP_FILE_SYSTEM *FileSystem,

src/dll/fsctl.c

@@ -32,7 +32,6 @@ static DWORD FspFsctlTransactCode = FSP_FSCTL_TRANSACT;
 static DWORD FspFsctlTransactBatchCode = FSP_FSCTL_TRANSACT_BATCH;
 
 static VOID FspFsctlServiceVersion(PUINT32 PVersion);
-static NTSTATUS FspFsctlStartService(VOID);
 
 FSP_API NTSTATUS FspFsctlCreateVolume(PWSTR DevicePath,
     const FSP_FSCTL_VOLUME_PARAMS *VolumeParams,
@@ -298,6 +297,56 @@ FSP_API NTSTATUS FspFsctlPreflight(PWSTR DevicePath)
     return STATUS_SUCCESS;
 }
 
+static NTSTATUS FspFsctlUnload(PWSTR DevicePath)
+{
+    NTSTATUS Result;
+    PWSTR DeviceRoot;
+    SIZE_T DeviceRootSize, DevicePathSize;
+    WCHAR DevicePathBuf[MAX_PATH], *DevicePathPtr;
+    HANDLE VolumeHandle = INVALID_HANDLE_VALUE;
+    DWORD Bytes;
+
+    /* check lengths; everything must fit within MAX_PATH */
+    DeviceRoot = L'\\' == DevicePath[0] ? GLOBALROOT : GLOBALROOT "\\Device\\";
+    DeviceRootSize = lstrlenW(DeviceRoot) * sizeof(WCHAR);
+    DevicePathSize = lstrlenW(DevicePath) * sizeof(WCHAR);
+    if (DeviceRootSize + DevicePathSize + sizeof(WCHAR) > sizeof DevicePathBuf)
+        return STATUS_INVALID_PARAMETER;
+
+    /* prepare the device path to be opened */
+    DevicePathPtr = DevicePathBuf;
+    memcpy(DevicePathPtr, DeviceRoot, DeviceRootSize);
+    DevicePathPtr = (PVOID)((PUINT8)DevicePathPtr + DeviceRootSize);
+    memcpy(DevicePathPtr, DevicePath, DevicePathSize);
+    DevicePathPtr = (PVOID)((PUINT8)DevicePathPtr + DevicePathSize);
+    *DevicePathPtr = L'\0';
+
+    VolumeHandle = CreateFileW(DevicePathBuf,
+        0, FILE_SHARE_READ | FILE_SHARE_WRITE, 0, OPEN_EXISTING, FILE_FLAG_OVERLAPPED, 0);
+    if (INVALID_HANDLE_VALUE == VolumeHandle)
+    {
+        Result = FspNtStatusFromWin32(GetLastError());
+        if (STATUS_OBJECT_PATH_NOT_FOUND == Result ||
+            STATUS_OBJECT_NAME_NOT_FOUND == Result)
+            Result = STATUS_NO_SUCH_DEVICE;
+        goto exit;
+    }
+
+    if (!DeviceIoControl(VolumeHandle, FSP_FSCTL_UNLOAD, 0, 0, 0, 0, &Bytes, 0))
+    {
+        Result = FspNtStatusFromWin32(GetLastError());
+        goto exit;
+    }
+
+    Result = STATUS_SUCCESS;
+
+exit:
+    if (INVALID_HANDLE_VALUE != VolumeHandle)
+        CloseHandle(VolumeHandle);
+
+    return Result;
+}
+
 static BOOL WINAPI FspFsctlServiceVersionInitialize(
     PINIT_ONCE InitOnce, PVOID Parameter, PVOID *Context)
 {
@@ -373,9 +422,24 @@ static VOID FspFsctlServiceVersion(PUINT32 PVersion)
         *PVersion = FspFsctlServiceVersionValue;
 }
 
-static NTSTATUS FspFsctlStartService(VOID)
+static SRWLOCK FspFsctlStartStopServiceLock = SRWLOCK_INIT;
+
+static BOOLEAN FspFsctlRunningInContainer(VOID)
+{
+    /* Determine if we are running inside container.
+     *
+     * See https://github.com/microsoft/perfview/blob/V1.9.65/src/TraceEvent/TraceEventSession.cs#L525
+     * See https://stackoverflow.com/a/50748300
+     */
+    return ERROR_SUCCESS == RegGetValueW(
+        HKEY_LOCAL_MACHINE, L"SYSTEM\\CurrentControlSet\\Control",
+        L"ContainerType",
+        RRF_RT_REG_DWORD, 0,
+        0, 0);
+}
+
+FSP_API NTSTATUS FspFsctlStartService(VOID)
 {
-    static SRWLOCK Lock = SRWLOCK_INIT;
     WCHAR DriverName[256];
     SC_HANDLE ScmHandle = 0;
     SC_HANDLE SvcHandle = 0;
@@ -385,19 +449,9 @@ static NTSTATUS FspFsctlStartService(VOID)
 
     wsprintfW(DriverName, L"" FSP_FSCTL_DRIVER_NAME "%s", FspSxsSuffix());
 
-    AcquireSRWLockExclusive(&Lock);
+    AcquireSRWLockExclusive(&FspFsctlStartStopServiceLock);
 
-    /* Determine if we are running inside container.
+    if (FspFsctlRunningInContainer())
-     *
-     * See https://github.com/microsoft/perfview/blob/V1.9.65/src/TraceEvent/TraceEventSession.cs#L525
-     * See https://stackoverflow.com/a/50748300
-     */
-    LastError = RegGetValueW(
-        HKEY_LOCAL_MACHINE, L"SYSTEM\\CurrentControlSet\\Control",
-        L"ContainerType",
-        RRF_RT_REG_DWORD, 0,
-        0, 0);
-    if (ERROR_SUCCESS == LastError)
     {
         Result = STATUS_SUCCESS;
         goto exit;
@@ -457,7 +511,97 @@ exit:
     if (0 != ScmHandle)
         CloseServiceHandle(ScmHandle);
 
-    ReleaseSRWLockExclusive(&Lock);
+    ReleaseSRWLockExclusive(&FspFsctlStartStopServiceLock);
+
+    return Result;
+}
+
+FSP_API NTSTATUS FspFsctlStopService(VOID)
+{
+    WCHAR DriverName[256];
+    HANDLE ThreadToken = 0, ProcessToken = 0;
+    BOOL DidSetThreadToken = FALSE, DidAdjustTokenPrivileges = FALSE;
+    TOKEN_PRIVILEGES Privileges, PreviousPrivileges;
+    PRIVILEGE_SET RequiredPrivileges;
+    DWORD PreviousPrivilegesLength;
+    BOOL PrivilegeCheckResult;
+    NTSTATUS Result;
+
+    wsprintfW(DriverName, L"" FSP_FSCTL_DRIVER_NAME "%s", FspSxsSuffix());
+
+    AcquireSRWLockExclusive(&FspFsctlStartStopServiceLock);
+
+    if (FspFsctlRunningInContainer())
+    {
+        Result = STATUS_SUCCESS;
+        goto exit;
+    }
+
+    /* enable and check SeLoadDriverPrivilege required for FSP_FSCTL_UNLOAD */
+    if (!OpenThreadToken(GetCurrentThread(), TOKEN_QUERY | TOKEN_ADJUST_PRIVILEGES, TRUE, &ThreadToken))
+    {
+        if (!OpenProcessToken(GetCurrentProcess(), MAXIMUM_ALLOWED, &ProcessToken) ||
+            !DuplicateToken(ProcessToken, SecurityDelegation, &ThreadToken) ||
+            !SetThreadToken(0, ThreadToken))
+        {
+            Result = FspNtStatusFromWin32(GetLastError());
+            goto exit;
+        }
+        DidSetThreadToken = TRUE;
+        CloseHandle(ThreadToken);
+        ThreadToken = 0;
+        if (!OpenThreadToken(GetCurrentThread(), TOKEN_QUERY | TOKEN_ADJUST_PRIVILEGES, TRUE, &ThreadToken))
+        {
+            Result = FspNtStatusFromWin32(GetLastError());
+            goto exit;
+        }
+    }
+    if (!LookupPrivilegeValueW(0, SE_LOAD_DRIVER_NAME, &Privileges.Privileges[0].Luid))
+    {
+        Result = FspNtStatusFromWin32(GetLastError());
+        goto exit;
+    }
+    Privileges.PrivilegeCount = 1;
+    Privileges.Privileges[0].Attributes = SE_PRIVILEGE_ENABLED;
+    if (!AdjustTokenPrivileges(ThreadToken, FALSE,
+        &Privileges, sizeof PreviousPrivileges, &PreviousPrivileges, &PreviousPrivilegesLength))
+    {
+        Result = FspNtStatusFromWin32(GetLastError());
+        goto exit;
+    }
+    DidAdjustTokenPrivileges = 0 == GetLastError();
+    RequiredPrivileges.PrivilegeCount = 1;
+    RequiredPrivileges.Control = PRIVILEGE_SET_ALL_NECESSARY;
+    RequiredPrivileges.Privilege[0].Attributes = 0;
+    RequiredPrivileges.Privilege[0].Luid = Privileges.Privileges[0].Luid;
+    if (!PrivilegeCheck(ThreadToken, &RequiredPrivileges, &PrivilegeCheckResult))
+    {
+        Result = FspNtStatusFromWin32(GetLastError());
+        goto exit;
+    }
+    if (!PrivilegeCheckResult)
+    {
+        Result = STATUS_PRIVILEGE_NOT_HELD;
+        goto exit;
+    }
+
+    Result = FspFsctlUnload(L"" FSP_FSCTL_DISK_DEVICE_NAME);
+    if (!NT_SUCCESS(Result) && STATUS_NO_SUCH_DEVICE != Result)
+        goto exit;
+
+    Result = STATUS_SUCCESS;
+
+exit:
+    if (DidAdjustTokenPrivileges)
+        AdjustTokenPrivileges(ThreadToken, FALSE, &PreviousPrivileges, 0, 0, 0);
+    if (DidSetThreadToken)
+        SetThreadToken(0, 0);
+    if (0 != ThreadToken)
+        CloseHandle(ThreadToken);
+    if (0 != ProcessToken)
+        CloseHandle(ProcessToken);
+
+    ReleaseSRWLockExclusive(&FspFsctlStartStopServiceLock);
 
     return Result;
 }
@@ -465,7 +609,9 @@ exit:
 static NTSTATUS FspFsctlFixServiceSecurity(HANDLE SvcHandle)
 {
     /*
-     * This function adds an ACE that allows Everyone to start a service.
+     * This function adds two ACE's:
+     * - An ACE that allows Everyone to start a service.
+     * - An ACE that denies Everyone (including Administrators) to stop a service.
      */
 
     PSID WorldSid;

src/sys/device.c

@@ -30,6 +30,7 @@ NTSTATUS FspDeviceCreate(UINT32 Kind, ULONG ExtraSize,
     PDEVICE_OBJECT *PDeviceObject);
 NTSTATUS FspDeviceInitialize(PDEVICE_OBJECT DeviceObject);
 VOID FspDeviceDelete(PDEVICE_OBJECT DeviceObject);
+VOID FspDeviceDoIoDeleteDevice(PDEVICE_OBJECT DeviceObject);
 BOOLEAN FspDeviceReference(PDEVICE_OBJECT DeviceObject);
 VOID FspDeviceDereference(PDEVICE_OBJECT DeviceObject);
 _IRQL_requires_(DISPATCH_LEVEL)
@@ -67,13 +68,13 @@ NTSTATUS FspDeviceCopyList(
     PDEVICE_OBJECT **PDeviceObjects, PULONG PDeviceObjectCount);
 VOID FspDeviceDeleteList(
     PDEVICE_OBJECT *DeviceObjects, ULONG DeviceObjectCount);
-VOID FspDeviceDeleteAll(VOID);
 
 #ifdef ALLOC_PRAGMA
 #pragma alloc_text(PAGE, FspDeviceCreateSecure)
 #pragma alloc_text(PAGE, FspDeviceCreate)
 #pragma alloc_text(PAGE, FspDeviceInitialize)
 #pragma alloc_text(PAGE, FspDeviceDelete)
+#pragma alloc_text(PAGE, FspDeviceDoIoDeleteDevice)
 #pragma alloc_text(PAGE, FspFsvolDeviceInit)
 #pragma alloc_text(PAGE, FspFsvolDeviceFini)
 #pragma alloc_text(PAGE, FspFsvolDeviceCopyContextList)
@@ -92,7 +93,6 @@ VOID FspDeviceDeleteAll(VOID);
 #pragma alloc_text(PAGE, FspFsmupDeviceFini)
 #pragma alloc_text(PAGE, FspDeviceCopyList)
 #pragma alloc_text(PAGE, FspDeviceDeleteList)
-#pragma alloc_text(PAGE, FspDeviceDeleteAll)
 #endif
 
 NTSTATUS FspDeviceCreateSecure(UINT32 Kind, ULONG ExtraSize,
@@ -218,13 +218,23 @@ VOID FspDeviceDelete(PDEVICE_OBJECT DeviceObject)
         return;
     }
 
+    FspDeviceDoIoDeleteDevice(DeviceObject);
+
 #if DBG
 #pragma prefast(suppress:28175, "Debugging only: ok to access DeviceObject->Size")
     RtlFillMemory(&DeviceExtension->Kind,
         (PUINT8)DeviceObject + DeviceObject->Size - (PUINT8)&DeviceExtension->Kind, 0xBD);
 #endif
+}
 
-    IoDeleteDevice(DeviceObject);
+VOID FspDeviceDoIoDeleteDevice(PDEVICE_OBJECT DeviceObject)
+{
+    PAGED_CODE();
+
+    FSP_DEVICE_EXTENSION *DeviceExtension = FspDeviceExtension(DeviceObject);
+
+    if (0 == InterlockedCompareExchange(&DeviceExtension->DidIoDeleteDevice, 1, 0))
+        IoDeleteDevice(DeviceObject);
 }
 
 BOOLEAN FspDeviceReference(PDEVICE_OBJECT DeviceObject)
@@ -942,22 +952,4 @@ VOID FspDeviceDeleteList(
     FspFree(DeviceObjects);
 }
 
-VOID FspDeviceDeleteAll(VOID)
-{
-    PAGED_CODE();
-
-    NTSTATUS Result;
-    PDEVICE_OBJECT *DeviceObjects = 0;
-    ULONG DeviceObjectCount = 0;
-
-    Result = FspDeviceCopyList(&DeviceObjects, &DeviceObjectCount);
-    if (!NT_SUCCESS(Result))
-        return;
-
-    for (ULONG i = 0; DeviceObjectCount > i; i++)
-        FspDeviceDelete(DeviceObjects[i]);
-
-    FspDeviceDeleteList(DeviceObjects, DeviceObjectCount);
-}
-
 FAST_MUTEX FspDeviceGlobalMutex;

src/sys/devtimer.c

@@ -55,6 +55,7 @@ NTSTATUS FspDeviceInitializeAllTimers(VOID)
 VOID FspDeviceFinalizeAllTimers(VOID)
 {
     KeCancelTimer(&FspDeviceTimer);
+    KeFlushQueuedDpcs();
 
 #if DBG
     KIRQL Irql;

src/sys/driver.c

@@ -21,27 +21,23 @@
 
 #include <sys/driver.h>
 
-/*
- * Define the following macro to include FspUnload and make the driver unloadable.
- */
-#define FSP_UNLOAD
-
 DRIVER_INITIALIZE DriverEntry;
-#if defined(FSP_UNLOAD)
+static DRIVER_UNLOAD DriverUnload;
-DRIVER_UNLOAD FspUnload;
-#endif
 static VOID FspDriverMultiVersionInitialize(VOID);
 static NTSTATUS FspDriverInitializeDevices(VOID);
 static VOID FspDriverFinalizeDevices(VOID);
+static VOID FspDriverFinalizeDevicesEx(BOOLEAN DeleteDevices);
+NTSTATUS FspDriverUnload(
+    PDEVICE_OBJECT FsctlDeviceObject, PIRP Irp, PIO_STACK_LOCATION IrpSp);
 
 #ifdef ALLOC_PRAGMA
 #pragma alloc_text(INIT, DriverEntry)
-#if defined(FSP_UNLOAD)
+#pragma alloc_text(PAGE, DriverUnload)
-#pragma alloc_text(PAGE, FspUnload)
-#endif
 #pragma alloc_text(INIT, FspDriverMultiVersionInitialize)
 #pragma alloc_text(PAGE, FspDriverInitializeDevices)
 #pragma alloc_text(PAGE, FspDriverFinalizeDevices)
+#pragma alloc_text(PAGE, FspDriverFinalizeDevicesEx)
+#pragma alloc_text(PAGE, FspDriverUnload)
 #endif
 
 NTSTATUS DriverEntry(
@@ -54,9 +50,7 @@ NTSTATUS DriverEntry(
     FspSxsIdentInitialize(&DriverObject->DriverName);
 
     /* setup the driver object */
-#if defined(FSP_UNLOAD)
+    DriverObject->DriverUnload = DriverUnload;
-    DriverObject->DriverUnload = FspUnload;
-#endif
     DriverObject->MajorFunction[IRP_MJ_CREATE] = FspCreate;
     DriverObject->MajorFunction[IRP_MJ_CLOSE] = FspClose;
     DriverObject->MajorFunction[IRP_MJ_READ] = FspRead;
@@ -144,6 +138,7 @@ NTSTATUS DriverEntry(
     FspDriverObject = DriverObject;
     FspDriverMultiVersionInitialize();
 
+    ExInitializeFastMutex(&FspDriverUnloadMutex);
     ExInitializeFastMutex(&FspDeviceGlobalMutex);
 
     Result = FspSiloInitialize(FspDriverInitializeDevices, FspDriverFinalizeDevices);
@@ -188,12 +183,13 @@ exit:
         &DriverObject->DriverName, RegistryPath);
 }
 
-#if defined(FSP_UNLOAD)
+VOID DriverUnload(
-VOID FspUnload(
     PDRIVER_OBJECT DriverObject)
 {
     FSP_ENTER_VOID(PAGED_CODE());
 
+    FspDriverFinalizeDevices();
+
     FspDeviceFinalizeAllTimers();
 
     FspProcessBufferFinalize();
@@ -206,7 +202,6 @@ VOID FspUnload(
     FSP_LEAVE_VOID("DriverName=\"%wZ\"",
         &DriverObject->DriverName);
 }
-#endif
 
 static VOID FspDriverMultiVersionInitialize(VOID)
 {
@@ -342,12 +337,25 @@ static NTSTATUS FspDriverInitializeDevices(VOID)
      * as a file system; we register with the MUP instead.
      */
     IoRegisterFileSystem(Globals->FsctlDiskDeviceObject);
+    Globals->InitDoneRegisterDisk = 1;
+
+    /*
+     * Reference primary device objects to allow for IoDeleteDevice during FspDriverUnload.
+     */
+    ObReferenceObject(Globals->FsctlDiskDeviceObject);
+    ObReferenceObject(Globals->FsctlNetDeviceObject);
+    ObReferenceObject(Globals->FsmupDeviceObject);
 
     Result = STATUS_SUCCESS;
 
 exit:
     if (!NT_SUCCESS(Result))
     {
+        if (Globals->InitDoneRegisterDisk)
+        {
+            IoUnregisterFileSystem(Globals->FsctlDiskDeviceObject);
+            Globals->InitDoneRegisterDisk = 0;
+        }
         if (0 != Globals->MupHandle)
         {
             FsRtlDeregisterUncProvider(Globals->MupHandle);
@@ -391,14 +399,24 @@ static VOID FspDriverFinalizeDevices(VOID)
 {
     PAGED_CODE();
 
+    FspDriverFinalizeDevicesEx(TRUE);
+}
+
+static VOID FspDriverFinalizeDevicesEx(BOOLEAN DeleteDevices)
+{
+    PAGED_CODE();
+
     FSP_SILO_GLOBALS *Globals;
     UNICODE_STRING SymlinkName;
 
     FspSiloGetGlobals(&Globals);
     ASSERT(0 != Globals);
 
-    IoUnregisterFileSystem(Globals->FsctlDiskDeviceObject);
+    if (Globals->InitDoneRegisterDisk)
-
+    {
+        IoUnregisterFileSystem(Globals->FsctlDiskDeviceObject);
+        Globals->InitDoneRegisterDisk = 0;
+    }
     if (0 != Globals->MupHandle)
     {
         FsRtlDeregisterUncProvider(Globals->MupHandle);
@@ -406,8 +424,14 @@ static VOID FspDriverFinalizeDevices(VOID)
     }
     if (0 != Globals->FsmupDeviceObject)
     {
-        FspDeviceDelete(Globals->FsmupDeviceObject);
+        if (DeleteDevices)
-        Globals->FsmupDeviceObject = 0;
+        {
+            FspDeviceDelete(Globals->FsmupDeviceObject);
+            ObDereferenceObject(Globals->FsmupDeviceObject);
+            Globals->FsmupDeviceObject = 0;
+        }
+        else
+            FspDeviceDoIoDeleteDevice(Globals->FsmupDeviceObject);
     }
     if (Globals->InitDoneSymlinkNet)
     {
@@ -417,8 +441,14 @@ static VOID FspDriverFinalizeDevices(VOID)
     }
     if (0 != Globals->FsctlNetDeviceObject)
     {
-        FspDeviceDelete(Globals->FsctlNetDeviceObject);
+        if (DeleteDevices)
-        Globals->FsctlNetDeviceObject = 0;
+        {
+            FspDeviceDelete(Globals->FsctlNetDeviceObject);
+            ObDereferenceObject(Globals->FsctlNetDeviceObject);
+            Globals->FsctlNetDeviceObject = 0;
+        }
+        else
+            FspDeviceDoIoDeleteDevice(Globals->FsctlNetDeviceObject);
     }
     if (Globals->InitDoneSymlinkDisk)
     {
@@ -428,16 +458,84 @@ static VOID FspDriverFinalizeDevices(VOID)
     }
     if (0 != Globals->FsctlDiskDeviceObject)
     {
-        FspDeviceDelete(Globals->FsctlDiskDeviceObject);
+        if (DeleteDevices)
-        Globals->FsctlDiskDeviceObject = 0;
+        {
+            FspDeviceDelete(Globals->FsctlDiskDeviceObject);
+            ObDereferenceObject(Globals->FsctlDiskDeviceObject);
+            Globals->FsctlDiskDeviceObject = 0;
+        }
+        else
+            FspDeviceDoIoDeleteDevice(Globals->FsctlDiskDeviceObject);
     }
 
     FspSiloDereferenceGlobals(Globals);
 }
 
+NTSTATUS FspDriverUnload(
+    PDEVICE_OBJECT FsctlDeviceObject, PIRP Irp, PIO_STACK_LOCATION IrpSp)
+{
+    PAGED_CODE();
+
+    ASSERT(IRP_MJ_FILE_SYSTEM_CONTROL == IrpSp->MajorFunction);
+    ASSERT(IRP_MN_USER_FS_REQUEST == IrpSp->MinorFunction);
+    ASSERT(FSP_FSCTL_UNLOAD == IrpSp->Parameters.FileSystemControl.FsControlCode);
+
+    NTSTATUS Result;
+    UNICODE_STRING DriverServiceName, DriverName, Remain;
+    WCHAR DriverServiceNameBuf[64 + 256];
+    PDEVICE_OBJECT *DeviceObjects = 0;
+    ULONG DeviceObjectCount = 0;
+
+    if (!FspSiloIsHost())
+        return STATUS_INVALID_DEVICE_REQUEST;
+
+    if (!SeSinglePrivilegeCheck(RtlConvertLongToLuid(SE_LOAD_DRIVER_PRIVILEGE), UserMode))
+        return STATUS_PRIVILEGE_NOT_HELD;
+
+    ExAcquireFastMutexUnsafe(&FspDriverUnloadMutex);
+
+    if (!FspDriverUnloadDone)
+    {
+        FspFileNameSuffix(&FspDriverObject->DriverName, &Remain, &DriverName);
+        RtlInitEmptyUnicodeString(&DriverServiceName, DriverServiceNameBuf, sizeof DriverServiceNameBuf);
+        Result = RtlUnicodeStringPrintf(&DriverServiceName,
+            L"\\Registry\\Machine\\System\\CurrentControlSet\\Services\\%wZ", &DriverName);
+        if (!NT_SUCCESS(Result))
+            goto exit;
+        Result = ZwUnloadDriver(&DriverServiceName);
+        if (!NT_SUCCESS(Result))
+            goto exit;
+
+        FspDriverFinalizeDevicesEx(FALSE);
+
+        Result = FspDeviceCopyList(&DeviceObjects, &DeviceObjectCount);
+        if (NT_SUCCESS(Result))
+        {
+            for (ULONG I = 0; DeviceObjectCount > I; I++)
+            {
+                FSP_DEVICE_EXTENSION *DeviceExtension = FspDeviceExtension(DeviceObjects[I]);
+                if (FspFsvolDeviceExtensionKind == DeviceExtension->Kind)
+                    FspIoqStop(((FSP_FSVOL_DEVICE_EXTENSION *)DeviceExtension)->Ioq, FALSE);
+            }
+            FspDeviceDeleteList(DeviceObjects, DeviceObjectCount);
+        }
+
+        FspDriverUnloadDone = TRUE;
+    }
+
+    Result = STATUS_SUCCESS;
+
+exit:
+    ExReleaseFastMutexUnsafe(&FspDriverUnloadMutex);
+
+    return Result;
+}
+
 PDRIVER_OBJECT FspDriverObject;
 FAST_IO_DISPATCH FspFastIoDispatch;
 CACHE_MANAGER_CALLBACKS FspCacheManagerCallbacks;
+FAST_MUTEX FspDriverUnloadMutex;
+BOOLEAN FspDriverUnloadDone;
 
 ULONG FspProcessorCount;
 FSP_MV_CcCoherencyFlushAndPurgeCache *FspMvCcCoherencyFlushAndPurgeCache;

src/sys/driver.h

@@ -364,6 +364,10 @@ VOID FspTraceNtStatus(const char *file, int line, const char *func, NTSTATUS Sta
 /* missing typedef */
 typedef const void *PCVOID;
 
+/* driver unload */
+NTSTATUS FspDriverUnload(
+    PDEVICE_OBJECT FsctlDeviceObject, PIRP Irp, PIO_STACK_LOCATION IrpSp);
+
 /* driver major functions */
 _Function_class_(DRIVER_DISPATCH)
 _IRQL_requires_max_(APC_LEVEL)
@@ -752,10 +756,11 @@ typedef struct
     PDEVICE_OBJECT FsmupDeviceObject;
     HANDLE MupHandle;
     WCHAR FsmupDeviceNameBuf[128];
-    UINT32 InitDoneSymlinkDisk:1, InitDoneSymlinkNet:1;
+    UINT32 InitDoneSymlinkDisk:1, InitDoneSymlinkNet:1, InitDoneRegisterDisk:1;
 } FSP_SILO_GLOBALS;
 typedef NTSTATUS (*FSP_SILO_INIT_CALLBACK)(VOID);
 typedef VOID (*FSP_SILO_FINI_CALLBACK)(VOID);
+BOOLEAN FspSiloIsHost(VOID);
 NTSTATUS FspSiloGetGlobals(FSP_SILO_GLOBALS **PGlobals);
 VOID FspSiloDereferenceGlobals(FSP_SILO_GLOBALS *Globals);
 VOID FspSiloGetContainerId(GUID *ContainerId);
@@ -1188,8 +1193,8 @@ typedef struct
     LONG RefCount;
     UINT32 Kind;
     GUID SiloContainerId;
-    /* IoTimer emulation */
+    FSP_DEVICE_TIMER DeviceTimer;       /* IoTimer emulation */
-    FSP_DEVICE_TIMER DeviceTimer;
+    LONG DidIoDeleteDevice;
 } FSP_DEVICE_EXTENSION;
 typedef struct
 {
@@ -1291,6 +1296,7 @@ NTSTATUS FspDeviceCreate(UINT32 Kind, ULONG ExtraSize,
     PDEVICE_OBJECT *PDeviceObject);
 NTSTATUS FspDeviceInitialize(PDEVICE_OBJECT DeviceObject);
 VOID FspDeviceDelete(PDEVICE_OBJECT DeviceObject);
+VOID FspDeviceDoIoDeleteDevice(PDEVICE_OBJECT DeviceObject);
 BOOLEAN FspDeviceReference(PDEVICE_OBJECT DeviceObject);
 VOID FspDeviceDereference(PDEVICE_OBJECT DeviceObject);
 static inline
@@ -1473,7 +1479,6 @@ NTSTATUS FspDeviceCopyList(
     PDEVICE_OBJECT **PDeviceObjects, PULONG PDeviceObjectCount);
 VOID FspDeviceDeleteList(
     PDEVICE_OBJECT *DeviceObjects, ULONG DeviceObjectCount);
-VOID FspDeviceDeleteAll(VOID);
 NTSTATUS FspDeviceInitializeAllTimers(VOID);
 VOID FspDeviceFinalizeAllTimers(VOID);
 NTSTATUS FspDeviceInitializeTimer(PDEVICE_OBJECT DeviceObject,
@@ -2005,6 +2010,8 @@ FSP_MV_CcCoherencyFlushAndPurgeCache(
 extern PDRIVER_OBJECT FspDriverObject;
 extern FAST_IO_DISPATCH FspFastIoDispatch;
 extern CACHE_MANAGER_CALLBACKS FspCacheManagerCallbacks;
+extern FAST_MUTEX FspDriverUnloadMutex;
+extern BOOLEAN FspDriverUnloadDone;
 extern FSP_IOPREP_DISPATCH *FspIopPrepareFunction[];
 extern FSP_IOCMPL_DISPATCH *FspIopCompleteFunction[];
 extern FAST_MUTEX FspDeviceGlobalMutex;

src/sys/fsctl.c

@@ -128,6 +128,9 @@ static NTSTATUS FspFsctlFileSystemControl(
             if (0 != IrpSp->FileObject->FsContext2)
                 Result = FspVolumeNotify(FsctlDeviceObject, Irp, IrpSp);
             break;
+        case FSP_FSCTL_UNLOAD:
+            Result = FspDriverUnload(FsctlDeviceObject, Irp, IrpSp);
+            break;
         default:
             if (CTL_CODE(0, 0xC00, 0, 0) ==
                 (IrpSp->Parameters.FileSystemControl.FsControlCode & CTL_CODE(0, 0xC00, 0, 0)))

src/sys/psbuffer.c

@@ -123,6 +123,27 @@ NTSTATUS FspProcessBufferInitialize(VOID)
 VOID FspProcessBufferFinalize(VOID)
 {
     PsSetCreateProcessNotifyRoutine(FspProcessBufferNotifyRoutine, TRUE);
+
+    /*
+     * Free any items in our process hash table.
+     * Any virtual memory was released when the corresponding processes went away.
+     */
+    for (ULONG HashIndex = 0; ProcessBufferBucketCount > HashIndex; HashIndex++)
+    {
+        for (FSP_PROCESS_BUFFER_ITEM *Item = ProcessBufferBuckets[HashIndex], *DictNext; Item; Item = DictNext)
+        {
+            for (FSP_PROCESS_BUFFER_LIST_ENTRY *P = Item->BufferList, *Next; P; P = Next)
+            {
+                Next = P->Next;
+                FspFree(P);
+            }
+
+            DictNext = Item->DictNext;
+            FspFree(Item);
+        }
+
+        ProcessBufferBuckets[HashIndex] = 0;
+    }
 }
 
 static VOID FspProcessBufferNotifyRoutine(HANDLE ParentId, HANDLE ProcessId, BOOLEAN Create)

src/sys/silo.c

@@ -123,6 +123,11 @@ static FSP_SILO_GLOBALS FspSiloHostGlobals;
     }
 #define CALL(n)                         (FspSilo ## n)
 
+BOOLEAN FspSiloIsHost(VOID)
+{
+    return !FspSiloInitDone || 0 == CALL(PsGetCurrentServerSilo)();
+}
+
 NTSTATUS FspSiloGetGlobals(FSP_SILO_GLOBALS **PGlobals)
 {
     FSP_PESILO Silo;

tst/memfs/memfs.cpp

@@ -40,6 +40,11 @@ FSP_FSCTL_STATIC_ASSERT(MEMFS_MAX_PATH > MAX_PATH,
  */
 //#define MEMFS_STANDALONE
 
+/*
+ * Define the MEMFS_DISPATCHER_STOPPED macro to include DispatcherStopped support.
+ */
+#define MEMFS_DISPATCHER_STOPPED
+
 /*
  * Define the MEMFS_NAME_NORMALIZATION macro to include name normalization support.
  */
@@ -2269,6 +2274,14 @@ static NTSTATUS SetEa(FSP_FILE_SYSTEM *FileSystem,
 }
 #endif
 
+#if defined(MEMFS_DISPATCHER_STOPPED)
+static void DispatcherStopped(FSP_FILE_SYSTEM *FileSystem,
+    BOOLEAN Normally)
+{
+    //FspDebugLog(__FUNCTION__ ": Normally=%d\n", Normally);
+}
+#endif
+
 static FSP_FILE_SYSTEM_INTERFACE MemfsInterface =
 {
     GetVolumeInfo,
@@ -2336,6 +2349,12 @@ static FSP_FILE_SYSTEM_INTERFACE MemfsInterface =
     0,
     0,
     0,
+#endif
+    0,
+#if defined(MEMFS_DISPATCHER_STOPPED)
+    DispatcherStopped,
+#else
+    0,
 #endif
 };
 

tst/winfsp-tests/loadun-test.c

@@ -0,0 +1,44 @@
+/**
+ * @file loadun-test.c
+ *
+ * @copyright 2015-2022 Bill Zissimopoulos
+ */
+/*
+ * This file is part of WinFsp.
+ *
+ * You can redistribute it and/or modify it under the terms of the GNU
+ * General Public License version 3 as published by the Free Software
+ * Foundation.
+ *
+ * Licensees holding a valid commercial license may use this software
+ * in accordance with the commercial license agreement provided in
+ * conjunction with the software.  The terms and conditions of any such
+ * commercial license agreement shall govern, supersede, and render
+ * ineffective any application of the GPLv3 license to this software,
+ * notwithstanding of any reference thereto in the software or
+ * associated repository.
+ */
+
+#include <winfsp/winfsp.h>
+#include <tlib/testsuite.h>
+
+#include "winfsp-tests.h"
+
+static void load_unload_test(void)
+{
+    /* this is not a real test! */
+
+    FspFsctlStartService();
+    FspFsctlStartService();
+
+    FspFsctlStopService();
+    FspFsctlStopService();
+}
+
+void load_unload_tests(void)
+{
+    if (OptExternal)
+        return;
+
+    TEST_OPT(load_unload_test);
+}

tst/winfsp-tests/winfsp-tests.c

@@ -220,6 +220,7 @@ LONG WINAPI UnhandledExceptionHandler(struct _EXCEPTION_POINTERS *ExceptionInfo)
     argv[argc] = 0
 int main(int argc, char *argv[])
 {
+    TESTSUITE(load_unload_tests);
     TESTSUITE(fuse_opt_tests);
     TESTSUITE(fuse_tests);
     TESTSUITE(posix_tests);