update Changelog for v1.5

Changelog.asciidoc

@@ -1,6 +1,42 @@
 = Changelog
 
 
+v1.5 (2019.3)::
+
+Changes since v1.4:
+
+* [GEN] WinFsp file systems can now be used by WSLinux. File systems must enable this support by setting the `FSP_FSCTL_VOLUME_PARAMS::WslFeatures` bit. Use the command `sudo mount -t drvfs x: /mnt/x` to mount.
+* [GEN] Extended attribute support has been added for all WinFsp API's: native, .NET, FUSE2 and FUSE3.
+* [GEN] Mount Manager support has been added and it works for current and new file systems:
+** If the file system mountpoint is in the syntax `\\.\X:` then the Mount Manager is used.
+** If the file system mountpoint is in the syntax `X:` then `DefineDosDeviceW` is used (i.e. same as today).
+** If the file system mountpoint is in the syntax `X:\DIR` then a reparse point is used and the file system is mounted as a directory (i.e. same as today).
+** Caveats:
+*** It requires Administrator access. This is because opening the `\\.\MountPointManager` device requires Administrator access.
+*** It currently works with drives (`\\.\X:`) but not directories (`\\.\X:\DIR`).
+*** Mount Manager drives created by WinFsp are transient. WinFsp takes various steps to ensure that this is the case.
+*** Mount Manager drives are global and are visible across Terminal Services sessions (they go into the `\GLOBAL??` portion of the NT namespace).
+* [FSD] Support for kernel-mode file systems on top of WinFsp has been added. See `FspFsextProvider`. This is in preparation for WinFuse - FUSE for Windows and WSLinux.
+* [FSD] FastIO support has been added. FastIO operations are enabled on cache-enabled file systems with the notable exception of `FastIoQueryOpen`, which allows opening files in kernel mode; this operation requires the file system to specify the `FSP_FSCTL_VOLUME_PARAMS::AllowOpenInKernelMode` flag.
+* [FSD] Support for `FileFsSectorSizeInformation` and `IOCTL_STORAGE_QUERY_PROPERTY / StorageAccessAlignmentProperty` has been added.
+* [DLL] The `FspFileSystemStartDispatcher` default number of threads (`ThreadCount==0`) has been changed. See commit 3902874ac93fe40685d9761f46a96358ba24f24c for more.
+* [FUSE] FUSE has new `-o UserName=DOMAIN+USERNAME` and `-o GroupName=DOMAIN+GROUPNAME` options. These function like the `-o uid=UID` and `-o gid=GID` options, but accept Windows user and groups names.
+* [FUSE] FUSE has new `-o dothidden` option that is used to add the Windows hidden file attribute to files that start with a dot.
+* [FUSE] FUSE has new `-o create_file_umask=nnn` and `-o create_dir_umask=nnn` options that allow for more control than the `-o create_umask=nnn` option.
+* [FUSE] FUSE has new `--ExactFileSystemName=FSNAME` option that removes the "FUSE-" prefix from the file system name. (Use with caution: see discussion in PR #251.) (Thanks @johntyner.)
+* [.NET] The .NET API now supports asynchronous handling of `Read`, `Write` and `ReadDirectory`. (Thanks @dworkin.)
+* [.NET] The .NET API now supports fine-grained timeouts (`VolumeInfoTimeout`, `DirInfoTimeout`, etc).
+* [.NET] The .NET API has new method `FileSystemHost.MountEx` that adds a `ThreadCount` parameter.
+* [LAUNCH] The Launcher can now rewrite path arguments passed to file systems during launching using "Path Transformation Language". See commit a73f1b95592617ac7484e16c2e642573a4d65644 for more.
+* [MEMFS] A new memfs FUSE3 file system written in C++ has been added. See `tst/memfs-fuse3`.
+* [AIRFS] John Oberschelp has done some fantastic work adding persistence to the airfs file system. (Thanks @JohnOberschelp.)
+* [FIX] Fixes for very large (> 4GiB) files. (Thanks @dworkin.)
+* [FIX] A fix for how FUSE handles the return value from `opendir`. (GitHub issue billziss-gh/sshfs-win#54)
+* [FIX] A fix for an invalid UID to SID mapping on domains with a lot of users. (Thanks @sganis.)
+* [FIX] A fix on the C++ layer. (Thanks @colatkinson.)
+* Other fixes and improvements.
+
+
 v1.5B4 (2019.3 B4)::
 
 Changes since v1.4:

build/VStudio/version.properties

@@ -18,8 +18,8 @@
 
         <MyCanonicalVersion>1.5</MyCanonicalVersion>
 
-        <MyProductVersion>2019.3 B4</MyProductVersion>
+        <MyProductVersion>2019.3</MyProductVersion>
-        <MyProductStage>Beta</MyProductStage>
+        <MyProductStage>Gold</MyProductStage>
 
         <MyVersion>$(MyCanonicalVersion).$(MyBuildNumber)</MyVersion>
         <MyVersionWithCommas>$(MyVersion.Replace('.',',')),0</MyVersionWithCommas>

src/ku/library.h

@@ -207,6 +207,11 @@ static inline int FspKuMultiByteToWideChar(
     return ByteCount / sizeof(WCHAR);
 }
 
+static inline PGENERIC_MAPPING FspGetFileGenericMapping(VOID)
+{
+    return IoGetFileObjectGenericMapping();
+}
+
 static inline void *MemAlloc(size_t Size)
 {
     return FspAlloc(Size);

src/ku/posix.c

@@ -725,6 +725,20 @@ lasterror:
     goto exit;
 }
 
+static inline ACCESS_MASK FspPosixCanonicalizeAccessMask(ACCESS_MASK AccessMask)
+{
+    PGENERIC_MAPPING Mapping = FspGetFileGenericMapping();
+    if (AccessMask & GENERIC_READ)
+        AccessMask |= Mapping->GenericRead;
+    if (AccessMask & GENERIC_WRITE)
+        AccessMask |= Mapping->GenericWrite;
+    if (AccessMask & GENERIC_EXECUTE)
+        AccessMask |= Mapping->GenericExecute;
+    if (AccessMask & GENERIC_ALL)
+        AccessMask |= Mapping->GenericAll;
+    return AccessMask & ~(GENERIC_READ | GENERIC_WRITE | GENERIC_EXECUTE | GENERIC_ALL);
+}
+
 static inline UINT32 FspPosixMapAccessMaskToPermission(ACCESS_MASK AccessMask)
 {
     /* [PERMS]
@@ -749,6 +763,14 @@ FSP_API NTSTATUS FspPosixMapSecurityDescriptorToPermissions(
 {
     FSP_KU_CODE;
 
+    BOOLEAN OwnerOptional = (UINT_PTR)PUid & 1;
+    PUid = (PVOID)((UINT_PTR)PUid & ~1);
+    UINT32 OrigUid = *PUid;
+
+    BOOLEAN GroupOptional = (UINT_PTR)PGid & 1;
+    PGid = (PVOID)((UINT_PTR)PGid & ~1);
+    UINT32 OrigGid = *PGid;
+
     PSID OwnerSid = 0, GroupSid = 0;
     BOOL Defaulted, DaclPresent;
     PACL Acl = 0;
@@ -757,6 +779,7 @@ FSP_API NTSTATUS FspPosixMapSecurityDescriptorToPermissions(
     PSID AceSid;
     DWORD AceAccessMask;
     DWORD OwnerAllow, OwnerDeny, GroupAllow, GroupDeny, WorldAllow, WorldDeny;
+    UINT32 AceUid = 0;
     UINT32 Uid, Gid, Mode;
     NTSTATUS Result;
 
@@ -771,13 +794,23 @@ FSP_API NTSTATUS FspPosixMapSecurityDescriptorToPermissions(
     if (!GetSecurityDescriptorDacl(SecurityDescriptor, &DaclPresent, &Acl, &Defaulted))
         goto lasterror;
 
-    Result = FspPosixMapSidToUid(OwnerSid, &Uid);
+    if (0 == OwnerSid && OwnerOptional)
-    if (!NT_SUCCESS(Result))
+        Uid = OrigUid;
-        goto exit;
+    else
+    {
+        Result = FspPosixMapSidToUid(OwnerSid, &Uid);
+        if (!NT_SUCCESS(Result))
+            goto exit;
+    }
 
-    Result = FspPosixMapSidToUid(GroupSid, &Gid);
+    if (0 == GroupSid && GroupOptional)
-    if (!NT_SUCCESS(Result))
+        Gid = OrigGid;
-        goto exit;
+    else
+    {
+        Result = FspPosixMapSidToUid(GroupSid, &Gid);
+        if (!NT_SUCCESS(Result))
+            goto exit;
+    }
 
     if (0 != Acl)
     {
@@ -810,6 +843,8 @@ FSP_API NTSTATUS FspPosixMapSecurityDescriptorToPermissions(
             else
                 continue;
 
+            AceAccessMask = FspPosixCanonicalizeAccessMask(AceAccessMask);
+
             /* [PERMS]
              * If the ACE contains the Authenticated Users SID or the World SID then
              * add the allowed or denied access right bits into the "owner", "group"
@@ -840,6 +875,9 @@ FSP_API NTSTATUS FspPosixMapSecurityDescriptorToPermissions(
             }
             else
             {
+                if (0 == OwnerSid || 0 == GroupSid)
+                    FspPosixMapSidToUid(AceSid, &AceUid);
+
                 /* [PERMS]
                  * Note that if the file owner and file group SIDs are the same,
                  * then the access rights are saved in both the "owner" and "group"
@@ -851,7 +889,7 @@ FSP_API NTSTATUS FspPosixMapSecurityDescriptorToPermissions(
                  * in the "group" collection as appropriate in the corresponding set of
                  * granted or denied rights (as described above).
                  */
-                if (EqualSid(GroupSid, AceSid))
+                if (0 != GroupSid ? EqualSid(GroupSid, AceSid) : (Gid == AceUid))
                 {
                     if (ACCESS_ALLOWED_ACE_TYPE == Ace->AceType)
                         GroupAllow |= AceAccessMask & ~GroupDeny;
@@ -864,7 +902,7 @@ FSP_API NTSTATUS FspPosixMapSecurityDescriptorToPermissions(
                  * in the "owner" collection as appropriate in the corresponding set of
                  * granted or denied rights (as described above).
                  */
-                if (EqualSid(OwnerSid, AceSid))
+                if (0 != OwnerSid ? EqualSid(OwnerSid, AceSid) : (Uid == AceUid))
                 {
                     if (ACCESS_ALLOWED_ACE_TYPE == Ace->AceType)
                         OwnerAllow |= AceAccessMask & ~OwnerDeny;

src/sys/callbacks.c

@@ -308,7 +308,7 @@ VOID FspPropagateTopFlags(PIRP Irp, PIRP TopLevelIrp)
         {
             DEBUGBREAK_EX(iorecu);
 
-            FspIrpSetTopFlags(Irp, FspIrpFlags(TopLevelIrp));
+            FspIrpSetTopFlags(Irp, FspIrpTopFlags(TopLevelIrp) | FspIrpFlags(TopLevelIrp));
         }
     }
 }

src/sys/file.c

@@ -1337,8 +1337,9 @@ NTSTATUS FspFileNodeRenameCheck(PDEVICE_OBJECT FsvolDeviceObject, PIRP OplockIrp
                 !MmFlushImageSection(&DescendantFileNode->NonPaged->SectionObjectPointers,
                     MmFlushForDelete)))
             {
-                /* release the FileNode in case of failure! */
+                /* release the FileNode and rename lock in case of failure! */
                 FspFileNodeReleaseF(FileNode, AcquireFlags);
+                FspFsvolDeviceFileRenameRelease(FsvolDeviceObject);
 
                 Result = STATUS_ACCESS_DENIED;
                 goto exit;
@@ -1441,8 +1442,9 @@ NTSTATUS FspFileNodeRenameCheck(PDEVICE_OBJECT FsvolDeviceObject, PIRP OplockIrp
 
     if (STATUS_OPLOCK_BREAK_IN_PROGRESS == Result || !NT_SUCCESS(Result))
     {
-        /* release the FileNode so that we can safely wait without deadlocks */
+        /* release the FileNode and rename lock so that we can safely wait without deadlocks */
         FspFileNodeReleaseF(FileNode, AcquireFlags);
+        FspFsvolDeviceFileRenameRelease(FsvolDeviceObject);
 
         /* wait for oplock breaks to finish */
         for (
@@ -1488,8 +1490,9 @@ NTSTATUS FspFileNodeRenameCheck(PDEVICE_OBJECT FsvolDeviceObject, PIRP OplockIrp
 
         if (DescendantFileNode != FileNode && 0 < DescendantFileNode->HandleCount)
         {
-            /* release the FileNode in case of failure! */
+            /* release the FileNode and rename lock in case of failure! */
             FspFileNodeReleaseF(FileNode, AcquireFlags);
+            FspFsvolDeviceFileRenameRelease(FsvolDeviceObject);
 
             Result = STATUS_ACCESS_DENIED;
             break;

src/sys/fileinfo.c

@@ -1576,8 +1576,8 @@ static NTSTATUS FspFsvolSetRenameInformation(
         ASSERT(TargetFileNode->IsDirectory);
     }
 
-    FspFsvolDeviceFileRenameAcquireExclusive(FsvolDeviceObject);
 retry:
+    FspFsvolDeviceFileRenameAcquireExclusive(FsvolDeviceObject);
     FspFileNodeAcquireExclusive(FileNode, Full);
 
     if (0 == Request)
@@ -1651,13 +1651,13 @@ retry:
     Result = FspFileNodeRenameCheck(FsvolDeviceObject, Irp,
         FileNode, FspFileNodeAcquireFull,
         &FileNode->FileName, TRUE);
-    /* FspFileNodeRenameCheck releases FileNode with STATUS_OPLOCK_BREAK_IN_PROGRESS or failure */
+    /* FspFileNodeRenameCheck releases FileNode and rename lock on failure */
     if (STATUS_OPLOCK_BREAK_IN_PROGRESS == Result)
         goto retry;
     if (!NT_SUCCESS(Result))
     {
         Result = STATUS_ACCESS_DENIED;
-        goto rename_unlock_exit;
+        goto exit;
     }
 
     if (0 != FspFileNameCompare(&FileNode->FileName, &NewFileName, !FileDesc->CaseSensitive, 0))
@@ -1665,13 +1665,13 @@ retry:
         Result = FspFileNodeRenameCheck(FsvolDeviceObject, Irp,
             FileNode, FspFileNodeAcquireFull,
             &NewFileName, FALSE);
-        /* FspFileNodeRenameCheck releases FileNode with STATUS_OPLOCK_BREAK_IN_PROGRESS or failure */
+        /* FspFileNodeRenameCheck releases FileNode and rename lock on failure */
         if (STATUS_OPLOCK_BREAK_IN_PROGRESS == Result)
             goto retry;
         if (!NT_SUCCESS(Result))
         {
             Result = STATUS_ACCESS_DENIED;
-            goto rename_unlock_exit;
+            goto exit;
         }
     }
     else
@@ -1713,9 +1713,9 @@ retry:
 
 unlock_exit:
     FspFileNodeRelease(FileNode, Full);
-rename_unlock_exit:
     FspFsvolDeviceFileRenameRelease(FsvolDeviceObject);
 
+exit:
     return Result;
 }
 

tools/build.bat

@@ -86,6 +86,13 @@ if not %signfail%==0 echo SIGNING FAILED! The product has been successfully buil
 for %%f in (build\%Configuration%\winfsp-*.msi) do set Version=%%~nf
 set Version=!Version:winfsp-=!
 
+if X%SignedPackage%==X (
+    pushd build\%Configuration%
+    powershell -command "Compress-Archive -Path winfsp-tests-*.exe,..\..\..\..\License.txt,..\..\..\..\tst\winfsp-tests\README.md -DestinationPath winfsp-tests-!Version!.zip"
+    if errorlevel 1 goto fail
+    popd
+)
+
 where /q choco.exe
 if %ERRORLEVEL% equ 0 (
     copy ..\choco\* build\%Configuration%
@@ -95,11 +102,6 @@ if %ERRORLEVEL% equ 0 (
     if errorlevel 1 goto fail
 )
 
-pushd build\%Configuration%
-powershell -command "Compress-Archive -Path winfsp-tests-*.exe,..\..\..\..\License.txt,..\..\..\..\tst\winfsp-tests\README.md -DestinationPath winfsp-tests-!Version!.zip"
-if errorlevel 1 goto fail
-popd
-
 exit /b 0
 
 :fail

tst/winfsp-tests/resilient.c

@@ -134,7 +134,9 @@ BOOL WINAPI ResilientRemoveDirectoryW(
     else
     {
         for (ULONG MaxTries = DeleteMaxTries;
-            !Success && ERROR_SHARING_VIOLATION == GetLastError() && 0 != MaxTries;
+            !Success &&
+            (ERROR_SHARING_VIOLATION == GetLastError() || ERROR_DIR_NOT_EMPTY == GetLastError()) &&
+            0 != MaxTries;
             MaxTries--)
         {
             Sleep(DeleteSleepTimeout);