build,changelog: version 2020.2 Gold

Changelog.asciidoc

@@ -1,6 +1,48 @@
 = Changelog
 
 
+v1.8 (2020.2)::
+
+Changes since v1.7:
+
+* [FSD] WinFsp now supports Windows containers. See the link:doc/WinFsp-Container-Support.asciidoc[WinFsp Container Support] document for details.
+
+* [FSD] The `FSP_FSCTL_QUERY_WINFSP` code provides a simple method to determine if
+the file system backing a file is a WinFsp file system. To use issue a
++
+----
+DeviceIoControl(Handle, FSP_FSCTL_QUERY_WINFSP, 0, 0, 0, 0, &Bytes, 0)
+----
++
+If the return value is TRUE this is a WinFsp file system.
+
+* [FSD] A fix regarding concurrency of READs on the same file: WinFsp was supposed to allow concurrent READ requests on the same file (e.g. two concurrent overlapped `ReadFile` requests on the same `HANDLE`) to be handled concurrently by the file system; unfortunately due to a problem in recent versions of WinFsp READ requests on the same file were serialized. This problem has now been fixed. See GitHub issue #291 for more details.
+** *NOTE*: It may be that some file system was inadvertently relying on WinFsp's implicit serialization of READs in this case. Please test your file system thoroughly against this version, especially with regard to READ serialization. Related XKCD: https://imgs.xkcd.com/comics/workflow.png
+
+* [FSD] When renaming files or directories NTFS allows the target name to contain a backslash at the end (even for files!) whereas WinFsp did not. This problem has been fixed and a test has been added in `winfsp-tests`.
+
+
+v1.8B3 (2020.2 B2)::
+
+Changes since v1.7:
+
+* [FSD] WinFsp now supports Windows containers. See the link:doc/WinFsp-Container-Support.asciidoc[WinFsp Container Support] document for details.
+
+* [FSD] The `FSP_FSCTL_QUERY_WINFSP` code provides a simple method to determine if
+the file system backing a file is a WinFsp file system. To use issue a
++
+----
+DeviceIoControl(Handle, FSP_FSCTL_QUERY_WINFSP, 0, 0, 0, 0, &Bytes, 0)
+----
++
+If the return value is TRUE this is a WinFsp file system.
+
+* [FSD] A fix regarding concurrency of READs on the same file: WinFsp was supposed to allow concurrent READ requests on the same file (e.g. two concurrent overlapped `ReadFile` requests on the same `HANDLE`) to be handled concurrently by the file system; unfortunately due to a problem in recent versions of WinFsp READ requests on the same file were serialized. This problem has now been fixed. See GitHub issue #291 for more details.
+** *NOTE*: It may be that some file system was inadvertently relying on WinFsp's implicit serialization of READs in this case. Please test your file system thoroughly against this version, especially with regard to READ serialization. Related XKCD: https://imgs.xkcd.com/comics/workflow.png
+
+* [FSD] When renaming files or directories NTFS allows the target name to contain a backslash at the end (even for files!) whereas WinFsp did not. This problem has been fixed and a test has been added in `winfsp-tests`.
+
+
 v1.8B2 (2020.2 B2)::
 
 Changes since v1.7:

build/VStudio/version.properties

@@ -18,8 +18,8 @@
 
         <MyCanonicalVersion>1.8</MyCanonicalVersion>
 
-        <MyProductVersion>2020.2 Beta2</MyProductVersion>
-        <MyProductStage>Beta</MyProductStage>
+        <MyProductVersion>2020.2</MyProductVersion>
+        <MyProductStage>Gold</MyProductStage>
 
         <MyVersion>$(MyCanonicalVersion).$(MyBuildNumber)</MyVersion>
         <MyVersionWithCommas>$(MyVersion.Replace('.',',')),0</MyVersionWithCommas>

src/sys/fileinfo.c

@@ -1589,6 +1589,10 @@ retry:
 
         Suffix.Length = (USHORT)Info->FileNameLength;
         Suffix.Buffer = Info->FileName;
+        /* remove any trailing backslash; NTFS allows it for both directories AND files! */
+        if (sizeof(WCHAR) * 2/* not empty or root */ <= Suffix.Length &&
+            L'\\' == Suffix.Buffer[Suffix.Length / sizeof(WCHAR) - 1])
+            Suffix.Length -= sizeof(WCHAR);
         /* if there is a backslash anywhere in the NewFileName get its suffix */
         for (PWSTR P = Suffix.Buffer, EndP = P + Suffix.Length / sizeof(WCHAR); EndP > P; P++)
             if (L'\\' == *P)

tools/parselog.nim

@@ -0,0 +1,345 @@
+# @file parselog.nim
+#
+# parse WinFsp debug logs
+#
+# @copyright 2015-2020 Bill Zissimopoulos
+
+import algorithm
+import macros
+import parseopt
+import parseutils
+import strformat
+import strscans
+import strutils
+import tables
+
+type
+    Req = ref object
+        fsname: string
+        tid: uint32
+        irp: uint64
+        op: string
+        inform: string
+        context: string
+        args: seq[(string, string)]
+    Rsp = ref object
+        fsname: string
+        tid: uint32
+        irp: uint64
+        op: string
+        status: uint32
+        inform: uint
+        context: string
+        args: seq[(string, string)]
+
+proc parseAndAddArg(prefix: string, rest: var string, args: var seq[(string, string)]) =
+    discard scanf(rest, ",$*", rest)
+    discard scanf(rest, "$s$*", rest)
+    var n, v: string
+    if scanf(rest, "\"$*\"$*", v, rest):
+        args.add((prefix & "", v))
+    elif scanf(rest, "$w=\"$*\"$*", n, v, rest):
+        args.add((prefix & n, v))
+    elif scanf(rest, "$w={$*}$*", n, v, rest):
+        while "" != v:
+            parseAndAddArg(n & ".", v, args)
+    elif scanf(rest, "$w=$*,$*", n, v, rest):
+        args.add((prefix & n, v))
+    elif scanf(rest, "$w=$*$.", n, v):
+        rest = ""
+        args.add((prefix & n, v))
+    elif scanf(rest, "$*,$*", v, rest):
+        args.add((prefix & "", v))
+    else:
+        v = rest
+        rest = ""
+        args.add((prefix & "", v))
+
+proc parseArgs(rest: var string): seq[(string, string)] =
+    while "" != rest:
+        parseAndAddArg("", rest, result)
+
+proc parseReq(op, rest: string): Req =
+    result = Req(op: op)
+    var rest = rest
+    var inform: string
+    if scanf(rest, "[$+]$s$*", inform, rest):
+        result.inform = inform
+    var c0, c1: uint64
+    if scanf(rest, "${parseHex[uint64]}:${parseHex[uint64]}$*", c0, c1, rest):
+        result.context = toHex(c0) & ":" & toHex(c1)
+    result.args = parseArgs(rest)
+
+proc parseRsp(op, rest: string): Rsp =
+    result = Rsp(op: op)
+    var rest = rest
+    var status: uint32
+    var inform: uint
+    if scanf(rest, "IoStatus=${parseHex[uint32]}[${parseUint}]$s$*", status, inform, rest):
+        result.status = status
+        result.inform = inform
+    var c0, c1: uint64
+    if scanf(rest, "UserContext=${parseHex[uint64]}:${parseHex[uint64]}$*", c0, c1, rest):
+        result.context = toHex(c0) & ":" & toHex(c1)
+    result.args = parseArgs(rest)
+
+proc parseLog(path: string, processReq: proc(req: Req), processRsp: proc(rsp: Rsp)) =
+    let file = open(path)
+    defer: file.close()
+    var lineno = 0
+    try:
+        for line in lines file:
+            inc lineno
+            var fsname, dir, op, rest: string
+            var tid: uint32
+            var irp: uint64
+            var req: Req
+            var rsp: Rsp
+            if scanf(line, "$+[TID=${parseHex[uint32]}]:$s${parseHex[uint64]}:$s$+ $*",
+                fsname, tid, irp, op, rest):
+                dir = op[0..1]
+                op = op[2..^1]
+                case dir
+                of ">>":
+                    req = parseReq(op, rest)
+                    req.fsname = fsname
+                    req.tid = tid
+                    req.irp = irp
+                    processReq(req)
+                of "<<":
+                    rsp = parseRsp(op, rest)
+                    rsp.fsname = fsname
+                    rsp.tid = tid
+                    rsp.irp = irp
+                    processRsp(rsp)
+                else:
+                    continue
+    except:
+        echo &"An exception has occurred while parsing file {path} line {lineno}"
+        raise
+
+type
+    Stat = ref object
+        ototal: int                     # open total
+        omulti: int                     # multiplicate open total
+        oerror: int                     # open error total
+        rtotal: int                     # read total
+        rnoaln: int                     # non-aligned read total
+        rbytes: uint64                  # read bytes
+        rerror: int                     # read error total
+        wtotal: int                     # write total
+        wnoaln: int                     # non-aligned write total
+        wbytes: uint64                  # write bytes
+        werror: int                     # write error total
+        dtotal: int                     # query directory total
+        dbytes: uint64                  # query directory bytes
+        derror: int                     # query directory error total
+        ptotal: int                     # query directory w/ pattern total
+        pbytes: uint64                  # query directory w/ pattern bytes
+        perror: int                     # query directory w/ pattern error total
+        ocount: int                     # current open count
+var
+    reqtab = newTable[uint64, Req]()
+    filetab = newTable[string, string]()
+    stattab = newOrderedTable[string, Stat]()
+    aggr = Stat()
+
+proc getArg(args: seq[(string, string)], name: string): string =
+    for n, v in items(args):
+        if name == n:
+            return v
+
+proc processReq(req: Req) =
+    reqtab[req.irp] = req
+    case req.op
+    of "Close":
+        var filename: string
+        if filetab.pop(req.context, filename):
+            var stat = stattab.mgetOrPut(filename, Stat())
+            stat.ocount -= 1
+
+proc processRsp(rsp: Rsp) =
+    var req: Req
+    if reqtab.pop(rsp.irp, req):
+        doAssert req.op == rsp.op
+        doAssert req.irp == rsp.irp
+        case req.op
+        of "Create":
+            var filename = getArg(req.args, "")
+            if "" != filename:
+                if 0 == rsp.status:
+                    filetab[rsp.context] = filename
+                    var stat = stattab.mgetOrPut(filename, Stat())
+                    stat.ototal += 1
+                    aggr.ototal += 1
+                    stat.ocount += 1
+                    if 2 == stat.ocount:
+                        stat.omulti += 1
+                        aggr.omulti += 1
+                else:
+                    var stat = stattab.mgetOrPut(filename, Stat())
+                    stat.oerror += 1
+                    aggr.oerror += 1
+        of "Read":
+            var filename = filetab[req.context]
+            var stat = stattab.mgetOrPut(filename, Stat())
+            if 0 == rsp.status or 0xC0000011u32 == rsp.status:
+                var oarg = getArg(req.args, "Offset")
+                var larg = getArg(req.args, "Length")
+                var hi, lo: uint32
+                var offset: uint64
+                var length: uint
+                if scanf(oarg, "${parseHex[uint32]}:${parseHex[uint32]}", hi, lo):
+                    offset = uint64(hi) * 0x100000000u64 + uint64(lo)
+                discard scanf(larg, "${parseUint}", length)
+                stat.rtotal += 1
+                stat.rbytes += rsp.inform
+                aggr.rtotal += 1
+                aggr.rbytes += rsp.inform
+                if 0 != offset mod 4096 or 0 != length mod 4096:
+                    stat.rnoaln += 1
+                    aggr.rnoaln += 1
+            else:
+                stat.rerror += 1
+                aggr.rerror += 1
+        of "Write":
+            var filename = filetab[req.context]
+            var stat = stattab.mgetOrPut(filename, Stat())
+            if 0 == rsp.status:
+                var oarg = getArg(req.args, "Offset")
+                var larg = getArg(req.args, "Length")
+                var hi, lo: uint32
+                var offset: uint64
+                var length: uint
+                if scanf(oarg, "${parseHex[uint32]}:${parseHex[uint32]}", hi, lo):
+                    offset = uint64(hi) * 0x100000000u64 + uint64(lo)
+                discard scanf(larg, "${parseUint}", length)
+                stat.wtotal += 1
+                stat.wbytes += rsp.inform
+                aggr.wtotal += 1
+                aggr.wbytes += rsp.inform
+                if 0 != offset mod 4096 or 0 != length mod 4096:
+                    stat.wnoaln += 1
+                    aggr.wnoaln += 1
+            else:
+                stat.werror += 1
+                aggr.werror += 1
+        of "QueryDirectory":
+            var filename = filetab[req.context]
+            var stat = stattab.mgetOrPut(filename, Stat())
+            var pattern = getArg(req.args, "Pattern")
+            if "NULL" == pattern:
+                if 0 == rsp.status:
+                    stat.dtotal += 1
+                    stat.dbytes += rsp.inform
+                    aggr.dtotal += 1
+                    aggr.dbytes += rsp.inform
+                else:
+                    stat.derror += 1
+                    aggr.derror += 1
+            else:
+                if 0 == rsp.status:
+                    stat.ptotal += 1
+                    stat.pbytes += rsp.inform
+                    aggr.ptotal += 1
+                    aggr.pbytes += rsp.inform
+                else:
+                    stat.perror += 1
+                    aggr.perror += 1
+
+macro identName(n: untyped): untyped =
+    result = n.strVal.newLit
+
+template dumpstat(F: untyped) =
+    stattab.sort(proc (x, y: (string, Stat)): int =
+        cmp(x[1].F, y[1].F), SortOrder.Descending)
+    var width, rows = 0
+    for filename, stat in stattab.pairs:
+        if 0 == width:
+            var s = identName(F).toUpperAscii()
+            width = len($aggr.F)
+            if width < len(s):
+                width = len(s)
+            var f: string
+            formatValue(f, s, ">" & $width)
+            echo f, "  PER% FILENAME"
+        var c0, c1: string
+        formatValue(c0, stat.F, $width)
+        if 0 != aggr.F:
+            formatValue(c1, 100.0 * float(stat.F) / float(aggr.F), "5.1f")
+        else:
+            c1 = "     "
+        echo c0, " ", c1, " ", filename
+        inc rows
+        if opttop == rows:
+            break
+    var c0: string
+    formatValue(c0, aggr.F, $width)
+    echo c0, " 100.0 TOTAL"
+
+proc main =
+    var filenames: seq[string]
+    var optstat: seq[string]
+    var opttop = 0
+    for kind, key, val in getopt(shortNoVal = {'Z'}, longNoVal = @["Zoo"]):
+        case kind
+        of cmdShortOption, cmdLongOption:
+            case key
+            of "stat":
+                optstat.add(val)
+            of "n":
+                opttop = parseInt(val)
+        of cmdArgument:
+            filenames.add(key)
+        else:
+            discard
+    if 0 == len(optstat):
+        optstat.add("ototal")
+
+    if 0 == len(filenames):
+        stderr.writeLine("usage: parselog [-nNN] [--stat ototal|rtotal|wtotal|dtotal|...] file...")
+        quit(2)
+
+    for filename in filenames:
+        parseLog filename, processReq, processRsp
+
+    for s in optstat:
+        case s
+        of "ototal":
+            dumpstat ototal
+        of "omulti":
+            dumpstat omulti
+        of "oerror":
+            dumpstat oerror
+        of "rtotal":
+            dumpstat rtotal
+        of "rnoaln":
+            dumpstat rnoaln
+        of "rbytes":
+            dumpstat rbytes
+        of "rerror":
+            dumpstat rerror
+        of "wtotal":
+            dumpstat wtotal
+        of "wnoaln":
+            dumpstat wnoaln
+        of "wbytes":
+            dumpstat wbytes
+        of "werror":
+            dumpstat werror
+        of "dtotal":
+            dumpstat dtotal
+        of "dbytes":
+            dumpstat dbytes
+        of "derror":
+            dumpstat derror
+        of "ptotal":
+            dumpstat ptotal
+        of "pbytes":
+            dumpstat pbytes
+        of "perror":
+            dumpstat perror
+        echo ""
+
+when isMainModule:
+    main()

tst/memfs-fuse3/Makefile

@@ -10,9 +10,9 @@ cygfuse3: memfs-cygfuse3
 winfsp-fuse3: memfs-winfsp-fuse3
 
 memfs-cygfuse3: memfs-fuse3.cpp
-	g++ $^ -o $@ -g -Wall `pkg-config fuse3 --cflags --libs`
+	g++ $^ -o $@ -g -Wall -std=gnu++17 `pkg-config fuse3 --cflags --libs`
 
 memfs-winfsp-fuse3: export PKG_CONFIG_PATH=$(PWD)/winfsp.install/lib
 memfs-winfsp-fuse3: memfs-fuse3.cpp
 	ln -nsf "`regtool --wow32 get '/HKLM/Software/WinFsp/InstallDir' | cygpath -au -f -`" winfsp.install
-	g++ $^ -o $@ -g -Wall `pkg-config fuse3 --cflags --libs`
+	g++ $^ -o $@ -g -Wall -std=gnu++17 `pkg-config fuse3 --cflags --libs`

tst/winfsp-tests/info-test.c

@@ -1025,6 +1025,76 @@ void rename_test(void)
     }
 }
 
+static void rename_backslash_dotest(ULONG Flags, PWSTR Prefix, ULONG FileInfoTimeout)
+{
+    void *memfs = memfs_start_ex(Flags, FileInfoTimeout);
+
+    HANDLE Handle;
+    BOOL Success;
+    WCHAR Dir1Path[MAX_PATH];
+    WCHAR Dir2Path[MAX_PATH];
+    WCHAR File0Path[MAX_PATH];
+    WCHAR File1Path[MAX_PATH];
+
+    StringCbPrintfW(Dir1Path, sizeof Dir1Path, L"%s%s\\dir1",
+        Prefix ? L"" : L"\\\\?\\GLOBALROOT", Prefix ? Prefix : memfs_volumename(memfs));
+
+    StringCbPrintfW(Dir2Path, sizeof Dir2Path, L"%s%s\\dir2\\",
+        Prefix ? L"" : L"\\\\?\\GLOBALROOT", Prefix ? Prefix : memfs_volumename(memfs));
+
+    StringCbPrintfW(File0Path, sizeof File0Path, L"%s%s\\dir1\\file0",
+        Prefix ? L"" : L"\\\\?\\GLOBALROOT", Prefix ? Prefix : memfs_volumename(memfs));
+
+    StringCbPrintfW(File1Path, sizeof File1Path, L"%s%s\\dir1\\file1\\",
+        Prefix ? L"" : L"\\\\?\\GLOBALROOT", Prefix ? Prefix : memfs_volumename(memfs));
+
+    Success = CreateDirectoryW(Dir1Path, 0);
+    ASSERT(Success);
+
+    Handle = CreateFileW(File0Path,
+        GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_WRITE, 0,
+        CREATE_NEW, FILE_ATTRIBUTE_NORMAL, 0);
+    ASSERT(INVALID_HANDLE_VALUE != Handle);
+    CloseHandle(Handle);
+
+    Success = MoveFileExW(File0Path, File1Path, 0);
+    ASSERT(Success);
+
+    Success = MoveFileExW(Dir1Path, Dir2Path, 0);
+    ASSERT(Success);
+
+    StringCbPrintfW(File1Path, sizeof File1Path, L"%s%s\\dir2\\file1",
+        Prefix ? L"" : L"\\\\?\\GLOBALROOT", Prefix ? Prefix : memfs_volumename(memfs));
+
+    Success = DeleteFileW(File1Path);
+    ASSERT(Success);
+
+    Success = RemoveDirectoryW(Dir2Path);
+    ASSERT(Success);
+
+    memfs_stop(memfs);
+}
+
+void rename_backslash_test(void)
+{
+    if (NtfsTests)
+    {
+        WCHAR DirBuf[MAX_PATH];
+        GetTestDirectory(DirBuf);
+        rename_backslash_dotest(-1, DirBuf, 0);
+    }
+    if (WinFspDiskTests)
+    {
+        rename_backslash_dotest(MemfsDisk, 0, 0);
+        rename_backslash_dotest(MemfsDisk, 0, 1000);
+    }
+    if (WinFspNetTests)
+    {
+        rename_backslash_dotest(MemfsNet, L"\\\\memfs\\share", 0);
+        rename_backslash_dotest(MemfsNet, L"\\\\memfs\\share", 1000);
+    }
+}
+
 static void rename_open_dotest(ULONG Flags, PWSTR Prefix, ULONG FileInfoTimeout)
 {
     void *memfs = memfs_start_ex(Flags, FileInfoTimeout);
@@ -1982,6 +2052,7 @@ void info_tests(void)
         TEST(delete_mmap_test);
     TEST(delete_standby_test);
     TEST(rename_test);
+    TEST(rename_backslash_test);
     TEST(rename_open_test);
     TEST(rename_caseins_test);
     if (!OptShareName)