doc: perf-tests: update Performance Testing document

doc/perf-tests.adoc

@@ -1,4 +1,5 @@
 = Performance Testing
+:caption:
 
 This document discusses performance testing for WinFsp. The goal of this performance testing is to discover optimization opportunities for WinFsp and compare its performance to that of NTFS and Dokany.
 
@@ -6,6 +7,14 @@ This document discusses performance testing for WinFsp. The goal of this perform
 
 This performance testing shows that WinFsp has excellent performance in all tested scenarios. It outperforms NTFS in most scenarios (an unfair comparison as NTFS is a disk file system and WinFsp is tested with an in-memory file system). It also outperforms Dokany in all scenarios, often by an order of magnitude.
 
+.File Tests
+ifdef::env-browser[chart::bar[data-uri="perf-tests/file_tests.csv",file="perf-tests/file_tests.png",opt="y-label=time"]]
+ifndef::env-browser[image::perf-tests/file_tests.png[]]
+
+.Read/Write Tests
+ifdef::env-browser[chart::bar[data-uri="perf-tests/rdwr_tests.csv",file="perf-tests/rdwr_tests.png",opt="y-label=time"]]
+ifndef::env-browser[image::perf-tests/rdwr_tests.png[]]
+
 == Fsbench
 
 All testing was performed using a new performance test suite developed as part of WinFsp, called https://github.com/billziss-gh/winfsp/blob/master/tst/fsbench/fsbench.c[fsbench]. Fsbench was developed because it allows the creation of tests that are important to file system developers; for example, it can answer questions of the type: "how long does it take to delete 1000 files" or "how long does it take to list a directory with 10000 files in it".

doc/perf-tests/file_mkdir_test.csv

@@ -1,30 +0,0 @@
-//ntfs
-1000,0.13
-2000,0.25
-3000,0.36
-4000,0.48
-5000,0.70
-//winfsp-t0
-1000,0.05
-2000,0.09
-3000,0.41
-4000,0.53
-5000,0.66
-//winfsp-t1
-1000,0.05
-2000,0.09
-3000,0.39
-4000,0.55
-5000,0.66
-//winfsp-tinf
-1000,0.05
-2000,0.09
-3000,0.39
-4000,0.66
-5000,0.66
-//dokany
-1000,0.08
-2000,0.33
-3000,0.48
-4000,0.66
-5000,0.83

doc/perf-tests/file_rmdir_test.csv

@@ -1,30 +0,0 @@
-//ntfs
-1000,0.11
-2000,0.28
-3000,0.36
-4000,0.41
-5000,0.59
-//winfsp-t0
-1000,0.08
-2000,0.14
-3000,0.20
-4000,0.27
-5000,0.34
-//winfsp-t1
-1000,0.06
-2000,0.12
-3000,0.17
-4000,0.22
-5000,0.28
-//winfsp-tinf
-1000,0.06
-2000,0.13
-3000,0.16
-4000,0.22
-5000,0.28
-//dokany
-1000,0.13
-2000,0.23
-3000,0.34
-4000,0.47
-5000,0.59

doc/perf-tests/file_tests.csv

@@ -0,0 +1,30 @@
+//ntfs
+file_create_test,1.94
+file_open_test,0.45
+file_overwrite_test,1.14
+file_list_test,0.41
+file_delete_test,1.03
+//winfsp-t0
+file_create_test,1.08
+file_open_test,0.28
+file_overwrite_test,0.30
+file_list_test,0.61
+file_delete_test,0.45
+//winfsp-t1
+file_create_test,0.95
+file_open_test,0.22
+file_overwrite_test,0.22
+file_list_test,0.59
+file_delete_test,0.34
+//winfsp-tinf
+file_create_test,0.97
+file_open_test,0.20
+file_overwrite_test,0.22
+file_list_test,0.59
+file_delete_test,0.33
+//dokany
+file_create_test,1.64
+file_open_test,0.67
+file_overwrite_test,1.64
+file_list_test,1.38
+file_delete_test,0.91

doc/perf-tests/munge.py

@@ -6,6 +6,22 @@
 import csv, os, sys
 
 snames = ["ntfs", "winfsp-t0", "winfsp-t1", "winfsp-tinf", "dokany"]
+file_tnames = [
+    "file_create_test",
+    "file_open_test",
+    "file_overwrite_test",
+    "file_list_test",
+    "file_delete_test"]
+    #"file_mkdir_test",
+    #"file_rmdir_test"]
+rdwr_tnames = [
+    "rdwr_cc_read_page_test",
+    "rdwr_cc_write_page_test",
+    "rdwr_nc_read_page_test",
+    "rdwr_nc_write_page_test",
+    "mmap_read_test",
+    "mmap_write_test"]
+tnames = file_tnames + rdwr_tnames
 aggregate = min
 
 tests = {}
@@ -18,28 +34,41 @@ for arg in sys.argv[1:]:
             tests.\
                 setdefault(row[0], {}).\
                 setdefault(name, {}).\
-                setdefault(row[1], []).\
-                append(row[2])
+                setdefault(int(row[1]), []).\
+                append(float(row[2]))
 
 if False:
-    for testname in sorted(tests.keys()):
-        print "%s:" % testname
-        test = tests[testname]
+    for tname in (tnames if tnames else sorted(tests.keys())):
+        print "%s:" % tname
+        test = tests[tname]
         for sname in (snames if snames else sorted(test.keys())):
             if sname not in test:
                 continue
             print "    %s:" % sname
             series = test[sname]
             for param in sorted(series.keys()):
-                print "        %s: %s -> %s" % (param, series[param], aggregate(series[param]))
+                print "        %s: %s -> %.2f" % (param, series[param], aggregate(series[param]))
 else:
-    for testname in sorted(tests.keys()):
-        with open(testname + ".csv", "w") as fout:
-            test = tests[testname]
+    for tname in (tnames if tnames else sorted(tests.keys())):
+        with open(tname + ".csv", "w") as fout:
+            test = tests[tname]
             for sname in (snames if snames else sorted(test.keys())):
                 if sname not in test:
                     continue
                 fout.write("//%s\r\n" % sname)
                 series = test[sname]
                 for param in sorted(series.keys()):
-                    fout.write("%s,%s\r\n" % (param, aggregate(series[param])))
+                    fout.write("%s,%.2f\r\n" % (param, aggregate(series[param])))
+    def master_write(fname, tnames):
+        with open(fname + ".csv", "w") as fout:
+            for sname in snames:
+                fout.write("//%s\r\n" % sname)
+                for tname in (tnames if tnames else sorted(tests.keys())):
+                    test = tests[tname]
+                    if sname not in test:
+                        continue
+                    series = test[sname]
+                    param = max(series.keys())
+                    fout.write("%s,%.2f\r\n" % (tname, aggregate(series[param])))
+    master_write("file_tests", file_tnames)
+    master_write("rdwr_tests", rdwr_tnames)

doc/perf-tests/rdwr_cc_read_large_test.csv

@@ -1,30 +0,0 @@
-//ntfs
-100,0.08
-200,0.16
-300,0.22
-400,0.28
-500,0.36
-//winfsp-t0
-100,0.17
-200,0.34
-300,0.50
-400,0.66
-500,0.83
-//winfsp-t1
-100,0.17
-200,0.34
-300,0.50
-400,0.67
-500,0.83
-//winfsp-tinf
-100,0.08
-200,0.22
-300,0.23
-400,0.33
-500,0.41
-//dokany
-100,0.28
-200,0.58
-300,0.86
-400,1.16
-500,1.45

doc/perf-tests/rdwr_cc_write_large_test.csv

@@ -1,30 +0,0 @@
-//ntfs
-100,0.09
-200,0.22
-300,0.28
-400,0.38
-500,0.48
-//winfsp-t0
-100,0.17
-200,0.31
-300,0.48
-400,0.63
-500,0.80
-//winfsp-t1
-100,0.16
-200,0.31
-300,0.47
-400,0.61
-500,0.78
-//winfsp-tinf
-100,0.11
-200,0.22
-300,0.34
-400,0.42
-500,0.53
-//dokany
-100,0.33
-200,0.59
-300,0.91
-400,1.20
-500,1.47

doc/perf-tests/rdwr_nc_read_large_test.csv

@@ -1,30 +0,0 @@
-//ntfs
-100,0.56
-200,1.16
-300,1.64
-400,2.20
-500,2.77
-//winfsp-t0
-100,0.16
-200,0.31
-300,0.47
-400,0.64
-500,0.78
-//winfsp-t1
-100,0.16
-200,0.33
-300,0.50
-400,0.67
-500,0.81
-//winfsp-tinf
-100,0.16
-200,0.31
-300,0.48
-400,0.64
-500,0.78
-//dokany
-100,0.30
-200,0.58
-300,0.84
-400,1.11
-500,1.41

doc/perf-tests/rdwr_nc_read_page_test.csv

@@ -1,5 +1,5 @@
 //ntfs
-100,10.14
+100,9.38
 200,21.39
 300,19.56
 400,26.11

doc/perf-tests/rdwr_nc_write_large_test.csv

@@ -1,30 +0,0 @@
-//ntfs
-100,0.86
-200,1.72
-300,2.53
-400,3.38
-500,4.33
-//winfsp-t0
-100,0.16
-200,0.31
-300,0.48
-400,0.63
-500,0.81
-//winfsp-t1
-100,0.19
-200,0.31
-300,0.48
-400,0.63
-500,0.81
-//winfsp-tinf
-100,0.17
-200,0.33
-300,0.47
-400,0.63
-500,0.78
-//dokany
-100,0.31
-200,0.61
-300,0.91
-400,1.22
-500,1.50

doc/perf-tests/rdwr_tests.csv

@@ -0,0 +1,33 @@
+//ntfs
+rdwr_cc_read_page_test,0.97
+rdwr_cc_write_page_test,1.19
+rdwr_nc_read_page_test,33.05
+rdwr_nc_write_page_test,35.45
+mmap_read_test,1.06
+mmap_write_test,0.72
+//winfsp-t0
+rdwr_cc_read_page_test,6.83
+rdwr_cc_write_page_test,6.12
+rdwr_nc_read_page_test,6.41
+rdwr_nc_write_page_test,6.41
+mmap_read_test,1.00
+mmap_write_test,0.72
+//winfsp-t1
+rdwr_cc_read_page_test,7.33
+rdwr_cc_write_page_test,6.17
+rdwr_nc_read_page_test,6.94
+rdwr_nc_write_page_test,6.51
+mmap_read_test,0.98
+mmap_write_test,0.73
+//winfsp-tinf
+rdwr_cc_read_page_test,1.48
+rdwr_cc_write_page_test,1.70
+rdwr_nc_read_page_test,6.58
+rdwr_nc_write_page_test,6.42
+mmap_read_test,0.98
+mmap_write_test,0.73
+//dokany
+rdwr_cc_read_page_test,11.20
+rdwr_cc_write_page_test,10.33
+rdwr_nc_read_page_test,10.59
+rdwr_nc_write_page_test,10.73