sys: FSP_META_CACHE: refactoring and start integration into Fsvol

inc/winfsp/fsctl.h

@@ -102,7 +102,7 @@ typedef struct
     UINT32 TransactTimeout;             /* FSP_FSCTL_TRANSACT timeout (millis; 1 sec - 10 sec) */
     UINT32 IrpTimeout;                  /* pending IRP timeout (millis; 1 min - 10 min) */
     UINT32 IrpCapacity;                 /* maximum number of pending IRP's (100 - 1000)*/
-    UINT32 FileInfoTimeout;             /* FileInfo/VolumeInfo timeout (millis) */
+    UINT32 FileInfoTimeout;             /* FileInfo/Security/VolumeInfo timeout (millis) */
     /* FILE_FS_ATTRIBUTE_INFORMATION::FileSystemAttributes */
     UINT32 CaseSensitiveSearch:1;       /* file system supports case-sensitive file names */
     UINT32 CasePreservedNames:1;        /* file system preserves the case of file names */

src/sys/device.c

@@ -290,6 +290,7 @@ static NTSTATUS FspFsvolDeviceInit(PDEVICE_OBJECT DeviceObject)
     NTSTATUS Result;
     FSP_FSVOL_DEVICE_EXTENSION *FsvolDeviceExtension = FspFsvolDeviceExtension(DeviceObject);
     LARGE_INTEGER IrpTimeout;
+    LARGE_INTEGER MetaTimeout;
 
     /*
      * Volume device initialization is a mess, because of the different ways of
@@ -325,6 +326,16 @@ static NTSTATUS FspFsvolDeviceInit(PDEVICE_OBJECT DeviceObject)
         return Result;
     FsvolDeviceExtension->InitDoneIoq = 1;
 
+    /* create our security meta cache */
+    MetaTimeout.QuadPart = FsvolDeviceExtension->VolumeParams.FileInfoTimeout * 10000ULL;
+        /* convert millis to nanos */
+    Result = FspMetaCacheCreate(
+        FspFsvolDeviceSecurityCacheCapacity, FspFsvolDeviceSecurityCacheItemSizeMax, &MetaTimeout,
+        &FsvolDeviceExtension->SecurityCache);
+    if (!NT_SUCCESS(Result))
+        return Result;
+    FsvolDeviceExtension->InitDoneSec = 1;
+
     /* initialize our context table */
     ExInitializeResourceLite(&FsvolDeviceExtension->FileRenameResource);
     ExInitializeResourceLite(&FsvolDeviceExtension->ContextTableResource);
@@ -374,6 +385,10 @@ static VOID FspFsvolDeviceFini(PDEVICE_OBJECT DeviceObject)
     if (FsvolDeviceExtension->InitDoneTimer)
         IoStopTimer(DeviceObject);
 
+    /* delete the security meta cache */
+    if (FsvolDeviceExtension->InitDoneSec)
+        FspMetaCacheDelete(FsvolDeviceExtension->SecurityCache);
+
     /* delete the Ioq */
     if (FsvolDeviceExtension->InitDoneIoq)
         FspIoqDelete(FsvolDeviceExtension->Ioq);
@@ -446,6 +461,7 @@ static VOID FspFsvolDeviceExpirationRoutine(PVOID Context)
     KIRQL Irql;
 
     InterruptTime = KeQueryInterruptTime();
+    FspMetaCacheInvalidateExpired(FsvolDeviceExtension->SecurityCache, InterruptTime);
     FspIoqRemoveExpired(FsvolDeviceExtension->Ioq, InterruptTime);
 
     KeAcquireSpinLock(&FsvolDeviceExtension->ExpirationLock, &Irql);

src/sys/driver.h

@@ -396,15 +396,15 @@ typedef struct
     ULONG ItemBucketCount;
     PVOID ItemBuckets[];
 } FSP_META_CACHE;
-NTSTATUS MetaCacheCreate(
+NTSTATUS FspMetaCacheCreate(
     ULONG MetaCapacity, ULONG ItemSizeMax, PLARGE_INTEGER MetaTimeout,
     FSP_META_CACHE **PMetaCache);
-VOID MetaCacheDelete(FSP_META_CACHE *MetaCache);
+VOID FspMetaCacheDelete(FSP_META_CACHE *MetaCache);
-VOID MetaCacheInvalidateExpired(FSP_META_CACHE *MetaCache, UINT64 ExpirationTime);
+VOID FspMetaCacheInvalidateExpired(FSP_META_CACHE *MetaCache, UINT64 ExpirationTime);
-PVOID MetaCacheReferenceItemBuffer(FSP_META_CACHE *MetaCache, UINT64 ItemIndex, PULONG PSize);
+PVOID FspMetaCacheReferenceItemBuffer(FSP_META_CACHE *MetaCache, UINT64 ItemIndex, PULONG PSize);
-VOID MetaCacheDereferenceItemBuffer(PVOID Buffer);
+VOID FspMetaCacheDereferenceItemBuffer(PVOID Buffer);
-UINT64 MetaCacheAddItem(FSP_META_CACHE *MetaCache, PVOID Buffer, ULONG Size);
+UINT64 FspMetaCacheAddItem(FSP_META_CACHE *MetaCache, PVOID Buffer, ULONG Size);
-VOID MetaCacheInvalidateItem(FSP_META_CACHE *MetaCache, UINT64 ItemIndex);
+VOID FspMetaCacheInvalidateItem(FSP_META_CACHE *MetaCache, UINT64 ItemIndex);
 
 /* I/O processing */
 #define FSP_FSCTL_WORK                  \
@@ -443,6 +443,11 @@ NTSTATUS FspIopDispatchComplete(PIRP Irp, const FSP_FSCTL_TRANSACT_RSP *Response
 
 /* device management */
 #define FSP_DEVICE_VOLUME_NAME_LENMAX   (FSP_FSCTL_VOLUME_NAME_SIZEMAX - sizeof(WCHAR))
+enum
+{
+    FspFsvolDeviceSecurityCacheCapacity = 100,
+    FspFsvolDeviceSecurityCacheItemSizeMax = 4096,
+};
 typedef struct
 {
     UINT64 Identifier;
@@ -478,8 +483,8 @@ typedef struct
 typedef struct
 {
     FSP_DEVICE_EXTENSION Base;
-    UINT32 InitDoneFsvrt:1, InitDoneDelRsc:1, InitDoneIoq:1, InitDoneCtxTab:1, InitDoneTimer:1,
+    UINT32 InitDoneFsvrt:1, InitDoneDelRsc:1, InitDoneIoq:1, InitDoneSec:1, InitDoneCtxTab:1,
-        InitDoneInfo:1;
+        InitDoneTimer:1, InitDoneInfo:1;
     PDEVICE_OBJECT FsctlDeviceObject;
     PDEVICE_OBJECT FsvrtDeviceObject;
     HANDLE MupHandle;
@@ -489,6 +494,7 @@ typedef struct
     FSP_FSCTL_VOLUME_PARAMS VolumeParams;
     UNICODE_STRING VolumePrefix;
     FSP_IOQ *Ioq;
+    FSP_META_CACHE *SecurityCache;
     KSPIN_LOCK ExpirationLock;
     WORK_QUEUE_ITEM ExpirationWorkItem;
     BOOLEAN ExpirationInProgress;

src/sys/meta.c

@@ -23,8 +23,7 @@ typedef struct
     __declspec(align(MEMORY_ALLOCATION_ALIGNMENT)) UINT8 Buffer[];
 } FSP_META_CACHE_ITEM_BUFFER;
 
-static inline
+static inline VOID FspMetaCacheDereferenceItem(FSP_META_CACHE_ITEM *Item)
-VOID MetaCacheDereferenceItem(FSP_META_CACHE_ITEM *Item)
 {
     LONG RefCount = InterlockedDecrement(&Item->RefCount);
     if (0 == RefCount)
@@ -35,186 +34,38 @@ VOID MetaCacheDereferenceItem(FSP_META_CACHE_ITEM *Item)
     }
 }
 
-NTSTATUS MetaCacheCreate(
+static inline FSP_META_CACHE_ITEM *FspMetaCacheLookupIndexedItemAtDpcLevel(FSP_META_CACHE *MetaCache,
-    ULONG MetaCapacity, ULONG ItemSizeMax, PLARGE_INTEGER MetaTimeout,
+    UINT64 ItemIndex)
-    FSP_META_CACHE **PMetaCache)
-{
-    *PMetaCache = 0;
-
-    FSP_META_CACHE *MetaCache;
-    ULONG BucketCount = (PAGE_SIZE - sizeof *MetaCache) / sizeof MetaCache->ItemBuckets[0];
-    MetaCache = FspAllocNonPaged(PAGE_SIZE);
-    if (0 == MetaCache)
-        return STATUS_INSUFFICIENT_RESOURCES;
-    RtlZeroMemory(MetaCache, PAGE_SIZE);
-
-    KeInitializeSpinLock(&MetaCache->SpinLock);
-    InitializeListHead(&MetaCache->ItemList);
-    MetaCache->MetaCapacity = MetaCapacity;
-    MetaCache->ItemSizeMax = ItemSizeMax;
-    MetaCache->MetaTimeout = MetaTimeout->QuadPart;
-    MetaCache->ItemBucketCount = BucketCount;
-
-    *PMetaCache = MetaCache;
-
-    return STATUS_SUCCESS;
-}
-
-VOID MetaCacheDelete(FSP_META_CACHE *MetaCache)
-{
-    MetaCacheInvalidateExpired(MetaCache, (UINT64)-1LL);
-    FspFree(MetaCache);
-}
-
-VOID MetaCacheInvalidateExpired(FSP_META_CACHE *MetaCache, UINT64 ExpirationTime)
-{
-    FSP_META_CACHE_ITEM *Item;
-    PLIST_ENTRY Head, Entry;
-    ULONG HashIndex;
-    KIRQL Irql;
-
-    for (;;)
-    {
-        KeAcquireSpinLock(&MetaCache->SpinLock, &Irql);
-        Item = 0;
-        Head = &MetaCache->ItemList;
-        Entry = Head->Flink;
-        if (Head != Entry)
-        {
-            Item = CONTAINING_RECORD(Entry, FSP_META_CACHE_ITEM, ListEntry);
-            if (Item->ExpirationTime <= ExpirationTime)
-            {
-                HashIndex = Item->ItemIndex % MetaCache->ItemBucketCount;
-                for (FSP_META_CACHE_ITEM **P = (PVOID)&MetaCache->ItemBuckets[HashIndex];
-                    *P; P = &(*P)->DictNext)
-                    if (*P == Item)
-                    {
-                        *P = (*P)->DictNext;
-                        break;
-                    }
-                RemoveEntryList(&Item->ListEntry);
-                MetaCache->ItemCount--;
-            }
-            else
-                Item = 0;
-        }
-        KeReleaseSpinLock(&MetaCache->SpinLock, Irql);
-
-        if (0 == Item)
-            break;
-
-        MetaCacheDereferenceItem(Item);
-    }
-}
-
-PVOID MetaCacheReferenceItemBuffer(FSP_META_CACHE *MetaCache, UINT64 ItemIndex, PULONG PSize)
 {
     FSP_META_CACHE_ITEM *Item = 0;
-    FSP_META_CACHE_ITEM_BUFFER *ItemBuffer;
+    ULONG HashIndex = ItemIndex % MetaCache->ItemBucketCount;
-    ULONG HashIndex;
-    KIRQL Irql;
-
-    KeAcquireSpinLock(&MetaCache->SpinLock, &Irql);
-    HashIndex = ItemIndex % MetaCache->ItemBucketCount;
     for (FSP_META_CACHE_ITEM *ItemX = MetaCache->ItemBuckets[HashIndex]; ItemX; ItemX = ItemX->DictNext)
         if (ItemX->ItemIndex == ItemIndex)
         {
             Item = ItemX;
-            InterlockedIncrement(&Item->RefCount);
             break;
         }
-    KeReleaseSpinLock(&MetaCache->SpinLock, Irql);
+    return Item;
-
-    if (0 == Item)
-    {
-        if (0 != PSize)
-            *PSize = 0;
-        return 0;
-    }
-
-    ItemBuffer = Item->ItemBuffer;
-    if (0 != PSize)
-        *PSize = ItemBuffer->Size;
-    return ItemBuffer->Buffer;
 }
 
-VOID MetaCacheDereferenceItemBuffer(PVOID Buffer)
+static inline VOID FspMetaCacheAddItemAtDpcLevel(FSP_META_CACHE *MetaCache, FSP_META_CACHE_ITEM *Item)
 {
-    FSP_META_CACHE_ITEM_BUFFER *ItemBuffer = (PVOID)((PUINT8)Buffer - sizeof *ItemBuffer);
+    ULONG HashIndex = Item->ItemIndex % MetaCache->ItemBucketCount;
-    MetaCacheDereferenceItem(ItemBuffer->Item);
-}
-
-UINT64 MetaCacheAddItem(FSP_META_CACHE *MetaCache, PVOID Buffer, ULONG Size)
-{
-    FSP_META_CACHE_ITEM *Item;
-    FSP_META_CACHE_ITEM_BUFFER *ItemBuffer;
-    UINT64 ItemIndex = 0;
-    ULONG HashIndex;
-    BOOLEAN HasCapacity;
-    KIRQL Irql;
-
-    if (sizeof *ItemBuffer + Size > MetaCache->ItemSizeMax)
-        return 0;
-
-    Item = FspAllocNonPaged(sizeof *Item);
-    if (0 == Item)
-        return 0;
-
-    ItemBuffer = FspAlloc(sizeof *ItemBuffer + Size);
-    if (0 == ItemBuffer)
-    {
-        FspFree(Item);
-        return 0;
-    }
-
-    RtlZeroMemory(Item, sizeof *Item);
-    Item->ItemBuffer = ItemBuffer;
-    Item->ExpirationTime = KeQueryInterruptTime() + MetaCache->MetaTimeout;
-    Item->RefCount = 1;
-
-    RtlZeroMemory(ItemBuffer, sizeof *ItemBuffer);
-    ItemBuffer->Item = Item;
-    ItemBuffer->Size = Size;
-    RtlCopyMemory(ItemBuffer->Buffer, Buffer, Size);
-
-    KeAcquireSpinLock(&MetaCache->SpinLock, &Irql);
-    HasCapacity = MetaCache->ItemCount < MetaCache->MetaCapacity;
-    if (HasCapacity)
-    {
-        ItemIndex = MetaCache->ItemIndex;
-        ItemIndex = (UINT64)-1LL == ItemIndex ? 1 : ItemIndex + 1;
-        MetaCache->ItemIndex = ItemIndex;
-        Item->ItemIndex = ItemIndex;
-
-        MetaCache->ItemCount++;
-        InsertTailList(&MetaCache->ItemList, &Item->ListEntry);
-        HashIndex = ItemIndex % MetaCache->ItemBucketCount;
 #if DBG
-        for (FSP_META_CACHE_ITEM *ItemX = MetaCache->ItemBuckets[HashIndex]; ItemX; ItemX = ItemX->DictNext)
+    for (FSP_META_CACHE_ITEM *ItemX = MetaCache->ItemBuckets[HashIndex]; ItemX; ItemX = ItemX->DictNext)
-            ASSERT(ItemX->ItemIndex != ItemIndex);
+        ASSERT(ItemX->ItemIndex != Item->ItemIndex);
 #endif
-        Item->DictNext = MetaCache->ItemBuckets[HashIndex];
+    Item->DictNext = MetaCache->ItemBuckets[HashIndex];
-        MetaCache->ItemBuckets[HashIndex] = Item;
+    MetaCache->ItemBuckets[HashIndex] = Item;
-    }
+    InsertTailList(&MetaCache->ItemList, &Item->ListEntry);
-    KeReleaseSpinLock(&MetaCache->SpinLock, Irql);
+    MetaCache->ItemCount++;
-
-    if (!HasCapacity)
-    {
-        FspFree(ItemBuffer);
-        FspFree(Item);
-    }
-
-    return ItemIndex;
 }
 
-VOID MetaCacheInvalidateItem(FSP_META_CACHE *MetaCache, UINT64 ItemIndex)
+static inline FSP_META_CACHE_ITEM *FspMetaCacheRemoveIndexedItemAtDpcLevel(FSP_META_CACHE *MetaCache,
+    UINT64 ItemIndex)
 {
     FSP_META_CACHE_ITEM *Item = 0;
-    ULONG HashIndex;
+    ULONG HashIndex = ItemIndex % MetaCache->ItemBucketCount;
-    KIRQL Irql;
-
-    KeAcquireSpinLock(&MetaCache->SpinLock, &Irql);
-    HashIndex = ItemIndex % MetaCache->ItemBucketCount;
     for (FSP_META_CACHE_ITEM **P = (PVOID)&MetaCache->ItemBuckets[HashIndex]; *P; P = &(*P)->DictNext)
         if ((*P)->ItemIndex == ItemIndex)
         {
@@ -224,8 +75,156 @@ VOID MetaCacheInvalidateItem(FSP_META_CACHE *MetaCache, UINT64 ItemIndex)
             MetaCache->ItemCount--;
             break;
         }
-    KeReleaseSpinLock(&MetaCache->SpinLock, Irql);
+    return Item;
-
+}
-    if (0 != Item)
+
-        MetaCacheDereferenceItem(Item);
+static inline FSP_META_CACHE_ITEM *FspMetaCacheRemoveExpiredItemAtDpcLevel(FSP_META_CACHE *MetaCache,
+    UINT64 ExpirationTime)
+{
+    PLIST_ENTRY Head = &MetaCache->ItemList;
+    PLIST_ENTRY Entry = Head->Flink;
+    if (Head == Entry)
+        return 0;
+    FSP_META_CACHE_ITEM *Item = CONTAINING_RECORD(Entry, FSP_META_CACHE_ITEM, ListEntry);
+    if (Item->ExpirationTime > ExpirationTime)
+        return 0;
+    ULONG HashIndex = Item->ItemIndex % MetaCache->ItemBucketCount;
+    for (FSP_META_CACHE_ITEM **P = (PVOID)&MetaCache->ItemBuckets[HashIndex]; *P; P = &(*P)->DictNext)
+        if (*P == Item)
+        {
+            *P = (*P)->DictNext;
+            break;
+        }
+    RemoveEntryList(&Item->ListEntry);
+    MetaCache->ItemCount--;
+    return Item;
+}
+
+NTSTATUS FspMetaCacheCreate(
+    ULONG MetaCapacity, ULONG ItemSizeMax, PLARGE_INTEGER MetaTimeout,
+    FSP_META_CACHE **PMetaCache)
+{
+    *PMetaCache = 0;
+    if (0 == MetaCapacity || 0 == ItemSizeMax || 0 == MetaTimeout->QuadPart)
+        return STATUS_SUCCESS;
+    FSP_META_CACHE *MetaCache;
+    ULONG BucketCount = (PAGE_SIZE - sizeof *MetaCache) / sizeof MetaCache->ItemBuckets[0];
+    MetaCache = FspAllocNonPaged(PAGE_SIZE);
+    if (0 == MetaCache)
+        return STATUS_INSUFFICIENT_RESOURCES;
+    RtlZeroMemory(MetaCache, PAGE_SIZE);
+    KeInitializeSpinLock(&MetaCache->SpinLock);
+    InitializeListHead(&MetaCache->ItemList);
+    MetaCache->MetaCapacity = MetaCapacity;
+    MetaCache->ItemSizeMax = ItemSizeMax;
+    MetaCache->MetaTimeout = MetaTimeout->QuadPart;
+    MetaCache->ItemBucketCount = BucketCount;
+    *PMetaCache = MetaCache;
+    return STATUS_SUCCESS;
+}
+
+VOID FspMetaCacheDelete(FSP_META_CACHE *MetaCache)
+{
+    if (0 == MetaCache)
+        return;
+    FspMetaCacheInvalidateExpired(MetaCache, (UINT64)-1LL);
+    FspFree(MetaCache);
+}
+
+VOID FspMetaCacheInvalidateExpired(FSP_META_CACHE *MetaCache, UINT64 ExpirationTime)
+{
+    if (0 == MetaCache)
+        return;
+    FSP_META_CACHE_ITEM *Item;
+    KIRQL Irql;
+    for (;;)
+    {
+        KeAcquireSpinLock(&MetaCache->SpinLock, &Irql);
+        Item = FspMetaCacheRemoveExpiredItemAtDpcLevel(MetaCache, ExpirationTime);
+        KeReleaseSpinLock(&MetaCache->SpinLock, Irql);
+        if (0 == Item)
+            break;
+        FspMetaCacheDereferenceItem(Item);
+    }
+}
+
+PVOID FspMetaCacheReferenceItemBuffer(FSP_META_CACHE *MetaCache, UINT64 ItemIndex, PULONG PSize)
+{
+    if (0 != PSize)
+        *PSize = 0;
+    if (0 == MetaCache || 0 == ItemIndex)
+        return 0;
+    FSP_META_CACHE_ITEM *Item = 0;
+    FSP_META_CACHE_ITEM_BUFFER *ItemBuffer;
+    KIRQL Irql;
+    KeAcquireSpinLock(&MetaCache->SpinLock, &Irql);
+    Item = FspMetaCacheLookupIndexedItemAtDpcLevel(MetaCache, ItemIndex);
+    if (0 == Item)
+    {
+        KeReleaseSpinLock(&MetaCache->SpinLock, Irql);
+        return 0;
+    }
+    InterlockedIncrement(&Item->RefCount);
+    KeReleaseSpinLock(&MetaCache->SpinLock, Irql);
+    ItemBuffer = Item->ItemBuffer;
+    if (0 != PSize)
+        *PSize = ItemBuffer->Size;
+    return ItemBuffer->Buffer;
+}
+
+VOID FspMetaCacheDereferenceItemBuffer(PVOID Buffer)
+{
+    FSP_META_CACHE_ITEM_BUFFER *ItemBuffer = (PVOID)((PUINT8)Buffer - sizeof *ItemBuffer);
+    FspMetaCacheDereferenceItem(ItemBuffer->Item);
+}
+
+UINT64 FspMetaCacheAddItem(FSP_META_CACHE *MetaCache, PVOID Buffer, ULONG Size)
+{
+    if (0 == MetaCache)
+        return 0;
+    FSP_META_CACHE_ITEM *Item;
+    FSP_META_CACHE_ITEM_BUFFER *ItemBuffer;
+    UINT64 ItemIndex = 0;
+    KIRQL Irql;
+    if (sizeof *ItemBuffer + Size > MetaCache->ItemSizeMax)
+        return 0;
+    Item = FspAllocNonPaged(sizeof *Item);
+    if (0 == Item)
+        return 0;
+    ItemBuffer = FspAlloc(sizeof *ItemBuffer + Size);
+    if (0 == ItemBuffer)
+    {
+        FspFree(Item);
+        return 0;
+    }
+    RtlZeroMemory(Item, sizeof *Item);
+    RtlZeroMemory(ItemBuffer, sizeof *ItemBuffer);
+    Item->ItemBuffer = ItemBuffer;
+    Item->ExpirationTime = KeQueryInterruptTime() + MetaCache->MetaTimeout;
+    Item->RefCount = 1;
+    ItemBuffer->Item = Item;
+    ItemBuffer->Size = Size;
+    RtlCopyMemory(ItemBuffer->Buffer, Buffer, Size);
+    KeAcquireSpinLock(&MetaCache->SpinLock, &Irql);
+    if (MetaCache->ItemCount >= MetaCache->MetaCapacity)
+        FspMetaCacheRemoveExpiredItemAtDpcLevel(MetaCache, (UINT64)-1LL);
+    ItemIndex = MetaCache->ItemIndex;
+    ItemIndex = (UINT64)-1LL == ItemIndex ? 1 : ItemIndex + 1;
+    MetaCache->ItemIndex = Item->ItemIndex = ItemIndex;
+    FspMetaCacheAddItemAtDpcLevel(MetaCache, Item);
+    KeReleaseSpinLock(&MetaCache->SpinLock, Irql);
+    return ItemIndex;
+}
+
+VOID FspMetaCacheInvalidateItem(FSP_META_CACHE *MetaCache, UINT64 ItemIndex)
+{
+    if (0 == MetaCache || 0 == ItemIndex)
+        return;
+    FSP_META_CACHE_ITEM *Item;
+    KIRQL Irql;
+    KeAcquireSpinLock(&MetaCache->SpinLock, &Irql);
+    Item = FspMetaCacheRemoveIndexedItemAtDpcLevel(MetaCache, ItemIndex);
+    KeReleaseSpinLock(&MetaCache->SpinLock, Irql);
+    if (0 != Item)
+        FspMetaCacheDereferenceItem(Item);
 }