BlockStorage/repertory
1
0
Fork 0
Code Issues 7 Pull Requests Releases 6 Wiki Activity

[bug] Address slow directory responses in S3 mounts for deep nested directories #28

Browse Source
This commit is contained in:
Scott E. Graves 2024-12-31 14:22:55 -06:00
parent a251cf6abf
commit 7c5f074d5a
2 changed files with 120 additions and 134 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

82
repertory/librepertory/src/drives/fuse/fuse_drive.cpp
View File

@ -81,8 +81,8 @@ auto fuse_drive::chown_impl(std::string api_path, uid_t uid, gid_t gid,
struct fuse_file_info * /*file_info*/) struct fuse_file_info * /*file_info*/)
-> api_error { -> api_error {
#else #else
auto fuse_drive::chown_impl(std::string api_path, uid_t uid, auto fuse_drive::chown_impl(std::string api_path, uid_t uid, gid_t gid)
gid_t gid) -> api_error { -> api_error {
#endif #endif
return check_and_perform( return check_and_perform(
api_path, X_OK, [&](api_meta_map &meta) -> api_error { api_path, X_OK, [&](api_meta_map &meta) -> api_error {
@ -363,13 +363,8 @@ auto fuse_drive::fgetattr_impl(std::string api_path, struct stat *unix_st,
return res; return res;
} }
bool directory{};
res = provider_.is_directory(api_path, directory);
if (res != api_error::success) {
return res;
}
fuse_drive_base::populate_stat(api_path, open_file->get_file_size(), meta, fuse_drive_base::populate_stat(api_path, open_file->get_file_size(), meta,
directory, provider_, unix_st); open_file->is_directory(), provider_, unix_st);
return api_error::success; return api_error::success;
} }
@ -481,8 +476,8 @@ auto fuse_drive::getattr_impl(std::string api_path, struct stat *unix_st,
struct fuse_file_info * /*file_info*/) struct fuse_file_info * /*file_info*/)
-> api_error { -> api_error {
#else #else
auto fuse_drive::getattr_impl(std::string api_path, auto fuse_drive::getattr_impl(std::string api_path, struct stat *unix_st)
struct stat *unix_st) -> api_error { -> api_error {
#endif #endif
auto parent = utils::path::get_parent_api_path(api_path); auto parent = utils::path::get_parent_api_path(api_path);
@ -508,14 +503,9 @@ auto fuse_drive::getattr_impl(std::string api_path,
return res; return res;
} }
bool directory{}; fuse_drive_base::populate_stat(
res = provider_.is_directory(api_path, directory); api_path, utils::string::to_uint64(meta[META_SIZE]), meta,
if (res != api_error::success) { utils::string::to_bool(meta[META_DIRECTORY]), provider_, unix_st);
return res;
}
fuse_drive_base::populate_stat(api_path,
utils::string::to_uint64(meta[META_SIZE]),
meta, directory, provider_, unix_st);
} }
return api_error::success; return api_error::success;
@ -565,8 +555,8 @@ auto fuse_drive::getxtimes_impl(std::string api_path, struct timespec *bkuptime,
#endif // __APPLE__ #endif // __APPLE__
#if FUSE_USE_VERSION >= 30 #if FUSE_USE_VERSION >= 30
auto fuse_drive::init_impl(struct fuse_conn_info *conn, auto fuse_drive::init_impl(struct fuse_conn_info *conn, struct fuse_config *cfg)
struct fuse_config *cfg) -> void * { -> void * {
#else #else
void *fuse_drive::init_impl(struct fuse_conn_info *conn) { void *fuse_drive::init_impl(struct fuse_conn_info *conn) {
#endif #endif
@ -803,8 +793,9 @@ auto fuse_drive::release_impl(std::string /*api_path*/,
return api_error::success; return api_error::success;
} }
auto fuse_drive::releasedir_impl( auto fuse_drive::releasedir_impl(std::string /*api_path*/,
std::string /*api_path*/, struct fuse_file_info *file_info) -> api_error { struct fuse_file_info *file_info)
-> api_error {
auto iter = directory_cache_->get_directory(file_info->fh); auto iter = directory_cache_->get_directory(file_info->fh);
if (iter == nullptr) { if (iter == nullptr) {
return api_error::invalid_handle; return api_error::invalid_handle;
@ -822,8 +813,8 @@ auto fuse_drive::rename_directory(const std::string &from_api_path,
} }
auto fuse_drive::rename_file(const std::string &from_api_path, auto fuse_drive::rename_file(const std::string &from_api_path,
const std::string &to_api_path, const std::string &to_api_path, bool overwrite)
bool overwrite) -> int { -> int {
auto res = fm_->rename_file(from_api_path, to_api_path, overwrite); auto res = fm_->rename_file(from_api_path, to_api_path, overwrite);
errno = std::abs(utils::from_api_error(res)); errno = std::abs(utils::from_api_error(res));
return (res == api_error::success) ? 0 : -1; return (res == api_error::success) ? 0 : -1;
@ -833,8 +824,8 @@ auto fuse_drive::rename_file(const std::string &from_api_path,
auto fuse_drive::rename_impl(std::string from_api_path, std::string to_api_path, auto fuse_drive::rename_impl(std::string from_api_path, std::string to_api_path,
unsigned int /*flags*/) -> api_error { unsigned int /*flags*/) -> api_error {
#else #else
auto fuse_drive::rename_impl(std::string from_api_path, auto fuse_drive::rename_impl(std::string from_api_path, std::string to_api_path)
std::string to_api_path) -> api_error { -> api_error {
#endif #endif
auto res = check_parent_access(to_api_path, W_OK | X_OK); auto res = check_parent_access(to_api_path, W_OK | X_OK);
if (res != api_error::success) { if (res != api_error::success) {
@ -948,15 +939,15 @@ auto fuse_drive::getxattr_impl(std::string api_path, const char *name,
} }
#else // __APPLE__ #else // __APPLE__
auto fuse_drive::getxattr_impl(std::string api_path, const char *name, auto fuse_drive::getxattr_impl(std::string api_path, const char *name,
char *value, size_t size, char *value, size_t size, int &attribute_size)
int &attribute_size) -> api_error { -> api_error {
return getxattr_common(api_path, name, value, size, attribute_size, nullptr); return getxattr_common(api_path, name, value, size, attribute_size, nullptr);
} }
#endif // __APPLE__ #endif // __APPLE__
auto fuse_drive::listxattr_impl(std::string api_path, char *buffer, size_t size, auto fuse_drive::listxattr_impl(std::string api_path, char *buffer, size_t size,
int &required_size, int &required_size, bool &return_size)
bool &return_size) -> api_error { -> api_error {
auto check_size = (size == 0); auto check_size = (size == 0);
auto res = check_parent_access(api_path, X_OK); auto res = check_parent_access(api_path, X_OK);
@ -996,8 +987,8 @@ auto fuse_drive::listxattr_impl(std::string api_path, char *buffer, size_t size,
return res; return res;
} }
auto fuse_drive::removexattr_impl(std::string api_path, auto fuse_drive::removexattr_impl(std::string api_path, const char *name)
const char *name) -> api_error { -> api_error {
std::string attribute_name; std::string attribute_name;
#if defined(__APPLE__) #if defined(__APPLE__)
auto res = parse_xattr_parameters(name, 0, attribute_name, api_path); auto res = parse_xattr_parameters(name, 0, attribute_name, api_path);
@ -1025,8 +1016,8 @@ auto fuse_drive::setxattr_impl(std::string api_path, const char *name,
uint32_t position) -> api_error { uint32_t position) -> api_error {
#else // __APPLE__ #else // __APPLE__
auto fuse_drive::setxattr_impl(std::string api_path, const char *name, auto fuse_drive::setxattr_impl(std::string api_path, const char *name,
const char *value, size_t size, const char *value, size_t size, int flags)
int flags) -> api_error { -> api_error {
#endif #endif
std::string attribute_name; std::string attribute_name;
#if defined(__APPLE__) #if defined(__APPLE__)
@ -1104,8 +1095,8 @@ void fuse_drive::set_item_meta(const std::string &api_path,
} }
#if defined(__APPLE__) #if defined(__APPLE__)
auto fuse_drive::setattr_x_impl(std::string api_path, auto fuse_drive::setattr_x_impl(std::string api_path, struct setattr_x *attr)
struct setattr_x *attr) -> api_error { -> api_error {
bool exists{}; bool exists{};
auto res = provider_.is_file(api_path, exists); auto res = provider_.is_file(api_path, exists);
if (res != api_error::success) { if (res != api_error::success) {
@ -1159,7 +1150,7 @@ auto fuse_drive::setattr_x_impl(std::string api_path,
ts[0].tv_sec = attr->acctime.tv_sec; ts[0].tv_sec = attr->acctime.tv_sec;
ts[0].tv_nsec = attr->acctime.tv_nsec; ts[0].tv_nsec = attr->acctime.tv_nsec;
} else { } else {
struct timeval tv {}; struct timeval tv{};
gettimeofday(&tv, NULL); gettimeofday(&tv, NULL);
ts[0].tv_sec = tv.tv_sec; ts[0].tv_sec = tv.tv_sec;
ts[0].tv_nsec = tv.tv_usec * 1000; ts[0].tv_nsec = tv.tv_usec * 1000;
@ -1204,8 +1195,9 @@ auto fuse_drive::setattr_x_impl(std::string api_path,
return api_error::success; return api_error::success;
} }
auto fuse_drive::setbkuptime_impl( auto fuse_drive::setbkuptime_impl(std::string api_path,
std::string api_path, const struct timespec *bkuptime) -> api_error { const struct timespec *bkuptime)
-> api_error {
return check_and_perform( return check_and_perform(
api_path, X_OK, [&](api_meta_map &meta) -> api_error { api_path, X_OK, [&](api_meta_map &meta) -> api_error {
auto nanos = bkuptime->tv_nsec + auto nanos = bkuptime->tv_nsec +
@ -1241,8 +1233,8 @@ auto fuse_drive::setvolname_impl(const char * /*volname*/) -> api_error {
return api_error::success; return api_error::success;
} }
auto fuse_drive::statfs_x_impl(std::string /*api_path*/, auto fuse_drive::statfs_x_impl(std::string /*api_path*/, struct statfs *stbuf)
struct statfs *stbuf) -> api_error { -> api_error {
if (statfs(&config_.get_cache_directory()[0], stbuf) != 0) { if (statfs(&config_.get_cache_directory()[0], stbuf) != 0) {
return api_error::os_error; return api_error::os_error;
} }
@ -1267,8 +1259,8 @@ auto fuse_drive::statfs_x_impl(std::string /*api_path*/,
return api_error::success; return api_error::success;
} }
#else // __APPLE__ #else // __APPLE__
auto fuse_drive::statfs_impl(std::string /*api_path*/, auto fuse_drive::statfs_impl(std::string /*api_path*/, struct statvfs *stbuf)
struct statvfs *stbuf) -> api_error { -> api_error {
if (statvfs(config_.get_cache_directory().data(), stbuf) != 0) { if (statvfs(config_.get_cache_directory().data(), stbuf) != 0) {
return api_error::os_error; return api_error::os_error;
} }
@ -1352,8 +1344,8 @@ auto fuse_drive::utimens_impl(std::string api_path, const struct timespec tv[2],
struct fuse_file_info * /*file_info*/) struct fuse_file_info * /*file_info*/)
-> api_error { -> api_error {
#else #else
auto fuse_drive::utimens_impl(std::string api_path, auto fuse_drive::utimens_impl(std::string api_path, const struct timespec tv[2])
const struct timespec tv[2]) -> api_error { -> api_error {
#endif #endif
api_meta_map meta; api_meta_map meta;
auto res = provider_.get_item_meta(api_path, meta); auto res = provider_.get_item_meta(api_path, meta);

172
repertory/librepertory/src/drives/winfsp/winfsp_drive.cpp
View File

@ -896,103 +896,97 @@ auto winfsp_drive::ReadDirectory(PVOID /*file_node*/, PVOID file_desc,
std::shared_ptr<i_open_file> file; std::shared_ptr<i_open_file> file;
if (fm_->get_open_file(handle, false, file)) { if (fm_->get_open_file(handle, false, file)) {
api_path = file->get_api_path(); api_path = file->get_api_path();
bool exists{}; if (file->is_directory()) {
error = provider_.is_directory(api_path, exists); directory_item_list list{};
if (error == api_error::success) { error = provider_.get_directory_items(api_path, list);
if (exists) { if (error == api_error::success) {
directory_item_list list{}; directory_iterator iterator(std::move(list));
error = provider_.get_directory_items(api_path, list); auto status_result = STATUS_SUCCESS;
if (error == api_error::success) { auto **directory_buffer =
directory_iterator iterator(std::move(list)); &file->get_open_data(handle).directory_buffer;
auto status_result = STATUS_SUCCESS; if (FspFileSystemAcquireDirectoryBuffer(
auto **directory_buffer = directory_buffer, static_cast<BOOLEAN>(nullptr == marker),
&file->get_open_data(handle).directory_buffer; &status_result) != 0U) {
if (FspFileSystemAcquireDirectoryBuffer( directory_item dir_item{};
directory_buffer, static_cast<BOOLEAN>(nullptr == marker), auto offset =
&status_result) != 0U) { marker == nullptr
directory_item dir_item{}; ? 0U
auto offset = : iterator.get_next_directory_offset(
marker == nullptr utils::path::create_api_path(utils::path::combine(
? 0U api_path, {utils::string::to_utf8(marker)})));
: iterator.get_next_directory_offset( while ((error = iterator.get_directory_item(offset++, dir_item)) ==
utils::path::create_api_path(utils::path::combine( api_error::success) {
api_path, {utils::string::to_utf8(marker)}))); if (dir_item.api_path == ".") {
while ((error = iterator.get_directory_item( auto res = provider_.get_item_meta(api_path, dir_item.meta);
offset++, dir_item)) == api_error::success) { if (res != api_error::success) {
if (dir_item.api_path == ".") { error = res;
auto res = provider_.get_item_meta(api_path, dir_item.meta); utils::error::raise_api_path_error(function_name,
if (res != api_error::success) { dir_item.api_path, error,
error = res; "failed to get . meta");
utils::error::raise_api_path_error(function_name, break;
dir_item.api_path, error, }
"failed to get . meta"); } else if (dir_item.api_path == "..") {
break; // TODO handle '/' parent
} auto res = provider_.get_item_meta(
} else if (dir_item.api_path == "..") { utils::path::get_parent_api_path(api_path), dir_item.meta);
// TODO handle '/' parent if (res != api_error::success) {
auto res = provider_.get_item_meta( error = res;
utils::path::get_parent_api_path(api_path), utils::error::raise_api_path_error(function_name,
dir_item.meta); dir_item.api_path, error,
if (res != api_error::success) { "failed to get .. meta");
error = res;
utils::error::raise_api_path_error(function_name,
dir_item.api_path, error,
"failed to get .. meta");
break;
}
}
if (dir_item.meta.empty()) {
error = api_error::error;
utils::error::raise_api_path_error(
function_name, dir_item.api_path, api_error::error,
"item meta is empty");
break; break;
} }
auto display_name = utils::string::from_utf8(
utils::path::strip_to_file_name(dir_item.api_path));
union {
UINT8 B[FIELD_OFFSET(FSP_FSCTL_DIR_INFO, FileNameBuf) +
((repertory::max_path_length + 1U) * sizeof(WCHAR))];
FSP_FSCTL_DIR_INFO D;
} directory_info_buffer;
auto *directory_info = &directory_info_buffer.D;
::ZeroMemory(directory_info, sizeof(*directory_info));
directory_info->Size = static_cast<UINT16>(
FIELD_OFFSET(FSP_FSCTL_DIR_INFO, FileNameBuf) +
(std::min(static_cast<size_t>(repertory::max_path_length),
display_name.size()) *
sizeof(WCHAR)));
populate_file_info(dir_item.size, dir_item.meta,
directory_info->FileInfo);
::wcscpy_s(&directory_info->FileNameBuf[0U],
repertory::max_path_length, display_name.data());
FspFileSystemFillDirectoryBuffer(
directory_buffer, directory_info, &status_result);
} }
FspFileSystemReleaseDirectoryBuffer(directory_buffer); if (dir_item.meta.empty()) {
error = api_error::error;
utils::error::raise_api_path_error(
function_name, dir_item.api_path, api_error::error,
"item meta is empty");
break;
}
auto display_name = utils::string::from_utf8(
utils::path::strip_to_file_name(dir_item.api_path));
union {
UINT8 B[FIELD_OFFSET(FSP_FSCTL_DIR_INFO, FileNameBuf) +
((repertory::max_path_length + 1U) * sizeof(WCHAR))];
FSP_FSCTL_DIR_INFO D;
} directory_info_buffer;
auto *directory_info = &directory_info_buffer.D;
::ZeroMemory(directory_info, sizeof(*directory_info));
directory_info->Size = static_cast<UINT16>(
FIELD_OFFSET(FSP_FSCTL_DIR_INFO, FileNameBuf) +
(std::min(static_cast<size_t>(repertory::max_path_length),
display_name.size()) *
sizeof(WCHAR)));
populate_file_info(dir_item.size, dir_item.meta,
directory_info->FileInfo);
::wcscpy_s(&directory_info->FileNameBuf[0U],
repertory::max_path_length, display_name.data());
FspFileSystemFillDirectoryBuffer(directory_buffer, directory_info,
&status_result);
} }
if (status_result == STATUS_SUCCESS) { FspFileSystemReleaseDirectoryBuffer(directory_buffer);
FspFileSystemReadDirectoryBuffer(directory_buffer, marker, buffer, }
buffer_length,
bytes_transferred); if (status_result == STATUS_SUCCESS) {
if (error == api_error::directory_end_of_files) { FspFileSystemReadDirectoryBuffer(directory_buffer, marker, buffer,
error = api_error::success; buffer_length, bytes_transferred);
} if (error == api_error::directory_end_of_files) {
} else { error = api_error::success;
RAISE_WINFSP_EVENT(function_name, api_path, status_result); }
return status_result; } else {
} RAISE_WINFSP_EVENT(function_name, api_path, status_result);
return status_result;
} }
} else {
error = api_error::directory_not_found;
} }
} else {
error = api_error::directory_not_found;
} }
} }
} }