BlockStorage/repertory
1
0
Fork 0
Code Issues 7 Pull Requests Releases 6 Wiki Activity

[bug] Address slow directory responses in S3 mounts for deep nested directories #28

Browse Source
This commit is contained in:
Scott E. Graves 2024-12-31 14:22:55 -06:00
parent a251cf6abf
commit 7c5f074d5a
2 changed files with 120 additions and 134 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

82
repertory/librepertory/src/drives/fuse/fuse_drive.cpp
View File

@ -81,8 +81,8 @@ auto fuse_drive::chown_impl(std::string api_path, uid_t uid, gid_t gid,
struct fuse_file_info * /*file_info*/)
-> api_error {
#else
auto fuse_drive::chown_impl(std::string api_path, uid_t uid,
gid_t gid) -> api_error {
auto fuse_drive::chown_impl(std::string api_path, uid_t uid, gid_t gid)
-> api_error {
#endif
return check_and_perform(
api_path, X_OK, [&](api_meta_map &meta) -> api_error {
@ -363,13 +363,8 @@ auto fuse_drive::fgetattr_impl(std::string api_path, struct stat *unix_st,
return res;
}
bool directory{};
res = provider_.is_directory(api_path, directory);
if (res != api_error::success) {
return res;
}
fuse_drive_base::populate_stat(api_path, open_file->get_file_size(), meta,
directory, provider_, unix_st);
open_file->is_directory(), provider_, unix_st);
return api_error::success;
}
@ -481,8 +476,8 @@ auto fuse_drive::getattr_impl(std::string api_path, struct stat *unix_st,
struct fuse_file_info * /*file_info*/)
-> api_error {
#else
auto fuse_drive::getattr_impl(std::string api_path,
struct stat *unix_st) -> api_error {
auto fuse_drive::getattr_impl(std::string api_path, struct stat *unix_st)
-> api_error {
#endif
auto parent = utils::path::get_parent_api_path(api_path);
@ -508,14 +503,9 @@ auto fuse_drive::getattr_impl(std::string api_path,
return res;
}
bool directory{};
res = provider_.is_directory(api_path, directory);
if (res != api_error::success) {
return res;
}
fuse_drive_base::populate_stat(api_path,
utils::string::to_uint64(meta[META_SIZE]),
meta, directory, provider_, unix_st);
fuse_drive_base::populate_stat(
api_path, utils::string::to_uint64(meta[META_SIZE]), meta,
utils::string::to_bool(meta[META_DIRECTORY]), provider_, unix_st);
}
return api_error::success;
@ -565,8 +555,8 @@ auto fuse_drive::getxtimes_impl(std::string api_path, struct timespec *bkuptime,
#endif // __APPLE__
#if FUSE_USE_VERSION >= 30
auto fuse_drive::init_impl(struct fuse_conn_info *conn,
struct fuse_config *cfg) -> void * {
auto fuse_drive::init_impl(struct fuse_conn_info *conn, struct fuse_config *cfg)
-> void * {
#else
void *fuse_drive::init_impl(struct fuse_conn_info *conn) {
#endif
@ -803,8 +793,9 @@ auto fuse_drive::release_impl(std::string /*api_path*/,
return api_error::success;
}
auto fuse_drive::releasedir_impl(
std::string /*api_path*/, struct fuse_file_info *file_info) -> api_error {
auto fuse_drive::releasedir_impl(std::string /*api_path*/,
struct fuse_file_info *file_info)
-> api_error {
auto iter = directory_cache_->get_directory(file_info->fh);
if (iter == nullptr) {
return api_error::invalid_handle;
@ -822,8 +813,8 @@ auto fuse_drive::rename_directory(const std::string &from_api_path,
}
auto fuse_drive::rename_file(const std::string &from_api_path,
const std::string &to_api_path,
bool overwrite) -> int {
const std::string &to_api_path, bool overwrite)
-> int {
auto res = fm_->rename_file(from_api_path, to_api_path, overwrite);
errno = std::abs(utils::from_api_error(res));
return (res == api_error::success) ? 0 : -1;
@ -833,8 +824,8 @@ auto fuse_drive::rename_file(const std::string &from_api_path,
auto fuse_drive::rename_impl(std::string from_api_path, std::string to_api_path,
unsigned int /*flags*/) -> api_error {
#else
auto fuse_drive::rename_impl(std::string from_api_path,
std::string to_api_path) -> api_error {
auto fuse_drive::rename_impl(std::string from_api_path, std::string to_api_path)
-> api_error {
#endif
auto res = check_parent_access(to_api_path, W_OK | X_OK);
if (res != api_error::success) {
@ -948,15 +939,15 @@ auto fuse_drive::getxattr_impl(std::string api_path, const char *name,
}
#else // __APPLE__
auto fuse_drive::getxattr_impl(std::string api_path, const char *name,
char *value, size_t size,
int &attribute_size) -> api_error {
char *value, size_t size, int &attribute_size)
-> api_error {
return getxattr_common(api_path, name, value, size, attribute_size, nullptr);
}
#endif // __APPLE__
auto fuse_drive::listxattr_impl(std::string api_path, char *buffer, size_t size,
int &required_size,
bool &return_size) -> api_error {
int &required_size, bool &return_size)
-> api_error {
auto check_size = (size == 0);
auto res = check_parent_access(api_path, X_OK);
@ -996,8 +987,8 @@ auto fuse_drive::listxattr_impl(std::string api_path, char *buffer, size_t size,
return res;
}
auto fuse_drive::removexattr_impl(std::string api_path,
const char *name) -> api_error {
auto fuse_drive::removexattr_impl(std::string api_path, const char *name)
-> api_error {
std::string attribute_name;
#if defined(__APPLE__)
auto res = parse_xattr_parameters(name, 0, attribute_name, api_path);
@ -1025,8 +1016,8 @@ auto fuse_drive::setxattr_impl(std::string api_path, const char *name,
uint32_t position) -> api_error {
#else // __APPLE__
auto fuse_drive::setxattr_impl(std::string api_path, const char *name,
const char *value, size_t size,
int flags) -> api_error {
const char *value, size_t size, int flags)
-> api_error {
#endif
std::string attribute_name;
#if defined(__APPLE__)
@ -1104,8 +1095,8 @@ void fuse_drive::set_item_meta(const std::string &api_path,
}
#if defined(__APPLE__)
auto fuse_drive::setattr_x_impl(std::string api_path,
struct setattr_x *attr) -> api_error {
auto fuse_drive::setattr_x_impl(std::string api_path, struct setattr_x *attr)
-> api_error {
bool exists{};
auto res = provider_.is_file(api_path, exists);
if (res != api_error::success) {
@ -1159,7 +1150,7 @@ auto fuse_drive::setattr_x_impl(std::string api_path,
ts[0].tv_sec = attr->acctime.tv_sec;
ts[0].tv_nsec = attr->acctime.tv_nsec;
} else {
struct timeval tv {};
struct timeval tv{};
gettimeofday(&tv, NULL);
ts[0].tv_sec = tv.tv_sec;
ts[0].tv_nsec = tv.tv_usec * 1000;
@ -1204,8 +1195,9 @@ auto fuse_drive::setattr_x_impl(std::string api_path,
return api_error::success;
}
auto fuse_drive::setbkuptime_impl(
std::string api_path, const struct timespec *bkuptime) -> api_error {
auto fuse_drive::setbkuptime_impl(std::string api_path,
const struct timespec *bkuptime)
-> api_error {
return check_and_perform(
api_path, X_OK, [&](api_meta_map &meta) -> api_error {
auto nanos = bkuptime->tv_nsec +
@ -1241,8 +1233,8 @@ auto fuse_drive::setvolname_impl(const char * /*volname*/) -> api_error {
return api_error::success;
}
auto fuse_drive::statfs_x_impl(std::string /*api_path*/,
struct statfs *stbuf) -> api_error {
auto fuse_drive::statfs_x_impl(std::string /*api_path*/, struct statfs *stbuf)
-> api_error {
if (statfs(&config_.get_cache_directory()[0], stbuf) != 0) {
return api_error::os_error;
}
@ -1267,8 +1259,8 @@ auto fuse_drive::statfs_x_impl(std::string /*api_path*/,
return api_error::success;
}
#else // __APPLE__
auto fuse_drive::statfs_impl(std::string /*api_path*/,
struct statvfs *stbuf) -> api_error {
auto fuse_drive::statfs_impl(std::string /*api_path*/, struct statvfs *stbuf)
-> api_error {
if (statvfs(config_.get_cache_directory().data(), stbuf) != 0) {
return api_error::os_error;
}
@ -1352,8 +1344,8 @@ auto fuse_drive::utimens_impl(std::string api_path, const struct timespec tv[2],
struct fuse_file_info * /*file_info*/)
-> api_error {
#else
auto fuse_drive::utimens_impl(std::string api_path,
const struct timespec tv[2]) -> api_error {
auto fuse_drive::utimens_impl(std::string api_path, const struct timespec tv[2])
-> api_error {
#endif
api_meta_map meta;
auto res = provider_.get_item_meta(api_path, meta);

172
repertory/librepertory/src/drives/winfsp/winfsp_drive.cpp
View File

@ -896,103 +896,97 @@ auto winfsp_drive::ReadDirectory(PVOID /*file_node*/, PVOID file_desc,
std::shared_ptr<i_open_file> file;
if (fm_->get_open_file(handle, false, file)) {
api_path = file->get_api_path();
bool exists{};
error = provider_.is_directory(api_path, exists);
if (error == api_error::success) {
if (exists) {
directory_item_list list{};
error = provider_.get_directory_items(api_path, list);
if (error == api_error::success) {
directory_iterator iterator(std::move(list));
auto status_result = STATUS_SUCCESS;
auto **directory_buffer =
&file->get_open_data(handle).directory_buffer;
if (FspFileSystemAcquireDirectoryBuffer(
directory_buffer, static_cast<BOOLEAN>(nullptr == marker),
&status_result) != 0U) {
directory_item dir_item{};
auto offset =
marker == nullptr
? 0U
: iterator.get_next_directory_offset(
utils::path::create_api_path(utils::path::combine(
api_path, {utils::string::to_utf8(marker)})));
while ((error = iterator.get_directory_item(
offset++, dir_item)) == api_error::success) {
if (dir_item.api_path == ".") {
auto res = provider_.get_item_meta(api_path, dir_item.meta);
if (res != api_error::success) {
error = res;
utils::error::raise_api_path_error(function_name,
dir_item.api_path, error,
"failed to get . meta");
break;
}
} else if (dir_item.api_path == "..") {
// TODO handle '/' parent
auto res = provider_.get_item_meta(
utils::path::get_parent_api_path(api_path),
dir_item.meta);
if (res != api_error::success) {
error = res;
utils::error::raise_api_path_error(function_name,
dir_item.api_path, error,
"failed to get .. meta");
break;
}
}
if (dir_item.meta.empty()) {
error = api_error::error;
utils::error::raise_api_path_error(
function_name, dir_item.api_path, api_error::error,
"item meta is empty");
if (file->is_directory()) {
directory_item_list list{};
error = provider_.get_directory_items(api_path, list);
if (error == api_error::success) {
directory_iterator iterator(std::move(list));
auto status_result = STATUS_SUCCESS;
auto **directory_buffer =
&file->get_open_data(handle).directory_buffer;
if (FspFileSystemAcquireDirectoryBuffer(
directory_buffer, static_cast<BOOLEAN>(nullptr == marker),
&status_result) != 0U) {
directory_item dir_item{};
auto offset =
marker == nullptr
? 0U
: iterator.get_next_directory_offset(
utils::path::create_api_path(utils::path::combine(
api_path, {utils::string::to_utf8(marker)})));
while ((error = iterator.get_directory_item(offset++, dir_item)) ==
api_error::success) {
if (dir_item.api_path == ".") {
auto res = provider_.get_item_meta(api_path, dir_item.meta);
if (res != api_error::success) {
error = res;
utils::error::raise_api_path_error(function_name,
dir_item.api_path, error,
"failed to get . meta");
break;
}
} else if (dir_item.api_path == "..") {
// TODO handle '/' parent
auto res = provider_.get_item_meta(
utils::path::get_parent_api_path(api_path), dir_item.meta);
if (res != api_error::success) {
error = res;
utils::error::raise_api_path_error(function_name,
dir_item.api_path, error,
"failed to get .. meta");
break;
}
auto display_name = utils::string::from_utf8(
utils::path::strip_to_file_name(dir_item.api_path));
union {
UINT8 B[FIELD_OFFSET(FSP_FSCTL_DIR_INFO, FileNameBuf) +
((repertory::max_path_length + 1U) * sizeof(WCHAR))];
FSP_FSCTL_DIR_INFO D;
} directory_info_buffer;
auto *directory_info = &directory_info_buffer.D;
::ZeroMemory(directory_info, sizeof(*directory_info));
directory_info->Size = static_cast<UINT16>(
FIELD_OFFSET(FSP_FSCTL_DIR_INFO, FileNameBuf) +
(std::min(static_cast<size_t>(repertory::max_path_length),
display_name.size()) *
sizeof(WCHAR)));
populate_file_info(dir_item.size, dir_item.meta,
directory_info->FileInfo);
::wcscpy_s(&directory_info->FileNameBuf[0U],
repertory::max_path_length, display_name.data());
FspFileSystemFillDirectoryBuffer(
directory_buffer, directory_info, &status_result);
}
FspFileSystemReleaseDirectoryBuffer(directory_buffer);
if (dir_item.meta.empty()) {
error = api_error::error;
utils::error::raise_api_path_error(
function_name, dir_item.api_path, api_error::error,
"item meta is empty");
break;
}
auto display_name = utils::string::from_utf8(
utils::path::strip_to_file_name(dir_item.api_path));
union {
UINT8 B[FIELD_OFFSET(FSP_FSCTL_DIR_INFO, FileNameBuf) +
((repertory::max_path_length + 1U) * sizeof(WCHAR))];
FSP_FSCTL_DIR_INFO D;
} directory_info_buffer;
auto *directory_info = &directory_info_buffer.D;
::ZeroMemory(directory_info, sizeof(*directory_info));
directory_info->Size = static_cast<UINT16>(
FIELD_OFFSET(FSP_FSCTL_DIR_INFO, FileNameBuf) +
(std::min(static_cast<size_t>(repertory::max_path_length),
display_name.size()) *
sizeof(WCHAR)));
populate_file_info(dir_item.size, dir_item.meta,
directory_info->FileInfo);
::wcscpy_s(&directory_info->FileNameBuf[0U],
repertory::max_path_length, display_name.data());
FspFileSystemFillDirectoryBuffer(directory_buffer, directory_info,
&status_result);
}
if (status_result == STATUS_SUCCESS) {
FspFileSystemReadDirectoryBuffer(directory_buffer, marker, buffer,
buffer_length,
bytes_transferred);
if (error == api_error::directory_end_of_files) {
error = api_error::success;
}
} else {
RAISE_WINFSP_EVENT(function_name, api_path, status_result);
return status_result;
}
FspFileSystemReleaseDirectoryBuffer(directory_buffer);
}
if (status_result == STATUS_SUCCESS) {
FspFileSystemReadDirectoryBuffer(directory_buffer, marker, buffer,
buffer_length, bytes_transferred);
if (error == api_error::directory_end_of_files) {
error = api_error::success;
}
} else {
RAISE_WINFSP_EVENT(function_name, api_path, status_result);
return status_result;
}
} else {
error = api_error::directory_not_found;
}
} else {
error = api_error::directory_not_found;
}
}
}