BlockStorage/repertory
1
0
Fork 0
Code Issues 7 Pull Requests Releases 6 Wiki Activity

[bug] Address slow directory responses in S3 mounts for deep nested directories #28

Browse Source
This commit is contained in:
Scott E. Graves 2024-12-31 14:22:55 -06:00
parent a251cf6abf
commit 7c5f074d5a
2 changed files with 120 additions and 134 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

82
repertory/librepertory/src/drives/fuse/fuse_drive.cpp
View File

@ -81,8 +81,8 @@ auto fuse_drive::chown_impl(std::string api_path, uid_t uid, gid_t gid,
struct fuse_file_info * /*file_info*/) struct fuse_file_info * /*file_info*/)
-> api_error { -> api_error {
#else #else
auto fuse_drive::chown_impl(std::string api_path, uid_t uid, auto fuse_drive::chown_impl(std::string api_path, uid_t uid, gid_t gid)
gid_t gid) -> api_error { -> api_error {
#endif #endif
return check_and_perform( return check_and_perform(
api_path, X_OK, [&](api_meta_map &meta) -> api_error { api_path, X_OK, [&](api_meta_map &meta) -> api_error {
@ -363,13 +363,8 @@ auto fuse_drive::fgetattr_impl(std::string api_path, struct stat *unix_st,
return res; return res;
} }
bool directory{};
res = provider_.is_directory(api_path, directory);
if (res != api_error::success) {
return res;
}
fuse_drive_base::populate_stat(api_path, open_file->get_file_size(), meta, fuse_drive_base::populate_stat(api_path, open_file->get_file_size(), meta,
directory, provider_, unix_st); open_file->is_directory(), provider_, unix_st);
return api_error::success; return api_error::success;
} }
@ -481,8 +476,8 @@ auto fuse_drive::getattr_impl(std::string api_path, struct stat *unix_st,
struct fuse_file_info * /*file_info*/) struct fuse_file_info * /*file_info*/)
-> api_error { -> api_error {
#else #else
auto fuse_drive::getattr_impl(std::string api_path, auto fuse_drive::getattr_impl(std::string api_path, struct stat *unix_st)
struct stat *unix_st) -> api_error { -> api_error {
#endif #endif
auto parent = utils::path::get_parent_api_path(api_path); auto parent = utils::path::get_parent_api_path(api_path);
@ -508,14 +503,9 @@ auto fuse_drive::getattr_impl(std::string api_path,
return res; return res;
} }
bool directory{}; fuse_drive_base::populate_stat(
res = provider_.is_directory(api_path, directory); api_path, utils::string::to_uint64(meta[META_SIZE]), meta,
if (res != api_error::success) { utils::string::to_bool(meta[META_DIRECTORY]), provider_, unix_st);
return res;
}
fuse_drive_base::populate_stat(api_path,
utils::string::to_uint64(meta[META_SIZE]),
meta, directory, provider_, unix_st);
} }
return api_error::success; return api_error::success;
@ -565,8 +555,8 @@ auto fuse_drive::getxtimes_impl(std::string api_path, struct timespec *bkuptime,
#endif // __APPLE__ #endif // __APPLE__
#if FUSE_USE_VERSION >= 30 #if FUSE_USE_VERSION >= 30
auto fuse_drive::init_impl(struct fuse_conn_info *conn, auto fuse_drive::init_impl(struct fuse_conn_info *conn, struct fuse_config *cfg)
struct fuse_config *cfg) -> void * { -> void * {
#else #else
void *fuse_drive::init_impl(struct fuse_conn_info *conn) { void *fuse_drive::init_impl(struct fuse_conn_info *conn) {
#endif #endif
@ -803,8 +793,9 @@ auto fuse_drive::release_impl(std::string /*api_path*/,
return api_error::success; return api_error::success;
} }
auto fuse_drive::releasedir_impl( auto fuse_drive::releasedir_impl(std::string /*api_path*/,
std::string /*api_path*/, struct fuse_file_info *file_info) -> api_error { struct fuse_file_info *file_info)
-> api_error {
auto iter = directory_cache_->get_directory(file_info->fh); auto iter = directory_cache_->get_directory(file_info->fh);
if (iter == nullptr) { if (iter == nullptr) {
return api_error::invalid_handle; return api_error::invalid_handle;
@ -822,8 +813,8 @@ auto fuse_drive::rename_directory(const std::string &from_api_path,
} }
auto fuse_drive::rename_file(const std::string &from_api_path, auto fuse_drive::rename_file(const std::string &from_api_path,
const std::string &to_api_path, const std::string &to_api_path, bool overwrite)
bool overwrite) -> int { -> int {
auto res = fm_->rename_file(from_api_path, to_api_path, overwrite); auto res = fm_->rename_file(from_api_path, to_api_path, overwrite);
errno = std::abs(utils::from_api_error(res)); errno = std::abs(utils::from_api_error(res));
return (res == api_error::success) ? 0 : -1; return (res == api_error::success) ? 0 : -1;
@ -833,8 +824,8 @@ auto fuse_drive::rename_file(const std::string &from_api_path,
auto fuse_drive::rename_impl(std::string from_api_path, std::string to_api_path, auto fuse_drive::rename_impl(std::string from_api_path, std::string to_api_path,
unsigned int /*flags*/) -> api_error { unsigned int /*flags*/) -> api_error {
#else #else
auto fuse_drive::rename_impl(std::string from_api_path, auto fuse_drive::rename_impl(std::string from_api_path, std::string to_api_path)
std::string to_api_path) -> api_error { -> api_error {
#endif #endif
auto res = check_parent_access(to_api_path, W_OK | X_OK); auto res = check_parent_access(to_api_path, W_OK | X_OK);
if (res != api_error::success) { if (res != api_error::success) {
@ -948,15 +939,15 @@ auto fuse_drive::getxattr_impl(std::string api_path, const char *name,
} }
#else // __APPLE__ #else // __APPLE__
auto fuse_drive::getxattr_impl(std::string api_path, const char *name, auto fuse_drive::getxattr_impl(std::string api_path, const char *name,
char *value, size_t size, char *value, size_t size, int &attribute_size)
int &attribute_size) -> api_error { -> api_error {
return getxattr_common(api_path, name, value, size, attribute_size, nullptr); return getxattr_common(api_path, name, value, size, attribute_size, nullptr);
} }
#endif // __APPLE__ #endif // __APPLE__
auto fuse_drive::listxattr_impl(std::string api_path, char *buffer, size_t size, auto fuse_drive::listxattr_impl(std::string api_path, char *buffer, size_t size,
int &required_size, int &required_size, bool &return_size)
bool &return_size) -> api_error { -> api_error {
auto check_size = (size == 0); auto check_size = (size == 0);
auto res = check_parent_access(api_path, X_OK); auto res = check_parent_access(api_path, X_OK);
@ -996,8 +987,8 @@ auto fuse_drive::listxattr_impl(std::string api_path, char *buffer, size_t size,
return res; return res;
} }
auto fuse_drive::removexattr_impl(std::string api_path, auto fuse_drive::removexattr_impl(std::string api_path, const char *name)
const char *name) -> api_error { -> api_error {
std::string attribute_name; std::string attribute_name;
#if defined(__APPLE__) #if defined(__APPLE__)
auto res = parse_xattr_parameters(name, 0, attribute_name, api_path); auto res = parse_xattr_parameters(name, 0, attribute_name, api_path);
@ -1025,8 +1016,8 @@ auto fuse_drive::setxattr_impl(std::string api_path, const char *name,
uint32_t position) -> api_error { uint32_t position) -> api_error {
#else // __APPLE__ #else // __APPLE__
auto fuse_drive::setxattr_impl(std::string api_path, const char *name, auto fuse_drive::setxattr_impl(std::string api_path, const char *name,
const char *value, size_t size, const char *value, size_t size, int flags)
int flags) -> api_error { -> api_error {
#endif #endif
std::string attribute_name; std::string attribute_name;
#if defined(__APPLE__) #if defined(__APPLE__)
@ -1104,8 +1095,8 @@ void fuse_drive::set_item_meta(const std::string &api_path,
} }
#if defined(__APPLE__) #if defined(__APPLE__)
auto fuse_drive::setattr_x_impl(std::string api_path, auto fuse_drive::setattr_x_impl(std::string api_path, struct setattr_x *attr)
struct setattr_x *attr) -> api_error { -> api_error {
bool exists{}; bool exists{};
auto res = provider_.is_file(api_path, exists); auto res = provider_.is_file(api_path, exists);
if (res != api_error::success) { if (res != api_error::success) {
@ -1159,7 +1150,7 @@ auto fuse_drive::setattr_x_impl(std::string api_path,
ts[0].tv_sec = attr->acctime.tv_sec; ts[0].tv_sec = attr->acctime.tv_sec;
ts[0].tv_nsec = attr->acctime.tv_nsec; ts[0].tv_nsec = attr->acctime.tv_nsec;
} else { } else {
struct timeval tv {}; struct timeval tv{};
gettimeofday(&tv, NULL); gettimeofday(&tv, NULL);
ts[0].tv_sec = tv.tv_sec; ts[0].tv_sec = tv.tv_sec;
ts[0].tv_nsec = tv.tv_usec * 1000; ts[0].tv_nsec = tv.tv_usec * 1000;
@ -1204,8 +1195,9 @@ auto fuse_drive::setattr_x_impl(std::string api_path,
return api_error::success; return api_error::success;
} }
auto fuse_drive::setbkuptime_impl( auto fuse_drive::setbkuptime_impl(std::string api_path,
std::string api_path, const struct timespec *bkuptime) -> api_error { const struct timespec *bkuptime)
-> api_error {
return check_and_perform( return check_and_perform(
api_path, X_OK, [&](api_meta_map &meta) -> api_error { api_path, X_OK, [&](api_meta_map &meta) -> api_error {
auto nanos = bkuptime->tv_nsec + auto nanos = bkuptime->tv_nsec +
@ -1241,8 +1233,8 @@ auto fuse_drive::setvolname_impl(const char * /*volname*/) -> api_error {
return api_error::success; return api_error::success;
} }
auto fuse_drive::statfs_x_impl(std::string /*api_path*/, auto fuse_drive::statfs_x_impl(std::string /*api_path*/, struct statfs *stbuf)
struct statfs *stbuf) -> api_error { -> api_error {
if (statfs(&config_.get_cache_directory()[0], stbuf) != 0) { if (statfs(&config_.get_cache_directory()[0], stbuf) != 0) {
return api_error::os_error; return api_error::os_error;
} }
@ -1267,8 +1259,8 @@ auto fuse_drive::statfs_x_impl(std::string /*api_path*/,
return api_error::success; return api_error::success;
} }
#else // __APPLE__ #else // __APPLE__
auto fuse_drive::statfs_impl(std::string /*api_path*/, auto fuse_drive::statfs_impl(std::string /*api_path*/, struct statvfs *stbuf)
struct statvfs *stbuf) -> api_error { -> api_error {
if (statvfs(config_.get_cache_directory().data(), stbuf) != 0) { if (statvfs(config_.get_cache_directory().data(), stbuf) != 0) {
return api_error::os_error; return api_error::os_error;
} }
@ -1352,8 +1344,8 @@ auto fuse_drive::utimens_impl(std::string api_path, const struct timespec tv[2],
struct fuse_file_info * /*file_info*/) struct fuse_file_info * /*file_info*/)
-> api_error { -> api_error {
#else #else
auto fuse_drive::utimens_impl(std::string api_path, auto fuse_drive::utimens_impl(std::string api_path, const struct timespec tv[2])
const struct timespec tv[2]) -> api_error { -> api_error {
#endif #endif
api_meta_map meta; api_meta_map meta;
auto res = provider_.get_item_meta(api_path, meta); auto res = provider_.get_item_meta(api_path, meta);

20
repertory/librepertory/src/drives/winfsp/winfsp_drive.cpp
View File

@ -896,10 +896,7 @@ auto winfsp_drive::ReadDirectory(PVOID /*file_node*/, PVOID file_desc,
std::shared_ptr<i_open_file> file; std::shared_ptr<i_open_file> file;
if (fm_->get_open_file(handle, false, file)) { if (fm_->get_open_file(handle, false, file)) {
api_path = file->get_api_path(); api_path = file->get_api_path();
bool exists{}; if (file->is_directory()) {
error = provider_.is_directory(api_path, exists);
if (error == api_error::success) {
if (exists) {
directory_item_list list{}; directory_item_list list{};
error = provider_.get_directory_items(api_path, list); error = provider_.get_directory_items(api_path, list);
if (error == api_error::success) { if (error == api_error::success) {
@ -917,8 +914,8 @@ auto winfsp_drive::ReadDirectory(PVOID /*file_node*/, PVOID file_desc,
: iterator.get_next_directory_offset( : iterator.get_next_directory_offset(
utils::path::create_api_path(utils::path::combine( utils::path::create_api_path(utils::path::combine(
api_path, {utils::string::to_utf8(marker)}))); api_path, {utils::string::to_utf8(marker)})));
while ((error = iterator.get_directory_item( while ((error = iterator.get_directory_item(offset++, dir_item)) ==
offset++, dir_item)) == api_error::success) { api_error::success) {
if (dir_item.api_path == ".") { if (dir_item.api_path == ".") {
auto res = provider_.get_item_meta(api_path, dir_item.meta); auto res = provider_.get_item_meta(api_path, dir_item.meta);
if (res != api_error::success) { if (res != api_error::success) {
@ -931,8 +928,7 @@ auto winfsp_drive::ReadDirectory(PVOID /*file_node*/, PVOID file_desc,
} else if (dir_item.api_path == "..") { } else if (dir_item.api_path == "..") {
// TODO handle '/' parent // TODO handle '/' parent
auto res = provider_.get_item_meta( auto res = provider_.get_item_meta(
utils::path::get_parent_api_path(api_path), utils::path::get_parent_api_path(api_path), dir_item.meta);
dir_item.meta);
if (res != api_error::success) { if (res != api_error::success) {
error = res; error = res;
utils::error::raise_api_path_error(function_name, utils::error::raise_api_path_error(function_name,
@ -971,8 +967,8 @@ auto winfsp_drive::ReadDirectory(PVOID /*file_node*/, PVOID file_desc,
::wcscpy_s(&directory_info->FileNameBuf[0U], ::wcscpy_s(&directory_info->FileNameBuf[0U],
repertory::max_path_length, display_name.data()); repertory::max_path_length, display_name.data());
FspFileSystemFillDirectoryBuffer( FspFileSystemFillDirectoryBuffer(directory_buffer, directory_info,
directory_buffer, directory_info, &status_result); &status_result);
} }
FspFileSystemReleaseDirectoryBuffer(directory_buffer); FspFileSystemReleaseDirectoryBuffer(directory_buffer);
@ -980,8 +976,7 @@ auto winfsp_drive::ReadDirectory(PVOID /*file_node*/, PVOID file_desc,
if (status_result == STATUS_SUCCESS) { if (status_result == STATUS_SUCCESS) {
FspFileSystemReadDirectoryBuffer(directory_buffer, marker, buffer, FspFileSystemReadDirectoryBuffer(directory_buffer, marker, buffer,
buffer_length, buffer_length, bytes_transferred);
bytes_transferred);
if (error == api_error::directory_end_of_files) { if (error == api_error::directory_end_of_files) {
error = api_error::success; error = api_error::success;
} }
@ -995,7 +990,6 @@ auto winfsp_drive::ReadDirectory(PVOID /*file_node*/, PVOID file_desc,
} }
} }
} }
}
auto ret = utils::from_api_error(error); auto ret = utils::from_api_error(error);
RAISE_WINFSP_EVENT(function_name, api_path, ret); RAISE_WINFSP_EVENT(function_name, api_path, ret);