BlockStorage/repertory
1
0
Fork 0
Code Issues 8 Pull Requests Releases 6 Wiki Activity

refactor

Browse Source
This commit is contained in:
Scott E. Graves 2024-07-25 12:17:12 -05:00
parent 5a928208fd
commit de6a7beb5e
14 changed files with 148 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
repertory/librepertory/include/events/event.hpp
View File

@ -24,16 +24,17 @@
namespace repertory { namespace repertory {
enum class event_level { enum class event_level {
critical,
error, error,
warn, warn,
normal, info,
debug, debug,
verbose, trace,
}; };
auto event_level_from_string(std::string level) -> event_level; auto event_level_from_string(std::string level) -> event_level;
auto event_level_to_string(const event_level &level) -> std::string; auto event_level_to_string(event_level level) -> std::string;
class event { class event {
protected: protected:

129
repertory/librepertory/src/app_config.cpp
View File

@ -72,7 +72,7 @@ app_config::app_config(const provider_type &prov,
enable_mount_manager_(false), enable_mount_manager_(false),
#endif #endif
enable_remote_mount_(false), enable_remote_mount_(false),
event_level_(event_level::normal), event_level_(event_level::info),
eviction_delay_mins_(default_eviction_delay_mins), eviction_delay_mins_(default_eviction_delay_mins),
eviction_uses_accessed_time_(false), eviction_uses_accessed_time_(false),
high_freq_interval_secs_(default_high_freq_interval_secs), high_freq_interval_secs_(default_high_freq_interval_secs),
@ -195,72 +195,71 @@ auto app_config::default_rpc_port(const provider_type &prov) -> std::uint16_t {
auto app_config::get_json() const -> json { auto app_config::get_json() const -> json {
json ret = { json ret = {
{"ApiAuth", api_auth_}, {"ApiAuth", api_auth_},
{"ApiPort", api_port_}, {"ApiPort", api_port_},
{"ApiUser", api_user_}, {"ApiUser", api_user_},
{"ChunkDownloaderTimeoutSeconds", download_timeout_secs_}, {"ChunkDownloaderTimeoutSeconds", download_timeout_secs_},
{"EnableChunkDownloaderTimeout", enable_chunk_downloader_timeout_}, {"EnableChunkDownloaderTimeout", enable_chunk_downloader_timeout_},
{"EnableCommDurationEvents", enable_comm_duration_events_}, {"EnableCommDurationEvents", enable_comm_duration_events_},
{"EnableDriveEvents", enable_drive_events_}, {"EnableDriveEvents", enable_drive_events_},
#if defined(_WIN32) #if defined(_WIN32)
{"EnableMountManager", enable_mount_manager_}, {"EnableMountManager", enable_mount_manager_},
#endif #endif
{"EnableMaxCacheSize", enable_max_cache_size_}, {"EnableMaxCacheSize", enable_max_cache_size_},
{"EncryptConfig", {"EncryptConfig",
{ {
{"EncryptionToken", encrypt_config_.encryption_token}, {"EncryptionToken", encrypt_config_.encryption_token},
{"Path", encrypt_config_.path}, {"Path", encrypt_config_.path},
}}, }},
{"EventLevel", event_level_to_string(event_level_)}, {"EventLevel", event_level_to_string(event_level_)},
{"EvictionDelayMinutes", eviction_delay_mins_}, {"EvictionDelayMinutes", eviction_delay_mins_},
{"EvictionUsesAccessedTime", eviction_uses_accessed_time_}, {"EvictionUsesAccessedTime", eviction_uses_accessed_time_},
{"HighFreqIntervalSeconds", high_freq_interval_secs_}, {"HighFreqIntervalSeconds", high_freq_interval_secs_},
{"HostConfig", {"HostConfig",
{ {
{"AgentString", hc_.agent_string}, {"AgentString", hc_.agent_string},
{"ApiPassword", hc_.api_password}, {"ApiPassword", hc_.api_password},
{"ApiPort", hc_.api_port}, {"ApiPort", hc_.api_port},
{"HostNameOrIp", hc_.host_name_or_ip}, {"HostNameOrIp", hc_.host_name_or_ip},
{"TimeoutMs", hc_.timeout_ms}, {"TimeoutMs", hc_.timeout_ms},
}}, }},
{"LowFreqIntervalSeconds", low_freq_interval_secs_}, {"LowFreqIntervalSeconds", low_freq_interval_secs_},
{"MaxCacheSizeBytes", max_cache_size_bytes_}, {"MaxCacheSizeBytes", max_cache_size_bytes_},
{"MaxUploadCount", max_upload_count_}, {"MaxUploadCount", max_upload_count_},
{"OnlineCheckRetrySeconds", online_check_retry_secs_}, {"OnlineCheckRetrySeconds", online_check_retry_secs_},
{"OrphanedFileRetentionDays", orphaned_file_retention_days_}, {"OrphanedFileRetentionDays", orphaned_file_retention_days_},
{"PreferredDownloadType", preferred_download_type_}, {"PreferredDownloadType", preferred_download_type_},
{"ReadAheadCount", read_ahead_count_}, {"ReadAheadCount", read_ahead_count_},
{ {
"RemoteMount", "RemoteMount",
{ {
{"EnableRemoteMount", enable_remote_mount_}, {"EnableRemoteMount", enable_remote_mount_},
{"IsRemoteMount", is_remote_mount_}, {"IsRemoteMount", is_remote_mount_},
{"RemoteClientPoolSize", remote_client_pool_size_}, {"RemoteClientPoolSize", remote_client_pool_size_},
{"RemoteMaxConnections", remote_max_connections_}, {"RemoteMaxConnections", remote_max_connections_},
{"RemoteHostNameOrIp", remote_host_name_or_ip_}, {"RemoteHostNameOrIp", remote_host_name_or_ip_},
{"RemotePort", remote_port_}, {"RemotePort", remote_port_},
{"RemoteReceiveTimeoutSeconds", remote_receive_timeout_secs_}, {"RemoteReceiveTimeoutSeconds", remote_receive_timeout_secs_},
{"RemoteSendTimeoutSeconds", remote_send_timeout_secs_}, {"RemoteSendTimeoutSeconds", remote_send_timeout_secs_},
{"RemoteToken", remote_token_}, {"RemoteToken", remote_token_},
}, },
}, },
{"RetryReadCount", retry_read_count_}, {"RetryReadCount", retry_read_count_},
{"RingBufferFileSize", ring_buffer_file_size_}, {"RingBufferFileSize", ring_buffer_file_size_},
{"S3Config", {"S3Config",
{ {
{"AccessKey", s3_config_.access_key}, {"AccessKey", s3_config_.access_key},
{"Bucket", s3_config_.bucket}, {"Bucket", s3_config_.bucket},
{"CacheTimeoutSeconds", s3_config_.cache_timeout_secs}, {"CacheTimeoutSeconds", s3_config_.cache_timeout_secs},
{"EncryptionToken", s3_config_.encryption_token}, {"EncryptionToken", s3_config_.encryption_token},
{"Region", s3_config_.region}, {"Region", s3_config_.region},
{"SecretKey", s3_config_.secret_key}, {"SecretKey", s3_config_.secret_key},
{"TimeoutMs", s3_config_.timeout_ms}, {"TimeoutMs", s3_config_.timeout_ms},
{"URL", s3_config_.url}, {"URL", s3_config_.url},
{"UsePathStyle", s3_config_.use_path_style}, {"UsePathStyle", s3_config_.use_path_style},
{"UseRegionInURL", s3_config_.use_region_in_url}, {"UseRegionInURL", s3_config_.use_region_in_url},
}}, }},
{"Version", version_} {"Version", version_}};
};
if (prov_ == provider_type::encrypt) { if (prov_ == provider_type::encrypt) {
ret.erase("ChunkDownloaderTimeoutSeconds"); ret.erase("ChunkDownloaderTimeoutSeconds");

2
repertory/librepertory/src/drives/fuse/fuse_base.cpp
View File

@ -680,7 +680,7 @@ void fuse_base::raise_fuse_event(std::string function_name,
} }
if (((config_.get_event_level() >= fuse_event::level) && (ret != 0)) || if (((config_.get_event_level() >= fuse_event::level) && (ret != 0)) ||
(config_.get_event_level() >= event_level::verbose)) { (config_.get_event_level() >= event_level::trace)) {
event_system::instance().raise<fuse_event>( event_system::instance().raise<fuse_event>(
utils::string::right_trim(function_name, '_'), api_path, ret); utils::string::right_trim(function_name, '_'), api_path, ret);
} }

2
repertory/librepertory/src/drives/fuse/remotefuse/remote_server.cpp
View File

@ -45,7 +45,7 @@ namespace repertory::remote_fuse {
if (config_.get_enable_drive_events() && \ if (config_.get_enable_drive_events() && \
(((config_.get_event_level() >= remote_fuse_server_event::level) && \ (((config_.get_event_level() >= remote_fuse_server_event::level) && \
(ret < 0)) || \ (ret < 0)) || \
(config_.get_event_level() >= event_level::verbose))) \ (config_.get_event_level() >= event_level::trace))) \
event_system::instance().raise<remote_fuse_server_event>(func, file, ret) event_system::instance().raise<remote_fuse_server_event>(func, file, ret)
// clang-format off // clang-format off

2
repertory/librepertory/src/drives/winfsp/remotewinfsp/remote_client.cpp
View File

@ -34,7 +34,7 @@ namespace repertory::remote_winfsp {
if (config_.get_enable_drive_events() && \ if (config_.get_enable_drive_events() && \
(((config_.get_event_level() >= remote_winfsp_client_event::level) && \ (((config_.get_event_level() >= remote_winfsp_client_event::level) && \
(ret != STATUS_SUCCESS)) || \ (ret != STATUS_SUCCESS)) || \
(config_.get_event_level() >= event_level::verbose))) \ (config_.get_event_level() >= event_level::trace))) \
event_system::instance().raise<remote_winfsp_client_event>(func, file, ret) event_system::instance().raise<remote_winfsp_client_event>(func, file, ret)
// clang-format off // clang-format off

2
repertory/librepertory/src/drives/winfsp/remotewinfsp/remote_server.cpp
View File

@ -52,7 +52,7 @@ namespace repertory::remote_winfsp {
if (config_.get_enable_drive_events() && \ if (config_.get_enable_drive_events() && \
(((config_.get_event_level() >= remote_winfsp_server_event::level) && \ (((config_.get_event_level() >= remote_winfsp_server_event::level) && \
(ret != STATUS_SUCCESS)) || \ (ret != STATUS_SUCCESS)) || \
(config_.get_event_level() >= event_level::verbose))) \ (config_.get_event_level() >= event_level::trace))) \
event_system::instance().raise<remote_winfsp_server_event>(func, file, ret) event_system::instance().raise<remote_winfsp_server_event>(func, file, ret)
// clang-format off // clang-format off

2
repertory/librepertory/src/drives/winfsp/winfsp_drive.cpp
View File

@ -51,7 +51,7 @@ E_SIMPLE3(winfsp_event, debug, true,
if (config_.get_enable_drive_events() && \ if (config_.get_enable_drive_events() && \
(((config_.get_event_level() >= winfsp_event::level) && \ (((config_.get_event_level() >= winfsp_event::level) && \
(ret != STATUS_SUCCESS)) || \ (ret != STATUS_SUCCESS)) || \
(config_.get_event_level() >= event_level::verbose))) \ (config_.get_event_level() >= event_level::trace))) \
event_system::instance().raise<winfsp_event>(func, file, ret) event_system::instance().raise<winfsp_event>(func, file, ret)
winfsp_drive::winfsp_service::winfsp_service( winfsp_drive::winfsp_service::winfsp_service(

49
repertory/librepertory/src/events/consumers/logging_consumer.cpp
View File

@ -21,6 +21,11 @@
*/ */
#include "events/consumers/logging_consumer.hpp" #include "events/consumers/logging_consumer.hpp"
#include "events/events.hpp"
#include "spdlog/spdlog.h"
#include "spdlog/async.h"
#include "spdlog/sinks/rotating_file_sink.h"
#include "spdlog/sinks/stdout_color_sinks.h"
#include "utils/path_utils.hpp" #include "utils/path_utils.hpp"
namespace repertory { namespace repertory {
@ -32,21 +37,45 @@ logging_consumer::logging_consumer(event_level level, std::string log_directory)
[this](const event_level_changed &changed) { [this](const event_level_changed &changed) {
event_level_ = event_level_from_string( event_level_ = event_level_from_string(
changed.get_new_event_level().get<std::string>()); changed.get_new_event_level().get<std::string>());
switch (event_level_) {}
}); });
spdlog::init_thread_pool(8192, 1);
std::vector<spdlog::sink_ptr> sinks{
std::make_shared<spdlog::sinks::stdout_color_sink_mt>(),
std::make_shared<spdlog::sinks::rotating_file_sink_mt>(
utils::path::combine(log_directory_, {"repertory.log"}),
MAX_LOG_FILE_SIZE, MAX_LOG_FILES),
};
spdlog::register_logger(std::make_shared<spdlog::async_logger>(
"logger", sinks.begin(), sinks.end(), spdlog::thread_pool(),
spdlog::async_overflow_policy::block));
} }
logging_consumer::~logging_consumer() { E_CONSUMER_RELEASE(); } logging_consumer::~logging_consumer() { E_CONSUMER_RELEASE(); }
void logging_consumer::process_event(const event &event) { void logging_consumer::process_event(const event &event) {
/* auto msg = ([&]() -> std::string { switch (event.get_event_level()) {
struct tm local_time {}; case event_level::critical:
utils::get_local_time_now(local_time); spdlog::critical(event.get_single_line());
break;
std::stringstream ss; case event_level::error:
ss << std::put_time(&local_time, "%F %T") << "|" spdlog::error(event.get_single_line());
<< event_level_to_string(event->get_event_level()).c_str() << "|" break;
<< event->get_single_line().c_str() << std::endl; case event_level::warn:
return ss.str(); spdlog::warn(event.get_single_line());
})(); */ break;
case event_level::info:
spdlog::info(event.get_single_line());
break;
case event_level::debug:
spdlog::debug(event.get_single_line());
break;
case event_level::trace:
default:
spdlog::trace(event.get_single_line());
break;
}
} }
} // namespace repertory } // namespace repertory

45
repertory/librepertory/src/events/event.cpp
View File

@ -26,34 +26,49 @@
namespace repertory { namespace repertory {
auto event_level_from_string(std::string level) -> event_level { auto event_level_from_string(std::string level) -> event_level {
level = utils::string::to_lower(level); level = utils::string::to_lower(level);
if (level == "critical" || level == "event_level::critical") {
return event_level::critical;
}
if (level == "debug" || level == "event_level::debug") { if (level == "debug" || level == "event_level::debug") {
return event_level::debug; return event_level::debug;
} else if (level == "warn" || level == "event_level::warn") {
return event_level::warn;
} else if (level == "normal" || level == "event_level::normal") {
return event_level::normal;
} else if (level == "error" || level == "event_level::error") {
return event_level::error;
} else if (level == "verbose" || level == "event_level::verbose") {
return event_level::verbose;
} }
return event_level::normal;
if (level == "warn" || level == "event_level::warn") {
return event_level::warn;
}
if (level == "info" || level == "event_level::info") {
return event_level::info;
}
if (level == "error" || level == "event_level::error") {
return event_level::error;
}
if (level == "trace" || level == "event_level::trace") {
return event_level::trace;
}
return event_level::info;
} }
auto event_level_to_string(const event_level &level) -> std::string { auto event_level_to_string(event_level level) -> std::string {
switch (level) { switch (level) {
case event_level::critical:
return "critical";
case event_level::debug: case event_level::debug:
return "debug"; return "debug";
case event_level::error: case event_level::error:
return "error"; return "error";
case event_level::normal: case event_level::info:
return "normal"; return "info";
case event_level::warn: case event_level::warn:
return "warn"; return "warn";
case event_level::verbose: case event_level::trace:
return "verbose"; return "trace";
default: default:
return "normal"; return "info";
} }
} }
} // namespace repertory } // namespace repertory

4
repertory/librepertory/src/utils/polling.cpp
View File

@ -38,12 +38,12 @@ void polling::frequency_thread(
if (kv.second.freq == freq) { if (kv.second.freq == freq) {
futures.emplace_back( futures.emplace_back(
std::async(std::launch::async, [this, &freq, kv]() -> void { std::async(std::launch::async, [this, &freq, kv]() -> void {
if (config_->get_event_level() == event_level::verbose || if (config_->get_event_level() == event_level::trace ||
freq != frequency::second) { freq != frequency::second) {
event_system::instance().raise<polling_item_begin>(kv.first); event_system::instance().raise<polling_item_begin>(kv.first);
} }
kv.second.action(); kv.second.action();
if (config_->get_event_level() == event_level::verbose || if (config_->get_event_level() == event_level::trace ||
freq != frequency::second) { freq != frequency::second) {
event_system::instance().raise<polling_item_end>(kv.first); event_system::instance().raise<polling_item_end>(kv.first);
} }

2
repertory/repertory_test/include/fixtures/winfsp_fixture.hpp
View File

@ -78,7 +78,7 @@ protected:
.empty()); .empty());
EXPECT_FALSE( EXPECT_FALSE(
config->set_value_by_name("S3Config.Bucket", "repertory").empty()); config->set_value_by_name("S3Config.Bucket", "repertory").empty());
config->set_event_level(event_level::verbose); config->set_event_level(event_level::trace);
config->set_enable_drive_events(true); config->set_enable_drive_events(true);
event_system::instance().start(); event_system::instance().start();

2
repertory/repertory_test/src/fuse_drive_test.cpp
View File

@ -541,7 +541,7 @@ TEST(fuse_drive, all_tests) {
app_config src_cfg(provider_type::s3, app_config src_cfg(provider_type::s3,
utils::path::combine(get_test_dir(), {"storj"})); utils::path::combine(get_test_dir(), {"storj"}));
config_ptr->set_enable_drive_events(true); config_ptr->set_enable_drive_events(true);
config_ptr->set_event_level(event_level::verbose); config_ptr->set_event_level(event_level::trace);
config_ptr->set_s3_config(src_cfg.get_s3_config()); config_ptr->set_s3_config(src_cfg.get_s3_config());
} }

2
repertory/repertory_test/src/remote_fuse_test.cpp
View File

@ -926,7 +926,7 @@ TEST(remote_fuse, all_tests) {
config.set_remote_port(port); config.set_remote_port(port);
config.set_remote_token("testtoken"); config.set_remote_token("testtoken");
config.set_enable_drive_events(true); config.set_enable_drive_events(true);
config.set_event_level(event_level::verbose); config.set_event_level(event_level::trace);
event_system::instance().start(); event_system::instance().start();
#if defined(_WIN32) #if defined(_WIN32)

2
repertory/repertory_test/src/remote_winfsp_test.cpp
View File

@ -498,7 +498,7 @@ TEST(remote_winfsp, all_tests) {
config.set_remote_port(port); config.set_remote_port(port);
config.set_remote_token("testtoken"); config.set_remote_token("testtoken");
config.set_enable_drive_events(true); config.set_enable_drive_events(true);
config.set_event_level(event_level::verbose); config.set_event_level(event_level::trace);
event_system::instance().start(); event_system::instance().start();
#if defined(_WIN32) #if defined(_WIN32)