From 733a4a286990939b07d1bce26059f3d4244e1480 Mon Sep 17 00:00:00 2001 From: orignal Date: Sat, 27 Apr 2024 16:00:43 -0400 Subject: [PATCH] moved save/delete profile disk oprations to separate threads --- libi2pd/NetDb.cpp | 35 +++++++++++++++++++++++++---- libi2pd/NetDb.hpp | 2 ++ libi2pd/Profiling.cpp | 51 +++++++++++++++++++++++++++---------------- libi2pd/Profiling.h | 11 ++++++---- 4 files changed, 72 insertions(+), 27 deletions(-) diff --git a/libi2pd/NetDb.cpp b/libi2pd/NetDb.cpp index bf8f6f3d..1c874e3d 100644 --- a/libi2pd/NetDb.cpp +++ b/libi2pd/NetDb.cpp @@ -107,8 +107,8 @@ namespace data i2p::util::SetThreadName("NetDB"); uint64_t lastManage = 0, lastExploratory = 0, lastManageRequest = 0; - uint64_t lastProfilesCleanup = i2p::util::GetMonotonicMilliseconds (); - int16_t profilesCleanupVariance = 0; + uint64_t lastProfilesCleanup = i2p::util::GetMonotonicMilliseconds (), lastObsoleteProfilesCleanup = lastProfilesCleanup; + int16_t profilesCleanupVariance = 0, obsoleteProfilesCleanVariance = 0; while (m_IsRunning) { @@ -171,12 +171,39 @@ namespace data if (mts >= lastProfilesCleanup + (uint64_t)(i2p::data::PEER_PROFILE_AUTOCLEAN_TIMEOUT + profilesCleanupVariance)*1000) { m_RouterProfilesPool.CleanUpMt (); - if (m_PersistProfiles) PersistProfiles (); - DeleteObsoleteProfiles (); + if (m_PersistProfiles) + { + bool isSaving = m_SavingProfiles.valid (); + if (isSaving && m_SavingProfiles.wait_for(std::chrono::seconds(0)) == std::future_status::ready) // still active? + { + m_SavingProfiles.get (); + isSaving = false; + } + if (!isSaving) + m_SavingProfiles = PersistProfiles (); + else + LogPrint (eLogWarning, "NetDb: Can't persist profiles. Profiles are being saved to disk"); + } lastProfilesCleanup = mts; profilesCleanupVariance = (rand () % (2 * i2p::data::PEER_PROFILE_AUTOCLEAN_VARIANCE) - i2p::data::PEER_PROFILE_AUTOCLEAN_VARIANCE); } + if (mts >= lastObsoleteProfilesCleanup + (uint64_t)(i2p::data::PEER_PROFILE_OBSOLETE_PROFILES_CLEAN_TIMEOUT + obsoleteProfilesCleanVariance)*1000) + { + bool isDeleting = m_DeletingProfiles.valid (); + if (isDeleting && m_DeletingProfiles.wait_for(std::chrono::seconds(0)) == std::future_status::ready) // still active? + { + m_DeletingProfiles.get (); + isDeleting = false; + } + if (!isDeleting) + m_DeletingProfiles = DeleteObsoleteProfiles (); + else + LogPrint (eLogWarning, "NetDb: Can't delete profiles. Profiles are being deleted from disk"); + lastObsoleteProfilesCleanup = mts; + obsoleteProfilesCleanVariance = (rand () % (2 * i2p::data::PEER_PROFILE_OBSOLETE_PROFILES_CLEAN_VARIANCE) - i2p::data::PEER_PROFILE_OBSOLETE_PROFILES_CLEAN_TIMEOUT); + } + if (mts >= lastExploratory + 30000) // exploratory every 30 seconds { auto numRouters = m_RouterInfos.size (); diff --git a/libi2pd/NetDb.hpp b/libi2pd/NetDb.hpp index fb440d08..e0349ad0 100644 --- a/libi2pd/NetDb.hpp +++ b/libi2pd/NetDb.hpp @@ -15,6 +15,7 @@ #include #include #include +#include #include "Base.h" #include "Gzip.h" @@ -177,6 +178,7 @@ namespace data NetDbRequests m_Requests; bool m_PersistProfiles; + std::future m_SavingProfiles, m_DeletingProfiles; /** router info we are bootstrapping from or nullptr if we are not currently doing that*/ std::shared_ptr m_FloodfillBootstrap; diff --git a/libi2pd/Profiling.cpp b/libi2pd/Profiling.cpp index 3a2db9a5..264dd5d9 100644 --- a/libi2pd/Profiling.cpp +++ b/libi2pd/Profiling.cpp @@ -260,7 +260,13 @@ namespace data g_ProfilesStorage.Init(i2p::data::GetBase64SubstitutionTable(), 64); } - void PersistProfiles () + static void SaveProfilesToDisk (std::list > >&& profiles) + { + for (auto& it: profiles) + if (it.second) it.second->Save (it.first); + } + + std::future PersistProfiles () { auto ts = GetTime (); std::list > > tmp; @@ -278,8 +284,9 @@ namespace data it++; } } - for (auto& it: tmp) - if (it.second) it.second->Save (it.first); + if (!tmp.empty ()) + return std::async (std::launch::async, SaveProfilesToDisk, std::move (tmp)); + return std::future(); } void SaveProfiles () @@ -287,8 +294,7 @@ namespace data std::unordered_map > tmp; { std::unique_lock l(g_ProfilesMutex); - tmp = g_Profiles; - g_Profiles.clear (); + std::swap (tmp, g_Profiles); } auto ts = GetTime (); for (auto& it: tmp) @@ -296,7 +302,26 @@ namespace data it.second->Save (it.first); } - void DeleteObsoleteProfiles () + static void DeleteFilesFromDisk (std::vector&& files) + { + struct stat st; + std::time_t now = std::time(nullptr); + for (const auto& path: files) + { + if (stat(path.c_str(), &st) != 0) + { + LogPrint(eLogWarning, "Profiling: Can't stat(): ", path); + continue; + } + if (now - st.st_mtime >= PEER_PROFILE_EXPIRATION_TIMEOUT*3600) + { + LogPrint(eLogDebug, "Profiling: Removing expired peer profile: ", path); + i2p::fs::Remove(path); + } + } + } + + std::future DeleteObsoleteProfiles () { { auto ts = GetTime (); @@ -310,21 +335,9 @@ namespace data } } - struct stat st; - std::time_t now = std::time(nullptr); - std::vector files; g_ProfilesStorage.Traverse(files); - for (const auto& path: files) { - if (stat(path.c_str(), &st) != 0) { - LogPrint(eLogWarning, "Profiling: Can't stat(): ", path); - continue; - } - if (now - st.st_mtime >= PEER_PROFILE_EXPIRATION_TIMEOUT*3600) { - LogPrint(eLogDebug, "Profiling: Removing expired peer profile: ", path); - i2p::fs::Remove(path); - } - } + return std::async (std::launch::async, DeleteFilesFromDisk, std::move (files)); } } } diff --git a/libi2pd/Profiling.h b/libi2pd/Profiling.h index c4aa6a9a..f4c4a4da 100644 --- a/libi2pd/Profiling.h +++ b/libi2pd/Profiling.h @@ -10,6 +10,7 @@ #define PROFILING_H__ #include +#include #include #include "Identity.h" @@ -31,8 +32,10 @@ namespace data const char PEER_PROFILE_USAGE_CONNECTED[] = "connected"; const int PEER_PROFILE_EXPIRATION_TIMEOUT = 36; // in hours (1.5 days) - const int PEER_PROFILE_AUTOCLEAN_TIMEOUT = 3 * 3600; // in seconds (3 hours) - const int PEER_PROFILE_AUTOCLEAN_VARIANCE = 3600; // in seconds (1 hour) + const int PEER_PROFILE_AUTOCLEAN_TIMEOUT = 1500; // in seconds (25 minutes) + const int PEER_PROFILE_AUTOCLEAN_VARIANCE = 900; // in seconds (15 minutes) + const int PEER_PROFILE_OBSOLETE_PROFILES_CLEAN_TIMEOUT = 5400; // in seconds (1.5 hours) + const int PEER_PROFILE_OBSOLETE_PROFILES_CLEAN_VARIANCE = 2400; // in seconds (40 minutes) const int PEER_PROFILE_DECLINED_RECENTLY_INTERVAL = 150; // in seconds (2.5 minutes) const int PEER_PROFILE_PERSIST_INTERVAL = 3300; // in seconds (55 minutes) const int PEER_PROFILE_UNREACHABLE_INTERVAL = 480; // in seconds (8 minutes) @@ -89,9 +92,9 @@ namespace data std::shared_ptr GetRouterProfile (const IdentHash& identHash); bool IsRouterBanned (const IdentHash& identHash); // check only existing profiles void InitProfilesStorage (); - void DeleteObsoleteProfiles (); + std::future DeleteObsoleteProfiles (); void SaveProfiles (); - void PersistProfiles (); + std::future PersistProfiles (); } }