From 5b95a190e8d7059039ce61e808d494dcf89ebb3b Mon Sep 17 00:00:00 2001 From: Alex Morcos Date: Fri, 31 Mar 2017 10:17:13 -0400 Subject: [PATCH 1/3] Make pcoinsTip memory calculations consistent Since we are more accurately measuring pcoinsTip peak usage at twice the current in dynamic usage, it makes sense to double the default (this will lead to the same effective usage and peak usage as previously). We should also double the buffer used to avoid flushing if above 90% but still sufficient space remaining. --- src/txdb.h | 6 +++++- src/validation.cpp | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/txdb.h b/src/txdb.h index 7f5cf2b583f..963f1006eaa 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -21,8 +21,12 @@ class CBlockIndex; class CCoinsViewDBCursor; class uint256; +//! Compensate for extra memory peak (x1.5-x1.9) at flush time. +static constexpr int DB_PEAK_USAGE_FACTOR = 2; +//! No need to flush if at least this much space still available. +static constexpr int MAX_BLOCK_COINSDB_USAGE = 100 * DB_PEAK_USAGE_FACTOR; //! -dbcache default (MiB) -static const int64_t nDefaultDbCache = 300; +static const int64_t nDefaultDbCache = 600; //! max. -dbcache (MiB) static const int64_t nMaxDbCache = sizeof(void*) > 4 ? 16384 : 1024; //! min. -dbcache (MiB) diff --git a/src/validation.cpp b/src/validation.cpp index 7737c13e6ed..390801d34f4 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -2005,10 +2005,10 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode, int n nLastSetChain = nNow; } int64_t nMempoolSizeMax = GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; - int64_t cacheSize = pcoinsTip->DynamicMemoryUsage() * 2; // Compensate for extra memory peak (x1.5-x1.9) at flush time. + int64_t cacheSize = pcoinsTip->DynamicMemoryUsage() * DB_PEAK_USAGE_FACTOR; int64_t nTotalSpace = nCoinCacheUsage + std::max(nMempoolSizeMax - nMempoolUsage, 0); // The cache is large and we're within 10% and 100 MiB of the limit, but we have time now (not in the middle of a block processing). - bool fCacheLarge = mode == FLUSH_STATE_PERIODIC && cacheSize > std::max((9 * nTotalSpace) / 10, nTotalSpace - 100 * 1024 * 1024); + bool fCacheLarge = mode == FLUSH_STATE_PERIODIC && cacheSize > std::max((9 * nTotalSpace) / 10, nTotalSpace - MAX_BLOCK_COINSDB_USAGE * 1024 * 1024); // The cache is over the limit, we have to write now. bool fCacheCritical = mode == FLUSH_STATE_IF_NEEDED && cacheSize > nTotalSpace; // It's been a while since we wrote the block index to disk. Do this frequently, so we don't need to redownload after a crash. From f33afd3b2be1bcabeb10168a53835359c9ff4a3e Mon Sep 17 00:00:00 2001 From: Alex Morcos Date: Fri, 31 Mar 2017 10:25:39 -0400 Subject: [PATCH 2/3] Lower default memory footprint slightly --- src/txdb.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/txdb.h b/src/txdb.h index 963f1006eaa..337942ffaf3 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -26,7 +26,7 @@ static constexpr int DB_PEAK_USAGE_FACTOR = 2; //! No need to flush if at least this much space still available. static constexpr int MAX_BLOCK_COINSDB_USAGE = 100 * DB_PEAK_USAGE_FACTOR; //! -dbcache default (MiB) -static const int64_t nDefaultDbCache = 600; +static const int64_t nDefaultDbCache = 450; //! max. -dbcache (MiB) static const int64_t nMaxDbCache = sizeof(void*) > 4 ? 16384 : 1024; //! min. -dbcache (MiB) From 1b55e07b7a61a9e6c299cf4c40fde80fa715d440 Mon Sep 17 00:00:00 2001 From: Alex Morcos Date: Fri, 31 Mar 2017 10:26:25 -0400 Subject: [PATCH 3/3] Make threshold for flushing more conservative. Always leave a reasonable buffer of 50MB for usage from newly connected block (once over 50%) and increase the high water mark buffer to 200MB. --- src/txdb.h | 6 ++++-- src/validation.cpp | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/txdb.h b/src/txdb.h index 337942ffaf3..d9214ba6185 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -23,8 +23,10 @@ class uint256; //! Compensate for extra memory peak (x1.5-x1.9) at flush time. static constexpr int DB_PEAK_USAGE_FACTOR = 2; -//! No need to flush if at least this much space still available. -static constexpr int MAX_BLOCK_COINSDB_USAGE = 100 * DB_PEAK_USAGE_FACTOR; +//! No need to periodic flush if at least this much space still available. +static constexpr int MAX_BLOCK_COINSDB_USAGE = 200 * DB_PEAK_USAGE_FACTOR; +//! Always periodic flush if less than this much space still available. +static constexpr int MIN_BLOCK_COINSDB_USAGE = 50 * DB_PEAK_USAGE_FACTOR; //! -dbcache default (MiB) static const int64_t nDefaultDbCache = 450; //! max. -dbcache (MiB) diff --git a/src/validation.cpp b/src/validation.cpp index 390801d34f4..fbfeffbaab8 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -2007,8 +2007,9 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode, int n int64_t nMempoolSizeMax = GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000; int64_t cacheSize = pcoinsTip->DynamicMemoryUsage() * DB_PEAK_USAGE_FACTOR; int64_t nTotalSpace = nCoinCacheUsage + std::max(nMempoolSizeMax - nMempoolUsage, 0); - // The cache is large and we're within 10% and 100 MiB of the limit, but we have time now (not in the middle of a block processing). - bool fCacheLarge = mode == FLUSH_STATE_PERIODIC && cacheSize > std::max((9 * nTotalSpace) / 10, nTotalSpace - MAX_BLOCK_COINSDB_USAGE * 1024 * 1024); + // The cache is large and we're within 10% and 200 MiB or 50% and 50MiB of the limit, but we have time now (not in the middle of a block processing). + bool fCacheLarge = mode == FLUSH_STATE_PERIODIC && cacheSize > std::min(std::max(nTotalSpace / 2, nTotalSpace - MIN_BLOCK_COINSDB_USAGE * 1024 * 1024), + std::max((9 * nTotalSpace) / 10, nTotalSpace - MAX_BLOCK_COINSDB_USAGE * 1024 * 1024)); // The cache is over the limit, we have to write now. bool fCacheCritical = mode == FLUSH_STATE_IF_NEEDED && cacheSize > nTotalSpace; // It's been a while since we wrote the block index to disk. Do this frequently, so we don't need to redownload after a crash.