Add a pool for locked memory chunks, replacing LockedPageManager. This is something I've been wanting to do for a long time. The current approach of locking objects where they happen to be on the stack or heap in-place causes a lot of mlock/munlock system call overhead, slowing down any handling of keys. Also locked memory is a limited resource on many operating systems (and using a lot of it bogs down the system), so the previous approach of locking every page that may contain any key information (but also other information) is wasteful.pull/308/head
Wladimir J. van der Laan
8 years ago
parent
f4d1fc259b
commit
4536148b15
@ -0,0 +1,383 @@
|
||||
// Copyright (c) 2016 The Bitcoin Core developers
|
||||
// Distributed under the MIT software license, see the accompanying
|
||||
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
|
||||
|
||||
#include "support/lockedpool.h"
|
||||
#include "support/cleanse.h"
|
||||
|
||||
#if defined(HAVE_CONFIG_H)
|
||||
#include "config/bitcoin-config.h"
|
||||
#endif
|
||||
|
||||
#ifdef WIN32
|
||||
#ifdef _WIN32_WINNT
|
||||
#undef _WIN32_WINNT
|
||||
#endif
|
||||
#define _WIN32_WINNT 0x0501
|
||||
#define WIN32_LEAN_AND_MEAN 1
|
||||
#ifndef NOMINMAX
|
||||
#define NOMINMAX
|
||||
#endif
|
||||
#include <windows.h>
|
||||
#else
|
||||
#include <sys/mman.h> // for mmap
|
||||
#include <sys/resource.h> // for getrlimit
|
||||
#include <limits.h> // for PAGESIZE
|
||||
#include <unistd.h> // for sysconf
|
||||
#endif
|
||||
|
||||
LockedPoolManager* LockedPoolManager::_instance = NULL;
|
||||
std::once_flag LockedPoolManager::init_flag;
|
||||
|
||||
/*******************************************************************************/
|
||||
// Utilities
|
||||
//
|
||||
/** Align up to power of 2 */
|
||||
static inline size_t align_up(size_t x, size_t align)
|
||||
{
|
||||
return (x + align - 1) & ~(align - 1);
|
||||
}
|
||||
|
||||
/*******************************************************************************/
|
||||
// Implementation: Arena
|
||||
|
||||
Arena::Arena(void *base_in, size_t size_in, size_t alignment_in):
|
||||
base(static_cast<char*>(base_in)), end(static_cast<char*>(base_in) + size_in), alignment(alignment_in)
|
||||
{
|
||||
// Start with one free chunk that covers the entire arena
|
||||
chunks.emplace(base, Chunk(size_in, false));
|
||||
}
|
||||
|
||||
Arena::~Arena()
|
||||
{
|
||||
}
|
||||
|
||||
void* Arena::alloc(size_t size)
|
||||
{
|
||||
// Round to next multiple of alignment
|
||||
size = align_up(size, alignment);
|
||||
|
||||
// Don't handle zero-sized chunks, or those bigger than MAX_SIZE
|
||||
if (size == 0 || size >= Chunk::MAX_SIZE) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
for (auto& chunk: chunks) {
|
||||
if (!chunk.second.isInUse() && size <= chunk.second.getSize()) {
|
||||
char* base = chunk.first;
|
||||
size_t leftover = chunk.second.getSize() - size;
|
||||
if (leftover > 0) { // Split chunk
|
||||
chunks.emplace(base + size, Chunk(leftover, false));
|
||||
chunk.second.setSize(size);
|
||||
}
|
||||
chunk.second.setInUse(true);
|
||||
return reinterpret_cast<void*>(base);
|
||||
}
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
void Arena::free(void *ptr)
|
||||
{
|
||||
// Freeing the NULL pointer is OK.
|
||||
if (ptr == nullptr) {
|
||||
return;
|
||||
}
|
||||
auto i = chunks.find(static_cast<char*>(ptr));
|
||||
if (i == chunks.end() || !i->second.isInUse()) {
|
||||
throw std::runtime_error("Arena: invalid or double free");
|
||||
}
|
||||
|
||||
i->second.setInUse(false);
|
||||
|
||||
if (i != chunks.begin()) { // Absorb into previous chunk if exists and free
|
||||
auto prev = i;
|
||||
--prev;
|
||||
if (!prev->second.isInUse()) {
|
||||
// Absorb current chunk size into previous chunk.
|
||||
prev->second.setSize(prev->second.getSize() + i->second.getSize());
|
||||
// Erase current chunk. Erasing does not invalidate current
|
||||
// iterators for a map, except for that pointing to the object
|
||||
// itself, which will be overwritten in the next statement.
|
||||
chunks.erase(i);
|
||||
// From here on, the previous chunk is our current chunk.
|
||||
i = prev;
|
||||
}
|
||||
}
|
||||
auto next = i;
|
||||
++next;
|
||||
if (next != chunks.end()) { // Absorb next chunk if exists and free
|
||||
if (!next->second.isInUse()) {
|
||||
// Absurb next chunk size into current chunk
|
||||
i->second.setSize(i->second.getSize() + next->second.getSize());
|
||||
// Erase next chunk.
|
||||
chunks.erase(next);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Arena::Stats Arena::stats() const
|
||||
{
|
||||
Arena::Stats r;
|
||||
r.used = r.free = r.total = r.chunks_used = r.chunks_free = 0;
|
||||
for (const auto& chunk: chunks) {
|
||||
if (chunk.second.isInUse()) {
|
||||
r.used += chunk.second.getSize();
|
||||
r.chunks_used += 1;
|
||||
} else {
|
||||
r.free += chunk.second.getSize();
|
||||
r.chunks_free += 1;
|
||||
}
|
||||
r.total += chunk.second.getSize();
|
||||
}
|
||||
return r;
|
||||
}
|
||||
|
||||
#ifdef ARENA_DEBUG
|
||||
void Arena::walk() const
|
||||
{
|
||||
for (const auto& chunk: chunks) {
|
||||
std::cout <<
|
||||
"0x" << std::hex << std::setw(16) << std::setfill('0') << chunk.first <<
|
||||
" 0x" << std::hex << std::setw(16) << std::setfill('0') << chunk.second.getSize() <<
|
||||
" 0x" << chunk.second.isInUse() << std::endl;
|
||||
}
|
||||
std::cout << std::endl;
|
||||
}
|
||||
#endif
|
||||
|
||||
/*******************************************************************************/
|
||||
// Implementation: Win32LockedPageAllocator
|
||||
|
||||
#ifdef WIN32
|
||||
/** LockedPageAllocator specialized for Windows.
|
||||
*/
|
||||
class Win32LockedPageAllocator: public LockedPageAllocator
|
||||
{
|
||||
public:
|
||||
Win32LockedPageAllocator();
|
||||
void* AllocateLocked(size_t len, bool *lockingSuccess);
|
||||
void FreeLocked(void* addr, size_t len);
|
||||
size_t GetLimit();
|
||||
private:
|
||||
size_t page_size;
|
||||
};
|
||||
|
||||
Win32LockedPageAllocator::Win32LockedPageAllocator()
|
||||
{
|
||||
// Determine system page size in bytes
|
||||
SYSTEM_INFO sSysInfo;
|
||||
GetSystemInfo(&sSysInfo);
|
||||
page_size = sSysInfo.dwPageSize;
|
||||
}
|
||||
void *Win32LockedPageAllocator::AllocateLocked(size_t len, bool *lockingSuccess)
|
||||
{
|
||||
len = align_up(len, page_size);
|
||||
void *addr = VirtualAlloc(nullptr, len, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
|
||||
if (addr) {
|
||||
// VirtualLock is used to attempt to keep keying material out of swap. Note
|
||||
// that it does not provide this as a guarantee, but, in practice, memory
|
||||
// that has been VirtualLock'd almost never gets written to the pagefile
|
||||
// except in rare circumstances where memory is extremely low.
|
||||
*lockingSuccess = VirtualLock(const_cast<void*>(addr), len) != 0;
|
||||
}
|
||||
return addr;
|
||||
}
|
||||
void Win32LockedPageAllocator::FreeLocked(void* addr, size_t len)
|
||||
{
|
||||
len = align_up(len, page_size);
|
||||
memory_cleanse(addr, len);
|
||||
VirtualUnlock(const_cast<void*>(addr), len);
|
||||
}
|
||||
|
||||
size_t Win32LockedPageAllocator::GetLimit()
|
||||
{
|
||||
// TODO is there a limit on windows, how to get it?
|
||||
return std::numeric_limits<size_t>::max();
|
||||
}
|
||||
#endif
|
||||
|
||||
/*******************************************************************************/
|
||||
// Implementation: PosixLockedPageAllocator
|
||||
|
||||
#ifndef WIN32
|
||||
/** LockedPageAllocator specialized for OSes that don't try to be
|
||||
* special snowflakes.
|
||||
*/
|
||||
class PosixLockedPageAllocator: public LockedPageAllocator
|
||||
{
|
||||
public:
|
||||
PosixLockedPageAllocator();
|
||||
void* AllocateLocked(size_t len, bool *lockingSuccess);
|
||||
void FreeLocked(void* addr, size_t len);
|
||||
size_t GetLimit();
|
||||
private:
|
||||
size_t page_size;
|
||||
};
|
||||
|
||||
PosixLockedPageAllocator::PosixLockedPageAllocator()
|
||||
{
|
||||
// Determine system page size in bytes
|
||||
#if defined(PAGESIZE) // defined in limits.h
|
||||
page_size = PAGESIZE;
|
||||
#else // assume some POSIX OS
|
||||
page_size = sysconf(_SC_PAGESIZE);
|
||||
#endif
|
||||
}
|
||||
void *PosixLockedPageAllocator::AllocateLocked(size_t len, bool *lockingSuccess)
|
||||
{
|
||||
void *addr;
|
||||
len = align_up(len, page_size);
|
||||
addr = mmap(nullptr, len, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
|
||||
if (addr) {
|
||||
*lockingSuccess = mlock(addr, len) == 0;
|
||||
}
|
||||
return addr;
|
||||
}
|
||||
void PosixLockedPageAllocator::FreeLocked(void* addr, size_t len)
|
||||
{
|
||||
len = align_up(len, page_size);
|
||||
memory_cleanse(addr, len);
|
||||
munlock(addr, len);
|
||||
munmap(addr, len);
|
||||
}
|
||||
size_t PosixLockedPageAllocator::GetLimit()
|
||||
{
|
||||
#ifdef RLIMIT_MEMLOCK
|
||||
struct rlimit rlim;
|
||||
if (getrlimit(RLIMIT_MEMLOCK, &rlim) == 0) {
|
||||
if (rlim.rlim_cur != RLIM_INFINITY) {
|
||||
return rlim.rlim_cur;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
return std::numeric_limits<size_t>::max();
|
||||
}
|
||||
#endif
|
||||
|
||||
/*******************************************************************************/
|
||||
// Implementation: LockedPool
|
||||
|
||||
LockedPool::LockedPool(std::unique_ptr<LockedPageAllocator> allocator_in, LockingFailed_Callback lf_cb_in):
|
||||
allocator(std::move(allocator_in)), lf_cb(lf_cb_in), cumulative_bytes_locked(0)
|
||||
{
|
||||
}
|
||||
|
||||
LockedPool::~LockedPool()
|
||||
{
|
||||
}
|
||||
void* LockedPool::alloc(size_t size)
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(mutex);
|
||||
// Try allocating from each current arena
|
||||
for (auto &arena: arenas) {
|
||||
void *addr = arena.alloc(size);
|
||||
if (addr) {
|
||||
return addr;
|
||||
}
|
||||
}
|
||||
// If that fails, create a new one
|
||||
if (new_arena(ARENA_SIZE, ARENA_ALIGN)) {
|
||||
return arenas.back().alloc(size);
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
void LockedPool::free(void *ptr)
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(mutex);
|
||||
// TODO we can do better than this linear search by keeping a map of arena
|
||||
// extents to arena, and looking up the address.
|
||||
for (auto &arena: arenas) {
|
||||
if (arena.addressInArena(ptr)) {
|
||||
arena.free(ptr);
|
||||
return;
|
||||
}
|
||||
}
|
||||
throw std::runtime_error("LockedPool: invalid address not pointing to any arena");
|
||||
}
|
||||
|
||||
LockedPool::Stats LockedPool::stats() const
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(mutex);
|
||||
LockedPool::Stats r;
|
||||
r.used = r.free = r.total = r.chunks_used = r.chunks_free = 0;
|
||||
r.locked = cumulative_bytes_locked;
|
||||
for (const auto &arena: arenas) {
|
||||
Arena::Stats i = arena.stats();
|
||||
r.used += i.used;
|
||||
r.free += i.free;
|
||||
r.total += i.total;
|
||||
r.chunks_used += i.chunks_used;
|
||||
r.chunks_free += i.chunks_free;
|
||||
}
|
||||
return r;
|
||||
}
|
||||
|
||||
bool LockedPool::new_arena(size_t size, size_t align)
|
||||
{
|
||||
bool locked;
|
||||
// If this is the first arena, handle this specially: Cap the upper size
|
||||
// by the process limit. This makes sure that the first arena will at least
|
||||
// be locked. An exception to this is if the process limit is 0:
|
||||
// in this case no memory can be locked at all so we'll skip past this logic.
|
||||
if (arenas.empty()) {
|
||||
size_t limit = allocator->GetLimit();
|
||||
if (limit > 0) {
|
||||
size = std::min(size, limit);
|
||||
}
|
||||
}
|
||||
void *addr = allocator->AllocateLocked(size, &locked);
|
||||
if (!addr) {
|
||||
return false;
|
||||
}
|
||||
if (locked) {
|
||||
cumulative_bytes_locked += size;
|
||||
} else if (lf_cb) { // Call the locking-failed callback if locking failed
|
||||
if (!lf_cb()) { // If the callback returns false, free the memory and fail, otherwise consider the user warned and proceed.
|
||||
allocator->FreeLocked(addr, size);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
arenas.emplace_back(allocator.get(), addr, size, align);
|
||||
return true;
|
||||
}
|
||||
|
||||
LockedPool::LockedPageArena::LockedPageArena(LockedPageAllocator *allocator_in, void *base_in, size_t size_in, size_t align_in):
|
||||
Arena(base_in, size_in, align_in), base(base_in), size(size_in), allocator(allocator_in)
|
||||
{
|
||||
}
|
||||
LockedPool::LockedPageArena::~LockedPageArena()
|
||||
{
|
||||
allocator->FreeLocked(base, size);
|
||||
}
|
||||
|
||||
/*******************************************************************************/
|
||||
// Implementation: LockedPoolManager
|
||||
//
|
||||
LockedPoolManager::LockedPoolManager(std::unique_ptr<LockedPageAllocator> allocator):
|
||||
LockedPool(std::move(allocator), &LockedPoolManager::LockingFailed)
|
||||
{
|
||||
}
|
||||
|
||||
bool LockedPoolManager::LockingFailed()
|
||||
{
|
||||
// TODO: log something but how? without including util.h
|
||||
return true;
|
||||
}
|
||||
|
||||
void LockedPoolManager::CreateInstance()
|
||||
{
|
||||
// Using a local static instance guarantees that the object is initialized
|
||||
// when it's first needed and also deinitialized after all objects that use
|
||||
// it are done with it. I can think of one unlikely scenario where we may
|
||||
// have a static deinitialization order/problem, but the check in
|
||||
// LockedPoolManagerBase's destructor helps us detect if that ever happens.
|
||||
#ifdef WIN32
|
||||
std::unique_ptr<LockedPageAllocator> allocator(new Win32LockedPageAllocator());
|
||||
#else
|
||||
std::unique_ptr<LockedPageAllocator> allocator(new PosixLockedPageAllocator());
|
||||
#endif
|
||||
static LockedPoolManager instance(std::move(allocator));
|
||||
LockedPoolManager::_instance = &instance;
|
||||
}
|
||||
@ -0,0 +1,251 @@
|
||||
// Copyright (c) 2016 The Bitcoin Core developers
|
||||
// Distributed under the MIT software license, see the accompanying
|
||||
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
|
||||
|
||||
#ifndef BITCOIN_SUPPORT_LOCKEDPOOL_H
|
||||
#define BITCOIN_SUPPORT_LOCKEDPOOL_H
|
||||
|
||||
#include <stdint.h>
|
||||
#include <list>
|
||||
#include <map>
|
||||
#include <mutex>
|
||||
#include <memory>
|
||||
|
||||
/**
|
||||
* OS-dependent allocation and deallocation of locked/pinned memory pages.
|
||||
* Abstract base class.
|
||||
*/
|
||||
class LockedPageAllocator
|
||||
{
|
||||
public:
|
||||
virtual ~LockedPageAllocator() {}
|
||||
/** Allocate and lock memory pages.
|
||||
* If len is not a multiple of the system page size, it is rounded up.
|
||||
* Returns 0 in case of allocation failure.
|
||||
*
|
||||
* If locking the memory pages could not be accomplished it will still
|
||||
* return the memory, however the lockingSuccess flag will be false.
|
||||
* lockingSuccess is undefined if the allocation fails.
|
||||
*/
|
||||
virtual void* AllocateLocked(size_t len, bool *lockingSuccess) = 0;
|
||||
|
||||
/** Unlock and free memory pages.
|
||||
* Clear the memory before unlocking.
|
||||
*/
|
||||
virtual void FreeLocked(void* addr, size_t len) = 0;
|
||||
|
||||
/** Get the total limit on the amount of memory that may be locked by this
|
||||
* process, in bytes. Return size_t max if there is no limit or the limit
|
||||
* is unknown. Return 0 if no memory can be locked at all.
|
||||
*/
|
||||
virtual size_t GetLimit() = 0;
|
||||
};
|
||||
|
||||
/* An arena manages a contiguous region of memory by dividing it into
|
||||
* chunks.
|
||||
*/
|
||||
class Arena
|
||||
{
|
||||
public:
|
||||
Arena(void *base, size_t size, size_t alignment);
|
||||
virtual ~Arena();
|
||||
|
||||
/** A chunk of memory.
|
||||
*/
|
||||
struct Chunk
|
||||
{
|
||||
/** Most significant bit of size_t. This is used to mark
|
||||
* in-usedness of chunk.
|
||||
*/
|
||||
const static size_t SIZE_MSB = 1LLU << ((sizeof(size_t)*8)-1);
|
||||
/** Maximum size of a chunk */
|
||||
const static size_t MAX_SIZE = SIZE_MSB - 1;
|
||||
|
||||
Chunk(size_t size_in, bool used_in):
|
||||
size(size_in | (used_in ? SIZE_MSB : 0)) {}
|
||||
|
||||
bool isInUse() const { return size & SIZE_MSB; }
|
||||
void setInUse(bool used_in) { size = (size & ~SIZE_MSB) | (used_in ? SIZE_MSB : 0); }
|
||||
size_t getSize() const { return size & ~SIZE_MSB; }
|
||||
void setSize(size_t size_in) { size = (size & SIZE_MSB) | size_in; }
|
||||
private:
|
||||
size_t size;
|
||||
};
|
||||
/** Memory statistics. */
|
||||
struct Stats
|
||||
{
|
||||
size_t used;
|
||||
size_t free;
|
||||
size_t total;
|
||||
size_t chunks_used;
|
||||
size_t chunks_free;
|
||||
};
|
||||
|
||||
/** Allocate size bytes from this arena.
|
||||
* Returns pointer on success, or 0 if memory is full or
|
||||
* the application tried to allocate 0 bytes.
|
||||
*/
|
||||
void* alloc(size_t size);
|
||||
|
||||
/** Free a previously allocated chunk of memory.
|
||||
* Freeing the zero pointer has no effect.
|
||||
* Raises std::runtime_error in case of error.
|
||||
*/
|
||||
void free(void *ptr);
|
||||
|
||||
/** Get arena usage statistics */
|
||||
Stats stats() const;
|
||||
|
||||
#ifdef ARENA_DEBUG
|
||||
void walk() const;
|
||||
#endif
|
||||
|
||||
/** Return whether a pointer points inside this arena.
|
||||
* This returns base <= ptr < (base+size) so only use it for (inclusive)
|
||||
* chunk starting addresses.
|
||||
*/
|
||||
bool addressInArena(void *ptr) const { return ptr >= base && ptr < end; }
|
||||
private:
|
||||
Arena(const Arena& other) = delete; // non construction-copyable
|
||||
Arena& operator=(const Arena&) = delete; // non copyable
|
||||
|
||||
/** Map of chunk address to chunk information. This class makes use of the
|
||||
* sorted order to merge previous and next chunks during deallocation.
|
||||
*/
|
||||
std::map<char*, Chunk> chunks;
|
||||
/** Base address of arena */
|
||||
char* base;
|
||||
/** End address of arena */
|
||||
char* end;
|
||||
/** Minimum chunk alignment */
|
||||
size_t alignment;
|
||||
};
|
||||
|
||||
/** Pool for locked memory chunks.
|
||||
*
|
||||
* To avoid sensitive key data from being swapped to disk, the memory in this pool
|
||||
* is locked/pinned.
|
||||
*
|
||||
* An arena manages a contiguous region of memory. The pool starts out with one arena
|
||||
* but can grow to multiple arenas if the need arises.
|
||||
*
|
||||
* Unlike a normal C heap, the administrative structures are seperate from the managed
|
||||
* memory. This has been done as the sizes and bases of objects are not in themselves sensitive
|
||||
* information, as to conserve precious locked memory. In some operating systems
|
||||
* the amount of memory that can be locked is small.
|
||||
*/
|
||||
class LockedPool
|
||||
{
|
||||
public:
|
||||
/** Size of one arena of locked memory. This is a compromise.
|
||||
* Do not set this too low, as managing many arenas will increase
|
||||
* allocation and deallocation overhead. Setting it too high allocates
|
||||
* more locked memory from the OS than strictly necessary.
|
||||
*/
|
||||
static const size_t ARENA_SIZE = 256*1024;
|
||||
/** Chunk alignment. Another compromise. Setting this too high will waste
|
||||
* memory, setting it too low will facilitate fragmentation.
|
||||
*/
|
||||
static const size_t ARENA_ALIGN = 16;
|
||||
|
||||
/** Callback when allocation succeeds but locking fails.
|
||||
*/
|
||||
typedef bool (*LockingFailed_Callback)();
|
||||
|
||||
/** Memory statistics. */
|
||||
struct Stats
|
||||
{
|
||||
size_t used;
|
||||
size_t free;
|
||||
size_t total;
|
||||
size_t locked;
|
||||
size_t chunks_used;
|
||||
size_t chunks_free;
|
||||
};
|
||||
|
||||
/** Create a new LockedPool. This takes ownership of the MemoryPageLocker,
|
||||
* you can only instantiate this with LockedPool(std::move(...)).
|
||||
*
|
||||
* The second argument is an optional callback when locking a newly allocated arena failed.
|
||||
* If this callback is provided and returns false, the allocation fails (hard fail), if
|
||||
* it returns true the allocation proceeds, but it could warn.
|
||||
*/
|
||||
LockedPool(std::unique_ptr<LockedPageAllocator> allocator, LockingFailed_Callback lf_cb_in = 0);
|
||||
~LockedPool();
|
||||
|
||||
/** Allocate size bytes from this arena.
|
||||
* Returns pointer on success, or 0 if memory is full or
|
||||
* the application tried to allocate 0 bytes.
|
||||
*/
|
||||
void* alloc(size_t size);
|
||||
|
||||
/** Free a previously allocated chunk of memory.
|
||||
* Freeing the zero pointer has no effect.
|
||||
* Raises std::runtime_error in case of error.
|
||||
*/
|
||||
void free(void *ptr);
|
||||
|
||||
/** Get pool usage statistics */
|
||||
Stats stats() const;
|
||||
private:
|
||||
LockedPool(const LockedPool& other) = delete; // non construction-copyable
|
||||
LockedPool& operator=(const LockedPool&) = delete; // non copyable
|
||||
|
||||
std::unique_ptr<LockedPageAllocator> allocator;
|
||||
|
||||
/** Create an arena from locked pages */
|
||||
class LockedPageArena: public Arena
|
||||
{
|
||||
public:
|
||||
LockedPageArena(LockedPageAllocator *alloc_in, void *base_in, size_t size, size_t align);
|
||||
~LockedPageArena();
|
||||
private:
|
||||
void *base;
|
||||
size_t size;
|
||||
LockedPageAllocator *allocator;
|
||||
};
|
||||
|
||||
bool new_arena(size_t size, size_t align);
|
||||
|
||||
std::list<LockedPageArena> arenas;
|
||||
LockingFailed_Callback lf_cb;
|
||||
size_t cumulative_bytes_locked;
|
||||
/** Mutex protects access to this pool's data structures, including arenas.
|
||||
*/
|
||||
mutable std::mutex mutex;
|
||||
};
|
||||
|
||||
/**
|
||||
* Singleton class to keep track of locked (ie, non-swappable) memory, for use in
|
||||
* std::allocator templates.
|
||||
*
|
||||
* Some implementations of the STL allocate memory in some constructors (i.e., see
|
||||
* MSVC's vector<T> implementation where it allocates 1 byte of memory in the allocator.)
|
||||
* Due to the unpredictable order of static initializers, we have to make sure the
|
||||
* LockedPoolManager instance exists before any other STL-based objects that use
|
||||
* secure_allocator are created. So instead of having LockedPoolManager also be
|
||||
* static-initialized, it is created on demand.
|
||||
*/
|
||||
class LockedPoolManager : public LockedPool
|
||||
{
|
||||
public:
|
||||
/** Return the current instance, or create it once */
|
||||
static LockedPoolManager& Instance()
|
||||
{
|
||||
std::call_once(LockedPoolManager::init_flag, LockedPoolManager::CreateInstance);
|
||||
return *LockedPoolManager::_instance;
|
||||
}
|
||||
|
||||
private:
|
||||
LockedPoolManager(std::unique_ptr<LockedPageAllocator> allocator);
|
||||
|
||||
/** Create a new LockedPoolManager specialized to the OS */
|
||||
static void CreateInstance();
|
||||
/** Called when locking fails, warn the user here */
|
||||
static bool LockingFailed();
|
||||
|
||||
static LockedPoolManager* _instance;
|
||||
static std::once_flag init_flag;
|
||||
};
|
||||
|
||||
#endif // BITCOIN_SUPPORT_LOCKEDPOOL_H
|
||||
@ -1,70 +0,0 @@
|
||||
// Copyright (c) 2009-2015 The Bitcoin Core developers
|
||||
// Distributed under the MIT software license, see the accompanying
|
||||
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
|
||||
|
||||
#include "support/pagelocker.h"
|
||||
|
||||
#if defined(HAVE_CONFIG_H)
|
||||
#include "config/bitcoin-config.h"
|
||||
#endif
|
||||
|
||||
#ifdef WIN32
|
||||
#ifdef _WIN32_WINNT
|
||||
#undef _WIN32_WINNT
|
||||
#endif
|
||||
#define _WIN32_WINNT 0x0501
|
||||
#define WIN32_LEAN_AND_MEAN 1
|
||||
#ifndef NOMINMAX
|
||||
#define NOMINMAX
|
||||
#endif
|
||||
#include <windows.h>
|
||||
// This is used to attempt to keep keying material out of swap
|
||||
// Note that VirtualLock does not provide this as a guarantee on Windows,
|
||||
// but, in practice, memory that has been VirtualLock'd almost never gets written to
|
||||
// the pagefile except in rare circumstances where memory is extremely low.
|
||||
#else
|
||||
#include <sys/mman.h>
|
||||
#include <limits.h> // for PAGESIZE
|
||||
#include <unistd.h> // for sysconf
|
||||
#endif
|
||||
|
||||
LockedPageManager* LockedPageManager::_instance = NULL;
|
||||
boost::once_flag LockedPageManager::init_flag = BOOST_ONCE_INIT;
|
||||
|
||||
/** Determine system page size in bytes */
|
||||
static inline size_t GetSystemPageSize()
|
||||
{
|
||||
size_t page_size;
|
||||
#if defined(WIN32)
|
||||
SYSTEM_INFO sSysInfo;
|
||||
GetSystemInfo(&sSysInfo);
|
||||
page_size = sSysInfo.dwPageSize;
|
||||
#elif defined(PAGESIZE) // defined in limits.h
|
||||
page_size = PAGESIZE;
|
||||
#else // assume some POSIX OS
|
||||
page_size = sysconf(_SC_PAGESIZE);
|
||||
#endif
|
||||
return page_size;
|
||||
}
|
||||
|
||||
bool MemoryPageLocker::Lock(const void* addr, size_t len)
|
||||
{
|
||||
#ifdef WIN32
|
||||
return VirtualLock(const_cast<void*>(addr), len) != 0;
|
||||
#else
|
||||
return mlock(addr, len) == 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
bool MemoryPageLocker::Unlock(const void* addr, size_t len)
|
||||
{
|
||||
#ifdef WIN32
|
||||
return VirtualUnlock(const_cast<void*>(addr), len) != 0;
|
||||
#else
|
||||
return munlock(addr, len) == 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
LockedPageManager::LockedPageManager() : LockedPageManagerBase<MemoryPageLocker>(GetSystemPageSize())
|
||||
{
|
||||
}
|
||||
@ -1,160 +0,0 @@
|
||||
// Copyright (c) 2009-2010 Satoshi Nakamoto
|
||||
// Copyright (c) 2009-2015 The Bitcoin Core developers
|
||||
// Distributed under the MIT software license, see the accompanying
|
||||
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
|
||||
|
||||
#ifndef BITCOIN_SUPPORT_PAGELOCKER_H
|
||||
#define BITCOIN_SUPPORT_PAGELOCKER_H
|
||||
|
||||
#include "support/cleanse.h"
|
||||
|
||||
#include <map>
|
||||
|
||||
#include <boost/thread/mutex.hpp>
|
||||
#include <boost/thread/once.hpp>
|
||||
|
||||
/**
|
||||
* Thread-safe class to keep track of locked (ie, non-swappable) memory pages.
|
||||
*
|
||||
* Memory locks do not stack, that is, pages which have been locked several times by calls to mlock()
|
||||
* will be unlocked by a single call to munlock(). This can result in keying material ending up in swap when
|
||||
* those functions are used naively. This class simulates stacking memory locks by keeping a counter per page.
|
||||
*
|
||||
* @note By using a map from each page base address to lock count, this class is optimized for
|
||||
* small objects that span up to a few pages, mostly smaller than a page. To support large allocations,
|
||||
* something like an interval tree would be the preferred data structure.
|
||||
*/
|
||||
template <class Locker>
|
||||
class LockedPageManagerBase
|
||||
{
|
||||
public:
|
||||
LockedPageManagerBase(size_t _page_size) : page_size(_page_size)
|
||||
{
|
||||
// Determine bitmask for extracting page from address
|
||||
assert(!(_page_size & (_page_size - 1))); // size must be power of two
|
||||
page_mask = ~(_page_size - 1);
|
||||
}
|
||||
|
||||
~LockedPageManagerBase()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
// For all pages in affected range, increase lock count
|
||||
void LockRange(void* p, size_t size)
|
||||
{
|
||||
boost::mutex::scoped_lock lock(mutex);
|
||||
if (!size)
|
||||
return;
|
||||
const size_t base_addr = reinterpret_cast<size_t>(p);
|
||||
const size_t start_page = base_addr & page_mask;
|
||||
const size_t end_page = (base_addr + size - 1) & page_mask;
|
||||
for (size_t page = start_page; page <= end_page; page += page_size) {
|
||||
Histogram::iterator it = histogram.find(page);
|
||||
if (it == histogram.end()) // Newly locked page
|
||||
{
|
||||
locker.Lock(reinterpret_cast<void*>(page), page_size);
|
||||
histogram.insert(std::make_pair(page, 1));
|
||||
} else // Page was already locked; increase counter
|
||||
{
|
||||
it->second += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// For all pages in affected range, decrease lock count
|
||||
void UnlockRange(void* p, size_t size)
|
||||
{
|
||||
boost::mutex::scoped_lock lock(mutex);
|
||||
if (!size)
|
||||
return;
|
||||
const size_t base_addr = reinterpret_cast<size_t>(p);
|
||||
const size_t start_page = base_addr & page_mask;
|
||||
const size_t end_page = (base_addr + size - 1) & page_mask;
|
||||
for (size_t page = start_page; page <= end_page; page += page_size) {
|
||||
Histogram::iterator it = histogram.find(page);
|
||||
assert(it != histogram.end()); // Cannot unlock an area that was not locked
|
||||
// Decrease counter for page, when it is zero, the page will be unlocked
|
||||
it->second -= 1;
|
||||
if (it->second == 0) // Nothing on the page anymore that keeps it locked
|
||||
{
|
||||
// Unlock page and remove the count from histogram
|
||||
locker.Unlock(reinterpret_cast<void*>(page), page_size);
|
||||
histogram.erase(it);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Get number of locked pages for diagnostics
|
||||
int GetLockedPageCount()
|
||||
{
|
||||
boost::mutex::scoped_lock lock(mutex);
|
||||
return histogram.size();
|
||||
}
|
||||
|
||||
private:
|
||||
Locker locker;
|
||||
boost::mutex mutex;
|
||||
size_t page_size, page_mask;
|
||||
// map of page base address to lock count
|
||||
typedef std::map<size_t, int> Histogram;
|
||||
Histogram histogram;
|
||||
};
|
||||
|
||||
|
||||
/**
|
||||
* OS-dependent memory page locking/unlocking.
|
||||
* Defined as policy class to make stubbing for test possible.
|
||||
*/
|
||||
class MemoryPageLocker
|
||||
{
|
||||
public:
|
||||
/** Lock memory pages.
|
||||
* addr and len must be a multiple of the system page size
|
||||
*/
|
||||
bool Lock(const void* addr, size_t len);
|
||||
/** Unlock memory pages.
|
||||
* addr and len must be a multiple of the system page size
|
||||
*/
|
||||
bool Unlock(const void* addr, size_t len);
|
||||
};
|
||||
|
||||
/**
|
||||
* Singleton class to keep track of locked (ie, non-swappable) memory pages, for use in
|
||||
* std::allocator templates.
|
||||
*
|
||||
* Some implementations of the STL allocate memory in some constructors (i.e., see
|
||||
* MSVC's vector<T> implementation where it allocates 1 byte of memory in the allocator.)
|
||||
* Due to the unpredictable order of static initializers, we have to make sure the
|
||||
* LockedPageManager instance exists before any other STL-based objects that use
|
||||
* secure_allocator are created. So instead of having LockedPageManager also be
|
||||
* static-initialized, it is created on demand.
|
||||
*/
|
||||
class LockedPageManager : public LockedPageManagerBase<MemoryPageLocker>
|
||||
{
|
||||
public:
|
||||
static LockedPageManager& Instance()
|
||||
{
|
||||
boost::call_once(LockedPageManager::CreateInstance, LockedPageManager::init_flag);
|
||||
return *LockedPageManager::_instance;
|
||||
}
|
||||
|
||||
private:
|
||||
LockedPageManager();
|
||||
|
||||
static void CreateInstance()
|
||||
{
|
||||
// Using a local static instance guarantees that the object is initialized
|
||||
// when it's first needed and also deinitialized after all objects that use
|
||||
// it are done with it. I can think of one unlikely scenario where we may
|
||||
// have a static deinitialization order/problem, but the check in
|
||||
// LockedPageManagerBase's destructor helps us detect if that ever happens.
|
||||
static LockedPageManager instance;
|
||||
LockedPageManager::_instance = &instance;
|
||||
}
|
||||
|
||||
static LockedPageManager* _instance;
|
||||
static boost::once_flag init_flag;
|
||||
};
|
||||
|
||||
#endif // BITCOIN_SUPPORT_PAGELOCKER_H
|
||||
Loading…
Reference in new issue