Cypher_Stack/wownero
2
0
Fork 0
mirror of https://git.wownero.com/wownero/wownero.git synced 2024-12-25 00:48:52 +00:00
Code Issues Packages Projects Releases Wiki Activity
f7e99047e4
wownero/src/blockchain_db/berkeleydb/db_bdb.cpp

2062 lines
61 KiB
C++
Raw Blame History

// Copyright (c) 2014, The Monero Project
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "db_bdb.h"
#include <boost/filesystem.hpp>
#include <memory> // std::unique_ptr
#include <cstring> // memcpy
#include "cryptonote_core/cryptonote_format_utils.h"
#include "crypto/crypto.h"
#include "profile_tools.h"
using epee::string_tools::pod_to_hex;
#define DB_DEFAULT_TX (m_write_txn != nullptr ? *m_write_txn : (DbTxn*) nullptr)
namespace
{
template <typename T>
inline void throw0(const T &e)
{
LOG_PRINT_L0(e.what());
throw e;
}
template <typename T>
inline void throw1(const T &e)
{
LOG_PRINT_L1(e.what());
throw e;
}
// cursor needs to be closed when it goes out of scope,
// this helps if the function using it throws
struct bdb_cur
{
bdb_cur(DbTxn* txn, Db* dbi)
{
if (dbi->cursor(txn, &m_cur, 0))
throw0(cryptonote::DB_ERROR("Error opening db cursor"));
done = false;
}
~bdb_cur()
{
close();
}
operator Dbc*()
{
return m_cur;
}
operator Dbc**()
{
return &m_cur;
}
Dbc* operator->()
{
return m_cur;
}
void close()
{
if (!done)
{
m_cur->close();
done = true;
}
}
private:
Dbc* m_cur;
bool done;
};
const char* const BDB_BLOCKS = "blocks";
const char* const BDB_BLOCK_TIMESTAMPS = "block_timestamps";
const char* const BDB_BLOCK_HEIGHTS = "block_heights";
const char* const BDB_BLOCK_HASHES = "block_hashes";
const char* const BDB_BLOCK_SIZES = "block_sizes";
const char* const BDB_BLOCK_DIFFS = "block_diffs";
const char* const BDB_BLOCK_COINS = "block_coins";
const char* const BDB_TXS = "txs";
const char* const BDB_TX_UNLOCKS = "tx_unlocks";
const char* const BDB_TX_HEIGHTS = "tx_heights";
const char* const BDB_TX_OUTPUTS = "tx_outputs";
const char* const BDB_OUTPUT_TXS = "output_txs";
const char* const BDB_OUTPUT_INDICES = "output_indices";
const char* const BDB_OUTPUT_AMOUNTS = "output_amounts";
const char* const BDB_OUTPUT_KEYS = "output_keys";
const char* const BDB_SPENT_KEYS = "spent_keys";
const char* const BDB_HF_STARTING_HEIGHTS = "hf_starting_heights";
const char* const BDB_HF_VERSIONS = "hf_versions";
const unsigned int MB = 1024 * 1024;
// ND: FIXME: db keeps running out of locks when doing full syncs. Possible bug??? Set it to 5K for now.
const unsigned int DB_MAX_LOCKS = 5000;
const unsigned int DB_BUFFER_LENGTH = 32 * MB;
// 256MB cache adjust as necessary using DB_CONFIG
const unsigned int DB_DEF_CACHESIZE = 256 * MB;
#if defined(BDB_BULK_CAN_THREAD)
const unsigned int DB_BUFFER_COUNT = std::thread::hardware_concurrency();
#else
const unsigned int DB_BUFFER_COUNT = 1;
#endif
template<typename T>
struct Dbt_copy: public Dbt
{
Dbt_copy(const T &t) :
t_copy(t)
{
init();
}
Dbt_copy()
{
init();
}
void init()
{
set_data(&t_copy);
set_size(sizeof(T));
set_ulen(sizeof(T));
set_flags(DB_DBT_USERMEM);
}
operator T()
{
return t_copy;
}
private:
T t_copy;
};
template<>
struct Dbt_copy<cryptonote::blobdata>: public Dbt
{
Dbt_copy(const cryptonote::blobdata &bd) :
m_data(new char[bd.size()])
{
memcpy(m_data.get(), bd.data(), bd.size());
set_data(m_data.get());
set_size(bd.size());
set_ulen(bd.size());
set_flags(DB_DBT_USERMEM);
}
private:
std::unique_ptr<char[]> m_data;
};
struct Dbt_safe : public Dbt
{
Dbt_safe()
{
set_data(NULL);
set_flags(DB_DBT_MALLOC);
}
~Dbt_safe()
{
void* buf = get_data();
if (buf != NULL)
{
free(buf);
}
}
};
} // anonymous namespace
namespace cryptonote
{
void BlockchainBDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const crypto::hash& blk_hash)
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
Dbt_copy<crypto::hash> val_h(blk_hash);
if (m_block_heights->exists(DB_DEFAULT_TX, &val_h, 0) == 0)
throw1(BLOCK_EXISTS("Attempting to add block that's already in the db"));
if (m_height > 0)
{
Dbt_copy<crypto::hash> parent_key(blk.prev_id);
Dbt_copy<uint32_t> parent_h;
if (m_block_heights->get(DB_DEFAULT_TX, &parent_key, &parent_h, 0))
{
LOG_PRINT_L3("m_height: " << m_height);
LOG_PRINT_L3("parent_key: " << blk.prev_id);
throw0(DB_ERROR("Failed to get top block hash to check for new block's parent"));
}
uint32_t parent_height = parent_h;
if (parent_height != m_height)
throw0(BLOCK_PARENT_DNE("Top block is not new block's parent"));
}
Dbt_copy<uint32_t> key(m_height + 1);
Dbt_copy<blobdata> blob(block_to_blob(blk));
auto res = m_blocks->put(DB_DEFAULT_TX, &key, &blob, 0);
if (res)
throw0(DB_ERROR("Failed to add block blob to db transaction."));
Dbt_copy<size_t> sz(block_size);
if (m_block_sizes->put(DB_DEFAULT_TX, &key, &sz, 0))
throw0(DB_ERROR("Failed to add block size to db transaction."));
Dbt_copy<uint64_t> ts(blk.timestamp);
if (m_block_timestamps->put(DB_DEFAULT_TX, &key, &ts, 0))
throw0(DB_ERROR("Failed to add block timestamp to db transaction."));
Dbt_copy<difficulty_type> diff(cumulative_difficulty);
if (m_block_diffs->put(DB_DEFAULT_TX, &key, &diff, 0))
throw0(DB_ERROR("Failed to add block cumulative difficulty to db transaction."));
Dbt_copy<uint64_t> coinsgen(coins_generated);
if (m_block_coins->put(DB_DEFAULT_TX, &key, &coinsgen, 0))
throw0(DB_ERROR("Failed to add block total generated coins to db transaction."));
if (m_block_heights->put(DB_DEFAULT_TX, &val_h, &key, 0))
throw0(DB_ERROR("Failed to add block height by hash to db transaction."));
if (m_block_hashes->put(DB_DEFAULT_TX, &key, &val_h, 0))
throw0(DB_ERROR("Failed to add block hash to db transaction."));
}
void BlockchainBDB::remove_block()
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
if (m_height == 0)
throw0(BLOCK_DNE ("Attempting to remove block from an empty blockchain"));
Dbt_copy<uint32_t> k(m_height);
Dbt_copy<crypto::hash> h;
if (m_block_hashes->get(DB_DEFAULT_TX, &k, &h, 0))
throw1(BLOCK_DNE("Attempting to remove block that's not in the db"));
if (m_blocks->del(DB_DEFAULT_TX, &k, 0))
throw1(DB_ERROR("Failed to add removal of block to db transaction"));
if (m_block_sizes->del(DB_DEFAULT_TX, &k, 0))
throw1(DB_ERROR("Failed to add removal of block size to db transaction"));
if (m_block_diffs->del(DB_DEFAULT_TX, &k, 0))
throw1(DB_ERROR("Failed to add removal of block cumulative difficulty to db transaction"));
if (m_block_coins->del(DB_DEFAULT_TX, &k, 0))
throw1(DB_ERROR("Failed to add removal of block total generated coins to db transaction"));
if (m_block_timestamps->del(DB_DEFAULT_TX, &k, 0))
throw1(DB_ERROR("Failed to add removal of block timestamp to db transaction"));
if (m_block_heights->del(DB_DEFAULT_TX, &h, 0))
throw1(DB_ERROR("Failed to add removal of block height by hash to db transaction"));
if (m_block_hashes->del(DB_DEFAULT_TX, &k, 0))
throw1(DB_ERROR("Failed to add removal of block hash to db transaction"));
}
void BlockchainBDB::add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash)
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
Dbt_copy<crypto::hash> val_h(tx_hash);
if (m_txs->exists(DB_DEFAULT_TX, &val_h, 0) == 0)
throw1(TX_EXISTS("Attempting to add transaction that's already in the db"));
Dbt_copy<blobdata> blob(tx_to_blob(tx));
if (m_txs->put(DB_DEFAULT_TX, &val_h, &blob, 0))
throw0(DB_ERROR("Failed to add tx blob to db transaction"));
Dbt_copy<uint64_t> height(m_height + 1);
if (m_tx_heights->put(DB_DEFAULT_TX, &val_h, &height, 0))
throw0(DB_ERROR("Failed to add tx block height to db transaction"));
Dbt_copy<uint64_t> unlock_time(tx.unlock_time);
if (m_tx_unlocks->put(DB_DEFAULT_TX, &val_h, &unlock_time, 0))
throw0(DB_ERROR("Failed to add tx unlock time to db transaction"));
}
void BlockchainBDB::remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx)
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
Dbt_copy<crypto::hash> val_h(tx_hash);
if (m_txs->exists(DB_DEFAULT_TX, &val_h, 0))
throw1(TX_DNE("Attempting to remove transaction that isn't in the db"));
if (m_txs->del(DB_DEFAULT_TX, &val_h, 0))
throw1(DB_ERROR("Failed to add removal of tx to db transaction"));
if (m_tx_unlocks->del(DB_DEFAULT_TX, &val_h, 0))
throw1(DB_ERROR("Failed to add removal of tx unlock time to db transaction"));
if (m_tx_heights->del(DB_DEFAULT_TX, &val_h, 0))
throw1(DB_ERROR("Failed to add removal of tx block height to db transaction"));
remove_tx_outputs(tx_hash, tx);
if (m_tx_outputs->del(DB_DEFAULT_TX, &val_h, 0))
throw1(DB_ERROR("Failed to add removal of tx outputs to db transaction"));
}
void BlockchainBDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time)
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
Dbt_copy<uint32_t> k(m_num_outputs + 1);
Dbt_copy<crypto::hash> v(tx_hash);
if (m_output_txs->put(DB_DEFAULT_TX, &k, &v, 0))
throw0(DB_ERROR("Failed to add output tx hash to db transaction"));
if (m_tx_outputs->put(DB_DEFAULT_TX, &v, &k, 0))
throw0(DB_ERROR("Failed to add tx output index to db transaction"));
Dbt_copy<uint64_t> val_local_index(local_index);
if (m_output_indices->put(DB_DEFAULT_TX, &k, &val_local_index, 0))
throw0(DB_ERROR("Failed to add tx output index to db transaction"));
Dbt_copy<uint64_t> val_amount(tx_output.amount);
if (m_output_amounts->put(DB_DEFAULT_TX, &val_amount, &k, 0))
throw0(DB_ERROR("Failed to add output amount to db transaction."));
if (tx_output.target.type() == typeid(txout_to_key))
{
output_data_t od;
od.pubkey = boost::get < txout_to_key > (tx_output.target).key;
od.unlock_time = unlock_time;
od.height = m_height;
Dbt_copy<output_data_t> data(od);
if (m_output_keys->put(DB_DEFAULT_TX, &k, &data, 0))
throw0(DB_ERROR("Failed to add output pubkey to db transaction"));
}
m_num_outputs++;
}
void BlockchainBDB::remove_tx_outputs(const crypto::hash& tx_hash, const transaction& tx)
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
bdb_cur cur(DB_DEFAULT_TX, m_tx_outputs);
Dbt_copy<crypto::hash> k(tx_hash);
Dbt_copy<uint32_t> v;
auto result = cur->get(&k, &v, DB_SET);
if (result == DB_NOTFOUND)
{
throw0(OUTPUT_DNE("Attempting to remove a tx's outputs, but none found."));
}
else if (result)
{
throw0(DB_ERROR("DB error attempting to get an output"));
}
else
{
db_recno_t num_elems = 0;
cur->count(&num_elems, 0);
for (uint64_t i = 0; i < num_elems; ++i)
{
const tx_out tx_output = tx.vout[i];
remove_output(v, tx_output.amount);
if (i < num_elems - 1)
{
cur->get(&k, &v, DB_NEXT_DUP);
}
}
}
cur.close();
}
// TODO: probably remove this function
void BlockchainBDB::remove_output(const tx_out& tx_output)
{
LOG_PRINT_L3("BlockchainBDB::" << __func__ << " (unused version - does nothing)");
return;
}
void BlockchainBDB::remove_output(const uint64_t& out_index, const uint64_t amount)
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
Dbt_copy<uint32_t> k(out_index);
auto result = m_output_indices->del(DB_DEFAULT_TX, &k, 0);
if (result == DB_NOTFOUND)
{
LOG_PRINT_L0("Unexpected: global output index not found in m_output_indices");
}
else if (result)
{
throw1(DB_ERROR("Error adding removal of output tx index to db transaction"));
}
result = m_output_txs->del(DB_DEFAULT_TX, &k, 0);
// if (result != 0 && result != DB_NOTFOUND)
// throw1(DB_ERROR("Error adding removal of output tx hash to db transaction"));
if (result == DB_NOTFOUND)
{
LOG_PRINT_L0("Unexpected: global output index not found in m_output_txs");
}
else if (result)
{
throw1(DB_ERROR("Error adding removal of output tx hash to db transaction"));
}
result = m_output_keys->del(DB_DEFAULT_TX, &k, 0);
if (result == DB_NOTFOUND)
{
LOG_PRINT_L0("Unexpected: global output index not found in m_output_keys");
}
else if (result)
throw1(DB_ERROR("Error adding removal of output pubkey to db transaction"));
remove_amount_output_index(amount, out_index);
m_num_outputs--;
}
void BlockchainBDB::remove_amount_output_index(const uint64_t amount, const uint64_t global_output_index)
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
bdb_cur cur(DB_DEFAULT_TX, m_output_amounts);
Dbt_copy<uint64_t> k(amount);
Dbt_copy<uint32_t> v;
auto result = cur->get(&k, &v, DB_SET);
if (result == DB_NOTFOUND)
throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found"));
else if (result)
throw0(DB_ERROR("DB error attempting to get an output"));
db_recno_t num_elems = 0;
cur->count(&num_elems, 0);
uint64_t amount_output_index = 0;
uint64_t goi = 0;
bool found_index = false;
for (uint64_t i = 0; i < num_elems; ++i)
{
goi = v;
if (goi == global_output_index)
{
amount_output_index = i;
found_index = true;
break;
}
cur->get(&k, &v, DB_NEXT_DUP);
}
if (found_index)
{
// found the amount output index
// now delete it
result = cur->del(0);
if (result)
throw0(DB_ERROR(std::string("Error deleting amount output index ").append(boost::lexical_cast<std::string>(amount_output_index)).c_str()));
}
else
{
// not found
throw1(OUTPUT_DNE("Failed to find amount output index"));
}
cur.close();
}
void BlockchainBDB::add_spent_key(const crypto::key_image& k_image)
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
Dbt_copy<crypto::key_image> val_key(k_image);
if (m_spent_keys->exists(DB_DEFAULT_TX, &val_key, 0) == 0)
throw1(KEY_IMAGE_EXISTS("Attempting to add spent key image that's already in the db"));
Dbt_copy<char> val('\0');
if (m_spent_keys->put(DB_DEFAULT_TX, &val_key, &val, 0))
throw1(DB_ERROR("Error adding spent key image to db transaction."));
}
void BlockchainBDB::remove_spent_key(const crypto::key_image& k_image)
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
Dbt_copy<crypto::key_image> k(k_image);
auto result = m_spent_keys->del(DB_DEFAULT_TX, &k, 0);
if (result != 0 && result != DB_NOTFOUND)
throw1(DB_ERROR("Error adding removal of key image to db transaction"));
}
bool BlockchainBDB::for_all_key_images(std::function<bool(const crypto::key_image&)> f) const
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
bdb_cur cur(DB_DEFAULT_TX, m_spent_keys);
Dbt_copy<crypto::key_image> k;
Dbt_copy<char> v;
bool ret = true;
int result;
while ((result = cur->get(&k, &v, DB_NEXT)) == 0)
{
if (!f(k))
{
ret = false;
break;
}
}
if (result != DB_NOTFOUND)
ret = false;
cur.close();
return ret;
}
bool BlockchainBDB::for_all_blocks(std::function<bool(uint64_t, const crypto::hash&, const cryptonote::block&)> f) const
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
bdb_cur cur(DB_DEFAULT_TX, m_blocks);
Dbt_copy<uint64_t> k;
Dbt_safe v;
bool ret = true;
int result;
while ((result = cur->get(&k, &v, DB_NEXT)) == 0)
{
uint64_t height = k - 1;
blobdata bd;
bd.assign(reinterpret_cast<char*>(v.get_data()), v.get_size());
block b;
if (!parse_and_validate_block_from_blob(bd, b))
throw0(DB_ERROR("Failed to parse block from blob retrieved from the db"));
crypto::hash hash;
if (!get_block_hash(b, hash))
throw0(DB_ERROR("Failed to get block hash from blob retrieved from the db"));
if (!f(height, hash, b))
{
ret = false;
break;
}
}
if (result != DB_NOTFOUND)
ret = false;
cur.close();
return ret;
}
bool BlockchainBDB::for_all_transactions(std::function<bool(const crypto::hash&, const cryptonote::transaction&)> f) const
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
bdb_cur cur(DB_DEFAULT_TX, m_txs);
Dbt_copy<crypto::hash> k;
Dbt_safe v;
bool ret = true;
int result;
while ((result = cur->get(&k, &v, DB_NEXT)) == 0)
{
blobdata bd;
bd.assign(reinterpret_cast<char*>(v.get_data()), v.get_size());
transaction tx;
if (!parse_and_validate_tx_from_blob(bd, tx))
throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db"));
if (!f(k, tx))
{
ret = false;
break;
}
}
if (result != DB_NOTFOUND)
ret = false;
cur.close();
return ret;
}
bool BlockchainBDB::for_all_outputs(std::function<bool(uint64_t amount, const crypto::hash &tx_hash, size_t tx_idx)> f) const
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
bdb_cur cur(DB_DEFAULT_TX, m_output_amounts);
Dbt_copy<uint64_t> k;
Dbt_copy<uint32_t> v;
bool ret = true;
int result;
while ((result = cur->get(&k, &v, DB_NEXT)) == 0)
{
uint32_t global_index = v - 1;
tx_out_index toi = get_output_tx_and_index_from_global(global_index);
if (!f(k, toi.first, toi.second))
{
ret = false;
break;
}
}
if (result != DB_NOTFOUND)
ret = false;
cur.close();
return ret;
}
blobdata BlockchainBDB::output_to_blob(const tx_out& output) const
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
blobdata b;
if (!t_serializable_object_to_blob(output, b))
throw1(DB_ERROR("Error serializing output to blob"));
return b;
}
tx_out BlockchainBDB::output_from_blob(const blobdata& blob) const
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
std::stringstream ss;
ss << blob;
binary_archive<false> ba(ss);
tx_out o;
if (!(::serialization::serialize(ba, o)))
throw1(DB_ERROR("Error deserializing tx output blob"));
return o;
}
uint64_t BlockchainBDB::get_output_global_index(const uint64_t& amount, const uint64_t& index)
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
std::vector < uint64_t > offsets;
std::vector < uint64_t > global_indices;
offsets.push_back(index);
get_output_global_indices(amount, offsets, global_indices);
if (!global_indices.size())
throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found"));
return global_indices[0];
}
void BlockchainBDB::check_open() const
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
if (!m_open)
throw0(DB_ERROR("DB operation attempted on a not-open DB instance"));
}
BlockchainBDB::~BlockchainBDB()
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
if (m_open)
{
close();
}
}
BlockchainBDB::BlockchainBDB(bool batch_transactions) :
m_buffer(DB_BUFFER_COUNT, DB_BUFFER_LENGTH)
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
// initialize folder to something "safe" just in case
// someone accidentally misuses this class...
m_folder = "thishsouldnotexistbecauseitisgibberish";
m_open = false;
m_run_checkpoint = 0;
m_batch_transactions = batch_transactions;
m_write_txn = nullptr;
m_height = 0;
}
void BlockchainBDB::open(const std::string& filename, const int db_flags)
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
if (m_open)
throw0(DB_OPEN_FAILURE("Attempted to open db, but it's already open"));
boost::filesystem::path direc(filename);
if (boost::filesystem::exists(direc))
{
if (!boost::filesystem::is_directory(direc))
throw0(DB_OPEN_FAILURE("DB needs a directory path, but a file was passed"));
}
else
{
if (!boost::filesystem::create_directory(direc))
throw0(DB_OPEN_FAILURE(std::string("Failed to create directory ").append(filename).c_str()));
}
m_folder = filename;
try
{
//Create BerkeleyDB environment
m_env = new DbEnv(0); // no flags needed for DbEnv
uint32_t db_env_open_flags = DB_CREATE | DB_INIT_MPOOL | DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_TXN | DB_RECOVER | DB_THREAD;
// Set some default values for these parameters.
// They can be overridden using the DB_CONFIG file.
m_env->set_cachesize(0, DB_DEF_CACHESIZE, 1);
m_env->set_lk_max_locks(DB_MAX_LOCKS);
m_env->set_lk_max_lockers(DB_MAX_LOCKS);
m_env->set_lk_max_objects(DB_MAX_LOCKS);
// last parameter left 0, files will be created with default rw access
m_env->open(filename.c_str(), db_env_open_flags, 0);
m_env->set_flags(db_flags, 1);
if(m_auto_remove_logs)
m_env->log_set_config(DB_LOG_AUTO_REMOVE, 1);
// begin transaction to init dbs
bdb_txn_safe txn;
m_env->txn_begin(NULL, txn, 0);
// create Dbs in the environment
m_blocks = new Db(m_env, 0);
m_block_heights = new Db(m_env, 0);
m_block_hashes = new Db(m_env, 0);
m_block_timestamps = new Db(m_env, 0);
m_block_sizes = new Db(m_env, 0);
m_block_diffs = new Db(m_env, 0);
m_block_coins = new Db(m_env, 0);
m_txs = new Db(m_env, 0);
m_tx_unlocks = new Db(m_env, 0);
m_tx_heights = new Db(m_env, 0);
m_tx_outputs = new Db(m_env, 0);
m_output_txs = new Db(m_env, 0);
m_output_indices = new Db(m_env, 0);
m_output_amounts = new Db(m_env, 0);
m_output_keys = new Db(m_env, 0);
m_spent_keys = new Db(m_env, 0);
m_hf_starting_heights = new Db(m_env, 0);
m_hf_versions = new Db(m_env, 0);
// Tell DB about Dbs that need duplicate support
// Note: no need to tell about sorting,
// as the default is insertion order, which we want
m_tx_outputs->set_flags(DB_DUP);
m_output_amounts->set_flags(DB_DUP);
// Tell DB about fixed-size values.
m_block_hashes->set_re_len(sizeof(crypto::hash));
m_block_timestamps->set_re_len(sizeof(uint64_t));
m_block_sizes->set_re_len(sizeof(size_t)); // should really store block size as uint64_t...
m_block_diffs->set_re_len(sizeof(difficulty_type));
m_block_coins->set_re_len(sizeof(uint64_t));
m_output_txs->set_re_len(sizeof(crypto::hash));
m_output_indices->set_re_len(sizeof(uint64_t));
m_output_keys->set_re_len(sizeof(output_data_t));
m_hf_starting_heights->set_re_len(sizeof(uint64_t));
m_hf_versions->set_re_len(sizeof(uint8_t));
//TODO: Find out if we need to do Db::set_flags(DB_RENUMBER)
// for the RECNO databases. We shouldn't as we're only
// inserting/removing from the end, but we'll see.
// open Dbs in the environment
// m_tx_outputs and m_output_amounts must be DB_HASH or DB_BTREE
// because they need duplicate entry support. The rest are DB_RECNO,
// as it seems that will be the most performant choice.
m_blocks->open(txn, BDB_BLOCKS, NULL, DB_RECNO, DB_CREATE, 0);
m_block_timestamps->open(txn, BDB_BLOCK_TIMESTAMPS, NULL, DB_RECNO, DB_CREATE, 0);
m_block_heights->open(txn, BDB_BLOCK_HEIGHTS, NULL, DB_HASH, DB_CREATE, 0);
m_block_hashes->open(txn, BDB_BLOCK_HASHES, NULL, DB_RECNO, DB_CREATE, 0);
m_block_sizes->open(txn, BDB_BLOCK_SIZES, NULL, DB_RECNO, DB_CREATE, 0);
m_block_diffs->open(txn, BDB_BLOCK_DIFFS, NULL, DB_RECNO, DB_CREATE, 0);
m_block_coins->open(txn, BDB_BLOCK_COINS, NULL, DB_RECNO, DB_CREATE, 0);
m_txs->open(txn, BDB_TXS, NULL, DB_HASH, DB_CREATE, 0);
m_tx_unlocks->open(txn, BDB_TX_UNLOCKS, NULL, DB_HASH, DB_CREATE, 0);
m_tx_heights->open(txn, BDB_TX_HEIGHTS, NULL, DB_HASH, DB_CREATE, 0);
m_tx_outputs->open(txn, BDB_TX_OUTPUTS, NULL, DB_HASH, DB_CREATE, 0);
m_output_txs->open(txn, BDB_OUTPUT_TXS, NULL, DB_RECNO, DB_CREATE, 0);
m_output_indices->open(txn, BDB_OUTPUT_INDICES, NULL, DB_RECNO, DB_CREATE, 0);
m_output_amounts->open(txn, BDB_OUTPUT_AMOUNTS, NULL, DB_HASH, DB_CREATE, 0);
m_output_keys->open(txn, BDB_OUTPUT_KEYS, NULL, DB_RECNO, DB_CREATE, 0);
m_spent_keys->open(txn, BDB_SPENT_KEYS, NULL, DB_HASH, DB_CREATE, 0);
m_hf_starting_heights->open(txn, BDB_HF_STARTING_HEIGHTS, NULL, DB_RECNO, DB_CREATE, 0);
m_hf_versions->open(txn, BDB_HF_VERSIONS, NULL, DB_RECNO, DB_CREATE, 0);
txn.commit();
DB_BTREE_STAT* stats;
// DB_FAST_STAT can apparently cause an incorrect number of records
// to be returned. The flag should be set to 0 instead if this proves
// to be the case.
// ND: The bug above can occur when a block is popped and the application
// exits without pushing a new block to the db. Set txn to NULL and DB_FAST_STAT
// to zero (0) for reliability.
m_blocks->stat(NULL, &stats, 0);
m_height = stats->bt_nkeys;
delete stats;
// see above comment about DB_FAST_STAT
m_output_indices->stat(NULL, &stats, 0);
m_num_outputs = stats->bt_nkeys;
delete stats;
// run checkpoint thread
m_run_checkpoint = true;
m_checkpoint_thread.reset(new boost::thread(&BlockchainBDB::checkpoint_worker, this));
}
catch (const std::exception& e)
{
throw0(DB_OPEN_FAILURE(e.what()));
}
m_open = true;
}
void BlockchainBDB::close()
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
this->sync();
m_run_checkpoint = false;
m_checkpoint_thread->join();
m_checkpoint_thread.reset();
// FIXME: not yet thread safe!!! Use with care.
m_open = false;
m_env->close(DB_FORCESYNC);
}
void BlockchainBDB::sync()
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
try
{
m_blocks->sync(0);
m_block_heights->sync(0);
m_block_hashes->sync(0);
m_block_timestamps->sync(0);
m_block_sizes->sync(0);
m_block_diffs->sync(0);
m_block_coins->sync(0);
m_txs->sync(0);
m_tx_unlocks->sync(0);
m_tx_heights->sync(0);
m_tx_outputs->sync(0);
m_output_txs->sync(0);
m_output_indices->sync(0);
m_output_amounts->sync(0);
m_output_keys->sync(0);
m_spent_keys->sync(0);
m_hf_starting_heights->sync(0);
m_hf_versions->sync(0);
}
catch (const std::exception& e)
{
throw0(DB_ERROR(std::string("Failed to sync database: ").append(e.what()).c_str()));
}
}
void BlockchainBDB::reset()
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
// TODO: this
}
std::vector<std::string> BlockchainBDB::get_filenames() const
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
std::vector<std::string> filenames;
char *fname, *dbname;
const char **pfname, **pdbname;
pfname = (const char **)&fname;
pdbname = (const char **)&dbname;
m_blocks->get_dbname(pfname, pdbname);
filenames.push_back(fname);
m_block_heights->get_dbname(pfname, pdbname);
filenames.push_back(fname);
m_block_hashes->get_dbname(pfname, pdbname);
filenames.push_back(fname);
m_block_timestamps->get_dbname(pfname, pdbname);
filenames.push_back(fname);
m_block_sizes->get_dbname(pfname, pdbname);
filenames.push_back(fname);
m_block_diffs->get_dbname(pfname, pdbname);
filenames.push_back(fname);
m_block_coins->get_dbname(pfname, pdbname);
filenames.push_back(fname);
m_txs->get_dbname(pfname, pdbname);
filenames.push_back(fname);
m_tx_unlocks->get_dbname(pfname, pdbname);
filenames.push_back(fname);
m_tx_heights->get_dbname(pfname, pdbname);
filenames.push_back(fname);
m_tx_outputs->get_dbname(pfname, pdbname);
filenames.push_back(fname);
m_output_txs->get_dbname(pfname, pdbname);
filenames.push_back(fname);
m_output_indices->get_dbname(pfname, pdbname);
filenames.push_back(fname);
m_output_amounts->get_dbname(pfname, pdbname);
filenames.push_back(fname);
m_output_keys->get_dbname(pfname, pdbname);
filenames.push_back(fname);
m_spent_keys->get_dbname(pfname, pdbname);
filenames.push_back(fname);
m_hf_starting_heights->get_dbname(pfname, pdbname);
filenames.push_back(fname);
m_hf_versions->get_dbname(pfname, pdbname);
filenames.push_back(fname);
std::vector<std::string> full_paths;
for (auto& filename : filenames)
{
boost::filesystem::path p(m_folder);
p /= filename;
full_paths.push_back(p.string());
}
return full_paths;
}
std::string BlockchainBDB::get_db_name() const
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
return std::string("BerkeleyDB");
}
// TODO: this?
bool BlockchainBDB::lock()
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
return false;
}
// TODO: this?
void BlockchainBDB::unlock()
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
}
bool BlockchainBDB::block_exists(const crypto::hash& h) const
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
Dbt_copy<crypto::hash> key(h);
auto get_result = m_block_heights->exists(DB_DEFAULT_TX, &key, 0);
if (get_result == DB_NOTFOUND)
{
LOG_PRINT_L3("Block with hash " << epee::string_tools::pod_to_hex(h) << " not found in db");
return false;
}
else if (get_result)
throw0(DB_ERROR("DB error attempting to fetch block index from hash"));
return true;
}
block BlockchainBDB::get_block(const crypto::hash& h) const
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
return get_block_from_height(get_block_height(h));
}
uint64_t BlockchainBDB::get_block_height(const crypto::hash& h) const
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
Dbt_copy<crypto::hash> key(h);
Dbt_copy<uint32_t> result;
auto get_result = m_block_heights->get(DB_DEFAULT_TX, &key, &result, 0);
if (get_result == DB_NOTFOUND)
throw1(BLOCK_DNE("Attempted to retrieve non-existent block height"));
else if (get_result)
throw0(DB_ERROR("Error attempting to retrieve a block height from the db"));
return result - 1;
}
block_header BlockchainBDB::get_block_header(const crypto::hash& h) const
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
// block_header object is automatically cast from block object
return get_block(h);
}
block BlockchainBDB::get_block_from_height(const uint64_t& height) const
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
Dbt_copy<uint32_t> key(height + 1);
Dbt_safe result;
auto get_result = m_blocks->get(DB_DEFAULT_TX, &key, &result, 0);
if (get_result == DB_NOTFOUND)
{
throw0(BLOCK_DNE(std::string("Attempt to get block from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- block not in db").c_str()));
}
else if (get_result)
throw0(DB_ERROR("Error attempting to retrieve a block from the db"));
blobdata bd;
bd.assign(reinterpret_cast<char*>(result.get_data()), result.get_size());
block b;
if (!parse_and_validate_block_from_blob(bd, b))
throw0(DB_ERROR("Failed to parse block from blob retrieved from the db"));
return b;
}
uint64_t BlockchainBDB::get_block_timestamp(const uint64_t& height) const
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
Dbt_copy<uint32_t> key(height + 1);
Dbt_copy<uint64_t> result;
auto get_result = m_block_timestamps->get(DB_DEFAULT_TX, &key, &result, 0);
if (get_result == DB_NOTFOUND)
{
throw0(BLOCK_DNE(std::string("Attempt to get timestamp from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- timestamp not in db").c_str()));
}
else if (get_result)
throw0(DB_ERROR("Error attempting to retrieve a timestamp from the db"));
return result;
}
uint64_t BlockchainBDB::get_top_block_timestamp() const
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
// if no blocks, return 0
if (m_height == 0)
{
return 0;
}
return get_block_timestamp(m_height - 1);
}
size_t BlockchainBDB::get_block_size(const uint64_t& height) const
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
Dbt_copy<uint32_t> key(height + 1);
Dbt_copy<size_t> result;
auto get_result = m_block_sizes->get(DB_DEFAULT_TX, &key, &result, 0);
if (get_result == DB_NOTFOUND)
{
throw0(BLOCK_DNE(std::string("Attempt to get block size from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- block size not in db").c_str()));
}
else if (get_result)
throw0(DB_ERROR("Error attempting to retrieve a block size from the db"));
return result;
}
difficulty_type BlockchainBDB::get_block_cumulative_difficulty(const uint64_t& height) const
{
LOG_PRINT_L3("BlockchainBDB::" << __func__ << " height: " << height);
check_open();
Dbt_copy<uint32_t> key(height + 1);
Dbt_copy<difficulty_type> result;
auto get_result = m_block_diffs->get(DB_DEFAULT_TX, &key, &result, 0);
if (get_result == DB_NOTFOUND)
{
throw0(BLOCK_DNE(std::string("Attempt to get cumulative difficulty from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- difficulty not in db").c_str()));
}
else if (get_result)
throw0(DB_ERROR("Error attempting to retrieve a cumulative difficulty from the db"));
return result;
}
difficulty_type BlockchainBDB::get_block_difficulty(const uint64_t& height) const
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
difficulty_type diff1 = 0;
difficulty_type diff2 = 0;
diff1 = get_block_cumulative_difficulty(height);
if (height != 0)
{
diff2 = get_block_cumulative_difficulty(height - 1);
}
return diff1 - diff2;
}
uint64_t BlockchainBDB::get_block_already_generated_coins(const uint64_t& height) const
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
Dbt_copy<uint32_t> key(height + 1);
Dbt_copy<uint64_t> result;
auto get_result = m_block_coins->get(DB_DEFAULT_TX, &key, &result, 0);
if (get_result == DB_NOTFOUND)
{
throw0(BLOCK_DNE(std::string("Attempt to get generated coins from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- block size not in db").c_str()));
}
else if (get_result)
throw0(DB_ERROR("Error attempting to retrieve a total generated coins from the db"));
return result;
}
crypto::hash BlockchainBDB::get_block_hash_from_height(const uint64_t& height) const
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
Dbt_copy<uint32_t> key(height + 1);
Dbt_copy<crypto::hash> result;
auto get_result = m_block_hashes->get(DB_DEFAULT_TX, &key, &result, 0);
if (get_result == DB_NOTFOUND)
{
throw0(BLOCK_DNE(std::string("Attempt to get hash from height ").append(boost::lexical_cast<std::string>(height)).append(" failed -- hash not in db").c_str()));
}
else if (get_result)
throw0(DB_ERROR("Error attempting to retrieve a block hash from the db."));
return result;
}
std::vector<block> BlockchainBDB::get_blocks_range(const uint64_t& h1, const uint64_t& h2) const
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
std::vector<block> v;
for (uint64_t height = h1; height <= h2; ++height)
{
v.push_back(get_block_from_height(height));
}
return v;
}
std::vector<crypto::hash> BlockchainBDB::get_hashes_range(const uint64_t& h1, const uint64_t& h2) const
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
std::vector<crypto::hash> v;
for (uint64_t height = h1; height <= h2; ++height)
{
v.push_back(get_block_hash_from_height(height));
}
return v;
}
crypto::hash BlockchainBDB::top_block_hash() const
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
if (m_height > 0)
{
return get_block_hash_from_height(m_height - 1);
}
return null_hash;
}
block BlockchainBDB::get_top_block() const
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
if (m_height > 0)
{
return get_block_from_height(m_height - 1);
}
block b;
return b;
}
uint64_t BlockchainBDB::height() const
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
return m_height;
}
bool BlockchainBDB::tx_exists(const crypto::hash& h) const
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
Dbt_copy<crypto::hash> key(h);
TIME_MEASURE_START(time1);
auto get_result = m_txs->exists(DB_DEFAULT_TX, &key, 0);
TIME_MEASURE_FINISH(time1);
time_tx_exists += time1;
if (get_result == DB_NOTFOUND)
{
LOG_PRINT_L1("transaction with hash " << epee::string_tools::pod_to_hex(h) << " not found in db");
return false;
}
else if (get_result)
throw0(DB_ERROR("DB error attempting to fetch transaction from hash"));
return true;
}
uint64_t BlockchainBDB::get_tx_unlock_time(const crypto::hash& h) const
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
Dbt_copy<crypto::hash> key(h);
Dbt_copy<uint64_t> result;
auto get_result = m_tx_unlocks->get(DB_DEFAULT_TX, &key, &result, 0);
if (get_result == DB_NOTFOUND)
throw1(TX_DNE(std::string("tx unlock time with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str()));
else if (get_result)
throw0(DB_ERROR("DB error attempting to fetch tx unlock time from hash"));
return result;
}
transaction BlockchainBDB::get_tx(const crypto::hash& h) const
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
Dbt_copy<crypto::hash> key(h);
Dbt_safe result;
auto get_result = m_txs->get(DB_DEFAULT_TX, &key, &result, 0);
if (get_result == DB_NOTFOUND)
throw1(TX_DNE(std::string("tx with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str()));
else if (get_result)
throw0(DB_ERROR("DB error attempting to fetch tx from hash"));
blobdata bd;
bd.assign(reinterpret_cast<char*>(result.get_data()), result.get_size());
transaction tx;
if (!parse_and_validate_tx_from_blob(bd, tx))
throw0(DB_ERROR("Failed to parse tx from blob retrieved from the db"));
return tx;
}
uint64_t BlockchainBDB::get_tx_count() const
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
DB_BTREE_STAT* stats;
// DB_FAST_STAT can apparently cause an incorrect number of records
// to be returned. The flag should be set to 0 instead if this proves
// to be the case.
m_txs->stat(DB_DEFAULT_TX, &stats, DB_FAST_STAT);
auto num_txs = stats->bt_nkeys;
delete stats;
return num_txs;
}
std::vector<transaction> BlockchainBDB::get_tx_list(const std::vector<crypto::hash>& hlist) const
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
std::vector<transaction> v;
for (auto& h : hlist)
{
v.push_back(get_tx(h));
}
return v;
}
uint64_t BlockchainBDB::get_tx_block_height(const crypto::hash& h) const
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
Dbt_copy<crypto::hash> key(h);
Dbt_copy<uint64_t> result;
auto get_result = m_tx_heights->get(DB_DEFAULT_TX, &key, &result, 0);
if (get_result == DB_NOTFOUND)
{
throw1(TX_DNE(std::string("tx height with hash ").append(epee::string_tools::pod_to_hex(h)).append(" not found in db").c_str()));
}
else if (get_result)
throw0(DB_ERROR("DB error attempting to fetch tx height from hash"));
return (uint64_t)result - 1;
}
uint64_t BlockchainBDB::get_num_outputs(const uint64_t& amount) const
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
bdb_cur cur(DB_DEFAULT_TX, m_output_amounts);
Dbt_copy<uint64_t> k(amount);
Dbt_copy<uint32_t> v;
auto result = cur->get(&k, &v, DB_SET);
if (result == DB_NOTFOUND)
{
return 0;
}
else if (result)
throw0(DB_ERROR("DB error attempting to get number of outputs of an amount"));
db_recno_t num_elems = 0;
cur->count(&num_elems, 0);
cur.close();
return num_elems;
}
output_data_t BlockchainBDB::get_output_key(const uint64_t& global_index) const
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
Dbt_copy<uint32_t> k(global_index + 1);
Dbt_copy<output_data_t> v;
auto get_result = m_output_keys->get(DB_DEFAULT_TX, &k, &v, 0);
if (get_result == DB_NOTFOUND)
throw1(OUTPUT_DNE("Attempting to get output pubkey by global index, but key does not exist"));
else if (get_result)
throw0(DB_ERROR("Error attempting to retrieve an output pubkey from the db"));
return v;
}
output_data_t BlockchainBDB::get_output_key(const uint64_t& amount, const uint64_t& index)
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
uint64_t glob_index = get_output_global_index(amount, index);
return get_output_key(glob_index);
}
// As this is not used, its return is now a blank output.
// This will save on space in the db.
tx_out BlockchainBDB::get_output(const crypto::hash& h, const uint64_t& index) const
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
return tx_out();
}
// As this is not used, its return is now a blank output.
// This will save on space in the db.
tx_out BlockchainBDB::get_output(const uint64_t& index) const
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
return tx_out();
}
tx_out_index BlockchainBDB::get_output_tx_and_index(const uint64_t& amount, const uint64_t& index)
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
std::vector < uint64_t > offsets;
std::vector<tx_out_index> indices;
offsets.push_back(index);
get_output_tx_and_index(amount, offsets, indices);
if (!indices.size())
throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found"));
return indices[0];
}
std::vector<uint64_t> BlockchainBDB::get_tx_output_indices(const crypto::hash& h) const
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
std::vector<uint64_t> index_vec;
bdb_cur cur(DB_DEFAULT_TX, m_tx_outputs);
Dbt_copy<crypto::hash> k(h);
Dbt_copy<uint32_t> v;
auto result = cur->get(&k, &v, DB_SET);
if (result == DB_NOTFOUND)
throw1(OUTPUT_DNE("Attempting to get an output by tx hash and tx index, but output not found"));
else if (result)
throw0(DB_ERROR("DB error attempting to get an output"));
db_recno_t num_elems = 0;
cur->count(&num_elems, 0);
for (uint64_t i = 0; i < num_elems; ++i)
{
index_vec.push_back(v);
cur->get(&k, &v, DB_NEXT_DUP);
}
cur.close();
return index_vec;
}
std::vector<uint64_t> BlockchainBDB::get_tx_amount_output_indices(const crypto::hash& h) const
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
std::vector<uint64_t> index_vec;
std::vector<uint64_t> index_vec2;
// get the transaction's global output indices first
index_vec = get_tx_output_indices(h);
// these are next used to obtain the amount output indices
transaction tx = get_tx(h);
uint64_t i = 0;
uint64_t global_index;
for (const auto& vout : tx.vout)
{
uint64_t amount = vout.amount;
global_index = index_vec[i];
bdb_cur cur(DB_DEFAULT_TX, m_output_amounts);
Dbt_copy<uint64_t> k(amount);
Dbt_copy<uint32_t> v;
auto result = cur->get(&k, &v, DB_SET);
if (result == DB_NOTFOUND)
throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found"));
else if (result)
throw0(DB_ERROR("DB error attempting to get an output"));
db_recno_t num_elems = 0;
cur->count(&num_elems, 0);
uint64_t amount_output_index = 0;
uint64_t output_index = 0;
bool found_index = false;
for (uint64_t j = 0; j < num_elems; ++j)
{
output_index = v;
if (output_index == global_index)
{
amount_output_index = j;
found_index = true;
break;
}
cur->get(&k, &v, DB_NEXT_DUP);
}
if (found_index)
{
index_vec2.push_back(amount_output_index);
}
else
{
// not found
cur.close();
throw1(OUTPUT_DNE("specified output not found in db"));
}
cur.close();
++i;
}
return index_vec2;
}
tx_out_index BlockchainBDB::get_output_tx_and_index_from_global(const uint64_t& index) const
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
Dbt_copy<uint32_t> k(index + 1);
Dbt_copy<crypto::hash > v;
auto get_result = m_output_txs->get(DB_DEFAULT_TX, &k, &v, 0);
if (get_result == DB_NOTFOUND)
throw1(OUTPUT_DNE("output with given index not in db"));
else if (get_result)
throw0(DB_ERROR("DB error attempting to fetch output tx hash"));
crypto::hash tx_hash = v;
Dbt_copy<uint64_t> result;
get_result = m_output_indices->get(DB_DEFAULT_TX, &k, &result, 0);
if (get_result == DB_NOTFOUND)
throw1(OUTPUT_DNE("output with given index not in db"));
else if (get_result)
throw0(DB_ERROR("DB error attempting to fetch output tx index"));
return tx_out_index(tx_hash, result);
}
bool BlockchainBDB::has_key_image(const crypto::key_image& img) const
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
Dbt_copy<crypto::key_image> val_key(img);
if (m_spent_keys->exists(DB_DEFAULT_TX, &val_key, 0) == 0)
{
return true;
}
return false;
}
// Ostensibly BerkeleyDB has batch transaction support built-in,
// so the following few functions will be NOP.
void BlockchainBDB::batch_start(uint64_t batch_num_blocks)
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
}
void BlockchainBDB::batch_commit()
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
}
void BlockchainBDB::batch_stop()
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
}
void BlockchainBDB::batch_abort()
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
}
void BlockchainBDB::set_batch_transactions(bool batch_transactions)
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
m_batch_transactions = batch_transactions;
LOG_PRINT_L3("batch transactions " << (m_batch_transactions ? "enabled" : "disabled"));
}
uint64_t BlockchainBDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const std::vector<transaction>& txs)
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
bdb_txn_safe txn;
if (m_env->txn_begin(NULL, txn, 0))
throw0(DB_ERROR("Failed to create a transaction for the db"));
m_write_txn = &txn;
uint64_t num_outputs = m_num_outputs;
try
{
BlockchainDB::add_block(blk, block_size, cumulative_difficulty, coins_generated, txs);
m_write_txn = NULL;
TIME_MEASURE_START(time1);
txn.commit();
TIME_MEASURE_FINISH(time1);
time_commit1 += time1;
}
catch (const std::exception& e)
{
m_num_outputs = num_outputs;
m_write_txn = NULL;
throw;
}
return ++m_height;
}
void BlockchainBDB::pop_block(block& blk, std::vector<transaction>& txs)
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
bdb_txn_safe txn;
if (m_env->txn_begin(NULL, txn, 0))
throw0(DB_ERROR("Failed to create a transaction for the db"));
m_write_txn = &txn;
uint64_t num_outputs = m_num_outputs;
try
{
BlockchainDB::pop_block(blk, txs);
m_write_txn = NULL;
txn.commit();
}
catch (...)
{
m_num_outputs = num_outputs;
m_write_txn = NULL;
throw;
}
--m_height;
}
void BlockchainBDB::get_output_tx_and_index_from_global(const std::vector<uint64_t> &global_indices, std::vector<tx_out_index> &tx_out_indices) const
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
tx_out_indices.clear();
for (const uint64_t &index : global_indices)
{
Dbt_copy<uint32_t> k(index);
Dbt_copy<crypto::hash> v;
auto get_result = m_output_txs->get(DB_DEFAULT_TX, &k, &v, 0);
if (get_result == DB_NOTFOUND)
throw1(OUTPUT_DNE("output with given index not in db"));
else if (get_result)
throw0(DB_ERROR("DB error attempting to fetch output tx hash"));
crypto::hash tx_hash = v;
Dbt_copy<uint64_t> result;
get_result = m_output_indices->get(DB_DEFAULT_TX, &k, &result, 0);
if (get_result == DB_NOTFOUND)
throw1(OUTPUT_DNE("output with given index not in db"));
else if (get_result)
throw0(DB_ERROR("DB error attempting to fetch output tx index"));
auto hashindex = tx_out_index(tx_hash, result);
tx_out_indices.push_back(hashindex);
}
}
void BlockchainBDB::get_output_global_indices(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<uint64_t> &global_indices)
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
TIME_MEASURE_START(txx);
check_open();
bdb_cur cur(DB_DEFAULT_TX, m_output_amounts);
uint64_t max = 0;
for (const uint64_t& index : offsets)
{
if (index > max)
max = index;
}
// get returned keypairs count
#define DB_COUNT_RECORDS(dbt, cnt) \
do { \
uint32_t *_p = (uint32_t *) ((uint8_t *)(dbt)->data + \
(dbt)->ulen - sizeof(uint32_t)); \
cnt = 0; \
while(*_p != (uint32_t) -1) { \
_p -= 2; \
++cnt; \
} \
} while(0); \
Dbt_copy<uint64_t> k(amount);
Dbt_copy<uint32_t> v;
uint64_t buflen = 0;
uint64_t t_dbmul = 0;
uint64_t t_dbscan = 0;
auto result = cur->get(&k, &v, DB_SET);
if (result == DB_NOTFOUND)
throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found"));
else if (result)
throw0(DB_ERROR("DB error attempting to get an output"));
db_recno_t num_elems = 0;
cur->count(&num_elems, 0);
if (max <= 1 && num_elems <= max)
throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found"));
TIME_MEASURE_START(db2);
if (max <= 1)
{
for (const uint64_t& index : offsets)
{
TIME_MEASURE_START(t_seek);
auto result = cur->get(&k, &v, DB_SET);
if (result == DB_NOTFOUND)
throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found"));
else if (result)
throw0(DB_ERROR("DB error attempting to get an output"));
for (uint64_t i = 0; i < index; ++i)
cur->get(&k, &v, DB_NEXT_DUP);
uint64_t glob_index = v;
LOG_PRINT_L3("L0->v: " << glob_index);
global_indices.push_back(glob_index);
TIME_MEASURE_FINISH(t_seek);
}
}
else
{
// setup a 256KB minimum buffer size
uint32_t pagesize = 256 * 1024;
// Retrieve only a suitable portion of the kvp data, up to somewhere near
// the maximum offset value being retrieved
buflen = (max + 1) * 4 * sizeof(uint64_t);
buflen = ((buflen / pagesize) + ((buflen % pagesize) > 0 ? 1 : 0)) * pagesize;
bool nomem = false;
Dbt data;
bool singlebuff = buflen <= m_buffer.get_buffer_size();
buflen = buflen < m_buffer.get_buffer_size() ? buflen : m_buffer.get_buffer_size();
bdb_safe_buffer_t::type buffer = nullptr;
bdb_safe_buffer_autolock<bdb_safe_buffer_t> lock(m_buffer, buffer);
data.set_data(buffer);
data.set_ulen(buflen);
data.set_size(buflen);
data.set_flags(DB_DBT_USERMEM);
uint32_t curcount = 0;
uint32_t blockstart = 0;
for (const uint64_t& index : offsets)
{
if (index >= num_elems)
{
LOG_PRINT_L1("Index: " << index << " Elems: " << num_elems << " partial results found for get_output_tx_and_index");
break;
}
// fixme! for whatever reason, the first call to DB_MULTIPLE | DB_SET does not
// retrieve the first value.
if (index <= 1 || nomem)
{
auto result = cur->get(&k, &v, DB_SET);
if (result == DB_NOTFOUND)
{
throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found"));
}
else if (result)
{
throw0(DB_ERROR("DB error attempting to get an output"));
}
for (uint64_t i = 0; i < index; ++i)
cur->get(&k, &v, DB_NEXT_DUP);
}
else
{
while (index >= curcount)
{
TIME_MEASURE_START(t_db1);
try
{
cur->get(&k, &data, DB_MULTIPLE | (curcount == 0 ? DB_SET : DB_NEXT_DUP));
blockstart = curcount;
int count = 0;
DB_COUNT_RECORDS((DBT * ) &data, count);
curcount += count;
}
catch (const std::exception &e)
{
cur.close();
throw0(DB_ERROR(std::string("Failed on DB_MULTIPLE: ").append(e.what()).c_str()));
}
TIME_MEASURE_FINISH(t_db1);
t_dbmul += t_db1;
if (singlebuff)
break;
}
LOG_PRINT_L3("Records returned: " << curcount << " Index: " << index);
TIME_MEASURE_START(t_db2);
DBT *pdata = (DBT *) &data;
uint8_t *value;
uint64_t dlen = 0;
void *pbase = ((uint8_t *) (pdata->data)) + pdata->ulen - sizeof(uint32_t);
uint32_t *p = (uint32_t *) pbase;
if (*p == (uint32_t) -1)
{
value = NULL;
}
else
{
p -= (index - blockstart) * 2; // index * 4 + 2; <- if DB_MULTIPLE_KEY
value = (uint8_t *) pdata->data + *p--;
dlen = *p--;
if (value == (uint8_t *) pdata->data)
value = NULL;
}
if (value != NULL)
{
v = dlen == sizeof(uint64_t) ? *((uint64_t *) value) : *((uint32_t *) value);
}
TIME_MEASURE_FINISH(t_db2);
t_dbscan += t_db2;
}
uint64_t glob_index = v;
LOG_PRINT_L3("L1->v: " << glob_index);
global_indices.push_back(glob_index);
}
}
TIME_MEASURE_FINISH(db2);
cur.close();
TIME_MEASURE_FINISH(txx);
LOG_PRINT_L3("blen: " << buflen << " txx: " << txx << " db1: " << t_dbmul << " db2: " << t_dbscan);
}
void BlockchainBDB::get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs)
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
TIME_MEASURE_START(txx);
outputs.clear();
std::vector < uint64_t > global_indices;
get_output_global_indices(amount, offsets, global_indices);
TIME_MEASURE_START(db3);
if (global_indices.size() > 0)
{
for (const uint64_t &index : global_indices)
{
Dbt_copy<uint32_t> k(index + 1);
Dbt_copy<output_data_t> v;
auto get_result = m_output_keys->get(DB_DEFAULT_TX, &k, &v, 0);
if (get_result == DB_NOTFOUND)
throw1(OUTPUT_DNE("output with given index not in db"));
else if (get_result)
throw0(DB_ERROR("DB error attempting to fetch output tx hash"));
output_data_t data = *(output_data_t *) v.get_data();
outputs.push_back(data);
}
}
TIME_MEASURE_FINISH(txx);
LOG_PRINT_L3("db3: " << db3);
}
void BlockchainBDB::get_output_tx_and_index(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<tx_out_index> &indices)
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
std::vector < uint64_t > global_indices;
get_output_global_indices(amount, offsets, global_indices);
TIME_MEASURE_START(db3);
if (global_indices.size() > 0)
get_output_tx_and_index_from_global(global_indices, indices);
TIME_MEASURE_FINISH(db3);
LOG_PRINT_L3("db3: " << db3);
}
void BlockchainBDB::set_hard_fork_starting_height(uint8_t version, uint64_t height)
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
Dbt_copy<uint32_t> val_key(version + 1);
Dbt_copy<uint64_t> val(height);
if (m_hf_starting_heights->put(DB_DEFAULT_TX, &val_key, &val, 0))
throw1(DB_ERROR("Error adding hard fork starting height to db transaction."));
}
uint64_t BlockchainBDB::get_hard_fork_starting_height(uint8_t version) const
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
Dbt_copy<uint32_t> key(version + 1);
Dbt_copy<uint64_t> result;
auto get_result = m_hf_starting_heights->get(DB_DEFAULT_TX, &key, &result, 0);
if (get_result == DB_NOTFOUND || get_result == DB_KEYEMPTY)
return std::numeric_limits<uint64_t>::max();
else if (get_result)
throw0(DB_ERROR("Error attempting to retrieve hard fork starting height from the db"));
return result;
}
void BlockchainBDB::set_hard_fork_version(uint64_t height, uint8_t version)
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
Dbt_copy<uint32_t> val_key(height + 1);
Dbt_copy<uint8_t> val(version);
if (m_hf_versions->put(DB_DEFAULT_TX, &val_key, &val, 0))
throw1(DB_ERROR("Error adding hard fork version to db transaction."));
}
uint8_t BlockchainBDB::get_hard_fork_version(uint64_t height) const
{
LOG_PRINT_L3("BlockchainBDB::" << __func__);
check_open();
Dbt_copy<uint32_t> key(height + 1);
Dbt_copy<uint8_t> result;
auto get_result = m_hf_versions->get(DB_DEFAULT_TX, &key, &result, 0);
if (get_result == DB_NOTFOUND || get_result == DB_KEYEMPTY)
throw0(OUTPUT_DNE("Error attempting to retrieve hard fork version from the db"));
else if (get_result)
throw0(DB_ERROR("Error attempting to retrieve hard fork version from the db"));
return result;
}
void BlockchainBDB::checkpoint_worker() const
{
LOG_PRINT_L0("Entering BDB checkpoint thread.")
int count = 0;
while(m_run_checkpoint && m_open)
{
// sleep every second, so we don't delay exit condition m_run_checkpoint = false
sleep(1);
// checkpoint every 5 minutes
if(count++ >= 300)
{
count = 0;
if(m_env->txn_checkpoint(0, 0, 0) != 0)
{
LOG_PRINT_L0("BDB txn_checkpoint failed.")
break;
}
}
}
LOG_PRINT_L0("Leaving BDB checkpoint thread.")
}
} // namespace cryptonote
Reference in New Issue View Git Blame Copy Permalink