Merge BerkeleyDB blockchain db implementation

CMakeLists.txt

@@ -177,6 +177,9 @@ include_directories(external/rapidjson)
 # Final setup for liblmdb
 include_directories(${LMDB_INCLUDE})
 
+# Final setup for Berkeley DB
+include_directories(${BDB_INCLUDE})
+
 if(MSVC)
   add_definitions("/bigobj /MP /W3 /GS- /D_CRT_SECURE_NO_WARNINGS /wd4996 /wd4345 /D_WIN32_WINNT=0x0600 /DWIN32_LEAN_AND_MEAN /DGTEST_HAS_TR1_TUPLE=0 /FIinline_c.h /D__SSE4_1__")
   # set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /Dinline=__inline")

cmake/FindBerkeleyDB.cmake

@@ -0,0 +1,25 @@
+# - Try to find Berkeley DB
+# Once done this will define
+#
+#  BERKELEY_DB_FOUND - system has Berkeley DB
+#  BERKELEY_DB_INCLUDE_DIR - the Berkeley DB include directory
+#  BERKELEY_DB_LIBRARIES - Link these to use Berkeley DB
+#  BERKELEY_DB_DEFINITIONS - Compiler switches required for using Berkeley DB
+
+# Copyright (c) 2006, Alexander Dymo, <adymo@kdevelop.org>
+#
+# Redistribution and use is allowed according to the terms of the BSD license.
+# For details see the accompanying COPYING-CMAKE-SCRIPTS file.
+
+find_path(BERKELEY_DB_INCLUDE_DIR db_cxx.h
+  /usr/include/db4
+  /usr/local/include/db4
+)
+
+find_library(BERKELEY_DB_LIBRARIES NAMES db_cxx )
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(Berkeley "Could not find Berkeley DB >= 4.1" BERKELEY_DB_INCLUDE_DIR BERKELEY_DB_LIBRARIES)
+# show the BERKELEY_DB_INCLUDE_DIR and BERKELEY_DB_LIBRARIES variables only in the advanced view
+mark_as_advanced(BERKELEY_DB_INCLUDE_DIR BERKELEY_DB_LIBRARIES )
+

external/db_drivers/CMakeLists.txt

@@ -33,3 +33,25 @@ add_subdirectory(liblmdb${ARCH_WIDTH})
 set(LMDB_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/liblmdb${ARCH_WIDTH}" CACHE STRING "LMDB Include path")
 
 set(LMDB_LIBRARY "lmdb" CACHE STRING "LMDB Library name")
+
+find_package(BerkeleyDB)
+
+if(NOT BERKELEY_DB_LIBRARIES OR STATIC)
+  add_subdirectory(libdb)
+  message(STATUS "BerkeleyDB not found, building from src tree")
+
+  set(BDB_STATIC true CACHE BOOL "BDB Static flag")
+  set(BDB_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/libdb" CACHE STRING "BDB include path")
+  set(BDB_LIBRARY "db" CACHE STRING "BDB library name")
+else()
+  message(STATUS "Found BerkeleyDB include (db.h) in ${BERKELEY_DB_INCLUDE_DIR}")
+  if(BERKELEY_DB_LIBRARIES)
+    message(STATUS "Found BerkeleyDB shared library")
+    set(BDB_STATIC false CACHE BOOL "BDB Static flag")
+    set(BDB_INCLUDE ${BERKELEY_DB_INCLUDE_DIR} CACHE STRING "BDB include path")
+    set(BDB_LIBRARY ${BERKELEY_DB_LIBRARIES} CACHE STRING "BDB library name")
+    set(BDB_LIBRARY_DIRS "" CACHE STRING "BDB Library dirs")
+  else()
+    die("Found BerkeleyDB includes, but could not find BerkeleyDB library. Please make sure you have installed libdb and libdb-dev or the equivalent")
+  endif()
+endif()

src/blockchain_converter/fake_core.h

@@ -29,8 +29,10 @@
 #pragma once
 
 #include <boost/filesystem.hpp>
-#include "cryptonote_core/blockchain.h" // BlockchainDB and LMDB
+#include "cryptonote_core/blockchain.h" // BlockchainDB
 #include "cryptonote_core/blockchain_storage.h" // in-memory DB
+#include "blockchain_db/blockchain_db.h"
+#include "blockchain_db/lmdb/db_lmdb.h"
 
 using namespace cryptonote;
 
@@ -53,7 +55,27 @@ struct fake_core_lmdb
 #endif
   {
     m_pool.init(path.string());
-    m_storage.init(path.string(), use_testnet, mdb_flags);
+
+    BlockchainDB* db = new BlockchainLMDB();
+
+    boost::filesystem::path folder(path);
+
+    folder /= db->get_db_name();
+
+    LOG_PRINT_L0("Loading blockchain from folder " << folder.c_str() << " ...");
+
+    const std::string filename = folder.string();
+    try
+    {
+      db->open(filename, mdb_flags);
+    }
+    catch (const std::exception& e)
+    {
+      LOG_PRINT_L0("Error opening database: " << e.what());
+      throw;
+    }
+
+    m_storage.init(db, use_testnet);
     if (do_batch)
       m_storage.get_db().set_batch_transactions(do_batch);
     support_batch = true;

src/blockchain_db/CMakeLists.txt

@@ -29,6 +29,7 @@
 set(blockchain_db_sources
   blockchain_db.cpp
   lmdb/db_lmdb.cpp
+  berkeleydb/db_bdb.cpp
   )
 
 set(blockchain_db_headers)
@@ -36,6 +37,7 @@ set(blockchain_db_headers)
 set(blockchain_db_private_headers
   blockchain_db.h
   lmdb/db_lmdb.h
+  berkeleydb/db_bdb.h
   )
 
 bitmonero_private_headers(blockchain_db
@@ -52,9 +54,10 @@ target_link_libraries(blockchain_db
     ${Boost_DATE_TIME_LIBRARY}
     ${Boost_PROGRAM_OPTIONS_LIBRARY}
     ${Boost_SERIALIZATION_LIBRARY}
+    ${LMDB_LIBRARY}
+    ${BDB_LIBRARY}
   LINK_PRIVATE
     ${Boost_FILESYSTEM_LIBRARY}
     ${Boost_SYSTEM_LIBRARY}
     ${Boost_THREAD_LIBRARY}
-    ${LMDB_LIBRARY}
     ${EXTRA_LIBRARIES})

src/blockchain_db/berkeleydb/db_bdb.h

@@ -0,0 +1,288 @@
+// Copyright (c) 2014, The Monero Project
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+// 
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+// 
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+// 
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <db_cxx.h>
+
+#include "blockchain_db/blockchain_db.h"
+#include "cryptonote_protocol/blobdatatype.h" // for type blobdata
+
+namespace cryptonote
+{
+
+struct bdb_txn_safe
+{
+  bdb_txn_safe() : m_txn(NULL) { }
+  ~bdb_txn_safe()
+  {
+    LOG_PRINT_L3("bdb_txn_safe: destructor");
+
+    if (m_txn != NULL)
+      abort();
+  }
+
+  void commit(std::string message = "")
+  {
+    if (message.size() == 0)
+    {
+      message = "Failed to commit a transaction to the db";
+    }
+
+    if (m_txn->commit(0))
+    {
+      m_txn = NULL;
+      LOG_PRINT_L0(message);
+      throw DB_ERROR(message.c_str());
+    }
+    m_txn = NULL;
+  }
+
+  void abort()
+  {
+    LOG_PRINT_L3("bdb_txn_safe: abort()");
+    if(m_txn != NULL)
+    {
+      m_txn->abort();
+      m_txn = NULL;
+    }
+    else
+    {
+      LOG_PRINT_L0("WARNING: bdb_txn_safe: abort() called, but m_txn is NULL");
+    }
+  }
+
+  operator DbTxn*()
+  {
+    return m_txn;
+  }
+
+  operator DbTxn**()
+  {
+    return &m_txn;
+  }
+
+  DbTxn* m_txn;
+};
+
+class BlockchainBDB : public BlockchainDB
+{
+public:
+  BlockchainBDB(bool batch_transactions=false);
+  ~BlockchainBDB();
+
+  virtual void open(const std::string& filename, const int db_flags);
+
+  virtual void close();
+
+  virtual void sync();
+
+  virtual void reset();
+
+  virtual std::vector<std::string> get_filenames() const;
+
+  virtual std::string get_db_name() const;
+
+  virtual bool lock();
+
+  virtual void unlock();
+
+  virtual bool block_exists(const crypto::hash& h) const;
+
+  virtual block get_block(const crypto::hash& h) const;
+
+  virtual uint64_t get_block_height(const crypto::hash& h) const;
+
+  virtual block_header get_block_header(const crypto::hash& h) const;
+
+  virtual block get_block_from_height(const uint64_t& height) const;
+
+  virtual uint64_t get_block_timestamp(const uint64_t& height) const;
+
+  virtual uint64_t get_top_block_timestamp() const;
+
+  virtual size_t get_block_size(const uint64_t& height) const;
+
+  virtual difficulty_type get_block_cumulative_difficulty(const uint64_t& height) const;
+
+  virtual difficulty_type get_block_difficulty(const uint64_t& height) const;
+
+  virtual uint64_t get_block_already_generated_coins(const uint64_t& height) const;
+
+  virtual crypto::hash get_block_hash_from_height(const uint64_t& height) const;
+
+  virtual std::vector<block> get_blocks_range(const uint64_t& h1, const uint64_t& h2) const;
+
+  virtual std::vector<crypto::hash> get_hashes_range(const uint64_t& h1, const uint64_t& h2) const;
+
+  virtual crypto::hash top_block_hash() const;
+
+  virtual block get_top_block() const;
+
+  virtual uint64_t height() const;
+
+  virtual bool tx_exists(const crypto::hash& h) const;
+
+  virtual uint64_t get_tx_unlock_time(const crypto::hash& h) const;
+
+  virtual transaction get_tx(const crypto::hash& h) const;
+
+  virtual uint64_t get_tx_count() const;
+
+  virtual std::vector<transaction> get_tx_list(const std::vector<crypto::hash>& hlist) const;
+
+  virtual uint64_t get_tx_block_height(const crypto::hash& h) const;
+
+  virtual uint64_t get_random_output(const uint64_t& amount) const;
+
+  virtual uint64_t get_num_outputs(const uint64_t& amount) const;
+
+  virtual crypto::public_key get_output_key(const uint64_t& amount, const uint64_t& index) const;
+
+  virtual tx_out get_output(const crypto::hash& h, const uint64_t& index) const;
+
+  /**
+   * @brief get an output from its global index
+   *
+   * @param index global index of the output desired
+   *
+   * @return the output associated with the index.
+   * Will throw OUTPUT_DNE if not output has that global index.
+   * Will throw DB_ERROR if there is a non-specific LMDB error in fetching
+   */
+  tx_out get_output(const uint64_t& index) const;
+
+  virtual tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const;
+
+  virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const;
+
+  virtual std::vector<uint64_t> get_tx_output_indices(const crypto::hash& h) const;
+  virtual std::vector<uint64_t> get_tx_amount_output_indices(const crypto::hash& h) const;
+
+  virtual bool has_key_image(const crypto::key_image& img) const;
+
+  virtual uint64_t add_block( const block& blk
+                            , const size_t& block_size
+                            , const difficulty_type& cumulative_difficulty
+                            , const uint64_t& coins_generated
+                            , const std::vector<transaction>& txs
+                            );
+
+  virtual void set_batch_transactions(bool batch_transactions);
+  virtual void batch_start();
+  virtual void batch_commit();
+  virtual void batch_stop();
+  virtual void batch_abort();
+
+  virtual void pop_block(block& blk, std::vector<transaction>& txs);
+
+private:
+  virtual void add_block( const block& blk
+                , const size_t& block_size
+                , const difficulty_type& cumulative_difficulty
+                , const uint64_t& coins_generated
+                , const crypto::hash& block_hash
+                );
+
+  virtual void remove_block();
+
+  virtual void add_transaction_data(const crypto::hash& blk_hash, const transaction& tx, const crypto::hash& tx_hash);
+
+  virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx);
+
+  virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index);
+
+  virtual void remove_output(const tx_out& tx_output);
+
+  void remove_tx_outputs(const crypto::hash& tx_hash, const transaction& tx);
+
+  void remove_output(const uint64_t& out_index, const uint64_t amount);
+  void remove_amount_output_index(const uint64_t amount, const uint64_t global_output_index);
+
+  virtual void add_spent_key(const crypto::key_image& k_image);
+
+  virtual void remove_spent_key(const crypto::key_image& k_image);
+
+  /**
+   * @brief convert a tx output to a blob for storage
+   *
+   * @param output the output to convert
+   *
+   * @return the resultant blob
+   */
+  blobdata output_to_blob(const tx_out& output);
+
+  /**
+   * @brief convert a tx output blob to a tx output
+   *
+   * @param blob the blob to convert
+   *
+   * @return the resultant tx output
+   */
+  tx_out output_from_blob(const blobdata& blob) const;
+
+  /**
+   * @brief get the global index of the index-th output of the given amount
+   *
+   * @param amount the output amount
+   * @param index the index into the set of outputs of that amount
+   *
+   * @return the global index of the desired output
+   */
+  uint64_t get_output_global_index(const uint64_t& amount, const uint64_t& index) const;
+
+  void check_open() const;
+
+  DbEnv* m_env;
+
+  Db* m_blocks;
+  Db* m_block_heights;
+  Db* m_block_hashes;
+  Db* m_block_timestamps;
+  Db* m_block_sizes;
+  Db* m_block_diffs;
+  Db* m_block_coins;
+
+  Db* m_txs;
+  Db* m_tx_unlocks;
+  Db* m_tx_heights;
+  Db* m_tx_outputs;
+
+  Db* m_output_txs;
+  Db* m_output_indices;
+  Db* m_output_amounts;
+  Db* m_output_keys;
+
+  Db* m_spent_keys;
+
+  uint64_t m_height;
+  uint64_t m_num_outputs;
+  std::string m_folder;
+  bdb_txn_safe *m_write_txn;
+
+  bool m_batch_transactions; // support for batch transactions
+};
+
+}  // namespace cryptonote

src/blockchain_db/blockchain_db.cpp

@@ -129,6 +129,11 @@ void BlockchainDB::pop_block(block& blk, std::vector<transaction>& txs)
   }
 }
 
+bool BlockchainDB::is_open()
+{
+  return m_open;
+}
+
 void BlockchainDB::remove_transaction(const crypto::hash& tx_hash)
 {
   transaction tx = get_tx(tx_hash);

src/blockchain_db/blockchain_db.h

@@ -62,6 +62,7 @@
  *
  * General:
  *   open()
+ *   is_open()
  *   close()
  *   sync()
  *   reset()
@@ -328,8 +329,8 @@ public:
   // open the db at location <filename>, or create it if there isn't one.
   virtual void open(const std::string& filename, const int db_flags = 0) = 0;
 
-  // make sure implementation has a create function as well
-  virtual void create(const std::string& filename) = 0;
+  // returns true of the db is open/ready, else false
+  bool is_open();
 
   // close and sync the db
   virtual void close() = 0;
@@ -482,6 +483,7 @@ public:
   // returns true if key image <img> is present in spent key images storage
   virtual bool has_key_image(const crypto::key_image& img) const = 0;
 
+  bool m_open;
 };  // class BlockchainDB
 
 

src/blockchain_db/db_types.h

@@ -0,0 +1,40 @@
+// Copyright (c) 2015, The Monero Project
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
+#pragma once
+
+namespace cryptonote
+{
+
+  const std::unordered_set<std::string> blockchain_db_types = 
+  { "lmdb"
+  , "berkeley"
+  };
+
+} // namespace cryptonote

src/blockchain_db/lmdb/db_lmdb.cpp

@@ -567,7 +567,7 @@ uint64_t BlockchainLMDB::get_output_global_index(const uint64_t& amount, const u
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
 
-  txn_safe txn;
+  mdb_txn_safe txn;
   if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
     throw0(DB_ERROR("Failed to create a transaction for the db"));
 
@@ -679,7 +679,7 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags)
     throw0(DB_ERROR(std::string("Failed to open lmdb environment: ").append(mdb_strerror(result)).c_str()));
 
   // get a read/write MDB_txn
-  txn_safe txn;
+  mdb_txn_safe txn;
   if (mdb_txn_begin(m_env, NULL, 0, txn))
     throw0(DB_ERROR("Failed to create a transaction for the db"));
 
@@ -733,13 +733,6 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags)
   // from here, init should be finished
 }
 
-// unused for now, create will happen on open if doesn't exist
-void BlockchainLMDB::create(const std::string& filename)
-{
-  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
-  throw DB_CREATE_FAILURE("create() is not implemented for this BlockchainDB, open() will create files if needed.");
-}
-
 void BlockchainLMDB::close()
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
@@ -816,7 +809,7 @@ bool BlockchainLMDB::block_exists(const crypto::hash& h) const
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
 
-  txn_safe txn;
+  mdb_txn_safe txn;
   if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
     throw0(DB_ERROR("Failed to create a transaction for the db"));
 
@@ -849,7 +842,7 @@ uint64_t BlockchainLMDB::get_block_height(const crypto::hash& h) const
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
 
-  txn_safe txn;
+  mdb_txn_safe txn;
   if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
     throw0(DB_ERROR("Failed to create a transaction for the db"));
 
@@ -879,7 +872,7 @@ block BlockchainLMDB::get_block_from_height(const uint64_t& height) const
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
 
-  txn_safe txn;
+  mdb_txn_safe txn;
   if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
     throw0(DB_ERROR("Failed to create a transaction for the db"));
 
@@ -910,8 +903,8 @@ uint64_t BlockchainLMDB::get_block_timestamp(const uint64_t& height) const
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
 
-  txn_safe txn;
-  txn_safe* txn_ptr = &txn;
+  mdb_txn_safe txn;
+  mdb_txn_safe* txn_ptr = &txn;
   if (m_batch_active)
     txn_ptr = m_write_txn;
   else
@@ -953,8 +946,8 @@ size_t BlockchainLMDB::get_block_size(const uint64_t& height) const
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
 
-  txn_safe txn;
-  txn_safe* txn_ptr = &txn;
+  mdb_txn_safe txn;
+  mdb_txn_safe* txn_ptr = &txn;
   if (m_batch_active)
     txn_ptr = m_write_txn;
   else
@@ -983,8 +976,8 @@ difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t&
   LOG_PRINT_L3("BlockchainLMDB::" << __func__ << "  height: " << height);
   check_open();
 
-  txn_safe txn;
-  txn_safe* txn_ptr = &txn;
+  mdb_txn_safe txn;
+  mdb_txn_safe* txn_ptr = &txn;
   if (m_batch_active)
     txn_ptr = m_write_txn;
   else
@@ -1029,8 +1022,8 @@ uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& heigh
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
 
-  txn_safe txn;
-  txn_safe* txn_ptr = &txn;
+  mdb_txn_safe txn;
+  mdb_txn_safe* txn_ptr = &txn;
   if (m_batch_active)
     txn_ptr = m_write_txn;
   else
@@ -1059,8 +1052,8 @@ crypto::hash BlockchainLMDB::get_block_hash_from_height(const uint64_t& height)
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
 
-  txn_safe txn;
-  txn_safe* txn_ptr = &txn;
+  mdb_txn_safe txn;
+  mdb_txn_safe* txn_ptr = &txn;
   if (m_batch_active)
     txn_ptr = m_write_txn;
   else
@@ -1152,8 +1145,8 @@ bool BlockchainLMDB::tx_exists(const crypto::hash& h) const
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
 
-  txn_safe txn;
-  txn_safe* txn_ptr = &txn;
+  mdb_txn_safe txn;
+  mdb_txn_safe* txn_ptr = &txn;
   if (m_batch_active)
     txn_ptr = m_write_txn;
   else
@@ -1187,7 +1180,7 @@ uint64_t BlockchainLMDB::get_tx_unlock_time(const crypto::hash& h) const
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
 
-  txn_safe txn;
+  mdb_txn_safe txn;
   if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
     throw0(DB_ERROR("Failed to create a transaction for the db"));
 
@@ -1207,8 +1200,8 @@ transaction BlockchainLMDB::get_tx(const crypto::hash& h) const
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
 
-  txn_safe txn;
-  txn_safe* txn_ptr = &txn;
+  mdb_txn_safe txn;
+  mdb_txn_safe* txn_ptr = &txn;
   if (m_batch_active)
     txn_ptr = m_write_txn;
   else
@@ -1242,7 +1235,7 @@ uint64_t BlockchainLMDB::get_tx_count() const
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
 
-  txn_safe txn;
+  mdb_txn_safe txn;
   if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
     throw0(DB_ERROR("Failed to create a transaction for the db"));
 
@@ -1274,8 +1267,8 @@ uint64_t BlockchainLMDB::get_tx_block_height(const crypto::hash& h) const
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
 
-  txn_safe txn;
-  txn_safe* txn_ptr = &txn;
+  mdb_txn_safe txn;
+  mdb_txn_safe* txn_ptr = &txn;
   if (m_batch_active)
     txn_ptr = m_write_txn;
   else
@@ -1318,7 +1311,7 @@ uint64_t BlockchainLMDB::get_num_outputs(const uint64_t& amount) const
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
 
-  txn_safe txn;
+  mdb_txn_safe txn;
   if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
     throw0(DB_ERROR("Failed to create a transaction for the db"));
 
@@ -1349,7 +1342,7 @@ crypto::public_key BlockchainLMDB::get_output_key(const uint64_t& amount, const
 
   uint64_t glob_index = get_output_global_index(amount, index);
 
-  txn_safe txn;
+  mdb_txn_safe txn;
   if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
     throw0(DB_ERROR("Failed to create a transaction for the db"));
 
@@ -1369,7 +1362,7 @@ tx_out BlockchainLMDB::get_output(const crypto::hash& h, const uint64_t& index)
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
 
-  txn_safe txn;
+  mdb_txn_safe txn;
   if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
     throw0(DB_ERROR("Failed to create a transaction for the db"));
 
@@ -1414,7 +1407,7 @@ tx_out BlockchainLMDB::get_output(const uint64_t& index) const
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
 
-  txn_safe txn;
+  mdb_txn_safe txn;
   if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
     throw0(DB_ERROR("Failed to create a transaction for the db"));
 
@@ -1438,8 +1431,8 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index_from_global(const uint64_t&
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
 
-  txn_safe txn;
-  txn_safe* txn_ptr = &txn;
+  mdb_txn_safe txn;
+  mdb_txn_safe* txn_ptr = &txn;
   if (m_batch_active)
     txn_ptr = m_write_txn;
   else
@@ -1474,8 +1467,8 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, con
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
 
-  txn_safe txn;
-  txn_safe* txn_ptr = &txn;
+  mdb_txn_safe txn;
+  mdb_txn_safe* txn_ptr = &txn;
   if (m_batch_active)
     txn_ptr = m_write_txn;
   else
@@ -1524,7 +1517,7 @@ std::vector<uint64_t> BlockchainLMDB::get_tx_output_indices(const crypto::hash&
   check_open();
   std::vector<uint64_t> index_vec;
 
-  txn_safe txn;
+  mdb_txn_safe txn;
   if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
     throw0(DB_ERROR("Failed to create a transaction for the db"));
 
@@ -1569,7 +1562,7 @@ std::vector<uint64_t> BlockchainLMDB::get_tx_amount_output_indices(const crypto:
 
   transaction tx = get_tx(h);
 
-  txn_safe txn;
+  mdb_txn_safe txn;
   if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
     throw0(DB_ERROR("Failed to create a transaction for the db"));
 
@@ -1640,7 +1633,7 @@ bool BlockchainLMDB::has_key_image(const crypto::key_image& img) const
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
 
-  txn_safe txn;
+  mdb_txn_safe txn;
   if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
     throw0(DB_ERROR("Failed to create a transaction for the db"));
 
@@ -1751,7 +1744,7 @@ uint64_t BlockchainLMDB::add_block( const block& blk
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
 
-  txn_safe txn;
+  mdb_txn_safe txn;
   if (! m_batch_active)
   {
     if (mdb_txn_begin(m_env, NULL, 0, txn))
@@ -1789,7 +1782,7 @@ void BlockchainLMDB::pop_block(block& blk, std::vector<transaction>& txs)
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
 
-  txn_safe txn;
+  mdb_txn_safe txn;
   if (! m_batch_active)
   {
     if (mdb_txn_begin(m_env, NULL, 0, txn))

src/blockchain_db/lmdb/db_lmdb.h

@@ -24,6 +24,7 @@
 // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 // STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
 // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#pragma once
 
 #include "blockchain_db/blockchain_db.h"
 #include "cryptonote_protocol/blobdatatype.h" // for type blobdata
@@ -33,17 +34,17 @@
 namespace cryptonote
 {
 
-struct txn_safe
+struct mdb_txn_safe
 {
-  txn_safe() : m_txn(NULL) { }
-  ~txn_safe()
+  mdb_txn_safe() : m_txn(NULL) { }
+  ~mdb_txn_safe()
   {
-    LOG_PRINT_L3("txn_safe: destructor");
+    LOG_PRINT_L3("mdb_txn_safe: destructor");
     if (m_txn != NULL)
     {
       if (m_batch_txn) // this is a batch txn and should have been handled before this point for safety
       {
-        LOG_PRINT_L0("WARNING: txn_safe: m_txn is a batch txn and it's not NULL in destructor - calling mdb_txn_abort()");
+        LOG_PRINT_L0("WARNING: mdb_txn_safe: m_txn is a batch txn and it's not NULL in destructor - calling mdb_txn_abort()");
       }
       else
       {
@@ -53,7 +54,7 @@ struct txn_safe
         //
         // NOTE: not sure if this is ever reached for a non-batch write
         // transaction, but it's probably not ideal if it did.
-        LOG_PRINT_L3("txn_safe: m_txn not NULL in destructor - calling mdb_txn_abort()");
+        LOG_PRINT_L3("mdb_txn_safe: m_txn not NULL in destructor - calling mdb_txn_abort()");
       }
       mdb_txn_abort(m_txn);
     }
@@ -77,11 +78,11 @@ struct txn_safe
 
   // This should only be needed for batch transaction which must be ensured to
   // be aborted before mdb_env_close, not after. So we can't rely on
-  // BlockchainLMDB destructor to call txn_safe destructor, as that's too late
+  // BlockchainLMDB destructor to call mdb_txn_safe destructor, as that's too late
   // to properly abort, since mdb_env_close would have been called earlier.
   void abort()
   {
-    LOG_PRINT_L3("txn_safe: abort()");
+    LOG_PRINT_L3("mdb_txn_safe: abort()");
     if(m_txn != NULL)
     {
       mdb_txn_abort(m_txn);
@@ -89,7 +90,7 @@ struct txn_safe
     }
     else
     {
-      LOG_PRINT_L0("WARNING: txn_safe: abort() called, but m_txn is NULL");
+      LOG_PRINT_L0("WARNING: mdb_txn_safe: abort() called, but m_txn is NULL");
     }
   }
 
@@ -116,8 +117,6 @@ public:
 
   virtual void open(const std::string& filename, const int mdb_flags=0);
 
-  virtual void create(const std::string& filename);
-
   virtual void close();
 
   virtual void sync();
@@ -302,12 +301,11 @@ private:
 
   MDB_dbi m_spent_keys;
 
-  bool m_open;
   uint64_t m_height;
   uint64_t m_num_outputs;
   std::string m_folder;
-  txn_safe* m_write_txn; // may point to either a short-lived txn or a batch txn
-  txn_safe m_write_batch_txn; // persist batch txn outside of BlockchainLMDB
+  mdb_txn_safe* m_write_txn; // may point to either a short-lived txn or a batch txn
+  mdb_txn_safe m_write_batch_txn; // persist batch txn outside of BlockchainLMDB
 
   bool m_batch_transactions; // support for batch transactions
   bool m_batch_active; // whether batch transaction is in progress

src/cryptonote_core/blockchain.cpp

@@ -39,7 +39,6 @@
 #include "tx_pool.h"
 #include "blockchain.h"
 #include "blockchain_db/blockchain_db.h"
-#include "blockchain_db/lmdb/db_lmdb.h"
 #include "cryptonote_format_utils.h"
 #include "cryptonote_boost_serialization.h"
 #include "cryptonote_config.h"
@@ -65,8 +64,7 @@ using epee::string_tools::pod_to_hex;
 DISABLE_VS_WARNINGS(4267)
 
 //------------------------------------------------------------------
-// TODO: initialize m_db with a concrete implementation of BlockchainDB
-Blockchain::Blockchain(tx_memory_pool& tx_pool):m_db(), m_tx_pool(tx_pool), m_current_block_cumul_sz_limit(0), m_is_in_checkpoint_zone(false), m_is_blockchain_storing(false), m_testnet(false), m_enforce_dns_checkpoints(false)
+Blockchain::Blockchain(tx_memory_pool& tx_pool):m_db(), m_tx_pool(tx_pool), m_current_block_cumul_sz_limit(0), m_is_in_checkpoint_zone(false), m_is_blockchain_storing(false), m_enforce_dns_checkpoints(false)
 {
   LOG_PRINT_L3("Blockchain::" << __func__);
 }
@@ -226,43 +224,24 @@ uint64_t Blockchain::get_current_blockchain_height() const
 //------------------------------------------------------------------
 //FIXME: possibly move this into the constructor, to avoid accidentally
 //       dereferencing a null BlockchainDB pointer
-bool Blockchain::init(const std::string& config_folder, const bool testnet, const int db_flags)
+bool Blockchain::init(BlockchainDB* db, const bool testnet)
 {
   LOG_PRINT_L3("Blockchain::" << __func__);
   CRITICAL_REGION_LOCAL(m_blockchain_lock);
 
-  // TODO: make this configurable
-  m_db = new BlockchainLMDB();
-
-  m_config_folder = config_folder;
-  m_testnet = testnet;
-
-  boost::filesystem::path folder(m_config_folder);
-
-  folder /= m_db->get_db_name();
-
-  LOG_PRINT_L0("Loading blockchain from folder " << folder.c_str() << " ...");
-
-  const std::string filename = folder.string();
-  try
+  if (db == nullptr)
   {
-    m_db->open(filename, db_flags);
-  }
-  catch (const DB_OPEN_FAILURE& e)
-  {
-    LOG_PRINT_L0("No blockchain file found, attempting to create one.");
-    try
-    {
-      m_db->create(filename);
-    }
-    catch (const DB_CREATE_FAILURE& db_create_error)
-    {
-      LOG_PRINT_L0("Unable to create BlockchainDB!  This is not good...");
-      //TODO: make sure whatever calls this handles the return value properly
+    LOG_ERROR("Attempted to init Blockchain with null DB");
     return false;
   }
+  if (!db->is_open())
+  {
+    LOG_ERROR("Attempted to init Blockchain with unopened DB");
+    return false;
   }
 
+  m_db = db;
+
   // if the blockchain is new, add the genesis block
   // this feels kinda kludgy to do it this way, but can be looked at later.
   // TODO: add function to create and store genesis block,

src/cryptonote_core/blockchain.h

@@ -81,7 +81,7 @@ namespace cryptonote
 
     Blockchain(tx_memory_pool& tx_pool);
 
-    bool init(const std::string& config_folder, const bool testnet = false, const int db_flags = 0);
+    bool init(BlockchainDB* db, const bool testnet = false);
     bool deinit();
 
     void set_checkpoints(checkpoints&& chk_pts) { m_checkpoints = chk_pts; }
@@ -180,11 +180,9 @@ namespace cryptonote
     outputs_container m_outputs;
 
 
-    std::string m_config_folder;
     checkpoints m_checkpoints;
     std::atomic<bool> m_is_in_checkpoint_zone;
     std::atomic<bool> m_is_blockchain_storing;
-    bool m_testnet;
     bool m_enforce_dns_checkpoints;
 
     bool switch_to_alternative_blockchain(std::list<blocks_ext_by_hash::iterator>& alt_chain, bool discard_disconnected_chain);

src/cryptonote_core/cryptonote_core.cpp

@@ -44,6 +44,9 @@ using namespace epee;
 #include <csignal>
 #include "daemon/command_line_args.h"
 #include "cryptonote_core/checkpoints_create.h"
+#include "blockchain_db/blockchain_db.h"
+#include "blockchain_db/lmdb/db_lmdb.h"
+#include "blockchain_db/berkeleydb/db_bdb.h"
 
 DISABLE_VS_WARNINGS(4355)
 
@@ -194,7 +197,45 @@ namespace cryptonote
     r = m_mempool.init(m_config_folder);
     CHECK_AND_ASSERT_MES(r, false, "Failed to initialize memory pool");
 
+#if BLOCKCHAIN_DB == DB_LMDB
+    std::string db_type = command_line::get_arg(vm, daemon_args::arg_db_type);
+
+    BlockchainDB* db = nullptr;
+    if (db_type == "lmdb")
+    {
+      db = new BlockchainLMDB();
+    }
+    else if (db_type == "berkeley")
+    {
+      db = new BlockchainBDB();
+    }
+    else
+    {
+      LOG_ERROR("Attempted to use non-existant database type");
+      return false;
+    }
+
+    boost::filesystem::path folder(m_config_folder);
+
+    folder /= db->get_db_name();
+
+    LOG_PRINT_L0("Loading blockchain from folder " << folder.c_str() << " ...");
+
+    const std::string filename = folder.string();
+    try
+    {
+      db->open(filename);
+    }
+    catch (const DB_ERROR& e)
+    {
+      LOG_PRINT_L0("Error opening database: " << e.what());
+      return false;
+    }
+
+    r = m_blockchain_storage.init(db, m_testnet);
+#else
     r = m_blockchain_storage.init(m_config_folder, m_testnet);
+#endif
     CHECK_AND_ASSERT_MES(r, false, "Failed to initialize blockchain storage");
 
     // load json & DNS checkpoints, and verify them

src/daemon/command_line_args.h

@@ -70,6 +70,11 @@ namespace daemon_args
   , "checkpoints from DNS server will be enforced"
   , false
   };
+  const command_line::arg_descriptor<std::string> arg_db_type = {
+    "db-type"
+  , "Specify database type"
+  , "lmdb"
+  };
 
 }  // namespace daemon_args
 

src/daemon/main.cpp

@@ -42,6 +42,7 @@
 #include "rpc/core_rpc_server.h"
 #include <boost/program_options.hpp>
 #include "daemon/command_line_args.h"
+#include "blockchain_db/db_types.h"
 
 namespace po = boost::program_options;
 namespace bf = boost::filesystem;
@@ -78,6 +79,7 @@ int main(int argc, char const * argv[])
       command_line::add_arg(core_settings, daemon_args::arg_log_level);
       command_line::add_arg(core_settings, daemon_args::arg_testnet_on);
       command_line::add_arg(core_settings, daemon_args::arg_dns_checkpoints);
+      command_line::add_arg(core_settings, daemon_args::arg_db_type);
       daemonizer::init_options(hidden_options, visible_options);
       daemonize::t_executor::init_options(core_settings);
 
@@ -128,6 +130,19 @@ int main(int argc, char const * argv[])
       return 0;
     }
 
+    std::string db_type = command_line::get_arg(vm, daemon_args::arg_db_type);
+
+    // verify that blockchaindb type is valid
+    if(cryptonote::blockchain_db_types.count(db_type) == 0)
+    {
+      std::cout << "Invalid database type (" << db_type << "), available types are:" << std::endl;
+      for (const auto& type : cryptonote::blockchain_db_types)
+      {
+        std::cout << "\t" << type << std::endl;
+      }
+      return 0;
+    }
+
     bool testnet_mode = command_line::get_arg(vm, daemon_args::arg_testnet_on);
 
     auto data_dir_arg = testnet_mode ? command_line::arg_testnet_data_dir : command_line::arg_data_dir;

tests/unit_tests/BlockchainDB.cpp

@@ -30,11 +30,14 @@
 #include <boost/algorithm/string/predicate.hpp>
 #include <cstdio>
 #include <iostream>
+#include <chrono>
+#include <thread>
 
 #include "gtest/gtest.h"
 
 #include "blockchain_db/blockchain_db.h"
 #include "blockchain_db/lmdb/db_lmdb.h"
+#include "blockchain_db/berkeleydb/db_bdb.h"
 #include "cryptonote_core/cryptonote_format_utils.h"
 
 using namespace cryptonote;
@@ -87,6 +90,7 @@ bool compare_blocks(const block& a, const block& b)
   return hash_a == hash_b;
 }
 
+/*
 void print_block(const block& blk, const std::string& prefix = "")
 {
   std::cerr << prefix << ": " << std::endl
@@ -105,6 +109,7 @@ bool compare_txs(const transaction& a, const transaction& b)
 
   return ab == bb;
 }
+*/
 
 // convert hex string to string that has values based on that hex
 // thankfully should automatically ignore null-terminator.
@@ -172,20 +177,29 @@ protected:
   }
 
   ~BlockchainDBTest() {
-    auto files = m_db->get_filenames();
     delete m_db;
-    remove_files(files);
+    remove_files();
   }
 
   BlockchainDB* m_db;
   std::string m_prefix;
   std::vector<block> m_blocks;
   std::vector<std::vector<transaction> > m_txs;
+  std::vector<std::string> m_filenames;
 
-  void remove_files(const std::vector<std::string>& files)
+  void get_filenames()
+  {
+    m_filenames = m_db->get_filenames();
+    for (auto& f : m_filenames)
+    {
+      std::cerr << "File created by test: " << f << std::endl;
+    }
+  }
+
+  void remove_files()
   {
     // remove each file the db created, making sure it starts with fname.
-    for (auto& f : files)
+    for (auto& f : m_filenames)
     {
       if (boost::starts_with(f, m_prefix))
       {
@@ -198,7 +212,7 @@ protected:
     }
 
     // remove directory if it still exists
-    boost::filesystem::remove(m_prefix);
+    boost::filesystem::remove_all(m_prefix);
   }
 
   void set_prefix(const std::string& prefix)
@@ -209,7 +223,7 @@ protected:
 
 using testing::Types;
 
-typedef Types<BlockchainLMDB> implementations;
+typedef Types<BlockchainLMDB, BlockchainBDB> implementations;
 
 TYPED_TEST_CASE(BlockchainDBTest, implementations);
 
@@ -221,6 +235,7 @@ TYPED_TEST(BlockchainDBTest, OpenAndClose)
 
   // make sure open does not throw
   ASSERT_NO_THROW(this->m_db->open(fname));
+  this->get_filenames();
 
   // make sure open when already open DOES throw
   ASSERT_THROW(this->m_db->open(fname), DB_OPEN_FAILURE);
@@ -231,9 +246,11 @@ TYPED_TEST(BlockchainDBTest, OpenAndClose)
 TYPED_TEST(BlockchainDBTest, AddBlock)
 {
   std::string fname(tmpnam(NULL));
+  this->set_prefix(fname);
 
   // make sure open does not throw
   ASSERT_NO_THROW(this->m_db->open(fname));
+  this->get_filenames();
 
   // adding a block with no parent in the blockchain should throw.
   // note: this shouldn't be possible, but is a good (and cheap) failsafe.
@@ -272,9 +289,11 @@ TYPED_TEST(BlockchainDBTest, AddBlock)
 TYPED_TEST(BlockchainDBTest, RetrieveBlockData)
 {
   std::string fname(tmpnam(NULL));
+  this->set_prefix(fname);
 
   // make sure open does not throw
   ASSERT_NO_THROW(this->m_db->open(fname));
+  this->get_filenames();
 
   ASSERT_NO_THROW(this->m_db->add_block(this->m_blocks[0], t_sizes[0], t_diffs[0], t_coins[0], this->m_txs[0]));
 

tests/unit_tests/CMakeLists.txt

@@ -57,6 +57,7 @@ add_executable(unit_tests
 target_link_libraries(unit_tests
   LINK_PRIVATE
     cryptonote_core
+    blockchain_db
     rpc
     wallet
     ${GTEST_MAIN_LIBRARIES}