Merge pull request #129 from wowario/upstream

Upstream

cmake/CheckTrezor.cmake

@@ -56,7 +56,7 @@ if(Protobuf_FOUND AND USE_DEVICE_TREZOR AND TREZOR_PYTHON)
         endif()
 
         if(USE_DEVICE_TREZOR_UDP_RELEASE)
-            add_definitions(-DWITH_DEVICE_TREZOR_UDP_RELEASE=1)
+            add_definitions(-DUSE_DEVICE_TREZOR_UDP_RELEASE=1)
         endif()
 
         if (Protobuf_INCLUDE_DIR)

contrib/depends/packages/bdb.mk

@@ -1,6 +1,6 @@
 package=bdb
 $(package)_version=4.8.30
-$(package)_download_path=http://download.oracle.com/berkeley-db
+$(package)_download_path=https://download.oracle.com/berkeley-db
 $(package)_file_name=db-$($(package)_version).NC.tar.gz
 $(package)_sha256_hash=12edc0df75bf9abd7f82f821795bcee50f42cb2e5f76a6a281b85732798364ef
 $(package)_build_subdir=build_unix

contrib/depends/packages/ldns.mk

@@ -1,6 +1,6 @@
 package=ldns
 $(package)_version=1.6.17
-$(package)_download_path=http://www.nlnetlabs.nl/downloads/ldns/
+$(package)_download_path=https://www.nlnetlabs.nl/downloads/ldns/
 $(package)_file_name=$(package)-$($(package)_version).tar.gz
 $(package)_sha256_hash=8b88e059452118e8949a2752a55ce59bc71fa5bc414103e17f5b6b06f9bcc8cd
 $(package)_dependencies=openssl

contrib/depends/packages/libICE.mk

@@ -1,6 +1,6 @@
 package=libICE
 $(package)_version=1.0.9
-$(package)_download_path=http://xorg.freedesktop.org/releases/individual/lib/
+$(package)_download_path=https://xorg.freedesktop.org/releases/individual/lib/
 $(package)_file_name=$(package)-$($(package)_version).tar.bz2
 $(package)_sha256_hash=8f7032f2c1c64352b5423f6b48a8ebdc339cc63064af34d66a6c9aa79759e202
 $(package)_dependencies=xtrans xproto

contrib/depends/packages/libSM.mk

@@ -1,6 +1,6 @@
 package=libSM
 $(package)_version=1.2.2
-$(package)_download_path=http://xorg.freedesktop.org/releases/individual/lib/
+$(package)_download_path=https://xorg.freedesktop.org/releases/individual/lib/
 $(package)_file_name=$(package)-$($(package)_version).tar.bz2
 $(package)_sha256_hash=0baca8c9f5d934450a70896c4ad38d06475521255ca63b717a6510fdb6e287bd
 $(package)_dependencies=xtrans xproto libICE

contrib/depends/packages/libusb.mk

@@ -1,6 +1,6 @@
 package=libusb
 $(package)_version=1.0.22
-$(package)_download_path=http://sourceforge.net/projects/libusb/files/libusb-1.0/libusb-$($(package)_version)/
+$(package)_download_path=https://sourceforge.net/projects/libusb/files/libusb-1.0/libusb-$($(package)_version)/
 $(package)_file_name=$(package)-$($(package)_version).tar.bz2
 $(package)_sha256_hash=75aeb9d59a4fdb800d329a545c2e6799f732362193b465ea198f2aa275518157
 

contrib/depends/packages/native_cdrkit.mk

@@ -1,6 +1,6 @@
 package=native_cdrkit
 $(package)_version=1.1.11
-$(package)_download_path=http://distro.ibiblio.org/fatdog/source/600/c
+$(package)_download_path=https://distro.ibiblio.org/fatdog/source/600/c
 $(package)_file_name=cdrkit-$($(package)_version).tar.bz2
 $(package)_sha256_hash=b50d64c214a65b1a79afe3a964c691931a4233e2ba605d793eb85d0ac3652564
 $(package)_patches=cdrkit-deterministic.patch

contrib/depends/packages/qt.mk

@@ -1,6 +1,6 @@
 PACKAGE=qt
 $(package)_version=5.7.1
-$(package)_download_path=http://download.qt.io/official_releases/qt/5.7/$($(package)_version)/submodules
+$(package)_download_path=https://download.qt.io/archive/qt/5.7/5.7.1/submodules
 $(package)_suffix=opensource-src-$($(package)_version).tar.gz
 $(package)_file_name=qtbase-$($(package)_suffix)
 $(package)_sha256_hash=95f83e532d23b3ddbde7973f380ecae1bac13230340557276f75f2e37984e410

contrib/depends/packages/unbound.mk

@@ -1,6 +1,6 @@
 package=unbound
 $(package)_version=1.6.8
-$(package)_download_path=http://www.unbound.net/downloads/
+$(package)_download_path=https://www.unbound.net/downloads/
 $(package)_file_name=$(package)-$($(package)_version).tar.gz
 $(package)_sha256_hash=e3b428e33f56a45417107448418865fe08d58e0e7fea199b855515f60884dd49
 $(package)_dependencies=openssl expat ldns

contrib/depends/packages/unwind.mk

@@ -1,6 +1,6 @@
 package=unwind
 $(package)_version=1.2
-$(package)_download_path=http://download.savannah.nongnu.org/releases/libunwind
+$(package)_download_path=https://download.savannah.nongnu.org/releases/libunwind
 $(package)_file_name=lib$(package)-$($(package)_version).tar.gz
 $(package)_sha256_hash=1de38ffbdc88bd694d10081865871cd2bfbb02ad8ef9e1606aee18d65532b992
 

contrib/depends/packages/xproto.mk

@@ -1,6 +1,6 @@
 package=xproto
 $(package)_version=7.0.26
-$(package)_download_path=http://xorg.freedesktop.org/releases/individual/proto
+$(package)_download_path=https://xorg.freedesktop.org/releases/individual/proto
 $(package)_file_name=$(package)-$($(package)_version).tar.bz2
 $(package)_sha256_hash=636162c1759805a5a0114a369dffdeccb8af8c859ef6e1445f26a4e6e046514f
 

contrib/depends/packages/zlib.mk

@@ -1,6 +1,6 @@
 package=zlib
 $(package)_version=1.2.11
-$(package)_download_path=http://www.zlib.net
+$(package)_download_path=https://www.zlib.net
 $(package)_file_name=$(package)-$($(package)_version).tar.gz
 $(package)_sha256_hash=c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1
 

contrib/epee/src/mlocker.cpp

@@ -53,7 +53,6 @@ static size_t query_page_size()
     MERROR("Failed to determine page size");
     return 0;
   }
-  MINFO("Page size: " << ret);
   return ret;
 #else
 #warning Missing query_page_size implementation

contrib/valgrind/monero.supp

@@ -17,3 +17,12 @@
    fun:maybe_unlock_and_signal_one<boost::asio::detail::scoped_lock<boost::asio::detail::posix_mutex> >
    ...
 }
+
+{
+   we leak the logger, for performance reasons in on-the-fly init
+   Memcheck:Leak
+   match-leak-kinds: definite
+   fun:_Znwm
+   fun:_ZN2el4base7Storage7getELPPEv
+   ...
+}

external/easylogging++/easylogging++.cc

@@ -2191,17 +2191,13 @@ void VRegistry::setFromArgs(const base::utils::CommandLineArgs* commandLineArgs)
 #   define ELPP_DEFAULT_LOGGING_FLAGS 0x0
 #endif // !defined(ELPP_DEFAULT_LOGGING_FLAGS)
 // Storage
-el::base::type::StoragePointer getresetELPP(bool reset)
+el::base::type::StoragePointer &el::base::Storage::getELPP()
 {
-  static el::base::type::StoragePointer p(new el::base::Storage(el::LogBuilderPtr(new el::base::DefaultLogBuilder())));
+  if (!el::base::elStorage)
-  if (reset)
+    el::base::elStorage = new el::base::Storage(el::LogBuilderPtr(new el::base::DefaultLogBuilder()));
-    p = NULL;
+  return el::base::elStorage;
-  return p;
-}
-el::base::type::StoragePointer el::base::Storage::getELPP()
-{
-  return getresetELPP(false);
 }
+static struct EnsureELPP { EnsureELPP() { el::base::Storage::getELPP(); } } ensureELPP;
 #if ELPP_ASYNC_LOGGING
 Storage::Storage(const LogBuilderPtr& defaultLogBuilder, base::IWorker* asyncDispatchWorker) :
 #else
@@ -2250,7 +2246,6 @@ Storage::Storage(const LogBuilderPtr& defaultLogBuilder) :
 
 Storage::~Storage(void) {
   ELPP_INTERNAL_INFO(4, "Destroying storage");
-  getresetELPP(true);
 #if ELPP_ASYNC_LOGGING
   ELPP_INTERNAL_INFO(5, "Replacing log dispatch callback to synchronous");
   uninstallLogDispatchCallback<base::AsyncLogDispatchCallback>(std::string("AsyncLogDispatchCallback"));

external/easylogging++/easylogging++.h

@@ -552,7 +552,7 @@ typedef std::ostream ostream_t;
 typedef unsigned int EnumType;
 typedef unsigned short VerboseLevel;
 typedef unsigned long int LineNumber;
-typedef std::shared_ptr<base::Storage> StoragePointer;
+typedef base::Storage *StoragePointer;
 typedef std::shared_ptr<LogDispatchCallback> LogDispatchCallbackPtr;
 typedef std::shared_ptr<PerformanceTrackingCallback> PerformanceTrackingCallbackPtr;
 typedef std::shared_ptr<LoggerRegistrationCallback> LoggerRegistrationCallbackPtr;
@@ -2734,7 +2734,7 @@ class Storage : base::NoCopy, public base::threading::ThreadSafe {
     return it->second;
   }
 
-  static el::base::type::StoragePointer getELPP();
+  static el::base::type::StoragePointer &getELPP();
 
  private:
   base::RegisteredHitCounters* m_registeredHitCounters;
@@ -4613,9 +4613,10 @@ el::base::debug::CrashHandler elCrashHandler(ELPP_USE_DEF_CRASH_HANDLER); \
 }
 
 #if ELPP_ASYNC_LOGGING
-#  define INITIALIZE_EASYLOGGINGPP ELPP_INIT_EASYLOGGINGPP(NULL)
+#  define INITIALIZE_EASYLOGGINGPP ELPP_INIT_EASYLOGGINGPP(new el::base::Storage(el::LogBuilderPtr(new el::base::DefaultLogBuilder()),\
+new el::base::AsyncDispatchWorker()))
 #else
-#  define INITIALIZE_EASYLOGGINGPP ELPP_INIT_EASYLOGGINGPP(NULL)
+#  define INITIALIZE_EASYLOGGINGPP ELPP_INIT_EASYLOGGINGPP(new el::base::Storage(el::LogBuilderPtr(new el::base::DefaultLogBuilder())))
 #endif  // ELPP_ASYNC_LOGGING
 #define INITIALIZE_NULL_EASYLOGGINGPP \
 namespace el {\

src/blockchain_db/blockchain_db.h

@@ -1404,6 +1404,13 @@ public:
    */
   virtual cryptonote::blobdata get_txpool_tx_blob(const crypto::hash& txid) const = 0;
 
+  /**
+   * @brief prune output data for the given amount
+   *
+   * @param amount the amount for which to prune data
+   */
+  virtual void prune_outputs(uint64_t amount) = 0;
+
   /**
    * @brief runs a function over all txpool transactions
    *

src/blockchain_db/lmdb/db_lmdb.cpp

@@ -1075,6 +1075,60 @@ void BlockchainLMDB::remove_output(const uint64_t amount, const uint64_t& out_in
     throw0(DB_ERROR(lmdb_error(std::string("Error deleting amount for output index ").append(boost::lexical_cast<std::string>(out_index).append(": ")).c_str(), result).c_str()));
 }
 
+void BlockchainLMDB::prune_outputs(uint64_t amount)
+{
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  check_open();
+  mdb_txn_cursors *m_cursors = &m_wcursors;
+  CURSOR(output_amounts);
+  CURSOR(output_txs);
+
+  MINFO("Pruning outputs for amount " << amount);
+
+  MDB_val v;
+  MDB_val_set(k, amount);
+  int result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_SET);
+  if (result == MDB_NOTFOUND)
+    return;
+  if (result)
+    throw0(DB_ERROR(lmdb_error("Error looking up outputs: ", result).c_str()));
+
+  // gather output ids
+  mdb_size_t num_elems;
+  mdb_cursor_count(m_cur_output_amounts, &num_elems);
+  MINFO(num_elems << " outputs found");
+  std::vector<uint64_t> output_ids;
+  output_ids.reserve(num_elems);
+  while (1)
+  {
+    const pre_rct_outkey *okp = (const pre_rct_outkey *)v.mv_data;
+    output_ids.push_back(okp->output_id);
+    MDEBUG("output id " << okp->output_id);
+    result = mdb_cursor_get(m_cur_output_amounts, &k, &v, MDB_NEXT_DUP);
+    if (result == MDB_NOTFOUND)
+      break;
+    if (result)
+      throw0(DB_ERROR(lmdb_error("Error counting outputs: ", result).c_str()));
+  }
+  if (output_ids.size() != num_elems)
+    throw0(DB_ERROR("Unexpected number of outputs"));
+
+  result = mdb_cursor_del(m_cur_output_amounts, MDB_NODUPDATA);
+  if (result)
+    throw0(DB_ERROR(lmdb_error("Error deleting outputs: ", result).c_str()));
+
+  for (uint64_t output_id: output_ids)
+  {
+    MDB_val_set(v, output_id);
+    result = mdb_cursor_get(m_cur_output_txs, (MDB_val *)&zerokval, &v, MDB_GET_BOTH);
+    if (result)
+      throw0(DB_ERROR(lmdb_error("Error looking up output: ", result).c_str()));
+    result = mdb_cursor_del(m_cur_output_txs, 0);
+    if (result)
+      throw0(DB_ERROR(lmdb_error("Error deleting output: ", result).c_str()));
+  }
+}
+
 void BlockchainLMDB::add_spent_key(const crypto::key_image& k_image)
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
@@ -2229,11 +2283,19 @@ uint64_t BlockchainLMDB::num_outputs() const
   TXN_PREFIX_RDONLY();
   int result;
 
-  // get current height
+  RCURSOR(output_txs)
-  MDB_stat db_stats;
+
-  if ((result = mdb_stat(m_txn, m_output_txs, &db_stats)))
+  uint64_t num = 0;
+  MDB_val k, v;
+  result = mdb_cursor_get(m_cur_output_txs, &k, &v, MDB_LAST);
+  if (result == MDB_NOTFOUND)
+    num = 0;
+  else if (result == 0)
+    num = 1 + ((const outtx*)v.mv_data)->output_id;
+  else
     throw0(DB_ERROR(lmdb_error("Failed to query m_output_txs: ", result).c_str()));
-  return db_stats.ms_entries;
+
+  return num;
 }
 
 bool BlockchainLMDB::tx_exists(const crypto::hash& h) const

src/blockchain_db/lmdb/db_lmdb.h

@@ -345,6 +345,8 @@ private:
 
   void remove_output(const uint64_t amount, const uint64_t& out_index);
 
+  virtual void prune_outputs(uint64_t amount);
+
   virtual void add_spent_key(const crypto::key_image& k_image);
 
   virtual void remove_spent_key(const crypto::key_image& k_image);

src/blockchain_utilities/CMakeLists.txt

@@ -81,6 +81,17 @@ monero_private_headers(blockchain_usage
 
 
 
+set(blockchain_prune_known_spent_data_sources
+  blockchain_prune_known_spent_data.cpp
+  )
+
+set(blockchain_prune_known_spent_data_private_headers)
+
+monero_private_headers(blockchain_prune_known_spent_data
+	  ${blockchain_prune_known_spent_data_private_headers})
+
+
+
 set(blockchain_ancestry_sources
   blockchain_ancestry.cpp
   )
@@ -265,3 +276,25 @@ set_property(TARGET blockchain_stats
 	PROPERTY
 	OUTPUT_NAME "wownero-blockchain-stats")
 install(TARGETS blockchain_stats DESTINATION bin)
+
+monero_add_executable(blockchain_prune_known_spent_data
+  ${blockchain_prune_known_spent_data_sources}
+  ${blockchain_prune_known_spent_data_private_headers})
+
+target_link_libraries(blockchain_prune_known_spent_data
+  PRIVATE
+    cryptonote_core
+    blockchain_db
+    p2p
+    version
+    epee
+    ${Boost_FILESYSTEM_LIBRARY}
+    ${Boost_SYSTEM_LIBRARY}
+    ${Boost_THREAD_LIBRARY}
+    ${CMAKE_THREAD_LIBS_INIT}
+    ${EXTRA_LIBRARIES})
+
+set_property(TARGET blockchain_prune_known_spent_data
+	PROPERTY
+	OUTPUT_NAME "wownero-blockchain-prune-known-spent-data")
+install(TARGETS blockchain_prune_known_spent_data DESTINATION bin)

src/blockchain_utilities/blockchain_blackball.cpp

@@ -59,6 +59,7 @@ static MDB_dbi dbi_relative_rings;
 static MDB_dbi dbi_outputs;
 static MDB_dbi dbi_processed_txidx;
 static MDB_dbi dbi_spent;
+static MDB_dbi dbi_per_amount;
 static MDB_dbi dbi_ring_instances;
 static MDB_dbi dbi_stats;
 static MDB_env *env = NULL;
@@ -238,7 +239,7 @@ static void init(std::string cache_filename)
 
   dbr = mdb_env_create(&env);
   CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to create LDMB environment: " + std::string(mdb_strerror(dbr)));
-  dbr = mdb_env_set_maxdbs(env, 6);
+  dbr = mdb_env_set_maxdbs(env, 7);
   CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to set max env dbs: " + std::string(mdb_strerror(dbr)));
   const std::string actual_filename = get_cache_filename(cache_filename); 
   dbr = mdb_env_open(env, actual_filename.c_str(), flags, 0664);
@@ -265,6 +266,10 @@ static void init(std::string cache_filename)
   CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
   mdb_set_dupsort(txn, dbi_spent, compare_uint64);
 
+  dbr = mdb_dbi_open(txn, "per_amount", MDB_CREATE | MDB_INTEGERKEY, &dbi_per_amount);
+  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
+  mdb_set_compare(txn, dbi_per_amount, compare_uint64);
+
   dbr = mdb_dbi_open(txn, "ring_instances", MDB_CREATE, &dbi_ring_instances);
   CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open LMDB dbi: " + std::string(mdb_strerror(dbr)));
 
@@ -283,6 +288,7 @@ static void close()
     mdb_dbi_close(env, dbi_relative_rings);
     mdb_dbi_close(env, dbi_outputs);
     mdb_dbi_close(env, dbi_processed_txidx);
+    mdb_dbi_close(env, dbi_per_amount);
     mdb_dbi_close(env, dbi_spent);
     mdb_dbi_close(env, dbi_ring_instances);
     mdb_dbi_close(env, dbi_stats);
@@ -585,6 +591,55 @@ static std::vector<output_data> get_spent_outputs(MDB_txn *txn)
   return outs;
 }
 
+static void get_per_amount_outputs(MDB_txn *txn, uint64_t amount, uint64_t &total, uint64_t &spent)
+{
+  MDB_cursor *cur;
+  int dbr = mdb_cursor_open(txn, dbi_per_amount, &cur);
+  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open cursor for per amount outputs: " + std::string(mdb_strerror(dbr)));
+  MDB_val k, v;
+  mdb_size_t count = 0;
+  k.mv_size = sizeof(uint64_t);
+  k.mv_data = (void*)&amount;
+  dbr = mdb_cursor_get(cur, &k, &v, MDB_SET);
+  if (dbr == MDB_NOTFOUND)
+  {
+    total = spent = 0;
+  }
+  else
+  {
+    CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to get per amount outputs: " + std::string(mdb_strerror(dbr)));
+    total = ((const uint64_t*)v.mv_data)[0];
+    spent = ((const uint64_t*)v.mv_data)[1];
+  }
+  mdb_cursor_close(cur);
+}
+
+static void inc_per_amount_outputs(MDB_txn *txn, uint64_t amount, uint64_t total, uint64_t spent)
+{
+  MDB_cursor *cur;
+  int dbr = mdb_cursor_open(txn, dbi_per_amount, &cur);
+  CHECK_AND_ASSERT_THROW_MES(!dbr, "Failed to open cursor for per amount outputs: " + std::string(mdb_strerror(dbr)));
+  MDB_val k, v;
+  mdb_size_t count = 0;
+  k.mv_size = sizeof(uint64_t);
+  k.mv_data = (void*)&amount;
+  dbr = mdb_cursor_get(cur, &k, &v, MDB_SET);
+  if (dbr == 0)
+  {
+    total += ((const uint64_t*)v.mv_data)[0];
+    spent += ((const uint64_t*)v.mv_data)[1];
+  }
+  else
+  {
+    CHECK_AND_ASSERT_THROW_MES(dbr == MDB_NOTFOUND, "Failed to get per amount outputs: " + std::string(mdb_strerror(dbr)));
+  }
+  uint64_t data[2] = {total, spent};
+  v.mv_size = 2 * sizeof(uint64_t);
+  v.mv_data = (void*)data;
+  dbr = mdb_cursor_put(cur, &k, &v, 0);
+  mdb_cursor_close(cur);
+}
+
 static uint64_t get_processed_txidx(const std::string &name)
 {
   MDB_txn *txn;
@@ -1193,6 +1248,7 @@ int main(int argc, char* argv[])
     for_all_transactions(filename, start_idx, n_txes, [&](const cryptonote::transaction_prefix &tx)->bool
     {
       std::cout << "\r" << start_idx << "/" << n_txes << "         \r" << std::flush;
+      const bool miner_tx = tx.vin.size() == 1 && tx.vin[0].type() == typeid(txin_gen);
       for (const auto &in: tx.vin)
       {
         if (in.type() != typeid(txin_to_key))
@@ -1210,6 +1266,9 @@ int main(int argc, char* argv[])
         std::vector<uint64_t> new_ring = canonicalize(txin.key_offsets);
         const uint32_t ring_size = txin.key_offsets.size();
         const uint64_t instances = inc_ring_instances(txn, txin.amount, new_ring);
+        uint64_t pa_total = 0, pa_spent = 0;
+        if (!opt_rct_only)
+          get_per_amount_outputs(txn, txin.amount, pa_total, pa_spent);
         if (n == 0 && ring_size == 1)
         {
           const std::pair<uint64_t, uint64_t> output = std::make_pair(txin.amount, absolute[0]);
@@ -1237,6 +1296,21 @@ int main(int argc, char* argv[])
               inc_stat(txn, txin.amount ? "pre-rct-duplicate-rings" : "rct-duplicate-rings");
           }
         }
+        else if (n == 0 && !opt_rct_only && pa_spent + 1 == pa_total)
+        {
+          for (size_t o = 0; o < pa_total; ++o)
+          {
+            const std::pair<uint64_t, uint64_t> output = std::make_pair(txin.amount, o);
+            if (opt_verbose)
+            {
+              MINFO("Marking output " << output.first << "/" << output.second << " as spent, due to as many outputs of that amount being spent as exist so far");
+              std::cout << "\r" << start_idx << "/" << n_txes << "         \r" << std::flush;
+            }
+            blackballs.push_back(output);
+            if (add_spent_output(cur, output_data(txin.amount, o)))
+              inc_stat(txn, txin.amount ? "pre-rct-full-count" : "rct-full-count");
+          }
+        }
         else if (n == 0 && opt_check_subsets && get_ring_subset_instances(txn, txin.amount, new_ring) >= new_ring.size())
         {
           for (size_t o = 0; o < new_ring.size(); ++o)
@@ -1299,9 +1373,28 @@ int main(int argc, char* argv[])
           }
         }
         if (n == 0)
+        {
           set_relative_ring(txn, txin.k_image, new_ring);
+          if (!opt_rct_only)
+            inc_per_amount_outputs(txn, txin.amount, 0, 1);
+        }
       }
       set_processed_txidx(txn, canonical, start_idx+1);
+      if (!opt_rct_only)
+      {
+        for (const auto &out: tx.vout)
+        {
+          uint64_t amount = out.amount;
+          if (miner_tx && tx.version >= 2)
+            amount = 0;
+
+          if (opt_rct_only && amount != 0)
+            continue;
+          if (out.target.type() != typeid(txout_to_key))
+            continue;
+          inc_per_amount_outputs(txn, amount, 1, 0);
+        }
+      }
 
       ++records;
       if (records >= records_per_sync)
@@ -1433,6 +1526,7 @@ skip_secondary_passes:
     { "pre-rct-ring-size-1", pre_rct }, { "rct-ring-size-1", rct },
     { "pre-rct-duplicate-rings", pre_rct }, { "rct-duplicate-rings", rct },
     { "pre-rct-subset-rings", pre_rct }, { "rct-subset-rings", rct },
+    { "pre-rct-full-count", pre_rct }, { "rct-full-count", rct },
     { "pre-rct-key-image-attack", pre_rct }, { "rct-key-image-attack", rct },
     { "pre-rct-extra", pre_rct }, { "rct-ring-extra", rct },
     { "pre-rct-chain-reaction", pre_rct }, { "rct-chain-reaction", rct },

src/blockchain_utilities/blockchain_prune_known_spent_data.cpp

@@ -0,0 +1,305 @@
+// Copyright (c) 2014-2018, The Monero Project
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <boost/algorithm/string.hpp>
+#include "common/command_line.h"
+#include "serialization/crypto.h"
+#include "cryptonote_core/tx_pool.h"
+#include "cryptonote_core/cryptonote_core.h"
+#include "cryptonote_core/blockchain.h"
+#include "blockchain_db/blockchain_db.h"
+#include "blockchain_db/db_types.h"
+#include "version.h"
+
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "bcutil"
+
+namespace po = boost::program_options;
+using namespace epee;
+using namespace cryptonote;
+
+static std::map<uint64_t, uint64_t> load_outputs(const std::string &filename)
+{
+  std::map<uint64_t, uint64_t> outputs;
+  uint64_t amount = std::numeric_limits<uint64_t>::max();
+  FILE *f;
+
+  f = fopen(filename.c_str(), "r");
+  if (!f)
+  {
+    MERROR("Failed to load outputs from " << filename << ": " << strerror(errno));
+    return {};
+  }
+  while (1)
+  {
+    char s[256];
+    if (!fgets(s, sizeof(s), f))
+      break;
+    if (feof(f))
+      break;
+    const size_t len = strlen(s);
+    if (len > 0 && s[len - 1] == '\n')
+      s[len - 1] = 0;
+    if (!s[0])
+      continue;
+    std::pair<uint64_t, uint64_t> output;
+    uint64_t offset, num_offsets;
+    if (sscanf(s, "@%" PRIu64, &amount) == 1)
+    {
+      continue;
+    }
+    if (amount == std::numeric_limits<uint64_t>::max())
+    {
+      MERROR("Bad format in " << filename);
+      continue;
+    }
+    if (sscanf(s, "%" PRIu64 "*%" PRIu64, &offset, &num_offsets) == 2 && num_offsets < std::numeric_limits<uint64_t>::max() - offset)
+    {
+      outputs[amount] += num_offsets;
+    }
+    else if (sscanf(s, "%" PRIu64, &offset) == 1)
+    {
+      outputs[amount] += 1;
+    }
+    else
+    {
+      MERROR("Bad format in " << filename);
+      continue;
+    }
+  }
+  fclose(f);
+  return outputs;
+}
+
+int main(int argc, char* argv[])
+{
+  TRY_ENTRY();
+
+  epee::string_tools::set_module_name_and_folder(argv[0]);
+
+  std::string default_db_type = "lmdb";
+
+  std::string available_dbs = cryptonote::blockchain_db_types(", ");
+  available_dbs = "available: " + available_dbs;
+
+  uint32_t log_level = 0;
+
+  tools::on_startup();
+
+  po::options_description desc_cmd_only("Command line options");
+  po::options_description desc_cmd_sett("Command line options and settings options");
+  const command_line::arg_descriptor<std::string> arg_log_level  = {"log-level",  "0-4 or categories", ""};
+  const command_line::arg_descriptor<std::string> arg_database = {
+    "database", available_dbs.c_str(), default_db_type
+  };
+  const command_line::arg_descriptor<bool> arg_verbose  = {"verbose", "Verbose output", false};
+  const command_line::arg_descriptor<bool> arg_dry_run  = {"dry-run", "Do not actually prune", false};
+  const command_line::arg_descriptor<std::string> arg_input = {"input", "Path to the known spent outputs file"};
+
+  command_line::add_arg(desc_cmd_sett, cryptonote::arg_data_dir);
+  command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on);
+  command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on);
+  command_line::add_arg(desc_cmd_sett, arg_log_level);
+  command_line::add_arg(desc_cmd_sett, arg_database);
+  command_line::add_arg(desc_cmd_sett, arg_verbose);
+  command_line::add_arg(desc_cmd_sett, arg_dry_run);
+  command_line::add_arg(desc_cmd_sett, arg_input);
+  command_line::add_arg(desc_cmd_only, command_line::arg_help);
+
+  po::options_description desc_options("Allowed options");
+  desc_options.add(desc_cmd_only).add(desc_cmd_sett);
+
+  po::variables_map vm;
+  bool r = command_line::handle_error_helper(desc_options, [&]()
+  {
+    auto parser = po::command_line_parser(argc, argv).options(desc_options);
+    po::store(parser.run(), vm);
+    po::notify(vm);
+    return true;
+  });
+  if (! r)
+    return 1;
+
+  if (command_line::get_arg(vm, command_line::arg_help))
+  {
+    std::cout << "Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")" << ENDL << ENDL;
+    std::cout << desc_options << std::endl;
+    return 1;
+  }
+
+  mlog_configure(mlog_get_default_log_path("wownero-blockchain-prune-known-spent-data.log"), true);
+  if (!command_line::is_arg_defaulted(vm, arg_log_level))
+    mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str());
+  else
+    mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str());
+
+  LOG_PRINT_L0("Starting...");
+
+  std::string opt_data_dir = command_line::get_arg(vm, cryptonote::arg_data_dir);
+  bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on);
+  bool opt_stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on);
+  network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET;
+  bool opt_verbose = command_line::get_arg(vm, arg_verbose);
+  bool opt_dry_run = command_line::get_arg(vm, arg_dry_run);
+
+  std::string db_type = command_line::get_arg(vm, arg_database);
+  if (!cryptonote::blockchain_valid_db_type(db_type))
+  {
+    std::cerr << "Invalid database type: " << db_type << std::endl;
+    return 1;
+  }
+
+  const std::string input = command_line::get_arg(vm, arg_input);
+
+  LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)");
+  std::unique_ptr<Blockchain> core_storage;
+  tx_memory_pool m_mempool(*core_storage);
+  core_storage.reset(new Blockchain(m_mempool));
+  BlockchainDB *db = new_db(db_type);
+  if (db == NULL)
+  {
+    LOG_ERROR("Attempted to use non-existent database type: " << db_type);
+    throw std::runtime_error("Attempting to use non-existent database type");
+  }
+  LOG_PRINT_L0("database: " << db_type);
+
+  const std::string filename = (boost::filesystem::path(opt_data_dir) / db->get_db_name()).string();
+  LOG_PRINT_L0("Loading blockchain from folder " << filename << " ...");
+
+  try
+  {
+    db->open(filename, 0);
+  }
+  catch (const std::exception& e)
+  {
+    LOG_PRINT_L0("Error opening database: " << e.what());
+    return 1;
+  }
+  r = core_storage->init(db, net_type);
+
+  CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage");
+  LOG_PRINT_L0("Source blockchain storage initialized OK");
+
+  std::map<uint64_t, uint64_t> known_spent_outputs;
+  if (input.empty())
+  {
+    std::map<uint64_t, std::pair<uint64_t, uint64_t>> outputs;
+
+    LOG_PRINT_L0("Scanning for known spent data...");
+    db->for_all_transactions([&](const crypto::hash &txid, const cryptonote::transaction &tx){
+      const bool miner_tx = tx.vin.size() == 1 && tx.vin[0].type() == typeid(txin_gen);
+      for (const auto &in: tx.vin)
+      {
+        if (in.type() != typeid(txin_to_key))
+          continue;
+        const auto &txin = boost::get<txin_to_key>(in);
+        if (txin.amount == 0)
+          continue;
+
+        outputs[txin.amount].second++;
+      }
+
+      for (const auto &out: tx.vout)
+      {
+        uint64_t amount = out.amount;
+        if (miner_tx && tx.version >= 2)
+          amount = 0;
+        if (amount == 0)
+          continue;
+        if (out.target.type() != typeid(txout_to_key))
+          continue;
+
+        outputs[amount].first++;
+      }
+      return true;
+    }, true);
+
+    for (const auto &i: outputs)
+    {
+      known_spent_outputs[i.first] = i.second.second;
+    }
+  }
+  else
+  {
+    LOG_PRINT_L0("Loading known spent data...");
+    known_spent_outputs = load_outputs(input);
+  }
+
+  LOG_PRINT_L0("Pruning known spent data...");
+
+  bool stop_requested = false;
+  tools::signal_handler::install([&stop_requested](int type) {
+    stop_requested = true;
+  });
+
+  db->batch_start();
+
+  size_t num_total_outputs = 0, num_prunable_outputs = 0, num_known_spent_outputs = 0, num_eligible_outputs = 0, num_eligible_known_spent_outputs = 0;
+  for (auto i = known_spent_outputs.begin(); i != known_spent_outputs.end(); ++i)
+  {
+    uint64_t num_outputs = db->get_num_outputs(i->first);
+    num_total_outputs += num_outputs;
+    num_known_spent_outputs += i->second;
+    if (i->first == 0 || is_valid_decomposed_amount(i->first))
+    {
+      if (opt_verbose)
+        MINFO("Ignoring output value " << i->first << ", with " << num_outputs << " outputs");
+      continue;
+    }
+    num_eligible_outputs += num_outputs;
+    num_eligible_known_spent_outputs += i->second;
+    if (opt_verbose)
+      MINFO(i->first << ": " << i->second << "/" << num_outputs);
+    if (num_outputs > i->second)
+      continue;
+    if (num_outputs && num_outputs < i->second)
+    {
+      MERROR("More outputs are spent than known for amount " << i->first << ", not touching");
+      continue;
+    }
+    if (opt_verbose)
+      MINFO("Pruning data for " << num_outputs << " outputs");
+    if (!opt_dry_run)
+      db->prune_outputs(i->first);
+    num_prunable_outputs += i->second;
+  }
+
+  db->batch_stop();
+
+  MINFO("Total outputs: " << num_total_outputs);
+  MINFO("Known spent outputs: " << num_known_spent_outputs);
+  MINFO("Eligible outputs: " << num_eligible_outputs);
+  MINFO("Eligible known spent outputs: " << num_eligible_known_spent_outputs);
+  MINFO("Prunable outputs: " << num_prunable_outputs);
+
+  LOG_PRINT_L0("Blockchain known spent data pruned OK");
+  core_storage->deinit();
+  return 0;
+
+  CATCH_ENTRY("Error", 1);
+}

src/cryptonote_basic/cryptonote_basic.h

@@ -175,7 +175,15 @@ namespace cryptonote
     END_SERIALIZE()
 
   public:
-    transaction_prefix(){}
+    transaction_prefix(){ set_null(); }
+    void set_null()
+    {
+      version = 1;
+      unlock_time = 0;
+      vin.clear();
+      vout.clear();
+      extra.clear();
+    }
   };
 
   class transaction: public transaction_prefix
@@ -302,17 +310,12 @@ namespace cryptonote
   inline
   transaction::~transaction()
   {
-    //set_null();
   }
 
   inline
   void transaction::set_null()
   {
-    version = 1;
+    transaction_prefix::set_null();
-    unlock_time = 0;
-    vin.clear();
-    vout.clear();
-    extra.clear();
     signatures.clear();
     rct_signatures.type = rct::RCTTypeNull;
     set_hash_valid(false);

src/cryptonote_core/blockchain.cpp

@@ -535,6 +535,38 @@ bool Blockchain::deinit()
   return true;
 }
 //------------------------------------------------------------------
+// This function removes blocks from the top of blockchain.
+// It starts a batch and calls private method pop_block_from_blockchain().
+void Blockchain::pop_blocks(uint64_t nblocks)
+{
+  uint64_t i;
+  CRITICAL_REGION_LOCAL(m_tx_pool);
+  CRITICAL_REGION_LOCAL1(m_blockchain_lock);
+
+  while (!m_db->batch_start())
+  {
+    m_blockchain_lock.unlock();
+    m_tx_pool.unlock();
+    epee::misc_utils::sleep_no_w(1000);
+    m_tx_pool.lock();
+    m_blockchain_lock.lock();
+  }
+
+  try
+  {
+    for (i=0; i < nblocks; ++i)
+    {
+      pop_block_from_blockchain();
+    }
+  }
+  catch (const std::exception& e)
+  {
+    LOG_ERROR("Error when popping blocks, only " << i << " blocks are popped: " << e.what());
+  }
+
+  m_db->batch_stop();
+}
+//------------------------------------------------------------------
 // This function tells BlockchainDB to remove the top block from the
 // blockchain and then returns all transactions (except the miner tx, of course)
 // from it to the tx_pool
@@ -3732,7 +3764,7 @@ void Blockchain::set_enforce_dns_checkpoints(bool enforce_checkpoints)
 }
 
 //------------------------------------------------------------------
-void Blockchain::block_longhash_worker(uint64_t height, const std::vector<block> &blocks, std::unordered_map<crypto::hash, crypto::hash> &map) const
+void Blockchain::block_longhash_worker(uint64_t height, const epee::span<const block> &blocks, std::unordered_map<crypto::hash, crypto::hash> &map) const
 {
   TIME_MEASURE_START(t);
   slow_hash_allocate_state();
@@ -3818,11 +3850,33 @@ bool Blockchain::cleanup_handle_incoming_blocks(bool force_sync)
 }
 
 //------------------------------------------------------------------
-void Blockchain::output_scan_worker(const uint64_t amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs) const
+void Blockchain::output_scan_worker(const uint64_t amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs, const std::vector<output_data_t> &extra_tx_map) const
 {
   try
   {
     m_db->get_output_key(epee::span<const uint64_t>(&amount, 1), offsets, outputs, true);
+    if (outputs.size() < offsets.size())
+    {
+      const uint64_t n_outputs = m_db->get_num_outputs(amount);
+      for (size_t i = outputs.size(); i < offsets.size(); ++i)
+      {
+        uint64_t idx = offsets[i];
+        if (idx < n_outputs)
+        {
+          MWARNING("Index " << idx << " not found in db for amount " << amount << ", but it is less than the number of entries");
+          break;
+        }
+        else if (idx < n_outputs + extra_tx_map.size())
+        {
+          outputs.push_back(extra_tx_map[idx - n_outputs]);
+        }
+        else
+        {
+          MWARNING("missed " << amount << "/" << idx << " in " << extra_tx_map.size() << " (chain " << n_outputs << ")");
+          break;
+        }
+      }
+    }
   }
   catch (const std::exception& e)
   {
@@ -3937,6 +3991,34 @@ uint64_t Blockchain::prevalidate_block_hashes(uint64_t height, const std::vector
 //    vs [k_image, output_keys] (m_scan_table). This is faster because it takes advantage of bulk queries
 //    and is threaded if possible. The table (m_scan_table) will be used later when querying output
 //    keys.
+static bool update_output_map(std::map<uint64_t, std::vector<output_data_t>> &extra_tx_map, const transaction &tx, uint64_t height, bool miner)
+{
+  MTRACE("Blockchain::" << __func__);
+  for (size_t i = 0; i < tx.vout.size(); ++i)
+  {
+    const auto &out = tx.vout[i];
+    if (out.target.type() != typeid(txout_to_key))
+      continue;
+    const txout_to_key &out_to_key = boost::get<txout_to_key>(out.target);
+    rct::key commitment;
+    uint64_t amount = out.amount;
+    if (miner && tx.version == 2)
+    {
+      commitment = rct::zeroCommit(amount);
+      amount = 0;
+    }
+    else if (tx.version > 1)
+    {
+      CHECK_AND_ASSERT_MES(i < tx.rct_signatures.outPk.size(), false, "Invalid outPk size");
+      commitment = tx.rct_signatures.outPk[i].mask;
+    }
+    else
+      commitment = rct::zero();
+    extra_tx_map[amount].push_back(output_data_t{out_to_key.key, tx.unlock_time, height, commitment});
+  }
+  return true;
+}
+
 bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete_entry> &blocks_entry)
 {
   MTRACE("Blockchain::" << __func__);
@@ -3983,42 +4065,40 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete
     m_blockchain_lock.lock();
   }
 
-  if ((m_db->height() + blocks_entry.size()) < m_blocks_hash_check.size())
+  const uint64_t height = m_db->height();
+  if ((height + blocks_entry.size()) < m_blocks_hash_check.size())
     return true;
 
   bool blocks_exist = false;
   tools::threadpool& tpool = tools::threadpool::getInstance();
-  uint64_t threads = tpool.get_max_concurrency();
+  unsigned threads = tpool.get_max_concurrency();
+  std::vector<block> blocks;
+  blocks.resize(blocks_entry.size());
 
-  if (blocks_entry.size() > 1 && threads > 1 && m_max_prepare_blocks_threads > 1)
+  if (1)
   {
     // limit threads, default limit = 4
     if(threads > m_max_prepare_blocks_threads)
       threads = m_max_prepare_blocks_threads;
 
-    uint64_t height = m_db->height();
+    unsigned int batches = blocks_entry.size() / threads;
-    int batches = blocks_entry.size() / threads;
+    unsigned int extra = blocks_entry.size() % threads;
-    int extra = blocks_entry.size() % threads;
     MDEBUG("block_batches: " << batches);
     std::vector<std::unordered_map<crypto::hash, crypto::hash>> maps(threads);
-    std::vector < std::vector < block >> blocks(threads);
     auto it = blocks_entry.begin();
+    unsigned blockidx = 0;
 
-    for (uint64_t i = 0; i < threads; i++)
+    for (unsigned i = 0; i < threads; i++)
     {
-      blocks[i].reserve(batches + 1);
+      for (unsigned int j = 0; j < batches; j++, ++blockidx)
-      for (int j = 0; j < batches; j++)
       {
-        block block;
+        block &block = blocks[blockidx];
 
         if (!parse_and_validate_block_from_blob(it->block, block))
-        {
+          return false;
-          std::advance(it, 1);
-          continue;
-        }
 
         // check first block and skip all blocks if its not chained properly
-        if (i == 0 && j == 0)
+        if (blockidx == 0)
         {
           crypto::hash tophash = m_db->top_block_hash();
           if (block.prev_id != tophash)
@@ -4033,20 +4113,16 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete
           break;
         }
 
-        blocks[i].push_back(std::move(block));
         std::advance(it, 1);
       }
     }
 
-    for (int i = 0; i < extra && !blocks_exist; i++)
+    for (unsigned i = 0; i < extra && !blocks_exist; i++, blockidx++)
     {
-      block block;
+      block &block = blocks[blockidx];
 
       if (!parse_and_validate_block_from_blob(it->block, block))
-      {
+        return false;
-        std::advance(it, 1);
-        continue;
-      }
 
       if (have_block(get_block_hash(block)))
       {
@@ -4054,7 +4130,6 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete
         break;
       }
 
-      blocks[i].push_back(std::move(block));
       std::advance(it, 1);
     }
 
@@ -4063,10 +4138,13 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete
       m_blocks_longhash_table.clear();
       uint64_t thread_height = height;
       tools::threadpool::waiter waiter;
-      for (uint64_t i = 0; i < threads; i++)
+      for (unsigned int i = 0; i < threads; i++)
       {
-        tpool.submit(&waiter, boost::bind(&Blockchain::block_longhash_worker, this, thread_height, std::cref(blocks[i]), std::ref(maps[i])), true);
+        unsigned nblocks = batches;
-        thread_height += blocks[i].size();
+        if (i < extra)
+          ++nblocks;
+        tpool.submit(&waiter, boost::bind(&Blockchain::block_longhash_worker, this, thread_height, epee::span<const block>(&blocks[i], nblocks), std::ref(maps[i])), true);
+        thread_height += nblocks;
       }
 
       waiter.wait(&tpool);
@@ -4109,7 +4187,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete
   // [input] stores all absolute_offsets for each amount
   std::map<uint64_t, std::vector<uint64_t>> offset_map;
   // [output] stores all output_data_t for each absolute_offset
-  std::map<uint64_t, std::vector<output_data_t>> tx_map;
+  std::map<uint64_t, std::vector<output_data_t>> tx_map, extra_tx_map;
   std::vector<std::pair<cryptonote::transaction, crypto::hash>> txes(total_txs);
 
 #define SCAN_TABLE_QUIT(m) \
@@ -4120,12 +4198,14 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete
         } while(0); \
 
   // generate sorted tables for all amounts and absolute offsets
-  size_t tx_index = 0;
+  size_t tx_index = 0, block_index = 0;
   for (const auto &entry : blocks_entry)
   {
     if (m_cancel)
       return false;
 
+    if (!update_output_map(extra_tx_map, blocks[block_index].miner_tx, height + block_index, true))
+      SCAN_TABLE_QUIT("Error building extra tx map.");
     for (const auto &tx_blob : entry.txs)
     {
       if (tx_index >= txes.size())
@@ -4184,7 +4264,10 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete
           offset_map[in_to_key.amount].push_back(offset);
 
       }
+      if (!update_output_map(extra_tx_map, tx, height + block_index, false))
+        SCAN_TABLE_QUIT("Error building extra tx map.");
     }
+    ++block_index;
   }
 
   // sort and remove duplicate absolute_offsets in offset_map
@@ -4207,7 +4290,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete
     for (size_t i = 0; i < amounts.size(); i++)
     {
       uint64_t amount = amounts[i];
-      tpool.submit(&waiter, boost::bind(&Blockchain::output_scan_worker, this, amount, std::cref(offset_map[amount]), std::ref(tx_map[amount])), true);
+      tpool.submit(&waiter, boost::bind(&Blockchain::output_scan_worker, this, amount, std::cref(offset_map[amount]), std::ref(tx_map[amount]), std::cref(extra_tx_map[amount])), true);
     }
     waiter.wait(&tpool);
   }
@@ -4216,7 +4299,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete
     for (size_t i = 0; i < amounts.size(); i++)
     {
       uint64_t amount = amounts[i];
-      output_scan_worker(amount, offset_map[amount], tx_map[amount]);
+      output_scan_worker(amount, offset_map[amount], tx_map[amount], extra_tx_map[amount]);
     }
   }
 

src/cryptonote_core/blockchain.h

@@ -918,7 +918,7 @@ namespace cryptonote
      * @param outputs return-by-reference the outputs collected
      */
     void output_scan_worker(const uint64_t amount,const std::vector<uint64_t> &offsets,
-        std::vector<output_data_t> &outputs) const;
+        std::vector<output_data_t> &outputs, const std::vector<output_data_t> &extra_tx_map) const;
 
     /**
      * @brief computes the "short" and "long" hashes for a set of blocks
@@ -927,7 +927,7 @@ namespace cryptonote
      * @param blocks the blocks to be hashed
      * @param map return-by-reference the hashes for each block
      */
-    void block_longhash_worker(uint64_t height, const std::vector<block> &blocks,
+    void block_longhash_worker(uint64_t height, const epee::span<const block> &blocks,
         std::unordered_map<crypto::hash, crypto::hash> &map) const;
 
     /**
@@ -967,6 +967,13 @@ namespace cryptonote
      */
     std::vector<time_t> get_last_block_timestamps(unsigned int blocks) const;
 
+    /**
+     * @brief removes blocks from the top of the blockchain
+     *
+     * @param nblocks number of blocks to be removed
+     */
+    void pop_blocks(uint64_t nblocks);
+
   private:
 
     // TODO: evaluate whether or not each of these typedefs are left over from blockchain_storage

src/cryptonote_core/cryptonote_core.cpp

@@ -160,6 +160,11 @@ namespace cryptonote
   , "Relay blocks as normal blocks"
   , false
   };
+  static const command_line::arg_descriptor<bool> arg_pad_transactions  = {
+    "pad-transactions"
+  , "Pad relayed transactions to help defend against traffic volume analysis"
+  , false
+  };
   static const command_line::arg_descriptor<size_t> arg_max_txpool_weight  = {
     "max-txpool-weight"
   , "Set maximum txpool weight in bytes."
@@ -185,7 +190,8 @@ namespace cryptonote
               m_disable_dns_checkpoints(false),
               m_update_download(0),
               m_nettype(UNDEFINED),
-              m_update_available(false)
+              m_update_available(false),
+              m_pad_transactions(false)
   {
     m_checkpoints_updating.clear();
     set_cryptonote_protocol(pprotocol);
@@ -244,6 +250,7 @@ namespace cryptonote
   //-----------------------------------------------------------------------------------
   void core::stop()
   {
+    m_miner.stop();
     m_blockchain_storage.cancel();
 
     tools::download_async_handle handle;
@@ -279,6 +286,7 @@ namespace cryptonote
     command_line::add_arg(desc, arg_offline);
     command_line::add_arg(desc, arg_disable_dns_checkpoints);
     command_line::add_arg(desc, arg_max_txpool_weight);
+    command_line::add_arg(desc, arg_pad_transactions);
     command_line::add_arg(desc, arg_block_notify);
 
     miner::init_options(desc);
@@ -317,6 +325,7 @@ namespace cryptonote
     set_enforce_dns_checkpoints(command_line::get_arg(vm, arg_dns_checkpoints));
     test_drop_download_height(command_line::get_arg(vm, arg_test_drop_download_height));
     m_fluffy_blocks_enabled = !get_arg(vm, arg_no_fluffy_blocks);
+    m_pad_transactions = get_arg(vm, arg_pad_transactions);
     m_offline = get_arg(vm, arg_offline);
     m_disable_dns_checkpoints = get_arg(vm, arg_disable_dns_checkpoints);
     if (!command_line::is_arg_defaulted(vm, arg_fluffy_blocks))
@@ -893,13 +902,15 @@ namespace cryptonote
     bool ok = true;
     it = tx_blobs.begin();
     for (size_t i = 0; i < tx_blobs.size(); i++, ++it) {
-      if (already_have[i])
-        continue;
       if (!results[i].res)
       {
         ok = false;
         continue;
       }
+      if (keeped_by_block)
+        get_blockchain_storage().on_new_tx_from_block(results[i].tx);
+      if (already_have[i])
+        continue;
 
       const size_t weight = get_transaction_weight(results[i].tx, it->size());
       ok &= add_new_tx(results[i].tx, results[i].hash, tx_blobs[i], results[i].prefix_hash, weight, tvc[i], keeped_by_block, relayed, do_not_relay);
@@ -1130,9 +1141,6 @@ namespace cryptonote
   //-----------------------------------------------------------------------------------------------
   bool core::add_new_tx(transaction& tx, const crypto::hash& tx_hash, const cryptonote::blobdata &blob, const crypto::hash& tx_prefix_hash, size_t tx_weight, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay)
   {
-    if (keeped_by_block)
-      get_blockchain_storage().on_new_tx_from_block(tx);
-
     if(m_mempool.have_tx(tx_hash))
     {
       LOG_PRINT_L2("tx " << tx_hash << "already have transaction in tx_pool");

src/cryptonote_core/cryptonote_core.h

@@ -754,6 +754,13 @@ namespace cryptonote
       */
      bool fluffy_blocks_enabled() const { return m_fluffy_blocks_enabled; }
 
+     /**
+      * @brief get whether transaction relay should be padded
+      *
+      * @return whether transaction relay should be padded
+      */
+     bool pad_transactions() const { return m_pad_transactions; }
+
      /**
       * @brief check a set of hashes against the precompiled hash set
       *
@@ -1013,6 +1020,7 @@ namespace cryptonote
 
      bool m_fluffy_blocks_enabled;
      bool m_offline;
+     bool m_pad_transactions;
    };
 }
 

src/cryptonote_protocol/cryptonote_protocol_defs.h

@@ -146,9 +146,11 @@ namespace cryptonote
     struct request
     {
       std::vector<blobdata>   txs;
+      std::string _; // padding
 
       BEGIN_KV_SERIALIZE_MAP()
         KV_SERIALIZE(txs)
+        KV_SERIALIZE(_)
       END_KV_SERIALIZE_MAP()
     };
   };

src/cryptonote_protocol/cryptonote_protocol_handler.inl

@@ -1726,8 +1726,39 @@ skip:
   bool t_cryptonote_protocol_handler<t_core>::relay_transactions(NOTIFY_NEW_TRANSACTIONS::request& arg, cryptonote_connection_context& exclude_context)
   {
     // no check for success, so tell core they're relayed unconditionally
+    const bool pad_transactions = m_core.pad_transactions();
+    size_t bytes = pad_transactions ? 9 /* header */ + 4 /* 1 + 'txs' */ + tools::get_varint_data(arg.txs.size()).size() : 0;
     for(auto tx_blob_it = arg.txs.begin(); tx_blob_it!=arg.txs.end(); ++tx_blob_it)
+    {
       m_core.on_transaction_relayed(*tx_blob_it);
+      if (pad_transactions)
+        bytes += tools::get_varint_data(tx_blob_it->size()).size() + tx_blob_it->size();
+    }
+
+    if (pad_transactions)
+    {
+      // stuff some dummy bytes in to stay safe from traffic volume analysis
+      static constexpr size_t granularity = 1024;
+      size_t padding = granularity - bytes % granularity;
+      const size_t overhead = 2 /* 1 + '_' */ + tools::get_varint_data(padding).size();
+      if (overhead > padding)
+        padding = 0;
+      else
+        padding -= overhead;
+      arg._ = std::string(padding, ' ');
+
+      std::string arg_buff;
+      epee::serialization::store_t_to_binary(arg, arg_buff);
+
+      // we probably lowballed the payload size a bit, so added a but too much. Fix this now.
+      size_t remove = arg_buff.size() % granularity;
+      if (remove > arg._.size())
+        arg._.clear();
+      else
+        arg._.resize(arg._.size() - remove);
+      // if the size of _ moved enough, we might lose byte in size encoding, we don't care
+    }
+
     return relay_post_notify<NOTIFY_NEW_TRANSACTIONS>(arg, exclude_context);
   }
   //------------------------------------------------------------------------------------------------------------------------

src/daemon/command_parser_executor.cpp

@@ -674,6 +674,31 @@ bool t_command_parser_executor::sync_info(const std::vector<std::string>& args)
   return m_executor.sync_info();
 }
 
+bool t_command_parser_executor::pop_blocks(const std::vector<std::string>& args)
+{
+  if (args.size() != 1)
+  {
+    std::cout << "Exactly one parameter is needed" << std::endl;
+    return false;
+  }
+
+  try
+  {
+    uint64_t nblocks = boost::lexical_cast<uint64_t>(args[0]);
+    if (nblocks < 1)
+    {
+      std::cout << "number of blocks must be greater than 0" << std::endl;
+      return false;
+    }
+    return m_executor.pop_blocks(nblocks);
+  }
+  catch (const boost::bad_lexical_cast&)
+  {
+    std::cout << "number of blocks must be a number greater than 0" << std::endl;
+  }
+  return false;
+}
+
 bool t_command_parser_executor::version(const std::vector<std::string>& args)
 {
   std::cout << "Wownero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")" << std::endl;

src/daemon/command_parser_executor.h

@@ -139,6 +139,8 @@ public:
 
   bool sync_info(const std::vector<std::string>& args);
 
+  bool pop_blocks(const std::vector<std::string>& args);
+
   bool version(const std::vector<std::string>& args);
 };
 

src/daemon/command_server.cpp

@@ -281,6 +281,12 @@ t_command_server::t_command_server(
     , std::bind(&t_command_parser_executor::sync_info, &m_parser, p::_1)
     , "Print information about the blockchain sync state."
     );
+    m_command_lookup.set_handler(
+      "pop_blocks"
+    , std::bind(&t_command_parser_executor::pop_blocks, &m_parser, p::_1)
+    , "pop_blocks <nblocks>"
+    , "Remove blocks from end of blockchain"
+    );
     m_command_lookup.set_handler(
       "version"
     , std::bind(&t_command_parser_executor::version, &m_parser, p::_1)

src/daemon/daemon.cpp

@@ -195,7 +195,6 @@ bool t_daemon::run(bool interactive)
 
     for(auto& rpc : mp_internals->rpcs)
       rpc->stop();
-    mp_internals->core.get().get_miner().stop();
     MGINFO("Node stopped.");
     return true;
   }
@@ -217,7 +216,6 @@ void t_daemon::stop()
   {
     throw std::runtime_error{"Can't stop stopped daemon"};
   }
-  mp_internals->core.get().get_miner().stop();
   mp_internals->p2p.stop();
   for(auto& rpc : mp_internals->rpcs)
     rpc->stop();

src/daemon/main.cpp

@@ -216,6 +216,16 @@ int main(int argc, char const * argv[])
     // after logs initialized
     tools::create_directories_if_necessary(data_dir.string());
 
+#ifdef STACK_TRACE
+    tools::set_stack_trace_log(log_file_path.filename().string());
+#endif // STACK_TRACE
+
+    if (!command_line::is_arg_defaulted(vm, daemon_args::arg_max_concurrency))
+      tools::set_max_concurrency(command_line::get_arg(vm, daemon_args::arg_max_concurrency));
+
+    // logging is now set up
+    MGINFO("Wownero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")");
+
     // If there are positional options, we're running a daemon command
     {
       auto command = command_line::get_arg(vm, daemon_args::arg_command);
@@ -276,16 +286,6 @@ int main(int argc, char const * argv[])
       }
     }
 
-#ifdef STACK_TRACE
-    tools::set_stack_trace_log(log_file_path.filename().string());
-#endif // STACK_TRACE
-
-    if (!command_line::is_arg_defaulted(vm, daemon_args::arg_max_concurrency))
-      tools::set_max_concurrency(command_line::get_arg(vm, daemon_args::arg_max_concurrency));
-
-    // logging is now set up
-    MGINFO("Wownero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")");
-
     MINFO("Moving from main() into the daemonize now.");
 
     return daemonizer::daemonize(argc, argv, daemonize::t_executor{}, vm) ? 0 : 1;

src/daemon/rpc_command_executor.cpp

@@ -1967,4 +1967,31 @@ bool t_rpc_command_executor::sync_info()
     return true;
 }
 
+bool t_rpc_command_executor::pop_blocks(uint64_t num_blocks)
+{
+  cryptonote::COMMAND_RPC_POP_BLOCKS::request req;
+  cryptonote::COMMAND_RPC_POP_BLOCKS::response res;
+  std::string fail_message = "pop_blocks failed";
+
+  req.nblocks = num_blocks;
+  if (m_is_rpc)
+  {
+    if (!m_rpc_client->rpc_request(req, res, "/pop_blocks", fail_message.c_str()))
+    {
+      return true;
+    }
+  }
+  else
+  {
+    if (!m_rpc_server->on_pop_blocks(req, res) || res.status != CORE_RPC_STATUS_OK)
+    {
+      tools::fail_msg_writer() << make_error(fail_message, res.status);
+      return true;
+    }
+  }
+  tools::success_msg_writer() << "new height: " << res.height;
+
+  return true;
+}
+
 }// namespace daemonize

src/daemon/rpc_command_executor.h

@@ -152,6 +152,8 @@ public:
   bool relay_tx(const std::string &txid);
 
   bool sync_info();
+
+  bool pop_blocks(uint64_t num_blocks);
 };
 
 } // namespace daemonize

src/device/device.hpp

@@ -80,6 +80,14 @@ namespace hw {
            return false;
     }
 
+    class i_device_callback {
+    public:
+        virtual void on_button_request() {}
+        virtual void on_pin_request(epee::wipeable_string & pin) {}
+        virtual void on_passphrase_request(bool on_device, epee::wipeable_string & passphrase) {}
+        virtual ~i_device_callback() = default;
+    };
+
     class device {
     protected:
         std::string  name;
@@ -129,6 +137,8 @@ namespace hw {
         virtual device_type get_type() const = 0;
 
         virtual device_protocol_t device_protocol() const { return PROTOCOL_DEFAULT; };
+        virtual void set_callback(i_device_callback * callback) {};
+        virtual void set_derivation_path(const std::string &derivation_path) {};
 
         /* ======================================================================= */
         /*  LOCKER                                                                 */

src/device_trezor/device_trezor.cpp

@@ -121,7 +121,8 @@ namespace trezor {
         const boost::optional<cryptonote::network_type> & network_type){
       AUTO_LOCK_CMD();
       require_connected();
-      test_ping();
+      device_state_reset_unsafe();
+      require_initialized();
 
       auto req = std::make_shared<messages::monero::MoneroGetAddress>();
       this->set_msg_addr<messages::monero::MoneroGetAddress>(req.get(), path, network_type);
@@ -136,7 +137,8 @@ namespace trezor {
         const boost::optional<cryptonote::network_type> & network_type){
       AUTO_LOCK_CMD();
       require_connected();
-      test_ping();
+      device_state_reset_unsafe();
+      require_initialized();
 
       auto req = std::make_shared<messages::monero::MoneroGetWatchKey>();
       this->set_msg_addr<messages::monero::MoneroGetWatchKey>(req.get(), path, network_type);
@@ -152,7 +154,8 @@ namespace trezor {
     {
       AUTO_LOCK_CMD();
       require_connected();
-      test_ping();
+      device_state_reset_unsafe();
+      require_initialized();
 
       std::shared_ptr<messages::monero::MoneroKeyImageExportInitRequest> req;
 
@@ -238,12 +241,11 @@ namespace trezor {
         cpend.construction_data = cdata.tx_data;
 
         // Transaction check
-        cryptonote::blobdata tx_blob;
+        try {
-        cryptonote::transaction tx_deserialized;
+          transaction_check(cdata, aux_data);
-        bool r = cryptonote::t_serializable_object_to_blob(cpend.tx, tx_blob);
+        } catch(const std::exception &e){
-        CHECK_AND_ASSERT_THROW_MES(r, "Transaction serialization failed");
+          throw exc::ProtocolException(std::string("Transaction verification failed: ") + e.what());
-        r = cryptonote::parse_and_validate_tx_from_blob(tx_blob, tx_deserialized);
+        }
-        CHECK_AND_ASSERT_THROW_MES(r, "Transaction deserialization failed");
 
         std::string key_images;
         bool all_are_txin_to_key = std::all_of(cdata.tx.vin.begin(), cdata.tx.vin.end(), [&](const cryptonote::txin_v& s_e) -> bool
@@ -283,7 +285,8 @@ namespace trezor {
     {
       AUTO_LOCK_CMD();
       require_connected();
-      test_ping();
+      device_state_reset_unsafe();
+      require_initialized();
 
       CHECK_AND_ASSERT_THROW_MES(idx < unsigned_tx.txes.size(), "Invalid transaction index");
       signer = std::make_shared<protocol::tx::Signer>(wallet, &unsigned_tx, idx, &aux_data);
@@ -294,6 +297,7 @@ namespace trezor {
       // Step: Init
       auto init_msg = signer->step_init();
       this->set_msg_addr(init_msg.get());
+      transaction_pre_check(init_msg);
 
       auto response = this->client_exchange<messages::monero::MoneroTransactionInitAck>(init_msg);
       signer->step_init_ack(response);
@@ -351,6 +355,59 @@ namespace trezor {
       signer->step_final_ack(ack_final);
     }
 
+    void device_trezor::transaction_pre_check(std::shared_ptr<messages::monero::MoneroTransactionInitRequest> init_msg)
+    {
+      CHECK_AND_ASSERT_THROW_MES(init_msg, "TransactionInitRequest is empty");
+      CHECK_AND_ASSERT_THROW_MES(init_msg->has_tsx_data(), "TransactionInitRequest has no transaction data");
+      CHECK_AND_ASSERT_THROW_MES(m_features, "Device state not initialized");  // make sure the caller did not reset features
+      const bool nonce_required = init_msg->tsx_data().has_payment_id() && init_msg->tsx_data().payment_id().size() > 0;
+
+      if (nonce_required){
+        // Versions 2.0.9 and lower do not support payment ID
+        CHECK_AND_ASSERT_THROW_MES(m_features->has_major_version() && m_features->has_minor_version() && m_features->has_patch_version(), "Invalid Trezor firmware version information");
+        const uint32_t vma = m_features->major_version();
+        const uint32_t vmi = m_features->minor_version();
+        const uint32_t vpa = m_features->patch_version();
+        if (vma < 2 || (vma == 2 && vmi == 0 && vpa <= 9)) {
+          throw exc::TrezorException("Trezor firmware 2.0.9 and lower does not support transactions with short payment IDs or integrated addresses. Please update.");
+        }
+      }
+    }
+
+    void device_trezor::transaction_check(const protocol::tx::TData & tdata, const hw::tx_aux_data & aux_data)
+    {
+      // Simple serialization check
+      cryptonote::blobdata tx_blob;
+      cryptonote::transaction tx_deserialized;
+      bool r = cryptonote::t_serializable_object_to_blob(tdata.tx, tx_blob);
+      CHECK_AND_ASSERT_THROW_MES(r, "Transaction serialization failed");
+      r = cryptonote::parse_and_validate_tx_from_blob(tx_blob, tx_deserialized);
+      CHECK_AND_ASSERT_THROW_MES(r, "Transaction deserialization failed");
+
+      // Extras check
+      std::vector<cryptonote::tx_extra_field> tx_extra_fields;
+      cryptonote::tx_extra_nonce nonce;
+
+      r = cryptonote::parse_tx_extra(tdata.tx.extra, tx_extra_fields);
+      CHECK_AND_ASSERT_THROW_MES(r, "tx.extra parsing failed");
+
+      const bool nonce_required = tdata.tsx_data.has_payment_id() && tdata.tsx_data.payment_id().size() > 0;
+      const bool has_nonce = cryptonote::find_tx_extra_field_by_type(tx_extra_fields, nonce);
+      CHECK_AND_ASSERT_THROW_MES(has_nonce == nonce_required, "Transaction nonce presence inconsistent");
+
+      if (nonce_required){
+        const std::string & payment_id = tdata.tsx_data.payment_id();
+        if (payment_id.size() == 32){
+          crypto::hash payment_id_long{};
+          CHECK_AND_ASSERT_THROW_MES(cryptonote::get_payment_id_from_tx_extra_nonce(nonce.nonce, payment_id_long), "Long payment ID not present");
+
+        } else if (payment_id.size() == 8){
+          crypto::hash8 payment_id_short{};
+          CHECK_AND_ASSERT_THROW_MES(cryptonote::get_encrypted_payment_id_from_tx_extra_nonce(nonce.nonce, payment_id_short), "Short payment ID not present");
+        }
+      }
+    }
+
 #else //WITH_DEVICE_TREZOR
 
     void register_all(std::map<std::string, std::unique_ptr<device>> &registry) {

src/device_trezor/device_trezor.hpp

@@ -57,9 +57,8 @@ namespace trezor {
    */
   class device_trezor : public hw::trezor::device_trezor_base, public hw::device_cold {
     protected:
-      // To speed up blockchain parsing the view key maybe handle here.
+      void transaction_pre_check(std::shared_ptr<messages::monero::MoneroTransactionInitRequest> init_msg);
-      crypto::secret_key viewkey;
+      void transaction_check(const protocol::tx::TData & tdata, const hw::tx_aux_data & aux_data);
-      bool has_view_key;
 
     public:
       device_trezor();

src/device_trezor/device_trezor_base.cpp

@@ -28,6 +28,9 @@
 //
 
 #include "device_trezor_base.hpp"
+#include <boost/algorithm/string/classification.hpp>
+#include <boost/algorithm/string/split.hpp>
+#include <boost/regex.hpp>
 
 namespace hw {
 namespace trezor {
@@ -36,10 +39,11 @@ namespace trezor {
 
 #undef MONERO_DEFAULT_LOG_CATEGORY
 #define MONERO_DEFAULT_LOG_CATEGORY "device.trezor"
+#define TREZOR_BIP44_HARDENED_ZERO 0x80000000
 
-    const uint32_t device_trezor_base::DEFAULT_BIP44_PATH[] = {0x8000002c, 0x80000080, 0x80000000};
+    const uint32_t device_trezor_base::DEFAULT_BIP44_PATH[] = {0x8000002c, 0x80000080};
 
-    device_trezor_base::device_trezor_base() {
+    device_trezor_base::device_trezor_base(): m_callback(nullptr) {
 
     }
 
@@ -61,7 +65,7 @@ namespace trezor {
     }
 
     bool device_trezor_base::set_name(const std::string & name) {
-      this->full_name = name;
+      this->m_full_name = name;
       this->name = "";
 
       auto delim = name.find(':');
@@ -73,10 +77,10 @@ namespace trezor {
     }
 
     const std::string device_trezor_base::get_name() const {
-      if (this->full_name.empty()) {
+      if (this->m_full_name.empty()) {
         return std::string("<disconnected:").append(this->name).append(">");
       }
-      return this->full_name;
+      return this->m_full_name;
     }
 
     bool device_trezor_base::init() {
@@ -135,6 +139,9 @@ namespace trezor {
     }
 
     bool device_trezor_base::disconnect() {
+      m_device_state.clear();
+      m_features.reset();
+
       if (m_transport){
         try {
           m_transport->close();
@@ -189,6 +196,25 @@ namespace trezor {
       }
     }
 
+    void device_trezor_base::require_initialized(){
+      if (!m_features){
+        throw exc::TrezorException("Device state not initialized");
+      }
+
+      if (m_features->has_bootloader_mode() && m_features->bootloader_mode()){
+        throw exc::TrezorException("Device is in the bootloader mode");
+      }
+
+      if (m_features->has_firmware_present() && !m_features->firmware_present()){
+        throw exc::TrezorException("Device has no firmware loaded");
+      }
+
+      // Hard requirement on initialized field, has to be there.
+      if (!m_features->has_initialized() || !m_features->initialized()){
+        throw exc::TrezorException("Device is not initialized");
+      }
+    }
+
     void device_trezor_base::call_ping_unsafe(){
       auto pingMsg = std::make_shared<messages::management::Ping>();
       pingMsg->set_message("PING");
@@ -213,7 +239,7 @@ namespace trezor {
     void device_trezor_base::write_raw(const google::protobuf::Message * msg){
       require_connected();
       CHECK_AND_ASSERT_THROW_MES(msg, "Empty message");
-      this->getTransport()->write(*msg);
+      this->get_transport()->write(*msg);
     }
 
     GenericMessage device_trezor_base::read_raw(){
@@ -221,7 +247,7 @@ namespace trezor {
       std::shared_ptr<google::protobuf::Message> msg_resp;
       hw::trezor::messages::MessageType msg_resp_type;
 
-      this->getTransport()->read(msg_resp, &msg_resp_type);
+      this->get_transport()->read(msg_resp, &msg_resp_type);
       return GenericMessage(msg_resp_type, msg_resp);
     }
 
@@ -252,6 +278,39 @@ namespace trezor {
       }
     }
 
+    void device_trezor_base::ensure_derivation_path() noexcept {
+      if (m_wallet_deriv_path.empty()){
+        m_wallet_deriv_path.push_back(TREZOR_BIP44_HARDENED_ZERO);  // default 0'
+      }
+    }
+
+    void device_trezor_base::set_derivation_path(const std::string &deriv_path){
+      this->m_wallet_deriv_path.clear();
+
+      if (deriv_path.empty() || deriv_path == "-"){
+        ensure_derivation_path();
+        return;
+      }
+
+      CHECK_AND_ASSERT_THROW_MES(deriv_path.size() <= 255, "Derivation path is too long");
+
+      std::vector<std::string> fields;
+      boost::split(fields, deriv_path, boost::is_any_of("/"));
+      CHECK_AND_ASSERT_THROW_MES(fields.size() <= 10, "Derivation path is too long");
+
+      boost::regex rgx("^([0-9]+)'?$");
+      boost::cmatch match;
+
+      this->m_wallet_deriv_path.reserve(fields.size());
+      for(const std::string & cur : fields){
+        const bool ok = boost::regex_match(cur.c_str(), match, rgx);
+        CHECK_AND_ASSERT_THROW_MES(ok, "Invalid wallet code: " << deriv_path << ". Invalid path element: " << cur);
+        CHECK_AND_ASSERT_THROW_MES(match[0].length() > 0, "Invalid wallet code: " << deriv_path << ". Invalid path element: " << cur);
+
+        const unsigned long cidx = std::stoul(match[0].str()) | TREZOR_BIP44_HARDENED_ZERO;
+        this->m_wallet_deriv_path.push_back((unsigned int)cidx);
+      }
+    }
 
     /* ======================================================================= */
     /*                              TREZOR PROTOCOL                            */
@@ -277,6 +336,25 @@ namespace trezor {
       return false;
     }
 
+    void device_trezor_base::device_state_reset_unsafe()
+    {
+      require_connected();
+      auto initMsg = std::make_shared<messages::management::Initialize>();
+
+      if(!m_device_state.empty()) {
+        initMsg->set_allocated_state(&m_device_state);
+      }
+
+      m_features = this->client_exchange<messages::management::Features>(initMsg);
+      initMsg->release_state();
+    }
+
+    void device_trezor_base::device_state_reset()
+    {
+      AUTO_LOCK_CMD();
+      device_state_reset_unsafe();
+    }
+
     void device_trezor_base::on_button_request(GenericMessage & resp, const messages::common::ButtonRequest * msg)
     {
       CHECK_AND_ASSERT_THROW_MES(msg, "Empty message");
@@ -324,7 +402,13 @@ namespace trezor {
         // TODO: remove passphrase from memory
         m.set_passphrase(passphrase.data(), passphrase.size());
       }
+
+      if (!m_device_state.empty()){
+        m.set_allocated_state(&m_device_state);
+      }
+
       resp = call_raw(&m);
+      m.release_state();
     }
 
     void device_trezor_base::on_passphrase_state_request(GenericMessage & resp, const messages::common::PassphraseStateRequest * msg)
@@ -332,10 +416,7 @@ namespace trezor {
       MDEBUG("on_passhprase_state_request");
       CHECK_AND_ASSERT_THROW_MES(msg, "Empty message");
 
-      if (m_callback){
+      m_device_state = msg->state();
-        m_callback->on_passphrase_state_request(msg->state());
-      }
-
       messages::common::PassphraseStateAck m;
       resp = call_raw(&m);
     }

src/device_trezor/device_trezor_base.hpp

@@ -57,17 +57,6 @@ namespace trezor {
 #ifdef WITH_DEVICE_TREZOR
   class device_trezor_base;
 
-  /**
-   * Trezor device callbacks
-   */
-  class trezor_callback {
-  public:
-    virtual void on_button_request() {};
-    virtual void on_pin_request(epee::wipeable_string & pin) {};
-    virtual void on_passphrase_request(bool on_device, epee::wipeable_string & passphrase) {};
-    virtual void on_passphrase_state_request(const std::string & state) {};
-  };
-
   /**
    * TREZOR device template with basic functions
    */
@@ -79,9 +68,12 @@ namespace trezor {
       mutable boost::mutex  command_locker;
 
       std::shared_ptr<Transport> m_transport;
-      std::shared_ptr<trezor_callback> m_callback;
+      i_device_callback * m_callback;
 
-      std::string full_name;
+      std::string m_full_name;
+      std::vector<unsigned int> m_wallet_deriv_path;
+      std::string m_device_state;  // returned after passphrase entry, session
+      std::shared_ptr<messages::management::Features> m_features;  // features from the last device reset
 
       cryptonote::network_type network_type;
 
@@ -90,8 +82,11 @@ namespace trezor {
       //
 
       void require_connected();
+      void require_initialized();
       void call_ping_unsafe();
       void test_ping();
+      void device_state_reset_unsafe();
+      void ensure_derivation_path() noexcept;
 
       // Communication methods
 
@@ -139,7 +134,7 @@ namespace trezor {
         // Scoped session closer
         BOOST_SCOPE_EXIT_ALL(&, this) {
           if (open_session){
-            this->getTransport()->close();
+            this->get_transport()->close();
           }
         };
 
@@ -187,9 +182,13 @@ namespace trezor {
             msg->add_address_n(x);
           }
         } else {
+          ensure_derivation_path();
           for (unsigned int i : DEFAULT_BIP44_PATH) {
             msg->add_address_n(i);
           }
+          for (unsigned int i : m_wallet_deriv_path) {
+            msg->add_address_n(i);
+          }
         }
 
         if (network_type){
@@ -212,16 +211,26 @@ namespace trezor {
     bool reset();
 
     // Default derivation path for Monero
-    static const uint32_t DEFAULT_BIP44_PATH[3];
+    static const uint32_t DEFAULT_BIP44_PATH[2];
 
-    std::shared_ptr<Transport> getTransport(){
+    std::shared_ptr<Transport> get_transport(){
       return m_transport;
     }
 
-    std::shared_ptr<trezor_callback> getCallback(){
+    void set_callback(i_device_callback * callback) override {
+      m_callback = callback;
+    }
+
+    i_device_callback * get_callback(){
       return m_callback;
     }
 
+    std::shared_ptr<messages::management::Features> & get_features() {
+      return m_features;
+    }
+
+    void set_derivation_path(const std::string &deriv_path) override;
+
     /* ======================================================================= */
     /*                              SETUP/TEARDOWN                             */
     /* ======================================================================= */
@@ -249,6 +258,11 @@ namespace trezor {
      */
     bool ping();
 
+    /**
+     * Performs Initialize call to the Trezor, resets to known state.
+     */
+    void device_state_reset();
+
     // Protocol callbacks
     void on_button_request(GenericMessage & resp, const messages::common::ButtonRequest * msg);
     void on_pin_request(GenericMessage & resp, const messages::common::PinMatrixRequest * msg);

src/device_trezor/trezor/tools/README.md

@@ -10,14 +10,28 @@ Install `protoc` for your distribution. Requirements:
 
 
 Soft requirement: Python 3, can be easily installed with [pyenv].
+If Python 3 is used there are no additional python dependencies.
 
-### Python 2
+Since Cmake 3.12 the `FindPython` module is used to locate the Python 
+interpreter in your system. It preferably searches for Python 3, if none
+is found, it searches for Python 2. 
 
-Workaround if there is no Python3 available:
+Lower version of the cmake uses another module which does not guarantee 
+ordering. If you want to override the selected python you can do it in 
+the following way:
 
 ```bash
-pip install backports.tempfile
+export TREZOR_PYTHON=`which python3`
-```
+``` 
+ 
+
+### Python 2.7+
+
+Python 3 has `tempfile.TemporaryDirectory` available but Python 2 lacks
+this class so the message generation code uses `backports.tempfile` package
+bundled in the repository.
+
+The minimal Python versions are 2.7 and 3.4
 
 ### Regenerate messages
 

src/device_trezor/trezor/tools/pb2cpp.py

@@ -14,12 +14,18 @@ import hashlib
 try:
     from tempfile import TemporaryDirectory
 except:
-    # Py2 backward compatibility, optionally installed by user
+    # Py2 backward compatibility, using bundled sources.
-    # pip install backports.tempfile
+    # Original source: pip install backports.tempfile
     try:
-        from backports.tempfile import TemporaryDirectory
+        # Try bundled python version
+        import sys
+        sys.path.append(os.path.dirname(__file__))
+        from py2backports.tempfile import TemporaryDirectory
+
     except:
-        raise EnvironmentError('TemporaryDirectory could not be imported. Try: pip install backports.tempfile')
+        raise EnvironmentError('Python 2.7+ or 3.4+ is required. '
+                               'TemporaryDirectory is not available in Python 2.'
+                               'Try to specify python to use, e.g.: "export TREZOR_PYTHON=`which python3`"')
 
 
 AUTO_HEADER = "# Automatically generated by pb2cpp\n"

src/device_trezor/trezor/tools/py2backports/tempfile.py

@@ -0,0 +1,72 @@
+"""
+https://github.com/pjdelport/backports.tempfile/blob/master/src/backports/tempfile.py
+Partial backport of Python 3.5's tempfile module:
+    TemporaryDirectory
+Backport modifications are marked with marked with "XXX backport".
+"""
+from __future__ import absolute_import
+
+import sys
+import warnings as _warnings
+from shutil import rmtree as _rmtree
+
+from py2backports.weakref import finalize
+
+
+# XXX backport:  Rather than backporting all of mkdtemp(), we just create a
+# thin wrapper implementing its Python 3.5 signature.
+if sys.version_info < (3, 5):
+    from tempfile import mkdtemp as old_mkdtemp
+
+    def mkdtemp(suffix=None, prefix=None, dir=None):
+        """
+        Wrap `tempfile.mkdtemp()` to make the suffix and prefix optional (like Python 3.5).
+        """
+        kwargs = {k: v for (k, v) in
+                  dict(suffix=suffix, prefix=prefix, dir=dir).items()
+                  if v is not None}
+        return old_mkdtemp(**kwargs)
+
+else:
+    from tempfile import mkdtemp
+
+
+# XXX backport: ResourceWarning was added in Python 3.2.
+# For earlier versions, fall back to RuntimeWarning instead.
+_ResourceWarning = RuntimeWarning if sys.version_info < (3, 2) else ResourceWarning
+
+
+class TemporaryDirectory(object):
+    """Create and return a temporary directory.  This has the same
+    behavior as mkdtemp but can be used as a context manager.  For
+    example:
+        with TemporaryDirectory() as tmpdir:
+            ...
+    Upon exiting the context, the directory and everything contained
+    in it are removed.
+    """
+
+    def __init__(self, suffix=None, prefix=None, dir=None):
+        self.name = mkdtemp(suffix, prefix, dir)
+        self._finalizer = finalize(
+            self, self._cleanup, self.name,
+            warn_message="Implicitly cleaning up {!r}".format(self))
+
+    @classmethod
+    def _cleanup(cls, name, warn_message):
+        _rmtree(name)
+        _warnings.warn(warn_message, _ResourceWarning)
+
+
+    def __repr__(self):
+        return "<{} {!r}>".format(self.__class__.__name__, self.name)
+
+    def __enter__(self):
+        return self.name
+
+    def __exit__(self, exc, value, tb):
+        self.cleanup()
+
+    def cleanup(self):
+        if self._finalizer.detach():
+            _rmtree(self.name)

src/device_trezor/trezor/tools/py2backports/weakref.py

@@ -0,0 +1,148 @@
+"""
+https://github.com/pjdelport/backports.weakref/blob/master/src/backports/weakref.py
+Partial backport of Python 3.6's weakref module:
+    finalize (new in Python 3.4)
+Backport modifications are marked with "XXX backport".
+"""
+from __future__ import absolute_import
+
+import itertools
+import sys
+from weakref import ref
+
+__all__ = ['finalize']
+
+
+class finalize(object):
+    """Class for finalization of weakrefable objects
+    finalize(obj, func, *args, **kwargs) returns a callable finalizer
+    object which will be called when obj is garbage collected. The
+    first time the finalizer is called it evaluates func(*arg, **kwargs)
+    and returns the result. After this the finalizer is dead, and
+    calling it just returns None.
+    When the program exits any remaining finalizers for which the
+    atexit attribute is true will be run in reverse order of creation.
+    By default atexit is true.
+    """
+
+    # Finalizer objects don't have any state of their own.  They are
+    # just used as keys to lookup _Info objects in the registry.  This
+    # ensures that they cannot be part of a ref-cycle.
+
+    __slots__ = ()
+    _registry = {}
+    _shutdown = False
+    _index_iter = itertools.count()
+    _dirty = False
+    _registered_with_atexit = False
+
+    class _Info(object):
+        __slots__ = ("weakref", "func", "args", "kwargs", "atexit", "index")
+
+    def __init__(self, obj, func, *args, **kwargs):
+        if not self._registered_with_atexit:
+            # We may register the exit function more than once because
+            # of a thread race, but that is harmless
+            import atexit
+            atexit.register(self._exitfunc)
+            finalize._registered_with_atexit = True
+        info = self._Info()
+        info.weakref = ref(obj, self)
+        info.func = func
+        info.args = args
+        info.kwargs = kwargs or None
+        info.atexit = True
+        info.index = next(self._index_iter)
+        self._registry[self] = info
+        finalize._dirty = True
+
+    def __call__(self, _=None):
+        """If alive then mark as dead and return func(*args, **kwargs);
+        otherwise return None"""
+        info = self._registry.pop(self, None)
+        if info and not self._shutdown:
+            return info.func(*info.args, **(info.kwargs or {}))
+
+    def detach(self):
+        """If alive then mark as dead and return (obj, func, args, kwargs);
+        otherwise return None"""
+        info = self._registry.get(self)
+        obj = info and info.weakref()
+        if obj is not None and self._registry.pop(self, None):
+            return (obj, info.func, info.args, info.kwargs or {})
+
+    def peek(self):
+        """If alive then return (obj, func, args, kwargs);
+        otherwise return None"""
+        info = self._registry.get(self)
+        obj = info and info.weakref()
+        if obj is not None:
+            return (obj, info.func, info.args, info.kwargs or {})
+
+    @property
+    def alive(self):
+        """Whether finalizer is alive"""
+        return self in self._registry
+
+    @property
+    def atexit(self):
+        """Whether finalizer should be called at exit"""
+        info = self._registry.get(self)
+        return bool(info) and info.atexit
+
+    @atexit.setter
+    def atexit(self, value):
+        info = self._registry.get(self)
+        if info:
+            info.atexit = bool(value)
+
+    def __repr__(self):
+        info = self._registry.get(self)
+        obj = info and info.weakref()
+        if obj is None:
+            return '<%s object at %#x; dead>' % (type(self).__name__, id(self))
+        else:
+            return '<%s object at %#x; for %r at %#x>' % \
+                   (type(self).__name__, id(self), type(obj).__name__, id(obj))
+
+    @classmethod
+    def _select_for_exit(cls):
+        # Return live finalizers marked for exit, oldest first
+        L = [(f,i) for (f,i) in cls._registry.items() if i.atexit]
+        L.sort(key=lambda item:item[1].index)
+        return [f for (f,i) in L]
+
+    @classmethod
+    def _exitfunc(cls):
+        # At shutdown invoke finalizers for which atexit is true.
+        # This is called once all other non-daemonic threads have been
+        # joined.
+        reenable_gc = False
+        try:
+            if cls._registry:
+                import gc
+                if gc.isenabled():
+                    reenable_gc = True
+                    gc.disable()
+                pending = None
+                while True:
+                    if pending is None or finalize._dirty:
+                        pending = cls._select_for_exit()
+                        finalize._dirty = False
+                    if not pending:
+                        break
+                    f = pending.pop()
+                    try:
+                        # gc is disabled, so (assuming no daemonic
+                        # threads) the following is the only line in
+                        # this function which might trigger creation
+                        # of a new finalizer
+                        f()
+                    except Exception:
+                        sys.excepthook(*sys.exc_info())
+                    assert f not in cls._registry
+        finally:
+            # prevent any more finalizers from executing during shutdown
+            finalize._shutdown = True
+            if reenable_gc:
+                gc.enable()

src/device_trezor/trezor/transport.cpp

@@ -840,7 +840,7 @@ namespace trezor{
       throw exc::DeviceAcquireException("Unable to claim libusb device");
     }
 
-    m_conn_count += 1;
+    m_conn_count = 1;
     m_proto->session_begin(*this);
     
 #undef TREZOR_DESTROY_SESSION

src/ringct/rctSigs.cpp

@@ -46,13 +46,34 @@ using namespace std;
 
 namespace
 {
-    rct::Bulletproof make_dummy_bulletproof(size_t n_outs)
+    rct::Bulletproof make_dummy_bulletproof(const std::vector<uint64_t> &outamounts, rct::keyV &C, rct::keyV &masks)
     {
+        const size_t n_outs = outamounts.size();
         const rct::key I = rct::identity();
         size_t nrl = 0;
         while ((1u << nrl) < n_outs)
           ++nrl;
         nrl += 6;
+
+        C.resize(n_outs);
+        masks.resize(n_outs);
+        for (size_t i = 0; i < n_outs; ++i)
+        {
+            masks[i] = I;
+            rct::key sv8, sv;
+            sv = rct::zero();
+            sv.bytes[0] = outamounts[i] & 255;
+            sv.bytes[1] = (outamounts[i] >> 8) & 255;
+            sv.bytes[2] = (outamounts[i] >> 16) & 255;
+            sv.bytes[3] = (outamounts[i] >> 24) & 255;
+            sv.bytes[4] = (outamounts[i] >> 32) & 255;
+            sv.bytes[5] = (outamounts[i] >> 40) & 255;
+            sv.bytes[6] = (outamounts[i] >> 48) & 255;
+            sv.bytes[7] = (outamounts[i] >> 56) & 255;
+            sc_mul(sv8.bytes, sv.bytes, rct::INV_EIGHT.bytes);
+            rct::addKeys2(C[i], rct::INV_EIGHT, sv8, rct::H);
+        }
+
         return rct::Bulletproof{rct::keyV(n_outs, I), I, I, I, I, I, I, rct::keyV(nrl, I), rct::keyV(nrl, I), I, I, I};
     }
 }
@@ -769,9 +790,7 @@ namespace rct {
                 if (hwdev.get_mode() == hw::device::TRANSACTION_CREATE_FAKE)
                 {
                     // use a fake bulletproof for speed
-                    rv.p.bulletproofs.push_back(make_dummy_bulletproof(outamounts.size()));
+                    rv.p.bulletproofs.push_back(make_dummy_bulletproof(outamounts, C, masks));
-                    C = rct::keyV(outamounts.size(), I);
-                    masks = rct::keyV(outamounts.size(), I);
                 }
                 else
                 {
@@ -799,9 +818,7 @@ namespace rct {
                 if (hwdev.get_mode() == hw::device::TRANSACTION_CREATE_FAKE)
                 {
                     // use a fake bulletproof for speed
-                    rv.p.bulletproofs.push_back(make_dummy_bulletproof(batch_amounts.size()));
+                    rv.p.bulletproofs.push_back(make_dummy_bulletproof(batch_amounts, C, masks));
-                    C = rct::keyV(batch_amounts.size(), I);
-                    masks = rct::keyV(batch_amounts.size(), I);
                 }
                 else
                 {

src/ringct/rctTypes.h

@@ -187,7 +187,8 @@ namespace rct {
       rct::keyV L, R;
       rct::key a, b, t;
 
-      Bulletproof() {}
+      Bulletproof():
+        A({}), S({}), T1({}), T2({}), taux({}), mu({}), a({}), b({}), t({}) {}
       Bulletproof(const rct::key &V, const rct::key &A, const rct::key &S, const rct::key &T1, const rct::key &T2, const rct::key &taux, const rct::key &mu, const rct::keyV &L, const rct::keyV &R, const rct::key &a, const rct::key &b, const rct::key &t):
         V({V}), A(A), S(S), T1(T1), T2(T2), taux(taux), mu(mu), L(L), R(R), a(a), b(b), t(t) {}
       Bulletproof(const rct::keyV &V, const rct::key &A, const rct::key &S, const rct::key &T1, const rct::key &T2, const rct::key &taux, const rct::key &mu, const rct::keyV &L, const rct::keyV &R, const rct::key &a, const rct::key &b, const rct::key &t):

src/rpc/core_rpc_server.cpp

@@ -2032,6 +2032,18 @@ namespace cryptonote
     return true;
   }
   //------------------------------------------------------------------------------------------------------------------------------
+  bool core_rpc_server::on_pop_blocks(const COMMAND_RPC_POP_BLOCKS::request& req, COMMAND_RPC_POP_BLOCKS::response& res)
+  {
+    PERF_TIMER(on_pop_blocks);
+
+    m_core.get_blockchain_storage().pop_blocks(req.nblocks);
+
+    res.height = m_core.get_current_blockchain_height();
+    res.status = CORE_RPC_STATUS_OK;
+
+    return true;
+  }
+  //------------------------------------------------------------------------------------------------------------------------------
   bool core_rpc_server::on_relay_tx(const COMMAND_RPC_RELAY_TX::request& req, COMMAND_RPC_RELAY_TX::response& res, epee::json_rpc::error& error_resp)
   {
     PERF_TIMER(on_relay_tx);

src/rpc/core_rpc_server.h

@@ -117,6 +117,7 @@ namespace cryptonote
       MAP_URI_AUTO_JON2("/get_outs", on_get_outs, COMMAND_RPC_GET_OUTPUTS)      
       MAP_URI_AUTO_JON2_IF("/update", on_update, COMMAND_RPC_UPDATE, !m_restricted)
       MAP_URI_AUTO_BIN2("/get_output_distribution.bin", on_get_output_distribution_bin, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION)
+      MAP_URI_AUTO_JON2_IF("/pop_blocks", on_pop_blocks, COMMAND_RPC_POP_BLOCKS, !m_restricted)
       BEGIN_JSON_RPC_MAP("/json_rpc")
         MAP_JON_RPC("get_block_count",           on_getblockcount,              COMMAND_RPC_GETBLOCKCOUNT)
         MAP_JON_RPC("getblockcount",             on_getblockcount,              COMMAND_RPC_GETBLOCKCOUNT)
@@ -188,6 +189,7 @@ namespace cryptonote
     bool on_stop_save_graph(const COMMAND_RPC_STOP_SAVE_GRAPH::request& req, COMMAND_RPC_STOP_SAVE_GRAPH::response& res);
     bool on_update(const COMMAND_RPC_UPDATE::request& req, COMMAND_RPC_UPDATE::response& res);
     bool on_get_output_distribution_bin(const COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::request& req, COMMAND_RPC_GET_OUTPUT_DISTRIBUTION::response& res);
+    bool on_pop_blocks(const COMMAND_RPC_POP_BLOCKS::request& req, COMMAND_RPC_POP_BLOCKS::response& res);
     
     //json_rpc
     bool on_getblockcount(const COMMAND_RPC_GETBLOCKCOUNT::request& req, COMMAND_RPC_GETBLOCKCOUNT::response& res);

src/rpc/core_rpc_server_commands_defs.h

@@ -2328,4 +2328,27 @@ namespace cryptonote
     };
   };
 
+  struct COMMAND_RPC_POP_BLOCKS
+  {
+    struct request
+    {
+      uint64_t nblocks;
+
+      BEGIN_KV_SERIALIZE_MAP()
+        KV_SERIALIZE(nblocks);
+      END_KV_SERIALIZE_MAP()
+    };
+
+    struct response
+    {
+      std::string status;
+      uint64_t height;
+
+      BEGIN_KV_SERIALIZE_MAP()
+        KV_SERIALIZE(status)
+        KV_SERIALIZE(height)
+      END_KV_SERIALIZE_MAP()
+    };
+  };
+
 }

src/rpc/rpc_handler.cpp

@@ -43,8 +43,25 @@ namespace rpc
 
       std::vector<std::uint64_t> distribution;
       std::uint64_t start_height, base;
-      if (!f(amount, from_height, to_height, start_height, distribution, base))
+
-        return boost::none;
+      // see if we can extend the cache - a common case
+      if (d.cached && amount == 0 && d.cached_from == from_height && to_height > d.cached_to)
+      {
+        std::vector<std::uint64_t> new_distribution;
+        if (!f(amount, d.cached_to + 1, to_height, start_height, new_distribution, base))
+          return boost::none;
+        distribution = d.cached_distribution;
+        distribution.reserve(distribution.size() + new_distribution.size());
+        for (const auto &e: new_distribution)
+          distribution.push_back(e);
+        start_height = d.cached_start_height;
+        base = d.cached_base;
+      }
+      else
+      {
+        if (!f(amount, from_height, to_height, start_height, distribution, base))
+          return boost::none;
+      }
 
       if (to_height > 0 && to_height >= from_height)
       {

src/simplewallet/simplewallet.cpp

@@ -3790,6 +3790,7 @@ boost::optional<epee::wipeable_string> simple_wallet::new_wallet(const boost::pr
 {
   auto rc = tools::wallet2::make_new(vm, false, password_prompter);
   m_wallet = std::move(rc.first);
+  m_wallet->callback(this);
   if (!m_wallet)
   {
     return {};
@@ -3807,9 +3808,11 @@ boost::optional<epee::wipeable_string> simple_wallet::new_wallet(const boost::pr
     m_wallet->set_refresh_from_block_height(m_restore_height);
 
   auto device_desc = tools::wallet2::device_name_option(vm);
+  auto device_derivation_path = tools::wallet2::device_derivation_path_option(vm);
   try
   {
     bool create_address_file = command_line::get_arg(vm, arg_create_address_file);
+    m_wallet->device_derivation_path(device_derivation_path);
     m_wallet->restore(m_wallet_file, std::move(rc.second).password(), device_desc.empty() ? "Ledger" : device_desc, create_address_file);
     message_writer(console_color_white, true) << tr("Generated new wallet on hw device: ")
       << m_wallet->get_account().get_public_address_str(m_wallet->nettype());
@@ -3897,7 +3900,7 @@ bool simple_wallet::open_wallet(const boost::program_options::variables_map& vm)
   epee::wipeable_string password;
   try
   {
-    auto rc = tools::wallet2::make_from_file(vm, false, m_wallet_file, password_prompter);
+    auto rc = tools::wallet2::make_from_file(vm, false, "", password_prompter);
     m_wallet = std::move(rc.first);
     password = std::move(std::move(rc.second).password());
     if (!m_wallet)
@@ -3905,6 +3908,8 @@ bool simple_wallet::open_wallet(const boost::program_options::variables_map& vm)
       return false;
     }
 
+    m_wallet->callback(this);
+    m_wallet->load(m_wallet_file, password);
     std::string prefix;
     bool ready;
     uint32_t threshold, total;
@@ -4308,6 +4313,61 @@ boost::optional<epee::wipeable_string> simple_wallet::on_get_password(const char
   return pwd_container->password();
 }
 //----------------------------------------------------------------------------------------------------
+void simple_wallet::on_button_request()
+{
+  message_writer(console_color_white, false) << tr("Device requires attention");
+}
+//----------------------------------------------------------------------------------------------------
+void simple_wallet::on_pin_request(epee::wipeable_string & pin)
+{
+#ifdef HAVE_READLINE
+  rdln::suspend_readline pause_readline;
+#endif
+  std::string msg = tr("Enter device PIN");
+  auto pwd_container = tools::password_container::prompt(false, msg.c_str());
+  THROW_WALLET_EXCEPTION_IF(!pwd_container, tools::error::password_entry_failed, tr("Failed to read device PIN"));
+  pin = pwd_container->password();
+}
+//----------------------------------------------------------------------------------------------------
+void simple_wallet::on_passphrase_request(bool on_device, epee::wipeable_string & passphrase)
+{
+  if (on_device){
+    message_writer(console_color_white, true) << tr("Please enter the device passphrase on the device");
+    return;
+  }
+
+#ifdef HAVE_READLINE
+  rdln::suspend_readline pause_readline;
+#endif
+  std::string msg = tr("Enter device passphrase");
+  auto pwd_container = tools::password_container::prompt(false, msg.c_str());
+  THROW_WALLET_EXCEPTION_IF(!pwd_container, tools::error::password_entry_failed, tr("Failed to read device passphrase"));
+  passphrase = pwd_container->password();
+}
+//----------------------------------------------------------------------------------------------------
+void simple_wallet::on_refresh_finished(uint64_t start_height, uint64_t fetched_blocks, bool is_init, bool received_money)
+{
+  // Key image sync after the first refresh
+  if (!m_wallet->get_account().get_device().has_tx_cold_sign()) {
+    return;
+  }
+
+  if (!received_money || m_wallet->get_device_last_key_image_sync() != 0) {
+    return;
+  }
+
+  // Finished first refresh for HW device and money received -> KI sync
+  message_writer() << "\n" << tr("The first refresh has finished for the HW-based wallet with received money. hw_key_images_sync is needed. ");
+
+  std::string accepted = input_line(tr("Do you want to do it now? (Y/Yes/N/No): "));
+  if (std::cin.eof() || !command_line::is_yes(accepted)) {
+    message_writer(console_color_red, false) << tr("hw_key_images_sync skipped. Run command manually before a transfer.");
+    return;
+  }
+
+  key_images_sync_intern();
+}
+//----------------------------------------------------------------------------------------------------
 bool simple_wallet::refresh_main(uint64_t start_height, enum ResetType reset, bool is_init)
 {
   if (!try_connect_to_daemon(is_init))
@@ -4325,13 +4385,14 @@ bool simple_wallet::refresh_main(uint64_t start_height, enum ResetType reset, bo
   message_writer() << tr("Starting refresh...");
 
   uint64_t fetched_blocks = 0;
+  bool received_money = false;
   bool ok = false;
   std::ostringstream ss;
   try
   {
     m_in_manual_refresh.store(true, std::memory_order_relaxed);
     epee::misc_utils::auto_scope_leave_caller scope_exit_handler = epee::misc_utils::create_scope_leave_handler([&](){m_in_manual_refresh.store(false, std::memory_order_relaxed);});
-    m_wallet->refresh(m_wallet->is_trusted_daemon(), start_height, fetched_blocks);
+    m_wallet->refresh(m_wallet->is_trusted_daemon(), start_height, fetched_blocks, received_money);
     ok = true;
     // Clear line "Height xxx of xxx"
     std::cout << "\r                                                                \r";
@@ -4339,6 +4400,7 @@ bool simple_wallet::refresh_main(uint64_t start_height, enum ResetType reset, bo
     if (is_init)
       print_accounts();
     show_balance_unlocked();
+    on_refresh_finished(start_height, fetched_blocks, is_init, received_money);
   }
   catch (const tools::error::daemon_busy&)
   {
@@ -5946,12 +6008,6 @@ bool simple_wallet::sweep_below(const std::vector<std::string> &args_)
 //----------------------------------------------------------------------------------------------------
 bool simple_wallet::donate(const std::vector<std::string> &args_)
 {
-  if(m_wallet->nettype() != cryptonote::MAINNET)
-  {
-    fail_msg_writer() << tr("donations are not enabled on the testnet or on the stagenet");
-    return true;
-  }
-
   std::vector<std::string> local_args = args_;
   if(local_args.empty() || local_args.size() > 5)
   {
@@ -5973,11 +6029,30 @@ bool simple_wallet::donate(const std::vector<std::string> &args_)
   amount_str = local_args.back();
   local_args.pop_back();
   // push back address, amount, payment id
-  local_args.push_back(MONERO_DONATION_ADDR);
+  std::string address_str;
+  if (m_wallet->nettype() != cryptonote::MAINNET)
+  {
+    // if not mainnet, convert donation address string to the relevant network type
+    address_parse_info info;
+    if (!cryptonote::get_account_address_from_str(info, cryptonote::MAINNET, MONERO_DONATION_ADDR))
+    {
+      fail_msg_writer() << tr("Failed to parse donation address: ") << MONERO_DONATION_ADDR;
+      return true;
+    }
+    address_str = cryptonote::get_account_address_as_str(m_wallet->nettype(), info.is_subaddress, info.address);
+  }
+  else
+  {
+    address_str = MONERO_DONATION_ADDR;
+  }
+  local_args.push_back(address_str);
   local_args.push_back(amount_str);
   if (!payment_id_str.empty())
     local_args.push_back(payment_id_str);
-  message_writer() << (boost::format(tr("Donating %s %s to The Monero Project (donate.getmonero.org or %s).")) % amount_str % cryptonote::get_unit(cryptonote::get_default_decimal_point()) % MONERO_DONATION_ADDR).str();
+  if (m_wallet->nettype() == cryptonote::MAINNET)
+    message_writer() << (boost::format(tr("Donating %s %s to The Monero Project (donate.getmonero.org or %s).")) % amount_str % cryptonote::get_unit(cryptonote::get_default_decimal_point()) % MONERO_DONATION_ADDR).str();
+  else
+    message_writer() << (boost::format(tr("Donating %s %s to %s.")) % amount_str % cryptonote::get_unit(cryptonote::get_default_decimal_point()) % address_str).str();
   transfer(local_args);
   return true;
 }
@@ -8096,13 +8171,13 @@ bool simple_wallet::hw_key_images_sync(const std::vector<std::string> &args)
     fail_msg_writer() << tr("hw wallet does not support cold KI sync");
     return true;
   }
-  if (!m_wallet->is_trusted_daemon())
-  {
-    fail_msg_writer() << tr("this command requires a trusted daemon. Enable with --trusted-daemon");
-    return true;
-  }
 
   LOCK_IDLE_SCOPE();
+  key_images_sync_intern();
+  return true;
+}
+//----------------------------------------------------------------------------------------------------
+void simple_wallet::key_images_sync_intern(){
   try
   {
     message_writer(console_color_white, false) << tr("Please confirm the key image sync on the device");
@@ -8111,19 +8186,23 @@ bool simple_wallet::hw_key_images_sync(const std::vector<std::string> &args)
     uint64_t height = m_wallet->cold_key_image_sync(spent, unspent);
     if (height > 0)
     {
-      success_msg_writer() << tr("Signed key images imported to height ") << height << ", "
+      success_msg_writer() << tr("Key images synchronized to height ") << height;
-          << print_money(spent) << tr(" spent, ") << print_money(unspent) << tr(" unspent");
+      if (!m_wallet->is_trusted_daemon())
-    } else {
+      {
+        message_writer() << tr("Running untrusted daemon, cannot determine which transaction output is spent. Use a trusted daemon with --trusted-daemon and run rescan_spent");
+      } else
+      {
+        success_msg_writer() << print_money(spent) << tr(" spent, ") << print_money(unspent) << tr(" unspent");
+      }
+    }
+    else {
       fail_msg_writer() << tr("Failed to import key images");
     }
   }
   catch (const std::exception &e)
   {
     fail_msg_writer() << tr("Failed to import key images: ") << e.what();
-    return true;
   }
-
-  return true;
 }
 //----------------------------------------------------------------------------------------------------
 bool simple_wallet::hw_reconnect(const std::vector<std::string> &args)

src/simplewallet/simplewallet.h

@@ -241,6 +241,8 @@ namespace cryptonote
     bool print_ring_members(const std::vector<tools::wallet2::pending_tx>& ptx_vector, std::ostream& ostr);
     std::string get_prompt() const;
     bool print_seed(bool encrypted);
+    void key_images_sync_intern();
+    void on_refresh_finished(uint64_t start_height, uint64_t fetched_blocks, bool is_init, bool received_money);
 
     struct transfer_view
     {
@@ -287,6 +289,9 @@ namespace cryptonote
     virtual void on_money_spent(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& in_tx, uint64_t amount, const cryptonote::transaction& spend_tx, const cryptonote::subaddress_index& subaddr_index);
     virtual void on_skip_transaction(uint64_t height, const crypto::hash &txid, const cryptonote::transaction& tx);
     virtual boost::optional<epee::wipeable_string> on_get_password(const char *reason);
+    virtual void on_button_request();
+    virtual void on_pin_request(epee::wipeable_string & pin);
+    virtual void on_passphrase_request(bool on_device, epee::wipeable_string & passphrase);
     //----------------------------------------------------------
 
     friend class refresh_progress_reporter_t;

src/wallet/wallet2.cpp

@@ -222,6 +222,7 @@ struct options {
   };
   const command_line::arg_descriptor<uint64_t> kdf_rounds = {"kdf-rounds", tools::wallet2::tr("Number of rounds for the key derivation function"), 1};
   const command_line::arg_descriptor<std::string> hw_device = {"hw-device", tools::wallet2::tr("HW device to use"), ""};
+  const command_line::arg_descriptor<std::string> hw_device_derivation_path = {"hw-device-deriv-path", tools::wallet2::tr("HW device wallet derivation path (e.g., SLIP-10)"), ""};
   const command_line::arg_descriptor<std::string> tx_notify = { "tx-notify" , "Run a program for each new incoming transaction, '%s' will be replaced by the transaction hash" , "" };
 };
 
@@ -274,6 +275,7 @@ std::unique_ptr<tools::wallet2> make_basic(const boost::program_options::variabl
   auto daemon_host = command_line::get_arg(vm, opts.daemon_host);
   auto daemon_port = command_line::get_arg(vm, opts.daemon_port);
   auto device_name = command_line::get_arg(vm, opts.hw_device);
+  auto device_derivation_path = command_line::get_arg(vm, opts.hw_device_derivation_path);
 
   THROW_WALLET_EXCEPTION_IF(!daemon_address.empty() && !daemon_host.empty() && 0 != daemon_port,
       tools::error::wallet_internal_error, tools::wallet2::tr("can't specify daemon host or port more than once"));
@@ -329,6 +331,7 @@ std::unique_ptr<tools::wallet2> make_basic(const boost::program_options::variabl
   boost::filesystem::path ringdb_path = command_line::get_arg(vm, opts.shared_ringdb_dir);
   wallet->set_ring_database(ringdb_path.string());
   wallet->device_name(device_name);
+  wallet->device_derivation_path(device_derivation_path);
 
   try
   {
@@ -838,6 +841,24 @@ wallet_keys_unlocker::~wallet_keys_unlocker()
   }
 }
 
+void wallet_device_callback::on_button_request()
+{
+  if (wallet)
+    wallet->on_button_request();
+}
+
+void wallet_device_callback::on_pin_request(epee::wipeable_string & pin)
+{
+  if (wallet)
+    wallet->on_pin_request(pin);
+}
+
+void wallet_device_callback::on_passphrase_request(bool on_device, epee::wipeable_string & passphrase)
+{
+  if (wallet)
+    wallet->on_passphrase_request(on_device, passphrase);
+}
+
 wallet2::wallet2(network_type nettype, uint64_t kdf_rounds, bool unattended):
   m_multisig_rescan_info(NULL),
   m_multisig_rescan_k(NULL),
@@ -891,7 +912,8 @@ wallet2::wallet2(network_type nettype, uint64_t kdf_rounds, bool unattended):
   m_ringdb(),
   m_last_block_reward(0),
   m_encrypt_keys_after_refresh(boost::none),
-  m_unattended(unattended)
+  m_unattended(unattended),
+  m_device_last_key_image_sync(0)
 {
 }
 
@@ -914,6 +936,11 @@ std::string wallet2::device_name_option(const boost::program_options::variables_
   return command_line::get_arg(vm, options().hw_device);
 }
 
+std::string wallet2::device_derivation_path_option(const boost::program_options::variables_map &vm)
+{
+  return command_line::get_arg(vm, options().hw_device_derivation_path);
+}
+
 void wallet2::init_options(boost::program_options::options_description& desc_params)
 {
   const options opts{};
@@ -930,6 +957,7 @@ void wallet2::init_options(boost::program_options::options_description& desc_par
   command_line::add_arg(desc_params, opts.shared_ringdb_dir);
   command_line::add_arg(desc_params, opts.kdf_rounds);
   command_line::add_arg(desc_params, opts.hw_device);
+  command_line::add_arg(desc_params, opts.hw_device_derivation_path);
   command_line::add_arg(desc_params, opts.tx_notify);
 }
 
@@ -949,7 +977,7 @@ std::pair<std::unique_ptr<wallet2>, password_container> wallet2::make_from_file(
     return {nullptr, password_container{}};
   }
   auto wallet = make_basic(vm, unattended, opts, password_prompter);
-  if (wallet)
+  if (wallet && !wallet_file.empty())
   {
     wallet->load(wallet_file, pwd->password());
   }
@@ -1091,15 +1119,17 @@ bool wallet2::reconnect_device()
   hw::device &hwdev = lookup_device(m_device_name);
   hwdev.set_name(m_device_name);
   hwdev.set_network_type(m_nettype);
+  hwdev.set_derivation_path(m_device_derivation_path);
+  hwdev.set_callback(get_device_callback());
   r = hwdev.init();
   if (!r){
-    LOG_PRINT_L2("Could not init device");
+    MERROR("Could not init device");
     return false;
   }
 
   r = hwdev.connect();
   if (!r){
-    LOG_PRINT_L2("Could not connect to the device");
+    MERROR("Could not connect to the device");
     return false;
   }
 
@@ -2998,6 +3028,7 @@ bool wallet2::clear()
   m_subaddresses.clear();
   m_subaddress_labels.clear();
   m_multisig_rounds_passed = 0;
+  m_device_last_key_image_sync = 0;
   return true;
 }
 
@@ -3159,6 +3190,9 @@ bool wallet2::store_keys(const std::string& keys_file_name, const epee::wipeable
   value.SetString(m_device_name.c_str(), m_device_name.size());
   json.AddMember("device_name", value, json.GetAllocator());
 
+  value.SetString(m_device_derivation_path.c_str(), m_device_derivation_path.size());
+  json.AddMember("device_derivation_path", value, json.GetAllocator());
+
   // Serialize the JSON object
   rapidjson::StringBuffer buffer;
   rapidjson::Writer<rapidjson::StringBuffer> writer(buffer);
@@ -3278,6 +3312,7 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_
     m_subaddress_lookahead_major = SUBADDRESS_LOOKAHEAD_MAJOR;
     m_subaddress_lookahead_minor = SUBADDRESS_LOOKAHEAD_MINOR;
     m_device_name = "";
+    m_device_derivation_path = "";
     m_key_device_type = hw::device::device_type::SOFTWARE;
     encrypted_secret_keys = false;
   }
@@ -3445,6 +3480,9 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_
         m_device_name = m_key_device_type == hw::device::device_type::LEDGER ? "Ledger" : "default";
       }
     }
+
+    GET_FIELD_FROM_JSON_RETURN_ON_ERROR(json, device_derivation_path, std::string, String, false, std::string());
+    m_device_derivation_path = field_device_derivation_path;
   }
   else
   {
@@ -3459,6 +3497,8 @@ bool wallet2::load_keys(const std::string& keys_file_name, const epee::wipeable_
     hw::device &hwdev = lookup_device(m_device_name);
     THROW_WALLET_EXCEPTION_IF(!hwdev.set_name(m_device_name), error::wallet_internal_error, "Could not set device name " + m_device_name);
     hwdev.set_network_type(m_nettype);
+    hwdev.set_derivation_path(m_device_derivation_path);
+    hwdev.set_callback(get_device_callback());
     THROW_WALLET_EXCEPTION_IF(!hwdev.init(), error::wallet_internal_error, "Could not initialize the device " + m_device_name);
     THROW_WALLET_EXCEPTION_IF(!hwdev.connect(), error::wallet_internal_error, "Could not connect to the device " + m_device_name);
     m_account.set_device(hwdev);
@@ -3965,6 +4005,8 @@ void wallet2::restore(const std::string& wallet_, const epee::wipeable_string& p
   auto &hwdev = lookup_device(device_name);
   hwdev.set_name(device_name);
   hwdev.set_network_type(m_nettype);
+  hwdev.set_derivation_path(m_device_derivation_path);
+  hwdev.set_callback(get_device_callback());
 
   m_account.create_from_device(hwdev);
   m_key_device_type = m_account.get_device().get_type();
@@ -9232,9 +9274,7 @@ void wallet2::cold_sign_tx(const std::vector<pending_tx>& ptx_vector, signed_tx_
 //----------------------------------------------------------------------------------------------------
 uint64_t wallet2::cold_key_image_sync(uint64_t &spent, uint64_t &unspent) {
   auto & hwdev = get_account().get_device();
-  if (!hwdev.has_ki_cold_sync()){
+  CHECK_AND_ASSERT_THROW_MES(hwdev.has_ki_cold_sync(), "Device does not support cold ki sync protocol");
-    throw std::invalid_argument("Device does not support cold ki sync protocol");
-  }
 
   auto dev_cold = dynamic_cast<::hw::device_cold*>(&hwdev);
   CHECK_AND_ASSERT_THROW_MES(dev_cold, "Device does not implement cold signing interface");
@@ -9245,7 +9285,11 @@ uint64_t wallet2::cold_key_image_sync(uint64_t &spent, uint64_t &unspent) {
 
   dev_cold->ki_sync(&wallet_shim, m_transfers, ski);
 
-  return import_key_images(ski, 0, spent, unspent);
+  // Call COMMAND_RPC_IS_KEY_IMAGE_SPENT only if daemon is trusted.
+  uint64_t import_res = import_key_images(ski, 0, spent, unspent, is_trusted_daemon());
+  m_device_last_key_image_sync = time(NULL);
+
+  return import_res;
 }
 //----------------------------------------------------------------------------------------------------
 void wallet2::get_hard_fork_info(uint8_t version, uint64_t &earliest_height) const
@@ -11975,4 +12019,29 @@ uint64_t wallet2::get_segregation_fork_height() const
 void wallet2::generate_genesis(cryptonote::block& b) const {
   cryptonote::generate_genesis_block(b, get_config(m_nettype).GENESIS_TX, get_config(m_nettype).GENESIS_NONCE);
 }
+//----------------------------------------------------------------------------------------------------
+wallet_device_callback * wallet2::get_device_callback()
+{
+  if (!m_device_callback){
+    m_device_callback.reset(new wallet_device_callback(this));
+  }
+  return m_device_callback.get();
+}//----------------------------------------------------------------------------------------------------
+void wallet2::on_button_request()
+{
+  if (0 != m_callback)
+    m_callback->on_button_request();
+}
+//----------------------------------------------------------------------------------------------------
+void wallet2::on_pin_request(epee::wipeable_string & pin)
+{
+  if (0 != m_callback)
+    m_callback->on_pin_request(pin);
+}
+//----------------------------------------------------------------------------------------------------
+void wallet2::on_passphrase_request(bool on_device, epee::wipeable_string & passphrase)
+{
+  if (0 != m_callback)
+    m_callback->on_passphrase_request(on_device, passphrase);
+}
 }

src/wallet/wallet2.h

@@ -100,11 +100,26 @@ namespace tools
     virtual void on_lw_money_received(uint64_t height, const crypto::hash &txid, uint64_t amount) {}
     virtual void on_lw_unconfirmed_money_received(uint64_t height, const crypto::hash &txid, uint64_t amount) {}
     virtual void on_lw_money_spent(uint64_t height, const crypto::hash &txid, uint64_t amount) {}
+    // Device callbacks
+    virtual void on_button_request() {}
+    virtual void on_pin_request(epee::wipeable_string & pin) {}
+    virtual void on_passphrase_request(bool on_device, epee::wipeable_string & passphrase) {}
     // Common callbacks
     virtual void on_pool_tx_removed(const crypto::hash &txid) {}
     virtual ~i_wallet2_callback() {}
   };
 
+  class wallet_device_callback : public hw::i_device_callback
+  {
+  public:
+    wallet_device_callback(wallet2 * wallet): wallet(wallet) {};
+    void on_button_request() override;
+    void on_pin_request(epee::wipeable_string & pin) override;
+    void on_passphrase_request(bool on_device, epee::wipeable_string & passphrase) override;
+  private:
+    wallet2 * wallet;
+  };
+
   struct tx_dust_policy
   {
     uint64_t dust_threshold;
@@ -156,6 +171,7 @@ namespace tools
   {
     friend class ::Serialization_portability_wallet_Test;
     friend class wallet_keys_unlocker;
+    friend class wallet_device_callback;
   public:
     static constexpr const std::chrono::seconds rpc_timeout = std::chrono::minutes(3) + std::chrono::seconds(30);
 
@@ -177,6 +193,7 @@ namespace tools
     static bool has_testnet_option(const boost::program_options::variables_map& vm);
     static bool has_stagenet_option(const boost::program_options::variables_map& vm);
     static std::string device_name_option(const boost::program_options::variables_map& vm);
+    static std::string device_derivation_path_option(const boost::program_options::variables_map &vm);
     static void init_options(boost::program_options::options_description& desc_params);
 
     //! Uses stdin and stdout. Returns a wallet2 if no errors.
@@ -795,6 +812,7 @@ namespace tools
     bool is_transfer_unlocked(uint64_t unlock_time, uint64_t block_height) const;
 
     uint64_t get_last_block_reward() const { return m_last_block_reward; }
+    uint64_t get_device_last_key_image_sync() const { return m_device_last_key_image_sync; }
 
     template <class t_archive>
     inline void serialize(t_archive &a, const unsigned int ver)
@@ -903,6 +921,9 @@ namespace tools
       if(ver < 26)
         return;
       a & m_tx_device;
+      if(ver < 27)
+        return;
+      a & m_device_last_key_image_sync;
     }
 
     /*!
@@ -964,6 +985,8 @@ namespace tools
     void confirm_non_default_ring_size(bool always) { m_confirm_non_default_ring_size = always; }
     const std::string & device_name() const { return m_device_name; }
     void device_name(const std::string & device_name) { m_device_name = device_name; }
+    const std::string & device_derivation_path() const { return m_device_derivation_path; }
+    void device_derivation_path(const std::string &device_derivation_path) { m_device_derivation_path = device_derivation_path; }
 
     bool get_tx_key(const crypto::hash &txid, crypto::secret_key &tx_key, std::vector<crypto::secret_key> &additional_tx_keys) const;
     void set_tx_key(const crypto::hash &txid, const crypto::secret_key &tx_key, const std::vector<crypto::secret_key> &additional_tx_keys);
@@ -1287,6 +1310,11 @@ namespace tools
     void setup_new_blockchain();
     void create_keys_file(const std::string &wallet_, bool watch_only, const epee::wipeable_string &password, bool create_address_file);
 
+    wallet_device_callback * get_device_callback();
+    void on_button_request();
+    void on_pin_request(epee::wipeable_string & pin);
+    void on_passphrase_request(bool on_device, epee::wipeable_string & passphrase);
+
     cryptonote::account_base m_account;
     boost::optional<epee::net_utils::http::login> m_daemon_login;
     std::string m_daemon_address;
@@ -1365,6 +1393,8 @@ namespace tools
     std::unordered_set<crypto::hash> m_scanned_pool_txs[2];
     size_t m_subaddress_lookahead_major, m_subaddress_lookahead_minor;
     std::string m_device_name;
+    std::string m_device_derivation_path;
+    uint64_t m_device_last_key_image_sync;
 
     // Aux transaction data from device
     std::unordered_map<crypto::hash, std::string> m_tx_device;
@@ -1398,9 +1428,10 @@ namespace tools
     bool m_devices_registered;
 
     std::shared_ptr<tools::Notify> m_tx_notify;
+    std::unique_ptr<wallet_device_callback> m_device_callback;
   };
 }
-BOOST_CLASS_VERSION(tools::wallet2, 26)
+BOOST_CLASS_VERSION(tools::wallet2, 27)
 BOOST_CLASS_VERSION(tools::wallet2::transfer_details, 10)
 BOOST_CLASS_VERSION(tools::wallet2::multisig_info, 1)
 BOOST_CLASS_VERSION(tools::wallet2::multisig_info::LR, 0)

src/wallet/wallet_errors.h

@@ -219,6 +219,14 @@ namespace tools
       }
     };
     //----------------------------------------------------------------------------------------------------
+    struct password_entry_failed : public wallet_runtime_error
+    {
+      explicit password_entry_failed(std::string&& loc, const std::string &msg = "Password entry failed")
+        : wallet_runtime_error(std::move(loc), msg)
+      {
+      }
+    };
+    //----------------------------------------------------------------------------------------------------
     const char* const file_error_messages[] = {
       "file already exists",
       "file not found",

tests/core_proxy/core_proxy.h

@@ -104,5 +104,6 @@ namespace tests
     cryptonote::difficulty_type get_block_cumulative_difficulty(uint64_t height) const { return 0; }
     bool fluffy_blocks_enabled() const { return false; }
     uint64_t prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes) { return 0; }
+    bool pad_transactions() const { return false; }
   };
 }

tests/unit_tests/ban.cpp

@@ -83,6 +83,7 @@ public:
   cryptonote::difficulty_type get_block_cumulative_difficulty(uint64_t height) const { return 0; }
   bool fluffy_blocks_enabled() const { return false; }
   uint64_t prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes) { return 0; }
+  bool pad_transactions() { return false; }
   void stop() {}
 };
 

tests/unit_tests/testdb.h

@@ -142,5 +142,6 @@ public:
   virtual bool prune_blockchain(uint32_t pruning_seed = 0) { return true; }
   virtual bool update_pruning() { return true; }
   virtual bool check_pruning() { return true; }
+  virtual void prune_outputs(uint64_t amount) {}
 };