Merge pull request #5383

0575794f console: simple shell over console.py (moneromooo-monero)
047af5c3 console.py: can now connect to several daemons/wallets (moneromooo-monero)
9f9571aa cmake: always detect python, it's neeed for some tests (moneromooo-monero)
8646bd00 functional_tests: exit with 1 if any test fails (moneromooo-monero)
6fd8834d console.py: add tab completion (moneromooo-monero)
04a20cb2 functional_tests: cold signing key images/outputs import/export (moneromooo-monero)
798e3cad functional_tests: add double spend detection tests (moneromooo-monero)
7c657bb2 functional_tests: add alt chains tests (moneromooo-monero)
f8be31d2 functional_tests: add wallet creation language tests (moneromooo-monero)
2d68b31f functional_tests: add more wallet tests (moneromooo-monero)
23f86dad python-rpc: add set_log_level and set_log_categories (moneromooo-monero)
b3a32d55 functional_tests: add describe_transfer tests (moneromooo-monero)
108f4375 console.py: support connecting to any host, not just 127.0.0.1 (moneromooo-monero)
064ab123 functional_tests: add more blockchain related tests (moneromooo-monero)
21b1ac1d functional_tests: add bans tests (moneromooo-monero)

CMakeLists.txt

@@ -1028,3 +1028,5 @@ option(INSTALL_VENDORED_LIBUNBOUND "Install libunbound binary built from source
 
 
 CHECK_C_COMPILER_FLAG(-std=c11 HAVE_C11)
+
+find_package(PythonInterp)

src/cryptonote_basic/cryptonote_format_utils.h

@@ -142,6 +142,16 @@ namespace cryptonote
   std::string print_money(uint64_t amount, unsigned int decimal_point = -1);
   //---------------------------------------------------------------
   template<class t_object>
+  bool t_serializable_object_from_blob(t_object& to, const blobdata& b_blob)
+  {
+    std::stringstream ss;
+    ss << b_blob;
+    binary_archive<false> ba(ss);
+    bool r = ::serialization::serialize(ba, to);
+    return r;
+  }
+  //---------------------------------------------------------------
+  template<class t_object>
   bool t_serializable_object_to_blob(const t_object& to, blobdata& b_blob)
   {
     std::stringstream ss;

src/cryptonote_basic/miner.cpp

@@ -576,7 +576,8 @@ namespace cryptonote
         //we lucky!
         ++m_config.current_extra_message_index;
         MGINFO_GREEN("Found block " << get_block_hash(b) << " at height " << height << " for difficulty: " << local_diff);
-        if(!m_phandler->handle_block_found(b))
+        cryptonote::block_verification_context bvc;
+        if(!m_phandler->handle_block_found(b, bvc) || !bvc.m_added_to_main_chain)
         {
           --m_config.current_extra_message_index;
         }else

src/cryptonote_basic/miner.h

@@ -34,6 +34,7 @@
 #include <boost/logic/tribool_fwd.hpp>
 #include <atomic>
 #include "cryptonote_basic.h"
+#include "verification_context.h"
 #include "difficulty.h"
 #include "math_helper.h"
 #ifdef _WIN32
@@ -45,7 +46,7 @@ namespace cryptonote
 
   struct i_miner_handler
   {
-    virtual bool handle_block_found(block& b) = 0;
+    virtual bool handle_block_found(block& b, block_verification_context &bvc) = 0;
     virtual bool get_block_template(block& b, const account_public_address& adr, difficulty_type& diffic, uint64_t& height, uint64_t& expected_reward, const blobdata& ex_nonce) = 0;
   protected:
     ~i_miner_handler(){};

src/cryptonote_core/blockchain.cpp

@@ -1008,7 +1008,7 @@ bool Blockchain::rollback_blockchain_switching(std::list<block>& original_chain,
 //------------------------------------------------------------------
 // This function attempts to switch to an alternate chain, returning
 // boolean based on success therein.
-bool Blockchain::switch_to_alternative_blockchain(std::list<blocks_ext_by_hash::iterator>& alt_chain, bool discard_disconnected_chain)
+bool Blockchain::switch_to_alternative_blockchain(std::list<blocks_ext_by_hash::const_iterator>& alt_chain, bool discard_disconnected_chain)
 {
   LOG_PRINT_L3("Blockchain::" << __func__);
   CRITICAL_REGION_LOCAL(m_blockchain_lock);
@@ -1109,7 +1109,7 @@ bool Blockchain::switch_to_alternative_blockchain(std::list<blocks_ext_by_hash::
 //------------------------------------------------------------------
 // This function calculates the difficulty target for the block being added to
 // an alternate chain.
-difficulty_type Blockchain::get_next_difficulty_for_alternative_chain(const std::list<blocks_ext_by_hash::iterator>& alt_chain, block_extended_info& bei) const
+difficulty_type Blockchain::get_next_difficulty_for_alternative_chain(const std::list<blocks_ext_by_hash::const_iterator>& alt_chain, block_extended_info& bei) const
 {
   if (m_fixed_difficulty)
   {
@@ -1351,7 +1351,7 @@ uint64_t Blockchain::get_current_cumulative_block_weight_median() const
 // in a lot of places.  That flag is not referenced in any of the code
 // nor any of the makefiles, howeve.  Need to look into whether or not it's
 // necessary at all.
-bool Blockchain::create_block_template(block& b, const account_public_address& miner_address, difficulty_type& diffic, uint64_t& height, uint64_t& expected_reward, const blobdata& ex_nonce)
+bool Blockchain::create_block_template(block& b, const crypto::hash *from_block, const account_public_address& miner_address, difficulty_type& diffic, uint64_t& height, uint64_t& expected_reward, const blobdata& ex_nonce)
 {
   LOG_PRINT_L3("Blockchain::" << __func__);
   size_t median_weight;
@@ -1361,8 +1361,7 @@ bool Blockchain::create_block_template(block& b, const account_public_address& m
   m_tx_pool.lock();
   const auto unlock_guard = epee::misc_utils::create_scope_leave_handler([&]() { m_tx_pool.unlock(); });
   CRITICAL_REGION_LOCAL(m_blockchain_lock);
-  height = m_db->height();
+  if (m_btc_valid && !from_block) {
-  if (m_btc_valid) {
     // The pool cookie is atomic. The lack of locking is OK, as if it changes
     // just as we compare it, we'll just use a slightly old template, but
     // this would be the case anyway if we'd lock, and the change happened
@@ -1376,13 +1375,75 @@ bool Blockchain::create_block_template(block& b, const account_public_address& m
       expected_reward = m_btc_expected_reward;
       return true;
     }
-    MDEBUG("Not using cached template: address " << (!memcmp(&miner_address, &m_btc_address, sizeof(cryptonote::account_public_address))) << ", nonce " << (m_btc_nonce == ex_nonce) << ", cookie " << (m_btc_pool_cookie == m_tx_pool.cookie()));
+    MDEBUG("Not using cached template: address " << (!memcmp(&miner_address, &m_btc_address, sizeof(cryptonote::account_public_address))) << ", nonce " << (m_btc_nonce == ex_nonce) << ", cookie " << (m_btc_pool_cookie == m_tx_pool.cookie()) << ", from_block " << (!!from_block));
     invalidate_block_template_cache();
   }
 
+  if (from_block)
+  {
+    //build alternative subchain, front -> mainchain, back -> alternative head
+    //block is not related with head of main chain
+    //first of all - look in alternative chains container
+    auto it_prev = m_alternative_chains.find(*from_block);
+    bool parent_in_main = m_db->block_exists(*from_block);
+    if(it_prev == m_alternative_chains.end() && !parent_in_main)
+    {
+      MERROR("Unknown from block");
+      return false;
+    }
+
+    //we have new block in alternative chain
+    std::list<blocks_ext_by_hash::const_iterator> alt_chain;
+    block_verification_context bvc = boost::value_initialized<block_verification_context>();
+    std::vector<uint64_t> timestamps;
+    if (!build_alt_chain(*from_block, alt_chain, timestamps, bvc))
+      return false;
+
+    if (parent_in_main)
+    {
+      cryptonote::block prev_block;
+      CHECK_AND_ASSERT_MES(get_block_by_hash(*from_block, prev_block), false, "From block not found"); // TODO
+      uint64_t from_block_height = cryptonote::get_block_height(prev_block);
+      height = from_block_height + 1;
+    }
+    else
+    {
+      height = alt_chain.back()->second.height + 1;
+    }
+    b.major_version = m_hardfork->get_ideal_version(height);
+    b.minor_version = m_hardfork->get_ideal_version();
+    b.prev_id = *from_block;
+
+    // cheat and use the weight of the block we start from, virtually certain to be acceptable
+    // and use 1.9 times rather than 2 times so we're even more sure
+    if (parent_in_main)
+    {
+      median_weight = m_db->get_block_weight(height - 1);
+      already_generated_coins = m_db->get_block_already_generated_coins(height - 1);
+    }
+    else
+    {
+      median_weight = it_prev->second.block_cumulative_weight - it_prev->second.block_cumulative_weight / 20;
+      already_generated_coins = alt_chain.back()->second.already_generated_coins;
+    }
+
+    // FIXME: consider moving away from block_extended_info at some point
+    block_extended_info bei = boost::value_initialized<block_extended_info>();
+    bei.bl = b;
+    bei.height = alt_chain.size() ? it_prev->second.height + 1 : m_db->get_block_height(*from_block) + 1;
+
+    diffic = get_next_difficulty_for_alternative_chain(alt_chain, bei);
+  }
+  else
+  {
+    height = m_db->height();
     b.major_version = m_hardfork->get_current_version();
     b.minor_version = m_hardfork->get_ideal_version();
     b.prev_id = get_tail_id();
+    median_weight = m_current_block_cumul_weight_limit / 2;
+    diffic = get_difficulty_for_next_block();
+    already_generated_coins = m_db->get_block_already_generated_coins(height - 1);
+  }
   b.timestamp = time(NULL);
 
   uint64_t median_ts;
@@ -1391,15 +1452,11 @@ bool Blockchain::create_block_template(block& b, const account_public_address& m
     b.timestamp = median_ts;
   }
 
-  diffic = get_difficulty_for_next_block();
   CHECK_AND_ASSERT_MES(diffic, false, "difficulty overhead.");
 
-  median_weight = m_current_block_cumul_weight_limit / 2;
-  already_generated_coins = m_db->get_block_already_generated_coins(height - 1);
-
   size_t txs_weight;
   uint64_t fee;
-  if (!m_tx_pool.fill_block_template(b, median_weight, already_generated_coins, txs_weight, fee, expected_reward, m_hardfork->get_current_version()))
+  if (!m_tx_pool.fill_block_template(b, median_weight, already_generated_coins, txs_weight, fee, expected_reward, b.major_version))
   {
     return false;
   }
@@ -1462,7 +1519,7 @@ bool Blockchain::create_block_template(block& b, const account_public_address& m
    block weight, so first miner transaction generated with fake amount of money, and with phase we know think we know expected block weight
    */
   //make blocks coin-base tx looks close to real coinbase tx to get truthful blob weight
-  uint8_t hf_version = m_hardfork->get_current_version();
+  uint8_t hf_version = b.major_version;
   size_t max_outs = hf_version >= 4 ? 1 : 11;
   bool r = construct_miner_tx(height, median_weight, already_generated_coins, txs_weight, fee, miner_address, b.miner_tx, ex_nonce, max_outs, hf_version);
   CHECK_AND_ASSERT_MES(r, false, "Failed to construct miner tx, first chance");
@@ -1517,6 +1574,7 @@ bool Blockchain::create_block_template(block& b, const account_public_address& m
         ", cumulative weight " << cumulative_weight << " is now good");
 #endif
 
+    if (!from_block)
       cache_block_template(b, miner_address, ex_nonce, diffic, height, expected_reward, pool_cookie);
     return true;
   }
@@ -1524,9 +1582,14 @@ bool Blockchain::create_block_template(block& b, const account_public_address& m
   return false;
 }
 //------------------------------------------------------------------
+bool Blockchain::create_block_template(block& b, const account_public_address& miner_address, difficulty_type& diffic, uint64_t& height, uint64_t& expected_reward, const blobdata& ex_nonce)
+{
+  return create_block_template(b, NULL, miner_address, diffic, height, expected_reward, ex_nonce);
+}
+//------------------------------------------------------------------
 // for an alternate chain, get the timestamps from the main chain to complete
 // the needed number of timestamps for the BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW.
-bool Blockchain::complete_timestamps_vector(uint64_t start_top_height, std::vector<uint64_t>& timestamps)
+bool Blockchain::complete_timestamps_vector(uint64_t start_top_height, std::vector<uint64_t>& timestamps) const
 {
   LOG_PRINT_L3("Blockchain::" << __func__);
 
@@ -1546,6 +1609,52 @@ bool Blockchain::complete_timestamps_vector(uint64_t start_top_height, std::vect
   return true;
 }
 //------------------------------------------------------------------
+bool Blockchain::build_alt_chain(const crypto::hash &prev_id, std::list<blocks_ext_by_hash::const_iterator>& alt_chain, std::vector<uint64_t> &timestamps, block_verification_context& bvc) const
+{
+    //build alternative subchain, front -> mainchain, back -> alternative head
+    blocks_ext_by_hash::const_iterator alt_it = m_alternative_chains.find(prev_id);
+    timestamps.clear();
+    while(alt_it != m_alternative_chains.end())
+    {
+      alt_chain.push_front(alt_it);
+      timestamps.push_back(alt_it->second.bl.timestamp);
+      alt_it = m_alternative_chains.find(alt_it->second.bl.prev_id);
+    }
+
+    // if block to be added connects to known blocks that aren't part of the
+    // main chain -- that is, if we're adding on to an alternate chain
+    if(!alt_chain.empty())
+    {
+      // make sure alt chain doesn't somehow start past the end of the main chain
+      CHECK_AND_ASSERT_MES(m_db->height() > alt_chain.front()->second.height, false, "main blockchain wrong height");
+
+      // make sure that the blockchain contains the block that should connect
+      // this alternate chain with it.
+      if (!m_db->block_exists(alt_chain.front()->second.bl.prev_id))
+      {
+        MERROR("alternate chain does not appear to connect to main chain...");
+        return false;
+      }
+
+      // make sure block connects correctly to the main chain
+      auto h = m_db->get_block_hash_from_height(alt_chain.front()->second.height - 1);
+      CHECK_AND_ASSERT_MES(h == alt_chain.front()->second.bl.prev_id, false, "alternative chain has wrong connection to main chain");
+      complete_timestamps_vector(m_db->get_block_height(alt_chain.front()->second.bl.prev_id), timestamps);
+    }
+    // if block not associated with known alternate chain
+    else
+    {
+      // if block parent is not part of main chain or an alternate chain,
+      // we ignore it
+      bool parent_in_main = m_db->block_exists(prev_id);
+      CHECK_AND_ASSERT_MES(parent_in_main, false, "internal error: broken imperative condition: parent_in_main");
+
+      complete_timestamps_vector(m_db->get_block_height(prev_id), timestamps);
+    }
+
+    return true;
+}
+//------------------------------------------------------------------
 // If a block is to be added and its parent block is not the current
 // main chain top block, then we need to see if we know about its parent block.
 // If its parent block is part of a known forked chain, then we need to see
@@ -1590,47 +1699,18 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id
   if(it_prev != m_alternative_chains.end() || parent_in_main)
   {
     //we have new block in alternative chain
-
+    std::list<blocks_ext_by_hash::const_iterator> alt_chain;
-    //build alternative subchain, front -> mainchain, back -> alternative head
-    blocks_ext_by_hash::iterator alt_it = it_prev; //m_alternative_chains.find()
-    std::list<blocks_ext_by_hash::iterator> alt_chain;
     std::vector<uint64_t> timestamps;
-    while(alt_it != m_alternative_chains.end())
+    if (!build_alt_chain(b.prev_id, alt_chain, timestamps, bvc))
-    {
-      alt_chain.push_front(alt_it);
-      timestamps.push_back(alt_it->second.bl.timestamp);
-      alt_it = m_alternative_chains.find(alt_it->second.bl.prev_id);
-    }
-
-    // if block to be added connects to known blocks that aren't part of the
-    // main chain -- that is, if we're adding on to an alternate chain
-    if(!alt_chain.empty())
-    {
-      // make sure alt chain doesn't somehow start past the end of the main chain
-      CHECK_AND_ASSERT_MES(m_db->height() > alt_chain.front()->second.height, false, "main blockchain wrong height");
-
-      // make sure that the blockchain contains the block that should connect
-      // this alternate chain with it.
-      if (!m_db->block_exists(alt_chain.front()->second.bl.prev_id))
-      {
-        MERROR("alternate chain does not appear to connect to main chain...");
       return false;
-      }
 
-      // make sure block connects correctly to the main chain
+    // FIXME: consider moving away from block_extended_info at some point
-      auto h = m_db->get_block_hash_from_height(alt_chain.front()->second.height - 1);
+    block_extended_info bei = boost::value_initialized<block_extended_info>();
-      CHECK_AND_ASSERT_MES(h == alt_chain.front()->second.bl.prev_id, false, "alternative chain has wrong connection to main chain");
+    bei.bl = b;
-      complete_timestamps_vector(m_db->get_block_height(alt_chain.front()->second.bl.prev_id), timestamps);
+    const uint64_t prev_height = alt_chain.size() ? it_prev->second.height : m_db->get_block_height(b.prev_id);
-    }
+    bei.height = prev_height + 1;
-    // if block not associated with known alternate chain
+    uint64_t block_reward = get_outs_money_amount(b.miner_tx);
-    else
+    bei.already_generated_coins = block_reward + (alt_chain.size() ? it_prev->second.already_generated_coins : m_db->get_block_already_generated_coins(prev_height));
-    {
-      // if block parent is not part of main chain or an alternate chain,
-      // we ignore it
-      CHECK_AND_ASSERT_MES(parent_in_main, false, "internal error: broken imperative condition: parent_in_main");
-
-      complete_timestamps_vector(m_db->get_block_height(b.prev_id), timestamps);
-    }
 
     // verify that the block's timestamp is within the acceptable range
     // (not earlier than the median of the last X blocks)
@@ -1641,11 +1721,6 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id
       return false;
     }
 
-    // FIXME: consider moving away from block_extended_info at some point
-    block_extended_info bei = boost::value_initialized<block_extended_info>();
-    bei.bl = b;
-    bei.height = alt_chain.size() ? it_prev->second.height + 1 : m_db->get_block_height(b.prev_id) + 1;
-
     bool is_a_checkpoint;
     if(!m_checkpoints.check_block(bei.height, id, is_a_checkpoint))
     {

src/cryptonote_core/blockchain.h

@@ -336,6 +336,7 @@ namespace cryptonote
      * @brief creates a new block to mine against
      *
      * @param b return-by-reference block to be filled in
+     * @param from_block optional block hash to start mining from (main chain tip if NULL)
      * @param miner_address address new coins for the block will go to
      * @param di return-by-reference tells the miner what the difficulty target is
      * @param height return-by-reference tells the miner what height it's mining against
@@ -345,6 +346,7 @@ namespace cryptonote
      * @return true if block template filled in successfully, else false
      */
     bool create_block_template(block& b, const account_public_address& miner_address, difficulty_type& di, uint64_t& height, uint64_t& expected_reward, const blobdata& ex_nonce);
+    bool create_block_template(block& b, const crypto::hash *from_block, const account_public_address& miner_address, difficulty_type& di, uint64_t& height, uint64_t& expected_reward, const blobdata& ex_nonce);
 
     /**
      * @brief checks if a block is known about with a given hash
@@ -1180,7 +1182,7 @@ namespace cryptonote
      *
      * @return false if the reorganization fails, otherwise true
      */
-    bool switch_to_alternative_blockchain(std::list<blocks_ext_by_hash::iterator>& alt_chain, bool discard_disconnected_chain);
+    bool switch_to_alternative_blockchain(std::list<blocks_ext_by_hash::const_iterator>& alt_chain, bool discard_disconnected_chain);
 
     /**
      * @brief removes the most recent block from the blockchain
@@ -1233,6 +1235,18 @@ namespace cryptonote
      */
     bool handle_alternative_block(const block& b, const crypto::hash& id, block_verification_context& bvc);
 
+    /**
+     * @brief builds a list of blocks connecting a block to the main chain
+     *
+     * @param prev_id the block hash of the tip of the alt chain
+     * @param alt_chain the chain to be added to
+     * @param timestamps returns the timestamps of previous blocks
+     * @param bvc the block verification context for error return
+     *
+     * @return true on success, false otherwise
+     */
+    bool build_alt_chain(const crypto::hash &prev_id, std::list<blocks_ext_by_hash::const_iterator>& alt_chain, std::vector<uint64_t> &timestamps, block_verification_context& bvc) const;
+
     /**
      * @brief gets the difficulty requirement for a new block on an alternate chain
      *
@@ -1241,7 +1255,7 @@ namespace cryptonote
      *
      * @return the difficulty requirement
      */
-    difficulty_type get_next_difficulty_for_alternative_chain(const std::list<blocks_ext_by_hash::iterator>& alt_chain, block_extended_info& bei) const;
+    difficulty_type get_next_difficulty_for_alternative_chain(const std::list<blocks_ext_by_hash::const_iterator>& alt_chain, block_extended_info& bei) const;
 
     /**
      * @brief sanity checks a miner transaction before validating an entire block
@@ -1401,7 +1415,7 @@ namespace cryptonote
      *
      * @return true unless start_height is greater than the current blockchain height
      */
-    bool complete_timestamps_vector(uint64_t start_height, std::vector<uint64_t>& timestamps);
+    bool complete_timestamps_vector(uint64_t start_height, std::vector<uint64_t>& timestamps) const;
 
     /**
      * @brief calculate the block weight limit for the next block to be added

src/cryptonote_core/cryptonote_core.cpp

@@ -1268,6 +1268,11 @@ namespace cryptonote
     return m_blockchain_storage.create_block_template(b, adr, diffic, height, expected_reward, ex_nonce);
   }
   //-----------------------------------------------------------------------------------------------
+  bool core::get_block_template(block& b, const crypto::hash *prev_block, const account_public_address& adr, difficulty_type& diffic, uint64_t& height, uint64_t& expected_reward, const blobdata& ex_nonce)
+  {
+    return m_blockchain_storage.create_block_template(b, prev_block, adr, diffic, height, expected_reward, ex_nonce);
+  }
+  //-----------------------------------------------------------------------------------------------
   bool core::find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) const
   {
     return m_blockchain_storage.find_blockchain_supplement(qblock_ids, resp);
@@ -1321,9 +1326,9 @@ namespace cryptonote
     return bce;
   }
   //-----------------------------------------------------------------------------------------------
-  bool core::handle_block_found(block& b)
+  bool core::handle_block_found(block& b, block_verification_context &bvc)
   {
-    block_verification_context bvc = boost::value_initialized<block_verification_context>();
+    bvc = boost::value_initialized<block_verification_context>();
     m_miner.pause();
     std::vector<block_complete_entry> blocks;
     try
@@ -1373,7 +1378,7 @@ namespace cryptonote
 
       m_pprotocol->relay_block(arg, exclude_context);
     }
-    return bvc.m_added_to_main_chain;
+    return true;
   }
   //-----------------------------------------------------------------------------------------------
   void core::on_synchronized()

src/cryptonote_core/cryptonote_core.h

@@ -195,10 +195,11 @@ namespace cryptonote
       * the network.
       *
       * @param b the block found
+      * @param bvc returns the block verification flags
       *
       * @return true if the block was added to the main chain, otherwise false
       */
-     virtual bool handle_block_found( block& b);
+     virtual bool handle_block_found(block& b, block_verification_context &bvc);
 
      /**
       * @copydoc Blockchain::create_block_template
@@ -206,6 +207,7 @@ namespace cryptonote
       * @note see Blockchain::create_block_template
       */
      virtual bool get_block_template(block& b, const account_public_address& adr, difficulty_type& diffic, uint64_t& height, uint64_t& expected_reward, const blobdata& ex_nonce);
+     virtual bool get_block_template(block& b, const crypto::hash *prev_block, const account_public_address& adr, difficulty_type& diffic, uint64_t& height, uint64_t& expected_reward, const blobdata& ex_nonce);
 
      /**
       * @brief called when a transaction is relayed

src/rpc/core_rpc_server.cpp

@@ -496,6 +496,7 @@ namespace cryptonote
 
     cryptonote::COMMAND_RPC_GET_OUTPUTS_BIN::request req_bin;
     req_bin.outputs = req.outputs;
+    req_bin.get_txid = req.get_txid;
     cryptonote::COMMAND_RPC_GET_OUTPUTS_BIN::response res_bin;
     if(!m_core.get_outs(req_bin, res_bin))
     {
@@ -1259,7 +1260,17 @@ namespace cryptonote
     cryptonote::blobdata blob_reserve;
     blob_reserve.resize(req.reserve_size, 0);
     cryptonote::difficulty_type wdiff;
-    if(!m_core.get_block_template(b, info.address, wdiff, res.height, res.expected_reward, blob_reserve))
+    crypto::hash prev_block;
+    if (!req.prev_block.empty())
+    {
+      if (!epee::string_tools::hex_to_pod(req.prev_block, prev_block))
+      {
+        error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR;
+        error_resp.message = "Invalid prev_block";
+        return false;
+      }
+    }
+    if(!m_core.get_block_template(b, req.prev_block.empty() ? NULL : &prev_block, info.address, wdiff, res.height, res.expected_reward, blob_reserve))
     {
       error_resp.code = CORE_RPC_ERROR_CODE_INTERNAL_ERROR;
       error_resp.message = "Internal error: failed to create block template";
@@ -1345,7 +1356,8 @@ namespace cryptonote
       return false;
     }
 
-    if(!m_core.handle_block_found(b))
+    block_verification_context bvc;
+    if(!m_core.handle_block_found(b, bvc))
     {
       error_resp.code = CORE_RPC_ERROR_CODE_BLOCK_NOT_ACCEPTED;
       error_resp.message = "Block not accepted";
@@ -1377,15 +1389,17 @@ namespace cryptonote
 
     template_req.reserve_size = 1;
     template_req.wallet_address = req.wallet_address;
+    template_req.prev_block = req.prev_block;
     submit_req.push_back(boost::value_initialized<std::string>());
     res.height = m_core.get_blockchain_storage().get_current_blockchain_height();
 
-    bool r;
+    bool r = CORE_RPC_STATUS_OK;
 
     for(size_t i = 0; i < req.amount_of_blocks; i++)
     {
       r = on_getblocktemplate(template_req, template_res, error_resp, ctx);
       res.status = template_res.status;
+      template_req.prev_block.clear();
       
       if (!r) return false;
 
@@ -1403,6 +1417,7 @@ namespace cryptonote
         error_resp.message = "Wrong block blob";
         return false;
       }
+      b.nonce = req.starting_nonce;
       miner::find_nonce_for_given_block(b, template_res.difficulty, template_res.height);
 
       submit_req.front() = string_tools::buff_to_hex_nodelimer(block_to_blob(b));
@@ -1411,6 +1426,8 @@ namespace cryptonote
 
       if (!r) return false;
 
+      res.blocks.push_back(epee::string_tools::pod_to_hex(get_block_hash(b)));
+      template_req.prev_block = res.blocks.back();
       res.height = template_res.height;
     }
 

src/rpc/core_rpc_server_commands_defs.h

@@ -530,9 +530,11 @@ namespace cryptonote
     struct request_t
     {
       std::vector<get_outputs_out> outputs;
+      bool get_txid;
 
       BEGIN_KV_SERIALIZE_MAP()
         KV_SERIALIZE(outputs)
+        KV_SERIALIZE(get_txid)
       END_KV_SERIALIZE_MAP()
     };
     typedef epee::misc_utils::struct_init<request_t> request;
@@ -904,10 +906,12 @@ namespace cryptonote
     {
       uint64_t reserve_size;       //max 255 bytes
       std::string wallet_address;
+      std::string prev_block;
 
       BEGIN_KV_SERIALIZE_MAP()
         KV_SERIALIZE(reserve_size)
         KV_SERIALIZE(wallet_address)
+        KV_SERIALIZE(prev_block)
       END_KV_SERIALIZE_MAP()
     };
     typedef epee::misc_utils::struct_init<request_t> request;
@@ -964,10 +968,14 @@ namespace cryptonote
     {
       uint64_t amount_of_blocks;
       std::string wallet_address;
+      std::string prev_block;
+      uint32_t starting_nonce;
 
       BEGIN_KV_SERIALIZE_MAP()
         KV_SERIALIZE(amount_of_blocks)
         KV_SERIALIZE(wallet_address)
+        KV_SERIALIZE(prev_block)
+        KV_SERIALIZE_OPT(starting_nonce, (uint32_t)0)
       END_KV_SERIALIZE_MAP()
     };
     typedef epee::misc_utils::struct_init<request_t> request;
@@ -975,10 +983,12 @@ namespace cryptonote
     struct response_t
     {
       uint64_t height;
+      std::vector<std::string> blocks;
       std::string status;
       
       BEGIN_KV_SERIALIZE_MAP()
         KV_SERIALIZE(height)
+        KV_SERIALIZE(blocks)
         KV_SERIALIZE(status)
       END_KV_SERIALIZE_MAP()
     };

tests/functional_tests/bans.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+
+# Copyright (c) 2019 The Monero Project
+# 
+# All rights reserved.
+# 
+# Redistribution and use in source and binary forms, with or without modification, are
+# permitted provided that the following conditions are met:
+# 
+# 1. Redistributions of source code must retain the above copyright notice, this list of
+#    conditions and the following disclaimer.
+# 
+# 2. Redistributions in binary form must reproduce the above copyright notice, this list
+#    of conditions and the following disclaimer in the documentation and/or other
+#    materials provided with the distribution.
+# 
+# 3. Neither the name of the copyright holder nor the names of its contributors may be
+#    used to endorse or promote products derived from this software without specific
+#    prior written permission.
+# 
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import time
+
+"""Test peer baning RPC calls
+
+Test the following RPCs:
+    - set_bans
+    - get_bans
+
+"""
+
+from framework.daemon import Daemon
+
+class BanTest():
+    def run_test(self):
+        print 'Testing bans'
+
+        daemon = Daemon()
+        res = daemon.get_bans()
+        assert 'bans' not in res or len(res.bans) == 0
+
+        daemon.set_bans([{'host': '1.2.3.4', 'ban': True, 'seconds': 100}])
+        res = daemon.get_bans()
+        assert len(res.bans) == 1
+        assert res.bans[0].host == '1.2.3.4'
+        assert res.bans[0].seconds >= 98 and res.bans[0].seconds <= 100 # allow for slow RPC
+
+        daemon.set_bans([{'host': '5.6.7.8', 'ban': True, 'seconds': 100}])
+        res = daemon.get_bans()
+        assert len(res.bans) == 2
+        for i in range(2):
+          assert res.bans[i].host == '1.2.3.4' or res.bans[i].host == '5.6.7.8'
+          assert res.bans[i].seconds >= 7 and res.bans[0].seconds <= 100 # allow for slow RPC
+
+        daemon.set_bans([{'host': '1.2.3.4', 'ban': False}])
+        res = daemon.get_bans()
+        assert len(res.bans) == 1
+        assert res.bans[0].host == '5.6.7.8'
+        assert res.bans[0].seconds >= 98 and res.bans[0].seconds <= 100 # allow for slow RPC
+
+        time.sleep(2)
+
+        res = daemon.get_bans()
+        assert len(res.bans) == 1
+        assert res.bans[0].host == '5.6.7.8'
+        assert res.bans[0].seconds >= 96 and res.bans[0].seconds <= 98 # allow for slow RPC
+
+        daemon.set_bans([{'host': '3.4.5.6', 'ban': False}])
+        res = daemon.get_bans()
+        assert len(res.bans) == 1
+        assert res.bans[0].host == '5.6.7.8'
+        assert res.bans[0].seconds >= 96 and res.bans[0].seconds <= 98 # allow for slow RPC
+
+        daemon.set_bans([{'host': '3.4.5.6', 'ban': True, 'seconds': 2}])
+        res = daemon.get_bans()
+        assert len(res.bans) == 2
+        for i in range(2):
+          assert res.bans[i].host == '5.6.7.8' or res.bans[i].host == '3.4.5.6'
+          if res.bans[i].host == '5.6.7.8':
+            assert res.bans[i].seconds >= 96 and res.bans[0].seconds <= 98 # allow for slow RPC
+          else:
+            assert res.bans[i].seconds >= 1 and res.bans[0].seconds <= 2 # allow for slow RPC
+
+        time.sleep(2)
+        res = daemon.get_bans()
+        assert len(res.bans) == 1
+        assert res.bans[0].host == '5.6.7.8'
+        assert res.bans[0].seconds >= 94 and res.bans[0].seconds <= 96 # allow for slow RPC
+
+        daemon.set_bans([{'host': '5.6.7.8', 'ban': True, 'seconds': 20}])
+        res = daemon.get_bans()
+        assert len(res.bans) == 1
+        assert res.bans[0].host == '5.6.7.8'
+        assert res.bans[0].seconds >= 18 and res.bans[0].seconds <= 20 # allow for slow RPC
+
+        daemon.set_bans([{'host': '5.6.7.8', 'ban': True, 'seconds': 200}])
+        res = daemon.get_bans()
+        assert len(res.bans) == 1
+        assert res.bans[0].host == '5.6.7.8'
+        assert res.bans[0].seconds >= 198 and res.bans[0].seconds <= 200 # allow for slow RPC
+
+        daemon.set_bans([{'host': '5.6.7.8', 'ban': False}])
+        res = daemon.get_bans()
+        assert 'bans' not in res or len(res.bans) == 0
+
+
+if __name__ == '__main__':
+    BanTest().run_test()

tests/functional_tests/blockchain.py

@@ -46,6 +46,7 @@ from framework.daemon import Daemon
 class BlockchainTest():
     def run_test(self):
         self._test_generateblocks(5)
+        self._test_alt_chains()
 
     def _test_generateblocks(self, blocks):
         assert blocks >= 2
@@ -152,6 +153,163 @@ class BlockchainTest():
         except: ok = True
         assert ok
 
+        # get transactions
+        res = daemon.get_info()
+        assert res.height == height + blocks - 1
+        nblocks = height + blocks - 1
+        res = daemon.getblockheadersrange(0, nblocks - 1)
+        assert len(res.headers) == nblocks
+        assert res.headers[-1] == block_header
+        txids = [x.miner_tx_hash for x in res.headers]
+        res = daemon.get_transactions(txs_hashes = txids)
+        assert len(res.txs) == nblocks
+        assert not 'missed_txs' in res or len(res.missed_txs) == 0
+        running_output_index = 0
+        for i in range(len(txids)):
+            tx = res.txs[i]
+            assert tx.tx_hash == txids[i]
+            assert not tx.double_spend_seen
+            assert not tx.in_pool
+            assert tx.block_height == i
+            if i > 0:
+                for idx in tx.output_indices:
+                    assert idx == running_output_index
+                    running_output_index += 1
+                res_out = daemon.get_outs([{'amount': 0, 'index': i} for i in tx.output_indices], get_txid = True)
+                assert len(res_out.outs) == len(tx.output_indices)
+                for out in res_out.outs:
+                    assert len(out.key) == 64
+                    assert len(out.mask) == 64
+                    assert not out.unlocked
+                    assert out.height == i + 1
+                    assert out.txid == txids[i + 1]
+
+        for i in range(height + nblocks - 1):
+            res_sum = daemon.get_coinbase_tx_sum(i, 1)
+            res_header = daemon.getblockheaderbyheight(i)
+            assert res_sum.emission_amount == res_header.block_header.reward
+
+        res = daemon.get_coinbase_tx_sum(0, 1)
+        assert res.emission_amount == 17592186044415
+        assert res.fee_amount == 0
+        sum_blocks = height + nblocks - 1
+        res = daemon.get_coinbase_tx_sum(0, sum_blocks)
+        extrapolated = 17592186044415 + 17592186044415 * 2 * (sum_blocks - 1)
+        assert res.emission_amount < extrapolated and res.emission_amount > extrapolated - 1e12
+        assert res.fee_amount == 0
+        sum_blocks_emission = res.emission_amount
+        res = daemon.get_coinbase_tx_sum(1, sum_blocks)
+        assert res.emission_amount == sum_blocks_emission - 17592186044415
+        assert res.fee_amount == 0
+
+        res = daemon.get_output_distribution([0, 1, 17592186044415], 0, 0)
+        assert len(res.distributions) == 3
+        for a in range(3):
+            assert res.distributions[a].amount == [0, 1, 17592186044415][a]
+            assert res.distributions[a].start_height == 0
+            assert res.distributions[a].base == 0
+            assert len(res.distributions[a].distribution) == height + nblocks - 1
+            assert res.distributions[a].binary == False
+            for i in range(height + nblocks - 1):
+                assert res.distributions[a].distribution[i] == (1 if i > 0 and a == 0 else 1 if a == 2 and i == 0 else 0)
+
+        res = daemon.get_output_histogram([], min_count = 0, max_count = 0)
+        assert len(res.histogram) == 2
+        for i in range(2):
+            assert res.histogram[i].amount in [0, 17592186044415]
+            assert res.histogram[i].total_instances in [height + nblocks - 2, 1]
+            assert res.histogram[i].unlocked_instances == 0
+            assert res.histogram[i].recent_instances == 0
+
+    def _test_alt_chains(self):
+        print('Testing alt chains')
+        daemon = Daemon()
+        res = daemon.get_alt_blocks_hashes()
+        starting_alt_blocks = res.blks_hashes if 'blks_hashes' in res else []
+        res = daemon.get_info()
+        root_block_hash = res.top_block_hash
+        height = res.height
+        prev_hash = res.top_block_hash
+        res_template = daemon.getblocktemplate('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm')
+        nonce = 0
+
+        # 5 siblings
+        alt_blocks = [None] * 5
+        for i in range(len(alt_blocks)):
+            res = daemon.generateblocks('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 1, prev_block = prev_hash, starting_nonce = nonce)
+            assert res.height == height
+            assert len(res.blocks) == 1
+            txid = res.blocks[0]
+            res = daemon.getblockheaderbyhash(txid)
+            nonce = res.block_header.nonce
+            print('mined ' + ('alt' if res.block_header.orphan_status else 'tip') + ' block ' + str(height) + ', nonce ' + str(nonce))
+            assert res.block_header.prev_hash == prev_hash
+            assert res.block_header.orphan_status == (i > 0)
+            alt_blocks[i] = txid
+            nonce += 1
+
+        print 'mining 3 on 1'
+        # three more on [1]
+        chain1 = []
+        res = daemon.generateblocks('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 3, prev_block = alt_blocks[1], starting_nonce = nonce)
+        assert res.height == height + 3
+        assert len(res.blocks) == 3
+        blk_hash = res.blocks[2]
+        res = daemon.getblockheaderbyhash(blk_hash)
+        nonce = res.block_header.nonce
+        assert not res.block_header.orphan_status
+        nonce += 1
+        chain1.append(blk_hash)
+        chain1.append(res.block_header.prev_hash)
+
+        print('Checking alt blocks match')
+        res = daemon.get_alt_blocks_hashes()
+        assert len(res.blks_hashes) == len(starting_alt_blocks) + 4
+        for txid in alt_blocks:
+            assert txid in res.blks_hashes or txid == alt_blocks[1]
+
+        print 'mining 4 on 3'
+        # 4 more on [3], the chain will reorg when we mine the 4th
+        top_block_hash = blk_hash
+        prev_block = alt_blocks[3]
+        for i in range(4):
+            res = daemon.generateblocks('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 1, prev_block = prev_block)
+            assert res.height == height + 1 + i
+            assert len(res.blocks) == 1
+            prev_block = res.blocks[-1]
+            res = daemon.getblockheaderbyhash(res.blocks[-1])
+            assert res.block_header.orphan_status == (i < 3)
+
+            res = daemon.get_info()
+            assert res.height == ((height + 4) if i < 3 else height + 5)
+            assert res.top_block_hash == (top_block_hash if i < 3 else prev_block)
+
+        res = daemon.get_info()
+        assert res.height == height + 5
+        assert res.top_block_hash == prev_block
+
+        print('Checking alt blocks match')
+        res = daemon.get_alt_blocks_hashes()
+        blks_hashes = res.blks_hashes
+        assert len(blks_hashes) == len(starting_alt_blocks) + 7
+        for txid in alt_blocks:
+            assert txid in blks_hashes or txid == alt_blocks[3]
+        for txid in chain1:
+            assert txid in blks_hashes
+
+        res = daemon.get_alternate_chains()
+        assert len(res.chains) == 4
+        tips = [chain.block_hash for chain in res.chains]
+        for txid in tips:
+            assert txid in blks_hashes
+        for chain in res.chains:
+            assert chain.length in [1, 4]
+            assert chain.length == len(chain.block_hashes)
+            assert chain.height == height + chain.length - 1 # all happen start at the same height
+            assert chain.main_chain_parent_block == root_block_hash
+        for txid in [alt_blocks[0], alt_blocks[2], alt_blocks[4]]:
+          assert len([chain for chain in res.chains if chain.block_hash == txid]) == 1
+
 
 if __name__ == '__main__':
     BlockchainTest().run_test()

tests/functional_tests/cold_signing.py

@@ -89,6 +89,12 @@ class ColdSigningTest():
         dst = {'address': '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 'amount': 1000000000000}
         payment_id = '1234500000012345abcde00000abcdeff1234500000012345abcde00000abcde'
 
+        self.hot_wallet.refresh()
+        res = self.hot_wallet.export_outputs()
+        self.cold_wallet.import_outputs(res.outputs_data_hex)
+        res = self.cold_wallet.export_key_images(True)
+        self.hot_wallet.import_key_images(res.signed_key_images, offset = res.offset)
+
         res = self.hot_wallet.transfer([dst], ring_size = 11, payment_id = payment_id, get_tx_key = False)
         assert len(res.tx_hash) == 32*2
         txid = res.tx_hash
@@ -104,6 +110,22 @@ class ColdSigningTest():
         unsigned_txset = res.unsigned_txset
 
         print 'Signing transaction with cold wallet'
+        res = self.cold_wallet.describe_transfer(unsigned_txset = unsigned_txset)
+        assert len(res.desc) == 1
+        desc = res.desc[0]
+        assert desc.amount_in >= amount + fee
+        assert desc.amount_out == desc.amount_in - fee
+        assert desc.ring_size == 11
+        assert desc.unlock_time == 0
+        assert desc.payment_id == payment_id
+        assert desc.change_amount == desc.amount_in - 1000000000000 - fee
+        assert desc.change_address == '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm'
+        assert desc.fee == fee
+        assert len(desc.recipients) == 1
+        rec = desc.recipients[0]
+        assert rec.address == '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm'
+        assert rec.amount == 1000000000000
+
         res = self.cold_wallet.sign_transfer(unsigned_txset)
         assert len(res.signed_txset) > 0
         signed_txset = res.signed_txset

tests/functional_tests/functional_tests_rpc.py

@@ -133,3 +133,5 @@ if len(FAIL) == 0:
   print('Done, ' + str(len(PASS)) + '/' + str(len(tests)) + ' tests passed')
 else:
   print('Done, ' + str(len(FAIL)) + '/' + str(len(tests)) + ' tests failed: ' + string.join(FAIL, ', '))
+
+sys.exit(0 if len(FAIL) == 0 else 1)

tests/functional_tests/multisig.py

@@ -129,6 +129,7 @@ class MultisigTest():
           addresses.append(res.address)
       for i in range(N_total):
         assert addresses[i] == expected_address
+      self.wallet_address = expected_address
 
       for i in range(N_total):
         res = self.wallet[i].is_multisig()
@@ -181,6 +182,22 @@ class MultisigTest():
 
         for i in range(len(signers[1:])):
           print('Signing multisig transaction with wallet ' + str(signers[i+1]))
+          res = self.wallet[signers[i+1]].describe_transfer(multisig_txset = multisig_txset)
+          assert len(res.desc) == 1
+          desc = res.desc[0]
+          assert desc.amount_in >= amount + fee
+          assert desc.amount_out == desc.amount_in - fee
+          assert desc.ring_size == 11
+          assert desc.unlock_time == 0
+          assert desc.payment_id == '0000000000000000'
+          assert desc.change_amount == desc.amount_in - 1000000000000 - fee
+          assert desc.change_address == self.wallet_address
+          assert desc.fee == fee
+          assert len(desc.recipients) == 1
+          rec = desc.recipients[0]
+          assert rec.address == '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm'
+          assert rec.amount == 1000000000000
+
           res = self.wallet[signers[i+1]].sign_multisig(multisig_txset)
           multisig_txset = res.tx_data_hex
           assert len(res.tx_hash_list if 'tx_hash_list' in res else []) == (i == len(signers[1:]) - 1)

tests/functional_tests/transfer.py

@@ -42,6 +42,7 @@ class TransferTest():
         self.mine()
         self.transfer()
         self.check_get_bulk_payments()
+        self.check_double_spend_detection()
 
     def create(self):
         print 'Creating wallets'
@@ -62,9 +63,14 @@ class TransferTest():
         print("Mining some blocks")
         daemon = Daemon()
 
+        res = daemon.get_info()
+        height = res.height
+
         daemon.generateblocks('42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 80)
         for i in range(len(self.wallet)):
             self.wallet[i].refresh()
+            res = self.wallet[i].get_height()
+            assert res.height == height + 80
 
     def transfer(self):
         daemon = Daemon()
@@ -169,6 +175,27 @@ class TransferTest():
         assert e.double_spend_seen == False
         assert e.confirmations == 1
 
+        res = self.wallet[0].get_height()
+        wallet_height = res.height
+        res = self.wallet[0].get_transfer_by_txid(txid)
+        assert len(res.transfers) == 1
+        assert res.transfers[0] == res.transfer
+        t = res.transfer
+        assert t.txid == txid
+        assert t.payment_id == payment_id
+        assert t.height == wallet_height - 1
+        assert t.timestamp > 0
+        assert t.amount == 0 # to self, so it's just "pay a fee" really
+        assert t.fee == fee
+        assert t.note == ''
+        assert len(t.destinations) == 1
+        assert t.destinations[0] == {'address': '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm', 'amount': 1000000000000}
+        assert t.type == 'out'
+        assert t.unlock_time == 0
+        assert t.address == '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm'
+        assert t.double_spend_seen == False
+        assert t.confirmations == 1
+
         res = self.wallet[0].get_balance()
         assert res.balance == running_balances[0]
         assert res.unlocked_balance <= res.balance
@@ -483,5 +510,65 @@ class TransferTest():
         res = self.wallet[2].get_bulk_payments(payment_ids = ['1'*64, '1234500000012345abcde00000abcdeff1234500000012345abcde00000abcde', '2'*64])
         assert len(res.payments) >= 1 # one tx was sent
 
+    def check_double_spend_detection(self):
+        print('Checking double spend detection')
+        txes = [[None, None], [None, None]]
+        for i in range(2):
+            self.wallet[0].restore_deterministic_wallet(seed = 'velvet lymph giddy number token physics poetry unquoted nibs useful sabotage limits benches lifestyle eden nitrogen anvil fewest avoid batch vials washing fences goat unquoted')
+            self.wallet[0].refresh()
+            res = self.wallet[0].get_balance()
+            unlocked_balance = res.unlocked_balance
+            res = self.wallet[0].sweep_all(address = '44Kbx4sJ7JDRDV5aAhLJzQCjDz2ViLRduE3ijDZu3osWKBjMGkV1XPk4pfDUMqt1Aiezvephdqm6YD19GKFD9ZcXVUTp6BW', do_not_relay = True, get_tx_hex = True)
+            assert len(res.tx_hash_list) == 1
+            assert len(res.tx_hash_list[0]) == 32*2
+            txes[i][0] = res.tx_hash_list[0]
+            assert len(res.fee_list) == 1
+            assert res.fee_list[0] > 0
+            assert len(res.amount_list) == 1
+            assert res.amount_list[0] == unlocked_balance - res.fee_list[0]
+            assert len(res.tx_blob_list) > 0
+            assert len(res.tx_blob_list[0]) > 0
+            assert not 'tx_metadata_list' in res or len(res.tx_metadata_list) == 0
+            assert not 'multisig_txset' in res or len(res.multisig_txset) == 0
+            assert not 'unsigned_txset' in res or len(res.unsigned_txset) == 0
+            assert len(res.tx_blob_list) == 1
+            txes[i][1] = res.tx_blob_list[0]
+
+        daemon = Daemon()
+        res = daemon.send_raw_transaction(txes[0][1])
+        assert res.not_relayed == False
+        assert res.low_mixin == False
+        assert res.double_spend == False
+        assert res.invalid_input == False
+        assert res.invalid_output == False
+        assert res.too_big == False
+        assert res.overspend == False
+        assert res.fee_too_low == False
+        assert res.not_rct == False
+
+        res = daemon.get_transactions([txes[0][0]])
+        assert len(res.txs) >= 1
+        tx = [tx for tx in res.txs if tx.tx_hash == txes[0][0]][0]
+        assert tx.in_pool
+        assert not tx.double_spend_seen
+
+        res = daemon.send_raw_transaction(txes[1][1])
+        assert res.not_relayed == False
+        assert res.low_mixin == False
+        assert res.double_spend == True
+        assert res.invalid_input == False
+        assert res.invalid_output == False
+        assert res.too_big == False
+        assert res.overspend == False
+        assert res.fee_too_low == False
+        assert res.not_rct == False
+
+        res = daemon.get_transactions([txes[0][0]])
+        assert len(res.txs) >= 1
+        tx = [tx for tx in res.txs if tx.tx_hash == txes[0][0]][0]
+        assert tx.in_pool
+        assert tx.double_spend_seen
+
+
 if __name__ == '__main__':
     TransferTest().run_test()

tests/functional_tests/wallet_address.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+#encoding=utf-8
 
 # Copyright (c) 2019 The Monero Project
 # 
@@ -45,6 +46,8 @@ class WalletAddressTest():
       self.check_main_address()
       self.check_keys()
       self.create_subaddresses()
+      self.open_close()
+      self.languages()
 
     def create(self):
         print 'Creating wallet'
@@ -148,5 +151,52 @@ class WalletAddressTest():
         res = wallet.get_address_index('82pP87g1Vkd3LUMssBCumk3MfyEsFqLAaGDf6oxddu61EgSFzt8gCwUD4tr3kp9TUfdPs2CnpD7xLZzyC1Ei9UsW3oyCWDf')
         assert res.index == {'major': 1, 'minor': 0}
 
+    def open_close(self):
+        print 'Testing open/close'
+        wallet = Wallet()
+
+        res = wallet.get_address()
+        assert res.address == '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm'
+
+        wallet.close_wallet()
+        ok = False
+        try: res = wallet.get_address()
+        except: ok = True
+        assert ok
+
+        wallet.restore_deterministic_wallet(seed = 'peeled mixture ionic radar utopia puddle buying illness nuns gadget river spout cavernous bounced paradise drunk looking cottage jump tequila melting went winter adjust spout')
+        res = wallet.get_address()
+        assert res.address == '44Kbx4sJ7JDRDV5aAhLJzQCjDz2ViLRduE3ijDZu3osWKBjMGkV1XPk4pfDUMqt1Aiezvephdqm6YD19GKFD9ZcXVUTp6BW'
+
+        wallet.close_wallet()
+        ok = False
+        try: wallet.get_address()
+        except: ok = True
+        assert ok
+
+        wallet.restore_deterministic_wallet(seed = 'velvet lymph giddy number token physics poetry unquoted nibs useful sabotage limits benches lifestyle eden nitrogen anvil fewest avoid batch vials washing fences goat unquoted')
+        res = wallet.get_address()
+        assert res.address == '42ey1afDFnn4886T7196doS9GPMzexD9gXpsZJDwVjeRVdFCSoHnv7KPbBeGpzJBzHRCAs9UxqeoyFQMYbqSWYTfJJQAWDm'
+
+    def languages(self):
+        print('Testing languages')
+        wallet = Wallet()
+        res = wallet.get_languages()
+        assert 'English' in res.languages
+        assert 'English' in res.languages_local
+        assert 'Dutch' in res.languages
+        assert 'Nederlands' in res.languages_local
+        assert 'Japanese' in res.languages
+        assert u'日本語' in res.languages_local
+        try: wallet.close_wallet()
+        except: pass
+        languages = res.languages
+        for language in languages:
+            print 'Creating ' + str(language) + ' wallet'
+            wallet.create_wallet(filename = '', language = language)
+            res = wallet.query_key('mnemonic')
+            wallet.close_wallet()
+
+
 if __name__ == '__main__':
     WalletAddressTest().run_test()

utils/python-rpc/console

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+env python -i $(dirname $0)/console.py "$@"

utils/python-rpc/console.py

@@ -4,46 +4,85 @@ from __future__ import print_function
 import sys
 import subprocess
 import socket
-from framework import rpc
+import urlparse
-from framework import wallet
+import framework.rpc
-from framework import daemon
+import framework.daemon
+import framework.wallet
 
-USAGE = 'usage: python -i console.py <port>'
+USAGE = 'usage: python -i console.py [[[scheme]<host>:]<port> [[[scheme]<host>:]<port>...]]'
-try:
+daemons = []
-  port = int(sys.argv[1])
+wallets = []
-except:
+rpcs = []
-  print(USAGE)
+for n in range(1, len(sys.argv)):
-  sys.exit(1)
+  scheme='http'
+  host='127.0.0.1'
+  port=None
+  try:
+    try:
+      port = int(sys.argv[n])
+    except:
+      t = urlparse.urlparse(sys.argv[n], allow_fragments = False)
+      scheme = t.scheme or scheme
+      host = t.hostname or host
+      port = t.port or port
+      if scheme != 'http' and scheme != 'https':
+        raise Exception(USAGE)
+      if port <= 0 or port > 65535:
+        raise Exception(USAGE)
+  except Exception, e:
+    print('Error: ' + str(e))
+    raise Exception(USAGE)
 
-# check for open port
+  # check for open port
-s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+  s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-s.settimeout(1)
+  s.settimeout(1)
-if s.connect_ex(('127.0.0.1', port)) != 0:
+  if s.connect_ex((host, port)) != 0:
-  print('No wallet or daemon RPC on port ' + str(port))
+    raise Exception('No wallet or daemon RPC on port ' + str(port))
-  sys.exit(1)
+  s.close()
-s.close()
 
-# both wallet and daemon have a get_version JSON RPC
+  # both wallet and daemon have a get_version JSON RPC
-rpc = rpc.JSONRPC('{protocol}://{host}:{port}'.format(protocol='http', host='127.0.0.1', port=port))
+  rpc = framework.rpc.JSONRPC('{protocol}://{host}:{port}'.format(protocol=scheme, host=host, port=port))
-get_version = {
+  get_version = {
       'method': 'get_version',
       'jsonrpc': '2.0', 
       'id': '0'
-}
+  }
-try:
+  try:
     res = rpc.send_json_rpc_request(get_version)
-except Exception, e:
+  except Exception, e:
-  print('Failed to call version RPC: ' + str(e))
+    raise Exception('Failed to call version RPC: ' + str(e))
-  sys.exit(1)
 
-if 'version' not in res:
+  if 'version' not in res:
-  print('Server is not a monero process')
+    raise Exception('Server is not a Monero process')
-  sys.exit(1)
 
-if 'status' in res:
+  if 'status' in res:
-  rpc = daemon.Daemon(port=port)
+    daemons.append(framework.daemon.Daemon(port=port))
+    rpcs.append(daemons[-1])
+  else:
+    wallets.append(framework.wallet.Wallet(port=port))
+    rpcs.append(wallets[-1])
+
+# add tab completion if we can: https://stackoverflow.com/questions/246725
+try:
+  import readline
+except:
+  pass
 else:
-  rpc = wallet.Wallet(port=port)
+  import rlcompleter
+  readline.parse_and_bind('tab: complete')
 
-print('Connected to %s RPC on port %u' % ('daemon' if 'status' in res else 'wallet', port))
+if len(daemons) == 1:
-print('The \'rpc\' object may now be used to use the API')
+  daemon = daemons[0]
+if len(wallets) == 1:
+  wallet = wallets[0]
+
+didx = 0
+widx = 0
+for rpc in rpcs:
+  if type(rpc) == framework.daemon.Daemon:
+    var = "daemon" if len(daemons) == 1 else "daemons[" + str(didx) + "]"
+    didx += 1
+  else:
+    var = "wallet" if len(wallets) == 1 else "wallets[" + str(widx) + "]"
+    widx += 1
+  print('Variable \'%s\' connected to %s RPC on %s:%u' % (var, 'daemon' if type(rpc) == framework.daemon.Daemon else 'wallet', rpc.host ,rpc.port))

utils/python-rpc/framework/daemon.py

@@ -33,14 +33,17 @@ from .rpc import JSONRPC
 class Daemon(object):
 
     def __init__(self, protocol='http', host='127.0.0.1', port=0, idx=0):
+        self.host = host
+        self.port = port
         self.rpc = JSONRPC('{protocol}://{host}:{port}'.format(protocol=protocol, host=host, port=port if port else 18180+idx))
 
-    def getblocktemplate(self, address):
+    def getblocktemplate(self, address, prev_block = ""):
         getblocktemplate = {
             'method': 'getblocktemplate',
             'params': {
                 'wallet_address': address,
-                'reserve_size' : 1
+                'reserve_size' : 1,
+                'prev_block' : prev_block,
             },
             'jsonrpc': '2.0', 
             'id': '0'
@@ -143,13 +146,15 @@ class Daemon(object):
         }    
         return self.rpc.send_json_rpc_request(hard_fork_info)
 
-    def generateblocks(self, address, blocks=1):
+    def generateblocks(self, address, blocks=1, prev_block = "", starting_nonce = 0):
         generateblocks = {
             'method': 'generateblocks',
             'params': {
                 'amount_of_blocks' : blocks,
                 'reserve_size' : 20,
-                'wallet_address': address
+                'wallet_address': address,
+                'prev_block': prev_block,
+                'starting_nonce': starting_nonce,
             },
             'jsonrpc': '2.0', 
             'id': '0'
@@ -217,3 +222,110 @@ class Daemon(object):
             'id': '0'
         }
         return self.rpc.send_json_rpc_request(get_version)
+
+    def get_bans(self):
+        get_bans = {
+            'method': 'get_bans',
+            'params': {
+            },
+            'jsonrpc': '2.0',
+            'id': '0'
+        }
+        return self.rpc.send_json_rpc_request(get_bans)
+
+    def set_bans(self, bans = []):
+        set_bans = {
+            'method': 'set_bans',
+            'params': {
+                'bans': bans
+            },
+            'jsonrpc': '2.0',
+            'id': '0'
+        }
+        return self.rpc.send_json_rpc_request(set_bans)
+
+    def get_transactions(self, txs_hashes = [], decode_as_json = False, prune = False, split = False):
+        get_transactions = {
+            'txs_hashes': txs_hashes,
+            'decode_as_json': decode_as_json,
+            'prune': prune,
+            'split': split,
+        }
+        return self.rpc.send_request('/get_transactions', get_transactions)
+
+    def get_outs(self, outputs = [], get_txid = False):
+        get_outs = {
+            'outputs': outputs,
+            'get_txid': get_txid,
+        }
+        return self.rpc.send_request('/get_outs', get_outs)
+
+    def get_coinbase_tx_sum(self, height, count):
+        get_coinbase_tx_sum = {
+            'method': 'get_coinbase_tx_sum',
+            'params': {
+                'height': height,
+                'count': count,
+            },
+            'jsonrpc': '2.0',
+            'id': '0'
+        }
+        return self.rpc.send_json_rpc_request(get_coinbase_tx_sum)
+
+    def get_output_distribution(self, amounts = [], from_height = 0, to_height = 0, cumulative = False, binary = False, compress = False):
+        get_output_distribution = {
+            'method': 'get_output_distribution',
+            'params': {
+                'amounts': amounts,
+                'from_height': from_height,
+                'to_height': to_height,
+                'cumulative': cumulative,
+                'binary': binary,
+                'compress': compress,
+            },
+            'jsonrpc': '2.0',
+            'id': '0'
+        }
+        return self.rpc.send_json_rpc_request(get_output_distribution)
+
+    def get_output_histogram(self, amounts = [], min_count = 0, max_count = 0, unlocked = False, recent_cutoff = 0):
+        get_output_histogram = {
+            'method': 'get_output_histogram',
+            'params': {
+                'amounts': amounts,
+                'min_count': min_count,
+                'max_count': max_count,
+                'unlocked': unlocked,
+                'recent_cutoff': recent_cutoff,
+            },
+            'jsonrpc': '2.0',
+            'id': '0'
+        }
+        return self.rpc.send_json_rpc_request(get_output_histogram)
+
+    def set_log_level(self, level):
+        set_log_level = {
+            'level': level,
+        }
+        return self.rpc.send_request('/set_log_level', set_log_level)
+
+    def set_log_categories(self, categories = ''):
+        set_log_categories = {
+            'categories': categories,
+        }
+        return self.rpc.send_request('/set_log_categories', set_log_categories)
+
+    def get_alt_blocks_hashes(self):
+        get_alt_blocks_hashes = {
+        }
+        return self.rpc.send_request('/get_alt_blocks_hashes', get_alt_blocks_hashes)
+
+    def get_alternate_chains(self):
+        get_alternate_chains = {
+            'method': 'get_alternate_chains',
+            'params': {
+            },
+            'jsonrpc': '2.0',
+            'id': '0'
+        }
+        return self.rpc.send_json_rpc_request(get_alternate_chains)

utils/python-rpc/framework/wallet.py

@@ -33,6 +33,8 @@ from .rpc import JSONRPC
 class Wallet(object):
 
     def __init__(self, protocol='http', host='127.0.0.1', port=0, idx=0):
+        self.host = host
+        self.port = port
         self.rpc = JSONRPC('{protocol}://{host}:{port}'.format(protocol=protocol, host=host, port=port if port else 18090+idx))
 
     def make_uniform_destinations(self, address, transfer_amount, transfer_number_of_destinations=1):
@@ -89,6 +91,18 @@ class Wallet(object):
         }
         return self.rpc.send_json_rpc_request(transfer)   
 
+    def get_transfer_by_txid(self, txid, account_index = 0):
+        get_transfer_by_txid = {
+            'method': 'get_transfer_by_txid',
+            'params': {
+                'txid': txid,
+                'account_index': account_index,
+            },
+            'jsonrpc': '2.0', 
+            'id': '0'
+        }
+        return self.rpc.send_json_rpc_request(get_transfer_by_txid)
+
     def get_bulk_payments(self, payment_ids = [], min_block_height = 0):
         get_bulk_payments = {
             'method': 'get_bulk_payments',
@@ -101,24 +115,25 @@ class Wallet(object):
         }
         return self.rpc.send_json_rpc_request(get_bulk_payments)
 
-    def describe_transfer(self, unsigned_txset):
+    def describe_transfer(self, unsigned_txset = '', multisig_txset = ''):
         describe_transfer = {
             'method': 'describe_transfer',
             'params': {
                 'unsigned_txset': unsigned_txset,
+                'multisig_txset': multisig_txset,
             },
             'jsonrpc': '2.0', 
             'id': '0'
         }
         return self.rpc.send_json_rpc_request(describe_transfer)
 
-    def create_wallet(self, index=''):
+    def create_wallet(self, filename='', password = '', language = 'English'):
         create_wallet = {
             'method': 'create_wallet',
             'params': {
-                'filename': 'testWallet' + index,
+                'filename': filename,
-                'password' : '',
+                'password': password,
-                'language' : 'English'
+                'language': language
             },
             'jsonrpc': '2.0', 
             'id': '0'
@@ -146,11 +161,23 @@ class Wallet(object):
         }
         return self.rpc.send_json_rpc_request(sweep_dust)
 
-    def sweep_all(self, address):
+    def sweep_all(self, address = '', account_index = 0, subaddr_indices = [], priority = 0, ring_size = 0, outputs = 1, unlock_time = 0, payment_id = '', get_tx_keys = False, below_amount = 0, do_not_relay = False, get_tx_hex = False, get_tx_metadata = False):
         sweep_all = {
             'method': 'sweep_all',
             'params' : {
-                'address' : ''
+                'address' : address,
+                'account_index' : account_index,
+                'subaddr_indices' : subaddr_indices,
+                'priority' : priority,
+                'ring_size' : ring_size,
+                'outputs' : outputs,
+                'unlock_time' : unlock_time,
+                'payment_id' : payment_id,
+                'get_tx_keys' : get_tx_keys,
+                'below_amount' : below_amount,
+                'do_not_relay' : do_not_relay,
+                'get_tx_hex' : get_tx_hex,
+                'get_tx_metadata' : get_tx_metadata,
             },
             'jsonrpc': '2.0', 
             'id': '0'
@@ -591,6 +618,79 @@ class Wallet(object):
         }
         return self.rpc.send_json_rpc_request(verify)
 
+    def get_height(self):
+        get_height = {
+            'method': 'get_height',
+            'jsonrpc': '2.0', 
+            'id': '0'
+        }
+        return self.rpc.send_json_rpc_request(get_height)
+
+    def relay_tx(self, hex_):
+        relay_tx = {
+            'method': 'relay_tx',
+            'params': {
+                'hex': hex_,
+            },
+            'jsonrpc': '2.0', 
+            'id': '0'
+        }
+        return self.rpc.send_json_rpc_request(relay_tx)
+
+    def get_languages(self):
+        get_languages = {
+            'method': 'get_languages',
+            'params': {
+            },
+            'jsonrpc': '2.0', 
+            'id': '0'
+        }
+        return self.rpc.send_json_rpc_request(get_languages)
+
+    def export_outputs(self):
+        export_outputs = {
+            'method': 'export_outputs',
+            'params': {
+            },
+            'jsonrpc': '2.0', 
+            'id': '0'
+        }
+        return self.rpc.send_json_rpc_request(export_outputs)
+
+    def import_outputs(self, outputs_data_hex):
+        import_outputs = {
+            'method': 'import_outputs',
+            'params': {
+                'outputs_data_hex': outputs_data_hex
+            },
+            'jsonrpc': '2.0', 
+            'id': '0'
+        }
+        return self.rpc.send_json_rpc_request(import_outputs)
+
+    def export_key_images(self, all_ = False):
+        export_key_images = {
+            'method': 'export_key_images',
+            'params': {
+                'all': all_
+            },
+            'jsonrpc': '2.0', 
+            'id': '0'
+        }
+        return self.rpc.send_json_rpc_request(export_key_images)
+
+    def import_key_images(self, signed_key_images, offset = 0):
+        import_key_images = {
+            'method': 'import_key_images',
+            'params': {
+                'offset': offset,
+                'signed_key_images': signed_key_images,
+            },
+            'jsonrpc': '2.0', 
+            'id': '0'
+        }
+        return self.rpc.send_json_rpc_request(import_key_images)
+
     def get_version(self):
         get_version = {
             'method': 'get_version',