Merge pull request #155 from wowario/upstream

Upstream

CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2018, The Monero Project
+# Copyright (c) 2014-2019, The Monero Project
 #
 # All rights reserved.
 #

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2014-2018, The Monero Project
+Copyright (c) 2014-2019, The Monero Project
 
 All rights reserved.
 

Makefile

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2018, The Monero Project
+# Copyright (c) 2014-2019, The Monero Project
 #
 # All rights reserved.
 #

cmake/32-bit-toolchain.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2018, The Monero Project
+# Copyright (c) 2014-2019, The Monero Project
 # 
 # All rights reserved.
 # 

cmake/64-bit-toolchain.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2018, The Monero Project
+# Copyright (c) 2014-2019, The Monero Project
 # 
 # All rights reserved.
 # 

cmake/FindUnbound.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2018, The Monero Project
+# Copyright (c) 2014-2019, The Monero Project
 # All rights reserved.
 # 
 # Redistribution and use in source and binary forms, with or without modification, are

cmake/GenVersion.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2018, The Monero Project
+# Copyright (c) 2014-2019, The Monero Project
 # 
 # All rights reserved.
 # 

cmake/Version.cmake

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2018, The Monero Project
+# Copyright (c) 2014-2019, The Monero Project
 # 
 # All rights reserved.
 # 

cmake/test-libusb-version.c

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 // 
 // All rights reserved.
 // 

cmake/test-protobuf.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 // 
 // All rights reserved.
 // 

cmake/test-static-assert.c

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 // 
 // All rights reserved.
 // 

cmake/test-static-assert.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 // 
 // All rights reserved.
 // 

contrib/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2018, The Monero Project
+# Copyright (c) 2014-2019, The Monero Project
 # 
 # All rights reserved.
 # 

contrib/depends/packages/boost.mk

@@ -20,7 +20,7 @@ $(package)_archiver_$(host_os)=$($(package)_ar)
 $(package)_toolset_darwin=darwin
 $(package)_archiver_darwin=$($(package)_libtool)
 $(package)_config_libraries=chrono,filesystem,program_options,system,thread,test,date_time,regex,serialization,locale
-$(package)_cxxflags=-std=c++11 -fvisibility=hidden
+$(package)_cxxflags=-std=c++11
 $(package)_cxxflags_linux=-fPIC
 endef
 

contrib/epee/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2018, The Monero Project
+# Copyright (c) 2014-2019, The Monero Project
 # 
 # All rights reserved.
 # 

contrib/epee/include/int-util.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 // 
 // All rights reserved.
 // 

contrib/epee/include/net/buffer.h

@@ -0,0 +1,62 @@
+// Copyright (c) 2018, The Monero Project
+// 
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+// 
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+// 
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+// 
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include <vector>
+#include "misc_log_ex.h"
+#include "span.h"
+
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "net.buffer"
+
+//#define NET_BUFFER_LOG(x) MDEBUG(x)
+#define NET_BUFFER_LOG(x) ((void)0)
+
+namespace epee
+{
+namespace net_utils
+{
+class buffer
+{
+public:
+  buffer(size_t reserve = 0): offset(0) { storage.reserve(reserve); }
+
+  void append(const void *data, size_t sz);
+  void erase(size_t sz) { NET_BUFFER_LOG("erasing " << sz << "/" << size()); CHECK_AND_ASSERT_THROW_MES(offset + sz <= storage.size(), "erase: sz too large"); offset += sz; if (offset == storage.size()) { storage.resize(0); offset = 0; } }
+  epee::span<const uint8_t> span(size_t sz) const { CHECK_AND_ASSERT_THROW_MES(sz <= size(), "span is too large"); return epee::span<const uint8_t>(storage.data() + offset, sz); }
+  // carve must keep the data in scope till next call, other API calls (such as append, erase) can invalidate the carved buffer
+  epee::span<const uint8_t> carve(size_t sz) { CHECK_AND_ASSERT_THROW_MES(sz <= size(), "span is too large"); offset += sz; return epee::span<const uint8_t>(storage.data() + offset - sz, sz); }
+  size_t size() const { return storage.size() - offset; }
+
+private:
+  std::vector<uint8_t> storage;
+  size_t offset;
+};
+}
+}

contrib/epee/include/net/connection_basic.hpp

@@ -8,7 +8,7 @@
 // ! (how ever if in some wonderful juristdictions that is not the case, then why not make another sub-class withat that members and licence it as epee part)
 // ! Working on above premise, IF this is valid in your juristdictions, then consider this code as released as:
 
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 // 
 // All rights reserved.
 // 

contrib/epee/include/net/http_auth.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 //
 // All rights reserved.
 //

contrib/epee/include/net/http_server_handlers_map2.h

@@ -92,7 +92,7 @@
       handled = true; \
       uint64_t ticks = misc_utils::get_tick_count(); \
       boost::value_initialized<command_type::request> req; \
-      bool parse_res = epee::serialization::load_t_from_binary(static_cast<command_type::request&>(req), query_info.m_body); \
+      bool parse_res = epee::serialization::load_t_from_binary(static_cast<command_type::request&>(req), epee::strspan<uint8_t>(query_info.m_body)); \
       CHECK_AND_ASSERT_MES(parse_res, false, "Failed to parse bin body data, body size=" << query_info.m_body.size()); \
       uint64_t ticks1 = misc_utils::get_tick_count(); \
       boost::value_initialized<command_type::response> resp;\

contrib/epee/include/net/levin_base.h

@@ -80,8 +80,8 @@ namespace levin
   template<class t_connection_context = net_utils::connection_context_base>
   struct levin_commands_handler
   {
-    virtual int invoke(int command, const std::string& in_buff, std::string& buff_out, t_connection_context& context)=0;
+    virtual int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, t_connection_context& context)=0;
-    virtual int notify(int command, const std::string& in_buff, t_connection_context& context)=0;
+    virtual int notify(int command, const epee::span<const uint8_t> in_buff, t_connection_context& context)=0;
     virtual void callback(t_connection_context& context){};
 
     virtual void on_connection_new(t_connection_context& context){};

contrib/epee/include/net/levin_client.h

@@ -57,7 +57,7 @@ namespace levin
 		bool is_connected();
 		bool disconnect();
 
-		virtual int invoke(int command, const std::string& in_buff, std::string& buff_out);
+		virtual int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out);
 		virtual int notify(int command, const std::string& in_buff);
 
 	protected: 
@@ -72,7 +72,7 @@ namespace levin
   {
   public:
 
-    int invoke(int command, const std::string& in_buff, std::string& buff_out);
+    int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out);
     int notify(int command, const std::string& in_buff);
   };
 

contrib/epee/include/net/levin_client.inl

@@ -74,7 +74,7 @@ levin_client_impl::~levin_client_impl()
 }
 //------------------------------------------------------------------------------
 inline
-int levin_client_impl::invoke(int command, const std::string& in_buff, std::string& buff_out)
+int levin_client_impl::invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out)
 {
 	if(!is_connected())
 		return -1;
@@ -133,7 +133,7 @@ int levin_client_impl::notify(int command, const std::string& in_buff)
 //------------------------------------------------------------------------------
 //------------------------------------------------------------------------------
 inline
-  int levin_client_impl2::invoke(int command, const std::string& in_buff, std::string& buff_out)
+  int levin_client_impl2::invoke(int command, epee::span<const uint8_t>string& in_buff, std::string& buff_out)
 {
   if(!is_connected())
     return -1;

contrib/epee/include/net/levin_protocol_handler_async.h

@@ -34,6 +34,7 @@
 #include <atomic>
 
 #include "levin_base.h"
+#include "buffer.h"
 #include "misc_language.h"
 #include "syncobj.h"
 #include "misc_os_dependent.h"
@@ -85,11 +86,11 @@ public:
   uint64_t m_max_packet_size; 
   uint64_t m_invoke_timeout;
 
-  int invoke(int command, const std::string& in_buff, std::string& buff_out, boost::uuids::uuid connection_id);
+  int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, boost::uuids::uuid connection_id);
   template<class callback_t>
-  int invoke_async(int command, const std::string& in_buff, boost::uuids::uuid connection_id, const callback_t &cb, size_t timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED);
+  int invoke_async(int command, const epee::span<const uint8_t> in_buff, boost::uuids::uuid connection_id, const callback_t &cb, size_t timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED);
 
-  int notify(int command, const std::string& in_buff, boost::uuids::uuid connection_id);
+  int notify(int command, const epee::span<const uint8_t> in_buff, boost::uuids::uuid connection_id);
   bool close(boost::uuids::uuid connection_id);
   bool update_connection_context(const t_connection_context& contxt);
   bool request_callback(boost::uuids::uuid connection_id);
@@ -143,7 +144,7 @@ public:
   config_type& m_config;
   t_connection_context& m_connection_context;
 
-  std::string m_cache_in_buffer;
+  net_utils::buffer m_cache_in_buffer;
   stream_state m_state;
 
   int32_t m_oponent_protocol_ver;
@@ -151,7 +152,7 @@ public:
 
   struct invoke_response_handler_base
   {
-    virtual bool handle(int res, const std::string& buff, connection_context& context)=0;
+    virtual bool handle(int res, const epee::span<const uint8_t> buff, connection_context& context)=0;
     virtual bool is_timer_started() const=0;
     virtual void cancel()=0;
     virtual bool cancel_timer()=0;
@@ -173,7 +174,7 @@ public:
           if(ec == boost::asio::error::operation_aborted)
             return;
           MINFO(con.get_context_ref() << "Timeout on invoke operation happened, command: " << command << " timeout: " << timeout);
-          std::string fake;
+          epee::span<const uint8_t> fake;
           cb(LEVIN_ERROR_CONNECTION_TIMEDOUT, fake, con.get_context_ref());
           con.close();
           con.finish_outer_call();
@@ -191,7 +192,7 @@ public:
     bool m_timer_cancelled;
     uint64_t m_timeout;
     int m_command;
-    virtual bool handle(int res, const std::string& buff, typename async_protocol_handler::connection_context& context)
+    virtual bool handle(int res, const epee::span<const uint8_t> buff, typename async_protocol_handler::connection_context& context)
     {
       if(!cancel_timer())
         return false;
@@ -207,7 +208,7 @@ public:
     {
       if(cancel_timer())
       {
-        std::string fake;
+        epee::span<const uint8_t> fake;
         m_cb(LEVIN_ERROR_CONNECTION_DESTROYED, fake, m_con.get_context_ref());
         m_con.finish_outer_call();
       }
@@ -237,7 +238,7 @@ public:
           if(ec == boost::asio::error::operation_aborted)
             return;
           MINFO(con.get_context_ref() << "Timeout on invoke operation happened, command: " << command << " timeout: " << timeout);
-          std::string fake;
+          epee::span<const uint8_t> fake;
           cb(LEVIN_ERROR_CONNECTION_TIMEDOUT, fake, con.get_context_ref());
           con.close();
           con.finish_outer_call();
@@ -265,6 +266,7 @@ public:
             m_pservice_endpoint(psnd_hndlr), 
             m_config(config), 
             m_connection_context(conn_context), 
+            m_cache_in_buffer(256 * 1024),
             m_state(stream_state_head)
   {
     m_close_called = 0;
@@ -405,14 +407,7 @@ public:
           break;
         }
         {
-          std::string buff_to_invoke;
+          epee::span<const uint8_t> buff_to_invoke = m_cache_in_buffer.carve((std::string::size_type)m_current_head.m_cb);
-          if(m_cache_in_buffer.size()  == m_current_head.m_cb)
-            buff_to_invoke.swap(m_cache_in_buffer);
-          else
-          {
-            buff_to_invoke.assign(m_cache_in_buffer, 0, (std::string::size_type)m_current_head.m_cb);
-            m_cache_in_buffer.erase(0, (std::string::size_type)m_current_head.m_cb);
-          }
 
           bool is_response = (m_oponent_protocol_ver == LEVIN_PROTOCOL_VER_1 && m_current_head.m_flags&LEVIN_PACKET_RESPONSE);
 
@@ -449,8 +444,8 @@ public:
               }else
               {
                 CRITICAL_REGION_BEGIN(m_local_inv_buff_lock);
-                buff_to_invoke.swap(m_local_inv_buff);
+                m_local_inv_buff = std::string((const char*)buff_to_invoke.data(), buff_to_invoke.size());
-                buff_to_invoke.clear();
+                buff_to_invoke = epee::span<const uint8_t>((const uint8_t*)NULL, 0);
                 m_invoke_result_code = m_current_head.m_return_code;
                 CRITICAL_REGION_END();
                 boost::interprocess::ipcdetail::atomic_write32(&m_invoke_buf_ready, 1);
@@ -503,7 +498,7 @@ public:
         {
           if(m_cache_in_buffer.size() < sizeof(bucket_head2))
           {
-            if(m_cache_in_buffer.size() >= sizeof(uint64_t) && *((uint64_t*)m_cache_in_buffer.data()) != SWAP64LE(LEVIN_SIGNATURE))
+            if(m_cache_in_buffer.size() >= sizeof(uint64_t) && *((uint64_t*)m_cache_in_buffer.span(8).data()) != SWAP64LE(LEVIN_SIGNATURE))
             {
               MWARNING(m_connection_context << "Signature mismatch, connection will be closed");
               return false;
@@ -513,9 +508,9 @@ public:
           }
 
 #if BYTE_ORDER == LITTLE_ENDIAN
-          bucket_head2& phead = *(bucket_head2*)m_cache_in_buffer.data();
+          bucket_head2& phead = *(bucket_head2*)m_cache_in_buffer.span(sizeof(bucket_head2)).data();
 #else
-          bucket_head2 phead = *(bucket_head2*)m_cache_in_buffer.data();
+          bucket_head2 phead = *(bucket_head2*)m_cache_in_buffer.span(sizeof(bucket_head2)).data();
           phead.m_signature = SWAP64LE(phead.m_signature);
           phead.m_cb = SWAP64LE(phead.m_cb);
           phead.m_command = SWAP32LE(phead.m_command);
@@ -530,7 +525,7 @@ public:
           }
           m_current_head = phead;
 
-          m_cache_in_buffer.erase(0, sizeof(bucket_head2));
+          m_cache_in_buffer.erase(sizeof(bucket_head2));
           m_state = stream_state_body;
           m_oponent_protocol_ver = m_current_head.m_protocol_version;
           if(m_current_head.m_cb > m_config.m_max_packet_size)
@@ -562,7 +557,7 @@ public:
   }
 
   template<class callback_t>
-  bool async_invoke(int command, const std::string& in_buff, const callback_t &cb, size_t timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED)
+  bool async_invoke(int command, const epee::span<const uint8_t> in_buff, const callback_t &cb, size_t timeout = LEVIN_DEFAULT_TIMEOUT_PRECONFIGURED)
   {
     misc_utils::auto_scope_leave_caller scope_exit_handler = misc_utils::create_scope_leave_handler(
       boost::bind(&async_protocol_handler::finish_outer_call, this));
@@ -606,7 +601,7 @@ public:
         break;
       }
 
-      if(!m_pservice_endpoint->do_send(in_buff.data(), (int)in_buff.size()))
+      if(!m_pservice_endpoint->do_send(in_buff.data(), in_buff.size()))
       {
         LOG_ERROR_CC(m_connection_context, "Failed to do_send");
         err_code = LEVIN_ERROR_CONNECTION;
@@ -623,7 +618,7 @@ public:
 
     if (LEVIN_OK != err_code)
     {
-      std::string stub_buff;
+      epee::span<const uint8_t> stub_buff{(const uint8_t*)"", 0};
       // Never call callback inside critical section, that can cause deadlock
       cb(err_code, stub_buff, m_connection_context);
       return false;
@@ -632,7 +627,7 @@ public:
     return true;
   }
 
-  int invoke(int command, const std::string& in_buff, std::string& buff_out)
+  int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out)
   {
     misc_utils::auto_scope_leave_caller scope_exit_handler = misc_utils::create_scope_leave_handler(
                                       boost::bind(&async_protocol_handler::finish_outer_call, this));
@@ -662,7 +657,7 @@ public:
       return LEVIN_ERROR_CONNECTION;
     }
 
-    if(!m_pservice_endpoint->do_send(in_buff.data(), (int)in_buff.size()))
+    if(!m_pservice_endpoint->do_send(in_buff.data(), in_buff.size()))
     {
       LOG_ERROR_CC(m_connection_context, "Failed to do_send");
       return LEVIN_ERROR_CONNECTION;
@@ -706,7 +701,7 @@ public:
     return m_invoke_result_code;
   }
 
-  int notify(int command, const std::string& in_buff)
+  int notify(int command, const epee::span<const uint8_t> in_buff)
   {
     misc_utils::auto_scope_leave_caller scope_exit_handler = misc_utils::create_scope_leave_handler(
                           boost::bind(&async_protocol_handler::finish_outer_call, this));
@@ -734,7 +729,7 @@ public:
       return -1;
     }
 
-    if(!m_pservice_endpoint->do_send(in_buff.data(), (int)in_buff.size()))
+    if(!m_pservice_endpoint->do_send(in_buff.data(), in_buff.size()))
     {
       LOG_ERROR_CC(m_connection_context, "Failed to do_send()");
       return -1;
@@ -839,7 +834,7 @@ int async_protocol_handler_config<t_connection_context>::find_and_lock_connectio
 }
 //------------------------------------------------------------------------------------------
 template<class t_connection_context>
-int async_protocol_handler_config<t_connection_context>::invoke(int command, const std::string& in_buff, std::string& buff_out, boost::uuids::uuid connection_id)
+int async_protocol_handler_config<t_connection_context>::invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, boost::uuids::uuid connection_id)
 {
   async_protocol_handler<t_connection_context>* aph;
   int r = find_and_lock_connection(connection_id, aph);
@@ -847,7 +842,7 @@ int async_protocol_handler_config<t_connection_context>::invoke(int command, con
 }
 //------------------------------------------------------------------------------------------
 template<class t_connection_context> template<class callback_t>
-int async_protocol_handler_config<t_connection_context>::invoke_async(int command, const std::string& in_buff, boost::uuids::uuid connection_id, const callback_t &cb, size_t timeout)
+int async_protocol_handler_config<t_connection_context>::invoke_async(int command, const epee::span<const uint8_t> in_buff, boost::uuids::uuid connection_id, const callback_t &cb, size_t timeout)
 {
   async_protocol_handler<t_connection_context>* aph;
   int r = find_and_lock_connection(connection_id, aph);
@@ -896,7 +891,7 @@ void async_protocol_handler_config<t_connection_context>::set_handler(levin_comm
 }
 //------------------------------------------------------------------------------------------
 template<class t_connection_context>
-int async_protocol_handler_config<t_connection_context>::notify(int command, const std::string& in_buff, boost::uuids::uuid connection_id)
+int async_protocol_handler_config<t_connection_context>::notify(int command, const epee::span<const uint8_t> in_buff, boost::uuids::uuid connection_id)
 {
   async_protocol_handler<t_connection_context>* aph;
   int r = find_and_lock_connection(connection_id, aph);

contrib/epee/include/net/network_throttle-detail.hpp

@@ -2,7 +2,7 @@
 /// @author rfree (current maintainer in monero.cc project)
 /// @brief implementaion for throttling of connection (count and rate-limit speed etc)
 
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 // 
 // All rights reserved.
 // 

contrib/epee/include/net/network_throttle.hpp

@@ -2,7 +2,7 @@
 /// @author rfree (current maintainer in monero.cc project)
 /// @brief interface for throttling of connection (count and rate-limit speed etc)
 
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 // 
 // All rights reserved.
 // 

contrib/epee/include/span.h

@@ -163,4 +163,12 @@ namespace epee
     static_assert(!has_padding<T>(), "source type may have padding");
     return {reinterpret_cast<std::uint8_t*>(std::addressof(src)), sizeof(T)};
   }
+
+  //! make a span from a std::string
+  template<typename T>
+  span<const T> strspan(const std::string &s) noexcept
+  {
+    static_assert(std::is_same<T, char>() || std::is_same<T, unsigned char>() || std::is_same<T, int8_t>() || std::is_same<T, uint8_t>(), "Unexpected type");
+    return {reinterpret_cast<const T*>(s.data()), s.size()};
+  }
 }

contrib/epee/include/storages/http_abstract_invoke.h

@@ -97,7 +97,7 @@ namespace epee
         return false;
       }
 
-      return serialization::load_t_from_binary(result_struct, pri->m_body);
+      return serialization::load_t_from_binary(result_struct, epee::strspan<uint8_t>(pri->m_body));
     }
 
     template<class t_request, class t_response, class t_transport>

contrib/epee/include/storages/levin_abstract_invoke2.h

@@ -28,6 +28,7 @@
 
 #include "portable_storage_template_helper.h"
 #include <boost/utility/value_init.hpp>
+#include "span.h"
 #include "net/levin_base.h"
 
 #undef MONERO_DEFAULT_LOG_CATEGORY
@@ -114,7 +115,7 @@ namespace epee
       const_cast<t_arg&>(out_struct).store(stg);//TODO: add true const support to searilzation
       std::string buff_to_send;
       stg.store_to_binary(buff_to_send);
-      int res = transport.invoke_async(command, buff_to_send, conn_id, [cb, command](int code, const std::string& buff, typename t_transport::connection_context& context)->bool 
+      int res = transport.invoke_async(command, epee::strspan<uint8_t>(buff_to_send), conn_id, [cb, command](int code, const epee::span<const uint8_t> buff, typename t_transport::connection_context& context)->bool 
       {
         t_result result_struct = AUTO_VAL_INIT(result_struct);
         if( code <=0 )
@@ -156,7 +157,7 @@ namespace epee
       std::string buff_to_send;
       stg.store_to_binary(buff_to_send);
 
-      int res = transport.notify(command, buff_to_send, conn_id);
+      int res = transport.notify(command, epee::strspan<uint8_t>(buff_to_send), conn_id);
       if(res <=0 )
       {
         MERROR("Failed to notify command " << command << " return code " << res);
@@ -167,7 +168,7 @@ namespace epee
     //----------------------------------------------------------------------------------------------------
     //----------------------------------------------------------------------------------------------------
     template<class t_owner, class t_in_type, class t_out_type, class t_context, class callback_t>
-    int buff_to_t_adapter(int command, const std::string& in_buff, std::string& buff_out, callback_t cb, t_context& context )
+    int buff_to_t_adapter(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, callback_t cb, t_context& context )
     {
       serialization::portable_storage strg;
       if(!strg.load_from_binary(in_buff))
@@ -197,7 +198,7 @@ namespace epee
     }
 
     template<class t_owner, class t_in_type, class t_context, class callback_t>
-    int buff_to_t_adapter(t_owner* powner, int command, const std::string& in_buff, callback_t cb, t_context& context)
+    int buff_to_t_adapter(t_owner* powner, int command, const epee::span<const uint8_t> in_buff, callback_t cb, t_context& context)
     {
       serialization::portable_storage strg;
       if(!strg.load_from_binary(in_buff))
@@ -215,14 +216,14 @@ namespace epee
     }
 
 #define CHAIN_LEVIN_INVOKE_MAP2(context_type) \
-  int invoke(int command, const std::string& in_buff, std::string& buff_out, context_type& context) \
+  int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, context_type& context) \
   { \
   bool handled = false; \
   return handle_invoke_map(false, command, in_buff, buff_out, context, handled); \
   } 
 
 #define CHAIN_LEVIN_NOTIFY_MAP2(context_type) \
-  int notify(int command, const std::string& in_buff, context_type& context) \
+  int notify(int command, const epee::span<const uint8_t> in_buff, context_type& context) \
   { \
   bool handled = false; std::string fake_str;\
   return handle_invoke_map(true, command, in_buff, fake_str, context, handled); \
@@ -230,27 +231,27 @@ namespace epee
 
 
 #define CHAIN_LEVIN_INVOKE_MAP() \
-  int invoke(int command, const std::string& in_buff, std::string& buff_out, epee::net_utils::connection_context_base& context) \
+  int invoke(int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, epee::net_utils::connection_context_base& context) \
   { \
   bool handled = false; \
   return handle_invoke_map(false, command, in_buff, buff_out, context, handled); \
   } 
 
 #define CHAIN_LEVIN_NOTIFY_MAP() \
-  int notify(int command, const std::string& in_buff, epee::net_utils::connection_context_base& context) \
+  int notify(int command, const epee::span<const uint8_t> in_buff, epee::net_utils::connection_context_base& context) \
   { \
   bool handled = false; std::string fake_str;\
   return handle_invoke_map(true, command, in_buff, fake_str, context, handled); \
   } 
 
 #define CHAIN_LEVIN_NOTIFY_STUB() \
-  int notify(int command, const std::string& in_buff, epee::net_utils::connection_context_base& context) \
+  int notify(int command, const epee::span<const uint8_t> in_buff, epee::net_utils::connection_context_base& context) \
   { \
   return -1; \
   } 
 
 #define BEGIN_INVOKE_MAP2(owner_type) \
-  template <class t_context> int handle_invoke_map(bool is_notify, int command, const std::string& in_buff, std::string& buff_out, t_context& context, bool& handled) \
+  template <class t_context> int handle_invoke_map(bool is_notify, int command, const epee::span<const uint8_t> in_buff, std::string& buff_out, t_context& context, bool& handled) \
   { \
   typedef owner_type internal_owner_type_name;
 

contrib/epee/include/storages/portable_storage.h

@@ -35,6 +35,7 @@
 #include "portable_storage_to_json.h"
 #include "portable_storage_from_json.h"
 #include "portable_storage_val_converters.h"
+#include "span.h"
 #include "int-util.h"
 
 namespace epee
@@ -81,7 +82,8 @@ namespace epee
 
       //-------------------------------------------------------------------------------
       bool		store_to_binary(binarybuffer& target);
-      bool		load_from_binary(const binarybuffer& target);
+      bool		load_from_binary(const epee::span<const uint8_t> target);
+      bool		load_from_binary(const std::string& target) { return load_from_binary(epee::strspan<uint8_t>(target)); }
       template<class trace_policy>
       bool		  dump_as_xml(std::string& targetObj, const std::string& root_name = "");
       bool		  dump_as_json(std::string& targetObj, size_t indent = 0, bool insert_newlines = true);
@@ -146,7 +148,7 @@ namespace epee
       CATCH_ENTRY("portable_storage::store_to_binary", false)
     }
     inline
-    bool portable_storage::load_from_binary(const binarybuffer& source)
+    bool portable_storage::load_from_binary(const epee::span<const uint8_t> source)
     {
       m_root.m_entries.clear();
       if(source.size() < sizeof(storage_block_header))

contrib/epee/include/storages/portable_storage_base.h

@@ -31,7 +31,8 @@
 #include <boost/variant.hpp>
 #include <boost/any.hpp>
 #include <string>
-#include <list>
+#include <vector>
+#include <deque>
 
 #define PORTABLE_STORAGE_SIGNATUREA 0x01011101
 #define PORTABLE_STORAGE_SIGNATUREB 0x01020101 // bender's nightmare 
@@ -71,6 +72,9 @@ namespace epee
   {
     struct section;
 
+    template<typename T> struct entry_container { typedef std::vector<T> type; static void reserve(type &t, size_t n) { t.reserve(n); } };
+    template<> struct entry_container<bool> { typedef std::deque<bool> type; static void reserve(type &t, size_t n) {} };
+
     /************************************************************************/
     /*                                                                      */
     /************************************************************************/
@@ -119,8 +123,13 @@ namespace epee
         return m_array.back();
       }
 
-      std::list<t_entry_type> m_array;
+      void reserve(size_t n)
-      mutable typename std::list<t_entry_type>::const_iterator m_it;
+      {
+        entry_container<t_entry_type>::reserve(m_array, n);
+      }
+
+      typename entry_container<t_entry_type>::type m_array;
+      mutable typename entry_container<t_entry_type>::type::const_iterator m_it;
     };
 
 

contrib/epee/include/storages/portable_storage_from_bin.h

@@ -136,6 +136,7 @@ namespace epee
       //for pod types
       array_entry_t<type_name> sa;
       size_t size = read_varint();
+      sa.reserve(size);
       //TODO: add some optimization here later
       while(size--)
         sa.m_array.push_back(read<type_name>());        

contrib/epee/include/storages/portable_storage_template_helper.h

@@ -84,7 +84,7 @@ namespace epee
     }
     //-----------------------------------------------------------------------------------------------------------
     template<class t_struct>
-    bool load_t_from_binary(t_struct& out, const std::string& binary_buff)
+    bool load_t_from_binary(t_struct& out, const epee::span<const uint8_t> binary_buff)
     {
       portable_storage ps;
       bool rs = ps.load_from_binary(binary_buff);
@@ -95,6 +95,12 @@ namespace epee
     }
     //-----------------------------------------------------------------------------------------------------------
     template<class t_struct>
+    bool load_t_from_binary(t_struct& out, const std::string& binary_buff)
+    {
+      return load_t_from_binary(out, epee::strspan<uint8_t>(binary_buff));
+    }
+    //-----------------------------------------------------------------------------------------------------------
+    template<class t_struct>
     bool load_t_from_binary_file(t_struct& out, const std::string& binary_file)
     {
       std::string f_buff;

contrib/epee/include/string_tools.h

@@ -40,8 +40,6 @@
 #include <cstdlib>
 #include <string>
 #include <type_traits>
-#include <boost/uuid/uuid.hpp>
-#include <boost/uuid/uuid_io.hpp>
 #include <boost/lexical_cast.hpp>
 #include <boost/algorithm/string/predicate.hpp>
 #include "hex.h"
@@ -84,34 +82,6 @@ namespace epee
 namespace string_tools
 {
   //----------------------------------------------------------------------------
-	inline std::string get_str_from_guid_a(const boost::uuids::uuid& rid)
-	{
-		return boost::lexical_cast<std::string>(rid);
-	}
-	//----------------------------------------------------------------------------
-	inline bool get_guid_from_string(OUT boost::uuids::uuid& inetifer, const std::string& str_id)
-	{
-		std::string local_str_id = str_id;
-		if(local_str_id.size() < 36)
-			return false;
-
-		if('{' == *local_str_id.begin())
-			local_str_id.erase(0, 1);
-
-		if('}' == *(--local_str_id.end()))
-			local_str_id.erase(--local_str_id.end());
-
-		try
-		{
-			inetifer = boost::lexical_cast<boost::uuids::uuid>(local_str_id);
-			return true;
-		}
-		catch(...)
-		{
-			return false;
-		}
-	}
-  //----------------------------------------------------------------------------
   inline std::string buff_to_hex_nodelimer(const std::string& src)
   {
     return to_hex::string(to_byte_span(to_span(src)));

contrib/epee/src/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2018, The Monero Project
+# Copyright (c) 2014-2019, The Monero Project
 # 
 # All rights reserved.
 # 
@@ -27,7 +27,7 @@
 # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 add_library(epee STATIC hex.cpp http_auth.cpp mlog.cpp net_utils_base.cpp string_tools.cpp wipeable_string.cpp memwipe.c
-    connection_basic.cpp network_throttle.cpp network_throttle-detail.cpp mlocker.cpp)
+    connection_basic.cpp network_throttle.cpp network_throttle-detail.cpp mlocker.cpp buffer.cpp)
 if (USE_READLINE AND GNU_READLINE_FOUND)
   add_library(epee_readline STATIC readline_buffer.cpp)
 endif()

contrib/epee/src/buffer.cpp

@@ -0,0 +1,97 @@
+// Copyright (c) 2018, The Monero Project
+// 
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+// 
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+// 
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+// 
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <string.h>
+#include "net/buffer.h"
+
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "net.buffer"
+
+namespace epee
+{
+namespace net_utils
+{
+
+void buffer::append(const void *data, size_t sz)
+{
+  const size_t capacity = storage.capacity();
+  const size_t avail = capacity - storage.size();
+
+  CHECK_AND_ASSERT_THROW_MES(storage.size() < std::numeric_limits<size_t>::max() - sz, "Too much data to append");
+
+  // decide when to move
+  if (sz > avail)
+  {
+    // we have to reallocate or move
+    const bool move = size() + sz <= capacity;
+    if (move)
+    {
+      const size_t bytes = storage.size() - offset;
+      NET_BUFFER_LOG("appending " << sz << " from " << size() << " by moving " << bytes << " from offset " << offset << " first (forced)");
+      memmove(storage.data(), storage.data() + offset, bytes);
+      storage.resize(bytes);
+      offset = 0;
+    }
+    else
+    {
+      NET_BUFFER_LOG("appending " << sz << " from " << size() << " by reallocating");
+      std::vector<uint8_t> new_storage;
+      size_t reserve = (((size() + sz) * 3 / 2) + 4095) & ~4095;
+      new_storage.reserve(reserve);
+      new_storage.resize(size());
+      memcpy(new_storage.data(), storage.data() + offset, storage.size() - offset);
+      offset = 0;
+      std::swap(storage, new_storage);
+    }
+  }
+  else
+  {
+    // we have space already
+    if (size() <= 4096 && offset > 4096 * 16 && offset >= capacity / 2)
+    {
+      // we have little to move, and we're far enough into the buffer that it's probably a win to move anyway
+      const size_t pos = storage.size() - offset;
+      NET_BUFFER_LOG("appending " << sz << " from " << size() << " by moving " << pos << " from offset " << offset << " first (unforced)");
+      memmove(storage.data(), storage.data() + offset, storage.size() - offset);
+      storage.resize(pos);
+      offset = 0;
+    }
+    else
+    {
+      NET_BUFFER_LOG("appending " << sz << " from " << size() << " by writing to existing capacity");
+    }
+  }
+
+  // add the new data
+  storage.insert(storage.end(), (const uint8_t*)data, (const uint8_t*)data + sz);
+
+  NET_BUFFER_LOG("storage now " << offset << "/" << storage.size() << "/" << storage.capacity());
+}
+
+}
+}

contrib/epee/src/connection_basic.cpp

@@ -2,7 +2,7 @@
 /// @author rfree (current maintainer in monero.cc project)
 /// @brief base for connection, contains e.g. the ratelimit hooks
 
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 // 
 // All rights reserved.
 // 

contrib/epee/src/http_auth.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 //
 // All rights reserved.
 //

contrib/epee/src/net_utils_base.cpp

@@ -1,7 +1,9 @@
 
 #include "net/net_utils_base.h"
-#include "string_tools.h"
 
+#include <boost/uuid/uuid_io.hpp>
+
+#include "string_tools.h"
 #include "net/local_ip.h"
 
 namespace epee { namespace net_utils
@@ -73,7 +75,7 @@ namespace epee { namespace net_utils
   std::string print_connection_context(const connection_context_base& ctx)
   {
     std::stringstream ss;
-    ss << ctx.m_remote_address.str() << " " << epee::string_tools::get_str_from_guid_a(ctx.m_connection_id) << (ctx.m_is_income ? " INC":" OUT");
+    ss << ctx.m_remote_address.str() << " " << ctx.m_connection_id << (ctx.m_is_income ? " INC":" OUT");
     return ss.str();
   }
 

contrib/epee/src/network_throttle-detail.cpp

@@ -2,7 +2,7 @@
 /// @author rfree (current maintainer in monero.cc project)
 /// @brief implementaion for throttling of connection (count and rate-limit speed etc)
 
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 // 
 // All rights reserved.
 // 

contrib/epee/src/network_throttle.cpp

@@ -26,7 +26,7 @@ Throttling work by:
 
 */
 
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 // 
 // All rights reserved.
 // 

external/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2018, The Monero Project
+# Copyright (c) 2014-2019, The Monero Project
 # 
 # All rights reserved.
 # 

external/db_drivers/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2018, The Monero Project
+# Copyright (c) 2014-2019, The Monero Project
 # 
 # All rights reserved.
 # 

external/db_drivers/liblmdb/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2018, The Monero Project
+# Copyright (c) 2014-2019, The Monero Project
 # 
 # All rights reserved.
 # 

external/easylogging++/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2018, The Monero Project
+# Copyright (c) 2014-2019, The Monero Project
 # 
 # All rights reserved.
 # 

include/INode.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 // 
 // All rights reserved.
 // 

include/IWallet.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 // 
 // All rights reserved.
 // 

src/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2018, The Monero Project
+# Copyright (c) 2014-2019, The Monero Project
 #
 # All rights reserved.
 #

src/blockchain_db/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2018, The Monero Project
+# Copyright (c) 2014-2019, The Monero Project
 #
 # All rights reserved.
 #

src/blockchain_db/berkeleydb/db_bdb.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without modification, are

src/blockchain_db/berkeleydb/db_bdb.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without modification, are

src/blockchain_db/blockchain_db.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 // 
 // All rights reserved.
 // 

src/blockchain_db/blockchain_db.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 //
 // All rights reserved.
 //
@@ -1329,10 +1329,11 @@ public:
    * If an output cannot be found, the subclass should throw OUTPUT_DNE.
    *
    * @param tx_id a transaction ID
+   * @param n_txes how many txes to get data for, starting with tx_id
    *
    * @return a list of amount-specific output indices
    */
-  virtual std::vector<uint64_t> get_tx_amount_output_indices(const uint64_t tx_id) const = 0;
+  virtual std::vector<std::vector<uint64_t>> get_tx_amount_output_indices(const uint64_t tx_id, size_t n_txes = 1) const = 0;
 
   /**
    * @brief check if a key image is stored as spent

src/blockchain_db/db_types.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 //
 // All rights reserved.
 //

src/blockchain_db/lmdb/db_lmdb.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without modification, are
@@ -1025,7 +1025,8 @@ void BlockchainLMDB::remove_tx_outputs(const uint64_t tx_id, const transaction&
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
 
-  std::vector<uint64_t> amount_output_indices = get_tx_amount_output_indices(tx_id);
+  std::vector<std::vector<uint64_t>> amount_output_indices_set = get_tx_amount_output_indices(tx_id, 1);
+  const std::vector<uint64_t> &amount_output_indices = amount_output_indices_set.front();
 
   if (amount_output_indices.empty())
   {
@@ -2604,7 +2605,7 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, con
   return indices[0];
 }
 
-std::vector<uint64_t> BlockchainLMDB::get_tx_amount_output_indices(const uint64_t tx_id) const
+std::vector<std::vector<uint64_t>> BlockchainLMDB::get_tx_amount_output_indices(uint64_t tx_id, size_t n_txes) const
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
 
@@ -2613,12 +2614,15 @@ std::vector<uint64_t> BlockchainLMDB::get_tx_amount_output_indices(const uint64_
   TXN_PREFIX_RDONLY();
   RCURSOR(tx_outputs);
 
-  int result = 0;
   MDB_val_set(k_tx_id, tx_id);
   MDB_val v;
-  std::vector<uint64_t> amount_output_indices;
+  std::vector<std::vector<uint64_t>> amount_output_indices_set;
+  amount_output_indices_set.reserve(n_txes);
 
-  result = mdb_cursor_get(m_cur_tx_outputs, &k_tx_id, &v, MDB_SET);
+  MDB_cursor_op op = MDB_SET;
+  while (n_txes-- > 0)
+  {
+    int result = mdb_cursor_get(m_cur_tx_outputs, &k_tx_id, &v, op);
     if (result == MDB_NOTFOUND)
       LOG_PRINT_L0("WARNING: Unexpected: tx has no amount indices stored in "
           "tx_outputs, but it should have an empty entry even if it's a tx without "
@@ -2626,22 +2630,24 @@ std::vector<uint64_t> BlockchainLMDB::get_tx_amount_output_indices(const uint64_
     else if (result)
       throw0(DB_ERROR(lmdb_error("DB error attempting to get data for tx_outputs[tx_index]", result).c_str()));
 
-  const uint64_t* indices = (const uint64_t*)v.mv_data;
+    op = MDB_NEXT;
-  int num_outputs = v.mv_size / sizeof(uint64_t);
 
+    const uint64_t* indices = (const uint64_t*)v.mv_data;
+    size_t num_outputs = v.mv_size / sizeof(uint64_t);
+
+    amount_output_indices_set.resize(amount_output_indices_set.size() + 1);
+    std::vector<uint64_t> &amount_output_indices = amount_output_indices_set.back();
     amount_output_indices.reserve(num_outputs);
-  for (int i = 0; i < num_outputs; ++i)
+    for (size_t i = 0; i < num_outputs; ++i)
     {
-    // LOG_PRINT_L0("amount output index[" << 2*i << "]" << ": " << paired_indices[2*i] << "  global output index: " << paired_indices[2*i+1]);
       amount_output_indices.push_back(indices[i]);
     }
-  indices = nullptr;
+  }
 
   TXN_POSTFIX_RDONLY();
-  return amount_output_indices;
+  return amount_output_indices_set;
 }
 
-
 bool BlockchainLMDB::has_key_image(const crypto::key_image& img) const
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);

src/blockchain_db/lmdb/db_lmdb.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without modification, are
@@ -252,7 +252,7 @@ public:
   virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const;
   virtual void get_output_tx_and_index(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<tx_out_index> &indices) const;
 
-  virtual std::vector<uint64_t> get_tx_amount_output_indices(const uint64_t tx_id) const;
+  virtual std::vector<std::vector<uint64_t>> get_tx_amount_output_indices(const uint64_t tx_id, size_t n_txes) const;
 
   virtual bool has_key_image(const crypto::key_image& img) const;
 

src/blockchain_utilities/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2018, The Monero Project
+# Copyright (c) 2014-2019, The Monero Project
 #
 # All rights reserved.
 #

src/blockchain_utilities/README.md

@@ -1,6 +1,6 @@
 # Monero Blockchain Utilities
 
-Copyright (c) 2014-2018, The Monero Project
+Copyright (c) 2014-2019, The Monero Project
 
 ## Introduction
 

src/blockchain_utilities/blockchain_ancestry.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 //
 // All rights reserved.
 //

src/blockchain_utilities/blockchain_blackball.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 //
 // All rights reserved.
 //
@@ -1131,7 +1131,7 @@ int main(int argc, char* argv[])
     return 1;
   }
 
-  mlog_configure(mlog_get_default_log_path("wownero-blockchain-find-spent-outputs.log"), true);
+  mlog_configure(mlog_get_default_log_path("wownero-blockchain-mark-spent-outputs.log"), true);
   if (!command_line::is_arg_defaulted(vm, arg_log_level))
     mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str());
   else

src/blockchain_utilities/blockchain_depth.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 //
 // All rights reserved.
 //

src/blockchain_utilities/blockchain_export.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 //
 // All rights reserved.
 //

src/blockchain_utilities/blockchain_import.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 //
 // All rights reserved.
 //

src/blockchain_utilities/blockchain_prune_known_spent_data.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 //
 // All rights reserved.
 //

src/blockchain_utilities/blockchain_stats.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 //
 // All rights reserved.
 //

src/blockchain_utilities/blockchain_usage.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 //
 // All rights reserved.
 //

src/blockchain_utilities/blockchain_utilities.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 //
 // All rights reserved.
 //

src/blockchain_utilities/blocksdat_file.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 //
 // All rights reserved.
 //

src/blockchain_utilities/blocksdat_file.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 //
 // All rights reserved.
 //

src/blockchain_utilities/bootstrap_file.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 //
 // All rights reserved.
 //

src/blockchain_utilities/bootstrap_file.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 //
 // All rights reserved.
 //

src/blockchain_utilities/bootstrap_serialization.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 //
 // All rights reserved.
 //

src/blocks/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2018, The Monero Project
+# Copyright (c) 2014-2019, The Monero Project
 # 
 # All rights reserved.
 # 

src/checkpoints/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2018, The Monero Project
+# Copyright (c) 2014-2019, The Monero Project
 #
 # All rights reserved.
 #

src/checkpoints/checkpoints.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 //
 // All rights reserved.
 //

src/checkpoints/checkpoints.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 //
 // All rights reserved.
 //

src/common/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2014-2018, The Monero Project
+# Copyright (c) 2014-2019, The Monero Project
 #
 # All rights reserved.
 #

src/common/base58.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 // 
 // All rights reserved.
 // 

src/common/base58.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 // 
 // All rights reserved.
 // 

src/common/boost_serialization_helper.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 // 
 // All rights reserved.
 // 

src/common/command_line.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 // 
 // All rights reserved.
 // 

src/common/command_line.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 // 
 // All rights reserved.
 // 

src/common/common_fwd.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 //
 // All rights reserved.
 //

src/common/dns_utils.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 //
 // All rights reserved.
 //

src/common/dns_utils.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 // 
 // All rights reserved.
 // 

src/common/http_connection.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 // 
 // All rights reserved.
 // 

src/common/i18n.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 // 
 // All rights reserved.
 // 
@@ -38,6 +38,8 @@
 #undef MONERO_DEFAULT_LOG_CATEGORY
 #define MONERO_DEFAULT_LOG_CATEGORY "i18n"
 
+#define MAX_LANGUAGE_SIZE 16
+
 static const unsigned char qm_magic[16] = {0x3c, 0xb8, 0x64, 0x18, 0xca, 0xef, 0x9c, 0x95, 0xcd, 0x21, 0x1c, 0xbf, 0x60, 0xa1, 0xbd, 0xdd};
 
 static std::map<std::string,std::string> i18n_entries;
@@ -62,7 +64,19 @@ std::string i18n_get_language()
 
   std::string language = e;
   language = language.substr(0, language.find("."));
+  language = language.substr(0, language.find("@"));
+
+  // check valid values
+  for (char c: language)
+    if (!strchr("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_-.@", c))
+      return "en";
+
   std::transform(language.begin(), language.end(), language.begin(), tolower);
+  if (language.size() > MAX_LANGUAGE_SIZE)
+  {
+    i18n_log("Language from LANG/LC_ALL suspiciously long, defaulting to en");
+    return "en";
+  }
   return language;
 }
 

src/common/i18n.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 // 
 // All rights reserved.
 // 

src/common/notify.cpp

@@ -27,11 +27,15 @@
 // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <boost/algorithm/string.hpp>
+#include <stdarg.h>
 #include "misc_log_ex.h"
 #include "file_io_utils.h"
 #include "spawn.h"
 #include "notify.h"
 
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "notify"
+
 namespace tools
 {
 
@@ -46,15 +50,32 @@ Notify::Notify(const char *spec)
 
   boost::split(args, spec, boost::is_any_of(" "));
   CHECK_AND_ASSERT_THROW_MES(args.size() > 0, "Failed to parse spec");
+  if (strchr(spec, '\'') || strchr(spec, '\"') || strchr(spec, '\\'))
+    MWARNING("A notification spec contains a quote or backslash: note that these are handled verbatim, which may not be the intent");
   filename = args[0];
   CHECK_AND_ASSERT_THROW_MES(epee::file_io_utils::is_file_exist(filename), "File not found: " << filename);
 }
 
-int Notify::notify(const char *parameter)
+static void replace(std::vector<std::string> &v, const char *tag, const char *s)
+{
+  for (std::string &str: v)
+    boost::replace_all(str, tag, s);
+}
+
+int Notify::notify(const char *tag, const char *s, ...)
 {
   std::vector<std::string> margs = args;
-  for (std::string &s: margs)
+
-    boost::replace_all(s, "%s", parameter);
+  replace(margs, tag, s);
+
+  va_list ap;
+  va_start(ap, s);
+  while ((tag = va_arg(ap, const char*)))
+  {
+    s = va_arg(ap, const char*);
+    replace(margs, tag, s);
+  }
+  va_end(ap);
 
   return tools::spawn(filename.c_str(), margs, false);
 }

src/common/notify.h

@@ -39,7 +39,7 @@ class Notify
 public:
   Notify(const char *spec);
 
-  int notify(const char *parameter);
+  int notify(const char *tag, const char *s, ...);
 
 private:
   std::string filename;

src/common/password.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 // 
 // All rights reserved.
 // 

src/common/password.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 // 
 // All rights reserved.
 // 

src/common/pod-class.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 // 
 // All rights reserved.
 // 

src/common/rpc_client.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 // 
 // All rights reserved.
 // 

src/common/scoped_message_writer.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 // 
 // All rights reserved.
 // 

src/common/spawn.cpp

@@ -42,6 +42,9 @@
 #include "util.h"
 #include "spawn.h"
 
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "spawn"
+
 namespace tools
 {
 

src/common/unordered_containers_boost_serialization.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 // 
 // All rights reserved.
 // 

src/common/util.cpp

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 // 
 // All rights reserved.
 // 

src/common/util.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 // 
 // All rights reserved.
 // 

src/common/varint.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2014-2018, The Monero Project
+// Copyright (c) 2014-2019, The Monero Project
 // 
 // All rights reserved.
 //