From 2bce229efef1a7e715dd7b03144353bac412b368 Mon Sep 17 00:00:00 2001 From: Dmitry Tarasov Date: Thu, 31 May 2018 08:51:16 +0500 Subject: [PATCH] *Merge code with bytecoin v3.1.0 *Minor changes on Bitsum lib +Explorer --- .clang-format | 2 +- CMakeLists.txt | 19 +- README.md | 39 +-- ReleaseNotes.md | 3 + src/Core/BlockChain.cpp | 399 +++++++++++++++++++----- src/Core/BlockChain.hpp | 29 +- src/Core/BlockChainState.cpp | 482 ++++++++++++++++++----------- src/Core/BlockChainState.hpp | 45 ++- src/Core/DifficultyCheck.cpp | 8 + src/Core/DifficultyCheck.hpp | 14 + src/Core/Node.cpp | 134 +++++--- src/Core/Node.hpp | 33 +- src/Core/NodeDownloader.cpp | 328 +++++++++++++++++--- src/Core/TransactionBuilder.cpp | 42 ++- src/Core/TransactionBuilder.hpp | 2 +- src/Core/Wallet.cpp | 37 ++- src/Core/WalletNode.cpp | 62 ++-- src/Core/WalletState.cpp | 72 +++-- src/Core/WalletState.hpp | 4 +- src/Core/WalletSync.cpp | 42 ++- src/Core/rpc_api_serialization.cpp | 23 +- src/bitsum/Common.hpp | 24 ++ src/bitsum/CommonWrapper.cpp | 34 ++ src/bitsum/Explorer.cpp | 46 +++ src/bitsum/Explorer.hpp | 34 ++ src/bitsum/NodeWrapper.cpp | 84 +++++ src/bitsum/WalletWrapper.cpp | 312 +++++++++++++++++++ src/bitsum/bitsum.cpp | 323 ++++++------------- src/common/ConsoleTools.cpp | 1 - src/common/JsonValue.cpp | 1 + src/http/Server.cpp | 3 +- src/logging/FileLogger.cpp | 16 +- src/logging/FileLogger.hpp | 2 + src/logging/LoggerManager.cpp | 7 +- src/main_bitsumd.cpp | 2 +- src/main_wallet_rpc.cpp | 17 +- src/p2p/P2pSeria.cpp | 4 + src/platform/DBlmdb.cpp | 3 +- src/platform/Files.cpp | 2 +- src/platform/Files.hpp | 2 +- src/platform/Network.cpp | 57 +++- src/rpc_api.hpp | 44 ++- src/version.hpp | 4 +- 43 files changed, 2061 insertions(+), 780 deletions(-) create mode 100644 src/Core/DifficultyCheck.cpp create mode 100644 src/Core/DifficultyCheck.hpp create mode 100644 src/bitsum/Common.hpp create mode 100644 src/bitsum/CommonWrapper.cpp create mode 100644 src/bitsum/Explorer.cpp create mode 100644 src/bitsum/Explorer.hpp create mode 100644 src/bitsum/NodeWrapper.cpp create mode 100644 src/bitsum/WalletWrapper.cpp diff --git a/.clang-format b/.clang-format index 3ad5c4dd..015c5528 100644 --- a/.clang-format +++ b/.clang-format @@ -24,7 +24,7 @@ BreakBeforeBraces: Attach BreakBeforeTernaryOperators: true BreakConstructorInitializersBeforeComma: true BreakAfterJavaFieldAnnotations: false -BreakStringLiterals: true +BreakStringLiterals: false ColumnLimit: 120 CommentPragmas: '^ IWYU pragma:' ConstructorInitializerAllOnOneLineOrOnePerLine: true diff --git a/CMakeLists.txt b/CMakeLists.txt index 17103a64..c85de8b0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,8 +26,8 @@ if(WIN32) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}") add_definitions(-D_SCL_SECURE_NO_WARNINGS=1 -D_CRT_SECURE_NO_WARNINGS=1 -D_WIN32_WINNT=0x0501) else() - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -maes -g -O2 -Wall -Wextra -Werror=return-type -Wno-unused-parameter") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -maes -g -O2 -Wall -Wextra -Werror=return-type -Wno-unused-parameter") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -maes -g -O2 -Wall -Wextra -Werror=return-type -Wno-unused-parameter -Wno-unknown-pragmas") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -maes -g -O2 -Wall -Wextra -Werror=return-type -Wno-unused-parameter -Wno-unknown-pragmas") message(STATUS "Instrumentation usage: " ${USE_INSTRUMENTATION}) message(STATUS "Thread sanitizer usage: " ${WITH_THREAD_SANITIZER}) if(USE_INSTRUMENTATION) @@ -170,12 +170,25 @@ else() find_package(Boost 1.65 REQUIRED COMPONENTS system) include_directories(SYSTEM ${Boost_INCLUDE_DIRS}) set(Boost_LIBRARIES "${Boost_LIBRARIES}") + if(APPLE) + set(BOOST_ROOT ../boost) + find_package(Boost 1.62 REQUIRED COMPONENTS system) + message(STATUS "Boost_INCLUDE_DIRS: " ${Boost_INCLUDE_DIRS}) + include_directories(SYSTEM ${Boost_INCLUDE_DIRS}) + #set(Boost_LIBRARIES "${Boost_LIBRARIES}") + else() + message(STATUS "Boost_INCLUDE_DIRS: ../boost") + include_directories(SYSTEM ../boost) + set(Boost_LIBRARIES "${CMAKE_CURRENT_SOURCE_DIR}/../boost/stage/lib/libboost_system.a") + endif() + message(STATUS "Boost_LIBRARIES: " ${Boost_LIBRARIES}) if(APPLE) set(CMAKE_OSX_DEPLOYMENT_TARGET "10.11") target_link_libraries(wallet-rpc "-framework Foundation" "-framework IOKit") target_link_libraries(bitsumd "-framework Foundation" "-framework IOKit") endif() - target_link_libraries(wallet-rpc ${Boost_LIBRARIES} ${LINK_OPENSSL} dl pthread) + target_link_libraries(bitsum ${Boost_LIBRARIES} ${LINK_OPENSSL} dl pthread) + target_link_libraries(wallet-rpc ${Boost_LIBRARIES} ${LINK_OPENSSL} dl pthread) target_link_libraries(bitsumd ${Boost_LIBRARIES} ${LINK_OPENSSL} dl pthread) #target_link_libraries(tests ${Boost_LIBRARIES} ${LINK_OPENSSL} dl pthread) endif() diff --git a/README.md b/README.md index eef9e0ea..dfb03112 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ $> mkdir bitsum $> cd bitsum ``` -To go futher you have to have a number of packages and utilities. +To go futher you have to have a number of packages and utilities. You need at least gcc 5.4. * `build-essential` package: ``` @@ -37,16 +37,15 @@ To go futher you have to have a number of packages and utilities. If version is too old, follow instructions on [the official site](https://cmake.org/download/). * Boost (1.62 or newer): + You need boost in `bcndev` folder. We do not configure to use boost installed by `apt-get`, because it is sometimes updated without your control by installing some unrelated packages. Also some users reported crashes after `find_package` finds headers from one version of boost and libraries from different version, or if installed boost uses dynamic linking. ``` - $> sudo apt-get install libboost-all-dev - $> cat /usr/include/boost/version.hpp | grep "BOOST_LIB_VERSION" - ``` - If version is too old, download boost from [boost.org](https://boost.org), unpack it into a folder inside `bitsum` and rename it from `boost_1_66_0` or similar to just `boost` - Build boost - ``` - $> cd boost - $bitsum/boost> ./bootstrap.sh - $bitsum/boost> ./b2 link=static -j 8 --build-dir=build64 --stagedir=stage + $bcndev> wget -c 'http://sourceforge.net/projects/boost/files/boost/1.67.0/boost_1_67_0.tar.bz2/download' + $bcndev> tar xf download + $bcndev> rm download + $bcndev> mv boost_1_67_0 boost + $bcndev> cd boost + $bcndev/boost> ./bootstrap.sh + $bcndev/boost> ./b2 link=static cxxflags="-fPIC" linkflags="-pie" -j 8 --build-dir=build64 --stagedir=stage cd .. ``` @@ -64,14 +63,11 @@ Create build directory inside bytecoin, go there and run CMake and Make: ``` $bitsum> mkdir bitsum_core/build $bitsum> cd bitsum_core/build -$bitsum/bitsum_core/build> cmake -DUSE_SSL=0 .. +$bitsum/bitsum_core/build> cmake .. $bitsum/bitsum_core/build> time make -j4 ``` -Check built binaries by running them from `../bin` folder -``` -$bitsum/bitsum_core/build> ../bin/bytecoind -v -``` + ### Building with specific options @@ -130,11 +126,6 @@ $bitsum/bitsum_core/build> cmake -DUSE_SSL=0 .. $bitsum/bitsum_core/build> time make -j4 ``` -Check built binaries by running them from `../bin` folder: -``` -$bitsum/bitsum_core/build> ../bin/bitsumd -v -``` - ### Building with specific options Binaries linked with Boost installed by Homebrew will work only on your computer's OS X version or newer, but not on older versions like El Capitan. @@ -147,9 +138,9 @@ Download and unpack [Boost](https://boost.org) to `Downloads` folder. Then build and install Boost: ``` -$~> cd ~/Downloads/boost_1_58_0/ -$~/Downloads/boost_1_58_0> ./bootstrap.sh -$~/Downloads/boost_1_58_0> ./b2 -a -j 4 cxxflags="-stdlib=libc++ -std=c++14 -mmacosx-version-min=10.11 -isysroot/Users/user/Downloads/MacOSX10.11.sdk" install` +$~> cd ~/Downloads/boost_1_67_0/ +$~/Downloads/boost_1_67_0> ./bootstrap.sh +$~/Downloads/boost_1_67_0> ./b2 -a -j 4 cxxflags="-stdlib=libc++ -std=c++14 -mmacosx-version-min=10.11 -isysroot/Users/user/Downloads/MacOSX10.11.sdk" install` ``` Install OpenSSL to `bitsum/openssl` folder: @@ -185,7 +176,7 @@ $bitsum/bitsum_core/build> time make -j4 ## Building on Windows -You need Microsoft Visual Studio Community 2017. [Download](https://microsoft.com) and install it selecting `C++`, `git`, `cmake integration` packages. +You need Microsoft Visual Studio Community 2017. [Download](https://www.visualstudio.com/vs/) and install it selecting `C++`, `git`, `cmake integration` packages. Run `Visual Studio x64 command prompt` from start menu. Create directory `bitsum` somewhere: diff --git a/ReleaseNotes.md b/ReleaseNotes.md index 3f8a330a..e9fdb3a4 100644 --- a/ReleaseNotes.md +++ b/ReleaseNotes.md @@ -1,5 +1,8 @@ ## Release Notes +### v1.18.5.25 +- Merge code with bytecoin v3.1.0 + ### v1.18.5.10 - Merge code with bytecoin v3.0.4 diff --git a/src/Core/BlockChain.cpp b/src/Core/BlockChain.cpp index 8573f3ea..67151095 100644 --- a/src/Core/BlockChain.cpp +++ b/src/Core/BlockChain.cpp @@ -7,6 +7,7 @@ #include #include #include "CryptoNoteTools.hpp" +#include "DifficultyCheck.hpp" #include "TransactionExtra.hpp" #include "common/StringTools.hpp" #include "common/Varint.hpp" @@ -18,7 +19,9 @@ using namespace bytecoin; using namespace platform; static const std::string previous_versions[] = {"B"}; // most recent previous version should be first in list -static const std::string version_current = "1"; + // 1 -> 2 we fix difficulty for consensus +static const std::string version_1 = "1"; +const std::string BlockChain::version_current = "2"; // We increment when making incompatible changes to indices. // We use suffixes so all keys related to the same block are close to each other @@ -37,6 +40,9 @@ static const std::string CD_TIPS_PREFIX = "x-tips/"; // We store bid->children counter, with counter=1 default (absent from index) // We store cumulative_difficulty->bid for bids with no children +static const size_t HEADER_CACHE_MAX_SIZE = 100000; // when lots of read_header called without commit +static const std::string delete_blockchain_message = "database corrupted, please delete "; + bool Block::from_raw_block(const RawBlock &raw_block) { try { BlockTemplate &bheader = header; @@ -97,8 +103,15 @@ PreparedBlock::PreparedBlock(RawBlock &&rba, crypto::CryptoNightContext *context BlockChain::BlockChain(const Hash &genesis_bid, const std::string &coin_folder) : m_genesis_bid(genesis_bid), m_coin_folder(coin_folder), m_db(coin_folder + "/blockchain") { std::string version; - m_db.get("$version", version); - if (version != version_current) { + if (!m_db.get("$version", version)) { + DB::Cursor cur = m_db.begin(std::string()); + if (!cur.end()) + throw std::runtime_error( + "Blockchain database format unknown version, please delete " + coin_folder + "/blockchain"); + version = version_current; + m_db.put("$version", version, false); + } + if (version == previous_versions[0]) { std::cout << "Data format changed, old version=" << version << " current version=" << version_current << ", deleting bytecoind cache..." << std::endl; std::set main_chain_bids; @@ -135,7 +148,8 @@ BlockChain::BlockChain(const Hash &genesis_bid, const std::string &coin_folder) cur.erase(); erased += 1; } - m_db.put("$version", version_current, true); + version = "2"; // Deleting all block headers fixes difficulty + m_db.put("$version", version, false); std::cout << "Deleted " << erased << " records, skipped " << skipped << " records" << std::endl; db_commit(); } @@ -152,9 +166,10 @@ BlockChain::BlockChain(const Hash &genesis_bid, const std::string &coin_folder) } void BlockChain::db_commit() { - std::cout << "BlockChain::db_commit started... tip_height=" << m_tip_height << " header_cache.size=" << header_cache.size() << std::endl; + std::cout << "BlockChain::db_commit started... tip_height=" << m_tip_height + << " header_cache.size=" << header_cache.size() << std::endl; m_db.commit_db_txn(); - header_cache.clear(); // Most simple cache policy ever + header_cache.clear(); // Most simple cache policy ever std::cout << "BlockChain::db_commit finished..." << std::endl; } @@ -177,8 +192,19 @@ BroadcastAction BlockChain::add_block(const PreparedBlock &pb, api::BlockHeader info.hash = pb.bid; info.height = prev_info.height + 1; // Rest fields are filled by check_standalone_consensus - if (!check_standalone_consensus(pb, info, prev_info)) - return BroadcastAction::BAN; + std::string check_error = check_standalone_consensus(pb, info, prev_info); + //Hash first_difficulty_check_hash; + //if (!common::pod_from_hex(difficulty_check[0].hash, first_difficulty_check_hash)) + // throw std::logic_error("DifficultyCheck table corrupted"); + //if (info.hash == first_difficulty_check_hash && + // info.cumulative_difficulty != difficulty_check[0].cumulative_difficulty) { + // std::cout << "Reached first difficulty checkpoint with wrong cumulative_difficulty " + // << info.cumulative_difficulty << ", should be " << difficulty_check[0].cumulative_difficulty << ", " + // << delete_blockchain_message << m_coin_folder << "/blockchain" << std::endl; + // std::exit(api::BYTECOIND_DATABASE_ERROR); + //} + if (!check_error.empty()) + return BroadcastAction::BAN; // TODO - return check_error try { if (!have_block) store_block(pb.bid, pb.block_data); // Do not commit between here and @@ -199,13 +225,15 @@ BroadcastAction BlockChain::add_block(const PreparedBlock &pb, api::BlockHeader if (!redo_block(pb.bid, pb.raw_block, pb.block, info, pb.base_transaction_hash)) return BroadcastAction::BAN; push_chain(pb.bid, info.cumulative_difficulty); + // std::map> undone_transactions; + // on_reorganization(undone_transactions, true); } else reorganize_blocks(pb.bid, pb, info); } } catch (const std::exception &ex) { std::cout << "Exception while reorganizing blockchain, probably out of " "disk space ex.what=" - << ex.what() << std::endl; + << ex.what() << ", " << delete_blockchain_message << m_coin_folder << "/blockchain" << std::endl; std::exit(api::BYTECOIND_DATABASE_ERROR); } if (get_tip_height() % 50000 == 0) @@ -219,15 +247,24 @@ bool BlockChain::reorganize_blocks(const Hash &switch_to_chain, // Header chain is better than block chain, undo upto splitting block std::vector chain1, chain2; Hash common = get_common_block(m_tip_bid, switch_to_chain, &chain1, &chain2); - for (auto &&chha : chain2) + for (auto &&chha : chain2) { if (!has_block(chha)) return false; // Full new chain not yet downloaded + } + std::map> undone_transactions; + bool undone_blocks = false; while (m_tip_bid != common) { RawBlock raw_block; Block block; if (!read_block(m_tip_bid, raw_block) || !block.from_raw_block(raw_block)) throw std::logic_error("Block to undo not found or failed to convert" + common::pod_to_hex(m_tip_bid)); + undone_blocks = true; undo_block(m_tip_bid, raw_block, block, m_tip_height); + for (size_t tx_index = 0; tx_index != block.transactions.size(); ++tx_index) { + Hash tid = block.header.transaction_hashes.at(tx_index); + undone_transactions.insert(std::make_pair(tid, std::make_pair(std::move(block.transactions.at(tx_index)), + std::move(raw_block.transactions.at(tx_index))))); + } pop_chain(); m_tip_bid = block.header.previous_block_hash; api::BlockHeader info = get_tip(); @@ -238,37 +275,49 @@ bool BlockChain::reorganize_blocks(const Hash &switch_to_chain, tip_changed(); } // Now redo all blocks we have in storage, will ask for the rest of blocks + bool result = true; while (!chain2.empty()) { Hash chha = chain2.back(); chain2.pop_back(); if (chha == recent_pb.bid) { if (recent_pb.block.header.previous_block_hash != m_tip_bid) throw std::logic_error("Unexpected block prev, invariant dead"); - if (!redo_block( - recent_pb.bid, recent_pb.raw_block, recent_pb.block, recent_info, recent_pb.base_transaction_hash)) + if (!redo_block(recent_pb.bid, recent_pb.raw_block, recent_pb.block, recent_info, + recent_pb.base_transaction_hash)) { // invalid block on longest subchain, make no attempt to download the // rest // we will forever stuck on this block until longer chain appears, that // does not include it - return false; + result = false; + break; + } push_chain(chha, recent_info.cumulative_difficulty); + for (auto &&tid : recent_pb.block.header.transaction_hashes) + undone_transactions.erase(tid); } else { RawBlock raw_block; Block block; - if (!read_block(chha, raw_block) || !block.from_raw_block(raw_block)) - return false; // Strange, we checked has_block, somehow "bad block" got - // into DB. TODO - throw? + if (!read_block(chha, raw_block) || !block.from_raw_block(raw_block)) { + result = false; + break; // Strange, we checked has_block, somehow "bad block" got + // into DB. TODO - throw? + } if (block.header.previous_block_hash != m_tip_bid) throw std::logic_error("Unexpected block prev, invariant dead"); api::BlockHeader info = read_header(chha); // if redo fails, we will forever stuck on this block until longer chain // appears, that does not include it - if (!redo_block(chha, raw_block, block, info, get_transaction_hash(block.header.base_transaction))) - return false; + if (!redo_block(chha, raw_block, block, info, get_transaction_hash(block.header.base_transaction))) { + result = false; + break; + } push_chain(chha, info.cumulative_difficulty); + for (auto &&tid : block.header.transaction_hashes) + undone_transactions.erase(tid); } } - return true; + on_reorganization(undone_transactions, undone_blocks); + return result; } Hash BlockChain::get_common_block( @@ -420,30 +469,34 @@ void ser_members(APITransactionPos &v, ISeria &s) { } } // namespace seria -bool BlockChain::read_transaction(const Hash &tid, Transaction &tx, Height &height, size_t &index_in_block) const { +bool BlockChain::read_transaction( + const Hash &tid, Transaction &tx, Height &block_height, Hash &block_hash, size_t &index_in_block) const { auto txkey = TRANSATION_PREFIX + DB::to_binary_key(tid.data, TRANSACTION_PREFIX_BYTES); for (DB::Cursor cur = m_db.begin(txkey); !cur.end(); cur.next()) { const std::string &suf = cur.get_suffix(); const char *be = suf.data(); const char *en = be + suf.size(); - height = boost::lexical_cast(common::read_varint_sqlite4(be, en)); - index_in_block = boost::lexical_cast(common::read_varint_sqlite4(be, en)); + Height ha = boost::lexical_cast(common::read_varint_sqlite4(be, en)); + size_t in = boost::lexical_cast(common::read_varint_sqlite4(be, en)); Hash bid; - if (!read_chain(height, bid)) + if (!read_chain(ha, bid)) throw std::logic_error("transaction index corrupted while reading tid=" + common::pod_to_hex(tid)); RawBlock rb; Block block; if (!read_block(bid, rb) || !block.from_raw_block(rb)) throw std::logic_error("transaction index corrupted while reading bid=" + common::pod_to_hex(bid)); - if (index_in_block == 0) { + if (in == 0) { if (get_transaction_hash(block.header.base_transaction) != tid) continue; tx = block.header.base_transaction; - return true; + } else { + if (block.header.transaction_hashes.at(in - 1) != tid) + continue; + tx = block.transactions.at(in - 1); } - if (block.header.transaction_hashes.at(index_in_block - 1) != tid) - continue; - tx = block.transactions.at(index_in_block - 1); + block_hash = bid; + block_height = ha; + index_in_block = in; return true; } return false; @@ -467,14 +520,14 @@ bool BlockChain::redo_block(const Hash &bhash, const RawBlock &, const Block &bl m_db.put(bkey, std::string(), true); } -// m_tip_segment.push_back(info); -// if (m_tip_segment.size() > 2048) // TODO - should be enough for all block windows we use -// m_tip_segment.pop_front(); + // m_tip_segment.push_back(info); + // if (m_tip_segment.size() > 2048) // TODO - should be enough for all block windows we use + // m_tip_segment.pop_front(); return true; } void BlockChain::undo_block(const Hash &bhash, const RawBlock &, const Block &block, Height height) { -// if (!m_tip_segment.empty()) -// m_tip_segment.pop_back(); + // if (!m_tip_segment.empty()) + // m_tip_segment.pop_back(); undo_block(bhash, block, height); auto tikey = TIMESTAMP_BLOCK_PREFIX + common::write_varint_sqlite4(block.header.timestamp) + @@ -523,19 +576,19 @@ void BlockChain::store_header(const Hash &bid, const api::BlockHeader &header) { bool BlockChain::read_header(const Hash &bid, api::BlockHeader &header) const { auto cit = header_cache.find(bid); - if( cit != header_cache.end() ){ + if (cit != header_cache.end()) { header = cit->second; return true; } -// if (bid == m_tip_bid && !m_tip_segment.empty()) { -// header = m_tip_segment.back(); -// return true; -// } + if (header_cache.size() > HEADER_CACHE_MAX_SIZE) { + std::cout << "HEADER_CACHE_MAX_SIZE cleared" << std::endl; + header_cache.clear(); // very simple policy + } BinaryArray rb; auto key = HEADER_PREFIX + DB::to_binary_key(bid.data, sizeof(bid.data)) + HEADER_SUFFIX; if (!m_db.get(key, rb)) return false; - Hash bbid = bid; + Hash bbid = bid; // next line can modify bid, because it can be reference to header.previous_block_hash seria::from_binary(header, rb); header_cache.insert(std::make_pair(bbid, header)); return true; @@ -550,52 +603,53 @@ api::BlockHeader BlockChain::read_header(const Hash &bid) const { const api::BlockHeader &BlockChain::get_tip() const { auto cit = header_cache.find(get_tip_bid()); - if( cit != header_cache.end() ){ + if (cit != header_cache.end()) { return cit->second; } read_header(get_tip_bid()); cit = header_cache.find(get_tip_bid()); - if( cit == header_cache.end() ) + if (cit == header_cache.end()) throw std::logic_error("After read_header, header should be in header_cache"); return cit->second; -// if (m_tip_segment.empty()) -// m_tip_segment.push_back(read_header(get_tip_bid())); -// return m_tip_segment.back(); + // if (m_tip_segment.empty()) + // m_tip_segment.push_back(read_header(get_tip_bid())); + // return m_tip_segment.back(); } -std::vector -BlockChain::get_tip_segment(const api::BlockHeader & prev_info, Height window, bool add_genesis) const { +std::vector BlockChain::get_tip_segment( + const api::BlockHeader &prev_info, Height window, bool add_genesis) const { std::vector result; - if( prev_info.height == Height(-1)) + if (prev_info.height == Height(-1)) return result; api::BlockHeader pi = prev_info; - while(result.size() < window && pi.height != 0){ + while (result.size() < window && pi.height != 0) { result.push_back(pi); - if( !read_header(pi.previous_block_hash, pi) ) + if (!read_header(pi.previous_block_hash, pi)) throw std::logic_error("Invariant dead - previous block header not found in get_tip_segment"); } - if( result.size() < window && add_genesis){ - if( pi.height != 0) - throw std::logic_error("Invariant dead - window size not reached, but genesis not found in get_tip_segment"); + if (result.size() < window && add_genesis) { + if (pi.height != 0) + throw std::logic_error( + "Invariant dead - window size not reached, but genesis not found in get_tip_segment"); result.push_back(pi); } -// if (get_tip_height() == (Height)-1 || height_delta > get_tip_height()) -// return std::make_pair(m_tip_segment.end(), m_tip_segment.end()); -// while (m_tip_segment.size() < height_delta + window && m_tip_segment.size() < m_tip_height + 1) { -// Hash ha = read_chain(static_cast(m_tip_height - m_tip_segment.size())); -// m_tip_segment.push_front(read_header(ha)); -// } -// if (m_tip_height + 1 <= height_delta + window) { -// if( m_tip_segment.size() == m_tip_height + 1 ) { - // if (height_delta + window >= m_tip_segment.size()) { -// return std::make_pair(m_tip_segment.begin() + (add_genesis ? 0 : 1), m_tip_segment.end() - height_delta); -// } + // if (get_tip_height() == (Height)-1 || height_delta > get_tip_height()) + // return std::make_pair(m_tip_segment.end(), m_tip_segment.end()); + // while (m_tip_segment.size() < height_delta + window && m_tip_segment.size() < m_tip_height + 1) { + // Hash ha = read_chain(static_cast(m_tip_height - m_tip_segment.size())); + // m_tip_segment.push_front(read_header(ha)); + // } + // if (m_tip_height + 1 <= height_delta + window) { + // if( m_tip_segment.size() == m_tip_height + 1 ) { + // if (height_delta + window >= m_tip_segment.size()) { + // return std::make_pair(m_tip_segment.begin() + (add_genesis ? 0 : 1), m_tip_segment.end() - height_delta); + // } std::reverse(result.begin(), result.end()); - return result;// std::make_pair(m_tip_segment.end() - window - height_delta, m_tip_segment.end() - height_delta); + return result; // std::make_pair(m_tip_segment.end() - window - height_delta, m_tip_segment.end() - height_delta); } void BlockChain::read_tip() { - DB::Cursor cur2 = m_db.rbegin(TIP_CHAIN_PREFIX + version_current + "/"); + DB::Cursor cur2 = m_db.rbegin(TIP_CHAIN_PREFIX + version_1 + "/"); m_tip_height = cur2.end() ? -1 : boost::lexical_cast(common::read_varint_sqlite4(cur2.get_suffix())); seria::from_binary(m_tip_bid, cur2.get_value_array()); api::BlockHeader tip_block = read_header(m_tip_bid); @@ -605,7 +659,7 @@ void BlockChain::read_tip() { void BlockChain::push_chain(Hash bid, Difficulty cumulative_difficulty) { m_tip_height += 1; BinaryArray ba = seria::to_binary(bid); - m_db.put(TIP_CHAIN_PREFIX + version_current + "/" + common::write_varint_sqlite4(m_tip_height), ba, true); + m_db.put(TIP_CHAIN_PREFIX + version_1 + "/" + common::write_varint_sqlite4(m_tip_height), ba, true); m_tip_bid = bid; m_tip_cumulative_difficulty = cumulative_difficulty; tip_changed(); @@ -614,13 +668,13 @@ void BlockChain::push_chain(Hash bid, Difficulty cumulative_difficulty) { void BlockChain::pop_chain() { if (m_tip_height == 0) throw std::logic_error("pop_chain tip_height == 0"); - m_db.del(TIP_CHAIN_PREFIX + version_current + "/" + common::write_varint_sqlite4(m_tip_height), true); + m_db.del(TIP_CHAIN_PREFIX + version_1 + "/" + common::write_varint_sqlite4(m_tip_height), true); m_tip_height -= 1; } bool BlockChain::read_chain(uint32_t height, Hash &bid) const { BinaryArray ba; - if (!m_db.get(TIP_CHAIN_PREFIX + version_current + "/" + common::write_varint_sqlite4(height), ba)) + if (!m_db.get(TIP_CHAIN_PREFIX + version_1 + "/" + common::write_varint_sqlite4(height), ba)) return false; seria::from_binary(bid, ba); return true; @@ -708,11 +762,24 @@ void BlockChain::test_prune_oldest() { } } -void BlockChain::test_print_structure() const { +void BlockChain::test_print_structure(Height n_confirmations) const { Difficulty ocd; Hash obid; if (get_oldest_tip(ocd, obid)) std::cout << "oldest tip cd=" << ocd << " bid=" << common::pod_to_hex(obid) << std::endl; + for (DB::Cursor cur = m_db.begin(CHILDREN_PREFIX); !cur.end(); cur.next()) { + Hash bid; + DB::from_binary_key(cur.get_suffix(), 0, bid.data, sizeof(bid.data)); + uint32_t counter = 1; + seria::from_binary(counter, cur.get_value_array()); + + std::cout << "children counter=" << counter << " bid=" << common::pod_to_hex(bid) << std::endl; + } + size_t total_forked_transactions = 0; + size_t total_possible_ds_transactions = 0; + Amount total_possible_ds_amount = 0; + size_t total_forked_blocks = 0; + size_t total_forked_blocks_not_found = 0; for (DB::Cursor cur = m_db.begin(CD_TIPS_PREFIX); !cur.end(); cur.next()) { const std::string &suf = cur.get_suffix(); const char *be = suf.data(); @@ -723,15 +790,191 @@ void BlockChain::test_print_structure() const { throw std::logic_error("CD_TIPS_PREFIX corrupted"); DB::from_binary_key(cur.get_suffix(), cur.get_suffix().size() - sizeof(bid.data), bid.data, sizeof(bid.data)); std::cout << "tip cd=" << cd << " bid=" << common::pod_to_hex(bid) << std::endl; + Height t_height = Height(-1); + while (true) { + api::BlockHeader header = read_header(bid); + if (t_height == Height(-1)) + t_height = header.height; + Hash main_bid; + if (read_chain(header.height, main_bid) && main_bid == header.hash) + break; // Reached main trunk + const bool confirmed = t_height >= header.height + n_confirmations; + std::cout << " fork height=" << header.height << " confirmed=" << confirmed + << " bid=" << common::pod_to_hex(bid) << std::endl; + RawBlock rb; + Block block; + if (confirmed) { + total_forked_blocks += 1; + if (read_block(bid, rb) && block.from_raw_block(rb)) { + for (size_t tx_pos = 0; tx_pos != block.header.transaction_hashes.size(); ++tx_pos) { + Hash tid = block.header.transaction_hashes.at(tx_pos); + total_forked_transactions += 1; + Transaction tx; + Height height = 0; + Hash block_hash; + size_t index_in_block = 0; + if (!read_transaction(tid, tx, height, block_hash, index_in_block)) { + Amount input_amount = 0; + for (const auto &input : block.transactions.at(tx_pos).inputs) + if (input.type() == typeid(KeyInput)) { + const KeyInput &in = boost::get(input); + input_amount += in.amount; + } + total_possible_ds_transactions += 1; + total_possible_ds_amount += input_amount; + std::cout << " Potential ds tx amount=" << input_amount + << " tid=" << common::pod_to_hex(tid) << std::endl; + } + } + } else + total_forked_blocks_not_found += 1; + } + bid = header.previous_block_hash; + } } - for (DB::Cursor cur = m_db.begin(CHILDREN_PREFIX); !cur.end(); cur.next()) { + std::cout << "n_confirmations=" << n_confirmations << std::endl; + std::cout << "total forked blocks=" << total_forked_blocks << ", not found " << total_forked_blocks_not_found + << std::endl; + std::cout << "total forked transactions=" << total_forked_transactions << ", possible ds " + << total_possible_ds_transactions << " total amount=" << total_possible_ds_amount << std::endl; +} + +bool BlockChain::fix_consensus(Hash bid, const api::BlockHeader &was_info) { + RawBlock rb; + Block block; + if (!read_block(bid, rb)) + throw std::logic_error("Block for fix consensus not found - please delete " + m_coin_folder + "/blockchain"); + PreparedBlock pb(std::move(rb), nullptr); + + api::BlockHeader info; + api::BlockHeader prev_info; + prev_info.height = -1; + if (pb.bid != m_genesis_bid) + prev_info = read_header(pb.block.header.previous_block_hash); + info.major_version = pb.block.header.major_version; + info.minor_version = pb.block.header.minor_version; + info.timestamp = pb.block.header.timestamp; + info.previous_block_hash = pb.block.header.previous_block_hash; + info.nonce = pb.block.header.nonce; + info.hash = pb.bid; + info.height = prev_info.height + 1; + + std::string check_error = check_standalone_consensus(pb, info, prev_info); + if (!check_error.empty()) + throw std::runtime_error("Failed to fix consensus differences for block " + common::pod_to_hex(bid) + + ", because " + check_error + ", " + delete_blockchain_message + m_coin_folder + + "/blockchain"); + if (info.cumulative_difficulty != was_info.cumulative_difficulty) { + //std::cout << "Cumulative difficulty difference for height=" << start_height << " mustbe=" + // << info.cumulative_difficulty << " was=" << was_info.cumulative_difficulty << std::endl; + auto key = HEADER_PREFIX + DB::to_binary_key(bid.data, sizeof(bid.data)) + HEADER_SUFFIX; + BinaryArray ba = seria::to_binary(info); + m_db.put(key, ba, false); + header_cache.erase(bid); + return true; + } + return false; +} + +void BlockChain::test_consensus(Height start_height) { + int fixed_counter = 0; + while (true) { Hash bid; - DB::from_binary_key(cur.get_suffix(), 0, bid.data, sizeof(bid.data)); - uint32_t counter = 1; - seria::from_binary(counter, cur.get_value_array()); + if (!read_chain(start_height, bid)) + break; // Reached main trunk + api::BlockHeader was_info = read_header(bid); + if (start_height % 100 == 0) + std::cout << "Testing consensus difference. Will take several minutes. height=" << start_height + << " fixed so far " << fixed_counter << std::endl; + if (fix_consensus(bid, was_info)) + fixed_counter += 1; + start_height += 1; + } + if (fixed_counter != 0) + throw std::runtime_error( + "test_consensus fixed_counter=" + std::to_string(fixed_counter) + " will not commit changes."); + std::cout << "Testing consesus complete, no differences" << std::endl; +} - std::cout << "children counter=" << counter << " bid=" << common::pod_to_hex(bid) << std::endl; +void BlockChain::fix_consensus() { + // const size_t max_ch = sizeof(difficulty_check)/sizeof(*difficulty_check); + int fixed_counter = 0; + std::set fixed_headers; + for (size_t ch = 0; ch != difficulty_check_count; ++ch) { + Hash bid; + if (!common::pod_from_hex(difficulty_check[ch].hash, bid)) + throw std::logic_error("DifficultyCheck table corrupted"); + api::BlockHeader was_info; + if (!read_header(bid, was_info)) + break; + if (ch % 1000 == 0) + std::cout << "Checking consensus difference. Will take several minutes. Remained blocks " + << difficulty_check_count - ch << " fixed so far " << fixed_counter << std::endl; + fixed_headers.insert(bid); + if (was_info.cumulative_difficulty != difficulty_check[ch].cumulative_difficulty) { + if (ch == 0) // We are not sure previous block has right difficulty, aborting + throw std::runtime_error("Failed to fix zero consensus differences, " + delete_blockchain_message + + m_coin_folder + "/blockchain"); + if (fixed_counter % 100 == 0) + std::cout << "Fixing consensus difference for height " << difficulty_check[ch].height + << ". Will take several minutes. remained blocks " << difficulty_check_count - ch + << " fixed so far " << fixed_counter << std::endl; + fixed_counter += 1; + fix_consensus(bid, was_info); + api::BlockHeader now_info = read_header(bid); + if (now_info.cumulative_difficulty != difficulty_check[ch].cumulative_difficulty) { + throw std::runtime_error("Failed to fix consensus differences, " + delete_blockchain_message + + m_coin_folder + "/blockchain"); + } + } } + std::cout << "Checked " << difficulty_check_count << " headers for consensus differences, fixed " << fixed_counter + << std::endl; + // start_height = difficulty_check[0].height; + // std::cout << "Found header with consensus differences at height"<< start_height << ", fixing (can take several + //minutes)..." << std::endl; + int progress_counter = 0; + for (DB::Cursor cur = m_db.rbegin(CD_TIPS_PREFIX); !cur.end(); cur.next()) { + const std::string &suf = cur.get_suffix(); + const char *be = suf.data(); + const char *en = be + suf.size(); + Difficulty cd = common::read_varint_sqlite4(be, en); + Hash tip_bid; + if (en - be != sizeof(tip_bid.data)) + throw std::logic_error("CD_TIPS_PREFIX corrupted"); + DB::from_binary_key( + cur.get_suffix(), cur.get_suffix().size() - sizeof(tip_bid.data), tip_bid.data, sizeof(tip_bid.data)); + // if( tip_bid == get_tip_bid()) + // std::cout << "Main chain" << std::endl; + std::vector side_chain; + for (Hash bid = tip_bid;;) { + api::BlockHeader header = read_header(bid); + Hash main_bid; + if (fixed_headers.count(bid) != 0 || header.height < difficulty_check[0].height) + break; // Do not fix beyond table start, or beyond fixed header + side_chain.push_back(bid); + bid = header.previous_block_hash; + } + // std::reverse(side_chain.begin(), side_chain.end()); + std::cout << "tip cd=" << cd << " len=" << side_chain.size() << " bid=" << common::pod_to_hex(tip_bid) + << std::endl; + for (; !side_chain.empty(); side_chain.pop_back()) { + Hash bid = side_chain.back(); + api::BlockHeader was_info = read_header(bid); + if (progress_counter % 100 == 0) + std::cout << "Fixing consensus difference. Will take several minutes. Remained blocks " + << side_chain.size() << " fixed so far " << fixed_counter << std::endl; + progress_counter += 1; + if (fix_consensus(bid, was_info)) + fixed_counter += 1; + fixed_headers.insert(bid); + } + } + api::BlockHeader tip_block = read_header(m_tip_bid); + m_tip_cumulative_difficulty = tip_block.cumulative_difficulty; + tip_changed(); + std::cout << "Checked consensus, made " << fixed_counter << " fixes" << std::endl; + // throw std::runtime_error("Preventing DB commit"); } bool BlockChain::read_next_internal_block(Hash &bid) const { @@ -787,13 +1030,13 @@ void BlockChain::test_undo_everything() { if (get_tip_bid() == m_genesis_bid) break; pop_chain(); - m_tip_bid = block.header.previous_block_hash; - api::BlockHeader info = get_tip(); + m_tip_bid = block.header.previous_block_hash; + api::BlockHeader info = get_tip(); m_tip_cumulative_difficulty = info.cumulative_difficulty; tip_changed(); - if (get_tip_height() % 50000 == 1 ) + if (get_tip_height() % 50000 == 1) db_commit(); - if( get_tip_height() <= 1525025 ) + if (get_tip_height() <= 1525025) return; } std::cout << "---- After undo everything ---- " << std::endl; diff --git a/src/Core/BlockChain.hpp b/src/Core/BlockChain.hpp index 45bde967..2b61e829 100644 --- a/src/Core/BlockChain.hpp +++ b/src/Core/BlockChain.hpp @@ -14,6 +14,14 @@ namespace bytecoin { enum class BroadcastAction { BROADCAST_ALL, NOTHING, BAN }; +enum class AddTransactionResult { + BAN, + BROADCAST_ALL, + ALREADY_IN_POOL, + INCREASE_FEE, + FAILED_TO_REDO, + OUTPUT_ALREADY_SPENT +}; struct PreparedBlock { BinaryArray block_data; @@ -44,14 +52,15 @@ class BlockChain { Height get_tip_height() const { return m_tip_height; } const api::BlockHeader &get_tip() const; - std::vector - get_tip_segment(const api::BlockHeader & prev_info, Height window, bool add_genesis) const; + std::vector get_tip_segment( + const api::BlockHeader &prev_info, Height window, bool add_genesis) const; bool read_chain(Height height, Hash &bid) const; bool read_block(const Hash &bid, RawBlock &rb) const; bool has_block(const Hash &bid) const; bool read_header(const Hash &bid, api::BlockHeader &info) const; - bool read_transaction(const Hash &tid, Transaction &tx, Height &height, size_t &index_in_block) const; + bool read_transaction( + const Hash &tid, Transaction &tx, Height &block_height, Hash &block_hash, size_t &index_in_block) const; // Modify blockchain state. bytecoin header does not contain enough info for consensus calcs, so we cannot have // header chain without block chain @@ -67,16 +76,21 @@ class BlockChain { Height get_timestamp_lower_bound_block_index(Timestamp) const; void test_undo_everything(); - void test_print_structure() const; + void test_print_structure(Height n_confirmations) const; + + void fix_consensus(); + void test_consensus(Height start_height); void test_prune_oldest(); + void db_commit(); bool internal_import(); // import some existing blocks from inside DB Height internal_import_known_height() const { return m_internal_import_known_height; } protected: + Difficulty get_tip_cumulative_difficulty() const { return m_tip_cumulative_difficulty; } bool read_next_internal_block(Hash &bid) const; - virtual bool check_standalone_consensus( + virtual std::string check_standalone_consensus( const PreparedBlock &pb, api::BlockHeader &info, const api::BlockHeader &prev_info) const = 0; virtual bool redo_block(const Hash &bhash, const Block &block, const api::BlockHeader &info) = 0; virtual void undo_block(const Hash &bhash, const Block &block, Height height) = 0; @@ -84,6 +98,8 @@ class BlockChain { const Hash &base_transaction_hash); void undo_block(const Hash &bhash, const RawBlock &raw_block, const Block &block, Height height); virtual void tip_changed() {} // Quick hack to allow BlockChainState to update next block params + virtual void on_reorganization( + const std::map> &undone_transactions, bool undone_blocks) = 0; const Hash m_genesis_bid; const std::string m_coin_folder; @@ -95,6 +111,8 @@ class BlockChain { Hash read_chain(Height height) const; api::BlockHeader read_header(const Hash &bid) const; + static const std::string version_current; + private: Hash m_tip_bid; Difficulty m_tip_cumulative_difficulty = 0; @@ -118,6 +136,7 @@ class BlockChain { void modify_children_counter(Difficulty cd, const Hash &bid, int delta); bool get_oldest_tip(Difficulty &cd, Hash &bid) const; bool prune_branch(Difficulty cd, Hash bid); + bool fix_consensus(Hash bid, const api::BlockHeader &was_info); }; } // namespace bytecoin diff --git a/src/Core/BlockChainState.cpp b/src/Core/BlockChainState.cpp index c6ff38fe..4a022a7f 100644 --- a/src/Core/BlockChainState.cpp +++ b/src/Core/BlockChainState.cpp @@ -24,23 +24,13 @@ static const std::string UNLOCK_BLOCK_PREFIX = "u"; static const std::string UNLOCK_TIME_PREFIX = "U"; // We store locked outputs in separate indexes -const size_t MAX_POOL_COMPLEXITY = 100000; // ~1000 "normal" transactions +const size_t MAX_POOL_SIZE = 2000000; // ~1000 "normal" transactions with 10 inputs and 10 outputs using namespace bytecoin; using namespace platform; -static size_t get_complexity(const Transaction &tx) { // For pool - size_t result = 0; - result += 20 * tx.inputs.size(); // ring signature checking is expensive - result += tx.outputs.size(); - for (const auto &input : tx.inputs) { - if (input.type() == typeid(KeyInput)) { - const KeyInput &in = boost::get(input); - result += 10 * in.output_indexes.size(); // ring signature checking is expensive - } - } - return result; -} +BlockChainState::PoolTransaction::PoolTransaction(const Transaction &tx, const BinaryArray &binary_tx, Amount fee) + : tx(tx), binary_tx(binary_tx), fee(fee) {} void BlockChainState::DeltaState::store_keyimage(const KeyImage &keyimage, Height height) { if (!m_keyimages.insert(std::make_pair(keyimage, height)).second) @@ -123,12 +113,67 @@ api::BlockHeader BlockChainState::fill_genesis(Hash genesis_bid, const BlockTemp return result; } +static std::string validate_semantic(bool generating, const Transaction &tx, uint64_t &fee, bool check_output_key) { + if (tx.inputs.empty()) + return "EMPTY_INPUTS"; + uint64_t summary_output_amount = 0; + for (const auto &output : tx.outputs) { + if (output.amount == 0) + return "OUTPUT_ZERO_AMOUNT"; + if (output.target.type() == typeid(KeyOutput)) { + const KeyOutput &key_output = boost::get(output.target); + if (check_output_key && !key_isvalid(key_output.key)) + return "OUTPUT_INVALID_KEY"; + } else + return "OUTPUT_UNKNOWN_TYPE"; + if (std::numeric_limits::max() - output.amount < summary_output_amount) + return "OUTPUTS_AMOUNT_OVERFLOW"; + summary_output_amount += output.amount; + } + uint64_t summary_input_amount = 0; + std::unordered_set ki; + std::set> outputs_usage; + for (const auto &input : tx.inputs) { + uint64_t amount = 0; + if (input.type() == typeid(CoinbaseInput)) { + if (!generating) + return "INPUT_UNKNOWN_TYPE"; + } else if (input.type() == typeid(KeyInput)) { + if (generating) + return "INPUT_UNKNOWN_TYPE"; + const KeyInput &in = boost::get(input); + amount = in.amount; + if (!ki.insert(in.key_image).second) + return "INPUT_IDENTICAL_KEYIMAGES"; + if (in.output_indexes.empty()) + return "INPUT_EMPTY_OUTPUT_USAGE"; + // output_indexes are packed here, first is absolute, others are offsets to + // previous, so first can be zero, others can't + if (std::find(++std::begin(in.output_indexes), std::end(in.output_indexes), 0) != + std::end(in.output_indexes)) { + return "INPUT_IDENTICAL_OUTPUT_INDEXES"; + } + } else + return "INPUT_UNKNOWN_TYPE"; + if (std::numeric_limits::max() - amount < summary_input_amount) + return "INPUTS_AMOUNT_OVERFLOW"; + summary_input_amount += amount; + } + if (summary_output_amount > summary_input_amount && !generating) + return "WRONG_AMOUNT"; + if (tx.signatures.size() != tx.inputs.size() && !generating) + return "INPUT_UNKNOWN_TYPE"; + if (!tx.signatures.empty() && generating) + return "INPUT_UNKNOWN_TYPE"; + fee = summary_input_amount - summary_output_amount; + return std::string(); +} + BlockChainState::BlockChainState(logging::ILogger &log, const Config &config, const Currency ¤cy) : BlockChain(currency.genesis_block_hash, config.get_data_folder()) - , m_config(config) + // , m_config(config) , m_currency(currency) , m_log(log, "BlockChainState") - , m_memory_state_total_complexity(0) , log_redo_block_timestamp(std::chrono::steady_clock::now()) { if (get_tip_height() == (Height)-1) { Block genesis_block; @@ -141,17 +186,25 @@ BlockChainState::BlockChainState(logging::ILogger &log, const Config &config, co if (add_block(pb, info) == BroadcastAction::BAN) throw std::logic_error("Genesis block failed to add"); } - m_log(logging::INFO) << "BlockChainState::BlockChainState height=" << get_tip_height() << " bid=" << common::pod_to_hex(get_tip_bid()) << std::endl; BlockChainState::tip_changed(); + std::string version; + m_db.get("$version", version); + if (version == "1") { // 1 -> 2 + std::cout << "Database version 1, advancing to version 2" << std::endl; + //fix_consensus(); + version = "2"; + m_db.put("$version", version, false); + db_commit(); + } + if (version != version_current) + throw std::runtime_error("Blockchain database format unknown (version=" + version + "), please delete " + + config.get_data_folder() + "/blockchain"); + m_log(logging::INFO) << "BlockChainState::BlockChainState height=" << get_tip_height() + << " cumulative_difficulty=" << get_tip_cumulative_difficulty() + << " bid=" << common::pod_to_hex(get_tip_bid()) << std::endl; } -bool BlockChainState::check_standalone_consensus( - const PreparedBlock &pb, api::BlockHeader &info, const api::BlockHeader &prev_info) const { - auto err = get_standalone_consensus_error(pb, info, prev_info); - return err.empty(); -} - -std::string BlockChainState::get_standalone_consensus_error( +std::string BlockChainState::check_standalone_consensus( const PreparedBlock &pb, api::BlockHeader &info, const api::BlockHeader &prev_info) const { const auto &block = pb.block; if (block.transactions.size() != block.header.transaction_hashes.size()) @@ -212,7 +265,7 @@ std::string BlockChainState::get_standalone_consensus_error( const bool check_keys = !m_currency.is_in_checkpoint_zone(info.height); uint64_t miner_reward = 0; - for (const auto &output : block.header.base_transaction.outputs) { + for (const auto &output : block.header.base_transaction.outputs) { // TODO - call validate_semantic if (output.amount == 0) return "OUTPUT_ZERO_AMOUNT"; if (output.target.type() == typeid(KeyOutput)) { @@ -274,6 +327,12 @@ std::string BlockChainState::get_standalone_consensus_error( info.already_generated_transactions = prev_info.already_generated_transactions + block.transactions.size() + 1; info.total_fee_amount = cumulative_fee; info.transactions_cumulative_size = static_cast(cumulative_size); + for (auto &&tx : pb.block.transactions) { + Amount tx_fee = 0; + std::string tx_result = validate_semantic(false, tx, tx_fee, check_keys); + if (!tx_result.empty()) + return tx_result; + } bool is_checkpoint; if (m_currency.is_in_checkpoint_zone(info.height)) { if (!m_currency.check_block_checkpoint(info.height, info.hash, is_checkpoint)) @@ -287,7 +346,7 @@ std::string BlockChainState::get_standalone_consensus_error( return std::string(); } -void BlockChainState::calculate_consensus_values(const api::BlockHeader & prev_info, uint32_t &next_median_size, +void BlockChainState::calculate_consensus_values(const api::BlockHeader &prev_info, uint32_t &next_median_size, Timestamp &next_median_timestamp, Timestamp &next_unlock_timestamp) const { std::vector last_blocks_sizes; auto window = get_tip_segment(prev_info, m_currency.reward_blocks_window, true); @@ -318,7 +377,8 @@ void BlockChainState::calculate_consensus_values(const api::BlockHeader & prev_i } void BlockChainState::tip_changed() { - calculate_consensus_values(read_header(get_tip_bid()), m_next_median_size, m_next_median_timestamp, m_next_unlock_timestamp); + calculate_consensus_values( + read_header(get_tip_bid()), m_next_median_size, m_next_median_timestamp, m_next_unlock_timestamp); } bool BlockChainState::create_mining_block_template(BlockTemplate &b, const AccountPublicAddress &adr, @@ -397,16 +457,14 @@ bool BlockChainState::create_mining_block_template(BlockTemplate &b, const Accou assert(false); continue; } - size_t block_size_limit = max_total_size; - size_t tx_size = seria::binary_size(tit->second); // TODO - remember size in tx index + const size_t block_size_limit = max_total_size; + const size_t tx_size = tit->second.binary_tx.size(); if (txs_size + tx_size > block_size_limit) continue; - Amount single_fee = 0; - bool fatal = false; + Amount single_fee = tit->second.fee; BlockGlobalIndices global_indices; - std::string result = - redo_transaction_get_error(false, tit->second, &memory_state, global_indices, true, single_fee, - fatal); // next_median_timestamp + const std::string result = + redo_transaction_get_error(false, tit->second.tx, &memory_state, global_indices, true); if (!result.empty()) { m_log(logging::ERROR) << "Transaction " << common::pod_to_hex(tit->first) << " is in pool, but could not be redone result=" << result << std::endl; @@ -415,7 +473,7 @@ bool BlockChainState::create_mining_block_template(BlockTemplate &b, const Accou txs_size += tx_size; fee += single_fee; b.transaction_hashes.emplace_back(tit->first); - m_mining_transactions.insert(std::make_pair(tit->first, std::make_pair(tit->second, height))); + m_mining_transactions.insert(std::make_pair(tit->first, std::make_pair(tit->second.binary_tx, height))); m_log(logging::TRACE) << "Transaction " << common::pod_to_hex(tit->first) << " included to block template"; } @@ -477,7 +535,7 @@ bool BlockChainState::create_mining_block_template(BlockTemplate &b, const Accou continue; } - m_log(logging::DEBUGGING) + m_log(logging::TRACE) << logging::BrightGreen << "Setting extra for block: " << b.base_transaction.extra.size() << ", try_count=" << try_count; } @@ -511,10 +569,10 @@ BroadcastAction BlockChainState::add_mined_block( raw_block.transactions.reserve(block_template.transaction_hashes.size()); raw_block.transactions.clear(); for (const auto &tx_hash : block_template.transaction_hashes) { - auto tit = m_memory_state_tx.find(tx_hash); - const Transaction *tx = nullptr; + auto tit = m_memory_state_tx.find(tx_hash); + const BinaryArray *binary_tx = nullptr; if (tit != m_memory_state_tx.end()) - tx = &(tit->second); + binary_tx = &(tit->second.binary_tx); else { auto tit2 = m_mining_transactions.find(tx_hash); if (tit2 == m_mining_transactions.end()) { @@ -522,9 +580,9 @@ BroadcastAction BlockChainState::add_mined_block( << " is absent in transaction pool on submit mined block"; return BroadcastAction::NOTHING; } - tx = &(tit2->second.first); + binary_tx = &(tit2->second.first); } - raw_block.transactions.emplace_back(seria::to_binary(*tx)); + raw_block.transactions.emplace_back(*binary_tx); } PreparedBlock pb(std::move(raw_block), nullptr); raw_block = pb.raw_block; @@ -539,132 +597,165 @@ void BlockChainState::clear_mining_transactions() const { ++tit; } -static std::string validate_semantic(bool generating, const Transaction &tx, uint64_t &fee, bool check_output_key) { - if (tx.inputs.empty()) - return "EMPTY_INPUTS"; - uint64_t summary_output_amount = 0; - for (const auto &output : tx.outputs) { - if (output.amount == 0) - return "OUTPUT_ZERO_AMOUNT"; - if (output.target.type() == typeid(KeyOutput)) { - const KeyOutput &key_output = boost::get(output.target); - if (check_output_key && !key_isvalid(key_output.key)) - return "OUTPUT_INVALID_KEY"; - } else - return "OUTPUT_UNKNOWN_TYPE"; - if (std::numeric_limits::max() - output.amount < summary_output_amount) - return "OUTPUTS_AMOUNT_OVERFLOW"; - summary_output_amount += output.amount; +Amount BlockChainState::minimum_pool_fee_per_byte(Hash &minimal_tid) const { + if (m_memory_state_fee_tx.empty()) { + minimal_tid = Hash(); + return 0; } - uint64_t summary_input_amount = 0; - std::unordered_set ki; - std::set> outputs_usage; - for (const auto &input : tx.inputs) { - uint64_t amount = 0; - if (input.type() == typeid(CoinbaseInput)) { - if (!generating) - return "INPUT_UNKNOWN_TYPE"; - } else if (input.type() == typeid(KeyInput)) { - if (generating) - return "INPUT_UNKNOWN_TYPE"; - const KeyInput &in = boost::get(input); - amount = in.amount; - if (!ki.insert(in.key_image).second) - return "INPUT_IDENTICAL_KEYIMAGES"; - if (in.output_indexes.empty()) - return "INPUT_EMPTY_OUTPUT_USAGE"; - // output_indexes are packed here, first is absolute, others are offsets to - // previous, - // so first can be zero, others can't - if (std::find(++std::begin(in.output_indexes), std::end(in.output_indexes), 0) != - std::end(in.output_indexes)) { - return "INPUT_IDENTICAL_OUTPUT_INDEXES"; - } - } else - return "INPUT_UNKNOWN_TYPE"; - if (std::numeric_limits::max() - amount < summary_input_amount) - return "INPUTS_AMOUNT_OVERFLOW"; - summary_input_amount += amount; + auto be = m_memory_state_fee_tx.begin(); + if (be->second.empty()) + throw std::logic_error("Invariant dead, memory_state_fee_tx empty set"); + minimal_tid = *(be->second.begin()); + return be->first; +} + +void BlockChainState::on_reorganization( + const std::map> &undone_transactions, bool undone_blocks) { + // TODO - remove/add only those transactions that could have their referenced output keys changed + if (undone_blocks) { + PoolTransMap old_memory_state_tx; + std::swap(old_memory_state_tx, m_memory_state_tx); + m_memory_state_ki_tx.clear(); + m_memory_state_fee_tx.clear(); + m_memory_state_total_size = 0; + for (auto &&msf : old_memory_state_tx) { + add_transaction( + msf.first, msf.second.tx, msf.second.binary_tx, get_tip_height() + 1, get_tip().timestamp, true); + } } - if (summary_output_amount > summary_input_amount && !generating) - return "WRONG_AMOUNT"; - if (tx.signatures.size() != tx.inputs.size() && !generating) - return "INPUT_UNKNOWN_TYPE"; - if (!tx.signatures.empty() && generating) - return "INPUT_UNKNOWN_TYPE"; - fee = summary_input_amount - summary_output_amount; - return std::string(); + for (auto ud : undone_transactions) { + add_transaction(ud.first, ud.second.first, ud.second.second, get_tip_height() + 1, get_tip().timestamp, true); + } + m_tx_pool_version = 2; // add_transaction will erroneously increase } -BroadcastAction BlockChainState::add_transaction(const Transaction &tx, Timestamp now) { - auto thash = get_transaction_hash(tx); - Timestamp g_timestamp = read_first_seen_timestamp(thash); - if (g_timestamp != 0 && now > g_timestamp + m_config.mempool_tx_live_time) - return BroadcastAction::NOTHING; - return add_transaction(thash, tx, get_tip_height() + 1, get_tip().timestamp, MAX_POOL_COMPLEXITY, true); +AddTransactionResult BlockChainState::add_transaction( + const Hash &tid, const Transaction &tx, const BinaryArray &binary_tx, Timestamp now) { + // Timestamp g_timestamp = read_first_seen_timestamp(tid); + // if (g_timestamp != 0 && now > g_timestamp + m_config.mempool_tx_live_time) + // return AddTransactionResult::TOO_OLD; + return add_transaction(tid, tx, binary_tx, get_tip_height() + 1, get_tip().timestamp, true); } -BroadcastAction BlockChainState::add_transaction(const Hash &tid, const Transaction &tx, Height unlock_height, - Timestamp unlock_timestamp, size_t max_pool_complexity, bool check_sigs) { - const size_t my_size = seria::binary_size(tx); +AddTransactionResult BlockChainState::add_transaction(const Hash &tid, const Transaction &tx, + const BinaryArray &binary_tx, Height unlock_height, Timestamp unlock_timestamp, bool check_sigs) { if (m_memory_state_tx.count(tid) != 0) - return BroadcastAction::NOTHING; - DeltaState memory_state(unlock_height, unlock_timestamp, this); // timestamp_unlock after fork - - Amount my_fee = 0; - bool fatal = false; + return AddTransactionResult::ALREADY_IN_POOL; + // std::cout << "add_transaction " << common::pod_to_hex(tid) << std::endl; + const size_t my_size = binary_tx.size(); + const Amount my_fee = bytecoin::get_tx_fee(tx); + const Amount my_fee_per_byte = my_fee / my_size; + Hash minimal_tid; + Amount minimal_fee = minimum_pool_fee_per_byte(minimal_tid); + // Invariant is if 1 byte of cheapest transaction fits, then all transaction fits + if (m_memory_state_total_size >= MAX_POOL_SIZE && my_fee_per_byte < minimal_fee) + return AddTransactionResult::INCREASE_FEE; + // Deterministic behaviour here and below so tx pools have tendency to stay the same + if (m_memory_state_total_size >= MAX_POOL_SIZE && my_fee_per_byte == minimal_fee && tid < minimal_tid) + return AddTransactionResult::INCREASE_FEE; + for (const auto &input : tx.inputs) { + if (input.type() == typeid(KeyInput)) { + const KeyInput &in = boost::get(input); + auto tit = m_memory_state_ki_tx.find(in.key_image); + if (tit == m_memory_state_ki_tx.end()) + continue; + const PoolTransaction &other_tx = m_memory_state_tx.at(tit->second); + const Amount other_fee_per_byte = other_tx.fee_per_byte(); + if (my_fee_per_byte < other_fee_per_byte) + return AddTransactionResult::INCREASE_FEE; + if (my_fee_per_byte == other_fee_per_byte && tid < tit->second) + return AddTransactionResult::INCREASE_FEE; + break; // Can displace another transaction from the pool, Will have to make heavy-lifting for this tx + } + } + for (const auto &input : tx.inputs) { + if (input.type() == typeid(KeyInput)) { + const KeyInput &in = boost::get(input); + if (read_keyimage(in.key_image)) { + // std::cout << common::pod_to_hex(tid) << " " << common::pod_to_hex(in.key_image) << + //std::endl; + return AddTransactionResult::OUTPUT_ALREADY_SPENT; // Already spent in main chain + } + } + } + Amount my_fee3 = 0; + const std::string validate_result = validate_semantic(false, tx, my_fee3, check_sigs); + if (!validate_result.empty()) { + m_log(logging::ERROR) << "add_transaction validation failed " << validate_result << " in transaction " + << common::pod_to_hex(tid) << std::endl; + return AddTransactionResult::BAN; + } + DeltaState memory_state(unlock_height, unlock_timestamp, this); BlockGlobalIndices global_indices; - std::string result = redo_transaction_get_error(false, tx, &memory_state, global_indices, check_sigs, my_fee, - fatal); // next_median_timestamp - if (!result.empty()) { - return BroadcastAction::NOTHING; // TODO - ban if fatal + const std::string redo_result = redo_transaction_get_error(false, tx, &memory_state, global_indices, check_sigs); + if (!redo_result.empty()) { + // std::cout << "Addding anyway for test " << std::endl; + m_log(logging::TRACE) << "add_transaction redo failed " << redo_result << " in transaction " + << common::pod_to_hex(tid) << std::endl; + return AddTransactionResult::FAILED_TO_REDO; // Not a ban because reorg can change indices } + if (my_fee != my_fee3) + m_log(logging::ERROR) << "Inconsistent fees " << my_fee << ", " << my_fee3 << " in transaction " + << common::pod_to_hex(tid) << std::endl; // Only good transactions are recorded in tx_first_seen, because they require // space there update_first_seen_timestamp(tid, unlock_timestamp); - const Amount my_fee_per_byte = my_fee / my_size; for (auto &&ki : memory_state.get_keyimages()) { auto tit = m_memory_state_ki_tx.find(ki.first); if (tit == m_memory_state_ki_tx.end()) continue; - const Transaction &other_tx = m_memory_state_tx.at(tit->second); - const Amount other_fee = bytecoin::get_tx_fee(other_tx); // TODO - optimize get fee - const size_t other_size = seria::binary_size(other_tx); - const Amount other_fee_per_byte = other_fee / other_size; + const PoolTransaction &other_tx = m_memory_state_tx.at(tit->second); + const Amount other_fee_per_byte = other_tx.fee_per_byte(); if (my_fee_per_byte < other_fee_per_byte) - return BroadcastAction::NOTHING; - if (my_fee_per_byte == other_fee_per_byte && - tid < tit->second) // Deterministic behaviour so tx pools have tendency - // to stay the same - return BroadcastAction::NOTHING; - Hash rhash = tit->second; - remove_from_pool(rhash); + return AddTransactionResult::INCREASE_FEE; // Never because checked above + if (my_fee_per_byte == other_fee_per_byte && tid < tit->second) + return AddTransactionResult::INCREASE_FEE; // Never because checked above + remove_from_pool(tit->second); } bool all_inserted = true; for (auto &&ki : memory_state.get_keyimages()) { if (!m_memory_state_ki_tx.insert(std::make_pair(ki.first, tid)).second) all_inserted = false; } - if (!m_memory_state_tx.insert(std::make_pair(tid, tx)).second) + if (!m_memory_state_tx.insert(std::make_pair(tid, PoolTransaction(tx, binary_tx, my_fee))).second) all_inserted = false; if (!m_memory_state_fee_tx[my_fee_per_byte].insert(tid).second) all_inserted = false; if (!all_inserted) // insert all before throw throw std::logic_error("Invariant dead, memory_state_fee_tx empty"); - max_pool_complexity = std::max(max_pool_complexity, m_memory_state_total_complexity); - // We do not reset pool complexity immediately after undo_block - m_memory_state_total_complexity += get_complexity(tx); - while (m_memory_state_total_complexity > max_pool_complexity) { + m_memory_state_total_size += my_size; + while (m_memory_state_total_size > MAX_POOL_SIZE) { if (m_memory_state_fee_tx.empty()) throw std::logic_error("Invariant dead, memory_state_fee_tx empty"); auto &be = m_memory_state_fee_tx.begin()->second; if (be.empty()) throw std::logic_error("Invariant dead, memory_state_fee_tx empty set"); - Hash rhash = *(be.begin()); + Hash rhash = *(be.begin()); + const PoolTransaction &minimal_tx = m_memory_state_tx.at(rhash); + if (m_memory_state_total_size < MAX_POOL_SIZE + minimal_tx.binary_tx.size()) + break; // Removing would diminish pool below max size remove_from_pool(rhash); } + auto min_size = m_memory_state_fee_tx.empty() || m_memory_state_fee_tx.begin()->second.empty() + ? 0 + : m_memory_state_tx.at(*(m_memory_state_fee_tx.begin()->second.begin())).binary_tx.size(); + auto min_fee_per_byte = m_memory_state_fee_tx.empty() || m_memory_state_fee_tx.begin()->second.empty() + ? 0 + : m_memory_state_fee_tx.begin()->first; + // if( m_memory_state_total_size-min_size >= MAX_POOL_SIZE) + // std::cout << "Aha" << std::endl; + m_log(logging::INFO) << "TX+ h=" << common::pod_to_hex(tid) << " s=" << my_size + << " fee=" << my_fee << " f/b=" << my_fee_per_byte << " ns=(" + << m_memory_state_total_size - min_size << "+" << min_size << ")=" << m_memory_state_total_size + << " count=" << m_memory_state_tx.size() << " min f/b=" << min_fee_per_byte << std::endl; + // for(auto && bb : m_memory_state_fee_tx) + // for(auto ff : bb.second){ + // const PoolTransaction &other_tx = m_memory_state_tx.at(ff); + // std::cout << "\t" << other_tx.fee_per_byte() << "\t" << other_tx.binary_tx.size() << "\t" << + //common::pod_to_hex(ff) << std::endl; + // } m_tx_pool_version += 1; - return BroadcastAction::BROADCAST_ALL; + return AddTransactionResult::BROADCAST_ALL; } void BlockChainState::remove_from_pool(Hash tid) { @@ -672,7 +763,7 @@ void BlockChainState::remove_from_pool(Hash tid) { if (tit == m_memory_state_tx.end()) return; bool all_erased = true; - const Transaction &tx = tit->second; + const Transaction &tx = tit->second.tx; for (const auto &input : tx.inputs) { if (input.type() == typeid(KeyInput)) { const KeyInput &in = boost::get(input); @@ -680,19 +771,28 @@ void BlockChainState::remove_from_pool(Hash tid) { all_erased = false; } } - const Amount my_fee = bytecoin::get_tx_fee(tx); // TODO - optimize get fee - const size_t my_size = seria::binary_size(tx); - const Amount my_fee_per_byte = my_fee / my_size; + const size_t my_size = tit->second.binary_tx.size(); + const Amount my_fee_per_byte = tit->second.fee_per_byte(); if (m_memory_state_fee_tx[my_fee_per_byte].erase(tid) != 1) all_erased = false; if (m_memory_state_fee_tx[my_fee_per_byte].empty()) m_memory_state_fee_tx.erase(my_fee_per_byte); - m_memory_state_total_complexity -= get_complexity(tx); + m_memory_state_total_size -= my_size; m_memory_state_tx.erase(tit); if (!all_erased) throw std::logic_error("Invariant dead, remove_memory_pool failed to erase everything"); // We do not increment m_tx_pool_version, because removing tx from pool is // always followed by reset or increment + auto min_size = m_memory_state_fee_tx.empty() || m_memory_state_fee_tx.begin()->second.empty() + ? 0 + : m_memory_state_tx.at(*(m_memory_state_fee_tx.begin()->second.begin())).binary_tx.size(); + auto min_fee_per_byte = m_memory_state_fee_tx.empty() || m_memory_state_fee_tx.begin()->second.empty() + ? 0 + : m_memory_state_fee_tx.begin()->first; + m_log(logging::INFO) << "TX- h=" << common::pod_to_hex(tid) << " s=" << my_size + << " ns=(" << m_memory_state_total_size - min_size << "+" << min_size << "=" + << m_memory_state_total_size << ") count=" << m_memory_state_tx.size() + << " min f/b=" << min_fee_per_byte << std::endl; } Timestamp BlockChainState::read_first_seen_timestamp(const Hash &tid) const { @@ -837,14 +937,10 @@ bool RingCheckerMulticore::signatures_valid() const { } } +// Called only on transactions which passed validate_semantic() std::string BlockChainState::redo_transaction_get_error(bool generating, const Transaction &transaction, - DeltaState *delta_state, BlockGlobalIndices &global_indices, bool check_sigs, Amount &fee, bool &fatal) const { + DeltaState *delta_state, BlockGlobalIndices &global_indices, bool check_sigs) const { const bool check_outputs = check_sigs; - std::string error = validate_semantic(generating, transaction, fee, check_outputs); - fatal = false; // for now we do not distinguish between fatal and non-fatal - // errors - if (!error.empty()) - return error; Hash tx_prefix_hash; if (check_sigs) tx_prefix_hash = get_transaction_prefix_hash(transaction); @@ -855,15 +951,14 @@ std::string BlockChainState::redo_transaction_get_error(bool generating, const T size_t input_index = 0; for (const auto &input : transaction.inputs) { - if (input.type() == typeid(CoinbaseInput)) { - } else if (input.type() == typeid(KeyInput)) { + if (input.type() == typeid(KeyInput)) { const KeyInput &in = boost::get(input); if (check_sigs || check_outputs) { if (tx_delta.read_keyimage(in.key_image)) return "INPUT_KEYIMAGE_ALREADY_SPENT"; if (in.output_indexes.empty()) - return "INPUT_UNKNOWN_TYPE"; + return "INPUT_UNKNOWN_TYPE"; // Never - checked in validate_semantic std::vector global_indexes(in.output_indexes.size()); global_indexes[0] = in.output_indexes[0]; for (size_t i = 1; i < in.output_indexes.size(); ++i) { @@ -892,8 +987,7 @@ std::string BlockChainState::redo_transaction_get_error(bool generating, const T } } tx_delta.store_keyimage(in.key_image, delta_state->get_block_height()); - } else - return "INPUT_UNKNOWN_TYPE"; + } input_index++; } for (const auto &output : transaction.outputs) { @@ -902,11 +996,10 @@ std::string BlockChainState::redo_transaction_get_error(bool generating, const T auto global_index = tx_delta.push_amount_output(output.amount, transaction.unlock_time, 0, 0, key_output.key); // DeltaState ignores unlock point my_indices.push_back(global_index); - } else - return "OUTPUT_UNKNOWN_TYPE"; + } } - tx_delta.apply(delta_state); // delta_state might be memory pool, we protect - // it from half-modification + tx_delta.apply(delta_state); + // delta_state might be memory pool, we protect it from half-modification return std::string(); } @@ -928,19 +1021,14 @@ bool BlockChainState::redo_block(const Block &block, const api::BlockHeader &info, DeltaState *delta_state, BlockGlobalIndices &global_indices) const { - // We will need info.timestamp_unlock after first hard fork - Amount fee = 0; - bool fatal = false; std::string result = - redo_transaction_get_error(true, block.header.base_transaction, delta_state, global_indices, false, fee, fatal); + redo_transaction_get_error(true, block.header.base_transaction, delta_state, global_indices, false); if (!result.empty()) return false; - SignedAmount sum_fee = 0; for (auto tit = block.transactions.begin(); tit != block.transactions.end(); ++tit) { - std::string result = redo_transaction_get_error(false, *tit, delta_state, global_indices, false, fee, fatal); + std::string result = redo_transaction_get_error(false, *tit, delta_state, global_indices, false); if (!result.empty()) return false; - sum_fee += fee; } return true; } @@ -956,7 +1044,7 @@ bool BlockChainState::redo_block(const Hash &bhash, const Block &block, const ap return false; if (check_sigs && !ring_checker.signatures_valid()) return false; - delta.apply(this); + delta.apply(this); // Will remove from pool by keyimage m_tx_pool_version = 2; auto key = @@ -965,7 +1053,6 @@ bool BlockChainState::redo_block(const Hash &bhash, const Block &block, const ap m_db.put(key, ba, true); for (auto th : block.header.transaction_hashes) { - remove_from_pool(th); update_first_seen_timestamp(th, 0); } auto now = std::chrono::steady_clock::now(); @@ -975,16 +1062,16 @@ bool BlockChainState::redo_block(const Hash &bhash, const Block &block, const ap << " bid=" << common::pod_to_hex(bhash) << std::endl; } m_log(logging::TRACE) << "redo_block {" << block.transactions.size() << "} height=" << info.height - << " bid=" << common::pod_to_hex(bhash) << std::endl; + << " bid=" << common::pod_to_hex(bhash) << std::endl; return true; } void BlockChainState::undo_block(const Hash &bhash, const Block &block, Height height) { -// if (height % 100 == 0) -// std::cout << "undo_block height=" << height << " bid=" << common::pod_to_hex(bhash) -// << " new tip_bid=" << common::pod_to_hex(block.header.previous_block_hash) << std::endl; + // if (height % 100 == 0) + // std::cout << "undo_block height=" << height << " bid=" << common::pod_to_hex(bhash) + // << " new tip_bid=" << common::pod_to_hex(block.header.previous_block_hash) << std::endl; m_log(logging::INFO) << "undo_block height=" << height << " bid=" << common::pod_to_hex(bhash) - << " new tip_bid=" << common::pod_to_hex(block.header.previous_block_hash) << std::endl; + << " new tip_bid=" << common::pod_to_hex(block.header.previous_block_hash) << std::endl; for (auto tit = block.transactions.rbegin(); tit != block.transactions.rend(); ++tit) { undo_transaction(this, height, *tit); } @@ -993,21 +1080,6 @@ void BlockChainState::undo_block(const Hash &bhash, const Block &block, Height h auto key = BLOCK_GLOBAL_INDICES_PREFIX + DB::to_binary_key(bhash.data, sizeof(bhash.data)) + BLOCK_GLOBAL_INDICES_SUFFIX; m_db.del(key, true); - - // Now put transactions to pool - for (auto tit = block.transactions.begin(); tit != block.transactions.end(); ++tit) { - if (m_memory_state_total_complexity < MAX_POOL_COMPLEXITY * 2) { - auto thash = get_transaction_hash(*tit); - // undo is rare, otherwise we'd optimize to use - // block.header.transaction_hashes - add_transaction(thash, *tit, height, get_tip().timestamp + m_currency.block_future_time_limit * 2, - std::numeric_limits::max(), false); - // we use increased timestamp so that just unlocked transactions will not - // be thrown out of the pool - // we set no limit on complexity to overshoot MAX_POOL_COMPLEXITY * 2 and - // stop doing work in case of large undo - } - } } bool BlockChainState::read_block_output_global_indices(const Hash &bid, BlockGlobalIndices &indices) const { @@ -1058,6 +1130,10 @@ std::vector BlockChainState::get_outputs_by_amount( void BlockChainState::store_keyimage(const KeyImage &keyimage, Height height) { auto key = KEYIMAGE_PREFIX + DB::to_binary_key(keyimage.data, sizeof(keyimage.data)); m_db.put(key, std::string(), true); + auto tit = m_memory_state_ki_tx.find(keyimage); + if (tit == m_memory_state_ki_tx.end()) + return; + remove_from_pool(tit->second); } void BlockChainState::delete_keyimage(const KeyImage &keyimage) { @@ -1150,3 +1226,51 @@ bool BlockChainState::read_amount_output( pk = was.second; return true; } + +void BlockChainState::test_print_outputs() { + Amount previous_amount = (Amount)-1; + uint32_t next_global_index = 0; + int total_counter = 0; + std::map coins; + for (DB::Cursor cur = m_db.begin(AMOUNT_OUTPUT_PREFIX); !cur.end(); cur.next()) { + const char *be = cur.get_suffix().data(); + const char *en = be + cur.get_suffix().size(); + auto amount = common::read_varint_sqlite4(be, en); + uint32_t global_index = boost::lexical_cast(common::read_varint_sqlite4(be, en)); + if (be != en) + std::cout << "Excess value bytes for amount=" << amount << " global_index=" << global_index << std::endl; + if (amount != previous_amount) { + if (previous_amount != (Amount)-1) { + if (!coins.insert(std::make_pair(previous_amount, next_global_index)).second) { + std::cout << "Duplicate amount for previous_amount=" << previous_amount + << " next_global_index=" << next_global_index << std::endl; + } + } + previous_amount = amount; + next_global_index = 0; + } + if (global_index != next_global_index) { + std::cout << "Bad output index for amount=" << amount << " global_index=" << global_index << std::endl; + } + next_global_index += 1; + if (++total_counter % 2000000 == 0) + std::cout << "Working on amount=" << amount << " global_index=" << global_index << std::endl; + } + total_counter = 0; + std::cout << "Total coins=" << total_counter << " total stacks=" << coins.size() << std::endl; + for (auto &&co : coins) { + auto total_count = next_global_index_for_amount(co.first); + if (total_count != co.second) + std::cout << "Wrong next_global_index_for_amount amount=" << co.first << " total_count=" << total_count + << " should be " << co.second << std::endl; + for (uint32_t i = 0; i != total_count; ++i) { + api::Output item; + item.amount = co.first; + item.global_index = i; + if (!read_amount_output(co.first, i, item.unlock_time, item.public_key)) + std::cout << "Failed to read amount=" << co.first << " global_index=" << i << std::endl; + if (++total_counter % 1000000 == 0) + std::cout << "Working on amount=" << co.first << " global_index=" << i << std::endl; + } + } +} diff --git a/src/Core/BlockChainState.hpp b/src/Core/BlockChainState.hpp index 0f11a71b..e8ded031 100644 --- a/src/Core/BlockChainState.hpp +++ b/src/Core/BlockChainState.hpp @@ -73,10 +73,22 @@ class BlockChainState : public BlockChain, private IBlockChainState { typedef std::vector> BlockGlobalIndices; bool read_block_output_global_indices(const Hash &bid, BlockGlobalIndices &) const; - BroadcastAction add_transaction(const Transaction &, Timestamp now); + Amount minimum_pool_fee_per_byte(Hash &minimal_tid) const; + AddTransactionResult add_transaction( + const Hash &tid, const Transaction &, const BinaryArray &binary_tx, Timestamp now); + uint32_t get_tx_pool_version() const { return m_tx_pool_version; } + struct PoolTransaction { + Transaction tx; + BinaryArray binary_tx; + Amount fee; + + PoolTransaction(const Transaction &tx, const BinaryArray &binary_tx, Amount fee); + Amount fee_per_byte() const { return fee / binary_tx.size(); } + }; typedef std::map TransMap; - const TransMap &get_memory_state_transactions() const { return m_memory_state_tx; } + typedef std::map PoolTransMap; + const PoolTransMap &get_memory_state_transactions() const { return m_memory_state_tx; } bool create_mining_block_template( BlockTemplate &, const AccountPublicAddress &, const BinaryArray &extra_nonce, Difficulty &, Height &) const; @@ -85,10 +97,10 @@ class BlockChainState : public BlockChain, private IBlockChainState { static api::BlockHeader fill_genesis(Hash genesis_bid, const BlockTemplate &); + void test_print_outputs(); + protected: - std::string get_standalone_consensus_error( - const PreparedBlock &pb, api::BlockHeader &info, const api::BlockHeader &prev_info) const; - virtual bool check_standalone_consensus( + virtual std::string check_standalone_consensus( const PreparedBlock &pb, api::BlockHeader &info, const api::BlockHeader &prev_info) const override; virtual bool redo_block(const Hash &bhash, const Block &, const api::BlockHeader &) override; virtual void undo_block(const Hash &bhash, const Block &, Height) override; @@ -131,13 +143,13 @@ class BlockChainState : public BlockChain, private IBlockChainState { virtual uint32_t next_global_index_for_amount(Amount) const override; virtual bool read_amount_output(Amount, uint32_t global_index, UnlockMoment &, PublicKey &) const override; - std::string redo_transaction_get_error(bool generating, const Transaction &, DeltaState *, BlockGlobalIndices &, - bool check_sigs, Amount &fee, bool &fatal) const; + std::string redo_transaction_get_error( + bool generating, const Transaction &, DeltaState *, BlockGlobalIndices &, bool check_sigs) const; bool redo_block(const Block &, const api::BlockHeader &, DeltaState *, BlockGlobalIndices &) const; void undo_transaction(IBlockChainState *delta_state, Height, const Transaction &); - const Config &m_config; + // const Config &m_config; const Currency &m_currency; logging::LoggerRef m_log; mutable crypto::CryptoNightContext m_hash_crypto_context; @@ -146,17 +158,18 @@ class BlockChainState : public BlockChain, private IBlockChainState { void update_first_seen_timestamp(const Hash &tid, Timestamp now); // 0 to delete - BroadcastAction add_transaction(const Hash &tid, const Transaction &tx, Height unlock_height, - Timestamp unlock_timestamp, size_t max_pool_complexity, bool check_sigs); + AddTransactionResult add_transaction(const Hash &tid, const Transaction &tx, const BinaryArray &binary_tx, + Height unlock_height, Timestamp unlock_timestamp, bool check_sigs); void remove_from_pool(Hash tid); uint32_t m_tx_pool_version = 2; // Incremented every time pool changes, reset to 2 on redo block. 2 is selected // because wallet resets to 1, so after both reset pool versions do not equal - TransMap m_memory_state_tx; + PoolTransMap m_memory_state_tx; std::map m_memory_state_ki_tx; std::map> m_memory_state_fee_tx; - size_t m_memory_state_total_complexity; - mutable std::map> m_mining_transactions; + size_t m_memory_state_total_size = 0; + + mutable std::map> m_mining_transactions; // We remember them for several blocks void clear_mining_transactions() const; @@ -164,8 +177,10 @@ class BlockChainState : public BlockChain, private IBlockChainState { Timestamp m_next_unlock_timestamp = 0; uint32_t m_next_median_size = 0; virtual void tip_changed() override; // Updates values above - void calculate_consensus_values(const api::BlockHeader & prev_info, uint32_t &next_median_size, Timestamp &next_median_timestamp, - Timestamp &next_unlock_timestamp) const; + virtual void on_reorganization( + const std::map> &undone_transactions, bool undone_blocks) override; + void calculate_consensus_values(const api::BlockHeader &prev_info, uint32_t &next_median_size, + Timestamp &next_median_timestamp, Timestamp &next_unlock_timestamp) const; RingCheckerMulticore ring_checker; std::chrono::steady_clock::time_point log_redo_block_timestamp; diff --git a/src/Core/DifficultyCheck.cpp b/src/Core/DifficultyCheck.cpp new file mode 100644 index 00000000..989ac87d --- /dev/null +++ b/src/Core/DifficultyCheck.cpp @@ -0,0 +1,8 @@ +#include "DifficultyCheck.hpp" + +static const bytecoin::DifficultyCheck table[1] = { + {UINT32_MAX, "", 0}, + }; + +const bytecoin::DifficultyCheck *bytecoin::difficulty_check = table; +const size_t bytecoin::difficulty_check_count = sizeof(table) / sizeof(*table); diff --git a/src/Core/DifficultyCheck.hpp b/src/Core/DifficultyCheck.hpp new file mode 100644 index 00000000..85e30cc9 --- /dev/null +++ b/src/Core/DifficultyCheck.hpp @@ -0,0 +1,14 @@ +#pragma once + +#include "CryptoNote.hpp" + +namespace bytecoin { +struct DifficultyCheck { + Height height; + const char *hash; + Difficulty cumulative_difficulty; +}; + +extern const DifficultyCheck *difficulty_check; +extern const size_t difficulty_check_count; +} diff --git a/src/Core/Node.cpp b/src/Core/Node.cpp index 22870e21..c7754c55 100644 --- a/src/Core/Node.cpp +++ b/src/Core/Node.cpp @@ -180,9 +180,11 @@ void Node::P2PClientBytecoin::on_msg_notify_request_tx_pool(NOTIFY_REQUEST_TX_PO auto it = std::lower_bound(req.txs.begin(), req.txs.end(), tx.first); if (it != req.txs.end() && *it == tx.first) continue; - BinaryArray raw_tx = seria::to_binary(tx.second); - msg.txs.push_back(std::move(raw_tx)); + msg.txs.push_back(tx.second.binary_tx); } + m_node->m_log(logging::TRACE) << "on_msg_notify_request_tx_pool from " << get_address() + << " peer sent=" << req.txs.size() << " we are relaying=" << msg.txs.size() + << std::endl; if (msg.txs.empty()) return; BinaryArray raw_msg = LevinProtocol::send_message(NOTIFY_NEW_TRANSACTIONS::ID, LevinProtocol::encode(msg), false); @@ -222,6 +224,7 @@ void Node::P2PClientBytecoin::on_msg_notify_new_transactions(NOTIFY_NEW_TRANSACT m_node->m_block_chain.get_tip_height() < m_node->m_block_chain.internal_import_known_height()) return; // We cannot check tx while downloading anyway NOTIFY_NEW_TRANSACTIONS::request msg; + Hash any_tid; for (auto &&raw_tx : req.txs) { Transaction tx; try { @@ -230,18 +233,27 @@ void Node::P2PClientBytecoin::on_msg_notify_new_transactions(NOTIFY_NEW_TRANSACT disconnect("NOTIFY_NEW_TRANSACTIONS add_transaction BAN from_binary failed " + std::string(ex.what())); return; } - auto action = m_node->m_block_chain.add_transaction(tx, m_node->m_p2p.get_local_time()); + const Hash tid = get_transaction_hash(tx); + any_tid = tid; + auto action = m_node->m_block_chain.add_transaction(tid, tx, raw_tx, m_node->m_p2p.get_local_time()); switch (action) { - case BroadcastAction::BAN: + case AddTransactionResult::BAN: disconnect("NOTIFY_NEW_TRANSACTIONS add_transaction BAN"); return; - case BroadcastAction::BROADCAST_ALL: + case AddTransactionResult::BROADCAST_ALL: msg.txs.push_back(raw_tx); break; - case BroadcastAction::NOTHING: + case AddTransactionResult::ALREADY_IN_POOL: + case AddTransactionResult::INCREASE_FEE: + case AddTransactionResult::FAILED_TO_REDO: + case AddTransactionResult::OUTPUT_ALREADY_SPENT: break; } } + m_node->m_log(logging::TRACE) << "on_msg_notify_new_transactions from " << get_address() + << " got=" << req.txs.size() << " relaying=" << msg.txs.size() + << (req.txs.size() > 1 ? " notify_tx_reply (?) " : " ") + << (any_tid == Hash{} ? "" : common::pod_to_hex(any_tid)) << std::endl; if (msg.txs.empty()) return; BinaryArray raw_msg = LevinProtocol::send_message(NOTIFY_NEW_TRANSACTIONS::ID, LevinProtocol::encode(msg), false); @@ -458,6 +470,7 @@ const std::unordered_map Node::m_json {api::bytecoind::SendTransaction::method(), json_rpc::make_member_method(&Node::handle_send_transaction3)}, {api::bytecoind::CheckSendProof::method(), json_rpc::make_member_method(&Node::handle_check_send_proof3)}, {api::bytecoind::SyncBlocks::method(), json_rpc::make_member_method(&Node::on_wallet_sync3)}, + {api::bytecoind::GetRawTransaction::method(), json_rpc::make_member_method(&Node::on_get_raw_transaction3)}, {api::bytecoind::SyncMemPool::method(), json_rpc::make_member_method(&Node::on_sync_mempool3)}}; bool Node::on_get_random_outputs3(http::Client *, http::RequestData &&, json_rpc::Request &&, @@ -516,19 +529,14 @@ bool Node::on_get_status3(http::Client *who, http::RequestData &&raw_request, js bool Node::on_wallet_sync3(http::Client *, http::RequestData &&, json_rpc::Request &&json_req, api::bytecoind::SyncBlocks::Request &&req, api::bytecoind::SyncBlocks::Response &res) { - if (req.sparse_chain.empty()) { - // res.status = "Empty sparse chain"; - return true; - } - - if (req.sparse_chain.back() != m_block_chain.get_genesis_bid()) { - // res.status = "Different currency"; - return true; - } - if (req.max_count > api::bytecoind::SyncBlocks::Request::MAX_COUNT) { - // res.status = "max_count too big"; - return true; - } + if (req.sparse_chain.empty()) + throw std::runtime_error("Empty sparse chain - must include at least genesis block"); + if (req.sparse_chain.back() != m_block_chain.get_genesis_bid()) + throw std::runtime_error( + "Wrong currency - different genesis block. Must be " + common::pod_to_hex(m_block_chain.get_genesis_bid())); + if (req.max_count > api::bytecoind::SyncBlocks::Request::MAX_COUNT) + throw std::runtime_error( + "Too big max_count - must be < " + std::to_string(api::bytecoind::SyncBlocks::Request::MAX_COUNT)); auto first_block_timestamp = req.first_block_timestamp < m_block_chain.get_currency().block_future_time_limit ? 0 : req.first_block_timestamp - m_block_chain.get_currency().block_future_time_limit; @@ -567,10 +575,10 @@ bool Node::on_wallet_sync3(http::Client *, http::RequestData &&, json_rpc::Reque if (!block.from_raw_block(rb)) throw std::logic_error("RawBlock failed to convert into block"); res.blocks[i].base_transaction_hash = get_transaction_hash(block.header.base_transaction); - res.blocks[i].bc_header = std::move(block.header); - res.blocks[i].bc_transactions.reserve(block.transactions.size()); + res.blocks[i].raw_header = std::move(block.header); + res.blocks[i].raw_transactions.reserve(block.transactions.size()); for (auto &&tx : block.transactions) - res.blocks[i].bc_transactions.push_back(std::move(tx)); + res.blocks[i].raw_transactions.push_back(std::move(tx)); if (!m_block_chain.read_block_output_global_indices(bhash, res.blocks[i].global_indices)) throw std::logic_error( "Invariant dead - bid is in chain but " @@ -590,29 +598,72 @@ bool Node::on_sync_mempool3(http::Client *, http::RequestData &&, json_rpc::Requ for (auto &&tx : pool) if (!std::binary_search(req.known_hashes.begin(), req.known_hashes.end(), tx.first)) { // res.added_binary_transactions.push_back(seria::to_binary(tx.second)); - res.added_bc_transactions.push_back(tx.second); + res.added_raw_transactions.push_back(tx.second.tx); res.added_transactions.push_back(api::Transaction{}); res.added_transactions.back().hash = tx.first; res.added_transactions.back().timestamp = m_block_chain.read_first_seen_timestamp(tx.first); + res.added_transactions.back().fee = tx.second.fee; } res.status = create_status_response3(); return true; } +bool Node::on_get_raw_transaction3(http::Client *, http::RequestData &&, json_rpc::Request &&, + api::bytecoind::GetRawTransaction::Request &&req, api::bytecoind::GetRawTransaction::Response &res) { + const auto &pool = m_block_chain.get_memory_state_transactions(); + auto tit = pool.find(req.hash); + if (tit != pool.end()) { + res.raw_transaction = static_cast(tit->second.tx); + res.transaction.hash = req.hash; + res.transaction.block_height = m_block_chain.get_tip_height() + 1; + res.transaction.timestamp = m_block_chain.read_first_seen_timestamp(req.hash); + return true; + } + Transaction tx; + Hash block_hash; + Height block_height = 0; + size_t index_in_block = 0; + if (m_block_chain.read_transaction(req.hash, tx, block_height, block_hash, index_in_block)) { + res.raw_transaction = static_cast(tx); // TODO - std::move? + res.transaction.hash = req.hash; + res.transaction.block_hash = block_hash; + res.transaction.block_height = block_height; + // TODO - add more parameters + return true; + } + return true; +} + bool Node::handle_send_transaction3(http::Client *, http::RequestData &&, json_rpc::Request &&, api::bytecoind::SendTransaction::Request &&request, api::bytecoind::SendTransaction::Response &response) { NOTIFY_NEW_TRANSACTIONS::request msg; Transaction tx; seria::from_binary(tx, request.binary_transaction); - if (m_block_chain.add_transaction(tx, m_p2p.get_local_time()) != BroadcastAction::BROADCAST_ALL) { - response.send_result = "Failed to be broadcasted"; // TODO - process error - return true; + const Hash tid = get_transaction_hash(tx); + auto action = m_block_chain.add_transaction(tid, tx, request.binary_transaction, m_p2p.get_local_time()); + switch (action) { + case AddTransactionResult::BAN: + throw json_rpc::Error(-101, "Binary transaction format is wrong"); + case AddTransactionResult::BROADCAST_ALL: { + msg.txs.push_back(request.binary_transaction); + BinaryArray raw_msg = + LevinProtocol::send_message(NOTIFY_NEW_TRANSACTIONS::ID, LevinProtocol::encode(msg), false); + m_p2p.broadcast(nullptr, raw_msg); + response.send_result = "broadcast"; // Success + advance_long_poll(); + break; + } + case AddTransactionResult::ALREADY_IN_POOL: + response.send_result = "broadcast"; // Was broadcasted already + break; + case AddTransactionResult::INCREASE_FEE: + response.send_result = "increasefee"; + break; + case AddTransactionResult::FAILED_TO_REDO: + throw json_rpc::Error(-102, "Transaction references outputs changed during reorganization or signature wrong"); + case AddTransactionResult::OUTPUT_ALREADY_SPENT: + throw json_rpc::Error(-103, "One of referenced outputs is already spent"); } - msg.txs.push_back(request.binary_transaction); - BinaryArray raw_msg = LevinProtocol::send_message(NOTIFY_NEW_TRANSACTIONS::ID, LevinProtocol::encode(msg), false); - m_p2p.broadcast(nullptr, raw_msg); - response.send_result = "broadcast"; // Success - advance_long_poll(); return true; } @@ -623,21 +674,19 @@ bool Node::handle_check_send_proof3(http::Client *, http::RequestData &&, json_r try { seria::from_json_value(sp, common::JsonValue::from_string(request.send_proof)); } catch (const std::exception &ex) { - response.validation_error = "Failed to parse proof object ex.what=" + std::string(ex.what()); - return true; + throw json_rpc::Error(-201, "Failed to parse proof object ex.what=" + std::string(ex.what())); } - Height height = 0; + Height height = 0; + Hash block_hash; size_t index_in_block = 0; - if (!m_block_chain.read_transaction(sp.transaction_hash, tx, height, index_in_block)) { - response.validation_error = "transaction is not in main chain"; - return true; + if (!m_block_chain.read_transaction(sp.transaction_hash, tx, height, block_hash, index_in_block)) { + throw json_rpc::Error(-202, "Transaction is not in main chain"); } PublicKey tx_public_key = get_transaction_public_key_from_extra(tx.extra); Hash message_hash = crypto::cn_fast_hash(sp.message.data(), sp.message.size()); if (!crypto::check_send_proof( tx_public_key, sp.address.view_public_key, sp.derivation, message_hash, sp.signature)) { - response.validation_error = "proof object does not match transaction, address or message"; - return true; + throw json_rpc::Error(-203, "Proof object does not match transaction or was tampered with"); } Amount total_amount = 0; size_t key_index = 0; @@ -655,9 +704,8 @@ bool Node::handle_check_send_proof3(http::Client *, http::RequestData &&, json_r ++out_index; } if (total_amount == 0) - response.validation_error = "no outputs found in transaction for the address being proofed"; - else if (total_amount != sp.amount) - response.validation_error = "wrong amount in outputs, actual amount is " + std::to_string(total_amount); - // here proof is checked, validation_error is empty + throw json_rpc::Error(-204, "No outputs found in transaction for the address being proofed"); + if (total_amount != sp.amount) + throw json_rpc::Error(-205, "Wrong amount in outputs, actual amount is " + std::to_string(total_amount)); return true; } diff --git a/src/Core/Node.hpp b/src/Core/Node.hpp index 6af915aa..a9b13e75 100644 --- a/src/Core/Node.hpp +++ b/src/Core/Node.hpp @@ -43,6 +43,8 @@ class Node { api::bytecoind::SyncBlocks::Request &&, api::bytecoind::SyncBlocks::Response &); bool on_sync_mempool3(http::Client *, http::RequestData &&, json_rpc::Request &&, api::bytecoind::SyncMemPool::Request &&, api::bytecoind::SyncMemPool::Response &); + bool on_get_raw_transaction3(http::Client *, http::RequestData &&, json_rpc::Request &&, + api::bytecoind::GetRawTransaction::Request &&, api::bytecoind::GetRawTransaction::Response &); api::bytecoind::GetStatus::Response create_status_response3() const; // json_rpc_node @@ -203,7 +205,36 @@ class Node { void on_msg_notify_request_objects(P2PClientBytecoin *, const NOTIFY_RESPONSE_GET_OBJECTS::request &); void on_msg_timed_sync(const CORE_SYNC_DATA &payload_data); }; - + /* class DownloaderV1 { // sync&download from legacy v1 clients + Node *const m_node; + BlockChainState &m_block_chain; + + size_t m_ask_blocks_count; + std::map m_good_clients; + P2PClientBytecoin *m_sync_client = nullptr; + platform::Timer m_sync_timer; // If sync_client does not respond for long, disconnect it + bool m_sync_sent = false; + + std::deque m_wanted_blocks; // If it is not empty, we are downloading first part + Height m_wanted_start_height = 0; + + void reset_sync_client(); + void on_sync_timer(); + + public: + DownloaderV1(Node *node, BlockChainState &block_chain); + + void advance_download(Hash last_downloaded_block); + bool on_idle() { return false; } + + uint32_t get_known_block_count(uint32_t my) const; + void on_connect(P2PClientBytecoin *); + void on_disconnect(P2PClientBytecoin *); + const std::map &get_good_clients() const { return m_good_clients; } + void on_msg_notify_request_chain(P2PClientBytecoin *, const NOTIFY_RESPONSE_CHAIN_ENTRY::request &); + void on_msg_notify_request_objects(P2PClientBytecoin *, const NOTIFY_RESPONSE_GET_OBJECTS::request &); + void on_msg_timed_sync(const CORE_SYNC_DATA &payload_data); + };*/ DownloaderV11 m_downloader; bool on_api_http_request(http::Client *, http::RequestData &&, http::ResponseData &); diff --git a/src/Core/NodeDownloader.cpp b/src/Core/NodeDownloader.cpp index 7aac0c20..18f998e8 100644 --- a/src/Core/NodeDownloader.cpp +++ b/src/Core/NodeDownloader.cpp @@ -82,9 +82,11 @@ void Node::DownloaderV11::on_connect(P2PClientBytecoin *who) { m_node->m_log(logging::TRACE) << "DownloaderV11::on_connect " << who->get_address() << std::endl; if (who->get_version() == 1) { m_good_clients.insert(std::make_pair(who, 0)); - if (who->get_last_received_sync_data().top_id == m_block_chain.get_tip_bid()) { - m_node->m_log(logging::TRACE) << "DownloaderV11::on_connect sync_transactions to " << who->get_address() << std::endl; - who->get_node()->sync_transactions(who); + if (who->get_last_received_sync_data().current_height == m_block_chain.get_tip_height()) { + m_node->m_log(logging::TRACE) + << "DownloaderV11::on_connect sync_transactions to " << who->get_address() + << " our pool size=" << m_node->m_block_chain.get_memory_state_transactions().size() << std::endl; + m_node->sync_transactions(who); // If we at same height, sync tx now, otherwise will sync after we reach same height } advance_download(Hash{}); @@ -130,8 +132,9 @@ void Node::DownloaderV11::on_msg_notify_request_chain(P2PClientBytecoin *who, return; // TODO - who just sent us chain we did not ask, ban std::cout << "Received chain from " << who->get_address() << " start_height=" << req.start_height << " length=" << req.m_block_ids.size() << std::endl; - m_node->m_log(logging::TRACE) << "DownloaderV11::on_msg_notify_request_chain from " << who->get_address() << " start_height=" << req.start_height - << " length=" << req.m_block_ids.size() << std::endl; + m_node->m_log(logging::TRACE) << "DownloaderV11::on_msg_notify_request_chain from " << who->get_address() + << " start_height=" << req.start_height << " length=" << req.m_block_ids.size() + << std::endl; m_chain_start_height = req.start_height; chain_source = m_chain_client->get_address(); m_chain.assign(req.m_block_ids.begin(), req.m_block_ids.end()); @@ -156,22 +159,25 @@ void Node::DownloaderV11::on_msg_notify_request_chain(P2PClientBytecoin *who, << " remote height=" << m_chain_client->get_last_received_sync_data().current_height << " our height=" << m_block_chain.get_tip_height() << " jumping from " << common::pod_to_hex(last_downloaded_block) << std::endl; - m_node->m_log(logging::TRACE) << "DownloaderV11::on_msg_notify_request_chain requesting more chain from " << m_chain_client->get_address() - << " remote height=" << m_chain_client->get_last_received_sync_data().current_height - << " our height=" << m_block_chain.get_tip_height() << " jumping from " - << common::pod_to_hex(last_downloaded_block) << std::endl; + m_node->m_log(logging::TRACE) << "DownloaderV11::on_msg_notify_request_chain requesting more chain from " + << m_chain_client->get_address() << " remote height=" + << m_chain_client->get_last_received_sync_data().current_height + << " our height=" << m_block_chain.get_tip_height() << " jumping from " + << common::pod_to_hex(last_downloaded_block) << std::endl; BinaryArray raw_msg = LevinProtocol::send_message(NOTIFY_REQUEST_CHAIN::ID, LevinProtocol::encode(msg), false); m_chain_client->send(std::move(raw_msg)); m_chain_timer.once(SYNC_TIMEOUT); return; } - if (req.m_block_ids.size() != m_chain.size() + 1){ + if (req.m_block_ids.size() != m_chain.size() + 1) { std::cout << " truncated chain length=" << m_chain.size() << std::endl; - m_node->m_log(logging::TRACE) << "DownloaderV11::on_msg_notify_request_chain truncated chain length=" << m_chain.size() << std::endl; + m_node->m_log(logging::TRACE) << "DownloaderV11::on_msg_notify_request_chain truncated chain length=" + << m_chain.size() << std::endl; } m_chain_client = nullptr; m_chain_timer.cancel(); - m_node->m_log(logging::TRACE) << "DownloaderV11::on_msg_notify_request_chain m_chain_client reset to 0" << std::endl; + m_node->m_log(logging::TRACE) << "DownloaderV11::on_msg_notify_request_chain m_chain_client reset to 0" + << std::endl; advance_download(Hash{}); } @@ -192,33 +198,31 @@ void Node::DownloaderV11::advance_chain() { std::sort(sorted_clients.begin(), sorted_clients.end(), [](P2PClientBytecoin *a, P2PClientBytecoin *b) -> bool { return a->get_last_received_sync_data().current_height < b->get_last_received_sync_data().current_height; }); - if (!lagging_clients.empty()) { + if (!sorted_clients.empty() && + sorted_clients.back()->get_last_received_sync_data().current_height > + m_block_chain.get_tip_height() + m_download_chain.size()) { + m_chain_client = sorted_clients.back(); + NOTIFY_REQUEST_CHAIN::request msg; + msg.block_ids = m_block_chain.get_sparse_chain(); + + m_node->m_log(logging::INFO) << "DownloaderV11::advance_chain Requesting chain from " + << m_chain_client->get_address() << " remote height=" + << m_chain_client->get_last_received_sync_data().current_height + << " our height=" << m_block_chain.get_tip_height() << std::endl; + BinaryArray raw_msg = LevinProtocol::send_message(NOTIFY_REQUEST_CHAIN::ID, LevinProtocol::encode(msg), false); + m_chain_client->send(std::move(raw_msg)); + m_chain_timer.once(SYNC_TIMEOUT); + } + if (lagging_clients.size() > m_node->m_config.p2p_default_connections_count / 4) { auto who = lagging_clients.front(); m_node->m_peer_db.delay_connection_attempt(who->get_address(), now); std::cout << "Disconnecting lagging client " << who->get_address() << std::endl; - m_node->m_log(logging::TRACE) << "DownloaderV11::advance_chain disconnecting lagging client " << who->get_address() << std::endl; + m_node->m_log(logging::TRACE) << "DownloaderV11::advance_chain disconnecting lagging client " + << who->get_address() << std::endl; who->disconnect(std::string()); // Will recursively call advance_chain again return; } - if (sorted_clients.empty() || - sorted_clients.back()->get_last_received_sync_data().current_height <= - m_block_chain.get_tip_height() + m_download_chain.size()) { - return; // If m_download_chain is not empty, it will become empty soon and - // we will ask for chain again - } - m_chain_client = sorted_clients.back(); - NOTIFY_REQUEST_CHAIN::request msg; - msg.block_ids = m_block_chain.get_sparse_chain(); - - std::cout << "Requesting chain from " << m_chain_client->get_address() - << " remote height=" << m_chain_client->get_last_received_sync_data().current_height - << " our height=" << m_block_chain.get_tip_height() << std::endl; - m_node->m_log(logging::TRACE) << "DownloaderV11::advance_chain Requesting chain from " << m_chain_client->get_address() - << " remote height=" << m_chain_client->get_last_received_sync_data().current_height - << " our height=" << m_block_chain.get_tip_height() << std::endl; - BinaryArray raw_msg = LevinProtocol::send_message(NOTIFY_REQUEST_CHAIN::ID, LevinProtocol::encode(msg), false); - m_chain_client->send(std::move(raw_msg)); - m_chain_timer.once(SYNC_TIMEOUT); + // If m_download_chain is not empty, it will become empty soon and we will ask for chain again } void Node::DownloaderV11::on_msg_timed_sync(const CORE_SYNC_DATA &payload_data) { advance_download(Hash{}); } @@ -257,8 +261,10 @@ void Node::DownloaderV11::on_msg_notify_request_objects(P2PClientBytecoin *who, std::cout << "Received block with height=" << dc.expected_height << " (queue=" << total_downloading_blocks << ") from " << who->get_address() << std::endl; } - m_node->m_log(logging::TRACE) << "DownloaderV11::on_msg_notify_request_objects received block with height=" << dc.expected_height << " hash=" << common::pod_to_hex(dc.bid) - << " (queue=" << total_downloading_blocks << ") from " << who->get_address() << std::endl; + m_node->m_log(logging::TRACE) + << "DownloaderV11::on_msg_notify_request_objects received block with height=" << dc.expected_height + << " hash=" << common::pod_to_hex(dc.bid) << " (queue=" << total_downloading_blocks << ") from " + << who->get_address() << std::endl; cell_found = true; if (multicore) { dc.status = DownloadCell::PREPARING; @@ -272,7 +278,8 @@ void Node::DownloaderV11::on_msg_notify_request_objects(P2PClientBytecoin *who, } if (!cell_found) { std::cout << "Received stray block from " << who->get_address() << " banning..." << std::endl; - m_node->m_log(logging::TRACE) << "DownloaderV11::on_msg_notify_request_objects received stray block from " << who->get_address() << " banning..." << std::endl; + m_node->m_log(logging::TRACE) << "DownloaderV11::on_msg_notify_request_objects received stray block from " + << who->get_address() << " banning..." << std::endl; who->disconnect(std::string()); break; } @@ -302,8 +309,8 @@ bool Node::DownloaderV11::on_idle() { if (m_block_chain.add_block(dc.pb, info) == BroadcastAction::BAN) { std::cout << "DownloadCell BAN height=" << dc.expected_height << " wb=" << common::pod_to_hex(dc.bid) << std::endl; - m_node->m_log(logging::TRACE) << "DownloaderV11::on_idle DownloadCell BAN height=" << dc.expected_height << " wb=" << common::pod_to_hex(dc.bid) - << std::endl; + m_node->m_log(logging::TRACE) << "DownloaderV11::on_idle DownloadCell BAN height=" << dc.expected_height + << " wb=" << common::pod_to_hex(dc.bid) << std::endl; // TODO - ban client who gave us chain // continue; } @@ -318,10 +325,13 @@ bool Node::DownloaderV11::on_idle() { advance_download(Hash{}); if (m_download_chain.empty()) for (auto &&who : m_good_clients) { - if (who.first->get_last_received_sync_data().top_id == m_node->m_block_chain.get_tip_bid()) { - m_node->m_log(logging::TRACE) << "DownloaderV11::on_idle sync_transactions to " << who.first->get_address() << std::endl; + if (who.first->get_last_received_sync_data().current_height == m_node->m_block_chain.get_tip_height()) { + m_node->m_log(logging::TRACE) + << "DownloaderV11::on_idle sync_transactions to " << who.first->get_address() + << " our pool size=" << m_node->m_block_chain.get_memory_state_transactions().size() + << std::endl; m_node->sync_transactions(who.first); - break; + break; // TODO - sync with all nodes } } } @@ -339,7 +349,8 @@ void Node::DownloaderV11::on_download_timer() { auto who = m_download_chain.front().downloading_client; m_node->m_peer_db.delay_connection_attempt(who->get_address(), m_node->m_p2p.get_local_time()); std::cout << "Disconnecting protected slacker " << who->get_address() << std::endl; - m_node->m_log(logging::TRACE) << "DownloaderV11::on_download_timer disconnecting protected slacker " << who->get_address() << std::endl; + m_node->m_log(logging::TRACE) << "DownloaderV11::on_download_timer disconnecting protected slacker " + << who->get_address() << std::endl; who->disconnect(std::string()); } } @@ -394,25 +405,26 @@ void Node::DownloaderV11::advance_download(Hash last_downloaded_block) { total_downloading_blocks += 1; NOTIFY_REQUEST_GET_OBJECTS::request msg; msg.blocks.push_back(dc.bid); -// auto now = std::chrono::steady_clock::now(); + // auto now = std::chrono::steady_clock::now(); if (std::chrono::duration_cast(idea_now - log_request_timestamp).count() > 1000) { log_request_timestamp = idea_now; std::cout << "Requesting block " << dc.expected_height << " from " << ready_client->get_address() << std::endl; } - m_node->m_log(logging::TRACE) << "DownloaderV11::advance_download requesting block " << dc.expected_height << " hash=" << common::pod_to_hex(dc.bid) << " from " << ready_client->get_address() - << std::endl; + m_node->m_log(logging::TRACE) << "DownloaderV11::advance_download requesting block " << dc.expected_height + << " hash=" << common::pod_to_hex(dc.bid) << " from " + << ready_client->get_address() << std::endl; BinaryArray raw_msg = LevinProtocol::send_message(NOTIFY_REQUEST_GET_OBJECTS::ID, LevinProtocol::encode(msg), false); ready_client->send(std::move(raw_msg)); } - const bool bad_timeout = !m_download_chain.empty() && m_download_chain.front().status == DownloadCell::DOWNLOADING && + const bool bad_timeout = + !m_download_chain.empty() && m_download_chain.front().status == DownloadCell::DOWNLOADING && m_download_chain.front().downloading_client && !m_download_chain.front().protect_from_disconnect && std::chrono::duration_cast(idea_now - m_download_chain.front().request_time).count() > 2 * SYNC_TIMEOUT; - if( bad_timeout ) - std::cout << "Aha" << std::endl; - const bool bad_relatively_slow = total_downloading_blocks < TOTAL_DOWNLOAD_BLOCKS && m_download_chain.size() >= TOTAL_DOWNLOAD_WINDOW && + const bool bad_relatively_slow = + total_downloading_blocks < TOTAL_DOWNLOAD_BLOCKS && m_download_chain.size() >= TOTAL_DOWNLOAD_WINDOW && m_good_clients.size() > 1 && m_download_chain.front().status == DownloadCell::DOWNLOADING && m_download_chain.front().downloading_client && !m_download_chain.front().protect_from_disconnect; if (bad_relatively_slow || bad_timeout) { @@ -422,7 +434,223 @@ void Node::DownloaderV11::advance_download(Hash last_downloaded_block) { dc.protect_from_disconnect = true; m_node->m_peer_db.delay_connection_attempt(who->get_address(), m_node->m_p2p.get_local_time()); std::cout << "Disconnecting slacker " << who->get_address() << std::endl; - m_node->m_log(logging::TRACE) << "DownloaderV11::advance_download disconnecting slacker " << who->get_address() << std::endl; + m_node->m_log(logging::TRACE) << "DownloaderV11::advance_download disconnecting slacker " << who->get_address() + << std::endl; who->disconnect(std::string()); } } + +/*Node::DownloaderV1::DownloaderV1(Node *node, BlockChainState &block_chain) + : m_node(node) + , m_block_chain(block_chain) + , m_ask_blocks_count(node->m_config.p2p_blocks_sync_default_count) + , m_sync_timer(std::bind(&DownloaderV1::on_sync_timer, this)) {} + + uint32_t Node::DownloaderV1::get_known_block_count(uint32_t my) const { + for (auto &&gc : m_good_clients) + my = std::max(my, gc.first->get_last_received_sync_data().current_height); + return my; + } + + void Node::DownloaderV1::reset_sync_client() { + m_sync_timer.cancel(); + + m_sync_client = nullptr; + m_sync_sent = false; + m_wanted_start_height = 0; + m_wanted_blocks.clear(); + advance_download(Hash{}); +} + +void Node::DownloaderV1::on_sync_timer() { + if (m_sync_client) { + m_sync_sent = false; + m_ask_blocks_count /= 2; + m_ask_blocks_count += 5; // TODO - constant + m_node->m_log(logging::INFO) << "Adjusted ask_blocks_count=" << m_ask_blocks_count << std::endl; + m_sync_client->disconnect(std::string()); + } +} + +void Node::DownloaderV1::on_connect(P2PClientBytecoin *who) { + if (who->is_incoming()) // Never sync headers from incoming + return; + if (who->get_version() == 1) { + m_good_clients.insert(std::make_pair(who, 0)); + if (who->get_last_received_sync_data().current_height == m_block_chain.get_tip_height()) { + who->get_node()->sync_transactions(who); // If we at same height, sync tx + // now, otherwise will sync + // after we + // reach same height in advance_download + } + advance_download(Hash{}); + } +} + +void Node::DownloaderV1::on_disconnect(P2PClientBytecoin *who) { + if (who->is_incoming()) + return; + m_good_clients.erase(who); + if (m_sync_client == who) { + reset_sync_client(); + } +} + +void Node::DownloaderV1::advance_download(Hash last_downloaded_block) { + if (m_node->m_block_chain_reader1 || m_node->m_block_chain_reader2 || + m_block_chain.get_tip_height() < m_block_chain.internal_import_known_height()) + return; + if (m_sync_sent) + return; + if (!m_sync_client) { // start sync with some client who has better chain + P2PClientBytecoin *worst_client = nullptr; + for (auto &&who : m_good_clients) { + if (!worst_client || + who.first->get_last_received_sync_data().current_height < + worst_client->get_last_received_sync_data().current_height) + worst_client = who.first; + if (who.first->get_last_received_sync_data().current_height > m_block_chain.get_tip_height()) { + m_sync_client = who.first; + NOTIFY_REQUEST_CHAIN::request msg; + msg.block_ids = m_block_chain.get_sparse_chain(); + + m_node->m_log(logging::INFO) << "Sending initial NOTIFY_REQUEST_CHAIN with lsd.current_height=" + << m_sync_client->get_last_received_sync_data().current_height + << " lsd.bid=" << common::pod_to_hex(m_sync_client->get_last_received_sync_data().top_id) + << std::endl; + m_node->m_log(logging::INFO) << " block_chain.get_tip_height=" << m_block_chain.get_tip_height() + << " bid=" << common::pod_to_hex(m_block_chain.get_tip_bid()) << std::endl; + m_node->m_log(logging::INFO) << " last_downloaded_block=" << +common::pod_to_hex(last_downloaded_block) << std::endl; + BinaryArray raw_msg = + LevinProtocol::send_message(NOTIFY_REQUEST_CHAIN::ID, LevinProtocol::encode(msg), false); + m_sync_client->send(std::move(raw_msg)); + m_sync_sent = true; + m_sync_timer.once(SYNC_TIMEOUT); + return; + } + } + auto now = m_node->m_p2p.get_local_time(); + if (worst_client && + m_block_chain.get_tip().timestamp + m_block_chain.get_currency().block_future_time_limit < now && + m_good_clients.size() > m_node->m_config.p2p_default_connections_count / 2) { + m_node->m_log(logging::INFO) << "All outgoing peers are far behind, disconnecting one" << std::endl; + m_node->m_peer_db.delay_connection_attempt(worst_client->get_address(), now); + worst_client->disconnect(std::string()); + } + return; + } + if (m_wanted_blocks.empty()) { + if (m_sync_client->get_last_received_sync_data().current_height > + m_block_chain.get_tip_height()) { // Continue download from sync_client + NOTIFY_REQUEST_CHAIN::request msg; + msg.block_ids.push_back(last_downloaded_block); + msg.block_ids.push_back(m_block_chain.get_genesis_bid()); // So we are not + // banned for + // not following + // protocol + + m_node->m_log(logging::INFO) << "Sending NOTIFY_REQUEST_CHAIN with lsd.current_height=" + << m_sync_client->get_last_received_sync_data().current_height + << " lsd.bid=" << common::pod_to_hex(m_sync_client->get_last_received_sync_data().top_id) + << std::endl; + m_node->m_log(logging::INFO) << " block_chain.get_tip_height=" << m_block_chain.get_tip_height() + << " bid=" << common::pod_to_hex(m_block_chain.get_tip_bid()) << std::endl; + m_node->m_log(logging::INFO) << " last_downloaded_block=" << common::pod_to_hex(last_downloaded_block) << +std::endl; + BinaryArray raw_msg = + LevinProtocol::send_message(NOTIFY_REQUEST_CHAIN::ID, LevinProtocol::encode(msg), false); + m_sync_client->send(std::move(raw_msg)); + m_sync_sent = true; + m_sync_timer.once(SYNC_TIMEOUT); + return; + } +// m_block_chain.db_commit(); // finished download from this client, find +// another one with better chain + m_sync_client->get_node()->sync_transactions(m_sync_client); + m_sync_client = nullptr; + advance_download(Hash{}); + return; + } + m_node->m_log(logging::INFO) << "Sending NOTIFY_REQUEST_GET_OBJECTS with wanted_start_height=" << +m_wanted_start_height + << " lsd.current_height=" << m_sync_client->get_last_received_sync_data().current_height + << " lsd.bid=" << common::pod_to_hex(m_sync_client->get_last_received_sync_data().top_id) << std::endl; + NOTIFY_REQUEST_GET_OBJECTS::request msg; + for (auto &&bh : m_wanted_blocks) { + if (msg.blocks.size() >= m_ask_blocks_count) + break; + msg.blocks.push_back(bh); + } + BinaryArray raw_msg = + LevinProtocol::send_message(NOTIFY_REQUEST_GET_OBJECTS::ID, LevinProtocol::encode(msg), false); + m_sync_client->send(std::move(raw_msg)); + m_sync_sent = true; + m_sync_timer.once(SYNC_TIMEOUT); +} + +void Node::DownloaderV1::on_msg_notify_request_chain(P2PClientBytecoin *who, + const NOTIFY_RESPONSE_CHAIN_ENTRY::request &req) { + if (m_sync_client != who || !m_sync_sent) + return; // TODO - who just sent us chain we did not ask, ban + m_sync_sent = false; + m_sync_timer.cancel(); + if (!m_wanted_blocks.empty()) + return; // TODO - who just sent us chain we did not ask, ban + m_node->m_log(logging::INFO) << "Received NOTIFY_REQUEST_CHAIN with lsd.current_height=" + << m_sync_client->get_last_received_sync_data().current_height + << " lsd.bid=" << common::pod_to_hex(m_sync_client->get_last_received_sync_data().top_id) << std::endl; + m_wanted_start_height = req.start_height; + m_wanted_blocks.assign(req.m_block_ids.begin(), req.m_block_ids.end()); + if (!m_wanted_blocks.empty()) { + m_wanted_blocks.pop_front(); // first block is common + m_wanted_start_height += 1; + } + m_node->m_log(logging::INFO) << " wanted_start_height=" << m_wanted_start_height << std::endl; + advance_download(Hash{}); +} + +void Node::DownloaderV1::on_msg_timed_sync(const CORE_SYNC_DATA &) { advance_download(Hash{}); } + +void Node::DownloaderV1::on_msg_notify_request_objects(P2PClientBytecoin *, + const NOTIFY_RESPONSE_GET_OBJECTS::request &req) { + if (!m_sync_sent) + return; + if (!req.missed_ids.empty()) { + reset_sync_client(); + return; + } + m_sync_sent = false; + m_sync_timer.cancel(); + Hash last_downloaded_block{}; + m_node->m_log(logging::INFO) << "Received NOTIFY_REQUEST_GET_OBJECTS with wanted_start_height=" << +m_wanted_start_height << std::endl; + for (auto &&rb : req.blocks) { + RawBlock raw_block{rb.block, rb.transactions}; + PreparedBlock pb(std::move(raw_block), nullptr); + last_downloaded_block = pb.bid; + api::BlockHeader info; + if (m_block_chain.add_block(pb, info) == BroadcastAction::BAN) { + m_node->m_log(logging::INFO) << " BAN height=" << info.height << " wb=" << +common::pod_to_hex(last_downloaded_block) + << std::endl; + // TODO - ban sync_client for not following protocol + reset_sync_client(); + return; + } + m_node->m_log(logging::INFO) << " height=" << info.height << " wb=" << +common::pod_to_hex(last_downloaded_block) << std::endl; + if (m_wanted_blocks.empty() || last_downloaded_block != m_wanted_blocks.front() || + info.height != m_wanted_start_height) { + reset_sync_client(); + return; + } + m_wanted_blocks.pop_front(); + m_wanted_start_height += 1; + } + m_ask_blocks_count = + std::min(m_node->m_config.p2p_blocks_sync_default_count, (m_ask_blocks_count + 2) * 5 / 4); + m_node->m_log(logging::INFO) << "Adjusted ask_blocks_count=" << m_ask_blocks_count << std::endl; + m_node->advance_long_poll(); + advance_download(last_downloaded_block); +}*/ diff --git a/src/Core/TransactionBuilder.cpp b/src/Core/TransactionBuilder.cpp index 7b8c9c90..7172be37 100644 --- a/src/Core/TransactionBuilder.cpp +++ b/src/Core/TransactionBuilder.cpp @@ -216,14 +216,14 @@ constexpr Amount fake_large = 1000000000000000000; // optimize negative amounts // number to them constexpr size_t OPTIMIZATIONS_PER_TX = 50; constexpr size_t OPTIMIZATIONS_PER_TX_AGGRESSIVE = 200; -constexpr size_t MEDIAN_PERCENT = 25; // make tx up to 25% of block -constexpr size_t MEDIAN_PERCENT_AGGRESSIVE = 50; // make tx up to 50% of block +constexpr size_t MEDIAN_PERCENT = 5; // make tx up to 25% of block +constexpr size_t MEDIAN_PERCENT_AGGRESSIVE = 10; // make tx up to 50% of block constexpr size_t STACK_OPTIMIZATION_THRESHOLD = 20; // If any coin stack is larger, we will spend 10 coins. constexpr size_t TWO_THRESHOLD = 10; // if any of 2 coin stacks is larger, we // will use 2 coins to cover single digit // (e.g. 7 + 9 for 6) -bool UnspentSelector::select_optimal_outputs(Height block_height, Timestamp block_time, Height confirmed_height, +std::string UnspentSelector::select_optimal_outputs(Height block_height, Timestamp block_time, Height confirmed_height, size_t effective_median_size, size_t anonymity, Amount total_amount, size_t total_outputs, Amount fee_per_byte, std::string optimization_level, Amount &change) { HaveCoins have_coins; @@ -241,7 +241,7 @@ bool UnspentSelector::select_optimal_outputs(Height block_height, Timestamp bloc const size_t optimization_median = effective_median_size * optimization_median_percent / 100; while (true) { if (!select_optimal_outputs(have_coins, dust_coins, max_digits, total_amount + fee, anonymity, optimizations)) - return false; + return "NOT_ENOUGH_FUNDS"; Amount change_dust_fee = (m_used_total - total_amount - fee) % m_currency.default_dust_threshold; size_t tx_size = get_maximum_tx_size(m_inputs_count, total_outputs + 8, anonymity); // TODO - 8 is expected max change outputs @@ -253,12 +253,17 @@ bool UnspentSelector::select_optimal_outputs(Height block_height, Timestamp bloc continue; } if (tx_size > effective_median_size) - return false; // Out of size + return "TRANSACTION_DOES_NOT_FIT_IN_BLOCK"; Amount size_fee = fee_per_byte * tx_size; if (fee + change_dust_fee >= size_fee) { change = m_used_total - total_amount - fee - change_dust_fee; combine_optimized_unspents(); - return true; + std::string final_coins; + for (auto uu : m_used_unspents) + final_coins += " " + std::to_string(uu.amount); + std::cout << "Selected used_total=" << m_used_total << " for total_amount=" << total_amount + << ", final coins" << final_coins << std::endl; + return std::string(); } fee = ((size_fee - change_dust_fee + m_currency.default_dust_threshold - 1) / m_currency.default_dust_threshold) * @@ -318,7 +323,8 @@ void UnspentSelector::unoptimize_amounts(HaveCoins &have_coins, DustCoins &dust_ } void UnspentSelector::optimize_amounts(HaveCoins &have_coins, size_t max_digit, Amount total_amount) { - std::cout << "Sub optimizing am=" << fake_large + total_amount - m_used_total << std::endl; + std::cout << "Sub optimizing amount=" << fake_large + total_amount - m_used_total + << " total_amount=" << total_amount << " used_total=" << m_used_total << std::endl; Amount digit_amount = 1; for (size_t digit = 0; digit != max_digit + 1; ++digit, digit_amount *= 10) { if (m_used_total >= total_amount && digit_amount > m_used_total) // No optimization far beyond requested sum @@ -340,8 +346,8 @@ void UnspentSelector::optimize_amounts(HaveCoins &have_coins, size_t max_digit, } } if (best_weight != 0) { - std::cout << "Found pair for digit=" << digit << " am=" << am << " cc=(" << best_two_counts[0] << ", " - << best_two_counts[1] << ") w=" << best_weight << std::endl; + std::cout << "Found pair for digit=" << digit << " am=" << 10 - am << " coins=(" << best_two_counts[0] + << ", " << best_two_counts[1] << ") sum weight=" << best_weight << std::endl; for (size_t i = 0; i != 2; ++i) { auto &uns = dit->second[best_two_counts[i]]; auto &un = uns.back(); @@ -369,7 +375,8 @@ void UnspentSelector::optimize_amounts(HaveCoins &have_coins, size_t max_digit, best_single = ait.first; } if (best_single != 0) { - std::cout << "Found single for digit=" << digit << " am=" << am << " cc=" << best_single << std::endl; + std::cout << "Found single for digit=" << digit << " am=" << 10 - am << " coin=" << best_single + << " weight=" << best_weight << std::endl; auto &uns = dit->second[best_single]; auto &un = uns.back(); m_optimization_unspents.push_back(un); @@ -391,7 +398,8 @@ bool UnspentSelector::select_optimal_outputs(HaveCoins &have_coins, DustCoins &d Amount total_amount, size_t anonymity, size_t optimization_count) { // Optimize for roundness of used_total - total_amount; // [digit:size:outputs] - std::cout << "Optimizing am=" << fake_large + total_amount - m_used_total << std::endl; + std::cout << "Optimizing amount=" << fake_large + total_amount - m_used_total << " total_amount=" << total_amount + << " used_total=" << m_used_total << std::endl; if (anonymity == 0) { if (m_used_total < total_amount) { // Find smallest dust coin >= total_amount - used_total, it can be very @@ -399,7 +407,7 @@ bool UnspentSelector::select_optimal_outputs(HaveCoins &have_coins, DustCoins &d auto duit = dust_coins.lower_bound(total_amount - m_used_total); if (duit != dust_coins.end()) { auto &un = duit->second.back(); - std::cout << "Found single large dust coin am=" << un.amount << std::endl; + std::cout << "Found single large dust coin=" << un.amount << std::endl; m_optimization_unspents.push_back(un); m_used_total += un.amount; m_inputs_count += 1; @@ -412,6 +420,7 @@ bool UnspentSelector::select_optimal_outputs(HaveCoins &have_coins, DustCoins &d while (m_used_total < total_amount && !dust_coins.empty() && optimization_count >= 1) { auto duit = --dust_coins.end(); auto &un = duit->second.back(); + std::cout << "Found optimization dust coin=" << un.amount << std::endl; m_optimization_unspents.push_back(un); m_used_total += un.amount; m_inputs_count += 1; @@ -419,7 +428,6 @@ bool UnspentSelector::select_optimal_outputs(HaveCoins &have_coins, DustCoins &d duit->second.pop_back(); if (duit->second.empty()) dust_coins.erase(duit); - std::cout << "Found optimization dust coin amount=" << un.amount << std::endl; } } // Add coins from large stacks, up to optimization_count @@ -436,7 +444,7 @@ bool UnspentSelector::select_optimal_outputs(HaveCoins &have_coins, DustCoins &d break; for (int i = 0; i != 10; ++i) { auto &un = best_stack->back(); - std::cout << "Found optimization coin amount=" << un.amount << std::endl; + std::cout << "Found optimization stack for coin=" << un.amount << std::endl; m_optimization_unspents.push_back(un); m_used_total += un.amount; m_inputs_count += 1; @@ -456,7 +464,7 @@ bool UnspentSelector::select_optimal_outputs(HaveCoins &have_coins, DustCoins &d continue; for (auto ait = dit->second.begin(); ait != dit->second.end(); ++ait) if (!ait->second.empty() && ait->first * digit_amount >= total_amount - m_used_total) { - std::cout << "Found single large coin for digit=" << digit << " cc=" << ait->first << std::endl; + std::cout << "Found single large coin for digit=" << digit << " coin=" << ait->first << std::endl; auto &uns = dit->second[ait->first]; auto &un = uns.back(); m_optimization_unspents.push_back(un); @@ -494,7 +502,7 @@ bool UnspentSelector::select_optimal_outputs(HaveCoins &have_coins, DustCoins &d auto ait = --dit->second.end(); auto &uns = ait->second; auto &un = uns.back(); - std::cout << "Found filler coin amount=" << un.amount << std::endl; + std::cout << "Found filler coin=" << un.amount << std::endl; m_optimization_unspents.push_back(un); m_used_total += un.amount; m_inputs_count += 1; @@ -506,7 +514,7 @@ bool UnspentSelector::select_optimal_outputs(HaveCoins &have_coins, DustCoins &d } else { auto duit = --dust_coins.end(); auto &un = duit->second.back(); - std::cout << "Found filler dust coin amount=" << un.amount << std::endl; + std::cout << "Found filler dust coin=" << un.amount << std::endl; m_optimization_unspents.push_back(un); m_used_total += un.amount; m_inputs_count += 1; diff --git a/src/Core/TransactionBuilder.hpp b/src/Core/TransactionBuilder.hpp index 99f8d57e..f0fca740 100644 --- a/src/Core/TransactionBuilder.hpp +++ b/src/Core/TransactionBuilder.hpp @@ -87,7 +87,7 @@ class UnspentSelector { const std::unordered_map &wallet_records, TransactionBuilder &builder, uint32_t anonymity, api::bytecoind::GetRandomOutputs::Response &&ra_response); - bool select_optimal_outputs(Height block_height, Timestamp block_time, Height confirmed_height, + std::string select_optimal_outputs(Height block_height, Timestamp block_time, Height confirmed_height, size_t effective_median_size, size_t anonymity, Amount total_amount, size_t total_outputs, Amount fee_per_byte, std::string optimization_level, Amount &change); Amount get_used_total() const { return m_used_total; } diff --git a/src/Core/Wallet.cpp b/src/Core/Wallet.cpp index 3a1a55d9..345a4ccc 100644 --- a/src/Core/Wallet.cpp +++ b/src/Core/Wallet.cpp @@ -152,17 +152,30 @@ Wallet::Wallet(const std::string &path, const std::string &password, bool create generate_new_address(SecretKey{}, m_oldest_timestamp); first_record = m_wallet_records.begin()->second; } else { - if (import_keys.size() != 128) - throw Exception(api::WALLET_FILE_DECRYPT_ERROR, "Imported keys should be exactly 128 hex bytes"); + + if (import_keys.size() != 256 || import_keys.size() != 128) + throw Exception(api::WALLET_FILE_DECRYPT_ERROR, "Imported keys should be exactly 128 or 256 hex bytes"); + WalletRecord record{}; - if (//!common::pod_from_hex(import_keys.substr(0, 64), record.spend_public_key) || - //!common::pod_from_hex(import_keys.substr(64, 64), m_view_public_key) || - !common::pod_from_hex(import_keys.substr(0, 64), record.spend_secret_key) || - !common::pod_from_hex(import_keys.substr(64, 64), m_view_secret_key)) - throw Exception(api::WALLET_FILE_DECRYPT_ERROR, "Imported keys should contain only hex bytes"); - crypto::secret_key_to_public_key(record.spend_secret_key, record.spend_public_key); - crypto::secret_key_to_public_key(m_view_secret_key, m_view_public_key); + if (import_keys.size() == 128) + { + if (!common::pod_from_hex(import_keys.substr(0, 64), record.spend_secret_key) || + !common::pod_from_hex(import_keys.substr(64, 64), m_view_secret_key)) + throw Exception(api::WALLET_FILE_DECRYPT_ERROR, "Imported keys should contain only hex bytes"); + + crypto::secret_key_to_public_key(record.spend_secret_key, record.spend_public_key); + crypto::secret_key_to_public_key(m_view_secret_key, m_view_public_key); + } + else + { + if (!common::pod_from_hex(import_keys.substr(0, 64), record.spend_public_key) || + !common::pod_from_hex(import_keys.substr(64, 64), m_view_public_key) || + !common::pod_from_hex(import_keys.substr(128, 64), record.spend_secret_key) || + !common::pod_from_hex(import_keys.substr(192, 64), m_view_secret_key)) + throw Exception(api::WALLET_FILE_DECRYPT_ERROR, "Imported keys should contain only hex bytes"); + } + if (!keys_match(m_view_secret_key, m_view_public_key)) throw Exception( api::WALLET_FILE_DECRYPT_ERROR, "Imported secret view key does not match corresponding public key"); @@ -419,8 +432,10 @@ bool Wallet::save_history(const Hash &bid, const History &used_addresses) const common::append(filename_data, std::begin(m_history_filename_seed.data), std::end(m_history_filename_seed.data)); Hash filename_hash = crypto::cn_fast_hash(filename_data.data(), filename_data.size()); - return common::save_file(history_folder + "/" + common::pod_to_hex(filename_hash) + ".txh", encrypted_data.data(), - encrypted_data.size()); + const auto tmp_path = history_folder + "/_tmp.txh"; + if( !common::save_file(tmp_path, encrypted_data.data(), encrypted_data.size()) ) + return false; + return platform::atomic_replace_file(tmp_path, history_folder + "/" + common::pod_to_hex(filename_hash) + ".txh"); } Wallet::History Wallet::load_history(const Hash &bid) const { diff --git a/src/Core/WalletNode.cpp b/src/Core/WalletNode.cpp index 04ccd58b..652936d0 100644 --- a/src/Core/WalletNode.cpp +++ b/src/Core/WalletNode.cpp @@ -268,6 +268,13 @@ bool WalletNode::handle_get_transfers3(http::Client *, http::RequestData &&, jso bool WalletNode::handle_create_transaction3(http::Client *who, http::RequestData &&raw_request, json_rpc::Request &&raw_js_request, api::walletd::CreateTransaction::Request &&request, api::walletd::CreateTransaction::Response &response) { + for (auto &&tid : request.prevent_conflict_with_transactions) { + if (m_wallet_state.api_has_transaction(tid)) + continue; + response.transactions_required.push_back(tid); + } + if (!response.transactions_required.empty()) + return true; if (request.confirmed_height_or_depth < 0) request.confirmed_height_or_depth = std::max(0, static_cast(m_wallet_state.get_tip_height()) + 1 - request.confirmed_height_or_depth); @@ -338,17 +345,20 @@ bool WalletNode::handle_create_transaction3(http::Client *who, http::RequestData if (!request.spend_addresses.empty()) for (auto &&ad : request.spend_addresses) { if (!m_wallet_state.api_add_unspent( - unspents, total_unspents, ad, request.confirmed_height_or_depth, sum_positive_transfers * 2)) + unspents, total_unspents, ad, request.confirmed_height_or_depth, sum_positive_transfers * 2)) break; // found enough funds } else - m_wallet_state.api_add_unspent( unspents, total_unspents, std::string(), request.confirmed_height_or_depth, sum_positive_transfers * 2); + m_wallet_state.api_add_unspent( + unspents, total_unspents, std::string(), request.confirmed_height_or_depth, sum_positive_transfers * 2); UnspentSelector selector(m_wallet_state.get_currency(), std::move(unspents)); // First we select just outputs with sum = 2x requires sum - if (!selector.select_optimal_outputs(m_wallet_state.get_tip_height(), m_wallet_state.get_tip().timestamp, - request.confirmed_height_or_depth, m_last_node_status.next_block_effective_median_size, - request.transaction.anonymity, sum_positive_transfers, total_outputs, request.fee_per_byte, optimization, - change)) { + if (!selector + .select_optimal_outputs(m_wallet_state.get_tip_height(), m_wallet_state.get_tip().timestamp, + request.confirmed_height_or_depth, m_last_node_status.next_block_effective_median_size, + request.transaction.anonymity, sum_positive_transfers, total_outputs, request.fee_per_byte, + optimization, change) + .empty()) { // If selected outputs do not fit in next_block_effective_median_size, we try all outputs unspents.clear(); total_unspents = 0; @@ -357,14 +367,14 @@ bool WalletNode::handle_create_transaction3(http::Client *who, http::RequestData m_wallet_state.api_add_unspent(unspents, total_unspents, ad, request.confirmed_height_or_depth); } else - m_wallet_state.api_add_unspent( unspents, total_unspents, std::string(), request.confirmed_height_or_depth); + m_wallet_state.api_add_unspent(unspents, total_unspents, std::string(), request.confirmed_height_or_depth); selector.reset(std::move(unspents)); - if (!selector.select_optimal_outputs(m_wallet_state.get_tip_height(), m_wallet_state.get_tip().timestamp, - request.confirmed_height_or_depth, m_last_node_status.next_block_effective_median_size, - request.transaction.anonymity, sum_positive_transfers, total_outputs, request.fee_per_byte, - optimization, change)) - throw json_rpc::Error( - json_rpc::INVALID_PARAMS, "Not enough funds on selected addresses with desired confirmations"); + std::string error = selector.select_optimal_outputs(m_wallet_state.get_tip_height(), + m_wallet_state.get_tip().timestamp, request.confirmed_height_or_depth, + m_last_node_status.next_block_effective_median_size, request.transaction.anonymity, sum_positive_transfers, + total_outputs, request.fee_per_byte, optimization, change); + if (!error.empty()) + throw json_rpc::Error(json_rpc::INVALID_PARAMS, "Outputs cannot be selected for transaction " + error); } // Selector ensures the change should be as "round" as possible if (change > 0) { @@ -393,8 +403,7 @@ bool WalletNode::handle_create_transaction3(http::Client *who, http::RequestData response.binary_transaction = seria::to_binary(tx); Hash transaction_hash = get_transaction_hash(tx); if (request.save_history && !m_wallet_state.get_wallet().save_history(transaction_hash, history)) { - m_log(logging::ERROR) << "Saving transaction history failed, proof of " - "sending will be unavailable for tx=" + m_log(logging::ERROR) << "Saving transaction history failed, you will need to pass list of destination addresses to generate sending proof for tx=" << common::pod_to_hex(transaction_hash) << std::endl; response.save_history_error = true; } @@ -410,9 +419,14 @@ bool WalletNode::handle_create_transaction3(http::Client *who, http::RequestData api::walletd::CreateTransaction::Request request_copy = request; // TODO ??? http::RequestData new_request = json_rpc::create_request(api::bytecoind::url(), api::bytecoind::GetRandomOutputs::method(), ra_request); + new_request.r.basic_authorization = m_config.bytecoind_authorization; add_waiting_command(who, std::move(raw_request), raw_js_request.get_id(), std::move(new_request), - [=](const WaitingClient &wc, const http::ResponseData &random_response) mutable { - m_log(logging::INFO) << "got random response" << std::endl; + [=](const WaitingClient &wc, const http::ResponseData &&random_response) mutable { + m_log(logging::INFO) << "got response to get_random_outputs, status=" << random_response.r.status + << " body starts with " << random_response.body.substr(0, 100) << std::endl; + if (random_response.r.status != 200) { + throw json_rpc::Error(json_rpc::INTERNAL_ERROR, "got error as response on get_random_outputs"); + } Transaction tx{}; api::walletd::CreateTransaction::Response last_response; Hash tx_hash{}; @@ -426,19 +440,18 @@ bool WalletNode::handle_create_transaction3(http::Client *who, http::RequestData last_response.binary_transaction = seria::to_binary(tx); tx_hash = get_transaction_hash(tx); if (request.save_history && !m_wallet_state.get_wallet().save_history(tx_hash, history)) { - m_log(logging::ERROR) << "Saving transaction history failed, proof " - "of sending will not be available for tx=" - << common::pod_to_hex(tx_hash) << std::endl; + m_log(logging::ERROR) << "Saving transaction history failed, you will need to pass list of destination addresses to generate sending proof for tx=" + << common::pod_to_hex(tx_hash) << std::endl; last_response.save_history_error = true; } - if (!m_wallet_state.parse_raw_transaction(last_response.transaction, tx, tx_hash)) { - // TODO - process error - } + if (!m_wallet_state.parse_raw_transaction(last_response.transaction, tx, tx_hash)) + throw json_rpc::Error(json_rpc::INTERNAL_ERROR, "Created trsnsaction cannot be parsed"); http::ResponseData last_http_response = json_rpc::create_response(wc.original_request, last_response, wc.original_jsonrpc_id); wc.original_who->write(std::move(last_http_response)); }, [=](const WaitingClient &wc, std::string err) mutable { + m_log(logging::INFO) << "got error to get_random_outputs from bytecoind, " << err << std::endl; http::ResponseData last_http_response = json_rpc::create_error_response( wc.original_request, json_rpc::Error(json_rpc::INTERNAL_ERROR, err), wc.original_jsonrpc_id); wc.original_who->write(std::move(last_http_response)); @@ -490,8 +503,9 @@ bool WalletNode::handle_send_transaction3(http::Client *who, http::RequestData & new_request.set_body(std::move(raw_request.body)); // We save on copying body here new_request.r.set_firstline("POST", api::bytecoind::url(), 1, 1); transient_transactions_counter += 1; + new_request.r.basic_authorization = m_config.bytecoind_authorization; add_waiting_command(who, std::move(raw_request), raw_js_request.get_id(), std::move(new_request), - [=](const WaitingClient &wc2, const http::ResponseData &send_response) mutable { + [=](const WaitingClient &wc2, const http::ResponseData &&send_response) mutable { transient_transactions_counter -= 1; try { // Manual try to prevent double decrement of transient_transactions_counter advance_sync(); diff --git a/src/Core/WalletState.cpp b/src/Core/WalletState.cpp index 2f92d5fc..9866669b 100644 --- a/src/Core/WalletState.cpp +++ b/src/Core/WalletState.cpp @@ -77,14 +77,14 @@ PreparedWalletTransaction::PreparedWalletTransaction(TransactionPrefix &&ttx, co ++out_index; } } -PreparedWalletBlock::PreparedWalletBlock(BlockTemplate &&bc_header, std::vector &&bc_transactions, +PreparedWalletBlock::PreparedWalletBlock(BlockTemplate &&bc_header, std::vector &&raw_transactions, Hash base_transaction_hash, const SecretKey &view_secret_key) : base_transaction_hash(base_transaction_hash) { header = bc_header; base_transaction = PreparedWalletTransaction(std::move(bc_header.base_transaction), view_secret_key); - transactions.reserve(bc_transactions.size()); - for (size_t tx_index = 0; tx_index != bc_transactions.size(); ++tx_index) { - transactions.emplace_back(std::move(bc_transactions.at(tx_index)), view_secret_key); + transactions.reserve(raw_transactions.size()); + for (size_t tx_index = 0; tx_index != raw_transactions.size(); ++tx_index) { + transactions.emplace_back(std::move(raw_transactions.at(tx_index)), view_secret_key); } } @@ -110,7 +110,7 @@ void WalletPreparatorMulticore::thread_run() { work.start_height += 1; work.blocks.erase(work.blocks.begin()); } - PreparedWalletBlock result(std::move(sync_block.bc_header), std::move(sync_block.bc_transactions), + PreparedWalletBlock result(std::move(sync_block.raw_header), std::move(sync_block.raw_transactions), sync_block.base_transaction_hash, view_secret_key); { std::unique_lock lock(mu); @@ -159,9 +159,9 @@ std::string to_binary_key(const T &s) { } template -void from_binary_key(const std::string & key, T & s) { +void from_binary_key(const std::string &key, T &s) { static_assert(std::is_standard_layout::value, "T must be Standard Layout"); - if( !common::pod_from_hex(key, s) ) // WalletState::DB::to_binary_key((const unsigned char *)&s, sizeof(s)); + if (!common::pod_from_hex(key, s)) // WalletState::DB::to_binary_key((const unsigned char *)&s, sizeof(s)); throw std::logic_error("from_binary_key failed for key " + key); } @@ -182,9 +182,8 @@ void WalletState::DeltaState::redo_keyimage_output( } void WalletState::DeltaState::undo_keyimage_output(const api::Output &output) { - throw std::logic_error("DeltaState::undo_keyimage_output"); // We do not call - // it on memory - // states + throw std::logic_error("DeltaState::undo_keyimage_output"); + // We do not call it on memory states } void WalletState::DeltaState::redo_height_keyimage(Height height, const KeyImage &keyimage) { @@ -199,8 +198,7 @@ void WalletState::DeltaState::undo_height_keyimage(Height height, const KeyImage } kit->second -= 1; if (kit->second < 0) - std::cout << "DeltaState::undo_height_keyimage more keyimages undone than " - "redone 2" + std::cout << "DeltaState::undo_height_keyimage more keyimages undone than redone 2" << std::endl; if (kit->second <= 0) kit = m_used_keyimages.erase(kit); @@ -227,9 +225,8 @@ void WalletState::DeltaState::undo_transaction(const Hash &tid) { auto uit = m_unspents.find(key_output.key); if (uit == m_unspents.end() || uit->second.empty()) // Actually should never be empty continue; // Not our output - uit->second.pop_back(); // We can pop wrong output, but this is not - // important - situation arises only in pool and - // only during attack + uit->second.pop_back(); + // We can pop wrong output, but this is not important - situation arises only in pool and only during attack if (uit->second.empty()) uit = m_unspents.erase(uit); } @@ -387,14 +384,14 @@ bool WalletState::sync_with_blockchain(api::bytecoind::SyncBlocks::Response &res if (m_tip_height + 1 != m_tail_height && header.previous_block_hash != m_tip.hash) return false; if (header.timestamp + m_currency.block_future_time_limit >= m_wallet.get_oldest_timestamp()) { - const auto &block_gi = resp.blocks.at(bin).global_indices; + const auto &block_gi = resp.blocks.at(bin).global_indices; PreparedWalletBlock pb = preparator.get_ready_work(m_tip_height + 1); // PreparedWalletBlock pb(std::move(resp.blocks.at(bin).block), m_wallet.get_view_secret_key()); redo_block(header, pb, block_gi, m_tip_height + 1); -// push_chain(header); -// undo_block(m_tip_height); -// pop_chain(); -// redo_block(header, pb, block_gi, m_tip_height + 1); + // push_chain(header); + // undo_block(m_tip_height); + // pop_chain(); + // redo_block(header, pb, block_gi, m_tip_height + 1); auto now = std::chrono::steady_clock::now(); if (std::chrono::duration_cast(now - log_redo_block).count() > 1000) { log_redo_block = now; @@ -418,8 +415,8 @@ bool WalletState::sync_with_blockchain(api::bytecoind::SyncMemPool::Response &re } m_memory_state.undo_transaction(tid); } - for (size_t i = 0; i != resp.added_bc_transactions.size(); ++i) { - TransactionPrefix &tx = resp.added_bc_transactions[i]; + for (size_t i = 0; i != resp.added_raw_transactions.size(); ++i) { + TransactionPrefix &tx = resp.added_raw_transactions[i]; // seria::from_binary(tx, resp.added_binary_transactions[i]); std::vector global_indices(tx.outputs.size(), 0); Hash tid = resp.added_transactions.at(i).hash; // get_transaction_hash(tx); @@ -589,12 +586,12 @@ bool WalletState::parse_raw_transaction(api::Transaction &ptx, const Transaction if (m_db.get(key, rb)) { api::Output output; seria::from_binary(output, rb); - + api::Transfer &transfer = transfer_map2[output.address]; transfer.amount -= static_cast(output.amount); transfer.ours = true; transfer.outputs.push_back(output); - }else + } else input_transfer.amount -= static_cast(in.amount); } } @@ -620,6 +617,9 @@ bool WalletState::parse_raw_transaction(api::Transaction &ptx, Amount &output_am if (pwtx.derivation == KeyDerivation{}) return false; Wallet::History history = m_wallet.load_history(tid); +// if(!history.empty()){ +// std::cout << "Found history for transaction " << common::pod_to_hex(tid) << std::endl; +// } KeyPair tx_keys; ptx.hash = tid; ptx.block_height = block_height; @@ -1216,7 +1216,22 @@ std::vector WalletState::api_get_transfers( return result; } +bool WalletState::api_has_transaction(Hash tid) const { + auto mit = m_memory_state.get_transactions().find(tid); + if (mit != m_memory_state.get_transactions().end()) + return true; + auto trkey = TRANSACTION_PREFIX + to_binary_key(tid); + BinaryArray data; + return m_db.get(trkey, data); +} + bool WalletState::api_get_transaction(Hash tid, TransactionPrefix &tx, api::Transaction &ptx) const { + auto mit = m_memory_state.get_transactions().find(tid); + if (mit != m_memory_state.get_transactions().end()) { + tx = mit->second.first; + ptx = mit->second.second; + return true; + } auto trkey = TRANSACTION_PREFIX + to_binary_key(tid); BinaryArray data; if (!m_db.get(trkey, data)) @@ -1231,13 +1246,8 @@ bool WalletState::api_get_transaction(Hash tid, TransactionPrefix &tx, api::Tran bool WalletState::api_create_proof(SendProof &sp) const { TransactionPrefix tx; api::Transaction ptx; - if (!api_get_transaction(sp.transaction_hash, tx, ptx)) { - auto mit = m_memory_state.get_transactions().find(sp.transaction_hash); - if (mit == m_memory_state.get_transactions().end()) - return false; - tx = mit->second.first; - ptx = mit->second.second; - } + if (!api_get_transaction(sp.transaction_hash, tx, ptx)) + return false; KeyPair tx_keys = TransactionBuilder::deterministic_keys_from_seed(tx, m_wallet.get_tx_derivation_seed()); if (!crypto::generate_key_derivation(sp.address.view_public_key, tx_keys.secret_key, sp.derivation)) return false; diff --git a/src/Core/WalletState.hpp b/src/Core/WalletState.hpp index 1fa5939d..ba02cc84 100644 --- a/src/Core/WalletState.hpp +++ b/src/Core/WalletState.hpp @@ -48,7 +48,7 @@ struct PreparedWalletBlock { Hash base_transaction_hash; std::vector transactions; PreparedWalletBlock() {} - PreparedWalletBlock(BlockTemplate &&bc_header, std::vector &&bc_transactions, + PreparedWalletBlock(BlockTemplate &&bc_header, std::vector &&raw_transactions, Hash base_transaction_hash, const SecretKey &view_secret_key); }; @@ -133,6 +133,7 @@ class WalletState : private IWalletState { std::vector api_get_transfers(const std::string &address, Height &from_height, Height &to_height, bool forward, uint32_t desired_tx_count = std::numeric_limits::max()) const; bool api_get_transaction(Hash tid, TransactionPrefix &tx, api::Transaction &ptx) const; + bool api_has_transaction(Hash tid) const; bool api_create_proof(SendProof &sp) const; api::Block api_get_pool_as_history(const std::string &address) const; std::map, api::Output> api_get_unlocked_outputs( @@ -190,7 +191,6 @@ class WalletState : private IWalletState { const Currency &m_currency; logging::ILogger &m_log; Wallet &m_wallet; - private: void modify_balance(const api::Output &output, int locked_op, int spendable_op); DB m_db; diff --git a/src/Core/WalletSync.cpp b/src/Core/WalletSync.cpp index 5992539f..55232195 100644 --- a/src/Core/WalletSync.cpp +++ b/src/Core/WalletSync.cpp @@ -105,15 +105,20 @@ void WalletSync::send_sync_pool() { m_sync_request = std::make_unique(m_sync_agent, std::move(req_header), [&](http::ResponseData &&response) { m_sync_request.reset(); - api::bytecoind::SyncMemPool::Response resp; - seria::from_binary(resp, response.body); - m_last_node_status = resp.status; - m_sync_error = "WRONG_BLOCKCHAIN"; - if (m_wallet_state.sync_with_blockchain(resp)) { - m_sync_error = std::string(); - advance_sync(); - } else + if (response.r.status == 200) { + m_sync_error = "WRONG_BLOCKCHAIN"; + api::bytecoind::SyncMemPool::Response resp; + seria::from_binary(resp, response.body); + m_last_node_status = resp.status; + if (m_wallet_state.sync_with_blockchain(resp)) { + m_sync_error = std::string(); + advance_sync(); + } else + m_status_timer.once(STATUS_ERROR_PERIOD); + } else { + m_sync_error = response.body; m_status_timer.once(STATUS_ERROR_PERIOD); + } m_state_changed_handler(); }, [&](std::string err) { @@ -135,15 +140,20 @@ void WalletSync::send_get_blocks() { m_sync_request = std::make_unique(m_sync_agent, std::move(req_header), [&](http::ResponseData &&response) { m_sync_request.reset(); - api::bytecoind::SyncBlocks::Response resp; - seria::from_binary(resp, response.body); - m_last_node_status = resp.status; - m_sync_error = "WRONG_BLOCKCHAIN"; - if (m_wallet_state.sync_with_blockchain(resp)) { - m_sync_error = std::string(); - advance_sync(); - } else + if (response.r.status == 200) { + m_sync_error = "WRONG_BLOCKCHAIN"; + api::bytecoind::SyncBlocks::Response resp; + seria::from_binary(resp, response.body); + m_last_node_status = resp.status; + if (m_wallet_state.sync_with_blockchain(resp)) { + m_sync_error = std::string(); + advance_sync(); + } else + m_status_timer.once(STATUS_ERROR_PERIOD); + } else { + m_sync_error = response.body; m_status_timer.once(STATUS_ERROR_PERIOD); + } m_state_changed_handler(); }, [&](std::string err) { diff --git a/src/Core/rpc_api_serialization.cpp b/src/Core/rpc_api_serialization.cpp index ce5a9776..1fcecf00 100644 --- a/src/Core/rpc_api_serialization.cpp +++ b/src/Core/rpc_api_serialization.cpp @@ -368,14 +368,13 @@ void ser_members(api::walletd::CreateTransaction::Request &v, ISeria &s) { seria_kv("fee_per_byte", v.fee_per_byte, s); seria_kv("optimization", v.optimization, s); seria_kv("save_history", v.save_history, s); - // seria_kv("send_immediately", v.send_immediately, s); + seria_kv("prevent_conflict_with_transactions", v.prevent_conflict_with_transactions, s); } void ser_members(api::walletd::CreateTransaction::Response &v, ISeria &s) { seria_kv("transaction", v.transaction, s); seria_kv("binary_transaction", v.binary_transaction, s); seria_kv("save_history_error", v.save_history_error, s); - // seria_kv("transaction_hash", v.transaction_hash, s); - // seria_kv("send_result", v.send_result, s); + seria_kv("transactions_required", v.transactions_required, s); } void ser_members(api::walletd::CreateSendProof::Request &v, ISeria &s) { seria_kv("transaction_hash", v.transaction_hash, s); @@ -409,8 +408,8 @@ void ser_members(api::bytecoind::SyncBlocks::Request &v, ISeria &s) { } void ser_members(bytecoin::api::bytecoind::SyncBlocks::SyncBlock &v, ISeria &s) { seria_kv("header", v.header, s); - seria_kv("bc_header", v.bc_header, s); - seria_kv("bc_transactions", v.bc_transactions, s); + seria_kv("raw_header", v.raw_header, s); + seria_kv("raw_transactions", v.raw_transactions, s); seria_kv("base_transaction_hash", v.base_transaction_hash, s); seria_kv("global_indices", v.global_indices, s); } @@ -419,6 +418,12 @@ void ser_members(api::bytecoind::SyncBlocks::Response &v, ISeria &s) { seria_kv("start_height", v.start_height, s); seria_kv("status", v.status, s); } +void ser_members(api::bytecoind::GetRawTransaction::Request &v, ISeria &s) { seria_kv("hash", v.hash, s); } +void ser_members(api::bytecoind::GetRawTransaction::Response &v, ISeria &s) { + seria_kv("transaction", v.transaction, s); + seria_kv("raw_transaction", v.raw_transaction, s); +} + void ser_members(api::bytecoind::SyncMemPool::Request &v, ISeria &s) { if (!s.is_input()) std::sort(v.known_hashes.begin(), v.known_hashes.end()); @@ -428,7 +433,7 @@ void ser_members(api::bytecoind::SyncMemPool::Request &v, ISeria &s) { } void ser_members(api::bytecoind::SyncMemPool::Response &v, ISeria &s) { seria_kv("removed_hashes", v.removed_hashes, s); - seria_kv("added_bc_transactions", v.added_bc_transactions, s); + seria_kv("added_raw_transactions", v.added_raw_transactions, s); seria_kv("added_transactions", v.added_transactions, s); seria_kv("status", v.status, s); } @@ -445,9 +450,9 @@ void ser_members(api::bytecoind::SendTransaction::Response &v, ISeria &s) { seri void ser_members(bytecoin::api::bytecoind::CheckSendProof::Request &v, ISeria &s) { seria_kv("send_proof", v.send_proof, s); } -void ser_members(bytecoin::api::bytecoind::CheckSendProof::Response &v, ISeria &s) { - seria_kv("validation_error", v.validation_error, s); -} +// void ser_members(bytecoin::api::bytecoind::CheckSendProof::Response &v, ISeria &s) { +// seria_kv("validation_error", v.validation_error, s); +//} /*void ser_members(bytecoin::api::walletd::GetBlock::Request &v, ISeria &s) { seria_kv("hash", v.hash, s); seria_kv("height", v.height, s); diff --git a/src/bitsum/Common.hpp b/src/bitsum/Common.hpp new file mode 100644 index 00000000..452d2f5a --- /dev/null +++ b/src/bitsum/Common.hpp @@ -0,0 +1,24 @@ +#pragma once + +namespace bitsum { + +#if defined(_MSC_VER) + #define EXPORT extern "C" __declspec(dllexport) +#elif defined(__GNUC__) + #define EXPORT extern "C" +#else + #pragma warning Unknown dynamic link import/export semantics. +#endif + +#if defined(__GNUC__) +#define ERROR_SUCCESS 0; +#endif +#define ERROR_UNKNOWN 1; +#define ERROR_NODE_NOT_INITIALIZED 2; +#define ERROR_WALLET_NOT_INITIALIZED 3; + + struct PrivateKeyPair { + unsigned char spend_key[32]; + unsigned char view_key[32]; + }; +} diff --git a/src/bitsum/CommonWrapper.cpp b/src/bitsum/CommonWrapper.cpp new file mode 100644 index 00000000..bc916b95 --- /dev/null +++ b/src/bitsum/CommonWrapper.cpp @@ -0,0 +1,34 @@ +#include "Common.hpp" +#include "crypto/crypto.hpp" +#include "CryptoNote.hpp" +#include "common/Base58.hpp" +#include "seria/BinaryInputStream.hpp" + +using namespace bytecoin; + +namespace bitsum { + + EXPORT bool CheckLib() + { + return true; + } + + EXPORT bool CheckAddress(char address[]) + { + BinaryArray data; + uint64_t prefix = 154; + AccountPublicAddress adr; + + try + { + if (!common::base58::decode_addr(address, prefix, data)) + return false; + seria::from_binary(adr, data); + return key_isvalid(adr.spend_public_key) && key_isvalid(adr.view_public_key); + } + catch (const std::exception) + { + return false; + } + } +} \ No newline at end of file diff --git a/src/bitsum/Explorer.cpp b/src/bitsum/Explorer.cpp new file mode 100644 index 00000000..d49d7c19 --- /dev/null +++ b/src/bitsum/Explorer.cpp @@ -0,0 +1,46 @@ +#include "Explorer.hpp" + +using namespace crypto; + +namespace explorer { + + Explorer::Explorer(bytecoin::BlockChainState* blockChain) + { + _blockChain = blockChain; + } + + crypto::Hash Explorer::GetBlockHash(uint32_t height) + { + crypto::Hash result; + _blockChain->read_chain(height, result); + + return result; + } + + BlockPreview Explorer::GetBlockPreview(uint32_t height) + { + crypto::Hash bid = GetBlockHash(height); + + BlockPreview result; + RawBlock rb; + api::BlockHeader bh; + + _blockChain->read_block(bid, rb); + _blockChain->read_header(bid, bh); + + result.height = height; + std::copy(std::begin(bid.data), std::end(bid.data), std::begin(result.hash)); + result.timestamp = bh.timestamp; + result.size = bh.block_size + ((uint32_t)rb.block.size() - (bh.block_size - bh.transactions_cumulative_size)); + result.tx_count = (uint32_t)rb.transactions.capacity() + 1; + + return result; + } + + + Explorer::~Explorer() + { + } +} + + diff --git a/src/bitsum/Explorer.hpp b/src/bitsum/Explorer.hpp new file mode 100644 index 00000000..19ee1d84 --- /dev/null +++ b/src/bitsum/Explorer.hpp @@ -0,0 +1,34 @@ +#pragma once +#include +#include "crypto/types.hpp" +#include "CryptoNote.hpp" +#include "Core/BlockChainState.hpp" +#include "rpc_api.hpp" + +namespace explorer { + + using namespace crypto; + using namespace bytecoin; + + typedef struct Hash { uint8_t bytes[32]; } Hash; + + struct BlockPreview { + uint8_t hash[32]; + uint32_t height; + uint32_t timestamp; + uint32_t size; + uint32_t tx_count; + }; + + class Explorer + { + bytecoin::BlockChainState* _blockChain; + public: + Explorer(bytecoin::BlockChainState* blockChain); + crypto::Hash GetBlockHash(uint32_t height); + BlockPreview GetBlockPreview(uint32_t height); + ~Explorer(); + }; + +} + diff --git a/src/bitsum/NodeWrapper.cpp b/src/bitsum/NodeWrapper.cpp new file mode 100644 index 00000000..5cbaeaa8 --- /dev/null +++ b/src/bitsum/NodeWrapper.cpp @@ -0,0 +1,84 @@ +#include "Common.hpp" +#include "common/CommandLine.hpp" +#include "common/ConsoleTools.hpp" +#include "Core/Node.hpp" +#include "Core/WalletNode.hpp" +#include "Core/Config.hpp" +#include "platform/ExclusiveLock.hpp" +#include "logging/LoggerManager.hpp" +#include "platform/Network.hpp" +#include "version.hpp" +#include "common/Base58.hpp" +#include "seria/BinaryInputStream.hpp" +#include "seria/BinaryOutputStream.hpp" + +using namespace bytecoin; +using namespace common; + +namespace bitsum { + + boost::asio::io_service node_io; + std::unique_ptr node_lock; + std::unique_ptr blockchain_state; + + EXPORT int StartNode(bool coutRedirect = true) + { + try { + if (coutRedirect) { + std::cout.rdbuf(nullptr); + } + else { + common::console::UnicodeConsoleSetup console_setup; + } + + auto idea_start = std::chrono::high_resolution_clock::now(); + + common::CommandLine cmd(1, 0); + bytecoin::Config config(cmd); + bytecoin::Currency currency(config.is_testnet); + const std::string coinFolder = config.get_data_folder(); + + // platform::ExclusiveLock coin_lock(coinFolder, "bitsumd.lock"); + node_lock = std::make_unique(coinFolder, "bitsumd.lock"); + + logging::LoggerManager logManager; + logManager.configure_default(config.get_data_folder("logs"), "bitsumd-"); + + // BlockChainState block_chain(logManager, config, currency); + // std::unique_ptr block_chain;// (logManager, config, currency); + blockchain_state = std::make_unique(logManager, config, currency); + + // boost::asio::io_service io; + platform::EventLoop run_loop(node_io); + + Node node(logManager, config, *(blockchain_state.get())); + + auto idea_ms = std::chrono::duration_cast( + std::chrono::high_resolution_clock::now() - idea_start); + std::cout << "bitsumd started seconds=" << double(idea_ms.count()) / 1000 << std::endl; + while (!node_io.stopped()) { + if (node.on_idle()) // Using it to load blockchain + node_io.poll(); + else + node_io.run_one(); + } + + return 0; + } + catch (const std::exception &) { + return 1; + } + } + + EXPORT int StopNode() + { + try { + node_io.stop(); + node_lock->~ExclusiveLock(); + return 0; + } + catch (const std::exception) { + return -1; + } + } +} \ No newline at end of file diff --git a/src/bitsum/WalletWrapper.cpp b/src/bitsum/WalletWrapper.cpp new file mode 100644 index 00000000..fe0637ae --- /dev/null +++ b/src/bitsum/WalletWrapper.cpp @@ -0,0 +1,312 @@ +#include "Common.hpp" +#include "common/CommandLine.hpp" +#include "common/ConsoleTools.hpp" +#include "Core/Node.hpp" +#include "Core/WalletNode.hpp" +#include "Core/Config.hpp" +#include "platform/ExclusiveLock.hpp" +#include "logging/LoggerManager.hpp" +#include "platform/Network.hpp" +#include "version.hpp" +#include "common/Base58.hpp" +#include "seria/BinaryInputStream.hpp" +#include "seria/BinaryOutputStream.hpp" + +using namespace bytecoin; +using namespace common; + +namespace bitsum { + + boost::asio::io_service wallet_io; + std::unique_ptr wallet_lock; + std::unique_ptr wallet; + std::unique_ptr wallet_state; + + EXPORT int StartWallet(char path[], char password[], bool coutRedirect = true) + { + if (coutRedirect) { + std::cout.rdbuf(nullptr); + } + else { + common::console::UnicodeConsoleSetup console_setup; + } + auto idea_start = std::chrono::high_resolution_clock::now(); + common::CommandLine cmd(1, 0); + bytecoin::Config config(cmd); + bytecoin::Currency currency(config.is_testnet); + + const std::string coinFolder = config.get_data_folder(); + // std::unique_ptr walletcache_lock; + //std::unique_ptr wallet; + + try { + wallet = std::make_unique(path, password, false); + wallet_lock = std::make_unique( + config.get_data_folder("wallet_cache"), wallet->get_cache_name() + ".lock"); + } + catch (const std::ios_base::failure &ex) { + std::cout << ex.what() << std::endl; + return api::WALLET_FILE_READ_ERROR; + } + catch (const platform::ExclusiveLock::FailedToLock &ex) { + std::cout << ex.what() << std::endl; + return api::WALLET_WITH_THE_SAME_VIEWKEY_IN_USE; + } + catch (const Wallet::Exception &ex) { + std::cout << ex.what() << std::endl; + return ex.return_code; + } + + logging::LoggerManager logManagerWalletNode; + logManagerWalletNode.configure_default(config.get_data_folder("logs"), "wallet-"); + + //WalletState wallet_state(*wallet, logManagerWalletNode, config, currency); + wallet_state = std::make_unique(*wallet, logManagerWalletNode, config, currency); + + // boost::asio::io_service _io; + wallet_io.reset(); + platform::EventLoop run_loop(wallet_io); + + std::unique_ptr node; + std::unique_ptr wallet_node; + try { + wallet_node = std::make_unique(nullptr, logManagerWalletNode, config, *(wallet_state.get())); + } + catch (const boost::system::system_error &ex) { + std::cout << ex.what() << std::endl; + return api::WALLETD_BIND_PORT_IN_USE; + } + + auto idea_ms = + std::chrono::duration_cast(std::chrono::high_resolution_clock::now() - idea_start); + std::cout << "wallet-rpc started seconds=" << double(idea_ms.count()) / 1000 << std::endl; + + while (!wallet_io.stopped()) { + if (node && node->on_idle()) // We load blockchain there + wallet_io.poll(); + else + wallet_io.run_one(); + } + + //std::cout << "WEX"; + return 0; + } + + EXPORT int StopWallet() + { + try { + wallet_io.stop(); + + delete wallet_state.release(); + delete wallet.release(); + delete wallet_lock.release(); + + return ERROR_SUCCESS; + } + catch (const std::exception &ex) { + std::cout << ex.what() << std::endl; + return ERROR_UNKNOWN; + } + } + + EXPORT bool CheckWalletPassword(char path[], char password[]) + { + std::unique_ptr wallet; + try { + wallet = std::make_unique(path, password, false); + return true; + } + catch (const std::exception &) { + return false; + } + } + + EXPORT int CreateWallet(char path[], char password[]) + { + std::unique_ptr wallet; + + try { + wallet = std::make_unique(path, password, true); + + return 0; + } + catch (const std::ios_base::failure &ex) { + std::cout << ex.what() << std::endl; + return api::WALLET_FILE_READ_ERROR; + } + catch (const platform::ExclusiveLock::FailedToLock &ex) { + std::cout << ex.what() << std::endl; + return api::WALLET_WITH_THE_SAME_VIEWKEY_IN_USE; + } + catch (const Wallet::Exception &ex) { + std::cout << ex.what() << std::endl; + return ex.return_code; + } + } + + EXPORT int ImportWallet(char path[], char password[], char keys[]) + { + std::unique_ptr wallet; + + try { + wallet = std::make_unique(path, password, true, keys); + // auto addr = wallet->get_first_address(); + // std::cout << Currency::get_account_address_as_str(154, addr); + return 0; + } + catch (const std::ios_base::failure &ex) { + std::cout << ex.what() << std::endl; + return api::WALLET_FILE_READ_ERROR; + } + catch (const platform::ExclusiveLock::FailedToLock &ex) { + std::cout << ex.what() << std::endl; + return api::WALLET_WITH_THE_SAME_VIEWKEY_IN_USE; + } + catch (const Wallet::Exception &ex) { + std::cout << ex.what() << std::endl; + return ex.return_code; + } + } + + EXPORT int ChangeWalletPassword(char newPassword[]) + { + // std::unique_ptr wallet; + + try { + // wallet = std::make_unique(path, oldPassword, false); + wallet_state->get_wallet().set_password(newPassword); + return 0; + } + catch (const std::ios_base::failure &ex) { + std::cout << ex.what() << std::endl; + return api::WALLET_FILE_READ_ERROR; + } + catch (const platform::ExclusiveLock::FailedToLock &ex) { + std::cout << ex.what() << std::endl; + return api::WALLET_WITH_THE_SAME_VIEWKEY_IN_USE; + } + catch (const Wallet::Exception &ex) { + std::cout << ex.what() << std::endl; + return ex.return_code; + } + } + + EXPORT int ChangeContainerPassword(char path[], char oldPassword[], char newPassword[]) + { + std::unique_ptr wallet; + + try { + wallet = std::make_unique(path, oldPassword, false); + wallet->set_password(newPassword); + return 0; + } + catch (const std::ios_base::failure &ex) { + std::cout << ex.what() << std::endl; + return api::WALLET_FILE_READ_ERROR; + } + catch (const platform::ExclusiveLock::FailedToLock &ex) { + std::cout << ex.what() << std::endl; + return api::WALLET_WITH_THE_SAME_VIEWKEY_IN_USE; + } + catch (const Wallet::Exception &ex) { + std::cout << ex.what() << std::endl; + return ex.return_code; + } + } + + std::string get_first_address() + { + AccountPublicAddress fa = wallet->get_first_address(); + BinaryArray ba = seria::to_binary(fa); + return common::base58::encode_addr(154, ba); + } + + EXPORT int GetFirstAddress(char * &address) + { + if (wallet == nullptr) + { + return ERROR_WALLET_NOT_INITIALIZED; + } + + try { + std::string sa = get_first_address(); + + address = new char[sa.length() + 1]; + strcpy(address, sa.c_str()); + + return ERROR_SUCCESS; + } + catch (const std::exception &ex) { + std::cout << ex.what() << std::endl; + return ERROR_UNKNOWN; + } + } + + EXPORT int GetBalance(char address[], bool total, api::Balance &balance) + { + try { + if (wallet_state == nullptr) + { + return ERROR_WALLET_NOT_INITIALIZED; + } + + std::string sa; + std::cout << "1"; + if (!total) + { + std::cout << ">1"; + if (address != nullptr) + { + std::cout << "2"; + sa = std::string(address); + } + else + { + std::cout << "3"; + sa = get_first_address(); + } + } + std::cout << "4"; + std::cout << sa << "get_balance(sa, -6); + + return ERROR_SUCCESS; + } + catch (const std::exception &ex) { + std::cout << ex.what() << std::endl; + return ERROR_UNKNOWN; + } + } + + EXPORT int GetKeys(char address[], PrivateKeyPair &keys) { + if (wallet == nullptr) + { + return ERROR_WALLET_NOT_INITIALIZED; + } + + try { + keys = PrivateKeyPair(); + + SecretKey vk = wallet->get_view_secret_key(); + std::copy(std::begin(vk.data), std::end(vk.data), std::begin(keys.view_key)); + + BinaryArray data; + uint64_t prefix = 154; + AccountPublicAddress adr; + AccountKeys ak; + + if (!common::base58::decode_addr(address, prefix, data)) return false; // parsing error + seria::from_binary(adr, data); + wallet->spend_keys_for_address(adr, ak); + SecretKey sk = ak.spend_secret_key; + std::copy(std::begin(sk.data), std::end(sk.data), std::begin(keys.spend_key)); + + return ERROR_SUCCESS; + } + catch (const std::exception &ex) { + std::cout << ex.what() << std::endl; + return ERROR_UNKNOWN; + } + } + +} \ No newline at end of file diff --git a/src/bitsum/bitsum.cpp b/src/bitsum/bitsum.cpp index 572a145f..647652dc 100644 --- a/src/bitsum/bitsum.cpp +++ b/src/bitsum/bitsum.cpp @@ -10,240 +10,97 @@ #include "common/Base58.hpp" #include "seria/BinaryInputStream.hpp" #include "seria/BinaryOutputStream.hpp" +#include "Explorer.hpp" using namespace bytecoin; using namespace common; -#if defined(_WIN32) -extern "C" _declspec(dllexport) int __DaemonRun() -#else -extern "C" int __DaemonRun() -#endif -{ - try - { - //common::console::UnicodeConsoleSetup console_setup; - std::cout.rdbuf(nullptr); - auto idea_start = std::chrono::high_resolution_clock::now(); - - common::CommandLine cmd(1, 0); - bytecoin::Config config(cmd); - bytecoin::Currency currency(config.is_testnet); - const std::string coinFolder = config.get_data_folder(); - - platform::ExclusiveLock coin_lock(coinFolder, "bitsumd.lock"); - - logging::LoggerManager logManager; - logManager.configure_default(config.get_data_folder("logs"), "bitsumd-"); - - BlockChainState block_chain(logManager, config, currency); - - boost::asio::io_service io; - platform::EventLoop run_loop(io); - - Node node(logManager, config, block_chain); - - auto idea_ms = - std::chrono::duration_cast(std::chrono::high_resolution_clock::now() - idea_start); - std::cout << "bitsumd started seconds=" << double(idea_ms.count()) / 1000 << std::endl; - while (!io.stopped()) { - if (node.on_idle()) // Using it to load blockchain - io.poll(); - else - io.run_one(); - } - - return 0; - } - catch (const std::exception&) - { - return 1; - } -} - -#if defined(_WIN32) -extern "C" _declspec(dllexport) int __WalletRun(char path[], char password[]) -#else -extern "C" int __WalletRun(char path[], char password[]) -#endif -{ - //common::console::UnicodeConsoleSetup console_setup; - std::cout.rdbuf(nullptr); - auto idea_start = std::chrono::high_resolution_clock::now(); - common::CommandLine cmd(1, 0); - bytecoin::Config config(cmd); - bytecoin::Currency currency(config.is_testnet); - - const std::string coinFolder = config.get_data_folder(); - std::unique_ptr walletcache_lock; - std::unique_ptr wallet; - - try { - wallet = std::make_unique(path, password, false); - walletcache_lock = std::make_unique( - config.get_data_folder("wallet_cache"), wallet->get_cache_name() + ".lock"); - } - catch (const std::ios_base::failure &ex) { - std::cout << ex.what() << std::endl; - return api::WALLET_FILE_READ_ERROR; - } - catch (const platform::ExclusiveLock::FailedToLock &ex) { - std::cout << ex.what() << std::endl; - return api::WALLET_WITH_THE_SAME_VIEWKEY_IN_USE; - } - catch (const Wallet::Exception &ex) { - std::cout << ex.what() << std::endl; - return ex.return_code; - } - - logging::LoggerManager logManagerWalletNode; - logManagerWalletNode.configure_default(config.get_data_folder("logs"), "wallet-"); - - WalletState wallet_state(*wallet, logManagerWalletNode, config, currency); - boost::asio::io_service io; - platform::EventLoop run_loop(io); - - std::unique_ptr node; - std::unique_ptr wallet_node; - try { - wallet_node = std::make_unique(nullptr, logManagerWalletNode, config, wallet_state); - } - catch (const boost::system::system_error &ex) { - std::cout << ex.what() << std::endl; - return api::WALLETD_BIND_PORT_IN_USE; - } - - auto idea_ms = - std::chrono::duration_cast(std::chrono::high_resolution_clock::now() - idea_start); - std::cout << "wallet-rpc started seconds=" << double(idea_ms.count()) / 1000 << std::endl; - - while (!io.stopped()) { - if (node && node->on_idle()) // We load blockchain there - io.poll(); - else - io.run_one(); - } - - return 0; -} - -#if defined(_WIN32) -extern "C" _declspec(dllexport) int __WalletCreateContainer(char path[], char password[]) -#else -extern "C" int __WalletCreateContainer(char path[], char password[]) -#endif -{ - std::unique_ptr wallet; - - try { - wallet = std::make_unique(path, password, true); - - return 0; - } - catch (const std::ios_base::failure & ex) { - std::cout << ex.what() << std::endl; - return api::WALLET_FILE_READ_ERROR; - } - catch (const platform::ExclusiveLock::FailedToLock & ex) { - std::cout << ex.what() << std::endl; - return api::WALLET_WITH_THE_SAME_VIEWKEY_IN_USE; - } - catch (const Wallet::Exception & ex) { - std::cout << ex.what() << std::endl; - return ex.return_code; - } -} - -#if defined(_WIN32) -extern "C" _declspec(dllexport) int __WalletImportContainer(char path[], char password[], char keys[]) -#else -extern "C" int __WalletImportContainer(char path[], char password[], char keys[]) -#endif -{ - std::unique_ptr wallet; - - try { - wallet = std::make_unique(path, password, true, keys); - //auto addr = wallet->get_first_address(); - //std::cout << Currency::get_account_address_as_str(154, addr); - return 0; - } - catch (const std::ios_base::failure & ex) { - std::cout << ex.what() << std::endl; - return api::WALLET_FILE_READ_ERROR; - } - catch (const platform::ExclusiveLock::FailedToLock & ex) { - std::cout << ex.what() << std::endl; - return api::WALLET_WITH_THE_SAME_VIEWKEY_IN_USE; - } - catch (const Wallet::Exception & ex) { - std::cout << ex.what() << std::endl; - return ex.return_code; - } -} - -#if defined(_WIN32) -extern "C" _declspec(dllexport) int __WalletChangeContainerPassword(char path[], char oldPassword[], char newPassword[]) -#else -extern "C" int __WalletChangeContainerPassword(char path[], char oldPassword[], char newPassword[]) -#endif -{ - std::unique_ptr wallet; - - try { - wallet = std::make_unique(path, oldPassword, false); - wallet->set_password(newPassword); - return 0; - } - catch (const std::ios_base::failure & ex) { - std::cout << ex.what() << std::endl; - return api::WALLET_FILE_READ_ERROR; - } - catch (const platform::ExclusiveLock::FailedToLock & ex) { - std::cout << ex.what() << std::endl; - return api::WALLET_WITH_THE_SAME_VIEWKEY_IN_USE; - } - catch (const Wallet::Exception & ex) { - std::cout << ex.what() << std::endl; - return ex.return_code; - } -} - -#if defined(_WIN32) -extern "C" _declspec(dllexport) bool __CheckAddress(char address[]) -#else -extern "C" bool __CheckAddress(char address[]) -#endif -{ - BinaryArray data; - uint64_t prefix = 154; - AccountPublicAddress adr; - - if (!common::base58::decode_addr(address, prefix, data)) - return false; - try { - seria::from_binary(adr, data); - } - catch (const std::exception &) { - return false; - } - return key_isvalid(adr.spend_public_key) && key_isvalid(adr.view_public_key); -} - -#if defined(_WIN32) -extern "C" _declspec(dllexport) bool __WalletCheckPassword(char path[], char password[]) -#else -extern "C" bool __WalletCheckPassword(char path[], char password[]) -#endif -{ - std::unique_ptr wallet; - try - { - wallet = std::make_unique(path, password, false); - return true; - } - catch (const std::exception&) - { - return false; - } -} \ No newline at end of file + +//boost::asio::io_service node_io; +//std::unique_ptr node_lock; +//std::unique_ptr blockchain_state; + + +//std::unique_ptr explorer; + +//struct BlockHeader_ { +// //explorer::Hash hashes[]; +// //std::vector has; +// std::vector has{}; +//}; +//extern "C" _declspec(dllexport) void __Test(BlockPreview &bp) +//{ +// try +// { +// uint8_t buf[1]; +// crypto::Hash h = cn_fast_hash(buf, 1); +// std::cout << common::pod_to_hex(h); +// +// bp.height = 100; +// std::copy(std::begin(h.data), std::end(h.data), std::begin(bp.hash)); +// bp.size = 101; +// bp.timestamp = 102; +// bp.tx_count = 103; +// +// } +// catch (const std::exception& ex) +// { +// std::cout << ex.what(); +// } +// +//} +// +//extern "C" _declspec(dllexport) void __Test2(BlockHeader_ &bp) +//{ +// try +// { +// uint8_t buf[1]; +// crypto::Hash h = cn_fast_hash(buf, 1); +// std::cout << common::pod_to_hex(h); +// +// //std::vector has; +// for (uint8_t i = 0; i < 5; i++) +// { +// //crypto::Hash h = cn_fast_hash(buf, 1); +// //std::cout << common::pod_to_hex(h); +// +// //explorer::Hash th = explorer::Hash(); +// //std::copy(std::begin(h.data), std::end(h.data), std::begin(th.bytes)); +// +// bp.has.push_back(uint8_t()); +// } +// +// //bp.height = 100; +// //std::copy(std::begin(h.data), std::end(h.data), std::begin(bp.hash)); +// //bp.size = 101; +// //bp.timestamp = 102; +// //bp.tx_count = 103; +// +//} +// catch (const std::exception& ex) +// { +// std::cout << ex.what(); +// } +// +//} + + + + + + + + + + + + + + + + + + + + + diff --git a/src/common/ConsoleTools.cpp b/src/common/ConsoleTools.cpp index a1401c94..c68b7d56 100644 --- a/src/common/ConsoleTools.cpp +++ b/src/common/ConsoleTools.cpp @@ -72,7 +72,6 @@ void set_text_color(Color color) { UnicodeConsoleSetup::UnicodeConsoleSetup() { #ifdef _WIN32 SetConsoleOutputCP(CP_UTF8); - //std::cout.rdbuf(nullptr); old_buf = std::cout.rdbuf(this); #else boost::ignore_unused_variable_warning(old_buf); diff --git a/src/common/JsonValue.cpp b/src/common/JsonValue.cpp index beb12c18..dfd1dbeb 100644 --- a/src/common/JsonValue.cpp +++ b/src/common/JsonValue.cpp @@ -202,6 +202,7 @@ JsonValue &JsonValue::operator=(const JsonValue &other) { return *this; } +// Analysers might warn about absense of this != &other JsonValue &JsonValue::operator=(JsonValue &&other) { if (type != other.type) { destruct_value(); diff --git a/src/http/Server.cpp b/src/http/Server.cpp index c996f156..d7a669f0 100644 --- a/src/http/Server.cpp +++ b/src/http/Server.cpp @@ -138,11 +138,10 @@ void Server::on_client_handler(Client *who) { } catch (const std::exception &e) { std::cout << "HTTP request leads to throw/catch, what=" << e.what() << std::endl; response.r.status = 422; - response.set_body(std::string()); + response.set_body(e.what()); } catch (...) { std::cout << "HTTP request leads to throw/catch" << std::endl; response.r.status = 422; - response.set_body(std::string()); } if (result) who->write(std::move(response)); diff --git a/src/logging/FileLogger.cpp b/src/logging/FileLogger.cpp index 9c4c7c21..786197e7 100644 --- a/src/logging/FileLogger.cpp +++ b/src/logging/FileLogger.cpp @@ -7,14 +7,14 @@ namespace logging { FileLogger::FileLogger(const std::string &fullfilenamenoext, size_t max_size, Level level) - : CommonLogger(level), max_size(max_size), fullfilenamenoext(fullfilenamenoext) { + : CommonLogger(level), initial_max_size(max_size), max_size(max_size), fullfilenamenoext(fullfilenamenoext) { try { file_stream = std::make_unique( - this->fullfilenamenoext + "_0.log", platform::FileStream::READ_WRITE_EXISTING); + this->fullfilenamenoext + ".log", platform::FileStream::READ_WRITE_EXISTING); file_stream->seek(0, SEEK_END); } catch (const std::exception &) { file_stream = std::make_unique( - this->fullfilenamenoext + "_0.log", platform::FileStream::TRUNCATE_READ_WRITE); + this->fullfilenamenoext + ".log", platform::FileStream::TRUNCATE_READ_WRITE); } } @@ -37,16 +37,20 @@ void FileLogger::do_log_string(const std::string &message) { } if (file_stream && file_stream->tellp() >= max_size) { - std::string cur = fullfilenamenoext + "_0.log"; - std::string prev = fullfilenamenoext + "_1.log"; - if (!platform::atomic_replace_file(cur, prev)) { + std::string cur = fullfilenamenoext + ".log"; + std::string prev = fullfilenamenoext + "_prev.log"; + if (!using_prev && !platform::atomic_replace_file(cur, prev)) { // StreamLogger::do_log_string("FileLogger failed to rotate log file, doubling size of next rotation..."); max_size *= 2; return; } try { file_stream = std::make_unique(cur, platform::FileStream::TRUNCATE_READ_WRITE); + using_prev = false; + max_size = initial_max_size; } catch (...) { // Will continue using old one if new one fails to open + using_prev = true; + max_size *= 2; // doubling size of next rotation... } } } diff --git a/src/logging/FileLogger.hpp b/src/logging/FileLogger.hpp index fbe58ca0..11826a2d 100644 --- a/src/logging/FileLogger.hpp +++ b/src/logging/FileLogger.hpp @@ -18,7 +18,9 @@ class FileLogger : public CommonLogger { private: std::mutex mutex; + const size_t initial_max_size; size_t max_size; + bool using_prev = false; // atomic_replaced success, but create new file failed const std::string fullfilenamenoext; std::unique_ptr file_stream; }; diff --git a/src/logging/LoggerManager.cpp b/src/logging/LoggerManager.cpp index af733a75..ec984347 100644 --- a/src/logging/LoggerManager.cpp +++ b/src/logging/LoggerManager.cpp @@ -26,11 +26,11 @@ void LoggerManager::configure_default(const std::string &log_folder, const std:: LoggerGroup::loggers.clear(); std::unique_ptr logger( - new FileLogger(log_folder + "/" + log_prefix + "verbose", 128 * 1024, TRACE)); + new FileLogger(log_folder + "/" + log_prefix + "verbose", 1024 * 1024, TRACE)); loggers.emplace_back(std::move(logger)); add_logger(*loggers.back()); - logger.reset(new FileLogger(log_folder + "/" + log_prefix + "errors", 128 * 1024, WARNING)); + logger.reset(new FileLogger(log_folder + "/" + log_prefix + "errors", 1024 * 1024, ERROR)); loggers.emplace_back(std::move(logger)); add_logger(*loggers.back()); @@ -39,7 +39,8 @@ void LoggerManager::configure_default(const std::string &log_folder, const std:: loggers.emplace_back(std::move(logger)); add_logger(*loggers.back()); } - (*this)("START", TRACE, boost::posix_time::microsec_clock::local_time(), "----------------------------------------\n"); + (*this)( + "START", TRACE, boost::posix_time::microsec_clock::local_time(), "----------------------------------------\n"); } void LoggerManager::configure(const JsonValue &val) { diff --git a/src/main_bitsumd.cpp b/src/main_bitsumd.cpp index ec0b1fc6..4ec15403 100644 --- a/src/main_bitsumd.cpp +++ b/src/main_bitsumd.cpp @@ -25,7 +25,7 @@ static const char USAGE[] = Options: --export-blocks= Export blockchain into specified directory as blocks.bin and blockindexes.bin, then exit. This overwrites existing files. --allow-local-ip Allow local ip add to peer list, mostly in debug purposes. - --testnet Configure for testnet. + --p2p-bind-address= Interface and port for P2P network protocol [default: 0.0.0.0:28080]. --p2p-external-port= External port for P2P network protocol, if port forwarding used with NAT [default: 28080]. --daemon-rpc-bind-address= Interface and port for bitsumd RPC [default: 127.0.0.1:28081]. diff --git a/src/main_wallet_rpc.cpp b/src/main_wallet_rpc.cpp index 5b24f876..fcf8265a 100644 --- a/src/main_wallet_rpc.cpp +++ b/src/main_wallet_rpc.cpp @@ -33,7 +33,7 @@ static const char USAGE[] = --set-password Read new password as a line from stdin (twice) and reencrypt wallet file. --export-view-only= Export view-only version of wallet file with the same password, then exit. --export-keys Export wallet keys to stdout, then exit. - --testnet Configure for testnet. + --wallet-rpc-bind-address= Interface and port for wallet-rpc [default: 127.0.0.1:28082]. --data-folder= Folder for wallet cache, blockchain, logs and peer DB [default: )" platform_DEFAULT_DATA_FOLDER_PATH_PREFIX R"(bitsum]. @@ -60,7 +60,12 @@ int main(int argc, const char *argv[]) try { bool ask_password = true; const bool export_keys = cmd.get_bool("--export-keys"); const bool create_wallet = cmd.get_bool("--create-wallet"); - const bool import_keys = create_wallet && cmd.get_bool("--import-keys"); + const bool import_keys = cmd.get_bool("--import-keys"); + if (import_keys && !create_wallet){ + std::cout << "When importing keys, you should use --create-wallet. You cannot import into existing wallet." + << std::endl; + return api::WALLETD_WRONG_ARGS; + } if (const char *pa = cmd.get("--wallet-file")) wallet_file = pa; if (const char *pa = cmd.get("--export-view-only")) @@ -88,7 +93,7 @@ int main(int argc, const char *argv[]) try { std::cout << "--wallet-file= argument is mandatory" << std::endl; return api::WALLETD_WRONG_ARGS; } - if (create_wallet && import_keys && import_keys_value.empty()) { + if (create_wallet && import_keys && import_keys_value.empty()) { // TODO import_keys_value always empty std::cout << "Enter imported keys as hex bytes (05AB6F... etc.): " << std::flush; if (!std::getline(std::cin, import_keys_value)) { std::cout << "Unexpected end of stdin" << std::endl; @@ -136,6 +141,8 @@ int main(int argc, const char *argv[]) try { try { wallet = std::make_unique( wallet_file, create_wallet ? new_password : password, create_wallet, import_keys_value); + std::cout << "Using wallet cache " << config.get_data_folder("wallet_cache") << "/" << wallet->get_cache_name() + << std::endl; } catch (const common::StreamError &ex) { std::cout << ex.what() << std::endl; return api::WALLET_FILE_READ_ERROR; @@ -246,9 +253,11 @@ int main(int argc, const char *argv[]) try { if (bytecoind_thread.joinable()) bytecoind_thread.join(); // otherwise terminate will be called in ~thread return api::BYTECOIND_BIND_PORT_IN_USE; + } catch (const std::exception &ex) { // On Windows what() is not printed if thrown from main + std::cout << "Exception in main() - " << ex.what() << std::endl; + throw; } } - std::unique_ptr wallet_node; try { wallet_node = std::make_unique(nullptr, logManagerWalletNode, config, wallet_state); diff --git a/src/p2p/P2pSeria.cpp b/src/p2p/P2pSeria.cpp index 74f943ba..34f8abef 100644 --- a/src/p2p/P2pSeria.cpp +++ b/src/p2p/P2pSeria.cpp @@ -40,6 +40,10 @@ void ser_members(bytecoin::PeerlistEntry &v, seria::ISeria &s) { seria_kv("adr", v.adr, s); seria_kv("id", v.id, s); seria_kv("last_seen", v.last_seen, s); + // uint64_t last_seen_64 = v.last_seen; + // seria_kv("last_seen", last_seen_64, s); + // if (s.is_input()) + // v.last_seen = static_cast(last_seen_64); // seria_kv("reserved", v.reserved, s); } diff --git a/src/platform/DBlmdb.cpp b/src/platform/DBlmdb.cpp index 86faef2d..d265a230 100644 --- a/src/platform/DBlmdb.cpp +++ b/src/platform/DBlmdb.cpp @@ -68,8 +68,9 @@ platform::lmdb::Cur::~Cur() { } DBlmdb::DBlmdb(const std::string &full_path, uint64_t max_db_size) : full_path(full_path) { - std::cout << "lmdb libversion=" << mdb_version(nullptr, nullptr, nullptr) << std::endl; + // std::cout << "lmdb libversion=" << mdb_version(nullptr, nullptr, nullptr) << std::endl; lmdb_check(::mdb_env_set_mapsize(db_env.handle, max_db_size), "mdb_env_set_mapsize "); + // VALGRIND is limited to 32GB, modify line above to use (max_db_size > 28000000000 ? 28000000000 : max_db_size) create_directories_if_necessary(full_path); lmdb_check(::mdb_env_open(db_env.handle, full_path.c_str(), MDB_NOMETASYNC, 0644), "mdb_env_open "); // MDB_NOMETASYNC - We agree to trade chance of losing 1 last transaction for 2x performance boost diff --git a/src/platform/Files.cpp b/src/platform/Files.cpp index 958a711b..cbdb5489 100644 --- a/src/platform/Files.cpp +++ b/src/platform/Files.cpp @@ -41,7 +41,7 @@ FileStream::~FileStream() { handle = nullptr; #else close(fd); - fd = 0; + fd = -1; #endif } diff --git a/src/platform/Files.hpp b/src/platform/Files.hpp index fbdea659..7c773e9f 100644 --- a/src/platform/Files.hpp +++ b/src/platform/Files.hpp @@ -34,7 +34,7 @@ class FileStream : public common::IOutputStream, public common::IInputStream, pr #ifdef _WIN32 void *handle = nullptr; #else - int fd = 0; + int fd = -1; #endif }; } diff --git a/src/platform/Network.cpp b/src/platform/Network.cpp index efbd1e6d..fb21f13e 100644 --- a/src/platform/Network.cpp +++ b/src/platform/Network.cpp @@ -305,6 +305,10 @@ void TCPSocket::write_callback(CFWriteStreamRef stream, CFStreamEventType event, namespace ssl = boost::asio::ssl; typedef ssl::stream SSLSocket; +// We need a timer fix from 1.62 to prevent (very rare) segfaults in asio::detail::timer_queue::remove_timer +static_assert(BOOST_VERSION / 100000 == 1 && ((BOOST_VERSION / 100) % 1000) >= 62, + "You need at least boost 1.62, you are compiling with " BOOST_LIB_VERSION); + #ifdef _WIN32 #include #pragma comment(lib, "libcrypto.lib") // OpenSSL library @@ -426,9 +430,23 @@ void EventLoop::wake() { io_service.post([](void) {}); } +// static bool ispowerof2(unsigned int x) { +// return x && !(x & (x - 1)); +//} +// static unsigned global_timer_impl_counter = 0; + class Timer::Impl { public: - explicit Impl(Timer *owner) : owner(owner), pending_wait(false), timer(EventLoop::current()->io()) {} + explicit Impl(Timer *owner) : owner(owner), pending_wait(false), timer(EventLoop::current()->io()) { + // global_timer_impl_counter += 1; + // if( ispowerof2(global_timer_impl_counter) ) + // std::cout << "++Timer::Impl::counter=" << global_timer_impl_counter << std::endl; + } + ~Impl() { + // if( ispowerof2(global_timer_impl_counter) ) + // std::cout << "--Timer::Impl::counter=" << global_timer_impl_counter << std::endl; + // global_timer_impl_counter -= 1; + } Timer *owner; bool pending_wait; boost::asio::deadline_timer timer; @@ -438,6 +456,8 @@ class Timer::Impl { if (pending_wait) { owner = nullptr; was_owner->impl.reset(); + boost::system::error_code ec; + timer.cancel(ec); // Prevent exceptions } } void handle_timeout(const boost::system::error_code &e) { @@ -840,8 +860,10 @@ bool TCPAcceptor::accept(TCPSocket &socket, std::string &accepted_addr) { auto endpoint = socket.impl->socket.remote_endpoint(ec); #endif - if (ec) + if (ec) { + impl->start_accept(); return false; + } accepted_addr = endpoint.address().to_string(); #if platform_USE_SSL if (impl->ssl) { @@ -861,3 +883,34 @@ bool TCPAcceptor::accept(TCPSocket &socket, std::string &accepted_addr) { } #endif // #if TARGET_OS_IPHONE + +// Code to stress-test timers +// std::vector> timers; +// +// void timers_handler(size_t pos){ +// std::cout << "Fired " << pos << std::endl; +// size_t rand1 = crypto::rand() % timers.size(); +// size_t rand2 = crypto::rand() % timers.size(); +// size_t rand3 = crypto::rand() % timers.size(); +// size_t rand4 = crypto::rand() % timers.size(); +// float rand_t = (crypto::rand() % 5000) / 1000.0f; +// timers.at(rand1)->cancel(); +// timers.at(rand2)->cancel(); +// timers.at(rand3)->once(rand_t); +// timers.at(rand3)->once(rand_t); +// timers.at(rand4)->once(rand_t + 0.5f); +//} +// +// static int test_timers(){ +// boost::asio::io_service io; +// platform::EventLoop run_loop(io); +// +// for(size_t i = 0; i != 50000; ++i) +// timers.push_back( std::make_unique(std::bind(&timers_handler, i)) ); +// timers.at(0)->once(1); +// +// while (!io.stopped()) { +// io.run_one(); +// } +// return 0; +//} diff --git a/src/rpc_api.hpp b/src/rpc_api.hpp index 1128b643..095bad21 100644 --- a/src/rpc_api.hpp +++ b/src/rpc_api.hpp @@ -307,11 +307,17 @@ struct CreateTransaction { bool save_history = true; // If true, wallet will save encrypted transaction data (~100 bytes per used address) // in .history/. With this data it is possible to generate // public-checkable proofs of sending funds to specific addresses. + std::vector + prevent_conflict_with_transactions; // Experimental API for guaranteed payouts under any circumstances }; struct Response { - BinaryArray binary_transaction; // Empty if error - api::Transaction transaction; // contains only fee, hash, blockIndex and anonymity for now... - bool save_history_error = false; // Read-only media + BinaryArray binary_transaction; // Empty if error + api::Transaction + transaction; // block_hash will be empty, block_height set to current pool height (may change later) + bool save_history_error = false; // When wallet on read-only media. Most clients should ignore this + std::vector transactions_required; // Works together with prevent_conflict_with_transactions + // If not empty, you should resend those transactions before trying create_transaction again to prevent + // conflicts }; }; @@ -388,9 +394,9 @@ struct SyncBlocks { // Used by walletd, block explorer, etc to sync to bytecoin }; struct SyncBlock { // Signatures are checked by bytecoind so usually they are of no interest api::BlockHeader header; - bytecoin::BlockTemplate bc_header; + bytecoin::BlockTemplate raw_header; // the only method returning actual BlockHeader from blockchain, not api::BlockHeader - std::vector bc_transactions; + std::vector raw_transactions; // the only method returning actual Transaction from blockchain, not api::Transaction Hash base_transaction_hash; // BlockTemplate does not contain it std::vector> global_indices; // for each transaction @@ -402,6 +408,19 @@ struct SyncBlocks { // Used by walletd, block explorer, etc to sync to bytecoin }; }; +struct GetRawTransaction { + static std::string method() { return "get_raw_transaction"; } + struct Request { + Hash hash; + }; + struct Response { + api::Transaction transaction; + // only hash, block_height and block_hash returned in transaction + // empty transaction with no hash returned if not in blockchain/mempool + bytecoin::TransactionPrefix raw_transaction; + }; +}; + // Signature of this method will stabilize to the end of beta struct SyncMemPool { // Used by walletd sync process static std::string method() { return "sync_mem_pool"; } @@ -410,10 +429,10 @@ struct SyncMemPool { // Used by walletd sync process std::vector known_hashes; // Should be sent sorted }; struct Response { - std::vector removed_hashes; // Hashes no more in pool - std::vector added_bc_transactions; // New raw transactions in pool + std::vector removed_hashes; // Hashes no more in pool + std::vector added_raw_transactions; // New raw transactions in pool std::vector added_transactions; - // binary version of this method returns only hash and timestamp here + // binary version of this method returns only hash, timestamp and fee here GetStatus::Response status; // We save roundtrip during sync by also sending status here }; }; @@ -442,10 +461,7 @@ struct CheckSendProof { struct Request { std::string send_proof; }; - - struct Response { - std::string validation_error; // empty when sucessfully validated - }; + typedef EmptyStruct Response; // All errors are reported as json rpc errors }; // Methods below are used by miners @@ -561,6 +577,8 @@ void ser_members(bytecoin::api::bytecoind::GetStatus::Response &v, ISeria &s); void ser_members(bytecoin::api::bytecoind::SyncBlocks::Request &v, ISeria &s); void ser_members(bytecoin::api::bytecoind::SyncBlocks::SyncBlock &v, ISeria &s); void ser_members(bytecoin::api::bytecoind::SyncBlocks::Response &v, ISeria &s); +void ser_members(bytecoin::api::bytecoind::GetRawTransaction::Request &v, ISeria &s); +void ser_members(bytecoin::api::bytecoind::GetRawTransaction::Response &v, ISeria &s); void ser_members(bytecoin::api::bytecoind::SyncMemPool::Request &v, ISeria &s); void ser_members(bytecoin::api::bytecoind::SyncMemPool::Response &v, ISeria &s); void ser_members(bytecoin::api::bytecoind::GetRandomOutputs::Request &v, ISeria &s); @@ -568,7 +586,7 @@ void ser_members(bytecoin::api::bytecoind::GetRandomOutputs::Response &v, ISeria void ser_members(bytecoin::api::bytecoind::SendTransaction::Request &v, ISeria &s); void ser_members(bytecoin::api::bytecoind::SendTransaction::Response &v, ISeria &s); void ser_members(bytecoin::api::bytecoind::CheckSendProof::Request &v, ISeria &s); -void ser_members(bytecoin::api::bytecoind::CheckSendProof::Response &v, ISeria &s); +// void ser_members(bytecoin::api::bytecoind::CheckSendProof::Response &v, ISeria &s); void ser_members(bytecoin::api::bytecoind::GetBlockTemplate::Request &v, ISeria &s); void ser_members(bytecoin::api::bytecoind::GetBlockTemplate::Response &v, ISeria &s); void ser_members(bytecoin::api::bytecoind::GetCurrencyId::Response &v, ISeria &s); diff --git a/src/version.hpp b/src/version.hpp index 5b551e98..96dc3c4a 100644 --- a/src/version.hpp +++ b/src/version.hpp @@ -4,8 +4,8 @@ #pragma once // defines are for Windows resource compiler -#define bytecoin_VERSION_WINDOWS_COMMA 1, 18, 5, 10 -#define bytecoin_VERSION_STRING "1.18.5.10" +#define bytecoin_VERSION_WINDOWS_COMMA 1, 18, 5, 25 +#define bytecoin_VERSION_STRING "1.18.5.25" #ifndef RC_INVOKED // Windows resource compiler