*Merge code with bytecoin v3.0.4

*Switch to SQLITE
*Seed nodes
+Checkpoints

CMakeLists.txt

@@ -11,7 +11,7 @@ option(USE_INSTRUMENTATION "For testing - builds with address sanitizer instrume
 option(WITH_THREAD_SANITIZER "For testing - builds with thread sanitizer instrument, USE_INSTRUMENTATION must be also set" OFF)
 option(USE_SSL "Builds with support of https between walletd and bytecoind" OFF)
 if(CMAKE_SIZEOF_VOID_P EQUAL 8)
-    option(USE_SQLITE "Builds with SQLite instead of LMDB. 4x slower, but works on 32-bit and mobile platforms" OFF)
+    option(USE_SQLITE "Builds with SQLite instead of LMDB. 4x slower, but works on 32-bit and mobile platforms" ON)
     set(OPENSSL_ROOT ../openssl)
 else()
     option(USE_SQLITE "Builds with sqlite instead of LMDB. 4x slower, but works on 32-bit and mobile platforms" ON)
@@ -124,9 +124,15 @@ set(SOURCE_FILES
     src/rpc_api.hpp
     src/version.hpp
 )
-set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/bin")
-set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/libs")
-set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/libs")
+if(CMAKE_SIZEOF_VOID_P EQUAL 8)
+set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/bin/x64")
+set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/libs/x64")
+set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/libs/x64")
+else()
+set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/bin/x86")
+set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/libs/x86")
+set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/libs/x86")
+endif()
 if(WIN32)
     include_directories(SYSTEM ../boost)
     if(CMAKE_SIZEOF_VOID_P EQUAL 8)
@@ -157,7 +163,7 @@ if(WIN32)
 else()
     set(BOOST_ROOT ../boost)
     set(Boost_USE_MULTITHREADED OFF) # all boost libraries are multithreaded since some version
-    find_package(Boost 1.62 REQUIRED COMPONENTS system)
+    find_package(Boost 1.67 REQUIRED COMPONENTS system)
     include_directories(SYSTEM ${Boost_INCLUDE_DIRS})
     set(Boost_LIBRARIES "${Boost_LIBRARIES}")
     if(APPLE)

ReleaseNotes.md

@@ -1,5 +1,8 @@
 ## Release Notes
 
+### v1.18.5.10
+- Merge code with bytecoin v3.0.4
+
 ### v1.18.4.20
 - Merge code with bytecoin v3.0.2
 

src/Core/BlockChain.cpp

@@ -152,8 +152,9 @@ BlockChain::BlockChain(const Hash &genesis_bid, const std::string &coin_folder)
 }
 
 void BlockChain::db_commit() {
-	std::cout << "BlockChain::db_commit started... tip_height=" << m_tip_height << std::endl;
+	std::cout << "BlockChain::db_commit started... tip_height=" << m_tip_height << " header_cache.size=" << header_cache.size() << std::endl;
 	m_db.commit_db_txn();
+	header_cache.clear(); // Most simple cache policy ever
 	std::cout << "BlockChain::db_commit finished..." << std::endl;
 }
 
@@ -234,6 +235,7 @@ bool BlockChain::reorganize_blocks(const Hash &switch_to_chain,
 		if (read_chain(m_tip_height) != m_tip_bid)
 			throw std::logic_error(
 			    "Invariant dead - after undo tip does not match read_chain" + common::pod_to_hex(m_tip_bid));
+		tip_changed();
 	}
 	// Now redo all blocks we have in storage, will ask for the rest of blocks
 	while (!chain2.empty()) {
@@ -465,15 +467,14 @@ bool BlockChain::redo_block(const Hash &bhash, const RawBlock &, const Block &bl
 		m_db.put(bkey, std::string(), true);
 	}
 
-	m_tip_segment.push_back(info);
-	if (m_tip_segment.size() > 2048)  // TODO - should be enough for all block windows we use
-		m_tip_segment.pop_front();
-
+//	m_tip_segment.push_back(info);
+//	if (m_tip_segment.size() > 2048)  // TODO - should be enough for all block windows we use
+//		m_tip_segment.pop_front();
 	return true;
 }
 void BlockChain::undo_block(const Hash &bhash, const RawBlock &, const Block &block, Height height) {
-	if (!m_tip_segment.empty())
-		m_tip_segment.pop_back();
+//	if (!m_tip_segment.empty())
+//		m_tip_segment.pop_back();
 	undo_block(bhash, block, height);
 
 	auto tikey = TIMESTAMP_BLOCK_PREFIX + common::write_varint_sqlite4(block.header.timestamp) +
@@ -521,15 +522,22 @@ void BlockChain::store_header(const Hash &bid, const api::BlockHeader &header) {
 }
 
 bool BlockChain::read_header(const Hash &bid, api::BlockHeader &header) const {
-	if (bid == m_tip_bid && !m_tip_segment.empty()) {
-		header = m_tip_segment.back();
+	auto cit = header_cache.find(bid);
+	if( cit != header_cache.end() ){
+		header = cit->second;
 		return true;
 	}
+//	if (bid == m_tip_bid && !m_tip_segment.empty()) {
+//		header = m_tip_segment.back();
+//		return true;
+//	}
 	BinaryArray rb;
 	auto key = HEADER_PREFIX + DB::to_binary_key(bid.data, sizeof(bid.data)) + HEADER_SUFFIX;
 	if (!m_db.get(key, rb))
 		return false;
+	Hash bbid = bid;
 	seria::from_binary(header, rb);
+	header_cache.insert(std::make_pair(bbid, header));
 	return true;
 }
 
@@ -541,25 +549,49 @@ api::BlockHeader BlockChain::read_header(const Hash &bid) const {
 }
 
 const api::BlockHeader &BlockChain::get_tip() const {
-	if (m_tip_segment.empty())
-		m_tip_segment.push_back(read_header(get_tip_bid()));
-	return m_tip_segment.back();
-}
-
-std::pair<std::deque<api::BlockHeader>::const_iterator, std::deque<api::BlockHeader>::const_iterator>
-BlockChain::get_tip_segment(Height height_delta, Height window, bool add_genesis) const {
-	if (get_tip_height() == (Height)-1 || height_delta > get_tip_height())
-		return std::make_pair(m_tip_segment.end(), m_tip_segment.end());
-	while (m_tip_segment.size() < height_delta + window && m_tip_segment.size() < m_tip_height + 1) {
-		Hash ha = read_chain(static_cast<uint32_t>(m_tip_height - m_tip_segment.size()));
-		m_tip_segment.push_front(read_header(ha));
+	auto cit = header_cache.find(get_tip_bid());
+	if( cit != header_cache.end() ){
+		return cit->second;
 	}
-	if (m_tip_height + 1 <= height_delta + window) {
-		//	if( m_tip_segment.size() == m_tip_height + 1 ) {
-		//	if (height_delta + window >= m_tip_segment.size()) {
-		return std::make_pair(m_tip_segment.begin() + (add_genesis ? 0 : 1), m_tip_segment.end() - height_delta);
+	read_header(get_tip_bid());
+	cit = header_cache.find(get_tip_bid());
+	if( cit == header_cache.end() )
+		throw std::logic_error("After read_header, header should be in header_cache");
+	return cit->second;
+//	if (m_tip_segment.empty())
+//		m_tip_segment.push_back(read_header(get_tip_bid()));
+//	return m_tip_segment.back();
+}
+
+std::vector<api::BlockHeader>
+BlockChain::get_tip_segment(const api::BlockHeader & prev_info, Height window, bool add_genesis) const {
+	std::vector<api::BlockHeader> result;
+	if( prev_info.height == Height(-1))
+		return result;
+	api::BlockHeader pi = prev_info;
+	while(result.size() < window && pi.height != 0){
+		result.push_back(pi);
+		if( !read_header(pi.previous_block_hash, pi) )
+			throw std::logic_error("Invariant dead - previous block header not found in get_tip_segment");
+	}
+	if( result.size() < window && add_genesis){
+		if( pi.height != 0)
+			throw std::logic_error("Invariant dead - window size not reached, but genesis not found in get_tip_segment");
+		result.push_back(pi);
 	}
-	return std::make_pair(m_tip_segment.end() - window - height_delta, m_tip_segment.end() - height_delta);
+//	if (get_tip_height() == (Height)-1 || height_delta > get_tip_height())
+//		return std::make_pair(m_tip_segment.end(), m_tip_segment.end());
+//	while (m_tip_segment.size() < height_delta + window && m_tip_segment.size() < m_tip_height + 1) {
+//		Hash ha = read_chain(static_cast<uint32_t>(m_tip_height - m_tip_segment.size()));
+//		m_tip_segment.push_front(read_header(ha));
+//	}
+//	if (m_tip_height + 1 <= height_delta + window) {
+//			if( m_tip_segment.size() == m_tip_height + 1 ) {
+		//	if (height_delta + window >= m_tip_segment.size()) {
+//		return std::make_pair(m_tip_segment.begin() + (add_genesis ? 0 : 1), m_tip_segment.end() - height_delta);
+//	}
+	std::reverse(result.begin(), result.end());
+	return result;// std::make_pair(m_tip_segment.end() - window - height_delta, m_tip_segment.end() - height_delta);
 }
 
 void BlockChain::read_tip() {
@@ -584,7 +616,6 @@ void BlockChain::pop_chain() {
 		throw std::logic_error("pop_chain tip_height == 0");
 	m_db.del(TIP_CHAIN_PREFIX + version_current + "/" + common::write_varint_sqlite4(m_tip_height), true);
 	m_tip_height -= 1;
-	tip_changed();
 }
 
 bool BlockChain::read_chain(uint32_t height, Hash &bid) const {
@@ -756,11 +787,14 @@ void BlockChain::test_undo_everything() {
 		if (get_tip_bid() == m_genesis_bid)
 			break;
 		pop_chain();
-		api::BlockHeader info = get_tip();
 		m_tip_bid             = block.header.previous_block_hash;
-		m_tip_cumulative_difficulty -= info.cumulative_difficulty;
-		if (get_tip_height() % 50000 == 1)
+		api::BlockHeader info = get_tip();
+		m_tip_cumulative_difficulty = info.cumulative_difficulty;
+		tip_changed();
+		if (get_tip_height() % 50000 == 1 )
 			db_commit();
+		if( get_tip_height() <= 1525025 )
+			return;
 	}
 	std::cout << "---- After undo everything ---- " << std::endl;
 	int counter = 0;

src/Core/BlockChain.hpp

@@ -4,6 +4,7 @@
 #pragma once
 
 #include <deque>
+#include <unordered_map>
 #include "CryptoNote.hpp"
 #include "Currency.hpp"
 #include "platform/DB.hpp"
@@ -43,8 +44,8 @@ class BlockChain {
 	Height get_tip_height() const { return m_tip_height; }
 	const api::BlockHeader &get_tip() const;
 
-	std::pair<std::deque<api::BlockHeader>::const_iterator, std::deque<api::BlockHeader>::const_iterator>
-	get_tip_segment(Height height_delta, Height window, bool add_genesis) const;
+	std::vector<api::BlockHeader>
+	get_tip_segment(const api::BlockHeader & prev_info, Height window, bool add_genesis) const;
 
 	bool read_chain(Height height, Hash &bid) const;
 	bool read_block(const Hash &bid, RawBlock &rb) const;
@@ -102,7 +103,7 @@ class BlockChain {
 	void read_tip();
 	void push_chain(Hash bid, Difficulty cumulative_difficulty);
 	void pop_chain();
-	mutable std::deque<api::BlockHeader> m_tip_segment;
+	mutable std::unordered_map<Hash, api::BlockHeader> header_cache;
 	// We cache recent headers for quick calculation in block windows
 
 	void store_block(const Hash &bid, const BinaryArray &block_data);

src/Core/BlockChainFileFormat.cpp

@@ -11,7 +11,7 @@ using namespace common;
 using namespace bytecoin;
 
 // Example
-//	LegacyBlockChainReader reader(import_path + "/blockindexesx.bin", import_path + "/blocksx.bin");
+//	LegacyBlockChainReader reader(import_path + "/blockindexes.bin", import_path + "/blocks.bin");
 //	std::cout << "Importing blocks count=" << reader.get_block_count() << std::endl;
 //	for(Height h = 0; h != reader.get_block_count(); ++h){
 //		PreparedBlock pb = reader.get_prepared_block_by_index(h);
@@ -165,11 +165,11 @@ bool LegacyBlockChainReader::import_blockchain2(const std::string &coin_folder,
 	//	           "difference\tMedian - Timestamp"
 	//	        << std::endl;
 
-	LegacyBlockChainReader reader(coin_folder + "/blockindexesx.bin", coin_folder + "/blocksx.bin");
+	LegacyBlockChainReader reader(coin_folder + "/blockindexes.bin", coin_folder + "/blocks.bin");
 	const size_t bs_count = reader.get_block_count();
 	if (block_chain.get_tip_height() > bs_count) {
 		std::cout << "Skipping block chain import - we have more blocks than "
-		             "blocksx.bin tip_height="
+		             "blocks.bin tip_height="
 		          << block_chain.get_tip_height() << " bs_count=" << bs_count << std::endl;
 		return true;
 	}
@@ -226,7 +226,7 @@ bool LegacyBlockChainWriter::export_blockchain2(const std::string &export_folder
 	auto idea_start = std::chrono::high_resolution_clock::now();
 	std::cout << "Start exporting blocks" << std::endl;
 	LegacyBlockChainWriter writer(
-	    export_folder + "/blockindexesx.bin", export_folder + "/blocksx.bin", block_chain.get_tip_height() + 1);
+	    export_folder + "/blockindexes.bin", export_folder + "/blocks.bin", block_chain.get_tip_height() + 1);
 	for (Height ha = 0; ha != block_chain.get_tip_height() + 1; ++ha) {
 		Hash bid{};
 		RawBlock raw_block;

src/Core/BlockChainState.cpp

@@ -141,6 +141,7 @@ BlockChainState::BlockChainState(logging::ILogger &log, const Config &config, co
 		if (add_block(pb, info) == BroadcastAction::BAN)
 			throw std::logic_error("Genesis block failed to add");
 	}
+	m_log(logging::INFO) << "BlockChainState::BlockChainState height=" << get_tip_height() << " bid=" << common::pod_to_hex(get_tip_bid()) << std::endl;
 	BlockChainState::tip_changed();
 }
 
@@ -159,9 +160,8 @@ std::string BlockChainState::get_standalone_consensus_error(
 	info.timestamp_median = m_next_median_timestamp;
 	info.timestamp_unlock = m_next_unlock_timestamp;
 
-	if (get_tip_height() != prev_info.height)  // Optimization for most common case
-		calculate_consensus_values(
-		    get_tip_height() - prev_info.height, info.size_median, info.timestamp_median, info.timestamp_unlock);
+	if (get_tip_bid() != prev_info.hash)  // Optimization for most common case
+		calculate_consensus_values(prev_info, info.size_median, info.timestamp_median, info.timestamp_unlock);
 
 	auto next_block_granted_full_reward_zone = m_currency.block_granted_full_reward_zone_by_block_version(
 	    block.header.major_version);  // We will check version later in this fun
@@ -229,12 +229,12 @@ std::string BlockChainState::get_standalone_consensus_error(
 		std::vector<Timestamp> timestamps;
 		std::vector<Difficulty> difficulties;
 		Height blocks_count    = std::min(prev_info.height, m_currency.difficulty_blocks_count());
-		auto timestamps_window = get_tip_segment(get_tip_height() - prev_info.height, blocks_count, false);
-		size_t actual_count    = timestamps_window.second - timestamps_window.first;
+		auto timestamps_window = get_tip_segment(prev_info, blocks_count, false);
+		size_t actual_count    = timestamps_window.size();
 		timestamps.resize(actual_count);
 		difficulties.resize(actual_count);
 		size_t pos = 0;
-		for (auto it = timestamps_window.first; it != timestamps_window.second; ++it, ++pos) {
+		for (auto it = timestamps_window.begin(); it != timestamps_window.end(); ++it, ++pos) {
 			timestamps.at(pos)   = it->timestamp;
 			difficulties.at(pos) = it->cumulative_difficulty;
 		}
@@ -287,20 +287,20 @@ std::string BlockChainState::get_standalone_consensus_error(
 	return std::string();
 }
 
-void BlockChainState::calculate_consensus_values(Height height_delta, uint32_t &next_median_size,
+void BlockChainState::calculate_consensus_values(const api::BlockHeader & prev_info, uint32_t &next_median_size,
     Timestamp &next_median_timestamp, Timestamp &next_unlock_timestamp) const {
 	std::vector<uint32_t> last_blocks_sizes;
-	auto window = get_tip_segment(height_delta, m_currency.reward_blocks_window, true);
+	auto window = get_tip_segment(prev_info, m_currency.reward_blocks_window, true);
 	last_blocks_sizes.reserve(m_currency.reward_blocks_window);
-	for (auto it = window.first; it != window.second; ++it)
+	for (auto it = window.begin(); it != window.end(); ++it)
 		last_blocks_sizes.push_back(it->block_size);
 	next_median_size = common::median_value(last_blocks_sizes);
 
-	window = get_tip_segment(height_delta, m_currency.timestamp_check_window, false);
-	if (window.second - window.first >= static_cast<ptrdiff_t>(m_currency.timestamp_check_window)) {
+	window = get_tip_segment(prev_info, m_currency.timestamp_check_window, false);
+	if (window.size() >= m_currency.timestamp_check_window) {
 		std::vector<Timestamp> timestamps;
 		timestamps.reserve(m_currency.timestamp_check_window);
-		for (auto it = window.first; it != window.second; ++it)
+		for (auto it = window.begin(); it != window.end(); ++it)
 			timestamps.push_back(it->timestamp);
 		next_median_timestamp = common::median_value(timestamps);  // sorts timestamps
 		next_unlock_timestamp = timestamps[timestamps.size() / 2];
@@ -318,7 +318,7 @@ void BlockChainState::calculate_consensus_values(Height height_delta, uint32_t &
 }
 
 void BlockChainState::tip_changed() {
-	calculate_consensus_values(0, m_next_median_size, m_next_median_timestamp, m_next_unlock_timestamp);
+	calculate_consensus_values(read_header(get_tip_bid()), m_next_median_size, m_next_median_timestamp, m_next_unlock_timestamp);
 }
 
 bool BlockChainState::create_mining_block_template(BlockTemplate &b, const AccountPublicAddress &adr,
@@ -331,8 +331,8 @@ bool BlockChainState::create_mining_block_template(BlockTemplate &b, const Accou
 		Height blocks_count = std::min(get_tip_height(), m_currency.difficulty_blocks_count());
 		timestamps.reserve(blocks_count);
 		difficulties.reserve(blocks_count);
-		auto timestamps_window = get_tip_segment(0, blocks_count, false);
-		for (auto it = timestamps_window.first; it != timestamps_window.second; ++it) {
+		auto timestamps_window = get_tip_segment(read_header(get_tip_bid()), blocks_count, false);
+		for (auto it = timestamps_window.begin(); it != timestamps_window.end(); ++it) {
 			timestamps.push_back(it->timestamp);
 			difficulties.push_back(it->cumulative_difficulty);
 		}
@@ -972,15 +972,18 @@ bool BlockChainState::redo_block(const Hash &bhash, const Block &block, const ap
 	if (std::chrono::duration_cast<std::chrono::milliseconds>(now - log_redo_block_timestamp).count() > 1000) {
 		log_redo_block_timestamp = now;
 		std::cout << "redo_block {" << block.transactions.size() << "} height=" << info.height
-		          << " bid=" << common::pod_to_hex(bhash) << " old tip_bid=" << common::pod_to_hex(get_tip_bid())
-		          << std::endl;
+		          << " bid=" << common::pod_to_hex(bhash) << std::endl;
 	}
+	m_log(logging::TRACE) << "redo_block {" << block.transactions.size() << "} height=" << info.height
+		          << " bid=" << common::pod_to_hex(bhash) << std::endl;
 	return true;
 }
 
 void BlockChainState::undo_block(const Hash &bhash, const Block &block, Height height) {
-	if (height % 100 == 0)
-		std::cout << "undo_block height=" << height << " bid=" << common::pod_to_hex(bhash)
+//	if (height % 100 == 0)
+//		std::cout << "undo_block height=" << height << " bid=" << common::pod_to_hex(bhash)
+//		          << " new tip_bid=" << common::pod_to_hex(block.header.previous_block_hash) << std::endl;
+	m_log(logging::INFO) << "undo_block height=" << height << " bid=" << common::pod_to_hex(bhash)
 		          << " new tip_bid=" << common::pod_to_hex(block.header.previous_block_hash) << std::endl;
 	for (auto tit = block.transactions.rbegin(); tit != block.transactions.rend(); ++tit) {
 		undo_transaction(this, height, *tit);

src/Core/BlockChainState.hpp

@@ -164,7 +164,7 @@ class BlockChainState : public BlockChain, private IBlockChainState {
 	Timestamp m_next_unlock_timestamp = 0;
 	uint32_t m_next_median_size       = 0;
 	virtual void tip_changed() override;  // Updates values above
-	void calculate_consensus_values(Height height_delta, uint32_t &next_median_size, Timestamp &next_median_timestamp,
+	void calculate_consensus_values(const api::BlockHeader & prev_info, uint32_t &next_median_size, Timestamp &next_median_timestamp,
 	    Timestamp &next_unlock_timestamp) const;
 
 	RingCheckerMulticore ring_checker;

src/Core/Node.cpp

@@ -31,12 +31,12 @@ Node::Node(logging::ILogger &log, const Config &config, BlockChainState &block_c
 
 	if (!config.is_testnet) {
 		m_block_chain_reader1 =
-		    std::make_unique<LegacyBlockChainReader>(new_path + "/blockindexesx.bin", new_path + "/blocksx.bin");
+		    std::make_unique<LegacyBlockChainReader>(new_path + "/blockindexes.bin", new_path + "/blocks.bin");
 		if (m_block_chain_reader1->get_block_count() <= block_chain.get_tip_height())
 			m_block_chain_reader1.reset();
 		if (new_path != old_path) {  // Current situation on Linux
 			m_block_chain_reader2 =
-			    std::make_unique<LegacyBlockChainReader>(old_path + "/blockindexesx.bin", old_path + "/blocksx.bin");
+			    std::make_unique<LegacyBlockChainReader>(old_path + "/blockindexes.bin", old_path + "/blocks.bin");
 			if (m_block_chain_reader2->get_block_count() <= block_chain.get_tip_height())
 				m_block_chain_reader2.reset();
 		}