[NOD-289] Implement API-Server bootstrapping and booting after downti…

…me (#408)

* [NOD-289] Implemented database isCurrent checking and connection.

* [NOD-289] Added GetChainFromBlock to RPCClient.

* [NOD-289] Limited the amount of blocks in GetChainFromBlockResponse.

* [NOD-289] Fixed various issues that were keeping GetChainFromBlocks from working properly.

* [NOD-289] Created blockloop.go.

* [NOD-289] Updated go.mod after merge.

* [NOD-289] Implemented collection of current selected parent chain.

* [NOD-289] Fixed test. Reverted not deleting utxoDiffData from the DB.

* [NOD-289] Implemented GetBlocks.

* [NOD-289] Added comment to BlockHashesFrom.

* [NOD-289] Added GetBlocks to rpcclient.

* [NOD-289] Added verboseBlocks to GetBlocks.

* [NOD-289] Implemented block insertion.

* [NOD-289] Added AUTO_INCREMENT to tables that were missing it.

* [NOD-289] Made gasLimit in subnetwork nullable.

* [NOD-289] Renamed transactions_outputs to transaction_outputs.

* [NOD-289] Fixed weird coinbase behavior in vin.

* [NOD-289] Made collectCurrentBlocks start from the most recent startHash.

* [NOD-289] Added IsChainBlock to GetBlockVerboseResult.

* [NOD-289] Implemented adding a block from onBlockAdded.

* [NOD-289] Added removedParentChainHashes to getChainFromBlock.

* [NOD-289] Implemented updating the selected parent chain from onChainChanged.

* [NOD-289] Implemented some initial logic for updating the UTXO.

* [NOD-289] Fixed merge errors.

* [NOD-326] Fixed some more merge errors.

* [NOD-289] Added error handling for missing required records.

* [NOD-289] Implemented handling removedChainHashes.

* [NOD-289] Implemented handling addedChainBlocks.

* [NOD-289] Fixed incorrect coinbase check.

* [NOD-289] Implemented inserting the transaction output address.

* [NOD-289] Added updating block.IsChainBlock.

* [NOD-289] Split insertBlock into many small functions.

* [NOD-289] Split updateSelectedParentChain into smaller functions.

* [NOD-289] Fixed pointer errors.

* [NOD-289] Fixed a bad exists check.

* [NOD-289] Fixed a couple of small bugs.

* [NOD-289] Fixed a TxID/Hash mixup.

* [NOD-289] Added block/tx mass to getBlockVerboseResponse.

* [NOD-289] Renamed blockLoop.go to sync.go. Added comments.

* [NOD-289] Deleted apiserver README.

* [NOD-289] Fixed golint errors.

* [NOD-289] Renamed findMostRecentBlockHash to findHashOfBluestBlock.

* [NOD-289] Fixed style in syncBlocks and fixed a comment.

* [NOD-289] Copied NewErrorFromDBErrors over from NOD-324.

* [NOD-289] Created a couple of utils to make error handling with gorm slightly less painful.

* [NOD-289] Added error handling for database calls.

* [NOD-289] Fixed some more style/comments.

* [NOD-289] Fixed comments.

* [NOD-289] Renamed TransactionInput.TransactionOutput to TransactionInput.PreviousTransactionOutput.

* [NOD-289] Added a commends about pagination in getBlocks and getChainFromBlock.

* [NOD-289] Removed the coinbase field from Vin.

* [NOD-289] Deferred handling chainChangedMsgs until we have the appropriate data.

* [NOD-289] Optimized queries in updateRemovedChainHashes and updateAddedChainBlocks.

* [NOD-289] Optimized queries in insertBlockParents.

* [NOD-289] Optimized queries in insertTransactionInput.

* [NOD-289] Split Where calls to separate lines.

* [NOD-289] Fixed merge errors.

* [NOD-289] Exited early from insertBlockParents if we're the genesis block.

* [NOD-289] Improved nextChainChangedChan mechanism.

* [NOD-289] Fixed the above sync mechanism a bit.

* [NOD-289] Renamed IsDBRecordNotFoundError to HasDBRecordNotFoundError and IsDBError to HasDBError.

* [NOD-289] Replaced old error handling for db errors with the lovely new stuff.

* [NOD-289] Exited early if we already inserted a block. This saves us checking if a record already exists for some record types.

* [NOD-289] Decoupled syncBlocks from syncSelectedParentChain.

* [NOD-289] Made a comment more explicit.

* [NOD-289] Extracted net resolution to a separate function.

* [NOD-289] Extracted syncing to a separate function.

* [NOD-289] Fixed a comment.

* [NOD-289] Fixed merge erros.

* [NOD-289] Fixed a couple of bugs.

* [NOD-289] Fixed another bug.

* [NOD-289] Extracted ChainChangedMsg conversion to a separate function.

* [NOD-289] Optimized queries in canHandleChainChangedMsg.

* [NOD-289] Moved the sync function closer to its call site.

* [NOD-289] Renamed HasDBRecordNotFoundError to IsDBRecordNotFoundError.

* [NOD-289] Used count instead of first.

* [NOD-289] Renamed address to hexAddress.

apiserver/config/config.go

@@ -3,6 +3,7 @@ package config
 import (
 	"errors"
 	"github.com/daglabs/btcd/apiserver/logger"
+	"github.com/daglabs/btcd/dagconfig"
 	"github.com/daglabs/btcd/util"
 	"github.com/jessevdk/go-flags"
 	"path/filepath"
@@ -13,10 +14,15 @@ const (
 	defaultErrLogFilename = "apiserver_err.log"
 )
 
+var (
+	// activeNetParams are the currently active net params
+	activeNetParams dagconfig.Params
+)
+
 var (
 	// Default configuration options
 	defaultLogDir     = util.AppDataDir("apiserver", false)
-	defaultDBAddr     = "localhost:3306"
+	defaultDBAddress  = "localhost:3306"
 	defaultHTTPListen = "0.0.0.0:8080"
 )
 
@@ -34,13 +40,16 @@ type Config struct {
 	DBName      string `long:"dbname" description:"Database name" required:"true"`
 	HTTPListen  string `long:"listen" description:"HTTP address to listen on (default: 0.0.0.0:8080)"`
 	Migrate     bool   `long:"migrate" description:"Migrate the database to the latest version. The server will not start when using this flag."`
+	TestNet     bool   `long:"testnet" description:"Connect to testnet"`
+	SimNet      bool   `long:"simnet" description:"Connect to the simulation test network"`
+	DevNet      bool   `long:"devnet" description:"Connect to the development test network"`
 }
 
 // Parse parses the CLI arguments and returns a config struct.
 func Parse() (*Config, error) {
 	cfg := &Config{
 		LogDir:     defaultLogDir,
-		DBAddress:  defaultDBAddr,
+		DBAddress:  defaultDBAddress,
 		HTTPListen: defaultHTTPListen,
 	}
 	parser := flags.NewParser(cfg, flags.PrintErrors|flags.HelpFlag)
@@ -69,9 +78,49 @@ func Parse() (*Config, error) {
 		return nil, errors.New("--cert should be omitted if --notls is used")
 	}
 
+	err = resolveNetwork(cfg)
+	if err != nil {
+		return nil, err
+	}
+
 	logFile := filepath.Join(cfg.LogDir, defaultLogFilename)
 	errLogFile := filepath.Join(cfg.LogDir, defaultErrLogFilename)
 	logger.InitLog(logFile, errLogFile)
 
 	return cfg, nil
 }
+
+func resolveNetwork(cfg *Config) error {
+	// Multiple networks can't be selected simultaneously.
+	numNets := 0
+	if cfg.TestNet {
+		numNets++
+	}
+	if cfg.SimNet {
+		numNets++
+	}
+	if cfg.DevNet {
+		numNets++
+	}
+	if numNets > 1 {
+		return errors.New("multiple net params (testnet, simnet, devnet, etc.) can't be used " +
+			"together -- choose one of them")
+	}
+
+	activeNetParams = dagconfig.MainNetParams
+	switch {
+	case cfg.TestNet:
+		activeNetParams = dagconfig.TestNet3Params
+	case cfg.SimNet:
+		activeNetParams = dagconfig.SimNetParams
+	case cfg.DevNet:
+		activeNetParams = dagconfig.DevNetParams
+	}
+
+	return nil
+}
+
+// ActiveNetParams returns the currently active net params
+func ActiveNetParams() *dagconfig.Params {
+	return &activeNetParams
+}

apiserver/controllers/block.go

@@ -39,10 +39,11 @@ func GetBlockByHashHandler(blockHash string) (interface{}, *utils.HandlerError)
 
 	block := &models.Block{}
 	dbResult := db.Where(&models.Block{BlockHash: blockHash}).Preload("AcceptingBlock").First(block)
-	if dbResult.RecordNotFound() && len(dbResult.GetErrors()) == 1 {
+	dbErrors := dbResult.GetErrors()
+	if utils.IsDBRecordNotFoundError(dbErrors) {
 		return nil, utils.NewHandlerError(http.StatusNotFound, "No block with the given block hash was found.")
 	}
-	if len(dbResult.GetErrors()) > 0 {
+	if utils.HasDBError(dbErrors) {
 		return nil, utils.NewHandlerErrorFromDBErrors("Some errors where encountered when loading transactions from the database:", dbResult.GetErrors())
 	}
 	return convertBlockModelToBlockResponse(block), nil

apiserver/controllers/response_types.go

@@ -82,11 +82,11 @@ func convertTxModelToTxResponse(tx *models.Transaction) *transactionResponse {
 	}
 	for i, txIn := range tx.TransactionInputs {
 		txRes.Inputs[i] = &transactionInputResponse{
-			PreviousTransactionID:          txIn.TransactionOutput.Transaction.TransactionID,
-			PreviousTransactionOutputIndex: txIn.TransactionOutput.Index,
+			PreviousTransactionID:          txIn.PreviousTransactionOutput.Transaction.TransactionID,
+			PreviousTransactionOutputIndex: txIn.PreviousTransactionOutput.Index,
 			SignatureScript:                hex.EncodeToString(txIn.SignatureScript),
 			Sequence:                       txIn.Sequence,
-			Address:                        txIn.TransactionOutput.Address.Address,
+			Address:                        txIn.PreviousTransactionOutput.Address.Address,
 		}
 	}
 	return txRes

apiserver/controllers/transaction.go

@@ -34,11 +34,12 @@ func GetTransactionByIDHandler(txID string) (interface{}, *utils.HandlerError) {
 	tx := &models.Transaction{}
 	query := db.Where(&models.Transaction{TransactionID: txID})
 	dbResult := addTxPreloadedFields(query).First(&tx)
-	if dbResult.RecordNotFound() && len(dbResult.GetErrors()) == 1 {
+	dbErrors := dbResult.GetErrors()
+	if utils.IsDBRecordNotFoundError(dbErrors) {
 		return nil, utils.NewHandlerError(http.StatusNotFound, "No transaction with the given txid was found.")
 	}
-	if len(dbResult.GetErrors()) > 0 {
-		return nil, utils.NewHandlerErrorFromDBErrors("Some errors where encountered when loading transaction from the database:", dbResult.GetErrors())
+	if utils.HasDBError(dbErrors) {
+		return nil, utils.NewHandlerErrorFromDBErrors("Some errors where encountered when loading transaction from the database:", dbErrors)
 	}
 	return convertTxModelToTxResponse(tx), nil
 }
@@ -58,11 +59,12 @@ func GetTransactionByHashHandler(txHash string) (interface{}, *utils.HandlerErro
 	tx := &models.Transaction{}
 	query := db.Where(&models.Transaction{TransactionHash: txHash})
 	dbResult := addTxPreloadedFields(query).First(&tx)
-	if dbResult.RecordNotFound() && len(dbResult.GetErrors()) == 1 {
+	dbErrors := dbResult.GetErrors()
+	if utils.IsDBRecordNotFoundError(dbErrors) {
 		return nil, utils.NewHandlerError(http.StatusNotFound, "No transaction with the given txhash was found.")
 	}
-	if len(dbResult.GetErrors()) > 0 {
-		return nil, utils.NewHandlerErrorFromDBErrors("Some errors where encountered when loading transaction from the database:", dbResult.GetErrors())
+	if utils.HasDBError(dbErrors) {
+		return nil, utils.NewHandlerErrorFromDBErrors("Some errors where encountered when loading transaction from the database:", dbErrors)
 	}
 	return convertTxModelToTxResponse(tx), nil
 }
@@ -92,8 +94,9 @@ func GetTransactionsByAddressHandler(address string, skip uint64, limit uint64)
 		Limit(limit).
 		Offset(skip).
 		Order("`transactions`.`id` ASC")
-	dbErrors := addTxPreloadedFields(query).Find(&txs).GetErrors()
-	if len(dbErrors) > 0 {
+	dbResult := addTxPreloadedFields(query).Find(&txs)
+	dbErrors := dbResult.GetErrors()
+	if utils.HasDBError(dbErrors) {
 		return nil, utils.NewHandlerErrorFromDBErrors("Some errors where encountered when loading transactions from the database:", dbErrors)
 	}
 	txResponses := make([]*transactionResponse, len(txs))
@@ -137,8 +140,8 @@ func addTxPreloadedFields(query *gorm.DB) *gorm.DB {
 		Preload("Subnetwork").
 		Preload("TransactionOutputs").
 		Preload("TransactionOutputs.Address").
-		Preload("TransactionInputs.TransactionOutput.Transaction").
-		Preload("TransactionInputs.TransactionOutput.Address")
+		Preload("TransactionInputs.PreviousTransactionOutput.Transaction").
+		Preload("TransactionInputs.PreviousTransactionOutput.Address")
 }
 
 // PostTransaction forwards a raw transaction to the JSON-RPC API server

apiserver/jsonrpc/client.go

@@ -17,8 +17,8 @@ import (
 // Client represents a connection to the JSON-RPC API of a full node
 type Client struct {
 	*rpcclient.Client
-	onBlockAdded   chan *blockAddedMsg
-	onChainChanged chan *chainChangedMsg
+	OnBlockAdded   chan *BlockAddedMsg
+	OnChainChanged chan *ChainChangedMsg
 }
 
 var client *Client
@@ -32,14 +32,16 @@ func GetClient() (*Client, error) {
 	return client, nil
 }
 
-type blockAddedMsg struct {
-	chainHeight uint64
-	header      *wire.BlockHeader
+// BlockAddedMsg defines the message received in onBlockAdded
+type BlockAddedMsg struct {
+	ChainHeight uint64
+	Header      *wire.BlockHeader
 }
 
-type chainChangedMsg struct {
-	removedChainBlockHashes []*daghash.Hash
-	addedChainBlocks        []*rpcclient.ChainBlock
+// ChainChangedMsg defines the message received in onChainChanged
+type ChainChangedMsg struct {
+	RemovedChainBlockHashes []*daghash.Hash
+	AddedChainBlocks        []*rpcclient.ChainBlock
 }
 
 // Close closes the connection to the JSON-RPC API server
@@ -87,22 +89,22 @@ func Connect(cfg *config.Config) error {
 
 func newClient(connCfg *rpcclient.ConnConfig) (*Client, error) {
 	client = &Client{
-		onBlockAdded:   make(chan *blockAddedMsg),
-		onChainChanged: make(chan *chainChangedMsg),
+		OnBlockAdded:   make(chan *BlockAddedMsg),
+		OnChainChanged: make(chan *ChainChangedMsg),
 	}
 	notificationHandlers := &rpcclient.NotificationHandlers{
 		OnFilteredBlockAdded: func(height uint64, header *wire.BlockHeader,
 			txs []*util.Tx) {
-			client.onBlockAdded <- &blockAddedMsg{
-				chainHeight: height,
-				header:      header,
+			client.OnBlockAdded <- &BlockAddedMsg{
+				ChainHeight: height,
+				Header:      header,
 			}
 		},
 		OnChainChanged: func(removedChainBlockHashes []*daghash.Hash,
 			addedChainBlocks []*rpcclient.ChainBlock) {
-			client.onChainChanged <- &chainChangedMsg{
-				removedChainBlockHashes: removedChainBlockHashes,
-				addedChainBlocks:        addedChainBlocks,
+			client.OnChainChanged <- &ChainChangedMsg{
+				RemovedChainBlockHashes: removedChainBlockHashes,
+				AddedChainBlocks:        addedChainBlocks,
 			}
 		},
 	}

apiserver/main.go

@@ -57,6 +57,17 @@ func main() {
 	shutdownServer := server.Start(cfg.HTTPListen)
 	defer shutdownServer()
 
+	doneChan := make(chan struct{}, 1)
+	spawn(func() {
+		err := startSync(doneChan)
+		if err != nil {
+			panic(err)
+		}
+	})
+
 	interrupt := signal.InterruptListener()
 	<-interrupt
+
+	// Gracefully stop syncing
+	doneChan <- struct{}{}
 }

apiserver/migrations/000004_create_subnetworks_table.up.sql

@@ -1,8 +1,8 @@
 CREATE TABLE `subnetworks`
 (
-    `id`            BIGINT UNSIGNED NOT NULL,
+    `id`            BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
     `subnetwork_id` CHAR(64)     NOT NULL,
-    `gas_limit`     BIGINT UNSIGNED NOT NULL,
+    `gas_limit`     BIGINT UNSIGNED NULL,
     PRIMARY KEY (`id`),
     UNIQUE INDEX `idx_subnetworks_subnetwork_id` (`subnetwork_id`)
 );

apiserver/migrations/000009_create_transaction_inputs_table.up.sql

@@ -1,18 +1,18 @@
 CREATE TABLE `transaction_inputs`
 (
-    `id`                    BIGINT UNSIGNED NOT NULL,
-    `transaction_id`        BIGINT UNSIGNED NULL,
-    `transaction_output_id` BIGINT UNSIGNED NOT NULL,
-    `index`                 INT UNSIGNED    NOT NULL,
-    `signature_script`      BLOB            NOT NULL,
-    `sequence`              BIGINT UNSIGNED NOT NULL,
+    `id`                             BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
+    `transaction_id`                 BIGINT UNSIGNED NULL,
+    `previous_transaction_output_id` BIGINT UNSIGNED NOT NULL,
+    `index`                          INT UNSIGNED    NOT NULL,
+    `signature_script`               BLOB            NOT NULL,
+    `sequence`                       BIGINT UNSIGNED NOT NULL,
     PRIMARY KEY (`id`),
     INDEX `idx_transaction_inputs_transaction_id` (`transaction_id`),
-    INDEX `idx_transaction_inputs_transaction_output_id` (`transaction_output_id`),
+    INDEX `idx_transaction_inputs_previous_transaction_output_id` (`previous_transaction_output_id`),
     CONSTRAINT `fk_transaction_inputs_transaction_id`
         FOREIGN KEY (`transaction_id`)
             REFERENCES `transactions` (`id`),
-    CONSTRAINT `fk_transaction_inputs_transaction_output_id`
-        FOREIGN KEY (`transaction_output_id`)
+    CONSTRAINT `fk_transaction_inputs_previous_transaction_output_id`
+        FOREIGN KEY (`previous_transaction_output_id`)
             REFERENCES `transaction_outputs` (`id`)
 );

apiserver/models/models.go

@@ -8,7 +8,7 @@ import (
 type Block struct {
 	ID                   uint64 `gorm:"primary_key"`
 	BlockHash            string
-	AcceptingBlockID     uint64
+	AcceptingBlockID     *uint64
 	AcceptingBlock       *Block
 	Version              int32
 	HashMerkleRoot       string
@@ -42,13 +42,14 @@ type RawBlock struct {
 type Subnetwork struct {
 	ID           uint64 `gorm:"primary_key"`
 	SubnetworkID string
+	GasLimit     *uint64
 }
 
 // Transaction is the gorm model for the 'transactions' table
 type Transaction struct {
 	ID                 uint64 `gorm:"primary_key"`
-	AcceptingBlockID   uint64
-	AcceptingBlock     Block
+	AcceptingBlockID   *uint64
+	AcceptingBlock     *Block
 	TransactionHash    string
 	TransactionID      string
 	LockTime           uint64
@@ -93,14 +94,14 @@ type TransactionOutput struct {
 
 // TransactionInput is the gorm model for the 'transaction_inputs' table
 type TransactionInput struct {
-	ID                  uint64 `gorm:"primary_key"`
-	TransactionID       uint64
-	Transaction         Transaction
-	TransactionOutputID uint64
-	TransactionOutput   TransactionOutput
-	Index               uint32
-	SignatureScript     []byte
-	Sequence            uint64
+	ID                          uint64 `gorm:"primary_key"`
+	TransactionID               uint64
+	Transaction                 Transaction
+	PreviousTransactionOutputID uint64
+	PreviousTransactionOutput   TransactionOutput
+	Index                       uint32
+	SignatureScript             []byte
+	Sequence                    uint64
 }
 
 // Address is the gorm model for the 'utxos' table