From 62e56fb5d52cf997ccb5535f83d5ea5bfb9fb24b Mon Sep 17 00:00:00 2001 From: Santiago Carmuega Date: Thu, 14 Dec 2023 19:00:57 -0300 Subject: [PATCH] feat: implement protocol params update logic --- Cargo.lock | 29 ++--- Cargo.toml | 1 + examples/sync-mainnet/dolos.toml | 3 + examples/sync-mainnet/shelley.json | 68 ++++++++++++ examples/sync-preprod/dolos.toml | 3 + examples/sync-preprod/shelley.json | 72 +++++++++++++ examples/sync-preview/dolos.toml | 3 + examples/sync-preview/shelley.json | 68 ++++++++++++ src/bin/dolos/common.rs | 7 +- src/bin/dolos/daemon.rs | 5 + src/bin/dolos/eval.rs | 33 ++++-- src/bin/dolos/main.rs | 1 + src/bin/dolos/sync.rs | 4 + src/storage/applydb/mod.rs | 129 ++++++++++++----------- src/storage/kvtable.rs | 13 +++ src/sync/ledger.rs | 139 +++++++++++++++--------- src/sync/mod.rs | 8 +- src/sync/pparams.rs | 164 +++++++++++++++++++++++++++++ 18 files changed, 606 insertions(+), 144 deletions(-) create mode 100644 examples/sync-mainnet/shelley.json create mode 100644 examples/sync-preprod/shelley.json create mode 100644 examples/sync-preview/shelley.json create mode 100644 src/sync/pparams.rs diff --git a/Cargo.lock b/Cargo.lock index f19cc91f..c9d82b63 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1901,7 +1901,7 @@ checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" [[package]] name = "pallas" version = "0.20.0" -source = "git+https://github.com/txpipe/pallas.git#04232c6a4ceea115293cb13bd2928ae2eee88daf" +source = "git+https://github.com/txpipe/pallas.git#408a41a9ea8466111620150bd59347d2c8615e98" dependencies = [ "pallas-addresses", "pallas-applying", @@ -1921,7 +1921,7 @@ dependencies = [ [[package]] name = "pallas-addresses" version = "0.20.0" -source = "git+https://github.com/txpipe/pallas.git#04232c6a4ceea115293cb13bd2928ae2eee88daf" +source = "git+https://github.com/txpipe/pallas.git#408a41a9ea8466111620150bd59347d2c8615e98" dependencies = [ "base58", "bech32 0.9.1", @@ -1936,7 +1936,7 @@ dependencies = [ [[package]] name = "pallas-applying" version = "0.20.0" -source = "git+https://github.com/txpipe/pallas.git#04232c6a4ceea115293cb13bd2928ae2eee88daf" +source = "git+https://github.com/txpipe/pallas.git#408a41a9ea8466111620150bd59347d2c8615e98" dependencies = [ "pallas-addresses", "pallas-codec 0.20.0 (git+https://github.com/txpipe/pallas.git)", @@ -1961,7 +1961,7 @@ dependencies = [ [[package]] name = "pallas-codec" version = "0.20.0" -source = "git+https://github.com/txpipe/pallas.git#04232c6a4ceea115293cb13bd2928ae2eee88daf" +source = "git+https://github.com/txpipe/pallas.git#408a41a9ea8466111620150bd59347d2c8615e98" dependencies = [ "hex", "minicbor", @@ -1972,7 +1972,7 @@ dependencies = [ [[package]] name = "pallas-configs" version = "0.20.0" -source = "git+https://github.com/txpipe/pallas.git#04232c6a4ceea115293cb13bd2928ae2eee88daf" +source = "git+https://github.com/txpipe/pallas.git#408a41a9ea8466111620150bd59347d2c8615e98" dependencies = [ "base64 0.21.5", "hex", @@ -2000,7 +2000,7 @@ dependencies = [ [[package]] name = "pallas-crypto" version = "0.20.0" -source = "git+https://github.com/txpipe/pallas.git#04232c6a4ceea115293cb13bd2928ae2eee88daf" +source = "git+https://github.com/txpipe/pallas.git#408a41a9ea8466111620150bd59347d2c8615e98" dependencies = [ "cryptoxide", "hex", @@ -2013,7 +2013,7 @@ dependencies = [ [[package]] name = "pallas-hardano" version = "0.20.0" -source = "git+https://github.com/txpipe/pallas.git#04232c6a4ceea115293cb13bd2928ae2eee88daf" +source = "git+https://github.com/txpipe/pallas.git#408a41a9ea8466111620150bd59347d2c8615e98" dependencies = [ "binary-layout", "tap", @@ -2039,7 +2039,7 @@ dependencies = [ [[package]] name = "pallas-network" version = "0.20.0" -source = "git+https://github.com/txpipe/pallas.git#04232c6a4ceea115293cb13bd2928ae2eee88daf" +source = "git+https://github.com/txpipe/pallas.git#408a41a9ea8466111620150bd59347d2c8615e98" dependencies = [ "byteorder", "hex", @@ -2054,7 +2054,7 @@ dependencies = [ [[package]] name = "pallas-primitives" version = "0.20.0" -source = "git+https://github.com/txpipe/pallas.git#04232c6a4ceea115293cb13bd2928ae2eee88daf" +source = "git+https://github.com/txpipe/pallas.git#408a41a9ea8466111620150bd59347d2c8615e98" dependencies = [ "base58", "bech32 0.9.1", @@ -2069,7 +2069,7 @@ dependencies = [ [[package]] name = "pallas-rolldb" version = "0.20.0" -source = "git+https://github.com/txpipe/pallas.git#04232c6a4ceea115293cb13bd2928ae2eee88daf" +source = "git+https://github.com/txpipe/pallas.git#408a41a9ea8466111620150bd59347d2c8615e98" dependencies = [ "async-stream", "bincode", @@ -2086,13 +2086,14 @@ dependencies = [ [[package]] name = "pallas-traverse" version = "0.20.0" -source = "git+https://github.com/txpipe/pallas.git#04232c6a4ceea115293cb13bd2928ae2eee88daf" +source = "git+https://github.com/txpipe/pallas.git#408a41a9ea8466111620150bd59347d2c8615e98" dependencies = [ "hex", "pallas-addresses", "pallas-codec 0.20.0 (git+https://github.com/txpipe/pallas.git)", "pallas-crypto 0.20.0 (git+https://github.com/txpipe/pallas.git)", "pallas-primitives", + "paste", "serde", "thiserror", ] @@ -2100,7 +2101,7 @@ dependencies = [ [[package]] name = "pallas-txbuilder" version = "0.20.0" -source = "git+https://github.com/txpipe/pallas.git#04232c6a4ceea115293cb13bd2928ae2eee88daf" +source = "git+https://github.com/txpipe/pallas.git#408a41a9ea8466111620150bd59347d2c8615e98" dependencies = [ "hex", "pallas-addresses", @@ -2116,7 +2117,7 @@ dependencies = [ [[package]] name = "pallas-utxorpc" version = "0.20.0" -source = "git+https://github.com/txpipe/pallas.git#04232c6a4ceea115293cb13bd2928ae2eee88daf" +source = "git+https://github.com/txpipe/pallas.git#408a41a9ea8466111620150bd59347d2c8615e98" dependencies = [ "pallas-codec 0.20.0 (git+https://github.com/txpipe/pallas.git)", "pallas-primitives", @@ -2127,7 +2128,7 @@ dependencies = [ [[package]] name = "pallas-wallet" version = "0.20.0" -source = "git+https://github.com/txpipe/pallas.git#04232c6a4ceea115293cb13bd2928ae2eee88daf" +source = "git+https://github.com/txpipe/pallas.git#408a41a9ea8466111620150bd59347d2c8615e98" dependencies = [ "bech32 0.9.1", "bip39", diff --git a/Cargo.toml b/Cargo.toml index 287fef35..388ea92e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,7 @@ authors = ["Santiago Carmuega "] [dependencies] pallas = { git = "https://github.com/txpipe/pallas.git", features = ["unstable"] } +# pallas = { path = "../pallas/pallas", features = ["unstable"] } gasket = { version = "^0.5", features = ["derive"] } # gasket = { path = "../../construkts/gasket-rs/gasket", features = ["derive"] } diff --git a/examples/sync-mainnet/dolos.toml b/examples/sync-mainnet/dolos.toml index c1acffd6..569fa2bf 100644 --- a/examples/sync-mainnet/dolos.toml +++ b/examples/sync-mainnet/dolos.toml @@ -16,3 +16,6 @@ magic = 0 [byron] path = "./byron.json" + +[shelley] +path = "./shelley.json" diff --git a/examples/sync-mainnet/shelley.json b/examples/sync-mainnet/shelley.json new file mode 100644 index 00000000..e8cc20dc --- /dev/null +++ b/examples/sync-mainnet/shelley.json @@ -0,0 +1,68 @@ +{ + "activeSlotsCoeff": 0.05, + "protocolParams": { + "protocolVersion": { + "minor": 0, + "major": 2 + }, + "decentralisationParam": 1, + "eMax": 18, + "extraEntropy": { + "tag": "NeutralNonce" + }, + "maxTxSize": 16384, + "maxBlockBodySize": 65536, + "maxBlockHeaderSize": 1100, + "minFeeA": 44, + "minFeeB": 155381, + "minUTxOValue": 1000000, + "poolDeposit": 500000000, + "minPoolCost": 340000000, + "keyDeposit": 2000000, + "nOpt": 150, + "rho": 0.003, + "tau": 0.20, + "a0": 0.3 + }, + "genDelegs": { + "ad5463153dc3d24b9ff133e46136028bdc1edbb897f5a7cf1b37950c": { + "delegate": "d9e5c76ad5ee778960804094a389f0b546b5c2b140a62f8ec43ea54d", + "vrf": "64fa87e8b29a5b7bfbd6795677e3e878c505bc4a3649485d366b50abadec92d7" + }, + "b9547b8a57656539a8d9bc42c008e38d9c8bd9c8adbb1e73ad529497": { + "delegate": "855d6fc1e54274e331e34478eeac8d060b0b90c1f9e8a2b01167c048", + "vrf": "66d5167a1f426bd1adcc8bbf4b88c280d38c148d135cb41e3f5a39f948ad7fcc" + }, + "60baee25cbc90047e83fd01e1e57dc0b06d3d0cb150d0ab40bbfead1": { + "delegate": "7f72a1826ae3b279782ab2bc582d0d2958de65bd86b2c4f82d8ba956", + "vrf": "c0546d9aa5740afd569d3c2d9c412595cd60822bb6d9a4e8ce6c43d12bd0f674" + }, + "f7b341c14cd58fca4195a9b278cce1ef402dc0e06deb77e543cd1757": { + "delegate": "69ae12f9e45c0c9122356c8e624b1fbbed6c22a2e3b4358cf0cb5011", + "vrf": "6394a632af51a32768a6f12dac3485d9c0712d0b54e3f389f355385762a478f2" + }, + "162f94554ac8c225383a2248c245659eda870eaa82d0ef25fc7dcd82": { + "delegate": "4485708022839a7b9b8b639a939c85ec0ed6999b5b6dc651b03c43f6", + "vrf": "aba81e764b71006c515986bf7b37a72fbb5554f78e6775f08e384dbd572a4b32" + }, + "2075a095b3c844a29c24317a94a643ab8e22d54a3a3a72a420260af6": { + "delegate": "6535db26347283990a252313a7903a45e3526ec25ddba381c071b25b", + "vrf": "fcaca997b8105bd860876348fc2c6e68b13607f9bbd23515cd2193b555d267af" + }, + "268cfc0b89e910ead22e0ade91493d8212f53f3e2164b2e4bef0819b": { + "delegate": "1d4f2e1fda43070d71bb22a5522f86943c7c18aeb4fa47a362c27e23", + "vrf": "63ef48bc5355f3e7973100c371d6a095251c80ceb40559f4750aa7014a6fb6db" + } + }, + "updateQuorum": 5, + "networkId": "Mainnet", + "initialFunds": {}, + "maxLovelaceSupply": 45000000000000000, + "networkMagic": 764824073, + "epochLength": 432000, + "systemStart": "2017-09-23T21:44:51Z", + "slotsPerKESPeriod": 129600, + "slotLength": 1, + "maxKESEvolutions": 62, + "securityParam": 2160 +} diff --git a/examples/sync-preprod/dolos.toml b/examples/sync-preprod/dolos.toml index 1fe33d59..f9739e24 100644 --- a/examples/sync-preprod/dolos.toml +++ b/examples/sync-preprod/dolos.toml @@ -17,5 +17,8 @@ magic = 0 [byron] path = "./byron.json" +[shelley] +path = "./shelley.json" + [logging] max_level = "debug" diff --git a/examples/sync-preprod/shelley.json b/examples/sync-preprod/shelley.json new file mode 100644 index 00000000..24862eb8 --- /dev/null +++ b/examples/sync-preprod/shelley.json @@ -0,0 +1,72 @@ +{ + "activeSlotsCoeff": 0.05, + "epochLength": 432000, + "genDelegs": { + "637f2e950b0fd8f8e3e811c5fbeb19e411e7a2bf37272b84b29c1a0b": { + "delegate": "aae9293510344ddd636364c2673e34e03e79e3eefa8dbaa70e326f7d", + "vrf": "227116365af2ed943f1a8b5e6557bfaa34996f1578eec667a5e2b361c51e4ce7" + }, + "8a4b77c4f534f8b8cc6f269e5ebb7ba77fa63a476e50e05e66d7051c": { + "delegate": "d15422b2e8b60e500a82a8f4ceaa98b04e55a0171d1125f6c58f8758", + "vrf": "0ada6c25d62db5e1e35d3df727635afa943b9e8a123ab83785e2281605b09ce2" + }, + "b00470cd193d67aac47c373602fccd4195aad3002c169b5570de1126": { + "delegate": "b3b539e9e7ed1b32fbf778bf2ebf0a6b9f980eac90ac86623d11881a", + "vrf": "0ff0ce9b820376e51c03b27877cd08f8ba40318f1a9f85a3db0b60dd03f71a7a" + }, + "b260ffdb6eba541fcf18601923457307647dce807851b9d19da133ab": { + "delegate": "7c64eb868b4ef566391a321c85323f41d2b95480d7ce56ad2abcb022", + "vrf": "7fb22abd39d550c9a022ec8104648a26240a9ff9c88b8b89a6e20d393c03098e" + }, + "ced1599fd821a39593e00592e5292bdc1437ae0f7af388ef5257344a": { + "delegate": "de7ca985023cf892f4de7f5f1d0a7181668884752d9ebb9e96c95059", + "vrf": "c301b7fc4d1b57fb60841bcec5e3d2db89602e5285801e522fce3790987b1124" + }, + "dd2a7d71a05bed11db61555ba4c658cb1ce06c8024193d064f2a66ae": { + "delegate": "1e113c218899ee7807f4028071d0e108fc790dade9fd1a0d0b0701ee", + "vrf": "faf2702aa4893c877c622ab22dfeaf1d0c8aab98b837fe2bf667314f0d043822" + }, + "f3b9e74f7d0f24d2314ea5dfbca94b65b2059d1ff94d97436b82d5b4": { + "delegate": "fd637b08cc379ef7b99c83b416458fcda8a01a606041779331008fb9", + "vrf": "37f2ea7c843a688159ddc2c38a2f997ab465150164a9136dca69564714b73268" + } + }, + "initialFunds": {}, + "maxKESEvolutions": 62, + "maxLovelaceSupply": 45000000000000000, + "networkId": "Testnet", + "networkMagic": 1, + "protocolParams": { + "protocolVersion": { + "minor": 0, + "major": 2 + }, + "decentralisationParam": 1, + "eMax": 18, + "extraEntropy": { + "tag": "NeutralNonce" + }, + "maxTxSize": 16384, + "maxBlockBodySize": 65536, + "maxBlockHeaderSize": 1100, + "minFeeA": 44, + "minFeeB": 155381, + "minUTxOValue": 1000000, + "poolDeposit": 500000000, + "minPoolCost": 340000000, + "keyDeposit": 2000000, + "nOpt": 150, + "rho": 0.003, + "tau": 0.20, + "a0": 0.3 + }, + "securityParam": 2160, + "slotLength": 1, + "slotsPerKESPeriod": 129600, + "staking": { + "pools": {}, + "stake": {} + }, + "systemStart": "2022-06-01T00:00:00Z", + "updateQuorum": 5 +} diff --git a/examples/sync-preview/dolos.toml b/examples/sync-preview/dolos.toml index 7bbec484..c9eb2268 100644 --- a/examples/sync-preview/dolos.toml +++ b/examples/sync-preview/dolos.toml @@ -17,5 +17,8 @@ magic = 0 [byron] path = "./byron.json" +[shelley] +path = "./shelley.json" + [logging] max_level = "debug" diff --git a/examples/sync-preview/shelley.json b/examples/sync-preview/shelley.json new file mode 100644 index 00000000..20abba3f --- /dev/null +++ b/examples/sync-preview/shelley.json @@ -0,0 +1,68 @@ +{ + "activeSlotsCoeff": 0.05, + "epochLength": 86400, + "genDelegs": { + "12b0f443d02861948a0fce9541916b014e8402984c7b83ad70a834ce": { + "delegate": "7c54a168c731f2f44ced620f3cca7c2bd90731cab223d5167aa994e6", + "vrf": "62d546a35e1be66a2b06e29558ef33f4222f1c466adbb59b52d800964d4e60ec" + }, + "3df542796a64e399b60c74acfbdb5afa1e114532fa36b46d6368ef3a": { + "delegate": "c44bc2f3cc7e98c0f227aa399e4035c33c0d775a0985875fff488e20", + "vrf": "4f9d334decadff6eba258b2df8ae1f02580a2628bce47ae7d957e1acd3f42a3c" + }, + "93fd5083ff20e7ab5570948831730073143bea5a5d5539852ed45889": { + "delegate": "82a02922f10105566b70366b07c758c8134fa91b3d8ae697dfa5e8e0", + "vrf": "8a57e94a9b4c65ec575f35d41edb1df399fa30fdf10775389f5d1ef670ca3f9f" + }, + "a86cab3ea72eabb2e8aafbbf4abbd2ba5bdfd04eea26a39b126a78e4": { + "delegate": "10257f6d3bae913514bdc96c9170b3166bf6838cca95736b0e418426", + "vrf": "1b54aad6b013145a0fc74bb5c2aa368ebaf3999e88637d78e09706d0cc29874a" + }, + "b799804a28885bd49c0e1b99d8b3b26de0fac17a5cf651ecf0c872f0": { + "delegate": "ebe606e22d932d51be2c1ce87e7d7e4c9a7d1f7df4a5535c29e23d22", + "vrf": "b3fc06a1f8ee69ff23185d9af453503be8b15b2652e1f9fb7c3ded6797a2d6f9" + }, + "d125812d6ab973a2c152a0525b7fd32d36ff13555a427966a9cac9b1": { + "delegate": "e302198135fb5b00bfe0b9b5623426f7cf03179ab7ba75f945d5b79b", + "vrf": "b45ca2ed95f92248fa0322ce1fc9f815a5a5aa2f21f1adc2c42c4dccfc7ba631" + }, + "ef27651990a26449a40767d5e06cdef1670a3f3ff4b951d385b51787": { + "delegate": "0e0b11e80d958732e587585d30978d683a061831d1b753878f549d05", + "vrf": "b860ec844f6cd476c4fabb4aa1ca72d5c74d82f3835aed3c9515a35b6e048719" + } + }, + "initialFunds": {}, + "maxKESEvolutions": 62, + "maxLovelaceSupply": 45000000000000000, + "networkId": "Testnet", + "networkMagic": 2, + "protocolParams": { + "protocolVersion": { + "minor": 0, + "major": 6 + }, + "decentralisationParam": 1, + "eMax": 18, + "extraEntropy": { + "tag": "NeutralNonce" + }, + "maxTxSize": 16384, + "maxBlockBodySize": 65536, + "maxBlockHeaderSize": 1100, + "minFeeA": 44, + "minFeeB": 155381, + "minUTxOValue": 1000000, + "poolDeposit": 500000000, + "minPoolCost": 340000000, + "keyDeposit": 2000000, + "nOpt": 150, + "rho": 0.003, + "tau": 0.20, + "a0": 0.3 + }, + "securityParam": 432, + "slotLength": 1, + "slotsPerKESPeriod": 129600, + "systemStart": "2022-10-25T00:00:00Z", + "updateQuorum": 5 +} diff --git a/src/bin/dolos/common.rs b/src/bin/dolos/common.rs index e89f229c..29687129 100644 --- a/src/bin/dolos/common.rs +++ b/src/bin/dolos/common.rs @@ -28,12 +28,7 @@ pub fn open_data_stores(config: &crate::Config) -> Result { let chain = chain::Store::open(rolldb_path.join("chain")).map_err(Error::storage)?; - let ledger = ApplyDB::open( - rolldb_path.join("ledger"), - config.upstream.network_magic as u32, - config.upstream.network_id, - ) - .map_err(Error::storage)?; + let ledger = ApplyDB::open(rolldb_path.join("ledger")).map_err(Error::storage)?; Ok((wal, chain, ledger)) } diff --git a/src/bin/dolos/daemon.rs b/src/bin/dolos/daemon.rs index 73007d68..8fa708cd 100644 --- a/src/bin/dolos/daemon.rs +++ b/src/bin/dolos/daemon.rs @@ -13,6 +13,10 @@ pub async fn run(config: super::Config, _args: &Args) -> miette::Result<()> { .into_diagnostic() .context("loading byron genesis config")?; + let shelley_genesis = pallas::ledger::configs::shelley::from_file(&config.shelley.path) + .into_diagnostic() + .context("loading shelley genesis config")?; + let server = tokio::spawn(dolos::serve::serve( config.serve, wal.clone(), @@ -25,6 +29,7 @@ pub async fn run(config: super::Config, _args: &Args) -> miette::Result<()> { chain, ledger, byron_genesis, + shelley_genesis, &config.retries, ) .into_diagnostic() diff --git a/src/bin/dolos/eval.rs b/src/bin/dolos/eval.rs index 19d6ad20..aa9d8375 100644 --- a/src/bin/dolos/eval.rs +++ b/src/bin/dolos/eval.rs @@ -1,6 +1,6 @@ use miette::{Context, IntoDiagnostic}; use pallas::{ - applying::{validate, Environment, UTxOs}, + applying::{validate, UTxOs}, ledger::traverse::{Era, MultiEraInput, MultiEraOutput}, }; use std::{borrow::Cow, collections::HashMap, path::PathBuf}; @@ -14,7 +14,10 @@ pub struct Args { era: u16, #[arg(long, short)] - slot: u64, + epoch: u64, + + #[arg(long, short)] + network_id: u8, } pub fn run(config: &super::Config, args: &Args) -> miette::Result<()> { @@ -42,6 +45,14 @@ pub fn run(config: &super::Config, args: &Args) -> miette::Result<()> { .into_diagnostic() .context("resolving tx inputs")?; + let byron_genesis = pallas::ledger::configs::byron::from_file(&config.byron.path) + .into_diagnostic() + .context("loading byron genesis")?; + + let shelley_genesis = pallas::ledger::configs::shelley::from_file(&config.shelley.path) + .into_diagnostic() + .context("loading shelley genesis")?; + let mut utxos2 = UTxOs::new(); for (ref_, output) in utxos.iter() { @@ -64,12 +75,18 @@ pub fn run(config: &super::Config, args: &Args) -> miette::Result<()> { utxos2.insert(key, value); } - let env: Environment = ledger - .get_active_pparams(args.slot) - .into_diagnostic() - .context("resolving pparams")?; - - validate(&tx, &utxos2, &env).unwrap(); + let pparams = dolos::sync::pparams::compute_pparams( + dolos::sync::pparams::Genesis { + byron: &byron_genesis, + shelley: &shelley_genesis, + }, + &ledger, + args.epoch, + ) + .into_diagnostic() + .context("computing protocol params")?; + + validate(&tx, &utxos2, &pparams).unwrap(); Ok(()) } diff --git a/src/bin/dolos/main.rs b/src/bin/dolos/main.rs index b69291ef..7b40bfcb 100644 --- a/src/bin/dolos/main.rs +++ b/src/bin/dolos/main.rs @@ -66,6 +66,7 @@ pub struct Config { pub serve: dolos::serve::Config, pub retries: Option, pub byron: GenesisFileRef, + pub shelley: GenesisFileRef, #[serde(default)] pub logging: LoggingConfig, } diff --git a/src/bin/dolos/sync.rs b/src/bin/dolos/sync.rs index a7c98d81..32f985ee 100644 --- a/src/bin/dolos/sync.rs +++ b/src/bin/dolos/sync.rs @@ -12,12 +12,16 @@ pub fn run(config: &super::Config, _args: &Args) -> miette::Result<()> { let byron_genesis = pallas::ledger::configs::byron::from_file(&config.byron.path).map_err(Error::config)?; + let shelley_genesis = + pallas::ledger::configs::shelley::from_file(&config.shelley.path).map_err(Error::config)?; + dolos::sync::pipeline( &config.upstream, wal, chain, ledger, byron_genesis, + shelley_genesis, &config.retries, ) .into_diagnostic() diff --git a/src/storage/applydb/mod.rs b/src/storage/applydb/mod.rs index 4e019085..7f74a696 100644 --- a/src/storage/applydb/mod.rs +++ b/src/storage/applydb/mod.rs @@ -1,7 +1,4 @@ -pub mod genesis; - use pallas::{ - applying::types::{ByronProtParams, Environment, FeePolicy, MultiEraProtParams}, crypto::hash::Hash, ledger::traverse::{MultiEraBlock, MultiEraTx}, }; @@ -20,10 +17,15 @@ use crate::prelude::BlockHash; use super::kvtable::*; +pub mod genesis; + +type Epoch = u64; +type Era = u16; type TxHash = Hash<32>; type OutputIndex = u64; -type UtxoBody = (u16, Vec); +type UtxoBody = (Era, Vec); type BlockSlot = u64; +type UpdateBody = (Era, BlockHash, Vec); #[derive(Error, Debug)] pub enum Error { @@ -36,6 +38,9 @@ pub enum Error { #[error("missing stxi {0}#{1}")] MissingStxi(TxHash, OutputIndex), + #[error("missing pparams")] + MissingPParams, + #[error("cbor decoding")] Cbor, @@ -78,6 +83,13 @@ impl KVTable> for SlotKV { const CF_NAME: &'static str = "SlotKV"; } +// Spent transaction inputs +pub struct PParamsKV; + +impl KVTable> for PParamsKV { + const CF_NAME: &'static str = "PParamsKV"; +} + pub struct ApplyBatch<'a> { db: &'a rocksdb::DB, block_slot: BlockSlot, @@ -85,6 +97,7 @@ pub struct ApplyBatch<'a> { utxo_inserts: HashMap, stxi_inserts: HashMap, utxo_deletes: HashMap, + pparams_update: Option<(Epoch, UpdateBody)>, } impl<'a> ApplyBatch<'a> { @@ -96,6 +109,7 @@ impl<'a> ApplyBatch<'a> { utxo_inserts: HashMap::new(), stxi_inserts: HashMap::new(), utxo_deletes: HashMap::new(), + pparams_update: None, } } @@ -141,6 +155,10 @@ impl<'a> ApplyBatch<'a> { self.stxi_inserts.insert(k.clone(), body.clone()); self.utxo_deletes.insert(k.clone(), body); } + + pub fn update_pparams(&mut self, epoch: Epoch, era: Era, block: BlockHash, content: Vec) { + self.pparams_update = Some((epoch, (era, block, content))); + } } impl<'a> From> for WriteBatch { @@ -169,6 +187,10 @@ impl<'a> From> for WriteBatch { SlotKV::stage_upsert(from.db, k, v, &mut batch); + if let Some((key, value)) = from.pparams_update { + PParamsKV::stage_upsert(from.db, DBInt(key), DBSerde(value), &mut batch); + } + batch } } @@ -255,14 +277,10 @@ impl<'a> From> for WriteBatch { #[derive(Clone)] pub struct ApplyDB { db: Arc, - - // TODO: should be extracted from genesis config data - prot_magic: u32, - network_id: u8, } impl ApplyDB { - pub fn open(path: impl AsRef, prot_magic: u32, network_id: u8) -> Result { + pub fn open(path: impl AsRef) -> Result { let mut opts = Options::default(); opts.create_if_missing(true); opts.create_missing_column_families(true); @@ -270,15 +288,16 @@ impl ApplyDB { let db = DB::open_cf( &opts, path, - [UtxoKV::CF_NAME, StxiKV::CF_NAME, SlotKV::CF_NAME], + [ + UtxoKV::CF_NAME, + StxiKV::CF_NAME, + SlotKV::CF_NAME, + PParamsKV::CF_NAME, + ], ) .map_err(|_| super::kvtable::Error::IO)?; - Ok(Self { - db: Arc::new(db), - prot_magic, - network_id, - }) + Ok(Self { db: Arc::new(db) }) } pub fn is_empty(&self) -> bool { @@ -378,6 +397,25 @@ impl ApplyDB { batch.spend_utxo(hash, idx, utxo); }; } + + if let Some(update) = tx.update() { + batch.update_pparams( + update.epoch(), + block.era().into(), + block.hash(), + update.encode(), + ); + } + } + + // check block-level updates (because of f#!@#@ byron) + if let Some(update) = block.update() { + batch.update_pparams( + update.epoch(), + block.era().into(), + block.hash(), + update.encode(), + ); } let batch = WriteBatch::from(batch); @@ -389,54 +427,17 @@ impl ApplyDB { Ok(()) } - pub fn get_active_pparams(&self, block_slot: u64) -> Result { - if block_slot <= 322876 { - // These are the genesis values. - Ok(Environment { - prot_params: MultiEraProtParams::Byron(ByronProtParams { - fee_policy: FeePolicy { - summand: 155381, - multiplier: 44, - }, - max_tx_size: 4096, - }), - block_slot, - prot_magic: self.prot_magic, - network_id: self.network_id, - }) - } else if block_slot > 322876 && block_slot <= 1784895 { - // Block hash were the update proposal was submitted: - // 850805044e0df6c13ced2190db7b11489672b0225d478a35a6db71fbfb33afc0 - Ok(Environment { - prot_params: MultiEraProtParams::Byron(ByronProtParams { - fee_policy: FeePolicy { - summand: 155381, - multiplier: 44, - }, - max_tx_size: 65536, - }), - block_slot, - prot_magic: self.prot_magic, - network_id: self.network_id, - }) - } else if block_slot < 4492800 { - // Block hash were the update proposal was submitted: - // d798a8d617b25fc6456ffe2d90895a2c15a7271b671dab2d18d46f3d0e4ef495 - Ok(Environment { - prot_params: MultiEraProtParams::Byron(ByronProtParams { - fee_policy: FeePolicy { - summand: 155381, - multiplier: 44, - }, - max_tx_size: 8192, - }), - block_slot, - prot_magic: self.prot_magic, - network_id: self.network_id, - }) - } else { - Err(Error::UnimplementedEra) - } + pub fn get_pparams_updates(&self, until: Epoch) -> Result, Error> { + let matches = PParamsKV::iter_entries_start(&self.db) + .map(|x| x.map(|(k, v)| (k.0, v.unwrap()))) + .collect::, _>>() + .map_err(Error::Data)? + .into_iter() + .filter(|(x, _)| *x <= until) + .map(|(_, v)| v) + .collect(); + + Ok(matches) } pub fn undo_block(&mut self, cbor: &[u8]) -> Result<(), Error> { @@ -525,7 +526,7 @@ mod tests { pub fn with_tmp_db(op: fn(db: ApplyDB) -> ()) { let path = tempfile::tempdir().unwrap().into_path(); - let db = ApplyDB::open(path.clone(), 764824073, 1).unwrap(); + let db = ApplyDB::open(path.clone()).unwrap(); op(db); diff --git a/src/storage/kvtable.rs b/src/storage/kvtable.rs index ba2f77b5..e7ee80ad 100644 --- a/src/storage/kvtable.rs +++ b/src/storage/kvtable.rs @@ -153,6 +153,12 @@ where } } +impl DBSerde { + pub fn unwrap(self) -> T { + self.0 + } +} + pub struct WithDBIntPrefix(pub u64, pub T); impl From> for Box<[u8]> @@ -349,6 +355,13 @@ where Self::iter_values(db, mode) } + fn iter_values_from_reverse(db: &rocksdb::DB, from: K) -> ValueIterator<'_, V> { + let from_raw = Box::<[u8]>::from(from); + let mode = rocksdb::IteratorMode::From(&from_raw, rocksdb::Direction::Reverse); + + Self::iter_values(db, mode) + } + fn iter_entries<'a>( db: &'a rocksdb::DB, mode: rocksdb::IteratorMode, diff --git a/src/sync/ledger.rs b/src/sync/ledger.rs index 4fc5ffa2..3a1e9c1b 100644 --- a/src/sync/ledger.rs +++ b/src/sync/ledger.rs @@ -1,61 +1,28 @@ -use std::borrow::Cow; -use std::collections::HashMap; - use gasket::framework::*; -use pallas::applying::{validate, UTxOs}; -use pallas::ledger::configs::byron::GenesisFile; +use pallas::applying::{validate, Environment, UTxOs}; +use pallas::ledger::configs::{byron, shelley}; +use pallas::ledger::traverse::wellknown::GenesisValues; use pallas::ledger::traverse::{Era, MultiEraBlock, MultiEraInput, MultiEraOutput}; +use std::borrow::Cow; +use std::collections::HashMap; use tracing::{info, warn}; use crate::prelude::*; use crate::storage::applydb::ApplyDB; -pub fn execute_phase1_validation( - ledger: &ApplyDB, - block: &MultiEraBlock<'_>, -) -> Result<(), WorkerError> { - let mut utxos = HashMap::new(); - ledger - .resolve_inputs_for_block(&block, &mut utxos) - .or_panic()?; - - let mut utxos2 = UTxOs::new(); - - for (ref_, output) in utxos.iter() { - let txin = pallas::ledger::primitives::byron::TxIn::Variant0( - pallas::codec::utils::CborWrap((ref_.0.clone(), ref_.1 as u32)), - ); - - let key = MultiEraInput::Byron( - >>::from(Cow::Owned(txin)), - ); - - let era = Era::try_from(output.0).or_panic()?; - let value = MultiEraOutput::decode(era, &output.1).or_panic()?; - - utxos2.insert(key, value); - } - - let env = ledger.get_active_pparams(block.slot()).or_panic()?; - - for tx in block.txs().iter() { - let res = validate(&tx, &utxos2, &env); - - if let Err(err) = res { - warn!(?err, "validation error"); - } - } - - Ok(()) -} - pub type UpstreamPort = gasket::messaging::tokio::InputPort; #[derive(Stage)] #[stage(name = "ledger", unit = "RollEvent", worker = "Worker")] pub struct Stage { ledger: ApplyDB, - genesis: GenesisFile, + byron: byron::GenesisFile, + shelley: shelley::GenesisFile, + + current_pparams: Option<(u64, Environment)>, + + // HACK: until multi-era genesis + genesis_values: GenesisValues, pub upstream: UpstreamPort, @@ -67,15 +34,87 @@ pub struct Stage { } impl Stage { - pub fn new(ledger: ApplyDB, genesis: GenesisFile) -> Self { + pub fn new(ledger: ApplyDB, byron: byron::GenesisFile, shelley: shelley::GenesisFile) -> Self { Self { ledger, - genesis, + genesis_values: pallas::ledger::traverse::wellknown::GenesisValues::from_magic( + byron.protocol_consts.protocol_magic as u64, + ) + .unwrap(), + byron, + shelley, + current_pparams: None, upstream: Default::default(), block_count: Default::default(), wal_count: Default::default(), } } + + fn ensure_pparams(&mut self, epoch: u64) -> Result<(), WorkerError> { + if self + .current_pparams + .as_ref() + .is_some_and(|(current, _)| *current == epoch) + { + return Ok(()); + } + + let pparams = super::pparams::compute_pparams( + super::pparams::Genesis { + byron: &self.byron, + shelley: &self.shelley, + }, + &self.ledger, + epoch, + )?; + + warn!(?pparams, "pparams for new epoch"); + + self.current_pparams = Some((epoch, pparams)); + + Ok(()) + } + + pub fn execute_phase1_validation(&self, block: &MultiEraBlock<'_>) -> Result<(), WorkerError> { + let mut utxos = HashMap::new(); + self.ledger + .resolve_inputs_for_block(&block, &mut utxos) + .or_panic()?; + + let mut utxos2 = UTxOs::new(); + + for (ref_, output) in utxos.iter() { + let txin = pallas::ledger::primitives::byron::TxIn::Variant0( + pallas::codec::utils::CborWrap((ref_.0.clone(), ref_.1 as u32)), + ); + + let key = MultiEraInput::Byron( + >>::from(Cow::Owned(txin)), + ); + + let era = Era::try_from(output.0).or_panic()?; + let value = MultiEraOutput::decode(era, &output.1).or_panic()?; + + utxos2.insert(key, value); + } + + let pparams = self + .current_pparams + .as_ref() + .map(|(_, x)| x) + .ok_or("no pparams available") + .or_panic()?; + + for tx in block.txs().iter() { + let res = validate(&tx, &utxos2, &pparams); + + if let Err(err) = res { + warn!(?err, "validation error"); + } + } + + Ok(()) + } } pub struct Worker; @@ -101,8 +140,10 @@ impl gasket::framework::Worker for Worker { info!(slot, "applying block"); let block = MultiEraBlock::decode(cbor).or_panic()?; + let (epoch, _) = block.epoch(&stage.genesis_values); + stage.ensure_pparams(epoch)?; - execute_phase1_validation(&stage.ledger, &block)?; + stage.execute_phase1_validation(&block)?; stage.ledger.apply_block(&block).or_panic()?; } @@ -112,7 +153,7 @@ impl gasket::framework::Worker for Worker { } RollEvent::Origin => { info!("applying origin"); - stage.ledger.apply_origin(&stage.genesis).or_panic()?; + stage.ledger.apply_origin(&stage.byron).or_panic()?; } }; diff --git a/src/sync/mod.rs b/src/sync/mod.rs index 01c53ccf..efa3478b 100644 --- a/src/sync/mod.rs +++ b/src/sync/mod.rs @@ -1,7 +1,7 @@ use std::time::Duration; use gasket::messaging::{RecvPort, SendPort}; -use pallas::ledger::configs::byron::GenesisFile; +use pallas::ledger::configs::{byron, shelley}; use pallas::storage::rolldb::chain::Store as ChainStore; use pallas::storage::rolldb::wal::Store as WalStore; use serde::Deserialize; @@ -12,6 +12,7 @@ use crate::storage::applydb::ApplyDB; pub mod chain; pub mod ledger; +pub mod pparams; pub mod pull; pub mod roll; @@ -47,7 +48,8 @@ pub fn pipeline( wal: WalStore, chain: ChainStore, ledger: ApplyDB, - genesis: GenesisFile, + byron: byron::GenesisFile, + shelley: shelley::GenesisFile, retries: &Option, ) -> Result { let pull_cursor = wal @@ -71,7 +73,7 @@ pub fn pipeline( let mut roll = roll::Stage::new(wal, cursor_chain, cursor_ledger); let mut chain = chain::Stage::new(chain); - let mut ledger = ledger::Stage::new(ledger, genesis); + let mut ledger = ledger::Stage::new(ledger, byron, shelley); let (to_roll, from_pull) = gasket::messaging::tokio::mpsc_channel(50); pull.downstream.connect(to_roll); diff --git a/src/sync/pparams.rs b/src/sync/pparams.rs new file mode 100644 index 00000000..bc5cc82d --- /dev/null +++ b/src/sync/pparams.rs @@ -0,0 +1,164 @@ +use gasket::framework::{AsWorkError, WorkerError}; +use pallas::{ + applying::{Environment, MultiEraProtParams}, + ledger::{ + configs::{byron, shelley}, + traverse::{Era, MultiEraUpdate}, + }, +}; +use tracing::{info, warn}; + +use crate::storage::applydb::ApplyDB; + +pub struct Genesis<'a> { + pub byron: &'a byron::GenesisFile, + pub shelley: &'a shelley::GenesisFile, +} + +fn pparams_from_byron_genesis( + byron: &byron::GenesisFile, +) -> Result { + let out = + pallas::applying::MultiEraProtParams::Byron(pallas::applying::types::ByronProtParams { + fee_policy: pallas::applying::types::FeePolicy { + summand: byron + .block_version_data + .tx_fee_policy + .summand + .parse() + .or_panic()?, + multiplier: byron + .block_version_data + .tx_fee_policy + .multiplier + .parse() + .or_panic()?, + }, + max_tx_size: byron.block_version_data.max_tx_size.parse().or_panic()?, + }); + + Ok(out) +} + +fn pparams_from_shelley_genesis( + shelley: &shelley::GenesisFile, +) -> Result { + let out = + pallas::applying::MultiEraProtParams::Shelley(pallas::applying::types::ShelleyProtParams { + fee_policy: pallas::applying::types::FeePolicy { + summand: shelley.protocol_params.min_fee_a, + multiplier: shelley.protocol_params.min_fee_b, + }, + max_tx_size: shelley.protocol_params.max_tx_size, + min_lovelace: shelley.protocol_params.min_u_tx_o_value, + }); + + Ok(out) +} + +fn apply_era_hardfork( + genesis: &Genesis, + new_protocol: u64, +) -> Result { + match new_protocol { + 1 => pparams_from_byron_genesis(&genesis.byron), + 2 | 3 | 4 => pparams_from_shelley_genesis(&genesis.shelley), + x => { + unimplemented!("don't know how to handle hardfork for protocol {x}"); + } + } +} + +fn apply_param_update( + genesis: &Genesis, + era: Era, + current: MultiEraProtParams, + update: MultiEraUpdate, +) -> Result { + match current { + MultiEraProtParams::Byron(mut pparams) => { + assert_eq!(u16::from(era), 1, "pparam update doesn't match era"); + + if let Some((major, _, _)) = update.byron_proposed_block_version() { + warn!(major, "found new byron protocol update proposal"); + return apply_era_hardfork(genesis, major as u64); + } + + if let Some(pallas::ledger::primitives::byron::TxFeePol::Variant0(new)) = + update.byron_proposed_fee_policy() + { + warn!("found new byron fee policy update proposal"); + + let new = new.unwrap(); + pparams.fee_policy = pallas::applying::types::FeePolicy { + summand: new.0 as u64, + multiplier: new.1 as u64, + }; + } + + if let Some(new) = update.byron_proposed_max_tx_size() { + warn!("found new byron max tx size update proposal"); + pparams.max_tx_size = new; + } + + Ok(MultiEraProtParams::Byron(pparams)) + } + MultiEraProtParams::Shelley(mut pparams) => { + assert_eq!(u16::from(era), 2, "pparam update doesn't match era"); + + if let Some((major, _)) = update.first_proposed_protocol_version() { + warn!(major, "found new shelley protocol update proposal"); + return apply_era_hardfork(genesis, major); + } + + if let Some(x) = update.first_proposed_minfee_a() { + warn!(x, "found new minfee a update proposal"); + pparams.fee_policy.summand = x as u64; + } + + if let Some(x) = update.first_proposed_minfee_b() { + warn!(x, "found new minfee b update proposal"); + pparams.fee_policy.multiplier = x as u64; + } + + if let Some(x) = update.first_proposed_max_transaction_size() { + warn!(x, "found new max tx size update proposal"); + pparams.max_tx_size = x as u64; + } + + // TODO: where's the min utxo value in the network primitives for shelley? do we + // have them wrong in Pallas? + + Ok(MultiEraProtParams::Shelley(pparams)) + } + _ => unimplemented!(), + } +} + +pub fn compute_pparams( + genesis: Genesis, + ledger: &ApplyDB, + epoch: u64, +) -> Result { + let mut out = Environment { + block_slot: 0, + prot_magic: genesis.byron.protocol_consts.protocol_magic, + network_id: match genesis.shelley.network_id.as_deref() { + Some("Mainnet") => 0, + _ => 1, + }, + prot_params: apply_era_hardfork(&genesis, 1)?, + }; + + let updates = ledger.get_pparams_updates(epoch).or_panic()?; + + info!(epoch, updates = updates.len(), "computing pparams"); + + for (era, _, cbor) in updates { + let era = Era::try_from(era).or_panic()?; + let update = MultiEraUpdate::decode_for_era(era, &cbor).or_panic()?; + out.prot_params = apply_param_update(&genesis, era, out.prot_params, update)?; + } + + Ok(out) +}