From 43f61a736e05733bc5ea75fffd7001e2e9b6b49c Mon Sep 17 00:00:00 2001 From: Alejandro Avagnina Date: Thu, 30 May 2024 20:09:46 -0300 Subject: [PATCH] chore: instances bootstrap + blue-green proxy (#47) --- bootstrap/configs/main.tf | 35 +++ bootstrap/configs/mainnet/config.json | 103 ++++++++ bootstrap/configs/mainnet/topology.json | 9 + bootstrap/configs/preprod-local/config.json | 107 ++++++++ bootstrap/configs/preprod-local/topology.json | 21 ++ bootstrap/configs/preprod/config.json | 107 ++++++++ bootstrap/configs/preprod/topology.json | 21 ++ bootstrap/configs/preview/config.json | 113 ++++++++ bootstrap/configs/preview/topology.json | 21 ++ bootstrap/configs/sanchonet/config.json | 114 ++++++++ bootstrap/configs/sanchonet/topology.json | 21 ++ bootstrap/configs/testnet/config.json | 101 +++++++ bootstrap/configs/testnet/db-sync-config.json | 114 ++++++++ .../configs/testnet/submit-api-config.json | 113 ++++++++ bootstrap/configs/testnet/topology.json | 9 + bootstrap/configs/vector-testnet/config.json | 101 +++++++ .../configs/vector-testnet/topology.json | 24 ++ bootstrap/feature/main.tf | 12 +- bootstrap/feature/operator.tf | 4 +- bootstrap/instance/main.tf | 89 +++++++ bootstrap/instance/monitor.tf | 33 +++ bootstrap/instance/nginx.conf | 14 + bootstrap/instance/node.tf | 248 ++++++++++++++++++ bootstrap/main.tf | 85 +++++- bootstrap/proxy/cert.tf | 2 +- bootstrap/proxy/config.tf | 48 ++-- bootstrap/proxy/deployment.tf | 22 +- bootstrap/proxy/main.tf | 25 ++ bootstrap/proxy/monitor.tf | 6 +- bootstrap/proxy/proxy-config.toml.tftpl | 1 + bootstrap/proxy/service.tf | 5 +- bootstrap/service/main.tf | 54 ++++ bootstrap/variables.tf | 87 +++++- 33 files changed, 1789 insertions(+), 80 deletions(-) create mode 100644 bootstrap/configs/main.tf create mode 100644 bootstrap/configs/mainnet/config.json create mode 100644 bootstrap/configs/mainnet/topology.json create mode 100644 bootstrap/configs/preprod-local/config.json create mode 100644 bootstrap/configs/preprod-local/topology.json create mode 100644 bootstrap/configs/preprod/config.json create mode 100644 bootstrap/configs/preprod/topology.json create mode 100644 bootstrap/configs/preview/config.json create mode 100644 bootstrap/configs/preview/topology.json create mode 100644 bootstrap/configs/sanchonet/config.json create mode 100644 bootstrap/configs/sanchonet/topology.json create mode 100644 bootstrap/configs/testnet/config.json create mode 100644 bootstrap/configs/testnet/db-sync-config.json create mode 100644 bootstrap/configs/testnet/submit-api-config.json create mode 100644 bootstrap/configs/testnet/topology.json create mode 100644 bootstrap/configs/vector-testnet/config.json create mode 100644 bootstrap/configs/vector-testnet/topology.json create mode 100644 bootstrap/instance/main.tf create mode 100644 bootstrap/instance/monitor.tf create mode 100644 bootstrap/instance/nginx.conf create mode 100644 bootstrap/instance/node.tf create mode 100644 bootstrap/service/main.tf diff --git a/bootstrap/configs/main.tf b/bootstrap/configs/main.tf new file mode 100644 index 0000000..87c80a2 --- /dev/null +++ b/bootstrap/configs/main.tf @@ -0,0 +1,35 @@ +terraform { + required_providers { + kubernetes = { + source = "hashicorp/kubernetes" + } + } +} + +variable "network" { + description = "cardano node network" +} + +variable "namespace" { + description = "the namespace where the resources will be created" +} + +variable "salt" { + description = "the salt to use for the network" +} + +resource "kubernetes_config_map" "node-config" { + metadata { + namespace = var.namespace + name = "configs-${var.network}-${var.salt}" + } + + data = { + "config.json" = "${file("${path.module}/${var.network}/config.json")}" + "topology.json" = "${file("${path.module}/${var.network}/topology.json")}" + } +} + +output "cm_name" { + value = "configs-${var.network}-${var.salt}" +} diff --git a/bootstrap/configs/mainnet/config.json b/bootstrap/configs/mainnet/config.json new file mode 100644 index 0000000..fef7e9e --- /dev/null +++ b/bootstrap/configs/mainnet/config.json @@ -0,0 +1,103 @@ +{ + "AlonzoGenesisFile": "/genesis/mainnet/alonzo-genesis.json", + "AlonzoGenesisHash": "7e94a15f55d1e82d10f09203fa1d40f8eede58fd8066542cf6566008068ed874", + "ApplicationName": "cardano-sl", + "ApplicationVersion": 1, + "ByronGenesisFile": "/genesis/mainnet/byron-genesis.json", + "ByronGenesisHash": "5f20df933584822601f9e3f8c024eb5eb252fe8cefb24d1317dc3d432e940ebb", + "ConwayGenesisFile": "/genesis/mainnet/conway-genesis.json", + "ConwayGenesisHash": "f28f1c1280ea0d32f8cd3143e268650d6c1a8e221522ce4a7d20d62fc09783e1", + "LastKnownBlockVersion-Alt": 0, + "LastKnownBlockVersion-Major": 3, + "LastKnownBlockVersion-Minor": 0, + "MaxKnownMajorProtocolVersion": 2, + "Protocol": "Cardano", + "RequiresNetworkMagic": "RequiresNoMagic", + "ShelleyGenesisFile": "/genesis/mainnet/shelley-genesis.json", + "ShelleyGenesisHash": "1a3be38bcbb7911969283716ad7aa550250226b76a61fc51cc9a9a35d9276d81", + "TraceAcceptPolicy": true, + "TraceBlockFetchClient": false, + "TraceBlockFetchDecisions": false, + "TraceBlockFetchProtocol": false, + "TraceBlockFetchProtocolSerialised": false, + "TraceBlockFetchServer": false, + "TraceChainDb": true, + "TraceChainSyncBlockServer": false, + "TraceChainSyncClient": false, + "TraceChainSyncHeaderServer": false, + "TraceChainSyncProtocol": false, + "TraceConnectionManager": true, + "TraceDNSResolver": true, + "TraceDNSSubscription": true, + "TraceDiffusionInitialization": true, + "TraceErrorPolicy": true, + "TraceForge": true, + "TraceHandshake": false, + "TraceInboundGovernor": true, + "TraceIpSubscription": true, + "TraceLedgerPeers": true, + "TraceLocalChainSyncProtocol": false, + "TraceLocalErrorPolicy": true, + "TraceLocalHandshake": false, + "TraceLocalRootPeers": true, + "TraceLocalTxSubmissionProtocol": false, + "TraceLocalTxSubmissionServer": false, + "TraceMempool": true, + "TraceMux": false, + "TracePeerSelection": true, + "TracePeerSelectionActions": true, + "TracePublicRootPeers": true, + "TraceServer": true, + "TraceTxInbound": false, + "TraceTxOutbound": false, + "TraceTxSubmissionProtocol": false, + "TracingVerbosity": "NormalVerbosity", + "TurnOnLogMetrics": true, + "TurnOnLogging": true, + "defaultBackends": [ + "KatipBK" + ], + "defaultScribes": [ + [ + "StdoutSK", + "stdout" + ] + ], + "hasEKG": 12788, + "hasPrometheus": [ + "0.0.0.0", + 12798 + ], + "minSeverity": "Info", + "options": { + "mapBackends": { + "cardano.node.metrics": [ + "EKGViewBK" + ], + "cardano.node.resources": [ + "EKGViewBK" + ] + }, + "mapSubtrace": { + "cardano.node.metrics": { + "subtrace": "Neutral" + } + } + }, + "rotation": { + "rpKeepFilesNum": 10, + "rpLogLimitBytes": 5000000, + "rpMaxAgeHours": 24 + }, + "setupBackends": [ + "KatipBK" + ], + "setupScribes": [ + { + "scFormat": "ScText", + "scKind": "StdoutSK", + "scName": "stdout", + "scRotation": null + } + ] +} \ No newline at end of file diff --git a/bootstrap/configs/mainnet/topology.json b/bootstrap/configs/mainnet/topology.json new file mode 100644 index 0000000..1d90ef2 --- /dev/null +++ b/bootstrap/configs/mainnet/topology.json @@ -0,0 +1,9 @@ +{ + "Producers": [ + { + "addr": "relays-new.cardano-mainnet.iohk.io", + "port": 3001, + "valency": 2 + } + ] +} diff --git a/bootstrap/configs/preprod-local/config.json b/bootstrap/configs/preprod-local/config.json new file mode 100644 index 0000000..87c75d7 --- /dev/null +++ b/bootstrap/configs/preprod-local/config.json @@ -0,0 +1,107 @@ +{ + "AlonzoGenesisFile": "/genesis/preprod/alonzo-genesis.json", + "AlonzoGenesisHash": "7e94a15f55d1e82d10f09203fa1d40f8eede58fd8066542cf6566008068ed874", + "ApplicationName": "cardano-sl", + "ApplicationVersion": 0, + "ByronGenesisFile": "/genesis/preprod/byron-genesis.json", + "ByronGenesisHash": "d4b8de7a11d929a323373cbab6c1a9bdc931beffff11db111cf9d57356ee1937", + "ConwayGenesisFile": "/genesis/preprod/conway-genesis.json", + "ConwayGenesisHash": "f28f1c1280ea0d32f8cd3143e268650d6c1a8e221522ce4a7d20d62fc09783e1", + "EnableP2P": true, + "LastKnownBlockVersion-Alt": 0, + "LastKnownBlockVersion-Major": 2, + "LastKnownBlockVersion-Minor": 0, + "Protocol": "Cardano", + "RequiresNetworkMagic": "RequiresMagic", + "ShelleyGenesisFile": "/genesis/preprod/shelley-genesis.json", + "ShelleyGenesisHash": "162d29c4e1cf6b8a84f2d692e67a3ac6bc7851bc3e6e4afe64d15778bed8bd86", + "TargetNumberOfActivePeers": 20, + "TargetNumberOfEstablishedPeers": 50, + "TargetNumberOfKnownPeers": 100, + "TargetNumberOfRootPeers": 100, + "TraceAcceptPolicy": true, + "TraceBlockFetchClient": false, + "TraceBlockFetchDecisions": false, + "TraceBlockFetchProtocol": false, + "TraceBlockFetchProtocolSerialised": false, + "TraceBlockFetchServer": false, + "TraceChainDb": true, + "TraceChainSyncBlockServer": false, + "TraceChainSyncClient": false, + "TraceChainSyncHeaderServer": false, + "TraceChainSyncProtocol": false, + "TraceConnectionManager": true, + "TraceDNSResolver": true, + "TraceDNSSubscription": true, + "TraceDiffusionInitialization": true, + "TraceErrorPolicy": true, + "TraceForge": true, + "TraceHandshake": false, + "TraceInboundGovernor": true, + "TraceIpSubscription": true, + "TraceLedgerPeers": true, + "TraceLocalChainSyncProtocol": false, + "TraceLocalErrorPolicy": true, + "TraceLocalHandshake": false, + "TraceLocalRootPeers": true, + "TraceLocalTxSubmissionProtocol": false, + "TraceLocalTxSubmissionServer": false, + "TraceMempool": true, + "TraceMux": false, + "TracePeerSelection": true, + "TracePeerSelectionActions": true, + "TracePublicRootPeers": true, + "TraceServer": true, + "TraceTxInbound": false, + "TraceTxOutbound": false, + "TraceTxSubmissionProtocol": false, + "TracingVerbosity": "NormalVerbosity", + "TurnOnLogMetrics": true, + "TurnOnLogging": true, + "defaultBackends": [ + "KatipBK" + ], + "defaultScribes": [ + [ + "StdoutSK", + "stdout" + ] + ], + "hasEKG": 12788, + "hasPrometheus": [ + "0.0.0.0", + 12798 + ], + "minSeverity": "Info", + "options": { + "mapBackends": { + "cardano.node.metrics": [ + "EKGViewBK" + ], + "cardano.node.resources": [ + "EKGViewBK" + ] + }, + "mapSubtrace": { + "cardano.node.metrics": { + "subtrace": "Neutral" + } + } + }, + "rotation": { + "rpKeepFilesNum": 10, + "rpLogLimitBytes": 5000000, + "rpMaxAgeHours": 24 + }, + "setupBackends": [ + "KatipBK" + ], + "setupScribes": [ + { + "scFormat": "ScText", + "scKind": "StdoutSK", + "scName": "stdout", + "scRotation": null + } + ] +} \ No newline at end of file diff --git a/bootstrap/configs/preprod-local/topology.json b/bootstrap/configs/preprod-local/topology.json new file mode 100644 index 0000000..1b60c2e --- /dev/null +++ b/bootstrap/configs/preprod-local/topology.json @@ -0,0 +1,21 @@ +{ + "localRoots": [ + { + "accessPoints": [], + "advertise": false, + "valency": 1 + } + ], + "publicRoots": [ + { + "accessPoints": [ + { + "address": "node-preprod-stable.ftr-nodes-v2.svc.cluster.local", + "port": 3000 + } + ], + "advertise": false + } + ], + "useLedgerAfterSlot": 4642000 +} diff --git a/bootstrap/configs/preprod/config.json b/bootstrap/configs/preprod/config.json new file mode 100644 index 0000000..87c75d7 --- /dev/null +++ b/bootstrap/configs/preprod/config.json @@ -0,0 +1,107 @@ +{ + "AlonzoGenesisFile": "/genesis/preprod/alonzo-genesis.json", + "AlonzoGenesisHash": "7e94a15f55d1e82d10f09203fa1d40f8eede58fd8066542cf6566008068ed874", + "ApplicationName": "cardano-sl", + "ApplicationVersion": 0, + "ByronGenesisFile": "/genesis/preprod/byron-genesis.json", + "ByronGenesisHash": "d4b8de7a11d929a323373cbab6c1a9bdc931beffff11db111cf9d57356ee1937", + "ConwayGenesisFile": "/genesis/preprod/conway-genesis.json", + "ConwayGenesisHash": "f28f1c1280ea0d32f8cd3143e268650d6c1a8e221522ce4a7d20d62fc09783e1", + "EnableP2P": true, + "LastKnownBlockVersion-Alt": 0, + "LastKnownBlockVersion-Major": 2, + "LastKnownBlockVersion-Minor": 0, + "Protocol": "Cardano", + "RequiresNetworkMagic": "RequiresMagic", + "ShelleyGenesisFile": "/genesis/preprod/shelley-genesis.json", + "ShelleyGenesisHash": "162d29c4e1cf6b8a84f2d692e67a3ac6bc7851bc3e6e4afe64d15778bed8bd86", + "TargetNumberOfActivePeers": 20, + "TargetNumberOfEstablishedPeers": 50, + "TargetNumberOfKnownPeers": 100, + "TargetNumberOfRootPeers": 100, + "TraceAcceptPolicy": true, + "TraceBlockFetchClient": false, + "TraceBlockFetchDecisions": false, + "TraceBlockFetchProtocol": false, + "TraceBlockFetchProtocolSerialised": false, + "TraceBlockFetchServer": false, + "TraceChainDb": true, + "TraceChainSyncBlockServer": false, + "TraceChainSyncClient": false, + "TraceChainSyncHeaderServer": false, + "TraceChainSyncProtocol": false, + "TraceConnectionManager": true, + "TraceDNSResolver": true, + "TraceDNSSubscription": true, + "TraceDiffusionInitialization": true, + "TraceErrorPolicy": true, + "TraceForge": true, + "TraceHandshake": false, + "TraceInboundGovernor": true, + "TraceIpSubscription": true, + "TraceLedgerPeers": true, + "TraceLocalChainSyncProtocol": false, + "TraceLocalErrorPolicy": true, + "TraceLocalHandshake": false, + "TraceLocalRootPeers": true, + "TraceLocalTxSubmissionProtocol": false, + "TraceLocalTxSubmissionServer": false, + "TraceMempool": true, + "TraceMux": false, + "TracePeerSelection": true, + "TracePeerSelectionActions": true, + "TracePublicRootPeers": true, + "TraceServer": true, + "TraceTxInbound": false, + "TraceTxOutbound": false, + "TraceTxSubmissionProtocol": false, + "TracingVerbosity": "NormalVerbosity", + "TurnOnLogMetrics": true, + "TurnOnLogging": true, + "defaultBackends": [ + "KatipBK" + ], + "defaultScribes": [ + [ + "StdoutSK", + "stdout" + ] + ], + "hasEKG": 12788, + "hasPrometheus": [ + "0.0.0.0", + 12798 + ], + "minSeverity": "Info", + "options": { + "mapBackends": { + "cardano.node.metrics": [ + "EKGViewBK" + ], + "cardano.node.resources": [ + "EKGViewBK" + ] + }, + "mapSubtrace": { + "cardano.node.metrics": { + "subtrace": "Neutral" + } + } + }, + "rotation": { + "rpKeepFilesNum": 10, + "rpLogLimitBytes": 5000000, + "rpMaxAgeHours": 24 + }, + "setupBackends": [ + "KatipBK" + ], + "setupScribes": [ + { + "scFormat": "ScText", + "scKind": "StdoutSK", + "scName": "stdout", + "scRotation": null + } + ] +} \ No newline at end of file diff --git a/bootstrap/configs/preprod/topology.json b/bootstrap/configs/preprod/topology.json new file mode 100644 index 0000000..99a3644 --- /dev/null +++ b/bootstrap/configs/preprod/topology.json @@ -0,0 +1,21 @@ +{ + "localRoots": [ + { + "accessPoints": [], + "advertise": false, + "valency": 1 + } + ], + "publicRoots": [ + { + "accessPoints": [ + { + "address": "preprod-node.world.dev.cardano.org", + "port": 30000 + } + ], + "advertise": false + } + ], + "useLedgerAfterSlot": 4642000 +} diff --git a/bootstrap/configs/preview/config.json b/bootstrap/configs/preview/config.json new file mode 100644 index 0000000..57235c8 --- /dev/null +++ b/bootstrap/configs/preview/config.json @@ -0,0 +1,113 @@ +{ + "AlonzoGenesisFile": "/genesis/preview/alonzo-genesis.json", + "AlonzoGenesisHash": "7e94a15f55d1e82d10f09203fa1d40f8eede58fd8066542cf6566008068ed874", + "ApplicationName": "cardano-sl", + "ApplicationVersion": 0, + "ByronGenesisFile": "/genesis/preview/byron-genesis.json", + "ByronGenesisHash": "83de1d7302569ad56cf9139a41e2e11346d4cb4a31c00142557b6ab3fa550761", + "ConwayGenesisFile": "/genesis/preview/conway-genesis.json", + "ConwayGenesisHash": "f28f1c1280ea0d32f8cd3143e268650d6c1a8e221522ce4a7d20d62fc09783e1", + "EnableP2P": true, + "ExperimentalHardForksEnabled": false, + "ExperimentalProtocolsEnabled": false, + "LastKnownBlockVersion-Alt": 0, + "LastKnownBlockVersion-Major": 3, + "LastKnownBlockVersion-Minor": 1, + "Protocol": "Cardano", + "RequiresNetworkMagic": "RequiresMagic", + "ShelleyGenesisFile": "/genesis/preview/shelley-genesis.json", + "ShelleyGenesisHash": "363498d1024f84bb39d3fa9593ce391483cb40d479b87233f868d6e57c3a400d", + "TargetNumberOfActivePeers": 20, + "TargetNumberOfEstablishedPeers": 50, + "TargetNumberOfKnownPeers": 100, + "TargetNumberOfRootPeers": 100, + "TestAllegraHardForkAtEpoch": 0, + "TestAlonzoHardForkAtEpoch": 0, + "TestMaryHardForkAtEpoch": 0, + "TestShelleyHardForkAtEpoch": 0, + "TraceAcceptPolicy": true, + "TraceBlockFetchClient": false, + "TraceBlockFetchDecisions": false, + "TraceBlockFetchProtocol": false, + "TraceBlockFetchProtocolSerialised": false, + "TraceBlockFetchServer": false, + "TraceChainDb": true, + "TraceChainSyncBlockServer": false, + "TraceChainSyncClient": false, + "TraceChainSyncHeaderServer": false, + "TraceChainSyncProtocol": false, + "TraceConnectionManager": true, + "TraceDNSResolver": true, + "TraceDNSSubscription": true, + "TraceDiffusionInitialization": true, + "TraceErrorPolicy": true, + "TraceForge": true, + "TraceHandshake": false, + "TraceInboundGovernor": true, + "TraceIpSubscription": true, + "TraceLedgerPeers": true, + "TraceLocalChainSyncProtocol": false, + "TraceLocalErrorPolicy": true, + "TraceLocalHandshake": false, + "TraceLocalRootPeers": true, + "TraceLocalTxSubmissionProtocol": false, + "TraceLocalTxSubmissionServer": false, + "TraceMempool": true, + "TraceMux": false, + "TracePeerSelection": true, + "TracePeerSelectionActions": true, + "TracePublicRootPeers": true, + "TraceServer": true, + "TraceTxInbound": false, + "TraceTxOutbound": false, + "TraceTxSubmissionProtocol": false, + "TracingVerbosity": "NormalVerbosity", + "TurnOnLogMetrics": true, + "TurnOnLogging": true, + "defaultBackends": [ + "KatipBK" + ], + "defaultScribes": [ + [ + "StdoutSK", + "stdout" + ] + ], + "hasEKG": 12788, + "hasPrometheus": [ + "0.0.0.0", + 12798 + ], + "minSeverity": "Info", + "options": { + "mapBackends": { + "cardano.node.metrics": [ + "EKGViewBK" + ], + "cardano.node.resources": [ + "EKGViewBK" + ] + }, + "mapSubtrace": { + "cardano.node.metrics": { + "subtrace": "Neutral" + } + } + }, + "rotation": { + "rpKeepFilesNum": 10, + "rpLogLimitBytes": 5000000, + "rpMaxAgeHours": 24 + }, + "setupBackends": [ + "KatipBK" + ], + "setupScribes": [ + { + "scFormat": "ScText", + "scKind": "StdoutSK", + "scName": "stdout", + "scRotation": null + } + ] +} \ No newline at end of file diff --git a/bootstrap/configs/preview/topology.json b/bootstrap/configs/preview/topology.json new file mode 100644 index 0000000..32cb61b --- /dev/null +++ b/bootstrap/configs/preview/topology.json @@ -0,0 +1,21 @@ +{ + "localRoots": [ + { + "accessPoints": [], + "advertise": false, + "valency": 1 + } + ], + "publicRoots": [ + { + "accessPoints": [ + { + "address": "preview-node.world.dev.cardano.org", + "port": 30002 + } + ], + "advertise": false + } + ], + "useLedgerAfterSlot": 322000 +} diff --git a/bootstrap/configs/sanchonet/config.json b/bootstrap/configs/sanchonet/config.json new file mode 100644 index 0000000..f4f9190 --- /dev/null +++ b/bootstrap/configs/sanchonet/config.json @@ -0,0 +1,114 @@ +{ + "AlonzoGenesisFile": "/genesis/sanchonet/alonzo-genesis.json", + "AlonzoGenesisHash": "8bedcaea62107d8a79ed5293b0027b3f8706a4bc2422f33380cb1fd01c6fa6ec", + "ByronGenesisFile": "/genesis/sanchonet/byron-genesis.json", + "ByronGenesisHash": "785eb88427e136378a15b0a152a8bfbeec7a611529ccda29c43a1e60ffb48eaa", + "ConwayGenesisFile": "/genesis/sanchonet/conway-genesis.json", + "ConwayGenesisHash": "79766b0c18076e9abefc806526714f30d42b14dba9f187fb316bd4e70c815ef4", + "EnableP2P": true, + "ExperimentalHardForksEnabled": true, + "ExperimentalProtocolsEnabled": true, + "LastKnownBlockVersion-Alt": 0, + "LastKnownBlockVersion-Major": 3, + "LastKnownBlockVersion-Minor": 1, + "MinNodeVersion": "8.11.0", + "PeerSharing": true, + "Protocol": "Cardano", + "RequiresNetworkMagic": "RequiresMagic", + "ShelleyGenesisFile": "/genesis/sanchonet/shelley-genesis.json", + "ShelleyGenesisHash": "f94457ec45a0c6773057a529533cf7ccf746cb44dabd56ae970e1dbfb55bfdb2", + "TargetNumberOfActivePeers": 20, + "TargetNumberOfEstablishedPeers": 50, + "TargetNumberOfKnownPeers": 100, + "TargetNumberOfRootPeers": 60, + "TestAllegraHardForkAtEpoch": 0, + "TestAlonzoHardForkAtEpoch": 0, + "TestMaryHardForkAtEpoch": 0, + "TestShelleyHardForkAtEpoch": 0, + "TraceAcceptPolicy": true, + "TraceBlockFetchClient": false, + "TraceBlockFetchDecisions": false, + "TraceBlockFetchProtocol": false, + "TraceBlockFetchProtocolSerialised": false, + "TraceBlockFetchServer": false, + "TraceChainDb": true, + "TraceChainSyncBlockServer": false, + "TraceChainSyncClient": false, + "TraceChainSyncHeaderServer": false, + "TraceChainSyncProtocol": false, + "TraceConnectionManager": true, + "TraceDNSResolver": true, + "TraceDNSSubscription": true, + "TraceDiffusionInitialization": true, + "TraceErrorPolicy": true, + "TraceForge": true, + "TraceHandshake": true, + "TraceInboundGovernor": true, + "TraceIpSubscription": true, + "TraceLedgerPeers": true, + "TraceLocalChainSyncProtocol": false, + "TraceLocalConnectionManager": true, + "TraceLocalErrorPolicy": true, + "TraceLocalHandshake": true, + "TraceLocalRootPeers": true, + "TraceLocalTxSubmissionProtocol": false, + "TraceLocalTxSubmissionServer": false, + "TraceMempool": true, + "TraceMux": false, + "TracePeerSelection": true, + "TracePeerSelectionActions": true, + "TracePublicRootPeers": true, + "TraceServer": true, + "TraceTxInbound": false, + "TraceTxOutbound": false, + "TraceTxSubmissionProtocol": false, + "TracingVerbosity": "NormalVerbosity", + "TurnOnLogMetrics": true, + "TurnOnLogging": true, + "defaultBackends": [ + "KatipBK" + ], + "defaultScribes": [ + [ + "StdoutSK", + "stdout" + ] + ], + "hasEKG": 12788, + "hasPrometheus": [ + "0.0.0.0", + 12798 + ], + "minSeverity": "Info", + "options": { + "mapBackends": { + "cardano.node.metrics": [ + "EKGViewBK" + ], + "cardano.node.resources": [ + "EKGViewBK" + ] + }, + "mapSubtrace": { + "cardano.node.metrics": { + "subtrace": "Neutral" + } + } + }, + "rotation": { + "rpKeepFilesNum": 10, + "rpLogLimitBytes": 5000000, + "rpMaxAgeHours": 24 + }, + "setupBackends": [ + "KatipBK" + ], + "setupScribes": [ + { + "scFormat": "ScText", + "scKind": "StdoutSK", + "scName": "stdout", + "scRotation": null + } + ] +} diff --git a/bootstrap/configs/sanchonet/topology.json b/bootstrap/configs/sanchonet/topology.json new file mode 100644 index 0000000..a20dc7c --- /dev/null +++ b/bootstrap/configs/sanchonet/topology.json @@ -0,0 +1,21 @@ +{ + "localRoots": [ + { + "accessPoints": [], + "advertise": false, + "valency": 1 + } + ], + "publicRoots": [ + { + "accessPoints": [ + { + "address": "sanchonet-node.play.dev.cardano.org", + "port": 3001 + } + ], + "advertise": false + } + ], + "useLedgerAfterSlot": 14601600 +} diff --git a/bootstrap/configs/testnet/config.json b/bootstrap/configs/testnet/config.json new file mode 100644 index 0000000..34be9ae --- /dev/null +++ b/bootstrap/configs/testnet/config.json @@ -0,0 +1,101 @@ +{ + "AlonzoGenesisFile": "/genesis/testnet/alonzo-genesis.json", + "AlonzoGenesisHash": "7e94a15f55d1e82d10f09203fa1d40f8eede58fd8066542cf6566008068ed874", + "ApplicationName": "cardano-sl", + "ApplicationVersion": 0, + "ByronGenesisFile": "/genesis/testnet/byron-genesis.json", + "ByronGenesisHash": "96fceff972c2c06bd3bb5243c39215333be6d56aaf4823073dca31afe5038471", + "LastKnownBlockVersion-Alt": 0, + "LastKnownBlockVersion-Major": 3, + "LastKnownBlockVersion-Minor": 0, + "MaxKnownMajorProtocolVersion": 2, + "Protocol": "Cardano", + "RequiresNetworkMagic": "RequiresMagic", + "ShelleyGenesisFile": "/genesis/testnet/shelley-genesis.json", + "ShelleyGenesisHash": "849a1764f152e1b09c89c0dfdbcbdd38d711d1fec2db5dfa0f87cf2737a0eaf4", + "TraceAcceptPolicy": true, + "TraceBlockFetchClient": false, + "TraceBlockFetchDecisions": false, + "TraceBlockFetchProtocol": false, + "TraceBlockFetchProtocolSerialised": false, + "TraceBlockFetchServer": false, + "TraceChainDb": true, + "TraceChainSyncBlockServer": false, + "TraceChainSyncClient": false, + "TraceChainSyncHeaderServer": false, + "TraceChainSyncProtocol": false, + "TraceConnectionManager": true, + "TraceDNSResolver": true, + "TraceDNSSubscription": true, + "TraceDiffusionInitialization": true, + "TraceErrorPolicy": true, + "TraceForge": true, + "TraceHandshake": false, + "TraceInboundGovernor": true, + "TraceIpSubscription": true, + "TraceLedgerPeers": true, + "TraceLocalChainSyncProtocol": false, + "TraceLocalErrorPolicy": true, + "TraceLocalHandshake": false, + "TraceLocalRootPeers": true, + "TraceLocalTxSubmissionProtocol": false, + "TraceLocalTxSubmissionServer": false, + "TraceMempool": true, + "TraceMux": false, + "TracePeerSelection": true, + "TracePeerSelectionActions": true, + "TracePublicRootPeers": true, + "TraceServer": true, + "TraceTxInbound": false, + "TraceTxOutbound": false, + "TraceTxSubmissionProtocol": false, + "TracingVerbosity": "NormalVerbosity", + "TurnOnLogMetrics": true, + "TurnOnLogging": true, + "defaultBackends": [ + "KatipBK" + ], + "defaultScribes": [ + [ + "StdoutSK", + "stdout" + ] + ], + "hasEKG": 12788, + "hasPrometheus": [ + "0.0.0.0", + 12798 + ], + "minSeverity": "Info", + "options": { + "mapBackends": { + "cardano.node.metrics": [ + "EKGViewBK" + ], + "cardano.node.resources": [ + "EKGViewBK" + ] + }, + "mapSubtrace": { + "cardano.node.metrics": { + "subtrace": "Neutral" + } + } + }, + "rotation": { + "rpKeepFilesNum": 10, + "rpLogLimitBytes": 5000000, + "rpMaxAgeHours": 24 + }, + "setupBackends": [ + "KatipBK" + ], + "setupScribes": [ + { + "scFormat": "ScText", + "scKind": "StdoutSK", + "scName": "stdout", + "scRotation": null + } + ] +} \ No newline at end of file diff --git a/bootstrap/configs/testnet/db-sync-config.json b/bootstrap/configs/testnet/db-sync-config.json new file mode 100644 index 0000000..e1d1ef7 --- /dev/null +++ b/bootstrap/configs/testnet/db-sync-config.json @@ -0,0 +1,114 @@ +{ + "EnableLogMetrics": false, + "EnableLogging": true, + "NetworkName": "testnet", + "NodeConfigFile": "config.json", + "PrometheusPort": 8080, + "RequiresNetworkMagic": "RequiresMagic", + "defaultBackends": [ + "KatipBK" + ], + "defaultScribes": [ + [ + "StdoutSK", + "stdout" + ] + ], + "minSeverity": "Info", + "options": { + "cfokey": { + "value": "Release-1.0.0" + }, + "mapBackends": {}, + "mapSeverity": { + "db-sync-node": "Info", + "db-sync-node.Mux": "Error", + "db-sync-node.Subscription": "Error" + }, + "mapSubtrace": { + "#ekgview": { + "contents": [ + [ + { + "contents": "cardano.epoch-validation.benchmark", + "tag": "Contains" + }, + [ + { + "contents": ".monoclock.basic.", + "tag": "Contains" + } + ] + ], + [ + { + "contents": "cardano.epoch-validation.benchmark", + "tag": "Contains" + }, + [ + { + "contents": "diff.RTS.cpuNs.timed.", + "tag": "Contains" + } + ] + ], + [ + { + "contents": "#ekgview.#aggregation.cardano.epoch-validation.benchmark", + "tag": "StartsWith" + }, + [ + { + "contents": "diff.RTS.gcNum.timed.", + "tag": "Contains" + } + ] + ] + ], + "subtrace": "FilterTrace" + }, + "#messagecounters.aggregation": { + "subtrace": "NoTrace" + }, + "#messagecounters.ekgview": { + "subtrace": "NoTrace" + }, + "#messagecounters.katip": { + "subtrace": "NoTrace" + }, + "#messagecounters.monitoring": { + "subtrace": "NoTrace" + }, + "#messagecounters.switchboard": { + "subtrace": "NoTrace" + }, + "benchmark": { + "contents": [ + "GhcRtsStats", + "MonotonicClock" + ], + "subtrace": "ObservableTrace" + }, + "cardano.epoch-validation.utxo-stats": { + "subtrace": "NoTrace" + } + } + }, + "rotation": { + "rpKeepFilesNum": 10, + "rpLogLimitBytes": 5000000, + "rpMaxAgeHours": 24 + }, + "setupBackends": [ + "AggregationBK", + "KatipBK" + ], + "setupScribes": [ + { + "scFormat": "ScText", + "scKind": "StdoutSK", + "scName": "stdout", + "scRotation": null + } + ] +} diff --git a/bootstrap/configs/testnet/submit-api-config.json b/bootstrap/configs/testnet/submit-api-config.json new file mode 100644 index 0000000..18c45f7 --- /dev/null +++ b/bootstrap/configs/testnet/submit-api-config.json @@ -0,0 +1,113 @@ +{ + "EnableLogMetrics": false, + "EnableLogging": true, + "GenesisHash": "96fceff972c2c06bd3bb5243c39215333be6d56aaf4823073dca31afe5038471", + "PrometheusPort": 8080, + "RequiresNetworkMagic": "RequiresMagic", + "defaultBackends": [ + "KatipBK" + ], + "defaultScribes": [ + [ + "StdoutSK", + "stdout" + ] + ], + "minSeverity": "Info", + "options": { + "cfokey": { + "value": "Release-1.0.0" + }, + "mapBackends": {}, + "mapSeverity": { + "db-sync-node": "Info", + "db-sync-node.Mux": "Error", + "db-sync-node.Subscription": "Error" + }, + "mapSubtrace": { + "#ekgview": { + "contents": [ + [ + { + "contents": "cardano.epoch-validation.benchmark", + "tag": "Contains" + }, + [ + { + "contents": ".monoclock.basic.", + "tag": "Contains" + } + ] + ], + [ + { + "contents": "cardano.epoch-validation.benchmark", + "tag": "Contains" + }, + [ + { + "contents": "diff.RTS.cpuNs.timed.", + "tag": "Contains" + } + ] + ], + [ + { + "contents": "#ekgview.#aggregation.cardano.epoch-validation.benchmark", + "tag": "StartsWith" + }, + [ + { + "contents": "diff.RTS.gcNum.timed.", + "tag": "Contains" + } + ] + ] + ], + "subtrace": "FilterTrace" + }, + "#messagecounters.aggregation": { + "subtrace": "NoTrace" + }, + "#messagecounters.ekgview": { + "subtrace": "NoTrace" + }, + "#messagecounters.katip": { + "subtrace": "NoTrace" + }, + "#messagecounters.monitoring": { + "subtrace": "NoTrace" + }, + "#messagecounters.switchboard": { + "subtrace": "NoTrace" + }, + "benchmark": { + "contents": [ + "GhcRtsStats", + "MonotonicClock" + ], + "subtrace": "ObservableTrace" + }, + "cardano.epoch-validation.utxo-stats": { + "subtrace": "NoTrace" + } + } + }, + "rotation": { + "rpKeepFilesNum": 10, + "rpLogLimitBytes": 5000000, + "rpMaxAgeHours": 24 + }, + "setupBackends": [ + "AggregationBK", + "KatipBK" + ], + "setupScribes": [ + { + "scFormat": "ScText", + "scKind": "StdoutSK", + "scName": "stdout", + "scRotation": null + } + ] +} diff --git a/bootstrap/configs/testnet/topology.json b/bootstrap/configs/testnet/topology.json new file mode 100644 index 0000000..0174fd5 --- /dev/null +++ b/bootstrap/configs/testnet/topology.json @@ -0,0 +1,9 @@ +{ + "Producers": [ + { + "addr": "relays-new.cardano-testnet.iohkdev.io", + "port": 3001, + "valency": 2 + } + ] +} diff --git a/bootstrap/configs/vector-testnet/config.json b/bootstrap/configs/vector-testnet/config.json new file mode 100644 index 0000000..5f9f78a --- /dev/null +++ b/bootstrap/configs/vector-testnet/config.json @@ -0,0 +1,101 @@ +{ + "ByronGenesisFile": "/genesis/vector-testnet/byron.json", + "ShelleyGenesisFile": "/genesis/vector-testnet/shelley.json", + "AlonzoGenesisFile": "/genesis/vector-testnet/alonzo.json", + "ConwayGenesisFile": "/genesis/vector-testnet/conway.json", + "SocketPath": "db/node.socket", + "PBftSignatureThreshold": 0.6, + "MaxConcurrencyBulkSync": 1, + "MaxConcurrencyDeadline": 2, + "Protocol": "Cardano", + "RequiresNetworkMagic": "RequiresMagic", + "LastKnownBlockVersion-Major": 6, + "LastKnownBlockVersion-Minor": 0, + "LastKnownBlockVersion-Alt": 0, + "TurnOnLogging": true, + "TurnOnLogMetrics": true, + "minSeverity": "Debug", + "TracingVerbosity": "NormalVerbosity", + "setupBackends": ["KatipBK"], + "defaultBackends": ["KatipBK"], + "hasEKG": 12788, + "hasPrometheus": [ + "0.0.0.0", + 12798 + ], + "setupScribes": [ + { + "scKind": "FileSK", + "scName": "logs/mainnet.log", + "scFormat": "ScText" + }, + { + "scKind": "StdoutSK", + "scName": "stdout", + "scFormat": "ScText" + } + ], + "defaultScribes": [ + ["FileSK", "logs/mainnet.log"], + ["StdoutSK", "stdout"] + ], + "rotation": { + "rpLogLimitBytes": 5000000, + "rpKeepFilesNum": 3, + "rpMaxAgeHours": 24 + }, + "TraceBlockFetchClient": false, + "TraceBlockFetchDecisions": false, + "TraceBlockFetchProtocol": false, + "TraceBlockFetchProtocolSerialised": false, + "TraceBlockFetchServer": false, + "TraceBlockchainTime": false, + "TraceChainDb": true, + "TraceChainSyncClient": false, + "TraceChainSyncBlockServer": false, + "TraceChainSyncHeaderServer": false, + "TraceChainSyncProtocol": false, + "TraceDNSResolver": true, + "TraceDNSSubscription": true, + "TraceErrorPolicy": true, + "TraceLocalErrorPolicy": true, + "TraceForge": true, + "TraceHandshake": false, + "TraceIpSubscription": true, + "TraceLocalRootPeers": true, + "TracePublicRootPeers": true, + "TracePeerSelection": true, + "TraceDebugPeerSelection": false, + "TracePeerSelectionActions": true, + "TraceConnectionManager": true, + "TraceServer": true, + "TraceLocalConnectionManager": false, + "TraceLocalServer": false, + "TraceLocalChainSyncProtocol": false, + "TraceLocalHandshake": false, + "TraceLocalTxSubmissionProtocol": false, + "TraceLocalTxSubmissionServer": false, + "TraceMempool": true, + "TraceMux": false, + "TraceTxInbound": false, + "TraceTxOutbound": false, + "TraceTxSubmissionProtocol": false, + "options": { + "mapBackends": { + "cardano.node.metrics": ["EKGViewBK"] + }, + "mapScribes": { + "cardano.node.metrics": ["FileSK::logs/mainnet.log"] + }, + "mapSeverity": { + "cardano.node.ChainDB": "Notice", + "cardano.node.DnsSubscription": "Debug" + } + }, + "TestShelleyHardForkAtEpoch": 0, + "TestAllegraHardForkAtEpoch": 0, + "TestMaryHardForkAtEpoch": 0, + "TestAlonzoHardForkAtEpoch": 0, + "TestBabbageHardForkAtEpoch": 0, + "ExperimentalProtocolsEnabled": true +} diff --git a/bootstrap/configs/vector-testnet/topology.json b/bootstrap/configs/vector-testnet/topology.json new file mode 100644 index 0000000..34d79e3 --- /dev/null +++ b/bootstrap/configs/vector-testnet/topology.json @@ -0,0 +1,24 @@ +{ + "Producers": [ + { + "addr": "85.90.225.26", + "port": 7532, + "valency": 1 + }, + { + "addr": "62.216.254.144", + "port": 7532, + "valency": 1 + }, + { + "addr": "62.216.254.145", + "port": 7532, + "valency": 1 + }, + { + "addr": "62.216.254.146", + "port": 7532, + "valency": 1 + } + ] +} \ No newline at end of file diff --git a/bootstrap/feature/main.tf b/bootstrap/feature/main.tf index 3250ce1..e88b40b 100644 --- a/bootstrap/feature/main.tf +++ b/bootstrap/feature/main.tf @@ -23,14 +23,14 @@ variable "metrics_delay" { default = "30" } -variable "dcu_per_package" { +variable "dcu_per_second" { type = map(string) default = { - "mainnet" = "10" - "preprod" = "5" - "preview" = "5" - "sanchonet" = "5" - "vector-testnet" = "5" + "mainnet" = "1" + "preprod" = "1" + "preview" = "1" + "sanchonet" = "1" + "vector-testnet" = "1" } } diff --git a/bootstrap/feature/operator.tf b/bootstrap/feature/operator.tf index 29dcfc9..55b61fe 100644 --- a/bootstrap/feature/operator.tf +++ b/bootstrap/feature/operator.tf @@ -72,8 +72,8 @@ resource "kubernetes_deployment_v1" "operator" { } env { - name = "DCU_PER_PACKAGE" - value = "mainnet=${var.dcu_per_package["mainnet"]},preprod=${var.dcu_per_package["preprod"]},preview=${var.dcu_per_package["preview"]},sanchonet=${var.dcu_per_package["sanchonet"]},vector-testnet=${var.dcu_per_package["vector-testnet"]}" + name = "DCU_PER_SECOND" + value = "mainnet=${var.dcu_per_second["mainnet"]},preprod=${var.dcu_per_second["preprod"]},preview=${var.dcu_per_second["preview"]},sanchonet=${var.dcu_per_second["sanchonet"]},vector-testnet=${var.dcu_per_second["vector-testnet"]}" } resources { diff --git a/bootstrap/instance/main.tf b/bootstrap/instance/main.tf new file mode 100644 index 0000000..f8c87ca --- /dev/null +++ b/bootstrap/instance/main.tf @@ -0,0 +1,89 @@ +terraform { + required_providers { + kubernetes = { + source = "hashicorp/kubernetes" + } + } +} + +variable "replicas" { + description = "the number of replicas for the node STS" + default = 1 +} + +variable "node_resources" { + type = object({ + requests = map(string) + limits = map(string) + }) + + default = { + limits = { + memory = "2Gi" + } + requests = { + cpu = "100m" + memory = "2Gi" + } + } +} + +variable "storage_size" { + default = "50Gi" +} + +variable "node_image" { + description = "the OCI image of the cardano-node" +} + +variable "node_image_tag" { + description = "the tag of the cardano-node OCI image" +} + +variable "release" { + description = "the version of the cardano-node being deployed in a k8s-friendly syntax" +} + +variable "network" { + description = "cardano node network name (mainnet, preprod, preview)" +} + +variable "magic" { + description = "cardano node network magic (int)" +} + +variable "namespace" { + description = "the namespace where the resources will be created" +} + +variable "topology_zone" {} + +variable "salt" {} + +variable "sync_status" { + default = "pending" +} + +variable "compute_arch" { + default = "arm64" +} + +variable "compute_profile" { + default = "mem-intensive" +} + +variable "availability_sla" { + default = "consistent" +} + +variable "node_version" { + type = string +} + +variable "restore" { + default = false +} + +variable "is_custom" { + default = false +} diff --git a/bootstrap/instance/monitor.tf b/bootstrap/instance/monitor.tf new file mode 100644 index 0000000..d6a7532 --- /dev/null +++ b/bootstrap/instance/monitor.tf @@ -0,0 +1,33 @@ +resource "kubernetes_manifest" "podmonitor" { + manifest = { + apiVersion = "monitoring.coreos.com/v1" + kind = "PodMonitor" + metadata = { + labels = { + "app.kubernetes.io/component" = "o11y" + "app.kubernetes.io/part-of" = "demeter" + } + name = "node-${var.network}-${var.salt}" + namespace = var.namespace + } + spec = { + selector = { + matchLabels = { + role = "node" + network = var.network + salt = var.salt + } + } + podMetricsEndpoints = [ + { + port = "metrics", + path = "/metrics" + }, + { + port = "ogmios", + path = "/metrics" + } + ] + } + } +} diff --git a/bootstrap/instance/nginx.conf b/bootstrap/instance/nginx.conf new file mode 100644 index 0000000..758c256 --- /dev/null +++ b/bootstrap/instance/nginx.conf @@ -0,0 +1,14 @@ +user root; + +events { + worker_connections 1024; +} + +stream { + server { + listen 3307; + proxy_pass unix:/ipc/node.socket; + proxy_connect_timeout 30s; + proxy_timeout 180m; + } +} diff --git a/bootstrap/instance/node.tf b/bootstrap/instance/node.tf new file mode 100644 index 0000000..878585f --- /dev/null +++ b/bootstrap/instance/node.tf @@ -0,0 +1,248 @@ +locals { + custom_arguments = [ + "run", + "--config", + "/configuration/config.json", + "--topology", + "/configuration/topology.json", + "--database-path", + "/data/db", + "--socket-path", + "/ipc/node.socket", + "--port", + "3000" + ] + default_arguments = [ + "run", + "--database-path", + "/data/db", + "--socket-path", + "/ipc/node.socket", + "--port", + "3000" + ] + arguments = var.is_custom == true ? local.custom_arguments : local.default_arguments +} + + +resource "kubernetes_config_map" "proxy-config" { + metadata { + namespace = var.namespace + name = "proxy-${var.network}-${var.salt}" + } + + data = { + "nginx.conf" = "${file("${path.module}/nginx.conf")}" + } +} + +resource "kubernetes_stateful_set_v1" "node" { + wait_for_rollout = false + + metadata { + namespace = var.namespace + name = "node-${var.network}-${var.salt}" + labels = { + network = var.network + release = var.release + salt = var.salt + role = "node" + node-version = var.node_version + } + } + + spec { + replicas = var.replicas + + service_name = "nodes-${var.salt}" + + selector { + match_labels = { + network = var.network + release = var.release + salt = var.salt + role = "node" + } + } + + volume_claim_template { + metadata { + name = "data" + } + spec { + access_modes = ["ReadWriteOnce"] + storage_class_name = "gp3" + resources { + requests = { + storage = var.storage_size + } + } + } + } + + template { + metadata { + labels = { + network = var.network + release = var.release + salt = var.salt + sync = var.sync_status + node-version = var.node_version + role = "node" + } + } + + spec { + dynamic "affinity" { + for_each = var.topology_zone != null ? toset([1]) : toset([]) + + content { + node_affinity { + required_during_scheduling_ignored_during_execution { + node_selector_term { + match_expressions { + key = "topology.kubernetes.io/zone" + operator = "In" + values = [var.topology_zone] + } + } + } + } + } + } + + toleration { + effect = "NoSchedule" + key = "demeter.run/compute-profile" + operator = "Equal" + value = var.compute_profile + } + + toleration { + effect = "NoSchedule" + key = "demeter.run/compute-arch" + operator = "Equal" + value = var.compute_arch + } + + toleration { + effect = "NoSchedule" + key = "demeter.run/availability-sla" + operator = "Equal" + value = var.availability_sla + } + + volume { + name = "ipc" + empty_dir {} + } + + volume { + name = "proxy-config" + config_map { + name = "proxy-${var.network}-${var.salt}" + } + } + + dynamic "volume" { + for_each = var.is_custom == true ? toset([1]) : toset([]) + + content { + name = "network-config" + config_map { + name = "configs-${var.network}-${var.salt}" + } + } + } + + container { + image = "${var.node_image}:${var.node_image_tag}" + name = "main" + + args = local.arguments + + env { + name = "CARDANO_NETWORK" + value = var.network + } + + env { + name = "RESTORE_SNAPSHOT" + value = var.restore + } + + resources { + limits = var.node_resources.limits + requests = var.node_resources.requests + } + + port { + name = "n2n" + container_port = 3000 + } + + port { + name = "metrics" + container_port = 12798 + } + + volume_mount { + mount_path = "/data" + name = "data" + } + + volume_mount { + mount_path = "/ipc" + name = "ipc" + } + + dynamic "volume_mount" { + for_each = var.is_custom == true ? toset([1]) : toset([]) + + content { + mount_path = "/configuration" + name = "network-config" + } + } + + readiness_probe { + initial_delay_seconds = 20 + exec { + command = ["test", "-S", "/ipc/node.socket"] + } + } + } + + container { + name = "nginx" + image = "nginx" + + resources { + limits = { + memory = "100Mi" + } + requests = { + cpu = "10m" + memory = "100Mi" + } + } + + port { + name = "n2c" + container_port = 3307 + } + + volume_mount { + mount_path = "/ipc" + name = "ipc" + } + + volume_mount { + mount_path = "/etc/nginx" + name = "proxy-config" + } + } + } + } + } +} diff --git a/bootstrap/main.tf b/bootstrap/main.tf index 0d08211..74ad733 100644 --- a/bootstrap/main.tf +++ b/bootstrap/main.tf @@ -13,18 +13,95 @@ module "node_v1_feature" { extension_name = var.extension_name dns_zone = var.dns_zone api_key_salt = var.api_key_salt - dcu_per_package = var.dcu_per_package + dcu_per_second = var.dcu_per_second resources = var.operator_resources } +// blue (once we have a green, we can update its name to proxy-blue) module "node_v1_proxy" { depends_on = [kubernetes_namespace.namespace] source = "./proxy" namespace = var.namespace - replicas = var.proxy_replicas + replicas = var.proxy_blue_replicas extension_name = var.extension_name dns_zone = var.dns_zone - proxy_image_tag = var.proxy_image_tag + proxy_image_tag = var.proxy_blue_image_tag resources = var.proxy_resources - instances_namespace = var.instances_namespace + instances_namespace = var.proxy_blue_instances_namespace + healthcheck_port = var.proxy_blue_healthcheck_port + name = "proxy" +} + +module "node_v1_proxy_green" { + depends_on = [kubernetes_namespace.namespace] + source = "./proxy" + namespace = var.namespace + replicas = var.proxy_green_replicas + extension_name = var.extension_name + dns_zone = var.dns_zone + proxy_image_tag = var.proxy_green_image_tag + resources = var.proxy_resources + instances_namespace = var.proxy_green_instances_namespace + healthcheck_port = var.proxy_green_healthcheck_port + environment = "green" + name = "proxy-green" +} + + +module "instances" { + depends_on = [kubernetes_namespace.namespace] + source = "./instance" + for_each = var.instances + + namespace = var.namespace + replicas = each.value.replicas + node_resources = coalesce(each.value.node_resources, { + limits = { + "memory" = "2Gi" + "cpu" = "8" + } + requests = { + "memory" = "2Gi" + "cpu" = "100m" + } + }) + storage_size = coalesce(each.value.storage_size, "50Gi") + node_image = each.value.node_image + node_image_tag = each.value.image_tag + release = each.value.release + network = each.value.network + magic = each.value.magic + topology_zone = each.value.topology_zone + salt = each.value.salt + compute_arch = coalesce(each.value.compute_arch, "arm64") + compute_profile = coalesce(each.value.compute_profile, "mem-intensive") + availability_sla = coalesce(each.value.availability_sla, "consistent") + node_version = each.value.node_version + restore = coalesce(each.value.restore, false) + is_custom = coalesce(each.value.is_custom, false) +} + + +module "custom_configs" { + depends_on = [kubernetes_namespace.namespace] + source = "./configs" + for_each = { + for key, instance in var.instances : key => instance + if instance.is_custom == true + } + + namespace = var.namespace + network = each.value.network + salt = each.value.salt +} + +module "services" { + depends_on = [kubernetes_namespace.namespace] + for_each = var.services + source = "./service" + + namespace = var.namespace + network = each.value.network + release = each.value.release + active_salt = each.value.active_salt } diff --git a/bootstrap/proxy/cert.tf b/bootstrap/proxy/cert.tf index d143b2f..aaf811f 100644 --- a/bootstrap/proxy/cert.tf +++ b/bootstrap/proxy/cert.tf @@ -1,7 +1,7 @@ locals { # Add the extra URL to the list of generated URLs dns_names = ["${var.extension_name}.${var.dns_zone}", "*.${var.extension_name}.${var.dns_zone}"] - cert_secret_name = "${var.extension_name}-proxy-wildcard-tls" + cert_secret_name = var.environment != null ? "${var.extension_name}-${var.environment}-proxy-wildcard-tls" : "${var.extension_name}-proxy-wildcard-tls" } resource "kubernetes_manifest" "certificate_cluster_wildcard_tls" { diff --git a/bootstrap/proxy/config.tf b/bootstrap/proxy/config.tf index ed6a014..72f5843 100644 --- a/bootstrap/proxy/config.tf +++ b/bootstrap/proxy/config.tf @@ -3,56 +3,46 @@ // and multiplied for simplicity. Example: 1Mb/s => 1 * 1024 * 60 for the 1m // limiter. locals { + config_map_name = var.environment != null ? "${var.environment}-proxy-config" : "proxy-config" + tiers = [ { - "name" = "0", + "name" = "0", + "max_connections" = 1 "rates" = [ { - "interval" = "1m", - "limit" = floor(1 * 1024 * 1024 * 60) - }, - { - "interval" = "1d", - "limit" = floor(1 * 1024 * 1024 * 60 * 60 * 24) + "interval" = "1s", + "limit" = 1024 * 1024 } ] }, { - "name" = "1", + "name" = "1", + "max_connections" = 5 "rates" = [ { - "interval" = "1m", - "limit" = floor(5 * 1024 * 1024 * 60) - }, - { - "interval" = "1d", - "limit" = floor(5 * 1024 * 1024 * 60 * 60 * 24) + "interval" = "1s", + "limit" = 1024 * 1024 } ] }, { - "name" = "2", + "name" = "2", + "max_connections" = 25 "rates" = [ { - "interval" = "1m", - "limit" = floor(50 * 1024 * 1024 * 60) - }, - { - "interval" = "1d", - "limit" = floor(50 * 1024 * 1024 * 60 * 60 * 24) + "interval" = "1s", + "limit" = 1024 * 1024 } ] }, { - "name" = "3", + "name" = "3", + "max_connections" = 75 "rates" = [ { - "interval" = "1m", - "limit" = floor(100 * 1024 * 1024 * 60) - }, - { - "interval" = "1d", - "limit" = floor(100 * 1024 * 1024 * 60 * 60 * 24) + "interval" = "1s", + "limit" = 1024 * 1024 } ] } @@ -62,7 +52,7 @@ locals { resource "kubernetes_config_map" "proxy" { metadata { namespace = var.namespace - name = "proxy-config" + name = local.config_map_name } data = { diff --git a/bootstrap/proxy/deployment.tf b/bootstrap/proxy/deployment.tf index 396500a..b0d27f9 100644 --- a/bootstrap/proxy/deployment.tf +++ b/bootstrap/proxy/deployment.tf @@ -1,13 +1,3 @@ -locals { - name = "proxy" - role = "proxy" - - prometheus_port = 9187 - prometheus_addr = "0.0.0.0:${local.prometheus_port}" - proxy_port = 8080 - proxy_addr = "0.0.0.0:${local.proxy_port}" -} - resource "kubernetes_deployment_v1" "node_proxy" { wait_for_rollout = false depends_on = [kubernetes_manifest.certificate_cluster_wildcard_tls] @@ -15,23 +5,17 @@ resource "kubernetes_deployment_v1" "node_proxy" { metadata { name = local.name namespace = var.namespace - labels = { - role = local.role - } + labels = local.proxy_labels } spec { replicas = var.replicas selector { - match_labels = { - role = local.role - } + match_labels = local.proxy_labels } template { metadata { name = local.name - labels = { - role = local.role - } + labels = local.proxy_labels } spec { container { diff --git a/bootstrap/proxy/main.tf b/bootstrap/proxy/main.tf index 7cd49ca..a07f484 100644 --- a/bootstrap/proxy/main.tf +++ b/bootstrap/proxy/main.tf @@ -1,3 +1,24 @@ +locals { + name = var.name + role = "proxy" + + prometheus_port = 9187 + prometheus_addr = "0.0.0.0:${local.prometheus_port}" + proxy_port = 8080 + proxy_addr = "0.0.0.0:${local.proxy_port}" + proxy_labels = var.environment != null ? { role = "${local.role}-${var.environment}" } : { role = local.role } +} + +variable "name" { + type = string + default = "proxy" +} + +// blue - green +variable "environment" { + default = null +} + variable "namespace" { type = string } @@ -60,3 +81,7 @@ variable "dns_zone" { type = string default = "demeter.run" } + +variable "healthcheck_port" { + type = number +} \ No newline at end of file diff --git a/bootstrap/proxy/monitor.tf b/bootstrap/proxy/monitor.tf index 337b784..7427c67 100644 --- a/bootstrap/proxy/monitor.tf +++ b/bootstrap/proxy/monitor.tf @@ -7,14 +7,12 @@ resource "kubernetes_manifest" "proxy_monitor" { "app.kubernetes.io/component" = "o11y" "app.kubernetes.io/part-of" = "demeter" } - name = "proxy" + name = local.name namespace = var.namespace } spec = { selector = { - matchLabels = { - role = "proxy" - } + matchLabels = local.proxy_labels } podMetricsEndpoints = [ { diff --git a/bootstrap/proxy/proxy-config.toml.tftpl b/bootstrap/proxy/proxy-config.toml.tftpl index 8b20a49..4bb1587 100644 --- a/bootstrap/proxy/proxy-config.toml.tftpl +++ b/bootstrap/proxy/proxy-config.toml.tftpl @@ -1,6 +1,7 @@ %{ for tier in tiers ~} [[tiers]] name = "${tier.name}" +max_connections = ${tier.max_connections} %{ for rate in tier.rates ~} [[tiers.rates]] interval = "${rate.interval}" diff --git a/bootstrap/proxy/service.tf b/bootstrap/proxy/service.tf index 35c93e3..9d3e8dc 100644 --- a/bootstrap/proxy/service.tf +++ b/bootstrap/proxy/service.tf @@ -8,14 +8,13 @@ resource "kubernetes_service_v1" "proxy_service" { "service.beta.kubernetes.io/aws-load-balancer-type" : "external" "service.beta.kubernetes.io/aws-load-balancer-healthcheck-protocol" : "HTTP" "service.beta.kubernetes.io/aws-load-balancer-healthcheck-path" : "/health" + "service.beta.kubernetes.io/aws-load-balancer-healthcheck-port" : var.healthcheck_port != null ? var.healthcheck_port : "traffic-port" } } spec { load_balancer_class = "service.k8s.aws/nlb" - selector = { - role = local.role - } + selector = local.proxy_labels port { name = "proxy" diff --git a/bootstrap/service/main.tf b/bootstrap/service/main.tf new file mode 100644 index 0000000..e48d70d --- /dev/null +++ b/bootstrap/service/main.tf @@ -0,0 +1,54 @@ +variable "namespace" { + description = "the namespace where the resources will be created" +} + +variable "network" { + description = "the network where the resources will be created" +} + +variable "release" { + description = "the release where the resources will be created" +} + +variable "active_salt" { + description = "the salt to use for the active network" + default = "" +} + +locals { + selector = length(var.active_salt) > 0 ? { + "role" = "node" + "network" = var.network + "release" = var.release + "salt" = var.active_salt + } : { + "role" = "node" + "network" = var.network + "release" = var.release + } +} + +resource "kubernetes_service_v1" "well_known_service" { + metadata { + name = "node-${var.network}-${var.release}" + namespace = var.namespace + } + + spec { + port { + name = "n2c" + protocol = "TCP" + port = 3307 + } + + port { + name = "n2n" + protocol = "TCP" + port = 3000 + } + + selector = local.selector + + type = "ClusterIP" + } +} diff --git a/bootstrap/variables.tf b/bootstrap/variables.tf index bbdb880..59bb058 100644 --- a/bootstrap/variables.tf +++ b/bootstrap/variables.tf @@ -12,10 +12,6 @@ variable "extension_name" { default = "nodes-m1" } -variable "instances_namespace" { - type = string -} - // Operator variable "operator_image_tag" { type = string @@ -25,14 +21,14 @@ variable "api_key_salt" { type = string } -variable "dcu_per_package" { +variable "dcu_per_second" { type = map(string) default = { - "mainnet" = "10" - "preprod" = "5" - "preview" = "5" - "sanchonet" = "5" - "vector-testnet" = "5" + "mainnet" = "1" + "preprod" = "1" + "preview" = "1" + "sanchonet" = "1" + "vector-testnet" = "1" } } @@ -65,15 +61,44 @@ variable "operator_resources" { } // Proxy -variable "proxy_image_tag" { +variable "proxy_green_image_tag" { + type = string +} + +variable "proxy_green_replicas" { + type = number + default = 1 +} + +variable "proxy_green_healthcheck_port" { + type = number + description = "The port the loadbalancer assigned to the HTTP endpoint of the service. Usually known after the service is created. The default is the target-port." + default = null +} + +variable "proxy_green_instances_namespace" { type = string } -variable "proxy_replicas" { +variable "proxy_blue_image_tag" { + type = string +} + +variable "proxy_blue_replicas" { type = number default = 1 } +variable "proxy_blue_healthcheck_port" { + type = number + description = "The port the loadbalancer assigned to the HTTP endpoint of the service. Usually known after the service is created. The default is the target-port." + default = null +} + +variable "proxy_blue_instances_namespace" { + type = string +} + variable "proxy_resources" { type = object({ limits = object({ @@ -100,3 +125,41 @@ variable "proxy_resources" { } } } + +variable "instances" { + type = map(object({ + node_image = string + image_tag = string + network = string + salt = string + release = string + magic = number + topology_zone = string + node_resources = optional(object({ + limits = object({ + cpu = string + memory = string + }) + requests = object({ + cpu = string + memory = string + }) + })) + storage_size = optional(string) + node_version = string + replicas = number + restore = optional(bool) + compute_arch = optional(string) + compute_profile = optional(string) + availability_sla = optional(string) + is_custom = optional(bool) + })) +} + +variable "services" { + type = map(object({ + network = string + release = string + active_salt = optional(string) + })) +}