diff --git a/Cargo.lock b/Cargo.lock index 933204fd..34ef03e7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -76,9 +76,9 @@ dependencies = [ [[package]] name = "ahash" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72832d73be48bac96a5d7944568f305d829ed55b0ce3b483647089dfaf6cf704" +checksum = "cd7d5a2cecb58716e47d67d5703a249964b14c7be1ec3cad3affc295b2d1c35d" dependencies = [ "cfg-if", "getrandom 0.2.12", @@ -1792,13 +1792,14 @@ dependencies = [ [[package]] name = "geyser-grpc-connector" -version = "0.10.1+yellowstone.1.12" -source = "git+https://github.com/blockworks-foundation/geyser-grpc-connector.git?tag=v0.10.3+yellowstone.1.12+solana.1.17.15-hacked-windowsize-with-broadcast-exit#688e4d241dd18d18f57345d592e803aa673fcd96" +version = "0.10.6+yellowstone.1.13" +source = "git+https://github.com/blockworks-foundation/geyser-grpc-connector.git?tag=v0.10.6+yellowstone.1.13+solana.1.17.28#20f29c1e21e1682f124c0a3386cd4a85b1854f01" dependencies = [ "anyhow", "async-stream", "base64 0.21.7", "bincode", + "csv", "derive_more", "futures", "itertools 0.10.5", @@ -1925,7 +1926,7 @@ version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" dependencies = [ - "ahash 0.8.4", + "ahash 0.8.5", ] [[package]] @@ -4249,9 +4250,9 @@ dependencies = [ [[package]] name = "solana-account-decoder" -version = "1.17.15" +version = "1.17.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ea4bedfcc8686ae6d01a3d8288f5b9746cd00ec63f0ce9a6415849d35add50" +checksum = "c4e29f060cabd0e1bd90a63f8e1517ddd3365d3dc2eaa05f9a9fa542f4adeaaa" dependencies = [ "Inflector", "base64 0.21.7", @@ -4274,9 +4275,9 @@ dependencies = [ [[package]] name = "solana-address-lookup-table-program" -version = "1.17.15" +version = "1.17.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d89cd9fd1668735eab8c83407bde1d073a35c0486c0d9f1afc9ef75b9b726f94" +checksum = "71a93d1c1a0c13de20e13a0c9bd7275b00bb901b8b7c55424fea98f71cb37778" dependencies = [ "bincode", "bytemuck", @@ -4295,9 +4296,9 @@ dependencies = [ [[package]] name = "solana-clap-utils" -version = "1.17.15" +version = "1.17.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eba77d79b1202853954c7a8cc4261bc50b39f99872d09fd6bbd22373df161171" +checksum = "62e5cdc0ae0c8ae79c39a4a362066d0d61764bc7ea7e033961fd7510fd24da2a" dependencies = [ "chrono", "clap 2.34.0", @@ -4312,9 +4313,9 @@ dependencies = [ [[package]] name = "solana-client" -version = "1.17.15" +version = "1.17.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6948f1741ea197c04a989510b6810e1593a694848d54f9a128dc15a840484c1f" +checksum = "a1e2301c2af7e5a1dba0855f710329a2bb993829ed9fdf8f6207d02ee6fc54a4" dependencies = [ "async-trait", "bincode", @@ -4345,9 +4346,9 @@ dependencies = [ [[package]] name = "solana-config-program" -version = "1.17.15" +version = "1.17.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de23cd0dd8673f4590e90bfa47ff19eb629f4b7dc15a3fb173a62d932801d07" +checksum = "595118948b966b110aad3f9d8d8464958abe379ecfa7a813b4fc82659c8259bc" dependencies = [ "bincode", "chrono", @@ -4359,9 +4360,9 @@ dependencies = [ [[package]] name = "solana-connection-cache" -version = "1.17.15" +version = "1.17.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9300b6a51c990fdd16918a522258c384582ad63e2fadcfb9ad1574e4b315e937" +checksum = "d363d6bb43e618b6010b47c2eb0579777ce4ed388ca15b84a610a738edf0b97e" dependencies = [ "async-trait", "bincode", @@ -4381,11 +4382,11 @@ dependencies = [ [[package]] name = "solana-frozen-abi" -version = "1.17.15" +version = "1.17.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4090f2ac64149ce1fbabd5277f41e278edc1f38121927fe8f6355e67ead3e199" +checksum = "96734b05823c8b515f8e3cc02641a27aee2c9760b1a43c74cb20f2a1ab0ab76c" dependencies = [ - "ahash 0.8.4", + "ahash 0.8.5", "blake3", "block-buffer 0.10.4", "bs58", @@ -4411,9 +4412,9 @@ dependencies = [ [[package]] name = "solana-frozen-abi-macro" -version = "1.17.15" +version = "1.17.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "765bcdc1ecc31ea5d3d7ddb680ffa6645809c122b4ffdc223b161850e6ba352b" +checksum = "9a0f1291a464fd046135d019d57a81be165ee3d23aa7df880b47dac683a0582a" dependencies = [ "proc-macro2", "quote", @@ -4501,7 +4502,6 @@ dependencies = [ "solana-version", "thiserror", "tokio", - "yellowstone-grpc-client", "yellowstone-grpc-proto", ] @@ -4648,7 +4648,6 @@ dependencies = [ "tokio-stream", "tonic-health", "tracing", - "yellowstone-grpc-client", "yellowstone-grpc-proto", ] @@ -4858,9 +4857,9 @@ dependencies = [ [[package]] name = "solana-logger" -version = "1.17.15" +version = "1.17.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c7f3cad088bc5f00569cb5b4c3aaba8d935f8f7cc25c91cc0c55a8a7de2b137" +checksum = "c5977c8f24b83cf50e7139ffdb25d70bad6a177f18ccc79ca2293d6a987fa81c" dependencies = [ "env_logger", "lazy_static", @@ -4869,9 +4868,9 @@ dependencies = [ [[package]] name = "solana-measure" -version = "1.17.15" +version = "1.17.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2de5041d16120852c0deea047c024e1fad8819e49041491f0cca6c91c243fd5d" +checksum = "7a39ef01b2c65552d05013b2642ffd73258f2c80e3a59e44c499762047df9456" dependencies = [ "log", "solana-sdk", @@ -4879,9 +4878,9 @@ dependencies = [ [[package]] name = "solana-metrics" -version = "1.17.15" +version = "1.17.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fd6f25f0076b6eb873f7e2a85e53191ac2affe6782131be1a2867d057307e20" +checksum = "9ad30ff3775412f2929d440446aef8b070676920bc5df495ea6398a8f28ce91f" dependencies = [ "crossbeam-channel", "gethostname", @@ -4894,9 +4893,9 @@ dependencies = [ [[package]] name = "solana-net-utils" -version = "1.17.15" +version = "1.17.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12ff6114e678f321b3d421288dc12311e6e5ca5b72eadd962d4239276b3d66d2" +checksum = "6eafd5178a38a039e12c14780f1b6a74f1e672d62357343e0aee6d0fc7e5bd18" dependencies = [ "bincode", "clap 3.2.25", @@ -4916,11 +4915,11 @@ dependencies = [ [[package]] name = "solana-perf" -version = "1.17.15" +version = "1.17.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34b28f2db62b93cb04b56d610ac8736ee4fb89b4a030c55935b646b7212b6556" +checksum = "10d6293cddcc98ae092d00f43f741405da30aa083acb96666606130810b064f3" dependencies = [ - "ahash 0.8.4", + "ahash 0.8.5", "bincode", "bv", "caps", @@ -4945,9 +4944,9 @@ dependencies = [ [[package]] name = "solana-program" -version = "1.17.15" +version = "1.17.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1141d1dffbe68852128f7bbcc3c43a5d2cb715ecffeeb64eb81bb93cbaf80bb" +checksum = "6412447793f8a3ef7526655906728325093b472e481791ac5c584e8d272166dc" dependencies = [ "ark-bn254", "ark-ec", @@ -4999,9 +4998,9 @@ dependencies = [ [[package]] name = "solana-program-runtime" -version = "1.17.15" +version = "1.17.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "942de577a2865cec28fc174575c9bd6cf7af815832af67fe40ca856075550998" +checksum = "1977e741a6793fca27413507457d797df0f41bc0ae634247d112bc77ab2b0325" dependencies = [ "base64 0.21.7", "bincode", @@ -5027,9 +5026,9 @@ dependencies = [ [[package]] name = "solana-pubsub-client" -version = "1.17.15" +version = "1.17.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7ba8af6bfcb21abdde0a9fc2bade99e5e7c8fadab7f113ac7bceb408296ae26" +checksum = "1ad21dd5d6fe09116dbc29aec279b7cf08d250b564899dc87437bd780ed26290" dependencies = [ "crossbeam-channel", "futures-util", @@ -5052,9 +5051,9 @@ dependencies = [ [[package]] name = "solana-quic-client" -version = "1.17.15" +version = "1.17.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "460b3265aec34d9a746bec2b64e2fb2890d7af694c81a494aceddeb9fb77182c" +checksum = "6201869768fe133ce9b8088e4f718f53ff164b8e5df3d0d46a6563a22545924f" dependencies = [ "async-mutex", "async-trait", @@ -5079,9 +5078,9 @@ dependencies = [ [[package]] name = "solana-rayon-threadlimit" -version = "1.17.15" +version = "1.17.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dda21485597afb5edb4080b854d0e2a7d2d6a8e05b0f1f0abf8cb855b44b9b6c" +checksum = "1f100d0c3214d67bb847a1eefc7079f6bb755534266423f4c994ad3b40c685ed" dependencies = [ "lazy_static", "num_cpus", @@ -5089,9 +5088,9 @@ dependencies = [ [[package]] name = "solana-remote-wallet" -version = "1.17.15" +version = "1.17.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2b32421f5ce524405db85b3227e57386a245ac75fbf7e0a2407b2e35c789968" +checksum = "3328c891079086b408a04e701470a346d517c9c51c0a96f2f166f616a3e1c3c8" dependencies = [ "console", "dialoguer", @@ -5108,9 +5107,9 @@ dependencies = [ [[package]] name = "solana-rpc-client" -version = "1.17.15" +version = "1.17.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09220386ed831960f32119e6b87100588269712e764e4838e01f78dc11b1ec1d" +checksum = "bfacf1163a375d98c29779a03ba278b2ef43494f77e33826a33f9460563c0887" dependencies = [ "async-trait", "base64 0.21.7", @@ -5134,9 +5133,9 @@ dependencies = [ [[package]] name = "solana-rpc-client-api" -version = "1.17.15" +version = "1.17.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2ef8ffa33755c1e648be2c7433404a275a34b951787770970b77bc682ff12c5" +checksum = "7fab293a88113511e66607d76bd027edfe0b1372b467fd76bbb5af03448539a2" dependencies = [ "base64 0.21.7", "bs58", @@ -5156,9 +5155,9 @@ dependencies = [ [[package]] name = "solana-rpc-client-nonce-utils" -version = "1.17.15" +version = "1.17.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f46040b32f0e8b8e215caa7d43e2b879e1c1e2cade205bc3edfb3dcfa0632ac5" +checksum = "1e43cb51374a6ec8fd401b3387334ef93e04f6d8ae87bbb29892aff42aeb1061" dependencies = [ "clap 2.34.0", "solana-clap-utils", @@ -5169,9 +5168,9 @@ dependencies = [ [[package]] name = "solana-sdk" -version = "1.17.15" +version = "1.17.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "278a95acb99377dd4585599fdbec23d0a6fcb94ec78285283723fdd365fe885e" +checksum = "de1ce8848de4198f9bc7e4574252be02b1ed86ecbc2fff506780d5f8d6e4c4a8" dependencies = [ "assert_matches", "base64 0.21.7", @@ -5223,9 +5222,9 @@ dependencies = [ [[package]] name = "solana-sdk-macro" -version = "1.17.15" +version = "1.17.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92dbaf563210f61828800f2a3d8c188fa2afede91920d364982e280318db2eb5" +checksum = "bc5cc46bbda0a5472d8d0a4c846b22941436ac45c31456d3e885a387a5f264f7" dependencies = [ "bs58", "proc-macro2", @@ -5242,9 +5241,9 @@ checksum = "468aa43b7edb1f9b7b7b686d5c3aeb6630dc1708e86e31343499dd5c4d775183" [[package]] name = "solana-streamer" -version = "1.17.15" +version = "1.17.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8315b493d109f89842dcc9df02b4444beb182b810e3256f69a777c6f2a8147e" +checksum = "46f02b475fc20c55ebbcfa5638ff93f9b780414cc6185e3a6d0992bca0ae81ee" dependencies = [ "async-channel", "bytes", @@ -5264,6 +5263,7 @@ dependencies = [ "rand 0.8.5", "rcgen 0.10.0", "rustls", + "smallvec", "solana-metrics", "solana-perf", "solana-sdk", @@ -5274,9 +5274,9 @@ dependencies = [ [[package]] name = "solana-thin-client" -version = "1.17.15" +version = "1.17.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3db2f09ad842ede0c354a2e870ed8d7660aa9d85eabbff2ced6fc7b8ba409a90" +checksum = "9b6ce2304764b8bb699db734fde9cd19ace038d3895d828a557ea0ec2a9e0ecd" dependencies = [ "bincode", "log", @@ -5289,9 +5289,9 @@ dependencies = [ [[package]] name = "solana-tpu-client" -version = "1.17.15" +version = "1.17.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9263b1c6647258a5c36a54b8d79e1c366180b1eda65150b0cafb7b469c68f5e1" +checksum = "aa3e2351625e26f55e5e08f8e5aadaa2380fd0649f25641d6ba3f3848dbe5c9a" dependencies = [ "async-trait", "bincode", @@ -5313,9 +5313,9 @@ dependencies = [ [[package]] name = "solana-transaction-status" -version = "1.17.15" +version = "1.17.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e2031070cba17802f7108b53f6db01b82cdfb0360b0a8b9d51c584f2e9dd9e4" +checksum = "0841bbd1845c87043e4184961e45cc7c08b36d96d0d146256b26ea5c74630a0f" dependencies = [ "Inflector", "base64 0.21.7", @@ -5338,9 +5338,9 @@ dependencies = [ [[package]] name = "solana-udp-client" -version = "1.17.15" +version = "1.17.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "829c54d9706b8da0fe81b5b79c4291c4a7d366c16b3e19bcfb44125e1cc4741b" +checksum = "bae54a100f0b0b5be065f5d05f2259f6d4a7b39f5866d579927f3ca35a01773b" dependencies = [ "async-trait", "solana-connection-cache", @@ -5353,9 +5353,9 @@ dependencies = [ [[package]] name = "solana-version" -version = "1.17.15" +version = "1.17.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1dff08def0cc14d1ab26916ffdc11a1456620d6590c43c569c08a98fac91f7d" +checksum = "5f69945e38d7440221e2fac0aaa57a9d72adb329b0de705ca5bd9ba981aedc16" dependencies = [ "log", "rustc_version", @@ -5369,9 +5369,9 @@ dependencies = [ [[package]] name = "solana-vote-program" -version = "1.17.15" +version = "1.17.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c144aee890e2e62bcfab7079d0a5c5795b84c5cd4cbe904c64b8cd868b131a05" +checksum = "e574aafc3c5adc7106ab4605d8ad378c9a12f2cf1dec2e8ba1aa6fd97a5d5490" dependencies = [ "bincode", "log", @@ -5391,9 +5391,9 @@ dependencies = [ [[package]] name = "solana-zk-token-sdk" -version = "1.17.15" +version = "1.17.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef26fb44734aa940e6648bbbeead677edc68c7e1ec09128e5f16a8924c389a38" +checksum = "597dddc8ab46852dea7fc3d22e031fa4ffdb1b2291ac24d960605424a510a5f5" dependencies = [ "aes-gcm-siv", "base64 0.21.7", @@ -6787,8 +6787,9 @@ dependencies = [ [[package]] name = "yellowstone-grpc-client" -version = "1.13.0+solana.1.17.15" -source = "git+https://github.com/rpcpool/yellowstone-grpc.git?tag=v1.12.0+solana.1.17.15#c7b72cc8781c2dc48e4a7c94e411f95df495cf2f" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f4f6d836d214cb8789002d039412da354049e9ffe983c643ec492c4d934698f" dependencies = [ "bytes", "futures", @@ -6801,8 +6802,9 @@ dependencies = [ [[package]] name = "yellowstone-grpc-proto" -version = "1.12.0+solana.1.17.15" -source = "git+https://github.com/rpcpool/yellowstone-grpc.git?tag=v1.12.0+solana.1.17.15#c7b72cc8781c2dc48e4a7c94e411f95df495cf2f" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c00b66d40d85c405f14b420d7674e98c70d06f6b673f36c9e0285f81b9b797d" dependencies = [ "anyhow", "bincode", diff --git a/Cargo.toml b/Cargo.toml index 9cc87b93..804105a7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,19 +28,19 @@ license = "AGPL" edition = "2021" [workspace.dependencies] -solana-sdk = "~1.17.15" -solana-rpc-client = "~1.17.15" -solana-rpc-client-api = "~1.17.15" -solana-transaction-status = "~1.17.15" -solana-version = "~1.17.15" -solana-client = "~1.17.15" -solana-net-utils = "~1.17.15" -solana-pubsub-client = "~1.17.15" -solana-streamer = "~1.17.15" -solana-account-decoder = "~1.17.15" -solana-ledger = "~1.17.15" -solana-program = "~1.17.15" -solana-address-lookup-table-program = "~1.17.15" +solana-sdk = "~1.17.28" +solana-rpc-client = "~1.17.28" +solana-rpc-client-api = "~1.17.28" +solana-transaction-status = "~1.17.28" +solana-version = "~1.17.28" +solana-client = "~1.17.28" +solana-net-utils = "~1.17.28" +solana-pubsub-client = "~1.17.28" +solana-streamer = "~1.17.28" +solana-account-decoder = "~1.17.28" +solana-ledger = "~1.17.28" +solana-program = "~1.17.28" +solana-address-lookup-table-program = "~1.17.28" itertools = "0.10.5" rangetools = "0.1.4" serde = { version = "1.0.160", features = ["derive"] } @@ -87,7 +87,10 @@ solana-lite-rpc-accounts = {path = "accounts", version = "0.2.4"} solana-lite-rpc-accounts-on-demand = {path = "accounts-on-demand", version = "0.2.4"} bench = { path = "bench", version="0.2.4" } +yellowstone-grpc-proto = "1.13.0" +#geyser-grpc-connector = { path = "../../geyser-grpc-connector" } +geyser-grpc-connector = { tag = "v0.10.6+yellowstone.1.13+solana.1.17.28", git = "https://github.com/blockworks-foundation/geyser-grpc-connector.git" } + + async-trait = "0.1.68" -yellowstone-grpc-client = { git = "https://github.com/rpcpool/yellowstone-grpc.git", tag = "v1.12.0+solana.1.17.15" } -yellowstone-grpc-proto = { git = "https://github.com/rpcpool/yellowstone-grpc.git", tag = "v1.12.0+solana.1.17.15" } tonic-health = "0.10" diff --git a/accounts-on-demand/Cargo.toml b/accounts-on-demand/Cargo.toml index fb4f2800..21219332 100644 --- a/accounts-on-demand/Cargo.toml +++ b/accounts-on-demand/Cargo.toml @@ -45,7 +45,6 @@ solana-lite-rpc-core = { workspace = true } solana-lite-rpc-accounts = { workspace = true } solana-lite-rpc-cluster-endpoints = { workspace = true } -yellowstone-grpc-client = { workspace = true } yellowstone-grpc-proto = { workspace = true } [dev-dependencies] diff --git a/cluster-endpoints/Cargo.toml b/cluster-endpoints/Cargo.toml index f1c91319..2c0619e3 100644 --- a/cluster-endpoints/Cargo.toml +++ b/cluster-endpoints/Cargo.toml @@ -8,8 +8,7 @@ repository = "https://github.com/blockworks-foundation/lite-rpc" license = "AGPL" [dependencies] -#geyser-grpc-connector = { path = "../../geyser-grpc-connector" } -geyser-grpc-connector = { tag = "v0.10.3+yellowstone.1.12+solana.1.17.15-hacked-windowsize-with-broadcast-exit", git = "https://github.com/blockworks-foundation/geyser-grpc-connector.git" } +geyser-grpc-connector = { workspace = true } solana-sdk = { workspace = true } solana-rpc-client-api = { workspace = true } @@ -42,7 +41,6 @@ derive_more = "0.99.17" async-channel = { workspace = true } solana-lite-rpc-core = { workspace = true } solana-lite-rpc-util = { workspace = true } -yellowstone-grpc-client = { workspace = true } yellowstone-grpc-proto = { workspace = true } itertools = {workspace = true} prometheus = { workspace = true } diff --git a/cluster-endpoints/src/grpc/grpc_accounts_streaming.rs b/cluster-endpoints/src/grpc/grpc_accounts_streaming.rs index a52778cf..7713de61 100644 --- a/cluster-endpoints/src/grpc/grpc_accounts_streaming.rs +++ b/cluster-endpoints/src/grpc/grpc_accounts_streaming.rs @@ -6,7 +6,10 @@ use std::{ time::Duration, }; -use geyser_grpc_connector::GrpcSourceConfig; +use geyser_grpc_connector::yellowstone_grpc_util::{ + connect_with_timeout_with_buffers, GeyserGrpcClientBufferConfig, +}; +use geyser_grpc_connector::{GeyserGrpcClient, GeyserGrpcClientResult, GrpcSourceConfig}; use itertools::Itertools; use solana_lite_rpc_core::{ commitment_utils::Commitment, @@ -24,8 +27,7 @@ use yellowstone_grpc_proto::geyser::{ SubscribeRequest, SubscribeRequestFilterAccounts, SubscribeRequestFilterAccountsFilter, SubscribeRequestFilterAccountsFilterMemcmp, }; - -use crate::grpc::grpc_utils::connect_with_timeout_hacked; +use yellowstone_grpc_proto::tonic::service::Interceptor; pub fn start_account_streaming_tasks( grpc_config: GrpcSourceConfig, @@ -111,11 +113,8 @@ pub fn start_account_streaming_tasks( ping: None, }; - let mut client = connect_with_timeout_hacked( - grpc_config.grpc_addr.clone(), - grpc_config.grpc_x_token.clone(), - ) - .await?; + let mut client = create_connection(&grpc_config).await?; + let account_stream = client.subscribe_once2(program_subscription).await.unwrap(); // each account subscription batch will require individual stream @@ -136,11 +135,7 @@ pub fn start_account_streaming_tasks( filters: vec![], }, ); - let mut client = connect_with_timeout_hacked( - grpc_config.grpc_addr.clone(), - grpc_config.grpc_x_token.clone(), - ) - .await?; + let mut client = create_connection(&grpc_config).await?; let account_request = SubscribeRequest { accounts: accounts_subscription, @@ -219,6 +214,24 @@ pub fn start_account_streaming_tasks( }) } +async fn create_connection( + grpc_config: &GrpcSourceConfig, +) -> GeyserGrpcClientResult> { + connect_with_timeout_with_buffers( + grpc_config.grpc_addr.clone(), + grpc_config.grpc_x_token.clone(), + None, + Some(Duration::from_secs(10)), + Some(Duration::from_secs(10)), + GeyserGrpcClientBufferConfig { + buffer_size: Some(65536), + conn_window: Some(5242880), + stream_window: Some(4194304), + }, + ) + .await +} + pub fn create_grpc_account_streaming( grpc_sources: Vec, accounts_filters: AccountFilters, diff --git a/cluster-endpoints/src/grpc/grpc_utils.rs b/cluster-endpoints/src/grpc/grpc_utils.rs deleted file mode 100644 index 547ae181..00000000 --- a/cluster-endpoints/src/grpc/grpc_utils.rs +++ /dev/null @@ -1,38 +0,0 @@ -use bytes::Bytes; -use std::time::Duration; -use tonic::metadata::{errors::InvalidMetadataValue, AsciiMetadataValue}; -use tonic::service::Interceptor; -use tonic::transport::ClientTlsConfig; -use tonic_health::pb::health_client::HealthClient; -use yellowstone_grpc_client::{GeyserGrpcClient, InterceptorXToken}; -use yellowstone_grpc_proto::geyser::geyser_client::GeyserClient; -use yellowstone_grpc_proto::tonic; - -pub async fn connect_with_timeout_hacked( - endpoint: E, - x_token: Option, -) -> anyhow::Result> -where - E: Into, - T: TryInto, -{ - let endpoint = tonic::transport::Endpoint::from_shared(endpoint)? - .buffer_size(Some(65536)) - .initial_connection_window_size(4194304) - .initial_stream_window_size(4194304) - .connect_timeout(Duration::from_secs(10)) - .timeout(Duration::from_secs(10)) - // .http2_adaptive_window() - .tls_config(ClientTlsConfig::new())?; - - let x_token: Option = x_token.map(|v| v.try_into()).transpose()?; - let interceptor = InterceptorXToken { x_token }; - - let channel = endpoint.connect_lazy(); - let client = GeyserGrpcClient::new( - HealthClient::with_interceptor(channel.clone(), interceptor.clone()), - GeyserClient::with_interceptor(channel, interceptor) - .max_decoding_message_size(GeyserGrpcClient::max_decoding_message_size()), - ); - Ok(client) -} diff --git a/cluster-endpoints/src/grpc/mod.rs b/cluster-endpoints/src/grpc/mod.rs index 448512c8..6ba29f85 100644 --- a/cluster-endpoints/src/grpc/mod.rs +++ b/cluster-endpoints/src/grpc/mod.rs @@ -1,2 +1 @@ pub mod grpc_accounts_streaming; -pub mod grpc_utils; diff --git a/cluster-endpoints/src/grpc_subscription.rs b/cluster-endpoints/src/grpc_subscription.rs index 6e6a5228..7f0150bf 100644 --- a/cluster-endpoints/src/grpc_subscription.rs +++ b/cluster-endpoints/src/grpc_subscription.rs @@ -1,11 +1,13 @@ use crate::endpoint_stremers::EndpointStreaming; use crate::grpc::grpc_accounts_streaming::create_grpc_account_streaming; -use crate::grpc::grpc_utils::connect_with_timeout_hacked; use crate::grpc_multiplex::{ create_grpc_multiplex_blocks_subscription, create_grpc_multiplex_processed_slots_subscription, }; use anyhow::Context; use futures::StreamExt; +use geyser_grpc_connector::yellowstone_grpc_util::{ + connect_with_timeout_with_buffers, GeyserGrpcClientBufferConfig, +}; use geyser_grpc_connector::GrpcSourceConfig; use itertools::Itertools; use log::trace; @@ -36,13 +38,11 @@ use solana_transaction_status::{Reward, RewardType}; use std::cell::OnceCell; use std::collections::HashMap; use std::sync::Arc; +use std::time::Duration; use tokio::sync::{broadcast, Notify}; use tracing::trace_span; -use yellowstone_grpc_client::GeyserGrpcClient; use yellowstone_grpc_proto::geyser::subscribe_update::UpdateOneof; -use yellowstone_grpc_proto::geyser::{ - CommitmentLevel, SubscribeRequestFilterBlocks, SubscribeRequestFilterSlots, SubscribeUpdateSlot, -}; +use yellowstone_grpc_proto::geyser::{CommitmentLevel, SubscribeRequestFilterBlocks}; use crate::rpc_polling::vote_accounts_and_cluster_info_polling::{ poll_cluster_info, poll_vote_accounts, @@ -291,8 +291,19 @@ pub fn create_block_processing_task( ); // connect to grpc - let mut client = - connect_with_timeout_hacked(grpc_addr.clone(), grpc_x_token.clone()).await?; + let mut client = connect_with_timeout_with_buffers( + grpc_addr.clone(), + grpc_x_token.clone(), + None, + Some(Duration::from_secs(10)), + Some(Duration::from_secs(10)), + GeyserGrpcClientBufferConfig { + buffer_size: Some(65536), + conn_window: Some(5242880), + stream_window: Some(4194304), + }, + ) + .await?; let mut stream = tokio::select! { res = client .subscribe_once( @@ -358,68 +369,6 @@ pub fn create_block_processing_task( }) } -// not used -pub fn create_slot_stream_task( - grpc_addr: String, - grpc_x_token: Option, - slot_sx: tokio::sync::mpsc::Sender, - commitment_level: CommitmentLevel, -) -> AnyhowJoinHandle { - tokio::spawn(async move { - loop { - let mut slots = HashMap::new(); - slots.insert( - "client_slot".to_string(), - SubscribeRequestFilterSlots { - filter_by_commitment: Some(true), - }, - ); - - // connect to grpc - let mut client = - GeyserGrpcClient::connect(grpc_addr.clone(), grpc_x_token.clone(), None)?; - let mut stream = client - .subscribe_once( - slots, - Default::default(), - HashMap::new(), - Default::default(), - HashMap::new(), - Default::default(), - Some(commitment_level), - Default::default(), - None, - ) - .await?; - - while let Some(message) = stream.next().await { - let message = message?; - - let Some(update) = message.update_oneof else { - continue; - }; - - match update { - UpdateOneof::Slot(slot) => { - slot_sx - .send(slot) - .await - .context("Problem sending on block channel")?; - } - UpdateOneof::Ping(_) => { - log::trace!("GRPC Ping"); - } - _ => { - log::trace!("unknown GRPC notification"); - } - }; - } - log::error!("Grpc block subscription broken (resubscribing)"); - tokio::time::sleep(std::time::Duration::from_secs(1)).await; - } - }) -} - pub fn create_grpc_subscription( rpc_client: Arc, grpc_sources: Vec, diff --git a/cluster-endpoints/src/lib.rs b/cluster-endpoints/src/lib.rs index 5dd408c4..91b69404 100644 --- a/cluster-endpoints/src/lib.rs +++ b/cluster-endpoints/src/lib.rs @@ -10,4 +10,3 @@ pub mod json_rpc_subscription; pub mod rpc_polling; pub use geyser_grpc_connector; -pub use yellowstone_grpc_proto::geyser::CommitmentLevel; diff --git a/config.example.json b/config.example.json index 7f67ed02..4f023ea7 100644 --- a/config.example.json +++ b/config.example.json @@ -1,6 +1,6 @@ { - "rpc_addr": "http://0.0.0.0:8899", - "ws_addr": "ws://0.0.0.0:8900", + "rpc_addr": "http://rpcnode-upstream:8899", + "ws_addr": "ws://rpcnode-upstream:8900", "lite_rpc_http_addr": "[::]:8890", "lite_rpc_ws_addr": "[::]:8891", "fanout_size": 18, @@ -11,7 +11,7 @@ "quic_proxy_addr": null, "use_grpc": false, "calculate_leader_schedule_from_geyser": false, - "grpc_addr": "http://127.0.0.0:10000", + "grpc_addr": "http://yellowstone-grpc-upstream:10000", "grpc_x_token": null, "postgres": { "pg_config": "your_postgres_config", diff --git a/lite-rpc/src/main.rs b/lite-rpc/src/main.rs index 466879e8..0640eb35 100644 --- a/lite-rpc/src/main.rs +++ b/lite-rpc/src/main.rs @@ -145,10 +145,10 @@ pub async fn start_lite_rpc(args: Config, rpc_client: Arc) -> anyhow: } let timeouts = GrpcConnectionTimeouts { - connect_timeout: Duration::from_secs(5), - request_timeout: Duration::from_secs(5), - subscribe_timeout: Duration::from_secs(5), - receive_timeout: Duration::from_secs(5), + connect_timeout: Duration::from_secs(15), + request_timeout: Duration::from_secs(15), + subscribe_timeout: Duration::from_secs(15), + receive_timeout: Duration::from_secs(15), }; let gprc_sources = grpc_sources diff --git a/stake_vote/Cargo.toml b/stake_vote/Cargo.toml index 9186ea38..84811433 100644 --- a/stake_vote/Cargo.toml +++ b/stake_vote/Cargo.toml @@ -19,8 +19,8 @@ itertools = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } -yellowstone-grpc-client = { workspace = true } -yellowstone-grpc-proto = { workspace = true } +#yellowstone-grpc-client = { workspace = true } +#yellowstone-grpc-proto = { workspace = true } solana-sdk = { workspace = true } solana-client = { workspace = true } solana-ledger = { workspace = true }