From e48e30a2f211904bab860e3e1b01767c097beecc Mon Sep 17 00:00:00 2001 From: Isaac-DeFrain Date: Wed, 17 May 2023 00:40:56 +0000 Subject: [PATCH 1/6] (precomputed blocks) Add local logging to individual files --- .../src/cli_entrypoint/mina_cli_entrypoint.ml | 14 ++- src/lib/mina_lib/coda_subscriptions.ml | 107 ++++++++++++++---- src/lib/mina_lib/mina_lib.ml | 11 +- 3 files changed, 106 insertions(+), 26 deletions(-) diff --git a/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml b/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml index a78496da6bb..e9114ae069a 100644 --- a/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml +++ b/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml @@ -353,15 +353,21 @@ let setup_daemon logger = dummy proofs (none)" and plugins = plugin_flag and precomputed_blocks_path = - flag "--precomputed-blocks-file" - ~aliases:[ "precomputed-blocks-file" ] + flag "--precomputed-blocks-path" + ~aliases:[ "precomputed-blocks-path" ] (optional string) - ~doc:"PATH Path to write precomputed blocks to, for replay or archiving" + ~doc: + "PATH Path to write precomputed blocks to, for replay or archiving. If \ + PATH is a directory, precomputed blocks will be logged to individual \ + files within this directory. Otherwise, they will be appended to the \ + same file." and log_precomputed_blocks = flag "--log-precomputed-blocks" ~aliases:[ "log-precomputed-blocks" ] (optional_with_default false bool) - ~doc:"true|false Include precomputed blocks in the log (default: false)" + ~doc: + "true|false Include precomputed blocks in the log (default: false). \ + See also --precomputed-block-path for additional functionality." and block_reward_threshold = flag "--minimum-block-reward" ~aliases:[ "minimum-block-reward" ] ~doc: diff --git a/src/lib/mina_lib/coda_subscriptions.ml b/src/lib/mina_lib/coda_subscriptions.ml index 1c270b6225f..8d956c46cf1 100644 --- a/src/lib/mina_lib/coda_subscriptions.ml +++ b/src/lib/mina_lib/coda_subscriptions.ml @@ -42,7 +42,7 @@ let add_new_subscription (t : t) ~pk = let create ~logger ~constraint_constants ~wallets ~new_blocks ~transition_frontier ~is_storing_all ~time_controller - ~upload_blocks_to_gcloud ~precomputed_block_writer = + ~upload_blocks_to_gcloud ~precomputed_block_writer ~log_precomputed_blocks = let subscribed_block_users = Optional_public_key.Table.of_alist_multi @@ List.map (Secrets.Wallets.pks wallets) ~f:(fun wallet -> @@ -101,13 +101,26 @@ let create ~logger ~constraint_constants ~wallets ~new_blocks let gcloud_keyfile = match Core.Sys.getenv "GCLOUD_KEYFILE" with | Some keyfile -> + if upload_blocks_to_gcloud then + [%log info] "GCLOUD_KEYFILE environment variable set to %s" keyfile ; Some keyfile | _ -> - [%log warn] - "GCLOUD_KEYFILE environment variable not set. Must be set to use \ - upload_blocks_to_gcloud" ; + if upload_blocks_to_gcloud then + [%log warn] + "GCLOUD_KEYFILE environment variable not set. Must be set to use \ + upload_blocks_to_gcloud" ; None in + Option.iter (fst !precomputed_block_writer) ~f:(fun path -> + match path with + | `Path_dir path -> + [%log info] + ~metadata:[ ("path", `String path) ] + "Precomputed blocks will be logged to individual files in $path" + | `Path path -> + [%log info] + ~metadata:[ ("path", `String path) ] + "Precomputed blocks will be logged to the same file $path" ) ; Option.iter gcloud_keyfile ~f:(fun path -> ignore ( Core.Sys.command @@ -121,7 +134,7 @@ let create ~logger ~constraint_constants ~wallets ~new_blocks Mina_block.Validated.forget new_block |> State_hash.With_state_hashes.state_hash in - (let path, log = !precomputed_block_writer in + (let path, _log = !precomputed_block_writer in let precomputed_block = lazy (let scheduled_time = Block_time.now time_controller in @@ -132,8 +145,8 @@ let create ~logger ~constraint_constants ~wallets ~new_blocks in Precomputed.to_yojson precomputed_block ) in - if upload_blocks_to_gcloud then ( - [%log info] "log" ; + (* Upload precomputed blocks to gcloud *) + ( if upload_blocks_to_gcloud then let json = Yojson.Safe.to_string (Lazy.force precomputed_block) in let network = match Core.Sys.getenv "NETWORK_NAME" with @@ -158,6 +171,12 @@ let create ~logger ~constraint_constants ~wallets ~new_blocks match (gcloud_keyfile, network, bucket) with | Some _, Some network, Some bucket -> let hash_string = State_hash.to_base58_check hash in + [%log info] + ~metadata: + [ ("hash", `String hash_string) + ; ("bucket", `String bucket) + ] + "Uploading precomputed block with $hash to gcloud $bucket" ; let height = Mina_block.Validated.forget new_block |> With_hash.data |> Mina_block.blockchain_length @@ -217,19 +236,67 @@ let create ~logger ~constraint_constants ~wallets ~new_blocks ) | _ -> () ) ; - Option.iter path ~f:(fun (`Path path) -> - Out_channel.with_file ~append:true path ~f:(fun out_channel -> - Out_channel.output_lines out_channel - [ Yojson.Safe.to_string (Lazy.force precomputed_block) ] ) ) ; - [%log info] "Saw block with state hash $state_hash" - ~metadata: - (let state_hash_data = - [ ("state_hash", `String (State_hash.to_base58_check hash)) ] - in - if is_some log then - state_hash_data - @ [ ("precomputed_block", Lazy.force precomputed_block) ] - else state_hash_data ) ) ; + (* Log precomputed blocks locally *) + Option.iter path ~f:(fun path -> + if log_precomputed_blocks then + let json = + Yojson.Safe.to_string (Lazy.force precomputed_block) + in + match path with + | `Path path -> + (* original logging functionality, appends to single file *) + Out_channel.with_file ~append:true path + ~f:(fun out_channel -> + Out_channel.output_lines out_channel [ json ] ) + | `Path_dir path -> ( + (* log precomputed blocks to individual files in the directory *) + let network = + match Core.Sys.getenv "NETWORK_NAME" with + | Some network -> + Some network + | _ -> + [%log warn] + "NETWORK_NAME environment variable not set. \ + Default to 'mainnet'" ; + Some "mainnet" + in + match network with + | Some network -> + let hash_string = State_hash.to_base58_check hash in + let height = + Mina_block.Validated.forget new_block + |> With_hash.data |> Mina_block.blockchain_length + |> Mina_numbers.Length.to_string + in + let name = + sprintf "%s-%s-%s.json" network height hash_string + in + let fpath = + Core.Filename.(parts path @ [ name ] |> of_parts) + in + Out_channel.with_file ~append:false fpath + ~f:(fun out_channel -> + Out_channel.output_lines out_channel [ json ] ) ; + [%log info] + ~metadata: + [ ("block", `String name) + ; ("path", `String path) + ; ( "time" + , `String + Time.( + now () + |> to_string_iso8601_basic ~zone:Zone.utc) + ) + ] + "Logged precomputed $block to $path at $time" + | None -> + () ) + else + [%log info] + ~metadata: + [ ("state_hash", `String (State_hash.to_base58_check hash)) + ] + "Saw block with state hash $state_hash" ) ) ; let new_block_no_hash = Mina_block.Validated.forget new_block |> With_hash.data in diff --git a/src/lib/mina_lib/mina_lib.ml b/src/lib/mina_lib/mina_lib.ml index 24cdb3dcb26..5e0da4a2cce 100644 --- a/src/lib/mina_lib/mina_lib.ml +++ b/src/lib/mina_lib/mina_lib.ml @@ -105,7 +105,7 @@ type t = ; subscriptions : Coda_subscriptions.t ; sync_status : Sync_status.t Mina_incremental.Status.Observer.t ; precomputed_block_writer : - ([ `Path of string ] option * [ `Log ] option) ref + ([ `Path of string | `Path_dir of string ] option * [ `Log ] option) ref ; block_production_status : [ `Producing | `Producing_in_ms of float | `Free ] ref ; in_memory_reverse_structured_log_messages_for_integration_test : @@ -1986,10 +1986,16 @@ let create ?wallets (config : Config.t) = Archive_client.run ~logger:config.logger ~frontier_broadcast_pipe:frontier_broadcast_pipe_r archive_process_port ) ; + (* To log precomputed blocks to individual files, set both + --precomputed-blocks-path DIR and --log-precomputed-blocks true *) let precomputed_block_writer = ref ( Option.map config.precomputed_blocks_path ~f:(fun path -> - `Path path ) + match Core.Unix.(stat path).st_kind with + | S_DIR -> + `Path_dir path + | _ -> + `Path path ) , if config.log_precomputed_blocks then Some `Log else None ) in let subscriptions = @@ -1999,6 +2005,7 @@ let create ?wallets (config : Config.t) = ~is_storing_all:config.is_archive_rocksdb ~upload_blocks_to_gcloud:config.upload_blocks_to_gcloud ~time_controller:config.time_controller ~precomputed_block_writer + ~log_precomputed_blocks:config.log_precomputed_blocks in let open Mina_incremental.Status in let transition_frontier_incr = From ced6aa9db985291d17fcfcae7e83439059750af6 Mon Sep 17 00:00:00 2001 From: Isaac-DeFrain Date: Fri, 19 May 2023 00:30:52 +0000 Subject: [PATCH 2/6] (precomputed blocks) Handle only regular paths --- src/lib/mina_lib/mina_lib.ml | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/lib/mina_lib/mina_lib.ml b/src/lib/mina_lib/mina_lib.ml index 5e0da4a2cce..37af10f8667 100644 --- a/src/lib/mina_lib/mina_lib.ml +++ b/src/lib/mina_lib/mina_lib.ml @@ -1990,12 +1990,20 @@ let create ?wallets (config : Config.t) = --precomputed-blocks-path DIR and --log-precomputed-blocks true *) let precomputed_block_writer = ref - ( Option.map config.precomputed_blocks_path ~f:(fun path -> - match Core.Unix.(stat path).st_kind with - | S_DIR -> - `Path_dir path - | _ -> - `Path path ) + ( ( try + Option.map config.precomputed_blocks_path ~f:(fun path -> + match Core.Unix.(lstat path).st_kind with + | S_DIR -> + `Path_dir path + | S_REG -> + `Path path + | _ -> + [%log' error config.logger] + ~metadata:[ ("path", `String path) ] + "$path is not a regular Unix file or directory. \ + Local precomputed block logging disabled." ; + failwith "No precomputed block logging" ) + with _ -> None ) , if config.log_precomputed_blocks then Some `Log else None ) in let subscriptions = From 867831f4af15eb94f9e843fe769432880022a8ba Mon Sep 17 00:00:00 2001 From: Isaac-DeFrain Date: Fri, 19 May 2023 00:31:34 +0000 Subject: [PATCH 3/6] (precomputed blocks) Improve logging and cli flags --- .../src/cli_entrypoint/mina_cli_entrypoint.ml | 10 +- src/lib/mina_lib/coda_subscriptions.ml | 155 +++++++++--------- 2 files changed, 77 insertions(+), 88 deletions(-) diff --git a/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml b/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml index e9114ae069a..00ed3a1bdc2 100644 --- a/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml +++ b/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml @@ -358,16 +358,14 @@ let setup_daemon logger = (optional string) ~doc: "PATH Path to write precomputed blocks to, for replay or archiving. If \ - PATH is a directory, precomputed blocks will be logged to individual \ - files within this directory. Otherwise, they will be appended to the \ - same file." + path is a directory, precomputed blocks will be logged to individual \ + files within this directory. If path is a file, they will be appended \ + to this file. Otherwise, precomputed blocks will not be dumped." and log_precomputed_blocks = flag "--log-precomputed-blocks" ~aliases:[ "log-precomputed-blocks" ] (optional_with_default false bool) - ~doc: - "true|false Include precomputed blocks in the log (default: false). \ - See also --precomputed-block-path for additional functionality." + ~doc:"true|false Include precomputed blocks in the log (default: false)" and block_reward_threshold = flag "--minimum-block-reward" ~aliases:[ "minimum-block-reward" ] ~doc: diff --git a/src/lib/mina_lib/coda_subscriptions.ml b/src/lib/mina_lib/coda_subscriptions.ml index 8d956c46cf1..f44fd9b32bc 100644 --- a/src/lib/mina_lib/coda_subscriptions.ml +++ b/src/lib/mina_lib/coda_subscriptions.ml @@ -98,6 +98,21 @@ let create ~logger ~constraint_constants ~wallets ~new_blocks Pipe.write_without_pushback writer { With_hash.data; hash } ) ) ~if_not_found:ignore in + let dump_precomputed_blocks = + Option.is_some (fst !precomputed_block_writer) + in + let network = + match Core.Sys.getenv "NETWORK_NAME" with + | Some network -> + if upload_blocks_to_gcloud || dump_precomputed_blocks then + [%log info] "NETWORK_NAME environment variable set to %s" network ; + network + | _ -> + if log_precomputed_blocks || dump_precomputed_blocks then + [%log warn] + "NETWORK_NAME environment variable not set. Default to 'mainnet'" ; + "mainnet" + in let gcloud_keyfile = match Core.Sys.getenv "GCLOUD_KEYFILE" with | Some keyfile -> @@ -111,6 +126,20 @@ let create ~logger ~constraint_constants ~wallets ~new_blocks upload_blocks_to_gcloud" ; None in + let gcloud_bucket = + match Core.Sys.getenv "GCLOUD_BLOCK_UPLOAD_BUCKET" with + | Some bucket -> + if upload_blocks_to_gcloud then + [%log info] + "GCLOUD_BLOCK_UPLOAD_BUCKET environment variable set to %s" bucket ; + Some bucket + | _ -> + if upload_blocks_to_gcloud then + [%log warn] + "GCLOUD_BLOCK_UPLOAD_BUCKET environment variable not set. Must be \ + set to use upload_blocks_to_gcloud" ; + None + in Option.iter (fst !precomputed_block_writer) ~f:(fun path -> match path with | `Path_dir path -> @@ -134,7 +163,7 @@ let create ~logger ~constraint_constants ~wallets ~new_blocks Mina_block.Validated.forget new_block |> State_hash.With_state_hashes.state_hash in - (let path, _log = !precomputed_block_writer in + (let path, log = !precomputed_block_writer in let precomputed_block = lazy (let scheduled_time = Block_time.now time_controller in @@ -148,28 +177,8 @@ let create ~logger ~constraint_constants ~wallets ~new_blocks (* Upload precomputed blocks to gcloud *) ( if upload_blocks_to_gcloud then let json = Yojson.Safe.to_string (Lazy.force precomputed_block) in - let network = - match Core.Sys.getenv "NETWORK_NAME" with - | Some network -> - Some network - | _ -> - [%log warn] - "NETWORK_NAME environment variable not set. Must be set \ - to use upload_blocks_to_gcloud" ; - None - in - let bucket = - match Core.Sys.getenv "GCLOUD_BLOCK_UPLOAD_BUCKET" with - | Some bucket -> - Some bucket - | _ -> - [%log warn] - "GCLOUD_BLOCK_UPLOAD_BUCKET environment variable not set. \ - Must be set to use upload_blocks_to_gcloud" ; - None - in - match (gcloud_keyfile, network, bucket) with - | Some _, Some network, Some bucket -> + match (gcloud_keyfile, gcloud_bucket) with + | Some _, Some bucket -> let hash_string = State_hash.to_base58_check hash in [%log info] ~metadata: @@ -238,65 +247,47 @@ let create ~logger ~constraint_constants ~wallets ~new_blocks () ) ; (* Log precomputed blocks locally *) Option.iter path ~f:(fun path -> - if log_precomputed_blocks then - let json = - Yojson.Safe.to_string (Lazy.force precomputed_block) - in - match path with - | `Path path -> - (* original logging functionality, appends to single file *) - Out_channel.with_file ~append:true path - ~f:(fun out_channel -> - Out_channel.output_lines out_channel [ json ] ) - | `Path_dir path -> ( - (* log precomputed blocks to individual files in the directory *) - let network = - match Core.Sys.getenv "NETWORK_NAME" with - | Some network -> - Some network - | _ -> - [%log warn] - "NETWORK_NAME environment variable not set. \ - Default to 'mainnet'" ; - Some "mainnet" - in - match network with - | Some network -> - let hash_string = State_hash.to_base58_check hash in - let height = - Mina_block.Validated.forget new_block - |> With_hash.data |> Mina_block.blockchain_length - |> Mina_numbers.Length.to_string - in - let name = - sprintf "%s-%s-%s.json" network height hash_string - in - let fpath = - Core.Filename.(parts path @ [ name ] |> of_parts) - in - Out_channel.with_file ~append:false fpath - ~f:(fun out_channel -> - Out_channel.output_lines out_channel [ json ] ) ; - [%log info] - ~metadata: - [ ("block", `String name) - ; ("path", `String path) - ; ( "time" - , `String - Time.( - now () - |> to_string_iso8601_basic ~zone:Zone.utc) - ) - ] - "Logged precomputed $block to $path at $time" - | None -> - () ) - else - [%log info] - ~metadata: - [ ("state_hash", `String (State_hash.to_base58_check hash)) - ] - "Saw block with state hash $state_hash" ) ) ; + let json = + Yojson.Safe.to_string (Lazy.force precomputed_block) + in + match path with + | `Path path -> + (* original logging functionality, appends to single file *) + Out_channel.with_file ~append:true path + ~f:(fun out_channel -> + Out_channel.output_lines out_channel [ json ] ) + | `Path_dir path -> + (* log precomputed blocks to individual files in the directory *) + let hash_string = State_hash.to_base58_check hash in + let height = + Mina_block.Validated.forget new_block + |> With_hash.data |> Mina_block.blockchain_length + |> Mina_numbers.Length.to_string + in + let name = + sprintf "%s-%s-%s.json" network height hash_string + in + let fpath = + Core.Filename.(parts path @ [ name ] |> of_parts) + in + Out_channel.with_file ~append:false fpath + ~f:(fun out_channel -> + Out_channel.output_lines out_channel [ json ] ) ; + [%log info] + ~metadata: + [ ("block", `String name); ("path", `String path) ] + "Logged precomputed $block to $path" ) ; + if log_precomputed_blocks then + [%log info] "Saw block with state hash $state_hash" + ~metadata: + (let state_hash_data = + [ ("state_hash", `String (State_hash.to_base58_check hash)) + ] + in + if is_some log then + state_hash_data + @ [ ("precomputed_block", Lazy.force precomputed_block) ] + else state_hash_data ) ) ; let new_block_no_hash = Mina_block.Validated.forget new_block |> With_hash.data in From 7512d30f134e6bae375fec7eb3efb3c82dc9b04c Mon Sep 17 00:00:00 2001 From: Isaac-DeFrain Date: Mon, 29 May 2023 19:59:32 +0000 Subject: [PATCH 4/6] (precomputed blocks) Add precomputed-blocks-dir CLI flag --- .../src/cli_entrypoint/mina_cli_entrypoint.ml | 18 +++++++++--------- src/lib/mina_lib/config.ml | 3 ++- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml b/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml index 00ed3a1bdc2..7bfa718723a 100644 --- a/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml +++ b/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml @@ -352,15 +352,15 @@ let setup_daemon logger = with full proving (full), snark-testing with dummy proofs (check), or \ dummy proofs (none)" and plugins = plugin_flag - and precomputed_blocks_path = - flag "--precomputed-blocks-path" - ~aliases:[ "precomputed-blocks-path" ] + and precomputed_blocks_file = + flag "--precomputed-blocks-file" + ~aliases:[ "precomputed-blocks-file" ] (optional string) - ~doc: - "PATH Path to write precomputed blocks to, for replay or archiving. If \ - path is a directory, precomputed blocks will be logged to individual \ - files within this directory. If path is a file, they will be appended \ - to this file. Otherwise, precomputed blocks will not be dumped." + ~doc:"PATH File to append precomputed blocks to, for replay or archiving." + and precomputed_blocks_dir = + flag "--precomputed-blocks-dir" + ~aliases:[ "precomputed-blocks-dir" ] + (optional string) ~doc:"PATH Directory to dump precomputed blocks to." and log_precomputed_blocks = flag "--log-precomputed-blocks" ~aliases:[ "log-precomputed-blocks" ] @@ -1308,7 +1308,7 @@ Pass one of -peer, -peer-list-file, -seed, -peer-list-url.|} ; ~consensus_local_state ~is_archive_rocksdb ~work_reassignment_wait ~archive_process_location ~log_block_creation ~precomputed_values ~start_time - ?precomputed_blocks_path ~log_precomputed_blocks + ?precomputed_blocks_file ?precomputed_blocks_dir ~log_precomputed_blocks ~upload_blocks_to_gcloud ~block_reward_threshold ~uptime_url ~uptime_submitter_keypair ~stop_time ~node_status_url ~node_status_type () ) diff --git a/src/lib/mina_lib/config.ml b/src/lib/mina_lib/config.ml index 7a049af7150..a805635606e 100644 --- a/src/lib/mina_lib/config.ml +++ b/src/lib/mina_lib/config.ml @@ -51,7 +51,8 @@ type t = ; log_block_creation : bool [@default false] ; precomputed_values : Precomputed_values.t ; start_time : Time.t - ; precomputed_blocks_path : string option + ; precomputed_blocks_file : string option + ; precomputed_blocks_dir : string option ; log_precomputed_blocks : bool ; upload_blocks_to_gcloud : bool ; block_reward_threshold : Currency.Amount.t option [@default None] From 5a6ce7eb17b4109bd504242e6777ca0bcf990089 Mon Sep 17 00:00:00 2001 From: Isaac-DeFrain Date: Mon, 29 May 2023 20:00:38 +0000 Subject: [PATCH 5/6] (precomputed blocks) Separate writing to file and dir --- src/lib/mina_lib/coda_subscriptions.ml | 115 +++++++++++++------------ src/lib/mina_lib/mina_lib.ml | 53 +++++++----- src/lib/mina_lib/mina_lib.mli | 1 + 3 files changed, 94 insertions(+), 75 deletions(-) diff --git a/src/lib/mina_lib/coda_subscriptions.ml b/src/lib/mina_lib/coda_subscriptions.ml index f44fd9b32bc..05bb18b794a 100644 --- a/src/lib/mina_lib/coda_subscriptions.ml +++ b/src/lib/mina_lib/coda_subscriptions.ml @@ -14,6 +14,15 @@ module Optional_public_key = struct include Hashable.Make (T) end +module Precomputed_block_writer = struct + type t = + { file : string option + ; dir : string option + ; log : bool + } + [@@deriving fields] +end + type t = { subscribed_payment_users : Signed_command.t reader_and_writer Public_key.Compressed.Table.t @@ -42,7 +51,7 @@ let add_new_subscription (t : t) ~pk = let create ~logger ~constraint_constants ~wallets ~new_blocks ~transition_frontier ~is_storing_all ~time_controller - ~upload_blocks_to_gcloud ~precomputed_block_writer ~log_precomputed_blocks = + ~upload_blocks_to_gcloud ~precomputed_block_writer = let subscribed_block_users = Optional_public_key.Table.of_alist_multi @@ List.map (Secrets.Wallets.pks wallets) ~f:(fun wallet -> @@ -99,8 +108,7 @@ let create ~logger ~constraint_constants ~wallets ~new_blocks ~if_not_found:ignore in let dump_precomputed_blocks = - Option.is_some (fst !precomputed_block_writer) - in + is_some (!precomputed_block_writer : Precomputed_block_writer.t).dir in let network = match Core.Sys.getenv "NETWORK_NAME" with | Some network -> @@ -108,7 +116,7 @@ let create ~logger ~constraint_constants ~wallets ~new_blocks [%log info] "NETWORK_NAME environment variable set to %s" network ; network | _ -> - if log_precomputed_blocks || dump_precomputed_blocks then + if upload_blocks_to_gcloud || dump_precomputed_blocks then [%log warn] "NETWORK_NAME environment variable not set. Default to 'mainnet'" ; "mainnet" @@ -140,16 +148,14 @@ let create ~logger ~constraint_constants ~wallets ~new_blocks set to use upload_blocks_to_gcloud" ; None in - Option.iter (fst !precomputed_block_writer) ~f:(fun path -> - match path with - | `Path_dir path -> - [%log info] - ~metadata:[ ("path", `String path) ] - "Precomputed blocks will be logged to individual files in $path" - | `Path path -> - [%log info] - ~metadata:[ ("path", `String path) ] - "Precomputed blocks will be logged to the same file $path" ) ; + Option.iter (!precomputed_block_writer).file ~f:(fun path -> + [%log info] + ~metadata:[ ("path", `String path) ] + "Precomputed blocks will be logged to the same file $path" ) ; + Option.iter (!precomputed_block_writer).dir ~f:(fun path -> + [%log info] + ~metadata:[ ("path", `String path) ] + "Precomputed blocks will be logged to individual files in $path" ) ; Option.iter gcloud_keyfile ~f:(fun path -> ignore ( Core.Sys.command @@ -163,7 +169,7 @@ let create ~logger ~constraint_constants ~wallets ~new_blocks Mina_block.Validated.forget new_block |> State_hash.With_state_hashes.state_hash in - (let path, log = !precomputed_block_writer in + (let Precomputed_block_writer.{ file; dir; log } = !precomputed_block_writer in let precomputed_block = lazy (let scheduled_time = Block_time.now time_controller in @@ -180,20 +186,21 @@ let create ~logger ~constraint_constants ~wallets ~new_blocks match (gcloud_keyfile, gcloud_bucket) with | Some _, Some bucket -> let hash_string = State_hash.to_base58_check hash in + let height = + Mina_block.Validated.forget new_block + |> With_hash.data |> Mina_block.blockchain_length + |> Mina_numbers.Length.to_string + in + let name = + sprintf "%s-%s-%s.json" network height hash_string + in [%log info] ~metadata: [ ("hash", `String hash_string) + ; ("height", `String height) ; ("bucket", `String bucket) ] "Uploading precomputed block with $hash to gcloud $bucket" ; - let height = - Mina_block.Validated.forget new_block - |> With_hash.data |> Mina_block.blockchain_length - |> Mina_numbers.Length.to_string - in - let name = - sprintf "%s-%s-%s.json" network height hash_string - in (* TODO: Use a pipe to queue this if these are building up *) don't_wait_for ( Mina_metrics.( @@ -245,46 +252,46 @@ let create ~logger ~constraint_constants ~wallets ~new_blocks ) | _ -> () ) ; - (* Log precomputed blocks locally *) - Option.iter path ~f:(fun path -> + (* Write precomputed blocks locally *) + Option.iter file ~f:(fun path -> let json = Yojson.Safe.to_string (Lazy.force precomputed_block) in - match path with - | `Path path -> - (* original logging functionality, appends to single file *) - Out_channel.with_file ~append:true path - ~f:(fun out_channel -> - Out_channel.output_lines out_channel [ json ] ) - | `Path_dir path -> - (* log precomputed blocks to individual files in the directory *) - let hash_string = State_hash.to_base58_check hash in - let height = - Mina_block.Validated.forget new_block - |> With_hash.data |> Mina_block.blockchain_length - |> Mina_numbers.Length.to_string - in - let name = - sprintf "%s-%s-%s.json" network height hash_string - in - let fpath = - Core.Filename.(parts path @ [ name ] |> of_parts) - in - Out_channel.with_file ~append:false fpath - ~f:(fun out_channel -> - Out_channel.output_lines out_channel [ json ] ) ; - [%log info] - ~metadata: - [ ("block", `String name); ("path", `String path) ] - "Logged precomputed $block to $path" ) ; - if log_precomputed_blocks then + (* original functionality, appends to single file *) + Out_channel.with_file ~append:true path + ~f:(fun out_channel -> + Out_channel.output_lines out_channel [ json ] ) ) ; + Option.iter dir ~f:(fun path -> + (* write precomputed blocks to individual files in the directory *) + let json = + Yojson.Safe.to_string (Lazy.force precomputed_block) + in + let hash_string = State_hash.to_base58_check hash in + let height = + Mina_block.Validated.forget new_block + |> With_hash.data |> Mina_block.blockchain_length + |> Mina_numbers.Length.to_string + in + let name = + sprintf "%s-%s-%s.json" network height hash_string + in + let fpath = + Core.Filename.(parts path @ [ name ] |> of_parts) + in + Out_channel.with_file ~append:false fpath + ~f:(fun out_channel -> + Out_channel.output_lines out_channel [ json ] ) ; + [%log info] + ~metadata: + [ ("block", `String name); ("path", `String path) ] + "Logged precomputed $block to $path" ) ; [%log info] "Saw block with state hash $state_hash" ~metadata: (let state_hash_data = [ ("state_hash", `String (State_hash.to_base58_check hash)) ] in - if is_some log then + if log then state_hash_data @ [ ("precomputed_block", Lazy.force precomputed_block) ] else state_hash_data ) ) ; diff --git a/src/lib/mina_lib/mina_lib.ml b/src/lib/mina_lib/mina_lib.ml index 37af10f8667..d3637a57c54 100644 --- a/src/lib/mina_lib/mina_lib.ml +++ b/src/lib/mina_lib/mina_lib.ml @@ -10,6 +10,7 @@ module Archive_client = Archive_client module Config = Config module Conf_dir = Conf_dir module Subscriptions = Coda_subscriptions +module Precomputed_block_writer = Subscriptions.Precomputed_block_writer module Snark_worker_lib = Snark_worker module Timeout = Timeout_lib.Core_time @@ -104,8 +105,7 @@ type t = Daemon_rpcs.Types.Status.Next_producer_timing.t option ; subscriptions : Coda_subscriptions.t ; sync_status : Sync_status.t Mina_incremental.Status.Observer.t - ; precomputed_block_writer : - ([ `Path of string | `Path_dir of string ] option * [ `Log ] option) ref + ; precomputed_block_writer : Precomputed_block_writer.t ref ; block_production_status : [ `Producing | `Producing_in_ms of float | `Free ] ref ; in_memory_reverse_structured_log_messages_for_integration_test : @@ -1986,25 +1986,37 @@ let create ?wallets (config : Config.t) = Archive_client.run ~logger:config.logger ~frontier_broadcast_pipe:frontier_broadcast_pipe_r archive_process_port ) ; - (* To log precomputed blocks to individual files, set both - --precomputed-blocks-path DIR and --log-precomputed-blocks true *) + (* Local block writing *) let precomputed_block_writer = - ref - ( ( try - Option.map config.precomputed_blocks_path ~f:(fun path -> - match Core.Unix.(lstat path).st_kind with - | S_DIR -> - `Path_dir path - | S_REG -> - `Path path - | _ -> - [%log' error config.logger] - ~metadata:[ ("path", `String path) ] - "$path is not a regular Unix file or directory. \ - Local precomputed block logging disabled." ; - failwith "No precomputed block logging" ) - with _ -> None ) - , if config.log_precomputed_blocks then Some `Log else None ) + let file = + try + Option.map config.precomputed_blocks_file ~f:(fun path -> + match Core.Unix.(lstat path).st_kind with + | S_REG -> + path + | _ -> + [%log' error config.logger] + ~metadata:[ ("path", `String path) ] + "$path is not a regular Unix file. \ + Local precomputed block appending disabled." ; + failwith "No precomputed block appending" ) + with _ -> None + in + let dir = + try + Option.map config.precomputed_blocks_dir ~f:(fun path -> + match Core.Unix.(lstat path).st_kind with + | S_DIR -> + path + | _ -> + [%log' error config.logger] + ~metadata:[ ("path", `String path) ] + "$path is not a regular Unix directory. \ + Local precomputed block dumping disabled." ; + failwith "No precomputed block dumping" ) + with _ -> None + in + ref { Precomputed_block_writer.file; dir; log = config.log_precomputed_blocks } in let subscriptions = Coda_subscriptions.create ~logger:config.logger @@ -2013,7 +2025,6 @@ let create ?wallets (config : Config.t) = ~is_storing_all:config.is_archive_rocksdb ~upload_blocks_to_gcloud:config.upload_blocks_to_gcloud ~time_controller:config.time_controller ~precomputed_block_writer - ~log_precomputed_blocks:config.log_precomputed_blocks in let open Mina_incremental.Status in let transition_frontier_incr = diff --git a/src/lib/mina_lib/mina_lib.mli b/src/lib/mina_lib/mina_lib.mli index f47de5999a2..94a1e91c018 100644 --- a/src/lib/mina_lib/mina_lib.mli +++ b/src/lib/mina_lib/mina_lib.mli @@ -9,6 +9,7 @@ module Archive_client = Archive_client module Config = Config module Conf_dir = Conf_dir module Subscriptions = Coda_subscriptions +module Precomputed_block_writer = Subscriptions.Precomputed_block_writer type t From b396c19a3b62e9d5cda96656b6d4f5156684bb4b Mon Sep 17 00:00:00 2001 From: Isaac-DeFrain Date: Wed, 31 May 2023 21:13:42 +0000 Subject: [PATCH 6/6] (precomputed blocks) Reformat --- .../src/cli_entrypoint/mina_cli_entrypoint.ml | 8 +- src/lib/mina_lib/coda_subscriptions.ml | 87 +++++++++---------- src/lib/mina_lib/mina_lib.ml | 14 +-- 3 files changed, 52 insertions(+), 57 deletions(-) diff --git a/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml b/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml index 7bfa718723a..2832fa96a82 100644 --- a/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml +++ b/src/app/cli/src/cli_entrypoint/mina_cli_entrypoint.ml @@ -1308,10 +1308,10 @@ Pass one of -peer, -peer-list-file, -seed, -peer-list-url.|} ; ~consensus_local_state ~is_archive_rocksdb ~work_reassignment_wait ~archive_process_location ~log_block_creation ~precomputed_values ~start_time - ?precomputed_blocks_file ?precomputed_blocks_dir ~log_precomputed_blocks - ~upload_blocks_to_gcloud ~block_reward_threshold ~uptime_url - ~uptime_submitter_keypair ~stop_time ~node_status_url - ~node_status_type () ) + ?precomputed_blocks_file ?precomputed_blocks_dir + ~log_precomputed_blocks ~upload_blocks_to_gcloud + ~block_reward_threshold ~uptime_url ~uptime_submitter_keypair + ~stop_time ~node_status_url () ) in { Coda_initialization.coda ; client_trustlist diff --git a/src/lib/mina_lib/coda_subscriptions.ml b/src/lib/mina_lib/coda_subscriptions.ml index 05bb18b794a..baee582c339 100644 --- a/src/lib/mina_lib/coda_subscriptions.ml +++ b/src/lib/mina_lib/coda_subscriptions.ml @@ -15,11 +15,7 @@ module Optional_public_key = struct end module Precomputed_block_writer = struct - type t = - { file : string option - ; dir : string option - ; log : bool - } + type t = { file : string option; dir : string option; log : bool } [@@deriving fields] end @@ -108,7 +104,8 @@ let create ~logger ~constraint_constants ~wallets ~new_blocks ~if_not_found:ignore in let dump_precomputed_blocks = - is_some (!precomputed_block_writer : Precomputed_block_writer.t).dir in + is_some (!precomputed_block_writer : Precomputed_block_writer.t).dir + in let network = match Core.Sys.getenv "NETWORK_NAME" with | Some network -> @@ -145,14 +142,14 @@ let create ~logger ~constraint_constants ~wallets ~new_blocks if upload_blocks_to_gcloud then [%log warn] "GCLOUD_BLOCK_UPLOAD_BUCKET environment variable not set. Must be \ - set to use upload_blocks_to_gcloud" ; + set to use upload_blocks_to_gcloud" ; None in - Option.iter (!precomputed_block_writer).file ~f:(fun path -> + Option.iter !precomputed_block_writer.file ~f:(fun path -> [%log info] ~metadata:[ ("path", `String path) ] "Precomputed blocks will be logged to the same file $path" ) ; - Option.iter (!precomputed_block_writer).dir ~f:(fun path -> + Option.iter !precomputed_block_writer.dir ~f:(fun path -> [%log info] ~metadata:[ ("path", `String path) ] "Precomputed blocks will be logged to individual files in $path" ) ; @@ -169,7 +166,9 @@ let create ~logger ~constraint_constants ~wallets ~new_blocks Mina_block.Validated.forget new_block |> State_hash.With_state_hashes.state_hash in - (let Precomputed_block_writer.{ file; dir; log } = !precomputed_block_writer in + (let Precomputed_block_writer.{ file; dir; log } = + !precomputed_block_writer + in let precomputed_block = lazy (let scheduled_time = Block_time.now time_controller in @@ -187,9 +186,9 @@ let create ~logger ~constraint_constants ~wallets ~new_blocks | Some _, Some bucket -> let hash_string = State_hash.to_base58_check hash in let height = - Mina_block.Validated.forget new_block - |> With_hash.data |> Mina_block.blockchain_length - |> Mina_numbers.Length.to_string + Mina_block.Validated.forget new_block + |> With_hash.data |> Mina_block.blockchain_length + |> Mina_numbers.Length.to_string in let name = sprintf "%s-%s-%s.json" network height hash_string @@ -257,44 +256,36 @@ let create ~logger ~constraint_constants ~wallets ~new_blocks let json = Yojson.Safe.to_string (Lazy.force precomputed_block) in - (* original functionality, appends to single file *) - Out_channel.with_file ~append:true path - ~f:(fun out_channel -> - Out_channel.output_lines out_channel [ json ] ) ) ; + (* original functionality, appends to single file *) + Out_channel.with_file ~append:true path ~f:(fun out_channel -> + Out_channel.output_lines out_channel [ json ] ) ) ; Option.iter dir ~f:(fun path -> (* write precomputed blocks to individual files in the directory *) - let json = - Yojson.Safe.to_string (Lazy.force precomputed_block) - in - let hash_string = State_hash.to_base58_check hash in - let height = - Mina_block.Validated.forget new_block - |> With_hash.data |> Mina_block.blockchain_length - |> Mina_numbers.Length.to_string - in - let name = - sprintf "%s-%s-%s.json" network height hash_string - in - let fpath = - Core.Filename.(parts path @ [ name ] |> of_parts) + let json = + Yojson.Safe.to_string (Lazy.force precomputed_block) + in + let hash_string = State_hash.to_base58_check hash in + let height = + Mina_block.Validated.forget new_block + |> With_hash.data |> Mina_block.blockchain_length + |> Mina_numbers.Length.to_string + in + let name = sprintf "%s-%s-%s.json" network height hash_string in + let fpath = Core.Filename.(parts path @ [ name ] |> of_parts) in + Out_channel.with_file ~append:false fpath ~f:(fun out_channel -> + Out_channel.output_lines out_channel [ json ] ) ; + [%log info] + ~metadata:[ ("block", `String name); ("path", `String path) ] + "Logged precomputed $block to $path" ) ; + [%log info] "Saw block with state hash $state_hash" + ~metadata: + (let state_hash_data = + [ ("state_hash", `String (State_hash.to_base58_check hash)) ] in - Out_channel.with_file ~append:false fpath - ~f:(fun out_channel -> - Out_channel.output_lines out_channel [ json ] ) ; - [%log info] - ~metadata: - [ ("block", `String name); ("path", `String path) ] - "Logged precomputed $block to $path" ) ; - [%log info] "Saw block with state hash $state_hash" - ~metadata: - (let state_hash_data = - [ ("state_hash", `String (State_hash.to_base58_check hash)) - ] - in - if log then - state_hash_data - @ [ ("precomputed_block", Lazy.force precomputed_block) ] - else state_hash_data ) ) ; + if log then + state_hash_data + @ [ ("precomputed_block", Lazy.force precomputed_block) ] + else state_hash_data ) ) ; let new_block_no_hash = Mina_block.Validated.forget new_block |> With_hash.data in diff --git a/src/lib/mina_lib/mina_lib.ml b/src/lib/mina_lib/mina_lib.ml index d3637a57c54..dca076f6eb8 100644 --- a/src/lib/mina_lib/mina_lib.ml +++ b/src/lib/mina_lib/mina_lib.ml @@ -1997,8 +1997,8 @@ let create ?wallets (config : Config.t) = | _ -> [%log' error config.logger] ~metadata:[ ("path", `String path) ] - "$path is not a regular Unix file. \ - Local precomputed block appending disabled." ; + "$path is not a regular Unix file. Local precomputed \ + block appending disabled." ; failwith "No precomputed block appending" ) with _ -> None in @@ -2011,12 +2011,16 @@ let create ?wallets (config : Config.t) = | _ -> [%log' error config.logger] ~metadata:[ ("path", `String path) ] - "$path is not a regular Unix directory. \ - Local precomputed block dumping disabled." ; + "$path is not a regular Unix directory. Local \ + precomputed block dumping disabled." ; failwith "No precomputed block dumping" ) with _ -> None in - ref { Precomputed_block_writer.file; dir; log = config.log_precomputed_blocks } + ref + { Precomputed_block_writer.file + ; dir + ; log = config.log_precomputed_blocks + } in let subscriptions = Coda_subscriptions.create ~logger:config.logger