diff --git a/docs/troubleshooting/logging-reference.md b/docs/troubleshooting/logging-reference.md index 977abcbb44..84b8e96364 100644 --- a/docs/troubleshooting/logging-reference.md +++ b/docs/troubleshooting/logging-reference.md @@ -3,15 +3,18 @@ title: Logging Reference slug: /troubleshooting/logging-reference --- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + Chia blockchain nodes consist of several components that each handle different aspects of farming, harvesting, the wallet and general management of a node. Each component creates entries in a single log file `debug.log`. ## Log file Location: -| OS | Location | -| ------- | ---------------------------------------------------------------------------------- | -| Linux | `~/.chia/mainnet/log/debug.log` | -| Windows | `%systemdrive% %homepath% \.chia\mainnet\debug.log` (`C:\Users\\.chia…`) | -| MacOS | `/Users//.chia/mainnet/log/debug.log` | +| OS | Location | +| ------- | ----------------------------------------------------------------------------- | +| Linux | `~/.chia/mainnet/log/debug.log` | +| Windows | `$env:USERPROFILE\.chia\mainnet\log\debug.log` (`C:\Users\\.chia…`) | +| MacOS | `$HOME/.chia/mainnet/log/debug.log` (`/Users//.chia…`) | ## Log File Management: @@ -73,58 +76,58 @@ Another log message you will see is `Updated Wallet peak to height x, weight y` ## Other Normal Log Messages: -| Component | Message | Direction | Destination | Cross component | Comment | -| ----------------- | ------------------------------------------------ | --------- | ----------- | -------------------- | ------- | -| mempool_manager | add_spendbundle took x seconds | | | | | -| mempool_manager | It took x to pre validate transaction | | | | | -| full_node | Added unfinished_block x, not farmed by us | | | | | -| full_node | Already compactified block: | | | | | +| Component | Message | Direction | Destination | Cross component | +| ----------------- | ------------------------------------------------ | --------- | ----------- | -------------------- | +| mempool_manager | add_spendbundle took x seconds | | | | +| mempool_manager | It took x to pre validate transaction | | | | +| full_node | Added unfinished_block x, not farmed by us | | | | +| full_node | Already compactified block: | | | | | full_node | Duplicate compact proof. Height: x | | | | | full_node | Finished signage point x/64: | | | | | full_node | Scanning the blockchain for uncompact blocks. | | | | -| full_node | Updated peak to height x | | | | | -| full_node_server | new_compact_vdf | to/from | peer | | | -| full_node_server | new_peak | to/from | peer | | | -| full_node_server | new_peak_timelord | to | localhost | from timelord_server | | -| full_node_server | new_peak_wallet | to | localhost | from wallet_server | | -| full_node_server | new_signage_point | to | localhost | from farmer_server | | -| full_node_server | new_signage_point_or_end_of_sub_slot | to/from | peer | | | -| full_node_server | new_transaction | to/from | peer | | | -| full_node_server | new_unfinished_block | to/from | peer | | | -| full_node_server | new_unfinished_block_timelord | to/from | localhost | | | -| full_node_server | request_block | to/from | peer | | | -| full_node_server | request_block_header | from | localhost | to wallet_server | | -| full_node_server | request_compact_vdf | to/from | peer | | | -| full_node_server | request_compact_proof_of_time | to | localhost | from timelord_server | | -| full_node_server | request_signage_point_or_end_of_sub_slot | to/from | peer | | | -| full_node_server | request_transaction | to/from | peer | | | -| full_node_server | request_unfinished_block | to/from | peer | | | -| full_node_server | respond_block | to/from | peer | | | -| full_node_server | respond_compact_vdf | to/from | peer | | | -| full_node_server | respond_signage_point | to/from | peer | | | -| full_node_server | respond_transaction | to/from | peer | | | -| full_node_server | respond_unfinished_block | to/from | peer | | | -| wallet_server | request_block_header | to | localhost | from full_node | | -| wallet_server | respond_block_header | from | localhost | to full_node | | -| wallet_server | new_peak_wallet | from | localhost | to full_node | | +| full_node | Updated peak to height x | | | | +| full_node_server | new_compact_vdf | to/from | peer | | +| full_node_server | new_peak | to/from | peer | | +| full_node_server | new_peak_timelord | to | localhost | from timelord_server | +| full_node_server | new_peak_wallet | to | localhost | from wallet_server | +| full_node_server | new_signage_point | to | localhost | from farmer_server | +| full_node_server | new_signage_point_or_end_of_sub_slot | to/from | peer | | +| full_node_server | new_transaction | to/from | peer | | +| full_node_server | new_unfinished_block | to/from | peer | | +| full_node_server | new_unfinished_block_timelord | to/from | localhost | | +| full_node_server | request_block | to/from | peer | | +| full_node_server | request_block_header | from | localhost | to wallet_server | +| full_node_server | request_compact_vdf | to/from | peer | | +| full_node_server | request_compact_proof_of_time | to | localhost | from timelord_server | +| full_node_server | request_signage_point_or_end_of_sub_slot | to/from | peer | | +| full_node_server | request_transaction | to/from | peer | | +| full_node_server | request_unfinished_block | to/from | peer | | +| full_node_server | respond_block | to/from | peer | | +| full_node_server | respond_compact_vdf | to/from | peer | | +| full_node_server | respond_signage_point | to/from | peer | | +| full_node_server | respond_transaction | to/from | peer | | +| full_node_server | respond_unfinished_block | to/from | peer | | +| wallet_server | request_block_header | to | localhost | from full_node | +| wallet_server | respond_block_header | from | localhost | to full_node | +| wallet_server | new_peak_wallet | from | localhost | to full_node | | wallet_blockchain | Updated Wallet peak to height x, weight y | | | | -| timelord_server | new_peak_timelord | from | localhost | to full_node | | -| timelord_server | new_unfinished_block_timelord | from | localhost | to full_node | | +| timelord_server | new_peak_timelord | from | localhost | to full_node | +| timelord_server | new_unfinished_block_timelord | from | localhost | to full_node | | timelord_launcher | VDF client x: VDF Client: Discriminant = | | | | | VDF Client | Sending Proof, Sent Proof, Stopped everything! | | | | -| harvester_server | farming_info | to/from | localhost | | | -| harvester_server | new_signage_point_harvester | from | localhost | to farmer_server | | +| harvester_server | farming_info | to/from | localhost | | +| harvester_server | new_signage_point_harvester | from | localhost | to farmer_server | | harvester | x plots were eligible for farming | | | | | plot_tools | Loaded a total of x plots of size y in z seconds | | | | | plot_tools | Searching directories | | | | -| farmer_server | new_signage_point | from | localhost | to full_node | | -| farmer_server | farming_info | from | localhost | to full_node | | -| farmer_server | new_signage_point_harvester | to | localhost | from harvester | | +| farmer_server | new_signage_point | from | localhost | to full_node | +| farmer_server | farming_info | from | localhost | to full_node | +| farmer_server | new_signage_point_harvester | to | localhost | from harvester | | Source | Level | Message | Description | | ----------------------------------- | ------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| daemon asyncio | ERROR | Task exception was never retrieved future: ` exception=ValueError('list.remove(x): x not in list')>` | -| full_node asyncio | ERROR | SSL error in data received protocol: `` | +| daemon asyncio | ERROR | Task exception was never retrieved future: ` exception=ValueError('list.remove(x): x not in list')>` | | +| full_node asyncio | ERROR | SSL error in data received protocol: `` | | | full_node full_node_server | ERROR | Exception: Failed to fetch block `N` from \{'host': `IP ADDRESS`, 'port': 8444\}, timed out, \{'host': `IP ADDRESS`, 'port': 8444\}. | Peer disconnected, other peer connections will take over | | full_node full_node_server | ERROR | Exception: ``, closing connection None. | Peer disconnected | | full_node full_node_server | WARNING | [Errno 32] Broken pipe `IP Address` | Peer disconnected | @@ -134,9 +137,119 @@ Another log message you will see is `Updated Wallet peak to height x, weight y` | harvester src.plotting.plot_tools | WARNING | Have multiple copies of the plot `plotfilename`, not adding it. | | | harvester src.plotting.plot_tools | INFO | Not checking subdirectory `Dir1/directory`, subdirectories not added by default | | | full_node full_node_server | INFO | Connection closed: `IP Address`, node id: `hex` | Peer disconnected | -| full_node src.full_node.full_node | INFO | ⏲️ Finished signage point `N`/64: `hex` | -| full_node src.full_node.full_node | INFO | Added unfinished_block `hex`, not farmed | +| full_node src.full_node.full_node | INFO | ⏲️ Finished signage point `N`/64: `hex` | | +| full_node src.full_node.full_node | INFO | Added unfinished_block `hex`, not farmed | | | harvester src.plotting.plot_tools | INFO | Searching directories [`Dir1`,`Dir2`] | | | harvester src.plotting.plot_tools | INFO | Loaded a total of `N` plots of size `size` TiB, in `time` seconds | | | harvester src.harvester.harvester | INFO | `X` plots were eligible for farming `hex`... Found `Y` proofs. Time: `Time` s. Total `Z` plots | This is a vital message and should be seen at regular intervals. Note that `Time` is ideally < 1s. If drive is in sleep mode, may show ~10 seconds, and should be prevented | | wallet src.wallet.wallet_blockchain | INFO | 💰 Updated wallet peak to height `HEIGHT`, weight `WEIGHT`, | | + +## Log Search Commands + + + + +### search linux or mac logs + +:::info +The below strings are an example of what can be used to search for specific log lines but they are not all-inclusive. +You can also use strings from any other log lines in the below command to search for other lines, just make sure you have the correct log level setting for the line that is being searched. +::: + +Functionality: Output log lines that contain the string in terminal + +Usage: `find "$HOME/.chia/mainnet/log" -type f -exec grep -Hn "[STRING]" {} \;` + +Options: + +| String | Minimum Required Log Level | Description | +| --------------- | -------------------------- | ---------------------------------------------------- | +| `ing service` | INFO | Outputs all starting and stopping services log lines | +| `: ERROR` | ERROR | Outputs all error log lines | +| `: WARNING` | WARNING | Outputs all warning log lines | +| `proofs. Time:` | INFO | Outputs all proof check log lines | + +
+Example + +Request: + +```bash +find "$HOME/.chia/mainnet/log" -type f -exec grep -Hn "ing service" {} \; +``` + +Response: + +:::info +The below output contains the `[file_path]:[log_line]:[timestamp] [service] [service] : [log_level] [log_information]`. + +The chia client automatically rotates log files so pay close attention to the file name at the end of the file path if you want to open the file and look for the logs. +::: + +``` +/Users/[username]/.chia/mainnet/log/debug.log:41432:2024-07-26T10:03:21.794 wallet wallet : INFO Starting service wallet ... +/Users/[username]/.chia/mainnet/log/debug.log:41437:2024-07-26T10:06:21.177 wallet wallet : INFO Stopping service wallet at port None ... +``` + +
+ +--- + + + + +### search windows logs + +:::info +The below strings are an example of what can be used to search for specific log lines but they are not all-inclusive. +You can also use strings from any other log lines in the below command to search for other lines, just make sure you have the correct log level setting for the line that is being searched. +::: + +Functionality: Output log lines that contain the string in powershell + +Usage: `Get-ChildItem -Path $env:USERPROFILE\.chia\mainnet\log\ -recurse | Select-String -Pattern "[STRING]"` + +Options: + +| String | Minimum Required Log Level | Description | +| --------------- | -------------------------- | ---------------------------------------------------- | +| `ing service` | INFO | Outputs all starting and stopping services log lines | +| `: ERROR` | ERROR | Outputs all error log lines | +| `: WARNING` | WARNING | Outputs all warning log lines | +| `proofs. Time:` | INFO | Outputs all proof check log lines | + +
+Example + +Request: + +```bash +Get-ChildItem -Path $env:USERPROFILE\.chia\mainnet\log\ -recurse | Select-String -Pattern "ing service" +``` + +Response: + +:::info +The below output contains the `[file_path]:[log_line]:[timestamp] [service] [service] : [log_level] [log_information]`. + +The chia client automatically rotates log files so pay close attention to the file name at the end of the file path if you want to open the file and look for the logs. +::: + +``` +C:\Users\[USERNAME]\.chia\mainnet +.chia\mainnet\log\debug.log:21:2024-07-26T10:03:21.794 wallet wallet : INFO Starting service wallet ... +.chia\mainnet\log\debug.log:10042:2024-07-26T10:06:21.177 wallet wallet : INFO Stopping service wallet at port None ... +``` + +
+ +--- + + +