From 21ac43aaa8e9c3b837438d92226bbd895e23c696 Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Tue, 12 Dec 2023 17:04:30 +1100 Subject: [PATCH 01/13] implement a logging proxy The logging proxy: - prevents the need to import chronicles (as well as export except toJson), - prevents the need to override `writeValue` or use or import nim-json-seralization elsewhere in the codebase, allowing for sole use of utils/json for de/serialization, - and handles json formatting correctly in chronicles json sinks --- codex.nim | 4 +- codex/blockexchange/engine/discovery.nim | 7 +- codex/blockexchange/engine/engine.nim | 2 +- codex/blockexchange/engine/pendingblocks.nim | 2 +- codex/blockexchange/network/network.nim | 2 +- codex/blockexchange/network/networkpeer.nim | 2 +- codex/blockexchange/peers/peercontext.nim | 2 +- codex/blockexchange/peers/peerctxstore.nim | 3 +- codex/blocktype.nim | 16 +- codex/chunker.nim | 2 +- codex/codex.nim | 2 +- codex/conf.nim | 6 +- codex/contracts/deployment.nim | 2 +- .../interactions/clientinteractions.nim | 4 +- .../interactions/hostinteractions.nim | 6 +- codex/contracts/market.nim | 2 +- codex/contracts/requests.nim | 20 +- codex/discovery.nim | 3 +- codex/erasure/erasure.nim | 2 +- codex/formats.nim | 28 --- codex/logging.nim | 89 ++++++++ codex/manifest/coders.nim | 2 +- codex/manifest/manifest.nim | 2 +- codex/merkletree/merkletree.nim | 2 +- codex/node.nim | 4 +- codex/purchasing/purchaseid.nim | 3 + codex/purchasing/states/cancelled.nim | 6 +- codex/purchasing/states/error.nim | 2 +- codex/purchasing/states/finished.nim | 3 +- codex/purchasing/states/started.nim | 3 +- codex/purchasing/states/submitted.nim | 3 +- codex/rest/api.nim | 2 +- codex/sales.nim | 4 +- codex/sales/reservations.nim | 12 +- codex/sales/salesagent.nim | 2 +- codex/sales/slotqueue.nim | 2 +- codex/sales/states/cancelled.nim | 6 +- codex/sales/states/downloading.nim | 3 +- codex/sales/states/errored.nim | 3 +- codex/sales/states/failed.nim | 4 +- codex/sales/states/filled.nim | 5 +- codex/sales/states/filling.nim | 4 +- codex/sales/states/finished.nim | 5 +- codex/sales/states/ignored.nim | 3 +- codex/sales/states/initialproving.nim | 6 +- codex/sales/states/payout.nim | 4 +- codex/sales/states/preparing.nim | 3 +- codex/sales/states/proving.nim | 13 +- codex/sales/states/provingsimulated.nim | 4 +- codex/sales/states/unknown.nim | 2 +- codex/stores/cachestore.nim | 2 +- codex/stores/maintenance.nim | 2 +- codex/stores/networkstore.nim | 13 +- codex/stores/repostore.nim | 2 +- codex/streams/asyncstreamwrapper.nim | 3 +- codex/streams/seekablestream.nim | 5 +- codex/streams/storestream.nim | 2 +- codex/units.nim | 17 +- codex/utils/asyncstatemachine.nim | 4 +- codex/utils/fileutils.nim | 9 +- codex/utils/json.nim | 2 +- codex/utils/keyutils.nim | 2 +- codex/utils/timer.nim | 3 +- codex/utils/trackedfutures.nim | 3 +- codex/validation.nim | 8 +- tests/codex/sales/testslotqueue.nim | 2 +- tests/codex/testchunking.nim | 2 +- tests/codex/testlogging.nim | 192 ++++++++++++++++++ tests/codex/testnode.nim | 2 +- tests/codex/utils/testjson.nim | 2 +- tests/integration/codexclient.nim | 2 +- tests/integration/multinodes.nim | 2 +- tests/integration/nodes.nim | 10 +- tests/integration/testIntegration.nim | 2 +- tests/integration/testproofs.nim | 6 +- tests/integration/twonodes.nim | 2 + tests/logging.nim | 2 +- tests/testCodex.nim | 2 + 78 files changed, 437 insertions(+), 191 deletions(-) delete mode 100644 codex/formats.nim create mode 100644 codex/logging.nim create mode 100644 tests/codex/testlogging.nim diff --git a/codex.nim b/codex.nim index 073331f14..9c110e581 100644 --- a/codex.nim +++ b/codex.nim @@ -7,7 +7,6 @@ ## This file may not be copied, modified, or distributed except according to ## those terms. -import pkg/chronicles import pkg/chronos import pkg/questionable import pkg/confutils @@ -21,10 +20,11 @@ import pkg/libp2p import ./codex/conf import ./codex/codex +import ./codex/logging import ./codex/units import ./codex/utils/keyutils -export codex, conf, libp2p, chronos, chronicles +export codex, conf, libp2p, chronos, logging when isMainModule: import std/sequtils diff --git a/codex/blockexchange/engine/discovery.nim b/codex/blockexchange/engine/discovery.nim index 8bb4dd4a6..9490d6a8c 100644 --- a/codex/blockexchange/engine/discovery.nim +++ b/codex/blockexchange/engine/discovery.nim @@ -10,22 +10,21 @@ import std/sequtils import pkg/chronos -import pkg/chronicles import pkg/libp2p/cid import pkg/metrics import pkg/questionable import pkg/questionable/results -import ../protobuf/presence +import ./pendingblocks +import ../protobuf/presence import ../network import ../peers import ../../utils import ../../discovery import ../../stores/blockstore - -import ./pendingblocks +import ../../logging logScope: topics = "codex discoveryengine" diff --git a/codex/blockexchange/engine/engine.nim b/codex/blockexchange/engine/engine.nim index ddff3c490..4386f65a1 100644 --- a/codex/blockexchange/engine/engine.nim +++ b/codex/blockexchange/engine/engine.nim @@ -14,7 +14,6 @@ import std/algorithm import std/sugar import pkg/chronos -import pkg/chronicles import pkg/libp2p/[cid, switch, multihash, multicodec] import pkg/metrics import pkg/stint @@ -23,6 +22,7 @@ import ../../stores/blockstore import ../../blocktype import ../../utils import ../../merkletree +import ../../logging import ../protobuf/blockexc import ../protobuf/presence diff --git a/codex/blockexchange/engine/pendingblocks.nim b/codex/blockexchange/engine/pendingblocks.nim index c47639982..4d149f057 100644 --- a/codex/blockexchange/engine/pendingblocks.nim +++ b/codex/blockexchange/engine/pendingblocks.nim @@ -14,7 +14,6 @@ import pkg/upraises push: {.upraises: [].} -import pkg/chronicles import pkg/chronos import pkg/libp2p import pkg/metrics @@ -23,6 +22,7 @@ import pkg/questionable/results import ../protobuf/blockexc import ../../blocktype import ../../merkletree +import ../../logging logScope: topics = "codex pendingblocks" diff --git a/codex/blockexchange/network/network.nim b/codex/blockexchange/network/network.nim index 0ce0c9ce0..287b6f065 100644 --- a/codex/blockexchange/network/network.nim +++ b/codex/blockexchange/network/network.nim @@ -10,7 +10,6 @@ import std/tables import std/sequtils -import pkg/chronicles import pkg/chronos import pkg/libp2p @@ -19,6 +18,7 @@ import pkg/questionable import pkg/questionable/results import ../../blocktype as bt +import ../../logging import ../protobuf/blockexc as pb import ../protobuf/payments diff --git a/codex/blockexchange/network/networkpeer.nim b/codex/blockexchange/network/networkpeer.nim index c3536f617..66bfa96e0 100644 --- a/codex/blockexchange/network/networkpeer.nim +++ b/codex/blockexchange/network/networkpeer.nim @@ -11,12 +11,12 @@ import pkg/upraises push: {.upraises: [].} import pkg/chronos -import pkg/chronicles import pkg/libp2p import ../protobuf/blockexc import ../protobuf/message import ../../errors +import ../../logging logScope: topics = "codex blockexcnetworkpeer" diff --git a/codex/blockexchange/peers/peercontext.nim b/codex/blockexchange/peers/peercontext.nim index 66418ddda..40b330016 100644 --- a/codex/blockexchange/peers/peercontext.nim +++ b/codex/blockexchange/peers/peercontext.nim @@ -12,7 +12,6 @@ import std/tables import std/sugar import std/sets -import pkg/chronicles import pkg/libp2p import pkg/chronos import pkg/nitro @@ -23,6 +22,7 @@ import ../protobuf/payments import ../protobuf/presence import ../../blocktype +import ../../logging export payments, nitro diff --git a/codex/blockexchange/peers/peerctxstore.nim b/codex/blockexchange/peers/peerctxstore.nim index f23415f62..1182a0b6d 100644 --- a/codex/blockexchange/peers/peerctxstore.nim +++ b/codex/blockexchange/peers/peerctxstore.nim @@ -16,11 +16,12 @@ import pkg/upraises push: {.upraises: [].} import pkg/chronos -import pkg/chronicles import pkg/libp2p import ../protobuf/blockexc import ../../blocktype +import ../../logging + import ./peercontext export peercontext diff --git a/codex/blocktype.nim b/codex/blocktype.nim index a26a31571..7046c5b0d 100644 --- a/codex/blocktype.nim +++ b/codex/blocktype.nim @@ -19,15 +19,14 @@ import pkg/libp2p/[cid, multicodec, multihash] import pkg/stew/byteutils import pkg/questionable import pkg/questionable/results -import pkg/chronicles import pkg/json_serialization import ./units import ./utils -import ./formats import ./errors +import ./logging -export errors, formats, units +export errors, logging, units const # Size of blocks for storage / network exchange, @@ -47,6 +46,11 @@ type else: cid*: Cid +logging.formatIt(BlockAddress): + if it.leaf: + "treeCid: " & shortLog($it.treeCid) & ", index: " & $it.index + else: + "cid: " & shortLog($it.cid) proc `==`*(a, b: BlockAddress): bool = a.leaf == b.leaf and @@ -63,12 +67,6 @@ proc `$`*(a: BlockAddress): string = else: "cid: " & $a.cid -proc writeValue*( - writer: var JsonWriter, - value: Cid -) {.upraises:[IOError].} = - writer.writeValue($value) - proc cidOrTreeCid*(a: BlockAddress): Cid = if a.leaf: a.treeCid diff --git a/codex/chunker.nim b/codex/chunker.nim index 58fcb4a3a..9b3660565 100644 --- a/codex/chunker.nim +++ b/codex/chunker.nim @@ -13,13 +13,13 @@ import pkg/upraises push: {.upraises: [].} -import pkg/chronicles import pkg/questionable import pkg/questionable/results import pkg/chronos import pkg/libp2p except shuffle import ./blocktype +import ./logging export blocktype diff --git a/codex/codex.nim b/codex/codex.nim index 2f5cf8603..dedfd219f 100644 --- a/codex/codex.nim +++ b/codex/codex.nim @@ -12,7 +12,6 @@ import std/strutils import std/os import std/tables -import pkg/chronicles import pkg/chronos import pkg/presto import pkg/libp2p @@ -39,6 +38,7 @@ import ./contracts/clock import ./contracts/deployment import ./utils/addrutils import ./namespaces +import ./logging logScope: topics = "codex node" diff --git a/codex/conf.nim b/codex/conf.nim index 4a0e70f47..003d3a2c1 100644 --- a/codex/conf.nim +++ b/codex/conf.nim @@ -18,7 +18,6 @@ import std/strutils import std/typetraits import pkg/chronos -import pkg/chronicles import pkg/chronicles/helpers import pkg/chronicles/topics_registry import pkg/confutils/defs @@ -35,6 +34,7 @@ import pkg/questionable import pkg/questionable/results import ./discovery +import ./logging import ./stores import ./units import ./utils @@ -276,6 +276,8 @@ type EthAddress* = ethers.Address +chronicles.formatIt(EthAddress): it.short0xHexLog + proc getCodexVersion(): string = let tag = strip(staticExec("git tag")) if tag.isEmptyOrWhitespace: @@ -407,7 +409,7 @@ proc completeCmdArg*(T: type Duration; val: string): seq[string] = discard # silly chronicles, colors is a compile-time property -proc stripAnsi(v: string): string = +proc stripAnsi*(v: string): string = var res = newStringOfCap(v.len) i: int diff --git a/codex/contracts/deployment.nim b/codex/contracts/deployment.nim index 9e5b9388c..c517b9c11 100644 --- a/codex/contracts/deployment.nim +++ b/codex/contracts/deployment.nim @@ -3,9 +3,9 @@ import std/os import std/tables import pkg/ethers import pkg/questionable -import pkg/chronicles import ../conf +import ../logging import ./marketplace type Deployment* = ref object diff --git a/codex/contracts/interactions/clientinteractions.nim b/codex/contracts/interactions/clientinteractions.nim index e2354d854..71a91e948 100644 --- a/codex/contracts/interactions/clientinteractions.nim +++ b/codex/contracts/interactions/clientinteractions.nim @@ -1,13 +1,13 @@ import pkg/ethers -import pkg/chronicles import ../../purchasing +import ../../logging import ../market import ../clock import ./interactions export purchasing -export chronicles except toJson +export logging type ClientInteractions* = ref object of ContractInteractions diff --git a/codex/contracts/interactions/hostinteractions.nim b/codex/contracts/interactions/hostinteractions.nim index e9749df5c..96bf8de22 100644 --- a/codex/contracts/interactions/hostinteractions.nim +++ b/codex/contracts/interactions/hostinteractions.nim @@ -1,11 +1,11 @@ -import pkg/ethers -import pkg/chronicles +import pkg/chronos +import ../../logging import ../../sales import ./interactions export sales -export chronicles except toJson +export logging type HostInteractions* = ref object of ContractInteractions diff --git a/codex/contracts/market.nim b/codex/contracts/market.nim index 538b44ac3..50acb497e 100644 --- a/codex/contracts/market.nim +++ b/codex/contracts/market.nim @@ -1,11 +1,11 @@ import std/sequtils import std/strutils import std/sugar -import pkg/chronicles import pkg/ethers import pkg/ethers/testing import pkg/upraises import pkg/questionable +import ../logging import ../market import ./marketplace diff --git a/codex/contracts/requests.nim b/codex/contracts/requests.nim index 60c3c5770..a4ac83d5c 100644 --- a/codex/contracts/requests.nim +++ b/codex/contracts/requests.nim @@ -1,4 +1,5 @@ import std/hashes +import std/sequtils import std/typetraits import pkg/contractabi import pkg/nimcrypto @@ -7,6 +8,7 @@ import pkg/questionable/results import pkg/stew/byteutils import pkg/json_serialization import pkg/upraises +import ../logging import ../utils/json export contractabi @@ -79,6 +81,10 @@ proc toHex*[T: distinct](id: T): string = type baseType = T.distinctBase baseType(id).toHex +logging.formatIt(Nonce): it.short0xHexLog +logging.formatIt(RequestId): it.short0xHexLog +logging.formatIt(SlotId): it.short0xHexLog + func fromTuple(_: type StorageRequest, tupl: tuple): StorageRequest = StorageRequest( client: tupl[0], @@ -176,17 +182,3 @@ func price*(request: StorageRequest): UInt256 = func size*(ask: StorageAsk): UInt256 = ask.slots.u256 * ask.slotSize - -proc writeValue*( - writer: var JsonWriter, - value: SlotId | RequestId) {.upraises:[IOError].} = - - mixin writeValue - writer.writeValue value.toArray - -proc readValue*[T: SlotId | RequestId]( - reader: var JsonReader, - value: var T) {.upraises: [SerializationError, IOError].} = - - mixin readValue - value = T reader.readValue(T.distinctBase) diff --git a/codex/discovery.nim b/codex/discovery.nim index 67aacd173..4a00bcbeb 100644 --- a/codex/discovery.nim +++ b/codex/discovery.nim @@ -11,7 +11,6 @@ import std/algorithm import std/sequtils import pkg/chronos -import pkg/chronicles import pkg/libp2p/[cid, multicodec, routing_record, signed_envelope] import pkg/questionable import pkg/questionable/results @@ -21,7 +20,7 @@ import pkg/codexdht/discv5/protocol as discv5 import ./rng import ./errors -import ./formats +import ./logging export discv5 diff --git a/codex/erasure/erasure.nim b/codex/erasure/erasure.nim index 1bab089a8..31ad5bd0a 100644 --- a/codex/erasure/erasure.nim +++ b/codex/erasure/erasure.nim @@ -15,10 +15,10 @@ import std/sequtils import std/sugar import pkg/chronos -import pkg/chronicles import pkg/libp2p/[multicodec, cid, multibase, multihash] import pkg/libp2p/protobuf/minprotobuf +import ../logging import ../manifest import ../merkletree import ../stores diff --git a/codex/formats.nim b/codex/formats.nim deleted file mode 100644 index 38881bc9d..000000000 --- a/codex/formats.nim +++ /dev/null @@ -1,28 +0,0 @@ -## Nim-Codex -## Copyright (c) 2022 Status Research & Development GmbH -## Licensed under either of -## * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) -## * MIT license ([LICENSE-MIT](LICENSE-MIT)) -## at your option. -## This file may not be copied, modified, or distributed except according to -## those terms. - -import std/strutils - -import pkg/chronicles -import pkg/libp2p/cid - -func shortLog*(cid: Cid): string = - ## Returns compact string representation of ``pid``. - var scid = $cid - if len(scid) > 10: - scid[3] = '*' - - when (NimMajor, NimMinor) > (1, 4): - scid.delete(4 .. scid.high - 6) - else: - scid.delete(4, scid.high - 6) - - scid - -chronicles.formatIt(Cid): shortLog(it) diff --git a/codex/logging.nim b/codex/logging.nim new file mode 100644 index 000000000..d30682273 --- /dev/null +++ b/codex/logging.nim @@ -0,0 +1,89 @@ +import std/options +import std/sequtils +import std/strutils +import std/sugar +import std/typetraits + +import pkg/chronicles +import pkg/faststreams +from pkg/libp2p import Cid, MultiAddress, `$` +import pkg/questionable +import pkg/stew/byteutils +import pkg/stint +import pkg/upraises + +export byteutils +export chronicles except toJson, formatIt +export questionable +export sequtils +export strutils +export sugar +export upraises + +func shortLog*(long: string, ellipses = "*", start = 3, stop = 6): string = + ## Returns compact string representation of ``long``. + var short = long + let minLen = start + ellipses.len + stop + if len(short) > minLen: + short.insert(ellipses, start) + + when (NimMajor, NimMinor) > (1, 4): + short.delete(start + ellipses.len .. short.high - stop) + else: + short.delete(start + ellipses.len, short.high - stop) + + short + +func shortHexLog*(long: string): string = + if long[0..1] == "0x": result &= "0x" + result &= long[2..long.high].shortLog("..", 4, 4) + +func short0xHexLog*[N: static[int], T: array[N, byte]](v: T): string = + v.to0xHex.shortHexLog + +func short0xHexLog*[T: distinct](v: T): string = + type BaseType = T.distinctBase + BaseType(v).short0xHexLog + +func short0xHexLog*[U: distinct, T: seq[U]](v: T): string = + type BaseType = U.distinctBase + "@[" & v.map(x => BaseType(x).short0xHexLog).join(",") & "]" + +proc formatSeq*(val: seq[string]): string = + "@[" & val.join(", ") & "]" + +template formatOption*(val, T, body): auto = + var v = "None(" & $T & ")" + if it =? val: + v = "Some(" & body & ")" # that I used to know :) + v + +template formatIt*(T: type, body: untyped) {.dirty.} = + chronicles.formatIt(T, body) + + proc writeValue*(writer: var JsonWriter, it: T) {.upraises:[IOError].} = + let formatted = body + writer.writeValue(formatted) + + proc setProperty*(r: var JsonRecord, key: string, it: T) = + let v = body + setProperty(r, key, v) + + proc setProperty*(r: var TextLineRecord, key: string, val: ?T) = + setProperty(r, key, val.formatOption(T, body)) + + proc setProperty*(r: var TextLineRecord, key: string, val: seq[?T]) = + let v = val.map(item => item.formatOption(T, body)).formatSeq + setProperty(r, key, v) + + proc setProperty*(r: var TextLineRecord, key: string, val: seq[T]) = + let v = val.map(it => body).formatSeq + setProperty(r, key, v) + + proc setProperty*(r: var TextLineRecord, key: string, it: T) = + let v = body + setProperty(r, key, v) + +formatIt(Cid): shortLog($it) +formatIt(UInt256): $it +formatIt(MultiAddress): $it diff --git a/codex/manifest/coders.nim b/codex/manifest/coders.nim index 7bf173f9d..35c52914d 100644 --- a/codex/manifest/coders.nim +++ b/codex/manifest/coders.nim @@ -19,12 +19,12 @@ import std/sequtils import pkg/libp2p import pkg/questionable import pkg/questionable/results -import pkg/chronicles import pkg/chronos import ./manifest import ../errors import ../blocktype +import ../logging import ./types proc encode*(_: DagPBCoder, manifest: Manifest): ?!seq[byte] = diff --git a/codex/manifest/manifest.nim b/codex/manifest/manifest.nim index 8f8df55c2..4e50d372e 100644 --- a/codex/manifest/manifest.nim +++ b/codex/manifest/manifest.nim @@ -17,13 +17,13 @@ import pkg/libp2p/protobuf/minprotobuf import pkg/libp2p import pkg/questionable import pkg/questionable/results -import pkg/chronicles import ../errors import ../utils import ../utils/json import ../units import ../blocktype +import ../logging import ./types export types diff --git a/codex/merkletree/merkletree.nim b/codex/merkletree/merkletree.nim index 74ddba6d8..b6840589b 100644 --- a/codex/merkletree/merkletree.nim +++ b/codex/merkletree/merkletree.nim @@ -13,7 +13,6 @@ import std/sequtils import std/sugar import std/algorithm -import pkg/chronicles import pkg/questionable import pkg/questionable/results import pkg/nimcrypto/sha2 @@ -21,6 +20,7 @@ import pkg/libp2p/[cid, multicodec, multihash, vbuffer] import pkg/stew/byteutils import ../errors +import ../logging logScope: topics = "codex merkletree" diff --git a/codex/node.nim b/codex/node.nim index 60606c36f..f21f3a37d 100644 --- a/codex/node.nim +++ b/codex/node.nim @@ -15,7 +15,6 @@ import std/sugar import pkg/questionable import pkg/questionable/results -import pkg/chronicles import pkg/chronos import pkg/libp2p/[switch, multicodec, multihash] @@ -39,6 +38,7 @@ import ./contracts import ./node/batch import ./utils import ./errors +import ./logging export batch @@ -483,7 +483,7 @@ proc start*(node: CodexNodeRef) {.async.} = node.contracts.validator = ValidatorInteractions.none node.networkId = node.switch.peerInfo.peerId - notice "Started codex node", id = $node.networkId, addrs = node.switch.peerInfo.addrs + notice "Started codex node", id = node.networkId, addrs = node.switch.peerInfo.addrs proc stop*(node: CodexNodeRef) {.async.} = trace "Stopping node" diff --git a/codex/purchasing/purchaseid.nim b/codex/purchasing/purchaseid.nim index ee5c3a168..0aff18f7a 100644 --- a/codex/purchasing/purchaseid.nim +++ b/codex/purchasing/purchaseid.nim @@ -1,8 +1,11 @@ import std/hashes import pkg/nimcrypto +import ../logging type PurchaseId* = distinct array[32, byte] +chronicles.formatIt(PurchaseId): it.short0xHexLog + proc hash*(x: PurchaseId): Hash {.borrow.} proc `==`*(x, y: PurchaseId): bool {.borrow.} proc toHex*(x: PurchaseId): string = array[32, byte](x).toHex diff --git a/codex/purchasing/states/cancelled.nim b/codex/purchasing/states/cancelled.nim index a4765276f..9e3aaf2c3 100644 --- a/codex/purchasing/states/cancelled.nim +++ b/codex/purchasing/states/cancelled.nim @@ -1,8 +1,8 @@ import pkg/metrics -import pkg/chronicles + +import ../../logging import ../statemachine import ./errorhandling -import ./error declareCounter(codex_purchases_cancelled, "codex purchases cancelled") @@ -18,7 +18,7 @@ method run*(state: PurchaseCancelled, machine: Machine): Future[?State] {.async. codex_purchases_cancelled.inc() let purchase = Purchase(machine) - warn "Request cancelled, withdrawing remaining funds", requestId = $purchase.requestId + warn "Request cancelled, withdrawing remaining funds", requestId = purchase.requestId await purchase.market.withdrawFunds(purchase.requestId) let error = newException(Timeout, "Purchase cancelled due to timeout") diff --git a/codex/purchasing/states/error.nim b/codex/purchasing/states/error.nim index baa0cceb2..73b1848b7 100644 --- a/codex/purchasing/states/error.nim +++ b/codex/purchasing/states/error.nim @@ -1,7 +1,7 @@ import pkg/metrics -import pkg/chronicles import ../statemachine import ../../utils/exceptions +import ../../logging declareCounter(codex_purchases_error, "codex purchases error") diff --git a/codex/purchasing/states/finished.nim b/codex/purchasing/states/finished.nim index 4a2662cf0..cdf43ebc9 100644 --- a/codex/purchasing/states/finished.nim +++ b/codex/purchasing/states/finished.nim @@ -1,6 +1,7 @@ import pkg/metrics -import pkg/chronicles + import ../statemachine +import ../../logging declareCounter(codex_purchases_finished, "codex purchases finished") diff --git a/codex/purchasing/states/started.nim b/codex/purchasing/states/started.nim index 04736a134..0ea660766 100644 --- a/codex/purchasing/states/started.nim +++ b/codex/purchasing/states/started.nim @@ -1,5 +1,6 @@ import pkg/metrics -import pkg/chronicles + +import ../../logging import ../statemachine import ./errorhandling import ./finished diff --git a/codex/purchasing/states/submitted.nim b/codex/purchasing/states/submitted.nim index f3a68512d..b8617136c 100644 --- a/codex/purchasing/states/submitted.nim +++ b/codex/purchasing/states/submitted.nim @@ -1,5 +1,6 @@ import pkg/metrics -import pkg/chronicles + +import ../../logging import ../statemachine import ./errorhandling import ./started diff --git a/codex/rest/api.nim b/codex/rest/api.nim index 2f1b192cd..787762f65 100644 --- a/codex/rest/api.nim +++ b/codex/rest/api.nim @@ -16,7 +16,6 @@ import std/sequtils import pkg/questionable import pkg/questionable/results -import pkg/chronicles except toJson import pkg/chronos import pkg/presto except toJson import pkg/metrics except toJson @@ -28,6 +27,7 @@ import pkg/libp2p import pkg/libp2p/routing_record import pkg/codexdht/discv5/spr as spr +import ../logging import ../node import ../blocktype import ../conf diff --git a/codex/sales.nim b/codex/sales.nim index 17a97f77c..14da3478c 100644 --- a/codex/sales.nim +++ b/codex/sales.nim @@ -3,13 +3,13 @@ import std/sugar import pkg/questionable import pkg/questionable/results import pkg/stint -import pkg/chronicles import pkg/datastore import ./market import ./clock import ./stores import ./contracts/requests import ./contracts/marketplace +import ./logging import ./sales/salescontext import ./sales/salesagent import ./sales/statemachine @@ -165,7 +165,7 @@ proc filled( processing.complete() proc processSlot(sales: Sales, item: SlotQueueItem, done: Future[void]) = - debug "processing slot from queue", requestId = $item.requestId, + debug "processing slot from queue", requestId = item.requestId, slot = item.slotIndex let agent = newSalesAgent( diff --git a/codex/sales/reservations.nim b/codex/sales/reservations.nim index 611c1ad6c..b0c68fc3c 100644 --- a/codex/sales/reservations.nim +++ b/codex/sales/reservations.nim @@ -28,19 +28,19 @@ push: {.upraises: [].} import std/typetraits import pkg/chronos -import pkg/chronicles except toJson import pkg/datastore import pkg/nimcrypto import pkg/questionable import pkg/questionable/results import pkg/stint import pkg/stew/byteutils +import ../logging import ../stores import ../contracts/requests import ../utils/json export requests -export chronicles except toJson +export logging logScope: topics = "sales reservations" @@ -139,13 +139,7 @@ proc toErr[E1: ref CatchableError, E2: ReservationsError]( return newException(E2, msg, e1) -proc writeValue*( - writer: var JsonWriter, - value: SomeStorableId) {.upraises:[IOError].} = - ## used for chronicles' logs - - mixin writeValue - writer.writeValue %value +logging.formatIt(SomeStorableId): it.toJson proc `onAvailabilityAdded=`*(self: Reservations, onAvailabilityAdded: OnAvailabilityAdded) = diff --git a/codex/sales/salesagent.nim b/codex/sales/salesagent.nim index 08d3ab0e6..45365f9f2 100644 --- a/codex/sales/salesagent.nim +++ b/codex/sales/salesagent.nim @@ -1,11 +1,11 @@ import pkg/chronos -import pkg/chronicles import pkg/questionable import pkg/questionable/results import pkg/stint import pkg/upraises import ../contracts/requests import ../errors +import ../logging import ./statemachine import ./salescontext import ./salesdata diff --git a/codex/sales/slotqueue.nim b/codex/sales/slotqueue.nim index fa8218647..e3db45272 100644 --- a/codex/sales/slotqueue.nim +++ b/codex/sales/slotqueue.nim @@ -1,11 +1,11 @@ import std/sequtils import std/tables -import pkg/chronicles import pkg/chronos import pkg/questionable import pkg/questionable/results import pkg/upraises import ../errors +import ../logging import ../rng import ../utils import ../contracts/requests diff --git a/codex/sales/states/cancelled.nim b/codex/sales/states/cancelled.nim index f1f754071..c6bdd082c 100644 --- a/codex/sales/states/cancelled.nim +++ b/codex/sales/states/cancelled.nim @@ -1,4 +1,4 @@ -import pkg/chronicles +import ../../logging import ../salesagent import ../statemachine import ./errorhandling @@ -21,7 +21,7 @@ method run*(state: SaleCancelled, machine: Machine): Future[?State] {.async.} = raiseAssert "no sale request" let slot = Slot(request: request, slotIndex: data.slotIndex) - debug "Collecting collateral and partial payout", requestId = $data.requestId, slotIndex = $data.slotIndex + debug "Collecting collateral and partial payout", requestId = data.requestId, slotIndex = data.slotIndex await market.freeSlot(slot.id) if onClear =? agent.context.onClear and @@ -31,4 +31,4 @@ method run*(state: SaleCancelled, machine: Machine): Future[?State] {.async.} = if onCleanUp =? agent.onCleanUp: await onCleanUp(returnBytes = true) - warn "Sale cancelled due to timeout", requestId = $data.requestId, slotIndex = $data.slotIndex + warn "Sale cancelled due to timeout", requestId = data.requestId, slotIndex = data.slotIndex diff --git a/codex/sales/states/downloading.nim b/codex/sales/states/downloading.nim index f79f0d687..8062e6ebe 100644 --- a/codex/sales/states/downloading.nim +++ b/codex/sales/states/downloading.nim @@ -1,7 +1,8 @@ -import pkg/chronicles import pkg/questionable import pkg/questionable/results + import ../../blocktype as bt +import ../../logging import ../../market import ../salesagent import ../statemachine diff --git a/codex/sales/states/errored.nim b/codex/sales/states/errored.nim index 59754eca2..3a909250a 100644 --- a/codex/sales/states/errored.nim +++ b/codex/sales/states/errored.nim @@ -1,9 +1,10 @@ import pkg/questionable import pkg/questionable/results import pkg/upraises -import pkg/chronicles + import ../statemachine import ../salesagent +import ../../logging import ../../utils/exceptions logScope: diff --git a/codex/sales/states/failed.nim b/codex/sales/states/failed.nim index cd954be41..ffd7da663 100644 --- a/codex/sales/states/failed.nim +++ b/codex/sales/states/failed.nim @@ -1,4 +1,4 @@ -import pkg/chronicles +import ../../logging import ../salesagent import ../statemachine import ./errorhandling @@ -21,7 +21,7 @@ method run*(state: SaleFailed, machine: Machine): Future[?State] {.async.} = raiseAssert "no sale request" let slot = Slot(request: request, slotIndex: data.slotIndex) - debug "Removing slot from mySlots", requestId = $data.requestId, slotIndex = $data.slotIndex + debug "Removing slot from mySlots", requestId = data.requestId, slotIndex = data.slotIndex await market.freeSlot(slot.id) let error = newException(SaleFailedError, "Sale failed") diff --git a/codex/sales/states/filled.nim b/codex/sales/states/filled.nim index c5bc4c27d..0e26ae94f 100644 --- a/codex/sales/states/filled.nim +++ b/codex/sales/states/filled.nim @@ -1,7 +1,8 @@ import pkg/questionable import pkg/questionable/results -import pkg/chronicles + import ../../conf +import ../../logging import ../statemachine import ../salesagent import ./errorhandling @@ -36,7 +37,7 @@ method run*(state: SaleFilled, machine: Machine): Future[?State] {.async.} = let me = await market.getSigner() if host == me.some: - info "Slot succesfully filled", requestId = $data.requestId, slotIndex = $data.slotIndex + info "Slot succesfully filled", requestId = data.requestId, slotIndex = data.slotIndex without request =? data.request: raiseAssert "no sale request" diff --git a/codex/sales/states/filling.nim b/codex/sales/states/filling.nim index b1e8471c1..e45302da5 100644 --- a/codex/sales/states/filling.nim +++ b/codex/sales/states/filling.nim @@ -1,4 +1,4 @@ -import pkg/chronicles +import ../../logging import ../../market import ../statemachine import ../salesagent @@ -32,5 +32,5 @@ method run(state: SaleFilling, machine: Machine): Future[?State] {.async.} = without (collateral =? data.request.?ask.?collateral): raiseAssert "Request not set" - debug "Filling slot", requestId = $data.requestId, slotIndex = $data.slotIndex + debug "Filling slot", requestId = data.requestId, slotIndex = data.slotIndex await market.fillSlot(data.requestId, data.slotIndex, state.proof, collateral) diff --git a/codex/sales/states/finished.nim b/codex/sales/states/finished.nim index 539cde62b..3d473aaf0 100644 --- a/codex/sales/states/finished.nim +++ b/codex/sales/states/finished.nim @@ -1,5 +1,6 @@ import pkg/chronos -import pkg/chronicles + +import ../../logging import ../statemachine import ../salesagent import ./errorhandling @@ -27,7 +28,7 @@ method run*(state: SaleFinished, machine: Machine): Future[?State] {.async.} = without request =? data.request: raiseAssert "no sale request" - info "Slot finished and paid out", requestId = $data.requestId, slotIndex = $data.slotIndex + info "Slot finished and paid out", requestId = data.requestId, slotIndex = data.slotIndex if onCleanUp =? agent.onCleanUp: await onCleanUp() diff --git a/codex/sales/states/ignored.nim b/codex/sales/states/ignored.nim index fa0641ccc..52b01e6ff 100644 --- a/codex/sales/states/ignored.nim +++ b/codex/sales/states/ignored.nim @@ -1,5 +1,6 @@ -import pkg/chronicles import pkg/chronos + +import ../../logging import ../statemachine import ../salesagent import ./errorhandling diff --git a/codex/sales/states/initialproving.nim b/codex/sales/states/initialproving.nim index d44b058b2..f5c370f27 100644 --- a/codex/sales/states/initialproving.nim +++ b/codex/sales/states/initialproving.nim @@ -1,4 +1,4 @@ -import pkg/chronicles +import ../../logging import ../statemachine import ../salesagent import ./errorhandling @@ -30,11 +30,11 @@ method run*(state: SaleInitialProving, machine: Machine): Future[?State] {.async without onProve =? context.onProve: raiseAssert "onProve callback not set" - debug "Generating initial proof", requestId = $data.requestId + debug "Generating initial proof", requestId = data.requestId let slot = Slot(request: request, slotIndex: data.slotIndex) challenge = await context.market.getChallenge(slot.id) proof = await onProve(slot, challenge) - debug "Finished proof calculation", requestId = $data.requestId + debug "Finished proof calculation", requestId = data.requestId return some State(SaleFilling(proof: proof)) diff --git a/codex/sales/states/payout.nim b/codex/sales/states/payout.nim index a70f7eac6..44bca5d02 100644 --- a/codex/sales/states/payout.nim +++ b/codex/sales/states/payout.nim @@ -1,4 +1,4 @@ -import pkg/chronicles +import ../../logging import ../../market import ../statemachine import ../salesagent @@ -29,7 +29,7 @@ method run(state: SalePayout, machine: Machine): Future[?State] {.async.} = raiseAssert "no sale request" let slot = Slot(request: request, slotIndex: data.slotIndex) - debug "Collecting finished slot's reward", requestId = $data.requestId, slotIndex = $data.slotIndex + debug "Collecting finished slot's reward", requestId = data.requestId, slotIndex = data.slotIndex await market.freeSlot(slot.id) return some State(SaleFinished()) diff --git a/codex/sales/states/preparing.nim b/codex/sales/states/preparing.nim index bb0fe7439..6bc6828bc 100644 --- a/codex/sales/states/preparing.nim +++ b/codex/sales/states/preparing.nim @@ -1,6 +1,7 @@ -import pkg/chronicles import pkg/questionable import pkg/questionable/results + +import ../../logging import ../../market import ../salesagent import ../statemachine diff --git a/codex/sales/states/proving.nim b/codex/sales/states/proving.nim index 367eaa9e3..e0c589cdb 100644 --- a/codex/sales/states/proving.nim +++ b/codex/sales/states/proving.nim @@ -1,6 +1,7 @@ import std/options -import pkg/chronicles + import ../../clock +import ../../logging import ../statemachine import ../salesagent import ../salescontext @@ -28,7 +29,7 @@ method prove*( ) {.base, async.} = try: let proof = await onProve(slot, challenge) - debug "Submitting proof", currentPeriod = currentPeriod, slotId = $slot.id + debug "Submitting proof", currentPeriod = currentPeriod, slotId = slot.id await market.submitProof(slot.id, proof) except CatchableError as e: error "Submitting proof failed", msg = e.msg @@ -47,9 +48,9 @@ proc proveLoop( logScope: period = currentPeriod - requestId = $request.id + requestId = request.id slotIndex - slotId = $slot.id + slotId = slot.id proc getCurrentPeriod(): Future[Period] {.async.} = let periodicity = await market.periodicity() @@ -106,7 +107,7 @@ method run*(state: SaleProving, machine: Machine): Future[?State] {.async.} = without clock =? context.clock: raiseAssert("clock not set") - debug "Start proving", requestId = $data.requestId, slotIndex = $data.slotIndex + debug "Start proving", requestId = data.requestId, slotIndex = data.slotIndex try: let loop = state.proveLoop(market, clock, request, data.slotIndex, onProve) state.loop = loop @@ -118,7 +119,7 @@ method run*(state: SaleProving, machine: Machine): Future[?State] {.async.} = return some State(SaleErrored(error: e)) finally: # Cleanup of the proving loop - debug "Stopping proving.", requestId = $data.requestId, slotIndex = $data.slotIndex + debug "Stopping proving.", requestId = data.requestId, slotIndex = data.slotIndex if not state.loop.isNil: if not state.loop.finished: diff --git a/codex/sales/states/provingsimulated.nim b/codex/sales/states/provingsimulated.nim index e3f5b2c2e..8722df6ce 100644 --- a/codex/sales/states/provingsimulated.nim +++ b/codex/sales/states/provingsimulated.nim @@ -1,12 +1,12 @@ import ../../conf when codex_enable_proof_failures: import std/strutils - import pkg/chronicles import pkg/stint import pkg/ethers import pkg/ethers/testing import ../../contracts/requests + import ../../logging import ../../market import ../salescontext import ./proving @@ -20,7 +20,7 @@ when codex_enable_proof_failures: proofCount: int proc onSubmitProofError(error: ref CatchableError, period: UInt256, slotId: SlotId) = - error "Submitting invalid proof failed", period = period, slotId = $slotId, msg = error.msg + error "Submitting invalid proof failed", period = period, slotId, msg = error.msg method prove*(state: SaleProvingSimulated, slot: Slot, challenge: ProofChallenge, onProve: OnProve, market: Market, currentPeriod: Period) {.async.} = trace "Processing proving in simulated mode" diff --git a/codex/sales/states/unknown.nim b/codex/sales/states/unknown.nim index 3672ed967..d68647741 100644 --- a/codex/sales/states/unknown.nim +++ b/codex/sales/states/unknown.nim @@ -1,4 +1,4 @@ -import pkg/chronicles +import ../../logging import ../statemachine import ../salesagent import ./filled diff --git a/codex/stores/cachestore.nim b/codex/stores/cachestore.nim index 13308c7a3..e57f22cc9 100644 --- a/codex/stores/cachestore.nim +++ b/codex/stores/cachestore.nim @@ -13,7 +13,6 @@ push: {.upraises: [].} import std/options -import pkg/chronicles import pkg/chronos import pkg/libp2p import pkg/lrucache @@ -24,6 +23,7 @@ import ./blockstore import ../units import ../chunker import ../errors +import ../logging import ../manifest import ../merkletree import ../utils diff --git a/codex/stores/maintenance.nim b/codex/stores/maintenance.nim index 98b6fab95..137809cd4 100644 --- a/codex/stores/maintenance.nim +++ b/codex/stores/maintenance.nim @@ -11,7 +11,6 @@ ## Looks for and removes expired blocks from blockstores. import pkg/chronos -import pkg/chronicles import pkg/questionable import pkg/questionable/results @@ -19,6 +18,7 @@ import ./repostore import ../utils/timer import ../utils/asynciter import ../clock +import ../logging import ../systemclock const diff --git a/codex/stores/networkstore.nim b/codex/stores/networkstore.nim index 6da1465c8..8ceae42cd 100644 --- a/codex/stores/networkstore.nim +++ b/codex/stores/networkstore.nim @@ -7,25 +7,22 @@ ## This file may not be copied, modified, or distributed except according to ## those terms. -import pkg/upraises +import pkg/upraises push: {.upraises: [].} -import std/sugar - -import pkg/chronicles import pkg/chronos import pkg/libp2p import pkg/questionable/results -import ../utils/asyncheapqueue -import ../utils/asynciter import ../clock - import ../blocktype -import ./blockstore import ../blockexchange +import ../logging import ../merkletree +import ../utils/asyncheapqueue +import ../utils/asynciter +import ./blockstore export blockstore, blockexchange, asyncheapqueue diff --git a/codex/stores/repostore.nim b/codex/stores/repostore.nim index db53ae6f4..f73572b2d 100644 --- a/codex/stores/repostore.nim +++ b/codex/stores/repostore.nim @@ -13,7 +13,6 @@ push: {.upraises: [].} import pkg/chronos import pkg/chronos/futures -import pkg/chronicles import pkg/libp2p/[cid, multicodec, multihash] import pkg/lrucache import pkg/metrics @@ -27,6 +26,7 @@ import ./keyutils import ../blocktype import ../clock import ../systemclock +import ../logging import ../merkletree import ../utils diff --git a/codex/streams/asyncstreamwrapper.nim b/codex/streams/asyncstreamwrapper.nim index f2491e8d4..65d9a6d64 100644 --- a/codex/streams/asyncstreamwrapper.nim +++ b/codex/streams/asyncstreamwrapper.nim @@ -11,9 +11,10 @@ import pkg/upraises push: {.upraises: [].} import pkg/chronos -import pkg/chronicles import pkg/libp2p +import ../logging + logScope: topics = "libp2p asyncstreamwrapper" diff --git a/codex/streams/seekablestream.nim b/codex/streams/seekablestream.nim index 785d9afea..0a587d01b 100644 --- a/codex/streams/seekablestream.nim +++ b/codex/streams/seekablestream.nim @@ -9,9 +9,10 @@ import pkg/libp2p/stream/lpstream import pkg/chronos -import pkg/chronicles -export lpstream, chronos, chronicles +import ../logging + +export lpstream, chronos, logging logScope: topics = "codex seekablestream" diff --git a/codex/streams/storestream.nim b/codex/streams/storestream.nim index 63406a5f5..c2a4e3228 100644 --- a/codex/streams/storestream.nim +++ b/codex/streams/storestream.nim @@ -14,12 +14,12 @@ import pkg/upraises push: {.upraises: [].} import pkg/chronos -import pkg/chronicles import pkg/stew/ptrops import ../stores import ../manifest import ../blocktype +import ../logging import ../utils import ./seekablestream diff --git a/codex/units.nim b/codex/units.nim index dcda8152d..e8dad5d03 100644 --- a/codex/units.nim +++ b/codex/units.nim @@ -12,8 +12,8 @@ import std/hashes import std/strutils import pkg/upraises -import pkg/json_serialization -import pkg/json_serialization/std/options + +import ./logging type NBytes* = distinct Natural @@ -42,6 +42,7 @@ divMaths(NBytes) proc `$`*(ts: NBytes): string = $(int(ts)) & "'NByte" proc `'nb`*(n: string): NBytes = parseInt(n).NBytes +logging.formatIt(NBytes): $it const MiB = 1024.NBytes * 1024.NBytes # ByteSz, 1 mebibyte = 1,048,576 ByteSz @@ -53,18 +54,6 @@ func divUp*[T: NBytes](a, b : T): int = assert(b != T(0)) if a==T(0): int(0) else: int( ((a - T(1)) div b) + 1 ) -proc writeValue*( - writer: var JsonWriter, - value: NBytes -) {.upraises:[IOError].} = - writer.writeValue value.int - -proc readValue*( - reader: var JsonReader, - value: var NBytes -) {.upraises: [SerializationError, IOError].} = - value = NBytes reader.readValue(int) - when isMainModule: import unittest2 diff --git a/codex/utils/asyncstatemachine.nim b/codex/utils/asyncstatemachine.nim index 512617ee9..0a3d0d3d3 100644 --- a/codex/utils/asyncstatemachine.nim +++ b/codex/utils/asyncstatemachine.nim @@ -1,10 +1,10 @@ import std/sugar import pkg/questionable import pkg/chronos -import pkg/chronicles import pkg/upraises -import ./trackedfutures +import ../logging import ./then +import ./trackedfutures push: {.upraises:[].} diff --git a/codex/utils/fileutils.nim b/codex/utils/fileutils.nim index 03cffeba4..ac5b6fe18 100644 --- a/codex/utils/fileutils.nim +++ b/codex/utils/fileutils.nim @@ -9,16 +9,17 @@ ## Partially taken from nim beacon chain -import std/strutils import pkg/upraises push: {.upraises: [].} -import pkg/chronicles -import stew/io2 +import std/strutils +import pkg/stew/io2 + +import ../logging export io2 -export chronicles except toJson +export logging when defined(windows): import stew/[windows/acl] diff --git a/codex/utils/json.nim b/codex/utils/json.nim index ade6936da..58c6ba402 100644 --- a/codex/utils/json.nim +++ b/codex/utils/json.nim @@ -6,13 +6,13 @@ import std/strutils import std/strformat import std/tables import std/typetraits -import pkg/chronicles from pkg/libp2p import Cid, init import pkg/contractabi import pkg/stew/byteutils import pkg/stint import pkg/questionable/results import ../errors +import ../logging export json except `%`, `%*` diff --git a/codex/utils/keyutils.nim b/codex/utils/keyutils.nim index ef6f6246d..b3a5e453e 100644 --- a/codex/utils/keyutils.nim +++ b/codex/utils/keyutils.nim @@ -10,12 +10,12 @@ import pkg/upraises push: {.upraises: [].} -import pkg/chronicles import pkg/questionable/results import pkg/libp2p/crypto/crypto import ./fileutils import ../errors +import ../logging import ../rng export crypto diff --git a/codex/utils/timer.nim b/codex/utils/timer.nim index 4eefc5991..f97b020e1 100644 --- a/codex/utils/timer.nim +++ b/codex/utils/timer.nim @@ -11,9 +11,10 @@ ## Used to execute a callback in a loop import pkg/chronos -import pkg/chronicles import pkg/upraises +import ../logging + type TimerCallback* = proc(): Future[void] {.gcsafe, upraises:[].} Timer* = ref object of RootObj diff --git a/codex/utils/trackedfutures.nim b/codex/utils/trackedfutures.nim index 064bf9e3e..e99decd4a 100644 --- a/codex/utils/trackedfutures.nim +++ b/codex/utils/trackedfutures.nim @@ -1,7 +1,8 @@ import std/sugar import std/tables -import pkg/chronicles import pkg/chronos + +import ../logging import ../utils/then type diff --git a/codex/validation.nim b/codex/validation.nim index 2be399757..a8e76eab6 100644 --- a/codex/validation.nim +++ b/codex/validation.nim @@ -1,9 +1,9 @@ import std/sets import std/sequtils import pkg/chronos -import pkg/chronicles import ./market import ./clock +import ./logging export market export sets @@ -48,7 +48,7 @@ proc subscribeSlotFilled(validation: Validation) {.async.} = let slotId = slotId(requestId, slotIndex) if slotId notin validation.slots: if validation.slots.len < validation.maxSlots: - trace "Adding slot", slotId = $slotId + trace "Adding slot", slotId validation.slots.incl(slotId) let subscription = await validation.market.subscribeSlotFilled(onSlotFilled) validation.subscriptions.add(subscription) @@ -58,7 +58,7 @@ proc removeSlotsThatHaveEnded(validation: Validation) {.async.} = for slotId in validation.slots: let state = await validation.market.slotState(slotId) if state != SlotState.Filled: - trace "Removing slot", slot = $slotId + trace "Removing slot", slotId ended.incl(slotId) validation.slots.excl(ended) @@ -70,7 +70,7 @@ proc markProofAsMissing(validation: Validation, try: if await validation.market.canProofBeMarkedAsMissing(slotId, period): - trace "Marking proof as missing", slotId = $slotId, periodProofMissed = period + trace "Marking proof as missing", slotId, periodProofMissed = period await validation.market.markProofAsMissing(slotId, period) else: let inDowntime {.used.} = await validation.market.inDowntime(slotId) diff --git a/tests/codex/sales/testslotqueue.nim b/tests/codex/sales/testslotqueue.nim index 706bcdbd6..4ae0abda2 100644 --- a/tests/codex/sales/testslotqueue.nim +++ b/tests/codex/sales/testslotqueue.nim @@ -1,11 +1,11 @@ import std/sequtils import pkg/asynctest -import pkg/chronicles import pkg/chronos import pkg/datastore import pkg/questionable import pkg/questionable/results +import pkg/codex/logging import pkg/codex/sales/slotqueue import ../helpers diff --git a/tests/codex/testchunking.nim b/tests/codex/testchunking.nim index 860e52b68..49a96fd61 100644 --- a/tests/codex/testchunking.nim +++ b/tests/codex/testchunking.nim @@ -2,7 +2,7 @@ import pkg/asynctest import pkg/stew/byteutils import pkg/codex/chunker -import pkg/chronicles +import pkg/codex/logging import pkg/chronos import ./helpers diff --git a/tests/codex/testlogging.nim b/tests/codex/testlogging.nim new file mode 100644 index 000000000..54d9fbf94 --- /dev/null +++ b/tests/codex/testlogging.nim @@ -0,0 +1,192 @@ +import std/json +import std/options +import std/nre +import std/strutils +import std/unittest +import pkg/codex/blocktype +import pkg/codex/conf +import pkg/codex/contracts/requests +import pkg/codex/logging +import pkg/codex/purchasing/purchaseid +import pkg/codex/units +import pkg/codex/utils/json +import pkg/libp2p/cid +import pkg/libp2p/multiaddress +import pkg/stew/byteutils +import pkg/stint +import ../checktest +import ../examples +import ./examples + +export logging + +logStream testlines[textlines[notimestamps,dynamic]] +logStream testjson[json[notimestamps,dynamic]] + +type + ObjectType = object + a: string + DistinctType {.borrow: `.`.} = distinct ObjectType + RefType = ref object + a: string + +# must be defined at the top-level +proc `$`*(t: ObjectType): string = "used `$`" +logging.formatIt(ObjectType): "formatted_" & it.a +logging.formatIt(RefType): "formatted_" & it.a +logging.formatIt(DistinctType): "formatted_" & it.a + +checksuite "Test logging outputLines": + var outputLines: string + var outputJson: string + + proc writeToLines(logLevel: LogLevel, msg: LogOutputStr) = + outputLines &= stripAnsi(msg) # nocolors in the stream definition doesn't seem to work + + proc writeToJson(logLevel: LogLevel, msg: LogOutputStr) = + outputJson &= stripAnsi(msg) + + setup: + outputLines = "" + outputJson = "" + testlines.outputs[0].writer = writeToLines + testjson.outputs[0].writer = writeToJson + + template logged(prop, expected): auto = + let regex = prop & """\=\"?""" & expected.escapeRe & """\"?""" + outputLines.contains(re(regex)) + + template loggedJson(prop, expected): auto = + let json = $ parseJson(outputJson){prop} + json == expected + + template log(val) = + testlines.trace "test", val + testjson.trace "test", val + + test "logs objects": + let t = ObjectType(a: "a") + log t + check logged("t", "formatted_a") + check loggedJson("t", "\"formatted_a\"") + + test "logs sequences of objects": + let t1 = ObjectType(a: "a") + let t2 = ObjectType(a: "b") + let t = @[t1, t2] + log t + check logged("t", "@[formatted_a, formatted_b]") + check loggedJson("t", "[\"formatted_a\",\"formatted_b\"]") + + test "logs ref types": + let t = RefType(a: "a") + log t + check logged("t", "formatted_a") + check loggedJson("t", "\"formatted_a\"") + + test "logs sequences of ref types": + let t1 = RefType(a: "a") + let t2 = RefType(a: "b") + let t = @[t1, t2] + log t + check logged("t", "@[formatted_a, formatted_b]") + check loggedJson("t", "[\"formatted_a\",\"formatted_b\"]") + + test "logs distinct types": + let t = DistinctType(ObjectType(a: "a")) + log t + check logged("t", "formatted_a") + check loggedJson("t", "\"formatted_a\"") + + test "logs sequences of distinct types": + let t1 = DistinctType(ObjectType(a: "a")) + let t2 = DistinctType(ObjectType(a: "b")) + let t = @[t1, t2] + log t + check logged("t", "@[formatted_a, formatted_b]") + check loggedJson("t", "[\"formatted_a\",\"formatted_b\"]") + + test "logs Option types": + let t = some ObjectType(a: "a") + log t + check logged("t", "Some(formatted_a)") + check loggedJson("t", """{"val":"formatted_a","has":true}""") + + test "logs sequences of Option types": + let t1 = some ObjectType(a: "a") + let t2 = none ObjectType + let t = @[t1, t2] + log t + check logged("t", "@[Some(formatted_a), None(ObjectType)]") + check loggedJson("t", """[{"val":"formatted_a","has":true},{"val":"formatted_","has":false}]""") + + test "can define `$` override for T": + let o = ObjectType() + check $o == "used `$`" + + test "logs NByte correctly": + let nb = 12345.NBytes + log nb + check logged("nb", "12345\'NByte") + check loggedJson("nb", "\"12345\'NByte\"") + + test "logs BlockAddress correctly": + let cid = Cid.init("zb2rhgsDE16rLtbwTFeNKbdSobtKiWdjJPvKEuPgrQAfndjU1").tryGet + let ba = BlockAddress.init(cid, 0) + log ba + check logged("ba", "treeCid: zb2*fndjU1, index: 0") + check loggedJson("ba", "\"treeCid: zb2*fndjU1, index: 0\"") + + test "logs Cid correctly": + let cid = Cid.init("zb2rhmfWaXASbyi15iLqbz5yp3awnSyecpt9jcFnc2YA5TgiD").tryGet + log cid + check logged("cid", "zb2*A5TgiD") + + test "logs StUint correctly": + let stint = 12345678901234.u256 + log stint + check logged("stint", "12345678901234") + + test "logs EthAddress correctly": + let address = EthAddress.fromHex("0xf75e076f650cd51dbfa0fd9c465d5037f22e1b1b") + log address + check logged("address", "0xf75e..1b1b") + + test "logs PurchaseId correctly": + let id = PurchaseId.fromHex("0x712003bdfc0db9abf21e7fbb7119cd52ff221c96714d21d39e782d7c744d3dea") + log id + check logged("id", "0x7120..3dea") + + test "logs RequestId correctly": + let id = RequestId.fromHex("0x712003bdfc0db9abf21e7fbb7119cd52ff221c96714d21d39e782d7c744d3dea") + log id + check logged("id", "0x7120..3dea") + + test "logs seq[RequestId] correctly": + let id = RequestId.fromHex("0x712003bdfc0db9abf21e7fbb7119cd52ff221c96714d21d39e782d7c744d3dea") + let id2 = RequestId.fromHex("0x9ab2c4d102a95d990facb022d67b3c9b39052597c006fddf122bed2cb594c282") + let ids = @[id, id2] + log ids + check logged("ids", "@[0x7120..3dea, 0x9ab2..c282]") + check loggedJson("ids", """["0x7120..3dea","0x9ab2..c282"]""") + + test "logs SlotId correctly": + let id = SlotId.fromHex("0x9ab2c4d102a95d990facb022d67b3c9b39052597c006fddf122bed2cb594c282") + log id + check logged("id", "0x9ab2..c282") + + test "logs Nonce correctly": + let id = SlotId.fromHex("ce88f368a7b776172ebd29a212456eb66acb60f169ee76eae91935e7fafad6ea") + log id + check logged("id", "0xce88..d6ea") + + test "logs MultiAddress correctly": + let ma = MultiAddress.init("/ip4/127.0.0.1/tcp/0").tryGet + log ma + check logged("ma", "/ip4/127.0.0.1/tcp/0") + + test "logs seq[MultiAddress] correctly": + let ma = @[MultiAddress.init("/ip4/127.0.0.1/tcp/0").tryGet, + MultiAddress.init("/ip4/127.0.0.2/tcp/1").tryGet] + log ma + check logged("ma", "@[/ip4/127.0.0.1/tcp/0, /ip4/127.0.0.2/tcp/1]") diff --git a/tests/codex/testnode.nim b/tests/codex/testnode.nim index bb3876e35..6d1ee1491 100644 --- a/tests/codex/testnode.nim +++ b/tests/codex/testnode.nim @@ -5,7 +5,6 @@ import std/times import pkg/asynctest import pkg/chronos -import pkg/chronicles import pkg/stew/byteutils import pkg/datastore import pkg/questionable @@ -18,6 +17,7 @@ import pkg/codexdht/discv5/protocol as discv5 import pkg/codex/stores import pkg/codex/clock import pkg/codex/contracts +import pkg/codex/logging import pkg/codex/systemclock import pkg/codex/blockexchange import pkg/codex/chunker diff --git a/tests/codex/utils/testjson.nim b/tests/codex/utils/testjson.nim index aa3731e57..324ee0b98 100644 --- a/tests/codex/utils/testjson.nim +++ b/tests/codex/utils/testjson.nim @@ -3,11 +3,11 @@ import std/options import std/strformat import std/strutils import std/unittest -import pkg/chronicles except toJson import pkg/stew/byteutils import pkg/stint import pkg/codex/contracts/requests from pkg/codex/rest/json import RestPurchase +import pkg/codex/logging import pkg/codex/utils/json as utilsjson import pkg/questionable import pkg/questionable/results diff --git a/tests/integration/codexclient.nim b/tests/integration/codexclient.nim index dbddc2046..324d60429 100644 --- a/tests/integration/codexclient.nim +++ b/tests/integration/codexclient.nim @@ -3,9 +3,9 @@ import std/strutils import std/sequtils from pkg/libp2p import Cid, `$`, init -import pkg/chronicles import pkg/stint import pkg/questionable/results +import pkg/codex/logging import pkg/codex/rest/json import pkg/codex/purchasing import pkg/codex/errors diff --git a/tests/integration/multinodes.nim b/tests/integration/multinodes.nim index b601e6db8..4167377ef 100644 --- a/tests/integration/multinodes.nim +++ b/tests/integration/multinodes.nim @@ -2,7 +2,7 @@ import std/os import std/macros import std/json import std/httpclient -import pkg/chronicles +import pkg/codex/logging import ../ethertest import ./codexclient import ./nodes diff --git a/tests/integration/nodes.nim b/tests/integration/nodes.nim index 777de80b1..b89c9ff62 100644 --- a/tests/integration/nodes.nim +++ b/tests/integration/nodes.nim @@ -1,12 +1,12 @@ -import pkg/questionable -import pkg/confutils -import pkg/chronicles -import pkg/libp2p import std/osproc import std/os import std/streams import std/strutils -import codex/conf +import pkg/codex/conf +import pkg/codex/logging +import pkg/confutils +import pkg/libp2p +import pkg/questionable import ./codexclient export codexclient diff --git a/tests/integration/testIntegration.nim b/tests/integration/testIntegration.nim index eed181f55..c60fd0eff 100644 --- a/tests/integration/testIntegration.nim +++ b/tests/integration/testIntegration.nim @@ -20,7 +20,7 @@ import ./twonodes # You can also pass a string in same format like for the `--log-level` parameter # to enable custom logging levels for specific topics like: debug2 = "INFO; TRACE: marketplace" -twonodessuite "Integration tests", debug1 = false, debug2 = false: +twonodessuite "Integration tests", debug1 = true, debug2 = true: proc purchaseStateIs(client: CodexClient, id: PurchaseId, state: string): bool = without purchase =? client.getPurchase(id): diff --git a/tests/integration/testproofs.nim b/tests/integration/testproofs.nim index 86905f8f2..c58963e32 100644 --- a/tests/integration/testproofs.nim +++ b/tests/integration/testproofs.nim @@ -1,9 +1,9 @@ import std/sequtils import std/os from std/times import getTime, toUnix -import pkg/chronicles -import codex/contracts -import codex/periods +import pkg/codex/contracts +import pkg/codex/logging +import pkg/codex/periods import ../contracts/time import ../contracts/deployment import ./twonodes diff --git a/tests/integration/twonodes.nim b/tests/integration/twonodes.nim index fafaf5677..b09551e1e 100644 --- a/tests/integration/twonodes.nim +++ b/tests/integration/twonodes.nim @@ -40,6 +40,7 @@ template twonodessuite*(name: string, debug1, debug2: string, body) = "--disc-port=8090", "--listen-addrs=/ip4/127.0.0.1/tcp/0", "--persistence", + "--log-format=json", "--eth-account=" & $account1 ] @@ -60,6 +61,7 @@ template twonodessuite*(name: string, debug1, debug2: string, body) = "--listen-addrs=/ip4/127.0.0.1/tcp/0", "--bootstrap-node=" & bootstrap, "--persistence", + "--log-format=json", "--eth-account=" & $account2 ] diff --git a/tests/logging.nim b/tests/logging.nim index cf2633d34..f58c060cd 100644 --- a/tests/logging.nim +++ b/tests/logging.nim @@ -1,5 +1,5 @@ when not defined(nimscript): - import pkg/chronicles + import pkg/codex/logging proc ignoreLogging(level: LogLevel, message: LogOutputStr) = discard diff --git a/tests/testCodex.nim b/tests/testCodex.nim index db9076843..8c1ba87df 100644 --- a/tests/testCodex.nim +++ b/tests/testCodex.nim @@ -2,7 +2,9 @@ import ./codex/teststores import ./codex/testblockexchange import ./codex/testasyncheapqueue import ./codex/testchunking +import ./codex/testlogging import ./codex/testmanifest +import ./codex/testlogging import ./codex/testnode import ./codex/teststorestream import ./codex/testpurchasing From ce8070aa3a67554c4694f895a50abf4838fd8ced Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Wed, 13 Dec 2023 11:25:36 +1100 Subject: [PATCH 02/13] Rename logging -> logutils to avoid ambiguity with common names --- codex.nim | 4 ++-- codex/blockexchange/engine/discovery.nim | 2 +- codex/blockexchange/engine/engine.nim | 2 +- codex/blockexchange/engine/pendingblocks.nim | 2 +- codex/blockexchange/network/network.nim | 2 +- codex/blockexchange/network/networkpeer.nim | 2 +- codex/blockexchange/peers/peercontext.nim | 2 +- codex/blockexchange/peers/peerctxstore.nim | 2 +- codex/blocktype.nim | 6 +++--- codex/chunker.nim | 2 +- codex/codex.nim | 2 +- codex/conf.nim | 2 +- codex/contracts/deployment.nim | 2 +- .../contracts/interactions/clientinteractions.nim | 4 ++-- codex/contracts/interactions/hostinteractions.nim | 4 ++-- codex/contracts/market.nim | 2 +- codex/contracts/requests.nim | 8 ++++---- codex/discovery.nim | 2 +- codex/erasure/erasure.nim | 2 +- codex/{logging.nim => logutils.nim} | 9 ++++----- codex/manifest/coders.nim | 2 +- codex/merkletree/merkletree.nim | 2 +- codex/node.nim | 2 +- codex/purchasing/purchaseid.nim | 2 +- codex/purchasing/states/cancelled.nim | 2 +- codex/purchasing/states/error.nim | 2 +- codex/purchasing/states/finished.nim | 2 +- codex/purchasing/states/started.nim | 2 +- codex/purchasing/states/submitted.nim | 2 +- codex/rest/api.nim | 2 +- codex/sales.nim | 2 +- codex/sales/reservations.nim | 6 +++--- codex/sales/salesagent.nim | 2 +- codex/sales/slotqueue.nim | 2 +- codex/sales/states/cancelled.nim | 2 +- codex/sales/states/downloading.nim | 2 +- codex/sales/states/errored.nim | 2 +- codex/sales/states/failed.nim | 2 +- codex/sales/states/filled.nim | 2 +- codex/sales/states/filling.nim | 2 +- codex/sales/states/finished.nim | 2 +- codex/sales/states/ignored.nim | 2 +- codex/sales/states/initialproving.nim | 2 +- codex/sales/states/payout.nim | 2 +- codex/sales/states/preparing.nim | 2 +- codex/sales/states/proving.nim | 2 +- codex/sales/states/provingsimulated.nim | 2 +- codex/sales/states/unknown.nim | 2 +- codex/stores/cachestore.nim | 2 +- codex/stores/maintenance.nim | 2 +- codex/stores/networkstore.nim | 2 +- codex/stores/repostore.nim | 2 +- codex/streams/asyncstreamwrapper.nim | 4 ++-- codex/streams/seekablestream.nim | 4 ++-- codex/streams/storestream.nim | 6 +++--- codex/units.nim | 6 +++--- codex/utils/asyncstatemachine.nim | 2 +- codex/utils/fileutils.nim | 4 ++-- codex/utils/json.nim | 2 +- codex/utils/keyutils.nim | 4 ++-- codex/utils/timer.nim | 2 +- codex/utils/trackedfutures.nim | 2 +- codex/validation.nim | 2 +- tests/codex/sales/testslotqueue.nim | 2 +- tests/codex/testchunking.nim | 2 +- tests/codex/testlogging.nim | 14 +++++++------- tests/codex/testnode.nim | 2 +- tests/codex/utils/testjson.nim | 2 +- tests/integration/codexclient.nim | 2 +- tests/integration/multinodes.nim | 2 +- tests/integration/nodes.nim | 2 +- tests/integration/testproofs.nim | 2 +- tests/logging.nim | 2 +- 73 files changed, 100 insertions(+), 101 deletions(-) rename codex/{logging.nim => logutils.nim} (89%) diff --git a/codex.nim b/codex.nim index 9c110e581..f7215db61 100644 --- a/codex.nim +++ b/codex.nim @@ -20,11 +20,11 @@ import pkg/libp2p import ./codex/conf import ./codex/codex -import ./codex/logging +import ./codex/logutils import ./codex/units import ./codex/utils/keyutils -export codex, conf, libp2p, chronos, logging +export codex, conf, libp2p, chronos, logutils when isMainModule: import std/sequtils diff --git a/codex/blockexchange/engine/discovery.nim b/codex/blockexchange/engine/discovery.nim index 9490d6a8c..267d8f124 100644 --- a/codex/blockexchange/engine/discovery.nim +++ b/codex/blockexchange/engine/discovery.nim @@ -24,7 +24,7 @@ import ../peers import ../../utils import ../../discovery import ../../stores/blockstore -import ../../logging +import ../../logutils logScope: topics = "codex discoveryengine" diff --git a/codex/blockexchange/engine/engine.nim b/codex/blockexchange/engine/engine.nim index 4386f65a1..f3423434d 100644 --- a/codex/blockexchange/engine/engine.nim +++ b/codex/blockexchange/engine/engine.nim @@ -22,7 +22,7 @@ import ../../stores/blockstore import ../../blocktype import ../../utils import ../../merkletree -import ../../logging +import ../../logutils import ../protobuf/blockexc import ../protobuf/presence diff --git a/codex/blockexchange/engine/pendingblocks.nim b/codex/blockexchange/engine/pendingblocks.nim index 4d149f057..cd4c1ed7f 100644 --- a/codex/blockexchange/engine/pendingblocks.nim +++ b/codex/blockexchange/engine/pendingblocks.nim @@ -22,7 +22,7 @@ import pkg/questionable/results import ../protobuf/blockexc import ../../blocktype import ../../merkletree -import ../../logging +import ../../logutils logScope: topics = "codex pendingblocks" diff --git a/codex/blockexchange/network/network.nim b/codex/blockexchange/network/network.nim index 287b6f065..1a39364b0 100644 --- a/codex/blockexchange/network/network.nim +++ b/codex/blockexchange/network/network.nim @@ -18,7 +18,7 @@ import pkg/questionable import pkg/questionable/results import ../../blocktype as bt -import ../../logging +import ../../logutils import ../protobuf/blockexc as pb import ../protobuf/payments diff --git a/codex/blockexchange/network/networkpeer.nim b/codex/blockexchange/network/networkpeer.nim index 66bfa96e0..69a193867 100644 --- a/codex/blockexchange/network/networkpeer.nim +++ b/codex/blockexchange/network/networkpeer.nim @@ -16,7 +16,7 @@ import pkg/libp2p import ../protobuf/blockexc import ../protobuf/message import ../../errors -import ../../logging +import ../../logutils logScope: topics = "codex blockexcnetworkpeer" diff --git a/codex/blockexchange/peers/peercontext.nim b/codex/blockexchange/peers/peercontext.nim index 40b330016..70b139b7b 100644 --- a/codex/blockexchange/peers/peercontext.nim +++ b/codex/blockexchange/peers/peercontext.nim @@ -22,7 +22,7 @@ import ../protobuf/payments import ../protobuf/presence import ../../blocktype -import ../../logging +import ../../logutils export payments, nitro diff --git a/codex/blockexchange/peers/peerctxstore.nim b/codex/blockexchange/peers/peerctxstore.nim index 1182a0b6d..b07265b4e 100644 --- a/codex/blockexchange/peers/peerctxstore.nim +++ b/codex/blockexchange/peers/peerctxstore.nim @@ -20,7 +20,7 @@ import pkg/libp2p import ../protobuf/blockexc import ../../blocktype -import ../../logging +import ../../logutils import ./peercontext diff --git a/codex/blocktype.nim b/codex/blocktype.nim index 7046c5b0d..edf0f9ae7 100644 --- a/codex/blocktype.nim +++ b/codex/blocktype.nim @@ -24,9 +24,9 @@ import pkg/json_serialization import ./units import ./utils import ./errors -import ./logging +import ./logutils -export errors, logging, units +export errors, logutils, units const # Size of blocks for storage / network exchange, @@ -46,7 +46,7 @@ type else: cid*: Cid -logging.formatIt(BlockAddress): +logutils.formatIt(BlockAddress): if it.leaf: "treeCid: " & shortLog($it.treeCid) & ", index: " & $it.index else: diff --git a/codex/chunker.nim b/codex/chunker.nim index 9b3660565..36f28f7ad 100644 --- a/codex/chunker.nim +++ b/codex/chunker.nim @@ -19,7 +19,7 @@ import pkg/chronos import pkg/libp2p except shuffle import ./blocktype -import ./logging +import ./logutils export blocktype diff --git a/codex/codex.nim b/codex/codex.nim index dedfd219f..0b3176e71 100644 --- a/codex/codex.nim +++ b/codex/codex.nim @@ -38,7 +38,7 @@ import ./contracts/clock import ./contracts/deployment import ./utils/addrutils import ./namespaces -import ./logging +import ./logutils logScope: topics = "codex node" diff --git a/codex/conf.nim b/codex/conf.nim index 003d3a2c1..da4f0d866 100644 --- a/codex/conf.nim +++ b/codex/conf.nim @@ -34,7 +34,7 @@ import pkg/questionable import pkg/questionable/results import ./discovery -import ./logging +import ./logutils import ./stores import ./units import ./utils diff --git a/codex/contracts/deployment.nim b/codex/contracts/deployment.nim index c517b9c11..01bf5f813 100644 --- a/codex/contracts/deployment.nim +++ b/codex/contracts/deployment.nim @@ -5,7 +5,7 @@ import pkg/ethers import pkg/questionable import ../conf -import ../logging +import ../logutils import ./marketplace type Deployment* = ref object diff --git a/codex/contracts/interactions/clientinteractions.nim b/codex/contracts/interactions/clientinteractions.nim index 71a91e948..78b3bedf8 100644 --- a/codex/contracts/interactions/clientinteractions.nim +++ b/codex/contracts/interactions/clientinteractions.nim @@ -1,13 +1,13 @@ import pkg/ethers import ../../purchasing -import ../../logging +import ../../logutils import ../market import ../clock import ./interactions export purchasing -export logging +export logutils type ClientInteractions* = ref object of ContractInteractions diff --git a/codex/contracts/interactions/hostinteractions.nim b/codex/contracts/interactions/hostinteractions.nim index 96bf8de22..2decfa44a 100644 --- a/codex/contracts/interactions/hostinteractions.nim +++ b/codex/contracts/interactions/hostinteractions.nim @@ -1,11 +1,11 @@ import pkg/chronos -import ../../logging +import ../../logutils import ../../sales import ./interactions export sales -export logging +export logutils type HostInteractions* = ref object of ContractInteractions diff --git a/codex/contracts/market.nim b/codex/contracts/market.nim index 50acb497e..51b3577b0 100644 --- a/codex/contracts/market.nim +++ b/codex/contracts/market.nim @@ -5,7 +5,7 @@ import pkg/ethers import pkg/ethers/testing import pkg/upraises import pkg/questionable -import ../logging +import ../logutils import ../market import ./marketplace diff --git a/codex/contracts/requests.nim b/codex/contracts/requests.nim index a4ac83d5c..ad4700e04 100644 --- a/codex/contracts/requests.nim +++ b/codex/contracts/requests.nim @@ -8,7 +8,7 @@ import pkg/questionable/results import pkg/stew/byteutils import pkg/json_serialization import pkg/upraises -import ../logging +import ../logutils import ../utils/json export contractabi @@ -81,9 +81,9 @@ proc toHex*[T: distinct](id: T): string = type baseType = T.distinctBase baseType(id).toHex -logging.formatIt(Nonce): it.short0xHexLog -logging.formatIt(RequestId): it.short0xHexLog -logging.formatIt(SlotId): it.short0xHexLog +logutils.formatIt(Nonce): it.short0xHexLog +logutils.formatIt(RequestId): it.short0xHexLog +logutils.formatIt(SlotId): it.short0xHexLog func fromTuple(_: type StorageRequest, tupl: tuple): StorageRequest = StorageRequest( diff --git a/codex/discovery.nim b/codex/discovery.nim index 4a00bcbeb..ce0b4032b 100644 --- a/codex/discovery.nim +++ b/codex/discovery.nim @@ -20,7 +20,7 @@ import pkg/codexdht/discv5/protocol as discv5 import ./rng import ./errors -import ./logging +import ./logutils export discv5 diff --git a/codex/erasure/erasure.nim b/codex/erasure/erasure.nim index 31ad5bd0a..26be0473c 100644 --- a/codex/erasure/erasure.nim +++ b/codex/erasure/erasure.nim @@ -18,7 +18,7 @@ import pkg/chronos import pkg/libp2p/[multicodec, cid, multibase, multihash] import pkg/libp2p/protobuf/minprotobuf -import ../logging +import ../logutils import ../manifest import ../merkletree import ../stores diff --git a/codex/logging.nim b/codex/logutils.nim similarity index 89% rename from codex/logging.nim rename to codex/logutils.nim index d30682273..f1e660651 100644 --- a/codex/logging.nim +++ b/codex/logutils.nim @@ -53,19 +53,18 @@ proc formatSeq*(val: seq[string]): string = "@[" & val.join(", ") & "]" template formatOption*(val, T, body): auto = - var v = "None(" & $T & ")" + var v = "none(" & $T & ")" if it =? val: - v = "Some(" & body & ")" # that I used to know :) + v = "some(" & body & ")" # that I used to know :) v template formatIt*(T: type, body: untyped) {.dirty.} = - chronicles.formatIt(T, body) proc writeValue*(writer: var JsonWriter, it: T) {.upraises:[IOError].} = let formatted = body writer.writeValue(formatted) - proc setProperty*(r: var JsonRecord, key: string, it: T) = + proc setProperty*(r: var JsonRecord, key: string, it: T) {.upraises:[IOError].} = let v = body setProperty(r, key, v) @@ -80,7 +79,7 @@ template formatIt*(T: type, body: untyped) {.dirty.} = let v = val.map(it => body).formatSeq setProperty(r, key, v) - proc setProperty*(r: var TextLineRecord, key: string, it: T) = + proc setProperty*(r: var TextLineRecord, key: string, it: T) {.upraises:[ValueError].} = let v = body setProperty(r, key, v) diff --git a/codex/manifest/coders.nim b/codex/manifest/coders.nim index 35c52914d..24f5199ad 100644 --- a/codex/manifest/coders.nim +++ b/codex/manifest/coders.nim @@ -24,7 +24,7 @@ import pkg/chronos import ./manifest import ../errors import ../blocktype -import ../logging +import ../logutils import ./types proc encode*(_: DagPBCoder, manifest: Manifest): ?!seq[byte] = diff --git a/codex/merkletree/merkletree.nim b/codex/merkletree/merkletree.nim index b6840589b..afc49be18 100644 --- a/codex/merkletree/merkletree.nim +++ b/codex/merkletree/merkletree.nim @@ -20,7 +20,7 @@ import pkg/libp2p/[cid, multicodec, multihash, vbuffer] import pkg/stew/byteutils import ../errors -import ../logging +import ../logutils logScope: topics = "codex merkletree" diff --git a/codex/node.nim b/codex/node.nim index f21f3a37d..a4f40afa5 100644 --- a/codex/node.nim +++ b/codex/node.nim @@ -38,7 +38,7 @@ import ./contracts import ./node/batch import ./utils import ./errors -import ./logging +import ./logutils export batch diff --git a/codex/purchasing/purchaseid.nim b/codex/purchasing/purchaseid.nim index 0aff18f7a..7b762e27d 100644 --- a/codex/purchasing/purchaseid.nim +++ b/codex/purchasing/purchaseid.nim @@ -1,6 +1,6 @@ import std/hashes import pkg/nimcrypto -import ../logging +import ../logutils type PurchaseId* = distinct array[32, byte] diff --git a/codex/purchasing/states/cancelled.nim b/codex/purchasing/states/cancelled.nim index 9e3aaf2c3..f9bb1ecee 100644 --- a/codex/purchasing/states/cancelled.nim +++ b/codex/purchasing/states/cancelled.nim @@ -1,6 +1,6 @@ import pkg/metrics -import ../../logging +import ../../logutils import ../statemachine import ./errorhandling diff --git a/codex/purchasing/states/error.nim b/codex/purchasing/states/error.nim index 73b1848b7..0ebe1dbed 100644 --- a/codex/purchasing/states/error.nim +++ b/codex/purchasing/states/error.nim @@ -1,7 +1,7 @@ import pkg/metrics import ../statemachine import ../../utils/exceptions -import ../../logging +import ../../logutils declareCounter(codex_purchases_error, "codex purchases error") diff --git a/codex/purchasing/states/finished.nim b/codex/purchasing/states/finished.nim index cdf43ebc9..0f97150de 100644 --- a/codex/purchasing/states/finished.nim +++ b/codex/purchasing/states/finished.nim @@ -1,7 +1,7 @@ import pkg/metrics import ../statemachine -import ../../logging +import ../../logutils declareCounter(codex_purchases_finished, "codex purchases finished") diff --git a/codex/purchasing/states/started.nim b/codex/purchasing/states/started.nim index 0ea660766..fff1bbc8f 100644 --- a/codex/purchasing/states/started.nim +++ b/codex/purchasing/states/started.nim @@ -1,6 +1,6 @@ import pkg/metrics -import ../../logging +import ../../logutils import ../statemachine import ./errorhandling import ./finished diff --git a/codex/purchasing/states/submitted.nim b/codex/purchasing/states/submitted.nim index b8617136c..391dbf7cc 100644 --- a/codex/purchasing/states/submitted.nim +++ b/codex/purchasing/states/submitted.nim @@ -1,6 +1,6 @@ import pkg/metrics -import ../../logging +import ../../logutils import ../statemachine import ./errorhandling import ./started diff --git a/codex/rest/api.nim b/codex/rest/api.nim index 787762f65..6100f35c1 100644 --- a/codex/rest/api.nim +++ b/codex/rest/api.nim @@ -27,7 +27,7 @@ import pkg/libp2p import pkg/libp2p/routing_record import pkg/codexdht/discv5/spr as spr -import ../logging +import ../logutils import ../node import ../blocktype import ../conf diff --git a/codex/sales.nim b/codex/sales.nim index 14da3478c..dba4e18d4 100644 --- a/codex/sales.nim +++ b/codex/sales.nim @@ -9,7 +9,7 @@ import ./clock import ./stores import ./contracts/requests import ./contracts/marketplace -import ./logging +import ./logutils import ./sales/salescontext import ./sales/salesagent import ./sales/statemachine diff --git a/codex/sales/reservations.nim b/codex/sales/reservations.nim index b0c68fc3c..075dab7ab 100644 --- a/codex/sales/reservations.nim +++ b/codex/sales/reservations.nim @@ -34,13 +34,13 @@ import pkg/questionable import pkg/questionable/results import pkg/stint import pkg/stew/byteutils -import ../logging +import ../logutils import ../stores import ../contracts/requests import ../utils/json export requests -export logging +export logutils logScope: topics = "sales reservations" @@ -139,7 +139,7 @@ proc toErr[E1: ref CatchableError, E2: ReservationsError]( return newException(E2, msg, e1) -logging.formatIt(SomeStorableId): it.toJson +logutils.formatIt(SomeStorableId): it.toJson proc `onAvailabilityAdded=`*(self: Reservations, onAvailabilityAdded: OnAvailabilityAdded) = diff --git a/codex/sales/salesagent.nim b/codex/sales/salesagent.nim index 45365f9f2..21475426b 100644 --- a/codex/sales/salesagent.nim +++ b/codex/sales/salesagent.nim @@ -5,7 +5,7 @@ import pkg/stint import pkg/upraises import ../contracts/requests import ../errors -import ../logging +import ../logutils import ./statemachine import ./salescontext import ./salesdata diff --git a/codex/sales/slotqueue.nim b/codex/sales/slotqueue.nim index e3db45272..a875f9178 100644 --- a/codex/sales/slotqueue.nim +++ b/codex/sales/slotqueue.nim @@ -5,7 +5,7 @@ import pkg/questionable import pkg/questionable/results import pkg/upraises import ../errors -import ../logging +import ../logutils import ../rng import ../utils import ../contracts/requests diff --git a/codex/sales/states/cancelled.nim b/codex/sales/states/cancelled.nim index c6bdd082c..aafed177e 100644 --- a/codex/sales/states/cancelled.nim +++ b/codex/sales/states/cancelled.nim @@ -1,4 +1,4 @@ -import ../../logging +import ../../logutils import ../salesagent import ../statemachine import ./errorhandling diff --git a/codex/sales/states/downloading.nim b/codex/sales/states/downloading.nim index 8062e6ebe..7c71f249b 100644 --- a/codex/sales/states/downloading.nim +++ b/codex/sales/states/downloading.nim @@ -2,7 +2,7 @@ import pkg/questionable import pkg/questionable/results import ../../blocktype as bt -import ../../logging +import ../../logutils import ../../market import ../salesagent import ../statemachine diff --git a/codex/sales/states/errored.nim b/codex/sales/states/errored.nim index 3a909250a..51f34bc96 100644 --- a/codex/sales/states/errored.nim +++ b/codex/sales/states/errored.nim @@ -4,7 +4,7 @@ import pkg/upraises import ../statemachine import ../salesagent -import ../../logging +import ../../logutils import ../../utils/exceptions logScope: diff --git a/codex/sales/states/failed.nim b/codex/sales/states/failed.nim index ffd7da663..e32fbb58a 100644 --- a/codex/sales/states/failed.nim +++ b/codex/sales/states/failed.nim @@ -1,4 +1,4 @@ -import ../../logging +import ../../logutils import ../salesagent import ../statemachine import ./errorhandling diff --git a/codex/sales/states/filled.nim b/codex/sales/states/filled.nim index 0e26ae94f..6e3e0106e 100644 --- a/codex/sales/states/filled.nim +++ b/codex/sales/states/filled.nim @@ -2,7 +2,7 @@ import pkg/questionable import pkg/questionable/results import ../../conf -import ../../logging +import ../../logutils import ../statemachine import ../salesagent import ./errorhandling diff --git a/codex/sales/states/filling.nim b/codex/sales/states/filling.nim index e45302da5..b66467339 100644 --- a/codex/sales/states/filling.nim +++ b/codex/sales/states/filling.nim @@ -1,4 +1,4 @@ -import ../../logging +import ../../logutils import ../../market import ../statemachine import ../salesagent diff --git a/codex/sales/states/finished.nim b/codex/sales/states/finished.nim index 3d473aaf0..59e9244c1 100644 --- a/codex/sales/states/finished.nim +++ b/codex/sales/states/finished.nim @@ -1,6 +1,6 @@ import pkg/chronos -import ../../logging +import ../../logutils import ../statemachine import ../salesagent import ./errorhandling diff --git a/codex/sales/states/ignored.nim b/codex/sales/states/ignored.nim index 52b01e6ff..d757e9c18 100644 --- a/codex/sales/states/ignored.nim +++ b/codex/sales/states/ignored.nim @@ -1,6 +1,6 @@ import pkg/chronos -import ../../logging +import ../../logutils import ../statemachine import ../salesagent import ./errorhandling diff --git a/codex/sales/states/initialproving.nim b/codex/sales/states/initialproving.nim index f5c370f27..3496b2924 100644 --- a/codex/sales/states/initialproving.nim +++ b/codex/sales/states/initialproving.nim @@ -1,4 +1,4 @@ -import ../../logging +import ../../logutils import ../statemachine import ../salesagent import ./errorhandling diff --git a/codex/sales/states/payout.nim b/codex/sales/states/payout.nim index 44bca5d02..5c8c28590 100644 --- a/codex/sales/states/payout.nim +++ b/codex/sales/states/payout.nim @@ -1,4 +1,4 @@ -import ../../logging +import ../../logutils import ../../market import ../statemachine import ../salesagent diff --git a/codex/sales/states/preparing.nim b/codex/sales/states/preparing.nim index 6bc6828bc..973446e27 100644 --- a/codex/sales/states/preparing.nim +++ b/codex/sales/states/preparing.nim @@ -1,7 +1,7 @@ import pkg/questionable import pkg/questionable/results -import ../../logging +import ../../logutils import ../../market import ../salesagent import ../statemachine diff --git a/codex/sales/states/proving.nim b/codex/sales/states/proving.nim index e0c589cdb..6fba9096c 100644 --- a/codex/sales/states/proving.nim +++ b/codex/sales/states/proving.nim @@ -1,7 +1,7 @@ import std/options import ../../clock -import ../../logging +import ../../logutils import ../statemachine import ../salesagent import ../salescontext diff --git a/codex/sales/states/provingsimulated.nim b/codex/sales/states/provingsimulated.nim index 8722df6ce..78ce5ee58 100644 --- a/codex/sales/states/provingsimulated.nim +++ b/codex/sales/states/provingsimulated.nim @@ -6,7 +6,7 @@ when codex_enable_proof_failures: import pkg/ethers/testing import ../../contracts/requests - import ../../logging + import ../../logutils import ../../market import ../salescontext import ./proving diff --git a/codex/sales/states/unknown.nim b/codex/sales/states/unknown.nim index d68647741..db00f517d 100644 --- a/codex/sales/states/unknown.nim +++ b/codex/sales/states/unknown.nim @@ -1,4 +1,4 @@ -import ../../logging +import ../../logutils import ../statemachine import ../salesagent import ./filled diff --git a/codex/stores/cachestore.nim b/codex/stores/cachestore.nim index e57f22cc9..7a21ab8ac 100644 --- a/codex/stores/cachestore.nim +++ b/codex/stores/cachestore.nim @@ -23,7 +23,7 @@ import ./blockstore import ../units import ../chunker import ../errors -import ../logging +import ../logutils import ../manifest import ../merkletree import ../utils diff --git a/codex/stores/maintenance.nim b/codex/stores/maintenance.nim index 137809cd4..343fed8f7 100644 --- a/codex/stores/maintenance.nim +++ b/codex/stores/maintenance.nim @@ -18,7 +18,7 @@ import ./repostore import ../utils/timer import ../utils/asynciter import ../clock -import ../logging +import ../logutils import ../systemclock const diff --git a/codex/stores/networkstore.nim b/codex/stores/networkstore.nim index 8ceae42cd..2f0f1f87e 100644 --- a/codex/stores/networkstore.nim +++ b/codex/stores/networkstore.nim @@ -18,7 +18,7 @@ import pkg/questionable/results import ../clock import ../blocktype import ../blockexchange -import ../logging +import ../logutils import ../merkletree import ../utils/asyncheapqueue import ../utils/asynciter diff --git a/codex/stores/repostore.nim b/codex/stores/repostore.nim index f73572b2d..b07e148f8 100644 --- a/codex/stores/repostore.nim +++ b/codex/stores/repostore.nim @@ -26,7 +26,7 @@ import ./keyutils import ../blocktype import ../clock import ../systemclock -import ../logging +import ../logutils import ../merkletree import ../utils diff --git a/codex/streams/asyncstreamwrapper.nim b/codex/streams/asyncstreamwrapper.nim index 65d9a6d64..a8a559557 100644 --- a/codex/streams/asyncstreamwrapper.nim +++ b/codex/streams/asyncstreamwrapper.nim @@ -13,7 +13,7 @@ push: {.upraises: [].} import pkg/chronos import pkg/libp2p -import ../logging +import ../logutils logScope: topics = "libp2p asyncstreamwrapper" @@ -38,7 +38,7 @@ proc new*( writer: AsyncStreamWriter = nil ): AsyncStreamWrapper = ## Create new instance of an asynchronous stream wrapper - ## + ## let stream = C(reader: reader, writer: writer) diff --git a/codex/streams/seekablestream.nim b/codex/streams/seekablestream.nim index 0a587d01b..b07e11163 100644 --- a/codex/streams/seekablestream.nim +++ b/codex/streams/seekablestream.nim @@ -10,9 +10,9 @@ import pkg/libp2p/stream/lpstream import pkg/chronos -import ../logging +import ../logutils -export lpstream, chronos, logging +export lpstream, chronos, logutils logScope: topics = "codex seekablestream" diff --git a/codex/streams/storestream.nim b/codex/streams/storestream.nim index c2a4e3228..d3b8c0360 100644 --- a/codex/streams/storestream.nim +++ b/codex/streams/storestream.nim @@ -19,7 +19,7 @@ import pkg/stew/ptrops import ../stores import ../manifest import ../blocktype -import ../logging +import ../logutils import ../utils import ./seekablestream @@ -53,7 +53,7 @@ proc new*( pad = true ): StoreStream = ## Create a new StoreStream instance for a given store and manifest - ## + ## result = StoreStream( store: store, manifest: manifest, @@ -80,7 +80,7 @@ method readOnce*( ## Read `nbytes` from current position in the StoreStream into output buffer pointed by `pbytes`. ## Return how many bytes were actually read before EOF was encountered. ## Raise exception if we are already at EOF. - ## + ## trace "Reading from manifest", cid = self.manifest.cid.get(), blocks = self.manifest.blocksCount if self.atEof: diff --git a/codex/units.nim b/codex/units.nim index e8dad5d03..57b52ed66 100644 --- a/codex/units.nim +++ b/codex/units.nim @@ -6,14 +6,14 @@ ## at your option. ## This file may not be copied, modified, or distributed except according to ## those terms. -## +## import std/hashes import std/strutils import pkg/upraises -import ./logging +import ./logutils type NBytes* = distinct Natural @@ -42,7 +42,7 @@ divMaths(NBytes) proc `$`*(ts: NBytes): string = $(int(ts)) & "'NByte" proc `'nb`*(n: string): NBytes = parseInt(n).NBytes -logging.formatIt(NBytes): $it +logutils.formatIt(NBytes): $it const MiB = 1024.NBytes * 1024.NBytes # ByteSz, 1 mebibyte = 1,048,576 ByteSz diff --git a/codex/utils/asyncstatemachine.nim b/codex/utils/asyncstatemachine.nim index 0a3d0d3d3..9aeb3eab6 100644 --- a/codex/utils/asyncstatemachine.nim +++ b/codex/utils/asyncstatemachine.nim @@ -2,7 +2,7 @@ import std/sugar import pkg/questionable import pkg/chronos import pkg/upraises -import ../logging +import ../logutils import ./then import ./trackedfutures diff --git a/codex/utils/fileutils.nim b/codex/utils/fileutils.nim index ac5b6fe18..847ce06c0 100644 --- a/codex/utils/fileutils.nim +++ b/codex/utils/fileutils.nim @@ -16,10 +16,10 @@ push: {.upraises: [].} import std/strutils import pkg/stew/io2 -import ../logging +import ../logutils export io2 -export logging +export logutils when defined(windows): import stew/[windows/acl] diff --git a/codex/utils/json.nim b/codex/utils/json.nim index 58c6ba402..ed403acdf 100644 --- a/codex/utils/json.nim +++ b/codex/utils/json.nim @@ -12,7 +12,7 @@ import pkg/stew/byteutils import pkg/stint import pkg/questionable/results import ../errors -import ../logging +import ../logutils export json except `%`, `%*` diff --git a/codex/utils/keyutils.nim b/codex/utils/keyutils.nim index b3a5e453e..c7f762630 100644 --- a/codex/utils/keyutils.nim +++ b/codex/utils/keyutils.nim @@ -15,7 +15,7 @@ import pkg/libp2p/crypto/crypto import ./fileutils import ../errors -import ../logging +import ../logutils import ../rng export crypto @@ -39,6 +39,6 @@ proc setupKey*(path: string): ?!PrivateKey = warn "The network private key file is not safe, aborting" return failure newException( CodexKeyUnsafeError, "The network private key file is not safe") - + let kb = ? path.readAllBytes().mapFailure(CodexKeyError) return PrivateKey.init(kb).mapFailure(CodexKeyError) diff --git a/codex/utils/timer.nim b/codex/utils/timer.nim index f97b020e1..64180dc7d 100644 --- a/codex/utils/timer.nim +++ b/codex/utils/timer.nim @@ -13,7 +13,7 @@ import pkg/chronos import pkg/upraises -import ../logging +import ../logutils type TimerCallback* = proc(): Future[void] {.gcsafe, upraises:[].} diff --git a/codex/utils/trackedfutures.nim b/codex/utils/trackedfutures.nim index e99decd4a..f3fcdb2da 100644 --- a/codex/utils/trackedfutures.nim +++ b/codex/utils/trackedfutures.nim @@ -2,7 +2,7 @@ import std/sugar import std/tables import pkg/chronos -import ../logging +import ../logutils import ../utils/then type diff --git a/codex/validation.nim b/codex/validation.nim index a8e76eab6..b549997ef 100644 --- a/codex/validation.nim +++ b/codex/validation.nim @@ -3,7 +3,7 @@ import std/sequtils import pkg/chronos import ./market import ./clock -import ./logging +import ./logutils export market export sets diff --git a/tests/codex/sales/testslotqueue.nim b/tests/codex/sales/testslotqueue.nim index 4ae0abda2..18d1e4299 100644 --- a/tests/codex/sales/testslotqueue.nim +++ b/tests/codex/sales/testslotqueue.nim @@ -5,7 +5,7 @@ import pkg/datastore import pkg/questionable import pkg/questionable/results -import pkg/codex/logging +import pkg/codex/logutils import pkg/codex/sales/slotqueue import ../helpers diff --git a/tests/codex/testchunking.nim b/tests/codex/testchunking.nim index 49a96fd61..57babc799 100644 --- a/tests/codex/testchunking.nim +++ b/tests/codex/testchunking.nim @@ -2,7 +2,7 @@ import pkg/asynctest import pkg/stew/byteutils import pkg/codex/chunker -import pkg/codex/logging +import pkg/codex/logutils import pkg/chronos import ./helpers diff --git a/tests/codex/testlogging.nim b/tests/codex/testlogging.nim index 54d9fbf94..a81030f48 100644 --- a/tests/codex/testlogging.nim +++ b/tests/codex/testlogging.nim @@ -6,7 +6,7 @@ import std/unittest import pkg/codex/blocktype import pkg/codex/conf import pkg/codex/contracts/requests -import pkg/codex/logging +import pkg/codex/logutils import pkg/codex/purchasing/purchaseid import pkg/codex/units import pkg/codex/utils/json @@ -18,7 +18,7 @@ import ../checktest import ../examples import ./examples -export logging +export logutils logStream testlines[textlines[notimestamps,dynamic]] logStream testjson[json[notimestamps,dynamic]] @@ -32,9 +32,9 @@ type # must be defined at the top-level proc `$`*(t: ObjectType): string = "used `$`" -logging.formatIt(ObjectType): "formatted_" & it.a -logging.formatIt(RefType): "formatted_" & it.a -logging.formatIt(DistinctType): "formatted_" & it.a +logutils.formatIt(ObjectType): "formatted_" & it.a +logutils.formatIt(RefType): "formatted_" & it.a +logutils.formatIt(DistinctType): "formatted_" & it.a checksuite "Test logging outputLines": var outputLines: string @@ -109,7 +109,7 @@ checksuite "Test logging outputLines": test "logs Option types": let t = some ObjectType(a: "a") log t - check logged("t", "Some(formatted_a)") + check logged("t", "some(formatted_a)") check loggedJson("t", """{"val":"formatted_a","has":true}""") test "logs sequences of Option types": @@ -117,7 +117,7 @@ checksuite "Test logging outputLines": let t2 = none ObjectType let t = @[t1, t2] log t - check logged("t", "@[Some(formatted_a), None(ObjectType)]") + check logged("t", "@[some(formatted_a), none(ObjectType)]") check loggedJson("t", """[{"val":"formatted_a","has":true},{"val":"formatted_","has":false}]""") test "can define `$` override for T": diff --git a/tests/codex/testnode.nim b/tests/codex/testnode.nim index 6d1ee1491..7fb996fe5 100644 --- a/tests/codex/testnode.nim +++ b/tests/codex/testnode.nim @@ -17,7 +17,7 @@ import pkg/codexdht/discv5/protocol as discv5 import pkg/codex/stores import pkg/codex/clock import pkg/codex/contracts -import pkg/codex/logging +import pkg/codex/logutils import pkg/codex/systemclock import pkg/codex/blockexchange import pkg/codex/chunker diff --git a/tests/codex/utils/testjson.nim b/tests/codex/utils/testjson.nim index 324ee0b98..5ba7fff3a 100644 --- a/tests/codex/utils/testjson.nim +++ b/tests/codex/utils/testjson.nim @@ -7,7 +7,7 @@ import pkg/stew/byteutils import pkg/stint import pkg/codex/contracts/requests from pkg/codex/rest/json import RestPurchase -import pkg/codex/logging +import pkg/codex/logutils import pkg/codex/utils/json as utilsjson import pkg/questionable import pkg/questionable/results diff --git a/tests/integration/codexclient.nim b/tests/integration/codexclient.nim index 324d60429..7cb0b50e8 100644 --- a/tests/integration/codexclient.nim +++ b/tests/integration/codexclient.nim @@ -5,7 +5,7 @@ import std/sequtils from pkg/libp2p import Cid, `$`, init import pkg/stint import pkg/questionable/results -import pkg/codex/logging +import pkg/codex/logutils import pkg/codex/rest/json import pkg/codex/purchasing import pkg/codex/errors diff --git a/tests/integration/multinodes.nim b/tests/integration/multinodes.nim index 4167377ef..e7de7640a 100644 --- a/tests/integration/multinodes.nim +++ b/tests/integration/multinodes.nim @@ -2,7 +2,7 @@ import std/os import std/macros import std/json import std/httpclient -import pkg/codex/logging +import pkg/codex/logutils import ../ethertest import ./codexclient import ./nodes diff --git a/tests/integration/nodes.nim b/tests/integration/nodes.nim index b89c9ff62..a50f99744 100644 --- a/tests/integration/nodes.nim +++ b/tests/integration/nodes.nim @@ -3,7 +3,7 @@ import std/os import std/streams import std/strutils import pkg/codex/conf -import pkg/codex/logging +import pkg/codex/logutils import pkg/confutils import pkg/libp2p import pkg/questionable diff --git a/tests/integration/testproofs.nim b/tests/integration/testproofs.nim index c58963e32..0af0572ac 100644 --- a/tests/integration/testproofs.nim +++ b/tests/integration/testproofs.nim @@ -2,7 +2,7 @@ import std/sequtils import std/os from std/times import getTime, toUnix import pkg/codex/contracts -import pkg/codex/logging +import pkg/codex/logutils import pkg/codex/periods import ../contracts/time import ../contracts/deployment diff --git a/tests/logging.nim b/tests/logging.nim index f58c060cd..ece9c9b08 100644 --- a/tests/logging.nim +++ b/tests/logging.nim @@ -1,5 +1,5 @@ when not defined(nimscript): - import pkg/codex/logging + import pkg/codex/logutils proc ignoreLogging(level: LogLevel, message: LogOutputStr) = discard From 02058ce55244429d8bf41503a621e00525f4657f Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Wed, 13 Dec 2023 13:43:07 +1100 Subject: [PATCH 03/13] clean up --- codex/conf.nim | 2 +- codex/manifest/manifest.nim | 2 +- codex/node.nim | 1 + codex/purchasing/purchaseid.nim | 2 +- .../{testlogging.nim => testlogutils.nim} | 29 ++++++++++++++++++- tests/codex/testlogutils_helper.nim | 8 +++++ tests/testCodex.nim | 3 +- 7 files changed, 41 insertions(+), 6 deletions(-) rename tests/codex/{testlogging.nim => testlogutils.nim} (85%) create mode 100644 tests/codex/testlogutils_helper.nim diff --git a/codex/conf.nim b/codex/conf.nim index da4f0d866..303a5a2ce 100644 --- a/codex/conf.nim +++ b/codex/conf.nim @@ -276,7 +276,7 @@ type EthAddress* = ethers.Address -chronicles.formatIt(EthAddress): it.short0xHexLog +logutils.formatIt(EthAddress): it.short0xHexLog proc getCodexVersion(): string = let tag = strip(staticExec("git tag")) diff --git a/codex/manifest/manifest.nim b/codex/manifest/manifest.nim index 4e50d372e..f2bf319b9 100644 --- a/codex/manifest/manifest.nim +++ b/codex/manifest/manifest.nim @@ -23,7 +23,7 @@ import ../utils import ../utils/json import ../units import ../blocktype -import ../logging +import ../logutils import ./types export types diff --git a/codex/node.nim b/codex/node.nim index a4f40afa5..56893ca37 100644 --- a/codex/node.nim +++ b/codex/node.nim @@ -41,6 +41,7 @@ import ./errors import ./logutils export batch +export logutils logScope: topics = "codex node" diff --git a/codex/purchasing/purchaseid.nim b/codex/purchasing/purchaseid.nim index 7b762e27d..4b32f7136 100644 --- a/codex/purchasing/purchaseid.nim +++ b/codex/purchasing/purchaseid.nim @@ -4,7 +4,7 @@ import ../logutils type PurchaseId* = distinct array[32, byte] -chronicles.formatIt(PurchaseId): it.short0xHexLog +logutils.formatIt(PurchaseId): it.short0xHexLog proc hash*(x: PurchaseId): Hash {.borrow.} proc `==`*(x, y: PurchaseId): bool {.borrow.} diff --git a/tests/codex/testlogging.nim b/tests/codex/testlogutils.nim similarity index 85% rename from tests/codex/testlogging.nim rename to tests/codex/testlogutils.nim index a81030f48..43d94591d 100644 --- a/tests/codex/testlogging.nim +++ b/tests/codex/testlogutils.nim @@ -17,6 +17,7 @@ import pkg/stint import ../checktest import ../examples import ./examples +import ./testlogutils_helper export logutils @@ -36,7 +37,10 @@ logutils.formatIt(ObjectType): "formatted_" & it.a logutils.formatIt(RefType): "formatted_" & it.a logutils.formatIt(DistinctType): "formatted_" & it.a -checksuite "Test logging outputLines": +# must be defined at the top-level +logutils.formatIt(ObjectType2): "o2_formatted_" & it.a + +checksuite "Test logging output": var outputLines: string var outputJson: string @@ -58,6 +62,8 @@ checksuite "Test logging outputLines": template loggedJson(prop, expected): auto = let json = $ parseJson(outputJson){prop} + echo "json: ", json + echo "expected: ", expected json == expected template log(val) = @@ -161,6 +167,7 @@ checksuite "Test logging outputLines": let id = RequestId.fromHex("0x712003bdfc0db9abf21e7fbb7119cd52ff221c96714d21d39e782d7c744d3dea") log id check logged("id", "0x7120..3dea") + check loggedJson("id", "\"0x7120..3dea\"") test "logs seq[RequestId] correctly": let id = RequestId.fromHex("0x712003bdfc0db9abf21e7fbb7119cd52ff221c96714d21d39e782d7c744d3dea") @@ -174,19 +181,39 @@ checksuite "Test logging outputLines": let id = SlotId.fromHex("0x9ab2c4d102a95d990facb022d67b3c9b39052597c006fddf122bed2cb594c282") log id check logged("id", "0x9ab2..c282") + check loggedJson("id", "\"0x9ab2..c282\"") test "logs Nonce correctly": let id = SlotId.fromHex("ce88f368a7b776172ebd29a212456eb66acb60f169ee76eae91935e7fafad6ea") log id check logged("id", "0xce88..d6ea") + check loggedJson("id", "\"0xce88..d6ea\"") test "logs MultiAddress correctly": let ma = MultiAddress.init("/ip4/127.0.0.1/tcp/0").tryGet log ma check logged("ma", "/ip4/127.0.0.1/tcp/0") + check loggedJson("ma", "\"/ip4/127.0.0.1/tcp/0\"") test "logs seq[MultiAddress] correctly": let ma = @[MultiAddress.init("/ip4/127.0.0.1/tcp/0").tryGet, MultiAddress.init("/ip4/127.0.0.2/tcp/1").tryGet] log ma check logged("ma", "@[/ip4/127.0.0.1/tcp/0, /ip4/127.0.0.2/tcp/1]") + check loggedJson("ma", "[\"/ip4/127.0.0.1/tcp/0\",\"/ip4/127.0.0.2/tcp/1\"]") + + test "logs imported module correctly": + let o2 = ObjectType2(a: "1") + log o2 + check logged("o2", "o2_formatted_1") + check loggedJson("o2", "\"o2_formatted_1\"") + + test "logs sequence of imported module correctly": + let o21 = ObjectType2(a: "1") + let o22 = ObjectType2(a: "2") + let o2 = @[o21, o22] + log o2 + echo "outputLines: ", outputLines + echo "outputJson: ", outputJson + check logged("o2", "@[o2_formatted_1, o2_formatted_2]") + check loggedJson("o2", "[\"o2_formatted_1\",\"o2_formatted_2\"]") diff --git a/tests/codex/testlogutils_helper.nim b/tests/codex/testlogutils_helper.nim new file mode 100644 index 000000000..64ea474e4 --- /dev/null +++ b/tests/codex/testlogutils_helper.nim @@ -0,0 +1,8 @@ +import pkg/codex/logutils + +type + ObjectType2* = object + a*: string + +# must be defined at the top-level +logutils.formatIt(ObjectType2): "o2_formatted_" & it.a \ No newline at end of file diff --git a/tests/testCodex.nim b/tests/testCodex.nim index 8c1ba87df..745fb6fe8 100644 --- a/tests/testCodex.nim +++ b/tests/testCodex.nim @@ -2,9 +2,8 @@ import ./codex/teststores import ./codex/testblockexchange import ./codex/testasyncheapqueue import ./codex/testchunking -import ./codex/testlogging +import ./codex/testlogutils import ./codex/testmanifest -import ./codex/testlogging import ./codex/testnode import ./codex/teststorestream import ./codex/testpurchasing From b5f16c78f857b0ec8d7eb75854235cea18296e18 Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Wed, 13 Dec 2023 16:59:59 +1100 Subject: [PATCH 04/13] add setProperty for JsonRecord, remove nim-json-serialization conflict --- codex/contracts/requests.nim | 1 - codex/logutils.nim | 34 +++++++++++++++++++++-------- codex/utils/json.nim | 4 ---- tests/codex/testlogutils.nim | 33 +++++++--------------------- tests/codex/testlogutils_helper.nim | 8 ------- 5 files changed, 33 insertions(+), 47 deletions(-) delete mode 100644 tests/codex/testlogutils_helper.nim diff --git a/codex/contracts/requests.nim b/codex/contracts/requests.nim index ad4700e04..3dcd71a7a 100644 --- a/codex/contracts/requests.nim +++ b/codex/contracts/requests.nim @@ -6,7 +6,6 @@ import pkg/nimcrypto import pkg/ethers/fields import pkg/questionable/results import pkg/stew/byteutils -import pkg/json_serialization import pkg/upraises import ../logutils import ../utils/json diff --git a/codex/logutils.nim b/codex/logutils.nim index f1e660651..16e0f29a4 100644 --- a/codex/logutils.nim +++ b/codex/logutils.nim @@ -4,7 +4,7 @@ import std/strutils import std/sugar import std/typetraits -import pkg/chronicles +import pkg/chronicles except toJson import pkg/faststreams from pkg/libp2p import Cid, MultiAddress, `$` import pkg/questionable @@ -12,6 +12,8 @@ import pkg/stew/byteutils import pkg/stint import pkg/upraises +import ./utils/json + export byteutils export chronicles except toJson, formatIt export questionable @@ -49,34 +51,48 @@ func short0xHexLog*[U: distinct, T: seq[U]](v: T): string = type BaseType = U.distinctBase "@[" & v.map(x => BaseType(x).short0xHexLog).join(",") & "]" -proc formatSeq*(val: seq[string]): string = +proc formatTextLineSeq*(val: seq[string]): string = "@[" & val.join(", ") & "]" -template formatOption*(val, T, body): auto = +template formatTextLineOption*(val, T, body): auto = var v = "none(" & $T & ")" if it =? val: v = "some(" & body & ")" # that I used to know :) v +template formatJsonOption*(val, T, body): auto = + var v = none string + if it =? val: + v = some body # that I used to know :) + v + template formatIt*(T: type, body: untyped) {.dirty.} = - proc writeValue*(writer: var JsonWriter, it: T) {.upraises:[IOError].} = - let formatted = body - writer.writeValue(formatted) + proc setProperty*(r: var JsonRecord, key: string, val: ?T) = + let v = val.formatJsonOption(T, body) + setProperty(r, key, %v) + + proc setProperty*(r: var JsonRecord, key: string, val: seq[?T]) = + let v = val.map(it => it.formatJsonOption(T, body)) + setProperty(r, key, %v) + + proc setProperty*(r: var JsonRecord, key: string, val: seq[T]) = + let v = val.map(it => body) + setProperty(r, key, %v) proc setProperty*(r: var JsonRecord, key: string, it: T) {.upraises:[IOError].} = let v = body setProperty(r, key, v) proc setProperty*(r: var TextLineRecord, key: string, val: ?T) = - setProperty(r, key, val.formatOption(T, body)) + setProperty(r, key, val.formatTextLineOption(T, body)) proc setProperty*(r: var TextLineRecord, key: string, val: seq[?T]) = - let v = val.map(item => item.formatOption(T, body)).formatSeq + let v = val.map(item => item.formatTextLineOption(T, body)).formatTextLineSeq setProperty(r, key, v) proc setProperty*(r: var TextLineRecord, key: string, val: seq[T]) = - let v = val.map(it => body).formatSeq + let v = val.map(it => body).formatTextLineSeq setProperty(r, key, v) proc setProperty*(r: var TextLineRecord, key: string, it: T) {.upraises:[ValueError].} = diff --git a/codex/utils/json.nim b/codex/utils/json.nim index ed403acdf..635205899 100644 --- a/codex/utils/json.nim +++ b/codex/utils/json.nim @@ -12,13 +12,9 @@ import pkg/stew/byteutils import pkg/stint import pkg/questionable/results import ../errors -import ../logutils export json except `%`, `%*` -logScope: - topics = "json serialization" - type SerializationError = object of CodexError UnexpectedKindError = object of SerializationError diff --git a/tests/codex/testlogutils.nim b/tests/codex/testlogutils.nim index 43d94591d..242afc48f 100644 --- a/tests/codex/testlogutils.nim +++ b/tests/codex/testlogutils.nim @@ -1,4 +1,3 @@ -import std/json import std/options import std/nre import std/strutils @@ -17,7 +16,6 @@ import pkg/stint import ../checktest import ../examples import ./examples -import ./testlogutils_helper export logutils @@ -37,9 +35,6 @@ logutils.formatIt(ObjectType): "formatted_" & it.a logutils.formatIt(RefType): "formatted_" & it.a logutils.formatIt(DistinctType): "formatted_" & it.a -# must be defined at the top-level -logutils.formatIt(ObjectType2): "o2_formatted_" & it.a - checksuite "Test logging output": var outputLines: string var outputJson: string @@ -62,8 +57,6 @@ checksuite "Test logging output": template loggedJson(prop, expected): auto = let json = $ parseJson(outputJson){prop} - echo "json: ", json - echo "expected: ", expected json == expected template log(val) = @@ -116,7 +109,7 @@ checksuite "Test logging output": let t = some ObjectType(a: "a") log t check logged("t", "some(formatted_a)") - check loggedJson("t", """{"val":"formatted_a","has":true}""") + check loggedJson("t", "\"formatted_a\"") test "logs sequences of Option types": let t1 = some ObjectType(a: "a") @@ -124,7 +117,7 @@ checksuite "Test logging output": let t = @[t1, t2] log t check logged("t", "@[some(formatted_a), none(ObjectType)]") - check loggedJson("t", """[{"val":"formatted_a","has":true},{"val":"formatted_","has":false}]""") + check loggedJson("t", """["formatted_a",null]""") test "can define `$` override for T": let o = ObjectType() @@ -153,6 +146,12 @@ checksuite "Test logging output": log stint check logged("stint", "12345678901234") + test "logs int correctly": + let int = 123 + log int + check logged("int", "123") + check loggedJson("int", "123") + test "logs EthAddress correctly": let address = EthAddress.fromHex("0xf75e076f650cd51dbfa0fd9c465d5037f22e1b1b") log address @@ -201,19 +200,3 @@ checksuite "Test logging output": log ma check logged("ma", "@[/ip4/127.0.0.1/tcp/0, /ip4/127.0.0.2/tcp/1]") check loggedJson("ma", "[\"/ip4/127.0.0.1/tcp/0\",\"/ip4/127.0.0.2/tcp/1\"]") - - test "logs imported module correctly": - let o2 = ObjectType2(a: "1") - log o2 - check logged("o2", "o2_formatted_1") - check loggedJson("o2", "\"o2_formatted_1\"") - - test "logs sequence of imported module correctly": - let o21 = ObjectType2(a: "1") - let o22 = ObjectType2(a: "2") - let o2 = @[o21, o22] - log o2 - echo "outputLines: ", outputLines - echo "outputJson: ", outputJson - check logged("o2", "@[o2_formatted_1, o2_formatted_2]") - check loggedJson("o2", "[\"o2_formatted_1\",\"o2_formatted_2\"]") diff --git a/tests/codex/testlogutils_helper.nim b/tests/codex/testlogutils_helper.nim deleted file mode 100644 index 64ea474e4..000000000 --- a/tests/codex/testlogutils_helper.nim +++ /dev/null @@ -1,8 +0,0 @@ -import pkg/codex/logutils - -type - ObjectType2* = object - a*: string - -# must be defined at the top-level -logutils.formatIt(ObjectType2): "o2_formatted_" & it.a \ No newline at end of file From b54cd49d678f57057cf3eebc7c02421a15b22719 Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Thu, 14 Dec 2023 15:26:14 +1100 Subject: [PATCH 05/13] Allow specifying textlines and json format separately Not specifying a LogFormat will apply the formatting to both textlines and json sinks. Specifying a LogFormat will apply the formatting to only that sink. --- codex/blocktype.nim | 12 +++-- codex/conf.nim | 3 +- codex/contracts/requests.nim | 9 ++-- codex/logutils.nim | 93 +++++++++++++++++++-------------- codex/purchasing/purchaseid.nim | 3 +- codex/sales/reservations.nim | 3 +- codex/units.nim | 3 +- codex/utils/json.nim | 7 ++- tests/codex/testlogutils.nim | 40 +++++++++----- 9 files changed, 109 insertions(+), 64 deletions(-) diff --git a/codex/blocktype.nim b/codex/blocktype.nim index edf0f9ae7..786320c21 100644 --- a/codex/blocktype.nim +++ b/codex/blocktype.nim @@ -19,12 +19,12 @@ import pkg/libp2p/[cid, multicodec, multihash] import pkg/stew/byteutils import pkg/questionable import pkg/questionable/results -import pkg/json_serialization import ./units import ./utils import ./errors import ./logutils +import ./utils/json export errors, logutils, units @@ -41,17 +41,19 @@ type BlockAddress* = object case leaf*: bool of true: - treeCid*: Cid - index*: Natural + treeCid* {.serialize.}: Cid + index* {.serialize.}: Natural else: - cid*: Cid + cid* {.serialize.}: Cid -logutils.formatIt(BlockAddress): +logutils.formatIt(LogFormat.textLines, BlockAddress): if it.leaf: "treeCid: " & shortLog($it.treeCid) & ", index: " & $it.index else: "cid: " & shortLog($it.cid) +logutils.formatIt(LogFormat.json, BlockAddress): %it + proc `==`*(a, b: BlockAddress): bool = a.leaf == b.leaf and ( diff --git a/codex/conf.nim b/codex/conf.nim index 303a5a2ce..2c8744dc6 100644 --- a/codex/conf.nim +++ b/codex/conf.nim @@ -276,7 +276,8 @@ type EthAddress* = ethers.Address -logutils.formatIt(EthAddress): it.short0xHexLog +logutils.formatIt(LogFormat.textLines, EthAddress): it.short0xHexLog +logutils.formatIt(LogFormat.json, EthAddress): %it proc getCodexVersion(): string = let tag = strip(staticExec("git tag")) diff --git a/codex/contracts/requests.nim b/codex/contracts/requests.nim index 3dcd71a7a..1363fb9de 100644 --- a/codex/contracts/requests.nim +++ b/codex/contracts/requests.nim @@ -80,9 +80,12 @@ proc toHex*[T: distinct](id: T): string = type baseType = T.distinctBase baseType(id).toHex -logutils.formatIt(Nonce): it.short0xHexLog -logutils.formatIt(RequestId): it.short0xHexLog -logutils.formatIt(SlotId): it.short0xHexLog +logutils.formatIt(LogFormat.textLines, Nonce): it.short0xHexLog +logutils.formatIt(LogFormat.textLines, RequestId): it.short0xHexLog +logutils.formatIt(LogFormat.textLines, SlotId): it.short0xHexLog +logutils.formatIt(LogFormat.json, Nonce): it.to0xHexLog +logutils.formatIt(LogFormat.json, RequestId): it.to0xHexLog +logutils.formatIt(LogFormat.json, SlotId): it.to0xHexLog func fromTuple(_: type StorageRequest, tupl: tuple): StorageRequest = StorageRequest( diff --git a/codex/logutils.nim b/codex/logutils.nim index 16e0f29a4..a1a50db88 100644 --- a/codex/logutils.nim +++ b/codex/logutils.nim @@ -4,8 +4,8 @@ import std/strutils import std/sugar import std/typetraits -import pkg/chronicles except toJson -import pkg/faststreams +import pkg/chronicles except toJson, `%` +# import pkg/faststreams from pkg/libp2p import Cid, MultiAddress, `$` import pkg/questionable import pkg/stew/byteutils @@ -15,12 +15,13 @@ import pkg/upraises import ./utils/json export byteutils -export chronicles except toJson, formatIt +export chronicles except toJson, formatIt, `%` export questionable export sequtils export strutils export sugar export upraises +export json func shortLog*(long: string, ellipses = "*", start = 3, stop = 6): string = ## Returns compact string representation of ``long``. @@ -51,54 +52,70 @@ func short0xHexLog*[U: distinct, T: seq[U]](v: T): string = type BaseType = U.distinctBase "@[" & v.map(x => BaseType(x).short0xHexLog).join(",") & "]" +func to0xHexLog*[T: distinct](v: T): string = + type BaseType = T.distinctBase + BaseType(v).to0xHex + +func to0xHexLog*[U: distinct, T: seq[U]](v: T): string = + type BaseType = U.distinctBase + "@[" & v.map(x => BaseType(x).to0xHex).join(",") & "]" + proc formatTextLineSeq*(val: seq[string]): string = "@[" & val.join(", ") & "]" -template formatTextLineOption*(val, T, body): auto = - var v = "none(" & $T & ")" - if it =? val: - v = "some(" & body & ")" # that I used to know :) - v +template formatIt*(format: LogFormat, T: type, body: untyped) {.dirty.} = -template formatJsonOption*(val, T, body): auto = - var v = none string - if it =? val: - v = some body # that I used to know :) - v + when format == LogFormat.json: + proc formatJsonOption(val: ?T): JsonNode = + if it =? val: + json.`%`(body) + else: + newJNull() -template formatIt*(T: type, body: untyped) {.dirty.} = + proc setProperty*(r: var JsonRecord, key: string, opt: ?T) = + let v = opt.formatJsonOption + setProperty(r, key, v) + + proc setProperty*(r: var JsonRecord, key: string, opts: seq[?T]) = + let v = opts.map(opt => opt.formatJsonOption) + setProperty(r, key, json.`%`(v)) - proc setProperty*(r: var JsonRecord, key: string, val: ?T) = - let v = val.formatJsonOption(T, body) - setProperty(r, key, %v) + proc setProperty*(r: var JsonRecord, key: string, val: seq[T]) = + let v = val.map(it => body) + setProperty(r, key, json.`%`(v)) - proc setProperty*(r: var JsonRecord, key: string, val: seq[?T]) = - let v = val.map(it => it.formatJsonOption(T, body)) - setProperty(r, key, %v) + proc setProperty*(r: var JsonRecord, key: string, it: T) {.upraises:[ValueError, IOError].} = + let v = body + setProperty(r, key, json.`%`(v)) - proc setProperty*(r: var JsonRecord, key: string, val: seq[T]) = - let v = val.map(it => body) - setProperty(r, key, %v) + elif format == LogFormat.textLines: + proc formatTextLineOption*(val: ?T): string = + var v = "none(" & $T & ")" + if it =? val: + v = "some(" & $(body) & ")" # that I used to know :) + v - proc setProperty*(r: var JsonRecord, key: string, it: T) {.upraises:[IOError].} = - let v = body - setProperty(r, key, v) + proc setProperty*(r: var TextLineRecord, key: string, opt: ?T) = + let v = opt.formatTextLineOption + setProperty(r, key, v) - proc setProperty*(r: var TextLineRecord, key: string, val: ?T) = - setProperty(r, key, val.formatTextLineOption(T, body)) + proc setProperty*(r: var TextLineRecord, key: string, opts: seq[?T]) = + let v = opts.map(opt => opt.formatTextLineOption) + setProperty(r, key, v.formatTextLineSeq) - proc setProperty*(r: var TextLineRecord, key: string, val: seq[?T]) = - let v = val.map(item => item.formatTextLineOption(T, body)).formatTextLineSeq - setProperty(r, key, v) + proc setProperty*(r: var TextLineRecord, key: string, val: seq[T]) = + let v = val.map(it => body) + setProperty(r, key, v.formatTextLineSeq) - proc setProperty*(r: var TextLineRecord, key: string, val: seq[T]) = - let v = val.map(it => body).formatTextLineSeq - setProperty(r, key, v) + proc setProperty*(r: var TextLineRecord, key: string, it: T) {.upraises:[ValueError, IOError].} = + let v = body + setProperty(r, key, v) - proc setProperty*(r: var TextLineRecord, key: string, it: T) {.upraises:[ValueError].} = - let v = body - setProperty(r, key, v) +template formatIt*(T: type, body: untyped) {.dirty.} = + formatIt(LogFormat.textLines, T): body + formatIt(LogFormat.json, T): body -formatIt(Cid): shortLog($it) +formatIt(LogFormat.textLines, Cid): shortLog($it) +formatIt(LogFormat.json, Cid): $it formatIt(UInt256): $it formatIt(MultiAddress): $it diff --git a/codex/purchasing/purchaseid.nim b/codex/purchasing/purchaseid.nim index 4b32f7136..226fcbee5 100644 --- a/codex/purchasing/purchaseid.nim +++ b/codex/purchasing/purchaseid.nim @@ -4,7 +4,8 @@ import ../logutils type PurchaseId* = distinct array[32, byte] -logutils.formatIt(PurchaseId): it.short0xHexLog +logutils.formatIt(LogFormat.textLines, PurchaseId): it.short0xHexLog +logutils.formatIt(LogFormat.json, PurchaseId): it.to0xHexLog proc hash*(x: PurchaseId): Hash {.borrow.} proc `==`*(x, y: PurchaseId): bool {.borrow.} diff --git a/codex/sales/reservations.nim b/codex/sales/reservations.nim index 075dab7ab..6c68853ec 100644 --- a/codex/sales/reservations.nim +++ b/codex/sales/reservations.nim @@ -139,7 +139,8 @@ proc toErr[E1: ref CatchableError, E2: ReservationsError]( return newException(E2, msg, e1) -logutils.formatIt(SomeStorableId): it.toJson +logutils.formatIt(LogFormat.textLines, SomeStorableId): it.short0xHexLog +logutils.formatIt(LogFormat.json, SomeStorableId): it.to0xHexLog proc `onAvailabilityAdded=`*(self: Reservations, onAvailabilityAdded: OnAvailabilityAdded) = diff --git a/codex/units.nim b/codex/units.nim index 57b52ed66..d254712da 100644 --- a/codex/units.nim +++ b/codex/units.nim @@ -42,7 +42,8 @@ divMaths(NBytes) proc `$`*(ts: NBytes): string = $(int(ts)) & "'NByte" proc `'nb`*(n: string): NBytes = parseInt(n).NBytes -logutils.formatIt(NBytes): $it +logutils.formatIt(LogFormat.textLines, NBytes): $it +logutils.formatIt(LogFormat.json, NBytes): $it const MiB = 1024.NBytes * 1024.NBytes # ByteSz, 1 mebibyte = 1,048,576 ByteSz diff --git a/codex/utils/json.nim b/codex/utils/json.nim index 635205899..9a6f4cc64 100644 --- a/codex/utils/json.nim +++ b/codex/utils/json.nim @@ -6,7 +6,8 @@ import std/strutils import std/strformat import std/tables import std/typetraits -from pkg/libp2p import Cid, init +from pkg/ethers import Address +from pkg/libp2p import Cid, init, `$` import pkg/contractabi import pkg/stew/byteutils import pkg/stint @@ -299,6 +300,10 @@ func `%`*[T: distinct](id: T): JsonNode = type baseType = T.distinctBase % baseType(id) +func `%`*(cid: Cid): JsonNode = % $cid + +func `%`*(address: Address): JsonNode = % $address + func toJson*[T](item: T): string = $(%item) proc toJsnImpl(x: NimNode): NimNode = diff --git a/tests/codex/testlogutils.nim b/tests/codex/testlogutils.nim index 242afc48f..faf022653 100644 --- a/tests/codex/testlogutils.nim +++ b/tests/codex/testlogutils.nim @@ -8,14 +8,12 @@ import pkg/codex/contracts/requests import pkg/codex/logutils import pkg/codex/purchasing/purchaseid import pkg/codex/units -import pkg/codex/utils/json +# import pkg/codex/utils/json as jsonutils import pkg/libp2p/cid import pkg/libp2p/multiaddress import pkg/stew/byteutils import pkg/stint import ../checktest -import ../examples -import ./examples export logutils @@ -28,12 +26,19 @@ type DistinctType {.borrow: `.`.} = distinct ObjectType RefType = ref object a: string + AnotherType = object + a: int # must be defined at the top-level proc `$`*(t: ObjectType): string = "used `$`" -logutils.formatIt(ObjectType): "formatted_" & it.a -logutils.formatIt(RefType): "formatted_" & it.a -logutils.formatIt(DistinctType): "formatted_" & it.a +func `%`*(t: RefType): JsonNode = % t.a +logutils.formatIt(LogFormat.textLines, ObjectType): "formatted_" & it.a +logutils.formatIt(LogFormat.textLines, RefType): "formatted_" & it.a +logutils.formatIt(LogFormat.textLines, DistinctType): "formatted_" & it.a +logutils.formatIt(LogFormat.json, ObjectType): "formatted_" & it.a +logutils.formatIt(LogFormat.json, RefType): %it +logutils.formatIt(LogFormat.json, DistinctType): "formatted_" & it.a +logutils.formatIt(AnotherType): it.a checksuite "Test logging output": var outputLines: string @@ -81,7 +86,7 @@ checksuite "Test logging output": let t = RefType(a: "a") log t check logged("t", "formatted_a") - check loggedJson("t", "\"formatted_a\"") + check loggedJson("t", "\"a\"") test "logs sequences of ref types": let t1 = RefType(a: "a") @@ -89,7 +94,7 @@ checksuite "Test logging output": let t = @[t1, t2] log t check logged("t", "@[formatted_a, formatted_b]") - check loggedJson("t", "[\"formatted_a\",\"formatted_b\"]") + check loggedJson("t", "[\"a\",\"b\"]") test "logs distinct types": let t = DistinctType(ObjectType(a: "a")) @@ -105,6 +110,12 @@ checksuite "Test logging output": check logged("t", "@[formatted_a, formatted_b]") check loggedJson("t", "[\"formatted_a\",\"formatted_b\"]") + test "formatIt can return non-string types": + let t = AnotherType(a: 1) + log t + check logged("t", "1") + check loggedJson("t", "1") + test "logs Option types": let t = some ObjectType(a: "a") log t @@ -134,17 +145,19 @@ checksuite "Test logging output": let ba = BlockAddress.init(cid, 0) log ba check logged("ba", "treeCid: zb2*fndjU1, index: 0") - check loggedJson("ba", "\"treeCid: zb2*fndjU1, index: 0\"") + check loggedJson("ba", """{"treeCid":"zb2rhgsDE16rLtbwTFeNKbdSobtKiWdjJPvKEuPgrQAfndjU1","index":0}""") test "logs Cid correctly": let cid = Cid.init("zb2rhmfWaXASbyi15iLqbz5yp3awnSyecpt9jcFnc2YA5TgiD").tryGet log cid check logged("cid", "zb2*A5TgiD") + check loggedJson("cid", "\"zb2rhmfWaXASbyi15iLqbz5yp3awnSyecpt9jcFnc2YA5TgiD\"") test "logs StUint correctly": let stint = 12345678901234.u256 log stint check logged("stint", "12345678901234") + check loggedJson("stint", "\"12345678901234\"") test "logs int correctly": let int = 123 @@ -156,6 +169,7 @@ checksuite "Test logging output": let address = EthAddress.fromHex("0xf75e076f650cd51dbfa0fd9c465d5037f22e1b1b") log address check logged("address", "0xf75e..1b1b") + check loggedJson("address", "\"0xf75e076f650cd51dbfa0fd9c465d5037f22e1b1b\"") test "logs PurchaseId correctly": let id = PurchaseId.fromHex("0x712003bdfc0db9abf21e7fbb7119cd52ff221c96714d21d39e782d7c744d3dea") @@ -166,7 +180,7 @@ checksuite "Test logging output": let id = RequestId.fromHex("0x712003bdfc0db9abf21e7fbb7119cd52ff221c96714d21d39e782d7c744d3dea") log id check logged("id", "0x7120..3dea") - check loggedJson("id", "\"0x7120..3dea\"") + check loggedJson("id", "\"0x712003bdfc0db9abf21e7fbb7119cd52ff221c96714d21d39e782d7c744d3dea\"") test "logs seq[RequestId] correctly": let id = RequestId.fromHex("0x712003bdfc0db9abf21e7fbb7119cd52ff221c96714d21d39e782d7c744d3dea") @@ -174,19 +188,19 @@ checksuite "Test logging output": let ids = @[id, id2] log ids check logged("ids", "@[0x7120..3dea, 0x9ab2..c282]") - check loggedJson("ids", """["0x7120..3dea","0x9ab2..c282"]""") + check loggedJson("ids", """["0x712003bdfc0db9abf21e7fbb7119cd52ff221c96714d21d39e782d7c744d3dea","0x9ab2c4d102a95d990facb022d67b3c9b39052597c006fddf122bed2cb594c282"]""") test "logs SlotId correctly": let id = SlotId.fromHex("0x9ab2c4d102a95d990facb022d67b3c9b39052597c006fddf122bed2cb594c282") log id check logged("id", "0x9ab2..c282") - check loggedJson("id", "\"0x9ab2..c282\"") + check loggedJson("id", "\"0x9ab2c4d102a95d990facb022d67b3c9b39052597c006fddf122bed2cb594c282\"") test "logs Nonce correctly": let id = SlotId.fromHex("ce88f368a7b776172ebd29a212456eb66acb60f169ee76eae91935e7fafad6ea") log id check logged("id", "0xce88..d6ea") - check loggedJson("id", "\"0xce88..d6ea\"") + check loggedJson("id", "\"0xce88f368a7b776172ebd29a212456eb66acb60f169ee76eae91935e7fafad6ea\"") test "logs MultiAddress correctly": let ma = MultiAddress.init("/ip4/127.0.0.1/tcp/0").tryGet From 9b30f4330f37be991c7533da59035b523e908a47 Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Thu, 14 Dec 2023 15:26:54 +1100 Subject: [PATCH 06/13] remove unneeded usages of std/json We only need to import utils/json instead of std/json --- codex/contracts/deployment.nim | 1 - tests/contracts/testContracts.nim | 1 - tests/ethertest.nim | 2 +- tests/integration/multinodes.nim | 1 - tests/integration/twonodes.nim | 1 - 5 files changed, 1 insertion(+), 5 deletions(-) diff --git a/codex/contracts/deployment.nim b/codex/contracts/deployment.nim index 01bf5f813..46591100f 100644 --- a/codex/contracts/deployment.nim +++ b/codex/contracts/deployment.nim @@ -1,4 +1,3 @@ -import std/json import std/os import std/tables import pkg/ethers diff --git a/tests/contracts/testContracts.nim b/tests/contracts/testContracts.nim index 22aaeba03..4864f9926 100644 --- a/tests/contracts/testContracts.nim +++ b/tests/contracts/testContracts.nim @@ -1,4 +1,3 @@ -import std/json import pkg/chronos import pkg/ethers/testing import pkg/ethers/erc20 diff --git a/tests/ethertest.nim b/tests/ethertest.nim index 1166b0e02..5c6203106 100644 --- a/tests/ethertest.nim +++ b/tests/ethertest.nim @@ -1,4 +1,4 @@ -import std/json + import pkg/asynctest import pkg/ethers diff --git a/tests/integration/multinodes.nim b/tests/integration/multinodes.nim index e7de7640a..6c9a8bf8d 100644 --- a/tests/integration/multinodes.nim +++ b/tests/integration/multinodes.nim @@ -1,6 +1,5 @@ import std/os import std/macros -import std/json import std/httpclient import pkg/codex/logutils import ../ethertest diff --git a/tests/integration/twonodes.nim b/tests/integration/twonodes.nim index b09551e1e..fb99ed3f9 100644 --- a/tests/integration/twonodes.nim +++ b/tests/integration/twonodes.nim @@ -1,6 +1,5 @@ import std/os import std/macros -import std/json import std/httpclient import ../ethertest import ./codexclient From 8e67628c2644ddd6176a747ae8578f1b4106fd0d Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Thu, 14 Dec 2023 15:27:24 +1100 Subject: [PATCH 07/13] move serialization from rest/json to utils/json so it can be shared --- codex/rest/json.nim | 21 +-------------------- codex/utils/json.nim | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 22 deletions(-) diff --git a/codex/rest/json.nim b/codex/rest/json.nim index 078a01243..181ea584a 100644 --- a/codex/rest/json.nim +++ b/codex/rest/json.nim @@ -111,23 +111,4 @@ func `%`*(obj: StorageRequest | Slot): JsonNode = return jsonObj -func `%`*(obj: Cid): JsonNode = - % $obj - -func `%`*(obj: PeerId): JsonNode = - % $obj - -func `%`*(obj: RestNodeId): JsonNode = - % $obj.id - -func `%`*(obj: SignedPeerRecord): JsonNode = - % $obj - -func `%`*(obj: dn.Address): JsonNode = - % $obj - -func `%`*(obj: AddressInfo): JsonNode = - % $obj.address - -func `%`*(obj: MultiAddress): JsonNode = - % $obj +func `%`*(obj: RestNodeId): JsonNode = % $obj.id diff --git a/codex/utils/json.nim b/codex/utils/json.nim index 9a6f4cc64..080d4e89d 100644 --- a/codex/utils/json.nim +++ b/codex/utils/json.nim @@ -7,8 +7,9 @@ import std/strformat import std/tables import std/typetraits from pkg/ethers import Address -from pkg/libp2p import Cid, init, `$` +from pkg/libp2p import Cid, PeerId, SignedPeerRecord, MultiAddress, AddressInfo, init, `$` import pkg/contractabi +import pkg/codexdht/discv5/node as dn import pkg/stew/byteutils import pkg/stint import pkg/questionable/results @@ -302,7 +303,17 @@ func `%`*[T: distinct](id: T): JsonNode = func `%`*(cid: Cid): JsonNode = % $cid -func `%`*(address: Address): JsonNode = % $address +func `%`*(obj: PeerId): JsonNode = % $obj + +func `%`*(obj: SignedPeerRecord): JsonNode = % $obj + +func `%`*(obj: dn.Address): JsonNode = % $obj + +func `%`*(obj: AddressInfo): JsonNode = % $obj.address + +func `%`*(obj: MultiAddress): JsonNode = % $obj + +func `%`*(address: ethers.Address): JsonNode = % $address func toJson*[T](item: T): string = $(%item) From 742d3b328687b06a4d985d1c7dd0d8c6a2872b61 Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Thu, 14 Dec 2023 16:39:24 +1100 Subject: [PATCH 08/13] fix NoColors ambiguity Was causing unit tests to fail on Windows. --- codex/conf.nim | 2 +- tests/codex/testlogutils.nim | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/codex/conf.nim b/codex/conf.nim index 2c8744dc6..2ae404589 100644 --- a/codex/conf.nim +++ b/codex/conf.nim @@ -53,7 +53,7 @@ type noCommand, initNode - LogKind* = enum + LogKind* {.pure.} = enum Auto = "auto" Colors = "colors" NoColors = "nocolors" diff --git a/tests/codex/testlogutils.nim b/tests/codex/testlogutils.nim index faf022653..ce0a912c5 100644 --- a/tests/codex/testlogutils.nim +++ b/tests/codex/testlogutils.nim @@ -17,8 +17,8 @@ import ../checktest export logutils -logStream testlines[textlines[notimestamps,dynamic]] -logStream testjson[json[notimestamps,dynamic]] +logStream testlines[textlines[nocolors,notimestamps,dynamic]] +logStream testjson[json[nocolors,notimestamps,dynamic]] type ObjectType = object @@ -45,10 +45,10 @@ checksuite "Test logging output": var outputJson: string proc writeToLines(logLevel: LogLevel, msg: LogOutputStr) = - outputLines &= stripAnsi(msg) # nocolors in the stream definition doesn't seem to work + outputLines &= msg proc writeToJson(logLevel: LogLevel, msg: LogOutputStr) = - outputJson &= stripAnsi(msg) + outputJson &= msg setup: outputLines = "" From 992084b7058ba345c335bab7791401b81d7b7cce Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Thu, 14 Dec 2023 17:19:45 +1100 Subject: [PATCH 09/13] Remove nre usage to fix Windows error Windows was erroring with `could not load: pcre64.dll`. Instead of fixing that error, remove the pcre usage :) --- tests/codex/testlogutils.nim | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/tests/codex/testlogutils.nim b/tests/codex/testlogutils.nim index ce0a912c5..76923df64 100644 --- a/tests/codex/testlogutils.nim +++ b/tests/codex/testlogutils.nim @@ -1,5 +1,4 @@ import std/options -import std/nre import std/strutils import std/unittest import pkg/codex/blocktype @@ -8,7 +7,6 @@ import pkg/codex/contracts/requests import pkg/codex/logutils import pkg/codex/purchasing/purchaseid import pkg/codex/units -# import pkg/codex/utils/json as jsonutils import pkg/libp2p/cid import pkg/libp2p/multiaddress import pkg/stew/byteutils @@ -57,8 +55,8 @@ checksuite "Test logging output": testjson.outputs[0].writer = writeToJson template logged(prop, expected): auto = - let regex = prop & """\=\"?""" & expected.escapeRe & """\"?""" - outputLines.contains(re(regex)) + let toFind = prop & "=" & expected + outputLines.contains(toFind) template loggedJson(prop, expected): auto = let json = $ parseJson(outputJson){prop} @@ -79,7 +77,7 @@ checksuite "Test logging output": let t2 = ObjectType(a: "b") let t = @[t1, t2] log t - check logged("t", "@[formatted_a, formatted_b]") + check logged("t", "\"@[formatted_a, formatted_b]\"") check loggedJson("t", "[\"formatted_a\",\"formatted_b\"]") test "logs ref types": @@ -93,7 +91,7 @@ checksuite "Test logging output": let t2 = RefType(a: "b") let t = @[t1, t2] log t - check logged("t", "@[formatted_a, formatted_b]") + check logged("t", "\"@[formatted_a, formatted_b]\"") check loggedJson("t", "[\"a\",\"b\"]") test "logs distinct types": @@ -107,7 +105,7 @@ checksuite "Test logging output": let t2 = DistinctType(ObjectType(a: "b")) let t = @[t1, t2] log t - check logged("t", "@[formatted_a, formatted_b]") + check logged("t", "\"@[formatted_a, formatted_b]\"") check loggedJson("t", "[\"formatted_a\",\"formatted_b\"]") test "formatIt can return non-string types": @@ -127,7 +125,7 @@ checksuite "Test logging output": let t2 = none ObjectType let t = @[t1, t2] log t - check logged("t", "@[some(formatted_a), none(ObjectType)]") + check logged("t", "\"@[some(formatted_a), none(ObjectType)]\"") check loggedJson("t", """["formatted_a",null]""") test "can define `$` override for T": @@ -144,7 +142,7 @@ checksuite "Test logging output": let cid = Cid.init("zb2rhgsDE16rLtbwTFeNKbdSobtKiWdjJPvKEuPgrQAfndjU1").tryGet let ba = BlockAddress.init(cid, 0) log ba - check logged("ba", "treeCid: zb2*fndjU1, index: 0") + check logged("ba", "\"treeCid: zb2*fndjU1, index: 0\"") check loggedJson("ba", """{"treeCid":"zb2rhgsDE16rLtbwTFeNKbdSobtKiWdjJPvKEuPgrQAfndjU1","index":0}""") test "logs Cid correctly": @@ -187,7 +185,7 @@ checksuite "Test logging output": let id2 = RequestId.fromHex("0x9ab2c4d102a95d990facb022d67b3c9b39052597c006fddf122bed2cb594c282") let ids = @[id, id2] log ids - check logged("ids", "@[0x7120..3dea, 0x9ab2..c282]") + check logged("ids", "\"@[0x7120..3dea, 0x9ab2..c282]\"") check loggedJson("ids", """["0x712003bdfc0db9abf21e7fbb7119cd52ff221c96714d21d39e782d7c744d3dea","0x9ab2c4d102a95d990facb022d67b3c9b39052597c006fddf122bed2cb594c282"]""") test "logs SlotId correctly": @@ -212,5 +210,5 @@ checksuite "Test logging output": let ma = @[MultiAddress.init("/ip4/127.0.0.1/tcp/0").tryGet, MultiAddress.init("/ip4/127.0.0.2/tcp/1").tryGet] log ma - check logged("ma", "@[/ip4/127.0.0.1/tcp/0, /ip4/127.0.0.2/tcp/1]") + check logged("ma", "\"@[/ip4/127.0.0.1/tcp/0, /ip4/127.0.0.2/tcp/1]\"") check loggedJson("ma", "[\"/ip4/127.0.0.1/tcp/0\",\"/ip4/127.0.0.2/tcp/1\"]") From 2eb21327f0475a477a45bf889a696964af654438 Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Tue, 19 Dec 2023 12:27:22 +1100 Subject: [PATCH 10/13] Add logutils module doc --- codex/logutils.nim | 96 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 93 insertions(+), 3 deletions(-) diff --git a/codex/logutils.nim b/codex/logutils.nim index a1a50db88..61cb44af4 100644 --- a/codex/logutils.nim +++ b/codex/logutils.nim @@ -1,3 +1,90 @@ +# logutils is a module that has several goals: +# 1. Fix json logging output (run with `--log-format=json`) which was +# effectively broken for many types using default Chronicles json +# serialization. +# 2. Ability to specify log output for textlines and json sinks together or +# separately +# - This is useful if consuming json in some kind of log parser and need +# valid json with real values +# - eg a shortened Cid is nice to see in a text log in stdout, but won't +# provide a real Cid when parsed in json +# 4. Remove usages of `nim-json-serialization` from the codebase +# 5. Remove need to declare `writeValue` for new types +# 6. Remove need to [avoid importing or exporting `toJson`, `%`, `%*` to prevent +# conflicts](https://github.com/codex-storage/nim-codex/pull/645#issuecomment-1838834467) +# +# When declaring a new type, one should consider importing the `codex/logutils` +# module, and specifying `formatIt`. If textlines log output and json log output +# need to be different, overload `formatIt` and specify a `LogFormat`. If json +# serialization is needed, it can be declared with a `%` proc. `logutils` +# imports and exports `utils/json` which handles the de/serialization, examples +# below. **Only `codex/logutils` needs to be imported.** +# +# Using `logutils` in the Codex codebase: +# - Instead of importing `pkg/chronicles`, import `pkg/codex/logutils` +# - most of `chronicles` is exported by `logutils` +# - Instead of importing `std/json`, import `pkg/codex/utils/json` +# - `std/json` is exported by `utils/json` which is exported by `logutils` +# - Instead of importing `pkg/nim-json-serialization`, import +# `pkg/codex/utils/json` +# - one of the goals is to remove the use of `nim-json-serialization` +# +# ```nim +# import pkg/codex/logutils +# +# type +# BlockAddress* = object +# case leaf*: bool +# of true: +# treeCid* {.serialize.}: Cid +# index* {.serialize.}: Natural +# else: +# cid* {.serialize.}: Cid +# +# logutils.formatIt(LogFormat.textLines, BlockAddress): +# if it.leaf: +# "treeCid: " & shortLog($it.treeCid) & ", index: " & $it.index +# else: +# "cid: " & shortLog($it.cid) +# +# logutils.formatIt(LogFormat.json, BlockAddress): %it +# +# # chronicles textlines output +# TRC test tid=14397405 ba="treeCid: zb2*fndjU1, index: 0" +# # chronicles json output +# {"lvl":"TRC","msg":"test","tid":14397405,"ba":{"treeCid":"zb2rhgsDE16rLtbwTFeNKbdSobtKiWdjJPvKEuPgrQAfndjU1","index":0}} +# ``` +# In this case, `BlockAddress` is just an object, so `utils/json` can handle +# serializing it without issue (only fields annotated with `{.serialize.}` will +# serialize (aka opt-in serialization)). +# +# If one so wished, another option for the textlines log output, would be to +# simply `toString` the serialised json: +# ```nim +# logutils.formatIt(LogFormat.textLines, BlockAddress): $ %it +# # or, more succinctly: +# logutils.formatIt(LogFormat.textLines, BlockAddress): it.toJson +# ``` +# In that case, both the textlines and json sinks would have the same output, so +# we could reduce this even further by not specifying a `LogFormat`: +# ```nim +# type +# BlockAddress* = object +# case leaf*: bool +# of true: +# treeCid* {.serialize.}: Cid +# index* {.serialize.}: Natural +# else: +# cid* {.serialize.}: Cid +# +# logutils.formatIt(BlockAddress): %it +# +# # chronicles textlines output +# TRC test tid=14400673 ba="{\"treeCid\":\"zb2rhgsDE16rLtbwTFeNKbdSobtKiWdjJPvKEuPgrQAfndjU1\",\"index\":0}" +# # chronicles json output +# {"lvl":"TRC","msg":"test","tid":14400673,"ba":{"treeCid":"zb2rhgsDE16rLtbwTFeNKbdSobtKiWdjJPvKEuPgrQAfndjU1","index":0}} +# ``` + import std/options import std/sequtils import std/strutils @@ -5,7 +92,6 @@ import std/sugar import std/typetraits import pkg/chronicles except toJson, `%` -# import pkg/faststreams from pkg/libp2p import Cid, MultiAddress, `$` import pkg/questionable import pkg/stew/byteutils @@ -63,8 +149,12 @@ func to0xHexLog*[U: distinct, T: seq[U]](v: T): string = proc formatTextLineSeq*(val: seq[string]): string = "@[" & val.join(", ") & "]" -template formatIt*(format: LogFormat, T: type, body: untyped) {.dirty.} = - +template formatIt*(format: LogFormat, T: typedesc, body: untyped) {.dirty.} = + # Provides formatters for logging with Chronicles for the given type and + # `LogFormat`. + # NOTE: `seq[T]`, `Option[T]`, and `seq[Option[T]]` are overriddden + # since the base `setProperty` is generic using `auto` and conflicts with + # providing a generic `seq` and `Option` override. when format == LogFormat.json: proc formatJsonOption(val: ?T): JsonNode = if it =? val: From bf680ece7e7561822df47f2a1a1d7224caddcf98 Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Tue, 19 Dec 2023 12:28:00 +1100 Subject: [PATCH 11/13] Shorten logutils.formatIt for `NBytes` Both json and textlines formatIt were not needed, and could be combined into one formatIt --- codex/units.nim | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/codex/units.nim b/codex/units.nim index d254712da..57b52ed66 100644 --- a/codex/units.nim +++ b/codex/units.nim @@ -42,8 +42,7 @@ divMaths(NBytes) proc `$`*(ts: NBytes): string = $(int(ts)) & "'NByte" proc `'nb`*(n: string): NBytes = parseInt(n).NBytes -logutils.formatIt(LogFormat.textLines, NBytes): $it -logutils.formatIt(LogFormat.json, NBytes): $it +logutils.formatIt(NBytes): $it const MiB = 1024.NBytes * 1024.NBytes # ByteSz, 1 mebibyte = 1,048,576 ByteSz From 46ebefd771496f3b9d38c490f64f03ffb311bdee Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Tue, 19 Dec 2023 13:29:36 +1100 Subject: [PATCH 12/13] remove debug integration test config debug output and logformat of json for integration test logs --- tests/integration/testIntegration.nim | 2 +- tests/integration/twonodes.nim | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/integration/testIntegration.nim b/tests/integration/testIntegration.nim index c60fd0eff..eed181f55 100644 --- a/tests/integration/testIntegration.nim +++ b/tests/integration/testIntegration.nim @@ -20,7 +20,7 @@ import ./twonodes # You can also pass a string in same format like for the `--log-level` parameter # to enable custom logging levels for specific topics like: debug2 = "INFO; TRACE: marketplace" -twonodessuite "Integration tests", debug1 = true, debug2 = true: +twonodessuite "Integration tests", debug1 = false, debug2 = false: proc purchaseStateIs(client: CodexClient, id: PurchaseId, state: string): bool = without purchase =? client.getPurchase(id): diff --git a/tests/integration/twonodes.nim b/tests/integration/twonodes.nim index fb99ed3f9..9fa11f1ff 100644 --- a/tests/integration/twonodes.nim +++ b/tests/integration/twonodes.nim @@ -39,7 +39,6 @@ template twonodessuite*(name: string, debug1, debug2: string, body) = "--disc-port=8090", "--listen-addrs=/ip4/127.0.0.1/tcp/0", "--persistence", - "--log-format=json", "--eth-account=" & $account1 ] @@ -60,7 +59,6 @@ template twonodessuite*(name: string, debug1, debug2: string, body) = "--listen-addrs=/ip4/127.0.0.1/tcp/0", "--bootstrap-node=" & bootstrap, "--persistence", - "--log-format=json", "--eth-account=" & $account2 ] From d60d0b476197e6e9af61d4e4d21cd24523c12785 Mon Sep 17 00:00:00 2001 From: Eric <5089238+emizzle@users.noreply.github.com> Date: Wed, 20 Dec 2023 07:13:13 +1100 Subject: [PATCH 13/13] Use ## module doc to support docgen --- codex/logutils.nim | 172 ++++++++++++++++++++++----------------------- 1 file changed, 86 insertions(+), 86 deletions(-) diff --git a/codex/logutils.nim b/codex/logutils.nim index 61cb44af4..b178d38cb 100644 --- a/codex/logutils.nim +++ b/codex/logutils.nim @@ -1,89 +1,89 @@ -# logutils is a module that has several goals: -# 1. Fix json logging output (run with `--log-format=json`) which was -# effectively broken for many types using default Chronicles json -# serialization. -# 2. Ability to specify log output for textlines and json sinks together or -# separately -# - This is useful if consuming json in some kind of log parser and need -# valid json with real values -# - eg a shortened Cid is nice to see in a text log in stdout, but won't -# provide a real Cid when parsed in json -# 4. Remove usages of `nim-json-serialization` from the codebase -# 5. Remove need to declare `writeValue` for new types -# 6. Remove need to [avoid importing or exporting `toJson`, `%`, `%*` to prevent -# conflicts](https://github.com/codex-storage/nim-codex/pull/645#issuecomment-1838834467) -# -# When declaring a new type, one should consider importing the `codex/logutils` -# module, and specifying `formatIt`. If textlines log output and json log output -# need to be different, overload `formatIt` and specify a `LogFormat`. If json -# serialization is needed, it can be declared with a `%` proc. `logutils` -# imports and exports `utils/json` which handles the de/serialization, examples -# below. **Only `codex/logutils` needs to be imported.** -# -# Using `logutils` in the Codex codebase: -# - Instead of importing `pkg/chronicles`, import `pkg/codex/logutils` -# - most of `chronicles` is exported by `logutils` -# - Instead of importing `std/json`, import `pkg/codex/utils/json` -# - `std/json` is exported by `utils/json` which is exported by `logutils` -# - Instead of importing `pkg/nim-json-serialization`, import -# `pkg/codex/utils/json` -# - one of the goals is to remove the use of `nim-json-serialization` -# -# ```nim -# import pkg/codex/logutils -# -# type -# BlockAddress* = object -# case leaf*: bool -# of true: -# treeCid* {.serialize.}: Cid -# index* {.serialize.}: Natural -# else: -# cid* {.serialize.}: Cid -# -# logutils.formatIt(LogFormat.textLines, BlockAddress): -# if it.leaf: -# "treeCid: " & shortLog($it.treeCid) & ", index: " & $it.index -# else: -# "cid: " & shortLog($it.cid) -# -# logutils.formatIt(LogFormat.json, BlockAddress): %it -# -# # chronicles textlines output -# TRC test tid=14397405 ba="treeCid: zb2*fndjU1, index: 0" -# # chronicles json output -# {"lvl":"TRC","msg":"test","tid":14397405,"ba":{"treeCid":"zb2rhgsDE16rLtbwTFeNKbdSobtKiWdjJPvKEuPgrQAfndjU1","index":0}} -# ``` -# In this case, `BlockAddress` is just an object, so `utils/json` can handle -# serializing it without issue (only fields annotated with `{.serialize.}` will -# serialize (aka opt-in serialization)). -# -# If one so wished, another option for the textlines log output, would be to -# simply `toString` the serialised json: -# ```nim -# logutils.formatIt(LogFormat.textLines, BlockAddress): $ %it -# # or, more succinctly: -# logutils.formatIt(LogFormat.textLines, BlockAddress): it.toJson -# ``` -# In that case, both the textlines and json sinks would have the same output, so -# we could reduce this even further by not specifying a `LogFormat`: -# ```nim -# type -# BlockAddress* = object -# case leaf*: bool -# of true: -# treeCid* {.serialize.}: Cid -# index* {.serialize.}: Natural -# else: -# cid* {.serialize.}: Cid -# -# logutils.formatIt(BlockAddress): %it -# -# # chronicles textlines output -# TRC test tid=14400673 ba="{\"treeCid\":\"zb2rhgsDE16rLtbwTFeNKbdSobtKiWdjJPvKEuPgrQAfndjU1\",\"index\":0}" -# # chronicles json output -# {"lvl":"TRC","msg":"test","tid":14400673,"ba":{"treeCid":"zb2rhgsDE16rLtbwTFeNKbdSobtKiWdjJPvKEuPgrQAfndjU1","index":0}} -# ``` +## logutils is a module that has several goals: +## 1. Fix json logging output (run with `--log-format=json`) which was +## effectively broken for many types using default Chronicles json +## serialization. +## 2. Ability to specify log output for textlines and json sinks together or +## separately +## - This is useful if consuming json in some kind of log parser and need +## valid json with real values +## - eg a shortened Cid is nice to see in a text log in stdout, but won't +## provide a real Cid when parsed in json +## 4. Remove usages of `nim-json-serialization` from the codebase +## 5. Remove need to declare `writeValue` for new types +## 6. Remove need to [avoid importing or exporting `toJson`, `%`, `%*` to prevent +## conflicts](https://github.com/codex-storage/nim-codex/pull/645#issuecomment-1838834467) +## +## When declaring a new type, one should consider importing the `codex/logutils` +## module, and specifying `formatIt`. If textlines log output and json log output +## need to be different, overload `formatIt` and specify a `LogFormat`. If json +## serialization is needed, it can be declared with a `%` proc. `logutils` +## imports and exports `utils/json` which handles the de/serialization, examples +## below. **Only `codex/logutils` needs to be imported.** +## +## Using `logutils` in the Codex codebase: +## - Instead of importing `pkg/chronicles`, import `pkg/codex/logutils` +## - most of `chronicles` is exported by `logutils` +## - Instead of importing `std/json`, import `pkg/codex/utils/json` +## - `std/json` is exported by `utils/json` which is exported by `logutils` +## - Instead of importing `pkg/nim-json-serialization`, import +## `pkg/codex/utils/json` +## - one of the goals is to remove the use of `nim-json-serialization` +## +## ```nim +## import pkg/codex/logutils +## +## type +## BlockAddress* = object +## case leaf*: bool +## of true: +## treeCid* {.serialize.}: Cid +## index* {.serialize.}: Natural +## else: +## cid* {.serialize.}: Cid +## +## logutils.formatIt(LogFormat.textLines, BlockAddress): +## if it.leaf: +## "treeCid: " & shortLog($it.treeCid) & ", index: " & $it.index +## else: +## "cid: " & shortLog($it.cid) +## +## logutils.formatIt(LogFormat.json, BlockAddress): %it +## +## # chronicles textlines output +## TRC test tid=14397405 ba="treeCid: zb2*fndjU1, index: 0" +## # chronicles json output +## {"lvl":"TRC","msg":"test","tid":14397405,"ba":{"treeCid":"zb2rhgsDE16rLtbwTFeNKbdSobtKiWdjJPvKEuPgrQAfndjU1","index":0}} +## ``` +## In this case, `BlockAddress` is just an object, so `utils/json` can handle +## serializing it without issue (only fields annotated with `{.serialize.}` will +## serialize (aka opt-in serialization)). +## +## If one so wished, another option for the textlines log output, would be to +## simply `toString` the serialised json: +## ```nim +## logutils.formatIt(LogFormat.textLines, BlockAddress): $ %it +## # or, more succinctly: +## logutils.formatIt(LogFormat.textLines, BlockAddress): it.toJson +## ``` +## In that case, both the textlines and json sinks would have the same output, +## so we could reduce this even further by not specifying a `LogFormat`: +## ```nim +## type +## BlockAddress* = object +## case leaf*: bool +## of true: +## treeCid* {.serialize.}: Cid +## index* {.serialize.}: Natural +## else: +## cid* {.serialize.}: Cid +## +## logutils.formatIt(BlockAddress): %it +## +## # chronicles textlines output +## TRC test tid=14400673 ba="{\"treeCid\":\"zb2rhgsDE16rLtbwTFeNKbdSobtKiWdjJPvKEuPgrQAfndjU1\",\"index\":0}" +## # chronicles json output +## {"lvl":"TRC","msg":"test","tid":14400673,"ba":{"treeCid":"zb2rhgsDE16rLtbwTFeNKbdSobtKiWdjJPvKEuPgrQAfndjU1","index":0}} +## ``` import std/options import std/sequtils