Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Move Query Results to LedgerMetrics Periodical Tracer #6076

Draft
wants to merge 11 commits into
base: master
Choose a base branch
from
1 change: 1 addition & 0 deletions bench/tx-generator/src/Cardano/Benchmarking/Tracer.hs
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,7 @@ initialTraceConfig = TraceConfig {
, tcPeerFrequency = Just 2000 -- Every 2 seconds
, tcResourceFrequency = Just 1000 -- Every second
, tcMetricsPrefix = Nothing
, tcLedgerMetricsFrequency = Nothing
}
where
setMaxDetail :: Text -> ([Text], [ConfigOption])
Expand Down
2 changes: 1 addition & 1 deletion cardano-node/cardano-node.cabal
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ library
Cardano.Node.Tracing.Tracers.Diffusion
Cardano.Node.Tracing.Tracers.ForgingThreadStats
Cardano.Node.Tracing.Tracers.KESInfo
Cardano.Node.Tracing.Tracers.LedgerMetrics
Cardano.Node.Tracing.Tracers.NodeToClient
Cardano.Node.Tracing.Tracers.NodeToNode
Cardano.Node.Tracing.Tracers.NodeVersion
Expand All @@ -119,7 +120,6 @@ library
Cardano.Node.Tracing.Tracers.Peer
Cardano.Node.Tracing.Tracers.Resources
Cardano.Node.Tracing.Tracers.Shutdown
Cardano.Node.Tracing.Tracers.StartLeadershipCheck
Cardano.Node.Tracing.Tracers.Startup
Cardano.Node.Types
Cardano.Tracing.Config
Expand Down
5 changes: 3 additions & 2 deletions cardano-node/src/Cardano/Node/Tracing.hs
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,9 @@ import Cardano.Node.Startup (NodeInfo, NodeStartupInfo, StartupTrace (
import Cardano.Node.Tracing.StateRep (NodeState)
import Cardano.Node.Tracing.Tracers.ConsensusStartupException
(ConsensusStartupException (..))
import Cardano.Node.Tracing.Tracers.Peer (PeerT)
import Cardano.Node.Tracing.Tracers.LedgerMetrics (LedgerMetrics)
import Cardano.Node.Tracing.Tracers.NodeVersion (NodeVersionTrace)

import Cardano.Node.Tracing.Tracers.Peer (PeerT)
import qualified Ouroboros.Consensus.Network.NodeToClient as NodeToClient
import qualified Ouroboros.Consensus.Network.NodeToNode as NodeToNode
import qualified Ouroboros.Consensus.Node.Tracers as Consensus
Expand Down Expand Up @@ -55,4 +55,5 @@ data Tracers peer localPeer blk p2p = Tracers
, nodeStateTracer :: !(Tracer IO NodeState)
, resourcesTracer :: !(Tracer IO ResourceStats)
, peersTracer :: !(Tracer IO [PeerT blk])
, ledgerMetricsTracer :: !(Tracer IO LedgerMetrics)
}
6 changes: 6 additions & 0 deletions cardano-node/src/Cardano/Node/Tracing/API.hs
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import Cardano.Node.Tracing
import Cardano.Node.Tracing.DefaultTraceConfig (defaultCardanoConfig)
import Cardano.Node.Tracing.StateRep (NodeState (..))
import Cardano.Node.Tracing.Tracers
import Cardano.Node.Tracing.Tracers.LedgerMetrics
import Cardano.Node.Tracing.Tracers.Peer (startPeerTracer)
import Cardano.Node.Tracing.Tracers.Resources (startResourceTracer)
import Cardano.Node.Types
Expand Down Expand Up @@ -69,6 +70,11 @@ initTraceDispatcher nc p networkMagic nodeKernel p2pMode = do
(resourcesTracer tracers)
(fromMaybe 1000 (tcResourceFrequency trConfig))

startLedgerMetricsTracer
(ledgerMetricsTracer tracers)
(fromMaybe 1000 (tcLedgerMetricsFrequency trConfig))
nodeKernel

startPeerTracer
(peersTracer tracers)
nodeKernel
Expand Down
9 changes: 6 additions & 3 deletions cardano-node/src/Cardano/Node/Tracing/Consistency.hs
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,12 @@ import Cardano.Node.Tracing.Documentation (docTracersFirstPhase)
import Cardano.Node.Tracing.Formatting ()
import qualified Cardano.Node.Tracing.StateRep as SR
import Cardano.Node.Tracing.Tracers.BlockReplayProgress
import Cardano.Node.Tracing.Tracers.Consensus
import Cardano.Node.Tracing.Tracers.Diffusion ()
import Cardano.Node.Tracing.Tracers.KESInfo ()
import Cardano.Node.Tracing.Tracers.LedgerMetrics (LedgerMetrics)
import Cardano.Node.Tracing.Tracers.NodeToClient ()
import Cardano.Node.Tracing.Tracers.NodeToNode ()
import Cardano.Node.Tracing.Tracers.NodeVersion (NodeVersionTrace)

import Cardano.Node.Tracing.Tracers.NonP2P ()
import Cardano.Node.Tracing.Tracers.P2P ()
import Cardano.Node.Tracing.Tracers.Peer
Expand All @@ -46,6 +45,7 @@ import Ouroboros.Consensus.MiniProtocol.ChainSync.Client (TraceChainSy
import Ouroboros.Consensus.MiniProtocol.ChainSync.Server (TraceChainSyncServerEvent)
import Ouroboros.Consensus.MiniProtocol.LocalTxSubmission.Server
(TraceLocalTxSubmissionServerEvent (..))
import Ouroboros.Consensus.Node.Tracers (TraceForgeEvent)
import qualified Ouroboros.Consensus.Protocol.Ledger.HotKey as HotKey
import qualified Ouroboros.Consensus.Storage.ChainDB as ChainDB
import Ouroboros.Network.Block (Point (..), SlotNo, Tip)
Expand Down Expand Up @@ -128,6 +128,8 @@ getAllNamespaces =
(allNamespaces :: [Namespace [PeerT blk]])
resourcesNS = map nsGetTuple
(allNamespaces :: [Namespace ResourceStats])
ledgerMetricsNS = map nsGetTuple
(allNamespaces :: [Namespace LedgerMetrics])
startupNS = map (nsGetTuple . nsReplacePrefix ["Startup"])
(allNamespaces :: [Namespace (StartupTrace blk)])
shutdownNS = map (nsGetTuple . nsReplacePrefix ["Shutdown"])
Expand Down Expand Up @@ -180,7 +182,7 @@ getAllNamespaces =
mempoolNS = map (nsGetTuple . nsReplacePrefix ["Mempool"])
(allNamespaces :: [Namespace (TraceEventMempool blk)])
forgeNS = map (nsGetTuple . nsReplacePrefix ["Forge", "Loop"])
(allNamespaces :: [Namespace (ForgeTracerType blk)])
(allNamespaces :: [Namespace (TraceForgeEvent blk)])

blockchainTimeNS = map (nsGetTuple . nsReplacePrefix ["BlockchainTime"])
(allNamespaces :: [Namespace (TraceBlockchainTimeEvent RelativeTime)])
Expand Down Expand Up @@ -371,6 +373,7 @@ getAllNamespaces =
stateNS
<> peersNS
<> resourcesNS
<> ledgerMetricsNS
<> startupNS
<> shutdownNS
<> nodeVersionNS
Expand Down
11 changes: 9 additions & 2 deletions cardano-node/src/Cardano/Node/Tracing/Documentation.hs
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import Cardano.Node.Tracing.Tracers.Consensus
import Cardano.Node.Tracing.Tracers.Diffusion ()
import Cardano.Node.Tracing.Tracers.ForgingThreadStats (ForgeThreadStats)
import Cardano.Node.Tracing.Tracers.KESInfo ()
import Cardano.Node.Tracing.Tracers.LedgerMetrics (LedgerMetrics)
import Cardano.Node.Tracing.Tracers.NodeToClient ()
import Cardano.Node.Tracing.Tracers.NodeToNode ()
import Cardano.Node.Tracing.Tracers.NodeVersion (NodeVersionTrace)
Expand Down Expand Up @@ -231,6 +232,10 @@ docTracersFirstPhase condConfigFileName = do
configureTracers configReflection trConfig [resourcesTr]
resourcesTrDoc <- documentTracer (resourcesTr :: Trace IO ResourceStats)

ledgerMetricsTr <- mkCardanoTracer trBase trForward mbTrEKG []
configureTracers configReflection trConfig [ledgerMetricsTr]
ledgerMetricsTrDoc <- documentTracer (ledgerMetricsTr :: Trace IO LedgerMetrics)

-- Startup tracer
startupTr <- mkCardanoTracer
trBase trForward mbTrEKG
Expand Down Expand Up @@ -360,8 +365,7 @@ docTracersFirstPhase condConfigFileName = do
["Forge", "Loop"]
configureTracers configReflection trConfig [forgeTr]
forgeTrDoc <- documentTracer (forgeTr ::
Trace IO (ForgeTracerType blk))

Trace IO (Consensus.TraceForgeEvent blk))

forgeTr' <- mkCardanoTracer
trBase trForward mbTrEKG
Expand All @@ -377,6 +381,8 @@ docTracersFirstPhase condConfigFileName = do
blockchainTimeTrDoc <- documentTracer (blockchainTimeTr ::
Trace IO (TraceBlockchainTimeEvent RelativeTime))



-- Node to client

keepAliveClientTr <- mkCardanoTracer
Expand Down Expand Up @@ -699,6 +705,7 @@ docTracersFirstPhase condConfigFileName = do
<> nodeStartupInfoDpDoc
<> stateTrDoc
<> resourcesTrDoc
<> ledgerMetricsTrDoc
<> startupTrDoc
<> shutdownTrDoc
<> nodeVersionDoc
Expand Down
14 changes: 9 additions & 5 deletions cardano-node/src/Cardano/Node/Tracing/Tracers.hs
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import Cardano.Node.Tracing.Tracers.Consensus
import Cardano.Node.Tracing.Tracers.Diffusion ()
import Cardano.Node.Tracing.Tracers.ForgingThreadStats (forgeThreadStats)
import Cardano.Node.Tracing.Tracers.KESInfo
import Cardano.Node.Tracing.Tracers.LedgerMetrics ()
import Cardano.Node.Tracing.Tracers.NodeToClient ()
import Cardano.Node.Tracing.Tracers.NodeToNode ()
import Cardano.Node.Tracing.Tracers.NodeVersion (getNodeVersion)
Expand Down Expand Up @@ -117,6 +118,9 @@ mkDispatchTracers nodeKernel trBase trForward mbTrEKG trDataPoint trConfig enabl
!resourcesTr <- mkCardanoTracer trBase trForward mbTrEKG []
configureTracers configReflection trConfig [resourcesTr]

!ledgerMetricsTr <- mkCardanoTracer trBase trForward mbTrEKG []
configureTracers configReflection trConfig [ledgerMetricsTr]

!startupTr <- mkCardanoTracer trBase trForward mbTrEKG ["Startup"]
configureTracers configReflection trConfig [startupTr]

Expand Down Expand Up @@ -199,6 +203,7 @@ mkDispatchTracers nodeKernel trBase trForward mbTrEKG trDataPoint trConfig enabl
, resourcesTracer = Tracer (traceWith resourcesTr)
, peersTracer = Tracer (traceWith peersTr)
<> Tracer (traceNodePeers nodePeersDP)
, ledgerMetricsTracer = Tracer (traceWith ledgerMetricsTr)
}

mkConsensusTracers :: forall blk.
Expand All @@ -217,7 +222,7 @@ mkConsensusTracers :: forall blk.
-> TraceConfig
-> NodeKernelData blk
-> IO (Consensus.Tracers IO (ConnectionId RemoteAddress) (ConnectionId LocalAddress) blk)
mkConsensusTracers configReflection trBase trForward mbTrEKG _trDataPoint trConfig nodeKernel = do
mkConsensusTracers configReflection trBase trForward mbTrEKG _trDataPoint trConfig _nodeKernel = do
!chainSyncClientTr <- mkCardanoTracer
trBase trForward mbTrEKG
["ChainSync", "Client"]
Expand Down Expand Up @@ -303,10 +308,9 @@ mkConsensusTracers configReflection trBase trForward mbTrEKG _trDataPoint trConf
["Mempool"]
configureTracers configReflection trConfig [mempoolTr]

!forgeTr <- mkCardanoTracer'
!forgeTr <- mkCardanoTracer
trBase trForward mbTrEKG
["Forge", "Loop"]
(forgeTracerTransform nodeKernel)
configureTracers configReflection trConfig [forgeTr]

!forgeThreadStatsTr <- mkCardanoTracer'
Expand Down Expand Up @@ -366,9 +370,9 @@ mkConsensusTracers configReflection trBase trForward mbTrEKG _trDataPoint trConf
, Consensus.mempoolTracer = Tracer $
traceWith mempoolTr
, Consensus.forgeTracer =
Tracer (\(Consensus.TraceLabelCreds _ x) -> traceWith (contramap Left forgeTr) x)
Tracer (\(Consensus.TraceLabelCreds _ x) -> traceWith forgeTr x)
<>
Tracer (\(Consensus.TraceLabelCreds _ x) -> traceWith (contramap Left forgeThreadStatsTr) x)
Tracer (\(Consensus.TraceLabelCreds _ x) -> traceWith forgeThreadStatsTr x)
, Consensus.blockchainTimeTracer = Tracer $
traceWith blockchainTimeTr
, Consensus.keepAliveClientTracer = Tracer $
Expand Down
1 change: 1 addition & 0 deletions cardano-node/src/Cardano/Node/Tracing/Tracers/ChainDB.hs
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@

module Cardano.Node.Tracing.Tracers.ChainDB
( withAddedToCurrentChainEmptyLimited
, fragmentChainDensity
) where

import Cardano.Logging
Expand Down
102 changes: 2 additions & 100 deletions cardano-node/src/Cardano/Node/Tracing/Tracers/Consensus.hs
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,7 @@

module Cardano.Node.Tracing.Tracers.Consensus
(
TraceStartLeadershipCheckPlus (..)
, ForgeTracerType
, forgeTracerTransform
, initialClientMetrics
initialClientMetrics
, calculateBlockFetchClientMetrics
, servedBlockLatest
, ClientMetrics
Expand All @@ -31,7 +28,6 @@ import Cardano.Node.Tracing.Era.Shelley ()
import Cardano.Node.Tracing.Formatting ()
import Cardano.Node.Tracing.Render
import Cardano.Node.Tracing.Tracers.ConsensusStartupException ()
import Cardano.Node.Tracing.Tracers.StartLeadershipCheck
import Cardano.Protocol.TPraos.OCert (KESPeriod (..))
import Cardano.Slotting.Slot (WithOrigin (..))
import Ouroboros.Consensus.Block
Expand Down Expand Up @@ -74,7 +70,7 @@ import Ouroboros.Network.TxSubmission.Outbound

import Control.Monad (guard)
import Control.Monad.Class.MonadTime.SI (Time (..))
import Data.Aeson (ToJSON, Value (Number, String), toJSON, (.=))
import Data.Aeson (ToJSON, Value (..), toJSON, (.=))
import qualified Data.Aeson as Aeson
import Data.Foldable (Foldable (toList))
import Data.Int (Int64)
Expand Down Expand Up @@ -1359,100 +1355,6 @@ instance MetaTrace (TraceEventMempool blk) where
, Namespace [] ["Synced"]
]

--------------------------------------------------------------------------------
-- ForgeTracerType
--------------------------------------------------------------------------------

instance ( tx ~ GenTx blk
, ConvertRawHash blk
, GetHeader blk
, HasHeader blk
, HasKESInfo blk
, LedgerSupportsProtocol blk
, LedgerSupportsMempool blk
, SerialiseNodeToNodeConstraints blk
, HasTxId (GenTx blk)
, Show (ForgeStateUpdateError blk)
, Show (CannotForge blk)
, LogFormatting (InvalidBlockReason blk)
, LogFormatting (CannotForge blk)
, LogFormatting (ForgeStateUpdateError blk))
=> LogFormatting (ForgeTracerType blk) where
forMachine dtal (Left i) = forMachine dtal i
forMachine dtal (Right i) = forMachine dtal i
forHuman (Left i) = forHumanOrMachine i
forHuman (Right i) = forHumanOrMachine i
asMetrics (Left i) = asMetrics i
asMetrics (Right i) = asMetrics i

instance MetaTrace (ForgeTracerType blk) where
namespaceFor (Left ev) =
nsCast (namespaceFor ev)
namespaceFor (Right _ev) =
Namespace [] ["StartLeadershipCheckPlus"]

severityFor (Namespace _ ["StartLeadershipCheckPlus"]) _ = Just
Info
severityFor ns (Just (Left ev')) =
severityFor (nsCast ns) (Just ev')
severityFor ns Nothing =
severityFor (nsCast ns :: Namespace (TraceForgeEvent blk)) Nothing
severityFor _ _ = Nothing

detailsFor (Namespace _ ["StartLeadershipCheckPlus"]) _ = Just
DNormal
detailsFor ns (Just (Left ev')) =
detailsFor (nsCast ns) (Just ev')
detailsFor ns Nothing =
detailsFor (nsCast ns :: Namespace (TraceForgeEvent blk)) Nothing
detailsFor _ _ = Nothing

privacyFor (Namespace _ ["StartLeadershipCheckPlus"]) _ = Just
Public
privacyFor ns (Just (Left ev')) =
privacyFor (nsCast ns) (Just ev')
privacyFor ns Nothing =
privacyFor (nsCast ns :: Namespace (TraceForgeEvent blk)) Nothing
privacyFor _ _ = Nothing

metricsDocFor (Namespace _ ["StartLeadershipCheckPlus"]) =
[ ("Forge.UtxoSize", "UTxO set size")
, ("Forge.DelegMapSize", "Delegation map size")
]
metricsDocFor ns =
metricsDocFor (nsCast ns :: Namespace (TraceForgeEvent blk))

documentFor (Namespace _ ["StartLeadershipCheckPlus"]) = Just $ mconcat
[ "We adopted the block we produced, we also trace the transactions"
, " that were adopted."
]
documentFor ns =
documentFor (nsCast ns :: Namespace (TraceForgeEvent blk))

allNamespaces =
Namespace [] ["StartLeadershipCheckPlus"]
: map nsCast (allNamespaces :: [Namespace (TraceForgeEvent blk)])

--------------------------------------------------------------------------------
-- TraceStartLeadershipCheck
--------------------------------------------------------------------------------

instance LogFormatting TraceStartLeadershipCheckPlus where
forMachine _dtal TraceStartLeadershipCheckPlus {..} =
mconcat [ "kind" .= String "TraceStartLeadershipCheck"
, "slot" .= toJSON (unSlotNo tsSlotNo)
, "utxoSize" .= Number (fromIntegral tsUtxoSize)
, "delegMapSize" .= Number (fromIntegral tsDelegMapSize)
, "chainDensity" .= Number (fromRational (toRational tsChainDensity))
]
forHuman TraceStartLeadershipCheckPlus {..} =
"Checking for leadership in slot " <> showT (unSlotNo tsSlotNo)
<> " utxoSize " <> showT tsUtxoSize
<> " delegMapSize " <> showT tsDelegMapSize
<> " chainDensity " <> showT tsChainDensity
asMetrics TraceStartLeadershipCheckPlus {..} =
[IntM "utxoSize" (fromIntegral tsUtxoSize),
IntM "delegMapSize" (fromIntegral tsDelegMapSize)]


--------------------------------------------------------------------------------
Expand Down
Loading
Loading