From b1c480b1eec3de23b339b40f3d5ee4f312035597 Mon Sep 17 00:00:00 2001
From: Yonas Habteab <yonas.habteab@icinga.com>
Date: Thu, 29 Aug 2024 14:14:20 +0200
Subject: [PATCH] JsonRpcConnection: Log message processing stats

---
 lib/base/logger.hpp              | 10 ++++++++
 lib/remote/jsonrpcconnection.cpp | 39 ++++++++++++++++++++++++++++++--
 lib/remote/jsonrpcconnection.hpp |  2 +-
 3 files changed, 48 insertions(+), 3 deletions(-)

diff --git a/lib/base/logger.hpp b/lib/base/logger.hpp
index 10e0872ae68..2be9a40011c 100644
--- a/lib/base/logger.hpp
+++ b/lib/base/logger.hpp
@@ -118,6 +118,16 @@ class Log
 
 	~Log();
 
+	/**
+	 * Set the severity of the current Log instance.
+	 *
+	 * @param severity LogSeverity The new severity you want to override the current one with.
+	 */
+	void SetSeverity(LogSeverity severity)
+	{
+		m_Severity = severity;
+	}
+
 	template<typename T>
 	Log& operator<<(const T& val)
 	{
diff --git a/lib/remote/jsonrpcconnection.cpp b/lib/remote/jsonrpcconnection.cpp
index d5a8e46cb8a..59132c5be3e 100644
--- a/lib/remote/jsonrpcconnection.cpp
+++ b/lib/remote/jsonrpcconnection.cpp
@@ -60,6 +60,8 @@ void JsonRpcConnection::Start()
 
 void JsonRpcConnection::HandleIncomingMessages(boost::asio::yield_context yc)
 {
+	namespace  ch = std::chrono;
+
 	m_Stream->next_layer().SetSeen(&m_Seen);
 
 	for (;;) {
@@ -78,9 +80,36 @@ void JsonRpcConnection::HandleIncomingMessages(boost::asio::yield_context yc)
 		m_Seen = Utility::GetTime();
 
 		try {
+			auto start (ch::steady_clock::now());
+			ch::steady_clock::duration cpuBoundDuration;
+
+			String rpcMethod;
+
+			Log statsLog(LogDebug, "JsonRpcConnection");
+
+			Defer addLogStats ([this, &statsLog, &rpcMethod,  &start, &cpuBoundDuration]() {
+				statsLog << "Processing JSON-RPC '" << rpcMethod << "' message for identity '" << m_Identity << "'";
+
+				if (ch::duration_cast<ch::seconds>(cpuBoundDuration).count()) {
+					statsLog << " waited '" << ch::duration_cast<ch::milliseconds>(cpuBoundDuration).count() << "ms' on semaphore and";
+				}
+
+				auto duration = ch::steady_clock::now() - start;
+				if (duration >= ch::seconds(5)) {
+					// Processing that RPC message seems to take an unexpectedly long time,
+					// so promote the log entry from debug to warning.
+					statsLog.SetSeverity(LogWarning);
+				}
+
+				statsLog << " took total " << ch::duration_cast<ch::milliseconds>(duration).count() << "ms.";
+			});
+
 			CpuBoundWork handleMessage (yc);
 
-			MessageHandler(message);
+			// Cache the elapsed time to acquire a CPU semaphore used to detect extremely heavy workloads.
+			cpuBoundDuration = ch::steady_clock::now() - start;
+
+			MessageHandler(message, rpcMethod);
 
 			l_TaskStats.InsertValue(Utility::GetTime(), 1);
 		} catch (const std::exception& ex) {
@@ -245,9 +274,15 @@ void JsonRpcConnection::Disconnect()
 	});
 }
 
-void JsonRpcConnection::MessageHandler(const String& jsonString)
+void JsonRpcConnection::MessageHandler(const String& jsonString, String& rpcMethod)
 {
 	Dictionary::Ptr message = JsonRpc::DecodeMessage(jsonString);
+	Defer setRpcMethod ([&message, &rpcMethod]() {
+		rpcMethod = message->Get("method");
+		if (rpcMethod.IsEmpty()) {
+			rpcMethod = "UNKNOWN";
+		}
+	});
 
 	if (m_Endpoint && message->Contains("ts")) {
 		double ts = message->Get("ts");
diff --git a/lib/remote/jsonrpcconnection.hpp b/lib/remote/jsonrpcconnection.hpp
index 591ddcb1f0d..88d4502cf54 100644
--- a/lib/remote/jsonrpcconnection.hpp
+++ b/lib/remote/jsonrpcconnection.hpp
@@ -88,7 +88,7 @@ class JsonRpcConnection final : public Object
 	void CheckLiveness(boost::asio::yield_context yc);
 
 	bool ProcessMessage();
-	void MessageHandler(const String& jsonString);
+	void MessageHandler(const String& jsonString, String& rpcMethod);
 
 	void CertificateRequestResponseHandler(const Dictionary::Ptr& message);