Less logs for repeated GET request logs

src/api/exchanges/src/huobiprivateapi.cpp

@@ -61,20 +61,22 @@ namespace {
 
 string BuildParamStr(HttpRequestType requestType, std::string_view baseUrl, std::string_view method,
                      std::string_view postDataStr) {
-  std::string_view urlBaseWithoutHttps(baseUrl.begin() + std::string_view("https://").size(), baseUrl.end());
-  std::string_view requestTypeStr = IntegralToString(requestType);
+  const std::string_view urlBaseWithoutHttps(baseUrl.begin() + std::string_view("https://").size(), baseUrl.end());
+  const auto requestTypeStr = HttpRequestTypeToString(requestType);
+
   string paramsStr(requestTypeStr.size() + urlBaseWithoutHttps.size() + method.size() + postDataStr.size() + 3U, '\n');
 
   auto it = paramsStr.begin();
   it = std::ranges::copy(requestTypeStr, it).out;
   it = std::ranges::copy(urlBaseWithoutHttps, it + 1).out;
   it = std::ranges::copy(method, it + 1).out;
+
   std::ranges::copy(postDataStr, it + 1);
 
   return paramsStr;
 }
 
-CurlOptions::PostDataFormat ComputePostDataFormat(HttpRequestType requestType, const CurlPostData& postData) {
+auto ComputePostDataFormat(HttpRequestType requestType, const CurlPostData& postData) {
   CurlOptions::PostDataFormat postDataFormat = CurlOptions::PostDataFormat::kString;
   if (!postData.empty() && requestType != HttpRequestType::kGet) {
     postDataFormat = CurlOptions::PostDataFormat::kJson;

src/api/exchanges/src/kucoinprivateapi.cpp

@@ -64,7 +64,7 @@ json PrivateQuery(CurlHandle& curlHandle, const APIKey& apiKey, HttpRequestType
                   CurlPostData&& postData = CurlPostData(), bool throwIfError = true) {
   string strToSign(Nonce_TimeSinceEpochInMs());
   auto nonceSize = strToSign.size();
-  strToSign.append(IntegralToString(requestType));
+  strToSign.append(HttpRequestTypeToString(requestType));
   strToSign.append(endpoint);
 
   CurlOptions::PostDataFormat postDataFormat = CurlOptions::PostDataFormat::kString;
@@ -78,16 +78,14 @@ json PrivateQuery(CurlHandle& curlHandle, const APIKey& apiKey, HttpRequestType
     }
   }
 
-  auto signature = B64Encode(ssl::Sha256Bin(strToSign, apiKey.privateKey()));
-  auto passphrase = B64Encode(ssl::Sha256Bin(apiKey.passphrase(), apiKey.privateKey()));
-
   CurlOptions opts(requestType, std::move(postData), postDataFormat);
 
   auto& httpHeaders = opts.mutableHttpHeaders();
+
   httpHeaders.emplace_back("KC-API-KEY", apiKey.key());
-  httpHeaders.emplace_back("KC-API-SIGN", signature);
+  httpHeaders.emplace_back("KC-API-SIGN", B64Encode(ssl::Sha256Bin(strToSign, apiKey.privateKey())));
   httpHeaders.emplace_back("KC-API-TIMESTAMP", std::string_view(strToSign.data(), nonceSize));
-  httpHeaders.emplace_back("KC-API-PASSPHRASE", passphrase);
+  httpHeaders.emplace_back("KC-API-PASSPHRASE", B64Encode(ssl::Sha256Bin(apiKey.passphrase(), apiKey.privateKey())));
   httpHeaders.emplace_back("KC-API-KEY-VERSION", 2);
 
   json ret = json::parse(curlHandle.query(endpoint, opts));
@@ -111,6 +109,7 @@ json PrivateQuery(CurlHandle& curlHandle, const APIKey& apiKey, HttpRequestType
 void InnerTransfer(CurlHandle& curlHandle, const APIKey& apiKey, MonetaryAmount amount, std::string_view fromStr,
                    std::string_view toStr) {
   log::info("Perform inner transfer of {} to {} account", amount, toStr);
+
   PrivateQuery(curlHandle, apiKey, HttpRequestType::kPost, "/api/v2/accounts/inner-transfer",
                {{"clientOid", Nonce_TimeSinceEpochInMs()},  // Not really needed, but it's mandatory apparently
                 {"currency", amount.currencyStr()},

src/engine/src/coincenter.cpp

@@ -105,6 +105,7 @@ int Coincenter::process(const CoincenterCommands &coincenterCommands) {
   TimePoint lastCommandTime;
   for (int repeatPos = 0; repeatPos != nbRepeats && g_signalStatus == 0; ++repeatPos) {
     const auto earliestTimeNextCommand = lastCommandTime + repeatTime;
+    const bool doLog = nbRepeats != 1 && (repeatPos < 100 || repeatPos % 100 == 0);
 
     lastCommandTime = Clock::now();
 
@@ -113,10 +114,12 @@ int Coincenter::process(const CoincenterCommands &coincenterCommands) {
 
       lastCommandTime += waitingDuration;
 
-      log::debug("Sleep for {} before next command", DurationToString(waitingDuration));
+      if (doLog) {
+        log::debug("Sleep for {} before next command", DurationToString(waitingDuration));
+      }
       std::this_thread::sleep_for(waitingDuration);
     }
-    if (nbRepeats != 1 && (repeatPos < 100 || repeatPos % 100 == 0)) {
+    if (doLog) {
       if (nbRepeats == -1) {
         log::info("Process request {}", repeatPos + 1);
       } else {

src/http-request/include/httprequesttype.hpp

@@ -11,7 +11,7 @@ enum class HttpRequestType : int8_t { kGet, kPut, kPost, kDelete };
 static constexpr HttpRequestType kHttpRequestTypes[] = {HttpRequestType::kGet, HttpRequestType::kPost,
                                                         HttpRequestType::kPut, HttpRequestType::kDelete};
 
-constexpr std::string_view IntegralToString(HttpRequestType requestType) {
+constexpr std::string_view HttpRequestTypeToString(HttpRequestType requestType) {
   switch (requestType) {
     case HttpRequestType::kGet:
       return "GET";

src/http-request/src/curlhandle.cpp

@@ -65,6 +65,25 @@ void CurlSetLogIfError(CURL *curl, CURLoption curlOption, T value) {
     }
   }
 }
+
+curl_slist *ComputeCurlSListPtr(const CurlOptions::HttpHeaders &httpHeaders) {
+  curl_slist *curlListPtr = nullptr;
+  curl_slist *oldCurlListPtr = nullptr;
+  for (const auto &httpHeader : httpHeaders) {
+    // Trick: HttpHeaders is actually a FlatKeyValueString with '\0' as header separator and ':' as key / value
+    // separator. curl_slist_append expects a 'const char *' as HTTP header - it's possible here to just give the
+    // pointer to the beginning of the key as we know the bundle key/value ends with a null-terminating char
+    // (either there is at least one more key / value pair, either it's the last one and it's also fine as string is
+    // guaranteed to be null-terminated since C++11)
+    curlListPtr = curl_slist_append(curlListPtr, httpHeader.key().data());
+    if (curlListPtr == nullptr) {
+      curl_slist_free_all(oldCurlListPtr);
+      throw std::bad_alloc();
+    }
+    oldCurlListPtr = curlListPtr;
+  }
+  return curlListPtr;
+}
 }  // namespace
 
 string GetCurlVersionInfo() {
@@ -165,6 +184,7 @@ std::string_view CurlHandle::query(std::string_view endpoint, const CurlOptions
   const auto baseUrlPos = _bestURLPicker.nextBaseURLPos();
   const std::string_view baseUrl = _bestURLPicker.getBaseURL(baseUrlPos);
   const std::string_view postDataStr = postData.str();
+
   string modifiedURL(baseUrl.size() + endpoint.size() + (appendParametersInQueryStr ? (1U + postDataStr.size()) : 0U),
                      '?');
 
@@ -209,21 +229,7 @@ std::string_view CurlHandle::query(std::string_view endpoint, const CurlOptions
 
   CurlSetLogIfError(curl, CURLOPT_VERBOSE, opts.isVerbose() ? 1L : 0L);
 
-  curl_slist *curlListPtr = nullptr;
-  curl_slist *oldCurlListPtr = nullptr;
-  for (const auto &httpHeader : opts.httpHeaders()) {
-    // Trick: HttpHeaders is actually a FlatKeyValueString with '\0' as header separator and ':' as key / value
-    // separator. curl_slist_append expects a 'const char *' as HTTP header - it's possible here to just give the
-    // pointer to the beginning of the key as we know the bundle key/value ends with a null-terminating char
-    // (either there is at least one more key / value pair, either it's the last one and it's also fine as string is
-    // guaranteed to be null-terminated since C++11)
-    curlListPtr = curl_slist_append(curlListPtr, httpHeader.key().data());
-    if (curlListPtr == nullptr) {
-      curl_slist_free_all(oldCurlListPtr);
-      throw std::bad_alloc();
-    }
-    oldCurlListPtr = curlListPtr;
-  }
+  curl_slist *curlListPtr = ComputeCurlSListPtr(opts.httpHeaders());
 
   using CurlSlistDeleter = decltype([](curl_slist *hdrList) { curl_slist_free_all(hdrList); });
   using CurlListUniquePtr = std::unique_ptr<curl_slist, CurlSlistDeleter>;
@@ -249,8 +255,16 @@ std::string_view CurlHandle::query(std::string_view endpoint, const CurlOptions
     }
   }
 
-  log::log(_requestCallLogLevel, "{} {}{}{}", IntegralToString(opts.requestType()), modifiedURL,
-           optsStr.empty() ? "" : "?", optsStr);
+  auto nbRequestsDone = _bestURLPicker.nbRequestsDone();
+  static constexpr auto kLogRequestsThreshold = 100;
+
+  if (opts.requestType() != HttpRequestType::kGet || nbRequestsDone % kLogRequestsThreshold == 0) {
+    const auto httpRequestStr = HttpRequestTypeToString(opts.requestType());
+    log::log(_requestCallLogLevel, "{} {}{}{}", httpRequestStr, modifiedURL, optsStr.empty() ? "" : "?", optsStr);
+    if (nbRequestsDone == 0 && opts.requestType() == HttpRequestType::kGet) {
+      log::log(_requestCallLogLevel, "Will only log {} requests every {} calls", httpRequestStr, kLogRequestsThreshold);
+    }
+  }
 
   // Actually make the query, with a fast retry mechanism (to avoid random technical errors)
   Duration sleepingTime = milliseconds(100);
@@ -290,10 +304,12 @@ std::string_view CurlHandle::query(std::string_view endpoint, const CurlOptions
 
     // Periodic memory release to avoid memory leak for a very large number of requests
     static constexpr int kReleaseMemoryRequestsFrequency = 10000;
-    if ((_bestURLPicker.nbRequestsDone() % kReleaseMemoryRequestsFrequency) == 0) {
+    if ((nbRequestsDone % kReleaseMemoryRequestsFrequency) == 0) {
       _queryData.shrink_to_fit();
     }
 
+    ++nbRequestsDone;
+
   } while (res != CURLE_OK && ++retryPos <= _nbMaxRetries);
   if (retryPos > _nbMaxRetries) {
     switch (_tooManyErrorsPolicy) {

src/http-request/src/curlmetrics.cpp

@@ -9,7 +9,7 @@ namespace {
 MetricKeyPerRequestType DefineTypes(MetricKey &requestCountKey) {
   MetricKeyPerRequestType ret;
   for (HttpRequestType requestType : kHttpRequestTypes) {
-    requestCountKey.set("type", IntegralToString(requestType));
+    requestCountKey.set("type", HttpRequestTypeToString(requestType));
     ret.insert_or_assign(requestType, requestCountKey);
   }
   return ret;

src/tech/include/stringconv.hpp

@@ -71,7 +71,7 @@ Integral StringToIntegral(std::string_view str) {
   }
 
   if (ptr != endPtr) {
-    throw exception("Only {} out of {} chars decoded into integral {}", ptr - begPtr, str.size(), ret);
+    throw exception("Only {} chars from {} decoded into integral {}", ptr - begPtr, str, ret);
   }
   return ret;
 }