Remove Cryptowatch API (it has been switch off). Use another data sou…

…rce for fiats.

src/api/common/CMakeLists.txt

@@ -19,7 +19,7 @@ function(add_common_test name)
    add_unit_test(
      ${name} 
      ${MY_UNPARSED_ARGUMENTS}
-     src/cryptowatchapi.cpp
+     src/commonapi.cpp
      LIBRARIES
      coincenter_api-objects
      coincenter_http-request
@@ -28,8 +28,8 @@ function(add_common_test name)
 endfunction()
 
 add_common_test(
-  cryptowatchapi_test
-  test/cryptowatchapi_test.cpp
+  commonapi_test
+  test/commonapi_test.cpp
 )
 
 add_common_test(

src/api/common/include/commonapi.hpp

@@ -0,0 +1,58 @@
+#pragma once
+
+#include <chrono>
+#include <mutex>
+#include <optional>
+#include <unordered_map>
+
+#include "cachedresult.hpp"
+#include "cct_flatset.hpp"
+#include "cct_json.hpp"
+#include "curlhandle.hpp"
+#include "exchangebase.hpp"
+#include "market.hpp"
+#include "timedef.hpp"
+
+namespace cct {
+class CoincenterInfo;
+namespace api {
+/// Public API connected to different exchanges, providing fast methods to retrieve huge amount of data.
+class CommonAPI : public ExchangeBase {
+ public:
+  using Fiats = FlatSet<CurrencyCode>;
+
+  CommonAPI(const CoincenterInfo &config, Duration fiatsUpdateFrequency = std::chrono::hours(96),
+            bool loadFromFileCacheAtInit = true);
+
+  /// Returns a new set of fiat currencies.
+  Fiats queryFiats() {
+    std::lock_guard<std::mutex> guard(_fiatsMutex);
+    return _fiatsCache.get();
+  }
+
+  /// Tells whether given currency code is a fiat currency or not.
+  /// Fiat currencies are traditional currencies, such as EUR, USD, GBP, KRW, etc.
+  /// Information here: https://en.wikipedia.org/wiki/Fiat_money
+  bool queryIsCurrencyCodeFiat(CurrencyCode currencyCode) {
+    std::lock_guard<std::mutex> guard(_fiatsMutex);
+    return _fiatsCache.get().contains(currencyCode);
+  }
+
+  void updateCacheFile() const override;
+
+ private:
+  struct FiatsFunc {
+    FiatsFunc();
+
+    Fiats operator()();
+
+    CurlHandle _curlHandle;
+  };
+
+  CachedResultVault _cachedResultVault;
+  const CoincenterInfo &_coincenterInfo;
+  std::mutex _fiatsMutex;
+  CachedResult<FiatsFunc> _fiatsCache;
+};
+}  // namespace api
+}  // namespace cct

src/api/common/include/exchangepublicapi.hpp

@@ -4,7 +4,7 @@
 #include <string_view>
 
 #include "cct_string.hpp"
-#include "cryptowatchapi.hpp"
+#include "commonapi.hpp"
 #include "currencycode.hpp"
 #include "currencyexchangeflatset.hpp"
 #include "exchangebase.hpp"
@@ -28,7 +28,7 @@ class ExchangePublic : public ExchangeBase {
   static constexpr int kDefaultDepth = MarketOrderBook::kDefaultDepth;
   static constexpr int kNbLastTradesDefault = 100;
 
-  using Fiats = CryptowatchAPI::Fiats;
+  using Fiats = CommonAPI::Fiats;
 
   virtual ~ExchangePublic() = default;
 
@@ -57,15 +57,15 @@ class ExchangePublic : public ExchangeBase {
     Fiats fiats = queryFiats();
     MarketSet markets;
     MarketsPath conversionPath = findMarketsPath(a.currencyCode(), toCurrency, markets, fiats, true);
-    return convert(a, toCurrency, conversionPath, fiats, marketOrderBookMap, true, priceOptions);
+    return convert(a, toCurrency, conversionPath, fiats, marketOrderBookMap, priceOptions);
   }
 
   /// Attempts to convert amount into a target currency.
   /// Conversion is made according to given price options, which uses the 'Maker' prices by default.
   /// No external calls is made with this version, it has all what it needs
   std::optional<MonetaryAmount> convert(MonetaryAmount a, CurrencyCode toCurrency, const MarketsPath &conversionPath,
                                         const Fiats &fiats, MarketOrderBookMap &marketOrderBookMap,
-                                        bool canUseCryptowatchAPI, const PriceOptions &priceOptions = PriceOptions());
+                                        const PriceOptions &priceOptions = PriceOptions());
 
   /// Retrieve the fixed withdrawal fees per currency.
   /// Depending on the exchange, this could be retrieved dynamically,
@@ -97,7 +97,7 @@ class ExchangePublic : public ExchangeBase {
   /// Retrieve the last price of given market.
   virtual MonetaryAmount queryLastPrice(Market mk) = 0;
 
-  Fiats queryFiats() { return _cryptowatchApi.queryFiats(); }
+  Fiats queryFiats() { return _commonApi.queryFiats(); }
 
   /// Get the name of the exchange in lower case.
   std::string_view name() const { return _name; }
@@ -160,18 +160,18 @@ class ExchangePublic : public ExchangeBase {
 
   const ExchangeInfo &exchangeInfo() const { return _exchangeInfo; }
 
-  CryptowatchAPI &cryptowatchAPI() { return _cryptowatchApi; }
+  CommonAPI &commonAPI() { return _commonApi; }
 
  protected:
   friend class ExchangePrivate;
 
-  ExchangePublic(std::string_view name, FiatConverter &fiatConverter, CryptowatchAPI &cryptowatchApi,
+  ExchangePublic(std::string_view name, FiatConverter &fiatConverter, CommonAPI &commonApi,
                  const CoincenterInfo &coincenterInfo);
 
   string _name;
   CachedResultVault _cachedResultVault;
   FiatConverter &_fiatConverter;
-  CryptowatchAPI &_cryptowatchApi;
+  CommonAPI &_commonApi;
   const CoincenterInfo &_coincenterInfo;
   const ExchangeInfo &_exchangeInfo;
 };

src/api/common/include/exchangepublicapi_mock.hpp

@@ -2,17 +2,17 @@
 
 #include <gmock/gmock.h>
 
-#include "cryptowatchapi.hpp"
+#include "commonapi.hpp"
 #include "exchangepublicapi.hpp"
 #include "exchangepublicapitypes.hpp"
 #include "fiatconverter.hpp"
 
 namespace cct::api {
 class MockExchangePublic : public ExchangePublic {
  public:
-  MockExchangePublic(std::string_view name, FiatConverter &fiatConverter, CryptowatchAPI &cryptowatchApi,
+  MockExchangePublic(std::string_view name, FiatConverter &fiatConverter, CommonAPI &commonApi,
                      const CoincenterInfo &config)
-      : ExchangePublic(name, fiatConverter, cryptowatchApi, config) {}
+      : ExchangePublic(name, fiatConverter, commonApi, config) {}
 
   MOCK_METHOD(bool, healthCheck, (), (override));
   MOCK_METHOD(CurrencyExchangeFlatSet, queryTradableCurrencies, (), (override));

src/api/common/src/commonapi.cpp

@@ -0,0 +1,91 @@
+#include "commonapi.hpp"
+
+#include <cctype>
+#include <string_view>
+
+#include "cct_exception.hpp"
+#include "cct_json.hpp"
+#include "cct_log.hpp"
+#include "coincenterinfo.hpp"
+#include "curloptions.hpp"
+#include "file.hpp"
+#include "permanentcurloptions.hpp"
+#include "timedef.hpp"
+#include "toupperlower.hpp"
+
+namespace cct::api {
+namespace {
+
+File GetFiatCacheFile(std::string_view dataDir) {
+  return {dataDir, File::Type::kCache, "fiatcache.json", File::IfError::kNoThrow};
+}
+
+}  // namespace
+
+CommonAPI::CommonAPI(const CoincenterInfo& config, Duration fiatsUpdateFrequency, bool loadFromFileCacheAtInit)
+    : _coincenterInfo(config), _fiatsCache(CachedResultOptions(fiatsUpdateFrequency, _cachedResultVault)) {
+  if (loadFromFileCacheAtInit) {
+    json data = GetFiatCacheFile(_coincenterInfo.dataDir()).readAllJson();
+    if (!data.empty()) {
+      int64_t timeEpoch = data["timeepoch"].get<int64_t>();
+      auto& fiatsFile = data["fiats"];
+      Fiats fiats;
+      fiats.reserve(static_cast<Fiats::size_type>(fiatsFile.size()));
+      for (json& val : fiatsFile) {
+        log::trace("Reading fiat {} from cache file", val.get<std::string_view>());
+        fiats.emplace_hint(fiats.end(), std::move(val.get_ref<string&>()));
+      }
+      log::debug("Loaded {} fiats from cache file", fiats.size());
+      _fiatsCache.set(std::move(fiats), TimePoint(std::chrono::seconds(timeEpoch)));
+    }
+  }
+}
+
+namespace {
+constexpr std::string_view kFiatsUrl = "https://datahub.io/core/currency-codes/r/codes-all.json";
+}
+
+CommonAPI::FiatsFunc::FiatsFunc()
+    : _curlHandle(kFiatsUrl, nullptr, PermanentCurlOptions::Builder().setFollowLocation().build()) {}
+
+CommonAPI::Fiats CommonAPI::FiatsFunc::operator()() {
+  json dataCSV = json::parse(_curlHandle.query("", CurlOptions(HttpRequestType::kGet)));
+  Fiats fiats;
+  for (const json& fiatData : dataCSV) {
+    auto codeIt = fiatData.find("AlphabeticCode");
+    auto withdrawalDateIt = fiatData.find("WithdrawalDate");
+    if (codeIt != fiatData.end() && !codeIt->is_null() && withdrawalDateIt != fiatData.end() &&
+        withdrawalDateIt->is_null()) {
+      fiats.insert(CurrencyCode(codeIt->get<std::string_view>()));
+      log::debug("Stored {} fiat", codeIt->get<std::string_view>());
+    }
+  }
+  if (fiats.empty()) {
+    throw exception("Error parsing currency codes, no fiats found in {}", dataCSV.dump());
+  }
+
+  log::info("Stored {} fiats", fiats.size());
+  return fiats;
+}
+
+void CommonAPI::updateCacheFile() const {
+  File fiatsCacheFile = GetFiatCacheFile(_coincenterInfo.dataDir());
+  json data = fiatsCacheFile.readAllJson();
+  auto fiatsPtrLastUpdatedTimePair = _fiatsCache.retrieve();
+  auto timeEpochIt = data.find("timeepoch");
+  if (timeEpochIt != data.end()) {
+    int64_t lastTimeFileUpdated = timeEpochIt->get<int64_t>();
+    if (TimePoint(std::chrono::seconds(lastTimeFileUpdated)) >= fiatsPtrLastUpdatedTimePair.second) {
+      return;  // No update
+    }
+  }
+  data.clear();
+  if (fiatsPtrLastUpdatedTimePair.first != nullptr) {
+    for (CurrencyCode fiatCode : *fiatsPtrLastUpdatedTimePair.first) {
+      data["fiats"].emplace_back(fiatCode.str());
+    }
+    data["timeepoch"] = TimestampToS(fiatsPtrLastUpdatedTimePair.second);
+    fiatsCacheFile.write(data);
+  }
+}
+}  // namespace cct::api