Add operation count query (#873)

* Add some index for transaction listing

* Add count() to OperationQuery

-> To get the number of operations, WD AccountController currently fetch all operations (sql) and then count the operations (scala). This PR adds a count() method, thus the counting is done by the database wich is much faster.

It aims to fix a huge memory usage when fetching account information on huge accounts with thousands of transactions

* minor improvements

core/idl/wallet/common/wallet.djinni

@@ -106,6 +106,13 @@ Block = record {
     height: i64;
 }
 
+OperationCount = record {
+    # Type of operation
+    type: OperationType;
+    # Count of operations
+    count: i64;
+}
+
 # Class representing an operation.
 Operation = interface +c {
     # Get id's operation.
@@ -265,6 +272,9 @@ OperationQuery = interface +c {
     # Execute query to retrieve operations.
     # @param callback, if execute method succeed, ListCallback object returning a List of Operation objects
     execute(callback: ListCallback<Operation>);
+    # Count operations by type
+    # @param callback, if execute method succeed, ListCallback object returning a List of OperationCount objects
+    count(callback: ListCallback<OperationCount>);
 }
 
 # Structure of informations needed for account creation.

core/src/database/query/QueryBuilder.cpp

@@ -51,6 +51,10 @@ namespace ledger {
                 query << sFilter;
             }
 
+            if (!_group.empty()) {
+              query << " GROUP BY " << _group;
+            }
+
             if (!_order.empty()) {
                 query << " ORDER BY ";
                 for (auto it = _order.begin(); it != _order.end(); it++) {
@@ -133,5 +137,10 @@ namespace ledger {
             return *this;
         }
 
+        QueryBuilder &QueryBuilder::groupBy(std::string group) {
+          _group = std::move(group);
+          return *this;
+        }
+
     }
 }

core/src/database/query/QueryBuilder.h

@@ -55,6 +55,7 @@ namespace ledger {
             QueryBuilder& order(std::string&& keys, bool&& descending, std::string&& table);
             QueryBuilder& limit(int32_t limit);
             QueryBuilder& offset(int32_t offset);
+            QueryBuilder& groupBy(std::string group);
             soci::details::prepare_temp_type execute(soci::session& sql);
 
         private:
@@ -63,6 +64,7 @@ namespace ledger {
             std::string _keys;
             std::string _table;
             std::string _output;
+            std::string _group;
             std::list<std::tuple<std::string, bool, std::string>> _order;
             std::vector<Option<LeftOuterJoin>> _outerJoins;
             std::shared_ptr<QueryFilter> _filter;

core/src/jni/jni/OperationCount.cpp

@@ -0,0 +1,31 @@
+// AUTOGENERATED FILE - DO NOT MODIFY!
+// This file generated by Djinni from wallet.djinni
+
+#include "OperationCount.hpp"  // my header
+#include "Marshal.hpp"
+#include "OperationType.hpp"
+
+namespace djinni_generated {
+
+OperationCount::OperationCount() = default;
+
+OperationCount::~OperationCount() = default;
+
+auto OperationCount::fromCpp(JNIEnv* jniEnv, const CppType& c) -> ::djinni::LocalRef<JniType> {
+    const auto& data = ::djinni::JniClass<OperationCount>::get();
+    auto r = ::djinni::LocalRef<JniType>{jniEnv->NewObject(data.clazz.get(), data.jconstructor,
+                                                           ::djinni::get(::djinni_generated::OperationType::fromCpp(jniEnv, c.type)),
+                                                           ::djinni::get(::djinni::I64::fromCpp(jniEnv, c.count)))};
+    ::djinni::jniExceptionCheck(jniEnv);
+    return r;
+}
+
+auto OperationCount::toCpp(JNIEnv* jniEnv, JniType j) -> CppType {
+    ::djinni::JniLocalScope jscope(jniEnv, 3);
+    assert(j != nullptr);
+    const auto& data = ::djinni::JniClass<OperationCount>::get();
+    return {::djinni_generated::OperationType::toCpp(jniEnv, jniEnv->GetObjectField(j, data.field_type)),
+            ::djinni::I64::toCpp(jniEnv, jniEnv->GetLongField(j, data.field_count))};
+}
+
+}  // namespace djinni_generated

core/src/jni/jni/OperationCount.hpp

@@ -0,0 +1,35 @@
+// AUTOGENERATED FILE - DO NOT MODIFY!
+// This file generated by Djinni from wallet.djinni
+
+#ifndef DJINNI_GENERATED_OPERATIONCOUNT_HPP_JNI_
+#define DJINNI_GENERATED_OPERATIONCOUNT_HPP_JNI_
+
+#include "../../api/OperationCount.hpp"
+#include "djinni_support.hpp"
+
+namespace djinni_generated {
+
+class OperationCount final {
+public:
+    using CppType = ::ledger::core::api::OperationCount;
+    using JniType = jobject;
+
+    using Boxed = OperationCount;
+
+    ~OperationCount();
+
+    static CppType toCpp(JNIEnv* jniEnv, JniType j);
+    static ::djinni::LocalRef<JniType> fromCpp(JNIEnv* jniEnv, const CppType& c);
+
+private:
+    OperationCount();
+    friend ::djinni::JniClass<OperationCount>;
+
+    const ::djinni::GlobalRef<jclass> clazz { ::djinni::jniFindClass("co/ledger/core/OperationCount") };
+    const jmethodID jconstructor { ::djinni::jniGetMethodID(clazz.get(), "<init>", "(Lco/ledger/core/OperationType;J)V") };
+    const jfieldID field_type { ::djinni::jniGetFieldID(clazz.get(), "type", "Lco/ledger/core/OperationType;") };
+    const jfieldID field_count { ::djinni::jniGetFieldID(clazz.get(), "count", "J") };
+};
+
+}  // namespace djinni_generated
+#endif //DJINNI_GENERATED_OPERATIONCOUNT_HPP_JNI_

core/src/jni/jni/OperationCountListCallback.cpp

@@ -0,0 +1,29 @@
+// AUTOGENERATED FILE - DO NOT MODIFY!
+// This file generated by Djinni from callback.djinni
+
+#include "OperationCountListCallback.hpp"  // my header
+#include "Error.hpp"
+#include "Marshal.hpp"
+#include "OperationCount.hpp"
+
+namespace djinni_generated {
+
+OperationCountListCallback::OperationCountListCallback() : ::djinni::JniInterface<::ledger::core::api::OperationCountListCallback, OperationCountListCallback>() {}
+
+OperationCountListCallback::~OperationCountListCallback() = default;
+
+OperationCountListCallback::JavaProxy::JavaProxy(JniType j) : Handle(::djinni::jniGetThreadEnv(), j) { }
+
+OperationCountListCallback::JavaProxy::~JavaProxy() = default;
+
+void OperationCountListCallback::JavaProxy::onCallback(const std::experimental::optional<std::vector<::ledger::core::api::OperationCount>> & c_result, const std::experimental::optional<::ledger::core::api::Error> & c_error) {
+    auto jniEnv = ::djinni::jniGetThreadEnv();
+    ::djinni::JniLocalScope jscope(jniEnv, 10);
+    const auto& data = ::djinni::JniClass<::djinni_generated::OperationCountListCallback>::get();
+    jniEnv->CallVoidMethod(Handle::get().get(), data.method_onCallback,
+                           ::djinni::get(::djinni::Optional<std::experimental::optional, ::djinni::List<::djinni_generated::OperationCount>>::fromCpp(jniEnv, c_result)),
+                           ::djinni::get(::djinni::Optional<std::experimental::optional, ::djinni_generated::Error>::fromCpp(jniEnv, c_error)));
+    ::djinni::jniExceptionCheck(jniEnv);
+}
+
+}  // namespace djinni_generated

core/src/jni/jni/OperationCountListCallback.hpp

@@ -0,0 +1,48 @@
+// AUTOGENERATED FILE - DO NOT MODIFY!
+// This file generated by Djinni from callback.djinni
+
+#ifndef DJINNI_GENERATED_OPERATIONCOUNTLISTCALLBACK_HPP_JNI_
+#define DJINNI_GENERATED_OPERATIONCOUNTLISTCALLBACK_HPP_JNI_
+
+#include "../../api/OperationCountListCallback.hpp"
+#include "djinni_support.hpp"
+
+namespace djinni_generated {
+
+class OperationCountListCallback final : ::djinni::JniInterface<::ledger::core::api::OperationCountListCallback, OperationCountListCallback> {
+public:
+    using CppType = std::shared_ptr<::ledger::core::api::OperationCountListCallback>;
+    using CppOptType = std::shared_ptr<::ledger::core::api::OperationCountListCallback>;
+    using JniType = jobject;
+
+    using Boxed = OperationCountListCallback;
+
+    ~OperationCountListCallback();
+
+    static CppType toCpp(JNIEnv* jniEnv, JniType j) { return ::djinni::JniClass<OperationCountListCallback>::get()._fromJava(jniEnv, j); }
+    static ::djinni::LocalRef<JniType> fromCppOpt(JNIEnv* jniEnv, const CppOptType& c) { return {jniEnv, ::djinni::JniClass<OperationCountListCallback>::get()._toJava(jniEnv, c)}; }
+    static ::djinni::LocalRef<JniType> fromCpp(JNIEnv* jniEnv, const CppType& c) { return fromCppOpt(jniEnv, c); }
+
+private:
+    OperationCountListCallback();
+    friend ::djinni::JniClass<OperationCountListCallback>;
+    friend ::djinni::JniInterface<::ledger::core::api::OperationCountListCallback, OperationCountListCallback>;
+
+    class JavaProxy final : ::djinni::JavaProxyHandle<JavaProxy>, public ::ledger::core::api::OperationCountListCallback
+    {
+    public:
+        JavaProxy(JniType j);
+        ~JavaProxy();
+
+        void onCallback(const std::experimental::optional<std::vector<::ledger::core::api::OperationCount>> & result, const std::experimental::optional<::ledger::core::api::Error> & error) override;
+
+    private:
+        friend ::djinni::JniInterface<::ledger::core::api::OperationCountListCallback, ::djinni_generated::OperationCountListCallback>;
+    };
+
+    const ::djinni::GlobalRef<jclass> clazz { ::djinni::jniFindClass("co/ledger/core/OperationCountListCallback") };
+    const jmethodID method_onCallback { ::djinni::jniGetMethodID(clazz.get(), "onCallback", "(Ljava/util/ArrayList;Lco/ledger/core/Error;)V") };
+};
+
+}  // namespace djinni_generated
+#endif //DJINNI_GENERATED_OPERATIONCOUNTLISTCALLBACK_HPP_JNI_

core/src/jni/jni/OperationQuery.cpp

@@ -3,6 +3,7 @@
 
 #include "OperationQuery.hpp"  // my header
 #include "Marshal.hpp"
+#include "OperationCountListCallback.hpp"
 #include "OperationListCallback.hpp"
 #include "OperationOrderKey.hpp"
 #include "QueryFilter.hpp"
@@ -92,4 +93,13 @@ CJNIEXPORT void JNICALL Java_co_ledger_core_OperationQuery_00024CppProxy_native_
     } JNI_TRANSLATE_EXCEPTIONS_RETURN(jniEnv, )
 }
 
+CJNIEXPORT void JNICALL Java_co_ledger_core_OperationQuery_00024CppProxy_native_1count(JNIEnv* jniEnv, jobject /*this*/, jlong nativeRef, jobject j_callback)
+{
+    try {
+        DJINNI_FUNCTION_PROLOGUE1(jniEnv, nativeRef);
+        const auto& ref = ::djinni::objectFromHandleAddress<::ledger::core::api::OperationQuery>(nativeRef);
+        ref->count(::djinni_generated::OperationCountListCallback::toCpp(jniEnv, j_callback));
+    } JNI_TRANSLATE_EXCEPTIONS_RETURN(jniEnv, )
+}
+
 }  // namespace djinni_generated

core/src/wallet/common/OperationQuery.cpp

@@ -30,6 +30,7 @@
  */
 #include "OperationQuery.h"
 #include <api/OperationListCallback.hpp>
+#include <api/OperationCountListCallback.hpp>
 #include "Operation.h"
 #include <database/soci-date.h>
 #include <database/soci-option.h>
@@ -110,6 +111,41 @@ namespace ledger {
             return shared_from_this();
         }
 
+        void OperationQuery::count(
+            const std::shared_ptr<api::OperationCountListCallback> &callback) {
+          count().callback(_mainContext, callback);
+        }
+
+        Future<std::vector<api::OperationCount>>
+        OperationQuery::count() {
+          auto self = shared_from_this();
+          return async<std::vector<api::OperationCount>>([=] () {
+            std::vector<api::OperationCount> out;
+            self->performCount(out);
+            return out;
+          });
+        }
+
+        soci::rowset<soci::row> OperationQuery::performCount(soci::session &sql) {
+          return _builder.select("o.type, count(*)")
+              .from("operations").to("o")
+              .outerJoin("blocks AS b", "o.block_uid = b.uid")
+              .groupBy("o.type")
+              .execute(sql);
+        }
+
+        void OperationQuery::performCount(std::vector<api::OperationCount> &operations) {
+          soci::session sql(_pool->getPool());
+          const soci::rowset<soci::row> rows = performCount(sql);
+
+          for (const auto& row : rows) {
+            const auto type = api::from_string<api::OperationType >(row.get<std::string>(0));
+            const auto count = soci::get_number<int64_t>(row, 1);
+
+            operations.emplace_back(api::OperationCount{type, count});
+          }
+        }
+
         void OperationQuery::execute(const std::shared_ptr<api::OperationListCallback> &callback) {
            execute().callback(_mainContext, callback);
         }