From fcc740e8a3a5688d5253ef34f6e6936d0b05240a Mon Sep 17 00:00:00 2001 From: Jacob Date: Wed, 18 May 2022 14:20:45 -0500 Subject: [PATCH 01/31] rust experiments --- .gitignore | 2 + Cargo.lock | 2757 +++++++++++++++++ Cargo.toml | 4 + service/Cargo.toml | 21 + service/build.rs | 4 + .../20220516203513_initial.down.sql | 3 + .../migrations/20220516203513_initial.up.sql | 47 + service/rustfmt.toml | 1 + service/src/badge/mod.rs | 157 + service/src/badge/transfer.rs | 198 ++ service/src/indexer.rs | 79 + service/src/main.rs | 69 + service/src/rpc.rs | 26 + service/src/updater.rs | 50 + 14 files changed, 3418 insertions(+) create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 service/Cargo.toml create mode 100644 service/build.rs create mode 100644 service/migrations/20220516203513_initial.down.sql create mode 100644 service/migrations/20220516203513_initial.up.sql create mode 100644 service/rustfmt.toml create mode 100644 service/src/badge/mod.rs create mode 100644 service/src/badge/transfer.rs create mode 100644 service/src/indexer.rs create mode 100644 service/src/main.rs create mode 100644 service/src/rpc.rs create mode 100644 service/src/updater.rs diff --git a/.gitignore b/.gitignore index 53fe481..e1508aa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,11 @@ .DS_Store node_modules /dist +target # local env files +.env .env.local .env.*.local diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..592c6de --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,2757 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "actix" +version = "0.11.0-beta.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eb78f9871feb3519e06b947c2becbf2cc7f67ce786e510e6bd3f9a27da3dbf1" +dependencies = [ + "actix-rt", + "actix_derive", + "bitflags", + "bytes", + "crossbeam-channel", + "futures-core", + "futures-sink", + "futures-task", + "log", + "once_cell", + "parking_lot 0.11.2", + "pin-project-lite", + "smallvec", + "tokio", + "tokio-util 0.6.10", +] + +[[package]] +name = "actix-rt" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ea16c295198e958ef31930a6ef37d0fb64e9ca3b6116e6b93a8bdae96ee1000" +dependencies = [ + "futures-core", + "tokio", +] + +[[package]] +name = "actix_derive" +version = "0.6.0-beta.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae749cf2582eb83efd288edd4e9704600fdce1bc4f69aa0c86ca1368a3e4c13f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "addr2line" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ecd88a8c8378ca913a680cd98f0f13ac67383d35993f86c90a70e3f137816b" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "ahash" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" +dependencies = [ + "getrandom 0.2.6", + "once_cell", + "version_check", +] + +[[package]] +name = "aho-corasick" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" +dependencies = [ + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08f9b8508dccb7687a1d6c4ce66b2b0ecef467c94667de27d8d7fe1f8d2a9cdc" + +[[package]] +name = "arrayref" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" + +[[package]] +name = "arrayvec" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" + +[[package]] +name = "arrayvec" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" + +[[package]] +name = "async-trait" +version = "0.1.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed6aa3524a2dfcf9fe180c51eae2b58738348d819517ceadf95789c51fff7600" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atoi" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616896e05fc0e2649463a93a15183c6a16bf03413a7af88ef1285ddedfa9cda5" +dependencies = [ + "num-traits", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "backtrace" +version = "0.3.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11a17d453482a265fd5f8479f2a3f405566e6ca627837aaddb85af8b1ab8ef61" +dependencies = [ + "addr2line", + "cc", + "cfg-if 1.0.0", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "base64" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" + +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitvec" +version = "0.20.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7774144344a4faa177370406a7ff5f1da24303817368584c6206c8303eb07848" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + +[[package]] +name = "blake2" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a4e37d16930f5459780f5621038b6382b9bb37c19016f39fb6b5808d831f174" +dependencies = [ + "crypto-mac", + "digest 0.9.0", + "opaque-debug", +] + +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-buffer" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf7fe51849ea569fd452f37822f606a5cabb684dc918707a0193fd4664ff324" +dependencies = [ + "generic-array", +] + +[[package]] +name = "borsh" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15bf3650200d8bffa99015595e10f1fbd17de07abbc25bb067da79e769939bfa" +dependencies = [ + "borsh-derive", + "hashbrown", +] + +[[package]] +name = "borsh-derive" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6441c552f230375d18e3cc377677914d2ca2b0d36e52129fe15450a2dce46775" +dependencies = [ + "borsh-derive-internal", + "borsh-schema-derive-internal", + "proc-macro-crate 0.1.5", + "proc-macro2", + "syn", +] + +[[package]] +name = "borsh-derive-internal" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5449c28a7b352f2d1e592a8a28bf139bc71afb0764a14f3c02500935d8c44065" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "borsh-schema-derive-internal" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdbd5696d8bfa21d53d9fe39a714a18538bad11492a42d066dbbc395fb1951c0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "bs58" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "771fe0050b883fcc3ea2359b1a96bcfbc090b7116eae7c3c512c7a083fdf23d3" + +[[package]] +name = "bumpalo" +version = "3.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899" + +[[package]] +name = "byte-slice-cast" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87c5fdd0166095e1d463fc6cc01aa8ce547ad77a4e84d42eb6762b084e28067e" + +[[package]] +name = "byteorder" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" + +[[package]] +name = "bytes" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" + +[[package]] +name = "bytesize" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c58ec36aac5066d5ca17df51b3e70279f5670a72102f5752cb7e7c856adfc70" + +[[package]] +name = "c2-chacha" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d27dae93fe7b1e0424dc57179ac396908c26b035a87234809f5c4dfd1b47dc80" +dependencies = [ + "cipher", + "ppv-lite86", +] + +[[package]] +name = "cc" +version = "1.0.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +dependencies = [ + "libc", + "num-integer", + "num-traits", + "serde", + "time", + "winapi", +] + +[[package]] +name = "cipher" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f8e7987cbd042a63249497f41aed09f8e65add917ea6566effbc56578d6801" +dependencies = [ + "generic-array", +] + +[[package]] +name = "convert_case" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" + +[[package]] +name = "core-foundation" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" + +[[package]] +name = "cpufeatures" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59a6001667ab124aebae2a495118e11d30984c3a653e99d86d58971708cf5e4b" +dependencies = [ + "libc", +] + +[[package]] +name = "crc" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49fc9a695bca7f35f5f4c15cddc84415f66a74ea78eef08e90c5024f2b540e23" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccaeedb56da03b09f598226e25e80088cb4cd25f316e6e4df7d695f0feeb1403" + +[[package]] +name = "crossbeam-channel" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aaa7bd5fb665c6864b5f963dd9097905c54125909c7aa94c9e18507cdbe6c53" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f25d8400f4a7a5778f0e4e52384a48cbd9b5c495d110786187fc750075277a2" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38" +dependencies = [ + "cfg-if 1.0.0", + "lazy_static", +] + +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + +[[package]] +name = "crypto-common" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57952ca27b5e3606ff4dd79b0020231aaf9d6aa76dc05fd30137538c50bd3ce8" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "crypto-mac" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b584a330336237c1eecd3e94266efb216c56ed91225d634cb2991c5f3fd1aeab" +dependencies = [ + "generic-array", + "subtle", +] + +[[package]] +name = "curve25519-dalek" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f9d052967f590a76e62eb387bd0bbb1b000182c3cefe5364db6b7211651bc0" +dependencies = [ + "byteorder", + "digest 0.9.0", + "rand_core 0.5.1", + "subtle", + "zeroize", +] + +[[package]] +name = "derive_more" +version = "0.99.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "rustc_version", + "syn", +] + +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array", +] + +[[package]] +name = "digest" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506" +dependencies = [ + "block-buffer 0.10.2", + "crypto-common", + "subtle", +] + +[[package]] +name = "dirs" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + +[[package]] +name = "dotenv" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" + +[[package]] +name = "easy-ext" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53aff6fdc1b181225acdcb5b14c47106726fd8e486707315b1b138baed68ee31" + +[[package]] +name = "ed25519" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9c280362032ea4203659fc489832d0204ef09f247a0506f170dafcac08c369" +dependencies = [ + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c762bae6dcaf24c4c84667b8579785430908723d5c889f469d76a41d59cc7a9d" +dependencies = [ + "curve25519-dalek", + "ed25519", + "rand 0.7.3", + "serde", + "sha2 0.9.9", + "zeroize", +] + +[[package]] +name = "either" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" + +[[package]] +name = "encoding_rs" +version = "0.8.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "envy" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f47e0157f2cb54f5ae1bd371b30a2ae4311e1c028f575cd4e81de7353215965" +dependencies = [ + "serde", +] + +[[package]] +name = "event-listener" +version = "2.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77f3309417938f28bf8228fcff79a4a37103981e3e186d2ccd19c74b38f4eb71" + +[[package]] +name = "failure" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86" +dependencies = [ + "backtrace", + "failure_derive", +] + +[[package]] +name = "failure_derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "fastrand" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3fcf0cee53519c866c09b5de1f6c56ff9d647101f81c1964fa632e148896cdf" +dependencies = [ + "instant", +] + +[[package]] +name = "fixed-hash" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcf0ed7fe52a17a03854ec54a9f76d6d84508d1c0e66bc1793301c73fc8493c" +dependencies = [ + "byteorder", + "rand 0.8.5", + "rustc-hex", + "static_assertions", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" +dependencies = [ + "matches", + "percent-encoding", +] + +[[package]] +name = "funty" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed34cd105917e91daa4da6b3728c47b068749d6a62c59811f06ed2ac71d9da7" + +[[package]] +name = "futures" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f73fe65f54d1e12b726f517d3e2135ca3125a437b6d998caf1962961f7172d9e" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3083ce4b914124575708913bca19bfe887522d6e2e6d0952943f5eac4a74010" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c09fd04b7e4073ac7156a9539b57a484a8ea920f79c7c675d05d289ab6110d3" + +[[package]] +name = "futures-executor" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9420b90cfa29e327d0429f19be13e7ddb68fa1cccb09d65e5706b8c7a749b8a6" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-intrusive" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62007592ac46aa7c2b6416f7deb9a8a8f63a01e0f1d6e1787d5630170db2b63e" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot 0.11.2", +] + +[[package]] +name = "futures-io" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc4045962a5a5e935ee2fdedaa4e08284547402885ab326734432bed5d12966b" + +[[package]] +name = "futures-macro" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33c1e13800337f4d4d7a316bf45a567dbcb6ffe087f16424852d97e97a91f512" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21163e139fa306126e6eedaf49ecdb4588f939600f0b1e770f4205ee4b7fa868" + +[[package]] +name = "futures-task" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c66a976bf5909d801bbef33416c41372779507e7a6b3a5e25e4749c58f776a" + +[[package]] +name = "futures-util" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8b7abd5d659d9b90c8cba917f6ec750a74e2dc23902ef9cd4cc8c8b22e6036a" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd48d33ec7f05fbfa152300fdad764757cbded343c1aa1cff2fbaf4134851803" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "wasi 0.10.2+wasi-snapshot-preview1", +] + +[[package]] +name = "gimli" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78cc372d058dcf6d5ecd98510e7fbc9e5aec4d21de70f65fea8fecebcd881bd4" + +[[package]] +name = "h2" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37a82c6d637fc9515a4694bbf1cb2457b79d81ce52b3108bdeea58b07dd34a57" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util 0.7.2", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" +dependencies = [ + "ahash", +] + +[[package]] +name = "hashlink" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7249a3129cbc1ffccd74857f81464a323a152173cdb134e0fd81bc803b29facf" +dependencies = [ + "hashbrown", +] + +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "heck" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hkdf" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791a029f6b9fc27657f6f188ec6e5e43f6911f6f878e0dc5501396e09809d437" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.3", +] + +[[package]] +name = "http" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff8670570af52249509a86f5e3e18a08c60b177071826898fde8997cf5f6bfbb" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ff4f84919677303da5f147645dbea6b1881f368d03ac84e1dc09031ebd7b2c6" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "496ce29bb5a52785b44e0f7ca2847ae0bb839c9bd28f69acac9b99d461c0c04c" + +[[package]] +name = "httpdate" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" + +[[package]] +name = "hyper" +version = "0.14.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b26ae0a80afebe130861d90abf98e3814a4f28a4c6ffeb5ab8ebb2be311e0ef2" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + +[[package]] +name = "idna" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "impl-codec" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "161ebdfec3c8e3b52bf61c4f3550a1eea4f9579d10dc1b936f3171ebdcd6c443" +dependencies = [ + "parity-scale-codec", +] + +[[package]] +name = "impl-trait-for-tuples" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11d7a9f6330b71fea57921c9b61c47ee6e84f72d394754eff6163ae67e7395eb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "indexmap" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f647032dfaa1f8b6dc29bd3edb7bbef4861b8b8007ebb118d6db284fd59f6ee" +dependencies = [ + "autocfg", + "hashbrown", +] + +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "ipnet" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b" + +[[package]] +name = "itertools" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9a9d19fa1e79b6215ff29b9d6880b706147f16e9b1dbb1e4e5947b5b02bc5e3" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d" + +[[package]] +name = "js-sys" +version = "0.3.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "671a26f820db17c2a2750743f1dd03bafd15b98c9f30c7c2628c024c05d73397" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5916d2ae698f6de9bfb891ad7a8d65c09d232dc58cc4ac433c7da3b2fd84bc2b" + +[[package]] +name = "lock_api" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "327fa5b6a6940e4699ec49a9beae1ea4845c6bab9314e4f84ac68742139d8c53" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "matches" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" + +[[package]] +name = "md-5" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "658646b21e0b72f7866c7038ab086d3d5e1cd6271f060fd37defb241949d0582" +dependencies = [ + "digest 0.10.3", +] + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2b29bd4bc3f33391105ebee3589c19197c4271e3e5a9ec9bfe8127eeff8f082" +dependencies = [ + "adler", +] + +[[package]] +name = "mio" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "713d550d9b44d89174e066b7a6217ae06234c10cb47819a88290d2b353c31799" +dependencies = [ + "libc", + "log", + "wasi 0.11.0+wasi-snapshot-preview1", + "windows-sys", +] + +[[package]] +name = "native-tls" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd7e2f3618557f980e0b17e8856252eee3c97fa12c54dff0ca290fb6266ca4a9" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "near-account-id" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b81ee2cf429b8bc04046d94f725a4f192fe7b13f42d76649d0177fd9ea719d8" +dependencies = [ + "borsh", + "serde", +] + +[[package]] +name = "near-chain-configs" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a830a8ea64328e4d2553873df848a97a90d111cb838ff5e199bfce79b62b6c2b" +dependencies = [ + "anyhow", + "chrono", + "derive_more", + "near-crypto", + "near-primitives", + "num-rational", + "serde", + "serde_json", + "sha2 0.9.9", + "smart-default", + "tracing", +] + +[[package]] +name = "near-chain-primitives" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff7984007ab1c73f101c5acf60a1e1f86a4afd47048df1e2412d4501794b43da" +dependencies = [ + "chrono", + "failure", + "failure_derive", + "log", + "near-crypto", + "near-primitives", + "thiserror", +] + +[[package]] +name = "near-chunks-primitives" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "838c06b137c7452001fc826f2defa6328b2695aaa720df2f0d08e31e54f849f6" +dependencies = [ + "near-chain-primitives", +] + +[[package]] +name = "near-client-primitives" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffc73ebfbb59defac9a17883467af98a14a37f53079dbcbdeec57c37d329de2b" +dependencies = [ + "actix", + "chrono", + "near-chain-configs", + "near-chain-primitives", + "near-chunks-primitives", + "near-crypto", + "near-network-primitives", + "near-primitives", + "strum", + "thiserror", +] + +[[package]] +name = "near-crypto" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d301d3ca4f59ab69c4b8ff625e7b2ef624345f69ab651d15c5fd59382096d2b1" +dependencies = [ + "arrayref", + "blake2", + "borsh", + "bs58", + "c2-chacha", + "curve25519-dalek", + "derive_more", + "ed25519-dalek", + "libc", + "near-account-id", + "once_cell", + "parity-secp256k1", + "primitive-types", + "rand 0.7.3", + "rand_core 0.5.1", + "serde", + "serde_json", + "subtle", + "thiserror", +] + +[[package]] +name = "near-jsonrpc-client" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf00d43006d0768aba1ff4d24386192a7aed2ad7548e9301055b6494917b2cc7" +dependencies = [ + "borsh", + "lazy_static", + "near-chain-configs", + "near-jsonrpc-primitives", + "near-primitives", + "reqwest", + "serde", + "serde_json", + "thiserror", + "uuid", +] + +[[package]] +name = "near-jsonrpc-primitives" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33e636f8862e0f5f565fb02b752e16aa676fc684e81682bfbb39a964defea627" +dependencies = [ + "actix", + "near-chain-configs", + "near-client-primitives", + "near-crypto", + "near-metrics", + "near-network-primitives", + "near-primitives", + "near-primitives-core", + "near-rpc-error-macro", + "once_cell", + "serde", + "serde_json", + "thiserror", + "tracing", + "uuid", +] + +[[package]] +name = "near-metrics" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df4074b56b24c73073b576bc27133182c8654d414be09ef666ad1e1d5ea8adb" +dependencies = [ + "lazy_static", + "log", + "prometheus", +] + +[[package]] +name = "near-network-primitives" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f63ddab6f0376a0ce870947e1ea2e98eba746af2e7edfba5dc127bbd2a2508d6" +dependencies = [ + "actix", + "actix_derive", + "anyhow", + "borsh", + "chrono", + "near-crypto", + "near-primitives", + "strum", + "tokio", + "tracing", +] + +[[package]] +name = "near-primitives" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad6a0b49d322ebda81ec4fdd99dca804526539673d0b12133286e0d7934f4802" +dependencies = [ + "borsh", + "byteorder", + "bytesize", + "chrono", + "derive_more", + "easy-ext", + "hex", + "near-crypto", + "near-primitives-core", + "near-rpc-error-macro", + "near-vm-errors", + "num-rational", + "primitive-types", + "rand 0.7.3", + "reed-solomon-erasure", + "serde", + "serde_json", + "smart-default", +] + +[[package]] +name = "near-primitives-core" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aaae2d2271ecdb0e3d90c11724cdb46d7d5bb50838fdb3941bfc381634cf08a3" +dependencies = [ + "base64 0.11.0", + "borsh", + "bs58", + "derive_more", + "near-account-id", + "num-rational", + "serde", + "sha2 0.9.9", +] + +[[package]] +name = "near-rpc-error-core" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5b9fd54b177a2efc1203cd605dc3369289d9c2a74ddbfdfb372feb13be38060" +dependencies = [ + "quote", + "serde", + "syn", +] + +[[package]] +name = "near-rpc-error-macro" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a38f5469f86f3c8ce0f39828f10a9c6aca2581d71a5c176a1c9be7e7eb4511a3" +dependencies = [ + "near-rpc-error-core", + "serde", + "syn", +] + +[[package]] +name = "near-vm-errors" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5092bbad4e16e48423d9be92e18b84af8e128c263df66d867a3f99c560b3cb28" +dependencies = [ + "borsh", + "near-account-id", + "near-rpc-error-macro", + "serde", +] + +[[package]] +name = "nom" +version = "7.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "num-bigint" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6f7833f2cbf2360a6cfd58cd41a53aa7a90bd4c202f5b1c7dd2ed73c57b2c3" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +dependencies = [ + "autocfg", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12ac428b1cb17fce6f731001d307d351ec70a6d202fc2e60f7d4c5e42d8f4f07" +dependencies = [ + "autocfg", + "num-bigint", + "num-integer", + "num-traits", + "serde", +] + +[[package]] +name = "num-traits" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "object" +version = "0.28.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e42c982f2d955fac81dd7e1d0e1426a7d702acd9c98d19ab01083a6a0328c424" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9" + +[[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" + +[[package]] +name = "openssl" +version = "0.10.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb81a6430ac911acb25fe5ac8f1d2af1b4ea8a4fdfda0f1ee4292af2e2d8eb0e" +dependencies = [ + "bitflags", + "cfg-if 1.0.0", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5fd19fb3e0a8191c1e34935718976a3e70c112ab9a24af6d7cadccd9d90bc0" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "parity-scale-codec" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373b1a4c1338d9cd3d1fa53b3a11bdab5ab6bd80a20f7f7becd76953ae2be909" +dependencies = [ + "arrayvec 0.7.2", + "bitvec", + "byte-slice-cast", + "impl-trait-for-tuples", + "parity-scale-codec-derive", + "serde", +] + +[[package]] +name = "parity-scale-codec-derive" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1557010476e0595c9b568d16dcfb81b93cdeb157612726f5170d31aa707bed27" +dependencies = [ + "proc-macro-crate 1.1.3", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "parity-secp256k1" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fca4f82fccae37e8bbdaeb949a4a218a1bbc485d11598f193d2a908042e5fc1" +dependencies = [ + "arrayvec 0.5.2", + "cc", + "cfg-if 0.1.10", + "rand 0.7.3", +] + +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core 0.8.5", +] + +[[package]] +name = "parking_lot" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58" +dependencies = [ + "lock_api", + "parking_lot_core 0.9.3", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216" +dependencies = [ + "cfg-if 1.0.0", + "instant", + "libc", + "redox_syscall", + "smallvec", + "winapi", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "redox_syscall", + "smallvec", + "windows-sys", +] + +[[package]] +name = "paste" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c520e05135d6e763148b6426a837e239041653ba7becd2e538c076c738025fc" + +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + +[[package]] +name = "pin-project-lite" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" + +[[package]] +name = "ppv-lite86" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" + +[[package]] +name = "primitive-types" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05e4722c697a58a99d5d06a08c30821d7c082a4632198de1eaa5a6c22ef42373" +dependencies = [ + "fixed-hash", + "impl-codec", + "uint", +] + +[[package]] +name = "proc-macro-crate" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d6ea3c4595b96363c13943497db34af4460fb474a95c43f4446ad341b8c9785" +dependencies = [ + "toml", +] + +[[package]] +name = "proc-macro-crate" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17d47ce914bf4de440332250b0edd23ce48c005f59fab39d3335866b114f11a" +dependencies = [ + "thiserror", + "toml", +] + +[[package]] +name = "proc-macro2" +version = "1.0.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9027b48e9d4c9175fa2218adf3557f91c1137021739951d4932f5f8268ac48aa" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "prometheus" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8425533e7122f0c3cc7a37e6244b16ad3a2cc32ae7ac6276e2a75da0d9c200d" +dependencies = [ + "cfg-if 1.0.0", + "fnv", + "lazy_static", + "parking_lot 0.11.2", + "protobuf", + "regex", + "thiserror", +] + +[[package]] +name = "protobuf" +version = "2.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf7e6d18738ecd0902d30d1ad232c9125985a3422929b16c65517b38adc14f96" + +[[package]] +name = "quote" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "radium" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "643f8f41a8ebc4c5dc4515c82bb8abd397b527fc20fd681b7c011c2aee5d44fb" + +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom 0.1.16", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.3", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.3", +] + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom 0.1.16", +] + +[[package]] +name = "rand_core" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" +dependencies = [ + "getrandom 0.2.6", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "redox_syscall" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62f25bc4c7e55e0b0b7a1d43fb893f4fa1361d0abe38b9ce4f323c2adfe6ef42" +dependencies = [ + "bitflags", +] + +[[package]] +name = "redox_users" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +dependencies = [ + "getrandom 0.2.6", + "redox_syscall", + "thiserror", +] + +[[package]] +name = "reed-solomon-erasure" +version = "4.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a415a013dd7c5d4221382329a5a3482566da675737494935cbbbcdec04662f9d" +dependencies = [ + "smallvec", +] + +[[package]] +name = "regex" +version = "1.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a11647b6b25ff05a515cb92c365cec08801e83423a235b51e231e1808747286" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" + +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + +[[package]] +name = "reqwest" +version = "0.11.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46a1f7aa4f35e5e8b4160449f51afc758f0ce6454315a9fa7d0d113e958c41eb" +dependencies = [ + "base64 0.13.0", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "lazy_static", + "log", + "mime", + "native-tls", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "tokio", + "tokio-native-tls", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + +[[package]] +name = "rust_decimal" +version = "1.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22dc69eadbf0ee2110b8d20418c0c6edbaefec2811c4963dc17b6344e11fe0f8" +dependencies = [ + "arrayvec 0.7.2", + "num-traits", + "serde", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342" + +[[package]] +name = "rustc-hex" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e75f6a532d0fd9f7f13144f392b6ad56a32696bfcd9c78f797f16bbb6f072d6" + +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver", +] + +[[package]] +name = "ryu" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3f6f92acf49d1b98f7a81226834412ada05458b7364277387724a237f062695" + +[[package]] +name = "schannel" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" +dependencies = [ + "lazy_static", + "winapi", +] + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "security-framework" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dc14f172faf8a0194a3aded622712b0de276821addc574fa54fc0a1167e10dc" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0160a13a177a45bfb43ce71c01580998474f556ad854dcbca936dd2841a5c556" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cb243bdfdb5936c8dc3c45762a19d12ab4550cdc753bc247637d4ec35a040fd" + +[[package]] +name = "serde" +version = "1.0.137" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61ea8d54c77f8315140a05f4c7237403bf38b72704d031543aa1d16abbf517d1" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.137" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f26faba0c3959972377d3b2d306ee9f71faee9714294e41bb777f83f88578be" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.81" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "service" +version = "0.1.0" +dependencies = [ + "async-trait", + "dotenv", + "envy", + "futures", + "lazy_static", + "near-jsonrpc-client", + "near-jsonrpc-primitives", + "near-primitives", + "num-traits", + "serde", + "sqlx", + "thiserror", + "tokio", +] + +[[package]] +name = "sha-1" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "028f48d513f9678cda28f6e4064755b3fbb2af6acd672f2c209b62323f7aea0f" +dependencies = [ + "cfg-if 1.0.0", + "cpufeatures", + "digest 0.10.3", +] + +[[package]] +name = "sha2" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" +dependencies = [ + "block-buffer 0.9.0", + "cfg-if 1.0.0", + "cpufeatures", + "digest 0.9.0", + "opaque-debug", +] + +[[package]] +name = "sha2" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55deaec60f81eefe3cce0dc50bda92d6d8e88f2a27df7c5033b42afeb1ed2676" +dependencies = [ + "cfg-if 1.0.0", + "cpufeatures", + "digest 0.10.3", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" +dependencies = [ + "libc", +] + +[[package]] +name = "signature" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f054c6c1a6e95179d6f23ed974060dcefb2d9388bb7256900badad682c499de4" + +[[package]] +name = "slab" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb703cfe953bccee95685111adeedb76fabe4e97549a58d16f03ea7b9367bb32" + +[[package]] +name = "smallvec" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83" + +[[package]] +name = "smart-default" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "133659a15339456eeeb07572eb02a91c91e9815e9cbc89566944d2c8d3efdbf6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "socket2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66d72b759436ae32898a2af0a14218dbf55efde3feeb170eb623637db85ee1e0" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "sqlformat" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4b7922be017ee70900be125523f38bdd644f4f06a1b16e8fa5a8ee8c34bffd4" +dependencies = [ + "itertools", + "nom", + "unicode_categories", +] + +[[package]] +name = "sqlx" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "551873805652ba0d912fec5bbb0f8b4cdd96baf8e2ebf5970e5671092966019b" +dependencies = [ + "sqlx-core", + "sqlx-macros", +] + +[[package]] +name = "sqlx-core" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e48c61941ccf5ddcada342cd59e3e5173b007c509e1e8e990dafc830294d9dc5" +dependencies = [ + "ahash", + "atoi", + "base64 0.13.0", + "bitflags", + "byteorder", + "bytes", + "crc", + "crossbeam-queue", + "dirs", + "either", + "event-listener", + "futures-channel", + "futures-core", + "futures-intrusive", + "futures-util", + "hashlink", + "hex", + "hkdf", + "hmac", + "indexmap", + "itoa", + "libc", + "log", + "md-5", + "memchr", + "num-bigint", + "once_cell", + "paste", + "percent-encoding", + "rand 0.8.5", + "rust_decimal", + "serde", + "serde_json", + "sha-1", + "sha2 0.10.2", + "smallvec", + "sqlformat", + "sqlx-rt", + "stringprep", + "thiserror", + "tokio-stream", + "url", + "whoami", +] + +[[package]] +name = "sqlx-macros" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc0fba2b0cae21fc00fe6046f8baa4c7fcb49e379f0f592b04696607f69ed2e1" +dependencies = [ + "dotenv", + "either", + "heck 0.4.0", + "once_cell", + "proc-macro2", + "quote", + "sha2 0.10.2", + "sqlx-core", + "sqlx-rt", + "syn", + "url", +] + +[[package]] +name = "sqlx-rt" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4db708cd3e459078f85f39f96a00960bd841f66ee2a669e90bf36907f5a79aae" +dependencies = [ + "native-tls", + "once_cell", + "tokio", + "tokio-native-tls", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "stringprep" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee348cb74b87454fff4b551cbf727025810a004f88aeacae7f85b87f4e9a1c1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "strum" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7318c509b5ba57f18533982607f24070a55d353e90d4cae30c467cdb2ad5ac5c" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee8bc6b87a5112aeeab1f4a9f7ab634fe6cbefc4850006df31267f4cfb9e3149" +dependencies = [ + "heck 0.3.3", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "subtle" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" + +[[package]] +name = "syn" +version = "1.0.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a07e33e919ebcd69113d5be0e4d70c5707004ff45188910106854f38b960df4a" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "synstructure" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "unicode-xid", +] + +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + +[[package]] +name = "tempfile" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" +dependencies = [ + "cfg-if 1.0.0", + "fastrand", + "libc", + "redox_syscall", + "remove_dir_all", + "winapi", +] + +[[package]] +name = "thiserror" +version = "1.0.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd829fe32373d27f76265620b5309d0340cb8550f523c1dda251d6298069069a" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0396bc89e626244658bef819e22d0cc459e795a5ebe878e6ec336d1674a8d79a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "time" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + +[[package]] +name = "tokio" +version = "1.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4903bf0427cf68dddd5aa6a93220756f8be0c34fcfa9f5e6191e103e15a31395" +dependencies = [ + "bytes", + "libc", + "memchr", + "mio", + "num_cpus", + "once_cell", + "parking_lot 0.12.0", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "winapi", +] + +[[package]] +name = "tokio-macros" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b557f72f448c511a979e2564e55d74e6c4432fc96ff4f6241bc6bded342643b7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50145484efff8818b5ccd256697f36863f587da82cf8b409c53adf1e840798e3" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.6.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36943ee01a6d67977dd3f84a5a1d2efeb4ada3a1ae771cadfaa535d9d9fc6507" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "log", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f988a1a1adc2fb21f9c12aa96441da33a1728193ae0b95d2be22dbd17fcb4e5c" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "toml" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" +dependencies = [ + "serde", +] + +[[package]] +name = "tower-service" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" + +[[package]] +name = "tracing" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0ecdcb44a79f0fe9844f0c4f33a342cbcbb5117de8001e6ba0dc2351327d09" +dependencies = [ + "cfg-if 1.0.0", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6b8ad3567499f98a1db7a752b07a7c8c7c7c34c332ec00effb2b0027974b7c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f54c8ca710e81886d498c2fd3331b56c93aa248d49de2222ad2742247c60072f" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + +[[package]] +name = "typenum" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" + +[[package]] +name = "uint" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f03af7ccf01dd611cc450a0d10dbc9b745770d096473e2faf0ca6e2d66d1e0" +dependencies = [ + "byteorder", + "crunchy", + "hex", + "static_assertions", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" + +[[package]] +name = "unicode-normalization" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d54590932941a9e9266f0832deed84ebe1bf2e4c9e4a3554d393d18f5e854bf9" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e8820f5d777f6224dc4be3632222971ac30164d4a258d595640799554ebfd99" + +[[package]] +name = "unicode-xid" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04" + +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + +[[package]] +name = "url" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c" +dependencies = [ + "form_urlencoded", + "idna", + "matches", + "percent-encoding", +] + +[[package]] +name = "uuid" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" +dependencies = [ + "getrandom 0.2.6", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + +[[package]] +name = "wasi" +version = "0.10.2+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27370197c907c55e3f1a9fbe26f44e937fe6451368324e009cba39e139dc08ad" +dependencies = [ + "cfg-if 1.0.0", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e04185bfa3a779273da532f5025e33398409573f348985af9a1cbf3774d3f4" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f741de44b75e14c35df886aff5f1eb73aa114fa5d4d00dcd37b5e01259bf3b2" +dependencies = [ + "cfg-if 1.0.0", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17cae7ff784d7e83a2fe7611cfe766ecf034111b49deb850a3dc7699c08251f5" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99ec0dc7a4756fffc231aab1b9f2f578d23cd391390ab27f952ae0c9b3ece20b" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d554b7f530dee5964d9a9468d95c1f8b8acae4f282807e7d27d4b03099a46744" + +[[package]] +name = "web-sys" +version = "0.3.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b17e741662c70c8bd24ac5c5b18de314a2c26c32bf8346ee1e6f53de919c283" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "whoami" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524b58fa5a20a2fb3014dd6358b70e6579692a56ef6fce928834e488f42f65e8" +dependencies = [ + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" +dependencies = [ + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" + +[[package]] +name = "windows_i686_gnu" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" + +[[package]] +name = "windows_i686_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" + +[[package]] +name = "winreg" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +dependencies = [ + "winapi", +] + +[[package]] +name = "wyz" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85e60b0d1b5f99db2556934e21937020776a5d31520bf169e851ac44e6420214" + +[[package]] +name = "zeroize" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4756f7db3f7b5574938c3eb1c117038b8e07f95ee6718c0efad4ac21508f1efd" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f8f187641dad4f680d25c4bfc4225b418165984179f26ca76ec4fb6441d3a17" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..d5af5cb --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,4 @@ +[workspace] +members = [ + "service" +] diff --git a/service/Cargo.toml b/service/Cargo.toml new file mode 100644 index 0000000..173f451 --- /dev/null +++ b/service/Cargo.toml @@ -0,0 +1,21 @@ +[package] +edition = "2021" +name = "service" +version = "0.1.0" + +[dependencies] +async-trait = "0.1.53" +dotenv = "0.15.0" +envy = "0.4.2" +futures = "0.3.21" +lazy_static = "1.4.0" +near-jsonrpc-client = "0.3.0" +near-jsonrpc-primitives = "0.12.0" +near-primitives = "0.12.0" +num-traits = "0.2.15" +serde = "1.0.137" +sqlx = {version = "0.5", features = ["runtime-tokio-native-tls", "postgres", "decimal"]} +thiserror = "1.0.31" +tokio = {version = "1", features = ["full"]} + +[features] diff --git a/service/build.rs b/service/build.rs new file mode 100644 index 0000000..25c3936 --- /dev/null +++ b/service/build.rs @@ -0,0 +1,4 @@ +fn main() { + // trigger recompilation when a new migration is added + println!("cargo:rerun-if-changed=migrations"); +} diff --git a/service/migrations/20220516203513_initial.down.sql b/service/migrations/20220516203513_initial.down.sql new file mode 100644 index 0000000..f9590e1 --- /dev/null +++ b/service/migrations/20220516203513_initial.down.sql @@ -0,0 +1,3 @@ +DROP TABLE account, account_badge CASCADE; + +DROP FUNCTION IF EXISTS update_modified_column; diff --git a/service/migrations/20220516203513_initial.up.sql b/service/migrations/20220516203513_initial.up.sql new file mode 100644 index 0000000..4974f2a --- /dev/null +++ b/service/migrations/20220516203513_initial.up.sql @@ -0,0 +1,47 @@ +CREATE TABLE account ( + id text NOT NULL, + balance numeric(45,0), + score integer +); + +CREATE TABLE account_badge ( + id uuid DEFAULT gen_random_uuid() NOT NULL, + account_id text NOT NULL, + badge_id uuid NOT NULL +); + + +ALTER TABLE ONLY account_badge + ADD CONSTRAINT pk__account_badge PRIMARY KEY (id); + +ALTER TABLE ONLY account + ADD CONSTRAINT pk__account PRIMARY KEY (id); + +ALTER TABLE ONLY account_badge + ADD CONSTRAINT ux__account_badge__account_id__badge_id UNIQUE (account_id, badge_id); + +CREATE INDEX ix__account__balance ON account USING btree (balance DESC NULLS LAST); + +CREATE INDEX ix__account__score ON account USING btree (score DESC NULLS LAST); + +CREATE INDEX ix__account_badge__badge_id ON account_badge USING btree (badge_id); + +ALTER TABLE ONLY account_badge + ADD CONSTRAINT fk__account_badge__account FOREIGN KEY (account_id) REFERENCES account(id); + +ALTER TABLE IF EXISTS account + ADD COLUMN modified timestamp without time zone DEFAULT current_timestamp; + +-- https://stackoverflow.com/a/26284695 +create or replace function update_modified_column() +returns trigger as $$ +begin + new.modified = now(); + return new; +end; +$$ language 'plpgsql'; + +create trigger update_account_modified + before update on account + for each row + execute procedure update_modified_column(); diff --git a/service/rustfmt.toml b/service/rustfmt.toml new file mode 100644 index 0000000..b6f799d --- /dev/null +++ b/service/rustfmt.toml @@ -0,0 +1 @@ +tab_spaces = 4 diff --git a/service/src/badge/mod.rs b/service/src/badge/mod.rs new file mode 100644 index 0000000..f4f4037 --- /dev/null +++ b/service/src/badge/mod.rs @@ -0,0 +1,157 @@ +use std::{ + collections::{HashMap, HashSet}, + pin::Pin, +}; + +use async_trait::async_trait; +use futures::{Future, future::BoxFuture}; +use lazy_static::lazy_static; +use near_jsonrpc_client::JsonRpcClient; +use near_primitives::types::AccountId; +use sqlx::PgPool; + +use self::transfer::Transfer; + +pub type BadgeId = &'static str; + +#[derive(Clone)] +pub struct BadgeCheckResult { + pub awarded: HashSet, + pub checked: HashSet, +} + +pub type BadgeChecker = + fn( + indexer_pool: &PgPool, + rpc_client: &JsonRpcClient, + account_id: &AccountId, + ) -> Pin>>>>; + +pub struct BadgeRegistry { + registry: HashMap, +} + +impl BadgeRegistry { + pub fn new() -> Self { + Self { + registry: HashMap::new(), + } + } + + pub fn register(&mut self, badge_id: BadgeId, checker: BadgeChecker) { + if self.registry.contains_key(&badge_id) { + return; + } + + self.registry.insert(badge_id, checker); + } + + pub async fn check_account( + &self, + account_id: &AccountId, + indexer_pool: &PgPool, + rpc_client: &JsonRpcClient, + ignore_badge_ids: &HashSet, + ) -> BadgeCheckResult { + let mut ignore_badge_ids = ignore_badge_ids.clone(); + let mut awarded = HashSet::::new(); + let mut checked = HashSet::::new(); + + for badge_id in self.registry.keys() { + if ignore_badge_ids.contains(badge_id) { + continue; + } + + let checker = self.registry[badge_id]; + let result = checker(indexer_pool, rpc_client, account_id).await; + if let Ok(result) = result { + awarded.extend(result.awarded); + checked.extend(&result.checked); + ignore_badge_ids.extend(result.checked); + } // TODO: Err case + } + + BadgeCheckResult { awarded, checked } + } +} + +// pub fn badges() -> Vec> { +// vec![Box::new(Transfer)] +// } + +#[async_trait] +pub trait Badge { + fn badge_ids() -> &'static [&'static str]; + async fn check_account( + indexer_pool: &PgPool, + rpc_client: &JsonRpcClient, + account_id: &AccountId, + ) -> Vec<&'static str>; +} + +pub struct BadgeStruct<'a, 'b, Fut, E> +where + Fut: Future> + Send + Sync, + E: std::error::Error, + 'b: 'a, +{ + + pub badges: HashSet, + pub checker: + Box Fut>, +} + +#[async_trait] +pub trait BadgeTrait<'b> { + async fn check( + &'b self, + indexer_pool: &'b PgPool, + rpc_client: &'b JsonRpcClient, + account_id: &'b AccountId, + ) -> Result>; +} + +// impl BadgeTrait for String { +// fn check<'life0, 'life1, 'life2, 'life3, 'async_trait>( +// &'life0 self, +// indexer_pool: &'life1 PgPool, +// rpc_client: &'life2 JsonRpcClient, +// account_id: &'life3 AccountId, +// ) -> core::pin::Pin< +// Box< +// dyn core::future::Future>> +// + core::marker::Send +// + 'async_trait, +// >, +// > +// where +// 'life0: 'async_trait, +// 'life1: 'async_trait, +// 'life2: 'async_trait, +// 'life3: 'async_trait, +// Self: 'async_trait, +// { +// todo!() +// } +// } + +#[async_trait] +impl BadgeTrait<'b> for BadgeStruct<'a, 'b, Fut, E> +where + Fut: Future> + Send + Sync, + E: std::error::Error + 'static, + 'b: 'a, +{ + async fn check( + &'b self, + indexer_pool: &'b PgPool, + rpc_client: &'b JsonRpcClient, + account_id: &'b AccountId, + ) -> Result> { + (self.checker)(indexer_pool, rpc_client, account_id) + .await + .map_err(|e| Box::new(e) as Box) + } +} + +pub mod transfer; diff --git a/service/src/badge/transfer.rs b/service/src/badge/transfer.rs new file mode 100644 index 0000000..2b4fe20 --- /dev/null +++ b/service/src/badge/transfer.rs @@ -0,0 +1,198 @@ +use std::{collections::HashSet, pin::Pin, time::Duration}; + +use crate::badge::BadgeCheckResult; + +use super::{Badge, BadgeStruct, BadgeTrait}; +use async_trait::async_trait; +use futures::{future::BoxFuture, Future, FutureExt}; +use near_jsonrpc_client::JsonRpcClient; +use near_primitives::types::AccountId; +use sqlx::{postgres::PgRow, PgPool}; +use thiserror::Error; +use tokio::time::{sleep_until, Instant}; + +pub struct Transfer; + +pub static TRANSFER_1: &'static str = "b36c3dd2-b8b0-4098-b227-63290f009668"; +pub static TRANSFER_10: &'static str = "609b2017-9534-4737-b86b-6ee4897fc4f9"; +pub static TRANSFER_100: &'static str = "64ce9af9-52ac-49dc-96fd-031b4fa2efad"; +pub static BADGE_IDS: &'static [&'static str] = &[TRANSFER_1, TRANSFER_10, TRANSFER_100]; + +#[async_trait] +impl Badge for Transfer { + fn badge_ids() -> &'static [&'static str] { + BADGE_IDS + } + + async fn check_account( + indexer_pool: &PgPool, + _: &JsonRpcClient, + account_id: &AccountId, + ) -> Vec<&'static str> { + #[derive(sqlx::FromRow)] + struct WithResult { + pub result: i64, + } + + let result = sqlx::query_as::<_, WithResult>( + r#" +select count(*) as result + from action_receipt_actions + where action_kind = 'TRANSFER' + and receipt_predecessor_account_id = $1 + "#, + ) + .bind(account_id.to_string()) + .fetch_one(indexer_pool) + .await; + + match result { + Ok(result) => { + let total = result.result; + if total >= 100 { + BADGE_IDS.to_vec() + } else if total >= 10 { + BADGE_IDS[..2].to_vec() + } else if total >= 1 { + BADGE_IDS[..1].to_vec() + } else { + vec![] + } + } + _ => vec![], + } + } +} + +#[derive(Error, Debug)] +enum TransferBadgeError { + #[error("Query failure: {0}")] + QueryFailure(#[from] sqlx::Error), +} + +pub async fn check_account( + indexer_pool: &PgPool, + _: &JsonRpcClient, + account_id: &AccountId, +) -> Result { + #[derive(sqlx::FromRow)] + struct WithResult { + pub result: i64, + } + + let result = sqlx::query_as::<_, WithResult>( + r#" +select count(*) as result + from action_receipt_actions + where action_kind = 'TRANSFER' + and receipt_predecessor_account_id = $1 + "#, + ) + .bind(account_id.to_string()) + .fetch_one(indexer_pool) + .await; + + result.map(|result| { + let total = result.result; + let awarded = if total >= 100 { + BADGE_IDS + } else if total >= 10 { + &BADGE_IDS[..2] + } else if total >= 1 { + &BADGE_IDS[..1] + } else { + &[] + } + .to_vec(); + + BadgeCheckResult { + awarded: awarded.into_iter().collect(), + checked: BADGE_IDS.to_vec().into_iter().collect(), + } + }) +} + +fn test() { + // let x = BadgeStruct { + // checker: Box::new(|indexer_pool, _, account_id| async move { + // #[derive(sqlx::FromRow)] + // struct WithResult { + // pub result: i64, + // } + + // sqlx::query_as::<_, WithResult>( + // r#" + // select count(*) as result + // from action_receipt_actions + // where action_kind = 'TRANSFER' + // and receipt_predecessor_account_id = $1 + // "#, + // ) + // .bind(account_id.to_string()) + // .fetch_one(indexer_pool) + // .map(|result| { + // result + // .map(|r| { + // let total = r.result; + // let awarded = if total >= 100 { + // BADGE_IDS + // } else if total >= 10 { + // &BADGE_IDS[..2] + // } else if total >= 1 { + // &BADGE_IDS[..1] + // } else { + // &[] + // } + // .to_vec(); + + // BadgeCheckResult { + // awarded: awarded.into_iter().collect(), + // checked: BADGE_IDS.to_vec().into_iter().collect(), + // } + // }) + // .map_err(|e| TransferBadgeError::from(e)) + // }) + // .boxed() + // .shared() + // .await + + // // result.map(|result| { + // // let total = result.result; + // // let awarded = if total >= 100 { + // // BADGE_IDS + // // } else if total >= 10 { + // // &BADGE_IDS[..2] + // // } else if total >= 1 { + // // &BADGE_IDS[..1] + // // } else { + // // &[] + // // } + // // .to_vec(); + + // // BadgeCheckResult { + // // awarded: awarded.into_iter().collect(), + // // checked: BADGE_IDS.to_vec().into_iter().collect(), + // // } + // // }) + // }), + // badges: HashSet::new(), + // }; + + let y = BadgeStruct { + checker: Box::new(|_, _, _| async { + Ok(BadgeCheckResult { + awarded: HashSet::new(), + checked: HashSet::new(), + }) as Result<_, TransferBadgeError> + }), + badges: HashSet::new(), + }; + + // let t: Box = Box::new(x); + + // let x: Vec> = vec![Box::new(x), Box::new(y)]; +} + +fn x(y: Box, sqlx::Error>> + Sync>, z: Box) { + let x: Box = Box::new(y); +} diff --git a/service/src/indexer.rs b/service/src/indexer.rs new file mode 100644 index 0000000..e410ce1 --- /dev/null +++ b/service/src/indexer.rs @@ -0,0 +1,79 @@ +use futures::{StreamExt, TryFutureExt, TryStreamExt}; +use sqlx::PgPool; + +pub async fn get_recent_actors( + pool: &PgPool, + timestamp_nanoseconds: u64, +) -> Result, sqlx::Error> { + #[derive(sqlx::FromRow)] + struct AccountId { + pub account_id: String, + } + + sqlx::query_as::<_, AccountId>( + " +select distinct predecessor_account_id as account_id + from receipts + where included_in_block_timestamp > $1 + and length(predecessor_account_id) != 64 +union + select distinct receiver_account_id as account_id + from receipts + where included_in_block_timestamp > $1 + and length(receiver_account_id) != 64 +", + ) + .bind(timestamp_nanoseconds as i64) // for some reason Encode is not implemented for u64 on Postgres + .fetch(pool) + .map(|i| i.map(|a| a.account_id)) + .try_collect::>() + .await +} + +pub async fn calculate_account_score( + pool: &PgPool, + account_id: &str, +) -> Result { + #[derive(sqlx::FromRow)] + struct WithResult { + pub result: i64, + } + + sqlx::query_as::<_, WithResult>( + r#" +select coalesce(sum( + case + when action_kind = 'TRANSFER' + and signer_account_id = $1 + then 10 + when action_kind = 'TRANSFER' + and tx.receiver_account_id = $1 + then 2 + when action_kind = 'CREATE_ACCOUNT' + and signer_account_id = $1 + then 50 + when action_kind = 'FUNCTION_CALL' + and signer_account_id = $1 + then 10 + when action_kind = 'DEPLOY_CONTRACT' + and signer_account_id = $1 + then 100 + else 0 + end +), 0) as result +from ( + select * + from transactions + where (transactions.signer_account_id = $1 + or transactions.receiver_account_id = $1) + order by block_timestamp desc +) tx +inner join receipts on tx.converted_into_receipt_id = receipts.receipt_id +left outer join transaction_actions on tx.transaction_hash = transaction_actions.transaction_hash +"#, + ) + .bind(account_id) + .fetch_one(pool) + .map_ok(|a| a.result as u32) + .await +} diff --git a/service/src/main.rs b/service/src/main.rs new file mode 100644 index 0000000..2080492 --- /dev/null +++ b/service/src/main.rs @@ -0,0 +1,69 @@ +use dotenv::dotenv; + +use near_jsonrpc_client::JsonRpcClient; +use serde::Deserialize; +use sqlx::{migrate, postgres::PgPoolOptions}; + +use crate::badge::{BadgeRegistry, transfer}; + +#[derive(Deserialize)] +struct Configuration { + #[allow(unused)] // env var read by default by sqlx + pub database_url: String, + pub indexer_url: String, + pub rpc_url: String, +} + +mod badge; +mod indexer; +mod rpc; +mod updater; + +#[tokio::main] +async fn main() -> Result<(), Box> { + dotenv().ok(); + migrate!(); + + let config = envy::from_env::().expect("Missing environment variables"); + + let local_pool = PgPoolOptions::new() + .max_connections(5) + .connect(&config.database_url) + .await?; + + let indexer_pool = PgPoolOptions::new() + .max_connections(5) + .connect(&config.indexer_url) + .await?; + + let jsonrpc_client = JsonRpcClient::connect(&config.rpc_url); + + let mut badge_registry = BadgeRegistry::new(); + + // for badge_id in transfer::BADGE_IDS { + // badge_registry.register(badge_id, transfer::check_account); + // } + + + // updater::update_account( + // &local_pool, + // &indexer_pool, + // &jsonrpc_client, + // "hatchet.near".parse().unwrap(), + // ).await.unwrap(); + + // let result = indexer::get_recent_actors(&indexer_pool, 1652741757312000000).await?; + + // println!("{}", &result.len()); + + // let mut result = sqlx::query("select * from accounts limit 10").fetch(&pool); + + // while let Some(row) = result.try_next().await? { + // let account_id: Result = row.try_get("account_id"); + // if let Ok(account_id) = account_id { + // println!("Account ID: {account_id}"); + // } + // } + + Ok(()) +} diff --git a/service/src/rpc.rs b/service/src/rpc.rs new file mode 100644 index 0000000..3290758 --- /dev/null +++ b/service/src/rpc.rs @@ -0,0 +1,26 @@ +use near_jsonrpc_client::{ + errors::JsonRpcError, + methods::{self, query::RpcQueryError}, + JsonRpcClient, +}; +use near_jsonrpc_primitives::types::query::QueryResponseKind; +use near_primitives::{ + types::{AccountId, BlockReference, Finality}, + views::QueryRequest, +}; + +pub async fn get_account_balance( + client: &JsonRpcClient, + account_id: AccountId, +) -> Result> { + client + .call(methods::query::RpcQueryRequest { + block_reference: BlockReference::Finality(Finality::Final), + request: QueryRequest::ViewAccount { account_id }, + }) + .await + .map(|r| match r.kind { + QueryResponseKind::ViewAccount(view_account) => view_account.amount, + _ => 0, // unlikely unless RPC malfunction + }) +} diff --git a/service/src/updater.rs b/service/src/updater.rs new file mode 100644 index 0000000..de7b0d7 --- /dev/null +++ b/service/src/updater.rs @@ -0,0 +1,50 @@ +use near_jsonrpc_client::{errors::JsonRpcError, methods::query::RpcQueryError, JsonRpcClient}; +use near_primitives::types::AccountId; +use num_traits::FromPrimitive; +use sqlx::{types::Decimal, PgPool}; +use tokio::join; + +use crate::{indexer::calculate_account_score, rpc::get_account_balance}; + +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum UpdateAccountError { + #[error("Error calculating score: {0}")] + ScoreError(sqlx::Error), + #[error("Error retrieving balance: {0}")] + BalanceError(#[from] JsonRpcError), + #[error("Error running account update query: {0}")] + InsertQueryError(#[from] sqlx::Error), +} + +pub async fn update_account( + local_pool: &PgPool, + indexer_pool: &PgPool, + jsonrpc_client: &JsonRpcClient, + account_id: AccountId, +) -> Result<(), UpdateAccountError> { + let (score, balance) = join!( + calculate_account_score(indexer_pool, &account_id), + get_account_balance(jsonrpc_client, account_id.clone()) + ); + + let score = score.map_err(|e| UpdateAccountError::ScoreError(e))?; + let balance = Decimal::from_u128(balance?); + + sqlx::query!( + " +INSERT INTO account(id, balance, score) + VALUES ($1, $2, $3) + ON CONFLICT (id) DO + UPDATE SET balance = EXCLUDED.balance, score = EXCLUDED.score +", + account_id.to_string(), + balance, + score as i64 + ) + .execute(local_pool) + .await?; + + Ok(()) +} From 17d6f5bc518898475ccb1486f858b89dd61c349f Mon Sep 17 00:00:00 2001 From: Jacob Date: Thu, 19 May 2022 13:29:35 -0500 Subject: [PATCH 02/31] trying with tokio::spawn services instead --- service/src/badge/mod.rs | 79 +++++++++++++++++++++-------------- service/src/badge/transfer.rs | 63 ++++++++++++++++++++++++++-- 2 files changed, 108 insertions(+), 34 deletions(-) diff --git a/service/src/badge/mod.rs b/service/src/badge/mod.rs index f4f4037..7726b75 100644 --- a/service/src/badge/mod.rs +++ b/service/src/badge/mod.rs @@ -4,18 +4,19 @@ use std::{ }; use async_trait::async_trait; -use futures::{Future, future::BoxFuture}; -use lazy_static::lazy_static; +use futures::Future; + use near_jsonrpc_client::JsonRpcClient; use near_primitives::types::AccountId; use sqlx::PgPool; -use self::transfer::Transfer; +use tokio::sync::mpsc::{Receiver, Sender}; pub type BadgeId = &'static str; #[derive(Clone)] pub struct BadgeCheckResult { + pub account_id: AccountId, pub awarded: HashSet, pub checked: HashSet, } @@ -27,6 +28,23 @@ pub type BadgeChecker = account_id: &AccountId, ) -> Pin>>>>; +pub trait BadgeStart { + fn run( + &self, + indexer_pool: &PgPool, + rpc_client: &JsonRpcClient, + input: tokio::sync::watch::Receiver, + output: tokio::sync::watch::Sender, + ); +} + +pub type BadgeStartFn = fn( + indexer_pool: &PgPool, + rpc_client: &JsonRpcClient, + input: tokio::sync::watch::Receiver, + output: tokio::sync::watch::Sender, +); + pub struct BadgeRegistry { registry: HashMap, } @@ -46,33 +64,33 @@ impl BadgeRegistry { self.registry.insert(badge_id, checker); } - pub async fn check_account( - &self, - account_id: &AccountId, - indexer_pool: &PgPool, - rpc_client: &JsonRpcClient, - ignore_badge_ids: &HashSet, - ) -> BadgeCheckResult { - let mut ignore_badge_ids = ignore_badge_ids.clone(); - let mut awarded = HashSet::::new(); - let mut checked = HashSet::::new(); - - for badge_id in self.registry.keys() { - if ignore_badge_ids.contains(badge_id) { - continue; - } - - let checker = self.registry[badge_id]; - let result = checker(indexer_pool, rpc_client, account_id).await; - if let Ok(result) = result { - awarded.extend(result.awarded); - checked.extend(&result.checked); - ignore_badge_ids.extend(result.checked); - } // TODO: Err case - } - - BadgeCheckResult { awarded, checked } - } + // pub async fn check_account( + // &self, + // account_id: &AccountId, + // indexer_pool: &PgPool, + // rpc_client: &JsonRpcClient, + // ignore_badge_ids: &HashSet, + // ) -> BadgeCheckResult { + // let mut ignore_badge_ids = ignore_badge_ids.clone(); + // let mut awarded = HashSet::::new(); + // let mut checked = HashSet::::new(); + + // for badge_id in self.registry.keys() { + // if ignore_badge_ids.contains(badge_id) { + // continue; + // } + + // let checker = self.registry[badge_id]; + // let result = checker(indexer_pool, rpc_client, account_id).await; + // if let Ok(result) = result { + // awarded.extend(result.awarded); + // checked.extend(&result.checked); + // ignore_badge_ids.extend(result.checked); + // } // TODO: Err case + // } + + // BadgeCheckResult { awarded, checked } + // } } // pub fn badges() -> Vec> { @@ -95,7 +113,6 @@ where E: std::error::Error, 'b: 'a, { - pub badges: HashSet, pub checker: Box Fut>, diff --git a/service/src/badge/transfer.rs b/service/src/badge/transfer.rs index 2b4fe20..47f3748 100644 --- a/service/src/badge/transfer.rs +++ b/service/src/badge/transfer.rs @@ -2,7 +2,7 @@ use std::{collections::HashSet, pin::Pin, time::Duration}; use crate::badge::BadgeCheckResult; -use super::{Badge, BadgeStruct, BadgeTrait}; +use super::{Badge, BadgeStruct, BadgeTrait, BadgeStartFn}; use async_trait::async_trait; use futures::{future::BoxFuture, Future, FutureExt}; use near_jsonrpc_client::JsonRpcClient; @@ -106,6 +106,7 @@ select count(*) as result .to_vec(); BadgeCheckResult { + account_id: account_id.clone(), awarded: awarded.into_iter().collect(), checked: BADGE_IDS.to_vec().into_iter().collect(), } @@ -181,6 +182,7 @@ fn test() { let y = BadgeStruct { checker: Box::new(|_, _, _| async { Ok(BadgeCheckResult { + account_id: "account".parse().unwrap(), awarded: HashSet::new(), checked: HashSet::new(), }) as Result<_, TransferBadgeError> @@ -193,6 +195,61 @@ fn test() { // let x: Vec> = vec![Box::new(x), Box::new(y)]; } -fn x(y: Box, sqlx::Error>> + Sync>, z: Box) { - let x: Box = Box::new(y); +fn badge_checker_fn( + indexer_pool: &PgPool, + rpc_client: &JsonRpcClient, + input: &mut tokio::sync::watch::Receiver, + output: tokio::sync::watch::Sender, +) { + #[derive(sqlx::FromRow)] + struct WithResult { + pub result: i64, + } + + + tokio::spawn(async move { + // while let Some(x) = input.changed().await {} + + + sqlx::query_as::<_, WithResult>( + r#" + select count(*) as result + from action_receipt_actions + where action_kind = 'TRANSFER' + and receipt_predecessor_account_id = $1 + "#, + ) + .bind(account_id.to_string()) + .fetch_one(indexer_pool) + .map(|result| { + result + .map(|r| { + let total = r.result; + let awarded = if total >= 100 { + BADGE_IDS + } else if total >= 10 { + &BADGE_IDS[..2] + } else if total >= 1 { + &BADGE_IDS[..1] + } else { + &[] + } + .to_vec(); + + BadgeCheckResult { + account_id, + awarded: awarded.into_iter().collect(), + checked: BADGE_IDS.to_vec().into_iter().collect(), + } + }) + .map_err(|e| TransferBadgeError::from(e)) + }); + .boxed() + .shared() + .await + }); +} + +fn t() { + let x: BadgeStartFn = badge_checker_fn; } From f3e634058dde99761e97a34c59433841768a09ea Mon Sep 17 00:00:00 2001 From: Jacob Date: Thu, 19 May 2022 23:50:34 -0500 Subject: [PATCH 03/31] badge registry working --- Cargo.lock | 1 + service/Cargo.toml | 1 + service/src/badge/mod.rs | 88 +++++++++++-- service/src/badge/transfer.rs | 238 ++++++---------------------------- service/src/indexer.rs | 4 +- service/src/main.rs | 63 ++++++--- 6 files changed, 165 insertions(+), 230 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 592c6de..c9a0f02 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1993,6 +1993,7 @@ name = "service" version = "0.1.0" dependencies = [ "async-trait", + "chrono", "dotenv", "envy", "futures", diff --git a/service/Cargo.toml b/service/Cargo.toml index 173f451..2cfba99 100644 --- a/service/Cargo.toml +++ b/service/Cargo.toml @@ -5,6 +5,7 @@ version = "0.1.0" [dependencies] async-trait = "0.1.53" +chrono = "0.4.19" dotenv = "0.15.0" envy = "0.4.2" futures = "0.3.21" diff --git a/service/src/badge/mod.rs b/service/src/badge/mod.rs index 7726b75..de3ae72 100644 --- a/service/src/badge/mod.rs +++ b/service/src/badge/mod.rs @@ -1,6 +1,7 @@ use std::{ collections::{HashMap, HashSet}, pin::Pin, + sync::{atomic::{AtomicUsize, Ordering}, Arc}, }; use async_trait::async_trait; @@ -10,11 +11,15 @@ use near_jsonrpc_client::JsonRpcClient; use near_primitives::types::AccountId; use sqlx::PgPool; -use tokio::sync::mpsc::{Receiver, Sender}; +use tokio::sync::{ + broadcast, + mpsc::{Receiver, Sender}, Semaphore, +}; pub type BadgeId = &'static str; +pub type BadgeCheckerRegistrationId = usize; -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct BadgeCheckResult { pub account_id: AccountId, pub awarded: HashSet, @@ -33,35 +38,94 @@ pub trait BadgeStart { &self, indexer_pool: &PgPool, rpc_client: &JsonRpcClient, - input: tokio::sync::watch::Receiver, - output: tokio::sync::watch::Sender, + input: tokio::sync::broadcast::Receiver, + output: tokio::sync::broadcast::Sender, ); } pub type BadgeStartFn = fn( indexer_pool: &PgPool, rpc_client: &JsonRpcClient, - input: tokio::sync::watch::Receiver, - output: tokio::sync::watch::Sender, + input: tokio::sync::broadcast::Receiver, + output: tokio::sync::broadcast::Sender, ); pub struct BadgeRegistry { - registry: HashMap, + indexer_pool: PgPool, + rpc_client: JsonRpcClient, + result_send: broadcast::Sender, + registration_to_fn: HashMap, broadcast::Sender)>, + badge_to_registration: HashMap, } impl BadgeRegistry { - pub fn new() -> Self { + pub fn new(indexer_pool: PgPool, rpc_client: JsonRpcClient) -> Self { + let (result_send, _) = tokio::sync::broadcast::channel(32); + Self { - registry: HashMap::new(), + indexer_pool, + rpc_client, + result_send, + registration_to_fn: HashMap::new(), + badge_to_registration: HashMap::new(), } } - pub fn register(&mut self, badge_id: BadgeId, checker: BadgeChecker) { - if self.registry.contains_key(&badge_id) { + pub fn subscribe(&self) -> broadcast::Receiver { + self.result_send.subscribe() + } + + pub fn register(&mut self, badge_ids: &[BadgeId], start_checker: BadgeStartFn) { + static REGISTRATION_ID: AtomicUsize = AtomicUsize::new(0); + + let badge_ids = badge_ids.iter().collect(); + + if !self + .badge_to_registration + .keys() + .collect::>() + .is_disjoint(&badge_ids) + { return; } - self.registry.insert(badge_id, checker); + let registration_id = REGISTRATION_ID.fetch_add(1, Ordering::Relaxed); + let (account_send, account_recv) = tokio::sync::broadcast::channel(16); + + for badge_id in badge_ids { + self.badge_to_registration.insert(badge_id, registration_id); + } + + self.registration_to_fn + .insert(registration_id, (Arc::new(Semaphore::new(16)), account_send)); + + start_checker( + &self.indexer_pool, + &self.rpc_client, + account_recv, + self.result_send.clone(), + ); + } + + pub async fn queue_account( + &self, + account_id: AccountId, + ignore_badge_ids: HashSet, + ) { + let registration_ids = self.badge_to_registration.iter().filter_map(|(badge_id, registration_id)|{ + if ignore_badge_ids.contains(badge_id) { + None + } else { + Some(registration_id) + } + }).collect::>(); + + for registration_id in registration_ids { + let (ref semaphore, ref sender) = self.registration_to_fn[registration_id]; + let permit = semaphore.acquire().await.unwrap(); + sender.send(account_id.clone()).unwrap(); // TODO: Remove unwrap() + drop(permit); + } } // pub async fn check_account( diff --git a/service/src/badge/transfer.rs b/service/src/badge/transfer.rs index 47f3748..cce9b1c 100644 --- a/service/src/badge/transfer.rs +++ b/service/src/badge/transfer.rs @@ -1,216 +1,34 @@ -use std::{collections::HashSet, pin::Pin, time::Duration}; +use std::sync::Arc; use crate::badge::BadgeCheckResult; -use super::{Badge, BadgeStruct, BadgeTrait, BadgeStartFn}; -use async_trait::async_trait; -use futures::{future::BoxFuture, Future, FutureExt}; +use super::BadgeStartFn; +use futures::FutureExt; use near_jsonrpc_client::JsonRpcClient; use near_primitives::types::AccountId; -use sqlx::{postgres::PgRow, PgPool}; +use sqlx::PgPool; use thiserror::Error; -use tokio::time::{sleep_until, Instant}; - -pub struct Transfer; pub static TRANSFER_1: &'static str = "b36c3dd2-b8b0-4098-b227-63290f009668"; pub static TRANSFER_10: &'static str = "609b2017-9534-4737-b86b-6ee4897fc4f9"; pub static TRANSFER_100: &'static str = "64ce9af9-52ac-49dc-96fd-031b4fa2efad"; pub static BADGE_IDS: &'static [&'static str] = &[TRANSFER_1, TRANSFER_10, TRANSFER_100]; -#[async_trait] -impl Badge for Transfer { - fn badge_ids() -> &'static [&'static str] { - BADGE_IDS - } - - async fn check_account( - indexer_pool: &PgPool, - _: &JsonRpcClient, - account_id: &AccountId, - ) -> Vec<&'static str> { - #[derive(sqlx::FromRow)] - struct WithResult { - pub result: i64, - } - - let result = sqlx::query_as::<_, WithResult>( - r#" -select count(*) as result - from action_receipt_actions - where action_kind = 'TRANSFER' - and receipt_predecessor_account_id = $1 - "#, - ) - .bind(account_id.to_string()) - .fetch_one(indexer_pool) - .await; - - match result { - Ok(result) => { - let total = result.result; - if total >= 100 { - BADGE_IDS.to_vec() - } else if total >= 10 { - BADGE_IDS[..2].to_vec() - } else if total >= 1 { - BADGE_IDS[..1].to_vec() - } else { - vec![] - } - } - _ => vec![], - } - } -} - #[derive(Error, Debug)] enum TransferBadgeError { #[error("Query failure: {0}")] QueryFailure(#[from] sqlx::Error), } -pub async fn check_account( - indexer_pool: &PgPool, - _: &JsonRpcClient, - account_id: &AccountId, -) -> Result { +async fn perform_query( + indexer_pool: PgPool, + account_id: AccountId, +) -> Result { #[derive(sqlx::FromRow)] struct WithResult { pub result: i64, } - let result = sqlx::query_as::<_, WithResult>( - r#" -select count(*) as result - from action_receipt_actions - where action_kind = 'TRANSFER' - and receipt_predecessor_account_id = $1 - "#, - ) - .bind(account_id.to_string()) - .fetch_one(indexer_pool) - .await; - - result.map(|result| { - let total = result.result; - let awarded = if total >= 100 { - BADGE_IDS - } else if total >= 10 { - &BADGE_IDS[..2] - } else if total >= 1 { - &BADGE_IDS[..1] - } else { - &[] - } - .to_vec(); - - BadgeCheckResult { - account_id: account_id.clone(), - awarded: awarded.into_iter().collect(), - checked: BADGE_IDS.to_vec().into_iter().collect(), - } - }) -} - -fn test() { - // let x = BadgeStruct { - // checker: Box::new(|indexer_pool, _, account_id| async move { - // #[derive(sqlx::FromRow)] - // struct WithResult { - // pub result: i64, - // } - - // sqlx::query_as::<_, WithResult>( - // r#" - // select count(*) as result - // from action_receipt_actions - // where action_kind = 'TRANSFER' - // and receipt_predecessor_account_id = $1 - // "#, - // ) - // .bind(account_id.to_string()) - // .fetch_one(indexer_pool) - // .map(|result| { - // result - // .map(|r| { - // let total = r.result; - // let awarded = if total >= 100 { - // BADGE_IDS - // } else if total >= 10 { - // &BADGE_IDS[..2] - // } else if total >= 1 { - // &BADGE_IDS[..1] - // } else { - // &[] - // } - // .to_vec(); - - // BadgeCheckResult { - // awarded: awarded.into_iter().collect(), - // checked: BADGE_IDS.to_vec().into_iter().collect(), - // } - // }) - // .map_err(|e| TransferBadgeError::from(e)) - // }) - // .boxed() - // .shared() - // .await - - // // result.map(|result| { - // // let total = result.result; - // // let awarded = if total >= 100 { - // // BADGE_IDS - // // } else if total >= 10 { - // // &BADGE_IDS[..2] - // // } else if total >= 1 { - // // &BADGE_IDS[..1] - // // } else { - // // &[] - // // } - // // .to_vec(); - - // // BadgeCheckResult { - // // awarded: awarded.into_iter().collect(), - // // checked: BADGE_IDS.to_vec().into_iter().collect(), - // // } - // // }) - // }), - // badges: HashSet::new(), - // }; - - let y = BadgeStruct { - checker: Box::new(|_, _, _| async { - Ok(BadgeCheckResult { - account_id: "account".parse().unwrap(), - awarded: HashSet::new(), - checked: HashSet::new(), - }) as Result<_, TransferBadgeError> - }), - badges: HashSet::new(), - }; - - // let t: Box = Box::new(x); - - // let x: Vec> = vec![Box::new(x), Box::new(y)]; -} - -fn badge_checker_fn( - indexer_pool: &PgPool, - rpc_client: &JsonRpcClient, - input: &mut tokio::sync::watch::Receiver, - output: tokio::sync::watch::Sender, -) { - #[derive(sqlx::FromRow)] - struct WithResult { - pub result: i64, - } - - - tokio::spawn(async move { - // while let Some(x) = input.changed().await {} - - sqlx::query_as::<_, WithResult>( r#" select count(*) as result @@ -220,7 +38,7 @@ fn badge_checker_fn( "#, ) .bind(account_id.to_string()) - .fetch_one(indexer_pool) + .fetch_one(&indexer_pool) .map(|result| { result .map(|r| { @@ -243,13 +61,39 @@ fn badge_checker_fn( } }) .map_err(|e| TransferBadgeError::from(e)) - }); - .boxed() - .shared() + }) .await - }); } -fn t() { - let x: BadgeStartFn = badge_checker_fn; +pub fn start( + indexer_pool: &PgPool, + _: &JsonRpcClient, + mut input: tokio::sync::broadcast::Receiver, + output: tokio::sync::broadcast::Sender, +) { + let indexer_pool = indexer_pool.clone(); + + tokio::spawn(async move { + let concurrent = Arc::new(tokio::sync::Semaphore::new(5)); + + while let Ok(account_id) = input.recv().await { + let indexer_pool = indexer_pool.clone(); + let output = output.clone(); + let concurrent = concurrent.clone(); + + tokio::spawn(async move { + let permit = concurrent.acquire().await.unwrap(); + println!("Acquired for {account_id}"); + let result = perform_query(indexer_pool, account_id.clone()).await; + println!("Finished {account_id}"); + drop(permit); + match result { + Ok(result) => { + output.send(result).unwrap(); // TODO: Log instead of unwrap + } + Err(e) => println!("{e:?}"), + } + }); + } + }); } diff --git a/service/src/indexer.rs b/service/src/indexer.rs index e410ce1..5b8c748 100644 --- a/service/src/indexer.rs +++ b/service/src/indexer.rs @@ -16,11 +16,13 @@ select distinct predecessor_account_id as account_id from receipts where included_in_block_timestamp > $1 and length(predecessor_account_id) != 64 + and predecessor_account_id like '%.%' union - select distinct receiver_account_id as account_id +select distinct receiver_account_id as account_id from receipts where included_in_block_timestamp > $1 and length(receiver_account_id) != 64 + and receiver_account_id like '%.%' ", ) .bind(timestamp_nanoseconds as i64) // for some reason Encode is not implemented for u64 on Postgres diff --git a/service/src/main.rs b/service/src/main.rs index 2080492..11d36cb 100644 --- a/service/src/main.rs +++ b/service/src/main.rs @@ -1,10 +1,17 @@ +use std::{collections::HashSet, ops::Sub}; + +use chrono::Duration; use dotenv::dotenv; use near_jsonrpc_client::JsonRpcClient; +use near_primitives::types::AccountId; use serde::Deserialize; use sqlx::{migrate, postgres::PgPoolOptions}; -use crate::badge::{BadgeRegistry, transfer}; +use crate::{ + badge::{transfer, BadgeRegistry}, + indexer::get_recent_actors, updater::update_account, +}; #[derive(Deserialize)] struct Configuration { @@ -38,32 +45,48 @@ async fn main() -> Result<(), Box> { let jsonrpc_client = JsonRpcClient::connect(&config.rpc_url); - let mut badge_registry = BadgeRegistry::new(); + println!("Requesting accounts"); + + let accounts = get_recent_actors( + &indexer_pool, + chrono::Utc::now() + .sub(Duration::minutes(5)) + .timestamp_nanos() + .try_into() + .unwrap(), + ).await.unwrap(); + + println!("Checking {} accounts", accounts.len()); + + println!("Creating badge registry"); + + let mut badge_registry = BadgeRegistry::new(indexer_pool.clone(), jsonrpc_client.clone()); - // for badge_id in transfer::BADGE_IDS { - // badge_registry.register(badge_id, transfer::check_account); - // } + let mut r = badge_registry.subscribe(); + let target = accounts.len(); - // updater::update_account( - // &local_pool, - // &indexer_pool, - // &jsonrpc_client, - // "hatchet.near".parse().unwrap(), - // ).await.unwrap(); + let badge_handle = tokio::spawn(async move { + println!("Listening for badge results"); + let mut done = 0; + while let Ok(result) = r.recv().await { + done += 1; + let account_id = result.account_id; + println!("{done} / {target} - {account_id}"); + } + }); - // let result = indexer::get_recent_actors(&indexer_pool, 1652741757312000000).await?; + badge_registry.register(transfer::BADGE_IDS, transfer::start); - // println!("{}", &result.len()); + for account in accounts { + let account_id: AccountId = account.parse().unwrap(); + badge_registry.queue_account(account_id.clone(), HashSet::new()).await; + update_account(&local_pool, &indexer_pool, &jsonrpc_client, account_id).await.unwrap(); + } - // let mut result = sqlx::query("select * from accounts limit 10").fetch(&pool); + println!("Waiting."); - // while let Some(row) = result.try_next().await? { - // let account_id: Result = row.try_get("account_id"); - // if let Ok(account_id) = account_id { - // println!("Account ID: {account_id}"); - // } - // } + badge_handle.await.unwrap(); Ok(()) } From 68a986210a80b3c864ccae3b6f7f2458009f8720 Mon Sep 17 00:00:00 2001 From: Jacob Date: Fri, 20 May 2022 10:12:59 -0500 Subject: [PATCH 04/31] fmt --- package.json | 4 + service/src/badge/mod.rs | 170 +++++----------------------------- service/src/badge/transfer.rs | 1 - service/src/indexer.rs | 5 +- service/src/main.rs | 15 ++- 5 files changed, 39 insertions(+), 156 deletions(-) diff --git a/package.json b/package.json index 807fe81..a012cad 100644 --- a/package.json +++ b/package.json @@ -104,6 +104,10 @@ "*.{js,jsx,vue,ts,tsx}": [ "vue-cli-service lint", "git add" + ], + "*.{rs}": [ + "cargo fmt", + "git add" ] } } diff --git a/service/src/badge/mod.rs b/service/src/badge/mod.rs index de3ae72..a62b60c 100644 --- a/service/src/badge/mod.rs +++ b/service/src/badge/mod.rs @@ -1,20 +1,16 @@ use std::{ collections::{HashMap, HashSet}, - pin::Pin, - sync::{atomic::{AtomicUsize, Ordering}, Arc}, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, }; -use async_trait::async_trait; -use futures::Future; - use near_jsonrpc_client::JsonRpcClient; use near_primitives::types::AccountId; use sqlx::PgPool; -use tokio::sync::{ - broadcast, - mpsc::{Receiver, Sender}, Semaphore, -}; +use tokio::sync::{broadcast, Semaphore}; pub type BadgeId = &'static str; pub type BadgeCheckerRegistrationId = usize; @@ -26,23 +22,6 @@ pub struct BadgeCheckResult { pub checked: HashSet, } -pub type BadgeChecker = - fn( - indexer_pool: &PgPool, - rpc_client: &JsonRpcClient, - account_id: &AccountId, - ) -> Pin>>>>; - -pub trait BadgeStart { - fn run( - &self, - indexer_pool: &PgPool, - rpc_client: &JsonRpcClient, - input: tokio::sync::broadcast::Receiver, - output: tokio::sync::broadcast::Sender, - ); -} - pub type BadgeStartFn = fn( indexer_pool: &PgPool, rpc_client: &JsonRpcClient, @@ -54,7 +33,8 @@ pub struct BadgeRegistry { indexer_pool: PgPool, rpc_client: JsonRpcClient, result_send: broadcast::Sender, - registration_to_fn: HashMap, broadcast::Sender)>, + registration_to_fn: + HashMap, broadcast::Sender)>, badge_to_registration: HashMap, } @@ -96,8 +76,10 @@ impl BadgeRegistry { self.badge_to_registration.insert(badge_id, registration_id); } - self.registration_to_fn - .insert(registration_id, (Arc::new(Semaphore::new(16)), account_send)); + self.registration_to_fn.insert( + registration_id, + (Arc::new(Semaphore::new(16)), account_send), + ); start_checker( &self.indexer_pool, @@ -107,18 +89,18 @@ impl BadgeRegistry { ); } - pub async fn queue_account( - &self, - account_id: AccountId, - ignore_badge_ids: HashSet, - ) { - let registration_ids = self.badge_to_registration.iter().filter_map(|(badge_id, registration_id)|{ - if ignore_badge_ids.contains(badge_id) { - None - } else { - Some(registration_id) - } - }).collect::>(); + pub async fn queue_account(&self, account_id: AccountId, ignore_badge_ids: HashSet) { + let registration_ids = self + .badge_to_registration + .iter() + .filter_map(|(badge_id, registration_id)| { + if ignore_badge_ids.contains(badge_id) { + None + } else { + Some(registration_id) + } + }) + .collect::>(); for registration_id in registration_ids { let (ref semaphore, ref sender) = self.registration_to_fn[registration_id]; @@ -127,112 +109,6 @@ impl BadgeRegistry { drop(permit); } } - - // pub async fn check_account( - // &self, - // account_id: &AccountId, - // indexer_pool: &PgPool, - // rpc_client: &JsonRpcClient, - // ignore_badge_ids: &HashSet, - // ) -> BadgeCheckResult { - // let mut ignore_badge_ids = ignore_badge_ids.clone(); - // let mut awarded = HashSet::::new(); - // let mut checked = HashSet::::new(); - - // for badge_id in self.registry.keys() { - // if ignore_badge_ids.contains(badge_id) { - // continue; - // } - - // let checker = self.registry[badge_id]; - // let result = checker(indexer_pool, rpc_client, account_id).await; - // if let Ok(result) = result { - // awarded.extend(result.awarded); - // checked.extend(&result.checked); - // ignore_badge_ids.extend(result.checked); - // } // TODO: Err case - // } - - // BadgeCheckResult { awarded, checked } - // } -} - -// pub fn badges() -> Vec> { -// vec![Box::new(Transfer)] -// } - -#[async_trait] -pub trait Badge { - fn badge_ids() -> &'static [&'static str]; - async fn check_account( - indexer_pool: &PgPool, - rpc_client: &JsonRpcClient, - account_id: &AccountId, - ) -> Vec<&'static str>; -} - -pub struct BadgeStruct<'a, 'b, Fut, E> -where - Fut: Future> + Send + Sync, - E: std::error::Error, - 'b: 'a, -{ - pub badges: HashSet, - pub checker: - Box Fut>, -} - -#[async_trait] -pub trait BadgeTrait<'b> { - async fn check( - &'b self, - indexer_pool: &'b PgPool, - rpc_client: &'b JsonRpcClient, - account_id: &'b AccountId, - ) -> Result>; -} - -// impl BadgeTrait for String { -// fn check<'life0, 'life1, 'life2, 'life3, 'async_trait>( -// &'life0 self, -// indexer_pool: &'life1 PgPool, -// rpc_client: &'life2 JsonRpcClient, -// account_id: &'life3 AccountId, -// ) -> core::pin::Pin< -// Box< -// dyn core::future::Future>> -// + core::marker::Send -// + 'async_trait, -// >, -// > -// where -// 'life0: 'async_trait, -// 'life1: 'async_trait, -// 'life2: 'async_trait, -// 'life3: 'async_trait, -// Self: 'async_trait, -// { -// todo!() -// } -// } - -#[async_trait] -impl BadgeTrait<'b> for BadgeStruct<'a, 'b, Fut, E> -where - Fut: Future> + Send + Sync, - E: std::error::Error + 'static, - 'b: 'a, -{ - async fn check( - &'b self, - indexer_pool: &'b PgPool, - rpc_client: &'b JsonRpcClient, - account_id: &'b AccountId, - ) -> Result> { - (self.checker)(indexer_pool, rpc_client, account_id) - .await - .map_err(|e| Box::new(e) as Box) - } } pub mod transfer; diff --git a/service/src/badge/transfer.rs b/service/src/badge/transfer.rs index cce9b1c..5d4daad 100644 --- a/service/src/badge/transfer.rs +++ b/service/src/badge/transfer.rs @@ -2,7 +2,6 @@ use std::sync::Arc; use crate::badge::BadgeCheckResult; -use super::BadgeStartFn; use futures::FutureExt; use near_jsonrpc_client::JsonRpcClient; use near_primitives::types::AccountId; diff --git a/service/src/indexer.rs b/service/src/indexer.rs index 5b8c748..f46c1ff 100644 --- a/service/src/indexer.rs +++ b/service/src/indexer.rs @@ -32,10 +32,7 @@ select distinct receiver_account_id as account_id .await } -pub async fn calculate_account_score( - pool: &PgPool, - account_id: &str, -) -> Result { +pub async fn calculate_account_score(pool: &PgPool, account_id: &str) -> Result { #[derive(sqlx::FromRow)] struct WithResult { pub result: i64, diff --git a/service/src/main.rs b/service/src/main.rs index 11d36cb..0f8a80b 100644 --- a/service/src/main.rs +++ b/service/src/main.rs @@ -10,7 +10,8 @@ use sqlx::{migrate, postgres::PgPoolOptions}; use crate::{ badge::{transfer, BadgeRegistry}, - indexer::get_recent_actors, updater::update_account, + indexer::get_recent_actors, + updater::update_account, }; #[derive(Deserialize)] @@ -54,7 +55,9 @@ async fn main() -> Result<(), Box> { .timestamp_nanos() .try_into() .unwrap(), - ).await.unwrap(); + ) + .await + .unwrap(); println!("Checking {} accounts", accounts.len()); @@ -80,8 +83,12 @@ async fn main() -> Result<(), Box> { for account in accounts { let account_id: AccountId = account.parse().unwrap(); - badge_registry.queue_account(account_id.clone(), HashSet::new()).await; - update_account(&local_pool, &indexer_pool, &jsonrpc_client, account_id).await.unwrap(); + badge_registry + .queue_account(account_id.clone(), HashSet::new()) + .await; + update_account(&local_pool, &indexer_pool, &jsonrpc_client, account_id) + .await + .unwrap(); } println!("Waiting."); From c604fd966faab0be196d94b6a9992564dba07f63 Mon Sep 17 00:00:00 2001 From: Jacob Date: Fri, 20 May 2022 10:42:58 -0500 Subject: [PATCH 05/31] controlled semaphore + connections struct --- service/src/badge/mod.rs | 35 +++++++++++++++++++++-------------- service/src/badge/transfer.rs | 20 +++++++++++--------- service/src/main.rs | 7 +++++-- 3 files changed, 37 insertions(+), 25 deletions(-) diff --git a/service/src/badge/mod.rs b/service/src/badge/mod.rs index a62b60c..51874ff 100644 --- a/service/src/badge/mod.rs +++ b/service/src/badge/mod.rs @@ -12,6 +12,10 @@ use sqlx::PgPool; use tokio::sync::{broadcast, Semaphore}; +const RESULT_CHANNEL_SIZE: usize = 32; +const ACCOUNT_CHANNEL_SIZE: usize = 16; +const MAX_SIMULTANEOUS_WORKERS: usize = 5; + pub type BadgeId = &'static str; pub type BadgeCheckerRegistrationId = usize; @@ -22,16 +26,20 @@ pub struct BadgeCheckResult { pub checked: HashSet, } +pub struct Connections { + pub indexer_pool: PgPool, + pub rpc_client: JsonRpcClient, +} + pub type BadgeStartFn = fn( - indexer_pool: &PgPool, - rpc_client: &JsonRpcClient, - input: tokio::sync::broadcast::Receiver, - output: tokio::sync::broadcast::Sender, + semaphore: Semaphore, + connections: &Connections, + input: broadcast::Receiver, + output: broadcast::Sender, ); pub struct BadgeRegistry { - indexer_pool: PgPool, - rpc_client: JsonRpcClient, + connections: Connections, result_send: broadcast::Sender, registration_to_fn: HashMap, broadcast::Sender)>, @@ -39,12 +47,11 @@ pub struct BadgeRegistry { } impl BadgeRegistry { - pub fn new(indexer_pool: PgPool, rpc_client: JsonRpcClient) -> Self { - let (result_send, _) = tokio::sync::broadcast::channel(32); + pub fn new(connections: Connections) -> Self { + let (result_send, _) = broadcast::channel(RESULT_CHANNEL_SIZE); Self { - indexer_pool, - rpc_client, + connections, result_send, registration_to_fn: HashMap::new(), badge_to_registration: HashMap::new(), @@ -70,7 +77,7 @@ impl BadgeRegistry { } let registration_id = REGISTRATION_ID.fetch_add(1, Ordering::Relaxed); - let (account_send, account_recv) = tokio::sync::broadcast::channel(16); + let (account_send, account_recv) = broadcast::channel(ACCOUNT_CHANNEL_SIZE); for badge_id in badge_ids { self.badge_to_registration.insert(badge_id, registration_id); @@ -78,12 +85,12 @@ impl BadgeRegistry { self.registration_to_fn.insert( registration_id, - (Arc::new(Semaphore::new(16)), account_send), + (Arc::new(Semaphore::new(ACCOUNT_CHANNEL_SIZE)), account_send), ); start_checker( - &self.indexer_pool, - &self.rpc_client, + Semaphore::new(MAX_SIMULTANEOUS_WORKERS), + &self.connections, account_recv, self.result_send.clone(), ); diff --git a/service/src/badge/transfer.rs b/service/src/badge/transfer.rs index 5d4daad..5805b0f 100644 --- a/service/src/badge/transfer.rs +++ b/service/src/badge/transfer.rs @@ -3,10 +3,12 @@ use std::sync::Arc; use crate::badge::BadgeCheckResult; use futures::FutureExt; -use near_jsonrpc_client::JsonRpcClient; use near_primitives::types::AccountId; use sqlx::PgPool; use thiserror::Error; +use tokio::sync::{broadcast, Semaphore}; + +use super::Connections; pub static TRANSFER_1: &'static str = "b36c3dd2-b8b0-4098-b227-63290f009668"; pub static TRANSFER_10: &'static str = "609b2017-9534-4737-b86b-6ee4897fc4f9"; @@ -65,23 +67,23 @@ async fn perform_query( } pub fn start( - indexer_pool: &PgPool, - _: &JsonRpcClient, - mut input: tokio::sync::broadcast::Receiver, - output: tokio::sync::broadcast::Sender, + semaphore: Semaphore, + connections: &Connections, + mut input: broadcast::Receiver, + output: broadcast::Sender, ) { - let indexer_pool = indexer_pool.clone(); + let indexer_pool = connections.indexer_pool.clone(); tokio::spawn(async move { - let concurrent = Arc::new(tokio::sync::Semaphore::new(5)); + let semaphore = Arc::new(semaphore); while let Ok(account_id) = input.recv().await { let indexer_pool = indexer_pool.clone(); let output = output.clone(); - let concurrent = concurrent.clone(); + let semaphore = semaphore.clone(); tokio::spawn(async move { - let permit = concurrent.acquire().await.unwrap(); + let permit = semaphore.acquire().await.unwrap(); println!("Acquired for {account_id}"); let result = perform_query(indexer_pool, account_id.clone()).await; println!("Finished {account_id}"); diff --git a/service/src/main.rs b/service/src/main.rs index 0f8a80b..2cd56b8 100644 --- a/service/src/main.rs +++ b/service/src/main.rs @@ -9,7 +9,7 @@ use serde::Deserialize; use sqlx::{migrate, postgres::PgPoolOptions}; use crate::{ - badge::{transfer, BadgeRegistry}, + badge::{transfer, BadgeRegistry, Connections}, indexer::get_recent_actors, updater::update_account, }; @@ -63,7 +63,10 @@ async fn main() -> Result<(), Box> { println!("Creating badge registry"); - let mut badge_registry = BadgeRegistry::new(indexer_pool.clone(), jsonrpc_client.clone()); + let mut badge_registry = BadgeRegistry::new(Connections { + indexer_pool: indexer_pool.clone(), + rpc_client: jsonrpc_client.clone(), + }); let mut r = badge_registry.subscribe(); From 8f5bd5769ecaf568bcbe7046355e34bb0a6a8387 Mon Sep 17 00:00:00 2001 From: Jacob Date: Fri, 20 May 2022 11:15:44 -0500 Subject: [PATCH 06/31] badge worker macro --- service/src/badge/mod.rs | 45 +++++++++++++++++++++++++++++++---- service/src/badge/transfer.rs | 38 +++-------------------------- service/src/main.rs | 22 +++++++++++------ 3 files changed, 59 insertions(+), 46 deletions(-) diff --git a/service/src/badge/mod.rs b/service/src/badge/mod.rs index 51874ff..3c7fe2f 100644 --- a/service/src/badge/mod.rs +++ b/service/src/badge/mod.rs @@ -26,14 +26,15 @@ pub struct BadgeCheckResult { pub checked: HashSet, } +#[derive(Clone)] pub struct Connections { pub indexer_pool: PgPool, pub rpc_client: JsonRpcClient, } -pub type BadgeStartFn = fn( +pub type BadgeWorker = fn( semaphore: Semaphore, - connections: &Connections, + connections: Connections, input: broadcast::Receiver, output: broadcast::Sender, ); @@ -62,7 +63,7 @@ impl BadgeRegistry { self.result_send.subscribe() } - pub fn register(&mut self, badge_ids: &[BadgeId], start_checker: BadgeStartFn) { + pub fn register(&mut self, badge_ids: &[BadgeId], start_checker: BadgeWorker) { static REGISTRATION_ID: AtomicUsize = AtomicUsize::new(0); let badge_ids = badge_ids.iter().collect(); @@ -90,7 +91,7 @@ impl BadgeRegistry { start_checker( Semaphore::new(MAX_SIMULTANEOUS_WORKERS), - &self.connections, + self.connections.clone(), account_recv, self.result_send.clone(), ); @@ -118,4 +119,40 @@ impl BadgeRegistry { } } +#[macro_export] +macro_rules! create_badge_worker { + ($query_fn: ident) => { + pub fn run( + semaphore: Semaphore, + connections: Connections, + mut input: broadcast::Receiver, + output: broadcast::Sender, + ) { + tokio::spawn(async move { + let semaphore = Arc::new(semaphore); + + while let Ok(account_id) = input.recv().await { + let connections = connections.clone(); + let output = output.clone(); + let semaphore = semaphore.clone(); + + tokio::spawn(async move { + let permit = semaphore.acquire().await.unwrap(); + println!("Acquired for {account_id}"); + let result = $query_fn(connections, account_id.clone()).await; + println!("Finished {account_id}"); + drop(permit); + match result { + Ok(result) => { + output.send(result).unwrap(); // TODO: Log instead of unwrap + } + Err(e) => println!("{e:?}"), + } + }); + } + }); + } + }; +} + pub mod transfer; diff --git a/service/src/badge/transfer.rs b/service/src/badge/transfer.rs index 5805b0f..d6aafe5 100644 --- a/service/src/badge/transfer.rs +++ b/service/src/badge/transfer.rs @@ -4,7 +4,6 @@ use crate::badge::BadgeCheckResult; use futures::FutureExt; use near_primitives::types::AccountId; -use sqlx::PgPool; use thiserror::Error; use tokio::sync::{broadcast, Semaphore}; @@ -22,7 +21,7 @@ enum TransferBadgeError { } async fn perform_query( - indexer_pool: PgPool, + connections: Connections, account_id: AccountId, ) -> Result { #[derive(sqlx::FromRow)] @@ -39,7 +38,7 @@ async fn perform_query( "#, ) .bind(account_id.to_string()) - .fetch_one(&indexer_pool) + .fetch_one(&connections.indexer_pool) .map(|result| { result .map(|r| { @@ -66,35 +65,4 @@ async fn perform_query( .await } -pub fn start( - semaphore: Semaphore, - connections: &Connections, - mut input: broadcast::Receiver, - output: broadcast::Sender, -) { - let indexer_pool = connections.indexer_pool.clone(); - - tokio::spawn(async move { - let semaphore = Arc::new(semaphore); - - while let Ok(account_id) = input.recv().await { - let indexer_pool = indexer_pool.clone(); - let output = output.clone(); - let semaphore = semaphore.clone(); - - tokio::spawn(async move { - let permit = semaphore.acquire().await.unwrap(); - println!("Acquired for {account_id}"); - let result = perform_query(indexer_pool, account_id.clone()).await; - println!("Finished {account_id}"); - drop(permit); - match result { - Ok(result) => { - output.send(result).unwrap(); // TODO: Log instead of unwrap - } - Err(e) => println!("{e:?}"), - } - }); - } - }); -} +create_badge_worker!(perform_query); diff --git a/service/src/main.rs b/service/src/main.rs index 2cd56b8..fef26f2 100644 --- a/service/src/main.rs +++ b/service/src/main.rs @@ -7,6 +7,7 @@ use near_jsonrpc_client::JsonRpcClient; use near_primitives::types::AccountId; use serde::Deserialize; use sqlx::{migrate, postgres::PgPoolOptions}; +use tokio::join; use crate::{ badge::{transfer, BadgeRegistry, Connections}, @@ -82,16 +83,23 @@ async fn main() -> Result<(), Box> { } }); - badge_registry.register(transfer::BADGE_IDS, transfer::start); + badge_registry.register(transfer::BADGE_IDS, transfer::run); for account in accounts { let account_id: AccountId = account.parse().unwrap(); - badge_registry - .queue_account(account_id.clone(), HashSet::new()) - .await; - update_account(&local_pool, &indexer_pool, &jsonrpc_client, account_id) - .await - .unwrap(); + let (_, update) = join!( + badge_registry.queue_account(account_id.clone(), HashSet::new()), + update_account( + &local_pool, + &indexer_pool, + &jsonrpc_client, + account_id.clone() + ), + ); + + if let Err(..) = update { + println!("Error updating {}", &account_id); + } } println!("Waiting."); From 8bc49a2f133974e6eca1faf5e0400db1b06c15c6 Mon Sep 17 00:00:00 2001 From: Jacob Date: Fri, 20 May 2022 11:18:20 -0500 Subject: [PATCH 07/31] rename updater -> local --- service/src/{updater.rs => local.rs} | 0 service/src/main.rs | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) rename service/src/{updater.rs => local.rs} (100%) diff --git a/service/src/updater.rs b/service/src/local.rs similarity index 100% rename from service/src/updater.rs rename to service/src/local.rs diff --git a/service/src/main.rs b/service/src/main.rs index fef26f2..054c7ce 100644 --- a/service/src/main.rs +++ b/service/src/main.rs @@ -12,7 +12,7 @@ use tokio::join; use crate::{ badge::{transfer, BadgeRegistry, Connections}, indexer::get_recent_actors, - updater::update_account, + local::update_account, }; #[derive(Deserialize)] @@ -25,8 +25,8 @@ struct Configuration { mod badge; mod indexer; +mod local; mod rpc; -mod updater; #[tokio::main] async fn main() -> Result<(), Box> { From e864c20ab48035a0a0842b5df9ce395cedb78973 Mon Sep 17 00:00:00 2001 From: Jacob Date: Fri, 20 May 2022 13:06:22 -0500 Subject: [PATCH 08/31] consecutive errors --- ...20220520161908_consecutive-errors.down.sql | 2 ++ .../20220520161908_consecutive-errors.up.sql | 2 ++ service/src/local.rs | 29 +++++++++++++++++-- service/src/main.rs | 12 ++++++++ 4 files changed, 42 insertions(+), 3 deletions(-) create mode 100644 service/migrations/20220520161908_consecutive-errors.down.sql create mode 100644 service/migrations/20220520161908_consecutive-errors.up.sql diff --git a/service/migrations/20220520161908_consecutive-errors.down.sql b/service/migrations/20220520161908_consecutive-errors.down.sql new file mode 100644 index 0000000..ec223f5 --- /dev/null +++ b/service/migrations/20220520161908_consecutive-errors.down.sql @@ -0,0 +1,2 @@ +alter table account + drop column consecutive_errors; diff --git a/service/migrations/20220520161908_consecutive-errors.up.sql b/service/migrations/20220520161908_consecutive-errors.up.sql new file mode 100644 index 0000000..0168b3f --- /dev/null +++ b/service/migrations/20220520161908_consecutive-errors.up.sql @@ -0,0 +1,2 @@ +alter table account + add column consecutive_errors smallint default 0; diff --git a/service/src/local.rs b/service/src/local.rs index de7b0d7..0ec11cf 100644 --- a/service/src/local.rs +++ b/service/src/local.rs @@ -29,15 +29,38 @@ pub async fn update_account( get_account_balance(jsonrpc_client, account_id.clone()) ); + if score.is_err() || balance.is_err() { + println!("Error calculating score or balance for {account_id}: {:?}\n{:?}", &score, &balance); + let err_query_result = sqlx::query!( + r#" +INSERT INTO account(id, consecutive_errors) + VALUES ($1, 1) + ON CONFLICT (id) DO + UPDATE SET consecutive_errors = account.consecutive_errors + 1 + "#, + account_id.to_string() + ) + .execute(local_pool) + .await; + + match err_query_result { + Ok(..) => println!("Successfully updated consecutive_errors"), + Err(e) => println!("Failed to update consecutive_errors: {e:?}"), + }; + } + let score = score.map_err(|e| UpdateAccountError::ScoreError(e))?; let balance = Decimal::from_u128(balance?); sqlx::query!( " -INSERT INTO account(id, balance, score) - VALUES ($1, $2, $3) +INSERT INTO account(id, balance, score, consecutive_errors) + VALUES ($1, $2, $3, 0) ON CONFLICT (id) DO - UPDATE SET balance = EXCLUDED.balance, score = EXCLUDED.score + UPDATE SET + balance = EXCLUDED.balance, + score = EXCLUDED.score, + consecutive_errors = 0 ", account_id.to_string(), balance, diff --git a/service/src/main.rs b/service/src/main.rs index 054c7ce..77a42f1 100644 --- a/service/src/main.rs +++ b/service/src/main.rs @@ -60,6 +60,18 @@ async fn main() -> Result<(), Box> { .await .unwrap(); + let accounts = vec![ + "thorinoracle.near", + "igwt5.near", + "a0b86991c6218b36c1d19d4a2e9eb0ce3606eb48.factory.bridge.near", + "onebadman.near", + "poshat.near", + "shedevr4.near", + "kuki.near", + "artfrontirdao.near", + "0001k.near", + ]; + println!("Checking {} accounts", accounts.len()); println!("Creating badge registry"); From 957de2c019be617648af6ad4bddea9d0a1378109 Mon Sep 17 00:00:00 2001 From: Jacob Date: Fri, 20 May 2022 14:12:37 -0500 Subject: [PATCH 09/31] account update cooldown --- Cargo.lock | 1 + service/Cargo.toml | 2 +- service/src/local.rs | 80 +++++++++++++++++++++++++++++++++++++++++--- service/src/main.rs | 35 +++++++++---------- service/src/rpc.rs | 6 ++-- 5 files changed, 98 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c9a0f02..9d703c1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2124,6 +2124,7 @@ dependencies = [ "bitflags", "byteorder", "bytes", + "chrono", "crc", "crossbeam-queue", "dirs", diff --git a/service/Cargo.toml b/service/Cargo.toml index 2cfba99..111159b 100644 --- a/service/Cargo.toml +++ b/service/Cargo.toml @@ -15,7 +15,7 @@ near-jsonrpc-primitives = "0.12.0" near-primitives = "0.12.0" num-traits = "0.2.15" serde = "1.0.137" -sqlx = {version = "0.5", features = ["runtime-tokio-native-tls", "postgres", "decimal"]} +sqlx = {version = "0.5", features = ["runtime-tokio-native-tls", "postgres", "decimal", "chrono"]} thiserror = "1.0.31" tokio = {version = "1", features = ["full"]} diff --git a/service/src/local.rs b/service/src/local.rs index 0ec11cf..c67f8e6 100644 --- a/service/src/local.rs +++ b/service/src/local.rs @@ -1,9 +1,18 @@ +use std::{ops::Add, str::FromStr}; + +use chrono::{Duration, NaiveDateTime, TimeZone}; +use futures::FutureExt; use near_jsonrpc_client::{errors::JsonRpcError, methods::query::RpcQueryError, JsonRpcClient}; use near_primitives::types::AccountId; use num_traits::FromPrimitive; -use sqlx::{types::Decimal, PgPool}; +use sqlx::{types::Decimal, FromRow, PgPool}; use tokio::join; +/// Minimum amount of time between account updates +const ACCOUNT_UPDATE_COOLDOWN_MINUTES: i64 = 60 * 6; +/// Doubles every failure +const ACCOUNT_UPDATE_FAILURE_PENALTY_COEFFICIENT_MINUTES: i64 = 60 * 12; + use crate::{indexer::calculate_account_score, rpc::get_account_balance}; use thiserror::Error; @@ -18,19 +27,82 @@ pub enum UpdateAccountError { InsertQueryError(#[from] sqlx::Error), } +#[derive(FromRow)] +pub struct AccountRecordDb { + pub id: String, + pub balance: Option, + pub score: Option, + pub modified: Option, + pub consecutive_errors: Option, +} + +pub struct AccountRecord { + pub id: AccountId, + pub balance: Option, + pub score: Option, + pub modified: Option>, + pub consecutive_errors: u16, +} + +impl AccountRecord { + pub fn next_update_allowed_at(&self) -> Option> { + self.modified.map(|m| { + m + (Duration::minutes(ACCOUNT_UPDATE_COOLDOWN_MINUTES)) + + (Duration::minutes(ACCOUNT_UPDATE_FAILURE_PENALTY_COEFFICIENT_MINUTES) + * ((1 << self.consecutive_errors) - 1)) + }) + } +} + +impl From for AccountRecord { + fn from(record: AccountRecordDb) -> Self { + Self { + id: record.id.parse().unwrap_or( + AccountId::from_str( + "0000000000000000000000000000000000000000000000000000000000000000", + ) + .unwrap(), + ), + balance: record.balance.and_then(|b| u128::try_from(b).ok()), + score: record.score.map(|s| s as u32), + modified: record.modified.map(|m| chrono::Utc.from_utc_datetime(&m)), + consecutive_errors: record.consecutive_errors.unwrap_or(0) as u16, + } + } +} + +pub async fn query_account( + local_pool: &PgPool, + account_id: &AccountId, +) -> Result { + sqlx::query_as!( + AccountRecordDb, + r#" +select id, balance, score, modified, consecutive_errors from account where id = $1 + "#, + account_id.to_string() + ) + .fetch_one(local_pool) + .map(|a| a.map(AccountRecord::from)) + .await +} + pub async fn update_account( local_pool: &PgPool, indexer_pool: &PgPool, jsonrpc_client: &JsonRpcClient, - account_id: AccountId, + account_id: &AccountId, ) -> Result<(), UpdateAccountError> { let (score, balance) = join!( calculate_account_score(indexer_pool, &account_id), - get_account_balance(jsonrpc_client, account_id.clone()) + get_account_balance(jsonrpc_client, &account_id) ); if score.is_err() || balance.is_err() { - println!("Error calculating score or balance for {account_id}: {:?}\n{:?}", &score, &balance); + println!( + "Error calculating score or balance for {account_id}: {:?}\n{:?}", + &score, &balance + ); let err_query_result = sqlx::query!( r#" INSERT INTO account(id, consecutive_errors) diff --git a/service/src/main.rs b/service/src/main.rs index 77a42f1..eaae97e 100644 --- a/service/src/main.rs +++ b/service/src/main.rs @@ -12,7 +12,7 @@ use tokio::join; use crate::{ badge::{transfer, BadgeRegistry, Connections}, indexer::get_recent_actors, - local::update_account, + local::{query_account, update_account}, }; #[derive(Deserialize)] @@ -60,18 +60,6 @@ async fn main() -> Result<(), Box> { .await .unwrap(); - let accounts = vec![ - "thorinoracle.near", - "igwt5.near", - "a0b86991c6218b36c1d19d4a2e9eb0ce3606eb48.factory.bridge.near", - "onebadman.near", - "poshat.near", - "shedevr4.near", - "kuki.near", - "artfrontirdao.near", - "0001k.near", - ]; - println!("Checking {} accounts", accounts.len()); println!("Creating badge registry"); @@ -97,16 +85,25 @@ async fn main() -> Result<(), Box> { badge_registry.register(transfer::BADGE_IDS, transfer::run); + let now = chrono::Utc::now(); + for account in accounts { let account_id: AccountId = account.parse().unwrap(); + let account_record = query_account(&local_pool, &account_id).await; + let is_update_allowed = account_record + .ok() + .and_then(|r| r.next_update_allowed_at()) + .map(|cutoff| now >= cutoff) + .unwrap_or(true); + + if !is_update_allowed { + println!("Disallowing update for {account_id}"); + continue; + } + let (_, update) = join!( badge_registry.queue_account(account_id.clone(), HashSet::new()), - update_account( - &local_pool, - &indexer_pool, - &jsonrpc_client, - account_id.clone() - ), + update_account(&local_pool, &indexer_pool, &jsonrpc_client, &account_id), ); if let Err(..) = update { diff --git a/service/src/rpc.rs b/service/src/rpc.rs index 3290758..6284b7c 100644 --- a/service/src/rpc.rs +++ b/service/src/rpc.rs @@ -11,12 +11,14 @@ use near_primitives::{ pub async fn get_account_balance( client: &JsonRpcClient, - account_id: AccountId, + account_id: &AccountId, ) -> Result> { client .call(methods::query::RpcQueryRequest { block_reference: BlockReference::Finality(Finality::Final), - request: QueryRequest::ViewAccount { account_id }, + request: QueryRequest::ViewAccount { + account_id: account_id.clone(), + }, }) .await .map(|r| match r.kind { From ca65d871bd07a4f1e6f76a01c3cb18eaf5a7dfcd Mon Sep 17 00:00:00 2001 From: Jacob Date: Fri, 20 May 2022 14:51:31 -0500 Subject: [PATCH 10/31] multithreaded local updater --- service/src/badge/mod.rs | 10 ++------ service/src/connections.rs | 9 +++++++ service/src/local.rs | 50 +++++++++++++++++++++++++++++--------- service/src/main.rs | 35 ++++++++++++++++---------- 4 files changed, 72 insertions(+), 32 deletions(-) create mode 100644 service/src/connections.rs diff --git a/service/src/badge/mod.rs b/service/src/badge/mod.rs index 3c7fe2f..5a7a268 100644 --- a/service/src/badge/mod.rs +++ b/service/src/badge/mod.rs @@ -6,12 +6,12 @@ use std::{ }, }; -use near_jsonrpc_client::JsonRpcClient; use near_primitives::types::AccountId; -use sqlx::PgPool; use tokio::sync::{broadcast, Semaphore}; +use crate::connections::Connections; + const RESULT_CHANNEL_SIZE: usize = 32; const ACCOUNT_CHANNEL_SIZE: usize = 16; const MAX_SIMULTANEOUS_WORKERS: usize = 5; @@ -26,12 +26,6 @@ pub struct BadgeCheckResult { pub checked: HashSet, } -#[derive(Clone)] -pub struct Connections { - pub indexer_pool: PgPool, - pub rpc_client: JsonRpcClient, -} - pub type BadgeWorker = fn( semaphore: Semaphore, connections: Connections, diff --git a/service/src/connections.rs b/service/src/connections.rs new file mode 100644 index 0000000..0b47524 --- /dev/null +++ b/service/src/connections.rs @@ -0,0 +1,9 @@ +use near_jsonrpc_client::JsonRpcClient; +use sqlx::PgPool; + +#[derive(Clone)] +pub struct Connections { + pub local_pool: PgPool, + pub indexer_pool: PgPool, + pub rpc_client: JsonRpcClient, +} diff --git a/service/src/local.rs b/service/src/local.rs index c67f8e6..9351f61 100644 --- a/service/src/local.rs +++ b/service/src/local.rs @@ -1,19 +1,22 @@ -use std::{ops::Add, str::FromStr}; +use std::{str::FromStr, sync::Arc}; use chrono::{Duration, NaiveDateTime, TimeZone}; use futures::FutureExt; -use near_jsonrpc_client::{errors::JsonRpcError, methods::query::RpcQueryError, JsonRpcClient}; +use near_jsonrpc_client::{errors::JsonRpcError, methods::query::RpcQueryError}; use near_primitives::types::AccountId; use num_traits::FromPrimitive; use sqlx::{types::Decimal, FromRow, PgPool}; -use tokio::join; +use tokio::{ + join, + sync::{broadcast, Semaphore}, +}; /// Minimum amount of time between account updates const ACCOUNT_UPDATE_COOLDOWN_MINUTES: i64 = 60 * 6; /// Doubles every failure const ACCOUNT_UPDATE_FAILURE_PENALTY_COEFFICIENT_MINUTES: i64 = 60 * 12; -use crate::{indexer::calculate_account_score, rpc::get_account_balance}; +use crate::{connections::Connections, indexer::calculate_account_score, rpc::get_account_balance}; use thiserror::Error; @@ -88,14 +91,12 @@ select id, balance, score, modified, consecutive_errors from account where id = } pub async fn update_account( - local_pool: &PgPool, - indexer_pool: &PgPool, - jsonrpc_client: &JsonRpcClient, + connections: &Connections, account_id: &AccountId, ) -> Result<(), UpdateAccountError> { let (score, balance) = join!( - calculate_account_score(indexer_pool, &account_id), - get_account_balance(jsonrpc_client, &account_id) + calculate_account_score(&connections.indexer_pool, &account_id), + get_account_balance(&connections.rpc_client, &account_id) ); if score.is_err() || balance.is_err() { @@ -112,7 +113,7 @@ INSERT INTO account(id, consecutive_errors) "#, account_id.to_string() ) - .execute(local_pool) + .execute(&connections.local_pool) .await; match err_query_result { @@ -138,8 +139,35 @@ INSERT INTO account(id, balance, score, consecutive_errors) balance, score as i64 ) - .execute(local_pool) + .execute(&connections.local_pool) .await?; Ok(()) } + +pub fn start_local_updater( + semaphore: Semaphore, + connections: Connections, + mut input: broadcast::Receiver, +) { + tokio::spawn(async move { + let semaphore = Arc::new(semaphore); + + while let Ok(account_id) = input.recv().await { + let connections = connections.clone(); + let semaphore = semaphore.clone(); + + tokio::spawn(async move { + let permit = semaphore.acquire().await.unwrap(); + println!("Acquired for {account_id}"); + let result = update_account(&connections, &account_id).await; + println!("Finished {account_id}"); + drop(permit); + + if let Err(e) = result { + println!("Error updating account {account_id}: {e:?}"); + } + }); + } + }); +} diff --git a/service/src/main.rs b/service/src/main.rs index eaae97e..03a8c6c 100644 --- a/service/src/main.rs +++ b/service/src/main.rs @@ -7,12 +7,13 @@ use near_jsonrpc_client::JsonRpcClient; use near_primitives::types::AccountId; use serde::Deserialize; use sqlx::{migrate, postgres::PgPoolOptions}; -use tokio::join; +use tokio::sync::{broadcast, Semaphore}; use crate::{ - badge::{transfer, BadgeRegistry, Connections}, + badge::{transfer, BadgeRegistry}, + connections::Connections, indexer::get_recent_actors, - local::{query_account, update_account}, + local::query_account, }; #[derive(Deserialize)] @@ -24,6 +25,7 @@ struct Configuration { } mod badge; +mod connections; mod indexer; mod local; mod rpc; @@ -52,7 +54,7 @@ async fn main() -> Result<(), Box> { let accounts = get_recent_actors( &indexer_pool, chrono::Utc::now() - .sub(Duration::minutes(5)) + .sub(Duration::minutes(10)) .timestamp_nanos() .try_into() .unwrap(), @@ -64,10 +66,13 @@ async fn main() -> Result<(), Box> { println!("Creating badge registry"); - let mut badge_registry = BadgeRegistry::new(Connections { + let connections = Connections { + local_pool: local_pool.clone(), indexer_pool: indexer_pool.clone(), rpc_client: jsonrpc_client.clone(), - }); + }; + + let mut badge_registry = BadgeRegistry::new(connections.clone()); let mut r = badge_registry.subscribe(); @@ -80,9 +85,14 @@ async fn main() -> Result<(), Box> { done += 1; let account_id = result.account_id; println!("{done} / {target} - {account_id}"); + // TODO: write badges back to local db } }); + let local_semaphore = Semaphore::new(5); + let (account_send, account_recv) = broadcast::channel(16); + local::start_local_updater(local_semaphore, connections.clone(), account_recv); + badge_registry.register(transfer::BADGE_IDS, transfer::run); let now = chrono::Utc::now(); @@ -101,14 +111,13 @@ async fn main() -> Result<(), Box> { continue; } - let (_, update) = join!( - badge_registry.queue_account(account_id.clone(), HashSet::new()), - update_account(&local_pool, &indexer_pool, &jsonrpc_client, &account_id), - ); - - if let Err(..) = update { - println!("Error updating {}", &account_id); + if let Err(e) = account_send.send(account_id.clone()) { + println!("Error sending to local updater: {e:?}"); } + + badge_registry + .queue_account(account_id.clone(), HashSet::new()) + .await; } println!("Waiting."); From 550d1b011ad641c9b9574bd665b2cd7f552a8c30 Mon Sep 17 00:00:00 2001 From: Jacob Date: Fri, 20 May 2022 23:42:52 -0500 Subject: [PATCH 11/31] vscode extensions --- .vscode/extensions.json | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .vscode/extensions.json diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 0000000..cd446f6 --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,12 @@ +{ + "recommendations": [ + "qufiwefefwoyn.inline-sql-syntax", + "mikestead.dotenv", + "esbenp.prettier-vscode", + "matklad.rust-analyzer", + "bradlc.vscode-tailwindcss", + "netcorext.uuid-generator", + "octref.vetur", + "bungcip.better-toml" + ] +} From 7c03bbf33a1c090a2afda3a56935bf24eb80c11d Mon Sep 17 00:00:00 2001 From: Jacob Date: Fri, 20 May 2022 23:43:21 -0500 Subject: [PATCH 12/31] syntax highlighting for inline sql w/ extension --- service/src/badge/transfer.rs | 12 ++++++------ service/src/indexer.rs | 7 +++---- service/src/local.rs | 10 ++++++---- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/service/src/badge/transfer.rs b/service/src/badge/transfer.rs index d6aafe5..40f0d43 100644 --- a/service/src/badge/transfer.rs +++ b/service/src/badge/transfer.rs @@ -30,12 +30,12 @@ async fn perform_query( } sqlx::query_as::<_, WithResult>( - r#" - select count(*) as result - from action_receipt_actions - where action_kind = 'TRANSFER' - and receipt_predecessor_account_id = $1 - "#, + r#"--sql +select count(*) as result + from action_receipt_actions + where action_kind = 'TRANSFER' + and receipt_predecessor_account_id = $1 +"#, ) .bind(account_id.to_string()) .fetch_one(&connections.indexer_pool) diff --git a/service/src/indexer.rs b/service/src/indexer.rs index f46c1ff..1276756 100644 --- a/service/src/indexer.rs +++ b/service/src/indexer.rs @@ -11,7 +11,7 @@ pub async fn get_recent_actors( } sqlx::query_as::<_, AccountId>( - " + r#"--sql select distinct predecessor_account_id as account_id from receipts where included_in_block_timestamp > $1 @@ -23,7 +23,7 @@ select distinct receiver_account_id as account_id where included_in_block_timestamp > $1 and length(receiver_account_id) != 64 and receiver_account_id like '%.%' -", +"#, ) .bind(timestamp_nanoseconds as i64) // for some reason Encode is not implemented for u64 on Postgres .fetch(pool) @@ -39,7 +39,7 @@ pub async fn calculate_account_score(pool: &PgPool, account_id: &str) -> Result< } sqlx::query_as::<_, WithResult>( - r#" + r#"--sql select coalesce(sum( case when action_kind = 'TRANSFER' @@ -65,7 +65,6 @@ from ( from transactions where (transactions.signer_account_id = $1 or transactions.receiver_account_id = $1) - order by block_timestamp desc ) tx inner join receipts on tx.converted_into_receipt_id = receipts.receipt_id left outer join transaction_actions on tx.transaction_hash = transaction_actions.transaction_hash diff --git a/service/src/local.rs b/service/src/local.rs index 9351f61..38c2046 100644 --- a/service/src/local.rs +++ b/service/src/local.rs @@ -80,8 +80,10 @@ pub async fn query_account( ) -> Result { sqlx::query_as!( AccountRecordDb, - r#" -select id, balance, score, modified, consecutive_errors from account where id = $1 + r#"--sql +select id, balance, score, modified, consecutive_errors + from account + where id = $1 "#, account_id.to_string() ) @@ -105,7 +107,7 @@ pub async fn update_account( &score, &balance ); let err_query_result = sqlx::query!( - r#" + r#"--sql INSERT INTO account(id, consecutive_errors) VALUES ($1, 1) ON CONFLICT (id) DO @@ -126,7 +128,7 @@ INSERT INTO account(id, consecutive_errors) let balance = Decimal::from_u128(balance?); sqlx::query!( - " + "--sql INSERT INTO account(id, balance, score, consecutive_errors) VALUES ($1, $2, $3, 0) ON CONFLICT (id) DO From e24691c0058711e9b1302b6f2d1ebb3ae69d07a4 Mon Sep 17 00:00:00 2001 From: Jacob Date: Mon, 23 May 2022 16:45:35 -0500 Subject: [PATCH 13/31] badge db functions + use Uuid type instead of &'static str for badge id --- Cargo.lock | 1 + service/Cargo.toml | 2 +- ...dex-ix__account_badge__account_id.down.sql | 1 + ...index-ix__account_badge__account_id.up.sql | 1 + service/src/badge/mod.rs | 15 ++++-- service/src/badge/transfer.rs | 46 ++++++++++------ service/src/local.rs | 52 +++++++++++++++++-- service/src/main.rs | 7 ++- 8 files changed, 98 insertions(+), 27 deletions(-) create mode 100644 service/migrations/20220523200724_create-index-ix__account_badge__account_id.down.sql create mode 100644 service/migrations/20220523200724_create-index-ix__account_badge__account_id.up.sql diff --git a/Cargo.lock b/Cargo.lock index 9d703c1..2512d89 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2161,6 +2161,7 @@ dependencies = [ "thiserror", "tokio-stream", "url", + "uuid", "whoami", ] diff --git a/service/Cargo.toml b/service/Cargo.toml index 111159b..6147ee8 100644 --- a/service/Cargo.toml +++ b/service/Cargo.toml @@ -15,7 +15,7 @@ near-jsonrpc-primitives = "0.12.0" near-primitives = "0.12.0" num-traits = "0.2.15" serde = "1.0.137" -sqlx = {version = "0.5", features = ["runtime-tokio-native-tls", "postgres", "decimal", "chrono"]} +sqlx = {version = "0.5", features = ["runtime-tokio-native-tls", "postgres", "decimal", "chrono", "uuid"]} thiserror = "1.0.31" tokio = {version = "1", features = ["full"]} diff --git a/service/migrations/20220523200724_create-index-ix__account_badge__account_id.down.sql b/service/migrations/20220523200724_create-index-ix__account_badge__account_id.down.sql new file mode 100644 index 0000000..6a3a17d --- /dev/null +++ b/service/migrations/20220523200724_create-index-ix__account_badge__account_id.down.sql @@ -0,0 +1 @@ +DROP INDEX ix__account_badge__account_id ON account_badge; diff --git a/service/migrations/20220523200724_create-index-ix__account_badge__account_id.up.sql b/service/migrations/20220523200724_create-index-ix__account_badge__account_id.up.sql new file mode 100644 index 0000000..6428175 --- /dev/null +++ b/service/migrations/20220523200724_create-index-ix__account_badge__account_id.up.sql @@ -0,0 +1 @@ +CREATE INDEX ix__account_badge__account_id ON account_badge USING btree (account_id); diff --git a/service/src/badge/mod.rs b/service/src/badge/mod.rs index 5a7a268..941bb54 100644 --- a/service/src/badge/mod.rs +++ b/service/src/badge/mod.rs @@ -8,6 +8,7 @@ use std::{ use near_primitives::types::AccountId; +use sqlx::types::Uuid; use tokio::sync::{broadcast, Semaphore}; use crate::connections::Connections; @@ -16,14 +17,14 @@ const RESULT_CHANNEL_SIZE: usize = 32; const ACCOUNT_CHANNEL_SIZE: usize = 16; const MAX_SIMULTANEOUS_WORKERS: usize = 5; -pub type BadgeId = &'static str; +pub type BadgeId = sqlx::types::Uuid; pub type BadgeCheckerRegistrationId = usize; #[derive(Clone, Debug)] pub struct BadgeCheckResult { pub account_id: AccountId, pub awarded: HashSet, - pub checked: HashSet, + pub checked: &'static HashSet, } pub type BadgeWorker = fn( @@ -57,10 +58,13 @@ impl BadgeRegistry { self.result_send.subscribe() } - pub fn register(&mut self, badge_ids: &[BadgeId], start_checker: BadgeWorker) { + pub fn register<'a, T>(&mut self, badge_ids: T, start_checker: BadgeWorker) + where + T: IntoIterator, + { static REGISTRATION_ID: AtomicUsize = AtomicUsize::new(0); - let badge_ids = badge_ids.iter().collect(); + let badge_ids: HashSet<_> = badge_ids.into_iter().collect(); if !self .badge_to_registration @@ -75,7 +79,8 @@ impl BadgeRegistry { let (account_send, account_recv) = broadcast::channel(ACCOUNT_CHANNEL_SIZE); for badge_id in badge_ids { - self.badge_to_registration.insert(badge_id, registration_id); + self.badge_to_registration + .insert(*badge_id, registration_id); } self.registration_to_fn.insert( diff --git a/service/src/badge/transfer.rs b/service/src/badge/transfer.rs index 40f0d43..3dfa8c5 100644 --- a/service/src/badge/transfer.rs +++ b/service/src/badge/transfer.rs @@ -1,18 +1,31 @@ -use std::sync::Arc; +use std::{collections::HashSet, str::FromStr, sync::Arc}; use crate::badge::BadgeCheckResult; use futures::FutureExt; +use lazy_static::lazy_static; use near_primitives::types::AccountId; +use sqlx::types::Uuid; use thiserror::Error; use tokio::sync::{broadcast, Semaphore}; use super::Connections; -pub static TRANSFER_1: &'static str = "b36c3dd2-b8b0-4098-b227-63290f009668"; -pub static TRANSFER_10: &'static str = "609b2017-9534-4737-b86b-6ee4897fc4f9"; -pub static TRANSFER_100: &'static str = "64ce9af9-52ac-49dc-96fd-031b4fa2efad"; -pub static BADGE_IDS: &'static [&'static str] = &[TRANSFER_1, TRANSFER_10, TRANSFER_100]; +// Badge UUIDs must correlate with badge renderers on the frontend + +lazy_static! { + pub static ref TRANSFER_1: Uuid = + Uuid::from_str("b36c3dd2-b8b0-4098-b227-63290f009668").unwrap(); + pub static ref TRANSFER_10: Uuid = + Uuid::from_str("609b2017-9534-4737-b86b-6ee4897fc4f9").unwrap(); + pub static ref TRANSFER_100: Uuid = + Uuid::from_str("64ce9af9-52ac-49dc-96fd-031b4fa2efad").unwrap(); + pub static ref BADGE_IDS: HashSet = HashSet::from([ + *TRANSFER_1, + *TRANSFER_10, + *TRANSFER_100, + ]); +} #[derive(Error, Debug)] enum TransferBadgeError { @@ -43,21 +56,22 @@ select count(*) as result result .map(|r| { let total = r.result; - let awarded = if total >= 100 { - BADGE_IDS - } else if total >= 10 { - &BADGE_IDS[..2] - } else if total >= 1 { - &BADGE_IDS[..1] - } else { - &[] + let mut awarded = HashSet::new(); + + if total >= 100 { + awarded.insert(TRANSFER_100.to_owned()); + } + if total >= 10 { + awarded.insert(TRANSFER_10.to_owned()); + } + if total >= 1 { + awarded.insert(TRANSFER_1.to_owned()); } - .to_vec(); BadgeCheckResult { account_id, - awarded: awarded.into_iter().collect(), - checked: BADGE_IDS.to_vec().into_iter().collect(), + awarded: awarded, + checked: &BADGE_IDS, } }) .map_err(|e| TransferBadgeError::from(e)) diff --git a/service/src/local.rs b/service/src/local.rs index 38c2046..0b2c99f 100644 --- a/service/src/local.rs +++ b/service/src/local.rs @@ -1,11 +1,14 @@ -use std::{str::FromStr, sync::Arc}; +use std::{collections::HashSet, str::FromStr, sync::Arc}; use chrono::{Duration, NaiveDateTime, TimeZone}; -use futures::FutureExt; +use futures::{FutureExt, StreamExt, TryFutureExt, TryStreamExt}; use near_jsonrpc_client::{errors::JsonRpcError, methods::query::RpcQueryError}; use near_primitives::types::AccountId; use num_traits::FromPrimitive; -use sqlx::{types::Decimal, FromRow, PgPool}; +use sqlx::{ + types::{Decimal, Uuid}, + FromRow, PgPool, +}; use tokio::{ join, sync::{broadcast, Semaphore}, @@ -92,6 +95,49 @@ select id, balance, score, modified, consecutive_errors .await } +#[derive(FromRow)] +pub struct AccountBadgeRecordDb { + pub badge_id: Uuid, +} + +pub async fn add_badge_for_account( + local_pool: &PgPool, + account_id: &AccountId, + badge_id: &Uuid, +) -> Result<(), sqlx::Error> { + sqlx::query!( + r#"--sql +insert into account_badge(account_id, badge_id) + values ($1, $2) + on conflict (account_id, badge_id) do nothing +"#, + account_id.to_string(), + &badge_id + ) + .execute(local_pool) + .map_ok(|_| ()) + .await +} + +pub async fn get_badges_for_account( + local_pool: &PgPool, + account_id: &AccountId, +) -> Result, sqlx::Error> { + sqlx::query_as!( + AccountBadgeRecordDb, + r#"--sql +select badge_id + from account_badge + where account_id = $1 +"#, + account_id.to_string() + ) + .fetch(local_pool) + .map(|b| b.map(|a| a.badge_id)) + .try_collect::>() + .await +} + pub async fn update_account( connections: &Connections, account_id: &AccountId, diff --git a/service/src/main.rs b/service/src/main.rs index 03a8c6c..d507036 100644 --- a/service/src/main.rs +++ b/service/src/main.rs @@ -10,7 +10,10 @@ use sqlx::{migrate, postgres::PgPoolOptions}; use tokio::sync::{broadcast, Semaphore}; use crate::{ - badge::{transfer, BadgeRegistry}, + badge::{ + transfer::{self, TRANSFER_1}, + BadgeRegistry, + }, connections::Connections, indexer::get_recent_actors, local::query_account, @@ -93,7 +96,7 @@ async fn main() -> Result<(), Box> { let (account_send, account_recv) = broadcast::channel(16); local::start_local_updater(local_semaphore, connections.clone(), account_recv); - badge_registry.register(transfer::BADGE_IDS, transfer::run); + badge_registry.register(&*transfer::BADGE_IDS, transfer::run); let now = chrono::Utc::now(); From c69819148ed2f04f27728fcfe21149e1462ea014 Mon Sep 17 00:00:00 2001 From: Jacob Date: Mon, 23 May 2022 17:13:43 -0500 Subject: [PATCH 14/31] badge exclusion --- service/src/main.rs | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/service/src/main.rs b/service/src/main.rs index d507036..9806599 100644 --- a/service/src/main.rs +++ b/service/src/main.rs @@ -7,16 +7,16 @@ use near_jsonrpc_client::JsonRpcClient; use near_primitives::types::AccountId; use serde::Deserialize; use sqlx::{migrate, postgres::PgPoolOptions}; -use tokio::sync::{broadcast, Semaphore}; +use tokio::{ + join, + sync::{broadcast, Semaphore}, +}; use crate::{ - badge::{ - transfer::{self, TRANSFER_1}, - BadgeRegistry, - }, + badge::{transfer, BadgeRegistry}, connections::Connections, indexer::get_recent_actors, - local::query_account, + local::{get_badges_for_account, query_account}, }; #[derive(Deserialize)] @@ -102,7 +102,11 @@ async fn main() -> Result<(), Box> { for account in accounts { let account_id: AccountId = account.parse().unwrap(); - let account_record = query_account(&local_pool, &account_id).await; + let (account_record, existing_badges) = join!( + query_account(&local_pool, &account_id), + get_badges_for_account(&local_pool, &account_id), + ); + let is_update_allowed = account_record .ok() .and_then(|r| r.next_update_allowed_at()) @@ -119,7 +123,10 @@ async fn main() -> Result<(), Box> { } badge_registry - .queue_account(account_id.clone(), HashSet::new()) + .queue_account( + account_id.clone(), + existing_badges.unwrap_or_else(|_| HashSet::new()), + ) .await; } From bc6bfe096247d1f05cb231b93eb068b937946bc5 Mon Sep 17 00:00:00 2001 From: Jacob Date: Mon, 23 May 2022 17:52:02 -0500 Subject: [PATCH 15/31] removes fk restraint so badges can be added before normal acct update completes --- .../20220523224756_remove-fk__account_badge__account.down.sql | 2 ++ .../20220523224756_remove-fk__account_badge__account.up.sql | 1 + 2 files changed, 3 insertions(+) create mode 100644 service/migrations/20220523224756_remove-fk__account_badge__account.down.sql create mode 100644 service/migrations/20220523224756_remove-fk__account_badge__account.up.sql diff --git a/service/migrations/20220523224756_remove-fk__account_badge__account.down.sql b/service/migrations/20220523224756_remove-fk__account_badge__account.down.sql new file mode 100644 index 0000000..0bc7b2b --- /dev/null +++ b/service/migrations/20220523224756_remove-fk__account_badge__account.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE ONLY account_badge + ADD CONSTRAINT fk__account_badge__account FOREIGN KEY (account_id) REFERENCES account(id); diff --git a/service/migrations/20220523224756_remove-fk__account_badge__account.up.sql b/service/migrations/20220523224756_remove-fk__account_badge__account.up.sql new file mode 100644 index 0000000..5d39454 --- /dev/null +++ b/service/migrations/20220523224756_remove-fk__account_badge__account.up.sql @@ -0,0 +1 @@ +alter table account_badge drop constraint fk__account_badge__account; From 251b97ed83ee0374c0b48964351a84cb0bf4dbe1 Mon Sep 17 00:00:00 2001 From: Jacob Date: Mon, 23 May 2022 17:55:43 -0500 Subject: [PATCH 16/31] badge updater working --- service/src/badge/mod.rs | 63 +++++++++++++++++++++-------------- service/src/badge/transfer.rs | 10 ++---- service/src/main.rs | 30 +++++------------ 3 files changed, 50 insertions(+), 53 deletions(-) diff --git a/service/src/badge/mod.rs b/service/src/badge/mod.rs index 941bb54..97f3c02 100644 --- a/service/src/badge/mod.rs +++ b/service/src/badge/mod.rs @@ -9,11 +9,10 @@ use std::{ use near_primitives::types::AccountId; use sqlx::types::Uuid; -use tokio::sync::{broadcast, Semaphore}; +use tokio::sync::{broadcast, mpsc, Semaphore}; use crate::connections::Connections; -const RESULT_CHANNEL_SIZE: usize = 32; const ACCOUNT_CHANNEL_SIZE: usize = 16; const MAX_SIMULTANEOUS_WORKERS: usize = 5; @@ -30,34 +29,30 @@ pub struct BadgeCheckResult { pub type BadgeWorker = fn( semaphore: Semaphore, connections: Connections, - input: broadcast::Receiver, - output: broadcast::Sender, + input: broadcast::Receiver<(AccountId, mpsc::Sender)>, ); pub struct BadgeRegistry { connections: Connections, - result_send: broadcast::Sender, - registration_to_fn: - HashMap, broadcast::Sender)>, + registration_to_fn: HashMap< + BadgeCheckerRegistrationId, + ( + Arc, + broadcast::Sender<(AccountId, mpsc::Sender)>, + ), + >, badge_to_registration: HashMap, } impl BadgeRegistry { pub fn new(connections: Connections) -> Self { - let (result_send, _) = broadcast::channel(RESULT_CHANNEL_SIZE); - Self { connections, - result_send, registration_to_fn: HashMap::new(), badge_to_registration: HashMap::new(), } } - pub fn subscribe(&self) -> broadcast::Receiver { - self.result_send.subscribe() - } - pub fn register<'a, T>(&mut self, badge_ids: T, start_checker: BadgeWorker) where T: IntoIterator, @@ -92,11 +87,14 @@ impl BadgeRegistry { Semaphore::new(MAX_SIMULTANEOUS_WORKERS), self.connections.clone(), account_recv, - self.result_send.clone(), ); } - pub async fn queue_account(&self, account_id: AccountId, ignore_badge_ids: HashSet) { + pub async fn queue_account( + &self, + account_id: AccountId, + ignore_badge_ids: HashSet, + ) -> HashSet { let registration_ids = self .badge_to_registration .iter() @@ -109,12 +107,26 @@ impl BadgeRegistry { }) .collect::>(); + let (result_send, mut result_recv) = mpsc::channel(registration_ids.len()); + for registration_id in registration_ids { let (ref semaphore, ref sender) = self.registration_to_fn[registration_id]; let permit = semaphore.acquire().await.unwrap(); - sender.send(account_id.clone()).unwrap(); // TODO: Remove unwrap() + sender + .send((account_id.clone(), result_send.clone())) + .unwrap(); // TODO: Remove unwrap() drop(permit); } + + drop(result_send); + + let mut result = HashSet::new(); + + while let Some(part) = result_recv.recv().await { + result.extend(part.awarded); + } + + result } } @@ -122,17 +134,18 @@ impl BadgeRegistry { macro_rules! create_badge_worker { ($query_fn: ident) => { pub fn run( - semaphore: Semaphore, - connections: Connections, - mut input: broadcast::Receiver, - output: broadcast::Sender, + semaphore: tokio::sync::Semaphore, + connections: $crate::connections::Connections, + mut input: tokio::sync::broadcast::Receiver<( + near_primitives::types::AccountId, + tokio::sync::mpsc::Sender<$crate::badge::BadgeCheckResult>, + )>, ) { tokio::spawn(async move { - let semaphore = Arc::new(semaphore); + let semaphore = std::sync::Arc::new(semaphore); - while let Ok(account_id) = input.recv().await { + while let Ok((account_id, output)) = input.recv().await { let connections = connections.clone(); - let output = output.clone(); let semaphore = semaphore.clone(); tokio::spawn(async move { @@ -143,7 +156,7 @@ macro_rules! create_badge_worker { drop(permit); match result { Ok(result) => { - output.send(result).unwrap(); // TODO: Log instead of unwrap + output.send(result).await.unwrap(); // TODO: Log instead of unwrap } Err(e) => println!("{e:?}"), } diff --git a/service/src/badge/transfer.rs b/service/src/badge/transfer.rs index 3dfa8c5..4743278 100644 --- a/service/src/badge/transfer.rs +++ b/service/src/badge/transfer.rs @@ -1,4 +1,4 @@ -use std::{collections::HashSet, str::FromStr, sync::Arc}; +use std::{collections::HashSet, str::FromStr}; use crate::badge::BadgeCheckResult; @@ -7,7 +7,6 @@ use lazy_static::lazy_static; use near_primitives::types::AccountId; use sqlx::types::Uuid; use thiserror::Error; -use tokio::sync::{broadcast, Semaphore}; use super::Connections; @@ -20,11 +19,8 @@ lazy_static! { Uuid::from_str("609b2017-9534-4737-b86b-6ee4897fc4f9").unwrap(); pub static ref TRANSFER_100: Uuid = Uuid::from_str("64ce9af9-52ac-49dc-96fd-031b4fa2efad").unwrap(); - pub static ref BADGE_IDS: HashSet = HashSet::from([ - *TRANSFER_1, - *TRANSFER_10, - *TRANSFER_100, - ]); + pub static ref BADGE_IDS: HashSet = + HashSet::from([*TRANSFER_1, *TRANSFER_10, *TRANSFER_100,]); } #[derive(Error, Debug)] diff --git a/service/src/main.rs b/service/src/main.rs index 9806599..177c70f 100644 --- a/service/src/main.rs +++ b/service/src/main.rs @@ -16,7 +16,7 @@ use crate::{ badge::{transfer, BadgeRegistry}, connections::Connections, indexer::get_recent_actors, - local::{get_badges_for_account, query_account}, + local::{add_badge_for_account, get_badges_for_account, query_account}, }; #[derive(Deserialize)] @@ -77,21 +77,6 @@ async fn main() -> Result<(), Box> { let mut badge_registry = BadgeRegistry::new(connections.clone()); - let mut r = badge_registry.subscribe(); - - let target = accounts.len(); - - let badge_handle = tokio::spawn(async move { - println!("Listening for badge results"); - let mut done = 0; - while let Ok(result) = r.recv().await { - done += 1; - let account_id = result.account_id; - println!("{done} / {target} - {account_id}"); - // TODO: write badges back to local db - } - }); - let local_semaphore = Semaphore::new(5); let (account_send, account_recv) = broadcast::channel(16); local::start_local_updater(local_semaphore, connections.clone(), account_recv); @@ -122,17 +107,20 @@ async fn main() -> Result<(), Box> { println!("Error sending to local updater: {e:?}"); } - badge_registry + let awarded_badges = badge_registry .queue_account( account_id.clone(), existing_badges.unwrap_or_else(|_| HashSet::new()), ) .await; - } - - println!("Waiting."); - badge_handle.await.unwrap(); + for badge_id in awarded_badges { + match add_badge_for_account(&local_pool, &account_id, &badge_id).await { + Ok(_) => println!("Added badge {badge_id} to {account_id}"), + Err(e) => println!("Failed to add badge {badge_id} to {account_id}: {e:?}"), + } + } + } Ok(()) } From a46e07b7ea4ec9969b925c2664fb0010f5e272d6 Mon Sep 17 00:00:00 2001 From: Jacob Date: Mon, 23 May 2022 18:10:10 -0500 Subject: [PATCH 17/31] move constants to main.rs --- service/src/badge/mod.rs | 39 ++++++++++++++------------------------- service/src/main.rs | 9 +++++++++ 2 files changed, 23 insertions(+), 25 deletions(-) diff --git a/service/src/badge/mod.rs b/service/src/badge/mod.rs index 97f3c02..56b63d4 100644 --- a/service/src/badge/mod.rs +++ b/service/src/badge/mod.rs @@ -1,9 +1,6 @@ use std::{ collections::{HashMap, HashSet}, - sync::{ - atomic::{AtomicUsize, Ordering}, - Arc, - }, + sync::atomic::{AtomicUsize, Ordering}, }; use near_primitives::types::AccountId; @@ -11,10 +8,9 @@ use near_primitives::types::AccountId; use sqlx::types::Uuid; use tokio::sync::{broadcast, mpsc, Semaphore}; -use crate::connections::Connections; - -const ACCOUNT_CHANNEL_SIZE: usize = 16; -const MAX_SIMULTANEOUS_WORKERS: usize = 5; +use crate::{ + connections::Connections, MAX_SIMULTANEOUS_ACCOUNTS, MAX_SIMULTANEOUS_WORKERS_PER_BADGE, +}; pub type BadgeId = sqlx::types::Uuid; pub type BadgeCheckerRegistrationId = usize; @@ -36,10 +32,7 @@ pub struct BadgeRegistry { connections: Connections, registration_to_fn: HashMap< BadgeCheckerRegistrationId, - ( - Arc, - broadcast::Sender<(AccountId, mpsc::Sender)>, - ), + broadcast::Sender<(AccountId, mpsc::Sender)>, >, badge_to_registration: HashMap, } @@ -71,20 +64,18 @@ impl BadgeRegistry { } let registration_id = REGISTRATION_ID.fetch_add(1, Ordering::Relaxed); - let (account_send, account_recv) = broadcast::channel(ACCOUNT_CHANNEL_SIZE); + let (account_send, account_recv) = broadcast::channel(MAX_SIMULTANEOUS_ACCOUNTS); for badge_id in badge_ids { self.badge_to_registration .insert(*badge_id, registration_id); } - self.registration_to_fn.insert( - registration_id, - (Arc::new(Semaphore::new(ACCOUNT_CHANNEL_SIZE)), account_send), - ); + self.registration_to_fn + .insert(registration_id, account_send); start_checker( - Semaphore::new(MAX_SIMULTANEOUS_WORKERS), + Semaphore::new(MAX_SIMULTANEOUS_WORKERS_PER_BADGE), self.connections.clone(), account_recv, ); @@ -110,12 +101,10 @@ impl BadgeRegistry { let (result_send, mut result_recv) = mpsc::channel(registration_ids.len()); for registration_id in registration_ids { - let (ref semaphore, ref sender) = self.registration_to_fn[registration_id]; - let permit = semaphore.acquire().await.unwrap(); + let sender = &self.registration_to_fn[registration_id]; sender .send((account_id.clone(), result_send.clone())) .unwrap(); // TODO: Remove unwrap() - drop(permit); } drop(result_send); @@ -134,7 +123,7 @@ impl BadgeRegistry { macro_rules! create_badge_worker { ($query_fn: ident) => { pub fn run( - semaphore: tokio::sync::Semaphore, + simultaneous_workers: tokio::sync::Semaphore, connections: $crate::connections::Connections, mut input: tokio::sync::broadcast::Receiver<( near_primitives::types::AccountId, @@ -142,14 +131,14 @@ macro_rules! create_badge_worker { )>, ) { tokio::spawn(async move { - let semaphore = std::sync::Arc::new(semaphore); + let simultaneous_workers = std::sync::Arc::new(simultaneous_workers); while let Ok((account_id, output)) = input.recv().await { let connections = connections.clone(); - let semaphore = semaphore.clone(); + let simultaneous_workers = simultaneous_workers.clone(); tokio::spawn(async move { - let permit = semaphore.acquire().await.unwrap(); + let permit = simultaneous_workers.acquire().await.unwrap(); println!("Acquired for {account_id}"); let result = $query_fn(connections, account_id.clone()).await; println!("Finished {account_id}"); diff --git a/service/src/main.rs b/service/src/main.rs index 177c70f..75e5b78 100644 --- a/service/src/main.rs +++ b/service/src/main.rs @@ -19,6 +19,9 @@ use crate::{ local::{add_badge_for_account, get_badges_for_account, query_account}, }; +pub const MAX_SIMULTANEOUS_ACCOUNTS: usize = 16; +pub const MAX_SIMULTANEOUS_WORKERS_PER_BADGE: usize = 5; + #[derive(Deserialize)] struct Configuration { #[allow(unused)] // env var read by default by sqlx @@ -85,7 +88,12 @@ async fn main() -> Result<(), Box> { let now = chrono::Utc::now(); + let simultaneous_accounts = Semaphore::new(MAX_SIMULTANEOUS_ACCOUNTS); + for account in accounts { + let permit = simultaneous_accounts.acquire().await; + let local_pool = local_pool.clone(); + // tokio::spawn(async move { let account_id: AccountId = account.parse().unwrap(); let (account_record, existing_badges) = join!( query_account(&local_pool, &account_id), @@ -120,6 +128,7 @@ async fn main() -> Result<(), Box> { Err(e) => println!("Failed to add badge {badge_id} to {account_id}: {e:?}"), } } + // }); } Ok(()) From 4288e1f00a2fbb7b3a648a852a1360b10f4c4b0e Mon Sep 17 00:00:00 2001 From: Jacob Date: Mon, 23 May 2022 19:41:00 -0500 Subject: [PATCH 18/31] sharing w/ Arc instead of cloning + move max simul acct semaphore up higher --- service/src/badge/mod.rs | 18 +++--- service/src/badge/transfer.rs | 4 +- service/src/local.rs | 6 +- service/src/main.rs | 117 +++++++++++++++++++--------------- 4 files changed, 78 insertions(+), 67 deletions(-) diff --git a/service/src/badge/mod.rs b/service/src/badge/mod.rs index 56b63d4..93adb26 100644 --- a/service/src/badge/mod.rs +++ b/service/src/badge/mod.rs @@ -1,6 +1,6 @@ use std::{ collections::{HashMap, HashSet}, - sync::atomic::{AtomicUsize, Ordering}, + sync::{atomic::{AtomicUsize, Ordering}, Arc}, }; use near_primitives::types::AccountId; @@ -23,13 +23,13 @@ pub struct BadgeCheckResult { } pub type BadgeWorker = fn( - semaphore: Semaphore, - connections: Connections, + semaphore: Arc, + connections: Arc, input: broadcast::Receiver<(AccountId, mpsc::Sender)>, ); pub struct BadgeRegistry { - connections: Connections, + connections: Arc, registration_to_fn: HashMap< BadgeCheckerRegistrationId, broadcast::Sender<(AccountId, mpsc::Sender)>, @@ -38,7 +38,7 @@ pub struct BadgeRegistry { } impl BadgeRegistry { - pub fn new(connections: Connections) -> Self { + pub fn new(connections: Arc) -> Self { Self { connections, registration_to_fn: HashMap::new(), @@ -75,7 +75,7 @@ impl BadgeRegistry { .insert(registration_id, account_send); start_checker( - Semaphore::new(MAX_SIMULTANEOUS_WORKERS_PER_BADGE), + Arc::new(Semaphore::new(MAX_SIMULTANEOUS_WORKERS_PER_BADGE)), self.connections.clone(), account_recv, ); @@ -123,16 +123,14 @@ impl BadgeRegistry { macro_rules! create_badge_worker { ($query_fn: ident) => { pub fn run( - simultaneous_workers: tokio::sync::Semaphore, - connections: $crate::connections::Connections, + simultaneous_workers: std::sync::Arc, + connections: std::sync::Arc<$crate::connections::Connections>, mut input: tokio::sync::broadcast::Receiver<( near_primitives::types::AccountId, tokio::sync::mpsc::Sender<$crate::badge::BadgeCheckResult>, )>, ) { tokio::spawn(async move { - let simultaneous_workers = std::sync::Arc::new(simultaneous_workers); - while let Ok((account_id, output)) = input.recv().await { let connections = connections.clone(); let simultaneous_workers = simultaneous_workers.clone(); diff --git a/service/src/badge/transfer.rs b/service/src/badge/transfer.rs index 4743278..7ae3160 100644 --- a/service/src/badge/transfer.rs +++ b/service/src/badge/transfer.rs @@ -1,4 +1,4 @@ -use std::{collections::HashSet, str::FromStr}; +use std::{collections::HashSet, str::FromStr, sync::Arc}; use crate::badge::BadgeCheckResult; @@ -30,7 +30,7 @@ enum TransferBadgeError { } async fn perform_query( - connections: Connections, + connections: Arc, account_id: AccountId, ) -> Result { #[derive(sqlx::FromRow)] diff --git a/service/src/local.rs b/service/src/local.rs index 0b2c99f..9f6af94 100644 --- a/service/src/local.rs +++ b/service/src/local.rs @@ -194,13 +194,11 @@ INSERT INTO account(id, balance, score, consecutive_errors) } pub fn start_local_updater( - semaphore: Semaphore, - connections: Connections, + semaphore: Arc, + connections: Arc, mut input: broadcast::Receiver, ) { tokio::spawn(async move { - let semaphore = Arc::new(semaphore); - while let Ok(account_id) = input.recv().await { let connections = connections.clone(); let semaphore = semaphore.clone(); diff --git a/service/src/main.rs b/service/src/main.rs index 75e5b78..f8af20a 100644 --- a/service/src/main.rs +++ b/service/src/main.rs @@ -1,8 +1,9 @@ -use std::{collections::HashSet, ops::Sub}; +use std::{collections::HashSet, ops::Sub, sync::Arc}; use chrono::Duration; use dotenv::dotenv; +use futures::future::join_all; use near_jsonrpc_client::JsonRpcClient; use near_primitives::types::AccountId; use serde::Deserialize; @@ -43,17 +44,17 @@ async fn main() -> Result<(), Box> { let config = envy::from_env::().expect("Missing environment variables"); - let local_pool = PgPoolOptions::new() + let local_pool = (PgPoolOptions::new() .max_connections(5) .connect(&config.database_url) - .await?; + .await?); - let indexer_pool = PgPoolOptions::new() + let indexer_pool = (PgPoolOptions::new() .max_connections(5) .connect(&config.indexer_url) - .await?; + .await?); - let jsonrpc_client = JsonRpcClient::connect(&config.rpc_url); + let jsonrpc_client = (JsonRpcClient::connect(&config.rpc_url)); println!("Requesting accounts"); @@ -72,64 +73,78 @@ async fn main() -> Result<(), Box> { println!("Creating badge registry"); - let connections = Connections { - local_pool: local_pool.clone(), - indexer_pool: indexer_pool.clone(), - rpc_client: jsonrpc_client.clone(), - }; + let connections = Arc::new(Connections { + local_pool: (local_pool), + indexer_pool: (indexer_pool), + rpc_client: jsonrpc_client, + }); let mut badge_registry = BadgeRegistry::new(connections.clone()); - let local_semaphore = Semaphore::new(5); + let local_semaphore = Arc::new(Semaphore::new(5)); let (account_send, account_recv) = broadcast::channel(16); local::start_local_updater(local_semaphore, connections.clone(), account_recv); badge_registry.register(&*transfer::BADGE_IDS, transfer::run); - let now = chrono::Utc::now(); + let simultaneous_accounts = Arc::new(Semaphore::new(MAX_SIMULTANEOUS_ACCOUNTS)); - let simultaneous_accounts = Semaphore::new(MAX_SIMULTANEOUS_ACCOUNTS); + let badge_registry = Arc::new(badge_registry); + + let mut join_handles = Vec::new(); for account in accounts { - let permit = simultaneous_accounts.acquire().await; - let local_pool = local_pool.clone(); - // tokio::spawn(async move { - let account_id: AccountId = account.parse().unwrap(); - let (account_record, existing_badges) = join!( - query_account(&local_pool, &account_id), - get_badges_for_account(&local_pool, &account_id), - ); - - let is_update_allowed = account_record - .ok() - .and_then(|r| r.next_update_allowed_at()) - .map(|cutoff| now >= cutoff) - .unwrap_or(true); - - if !is_update_allowed { - println!("Disallowing update for {account_id}"); - continue; - } - - if let Err(e) = account_send.send(account_id.clone()) { - println!("Error sending to local updater: {e:?}"); - } - - let awarded_badges = badge_registry - .queue_account( - account_id.clone(), - existing_badges.unwrap_or_else(|_| HashSet::new()), - ) - .await; - - for badge_id in awarded_badges { - match add_badge_for_account(&local_pool, &account_id, &badge_id).await { - Ok(_) => println!("Added badge {badge_id} to {account_id}"), - Err(e) => println!("Failed to add badge {badge_id} to {account_id}: {e:?}"), + let simultaneous_accounts = Arc::clone(&simultaneous_accounts); + let permit = simultaneous_accounts.acquire_owned().await; + let account_send = account_send.clone(); + let connections = Arc::clone(&connections); + let badge_registry = Arc::clone(&badge_registry); + + let join_handle = tokio::spawn(async move { + let account_id: AccountId = account.parse().unwrap(); + let (account_record, existing_badges) = join!( + query_account(&connections.local_pool, &account_id), + get_badges_for_account(&connections.local_pool, &account_id), + ); + + let now = chrono::Utc::now(); + + let is_update_allowed = account_record + .ok() + .and_then(|r| r.next_update_allowed_at()) + .map(|cutoff| now >= cutoff) + .unwrap_or(true); + + if !is_update_allowed { + println!("Disallowing update for {account_id}"); + return; + } + + if let Err(e) = account_send.send(account_id.clone()) { + println!("Error sending to local updater: {e:?}"); + } + + let awarded_badges = badge_registry + .queue_account( + account_id.clone(), + existing_badges.unwrap_or_else(|_| HashSet::new()), + ) + .await; + + for badge_id in awarded_badges { + match add_badge_for_account(&connections.local_pool, &account_id, &badge_id).await { + Ok(_) => println!("Added badge {badge_id} to {account_id}"), + Err(e) => println!("Failed to add badge {badge_id} to {account_id}: {e:?}"), + } } - } - // }); + + drop(permit); + }); + + join_handles.push(join_handle); } + join_all(join_handles).await; + Ok(()) } From 587605641ef781403bae775ad48ceae1741f6b95 Mon Sep 17 00:00:00 2001 From: Jacob Date: Mon, 23 May 2022 20:10:48 -0500 Subject: [PATCH 19/31] reorg, fmt, rename --- service/src/badge/mod.rs | 5 ++++- service/src/indexer.rs | 11 +++++++---- service/src/main.rs | 36 ++++++++++++++++-------------------- 3 files changed, 27 insertions(+), 25 deletions(-) diff --git a/service/src/badge/mod.rs b/service/src/badge/mod.rs index 93adb26..aba67bd 100644 --- a/service/src/badge/mod.rs +++ b/service/src/badge/mod.rs @@ -1,6 +1,9 @@ use std::{ collections::{HashMap, HashSet}, - sync::{atomic::{AtomicUsize, Ordering}, Arc}, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, }; use near_primitives::types::AccountId; diff --git a/service/src/indexer.rs b/service/src/indexer.rs index 1276756..00edcca 100644 --- a/service/src/indexer.rs +++ b/service/src/indexer.rs @@ -2,7 +2,7 @@ use futures::{StreamExt, TryFutureExt, TryStreamExt}; use sqlx::PgPool; pub async fn get_recent_actors( - pool: &PgPool, + indexer_pool: &PgPool, timestamp_nanoseconds: u64, ) -> Result, sqlx::Error> { #[derive(sqlx::FromRow)] @@ -26,13 +26,16 @@ select distinct receiver_account_id as account_id "#, ) .bind(timestamp_nanoseconds as i64) // for some reason Encode is not implemented for u64 on Postgres - .fetch(pool) + .fetch(indexer_pool) .map(|i| i.map(|a| a.account_id)) .try_collect::>() .await } -pub async fn calculate_account_score(pool: &PgPool, account_id: &str) -> Result { +pub async fn calculate_account_score( + indexer_pool: &PgPool, + account_id: &str, +) -> Result { #[derive(sqlx::FromRow)] struct WithResult { pub result: i64, @@ -71,7 +74,7 @@ left outer join transaction_actions on tx.transaction_hash = transaction_actions "#, ) .bind(account_id) - .fetch_one(pool) + .fetch_one(indexer_pool) .map_ok(|a| a.result as u32) .await } diff --git a/service/src/main.rs b/service/src/main.rs index f8af20a..57e47c3 100644 --- a/service/src/main.rs +++ b/service/src/main.rs @@ -44,22 +44,28 @@ async fn main() -> Result<(), Box> { let config = envy::from_env::().expect("Missing environment variables"); - let local_pool = (PgPoolOptions::new() + let local_pool = PgPoolOptions::new() .max_connections(5) .connect(&config.database_url) - .await?); + .await?; - let indexer_pool = (PgPoolOptions::new() + let indexer_pool = PgPoolOptions::new() .max_connections(5) .connect(&config.indexer_url) - .await?); + .await?; - let jsonrpc_client = (JsonRpcClient::connect(&config.rpc_url)); + let rpc_client = JsonRpcClient::connect(&config.rpc_url); + + let connections = Arc::new(Connections { + local_pool, + indexer_pool, + rpc_client, + }); println!("Requesting accounts"); let accounts = get_recent_actors( - &indexer_pool, + &connections.indexer_pool, chrono::Utc::now() .sub(Duration::minutes(10)) .timestamp_nanos() @@ -73,27 +79,20 @@ async fn main() -> Result<(), Box> { println!("Creating badge registry"); - let connections = Arc::new(Connections { - local_pool: (local_pool), - indexer_pool: (indexer_pool), - rpc_client: jsonrpc_client, - }); - - let mut badge_registry = BadgeRegistry::new(connections.clone()); - let local_semaphore = Arc::new(Semaphore::new(5)); let (account_send, account_recv) = broadcast::channel(16); local::start_local_updater(local_semaphore, connections.clone(), account_recv); + let mut badge_registry = BadgeRegistry::new(connections.clone()); badge_registry.register(&*transfer::BADGE_IDS, transfer::run); + let badge_registry = Arc::new(badge_registry); let simultaneous_accounts = Arc::new(Semaphore::new(MAX_SIMULTANEOUS_ACCOUNTS)); - let badge_registry = Arc::new(badge_registry); - let mut join_handles = Vec::new(); for account in accounts { + let account_id: AccountId = account.parse().unwrap(); let simultaneous_accounts = Arc::clone(&simultaneous_accounts); let permit = simultaneous_accounts.acquire_owned().await; let account_send = account_send.clone(); @@ -101,18 +100,15 @@ async fn main() -> Result<(), Box> { let badge_registry = Arc::clone(&badge_registry); let join_handle = tokio::spawn(async move { - let account_id: AccountId = account.parse().unwrap(); let (account_record, existing_badges) = join!( query_account(&connections.local_pool, &account_id), get_badges_for_account(&connections.local_pool, &account_id), ); - let now = chrono::Utc::now(); - let is_update_allowed = account_record .ok() .and_then(|r| r.next_update_allowed_at()) - .map(|cutoff| now >= cutoff) + .map(|cutoff| chrono::Utc::now() >= cutoff) .unwrap_or(true); if !is_update_allowed { From 658f73fb9174e1452d7cf0c40262c075cd46239e Mon Sep 17 00:00:00 2001 From: Jacob Date: Mon, 23 May 2022 21:05:00 -0500 Subject: [PATCH 20/31] move constant to env var --- service/src/badge/mod.rs | 25 ++++-------- service/src/local.rs | 11 +----- service/src/main.rs | 82 +++++++++++++++++++++++++--------------- 3 files changed, 59 insertions(+), 59 deletions(-) diff --git a/service/src/badge/mod.rs b/service/src/badge/mod.rs index aba67bd..410c282 100644 --- a/service/src/badge/mod.rs +++ b/service/src/badge/mod.rs @@ -9,11 +9,9 @@ use std::{ use near_primitives::types::AccountId; use sqlx::types::Uuid; -use tokio::sync::{broadcast, mpsc, Semaphore}; +use tokio::sync::{broadcast, mpsc}; -use crate::{ - connections::Connections, MAX_SIMULTANEOUS_ACCOUNTS, MAX_SIMULTANEOUS_WORKERS_PER_BADGE, -}; +use crate::connections::Connections; pub type BadgeId = sqlx::types::Uuid; pub type BadgeCheckerRegistrationId = usize; @@ -26,12 +24,12 @@ pub struct BadgeCheckResult { } pub type BadgeWorker = fn( - semaphore: Arc, connections: Arc, input: broadcast::Receiver<(AccountId, mpsc::Sender)>, ); pub struct BadgeRegistry { + account_threads: usize, connections: Arc, registration_to_fn: HashMap< BadgeCheckerRegistrationId, @@ -41,8 +39,9 @@ pub struct BadgeRegistry { } impl BadgeRegistry { - pub fn new(connections: Arc) -> Self { + pub fn new(connections: Arc, account_threads: usize) -> Self { Self { + account_threads, connections, registration_to_fn: HashMap::new(), badge_to_registration: HashMap::new(), @@ -67,7 +66,7 @@ impl BadgeRegistry { } let registration_id = REGISTRATION_ID.fetch_add(1, Ordering::Relaxed); - let (account_send, account_recv) = broadcast::channel(MAX_SIMULTANEOUS_ACCOUNTS); + let (account_send, account_recv) = broadcast::channel(self.account_threads); for badge_id in badge_ids { self.badge_to_registration @@ -77,11 +76,7 @@ impl BadgeRegistry { self.registration_to_fn .insert(registration_id, account_send); - start_checker( - Arc::new(Semaphore::new(MAX_SIMULTANEOUS_WORKERS_PER_BADGE)), - self.connections.clone(), - account_recv, - ); + start_checker(self.connections.clone(), account_recv); } pub async fn queue_account( @@ -126,7 +121,6 @@ impl BadgeRegistry { macro_rules! create_badge_worker { ($query_fn: ident) => { pub fn run( - simultaneous_workers: std::sync::Arc, connections: std::sync::Arc<$crate::connections::Connections>, mut input: tokio::sync::broadcast::Receiver<( near_primitives::types::AccountId, @@ -136,14 +130,9 @@ macro_rules! create_badge_worker { tokio::spawn(async move { while let Ok((account_id, output)) = input.recv().await { let connections = connections.clone(); - let simultaneous_workers = simultaneous_workers.clone(); tokio::spawn(async move { - let permit = simultaneous_workers.acquire().await.unwrap(); - println!("Acquired for {account_id}"); let result = $query_fn(connections, account_id.clone()).await; - println!("Finished {account_id}"); - drop(permit); match result { Ok(result) => { output.send(result).await.unwrap(); // TODO: Log instead of unwrap diff --git a/service/src/local.rs b/service/src/local.rs index 9f6af94..dfe9f32 100644 --- a/service/src/local.rs +++ b/service/src/local.rs @@ -9,10 +9,7 @@ use sqlx::{ types::{Decimal, Uuid}, FromRow, PgPool, }; -use tokio::{ - join, - sync::{broadcast, Semaphore}, -}; +use tokio::{join, sync::broadcast}; /// Minimum amount of time between account updates const ACCOUNT_UPDATE_COOLDOWN_MINUTES: i64 = 60 * 6; @@ -194,21 +191,15 @@ INSERT INTO account(id, balance, score, consecutive_errors) } pub fn start_local_updater( - semaphore: Arc, connections: Arc, mut input: broadcast::Receiver, ) { tokio::spawn(async move { while let Ok(account_id) = input.recv().await { let connections = connections.clone(); - let semaphore = semaphore.clone(); tokio::spawn(async move { - let permit = semaphore.acquire().await.unwrap(); - println!("Acquired for {account_id}"); let result = update_account(&connections, &account_id).await; - println!("Finished {account_id}"); - drop(permit); if let Err(e) = result { println!("Error updating account {account_id}: {e:?}"); diff --git a/service/src/main.rs b/service/src/main.rs index 57e47c3..32fb572 100644 --- a/service/src/main.rs +++ b/service/src/main.rs @@ -10,7 +10,7 @@ use serde::Deserialize; use sqlx::{migrate, postgres::PgPoolOptions}; use tokio::{ join, - sync::{broadcast, Semaphore}, + sync::{broadcast, Mutex, Semaphore}, }; use crate::{ @@ -20,15 +20,27 @@ use crate::{ local::{add_badge_for_account, get_badges_for_account, query_account}, }; -pub const MAX_SIMULTANEOUS_ACCOUNTS: usize = 16; -pub const MAX_SIMULTANEOUS_WORKERS_PER_BADGE: usize = 5; +#[inline] +const fn default_account_threads() -> usize { + 16 +} + +#[inline] +const fn default_pool_connections() -> u32 { + 5 +} #[derive(Deserialize)] struct Configuration { - #[allow(unused)] // env var read by default by sqlx pub database_url: String, pub indexer_url: String, pub rpc_url: String, + #[serde(default = "default_account_threads")] + pub account_threads: usize, + #[serde(default = "default_pool_connections")] + pub local_pool_connections: u32, + #[serde(default = "default_pool_connections")] + pub indexer_pool_connections: u32, } mod badge; @@ -45,12 +57,12 @@ async fn main() -> Result<(), Box> { let config = envy::from_env::().expect("Missing environment variables"); let local_pool = PgPoolOptions::new() - .max_connections(5) + .max_connections(config.local_pool_connections) .connect(&config.database_url) .await?; let indexer_pool = PgPoolOptions::new() - .max_connections(5) + .max_connections(config.indexer_pool_connections) .connect(&config.indexer_url) .await?; @@ -77,27 +89,30 @@ async fn main() -> Result<(), Box> { println!("Checking {} accounts", accounts.len()); - println!("Creating badge registry"); - - let local_semaphore = Arc::new(Semaphore::new(5)); - let (account_send, account_recv) = broadcast::channel(16); - local::start_local_updater(local_semaphore, connections.clone(), account_recv); + let (account_send, account_recv) = broadcast::channel(config.account_threads); + local::start_local_updater(connections.clone(), account_recv); - let mut badge_registry = BadgeRegistry::new(connections.clone()); + let mut badge_registry = BadgeRegistry::new(connections.clone(), config.account_threads); badge_registry.register(&*transfer::BADGE_IDS, transfer::run); let badge_registry = Arc::new(badge_registry); - let simultaneous_accounts = Arc::new(Semaphore::new(MAX_SIMULTANEOUS_ACCOUNTS)); + let simultaneous_accounts = Arc::new(Semaphore::new(config.account_threads)); let mut join_handles = Vec::new(); + let num_completed = Arc::new(Mutex::new(0)); + let total = accounts.len(); + for account in accounts { - let account_id: AccountId = account.parse().unwrap(); + // Acquire semaphore first to reduce Future state overhead let simultaneous_accounts = Arc::clone(&simultaneous_accounts); let permit = simultaneous_accounts.acquire_owned().await; + + let account_id: AccountId = account.parse().unwrap(); let account_send = account_send.clone(); let connections = Arc::clone(&connections); let badge_registry = Arc::clone(&badge_registry); + let num_completed = num_completed.clone(); let join_handle = tokio::spawn(async move { let (account_record, existing_badges) = join!( @@ -113,27 +128,32 @@ async fn main() -> Result<(), Box> { if !is_update_allowed { println!("Disallowing update for {account_id}"); - return; - } - - if let Err(e) = account_send.send(account_id.clone()) { - println!("Error sending to local updater: {e:?}"); - } + } else { + if let Err(e) = account_send.send(account_id.clone()) { + println!("Error sending to local updater: {e:?}"); + } - let awarded_badges = badge_registry - .queue_account( - account_id.clone(), - existing_badges.unwrap_or_else(|_| HashSet::new()), - ) - .await; - - for badge_id in awarded_badges { - match add_badge_for_account(&connections.local_pool, &account_id, &badge_id).await { - Ok(_) => println!("Added badge {badge_id} to {account_id}"), - Err(e) => println!("Failed to add badge {badge_id} to {account_id}: {e:?}"), + let awarded_badges = badge_registry + .queue_account( + account_id.clone(), + existing_badges.unwrap_or_else(|_| HashSet::new()), + ) + .await; + + for badge_id in awarded_badges { + match add_badge_for_account(&connections.local_pool, &account_id, &badge_id) + .await + { + Ok(_) => println!("Added badge {badge_id} to {account_id}"), + Err(e) => println!("Failed to add badge {badge_id} to {account_id}: {e:?}"), + } } } + let mut num_completed = num_completed.lock().await; + *num_completed += 1; + println!("{num_completed} / {total}\t{account_id}"); + drop(permit); }); From 011c597bfc0b97a7b2d09045eaa95064eba262b8 Mon Sep 17 00:00:00 2001 From: Jacob Date: Tue, 24 May 2022 10:55:57 -0500 Subject: [PATCH 21/31] replace rust-analyzer with new identifier in extension recommendations --- .vscode/extensions.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.vscode/extensions.json b/.vscode/extensions.json index cd446f6..3acbc05 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -3,10 +3,10 @@ "qufiwefefwoyn.inline-sql-syntax", "mikestead.dotenv", "esbenp.prettier-vscode", - "matklad.rust-analyzer", "bradlc.vscode-tailwindcss", "netcorext.uuid-generator", "octref.vetur", - "bungcip.better-toml" + "bungcip.better-toml", + "rust-lang.rust-analyzer" ] } From 24299c8ebb48e299abfd1d64ef2b730775ef8825 Mon Sep 17 00:00:00 2001 From: Jacob Date: Tue, 24 May 2022 12:15:16 -0500 Subject: [PATCH 22/31] adds logging --- Cargo.lock | 61 ++++++++++++++++++++++++++++++++++++++ service/Cargo.toml | 2 ++ service/src/badge/mod.rs | 8 +++-- service/src/connections.rs | 2 +- service/src/indexer.rs | 2 ++ service/src/local.rs | 23 +++++++++----- service/src/main.rs | 18 ++++++----- service/src/rpc.rs | 1 + 8 files changed, 100 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2512d89..b6c7fb5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -81,6 +81,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + [[package]] name = "anyhow" version = "1.0.57" @@ -2006,6 +2015,8 @@ dependencies = [ "sqlx", "thiserror", "tokio", + "tracing", + "tracing-subscriber", ] [[package]] @@ -2043,6 +2054,15 @@ dependencies = [ "digest 0.10.3", ] +[[package]] +name = "sharded-slab" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" +dependencies = [ + "lazy_static", +] + [[package]] name = "signal-hook-registry" version = "1.4.0" @@ -2302,6 +2322,15 @@ dependencies = [ "syn", ] +[[package]] +name = "thread_local" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180" +dependencies = [ + "once_cell", +] + [[package]] name = "time" version = "0.1.43" @@ -2452,6 +2481,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f54c8ca710e81886d498c2fd3331b56c93aa248d49de2222ad2742247c60072f" dependencies = [ "lazy_static", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ddad33d2d10b1ed7eb9d1f518a5674713876e97e5bb9b7345a7984fbb4f922" +dependencies = [ + "lazy_static", + "log", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bc28f93baff38037f64e6f43d34cfa1605f27a49c34e8a04c5e78b0babf2596" +dependencies = [ + "ansi_term", + "sharded-slab", + "smallvec", + "thread_local", + "tracing-core", + "tracing-log", ] [[package]] @@ -2532,6 +2587,12 @@ dependencies = [ "getrandom 0.2.6", ] +[[package]] +name = "valuable" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" + [[package]] name = "vcpkg" version = "0.2.15" diff --git a/service/Cargo.toml b/service/Cargo.toml index 6147ee8..67d1ff1 100644 --- a/service/Cargo.toml +++ b/service/Cargo.toml @@ -18,5 +18,7 @@ serde = "1.0.137" sqlx = {version = "0.5", features = ["runtime-tokio-native-tls", "postgres", "decimal", "chrono", "uuid"]} thiserror = "1.0.31" tokio = {version = "1", features = ["full"]} +tracing = "0.1.34" +tracing-subscriber = "0.3.11" [features] diff --git a/service/src/badge/mod.rs b/service/src/badge/mod.rs index 410c282..a41eab3 100644 --- a/service/src/badge/mod.rs +++ b/service/src/badge/mod.rs @@ -28,6 +28,7 @@ pub type BadgeWorker = fn( input: broadcast::Receiver<(AccountId, mpsc::Sender)>, ); +#[derive(Debug)] pub struct BadgeRegistry { account_threads: usize, connections: Arc, @@ -48,9 +49,10 @@ impl BadgeRegistry { } } + #[tracing::instrument(skip(self))] pub fn register<'a, T>(&mut self, badge_ids: T, start_checker: BadgeWorker) where - T: IntoIterator, + T: IntoIterator + std::fmt::Debug, { static REGISTRATION_ID: AtomicUsize = AtomicUsize::new(0); @@ -79,6 +81,7 @@ impl BadgeRegistry { start_checker(self.connections.clone(), account_recv); } + #[tracing::instrument(skip(self))] pub async fn queue_account( &self, account_id: AccountId, @@ -120,6 +123,7 @@ impl BadgeRegistry { #[macro_export] macro_rules! create_badge_worker { ($query_fn: ident) => { + #[tracing::instrument(skip_all)] pub fn run( connections: std::sync::Arc<$crate::connections::Connections>, mut input: tokio::sync::broadcast::Receiver<( @@ -137,7 +141,7 @@ macro_rules! create_badge_worker { Ok(result) => { output.send(result).await.unwrap(); // TODO: Log instead of unwrap } - Err(e) => println!("{e:?}"), + Err(e) => tracing::error!("Error in badge worker result: {e:?}"), } }); } diff --git a/service/src/connections.rs b/service/src/connections.rs index 0b47524..49088b1 100644 --- a/service/src/connections.rs +++ b/service/src/connections.rs @@ -1,7 +1,7 @@ use near_jsonrpc_client::JsonRpcClient; use sqlx::PgPool; -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct Connections { pub local_pool: PgPool, pub indexer_pool: PgPool, diff --git a/service/src/indexer.rs b/service/src/indexer.rs index 00edcca..9119e39 100644 --- a/service/src/indexer.rs +++ b/service/src/indexer.rs @@ -1,6 +1,7 @@ use futures::{StreamExt, TryFutureExt, TryStreamExt}; use sqlx::PgPool; +#[tracing::instrument(skip(indexer_pool))] pub async fn get_recent_actors( indexer_pool: &PgPool, timestamp_nanoseconds: u64, @@ -32,6 +33,7 @@ select distinct receiver_account_id as account_id .await } +#[tracing::instrument(skip(indexer_pool))] pub async fn calculate_account_score( indexer_pool: &PgPool, account_id: &str, diff --git a/service/src/local.rs b/service/src/local.rs index dfe9f32..f0ca40a 100644 --- a/service/src/local.rs +++ b/service/src/local.rs @@ -10,6 +10,7 @@ use sqlx::{ FromRow, PgPool, }; use tokio::{join, sync::broadcast}; +use tracing::{error, info, warn}; /// Minimum amount of time between account updates const ACCOUNT_UPDATE_COOLDOWN_MINUTES: i64 = 60 * 6; @@ -74,6 +75,7 @@ impl From for AccountRecord { } } +#[tracing::instrument(skip(local_pool))] pub async fn query_account( local_pool: &PgPool, account_id: &AccountId, @@ -97,6 +99,7 @@ pub struct AccountBadgeRecordDb { pub badge_id: Uuid, } +#[tracing::instrument(skip(local_pool))] pub async fn add_badge_for_account( local_pool: &PgPool, account_id: &AccountId, @@ -116,6 +119,7 @@ insert into account_badge(account_id, badge_id) .await } +#[tracing::instrument(skip(local_pool))] pub async fn get_badges_for_account( local_pool: &PgPool, account_id: &AccountId, @@ -135,6 +139,7 @@ select badge_id .await } +#[tracing::instrument(skip(connections))] pub async fn update_account( connections: &Connections, account_id: &AccountId, @@ -145,10 +150,13 @@ pub async fn update_account( ); if score.is_err() || balance.is_err() { - println!( - "Error calculating score or balance for {account_id}: {:?}\n{:?}", - &score, &balance - ); + if let Err(ref error) = score { + warn!("Failed to calculate score for {account_id}: {error:?}"); + } + if let Err(ref error) = balance { + warn!("Failed to retrieve balance for {account_id}: {error:?}"); + } + let err_query_result = sqlx::query!( r#"--sql INSERT INTO account(id, consecutive_errors) @@ -162,8 +170,8 @@ INSERT INTO account(id, consecutive_errors) .await; match err_query_result { - Ok(..) => println!("Successfully updated consecutive_errors"), - Err(e) => println!("Failed to update consecutive_errors: {e:?}"), + Ok(..) => info!("Successfully updated consecutive_errors"), + Err(e) => error!("Failed to update consecutive_errors: {e:?}"), }; } @@ -190,6 +198,7 @@ INSERT INTO account(id, balance, score, consecutive_errors) Ok(()) } +#[tracing::instrument(skip_all)] pub fn start_local_updater( connections: Arc, mut input: broadcast::Receiver, @@ -202,7 +211,7 @@ pub fn start_local_updater( let result = update_account(&connections, &account_id).await; if let Err(e) = result { - println!("Error updating account {account_id}: {e:?}"); + warn!("Error updating account {account_id}: {e:?}"); } }); } diff --git a/service/src/main.rs b/service/src/main.rs index 32fb572..a5b3fd0 100644 --- a/service/src/main.rs +++ b/service/src/main.rs @@ -12,6 +12,7 @@ use tokio::{ join, sync::{broadcast, Mutex, Semaphore}, }; +use tracing::{error, info}; use crate::{ badge::{transfer, BadgeRegistry}, @@ -54,6 +55,9 @@ async fn main() -> Result<(), Box> { dotenv().ok(); migrate!(); + let subscriber = tracing_subscriber::fmt().finish(); + tracing::subscriber::set_global_default(subscriber)?; + let config = envy::from_env::().expect("Missing environment variables"); let local_pool = PgPoolOptions::new() @@ -74,7 +78,7 @@ async fn main() -> Result<(), Box> { rpc_client, }); - println!("Requesting accounts"); + info!("Requesting accounts"); let accounts = get_recent_actors( &connections.indexer_pool, @@ -87,7 +91,7 @@ async fn main() -> Result<(), Box> { .await .unwrap(); - println!("Checking {} accounts", accounts.len()); + info!("Checking {} accounts", accounts.len()); let (account_send, account_recv) = broadcast::channel(config.account_threads); local::start_local_updater(connections.clone(), account_recv); @@ -127,10 +131,10 @@ async fn main() -> Result<(), Box> { .unwrap_or(true); if !is_update_allowed { - println!("Disallowing update for {account_id}"); + info!("Disallowing update for {account_id}"); } else { if let Err(e) = account_send.send(account_id.clone()) { - println!("Error sending to local updater: {e:?}"); + error!("Error sending to local updater: {e:?}"); } let awarded_badges = badge_registry @@ -144,15 +148,15 @@ async fn main() -> Result<(), Box> { match add_badge_for_account(&connections.local_pool, &account_id, &badge_id) .await { - Ok(_) => println!("Added badge {badge_id} to {account_id}"), - Err(e) => println!("Failed to add badge {badge_id} to {account_id}: {e:?}"), + Ok(_) => info!("Added badge {badge_id} to {account_id}"), + Err(e) => error!("Failed to add badge {badge_id} to {account_id}: {e:?}"), } } } let mut num_completed = num_completed.lock().await; *num_completed += 1; - println!("{num_completed} / {total}\t{account_id}"); + info!("Completed {num_completed} / {total}\t{account_id}"); drop(permit); }); diff --git a/service/src/rpc.rs b/service/src/rpc.rs index 6284b7c..a59074f 100644 --- a/service/src/rpc.rs +++ b/service/src/rpc.rs @@ -9,6 +9,7 @@ use near_primitives::{ views::QueryRequest, }; +#[tracing::instrument(skip(client))] pub async fn get_account_balance( client: &JsonRpcClient, account_id: &AccountId, From 48c4593b6c0ea4bd4e7ee399f105e8f4d80dbea5 Mon Sep 17 00:00:00 2001 From: Jacob Date: Tue, 24 May 2022 12:24:58 -0500 Subject: [PATCH 23/31] removes some unwrap() calls --- service/src/badge/mod.rs | 13 +++++++++---- service/src/main.rs | 10 ++++++---- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/service/src/badge/mod.rs b/service/src/badge/mod.rs index a41eab3..0ab635d 100644 --- a/service/src/badge/mod.rs +++ b/service/src/badge/mod.rs @@ -10,6 +10,7 @@ use near_primitives::types::AccountId; use sqlx::types::Uuid; use tokio::sync::{broadcast, mpsc}; +use tracing::error; use crate::connections::Connections; @@ -103,9 +104,10 @@ impl BadgeRegistry { for registration_id in registration_ids { let sender = &self.registration_to_fn[registration_id]; - sender - .send((account_id.clone(), result_send.clone())) - .unwrap(); // TODO: Remove unwrap() + let send_result = sender.send((account_id.clone(), result_send.clone())); + if let Err(error) = send_result { + error!("Error sending {account_id} to channel: {error:?}"); + } } drop(result_send); @@ -139,7 +141,10 @@ macro_rules! create_badge_worker { let result = $query_fn(connections, account_id.clone()).await; match result { Ok(result) => { - output.send(result).await.unwrap(); // TODO: Log instead of unwrap + let send_result = output.send(result).await; + if let Err(e) = send_result { + tracing::error!("Error sending badge result: {e:?}"); + } } Err(e) => tracing::error!("Error in badge worker result: {e:?}"), } diff --git a/service/src/main.rs b/service/src/main.rs index a5b3fd0..e9fe220 100644 --- a/service/src/main.rs +++ b/service/src/main.rs @@ -108,11 +108,13 @@ async fn main() -> Result<(), Box> { let total = accounts.len(); for account in accounts { - // Acquire semaphore first to reduce Future state overhead let simultaneous_accounts = Arc::clone(&simultaneous_accounts); let permit = simultaneous_accounts.acquire_owned().await; - let account_id: AccountId = account.parse().unwrap(); + let account_id: AccountId = match account.parse() { + Ok(a) => a, + Err(..) => continue, + }; let account_send = account_send.clone(); let connections = Arc::clone(&connections); let badge_registry = Arc::clone(&badge_registry); @@ -154,11 +156,11 @@ async fn main() -> Result<(), Box> { } } + drop(permit); + let mut num_completed = num_completed.lock().await; *num_completed += 1; info!("Completed {num_completed} / {total}\t{account_id}"); - - drop(permit); }); join_handles.push(join_handle); From b63a381e797dae4e40a374fa0b27e29434274ec4 Mon Sep 17 00:00:00 2001 From: Jacob Date: Tue, 24 May 2022 12:29:53 -0500 Subject: [PATCH 24/31] adds update size config option --- service/src/main.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/service/src/main.rs b/service/src/main.rs index e9fe220..c6154d2 100644 --- a/service/src/main.rs +++ b/service/src/main.rs @@ -31,6 +31,11 @@ const fn default_pool_connections() -> u32 { 5 } +#[inline] +const fn default_update_size() -> i64 { + 10 // minutes +} + #[derive(Deserialize)] struct Configuration { pub database_url: String, @@ -42,6 +47,8 @@ struct Configuration { pub local_pool_connections: u32, #[serde(default = "default_pool_connections")] pub indexer_pool_connections: u32, + #[serde(default = "default_update_size")] + pub update_chunk_size_minutes: i64, } mod badge; @@ -83,7 +90,7 @@ async fn main() -> Result<(), Box> { let accounts = get_recent_actors( &connections.indexer_pool, chrono::Utc::now() - .sub(Duration::minutes(10)) + .sub(Duration::minutes(config.update_chunk_size_minutes)) .timestamp_nanos() .try_into() .unwrap(), From 0e13d69a40600a15eb02eb6871ff222026dff71f Mon Sep 17 00:00:00 2001 From: Jacob Date: Thu, 26 May 2022 02:26:04 -0500 Subject: [PATCH 25/31] clippy suggestions --- service/src/badge/transfer.rs | 6 +++--- service/src/local.rs | 24 ++++++++++++------------ 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/service/src/badge/transfer.rs b/service/src/badge/transfer.rs index 7ae3160..222104d 100644 --- a/service/src/badge/transfer.rs +++ b/service/src/badge/transfer.rs @@ -26,7 +26,7 @@ lazy_static! { #[derive(Error, Debug)] enum TransferBadgeError { #[error("Query failure: {0}")] - QueryFailure(#[from] sqlx::Error), + Query(#[from] sqlx::Error), } async fn perform_query( @@ -66,11 +66,11 @@ select count(*) as result BadgeCheckResult { account_id, - awarded: awarded, + awarded, checked: &BADGE_IDS, } }) - .map_err(|e| TransferBadgeError::from(e)) + .map_err(TransferBadgeError::from) }) .await } diff --git a/service/src/local.rs b/service/src/local.rs index f0ca40a..40fd13f 100644 --- a/service/src/local.rs +++ b/service/src/local.rs @@ -2,6 +2,7 @@ use std::{collections::HashSet, str::FromStr, sync::Arc}; use chrono::{Duration, NaiveDateTime, TimeZone}; use futures::{FutureExt, StreamExt, TryFutureExt, TryStreamExt}; +use lazy_static::lazy_static; use near_jsonrpc_client::{errors::JsonRpcError, methods::query::RpcQueryError}; use near_primitives::types::AccountId; use num_traits::FromPrimitive; @@ -17,6 +18,10 @@ const ACCOUNT_UPDATE_COOLDOWN_MINUTES: i64 = 60 * 6; /// Doubles every failure const ACCOUNT_UPDATE_FAILURE_PENALTY_COEFFICIENT_MINUTES: i64 = 60 * 12; +lazy_static! { + static ref NULL_ACCOUNT: AccountId = AccountId::from_str(&"0".repeat(64)).unwrap(); +} + use crate::{connections::Connections, indexer::calculate_account_score, rpc::get_account_balance}; use thiserror::Error; @@ -24,11 +29,11 @@ use thiserror::Error; #[derive(Error, Debug)] pub enum UpdateAccountError { #[error("Error calculating score: {0}")] - ScoreError(sqlx::Error), + Score(sqlx::Error), #[error("Error retrieving balance: {0}")] - BalanceError(#[from] JsonRpcError), + Balance(#[from] JsonRpcError), #[error("Error running account update query: {0}")] - InsertQueryError(#[from] sqlx::Error), + InsertQuery(#[from] sqlx::Error), } #[derive(FromRow)] @@ -61,12 +66,7 @@ impl AccountRecord { impl From for AccountRecord { fn from(record: AccountRecordDb) -> Self { Self { - id: record.id.parse().unwrap_or( - AccountId::from_str( - "0000000000000000000000000000000000000000000000000000000000000000", - ) - .unwrap(), - ), + id: record.id.parse().unwrap_or_else(|_| NULL_ACCOUNT.to_owned()), balance: record.balance.and_then(|b| u128::try_from(b).ok()), score: record.score.map(|s| s as u32), modified: record.modified.map(|m| chrono::Utc.from_utc_datetime(&m)), @@ -145,8 +145,8 @@ pub async fn update_account( account_id: &AccountId, ) -> Result<(), UpdateAccountError> { let (score, balance) = join!( - calculate_account_score(&connections.indexer_pool, &account_id), - get_account_balance(&connections.rpc_client, &account_id) + calculate_account_score(&connections.indexer_pool, account_id), + get_account_balance(&connections.rpc_client, account_id) ); if score.is_err() || balance.is_err() { @@ -175,7 +175,7 @@ INSERT INTO account(id, consecutive_errors) }; } - let score = score.map_err(|e| UpdateAccountError::ScoreError(e))?; + let score = score.map_err(UpdateAccountError::Score)?; let balance = Decimal::from_u128(balance?); sqlx::query!( From 578888a6a78fa8eb9e1180ff805617c17fb20e9b Mon Sep 17 00:00:00 2001 From: Jacob Date: Thu, 26 May 2022 09:32:23 -0500 Subject: [PATCH 26/31] merge clippy changes --- Cargo.lock | 12 ++++++ service/Cargo.toml | 1 + service/src/indexer.rs | 70 +++++++++++++++++++++++++++++++---- service/src/local.rs | 40 +++++++++++++++----- service/src/main.rs | 84 +++++++++++++++++++++++++++--------------- 5 files changed, 161 insertions(+), 46 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b6c7fb5..6c236b0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -114,6 +114,17 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" +[[package]] +name = "async-recursion" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cda8f4bcc10624c4e85bc66b3f452cca98cfa5ca002dc83a16aad2367641bea" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "async-trait" version = "0.1.53" @@ -2001,6 +2012,7 @@ dependencies = [ name = "service" version = "0.1.0" dependencies = [ + "async-recursion", "async-trait", "chrono", "dotenv", diff --git a/service/Cargo.toml b/service/Cargo.toml index 67d1ff1..398ec6e 100644 --- a/service/Cargo.toml +++ b/service/Cargo.toml @@ -4,6 +4,7 @@ name = "service" version = "0.1.0" [dependencies] +async-recursion = "1.0.0" async-trait = "0.1.53" chrono = "0.4.19" dotenv = "0.15.0" diff --git a/service/src/indexer.rs b/service/src/indexer.rs index 9119e39..fbfa374 100644 --- a/service/src/indexer.rs +++ b/service/src/indexer.rs @@ -1,5 +1,15 @@ +use async_recursion::async_recursion; use futures::{StreamExt, TryFutureExt, TryStreamExt}; +use near_primitives::types::AccountId; use sqlx::PgPool; +use thiserror::Error; +use tokio::join; +use tracing::info; + +const FIRST_BLOCK_TIMESTAMP: i64 = 1595368210762782796; +// const FIRST_BLOCK_HEIGHT: i32 = 9820214; +const ONE_DAY_NANOS: i64 = 1000000000 * 60 * 60 * 24; +const MINIMUM_CHECKABLE_DURATION_NANOS: i64 = ONE_DAY_NANOS * 7; #[tracing::instrument(skip(indexer_pool))] pub async fn get_recent_actors( @@ -33,17 +43,47 @@ select distinct receiver_account_id as account_id .await } +#[derive(Error, Debug)] +pub enum ScoreCalculationError { + #[error("Query error in calculating score: {0:?}")] + SqlError(#[from] sqlx::Error), + #[error("Over-recursion in calculating score")] + OverRecursionError, +} + #[tracing::instrument(skip(indexer_pool))] pub async fn calculate_account_score( indexer_pool: &PgPool, - account_id: &str, -) -> Result { + account_id: &AccountId, + from_timestamp: Option, +) -> Result { + calculate_account_score_rec( + indexer_pool, + account_id, + from_timestamp.unwrap_or(FIRST_BLOCK_TIMESTAMP), + chrono::Utc::now().timestamp_nanos(), + ) + .await +} + +#[async_recursion] +pub async fn calculate_account_score_rec( + indexer_pool: &PgPool, + account_id: &AccountId, + min_timestamp: i64, + max_timestamp: i64, +) -> Result { + info!("calculate_account_score_rec"); + if max_timestamp - min_timestamp < MINIMUM_CHECKABLE_DURATION_NANOS { + return Err(ScoreCalculationError::OverRecursionError); + } + #[derive(sqlx::FromRow)] struct WithResult { pub result: i64, } - sqlx::query_as::<_, WithResult>( + Ok(sqlx::query_as::<_, WithResult>( r#"--sql select coalesce(sum( case @@ -69,14 +109,30 @@ from ( select * from transactions where (transactions.signer_account_id = $1 - or transactions.receiver_account_id = $1) + or transactions.receiver_account_id = $1) + and transactions.block_timestamp >= $2 + and transactions.block_timestamp < $3 ) tx -inner join receipts on tx.converted_into_receipt_id = receipts.receipt_id left outer join transaction_actions on tx.transaction_hash = transaction_actions.transaction_hash "#, ) - .bind(account_id) + .bind(account_id.to_string()) + .bind(min_timestamp) + .bind(max_timestamp) .fetch_one(indexer_pool) .map_ok(|a| a.result as u32) - .await + .or_else(|_| async move { + let midpoint = (max_timestamp + min_timestamp) / 2; + info!("Score split for {account_id}: {min_timestamp} | {max_timestamp}"); + let (first, second) = join!( + calculate_account_score_rec(indexer_pool, account_id, min_timestamp, midpoint), + calculate_account_score_rec(indexer_pool, account_id, midpoint, max_timestamp), + ); + + match (first, second) { + (Ok(a), Ok(b)) => Ok(a + b), + (Err(e), _) | (_, Err(e)) => Err(e), + } + }) + .await?) } diff --git a/service/src/local.rs b/service/src/local.rs index 40fd13f..02814fb 100644 --- a/service/src/local.rs +++ b/service/src/local.rs @@ -10,7 +10,10 @@ use sqlx::{ types::{Decimal, Uuid}, FromRow, PgPool, }; -use tokio::{join, sync::broadcast}; +use tokio::{ + join, + sync::{broadcast, mpsc, oneshot}, +}; use tracing::{error, info, warn}; /// Minimum amount of time between account updates @@ -22,14 +25,18 @@ lazy_static! { static ref NULL_ACCOUNT: AccountId = AccountId::from_str(&"0".repeat(64)).unwrap(); } -use crate::{connections::Connections, indexer::calculate_account_score, rpc::get_account_balance}; +use crate::{ + connections::Connections, + indexer::{calculate_account_score, ScoreCalculationError}, + rpc::get_account_balance, +}; use thiserror::Error; #[derive(Error, Debug)] pub enum UpdateAccountError { #[error("Error calculating score: {0}")] - Score(sqlx::Error), + Score(#[from] ScoreCalculationError), #[error("Error retrieving balance: {0}")] Balance(#[from] JsonRpcError), #[error("Error running account update query: {0}")] @@ -66,7 +73,10 @@ impl AccountRecord { impl From for AccountRecord { fn from(record: AccountRecordDb) -> Self { Self { - id: record.id.parse().unwrap_or_else(|_| NULL_ACCOUNT.to_owned()), + id: record + .id + .parse() + .unwrap_or_else(|_| NULL_ACCOUNT.to_owned()), balance: record.balance.and_then(|b| u128::try_from(b).ok()), score: record.score.map(|s| s as u32), modified: record.modified.map(|m| chrono::Utc.from_utc_datetime(&m)), @@ -143,9 +153,14 @@ select badge_id pub async fn update_account( connections: &Connections, account_id: &AccountId, + last_successful_update_nanos: Option, ) -> Result<(), UpdateAccountError> { let (score, balance) = join!( - calculate_account_score(&connections.indexer_pool, account_id), + calculate_account_score( + &connections.indexer_pool, + account_id, + last_successful_update_nanos + ), get_account_balance(&connections.rpc_client, account_id) ); @@ -201,17 +216,22 @@ INSERT INTO account(id, balance, score, consecutive_errors) #[tracing::instrument(skip_all)] pub fn start_local_updater( connections: Arc, - mut input: broadcast::Receiver, + mut input: mpsc::Receiver<( + AccountId, + Option, + oneshot::Sender>, + )>, ) { tokio::spawn(async move { - while let Ok(account_id) = input.recv().await { + while let Some((account_id, last_successful_update_nanos, output)) = input.recv().await { let connections = connections.clone(); tokio::spawn(async move { - let result = update_account(&connections, &account_id).await; + let result = + update_account(&connections, &account_id, last_successful_update_nanos).await; - if let Err(e) = result { - warn!("Error updating account {account_id}: {e:?}"); + if let Err(e) = output.send(result) { + error!("Error sending output oneshot channel"); } }); } diff --git a/service/src/main.rs b/service/src/main.rs index c6154d2..54d4337 100644 --- a/service/src/main.rs +++ b/service/src/main.rs @@ -10,9 +10,9 @@ use serde::Deserialize; use sqlx::{migrate, postgres::PgPoolOptions}; use tokio::{ join, - sync::{broadcast, Mutex, Semaphore}, + sync::{broadcast, mpsc, oneshot, Mutex, Semaphore}, }; -use tracing::{error, info}; +use tracing::{error, info, warn}; use crate::{ badge::{transfer, BadgeRegistry}, @@ -87,20 +87,22 @@ async fn main() -> Result<(), Box> { info!("Requesting accounts"); - let accounts = get_recent_actors( - &connections.indexer_pool, - chrono::Utc::now() - .sub(Duration::minutes(config.update_chunk_size_minutes)) - .timestamp_nanos() - .try_into() - .unwrap(), - ) - .await - .unwrap(); + let accounts = vec!["x.paras.near"]; + + // let accounts = get_recent_actors( + // &connections.indexer_pool, + // chrono::Utc::now() + // .sub(Duration::minutes(config.update_chunk_size_minutes)) + // .timestamp_nanos() + // .try_into() + // .unwrap(), + // ) + // .await + // .unwrap(); info!("Checking {} accounts", accounts.len()); - let (account_send, account_recv) = broadcast::channel(config.account_threads); + let (account_send, account_recv) = mpsc::channel(config.account_threads); local::start_local_updater(connections.clone(), account_recv); let mut badge_registry = BadgeRegistry::new(connections.clone(), config.account_threads); @@ -134,33 +136,57 @@ async fn main() -> Result<(), Box> { ); let is_update_allowed = account_record + .as_ref() .ok() .and_then(|r| r.next_update_allowed_at()) .map(|cutoff| chrono::Utc::now() >= cutoff) .unwrap_or(true); + let is_update_allowed = true; + if !is_update_allowed { info!("Disallowing update for {account_id}"); } else { - if let Err(e) = account_send.send(account_id.clone()) { - error!("Error sending to local updater: {e:?}"); - } + let (send, recv) = oneshot::channel(); - let awarded_badges = badge_registry - .queue_account( + if let Err(e) = account_send + .send(( account_id.clone(), - existing_badges.unwrap_or_else(|_| HashSet::new()), - ) - .await; - - for badge_id in awarded_badges { - match add_badge_for_account(&connections.local_pool, &account_id, &badge_id) - .await - { - Ok(_) => info!("Added badge {badge_id} to {account_id}"), - Err(e) => error!("Failed to add badge {badge_id} to {account_id}: {e:?}"), - } + account_record.as_ref().ok().and_then(|r| { + if r.consecutive_errors == 0 { + r.modified.map(|t| t.timestamp_nanos()) + } else { + None + } + }), + send, + )) + .await + { + error!("Error sending to local updater: {e:?}"); } + + match recv.await { + Ok(Err(e)) => warn!("Failed to update account: {e:?}"), + Err(e) => error!("Error receiving update result from account updater: {e:?}"), + _ => {} // Success + }; + + // let awarded_badges = badge_registry + // .queue_account( + // account_id.clone(), + // existing_badges.unwrap_or_else(|_| HashSet::new()), + // ) + // .await; + + // for badge_id in awarded_badges { + // match add_badge_for_account(&connections.local_pool, &account_id, &badge_id) + // .await + // { + // Ok(_) => info!("Added badge {badge_id} to {account_id}"), + // Err(e) => error!("Failed to add badge {badge_id} to {account_id}: {e:?}"), + // } + // } } drop(permit); From 4d5311f92de919c373a17e2b23de1f5cd7f4008f Mon Sep 17 00:00:00 2001 From: Jacob Date: Thu, 2 Jun 2022 13:54:41 -0500 Subject: [PATCH 27/31] revamped scoring --- Cargo.lock | 1 + service/Cargo.toml | 1 + service/src/badge/mod.rs | 6 +++- service/src/indexer.rs | 65 +++++++++++++++--------------------- service/src/indexer/score.rs | 39 ++++++++++++++++++++++ service/src/main.rs | 55 +++++++++++++++--------------- 6 files changed, 99 insertions(+), 68 deletions(-) create mode 100644 service/src/indexer/score.rs diff --git a/Cargo.lock b/Cargo.lock index 6c236b0..47ed162 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2025,6 +2025,7 @@ dependencies = [ "num-traits", "serde", "sqlx", + "tap", "thiserror", "tokio", "tracing", diff --git a/service/Cargo.toml b/service/Cargo.toml index 398ec6e..398a2e5 100644 --- a/service/Cargo.toml +++ b/service/Cargo.toml @@ -17,6 +17,7 @@ near-primitives = "0.12.0" num-traits = "0.2.15" serde = "1.0.137" sqlx = {version = "0.5", features = ["runtime-tokio-native-tls", "postgres", "decimal", "chrono", "uuid"]} +tap = "1.0.1" thiserror = "1.0.31" tokio = {version = "1", features = ["full"]} tracing = "0.1.34" diff --git a/service/src/badge/mod.rs b/service/src/badge/mod.rs index 0ab635d..b62a767 100644 --- a/service/src/badge/mod.rs +++ b/service/src/badge/mod.rs @@ -83,7 +83,7 @@ impl BadgeRegistry { } #[tracing::instrument(skip(self))] - pub async fn queue_account( + pub async fn award_badges( &self, account_id: AccountId, ignore_badge_ids: HashSet, @@ -100,6 +100,10 @@ impl BadgeRegistry { }) .collect::>(); + if registration_ids.is_empty() { + return HashSet::new(); + } + let (result_send, mut result_recv) = mpsc::channel(registration_ids.len()); for registration_id in registration_ids { diff --git a/service/src/indexer.rs b/service/src/indexer.rs index fbfa374..3ffbf96 100644 --- a/service/src/indexer.rs +++ b/service/src/indexer.rs @@ -1,15 +1,21 @@ use async_recursion::async_recursion; +use chrono::TimeZone; use futures::{StreamExt, TryFutureExt, TryStreamExt}; use near_primitives::types::AccountId; use sqlx::PgPool; +use tap::Tap; use thiserror::Error; use tokio::join; -use tracing::info; +use tracing::{debug, error, info}; + +use crate::indexer::score::ActionCountRow; + +mod score; const FIRST_BLOCK_TIMESTAMP: i64 = 1595368210762782796; // const FIRST_BLOCK_HEIGHT: i32 = 9820214; const ONE_DAY_NANOS: i64 = 1000000000 * 60 * 60 * 24; -const MINIMUM_CHECKABLE_DURATION_NANOS: i64 = ONE_DAY_NANOS * 7; +const MINIMUM_CHECKABLE_DURATION_NANOS: i64 = ONE_DAY_NANOS / 2; #[tracing::instrument(skip(indexer_pool))] pub async fn get_recent_actors( @@ -66,6 +72,7 @@ pub async fn calculate_account_score( .await } +#[tracing::instrument(skip(indexer_pool))] #[async_recursion] pub async fn calculate_account_score_rec( indexer_pool: &PgPool, @@ -73,57 +80,36 @@ pub async fn calculate_account_score_rec( min_timestamp: i64, max_timestamp: i64, ) -> Result { - info!("calculate_account_score_rec"); if max_timestamp - min_timestamp < MINIMUM_CHECKABLE_DURATION_NANOS { + let min = chrono::Utc.timestamp_nanos(min_timestamp); + let max = chrono::Utc.timestamp_nanos(max_timestamp); + debug!("Over-recursion: {min} <> {max}"); return Err(ScoreCalculationError::OverRecursionError); } - #[derive(sqlx::FromRow)] - struct WithResult { - pub result: i64, - } - - Ok(sqlx::query_as::<_, WithResult>( + Ok(sqlx::query_as::<_, ActionCountRow>( r#"--sql -select coalesce(sum( - case - when action_kind = 'TRANSFER' - and signer_account_id = $1 - then 10 - when action_kind = 'TRANSFER' - and tx.receiver_account_id = $1 - then 2 - when action_kind = 'CREATE_ACCOUNT' - and signer_account_id = $1 - then 50 - when action_kind = 'FUNCTION_CALL' - and signer_account_id = $1 - then 10 - when action_kind = 'DEPLOY_CONTRACT' - and signer_account_id = $1 - then 100 - else 0 - end -), 0) as result -from ( - select * - from transactions - where (transactions.signer_account_id = $1 - or transactions.receiver_account_id = $1) +select action_kind, + count(*) as transaction_count +from transactions +left outer join transaction_actions on transactions.transaction_hash = transaction_actions.transaction_hash +where transactions.signer_account_id = $1 and transactions.block_timestamp >= $2 and transactions.block_timestamp < $3 -) tx -left outer join transaction_actions on tx.transaction_hash = transaction_actions.transaction_hash +group by action_kind "#, ) .bind(account_id.to_string()) .bind(min_timestamp) .bind(max_timestamp) - .fetch_one(indexer_pool) - .map_ok(|a| a.result as u32) - .or_else(|_| async move { + .fetch_all(indexer_pool) + .map_ok(|v| v.iter().map(|r| r.score_value()).sum()) + .or_else(|e| async move { + error!("Splitting because of error: {e:?}"); let midpoint = (max_timestamp + min_timestamp) / 2; info!("Score split for {account_id}: {min_timestamp} | {max_timestamp}"); + // let first = calculate_account_score_rec(indexer_pool, account_id, min_timestamp, midpoint).await; + // let second = calculate_account_score_rec(indexer_pool, account_id, midpoint, max_timestamp).await; let (first, second) = join!( calculate_account_score_rec(indexer_pool, account_id, min_timestamp, midpoint), calculate_account_score_rec(indexer_pool, account_id, midpoint, max_timestamp), @@ -135,4 +121,5 @@ left outer join transaction_actions on tx.transaction_hash = transaction_actions } }) .await?) + .tap(|r| info!("Score result: {r:?}")) } diff --git a/service/src/indexer/score.rs b/service/src/indexer/score.rs new file mode 100644 index 0000000..9da566d --- /dev/null +++ b/service/src/indexer/score.rs @@ -0,0 +1,39 @@ +use std::str::FromStr; + +use sqlx::FromRow; +use thiserror::Error; + +#[derive(Debug, sqlx::Type)] +#[sqlx(type_name = "action_kind")] +#[sqlx(rename_all = "SCREAMING_SNAKE_CASE")] +pub enum ActionKind { + CreateAccount, + DeleteAccount, + AddKey, + DeleteKey, + Transfer, + Stake, + DeployContract, + FunctionCall, +} + +#[derive(FromRow)] +pub struct ActionCountRow { + pub action_kind: ActionKind, + pub transaction_count: i64, +} + +impl ActionCountRow { + pub fn score_value(&self) -> u32 { + self.transaction_count as u32 + * match &self.action_kind { + ActionKind::CreateAccount => 50, + ActionKind::AddKey => 1, + ActionKind::DeleteKey => 25, + ActionKind::Transfer => 10, + ActionKind::DeployContract => 100, + ActionKind::FunctionCall => 10, + _ => 0, + } + } +} diff --git a/service/src/main.rs b/service/src/main.rs index 54d4337..b608696 100644 --- a/service/src/main.rs +++ b/service/src/main.rs @@ -74,6 +74,7 @@ async fn main() -> Result<(), Box> { let indexer_pool = PgPoolOptions::new() .max_connections(config.indexer_pool_connections) + .connect_timeout(std::time::Duration::from_secs(120)) .connect(&config.indexer_url) .await?; @@ -87,18 +88,18 @@ async fn main() -> Result<(), Box> { info!("Requesting accounts"); - let accounts = vec!["x.paras.near"]; + // let accounts = vec!["x.paras.near"]; - // let accounts = get_recent_actors( - // &connections.indexer_pool, - // chrono::Utc::now() - // .sub(Duration::minutes(config.update_chunk_size_minutes)) - // .timestamp_nanos() - // .try_into() - // .unwrap(), - // ) - // .await - // .unwrap(); + let accounts = get_recent_actors( + &connections.indexer_pool, + chrono::Utc::now() + .sub(Duration::minutes(config.update_chunk_size_minutes)) + .timestamp_nanos() + .try_into() + .unwrap(), + ) + .await + .unwrap(); info!("Checking {} accounts", accounts.len()); @@ -142,8 +143,6 @@ async fn main() -> Result<(), Box> { .map(|cutoff| chrono::Utc::now() >= cutoff) .unwrap_or(true); - let is_update_allowed = true; - if !is_update_allowed { info!("Disallowing update for {account_id}"); } else { @@ -172,21 +171,21 @@ async fn main() -> Result<(), Box> { _ => {} // Success }; - // let awarded_badges = badge_registry - // .queue_account( - // account_id.clone(), - // existing_badges.unwrap_or_else(|_| HashSet::new()), - // ) - // .await; - - // for badge_id in awarded_badges { - // match add_badge_for_account(&connections.local_pool, &account_id, &badge_id) - // .await - // { - // Ok(_) => info!("Added badge {badge_id} to {account_id}"), - // Err(e) => error!("Failed to add badge {badge_id} to {account_id}: {e:?}"), - // } - // } + let awarded_badges = badge_registry + .award_badges( + account_id.clone(), + existing_badges.unwrap_or_else(|_| HashSet::new()), + ) + .await; + + for badge_id in awarded_badges { + match add_badge_for_account(&connections.local_pool, &account_id, &badge_id) + .await + { + Ok(_) => info!("Added badge {badge_id} to {account_id}"), + Err(e) => error!("Failed to add badge {badge_id} to {account_id}: {e:?}"), + } + } } drop(permit); From b2c333898da7250865287a37af9b0b124ca6e027 Mon Sep 17 00:00:00 2001 From: Jacob Date: Thu, 2 Jun 2022 13:57:42 -0500 Subject: [PATCH 28/31] fmt + fix --- service/src/indexer/score.rs | 3 --- service/src/local.rs | 4 ++-- service/src/main.rs | 2 +- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/service/src/indexer/score.rs b/service/src/indexer/score.rs index 9da566d..c1052b7 100644 --- a/service/src/indexer/score.rs +++ b/service/src/indexer/score.rs @@ -1,7 +1,4 @@ -use std::str::FromStr; - use sqlx::FromRow; -use thiserror::Error; #[derive(Debug, sqlx::Type)] #[sqlx(type_name = "action_kind")] diff --git a/service/src/local.rs b/service/src/local.rs index 02814fb..dd1f0ee 100644 --- a/service/src/local.rs +++ b/service/src/local.rs @@ -12,7 +12,7 @@ use sqlx::{ }; use tokio::{ join, - sync::{broadcast, mpsc, oneshot}, + sync::{mpsc, oneshot}, }; use tracing::{error, info, warn}; @@ -230,7 +230,7 @@ pub fn start_local_updater( let result = update_account(&connections, &account_id, last_successful_update_nanos).await; - if let Err(e) = output.send(result) { + if let Err(_e) = output.send(result) { error!("Error sending output oneshot channel"); } }); diff --git a/service/src/main.rs b/service/src/main.rs index b608696..79060d4 100644 --- a/service/src/main.rs +++ b/service/src/main.rs @@ -10,7 +10,7 @@ use serde::Deserialize; use sqlx::{migrate, postgres::PgPoolOptions}; use tokio::{ join, - sync::{broadcast, mpsc, oneshot, Mutex, Semaphore}, + sync::{mpsc, oneshot, Mutex, Semaphore}, }; use tracing::{error, info, warn}; From 64a4b08b869c33c4e80c4650a3e3556300761a90 Mon Sep 17 00:00:00 2001 From: Jacob Date: Fri, 3 Jun 2022 12:01:43 -0500 Subject: [PATCH 29/31] rust api server initial commit --- Cargo.lock | 361 ++++++++++++++++++++++++++++++++++++++++++++- Cargo.toml | 3 +- server/Cargo.toml | 16 ++ server/src/main.rs | 33 +++++ 4 files changed, 411 insertions(+), 2 deletions(-) create mode 100644 server/Cargo.toml create mode 100644 server/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 47ed162..4a660ee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -25,6 +25,84 @@ dependencies = [ "tokio-util 0.6.10", ] +[[package]] +name = "actix-codec" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57a7559404a7f3573127aab53c08ce37a6c6a315c374a31070f3c91cd1b4a7fe" +dependencies = [ + "bitflags", + "bytes", + "futures-core", + "futures-sink", + "log", + "memchr", + "pin-project-lite", + "tokio", + "tokio-util 0.7.2", +] + +[[package]] +name = "actix-http" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5885cb81a0d4d0d322864bea1bb6c2a8144626b4fdc625d4c51eba197e7797a" +dependencies = [ + "actix-codec", + "actix-rt", + "actix-service", + "actix-utils", + "ahash", + "base64 0.13.0", + "bitflags", + "brotli", + "bytes", + "bytestring", + "derive_more", + "encoding_rs", + "flate2", + "futures-core", + "h2", + "http", + "httparse", + "httpdate", + "itoa", + "language-tags", + "local-channel", + "log", + "mime", + "percent-encoding", + "pin-project-lite", + "rand 0.8.5", + "sha-1", + "smallvec", + "zstd", +] + +[[package]] +name = "actix-macros" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "465a6172cf69b960917811022d8f29bc0b7fa1398bc4f78b3c466673db1213b6" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "actix-router" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb60846b52c118f2f04a56cc90880a274271c489b2498623d58176f8ca21fa80" +dependencies = [ + "bytestring", + "firestorm", + "http", + "log", + "regex", + "serde", +] + [[package]] name = "actix-rt" version = "2.7.0" @@ -35,6 +113,97 @@ dependencies = [ "tokio", ] +[[package]] +name = "actix-server" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0da34f8e659ea1b077bb4637948b815cd3768ad5a188fdcd74ff4d84240cd824" +dependencies = [ + "actix-rt", + "actix-service", + "actix-utils", + "futures-core", + "futures-util", + "mio", + "num_cpus", + "socket2", + "tokio", + "tracing", +] + +[[package]] +name = "actix-service" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b894941f818cfdc7ccc4b9e60fa7e53b5042a2e8567270f9147d5591893373a" +dependencies = [ + "futures-core", + "paste", + "pin-project-lite", +] + +[[package]] +name = "actix-utils" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e491cbaac2e7fc788dfff99ff48ef317e23b3cf63dbaf7aaab6418f40f92aa94" +dependencies = [ + "local-waker", + "pin-project-lite", +] + +[[package]] +name = "actix-web" +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4e5ebffd51d50df56a3ae0de0e59487340ca456f05dd0b90c0a7a6dd6a74d31" +dependencies = [ + "actix-codec", + "actix-http", + "actix-macros", + "actix-router", + "actix-rt", + "actix-server", + "actix-service", + "actix-utils", + "actix-web-codegen", + "ahash", + "bytes", + "bytestring", + "cfg-if 1.0.0", + "cookie", + "derive_more", + "encoding_rs", + "futures-core", + "futures-util", + "itoa", + "language-tags", + "log", + "mime", + "once_cell", + "pin-project-lite", + "regex", + "serde", + "serde_json", + "serde_urlencoded", + "smallvec", + "socket2", + "time 0.3.9", + "url", +] + +[[package]] +name = "actix-web-codegen" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7525bedf54704abb1d469e88d7e7e9226df73778798a69cea5022d53b2ae91bc" +dependencies = [ + "actix-router", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "actix_derive" version = "0.6.0-beta.1" @@ -81,6 +250,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloc-no-stdlib" +version = "2.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35ef4730490ad1c4eae5c4325b2a95f521d023e5c885853ff7aca0a6a1631db3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "697ed7edc0f1711de49ce108c541623a0af97c6c60b2f6e2b65229847ac843c2" +dependencies = [ + "alloc-no-stdlib", +] + [[package]] name = "ansi_term" version = "0.12.1" @@ -270,6 +454,27 @@ dependencies = [ "syn", ] +[[package]] +name = "brotli" +version = "3.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1a0b1dbcc8ae29329621f8d4f0d835787c1c38bb1401979b49d13b0b305ff68" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ad2d4653bf5ca36ae797b1f4bb4dbddb60ce49ca4aed8a2ce4829f60425b80" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bs58" version = "0.4.0" @@ -306,6 +511,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c58ec36aac5066d5ca17df51b3e70279f5670a72102f5752cb7e7c856adfc70" +[[package]] +name = "bytestring" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90706ba19e97b90786e19dc0d5e2abd80008d99d4c0c5d1ad0b5e72cec7c494d" +dependencies = [ + "bytes", +] + [[package]] name = "c2-chacha" version = "0.3.3" @@ -321,6 +535,9 @@ name = "cc" version = "1.0.73" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" +dependencies = [ + "jobserver", +] [[package]] name = "cfg-if" @@ -344,7 +561,7 @@ dependencies = [ "num-integer", "num-traits", "serde", - "time", + "time 0.1.43", "winapi", ] @@ -363,6 +580,17 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" +[[package]] +name = "cookie" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94d4706de1b0fa5b132270cddffa8585166037822e260a944fe161acd137ca05" +dependencies = [ + "percent-encoding", + "time 0.3.9", + "version_check", +] + [[package]] name = "core-foundation" version = "0.9.3" @@ -403,6 +631,15 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccaeedb56da03b09f598226e25e80088cb4cd25f316e6e4df7d695f0feeb1403" +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if 1.0.0", +] + [[package]] name = "crossbeam-channel" version = "0.5.4" @@ -621,6 +858,12 @@ dependencies = [ "instant", ] +[[package]] +name = "firestorm" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c5f6c2c942da57e2aaaa84b8a521489486f14e75e7fa91dab70aba913975f98" + [[package]] name = "fixed-hash" version = "0.7.0" @@ -633,6 +876,16 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "flate2" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1038,6 +1291,15 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d" +[[package]] +name = "jobserver" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af25a77299a7f711a01975c35a6a424eb6862092cc2d6c72c4ed6cbc56dfc1fa" +dependencies = [ + "libc", +] + [[package]] name = "js-sys" version = "0.3.57" @@ -1047,6 +1309,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "language-tags" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4345964bb142484797b161f473a503a434de77149dd8c7427788c6e13379388" + [[package]] name = "lazy_static" version = "1.4.0" @@ -1059,6 +1327,24 @@ version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5916d2ae698f6de9bfb891ad7a8d65c09d232dc58cc4ac433c7da3b2fd84bc2b" +[[package]] +name = "local-channel" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f303ec0e94c6c54447f84f3b0ef7af769858a9c4ef56ef2a986d3dcd4c3fc9c" +dependencies = [ + "futures-core", + "futures-sink", + "futures-util", + "local-waker", +] + +[[package]] +name = "local-waker" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e34f76eb3611940e0e7d53a9aaa4e6a3151f69541a282fd0dad5571420c53ff1" + [[package]] name = "lock_api" version = "0.4.7" @@ -1457,6 +1743,15 @@ dependencies = [ "libc", ] +[[package]] +name = "num_threads" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" +dependencies = [ + "libc", +] + [[package]] name = "object" version = "0.28.4" @@ -2008,6 +2303,22 @@ dependencies = [ "serde", ] +[[package]] +name = "server" +version = "0.1.0" +dependencies = [ + "actix-web", + "chrono", + "dotenv", + "envy", + "futures", + "serde", + "sqlx", + "tokio", + "tracing", + "tracing-subscriber", +] + [[package]] name = "service" version = "0.1.0" @@ -2354,6 +2665,24 @@ dependencies = [ "winapi", ] +[[package]] +name = "time" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2702e08a7a860f005826c6815dcac101b19b5eb330c27fe4a5928fec1d20ddd" +dependencies = [ + "itoa", + "libc", + "num_threads", + "time-macros", +] + +[[package]] +name = "time-macros" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42657b1a6f4d817cda8e7a0ace261fe0cc946cf3a80314390b22cc61ae080792" + [[package]] name = "tinyvec" version = "1.6.0" @@ -2471,6 +2800,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d0ecdcb44a79f0fe9844f0c4f33a342cbcbb5117de8001e6ba0dc2351327d09" dependencies = [ "cfg-if 1.0.0", + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -2832,3 +3162,32 @@ dependencies = [ "syn", "synstructure", ] + +[[package]] +name = "zstd" +version = "0.10.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f4a6bd64f22b5e3e94b4e238669ff9f10815c27a5180108b849d24174a83847" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "4.1.6+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94b61c51bb270702d6167b8ce67340d2754b088d0c091b06e593aa772c3ee9bb" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "1.6.3+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc49afa5c8d634e75761feda8c592051e7eeb4683ba827211eb0d731d3402ea8" +dependencies = [ + "cc", + "libc", +] diff --git a/Cargo.toml b/Cargo.toml index d5af5cb..b767cea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,4 +1,5 @@ [workspace] members = [ - "service" + "service", + "server" ] diff --git a/server/Cargo.toml b/server/Cargo.toml new file mode 100644 index 0000000..378b315 --- /dev/null +++ b/server/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "server" +version = "0.1.0" +edition = "2021" + +[dependencies] +actix-web = "4.0.1" +chrono = "0.4.19" +dotenv = "0.15.0" +envy = "0.4.2" +futures = "0.3.21" +serde = "1.0.137" +sqlx = {version = "0.5", features = ["runtime-tokio-native-tls", "postgres", "decimal", "chrono", "uuid"]} +tokio = {version = "1", features = ["full"]} +tracing = "0.1.34" +tracing-subscriber = "0.3.11" diff --git a/server/src/main.rs b/server/src/main.rs new file mode 100644 index 0000000..15b116d --- /dev/null +++ b/server/src/main.rs @@ -0,0 +1,33 @@ +use actix_web::{get, App, HttpServer, Responder}; +use dotenv::dotenv; +use serde::Deserialize; + +#[inline] +fn default_port() -> u16 { + 3000 +} + +#[derive(Deserialize)] +struct Configuration { + #[serde(default = "default_port")] + pub port: u16, + pub database_url: String, + pub indexer_url: String, +} + +#[get("/status")] +async fn status() -> impl Responder { + "ok" +} + +#[tokio::main] +async fn main() -> std::io::Result<()> { + dotenv().ok(); + + let config = envy::from_env::().unwrap(); + + HttpServer::new(|| App::new().service(status)) + .bind(("127.0.0.1", config.port))? + .run() + .await +} From 2b7ce8b37b8a87da1c735dc20b0f22622a7c6e80 Mon Sep 17 00:00:00 2001 From: Jacob Date: Tue, 7 Jun 2022 11:55:47 -0500 Subject: [PATCH 30/31] server --- Cargo.lock | 383 +++++++++++++++++- server/Cargo.toml | 9 +- server/src/indexer/access_keys.rs | 31 ++ server/src/indexer/account_changes.rs | 40 ++ server/src/indexer/accounts.rs | 29 ++ server/src/indexer/action_receipt_actions.rs | 31 ++ .../src/indexer/action_receipt_input_data.rs | 23 ++ .../src/indexer/action_receipt_output_data.rs | 25 ++ server/src/indexer/action_receipts.rs | 27 ++ .../indexer/aggregated_circulating_supply.rs | 33 ++ server/src/indexer/blocks.rs | 33 ++ server/src/indexer/chunks.rs | 33 ++ server/src/indexer/data_receipts.rs | 24 ++ .../src/indexer/execution_outcome_receipts.rs | 25 ++ server/src/indexer/execution_outcomes.rs | 36 ++ server/src/indexer/mod.rs | 21 + server/src/indexer/prelude.rs | 18 + server/src/indexer/receipts.rs | 36 ++ server/src/indexer/sea_orm_active_enums.rs | 104 +++++ server/src/indexer/transaction_actions.rs | 26 ++ server/src/indexer/transactions.rs | 46 +++ server/src/local/account.rs | 26 ++ server/src/local/account_badge.rs | 24 ++ server/src/local/mod.rs | 6 + server/src/local/prelude.rs | 4 + server/src/main.rs | 198 ++++++++- ...0603200449_modified-with-timezone.down.sql | 2 + ...220603200449_modified-with-timezone.up.sql | 2 + service/src/indexer.rs | 3 - service/src/local.rs | 6 +- 30 files changed, 1281 insertions(+), 23 deletions(-) create mode 100644 server/src/indexer/access_keys.rs create mode 100644 server/src/indexer/account_changes.rs create mode 100644 server/src/indexer/accounts.rs create mode 100644 server/src/indexer/action_receipt_actions.rs create mode 100644 server/src/indexer/action_receipt_input_data.rs create mode 100644 server/src/indexer/action_receipt_output_data.rs create mode 100644 server/src/indexer/action_receipts.rs create mode 100644 server/src/indexer/aggregated_circulating_supply.rs create mode 100644 server/src/indexer/blocks.rs create mode 100644 server/src/indexer/chunks.rs create mode 100644 server/src/indexer/data_receipts.rs create mode 100644 server/src/indexer/execution_outcome_receipts.rs create mode 100644 server/src/indexer/execution_outcomes.rs create mode 100644 server/src/indexer/mod.rs create mode 100644 server/src/indexer/prelude.rs create mode 100644 server/src/indexer/receipts.rs create mode 100644 server/src/indexer/sea_orm_active_enums.rs create mode 100644 server/src/indexer/transaction_actions.rs create mode 100644 server/src/indexer/transactions.rs create mode 100644 server/src/local/account.rs create mode 100644 server/src/local/account_badge.rs create mode 100644 server/src/local/mod.rs create mode 100644 server/src/local/prelude.rs create mode 100644 service/migrations/20220603200449_modified-with-timezone.down.sql create mode 100644 service/migrations/20220603200449_modified-with-timezone.up.sql diff --git a/Cargo.lock b/Cargo.lock index 4a660ee..9af7e63 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "Inflector" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" + [[package]] name = "actix" version = "0.11.0-beta.2" @@ -250,6 +256,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "aliasable" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" + [[package]] name = "alloc-no-stdlib" version = "2.0.3" @@ -309,6 +321,27 @@ dependencies = [ "syn", ] +[[package]] +name = "async-stream" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dad5c83079eae9969be7fadefe640a1c566901f05ff91ab221de4b6f68d9507e" +dependencies = [ + "async-stream-impl", + "futures-core", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10f203db73a71dfa2fb6dd22763990fa26f3d2625a6da2da900d23b87d26be27" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "async-trait" version = "0.1.53" @@ -350,6 +383,25 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "bae" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33b8de67cc41132507eeece2584804efcb15f85ba516e34c944b7667f480397a" +dependencies = [ + "heck 0.3.3", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "base-x" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc19a4937b4fbd3fe3379793130e42060d10627a360f2127802b10b87e7baf74" + [[package]] name = "base64" version = "0.11.0" @@ -574,6 +626,12 @@ dependencies = [ "generic-array", ] +[[package]] +name = "const_fn" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbdcdcb6d86f71c5e97409ad45898af11cbc995b4ee8112d59095a28d376c935" + [[package]] name = "convert_case" version = "0.4.0" @@ -718,7 +776,7 @@ dependencies = [ "convert_case", "proc-macro2", "quote", - "rustc_version", + "rustc_version 0.4.0", "syn", ] @@ -762,6 +820,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "discard" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" + [[package]] name = "dotenv" version = "0.15.0" @@ -1818,6 +1882,30 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "ouroboros" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f31a3b678685b150cba82b702dcdc5e155893f63610cf388d30cd988d4ca2bf" +dependencies = [ + "aliasable", + "ouroboros_macro", + "stable_deref_trait", +] + +[[package]] +name = "ouroboros_macro" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "084fd65d5dd8b3772edccb5ffd1e4b7eba43897ecd0f9401e330e8c542959408" +dependencies = [ + "Inflector", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "parity-scale-codec" version = "2.3.1" @@ -1970,6 +2058,36 @@ dependencies = [ "toml", ] +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro-hack" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" + [[package]] name = "proc-macro2" version = "1.0.38" @@ -2200,15 +2318,30 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e75f6a532d0fd9f7f13144f392b6ad56a32696bfcd9c78f797f16bbb6f072d6" +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver 0.9.0", +] + [[package]] name = "rustc_version" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver", + "semver 1.0.9", ] +[[package]] +name = "rustversion" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2cc38e8fa666e2de3c4aba7edeb5ffc5246c1c2ed0e3d17e560aeeba736b23f" + [[package]] name = "ryu" version = "1.0.10" @@ -2231,6 +2364,106 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "sea-orm" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51de529763804dd4f74c133055f53eccdda2221bdded94351009be28cc80d2fb" +dependencies = [ + "async-stream", + "async-trait", + "chrono", + "futures", + "futures-util", + "once_cell", + "ouroboros", + "rust_decimal", + "sea-orm-macros", + "sea-query", + "sea-strum", + "serde", + "serde_json", + "sqlx", + "time 0.2.27", + "tracing", + "url", + "uuid", +] + +[[package]] +name = "sea-orm-macros" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f9378e21366b119d281489013c8170c49972fd3709c2155eb4504a913715d2d" +dependencies = [ + "bae", + "heck 0.3.3", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sea-query" +version = "0.24.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b0fa62db5ae33dfc61e805b0b0c9d579c3733f1ed90326b3779f5b38f30fa2a" +dependencies = [ + "chrono", + "rust_decimal", + "sea-query-derive", + "sea-query-driver", + "serde_json", + "time 0.2.27", + "uuid", +] + +[[package]] +name = "sea-query-derive" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34cdc022b4f606353fe5dc85b09713a04e433323b70163e81513b141c6ae6eb5" +dependencies = [ + "heck 0.3.3", + "proc-macro2", + "quote", + "syn", + "thiserror", +] + +[[package]] +name = "sea-query-driver" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3953baee94dcb90f0e19e8b4b91b91e9394867b0fc1886d0221cfc6d0439f5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "sea-strum" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "391d06a6007842cfe79ac6f7f53911b76dfd69fc9a6769f1cf6569d12ce20e1b" +dependencies = [ + "sea-strum_macros", +] + +[[package]] +name = "sea-strum_macros" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b4397b825df6ccf1e98bcdabef3bbcfc47ff5853983467850eeab878384f21" +dependencies = [ + "heck 0.3.3", + "proc-macro2", + "quote", + "rustversion", + "syn", +] + [[package]] name = "security-framework" version = "2.6.1" @@ -2254,12 +2487,27 @@ dependencies = [ "libc", ] +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + [[package]] name = "semver" version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8cb243bdfdb5936c8dc3c45762a19d12ab4550cdc753bc247637d4ec35a040fd" +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + [[package]] name = "serde" version = "1.0.137" @@ -2312,8 +2560,11 @@ dependencies = [ "dotenv", "envy", "futures", + "mime", + "sea-orm", "serde", - "sqlx", + "serde_json", + "thiserror", "tokio", "tracing", "tracing-subscriber", @@ -2354,6 +2605,21 @@ dependencies = [ "digest 0.10.3", ] +[[package]] +name = "sha1" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1da05c97445caa12d05e848c4a4fcbbea29e748ac28f7e80e9b010392063770" +dependencies = [ + "sha1_smol", +] + +[[package]] +name = "sha1_smol" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" + [[package]] name = "sha2" version = "0.9.9" @@ -2503,6 +2769,7 @@ dependencies = [ "sqlx-rt", "stringprep", "thiserror", + "time 0.2.27", "tokio-stream", "url", "uuid", @@ -2521,6 +2788,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", + "serde_json", "sha2 0.10.2", "sqlx-core", "sqlx-rt", @@ -2540,12 +2808,76 @@ dependencies = [ "tokio-native-tls", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + +[[package]] +name = "standback" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e113fb6f3de07a243d434a56ec6f186dfd51cb08448239fe7bcae73f87ff28ff" +dependencies = [ + "version_check", +] + [[package]] name = "static_assertions" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "stdweb" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5" +dependencies = [ + "discard", + "rustc_version 0.2.3", + "stdweb-derive", + "stdweb-internal-macros", + "stdweb-internal-runtime", + "wasm-bindgen", +] + +[[package]] +name = "stdweb-derive" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" +dependencies = [ + "proc-macro2", + "quote", + "serde", + "serde_derive", + "syn", +] + +[[package]] +name = "stdweb-internal-macros" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11" +dependencies = [ + "base-x", + "proc-macro2", + "quote", + "serde", + "serde_derive", + "serde_json", + "sha1", + "syn", +] + +[[package]] +name = "stdweb-internal-runtime" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0" + [[package]] name = "stringprep" version = "0.1.2" @@ -2665,6 +2997,21 @@ dependencies = [ "winapi", ] +[[package]] +name = "time" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4752a97f8eebd6854ff91f1c1824cd6160626ac4bd44287f7f4ea2035a02a242" +dependencies = [ + "const_fn", + "libc", + "standback", + "stdweb", + "time-macros 0.1.1", + "version_check", + "winapi", +] + [[package]] name = "time" version = "0.3.9" @@ -2674,7 +3021,17 @@ dependencies = [ "itoa", "libc", "num_threads", - "time-macros", + "time-macros 0.2.4", +] + +[[package]] +name = "time-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957e9c6e26f12cb6d0dd7fc776bb67a706312e7299aed74c8dd5b17ebb27e2f1" +dependencies = [ + "proc-macro-hack", + "time-macros-impl", ] [[package]] @@ -2683,6 +3040,19 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42657b1a6f4d817cda8e7a0ace261fe0cc946cf3a80314390b22cc61ae080792" +[[package]] +name = "time-macros-impl" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3c141a1b43194f3f56a1411225df8646c55781d5f26db825b3d98507eb482f" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "standback", + "syn", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -2700,9 +3070,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.18.2" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4903bf0427cf68dddd5aa6a93220756f8be0c34fcfa9f5e6191e103e15a31395" +checksum = "0f392c8f16bda3456c0b00c6de39cb100449b98de55ac41c6cdd2bfcf53a1245" dependencies = [ "bytes", "libc", @@ -2928,6 +3298,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" dependencies = [ "getrandom 0.2.6", + "serde", ] [[package]] diff --git a/server/Cargo.toml b/server/Cargo.toml index 378b315..342d123 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -1,7 +1,7 @@ [package] +edition = "2021" name = "server" version = "0.1.0" -edition = "2021" [dependencies] actix-web = "4.0.1" @@ -9,8 +9,11 @@ chrono = "0.4.19" dotenv = "0.15.0" envy = "0.4.2" futures = "0.3.21" +mime = "0.3.16" +sea-orm = {version = "^0", features = ["sqlx-postgres", "runtime-tokio-native-tls", "macros"]} serde = "1.0.137" -sqlx = {version = "0.5", features = ["runtime-tokio-native-tls", "postgres", "decimal", "chrono", "uuid"]} -tokio = {version = "1", features = ["full"]} +serde_json = "1.0.81" +thiserror = "1.0.31" +tokio = "1.19.0" tracing = "0.1.34" tracing-subscriber = "0.3.11" diff --git a/server/src/indexer/access_keys.rs b/server/src/indexer/access_keys.rs new file mode 100644 index 0000000..457d651 --- /dev/null +++ b/server/src/indexer/access_keys.rs @@ -0,0 +1,31 @@ +//! SeaORM Entity. Generated by sea-orm-codegen 0.8.0 + +use super::sea_orm_active_enums::AccessKeyPermissionKind; +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel)] +#[sea_orm(table_name = "access_keys")] +pub struct Model { + #[sea_orm(primary_key, column_type = "Text")] + pub public_key: String, + #[sea_orm(primary_key, column_type = "Text")] + pub account_id: String, + #[sea_orm(column_type = "Text", nullable)] + pub created_by_receipt_id: Option, + #[sea_orm(column_type = "Text", nullable)] + pub deleted_by_receipt_id: Option, + pub permission_kind: AccessKeyPermissionKind, + #[sea_orm(column_type = "Decimal(Some((20, 0)))")] + pub last_update_block_height: Decimal, +} + +#[derive(Copy, Clone, Debug, EnumIter)] +pub enum Relation {} + +impl RelationTrait for Relation { + fn def(&self) -> RelationDef { + panic!("No RelationDef") + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/server/src/indexer/account_changes.rs b/server/src/indexer/account_changes.rs new file mode 100644 index 0000000..45d214e --- /dev/null +++ b/server/src/indexer/account_changes.rs @@ -0,0 +1,40 @@ +//! SeaORM Entity. Generated by sea-orm-codegen 0.8.0 + +use super::sea_orm_active_enums::StateChangeReasonKind; +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel)] +#[sea_orm(table_name = "account_changes")] +pub struct Model { + #[sea_orm(primary_key)] + pub id: i64, + #[sea_orm(column_type = "Text")] + pub affected_account_id: String, + #[sea_orm(column_type = "Decimal(Some((20, 0)))")] + pub changed_in_block_timestamp: Decimal, + #[sea_orm(column_type = "Text")] + pub changed_in_block_hash: String, + #[sea_orm(column_type = "Text", nullable)] + pub caused_by_transaction_hash: Option, + #[sea_orm(column_type = "Text", nullable)] + pub caused_by_receipt_id: Option, + pub update_reason: StateChangeReasonKind, + #[sea_orm(column_type = "Decimal(Some((45, 0)))")] + pub affected_account_nonstaked_balance: Decimal, + #[sea_orm(column_type = "Decimal(Some((45, 0)))")] + pub affected_account_staked_balance: Decimal, + #[sea_orm(column_type = "Decimal(Some((20, 0)))")] + pub affected_account_storage_usage: Decimal, + pub index_in_block: i32, +} + +#[derive(Copy, Clone, Debug, EnumIter)] +pub enum Relation {} + +impl RelationTrait for Relation { + fn def(&self) -> RelationDef { + panic!("No RelationDef") + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/server/src/indexer/accounts.rs b/server/src/indexer/accounts.rs new file mode 100644 index 0000000..bb3821f --- /dev/null +++ b/server/src/indexer/accounts.rs @@ -0,0 +1,29 @@ +//! SeaORM Entity. Generated by sea-orm-codegen 0.8.0 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel)] +#[sea_orm(table_name = "accounts")] +pub struct Model { + #[sea_orm(primary_key)] + pub id: i64, + #[sea_orm(column_type = "Text")] + pub account_id: String, + #[sea_orm(column_type = "Text", nullable)] + pub created_by_receipt_id: Option, + #[sea_orm(column_type = "Text", nullable)] + pub deleted_by_receipt_id: Option, + #[sea_orm(column_type = "Decimal(Some((20, 0)))")] + pub last_update_block_height: Decimal, +} + +#[derive(Copy, Clone, Debug, EnumIter)] +pub enum Relation {} + +impl RelationTrait for Relation { + fn def(&self) -> RelationDef { + panic!("No RelationDef") + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/server/src/indexer/action_receipt_actions.rs b/server/src/indexer/action_receipt_actions.rs new file mode 100644 index 0000000..d1935f2 --- /dev/null +++ b/server/src/indexer/action_receipt_actions.rs @@ -0,0 +1,31 @@ +//! SeaORM Entity. Generated by sea-orm-codegen 0.8.0 + +use super::sea_orm_active_enums::ActionKind; +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel)] +#[sea_orm(table_name = "action_receipt_actions")] +pub struct Model { + #[sea_orm(primary_key, column_type = "Text")] + pub receipt_id: String, + pub index_in_action_receipt: i32, + pub action_kind: ActionKind, + pub args: Json, + #[sea_orm(column_type = "Text")] + pub receipt_predecessor_account_id: String, + #[sea_orm(column_type = "Text")] + pub receipt_receiver_account_id: String, + #[sea_orm(column_type = "Decimal(Some((20, 0)))")] + pub receipt_included_in_block_timestamp: Decimal, +} + +#[derive(Copy, Clone, Debug, EnumIter)] +pub enum Relation {} + +impl RelationTrait for Relation { + fn def(&self) -> RelationDef { + panic!("No RelationDef") + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/server/src/indexer/action_receipt_input_data.rs b/server/src/indexer/action_receipt_input_data.rs new file mode 100644 index 0000000..788138a --- /dev/null +++ b/server/src/indexer/action_receipt_input_data.rs @@ -0,0 +1,23 @@ +//! SeaORM Entity. Generated by sea-orm-codegen 0.8.0 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel)] +#[sea_orm(table_name = "action_receipt_input_data")] +pub struct Model { + #[sea_orm(primary_key, column_type = "Text")] + pub input_data_id: String, + #[sea_orm(column_type = "Text")] + pub input_to_receipt_id: String, +} + +#[derive(Copy, Clone, Debug, EnumIter)] +pub enum Relation {} + +impl RelationTrait for Relation { + fn def(&self) -> RelationDef { + panic!("No RelationDef") + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/server/src/indexer/action_receipt_output_data.rs b/server/src/indexer/action_receipt_output_data.rs new file mode 100644 index 0000000..8f61551 --- /dev/null +++ b/server/src/indexer/action_receipt_output_data.rs @@ -0,0 +1,25 @@ +//! SeaORM Entity. Generated by sea-orm-codegen 0.8.0 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel)] +#[sea_orm(table_name = "action_receipt_output_data")] +pub struct Model { + #[sea_orm(primary_key, column_type = "Text")] + pub output_data_id: String, + #[sea_orm(column_type = "Text")] + pub output_from_receipt_id: String, + #[sea_orm(column_type = "Text")] + pub receiver_account_id: String, +} + +#[derive(Copy, Clone, Debug, EnumIter)] +pub enum Relation {} + +impl RelationTrait for Relation { + fn def(&self) -> RelationDef { + panic!("No RelationDef") + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/server/src/indexer/action_receipts.rs b/server/src/indexer/action_receipts.rs new file mode 100644 index 0000000..4e86e98 --- /dev/null +++ b/server/src/indexer/action_receipts.rs @@ -0,0 +1,27 @@ +//! SeaORM Entity. Generated by sea-orm-codegen 0.8.0 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel)] +#[sea_orm(table_name = "action_receipts")] +pub struct Model { + #[sea_orm(primary_key, column_type = "Text")] + pub receipt_id: String, + #[sea_orm(column_type = "Text")] + pub signer_account_id: String, + #[sea_orm(column_type = "Text")] + pub signer_public_key: String, + #[sea_orm(column_type = "Decimal(Some((45, 0)))")] + pub gas_price: Decimal, +} + +#[derive(Copy, Clone, Debug, EnumIter)] +pub enum Relation {} + +impl RelationTrait for Relation { + fn def(&self) -> RelationDef { + panic!("No RelationDef") + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/server/src/indexer/aggregated_circulating_supply.rs b/server/src/indexer/aggregated_circulating_supply.rs new file mode 100644 index 0000000..f123260 --- /dev/null +++ b/server/src/indexer/aggregated_circulating_supply.rs @@ -0,0 +1,33 @@ +//! SeaORM Entity. Generated by sea-orm-codegen 0.8.0 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel)] +#[sea_orm(table_name = "aggregated__circulating_supply")] +pub struct Model { + #[sea_orm(column_type = "Decimal(Some((20, 0)))")] + pub computed_at_block_timestamp: Decimal, + #[sea_orm(primary_key, column_type = "Text")] + pub computed_at_block_hash: String, + #[sea_orm(column_type = "Decimal(Some((45, 0)))")] + pub circulating_tokens_supply: Decimal, + #[sea_orm(column_type = "Decimal(Some((45, 0)))")] + pub total_tokens_supply: Decimal, + pub total_lockup_contracts_count: i32, + pub unfinished_lockup_contracts_count: i32, + #[sea_orm(column_type = "Decimal(Some((45, 0)))")] + pub foundation_locked_tokens: Decimal, + #[sea_orm(column_type = "Decimal(Some((45, 0)))")] + pub lockups_locked_tokens: Decimal, +} + +#[derive(Copy, Clone, Debug, EnumIter)] +pub enum Relation {} + +impl RelationTrait for Relation { + fn def(&self) -> RelationDef { + panic!("No RelationDef") + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/server/src/indexer/blocks.rs b/server/src/indexer/blocks.rs new file mode 100644 index 0000000..5d272c6 --- /dev/null +++ b/server/src/indexer/blocks.rs @@ -0,0 +1,33 @@ +//! SeaORM Entity. Generated by sea-orm-codegen 0.8.0 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel)] +#[sea_orm(table_name = "blocks")] +pub struct Model { + #[sea_orm(column_type = "Decimal(Some((20, 0)))")] + pub block_height: Decimal, + #[sea_orm(primary_key, column_type = "Text")] + pub block_hash: String, + #[sea_orm(column_type = "Text")] + pub prev_block_hash: String, + #[sea_orm(column_type = "Decimal(Some((20, 0)))")] + pub block_timestamp: Decimal, + #[sea_orm(column_type = "Decimal(Some((45, 0)))")] + pub total_supply: Decimal, + #[sea_orm(column_type = "Decimal(Some((45, 0)))")] + pub gas_price: Decimal, + #[sea_orm(column_type = "Text")] + pub author_account_id: String, +} + +#[derive(Copy, Clone, Debug, EnumIter)] +pub enum Relation {} + +impl RelationTrait for Relation { + fn def(&self) -> RelationDef { + panic!("No RelationDef") + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/server/src/indexer/chunks.rs b/server/src/indexer/chunks.rs new file mode 100644 index 0000000..7c00c9d --- /dev/null +++ b/server/src/indexer/chunks.rs @@ -0,0 +1,33 @@ +//! SeaORM Entity. Generated by sea-orm-codegen 0.8.0 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel)] +#[sea_orm(table_name = "chunks")] +pub struct Model { + #[sea_orm(column_type = "Text")] + pub included_in_block_hash: String, + #[sea_orm(primary_key, column_type = "Text")] + pub chunk_hash: String, + #[sea_orm(column_type = "Decimal(Some((20, 0)))")] + pub shard_id: Decimal, + #[sea_orm(column_type = "Text")] + pub signature: String, + #[sea_orm(column_type = "Decimal(Some((20, 0)))")] + pub gas_limit: Decimal, + #[sea_orm(column_type = "Decimal(Some((20, 0)))")] + pub gas_used: Decimal, + #[sea_orm(column_type = "Text")] + pub author_account_id: String, +} + +#[derive(Copy, Clone, Debug, EnumIter)] +pub enum Relation {} + +impl RelationTrait for Relation { + fn def(&self) -> RelationDef { + panic!("No RelationDef") + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/server/src/indexer/data_receipts.rs b/server/src/indexer/data_receipts.rs new file mode 100644 index 0000000..f981ffc --- /dev/null +++ b/server/src/indexer/data_receipts.rs @@ -0,0 +1,24 @@ +//! SeaORM Entity. Generated by sea-orm-codegen 0.8.0 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel)] +#[sea_orm(table_name = "data_receipts")] +pub struct Model { + #[sea_orm(primary_key, column_type = "Text")] + pub data_id: String, + #[sea_orm(primary_key, column_type = "Text")] + pub receipt_id: String, + pub data: Option>, +} + +#[derive(Copy, Clone, Debug, EnumIter)] +pub enum Relation {} + +impl RelationTrait for Relation { + fn def(&self) -> RelationDef { + panic!("No RelationDef") + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/server/src/indexer/execution_outcome_receipts.rs b/server/src/indexer/execution_outcome_receipts.rs new file mode 100644 index 0000000..a976ff0 --- /dev/null +++ b/server/src/indexer/execution_outcome_receipts.rs @@ -0,0 +1,25 @@ +//! SeaORM Entity. Generated by sea-orm-codegen 0.8.0 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel)] +#[sea_orm(table_name = "execution_outcome_receipts")] +pub struct Model { + #[sea_orm(primary_key, column_type = "Text")] + pub executed_receipt_id: String, + #[sea_orm(primary_key)] + pub index_in_execution_outcome: i32, + #[sea_orm(column_type = "Text")] + pub produced_receipt_id: String, +} + +#[derive(Copy, Clone, Debug, EnumIter)] +pub enum Relation {} + +impl RelationTrait for Relation { + fn def(&self) -> RelationDef { + panic!("No RelationDef") + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/server/src/indexer/execution_outcomes.rs b/server/src/indexer/execution_outcomes.rs new file mode 100644 index 0000000..5c4c7e1 --- /dev/null +++ b/server/src/indexer/execution_outcomes.rs @@ -0,0 +1,36 @@ +//! SeaORM Entity. Generated by sea-orm-codegen 0.8.0 + +use super::sea_orm_active_enums::ExecutionOutcomeStatus; +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel)] +#[sea_orm(table_name = "execution_outcomes")] +pub struct Model { + #[sea_orm(primary_key, column_type = "Text")] + pub receipt_id: String, + #[sea_orm(column_type = "Text")] + pub executed_in_block_hash: String, + #[sea_orm(column_type = "Decimal(Some((20, 0)))")] + pub executed_in_block_timestamp: Decimal, + pub index_in_chunk: i32, + #[sea_orm(column_type = "Decimal(Some((20, 0)))")] + pub gas_burnt: Decimal, + #[sea_orm(column_type = "Decimal(Some((45, 0)))")] + pub tokens_burnt: Decimal, + #[sea_orm(column_type = "Text")] + pub executor_account_id: String, + pub status: ExecutionOutcomeStatus, + #[sea_orm(column_type = "Decimal(Some((20, 0)))")] + pub shard_id: Decimal, +} + +#[derive(Copy, Clone, Debug, EnumIter)] +pub enum Relation {} + +impl RelationTrait for Relation { + fn def(&self) -> RelationDef { + panic!("No RelationDef") + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/server/src/indexer/mod.rs b/server/src/indexer/mod.rs new file mode 100644 index 0000000..9318367 --- /dev/null +++ b/server/src/indexer/mod.rs @@ -0,0 +1,21 @@ +//! SeaORM Entity. Generated by sea-orm-codegen 0.8.0 + +pub mod prelude; + +pub mod access_keys; +pub mod account_changes; +pub mod accounts; +pub mod action_receipt_actions; +pub mod action_receipt_input_data; +pub mod action_receipt_output_data; +pub mod action_receipts; +pub mod aggregated_circulating_supply; +pub mod blocks; +pub mod chunks; +pub mod data_receipts; +pub mod execution_outcome_receipts; +pub mod execution_outcomes; +pub mod receipts; +pub mod sea_orm_active_enums; +pub mod transaction_actions; +pub mod transactions; diff --git a/server/src/indexer/prelude.rs b/server/src/indexer/prelude.rs new file mode 100644 index 0000000..848ca59 --- /dev/null +++ b/server/src/indexer/prelude.rs @@ -0,0 +1,18 @@ +//! SeaORM Entity. Generated by sea-orm-codegen 0.8.0 + +pub use super::access_keys::Entity as AccessKeys; +pub use super::account_changes::Entity as AccountChanges; +pub use super::accounts::Entity as Accounts; +pub use super::action_receipt_actions::Entity as ActionReceiptActions; +pub use super::action_receipt_input_data::Entity as ActionReceiptInputData; +pub use super::action_receipt_output_data::Entity as ActionReceiptOutputData; +pub use super::action_receipts::Entity as ActionReceipts; +pub use super::aggregated_circulating_supply::Entity as AggregatedCirculatingSupply; +pub use super::blocks::Entity as Blocks; +pub use super::chunks::Entity as Chunks; +pub use super::data_receipts::Entity as DataReceipts; +pub use super::execution_outcome_receipts::Entity as ExecutionOutcomeReceipts; +pub use super::execution_outcomes::Entity as ExecutionOutcomes; +pub use super::receipts::Entity as Receipts; +pub use super::transaction_actions::Entity as TransactionActions; +pub use super::transactions::Entity as Transactions; diff --git a/server/src/indexer/receipts.rs b/server/src/indexer/receipts.rs new file mode 100644 index 0000000..9279125 --- /dev/null +++ b/server/src/indexer/receipts.rs @@ -0,0 +1,36 @@ +//! SeaORM Entity. Generated by sea-orm-codegen 0.8.0 + +use super::sea_orm_active_enums::ReceiptKind; +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel)] +#[sea_orm(table_name = "receipts")] +pub struct Model { + #[sea_orm(primary_key, column_type = "Text")] + pub receipt_id: String, + #[sea_orm(column_type = "Text")] + pub included_in_block_hash: String, + #[sea_orm(column_type = "Text")] + pub included_in_chunk_hash: String, + pub index_in_chunk: i32, + #[sea_orm(column_type = "Decimal(Some((20, 0)))")] + pub included_in_block_timestamp: Decimal, + #[sea_orm(column_type = "Text")] + pub predecessor_account_id: String, + #[sea_orm(column_type = "Text")] + pub receiver_account_id: String, + pub receipt_kind: ReceiptKind, + #[sea_orm(column_type = "Text")] + pub originated_from_transaction_hash: String, +} + +#[derive(Copy, Clone, Debug, EnumIter)] +pub enum Relation {} + +impl RelationTrait for Relation { + fn def(&self) -> RelationDef { + panic!("No RelationDef") + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/server/src/indexer/sea_orm_active_enums.rs b/server/src/indexer/sea_orm_active_enums.rs new file mode 100644 index 0000000..143a3b0 --- /dev/null +++ b/server/src/indexer/sea_orm_active_enums.rs @@ -0,0 +1,104 @@ +//! SeaORM Entity. Generated by sea-orm-codegen 0.8.0 + +use sea_orm::entity::prelude::*; + +#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum)] +#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "nft_event_kind")] +pub enum NftEventKind { + #[sea_orm(string_value = "BURN")] + Burn, + #[sea_orm(string_value = "MINT")] + Mint, + #[sea_orm(string_value = "TRANSFER")] + Transfer, +} +#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum)] +#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "ft_event_kind")] +pub enum FtEventKind { + #[sea_orm(string_value = "BURN")] + Burn, + #[sea_orm(string_value = "MINT")] + Mint, + #[sea_orm(string_value = "TRANSFER")] + Transfer, +} +#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum)] +#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "receipt_kind")] +pub enum ReceiptKind { + #[sea_orm(string_value = "ACTION")] + Action, + #[sea_orm(string_value = "DATA")] + Data, +} +#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum)] +#[sea_orm( + rs_type = "String", + db_type = "Enum", + enum_name = "execution_outcome_status" +)] +pub enum ExecutionOutcomeStatus { + #[sea_orm(string_value = "FAILURE")] + Failure, + #[sea_orm(string_value = "SUCCESS_RECEIPT_ID")] + SuccessReceiptId, + #[sea_orm(string_value = "SUCCESS_VALUE")] + SuccessValue, + #[sea_orm(string_value = "UNKNOWN")] + Unknown, +} +#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum)] +#[sea_orm( + rs_type = "String", + db_type = "Enum", + enum_name = "access_key_permission_kind" +)] +pub enum AccessKeyPermissionKind { + #[sea_orm(string_value = "FULL_ACCESS")] + FullAccess, + #[sea_orm(string_value = "FUNCTION_CALL")] + FunctionCall, +} +#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum)] +#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "action_kind")] +pub enum ActionKind { + #[sea_orm(string_value = "ADD_KEY")] + AddKey, + #[sea_orm(string_value = "CREATE_ACCOUNT")] + CreateAccount, + #[sea_orm(string_value = "DELETE_ACCOUNT")] + DeleteAccount, + #[sea_orm(string_value = "DELETE_KEY")] + DeleteKey, + #[sea_orm(string_value = "DEPLOY_CONTRACT")] + DeployContract, + #[sea_orm(string_value = "FUNCTION_CALL")] + FunctionCall, + #[sea_orm(string_value = "STAKE")] + Stake, + #[sea_orm(string_value = "TRANSFER")] + Transfer, +} +#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum)] +#[sea_orm( + rs_type = "String", + db_type = "Enum", + enum_name = "state_change_reason_kind" +)] +pub enum StateChangeReasonKind { + #[sea_orm(string_value = "ACTION_RECEIPT_GAS_REWARD")] + ActionReceiptGasReward, + #[sea_orm(string_value = "ACTION_RECEIPT_PROCESSING_STARTED")] + ActionReceiptProcessingStarted, + #[sea_orm(string_value = "MIGRATION")] + Migration, + #[sea_orm(string_value = "POSTPONED_RECEIPT")] + PostponedReceipt, + #[sea_orm(string_value = "RECEIPT_PROCESSING")] + ReceiptProcessing, + #[sea_orm(string_value = "TRANSACTION_PROCESSING")] + TransactionProcessing, + #[sea_orm(string_value = "UPDATED_DELAYED_RECEIPTS")] + UpdatedDelayedReceipts, + #[sea_orm(string_value = "VALIDATOR_ACCOUNTS_UPDATE")] + ValidatorAccountsUpdate, +} diff --git a/server/src/indexer/transaction_actions.rs b/server/src/indexer/transaction_actions.rs new file mode 100644 index 0000000..a801899 --- /dev/null +++ b/server/src/indexer/transaction_actions.rs @@ -0,0 +1,26 @@ +//! SeaORM Entity. Generated by sea-orm-codegen 0.8.0 + +use super::sea_orm_active_enums::ActionKind; +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel)] +#[sea_orm(table_name = "transaction_actions")] +pub struct Model { + #[sea_orm(primary_key, column_type = "Text")] + pub transaction_hash: String, + #[sea_orm(primary_key)] + pub index_in_transaction: i32, + pub action_kind: ActionKind, + pub args: Json, +} + +#[derive(Copy, Clone, Debug, EnumIter)] +pub enum Relation {} + +impl RelationTrait for Relation { + fn def(&self) -> RelationDef { + panic!("No RelationDef") + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/server/src/indexer/transactions.rs b/server/src/indexer/transactions.rs new file mode 100644 index 0000000..05addc2 --- /dev/null +++ b/server/src/indexer/transactions.rs @@ -0,0 +1,46 @@ +//! SeaORM Entity. Generated by sea-orm-codegen 0.8.0 + +use super::sea_orm_active_enums::ExecutionOutcomeStatus; +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel)] +#[sea_orm(table_name = "transactions")] +pub struct Model { + #[sea_orm(primary_key, column_type = "Text")] + pub transaction_hash: String, + #[sea_orm(column_type = "Text")] + pub included_in_block_hash: String, + #[sea_orm(column_type = "Text")] + pub included_in_chunk_hash: String, + pub index_in_chunk: i32, + #[sea_orm(column_type = "Decimal(Some((20, 0)))")] + pub block_timestamp: Decimal, + #[sea_orm(column_type = "Text")] + pub signer_account_id: String, + #[sea_orm(column_type = "Text")] + pub signer_public_key: String, + #[sea_orm(column_type = "Decimal(Some((20, 0)))")] + pub nonce: Decimal, + #[sea_orm(column_type = "Text")] + pub receiver_account_id: String, + #[sea_orm(column_type = "Text")] + pub signature: String, + pub status: ExecutionOutcomeStatus, + #[sea_orm(column_type = "Text")] + pub converted_into_receipt_id: String, + #[sea_orm(column_type = "Decimal(Some((20, 0)))", nullable)] + pub receipt_conversion_gas_burnt: Option, + #[sea_orm(column_type = "Decimal(Some((45, 0)))", nullable)] + pub receipt_conversion_tokens_burnt: Option, +} + +#[derive(Copy, Clone, Debug, EnumIter)] +pub enum Relation {} + +impl RelationTrait for Relation { + fn def(&self) -> RelationDef { + panic!("No RelationDef") + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/server/src/local/account.rs b/server/src/local/account.rs new file mode 100644 index 0000000..09ae316 --- /dev/null +++ b/server/src/local/account.rs @@ -0,0 +1,26 @@ +//! SeaORM Entity. Generated by sea-orm-codegen 0.8.0 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel)] +#[sea_orm(table_name = "account")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false, column_type = "Text")] + pub id: String, + #[sea_orm(column_type = "Decimal(Some((45, 0)))", nullable)] + pub balance: Option, + pub score: Option, + pub modified: Option, + pub consecutive_errors: Option, +} + +#[derive(Copy, Clone, Debug, EnumIter)] +pub enum Relation {} + +impl RelationTrait for Relation { + fn def(&self) -> RelationDef { + panic!("No RelationDef") + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/server/src/local/account_badge.rs b/server/src/local/account_badge.rs new file mode 100644 index 0000000..0cbc859 --- /dev/null +++ b/server/src/local/account_badge.rs @@ -0,0 +1,24 @@ +//! SeaORM Entity. Generated by sea-orm-codegen 0.8.0 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel)] +#[sea_orm(table_name = "account_badge")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub id: Uuid, + #[sea_orm(column_type = "Text")] + pub account_id: String, + pub badge_id: Uuid, +} + +#[derive(Copy, Clone, Debug, EnumIter)] +pub enum Relation {} + +impl RelationTrait for Relation { + fn def(&self) -> RelationDef { + panic!("No RelationDef") + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/server/src/local/mod.rs b/server/src/local/mod.rs new file mode 100644 index 0000000..e0f1085 --- /dev/null +++ b/server/src/local/mod.rs @@ -0,0 +1,6 @@ +//! SeaORM Entity. Generated by sea-orm-codegen 0.8.0 + +pub mod prelude; + +pub mod account; +pub mod account_badge; diff --git a/server/src/local/prelude.rs b/server/src/local/prelude.rs new file mode 100644 index 0000000..e82c198 --- /dev/null +++ b/server/src/local/prelude.rs @@ -0,0 +1,4 @@ +//! SeaORM Entity. Generated by sea-orm-codegen 0.8.0 + +pub use super::account::Entity as Account; +pub use super::account_badge::Entity as AccountBadge; diff --git a/server/src/main.rs b/server/src/main.rs index 15b116d..0a64f7f 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -1,6 +1,20 @@ -use actix_web::{get, App, HttpServer, Responder}; +use std::sync::Arc; + +use actix_web::{ + get, http::StatusCode, web, App, HttpRequest, HttpResponse, HttpServer, Responder, +}; use dotenv::dotenv; -use serde::Deserialize; +use indexer::prelude::*; +use local::prelude::*; +use sea_orm::{ + ColumnTrait, Condition, Database, DatabaseConnection, EntityTrait, JoinType, QueryFilter, + QueryOrder, QuerySelect, FromQueryResult, +}; +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +mod indexer; +mod local; #[inline] fn default_port() -> u16 { @@ -15,19 +29,187 @@ struct Configuration { pub indexer_url: String, } +#[derive(Serialize)] +struct AppResponse { + pub result: T, +} + +#[derive(Serialize, Error, Debug)] +#[serde(tag = "error", rename_all = "snake_case")] +enum AppError { + #[error("Database query error: {0:?}")] + Query( + #[from] + #[serde(skip)] // Don't expose database errors externallysea_orm::error::DbErr, + sea_orm::DbErr, + ), + #[error("Account not indexed: {account_id}")] + NotFound { account_id: String }, +} + +impl actix_web::error::ResponseError for AppError { + fn status_code(&self) -> StatusCode { + match self { + AppError::NotFound { .. } => StatusCode::NOT_FOUND, + _ => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + let serialized = serde_json::to_string(self); + match serialized { + Ok(serialized) => HttpResponse::build(self.status_code()) + .content_type(mime::APPLICATION_JSON) + .body(serialized), + Err(_) => HttpResponse::InternalServerError() + .content_type(mime::APPLICATION_JSON) + .body("error"), + } + } +} + +struct AppState { + pub local_pool: Arc, + pub indexer_pool: Arc, +} + +impl Clone for AppState { + fn clone(&self) -> Self { + Self { + local_pool: Arc::clone(&self.local_pool), + indexer_pool: Arc::clone(&self.indexer_pool), + } + } +} + #[get("/status")] async fn status() -> impl Responder { "ok" } -#[tokio::main] -async fn main() -> std::io::Result<()> { +#[derive(Deserialize)] +struct TimestampRange { + pub after_block_timestamp: Option, + pub before_block_timestamp: Option, +} + +#[get("account/{account_id}/actions")] +async fn account_actions( + state: web::Data, + req: HttpRequest, + account_id: web::Path, +) -> Result { + use indexer::*; + + #[derive(FromQueryResult, Serialize, Debug)] + struct AccountAction { + pub receipt_id: String, + pub index_in_action_receipt: i32, + pub transaction_hash: String, + pub action_kind: String, // TODO: Enum? + pub block_hash: String, + pub block_timestamp: u64, + pub predecessor_account_id: String, + pub receiver_account_id: String, + } + + let account_id = account_id.into_inner(); + let range = web::Query::::from_query(req.query_string()).unwrap(); // Should always be safe because req.query_string() should never return an invalid query string + + let condition = { + let mut condition = Condition::all() + .add(receipts::Column::ReceiptKind.eq("ACTION")) + .add( + Condition::any() + .add(receipts::Column::PredecessorAccountId.eq(account_id.clone())) + .add(receipts::Column::ReceiverAccountId.eq(account_id.clone())), + ); + + if let Some(after) = range.after_block_timestamp { + condition = condition.add(receipts::Column::IncludedInBlockTimestamp.gte(after)); + } + + if let Some(before) = range.before_block_timestamp { + condition = condition.add(receipts::Column::IncludedInBlockTimestamp.lt(before)); + } + + condition + }; + + let res = Receipts::find() + .column_as( + receipts::Column::OriginatedFromTransactionHash, + "transaction_hash", + ) + .column_as(receipts::Column::IncludedInBlockHash, "block_hash") + .column_as( + receipts::Column::IncludedInBlockTimestamp, + "block_timestamp", + ) + .column(action_receipt_actions::Column::IndexInActionReceipt) + .column(action_receipt_actions::Column::ActionKind) + .filter(condition) + .join_rev( + JoinType::LeftJoin, + action_receipt_actions::Entity::belongs_to(receipts::Entity) + .from(action_receipt_actions::Column::ReceiptId) + .to(receipts::Column::ReceiptId) + .into(), + ) + .order_by_desc(receipts::Column::IncludedInBlockTimestamp) + .limit(2000) + .into_model::() + .all(&*state.indexer_pool) + .await; + + if let Err(ref e) = res { + println!("{e:?}"); + } + + let res = res?; + + Ok(web::Json(res)) +} + +#[get("account/{account_id}/score")] +async fn account_score( + state: web::Data, + account_id: web::Path, +) -> Result { + let account_id = account_id.into_inner(); + let res = Account::find_by_id(account_id.clone()) + .one(&*state.local_pool) + .await?; + + if let Some(res) = res { + Ok(web::Json(AppResponse { result: res.score })) + } else { + Err(AppError::NotFound { account_id }) + } +} + +#[actix_web::main] +async fn main() -> std::result::Result<(), Box> { dotenv().ok(); let config = envy::from_env::().unwrap(); - HttpServer::new(|| App::new().service(status)) - .bind(("127.0.0.1", config.port))? - .run() - .await + let local_pool = Database::connect(&config.database_url).await?; + let indexer_pool = Database::connect(&config.indexer_url).await?; + + let state = AppState { + local_pool: Arc::new(local_pool), + indexer_pool: Arc::new(indexer_pool), + }; + + Ok(HttpServer::new(move || { + App::new() + .app_data(web::Data::new(state.clone())) + .service(status) + .service(account_score) + .service(account_actions) + }) + .bind(("127.0.0.1", config.port))? + .run() + .await?) } diff --git a/service/migrations/20220603200449_modified-with-timezone.down.sql b/service/migrations/20220603200449_modified-with-timezone.down.sql new file mode 100644 index 0000000..6fbe490 --- /dev/null +++ b/service/migrations/20220603200449_modified-with-timezone.down.sql @@ -0,0 +1,2 @@ +alter table account + alter column modified type timestamp without time zone; diff --git a/service/migrations/20220603200449_modified-with-timezone.up.sql b/service/migrations/20220603200449_modified-with-timezone.up.sql new file mode 100644 index 0000000..5384114 --- /dev/null +++ b/service/migrations/20220603200449_modified-with-timezone.up.sql @@ -0,0 +1,2 @@ +alter table account + alter column modified type timestamp with time zone; diff --git a/service/src/indexer.rs b/service/src/indexer.rs index 3ffbf96..e966f85 100644 --- a/service/src/indexer.rs +++ b/service/src/indexer.rs @@ -72,7 +72,6 @@ pub async fn calculate_account_score( .await } -#[tracing::instrument(skip(indexer_pool))] #[async_recursion] pub async fn calculate_account_score_rec( indexer_pool: &PgPool, @@ -108,8 +107,6 @@ group by action_kind error!("Splitting because of error: {e:?}"); let midpoint = (max_timestamp + min_timestamp) / 2; info!("Score split for {account_id}: {min_timestamp} | {max_timestamp}"); - // let first = calculate_account_score_rec(indexer_pool, account_id, min_timestamp, midpoint).await; - // let second = calculate_account_score_rec(indexer_pool, account_id, midpoint, max_timestamp).await; let (first, second) = join!( calculate_account_score_rec(indexer_pool, account_id, min_timestamp, midpoint), calculate_account_score_rec(indexer_pool, account_id, midpoint, max_timestamp), diff --git a/service/src/local.rs b/service/src/local.rs index dd1f0ee..0ba74d7 100644 --- a/service/src/local.rs +++ b/service/src/local.rs @@ -1,6 +1,6 @@ use std::{collections::HashSet, str::FromStr, sync::Arc}; -use chrono::{Duration, NaiveDateTime, TimeZone}; +use chrono::Duration; use futures::{FutureExt, StreamExt, TryFutureExt, TryStreamExt}; use lazy_static::lazy_static; use near_jsonrpc_client::{errors::JsonRpcError, methods::query::RpcQueryError}; @@ -48,7 +48,7 @@ pub struct AccountRecordDb { pub id: String, pub balance: Option, pub score: Option, - pub modified: Option, + pub modified: Option>, pub consecutive_errors: Option, } @@ -79,7 +79,7 @@ impl From for AccountRecord { .unwrap_or_else(|_| NULL_ACCOUNT.to_owned()), balance: record.balance.and_then(|b| u128::try_from(b).ok()), score: record.score.map(|s| s as u32), - modified: record.modified.map(|m| chrono::Utc.from_utc_datetime(&m)), + modified: record.modified, consecutive_errors: record.consecutive_errors.unwrap_or(0) as u16, } } From d8957d2229080a1fb46521a4a32f923abc2f3586 Mon Sep 17 00:00:00 2001 From: Jacob Date: Tue, 9 Aug 2022 20:58:05 +0900 Subject: [PATCH 31/31] upgrades libraries and solves seaorm column enum issue --- Cargo.lock | 607 +++++++++++------- server/Cargo.toml | 19 +- server/src/indexer/access_keys.rs | 5 +- server/src/indexer/account_changes.rs | 5 +- server/src/indexer/accounts.rs | 5 +- server/src/indexer/action_receipt_actions.rs | 5 +- .../src/indexer/action_receipt_input_data.rs | 5 +- .../src/indexer/action_receipt_output_data.rs | 5 +- server/src/indexer/action_receipts.rs | 5 +- .../indexer/aggregated_circulating_supply.rs | 5 +- server/src/indexer/blocks.rs | 5 +- server/src/indexer/chunks.rs | 5 +- server/src/indexer/data_receipts.rs | 5 +- .../src/indexer/execution_outcome_receipts.rs | 5 +- server/src/indexer/execution_outcomes.rs | 5 +- server/src/indexer/mod.rs | 2 +- server/src/indexer/prelude.rs | 2 +- server/src/indexer/receipts.rs | 5 +- server/src/indexer/sea_orm_active_enums.rs | 119 ++-- server/src/indexer/transaction_actions.rs | 5 +- server/src/indexer/transactions.rs | 5 +- server/src/main.rs | 141 +++- 22 files changed, 613 insertions(+), 357 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9af7e63..b7465a3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -160,9 +160,9 @@ dependencies = [ [[package]] name = "actix-web" -version = "4.0.1" +version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4e5ebffd51d50df56a3ae0de0e59487340ca456f05dd0b90c0a7a6dd6a74d31" +checksum = "a27e8fe9ba4ae613c21f677c2cfaf0696c3744030c6f485b34634e502d6bb379" dependencies = [ "actix-codec", "actix-http", @@ -362,6 +362,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "atoi" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c57d12312ff59c811c0643f4d80830505833c9ffaebd193d819392b265be8e" +dependencies = [ + "num-traits", +] + [[package]] name = "autocfg" version = "1.1.0" @@ -396,12 +405,6 @@ dependencies = [ "syn", ] -[[package]] -name = "base-x" -version = "0.2.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc19a4937b4fbd3fe3379793130e42060d10627a360f2127802b10b87e7baf74" - [[package]] name = "base64" version = "0.11.0" @@ -414,6 +417,23 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" +[[package]] +name = "bigdecimal" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6aaf33151a6429fe9211d1b276eafdf70cdff28b071e76c0b0e1503221ea3744" +dependencies = [ + "num-bigint 0.4.3", + "num-integer", + "num-traits", +] + +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + [[package]] name = "bitflags" version = "1.3.2" @@ -468,7 +488,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "15bf3650200d8bffa99015595e10f1fbd17de07abbc25bb067da79e769939bfa" dependencies = [ "borsh-derive", - "hashbrown", + "hashbrown 0.11.2", ] [[package]] @@ -533,6 +553,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "771fe0050b883fcc3ea2359b1a96bcfbc090b7116eae7c3c512c7a083fdf23d3" +[[package]] +name = "bstr" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" +dependencies = [ + "memchr", +] + [[package]] name = "bumpalo" version = "3.9.1" @@ -605,15 +634,16 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.19" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73" +checksum = "6127248204b9aba09a362f6c930ef6a78f2c1b2215f8a7b398c06e1083f17af0" dependencies = [ - "libc", + "js-sys", "num-integer", "num-traits", "serde", "time 0.1.43", + "wasm-bindgen", "winapi", ] @@ -626,12 +656,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "const_fn" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbdcdcb6d86f71c5e97409ad45898af11cbc995b4ee8112d59095a28d376c935" - [[package]] name = "convert_case" version = "0.4.0" @@ -680,7 +704,16 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49fc9a695bca7f35f5f4c15cddc84415f66a74ea78eef08e90c5024f2b540e23" dependencies = [ - "crc-catalog", + "crc-catalog 1.1.1", +] + +[[package]] +name = "crc" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53757d12b596c16c78b83458d732a5d1a17ab3f53f2f7412f6fb57cc8a140ab3" +dependencies = [ + "crc-catalog 2.1.0", ] [[package]] @@ -689,6 +722,12 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccaeedb56da03b09f598226e25e80088cb4cd25f316e6e4df7d695f0feeb1403" +[[package]] +name = "crc-catalog" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d0165d2900ae6778e36e80bbc4da3b5eefccee9ba939761f9c2882a5d9af3ff" + [[package]] name = "crc32fast" version = "1.3.2" @@ -776,7 +815,7 @@ dependencies = [ "convert_case", "proc-macro2", "quote", - "rustc_version 0.4.0", + "rustc_version", "syn", ] @@ -820,18 +859,21 @@ dependencies = [ "winapi", ] -[[package]] -name = "discard" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" - [[package]] name = "dotenv" version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" +[[package]] +name = "dotenvy" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e851a83c30366fd01d75b913588e95e74a1705c1ecc5d58b1f8e1a6d556525f" +dependencies = [ + "dirs", +] + [[package]] name = "easy-ext" version = "0.2.9" @@ -913,6 +955,12 @@ dependencies = [ "synstructure", ] +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + [[package]] name = "fastrand" version = "1.7.0" @@ -1125,6 +1173,19 @@ version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78cc372d058dcf6d5ecd98510e7fbc9e5aec4d21de70f65fea8fecebcd881bd4" +[[package]] +name = "git2" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0155506aab710a86160ddb504a480d2964d7ab5b9e62419be69e0032bc5931c" +dependencies = [ + "bitflags", + "libc", + "libgit2-sys", + "log", + "url", +] + [[package]] name = "h2" version = "0.3.13" @@ -1153,13 +1214,31 @@ dependencies = [ "ahash", ] +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash", +] + [[package]] name = "hashlink" version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7249a3129cbc1ffccd74857f81464a323a152173cdb134e0fd81bc803b29facf" dependencies = [ - "hashbrown", + "hashbrown 0.11.2", +] + +[[package]] +name = "hashlink" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d452c155cb93fecdfb02a73dd57b5d8e442c2063bd7aac72f1bc5e4263a43086" +dependencies = [ + "hashbrown 0.12.3", ] [[package]] @@ -1322,7 +1401,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f647032dfaa1f8b6dc29bd3edb7bbef4861b8b8007ebb118d6db284fd59f6ee" dependencies = [ "autocfg", - "hashbrown", + "hashbrown 0.11.2", ] [[package]] @@ -1340,6 +1419,12 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b" +[[package]] +name = "ipnetwork" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f84f1612606f3753f205a4e9a2efd6fe5b4c573a6269b2cc6c3003d44a0d127" + [[package]] name = "itertools" version = "0.10.3" @@ -1391,6 +1476,30 @@ version = "0.2.125" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5916d2ae698f6de9bfb891ad7a8d65c09d232dc58cc4ac433c7da3b2fd84bc2b" +[[package]] +name = "libgit2-sys" +version = "0.13.4+1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0fa6563431ede25f5cc7f6d803c6afbc1c5d3ad3d4925d12c882bf2b526f5d1" +dependencies = [ + "cc", + "libc", + "libz-sys", + "pkg-config", +] + +[[package]] +name = "libz-sys" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9702761c3935f8cc2f101793272e202c72b99da8f4224a19ddcf1279a6450bbf" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "local-channel" version = "0.1.3" @@ -1428,6 +1537,16 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "mac_address" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df1d1bc1084549d60725ccc53a2bfa07f67fe4689fda07b05a36531f2988104a" +dependencies = [ + "nix", + "winapi", +] + [[package]] name = "matches" version = "0.1.9" @@ -1449,6 +1568,15 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +[[package]] +name = "memoffset" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" +dependencies = [ + "autocfg", +] + [[package]] name = "mime" version = "0.3.16" @@ -1613,7 +1741,7 @@ dependencies = [ "serde", "serde_json", "thiserror", - "uuid", + "uuid 0.8.2", ] [[package]] @@ -1636,7 +1764,7 @@ dependencies = [ "serde_json", "thiserror", "tracing", - "uuid", + "uuid 0.8.2", ] [[package]] @@ -1744,6 +1872,19 @@ dependencies = [ "serde", ] +[[package]] +name = "nix" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f866317acbd3a240710c63f065ffb1e4fd466259045ccb504130b7f668f35c6" +dependencies = [ + "bitflags", + "cc", + "cfg-if 1.0.0", + "libc", + "memoffset", +] + [[package]] name = "nom" version = "7.1.1" @@ -1765,6 +1906,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + [[package]] name = "num-integer" version = "0.1.45" @@ -1782,7 +1934,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12ac428b1cb17fce6f731001d307d351ec70a6d202fc2e60f7d4c5e42d8f4f07" dependencies = [ "autocfg", - "num-bigint", + "num-bigint 0.3.3", "num-integer", "num-traits", "serde", @@ -1827,9 +1979,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.10.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9" +checksum = "18a6dbe30758c9f83eb00cbea4ac95966305f5a7772f3f42ebfc7fc7eddbd8e1" [[package]] name = "opaque-debug" @@ -2022,6 +2174,40 @@ version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" +[[package]] +name = "postgres-protocol" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "878c6cbf956e03af9aa8204b407b9cbf47c072164800aa918c516cd4b056c50c" +dependencies = [ + "base64 0.13.0", + "byteorder", + "bytes", + "fallible-iterator", + "hmac", + "md-5", + "memchr", + "rand 0.8.5", + "sha2 0.10.2", + "stringprep", +] + +[[package]] +name = "postgres-types" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebd6e8b7189a73169290e89bd24c771071f1012d8fe6f738f5226531f0b03d89" +dependencies = [ + "bytes", + "chrono", + "fallible-iterator", + "postgres-protocol", + "serde", + "serde_json", + "time 0.3.9", + "uuid 1.1.2", +] + [[package]] name = "ppv-lite86" version = "0.2.16" @@ -2082,12 +2268,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "proc-macro-hack" -version = "0.5.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" - [[package]] name = "proc-macro2" version = "1.0.38" @@ -2297,9 +2477,9 @@ dependencies = [ [[package]] name = "rust_decimal" -version = "1.23.1" +version = "1.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22dc69eadbf0ee2110b8d20418c0c6edbaefec2811c4963dc17b6344e11fe0f8" +checksum = "ee9164faf726e4f3ece4978b25ca877ddc6802fa77f38cdccb32c7f805ecd70c" dependencies = [ "arrayvec 0.7.2", "num-traits", @@ -2318,22 +2498,13 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e75f6a532d0fd9f7f13144f392b6ad56a32696bfcd9c78f797f16bbb6f072d6" -[[package]] -name = "rustc_version" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" -dependencies = [ - "semver 0.9.0", -] - [[package]] name = "rustc_version" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver 1.0.9", + "semver", ] [[package]] @@ -2366,15 +2537,16 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "sea-orm" -version = "0.8.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51de529763804dd4f74c133055f53eccdda2221bdded94351009be28cc80d2fb" +checksum = "e05961693fe32ef4dfc964e48dcd0363151e3fab0215e3f3e138e034710339ca" dependencies = [ "async-stream", "async-trait", "chrono", "futures", "futures-util", + "log", "once_cell", "ouroboros", "rust_decimal", @@ -2383,18 +2555,18 @@ dependencies = [ "sea-strum", "serde", "serde_json", - "sqlx", - "time 0.2.27", + "sqlx 0.6.1", + "time 0.3.9", "tracing", "url", - "uuid", + "uuid 1.1.2", ] [[package]] name = "sea-orm-macros" -version = "0.8.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f9378e21366b119d281489013c8170c49972fd3709c2155eb4504a913715d2d" +checksum = "6a8c8c00c8a57ae61991b34822f32ded409afb66a971d4952d857c329efbc355" dependencies = [ "bae", "heck 0.3.3", @@ -2405,17 +2577,18 @@ dependencies = [ [[package]] name = "sea-query" -version = "0.24.6" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b0fa62db5ae33dfc61e805b0b0c9d579c3733f1ed90326b3779f5b38f30fa2a" +checksum = "2997a3e57614c0ff2212890e143deadb4a31f8057cbe7fc163daa3f297a33527" dependencies = [ "chrono", + "postgres-types", "rust_decimal", "sea-query-derive", "sea-query-driver", "serde_json", - "time 0.2.27", - "uuid", + "time 0.3.9", + "uuid 1.1.2", ] [[package]] @@ -2433,9 +2606,9 @@ dependencies = [ [[package]] name = "sea-query-driver" -version = "0.1.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3953baee94dcb90f0e19e8b4b91b91e9394867b0fc1886d0221cfc6d0439f5" +checksum = "fbda46eb3484cae1efb7bc68bca50f553a5b42c076cf4cbfae05b27f707549d4" dependencies = [ "proc-macro2", "quote", @@ -2487,41 +2660,26 @@ dependencies = [ "libc", ] -[[package]] -name = "semver" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" -dependencies = [ - "semver-parser", -] - [[package]] name = "semver" version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8cb243bdfdb5936c8dc3c45762a19d12ab4550cdc753bc247637d4ec35a040fd" -[[package]] -name = "semver-parser" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" - [[package]] name = "serde" -version = "1.0.137" +version = "1.0.142" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61ea8d54c77f8315140a05f4c7237403bf38b72704d031543aa1d16abbf517d1" +checksum = "e590c437916fb6b221e1d00df6e3294f3fccd70ca7e92541c475d6ed6ef5fee2" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.137" +version = "1.0.142" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f26faba0c3959972377d3b2d306ee9f71faee9714294e41bb777f83f88578be" +checksum = "34b5b8d809babe02f538c2cfec6f2c1ed10804c0e5a6a041a049a4f5588ccc2e" dependencies = [ "proc-macro2", "quote", @@ -2530,9 +2688,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.81" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c" +checksum = "38dd04e3c8279e75b31ef29dbdceebfe5ad89f4d0937213c53f7d49d01b3d5a7" dependencies = [ "itoa", "ryu", @@ -2564,6 +2722,7 @@ dependencies = [ "sea-orm", "serde", "serde_json", + "sqlx 0.6.1", "thiserror", "tokio", "tracing", @@ -2586,7 +2745,7 @@ dependencies = [ "near-primitives", "num-traits", "serde", - "sqlx", + "sqlx 0.5.13", "tap", "thiserror", "tokio", @@ -2605,21 +2764,6 @@ dependencies = [ "digest 0.10.3", ] -[[package]] -name = "sha1" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1da05c97445caa12d05e848c4a4fcbbea29e748ac28f7e80e9b010392063770" -dependencies = [ - "sha1_smol", -] - -[[package]] -name = "sha1_smol" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" - [[package]] name = "sha2" version = "0.9.9" @@ -2718,8 +2862,18 @@ version = "0.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "551873805652ba0d912fec5bbb0f8b4cdd96baf8e2ebf5970e5671092966019b" dependencies = [ - "sqlx-core", - "sqlx-macros", + "sqlx-core 0.5.13", + "sqlx-macros 0.5.13", +] + +[[package]] +name = "sqlx" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "788841def501aabde58d3666fcea11351ec3962e6ea75dbcd05c84a71d68bcd1" +dependencies = [ + "sqlx-core 0.6.1", + "sqlx-macros 0.6.1", ] [[package]] @@ -2729,32 +2883,92 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e48c61941ccf5ddcada342cd59e3e5173b007c509e1e8e990dafc830294d9dc5" dependencies = [ "ahash", - "atoi", + "atoi 0.4.0", + "base64 0.13.0", + "bitflags", + "byteorder", + "bytes", + "chrono", + "crc 2.1.0", + "crossbeam-queue", + "dirs", + "either", + "event-listener", + "futures-channel", + "futures-core", + "futures-intrusive", + "futures-util", + "hashlink 0.7.0", + "hex", + "hkdf", + "hmac", + "indexmap", + "itoa", + "libc", + "log", + "md-5", + "memchr", + "num-bigint 0.3.3", + "once_cell", + "paste", + "percent-encoding", + "rand 0.8.5", + "rust_decimal", + "serde", + "serde_json", + "sha-1", + "sha2 0.10.2", + "smallvec", + "sqlformat", + "sqlx-rt 0.5.13", + "stringprep", + "thiserror", + "tokio-stream", + "url", + "uuid 0.8.2", + "whoami", +] + +[[package]] +name = "sqlx-core" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c21d3b5e7cadfe9ba7cdc1295f72cc556c750b4419c27c219c0693198901f8e" +dependencies = [ + "ahash", + "atoi 1.0.0", "base64 0.13.0", + "bigdecimal", + "bit-vec", "bitflags", + "bstr", "byteorder", "bytes", "chrono", - "crc", + "crc 3.0.0", "crossbeam-queue", "dirs", + "dotenvy", "either", "event-listener", "futures-channel", "futures-core", "futures-intrusive", "futures-util", - "hashlink", + "git2", + "hashlink 0.8.0", "hex", "hkdf", "hmac", "indexmap", + "ipnetwork", "itoa", "libc", "log", + "mac_address", "md-5", "memchr", - "num-bigint", + "num-bigint 0.4.3", "once_cell", "paste", "percent-encoding", @@ -2766,13 +2980,13 @@ dependencies = [ "sha2 0.10.2", "smallvec", "sqlformat", - "sqlx-rt", + "sqlx-rt 0.6.1", "stringprep", "thiserror", - "time 0.2.27", + "time 0.3.9", "tokio-stream", "url", - "uuid", + "uuid 1.1.2", "whoami", ] @@ -2788,10 +3002,29 @@ dependencies = [ "once_cell", "proc-macro2", "quote", + "sha2 0.10.2", + "sqlx-core 0.5.13", + "sqlx-rt 0.5.13", + "syn", + "url", +] + +[[package]] +name = "sqlx-macros" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4adfd2df3557bddd3b91377fc7893e8fa899e9b4061737cbade4e1bb85f1b45c" +dependencies = [ + "dotenvy", + "either", + "heck 0.4.0", + "once_cell", + "proc-macro2", + "quote", "serde_json", "sha2 0.10.2", - "sqlx-core", - "sqlx-rt", + "sqlx-core 0.6.1", + "sqlx-rt 0.6.1", "syn", "url", ] @@ -2809,19 +3042,22 @@ dependencies = [ ] [[package]] -name = "stable_deref_trait" -version = "1.2.0" +name = "sqlx-rt" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +checksum = "7be52fc7c96c136cedea840ed54f7d446ff31ad670c9dea95ebcb998530971a3" +dependencies = [ + "native-tls", + "once_cell", + "tokio", + "tokio-native-tls", +] [[package]] -name = "standback" -version = "0.2.17" +name = "stable_deref_trait" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e113fb6f3de07a243d434a56ec6f186dfd51cb08448239fe7bcae73f87ff28ff" -dependencies = [ - "version_check", -] +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] name = "static_assertions" @@ -2829,55 +3065,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" -[[package]] -name = "stdweb" -version = "0.4.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5" -dependencies = [ - "discard", - "rustc_version 0.2.3", - "stdweb-derive", - "stdweb-internal-macros", - "stdweb-internal-runtime", - "wasm-bindgen", -] - -[[package]] -name = "stdweb-derive" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" -dependencies = [ - "proc-macro2", - "quote", - "serde", - "serde_derive", - "syn", -] - -[[package]] -name = "stdweb-internal-macros" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11" -dependencies = [ - "base-x", - "proc-macro2", - "quote", - "serde", - "serde_derive", - "serde_json", - "sha1", - "syn", -] - -[[package]] -name = "stdweb-internal-runtime" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0" - [[package]] name = "stringprep" version = "0.1.2" @@ -2960,18 +3147,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.31" +version = "1.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd829fe32373d27f76265620b5309d0340cb8550f523c1dda251d6298069069a" +checksum = "f5f6586b7f764adc0231f4c79be7b920e766bb2f3e51b3661cdb263828f19994" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.31" +version = "1.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0396bc89e626244658bef819e22d0cc459e795a5ebe878e6ec336d1674a8d79a" +checksum = "12bafc5b54507e0149cdf1b145a5d80ab80a90bcd9275df43d4fff68460f6c21" dependencies = [ "proc-macro2", "quote", @@ -2997,21 +3184,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "time" -version = "0.2.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4752a97f8eebd6854ff91f1c1824cd6160626ac4bd44287f7f4ea2035a02a242" -dependencies = [ - "const_fn", - "libc", - "standback", - "stdweb", - "time-macros 0.1.1", - "version_check", - "winapi", -] - [[package]] name = "time" version = "0.3.9" @@ -3021,17 +3193,7 @@ dependencies = [ "itoa", "libc", "num_threads", - "time-macros 0.2.4", -] - -[[package]] -name = "time-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "957e9c6e26f12cb6d0dd7fc776bb67a706312e7299aed74c8dd5b17ebb27e2f1" -dependencies = [ - "proc-macro-hack", - "time-macros-impl", + "time-macros", ] [[package]] @@ -3040,19 +3202,6 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42657b1a6f4d817cda8e7a0ace261fe0cc946cf3a80314390b22cc61ae080792" -[[package]] -name = "time-macros-impl" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd3c141a1b43194f3f56a1411225df8646c55781d5f26db825b3d98507eb482f" -dependencies = [ - "proc-macro-hack", - "proc-macro2", - "quote", - "standback", - "syn", -] - [[package]] name = "tinyvec" version = "1.6.0" @@ -3070,10 +3219,11 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.19.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f392c8f16bda3456c0b00c6de39cb100449b98de55ac41c6cdd2bfcf53a1245" +checksum = "7a8325f63a7d4774dd041e363b2409ed1c5cbbd0f867795e661df066b2b0a581" dependencies = [ + "autocfg", "bytes", "libc", "memchr", @@ -3165,9 +3315,9 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" [[package]] name = "tracing" -version = "0.1.34" +version = "0.1.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d0ecdcb44a79f0fe9844f0c4f33a342cbcbb5117de8001e6ba0dc2351327d09" +checksum = "2fce9567bd60a67d08a16488756721ba392f24f29006402881e43b19aac64307" dependencies = [ "cfg-if 1.0.0", "log", @@ -3178,9 +3328,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.21" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc6b8ad3567499f98a1db7a752b07a7c8c7c7c34c332ec00effb2b0027974b7c" +checksum = "11c75893af559bc8e10716548bdef5cb2b983f8e637db9d0e15126b61b484ee2" dependencies = [ "proc-macro2", "quote", @@ -3189,11 +3339,11 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.26" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f54c8ca710e81886d498c2fd3331b56c93aa248d49de2222ad2742247c60072f" +checksum = "5aeea4303076558a00714b823f9ad67d58a3bbda1df83d8827d21193156e22f7" dependencies = [ - "lazy_static", + "once_cell", "valuable", ] @@ -3210,9 +3360,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.11" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bc28f93baff38037f64e6f43d34cfa1605f27a49c34e8a04c5e78b0babf2596" +checksum = "60db860322da191b40952ad9affe65ea23e7dd6a5c442c2c42865810c6ab8e6b" dependencies = [ "ansi_term", "sharded-slab", @@ -3296,6 +3446,15 @@ name = "uuid" version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" +dependencies = [ + "getrandom 0.2.6", +] + +[[package]] +name = "uuid" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd6469f4314d5f1ffec476e05f17cc9a78bc7a27a6a857842170bdf8d6f98d2f" dependencies = [ "getrandom 0.2.6", "serde", diff --git a/server/Cargo.toml b/server/Cargo.toml index 342d123..1b408b8 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -4,16 +4,17 @@ name = "server" version = "0.1.0" [dependencies] -actix-web = "4.0.1" -chrono = "0.4.19" +actix-web = "4.1.0" +chrono = "0.4.20" dotenv = "0.15.0" envy = "0.4.2" futures = "0.3.21" mime = "0.3.16" -sea-orm = {version = "^0", features = ["sqlx-postgres", "runtime-tokio-native-tls", "macros"]} -serde = "1.0.137" -serde_json = "1.0.81" -thiserror = "1.0.31" -tokio = "1.19.0" -tracing = "0.1.34" -tracing-subscriber = "0.3.11" +sea-orm = { version = "0.9.1", features = ["sqlx-postgres", "runtime-tokio-native-tls", "macros"] } +serde = "1.0.142" +serde_json = "1.0.83" +sqlx = { version = "0.6.1", features = ["all-types", "decimal"] } +thiserror = "1.0.32" +tokio = "1.20.1" +tracing = "0.1.36" +tracing-subscriber = "0.3.15" diff --git a/server/src/indexer/access_keys.rs b/server/src/indexer/access_keys.rs index 457d651..42d6d9b 100644 --- a/server/src/indexer/access_keys.rs +++ b/server/src/indexer/access_keys.rs @@ -1,9 +1,10 @@ -//! SeaORM Entity. Generated by sea-orm-codegen 0.8.0 +//! SeaORM Entity. Generated by sea-orm-codegen 0.9.1 use super::sea_orm_active_enums::AccessKeyPermissionKind; use sea_orm::entity::prelude::*; +use serde::Serialize; -#[derive(Clone, Debug, PartialEq, DeriveEntityModel)] +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Serialize)] #[sea_orm(table_name = "access_keys")] pub struct Model { #[sea_orm(primary_key, column_type = "Text")] diff --git a/server/src/indexer/account_changes.rs b/server/src/indexer/account_changes.rs index 45d214e..36578c2 100644 --- a/server/src/indexer/account_changes.rs +++ b/server/src/indexer/account_changes.rs @@ -1,9 +1,10 @@ -//! SeaORM Entity. Generated by sea-orm-codegen 0.8.0 +//! SeaORM Entity. Generated by sea-orm-codegen 0.9.1 use super::sea_orm_active_enums::StateChangeReasonKind; use sea_orm::entity::prelude::*; +use serde::Serialize; -#[derive(Clone, Debug, PartialEq, DeriveEntityModel)] +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Serialize)] #[sea_orm(table_name = "account_changes")] pub struct Model { #[sea_orm(primary_key)] diff --git a/server/src/indexer/accounts.rs b/server/src/indexer/accounts.rs index bb3821f..96ecd1b 100644 --- a/server/src/indexer/accounts.rs +++ b/server/src/indexer/accounts.rs @@ -1,8 +1,9 @@ -//! SeaORM Entity. Generated by sea-orm-codegen 0.8.0 +//! SeaORM Entity. Generated by sea-orm-codegen 0.9.1 use sea_orm::entity::prelude::*; +use serde::Serialize; -#[derive(Clone, Debug, PartialEq, DeriveEntityModel)] +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Serialize)] #[sea_orm(table_name = "accounts")] pub struct Model { #[sea_orm(primary_key)] diff --git a/server/src/indexer/action_receipt_actions.rs b/server/src/indexer/action_receipt_actions.rs index d1935f2..005b22c 100644 --- a/server/src/indexer/action_receipt_actions.rs +++ b/server/src/indexer/action_receipt_actions.rs @@ -1,9 +1,10 @@ -//! SeaORM Entity. Generated by sea-orm-codegen 0.8.0 +//! SeaORM Entity. Generated by sea-orm-codegen 0.9.1 use super::sea_orm_active_enums::ActionKind; use sea_orm::entity::prelude::*; +use serde::Serialize; -#[derive(Clone, Debug, PartialEq, DeriveEntityModel)] +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Serialize)] #[sea_orm(table_name = "action_receipt_actions")] pub struct Model { #[sea_orm(primary_key, column_type = "Text")] diff --git a/server/src/indexer/action_receipt_input_data.rs b/server/src/indexer/action_receipt_input_data.rs index 788138a..2fc2909 100644 --- a/server/src/indexer/action_receipt_input_data.rs +++ b/server/src/indexer/action_receipt_input_data.rs @@ -1,8 +1,9 @@ -//! SeaORM Entity. Generated by sea-orm-codegen 0.8.0 +//! SeaORM Entity. Generated by sea-orm-codegen 0.9.1 use sea_orm::entity::prelude::*; +use serde::Serialize; -#[derive(Clone, Debug, PartialEq, DeriveEntityModel)] +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Serialize)] #[sea_orm(table_name = "action_receipt_input_data")] pub struct Model { #[sea_orm(primary_key, column_type = "Text")] diff --git a/server/src/indexer/action_receipt_output_data.rs b/server/src/indexer/action_receipt_output_data.rs index 8f61551..0817139 100644 --- a/server/src/indexer/action_receipt_output_data.rs +++ b/server/src/indexer/action_receipt_output_data.rs @@ -1,8 +1,9 @@ -//! SeaORM Entity. Generated by sea-orm-codegen 0.8.0 +//! SeaORM Entity. Generated by sea-orm-codegen 0.9.1 use sea_orm::entity::prelude::*; +use serde::Serialize; -#[derive(Clone, Debug, PartialEq, DeriveEntityModel)] +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Serialize)] #[sea_orm(table_name = "action_receipt_output_data")] pub struct Model { #[sea_orm(primary_key, column_type = "Text")] diff --git a/server/src/indexer/action_receipts.rs b/server/src/indexer/action_receipts.rs index 4e86e98..54d96d0 100644 --- a/server/src/indexer/action_receipts.rs +++ b/server/src/indexer/action_receipts.rs @@ -1,8 +1,9 @@ -//! SeaORM Entity. Generated by sea-orm-codegen 0.8.0 +//! SeaORM Entity. Generated by sea-orm-codegen 0.9.1 use sea_orm::entity::prelude::*; +use serde::Serialize; -#[derive(Clone, Debug, PartialEq, DeriveEntityModel)] +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Serialize)] #[sea_orm(table_name = "action_receipts")] pub struct Model { #[sea_orm(primary_key, column_type = "Text")] diff --git a/server/src/indexer/aggregated_circulating_supply.rs b/server/src/indexer/aggregated_circulating_supply.rs index f123260..d0389b8 100644 --- a/server/src/indexer/aggregated_circulating_supply.rs +++ b/server/src/indexer/aggregated_circulating_supply.rs @@ -1,8 +1,9 @@ -//! SeaORM Entity. Generated by sea-orm-codegen 0.8.0 +//! SeaORM Entity. Generated by sea-orm-codegen 0.9.1 use sea_orm::entity::prelude::*; +use serde::Serialize; -#[derive(Clone, Debug, PartialEq, DeriveEntityModel)] +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Serialize)] #[sea_orm(table_name = "aggregated__circulating_supply")] pub struct Model { #[sea_orm(column_type = "Decimal(Some((20, 0)))")] diff --git a/server/src/indexer/blocks.rs b/server/src/indexer/blocks.rs index 5d272c6..2318ef2 100644 --- a/server/src/indexer/blocks.rs +++ b/server/src/indexer/blocks.rs @@ -1,8 +1,9 @@ -//! SeaORM Entity. Generated by sea-orm-codegen 0.8.0 +//! SeaORM Entity. Generated by sea-orm-codegen 0.9.1 use sea_orm::entity::prelude::*; +use serde::Serialize; -#[derive(Clone, Debug, PartialEq, DeriveEntityModel)] +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Serialize)] #[sea_orm(table_name = "blocks")] pub struct Model { #[sea_orm(column_type = "Decimal(Some((20, 0)))")] diff --git a/server/src/indexer/chunks.rs b/server/src/indexer/chunks.rs index 7c00c9d..de78ee8 100644 --- a/server/src/indexer/chunks.rs +++ b/server/src/indexer/chunks.rs @@ -1,8 +1,9 @@ -//! SeaORM Entity. Generated by sea-orm-codegen 0.8.0 +//! SeaORM Entity. Generated by sea-orm-codegen 0.9.1 use sea_orm::entity::prelude::*; +use serde::Serialize; -#[derive(Clone, Debug, PartialEq, DeriveEntityModel)] +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Serialize)] #[sea_orm(table_name = "chunks")] pub struct Model { #[sea_orm(column_type = "Text")] diff --git a/server/src/indexer/data_receipts.rs b/server/src/indexer/data_receipts.rs index f981ffc..92897c5 100644 --- a/server/src/indexer/data_receipts.rs +++ b/server/src/indexer/data_receipts.rs @@ -1,8 +1,9 @@ -//! SeaORM Entity. Generated by sea-orm-codegen 0.8.0 +//! SeaORM Entity. Generated by sea-orm-codegen 0.9.1 use sea_orm::entity::prelude::*; +use serde::Serialize; -#[derive(Clone, Debug, PartialEq, DeriveEntityModel)] +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Serialize)] #[sea_orm(table_name = "data_receipts")] pub struct Model { #[sea_orm(primary_key, column_type = "Text")] diff --git a/server/src/indexer/execution_outcome_receipts.rs b/server/src/indexer/execution_outcome_receipts.rs index a976ff0..1c24fd8 100644 --- a/server/src/indexer/execution_outcome_receipts.rs +++ b/server/src/indexer/execution_outcome_receipts.rs @@ -1,8 +1,9 @@ -//! SeaORM Entity. Generated by sea-orm-codegen 0.8.0 +//! SeaORM Entity. Generated by sea-orm-codegen 0.9.1 use sea_orm::entity::prelude::*; +use serde::Serialize; -#[derive(Clone, Debug, PartialEq, DeriveEntityModel)] +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Serialize)] #[sea_orm(table_name = "execution_outcome_receipts")] pub struct Model { #[sea_orm(primary_key, column_type = "Text")] diff --git a/server/src/indexer/execution_outcomes.rs b/server/src/indexer/execution_outcomes.rs index 5c4c7e1..8291966 100644 --- a/server/src/indexer/execution_outcomes.rs +++ b/server/src/indexer/execution_outcomes.rs @@ -1,9 +1,10 @@ -//! SeaORM Entity. Generated by sea-orm-codegen 0.8.0 +//! SeaORM Entity. Generated by sea-orm-codegen 0.9.1 use super::sea_orm_active_enums::ExecutionOutcomeStatus; use sea_orm::entity::prelude::*; +use serde::Serialize; -#[derive(Clone, Debug, PartialEq, DeriveEntityModel)] +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Serialize)] #[sea_orm(table_name = "execution_outcomes")] pub struct Model { #[sea_orm(primary_key, column_type = "Text")] diff --git a/server/src/indexer/mod.rs b/server/src/indexer/mod.rs index 9318367..3bf3d67 100644 --- a/server/src/indexer/mod.rs +++ b/server/src/indexer/mod.rs @@ -1,4 +1,4 @@ -//! SeaORM Entity. Generated by sea-orm-codegen 0.8.0 +//! SeaORM Entity. Generated by sea-orm-codegen 0.9.1 pub mod prelude; diff --git a/server/src/indexer/prelude.rs b/server/src/indexer/prelude.rs index 848ca59..5e307ea 100644 --- a/server/src/indexer/prelude.rs +++ b/server/src/indexer/prelude.rs @@ -1,4 +1,4 @@ -//! SeaORM Entity. Generated by sea-orm-codegen 0.8.0 +//! SeaORM Entity. Generated by sea-orm-codegen 0.9.1 pub use super::access_keys::Entity as AccessKeys; pub use super::account_changes::Entity as AccountChanges; diff --git a/server/src/indexer/receipts.rs b/server/src/indexer/receipts.rs index 9279125..ce525e1 100644 --- a/server/src/indexer/receipts.rs +++ b/server/src/indexer/receipts.rs @@ -1,9 +1,10 @@ -//! SeaORM Entity. Generated by sea-orm-codegen 0.8.0 +//! SeaORM Entity. Generated by sea-orm-codegen 0.9.1 use super::sea_orm_active_enums::ReceiptKind; use sea_orm::entity::prelude::*; +use serde::Serialize; -#[derive(Clone, Debug, PartialEq, DeriveEntityModel)] +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Serialize)] #[sea_orm(table_name = "receipts")] pub struct Model { #[sea_orm(primary_key, column_type = "Text")] diff --git a/server/src/indexer/sea_orm_active_enums.rs b/server/src/indexer/sea_orm_active_enums.rs index 143a3b0..8ac1800 100644 --- a/server/src/indexer/sea_orm_active_enums.rs +++ b/server/src/indexer/sea_orm_active_enums.rs @@ -1,28 +1,9 @@ -//! SeaORM Entity. Generated by sea-orm-codegen 0.8.0 +//! SeaORM Entity. Generated by sea-orm-codegen 0.9.1 use sea_orm::entity::prelude::*; +use serde::Serialize; -#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum)] -#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "nft_event_kind")] -pub enum NftEventKind { - #[sea_orm(string_value = "BURN")] - Burn, - #[sea_orm(string_value = "MINT")] - Mint, - #[sea_orm(string_value = "TRANSFER")] - Transfer, -} -#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum)] -#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "ft_event_kind")] -pub enum FtEventKind { - #[sea_orm(string_value = "BURN")] - Burn, - #[sea_orm(string_value = "MINT")] - Mint, - #[sea_orm(string_value = "TRANSFER")] - Transfer, -} -#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum)] +#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum, Serialize)] #[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "receipt_kind")] pub enum ReceiptKind { #[sea_orm(string_value = "ACTION")] @@ -30,23 +11,7 @@ pub enum ReceiptKind { #[sea_orm(string_value = "DATA")] Data, } -#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum)] -#[sea_orm( - rs_type = "String", - db_type = "Enum", - enum_name = "execution_outcome_status" -)] -pub enum ExecutionOutcomeStatus { - #[sea_orm(string_value = "FAILURE")] - Failure, - #[sea_orm(string_value = "SUCCESS_RECEIPT_ID")] - SuccessReceiptId, - #[sea_orm(string_value = "SUCCESS_VALUE")] - SuccessValue, - #[sea_orm(string_value = "UNKNOWN")] - Unknown, -} -#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum)] +#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum, Serialize)] #[sea_orm( rs_type = "String", db_type = "Enum", @@ -58,7 +23,51 @@ pub enum AccessKeyPermissionKind { #[sea_orm(string_value = "FUNCTION_CALL")] FunctionCall, } -#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum)] +#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum, Serialize)] +#[sea_orm( + rs_type = "String", + db_type = "Enum", + enum_name = "state_change_reason_kind" +)] +pub enum StateChangeReasonKind { + #[sea_orm(string_value = "ACTION_RECEIPT_GAS_REWARD")] + ActionReceiptGasReward, + #[sea_orm(string_value = "ACTION_RECEIPT_PROCESSING_STARTED")] + ActionReceiptProcessingStarted, + #[sea_orm(string_value = "MIGRATION")] + Migration, + #[sea_orm(string_value = "POSTPONED_RECEIPT")] + PostponedReceipt, + #[sea_orm(string_value = "RECEIPT_PROCESSING")] + ReceiptProcessing, + #[sea_orm(string_value = "TRANSACTION_PROCESSING")] + TransactionProcessing, + #[sea_orm(string_value = "UPDATED_DELAYED_RECEIPTS")] + UpdatedDelayedReceipts, + #[sea_orm(string_value = "VALIDATOR_ACCOUNTS_UPDATE")] + ValidatorAccountsUpdate, +} +#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum, Serialize)] +#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "nft_event_kind")] +pub enum NftEventKind { + #[sea_orm(string_value = "BURN")] + Burn, + #[sea_orm(string_value = "MINT")] + Mint, + #[sea_orm(string_value = "TRANSFER")] + Transfer, +} +#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum, Serialize)] +#[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "ft_event_kind")] +pub enum FtEventKind { + #[sea_orm(string_value = "BURN")] + Burn, + #[sea_orm(string_value = "MINT")] + Mint, + #[sea_orm(string_value = "TRANSFER")] + Transfer, +} +#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum, Serialize)] #[sea_orm(rs_type = "String", db_type = "Enum", enum_name = "action_kind")] pub enum ActionKind { #[sea_orm(string_value = "ADD_KEY")] @@ -78,27 +87,19 @@ pub enum ActionKind { #[sea_orm(string_value = "TRANSFER")] Transfer, } -#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum)] +#[derive(Debug, Clone, PartialEq, EnumIter, DeriveActiveEnum, Serialize)] #[sea_orm( rs_type = "String", db_type = "Enum", - enum_name = "state_change_reason_kind" + enum_name = "execution_outcome_status" )] -pub enum StateChangeReasonKind { - #[sea_orm(string_value = "ACTION_RECEIPT_GAS_REWARD")] - ActionReceiptGasReward, - #[sea_orm(string_value = "ACTION_RECEIPT_PROCESSING_STARTED")] - ActionReceiptProcessingStarted, - #[sea_orm(string_value = "MIGRATION")] - Migration, - #[sea_orm(string_value = "POSTPONED_RECEIPT")] - PostponedReceipt, - #[sea_orm(string_value = "RECEIPT_PROCESSING")] - ReceiptProcessing, - #[sea_orm(string_value = "TRANSACTION_PROCESSING")] - TransactionProcessing, - #[sea_orm(string_value = "UPDATED_DELAYED_RECEIPTS")] - UpdatedDelayedReceipts, - #[sea_orm(string_value = "VALIDATOR_ACCOUNTS_UPDATE")] - ValidatorAccountsUpdate, +pub enum ExecutionOutcomeStatus { + #[sea_orm(string_value = "FAILURE")] + Failure, + #[sea_orm(string_value = "SUCCESS_RECEIPT_ID")] + SuccessReceiptId, + #[sea_orm(string_value = "SUCCESS_VALUE")] + SuccessValue, + #[sea_orm(string_value = "UNKNOWN")] + Unknown, } diff --git a/server/src/indexer/transaction_actions.rs b/server/src/indexer/transaction_actions.rs index a801899..656f76a 100644 --- a/server/src/indexer/transaction_actions.rs +++ b/server/src/indexer/transaction_actions.rs @@ -1,9 +1,10 @@ -//! SeaORM Entity. Generated by sea-orm-codegen 0.8.0 +//! SeaORM Entity. Generated by sea-orm-codegen 0.9.1 use super::sea_orm_active_enums::ActionKind; use sea_orm::entity::prelude::*; +use serde::Serialize; -#[derive(Clone, Debug, PartialEq, DeriveEntityModel)] +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Serialize)] #[sea_orm(table_name = "transaction_actions")] pub struct Model { #[sea_orm(primary_key, column_type = "Text")] diff --git a/server/src/indexer/transactions.rs b/server/src/indexer/transactions.rs index 05addc2..968b59a 100644 --- a/server/src/indexer/transactions.rs +++ b/server/src/indexer/transactions.rs @@ -1,9 +1,10 @@ -//! SeaORM Entity. Generated by sea-orm-codegen 0.8.0 +//! SeaORM Entity. Generated by sea-orm-codegen 0.9.1 use super::sea_orm_active_enums::ExecutionOutcomeStatus; use sea_orm::entity::prelude::*; +use serde::Serialize; -#[derive(Clone, Debug, PartialEq, DeriveEntityModel)] +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Serialize)] #[sea_orm(table_name = "transactions")] pub struct Model { #[sea_orm(primary_key, column_type = "Text")] diff --git a/server/src/main.rs b/server/src/main.rs index 0a64f7f..d786414 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -7,12 +7,16 @@ use dotenv::dotenv; use indexer::prelude::*; use local::prelude::*; use sea_orm::{ - ColumnTrait, Condition, Database, DatabaseConnection, EntityTrait, JoinType, QueryFilter, - QueryOrder, QuerySelect, FromQueryResult, + prelude::{Decimal, Uuid}, + sea_query::{Alias, Expr, IntoColumnRef, SimpleExpr}, + ColumnTrait, Condition, Database, DatabaseConnection, EntityTrait, FromQueryResult, JoinType, + QueryFilter, QueryOrder, QuerySelect, }; use serde::{Deserialize, Serialize}; use thiserror::Error; +use crate::indexer::sea_orm_active_enums::ReceiptKind; + mod indexer; mod local; @@ -93,6 +97,79 @@ struct TimestampRange { pub before_block_timestamp: Option, } +/// For some reason enums don't work in SeaORM selects unless they're explicitly cast to text. +/// +/// This took forever to figure out. +/// +/// References: +/// - https://github.com/SeaQL/sea-orm/discussions/843 +/// - https://github.com/SeaQL/sea-orm/blob/4301383b409737f85378a1da8a4c122478248ba0/src/query/select.rs#L124-L134 +/// - https://www.sea-ql.org/SeaORM/docs/advanced-query/custom-select/#select-custom-expressions +pub fn col_enum(column: T) -> SimpleExpr +where + T: IntoColumnRef, +{ + Expr::col(column).as_enum(Alias::new("text")) +} + +#[get("account/{account_id}/info")] +async fn account_info( + state: web::Data, + account_id: web::Path, +) -> Result { + use local::*; + + #[derive(FromQueryResult, Serialize, Debug)] + struct BadgeIdQuery { + pub badge_id: Uuid, + } + + let account_id = account_id.into_inner(); + let res = account_badge::Entity::find() + .column(account_badge::Column::BadgeId) + .filter(account_badge::Column::AccountId.eq(account_id)) + .into_model::() + .all(&*state.local_pool) + .await; + + if let Err(ref e) = res { + println!("{e:?}"); + } + + let res = res?.into_iter().map(|b| b.badge_id).collect::>(); + + Ok(web::Json(res)) +} + +#[get("account/{account_id}/badges")] +async fn account_badges( + state: web::Data, + account_id: web::Path, +) -> Result { + use local::*; + + #[derive(FromQueryResult, Serialize, Debug)] + struct BadgeIdQuery { + pub badge_id: Uuid, + } + + let account_id = account_id.into_inner(); + let res = account_badge::Entity::find() + .column(account_badge::Column::BadgeId) + .filter(account_badge::Column::AccountId.eq(account_id)) + .into_model::() + .all(&*state.local_pool) + .await; + + if let Err(ref e) = res { + println!("{e:?}"); + } + + let res = res?.into_iter().map(|b| b.badge_id).collect::>(); + + Ok(web::Json(res)) +} + #[get("account/{account_id}/actions")] async fn account_actions( state: web::Data, @@ -103,38 +180,36 @@ async fn account_actions( #[derive(FromQueryResult, Serialize, Debug)] struct AccountAction { - pub receipt_id: String, - pub index_in_action_receipt: i32, - pub transaction_hash: String, - pub action_kind: String, // TODO: Enum? - pub block_hash: String, - pub block_timestamp: u64, - pub predecessor_account_id: String, - pub receiver_account_id: String, + pub receipt_id: String, + pub index_in_action_receipt: i32, + pub transaction_hash: String, + pub action_kind: String, + pub block_hash: String, + pub block_timestamp: Decimal, + pub predecessor_account_id: String, + pub receiver_account_id: String, } let account_id = account_id.into_inner(); let range = web::Query::::from_query(req.query_string()).unwrap(); // Should always be safe because req.query_string() should never return an invalid query string - let condition = { - let mut condition = Condition::all() - .add(receipts::Column::ReceiptKind.eq("ACTION")) - .add( - Condition::any() - .add(receipts::Column::PredecessorAccountId.eq(account_id.clone())) - .add(receipts::Column::ReceiverAccountId.eq(account_id.clone())), - ); - - if let Some(after) = range.after_block_timestamp { - condition = condition.add(receipts::Column::IncludedInBlockTimestamp.gte(after)); - } - - if let Some(before) = range.before_block_timestamp { - condition = condition.add(receipts::Column::IncludedInBlockTimestamp.lt(before)); - } - - condition - }; + let condition = Condition::all() + .add(receipts::Column::ReceiptKind.eq(ReceiptKind::Action)) + .add( + Condition::any() + .add(receipts::Column::PredecessorAccountId.eq(account_id.clone())) + .add(receipts::Column::ReceiverAccountId.eq(account_id.clone())), + ) + .add_option( + range + .after_block_timestamp + .map(|after| receipts::Column::IncludedInBlockTimestamp.gte(after)), + ) + .add_option( + range + .before_block_timestamp + .map(|before| receipts::Column::IncludedInBlockTimestamp.lt(before)), + ); let res = Receipts::find() .column_as( @@ -147,7 +222,10 @@ async fn account_actions( "block_timestamp", ) .column(action_receipt_actions::Column::IndexInActionReceipt) - .column(action_receipt_actions::Column::ActionKind) + .column_as( + col_enum(action_receipt_actions::Column::ActionKind), + "action_kind", + ) .filter(condition) .join_rev( JoinType::LeftJoin, @@ -165,7 +243,7 @@ async fn account_actions( if let Err(ref e) = res { println!("{e:?}"); } - + let res = res?; Ok(web::Json(res)) @@ -208,6 +286,7 @@ async fn main() -> std::result::Result<(), Box> { .service(status) .service(account_score) .service(account_actions) + .service(account_badges) }) .bind(("127.0.0.1", config.port))? .run()