transaction history: add input addresses, metadata and slots filter (#…

…173) * transaction history: add input addresses, metadata and slots filter * remove outputs field from the response moved to the client * update .nvmrc * make input addresses and metadata optional (as an arg) * reformat slotBoundsPagination.sql Co-authored-by: Sebastien Guillemot <[email protected]> * add more comments in the code * update slotBOundsPagination.queries.ts after reformatting * update .nvmrc to lts/hydrogen * optimize slot filter tx filter query * add support to lower slot limit as -1 to get the full range * update openapi.json * update tx_count change to cml multiera after rebase * fix genesis_block task: missing tx_count * add comment explaining the <= :low condition --------- Co-authored-by: Sebastien Guillemot <[email protected]>
dcSpark · Mar 12, 2024 · e5c5ab1 · e5c5ab1
1 parent 577b658
commit e5c5ab1
Show file tree

Hide file tree

Showing 17 changed files with 531 additions and 68 deletions.
diff --git a/docs/bin/openapi.json b/docs/bin/openapi.json
@@ -1111,6 +1111,16 @@
             },
             "TransactionInfo": {
                 "properties": {
+                    "inputCredentials": {
+                        "items": {
+                            "type": "string"
+                        },
+                        "type": "array"
+                    },
+                    "metadata": {
+                        "type": "string",
+                        "nullable": true
+                    },
                     "payload": {
                         "type": "string",
                         "description": "cbor-encoded transaction",
@@ -1166,10 +1176,35 @@
                 "description": "Filter which uses of the address are considered relevant for the query.\n\nThis is a bitmask, so you can combine multiple options\nex: `RelationFilterType.Input & RelationFilterType.Output`\n\nNote: relations only apply to credentials and not to full bech32 addresses",
                 "pattern": "([01]?[0-9]?[0-9]|2[0-4][0-9]|25[0-5])"
             },
+            "SlotLimits": {
+                "properties": {
+                    "to": {
+                        "type": "number",
+                        "format": "double"
+                    },
+                    "from": {
+                        "type": "number",
+                        "format": "double"
+                    }
+                },
+                "required": [
+                    "to",
+                    "from"
+                ],
+                "type": "object"
+            },
             "TransactionHistoryRequest": {
                 "allOf": [
                     {
                         "properties": {
+                            "withInputContext": {
+                                "type": "boolean",
+                                "description": "If this is set to true, the result includes the input addresses (which are\nnot part of the tx), and the metadata (if any)"
+                            },
+                            "slotLimits": {
+                                "$ref": "#/components/schemas/SlotLimits",
+                                "description": "This limits the transactions in the result to this range of slots.\nEverything else is filtered out"
+                            },
                             "limit": {
                                 "type": "number",
                                 "format": "double",

diff --git a/indexer/entity/src/block.rs b/indexer/entity/src/block.rs
@@ -12,6 +12,7 @@ pub struct Model {
     pub epoch: i32,
     pub slot: i32,
     pub payload: Option<Vec<u8>>,
+    pub tx_count: i32,
 }
 
 #[derive(Copy, Clone, Debug, DeriveRelation, EnumIter)]

diff --git a/indexer/migration/src/lib.rs b/indexer/migration/src/lib.rs
@@ -20,6 +20,7 @@ mod m20230223_000015_modify_block_table;
 mod m20230927_000016_create_stake_delegation_table;
 mod m20231025_000017_projected_nft;
 mod m20231220_000018_asset_utxo_table;
+mod m20240229_000019_add_block_tx_count_column;
 
 pub struct Migrator;
 
@@ -47,6 +48,7 @@ impl MigratorTrait for Migrator {
             Box::new(m20230927_000016_create_stake_delegation_table::Migration),
             Box::new(m20231025_000017_projected_nft::Migration),
             Box::new(m20231220_000018_asset_utxo_table::Migration),
+            Box::new(m20240229_000019_add_block_tx_count_column::Migration),
         ]
     }
 }
diff --git a/indexer/migration/src/m20240229_000019_add_block_tx_count_column.rs b/indexer/migration/src/m20240229_000019_add_block_tx_count_column.rs
@@ -0,0 +1,36 @@
+use sea_schema::migration::prelude::*;
+
+use entity::block::*;
+
+pub struct Migration;
+
+impl MigrationName for Migration {
+    fn name(&self) -> &str {
+        "m20240229_000019_add_block_tx_count_column"
+    }
+}
+
+#[async_trait::async_trait]
+impl MigrationTrait for Migration {
+    async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        manager
+            .alter_table(
+                Table::alter()
+                    .table(Entity)
+                    .add_column(ColumnDef::new(Column::TxCount).integer())
+                    .to_owned(),
+            )
+            .await
+    }
+
+    async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        manager
+            .alter_table(
+                Table::alter()
+                    .table(Entity)
+                    .drop_column(Column::TxCount)
+                    .to_owned(),
+            )
+            .await
+    }
+}
diff --git a/indexer/tasks/src/genesis/genesis_block.rs b/indexer/tasks/src/genesis/genesis_block.rs
@@ -47,6 +47,7 @@ async fn handle_block(
         epoch: Set(0),
         slot: Set(0),
         payload: Set(Some(block_payload)),
+        tx_count: Set(0),
         ..Default::default()
     };
 

diff --git a/indexer/tasks/src/multiera/multiera_block.rs b/indexer/tasks/src/multiera/multiera_block.rs
@@ -49,7 +49,25 @@ async fn handle_block(
         epoch: Set(block.2.epoch.unwrap() as i32),
         slot: Set(block.1.header().slot() as i32),
         payload: Set(Some(block_payload)),
+        tx_count: Set(block_tx_count(block.1) as i32),
         ..Default::default()
     };
     block.insert(db_tx).await
 }
+
+fn block_tx_count(block: &cml_multi_era::MultiEraBlock) -> usize {
+    match block {
+        cml_multi_era::MultiEraBlock::Byron(
+            cml_multi_era::byron::block::ByronBlock::EpochBoundary(_),
+        ) => 0,
+        cml_multi_era::MultiEraBlock::Byron(cml_multi_era::byron::block::ByronBlock::Main(
+            block,
+        )) => block.body.tx_payload.len(),
+        cml_multi_era::MultiEraBlock::Shelley(block) => block.transaction_bodies.len(),
+        cml_multi_era::MultiEraBlock::Allegra(block) => block.transaction_bodies.len(),
+        cml_multi_era::MultiEraBlock::Mary(block) => block.transaction_bodies.len(),
+        cml_multi_era::MultiEraBlock::Alonzo(block) => block.transaction_bodies.len(),
+        cml_multi_era::MultiEraBlock::Babbage(block) => block.transaction_bodies.len(),
+        cml_multi_era::MultiEraBlock::Conway(block) => block.transaction_bodies.len(),
+    }
+}
diff --git a/webserver/server/.nvmrc b/webserver/server/.nvmrc
@@ -1 +1 @@
-v14.17.0
+lts/hydrogen
diff --git a/webserver/server/app/controllers/TransactionHistoryController.ts b/webserver/server/app/controllers/TransactionHistoryController.ts
@@ -15,6 +15,7 @@ import { Routes } from '../../../shared/routes';
 import sortBy from 'lodash/sortBy';
 import { getAddressTypes } from '../models/utils';
 import { RelationFilterType } from '../../../shared/models/common';
+import { slotBoundsPagination } from '../models/pagination/slotBoundsPagination.queries';
 
 const route = Routes.transactionHistory;
 
@@ -32,7 +33,9 @@ export class TransactionHistoryController extends Controller {
     requestBody: EndpointTypes[typeof route]['input'],
     @Res()
     errorResponse: TsoaResponse<
-      StatusCodes.CONFLICT | StatusCodes.BAD_REQUEST | StatusCodes.UNPROCESSABLE_ENTITY,
+      | StatusCodes.CONFLICT
+      | StatusCodes.BAD_REQUEST
+      | StatusCodes.UNPROCESSABLE_ENTITY,
       ErrorShape
     >
   ): Promise<EndpointTypes[typeof route]['response']> {
@@ -62,7 +65,7 @@ export class TransactionHistoryController extends Controller {
     const cardanoTxs = await tx<
       ErrorShape | [TransactionHistoryResponse, TransactionHistoryResponse]
     >(pool, async dbTx => {
-      const [until, pageStart] = await Promise.all([
+      const [until, pageStart, slotBounds] = await Promise.all([
         resolveUntilTransaction({
           block_hash: Buffer.from(requestBody.untilBlock, 'hex'),
           dbTx,
@@ -74,6 +77,12 @@ export class TransactionHistoryController extends Controller {
               after_tx: Buffer.from(requestBody.after.tx, 'hex'),
               dbTx,
             }),
+        !requestBody.slotLimits
+          ? Promise.resolve(undefined)
+          : slotBoundsPagination.run(
+              { low: requestBody.slotLimits.from, high: requestBody.slotLimits.to },
+              dbTx
+            ),
       ]);
       if (until == null) {
         return genErrorMessage(Errors.BlockHashNotFound, {
@@ -87,11 +96,38 @@ export class TransactionHistoryController extends Controller {
         });
       }
 
+      let pageStartWithSlot = pageStart;
+
+      // if the slotLimits field is set, this shrinks the tx id range
+      // accordingly if necessary.
+      if (requestBody.slotLimits) {
+        const bounds = slotBounds ? slotBounds[0] : { min_tx_id: -1, max_tx_id: -2 };
+
+        const minTxId = Number(bounds.min_tx_id);
+
+        if (!pageStartWithSlot) {
+          pageStartWithSlot = {
+            // block_id is not really used by this query.
+            block_id: -1,
+            // if no *after* argument is provided, this starts the pagination
+            // from the corresponding slot. This allows skipping slots you are
+            // not interested in. If there is also no slotLimits specified this
+            // starts from the first tx because of the default of -1.
+            tx_id: minTxId,
+          };
+        } else {
+          pageStartWithSlot.tx_id = Math.max(Number(bounds.min_tx_id), pageStartWithSlot.tx_id);
+        }
+
+        until.tx_id = Math.min(until.tx_id, Number(bounds.max_tx_id));
+      }
+
       const commonRequest = {
-        after: pageStart,
+        after: pageStartWithSlot,
         limit: requestBody.limit ?? ADDRESS_LIMIT.RESPONSE,
         until,
         dbTx,
+        withInputContext: !!requestBody.withInputContext
       };
       const result = await Promise.all([
         historyForCredentials({

diff --git a/webserver/server/app/models/block/sqlBlockLatest.queries.ts b/webserver/server/app/models/block/sqlBlockLatest.queries.ts
@@ -17,6 +17,7 @@ export interface ISqlBlockLatestResult {
   id: number;
   payload: Buffer | null;
   slot: number;
+  tx_count: number | null;
 }
 
 /** 'SqlBlockLatest' query type */

diff --git a/webserver/server/app/models/pagination/slotBoundsPagination.queries.ts b/webserver/server/app/models/pagination/slotBoundsPagination.queries.ts
@@ -0,0 +1,87 @@
+/** Types generated for queries found in "app/models/pagination/slotBoundsPagination.sql" */
+import { PreparedQuery } from '@pgtyped/runtime';
+
+/** 'SlotBoundsPagination' parameters type */
+export interface ISlotBoundsPaginationParams {
+  high: number;
+  low: number;
+}
+
+/** 'SlotBoundsPagination' return type */
+export interface ISlotBoundsPaginationResult {
+  max_tx_id: string | null;
+  min_tx_id: string | null;
+}
+
+/** 'SlotBoundsPagination' query type */
+export interface ISlotBoundsPaginationQuery {
+  params: ISlotBoundsPaginationParams;
+  result: ISlotBoundsPaginationResult;
+}
+
+const slotBoundsPaginationIR: any = {"usedParamSet":{"low":true,"high":true},"params":[{"name":"low","required":true,"transform":{"type":"scalar"},"locs":[{"a":782,"b":786}]},{"name":"high","required":true,"transform":{"type":"scalar"},"locs":[{"a":1036,"b":1041}]}],"statement":"WITH\n    low_block AS (\n        SELECT\n            \"Block\".id,\n            \"Block\".slot\n        FROM\n            \"Block\"\n        WHERE\n                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  \n            slot <= :low! AND tx_count > 0\n        ORDER BY\n            \"Block\".id DESC\n        LIMIT\n            1\n    ),\n    high_block AS (\n        SELECT\n            \"Block\".id,\n            \"Block\".slot\n        FROM\n            \"Block\"\n        WHERE\n            slot <= :high! AND tx_count > 0\n        ORDER BY\n            \"Block\".id DESC\n        LIMIT 1\n    ),\n    min_hash AS (\n        (SELECT\n            COALESCE(MAX(\"Transaction\".id), -1) AS min_tx_id\n        FROM\n            \"Transaction\"\n            JOIN low_block ON \"Transaction\".block_id = low_block.id\n        GROUP BY\n            low_block.slot\n        LIMIT\n            1\n        )\n        UNION (SELECT min_tx_id FROM (values(-1)) s(min_tx_id))\n        ORDER BY min_tx_id DESC\n        LIMIT\n            1\n    ),\n    max_hash AS (\n        SELECT\n            COALESCE(MAX(\"Transaction\".id), -2) AS max_tx_id\n        FROM\n            \"Transaction\"\n            JOIN high_block ON \"Transaction\".block_id = high_block.id\n        GROUP BY\n            high_block.slot\n    )\nSELECT\n    *\nFROM min_hash\nLEFT JOIN max_hash ON 1 = 1"};
+
+/**
+ * Query generated from SQL:
+ * ```
+ * WITH
+ *     low_block AS (
+ *         SELECT
+ *             "Block".id,
+ *             "Block".slot
+ *         FROM
+ *             "Block"
+ *         WHERE
+ *                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   
+ *             slot <= :low! AND tx_count > 0
+ *         ORDER BY
+ *             "Block".id DESC
+ *         LIMIT
+ *             1
+ *     ),
+ *     high_block AS (
+ *         SELECT
+ *             "Block".id,
+ *             "Block".slot
+ *         FROM
+ *             "Block"
+ *         WHERE
+ *             slot <= :high! AND tx_count > 0
+ *         ORDER BY
+ *             "Block".id DESC
+ *         LIMIT 1
+ *     ),
+ *     min_hash AS (
+ *         (SELECT
+ *             COALESCE(MAX("Transaction".id), -1) AS min_tx_id
+ *         FROM
+ *             "Transaction"
+ *             JOIN low_block ON "Transaction".block_id = low_block.id
+ *         GROUP BY
+ *             low_block.slot
+ *         LIMIT
+ *             1
+ *         )
+ *         UNION (SELECT min_tx_id FROM (values(-1)) s(min_tx_id))
+ *         ORDER BY min_tx_id DESC
+ *         LIMIT
+ *             1
+ *     ),
+ *     max_hash AS (
+ *         SELECT
+ *             COALESCE(MAX("Transaction".id), -2) AS max_tx_id
+ *         FROM
+ *             "Transaction"
+ *             JOIN high_block ON "Transaction".block_id = high_block.id
+ *         GROUP BY
+ *             high_block.slot
+ *     )
+ * SELECT
+ *     *
+ * FROM min_hash
+ * LEFT JOIN max_hash ON 1 = 1
+ * ```
+ */
+export const slotBoundsPagination = new PreparedQuery<ISlotBoundsPaginationParams,ISlotBoundsPaginationResult>(slotBoundsPaginationIR);
+
+