make things compile (with todo!)

indexer/Cargo.toml

@@ -38,3 +38,6 @@ serde_yaml = { version = "0.9.17" }
 tokio = { version = "1.25.0", features = ["full"] }
 tracing = { version = "0.1.37" }
 tracing-subscriber = { version = "0.3.16" }
+
+cryptoxide = "0.4.4"
+cbor_event = "2.4.0"

indexer/src/sinks/cardano.rs

@@ -4,6 +4,7 @@ use crate::sink::Sink;
 use crate::types::{MultiEraBlock, StoppableService};
 use crate::{genesis, DbConfig, SinkConfig};
 use async_trait::async_trait;
+use cml_core::serialization::FromBytes;
 use dcspark_blockchain_source::cardano::Point;
 use dcspark_core::{BlockId, SlotNumber};
 use entity::sea_orm::Database;
@@ -18,7 +19,6 @@ use entity::{
 };
 use std::sync::Arc;
 use std::sync::Mutex;
-use cml_core::serialization::FromBytes;
 use tasks::byron::byron_executor::process_byron_block;
 use tasks::dsl::database_task::BlockGlobalInfo;
 use tasks::execution_plan::ExecutionPlan;
@@ -283,15 +283,15 @@ async fn insert_block(
     let multi_block = MultiEraBlock::from_bytes(block_payload).unwrap();
 
     let block_global_info = BlockGlobalInfo {
-        era: to_era_value(multi_block),
+        era: to_era_value(&multi_block),
         epoch,
         epoch_slot,
     };
 
     perf_aggregator.block_parse += block_parse_counter.elapsed();
 
     match &multi_block {
-        MultiEraBlock::Byron(byron) => {
+        MultiEraBlock::Byron(_byron) => {
             process_byron_block(
                 txn,
                 (&cbor_hex, &multi_block, &block_global_info),

indexer/tasks/Cargo.toml

@@ -32,6 +32,9 @@ toml = { version = "0.5", features = ["preserve_order"] }
 urlencoding = { version = "2.1.2" }
 tracing-subscriber = { version = "0.3.16" }
 tracing = { version = "0.1.37" }
+serde_json = { version = "1.0.92" }
+
+cbor_event = "2.4.0"
 
 [features]
 default = ["build_rust_task"]

indexer/tasks/src/byron/byron_address.rs

@@ -1,3 +1,5 @@
+use cml_core::serialization::ToBytes;
+use cml_multi_era::{byron::block::ByronBlock, MultiEraBlock};
 use pallas::ledger::primitives::{
     byron::{self},
     Fragment,
@@ -19,10 +21,10 @@ carp_task! {
   should_add_task |block, _properties| {
     // recall: txs may have no outputs if they just burn all inputs as fee
     match block.1 {
-        MultiEraBlock::Byron(main_block) => {
+        MultiEraBlock::Byron(ByronBlock::Main(main_block)) => {
             main_block
                 .body
-                .tx_payload.iter().any(|payload| payload.transaction.outputs.len() > 0)
+                .tx_payload.iter().any(|payload| !payload.byron_tx.outputs.is_empty())
         }
         _ => false,
     }
@@ -43,12 +45,12 @@ async fn handle_addresses(
     byron_txs: &[TransactionModel],
 ) -> Result<BTreeMap<Vec<u8>, AddressInBlock>, DbErr> {
     match &block.1 {
-        MultiEraBlock::Byron(main_block) => {
+        MultiEraBlock::Byron(cml_multi_era::byron::block::ByronBlock::Main(main_block)) => {
             let tx_outputs: Vec<_> = main_block
                 .body
                 .tx_payload
                 .iter()
-                .map(|payload| &payload.transaction.outputs)
+                .map(|payload| &payload.byron_tx.outputs)
                 .zip(byron_txs)
                 .collect();
 
@@ -60,7 +62,7 @@ async fn handle_addresses(
             for (address, tx_id) in tx_outputs
                 .iter()
                 .flat_map(|pair| pair.0.iter().zip(std::iter::repeat(pair.1.id)))
-                .map(|(output, tx_id)| (output.address.encode_fragment().unwrap(), tx_id))
+                .map(|(output, tx_id)| (output.address.to_bytes(), tx_id))
             {
                 // we want to keep track of the first tx for each address
                 queued_address.entry(address).or_insert(tx_id);

indexer/tasks/src/byron/byron_block.rs

@@ -1,6 +1,12 @@
-use crate::config::PayloadAndReadonlyConfig::PayloadAndReadonlyConfig;
 use crate::dsl::task_macro::*;
 use crate::era_common::block_from_hash;
+use crate::{
+    config::PayloadAndReadonlyConfig::PayloadAndReadonlyConfig,
+    multiera_block_ext::MultieraBlockExt,
+};
+use cml_crypto::Serialize;
+use cml_multi_era::byron::block::ByronBlockHeader;
+use cryptoxide::hashing::blake2b_256;
 use entity::{block::EraValue, sea_orm::Set};
 use hex::ToHex;
 use pallas::ledger::primitives::{byron, Fragment};
@@ -47,7 +53,7 @@ async fn handle_block(
         era: Set(EraValue::Byron.into()),
         hash: Set(hash),
         height: Set(block.1.number() as i32),
-        epoch: Set(block.1.header().as_byron().unwrap().consensus_data.0.epoch as i32),
+        epoch: Set(block.1.epoch() as i32),
         slot: Set(block.1.slot() as i32),
         payload: Set(Some(block_payload)),
         ..Default::default()

indexer/tasks/src/byron/byron_inputs.rs

@@ -1,12 +1,16 @@
+use crate::dsl::task_macro::*;
+use cml_chain::transaction::TransactionInput;
+use cml_crypto::chain_crypto::Blake2b256;
+use cml_crypto::{RawBytesEncoding, TransactionHash};
 use cml_multi_era::byron::block::ByronBlock;
 use cml_multi_era::byron::transaction::{ByronTx, ByronTxIn};
 use cml_multi_era::MultiEraBlock;
-use crate::dsl::task_macro::*;
 use pallas::ledger::primitives::byron::{self, TxIn};
 
 use super::byron_outputs::ByronOutputTask;
 use crate::config::EmptyConfig::EmptyConfig;
 use crate::dsl::database_task::BlockGlobalInfo;
+use crate::multiera_block_ext::MultieraBlockExt;
 
 carp_task! {
   name ByronInputTask;
@@ -35,63 +39,33 @@ async fn handle_inputs(
     block: BlockInfo<'_, MultiEraBlock, BlockGlobalInfo>,
     byron_txs: &[TransactionModel],
 ) -> Result<Vec<TransactionInputModel>, DbErr> {
-    match block.1 {
-        MultiEraBlock::Byron(ByronBlock::Main(block)) => {
-            block
-                .body
-                .tx_payload
-                .iter()
-                .map(|tx| {
-                    tx
-                        .byron_tx
-                        .inputs
-                        .iter()
-                        .map(|input| {
-                            match input {
-                                ByronTxIn::ByronTxInRegular(regular) => {
-                                    (regular.index_1.byron_tx_id, regular.index_1.u32)
-                                }
-                                ByronTxIn::ByronTxInGenesis(genesis) => {
-                                    genesis.
-                                }
-                            }
-                        })
-                })
-        }
-        MultiEraBlock::Shelley(block) => {
-            block.transaction_bodies
-                .iter()
-                .flat_map(|tx| tx.inputs
-                    .iter()
-                    .map(|input|
-                        (input.transaction_id, input.index)
-                    )
-                )
-        }
-        MultiEraBlock::Allegra(_) => {}
-        MultiEraBlock::Mary(_) => {}
-        MultiEraBlock::Alonzo(_) => {}
-        MultiEraBlock::Babbage(_) => {}
-        MultiEraBlock::Conway(_) => {}
-    }
-    let flattened_inputs: Vec<(Vec<pallas::ledger::traverse::OutputRef>, i64)> = block
-        .1
-        .txs()
-        .iter()
-        .zip(byron_txs)
-        .map(|(tx, cardano_tx_in_db)| {
-            let inputs: Vec<pallas::ledger::traverse::OutputRef> =
-                tx.inputs().iter().map(|x| x.output_ref()).collect();
+    let txs = match block.1 {
+        MultiEraBlock::Byron(ByronBlock::Main(block)) => block.body.tx_payload.iter().map(|tx| {
+            tx.byron_tx.inputs.iter().filter_map(|input| match input {
+                ByronTxIn::ByronTxInRegular(regular) => Some(TransactionInput::new(
+                    // TODO: is it fine to use this for byron? Otherwise we need another abstraction
+                    TransactionHash::from_raw_bytes(regular.index_1.byron_tx_id.to_raw_bytes())
+                        .unwrap(),
+                    regular.index_1.u32 as u64,
+                )),
+                ByronTxIn::ByronTxInGenesis(_) => None,
+            })
+        }),
+        _ => todo!("non byron block?"),
+    };
 
-            (inputs, cardano_tx_in_db.id)
-        })
+    let flattened_inputs: Vec<(Vec<_>, i64)> = txs
+        .zip(byron_txs)
+        .map(|(inputs, cardano_tx_in_db)| (inputs.collect::<Vec<_>>(), cardano_tx_in_db.id))
         .collect();
 
     let outputs_for_inputs =
-        crate::era_common::get_outputs_for_inputs(&flattened_inputs, db_tx).await?;
+        crate::era_common::get_outputs_for_inputs(&flattened_inputs[..], db_tx).await?;
 
     let input_to_output_map = crate::era_common::gen_input_to_output_map(&outputs_for_inputs);
     let result =
-        crate::era_common::insert_inputs(&flattened_inputs, &input_to_output_map, db_tx).await?;
+        crate::era_common::insert_inputs(&flattened_inputs[..], &input_to_output_map, db_tx)
+            .await?;
+
     Ok(result)
 }

indexer/tasks/src/byron/byron_outputs.rs

@@ -1,7 +1,11 @@
 use super::byron_address::ByronAddressTask;
 use crate::config::EmptyConfig::EmptyConfig;
 use crate::dsl::database_task::BlockGlobalInfo;
+use crate::multiera_block_ext::MultieraBlockExt;
 use crate::{dsl::task_macro::*, era_common::get_truncated_address};
+use cbor_event::cbor;
+use cml_chain::byron::ByronTxOut;
+use cml_multi_era::byron::block::ByronBlock;
 use entity::sea_orm::Set;
 use pallas::ledger::traverse::MultiEraOutput;
 use pallas::{
@@ -22,7 +26,7 @@ carp_task! {
   write [byron_outputs];
   should_add_task |block, _properties| {
     // recall: txs may have no outputs if they just burn all inputs as fee
-    block.1.txs().iter().any(|tx| tx.outputs().len() > 0)
+    block.1.txs().iter().any(|tx| !tx.outputs().is_empty())
   };
   execute |previous_data, task| handle_outputs(
       task.db_tx,
@@ -41,14 +45,16 @@ async fn handle_outputs(
     byron_txs: &[TransactionModel],
     byron_addresses: &BTreeMap<Vec<u8>, AddressInBlock>,
 ) -> Result<Vec<TransactionOutputModel>, DbErr> {
-    let tx_outputs: Vec<_> = block
-        .1
-        .as_byron()
-        .unwrap()
+    let block_byron = match block.1 {
+        cml_multi_era::MultiEraBlock::Byron(ByronBlock::Main(block_byron)) => block_byron,
+        _ => todo!("non byron block"),
+    };
+
+    let tx_outputs: Vec<_> = block_byron
         .body
         .tx_payload
         .iter()
-        .map(|payload| &payload.transaction.outputs)
+        .map(|payload| &payload.byron_tx.outputs)
         .zip(byron_txs)
         .collect();
 
@@ -57,25 +63,23 @@ async fn handle_outputs(
     }
 
     // note: outputs have to be added before inputs
-    insert_byron_outputs(db_tx, byron_addresses, &tx_outputs).await
+    insert_byron_outputs(db_tx, byron_addresses, &tx_outputs[..]).await
 }
 
 async fn insert_byron_outputs(
     txn: &DatabaseTransaction,
     address_map: &BTreeMap<Vec<u8>, AddressInBlock>,
-    outputs: &[(&MaybeIndefArray<TxOut>, &TransactionModel)],
+    outputs: &[(&Vec<ByronTxOut>, &TransactionModel)],
 ) -> Result<Vec<TransactionOutputModel>, DbErr> {
     let result = TransactionOutput::insert_many(
         outputs
             .iter()
             .flat_map(|pair| pair.0.iter().enumerate().zip(std::iter::repeat(pair.1)))
             .map(
                 |((output_index, output), tx_id)| TransactionOutputActiveModel {
-                    payload: Set(output.encode_fragment().unwrap()),
+                    payload: Set(cbor!(output).unwrap()),
                     address_id: Set(address_map
-                        .get(get_truncated_address(
-                            &output.address.encode_fragment().unwrap(),
-                        ))
+                        .get(get_truncated_address(&cbor!(output.address).unwrap()))
                         .unwrap()
                         .model
                         .id),

indexer/tasks/src/byron/byron_txs.rs

@@ -1,3 +1,4 @@
+use crate::multiera_block_ext::MultieraBlockExt;
 use crate::{dsl::task_macro::*, era_common::transactions_from_hashes, utils::blake2b256};
 use entity::sea_orm::Set;
 use pallas::ledger::primitives::{byron, Fragment};
@@ -20,7 +21,7 @@ carp_task! {
   execute |previous_data, task| handle_tx(
       task.db_tx,
       task.block,
-      &previous_data.byron_block.as_ref().unwrap(),
+      previous_data.byron_block.as_ref().unwrap(),
       task.config.readonly,
       task.config.include_payload
   );
@@ -53,7 +54,11 @@ async fn handle_tx(
 
     let transaction_inserts =
         Transaction::insert_many(block.1.txs().iter().enumerate().map(|(idx, tx)| {
-            let tx_payload = if include_payload { tx.encode() } else { vec![] };
+            let tx_payload = if include_payload {
+                tx.to_bytes()
+            } else {
+                vec![]
+            };
 
             TransactionActiveModel {
                 hash: Set(tx.hash().to_vec()),