WIP endorsement_with_dal

build/dipdup.testnet.yml

@@ -97,6 +97,7 @@ mempool:
       filters:
         kinds:
           - endorsement
+          - endorsement_with_dal
           - transaction
           - activate_account
           - ballot
@@ -132,6 +133,7 @@ mempool:
           - smart_rollup_recover_bond
           - smart_rollup_timeout
           - smart_rollup_cement
+          - dal_publish_commitment
       datasources:
         tzkt: parisnet_tzkt
         rpc: parisnet_rpc

cmd/mempool/handlers.go

@@ -221,6 +221,8 @@ func (indexer *Indexer) handleContent(ctx context.Context, tx bun.IDB, content n
 		return indexer.handleEndorsement(ctx, tx, content, operation)
 	case node.KindEndorsementWithSlot:
 		return indexer.handleEndorsementWithSlot(ctx, tx, content, operation)
+	case node.KindEndorsementWithDal:
+		return indexer.handleEndorsement(ctx, tx, content, operation)
 	case node.KindNonceRevelation:
 		var model models.NonceRevelation
 		return defaultHandler(ctx, tx, content, operation, &model)
@@ -306,6 +308,9 @@ func (indexer *Indexer) handleContent(ctx context.Context, tx bun.IDB, content n
 	case node.KindSrTimeout:
 		var model models.SmartRollupTimeout
 		return defaultHandler(ctx, tx, content, operation, &model)
+	case node.KindDalPublishCommitment:
+		var model models.DalPublishCommitment
+		return defaultHandler(ctx, tx, content, operation, &model)
 	case node.KindEvent:
 	default:
 		indexer.warn().Str("kind", content.Kind).Msg("unknown operation kind")
@@ -332,6 +337,20 @@ func (indexer *Indexer) handleEndorsement(ctx context.Context, tx bun.IDB, conte
 	return nil
 }
 
+func (indexer *Indexer) handleEndorsementWithDal(ctx context.Context, tx bun.IDB, content node.Content, operation models.MempoolOperation) error {
+	var endorsement models.Endorsement
+	if err := json.Unmarshal(content.Body, &endorsement); err != nil {
+		return err
+	}
+	endorsement.MempoolOperation = operation
+
+	if err := createModel(ctx, tx, &endorsement); err != nil {
+		return err
+	}
+	indexer.endorsements <- &endorsement
+	return nil
+}
+
 func (indexer *Indexer) handleEndorsementWithSlot(ctx context.Context, tx bun.IDB, content node.Content, operation models.MempoolOperation) error {
 	var endorsementWithSlot node.EndorsementWithSlot
 	if err := json.Unmarshal(content.Body, &endorsementWithSlot); err != nil {

cmd/mempool/models/dal_publish_commitment.go

@@ -0,0 +1,27 @@
+package models
+
+import "github.com/uptrace/bun"
+
+// DalPublishCommitment -
+type DalPublishCommitment struct {
+	bun.BaseModel `bun:"table:dal_publish_commitment"`
+
+	MempoolOperation
+	Fee          int64      `comment:"Fee to the baker, produced block, in which the operation was included (micro tez)." json:"fee,string"`
+	Counter      int64      `bun:",pk"                                                                                    comment:"An account nonce which is used to prevent operation replay." json:"counter,string"`
+	GasLimit     int64      `comment:"A cap on the amount of gas a given operation can consume."                          json:"gas_limit,string"`
+	StorageLimit int64      `comment:"A cap on the amount of storage a given operation can consume."                      json:"storage_limit,string"`
+	Source       string     `comment:"Address of the account who has sent the operation."                                 index:"sr_refute_source_idx"                                          json:"source,omitempty"`
+	SlotHeader   SlotHeader `comment:"Published slot header"                                                              json:"slot_header"`
+}
+
+
+type SlotHeader struct {
+	SlotIndex       int    `json:"slot_index"`
+	Commitment      string `json:"commitment"`
+	CommitmentProof string `json:"commitment_proof"`
+}
+// SetMempoolOperation -
+func (i *DalPublishCommitment) SetMempoolOperation(operaiton MempoolOperation) {
+	i.MempoolOperation = operaiton
+}

cmd/mempool/models/mempool_operation.go

@@ -22,12 +22,12 @@ const (
 
 // DefaultConstraint -
 type DefaultConstraint interface {
-	Ballot | ActivateAccount | Delegation | DoubleBaking | DoubleEndorsing | DoublePreendorsing | Endorsement |
+	Ballot | ActivateAccount | Delegation | DoubleBaking | DoubleEndorsing | DoublePreendorsing | Endorsement | 
 		NonceRevelation | Origination | Preendorsement | Proposal | RegisterGlobalConstant | Reveal | SetDepositsLimit |
 		Transaction | TransferTicket | TxRollupCommit | TxRollupDispatchTickets | TxRollupFinalizeCommitment | TxRollupOrigination |
 		TxRollupRejection | TxRollupRemoveCommitment | TxRollupReturnBond | TxRollupSubmitBatch | VdfRevelation | IncreasePaidStorage |
 		UpdateConsensusKey | DelegateDrain | SmartRollupAddMessage | SmartRollupCement | SmartRollupExecute |
-		SmartRollupOriginate | SmartRollupPublish | SmartRollupRecoverBond | SmartRollupRefute | SmartRollupTimeout
+		SmartRollupOriginate | SmartRollupPublish | SmartRollupRecoverBond | SmartRollupRefute | SmartRollupTimeout | DalPublishCommitment
 }
 
 // ChangableMempoolOperation -
@@ -268,6 +268,9 @@ func getModelByKind(kind string) (interface{}, error) {
 		return &SmartRollupRefute{}, nil
 	case node.KindSrTimeout:
 		return &SmartRollupTimeout{}, nil
+	case node.KindDalPublishCommitment:
+		return &DalPublishCommitment{}, nil
+
 
 	default:
 		return nil, errors.Wrap(node.ErrUnknownKind, kind)

cmd/mempool/tzkt/data.go

@@ -16,6 +16,7 @@ var toTzKTKinds = map[string]string{
 	node.KindDoubleBaking:               data.KindDoubleBaking,
 	node.KindDoubleEndorsing:            data.KindDoubleEndorsing,
 	node.KindEndorsement:                data.KindEndorsement,
+	node.KindEndorsementWithDal:         data.KindEndorsement,
 	node.KindNonceRevelation:            data.KindNonceRevelation,
 	node.KindOrigination:                data.KindOrigination,
 	node.KindProposal:                   data.KindProposal,
@@ -44,6 +45,7 @@ var toTzKTKinds = map[string]string{
 	node.KindSrRecoverBond:              data.KindSrRecoverBond,
 	node.KindSrRefute:                   data.KindSrRefute,
 	node.KindSrTimeout:                  data.KindSrRefute,
+	node.KindDalPublishCommitment:       data.KindDalPublishCommitment,
 }
 
 // OperationMessage -

cmd/mempool/tzkt/processors.go

@@ -341,3 +341,12 @@ func operationFromSrRefute(model data.SmartRollupRefute) data.Operation {
 		Hash:  model.Hash,
 	}
 }
+
+func operationFromDalPublishCommitment(model data.DalPublishCommitment) data.Operation {
+	return data.Operation{
+		Type:  node.KindDalPublishCommitment,
+		Level: model.Level,
+		ID:    model.ID,
+		Hash:  model.Hash,
+	}
+}

cmd/mempool/tzkt/tzkt.go

@@ -268,6 +268,9 @@ func (tzkt *TzKT) getAPIOperation(model interface{}) (data.Operation, error) {
 	case *data.SmartRollupRefute:
 		return operationFromSrRefute(*operation), nil
 
+	case *data.DalPublishCommitment:
+		return operationFromDalPublishCommitment(*operation), nil
+
 	default:
 		return data.Operation{}, errors.Wrapf(ErrInvalidOperationType, "%T", model)
 	}
@@ -591,6 +594,8 @@ func (tzkt *TzKT) getTableData(ctx context.Context, table *tableState, indexerSt
 		return getOperations(ctx, table, filters, tzkt.api.GetSmartRollupRecoverBond, operationFromSrRecoverBond)
 	case data.KindSrRefute:
 		return getOperations(ctx, table, filters, tzkt.api.GetSmartRollupRefute, operationFromSrRefute)
+	case data.KindDalPublishCommitment:
+		return getOperations(ctx, table, filters, tzkt.api.GetDalPublishCommitment, operationFromDalPublishCommitment)
 	default:
 		return errors.Wrap(ErrUnknownOperationKind, table.Table)
 	}

go.mod

@@ -5,17 +5,17 @@ go 1.22
 require (
 	github.com/btcsuite/btcutil v1.0.2
 	github.com/dipdup-io/workerpool v0.0.4
-	github.com/dipdup-net/go-lib v0.3.6
+	github.com/dipdup-net/go-lib v0.4.2
 	github.com/grafana/pyroscope-go v1.1.1
 	github.com/json-iterator/go v1.1.12
 	github.com/karlseguin/ccache v2.0.3+incompatible
 	github.com/lib/pq v1.10.9
 	github.com/pkg/errors v0.9.1
-	github.com/rs/zerolog v1.30.0
+	github.com/rs/zerolog v1.31.0
 	github.com/spf13/cobra v1.6.1
 	github.com/ubiq/go-ubiq v3.0.1+incompatible
-	github.com/uptrace/bun v1.1.14
-	golang.org/x/crypto v0.17.0
+	github.com/uptrace/bun v1.1.17
+	golang.org/x/crypto v0.21.0
 )
 
 require (
@@ -34,32 +34,23 @@ require (
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/ebellocchia/go-base58 v0.1.0 // indirect
-	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
-	github.com/go-pg/pg/v10 v10.10.6 // indirect
-	github.com/go-pg/zerochecker v0.2.0 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
-	github.com/go-playground/validator/v10 v10.15.0 // indirect
-	github.com/go-sql-driver/mysql v1.7.1 // indirect
+	github.com/go-playground/validator/v10 v10.18.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/grafana/pyroscope-go/godeltaprof v0.1.6 // indirect
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
-	github.com/jackc/pgpassfile v1.0.0 // indirect
-	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
-	github.com/jackc/pgx/v5 v5.5.4 // indirect
-	github.com/jackc/puddle/v2 v2.2.1 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
-	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/karlseguin/expect v1.0.8 // indirect
 	github.com/klauspost/compress v1.17.3 // indirect
-	github.com/leodido/go-urn v1.2.4 // indirect
+	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.19 // indirect
-	github.com/mattn/go-sqlite3 v1.14.19 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/moby/patternmatcher v0.5.0 // indirect
 	github.com/moby/sys/sequential v0.5.0 // indirect
@@ -84,25 +75,17 @@ require (
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.1 // indirect
 	github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect
-	github.com/uptrace/bun/dialect/pgdialect v1.1.14 // indirect
-	github.com/vmihailenco/bufpool v0.1.11 // indirect
-	github.com/vmihailenco/msgpack/v5 v5.4.0 // indirect
-	github.com/vmihailenco/tagparser v0.1.2 // indirect
+	github.com/uptrace/bun/dialect/pgdialect v1.1.17 // indirect
+	github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
 	golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea // indirect
 	golang.org/x/mod v0.11.0 // indirect
-	golang.org/x/net v0.17.0 // indirect
-	golang.org/x/sync v0.3.0 // indirect
-	golang.org/x/sys v0.15.0 // indirect
+	golang.org/x/net v0.23.0 // indirect
+	golang.org/x/sys v0.18.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/tools v0.10.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect
 	google.golang.org/grpc v1.58.3 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	gorm.io/driver/mysql v1.5.1 // indirect
-	gorm.io/driver/postgres v1.5.2 // indirect
-	gorm.io/driver/sqlite v1.5.2 // indirect
-	gorm.io/gorm v1.25.3 // indirect
-	mellium.im/sasl v0.3.1 // indirect
 )