tbcd, bfgd: add various hemi specific indexes

api/tbcapi/tbcapi.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2024 Hemi Labs, Inc.
+// Copyright (c) 2024-2025 Hemi Labs, Inc.
 // Use of this source code is governed by the MIT License,
 // which can be found in the LICENSE file.
 
@@ -15,6 +15,7 @@ import (
 
 	"github.com/hemilabs/heminetwork/api"
 	"github.com/hemilabs/heminetwork/api/protocol"
+	"github.com/hemilabs/heminetwork/hemi"
 )
 
 // XXX we should kill the wrapping types that are basically identical to wire.
@@ -76,6 +77,9 @@ const (
 
 	CmdBlockDownloadAsyncRawRequest  = "tbcapi-block-download-async-raw-request"
 	CmdBlockDownloadAsyncRawResponse = "tbcapi-block-download-async-raw-response"
+
+	CmdBlockKeystoneByL2KeystoneAbrevHashRequest  = "tbcapi-l2-keystone-abrev-by-abrev-hash-request"
+	CmdBlockKeystoneByL2KeystoneAbrevHashResponse = "tbcapi-l2-keystone-abrev-by-abrev-hash-response"
 )
 
 var (
@@ -284,6 +288,16 @@ type BlockInsertRawRequest struct {
 	Block api.ByteSlice `json:"block"`
 }
 
+type BlockKeystoneByL2KeystoneAbrevHashRequest struct {
+	L2KeystoneAbrevHash *chainhash.Hash `json:"l2_keystones_abrev_hash"`
+}
+
+type BlockKeystoneByL2KeystoneAbrevHashResponse struct {
+	L2KeystoneAbrev *hemi.L2KeystoneAbrev `json:"l2_keystone_abrev"`
+	BtcBlockHash    *chainhash.Hash       `json:"btc_block_hash"`
+	Error           *protocol.Error       `json:"error,omitempty"`
+}
+
 type BlockInsertRawResponse struct {
 	BlockHash *chainhash.Hash `json:"block_hash"`
 	Error     *protocol.Error `json:"error,omitempty"`
@@ -315,42 +329,44 @@ type BlockDownloadAsyncRawResponse struct {
 }
 
 var commands = map[protocol.Command]reflect.Type{
-	CmdPingRequest:                     reflect.TypeOf(PingRequest{}),
-	CmdPingResponse:                    reflect.TypeOf(PingResponse{}),
-	CmdBlockByHashRequest:              reflect.TypeOf(BlockByHashRequest{}),
-	CmdBlockByHashResponse:             reflect.TypeOf(BlockByHashResponse{}),
-	CmdBlockByHashRawRequest:           reflect.TypeOf(BlockByHashRawRequest{}),
-	CmdBlockByHashRawResponse:          reflect.TypeOf(BlockByHashRawResponse{}),
-	CmdBlockHeadersByHeightRawRequest:  reflect.TypeOf(BlockHeadersByHeightRawRequest{}),
-	CmdBlockHeadersByHeightRawResponse: reflect.TypeOf(BlockHeadersByHeightRawResponse{}),
-	CmdBlockHeadersByHeightRequest:     reflect.TypeOf(BlockHeadersByHeightRequest{}),
-	CmdBlockHeadersByHeightResponse:    reflect.TypeOf(BlockHeadersByHeightResponse{}),
-	CmdBlockHeaderBestRawRequest:       reflect.TypeOf(BlockHeaderBestRawRequest{}),
-	CmdBlockHeaderBestRawResponse:      reflect.TypeOf(BlockHeaderBestRawResponse{}),
-	CmdBlockHeaderBestRequest:          reflect.TypeOf(BlockHeaderBestRequest{}),
-	CmdBlockHeaderBestResponse:         reflect.TypeOf(BlockHeaderBestResponse{}),
-	CmdBalanceByAddressRequest:         reflect.TypeOf(BalanceByAddressRequest{}),
-	CmdBalanceByAddressResponse:        reflect.TypeOf(BalanceByAddressResponse{}),
-	CmdUTXOsByAddressRawRequest:        reflect.TypeOf(UTXOsByAddressRawRequest{}),
-	CmdUTXOsByAddressRawResponse:       reflect.TypeOf(UTXOsByAddressRawResponse{}),
-	CmdUTXOsByAddressRequest:           reflect.TypeOf(UTXOsByAddressRequest{}),
-	CmdUTXOsByAddressResponse:          reflect.TypeOf(UTXOsByAddressResponse{}),
-	CmdTxByIdRawRequest:                reflect.TypeOf(TxByIdRawRequest{}),
-	CmdTxByIdRawResponse:               reflect.TypeOf(TxByIdRawResponse{}),
-	CmdTxByIdRequest:                   reflect.TypeOf(TxByIdRequest{}),
-	CmdTxByIdResponse:                  reflect.TypeOf(TxByIdResponse{}),
-	CmdTxBroadcastRequest:              reflect.TypeOf(TxBroadcastRequest{}),
-	CmdTxBroadcastResponse:             reflect.TypeOf(TxBroadcastResponse{}),
-	CmdTxBroadcastRawRequest:           reflect.TypeOf(TxBroadcastRawRequest{}),
-	CmdTxBroadcastRawResponse:          reflect.TypeOf(TxBroadcastRawResponse{}),
-	CmdBlockInsertRequest:              reflect.TypeOf(BlockInsertRequest{}),
-	CmdBlockInsertResponse:             reflect.TypeOf(BlockInsertResponse{}),
-	CmdBlockInsertRawRequest:           reflect.TypeOf(BlockInsertRawRequest{}),
-	CmdBlockInsertRawResponse:          reflect.TypeOf(BlockInsertRawResponse{}),
-	CmdBlockDownloadAsyncRequest:       reflect.TypeOf(BlockDownloadAsyncRequest{}),
-	CmdBlockDownloadAsyncResponse:      reflect.TypeOf(BlockDownloadAsyncResponse{}),
-	CmdBlockDownloadAsyncRawRequest:    reflect.TypeOf(BlockDownloadAsyncRawRequest{}),
-	CmdBlockDownloadAsyncRawResponse:   reflect.TypeOf(BlockDownloadAsyncRawResponse{}),
+	CmdPingRequest:                                reflect.TypeOf(PingRequest{}),
+	CmdPingResponse:                               reflect.TypeOf(PingResponse{}),
+	CmdBlockByHashRequest:                         reflect.TypeOf(BlockByHashRequest{}),
+	CmdBlockByHashResponse:                        reflect.TypeOf(BlockByHashResponse{}),
+	CmdBlockByHashRawRequest:                      reflect.TypeOf(BlockByHashRawRequest{}),
+	CmdBlockByHashRawResponse:                     reflect.TypeOf(BlockByHashRawResponse{}),
+	CmdBlockHeadersByHeightRawRequest:             reflect.TypeOf(BlockHeadersByHeightRawRequest{}),
+	CmdBlockHeadersByHeightRawResponse:            reflect.TypeOf(BlockHeadersByHeightRawResponse{}),
+	CmdBlockHeadersByHeightRequest:                reflect.TypeOf(BlockHeadersByHeightRequest{}),
+	CmdBlockHeadersByHeightResponse:               reflect.TypeOf(BlockHeadersByHeightResponse{}),
+	CmdBlockHeaderBestRawRequest:                  reflect.TypeOf(BlockHeaderBestRawRequest{}),
+	CmdBlockHeaderBestRawResponse:                 reflect.TypeOf(BlockHeaderBestRawResponse{}),
+	CmdBlockHeaderBestRequest:                     reflect.TypeOf(BlockHeaderBestRequest{}),
+	CmdBlockHeaderBestResponse:                    reflect.TypeOf(BlockHeaderBestResponse{}),
+	CmdBalanceByAddressRequest:                    reflect.TypeOf(BalanceByAddressRequest{}),
+	CmdBalanceByAddressResponse:                   reflect.TypeOf(BalanceByAddressResponse{}),
+	CmdUTXOsByAddressRawRequest:                   reflect.TypeOf(UTXOsByAddressRawRequest{}),
+	CmdUTXOsByAddressRawResponse:                  reflect.TypeOf(UTXOsByAddressRawResponse{}),
+	CmdUTXOsByAddressRequest:                      reflect.TypeOf(UTXOsByAddressRequest{}),
+	CmdUTXOsByAddressResponse:                     reflect.TypeOf(UTXOsByAddressResponse{}),
+	CmdTxByIdRawRequest:                           reflect.TypeOf(TxByIdRawRequest{}),
+	CmdTxByIdRawResponse:                          reflect.TypeOf(TxByIdRawResponse{}),
+	CmdTxByIdRequest:                              reflect.TypeOf(TxByIdRequest{}),
+	CmdTxByIdResponse:                             reflect.TypeOf(TxByIdResponse{}),
+	CmdTxBroadcastRequest:                         reflect.TypeOf(TxBroadcastRequest{}),
+	CmdTxBroadcastResponse:                        reflect.TypeOf(TxBroadcastResponse{}),
+	CmdTxBroadcastRawRequest:                      reflect.TypeOf(TxBroadcastRawRequest{}),
+	CmdTxBroadcastRawResponse:                     reflect.TypeOf(TxBroadcastRawResponse{}),
+	CmdBlockInsertRequest:                         reflect.TypeOf(BlockInsertRequest{}),
+	CmdBlockInsertResponse:                        reflect.TypeOf(BlockInsertResponse{}),
+	CmdBlockInsertRawRequest:                      reflect.TypeOf(BlockInsertRawRequest{}),
+	CmdBlockInsertRawResponse:                     reflect.TypeOf(BlockInsertRawResponse{}),
+	CmdBlockDownloadAsyncRequest:                  reflect.TypeOf(BlockDownloadAsyncRequest{}),
+	CmdBlockDownloadAsyncResponse:                 reflect.TypeOf(BlockDownloadAsyncResponse{}),
+	CmdBlockDownloadAsyncRawRequest:               reflect.TypeOf(BlockDownloadAsyncRawRequest{}),
+	CmdBlockDownloadAsyncRawResponse:              reflect.TypeOf(BlockDownloadAsyncRawResponse{}),
+	CmdBlockKeystoneByL2KeystoneAbrevHashRequest:  reflect.TypeOf(BlockKeystoneByL2KeystoneAbrevHashRequest{}),
+	CmdBlockKeystoneByL2KeystoneAbrevHashResponse: reflect.TypeOf(BlockKeystoneByL2KeystoneAbrevHashResponse{}),
 }
 
 type tbcAPI struct{}

cmd/tbcd/tbcd.go

@@ -61,10 +61,16 @@ var (
 		},
 		"TBC_BLOCK_SANITY": config.Config{
 			Value:        &cfg.BlockSanity,
-			DefaultValue: false,
+			DefaultValue: true,
 			Help:         "enable/disable block sanity checks before inserting",
 			Print:        config.PrintAll,
 		},
+		"TBC_HEMI_INDEX": config.Config{
+			Value:        &cfg.HemiIndex,
+			DefaultValue: false,
+			Help:         "enable/disable various hemi related indexes",
+			Print:        config.PrintAll,
+		},
 		"TBC_LEVELDB_HOME": config.Config{
 			Value:        &cfg.LevelDBHome,
 			DefaultValue: defaultHome,
@@ -77,6 +83,12 @@ var (
 			Help:         "loglevel for various packages; INFO, DEBUG and TRACE",
 			Print:        config.PrintAll,
 		},
+		"TBC_MAX_CACHED_KEYSTONES": config.Config{
+			Value:        &cfg.MaxCachedKeystones,
+			DefaultValue: int(1e5),
+			Help:         "maximum cached keystones during indexing",
+			Print:        config.PrintAll,
+		},
 		"TBC_MAX_CACHED_TXS": config.Config{
 			Value:        &cfg.MaxCachedTxs,
 			DefaultValue: int(1e6),

database/level/level.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2024 Hemi Labs, Inc.
+// Copyright (c) 2024-2025 Hemi Labs, Inc.
 // Use of this source code is governed by the MIT License,
 // which can be found in the LICENSE file.
 
@@ -29,6 +29,7 @@ const (
 	BlockHeadersDB  = "blockheaders"
 	BlocksMissingDB = "blocksmissing"
 	MetadataDB      = "metadata"
+	KeystonesDB     = "keystones"
 	HeightHashDB    = "heighthash"
 	PeersDB         = "peers"
 	OutputsDB       = "outputs"
@@ -221,6 +222,10 @@ func New(ctx context.Context, home string, version int) (*Database, error) {
 	if err != nil {
 		return nil, fmt.Errorf("leveldb %v: %w", TransactionsDB, err)
 	}
+	err = l.openDB(KeystonesDB, nil)
+	if err != nil {
+		return nil, fmt.Errorf("leveldb %v: %w", KeystonesDB, err)
+	}
 
 	// Blocks database is special
 	err = l.openRawDB(BlocksDB, rawdb.DefaultMaxFileSize)

database/tbcd/database.go

@@ -118,6 +118,15 @@ type Database interface {
 	ScriptHashByOutpoint(ctx context.Context, op Outpoint) (*ScriptHash, error)
 	UtxosByScriptHash(ctx context.Context, sh ScriptHash, start uint64, count uint64) ([]Utxo, error)
 	UtxosByScriptHashCount(ctx context.Context, sh ScriptHash) (uint64, error)
+
+	// Hemi
+	BlockKeystoneUpdate(ctx context.Context, direction int, keystones map[chainhash.Hash]Keystone) error
+	BlockKeystoneByL2KeystoneAbrevHash(ctx context.Context, abrevhash chainhash.Hash) (*Keystone, error)
+}
+
+type Keystone struct {
+	BlockHash           chainhash.Hash // Block that contains abbreviated keystone
+	AbbreviatedKeystone []byte         // Abbreviated keystone
 }
 
 // XXX there exist various types in this file that need to be reevaluated.
@@ -225,13 +234,13 @@ func NewOutpoint(txid [32]byte, index uint32) (op Outpoint) {
 	return
 }
 
-// CacheOutput is a densely packed representation of a bitcoin UTXo. The fields are
-// script_hash + value + out_index. It is packed for
-// memory conservation reasons.
+// CacheOutput is a densely packed representation of a bitcoin UTXo. The fields
+// are script_hash + value + out_index. It is packed for memory conservation
+// reasons.
 type CacheOutput [32 + 8 + 4]byte // script_hash + value + out_idx
 
-// String returns pretty printable CacheOutput. Hash is not reversed since it is an
-// opaque pointer. It prints satoshis@script_hash:output_index
+// String returns pretty printable CacheOutput. Hash is not reversed since it
+// is an opaque pointer. It prints satoshis@script_hash:output_index
 func (c CacheOutput) String() string {
 	return fmt.Sprintf("%d @ %x:%d", binary.BigEndian.Uint64(c[32:40]),
 		c[0:32], binary.BigEndian.Uint32(c[40:]))
@@ -292,8 +301,8 @@ func NewDeleteCacheOutput(hash [32]byte, outIndex uint32) (co CacheOutput) {
 // Utxo packs a transaction id, the value and the out index.
 type Utxo [32 + 8 + 4]byte // tx_id + value + out_idx
 
-// String returns pretty printable CacheOutput. Hash is not reversed since it is an
-// opaque pointer. It prints satoshis@script_hash:output_index
+// String returns pretty printable CacheOutput. Hash is not reversed since it
+// is an opaque pointer. It prints satoshis@script_hash:output_index
 func (u Utxo) String() string {
 	ch, _ := chainhash.NewHash(u[0:32])
 	return fmt.Sprintf("%d @ %v:%d", binary.BigEndian.Uint64(u[32:40]),

database/tbcd/level/level.go

@@ -26,6 +26,7 @@ import (
 	"github.com/hemilabs/heminetwork/database"
 	"github.com/hemilabs/heminetwork/database/level"
 	"github.com/hemilabs/heminetwork/database/tbcd"
+	"github.com/hemilabs/heminetwork/hemi"
 )
 
 // Locking order:
@@ -282,6 +283,27 @@ func (l *ldb) MetadataBatchGet(ctx context.Context, allOrNone bool, keys [][]byt
 	return l.transactionBatchGet(ctx, mdDB, allOrNone, keys)
 }
 
+func (l *ldb) BlockKeystoneByL2KeystoneAbrevHash(ctx context.Context, abrevhash chainhash.Hash) (*tbcd.Keystone, error) {
+	log.Tracef("BlockKeystoneByL2KeystoneAbrevHash")
+	defer log.Tracef("BlockKeystoneByL2KeystoneAbrevHash exit")
+
+	kssDB := l.pool[level.KeystonesDB]
+	eks, err := kssDB.Get(abrevhash.CloneBytes(), nil)
+	if err != nil {
+		if errors.Is(err, leveldb.ErrNotFound) {
+			return nil, database.NotFoundError(fmt.Sprintf("l2 keystone not found: %v", abrevhash))
+		}
+		return nil, fmt.Errorf("l2 keystone get: %w", err)
+	}
+
+	ks := tbcd.Keystone{
+		BlockHash:           chainhash.Hash(eks[:32]),
+		AbbreviatedKeystone: eks[32:],
+	}
+
+	return &ks, nil
+}
+
 // BatchAppend appends rows to batch b.
 func BatchAppend(ctx context.Context, b *leveldb.Batch, rows []tbcd.Row) {
 	log.Tracef("BatchAppend")
@@ -1707,6 +1729,66 @@ func (l *ldb) BlockTxUpdate(ctx context.Context, direction int, txs map[tbcd.TxK
 	return nil
 }
 
+func (l *ldb) BlockKeystoneUpdate(ctx context.Context, direction int, keystones map[chainhash.Hash]tbcd.Keystone) error {
+	log.Tracef("BlockKeystoneUpdate")
+	defer log.Tracef("BlockKeystoneUpdate exit")
+
+	if !(direction == 1 || direction == -1) {
+		return fmt.Errorf("invalid direction: %v", direction)
+	}
+
+	// keystones
+	kssTx, kssCommit, kssDiscard, err := l.startTransaction(level.KeystonesDB)
+	if err != nil {
+		return fmt.Errorf("keystones open db transaction: %w", err)
+	}
+	defer kssDiscard()
+
+	kssBatch := new(leveldb.Batch)
+	for k, v := range keystones {
+		switch direction {
+		case -1:
+			foundKeystone, err := kssTx.Get(k[:], nil)
+			if err != nil {
+				continue
+			}
+			fbh, err := chainhash.NewHash(foundKeystone[0:32])
+			if err != nil {
+				return err // XXX really can't happen
+			}
+			// Only delete keystone if it is in the previously found block.
+			if fbh.IsEqual(&v.BlockHash) {
+				kssBatch.Delete(k[:])
+			}
+		case 1:
+			has, err := kssTx.Has(k[:], nil)
+			if err != nil {
+				return fmt.Errorf("keystone update has: %w", err)
+			}
+			if has {
+				// Only store unknown keystones
+				continue
+			}
+			var value [chainhash.HashSize + hemi.L2KeystoneAbrevSize]byte
+			copy(value[0:32], v.BlockHash[:])
+			copy(value[32:], v.AbbreviatedKeystone)
+			kssBatch.Put(k[:], value[:])
+		}
+	}
+
+	// Write keystones batch
+	if err = kssTx.Write(kssBatch, nil); err != nil {
+		return fmt.Errorf("keystones insert: %w", err)
+	}
+
+	// keystones commit
+	if err = kssCommit(); err != nil {
+		return fmt.Errorf("keystones commit: %w", err)
+	}
+
+	return nil
+}
+
 func (l *ldb) BlockHeaderCacheStats() tbcd.CacheStats {
 	if l.cfg.blockheaderCacheSize == 0 {
 		return noStats