Upgrade to go 1.19 and rocksdb 7.5.3

README.md

@@ -66,16 +66,6 @@ Check [this](https://github.com/trezor/blockbook/issues/89) or [this](https://gi
 
 Your coin's block/transaction data may not be compatible with `BitcoinParser` `ParseBlock`/`ParseTx`, which is used by default. In that case, implement your coin in a similar way we used in case of [zcash](https://github.com/trezor/blockbook/tree/master/bchain/coins/zec) and some other coins. The principle is not to parse the block/transaction data in Blockbook but instead to get parsed transactions as json from the backend.
 
-#### Cannot build Blockbook using `go build` command
-
-When building Blockbook I get error `not enough arguments in call to _Cfunc_rocksdb_approximate_sizes`.
-
-RocksDB version 6.16.0 changed the API in a backwards incompatible way. It is necessary to build Blockbook with the `rocksdb_6_16` tag to fix the compatibility problem. The correct way to build Blockbook is:
-
-```
-go build -tags rocksdb_6_16
-```
-
 ## Data storage in RocksDB
 
 Blockbook stores data the key-value store RocksDB. Database format is described [here](/docs/rocksdb.md).

bchain/coins/blockchain.go

@@ -73,6 +73,7 @@ func init() {
 	BlockChainFactories["Ethereum Testnet Ropsten"] = eth.NewEthereumRPC
 	BlockChainFactories["Ethereum Testnet Ropsten Archive"] = eth.NewEthereumRPC
 	BlockChainFactories["Ethereum Testnet Goerli"] = eth.NewEthereumRPC
+	BlockChainFactories["Ethereum Testnet Goerli Archive"] = eth.NewEthereumRPC
 	BlockChainFactories["Bcash"] = bch.NewBCashRPC
 	BlockChainFactories["Bcash Testnet"] = bch.NewBCashRPC
 	BlockChainFactories["Bgold"] = btg.NewBGoldRPC

build/docker/bin/Dockerfile

@@ -8,15 +8,15 @@ RUN apt-get update && \
     apt-get upgrade -y && \
     apt-get install -y build-essential git wget pkg-config lxc-dev libzmq3-dev \
                        libgflags-dev libsnappy-dev zlib1g-dev libbz2-dev \
-                       liblz4-dev graphviz && \
+                       libzstd-dev liblz4-dev graphviz && \
     apt-get clean
 ARG GOLANG_VERSION
-ENV GOLANG_VERSION=go1.17.1
-ENV ROCKSDB_VERSION=v6.22.1
+ENV GOLANG_VERSION=go1.19
+ENV ROCKSDB_VERSION=v7.5.3
 ENV GOPATH=/go
 ENV PATH=$PATH:$GOPATH/bin
 ENV CGO_CFLAGS="-I/opt/rocksdb/include"
-ENV CGO_LDFLAGS="-L/opt/rocksdb -ldl -lrocksdb -lstdc++ -lm -lz -lbz2 -lsnappy -llz4"
+ENV CGO_LDFLAGS="-L/opt/rocksdb -ldl -lrocksdb -lstdc++ -lm -lz -lbz2 -lsnappy -llz4 -lzstd"
 ARG TCMALLOC
 
 RUN mkdir /build

build/docker/bin/Makefile

@@ -10,12 +10,12 @@ ARGS ?=
 all: build tools
 
 build: prepare-sources
-	cd $(BLOCKBOOK_SRC) && go build -tags rocksdb_6_16 -o $(CURDIR)/blockbook -ldflags="-s -w $(LDFLAGS)" $(ARGS)
+	cd $(BLOCKBOOK_SRC) && go build -o $(CURDIR)/blockbook -ldflags="-s -w $(LDFLAGS)" $(ARGS)
 	cp $(CURDIR)/blockbook /out/blockbook
 	chown $(PACKAGER) /out/blockbook
 
 build-debug: prepare-sources
-	cd $(BLOCKBOOK_SRC) && go build -tags rocksdb_6_16 -o $(CURDIR)/blockbook -ldflags="$(LDFLAGS)" $(ARGS)
+	cd $(BLOCKBOOK_SRC) && go build -o $(CURDIR)/blockbook -ldflags="$(LDFLAGS)" $(ARGS)
 	cp $(CURDIR)/blockbook /out/blockbook
 	chown $(PACKAGER) /out/blockbook
 
@@ -24,13 +24,13 @@ tools:
 	chown $(PACKAGER) /out/{ldb,sst_dump}
 
 test: prepare-sources
-	cd $(BLOCKBOOK_SRC) && go test -tags 'rocksdb_6_16 unittest' `go list ./... | grep -vP '^github.com/trezor/blockbook/(contrib|tests)'` $(ARGS)
+	cd $(BLOCKBOOK_SRC) && go test -tags 'unittest' `go list ./... | grep -vP '^github.com/trezor/blockbook/(contrib|tests)'` $(ARGS)
 
 test-integration: prepare-sources
-	cd $(BLOCKBOOK_SRC) && go test -tags 'rocksdb_6_16 integration' `go list github.com/trezor/blockbook/tests/...` $(ARGS)
+	cd $(BLOCKBOOK_SRC) && go test -tags 'integration' `go list github.com/trezor/blockbook/tests/...` $(ARGS)
 
 test-all: prepare-sources
-	cd $(BLOCKBOOK_SRC) && go test -tags 'rocksdb_6_16 unittest integration' `go list ./... | grep -v '^github.com/trezor/blockbook/contrib'` $(ARGS)
+	cd $(BLOCKBOOK_SRC) && go test -tags 'unittest integration' `go list ./... | grep -v '^github.com/trezor/blockbook/contrib'` $(ARGS)
 
 prepare-sources:
 	@ [ -n "`ls /src 2> /dev/null`" ] || (echo "/src doesn't exist or is empty" 1>&2 && exit 1)

db/bulkconnect.go

@@ -3,8 +3,8 @@ package db
 import (
 	"time"
 
-	"github.com/flier/gorocksdb"
 	"github.com/golang/glog"
+	"github.com/linxGnu/grocksdb"
 	"github.com/trezor/blockbook/bchain"
 )
 
@@ -58,7 +58,7 @@ func (d *RocksDB) InitBulkConnect() (*BulkConnect, error) {
 	return b, nil
 }
 
-func (b *BulkConnect) storeTxAddresses(wb *gorocksdb.WriteBatch, all bool) (int, int, error) {
+func (b *BulkConnect) storeTxAddresses(wb *grocksdb.WriteBatch, all bool) (int, int, error) {
 	var txm map[string]*TxAddresses
 	var sp int
 	if all {
@@ -101,7 +101,7 @@ func (b *BulkConnect) storeTxAddresses(wb *gorocksdb.WriteBatch, all bool) (int,
 func (b *BulkConnect) parallelStoreTxAddresses(c chan error, all bool) {
 	defer close(c)
 	start := time.Now()
-	wb := gorocksdb.NewWriteBatch()
+	wb := grocksdb.NewWriteBatch()
 	defer wb.Destroy()
 	count, sp, err := b.storeTxAddresses(wb, all)
 	if err != nil {
@@ -116,7 +116,7 @@ func (b *BulkConnect) parallelStoreTxAddresses(c chan error, all bool) {
 	c <- nil
 }
 
-func (b *BulkConnect) storeBalances(wb *gorocksdb.WriteBatch, all bool) (int, error) {
+func (b *BulkConnect) storeBalances(wb *grocksdb.WriteBatch, all bool) (int, error) {
 	var bal map[string]*AddrBalance
 	if all {
 		bal = b.balances
@@ -141,7 +141,7 @@ func (b *BulkConnect) storeBalances(wb *gorocksdb.WriteBatch, all bool) (int, er
 func (b *BulkConnect) parallelStoreBalances(c chan error, all bool) {
 	defer close(c)
 	start := time.Now()
-	wb := gorocksdb.NewWriteBatch()
+	wb := grocksdb.NewWriteBatch()
 	defer wb.Destroy()
 	count, err := b.storeBalances(wb, all)
 	if err != nil {
@@ -156,7 +156,7 @@ func (b *BulkConnect) parallelStoreBalances(c chan error, all bool) {
 	c <- nil
 }
 
-func (b *BulkConnect) storeBulkAddresses(wb *gorocksdb.WriteBatch) error {
+func (b *BulkConnect) storeBulkAddresses(wb *grocksdb.WriteBatch) error {
 	for _, ba := range b.bulkAddresses {
 		if err := b.d.storeAddresses(wb, ba.bi.Height, ba.addresses); err != nil {
 			return err
@@ -202,7 +202,7 @@ func (b *BulkConnect) connectBlockBitcoinType(block *bchain.Block, storeBlockTxs
 	// open WriteBatch only if going to write
 	if sa || b.bulkAddressesCount > maxBulkAddresses || storeBlockTxs {
 		start := time.Now()
-		wb := gorocksdb.NewWriteBatch()
+		wb := grocksdb.NewWriteBatch()
 		defer wb.Destroy()
 		bac := b.bulkAddressesCount
 		if sa || b.bulkAddressesCount > maxBulkAddresses {
@@ -235,7 +235,7 @@ func (b *BulkConnect) connectBlockBitcoinType(block *bchain.Block, storeBlockTxs
 	return nil
 }
 
-func (b *BulkConnect) storeAddressContracts(wb *gorocksdb.WriteBatch, all bool) (int, error) {
+func (b *BulkConnect) storeAddressContracts(wb *grocksdb.WriteBatch, all bool) (int, error) {
 	var ac map[string]*AddrContracts
 	if all {
 		ac = b.addressContracts
@@ -260,7 +260,7 @@ func (b *BulkConnect) storeAddressContracts(wb *gorocksdb.WriteBatch, all bool)
 func (b *BulkConnect) parallelStoreAddressContracts(c chan error, all bool) {
 	defer close(c)
 	start := time.Now()
-	wb := gorocksdb.NewWriteBatch()
+	wb := grocksdb.NewWriteBatch()
 	defer wb.Destroy()
 	count, err := b.storeAddressContracts(wb, all)
 	if err != nil {
@@ -303,7 +303,7 @@ func (b *BulkConnect) connectBlockEthereumType(block *bchain.Block, storeBlockTx
 	// open WriteBatch only if going to write
 	if sa || b.bulkAddressesCount > maxBulkAddresses || storeBlockTxs {
 		start := time.Now()
-		wb := gorocksdb.NewWriteBatch()
+		wb := grocksdb.NewWriteBatch()
 		defer wb.Destroy()
 		bac := b.bulkAddressesCount
 		if sa || b.bulkAddressesCount > maxBulkAddresses {
@@ -333,7 +333,7 @@ func (b *BulkConnect) connectBlockEthereumType(block *bchain.Block, storeBlockTx
 		// if there are blockSpecificData, store them
 		blockSpecificData, _ := block.CoinSpecificData.(*bchain.EthereumBlockSpecificData)
 		if blockSpecificData != nil {
-			wb := gorocksdb.NewWriteBatch()
+			wb := grocksdb.NewWriteBatch()
 			defer wb.Destroy()
 			if err = b.d.storeBlockSpecificDataEthereumType(wb, block); err != nil {
 				return err
@@ -378,7 +378,7 @@ func (b *BulkConnect) Close() error {
 		storeAddressContractsChan = make(chan error)
 		go b.parallelStoreAddressContracts(storeAddressContractsChan, true)
 	}
-	wb := gorocksdb.NewWriteBatch()
+	wb := grocksdb.NewWriteBatch()
 	defer wb.Destroy()
 	bac := b.bulkAddressesCount
 	if err := b.storeBulkAddresses(wb); err != nil {

db/dboptions.go

@@ -2,31 +2,28 @@ package db
 
 // #include "rocksdb/c.h"
 import "C"
-
-import (
-	"github.com/flier/gorocksdb"
-)
+import "github.com/linxGnu/grocksdb"
 
 /*
-	possible additional tuning, using options not accessible by gorocksdb
+	possible additional tuning, using options not accessible by grocksdb
 
 // #include "rocksdb/c.h"
 import "C"
 
 	cNativeOpts := C.rocksdb_options_create()
-	opts := &gorocksdb.Options{}
+	opts := &grocksdb.Options{}
 	cField := reflect.Indirect(reflect.ValueOf(opts)).FieldByName("c")
 	cPtr := (**C.rocksdb_options_t)(unsafe.Pointer(cField.UnsafeAddr()))
 	*cPtr = cNativeOpts
 
 	cNativeBlockOpts := C.rocksdb_block_based_options_create()
-	blockOpts := &gorocksdb.BlockBasedTableOptions{}
+	blockOpts := &grocksdb.BlockBasedTableOptions{}
 	cBlockField := reflect.Indirect(reflect.ValueOf(blockOpts)).FieldByName("c")
 	cBlockPtr := (**C.rocksdb_block_based_table_options_t)(unsafe.Pointer(cBlockField.UnsafeAddr()))
 	*cBlockPtr = cNativeBlockOpts
 
 	// https://github.com/facebook/rocksdb/wiki/Partitioned-Index-Filters
-	blockOpts.SetIndexType(gorocksdb.KTwoLevelIndexSearchIndexType)
+	blockOpts.SetIndexType(grocksdb.KTwoLevelIndexSearchIndexType)
 	C.rocksdb_block_based_options_set_partition_filters(cNativeBlockOpts, boolToChar(true))
 	C.rocksdb_block_based_options_set_metadata_block_size(cNativeBlockOpts, C.uint64_t(4096))
 	C.rocksdb_block_based_options_set_cache_index_and_filter_blocks_with_high_priority(cNativeBlockOpts, boolToChar(true))
@@ -41,16 +38,16 @@ func boolToChar(b bool) C.uchar {
 }
 */
 
-func createAndSetDBOptions(bloomBits int, c *gorocksdb.Cache, maxOpenFiles int) *gorocksdb.Options {
-	blockOpts := gorocksdb.NewDefaultBlockBasedTableOptions()
+func createAndSetDBOptions(bloomBits int, c *grocksdb.Cache, maxOpenFiles int) *grocksdb.Options {
+	blockOpts := grocksdb.NewDefaultBlockBasedTableOptions()
 	blockOpts.SetBlockSize(32 << 10) // 32kB
 	blockOpts.SetBlockCache(c)
 	if bloomBits > 0 {
-		blockOpts.SetFilterPolicy(gorocksdb.NewBloomFilter(bloomBits))
+		blockOpts.SetFilterPolicy(grocksdb.NewBloomFilter(float64(bloomBits)))
 	}
 	blockOpts.SetFormatVersion(4)
 
-	opts := gorocksdb.NewDefaultOptions()
+	opts := grocksdb.NewDefaultOptions()
 	opts.SetBlockBasedTableFactory(blockOpts)
 	opts.SetCreateIfMissing(true)
 	opts.SetCreateIfMissingColumnFamilies(true)
@@ -60,6 +57,6 @@ func createAndSetDBOptions(bloomBits int, c *gorocksdb.Cache, maxOpenFiles int)
 	opts.SetWriteBufferSize(1 << 27)      // 128MB
 	opts.SetMaxBytesForLevelBase(1 << 27) // 128MB
 	opts.SetMaxOpenFiles(maxOpenFiles)
-	opts.SetCompression(gorocksdb.LZ4HCCompression)
+	opts.SetCompression(grocksdb.LZ4HCCompression)
 	return opts
 }

db/fiat.go

@@ -7,9 +7,9 @@ import (
 	"time"
 
 	vlq "github.com/bsm/go-vlq"
-	"github.com/flier/gorocksdb"
 	"github.com/golang/glog"
 	"github.com/juju/errors"
+	"github.com/linxGnu/grocksdb"
 )
 
 // FiatRatesTimeFormat is a format string for storing FiatRates timestamps in rocksdb
@@ -152,7 +152,7 @@ func FiatRatesConvertDate(date string) (*time.Time, error) {
 }
 
 // FiatRatesStoreTicker stores ticker data at the specified time
-func (d *RocksDB) FiatRatesStoreTicker(wb *gorocksdb.WriteBatch, ticker *CurrencyRatesTicker) error {
+func (d *RocksDB) FiatRatesStoreTicker(wb *grocksdb.WriteBatch, ticker *CurrencyRatesTicker) error {
 	if len(ticker.Rates) == 0 {
 		return errors.New("Error storing ticker: empty rates")
 	}
@@ -180,7 +180,7 @@ func isSuitableTicker(ticker *CurrencyRatesTicker, vsCurrency string, token stri
 	return true
 }
 
-func getTickerFromIterator(it *gorocksdb.Iterator, vsCurrency string, token string) (*CurrencyRatesTicker, error) {
+func getTickerFromIterator(it *grocksdb.Iterator, vsCurrency string, token string) (*CurrencyRatesTicker, error) {
 	timeObj, err := time.Parse(FiatRatesTimeFormat, string(it.Key().Data()))
 	if err != nil {
 		return nil, err

db/fiat_test.go

@@ -7,7 +7,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/flier/gorocksdb"
+	"github.com/linxGnu/grocksdb"
 )
 
 func TestRocksTickers(t *testing.T) {
@@ -60,7 +60,7 @@ func TestRocksTickers(t *testing.T) {
 		},
 	}
 
-	wb := gorocksdb.NewWriteBatch()
+	wb := grocksdb.NewWriteBatch()
 	defer wb.Destroy()
 	err := d.FiatRatesStoreTicker(wb, ticker1)
 	if err != nil {