-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Use a constant amount of memory for error slices and add error abstra…
…ction (#57) This PR changes the Info struct in the Ingester a bit so that we have an inner Errors struct for keeping track of errors. We add a method for resetting at, for observing errors, and for transforming to progress reports. We make sure that the error slices do not grow unbounded, but we keep track of the total error counts as well.
- Loading branch information
Showing
8 changed files
with
190 additions
and
69 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,117 @@ | ||
| package ingester | ||
|
|
||
| import ( | ||
| "time" | ||
|
|
||
| "github.com/duneanalytics/blockchain-ingester/models" | ||
| ) | ||
|
|
||
| type Info struct { | ||
| BlockchainName string | ||
| Stack string | ||
| LatestBlockNumber int64 | ||
| IngestedBlockNumber int64 | ||
| ConsumedBlockNumber int64 | ||
| Errors ErrorState | ||
| Since time.Time | ||
| } | ||
|
|
||
| func NewInfo(blockchain string, stack string) Info { | ||
| return Info{ | ||
| BlockchainName: blockchain, | ||
| Stack: stack, | ||
| Errors: ErrorState{ | ||
| RPCErrors: make([]ErrorInfo, 0, 100), | ||
| DuneErrors: make([]ErrorInfo, 0, 100), | ||
| RPCErrorCount: 0, | ||
| DuneErrorCount: 0, | ||
| }, | ||
| Since: time.Now(), | ||
| } | ||
| } | ||
|
|
||
| func (info *Info) ToProgressReport() models.BlockchainIndexProgress { | ||
| return models.BlockchainIndexProgress{ | ||
| BlockchainName: info.BlockchainName, | ||
| EVMStack: info.Stack, | ||
| LastIngestedBlockNumber: info.IngestedBlockNumber, | ||
| LatestBlockNumber: info.LatestBlockNumber, | ||
| Errors: info.ProgressReportErrors(), | ||
| DuneErrorCounts: info.Errors.DuneErrorCount, | ||
| RPCErrorCounts: info.Errors.RPCErrorCount, | ||
| Since: info.Since, | ||
| } | ||
| } | ||
|
|
||
| func (info *Info) ResetErrors() { | ||
| info.Since = time.Now() | ||
| info.Errors.Reset() | ||
| } | ||
|
|
||
| type ErrorState struct { | ||
| RPCErrors []ErrorInfo | ||
| DuneErrors []ErrorInfo | ||
| RPCErrorCount int | ||
| DuneErrorCount int | ||
| } | ||
|
|
||
| // ProgressReportErrors returns a combined list of errors from RPC requests and Dune requests | ||
| func (info Info) ProgressReportErrors() []models.BlockchainIndexError { | ||
| errors := make([]models.BlockchainIndexError, 0, len(info.Errors.RPCErrors)+len(info.Errors.DuneErrors)) | ||
| for _, e := range info.Errors.RPCErrors { | ||
| errors = append(errors, models.BlockchainIndexError{ | ||
| Timestamp: e.Timestamp, | ||
| BlockNumbers: e.BlockNumbers, | ||
| Error: e.Error.Error(), | ||
| Source: "rpc", | ||
| }) | ||
| } | ||
| for _, e := range info.Errors.DuneErrors { | ||
| errors = append(errors, models.BlockchainIndexError{ | ||
| Timestamp: e.Timestamp, | ||
| BlockNumbers: e.BlockNumbers, | ||
| Error: e.Error.Error(), | ||
| Source: "dune", | ||
| }) | ||
| } | ||
| return errors | ||
| } | ||
|
|
||
| func (es *ErrorState) Reset() { | ||
| es.RPCErrors = es.RPCErrors[:0] | ||
| es.DuneErrors = es.DuneErrors[:0] | ||
| es.RPCErrorCount = 0 | ||
| es.DuneErrorCount = 0 | ||
| } | ||
|
|
||
| func (es *ErrorState) ObserveRPCError(err ErrorInfo) { | ||
| es.RPCErrorCount++ | ||
| err.Timestamp = time.Now() | ||
|
|
||
| // If we have filled the slice, remove the oldest error | ||
| if len(es.RPCErrors) == cap(es.RPCErrors) { | ||
| tmp := make([]ErrorInfo, len(es.RPCErrors)-1, cap(es.RPCErrors)) | ||
| copy(tmp, es.RPCErrors[1:]) | ||
| es.RPCErrors = tmp | ||
| } | ||
| es.RPCErrors = append(es.RPCErrors, err) | ||
| } | ||
|
|
||
| func (es *ErrorState) ObserveDuneError(err ErrorInfo) { | ||
| es.DuneErrorCount++ | ||
| err.Timestamp = time.Now() | ||
|
|
||
| // If we have filled the slice, remove the oldest error | ||
| if len(es.DuneErrors) == cap(es.DuneErrors) { | ||
| tmp := make([]ErrorInfo, len(es.DuneErrors)-1, cap(es.DuneErrors)) | ||
| copy(tmp, es.DuneErrors[1:]) | ||
| es.DuneErrors = tmp | ||
| } | ||
| es.DuneErrors = append(es.DuneErrors, err) | ||
| } | ||
|
|
||
| type ErrorInfo struct { | ||
| Timestamp time.Time | ||
| BlockNumbers string | ||
| Error error | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,52 @@ | ||
| package ingester_test | ||
|
|
||
| import ( | ||
| "testing" | ||
|
|
||
| "github.com/duneanalytics/blockchain-ingester/ingester" | ||
| "github.com/go-errors/errors" | ||
| "github.com/stretchr/testify/require" | ||
| ) | ||
|
|
||
| // TestInfoErrors ensures that we never allow the error slices to grow indefinitely | ||
| func TestInfoErrors(t *testing.T) { | ||
| info := ingester.NewInfo("test", "test") | ||
| for j := 0; j < 2; j++ { | ||
| for i := 0; i < 200; i++ { | ||
| require.Len(t, info.Errors.RPCErrors, min(i, 100)) | ||
| require.Len(t, info.Errors.DuneErrors, min(i, 100)) | ||
| info.Errors.ObserveDuneError(ingester.ErrorInfo{}) | ||
| info.Errors.ObserveRPCError(ingester.ErrorInfo{}) | ||
| require.Equal(t, 100, cap(info.Errors.RPCErrors)) | ||
| require.Equal(t, 100, cap(info.Errors.DuneErrors)) | ||
| } | ||
| info.ResetErrors() | ||
| require.Len(t, info.Errors.RPCErrors, 0) | ||
| require.Len(t, info.Errors.DuneErrors, 0) | ||
| require.Equal(t, 100, cap(info.Errors.RPCErrors)) | ||
| require.Equal(t, 100, cap(info.Errors.DuneErrors)) | ||
| } | ||
| } | ||
|
|
||
| func TestProgressReportErrors(t *testing.T) { | ||
| info := ingester.NewInfo("test", "test") | ||
| info.Errors.ObserveDuneError(ingester.ErrorInfo{Error: errors.New("foo")}) | ||
| info.Errors.ObserveRPCError(ingester.ErrorInfo{Error: errors.New("bar")}) | ||
| errors := info.ProgressReportErrors() | ||
| require.Len(t, errors, 2) | ||
| } | ||
|
|
||
| func TestInfoToProgressReport(t *testing.T) { | ||
| info := ingester.NewInfo("test", "test") | ||
| info.IngestedBlockNumber = 1 | ||
| info.LatestBlockNumber = 2 | ||
| info.Errors.ObserveDuneError(ingester.ErrorInfo{Error: errors.New("foo")}) | ||
| report := info.ToProgressReport() | ||
| require.Equal(t, "test", report.BlockchainName) | ||
| require.Equal(t, "test", report.EVMStack) | ||
| require.Equal(t, int64(1), report.LastIngestedBlockNumber) | ||
| require.Equal(t, int64(2), report.LatestBlockNumber) | ||
| require.Len(t, report.Errors, 1) | ||
| require.Equal(t, 1, report.DuneErrorCounts) | ||
| require.Equal(t, 0, report.RPCErrorCounts) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters