Skip to content

Commit

Permalink
chore(x/tally): std dev filter max min check against pre-computed big…
Browse files Browse the repository at this point in the history 
… ints
  • Loading branch information
@hacheigriega
hacheigriega committed Jan 29, 2025
1 parent a448745 commit 1410351
Show file tree
Hide file tree
Showing 2 changed files with 49 additions and 29 deletions.
20 changes: 10 additions & 10 deletions x/tally/keeper/filter_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -562,7 +562,7 @@ func TestFilter(t *testing.T) {
{Reveal: `{"result": {"text": 5}}`},
{Reveal: `{"result": {"text": 6}}`},
{Reveal: `{"result": {"text": 7}}`},
{Reveal: `{"result": {"text": -9223372036854775808}}`}, // overflow
{Reveal: `{"result": {"text": -9223372036854775809}}`}, // overflow
{Reveal: `{"result": {"text": 9}}`},
},
consensus: true,
Expand Down Expand Up @@ -592,7 +592,7 @@ func TestFilter(t *testing.T) {
{Reveal: `{"result": {"text": 6}}`},
{Reveal: `{"result": {"text": 7}}`},
{Reveal: `{"result": {"text": 6}}`},
{Reveal: `{"result": {"text": 4294967296}}`}, // overflow
{Reveal: `{"result": {"text": 2147483648}}`}, // overflow
},
consensus: true,
consPubKeys: nil,
Expand Down Expand Up @@ -746,15 +746,15 @@ func TestFilter(t *testing.T) {
tallyInputAsHex: "0200000000000F424006000000000000000D242E726573756C742E74657874", // sigma_multiplier = 1.0, number_type = 0x06, json_path = $.result.text
outliers: []bool{true, false, false, false, false, true, false, false, false},
reveals: []types.RevealBody{
{Reveal: `{"result": {"text": -58896044618658097711785492504343953926634992332820282019728792003956564819968, "number": 0}}`}, // too small
{Reveal: `{"result": {"text": -57896044618658097711785492504343953926634992332820282019728792003956564819967, "number": 10}}`}, // ok (min of int256)
{Reveal: `{"result": {"text": -57896044618658097711785492504343953926634992332820282019728792003956564819967, "number": 101}}`}, // ok
{Reveal: `{"result": {"text": -57896044618658097711785492504343953926634992332820282019728792003956564819967, "number": 0}}`}, // ok
{Reveal: `{"result": {"text": -57896044618658097711785492504343953926634992332820282019728792003956564819967, "number": 0}}`}, // ok
{Reveal: `{"result": {"text": -57896044618658097711785492504343953926634992332820282019728792003956564819969, "number": 0}}`}, // too small
{Reveal: `{"result": {"text": -57896044618658097711785492504343953926634992332820282019728792003956564819968, "number": 10}}`}, // ok (min of int256)
{Reveal: `{"result": {"text": -57896044618658097711785492504343953926634992332820282019728792003956564819968, "number": 101}}`}, // ok
{Reveal: `{"result": {"text": -57896044618658097711785492504343953926634992332820282019728792003956564819968, "number": 0}}`}, // ok
{Reveal: `{"result": {"text": -57896044618658097711785492504343953926634992332820282019728792003956564819968, "number": 0}}`}, // ok
{Reveal: `{"result": {"text": 115792089237316195423570985008687907853269984665640564039457584007913129639936, "number": 0}}`}, // too large (max of uint256 + 1)
{Reveal: `{"result": {"text": -57896044618658097711785492504343953926634992332820282019728792003956564819967, "number": 0}}`}, // ok
{Reveal: `{"result": {"text": -57896044618658097711785492504343953926634992332820282019728792003956564819967, "number": 0}}`}, // ok
{Reveal: `{"result": {"text": -57896044618658097711785492504343953926634992332820282019728792003956564819967, "number": 0}}`}, // ok
{Reveal: `{"result": {"text": -57896044618658097711785492504343953926634992332820282019728792003956564819968, "number": 0}}`}, // ok
{Reveal: `{"result": {"text": -57896044618658097711785492504343953926634992332820282019728792003956564819968, "number": 0}}`}, // ok
{Reveal: `{"result": {"text": -57896044618658097711785492504343953926634992332820282019728792003956564819968, "number": 0}}`}, // ok
},
consensus: true,
consPubKeys: nil,
Expand Down
58 changes: 39 additions & 19 deletions x/tally/types/filters.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package types
import (
"bytes"
"encoding/binary"
"math"
"math/big"
)

Expand Down Expand Up @@ -94,11 +95,28 @@ func (f FilterMode) ApplyFilter(reveals []RevealBody, errors []bool) ([]bool, bo
type FilterStdDev struct {
sigmaMultiplier SigmaMultiplier
dataPath string // JSON path to reveal data
filterFunc func(dataList []string, sigmaMultiplier SigmaMultiplier, errors []bool, replicationFactor uint16, bitLenLimit int) ([]bool, bool)
replicationFactor uint16
bitLenLimit int
// The maximum and minimum values that can be represented by the number type as specified by the requestor
maxNumber *big.Int
minNumber *big.Int
}

var (
minUint = big.NewInt(0)
minInt32 = big.NewInt(0).SetInt64(math.MinInt32)
maxInt32 = big.NewInt(0).SetInt64(math.MaxInt32)
maxUint32 = big.NewInt(0).SetUint64(math.MaxUint32)
minInt64 = big.NewInt(0).SetInt64(math.MinInt64)
maxInt64 = big.NewInt(0).SetInt64(math.MaxInt64)
maxUint64 = big.NewInt(0).SetUint64(math.MaxUint64)
minInt128, _ = big.NewInt(0).SetString("-170141183460469231731687303715884105728", 10)
maxInt128, _ = big.NewInt(0).SetString("170141183460469231731687303715884105727", 10)
maxUint128, _ = big.NewInt(0).SetString("340282366920938463463374607431768211455", 10)
minInt256, _ = big.NewInt(0).SetString("-57896044618658097711785492504343953926634992332820282019728792003956564819968", 10)
maxInt256, _ = big.NewInt(0).SetString("57896044618658097711785492504343953926634992332820282019728792003956564819967", 10)
maxUint256, _ = big.NewInt(0).SetString("115792089237316195423570985008687907853269984665640564039457584007913129639935", 10)
)

// NewFilterStdDev constructs a new FilterStdDev object given a
// filter input.
// Standard deviation filter input looks as follows:
Expand All @@ -123,25 +141,32 @@ func NewFilterStdDev(input []byte, gasCostMultiplier uint64, replicationFactor u

switch input[9] {
case 0x00: // 32-bit signed integer
filter.bitLenLimit = 31
filter.maxNumber = maxInt32
filter.minNumber = minInt32
case 0x01: // 32-bit unsigned integer
filter.bitLenLimit = 32
filter.maxNumber = maxUint32
filter.minNumber = minUint
case 0x02: // 64-bit signed integer
filter.bitLenLimit = 63
filter.maxNumber = maxInt64
filter.minNumber = minInt64
case 0x03: // 64-bit unsigned integer
filter.bitLenLimit = 64
filter.maxNumber = maxUint64
filter.minNumber = minUint
case 0x04: // 128-bit signed integer
filter.bitLenLimit = 127
filter.maxNumber = maxInt128
filter.minNumber = minInt128
case 0x05: // 128-bit unsigned integer
filter.bitLenLimit = 128
filter.maxNumber = maxUint128
filter.minNumber = minUint
case 0x06: // 256-bit signed integer
filter.bitLenLimit = 255
filter.maxNumber = maxInt256
filter.minNumber = minInt256
case 0x07: // 256-bit unsigned integer
filter.bitLenLimit = 256
filter.maxNumber = maxUint256
filter.minNumber = minUint
default:
return filter, ErrInvalidNumberType
}
filter.filterFunc = detectOutliersBigInt

var pathLen uint64
err = binary.Read(bytes.NewReader(input[10:18]), binary.BigEndian, &pathLen)
Expand All @@ -164,10 +189,10 @@ func NewFilterStdDev(input []byte, gasCostMultiplier uint64, replicationFactor u
// by the given sigma multiplier value.
func (f FilterStdDev) ApplyFilter(reveals []RevealBody, errors []bool) ([]bool, bool) {
dataList, _ := parseReveals(reveals, f.dataPath, errors)
return f.filterFunc(dataList, f.sigmaMultiplier, errors, f.replicationFactor, f.bitLenLimit)
return detectOutliersBigInt(dataList, f.sigmaMultiplier, errors, f.replicationFactor, f.minNumber, f.maxNumber)
}

func detectOutliersBigInt(dataList []string, sigmaMultiplier SigmaMultiplier, errors []bool, replicationFactor uint16, bitLenLimit int) ([]bool, bool) {
func detectOutliersBigInt(dataList []string, sigmaMultiplier SigmaMultiplier, errors []bool, replicationFactor uint16, minNumber *big.Int, maxNumber *big.Int) ([]bool, bool) {
sum := new(big.Int)
nums := make([]*big.Int, 0, len(dataList))
corruptQueue := make([]int, 0, len(dataList)) // queue of corrupt indices in dataList
Expand All @@ -180,12 +205,7 @@ func detectOutliersBigInt(dataList []string, sigmaMultiplier SigmaMultiplier, er

num := new(big.Int)
_, ok := num.SetString(data, 10)
if !ok || num.BitLen() > bitLenLimit {
errors[i] = true
corruptQueue = append(corruptQueue, i)
continue
}
if bitLenLimit%2 == 0 && num.Sign() == -1 {
if !ok || num.Cmp(minNumber) < 0 || num.Cmp(maxNumber) > 0 {
errors[i] = true
corruptQueue = append(corruptQueue, i)
continue
Expand Down

0 comments on commit 1410351

Please sign in to comment.