diff --git a/.gitignore b/.gitignore index 82c9e8857..fa60f43a2 100644 --- a/.gitignore +++ b/.gitignore @@ -52,3 +52,4 @@ fuzz/testdata *__debug_bin* *pprof *coverage.txt +tools/venv/* \ No newline at end of file diff --git a/api.go b/api.go index 3a3d1d5fa..af6be70a4 100644 --- a/api.go +++ b/api.go @@ -84,6 +84,10 @@ type Config struct { // NoValidateJSONMarshaler indicates that the encoder should not validate the output string // after encoding the JSONMarshaler to JSON. NoValidateJSONMarshaler bool + + // NoValidateJSONSkip indicates the decoder should not validate the JSON value when skipping it, + // such as unknown-fields, mismatched-type, redundant elements.. + NoValidateJSONSkip bool // NoEncoderNewline indicates that the encoder should not add a newline after every message NoEncoderNewline bool @@ -109,6 +113,7 @@ var ( ConfigFastest = Config{ NoQuoteTextMarshaler: true, NoValidateJSONMarshaler: true, + NoValidateJSONSkip: true, }.Froze() ) diff --git a/decoder/decoder_compat.go b/decoder/decoder_compat.go index b3e634187..81e1ae4e3 100644 --- a/decoder/decoder_compat.go +++ b/decoder/decoder_compat.go @@ -42,6 +42,7 @@ const ( _F_use_number = types.B_USE_NUMBER _F_validate_string = types.B_VALIDATE_STRING _F_allow_control = types.B_ALLOW_CONTROL + _F_no_validate_json = types.B_NO_VALIDATE_JSON ) type Options uint64 @@ -53,6 +54,7 @@ const ( OptionDisableUnknown Options = 1 << _F_disable_unknown OptionCopyString Options = 1 << _F_copy_string OptionValidateString Options = 1 << _F_validate_string + OptionNoValidateJSON Options = 1 << _F_no_validate_json ) func (self *Decoder) SetOptions(opts Options) { diff --git a/decoder/decoder_native_test.go b/decoder/decoder_native_test.go index ffa659b51..d8dbcc7ad 100644 --- a/decoder/decoder_native_test.go +++ b/decoder/decoder_native_test.go @@ -1,7 +1,6 @@ //go:build (amd64 && go1.17 && !go1.24) || (arm64 && go1.20 && !go1.24) // +build amd64,go1.17,!go1.24 arm64,go1.20,!go1.24 - /* * Copyright 2021 ByteDance Inc. * @@ -21,15 +20,88 @@ package decoder import ( - `encoding/json` - _`strings` - `testing` - _`reflect` + "encoding/json" + "fmt" + _ "reflect" + "strings" + _ "strings" + "testing" + "time" - `github.com/bytedance/sonic/internal/rt` - `github.com/stretchr/testify/assert` + "github.com/bytedance/sonic/internal/rt" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) + +func BenchmarkSkipValidate(b *testing.B) { + type skiptype struct { + A int `json:"a"` // mismatched + B string `json:"-"` // ommited + C [1]int `json:"c"` // fast int + D struct {} `json:"d"` // empty struct + E map[string]int `json:"e"` // mismatched elem + // Unknonwn + } + type C struct { + name string + json string + expTime float64 + } + var sam = map[int]interface{}{} + for i := 0; i < 1; i++ { + sam[i] = _BindingValue + } + comptd, err := json.Marshal(sam) + if err != nil { + b.Fatal("invalid json") + } + compt := string(comptd) + var cases = []C{ + {"mismatched", `{"a":`+compt+`}`, 5}, + {"ommited", `{"b":`+compt+`}`, 5}, + {"number", `{"c":[`+strings.Repeat("-1.23456e-19,", 1000)+`1]}`, 1.5}, + {"unknown", `{"unknown":`+compt+`}`, 5}, + {"empty", `{"d":`+compt+`}`, 5}, + {"mismatched elem", `{"e":`+compt+`}`, 5}, + } + _ = NewDecoder(`{}`).Decode(&skiptype{}) + + var avg1, avg2 time.Duration + for _, c := range cases { + b.Run(c.name, func(b *testing.B) { + b.Run("validate", func(b *testing.B) { + b.ResetTimer() + t1 := time.Now() + for i := 0; i < b.N; i++ { + var obj1 = &skiptype{} + // validate skip + d := NewDecoder(c.json) + _ = d.Decode(obj1) + } + d1 := time.Since(t1) + avg1 = d1/time.Duration(b.N) + }) + b.Run("fast", func(b *testing.B) { + b.ResetTimer() + t2 := time.Now() + for i := 0; i < b.N; i++ { + var obj2 = &skiptype{} + // fask skip + d := NewDecoder(c.json) + d.SetOptions(OptionNoValidateJSON) + _ = d.Decode(obj2) + } + d2 := time.Since(t2) + avg2 = d2/time.Duration(b.N) + }) + // fast skip must be expTime x faster + require.True(b, float64(avg1)/float64(avg2) > c.expTime, fmt.Sprintf("%v/%v=%v", avg1, avg2, float64(avg1)/float64(avg2))) + }) + } +} + + func TestSkipMismatchTypeAmd64Error(t *testing.T) { // t.Run("struct", func(t *testing.T) { // println("TestSkipError") diff --git a/decoder/decoder_test.go b/decoder/decoder_test.go index 19af2986f..c58ed97a5 100644 --- a/decoder/decoder_test.go +++ b/decoder/decoder_test.go @@ -18,7 +18,6 @@ package decoder import ( "encoding/json" - "fmt" "runtime" "runtime/debug" "strings" @@ -86,74 +85,6 @@ func init() { _ = json.Unmarshal([]byte(TwitterJson), &_BindingValue) } -func BenchmarkSkipValidate(b *testing.B) { - type skiptype struct { - A int `json:"a"` // mismatched - B string `json:"-"` // ommited - C [1]int `json:"c"` // fast int - D struct {} `json:"d"` // empty struct - E map[string]int `json:"e"` // mismatched elem - // Unknonwn - } - type C struct { - name string - json string - expTime float64 - } - var sam = map[int]interface{}{} - for i := 0; i < 1; i++ { - sam[i] = _BindingValue - } - comptd, err := json.Marshal(sam) - if err != nil { - b.Fatal("invalid json") - } - compt := string(comptd) - var cases = []C{ - {"mismatched", `{"a":`+compt+`}`, 5}, - {"ommited", `{"b":`+compt+`}`, 5}, - {"number", `{"c":[`+strings.Repeat("-1.23456e-19,", 1000)+`1]}`, 1.5}, - {"unknown", `{"unknown":`+compt+`}`, 5}, - {"empty", `{"d":`+compt+`}`, 5}, - {"mismatched elem", `{"e":`+compt+`}`, 5}, - } - _ = NewDecoder(`{}`).Decode(&skiptype{}) - - var avg1, avg2 time.Duration - for _, c := range cases { - b.Run(c.name, func(b *testing.B) { - b.Run("validate", func(b *testing.B) { - b.ResetTimer() - t1 := time.Now() - for i := 0; i < b.N; i++ { - var obj1 = &skiptype{} - // validate skip - d := NewDecoder(c.json) - _ = d.Decode(obj1) - } - d1 := time.Since(t1) - avg1 = d1/time.Duration(b.N) - }) - b.Run("fast", func(b *testing.B) { - b.ResetTimer() - t2 := time.Now() - for i := 0; i < b.N; i++ { - var obj2 = &skiptype{} - // fask skip - d := NewDecoder(c.json) - d.SetOptions(OptionNoValidateJSON) - _ = d.Decode(obj2) - } - d2 := time.Since(t2) - avg2 = d2/time.Duration(b.N) - }) - // fast skip must be expTime x faster - require.True(b, float64(avg1)/float64(avg2) > c.expTime, fmt.Sprintf("%v/%v=%v", avg1, avg2, float64(avg1)/float64(avg2))) - }) - } -} - - func TestSkipMismatchTypeError(t *testing.T) { t.Run("struct", func(t *testing.T) { println("TestSkipError") diff --git a/sonic.go b/sonic.go index 72e568d48..a9adc8a8b 100644 --- a/sonic.go +++ b/sonic.go @@ -72,6 +72,9 @@ func (cfg Config) Froze() API { } // configure decoder options: + if cfg.NoValidateJSONSkip { + api.decoderOpts |= decoder.OptionNoValidateJSON + } if cfg.UseInt64 { api.decoderOpts |= decoder.OptionUseInt64 }