feat:(decoder) add option CaseSensitive (#709)

Co-authored-by: liuqiang.06 <[email protected]>
bytedance · Jan 13, 2025 · 59be92f · 59be92f
1 parent 1d71b4d
commit 59be92f
Show file tree

Hide file tree

Showing 14 changed files with 241 additions and 169 deletions.
diff --git a/decoder/decoder_compat.go b/decoder/decoder_compat.go
@@ -1,3 +1,4 @@
+//go:build (!amd64 && !arm64) || go1.24 || !go1.17 || (arm64 && !go1.20)
 // +build !amd64,!arm64 go1.24 !go1.17 arm64,!go1.20
 
 /*
@@ -19,30 +20,32 @@
 package decoder
 
 import (
-    `bytes`
-    `encoding/json`
-    `io`
-    `reflect`
-    `unsafe`
-
-    `github.com/bytedance/sonic/internal/native/types`
-    `github.com/bytedance/sonic/option`
+	"bytes"
+	"encoding/json"
+	"io"
+	"reflect"
+	"unsafe"
+
+	"github.com/bytedance/sonic/internal/decoder/consts"
+	"github.com/bytedance/sonic/internal/native/types"
+	"github.com/bytedance/sonic/option"
 )
 
 func init() {
      println("WARNING: sonic/decoder only supports (Go1.17~1.23 && CPU amd64) or (go1.20~1.23 && CPU arm64), but your environment is not suitable")
 }
 
 const (
-     _F_use_int64       = 0
-     _F_disable_urc     = 2
-     _F_disable_unknown = 3
-     _F_copy_string     = 4
+     _F_use_int64       = consts.F_use_int64
+     _F_disable_urc     = consts.F_disable_unknown
+     _F_disable_unknown = consts.F_disable_unknown
+     _F_copy_string     = consts.F_copy_string
 
-     _F_use_number      = types.B_USE_NUMBER
-     _F_validate_string = types.B_VALIDATE_STRING
-     _F_allow_control   = types.B_ALLOW_CONTROL
-     _F_no_validate_json   = types.B_NO_VALIDATE_JSON
+     _F_use_number      = consts.F_use_number
+     _F_validate_string = consts.F_validate_string
+     _F_allow_control   = consts.F_allow_control
+     _F_no_validate_json = consts.F_no_validate_json
+     _F_case_sensitive  = consts.F_case_sensitive
 )
 
 type Options uint64
@@ -55,6 +58,7 @@ const (
      OptionCopyString       Options = 1 << _F_copy_string
      OptionValidateString   Options = 1 << _F_validate_string
      OptionNoValidateJSON   Options = 1 << _F_no_validate_json
+     OptionCaseSensitive    Options = 1 << _F_case_sensitive
 )
 
 func (self *Decoder) SetOptions(opts Options) {

diff --git a/decoder/decoder_native.go b/decoder/decoder_native.go
@@ -44,6 +44,7 @@ const (
     OptionCopyString       Options = api.OptionCopyString
     OptionValidateString   Options = api.OptionValidateString
     OptionNoValidateJSON   Options = api.OptionNoValidateJSON
+    OptionCaseSensitive    Options = api.OptionCaseSensitive
 )
 
 // StreamDecoder is the decoder context object for streaming input.

diff --git a/decoder/decoder_native_test.go b/decoder/decoder_native_test.go
@@ -33,6 +33,67 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+func TestDecoder_OptionCaseSensitive(t *testing.T) {
+    var js = `{"a":1,"normallllll":1,"longllllllllllllllllllllllllllllllllll":1}`
+    type TS struct{
+        A int 
+        Normallllll int 
+        Longllllllllllllllllllllllllllllllllll int 
+    }
+    var obj = TS{}
+    d := NewDecoder(js)
+    d.SetOptions(OptionCaseSensitive)
+    err := d.Decode(&obj)
+    require.NoError(t, err)
+    require.Equal(t, TS{}, obj)
+}
+
+
+func TestDecoder_MapWithIndirectElement(t *testing.T) {
+    var v map[string]struct { A [129]byte }
+    _, err := decode(`{"":{"A":[1,2,3,4,5]}}`, &v, false)
+    require.NoError(t, err)
+    assert.Equal(t, [129]byte{1, 2, 3, 4, 5}, v[""].A)
+}
+
+func TestDecoder_OptionCaseSensitiveForManyKeys(t *testing.T) {
+    var js = `{"a":1,"b":2,"C":3,"DD":4,"eE":5,"fF":6,"G":7,"H":8,"I":9,"J":10,"K":11,"L":12,"M":13}`
+    type TS struct{
+        A int
+        B int
+        C int `json:"c"`
+        Dd int `json:"dd"`
+        Ee int `json:"ee"`
+        Ff int `json:"Ff"`
+        G int `json:"g"`
+        H int `json:"h"`
+        I int `json:"i"`
+        J int `json:"j"`
+        K int `json:"k"`
+        L int `json:"l"`
+        M int `json:"m"`
+    }
+
+    {
+        var obj = TS{}
+        err := json.Unmarshal([]byte(js), &obj)
+        require.NoError(t, err)
+
+        var obj2 = TS{}
+        d := NewDecoder(js)
+        err2 := d.Decode(&obj2)
+        require.NoError(t, err2)
+        require.Equal(t, obj, obj2)
+    }
+
+    var obj = TS{}
+    d := NewDecoder(js)
+    d.SetOptions(OptionCaseSensitive)
+    err := d.Decode(&obj)
+    require.NoError(t, err)
+    require.Equal(t, TS{}, obj)
+}
+
 
 func BenchmarkSkipValidate(b *testing.B) {
     type skiptype struct {

diff --git a/decoder/decoder_test.go b/decoder/decoder_test.go
@@ -26,7 +26,6 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 )
 
 func TestMain(m *testing.M) {
@@ -287,14 +286,6 @@ func TestDecoder_Binding(t *testing.T) {
     assert.Equal(t, _BindingValue, v, 0)
 }
 
-
-func TestDecoder_MapWithIndirectElement(t *testing.T) {
-    var v map[string]struct { A [129]byte }
-    _, err := decode(`{"":{"A":[1,2,3,4,5]}}`, &v, false)
-    require.NoError(t, err)
-    assert.Equal(t, [129]byte{1, 2, 3, 4, 5}, v[""].A)
-}
-
 func BenchmarkDecoder_Generic_Sonic(b *testing.B) {
     var w interface{}
     _, _ = decode(TwitterJson, &w, true)

diff --git a/internal/decoder/api/decoder.go b/internal/decoder/api/decoder.go
@@ -35,6 +35,7 @@ const (
 	_F_use_int64 = consts.F_use_int64
 	_F_use_number = consts.F_use_number
 	_F_validate_string = consts.F_validate_string
+    _F_case_sensitive = consts.F_case_sensitive
 
 	_MaxStack = consts.MaxStack
 
@@ -45,6 +46,7 @@ const (
     OptionCopyString       = consts.OptionCopyString
     OptionValidateString   = consts.OptionValidateString
     OptionNoValidateJSON   = consts.OptionNoValidateJSON
+    OptionCaseSensitive    = consts.OptionCaseSensitive
 )
 
 type (

diff --git a/internal/decoder/consts/option.go b/internal/decoder/consts/option.go
@@ -12,11 +12,11 @@ const (
     F_disable_unknown = 3
     F_copy_string     = 4
 
-
     F_use_number      = types.B_USE_NUMBER
     F_validate_string = types.B_VALIDATE_STRING
     F_allow_control   = types.B_ALLOW_CONTROL
     F_no_validate_json = types.B_NO_VALIDATE_JSON
+    F_case_sensitive = 7
 )
 
 type Options uint64
@@ -29,6 +29,7 @@ const (
     OptionCopyString       Options = 1 << F_copy_string
     OptionValidateString   Options = 1 << F_validate_string
     OptionNoValidateJSON   Options = 1 << F_no_validate_json
+    OptionCaseSensitive    Options = 1 << F_case_sensitive
 )
 
 const (

diff --git a/internal/decoder/jitdec/assembler_regabi_amd64.go b/internal/decoder/jitdec/assembler_regabi_amd64.go
@@ -1794,13 +1794,19 @@ func (self *_Assembler) _asm_OP_struct_field(p *_Instr) {
     self.Emit("MOVQ" , _R8, _VAR_sr)                            // MOVQ    R8, sr
     self.Sjmp("JMP"  , "_end_{n}")                              // JMP     _end_{n}
     self.Link("_try_lowercase_{n}")                             // _try_lowercase_{n}:
+    self.Emit("BTQ"  , jit.Imm(_F_case_sensitive), _ARG_fv)     // check if enable option CaseSensitive
+    self.Sjmp("JC"   , "_unknown_{n}")                         
     self.Emit("MOVQ" , jit.Imm(referenceFields(p.vf())), _AX)   // MOVQ    ${p.vf()}, AX
     self.Emit("MOVQ", _ARG_sv_p, _BX)                            // MOVQ   sv, BX
     self.Emit("MOVQ", _ARG_sv_n, _CX)                            // MOVQ   sv, CX
     self.call_go(_F_FieldMap_GetCaseInsensitive)                // CALL_GO FieldMap::GetCaseInsensitive
     self.Emit("MOVQ" , _AX, _VAR_sr)                            // MOVQ    AX, _VAR_sr
     self.Emit("TESTQ", _AX, _AX)                                // TESTQ   AX, AX
     self.Sjmp("JNS"  , "_end_{n}")                              // JNS     _end_{n}
+    self.Link("_unknown_{n}")
+    // HACK: because `_VAR_sr` maybe used in `F_vstring`, so we should clear here again for `_OP_switch`.
+    self.Emit("MOVQ" , jit.Imm(-1), _AX)                        // MOVQ    $-1, AX
+    self.Emit("MOVQ" , _AX, _VAR_sr)                            // MOVQ    AX, sr
     self.Emit("BTQ"  , jit.Imm(_F_disable_unknown), _ARG_fv)    // BTQ     ${_F_disable_unknown}, fv
     self.Sjmp("JC"   , _LB_field_error)                         // JC      _field_error
     self.Link("_end_{n}")                                       // _end_{n}:

diff --git a/internal/decoder/jitdec/decoder.go b/internal/decoder/jitdec/decoder.go
@@ -27,6 +27,7 @@ const (
 	_F_use_number = consts.F_use_number
 	_F_no_validate_json = consts.F_no_validate_json
 	_F_validate_string = consts.F_validate_string
+    _F_case_sensitive = consts.F_case_sensitive
 )
 
 var (

diff --git a/internal/decoder/optdec/structs.go b/internal/decoder/optdec/structs.go
@@ -4,6 +4,7 @@ import (
 	"reflect"
 	"unsafe"
 
+	"github.com/bytedance/sonic/internal/decoder/consts"
 	caching "github.com/bytedance/sonic/internal/optcaching"
 	"github.com/bytedance/sonic/internal/resolver"
 )
@@ -38,7 +39,7 @@ func (d *structDecoder) FromDom(vp unsafe.Pointer, node Node, ctx *context) erro
 		next = val.Next()
 
 		// find field idx
-		idx := d.fieldMap.Get(key)
+		idx := d.fieldMap.Get(key, ctx.Options()&uint64(consts.OptionCaseSensitive) != 0)
         if idx == -1 {
             if Options(ctx.Options())&OptionDisableUnknown != 0 {
                 return error_field(key)

diff --git a/internal/native/neon/f32toa_arm64.s b/internal/native/neon/f32toa_arm64.s
@@ -916,7 +916,7 @@ _Digits:
 	WORD $0x37393639  // .ascii 4, '96979899'
 	WORD $0x39393839  // .ascii 4, '9899'
 	WORD $0x00000000  // .p2align 3, 0x00
-_LB_15828841: // _pow10_ceil_sig_f32.g
+_LB_3c57fe76: // _pow10_ceil_sig_f32.g
 	WORD $0x4b43fcf5; WORD $0x81ceb32c  // .quad -9093133594791772939
 	WORD $0x5e14fc32; WORD $0xa2425ff7  // .quad -6754730975062328270
 	WORD $0x359a3b3f; WORD $0xcad2f7f5  // .quad -3831727700400522433

diff --git a/internal/native/neon/f64toa_arm64.s b/internal/native/neon/f64toa_arm64.s
@@ -1232,7 +1232,7 @@ _Digits:
 	WORD $0x37393639  // .ascii 4, '96979899'
 	WORD $0x39393839  // .ascii 4, '9899'
 	  // .p2align 3, 0x00
-_LB_3b41de77: // _pow10_ceil_sig.g
+_LB_f262bdcb: // _pow10_ceil_sig.g
 	WORD $0xbebcdc4f; WORD $0xff77b1fc  // .quad -38366372719436721
 	WORD $0x13bb0f7b; WORD $0x25e8e89c  // .quad 2731688931043774331
 	WORD $0xf73609b1; WORD $0x9faacf3d  // .quad -6941508010590729807