From 8396b0624de4baf09b7b59b12e00a2d92610122f Mon Sep 17 00:00:00 2001 From: "duanyi.aster" Date: Sun, 21 Jan 2024 22:06:12 +0800 Subject: [PATCH] fix: skip number may contains space --- ast/api_amd64.go | 50 +++++++--------------- ast/api_compat.go | 103 +++++----------------------------------------- ast/decode.go | 102 +++++++++++++++++++++++++++++++++++++++++---- ast/node.go | 5 +++ ast/raw.go | 12 +++--- ast/search.go | 32 ++++++-------- 6 files changed, 142 insertions(+), 162 deletions(-) diff --git a/ast/api_amd64.go b/ast/api_amd64.go index d08f1c2da..4631746ed 100644 --- a/ast/api_amd64.go +++ b/ast/api_amd64.go @@ -121,24 +121,32 @@ func (self *Parser) skipFast() (int, types.ParsingError) { return start, 0 } -func (self *Parser) getByPath(path ...interface{}) (int, types.ParsingError) { +func (self *Parser) getByPath(path ...interface{}) (int, types.ValueType, types.ParsingError) { fsm := types.NewStateMachine() start := native.GetByPath(&self.s, &self.p, &path, fsm) types.FreeStateMachine(fsm) runtime.KeepAlive(path) if start < 0 { - return self.p, types.ParsingError(-start) + return self.p, 0, types.ParsingError(-start) } - return start, 0 + t := switchRawType(self.s[start]) + if t == _V_NUMBER { + self.p = 1 + backward(self.s, self.p-1) + } + return start, t, 0 } -func (self *Parser) getByPathNoValidate(path ...interface{}) (int, types.ParsingError) { +func (self *Parser) getByPathNoValidate(path ...interface{}) (int, types.ValueType, types.ParsingError) { start := native.GetByPath(&self.s, &self.p, &path, nil) runtime.KeepAlive(path) if start < 0 { - return self.p, types.ParsingError(-start) + return self.p, 0, types.ParsingError(-start) } - return start, 0 + t := switchRawType(self.s[start]) + if t == _V_NUMBER { + self.p = 1 + backward(self.s, self.p-1) + } + return start, t, 0 } func DecodeString(src string, pos int) (ret int, v string) { @@ -163,33 +171,3 @@ func DecodeString(src string, pos int) (ret int, v string) { return -int(_ERR_UNSUPPORT_TYPE), "" } } - -func DecodeInt64(src string, pos int) (ret int, v int64, err error) { - p := NewParserObj(src) - p.p = pos - p.decodeNumber(true) - val := p.decodeValue() - p.decodeNumber(false) - switch val.Vt { - case types.V_INTEGER: - return p.p, val.Iv, nil - default: - return -int(_ERR_UNSUPPORT_TYPE), 0, nil - } -} - -func DecodeFloat64(src string, pos int) (ret int, v float64, err error) { - p := NewParserObj(src) - p.p = pos - p.decodeNumber(true) - val := p.decodeValue() - p.decodeNumber(false) - switch val.Vt { - case types.V_DOUBLE: - return p.p, val.Dv, nil - case types.V_INTEGER: - return p.p, float64(val.Iv), nil - default: - return -int(_ERR_UNSUPPORT_TYPE), 0, nil - } -} \ No newline at end of file diff --git a/ast/api_compat.go b/ast/api_compat.go index 039561a18..0ee3d4bca 100644 --- a/ast/api_compat.go +++ b/ast/api_compat.go @@ -23,7 +23,6 @@ import ( "encoding/base64" "encoding/json" "runtime" - "strconv" "unsafe" "github.com/bytedance/sonic/internal/native/types" @@ -91,15 +90,15 @@ func (self *Node) encodeInterface(buf *[]byte) error { return nil } -func (self *Parser) getByPath(path ...interface{}) (int, types.ParsingError) { +func (self *Parser) getByPath(path ...interface{}) (int, types.ValueType, types.ParsingError) { for _, p := range path { if idx, ok := p.(int); ok && idx >= 0 { if _, err := self.searchIndex(idx); err != 0 { - return self.p, err + return self.p, 0, err } } else if key, ok := p.(string); ok { if _, err := self.searchKey(key); err != 0 { - return self.p, err + return self.p, 0, err } } else { panic("path must be either int(>=0) or string") @@ -107,12 +106,16 @@ func (self *Parser) getByPath(path ...interface{}) (int, types.ParsingError) { } start, e := self.skip() if e != 0 { - return self.p, e + return self.p, 0, e } - return start, 0 + t := switchRawType(self.s[start]) + if t == _V_NUMBER { + self.p = 1 + backward(self.s, self.p-1) + } + return start, t, 0 } -func (self *Parser) getByPathNoValidate(path ...interface{}) (int, types.ParsingError) { +func (self *Parser) getByPathNoValidate(path ...interface{}) (int, types.ValueType, types.ParsingError) { return self.getByPath(path...) } @@ -133,89 +136,3 @@ func DecodeString(src string, pos int) (ret int, v string) { runtime.KeepAlive(src) return ret, rt.Mem2Str(vv) } - -//go:nocheckptr -func DecodeInt64(src string, pos int) (ret int, v int64, err error) { - sp := uintptr(rt.IndexChar(src, pos)) - ss := uintptr(sp) - se := uintptr(rt.IndexChar(src, len(src))) - if uintptr(sp) >= se { - return -int(types.ERR_EOF), 0, nil - } - - if c := *(*byte)(unsafe.Pointer(sp)); c == '-' { - sp += 1 - } - if sp == se { - return -int(types.ERR_EOF), 0, nil - } - - for ; sp < se; sp += uintptr(1) { - if !isDigit(*(*byte)(unsafe.Pointer(sp))) { - break - } - } - - if sp < se { - if c := *(*byte)(unsafe.Pointer(sp)); c == '.' || c == 'e' || c == 'E' { - return -int(types.ERR_INVALID_NUMBER_FMT), 0, nil - } - } - - var vv string - ret = int(uintptr(sp) - uintptr((*rt.GoString)(unsafe.Pointer(&src)).Ptr)) - (*rt.GoString)(unsafe.Pointer(&vv)).Ptr = unsafe.Pointer(ss) - (*rt.GoString)(unsafe.Pointer(&vv)).Len = ret - pos - - v, err = strconv.ParseInt(vv, 10, 64) - if err != nil { - //NOTICE: allow overflow here - if err.(*strconv.NumError).Err == strconv.ErrRange { - return ret, 0, err - } - return -int(types.ERR_INVALID_CHAR), 0, err - } - - runtime.KeepAlive(src) - return ret, v, nil -} - -//go:nocheckptr -func DecodeFloat64(src string, pos int) (ret int, v float64, err error) { - sp := uintptr(rt.IndexChar(src, pos)) - ss := uintptr(sp) - se := uintptr(rt.IndexChar(src, len(src))) - if uintptr(sp) >= se { - return -int(types.ERR_EOF), 0, nil - } - - if c := *(*byte)(unsafe.Pointer(sp)); c == '-' { - sp += 1 - } - if sp == se { - return -int(types.ERR_EOF), 0, nil - } - - for ; sp < se; sp += uintptr(1) { - if !isNumberChars(*(*byte)(unsafe.Pointer(sp))) { - break - } - } - - var vv string - ret = int(uintptr(sp) - uintptr((*rt.GoString)(unsafe.Pointer(&src)).Ptr)) - (*rt.GoString)(unsafe.Pointer(&vv)).Ptr = unsafe.Pointer(ss) - (*rt.GoString)(unsafe.Pointer(&vv)).Len = ret - pos - - v, err = strconv.ParseFloat(vv, 64) - if err != nil { - //NOTICE: allow overflow here - if err.(*strconv.NumError).Err == strconv.ErrRange { - return ret, 0, err - } - return -int(types.ERR_INVALID_CHAR), 0, err - } - - runtime.KeepAlive(src) - return ret, v, nil -} diff --git a/ast/decode.go b/ast/decode.go index 7f23484b5..958f6299c 100644 --- a/ast/decode.go +++ b/ast/decode.go @@ -17,12 +17,13 @@ package ast import ( - `encoding/base64` - `runtime` - `unsafe` + "encoding/base64" + "runtime" + "strconv" + "unsafe" - `github.com/bytedance/sonic/internal/native/types` - `github.com/bytedance/sonic/internal/rt` + "github.com/bytedance/sonic/internal/native/types" + "github.com/bytedance/sonic/internal/rt" ) const _blankCharsMask = (1 << ' ') | (1 << '\t') | (1 << '\r') | (1 << '\n') @@ -160,14 +161,14 @@ func decodeValue(src string, pos int, skipnum bool) (ret int, v types.JsonState) } } else { var iv int64 - ret, iv, _ = DecodeInt64(src, pos) + ret, iv, _ = decodeInt64(src, pos) if ret >= 0 { return ret, types.JsonState{Vt: types.V_INTEGER, Iv: iv, Ep: pos} } else if ret != -int(types.ERR_INVALID_NUMBER_FMT) { return ret, types.JsonState{Vt: types.ValueType(ret)} } var fv float64 - ret, fv, _ = DecodeFloat64(src, pos) + ret, fv, _ = decodeFloat64(src, pos) if ret >= 0 { return ret, types.JsonState{Vt: types.V_DOUBLE, Dv: fv, Ep: pos} } else { @@ -478,3 +479,90 @@ func skipArray(src string, pos int) (ret int, start int) { pos++ } } + + +//go:nocheckptr +func decodeInt64(src string, pos int) (ret int, v int64, err error) { + sp := uintptr(rt.IndexChar(src, pos)) + ss := uintptr(sp) + se := uintptr(rt.IndexChar(src, len(src))) + if uintptr(sp) >= se { + return -int(types.ERR_EOF), 0, nil + } + + if c := *(*byte)(unsafe.Pointer(sp)); c == '-' { + sp += 1 + } + if sp == se { + return -int(types.ERR_EOF), 0, nil + } + + for ; sp < se; sp += uintptr(1) { + if !isDigit(*(*byte)(unsafe.Pointer(sp))) { + break + } + } + + if sp < se { + if c := *(*byte)(unsafe.Pointer(sp)); c == '.' || c == 'e' || c == 'E' { + return -int(types.ERR_INVALID_NUMBER_FMT), 0, nil + } + } + + var vv string + ret = int(uintptr(sp) - uintptr((*rt.GoString)(unsafe.Pointer(&src)).Ptr)) + (*rt.GoString)(unsafe.Pointer(&vv)).Ptr = unsafe.Pointer(ss) + (*rt.GoString)(unsafe.Pointer(&vv)).Len = ret - pos + + v, err = strconv.ParseInt(vv, 10, 64) + if err != nil { + //NOTICE: allow overflow here + if err.(*strconv.NumError).Err == strconv.ErrRange { + return ret, 0, err + } + return -int(types.ERR_INVALID_CHAR), 0, err + } + + runtime.KeepAlive(src) + return ret, v, nil +} + +//go:nocheckptr +func decodeFloat64(src string, pos int) (ret int, v float64, err error) { + sp := uintptr(rt.IndexChar(src, pos)) + ss := uintptr(sp) + se := uintptr(rt.IndexChar(src, len(src))) + if uintptr(sp) >= se { + return -int(types.ERR_EOF), 0, nil + } + + if c := *(*byte)(unsafe.Pointer(sp)); c == '-' { + sp += 1 + } + if sp == se { + return -int(types.ERR_EOF), 0, nil + } + + for ; sp < se; sp += uintptr(1) { + if !isNumberChars(*(*byte)(unsafe.Pointer(sp))) { + break + } + } + + var vv string + ret = int(uintptr(sp) - uintptr((*rt.GoString)(unsafe.Pointer(&src)).Ptr)) + (*rt.GoString)(unsafe.Pointer(&vv)).Ptr = unsafe.Pointer(ss) + (*rt.GoString)(unsafe.Pointer(&vv)).Len = ret - pos + + v, err = strconv.ParseFloat(vv, 64) + if err != nil { + //NOTICE: allow overflow here + if err.(*strconv.NumError).Err == strconv.ErrRange { + return ret, 0, err + } + return -int(types.ERR_INVALID_CHAR), 0, err + } + + runtime.KeepAlive(src) + return ret, v, nil +} diff --git a/ast/node.go b/ast/node.go index 30ff4a4e8..25ecd25f0 100644 --- a/ast/node.go +++ b/ast/node.go @@ -1800,3 +1800,8 @@ var typeJumpTable = [256]types.ValueType{ func switchRawType(c byte) types.ValueType { return typeJumpTable[c] } + +func backward(src string, i int) int { + for ; i>=0 && isSpace(src[i]); i-- {} + return i +} \ No newline at end of file diff --git a/ast/raw.go b/ast/raw.go index 5bcc67f0d..7096dcb50 100644 --- a/ast/raw.go +++ b/ast/raw.go @@ -209,11 +209,11 @@ func (self Value) GetByPath(path ...interface{}) Value { return self } p := NewParserObj(self.js) - s, e := p.getByPathNoValidate(path...) + s, t, e := p.getByPathNoValidate(path...) if e != 0 { return errValue(p.ExportError(e)) } - return value(self.js[s:p.p]) + return Value{int(t), self.js[s:p.p]} } // SetAnyByPath set value on given path and create nodes on the json if not exist @@ -393,11 +393,11 @@ func (self Value) Get(key string) Value { return self } p := NewParserObj(self.js) - s, e := p.getByPathNoValidate(key) + s, t, e := p.getByPathNoValidate(key) if e != 0 { return errValue(p.ExportError(e)) } - return value(self.js[s:p.p]) + return Value{int(t), self.js[s:p.p]} } // GetMany retrieves all the keys in kvs and set found Value at correpsonding index @@ -474,11 +474,11 @@ func (self Value) Index(idx int) Value { return self } p := NewParserObj(self.js) - s, e := p.getByPathNoValidate(idx) + s, t, e := p.getByPathNoValidate(idx) if e != 0 { return errValue(p.ExportError(e)) } - return value(self.js[s:p.p]) + return Value{int(t), self.js[s:p.p]} } // GetMany retrieves all the indexes in ids and set found Value at correpsonding index of vals diff --git a/ast/search.go b/ast/search.go index b0256ff1a..1d6a4892b 100644 --- a/ast/search.go +++ b/ast/search.go @@ -54,12 +54,13 @@ func (self *Searcher) Copy(enable bool) { func (self *Searcher) GetByPath(path ...interface{}) (Node, error) { var err types.ParsingError var start int + var t types.ValueType self.parser.p = 0 if self.noValidate { - start, err = self.parser.getByPathNoValidate(path...) + start, t, err = self.parser.getByPathNoValidate(path...) } else { - start, err = self.parser.getByPath(path...) + start, t, err = self.parser.getByPath(path...) } if err != 0 { // for compatibility with old version @@ -72,11 +73,6 @@ func (self *Searcher) GetByPath(path ...interface{}) (Node, error) { return Node{}, self.parser.syntaxError(err) } - t := switchRawType(self.parser.s[start]) - if t == _V_NONE { - return Node{}, self.parser.ExportError(err) - } - var raw string if self.copy { raw = rt.Mem2Str([]byte(self.parser.s[start:self.parser.p])) @@ -87,13 +83,13 @@ func (self *Searcher) GetByPath(path ...interface{}) (Node, error) { } // Export Search API for other libs -func GetByPath(src string, path ...interface{}) (int, int, error) { +func GetByPath(src string, path ...interface{}) (int, int, int, error) { p := NewParserObj(src) - s, e := p.getByPathNoValidate(path...) + s, t, e := p.getByPathNoValidate(path...) if e != 0 { - return -1, -1, p.ExportError(e) + return -1, -1, 0, p.ExportError(e) } - return s, p.p, nil + return s, p.p, int(t), nil } // GetValueByPath searches in the json and located a Value at path @@ -106,22 +102,18 @@ func (self *Searcher) GetValueByPath(path ...interface{}) (Value, error) { self.parser.p = 0 var s int var err types.ParsingError + var t types.ValueType + if self.noValidate { - s, err = self.parser.getByPathNoValidate(path...) + s, t, err = self.parser.getByPathNoValidate(path...) } else { - s, err = self.parser.getByPath(path...) + s, t, err = self.parser.getByPath(path...) } if err != 0 { e := self.parser.ExportError(err) return errValue(e), e } - t := switchRawType(self.parser.s[s]) - if t == _V_NONE { - e := self.parser.ExportError(err) - return errValue(e), e - } - var raw string if self.copy { raw = rt.Mem2Str([]byte(self.parser.s[s:self.parser.p])) @@ -140,7 +132,7 @@ func (self *Searcher) SetValueByPath(val Value, path ...interface{}) (string, er } self.parser.p = 0 - s, err := self.parser.getByPathNoValidate(path...) + s, _, err := self.parser.getByPathNoValidate(path...) if err != 0 { if err != _ERR_NOT_FOUND {