fix: skip number may contains space

bytedance · Jan 21, 2024 · 8396b06 · 8396b06
1 parent 9142661
commit 8396b06
Show file tree

Hide file tree

Showing 6 changed files with 142 additions and 162 deletions.
diff --git a/ast/api_amd64.go b/ast/api_amd64.go
@@ -121,24 +121,32 @@ func (self *Parser) skipFast() (int, types.ParsingError) {
     return start, 0
 }
 
-func (self *Parser) getByPath(path ...interface{}) (int, types.ParsingError) {
+func (self *Parser) getByPath(path ...interface{}) (int, types.ValueType, types.ParsingError) {
     fsm := types.NewStateMachine()
     start := native.GetByPath(&self.s, &self.p, &path, fsm)
     types.FreeStateMachine(fsm)
     runtime.KeepAlive(path)
     if start < 0 {
-        return self.p, types.ParsingError(-start)
+        return self.p, 0, types.ParsingError(-start)
     }
-    return start, 0
+    t := switchRawType(self.s[start])
+    if t == _V_NUMBER {
+        self.p = 1 + backward(self.s, self.p-1)
+    }
+    return start, t, 0
 }
 
-func (self *Parser) getByPathNoValidate(path ...interface{}) (int, types.ParsingError) {
+func (self *Parser) getByPathNoValidate(path ...interface{}) (int, types.ValueType, types.ParsingError) {
     start := native.GetByPath(&self.s, &self.p, &path, nil)
     runtime.KeepAlive(path)
     if start < 0 {
-        return self.p, types.ParsingError(-start)
+        return self.p, 0, types.ParsingError(-start)
     }
-    return start, 0
+    t := switchRawType(self.s[start])
+    if t == _V_NUMBER {
+        self.p = 1 + backward(self.s, self.p-1)
+    }
+    return start, t, 0
 }
 
 func DecodeString(src string, pos int) (ret int, v string) {
@@ -163,33 +171,3 @@ func DecodeString(src string, pos int) (ret int, v string) {
 		return -int(_ERR_UNSUPPORT_TYPE), ""
 	}
 }
-
-func DecodeInt64(src string, pos int) (ret int, v int64, err error) {
-    p := NewParserObj(src)
-    p.p = pos
-    p.decodeNumber(true)
-    val := p.decodeValue()
-    p.decodeNumber(false)
-    switch val.Vt {
-    case types.V_INTEGER:
-        return p.p, val.Iv, nil
-    default:
-        return -int(_ERR_UNSUPPORT_TYPE), 0, nil
-    }
-}
-
-func DecodeFloat64(src string, pos int) (ret int, v float64, err error) {
-    p := NewParserObj(src)
-    p.p = pos
-    p.decodeNumber(true)
-    val := p.decodeValue()
-    p.decodeNumber(false)
-    switch val.Vt {
-    case types.V_DOUBLE:
-        return p.p, val.Dv, nil
-    case types.V_INTEGER:
-        return p.p, float64(val.Iv), nil
-    default:
-        return -int(_ERR_UNSUPPORT_TYPE), 0, nil
-    }
-}
diff --git a/ast/api_compat.go b/ast/api_compat.go
@@ -23,7 +23,6 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"runtime"
-	"strconv"
 	"unsafe"
 
 	"github.com/bytedance/sonic/internal/native/types"
@@ -91,28 +90,32 @@ func (self *Node) encodeInterface(buf *[]byte) error {
     return nil
 }
 
-func (self *Parser) getByPath(path ...interface{}) (int, types.ParsingError) {
+func (self *Parser) getByPath(path ...interface{}) (int, types.ValueType, types.ParsingError) {
     for _, p := range path {
         if idx, ok := p.(int); ok && idx >= 0 {
             if _, err := self.searchIndex(idx); err != 0 {
-                return self.p, err
+                return self.p, 0, err
             }
         } else if key, ok := p.(string); ok {
             if _, err := self.searchKey(key); err != 0 {
-                return self.p, err
+                return self.p, 0, err
             }
         } else {
             panic("path must be either int(>=0) or string")
         }
     }
     start, e := self.skip()
     if e != 0 {
-        return self.p, e
+        return self.p, 0, e
     }
-    return start, 0
+    t := switchRawType(self.s[start])
+    if t == _V_NUMBER {
+        self.p = 1 + backward(self.s, self.p-1)
+    }
+    return start, t, 0
 }
 
-func (self *Parser) getByPathNoValidate(path ...interface{}) (int, types.ParsingError) {
+func (self *Parser) getByPathNoValidate(path ...interface{}) (int, types.ValueType, types.ParsingError) {
     return self.getByPath(path...)
 }
 
@@ -133,89 +136,3 @@ func DecodeString(src string, pos int) (ret int, v string) {
     runtime.KeepAlive(src)
     return ret, rt.Mem2Str(vv)
 }
-
-//go:nocheckptr
-func DecodeInt64(src string, pos int) (ret int, v int64, err error) {
-    sp := uintptr(rt.IndexChar(src, pos))
-    ss := uintptr(sp)
-    se := uintptr(rt.IndexChar(src, len(src)))
-    if uintptr(sp) >= se {
-        return -int(types.ERR_EOF), 0, nil
-    }
-
-    if c := *(*byte)(unsafe.Pointer(sp)); c == '-' {
-        sp += 1
-    }
-    if sp == se {
-        return -int(types.ERR_EOF), 0, nil
-    }
-
-    for ; sp < se; sp += uintptr(1) {
-        if !isDigit(*(*byte)(unsafe.Pointer(sp))) {
-            break
-        }
-    }
-
-    if sp < se {
-        if c := *(*byte)(unsafe.Pointer(sp)); c == '.' || c == 'e' || c == 'E' {
-            return -int(types.ERR_INVALID_NUMBER_FMT), 0, nil
-        }
-    }
-
-    var vv string
-    ret = int(uintptr(sp) - uintptr((*rt.GoString)(unsafe.Pointer(&src)).Ptr))
-    (*rt.GoString)(unsafe.Pointer(&vv)).Ptr = unsafe.Pointer(ss)
-    (*rt.GoString)(unsafe.Pointer(&vv)).Len = ret - pos
-
-    v, err = strconv.ParseInt(vv, 10, 64)
-    if err != nil {
-        //NOTICE: allow overflow here
-        if err.(*strconv.NumError).Err == strconv.ErrRange {
-            return ret, 0, err
-        }
-        return -int(types.ERR_INVALID_CHAR), 0, err
-    }
-
-    runtime.KeepAlive(src)
-    return ret, v, nil
-}
-
-//go:nocheckptr
-func DecodeFloat64(src string, pos int) (ret int, v float64, err error) {
-    sp := uintptr(rt.IndexChar(src, pos))
-    ss := uintptr(sp)
-    se := uintptr(rt.IndexChar(src, len(src)))
-    if uintptr(sp) >= se {
-        return -int(types.ERR_EOF), 0, nil
-    }
-
-    if c := *(*byte)(unsafe.Pointer(sp)); c == '-' {
-        sp += 1
-    }
-    if sp == se {
-        return -int(types.ERR_EOF), 0, nil
-    }
-
-    for ; sp < se; sp += uintptr(1) {
-        if !isNumberChars(*(*byte)(unsafe.Pointer(sp))) {
-            break
-        }
-    }
-
-    var vv string
-    ret = int(uintptr(sp) - uintptr((*rt.GoString)(unsafe.Pointer(&src)).Ptr))
-    (*rt.GoString)(unsafe.Pointer(&vv)).Ptr = unsafe.Pointer(ss)
-    (*rt.GoString)(unsafe.Pointer(&vv)).Len = ret - pos
-
-    v, err = strconv.ParseFloat(vv, 64)
-    if err != nil {
-        //NOTICE: allow overflow here
-        if err.(*strconv.NumError).Err == strconv.ErrRange {
-            return ret, 0, err
-        }
-        return -int(types.ERR_INVALID_CHAR), 0, err
-    }
-
-    runtime.KeepAlive(src)
-    return ret, v, nil
-}
diff --git a/ast/decode.go b/ast/decode.go
@@ -17,12 +17,13 @@
 package ast
 
 import (
-    `encoding/base64`
-    `runtime`
-    `unsafe`
+	"encoding/base64"
+	"runtime"
+	"strconv"
+	"unsafe"
 
-    `github.com/bytedance/sonic/internal/native/types`
-    `github.com/bytedance/sonic/internal/rt`
+	"github.com/bytedance/sonic/internal/native/types"
+	"github.com/bytedance/sonic/internal/rt"
 )
 
 const _blankCharsMask = (1 << ' ') | (1 << '\t') | (1 << '\r') | (1 << '\n')
@@ -160,14 +161,14 @@ func decodeValue(src string, pos int, skipnum bool) (ret int, v types.JsonState)
             }
         } else {
             var iv int64
-            ret, iv, _ = DecodeInt64(src, pos)
+            ret, iv, _ = decodeInt64(src, pos)
             if ret >= 0 {
                 return ret, types.JsonState{Vt: types.V_INTEGER, Iv: iv, Ep: pos}
             } else if ret != -int(types.ERR_INVALID_NUMBER_FMT) {
                 return ret, types.JsonState{Vt: types.ValueType(ret)}
             }
             var fv float64
-            ret, fv, _ = DecodeFloat64(src, pos)
+            ret, fv, _ = decodeFloat64(src, pos)
             if ret >= 0 {
                 return ret, types.JsonState{Vt: types.V_DOUBLE, Dv: fv, Ep: pos}
             } else {
@@ -478,3 +479,90 @@ func skipArray(src string, pos int) (ret int, start int) {
         pos++
     }
 }
+
+
+//go:nocheckptr
+func decodeInt64(src string, pos int) (ret int, v int64, err error) {
+    sp := uintptr(rt.IndexChar(src, pos))
+    ss := uintptr(sp)
+    se := uintptr(rt.IndexChar(src, len(src)))
+    if uintptr(sp) >= se {
+        return -int(types.ERR_EOF), 0, nil
+    }
+
+    if c := *(*byte)(unsafe.Pointer(sp)); c == '-' {
+        sp += 1
+    }
+    if sp == se {
+        return -int(types.ERR_EOF), 0, nil
+    }
+
+    for ; sp < se; sp += uintptr(1) {
+        if !isDigit(*(*byte)(unsafe.Pointer(sp))) {
+            break
+        }
+    }
+
+    if sp < se {
+        if c := *(*byte)(unsafe.Pointer(sp)); c == '.' || c == 'e' || c == 'E' {
+            return -int(types.ERR_INVALID_NUMBER_FMT), 0, nil
+        }
+    }
+
+    var vv string
+    ret = int(uintptr(sp) - uintptr((*rt.GoString)(unsafe.Pointer(&src)).Ptr))
+    (*rt.GoString)(unsafe.Pointer(&vv)).Ptr = unsafe.Pointer(ss)
+    (*rt.GoString)(unsafe.Pointer(&vv)).Len = ret - pos
+
+    v, err = strconv.ParseInt(vv, 10, 64)
+    if err != nil {
+        //NOTICE: allow overflow here
+        if err.(*strconv.NumError).Err == strconv.ErrRange {
+            return ret, 0, err
+        }
+        return -int(types.ERR_INVALID_CHAR), 0, err
+    }
+
+    runtime.KeepAlive(src)
+    return ret, v, nil
+}
+
+//go:nocheckptr
+func decodeFloat64(src string, pos int) (ret int, v float64, err error) {
+    sp := uintptr(rt.IndexChar(src, pos))
+    ss := uintptr(sp)
+    se := uintptr(rt.IndexChar(src, len(src)))
+    if uintptr(sp) >= se {
+        return -int(types.ERR_EOF), 0, nil
+    }
+
+    if c := *(*byte)(unsafe.Pointer(sp)); c == '-' {
+        sp += 1
+    }
+    if sp == se {
+        return -int(types.ERR_EOF), 0, nil
+    }
+
+    for ; sp < se; sp += uintptr(1) {
+        if !isNumberChars(*(*byte)(unsafe.Pointer(sp))) {
+            break
+        }
+    }
+
+    var vv string
+    ret = int(uintptr(sp) - uintptr((*rt.GoString)(unsafe.Pointer(&src)).Ptr))
+    (*rt.GoString)(unsafe.Pointer(&vv)).Ptr = unsafe.Pointer(ss)
+    (*rt.GoString)(unsafe.Pointer(&vv)).Len = ret - pos
+
+    v, err = strconv.ParseFloat(vv, 64)
+    if err != nil {
+        //NOTICE: allow overflow here
+        if err.(*strconv.NumError).Err == strconv.ErrRange {
+            return ret, 0, err
+        }
+        return -int(types.ERR_INVALID_CHAR), 0, err
+    }
+
+    runtime.KeepAlive(src)
+    return ret, v, nil
+}
diff --git a/ast/node.go b/ast/node.go
@@ -1800,3 +1800,8 @@ var typeJumpTable = [256]types.ValueType{
 func switchRawType(c byte) types.ValueType {
     return typeJumpTable[c]
 }
+
+func backward(src string, i int) int {
+    for ; i>=0 && isSpace(src[i]); i-- {}
+    return i
+}
diff --git a/ast/raw.go b/ast/raw.go
@@ -209,11 +209,11 @@ func (self Value) GetByPath(path ...interface{}) Value {
 		return self
 	}
 	p := NewParserObj(self.js)
-	s, e := p.getByPathNoValidate(path...)
+	s, t, e := p.getByPathNoValidate(path...)
 	if e != 0 {
 		return errValue(p.ExportError(e))
 	}
-	return value(self.js[s:p.p])
+	return Value{int(t), self.js[s:p.p]}
 }
 
 // SetAnyByPath set value on given path and create nodes on the json if not exist
@@ -393,11 +393,11 @@ func (self Value) Get(key string) Value {
 		return self
 	}
 	p := NewParserObj(self.js)
-	s, e := p.getByPathNoValidate(key)
+	s, t, e := p.getByPathNoValidate(key)
 	if e != 0 {
 		return errValue(p.ExportError(e))
 	}
-	return value(self.js[s:p.p])
+	return Value{int(t), self.js[s:p.p]}
 }
 
 // GetMany retrieves all the keys in kvs and set found Value at correpsonding index
@@ -474,11 +474,11 @@ func (self Value) Index(idx int) Value {
 		return self
 	}
 	p := NewParserObj(self.js)
-	s, e := p.getByPathNoValidate(idx)
+	s, t, e := p.getByPathNoValidate(idx)
 	if e != 0 {
 		return errValue(p.ExportError(e))
 	}
-	return value(self.js[s:p.p])
+	return Value{int(t), self.js[s:p.p]}
 }
 
 // GetMany retrieves all the indexes in ids and set found Value at correpsonding index of vals