From ab45e09a189a01771f33f4be987faf9fad5b0852 Mon Sep 17 00:00:00 2001 From: "duanyi.aster" Date: Tue, 10 Oct 2023 19:54:58 +0800 Subject: [PATCH] only support >=go1.20 --- ast/api.go | 2 + ast/api_compat.go | 116 +++++++++++ ast/api_compat_test.go | 147 -------------- ast/{api_amd64_test.go => api_native_test.go} | 2 +- ast/api_test.go | 188 ------------------ ast/encode_test.go | 166 ++++++++++++++++ 6 files changed, 285 insertions(+), 336 deletions(-) create mode 100644 ast/api_compat.go delete mode 100644 ast/api_compat_test.go rename ast/{api_amd64_test.go => api_native_test.go} (98%) delete mode 100644 ast/api_test.go diff --git a/ast/api.go b/ast/api.go index fe010bbf8..378941c1d 100644 --- a/ast/api.go +++ b/ast/api.go @@ -1,3 +1,5 @@ +// +build amd64,go1.16,!go1.22 arm64,go1.20,!go1.22 + /* * Copyright 2022 ByteDance Inc. * diff --git a/ast/api_compat.go b/ast/api_compat.go new file mode 100644 index 000000000..4ddab814c --- /dev/null +++ b/ast/api_compat.go @@ -0,0 +1,116 @@ +// +build !amd64,!arm64 go1.22 !go1.16 arm64,!go1.20 + +/* + * Copyright 2022 ByteDance Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + package ast + + import ( + `encoding/json` + `fmt` + + `github.com/bytedance/sonic/internal/native/types` + `github.com/bytedance/sonic/internal/rt` + ) + + func init() { + println("WARNING: sonic only supports Go1.16~1.20 && CPU amd64, but your environment is not suitable") + } + + func quote(buf *[]byte, val string) { + quoteString(buf, val) + } + + func unquote(src string) (string, types.ParsingError) { + sp := rt.IndexChar(src, -1) + out, ok := unquoteBytes(rt.BytesFrom(sp, len(src)+2, len(src)+2)) + if !ok { + return "", types.ERR_INVALID_ESCAPE + } + return rt.Mem2Str(out), 0 + } + + + func (self *Parser) decodeValue() (val types.JsonState) { + e, v := decodeValue(self.s, self.p, self.dbuf == nil) + if e < 0 { + return v + } + self.p = e + return v + } + + func (self *Parser) skip() (int, types.ParsingError) { + e, s := skipValue(self.s, self.p) + if e < 0 { + return self.p, types.ParsingError(-e) + } + self.p = e + return s, 0 + } + + func (self *Parser) skipFast() (int, types.ParsingError) { + e, s := skipValueFast(self.s, self.p) + if e < 0 { + return self.p, types.ParsingError(-e) + } + self.p = e + return s, 0 + } + + func (self *Node) encodeInterface(buf *[]byte) error { + out, err := json.Marshal(self.packAny()) + if err != nil { + return err + } + *buf = append(*buf, out...) + return nil + } + + func (self *Searcher) GetByPath(path ...interface{}) (Node, error) { + self.parser.p = 0 + + var err types.ParsingError + for _, p := range path { + if idx, ok := p.(int); ok && idx >= 0 { + if err = self.parser.searchIndex(idx); err != 0 { + return Node{}, self.parser.ExportError(err) + } + } else if key, ok := p.(string); ok { + if err = self.parser.searchKey(key); err != 0 { + return Node{}, self.parser.ExportError(err) + } + } else { + panic("path must be either int(>=0) or string") + } + } + + var start = self.parser.p + if start, err = self.parser.skip(); err != 0 { + return Node{}, self.parser.ExportError(err) + } + ns := len(self.parser.s) + if self.parser.p > ns || start >= ns || start>=self.parser.p { + return Node{}, fmt.Errorf("skip %d char out of json boundary", start) + } + + t := switchRawType(self.parser.s[start]) + if t == _V_NONE { + return Node{}, self.parser.ExportError(err) + } + + return newRawNode(self.parser.s[start:self.parser.p], t), nil + } \ No newline at end of file diff --git a/ast/api_compat_test.go b/ast/api_compat_test.go deleted file mode 100644 index 31690e710..000000000 --- a/ast/api_compat_test.go +++ /dev/null @@ -1,147 +0,0 @@ -//go:build !amd64 || !go1.16 || go1.22 -// +build !amd64 !go1.16 go1.22 - -/* - * Copyright 2022 ByteDance Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package ast - -import ( - `encoding/json` - `fmt` - `reflect` - `runtime` - `runtime/debug` - `testing` - - `github.com/stretchr/testify/require` -) - -func TestSortNodeTwitter(t *testing.T) { - root, err := NewSearcher(_TwitterJson).GetByPath() - if err != nil { - t.Fatal(err) - } - obj, err := root.MapUseNumber() - if err != nil { - t.Fatal(err) - } - exp, err := json.Marshal(obj) - if err != nil { - t.Fatal(err) - } - var expObj interface{} - require.NoError(t, json.Unmarshal(exp, &expObj)) - - if err := root.SortKeys(true); err != nil { - t.Fatal(err) - } - act, err := root.MarshalJSON() - if err != nil { - t.Fatal(err) - } - var actObj interface{} - require.NoError(t, json.Unmarshal(act, &actObj)) - require.Equal(t, expObj, actObj) -} - -func TestNodeAny(t *testing.T) { - empty := Node{} - _,err := empty.SetAny("any", map[string]interface{}{"a": []int{0}}) - if err != nil { - t.Fatal(err) - } - if m, err := empty.Get("any").Interface(); err != nil { - t.Fatal(err) - } else if v, ok := m.(map[string]interface{}); !ok { - t.Fatal(v) - } - if buf, err := empty.MarshalJSON(); err != nil { - t.Fatal(err) - } else if string(buf) != `{"any":{"a":[0]} -}` { - t.Fatal(string(buf)) - } - if _, err := empty.Set("any2", Node{}); err != nil { - t.Fatal(err) - } - if err := empty.Get("any2").AddAny(nil); err != nil { - t.Fatal(err) - } - if buf, err := empty.MarshalJSON(); err != nil { - t.Fatal(err) - } else if string(buf) != `{"any":{"a":[0]} -,"any2":[null -]}` { - t.Fatal(string(buf)) - } - if _, err := empty.Get("any2").SetAnyByIndex(0, NewNumber("-0.0")); err != nil { - t.Fatal(err) - } - if buf, err := empty.MarshalJSON(); err != nil { - t.Fatal(err) - } else if string(buf) != `{"any":{"a":[0]} -,"any2":[-0.0]}` { - t.Fatal(string(buf)) - } -} - - -func TestTypeCast2(t *testing.T) { - type tcase struct { - method string - node Node - exp interface{} - err error - } - var cases = []tcase{ - - {"Raw", NewAny(""), "\"\"\n", nil}, - - } - - for i, c := range cases { - fmt.Println(i, c) - rt := reflect.ValueOf(&c.node) - m := rt.MethodByName(c.method) - rets := m.Call([]reflect.Value{}) - if len(rets) != 2 { - t.Fatal(i, rets) - } - require.Equal(t, rets[0].Interface(), c.exp) - v := rets[1].Interface(); - if v != c.err { - t.Fatal(i, v) - } - } -} - -func TestStackAny(t *testing.T) { - var obj = stackObj() - any := NewAny(obj) - fmt.Printf("any: %#v\n", any) - runtime.GC() - debug.FreeOSMemory() - println("finish GC") - buf, err := any.MarshalJSON() - println("finish marshal") - if err != nil { - t.Fatal(err) - } - if string(buf) != "1\n" { - t.Fatal(string(buf)) - } -} \ No newline at end of file diff --git a/ast/api_amd64_test.go b/ast/api_native_test.go similarity index 98% rename from ast/api_amd64_test.go rename to ast/api_native_test.go index 2859abec6..af09643b5 100644 --- a/ast/api_amd64_test.go +++ b/ast/api_native_test.go @@ -1,4 +1,4 @@ -// +build amd64,go1.16,!go1.22 +// +build amd64,go1.16,!go1.22 arm64,go1.20,!go1.22 /* * Copyright 2022 ByteDance Inc. diff --git a/ast/api_test.go b/ast/api_test.go deleted file mode 100644 index 801aa49a4..000000000 --- a/ast/api_test.go +++ /dev/null @@ -1,188 +0,0 @@ -/* - * Copyright 2022 ByteDance Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package ast - -import ( - `testing` - `strings` - - `github.com/stretchr/testify/assert` -) - -type Path = []interface{} - -type testGetApi struct { - json string - path Path -} - -type checkError func(error) bool - -func isSyntaxError(err error) bool { - if err == nil { - return false - } - return strings.HasPrefix(err.Error(), `"Syntax error at index`) -} - -func isEmptySource(err error) bool { - if err == nil { - return false - } - return strings.Contains(err.Error(), "no sources available") -} - -func isErrNotExist(err error) bool { - return err == ErrNotExist -} - -func isErrUnsupportType(err error) bool { - return err == ErrUnsupportType -} - -func testSyntaxJson(t *testing.T, json string, path ...interface{}) { - search := NewSearcher(json) - _, err := search.GetByPath(path...) - assert.True(t, isSyntaxError(err)) -} - -func TestGetFromEmptyJson(t *testing.T) { - tests := []testGetApi { - { "", nil }, - { "", Path{}}, - { "", Path{""}}, - { "", Path{0}}, - { "", Path{"", ""}}, - } - for _, test := range tests { - f := func(t *testing.T) { - search := NewSearcher(test.json) - _, err := search.GetByPath(test.path...) - assert.True(t, isEmptySource(err)) - } - t.Run(test.json, f) - } -} - -func TestGetFromSyntaxError(t *testing.T) { - tests := []testGetApi { - { " \r\n\f\t", Path{} }, - { "123.", Path{} }, - { "+124", Path{} }, - { "-", Path{} }, - { "-e123", Path{} }, - { "-1.e123", Path{} }, - { "-12e456.1", Path{} }, - { "-12e.1", Path{} }, - { "[", Path{} }, - { "{", Path{} }, - { "[}", Path{} }, - { "{]", Path{} }, - { "{,}", Path{} }, - { "[,]", Path{} }, - { "tru", Path{} }, - { "fals", Path{} }, - { "nul", Path{} }, - { `{"a":"`, Path{"a"} }, - { `{"`, Path{} }, - { `"`, Path{} }, - { `"\"`, Path{} }, - { `"\\\"`, Path{} }, - { `"hello`, Path{} }, - { `{{}}`, Path{} }, - { `{[]}`, Path{} }, - { `{:,}`, Path{} }, - { `{test:error}`, Path{} }, - { `{":true}`, Path{} }, - { `{"" false}`, Path{} }, - { `{ "" : "false }`, Path{} }, - { `{"":"",}`, Path{} }, - { `{ " test : true}`, Path{} }, - { `{ "test" : tru }`, Path{} }, - { `{ "test" : true , }`, Path{} }, - { `{ {"test" : true , } }`, Path{} }, - { `{"test":1. }`, Path{} }, - { `{"\\\""`, Path{} }, - { `{"\\\"":`, Path{} }, - { `{"\\\":",""}`, Path{} }, - { `[{]`, Path{} }, - { `[tru]`, Path{} }, - { `[-1.]`, Path{} }, - { `[[]`, Path{} }, - { `[[],`, Path{} }, - { `[ true , false , [ ]`, Path{} }, - { `[true, false, [],`, Path{} }, - { `[true, false, [],]`, Path{} }, - { `{"key": [true, false, []], "key2": {{}}`, Path{} }, - } - - for _, test := range tests { - f := func(t *testing.T) { - testSyntaxJson(t, test.json, test.path...) - path := append(Path{"key"}, test.path...) - testSyntaxJson(t, `{"key":` + test.json, path...) - path = append(Path{""}, test.path...) - testSyntaxJson(t, `{"":` + test.json, path...) - path = append(Path{1}, test.path...) - testSyntaxJson(t, `["",` + test.json, path...) - } - t.Run(test.json, f) - } -} - -// NOTE: GetByPath API not validate the undemanded fields for performance. -func TestGetWithInvalidUndemandedField(t *testing.T) { - type Any = interface{} - tests := []struct { - json string - path Path - exp Any - } { - { "-0xyz", Path{}, Any(float64(-0))}, - { "-12e4xyz", Path{}, Any(float64(-12e4))}, - { "truex", Path{}, Any(true)}, - { "false,", Path{}, Any(false)}, - { `{"a":{,xxx},"b":true}`, Path{"b"}, Any(true)}, - { `{"a":[,xxx],"b":true}`, Path{"b"}, Any(true)}, - } - - for _, test := range tests { - f := func(t *testing.T) { - search := NewSearcher(test.json) - node, err := search.GetByPath(test.path...) - assert.NoError(t, err) - v, err := node.Interface() - assert.NoError(t, err) - assert.Equal(t, v, test.exp) - } - t.Run(test.json, f) - } -} - -func TestGet_InvalidPathType(t *testing.T) { - assert.Panics(t, assert.PanicTestFunc(func() { - data := `{"a":[{"b":true}]}` - s := NewSearcher(data) - s.GetByPath("a", true) - - s = NewSearcher(data) - s.GetByPath("a", nil) - - s = NewSearcher(data) - s.GetByPath("a", -1) - })) -} diff --git a/ast/encode_test.go b/ast/encode_test.go index a333b0688..250f82b30 100644 --- a/ast/encode_test.go +++ b/ast/encode_test.go @@ -21,6 +21,7 @@ import ( `runtime` `sync` `testing` + `strings` `github.com/bytedance/sonic/internal/native/types` `github.com/stretchr/testify/assert` @@ -261,3 +262,168 @@ func TestEncodeNone(t *testing.T) { require.NoError(t, err) require.Equal(t, `[null]`, string(out)) } + + +type Path = []interface{} + +type testGetApi struct { + json string + path Path +} + +type checkError func(error) bool + +func isSyntaxError(err error) bool { + if err == nil { + return false + } + return strings.HasPrefix(err.Error(), `"Syntax error at index`) +} + +func isEmptySource(err error) bool { + if err == nil { + return false + } + return strings.Contains(err.Error(), "no sources available") +} + +func isErrNotExist(err error) bool { + return err == ErrNotExist +} + +func isErrUnsupportType(err error) bool { + return err == ErrUnsupportType +} + +func testSyntaxJson(t *testing.T, json string, path ...interface{}) { + search := NewSearcher(json) + _, err := search.GetByPath(path...) + assert.True(t, isSyntaxError(err)) +} + +func TestGetFromEmptyJson(t *testing.T) { + tests := []testGetApi { + { "", nil }, + { "", Path{}}, + { "", Path{""}}, + { "", Path{0}}, + { "", Path{"", ""}}, + } + for _, test := range tests { + f := func(t *testing.T) { + search := NewSearcher(test.json) + _, err := search.GetByPath(test.path...) + assert.True(t, isEmptySource(err)) + } + t.Run(test.json, f) + } +} + +func TestGetFromSyntaxError(t *testing.T) { + tests := []testGetApi { + { " \r\n\f\t", Path{} }, + { "123.", Path{} }, + { "+124", Path{} }, + { "-", Path{} }, + { "-e123", Path{} }, + { "-1.e123", Path{} }, + { "-12e456.1", Path{} }, + { "-12e.1", Path{} }, + { "[", Path{} }, + { "{", Path{} }, + { "[}", Path{} }, + { "{]", Path{} }, + { "{,}", Path{} }, + { "[,]", Path{} }, + { "tru", Path{} }, + { "fals", Path{} }, + { "nul", Path{} }, + { `{"a":"`, Path{"a"} }, + { `{"`, Path{} }, + { `"`, Path{} }, + { `"\"`, Path{} }, + { `"\\\"`, Path{} }, + { `"hello`, Path{} }, + { `{{}}`, Path{} }, + { `{[]}`, Path{} }, + { `{:,}`, Path{} }, + { `{test:error}`, Path{} }, + { `{":true}`, Path{} }, + { `{"" false}`, Path{} }, + { `{ "" : "false }`, Path{} }, + { `{"":"",}`, Path{} }, + { `{ " test : true}`, Path{} }, + { `{ "test" : tru }`, Path{} }, + { `{ "test" : true , }`, Path{} }, + { `{ {"test" : true , } }`, Path{} }, + { `{"test":1. }`, Path{} }, + { `{"\\\""`, Path{} }, + { `{"\\\"":`, Path{} }, + { `{"\\\":",""}`, Path{} }, + { `[{]`, Path{} }, + { `[tru]`, Path{} }, + { `[-1.]`, Path{} }, + { `[[]`, Path{} }, + { `[[],`, Path{} }, + { `[ true , false , [ ]`, Path{} }, + { `[true, false, [],`, Path{} }, + { `[true, false, [],]`, Path{} }, + { `{"key": [true, false, []], "key2": {{}}`, Path{} }, + } + + for _, test := range tests { + f := func(t *testing.T) { + testSyntaxJson(t, test.json, test.path...) + path := append(Path{"key"}, test.path...) + testSyntaxJson(t, `{"key":` + test.json, path...) + path = append(Path{""}, test.path...) + testSyntaxJson(t, `{"":` + test.json, path...) + path = append(Path{1}, test.path...) + testSyntaxJson(t, `["",` + test.json, path...) + } + t.Run(test.json, f) + } +} + +// NOTE: GetByPath API not validate the undemanded fields for performance. +func TestGetWithInvalidUndemandedField(t *testing.T) { + type Any = interface{} + tests := []struct { + json string + path Path + exp Any + } { + { "-0xyz", Path{}, Any(float64(-0))}, + { "-12e4xyz", Path{}, Any(float64(-12e4))}, + { "truex", Path{}, Any(true)}, + { "false,", Path{}, Any(false)}, + { `{"a":{,xxx},"b":true}`, Path{"b"}, Any(true)}, + { `{"a":[,xxx],"b":true}`, Path{"b"}, Any(true)}, + } + + for _, test := range tests { + f := func(t *testing.T) { + search := NewSearcher(test.json) + node, err := search.GetByPath(test.path...) + assert.NoError(t, err) + v, err := node.Interface() + assert.NoError(t, err) + assert.Equal(t, v, test.exp) + } + t.Run(test.json, f) + } +} + +func TestGet_InvalidPathType(t *testing.T) { + assert.Panics(t, assert.PanicTestFunc(func() { + data := `{"a":[{"b":true}]}` + s := NewSearcher(data) + s.GetByPath("a", true) + + s = NewSearcher(data) + s.GetByPath("a", nil) + + s = NewSearcher(data) + s.GetByPath("a", -1) + })) +}