diff --git a/benchmark/jsonresultset/.gitignore b/benchmark/jsonresultset/.gitignore deleted file mode 100644 index 5901ab5cd..000000000 --- a/benchmark/jsonresultset/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -*.test -*.out -prof_largeresults diff --git a/benchmark/jsonresultset/Makefile b/benchmark/jsonresultset/Makefile deleted file mode 100644 index d8cfc7a9b..000000000 --- a/benchmark/jsonresultset/Makefile +++ /dev/null @@ -1,35 +0,0 @@ -## Setup -SHELL := /bin/bash -SRC = $(shell find . -type f -name '*.go' -not -path "./vendor/*") - -setup: - go install golang.org/x/lint/golint@latest - go install github.com/Songmu/make2help/cmd/make2help@latest - -## Benchmark -profile: - go test -tags=sfdebug -v -run none -bench . -benchtime 3s -benchmem -cpuprofile cpu.out -memprofile mem.out -stderrthreshold=INFO -vmodule=*=2 - @echo "For CPU usage, run 'go tool pprof jsonresultset.test cpu.out'" - @echo "For Memory usage, run 'go tool pprof -alloc_space jsonresultset.test mem.out'" - -## Trace -trace: - go test -trace trace.out - @echo "Run 'go tool trace jsonresultset.test trace.out'" - -## Lint -lint: setup - go vet $(SRC) - for pkg in $$(go list ./... | grep -v vendor); do \ - golint -set_exit_status $$pkg || exit $$?; \ - done - -## Format source codes using gfmt -fmt: setup - @gofmt -l -w $(SRC) - -## Show help -help: - @make2help $(MAKEFILE_LIST) - -.PHONY: install run diff --git a/benchmark/jsonresultset/README.rst b/benchmark/jsonresultset/README.rst deleted file mode 100644 index 6593e269d..000000000 --- a/benchmark/jsonresultset/README.rst +++ /dev/null @@ -1,60 +0,0 @@ -******************************************************************************** -Benchmark Large Result Set -******************************************************************************** - -This folder includes a benchmark test case for "JSON Result Set", which refers -to a query result of more than 100 MB of JSON objects. This differs from the "Large -Result Set" case, since it benchmarks large strings with many escaped characters. - -Profiling -========= - -Using Go's profilers, you may see CPU and memory usage on each function/method. -This command instruments CPU and memory usage and save them into files. - -.. code-block:: bash - - SNOWFLAKE_TEST_ACCOUNT= \ - SNOWFLAKE_TEST_USER= \ - SNOWFLAKE_TEST_PASSWORD= \ - SNOWFLAKE_TEST_CUSTOME_JSON_DECODER_ENABLE= \ - SNOWFLAKE_TEST_MAX_CHUNK_DOWNLOAD_WORKERS= \ - make profile - -Check CPU usage on the web browser: - -.. code-block:: bash - - go tool pprof jsonresultset.test cpu.out - (pprof) web - -Check memory usage on the web browser: - -.. code-block:: bash - - go tool pprof -alloc_space jsonresultset.test mem.out - (pprof) web - -Note adjust SNOWFLAKE_TEST_CUSTOME_JSON_DECODER_ENABLE and SNOWFLAKE_TEST_MAX_CHUNK_DOWNLOAD_WORKERS to - -Tracing -======= - -Using Go's trace tool, you may see all of the goroutine's activity with timeline. - -.. code-block:: bash - - SNOWFLAKE_TEST_ACCOUNT= \ - SNOWFLAKE_TEST_USER= \ - SNOWFLAKE_TEST_PASSWORD= \ - SNOWFLAKE_TEST_CUSTOME_JSON_DECODER_ENABLE= \ - SNOWFLAKE_TEST_MAX_CHUNK_DOWNLOAD_WORKERS= \ - make trace - -Check goroutine's activities on web browser. - -.. code-block:: bash - - go tool trace jsonresultset.test trace.out - - diff --git a/benchmark/jsonresultset/jsonresultset.go b/benchmark/jsonresultset/jsonresultset.go deleted file mode 100644 index 4c10d309b..000000000 --- a/benchmark/jsonresultset/jsonresultset.go +++ /dev/null @@ -1,7 +0,0 @@ -// Package jsonresultset is a benchmark for large json result sets -// -// This exists to mitigate "no non-test Go files" in the latest Go -package jsonresultset - -func main() { -} diff --git a/benchmark/jsonresultset/jsonresultset_test.go b/benchmark/jsonresultset/jsonresultset_test.go deleted file mode 100644 index 6654d5923..000000000 --- a/benchmark/jsonresultset/jsonresultset_test.go +++ /dev/null @@ -1,117 +0,0 @@ -// This code is to profile a large json result set query. It is basically similar to selectmany example code but -// leverages benchmark framework. -package jsonresultset - -import ( - "flag" - "log" - _ "net/http/pprof" - "os" - "strconv" - "strings" - "testing" - - "database/sql" - - "context" - "os/signal" - - "runtime/debug" - - sf "github.com/snowflakedb/gosnowflake" -) - -func TestJsonResultSet(t *testing.T) { - runJSONResultSet() -} - -func BenchmarkJsonResultSet(*testing.B) { - runJSONResultSet() -} - -func runJSONResultSet() { - if !flag.Parsed() { - flag.Parse() - } - - // handler interrupt signal - ctx, cancel := context.WithCancel(context.Background()) - c := make(chan os.Signal, 1) - defer close(c) - signal.Notify(c, os.Interrupt) - defer func() { - signal.Stop(c) - }() - go func() { - select { - case <-c: - cancel() - case <-ctx.Done(): - } - }() - setCustomJSONDecoder() - setMaxChunkDownloadWorkers() - cfg, err := sf.GetConfigFromEnv([]*sf.ConfigParam{ - {Name: "Account", EnvName: "SNOWFLAKE_TEST_ACCOUNT", FailOnMissing: true}, - {Name: "User", EnvName: "SNOWFLAKE_TEST_USER", FailOnMissing: true}, - {Name: "Password", EnvName: "SNOWFLAKE_TEST_PASSWORD", FailOnMissing: true}, - {Name: "Role", EnvName: "SNOWFLAKE_TEST_ROLE", FailOnMissing: false}, - {Name: "Host", EnvName: "SNOWFLAKE_TEST_HOST", FailOnMissing: false}, - {Name: "Port", EnvName: "SNOWFLAKE_TEST_PORT", FailOnMissing: false}, - {Name: "Protocol", EnvName: "SNOWFLAKE_TEST_PROTOCOL", FailOnMissing: false}, - }) - if err != nil { - log.Fatalf("failed to create Config, err: %v", err) - } - dsn, err := sf.DSN(cfg) - if err != nil { - log.Fatalf("failed to create DSN from Config: %v, err: %v", cfg, err) - } - - db, err := sql.Open("snowflake", dsn) - defer db.Close() - if err != nil { - log.Fatalf("failed to connect. %v, err: %v", dsn, err) - } - - query := `SELECT V FROM SNOWFLAKE_SAMPLE_DATA.WEATHER.HOURLY_14_TOTAL LIMIT 100000` - rows, err := db.QueryContext(ctx, query) - if err != nil { - log.Fatalf("failed to run a query. %v, err: %v", query, err) - } - defer rows.Close() - var v1 string - counter := 0 - for rows.Next() { - err := rows.Scan(&v1) - if err != nil { - log.Fatalf("failed to get result. err: %v", err) - } - if counter%1000000 == 0 { - debug.FreeOSMemory() - } - counter++ - } -} - -func setMaxChunkDownloadWorkers() { - maxChunkDownloadWorkersStr, err := sf.GetFromEnv("SNOWFLAKE_TEST_MAX_CHUNK_DOWNLOAD_WORKERS", false) - if err != nil { - log.Fatal(err) - } - if maxChunkDownloadWorkersStr != "" { - maxChunkDownloadWorkers, err := strconv.Atoi(maxChunkDownloadWorkersStr) - if err != nil { - log.Fatalf("invalid value for SNOWFLAKE_TEST_MAX_CHUNK_DOWNLOAD_WORKERS: %v", maxChunkDownloadWorkers) - } - sf.MaxChunkDownloadWorkers = maxChunkDownloadWorkers - } -} - -func setCustomJSONDecoder() { - customJSONDecoderEnabledStr, err := sf.GetFromEnv("SNOWFLAKE_TEST_CUSTOME_JSON_DECODER_ENABLED", true) - if err != nil { - log.Fatal(err) - } - sf.CustomJSONDecoderEnabled = strings.EqualFold("true", customJSONDecoderEnabledStr) -} diff --git a/benchmark/largesetresult/.gitignore b/benchmark/largesetresult/.gitignore deleted file mode 100644 index 5901ab5cd..000000000 --- a/benchmark/largesetresult/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -*.test -*.out -prof_largeresults diff --git a/benchmark/largesetresult/Makefile b/benchmark/largesetresult/Makefile deleted file mode 100644 index c1ec2f6f7..000000000 --- a/benchmark/largesetresult/Makefile +++ /dev/null @@ -1,35 +0,0 @@ -## Setup -SHELL := /bin/bash -SRC = $(shell find . -type f -name '*.go' -not -path "./vendor/*") - -setup: - go install golang.org/x/lint/golint@latest - go install github.com/Songmu/make2help/cmd/make2help@latest - -## Benchmark -profile: - go test -run none -bench . -benchtime 3s -benchmem -cpuprofile cpu.out -memprofile mem.out - @echo "For CPU usage, run 'go tool pprof largesetresult.test cpu.out'" - @echo "For Memory usage, run 'go tool pprof -alloc_space largesetresult.test mem.out'" - -## Trace -trace: - go test -trace trace.out - @echo "Run 'go tool trace largesetresult.test trace.out'" - -## Lint -lint: setup - go vet $(SRC) - for pkg in $$(go list ./... | grep -v vendor); do \ - golint -set_exit_status $$pkg || exit $$?; \ - done - -## Format source codes using gfmt -fmt: setup - @gofmt -l -w $(SRC) - -## Show help -help: - @make2help $(MAKEFILE_LIST) - -.PHONY: install run diff --git a/benchmark/largesetresult/README.rst b/benchmark/largesetresult/README.rst deleted file mode 100644 index 7e8892a4c..000000000 --- a/benchmark/largesetresult/README.rst +++ /dev/null @@ -1,55 +0,0 @@ -******************************************************************************** -Benchmark Large Result Set -******************************************************************************** - -This folder includes a benchmark test case for "Large Result set", which refers -to the query result more than 100 MB as Snowflake clients fetches the first small -chunk of data (~100MB) from Snowflake DB and downloads the rest of data chunks -from AWS S3 bucket. - -Profiling -========= - -Using Go's profilers, you may see CPU and memory usage on each function/method. -This command instruments CPU and memory usage and save them into files. - -.. code-block:: bash - - SNOWFLAKE_TEST_ACCOUNT= \ - SNOWFLAKE_TEST_USER= \ - SNOWFLAKE_TEST_PASSWORD= \ - make profile - -Check CPU usage on the web browser: - -.. code-block:: bash - - go tool pprof largesetresult.test cpu.out - (pprof) web - -Check memory usage on the web browser: - -.. code-block:: bash - - go tool pprof -alloc_space largesetresult.test mem.out - (pprof) web - -Tracing -======= - -Using Go's trace tool, you may see all of the goroutine's activity with timeline. - -.. code-block:: bash - - SNOWFLAKE_TEST_ACCOUNT= \ - SNOWFLAKE_TEST_USER= \ - SNOWFLAKE_TEST_PASSWORD= \ - make trace - -Check goroutine's activities on web browser. - -.. code-block:: bash - - go tool trace largesetresult.test trace.out - - diff --git a/benchmark/largesetresult/largesetresult.go b/benchmark/largesetresult/largesetresult.go deleted file mode 100644 index a6e36413a..000000000 --- a/benchmark/largesetresult/largesetresult.go +++ /dev/null @@ -1,7 +0,0 @@ -// Package largesetresult is a benchmark for large result sets -// -// This exists to mitigate "no non-test Go files" in the latest Go -package largesetresult - -func main() { -} diff --git a/benchmark/largesetresult/largesetresult_test.go b/benchmark/largesetresult/largesetresult_test.go deleted file mode 100644 index 99cd4b929..000000000 --- a/benchmark/largesetresult/largesetresult_test.go +++ /dev/null @@ -1,97 +0,0 @@ -// This code is to profile a large result set query. It is basically similar to selectmany example code but -// leverages benchmark framework. -package largesetresult - -import ( - "flag" - "log" - _ "net/http/pprof" - "os" - "testing" - - "database/sql" - - "context" - "os/signal" - - "runtime/debug" - - sf "github.com/snowflakedb/gosnowflake" -) - -func TestLargeResultSet(t *testing.T) { - runLargeResultSet() -} - -func BenchmarkLargeResultSet(*testing.B) { - runLargeResultSet() -} - -func runLargeResultSet() { - if !flag.Parsed() { - flag.Parse() - } - - // handler interrupt signal - ctx, cancel := context.WithCancel(context.Background()) - c := make(chan os.Signal, 1) - defer close(c) - signal.Notify(c, os.Interrupt) - defer func() { - signal.Stop(c) - }() - go func() { - select { - case <-c: - cancel() - case <-ctx.Done(): - } - }() - - cfg, err := sf.GetConfigFromEnv([]*sf.ConfigParam{ - {Name: "Account", EnvName: "SNOWFLAKE_TEST_ACCOUNT", FailOnMissing: true}, - {Name: "User", EnvName: "SNOWFLAKE_TEST_USER", FailOnMissing: true}, - {Name: "Password", EnvName: "SNOWFLAKE_TEST_PASSWORD", FailOnMissing: true}, - {Name: "Host", EnvName: "SNOWFLAKE_TEST_HOST", FailOnMissing: false}, - {Name: "Port", EnvName: "SNOWFLAKE_TEST_PORT", FailOnMissing: false}, - {Name: "Protocol", EnvName: "SNOWFLAKE_TEST_PROTOCOL", FailOnMissing: false}, - }) - if err != nil { - log.Fatalf("failed to create Config, err: %v", err) - } - dsn, err := sf.DSN(cfg) - if err != nil { - log.Fatalf("failed to create DSN from Config: %v, err: %v", cfg, err) - } - - db, err := sql.Open("snowflake", dsn) - defer db.Close() - if err != nil { - log.Fatalf("failed to connect. err: %v", err) - } - - query := `select * from - (select 0 a union select 1 union select 2 union select 3 union select 4 union select 5 union select 6 union select 7 union select 8 union select 9) A, - (select 0 b union select 1 union select 2 union select 3 union select 4 union select 5 union select 6 union select 7 union select 8 union select 9) B, - (select 0 c union select 1 union select 2 union select 3 union select 4 union select 5 union select 6 union select 7 union select 8 union select 9) C, - (select 0 d union select 1 union select 2 union select 3 union select 4 union select 5 union select 6 union select 7 union select 8 union select 9) E, - (select 0 e union select 1 union select 2 union select 3 union select 4 union select 5 union select 6 union select 7 union select 8 union select 9) F, - (select 0 f union select 1 union select 2 union select 3 union select 4 union select 5 union select 6 union select 7 union select 8 union select 9) G, - (select 0 f union select 1 union select 2 union select 3 union select 4 union select 5 union select 6 union select 7 union select 8 union select 9) H` - rows, err := db.QueryContext(ctx, query) - if err != nil { - log.Fatalf("failed to run a query. %v, err: %v", query, err) - } - defer rows.Close() - var v1, v2, v3, v4, v5, v6, v7 int - counter := 0 - for rows.Next() { - if err = rows.Scan(&v1, &v2, &v3, &v4, &v5, &v6, &v7); err != nil { - log.Fatalf("failed to get result. err: %v", err) - } - if counter%1000000 == 0 { - debug.FreeOSMemory() - } - counter++ - } -}