From bdde0f6c7840f5977b375ac600dd03904d083710 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Fri, 6 Mar 2020 15:20:18 -0800 Subject: [PATCH] add system max resources (#75) --- .github/workflows/go.yml | 2 +- .github/workflows/release.yml | 2 +- cli/cli.go | 3 ++ cli/client.go | 1 - cli/rlimit.go | 56 +++++++++++++++++++++++++++++++++++ go.mod | 2 ++ go.sum | 4 +++ 7 files changed, 67 insertions(+), 3 deletions(-) create mode 100644 cli/rlimit.go diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index c8db2809..bb16d6cd 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -14,7 +14,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - go-version: [1.13.x] + go-version: [1.13.x, 1.14.x] os: [ubuntu-latest] steps: - name: Set up Go ${{ matrix.go-version }} on ${{ matrix.os }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 51fcdc06..ab63ac27 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -18,7 +18,7 @@ jobs: name: Set up Go uses: actions/setup-go@v1 with: - go-version: 1.13.x + go-version: 1.14.x - name: Run GoReleaser uses: goreleaser/goreleaser-action@53acad1befee355d46f71cccf6ab4d885eb4f77f diff --git a/cli/cli.go b/cli/cli.go index edfd8a3d..4c0d009e 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -53,6 +53,9 @@ const ( ) func Main(args []string) { + // Set system max resources as needed. + setMaxResources() + if len(args) > 1 { switch args[1] { case appName, filepath.Base(args[0]): diff --git a/cli/client.go b/cli/client.go index 2453082d..5a1adbe9 100644 --- a/cli/client.go +++ b/cli/client.go @@ -176,7 +176,6 @@ func clientTransport(ctx *cli.Context) http.RoundTripper { Timeout: 5 * time.Second, KeepAlive: 5 * time.Second, }).DialContext, - MaxIdleConns: ctx.Int("concurrent"), MaxIdleConnsPerHost: ctx.Int("concurrent"), IdleConnTimeout: 90 * time.Second, TLSHandshakeTimeout: 10 * time.Second, diff --git a/cli/rlimit.go b/cli/rlimit.go new file mode 100644 index 00000000..9e9e7ae4 --- /dev/null +++ b/cli/rlimit.go @@ -0,0 +1,56 @@ +/* + * Warp (C) 2019-2020 MinIO, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package cli + +import ( + "runtime/debug" + + "github.com/minio/minio/pkg/sys" +) + +func setMaxResources() (err error) { + // Set the Go runtime max threads threshold to 90% of kernel setting. + // Do not return when an error when encountered since it is not a crucial task. + sysMaxThreads, mErr := sys.GetMaxThreads() + if mErr == nil { + minioMaxThreads := (sysMaxThreads * 90) / 100 + // Only set max threads if it is greater than the default one + if minioMaxThreads > 10000 { + debug.SetMaxThreads(minioMaxThreads) + } + } + + var maxLimit uint64 + + // Set open files limit to maximum. + if _, maxLimit, err = sys.GetMaxOpenFileLimit(); err != nil { + return err + } + + if err = sys.SetMaxOpenFileLimit(maxLimit, maxLimit); err != nil { + return err + } + + // Set max memory limit as current memory limit. + if _, maxLimit, err = sys.GetMaxMemoryLimit(); err != nil { + return err + } + + err = sys.SetMaxMemoryLimit(maxLimit, maxLimit) + return err +} diff --git a/go.mod b/go.mod index f354e7fd..63263d62 100644 --- a/go.mod +++ b/go.mod @@ -15,4 +15,6 @@ require ( github.com/pkg/profile v1.3.0 github.com/posener/complete v1.2.2-0.20190702141536-6ffe496ea953 github.com/secure-io/sio-go v0.3.0 + github.com/shirou/gopsutil v2.20.2+incompatible // indirect + golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 // indirect ) diff --git a/go.sum b/go.sum index 7e4d3a68..ed289812 100644 --- a/go.sum +++ b/go.sum @@ -348,6 +348,8 @@ github.com/secure-io/sio-go v0.3.0 h1:QKGb6rGJeiExac9wSWxnWPYo8O8OFN7lxXQvHshX6v github.com/secure-io/sio-go v0.3.0/go.mod h1:D3KmXgKETffyYxBdFRN+Hpd2WzhzqS0EQwT3XWsAcBU= github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM= github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/gopsutil v2.20.2+incompatible h1:ucK79BhBpgqQxPASyS2cu9HX8cfDVljBN1WWFvbNvgY= +github.com/shirou/gopsutil v2.20.2+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/skyrings/skyring-common v0.0.0-20160929130248-d1c0bb1cbd5e h1:jrZSSgPUDtBeJbGXqgGUeupQH8I+ZvGXfhpIahye2Bc= @@ -451,6 +453,8 @@ golang.org/x/sys v0.0.0-20190523142557-0e01d883c5c5/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69 h1:rOhMmluY6kLMhdnrivzec6lLgaVbMHMn2ISQXJeJ5EM= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 h1:uYVVQ9WP/Ds2ROhcaGPeIdVq0RIXVLwsHlnvJ+cT1So= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=