From d14abccafb60c1dd500b60f969b2b7b301a44228 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Thu, 12 Jan 2023 09:30:38 -0500 Subject: [PATCH 01/86] runtime: replace panic(nil) with panic(new(runtime.PanicNilError)) Long ago we decided that panic(nil) was too unlikely to bother making a special case for purposes of recover. Unfortunately, it has turned out not to be a special case. There are many examples of code in the Go ecosystem where an author has written panic(nil) because they want to panic and don't care about the panic value. Using panic(nil) in this case has the unfortunate behavior of making recover behave as though the goroutine isn't panicking. As a result, code like: func f() { defer func() { if err := recover(); err != nil { log.Fatalf("panicked! %v", err) } }() call1() call2() } looks like it guarantees that call2 has been run any time f returns, but that turns out not to be strictly true. If call1 does panic(nil), then f returns "successfully", having recovered the panic, but without calling call2. Instead you have to write something like: func f() { done := false defer func() { if err := recover(); !done { log.Fatalf("panicked! %v", err) } }() call1() call2() done = true } which defeats nearly the whole point of recover. No one does this, with the result that almost all uses of recover are subtly broken. One specific broken use along these lines is in net/http, which recovers from panics in handlers and sends back an HTTP error. Users discovered in the early days of Go that panic(nil) was a convenient way to jump out of a handler up to the serving loop without sending back an HTTP error. This was a bug, not a feature. Go 1.8 added panic(http.ErrAbortHandler) as a better way to access the feature. Any lingering code that uses panic(nil) to abort an HTTP handler without a failure message should be changed to use http.ErrAbortHandler. Programs that need the old, unintended behavior from net/http or other packages can set GODEBUG=panicnil=1 to stop the run-time error. Uses of recover that want to detect panic(nil) in new programs can check for recover returning a value of type *runtime.PanicNilError. Because the new GODEBUG is used inside the runtime, we can't import internal/godebug, so there is some new machinery to cross-connect those in this CL, to allow a mutable GODEBUG setting. That won't be necessary if we add any other mutable GODEBUG settings in the future. The CL also corrects the handling of defaulted GODEBUG values in the runtime, for #56986. Fixes #25448. Change-Id: I2b39c7e83e4f7aa308777dabf2edae54773e03f5 Reviewed-on: https://go-review.googlesource.com/c/go/+/461956 Reviewed-by: Robert Griesemer Run-TryBot: Russ Cox TryBot-Result: Gopher Robot Auto-Submit: Russ Cox --- clientserver_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clientserver_test.go b/clientserver_test.go index da5671d9..e49bed11 100644 --- a/clientserver_test.go +++ b/clientserver_test.go @@ -1239,9 +1239,9 @@ func testTransportRejectsInvalidHeaders(t *testing.T, mode testMode) { func TestInterruptWithPanic(t *testing.T) { run(t, func(t *testing.T, mode testMode) { t.Run("boom", func(t *testing.T) { testInterruptWithPanic(t, mode, "boom") }) - t.Run("nil", func(t *testing.T) { testInterruptWithPanic(t, mode, nil) }) + t.Run("nil", func(t *testing.T) { t.Setenv("GODEBUG", "panicnil=1"); testInterruptWithPanic(t, mode, nil) }) t.Run("ErrAbortHandler", func(t *testing.T) { testInterruptWithPanic(t, mode, ErrAbortHandler) }) - }) + }, testNotParallel) } func testInterruptWithPanic(t *testing.T, mode testMode, panicValue any) { const msg = "hello" From edf0c4c685054b49f6609259ed688152226c7862 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Mon, 28 Nov 2022 13:59:49 -0500 Subject: [PATCH 02/86] internal/godebug: export non-default-behavior counters in runtime/metrics Allow GODEBUG users to report how many times a setting resulted in non-default behavior. Record non-default-behaviors for all existing GODEBUGs. Also rework tests to ensure that runtime is in sync with runtime/metrics.All, and generate docs mechanically from metrics.All. For #56986. Change-Id: Iefa1213e2a5c3f19ea16cd53298c487952ef05a4 Reviewed-on: https://go-review.googlesource.com/c/go/+/453618 TryBot-Result: Gopher Robot Auto-Submit: Russ Cox Run-TryBot: Russ Cox Reviewed-by: Michael Knyszek --- server.go | 6 +++++- transport.go | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/server.go b/server.go index c3c3f91d..a9ba911a 100644 --- a/server.go +++ b/server.go @@ -3319,7 +3319,11 @@ var http2server = godebug.New("http2server") // configured otherwise. (by setting srv.TLSNextProto non-nil) // It must only be called via srv.nextProtoOnce (use srv.setupHTTP2_*). func (srv *Server) onceSetNextProtoDefaults() { - if omitBundledHTTP2 || http2server.Value() == "0" { + if omitBundledHTTP2 { + return + } + if http2server.Value() == "0" { + http2server.IncNonDefault() return } // Enable HTTP/2 by default if the user hasn't otherwise diff --git a/transport.go b/transport.go index ddcb6481..a90f36ff 100644 --- a/transport.go +++ b/transport.go @@ -369,6 +369,7 @@ var http2client = godebug.New("http2client") func (t *Transport) onceSetNextProtoDefaults() { t.tlsNextProtoWasNil = (t.TLSNextProto == nil) if http2client.Value() == "0" { + http2client.IncNonDefault() return } From 3a8342a45e5b5223b6225627b3d22217f3e0dc60 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Wed, 11 Jan 2023 13:47:38 -0800 Subject: [PATCH 03/86] net/http: close Request.Body when pconn write loop exits early The pconn write loop closes a request's body after sending the request, but in the case where the write loop exits with an unsent request in writech the body is never closed. Close the request body in this case. Fixes #49621 Change-Id: Id94a92937bbfc0beb1396446f4dee32fd2059c7e Reviewed-on: https://go-review.googlesource.com/c/go/+/461675 TryBot-Result: Gopher Robot Run-TryBot: Damien Neil Reviewed-by: Cherry Mui Reviewed-by: Brad Fitzpatrick --- transport.go | 6 ++++++ transport_test.go | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/transport.go b/transport.go index a90f36ff..b8e4c4e9 100644 --- a/transport.go +++ b/transport.go @@ -622,6 +622,12 @@ func (t *Transport) roundTrip(req *Request) (*Response, error) { if e, ok := err.(transportReadFromServerError); ok { err = e.err } + if b, ok := req.Body.(*readTrackingBody); ok && !b.didClose { + // Issue 49621: Close the request body if pconn.roundTrip + // didn't do so already. This can happen if the pconn + // write loop exits without reading the write request. + req.closeBody() + } return nil, err } testHookRoundTripRetried() diff --git a/transport_test.go b/transport_test.go index 245f73bc..2879dee0 100644 --- a/transport_test.go +++ b/transport_test.go @@ -4092,6 +4092,45 @@ func testTransportDialCancelRace(t *testing.T, mode testMode) { } } +// https://go.dev/issue/49621 +func TestConnClosedBeforeRequestIsWritten(t *testing.T) { + run(t, testConnClosedBeforeRequestIsWritten, testNotParallel, []testMode{http1Mode}) +} +func testConnClosedBeforeRequestIsWritten(t *testing.T, mode testMode) { + ts := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) {}), + func(tr *Transport) { + tr.DialContext = func(_ context.Context, network, addr string) (net.Conn, error) { + // Connection immediately returns errors. + return &funcConn{ + read: func([]byte) (int, error) { + return 0, errors.New("error") + }, + write: func([]byte) (int, error) { + return 0, errors.New("error") + }, + }, nil + } + }, + ).ts + // Set a short delay in RoundTrip to give the persistConn time to notice + // the connection is broken. We want to exercise the path where writeLoop exits + // before it reads the request to send. If this delay is too short, we may instead + // exercise the path where writeLoop accepts the request and then fails to write it. + // That's fine, so long as we get the desired path often enough. + SetEnterRoundTripHook(func() { + time.Sleep(1 * time.Millisecond) + }) + defer SetEnterRoundTripHook(nil) + var closes int + _, err := ts.Client().Post(ts.URL, "text/plain", countCloseReader{&closes, strings.NewReader("hello")}) + if err == nil { + t.Fatalf("expected request to fail, but it did not") + } + if closes != 1 { + t.Errorf("after RoundTrip, request body was closed %v times; want 1", closes) + } +} + // logWritesConn is a net.Conn that logs each Write call to writes // and then proxies to w. // It proxies Read calls to a reader it receives from rch. From a958a40f5356b6a48fede9d1a82b5c9fed8dbc70 Mon Sep 17 00:00:00 2001 From: Gustavo Falco Date: Sun, 11 Dec 2022 02:39:20 +0000 Subject: [PATCH 04/86] net/http: keep sensitive headers on redirects to the same host Preserve sensitive headers on a redirect to a different port of the same host. Fixes #35104 Change-Id: I5ab57c414ce92a70e688ee684b9ff02fb062b3c6 GitHub-Last-Rev: 8d53e71e2243c141d70d27a503d0f7e6dee64c3c GitHub-Pull-Request: golang/go#54539 Reviewed-on: https://go-review.googlesource.com/c/go/+/424935 TryBot-Result: Gopher Robot Reviewed-by: Cherry Mui Reviewed-by: Damien Neil Run-TryBot: Damien Neil --- client.go | 4 ++-- client_test.go | 29 ++++++++++++++++++++++++----- transport.go | 10 +++++++--- 3 files changed, 33 insertions(+), 10 deletions(-) diff --git a/client.go b/client.go index 122e8d03..2f7d4f69 100644 --- a/client.go +++ b/client.go @@ -990,8 +990,8 @@ func shouldCopyHeaderOnRedirect(headerKey string, initial, dest *url.URL) bool { // directly, we don't know their scope, so we assume // it's for *.domain.com. - ihost := canonicalAddr(initial) - dhost := canonicalAddr(dest) + ihost := idnaASCIIFromURL(initial) + dhost := idnaASCIIFromURL(dest) return isDomainOrSubdomain(dhost, ihost) } // All other headers are copied: diff --git a/client_test.go b/client_test.go index 8b53c416..bb1ed6f1 100644 --- a/client_test.go +++ b/client_test.go @@ -1470,6 +1470,9 @@ func TestClientRedirectResponseWithoutRequest(t *testing.T) { } // Issue 4800: copy (some) headers when Client follows a redirect. +// Issue 35104: Since both URLs have the same host (localhost) +// but different ports, sensitive headers like Cookie and Authorization +// are preserved. func TestClientCopyHeadersOnRedirect(t *testing.T) { run(t, testClientCopyHeadersOnRedirect) } func testClientCopyHeadersOnRedirect(t *testing.T, mode testMode) { const ( @@ -1483,6 +1486,8 @@ func testClientCopyHeadersOnRedirect(t *testing.T, mode testMode) { "X-Foo": []string{xfoo}, "Referer": []string{ts2URL}, "Accept-Encoding": []string{"gzip"}, + "Cookie": []string{"foo=bar"}, + "Authorization": []string{"secretpassword"}, } if !reflect.DeepEqual(r.Header, want) { t.Errorf("Request.Header = %#v; want %#v", r.Header, want) @@ -1501,9 +1506,11 @@ func testClientCopyHeadersOnRedirect(t *testing.T, mode testMode) { c := ts1.Client() c.CheckRedirect = func(r *Request, via []*Request) error { want := Header{ - "User-Agent": []string{ua}, - "X-Foo": []string{xfoo}, - "Referer": []string{ts2URL}, + "User-Agent": []string{ua}, + "X-Foo": []string{xfoo}, + "Referer": []string{ts2URL}, + "Cookie": []string{"foo=bar"}, + "Authorization": []string{"secretpassword"}, } if !reflect.DeepEqual(r.Header, want) { t.Errorf("CheckRedirect Request.Header = %#v; want %#v", r.Header, want) @@ -1707,18 +1714,30 @@ func TestShouldCopyHeaderOnRedirect(t *testing.T) { {"cookie", "http://foo.com/", "http://bar.com/", false}, {"cookie2", "http://foo.com/", "http://bar.com/", false}, {"authorization", "http://foo.com/", "http://bar.com/", false}, + {"authorization", "http://foo.com/", "https://foo.com/", true}, + {"authorization", "http://foo.com:1234/", "http://foo.com:4321/", true}, {"www-authenticate", "http://foo.com/", "http://bar.com/", false}, // But subdomains should work: {"www-authenticate", "http://foo.com/", "http://foo.com/", true}, {"www-authenticate", "http://foo.com/", "http://sub.foo.com/", true}, {"www-authenticate", "http://foo.com/", "http://notfoo.com/", false}, - {"www-authenticate", "http://foo.com/", "https://foo.com/", false}, + {"www-authenticate", "http://foo.com/", "https://foo.com/", true}, {"www-authenticate", "http://foo.com:80/", "http://foo.com/", true}, {"www-authenticate", "http://foo.com:80/", "http://sub.foo.com/", true}, {"www-authenticate", "http://foo.com:443/", "https://foo.com/", true}, {"www-authenticate", "http://foo.com:443/", "https://sub.foo.com/", true}, - {"www-authenticate", "http://foo.com:1234/", "http://foo.com/", false}, + {"www-authenticate", "http://foo.com:1234/", "http://foo.com/", true}, + + {"authorization", "http://foo.com/", "http://foo.com/", true}, + {"authorization", "http://foo.com/", "http://sub.foo.com/", true}, + {"authorization", "http://foo.com/", "http://notfoo.com/", false}, + {"authorization", "http://foo.com/", "https://foo.com/", true}, + {"authorization", "http://foo.com:80/", "http://foo.com/", true}, + {"authorization", "http://foo.com:80/", "http://sub.foo.com/", true}, + {"authorization", "http://foo.com:443/", "https://foo.com/", true}, + {"authorization", "http://foo.com:443/", "https://sub.foo.com/", true}, + {"authorization", "http://foo.com:1234/", "http://foo.com/", true}, } for i, tt := range tests { u0, err := url.Parse(tt.initialURL) diff --git a/transport.go b/transport.go index b8e4c4e9..7561f7f5 100644 --- a/transport.go +++ b/transport.go @@ -2750,17 +2750,21 @@ var portMap = map[string]string{ "socks5": "1080", } -// canonicalAddr returns url.Host but always with a ":port" suffix. -func canonicalAddr(url *url.URL) string { +func idnaASCIIFromURL(url *url.URL) string { addr := url.Hostname() if v, err := idnaASCII(addr); err == nil { addr = v } + return addr +} + +// canonicalAddr returns url.Host but always with a ":port" suffix. +func canonicalAddr(url *url.URL) string { port := url.Port() if port == "" { port = portMap[url.Scheme] } - return net.JoinHostPort(addr, port) + return net.JoinHostPort(idnaASCIIFromURL(url), port) } // bodyEOFSignal is used by the HTTP/1 transport when reading response From 635b4d0b95369e71ad839056a853c03185af8bb9 Mon Sep 17 00:00:00 2001 From: Oleksii Sholik Date: Sat, 14 Jan 2023 22:03:38 +0000 Subject: [PATCH 05/86] net/http: fix missing word in ServeMux doc Change-Id: I324cedfcdfa1d7eac7be43dc89febb584359c8e3 GitHub-Last-Rev: 7724cd84f323b531215db0406f72481394d0b206 GitHub-Pull-Request: golang/go#57799 Reviewed-on: https://go-review.googlesource.com/c/go/+/462039 TryBot-Result: Gopher Robot Reviewed-by: Matthew Dempsky Reviewed-by: Damien Neil Auto-Submit: Damien Neil Run-TryBot: Damien Neil --- server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server.go b/server.go index a9ba911a..c15f0f58 100644 --- a/server.go +++ b/server.go @@ -2278,7 +2278,7 @@ func RedirectHandler(url string, code int) Handler { // Longer patterns take precedence over shorter ones, so that // if there are handlers registered for both "/images/" // and "/images/thumbnails/", the latter handler will be -// called for paths beginning "/images/thumbnails/" and the +// called for paths beginning with "/images/thumbnails/" and the // former will receive requests for any other paths in the // "/images/" subtree. // From 308f5570e05aa71427a04c0b518e6dae3b8c6b92 Mon Sep 17 00:00:00 2001 From: Johan Brandhorst-Satzkorn Date: Fri, 27 Jan 2023 22:50:54 -0800 Subject: [PATCH 06/86] net/http: disable fetch on NodeJS NodeJS 18 introduced support for the fetch API for making HTTP requests. This broke all wasm tests that were relying on NodeJS falling back to the fake network implementation in net_fake.go. Disable the fetch API on NodeJS to get tests passing. Fixes #57613 Change-Id: Icb2cce6d5289d812da798e07366f8ac26b5f82cb Reviewed-on: https://go-review.googlesource.com/c/go/+/463976 Reviewed-by: Evan Phoenix TryBot-Result: Gopher Robot Reviewed-by: Dmitri Shuralyov Reviewed-by: Dmitri Shuralyov Run-TryBot: Dmitri Shuralyov Reviewed-by: Michael Knyszek --- roundtrip_js.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/roundtrip_js.go b/roundtrip_js.go index 01c0600b..21d8df96 100644 --- a/roundtrip_js.go +++ b/roundtrip_js.go @@ -44,6 +44,12 @@ const jsFetchRedirect = "js.fetch:redirect" // the browser globals. var jsFetchMissing = js.Global().Get("fetch").IsUndefined() +// jsFetchDisabled will be true if the "process" global is present. +// We use this as an indicator that we're running in Node.js. We +// want to disable the Fetch API in Node.js because it breaks +// our wasm tests. See https://go.dev/issue/57613 for more information. +var jsFetchDisabled = !js.Global().Get("process").IsUndefined() + // RoundTrip implements the RoundTripper interface using the WHATWG Fetch API. func (t *Transport) RoundTrip(req *Request) (*Response, error) { // The Transport has a documented contract that states that if the DialContext or @@ -52,7 +58,7 @@ func (t *Transport) RoundTrip(req *Request) (*Response, error) { // though they are deprecated. Therefore, if any of these are set, we should obey // the contract and dial using the regular round-trip instead. Otherwise, we'll try // to fall back on the Fetch API, unless it's not available. - if t.Dial != nil || t.DialContext != nil || t.DialTLS != nil || t.DialTLSContext != nil || jsFetchMissing { + if t.Dial != nil || t.DialContext != nil || t.DialTLS != nil || t.DialTLSContext != nil || jsFetchMissing || jsFetchDisabled { return t.roundTrip(req) } From 7bc80a6165d01dfc4b724f8f8b84458ac6b90eb7 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Sat, 14 Jan 2023 14:03:03 -0500 Subject: [PATCH 07/86] net/http: add section headers to package doc Change-Id: I2379cceeb74cb8511058b24cdd100b21649505ce Reviewed-on: https://go-review.googlesource.com/c/go/+/462197 Run-TryBot: Russ Cox TryBot-Result: Gopher Robot Reviewed-by: Damien Neil --- doc.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/doc.go b/doc.go index 67c4246c..d9e6aafb 100644 --- a/doc.go +++ b/doc.go @@ -14,7 +14,7 @@ Get, Head, Post, and PostForm make HTTP (or HTTPS) requests: resp, err := http.PostForm("http://example.com/form", url.Values{"key": {"Value"}, "id": {"123"}}) -The client must close the response body when finished with it: +The caller must close the response body when finished with it: resp, err := http.Get("http://example.com/") if err != nil { @@ -24,6 +24,8 @@ The client must close the response body when finished with it: body, err := io.ReadAll(resp.Body) // ... +# Clients and Transports + For control over HTTP client headers, redirect policy, and other settings, create a Client: @@ -54,6 +56,8 @@ compression, and other settings, create a Transport: Clients and Transports are safe for concurrent use by multiple goroutines and for efficiency should only be created once and re-used. +# Servers + ListenAndServe starts an HTTP server with a given address and handler. The handler is usually nil, which means to use DefaultServeMux. Handle and HandleFunc add handlers to DefaultServeMux: @@ -78,11 +82,13 @@ custom Server: } log.Fatal(s.ListenAndServe()) +# HTTP/2 + Starting with Go 1.6, the http package has transparent support for the HTTP/2 protocol when using HTTPS. Programs that must disable HTTP/2 can do so by setting Transport.TLSNextProto (for clients) or Server.TLSNextProto (for servers) to a non-nil, empty -map. Alternatively, the following GODEBUG environment variables are +map. Alternatively, the following GODEBUG settings are currently supported: GODEBUG=http2client=0 # disable HTTP/2 client support @@ -90,9 +96,7 @@ currently supported: GODEBUG=http2debug=1 # enable verbose HTTP/2 debug logs GODEBUG=http2debug=2 # ... even more verbose, with frame dumps -The GODEBUG variables are not covered by Go's API compatibility -promise. Please report any issues before disabling HTTP/2 -support: https://golang.org/s/http2bug +Please report any issues before disabling HTTP/2 support: https://golang.org/s/http2bug The http package's Transport and Server both automatically enable HTTP/2 support for simple configurations. To enable HTTP/2 for more From bc52c2200e26d8d372d2d53d839752541cb62975 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Fri, 9 Dec 2022 09:56:15 -0800 Subject: [PATCH 08/86] io: allocate copy buffers from a pool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CopyBuffer allocates a 32k buffer when no buffer is available. Allocate these buffers from a sync.Pool. This removes an optimization where the copy buffer size was reduced when the source is a io.LimitedReader (including the case of CopyN) with a limit less than the default buffer size. This change could cause a program which only uses io.Copy with sources with a small limit to allocate unnecessarily large buffers. Programs which care about the transient buffer allocation can avoid this by providing their own buffer. name old time/op new time/op delta CopyNSmall-10 165ns ± 7% 117ns ± 7% -29.19% (p=0.001 n=7+7) CopyNLarge-10 7.33µs ±34% 4.07µs ± 2% -44.52% (p=0.001 n=7+7) name old alloc/op new alloc/op delta CopyNSmall-10 2.20kB ±12% 1.20kB ± 4% -45.24% (p=0.000 n=8+7) CopyNLarge-10 148kB ± 9% 81kB ± 4% -45.26% (p=0.000 n=8+7) name old allocs/op new allocs/op delta CopyNSmall-10 2.00 ± 0% 1.00 ± 0% -50.00% (p=0.000 n=8+8) CopyNLarge-10 2.00 ± 0% 1.00 ± 0% -50.00% (p=0.000 n=8+8) For #57202 Change-Id: I2292226da9ba1dc09a2543f5d74fe5da06080d49 Reviewed-on: https://go-review.googlesource.com/c/go/+/456555 TryBot-Result: Gopher Robot Run-TryBot: Damien Neil Reviewed-by: Thomas Austad Auto-Submit: Damien Neil Reviewed-by: Ian Lance Taylor --- server.go | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/server.go b/server.go index c15f0f58..bb31761a 100644 --- a/server.go +++ b/server.go @@ -567,16 +567,12 @@ type writerOnly struct { // to a *net.TCPConn with sendfile, or from a supported src type such // as a *net.TCPConn on Linux with splice. func (w *response) ReadFrom(src io.Reader) (n int64, err error) { - bufp := copyBufPool.Get().(*[]byte) - buf := *bufp - defer copyBufPool.Put(bufp) - // Our underlying w.conn.rwc is usually a *TCPConn (with its // own ReadFrom method). If not, just fall back to the normal // copy method. rf, ok := w.conn.rwc.(io.ReaderFrom) if !ok { - return io.CopyBuffer(writerOnly{w}, src, buf) + return io.Copy(writerOnly{w}, src) } // Copy the first sniffLen bytes before switching to ReadFrom. @@ -584,7 +580,7 @@ func (w *response) ReadFrom(src io.Reader) (n int64, err error) { // source is available (see golang.org/issue/5660) and provides // enough bytes to perform Content-Type sniffing when required. if !w.cw.wroteHeader { - n0, err := io.CopyBuffer(writerOnly{w}, io.LimitReader(src, sniffLen), buf) + n0, err := io.Copy(writerOnly{w}, io.LimitReader(src, sniffLen)) n += n0 if err != nil || n0 < sniffLen { return n, err @@ -602,7 +598,7 @@ func (w *response) ReadFrom(src io.Reader) (n int64, err error) { return n, err } - n0, err := io.CopyBuffer(writerOnly{w}, src, buf) + n0, err := io.Copy(writerOnly{w}, src) n += n0 return n, err } @@ -799,13 +795,6 @@ var ( bufioWriter4kPool sync.Pool ) -var copyBufPool = sync.Pool{ - New: func() any { - b := make([]byte, 32*1024) - return &b - }, -} - func bufioWriterPool(size int) *sync.Pool { switch size { case 2 << 10: From e689de49a1240654de025468dbe1c8842e8c96e3 Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Fri, 27 Mar 2020 17:16:26 -0400 Subject: [PATCH 09/86] net/http: remove another arbitrary timeout in TestTLSHandshakeTimeout Updates #37327 Change-Id: I87774be71ed54e9c45a27062122e6177888e890a Reviewed-on: https://go-review.googlesource.com/c/go/+/226137 Auto-Submit: Bryan Mills TryBot-Result: Gopher Robot Run-TryBot: Bryan Mills Reviewed-by: Damien Neil --- serve_test.go | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/serve_test.go b/serve_test.go index eac527b9..e11de660 100644 --- a/serve_test.go +++ b/serve_test.go @@ -1444,13 +1444,9 @@ func testTLSHandshakeTimeout(t *testing.T, mode testMode) { t.Errorf("Read = %d, %v; want an error and no bytes", n, err) } - select { - case v := <-errc: - if !strings.Contains(v, "timeout") && !strings.Contains(v, "TLS handshake") { - t.Errorf("expected a TLS handshake timeout error; got %q", v) - } - case <-time.After(5 * time.Second): - t.Errorf("timeout waiting for logged error") + v := <-errc + if !strings.Contains(v, "timeout") && !strings.Contains(v, "TLS handshake") { + t.Errorf("expected a TLS handshake timeout error; got %q", v) } } From ef3eb97dd99e07e7a61f0a97bb3302b28390bbdd Mon Sep 17 00:00:00 2001 From: Ethan Lowman Date: Fri, 3 Feb 2023 13:35:28 -0500 Subject: [PATCH 10/86] net/http/httputil: fix syntax in ReverseProxy example comment Change-Id: I7e71626246af94047fbd1abb9bb77f2cd9b281fb Reviewed-on: https://go-review.googlesource.com/c/go/+/465195 Reviewed-by: Dmitri Shuralyov Auto-Submit: Dmitri Shuralyov Reviewed-by: Damien Neil TryBot-Result: Gopher Robot Run-TryBot: Damien Neil --- httputil/reverseproxy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/httputil/reverseproxy.go b/httputil/reverseproxy.go index 58064a53..eece455a 100644 --- a/httputil/reverseproxy.go +++ b/httputil/reverseproxy.go @@ -257,7 +257,7 @@ func joinURLPath(a, b *url.URL) (path, rawpath string) { // Rewrite: func(r *ProxyRequest) { // r.SetURL(target) // r.Out.Host = r.In.Host // if desired -// } +// }, // } func NewSingleHostReverseProxy(target *url.URL) *ReverseProxy { director := func(req *http.Request) { From 010b00ce29cc8c5bc241f0a0cdaf60d1edf2ee4d Mon Sep 17 00:00:00 2001 From: Johan Brandhorst-Satzkorn Date: Fri, 27 Jan 2023 22:53:25 -0800 Subject: [PATCH 11/86] net/http: improve js fetch errors The Error type used in failed fetch invocations contain more information than we're currently extracting. Optimistically look for the cause field for extra context for errors. Change-Id: I6c8e4b230a21ec684af2107707aa605fc2148fcf Reviewed-on: https://go-review.googlesource.com/c/go/+/463978 TryBot-Result: Gopher Robot Reviewed-by: Dmitri Shuralyov Reviewed-by: David Chase Reviewed-by: Evan Phoenix Run-TryBot: Johan Brandhorst-Satzkorn Reviewed-by: Dmitri Shuralyov Auto-Submit: Johan Brandhorst-Satzkorn --- roundtrip_js.go | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/roundtrip_js.go b/roundtrip_js.go index 21d8df96..4f381247 100644 --- a/roundtrip_js.go +++ b/roundtrip_js.go @@ -191,7 +191,22 @@ func (t *Transport) RoundTrip(req *Request) (*Response, error) { failure = js.FuncOf(func(this js.Value, args []js.Value) any { success.Release() failure.Release() - errCh <- fmt.Errorf("net/http: fetch() failed: %s", args[0].Get("message").String()) + err := args[0] + // The error is a JS Error type + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Error + // We can use the toString() method to get a string representation of the error. + errMsg := err.Call("toString").String() + // Errors can optionally contain a cause. + if cause := err.Get("cause"); !cause.IsUndefined() { + // The exact type of the cause is not defined, + // but if it's another error, we can call toString() on it too. + if !cause.Get("toString").IsUndefined() { + errMsg += ": " + cause.Call("toString").String() + } else if cause.Type() == js.TypeString { + errMsg += ": " + cause.String() + } + } + errCh <- fmt.Errorf("net/http: fetch() failed: %s", errMsg) return nil }) From 3717927f588a64162b560e304a997e550d40083e Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Thu, 9 Feb 2023 14:24:46 -0800 Subject: [PATCH 12/86] Revert "io: allocate copy buffers from a pool" This reverts CL 456555. Reason for revert: This seems too likely to exercise race conditions in code where a Write call continues to access its buffer after returning. The HTTP/2 ResponseWriter is one such example. Reverting this change while we think about this some more. For #57202 Change-Id: Ic86823f81d7da410ea6b3f17fb5b3f9a979e3340 Reviewed-on: https://go-review.googlesource.com/c/go/+/467095 Reviewed-by: Bryan Mills Run-TryBot: Damien Neil TryBot-Result: Gopher Robot --- server.go | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/server.go b/server.go index bb31761a..c15f0f58 100644 --- a/server.go +++ b/server.go @@ -567,12 +567,16 @@ type writerOnly struct { // to a *net.TCPConn with sendfile, or from a supported src type such // as a *net.TCPConn on Linux with splice. func (w *response) ReadFrom(src io.Reader) (n int64, err error) { + bufp := copyBufPool.Get().(*[]byte) + buf := *bufp + defer copyBufPool.Put(bufp) + // Our underlying w.conn.rwc is usually a *TCPConn (with its // own ReadFrom method). If not, just fall back to the normal // copy method. rf, ok := w.conn.rwc.(io.ReaderFrom) if !ok { - return io.Copy(writerOnly{w}, src) + return io.CopyBuffer(writerOnly{w}, src, buf) } // Copy the first sniffLen bytes before switching to ReadFrom. @@ -580,7 +584,7 @@ func (w *response) ReadFrom(src io.Reader) (n int64, err error) { // source is available (see golang.org/issue/5660) and provides // enough bytes to perform Content-Type sniffing when required. if !w.cw.wroteHeader { - n0, err := io.Copy(writerOnly{w}, io.LimitReader(src, sniffLen)) + n0, err := io.CopyBuffer(writerOnly{w}, io.LimitReader(src, sniffLen), buf) n += n0 if err != nil || n0 < sniffLen { return n, err @@ -598,7 +602,7 @@ func (w *response) ReadFrom(src io.Reader) (n int64, err error) { return n, err } - n0, err := io.Copy(writerOnly{w}, src) + n0, err := io.CopyBuffer(writerOnly{w}, src, buf) n += n0 return n, err } @@ -795,6 +799,13 @@ var ( bufioWriter4kPool sync.Pool ) +var copyBufPool = sync.Pool{ + New: func() any { + b := make([]byte, 32*1024) + return &b + }, +} + func bufioWriterPool(size int) *sync.Pool { switch size { case 2 << 10: From 22e5e31c9fef047bc729247ea9916ba5fe715166 Mon Sep 17 00:00:00 2001 From: David Chase Date: Thu, 9 Feb 2023 13:04:21 -0500 Subject: [PATCH 13/86] vendor, cmd/vendor: update standard library dependencies Change-Id: I6facfae14e850f6c9bf3bcb53489c8b475bbb860 Reviewed-on: https://go-review.googlesource.com/c/go/+/467297 TryBot-Result: Gopher Robot Reviewed-by: Damien Neil Run-TryBot: David Chase --- h2_bundle.go | 269 ++++++++++++++++++++++++++++----------------------- 1 file changed, 149 insertions(+), 120 deletions(-) diff --git a/h2_bundle.go b/h2_bundle.go index 1e0b83d4..eb3fd159 100644 --- a/h2_bundle.go +++ b/h2_bundle.go @@ -1303,23 +1303,91 @@ var ( http2errPseudoAfterRegular = errors.New("pseudo header field after regular") ) -// flow is the flow control window's size. -type http2flow struct { +// inflowMinRefresh is the minimum number of bytes we'll send for a +// flow control window update. +const http2inflowMinRefresh = 4 << 10 + +// inflow accounts for an inbound flow control window. +// It tracks both the latest window sent to the peer (used for enforcement) +// and the accumulated unsent window. +type http2inflow struct { + avail int32 + unsent int32 +} + +// init sets the initial window. +func (f *http2inflow) init(n int32) { + f.avail = n +} + +// add adds n bytes to the window, with a maximum window size of max, +// indicating that the peer can now send us more data. +// For example, the user read from a {Request,Response} body and consumed +// some of the buffered data, so the peer can now send more. +// It returns the number of bytes to send in a WINDOW_UPDATE frame to the peer. +// Window updates are accumulated and sent when the unsent capacity +// is at least inflowMinRefresh or will at least double the peer's available window. +func (f *http2inflow) add(n int) (connAdd int32) { + if n < 0 { + panic("negative update") + } + unsent := int64(f.unsent) + int64(n) + // "A sender MUST NOT allow a flow-control window to exceed 2^31-1 octets." + // RFC 7540 Section 6.9.1. + const maxWindow = 1<<31 - 1 + if unsent+int64(f.avail) > maxWindow { + panic("flow control update exceeds maximum window size") + } + f.unsent = int32(unsent) + if f.unsent < http2inflowMinRefresh && f.unsent < f.avail { + // If there aren't at least inflowMinRefresh bytes of window to send, + // and this update won't at least double the window, buffer the update for later. + return 0 + } + f.avail += f.unsent + f.unsent = 0 + return int32(unsent) +} + +// take attempts to take n bytes from the peer's flow control window. +// It reports whether the window has available capacity. +func (f *http2inflow) take(n uint32) bool { + if n > uint32(f.avail) { + return false + } + f.avail -= int32(n) + return true +} + +// takeInflows attempts to take n bytes from two inflows, +// typically connection-level and stream-level flows. +// It reports whether both windows have available capacity. +func http2takeInflows(f1, f2 *http2inflow, n uint32) bool { + if n > uint32(f1.avail) || n > uint32(f2.avail) { + return false + } + f1.avail -= int32(n) + f2.avail -= int32(n) + return true +} + +// outflow is the outbound flow control window's size. +type http2outflow struct { _ http2incomparable // n is the number of DATA bytes we're allowed to send. - // A flow is kept both on a conn and a per-stream. + // An outflow is kept both on a conn and a per-stream. n int32 - // conn points to the shared connection-level flow that is - // shared by all streams on that conn. It is nil for the flow + // conn points to the shared connection-level outflow that is + // shared by all streams on that conn. It is nil for the outflow // that's on the conn directly. - conn *http2flow + conn *http2outflow } -func (f *http2flow) setConnFlow(cf *http2flow) { f.conn = cf } +func (f *http2outflow) setConnFlow(cf *http2outflow) { f.conn = cf } -func (f *http2flow) available() int32 { +func (f *http2outflow) available() int32 { n := f.n if f.conn != nil && f.conn.n < n { n = f.conn.n @@ -1327,7 +1395,7 @@ func (f *http2flow) available() int32 { return n } -func (f *http2flow) take(n int32) { +func (f *http2outflow) take(n int32) { if n > f.available() { panic("internal error: took too much") } @@ -1339,7 +1407,7 @@ func (f *http2flow) take(n int32) { // add adds n bytes (positive or negative) to the flow control window. // It returns false if the sum would exceed 2^31-1. -func (f *http2flow) add(n int32) bool { +func (f *http2outflow) add(n int32) bool { sum := f.n + n if (sum > n) == (f.n > 0) { f.n = sum @@ -4187,7 +4255,7 @@ func (s *http2Server) ServeConn(c net.Conn, opts *http2ServeConnOpts) { // configured value for inflow, that will be updated when we send a // WINDOW_UPDATE shortly after sending SETTINGS. sc.flow.add(http2initialWindowSize) - sc.inflow.add(http2initialWindowSize) + sc.inflow.init(http2initialWindowSize) sc.hpackEncoder = hpack.NewEncoder(&sc.headerWriteBuf) sc.hpackEncoder.SetMaxDynamicTableSizeLimit(s.maxEncoderHeaderTableSize()) @@ -4302,8 +4370,8 @@ type http2serverConn struct { wroteFrameCh chan http2frameWriteResult // from writeFrameAsync -> serve, tickles more frame writes bodyReadCh chan http2bodyReadMsg // from handlers -> serve serveMsgCh chan interface{} // misc messages & code to send to / run on the serve loop - flow http2flow // conn-wide (not stream-specific) outbound flow control - inflow http2flow // conn-wide inbound flow control + flow http2outflow // conn-wide (not stream-specific) outbound flow control + inflow http2inflow // conn-wide inbound flow control tlsState *tls.ConnectionState // shared by all handlers, like net/http remoteAddrStr string writeSched http2WriteScheduler @@ -4380,10 +4448,10 @@ type http2stream struct { cancelCtx func() // owned by serverConn's serve loop: - bodyBytes int64 // body bytes seen so far - declBodyBytes int64 // or -1 if undeclared - flow http2flow // limits writing from Handler to client - inflow http2flow // what the client is allowed to POST/etc to us + bodyBytes int64 // body bytes seen so far + declBodyBytes int64 // or -1 if undeclared + flow http2outflow // limits writing from Handler to client + inflow http2inflow // what the client is allowed to POST/etc to us state http2streamState resetQueued bool // RST_STREAM queued for write; set by sc.resetStream gotTrailerHeader bool // HEADER frame for trailers was seen @@ -5247,7 +5315,7 @@ func (sc *http2serverConn) processFrame(f http2Frame) error { if sc.inGoAway && (sc.goAwayCode != http2ErrCodeNo || f.Header().StreamID > sc.maxClientStreamID) { if f, ok := f.(*http2DataFrame); ok { - if sc.inflow.available() < int32(f.Length) { + if !sc.inflow.take(f.Length) { return sc.countError("data_flow", http2streamError(f.Header().StreamID, http2ErrCodeFlowControl)) } sc.sendWindowUpdate(nil, int(f.Length)) // conn-level @@ -5519,14 +5587,9 @@ func (sc *http2serverConn) processData(f *http2DataFrame) error { // But still enforce their connection-level flow control, // and return any flow control bytes since we're not going // to consume them. - if sc.inflow.available() < int32(f.Length) { + if !sc.inflow.take(f.Length) { return sc.countError("data_flow", http2streamError(id, http2ErrCodeFlowControl)) } - // Deduct the flow control from inflow, since we're - // going to immediately add it back in - // sendWindowUpdate, which also schedules sending the - // frames. - sc.inflow.take(int32(f.Length)) sc.sendWindowUpdate(nil, int(f.Length)) // conn-level if st != nil && st.resetQueued { @@ -5541,10 +5604,9 @@ func (sc *http2serverConn) processData(f *http2DataFrame) error { // Sender sending more than they'd declared? if st.declBodyBytes != -1 && st.bodyBytes+int64(len(data)) > st.declBodyBytes { - if sc.inflow.available() < int32(f.Length) { + if !sc.inflow.take(f.Length) { return sc.countError("data_flow", http2streamError(id, http2ErrCodeFlowControl)) } - sc.inflow.take(int32(f.Length)) sc.sendWindowUpdate(nil, int(f.Length)) // conn-level st.body.CloseWithError(fmt.Errorf("sender tried to send more than declared Content-Length of %d bytes", st.declBodyBytes)) @@ -5555,10 +5617,9 @@ func (sc *http2serverConn) processData(f *http2DataFrame) error { } if f.Length > 0 { // Check whether the client has flow control quota. - if st.inflow.available() < int32(f.Length) { + if !http2takeInflows(&sc.inflow, &st.inflow, f.Length) { return sc.countError("flow_on_data_length", http2streamError(id, http2ErrCodeFlowControl)) } - st.inflow.take(int32(f.Length)) if len(data) > 0 { wrote, err := st.body.Write(data) @@ -5574,10 +5635,12 @@ func (sc *http2serverConn) processData(f *http2DataFrame) error { // Return any padded flow control now, since we won't // refund it later on body reads. - if pad := int32(f.Length) - int32(len(data)); pad > 0 { - sc.sendWindowUpdate32(nil, pad) - sc.sendWindowUpdate32(st, pad) - } + // Call sendWindowUpdate even if there is no padding, + // to return buffered flow control credit if the sent + // window has shrunk. + pad := int32(f.Length) - int32(len(data)) + sc.sendWindowUpdate32(nil, pad) + sc.sendWindowUpdate32(st, pad) } if f.StreamEnded() { st.endStream() @@ -5849,8 +5912,7 @@ func (sc *http2serverConn) newStream(id, pusherID uint32, state http2streamState st.cw.Init() st.flow.conn = &sc.flow // link to conn-level counter st.flow.add(sc.initialStreamSendWindowSize) - st.inflow.conn = &sc.inflow // link to conn-level counter - st.inflow.add(sc.srv.initialStreamRecvWindowSize()) + st.inflow.init(sc.srv.initialStreamRecvWindowSize()) if sc.hs.WriteTimeout != 0 { st.writeDeadline = time.AfterFunc(sc.hs.WriteTimeout, st.onWriteTimeout) } @@ -5942,7 +6004,7 @@ func (sc *http2serverConn) newWriterAndRequestNoBody(st *http2stream, rp http2re tlsState = sc.tlsState } - needsContinue := rp.header.Get("Expect") == "100-continue" + needsContinue := httpguts.HeaderValuesContainsToken(rp.header["Expect"], "100-continue") if needsContinue { rp.header.Del("Expect") } @@ -6132,47 +6194,28 @@ func (sc *http2serverConn) noteBodyRead(st *http2stream, n int) { } // st may be nil for conn-level -func (sc *http2serverConn) sendWindowUpdate(st *http2stream, n int) { - sc.serveG.check() - // "The legal range for the increment to the flow control - // window is 1 to 2^31-1 (2,147,483,647) octets." - // A Go Read call on 64-bit machines could in theory read - // a larger Read than this. Very unlikely, but we handle it here - // rather than elsewhere for now. - const maxUint31 = 1<<31 - 1 - for n > maxUint31 { - sc.sendWindowUpdate32(st, maxUint31) - n -= maxUint31 - } - sc.sendWindowUpdate32(st, int32(n)) +func (sc *http2serverConn) sendWindowUpdate32(st *http2stream, n int32) { + sc.sendWindowUpdate(st, int(n)) } // st may be nil for conn-level -func (sc *http2serverConn) sendWindowUpdate32(st *http2stream, n int32) { +func (sc *http2serverConn) sendWindowUpdate(st *http2stream, n int) { sc.serveG.check() - if n == 0 { - return - } - if n < 0 { - panic("negative update") - } var streamID uint32 - if st != nil { + var send int32 + if st == nil { + send = sc.inflow.add(n) + } else { streamID = st.id + send = st.inflow.add(n) + } + if send == 0 { + return } sc.writeFrame(http2FrameWriteRequest{ - write: http2writeWindowUpdate{streamID: streamID, n: uint32(n)}, + write: http2writeWindowUpdate{streamID: streamID, n: uint32(send)}, stream: st, }) - var ok bool - if st == nil { - ok = sc.inflow.add(n) - } else { - ok = st.inflow.add(n) - } - if !ok { - panic("internal error; sent too many window updates without decrements?") - } } // requestBody is the Handler's Request.Body type. @@ -7004,10 +7047,6 @@ const ( // we buffer per stream. http2transportDefaultStreamFlow = 4 << 20 - // transportDefaultStreamMinRefresh is the minimum number of bytes we'll send - // a stream-level WINDOW_UPDATE for at a time. - http2transportDefaultStreamMinRefresh = 4 << 10 - http2defaultUserAgent = "Go-http-client/2.0" // initialMaxConcurrentStreams is a connections maxConcurrentStreams until @@ -7265,11 +7304,11 @@ type http2ClientConn struct { idleTimeout time.Duration // or 0 for never idleTimer *time.Timer - mu sync.Mutex // guards following - cond *sync.Cond // hold mu; broadcast on flow/closed changes - flow http2flow // our conn-level flow control quota (cs.flow is per stream) - inflow http2flow // peer's conn-level flow control - doNotReuse bool // whether conn is marked to not be reused for any future requests + mu sync.Mutex // guards following + cond *sync.Cond // hold mu; broadcast on flow/closed changes + flow http2outflow // our conn-level flow control quota (cs.outflow is per stream) + inflow http2inflow // peer's conn-level flow control + doNotReuse bool // whether conn is marked to not be reused for any future requests closing bool closed bool seenSettings bool // true if we've seen a settings frame, false otherwise @@ -7333,10 +7372,10 @@ type http2clientStream struct { respHeaderRecv chan struct{} // closed when headers are received res *Response // set if respHeaderRecv is closed - flow http2flow // guarded by cc.mu - inflow http2flow // guarded by cc.mu - bytesRemain int64 // -1 means unknown; owned by transportResponseBody.Read - readErr error // sticky read error; owned by transportResponseBody.Read + flow http2outflow // guarded by cc.mu + inflow http2inflow // guarded by cc.mu + bytesRemain int64 // -1 means unknown; owned by transportResponseBody.Read + readErr error // sticky read error; owned by transportResponseBody.Read reqBody io.ReadCloser reqBodyContentLength int64 // -1 means unknown @@ -7769,7 +7808,7 @@ func (t *http2Transport) newClientConn(c net.Conn, singleUse bool) (*http2Client cc.bw.Write(http2clientPreface) cc.fr.WriteSettings(initialSettings...) cc.fr.WriteWindowUpdate(0, http2transportDefaultConnFlow) - cc.inflow.add(http2transportDefaultConnFlow + http2initialWindowSize) + cc.inflow.init(http2transportDefaultConnFlow + http2initialWindowSize) cc.bw.Flush() if cc.werr != nil { cc.Close() @@ -8531,7 +8570,7 @@ func (cs *http2clientStream) cleanupWriteRequest(err error) { close(cs.donec) } -// awaitOpenSlotForStream waits until len(streams) < maxConcurrentStreams. +// awaitOpenSlotForStreamLocked waits until len(streams) < maxConcurrentStreams. // Must hold cc.mu. func (cc *http2ClientConn) awaitOpenSlotForStreamLocked(cs *http2clientStream) error { for { @@ -9031,8 +9070,7 @@ type http2resAndError struct { func (cc *http2ClientConn) addStreamLocked(cs *http2clientStream) { cs.flow.add(int32(cc.initialWindowSize)) cs.flow.setConnFlow(&cc.flow) - cs.inflow.add(http2transportDefaultStreamFlow) - cs.inflow.setConnFlow(&cc.inflow) + cs.inflow.init(http2transportDefaultStreamFlow) cs.ID = cc.nextStreamID cc.nextStreamID += 2 cc.streams[cs.ID] = cs @@ -9491,21 +9529,10 @@ func (b http2transportResponseBody) Read(p []byte) (n int, err error) { } cc.mu.Lock() - var connAdd, streamAdd int32 - // Check the conn-level first, before the stream-level. - if v := cc.inflow.available(); v < http2transportDefaultConnFlow/2 { - connAdd = http2transportDefaultConnFlow - v - cc.inflow.add(connAdd) - } + connAdd := cc.inflow.add(n) + var streamAdd int32 if err == nil { // No need to refresh if the stream is over or failed. - // Consider any buffered body data (read from the conn but not - // consumed by the client) when computing flow control for this - // stream. - v := int(cs.inflow.available()) + cs.bufPipe.Len() - if v < http2transportDefaultStreamFlow-http2transportDefaultStreamMinRefresh { - streamAdd = int32(http2transportDefaultStreamFlow - v) - cs.inflow.add(streamAdd) - } + streamAdd = cs.inflow.add(n) } cc.mu.Unlock() @@ -9533,17 +9560,15 @@ func (b http2transportResponseBody) Close() error { if unread > 0 { cc.mu.Lock() // Return connection-level flow control. - if unread > 0 { - cc.inflow.add(int32(unread)) - } + connAdd := cc.inflow.add(unread) cc.mu.Unlock() // TODO(dneil): Acquiring this mutex can block indefinitely. // Move flow control return to a goroutine? cc.wmu.Lock() // Return connection-level flow control. - if unread > 0 { - cc.fr.WriteWindowUpdate(0, uint32(unread)) + if connAdd > 0 { + cc.fr.WriteWindowUpdate(0, uint32(connAdd)) } cc.bw.Flush() cc.wmu.Unlock() @@ -9586,13 +9611,18 @@ func (rl *http2clientConnReadLoop) processData(f *http2DataFrame) error { // But at least return their flow control: if f.Length > 0 { cc.mu.Lock() - cc.inflow.add(int32(f.Length)) + ok := cc.inflow.take(f.Length) + connAdd := cc.inflow.add(int(f.Length)) cc.mu.Unlock() - - cc.wmu.Lock() - cc.fr.WriteWindowUpdate(0, uint32(f.Length)) - cc.bw.Flush() - cc.wmu.Unlock() + if !ok { + return http2ConnectionError(http2ErrCodeFlowControl) + } + if connAdd > 0 { + cc.wmu.Lock() + cc.fr.WriteWindowUpdate(0, uint32(connAdd)) + cc.bw.Flush() + cc.wmu.Unlock() + } } return nil } @@ -9623,9 +9653,7 @@ func (rl *http2clientConnReadLoop) processData(f *http2DataFrame) error { } // Check connection-level flow control. cc.mu.Lock() - if cs.inflow.available() >= int32(f.Length) { - cs.inflow.take(int32(f.Length)) - } else { + if !http2takeInflows(&cc.inflow, &cs.inflow, f.Length) { cc.mu.Unlock() return http2ConnectionError(http2ErrCodeFlowControl) } @@ -9647,19 +9675,20 @@ func (rl *http2clientConnReadLoop) processData(f *http2DataFrame) error { } } - if refund > 0 { - cc.inflow.add(int32(refund)) - if !didReset { - cs.inflow.add(int32(refund)) - } + sendConn := cc.inflow.add(refund) + var sendStream int32 + if !didReset { + sendStream = cs.inflow.add(refund) } cc.mu.Unlock() - if refund > 0 { + if sendConn > 0 || sendStream > 0 { cc.wmu.Lock() - cc.fr.WriteWindowUpdate(0, uint32(refund)) - if !didReset { - cc.fr.WriteWindowUpdate(cs.ID, uint32(refund)) + if sendConn > 0 { + cc.fr.WriteWindowUpdate(0, uint32(sendConn)) + } + if sendStream > 0 { + cc.fr.WriteWindowUpdate(cs.ID, uint32(sendStream)) } cc.bw.Flush() cc.wmu.Unlock() From 3d357e8106b89e15071ef27cbf739fb9fb6e4d48 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Mon, 13 Feb 2023 10:58:32 -0800 Subject: [PATCH 14/86] all: update vendored golang.org/x/net Pull in HTTP/2 fix to deflake builders: 547e7edf38 http2: avoid referencing ResponseWrite.Write parameter after returning For #58446 Change-Id: I7f3666bc1f20ee03a7ccf25f0e091033cbc635d7 Reviewed-on: https://go-review.googlesource.com/c/go/+/467657 Auto-Submit: Damien Neil Run-TryBot: Damien Neil Reviewed-by: Bryan Mills TryBot-Result: Gopher Robot --- h2_bundle.go | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/h2_bundle.go b/h2_bundle.go index eb3fd159..b451cee9 100644 --- a/h2_bundle.go +++ b/h2_bundle.go @@ -2063,6 +2063,15 @@ func (f *http2Framer) WriteData(streamID uint32, endStream bool, data []byte) er // It is the caller's responsibility not to violate the maximum frame size // and to not call other Write methods concurrently. func (f *http2Framer) WriteDataPadded(streamID uint32, endStream bool, data, pad []byte) error { + if err := f.startWriteDataPadded(streamID, endStream, data, pad); err != nil { + return err + } + return f.endWrite() +} + +// startWriteDataPadded is WriteDataPadded, but only writes the frame to the Framer's internal buffer. +// The caller should call endWrite to flush the frame to the underlying writer. +func (f *http2Framer) startWriteDataPadded(streamID uint32, endStream bool, data, pad []byte) error { if !http2validStreamID(streamID) && !f.AllowIllegalWrites { return http2errStreamID } @@ -2092,7 +2101,7 @@ func (f *http2Framer) WriteDataPadded(streamID uint32, endStream bool, data, pad } f.wbuf = append(f.wbuf, data...) f.wbuf = append(f.wbuf, pad...) - return f.endWrite() + return nil } // A SettingsFrame conveys configuration parameters that affect how @@ -4653,8 +4662,13 @@ type http2frameWriteResult struct { // and then reports when it's done. // At most one goroutine can be running writeFrameAsync at a time per // serverConn. -func (sc *http2serverConn) writeFrameAsync(wr http2FrameWriteRequest) { - err := wr.write.writeFrame(sc) +func (sc *http2serverConn) writeFrameAsync(wr http2FrameWriteRequest, wd *http2writeData) { + var err error + if wd == nil { + err = wr.write.writeFrame(sc) + } else { + err = sc.framer.endWrite() + } sc.wroteFrameCh <- http2frameWriteResult{wr: wr, err: err} } @@ -5063,9 +5077,16 @@ func (sc *http2serverConn) startFrameWrite(wr http2FrameWriteRequest) { sc.writingFrameAsync = false err := wr.write.writeFrame(sc) sc.wroteFrame(http2frameWriteResult{wr: wr, err: err}) + } else if wd, ok := wr.write.(*http2writeData); ok { + // Encode the frame in the serve goroutine, to ensure we don't have + // any lingering asynchronous references to data passed to Write. + // See https://go.dev/issue/58446. + sc.framer.startWriteDataPadded(wd.streamID, wd.endStream, wd.p, nil) + sc.writingFrameAsync = true + go sc.writeFrameAsync(wr, wd) } else { sc.writingFrameAsync = true - go sc.writeFrameAsync(wr) + go sc.writeFrameAsync(wr, nil) } } From 3a1e1be57225585cd70b4ebe50b1d2448238dd93 Mon Sep 17 00:00:00 2001 From: cui fliter Date: Tue, 14 Feb 2023 22:42:38 +0800 Subject: [PATCH 15/86] all: fix some comments Change-Id: I16ec916b47de2f417b681c8abff5a1375ddf491b Reviewed-on: https://go-review.googlesource.com/c/go/+/468055 Run-TryBot: Ian Lance Taylor TryBot-Result: Gopher Robot Auto-Submit: Ian Lance Taylor Reviewed-by: Ian Lance Taylor Reviewed-by: Dmitri Shuralyov --- server.go | 4 ++-- transfer.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server.go b/server.go index c15f0f58..1ac61f71 100644 --- a/server.go +++ b/server.go @@ -1749,7 +1749,7 @@ type closeWriter interface { var _ closeWriter = (*net.TCPConn)(nil) -// closeWrite flushes any outstanding data and sends a FIN packet (if +// closeWriteAndWait flushes any outstanding data and sends a FIN packet (if // client is connected via TCP), signaling that we're done. We then // pause for a bit, hoping the client processes it before any // subsequent RST. @@ -2990,7 +2990,7 @@ func (srv *Server) ListenAndServe() error { var testHookServerServe func(*Server, net.Listener) // used if non-nil -// shouldDoServeHTTP2 reports whether Server.Serve should configure +// shouldConfigureHTTP2ForServe reports whether Server.Serve should configure // automatic HTTP/2. (which sets up the srv.TLSNextProto map) func (srv *Server) shouldConfigureHTTP2ForServe() bool { if srv.TLSConfig == nil { diff --git a/transfer.go b/transfer.go index 7c7afd7d..d6f26a70 100644 --- a/transfer.go +++ b/transfer.go @@ -416,7 +416,7 @@ func (t *transferWriter) doBodyCopy(dst io.Writer, src io.Reader) (n int64, err return } -// unwrapBodyReader unwraps the body's inner reader if it's a +// unwrapBody unwraps the body's inner reader if it's a // nopCloser. This is to ensure that body writes sourced from local // files (*os.File types) are properly optimized. // From df079bc067db188d70de5326ebc19c8d4a9fd4cc Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Wed, 25 Jan 2023 09:27:01 -0800 Subject: [PATCH 16/86] mime/multipart: limit memory/inode consumption of ReadForm Reader.ReadForm is documented as storing "up to maxMemory bytes + 10MB" in memory. Parsed forms can consume substantially more memory than this limit, since ReadForm does not account for map entry overhead and MIME headers. In addition, while the amount of disk memory consumed by ReadForm can be constrained by limiting the size of the parsed input, ReadForm will create one temporary file per form part stored on disk, potentially consuming a large number of inodes. Update ReadForm's memory accounting to include part names, MIME headers, and map entry overhead. Update ReadForm to store all on-disk file parts in a single temporary file. Files returned by FileHeader.Open are documented as having a concrete type of *os.File when a file is stored on disk. The change to use a single temporary file for all parts means that this is no longer the case when a form contains more than a single file part stored on disk. The previous behavior of storing each file part in a separate disk file may be reenabled with GODEBUG=multipartfiles=distinct. Update Reader.NextPart and Reader.NextRawPart to set a 10MiB cap on the size of MIME headers. Thanks to Jakob Ackermann (@das7pad) for reporting this issue. Fixes #58006 Fixes CVE-2022-41725 Change-Id: Ibd780a6c4c83ac8bcfd3cbe344f042e9940f2eab Reviewed-on: https://team-review.git.corp.google.com/c/golang/go-private/+/1714276 Reviewed-by: Julie Qiu TryBot-Result: Security TryBots Reviewed-by: Roland Shoemaker Run-TryBot: Damien Neil Reviewed-on: https://go-review.googlesource.com/c/go/+/468124 Auto-Submit: Michael Pratt Run-TryBot: Michael Pratt Reviewed-by: Than McIntosh TryBot-Result: Gopher Robot --- request_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/request_test.go b/request_test.go index 686a8699..23e49d6b 100644 --- a/request_test.go +++ b/request_test.go @@ -1097,7 +1097,7 @@ func testMissingFile(t *testing.T, req *Request) { t.Errorf("FormFile file = %v, want nil", f) } if fh != nil { - t.Errorf("FormFile file header = %q, want nil", fh) + t.Errorf("FormFile file header = %v, want nil", fh) } if err != ErrMissingFile { t.Errorf("FormFile err = %q, want ErrMissingFile", err) From de31f4f8bdd9598dae48d5fd9342e5d8ba02c443 Mon Sep 17 00:00:00 2001 From: Will Hawkins Date: Mon, 19 Dec 2022 11:49:33 -0500 Subject: [PATCH 17/86] net/http: support streaming POST content in wasm With new releases of Chrome, Opera and Deno it is possible to stream the body of a POST request. Add support for using that interface when it is available. Change-Id: Ib23d63cd3dea634bd9e267abf4e9a9bfa9c525ad Reviewed-on: https://go-review.googlesource.com/c/go/+/458395 Auto-Submit: Johan Brandhorst-Satzkorn Reviewed-by: Michael Pratt Run-TryBot: Dmitri Shuralyov TryBot-Result: Gopher Robot Reviewed-by: Dmitri Shuralyov Reviewed-by: Johan Brandhorst-Satzkorn --- roundtrip_js.go | 112 +++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 96 insertions(+), 16 deletions(-) diff --git a/roundtrip_js.go b/roundtrip_js.go index 4f381247..f4d0b9d4 100644 --- a/roundtrip_js.go +++ b/roundtrip_js.go @@ -50,6 +50,38 @@ var jsFetchMissing = js.Global().Get("fetch").IsUndefined() // our wasm tests. See https://go.dev/issue/57613 for more information. var jsFetchDisabled = !js.Global().Get("process").IsUndefined() +// Determine whether the JS runtime supports streaming request bodies. +// Courtesy: https://developer.chrome.com/articles/fetch-streaming-requests/#feature-detection +func supportsPostRequestStreams() bool { + requestOpt := js.Global().Get("Object").New() + requestBody := js.Global().Get("ReadableStream").New() + + requestOpt.Set("method", "POST") + requestOpt.Set("body", requestBody) + + // There is quite a dance required to define a getter if you do not have the { get property() { ... } } + // syntax available. However, it is possible: + // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Functions/get#defining_a_getter_on_existing_objects_using_defineproperty + duplexCalled := false + duplexGetterObj := js.Global().Get("Object").New() + duplexGetterFunc := js.FuncOf(func(this js.Value, args []js.Value) any { + duplexCalled = true + return "half" + }) + defer duplexGetterFunc.Release() + duplexGetterObj.Set("get", duplexGetterFunc) + js.Global().Get("Object").Call("defineProperty", requestOpt, "duplex", duplexGetterObj) + + // Slight difference here between the aforementioned example: Non-browser-based runtimes + // do not have a non-empty API Base URL (https://html.spec.whatwg.org/multipage/webappapis.html#api-base-url) + // so we have to supply a valid URL here. + requestObject := js.Global().Get("Request").New("https://www.example.org", requestOpt) + + hasContentTypeHeader := requestObject.Get("headers").Call("has", "Content-Type").Bool() + + return duplexCalled && !hasContentTypeHeader +} + // RoundTrip implements the RoundTripper interface using the WHATWG Fetch API. func (t *Transport) RoundTrip(req *Request) (*Response, error) { // The Transport has a documented contract that states that if the DialContext or @@ -98,23 +130,60 @@ func (t *Transport) RoundTrip(req *Request) (*Response, error) { } opt.Set("headers", headers) + var readableStreamStart, readableStreamPull, readableStreamCancel js.Func if req.Body != nil { - // TODO(johanbrandhorst): Stream request body when possible. - // See https://bugs.chromium.org/p/chromium/issues/detail?id=688906 for Blink issue. - // See https://bugzilla.mozilla.org/show_bug.cgi?id=1387483 for Firefox issue. - // See https://github.com/web-platform-tests/wpt/issues/7693 for WHATWG tests issue. - // See https://developer.mozilla.org/en-US/docs/Web/API/Streams_API for more details on the Streams API - // and browser support. - body, err := io.ReadAll(req.Body) - if err != nil { - req.Body.Close() // RoundTrip must always close the body, including on errors. - return nil, err - } - req.Body.Close() - if len(body) != 0 { - buf := uint8Array.New(len(body)) - js.CopyBytesToJS(buf, body) - opt.Set("body", buf) + if !supportsPostRequestStreams() { + body, err := io.ReadAll(req.Body) + if err != nil { + req.Body.Close() // RoundTrip must always close the body, including on errors. + return nil, err + } + if len(body) != 0 { + buf := uint8Array.New(len(body)) + js.CopyBytesToJS(buf, body) + opt.Set("body", buf) + } + } else { + readableStreamCtorArg := js.Global().Get("Object").New() + readableStreamCtorArg.Set("type", "bytes") + readableStreamCtorArg.Set("autoAllocateChunkSize", t.writeBufferSize()) + + readableStreamPull = js.FuncOf(func(this js.Value, args []js.Value) any { + controller := args[0] + byobRequest := controller.Get("byobRequest") + if byobRequest.IsNull() { + controller.Call("close") + } + + byobRequestView := byobRequest.Get("view") + + bodyBuf := make([]byte, byobRequestView.Get("byteLength").Int()) + readBytes, readErr := io.ReadFull(req.Body, bodyBuf) + if readBytes > 0 { + buf := uint8Array.New(byobRequestView.Get("buffer")) + js.CopyBytesToJS(buf, bodyBuf) + byobRequest.Call("respond", readBytes) + } + + if readErr == io.EOF || readErr == io.ErrUnexpectedEOF { + controller.Call("close") + } else if readErr != nil { + readErrCauseObject := js.Global().Get("Object").New() + readErrCauseObject.Set("cause", readErr.Error()) + readErr := js.Global().Get("Error").New("io.ReadFull failed while streaming POST body", readErrCauseObject) + controller.Call("error", readErr) + } + // Note: This a return from the pull callback of the controller and *not* RoundTrip(). + return nil + }) + readableStreamCtorArg.Set("pull", readableStreamPull) + + opt.Set("body", js.Global().Get("ReadableStream").New(readableStreamCtorArg)) + // There is a requirement from the WHATWG fetch standard that the duplex property of + // the object given as the options argument to the fetch call be set to 'half' + // when the body property of the same options object is a ReadableStream: + // https://fetch.spec.whatwg.org/#dom-requestinit-duplex + opt.Set("duplex", "half") } } @@ -127,6 +196,11 @@ func (t *Transport) RoundTrip(req *Request) (*Response, error) { success = js.FuncOf(func(this js.Value, args []js.Value) any { success.Release() failure.Release() + readableStreamCancel.Release() + readableStreamPull.Release() + readableStreamStart.Release() + + req.Body.Close() result := args[0] header := Header{} @@ -191,6 +265,12 @@ func (t *Transport) RoundTrip(req *Request) (*Response, error) { failure = js.FuncOf(func(this js.Value, args []js.Value) any { success.Release() failure.Release() + readableStreamCancel.Release() + readableStreamPull.Release() + readableStreamStart.Release() + + req.Body.Close() + err := args[0] // The error is a JS Error type // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Error From 0c19d74bd3453d6d22c80114572f6723e1b0a427 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Thu, 26 Jan 2023 13:16:26 -0800 Subject: [PATCH 18/86] net/http: remove five second timeout from TestTransportClosesBodyOnError Wait forever and let the test time out with a stack trace if the expected response doesn't happen. Fixes #57990 Change-Id: I835def63db113752cdd06e03e258cb10d63a6a25 Reviewed-on: https://go-review.googlesource.com/c/go/+/463222 TryBot-Result: Gopher Robot Run-TryBot: Damien Neil Reviewed-by: Bryan Mills --- transport_test.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/transport_test.go b/transport_test.go index 2879dee0..cb5af755 100644 --- a/transport_test.go +++ b/transport_test.go @@ -3700,13 +3700,8 @@ func testTransportClosesBodyOnError(t *testing.T, mode testMode) { if err == nil || !strings.Contains(err.Error(), fakeErr.Error()) { t.Fatalf("Do error = %v; want something containing %q", err, fakeErr.Error()) } - select { - case err := <-readBody: - if err == nil { - t.Errorf("Unexpected success reading request body from handler; want 'unexpected EOF reading trailer'") - } - case <-time.After(5 * time.Second): - t.Error("timeout waiting for server handler to complete") + if err := <-readBody; err == nil { + t.Errorf("Unexpected success reading request body from handler; want 'unexpected EOF reading trailer'") } select { case <-didClose: From d1ca58d5fb2e9f79e7ecd23dba636fe6a6d154c1 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Wed, 22 Feb 2023 08:42:37 -0800 Subject: [PATCH 19/86] net/http: remove warning when parsing a query containing a semicolon It's been years since the behavior here was changed, and there's little point in continuing to warn users of it. Fixes #49399 Change-Id: I95f64ca14cacb64ebe78296593b1cc3d837e6b77 Reviewed-on: https://go-review.googlesource.com/c/go/+/470315 Run-TryBot: Damien Neil TryBot-Result: Gopher Robot Reviewed-by: Filippo Valsorda Reviewed-by: Than McIntosh --- serve_test.go | 28 +++++++++------------------- server.go | 17 ----------------- 2 files changed, 9 insertions(+), 36 deletions(-) diff --git a/serve_test.go b/serve_test.go index e11de660..b2bdeb10 100644 --- a/serve_test.go +++ b/serve_test.go @@ -6558,10 +6558,10 @@ func TestQuerySemicolon(t *testing.T) { t.Cleanup(func() { afterTest(t) }) tests := []struct { - query string - xNoSemicolons string - xWithSemicolons string - warning bool + query string + xNoSemicolons string + xWithSemicolons string + expectParseFormErr bool }{ {"?a=1;x=bad&x=good", "good", "bad", true}, {"?a=1;b=bad&x=good", "good", "good", true}, @@ -6573,20 +6573,20 @@ func TestQuerySemicolon(t *testing.T) { for _, tt := range tests { t.Run(tt.query+"/allow=false", func(t *testing.T) { allowSemicolons := false - testQuerySemicolon(t, mode, tt.query, tt.xNoSemicolons, allowSemicolons, tt.warning) + testQuerySemicolon(t, mode, tt.query, tt.xNoSemicolons, allowSemicolons, tt.expectParseFormErr) }) t.Run(tt.query+"/allow=true", func(t *testing.T) { - allowSemicolons, expectWarning := true, false - testQuerySemicolon(t, mode, tt.query, tt.xWithSemicolons, allowSemicolons, expectWarning) + allowSemicolons, expectParseFormErr := true, false + testQuerySemicolon(t, mode, tt.query, tt.xWithSemicolons, allowSemicolons, expectParseFormErr) }) } }) } -func testQuerySemicolon(t *testing.T, mode testMode, query string, wantX string, allowSemicolons, expectWarning bool) { +func testQuerySemicolon(t *testing.T, mode testMode, query string, wantX string, allowSemicolons, expectParseFormErr bool) { writeBackX := func(w ResponseWriter, r *Request) { x := r.URL.Query().Get("x") - if expectWarning { + if expectParseFormErr { if err := r.ParseForm(); err == nil || !strings.Contains(err.Error(), "semicolon") { t.Errorf("expected error mentioning semicolons from ParseForm, got %v", err) } @@ -6624,16 +6624,6 @@ func testQuerySemicolon(t *testing.T, mode testMode, query string, wantX string, if got, want := string(slurp), wantX; got != want { t.Errorf("Body = %q; want = %q", got, want) } - - if expectWarning { - if !strings.Contains(logBuf.String(), "semicolon") { - t.Errorf("got %q from ErrorLog, expected a mention of semicolons", logBuf.String()) - } - } else { - if strings.Contains(logBuf.String(), "semicolon") { - t.Errorf("got %q from ErrorLog, expected no mention of semicolons", logBuf.String()) - } - } } func TestMaxBytesHandler(t *testing.T) { diff --git a/server.go b/server.go index 1ac61f71..1b3b2f2e 100644 --- a/server.go +++ b/server.go @@ -2921,23 +2921,9 @@ func (sh serverHandler) ServeHTTP(rw ResponseWriter, req *Request) { handler = globalOptionsHandler{} } - if req.URL != nil && strings.Contains(req.URL.RawQuery, ";") { - var allowQuerySemicolonsInUse atomic.Bool - req = req.WithContext(context.WithValue(req.Context(), silenceSemWarnContextKey, func() { - allowQuerySemicolonsInUse.Store(true) - })) - defer func() { - if !allowQuerySemicolonsInUse.Load() { - sh.srv.logf("http: URL query contains semicolon, which is no longer a supported separator; parts of the query may be stripped when parsed; see golang.org/issue/25192") - } - }() - } - handler.ServeHTTP(rw, req) } -var silenceSemWarnContextKey = &contextKey{"silence-semicolons"} - // AllowQuerySemicolons returns a handler that serves requests by converting any // unescaped semicolons in the URL query to ampersands, and invoking the handler h. // @@ -2949,9 +2935,6 @@ var silenceSemWarnContextKey = &contextKey{"silence-semicolons"} // AllowQuerySemicolons should be invoked before Request.ParseForm is called. func AllowQuerySemicolons(h Handler) Handler { return HandlerFunc(func(w ResponseWriter, r *Request) { - if silenceSemicolonsWarning, ok := r.Context().Value(silenceSemWarnContextKey).(func()); ok { - silenceSemicolonsWarning() - } if strings.Contains(r.URL.RawQuery, ";") { r2 := new(Request) *r2 = *r From c693d2c35af55428316ce0fa026bd6d8e21ee4bc Mon Sep 17 00:00:00 2001 From: Andy Pan Date: Wed, 24 Aug 2022 23:02:16 +0800 Subject: [PATCH 20/86] net/http: reduce calls to append in hexEscapeNonASCII to gain a slight performance boost MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit goos: linux goarch: amd64 pkg: net/http cpu: DO-Premium-Intel │ old │ new │ │ sec/op │ sec/op vs base │ HexEscapeNonASCII-4 469.6n ± 20% 371.1n ± 9% -20.98% (p=0.000 n=10) │ old │ new │ │ B/op │ B/op vs base │ HexEscapeNonASCII-4 192.0 ± 0% 192.0 ± 0% ~ (p=1.000 n=10) ¹ ¹ all samples are equal │ old │ new │ │ allocs/op │ allocs/op vs base │ HexEscapeNonASCII-4 2.000 ± 0% 2.000 ± 0% ~ (p=1.000 n=10) ¹ ¹ all samples are equal Change-Id: Ic8d2b3ddcf2cf724dec3f51a2aba205f2c6e4fe6 Reviewed-on: https://go-review.googlesource.com/c/go/+/425786 Reviewed-by: Than McIntosh Auto-Submit: Ian Lance Taylor Run-TryBot: Andy Pan TryBot-Result: Gopher Robot Reviewed-by: Ian Lance Taylor Reviewed-by: Dmitri Shuralyov --- http.go | 10 ++++++++-- http_test.go | 10 ++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/http.go b/http.go index 101799f5..9b81654f 100644 --- a/http.go +++ b/http.go @@ -86,14 +86,20 @@ func hexEscapeNonASCII(s string) string { return s } b := make([]byte, 0, newLen) + var pos int for i := 0; i < len(s); i++ { if s[i] >= utf8.RuneSelf { + if pos < i { + b = append(b, s[pos:i]...) + } b = append(b, '%') b = strconv.AppendInt(b, int64(s[i]), 16) - } else { - b = append(b, s[i]) + pos = i + 1 } } + if pos < len(s) { + b = append(b, s[pos:]...) + } return string(b) } diff --git a/http_test.go b/http_test.go index 0d92fe5f..1c9fb33b 100644 --- a/http_test.go +++ b/http_test.go @@ -218,3 +218,13 @@ func TestNoUnicodeStrings(t *testing.T) { t.Fatal(err) } } + +const redirectURL = "/thisaredirect细雪withasciilettersのけぶabcdefghijk.html" + +func BenchmarkHexEscapeNonASCII(b *testing.B) { + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + hexEscapeNonASCII(redirectURL) + } +} From 84b7f6a472d95b650a68e9bd5e79a6e6ce020875 Mon Sep 17 00:00:00 2001 From: pgxiaolianzi Date: Mon, 27 Feb 2023 06:51:55 +0000 Subject: [PATCH 21/86] all: fix typos Change-Id: Ica8d5e5799a4de532764ae86cdb623508d3a8e18 GitHub-Last-Rev: 3e97cca9de3885f2fe0d7deb776e59cc1c73146d GitHub-Pull-Request: golang/go#58689 Reviewed-on: https://go-review.googlesource.com/c/go/+/471021 Auto-Submit: Ian Lance Taylor Reviewed-by: Ian Lance Taylor TryBot-Result: Gopher Robot Reviewed-by: Dmitri Shuralyov Run-TryBot: Ian Lance Taylor --- transport_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transport_test.go b/transport_test.go index cb5af755..cae98767 100644 --- a/transport_test.go +++ b/transport_test.go @@ -4969,7 +4969,7 @@ func testTLSHandshakeTrace(t *testing.T, mode testMode) { t.Fatal("Expected TLSHandshakeStart to be called, but wasn't") } if !done { - t.Fatal("Expected TLSHandshakeDone to be called, but wasnt't") + t.Fatal("Expected TLSHandshakeDone to be called, but wasn't") } } From f53bcdabb7d70de01fd0c4f234db38543509b651 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Wed, 1 Mar 2023 15:17:35 -0800 Subject: [PATCH 22/86] net/http: support full-duplex HTTP/1 responses Add support for concurrently reading from an HTTP/1 request body while writing the response. Normally, the HTTP/1 server automatically consumes any remaining request body before starting to write a response, to avoid deadlocking clients which attempt to write a complete request before reading the response. Add a ResponseController.EnableFullDuplex method which disables this behavior. For #15527 For #57786 Change-Id: Ie7ee8267d8333e9b32b82b9b84d4ad28ab8edf01 Reviewed-on: https://go-review.googlesource.com/c/go/+/472636 TryBot-Result: Gopher Robot Run-TryBot: Damien Neil Reviewed-by: Roland Shoemaker --- responsecontroller.go | 25 ++++++++++++++++++++ responsecontroller_test.go | 48 ++++++++++++++++++++++++++++++++++++++ server.go | 25 +++++++++++++------- 3 files changed, 90 insertions(+), 8 deletions(-) diff --git a/responsecontroller.go b/responsecontroller.go index 018bdc00..92276ffa 100644 --- a/responsecontroller.go +++ b/responsecontroller.go @@ -31,6 +31,7 @@ type ResponseController struct { // Hijack() (net.Conn, *bufio.ReadWriter, error) // SetReadDeadline(deadline time.Time) error // SetWriteDeadline(deadline time.Time) error +// EnableFullDuplex() error // // If the ResponseWriter does not support a method, ResponseController returns // an error matching ErrNotSupported. @@ -115,6 +116,30 @@ func (c *ResponseController) SetWriteDeadline(deadline time.Time) error { } } +// EnableFullDuplex indicates that the request handler will interleave reads from Request.Body +// with writes to the ResponseWriter. +// +// For HTTP/1 requests, the Go HTTP server by default consumes any unread portion of +// the request body before beginning to write the response, preventing handlers from +// concurrently reading from the request and writing the response. +// Calling EnableFullDuplex disables this behavior and permits handlers to continue to read +// from the request while concurrently writing the response. +// +// For HTTP/2 requests, the Go HTTP server always permits concurrent reads and responses. +func (c *ResponseController) EnableFullDuplex() error { + rw := c.rw + for { + switch t := rw.(type) { + case interface{ EnableFullDuplex() error }: + return t.EnableFullDuplex() + case rwUnwrapper: + rw = t.Unwrap() + default: + return errNotSupported() + } + } +} + // errNotSupported returns an error that Is ErrNotSupported, // but is not == to it. func errNotSupported() error { diff --git a/responsecontroller_test.go b/responsecontroller_test.go index 0dca7332..ee8b55a8 100644 --- a/responsecontroller_test.go +++ b/responsecontroller_test.go @@ -263,3 +263,51 @@ func testWrappedResponseController(t *testing.T, mode testMode) { io.Copy(io.Discard, res.Body) defer res.Body.Close() } + +func TestResponseControllerEnableFullDuplex(t *testing.T) { + run(t, testResponseControllerEnableFullDuplex) +} +func testResponseControllerEnableFullDuplex(t *testing.T, mode testMode) { + cst := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, req *Request) { + ctl := NewResponseController(w) + if err := ctl.EnableFullDuplex(); err != nil { + // TODO: Drop test for HTTP/2 when x/net is updated to support + // EnableFullDuplex. Since HTTP/2 supports full duplex by default, + // the rest of the test is fine; it's just the EnableFullDuplex call + // that fails. + if mode != http2Mode { + t.Errorf("ctl.EnableFullDuplex() = %v, want nil", err) + } + } + w.WriteHeader(200) + ctl.Flush() + for { + var buf [1]byte + n, err := req.Body.Read(buf[:]) + if n != 1 || err != nil { + break + } + w.Write(buf[:]) + ctl.Flush() + } + })) + pr, pw := io.Pipe() + res, err := cst.c.Post(cst.ts.URL, "text/apocryphal", pr) + if err != nil { + t.Fatal(err) + } + defer res.Body.Close() + for i := byte(0); i < 10; i++ { + if _, err := pw.Write([]byte{i}); err != nil { + t.Fatalf("Write: %v", err) + } + var buf [1]byte + if n, err := res.Body.Read(buf[:]); n != 1 || err != nil { + t.Fatalf("Read: %v, %v", n, err) + } + if buf[0] != i { + t.Fatalf("read byte %v, want %v", buf[0], i) + } + } + pw.Close() +} diff --git a/server.go b/server.go index 1b3b2f2e..9bd381ff 100644 --- a/server.go +++ b/server.go @@ -460,6 +460,10 @@ type response struct { // Content-Length. closeAfterReply bool + // When fullDuplex is false (the default), we consume any remaining + // request body before starting to write a response. + fullDuplex bool + // requestBodyLimitHit is set by requestTooLarge when // maxBytesReader hits its max size. It is checked in // WriteHeader, to make sure we don't consume the @@ -497,6 +501,11 @@ func (c *response) SetWriteDeadline(deadline time.Time) error { return c.conn.rwc.SetWriteDeadline(deadline) } +func (c *response) EnableFullDuplex() error { + c.fullDuplex = true + return nil +} + // TrailerPrefix is a magic prefix for ResponseWriter.Header map keys // that, if present, signals that the map entry is actually for // the response trailers, and not the response headers. The prefix @@ -1354,14 +1363,14 @@ func (cw *chunkWriter) writeHeader(p []byte) { w.closeAfterReply = true } - // Per RFC 2616, we should consume the request body before - // replying, if the handler hasn't already done so. But we - // don't want to do an unbounded amount of reading here for - // DoS reasons, so we only try up to a threshold. - // TODO(bradfitz): where does RFC 2616 say that? See Issue 15527 - // about HTTP/1.x Handlers concurrently reading and writing, like - // HTTP/2 handlers can do. Maybe this code should be relaxed? - if w.req.ContentLength != 0 && !w.closeAfterReply { + // We do this by default because there are a number of clients that + // send a full request before starting to read the response, and they + // can deadlock if we start writing the response with unconsumed body + // remaining. See Issue 15527 for some history. + // + // If full duplex mode has been enabled with ResponseController.EnableFullDuplex, + // then leave the request body alone. + if w.req.ContentLength != 0 && !w.closeAfterReply && !w.fullDuplex { var discard, tooBig bool switch bdy := w.req.Body.(type) { From 409674b835c9909db156a4fb84176bef5eb207b5 Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Wed, 8 Mar 2023 17:10:06 -0500 Subject: [PATCH 23/86] net/http: remove arbitrary timeout in TestServerAllowsBlockingRemoteAddr If the test actually deadlocks, we probably want a goroutine dump to debug it anyway. Otherwise, the arbitrary timeout can only cause spurious failures. Fixes #36179. Change-Id: Ic2037496959a38d3231eefdbc1dd5d45eebdf306 Reviewed-on: https://go-review.googlesource.com/c/go/+/474582 Auto-Submit: Bryan Mills Reviewed-by: Damien Neil Run-TryBot: Bryan Mills TryBot-Result: Gopher Robot --- serve_test.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/serve_test.go b/serve_test.go index b2bdeb10..343a358e 100644 --- a/serve_test.go +++ b/serve_test.go @@ -1354,13 +1354,7 @@ func testServerAllowsBlockingRemoteAddr(t *testing.T, mode testMode) { // Start another request and grab its connection response2c := make(chan string, 1) go fetch(2, response2c) - var conn2 net.Conn - - select { - case conn2 = <-conns: - case <-time.After(time.Second): - t.Fatal("Second Accept didn't happen") - } + conn2 := <-conns // Send a response on connection 2. conn2.(*blockingRemoteAddrConn).addrs <- &net.TCPAddr{ From 777735fcea82d2f81c32b2409bc684a7020dd62a Mon Sep 17 00:00:00 2001 From: Leo Antunes Date: Sun, 30 Oct 2022 10:15:27 +0000 Subject: [PATCH 24/86] net/http: use Copy in ServeContent if CopyN not needed This small PR allows optimizations made in io.Copy (like the use of io.WriterTo) to be used in one possible path of http.ServeContent (in case of a non-Range request). This, in turn, allows us to skip the buffer allocation in io.Copy. Change-Id: Ifa2ece206ecd4556aaaed15d663b65e95e00bb0a GitHub-Last-Rev: 94fc0318145ba1bd48502564f6488aade871c301 GitHub-Pull-Request: golang/go#56480 Reviewed-on: https://go-review.googlesource.com/c/go/+/446276 Reviewed-by: Bryan Mills TryBot-Result: Gopher Robot Auto-Submit: Damien Neil Reviewed-by: Damien Neil Run-TryBot: Damien Neil --- fs.go | 9 +++++++-- fs_test.go | 43 +++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/fs.go b/fs.go index 83459046..7f302491 100644 --- a/fs.go +++ b/fs.go @@ -349,8 +349,13 @@ func serveContent(w ResponseWriter, r *Request, name string, modtime time.Time, w.WriteHeader(code) - if r.Method != "HEAD" { - io.CopyN(w, sendContent, sendSize) + if r.Method != MethodHead { + if sendSize == size { + // use Copy in the non-range case to make use of WriterTo if available + io.Copy(w, sendContent) + } else { + io.CopyN(w, sendContent, sendSize) + } } } diff --git a/fs_test.go b/fs_test.go index 74f7a80e..ce429201 100644 --- a/fs_test.go +++ b/fs_test.go @@ -937,6 +937,7 @@ func testServeContent(t *testing.T, mode testMode) { wantContentType string wantContentRange string wantStatus int + wantContent []byte } htmlModTime := mustStat(t, "testdata/index.html").ModTime() tests := map[string]testCase{ @@ -1152,6 +1153,24 @@ func testServeContent(t *testing.T, mode testMode) { wantStatus: 412, wantLastMod: htmlModTime.UTC().Format(TimeFormat), }, + "uses_writeTo_if_available_and_non-range": { + content: &panicOnNonWriterTo{seekWriterTo: strings.NewReader("foobar")}, + serveContentType: "text/plain; charset=utf-8", + wantContentType: "text/plain; charset=utf-8", + wantStatus: StatusOK, + wantContent: []byte("foobar"), + }, + "do_not_use_writeTo_for_range_requests": { + content: &panicOnWriterTo{ReadSeeker: strings.NewReader("foobar")}, + serveContentType: "text/plain; charset=utf-8", + reqHeader: map[string]string{ + "Range": "bytes=0-4", + }, + wantContentType: "text/plain; charset=utf-8", + wantContentRange: "bytes 0-4/6", + wantStatus: StatusPartialContent, + wantContent: []byte("fooba"), + }, } for testName, tt := range tests { var content io.ReadSeeker @@ -1165,7 +1184,8 @@ func testServeContent(t *testing.T, mode testMode) { } else { content = tt.content } - for _, method := range []string{"GET", "HEAD"} { + contentOut := &strings.Builder{} + for _, method := range []string{MethodGet, MethodHead} { //restore content in case it is consumed by previous method if content, ok := content.(*strings.Reader); ok { content.Seek(0, io.SeekStart) @@ -1191,7 +1211,8 @@ func testServeContent(t *testing.T, mode testMode) { if err != nil { t.Fatal(err) } - io.Copy(io.Discard, res.Body) + contentOut.Reset() + io.Copy(contentOut, res.Body) res.Body.Close() if res.StatusCode != tt.wantStatus { t.Errorf("test %q using %q: got status = %d; want %d", testName, method, res.StatusCode, tt.wantStatus) @@ -1205,10 +1226,28 @@ func testServeContent(t *testing.T, mode testMode) { if g, e := res.Header.Get("Last-Modified"), tt.wantLastMod; g != e { t.Errorf("test %q using %q: got last-modified = %q, want %q", testName, method, g, e) } + if g, e := contentOut.String(), tt.wantContent; e != nil && method == MethodGet && g != string(e) { + t.Errorf("test %q using %q: got unexpected content %q, want %q", testName, method, g, e) + } } } } +type seekWriterTo interface { + io.Seeker + io.WriterTo +} + +type panicOnNonWriterTo struct { + io.Reader + seekWriterTo +} + +type panicOnWriterTo struct { + io.ReadSeeker + io.WriterTo +} + // Issue 12991 func TestServerFileStatError(t *testing.T) { rec := httptest.NewRecorder() From a964b89b83720be7c603de00825cdf3f3e9ddea4 Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Mon, 13 Mar 2023 16:16:32 -0400 Subject: [PATCH 25/86] net/http: remove more arbitrary timeouts from server tests This change eliminates the easy, arbitrary timouts that should never happen. It leaves in place a couple of more complicated ones that will probably need retry loops for robustness. For #49336. For #36179. Change-Id: I657ef223a66461413a915da5ce9150f49acec04a Reviewed-on: https://go-review.googlesource.com/c/go/+/476035 Run-TryBot: Bryan Mills Auto-Submit: Bryan Mills TryBot-Result: Gopher Robot Reviewed-by: Damien Neil --- serve_test.go | 210 ++++++++++---------------------------------------- 1 file changed, 39 insertions(+), 171 deletions(-) diff --git a/serve_test.go b/serve_test.go index 343a358e..8ee8107c 100644 --- a/serve_test.go +++ b/serve_test.go @@ -35,7 +35,6 @@ import ( "reflect" "regexp" "runtime" - "runtime/debug" "strconv" "strings" "sync" @@ -827,15 +826,7 @@ func testWriteDeadlineExtendedOnNewRequest(t *testing.T, mode testMode) { t.Fatal(err) } - // fail test if no response after 1 second - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) - defer cancel() - req = req.WithContext(ctx) - r, err := c.Do(req) - if ctx.Err() == context.DeadlineExceeded { - t.Fatalf("http2 Get #%d response timed out", i) - } if err != nil { t.Fatalf("http2 Get #%d: %v", i, err) } @@ -988,25 +979,19 @@ func testOnlyWriteTimeout(t *testing.T, mode testMode) { c := ts.Client() - errc := make(chan error, 1) - go func() { + err := func() error { res, err := c.Get(ts.URL) if err != nil { - errc <- err - return + return err } _, err = io.Copy(io.Discard, res.Body) res.Body.Close() - errc <- err + return err }() - select { - case err := <-errc: - if err == nil { - t.Errorf("expected an error from Get request") - } - case <-time.After(10 * time.Second): - t.Fatal("timeout waiting for Get error") + if err == nil { + t.Errorf("expected an error copying body from Get request") } + if err := <-afterTimeoutErrc; err == nil { t.Error("expected write error after timeout") } @@ -1133,21 +1118,10 @@ func testTCPConnectionCloses(t *testing.T, req string, h Handler) { t.Fatal("ReadResponse error:", err) } - didReadAll := make(chan bool, 1) - go func() { - select { - case <-time.After(5 * time.Second): - t.Error("body not closed after 5s") - return - case <-didReadAll: - } - }() - _, err = io.ReadAll(r) if err != nil { t.Fatal("read error:", err) } - didReadAll <- true if !res.Close { t.Errorf("Response.Close = false; want true") @@ -1323,7 +1297,6 @@ func testServerAllowsBlockingRemoteAddr(t *testing.T, mode testMode) { }).ts c := ts.Client() - c.Timeout = time.Second // Force separate connection for each: c.Transport.(*Transport).DisableKeepAlives = true @@ -1526,8 +1499,6 @@ func TestServeTLS(t *testing.T) { case err := <-errc: t.Fatalf("ServeTLS: %v", err) case <-serving: - case <-time.After(5 * time.Second): - t.Fatal("timeout") } c, err := tls.Dial("tcp", ln.Addr().String(), &tls.Config{ @@ -2820,18 +2791,7 @@ func testHandlerPanic(t *testing.T, withHijack bool, mode testMode, wrapper func return } - var delay time.Duration - if deadline, ok := t.Deadline(); ok { - delay = time.Until(deadline) - } else { - delay = 5 * time.Second - } - select { - case <-done: - return - case <-time.After(delay): - t.Fatal("expected server handler to log an error") - } + <-done } type terrorWriter struct{ t *testing.T } @@ -2871,11 +2831,7 @@ func testServerWriteHijackZeroBytes(t *testing.T, mode testMode) { t.Fatal(err) } res.Body.Close() - select { - case <-done: - case <-time.After(5 * time.Second): - t.Fatal("timeout") - } + <-done } func TestServerNoDate(t *testing.T) { @@ -3238,8 +3194,6 @@ For: diec <- true case <-sawClose: break For - case <-time.After(5 * time.Second): - t.Fatal("timeout") } } ts.Close() @@ -3295,9 +3249,6 @@ func testCloseNotifierPipelined(t *testing.T, mode testMode) { if closes > 1 { return } - case <-time.After(5 * time.Second): - ts.CloseClientConnections() - t.Fatal("timeout") } } } @@ -3408,12 +3359,8 @@ func testHijackBeforeRequestBodyRead(t *testing.T, mode testMode) { return } bodyOkay <- true - select { - case <-gone: - gotCloseNotify <- true - case <-time.After(5 * time.Second): - gotCloseNotify <- false - } + <-gone + gotCloseNotify <- true })).ts conn, err := net.Dial("tcp", ts.Listener.Addr().String()) @@ -3429,9 +3376,7 @@ func testHijackBeforeRequestBodyRead(t *testing.T, mode testMode) { return } conn.Close() - if !<-gotCloseNotify { - t.Error("timeout waiting for CloseNotify") - } + <-gotCloseNotify } func TestOptions(t *testing.T) { run(t, testOptions, []testMode{http1Mode}) } @@ -3507,13 +3452,8 @@ func testOptionsHandler(t *testing.T, mode testMode) { t.Fatal(err) } - select { - case got := <-rc: - if got.Method != "OPTIONS" || got.RequestURI != "*" { - t.Errorf("Expected OPTIONS * request, got %v", got) - } - case <-time.After(5 * time.Second): - t.Error("timeout") + if got := <-rc; got.Method != "OPTIONS" || got.RequestURI != "*" { + t.Errorf("Expected OPTIONS * request, got %v", got) } } @@ -3985,8 +3925,6 @@ func testTransportAndServerSharedBodyRace(t *testing.T, mode testMode) { } <-unblockBackend })) - var quitTimer *time.Timer - defer func() { quitTimer.Stop() }() defer backend.close() backendRespc := make(chan *Response, 1) @@ -4019,20 +3957,6 @@ func testTransportAndServerSharedBodyRace(t *testing.T, mode testMode) { rw.Write([]byte("OK")) })) defer proxy.close() - defer func() { - // Before we shut down our two httptest.Servers, start a timer. - // We choose 7 seconds because httptest.Server starts logging - // warnings to stderr at 5 seconds. If we don't disarm this bomb - // in 7 seconds (after the two httptest.Server.Close calls above), - // then we explode with stacks. - quitTimer = time.AfterFunc(7*time.Second, func() { - debug.SetTraceback("ALL") - stacks := make([]byte, 1<<20) - stacks = stacks[:runtime.Stack(stacks, true)] - fmt.Fprintf(os.Stderr, "%s", stacks) - log.Fatalf("Timeout.") - }) - }() defer close(unblockBackend) req, _ := NewRequest("POST", proxy.ts.URL, io.LimitReader(neverEnding('a'), bodySize)) @@ -4098,8 +4022,6 @@ func testRequestBodyCloseDoesntBlock(t *testing.T, mode testMode) { } case err := <-errCh: t.Error(err) - case <-time.After(5 * time.Second): - t.Error("timeout") } } @@ -4176,22 +4098,7 @@ func testServerConnState(t *testing.T, mode testMode) { doRequests() - stateDelay := 5 * time.Second - if deadline, ok := t.Deadline(); ok { - // Allow an arbitrarily long delay. - // This test was observed to be flaky on the darwin-arm64-corellium builder, - // so we're increasing the deadline to see if it starts passing. - // See https://golang.org/issue/37322. - const arbitraryCleanupMargin = 1 * time.Second - stateDelay = time.Until(deadline) - arbitraryCleanupMargin - } - timer := time.NewTimer(stateDelay) - select { - case <-timer.C: - t.Errorf("Timed out after %v waiting for connection to change state.", stateDelay) - case <-complete: - timer.Stop() - } + <-complete sl := <-activeLog if !reflect.DeepEqual(sl.got, sl.want) { t.Errorf("Request(s) produced unexpected state sequence.\nGot: %v\nWant: %v", sl.got, sl.want) @@ -4480,8 +4387,6 @@ func testServerKeepAliveAfterWriteError(t *testing.T, mode testMode) { } }() - timeout := time.NewTimer(numReq * 2 * time.Second) // 4x overkill - defer timeout.Stop() addrSeen := map[string]bool{} numOkay := 0 for { @@ -4501,8 +4406,6 @@ func testServerKeepAliveAfterWriteError(t *testing.T, mode testMode) { if err == nil { numOkay++ } - case <-timeout.C: - t.Fatal("timeout waiting for requests to complete") } } } @@ -4936,15 +4839,11 @@ func testServerContext_LocalAddrContextKey(t *testing.T, mode testMode) { } host := cst.ts.Listener.Addr().String() - select { - case got := <-ch: - if addr, ok := got.(net.Addr); !ok { - t.Errorf("local addr value = %T; want net.Addr", got) - } else if fmt.Sprint(addr) != host { - t.Errorf("local addr = %v; want %v", addr, host) - } - case <-time.After(5 * time.Second): - t.Error("timed out") + got := <-ch + if addr, ok := got.(net.Addr); !ok { + t.Errorf("local addr value = %T; want net.Addr", got) + } else if fmt.Sprint(addr) != host { + t.Errorf("local addr = %v; want %v", addr, host) } } @@ -5151,8 +5050,9 @@ func BenchmarkClient(b *testing.B) { } // Start server process. - cmd := exec.Command(os.Args[0], "-test.run=XXXX", "-test.bench=BenchmarkClient$") - cmd.Env = append(os.Environ(), "TEST_BENCH_SERVER=yes") + ctx, cancel := context.WithCancel(context.Background()) + cmd := testenv.CommandContext(b, ctx, os.Args[0], "-test.run=XXXX", "-test.bench=BenchmarkClient$") + cmd.Env = append(cmd.Environ(), "TEST_BENCH_SERVER=yes") cmd.Stderr = os.Stderr stdout, err := cmd.StdoutPipe() if err != nil { @@ -5161,35 +5061,28 @@ func BenchmarkClient(b *testing.B) { if err := cmd.Start(); err != nil { b.Fatalf("subprocess failed to start: %v", err) } - defer cmd.Process.Kill() + + done := make(chan error, 1) + go func() { + done <- cmd.Wait() + close(done) + }() + defer func() { + cancel() + <-done + }() // Wait for the server in the child process to respond and tell us // its listening address, once it's started listening: - timer := time.AfterFunc(10*time.Second, func() { - cmd.Process.Kill() - }) - defer timer.Stop() bs := bufio.NewScanner(stdout) if !bs.Scan() { b.Fatalf("failed to read listening URL from child: %v", bs.Err()) } url := "http://" + strings.TrimSpace(bs.Text()) + "/" - timer.Stop() if _, err := getNoBody(url); err != nil { b.Fatalf("initial probe of child process failed: %v", err) } - done := make(chan error) - stop := make(chan struct{}) - defer close(stop) - go func() { - select { - case <-stop: - return - case done <- cmd.Wait(): - } - }() - // Do b.N requests to the server. b.StartTimer() for i := 0; i < b.N; i++ { @@ -5210,13 +5103,8 @@ func BenchmarkClient(b *testing.B) { // Instruct server process to stop. getNoBody(url + "?stop=yes") - select { - case err := <-done: - if err != nil { - b.Fatalf("subprocess failed: %v", err) - } - case <-time.After(5 * time.Second): - b.Fatalf("subprocess did not stop") + if err := <-done; err != nil { + b.Fatalf("subprocess failed: %v", err) } } @@ -5426,8 +5314,6 @@ func benchmarkCloseNotifier(b *testing.B, mode testMode) { <-rw.(CloseNotifier).CloseNotify() sawClose <- true })).ts - tot := time.NewTimer(5 * time.Second) - defer tot.Stop() b.StartTimer() for i := 0; i < b.N; i++ { conn, err := net.Dial("tcp", ts.Listener.Addr().String()) @@ -5439,12 +5325,7 @@ func benchmarkCloseNotifier(b *testing.B, mode testMode) { b.Fatal(err) } conn.Close() - tot.Reset(5 * time.Second) - select { - case <-sawClose: - case <-tot.C: - b.Fatal("timeout") - } + <-sawClose } b.StopTimer() } @@ -5603,11 +5484,7 @@ func testServerShutdown(t *testing.T, mode testMode) { if err := <-shutdownRes; err != nil { t.Fatalf("Shutdown: %v", err) } - select { - case <-gotOnShutdown: - case <-time.After(5 * time.Second): - t.Errorf("onShutdown callback not called, RegisterOnShutdown broken?") - } + <-gotOnShutdown // Will hang if RegisterOnShutdown is broken. if states := <-statesRes; states[StateActive] != 1 { t.Errorf("connection in wrong state, %v", states) @@ -5678,13 +5555,8 @@ func testServerShutdownStateNew(t *testing.T, mode testMode) { // Wait for c.Read to unblock; should be already done at this point, // or within a few milliseconds. - select { - case err := <-readRes: - if err == nil { - t.Error("expected error from Read") - } - case <-time.After(2 * time.Second): - t.Errorf("timeout waiting for Read to unblock") + if err := <-readRes; err == nil { + t.Error("expected error from Read") } } @@ -5954,11 +5826,7 @@ func testServerHijackGetsBackgroundByte(t *testing.T, mode testMode) { if err := cn.(*net.TCPConn).CloseWrite(); err != nil { t.Fatal(err) } - select { - case <-done: - case <-time.After(2 * time.Second): - t.Error("timeout") - } + <-done } // Like TestServerHijackGetsBackgroundByte above but sending a From b74ca9b5b7f235a254b64cafe0616e9c4532b876 Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Mon, 13 Mar 2023 17:36:36 -0400 Subject: [PATCH 26/86] net/http: avoid making a request to a closed server in TestServerShutdown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As soon as the test server closes its listener, its port may be reused for another test server. On some platforms port reuse takes a long time, because they cycle through the available ports before reusing an old one. However, other platforms reuse ports much more aggressively. net/http shouldn't know or care which kind of platform it is on — dialing wild connections is risky and can interfere with other tests no matter what platform we do it on. Instead of making the second request after the server has completely shut down, we can start (and finish!) the entire request while we are certain that the listener has been closed but the port is still open serving an existing request. If everything synchronizes as we expect, that should guarantee that the second request fails. Fixes #56421. Change-Id: I56add243bb9f76ee04ead8f643118f9448fd1280 Reviewed-on: https://go-review.googlesource.com/c/go/+/476036 Auto-Submit: Bryan Mills Run-TryBot: Bryan Mills TryBot-Result: Gopher Robot Reviewed-by: Damien Neil --- serve_test.go | 77 +++++++++++++++++++++++++++++++++------------------ 1 file changed, 50 insertions(+), 27 deletions(-) diff --git a/serve_test.go b/serve_test.go index 8ee8107c..23aab290 100644 --- a/serve_test.go +++ b/serve_test.go @@ -5454,32 +5454,54 @@ func testServerSetKeepAlivesEnabledClosesConns(t *testing.T, mode testMode) { func TestServerShutdown(t *testing.T) { run(t, testServerShutdown) } func testServerShutdown(t *testing.T, mode testMode) { - var doShutdown func() // set later - var doStateCount func() - var shutdownRes = make(chan error, 1) - var statesRes = make(chan map[ConnState]int, 1) - var gotOnShutdown = make(chan struct{}, 1) + var cst *clientServerTest + + var once sync.Once + statesRes := make(chan map[ConnState]int, 1) + shutdownRes := make(chan error, 1) + gotOnShutdown := make(chan struct{}) handler := HandlerFunc(func(w ResponseWriter, r *Request) { - doStateCount() - go doShutdown() - // Shutdown is graceful, so it should not interrupt - // this in-flight response. Add a tiny sleep here to - // increase the odds of a failure if shutdown has - // bugs. - time.Sleep(20 * time.Millisecond) + first := false + once.Do(func() { + statesRes <- cst.ts.Config.ExportAllConnsByState() + go func() { + shutdownRes <- cst.ts.Config.Shutdown(context.Background()) + }() + first = true + }) + + if first { + // Shutdown is graceful, so it should not interrupt this in-flight response + // but should reject new requests. (Since this request is still in flight, + // the server's port should not be reused for another server yet.) + <-gotOnShutdown + // TODO(#59038): The HTTP/2 server empirically does not always reject new + // requests. As a workaround, loop until we see a failure. + for !t.Failed() { + res, err := cst.c.Get(cst.ts.URL) + if err != nil { + break + } + out, _ := io.ReadAll(res.Body) + res.Body.Close() + if mode == http2Mode { + t.Logf("%v: unexpected success (%q). Listener should be closed before OnShutdown is called.", cst.ts.URL, out) + t.Logf("Retrying to work around https://go.dev/issue/59038.") + continue + } + t.Errorf("%v: unexpected success (%q). Listener should be closed before OnShutdown is called.", cst.ts.URL, out) + } + } + io.WriteString(w, r.RemoteAddr) }) - cst := newClientServerTest(t, mode, handler, func(srv *httptest.Server) { - srv.Config.RegisterOnShutdown(func() { gotOnShutdown <- struct{}{} }) + + cst = newClientServerTest(t, mode, handler, func(srv *httptest.Server) { + srv.Config.RegisterOnShutdown(func() { close(gotOnShutdown) }) }) - doShutdown = func() { - shutdownRes <- cst.ts.Config.Shutdown(context.Background()) - } - doStateCount = func() { - statesRes <- cst.ts.Config.ExportAllConnsByState() - } - get(t, cst.c, cst.ts.URL) // calls t.Fail on failure + out := get(t, cst.c, cst.ts.URL) // calls t.Fail on failure + t.Logf("%v: %q", cst.ts.URL, out) if err := <-shutdownRes; err != nil { t.Fatalf("Shutdown: %v", err) @@ -5489,12 +5511,6 @@ func testServerShutdown(t *testing.T, mode testMode) { if states := <-statesRes; states[StateActive] != 1 { t.Errorf("connection in wrong state, %v", states) } - - res, err := cst.c.Get(cst.ts.URL) - if err == nil { - res.Body.Close() - t.Fatal("second request should fail. server should be shut down") - } } func TestServerShutdownStateNew(t *testing.T) { run(t, testServerShutdownStateNew) } @@ -5502,6 +5518,9 @@ func testServerShutdownStateNew(t *testing.T, mode testMode) { if testing.Short() { t.Skip("test takes 5-6 seconds; skipping in short mode") } + // The run helper runs the test in parallel only in short mode by default. + // Since this test has a very long latency, always run it in parallel. + t.Parallel() var connAccepted sync.WaitGroup ts := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { @@ -5538,7 +5557,11 @@ func testServerShutdownStateNew(t *testing.T, mode testMode) { readRes <- err }() + // TODO(#59037): This timeout is hard-coded in closeIdleConnections. + // It is undocumented, and some users may find it surprising. + // Either document it, or switch to a less surprising behavior. const expectTimeout = 5 * time.Second + t0 := time.Now() select { case got := <-shutdownRes: From 51163f3d1b90981cd48e6d38667be0cfe05530c7 Mon Sep 17 00:00:00 2001 From: Shang Ding Date: Wed, 15 Feb 2023 21:04:33 -0600 Subject: [PATCH 27/86] net/http/httputil: use response controller in reverse proxy Previously, the reverse proxy is unable to detect the support for hijack or flush if those things are residing in the response writer in a wrapped manner. The reverse proxy now makes use of the new http response controller as the means to discover the underlying flusher and hijacker associated with the response writer, allowing wrapped flusher and hijacker become discoverable. Change-Id: I53acbb12315c3897be068e8c00598ef42fc74649 Reviewed-on: https://go-review.googlesource.com/c/go/+/468755 Run-TryBot: Damien Neil Auto-Submit: Damien Neil TryBot-Result: Gopher Robot Reviewed-by: Damien Neil Reviewed-by: Cherry Mui --- httputil/reverseproxy.go | 61 ++++++++++++++++------------------- httputil/reverseproxy_test.go | 56 ++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 33 deletions(-) diff --git a/httputil/reverseproxy.go b/httputil/reverseproxy.go index eece455a..2a76b0b8 100644 --- a/httputil/reverseproxy.go +++ b/httputil/reverseproxy.go @@ -524,9 +524,7 @@ func (p *ReverseProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) { // Force chunking if we saw a response trailer. // This prevents net/http from calculating the length for short // bodies and adding a Content-Length. - if fl, ok := rw.(http.Flusher); ok { - fl.Flush() - } + http.NewResponseController(rw).Flush() } if len(res.Trailer) == announcedTrailers { @@ -601,21 +599,22 @@ func (p *ReverseProxy) flushInterval(res *http.Response) time.Duration { return p.FlushInterval } -func (p *ReverseProxy) copyResponse(dst io.Writer, src io.Reader, flushInterval time.Duration) error { +func (p *ReverseProxy) copyResponse(dst http.ResponseWriter, src io.Reader, flushInterval time.Duration) error { + var w io.Writer = dst + if flushInterval != 0 { - if wf, ok := dst.(writeFlusher); ok { - mlw := &maxLatencyWriter{ - dst: wf, - latency: flushInterval, - } - defer mlw.stop() + mlw := &maxLatencyWriter{ + dst: dst, + flush: http.NewResponseController(dst).Flush, + latency: flushInterval, + } + defer mlw.stop() - // set up initial timer so headers get flushed even if body writes are delayed - mlw.flushPending = true - mlw.t = time.AfterFunc(flushInterval, mlw.delayedFlush) + // set up initial timer so headers get flushed even if body writes are delayed + mlw.flushPending = true + mlw.t = time.AfterFunc(flushInterval, mlw.delayedFlush) - dst = mlw - } + w = mlw } var buf []byte @@ -623,7 +622,7 @@ func (p *ReverseProxy) copyResponse(dst io.Writer, src io.Reader, flushInterval buf = p.BufferPool.Get() defer p.BufferPool.Put(buf) } - _, err := p.copyBuffer(dst, src, buf) + _, err := p.copyBuffer(w, src, buf) return err } @@ -668,13 +667,9 @@ func (p *ReverseProxy) logf(format string, args ...any) { } } -type writeFlusher interface { - io.Writer - http.Flusher -} - type maxLatencyWriter struct { - dst writeFlusher + dst io.Writer + flush func() error latency time.Duration // non-zero; negative means to flush immediately mu sync.Mutex // protects t, flushPending, and dst.Flush @@ -687,7 +682,7 @@ func (m *maxLatencyWriter) Write(p []byte) (n int, err error) { defer m.mu.Unlock() n, err = m.dst.Write(p) if m.latency < 0 { - m.dst.Flush() + m.flush() return } if m.flushPending { @@ -708,7 +703,7 @@ func (m *maxLatencyWriter) delayedFlush() { if !m.flushPending { // if stop was called but AfterFunc already started this goroutine return } - m.dst.Flush() + m.flush() m.flushPending = false } @@ -739,17 +734,19 @@ func (p *ReverseProxy) handleUpgradeResponse(rw http.ResponseWriter, req *http.R return } - hj, ok := rw.(http.Hijacker) - if !ok { - p.getErrorHandler()(rw, req, fmt.Errorf("can't switch protocols using non-Hijacker ResponseWriter type %T", rw)) - return - } backConn, ok := res.Body.(io.ReadWriteCloser) if !ok { p.getErrorHandler()(rw, req, fmt.Errorf("internal error: 101 switching protocols response with non-writable body")) return } + rc := http.NewResponseController(rw) + conn, brw, hijackErr := rc.Hijack() + if errors.Is(hijackErr, http.ErrNotSupported) { + p.getErrorHandler()(rw, req, fmt.Errorf("can't switch protocols using non-Hijacker ResponseWriter type %T", rw)) + return + } + backConnCloseCh := make(chan bool) go func() { // Ensure that the cancellation of a request closes the backend. @@ -760,12 +757,10 @@ func (p *ReverseProxy) handleUpgradeResponse(rw http.ResponseWriter, req *http.R } backConn.Close() }() - defer close(backConnCloseCh) - conn, brw, err := hj.Hijack() - if err != nil { - p.getErrorHandler()(rw, req, fmt.Errorf("Hijack failed on protocol switch: %v", err)) + if hijackErr != nil { + p.getErrorHandler()(rw, req, fmt.Errorf("Hijack failed on protocol switch: %v", hijackErr)) return } defer conn.Close() diff --git a/httputil/reverseproxy_test.go b/httputil/reverseproxy_test.go index d5b0fb42..dd3330b6 100644 --- a/httputil/reverseproxy_test.go +++ b/httputil/reverseproxy_test.go @@ -478,6 +478,62 @@ func TestReverseProxyFlushInterval(t *testing.T) { } } +type mockFlusher struct { + http.ResponseWriter + flushed bool +} + +func (m *mockFlusher) Flush() { + m.flushed = true +} + +type wrappedRW struct { + http.ResponseWriter +} + +func (w *wrappedRW) Unwrap() http.ResponseWriter { + return w.ResponseWriter +} + +func TestReverseProxyResponseControllerFlushInterval(t *testing.T) { + const expected = "hi" + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(expected)) + })) + defer backend.Close() + + backendURL, err := url.Parse(backend.URL) + if err != nil { + t.Fatal(err) + } + + mf := &mockFlusher{} + proxyHandler := NewSingleHostReverseProxy(backendURL) + proxyHandler.FlushInterval = -1 // flush immediately + proxyWithMiddleware := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mf.ResponseWriter = w + w = &wrappedRW{mf} + proxyHandler.ServeHTTP(w, r) + }) + + frontend := httptest.NewServer(proxyWithMiddleware) + defer frontend.Close() + + req, _ := http.NewRequest("GET", frontend.URL, nil) + req.Close = true + res, err := frontend.Client().Do(req) + if err != nil { + t.Fatalf("Get: %v", err) + } + defer res.Body.Close() + if bodyBytes, _ := io.ReadAll(res.Body); string(bodyBytes) != expected { + t.Errorf("got body %q; expected %q", bodyBytes, expected) + } + if !mf.flushed { + t.Errorf("response writer was not flushed") + } +} + func TestReverseProxyFlushIntervalHeaders(t *testing.T) { const expected = "hi" stopCh := make(chan struct{}) From 7b654ba6e6c609f06ed6e4acc702b2475da1d13a Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Fri, 17 Mar 2023 12:42:02 -0400 Subject: [PATCH 28/86] net/http: eliminate more arbitrary timeouts in tests Change-Id: I5b3158ecd0eb20dc433a53a2b03eb4551cbb3f7d Reviewed-on: https://go-review.googlesource.com/c/go/+/477196 Auto-Submit: Bryan Mills TryBot-Result: Gopher Robot Run-TryBot: Bryan Mills Reviewed-by: Damien Neil --- clientserver_test.go | 22 +- main_test.go | 34 +- serve_test.go | 26 +- transport_test.go | 820 +++++++++++++++++++++---------------------- 4 files changed, 449 insertions(+), 453 deletions(-) diff --git a/clientserver_test.go b/clientserver_test.go index e49bed11..58321532 100644 --- a/clientserver_test.go +++ b/clientserver_test.go @@ -1283,24 +1283,28 @@ func testInterruptWithPanic(t *testing.T, mode testMode, panicValue any) { } wantStackLogged := panicValue != nil && panicValue != ErrAbortHandler - if err := waitErrCondition(5*time.Second, 10*time.Millisecond, func() error { + waitCondition(t, 10*time.Millisecond, func(d time.Duration) bool { gotLog := logOutput() if !wantStackLogged { if gotLog == "" { - return nil + return true } - return fmt.Errorf("want no log output; got: %s", gotLog) + t.Fatalf("want no log output; got: %s", gotLog) } if gotLog == "" { - return fmt.Errorf("wanted a stack trace logged; got nothing") + if d > 0 { + t.Logf("wanted a stack trace logged; got nothing after %v", d) + } + return false } if !strings.Contains(gotLog, "created by ") && strings.Count(gotLog, "\n") < 6 { - return fmt.Errorf("output doesn't look like a panic stack trace. Got: %s", gotLog) + if d > 0 { + t.Logf("output doesn't look like a panic stack trace after %v. Got: %s", d, gotLog) + } + return false } - return nil - }); err != nil { - t.Fatal(err) - } + return true + }) } type lockedBytesBuffer struct { diff --git a/main_test.go b/main_test.go index 27872b4e..1b2fa215 100644 --- a/main_test.go +++ b/main_test.go @@ -140,29 +140,15 @@ func afterTest(t testing.TB) { t.Errorf("Test appears to have leaked %s:\n%s", bad, stacks) } -// waitCondition reports whether fn eventually returned true, -// checking immediately and then every checkEvery amount, -// until waitFor has elapsed, at which point it returns false. -func waitCondition(waitFor, checkEvery time.Duration, fn func() bool) bool { - deadline := time.Now().Add(waitFor) - for time.Now().Before(deadline) { - if fn() { - return true - } - time.Sleep(checkEvery) - } - return false -} - -// waitErrCondition is like waitCondition but with errors instead of bools. -func waitErrCondition(waitFor, checkEvery time.Duration, fn func() error) error { - deadline := time.Now().Add(waitFor) - var err error - for time.Now().Before(deadline) { - if err = fn(); err == nil { - return nil - } - time.Sleep(checkEvery) +// waitCondition waits for fn to return true, +// checking immediately and then at exponentially increasing intervals. +func waitCondition(t testing.TB, delay time.Duration, fn func(time.Duration) bool) { + t.Helper() + start := time.Now() + var since time.Duration + for !fn(since) { + time.Sleep(delay) + delay = 2*delay - (delay / 2) // 1.5x, rounded up + since = time.Since(start) } - return err } diff --git a/serve_test.go b/serve_test.go index 23aab290..eb4660f8 100644 --- a/serve_test.go +++ b/serve_test.go @@ -5439,12 +5439,16 @@ func testServerSetKeepAlivesEnabledClosesConns(t *testing.T, mode testMode) { ts.Config.SetKeepAlivesEnabled(false) var idle1 int - if !waitCondition(2*time.Second, 10*time.Millisecond, func() bool { + waitCondition(t, 10*time.Millisecond, func(d time.Duration) bool { idle1 = tr.IdleConnCountForTesting("http", addr) - return idle1 == 0 - }) { - t.Fatalf("idle count after SetKeepAlivesEnabled called = %v; want 0", idle1) - } + if idle1 != 0 { + if d > 0 { + t.Logf("idle count %v after SetKeepAlivesEnabled called = %v; waiting for 0", d, idle1) + } + return false + } + return true + }) a3 := get() if a3 == a2 { @@ -5604,9 +5608,15 @@ func testServerKeepAlivesEnabled(t *testing.T, mode testMode) { srv := cst.ts.Config srv.SetKeepAlivesEnabled(false) for try := 0; try < 2; try++ { - if !waitCondition(2*time.Second, 10*time.Millisecond, srv.ExportAllConnsIdle) { - t.Fatalf("request %v: test server has active conns", try) - } + waitCondition(t, 10*time.Millisecond, func(d time.Duration) bool { + if !srv.ExportAllConnsIdle() { + if d > 0 { + t.Logf("test server still has active conns after %v", d) + } + return false + } + return true + }) conns := 0 var info httptrace.GotConnInfo ctx := httptrace.WithClientTrace(context.Background(), &httptrace.ClientTrace{ diff --git a/transport_test.go b/transport_test.go index cae98767..1abb0aab 100644 --- a/transport_test.go +++ b/transport_test.go @@ -125,6 +125,8 @@ func (tcs *testConnSet) check(t *testing.T) { continue } if i != 0 { + // TODO(bcmills): What is the Sleep here doing, and why is this + // Unlock/Sleep/Lock cycle needed at all? tcs.mu.Unlock() time.Sleep(50 * time.Millisecond) tcs.mu.Lock() @@ -763,12 +765,16 @@ func testTransportRemovesDeadIdleConnections(t *testing.T, mode testMode) { ts.CloseClientConnections() var keys2 []string - if !waitCondition(3*time.Second, 50*time.Millisecond, func() bool { + waitCondition(t, 10*time.Millisecond, func(d time.Duration) bool { keys2 = tr.IdleConnKeysForTesting() - return len(keys2) == 0 - }) { - t.Fatalf("Transport didn't notice idle connection's death.\nbefore: %q\n after: %q\n", keys1, keys2) - } + if len(keys2) != 0 { + if d > 0 { + t.Logf("Transport hasn't noticed idle connection's death in %v.\nbefore: %q\n after: %q\n", d, keys1, keys2) + } + return false + } + return true + }) second := doReq("second") if first == second { @@ -863,7 +869,8 @@ func testStressSurpriseServerCloses(t *testing.T, mode testMode) { numClients = 20 reqsPerClient = 25 ) - activityc := make(chan bool) + var wg sync.WaitGroup + wg.Add(numClients * reqsPerClient) for i := 0; i < numClients; i++ { go func() { for i := 0; i < reqsPerClient; i++ { @@ -877,22 +884,13 @@ func testStressSurpriseServerCloses(t *testing.T, mode testMode) { // where we won the race. res.Body.Close() } - if !<-activityc { // Receives false when close(activityc) is executed - return - } + wg.Done() } }() } // Make sure all the request come back, one way or another. - for i := 0; i < numClients*reqsPerClient; i++ { - select { - case activityc <- true: - case <-time.After(5 * time.Second): - close(activityc) - t.Fatalf("presumed deadlock; no HTTP client activity seen in awhile") - } - } + wg.Wait() } // TestTransportHeadResponses verifies that we deal with Content-Lengths @@ -1324,12 +1322,7 @@ func testSOCKS5Proxy(t *testing.T, mode testMode) { if r.Header.Get(sentinelHeader) != sentinelValue { t.Errorf("Failed to retrieve sentinel value") } - var got string - select { - case got = <-ch: - case <-time.After(5 * time.Second): - t.Fatal("timeout connecting to socks5 proxy") - } + got := <-ch ts.Close() tsu, err := url.Parse(ts.URL) if err != nil { @@ -1420,12 +1413,7 @@ func TestTransportProxy(t *testing.T) { if _, err := c.Head(ts.URL); err != nil { t.Error(err) } - var got *Request - select { - case got = <-proxyCh: - case <-time.After(5 * time.Second): - t.Fatal("timeout connecting to http proxy") - } + got := <-proxyCh c.Transport.(*Transport).CloseIdleConnections() ts.Close() proxy.Close() @@ -2329,67 +2317,81 @@ func testTransportResponseHeaderTimeout(t *testing.T, mode testMode) { if testing.Short() { t.Skip("skipping timeout test in -short mode") } - inHandler := make(chan bool, 1) - mux := NewServeMux() - mux.HandleFunc("/fast", func(w ResponseWriter, r *Request) { - inHandler <- true - }) - mux.HandleFunc("/slow", func(w ResponseWriter, r *Request) { - inHandler <- true - time.Sleep(2 * time.Second) - }) - ts := newClientServerTest(t, mode, mux).ts - - c := ts.Client() - c.Transport.(*Transport).ResponseHeaderTimeout = 500 * time.Millisecond - tests := []struct { - path string - want int - wantErr string - }{ - {path: "/fast", want: 200}, - {path: "/slow", wantErr: "timeout awaiting response headers"}, - {path: "/fast", want: 200}, - } - for i, tt := range tests { - req, _ := NewRequest("GET", ts.URL+tt.path, nil) - req = req.WithT(t) - res, err := c.Do(req) - select { - case <-inHandler: - case <-time.After(5 * time.Second): - t.Errorf("never entered handler for test index %d, %s", i, tt.path) - continue - } - if err != nil { - uerr, ok := err.(*url.Error) - if !ok { - t.Errorf("error is not an url.Error; got: %#v", err) - continue - } - nerr, ok := uerr.Err.(net.Error) - if !ok { - t.Errorf("error does not satisfy net.Error interface; got: %#v", err) + timeout := 2 * time.Millisecond + retry := true + for retry && !t.Failed() { + var srvWG sync.WaitGroup + inHandler := make(chan bool, 1) + mux := NewServeMux() + mux.HandleFunc("/fast", func(w ResponseWriter, r *Request) { + inHandler <- true + srvWG.Done() + }) + mux.HandleFunc("/slow", func(w ResponseWriter, r *Request) { + inHandler <- true + <-r.Context().Done() + srvWG.Done() + }) + ts := newClientServerTest(t, mode, mux).ts + + c := ts.Client() + c.Transport.(*Transport).ResponseHeaderTimeout = timeout + + retry = false + srvWG.Add(3) + tests := []struct { + path string + wantTimeout bool + }{ + {path: "/fast"}, + {path: "/slow", wantTimeout: true}, + {path: "/fast"}, + } + for i, tt := range tests { + req, _ := NewRequest("GET", ts.URL+tt.path, nil) + req = req.WithT(t) + res, err := c.Do(req) + <-inHandler + if err != nil { + uerr, ok := err.(*url.Error) + if !ok { + t.Errorf("error is not an url.Error; got: %#v", err) + continue + } + nerr, ok := uerr.Err.(net.Error) + if !ok { + t.Errorf("error does not satisfy net.Error interface; got: %#v", err) + continue + } + if !nerr.Timeout() { + t.Errorf("want timeout error; got: %q", nerr) + continue + } + if !tt.wantTimeout { + if !retry { + // The timeout may be set too short. Retry with a longer one. + t.Logf("unexpected timout for path %q after %v; retrying with longer timeout", tt.path, timeout) + timeout *= 2 + retry = true + } + } + if !strings.Contains(err.Error(), "timeout awaiting response headers") { + t.Errorf("%d. unexpected error: %v", i, err) + } continue } - if !nerr.Timeout() { - t.Errorf("want timeout error; got: %q", nerr) + if tt.wantTimeout { + t.Errorf(`no error for path %q; expected "timeout awaiting response headers"`, tt.path) continue } - if strings.Contains(err.Error(), tt.wantErr) { - continue + if res.StatusCode != 200 { + t.Errorf("%d for path %q status = %d; want 200", i, tt.path, res.StatusCode) } - t.Errorf("%d. unexpected error: %v", i, err) - continue - } - if tt.wantErr != "" { - t.Errorf("%d. no error. expected error: %v", i, tt.wantErr) - continue - } - if res.StatusCode != tt.want { - t.Errorf("%d for path %q status = %d; want %d", i, tt.path, res.StatusCode, tt.want) } + + srvWG.Wait() + ts.Close() } } @@ -2400,9 +2402,11 @@ func testTransportCancelRequest(t *testing.T, mode testMode) { if testing.Short() { t.Skip("skipping test in -short mode") } + + const msg = "Hello" unblockc := make(chan bool) ts := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { - fmt.Fprintf(w, "Hello") + io.WriteString(w, msg) w.(Flusher).Flush() // send headers and some body <-unblockc })).ts @@ -2416,35 +2420,32 @@ func testTransportCancelRequest(t *testing.T, mode testMode) { if err != nil { t.Fatal(err) } - go func() { - time.Sleep(1 * time.Second) - tr.CancelRequest(req) - }() - t0 := time.Now() - body, err := io.ReadAll(res.Body) - d := time.Since(t0) + body := make([]byte, len(msg)) + n, _ := io.ReadFull(res.Body, body) + if n != len(body) || !bytes.Equal(body, []byte(msg)) { + t.Errorf("Body = %q; want %q", body[:n], msg) + } + tr.CancelRequest(req) + tail, err := io.ReadAll(res.Body) + res.Body.Close() if err != ExportErrRequestCanceled { t.Errorf("Body.Read error = %v; want errRequestCanceled", err) + } else if len(tail) > 0 { + t.Errorf("Spurious bytes from Body.Read: %q", tail) } - if string(body) != "Hello" { - t.Errorf("Body = %q; want Hello", body) - } - if d < 500*time.Millisecond { - t.Errorf("expected ~1 second delay; got %v", d) - } + // Verify no outstanding requests after readLoop/writeLoop // goroutines shut down. - for tries := 5; tries > 0; tries-- { + waitCondition(t, 10*time.Millisecond, func(d time.Duration) bool { n := tr.NumPendingRequestsForTesting() - if n == 0 { - break - } - time.Sleep(100 * time.Millisecond) - if tries == 1 { - t.Errorf("pending requests = %d; want 0", n) + if n > 0 { + if d > 0 { + t.Logf("pending requests = %d after %v (want 0)", n, d) + } } - } + return true + }) } func testTransportCancelRequestInDo(t *testing.T, mode testMode, body io.Reader) { @@ -2466,18 +2467,20 @@ func testTransportCancelRequestInDo(t *testing.T, mode testMode, body io.Reader) defer close(donec) c.Do(req) }() - start := time.Now() - timeout := 10 * time.Second - for time.Since(start) < timeout { - time.Sleep(100 * time.Millisecond) + + unblockc <- true + waitCondition(t, 10*time.Millisecond, func(d time.Duration) bool { tr.CancelRequest(req) select { case <-donec: - return + return true default: + if d > 0 { + t.Logf("Do of canceled request has not returned after %v", d) + } + return false } - } - t.Errorf("Do of canceled request has not returned after %v", timeout) + }) } func TestTransportCancelRequestInDo(t *testing.T) { @@ -2523,22 +2526,22 @@ func TestTransportCancelRequestInDial(t *testing.T) { gotres <- true }() - select { - case inDial <- true: - case <-time.After(5 * time.Second): - close(inDial) - t.Fatal("timeout; never saw blocking dial") - } + inDial <- true eventLog.Printf("canceling") tr.CancelRequest(req) tr.CancelRequest(req) // used to panic on second call - select { - case <-gotres: - case <-time.After(5 * time.Second): - panic("hang. events are: " + logbuf.String()) + if d, ok := t.Deadline(); ok { + // When the test's deadline is about to expire, log the pending events for + // better debugging. + timeout := time.Until(d) * 19 / 20 // Allow 5% for cleanup. + timer := time.AfterFunc(timeout, func() { + panic(fmt.Sprintf("hang in %s. events are: %s", t.Name(), logbuf.String())) + }) + defer timer.Stop() } + <-gotres got := logbuf.String() want := `dial: blocking @@ -2555,9 +2558,11 @@ func testCancelRequestWithChannel(t *testing.T, mode testMode) { if testing.Short() { t.Skip("skipping test in -short mode") } - unblockc := make(chan bool) + + const msg = "Hello" + unblockc := make(chan struct{}) ts := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { - fmt.Fprintf(w, "Hello") + io.WriteString(w, msg) w.(Flusher).Flush() // send headers and some body <-unblockc })).ts @@ -2567,42 +2572,39 @@ func testCancelRequestWithChannel(t *testing.T, mode testMode) { tr := c.Transport.(*Transport) req, _ := NewRequest("GET", ts.URL, nil) - ch := make(chan struct{}) - req.Cancel = ch + cancel := make(chan struct{}) + req.Cancel = cancel res, err := c.Do(req) if err != nil { t.Fatal(err) } - go func() { - time.Sleep(1 * time.Second) - close(ch) - }() - t0 := time.Now() - body, err := io.ReadAll(res.Body) - d := time.Since(t0) + body := make([]byte, len(msg)) + n, _ := io.ReadFull(res.Body, body) + if n != len(body) || !bytes.Equal(body, []byte(msg)) { + t.Errorf("Body = %q; want %q", body[:n], msg) + } + close(cancel) + tail, err := io.ReadAll(res.Body) + res.Body.Close() if err != ExportErrRequestCanceled { t.Errorf("Body.Read error = %v; want errRequestCanceled", err) + } else if len(tail) > 0 { + t.Errorf("Spurious bytes from Body.Read: %q", tail) } - if string(body) != "Hello" { - t.Errorf("Body = %q; want Hello", body) - } - if d < 500*time.Millisecond { - t.Errorf("expected ~1 second delay; got %v", d) - } + // Verify no outstanding requests after readLoop/writeLoop // goroutines shut down. - for tries := 5; tries > 0; tries-- { + waitCondition(t, 10*time.Millisecond, func(d time.Duration) bool { n := tr.NumPendingRequestsForTesting() - if n == 0 { - break - } - time.Sleep(100 * time.Millisecond) - if tries == 1 { - t.Errorf("pending requests = %d; want 0", n) + if n > 0 { + if d > 0 { + t.Logf("pending requests = %d after %v (want 0)", n, d) + } } - } + return true + }) } func TestCancelRequestWithChannelBeforeDo_Cancel(t *testing.T) { @@ -2731,25 +2733,13 @@ func testTransportCloseResponseBody(t *testing.T, mode testMode) { if !bytes.Equal(buf, want) { t.Fatalf("read %q; want %q", buf, want) } - didClose := make(chan error, 1) - go func() { - didClose <- res.Body.Close() - }() - select { - case err := <-didClose: - if err != nil { - t.Errorf("Close = %v", err) - } - case <-time.After(10 * time.Second): - t.Fatal("too long waiting for close") + + if err := res.Body.Close(); err != nil { + t.Errorf("Close = %v", err) } - select { - case err := <-writeErr: - if err == nil { - t.Errorf("expected non-nil write error") - } - case <-time.After(10 * time.Second): - t.Fatal("too long waiting for write error") + + if err := <-writeErr; err == nil { + t.Errorf("expected non-nil write error") } } @@ -2827,13 +2817,21 @@ func testTransportSocketLateBinding(t *testing.T, mode testMode) { ts := newClientServerTest(t, mode, mux).ts dialGate := make(chan bool, 1) + dialing := make(chan bool) c := ts.Client() c.Transport.(*Transport).Dial = func(n, addr string) (net.Conn, error) { - if <-dialGate { - return net.Dial(n, addr) + for { + select { + case ok := <-dialGate: + if !ok { + return nil, errors.New("manually closed") + } + return net.Dial(n, addr) + case dialing <- true: + } } - return nil, errors.New("manually closed") } + defer close(dialGate) dialGate <- true // only allow one dial fooRes, err := c.Get(ts.URL + "/foo") @@ -2844,13 +2842,34 @@ func testTransportSocketLateBinding(t *testing.T, mode testMode) { if fooAddr == "" { t.Fatal("No addr on /foo request") } - time.AfterFunc(200*time.Millisecond, func() { - // let the foo response finish so we can use its - // connection for /bar + + fooDone := make(chan struct{}) + go func() { + // We know that the foo Dial completed and reached the handler because we + // read its header. Wait for the bar request to block in Dial, then + // let the foo response finish so we can use its connection for /bar. + + if mode == http2Mode { + // In HTTP/2 mode, the second Dial won't happen because the protocol + // multiplexes the streams by default. Just sleep for an arbitrary time; + // the test should pass regardless of how far the bar request gets by this + // point. + select { + case <-dialing: + t.Errorf("unexpected second Dial in HTTP/2 mode") + case <-time.After(10 * time.Millisecond): + } + } else { + <-dialing + } fooGate <- true io.Copy(io.Discard, fooRes.Body) fooRes.Body.Close() - }) + close(fooDone) + }() + defer func() { + <-fooDone + }() barRes, err := c.Get(ts.URL + "/bar") if err != nil { @@ -2861,7 +2880,6 @@ func testTransportSocketLateBinding(t *testing.T, mode testMode) { t.Fatalf("/foo came from conn %q; /bar came from %q instead", fooAddr, barAddr) } barRes.Body.Close() - dialGate <- false } // Issue 2184 @@ -3271,42 +3289,33 @@ func TestTransportTLSHandshakeTimeout(t *testing.T) { c.Close() }() - getdonec := make(chan struct{}) - go func() { - defer close(getdonec) - tr := &Transport{ - Dial: func(_, _ string) (net.Conn, error) { - return net.Dial("tcp", ln.Addr().String()) - }, - TLSHandshakeTimeout: 250 * time.Millisecond, - } - cl := &Client{Transport: tr} - _, err := cl.Get("https://dummy.tld/") - if err == nil { - t.Error("expected error") - return - } - ue, ok := err.(*url.Error) - if !ok { - t.Errorf("expected url.Error; got %#v", err) - return - } - ne, ok := ue.Err.(net.Error) - if !ok { - t.Errorf("expected net.Error; got %#v", err) - return - } - if !ne.Timeout() { - t.Errorf("expected timeout error; got %v", err) - } - if !strings.Contains(err.Error(), "handshake timeout") { - t.Errorf("expected 'handshake timeout' in error; got %v", err) - } - }() - select { - case <-getdonec: - case <-time.After(5 * time.Second): - t.Error("test timeout; TLS handshake hung?") + tr := &Transport{ + Dial: func(_, _ string) (net.Conn, error) { + return net.Dial("tcp", ln.Addr().String()) + }, + TLSHandshakeTimeout: 250 * time.Millisecond, + } + cl := &Client{Transport: tr} + _, err := cl.Get("https://dummy.tld/") + if err == nil { + t.Error("expected error") + return + } + ue, ok := err.(*url.Error) + if !ok { + t.Errorf("expected url.Error; got %#v", err) + return + } + ne, ok := ue.Err.(net.Error) + if !ok { + t.Errorf("expected net.Error; got %#v", err) + return + } + if !ne.Timeout() { + t.Errorf("expected timeout error; got %v", err) + } + if !strings.Contains(err.Error(), "handshake timeout") { + t.Errorf("expected 'handshake timeout' in error; got %v", err) } } @@ -3439,24 +3448,15 @@ func testTransportNoReuseAfterEarlyResponse(t *testing.T, mode testMode) { if err := wantBody(res, err, "foo"); err != nil { t.Errorf("POST response: %v", err) } - donec := make(chan bool) - go func() { - defer close(donec) - res, err = c.Get(ts.URL) - if err := wantBody(res, err, "bar"); err != nil { - t.Errorf("GET response: %v", err) - return - } - getOkay = true // suppress test noise - }() - time.AfterFunc(5*time.Second, closeConn) - select { - case <-donec: - finalBit <- 'x' // unblock the writeloop of the first Post - close(finalBit) - case <-time.After(7 * time.Second): - t.Fatal("timeout waiting for GET request to finish") + + res, err = c.Get(ts.URL) + if err := wantBody(res, err, "bar"); err != nil { + t.Errorf("GET response: %v", err) + return } + getOkay = true // suppress test noise + finalBit <- 'x' // unblock the writeloop of the first Post + close(finalBit) } // Tests that we don't leak Transport persistConn.readLoop goroutines @@ -3927,35 +3927,45 @@ func testTransportRemovesH2ConnsAfterIdle(t *testing.T, mode testMode) { t.Skip("skipping in short mode") } - trFunc := func(tr *Transport) { - tr.MaxConnsPerHost = 1 - tr.MaxIdleConnsPerHost = 1 - tr.IdleConnTimeout = 10 * time.Millisecond - } - cst := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) {}), trFunc) + timeout := 1 * time.Millisecond + retry := true + for retry { + trFunc := func(tr *Transport) { + tr.MaxConnsPerHost = 1 + tr.MaxIdleConnsPerHost = 1 + tr.IdleConnTimeout = timeout + } + cst := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) {}), trFunc) - if _, err := cst.c.Get(cst.ts.URL); err != nil { - t.Fatalf("got error: %s", err) - } + retry = false + tooShort := func(err error) bool { + if err == nil || !strings.Contains(err.Error(), "use of closed network connection") { + return false + } + if !retry { + t.Helper() + t.Logf("idle conn timeout %v may be too short; retrying with longer", timeout) + timeout *= 2 + retry = true + cst.close() + } + return true + } - time.Sleep(100 * time.Millisecond) - got := make(chan error) - go func() { if _, err := cst.c.Get(cst.ts.URL); err != nil { - got <- err + if tooShort(err) { + continue + } + t.Fatalf("got error: %s", err) } - close(got) - }() - timeout := time.NewTimer(5 * time.Second) - defer timeout.Stop() - select { - case err := <-got: - if err != nil { + time.Sleep(10 * timeout) + if _, err := cst.c.Get(cst.ts.URL); err != nil { + if tooShort(err) { + continue + } t.Fatalf("got error: %s", err) } - case <-timeout.C: - t.Fatal("request never completed") } } @@ -3965,9 +3975,13 @@ func testTransportRemovesH2ConnsAfterIdle(t *testing.T, mode testMode) { // golang.org/issue/8923 func TestTransportRangeAndGzip(t *testing.T) { run(t, testTransportRangeAndGzip) } func testTransportRangeAndGzip(t *testing.T, mode testMode) { - reqc := make(chan *Request, 1) ts := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { - reqc <- r + if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") { + t.Error("Transport advertised gzip support in the Accept header") + } + if r.Header.Get("Range") == "" { + t.Error("no Range in request") + } })).ts c := ts.Client() @@ -3977,18 +3991,6 @@ func testTransportRangeAndGzip(t *testing.T, mode testMode) { if err != nil { t.Fatal(err) } - - select { - case r := <-reqc: - if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") { - t.Error("Transport advertised gzip support in the Accept header") - } - if r.Header.Get("Range") == "" { - t.Error("no Range in request") - } - case <-time.After(10 * time.Second): - t.Fatal("timeout") - } res.Body.Close() } @@ -4242,12 +4244,8 @@ func testTransportFlushesRequestHeader(t *testing.T, mode testMode) { res.Body.Close() }() - select { - case <-gotReq: - pw.Close() - case <-time.After(5 * time.Second): - t.Fatal("timeout waiting for handler to get request") - } + <-gotReq + pw.Close() <-gotRes } @@ -4600,11 +4598,7 @@ func testTransportEventTrace(t *testing.T, mode testMode, noHooks bool) { t.Error(err) } if !noHooks { - select { - case <-gotWroteReqEvent: - case <-time.After(5 * time.Second): - t.Error("timeout waiting for WroteRequest event") - } + <-gotWroteReqEvent } io.WriteString(w, resBody) }), func(tr *Transport) { @@ -5033,57 +5027,80 @@ func testTransportIdleConnTimeout(t *testing.T, mode testMode) { t.Skip("skipping in short mode") } - const timeout = 1 * time.Second - - cst := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { - // No body for convenience. - })) - tr := cst.tr - tr.IdleConnTimeout = timeout - defer tr.CloseIdleConnections() - c := &Client{Transport: tr} + timeout := 1 * time.Millisecond +timeoutLoop: + for { + cst := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { + // No body for convenience. + })) + tr := cst.tr + tr.IdleConnTimeout = timeout + defer tr.CloseIdleConnections() + c := &Client{Transport: tr} - idleConns := func() []string { - if mode == http2Mode { - return tr.IdleConnStrsForTesting_h2() - } else { - return tr.IdleConnStrsForTesting() + idleConns := func() []string { + if mode == http2Mode { + return tr.IdleConnStrsForTesting_h2() + } else { + return tr.IdleConnStrsForTesting() + } } - } - var conn string - doReq := func(n int) { - req, _ := NewRequest("GET", cst.ts.URL, nil) - req = req.WithContext(httptrace.WithClientTrace(context.Background(), &httptrace.ClientTrace{ - PutIdleConn: func(err error) { - if err != nil { - t.Errorf("failed to keep idle conn: %v", err) + var conn string + doReq := func(n int) (ok bool) { + req, _ := NewRequest("GET", cst.ts.URL, nil) + req = req.WithContext(httptrace.WithClientTrace(context.Background(), &httptrace.ClientTrace{ + PutIdleConn: func(err error) { + if err != nil { + t.Errorf("failed to keep idle conn: %v", err) + } + }, + })) + res, err := c.Do(req) + if err != nil { + if strings.Contains(err.Error(), "use of closed network connection") { + t.Logf("req %v: connection closed prematurely", n) + return false } - }, - })) - res, err := c.Do(req) - if err != nil { - t.Fatal(err) - } - res.Body.Close() - conns := idleConns() - if len(conns) != 1 { - t.Fatalf("req %v: unexpected number of idle conns: %q", n, conns) - } - if conn == "" { - conn = conns[0] + } + res.Body.Close() + conns := idleConns() + if len(conns) != 1 { + if len(conns) == 0 { + t.Logf("req %v: no idle conns", n) + return false + } + t.Fatalf("req %v: unexpected number of idle conns: %q", n, conns) + } + if conn == "" { + conn = conns[0] + } + if conn != conns[0] { + t.Logf("req %v: cached connection changed; expected the same one throughout the test", n) + return false + } + return true } - if conn != conns[0] { - t.Fatalf("req %v: cached connection changed; expected the same one throughout the test", n) + for i := 0; i < 3; i++ { + if !doReq(i) { + t.Logf("idle conn timeout %v appears to be too short; retrying with longer", timeout) + timeout *= 2 + cst.close() + continue timeoutLoop + } + time.Sleep(timeout / 2) } - } - for i := 0; i < 3; i++ { - doReq(i) - time.Sleep(timeout / 2) - } - time.Sleep(timeout * 3 / 2) - if got := idleConns(); len(got) != 0 { - t.Errorf("idle conns = %q; want none", got) + + waitCondition(t, timeout/2, func(d time.Duration) bool { + if got := idleConns(); len(got) != 0 { + if d >= timeout*3/2 { + t.Logf("after %d, idle conns = %q", d, got) + } + return false + } + return true + }) + break } } @@ -5129,13 +5146,9 @@ func testIdleConnH2Crash(t *testing.T, mode testMode) { cancel() - failTimer := time.NewTimer(5 * time.Second) - defer failTimer.Stop() select { case <-sawDoErr: case <-testDone: - case <-failTimer.C: - t.Error("timeout in DialTLS, waiting too long for cst.c.Do to fail") } return c, nil } @@ -5295,16 +5308,13 @@ func testTransportProxyConnectHeader(t *testing.T, mode testMode) { res.Body.Close() t.Errorf("unexpected success") } - select { - case <-time.After(3 * time.Second): - t.Fatal("timeout") - case r := <-reqc: - if got, want := r.Header.Get("User-Agent"), "foo"; got != want { - t.Errorf("CONNECT request User-Agent = %q; want %q", got, want) - } - if got, want := r.Header.Get("Other"), "bar"; got != want { - t.Errorf("CONNECT request Other = %q; want %q", got, want) - } + + r := <-reqc + if got, want := r.Header.Get("User-Agent"), "foo"; got != want { + t.Errorf("CONNECT request User-Agent = %q; want %q", got, want) + } + if got, want := r.Header.Get("Other"), "bar"; got != want { + t.Errorf("CONNECT request Other = %q; want %q", got, want) } } @@ -5347,16 +5357,13 @@ func testTransportProxyGetConnectHeader(t *testing.T, mode testMode) { res.Body.Close() t.Errorf("unexpected success") } - select { - case <-time.After(3 * time.Second): - t.Fatal("timeout") - case r := <-reqc: - if got, want := r.Header.Get("User-Agent"), "foo2"; got != want { - t.Errorf("CONNECT request User-Agent = %q; want %q", got, want) - } - if got, want := r.Header.Get("Other"), "bar2"; got != want { - t.Errorf("CONNECT request Other = %q; want %q", got, want) - } + + r := <-reqc + if got, want := r.Header.Get("User-Agent"), "foo2"; got != want { + t.Errorf("CONNECT request User-Agent = %q; want %q", got, want) + } + if got, want := r.Header.Get("Other"), "bar2"; got != want { + t.Errorf("CONNECT request Other = %q; want %q", got, want) } } @@ -5575,46 +5582,48 @@ func TestClientTimeoutKillsConn_BeforeHeaders(t *testing.T) { run(t, testClientTimeoutKillsConn_BeforeHeaders, []testMode{http1Mode}) } func testClientTimeoutKillsConn_BeforeHeaders(t *testing.T, mode testMode) { - inHandler := make(chan net.Conn, 1) - handlerReadReturned := make(chan bool, 1) - cst := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { - conn, _, err := w.(Hijacker).Hijack() - if err != nil { - t.Error(err) - return - } - inHandler <- conn - n, err := conn.Read([]byte{0}) - if n != 0 || err != io.EOF { - t.Errorf("unexpected Read result: %v, %v", n, err) - } - handlerReadReturned <- true - })) + timeout := 1 * time.Millisecond + for { + inHandler := make(chan net.Conn, 1) + handlerReadReturned := make(chan bool, 1) + cst := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { + conn, _, err := w.(Hijacker).Hijack() + if err != nil { + t.Error(err) + return + } + inHandler <- conn + n, err := conn.Read([]byte{0}) + if n != 0 || err != io.EOF { + t.Errorf("unexpected Read result: %v, %v", n, err) + } + handlerReadReturned <- true + })) - const timeout = 50 * time.Millisecond - cst.c.Timeout = timeout + cst.c.Timeout = timeout - _, err := cst.c.Get(cst.ts.URL) - if err == nil { - t.Fatal("unexpected Get succeess") - } + _, err := cst.c.Get(cst.ts.URL) + if err == nil { + t.Fatal("unexpected Get succeess") + } - select { - case c := <-inHandler: + var c net.Conn + tooSlow := time.NewTimer(timeout * 10) select { - case <-handlerReadReturned: - // Success. - return - case <-time.After(5 * time.Second): - t.Error("Handler's conn.Read seems to be stuck in Read") - c.Close() // close it to unblock Handler + case <-tooSlow.C: + // If we didn't get into the Handler, that probably means the builder was + // just slow and the Get failed in that time but never made it to the + // server. That's fine; we'll try again with a longer timout. + t.Logf("no handler seen in %v; retrying with longer timout", timeout) + timeout *= 2 + cst.close() + continue + case c = <-inHandler: + tooSlow.Stop() } - case <-time.After(timeout * 10): - // If we didn't get into the Handler in 50ms, that probably means - // the builder was just slow and the Get failed in that time - // but never made it to the server. That's fine. We'll usually - // test the part above on faster machines. - t.Skip("skipping test on slow builder") + <-handlerReadReturned + c.Close() + break } } @@ -5678,21 +5687,11 @@ func testClientTimeoutKillsConn_AfterHeaders(t *testing.T, mode testMode) { t.Fatalf("unexpected success; read %q, nil", got) } - select { - case c := <-inHandler: - select { - case err := <-handlerResult: - if err != nil { - t.Errorf("handler: %v", err) - } - return - case <-time.After(5 * time.Second): - t.Error("Handler's conn.Read seems to be stuck in Read") - c.Close() // close it to unblock Handler - } - case <-time.After(5 * time.Second): - t.Fatal("timeout") + c := <-inHandler + if err := <-handlerResult; err != nil { + t.Errorf("handler: %v", err) } + c.Close() } func TestTransportResponseBodyWritableOnProtocolSwitch(t *testing.T) { @@ -6127,17 +6126,18 @@ func testTransportIgnores408(t *testing.T, mode testMode) { t.Fatalf("got %q; want ok", slurp) } - t0 := time.Now() - for i := 0; i < 50; i++ { - time.Sleep(time.Duration(i) * 5 * time.Millisecond) - if cst.tr.IdleConnKeyCountForTesting() == 0 { - if got := logout.String(); got != "" { - t.Fatalf("expected no log output; got: %s", got) + waitCondition(t, 1*time.Millisecond, func(d time.Duration) bool { + if n := cst.tr.IdleConnKeyCountForTesting(); n != 0 { + if d > 0 { + t.Logf("%v idle conns still present after %v", n, d) } - return + return false } + return true + }) + if got := logout.String(); got != "" { + t.Fatalf("expected no log output; got: %s", got) } - t.Fatalf("timeout after %v waiting for Transport connections to die off", time.Since(t0)) } func TestInvalidHeaderResponse(t *testing.T) { @@ -6410,26 +6410,22 @@ func TestAltProtoCancellation(t *testing.T) { Transport: tr, Timeout: time.Millisecond, } - tr.RegisterProtocol("timeout", timeoutProto{}) - _, err := c.Get("timeout://bar.com/path") + tr.RegisterProtocol("cancel", cancelProto{}) + _, err := c.Get("cancel://bar.com/path") if err == nil { t.Error("request unexpectedly succeeded") - } else if !strings.Contains(err.Error(), timeoutProtoErr.Error()) { - t.Errorf("got error %q, does not contain expected string %q", err, timeoutProtoErr) + } else if !strings.Contains(err.Error(), errCancelProto.Error()) { + t.Errorf("got error %q, does not contain expected string %q", err, errCancelProto) } } -var timeoutProtoErr = errors.New("canceled as expected") +var errCancelProto = errors.New("canceled as expected") -type timeoutProto struct{} +type cancelProto struct{} -func (timeoutProto) RoundTrip(req *Request) (*Response, error) { - select { - case <-req.Cancel: - return nil, timeoutProtoErr - case <-time.After(5 * time.Second): - return nil, errors.New("request was not canceled") - } +func (cancelProto) RoundTrip(req *Request) (*Response, error) { + <-req.Cancel + return nil, errCancelProto } type roundTripFunc func(r *Request) (*Response, error) From 36eda4b8684ce78d1c1a8db92d5cd4159ca25b51 Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Mon, 20 Mar 2023 16:19:45 -0400 Subject: [PATCH 29/86] net/http: in the IdleConnStrsForTesting_h2 helper, omit conns that cannot be reused In #59155, we observed that the IdleConnStrsForTesting_h2 helper function sometimes reported extra connections after a "client conn not usable" failure and retry. It turns out that that state corresponds exactly to the http2clientConnIdleState.canTakeNewRequest field, so (with a bit of extra nethttpomithttp2 plumbing) we can use that field in the helper to filter out the unusable connections. Fixes #59155. Change-Id: Ief6283c9c8c5ec47dd9f378beb0ddf720832484e Reviewed-on: https://go-review.googlesource.com/c/go/+/477856 Reviewed-by: Damien Neil TryBot-Result: Gopher Robot Run-TryBot: Bryan Mills Auto-Submit: Bryan Mills --- export_test.go | 8 +++++--- omithttp2.go | 10 +++++++++- transport_test.go | 4 ++-- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/export_test.go b/export_test.go index fb5ab939..8a61e651 100644 --- a/export_test.go +++ b/export_test.go @@ -142,9 +142,11 @@ func (t *Transport) IdleConnStrsForTesting_h2() []string { pool.mu.Lock() defer pool.mu.Unlock() - for k, cc := range pool.conns { - for range cc { - ret = append(ret, k) + for k, ccs := range pool.conns { + for _, cc := range ccs { + if cc.idleState().canTakeNewRequest { + ret = append(ret, k) + } } } diff --git a/omithttp2.go b/omithttp2.go index 3316f55c..ca08ddfa 100644 --- a/omithttp2.go +++ b/omithttp2.go @@ -42,9 +42,17 @@ type http2noDialClientConnPool struct { type http2clientConnPool struct { mu *sync.Mutex - conns map[string][]struct{} + conns map[string][]*http2clientConn } +type http2clientConn struct{} + +type http2clientConnIdleState struct { + canTakeNewRequest bool +} + +func (cc *http2clientConn) idleState() http2clientConnIdleState { return http2clientConnIdleState{} } + func http2configureTransports(*Transport) (*http2Transport, error) { panic(noHTTP2) } func http2isNoCachedConnError(err error) bool { diff --git a/transport_test.go b/transport_test.go index 1abb0aab..b82c6156 100644 --- a/transport_test.go +++ b/transport_test.go @@ -5047,7 +5047,7 @@ timeoutLoop: } var conn string - doReq := func(n int) (ok bool) { + doReq := func(n int) (timeoutOk bool) { req, _ := NewRequest("GET", cst.ts.URL, nil) req = req.WithContext(httptrace.WithClientTrace(context.Background(), &httptrace.ClientTrace{ PutIdleConn: func(err error) { @@ -5094,7 +5094,7 @@ timeoutLoop: waitCondition(t, timeout/2, func(d time.Duration) bool { if got := idleConns(); len(got) != 0 { if d >= timeout*3/2 { - t.Logf("after %d, idle conns = %q", d, got) + t.Logf("after %v, idle conns = %q", d, got) } return false } From 13da1c626e86903e97741cac573f491d9dbeebc4 Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Tue, 21 Mar 2023 16:14:53 -0400 Subject: [PATCH 30/86] net/http: simplify Conn lifetimes in TestClientTimeoutKillsConn tests This is intended to fix the failure mode observed in https://build.golang.org/log/f153e06ed547517fb2cddb0fa817fea40a6146f7, but I haven't been able to reproduce that failure mode locally so I'm not sure whether it actually does. Change-Id: Ib14378f1299a76b54013419bdc715a9dbdd94667 Reviewed-on: https://go-review.googlesource.com/c/go/+/478235 Auto-Submit: Bryan Mills Reviewed-by: Damien Neil Run-TryBot: Bryan Mills TryBot-Result: Gopher Robot --- transport_test.go | 78 ++++++++++++++++++++++++++++------------------- 1 file changed, 46 insertions(+), 32 deletions(-) diff --git a/transport_test.go b/transport_test.go index b82c6156..9465b93b 100644 --- a/transport_test.go +++ b/transport_test.go @@ -5584,30 +5584,40 @@ func TestClientTimeoutKillsConn_BeforeHeaders(t *testing.T) { func testClientTimeoutKillsConn_BeforeHeaders(t *testing.T, mode testMode) { timeout := 1 * time.Millisecond for { - inHandler := make(chan net.Conn, 1) - handlerReadReturned := make(chan bool, 1) + inHandler := make(chan bool) + cancelHandler := make(chan struct{}) + handlerDone := make(chan bool) cst := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { + <-r.Context().Done() + + select { + case <-cancelHandler: + return + case inHandler <- true: + } + defer func() { handlerDone <- true }() + + // Read from the conn until EOF to verify that it was correctly closed. conn, _, err := w.(Hijacker).Hijack() if err != nil { t.Error(err) return } - inHandler <- conn n, err := conn.Read([]byte{0}) if n != 0 || err != io.EOF { t.Errorf("unexpected Read result: %v, %v", n, err) } - handlerReadReturned <- true + conn.Close() })) cst.c.Timeout = timeout _, err := cst.c.Get(cst.ts.URL) if err == nil { + close(cancelHandler) t.Fatal("unexpected Get succeess") } - var c net.Conn tooSlow := time.NewTimer(timeout * 10) select { case <-tooSlow.C: @@ -5615,14 +5625,14 @@ func testClientTimeoutKillsConn_BeforeHeaders(t *testing.T, mode testMode) { // just slow and the Get failed in that time but never made it to the // server. That's fine; we'll try again with a longer timout. t.Logf("no handler seen in %v; retrying with longer timout", timeout) - timeout *= 2 + close(cancelHandler) cst.close() + timeout *= 2 continue - case c = <-inHandler: + case <-inHandler: tooSlow.Stop() + <-handlerDone } - <-handlerReadReturned - c.Close() break } } @@ -5636,18 +5646,27 @@ func TestClientTimeoutKillsConn_AfterHeaders(t *testing.T) { run(t, testClientTimeoutKillsConn_AfterHeaders, []testMode{http1Mode}) } func testClientTimeoutKillsConn_AfterHeaders(t *testing.T, mode testMode) { - inHandler := make(chan net.Conn, 1) - handlerResult := make(chan error, 1) + inHandler := make(chan bool) + cancelHandler := make(chan struct{}) + handlerDone := make(chan bool) cst := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { w.Header().Set("Content-Length", "100") w.(Flusher).Flush() + + select { + case <-cancelHandler: + return + case inHandler <- true: + } + defer func() { handlerDone <- true }() + conn, _, err := w.(Hijacker).Hijack() if err != nil { t.Error(err) return } conn.Write([]byte("foo")) - inHandler <- conn + n, err := conn.Read([]byte{0}) // The error should be io.EOF or "read tcp // 127.0.0.1:35827->127.0.0.1:40290: read: connection @@ -5655,43 +5674,38 @@ func testClientTimeoutKillsConn_AfterHeaders(t *testing.T, mode testMode) { // care that it returns at all. But if it returns with // data, that's weird. if n != 0 || err == nil { - handlerResult <- fmt.Errorf("unexpected Read result: %v, %v", n, err) - return + t.Errorf("unexpected Read result: %v, %v", n, err) } - handlerResult <- nil + conn.Close() })) // Set Timeout to something very long but non-zero to exercise // the codepaths that check for it. But rather than wait for it to fire // (which would make the test slow), we send on the req.Cancel channel instead, // which happens to exercise the same code paths. - cst.c.Timeout = time.Minute // just to be non-zero, not to hit it. + cst.c.Timeout = 24 * time.Hour // just to be non-zero, not to hit it. req, _ := NewRequest("GET", cst.ts.URL, nil) - cancel := make(chan struct{}) - req.Cancel = cancel + cancelReq := make(chan struct{}) + req.Cancel = cancelReq res, err := cst.c.Do(req) if err != nil { - select { - case <-inHandler: - t.Fatalf("Get error: %v", err) - default: - // Failed before entering handler. Ignore result. - t.Skip("skipping test on slow builder") - } + close(cancelHandler) + t.Fatalf("Get error: %v", err) } - close(cancel) + // Cancel the request while the handler is still blocked on sending to the + // inHandler channel. Then read it until it fails, to verify that the + // connection is broken before the handler itself closes it. + close(cancelReq) got, err := io.ReadAll(res.Body) if err == nil { - t.Fatalf("unexpected success; read %q, nil", got) + t.Errorf("unexpected success; read %q, nil", got) } - c := <-inHandler - if err := <-handlerResult; err != nil { - t.Errorf("handler: %v", err) - } - c.Close() + // Now unblock the handler and wait for it to complete. + <-inHandler + <-handlerDone } func TestTransportResponseBodyWritableOnProtocolSwitch(t *testing.T) { From 0c7faadbf980a6f9934124117ebd659ae32b6c98 Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Wed, 22 Mar 2023 12:19:23 -0400 Subject: [PATCH 31/86] net/http: improve logging in TestServerSetKeepAlivesEnabledClosesConns - Log the actual addresses reported, in case that information is relevant. - Keep going after the first error, so that we report more information about the idle connections after they have been used. (Was the first connection dropped completely, or did it later show up as idle?) - Remove the third request at the end of the test. It had been assuming that the address for a new connection would always be different from the address for the just-closed connection; however, that assumption does not hold in general. Removing the third request addresses one of the two failure modes seen in #55195. It may help in investigating the other failure mode, but I do not expect it to fix the failures entirely. (I suspect that the other failure mode is a synchronization bug in returning the idle connection from the first request.) For #55195. Change-Id: If9604ea68db0697268288ce9812dd57633e83fbd Reviewed-on: https://go-review.googlesource.com/c/go/+/478515 Reviewed-by: Damien Neil Auto-Submit: Bryan Mills TryBot-Result: Gopher Robot Run-TryBot: Bryan Mills --- serve_test.go | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/serve_test.go b/serve_test.go index eb4660f8..88184bcf 100644 --- a/serve_test.go +++ b/serve_test.go @@ -5421,39 +5421,36 @@ func testServerSetKeepAlivesEnabledClosesConns(t *testing.T, mode testMode) { get := func() string { return get(t, c, ts.URL) } a1, a2 := get(), get() - if a1 != a2 { - t.Fatal("expected first two requests on same connection") + if a1 == a2 { + t.Logf("made two requests from a single conn %q (as expected)", a1) + } else { + t.Errorf("server reported requests from %q and %q; expected same connection", a1, a2) } - addr := strings.TrimPrefix(ts.URL, "http://") // The two requests should have used the same connection, // and there should not have been a second connection that // was created by racing dial against reuse. // (The first get was completed when the second get started.) - n := tr.IdleConnCountForTesting("http", addr) - if n != 1 { - t.Fatalf("idle count for %q after 2 gets = %d, want 1", addr, n) + if conns := tr.IdleConnStrsForTesting(); len(conns) != 1 { + t.Errorf("found %d idle conns (%q); want 1", len(conns), conns) } // SetKeepAlivesEnabled should discard idle conns. ts.Config.SetKeepAlivesEnabled(false) - var idle1 int waitCondition(t, 10*time.Millisecond, func(d time.Duration) bool { - idle1 = tr.IdleConnCountForTesting("http", addr) - if idle1 != 0 { + if conns := tr.IdleConnStrsForTesting(); len(conns) > 0 { if d > 0 { - t.Logf("idle count %v after SetKeepAlivesEnabled called = %v; waiting for 0", d, idle1) + t.Logf("idle conns %v after SetKeepAlivesEnabled called = %q; waiting for empty", d, conns) } return false } return true }) - a3 := get() - if a3 == a2 { - t.Fatal("expected third request on new connection") - } + // If we make a third request it should use a new connection, but in general + // we have no way to verify that: the new connection could happen to reuse the + // exact same ports from the previous connection. } func TestServerShutdown(t *testing.T) { run(t, testServerShutdown) } From 2eca7122f1b0dca35b06a0de262a933d3175f185 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Matczuk?= Date: Thu, 23 Mar 2023 13:52:59 +0000 Subject: [PATCH 32/86] net/http: fix typo Change-Id: I6e39f14df65685451d3d98c6def1e0665da20590 GitHub-Last-Rev: d9de237bd24ab3deebdbdf1f75a1835169670c1d GitHub-Pull-Request: golang/go#59200 Reviewed-on: https://go-review.googlesource.com/c/go/+/478855 Reviewed-by: Damien Neil Run-TryBot: Damien Neil TryBot-Result: Gopher Robot Auto-Submit: Damien Neil Reviewed-by: Ian Lance Taylor --- transport.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transport.go b/transport.go index 7561f7f5..807cc8f0 100644 --- a/transport.go +++ b/transport.go @@ -173,7 +173,7 @@ type Transport struct { // If non-nil, HTTP/2 support may not be enabled by default. TLSClientConfig *tls.Config - // TLSHandshakeTimeout specifies the maximum amount of time waiting to + // TLSHandshakeTimeout specifies the maximum amount of time to // wait for a TLS handshake. Zero means no timeout. TLSHandshakeTimeout time.Duration From f57b8dd044b5daab04eef7681cbb8f879071f255 Mon Sep 17 00:00:00 2001 From: Paschalis Tsilias Date: Mon, 15 Feb 2021 15:55:28 +0200 Subject: [PATCH 33/86] net/http: continue using referer header if it's present Currently, net/http replaces the Referer header with the URL of the previous request, regardless of its status. This CL changes this behavior, respecting the Referer header for secure connections, if it is set. Fixes #44160 Change-Id: I2d7fe37dd681549136329e832188294691584870 Reviewed-on: https://go-review.googlesource.com/c/go/+/291636 Reviewed-by: Damien Neil Reviewed-by: Nick Craig-Wood Run-TryBot: Damien Neil TryBot-Result: Gopher Robot Auto-Submit: Damien Neil Reviewed-by: Ian Lance Taylor --- client.go | 9 +++++++-- client_test.go | 30 +++++++++++++++++++----------- 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/client.go b/client.go index 2f7d4f69..1e300acf 100644 --- a/client.go +++ b/client.go @@ -144,7 +144,8 @@ type RoundTripper interface { // refererForURL returns a referer without any authentication info or // an empty string if lastReq scheme is https and newReq scheme is http. -func refererForURL(lastReq, newReq *url.URL) string { +// If the referer was explicitly set, then it will continue to be used. +func refererForURL(lastReq, newReq *url.URL, explicitRef string) string { // https://tools.ietf.org/html/rfc7231#section-5.5.2 // "Clients SHOULD NOT include a Referer header field in a // (non-secure) HTTP request if the referring page was @@ -152,6 +153,10 @@ func refererForURL(lastReq, newReq *url.URL) string { if lastReq.Scheme == "https" && newReq.Scheme == "http" { return "" } + if explicitRef != "" { + return explicitRef + } + referer := lastReq.String() if lastReq.User != nil { // This is not very efficient, but is the best we can @@ -676,7 +681,7 @@ func (c *Client) do(req *Request) (retres *Response, reterr error) { // Add the Referer header from the most recent // request URL to the new one, if it's not https->http: - if ref := refererForURL(reqs[len(reqs)-1].URL, req.URL); ref != "" { + if ref := refererForURL(reqs[len(reqs)-1].URL, req.URL, req.Header.Get("Referer")); ref != "" { req.Header.Set("Referer", ref) } err = c.checkRedirect(req, reqs) diff --git a/client_test.go b/client_test.go index bb1ed6f1..b8c914bf 100644 --- a/client_test.go +++ b/client_test.go @@ -1411,24 +1411,32 @@ func (f eofReaderFunc) Read(p []byte) (n int, err error) { func TestReferer(t *testing.T) { tests := []struct { - lastReq, newReq string // from -> to URLs - want string + lastReq, newReq, explicitRef string // from -> to URLs, explicitly set Referer value + want string }{ // don't send user: - {"http://gopher@test.com", "http://link.com", "http://test.com"}, - {"https://gopher@test.com", "https://link.com", "https://test.com"}, + {lastReq: "http://gopher@test.com", newReq: "http://link.com", want: "http://test.com"}, + {lastReq: "https://gopher@test.com", newReq: "https://link.com", want: "https://test.com"}, // don't send a user and password: - {"http://gopher:go@test.com", "http://link.com", "http://test.com"}, - {"https://gopher:go@test.com", "https://link.com", "https://test.com"}, + {lastReq: "http://gopher:go@test.com", newReq: "http://link.com", want: "http://test.com"}, + {lastReq: "https://gopher:go@test.com", newReq: "https://link.com", want: "https://test.com"}, // nothing to do: - {"http://test.com", "http://link.com", "http://test.com"}, - {"https://test.com", "https://link.com", "https://test.com"}, + {lastReq: "http://test.com", newReq: "http://link.com", want: "http://test.com"}, + {lastReq: "https://test.com", newReq: "https://link.com", want: "https://test.com"}, // https to http doesn't send a referer: - {"https://test.com", "http://link.com", ""}, - {"https://gopher:go@test.com", "http://link.com", ""}, + {lastReq: "https://test.com", newReq: "http://link.com", want: ""}, + {lastReq: "https://gopher:go@test.com", newReq: "http://link.com", want: ""}, + + // https to http should remove an existing referer: + {lastReq: "https://test.com", newReq: "http://link.com", explicitRef: "https://foo.com", want: ""}, + {lastReq: "https://gopher:go@test.com", newReq: "http://link.com", explicitRef: "https://foo.com", want: ""}, + + // don't override an existing referer: + {lastReq: "https://test.com", newReq: "https://link.com", explicitRef: "https://foo.com", want: "https://foo.com"}, + {lastReq: "https://gopher:go@test.com", newReq: "https://link.com", explicitRef: "https://foo.com", want: "https://foo.com"}, } for _, tt := range tests { l, err := url.Parse(tt.lastReq) @@ -1439,7 +1447,7 @@ func TestReferer(t *testing.T) { if err != nil { t.Fatal(err) } - r := ExportRefererForURL(l, n) + r := ExportRefererForURL(l, n, tt.explicitRef) if r != tt.want { t.Errorf("refererForURL(%q, %q) = %q; want %q", tt.lastReq, tt.newReq, r, tt.want) } From 781d6ac84d4a42239fd0c1eedfebc4595fbfdfea Mon Sep 17 00:00:00 2001 From: Philippe Antoine Date: Thu, 23 Mar 2023 08:14:39 +0000 Subject: [PATCH 34/86] all: replace fmt.Sprintf("%d") with strconv.Itoa This was found by running `git grep 'fmt.Sprintf("%d",' | grep -v test | grep -v vendor` And this was automatically fixed with gotiti https://github.com/catenacyber/gotiti and using unconvert https://github.com/mdempsky/unconvert to check if there was (tool which fixed another useless cast) Change-Id: I023926bc4aa8d51de45f712ac739a0a80145c28c GitHub-Last-Rev: 1063e32e5b69b6f9bb17673887b8c4ebe5be8fe4 GitHub-Pull-Request: golang/go#59144 Reviewed-on: https://go-review.googlesource.com/c/go/+/477675 Run-TryBot: Ian Lance Taylor TryBot-Result: Gopher Robot Reviewed-by: Bryan Mills Run-TryBot: Ian Lance Taylor Auto-Submit: Ian Lance Taylor Reviewed-by: Ian Lance Taylor --- triv.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/triv.go b/triv.go index 32edbbb3..f614922c 100644 --- a/triv.go +++ b/triv.go @@ -39,7 +39,7 @@ type Counter struct { func (ctr *Counter) String() string { ctr.mu.Lock() defer ctr.mu.Unlock() - return fmt.Sprintf("%d", ctr.n) + return strconv.Itoa(ctr.n) } func (ctr *Counter) ServeHTTP(w http.ResponseWriter, req *http.Request) { From c23656c96d9da6695f608dce7a5c7f639c5dc524 Mon Sep 17 00:00:00 2001 From: cui fliter Date: Fri, 31 Mar 2023 00:00:19 +0800 Subject: [PATCH 35/86] all: fix misuses of "a" vs "an" Fixes the misuse of "a" vs "an", according to English grammatical expectations and using https://www.a-or-an.com/ Change-Id: I53ac724070e3ff3d33c304483fe72c023c7cda47 Reviewed-on: https://go-review.googlesource.com/c/go/+/480536 Run-TryBot: shuang cui Reviewed-by: Ian Lance Taylor TryBot-Result: Gopher Robot Auto-Submit: Ian Lance Taylor Run-TryBot: Ian Lance Taylor Reviewed-by: Michael Knyszek --- cookiejar/jar.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cookiejar/jar.go b/cookiejar/jar.go index 3d0ad19a..e1ba5971 100644 --- a/cookiejar/jar.go +++ b/cookiejar/jar.go @@ -365,7 +365,7 @@ func isIP(host string) bool { return net.ParseIP(host) != nil } -// defaultPath returns the directory part of an URL's path according to +// defaultPath returns the directory part of a URL's path according to // RFC 6265 section 5.1.4. func defaultPath(path string) string { if len(path) == 0 || path[0] != '/' { @@ -379,7 +379,7 @@ func defaultPath(path string) string { return path[:i] // Path is either of form "/abc/xyz" or "/abc/xyz/". } -// newEntry creates an entry from a http.Cookie c. now is the current time and +// newEntry creates an entry from an http.Cookie c. now is the current time and // is compared to c.Expires to determine deletion of c. defPath and host are the // default-path and the canonical host name of the URL c was received from. // From 18f3402293de8cdd8fc02d896e1497b54f6d5819 Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Tue, 4 Apr 2023 14:49:23 -0400 Subject: [PATCH 36/86] net/http: drop client address comparison in TestTransportRemovesDeadIdleConnections Since the first client connection is explicitly closed before making the second request, we cannot in general assume that the second request uses a different port (it is equally valid to open the new connection on the same port as the old one that was closed). Fixes #59438. Change-Id: I52d5fe493bd8b1b49270d3996d2019d38d375ce9 Reviewed-on: https://go-review.googlesource.com/c/go/+/482175 Auto-Submit: Bryan Mills Reviewed-by: Damien Neil Run-TryBot: Bryan Mills TryBot-Result: Gopher Robot --- transport_test.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/transport_test.go b/transport_test.go index 9465b93b..268b0a47 100644 --- a/transport_test.go +++ b/transport_test.go @@ -741,7 +741,7 @@ func testTransportRemovesDeadIdleConnections(t *testing.T, mode testMode) { c := ts.Client() tr := c.Transport.(*Transport) - doReq := func(name string) string { + doReq := func(name string) { // Do a POST instead of a GET to prevent the Transport's // idempotent request retry logic from kicking in... res, err := c.Post(ts.URL, "", nil) @@ -756,10 +756,10 @@ func testTransportRemovesDeadIdleConnections(t *testing.T, mode testMode) { if err != nil { t.Fatalf("%s: %v", name, err) } - return string(slurp) + t.Logf("%s: ok (%q)", name, slurp) } - first := doReq("first") + doReq("first") keys1 := tr.IdleConnKeysForTesting() ts.CloseClientConnections() @@ -776,10 +776,7 @@ func testTransportRemovesDeadIdleConnections(t *testing.T, mode testMode) { return true }) - second := doReq("second") - if first == second { - t.Errorf("expected a different connection between requests. got %q both times", first) - } + doReq("second") } // Test that the Transport notices when a server hangs up on its From a361bc49fc875421e23649acfbfd9b4947d61bc1 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Wed, 5 Apr 2023 10:19:44 -0700 Subject: [PATCH 37/86] Revert "net/http: FileServer method check + minimal OPTIONS implementation" This reverts https://go.dev/cl/413554 Reason for revert: Backwards-incompatible change in behavior. For #53501 For #59375 Change-Id: Ic3f63b378f9c819599b32e5e6e410f6163849317 Reviewed-on: https://go-review.googlesource.com/c/go/+/482635 Reviewed-by: Tatiana Bradley Run-TryBot: Damien Neil TryBot-Result: Gopher Robot --- fs.go | 20 +++++--------------- fs_test.go | 41 ----------------------------------------- 2 files changed, 5 insertions(+), 56 deletions(-) diff --git a/fs.go b/fs.go index 7f302491..55094400 100644 --- a/fs.go +++ b/fs.go @@ -863,22 +863,12 @@ func FileServer(root FileSystem) Handler { } func (f *fileHandler) ServeHTTP(w ResponseWriter, r *Request) { - const options = MethodOptions + ", " + MethodGet + ", " + MethodHead - - switch r.Method { - case MethodGet, MethodHead: - if !strings.HasPrefix(r.URL.Path, "/") { - r.URL.Path = "/" + r.URL.Path - } - serveFile(w, r, f.root, path.Clean(r.URL.Path), true) - - case MethodOptions: - w.Header().Set("Allow", options) - - default: - w.Header().Set("Allow", options) - Error(w, "read-only", StatusMethodNotAllowed) + upath := r.URL.Path + if !strings.HasPrefix(upath, "/") { + upath = "/" + upath + r.URL.Path = upath } + serveFile(w, r, f.root, path.Clean(upath), true) } // httpRange specifies the byte range to be sent to the client. diff --git a/fs_test.go b/fs_test.go index ce429201..e5fb52f3 100644 --- a/fs_test.go +++ b/fs_test.go @@ -24,7 +24,6 @@ import ( "reflect" "regexp" "runtime" - "sort" "strings" "testing" "time" @@ -420,46 +419,6 @@ func testFileServerImplicitLeadingSlash(t *testing.T, mode testMode) { } } -func TestFileServerMethodOptions(t *testing.T) { run(t, testFileServerMethodOptions) } -func testFileServerMethodOptions(t *testing.T, mode testMode) { - const want = "GET, HEAD, OPTIONS" - ts := newClientServerTest(t, mode, FileServer(Dir("."))).ts - - tests := []struct { - method string - wantStatus int - }{ - {MethodOptions, StatusOK}, - - {MethodDelete, StatusMethodNotAllowed}, - {MethodPut, StatusMethodNotAllowed}, - {MethodPost, StatusMethodNotAllowed}, - } - - for _, test := range tests { - req, err := NewRequest(test.method, ts.URL, nil) - if err != nil { - t.Fatal(err) - } - res, err := ts.Client().Do(req) - if err != nil { - t.Fatal(err) - } - defer res.Body.Close() - - if res.StatusCode != test.wantStatus { - t.Errorf("%s got status %q, want code %d", test.method, res.Status, test.wantStatus) - } - - a := strings.Split(res.Header.Get("Allow"), ", ") - sort.Strings(a) - got := strings.Join(a, ", ") - if got != want { - t.Errorf("%s got Allow header %q, want %q", test.method, got, want) - } - } -} - func TestDirJoin(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("skipping test on windows") From 5a69ad0f0876002fce827d0b936685b85fe9199b Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Thu, 6 Apr 2023 11:01:05 -0700 Subject: [PATCH 38/86] net/http: add tests covering non-GET methods for file serving ServeFile and FileServer will respond to methods such as DELETE by serving the file contents. This is surprising, but we don't want to change it without some consideration. Add tests covering the current behavior. For #59470 Change-Id: Ib6a2594c5b2b7f380149fc1628f7204b308161e1 Reviewed-on: https://go-review.googlesource.com/c/go/+/482876 Run-TryBot: Damien Neil Reviewed-by: Tatiana Bradley TryBot-Result: Gopher Robot --- fs_test.go | 83 ++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 78 insertions(+), 5 deletions(-) diff --git a/fs_test.go b/fs_test.go index e5fb52f3..3f0f864b 100644 --- a/fs_test.go +++ b/fs_test.go @@ -87,15 +87,39 @@ func testServeFile(t *testing.T, mode testMode) { if req.URL, err = url.Parse(ts.URL); err != nil { t.Fatal("ParseURL:", err) } - req.Method = "GET" - // straight GET - _, body := getBody(t, "straight get", req, c) - if !bytes.Equal(body, file) { - t.Fatalf("body mismatch: got %q, want %q", body, file) + // Get contents via various methods. + // + // See https://go.dev/issue/59471 for a proposal to limit the set of methods handled. + // For now, test the historical behavior. + for _, method := range []string{ + MethodGet, + MethodPost, + MethodPut, + MethodPatch, + MethodDelete, + MethodOptions, + MethodTrace, + } { + req.Method = method + _, body := getBody(t, method, req, c) + if !bytes.Equal(body, file) { + t.Fatalf("body mismatch for %v request: got %q, want %q", method, body, file) + } + } + + // HEAD request. + req.Method = MethodHead + resp, body := getBody(t, "HEAD", req, c) + if len(body) != 0 { + t.Fatalf("body mismatch for HEAD request: got %q, want empty", body) + } + if got, want := resp.Header.Get("Content-Length"), fmt.Sprint(len(file)); got != want { + t.Fatalf("Content-Length mismatch for HEAD request: got %v, want %v", got, want) } // Range tests + req.Method = MethodGet Cases: for _, rt := range ServeFileRangeTests { if rt.r != "" { @@ -1521,3 +1545,52 @@ func testServeFileRejectsInvalidSuffixLengths(t *testing.T, mode testMode) { }) } } + +func TestFileServerMethods(t *testing.T) { + run(t, testFileServerMethods) +} +func testFileServerMethods(t *testing.T, mode testMode) { + ts := newClientServerTest(t, mode, FileServer(Dir("testdata"))).ts + + file, err := os.ReadFile(testFile) + if err != nil { + t.Fatal("reading file:", err) + } + + // Get contents via various methods. + // + // See https://go.dev/issue/59471 for a proposal to limit the set of methods handled. + // For now, test the historical behavior. + for _, method := range []string{ + MethodGet, + MethodHead, + MethodPost, + MethodPut, + MethodPatch, + MethodDelete, + MethodOptions, + MethodTrace, + } { + req, _ := NewRequest(method, ts.URL+"/file", nil) + t.Log(req.URL) + res, err := ts.Client().Do(req) + if err != nil { + t.Fatal(err) + } + body, err := io.ReadAll(res.Body) + res.Body.Close() + if err != nil { + t.Fatal(err) + } + wantBody := file + if method == MethodHead { + wantBody = nil + } + if !bytes.Equal(body, wantBody) { + t.Fatalf("%v: got body %q, want %q", method, body, wantBody) + } + if got, want := res.Header.Get("Content-Length"), fmt.Sprint(len(file)); got != want { + t.Fatalf("%v: got Content-Length %q, want %q", method, got, want) + } + } +} From 3abc4992618e23cdcae7148e2043f0b7432f56fc Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Thu, 6 Apr 2023 12:09:04 -0700 Subject: [PATCH 39/86] net/http: improve failure mode for TestResponseControllerSetPastReadDeadline A test flake in #59447 seems to indicate that this test got stuck waiting for the test handler to close the readc channel. If the handler returns early due to an unexpected error, it might fail to close this channel. Add a second channel to act as a signal that the handler has given up and the test should stop. This won't fix whatever happened in the flake, but might help us debug it if it happens again. For #59447 Change-Id: I05d84c6176aa938887d93126a6f3bb4dc941c90d Reviewed-on: https://go-review.googlesource.com/c/go/+/482935 Reviewed-by: Bryan Mills TryBot-Result: Gopher Robot Auto-Submit: Damien Neil Run-TryBot: Damien Neil --- responsecontroller_test.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/responsecontroller_test.go b/responsecontroller_test.go index ee8b55a8..c560e4bc 100644 --- a/responsecontroller_test.go +++ b/responsecontroller_test.go @@ -156,7 +156,9 @@ func TestResponseControllerSetPastReadDeadline(t *testing.T) { } func testResponseControllerSetPastReadDeadline(t *testing.T, mode testMode) { readc := make(chan struct{}) + donec := make(chan struct{}) cst := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { + defer close(donec) ctl := NewResponseController(w) b := make([]byte, 3) n, err := io.ReadFull(r.Body, b) @@ -192,10 +194,15 @@ func testResponseControllerSetPastReadDeadline(t *testing.T, mode testMode) { wg.Add(1) go func() { defer wg.Done() + defer pw.Close() pw.Write([]byte("one")) - <-readc + select { + case <-readc: + case <-donec: + t.Errorf("server handler unexpectedly exited without closing readc") + return + } pw.Write([]byte("two")) - pw.Close() }() defer wg.Wait() res, err := cst.c.Post(cst.ts.URL, "text/foo", pr) From e2404c3e6f2b2fb56c575025dd6897cc4b83e508 Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Fri, 7 Apr 2023 15:37:18 +0000 Subject: [PATCH 40/86] net/http: fix a race in TestResponseControllerSetPastReadDeadline If the Write goroutine is delayed for long enough after its first Write, the handler may have closed both the readc and donec channels by the time it selects over them, and the donec case may be randomly chosen. Handle that case by explicitly checking readc as well. This fixes a race accidentally introduced in CL 482935 and observed in https://build.golang.org/log/fa684750994d1fda409722f144b90c65b4c52cf9. For #59447. Change-Id: I5c87a599910cf8c1d037e5bbce68bf35afd55d61 Reviewed-on: https://go-review.googlesource.com/c/go/+/483036 TryBot-Result: Gopher Robot Auto-Submit: Bryan Mills Reviewed-by: Damien Neil Run-TryBot: Bryan Mills --- responsecontroller_test.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/responsecontroller_test.go b/responsecontroller_test.go index c560e4bc..5828f379 100644 --- a/responsecontroller_test.go +++ b/responsecontroller_test.go @@ -199,8 +199,12 @@ func testResponseControllerSetPastReadDeadline(t *testing.T, mode testMode) { select { case <-readc: case <-donec: - t.Errorf("server handler unexpectedly exited without closing readc") - return + select { + case <-readc: + default: + t.Errorf("server handler unexpectedly exited without closing readc") + return + } } pw.Write([]byte("two")) }() From b98a52e1bff50127d2101d259b183308f93f306e Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Fri, 7 Apr 2023 10:04:42 -0700 Subject: [PATCH 41/86] net/http: wait forever for write results in tests After performing a round trip on a connection, the connection is usually returned to the idle connection pool. If the write of the request did not complete successfully, the connection is not returned. It is possible for the response to be read before the write goroutine has finished signalling that its write has completed. To allow for this, the check to see if the write completed successfully waits for 50ms for the write goroutine to report the result of the write. See comments in persistConn.wroteRequest for more details. On a slow builder, it is possible for the write goroutine to take longer than 50ms to report the status of its write, leading to test flakiness when successive requests unexpectedly use different connections. Set the timeout for waiting for the writer to an effectively infinite duration in tests. Fixes #51147 Fixes #56275 Fixes #56419 Fixes #56577 Fixes #57375 Fixes #57417 Fixes #57476 Fixes #57604 Fixes #57605 Change-Id: I5e92ffd66b676f3f976d8832c0910f27456a6991 Reviewed-on: https://go-review.googlesource.com/c/go/+/483116 TryBot-Result: Gopher Robot Reviewed-by: Bryan Mills Run-TryBot: Bryan Mills --- export_test.go | 2 +- main_test.go | 1 + transport.go | 5 ++++- transport_test.go | 10 +++++++--- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/export_test.go b/export_test.go index 8a61e651..5d198f3f 100644 --- a/export_test.go +++ b/export_test.go @@ -36,7 +36,7 @@ var ( Export_is408Message = is408Message ) -const MaxWriteWaitBeforeConnReuse = maxWriteWaitBeforeConnReuse +var MaxWriteWaitBeforeConnReuse = &maxWriteWaitBeforeConnReuse func init() { // We only want to pay for this cost during testing. diff --git a/main_test.go b/main_test.go index 1b2fa215..1e83ca3c 100644 --- a/main_test.go +++ b/main_test.go @@ -20,6 +20,7 @@ import ( var quietLog = log.New(io.Discard, "", 0) func TestMain(m *testing.M) { + *http.MaxWriteWaitBeforeConnReuse = 60 * time.Minute v := m.Run() if v == 0 && goroutineLeaked() { os.Exit(1) diff --git a/transport.go b/transport.go index 807cc8f0..8de63cdb 100644 --- a/transport.go +++ b/transport.go @@ -2452,7 +2452,10 @@ func (pc *persistConn) writeLoop() { // maxWriteWaitBeforeConnReuse is how long the a Transport RoundTrip // will wait to see the Request's Body.Write result after getting a // response from the server. See comments in (*persistConn).wroteRequest. -const maxWriteWaitBeforeConnReuse = 50 * time.Millisecond +// +// In tests, we set this to a large value to avoid flakiness from inconsistent +// recycling of connections. +var maxWriteWaitBeforeConnReuse = 50 * time.Millisecond // wroteRequest is a check before recycling a connection that the previous write // (from writeLoop above) happened and was successful. diff --git a/transport_test.go b/transport_test.go index 268b0a47..6f57629e 100644 --- a/transport_test.go +++ b/transport_test.go @@ -3402,9 +3402,13 @@ func (c byteFromChanReader) Read(p []byte) (n int, err error) { // questionable state. // golang.org/issue/7569 func TestTransportNoReuseAfterEarlyResponse(t *testing.T) { - run(t, testTransportNoReuseAfterEarlyResponse, []testMode{http1Mode}) + run(t, testTransportNoReuseAfterEarlyResponse, []testMode{http1Mode}, testNotParallel) } func testTransportNoReuseAfterEarlyResponse(t *testing.T, mode testMode) { + defer func(d time.Duration) { + *MaxWriteWaitBeforeConnReuse = d + }(*MaxWriteWaitBeforeConnReuse) + *MaxWriteWaitBeforeConnReuse = 10 * time.Millisecond var sconn struct { sync.Mutex c net.Conn @@ -3631,13 +3635,13 @@ func testRetryRequestsOnError(t *testing.T, mode testMode) { req := tc.req() res, err := c.Do(req) if err != nil { - if time.Since(t0) < MaxWriteWaitBeforeConnReuse/2 { + if time.Since(t0) < *MaxWriteWaitBeforeConnReuse/2 { mu.Lock() got := logbuf.String() mu.Unlock() t.Fatalf("i=%d: Do = %v; log:\n%s", i, err, got) } - t.Skipf("connection likely wasn't recycled within %d, interfering with actual test; skipping", MaxWriteWaitBeforeConnReuse) + t.Skipf("connection likely wasn't recycled within %d, interfering with actual test; skipping", *MaxWriteWaitBeforeConnReuse) } res.Body.Close() if res.Request != req { From b6fecf7f89d659d6bcaac8e605dd93e6b50bcbc0 Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Fri, 7 Apr 2023 01:19:21 +0000 Subject: [PATCH 42/86] net/http: expose "http: server gave HTTP response to HTTPS client" error Expose "http: server gave HTTP response to HTTPS client" error as `ErrSchemeMismatch`, so that it can be compared with `errors.Is` . Fixes #44855 Change-Id: If96e0d000fdef641fea407310faf9e1c4f7ad0f0 GitHub-Last-Rev: 22879fc88367d77817d7d96c9164f22e55f3a192 GitHub-Pull-Request: golang/go#50939 Reviewed-on: https://go-review.googlesource.com/c/go/+/382117 Run-TryBot: Damien Neil Auto-Submit: Damien Neil Reviewed-by: Damien Neil TryBot-Result: Gopher Robot Reviewed-by: Michael Knyszek --- client.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/client.go b/client.go index 1e300acf..2cab53a5 100644 --- a/client.go +++ b/client.go @@ -204,6 +204,9 @@ func (c *Client) transport() RoundTripper { return DefaultTransport } +// ErrSchemeMismatch is returned when a server returns an HTTP response to an HTTPS client. +var ErrSchemeMismatch = errors.New("http: server gave HTTP response to HTTPS client") + // send issues an HTTP request. // Caller should close resp.Body when done reading from it. func send(ireq *Request, rt RoundTripper, deadline time.Time) (resp *Response, didTimeout func() bool, err error) { @@ -265,7 +268,7 @@ func send(ireq *Request, rt RoundTripper, deadline time.Time) (resp *Response, d // response looks like HTTP and give a more helpful error. // See golang.org/issue/11111. if string(tlsErr.RecordHeader[:]) == "HTTP/" { - err = errors.New("http: server gave HTTP response to HTTPS client") + err = ErrSchemeMismatch } } return nil, didTimeout, err From b08fb835e261bb2875e160d015df03e1893eed34 Mon Sep 17 00:00:00 2001 From: Johan Brandhorst-Satzkorn Date: Sat, 25 Mar 2023 16:38:37 -0700 Subject: [PATCH 43/86] net/http, net/internal, net/smtp: add wasip1 For #58141 Co-authored-by: Richard Musiol Co-authored-by: Achille Roussel Co-authored-by: Julien Fabre Co-authored-by: Evan Phoenix Change-Id: Ib49b7ccabe18de544455f7d09c7d715d6564a73d Reviewed-on: https://go-review.googlesource.com/c/go/+/479625 Run-TryBot: Johan Brandhorst-Satzkorn Auto-Submit: Johan Brandhorst-Satzkorn Reviewed-by: Cherry Mui Reviewed-by: Ian Lance Taylor TryBot-Result: Gopher Robot --- roundtrip.go | 2 +- transport_default_other.go | 3 +-- transport_default_js.go => transport_default_wasm.go | 3 +-- 3 files changed, 3 insertions(+), 5 deletions(-) rename transport_default_js.go => transport_default_wasm.go (89%) diff --git a/roundtrip.go b/roundtrip.go index c4c5d3b6..49ea1a71 100644 --- a/roundtrip.go +++ b/roundtrip.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build !js || !wasm +//go:build !js package http diff --git a/transport_default_other.go b/transport_default_other.go index 8a2f1cc4..4f6c5c12 100644 --- a/transport_default_other.go +++ b/transport_default_other.go @@ -2,8 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build !(js && wasm) -// +build !js !wasm +//go:build !wasm package http diff --git a/transport_default_js.go b/transport_default_wasm.go similarity index 89% rename from transport_default_js.go rename to transport_default_wasm.go index c07d35ef..3946812d 100644 --- a/transport_default_js.go +++ b/transport_default_wasm.go @@ -2,8 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build js && wasm -// +build js,wasm +//go:build (js && wasm) || wasip1 package http From b63b058f393d5784754d696297b17620148baa59 Mon Sep 17 00:00:00 2001 From: Tobias Klauser Date: Wed, 12 Apr 2023 00:28:06 +0200 Subject: [PATCH 44/86] all: update vendored golang.org/x/net Pull in CL 483375. This also updates golang.org/x/sys to v0.7.0 and thus we also need to update it to that version in cmd to keep TestDependencyVersionsConsistent happy. Fixes #22927 Change-Id: Ice14cd66a5c2a621b373c3d29455c75494436045 Reviewed-on: https://go-review.googlesource.com/c/go/+/483595 Auto-Submit: Tobias Klauser Reviewed-by: Benny Siegert Run-TryBot: Tobias Klauser Reviewed-by: Ian Lance Taylor Reviewed-by: Bryan Mills TryBot-Result: Gopher Robot --- h2_bundle.go | 24 ++++++++++++------------ socks_bundle.go | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/h2_bundle.go b/h2_bundle.go index b451cee9..8ec90cda 100644 --- a/h2_bundle.go +++ b/h2_bundle.go @@ -3777,13 +3777,9 @@ func (p *http2pipe) Write(d []byte) (n int, err error) { p.c.L = &p.mu } defer p.c.Signal() - if p.err != nil { + if p.err != nil || p.breakErr != nil { return 0, http2errClosedPipeWrite } - if p.breakErr != nil { - p.unread += len(d) - return len(d), nil // discard when there is no reader - } return p.b.Write(d) } @@ -5643,15 +5639,18 @@ func (sc *http2serverConn) processData(f *http2DataFrame) error { } if len(data) > 0 { + st.bodyBytes += int64(len(data)) wrote, err := st.body.Write(data) if err != nil { + // The handler has closed the request body. + // Return the connection-level flow control for the discarded data, + // but not the stream-level flow control. sc.sendWindowUpdate(nil, int(f.Length)-wrote) - return sc.countError("body_write_err", http2streamError(id, http2ErrCodeStreamClosed)) + return nil } if wrote != len(data) { panic("internal error: bad Writer") } - st.bodyBytes += int64(len(data)) } // Return any padded flow control now, since we won't @@ -7582,10 +7581,11 @@ func (t *http2Transport) RoundTripOpt(req *Request, opt http2RoundTripOpt) (*Res http2traceGotConn(req, cc, reused) res, err := cc.RoundTrip(req) if err != nil && retry <= 6 { + roundTripErr := err if req, err = http2shouldRetryRequest(req, err); err == nil { // After the first retry, do exponential backoff with 10% jitter. if retry == 0 { - t.vlogf("RoundTrip retrying after failure: %v", err) + t.vlogf("RoundTrip retrying after failure: %v", roundTripErr) continue } backoff := float64(uint(1) << (uint(retry) - 1)) @@ -7594,7 +7594,7 @@ func (t *http2Transport) RoundTripOpt(req *Request, opt http2RoundTripOpt) (*Res timer := http2backoffNewTimer(d) select { case <-timer.C: - t.vlogf("RoundTrip retrying after failure: %v", err) + t.vlogf("RoundTrip retrying after failure: %v", roundTripErr) continue case <-req.Context().Done(): timer.Stop() @@ -9577,6 +9577,9 @@ func (b http2transportResponseBody) Close() error { cs := b.cs cc := cs.cc + cs.bufPipe.BreakWithError(http2errClosedResponseBody) + cs.abortStream(http2errClosedResponseBody) + unread := cs.bufPipe.Len() if unread > 0 { cc.mu.Lock() @@ -9595,9 +9598,6 @@ func (b http2transportResponseBody) Close() error { cc.wmu.Unlock() } - cs.bufPipe.BreakWithError(http2errClosedResponseBody) - cs.abortStream(http2errClosedResponseBody) - select { case <-cs.donec: case <-cs.ctx.Done(): diff --git a/socks_bundle.go b/socks_bundle.go index e4466695..776b03d9 100644 --- a/socks_bundle.go +++ b/socks_bundle.go @@ -445,7 +445,7 @@ func (up *socksUsernamePassword) Authenticate(ctx context.Context, rw io.ReadWri case socksAuthMethodNotRequired: return nil case socksAuthMethodUsernamePassword: - if len(up.Username) == 0 || len(up.Username) > 255 || len(up.Password) == 0 || len(up.Password) > 255 { + if len(up.Username) == 0 || len(up.Username) > 255 || len(up.Password) > 255 { return errors.New("invalid username/password") } b := []byte{socksauthUsernamePasswordVersion} From 187b354e78c5d2eb3673b81b2a023097e2a5f75d Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Wed, 12 Apr 2023 14:45:44 +0000 Subject: [PATCH 45/86] net/http: only report the first leak of each test run We don't have a way to terminate the leaked goroutines, and we can't wait forever for them to exit (or else we would risk timing out the test and losing the log line describing what exactly leaked). So we have reason to believe that they will remain leaked while we run the next test, and we don't want the goroutines from the first leak to generate a spurious error when the second test completes. This also removes a racy Parallel call I added in CL 476036, which was flagged by the race detector in the duplicate-suppression check. (I hadn't considered the potential interaction with the leak checker.) For #59526. Updates #56421. Change-Id: Ib1f759f102fb41ece114401680cd728343e58545 Reviewed-on: https://go-review.googlesource.com/c/go/+/483896 TryBot-Result: Gopher Robot Reviewed-by: Michael Pratt Auto-Submit: Bryan Mills Run-TryBot: Bryan Mills --- main_test.go | 20 ++++++++++++++++++++ serve_test.go | 3 --- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/main_test.go b/main_test.go index 1e83ca3c..ff56ef88 100644 --- a/main_test.go +++ b/main_test.go @@ -108,11 +108,30 @@ func runningBenchmarks() bool { return false } +var leakReported bool + func afterTest(t testing.TB) { http.DefaultTransport.(*http.Transport).CloseIdleConnections() if testing.Short() { return } + if leakReported { + // To avoid confusion, only report the first leak of each test run. + // After the first leak has been reported, we can't tell whether the leaked + // goroutines are a new leak from a subsequent test or just the same + // goroutines from the first leak still hanging around, and we may add a lot + // of latency waiting for them to exit at the end of each test. + return + } + + // We shouldn't be running the leak check for parallel tests, because we might + // report the goroutines from a test that is still running as a leak from a + // completely separate test that has just finished. So we use non-atomic loads + // and stores for the leakReported variable, and store every time we start a + // leak check so that the race detector will flag concurrent leak checks as a + // race even if we don't detect any leaks. + leakReported = true + var bad string badSubstring := map[string]string{ ").readLoop(": "a Transport", @@ -132,6 +151,7 @@ func afterTest(t testing.TB) { } } if bad == "" { + leakReported = false return } // Bad stuff found, but goroutines might just still be diff --git a/serve_test.go b/serve_test.go index 88184bcf..164b1828 100644 --- a/serve_test.go +++ b/serve_test.go @@ -5519,9 +5519,6 @@ func testServerShutdownStateNew(t *testing.T, mode testMode) { if testing.Short() { t.Skip("test takes 5-6 seconds; skipping in short mode") } - // The run helper runs the test in parallel only in short mode by default. - // Since this test has a very long latency, always run it in parallel. - t.Parallel() var connAccepted sync.WaitGroup ts := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { From 77d813a6ee151d184ff0d4324b214d5f43d70099 Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Wed, 12 Apr 2023 13:58:54 +0000 Subject: [PATCH 46/86] net/http: avoid leaking writer goroutines in tests In TestTransportPrefersResponseOverWriteError and TestMaxBytesHandler, the server may respond to an incoming request without ever reading the request body. The client's Do method will return as soon as the server's response headers are read, but the Transport will remain active until it notices that the server has closed the connection, which may be arbitrarily later. When the server has closed the connection, it will call the Close method on the request body (if it has such a method). So we can use that method to find out when the Transport is close enough to done for the test to complete without interfering too much with other tests. For #57612. For #59526. Change-Id: Iddc7a3b7b09429113ad76ccc1c090ebc9e1835a1 Reviewed-on: https://go-review.googlesource.com/c/go/+/483895 Run-TryBot: Bryan Mills Commit-Queue: Bryan Mills Auto-Submit: Bryan Mills TryBot-Result: Gopher Robot Reviewed-by: Michael Pratt --- serve_test.go | 25 +++++++++++++++++++++++-- transport_test.go | 34 +++++++++++++++++++++++++++++++++- 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/serve_test.go b/serve_test.go index 164b1828..9b8496e7 100644 --- a/serve_test.go +++ b/serve_test.go @@ -6546,9 +6546,30 @@ func testMaxBytesHandler(t *testing.T, mode testMode, maxSize, requestSize int64 defer ts.Close() c := ts.Client() + + body := strings.Repeat("a", int(requestSize)) + var wg sync.WaitGroup + defer wg.Wait() + getBody := func() (io.ReadCloser, error) { + wg.Add(1) + body := &wgReadCloser{ + Reader: strings.NewReader(body), + wg: &wg, + } + return body, nil + } + reqBody, _ := getBody() + req, err := NewRequest("POST", ts.URL, reqBody) + if err != nil { + reqBody.Close() + t.Fatal(err) + } + req.ContentLength = int64(len(body)) + req.GetBody = getBody + req.Header.Set("Content-Type", "text/plain") + var buf strings.Builder - body := strings.NewReader(strings.Repeat("a", int(requestSize))) - res, err := c.Post(ts.URL, "text/plain", body) + res, err := c.Do(req) if err != nil { t.Errorf("unexpected connection error: %v", err) } else { diff --git a/transport_test.go b/transport_test.go index 6f57629e..f9e8a285 100644 --- a/transport_test.go +++ b/transport_test.go @@ -4250,6 +4250,21 @@ func testTransportFlushesRequestHeader(t *testing.T, mode testMode) { <-gotRes } +type wgReadCloser struct { + io.Reader + wg *sync.WaitGroup + closed bool +} + +func (c *wgReadCloser) Close() error { + if c.closed { + return net.ErrClosed + } + c.closed = true + c.wg.Done() + return nil +} + // Issue 11745. func TestTransportPrefersResponseOverWriteError(t *testing.T) { run(t, testTransportPrefersResponseOverWriteError) @@ -4271,12 +4286,29 @@ func testTransportPrefersResponseOverWriteError(t *testing.T, mode testMode) { fail := 0 count := 100 + bigBody := strings.Repeat("a", contentLengthLimit*2) + var wg sync.WaitGroup + defer wg.Wait() + getBody := func() (io.ReadCloser, error) { + wg.Add(1) + body := &wgReadCloser{ + Reader: strings.NewReader(bigBody), + wg: &wg, + } + return body, nil + } + for i := 0; i < count; i++ { - req, err := NewRequest("PUT", ts.URL, strings.NewReader(bigBody)) + reqBody, _ := getBody() + req, err := NewRequest("PUT", ts.URL, reqBody) if err != nil { + reqBody.Close() t.Fatal(err) } + req.ContentLength = int64(len(bigBody)) + req.GetBody = getBody + resp, err := c.Do(req) if err != nil { fail++ From 117d4b5f1e9c129ecc63fc41c917e8a352349ebe Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Wed, 26 Apr 2023 15:38:08 -0400 Subject: [PATCH 47/86] net/http/cgi: propagate LD_LIBRARY_PATH on Android Android is functionally a variant on linux, and should be treated as such. Change-Id: I08056f00bf98c1935c8cc3c859a6c72fe1a48efa Reviewed-on: https://go-review.googlesource.com/c/go/+/489395 Run-TryBot: Bryan Mills TryBot-Result: Gopher Robot Reviewed-by: Ian Lance Taylor --- cgi/host.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cgi/host.go b/cgi/host.go index 349dda15..073952a7 100644 --- a/cgi/host.go +++ b/cgi/host.go @@ -39,7 +39,7 @@ var osDefaultInheritEnv = func() []string { switch runtime.GOOS { case "darwin", "ios": return []string{"DYLD_LIBRARY_PATH"} - case "linux", "freebsd", "netbsd", "openbsd": + case "android", "linux", "freebsd", "netbsd", "openbsd": return []string{"LD_LIBRARY_PATH"} case "hpux": return []string{"LD_LIBRARY_PATH", "SHLIB_PATH"} From 1d0595ae2718cbd7fc87c3cf73737324e9b81a78 Mon Sep 17 00:00:00 2001 From: Michael Fraenkel Date: Fri, 28 Apr 2023 07:14:08 -0600 Subject: [PATCH 48/86] net/http: avoid leaking the writing goroutine The test will wait for all goroutines. A race can occur if the writing goroutine uses the Log after the test exits. For #58264 For #59883 For #59884 Change-Id: I9b8ec7c9d024ff74b922b69efa438be5a4fa3483 Reviewed-on: https://go-review.googlesource.com/c/go/+/490255 Reviewed-by: Damien Neil Reviewed-by: Bryan Mills Run-TryBot: Damien Neil TryBot-Result: Gopher Robot Auto-Submit: Bryan Mills --- serve_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/serve_test.go b/serve_test.go index 9b8496e7..81915265 100644 --- a/serve_test.go +++ b/serve_test.go @@ -1744,7 +1744,13 @@ func testServerExpect(t *testing.T, mode testMode) { // that doesn't send 100-continue expectations. writeBody := test.contentLength != 0 && strings.ToLower(test.expectation) != "100-continue" + wg := sync.WaitGroup{} + wg.Add(1) + defer wg.Wait() + go func() { + defer wg.Done() + contentLen := fmt.Sprintf("Content-Length: %d", test.contentLength) if test.chunked { contentLen = "Transfer-Encoding: chunked" From dc1ac38029bbd161cd5d41c72450546b0077411e Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Mon, 1 May 2023 17:38:08 -0700 Subject: [PATCH 49/86] all: add String for fs.{FileInfo,DirEntry} implementations The new String methods use the new FormatFileInfo and FormatDirEntry functions. Fixes #54451 Change-Id: I414cdfc212ec3c316fb2734756d2117842a23631 Reviewed-on: https://go-review.googlesource.com/c/go/+/491175 Reviewed-by: Joseph Tsai Run-TryBot: Ian Lance Taylor TryBot-Result: Gopher Robot Reviewed-by: Ian Lance Taylor Run-TryBot: Ian Lance Taylor Auto-Submit: Ian Lance Taylor Reviewed-by: Bryan Mills --- fs_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fs_test.go b/fs_test.go index 3f0f864b..e37e0f04 100644 --- a/fs_test.go +++ b/fs_test.go @@ -768,6 +768,10 @@ func (f *fakeFileInfo) Mode() fs.FileMode { return 0644 } +func (f *fakeFileInfo) String() string { + return fs.FormatFileInfo(f) +} + type fakeFile struct { io.ReadSeeker fi *fakeFileInfo From 201fe0ae8cc16ecad86c293ef88359a7f19691eb Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Tue, 9 May 2023 09:00:02 -0400 Subject: [PATCH 50/86] net/http: regenerate h2_bundle.go The x/net version was updated in CL 493596; cmd/internal/moddeps catches the skew, but only runs on the -longtest builders (because it requires network access for the bundle tool and x/net dependency). Change-Id: I48891d51aab23b2ca6f4484215438c60bd8c8c21 Reviewed-on: https://go-review.googlesource.com/c/go/+/493875 TryBot-Result: Gopher Robot Run-TryBot: Bryan Mills Auto-Submit: Bryan Mills Reviewed-by: Alan Donovan --- h2_bundle.go | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/h2_bundle.go b/h2_bundle.go index 8ec90cda..ed8d53ab 100644 --- a/h2_bundle.go +++ b/h2_bundle.go @@ -8287,6 +8287,27 @@ func (cc *http2ClientConn) RoundTrip(req *Request) (*Response, error) { return res, nil } + cancelRequest := func(cs *http2clientStream, err error) error { + cs.cc.mu.Lock() + defer cs.cc.mu.Unlock() + cs.abortStreamLocked(err) + if cs.ID != 0 { + // This request may have failed because of a problem with the connection, + // or for some unrelated reason. (For example, the user might have canceled + // the request without waiting for a response.) Mark the connection as + // not reusable, since trying to reuse a dead connection is worse than + // unnecessarily creating a new one. + // + // If cs.ID is 0, then the request was never allocated a stream ID and + // whatever went wrong was unrelated to the connection. We might have + // timed out waiting for a stream slot when StrictMaxConcurrentStreams + // is set, for example, in which case retrying on a different connection + // will not help. + cs.cc.doNotReuse = true + } + return err + } + for { select { case <-cs.respHeaderRecv: @@ -8301,15 +8322,12 @@ func (cc *http2ClientConn) RoundTrip(req *Request) (*Response, error) { return handleResponseHeaders() default: waitDone() - return nil, cs.abortErr + return nil, cancelRequest(cs, cs.abortErr) } case <-ctx.Done(): - err := ctx.Err() - cs.abortStream(err) - return nil, err + return nil, cancelRequest(cs, ctx.Err()) case <-cs.reqCancel: - cs.abortStream(http2errRequestCanceled) - return nil, http2errRequestCanceled + return nil, cancelRequest(cs, http2errRequestCanceled) } } } From 4dd2af491a77479783adf3a643eebb02e6dc937e Mon Sep 17 00:00:00 2001 From: cui fliter Date: Sat, 6 May 2023 00:11:33 +0800 Subject: [PATCH 51/86] all: fix a lot of comments Fix comments, including duplicate is, wrong phrases and articles, misspellings, etc. Change-Id: I8bfea53b9b275e649757cc4bee6a8a026ed9c7a4 Reviewed-on: https://go-review.googlesource.com/c/go/+/493035 Reviewed-by: Benny Siegert Reviewed-by: Ian Lance Taylor Run-TryBot: shuang cui TryBot-Result: Gopher Robot Run-TryBot: Ian Lance Taylor Auto-Submit: Ian Lance Taylor --- cookiejar/jar_test.go | 4 ++-- request_test.go | 4 ++-- transport_test.go | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cookiejar/jar_test.go b/cookiejar/jar_test.go index 13d994aa..56d0695a 100644 --- a/cookiejar/jar_test.go +++ b/cookiejar/jar_test.go @@ -349,7 +349,7 @@ func expiresIn(delta int) string { return "expires=" + t.Format(time.RFC1123) } -// mustParseURL parses s to an URL and panics on error. +// mustParseURL parses s to a URL and panics on error. func mustParseURL(s string) *url.URL { u, err := url.Parse(s) if err != nil || u.Scheme == "" || u.Host == "" { @@ -670,7 +670,7 @@ var updateAndDeleteTests = [...]jarTest{ }, }, { - "Clear Secure flag from a http.", + "Clear Secure flag from an http.", "http://www.host.test/", []string{ "b=xx", diff --git a/request_test.go b/request_test.go index 23e49d6b..76c8790f 100644 --- a/request_test.go +++ b/request_test.go @@ -31,7 +31,7 @@ func TestQuery(t *testing.T) { } } -// Issue #25192: Test that ParseForm fails but still parses the form when an URL +// Issue #25192: Test that ParseForm fails but still parses the form when a URL // containing a semicolon is provided. func TestParseFormSemicolonSeparator(t *testing.T) { for _, method := range []string{"POST", "PATCH", "PUT", "GET"} { @@ -379,7 +379,7 @@ func TestMultipartRequest(t *testing.T) { } // Issue #25192: Test that ParseMultipartForm fails but still parses the -// multi-part form when an URL containing a semicolon is provided. +// multi-part form when a URL containing a semicolon is provided. func TestParseMultipartFormSemicolonSeparator(t *testing.T) { req := newTestMultipartRequest(t) req.URL = &url.URL{RawQuery: "q=foo;q=bar"} diff --git a/transport_test.go b/transport_test.go index f9e8a285..fdbc5daa 100644 --- a/transport_test.go +++ b/transport_test.go @@ -2353,7 +2353,7 @@ func testTransportResponseHeaderTimeout(t *testing.T, mode testMode) { if err != nil { uerr, ok := err.(*url.Error) if !ok { - t.Errorf("error is not an url.Error; got: %#v", err) + t.Errorf("error is not a url.Error; got: %#v", err) continue } nerr, ok := uerr.Err.(net.Error) @@ -3889,7 +3889,7 @@ func TestTransportCloseIdleConnsThenReturn(t *testing.T) { } // Test for issue 34282 -// Ensure that getConn doesn't call the GotConn trace hook on a HTTP/2 idle conn +// Ensure that getConn doesn't call the GotConn trace hook on an HTTP/2 idle conn func TestTransportTraceGotConnH2IdleConns(t *testing.T) { tr := &Transport{} wantIdle := func(when string, n int) bool { @@ -4952,7 +4952,7 @@ func TestTransportRejectsAlphaPort(t *testing.T) { } } -// Test the httptrace.TLSHandshake{Start,Done} hooks with a https http1 +// Test the httptrace.TLSHandshake{Start,Done} hooks with an https http1 // connections. The http2 test is done in TestTransportEventTrace_h2 func TestTLSHandshakeTrace(t *testing.T) { run(t, testTLSHandshakeTrace, []testMode{https1Mode, http2Mode}) From 61374c74d6e10f083e7ad8b76e6cdcb8b1b00697 Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Wed, 10 May 2023 12:47:06 -0700 Subject: [PATCH 52/86] net/http: let ErrNotSupported match errors.ErrUnsupported For #41198 Change-Id: Ibb030e94618a1f594cfd98ddea214ad7a88d2e73 Reviewed-on: https://go-review.googlesource.com/c/go/+/494122 Auto-Submit: Ian Lance Taylor Reviewed-by: Damien Neil Run-TryBot: Ian Lance Taylor Reviewed-by: Bryan Mills TryBot-Result: Gopher Robot --- request.go | 5 +++++ request_test.go | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/request.go b/request.go index a45c9e3d..4e919049 100644 --- a/request.go +++ b/request.go @@ -48,6 +48,11 @@ type ProtocolError struct { func (pe *ProtocolError) Error() string { return pe.ErrorString } +// Is lets http.ErrNotSupported match errors.ErrUnsupported. +func (pe *ProtocolError) Is(err error) bool { + return pe == ErrNotSupported && err == errors.ErrUnsupported +} + var ( // ErrNotSupported indicates that a feature is not supported. // diff --git a/request_test.go b/request_test.go index 76c8790f..78b968f2 100644 --- a/request_test.go +++ b/request_test.go @@ -10,6 +10,7 @@ import ( "context" "crypto/rand" "encoding/base64" + "errors" "fmt" "io" "math" @@ -1388,3 +1389,9 @@ func runFileAndServerBenchmarks(b *testing.B, mode testMode, f *os.File, n int64 b.SetBytes(n) } } + +func TestErrNotSupported(t *testing.T) { + if !errors.Is(ErrNotSupported, errors.ErrUnsupported) { + t.Error("errors.Is(ErrNotSupported, errors.ErrUnsupported) failed") + } +} From 720427665e55d9ad7bdfece61fd8c00c861f9810 Mon Sep 17 00:00:00 2001 From: Laurent Senta Date: Fri, 10 Mar 2023 12:33:59 +0000 Subject: [PATCH 53/86] net/http: do not force the Content-Length header if nilled According to the ResponseWriter documentation: To suppress automatic response headers (such as "Date"), set their value to nil. In some cases, this documentation is incorrect: chunkWriter writes a Content-Length header even if the value was set to nil. Meaning there is no way to suppress this header. This patch replaces the empty string comparison with a call to `header.has` which takes into account nil values as expected. This is similar to the way we handle the "Date" header. Change-Id: Ie10d54ab0bb7d41270bc944ff867e035fe2bd0c5 GitHub-Last-Rev: e0616dd46388a724df7c6ea821b3808ed1663cab GitHub-Pull-Request: golang/go#58578 Reviewed-on: https://go-review.googlesource.com/c/go/+/469095 Reviewed-by: Heschi Kreinick TryBot-Result: Gopher Robot Auto-Submit: Damien Neil Run-TryBot: Jorropo Reviewed-by: Damien Neil --- serve_test.go | 34 ++++++++++++++++++++++++++++++++++ server.go | 2 +- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/serve_test.go b/serve_test.go index 81915265..a21518b5 100644 --- a/serve_test.go +++ b/serve_test.go @@ -6756,3 +6756,37 @@ func testHeadBody(t *testing.T, mode testMode, chunked bool, method string) { } } } + +// TestContentLengthResponseCanBeNilled verifies that the Content-Length is set by default +// or disabled when the header is set to nil. +func TestDisableContentLength(t *testing.T) { run(t, testDisableContentLength) } +func testDisableContentLength(t *testing.T, mode testMode) { + if mode == http2Mode { + t.Skip("skipping until h2_bundle.go is updated; see https://go-review.googlesource.com/c/net/+/471535") + } + + noCL := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { + w.Header()["Content-Length"] = nil // disable the default Content-Length response + fmt.Fprintf(w, "OK") + })) + + res, err := noCL.c.Get(noCL.ts.URL) + if err != nil { + t.Error(err) + } + if got, haveCL := res.Header["Content-Length"]; haveCL { + t.Errorf("Unexpected Content-Length: %q", got) + } + + withCL := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { + fmt.Fprintf(w, "OK") + })) + + res, err = withCL.c.Get(withCL.ts.URL) + if err != nil { + t.Error(err) + } + if got := res.Header.Get("Content-Length"); got != "2" { + t.Errorf("Content-Length: %q; want 2", got) + } +} diff --git a/server.go b/server.go index 9bd381ff..e82669a1 100644 --- a/server.go +++ b/server.go @@ -1317,7 +1317,7 @@ func (cw *chunkWriter) writeHeader(p []byte) { // send a Content-Length header. // Further, we don't send an automatic Content-Length if they // set a Transfer-Encoding, because they're generally incompatible. - if w.handlerDone.Load() && !trailers && !hasTE && bodyAllowedForStatus(w.status) && header.get("Content-Length") == "" && (!isHEAD || len(p) > 0) { + if w.handlerDone.Load() && !trailers && !hasTE && bodyAllowedForStatus(w.status) && !header.has("Content-Length") && (!isHEAD || len(p) > 0) { w.contentLength = int64(len(p)) setHeader.contentLength = strconv.AppendInt(cw.res.clenBuf[:0], int64(len(p)), 10) } From d12298971152cdf7730b2129faeffb7c30034b94 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Tue, 18 Apr 2023 08:50:02 -0700 Subject: [PATCH 54/86] net/http: handle WriteHeader(101) as a non-informational header Prior to Go 1.19 adding support for sending 1xx informational headers with ResponseWriter.WriteHeader, WriteHeader(101) would send a 101 status and disable further writes to the response. This behavior was not documented, but is intentional: Writing to the response body explicitly checks to see if a 101 status has been sent before writing. Restore the pre-1.19 behavior when writing a 101 Switching Protocols header: The header is sent, no subsequent headers are sent, and subsequent writes to the response body fail. For #59564 Change-Id: I72c116f88405b1ef5067b510f8c7cff0b36951ee Reviewed-on: https://go-review.googlesource.com/c/go/+/485775 Reviewed-by: Bryan Mills Auto-Submit: Damien Neil Run-TryBot: Damien Neil TryBot-Result: Gopher Robot --- serve_test.go | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++ server.go | 7 ++++-- 2 files changed, 74 insertions(+), 2 deletions(-) diff --git a/serve_test.go b/serve_test.go index a21518b5..12f6b768 100644 --- a/serve_test.go +++ b/serve_test.go @@ -6431,6 +6431,75 @@ func testDisableKeepAliveUpgrade(t *testing.T, mode testMode) { } } +type tlogWriter struct{ t *testing.T } + +func (w tlogWriter) Write(p []byte) (int, error) { + w.t.Log(string(p)) + return len(p), nil +} + +func TestWriteHeaderSwitchingProtocols(t *testing.T) { + run(t, testWriteHeaderSwitchingProtocols, []testMode{http1Mode}) +} +func testWriteHeaderSwitchingProtocols(t *testing.T, mode testMode) { + const wantBody = "want" + const wantUpgrade = "someProto" + ts := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { + w.Header().Set("Connection", "Upgrade") + w.Header().Set("Upgrade", wantUpgrade) + w.WriteHeader(StatusSwitchingProtocols) + NewResponseController(w).Flush() + + // Writing headers or the body after sending a 101 header should fail. + w.WriteHeader(200) + if _, err := w.Write([]byte("x")); err == nil { + t.Errorf("Write to body after 101 Switching Protocols unexpectedly succeeded") + } + + c, _, err := NewResponseController(w).Hijack() + if err != nil { + t.Errorf("Hijack: %v", err) + return + } + defer c.Close() + if _, err := c.Write([]byte(wantBody)); err != nil { + t.Errorf("Write to hijacked body: %v", err) + } + }), func(ts *httptest.Server) { + // Don't spam log with warning about superfluous WriteHeader call. + ts.Config.ErrorLog = log.New(tlogWriter{t}, "log: ", 0) + }).ts + + conn, err := net.Dial("tcp", ts.Listener.Addr().String()) + if err != nil { + t.Fatalf("net.Dial: %v", err) + } + _, err = conn.Write([]byte("GET / HTTP/1.1\r\nHost: foo\r\n\r\n")) + if err != nil { + t.Fatalf("conn.Write: %v", err) + } + defer conn.Close() + + r := bufio.NewReader(conn) + res, err := ReadResponse(r, &Request{Method: "GET"}) + if err != nil { + t.Fatal("ReadResponse error:", err) + } + if res.StatusCode != StatusSwitchingProtocols { + t.Errorf("Response StatusCode=%v, want 101", res.StatusCode) + } + if got := res.Header.Get("Upgrade"); got != wantUpgrade { + t.Errorf("Response Upgrade header = %q, want %q", got, wantUpgrade) + } + body, err := io.ReadAll(r) + if err != nil { + t.Error(err) + } + if string(body) != wantBody { + t.Errorf("Response body = %q, want %q", string(body), wantBody) + } +} + func TestMuxRedirectRelative(t *testing.T) { setParallel(t) req, err := ReadRequest(bufio.NewReader(strings.NewReader("GET http://example.com HTTP/1.1\r\nHost: test\r\n\r\n"))) diff --git a/server.go b/server.go index e82669a1..680c5f68 100644 --- a/server.go +++ b/server.go @@ -1154,8 +1154,11 @@ func (w *response) WriteHeader(code int) { } checkWriteHeaderCode(code) - // Handle informational headers - if code >= 100 && code <= 199 { + // Handle informational headers. + // + // We shouldn't send any further headers after 101 Switching Protocols, + // so it takes the non-informational path. + if code >= 100 && code <= 199 && code != StatusSwitchingProtocols { // Prevent a potential race with an automatically-sent 100 Continue triggered by Request.Body.Read() if code == 100 && w.canWriteContinue.Load() { w.writeContinueMu.Lock() From 28a8291ab0eefa15f04796b83faac8d381692eca Mon Sep 17 00:00:00 2001 From: Austin Clements Date: Mon, 15 May 2023 21:19:08 +0000 Subject: [PATCH 55/86] Revert "net/http: do not force the Content-Length header if nilled" This reverts CL 469095. The newly added TestDisableContentLength is failing on all longtest builders. Change-Id: Id307df61c7bf80691d9c276e8d200eebf6d4a59c Reviewed-on: https://go-review.googlesource.com/c/go/+/495017 Auto-Submit: Austin Clements Run-TryBot: Austin Clements Reviewed-by: Damien Neil Reviewed-by: Heschi Kreinick TryBot-Result: Gopher Robot --- serve_test.go | 34 ---------------------------------- server.go | 2 +- 2 files changed, 1 insertion(+), 35 deletions(-) diff --git a/serve_test.go b/serve_test.go index 12f6b768..33a7b54d 100644 --- a/serve_test.go +++ b/serve_test.go @@ -6825,37 +6825,3 @@ func testHeadBody(t *testing.T, mode testMode, chunked bool, method string) { } } } - -// TestContentLengthResponseCanBeNilled verifies that the Content-Length is set by default -// or disabled when the header is set to nil. -func TestDisableContentLength(t *testing.T) { run(t, testDisableContentLength) } -func testDisableContentLength(t *testing.T, mode testMode) { - if mode == http2Mode { - t.Skip("skipping until h2_bundle.go is updated; see https://go-review.googlesource.com/c/net/+/471535") - } - - noCL := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { - w.Header()["Content-Length"] = nil // disable the default Content-Length response - fmt.Fprintf(w, "OK") - })) - - res, err := noCL.c.Get(noCL.ts.URL) - if err != nil { - t.Error(err) - } - if got, haveCL := res.Header["Content-Length"]; haveCL { - t.Errorf("Unexpected Content-Length: %q", got) - } - - withCL := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { - fmt.Fprintf(w, "OK") - })) - - res, err = withCL.c.Get(withCL.ts.URL) - if err != nil { - t.Error(err) - } - if got := res.Header.Get("Content-Length"); got != "2" { - t.Errorf("Content-Length: %q; want 2", got) - } -} diff --git a/server.go b/server.go index 680c5f68..efdc0317 100644 --- a/server.go +++ b/server.go @@ -1320,7 +1320,7 @@ func (cw *chunkWriter) writeHeader(p []byte) { // send a Content-Length header. // Further, we don't send an automatic Content-Length if they // set a Transfer-Encoding, because they're generally incompatible. - if w.handlerDone.Load() && !trailers && !hasTE && bodyAllowedForStatus(w.status) && !header.has("Content-Length") && (!isHEAD || len(p) > 0) { + if w.handlerDone.Load() && !trailers && !hasTE && bodyAllowedForStatus(w.status) && header.get("Content-Length") == "" && (!isHEAD || len(p) > 0) { w.contentLength = int64(len(p)) setHeader.contentLength = strconv.AppendInt(cw.res.clenBuf[:0], int64(len(p)), 10) } From a4d775ff511a2c78327ac4f32763cdd945cebd88 Mon Sep 17 00:00:00 2001 From: Sean Liao Date: Fri, 19 May 2023 19:39:29 +0100 Subject: [PATCH 56/86] net/http/pprof: document query params Fixes #59452 Change-Id: Ia0b5a03565f663190c480ef9e26309fa85ff192c Reviewed-on: https://go-review.googlesource.com/c/go/+/496144 Reviewed-by: Ian Lance Taylor TryBot-Result: Gopher Robot Reviewed-by: Dmitri Shuralyov Run-TryBot: Ian Lance Taylor Auto-Submit: Ian Lance Taylor --- pprof/pprof.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pprof/pprof.go b/pprof/pprof.go index db03af1c..385eb423 100644 --- a/pprof/pprof.go +++ b/pprof/pprof.go @@ -27,6 +27,15 @@ // If you are not using DefaultServeMux, you will have to register handlers // with the mux you are using. // +// # Parameters +// +// Parameters can be passed via GET query params: +// +// - debug=N (all profiles): response format: N = 0: binary (default), N > 0: plaintext +// - gc=N (heap profile): N > 0: run a garbage collection cycle before profiling +// - seconds=N (allocs, block, goroutine, heap, mutex, threadcreate profiles): return a delta profile +// - seconds=N (cpu (profile), trace profiles): profile for the given duration +// // # Usage examples // // Use the pprof tool to look at the heap profile: From 010cb5a488238f7624419da6ec52cffa49c2e4b4 Mon Sep 17 00:00:00 2001 From: Oleksandr Redko Date: Thu, 18 May 2023 11:12:23 +0300 Subject: [PATCH 57/86] net/http: fix spelling issues in comments and tests Change-Id: I1b90619fd073a0c41188278a50ed149b763f0fa8 Reviewed-on: https://go-review.googlesource.com/c/go/+/496135 Run-TryBot: Bryan Mills Reviewed-by: Ian Lance Taylor Reviewed-by: Bryan Mills Auto-Submit: Bryan Mills TryBot-Result: Gopher Robot --- client_test.go | 2 +- cookiejar/jar.go | 2 +- transport_test.go | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/client_test.go b/client_test.go index b8c914bf..0fe555af 100644 --- a/client_test.go +++ b/client_test.go @@ -1207,7 +1207,7 @@ func testClientTimeout(t *testing.T, mode testMode) { })) // Try to trigger a timeout after reading part of the response body. - // The initial timeout is emprically usually long enough on a decently fast + // The initial timeout is empirically usually long enough on a decently fast // machine, but if we undershoot we'll retry with exponentially longer // timeouts until the test either passes or times out completely. // This keeps the test reasonably fast in the typical case but allows it to diff --git a/cookiejar/jar.go b/cookiejar/jar.go index e1ba5971..d5731476 100644 --- a/cookiejar/jar.go +++ b/cookiejar/jar.go @@ -472,7 +472,7 @@ func (j *Jar) domainAndType(host, domain string) (string, bool, error) { // would be sent to every subdomain of bbc.co.uk. // It just doesn't make sense on IP addresses. // The other processing and validation steps in RFC 6265 just - // collaps to: + // collapse to: if host != domain { return "", false, errIllegalDomain } diff --git a/transport_test.go b/transport_test.go index fdbc5daa..172aba67 100644 --- a/transport_test.go +++ b/transport_test.go @@ -251,7 +251,7 @@ func testTransportConnectionCloseOnResponse(t *testing.T, mode testMode) { // an underlying TCP connection after making an http.Request with Request.Close set. // // It tests the behavior by making an HTTP request to a server which -// describes the source source connection it got (remote port number + +// describes the source connection it got (remote port number + // address of its net.Conn). func TestTransportConnectionCloseOnRequest(t *testing.T) { run(t, testTransportConnectionCloseOnRequest, []testMode{http1Mode}) @@ -2368,7 +2368,7 @@ func testTransportResponseHeaderTimeout(t *testing.T, mode testMode) { if !tt.wantTimeout { if !retry { // The timeout may be set too short. Retry with a longer one. - t.Logf("unexpected timout for path %q after %v; retrying with longer timeout", tt.path, timeout) + t.Logf("unexpected timeout for path %q after %v; retrying with longer timeout", tt.path, timeout) timeout *= 2 retry = true } @@ -5648,7 +5648,7 @@ func testClientTimeoutKillsConn_BeforeHeaders(t *testing.T, mode testMode) { _, err := cst.c.Get(cst.ts.URL) if err == nil { close(cancelHandler) - t.Fatal("unexpected Get succeess") + t.Fatal("unexpected Get success") } tooSlow := time.NewTimer(timeout * 10) @@ -5656,8 +5656,8 @@ func testClientTimeoutKillsConn_BeforeHeaders(t *testing.T, mode testMode) { case <-tooSlow.C: // If we didn't get into the Handler, that probably means the builder was // just slow and the Get failed in that time but never made it to the - // server. That's fine; we'll try again with a longer timout. - t.Logf("no handler seen in %v; retrying with longer timout", timeout) + // server. That's fine; we'll try again with a longer timeout. + t.Logf("no handler seen in %v; retrying with longer timeout", timeout) close(cancelHandler) cst.close() timeout *= 2 From 86e6435e870f6d295cb0b414f483f34442ebb60a Mon Sep 17 00:00:00 2001 From: Simon Kotwicz Date: Sun, 20 Feb 2022 08:24:08 +0000 Subject: [PATCH 58/86] net/http: add doc details regarding Transport retries Add doc details to Transport mentioning retries only occur if a connection has been already been used successfully. Change-Id: I37afbad50b885248e0e6cd5e799ad848bf97c86b GitHub-Last-Rev: 7c45c32aec2bd3266c525bf28ab1879acbecf193 GitHub-Pull-Request: golang/go#51273 Reviewed-on: https://go-review.googlesource.com/c/go/+/386994 Run-TryBot: Damien Neil Reviewed-by: Ian Lance Taylor TryBot-Result: Gopher Robot Reviewed-by: Damien Neil Auto-Submit: Dmitri Shuralyov --- transport.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/transport.go b/transport.go index 8de63cdb..3ed5068e 100644 --- a/transport.go +++ b/transport.go @@ -85,13 +85,13 @@ const DefaultMaxIdleConnsPerHost = 2 // ClientTrace.Got1xxResponse. // // Transport only retries a request upon encountering a network error -// if the request is idempotent and either has no body or has its -// Request.GetBody defined. HTTP requests are considered idempotent if -// they have HTTP methods GET, HEAD, OPTIONS, or TRACE; or if their -// Header map contains an "Idempotency-Key" or "X-Idempotency-Key" -// entry. If the idempotency key value is a zero-length slice, the -// request is treated as idempotent but the header is not sent on the -// wire. +// if the connection has been already been used successfully and if the +// request is idempotent and either has no body or has its Request.GetBody +// defined. HTTP requests are considered idempotent if they have HTTP methods +// GET, HEAD, OPTIONS, or TRACE; or if their Header map contains an +// "Idempotency-Key" or "X-Idempotency-Key" entry. If the idempotency key +// value is a zero-length slice, the request is treated as idempotent but the +// header is not sent on the wire. type Transport struct { idleMu sync.Mutex closeIdle bool // user has requested to close all idle conns From 933e9e586f4a072038a4fc79db117c01ddc09383 Mon Sep 17 00:00:00 2001 From: Joe Tsai Date: Mon, 20 Mar 2023 20:28:47 -0700 Subject: [PATCH 59/86] net/http/pprof: adjust URL in package doc The indentation makes the URL be treated as a code block, thus preventing automatic detection of this URL. Avoid using a code block for this. Change-Id: Ie37ae18ec0969ef2d5a6e3b92b2512dac093dbf6 Reviewed-on: https://go-review.googlesource.com/c/go/+/478015 TryBot-Result: Gopher Robot Run-TryBot: Ian Lance Taylor Reviewed-by: Cherry Mui Auto-Submit: Dmitri Shuralyov Reviewed-by: Ian Lance Taylor --- pprof/pprof.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pprof/pprof.go b/pprof/pprof.go index 385eb423..bc3225da 100644 --- a/pprof/pprof.go +++ b/pprof/pprof.go @@ -66,8 +66,7 @@ // in your browser. // // For a study of the facility in action, visit -// -// https://blog.golang.org/2011/06/profiling-go-programs.html +// https://blog.golang.org/2011/06/profiling-go-programs.html. package pprof import ( From dbd4e0ea38de5a216c0100842c72a6dc884d8f6c Mon Sep 17 00:00:00 2001 From: fangguizhen <1297394526@qq.com> Date: Tue, 3 Jan 2023 18:18:35 +0000 Subject: [PATCH 60/86] net/http: check for nil, nil return from DialContext as well as Dial Change-Id: I3b6dd9c40b3c10db2eda6a25b9d556c9c3733bbc GitHub-Last-Rev: fd9b0c4193511a75b4a0073f37aa3116db23a46f GitHub-Pull-Request: golang/go#57448 Reviewed-on: https://go-review.googlesource.com/c/go/+/458876 Reviewed-by: Javad Rajabzadeh Reviewed-by: Damien Neil Reviewed-by: Cherry Mui Run-TryBot: Dmitri Shuralyov TryBot-Result: Gopher Robot Auto-Submit: Dmitri Shuralyov --- transport.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/transport.go b/transport.go index 3ed5068e..c07352b0 100644 --- a/transport.go +++ b/transport.go @@ -1180,7 +1180,11 @@ var zeroDialer net.Dialer func (t *Transport) dial(ctx context.Context, network, addr string) (net.Conn, error) { if t.DialContext != nil { - return t.DialContext(ctx, network, addr) + c, err := t.DialContext(ctx, network, addr) + if c == nil && err == nil { + err = errors.New("net/http: Transport.DialContext hook returned (nil, nil)") + } + return c, err } if t.Dial != nil { c, err := t.Dial(network, addr) From be2a5d7ae9f14f5d958ba9218268f0071beb90e0 Mon Sep 17 00:00:00 2001 From: Jorropo Date: Tue, 16 May 2023 09:19:58 +0200 Subject: [PATCH 61/86] net/http: second do not force the Content-Length header if nilled This is a second round of CL 469095 which has been fixed after the issue discovered in the revert CL 495017. The issue was a missing res.Body.Close() in the newly added test. Change-Id: Ifd9d8458022e59f4486397443a2862d06383e990 Reviewed-on: https://go-review.googlesource.com/c/go/+/495115 Reviewed-by: Damien Neil Run-TryBot: Jorropo TryBot-Result: Gopher Robot Reviewed-by: Matthew Dempsky Auto-Submit: Dmitri Shuralyov --- serve_test.go | 40 ++++++++++++++++++++++++++++++++++++++++ server.go | 2 +- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/serve_test.go b/serve_test.go index 33a7b54d..b712f92c 100644 --- a/serve_test.go +++ b/serve_test.go @@ -6825,3 +6825,43 @@ func testHeadBody(t *testing.T, mode testMode, chunked bool, method string) { } } } + +// TestDisableContentLength verifies that the Content-Length is set by default +// or disabled when the header is set to nil. +func TestDisableContentLength(t *testing.T) { run(t, testDisableContentLength) } +func testDisableContentLength(t *testing.T, mode testMode) { + if mode == http2Mode { + t.Skip("skipping until h2_bundle.go is updated; see https://go-review.googlesource.com/c/net/+/471535") + } + + noCL := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { + w.Header()["Content-Length"] = nil // disable the default Content-Length response + fmt.Fprintf(w, "OK") + })) + + res, err := noCL.c.Get(noCL.ts.URL) + if err != nil { + t.Fatal(err) + } + if got, haveCL := res.Header["Content-Length"]; haveCL { + t.Errorf("Unexpected Content-Length: %q", got) + } + if err := res.Body.Close(); err != nil { + t.Fatal(err) + } + + withCL := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { + fmt.Fprintf(w, "OK") + })) + + res, err = withCL.c.Get(withCL.ts.URL) + if err != nil { + t.Fatal(err) + } + if got := res.Header.Get("Content-Length"); got != "2" { + t.Errorf("Content-Length: %q; want 2", got) + } + if err := res.Body.Close(); err != nil { + t.Fatal(err) + } +} diff --git a/server.go b/server.go index efdc0317..680c5f68 100644 --- a/server.go +++ b/server.go @@ -1320,7 +1320,7 @@ func (cw *chunkWriter) writeHeader(p []byte) { // send a Content-Length header. // Further, we don't send an automatic Content-Length if they // set a Transfer-Encoding, because they're generally incompatible. - if w.handlerDone.Load() && !trailers && !hasTE && bodyAllowedForStatus(w.status) && header.get("Content-Length") == "" && (!isHEAD || len(p) > 0) { + if w.handlerDone.Load() && !trailers && !hasTE && bodyAllowedForStatus(w.status) && !header.has("Content-Length") && (!isHEAD || len(p) > 0) { w.contentLength = int64(len(p)) setHeader.contentLength = strconv.AppendInt(cw.res.clenBuf[:0], int64(len(p)), 10) } From bcdf0304ae8b002c072e9468c831ead563dcc2ce Mon Sep 17 00:00:00 2001 From: Matthew Dempsky Date: Fri, 26 May 2023 14:34:56 -0700 Subject: [PATCH 62/86] all: update vendored dependencies Generated with x/build/cmd/updatestd. Updates #36905. Change-Id: Iecd76e820c5f40a0f5e013684f7e7bef4c3fd482 Reviewed-on: https://go-review.googlesource.com/c/go/+/498598 Run-TryBot: Dmitri Shuralyov Reviewed-by: Dmitri Shuralyov Reviewed-by: Dmitri Shuralyov Auto-Submit: Dmitri Shuralyov TryBot-Result: Gopher Robot --- h2_bundle.go | 142 ++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 135 insertions(+), 7 deletions(-) diff --git a/h2_bundle.go b/h2_bundle.go index ed8d53ab..ebc004a3 100644 --- a/h2_bundle.go +++ b/h2_bundle.go @@ -4253,7 +4253,7 @@ func (s *http2Server) ServeConn(c net.Conn, opts *http2ServeConnOpts) { if s.NewWriteScheduler != nil { sc.writeSched = s.NewWriteScheduler() } else { - sc.writeSched = http2NewPriorityWriteScheduler(nil) + sc.writeSched = http2newRoundRobinWriteScheduler() } // These start at the RFC-specified defaults. If there is a higher @@ -6246,7 +6246,7 @@ type http2requestBody struct { conn *http2serverConn closeOnce sync.Once // for use by Close only sawEOF bool // for use by Read only - pipe *http2pipe // non-nil if we have a HTTP entity message body + pipe *http2pipe // non-nil if we have an HTTP entity message body needsContinue bool // need to send a 100-continue } @@ -6386,7 +6386,8 @@ func (rws *http2responseWriterState) writeChunk(p []byte) (n int, err error) { clen = "" } } - if clen == "" && rws.handlerDone && http2bodyAllowedForStatus(rws.status) && (len(p) > 0 || !isHeadResp) { + _, hasContentLength := rws.snapHeader["Content-Length"] + if !hasContentLength && clen == "" && rws.handlerDone && http2bodyAllowedForStatus(rws.status) && (len(p) > 0 || !isHeadResp) { clen = strconv.Itoa(len(p)) } _, hasContentType := rws.snapHeader["Content-Type"] @@ -6591,7 +6592,7 @@ func (w *http2responseWriter) FlushError() error { err = rws.bw.Flush() } else { // The bufio.Writer won't call chunkWriter.Write - // (writeChunk with zero bytes, so we have to do it + // (writeChunk with zero bytes), so we have to do it // ourselves to force the HTTP response header and/or // final DATA frame (with END_STREAM) to be sent. _, err = http2chunkWriter{rws}.Write(nil) @@ -8289,8 +8290,8 @@ func (cc *http2ClientConn) RoundTrip(req *Request) (*Response, error) { cancelRequest := func(cs *http2clientStream, err error) error { cs.cc.mu.Lock() - defer cs.cc.mu.Unlock() cs.abortStreamLocked(err) + bodyClosed := cs.reqBodyClosed if cs.ID != 0 { // This request may have failed because of a problem with the connection, // or for some unrelated reason. (For example, the user might have canceled @@ -8305,6 +8306,23 @@ func (cc *http2ClientConn) RoundTrip(req *Request) (*Response, error) { // will not help. cs.cc.doNotReuse = true } + cs.cc.mu.Unlock() + // Wait for the request body to be closed. + // + // If nothing closed the body before now, abortStreamLocked + // will have started a goroutine to close it. + // + // Closing the body before returning avoids a race condition + // with net/http checking its readTrackingBody to see if the + // body was read from or closed. See golang/go#60041. + // + // The body is closed in a separate goroutine without the + // connection mutex held, but dropping the mutex before waiting + // will keep us from holding it indefinitely if the body + // close is slow for some reason. + if bodyClosed != nil { + <-bodyClosed + } return err } @@ -8920,7 +8938,7 @@ func (cc *http2ClientConn) encodeHeaders(req *Request, addGzipHeader bool, trail // 8.1.2.3 Request Pseudo-Header Fields // The :path pseudo-header field includes the path and query parts of the // target URI (the path-absolute production and optionally a '?' character - // followed by the query production (see Sections 3.3 and 3.4 of + // followed by the query production, see Sections 3.3 and 3.4 of // [RFC3986]). f(":authority", host) m := req.Method @@ -10728,7 +10746,8 @@ func (wr *http2FrameWriteRequest) replyToWriter(err error) { // writeQueue is used by implementations of WriteScheduler. type http2writeQueue struct { - s []http2FrameWriteRequest + s []http2FrameWriteRequest + prev, next *http2writeQueue } func (q *http2writeQueue) empty() bool { return len(q.s) == 0 } @@ -11306,3 +11325,112 @@ func (ws *http2randomWriteScheduler) Pop() (http2FrameWriteRequest, bool) { } return http2FrameWriteRequest{}, false } + +type http2roundRobinWriteScheduler struct { + // control contains control frames (SETTINGS, PING, etc.). + control http2writeQueue + + // streams maps stream ID to a queue. + streams map[uint32]*http2writeQueue + + // stream queues are stored in a circular linked list. + // head is the next stream to write, or nil if there are no streams open. + head *http2writeQueue + + // pool of empty queues for reuse. + queuePool http2writeQueuePool +} + +// newRoundRobinWriteScheduler constructs a new write scheduler. +// The round robin scheduler priorizes control frames +// like SETTINGS and PING over DATA frames. +// When there are no control frames to send, it performs a round-robin +// selection from the ready streams. +func http2newRoundRobinWriteScheduler() http2WriteScheduler { + ws := &http2roundRobinWriteScheduler{ + streams: make(map[uint32]*http2writeQueue), + } + return ws +} + +func (ws *http2roundRobinWriteScheduler) OpenStream(streamID uint32, options http2OpenStreamOptions) { + if ws.streams[streamID] != nil { + panic(fmt.Errorf("stream %d already opened", streamID)) + } + q := ws.queuePool.get() + ws.streams[streamID] = q + if ws.head == nil { + ws.head = q + q.next = q + q.prev = q + } else { + // Queues are stored in a ring. + // Insert the new stream before ws.head, putting it at the end of the list. + q.prev = ws.head.prev + q.next = ws.head + q.prev.next = q + q.next.prev = q + } +} + +func (ws *http2roundRobinWriteScheduler) CloseStream(streamID uint32) { + q := ws.streams[streamID] + if q == nil { + return + } + if q.next == q { + // This was the only open stream. + ws.head = nil + } else { + q.prev.next = q.next + q.next.prev = q.prev + if ws.head == q { + ws.head = q.next + } + } + delete(ws.streams, streamID) + ws.queuePool.put(q) +} + +func (ws *http2roundRobinWriteScheduler) AdjustStream(streamID uint32, priority http2PriorityParam) {} + +func (ws *http2roundRobinWriteScheduler) Push(wr http2FrameWriteRequest) { + if wr.isControl() { + ws.control.push(wr) + return + } + q := ws.streams[wr.StreamID()] + if q == nil { + // This is a closed stream. + // wr should not be a HEADERS or DATA frame. + // We push the request onto the control queue. + if wr.DataSize() > 0 { + panic("add DATA on non-open stream") + } + ws.control.push(wr) + return + } + q.push(wr) +} + +func (ws *http2roundRobinWriteScheduler) Pop() (http2FrameWriteRequest, bool) { + // Control and RST_STREAM frames first. + if !ws.control.empty() { + return ws.control.shift(), true + } + if ws.head == nil { + return http2FrameWriteRequest{}, false + } + q := ws.head + for { + if wr, ok := q.consume(math.MaxInt32); ok { + ws.head = q.next + return wr, true + } + q = q.next + if q == ws.head { + break + } + } + return http2FrameWriteRequest{}, false +} From 98dbdb00fe3bb35a3cc2dee27c448448c9082b13 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Wed, 7 Jun 2023 14:13:55 -0700 Subject: [PATCH 63/86] net/http: close response body in TestRequestBodyLimit Failing to close the response body before returning leaks the in-progress request past the test lifetime. Fixes #60264 Change-Id: Ic327d9f8e02e87ed656324aaa042f833d9ea18ca Reviewed-on: https://go-review.googlesource.com/c/go/+/501309 Run-TryBot: Damien Neil TryBot-Result: Gopher Robot Reviewed-by: Bryan Mills --- serve_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/serve_test.go b/serve_test.go index b712f92c..bb380cf4 100644 --- a/serve_test.go +++ b/serve_test.go @@ -3012,7 +3012,10 @@ func testRequestBodyLimit(t *testing.T, mode testMode) { // // But that's okay, since what we're really testing is that // the remote side hung up on us before we wrote too much. - _, _ = cst.c.Do(req) + resp, err := cst.c.Do(req) + if err == nil { + resp.Body.Close() + } if atomic.LoadInt64(nWritten) > limit*100 { t.Errorf("handler restricted the request body to %d bytes, but client managed to write %d", From 562482071a0b2043cc4f5bf5b16abd0f6e674da1 Mon Sep 17 00:00:00 2001 From: cui fliter Date: Fri, 5 May 2023 22:45:32 +0800 Subject: [PATCH 64/86] all: fix mismatched symbols There are some symbol mismatches in the comments, this commit attempts to fix them Change-Id: I5c9075e5218defe9233c075744d243b26ff68496 Reviewed-on: https://go-review.googlesource.com/c/go/+/492996 TryBot-Result: Gopher Robot Run-TryBot: shuang cui Reviewed-by: Michael Pratt Reviewed-by: Michael Knyszek Run-TryBot: Michael Pratt Auto-Submit: Michael Pratt --- cookiejar/jar.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cookiejar/jar.go b/cookiejar/jar.go index d5731476..273b54c8 100644 --- a/cookiejar/jar.go +++ b/cookiejar/jar.go @@ -465,7 +465,7 @@ func (j *Jar) domainAndType(host, domain string) (string, bool, error) { // dot in the domain-attribute before processing the cookie. // // Most browsers don't do that for IP addresses, only curl - // version 7.54) and IE (version 11) do not reject a + // (version 7.54) and IE (version 11) do not reject a // Set-Cookie: a=1; domain=.127.0.0.1 // This leading dot is optional and serves only as hint for // humans to indicate that a cookie with "domain=.bbc.co.uk" From a16a8cfc09810a4f892ba35a5d0c501c82a4afa6 Mon Sep 17 00:00:00 2001 From: Dmitri Shuralyov Date: Thu, 15 Jun 2023 02:08:47 -0400 Subject: [PATCH 65/86] net/http: only disable Fetch API in tests The Fetch API was meant to only be disabled in tests. Since wasm_exec.js defines a global 'process' object, it ended up being disabled anywhere that script is used. Make the heuristic stricter so that it's less likely to trigger anywhere but when testing js/wasm using Node.js. For #57613. Fixes #60808. Change-Id: Ief8def802b466ef4faad16daccefcfd72e4398b8 Reviewed-on: https://go-review.googlesource.com/c/go/+/503675 TryBot-Result: Gopher Robot Auto-Submit: Dmitri Shuralyov Reviewed-by: Johan Brandhorst-Satzkorn Run-TryBot: Dmitri Shuralyov Reviewed-by: Dmitri Shuralyov Reviewed-by: Damien Neil --- roundtrip_js.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/roundtrip_js.go b/roundtrip_js.go index f4d0b9d4..32337258 100644 --- a/roundtrip_js.go +++ b/roundtrip_js.go @@ -11,6 +11,7 @@ import ( "fmt" "io" "strconv" + "strings" "syscall/js" ) @@ -44,11 +45,15 @@ const jsFetchRedirect = "js.fetch:redirect" // the browser globals. var jsFetchMissing = js.Global().Get("fetch").IsUndefined() -// jsFetchDisabled will be true if the "process" global is present. -// We use this as an indicator that we're running in Node.js. We -// want to disable the Fetch API in Node.js because it breaks -// our wasm tests. See https://go.dev/issue/57613 for more information. -var jsFetchDisabled = !js.Global().Get("process").IsUndefined() +// jsFetchDisabled controls whether the use of Fetch API is disabled. +// It's set to true when we detect we're running in Node.js, so that +// RoundTrip ends up talking over the same fake network the HTTP servers +// currently use in various tests and examples. See go.dev/issue/57613. +// +// TODO(go.dev/issue/60810): See if it's viable to test the Fetch API +// code path. +var jsFetchDisabled = js.Global().Get("process").Type() == js.TypeObject && + strings.HasPrefix(js.Global().Get("process").Get("argv0").String(), "node") // Determine whether the JS runtime supports streaming request bodies. // Courtesy: https://developer.chrome.com/articles/fetch-streaming-requests/#feature-detection From 9b6d8e4eab5f56620e96f17da2b8d1f495b8e624 Mon Sep 17 00:00:00 2001 From: Dmitri Shuralyov Date: Thu, 15 Jun 2023 01:39:04 -0400 Subject: [PATCH 66/86] net/http: close req.Body only when it's non-nil on js The main change here is fixing the panic where it was called even when req.Body was nil. It might also work better to keep the req.Body.Close calls closer after req.Body is read, so do that too. Calling readableStreamPull.Release on a js.Func with a zero value is currently a no-op, but it seems better to avoid it anyway. Also remove readableStreamStart, readableStreamCancel while here. They were used in the initial but not final patch set of CL 458395. Fixes #60809. Change-Id: I6ff2e3b6ec2cd4b0c9c67939903e32908312db8d Reviewed-on: https://go-review.googlesource.com/c/go/+/503676 Reviewed-by: Dmitri Shuralyov Reviewed-by: Johan Brandhorst-Satzkorn Auto-Submit: Dmitri Shuralyov Reviewed-by: Bryan Mills Run-TryBot: Dmitri Shuralyov TryBot-Result: Gopher Robot --- roundtrip_js.go | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/roundtrip_js.go b/roundtrip_js.go index 32337258..2826383c 100644 --- a/roundtrip_js.go +++ b/roundtrip_js.go @@ -135,7 +135,6 @@ func (t *Transport) RoundTrip(req *Request) (*Response, error) { } opt.Set("headers", headers) - var readableStreamStart, readableStreamPull, readableStreamCancel js.Func if req.Body != nil { if !supportsPostRequestStreams() { body, err := io.ReadAll(req.Body) @@ -143,6 +142,7 @@ func (t *Transport) RoundTrip(req *Request) (*Response, error) { req.Body.Close() // RoundTrip must always close the body, including on errors. return nil, err } + req.Body.Close() if len(body) != 0 { buf := uint8Array.New(len(body)) js.CopyBytesToJS(buf, body) @@ -153,7 +153,7 @@ func (t *Transport) RoundTrip(req *Request) (*Response, error) { readableStreamCtorArg.Set("type", "bytes") readableStreamCtorArg.Set("autoAllocateChunkSize", t.writeBufferSize()) - readableStreamPull = js.FuncOf(func(this js.Value, args []js.Value) any { + readableStreamPull := js.FuncOf(func(this js.Value, args []js.Value) any { controller := args[0] byobRequest := controller.Get("byobRequest") if byobRequest.IsNull() { @@ -181,6 +181,10 @@ func (t *Transport) RoundTrip(req *Request) (*Response, error) { // Note: This a return from the pull callback of the controller and *not* RoundTrip(). return nil }) + defer func() { + readableStreamPull.Release() + req.Body.Close() + }() readableStreamCtorArg.Set("pull", readableStreamPull) opt.Set("body", js.Global().Get("ReadableStream").New(readableStreamCtorArg)) @@ -201,11 +205,6 @@ func (t *Transport) RoundTrip(req *Request) (*Response, error) { success = js.FuncOf(func(this js.Value, args []js.Value) any { success.Release() failure.Release() - readableStreamCancel.Release() - readableStreamPull.Release() - readableStreamStart.Release() - - req.Body.Close() result := args[0] header := Header{} @@ -270,11 +269,6 @@ func (t *Transport) RoundTrip(req *Request) (*Response, error) { failure = js.FuncOf(func(this js.Value, args []js.Value) any { success.Release() failure.Release() - readableStreamCancel.Release() - readableStreamPull.Release() - readableStreamStart.Release() - - req.Body.Close() err := args[0] // The error is a JS Error type From 46e3c4f764d1d3b6b9a808c55c5ec08295898d03 Mon Sep 17 00:00:00 2001 From: Al Cutter Date: Thu, 15 Jun 2023 15:23:12 +0000 Subject: [PATCH 67/86] net/http: check RemoteAddr isn't nil before dereferencing RemoteAddr can return nil in some cases, this fix prevents a panic. I chatted with @neild about this beforehand, but what's happening in our case is that a connection comes in to the HTTP server which is then immediately closed (we discovered this issue by accident using nmap). The network implementation that we're using (it happens to be gVisor via its gonet adaptor) is returning nil from RemoteAddr(), presumably as there is no remote at that point. But, ultimately, since RemoteAddr returns an interface it is always possible for it to return nil, and indeed conn.RemoteAddr in this file does exactly that if the conn is not ok. Change-Id: Ibe67ae6e30b68e2776df5ee2911bf5f1dc539641 GitHub-Last-Rev: ff3505d1d0b00ca16c68ec2a05f542978b79b170 GitHub-Pull-Request: golang/go#60823 Reviewed-on: https://go-review.googlesource.com/c/go/+/503656 Auto-Submit: Dmitri Shuralyov Reviewed-by: Damien Neil Run-TryBot: Damien Neil Reviewed-by: Bryan Mills TryBot-Result: Gopher Robot --- server.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server.go b/server.go index 680c5f68..8f63a902 100644 --- a/server.go +++ b/server.go @@ -1856,7 +1856,9 @@ func isCommonNetReadError(err error) bool { // Serve a new connection. func (c *conn) serve(ctx context.Context) { - c.remoteAddr = c.rwc.RemoteAddr().String() + if ra := c.rwc.RemoteAddr(); ra != nil { + c.remoteAddr = ra.String() + } ctx = context.WithValue(ctx, LocalAddrContextKey, c.rwc.LocalAddr()) var inFlightResponse *response defer func() { From c6163328ead9fa69227d38f6eb2e491e583cd55e Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Wed, 28 Jun 2023 13:20:08 -0700 Subject: [PATCH 68/86] net/http: validate Host header before sending Verify that the Host header we send is valid. Avoids surprising behavior such as a Host of "go.dev\r\nX-Evil:oops" adding an X-Evil header to HTTP/1 requests. Add a test, skip the test for HTTP/2. HTTP/2 is not vulnerable to header injection in the way HTTP/1 is, but x/net/http2 doesn't validate the header and will go into a retry loop when the server rejects it. CL 506995 adds the necessary validation to x/net/http2. For #60374 Change-Id: I05cb6866a9bead043101954dfded199258c6dd04 Reviewed-on: https://go-review.googlesource.com/c/go/+/506996 Reviewed-by: Tatiana Bradley TryBot-Result: Gopher Robot Run-TryBot: Damien Neil --- http_test.go | 29 ----------------------------- request.go | 47 ++++++++++------------------------------------- request_test.go | 11 ++--------- transport_test.go | 19 +++++++++++++++++++ 4 files changed, 31 insertions(+), 75 deletions(-) diff --git a/http_test.go b/http_test.go index 1c9fb33b..91bb1b26 100644 --- a/http_test.go +++ b/http_test.go @@ -48,35 +48,6 @@ func TestForeachHeaderElement(t *testing.T) { } } -func TestCleanHost(t *testing.T) { - tests := []struct { - in, want string - }{ - {"www.google.com", "www.google.com"}, - {"www.google.com foo", "www.google.com"}, - {"www.google.com/foo", "www.google.com"}, - {" first character is a space", ""}, - {"[1::6]:8080", "[1::6]:8080"}, - - // Punycode: - {"гофер.рф/foo", "xn--c1ae0ajs.xn--p1ai"}, - {"bücher.de", "xn--bcher-kva.de"}, - {"bücher.de:8080", "xn--bcher-kva.de:8080"}, - // Verify we convert to lowercase before punycode: - {"BÜCHER.de", "xn--bcher-kva.de"}, - {"BÜCHER.de:8080", "xn--bcher-kva.de:8080"}, - // Verify we normalize to NFC before punycode: - {"gophér.nfc", "xn--gophr-esa.nfc"}, // NFC input; no work needed - {"goph\u0065\u0301r.nfd", "xn--gophr-esa.nfd"}, // NFD input - } - for _, tt := range tests { - got := cleanHost(tt.in) - if tt.want != got { - t.Errorf("cleanHost(%q) = %q, want %q", tt.in, got, tt.want) - } - } -} - // Test that cmd/go doesn't link in the HTTP server. // // This catches accidental dependencies between the HTTP transport and diff --git a/request.go b/request.go index 4e919049..bd868373 100644 --- a/request.go +++ b/request.go @@ -17,7 +17,6 @@ import ( "io" "mime" "mime/multipart" - "net" "net/http/httptrace" "net/http/internal/ascii" "net/textproto" @@ -27,6 +26,7 @@ import ( "strings" "sync" + "golang.org/x/net/http/httpguts" "golang.org/x/net/idna" ) @@ -580,12 +580,19 @@ func (r *Request) write(w io.Writer, usingProxy bool, extraHeaders Header, waitF // is not given, use the host from the request URL. // // Clean the host, in case it arrives with unexpected stuff in it. - host := cleanHost(r.Host) + host := r.Host if host == "" { if r.URL == nil { return errMissingHost } - host = cleanHost(r.URL.Host) + host = r.URL.Host + } + host, err = httpguts.PunycodeHostPort(host) + if err != nil { + return err + } + if !httpguts.ValidHostHeader(host) { + return errors.New("http: invalid Host header") } // According to RFC 6874, an HTTP client, proxy, or other @@ -742,40 +749,6 @@ func idnaASCII(v string) (string, error) { return idna.Lookup.ToASCII(v) } -// cleanHost cleans up the host sent in request's Host header. -// -// It both strips anything after '/' or ' ', and puts the value -// into Punycode form, if necessary. -// -// Ideally we'd clean the Host header according to the spec: -// -// https://tools.ietf.org/html/rfc7230#section-5.4 (Host = uri-host [ ":" port ]") -// https://tools.ietf.org/html/rfc7230#section-2.7 (uri-host -> rfc3986's host) -// https://tools.ietf.org/html/rfc3986#section-3.2.2 (definition of host) -// -// But practically, what we are trying to avoid is the situation in -// issue 11206, where a malformed Host header used in the proxy context -// would create a bad request. So it is enough to just truncate at the -// first offending character. -func cleanHost(in string) string { - if i := strings.IndexAny(in, " /"); i != -1 { - in = in[:i] - } - host, port, err := net.SplitHostPort(in) - if err != nil { // input was just a host - a, err := idnaASCII(in) - if err != nil { - return in // garbage in, garbage out - } - return a - } - a, err := idnaASCII(host) - if err != nil { - return in // garbage in, garbage out - } - return net.JoinHostPort(a, port) -} - // removeZone removes IPv6 zone identifier from host. // E.g., "[fe80::1%en0]:8080" to "[fe80::1]:8080" func removeZone(host string) string { diff --git a/request_test.go b/request_test.go index 78b968f2..0892bc25 100644 --- a/request_test.go +++ b/request_test.go @@ -775,15 +775,8 @@ func TestRequestBadHost(t *testing.T) { } req.Host = "foo.com with spaces" req.URL.Host = "foo.com with spaces" - req.Write(logWrites{t, &got}) - want := []string{ - "GET /after HTTP/1.1\r\n", - "Host: foo.com\r\n", - "User-Agent: " + DefaultUserAgent + "\r\n", - "\r\n", - } - if !reflect.DeepEqual(got, want) { - t.Errorf("Writes = %q\n Want = %q", got, want) + if err := req.Write(logWrites{t, &got}); err == nil { + t.Errorf("Writing request with invalid Host: succeded, want error") } } diff --git a/transport_test.go b/transport_test.go index 172aba67..028fecc9 100644 --- a/transport_test.go +++ b/transport_test.go @@ -6731,3 +6731,22 @@ func testHandlerAbortRacesBodyRead(t *testing.T, mode testMode) { } wg.Wait() } + +func TestRequestSanitization(t *testing.T) { run(t, testRequestSanitization) } +func testRequestSanitization(t *testing.T, mode testMode) { + if mode == http2Mode { + // Remove this after updating x/net. + t.Skip("https://go.dev/issue/60374 test fails when run with HTTP/2") + } + ts := newClientServerTest(t, mode, HandlerFunc(func(rw ResponseWriter, req *Request) { + if h, ok := req.Header["X-Evil"]; ok { + t.Errorf("request has X-Evil header: %q", h) + } + })).ts + req, _ := NewRequest("GET", ts.URL, nil) + req.Host = "go.dev\r\nX-Evil:evil" + resp, _ := ts.Client().Do(req) + if resp != nil { + resp.Body.Close() + } +} From 1e530e076cfea91c7de8228dbcedb0db7aaa744b Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Tue, 11 Jul 2023 10:52:42 -0400 Subject: [PATCH 69/86] net/http/fcgi: eliminate goroutine leaks in tests Also fix a (minor) double-Close error in Serve that was exposed by the test fix. Serve accepts a net.Listener, which produces net.Conn instances. The documentation for net.Conn requires its methods to be safe for concurrent use, so most implementations likely allow Close to be called multiple times as a side effect of making it safe to call concurrently with other methods. However, the net.Conn interface is a superset of the io.Closer interface, io.Closer explicitly leaves the behavior of multiple Close calls undefined, and net.Conn does not explicitly document a stricter requirement. Perhaps more importantly, the test for the fcgi package calls unexported functions that accept an io.ReadWriteCloser (not a net.Conn), and at least one of the test-helper ReadWriteCloser implementations expects Close to be called only once. The goroutine leaks were exposed by a racy arbitrary timeout reported in #61271. Fixing the goroutine leak exposed the double-Close error: one of the leaked goroutines was blocked on reading from an unclosed pipe. Closing the pipe (to unblock the goroutine) triggered the second Close call. Fixes #61271. Change-Id: I5cfac8870e4bb4f13adeee48910d165dbd4b76fe Reviewed-on: https://go-review.googlesource.com/c/go/+/508815 Run-TryBot: Bryan Mills Reviewed-by: Ian Lance Taylor TryBot-Result: Gopher Robot --- fcgi/fcgi.go | 13 ++++++++--- fcgi/fcgi_test.go | 57 +++++++++++++++++++++++------------------------ 2 files changed, 38 insertions(+), 32 deletions(-) diff --git a/fcgi/fcgi.go b/fcgi/fcgi.go index fb822f8a..56f7d407 100644 --- a/fcgi/fcgi.go +++ b/fcgi/fcgi.go @@ -99,8 +99,10 @@ func (h *header) init(recType recType, reqId uint16, contentLength int) { // conn sends records over rwc type conn struct { - mutex sync.Mutex - rwc io.ReadWriteCloser + mutex sync.Mutex + rwc io.ReadWriteCloser + closeErr error + closed bool // to avoid allocations buf bytes.Buffer @@ -111,10 +113,15 @@ func newConn(rwc io.ReadWriteCloser) *conn { return &conn{rwc: rwc} } +// Close closes the conn if it is not already closed. func (c *conn) Close() error { c.mutex.Lock() defer c.mutex.Unlock() - return c.rwc.Close() + if !c.closed { + c.closeErr = c.rwc.Close() + c.closed = true + } + return c.closeErr } type record struct { diff --git a/fcgi/fcgi_test.go b/fcgi/fcgi_test.go index 7a344ff3..03c42242 100644 --- a/fcgi/fcgi_test.go +++ b/fcgi/fcgi_test.go @@ -241,7 +241,7 @@ func TestChildServeCleansUp(t *testing.T) { input := make([]byte, len(tt.input)) copy(input, tt.input) rc := nopWriteCloser{bytes.NewReader(input)} - done := make(chan bool) + done := make(chan struct{}) c := newChild(rc, http.HandlerFunc(func( w http.ResponseWriter, r *http.Request, @@ -252,9 +252,9 @@ func TestChildServeCleansUp(t *testing.T) { t.Errorf("Expected %#v, got %#v", tt.err, err) } // not reached if body of request isn't closed - done <- true + close(done) })) - go c.serve() + c.serve() // wait for body of request to be closed or all goroutines to block <-done } @@ -331,7 +331,7 @@ func TestChildServeReadsEnvVars(t *testing.T) { input := make([]byte, len(tt.input)) copy(input, tt.input) rc := nopWriteCloser{bytes.NewReader(input)} - done := make(chan bool) + done := make(chan struct{}) c := newChild(rc, http.HandlerFunc(func( w http.ResponseWriter, r *http.Request, @@ -343,9 +343,9 @@ func TestChildServeReadsEnvVars(t *testing.T) { } else if env[tt.envVar] != tt.expectedVal { t.Errorf("Expected %s, got %s", tt.expectedVal, env[tt.envVar]) } - done <- true + close(done) })) - go c.serve() + c.serve() <-done } } @@ -381,7 +381,7 @@ func TestResponseWriterSniffsContentType(t *testing.T) { input := make([]byte, len(streamFullRequestStdin)) copy(input, streamFullRequestStdin) rc := nopWriteCloser{bytes.NewReader(input)} - done := make(chan bool) + done := make(chan struct{}) var resp *response c := newChild(rc, http.HandlerFunc(func( w http.ResponseWriter, @@ -389,10 +389,9 @@ func TestResponseWriterSniffsContentType(t *testing.T) { ) { io.WriteString(w, tt.body) resp = w.(*response) - done <- true + close(done) })) - defer c.cleanUp() - go c.serve() + c.serve() <-done if got := resp.Header().Get("Content-Type"); got != tt.wantCT { t.Errorf("got a Content-Type of %q; expected it to start with %q", got, tt.wantCT) @@ -401,25 +400,27 @@ func TestResponseWriterSniffsContentType(t *testing.T) { } } -type signalingNopCloser struct { - io.Reader +type signalingNopWriteCloser struct { + io.ReadCloser closed chan bool } -func (*signalingNopCloser) Write(buf []byte) (int, error) { +func (*signalingNopWriteCloser) Write(buf []byte) (int, error) { return len(buf), nil } -func (rc *signalingNopCloser) Close() error { +func (rc *signalingNopWriteCloser) Close() error { close(rc.closed) - return nil + return rc.ReadCloser.Close() } // Test whether server properly closes connection when processing slow // requests func TestSlowRequest(t *testing.T) { pr, pw := io.Pipe() - go func(w io.Writer) { + + writerDone := make(chan struct{}) + go func() { for _, buf := range [][]byte{ streamBeginTypeStdin, makeRecord(typeStdin, 1, nil), @@ -427,9 +428,14 @@ func TestSlowRequest(t *testing.T) { pw.Write(buf) time.Sleep(100 * time.Millisecond) } - }(pw) - - rc := &signalingNopCloser{pr, make(chan bool)} + close(writerDone) + }() + defer func() { + <-writerDone + pw.Close() + }() + + rc := &signalingNopWriteCloser{pr, make(chan bool)} handlerDone := make(chan bool) c := newChild(rc, http.HandlerFunc(func( @@ -439,16 +445,9 @@ func TestSlowRequest(t *testing.T) { w.WriteHeader(200) close(handlerDone) })) - go c.serve() - defer c.cleanUp() - - timeout := time.After(2 * time.Second) + c.serve() <-handlerDone - select { - case <-rc.closed: - t.Log("FastCGI child closed connection") - case <-timeout: - t.Error("FastCGI child did not close socket after handling request") - } + <-rc.closed + t.Log("FastCGI child closed connection") } From 0dff642dfa19c536bac80b45052ba42ab5b8bf30 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Wed, 12 Jul 2023 14:08:12 -0400 Subject: [PATCH 70/86] all: update vendored dependencies Generated by: go install golang.org/x/tools/cmd/bundle@latest go install golang.org/x/build/cmd/updatestd@latest updatestd -goroot=$GOROOT -branch=master For #36905. For #55079. Fixes #61174 (vet checkers understanding Go language version). Fixes #61200 (slog InfoCtx -> InfoContext etc). Change-Id: I4f2c86960ce72d6df06e23da1b1297ab3ff2eecf Reviewed-on: https://go-review.googlesource.com/c/go/+/509099 Reviewed-by: Ian Lance Taylor Reviewed-by: Jonathan Amsterdam Reviewed-by: Alan Donovan TryBot-Result: Gopher Robot Reviewed-by: Heschi Kreinick Reviewed-by: Bryan Mills Reviewed-by: Damien Neil --- h2_bundle.go | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/h2_bundle.go b/h2_bundle.go index ebc004a3..dc3e099c 100644 --- a/h2_bundle.go +++ b/h2_bundle.go @@ -7540,11 +7540,14 @@ func (t *http2Transport) RoundTrip(req *Request) (*Response, error) { func http2authorityAddr(scheme string, authority string) (addr string) { host, port, err := net.SplitHostPort(authority) if err != nil { // authority didn't have a port + host = authority + port = "" + } + if port == "" { // authority's port was empty port = "443" if scheme == "http" { port = "80" } - host = authority } if a, err := idna.ToASCII(host); err == nil { host = a @@ -8290,22 +8293,7 @@ func (cc *http2ClientConn) RoundTrip(req *Request) (*Response, error) { cancelRequest := func(cs *http2clientStream, err error) error { cs.cc.mu.Lock() - cs.abortStreamLocked(err) bodyClosed := cs.reqBodyClosed - if cs.ID != 0 { - // This request may have failed because of a problem with the connection, - // or for some unrelated reason. (For example, the user might have canceled - // the request without waiting for a response.) Mark the connection as - // not reusable, since trying to reuse a dead connection is worse than - // unnecessarily creating a new one. - // - // If cs.ID is 0, then the request was never allocated a stream ID and - // whatever went wrong was unrelated to the connection. We might have - // timed out waiting for a stream slot when StrictMaxConcurrentStreams - // is set, for example, in which case retrying on a different connection - // will not help. - cs.cc.doNotReuse = true - } cs.cc.mu.Unlock() // Wait for the request body to be closed. // @@ -8340,11 +8328,14 @@ func (cc *http2ClientConn) RoundTrip(req *Request) (*Response, error) { return handleResponseHeaders() default: waitDone() - return nil, cancelRequest(cs, cs.abortErr) + return nil, cs.abortErr } case <-ctx.Done(): - return nil, cancelRequest(cs, ctx.Err()) + err := ctx.Err() + cs.abortStream(err) + return nil, cancelRequest(cs, err) case <-cs.reqCancel: + cs.abortStream(http2errRequestCanceled) return nil, cancelRequest(cs, http2errRequestCanceled) } } @@ -8902,6 +8893,9 @@ func (cc *http2ClientConn) encodeHeaders(req *Request, addGzipHeader bool, trail if err != nil { return nil, err } + if !httpguts.ValidHostHeader(host) { + return nil, errors.New("http2: invalid Host header") + } var path string if req.Method != "CONNECT" { From 269999560ddda15b10ffdd0680ba200c2d3dbb0d Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Wed, 12 Jul 2023 22:26:57 +0930 Subject: [PATCH 71/86] all: fix typos and remove repeated words Change-Id: I5f06a4ef1d827eb0fe32a8d98444142108b0d573 Reviewed-on: https://go-review.googlesource.com/c/go/+/508996 Run-TryBot: Ian Lance Taylor Auto-Submit: Keith Randall TryBot-Result: Gopher Robot Run-TryBot: Keith Randall Reviewed-by: Keith Randall Reviewed-by: Ian Lance Taylor Auto-Submit: Ian Lance Taylor Reviewed-by: Keith Randall --- h2_bundle.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/h2_bundle.go b/h2_bundle.go index dc3e099c..9c0d5920 100644 --- a/h2_bundle.go +++ b/h2_bundle.go @@ -8007,7 +8007,7 @@ func (cc *http2ClientConn) canTakeNewRequestLocked() bool { return st.canTakeNewRequest } -// tooIdleLocked reports whether this connection has been been sitting idle +// tooIdleLocked reports whether this connection has been sitting idle // for too much wall time. func (cc *http2ClientConn) tooIdleLocked() bool { // The Round(0) strips the monontonic clock reading so the From 54e00c47b1b6ad43269ec00847bacd5ed0318c0c Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Thu, 13 Jul 2023 10:55:55 -0400 Subject: [PATCH 72/86] net/http: revert stray edit to h2_bundle.go from CL 508996 h2_bundle.go is generated from x/net/http2, so it must not be edited manually. Change-Id: If76716ce8baf581321358734e5b8bef3541632ca Reviewed-on: https://go-review.googlesource.com/c/go/+/508922 Run-TryBot: Bryan Mills TryBot-Result: Gopher Robot Reviewed-by: Cherry Mui Auto-Submit: Bryan Mills --- h2_bundle.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/h2_bundle.go b/h2_bundle.go index 9c0d5920..dc3e099c 100644 --- a/h2_bundle.go +++ b/h2_bundle.go @@ -8007,7 +8007,7 @@ func (cc *http2ClientConn) canTakeNewRequestLocked() bool { return st.canTakeNewRequest } -// tooIdleLocked reports whether this connection has been sitting idle +// tooIdleLocked reports whether this connection has been been sitting idle // for too much wall time. func (cc *http2ClientConn) tooIdleLocked() bool { // The Round(0) strips the monontonic clock reading so the From 5dfebb4f844ec63ea3b61a544159533ca2112a7d Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Mon, 24 Jul 2023 16:19:49 +0000 Subject: [PATCH 73/86] [release-branch.go1.21] Revert "net/http: use Copy in ServeContent if CopyN not needed" This reverts CL 446276. Reason for revert: Causing surprising performance regression. Fixes #61530 Change-Id: Ic970f2e05d875b606ce274ea621f7e4c8c337481 Reviewed-on: https://go-review.googlesource.com/c/go/+/512615 Run-TryBot: Damien Neil Reviewed-by: Bryan Mills TryBot-Result: Gopher Robot (cherry picked from commit df0a1297899aff1c46b66523e75aa12b0ff5049f) Reviewed-on: https://go-review.googlesource.com/c/go/+/515795 Run-TryBot: Ian Lance Taylor Reviewed-by: David Chase Reviewed-by: Damien Neil --- fs.go | 9 ++------- fs_test.go | 43 ++----------------------------------------- 2 files changed, 4 insertions(+), 48 deletions(-) diff --git a/fs.go b/fs.go index 55094400..41e0b43a 100644 --- a/fs.go +++ b/fs.go @@ -349,13 +349,8 @@ func serveContent(w ResponseWriter, r *Request, name string, modtime time.Time, w.WriteHeader(code) - if r.Method != MethodHead { - if sendSize == size { - // use Copy in the non-range case to make use of WriterTo if available - io.Copy(w, sendContent) - } else { - io.CopyN(w, sendContent, sendSize) - } + if r.Method != "HEAD" { + io.CopyN(w, sendContent, sendSize) } } diff --git a/fs_test.go b/fs_test.go index e37e0f04..3fb9e012 100644 --- a/fs_test.go +++ b/fs_test.go @@ -924,7 +924,6 @@ func testServeContent(t *testing.T, mode testMode) { wantContentType string wantContentRange string wantStatus int - wantContent []byte } htmlModTime := mustStat(t, "testdata/index.html").ModTime() tests := map[string]testCase{ @@ -1140,24 +1139,6 @@ func testServeContent(t *testing.T, mode testMode) { wantStatus: 412, wantLastMod: htmlModTime.UTC().Format(TimeFormat), }, - "uses_writeTo_if_available_and_non-range": { - content: &panicOnNonWriterTo{seekWriterTo: strings.NewReader("foobar")}, - serveContentType: "text/plain; charset=utf-8", - wantContentType: "text/plain; charset=utf-8", - wantStatus: StatusOK, - wantContent: []byte("foobar"), - }, - "do_not_use_writeTo_for_range_requests": { - content: &panicOnWriterTo{ReadSeeker: strings.NewReader("foobar")}, - serveContentType: "text/plain; charset=utf-8", - reqHeader: map[string]string{ - "Range": "bytes=0-4", - }, - wantContentType: "text/plain; charset=utf-8", - wantContentRange: "bytes 0-4/6", - wantStatus: StatusPartialContent, - wantContent: []byte("fooba"), - }, } for testName, tt := range tests { var content io.ReadSeeker @@ -1171,8 +1152,7 @@ func testServeContent(t *testing.T, mode testMode) { } else { content = tt.content } - contentOut := &strings.Builder{} - for _, method := range []string{MethodGet, MethodHead} { + for _, method := range []string{"GET", "HEAD"} { //restore content in case it is consumed by previous method if content, ok := content.(*strings.Reader); ok { content.Seek(0, io.SeekStart) @@ -1198,8 +1178,7 @@ func testServeContent(t *testing.T, mode testMode) { if err != nil { t.Fatal(err) } - contentOut.Reset() - io.Copy(contentOut, res.Body) + io.Copy(io.Discard, res.Body) res.Body.Close() if res.StatusCode != tt.wantStatus { t.Errorf("test %q using %q: got status = %d; want %d", testName, method, res.StatusCode, tt.wantStatus) @@ -1213,28 +1192,10 @@ func testServeContent(t *testing.T, mode testMode) { if g, e := res.Header.Get("Last-Modified"), tt.wantLastMod; g != e { t.Errorf("test %q using %q: got last-modified = %q, want %q", testName, method, g, e) } - if g, e := contentOut.String(), tt.wantContent; e != nil && method == MethodGet && g != string(e) { - t.Errorf("test %q using %q: got unexpected content %q, want %q", testName, method, g, e) - } } } } -type seekWriterTo interface { - io.Seeker - io.WriterTo -} - -type panicOnNonWriterTo struct { - io.Reader - seekWriterTo -} - -type panicOnWriterTo struct { - io.ReadSeeker - io.WriterTo -} - // Issue 12991 func TestServerFileStatError(t *testing.T) { rec := httptest.NewRecorder() From f3e2fb3cca7f79274f6634fb7a785b465c135fca Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Wed, 19 Jul 2023 10:30:46 -0700 Subject: [PATCH 74/86] [release-branch.go1.21] net/http: permit requests with invalid Host headers Historically, the Transport has silently truncated invalid Host headers at the first '/' or ' ' character. CL 506996 changed this behavior to reject invalid Host headers entirely. Unfortunately, Docker appears to rely on the previous behavior. When sending a HTTP/1 request with an invalid Host, send an empty Host header. This is safer than truncation: If you care about the Host, then you should get the one you set; if you don't care, then an empty Host should be fine. Continue to fully validate Host headers sent to a proxy, since proxies generally can't productively forward requests without a Host. For #60374 Fixes #61431 Fixes #61904 Change-Id: If170c7dd860aa20eb58fe32990fc93af832742b6 Reviewed-on: https://go-review.googlesource.com/c/go/+/511155 TryBot-Result: Gopher Robot Reviewed-by: Roland Shoemaker Run-TryBot: Damien Neil (cherry picked from commit b9153f6ef338baee5fe02a867c8fbc83a8b29dd1) Reviewed-on: https://go-review.googlesource.com/c/go/+/518856 Auto-Submit: Dmitri Shuralyov Run-TryBot: Roland Shoemaker Reviewed-by: Russ Cox --- request.go | 23 ++++++++++++++++++++++- request_test.go | 17 ++++++++++++----- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/request.go b/request.go index bd868373..81f79566 100644 --- a/request.go +++ b/request.go @@ -591,8 +591,29 @@ func (r *Request) write(w io.Writer, usingProxy bool, extraHeaders Header, waitF if err != nil { return err } + // Validate that the Host header is a valid header in general, + // but don't validate the host itself. This is sufficient to avoid + // header or request smuggling via the Host field. + // The server can (and will, if it's a net/http server) reject + // the request if it doesn't consider the host valid. if !httpguts.ValidHostHeader(host) { - return errors.New("http: invalid Host header") + // Historically, we would truncate the Host header after '/' or ' '. + // Some users have relied on this truncation to convert a network + // address such as Unix domain socket path into a valid, ignored + // Host header (see https://go.dev/issue/61431). + // + // We don't preserve the truncation, because sending an altered + // header field opens a smuggling vector. Instead, zero out the + // Host header entirely if it isn't valid. (An empty Host is valid; + // see RFC 9112 Section 3.2.) + // + // Return an error if we're sending to a proxy, since the proxy + // probably can't do anything useful with an empty Host header. + if !usingProxy { + host = "" + } else { + return errors.New("http: invalid Host header") + } } // According to RFC 6874, an HTTP client, proxy, or other diff --git a/request_test.go b/request_test.go index 0892bc25..a32b583c 100644 --- a/request_test.go +++ b/request_test.go @@ -767,16 +767,23 @@ func TestRequestWriteBufferedWriter(t *testing.T) { } } -func TestRequestBadHost(t *testing.T) { +func TestRequestBadHostHeader(t *testing.T) { got := []string{} req, err := NewRequest("GET", "http://foo/after", nil) if err != nil { t.Fatal(err) } - req.Host = "foo.com with spaces" - req.URL.Host = "foo.com with spaces" - if err := req.Write(logWrites{t, &got}); err == nil { - t.Errorf("Writing request with invalid Host: succeded, want error") + req.Host = "foo.com\nnewline" + req.URL.Host = "foo.com\nnewline" + req.Write(logWrites{t, &got}) + want := []string{ + "GET /after HTTP/1.1\r\n", + "Host: \r\n", + "User-Agent: " + DefaultUserAgent + "\r\n", + "\r\n", + } + if !reflect.DeepEqual(got, want) { + t.Errorf("Writes = %q\n Want = %q", got, want) } } From 80280709b28a38a2ba999db35702dc756aa62737 Mon Sep 17 00:00:00 2001 From: haruyama480 Date: Fri, 25 Aug 2023 15:14:35 +0900 Subject: [PATCH 75/86] [release-branch.go1.21] net/http: revert "support streaming POST content in wasm" CL 458395 added support for streaming POST content in Wasm. Unfortunately, this breaks requests to servers that only support HTTP/1.1. Revert the change until a suitable fallback or opt-in strategy can be decided. For #61889. Fixes #62328. Change-Id: If53a77e1890132063b39abde867d34515d4ac2af Reviewed-on: https://go-review.googlesource.com/c/go/+/522955 Run-TryBot: Johan Brandhorst-Satzkorn Reviewed-by: Damien Neil TryBot-Result: Gopher Robot Reviewed-by: Bryan Mills Reviewed-by: Johan Brandhorst-Satzkorn Reviewed-on: https://go-review.googlesource.com/c/go/+/524855 Run-TryBot: Dmitri Shuralyov Commit-Queue: Dmitri Shuralyov Reviewed-by: Dmitri Shuralyov Auto-Submit: Dmitri Shuralyov --- roundtrip_js.go | 107 ++++++++---------------------------------------- 1 file changed, 18 insertions(+), 89 deletions(-) diff --git a/roundtrip_js.go b/roundtrip_js.go index 2826383c..9f9f0cb6 100644 --- a/roundtrip_js.go +++ b/roundtrip_js.go @@ -55,38 +55,6 @@ var jsFetchMissing = js.Global().Get("fetch").IsUndefined() var jsFetchDisabled = js.Global().Get("process").Type() == js.TypeObject && strings.HasPrefix(js.Global().Get("process").Get("argv0").String(), "node") -// Determine whether the JS runtime supports streaming request bodies. -// Courtesy: https://developer.chrome.com/articles/fetch-streaming-requests/#feature-detection -func supportsPostRequestStreams() bool { - requestOpt := js.Global().Get("Object").New() - requestBody := js.Global().Get("ReadableStream").New() - - requestOpt.Set("method", "POST") - requestOpt.Set("body", requestBody) - - // There is quite a dance required to define a getter if you do not have the { get property() { ... } } - // syntax available. However, it is possible: - // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Functions/get#defining_a_getter_on_existing_objects_using_defineproperty - duplexCalled := false - duplexGetterObj := js.Global().Get("Object").New() - duplexGetterFunc := js.FuncOf(func(this js.Value, args []js.Value) any { - duplexCalled = true - return "half" - }) - defer duplexGetterFunc.Release() - duplexGetterObj.Set("get", duplexGetterFunc) - js.Global().Get("Object").Call("defineProperty", requestOpt, "duplex", duplexGetterObj) - - // Slight difference here between the aforementioned example: Non-browser-based runtimes - // do not have a non-empty API Base URL (https://html.spec.whatwg.org/multipage/webappapis.html#api-base-url) - // so we have to supply a valid URL here. - requestObject := js.Global().Get("Request").New("https://www.example.org", requestOpt) - - hasContentTypeHeader := requestObject.Get("headers").Call("has", "Content-Type").Bool() - - return duplexCalled && !hasContentTypeHeader -} - // RoundTrip implements the RoundTripper interface using the WHATWG Fetch API. func (t *Transport) RoundTrip(req *Request) (*Response, error) { // The Transport has a documented contract that states that if the DialContext or @@ -136,63 +104,24 @@ func (t *Transport) RoundTrip(req *Request) (*Response, error) { opt.Set("headers", headers) if req.Body != nil { - if !supportsPostRequestStreams() { - body, err := io.ReadAll(req.Body) - if err != nil { - req.Body.Close() // RoundTrip must always close the body, including on errors. - return nil, err - } - req.Body.Close() - if len(body) != 0 { - buf := uint8Array.New(len(body)) - js.CopyBytesToJS(buf, body) - opt.Set("body", buf) - } - } else { - readableStreamCtorArg := js.Global().Get("Object").New() - readableStreamCtorArg.Set("type", "bytes") - readableStreamCtorArg.Set("autoAllocateChunkSize", t.writeBufferSize()) - - readableStreamPull := js.FuncOf(func(this js.Value, args []js.Value) any { - controller := args[0] - byobRequest := controller.Get("byobRequest") - if byobRequest.IsNull() { - controller.Call("close") - } - - byobRequestView := byobRequest.Get("view") - - bodyBuf := make([]byte, byobRequestView.Get("byteLength").Int()) - readBytes, readErr := io.ReadFull(req.Body, bodyBuf) - if readBytes > 0 { - buf := uint8Array.New(byobRequestView.Get("buffer")) - js.CopyBytesToJS(buf, bodyBuf) - byobRequest.Call("respond", readBytes) - } - - if readErr == io.EOF || readErr == io.ErrUnexpectedEOF { - controller.Call("close") - } else if readErr != nil { - readErrCauseObject := js.Global().Get("Object").New() - readErrCauseObject.Set("cause", readErr.Error()) - readErr := js.Global().Get("Error").New("io.ReadFull failed while streaming POST body", readErrCauseObject) - controller.Call("error", readErr) - } - // Note: This a return from the pull callback of the controller and *not* RoundTrip(). - return nil - }) - defer func() { - readableStreamPull.Release() - req.Body.Close() - }() - readableStreamCtorArg.Set("pull", readableStreamPull) - - opt.Set("body", js.Global().Get("ReadableStream").New(readableStreamCtorArg)) - // There is a requirement from the WHATWG fetch standard that the duplex property of - // the object given as the options argument to the fetch call be set to 'half' - // when the body property of the same options object is a ReadableStream: - // https://fetch.spec.whatwg.org/#dom-requestinit-duplex - opt.Set("duplex", "half") + // TODO(johanbrandhorst): Stream request body when possible. + // See https://bugs.chromium.org/p/chromium/issues/detail?id=688906 for Blink issue. + // See https://bugzilla.mozilla.org/show_bug.cgi?id=1387483 for Firefox issue. + // See https://github.com/web-platform-tests/wpt/issues/7693 for WHATWG tests issue. + // See https://developer.mozilla.org/en-US/docs/Web/API/Streams_API for more details on the Streams API + // and browser support. + // NOTE(haruyama480): Ensure HTTP/1 fallback exists. + // See https://go.dev/issue/61889 for discussion. + body, err := io.ReadAll(req.Body) + if err != nil { + req.Body.Close() // RoundTrip must always close the body, including on errors. + return nil, err + } + req.Body.Close() + if len(body) != 0 { + buf := uint8Array.New(len(body)) + js.CopyBytesToJS(buf, body) + opt.Set("body", buf) } } From 8dc5440965f883d156aa4587cb3187c48d74b800 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Fri, 6 Oct 2023 14:00:28 -0700 Subject: [PATCH 76/86] [release-branch.go1.21] net/http: regenerate h2_bundle.go Pull in a security fix from x/net/http2: http2: limit maximum handler goroutines to MaxConcurrentStreamso For #63417 Fixes #63427 Fixes CVE-2023-39325 Change-Id: I70626734e6d56edf508f27a5b055ddf96d806eeb Reviewed-on: https://team-review.git.corp.google.com/c/golang/go-private/+/2047402 Reviewed-by: Tatiana Bradley Run-TryBot: Damien Neil Reviewed-by: Ian Cottrell Reviewed-on: https://go-review.googlesource.com/c/go/+/534235 Auto-Submit: Dmitri Shuralyov Reviewed-by: Michael Pratt Reviewed-by: Damien Neil Reviewed-by: Dmitri Shuralyov LUCI-TryBot-Result: Go LUCI --- h2_bundle.go | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 64 insertions(+), 2 deletions(-) diff --git a/h2_bundle.go b/h2_bundle.go index dc3e099c..9cd6a349 100644 --- a/h2_bundle.go +++ b/h2_bundle.go @@ -4393,9 +4393,11 @@ type http2serverConn struct { advMaxStreams uint32 // our SETTINGS_MAX_CONCURRENT_STREAMS advertised the client curClientStreams uint32 // number of open streams initiated by the client curPushedStreams uint32 // number of open streams initiated by server push + curHandlers uint32 // number of running handler goroutines maxClientStreamID uint32 // max ever seen from client (odd), or 0 if there have been no client requests maxPushPromiseID uint32 // ID of the last push promise (even), or 0 if there have been no pushes streams map[uint32]*http2stream + unstartedHandlers []http2unstartedHandler initialStreamSendWindowSize int32 maxFrameSize int32 peerMaxHeaderListSize uint32 // zero means unknown (default) @@ -4796,6 +4798,8 @@ func (sc *http2serverConn) serve() { return case http2gracefulShutdownMsg: sc.startGracefulShutdownInternal() + case http2handlerDoneMsg: + sc.handlerDone() default: panic("unknown timer") } @@ -4843,6 +4847,7 @@ var ( http2idleTimerMsg = new(http2serverMessage) http2shutdownTimerMsg = new(http2serverMessage) http2gracefulShutdownMsg = new(http2serverMessage) + http2handlerDoneMsg = new(http2serverMessage) ) func (sc *http2serverConn) onSettingsTimer() { sc.sendServeMsg(http2settingsTimerMsg) } @@ -5842,8 +5847,7 @@ func (sc *http2serverConn) processHeaders(f *http2MetaHeadersFrame) error { } } - go sc.runHandler(rw, req, handler) - return nil + return sc.scheduleHandler(id, rw, req, handler) } func (sc *http2serverConn) upgradeRequest(req *Request) { @@ -5863,6 +5867,10 @@ func (sc *http2serverConn) upgradeRequest(req *Request) { sc.conn.SetReadDeadline(time.Time{}) } + // This is the first request on the connection, + // so start the handler directly rather than going + // through scheduleHandler. + sc.curHandlers++ go sc.runHandler(rw, req, sc.handler.ServeHTTP) } @@ -6103,8 +6111,62 @@ func (sc *http2serverConn) newResponseWriter(st *http2stream, req *Request) *htt return &http2responseWriter{rws: rws} } +type http2unstartedHandler struct { + streamID uint32 + rw *http2responseWriter + req *Request + handler func(ResponseWriter, *Request) +} + +// scheduleHandler starts a handler goroutine, +// or schedules one to start as soon as an existing handler finishes. +func (sc *http2serverConn) scheduleHandler(streamID uint32, rw *http2responseWriter, req *Request, handler func(ResponseWriter, *Request)) error { + sc.serveG.check() + maxHandlers := sc.advMaxStreams + if sc.curHandlers < maxHandlers { + sc.curHandlers++ + go sc.runHandler(rw, req, handler) + return nil + } + if len(sc.unstartedHandlers) > int(4*sc.advMaxStreams) { + return sc.countError("too_many_early_resets", http2ConnectionError(http2ErrCodeEnhanceYourCalm)) + } + sc.unstartedHandlers = append(sc.unstartedHandlers, http2unstartedHandler{ + streamID: streamID, + rw: rw, + req: req, + handler: handler, + }) + return nil +} + +func (sc *http2serverConn) handlerDone() { + sc.serveG.check() + sc.curHandlers-- + i := 0 + maxHandlers := sc.advMaxStreams + for ; i < len(sc.unstartedHandlers); i++ { + u := sc.unstartedHandlers[i] + if sc.streams[u.streamID] == nil { + // This stream was reset before its goroutine had a chance to start. + continue + } + if sc.curHandlers >= maxHandlers { + break + } + sc.curHandlers++ + go sc.runHandler(u.rw, u.req, u.handler) + sc.unstartedHandlers[i] = http2unstartedHandler{} // don't retain references + } + sc.unstartedHandlers = sc.unstartedHandlers[i:] + if len(sc.unstartedHandlers) == 0 { + sc.unstartedHandlers = nil + } +} + // Run on its own goroutine. func (sc *http2serverConn) runHandler(rw *http2responseWriter, req *Request, handler func(ResponseWriter, *Request)) { + defer sc.sendServeMsg(http2handlerDoneMsg) didPanic := true defer func() { rw.rws.stream.cancelCtx() From 999da33a5fc246b0d576be0079c489abd0f4400a Mon Sep 17 00:00:00 2001 From: Mauri de Souza Meneguzzo Date: Thu, 26 Oct 2023 01:52:57 +0000 Subject: [PATCH 77/86] [release-branch.go1.21] net/http: pull http2 underflow fix from x/net/http2 After CL 534295 was merged to fix a CVE it introduced an underflow when we try to decrement sc.curHandlers in handlerDone. Pull in a fix from x/net/http2: http2: fix underflow in http2 server push https://go-review.googlesource.com/c/net/+/535595 For #63511 Fixes #63560 Change-Id: I5c678ce7dcc53635f3ad5e4999857cb120dfc1ab GitHub-Last-Rev: 587ffa3cafbb9da6bc82ba8a5b83313f81e5c89b GitHub-Pull-Request: golang/go#63561 Reviewed-on: https://go-review.googlesource.com/c/go/+/535575 Run-TryBot: Mauri de Souza Meneguzzo Reviewed-by: Dmitri Shuralyov Reviewed-by: Dmitri Shuralyov Reviewed-by: David Chase Auto-Submit: Dmitri Shuralyov TryBot-Result: Gopher Robot (cherry picked from commit 0046c1414c4910dfe54abfcdbe18e565dd5a60f6) Reviewed-on: https://go-review.googlesource.com/c/go/+/537996 Reviewed-by: Cherry Mui LUCI-TryBot-Result: Go LUCI --- h2_bundle.go | 1 + 1 file changed, 1 insertion(+) diff --git a/h2_bundle.go b/h2_bundle.go index 9cd6a349..dd59e1f4 100644 --- a/h2_bundle.go +++ b/h2_bundle.go @@ -7012,6 +7012,7 @@ func (sc *http2serverConn) startPush(msg *http2startPushRequest) { panic(fmt.Sprintf("newWriterAndRequestNoBody(%+v): %v", msg.url, err)) } + sc.curHandlers++ go sc.runHandler(rw, req, sc.handler.ServeHTTP) return promisedID, nil } From d856809aab3a997b042569210d4acbb3cb4d0cc3 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Tue, 7 Nov 2023 10:47:56 -0800 Subject: [PATCH 78/86] [release-branch.go1.21] net/http: limit chunked data overhead The chunked transfer encoding adds some overhead to the content transferred. When writing one byte per chunk, for example, there are five bytes of overhead per byte of data transferred: "1\r\nX\r\n" to send "X". Chunks may include "chunk extensions", which we skip over and do not use. For example: "1;chunk extension here\r\nX\r\n". A malicious sender can use chunk extensions to add about 4k of overhead per byte of data. (The maximum chunk header line size we will accept.) Track the amount of overhead read in chunked data, and produce an error if it seems excessive. Updates #64433 Fixes #64435 Fixes CVE-2023-39326 Change-Id: I40f8d70eb6f9575fb43f506eb19132ccedafcf39 Reviewed-on: https://team-review.git.corp.google.com/c/golang/go-private/+/2076135 Reviewed-by: Tatiana Bradley Reviewed-by: Roland Shoemaker (cherry picked from commit 3473ae72ee66c60744665a24b2fde143e8964d4f) Reviewed-on: https://team-review.git.corp.google.com/c/golang/go-private/+/2095408 Run-TryBot: Roland Shoemaker Reviewed-by: Damien Neil Reviewed-on: https://go-review.googlesource.com/c/go/+/547356 Reviewed-by: Dmitri Shuralyov LUCI-TryBot-Result: Go LUCI --- internal/chunked.go | 34 +++++++++++++++++++---- internal/chunked_test.go | 59 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 6 deletions(-) diff --git a/internal/chunked.go b/internal/chunked.go index 5a174415..aad8e5aa 100644 --- a/internal/chunked.go +++ b/internal/chunked.go @@ -39,7 +39,8 @@ type chunkedReader struct { n uint64 // unread bytes in chunk err error buf [2]byte - checkEnd bool // whether need to check for \r\n chunk footer + checkEnd bool // whether need to check for \r\n chunk footer + excess int64 // "excessive" chunk overhead, for malicious sender detection } func (cr *chunkedReader) beginChunk() { @@ -49,10 +50,36 @@ func (cr *chunkedReader) beginChunk() { if cr.err != nil { return } + cr.excess += int64(len(line)) + 2 // header, plus \r\n after the chunk data + line = trimTrailingWhitespace(line) + line, cr.err = removeChunkExtension(line) + if cr.err != nil { + return + } cr.n, cr.err = parseHexUint(line) if cr.err != nil { return } + // A sender who sends one byte per chunk will send 5 bytes of overhead + // for every byte of data. ("1\r\nX\r\n" to send "X".) + // We want to allow this, since streaming a byte at a time can be legitimate. + // + // A sender can use chunk extensions to add arbitrary amounts of additional + // data per byte read. ("1;very long extension\r\nX\r\n" to send "X".) + // We don't want to disallow extensions (although we discard them), + // but we also don't want to allow a sender to reduce the signal/noise ratio + // arbitrarily. + // + // We track the amount of excess overhead read, + // and produce an error if it grows too large. + // + // Currently, we say that we're willing to accept 16 bytes of overhead per chunk, + // plus twice the amount of real data in the chunk. + cr.excess -= 16 + (2 * int64(cr.n)) + cr.excess = max(cr.excess, 0) + if cr.excess > 16*1024 { + cr.err = errors.New("chunked encoding contains too much non-data") + } if cr.n == 0 { cr.err = io.EOF } @@ -140,11 +167,6 @@ func readChunkLine(b *bufio.Reader) ([]byte, error) { if len(p) >= maxLineLength { return nil, ErrLineTooLong } - p = trimTrailingWhitespace(p) - p, err = removeChunkExtension(p) - if err != nil { - return nil, err - } return p, nil } diff --git a/internal/chunked_test.go b/internal/chunked_test.go index 5e29a786..b99090c1 100644 --- a/internal/chunked_test.go +++ b/internal/chunked_test.go @@ -239,3 +239,62 @@ func TestChunkEndReadError(t *testing.T) { t.Errorf("expected %v, got %v", readErr, err) } } + +func TestChunkReaderTooMuchOverhead(t *testing.T) { + // If the sender is sending 100x as many chunk header bytes as chunk data, + // we should reject the stream at some point. + chunk := []byte("1;") + for i := 0; i < 100; i++ { + chunk = append(chunk, 'a') // chunk extension + } + chunk = append(chunk, "\r\nX\r\n"...) + const bodylen = 1 << 20 + r := NewChunkedReader(&funcReader{f: func(i int) ([]byte, error) { + if i < bodylen { + return chunk, nil + } + return []byte("0\r\n"), nil + }}) + _, err := io.ReadAll(r) + if err == nil { + t.Fatalf("successfully read body with excessive overhead; want error") + } +} + +func TestChunkReaderByteAtATime(t *testing.T) { + // Sending one byte per chunk should not trip the excess-overhead detection. + const bodylen = 1 << 20 + r := NewChunkedReader(&funcReader{f: func(i int) ([]byte, error) { + if i < bodylen { + return []byte("1\r\nX\r\n"), nil + } + return []byte("0\r\n"), nil + }}) + got, err := io.ReadAll(r) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if len(got) != bodylen { + t.Errorf("read %v bytes, want %v", len(got), bodylen) + } +} + +type funcReader struct { + f func(iteration int) ([]byte, error) + i int + b []byte + err error +} + +func (r *funcReader) Read(p []byte) (n int, err error) { + if len(r.b) == 0 && r.err == nil { + r.b, r.err = r.f(r.i) + r.i++ + } + n = copy(p, r.b) + r.b = r.b[n:] + if len(r.b) > 0 { + return n, nil + } + return n, r.err +} From f22e59e834620018a47c42b1d71a2d0d5c473d99 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Thu, 11 Jan 2024 11:31:57 -0800 Subject: [PATCH 79/86] [release-branch.go1.21] net/http, net/http/cookiejar: avoid subdomain matches on IPv6 zones When deciding whether to forward cookies or sensitive headers across a redirect, do not attempt to interpret an IPv6 address as a domain name. Avoids a case where a maliciously-crafted redirect to an IPv6 address with a scoped addressing zone could be misinterpreted as a within-domain redirect. For example, we could interpret "::1%.www.example.com" as a subdomain of "www.example.com". Thanks to Juho Nurminen of Mattermost for reporting this issue. Fixes CVE-2023-45289 Fixes #65385 For #65065 Change-Id: I8f463f59f0e700c8a18733d2b264a8bcb3a19599 Reviewed-on: https://team-review.git.corp.google.com/c/golang/go-private/+/2131938 Reviewed-by: Tatiana Bradley Reviewed-by: Roland Shoemaker Reviewed-on: https://team-review.git.corp.google.com/c/golang/go-private/+/2173775 Reviewed-by: Carlos Amedee Reviewed-on: https://go-review.googlesource.com/c/go/+/569239 Reviewed-by: Carlos Amedee Auto-Submit: Michael Knyszek TryBot-Bypass: Michael Knyszek --- client.go | 6 ++++++ client_test.go | 1 + cookiejar/jar.go | 7 +++++++ cookiejar/jar_test.go | 10 ++++++++++ 4 files changed, 24 insertions(+) diff --git a/client.go b/client.go index 2cab53a5..77a701b8 100644 --- a/client.go +++ b/client.go @@ -1014,6 +1014,12 @@ func isDomainOrSubdomain(sub, parent string) bool { if sub == parent { return true } + // If sub contains a :, it's probably an IPv6 address (and is definitely not a hostname). + // Don't check the suffix in this case, to avoid matching the contents of a IPv6 zone. + // For example, "::1%.www.example.com" is not a subdomain of "www.example.com". + if strings.ContainsAny(sub, ":%") { + return false + } // If sub is "foo.example.com" and parent is "example.com", // that means sub must end in "."+parent. // Do it without allocating. diff --git a/client_test.go b/client_test.go index 0fe555af..fc1d7916 100644 --- a/client_test.go +++ b/client_test.go @@ -1725,6 +1725,7 @@ func TestShouldCopyHeaderOnRedirect(t *testing.T) { {"authorization", "http://foo.com/", "https://foo.com/", true}, {"authorization", "http://foo.com:1234/", "http://foo.com:4321/", true}, {"www-authenticate", "http://foo.com/", "http://bar.com/", false}, + {"authorization", "http://foo.com/", "http://[::1%25.foo.com]/", false}, // But subdomains should work: {"www-authenticate", "http://foo.com/", "http://foo.com/", true}, diff --git a/cookiejar/jar.go b/cookiejar/jar.go index 273b54c8..4b162660 100644 --- a/cookiejar/jar.go +++ b/cookiejar/jar.go @@ -362,6 +362,13 @@ func jarKey(host string, psl PublicSuffixList) string { // isIP reports whether host is an IP address. func isIP(host string) bool { + if strings.ContainsAny(host, ":%") { + // Probable IPv6 address. + // Hostnames can't contain : or %, so this is definitely not a valid host. + // Treating it as an IP is the more conservative option, and avoids the risk + // of interpeting ::1%.www.example.com as a subtomain of www.example.com. + return true + } return net.ParseIP(host) != nil } diff --git a/cookiejar/jar_test.go b/cookiejar/jar_test.go index 56d0695a..251f7c16 100644 --- a/cookiejar/jar_test.go +++ b/cookiejar/jar_test.go @@ -252,6 +252,7 @@ var isIPTests = map[string]bool{ "127.0.0.1": true, "1.2.3.4": true, "2001:4860:0:2001::68": true, + "::1%zone": true, "example.com": false, "1.1.1.300": false, "www.foo.bar.net": false, @@ -629,6 +630,15 @@ var basicsTests = [...]jarTest{ {"http://www.host.test:1234/", "a=1"}, }, }, + { + "IPv6 zone is not treated as a host.", + "https://example.com/", + []string{"a=1"}, + "a=1", + []query{ + {"https://[::1%25.example.com]:80/", ""}, + }, + }, } func TestBasics(t *testing.T) { From 7f0fa5fb4ac9a1231b6b1af5d437b9500b40ff71 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Wed, 27 Mar 2024 12:18:07 +0100 Subject: [PATCH 80/86] chore: prepare for upgrading to go1.21.8 --- UPSTREAM | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/UPSTREAM b/UPSTREAM index e36050d0..3e9ef2da 100644 --- a/UPSTREAM +++ b/UPSTREAM @@ -1 +1 @@ -go1.20.14 +go1.21.8 From 180c55efa8f3a62c5061f1d6b03140747a7d626b Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Wed, 27 Mar 2024 12:36:59 +0100 Subject: [PATCH 81/86] chore: use go1.21.8 in github workflow --- .github/workflows/go.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 3814f7ee..fc12031d 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -16,7 +16,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v4 with: - go-version: "1.20.14" + go-version: "1.21.8" - name: Build run: go build -v ./... From 9cae361d328bef04fa9b8f2f99573b300aeed847 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Wed, 27 Mar 2024 12:37:23 +0100 Subject: [PATCH 82/86] doc: improve merging procedure docs --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 3d18ebfa..1c622296 100644 --- a/README.md +++ b/README.md @@ -225,20 +225,20 @@ minor changes (e.g., updating docs) directly on the `main` branch. - [ ] update [UPSTREAM](UPSTREAM), commit the change, and then run the `./tools/merge.bash` script to merge from upstream; -- [ ] make sure you synch [./internal/safefilepath](./internal/safefilepath) with the -`./src/internal/safefilepath` of the Go release you're merging from; - - [ ] solve the very-likely merge conflicts and ensure [the original spirit of the patches](#patches) still hold; +- [ ] make sure you synch [./internal/safefilepath](./internal/safefilepath) with the +`./src/internal/safefilepath` of the Go release you're merging from; + - [ ] make sure the codebase does not assume `*tls.Conn` *anywhere* (`git grep -n '\*tls\.Conn'`) and otherwise replace `*tls.Conn` with `TLSConn`; - [ ] make sure the codebase does not call `tls.Client` *anywhere* except for `tlsconn.go` (`git grep -n 'tls\.Client'`) and otherwise replace `tls.Client` with `TLSClientFactory`; -- [ ] diff with upstream (`diff --color=never -ru .../golang/go/src/net/http .`) and -make sure what you see makes sense in terms of the original patches; +- [ ] diff with upstream (`./tools/compare.bash`) and make sure what you see +makes sense in terms of the original patches; - [ ] ensure `go build -v ./...` still works; From 67466024a32200fe97a9bab32e85df50b63bc44b Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Wed, 27 Mar 2024 12:37:39 +0100 Subject: [PATCH 83/86] feat(tools/compare.bash): better comparison Make sure we explain the provenance of each package inside internal and update the diff invocations to cover more packages. --- tools/compare.bash | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tools/compare.bash b/tools/compare.bash index 17d6796c..5dcb6606 100755 --- a/tools/compare.bash +++ b/tools/compare.bash @@ -9,4 +9,24 @@ test -d $upstreamrepo || git clone git@github.com:golang/go.git $upstreamrepo git pull git checkout $TAG ) + +# Classification of ./internal packages +# +# 1. packages that map directly and are captured by the following diff command +# +# - src/net/http/internal/ascii => ./internal/ascii +# +# - src/net/http/internal/testcert => ./internal/testcert + diff -ur $upstreamrepo/src/net/http . + +# 2. packages that we need to diff for explicitly +# +# - src/internal/safefilepath => ./internal/safefilepath + +diff -ur $upstreamrepo/src/internal/safefilepath ./internal/safefilepath + +# 3. replacement packages +# +# - ./internal/fakerace fakes out src/internal/race +# - ./internal/testenv fakes out src/internal/testenv From 50ad53e878f2fe34d54307e7254fb6d35b38666e Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Wed, 27 Mar 2024 12:42:08 +0100 Subject: [PATCH 84/86] fix(tools/compare.bash): show full diff While there, further refine the documentation. --- README.md | 3 ++- tools/compare.bash | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 1c622296..2e790225 100644 --- a/README.md +++ b/README.md @@ -238,7 +238,8 @@ and otherwise replace `*tls.Conn` with `TLSConn`; (`git grep -n 'tls\.Client'`) and otherwise replace `tls.Client` with `TLSClientFactory`; - [ ] diff with upstream (`./tools/compare.bash`) and make sure what you see -makes sense in terms of the original patches; +makes sense in terms of the original patches, save the diff, and include it into +the PR to document the actual changes between us and upstream. - [ ] ensure `go build -v ./...` still works; diff --git a/tools/compare.bash b/tools/compare.bash index 5dcb6606..cf48fac0 100755 --- a/tools/compare.bash +++ b/tools/compare.bash @@ -18,13 +18,13 @@ test -d $upstreamrepo || git clone git@github.com:golang/go.git $upstreamrepo # # - src/net/http/internal/testcert => ./internal/testcert -diff -ur $upstreamrepo/src/net/http . +diff -ur $upstreamrepo/src/net/http . || true # 2. packages that we need to diff for explicitly # # - src/internal/safefilepath => ./internal/safefilepath -diff -ur $upstreamrepo/src/internal/safefilepath ./internal/safefilepath +diff -ur $upstreamrepo/src/internal/safefilepath ./internal/safefilepath || true # 3. replacement packages # From c2ca4857a7daa4c105f8c0f9ae6624bd95900298 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Wed, 27 Mar 2024 12:50:03 +0100 Subject: [PATCH 85/86] fix(cgi/child_test.go): force naming imports This prevents VSCode from updating imports when saving the file. --- cgi/child_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cgi/child_test.go b/cgi/child_test.go index aa9b0010..b949bfec 100644 --- a/cgi/child_test.go +++ b/cgi/child_test.go @@ -12,8 +12,8 @@ import ( "strings" "testing" - "github.com/ooni/oohttp" - "github.com/ooni/oohttp/httptest" + http "github.com/ooni/oohttp" + httptest "github.com/ooni/oohttp/httptest" ) func TestRequest(t *testing.T) { From f09c2bb71bc6d041e4f5b5f1e4f28414f1054fbd Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Wed, 27 Mar 2024 13:07:29 +0100 Subject: [PATCH 86/86] fix(examples): run go mod tidy --- example/go.mod | 4 +++- example/go.sum | 7 +++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/example/go.mod b/example/go.mod index a7a11b9b..9b8f5b77 100644 --- a/example/go.mod +++ b/example/go.mod @@ -1,6 +1,8 @@ module github.com/ooni/oohttp/example -go 1.20 +go 1.21 + +toolchain go1.21.8 require ( github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 diff --git a/example/go.sum b/example/go.sum index 47022550..a72e40b7 100644 --- a/example/go.sum +++ b/example/go.sum @@ -19,7 +19,9 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -51,11 +53,14 @@ github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/onsi/ginkgo/v2 v2.9.5 h1:+6Hr4uxzP4XIUyAkg61dWBw8lb/gc4/X5luuxN/EC+Q= +github.com/onsi/ginkgo/v2 v2.9.5/go.mod h1:tvAoo1QUJwNEU2ITftXTpR7R1RbCzoZUOs3RonqW57k= github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= +github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= github.com/ooni/oohttp v0.6.8-0.20240322100813-2a1cdc95a941 h1:es2QWDycIahrNuEkXClWSNIs9ISSdNGVNGYSkMJv5Tc= github.com/ooni/oohttp v0.6.8-0.20240322100813-2a1cdc95a941/go.mod h1:Vipww76rE6i/Lyd+M8gec/ixPrsyPti1J8xTyqzFIHA= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -108,6 +113,7 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= +golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= @@ -138,5 +144,6 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=