diff options
| author | Lorry Tar Creator <lorry-tar-importer@lorry> | 2017-05-02 14:43:35 +0000 |
|---|---|---|
| committer | Lorry Tar Creator <lorry-tar-importer@lorry> | 2017-05-02 14:43:35 +0000 |
| commit | 34efdaf078b01a7387007c4e6bde6db86384c4b7 (patch) | |
| tree | d503eaf41d085669d1481bb46ec038bc866fece6 /libgo/go/net | |
| parent | f733cf303bcdc952c92b81dd62199a40a1f555ec (diff) | |
| download | gcc-tarball-master.tar.gz | |
gcc-7.1.0gcc-7.1.0
Diffstat (limited to 'libgo/go/net')
243 files changed, 29063 insertions, 7765 deletions
diff --git a/libgo/go/net/addrselect.go b/libgo/go/net/addrselect.go index 58ab7d706c..1ab9fc5326 100644 --- a/libgo/go/net/addrselect.go +++ b/libgo/go/net/addrselect.go @@ -188,33 +188,17 @@ func (s *byRFC6724) Less(i, j int) bool { // Rule 9: Use longest matching prefix. // When DA and DB belong to the same address family (both are IPv6 or - // both are IPv4): If CommonPrefixLen(Source(DA), DA) > + // both are IPv4 [but see below]): If CommonPrefixLen(Source(DA), DA) > // CommonPrefixLen(Source(DB), DB), then prefer DA. Similarly, if // CommonPrefixLen(Source(DA), DA) < CommonPrefixLen(Source(DB), DB), // then prefer DB. - da4 := DA.To4() != nil - db4 := DB.To4() != nil - if da4 == db4 { + // + // However, applying this rule to IPv4 addresses causes + // problems (see issues 13283 and 18518), so limit to IPv6. + if DA.To4() == nil && DB.To4() == nil { commonA := commonPrefixLen(SourceDA, DA) commonB := commonPrefixLen(SourceDB, DB) - // CommonPrefixLen doesn't really make sense for IPv4, and even - // causes problems for common load balancing practices - // (e.g., https://golang.org/issue/13283). Glibc instead only - // uses CommonPrefixLen for IPv4 when the source and destination - // addresses are on the same subnet, but that requires extra - // work to find the netmask for our source addresses. As a - // simpler heuristic, we limit its use to when the source and - // destination belong to the same special purpose block. - if da4 { - if !sameIPv4SpecialPurposeBlock(SourceDA, DA) { - commonA = 0 - } - if !sameIPv4SpecialPurposeBlock(SourceDB, DB) { - commonB = 0 - } - } - if commonA > commonB { return preferDA } @@ -404,28 +388,3 @@ func commonPrefixLen(a, b IP) (cpl int) { } return } - -// sameIPv4SpecialPurposeBlock reports whether a and b belong to the same -// address block reserved by the IANA IPv4 Special-Purpose Address Registry: -// http://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml -func sameIPv4SpecialPurposeBlock(a, b IP) bool { - a, b = a.To4(), b.To4() - if a == nil || b == nil || a[0] != b[0] { - return false - } - // IANA defines more special-purpose blocks, but these are the only - // ones likely to be relevant to typical Go systems. - switch a[0] { - case 10: // 10.0.0.0/8: Private-Use - return true - case 127: // 127.0.0.0/8: Loopback - return true - case 169: // 169.254.0.0/16: Link Local - return a[1] == 254 && b[1] == 254 - case 172: // 172.16.0.0/12: Private-Use - return a[1]&0xf0 == 16 && b[1]&0xf0 == 16 - case 192: // 192.168.0.0/16: Private-Use - return a[1] == 168 && b[1] == 168 - } - return false -} diff --git a/libgo/go/net/addrselect_test.go b/libgo/go/net/addrselect_test.go index 80aa4eb195..d6e0e63c3b 100644 --- a/libgo/go/net/addrselect_test.go +++ b/libgo/go/net/addrselect_test.go @@ -117,27 +117,6 @@ func TestSortByRFC6724(t *testing.T) { }, reverse: false, }, - - // Prefer longer common prefixes, but only for IPv4 address - // pairs in the same special-purpose block. - { - in: []IPAddr{ - {IP: ParseIP("1.2.3.4")}, - {IP: ParseIP("10.55.0.1")}, - {IP: ParseIP("10.66.0.1")}, - }, - srcs: []IP{ - ParseIP("1.2.3.5"), - ParseIP("10.66.1.2"), - ParseIP("10.66.1.2"), - }, - want: []IPAddr{ - {IP: ParseIP("10.66.0.1")}, - {IP: ParseIP("10.55.0.1")}, - {IP: ParseIP("1.2.3.4")}, - }, - reverse: true, - }, } for i, tt := range tests { inCopy := make([]IPAddr, len(tt.in)) @@ -268,67 +247,3 @@ func TestRFC6724CommonPrefixLength(t *testing.T) { } } - -func mustParseCIDRs(t *testing.T, blocks ...string) []*IPNet { - res := make([]*IPNet, len(blocks)) - for i, block := range blocks { - var err error - _, res[i], err = ParseCIDR(block) - if err != nil { - t.Fatalf("ParseCIDR(%s) failed: %v", block, err) - } - } - return res -} - -func TestSameIPv4SpecialPurposeBlock(t *testing.T) { - blocks := mustParseCIDRs(t, - "10.0.0.0/8", - "127.0.0.0/8", - "169.254.0.0/16", - "172.16.0.0/12", - "192.168.0.0/16", - ) - - addrs := []struct { - ip IP - block int // index or -1 - }{ - {IP{1, 2, 3, 4}, -1}, - {IP{2, 3, 4, 5}, -1}, - {IP{10, 2, 3, 4}, 0}, - {IP{10, 6, 7, 8}, 0}, - {IP{127, 0, 0, 1}, 1}, - {IP{127, 255, 255, 255}, 1}, - {IP{169, 254, 77, 99}, 2}, - {IP{169, 254, 44, 22}, 2}, - {IP{169, 255, 0, 1}, -1}, - {IP{172, 15, 5, 6}, -1}, - {IP{172, 16, 32, 41}, 3}, - {IP{172, 31, 128, 9}, 3}, - {IP{172, 32, 88, 100}, -1}, - {IP{192, 168, 1, 1}, 4}, - {IP{192, 168, 128, 42}, 4}, - {IP{192, 169, 1, 1}, -1}, - } - - for i, addr := range addrs { - for j, block := range blocks { - got := block.Contains(addr.ip) - want := addr.block == j - if got != want { - t.Errorf("%d/%d. %s.Contains(%s): got %v, want %v", i, j, block, addr.ip, got, want) - } - } - } - - for i, addr1 := range addrs { - for j, addr2 := range addrs { - got := sameIPv4SpecialPurposeBlock(addr1.ip, addr2.ip) - want := addr1.block >= 0 && addr1.block == addr2.block - if got != want { - t.Errorf("%d/%d. sameIPv4SpecialPurposeBlock(%s, %s): got %v, want %v", i, j, addr1.ip, addr2.ip, got, want) - } - } - } -} diff --git a/libgo/go/net/cgo_android.go b/libgo/go/net/cgo_android.go index fe9925b840..ab0368d14b 100644 --- a/libgo/go/net/cgo_android.go +++ b/libgo/go/net/cgo_android.go @@ -1,4 +1,4 @@ -// Copyright 2014 The Go Authors. All rights reserved. +// Copyright 2014 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/cgo_bsd.go b/libgo/go/net/cgo_bsd.go index ae1054b39a..c1adf20cd9 100644 --- a/libgo/go/net/cgo_bsd.go +++ b/libgo/go/net/cgo_bsd.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/cgo_linux.go b/libgo/go/net/cgo_linux.go index baf207257f..36bac3439f 100644 --- a/libgo/go/net/cgo_linux.go +++ b/libgo/go/net/cgo_linux.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/cgo_netbsd.go b/libgo/go/net/cgo_netbsd.go index 8a16871906..1ce0139237 100644 --- a/libgo/go/net/cgo_netbsd.go +++ b/libgo/go/net/cgo_netbsd.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/cgo_openbsd.go b/libgo/go/net/cgo_openbsd.go index 183091366c..4610246561 100644 --- a/libgo/go/net/cgo_openbsd.go +++ b/libgo/go/net/cgo_openbsd.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/cgo_solaris.go b/libgo/go/net/cgo_solaris.go index 05811c6c25..bd1e8f3ae8 100644 --- a/libgo/go/net/cgo_solaris.go +++ b/libgo/go/net/cgo_solaris.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Go Authors. All rights reserved. +// Copyright 2015 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/cgo_stub.go b/libgo/go/net/cgo_stub.go index b86ff7daf1..51259722ae 100644 --- a/libgo/go/net/cgo_stub.go +++ b/libgo/go/net/cgo_stub.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -6,6 +6,8 @@ package net +import "context" + func init() { netGo = true } type addrinfoErrno int @@ -14,22 +16,22 @@ func (eai addrinfoErrno) Error() string { return "<nil>" } func (eai addrinfoErrno) Temporary() bool { return false } func (eai addrinfoErrno) Timeout() bool { return false } -func cgoLookupHost(name string) (addrs []string, err error, completed bool) { +func cgoLookupHost(ctx context.Context, name string) (addrs []string, err error, completed bool) { return nil, nil, false } -func cgoLookupPort(network, service string) (port int, err error, completed bool) { +func cgoLookupPort(ctx context.Context, network, service string) (port int, err error, completed bool) { return 0, nil, false } -func cgoLookupIP(name string) (addrs []IPAddr, err error, completed bool) { +func cgoLookupIP(ctx context.Context, name string) (addrs []IPAddr, err error, completed bool) { return nil, nil, false } -func cgoLookupCNAME(name string) (cname string, err error, completed bool) { +func cgoLookupCNAME(ctx context.Context, name string) (cname string, err error, completed bool) { return "", nil, false } -func cgoLookupPTR(addr string) (ptrs []string, err error, completed bool) { +func cgoLookupPTR(ctx context.Context, addr string) (ptrs []string, err error, completed bool) { return nil, nil, false } diff --git a/libgo/go/net/cgo_unix.go b/libgo/go/net/cgo_unix.go index f634323ed8..a90aaa904e 100644 --- a/libgo/go/net/cgo_unix.go +++ b/libgo/go/net/cgo_unix.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -18,6 +18,7 @@ package net */ import ( + "context" "syscall" "unsafe" ) @@ -51,18 +52,31 @@ func (eai addrinfoErrno) Error() string { return bytePtrToString(libc_gai_stre func (eai addrinfoErrno) Temporary() bool { return eai == syscall.EAI_AGAIN } func (eai addrinfoErrno) Timeout() bool { return false } -func cgoLookupHost(name string) (hosts []string, err error, completed bool) { - addrs, err, completed := cgoLookupIP(name) +type portLookupResult struct { + port int + err error +} + +type ipLookupResult struct { + addrs []IPAddr + cname string + err error +} + +type reverseLookupResult struct { + names []string + err error +} + +func cgoLookupHost(ctx context.Context, name string) (hosts []string, err error, completed bool) { + addrs, err, completed := cgoLookupIP(ctx, name) for _, addr := range addrs { hosts = append(hosts, addr.String()) } return } -func cgoLookupPort(network, service string) (port int, err error, completed bool) { - acquireThread() - defer releaseThread() - +func cgoLookupPort(ctx context.Context, network, service string) (port int, err error, completed bool) { var hints syscall.Addrinfo switch network { case "": // no hints @@ -83,11 +97,35 @@ func cgoLookupPort(network, service string) (port int, err error, completed bool hints.Ai_family = syscall.AF_INET6 } } + if ctx.Done() == nil { + port, err := cgoLookupServicePort(&hints, network, service) + return port, err, true + } + result := make(chan portLookupResult, 1) + go cgoPortLookup(result, &hints, network, service) + select { + case r := <-result: + return r.port, r.err, true + case <-ctx.Done(): + // Since there isn't a portable way to cancel the lookup, + // we just let it finish and write to the buffered channel. + return 0, mapErr(ctx.Err()), false + } +} - s := syscall.StringBytePtr(service) +func cgoLookupServicePort(hints *syscall.Addrinfo, network, service string) (port int, err error) { + s, err := syscall.BytePtrFromString(service) + if err != nil { + return 0, err + } + // Lowercase the service name in the memory passed to C. + for i := 0; i < len(service); i++ { + bp := (*byte)(unsafe.Pointer(uintptr(unsafe.Pointer(s)) + uintptr(i))) + *bp = lowerASCII(*bp) + } var res *syscall.Addrinfo syscall.Entersyscall() - gerrno := libc_getaddrinfo(nil, s, &hints, &res) + gerrno := libc_getaddrinfo(nil, s, hints, &res) syscall.Exitsyscall() if gerrno != 0 { switch gerrno { @@ -100,7 +138,7 @@ func cgoLookupPort(network, service string) (port int, err error, completed bool default: err = addrinfoErrno(gerrno) } - return 0, &DNSError{Err: err.Error(), Name: network + "/" + service}, true + return 0, &DNSError{Err: err.Error(), Name: network + "/" + service} } defer libc_freeaddrinfo(res) @@ -109,17 +147,22 @@ func cgoLookupPort(network, service string) (port int, err error, completed bool case syscall.AF_INET: sa := (*syscall.RawSockaddrInet4)(unsafe.Pointer(r.Ai_addr)) p := (*[2]byte)(unsafe.Pointer(&sa.Port)) - return int(p[0])<<8 | int(p[1]), nil, true + return int(p[0])<<8 | int(p[1]), nil case syscall.AF_INET6: sa := (*syscall.RawSockaddrInet6)(unsafe.Pointer(r.Ai_addr)) p := (*[2]byte)(unsafe.Pointer(&sa.Port)) - return int(p[0])<<8 | int(p[1]), nil, true + return int(p[0])<<8 | int(p[1]), nil } } - return 0, &DNSError{Err: "unknown port", Name: network + "/" + service}, true + return 0, &DNSError{Err: "unknown port", Name: network + "/" + service} +} + +func cgoPortLookup(result chan<- portLookupResult, hints *syscall.Addrinfo, network, service string) { + port, err := cgoLookupServicePort(hints, network, service) + result <- portLookupResult{port, err} } -func cgoLookupIPCNAME(name string) (addrs []IPAddr, cname string, err error, completed bool) { +func cgoLookupIPCNAME(name string) (addrs []IPAddr, cname string, err error) { acquireThread() defer releaseThread() @@ -152,7 +195,7 @@ func cgoLookupIPCNAME(name string) (addrs []IPAddr, cname string, err error, com default: err = addrinfoErrno(gerrno) } - return nil, "", &DNSError{Err: err.Error(), Name: name}, true + return nil, "", &DNSError{Err: err.Error(), Name: name} } defer libc_freeaddrinfo(res) @@ -181,17 +224,42 @@ func cgoLookupIPCNAME(name string) (addrs []IPAddr, cname string, err error, com addrs = append(addrs, addr) } } - return addrs, cname, nil, true + return addrs, cname, nil } -func cgoLookupIP(name string) (addrs []IPAddr, err error, completed bool) { - addrs, _, err, completed = cgoLookupIPCNAME(name) - return +func cgoIPLookup(result chan<- ipLookupResult, name string) { + addrs, cname, err := cgoLookupIPCNAME(name) + result <- ipLookupResult{addrs, cname, err} } -func cgoLookupCNAME(name string) (cname string, err error, completed bool) { - _, cname, err, completed = cgoLookupIPCNAME(name) - return +func cgoLookupIP(ctx context.Context, name string) (addrs []IPAddr, err error, completed bool) { + if ctx.Done() == nil { + addrs, _, err = cgoLookupIPCNAME(name) + return addrs, err, true + } + result := make(chan ipLookupResult, 1) + go cgoIPLookup(result, name) + select { + case r := <-result: + return r.addrs, r.err, true + case <-ctx.Done(): + return nil, mapErr(ctx.Err()), false + } +} + +func cgoLookupCNAME(ctx context.Context, name string) (cname string, err error, completed bool) { + if ctx.Done() == nil { + _, cname, err = cgoLookupIPCNAME(name) + return cname, err, true + } + result := make(chan ipLookupResult, 1) + go cgoIPLookup(result, name) + select { + case r := <-result: + return r.cname, r.err, true + case <-ctx.Done(): + return "", mapErr(ctx.Err()), false + } } // These are roughly enough for the following: @@ -207,10 +275,7 @@ const ( maxNameinfoLen = 4096 ) -func cgoLookupPTR(addr string) ([]string, error, bool) { - acquireThread() - defer releaseThread() - +func cgoLookupPTR(ctx context.Context, addr string) (names []string, err error, completed bool) { var zone string ip := parseIPv4(addr) if ip == nil { @@ -223,9 +288,26 @@ func cgoLookupPTR(addr string) ([]string, error, bool) { if sa == nil { return nil, &DNSError{Err: "invalid address " + ip.String(), Name: addr}, true } - var err error - var b []byte + if ctx.Done() == nil { + names, err := cgoLookupAddrPTR(addr, sa, salen) + return names, err, true + } + result := make(chan reverseLookupResult, 1) + go cgoReverseLookup(result, addr, sa, salen) + select { + case r := <-result: + return r.names, r.err, true + case <-ctx.Done(): + return nil, mapErr(ctx.Err()), false + } +} + +func cgoLookupAddrPTR(addr string, sa *syscall.RawSockaddr, salen syscall.Socklen_t) (names []string, err error) { + acquireThread() + defer releaseThread() + var gerrno int + var b []byte for l := nameinfoLen; l <= maxNameinfoLen; l *= 2 { b = make([]byte, l) gerrno, err = cgoNameinfoPTR(b, sa, salen) @@ -242,16 +324,20 @@ func cgoLookupPTR(addr string) ([]string, error, bool) { default: err = addrinfoErrno(gerrno) } - return nil, &DNSError{Err: err.Error(), Name: addr}, true + return nil, &DNSError{Err: err.Error(), Name: addr} } - for i := 0; i < len(b); i++ { if b[i] == 0 { b = b[:i] break } } - return []string{absDomainName(b)}, nil, true + return []string{absDomainName(b)}, nil +} + +func cgoReverseLookup(result chan<- reverseLookupResult, addr string, sa *syscall.RawSockaddr, salen syscall.Socklen_t) { + names, err := cgoLookupAddrPTR(addr, sa, salen) + result <- reverseLookupResult{names, err} } func cgoSockaddr(ip IP, zone string) (*syscall.RawSockaddr, syscall.Socklen_t) { diff --git a/libgo/go/net/cgo_unix_test.go b/libgo/go/net/cgo_unix_test.go index 4d5ab23fd3..e861c7aa1f 100644 --- a/libgo/go/net/cgo_unix_test.go +++ b/libgo/go/net/cgo_unix_test.go @@ -7,18 +7,76 @@ package net -import "testing" +import ( + "context" + "testing" +) func TestCgoLookupIP(t *testing.T) { - host := "localhost" - _, err, ok := cgoLookupIP(host) + ctx := context.Background() + _, err, ok := cgoLookupIP(ctx, "localhost") if !ok { t.Errorf("cgoLookupIP must not be a placeholder") } if err != nil { t.Error(err) } - if _, err := goLookupIP(host); err != nil { +} + +func TestCgoLookupIPWithCancel(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + _, err, ok := cgoLookupIP(ctx, "localhost") + if !ok { + t.Errorf("cgoLookupIP must not be a placeholder") + } + if err != nil { + t.Error(err) + } +} + +func TestCgoLookupPort(t *testing.T) { + ctx := context.Background() + _, err, ok := cgoLookupPort(ctx, "tcp", "smtp") + if !ok { + t.Errorf("cgoLookupPort must not be a placeholder") + } + if err != nil { + t.Error(err) + } +} + +func TestCgoLookupPortWithCancel(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + _, err, ok := cgoLookupPort(ctx, "tcp", "smtp") + if !ok { + t.Errorf("cgoLookupPort must not be a placeholder") + } + if err != nil { + t.Error(err) + } +} + +func TestCgoLookupPTR(t *testing.T) { + ctx := context.Background() + _, err, ok := cgoLookupPTR(ctx, "127.0.0.1") + if !ok { + t.Errorf("cgoLookupPTR must not be a placeholder") + } + if err != nil { + t.Error(err) + } +} + +func TestCgoLookupPTRWithCancel(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + _, err, ok := cgoLookupPTR(ctx, "127.0.0.1") + if !ok { + t.Errorf("cgoLookupPTR must not be a placeholder") + } + if err != nil { t.Error(err) } } diff --git a/libgo/go/net/conf.go b/libgo/go/net/conf.go index ddaa978f4f..c10aafe63a 100644 --- a/libgo/go/net/conf.go +++ b/libgo/go/net/conf.go @@ -124,16 +124,17 @@ func (c *conf) hostLookupOrder(hostname string) (ret hostLookupOrder) { print("go package net: hostLookupOrder(", hostname, ") = ", ret.String(), "\n") }() } + fallbackOrder := hostLookupCgo if c.netGo { - return hostLookupFilesDNS + fallbackOrder = hostLookupFilesDNS } if c.forceCgoLookupHost || c.resolv.unknownOpt || c.goos == "android" { - return hostLookupCgo + return fallbackOrder } if byteIndex(hostname, '\\') != -1 || byteIndex(hostname, '%') != -1 { // Don't deal with special form hostnames with backslashes // or '%'. - return hostLookupCgo + return fallbackOrder } // OpenBSD is unique and doesn't use nsswitch.conf. @@ -154,7 +155,7 @@ func (c *conf) hostLookupOrder(hostname string) (ret hostLookupOrder) { return hostLookupDNSFiles } if len(lookup) < 1 || len(lookup) > 2 { - return hostLookupCgo + return fallbackOrder } switch lookup[0] { case "bind": @@ -162,7 +163,7 @@ func (c *conf) hostLookupOrder(hostname string) (ret hostLookupOrder) { if lookup[1] == "file" { return hostLookupDNSFiles } - return hostLookupCgo + return fallbackOrder } return hostLookupDNS case "file": @@ -170,26 +171,24 @@ func (c *conf) hostLookupOrder(hostname string) (ret hostLookupOrder) { if lookup[1] == "bind" { return hostLookupFilesDNS } - return hostLookupCgo + return fallbackOrder } return hostLookupFiles default: - return hostLookupCgo + return fallbackOrder } } - hasDot := byteIndex(hostname, '.') != -1 - // Canonicalize the hostname by removing any trailing dot. if stringsHasSuffix(hostname, ".") { hostname = hostname[:len(hostname)-1] } if stringsHasSuffixFold(hostname, ".local") { - // Per RFC 6762, the ".local" TLD is special. And + // Per RFC 6762, the ".local" TLD is special. And // because Go's native resolver doesn't do mDNS or // similar local resolution mechanisms, assume that // libc might (via Avahi, etc) and use cgo. - return hostLookupCgo + return fallbackOrder } nss := c.nss @@ -199,7 +198,7 @@ func (c *conf) hostLookupOrder(hostname string) (ret hostLookupOrder) { if os.IsNotExist(nss.err) || (nss.err == nil && len(srcs) == 0) { if c.goos == "solaris" { // illumos defaults to "nis [NOTFOUND=return] files" - return hostLookupCgo + return fallbackOrder } if c.goos == "linux" { // glibc says the default is "dns [!UNAVAIL=return] files" @@ -212,21 +211,25 @@ func (c *conf) hostLookupOrder(hostname string) (ret hostLookupOrder) { // We failed to parse or open nsswitch.conf, so // conservatively assume we should use cgo if it's // available. - return hostLookupCgo + return fallbackOrder } var mdnsSource, filesSource, dnsSource bool var first string for _, src := range srcs { if src.source == "myhostname" { - if hasDot { - continue + if isLocalhost(hostname) || isGateway(hostname) { + return fallbackOrder } - return hostLookupCgo + hn, err := getHostname() + if err != nil || stringsEqualFold(hostname, hn) { + return fallbackOrder + } + continue } if src.source == "files" || src.source == "dns" { if !src.standardCriteria() { - return hostLookupCgo // non-standard; let libc deal with it. + return fallbackOrder // non-standard; let libc deal with it. } if src.source == "files" { filesSource = true @@ -246,14 +249,14 @@ func (c *conf) hostLookupOrder(hostname string) (ret hostLookupOrder) { continue } // Some source we don't know how to deal with. - return hostLookupCgo + return fallbackOrder } // We don't parse mdns.allow files. They're rare. If one // exists, it might list other TLDs (besides .local) or even // '*', so just let libc deal with it. if mdnsSource && c.hasMDNSAllow { - return hostLookupCgo + return fallbackOrder } // Cases where Go can handle it without cgo and C thread @@ -272,7 +275,7 @@ func (c *conf) hostLookupOrder(hostname string) (ret hostLookupOrder) { } // Something weird. Let libc deal with it. - return hostLookupCgo + return fallbackOrder } // goDebugNetDNS parses the value of the GODEBUG "netdns" value. @@ -292,7 +295,7 @@ func goDebugNetDNS() (dnsMode string, debugLevel int) { return } if '0' <= s[0] && s[0] <= '9' { - debugLevel, _, _ = dtoi(s, 0) + debugLevel, _, _ = dtoi(s) } else { dnsMode = s } @@ -305,3 +308,15 @@ func goDebugNetDNS() (dnsMode string, debugLevel int) { parsePart(goDebug) return } + +// isLocalhost reports whether h should be considered a "localhost" +// name for the myhostname NSS module. +func isLocalhost(h string) bool { + return stringsEqualFold(h, "localhost") || stringsEqualFold(h, "localhost.localdomain") || stringsHasSuffixFold(h, ".localhost") || stringsHasSuffixFold(h, ".localhost.localdomain") +} + +// isGateway reports whether h should be considered a "gateway" +// name for the myhostname NSS module. +func isGateway(h string) bool { + return stringsEqualFold(h, "gateway") +} diff --git a/libgo/go/net/conf_netcgo.go b/libgo/go/net/conf_netcgo.go index b66bae3710..abc33cee47 100644 --- a/libgo/go/net/conf_netcgo.go +++ b/libgo/go/net/conf_netcgo.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Go Authors. All rights reserved. +// Copyright 2015 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/conf_test.go b/libgo/go/net/conf_test.go index 86904bffde..17d03f4b5f 100644 --- a/libgo/go/net/conf_test.go +++ b/libgo/go/net/conf_test.go @@ -13,8 +13,9 @@ import ( ) type nssHostTest struct { - host string - want hostLookupOrder + host string + localhost string + want hostLookupOrder } func nssStr(s string) *nssConf { return parseNSSConf(strings.NewReader(s)) } @@ -32,7 +33,6 @@ func TestConfHostLookupOrder(t *testing.T) { tests := []struct { name string c *conf - goos string hostTests []nssHostTest }{ { @@ -43,8 +43,30 @@ func TestConfHostLookupOrder(t *testing.T) { resolv: defaultResolvConf, }, hostTests: []nssHostTest{ - {"foo.local", hostLookupCgo}, - {"google.com", hostLookupCgo}, + {"foo.local", "myhostname", hostLookupCgo}, + {"google.com", "myhostname", hostLookupCgo}, + }, + }, + { + name: "netgo_dns_before_files", + c: &conf{ + netGo: true, + nss: nssStr("hosts: dns files"), + resolv: defaultResolvConf, + }, + hostTests: []nssHostTest{ + {"x.com", "myhostname", hostLookupDNSFiles}, + }, + }, + { + name: "netgo_fallback_on_cgo", + c: &conf{ + netGo: true, + nss: nssStr("hosts: dns files something_custom"), + resolv: defaultResolvConf, + }, + hostTests: []nssHostTest{ + {"x.com", "myhostname", hostLookupFilesDNS}, }, }, { @@ -54,11 +76,11 @@ func TestConfHostLookupOrder(t *testing.T) { resolv: defaultResolvConf, }, hostTests: []nssHostTest{ - {"foo.local", hostLookupCgo}, - {"foo.local.", hostLookupCgo}, - {"foo.LOCAL", hostLookupCgo}, - {"foo.LOCAL.", hostLookupCgo}, - {"google.com", hostLookupFilesDNS}, + {"foo.local", "myhostname", hostLookupCgo}, + {"foo.local.", "myhostname", hostLookupCgo}, + {"foo.LOCAL", "myhostname", hostLookupCgo}, + {"foo.LOCAL.", "myhostname", hostLookupCgo}, + {"google.com", "myhostname", hostLookupFilesDNS}, }, }, { @@ -68,7 +90,7 @@ func TestConfHostLookupOrder(t *testing.T) { nss: nssStr("foo: bar"), resolv: defaultResolvConf, }, - hostTests: []nssHostTest{{"google.com", hostLookupFilesDNS}}, + hostTests: []nssHostTest{{"google.com", "myhostname", hostLookupFilesDNS}}, }, // On OpenBSD, no resolv.conf means no DNS. { @@ -77,7 +99,7 @@ func TestConfHostLookupOrder(t *testing.T) { goos: "openbsd", resolv: defaultResolvConf, }, - hostTests: []nssHostTest{{"google.com", hostLookupFiles}}, + hostTests: []nssHostTest{{"google.com", "myhostname", hostLookupFiles}}, }, { name: "solaris_no_nsswitch", @@ -86,7 +108,7 @@ func TestConfHostLookupOrder(t *testing.T) { nss: &nssConf{err: os.ErrNotExist}, resolv: defaultResolvConf, }, - hostTests: []nssHostTest{{"google.com", hostLookupCgo}}, + hostTests: []nssHostTest{{"google.com", "myhostname", hostLookupCgo}}, }, { name: "openbsd_lookup_bind_file", @@ -95,8 +117,8 @@ func TestConfHostLookupOrder(t *testing.T) { resolv: &dnsConfig{lookup: []string{"bind", "file"}}, }, hostTests: []nssHostTest{ - {"google.com", hostLookupDNSFiles}, - {"foo.local", hostLookupDNSFiles}, + {"google.com", "myhostname", hostLookupDNSFiles}, + {"foo.local", "myhostname", hostLookupDNSFiles}, }, }, { @@ -105,7 +127,7 @@ func TestConfHostLookupOrder(t *testing.T) { goos: "openbsd", resolv: &dnsConfig{lookup: []string{"file", "bind"}}, }, - hostTests: []nssHostTest{{"google.com", hostLookupFilesDNS}}, + hostTests: []nssHostTest{{"google.com", "myhostname", hostLookupFilesDNS}}, }, { name: "openbsd_lookup_bind", @@ -113,7 +135,7 @@ func TestConfHostLookupOrder(t *testing.T) { goos: "openbsd", resolv: &dnsConfig{lookup: []string{"bind"}}, }, - hostTests: []nssHostTest{{"google.com", hostLookupDNS}}, + hostTests: []nssHostTest{{"google.com", "myhostname", hostLookupDNS}}, }, { name: "openbsd_lookup_file", @@ -121,7 +143,7 @@ func TestConfHostLookupOrder(t *testing.T) { goos: "openbsd", resolv: &dnsConfig{lookup: []string{"file"}}, }, - hostTests: []nssHostTest{{"google.com", hostLookupFiles}}, + hostTests: []nssHostTest{{"google.com", "myhostname", hostLookupFiles}}, }, { name: "openbsd_lookup_yp", @@ -129,7 +151,7 @@ func TestConfHostLookupOrder(t *testing.T) { goos: "openbsd", resolv: &dnsConfig{lookup: []string{"file", "bind", "yp"}}, }, - hostTests: []nssHostTest{{"google.com", hostLookupCgo}}, + hostTests: []nssHostTest{{"google.com", "myhostname", hostLookupCgo}}, }, { name: "openbsd_lookup_two", @@ -137,7 +159,7 @@ func TestConfHostLookupOrder(t *testing.T) { goos: "openbsd", resolv: &dnsConfig{lookup: []string{"file", "foo"}}, }, - hostTests: []nssHostTest{{"google.com", hostLookupCgo}}, + hostTests: []nssHostTest{{"google.com", "myhostname", hostLookupCgo}}, }, { name: "openbsd_lookup_empty", @@ -145,7 +167,7 @@ func TestConfHostLookupOrder(t *testing.T) { goos: "openbsd", resolv: &dnsConfig{lookup: nil}, }, - hostTests: []nssHostTest{{"google.com", hostLookupDNSFiles}}, + hostTests: []nssHostTest{{"google.com", "myhostname", hostLookupDNSFiles}}, }, // glibc lacking an nsswitch.conf, per // http://www.gnu.org/software/libc/manual/html_node/Notes-on-NSS-Configuration-File.html @@ -156,7 +178,7 @@ func TestConfHostLookupOrder(t *testing.T) { nss: &nssConf{err: os.ErrNotExist}, resolv: defaultResolvConf, }, - hostTests: []nssHostTest{{"google.com", hostLookupDNSFiles}}, + hostTests: []nssHostTest{{"google.com", "myhostname", hostLookupDNSFiles}}, }, { name: "files_mdns_dns", @@ -165,8 +187,8 @@ func TestConfHostLookupOrder(t *testing.T) { resolv: defaultResolvConf, }, hostTests: []nssHostTest{ - {"x.com", hostLookupFilesDNS}, - {"x.local", hostLookupCgo}, + {"x.com", "myhostname", hostLookupFilesDNS}, + {"x.local", "myhostname", hostLookupCgo}, }, }, { @@ -176,9 +198,9 @@ func TestConfHostLookupOrder(t *testing.T) { resolv: defaultResolvConf, }, hostTests: []nssHostTest{ - {"x.com", hostLookupDNS}, - {"x\\.com", hostLookupCgo}, // punt on weird glibc escape - {"foo.com%en0", hostLookupCgo}, // and IPv6 zones + {"x.com", "myhostname", hostLookupDNS}, + {"x\\.com", "myhostname", hostLookupCgo}, // punt on weird glibc escape + {"foo.com%en0", "myhostname", hostLookupCgo}, // and IPv6 zones }, }, { @@ -189,8 +211,8 @@ func TestConfHostLookupOrder(t *testing.T) { hasMDNSAllow: true, }, hostTests: []nssHostTest{ - {"x.com", hostLookupCgo}, - {"x.local", hostLookupCgo}, + {"x.com", "myhostname", hostLookupCgo}, + {"x.local", "myhostname", hostLookupCgo}, }, }, { @@ -200,9 +222,9 @@ func TestConfHostLookupOrder(t *testing.T) { resolv: defaultResolvConf, }, hostTests: []nssHostTest{ - {"x.com", hostLookupFilesDNS}, - {"x", hostLookupFilesDNS}, - {"x.local", hostLookupCgo}, + {"x.com", "myhostname", hostLookupFilesDNS}, + {"x", "myhostname", hostLookupFilesDNS}, + {"x.local", "myhostname", hostLookupCgo}, }, }, { @@ -212,9 +234,9 @@ func TestConfHostLookupOrder(t *testing.T) { resolv: defaultResolvConf, }, hostTests: []nssHostTest{ - {"x.com", hostLookupDNSFiles}, - {"x", hostLookupDNSFiles}, - {"x.local", hostLookupCgo}, + {"x.com", "myhostname", hostLookupDNSFiles}, + {"x", "myhostname", hostLookupDNSFiles}, + {"x.local", "myhostname", hostLookupCgo}, }, }, { @@ -224,7 +246,7 @@ func TestConfHostLookupOrder(t *testing.T) { resolv: defaultResolvConf, }, hostTests: []nssHostTest{ - {"x.com", hostLookupCgo}, + {"x.com", "myhostname", hostLookupCgo}, }, }, { @@ -234,8 +256,23 @@ func TestConfHostLookupOrder(t *testing.T) { resolv: defaultResolvConf, }, hostTests: []nssHostTest{ - {"x.com", hostLookupFilesDNS}, - {"somehostname", hostLookupCgo}, + {"x.com", "myhostname", hostLookupFilesDNS}, + {"myhostname", "myhostname", hostLookupCgo}, + {"myHostname", "myhostname", hostLookupCgo}, + {"myhostname.dot", "myhostname.dot", hostLookupCgo}, + {"myHostname.dot", "myhostname.dot", hostLookupCgo}, + {"gateway", "myhostname", hostLookupCgo}, + {"Gateway", "myhostname", hostLookupCgo}, + {"localhost", "myhostname", hostLookupCgo}, + {"Localhost", "myhostname", hostLookupCgo}, + {"anything.localhost", "myhostname", hostLookupCgo}, + {"Anything.localhost", "myhostname", hostLookupCgo}, + {"localhost.localdomain", "myhostname", hostLookupCgo}, + {"Localhost.Localdomain", "myhostname", hostLookupCgo}, + {"anything.localhost.localdomain", "myhostname", hostLookupCgo}, + {"Anything.Localhost.Localdomain", "myhostname", hostLookupCgo}, + {"somehostname", "myhostname", hostLookupFilesDNS}, + {"", "myhostname", hostLookupFilesDNS}, // Issue 13623 }, }, { @@ -245,8 +282,9 @@ func TestConfHostLookupOrder(t *testing.T) { resolv: defaultResolvConf, }, hostTests: []nssHostTest{ - {"x.com", hostLookupFilesDNS}, - {"somehostname", hostLookupCgo}, + {"x.com", "myhostname", hostLookupFilesDNS}, + {"somehostname", "myhostname", hostLookupFilesDNS}, + {"myhostname", "myhostname", hostLookupCgo}, }, }, // Debian Squeeze is just "dns,files", but lists all @@ -260,8 +298,8 @@ func TestConfHostLookupOrder(t *testing.T) { resolv: defaultResolvConf, }, hostTests: []nssHostTest{ - {"x.com", hostLookupDNSFiles}, - {"somehostname", hostLookupDNSFiles}, + {"x.com", "myhostname", hostLookupDNSFiles}, + {"somehostname", "myhostname", hostLookupDNSFiles}, }, }, { @@ -270,7 +308,7 @@ func TestConfHostLookupOrder(t *testing.T) { nss: nssStr("foo: bar"), resolv: &dnsConfig{servers: defaultNS, ndots: 1, timeout: 5, attempts: 2, unknownOpt: true}, }, - hostTests: []nssHostTest{{"google.com", hostLookupCgo}}, + hostTests: []nssHostTest{{"google.com", "myhostname", hostLookupCgo}}, }, // Android should always use cgo. { @@ -281,12 +319,18 @@ func TestConfHostLookupOrder(t *testing.T) { resolv: defaultResolvConf, }, hostTests: []nssHostTest{ - {"x.com", hostLookupCgo}, + {"x.com", "myhostname", hostLookupCgo}, }, }, } + + origGetHostname := getHostname + defer func() { getHostname = origGetHostname }() + for _, tt := range tests { for _, ht := range tt.hostTests { + getHostname = func() (string, error) { return ht.localhost, nil } + gotOrder := tt.c.hostLookupOrder(ht.host) if gotOrder != ht.want { t.Errorf("%s: hostLookupOrder(%q) = %v; want %v", tt.name, ht.host, gotOrder, ht.want) diff --git a/libgo/go/net/conn_test.go b/libgo/go/net/conn_test.go index 6995c110f2..16cf69ee16 100644 --- a/libgo/go/net/conn_test.go +++ b/libgo/go/net/conn_test.go @@ -1,4 +1,4 @@ -// Copyright 2012 The Go Authors. All rights reserved. +// Copyright 2012 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -43,7 +43,7 @@ func TestConnAndListener(t *testing.T) { t.Fatal(err) } defer c.Close() - if c.LocalAddr().Network() != network || c.LocalAddr().Network() != network { + if c.LocalAddr().Network() != network || c.RemoteAddr().Network() != network { t.Fatalf("got %s->%s; want %s->%s", c.LocalAddr().Network(), c.RemoteAddr().Network(), network, network) } c.SetDeadline(time.Now().Add(someTimeout)) diff --git a/libgo/go/net/dial.go b/libgo/go/net/dial.go index 193776fe41..50bba5a49e 100644 --- a/libgo/go/net/dial.go +++ b/libgo/go/net/dial.go @@ -1,11 +1,12 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package net import ( - "errors" + "context" + "internal/nettrace" "time" ) @@ -58,24 +59,47 @@ type Dialer struct { // that do not support keep-alives ignore this field. KeepAlive time.Duration + // Resolver optionally specifies an alternate resolver to use. + Resolver *Resolver + // Cancel is an optional channel whose closure indicates that // the dial should be canceled. Not all types of dials support // cancelation. + // + // Deprecated: Use DialContext instead. Cancel <-chan struct{} } -// Return either now+Timeout or Deadline, whichever comes first. -// Or zero, if neither is set. -func (d *Dialer) deadline(now time.Time) time.Time { - if d.Timeout == 0 { - return d.Deadline +func minNonzeroTime(a, b time.Time) time.Time { + if a.IsZero() { + return b } - timeoutDeadline := now.Add(d.Timeout) - if d.Deadline.IsZero() || timeoutDeadline.Before(d.Deadline) { - return timeoutDeadline - } else { - return d.Deadline + if b.IsZero() || a.Before(b) { + return a + } + return b +} + +// deadline returns the earliest of: +// - now+Timeout +// - d.Deadline +// - the context's deadline +// Or zero, if none of Timeout, Deadline, or context's deadline is set. +func (d *Dialer) deadline(ctx context.Context, now time.Time) (earliest time.Time) { + if d.Timeout != 0 { // including negative, for historical reasons + earliest = now.Add(d.Timeout) + } + if d, ok := ctx.Deadline(); ok { + earliest = minNonzeroTime(earliest, d) + } + return minNonzeroTime(earliest, d.Deadline) +} + +func (d *Dialer) resolver() *Resolver { + if d.Resolver != nil { + return d.Resolver } + return DefaultResolver } // partialDeadline returns the deadline to use for a single address, @@ -110,7 +134,7 @@ func (d *Dialer) fallbackDelay() time.Duration { } } -func parseNetwork(net string) (afnet string, proto int, err error) { +func parseNetwork(ctx context.Context, net string) (afnet string, proto int, err error) { i := last(net, ':') if i < 0 { // no colon switch net { @@ -127,9 +151,9 @@ func parseNetwork(net string) (afnet string, proto int, err error) { switch afnet { case "ip", "ip4", "ip6": protostr := net[i+1:] - proto, i, ok := dtoi(protostr, 0) + proto, i, ok := dtoi(protostr) if !ok || i != len(protostr) { - proto, err = lookupProtocol(protostr) + proto, err = lookupProtocol(ctx, protostr) if err != nil { return "", 0, err } @@ -139,8 +163,11 @@ func parseNetwork(net string) (afnet string, proto int, err error) { return "", 0, UnknownNetworkError(net) } -func resolveAddrList(op, net, addr string, deadline time.Time) (addrList, error) { - afnet, _, err := parseNetwork(net) +// resolveAddrList resolves addr using hint and returns a list of +// addresses. The result contains at least one address when error is +// nil. +func (r *Resolver) resolveAddrList(ctx context.Context, op, network, addr string, hint Addr) (addrList, error) { + afnet, _, err := parseNetwork(ctx, network) if err != nil { return nil, err } @@ -153,9 +180,59 @@ func resolveAddrList(op, net, addr string, deadline time.Time) (addrList, error) if err != nil { return nil, err } + if op == "dial" && hint != nil && addr.Network() != hint.Network() { + return nil, &AddrError{Err: "mismatched local address type", Addr: hint.String()} + } return addrList{addr}, nil } - return internetAddrList(afnet, addr, deadline) + addrs, err := r.internetAddrList(ctx, afnet, addr) + if err != nil || op != "dial" || hint == nil { + return addrs, err + } + var ( + tcp *TCPAddr + udp *UDPAddr + ip *IPAddr + wildcard bool + ) + switch hint := hint.(type) { + case *TCPAddr: + tcp = hint + wildcard = tcp.isWildcard() + case *UDPAddr: + udp = hint + wildcard = udp.isWildcard() + case *IPAddr: + ip = hint + wildcard = ip.isWildcard() + } + naddrs := addrs[:0] + for _, addr := range addrs { + if addr.Network() != hint.Network() { + return nil, &AddrError{Err: "mismatched local address type", Addr: hint.String()} + } + switch addr := addr.(type) { + case *TCPAddr: + if !wildcard && !addr.isWildcard() && !addr.IP.matchAddrFamily(tcp.IP) { + continue + } + naddrs = append(naddrs, addr) + case *UDPAddr: + if !wildcard && !addr.isWildcard() && !addr.IP.matchAddrFamily(udp.IP) { + continue + } + naddrs = append(naddrs, addr) + case *IPAddr: + if !wildcard && !addr.isWildcard() && !addr.IP.matchAddrFamily(ip.IP) { + continue + } + naddrs = append(naddrs, addr) + } + } + if len(naddrs) == 0 { + return nil, &AddrError{Err: errNoSuitableAddress.Error(), Addr: hint.String()} + } + return naddrs, nil } // Dial connects to the address on the named network. @@ -173,8 +250,8 @@ func resolveAddrList(op, net, addr string, deadline time.Time) (addrList, error) // If the host is empty, as in ":80", the local system is assumed. // // Examples: -// Dial("tcp", "12.34.56.78:80") -// Dial("tcp", "google.com:http") +// Dial("tcp", "192.0.2.1:80") +// Dial("tcp", "golang.org:http") // Dial("tcp", "[2001:db8::1]:http") // Dial("tcp", "[fe80::1%lo0]:80") // Dial("tcp", ":80") @@ -184,10 +261,13 @@ func resolveAddrList(op, net, addr string, deadline time.Time) (addrList, error) // literal IP address. // // Examples: -// Dial("ip4:1", "127.0.0.1") -// Dial("ip6:ospf", "::1") +// Dial("ip4:1", "192.0.2.1") +// Dial("ip6:ipv6-icmp", "2001:db8::1") // // For Unix networks, the address must be a file system path. +// +// If the host is resolved to multiple addresses, +// Dial will try each address in order until one succeeds. func Dial(network, address string) (Conn, error) { var d Dialer return d.Dial(network, address) @@ -200,11 +280,10 @@ func DialTimeout(network, address string, timeout time.Duration) (Conn, error) { return d.Dial(network, address) } -// dialContext holds common state for all dial operations. -type dialContext struct { +// dialParam contains a Dial's parameters and configuration. +type dialParam struct { Dialer network, address string - finalDeadline time.Time } // Dial connects to the address on the named network. @@ -212,17 +291,70 @@ type dialContext struct { // See func Dial for a description of the network and address // parameters. func (d *Dialer) Dial(network, address string) (Conn, error) { - finalDeadline := d.deadline(time.Now()) - addrs, err := resolveAddrList("dial", network, address, finalDeadline) + return d.DialContext(context.Background(), network, address) +} + +// DialContext connects to the address on the named network using +// the provided context. +// +// The provided Context must be non-nil. If the context expires before +// the connection is complete, an error is returned. Once successfully +// connected, any expiration of the context will not affect the +// connection. +// +// When using TCP, and the host in the address parameter resolves to multiple +// network addresses, any dial timeout (from d.Timeout or ctx) is spread +// over each consecutive dial, such that each is given an appropriate +// fraction of the time to connect. +// For example, if a host has 4 IP addresses and the timeout is 1 minute, +// the connect to each single address will be given 15 seconds to complete +// before trying the next one. +// +// See func Dial for a description of the network and address +// parameters. +func (d *Dialer) DialContext(ctx context.Context, network, address string) (Conn, error) { + if ctx == nil { + panic("nil context") + } + deadline := d.deadline(ctx, time.Now()) + if !deadline.IsZero() { + if d, ok := ctx.Deadline(); !ok || deadline.Before(d) { + subCtx, cancel := context.WithDeadline(ctx, deadline) + defer cancel() + ctx = subCtx + } + } + if oldCancel := d.Cancel; oldCancel != nil { + subCtx, cancel := context.WithCancel(ctx) + defer cancel() + go func() { + select { + case <-oldCancel: + cancel() + case <-subCtx.Done(): + } + }() + ctx = subCtx + } + + // Shadow the nettrace (if any) during resolve so Connect events don't fire for DNS lookups. + resolveCtx := ctx + if trace, _ := ctx.Value(nettrace.TraceKey{}).(*nettrace.Trace); trace != nil { + shadow := *trace + shadow.ConnectStart = nil + shadow.ConnectDone = nil + resolveCtx = context.WithValue(resolveCtx, nettrace.TraceKey{}, &shadow) + } + + addrs, err := d.resolver().resolveAddrList(resolveCtx, "dial", network, address, d.LocalAddr) if err != nil { return nil, &OpError{Op: "dial", Net: network, Source: nil, Addr: nil, Err: err} } - ctx := &dialContext{ - Dialer: *d, - network: network, - address: address, - finalDeadline: finalDeadline, + dp := &dialParam{ + Dialer: *d, + network: network, + address: address, } var primaries, fallbacks addrList @@ -233,116 +365,128 @@ func (d *Dialer) Dial(network, address string) (Conn, error) { } var c Conn - if len(fallbacks) == 0 { - // dialParallel can accept an empty fallbacks list, - // but this shortcut avoids the goroutine/channel overhead. - c, err = dialSerial(ctx, primaries, nil) + if len(fallbacks) > 0 { + c, err = dialParallel(ctx, dp, primaries, fallbacks) } else { - c, err = dialParallel(ctx, primaries, fallbacks) + c, err = dialSerial(ctx, dp, primaries) + } + if err != nil { + return nil, err } - if d.KeepAlive > 0 && err == nil { - if tc, ok := c.(*TCPConn); ok { - setKeepAlive(tc.fd, true) - setKeepAlivePeriod(tc.fd, d.KeepAlive) - testHookSetKeepAlive() - } + if tc, ok := c.(*TCPConn); ok && d.KeepAlive > 0 { + setKeepAlive(tc.fd, true) + setKeepAlivePeriod(tc.fd, d.KeepAlive) + testHookSetKeepAlive() } - return c, err + return c, nil } // dialParallel races two copies of dialSerial, giving the first a // head start. It returns the first established connection and // closes the others. Otherwise it returns an error from the first // primary address. -func dialParallel(ctx *dialContext, primaries, fallbacks addrList) (Conn, error) { - results := make(chan dialResult) // unbuffered, so dialSerialAsync can detect race loss & cleanup - cancel := make(chan struct{}) - defer close(cancel) - - // Spawn the primary racer. - go dialSerialAsync(ctx, primaries, nil, cancel, results) - - // Spawn the fallback racer. - fallbackTimer := time.NewTimer(ctx.fallbackDelay()) - go dialSerialAsync(ctx, fallbacks, fallbackTimer, cancel, results) - - var primaryErr error - for nracers := 2; nracers > 0; nracers-- { - res := <-results - // If we're still waiting for a connection, then hasten the delay. - // Otherwise, disable the Timer and let cancel take over. - if fallbackTimer.Stop() && res.error != nil { - fallbackTimer.Reset(0) - } - if res.error == nil { - return res.Conn, nil - } - if res.primary { - primaryErr = res.error - } +func dialParallel(ctx context.Context, dp *dialParam, primaries, fallbacks addrList) (Conn, error) { + if len(fallbacks) == 0 { + return dialSerial(ctx, dp, primaries) } - return nil, primaryErr -} -type dialResult struct { - Conn - error - primary bool -} + returned := make(chan struct{}) + defer close(returned) + + type dialResult struct { + Conn + error + primary bool + done bool + } + results := make(chan dialResult) // unbuffered -// dialSerialAsync runs dialSerial after some delay, and returns the -// resulting connection through a channel. When racing two connections, -// the primary goroutine uses a nil timer to omit the delay. -func dialSerialAsync(ctx *dialContext, ras addrList, timer *time.Timer, cancel <-chan struct{}, results chan<- dialResult) { - if timer != nil { - // We're in the fallback goroutine; sleep before connecting. + startRacer := func(ctx context.Context, primary bool) { + ras := primaries + if !primary { + ras = fallbacks + } + c, err := dialSerial(ctx, dp, ras) select { - case <-timer.C: - case <-cancel: - return + case results <- dialResult{Conn: c, error: err, primary: primary, done: true}: + case <-returned: + if c != nil { + c.Close() + } } } - c, err := dialSerial(ctx, ras, cancel) - select { - case results <- dialResult{c, err, timer == nil}: - // We won the race. - case <-cancel: - // The other goroutine won the race. - if c != nil { - c.Close() + + var primary, fallback dialResult + + // Start the main racer. + primaryCtx, primaryCancel := context.WithCancel(ctx) + defer primaryCancel() + go startRacer(primaryCtx, true) + + // Start the timer for the fallback racer. + fallbackTimer := time.NewTimer(dp.fallbackDelay()) + defer fallbackTimer.Stop() + + for { + select { + case <-fallbackTimer.C: + fallbackCtx, fallbackCancel := context.WithCancel(ctx) + defer fallbackCancel() + go startRacer(fallbackCtx, false) + + case res := <-results: + if res.error == nil { + return res.Conn, nil + } + if res.primary { + primary = res + } else { + fallback = res + } + if primary.done && fallback.done { + return nil, primary.error + } + if res.primary && fallbackTimer.Stop() { + // If we were able to stop the timer, that means it + // was running (hadn't yet started the fallback), but + // we just got an error on the primary path, so start + // the fallback immediately (in 0 nanoseconds). + fallbackTimer.Reset(0) + } } } } // dialSerial connects to a list of addresses in sequence, returning // either the first successful connection, or the first error. -func dialSerial(ctx *dialContext, ras addrList, cancel <-chan struct{}) (Conn, error) { +func dialSerial(ctx context.Context, dp *dialParam, ras addrList) (Conn, error) { var firstErr error // The error from the first address is most relevant. for i, ra := range ras { select { - case <-cancel: - return nil, &OpError{Op: "dial", Net: ctx.network, Source: ctx.LocalAddr, Addr: ra, Err: errCanceled} + case <-ctx.Done(): + return nil, &OpError{Op: "dial", Net: dp.network, Source: dp.LocalAddr, Addr: ra, Err: mapErr(ctx.Err())} default: } - partialDeadline, err := partialDeadline(time.Now(), ctx.finalDeadline, len(ras)-i) + deadline, _ := ctx.Deadline() + partialDeadline, err := partialDeadline(time.Now(), deadline, len(ras)-i) if err != nil { // Ran out of time. if firstErr == nil { - firstErr = &OpError{Op: "dial", Net: ctx.network, Source: ctx.LocalAddr, Addr: ra, Err: err} + firstErr = &OpError{Op: "dial", Net: dp.network, Source: dp.LocalAddr, Addr: ra, Err: err} } break } - - // dialTCP does not support cancelation (see golang.org/issue/11225), - // so if cancel fires, we'll continue trying to connect until the next - // timeout, or return a spurious connection for the caller to close. - dialer := func(d time.Time) (Conn, error) { - return dialSingle(ctx, ra, d) + dialCtx := ctx + if partialDeadline.Before(deadline) { + var cancel context.CancelFunc + dialCtx, cancel = context.WithDeadline(ctx, partialDeadline) + defer cancel() } - c, err := dial(ctx.network, ra, dialer, partialDeadline) + + c, err := dialSingle(dialCtx, dp, ra) if err == nil { return c, nil } @@ -352,37 +496,43 @@ func dialSerial(ctx *dialContext, ras addrList, cancel <-chan struct{}) (Conn, e } if firstErr == nil { - firstErr = &OpError{Op: "dial", Net: ctx.network, Source: nil, Addr: nil, Err: errMissingAddress} + firstErr = &OpError{Op: "dial", Net: dp.network, Source: nil, Addr: nil, Err: errMissingAddress} } return nil, firstErr } // dialSingle attempts to establish and returns a single connection to -// the destination address. This must be called through the OS-specific -// dial function, because some OSes don't implement the deadline feature. -func dialSingle(ctx *dialContext, ra Addr, deadline time.Time) (c Conn, err error) { - la := ctx.LocalAddr - if la != nil && la.Network() != ra.Network() { - return nil, &OpError{Op: "dial", Net: ctx.network, Source: la, Addr: ra, Err: errors.New("mismatched local address type " + la.Network())} +// the destination address. +func dialSingle(ctx context.Context, dp *dialParam, ra Addr) (c Conn, err error) { + trace, _ := ctx.Value(nettrace.TraceKey{}).(*nettrace.Trace) + if trace != nil { + raStr := ra.String() + if trace.ConnectStart != nil { + trace.ConnectStart(dp.network, raStr) + } + if trace.ConnectDone != nil { + defer func() { trace.ConnectDone(dp.network, raStr, err) }() + } } + la := dp.LocalAddr switch ra := ra.(type) { case *TCPAddr: la, _ := la.(*TCPAddr) - c, err = testHookDialTCP(ctx.network, la, ra, deadline, ctx.Cancel) + c, err = dialTCP(ctx, dp.network, la, ra) case *UDPAddr: la, _ := la.(*UDPAddr) - c, err = dialUDP(ctx.network, la, ra, deadline) + c, err = dialUDP(ctx, dp.network, la, ra) case *IPAddr: la, _ := la.(*IPAddr) - c, err = dialIP(ctx.network, la, ra, deadline) + c, err = dialIP(ctx, dp.network, la, ra) case *UnixAddr: la, _ := la.(*UnixAddr) - c, err = dialUnix(ctx.network, la, ra, deadline) + c, err = dialUnix(ctx, dp.network, la, ra) default: - return nil, &OpError{Op: "dial", Net: ctx.network, Source: la, Addr: ra, Err: &AddrError{Err: "unexpected address type", Addr: ctx.address}} + return nil, &OpError{Op: "dial", Net: dp.network, Source: la, Addr: ra, Err: &AddrError{Err: "unexpected address type", Addr: dp.address}} } if err != nil { - return nil, err // c is non-nil interface containing nil pointer + return nil, &OpError{Op: "dial", Net: dp.network, Source: la, Addr: ra, Err: err} // c is non-nil interface containing nil pointer } return c, nil } @@ -394,8 +544,11 @@ func dialSingle(ctx *dialContext, ra Addr, deadline time.Time) (c Conn, err erro // If host is omitted, as in ":8080", Listen listens on all available interfaces // instead of just the interface with the given host address. // See Dial for more details about address syntax. +// +// Listening on a hostname is not recommended because this creates a socket +// for at most one of its IP addresses. func Listen(net, laddr string) (Listener, error) { - addrs, err := resolveAddrList("listen", net, laddr, noDeadline) + addrs, err := DefaultResolver.resolveAddrList(context.Background(), "listen", net, laddr, nil) if err != nil { return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: nil, Err: err} } @@ -421,8 +574,11 @@ func Listen(net, laddr string) (Listener, error) { // If host is omitted, as in ":8080", ListenPacket listens on all available interfaces // instead of just the interface with the given host address. // See Dial for the syntax of laddr. +// +// Listening on a hostname is not recommended because this creates a socket +// for at most one of its IP addresses. func ListenPacket(net, laddr string) (PacketConn, error) { - addrs, err := resolveAddrList("listen", net, laddr, noDeadline) + addrs, err := DefaultResolver.resolveAddrList(context.Background(), "listen", net, laddr, nil) if err != nil { return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: nil, Err: err} } diff --git a/libgo/go/net/dial_gen.go b/libgo/go/net/dial_gen.go deleted file mode 100644 index a628f71483..0000000000 --- a/libgo/go/net/dial_gen.go +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2012 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build windows plan9 - -package net - -import "time" - -// dialChannel is the simple pure-Go implementation of dial, still -// used on operating systems where the deadline hasn't been pushed -// down into the pollserver. (Plan 9 and some old versions of Windows) -func dialChannel(net string, ra Addr, dialer func(time.Time) (Conn, error), deadline time.Time) (Conn, error) { - if deadline.IsZero() { - return dialer(noDeadline) - } - timeout := deadline.Sub(time.Now()) - if timeout <= 0 { - return nil, &OpError{Op: "dial", Net: net, Source: nil, Addr: ra, Err: errTimeout} - } - t := time.NewTimer(timeout) - defer t.Stop() - type racer struct { - Conn - error - } - ch := make(chan racer, 1) - go func() { - testHookDialChannel() - c, err := dialer(noDeadline) - ch <- racer{c, err} - }() - select { - case <-t.C: - return nil, &OpError{Op: "dial", Net: net, Source: nil, Addr: ra, Err: errTimeout} - case racer := <-ch: - return racer.Conn, racer.error - } -} diff --git a/libgo/go/net/dial_test.go b/libgo/go/net/dial_test.go index 2311b10824..9919d72ce3 100644 --- a/libgo/go/net/dial_test.go +++ b/libgo/go/net/dial_test.go @@ -5,6 +5,8 @@ package net import ( + "bufio" + "context" "internal/testenv" "io" "net/internal/socktest" @@ -23,13 +25,12 @@ var prohibitionaryDialArgTests = []struct { } func TestProhibitionaryDialArg(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + switch runtime.GOOS { case "plan9": t.Skipf("not supported on %s", runtime.GOOS) } - if testing.Short() || !*testExternal { - t.Skip("avoid external network") - } if !supportsIPv4map { t.Skip("mapping ipv4 address inside ipv6 address not supported") } @@ -54,56 +55,29 @@ func TestProhibitionaryDialArg(t *testing.T) { } } -func TestSelfConnect(t *testing.T) { - if runtime.GOOS == "windows" { - // TODO(brainman): do not know why it hangs. - t.Skip("known-broken test on windows") +func TestDialLocal(t *testing.T) { + ln, err := newLocalListener("tcp") + if err != nil { + t.Fatal(err) } - - // Test that Dial does not honor self-connects. - // See the comment in DialTCP. - - // Find a port that would be used as a local address. - l, err := Listen("tcp", "127.0.0.1:0") + defer ln.Close() + _, port, err := SplitHostPort(ln.Addr().String()) if err != nil { t.Fatal(err) } - c, err := Dial("tcp", l.Addr().String()) + c, err := Dial("tcp", JoinHostPort("", port)) if err != nil { t.Fatal(err) } - addr := c.LocalAddr().String() c.Close() - l.Close() - - // Try to connect to that address repeatedly. - n := 100000 - if testing.Short() { - n = 1000 - } - switch runtime.GOOS { - case "darwin", "dragonfly", "freebsd", "netbsd", "openbsd", "plan9", "solaris", "windows": - // Non-Linux systems take a long time to figure - // out that there is nothing listening on localhost. - n = 100 - } - for i := 0; i < n; i++ { - c, err := DialTimeout("tcp", addr, time.Millisecond) - if err == nil { - if c.LocalAddr().String() == addr { - t.Errorf("#%d: Dial %q self-connect", i, addr) - } else { - t.Logf("#%d: Dial %q succeeded - possibly racing with other listener", i, addr) - } - c.Close() - } - } } func TestDialTimeoutFDLeak(t *testing.T) { switch runtime.GOOS { case "plan9": t.Skipf("%s does not have full support of socktest", runtime.GOOS) + case "openbsd": + testenv.SkipFlaky(t, 15157) } const T = 100 * time.Millisecond @@ -130,17 +104,14 @@ func TestDialTimeoutFDLeak(t *testing.T) { // socktest.Switch. // It may happen when the Dial call bumps against TCP // simultaneous open. See selfConnect in tcpsock_posix.go. - defer func() { - sw.Set(socktest.FilterClose, nil) - forceCloseSockets() - }() + defer func() { sw.Set(socktest.FilterClose, nil) }() var mu sync.Mutex var attempts int sw.Set(socktest.FilterClose, func(so *socktest.Status) (socktest.AfterFilter, error) { mu.Lock() attempts++ mu.Unlock() - return nil, errTimedout + return nil, nil }) const N = 100 @@ -171,11 +142,19 @@ func TestDialerDualStackFDLeak(t *testing.T) { t.Skipf("%s does not have full support of socktest", runtime.GOOS) case "windows": t.Skipf("not implemented a way to cancel dial racers in TCP SYN-SENT state on %s", runtime.GOOS) + case "openbsd": + testenv.SkipFlaky(t, 15157) } if !supportsIPv4 || !supportsIPv6 { t.Skip("both IPv4 and IPv6 are required") } + closedPortDelay, expectClosedPortDelay := dialClosedPort() + if closedPortDelay > expectClosedPortDelay { + t.Errorf("got %v; want <= %v", closedPortDelay, expectClosedPortDelay) + } + + before := sw.Sockets() origTestHookLookupIP := testHookLookupIP defer func() { testHookLookupIP = origTestHookLookupIP }() testHookLookupIP = lookupLocalhost @@ -188,24 +167,19 @@ func TestDialerDualStackFDLeak(t *testing.T) { c.Close() } } - dss, err := newDualStackServer([]streamListener{ - {network: "tcp4", address: "127.0.0.1"}, - {network: "tcp6", address: "::1"}, - }) + dss, err := newDualStackServer() if err != nil { t.Fatal(err) } - defer dss.teardown() if err := dss.buildup(handler); err != nil { + dss.teardown() t.Fatal(err) } - before := sw.Sockets() - const T = 100 * time.Millisecond const N = 10 var wg sync.WaitGroup wg.Add(N) - d := &Dialer{DualStack: true, Timeout: T} + d := &Dialer{DualStack: true, Timeout: 100*time.Millisecond + closedPortDelay} for i := 0; i < N; i++ { go func() { defer wg.Done() @@ -218,7 +192,7 @@ func TestDialerDualStackFDLeak(t *testing.T) { }() } wg.Wait() - time.Sleep(2 * T) // wait for the dial racers to stop + dss.teardown() after := sw.Sockets() if len(after) != len(before) { t.Errorf("got %d; want %d", len(after), len(before)) @@ -229,18 +203,18 @@ func TestDialerDualStackFDLeak(t *testing.T) { // expected to hang until the timeout elapses. These addresses are reserved // for benchmarking by RFC 6890. const ( - slowDst4 = "192.18.0.254" - slowDst6 = "2001:2::254" - slowTimeout = 1 * time.Second + slowDst4 = "198.18.0.254" + slowDst6 = "2001:2::254" ) // In some environments, the slow IPs may be explicitly unreachable, and fail // more quickly than expected. This test hook prevents dialTCP from returning // before the deadline. -func slowDialTCP(net string, laddr, raddr *TCPAddr, deadline time.Time, cancel <-chan struct{}) (*TCPConn, error) { - c, err := dialTCP(net, laddr, raddr, deadline, cancel) +func slowDialTCP(ctx context.Context, net string, laddr, raddr *TCPAddr) (*TCPConn, error) { + c, err := doDialTCP(ctx, net, laddr, raddr) if ParseIP(slowDst4).Equal(raddr.IP) || ParseIP(slowDst6).Equal(raddr.IP) { - time.Sleep(deadline.Sub(time.Now())) + // Wait for the deadline, or indefinitely if none exists. + <-ctx.Done() } return c, err } @@ -278,9 +252,8 @@ func dialClosedPort() (actual, expected time.Duration) { } func TestDialParallel(t *testing.T) { - if testing.Short() || !*testExternal { - t.Skip("avoid external network") - } + testenv.MustHaveExternalNetwork(t) + if !supportsIPv4 || !supportsIPv6 { t.Skip("both IPv4 and IPv6 are required") } @@ -294,7 +267,7 @@ func TestDialParallel(t *testing.T) { const fallbackDelay = 200 * time.Millisecond // Some cases will run quickly when "connection refused" is fast, - // or trigger the fallbackDelay on Windows. This value holds the + // or trigger the fallbackDelay on Windows. This value holds the // lesser of the two delays. var closedPortOrFallbackDelay time.Duration if closedPortDelay < fallbackDelay { @@ -369,10 +342,7 @@ func TestDialParallel(t *testing.T) { } for i, tt := range testCases { - dss, err := newDualStackServer([]streamListener{ - {network: "tcp4", address: "127.0.0.1"}, - {network: "tcp6", address: "::1"}, - }) + dss, err := newDualStackServer() if err != nil { t.Fatal(err) } @@ -389,17 +359,15 @@ func TestDialParallel(t *testing.T) { fallbacks := makeAddrs(tt.fallbacks, dss.port) d := Dialer{ FallbackDelay: fallbackDelay, - Timeout: slowTimeout, - } - ctx := &dialContext{ - Dialer: d, - network: "tcp", - address: "?", - finalDeadline: d.deadline(time.Now()), } startTime := time.Now() - c, err := dialParallel(ctx, primaries, fallbacks) - elapsed := time.Now().Sub(startTime) + dp := &dialParam{ + Dialer: d, + network: "tcp", + address: "?", + } + c, err := dialParallel(context.Background(), dp, primaries, fallbacks) + elapsed := time.Since(startTime) if c != nil { c.Close() @@ -418,12 +386,30 @@ func TestDialParallel(t *testing.T) { } else if !(elapsed <= expectElapsedMax) { t.Errorf("#%d: got %v; want <= %v", i, elapsed, expectElapsedMax) } + + // Repeat each case, ensuring that it can be canceled quickly. + ctx, cancel := context.WithCancel(context.Background()) + var wg sync.WaitGroup + wg.Add(1) + go func() { + time.Sleep(5 * time.Millisecond) + cancel() + wg.Done() + }() + startTime = time.Now() + c, err = dialParallel(ctx, dp, primaries, fallbacks) + if c != nil { + c.Close() + } + elapsed = time.Now().Sub(startTime) + if elapsed > 100*time.Millisecond { + t.Errorf("#%d (cancel): got %v; want <= 100ms", i, elapsed) + } + wg.Wait() } - // Wait for any slowDst4/slowDst6 connections to timeout. - time.Sleep(slowTimeout * 3 / 2) } -func lookupSlowFast(fn func(string) ([]IPAddr, error), host string) ([]IPAddr, error) { +func lookupSlowFast(ctx context.Context, fn func(context.Context, string) ([]IPAddr, error), host string) ([]IPAddr, error) { switch host { case "slow6loopback4": // Returns a slow IPv6 address, and a local IPv4 address. @@ -432,14 +418,13 @@ func lookupSlowFast(fn func(string) ([]IPAddr, error), host string) ([]IPAddr, e {IP: ParseIP("127.0.0.1")}, }, nil default: - return fn(host) + return fn(ctx, host) } } func TestDialerFallbackDelay(t *testing.T) { - if testing.Short() || !*testExternal { - t.Skip("avoid external network") - } + testenv.MustHaveExternalNetwork(t) + if !supportsIPv4 || !supportsIPv6 { t.Skip("both IPv4 and IPv6 are required") } @@ -463,8 +448,6 @@ func TestDialerFallbackDelay(t *testing.T) { {true, 200 * time.Millisecond, 200 * time.Millisecond}, // The default is 300ms. {true, 0, 300 * time.Millisecond}, - // This case is last, in order to wait for hanging slowDst6 connections. - {false, 0, slowTimeout}, } handler := func(dss *dualStackServer, ln Listener) { @@ -476,9 +459,7 @@ func TestDialerFallbackDelay(t *testing.T) { c.Close() } } - dss, err := newDualStackServer([]streamListener{ - {network: "tcp", address: "127.0.0.1"}, - }) + dss, err := newDualStackServer() if err != nil { t.Fatal(err) } @@ -488,7 +469,7 @@ func TestDialerFallbackDelay(t *testing.T) { } for i, tt := range testCases { - d := &Dialer{DualStack: tt.dualstack, FallbackDelay: tt.delay, Timeout: slowTimeout} + d := &Dialer{DualStack: tt.dualstack, FallbackDelay: tt.delay} startTime := time.Now() c, err := d.Dial("tcp", JoinHostPort("slow6loopback4", dss.port)) @@ -509,46 +490,78 @@ func TestDialerFallbackDelay(t *testing.T) { } } -func TestDialSerialAsyncSpuriousConnection(t *testing.T) { - if runtime.GOOS == "plan9" { - t.Skip("skipping on plan9; no deadline support, golang.org/issue/11932") +func TestDialParallelSpuriousConnection(t *testing.T) { + if !supportsIPv4 || !supportsIPv6 { + t.Skip("both IPv4 and IPv6 are required") } - ln, err := newLocalListener("tcp") + + var wg sync.WaitGroup + wg.Add(2) + handler := func(dss *dualStackServer, ln Listener) { + // Accept one connection per address. + c, err := ln.Accept() + if err != nil { + t.Fatal(err) + } + // The client should close itself, without sending data. + c.SetReadDeadline(time.Now().Add(1 * time.Second)) + var b [1]byte + if _, err := c.Read(b[:]); err != io.EOF { + t.Errorf("got %v; want %v", err, io.EOF) + } + c.Close() + wg.Done() + } + dss, err := newDualStackServer() if err != nil { t.Fatal(err) } - defer ln.Close() + defer dss.teardown() + if err := dss.buildup(handler); err != nil { + t.Fatal(err) + } + + const fallbackDelay = 100 * time.Millisecond + + origTestHookDialTCP := testHookDialTCP + defer func() { testHookDialTCP = origTestHookDialTCP }() + testHookDialTCP = func(ctx context.Context, net string, laddr, raddr *TCPAddr) (*TCPConn, error) { + // Sleep long enough for Happy Eyeballs to kick in, and inhibit cancelation. + // This forces dialParallel to juggle two successful connections. + time.Sleep(fallbackDelay * 2) - d := Dialer{} - ctx := &dialContext{ - Dialer: d, - network: "tcp", - address: "?", - finalDeadline: d.deadline(time.Now()), + // Now ignore the provided context (which will be canceled) and use a + // different one to make sure this completes with a valid connection, + // which we hope to be closed below: + return doDialTCP(context.Background(), net, laddr, raddr) } - results := make(chan dialResult) - cancel := make(chan struct{}) + d := Dialer{ + FallbackDelay: fallbackDelay, + } + dp := &dialParam{ + Dialer: d, + network: "tcp", + address: "?", + } - // Spawn a connection in the background. - go dialSerialAsync(ctx, addrList{ln.Addr()}, nil, cancel, results) + makeAddr := func(ip string) addrList { + addr, err := ResolveTCPAddr("tcp", JoinHostPort(ip, dss.port)) + if err != nil { + t.Fatal(err) + } + return addrList{addr} + } - // Receive it at the server. - c, err := ln.Accept() + // dialParallel returns one connection (and closes the other.) + c, err := dialParallel(context.Background(), dp, makeAddr("127.0.0.1"), makeAddr("::1")) if err != nil { t.Fatal(err) } - defer c.Close() - - // Tell dialSerialAsync that someone else won the race. - close(cancel) + c.Close() - // The connection should close itself, without sending data. - c.SetReadDeadline(time.Now().Add(1 * time.Second)) - var b [1]byte - if _, err := c.Read(b[:]); err != io.EOF { - t.Errorf("got %v; want %v", err, io.EOF) - } + // The server should've seen both connections. + wg.Wait() } func TestDialerPartialDeadline(t *testing.T) { @@ -587,44 +600,125 @@ func TestDialerPartialDeadline(t *testing.T) { } func TestDialerLocalAddr(t *testing.T) { - ch := make(chan error, 1) - handler := func(ls *localServer, ln Listener) { - c, err := ln.Accept() - if err != nil { - ch <- err - return - } - defer c.Close() - ch <- nil - } - ls, err := newLocalServer("tcp") - if err != nil { - t.Fatal(err) + if !supportsIPv4 || !supportsIPv6 { + t.Skip("both IPv4 and IPv6 are required") } - defer ls.teardown() - if err := ls.buildup(handler); err != nil { - t.Fatal(err) + + type test struct { + network, raddr string + laddr Addr + error + } + var tests = []test{ + {"tcp4", "127.0.0.1", nil, nil}, + {"tcp4", "127.0.0.1", &TCPAddr{}, nil}, + {"tcp4", "127.0.0.1", &TCPAddr{IP: ParseIP("0.0.0.0")}, nil}, + {"tcp4", "127.0.0.1", &TCPAddr{IP: ParseIP("0.0.0.0").To4()}, nil}, + {"tcp4", "127.0.0.1", &TCPAddr{IP: ParseIP("::")}, &AddrError{Err: "some error"}}, + {"tcp4", "127.0.0.1", &TCPAddr{IP: ParseIP("127.0.0.1").To4()}, nil}, + {"tcp4", "127.0.0.1", &TCPAddr{IP: ParseIP("127.0.0.1").To16()}, nil}, + {"tcp4", "127.0.0.1", &TCPAddr{IP: IPv6loopback}, errNoSuitableAddress}, + {"tcp4", "127.0.0.1", &UDPAddr{}, &AddrError{Err: "some error"}}, + {"tcp4", "127.0.0.1", &UnixAddr{}, &AddrError{Err: "some error"}}, + + {"tcp6", "::1", nil, nil}, + {"tcp6", "::1", &TCPAddr{}, nil}, + {"tcp6", "::1", &TCPAddr{IP: ParseIP("0.0.0.0")}, nil}, + {"tcp6", "::1", &TCPAddr{IP: ParseIP("0.0.0.0").To4()}, nil}, + {"tcp6", "::1", &TCPAddr{IP: ParseIP("::")}, nil}, + {"tcp6", "::1", &TCPAddr{IP: ParseIP("127.0.0.1").To4()}, errNoSuitableAddress}, + {"tcp6", "::1", &TCPAddr{IP: ParseIP("127.0.0.1").To16()}, errNoSuitableAddress}, + {"tcp6", "::1", &TCPAddr{IP: IPv6loopback}, nil}, + {"tcp6", "::1", &UDPAddr{}, &AddrError{Err: "some error"}}, + {"tcp6", "::1", &UnixAddr{}, &AddrError{Err: "some error"}}, + + {"tcp", "127.0.0.1", nil, nil}, + {"tcp", "127.0.0.1", &TCPAddr{}, nil}, + {"tcp", "127.0.0.1", &TCPAddr{IP: ParseIP("0.0.0.0")}, nil}, + {"tcp", "127.0.0.1", &TCPAddr{IP: ParseIP("0.0.0.0").To4()}, nil}, + {"tcp", "127.0.0.1", &TCPAddr{IP: ParseIP("127.0.0.1").To4()}, nil}, + {"tcp", "127.0.0.1", &TCPAddr{IP: ParseIP("127.0.0.1").To16()}, nil}, + {"tcp", "127.0.0.1", &TCPAddr{IP: IPv6loopback}, errNoSuitableAddress}, + {"tcp", "127.0.0.1", &UDPAddr{}, &AddrError{Err: "some error"}}, + {"tcp", "127.0.0.1", &UnixAddr{}, &AddrError{Err: "some error"}}, + + {"tcp", "::1", nil, nil}, + {"tcp", "::1", &TCPAddr{}, nil}, + {"tcp", "::1", &TCPAddr{IP: ParseIP("0.0.0.0")}, nil}, + {"tcp", "::1", &TCPAddr{IP: ParseIP("0.0.0.0").To4()}, nil}, + {"tcp", "::1", &TCPAddr{IP: ParseIP("::")}, nil}, + {"tcp", "::1", &TCPAddr{IP: ParseIP("127.0.0.1").To4()}, errNoSuitableAddress}, + {"tcp", "::1", &TCPAddr{IP: ParseIP("127.0.0.1").To16()}, errNoSuitableAddress}, + {"tcp", "::1", &TCPAddr{IP: IPv6loopback}, nil}, + {"tcp", "::1", &UDPAddr{}, &AddrError{Err: "some error"}}, + {"tcp", "::1", &UnixAddr{}, &AddrError{Err: "some error"}}, + } + + if supportsIPv4map { + tests = append(tests, test{ + "tcp", "127.0.0.1", &TCPAddr{IP: ParseIP("::")}, nil, + }) + } else { + tests = append(tests, test{ + "tcp", "127.0.0.1", &TCPAddr{IP: ParseIP("::")}, &AddrError{Err: "some error"}, + }) } - laddr, err := ResolveTCPAddr(ls.Listener.Addr().Network(), ls.Listener.Addr().String()) - if err != nil { - t.Fatal(err) + origTestHookLookupIP := testHookLookupIP + defer func() { testHookLookupIP = origTestHookLookupIP }() + testHookLookupIP = lookupLocalhost + handler := func(ls *localServer, ln Listener) { + for { + c, err := ln.Accept() + if err != nil { + return + } + c.Close() + } } - laddr.Port = 0 - d := &Dialer{LocalAddr: laddr} - c, err := d.Dial(ls.Listener.Addr().Network(), ls.Addr().String()) - if err != nil { - t.Fatal(err) + var err error + var lss [2]*localServer + for i, network := range []string{"tcp4", "tcp6"} { + lss[i], err = newLocalServer(network) + if err != nil { + t.Fatal(err) + } + defer lss[i].teardown() + if err := lss[i].buildup(handler); err != nil { + t.Fatal(err) + } } - defer c.Close() - c.Read(make([]byte, 1)) - err = <-ch - if err != nil { - t.Error(err) + + for _, tt := range tests { + d := &Dialer{LocalAddr: tt.laddr} + var addr string + ip := ParseIP(tt.raddr) + if ip.To4() != nil { + addr = lss[0].Listener.Addr().String() + } + if ip.To16() != nil && ip.To4() == nil { + addr = lss[1].Listener.Addr().String() + } + c, err := d.Dial(tt.network, addr) + if err == nil && tt.error != nil || err != nil && tt.error == nil { + t.Errorf("%s %v->%s: got %v; want %v", tt.network, tt.laddr, tt.raddr, err, tt.error) + } + if err != nil { + if perr := parseDialError(err); perr != nil { + t.Error(perr) + } + continue + } + c.Close() } } func TestDialerDualStack(t *testing.T) { + // This test is known to be flaky. Don't frighten regular + // users about it; only fail on the build dashboard. + if testenv.Builder() == "" { + testenv.SkipFlaky(t, 13324) + } if !supportsIPv4 || !supportsIPv6 { t.Skip("both IPv4 and IPv6 are required") } @@ -649,10 +743,7 @@ func TestDialerDualStack(t *testing.T) { var timeout = 150*time.Millisecond + closedPortDelay for _, dualstack := range []bool{false, true} { - dss, err := newDualStackServer([]streamListener{ - {network: "tcp4", address: "127.0.0.1"}, - {network: "tcp6", address: "::1"}, - }) + dss, err := newDualStackServer() if err != nil { t.Fatal(err) } @@ -677,7 +768,6 @@ func TestDialerDualStack(t *testing.T) { c.Close() } } - time.Sleep(timeout * 3 / 2) // wait for the dial racers to stop } func TestDialerKeepAlive(t *testing.T) { @@ -719,14 +809,16 @@ func TestDialerKeepAlive(t *testing.T) { } func TestDialCancel(t *testing.T) { - if runtime.GOOS == "plan9" || runtime.GOOS == "nacl" { - // plan9 is not implemented and nacl doesn't have - // external network access. - t.Skipf("skipping on %s", runtime.GOOS) + switch testenv.Builder() { + case "linux-arm64-buildlet": + t.Skip("skipping on linux-arm64-buildlet; incompatible network config? issue 15191") + case "": + testenv.MustHaveExternalNetwork(t) } - onGoBuildFarm := testenv.Builder() != "" - if testing.Short() && !onGoBuildFarm { - t.Skip("skipping in short mode") + + if runtime.GOOS == "nacl" { + // nacl doesn't have external network access. + t.Skipf("skipping on %s", runtime.GOOS) } blackholeIPPort := JoinHostPort(slowDst4, "1234") @@ -781,3 +873,84 @@ func TestDialCancel(t *testing.T) { } } } + +func TestCancelAfterDial(t *testing.T) { + if testing.Short() { + t.Skip("avoiding time.Sleep") + } + + ln, err := newLocalListener("tcp") + if err != nil { + t.Fatal(err) + } + + var wg sync.WaitGroup + wg.Add(1) + defer func() { + ln.Close() + wg.Wait() + }() + + // Echo back the first line of each incoming connection. + go func() { + for { + c, err := ln.Accept() + if err != nil { + break + } + rb := bufio.NewReader(c) + line, err := rb.ReadString('\n') + if err != nil { + t.Error(err) + c.Close() + continue + } + if _, err := c.Write([]byte(line)); err != nil { + t.Error(err) + } + c.Close() + } + wg.Done() + }() + + try := func() { + cancel := make(chan struct{}) + d := &Dialer{Cancel: cancel} + c, err := d.Dial("tcp", ln.Addr().String()) + + // Immediately after dialing, request cancelation and sleep. + // Before Issue 15078 was fixed, this would cause subsequent operations + // to fail with an i/o timeout roughly 50% of the time. + close(cancel) + time.Sleep(10 * time.Millisecond) + + if err != nil { + t.Fatal(err) + } + defer c.Close() + + // Send some data to confirm that the connection is still alive. + const message = "echo!\n" + if _, err := c.Write([]byte(message)); err != nil { + t.Fatal(err) + } + + // The server should echo the line, and close the connection. + rb := bufio.NewReader(c) + line, err := rb.ReadString('\n') + if err != nil { + t.Fatal(err) + } + if line != message { + t.Errorf("got %q; want %q", line, message) + } + if _, err := rb.ReadByte(); err != io.EOF { + t.Errorf("got %v; want %v", err, io.EOF) + } + } + + // This bug manifested about 50% of the time, so try it a few times. + for i := 0; i < 10; i++ { + try() + } +} diff --git a/libgo/go/net/dial_unix_test.go b/libgo/go/net/dial_unix_test.go new file mode 100644 index 0000000000..4705254728 --- /dev/null +++ b/libgo/go/net/dial_unix_test.go @@ -0,0 +1,108 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux netbsd openbsd solaris + +package net + +import ( + "context" + "syscall" + "testing" + "time" +) + +// Issue 16523 +func TestDialContextCancelRace(t *testing.T) { + oldConnectFunc := connectFunc + oldGetsockoptIntFunc := getsockoptIntFunc + oldTestHookCanceledDial := testHookCanceledDial + defer func() { + connectFunc = oldConnectFunc + getsockoptIntFunc = oldGetsockoptIntFunc + testHookCanceledDial = oldTestHookCanceledDial + }() + + ln, err := newLocalListener("tcp") + if err != nil { + t.Fatal(err) + } + listenerDone := make(chan struct{}) + go func() { + defer close(listenerDone) + c, err := ln.Accept() + if err == nil { + c.Close() + } + }() + defer func() { <-listenerDone }() + defer ln.Close() + + sawCancel := make(chan bool, 1) + testHookCanceledDial = func() { + sawCancel <- true + } + + ctx, cancelCtx := context.WithCancel(context.Background()) + + connectFunc = func(fd int, addr syscall.Sockaddr) error { + err := oldConnectFunc(fd, addr) + t.Logf("connect(%d, addr) = %v", fd, err) + if err == nil { + // On some operating systems, localhost + // connects _sometimes_ succeed immediately. + // Prevent that, so we exercise the code path + // we're interested in testing. This seems + // harmless. It makes FreeBSD 10.10 work when + // run with many iterations. It failed about + // half the time previously. + return syscall.EINPROGRESS + } + return err + } + + getsockoptIntFunc = func(fd, level, opt int) (val int, err error) { + val, err = oldGetsockoptIntFunc(fd, level, opt) + t.Logf("getsockoptIntFunc(%d, %d, %d) = (%v, %v)", fd, level, opt, val, err) + if level == syscall.SOL_SOCKET && opt == syscall.SO_ERROR && err == nil && val == 0 { + t.Logf("canceling context") + + // Cancel the context at just the moment which + // caused the race in issue 16523. + cancelCtx() + + // And wait for the "interrupter" goroutine to + // cancel the dial by messing with its write + // timeout before returning. + select { + case <-sawCancel: + t.Logf("saw cancel") + case <-time.After(5 * time.Second): + t.Errorf("didn't see cancel after 5 seconds") + } + } + return + } + + var d Dialer + c, err := d.DialContext(ctx, "tcp", ln.Addr().String()) + if err == nil { + c.Close() + t.Fatal("unexpected successful dial; want context canceled error") + } + + select { + case <-ctx.Done(): + case <-time.After(5 * time.Second): + t.Fatal("expected context to be canceled") + } + + oe, ok := err.(*OpError) + if !ok || oe.Op != "dial" { + t.Fatalf("Dial error = %#v; want dial *OpError", err) + } + if oe.Err != ctx.Err() { + t.Errorf("DialContext = (%v, %v); want OpError with error %v", c, err, ctx.Err()) + } +} diff --git a/libgo/go/net/dnsclient.go b/libgo/go/net/dnsclient.go index 5dc2a0368c..2ab5639d62 100644 --- a/libgo/go/net/dnsclient.go +++ b/libgo/go/net/dnsclient.go @@ -45,7 +45,7 @@ func answer(name, server string, dns *dnsMsg, qtype uint16) (cname string, addrs } if dns.rcode != dnsRcodeSuccess { // None of the error codes make sense - // for the query we sent. If we didn't get + // for the query we sent. If we didn't get // a name error and we didn't get success, // the server is behaving incorrectly or // having temporary trouble. @@ -113,12 +113,20 @@ func equalASCIILabel(x, y string) bool { return true } +// isDomainName checks if a string is a presentation-format domain name +// (currently restricted to hostname-compatible "preferred name" LDH labels and +// SRV-like "underscore labels"; see golang.org/issue/12421). func isDomainName(s string) bool { // See RFC 1035, RFC 3696. - if len(s) == 0 { - return false - } - if len(s) > 255 { + // Presentation format has dots before every label except the first, and the + // terminal empty label is optional here because we assume fully-qualified + // (absolute) input. We must therefore reserve space for the first and last + // labels' length octets in wire format, where they are necessary and the + // maximum total length is 255. + // So our _effective_ maximum is 253, but 254 is not rejected if the last + // character is a dot. + l := len(s) + if l == 0 || l > 254 || l == 254 && s[l-1] != '.' { return false } @@ -161,7 +169,7 @@ func isDomainName(s string) bool { return ok } -// absDomainName returns an absoulte domain name which ends with a +// absDomainName returns an absolute domain name which ends with a // trailing dot to match pure Go reverse resolver and all other lookup // routines. // See golang.org/issue/12189. diff --git a/libgo/go/net/dnsclient_unix.go b/libgo/go/net/dnsclient_unix.go index 17188f0024..4dd4e16b0f 100644 --- a/libgo/go/net/dnsclient_unix.go +++ b/libgo/go/net/dnsclient_unix.go @@ -16,6 +16,7 @@ package net import ( + "context" "errors" "io" "math/rand" @@ -26,10 +27,10 @@ import ( // A dnsDialer provides dialing suitable for DNS queries. type dnsDialer interface { - dialDNS(string, string) (dnsConn, error) + dialDNS(ctx context.Context, network, addr string) (dnsConn, error) } -var testHookDNSDialer = func(d time.Duration) dnsDialer { return &Dialer{Timeout: d} } +var testHookDNSDialer = func() dnsDialer { return &Dialer{} } // A dnsConn represents a DNS transport endpoint. type dnsConn interface { @@ -37,46 +38,67 @@ type dnsConn interface { SetDeadline(time.Time) error - // readDNSResponse reads a DNS response message from the DNS - // transport endpoint and returns the received DNS response - // message. - readDNSResponse() (*dnsMsg, error) + // dnsRoundTrip executes a single DNS transaction, returning a + // DNS response message for the provided DNS query message. + dnsRoundTrip(query *dnsMsg) (*dnsMsg, error) +} - // writeDNSQuery writes a DNS query message to the DNS - // connection endpoint. - writeDNSQuery(*dnsMsg) error +func (c *UDPConn) dnsRoundTrip(query *dnsMsg) (*dnsMsg, error) { + return dnsRoundTripUDP(c, query) } -func (c *UDPConn) readDNSResponse() (*dnsMsg, error) { - b := make([]byte, 512) // see RFC 1035 - n, err := c.Read(b) - if err != nil { +// dnsRoundTripUDP implements the dnsRoundTrip interface for RFC 1035's +// "UDP usage" transport mechanism. c should be a packet-oriented connection, +// such as a *UDPConn. +func dnsRoundTripUDP(c io.ReadWriter, query *dnsMsg) (*dnsMsg, error) { + b, ok := query.Pack() + if !ok { + return nil, errors.New("cannot marshal DNS message") + } + if _, err := c.Write(b); err != nil { return nil, err } - msg := &dnsMsg{} - if !msg.Unpack(b[:n]) { - return nil, errors.New("cannot unmarshal DNS message") + + b = make([]byte, 512) // see RFC 1035 + for { + n, err := c.Read(b) + if err != nil { + return nil, err + } + resp := &dnsMsg{} + if !resp.Unpack(b[:n]) || !resp.IsResponseTo(query) { + // Ignore invalid responses as they may be malicious + // forgery attempts. Instead continue waiting until + // timeout. See golang.org/issue/13281. + continue + } + return resp, nil } - return msg, nil } -func (c *UDPConn) writeDNSQuery(msg *dnsMsg) error { - b, ok := msg.Pack() +func (c *TCPConn) dnsRoundTrip(out *dnsMsg) (*dnsMsg, error) { + return dnsRoundTripTCP(c, out) +} + +// dnsRoundTripTCP implements the dnsRoundTrip interface for RFC 1035's +// "TCP usage" transport mechanism. c should be a stream-oriented connection, +// such as a *TCPConn. +func dnsRoundTripTCP(c io.ReadWriter, query *dnsMsg) (*dnsMsg, error) { + b, ok := query.Pack() if !ok { - return errors.New("cannot marshal DNS message") + return nil, errors.New("cannot marshal DNS message") } + l := len(b) + b = append([]byte{byte(l >> 8), byte(l)}, b...) if _, err := c.Write(b); err != nil { - return err + return nil, err } - return nil -} -func (c *TCPConn) readDNSResponse() (*dnsMsg, error) { - b := make([]byte, 1280) // 1280 is a reasonable initial size for IP over Ethernet, see RFC 4035 + b = make([]byte, 1280) // 1280 is a reasonable initial size for IP over Ethernet, see RFC 4035 if _, err := io.ReadFull(c, b[:2]); err != nil { return nil, err } - l := int(b[0])<<8 | int(b[1]) + l = int(b[0])<<8 | int(b[1]) if l > len(b) { b = make([]byte, l) } @@ -84,27 +106,17 @@ func (c *TCPConn) readDNSResponse() (*dnsMsg, error) { if err != nil { return nil, err } - msg := &dnsMsg{} - if !msg.Unpack(b[:n]) { + resp := &dnsMsg{} + if !resp.Unpack(b[:n]) { return nil, errors.New("cannot unmarshal DNS message") } - return msg, nil -} - -func (c *TCPConn) writeDNSQuery(msg *dnsMsg) error { - b, ok := msg.Pack() - if !ok { - return errors.New("cannot marshal DNS message") - } - l := uint16(len(b)) - b = append([]byte{byte(l >> 8), byte(l)}, b...) - if _, err := c.Write(b); err != nil { - return err + if !resp.IsResponseTo(query) { + return nil, errors.New("invalid DNS response") } - return nil + return resp, nil } -func (d *Dialer) dialDNS(network, server string) (dnsConn, error) { +func (d *Dialer) dialDNS(ctx context.Context, network, server string) (dnsConn, error) { switch network { case "tcp", "tcp4", "tcp6", "udp", "udp4", "udp6": default: @@ -113,11 +125,11 @@ func (d *Dialer) dialDNS(network, server string) (dnsConn, error) { // Calling Dial here is scary -- we have to be sure not to // dial a name that will require a DNS lookup, or Dial will // call back here to translate it. The DNS config parser has - // already checked that all the cfg.servers[i] are IP + // already checked that all the cfg.servers are IP // addresses, which Dial will use without a DNS lookup. - c, err := d.Dial(network, server) + c, err := d.DialContext(ctx, network, server) if err != nil { - return nil, err + return nil, mapErr(err) } switch network { case "tcp", "tcp4", "tcp6": @@ -129,8 +141,8 @@ func (d *Dialer) dialDNS(network, server string) (dnsConn, error) { } // exchange sends a query on the connection and hopes for a response. -func exchange(server, name string, qtype uint16, timeout time.Duration) (*dnsMsg, error) { - d := testHookDNSDialer(timeout) +func exchange(ctx context.Context, server, name string, qtype uint16, timeout time.Duration) (*dnsMsg, error) { + d := testHookDNSDialer() out := dnsMsg{ dnsMsgHdr: dnsMsgHdr{ recursion_desired: true, @@ -140,24 +152,24 @@ func exchange(server, name string, qtype uint16, timeout time.Duration) (*dnsMsg }, } for _, network := range []string{"udp", "tcp"} { - c, err := d.dialDNS(network, server) + // TODO(mdempsky): Refactor so defers from UDP-based + // exchanges happen before TCP-based exchange. + + ctx, cancel := context.WithDeadline(ctx, time.Now().Add(timeout)) + defer cancel() + + c, err := d.dialDNS(ctx, network, server) if err != nil { return nil, err } defer c.Close() - if timeout > 0 { - c.SetDeadline(time.Now().Add(timeout)) + if d, ok := ctx.Deadline(); ok && !d.IsZero() { + c.SetDeadline(d) } out.id = uint16(rand.Int()) ^ uint16(time.Now().UnixNano()) - if err := c.writeDNSQuery(&out); err != nil { - return nil, err - } - in, err := c.readDNSResponse() + in, err := c.dnsRoundTrip(&out) if err != nil { - return nil, err - } - if in.id != out.id { - return nil, errors.New("DNS message ID mismatch") + return nil, mapErr(err) } if in.truncated { // see RFC 5966 continue @@ -169,16 +181,16 @@ func exchange(server, name string, qtype uint16, timeout time.Duration) (*dnsMsg // Do a lookup for a single name, which must be rooted // (otherwise answer will not find the answers). -func tryOneName(cfg *dnsConfig, name string, qtype uint16) (string, []dnsRR, error) { - if len(cfg.servers) == 0 { - return "", nil, &DNSError{Err: "no DNS servers", Name: name} - } - timeout := time.Duration(cfg.timeout) * time.Second +func tryOneName(ctx context.Context, cfg *dnsConfig, name string, qtype uint16) (string, []dnsRR, error) { var lastErr error + serverOffset := cfg.serverOffset() + sLen := uint32(len(cfg.servers)) + for i := 0; i < cfg.attempts; i++ { - for _, server := range cfg.servers { - server = JoinHostPort(server, "53") - msg, err := exchange(server, name, qtype, timeout) + for j := uint32(0); j < sLen; j++ { + server := cfg.servers[(serverOffset+j)%sLen] + + msg, err := exchange(ctx, server, name, qtype, cfg.timeout) if err != nil { lastErr = &DNSError{ Err: err.Error(), @@ -190,6 +202,12 @@ func tryOneName(cfg *dnsConfig, name string, qtype uint16) (string, []dnsRR, err } continue } + // libresolv continues to the next server when it receives + // an invalid referral response. See golang.org/issue/15434. + if msg.rcode == dnsRcodeSuccess && !msg.authoritative && !msg.recursion_available && len(msg.answer) == 0 && len(msg.extra) == 0 { + lastErr = &DNSError{Err: "lame referral", Name: name, Server: server} + continue + } cname, rrs, err := answer(name, server, msg, qtype) // If answer errored for rcodes dnsRcodeSuccess or dnsRcodeNameError, // it means the response in msg was not useful and trying another @@ -229,7 +247,6 @@ type resolverConfig struct { // time to recheck resolv.conf. ch chan struct{} // guards lastChecked and modTime lastChecked time.Time // last time resolv.conf was checked - modTime time.Time // time of resolv.conf modification mu sync.RWMutex // protects dnsConfig dnsConfig *dnsConfig // parsed resolv.conf structure used in lookups @@ -239,16 +256,12 @@ var resolvConf resolverConfig // init initializes conf and is only called via conf.initOnce. func (conf *resolverConfig) init() { - // Set dnsConfig, modTime, and lastChecked so we don't parse + // Set dnsConfig and lastChecked so we don't parse // resolv.conf twice the first time. conf.dnsConfig = systemConf().resolv if conf.dnsConfig == nil { conf.dnsConfig = dnsReadConfig("/etc/resolv.conf") } - - if fi, err := os.Stat("/etc/resolv.conf"); err == nil { - conf.modTime = fi.ModTime() - } conf.lastChecked = time.Now() // Prepare ch so that only one update of resolverConfig may @@ -274,17 +287,12 @@ func (conf *resolverConfig) tryUpdate(name string) { } conf.lastChecked = now + var mtime time.Time if fi, err := os.Stat(name); err == nil { - if fi.ModTime().Equal(conf.modTime) { - return - } - conf.modTime = fi.ModTime() - } else { - // If modTime wasn't set prior, assume nothing has changed. - if conf.modTime.IsZero() { - return - } - conf.modTime = time.Time{} + mtime = fi.ModTime() + } + if mtime.Equal(conf.dnsConfig.mtime) { + return } dnsConf := dnsReadConfig(name) @@ -306,16 +314,21 @@ func (conf *resolverConfig) releaseSema() { <-conf.ch } -func lookup(name string, qtype uint16) (cname string, rrs []dnsRR, err error) { +func lookup(ctx context.Context, name string, qtype uint16) (cname string, rrs []dnsRR, err error) { if !isDomainName(name) { - return "", nil, &DNSError{Err: "invalid domain name", Name: name} + // We used to use "invalid domain name" as the error, + // but that is a detail of the specific lookup mechanism. + // Other lookups might allow broader name syntax + // (for example Multicast DNS allows UTF-8; see RFC 6762). + // For consistency with libc resolvers, report no such host. + return "", nil, &DNSError{Err: errNoSuchHost.Error(), Name: name} } resolvConf.tryUpdate("/etc/resolv.conf") resolvConf.mu.RLock() conf := resolvConf.dnsConfig resolvConf.mu.RUnlock() for _, fqdn := range conf.nameList(name) { - cname, rrs, err = tryOneName(conf, fqdn, qtype) + cname, rrs, err = tryOneName(ctx, conf, fqdn, qtype) if err == nil { break } @@ -329,30 +342,57 @@ func lookup(name string, qtype uint16) (cname string, rrs []dnsRR, err error) { return } +// avoidDNS reports whether this is a hostname for which we should not +// use DNS. Currently this includes only .onion, per RFC 7686. See +// golang.org/issue/13705. Does not cover .local names (RFC 6762), +// see golang.org/issue/16739. +func avoidDNS(name string) bool { + if name == "" { + return true + } + if name[len(name)-1] == '.' { + name = name[:len(name)-1] + } + return stringsHasSuffixFold(name, ".onion") +} + // nameList returns a list of names for sequential DNS queries. func (conf *dnsConfig) nameList(name string) []string { + if avoidDNS(name) { + return nil + } + + // Check name length (see isDomainName). + l := len(name) + rooted := l > 0 && name[l-1] == '.' + if l > 254 || l == 254 && rooted { + return nil + } + // If name is rooted (trailing dot), try only that name. - rooted := len(name) > 0 && name[len(name)-1] == '.' if rooted { return []string{name} } + + hasNdots := count(name, '.') >= conf.ndots + name += "." + l++ + // Build list of search choices. names := make([]string, 0, 1+len(conf.search)) // If name has enough dots, try unsuffixed first. - if count(name, '.') >= conf.ndots { - names = append(names, name+".") + if hasNdots { + names = append(names, name) } - // Try suffixes. + // Try suffixes that are not too long (see isDomainName). for _, suffix := range conf.search { - suffixed := name + "." + suffix - if suffixed[len(suffixed)-1] != '.' { - suffixed += "." + if l+len(suffix) <= 254 { + names = append(names, name+suffix) } - names = append(names, suffixed) } // Try unsuffixed, if not tried first above. - if count(name, '.') < conf.ndots { - names = append(names, name+".") + if !hasNdots { + names = append(names, name) } return names } @@ -392,11 +432,11 @@ func (o hostLookupOrder) String() string { // Normally we let cgo use the C library resolver instead of // depending on our lookup code, so that Go and C get the same // answers. -func goLookupHost(name string) (addrs []string, err error) { - return goLookupHostOrder(name, hostLookupFilesDNS) +func goLookupHost(ctx context.Context, name string) (addrs []string, err error) { + return goLookupHostOrder(ctx, name, hostLookupFilesDNS) } -func goLookupHostOrder(name string, order hostLookupOrder) (addrs []string, err error) { +func goLookupHostOrder(ctx context.Context, name string, order hostLookupOrder) (addrs []string, err error) { if order == hostLookupFilesDNS || order == hostLookupFiles { // Use entries from /etc/hosts if they match. addrs = lookupStaticHost(name) @@ -404,7 +444,7 @@ func goLookupHostOrder(name string, order hostLookupOrder) (addrs []string, err return } } - ips, err := goLookupIPOrder(name, order) + ips, _, err := goLookupIPCNAMEOrder(ctx, name, order) if err != nil { return } @@ -430,27 +470,30 @@ func goLookupIPFiles(name string) (addrs []IPAddr) { // goLookupIP is the native Go implementation of LookupIP. // The libc versions are in cgo_*.go. -func goLookupIP(name string) (addrs []IPAddr, err error) { - return goLookupIPOrder(name, hostLookupFilesDNS) +func goLookupIP(ctx context.Context, host string) (addrs []IPAddr, err error) { + order := systemConf().hostLookupOrder(host) + addrs, _, err = goLookupIPCNAMEOrder(ctx, host, order) + return } -func goLookupIPOrder(name string, order hostLookupOrder) (addrs []IPAddr, err error) { +func goLookupIPCNAMEOrder(ctx context.Context, name string, order hostLookupOrder) (addrs []IPAddr, cname string, err error) { if order == hostLookupFilesDNS || order == hostLookupFiles { addrs = goLookupIPFiles(name) if len(addrs) > 0 || order == hostLookupFiles { - return addrs, nil + return addrs, name, nil } } if !isDomainName(name) { - return nil, &DNSError{Err: "invalid domain name", Name: name} + // See comment in func lookup above about use of errNoSuchHost. + return nil, "", &DNSError{Err: errNoSuchHost.Error(), Name: name} } resolvConf.tryUpdate("/etc/resolv.conf") resolvConf.mu.RLock() conf := resolvConf.dnsConfig resolvConf.mu.RUnlock() type racer struct { - fqdn string - rrs []dnsRR + cname string + rrs []dnsRR error } lane := make(chan racer, 1) @@ -459,20 +502,23 @@ func goLookupIPOrder(name string, order hostLookupOrder) (addrs []IPAddr, err er for _, fqdn := range conf.nameList(name) { for _, qtype := range qtypes { go func(qtype uint16) { - _, rrs, err := tryOneName(conf, fqdn, qtype) - lane <- racer{fqdn, rrs, err} + cname, rrs, err := tryOneName(ctx, conf, fqdn, qtype) + lane <- racer{cname, rrs, err} }(qtype) } for range qtypes { racer := <-lane if racer.error != nil { // Prefer error for original name. - if lastErr == nil || racer.fqdn == name+"." { + if lastErr == nil || fqdn == name+"." { lastErr = racer.error } continue } addrs = append(addrs, addrRecordList(racer.rrs)...) + if cname == "" { + cname = racer.cname + } } if len(addrs) > 0 { break @@ -490,24 +536,16 @@ func goLookupIPOrder(name string, order hostLookupOrder) (addrs []IPAddr, err er addrs = goLookupIPFiles(name) } if len(addrs) == 0 && lastErr != nil { - return nil, lastErr + return nil, "", lastErr } } - return addrs, nil + return addrs, cname, nil } -// goLookupCNAME is the native Go implementation of LookupCNAME. -// Used only if cgoLookupCNAME refuses to handle the request -// (that is, only if cgoLookupCNAME is the stub in cgo_stub.go). -// Normally we let cgo use the C library resolver instead of -// depending on our lookup code, so that Go and C get the same -// answers. -func goLookupCNAME(name string) (cname string, err error) { - _, rrs, err := lookup(name, dnsTypeCNAME) - if err != nil { - return - } - cname = rrs[0].(*dnsRR_CNAME).Cname +// goLookupCNAME is the native Go (non-cgo) implementation of LookupCNAME. +func goLookupCNAME(ctx context.Context, host string) (cname string, err error) { + order := systemConf().hostLookupOrder(host) + _, cname, err = goLookupIPCNAMEOrder(ctx, host, order) return } @@ -516,7 +554,7 @@ func goLookupCNAME(name string) (cname string, err error) { // only if cgoLookupPTR is the stub in cgo_stub.go). // Normally we let cgo use the C library resolver instead of depending // on our lookup code, so that Go and C get the same answers. -func goLookupPTR(addr string) ([]string, error) { +func goLookupPTR(ctx context.Context, addr string) ([]string, error) { names := lookupStaticAddr(addr) if len(names) > 0 { return names, nil @@ -525,7 +563,7 @@ func goLookupPTR(addr string) ([]string, error) { if err != nil { return nil, err } - _, rrs, err := lookup(arpa, dnsTypePTR) + _, rrs, err := lookup(ctx, arpa, dnsTypePTR) if err != nil { return nil, err } diff --git a/libgo/go/net/dnsclient_unix_test.go b/libgo/go/net/dnsclient_unix_test.go index 934f25b2c9..85267bbddc 100644 --- a/libgo/go/net/dnsclient_unix_test.go +++ b/libgo/go/net/dnsclient_unix_test.go @@ -7,7 +7,9 @@ package net import ( + "context" "fmt" + "internal/testenv" "io/ioutil" "os" "path" @@ -18,6 +20,9 @@ import ( "time" ) +// Test address from 192.0.2.0/24 block, reserved by RFC 5737 for documentation. +const TestAddr uint32 = 0xc0000201 + var dnsTransportFallbackTests = []struct { server string name string @@ -32,13 +37,12 @@ var dnsTransportFallbackTests = []struct { } func TestDNSTransportFallback(t *testing.T) { - if testing.Short() || !*testExternal { - t.Skip("avoid external network") - } + testenv.MustHaveExternalNetwork(t) for _, tt := range dnsTransportFallbackTests { - timeout := time.Duration(tt.timeout) * time.Second - msg, err := exchange(tt.server, tt.name, tt.qtype, timeout) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + msg, err := exchange(ctx, tt.server, tt.name, tt.qtype, time.Second) if err != nil { t.Error(err) continue @@ -67,20 +71,20 @@ var specialDomainNameTests = []struct { // Name resolution APIs and libraries should recognize the // followings as special and should not send any queries. - // Though, we test those names here for verifying nagative + // Though, we test those names here for verifying negative // answers at DNS query-response interaction level. {"localhost.", dnsTypeALL, dnsRcodeNameError}, {"invalid.", dnsTypeALL, dnsRcodeNameError}, } func TestSpecialDomainName(t *testing.T) { - if testing.Short() || !*testExternal { - t.Skip("avoid external network") - } + testenv.MustHaveExternalNetwork(t) server := "8.8.8.8:53" for _, tt := range specialDomainNameTests { - msg, err := exchange(server, tt.name, tt.qtype, 3*time.Second) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + msg, err := exchange(ctx, server, tt.name, tt.qtype, 3*time.Second) if err != nil { t.Error(err) continue @@ -94,6 +98,58 @@ func TestSpecialDomainName(t *testing.T) { } } +// Issue 13705: don't try to resolve onion addresses, etc +func TestAvoidDNSName(t *testing.T) { + tests := []struct { + name string + avoid bool + }{ + {"foo.com", false}, + {"foo.com.", false}, + + {"foo.onion.", true}, + {"foo.onion", true}, + {"foo.ONION", true}, + {"foo.ONION.", true}, + + // But do resolve *.local address; Issue 16739 + {"foo.local.", false}, + {"foo.local", false}, + {"foo.LOCAL", false}, + {"foo.LOCAL.", false}, + + {"", true}, // will be rejected earlier too + + // Without stuff before onion/local, they're fine to + // use DNS. With a search path, + // "onion.vegegtables.com" can use DNS. Without a + // search path (or with a trailing dot), the queries + // are just kinda useless, but don't reveal anything + // private. + {"local", false}, + {"onion", false}, + {"local.", false}, + {"onion.", false}, + } + for _, tt := range tests { + got := avoidDNS(tt.name) + if got != tt.avoid { + t.Errorf("avoidDNS(%q) = %v; want %v", tt.name, got, tt.avoid) + } + } +} + +// Issue 13705: don't try to resolve onion addresses, etc +func TestLookupTorOnion(t *testing.T) { + addrs, err := goLookupIP(context.Background(), "foo.onion") + if len(addrs) > 0 { + t.Errorf("unexpected addresses: %v", addrs) + } + if err != nil { + t.Fatalf("lookup = %v; want nil", err) + } +} + type resolvConfTest struct { dir string path string @@ -124,20 +180,20 @@ func (conf *resolvConfTest) writeAndUpdate(lines []string) error { return err } f.Close() - if err := conf.forceUpdate(conf.path); err != nil { + if err := conf.forceUpdate(conf.path, time.Now().Add(time.Hour)); err != nil { return err } return nil } -func (conf *resolvConfTest) forceUpdate(name string) error { +func (conf *resolvConfTest) forceUpdate(name string, lastChecked time.Time) error { dnsConf := dnsReadConfig(name) conf.mu.Lock() conf.dnsConfig = dnsConf conf.mu.Unlock() for i := 0; i < 5; i++ { if conf.tryAcquireSema() { - conf.lastChecked = time.Time{} + conf.lastChecked = lastChecked conf.releaseSema() return nil } @@ -153,7 +209,7 @@ func (conf *resolvConfTest) servers() []string { } func (conf *resolvConfTest) teardown() error { - err := conf.forceUpdate("/etc/resolv.conf") + err := conf.forceUpdate("/etc/resolv.conf", time.Time{}) os.RemoveAll(conf.dir) return err } @@ -166,7 +222,7 @@ var updateResolvConfTests = []struct { { name: "golang.org", lines: []string{"nameserver 8.8.8.8"}, - servers: []string{"8.8.8.8"}, + servers: []string{"8.8.8.8:53"}, }, { name: "", @@ -176,14 +232,12 @@ var updateResolvConfTests = []struct { { name: "www.example.com", lines: []string{"nameserver 8.8.4.4"}, - servers: []string{"8.8.4.4"}, + servers: []string{"8.8.4.4:53"}, }, } func TestUpdateResolvConf(t *testing.T) { - if testing.Short() || !*testExternal { - t.Skip("avoid external network") - } + testenv.MustHaveExternalNetwork(t) conf, err := newResolvConfTest() if err != nil { @@ -203,7 +257,7 @@ func TestUpdateResolvConf(t *testing.T) { for j := 0; j < N; j++ { go func(name string) { defer wg.Done() - ips, err := goLookupIP(name) + ips, err := goLookupIP(context.Background(), name) if err != nil { t.Error(err) return @@ -338,9 +392,7 @@ var goLookupIPWithResolverConfigTests = []struct { } func TestGoLookupIPWithResolverConfig(t *testing.T) { - if testing.Short() || !*testExternal { - t.Skip("avoid external network") - } + testenv.MustHaveExternalNetwork(t) conf, err := newResolvConfTest() if err != nil { @@ -353,10 +405,15 @@ func TestGoLookupIPWithResolverConfig(t *testing.T) { t.Error(err) continue } - conf.tryUpdate(conf.path) - addrs, err := goLookupIP(tt.name) + addrs, err := goLookupIP(context.Background(), tt.name) if err != nil { - if err, ok := err.(*DNSError); !ok || (err.Name != tt.error.(*DNSError).Name || err.Server != tt.error.(*DNSError).Server || err.IsTimeout != tt.error.(*DNSError).IsTimeout) { + // This test uses external network connectivity. + // We need to take care with errors on both + // DNS message exchange layer and DNS + // transport layer because goLookupIP may fail + // when the IP connectivity on node under test + // gets lost during its run. + if err, ok := err.(*DNSError); !ok || tt.error != nil && (err.Name != tt.error.(*DNSError).Name || err.Server != tt.error.(*DNSError).Server || err.IsTimeout != tt.error.(*DNSError).IsTimeout) { t.Errorf("got %v; want %v", err, tt.error) } continue @@ -380,9 +437,7 @@ func TestGoLookupIPWithResolverConfig(t *testing.T) { // Test that goLookupIPOrder falls back to the host file when no DNS servers are available. func TestGoLookupIPOrderFallbackToFile(t *testing.T) { - if testing.Short() || !*testExternal { - t.Skip("avoid external network") - } + testenv.MustHaveExternalNetwork(t) // Add a config that simulates no dns servers being available. conf, err := newResolvConfTest() @@ -392,7 +447,6 @@ func TestGoLookupIPOrderFallbackToFile(t *testing.T) { if err := conf.writeAndUpdate([]string{}); err != nil { t.Fatal(err) } - conf.tryUpdate(conf.path) // Redirect host file lookups. defer func(orig string) { testHookHostsPath = orig }(testHookHostsPath) testHookHostsPath = "testdata/hosts" @@ -400,15 +454,15 @@ func TestGoLookupIPOrderFallbackToFile(t *testing.T) { for _, order := range []hostLookupOrder{hostLookupFilesDNS, hostLookupDNSFiles} { name := fmt.Sprintf("order %v", order) - // First ensure that we get an error when contacting a non-existant host. - _, err := goLookupIPOrder("notarealhost", order) + // First ensure that we get an error when contacting a non-existent host. + _, _, err := goLookupIPCNAMEOrder(context.Background(), "notarealhost", order) if err == nil { t.Errorf("%s: expected error while looking up name not in hosts file", name) continue } // Now check that we get an address when the name appears in the hosts file. - addrs, err := goLookupIPOrder("thor", order) // entry is in "testdata/hosts" + addrs, _, err := goLookupIPCNAMEOrder(context.Background(), "thor", order) // entry is in "testdata/hosts" if err != nil { t.Errorf("%s: expected to successfully lookup host entry", name) continue @@ -444,10 +498,10 @@ func TestErrorForOriginalNameWhenSearching(t *testing.T) { t.Fatal(err) } - d := &fakeDNSConn{} - testHookDNSDialer = func(time.Duration) dnsDialer { return d } + d := &fakeDNSDialer{} + testHookDNSDialer = func() dnsDialer { return d } - d.rh = func(q *dnsMsg) (*dnsMsg, error) { + d.rh = func(s string, q *dnsMsg, _ time.Time) (*dnsMsg, error) { r := &dnsMsg{ dnsMsgHdr: dnsMsgHdr{ id: q.id, @@ -464,7 +518,7 @@ func TestErrorForOriginalNameWhenSearching(t *testing.T) { return r, nil } - _, err = goLookupIP(fqdn) + _, err = goLookupIP(context.Background(), fqdn) if err == nil { t.Fatal("expected an error") } @@ -475,19 +529,84 @@ func TestErrorForOriginalNameWhenSearching(t *testing.T) { } } +// Issue 15434. If a name server gives a lame referral, continue to the next. +func TestIgnoreLameReferrals(t *testing.T) { + origTestHookDNSDialer := testHookDNSDialer + defer func() { testHookDNSDialer = origTestHookDNSDialer }() + + conf, err := newResolvConfTest() + if err != nil { + t.Fatal(err) + } + defer conf.teardown() + + if err := conf.writeAndUpdate([]string{"nameserver 192.0.2.1", // the one that will give a lame referral + "nameserver 192.0.2.2"}); err != nil { + t.Fatal(err) + } + + d := &fakeDNSDialer{} + testHookDNSDialer = func() dnsDialer { return d } + + d.rh = func(s string, q *dnsMsg, _ time.Time) (*dnsMsg, error) { + t.Log(s, q) + r := &dnsMsg{ + dnsMsgHdr: dnsMsgHdr{ + id: q.id, + response: true, + }, + question: q.question, + } + + if s == "192.0.2.2:53" { + r.recursion_available = true + if q.question[0].Qtype == dnsTypeA { + r.answer = []dnsRR{ + &dnsRR_A{ + Hdr: dnsRR_Header{ + Name: q.question[0].Name, + Rrtype: dnsTypeA, + Class: dnsClassINET, + Rdlength: 4, + }, + A: TestAddr, + }, + } + } + } + + return r, nil + } + + addrs, err := goLookupIP(context.Background(), "www.golang.org") + if err != nil { + t.Fatal(err) + } + + if got := len(addrs); got != 1 { + t.Fatalf("got %d addresses, want 1", got) + } + + if got, want := addrs[0].String(), "192.0.2.1"; got != want { + t.Fatalf("got address %v, want %v", got, want) + } +} + func BenchmarkGoLookupIP(b *testing.B) { testHookUninstaller.Do(uninstallTestHooks) + ctx := context.Background() for i := 0; i < b.N; i++ { - goLookupIP("www.example.com") + goLookupIP(ctx, "www.example.com") } } func BenchmarkGoLookupIPNoSuchHost(b *testing.B) { testHookUninstaller.Do(uninstallTestHooks) + ctx := context.Background() for i := 0; i < b.N; i++ { - goLookupIP("some.nonexistent") + goLookupIP(ctx, "some.nonexistent") } } @@ -507,42 +626,237 @@ func BenchmarkGoLookupIPWithBrokenNameServer(b *testing.B) { if err := conf.writeAndUpdate(lines); err != nil { b.Fatal(err) } + ctx := context.Background() for i := 0; i < b.N; i++ { - goLookupIP("www.example.com") + goLookupIP(ctx, "www.example.com") } } -type fakeDNSConn struct { - // last query - qmu sync.Mutex // guards q - q *dnsMsg +type fakeDNSDialer struct { // reply handler - rh func(*dnsMsg) (*dnsMsg, error) + rh func(s string, q *dnsMsg, t time.Time) (*dnsMsg, error) } -func (f *fakeDNSConn) dialDNS(n, s string) (dnsConn, error) { - return f, nil +func (f *fakeDNSDialer) dialDNS(_ context.Context, n, s string) (dnsConn, error) { + return &fakeDNSConn{f.rh, s, time.Time{}}, nil +} + +type fakeDNSConn struct { + rh func(s string, q *dnsMsg, t time.Time) (*dnsMsg, error) + s string + t time.Time } func (f *fakeDNSConn) Close() error { return nil } -func (f *fakeDNSConn) SetDeadline(time.Time) error { +func (f *fakeDNSConn) SetDeadline(t time.Time) error { + f.t = t return nil } -func (f *fakeDNSConn) writeDNSQuery(q *dnsMsg) error { - f.qmu.Lock() - defer f.qmu.Unlock() - f.q = q - return nil +func (f *fakeDNSConn) dnsRoundTrip(q *dnsMsg) (*dnsMsg, error) { + return f.rh(f.s, q, f.t) } -func (f *fakeDNSConn) readDNSResponse() (*dnsMsg, error) { - f.qmu.Lock() - q := f.q - f.qmu.Unlock() - return f.rh(q) +// UDP round-tripper algorithm should ignore invalid DNS responses (issue 13281). +func TestIgnoreDNSForgeries(t *testing.T) { + c, s := Pipe() + go func() { + b := make([]byte, 512) + n, err := s.Read(b) + if err != nil { + t.Error(err) + return + } + + msg := &dnsMsg{} + if !msg.Unpack(b[:n]) { + t.Error("invalid DNS query") + return + } + + s.Write([]byte("garbage DNS response packet")) + + msg.response = true + msg.id++ // make invalid ID + b, ok := msg.Pack() + if !ok { + t.Error("failed to pack DNS response") + return + } + s.Write(b) + + msg.id-- // restore original ID + msg.answer = []dnsRR{ + &dnsRR_A{ + Hdr: dnsRR_Header{ + Name: "www.example.com.", + Rrtype: dnsTypeA, + Class: dnsClassINET, + Rdlength: 4, + }, + A: TestAddr, + }, + } + + b, ok = msg.Pack() + if !ok { + t.Error("failed to pack DNS response") + return + } + s.Write(b) + }() + + msg := &dnsMsg{ + dnsMsgHdr: dnsMsgHdr{ + id: 42, + }, + question: []dnsQuestion{ + { + Name: "www.example.com.", + Qtype: dnsTypeA, + Qclass: dnsClassINET, + }, + }, + } + + resp, err := dnsRoundTripUDP(c, msg) + if err != nil { + t.Fatalf("dnsRoundTripUDP failed: %v", err) + } + + if got := resp.answer[0].(*dnsRR_A).A; got != TestAddr { + t.Errorf("got address %v, want %v", got, TestAddr) + } +} + +// Issue 16865. If a name server times out, continue to the next. +func TestRetryTimeout(t *testing.T) { + origTestHookDNSDialer := testHookDNSDialer + defer func() { testHookDNSDialer = origTestHookDNSDialer }() + + conf, err := newResolvConfTest() + if err != nil { + t.Fatal(err) + } + defer conf.teardown() + + testConf := []string{ + "nameserver 192.0.2.1", // the one that will timeout + "nameserver 192.0.2.2", + } + if err := conf.writeAndUpdate(testConf); err != nil { + t.Fatal(err) + } + + d := &fakeDNSDialer{} + testHookDNSDialer = func() dnsDialer { return d } + + var deadline0 time.Time + + d.rh = func(s string, q *dnsMsg, deadline time.Time) (*dnsMsg, error) { + t.Log(s, q, deadline) + + if deadline.IsZero() { + t.Error("zero deadline") + } + + if s == "192.0.2.1:53" { + deadline0 = deadline + time.Sleep(10 * time.Millisecond) + return nil, errTimeout + } + + if deadline == deadline0 { + t.Error("deadline didn't change") + } + + return mockTXTResponse(q), nil + } + + _, err = LookupTXT("www.golang.org") + if err != nil { + t.Fatal(err) + } + + if deadline0.IsZero() { + t.Error("deadline0 still zero", deadline0) + } +} + +func TestRotate(t *testing.T) { + // without rotation, always uses the first server + testRotate(t, false, []string{"192.0.2.1", "192.0.2.2"}, []string{"192.0.2.1:53", "192.0.2.1:53", "192.0.2.1:53"}) + + // with rotation, rotates through back to first + testRotate(t, true, []string{"192.0.2.1", "192.0.2.2"}, []string{"192.0.2.1:53", "192.0.2.2:53", "192.0.2.1:53"}) +} + +func testRotate(t *testing.T, rotate bool, nameservers, wantServers []string) { + origTestHookDNSDialer := testHookDNSDialer + defer func() { testHookDNSDialer = origTestHookDNSDialer }() + + conf, err := newResolvConfTest() + if err != nil { + t.Fatal(err) + } + defer conf.teardown() + + var confLines []string + for _, ns := range nameservers { + confLines = append(confLines, "nameserver "+ns) + } + if rotate { + confLines = append(confLines, "options rotate") + } + + if err := conf.writeAndUpdate(confLines); err != nil { + t.Fatal(err) + } + + d := &fakeDNSDialer{} + testHookDNSDialer = func() dnsDialer { return d } + + var usedServers []string + d.rh = func(s string, q *dnsMsg, _ time.Time) (*dnsMsg, error) { + usedServers = append(usedServers, s) + return mockTXTResponse(q), nil + } + + // len(nameservers) + 1 to allow rotation to get back to start + for i := 0; i < len(nameservers)+1; i++ { + if _, err := LookupTXT("www.golang.org"); err != nil { + t.Fatal(err) + } + } + + if !reflect.DeepEqual(usedServers, wantServers) { + t.Errorf("rotate=%t got used servers:\n%v\nwant:\n%v", rotate, usedServers, wantServers) + } +} + +func mockTXTResponse(q *dnsMsg) *dnsMsg { + r := &dnsMsg{ + dnsMsgHdr: dnsMsgHdr{ + id: q.id, + response: true, + recursion_available: true, + }, + question: q.question, + answer: []dnsRR{ + &dnsRR_TXT{ + Hdr: dnsRR_Header{ + Name: q.question[0].Name, + Rrtype: dnsTypeTXT, + Class: dnsClassINET, + }, + Txt: "ok", + }, + }, + } + + return r } diff --git a/libgo/go/net/dnsconfig_unix.go b/libgo/go/net/dnsconfig_unix.go index 6073fdb6d8..9c8108d11c 100644 --- a/libgo/go/net/dnsconfig_unix.go +++ b/libgo/go/net/dnsconfig_unix.go @@ -8,36 +8,54 @@ package net -var defaultNS = []string{"127.0.0.1", "::1"} +import ( + "os" + "sync/atomic" + "time" +) + +var ( + defaultNS = []string{"127.0.0.1:53", "[::1]:53"} + getHostname = os.Hostname // variable for testing +) type dnsConfig struct { - servers []string // servers to use - search []string // suffixes to append to local name - ndots int // number of dots in name to trigger absolute lookup - timeout int // seconds before giving up on packet - attempts int // lost packets before giving up on server - rotate bool // round robin among servers - unknownOpt bool // anything unknown was encountered - lookup []string // OpenBSD top-level database "lookup" order - err error // any error that occurs during open of resolv.conf + servers []string // server addresses (in host:port form) to use + search []string // rooted suffixes to append to local name + ndots int // number of dots in name to trigger absolute lookup + timeout time.Duration // wait before giving up on a query, including retries + attempts int // lost packets before giving up on server + rotate bool // round robin among servers + unknownOpt bool // anything unknown was encountered + lookup []string // OpenBSD top-level database "lookup" order + err error // any error that occurs during open of resolv.conf + mtime time.Time // time of resolv.conf modification + soffset uint32 // used by serverOffset } // See resolv.conf(5) on a Linux machine. -// TODO(rsc): Supposed to call uname() and chop the beginning -// of the host name to get the default search domain. func dnsReadConfig(filename string) *dnsConfig { conf := &dnsConfig{ ndots: 1, - timeout: 5, + timeout: 5 * time.Second, attempts: 2, } file, err := open(filename) if err != nil { conf.servers = defaultNS + conf.search = dnsDefaultSearch() conf.err = err return conf } defer file.close() + if fi, err := file.file.Stat(); err == nil { + conf.mtime = fi.ModTime() + } else { + conf.servers = defaultNS + conf.search = dnsDefaultSearch() + conf.err = err + return conf + } for line, ok := file.readLine(); ok; line, ok = file.readLine() { if len(line) > 0 && (line[0] == ';' || line[0] == '#') { // comment. @@ -51,43 +69,45 @@ func dnsReadConfig(filename string) *dnsConfig { case "nameserver": // add one name server if len(f) > 1 && len(conf.servers) < 3 { // small, but the standard limit // One more check: make sure server name is - // just an IP address. Otherwise we need DNS + // just an IP address. Otherwise we need DNS // to look it up. if parseIPv4(f[1]) != nil { - conf.servers = append(conf.servers, f[1]) + conf.servers = append(conf.servers, JoinHostPort(f[1], "53")) } else if ip, _ := parseIPv6(f[1], true); ip != nil { - conf.servers = append(conf.servers, f[1]) + conf.servers = append(conf.servers, JoinHostPort(f[1], "53")) } } case "domain": // set search path to just this domain if len(f) > 1 { - conf.search = []string{f[1]} + conf.search = []string{ensureRooted(f[1])} } case "search": // set search path to given servers conf.search = make([]string, len(f)-1) for i := 0; i < len(conf.search); i++ { - conf.search[i] = f[i+1] + conf.search[i] = ensureRooted(f[i+1]) } case "options": // magic options for _, s := range f[1:] { switch { case hasPrefix(s, "ndots:"): - n, _, _ := dtoi(s, 6) - if n < 1 { - n = 1 + n, _, _ := dtoi(s[6:]) + if n < 0 { + n = 0 + } else if n > 15 { + n = 15 } conf.ndots = n case hasPrefix(s, "timeout:"): - n, _, _ := dtoi(s, 8) + n, _, _ := dtoi(s[8:]) if n < 1 { n = 1 } - conf.timeout = n + conf.timeout = time.Duration(n) * time.Second case hasPrefix(s, "attempts:"): - n, _, _ := dtoi(s, 9) + n, _, _ := dtoi(s[9:]) if n < 1 { n = 1 } @@ -112,9 +132,42 @@ func dnsReadConfig(filename string) *dnsConfig { if len(conf.servers) == 0 { conf.servers = defaultNS } + if len(conf.search) == 0 { + conf.search = dnsDefaultSearch() + } return conf } +// serverOffset returns an offset that can be used to determine +// indices of servers in c.servers when making queries. +// When the rotate option is enabled, this offset increases. +// Otherwise it is always 0. +func (c *dnsConfig) serverOffset() uint32 { + if c.rotate { + return atomic.AddUint32(&c.soffset, 1) - 1 // return 0 to start + } + return 0 +} + +func dnsDefaultSearch() []string { + hn, err := getHostname() + if err != nil { + // best effort + return nil + } + if i := byteIndex(hn, '.'); i >= 0 && i < len(hn)-1 { + return []string{ensureRooted(hn[i+1:])} + } + return nil +} + func hasPrefix(s, prefix string) bool { return len(s) >= len(prefix) && s[:len(prefix)] == prefix } + +func ensureRooted(s string) string { + if len(s) > 0 && s[len(s)-1] == '.' { + return s + } + return s + "." +} diff --git a/libgo/go/net/dnsconfig_unix_test.go b/libgo/go/net/dnsconfig_unix_test.go index c8eed61890..37bdeb04c8 100644 --- a/libgo/go/net/dnsconfig_unix_test.go +++ b/libgo/go/net/dnsconfig_unix_test.go @@ -7,9 +7,12 @@ package net import ( + "errors" "os" "reflect" + "strings" "testing" + "time" ) var dnsReadConfigTests = []struct { @@ -19,10 +22,10 @@ var dnsReadConfigTests = []struct { { name: "testdata/resolv.conf", want: &dnsConfig{ - servers: []string{"8.8.8.8", "2001:4860:4860::8888", "fe80::1%lo0"}, - search: []string{"localdomain"}, + servers: []string{"8.8.8.8:53", "[2001:4860:4860::8888]:53", "[fe80::1%lo0]:53"}, + search: []string{"localdomain."}, ndots: 5, - timeout: 10, + timeout: 10 * time.Second, attempts: 3, rotate: true, unknownOpt: true, // the "options attempts 3" line @@ -31,20 +34,20 @@ var dnsReadConfigTests = []struct { { name: "testdata/domain-resolv.conf", want: &dnsConfig{ - servers: []string{"8.8.8.8"}, - search: []string{"localdomain"}, + servers: []string{"8.8.8.8:53"}, + search: []string{"localdomain."}, ndots: 1, - timeout: 5, + timeout: 5 * time.Second, attempts: 2, }, }, { name: "testdata/search-resolv.conf", want: &dnsConfig{ - servers: []string{"8.8.8.8"}, - search: []string{"test", "invalid"}, + servers: []string{"8.8.8.8:53"}, + search: []string{"test.", "invalid."}, ndots: 1, - timeout: 5, + timeout: 5 * time.Second, attempts: 2, }, }, @@ -53,29 +56,65 @@ var dnsReadConfigTests = []struct { want: &dnsConfig{ servers: defaultNS, ndots: 1, - timeout: 5, + timeout: 5 * time.Second, attempts: 2, + search: []string{"domain.local."}, + }, + }, + { + name: "testdata/invalid-ndots-resolv.conf", + want: &dnsConfig{ + servers: defaultNS, + ndots: 0, + timeout: 5 * time.Second, + attempts: 2, + search: []string{"domain.local."}, + }, + }, + { + name: "testdata/large-ndots-resolv.conf", + want: &dnsConfig{ + servers: defaultNS, + ndots: 15, + timeout: 5 * time.Second, + attempts: 2, + search: []string{"domain.local."}, + }, + }, + { + name: "testdata/negative-ndots-resolv.conf", + want: &dnsConfig{ + servers: defaultNS, + ndots: 0, + timeout: 5 * time.Second, + attempts: 2, + search: []string{"domain.local."}, }, }, { name: "testdata/openbsd-resolv.conf", want: &dnsConfig{ ndots: 1, - timeout: 5, + timeout: 5 * time.Second, attempts: 2, lookup: []string{"file", "bind"}, - servers: []string{"169.254.169.254", "10.240.0.1"}, + servers: []string{"169.254.169.254:53", "10.240.0.1:53"}, search: []string{"c.symbolic-datum-552.internal."}, }, }, } func TestDNSReadConfig(t *testing.T) { + origGetHostname := getHostname + defer func() { getHostname = origGetHostname }() + getHostname = func() (string, error) { return "host.domain.local", nil } + for _, tt := range dnsReadConfigTests { conf := dnsReadConfig(tt.name) if conf.err != nil { t.Fatal(conf.err) } + conf.mtime = time.Time{} if !reflect.DeepEqual(conf, tt.want) { t.Errorf("%s:\ngot: %+v\nwant: %+v", tt.name, conf, tt.want) } @@ -83,6 +122,10 @@ func TestDNSReadConfig(t *testing.T) { } func TestDNSReadMissingFile(t *testing.T) { + origGetHostname := getHostname + defer func() { getHostname = origGetHostname }() + getHostname = func() (string, error) { return "host.domain.local", nil } + conf := dnsReadConfig("a-nonexistent-file") if !os.IsNotExist(conf.err) { t.Errorf("missing resolv.conf:\ngot: %v\nwant: %v", conf.err, os.ErrNotExist) @@ -91,10 +134,106 @@ func TestDNSReadMissingFile(t *testing.T) { want := &dnsConfig{ servers: defaultNS, ndots: 1, - timeout: 5, + timeout: 5 * time.Second, attempts: 2, + search: []string{"domain.local."}, } if !reflect.DeepEqual(conf, want) { t.Errorf("missing resolv.conf:\ngot: %+v\nwant: %+v", conf, want) } } + +var dnsDefaultSearchTests = []struct { + name string + err error + want []string +}{ + { + name: "host.long.domain.local", + want: []string{"long.domain.local."}, + }, + { + name: "host.local", + want: []string{"local."}, + }, + { + name: "host", + want: nil, + }, + { + name: "host.domain.local", + err: errors.New("errored"), + want: nil, + }, + { + // ensures we don't return []string{""} + // which causes duplicate lookups + name: "foo.", + want: nil, + }, +} + +func TestDNSDefaultSearch(t *testing.T) { + origGetHostname := getHostname + defer func() { getHostname = origGetHostname }() + + for _, tt := range dnsDefaultSearchTests { + getHostname = func() (string, error) { return tt.name, tt.err } + got := dnsDefaultSearch() + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("dnsDefaultSearch with hostname %q and error %+v = %q, wanted %q", tt.name, tt.err, got, tt.want) + } + } +} + +func TestDNSNameLength(t *testing.T) { + origGetHostname := getHostname + defer func() { getHostname = origGetHostname }() + getHostname = func() (string, error) { return "host.domain.local", nil } + + var char63 = "" + for i := 0; i < 63; i++ { + char63 += "a" + } + longDomain := strings.Repeat(char63+".", 5) + "example" + + for _, tt := range dnsReadConfigTests { + conf := dnsReadConfig(tt.name) + if conf.err != nil { + t.Fatal(conf.err) + } + + var shortestSuffix int + for _, suffix := range tt.want.search { + if shortestSuffix == 0 || len(suffix) < shortestSuffix { + shortestSuffix = len(suffix) + } + } + + // Test a name that will be maximally long when prefixing the shortest + // suffix (accounting for the intervening dot). + longName := longDomain[len(longDomain)-254+1+shortestSuffix:] + if longName[0] == '.' || longName[1] == '.' { + longName = "aa." + longName[3:] + } + for _, fqdn := range conf.nameList(longName) { + if len(fqdn) > 254 { + t.Errorf("got %d; want less than or equal to 254", len(fqdn)) + } + } + + // Now test a name that's too long for suffixing. + unsuffixable := "a." + longName[1:] + unsuffixableResults := conf.nameList(unsuffixable) + if len(unsuffixableResults) != 1 { + t.Errorf("suffixed names %v; want []", unsuffixableResults[1:]) + } + + // Now test a name that's too long for DNS. + tooLong := "a." + longDomain + tooLongResults := conf.nameList(tooLong) + if tooLongResults != nil { + t.Errorf("suffixed names %v; want nil", tooLongResults) + } + } +} diff --git a/libgo/go/net/dnsmsg.go b/libgo/go/net/dnsmsg.go index 93078fe849..8f6c7b6350 100644 --- a/libgo/go/net/dnsmsg.go +++ b/libgo/go/net/dnsmsg.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// DNS packet assembly. See RFC 1035. +// DNS packet assembly. See RFC 1035. // // This is intended to support name resolution during Dial. // It doesn't have to be blazing fast. @@ -18,7 +18,7 @@ // generic pack/unpack routines. // // TODO(rsc): There are enough names defined in this file that they're all -// prefixed with dns. Perhaps put this in its own package later. +// prefixed with dns. Perhaps put this in its own package later. package net @@ -69,7 +69,7 @@ const ( ) // A dnsStruct describes how to iterate over its fields to emulate -// reflective marshalling. +// reflective marshaling. type dnsStruct interface { // Walk iterates over fields of a structure and calls f // with a reference to that field, the name of the field @@ -109,7 +109,7 @@ const ( // DNS queries. type dnsQuestion struct { - Name string `net:"domain-name"` // `net:"domain-name"` specifies encoding; see packers below + Name string Qtype uint16 Qclass uint16 } @@ -124,7 +124,7 @@ func (q *dnsQuestion) Walk(f func(v interface{}, name, tag string) bool) bool { // There are many types of messages, // but they all share the same header. type dnsRR_Header struct { - Name string `net:"domain-name"` + Name string Rrtype uint16 Class uint16 Ttl uint32 @@ -152,7 +152,7 @@ type dnsRR interface { type dnsRR_CNAME struct { Hdr dnsRR_Header - Cname string `net:"domain-name"` + Cname string } func (rr *dnsRR_CNAME) Header() *dnsRR_Header { @@ -163,77 +163,10 @@ func (rr *dnsRR_CNAME) Walk(f func(v interface{}, name, tag string) bool) bool { return rr.Hdr.Walk(f) && f(&rr.Cname, "Cname", "domain") } -type dnsRR_HINFO struct { - Hdr dnsRR_Header - Cpu string - Os string -} - -func (rr *dnsRR_HINFO) Header() *dnsRR_Header { - return &rr.Hdr -} - -func (rr *dnsRR_HINFO) Walk(f func(v interface{}, name, tag string) bool) bool { - return rr.Hdr.Walk(f) && f(&rr.Cpu, "Cpu", "") && f(&rr.Os, "Os", "") -} - -type dnsRR_MB struct { - Hdr dnsRR_Header - Mb string `net:"domain-name"` -} - -func (rr *dnsRR_MB) Header() *dnsRR_Header { - return &rr.Hdr -} - -func (rr *dnsRR_MB) Walk(f func(v interface{}, name, tag string) bool) bool { - return rr.Hdr.Walk(f) && f(&rr.Mb, "Mb", "domain") -} - -type dnsRR_MG struct { - Hdr dnsRR_Header - Mg string `net:"domain-name"` -} - -func (rr *dnsRR_MG) Header() *dnsRR_Header { - return &rr.Hdr -} - -func (rr *dnsRR_MG) Walk(f func(v interface{}, name, tag string) bool) bool { - return rr.Hdr.Walk(f) && f(&rr.Mg, "Mg", "domain") -} - -type dnsRR_MINFO struct { - Hdr dnsRR_Header - Rmail string `net:"domain-name"` - Email string `net:"domain-name"` -} - -func (rr *dnsRR_MINFO) Header() *dnsRR_Header { - return &rr.Hdr -} - -func (rr *dnsRR_MINFO) Walk(f func(v interface{}, name, tag string) bool) bool { - return rr.Hdr.Walk(f) && f(&rr.Rmail, "Rmail", "domain") && f(&rr.Email, "Email", "domain") -} - -type dnsRR_MR struct { - Hdr dnsRR_Header - Mr string `net:"domain-name"` -} - -func (rr *dnsRR_MR) Header() *dnsRR_Header { - return &rr.Hdr -} - -func (rr *dnsRR_MR) Walk(f func(v interface{}, name, tag string) bool) bool { - return rr.Hdr.Walk(f) && f(&rr.Mr, "Mr", "domain") -} - type dnsRR_MX struct { Hdr dnsRR_Header Pref uint16 - Mx string `net:"domain-name"` + Mx string } func (rr *dnsRR_MX) Header() *dnsRR_Header { @@ -246,7 +179,7 @@ func (rr *dnsRR_MX) Walk(f func(v interface{}, name, tag string) bool) bool { type dnsRR_NS struct { Hdr dnsRR_Header - Ns string `net:"domain-name"` + Ns string } func (rr *dnsRR_NS) Header() *dnsRR_Header { @@ -259,7 +192,7 @@ func (rr *dnsRR_NS) Walk(f func(v interface{}, name, tag string) bool) bool { type dnsRR_PTR struct { Hdr dnsRR_Header - Ptr string `net:"domain-name"` + Ptr string } func (rr *dnsRR_PTR) Header() *dnsRR_Header { @@ -272,8 +205,8 @@ func (rr *dnsRR_PTR) Walk(f func(v interface{}, name, tag string) bool) bool { type dnsRR_SOA struct { Hdr dnsRR_Header - Ns string `net:"domain-name"` - Mbox string `net:"domain-name"` + Ns string + Mbox string Serial uint32 Refresh uint32 Retry uint32 @@ -315,7 +248,7 @@ func (rr *dnsRR_TXT) Walk(f func(v interface{}, name, tag string) bool) bool { if !f(&txt, "Txt", "") { return false } - // more bytes than rr.Hdr.Rdlength said there woudld be + // more bytes than rr.Hdr.Rdlength said there would be if rr.Hdr.Rdlength-n < uint16(len(txt))+1 { return false } @@ -330,7 +263,7 @@ type dnsRR_SRV struct { Priority uint16 Weight uint16 Port uint16 - Target string `net:"domain-name"` + Target string } func (rr *dnsRR_SRV) Header() *dnsRR_Header { @@ -347,7 +280,7 @@ func (rr *dnsRR_SRV) Walk(f func(v interface{}, name, tag string) bool) bool { type dnsRR_A struct { Hdr dnsRR_Header - A uint32 `net:"ipv4"` + A uint32 } func (rr *dnsRR_A) Header() *dnsRR_Header { @@ -360,7 +293,7 @@ func (rr *dnsRR_A) Walk(f func(v interface{}, name, tag string) bool) bool { type dnsRR_AAAA struct { Hdr dnsRR_Header - AAAA [16]byte `net:"ipv6"` + AAAA [16]byte } func (rr *dnsRR_AAAA) Header() *dnsRR_Header { @@ -376,17 +309,12 @@ func (rr *dnsRR_AAAA) Walk(f func(v interface{}, name, tag string) bool) bool { // All the packers and unpackers take a (msg []byte, off int) // and return (off1 int, ok bool). If they return ok==false, they // also return off1==len(msg), so that the next unpacker will -// also fail. This lets us avoid checks of ok until the end of a +// also fail. This lets us avoid checks of ok until the end of a // packing sequence. // Map of constructors for each RR wire type. var rr_mk = map[int]func() dnsRR{ dnsTypeCNAME: func() dnsRR { return new(dnsRR_CNAME) }, - dnsTypeHINFO: func() dnsRR { return new(dnsRR_HINFO) }, - dnsTypeMB: func() dnsRR { return new(dnsRR_MB) }, - dnsTypeMG: func() dnsRR { return new(dnsRR_MG) }, - dnsTypeMINFO: func() dnsRR { return new(dnsRR_MINFO) }, - dnsTypeMR: func() dnsRR { return new(dnsRR_MR) }, dnsTypeMX: func() dnsRR { return new(dnsRR_MX) }, dnsTypeNS: func() dnsRR { return new(dnsRR_NS) }, dnsTypePTR: func() dnsRR { return new(dnsRR_PTR) }, @@ -399,13 +327,20 @@ var rr_mk = map[int]func() dnsRR{ // Pack a domain name s into msg[off:]. // Domain names are a sequence of counted strings -// split at the dots. They end with a zero-length string. +// split at the dots. They end with a zero-length string. func packDomainName(s string, msg []byte, off int) (off1 int, ok bool) { // Add trailing dot to canonicalize name. if n := len(s); n == 0 || s[n-1] != '.' { s += "." } + // Allow root domain. + if s == "." { + msg[off] = 0 + off++ + return off, true + } + // Each dot ends a segment of the name. // We trade each dot byte for a length byte. // There is also a trailing zero. @@ -422,8 +357,13 @@ func packDomainName(s string, msg []byte, off int) (off1 int, ok bool) { if i-begin >= 1<<6 { // top two bits of length must be clear return len(msg), false } + if i-begin == 0 { + return len(msg), false + } + msg[off] = byte(i - begin) off++ + for j := begin; j < i; j++ { msg[off] = s[j] off++ @@ -440,8 +380,8 @@ func packDomainName(s string, msg []byte, off int) (off1 int, ok bool) { // In addition to the simple sequences of counted strings above, // domain names are allowed to refer to strings elsewhere in the // packet, to avoid repeating common suffixes when returning -// many entries in a single domain. The pointers are marked -// by a length byte with the top two bits set. Ignoring those +// many entries in a single domain. The pointers are marked +// by a length byte with the top two bits set. Ignoring those // two bits, that byte and the next give a 14 bit offset from msg[0] // where we should pick up the trail. // Note that if we jump elsewhere in the packet, @@ -494,6 +434,9 @@ Loop: return "", len(msg), false } } + if len(s) == 0 { + s = "." + } if ptr == 0 { off1 = off } @@ -803,20 +746,32 @@ func (dns *dnsMsg) Pack() (msg []byte, ok bool) { // Pack it in: header and then the pieces. off := 0 off, ok = packStruct(&dh, msg, off) + if !ok { + return nil, false + } for i := 0; i < len(question); i++ { off, ok = packStruct(&question[i], msg, off) + if !ok { + return nil, false + } } for i := 0; i < len(answer); i++ { off, ok = packRR(answer[i], msg, off) + if !ok { + return nil, false + } } for i := 0; i < len(ns); i++ { off, ok = packRR(ns[i], msg, off) + if !ok { + return nil, false + } } for i := 0; i < len(extra); i++ { off, ok = packRR(extra[i], msg, off) - } - if !ok { - return nil, false + if !ok { + return nil, false + } } return msg[0:off], true } @@ -848,6 +803,9 @@ func (dns *dnsMsg) Unpack(msg []byte) bool { for i := 0; i < len(dns.question); i++ { off, ok = unpackStruct(&dns.question[i], msg, off) + if !ok { + return false + } } for i := 0; i < int(dh.Ancount); i++ { rec, off, ok = unpackRR(msg, off) @@ -904,3 +862,23 @@ func (dns *dnsMsg) String() string { } return s } + +// IsResponseTo reports whether m is an acceptable response to query. +func (m *dnsMsg) IsResponseTo(query *dnsMsg) bool { + if !m.response { + return false + } + if m.id != query.id { + return false + } + if len(m.question) != len(query.question) { + return false + } + for i, q := range m.question { + q2 := query.question[i] + if !equalASCIILabel(q.Name, q2.Name) || q.Qtype != q2.Qtype || q.Qclass != q2.Qclass { + return false + } + } + return true +} diff --git a/libgo/go/net/dnsmsg_test.go b/libgo/go/net/dnsmsg_test.go index 1078d77ceb..2a25a21d4c 100644 --- a/libgo/go/net/dnsmsg_test.go +++ b/libgo/go/net/dnsmsg_test.go @@ -10,6 +10,103 @@ import ( "testing" ) +func TestStructPackUnpack(t *testing.T) { + want := dnsQuestion{ + Name: ".", + Qtype: dnsTypeA, + Qclass: dnsClassINET, + } + buf := make([]byte, 50) + n, ok := packStruct(&want, buf, 0) + if !ok { + t.Fatal("packing failed") + } + buf = buf[:n] + got := dnsQuestion{} + n, ok = unpackStruct(&got, buf, 0) + if !ok { + t.Fatal("unpacking failed") + } + if n != len(buf) { + t.Errorf("unpacked different amount than packed: got n = %d, want = %d", n, len(buf)) + } + if !reflect.DeepEqual(got, want) { + t.Errorf("got = %+v, want = %+v", got, want) + } +} + +func TestDomainNamePackUnpack(t *testing.T) { + tests := []struct { + in string + want string + ok bool + }{ + {"", ".", true}, + {".", ".", true}, + {"google..com", "", false}, + {"google.com", "google.com.", true}, + {"google..com.", "", false}, + {"google.com.", "google.com.", true}, + {".google.com.", "", false}, + {"www..google.com.", "", false}, + {"www.google.com.", "www.google.com.", true}, + } + + for _, test := range tests { + buf := make([]byte, 30) + n, ok := packDomainName(test.in, buf, 0) + if ok != test.ok { + t.Errorf("packing of %s: got ok = %t, want = %t", test.in, ok, test.ok) + continue + } + if !test.ok { + continue + } + buf = buf[:n] + got, n, ok := unpackDomainName(buf, 0) + if !ok { + t.Errorf("unpacking for %s failed", test.in) + continue + } + if n != len(buf) { + t.Errorf( + "unpacked different amount than packed for %s: got n = %d, want = %d", + test.in, + n, + len(buf), + ) + } + if got != test.want { + t.Errorf("unpacking packing of %s: got = %s, want = %s", test.in, got, test.want) + } + } +} + +func TestDNSPackUnpack(t *testing.T) { + want := dnsMsg{ + question: []dnsQuestion{{ + Name: ".", + Qtype: dnsTypeAAAA, + Qclass: dnsClassINET, + }}, + answer: []dnsRR{}, + ns: []dnsRR{}, + extra: []dnsRR{}, + } + b, ok := want.Pack() + if !ok { + t.Fatal("packing failed") + } + var got dnsMsg + ok = got.Unpack(b) + if !ok { + t.Fatal("unpacking failed") + } + if !reflect.DeepEqual(got, want) { + t.Errorf("got = %+v, want = %+v", got, want) + } +} + func TestDNSParseSRVReply(t *testing.T) { data, err := hex.DecodeString(dnsSRVReply) if err != nil { @@ -20,7 +117,7 @@ func TestDNSParseSRVReply(t *testing.T) { if !ok { t.Fatal("unpacking packet failed") } - msg.String() // exercise this code path + _ = msg.String() // exercise this code path if g, e := len(msg.answer), 5; g != e { t.Errorf("len(msg.answer) = %d; want %d", g, e) } @@ -68,7 +165,7 @@ func TestDNSParseCorruptSRVReply(t *testing.T) { if !ok { t.Fatal("unpacking packet failed") } - msg.String() // exercise this code path + _ = msg.String() // exercise this code path if g, e := len(msg.answer), 5; g != e { t.Errorf("len(msg.answer) = %d; want %d", g, e) } @@ -183,6 +280,124 @@ func TestDNSParseTXTCorruptTXTLengthReply(t *testing.T) { } } +func TestIsResponseTo(t *testing.T) { + // Sample DNS query. + query := dnsMsg{ + dnsMsgHdr: dnsMsgHdr{ + id: 42, + }, + question: []dnsQuestion{ + { + Name: "www.example.com.", + Qtype: dnsTypeA, + Qclass: dnsClassINET, + }, + }, + } + + resp := query + resp.response = true + if !resp.IsResponseTo(&query) { + t.Error("got false, want true") + } + + badResponses := []dnsMsg{ + // Different ID. + { + dnsMsgHdr: dnsMsgHdr{ + id: 43, + response: true, + }, + question: []dnsQuestion{ + { + Name: "www.example.com.", + Qtype: dnsTypeA, + Qclass: dnsClassINET, + }, + }, + }, + + // Different query name. + { + dnsMsgHdr: dnsMsgHdr{ + id: 42, + response: true, + }, + question: []dnsQuestion{ + { + Name: "www.google.com.", + Qtype: dnsTypeA, + Qclass: dnsClassINET, + }, + }, + }, + + // Different query type. + { + dnsMsgHdr: dnsMsgHdr{ + id: 42, + response: true, + }, + question: []dnsQuestion{ + { + Name: "www.example.com.", + Qtype: dnsTypeAAAA, + Qclass: dnsClassINET, + }, + }, + }, + + // Different query class. + { + dnsMsgHdr: dnsMsgHdr{ + id: 42, + response: true, + }, + question: []dnsQuestion{ + { + Name: "www.example.com.", + Qtype: dnsTypeA, + Qclass: dnsClassCSNET, + }, + }, + }, + + // No questions. + { + dnsMsgHdr: dnsMsgHdr{ + id: 42, + response: true, + }, + }, + + // Extra questions. + { + dnsMsgHdr: dnsMsgHdr{ + id: 42, + response: true, + }, + question: []dnsQuestion{ + { + Name: "www.example.com.", + Qtype: dnsTypeA, + Qclass: dnsClassINET, + }, + { + Name: "www.golang.org.", + Qtype: dnsTypeAAAA, + Qclass: dnsClassINET, + }, + }, + }, + } + + for i := range badResponses { + if badResponses[i].IsResponseTo(&query) { + t.Errorf("%v: got true, want false", i) + } + } +} + // Valid DNS SRV reply const dnsSRVReply = "0901818000010005000000000c5f786d70702d736572766572045f74637006676f6f67" + "6c6503636f6d0000210001c00c002100010000012c00210014000014950c786d70702d" + diff --git a/libgo/go/net/dnsname_test.go b/libgo/go/net/dnsname_test.go index be07dc6a16..e0f786dec8 100644 --- a/libgo/go/net/dnsname_test.go +++ b/libgo/go/net/dnsname_test.go @@ -15,7 +15,7 @@ type dnsNameTest struct { } var dnsNameTests = []dnsNameTest{ - // RFC2181, section 11. + // RFC 2181, section 11. {"_xmpp-server._tcp.google.com", true}, {"foo.com", true}, {"1foo.com", true}, @@ -32,14 +32,12 @@ var dnsNameTests = []dnsNameTest{ func emitDNSNameTest(ch chan<- dnsNameTest) { defer close(ch) - var char59 = "" var char63 = "" - var char64 = "" - for i := 0; i < 59; i++ { - char59 += "a" + for i := 0; i < 63; i++ { + char63 += "a" } - char63 = char59 + "aaaa" - char64 = char63 + "a" + char64 := char63 + "a" + longDomain := strings.Repeat(char63+".", 5) + "example" for _, tc := range dnsNameTests { ch <- tc @@ -47,14 +45,15 @@ func emitDNSNameTest(ch chan<- dnsNameTest) { ch <- dnsNameTest{char63 + ".com", true} ch <- dnsNameTest{char64 + ".com", false} - // 255 char name is fine: - ch <- dnsNameTest{char59 + "." + char63 + "." + char63 + "." + - char63 + ".com", - true} - // 256 char name is bad: - ch <- dnsNameTest{char59 + "a." + char63 + "." + char63 + "." + - char63 + ".com", - false} + + // Remember: wire format is two octets longer than presentation + // (length octets for the first and [root] last labels). + // 253 is fine: + ch <- dnsNameTest{longDomain[len(longDomain)-253:], true} + // A terminal dot doesn't contribute to length: + ch <- dnsNameTest{longDomain[len(longDomain)-253:] + ".", true} + // 254 is bad: + ch <- dnsNameTest{longDomain[len(longDomain)-254:], false} } func TestDNSName(t *testing.T) { diff --git a/libgo/go/net/error_plan9_test.go b/libgo/go/net/error_plan9_test.go index 495ea96534..d7c7f1487f 100644 --- a/libgo/go/net/error_plan9_test.go +++ b/libgo/go/net/error_plan9_test.go @@ -9,6 +9,8 @@ import "syscall" var ( errTimedout = syscall.ETIMEDOUT errOpNotSupported = syscall.EPLAN9 + + abortedConnRequestErrors []error ) func isPlatformError(err error) bool { diff --git a/libgo/go/net/error_posix_test.go b/libgo/go/net/error_posix_test.go index 981cc837ba..b411a378df 100644 --- a/libgo/go/net/error_posix_test.go +++ b/libgo/go/net/error_posix_test.go @@ -12,16 +12,6 @@ import ( "testing" ) -var ( - errTimedout = syscall.ETIMEDOUT - errOpNotSupported = syscall.EOPNOTSUPP -) - -func isPlatformError(err error) bool { - _, ok := err.(syscall.Errno) - return ok -} - func TestSpuriousENOTAVAIL(t *testing.T) { for _, tt := range []struct { error diff --git a/libgo/go/net/error_test.go b/libgo/go/net/error_test.go index 1aab14c449..c23da49fad 100644 --- a/libgo/go/net/error_test.go +++ b/libgo/go/net/error_test.go @@ -5,6 +5,7 @@ package net import ( + "context" "fmt" "io" "io/ioutil" @@ -91,9 +92,13 @@ second: case *os.SyscallError: nestedErr = err.Err goto third + case *os.PathError: // for Plan 9 + nestedErr = err.Err + goto third } switch nestedErr { - case errCanceled, errClosing, errMissingAddress: + case errCanceled, errClosing, errMissingAddress, errNoSuitableAddress, + context.DeadlineExceeded, context.Canceled: return nil } return fmt.Errorf("unexpected type on 2nd nested level: %T", nestedErr) @@ -135,7 +140,7 @@ func TestDialError(t *testing.T) { origTestHookLookupIP := testHookLookupIP defer func() { testHookLookupIP = origTestHookLookupIP }() - testHookLookupIP = func(fn func(string) ([]IPAddr, error), host string) ([]IPAddr, error) { + testHookLookupIP = func(ctx context.Context, fn func(context.Context, string) ([]IPAddr, error), host string) ([]IPAddr, error) { return nil, &DNSError{Err: "dial error test", Name: "name", Server: "server", IsTimeout: true} } sw.Set(socktest.FilterConnect, func(so *socktest.Status) (socktest.AfterFilter, error) { @@ -203,6 +208,62 @@ func TestProtocolDialError(t *testing.T) { } } +func TestDialAddrError(t *testing.T) { + switch runtime.GOOS { + case "nacl", "plan9": + t.Skipf("not supported on %s", runtime.GOOS) + } + if !supportsIPv4 || !supportsIPv6 { + t.Skip("both IPv4 and IPv6 are required") + } + + for _, tt := range []struct { + network string + lit string + addr *TCPAddr + }{ + {"tcp4", "::1", nil}, + {"tcp4", "", &TCPAddr{IP: IPv6loopback}}, + // We don't test the {"tcp6", "byte sequence", nil} + // case for now because there is no easy way to + // control name resolution. + {"tcp6", "", &TCPAddr{IP: IP{0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef}}}, + } { + var err error + var c Conn + var op string + if tt.lit != "" { + c, err = Dial(tt.network, JoinHostPort(tt.lit, "0")) + op = fmt.Sprintf("Dial(%q, %q)", tt.network, JoinHostPort(tt.lit, "0")) + } else { + c, err = DialTCP(tt.network, nil, tt.addr) + op = fmt.Sprintf("DialTCP(%q, %q)", tt.network, tt.addr) + } + if err == nil { + c.Close() + t.Errorf("%s succeeded, want error", op) + continue + } + if perr := parseDialError(err); perr != nil { + t.Errorf("%s: %v", op, perr) + continue + } + operr := err.(*OpError).Err + aerr, ok := operr.(*AddrError) + if !ok { + t.Errorf("%s: %v is %T, want *AddrError", op, err, operr) + continue + } + want := tt.lit + if tt.lit == "" { + want = tt.addr.IP.String() + } + if aerr.Addr != want { + t.Errorf("%s: %v, error Addr=%q, want %q", op, err, aerr.Addr, want) + } + } +} + var listenErrorTests = []struct { network, address string }{ @@ -228,7 +289,7 @@ func TestListenError(t *testing.T) { origTestHookLookupIP := testHookLookupIP defer func() { testHookLookupIP = origTestHookLookupIP }() - testHookLookupIP = func(fn func(string) ([]IPAddr, error), host string) ([]IPAddr, error) { + testHookLookupIP = func(_ context.Context, fn func(context.Context, string) ([]IPAddr, error), host string) ([]IPAddr, error) { return nil, &DNSError{Err: "listen error test", Name: "name", Server: "server", IsTimeout: true} } sw.Set(socktest.FilterListen, func(so *socktest.Status) (socktest.AfterFilter, error) { @@ -288,7 +349,7 @@ func TestListenPacketError(t *testing.T) { origTestHookLookupIP := testHookLookupIP defer func() { testHookLookupIP = origTestHookLookupIP }() - testHookLookupIP = func(fn func(string) ([]IPAddr, error), host string) ([]IPAddr, error) { + testHookLookupIP = func(_ context.Context, fn func(context.Context, string) ([]IPAddr, error), host string) ([]IPAddr, error) { return nil, &DNSError{Err: "listen error test", Name: "name", Server: "server", IsTimeout: true} } @@ -413,7 +474,7 @@ second: goto third } switch nestedErr { - case errCanceled, errClosing, errTimeout, ErrWriteToConnected, io.ErrUnexpectedEOF: + case errCanceled, errClosing, errMissingAddress, errTimeout, ErrWriteToConnected, io.ErrUnexpectedEOF: return nil } return fmt.Errorf("unexpected type on 2nd nested level: %T", nestedErr) @@ -465,6 +526,10 @@ third: if isPlatformError(nestedErr) { return nil } + switch nestedErr { + case os.ErrClosed: // for Plan 9 + return nil + } return fmt.Errorf("unexpected type on 3rd nested level: %T", nestedErr) } @@ -543,6 +608,9 @@ second: case *os.SyscallError: nestedErr = err.Err goto third + case *os.PathError: // for Plan 9 + nestedErr = err.Err + goto third } switch nestedErr { case errClosing, errTimeout: @@ -703,3 +771,17 @@ func TestFileError(t *testing.T) { ln.Close() } } + +func parseLookupPortError(nestedErr error) error { + if nestedErr == nil { + return nil + } + + switch nestedErr.(type) { + case *AddrError, *DNSError: + return nil + case *os.PathError: // for Plan 9 + return nil + } + return fmt.Errorf("unexpected type on 1st nested level: %T", nestedErr) +} diff --git a/libgo/go/net/error_unix_test.go b/libgo/go/net/error_unix_test.go new file mode 100644 index 0000000000..9ce9e12c5e --- /dev/null +++ b/libgo/go/net/error_unix_test.go @@ -0,0 +1,34 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !plan9,!windows + +package net + +import ( + "os" + "syscall" +) + +var ( + errTimedout = syscall.ETIMEDOUT + errOpNotSupported = syscall.EOPNOTSUPP + + abortedConnRequestErrors = []error{syscall.ECONNABORTED} // see accept in fd_unix.go +) + +func isPlatformError(err error) bool { + _, ok := err.(syscall.Errno) + return ok +} + +func samePlatformError(err, want error) bool { + if op, ok := err.(*OpError); ok { + err = op.Err + } + if sys, ok := err.(*os.SyscallError); ok { + err = sys.Err + } + return err == want +} diff --git a/libgo/go/net/error_windows_test.go b/libgo/go/net/error_windows_test.go new file mode 100644 index 0000000000..834a9de441 --- /dev/null +++ b/libgo/go/net/error_windows_test.go @@ -0,0 +1,19 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import "syscall" + +var ( + errTimedout = syscall.ETIMEDOUT + errOpNotSupported = syscall.EOPNOTSUPP + + abortedConnRequestErrors = []error{syscall.ERROR_NETNAME_DELETED, syscall.WSAECONNRESET} // see accept in fd_windows.go +) + +func isPlatformError(err error) bool { + _, ok := err.(syscall.Errno) + return ok +} diff --git a/libgo/go/net/external_test.go b/libgo/go/net/external_test.go index d5ff2be20a..e18b547cac 100644 --- a/libgo/go/net/external_test.go +++ b/libgo/go/net/external_test.go @@ -6,15 +6,15 @@ package net import ( "fmt" + "internal/testenv" "io" "strings" "testing" ) func TestResolveGoogle(t *testing.T) { - if testing.Short() || !*testExternal { - t.Skip("avoid external network") - } + testenv.MustHaveExternalNetwork(t) + if !supportsIPv4 || !supportsIPv6 || !*testIPv4 || !*testIPv6 { t.Skip("both IPv4 and IPv6 are required") } @@ -60,9 +60,8 @@ var dialGoogleTests = []struct { } func TestDialGoogle(t *testing.T) { - if testing.Short() || !*testExternal { - t.Skip("avoid external network") - } + testenv.MustHaveExternalNetwork(t) + if !supportsIPv4 || !supportsIPv6 || !*testIPv4 || !*testIPv6 { t.Skip("both IPv4 and IPv6 are required") } diff --git a/libgo/go/net/fd_io_plan9.go b/libgo/go/net/fd_io_plan9.go new file mode 100644 index 0000000000..76da0c546c --- /dev/null +++ b/libgo/go/net/fd_io_plan9.go @@ -0,0 +1,93 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "os" + "runtime" + "sync" + "syscall" +) + +// asyncIO implements asynchronous cancelable I/O. +// An asyncIO represents a single asynchronous Read or Write +// operation. The result is returned on the result channel. +// The undergoing I/O system call can either complete or be +// interrupted by a note. +type asyncIO struct { + res chan result + + // mu guards the pid field. + mu sync.Mutex + + // pid holds the process id of + // the process running the IO operation. + pid int +} + +// result is the return value of a Read or Write operation. +type result struct { + n int + err error +} + +// newAsyncIO returns a new asyncIO that performs an I/O +// operation by calling fn, which must do one and only one +// interruptible system call. +func newAsyncIO(fn func([]byte) (int, error), b []byte) *asyncIO { + aio := &asyncIO{ + res: make(chan result, 0), + } + aio.mu.Lock() + go func() { + // Lock the current goroutine to its process + // and store the pid in io so that Cancel can + // interrupt it. We ignore the "hangup" signal, + // so the signal does not take down the entire + // Go runtime. + runtime.LockOSThread() + runtime_ignoreHangup() + aio.pid = os.Getpid() + aio.mu.Unlock() + + n, err := fn(b) + + aio.mu.Lock() + aio.pid = -1 + runtime_unignoreHangup() + aio.mu.Unlock() + + aio.res <- result{n, err} + }() + return aio +} + +var hangupNote os.Signal = syscall.Note("hangup") + +// Cancel interrupts the I/O operation, causing +// the Wait function to return. +func (aio *asyncIO) Cancel() { + aio.mu.Lock() + defer aio.mu.Unlock() + if aio.pid == -1 { + return + } + proc, err := os.FindProcess(aio.pid) + if err != nil { + return + } + proc.Signal(hangupNote) +} + +// Wait for the I/O operation to complete. +func (aio *asyncIO) Wait() (int, error) { + res := <-aio.res + return res.n, res.err +} + +// The following functions, provided by the runtime, are used to +// ignore and unignore the "hangup" signal received by the process. +func runtime_ignoreHangup() +func runtime_unignoreHangup() diff --git a/libgo/go/net/fd_mutex.go b/libgo/go/net/fd_mutex.go index 6d5509d7f2..4591fd1cac 100644 --- a/libgo/go/net/fd_mutex.go +++ b/libgo/go/net/fd_mutex.go @@ -6,9 +6,9 @@ package net import "sync/atomic" -// fdMutex is a specialized synchronization primitive -// that manages lifetime of an fd and serializes access -// to Read and Write methods on netFD. +// fdMutex is a specialized synchronization primitive that manages +// lifetime of an fd and serializes access to Read, Write and Close +// methods on netFD. type fdMutex struct { state uint64 rsema uint32 @@ -34,18 +34,21 @@ const ( mutexWMask = (1<<20 - 1) << 43 ) -// Read operations must do RWLock(true)/RWUnlock(true). -// Write operations must do RWLock(false)/RWUnlock(false). -// Misc operations must do Incref/Decref. Misc operations include functions like -// setsockopt and setDeadline. They need to use Incref/Decref to ensure that -// they operate on the correct fd in presence of a concurrent Close call -// (otherwise fd can be closed under their feet). -// Close operation must do IncrefAndClose/Decref. +// Read operations must do rwlock(true)/rwunlock(true). +// +// Write operations must do rwlock(false)/rwunlock(false). +// +// Misc operations must do incref/decref. +// Misc operations include functions like setsockopt and setDeadline. +// They need to use incref/decref to ensure that they operate on the +// correct fd in presence of a concurrent close call (otherwise fd can +// be closed under their feet). +// +// Close operations must do increfAndClose/decref. -// RWLock/Incref return whether fd is open. -// RWUnlock/Decref return whether fd is closed and there are no remaining references. - -func (mu *fdMutex) Incref() bool { +// incref adds a reference to mu. +// It reports whether mu is available for reading or writing. +func (mu *fdMutex) incref() bool { for { old := atomic.LoadUint64(&mu.state) if old&mutexClosed != 0 { @@ -61,7 +64,9 @@ func (mu *fdMutex) Incref() bool { } } -func (mu *fdMutex) IncrefAndClose() bool { +// increfAndClose sets the state of mu to closed. +// It reports whether there is no remaining reference. +func (mu *fdMutex) increfAndClose() bool { for { old := atomic.LoadUint64(&mu.state) if old&mutexClosed != 0 { @@ -90,7 +95,9 @@ func (mu *fdMutex) IncrefAndClose() bool { } } -func (mu *fdMutex) Decref() bool { +// decref removes a reference from mu. +// It reports whether there is no remaining reference. +func (mu *fdMutex) decref() bool { for { old := atomic.LoadUint64(&mu.state) if old&mutexRefMask == 0 { @@ -103,7 +110,9 @@ func (mu *fdMutex) Decref() bool { } } -func (mu *fdMutex) RWLock(read bool) bool { +// lock adds a reference to mu and locks mu. +// It reports whether mu is available for reading or writing. +func (mu *fdMutex) rwlock(read bool) bool { var mutexBit, mutexWait, mutexMask uint64 var mutexSema *uint32 if read { @@ -146,7 +155,9 @@ func (mu *fdMutex) RWLock(read bool) bool { } } -func (mu *fdMutex) RWUnlock(read bool) bool { +// unlock removes a reference from mu and unlocks mu. +// It reports whether there is no remaining reference. +func (mu *fdMutex) rwunlock(read bool) bool { var mutexBit, mutexWait, mutexMask uint64 var mutexSema *uint32 if read { @@ -182,3 +193,57 @@ func (mu *fdMutex) RWUnlock(read bool) bool { // Implemented in runtime package. func runtime_Semacquire(sema *uint32) func runtime_Semrelease(sema *uint32) + +// incref adds a reference to fd. +// It returns an error when fd cannot be used. +func (fd *netFD) incref() error { + if !fd.fdmu.incref() { + return errClosing + } + return nil +} + +// decref removes a reference from fd. +// It also closes fd when the state of fd is set to closed and there +// is no remaining reference. +func (fd *netFD) decref() { + if fd.fdmu.decref() { + fd.destroy() + } +} + +// readLock adds a reference to fd and locks fd for reading. +// It returns an error when fd cannot be used for reading. +func (fd *netFD) readLock() error { + if !fd.fdmu.rwlock(true) { + return errClosing + } + return nil +} + +// readUnlock removes a reference from fd and unlocks fd for reading. +// It also closes fd when the state of fd is set to closed and there +// is no remaining reference. +func (fd *netFD) readUnlock() { + if fd.fdmu.rwunlock(true) { + fd.destroy() + } +} + +// writeLock adds a reference to fd and locks fd for writing. +// It returns an error when fd cannot be used for writing. +func (fd *netFD) writeLock() error { + if !fd.fdmu.rwlock(false) { + return errClosing + } + return nil +} + +// writeUnlock removes a reference from fd and unlocks fd for writing. +// It also closes fd when the state of fd is set to closed and there +// is no remaining reference. +func (fd *netFD) writeUnlock() { + if fd.fdmu.rwunlock(false) { + fd.destroy() + } +} diff --git a/libgo/go/net/fd_mutex_test.go b/libgo/go/net/fd_mutex_test.go index c34ec59b99..3542c70f9d 100644 --- a/libgo/go/net/fd_mutex_test.go +++ b/libgo/go/net/fd_mutex_test.go @@ -14,44 +14,44 @@ import ( func TestMutexLock(t *testing.T) { var mu fdMutex - if !mu.Incref() { + if !mu.incref() { t.Fatal("broken") } - if mu.Decref() { + if mu.decref() { t.Fatal("broken") } - if !mu.RWLock(true) { + if !mu.rwlock(true) { t.Fatal("broken") } - if mu.RWUnlock(true) { + if mu.rwunlock(true) { t.Fatal("broken") } - if !mu.RWLock(false) { + if !mu.rwlock(false) { t.Fatal("broken") } - if mu.RWUnlock(false) { + if mu.rwunlock(false) { t.Fatal("broken") } } func TestMutexClose(t *testing.T) { var mu fdMutex - if !mu.IncrefAndClose() { + if !mu.increfAndClose() { t.Fatal("broken") } - if mu.Incref() { + if mu.incref() { t.Fatal("broken") } - if mu.RWLock(true) { + if mu.rwlock(true) { t.Fatal("broken") } - if mu.RWLock(false) { + if mu.rwlock(false) { t.Fatal("broken") } - if mu.IncrefAndClose() { + if mu.increfAndClose() { t.Fatal("broken") } } @@ -59,10 +59,10 @@ func TestMutexClose(t *testing.T) { func TestMutexCloseUnblock(t *testing.T) { c := make(chan bool) var mu fdMutex - mu.RWLock(true) + mu.rwlock(true) for i := 0; i < 4; i++ { go func() { - if mu.RWLock(true) { + if mu.rwlock(true) { t.Error("broken") return } @@ -76,7 +76,7 @@ func TestMutexCloseUnblock(t *testing.T) { t.Fatal("broken") default: } - mu.IncrefAndClose() // Must unblock the readers. + mu.increfAndClose() // Must unblock the readers. for i := 0; i < 4; i++ { select { case <-c: @@ -84,10 +84,10 @@ func TestMutexCloseUnblock(t *testing.T) { t.Fatal("broken") } } - if mu.Decref() { + if mu.decref() { t.Fatal("broken") } - if !mu.RWUnlock(true) { + if !mu.rwunlock(true) { t.Fatal("broken") } } @@ -103,21 +103,21 @@ func TestMutexPanic(t *testing.T) { } var mu fdMutex - ensurePanics(func() { mu.Decref() }) - ensurePanics(func() { mu.RWUnlock(true) }) - ensurePanics(func() { mu.RWUnlock(false) }) + ensurePanics(func() { mu.decref() }) + ensurePanics(func() { mu.rwunlock(true) }) + ensurePanics(func() { mu.rwunlock(false) }) - ensurePanics(func() { mu.Incref(); mu.Decref(); mu.Decref() }) - ensurePanics(func() { mu.RWLock(true); mu.RWUnlock(true); mu.RWUnlock(true) }) - ensurePanics(func() { mu.RWLock(false); mu.RWUnlock(false); mu.RWUnlock(false) }) + ensurePanics(func() { mu.incref(); mu.decref(); mu.decref() }) + ensurePanics(func() { mu.rwlock(true); mu.rwunlock(true); mu.rwunlock(true) }) + ensurePanics(func() { mu.rwlock(false); mu.rwunlock(false); mu.rwunlock(false) }) // ensure that it's still not broken - mu.Incref() - mu.Decref() - mu.RWLock(true) - mu.RWUnlock(true) - mu.RWLock(false) - mu.RWUnlock(false) + mu.incref() + mu.decref() + mu.rwlock(true) + mu.rwunlock(true) + mu.rwlock(false) + mu.rwunlock(false) } func TestMutexStress(t *testing.T) { @@ -138,16 +138,16 @@ func TestMutexStress(t *testing.T) { for i := 0; i < N; i++ { switch r.Intn(3) { case 0: - if !mu.Incref() { + if !mu.incref() { t.Error("broken") return } - if mu.Decref() { + if mu.decref() { t.Error("broken") return } case 1: - if !mu.RWLock(true) { + if !mu.rwlock(true) { t.Error("broken") return } @@ -158,12 +158,12 @@ func TestMutexStress(t *testing.T) { } readState[0]++ readState[1]++ - if mu.RWUnlock(true) { + if mu.rwunlock(true) { t.Error("broken") return } case 2: - if !mu.RWLock(false) { + if !mu.rwlock(false) { t.Error("broken") return } @@ -174,7 +174,7 @@ func TestMutexStress(t *testing.T) { } writeState[0]++ writeState[1]++ - if mu.RWUnlock(false) { + if mu.rwunlock(false) { t.Error("broken") return } @@ -186,10 +186,10 @@ func TestMutexStress(t *testing.T) { for p := 0; p < P; p++ { <-done } - if !mu.IncrefAndClose() { + if !mu.increfAndClose() { t.Fatal("broken") } - if !mu.Decref() { + if !mu.decref() { t.Fatal("broken") } } diff --git a/libgo/go/net/fd_plan9.go b/libgo/go/net/fd_plan9.go index cec88609d0..300d8c4543 100644 --- a/libgo/go/net/fd_plan9.go +++ b/libgo/go/net/fd_plan9.go @@ -1,4 +1,4 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -7,21 +7,37 @@ package net import ( "io" "os" + "sync/atomic" "syscall" "time" ) +type atomicBool int32 + +func (b *atomicBool) isSet() bool { return atomic.LoadInt32((*int32)(b)) != 0 } +func (b *atomicBool) setFalse() { atomic.StoreInt32((*int32)(b), 0) } +func (b *atomicBool) setTrue() { atomic.StoreInt32((*int32)(b), 1) } + // Network file descriptor. type netFD struct { // locking/lifetime of sysfd + serialize access to Read and Write methods fdmu fdMutex // immutable until Close - net string - n string - dir string - ctl, data *os.File - laddr, raddr Addr + net string + n string + dir string + listen, ctl, data *os.File + laddr, raddr Addr + isStream bool + + // deadlines + raio *asyncIO + waio *asyncIO + rtimer *time.Timer + wtimer *time.Timer + rtimedout atomicBool // set true when read deadline has been reached + wtimedout atomicBool // set true when write deadline has been reached } var ( @@ -32,14 +48,16 @@ func sysInit() { netdir = "/net" } -func dial(net string, ra Addr, dialer func(time.Time) (Conn, error), deadline time.Time) (Conn, error) { - // On plan9, use the relatively inefficient - // goroutine-racing implementation. - return dialChannel(net, ra, dialer, deadline) -} - -func newFD(net, name string, ctl, data *os.File, laddr, raddr Addr) (*netFD, error) { - return &netFD{net: net, n: name, dir: netdir + "/" + net + "/" + name, ctl: ctl, data: data, laddr: laddr, raddr: raddr}, nil +func newFD(net, name string, listen, ctl, data *os.File, laddr, raddr Addr) (*netFD, error) { + return &netFD{ + net: net, + n: name, + dir: netdir + "/" + net + "/" + name, + listen: listen, + ctl: ctl, data: data, + laddr: laddr, + raddr: raddr, + }, nil } func (fd *netFD) init() error { @@ -70,60 +88,20 @@ func (fd *netFD) destroy() { err = err1 } } + if fd.listen != nil { + if err1 := fd.listen.Close(); err1 != nil && err == nil { + err = err1 + } + } fd.ctl = nil fd.data = nil -} - -// Add a reference to this fd. -// Returns an error if the fd cannot be used. -func (fd *netFD) incref() error { - if !fd.fdmu.Incref() { - return errClosing - } - return nil -} - -// Remove a reference to this FD and close if we've been asked to do so -// (and there are no references left). -func (fd *netFD) decref() { - if fd.fdmu.Decref() { - fd.destroy() - } -} - -// Add a reference to this fd and lock for reading. -// Returns an error if the fd cannot be used. -func (fd *netFD) readLock() error { - if !fd.fdmu.RWLock(true) { - return errClosing - } - return nil -} - -// Unlock for reading and remove a reference to this FD. -func (fd *netFD) readUnlock() { - if fd.fdmu.RWUnlock(true) { - fd.destroy() - } -} - -// Add a reference to this fd and lock for writing. -// Returns an error if the fd cannot be used. -func (fd *netFD) writeLock() error { - if !fd.fdmu.RWLock(false) { - return errClosing - } - return nil -} - -// Unlock for writing and remove a reference to this FD. -func (fd *netFD) writeUnlock() { - if fd.fdmu.RWUnlock(false) { - fd.destroy() - } + fd.listen = nil } func (fd *netFD) Read(b []byte) (n int, err error) { + if fd.rtimedout.isSet() { + return 0, errTimeout + } if !fd.ok() || fd.data == nil { return 0, syscall.EINVAL } @@ -131,7 +109,18 @@ func (fd *netFD) Read(b []byte) (n int, err error) { return 0, err } defer fd.readUnlock() - n, err = fd.data.Read(b) + if len(b) == 0 { + return 0, nil + } + fd.raio = newAsyncIO(fd.data.Read, b) + n, err = fd.raio.Wait() + fd.raio = nil + if isHangup(err) { + err = io.EOF + } + if isInterrupted(err) { + err = errTimeout + } if fd.net == "udp" && err == io.EOF { n = 0 err = nil @@ -140,6 +129,9 @@ func (fd *netFD) Read(b []byte) (n int, err error) { } func (fd *netFD) Write(b []byte) (n int, err error) { + if fd.wtimedout.isSet() { + return 0, errTimeout + } if !fd.ok() || fd.data == nil { return 0, syscall.EINVAL } @@ -147,7 +139,13 @@ func (fd *netFD) Write(b []byte) (n int, err error) { return 0, err } defer fd.writeUnlock() - return fd.data.Write(b) + fd.waio = newAsyncIO(fd.data.Write, b) + n, err = fd.waio.Wait() + fd.waio = nil + if isInterrupted(err) { + err = errTimeout + } + return } func (fd *netFD) closeRead() error { @@ -165,7 +163,7 @@ func (fd *netFD) closeWrite() error { } func (fd *netFD) Close() error { - if !fd.fdmu.IncrefAndClose() { + if !fd.fdmu.increfAndClose() { return errClosing } if !fd.ok() { @@ -173,11 +171,10 @@ func (fd *netFD) Close() error { } if fd.net == "tcp" { // The following line is required to unblock Reads. - // For some reason, WriteString returns an error: - // "write /net/tcp/39/listen: inappropriate use of fd" - // But without it, Reads on dead conns hang forever. - // See Issue 9554. - fd.ctl.WriteString("hangup") + _, err := fd.ctl.WriteString("close") + if err != nil { + return err + } } err := fd.ctl.Close() if fd.data != nil { @@ -185,8 +182,14 @@ func (fd *netFD) Close() error { err = err1 } } + if fd.listen != nil { + if err1 := fd.listen.Close(); err1 != nil && err == nil { + err = err1 + } + } fd.ctl = nil fd.data = nil + fd.listen = nil return err } @@ -206,9 +209,7 @@ func (l *TCPListener) dup() (*os.File, error) { } func (fd *netFD) file(f *os.File, s string) (*os.File, error) { - syscall.ForkLock.RLock() dfd, err := syscall.Dup(int(f.Fd()), -1) - syscall.ForkLock.RUnlock() if err != nil { return nil, os.NewSyscallError("dup", err) } @@ -216,15 +217,74 @@ func (fd *netFD) file(f *os.File, s string) (*os.File, error) { } func (fd *netFD) setDeadline(t time.Time) error { - return syscall.EPLAN9 + return setDeadlineImpl(fd, t, 'r'+'w') } func (fd *netFD) setReadDeadline(t time.Time) error { - return syscall.EPLAN9 + return setDeadlineImpl(fd, t, 'r') } func (fd *netFD) setWriteDeadline(t time.Time) error { - return syscall.EPLAN9 + return setDeadlineImpl(fd, t, 'w') +} + +func setDeadlineImpl(fd *netFD, t time.Time, mode int) error { + d := t.Sub(time.Now()) + if mode == 'r' || mode == 'r'+'w' { + fd.rtimedout.setFalse() + } + if mode == 'w' || mode == 'r'+'w' { + fd.wtimedout.setFalse() + } + if t.IsZero() || d < 0 { + // Stop timer + if mode == 'r' || mode == 'r'+'w' { + if fd.rtimer != nil { + fd.rtimer.Stop() + } + fd.rtimer = nil + } + if mode == 'w' || mode == 'r'+'w' { + if fd.wtimer != nil { + fd.wtimer.Stop() + } + fd.wtimer = nil + } + } else { + // Interrupt I/O operation once timer has expired + if mode == 'r' || mode == 'r'+'w' { + fd.rtimer = time.AfterFunc(d, func() { + fd.rtimedout.setTrue() + if fd.raio != nil { + fd.raio.Cancel() + } + }) + } + if mode == 'w' || mode == 'r'+'w' { + fd.wtimer = time.AfterFunc(d, func() { + fd.wtimedout.setTrue() + if fd.waio != nil { + fd.waio.Cancel() + } + }) + } + } + if !t.IsZero() && d < 0 { + // Interrupt current I/O operation + if mode == 'r' || mode == 'r'+'w' { + fd.rtimedout.setTrue() + if fd.raio != nil { + fd.raio.Cancel() + } + } + if mode == 'w' || mode == 'r'+'w' { + fd.wtimedout.setTrue() + if fd.waio != nil { + fd.waio.Cancel() + } + } + } + return nil } func setReadBuffer(fd *netFD, bytes int) error { @@ -234,3 +294,11 @@ func setReadBuffer(fd *netFD, bytes int) error { func setWriteBuffer(fd *netFD, bytes int) error { return syscall.EPLAN9 } + +func isHangup(err error) bool { + return err != nil && stringsHasSuffix(err.Error(), "Hangup") +} + +func isInterrupted(err error) bool { + return err != nil && stringsHasSuffix(err.Error(), "interrupted") +} diff --git a/libgo/go/net/fd_poll_nacl.go b/libgo/go/net/fd_poll_nacl.go index cdf14e32ce..83987602a5 100644 --- a/libgo/go/net/fd_poll_nacl.go +++ b/libgo/go/net/fd_poll_nacl.go @@ -1,10 +1,11 @@ -// Copyright 2013 The Go Authors. All rights reserved. +// Copyright 2013 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package net import ( + "runtime" "syscall" "time" ) @@ -14,44 +15,45 @@ type pollDesc struct { closing bool } -func (pd *pollDesc) Init(fd *netFD) error { pd.fd = fd; return nil } +func (pd *pollDesc) init(fd *netFD) error { pd.fd = fd; return nil } -func (pd *pollDesc) Close() {} +func (pd *pollDesc) close() {} -func (pd *pollDesc) Evict() { +func (pd *pollDesc) evict() { pd.closing = true if pd.fd != nil { syscall.StopIO(pd.fd.sysfd) + runtime.KeepAlive(pd.fd) } } -func (pd *pollDesc) Prepare(mode int) error { +func (pd *pollDesc) prepare(mode int) error { if pd.closing { return errClosing } return nil } -func (pd *pollDesc) PrepareRead() error { return pd.Prepare('r') } +func (pd *pollDesc) prepareRead() error { return pd.prepare('r') } -func (pd *pollDesc) PrepareWrite() error { return pd.Prepare('w') } +func (pd *pollDesc) prepareWrite() error { return pd.prepare('w') } -func (pd *pollDesc) Wait(mode int) error { +func (pd *pollDesc) wait(mode int) error { if pd.closing { return errClosing } return errTimeout } -func (pd *pollDesc) WaitRead() error { return pd.Wait('r') } +func (pd *pollDesc) waitRead() error { return pd.wait('r') } -func (pd *pollDesc) WaitWrite() error { return pd.Wait('w') } +func (pd *pollDesc) waitWrite() error { return pd.wait('w') } -func (pd *pollDesc) WaitCanceled(mode int) {} +func (pd *pollDesc) waitCanceled(mode int) {} -func (pd *pollDesc) WaitCanceledRead() {} +func (pd *pollDesc) waitCanceledRead() {} -func (pd *pollDesc) WaitCanceledWrite() {} +func (pd *pollDesc) waitCanceledWrite() {} func (fd *netFD) setDeadline(t time.Time) error { return setDeadlineImpl(fd, t, 'r'+'w') diff --git a/libgo/go/net/fd_poll_runtime.go b/libgo/go/net/fd_poll_runtime.go index 8522ccebfb..62b69fcbf1 100644 --- a/libgo/go/net/fd_poll_runtime.go +++ b/libgo/go/net/fd_poll_runtime.go @@ -7,6 +7,7 @@ package net import ( + "runtime" "sync" "syscall" "time" @@ -30,9 +31,10 @@ type pollDesc struct { var serverInit sync.Once -func (pd *pollDesc) Init(fd *netFD) error { +func (pd *pollDesc) init(fd *netFD) error { serverInit.Do(runtime_pollServerInit) ctx, errno := runtime_pollOpen(uintptr(fd.sysfd)) + runtime.KeepAlive(fd) if errno != 0 { return syscall.Errno(errno) } @@ -40,7 +42,7 @@ func (pd *pollDesc) Init(fd *netFD) error { return nil } -func (pd *pollDesc) Close() { +func (pd *pollDesc) close() { if pd.runtimeCtx == 0 { return } @@ -49,49 +51,49 @@ func (pd *pollDesc) Close() { } // Evict evicts fd from the pending list, unblocking any I/O running on fd. -func (pd *pollDesc) Evict() { +func (pd *pollDesc) evict() { if pd.runtimeCtx == 0 { return } runtime_pollUnblock(pd.runtimeCtx) } -func (pd *pollDesc) Prepare(mode int) error { +func (pd *pollDesc) prepare(mode int) error { res := runtime_pollReset(pd.runtimeCtx, mode) return convertErr(res) } -func (pd *pollDesc) PrepareRead() error { - return pd.Prepare('r') +func (pd *pollDesc) prepareRead() error { + return pd.prepare('r') } -func (pd *pollDesc) PrepareWrite() error { - return pd.Prepare('w') +func (pd *pollDesc) prepareWrite() error { + return pd.prepare('w') } -func (pd *pollDesc) Wait(mode int) error { +func (pd *pollDesc) wait(mode int) error { res := runtime_pollWait(pd.runtimeCtx, mode) return convertErr(res) } -func (pd *pollDesc) WaitRead() error { - return pd.Wait('r') +func (pd *pollDesc) waitRead() error { + return pd.wait('r') } -func (pd *pollDesc) WaitWrite() error { - return pd.Wait('w') +func (pd *pollDesc) waitWrite() error { + return pd.wait('w') } -func (pd *pollDesc) WaitCanceled(mode int) { +func (pd *pollDesc) waitCanceled(mode int) { runtime_pollWaitCanceled(pd.runtimeCtx, mode) } -func (pd *pollDesc) WaitCanceledRead() { - pd.WaitCanceled('r') +func (pd *pollDesc) waitCanceledRead() { + pd.waitCanceled('r') } -func (pd *pollDesc) WaitCanceledWrite() { - pd.WaitCanceled('w') +func (pd *pollDesc) waitCanceledWrite() { + pd.waitCanceled('w') } func convertErr(res int) error { @@ -120,7 +122,13 @@ func (fd *netFD) setWriteDeadline(t time.Time) error { } func setDeadlineImpl(fd *netFD, t time.Time, mode int) error { - d := runtimeNano() + int64(t.Sub(time.Now())) + diff := int64(time.Until(t)) + d := runtimeNano() + diff + if d <= 0 && diff > 0 { + // If the user has a deadline in the future, but the delay calculation + // overflows, then set the deadline to the maximum possible value. + d = 1<<63 - 1 + } if t.IsZero() { d = 0 } diff --git a/libgo/go/net/fd_select.go b/libgo/go/net/fd_select.go deleted file mode 100644 index 4103c57e2c..0000000000 --- a/libgo/go/net/fd_select.go +++ /dev/null @@ -1,182 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Waiting for FDs via select(2). - -package net - -import ( - "errors" - "os" - "syscall" -) - -type pollster struct { - readFds, writeFds, repeatFds *syscall.FdSet - maxFd int - readyReadFds, readyWriteFds *syscall.FdSet - nReady int - lastFd int - closed bool -} - -func newpollster() (p *pollster, err error) { - p = new(pollster) - p.readFds = new(syscall.FdSet) - p.writeFds = new(syscall.FdSet) - p.repeatFds = new(syscall.FdSet) - p.readyReadFds = new(syscall.FdSet) - p.readyWriteFds = new(syscall.FdSet) - p.maxFd = -1 - p.nReady = 0 - p.lastFd = 0 - return p, nil -} - -func (p *pollster) AddFD(fd int, mode int, repeat bool) (bool, error) { - // pollServer is locked. - - if p.closed { - return false, errors.New("pollster closed") - } - - if mode == 'r' { - syscall.FDSet(fd, p.readFds) - } else { - syscall.FDSet(fd, p.writeFds) - } - - if repeat { - syscall.FDSet(fd, p.repeatFds) - } - - if fd > p.maxFd { - p.maxFd = fd - } - - return true, nil -} - -func (p *pollster) DelFD(fd int, mode int) bool { - // pollServer is locked. - - if p.closed { - return false - } - - if mode == 'r' { - if !syscall.FDIsSet(fd, p.readFds) { - print("Select unexpected fd=", fd, " for read\n") - return false - } - syscall.FDClr(fd, p.readFds) - } else { - if !syscall.FDIsSet(fd, p.writeFds) { - print("Select unexpected fd=", fd, " for write\n") - return false - } - syscall.FDClr(fd, p.writeFds) - } - - // Doesn't matter if not already present. - syscall.FDClr(fd, p.repeatFds) - - // We don't worry about maxFd here. - - return true -} - -func (p *pollster) WaitFD(s *pollServer, nsec int64) (fd int, mode int, err error) { - if p.nReady == 0 { - var timeout *syscall.Timeval - var tv syscall.Timeval - timeout = nil - if nsec > 0 { - tv = syscall.NsecToTimeval(nsec) - timeout = &tv - } - - var n int - var e error - var tmpReadFds, tmpWriteFds syscall.FdSet - for { - if p.closed { - return -1, 0, errors.New("pollster closed") - } - - // Temporary syscall.FdSet's into which the values are copied - // because select mutates the values. - tmpReadFds = *p.readFds - tmpWriteFds = *p.writeFds - - s.Unlock() - n, e = syscall.Select(p.maxFd+1, &tmpReadFds, &tmpWriteFds, nil, timeout) - s.Lock() - - if e != syscall.EINTR { - break - } - } - if e == syscall.EBADF { - // Some file descriptor has been closed. - tmpReadFds = syscall.FdSet{} - tmpWriteFds = syscall.FdSet{} - n = 0 - for i := 0; i < p.maxFd+1; i++ { - if syscall.FDIsSet(i, p.readFds) { - var s syscall.Stat_t - if syscall.Fstat(i, &s) == syscall.EBADF { - syscall.FDSet(i, &tmpReadFds) - n++ - } - } else if syscall.FDIsSet(i, p.writeFds) { - var s syscall.Stat_t - if syscall.Fstat(i, &s) == syscall.EBADF { - syscall.FDSet(i, &tmpWriteFds) - n++ - } - } - } - } else if e != nil { - return -1, 0, os.NewSyscallError("select", e) - } - if n == 0 { - return -1, 0, nil - } - - p.nReady = n - *p.readyReadFds = tmpReadFds - *p.readyWriteFds = tmpWriteFds - p.lastFd = 0 - } - - flag := false - for i := p.lastFd; i < p.maxFd+1; i++ { - if syscall.FDIsSet(i, p.readyReadFds) { - flag = true - mode = 'r' - syscall.FDClr(i, p.readyReadFds) - } else if syscall.FDIsSet(i, p.readyWriteFds) { - flag = true - mode = 'w' - syscall.FDClr(i, p.readyWriteFds) - } - if flag { - if !syscall.FDIsSet(i, p.repeatFds) { - p.DelFD(i, mode) - } - p.nReady-- - p.lastFd = i - return i, mode, nil - } - } - - // Will not reach here. Just to shut up the compiler. - return -1, 0, nil -} - -func (p *pollster) Close() error { - p.closed = true - return nil -} diff --git a/libgo/go/net/fd_unix.go b/libgo/go/net/fd_unix.go index ff498c2bff..9bc5ebc7a0 100644 --- a/libgo/go/net/fd_unix.go +++ b/libgo/go/net/fd_unix.go @@ -7,12 +7,12 @@ package net import ( + "context" "io" "os" "runtime" "sync/atomic" "syscall" - "time" ) // Network file descriptor. @@ -24,11 +24,15 @@ type netFD struct { sysfd int family int sotype int + isStream bool isConnected bool net string laddr Addr raddr Addr + // writev cache. + iovecs *[]syscall.Iovec + // wait server pd pollDesc } @@ -36,16 +40,12 @@ type netFD struct { func sysInit() { } -func dial(network string, ra Addr, dialer func(time.Time) (Conn, error), deadline time.Time) (Conn, error) { - return dialer(deadline) -} - func newFD(sysfd, family, sotype int, net string) (*netFD, error) { - return &netFD{sysfd: sysfd, family: family, sotype: sotype, net: net}, nil + return &netFD{sysfd: sysfd, family: family, sotype: sotype, net: net, isStream: sotype == syscall.SOCK_STREAM}, nil } func (fd *netFD) init() error { - if err := fd.pd.Init(fd); err != nil { + if err := fd.pd.init(fd); err != nil { return err } return nil @@ -68,15 +68,17 @@ func (fd *netFD) name() string { return fd.net + ":" + ls + "->" + rs } -func (fd *netFD) connect(la, ra syscall.Sockaddr, deadline time.Time, cancel <-chan struct{}) error { +func (fd *netFD) connect(ctx context.Context, la, ra syscall.Sockaddr) (ret error) { // Do not need to call fd.writeLock here, // because fd is not yet accessible to user, // so no concurrent operations are possible. switch err := connectFunc(fd.sysfd, ra); err { case syscall.EINPROGRESS, syscall.EALREADY, syscall.EINTR: case nil, syscall.EISCONN: - if !deadline.IsZero() && deadline.Before(time.Now()) { - return errTimeout + select { + case <-ctx.Done(): + return mapErr(ctx.Err()) + default: } if err := fd.init(); err != nil { return err @@ -98,23 +100,50 @@ func (fd *netFD) connect(la, ra syscall.Sockaddr, deadline time.Time, cancel <-c if err := fd.init(); err != nil { return err } - if !deadline.IsZero() { + if deadline, _ := ctx.Deadline(); !deadline.IsZero() { fd.setWriteDeadline(deadline) defer fd.setWriteDeadline(noDeadline) } - if cancel != nil { - done := make(chan bool) - defer close(done) + + // Start the "interrupter" goroutine, if this context might be canceled. + // (The background context cannot) + // + // The interrupter goroutine waits for the context to be done and + // interrupts the dial (by altering the fd's write deadline, which + // wakes up waitWrite). + if ctx != context.Background() { + // Wait for the interrupter goroutine to exit before returning + // from connect. + done := make(chan struct{}) + interruptRes := make(chan error) + defer func() { + close(done) + if ctxErr := <-interruptRes; ctxErr != nil && ret == nil { + // The interrupter goroutine called setWriteDeadline, + // but the connect code below had returned from + // waitWrite already and did a successful connect (ret + // == nil). Because we've now poisoned the connection + // by making it unwritable, don't return a successful + // dial. This was issue 16523. + ret = ctxErr + fd.Close() // prevent a leak + } + }() go func() { select { - case <-cancel: - // Force the runtime's poller to immediately give - // up waiting for writability. + case <-ctx.Done(): + // Force the runtime's poller to immediately give up + // waiting for writability, unblocking waitWrite + // below. fd.setWriteDeadline(aLongTimeAgo) + testHookCanceledDial() + interruptRes <- ctx.Err() case <-done: + interruptRes <- nil } }() } + for { // Performing multiple connect system calls on a // non-blocking socket under Unix variants does not @@ -124,10 +153,10 @@ func (fd *netFD) connect(la, ra syscall.Sockaddr, deadline time.Time, cancel <-c // SO_ERROR socket option to see if the connection // succeeded or failed. See issue 7474 for further // details. - if err := fd.pd.WaitWrite(); err != nil { + if err := fd.pd.waitWrite(); err != nil { select { - case <-cancel: - return errCanceled + case <-ctx.Done(): + return mapErr(ctx.Err()) default: } return err @@ -139,7 +168,16 @@ func (fd *netFD) connect(la, ra syscall.Sockaddr, deadline time.Time, cancel <-c switch err := syscall.Errno(nerr); err { case syscall.EINPROGRESS, syscall.EALREADY, syscall.EINTR: case syscall.Errno(0), syscall.EISCONN: - return nil + if runtime.GOOS != "darwin" { + return nil + } + // See golang.org/issue/14548. + // On Darwin, multiple connect system calls on + // a non-blocking socket never harm SO_ERROR. + switch err := connectFunc(fd.sysfd, ra); err { + case nil, syscall.EISCONN: + return nil + } default: return os.NewSyscallError("getsockopt", err) } @@ -149,71 +187,22 @@ func (fd *netFD) connect(la, ra syscall.Sockaddr, deadline time.Time, cancel <-c func (fd *netFD) destroy() { // Poller may want to unregister fd in readiness notification mechanism, // so this must be executed before closeFunc. - fd.pd.Close() + fd.pd.close() closeFunc(fd.sysfd) fd.sysfd = -1 runtime.SetFinalizer(fd, nil) } -// Add a reference to this fd. -// Returns an error if the fd cannot be used. -func (fd *netFD) incref() error { - if !fd.fdmu.Incref() { - return errClosing - } - return nil -} - -// Remove a reference to this FD and close if we've been asked to do so -// (and there are no references left). -func (fd *netFD) decref() { - if fd.fdmu.Decref() { - fd.destroy() - } -} - -// Add a reference to this fd and lock for reading. -// Returns an error if the fd cannot be used. -func (fd *netFD) readLock() error { - if !fd.fdmu.RWLock(true) { - return errClosing - } - return nil -} - -// Unlock for reading and remove a reference to this FD. -func (fd *netFD) readUnlock() { - if fd.fdmu.RWUnlock(true) { - fd.destroy() - } -} - -// Add a reference to this fd and lock for writing. -// Returns an error if the fd cannot be used. -func (fd *netFD) writeLock() error { - if !fd.fdmu.RWLock(false) { - return errClosing - } - return nil -} - -// Unlock for writing and remove a reference to this FD. -func (fd *netFD) writeUnlock() { - if fd.fdmu.RWUnlock(false) { - fd.destroy() - } -} - func (fd *netFD) Close() error { - if !fd.fdmu.IncrefAndClose() { + if !fd.fdmu.increfAndClose() { return errClosing } // Unblock any I/O. Once it all unblocks and returns, // so that it cannot be referring to fd.sysfd anymore, - // the final decref will close fd.sysfd. This should happen + // the final decref will close fd.sysfd. This should happen // fairly quickly, since all the I/O is non-blocking, and any // attempts to block in the pollDesc will return errClosing. - fd.pd.Evict() + fd.pd.evict() fd.decref() return nil } @@ -239,15 +228,26 @@ func (fd *netFD) Read(p []byte) (n int, err error) { return 0, err } defer fd.readUnlock() - if err := fd.pd.PrepareRead(); err != nil { + if len(p) == 0 { + // If the caller wanted a zero byte read, return immediately + // without trying. (But after acquiring the readLock.) Otherwise + // syscall.Read returns 0, nil and eofError turns that into + // io.EOF. + // TODO(bradfitz): make it wait for readability? (Issue 15735) + return 0, nil + } + if err := fd.pd.prepareRead(); err != nil { return 0, err } + if fd.isStream && len(p) > 1<<30 { + p = p[:1<<30] + } for { n, err = syscall.Read(fd.sysfd, p) if err != nil { n = 0 if err == syscall.EAGAIN { - if err = fd.pd.WaitRead(); err == nil { + if err = fd.pd.waitRead(); err == nil { continue } } @@ -266,7 +266,7 @@ func (fd *netFD) readFrom(p []byte) (n int, sa syscall.Sockaddr, err error) { return 0, nil, err } defer fd.readUnlock() - if err := fd.pd.PrepareRead(); err != nil { + if err := fd.pd.prepareRead(); err != nil { return 0, nil, err } for { @@ -274,7 +274,7 @@ func (fd *netFD) readFrom(p []byte) (n int, sa syscall.Sockaddr, err error) { if err != nil { n = 0 if err == syscall.EAGAIN { - if err = fd.pd.WaitRead(); err == nil { + if err = fd.pd.waitRead(); err == nil { continue } } @@ -293,7 +293,7 @@ func (fd *netFD) readMsg(p []byte, oob []byte) (n, oobn, flags int, sa syscall.S return 0, 0, 0, nil, err } defer fd.readUnlock() - if err := fd.pd.PrepareRead(); err != nil { + if err := fd.pd.prepareRead(); err != nil { return 0, 0, 0, nil, err } for { @@ -301,7 +301,7 @@ func (fd *netFD) readMsg(p []byte, oob []byte) (n, oobn, flags int, sa syscall.S if err != nil { // TODO(dfc) should n and oobn be set to 0 if err == syscall.EAGAIN { - if err = fd.pd.WaitRead(); err == nil { + if err = fd.pd.waitRead(); err == nil { continue } } @@ -320,12 +320,16 @@ func (fd *netFD) Write(p []byte) (nn int, err error) { return 0, err } defer fd.writeUnlock() - if err := fd.pd.PrepareWrite(); err != nil { + if err := fd.pd.prepareWrite(); err != nil { return 0, err } for { var n int - n, err = syscall.Write(fd.sysfd, p[nn:]) + max := len(p) + if fd.isStream && max-nn > 1<<30 { + max = nn + 1<<30 + } + n, err = syscall.Write(fd.sysfd, p[nn:max]) if n > 0 { nn += n } @@ -333,7 +337,7 @@ func (fd *netFD) Write(p []byte) (nn int, err error) { break } if err == syscall.EAGAIN { - if err = fd.pd.WaitWrite(); err == nil { + if err = fd.pd.waitWrite(); err == nil { continue } } @@ -356,13 +360,13 @@ func (fd *netFD) writeTo(p []byte, sa syscall.Sockaddr) (n int, err error) { return 0, err } defer fd.writeUnlock() - if err := fd.pd.PrepareWrite(); err != nil { + if err := fd.pd.prepareWrite(); err != nil { return 0, err } for { err = syscall.Sendto(fd.sysfd, p, 0, sa) if err == syscall.EAGAIN { - if err = fd.pd.WaitWrite(); err == nil { + if err = fd.pd.waitWrite(); err == nil { continue } } @@ -382,13 +386,13 @@ func (fd *netFD) writeMsg(p []byte, oob []byte, sa syscall.Sockaddr) (n int, oob return 0, 0, err } defer fd.writeUnlock() - if err := fd.pd.PrepareWrite(); err != nil { + if err := fd.pd.prepareWrite(); err != nil { return 0, 0, err } for { n, err = syscall.SendmsgN(fd.sysfd, p, oob, sa, 0) if err == syscall.EAGAIN { - if err = fd.pd.WaitWrite(); err == nil { + if err = fd.pd.waitWrite(); err == nil { continue } } @@ -411,7 +415,7 @@ func (fd *netFD) accept() (netfd *netFD, err error) { var s int var rsa syscall.Sockaddr - if err = fd.pd.PrepareRead(); err != nil { + if err = fd.pd.prepareRead(); err != nil { return nil, err } for { @@ -423,7 +427,7 @@ func (fd *netFD) accept() (netfd *netFD, err error) { } switch nerr.Err { case syscall.EAGAIN: - if err = fd.pd.WaitRead(); err == nil { + if err = fd.pd.waitRead(); err == nil { continue } case syscall.ECONNABORTED: @@ -472,7 +476,7 @@ func dupCloseOnExec(fd int) (newfd int, err error) { // and fcntl there falls back (undocumented) // to doing an ioctl instead, returning EBADF // in this case because fd is not of the - // expected device fd type. Treat it as + // expected device fd type. Treat it as // EINVAL instead, so we fall back to the // normal dup path. // TODO: only do this on 10.6 if we can detect 10.6 diff --git a/libgo/go/net/fd_windows.go b/libgo/go/net/fd_windows.go index fd50d772d6..a976f2ac7f 100644 --- a/libgo/go/net/fd_windows.go +++ b/libgo/go/net/fd_windows.go @@ -5,12 +5,12 @@ package net import ( + "context" "internal/race" "os" "runtime" "sync" "syscall" - "time" "unsafe" ) @@ -42,11 +42,6 @@ func sysInit() { initErr = os.NewSyscallError("wsastartup", e) } canCancelIO = syscall.LoadCancelIoEx() == nil - if syscall.LoadGetAddrInfo() == nil { - lookupPort = newLookupPort - lookupIP = newLookupIP - } - hasLoadSetFileCompletionNotificationModes = syscall.LoadSetFileCompletionNotificationModes() == nil if hasLoadSetFileCompletionNotificationModes { // It's not safe to use FILE_SKIP_COMPLETION_PORT_ON_SUCCESS if non IFS providers are installed: @@ -69,22 +64,15 @@ func sysInit() { } } +// canUseConnectEx reports whether we can use the ConnectEx Windows API call +// for the given network type. func canUseConnectEx(net string) bool { switch net { - case "udp", "udp4", "udp6", "ip", "ip4", "ip6": - // ConnectEx windows API does not support connectionless sockets. - return false - } - return syscall.LoadConnectEx() == nil -} - -func dial(net string, ra Addr, dialer func(time.Time) (Conn, error), deadline time.Time) (Conn, error) { - if !canUseConnectEx(net) { - // Use the relatively inefficient goroutine-racing - // implementation of DialTimeout. - return dialChannel(net, ra, dialer, deadline) + case "tcp", "tcp4", "tcp6": + return true } - return dialer(deadline) + // ConnectEx windows API does not support connectionless sockets. + return false } // operation contains superset of data necessary to perform all async IO. @@ -108,6 +96,7 @@ type operation struct { rsan int32 handle syscall.Handle flags uint32 + bufs []syscall.WSABuf } func (o *operation) InitBuf(buf []byte) { @@ -118,6 +107,30 @@ func (o *operation) InitBuf(buf []byte) { } } +func (o *operation) InitBufs(buf *Buffers) { + if o.bufs == nil { + o.bufs = make([]syscall.WSABuf, 0, len(*buf)) + } else { + o.bufs = o.bufs[:0] + } + for _, b := range *buf { + var p *byte + if len(b) > 0 { + p = &b[0] + } + o.bufs = append(o.bufs, syscall.WSABuf{Len: uint32(len(b)), Buf: p}) + } +} + +// ClearBufs clears all pointers to Buffers parameter captured +// by InitBufs, so it can be released by garbage collector. +func (o *operation) ClearBufs() { + for i := range o.bufs { + o.bufs[i].Buf = nil + } + o.bufs = o.bufs[:0] +} + // ioSrv executes net IO requests. type ioSrv struct { req chan ioSrvReq @@ -152,7 +165,7 @@ func (s *ioSrv) ProcessRemoteIO() { func (s *ioSrv) ExecIO(o *operation, name string, submit func(o *operation) error) (int, error) { fd := o.fd // Notify runtime netpoll about starting IO. - err := fd.pd.Prepare(int(o.mode)) + err := fd.pd.prepare(int(o.mode)) if err != nil { return 0, err } @@ -180,7 +193,7 @@ func (s *ioSrv) ExecIO(o *operation, name string, submit func(o *operation) erro return 0, err } // Wait for our request to complete. - err = fd.pd.Wait(int(o.mode)) + err = fd.pd.wait(int(o.mode)) if err == nil { // All is good. Extract our IO results and return. if o.errno != 0 { @@ -210,7 +223,7 @@ func (s *ioSrv) ExecIO(o *operation, name string, submit func(o *operation) erro <-o.errc } // Wait for cancelation to complete. - fd.pd.WaitCanceled(int(o.mode)) + fd.pd.waitCanceled(int(o.mode)) if o.errno != 0 { err = syscall.Errno(o.errno) if err == syscall.ERROR_OPERATION_ABORTED { // IO Canceled @@ -251,6 +264,7 @@ type netFD struct { sysfd syscall.Handle family int sotype int + isStream bool isConnected bool skipSyncNotif bool net string @@ -269,11 +283,11 @@ func newFD(sysfd syscall.Handle, family, sotype int, net string) (*netFD, error) return nil, initErr } onceStartServer.Do(startServer) - return &netFD{sysfd: sysfd, family: family, sotype: sotype, net: net}, nil + return &netFD{sysfd: sysfd, family: family, sotype: sotype, net: net, isStream: sotype == syscall.SOCK_STREAM}, nil } func (fd *netFD) init() error { - if err := fd.pd.Init(fd); err != nil { + if err := fd.pd.init(fd); err != nil { return err } if hasLoadSetFileCompletionNotificationModes { @@ -320,19 +334,20 @@ func (fd *netFD) setAddr(laddr, raddr Addr) { runtime.SetFinalizer(fd, (*netFD).Close) } -func (fd *netFD) connect(la, ra syscall.Sockaddr, deadline time.Time, cancel <-chan struct{}) error { +func (fd *netFD) connect(ctx context.Context, la, ra syscall.Sockaddr) error { // Do not need to call fd.writeLock here, // because fd is not yet accessible to user, // so no concurrent operations are possible. if err := fd.init(); err != nil { return err } - if !deadline.IsZero() { + if deadline, ok := ctx.Deadline(); ok && !deadline.IsZero() { fd.setWriteDeadline(deadline) defer fd.setWriteDeadline(noDeadline) } if !canUseConnectEx(fd.net) { - return os.NewSyscallError("connect", connectFunc(fd.sysfd, ra)) + err := connectFunc(fd.sysfd, ra) + return os.NewSyscallError("connect", err) } // ConnectEx windows API requires an unconnected, previously bound socket. if la == nil { @@ -351,26 +366,30 @@ func (fd *netFD) connect(la, ra syscall.Sockaddr, deadline time.Time, cancel <-c // Call ConnectEx API. o := &fd.wop o.sa = ra - if cancel != nil { - done := make(chan struct{}) - defer close(done) - go func() { - select { - case <-cancel: - // Force the runtime's poller to immediately give - // up waiting for writability. - fd.setWriteDeadline(aLongTimeAgo) - case <-done: - } - }() - } + + // Wait for the goroutine converting context.Done into a write timeout + // to exist, otherwise our caller might cancel the context and + // cause fd.setWriteDeadline(aLongTimeAgo) to cancel a successful dial. + done := make(chan bool) // must be unbuffered + defer func() { done <- true }() + go func() { + select { + case <-ctx.Done(): + // Force the runtime's poller to immediately give + // up waiting for writability. + fd.setWriteDeadline(aLongTimeAgo) + <-done + case <-done: + } + }() + _, err := wsrv.ExecIO(o, "ConnectEx", func(o *operation) error { return connectExFunc(o.fd.sysfd, o.sa, nil, 0, nil, &o.o) }) if err != nil { select { - case <-cancel: - return errCanceled + case <-ctx.Done(): + return mapErr(ctx.Err()) default: if _, ok := err.(syscall.Errno); ok { err = os.NewSyscallError("connectex", err) @@ -388,68 +407,19 @@ func (fd *netFD) destroy() { } // Poller may want to unregister fd in readiness notification mechanism, // so this must be executed before closeFunc. - fd.pd.Close() + fd.pd.close() closeFunc(fd.sysfd) fd.sysfd = syscall.InvalidHandle // no need for a finalizer anymore runtime.SetFinalizer(fd, nil) } -// Add a reference to this fd. -// Returns an error if the fd cannot be used. -func (fd *netFD) incref() error { - if !fd.fdmu.Incref() { - return errClosing - } - return nil -} - -// Remove a reference to this FD and close if we've been asked to do so -// (and there are no references left). -func (fd *netFD) decref() { - if fd.fdmu.Decref() { - fd.destroy() - } -} - -// Add a reference to this fd and lock for reading. -// Returns an error if the fd cannot be used. -func (fd *netFD) readLock() error { - if !fd.fdmu.RWLock(true) { - return errClosing - } - return nil -} - -// Unlock for reading and remove a reference to this FD. -func (fd *netFD) readUnlock() { - if fd.fdmu.RWUnlock(true) { - fd.destroy() - } -} - -// Add a reference to this fd and lock for writing. -// Returns an error if the fd cannot be used. -func (fd *netFD) writeLock() error { - if !fd.fdmu.RWLock(false) { - return errClosing - } - return nil -} - -// Unlock for writing and remove a reference to this FD. -func (fd *netFD) writeUnlock() { - if fd.fdmu.RWUnlock(false) { - fd.destroy() - } -} - func (fd *netFD) Close() error { - if !fd.fdmu.IncrefAndClose() { + if !fd.fdmu.increfAndClose() { return errClosing } // unblock pending reader and writer - fd.pd.Evict() + fd.pd.evict() fd.decref() return nil } @@ -483,7 +453,9 @@ func (fd *netFD) Read(buf []byte) (int, error) { if race.Enabled { race.Acquire(unsafe.Pointer(&ioSync)) } - err = fd.eofError(n, err) + if len(buf) != 0 { + err = fd.eofError(n, err) + } if _, ok := err.(syscall.Errno); ok { err = os.NewSyscallError("wsarecv", err) } @@ -537,6 +509,42 @@ func (fd *netFD) Write(buf []byte) (int, error) { return n, err } +func (c *conn) writeBuffers(v *Buffers) (int64, error) { + if !c.ok() { + return 0, syscall.EINVAL + } + n, err := c.fd.writeBuffers(v) + if err != nil { + return n, &OpError{Op: "WSASend", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} + } + return n, nil +} + +func (fd *netFD) writeBuffers(buf *Buffers) (int64, error) { + if len(*buf) == 0 { + return 0, nil + } + if err := fd.writeLock(); err != nil { + return 0, err + } + defer fd.writeUnlock() + if race.Enabled { + race.ReleaseMerge(unsafe.Pointer(&ioSync)) + } + o := &fd.wop + o.InitBufs(buf) + n, err := wsrv.ExecIO(o, "WSASend", func(o *operation) error { + return syscall.WSASend(o.fd.sysfd, &o.bufs[0], uint32(len(*buf)), &o.qty, 0, &o.o, nil) + }) + o.ClearBufs() + if _, ok := err.(syscall.Errno); ok { + err = os.NewSyscallError("wsasend", err) + } + testHookDidWritev(n) + buf.consume(int64(n)) + return int64(n), err +} + func (fd *netFD) writeTo(buf []byte, sa syscall.Sockaddr) (int, error) { if len(buf) == 0 { return 0, nil @@ -579,7 +587,7 @@ func (fd *netFD) acceptOne(rawsa []syscall.RawSockaddrAny, o *operation) (*netFD o.handle = s o.rsan = int32(unsafe.Sizeof(rawsa[0])) _, err = rsrv.ExecIO(o, "AcceptEx", func(o *operation) error { - return syscall.AcceptEx(o.fd.sysfd, o.handle, (*byte)(unsafe.Pointer(&rawsa[0])), 0, uint32(o.rsan), uint32(o.rsan), &o.qty, &o.o) + return acceptFunc(o.fd.sysfd, o.handle, (*byte)(unsafe.Pointer(&rawsa[0])), 0, uint32(o.rsan), uint32(o.rsan), &o.qty, &o.o) }) if err != nil { netfd.Close() @@ -595,7 +603,7 @@ func (fd *netFD) acceptOne(rawsa []syscall.RawSockaddrAny, o *operation) (*netFD netfd.Close() return nil, os.NewSyscallError("setsockopt", err) } - + runtime.KeepAlive(fd) return netfd, nil } diff --git a/libgo/go/net/file.go b/libgo/go/net/file.go index 1aad477400..07099851cb 100644 --- a/libgo/go/net/file.go +++ b/libgo/go/net/file.go @@ -6,6 +6,9 @@ package net import "os" +// BUG(mikio): On NaCl and Windows, the FileConn, FileListener and +// FilePacketConn functions are not implemented. + type fileAddr string func (fileAddr) Network() string { return "file+net" } diff --git a/libgo/go/net/file_plan9.go b/libgo/go/net/file_plan9.go index 892775a024..d16e5a166c 100644 --- a/libgo/go/net/file_plan9.go +++ b/libgo/go/net/file_plan9.go @@ -50,9 +50,7 @@ func newFileFD(f *os.File) (net *netFD, err error) { name := comp[2] switch file := comp[n-1]; file { case "ctl", "clone": - syscall.ForkLock.RLock() fd, err := syscall.Dup(int(f.Fd()), -1) - syscall.ForkLock.RUnlock() if err != nil { return nil, os.NewSyscallError("dup", err) } @@ -60,7 +58,7 @@ func newFileFD(f *os.File) (net *netFD, err error) { dir := netdir + "/" + comp[n-2] ctl = os.NewFile(uintptr(fd), dir+"/"+file) - ctl.Seek(0, 0) + ctl.Seek(0, io.SeekStart) var buf [16]byte n, err := ctl.Read(buf[:]) if err != nil { @@ -83,7 +81,7 @@ func newFileFD(f *os.File) (net *netFD, err error) { if err != nil { return nil, err } - return newFD(comp[1], name, ctl, nil, laddr, nil) + return newFD(comp[1], name, nil, ctl, nil, laddr, nil) } func fileConn(f *os.File) (Conn, error) { diff --git a/libgo/go/net/hook.go b/libgo/go/net/hook.go index 9ab34c0e36..d7316ea438 100644 --- a/libgo/go/net/hook.go +++ b/libgo/go/net/hook.go @@ -4,9 +4,19 @@ package net +import "context" + var ( - testHookDialTCP = dialTCP - testHookHostsPath = "/etc/hosts" - testHookLookupIP = func(fn func(string) ([]IPAddr, error), host string) ([]IPAddr, error) { return fn(host) } + // if non-nil, overrides dialTCP. + testHookDialTCP func(ctx context.Context, net string, laddr, raddr *TCPAddr) (*TCPConn, error) + + testHookHostsPath = "/etc/hosts" + testHookLookupIP = func( + ctx context.Context, + fn func(context.Context, string) ([]IPAddr, error), + host string, + ) ([]IPAddr, error) { + return fn(ctx, host) + } testHookSetKeepAlive = func() {} ) diff --git a/libgo/go/net/hook_unix.go b/libgo/go/net/hook_unix.go index 361ca5980c..cf52567fcf 100644 --- a/libgo/go/net/hook_unix.go +++ b/libgo/go/net/hook_unix.go @@ -9,7 +9,8 @@ package net import "syscall" var ( - testHookDialChannel = func() {} // see golang.org/issue/5349 + testHookDialChannel = func() {} // for golang.org/issue/5349 + testHookCanceledDial = func() {} // for golang.org/issue/16523 // Placeholders for socket system calls. socketFunc func(int, int, int) (int, error) = syscall.Socket diff --git a/libgo/go/net/hook_windows.go b/libgo/go/net/hook_windows.go index 126b0ebdd1..63ea35ab8c 100644 --- a/libgo/go/net/hook_windows.go +++ b/libgo/go/net/hook_windows.go @@ -13,9 +13,10 @@ var ( testHookDialChannel = func() { time.Sleep(time.Millisecond) } // see golang.org/issue/5349 // Placeholders for socket system calls. - socketFunc func(int, int, int) (syscall.Handle, error) = syscall.Socket - closeFunc func(syscall.Handle) error = syscall.Closesocket - connectFunc func(syscall.Handle, syscall.Sockaddr) error = syscall.Connect - connectExFunc func(syscall.Handle, syscall.Sockaddr, *byte, uint32, *uint32, *syscall.Overlapped) error = syscall.ConnectEx - listenFunc func(syscall.Handle, int) error = syscall.Listen + socketFunc func(int, int, int) (syscall.Handle, error) = syscall.Socket + closeFunc func(syscall.Handle) error = syscall.Closesocket + connectFunc func(syscall.Handle, syscall.Sockaddr) error = syscall.Connect + connectExFunc func(syscall.Handle, syscall.Sockaddr, *byte, uint32, *uint32, *syscall.Overlapped) error = syscall.ConnectEx + listenFunc func(syscall.Handle, int) error = syscall.Listen + acceptFunc func(syscall.Handle, syscall.Handle, *byte, uint32, uint32, uint32, *uint32, *syscall.Overlapped) error = syscall.AcceptEx ) diff --git a/libgo/go/net/hosts.go b/libgo/go/net/hosts.go index c4de1b6a97..9c101c6ef5 100644 --- a/libgo/go/net/hosts.go +++ b/libgo/go/net/hosts.go @@ -110,7 +110,9 @@ func lookupStaticHost(host string) []string { lowerHost := []byte(host) lowerASCIIBytes(lowerHost) if ips, ok := hosts.byName[absDomainName(lowerHost)]; ok { - return ips + ipsCp := make([]string, len(ips)) + copy(ipsCp, ips) + return ipsCp } } return nil @@ -127,7 +129,9 @@ func lookupStaticAddr(addr string) []string { } if len(hosts.byAddr) != 0 { if hosts, ok := hosts.byAddr[addr]; ok { - return hosts + hostsCp := make([]string, len(hosts)) + copy(hostsCp, hosts) + return hostsCp } } return nil diff --git a/libgo/go/net/hosts_test.go b/libgo/go/net/hosts_test.go index 4c67bfa982..5d6c9cfe19 100644 --- a/libgo/go/net/hosts_test.go +++ b/libgo/go/net/hosts_test.go @@ -64,13 +64,17 @@ func TestLookupStaticHost(t *testing.T) { for _, tt := range lookupStaticHostTests { testHookHostsPath = tt.name for _, ent := range tt.ents { - ins := []string{ent.in, absDomainName([]byte(ent.in)), strings.ToLower(ent.in), strings.ToUpper(ent.in)} - for _, in := range ins { - addrs := lookupStaticHost(in) - if !reflect.DeepEqual(addrs, ent.out) { - t.Errorf("%s, lookupStaticHost(%s) = %v; want %v", tt.name, in, addrs, ent.out) - } - } + testStaticHost(t, tt.name, ent) + } + } +} + +func testStaticHost(t *testing.T, hostsPath string, ent staticHostEntry) { + ins := []string{ent.in, absDomainName([]byte(ent.in)), strings.ToLower(ent.in), strings.ToUpper(ent.in)} + for _, in := range ins { + addrs := lookupStaticHost(in) + if !reflect.DeepEqual(addrs, ent.out) { + t.Errorf("%s, lookupStaticHost(%s) = %v; want %v", hostsPath, in, addrs, ent.out) } } } @@ -129,13 +133,43 @@ func TestLookupStaticAddr(t *testing.T) { for _, tt := range lookupStaticAddrTests { testHookHostsPath = tt.name for _, ent := range tt.ents { - hosts := lookupStaticAddr(ent.in) - for i := range ent.out { - ent.out[i] = absDomainName([]byte(ent.out[i])) - } - if !reflect.DeepEqual(hosts, ent.out) { - t.Errorf("%s, lookupStaticAddr(%s) = %v; want %v", tt.name, ent.in, hosts, ent.out) - } + testStaticAddr(t, tt.name, ent) } } } + +func testStaticAddr(t *testing.T, hostsPath string, ent staticHostEntry) { + hosts := lookupStaticAddr(ent.in) + for i := range ent.out { + ent.out[i] = absDomainName([]byte(ent.out[i])) + } + if !reflect.DeepEqual(hosts, ent.out) { + t.Errorf("%s, lookupStaticAddr(%s) = %v; want %v", hostsPath, ent.in, hosts, ent.out) + } +} + +func TestHostCacheModification(t *testing.T) { + // Ensure that programs can't modify the internals of the host cache. + // See https://github.com/golang/go/issues/14212. + defer func(orig string) { testHookHostsPath = orig }(testHookHostsPath) + + testHookHostsPath = "testdata/ipv4-hosts" + ent := staticHostEntry{"localhost", []string{"127.0.0.1", "127.0.0.2", "127.0.0.3"}} + testStaticHost(t, testHookHostsPath, ent) + // Modify the addresses return by lookupStaticHost. + addrs := lookupStaticHost(ent.in) + for i := range addrs { + addrs[i] += "junk" + } + testStaticHost(t, testHookHostsPath, ent) + + testHookHostsPath = "testdata/ipv6-hosts" + ent = staticHostEntry{"::1", []string{"localhost"}} + testStaticAddr(t, testHookHostsPath, ent) + // Modify the hosts return by lookupStaticAddr. + hosts := lookupStaticAddr(ent.in) + for i := range hosts { + hosts[i] += "junk" + } + testStaticAddr(t, testHookHostsPath, ent) +} diff --git a/libgo/go/net/http/cgi/host.go b/libgo/go/net/http/cgi/host.go index 9b4d875418..58e9f7132a 100644 --- a/libgo/go/net/http/cgi/host.go +++ b/libgo/go/net/http/cgi/host.go @@ -10,7 +10,7 @@ // // Note that using CGI means starting a new process to handle each // request, which is typically less efficient than using a -// long-running server. This package is intended primarily for +// long-running server. This package is intended primarily for // compatibility with existing systems. package cgi @@ -58,6 +58,7 @@ type Handler struct { InheritEnv []string // environment variables to inherit from host, as "key" Logger *log.Logger // optional log for errors or nil to use log.Print Args []string // optional arguments to pass to child process + Stderr io.Writer // optional stderr for the child process; nil means os.Stderr // PathLocationHandler specifies the root http Handler that // should handle internal redirects when the CGI process @@ -70,6 +71,13 @@ type Handler struct { PathLocationHandler http.Handler } +func (h *Handler) stderr() io.Writer { + if h.Stderr != nil { + return h.Stderr + } + return os.Stderr +} + // removeLeadingDuplicates remove leading duplicate in environments. // It's possible to override environment like following. // cgi.Handler{ @@ -145,6 +153,10 @@ func (h *Handler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { for k, v := range req.Header { k = strings.Map(upperCaseAndUnderscore, k) + if k == "PROXY" { + // See Issue 16405 + continue + } joinStr := ", " if k == "COOKIE" { joinStr = "; " @@ -204,7 +216,7 @@ func (h *Handler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { Args: append([]string{h.Path}, h.Args...), Dir: cwd, Env: env, - Stderr: os.Stderr, // for now + Stderr: h.stderr(), } if req.ContentLength != 0 { cmd.Stdin = req.Body diff --git a/libgo/go/net/http/cgi/host_test.go b/libgo/go/net/http/cgi/host_test.go index fb7d66adb9..f0583729eb 100644 --- a/libgo/go/net/http/cgi/host_test.go +++ b/libgo/go/net/http/cgi/host_test.go @@ -8,6 +8,7 @@ package cgi import ( "bufio" + "bytes" "fmt" "io" "net" @@ -34,15 +35,18 @@ func newRequest(httpreq string) *http.Request { return req } -func runCgiTest(t *testing.T, h *Handler, httpreq string, expectedMap map[string]string) *httptest.ResponseRecorder { +func runCgiTest(t *testing.T, h *Handler, + httpreq string, + expectedMap map[string]string, checks ...func(reqInfo map[string]string)) *httptest.ResponseRecorder { rw := httptest.NewRecorder() req := newRequest(httpreq) h.ServeHTTP(rw, req) - runResponseChecks(t, rw, expectedMap) + runResponseChecks(t, rw, expectedMap, checks...) return rw } -func runResponseChecks(t *testing.T, rw *httptest.ResponseRecorder, expectedMap map[string]string) { +func runResponseChecks(t *testing.T, rw *httptest.ResponseRecorder, + expectedMap map[string]string, checks ...func(reqInfo map[string]string)) { // Make a map to hold the test map that the CGI returns. m := make(map[string]string) m["_body"] = rw.Body.String() @@ -80,6 +84,9 @@ readlines: t.Errorf("for key %q got %q; expected %q", key, got, expected) } } + for _, check := range checks { + check(m) + } } var cgiTested, cgiWorks bool @@ -235,6 +242,31 @@ func TestDupHeaders(t *testing.T) { expectedMap) } +// Issue 16405: CGI+http.Transport differing uses of HTTP_PROXY. +// Verify we don't set the HTTP_PROXY environment variable. +// Hope nobody was depending on it. It's not a known header, though. +func TestDropProxyHeader(t *testing.T) { + check(t) + h := &Handler{ + Path: "testdata/test.cgi", + } + expectedMap := map[string]string{ + "env-REQUEST_URI": "/myscript/bar?a=b", + "env-SCRIPT_FILENAME": "testdata/test.cgi", + "env-HTTP_X_FOO": "a", + } + runCgiTest(t, h, "GET /myscript/bar?a=b HTTP/1.0\n"+ + "X-Foo: a\n"+ + "Proxy: should_be_stripped\n"+ + "Host: example.com\n\n", + expectedMap, + func(reqInfo map[string]string) { + if v, ok := reqInfo["env-HTTP_PROXY"]; ok { + t.Errorf("HTTP_PROXY = %q; should be absent", v) + } + }) +} + func TestPathInfoNoRoot(t *testing.T) { check(t) h := &Handler{ @@ -501,6 +533,23 @@ func TestEnvOverride(t *testing.T) { runCgiTest(t, h, "GET /test.cgi HTTP/1.0\nHost: example.com\n\n", expectedMap) } +func TestHandlerStderr(t *testing.T) { + check(t) + var stderr bytes.Buffer + h := &Handler{ + Path: "testdata/test.cgi", + Root: "/test.cgi", + Stderr: &stderr, + } + + rw := httptest.NewRecorder() + req := newRequest("GET /test.cgi?writestderr=1 HTTP/1.0\nHost: example.com\n\n") + h.ServeHTTP(rw, req) + if got, want := stderr.String(), "Hello, stderr!\n"; got != want { + t.Errorf("Stderr = %q; want %q", got, want) + } +} + func TestRemoveLeadingDuplicates(t *testing.T) { tests := []struct { env []string diff --git a/libgo/go/net/http/cgi/testdata/test.cgi b/libgo/go/net/http/cgi/testdata/test.cgi index ec7ee6f386..667fce217e 100644 --- a/libgo/go/net/http/cgi/testdata/test.cgi +++ b/libgo/go/net/http/cgi/testdata/test.cgi @@ -23,6 +23,10 @@ print "X-CGI-Pid: $$\r\n"; print "X-Test-Header: X-Test-Value\r\n"; print "\r\n"; +if ($params->{"writestderr"}) { + print STDERR "Hello, stderr!\n"; +} + if ($params->{"bigresponse"}) { # 17 MB, for OS X: golang.org/issue/4958 for (1..(17 * 1024)) { diff --git a/libgo/go/net/http/client.go b/libgo/go/net/http/client.go index 3106d229da..0005538e70 100644 --- a/libgo/go/net/http/client.go +++ b/libgo/go/net/http/client.go @@ -18,6 +18,7 @@ import ( "io/ioutil" "log" "net/url" + "sort" "strings" "sync" "time" @@ -33,6 +34,25 @@ import ( // A Client is higher-level than a RoundTripper (such as Transport) // and additionally handles HTTP details such as cookies and // redirects. +// +// When following redirects, the Client will forward all headers set on the +// initial Request except: +// +// * when forwarding sensitive headers like "Authorization", +// "WWW-Authenticate", and "Cookie" to untrusted targets. +// These headers will be ignored when following a redirect to a domain +// that is not a subdomain match or exact match of the initial domain. +// For example, a redirect from "foo.com" to either "foo.com" or "sub.foo.com" +// will forward the sensitive headers, but a redirect to "bar.com" will not. +// +// * when forwarding the "Cookie" header with a non-nil cookie Jar. +// Since each redirect may mutate the state of the cookie jar, +// a redirect may possibly alter a cookie set in the initial request. +// When forwarding the "Cookie" header, any mutated cookies will be omitted, +// with the expectation that the Jar will insert those mutated cookies +// with the updated values (assuming the origin matches). +// If Jar is nil, the initial cookies are forwarded without change. +// type Client struct { // Transport specifies the mechanism by which individual // HTTP requests are made. @@ -44,17 +64,26 @@ type Client struct { // following an HTTP redirect. The arguments req and via are // the upcoming request and the requests made already, oldest // first. If CheckRedirect returns an error, the Client's Get - // method returns both the previous Response and - // CheckRedirect's error (wrapped in a url.Error) instead of - // issuing the Request req. + // method returns both the previous Response (with its Body + // closed) and CheckRedirect's error (wrapped in a url.Error) + // instead of issuing the Request req. + // As a special case, if CheckRedirect returns ErrUseLastResponse, + // then the most recent response is returned with its body + // unclosed, along with a nil error. // // If CheckRedirect is nil, the Client uses its default policy, // which is to stop after 10 consecutive requests. CheckRedirect func(req *Request, via []*Request) error // Jar specifies the cookie jar. - // If Jar is nil, cookies are not sent in requests and ignored - // in responses. + // + // The Jar is used to insert relevant cookies into every + // outbound Request and is updated with the cookie values + // of every inbound Response. The Jar is consulted for every + // redirect that the Client follows. + // + // If Jar is nil, cookies are only sent if they are explicitly + // set on the Request. Jar CookieJar // Timeout specifies a time limit for requests made by this @@ -110,10 +139,6 @@ type RoundTripper interface { RoundTrip(*Request) (*Response, error) } -// Given a string of the form "host", "host:port", or "[ipv6::address]:port", -// return true if the string includes a port. -func hasPort(s string) bool { return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") } - // refererForURL returns a referer without any authentication info or // an empty string if lastReq scheme is https and newReq scheme is http. func refererForURL(lastReq, newReq *url.URL) string { @@ -138,59 +163,23 @@ func refererForURL(lastReq, newReq *url.URL) string { return referer } -// Used in Send to implement io.ReadCloser by bundling together the -// bufio.Reader through which we read the response, and the underlying -// network connection. -type readClose struct { - io.Reader - io.Closer -} - -func (c *Client) send(req *Request, deadline time.Time) (*Response, error) { +// didTimeout is non-nil only if err != nil. +func (c *Client) send(req *Request, deadline time.Time) (resp *Response, didTimeout func() bool, err error) { if c.Jar != nil { for _, cookie := range c.Jar.Cookies(req.URL) { req.AddCookie(cookie) } } - resp, err := send(req, c.transport(), deadline) + resp, didTimeout, err = send(req, c.transport(), deadline) if err != nil { - return nil, err + return nil, didTimeout, err } if c.Jar != nil { if rc := resp.Cookies(); len(rc) > 0 { c.Jar.SetCookies(req.URL, rc) } } - return resp, err -} - -// Do sends an HTTP request and returns an HTTP response, following -// policy (e.g. redirects, cookies, auth) as configured on the client. -// -// An error is returned if caused by client policy (such as -// CheckRedirect), or if there was an HTTP protocol error. -// A non-2xx response doesn't cause an error. -// -// When err is nil, resp always contains a non-nil resp.Body. -// -// Callers should close resp.Body when done reading from it. If -// resp.Body is not closed, the Client's underlying RoundTripper -// (typically Transport) may not be able to re-use a persistent TCP -// connection to the server for a subsequent "keep-alive" request. -// -// The request Body, if non-nil, will be closed by the underlying -// Transport, even on errors. -// -// Generally Get, Post, or PostForm will be used instead of Do. -func (c *Client) Do(req *Request) (resp *Response, err error) { - method := valueOrDefault(req.Method, "GET") - if method == "GET" || method == "HEAD" { - return c.doFollowingRedirects(req, shouldRedirectGet) - } - if method == "POST" || method == "PUT" { - return c.doFollowingRedirects(req, shouldRedirectPost) - } - return c.send(req, c.deadline()) + return resp, nil, nil } func (c *Client) deadline() time.Time { @@ -209,22 +198,22 @@ func (c *Client) transport() RoundTripper { // send issues an HTTP request. // Caller should close resp.Body when done reading from it. -func send(ireq *Request, rt RoundTripper, deadline time.Time) (*Response, error) { +func send(ireq *Request, rt RoundTripper, deadline time.Time) (resp *Response, didTimeout func() bool, err error) { req := ireq // req is either the original request, or a modified fork if rt == nil { req.closeBody() - return nil, errors.New("http: no Client.Transport or DefaultTransport") + return nil, alwaysFalse, errors.New("http: no Client.Transport or DefaultTransport") } if req.URL == nil { req.closeBody() - return nil, errors.New("http: nil Request.URL") + return nil, alwaysFalse, errors.New("http: nil Request.URL") } if req.RequestURI != "" { req.closeBody() - return nil, errors.New("http: Request.RequestURI can't be set in client requests.") + return nil, alwaysFalse, errors.New("http: Request.RequestURI can't be set in client requests.") } // forkReq forks req into a shallow clone of ireq the first @@ -237,7 +226,7 @@ func send(ireq *Request, rt RoundTripper, deadline time.Time) (*Response, error) } // Most the callers of send (Get, Post, et al) don't need - // Headers, leaving it uninitialized. We guarantee to the + // Headers, leaving it uninitialized. We guarantee to the // Transport that this has been initialized, though. if req.Header == nil { forkReq() @@ -255,9 +244,9 @@ func send(ireq *Request, rt RoundTripper, deadline time.Time) (*Response, error) if !deadline.IsZero() { forkReq() } - stopTimer, wasCanceled := setRequestCancel(req, rt, deadline) + stopTimer, didTimeout := setRequestCancel(req, rt, deadline) - resp, err := rt.RoundTrip(req) + resp, err = rt.RoundTrip(req) if err != nil { stopTimer() if resp != nil { @@ -271,22 +260,27 @@ func send(ireq *Request, rt RoundTripper, deadline time.Time) (*Response, error) err = errors.New("http: server gave HTTP response to HTTPS client") } } - return nil, err + return nil, didTimeout, err } if !deadline.IsZero() { resp.Body = &cancelTimerBody{ - stop: stopTimer, - rc: resp.Body, - reqWasCanceled: wasCanceled, + stop: stopTimer, + rc: resp.Body, + reqDidTimeout: didTimeout, } } - return resp, nil + return resp, nil, nil } // setRequestCancel sets the Cancel field of req, if deadline is // non-zero. The RoundTripper's type is used to determine whether the legacy // CancelRequest behavior should be used. -func setRequestCancel(req *Request, rt RoundTripper, deadline time.Time) (stopTimer func(), wasCanceled func() bool) { +// +// As background, there are three ways to cancel a request: +// First was Transport.CancelRequest. (deprecated) +// Second was Request.Cancel (this mechanism). +// Third was Request.Context. +func setRequestCancel(req *Request, rt RoundTripper, deadline time.Time) (stopTimer func(), didTimeout func() bool) { if deadline.IsZero() { return nop, alwaysFalse } @@ -296,17 +290,8 @@ func setRequestCancel(req *Request, rt RoundTripper, deadline time.Time) (stopTi cancel := make(chan struct{}) req.Cancel = cancel - wasCanceled = func() bool { - select { - case <-cancel: - return true - default: - return false - } - } - doCancel := func() { - // The new way: + // The newer way (the second way in the func comment): close(cancel) // The legacy compatibility way, used only @@ -328,19 +313,23 @@ func setRequestCancel(req *Request, rt RoundTripper, deadline time.Time) (stopTi var once sync.Once stopTimer = func() { once.Do(func() { close(stopTimerCh) }) } - timer := time.NewTimer(deadline.Sub(time.Now())) + timer := time.NewTimer(time.Until(deadline)) + var timedOut atomicBool + go func() { select { case <-initialReqCancel: doCancel() + timer.Stop() case <-timer.C: + timedOut.setTrue() doCancel() case <-stopTimerCh: timer.Stop() } }() - return stopTimer, wasCanceled + return stopTimer, timedOut.isSet } // See 2 (end of page 4) http://www.ietf.org/rfc/rfc2617.txt @@ -353,26 +342,6 @@ func basicAuth(username, password string) string { return base64.StdEncoding.EncodeToString([]byte(auth)) } -// True if the specified HTTP status code is one for which the Get utility should -// automatically redirect. -func shouldRedirectGet(statusCode int) bool { - switch statusCode { - case StatusMovedPermanently, StatusFound, StatusSeeOther, StatusTemporaryRedirect: - return true - } - return false -} - -// True if the specified HTTP status code is one for which the Post utility should -// automatically redirect. -func shouldRedirectPost(statusCode int) bool { - switch statusCode { - case StatusFound, StatusSeeOther: - return true - } - return false -} - // Get issues a GET to the specified URL. If the response is one of // the following redirect codes, Get follows the redirect, up to a // maximum of 10 redirects: @@ -381,6 +350,7 @@ func shouldRedirectPost(statusCode int) bool { // 302 (Found) // 303 (See Other) // 307 (Temporary Redirect) +// 308 (Permanent Redirect) // // An error is returned if there were too many redirects or if there // was an HTTP protocol error. A non-2xx response doesn't cause an @@ -405,6 +375,7 @@ func Get(url string) (resp *Response, err error) { // 302 (Found) // 303 (See Other) // 307 (Temporary Redirect) +// 308 (Permanent Redirect) // // An error is returned if the Client's CheckRedirect function fails // or if there was an HTTP protocol error. A non-2xx response doesn't @@ -419,106 +390,290 @@ func (c *Client) Get(url string) (resp *Response, err error) { if err != nil { return nil, err } - return c.doFollowingRedirects(req, shouldRedirectGet) + return c.Do(req) } func alwaysFalse() bool { return false } -func (c *Client) doFollowingRedirects(ireq *Request, shouldRedirect func(int) bool) (resp *Response, err error) { - var base *url.URL - redirectChecker := c.CheckRedirect - if redirectChecker == nil { - redirectChecker = defaultCheckRedirect +// ErrUseLastResponse can be returned by Client.CheckRedirect hooks to +// control how redirects are processed. If returned, the next request +// is not sent and the most recent response is returned with its body +// unclosed. +var ErrUseLastResponse = errors.New("net/http: use last response") + +// checkRedirect calls either the user's configured CheckRedirect +// function, or the default. +func (c *Client) checkRedirect(req *Request, via []*Request) error { + fn := c.CheckRedirect + if fn == nil { + fn = defaultCheckRedirect } - var via []*Request + return fn(req, via) +} - if ireq.URL == nil { - ireq.closeBody() +// redirectBehavior describes what should happen when the +// client encounters a 3xx status code from the server +func redirectBehavior(reqMethod string, resp *Response, ireq *Request) (redirectMethod string, shouldRedirect, includeBody bool) { + switch resp.StatusCode { + case 301, 302, 303: + redirectMethod = reqMethod + shouldRedirect = true + includeBody = false + + // RFC 2616 allowed automatic redirection only with GET and + // HEAD requests. RFC 7231 lifts this restriction, but we still + // restrict other methods to GET to maintain compatibility. + // See Issue 18570. + if reqMethod != "GET" && reqMethod != "HEAD" { + redirectMethod = "GET" + } + case 307, 308: + redirectMethod = reqMethod + shouldRedirect = true + includeBody = true + + // Treat 307 and 308 specially, since they're new in + // Go 1.8, and they also require re-sending the request body. + if resp.Header.Get("Location") == "" { + // 308s have been observed in the wild being served + // without Location headers. Since Go 1.7 and earlier + // didn't follow these codes, just stop here instead + // of returning an error. + // See Issue 17773. + shouldRedirect = false + break + } + if ireq.GetBody == nil && ireq.outgoingLength() != 0 { + // We had a request body, and 307/308 require + // re-sending it, but GetBody is not defined. So just + // return this response to the user instead of an + // error, like we did in Go 1.7 and earlier. + shouldRedirect = false + } + } + return redirectMethod, shouldRedirect, includeBody +} + +// Do sends an HTTP request and returns an HTTP response, following +// policy (such as redirects, cookies, auth) as configured on the +// client. +// +// An error is returned if caused by client policy (such as +// CheckRedirect), or failure to speak HTTP (such as a network +// connectivity problem). A non-2xx status code doesn't cause an +// error. +// +// If the returned error is nil, the Response will contain a non-nil +// Body which the user is expected to close. If the Body is not +// closed, the Client's underlying RoundTripper (typically Transport) +// may not be able to re-use a persistent TCP connection to the server +// for a subsequent "keep-alive" request. +// +// The request Body, if non-nil, will be closed by the underlying +// Transport, even on errors. +// +// On error, any Response can be ignored. A non-nil Response with a +// non-nil error only occurs when CheckRedirect fails, and even then +// the returned Response.Body is already closed. +// +// Generally Get, Post, or PostForm will be used instead of Do. +// +// If the server replies with a redirect, the Client first uses the +// CheckRedirect function to determine whether the redirect should be +// followed. If permitted, a 301, 302, or 303 redirect causes +// subsequent requests to use HTTP method GET +// (or HEAD if the original request was HEAD), with no body. +// A 307 or 308 redirect preserves the original HTTP method and body, +// provided that the Request.GetBody function is defined. +// The NewRequest function automatically sets GetBody for common +// standard library body types. +func (c *Client) Do(req *Request) (*Response, error) { + if req.URL == nil { + req.closeBody() return nil, errors.New("http: nil Request.URL") } - req := ireq - deadline := c.deadline() + var ( + deadline = c.deadline() + reqs []*Request + resp *Response + copyHeaders = c.makeHeadersCopier(req) - urlStr := "" // next relative or absolute URL to fetch (after first request) - redirectFailed := false - for redirect := 0; ; redirect++ { - if redirect != 0 { - nreq := new(Request) - nreq.Cancel = ireq.Cancel - nreq.Method = ireq.Method - if ireq.Method == "POST" || ireq.Method == "PUT" { - nreq.Method = "GET" + // Redirect behavior: + redirectMethod string + includeBody bool + ) + uerr := func(err error) error { + req.closeBody() + method := valueOrDefault(reqs[0].Method, "GET") + var urlStr string + if resp != nil && resp.Request != nil { + urlStr = resp.Request.URL.String() + } else { + urlStr = req.URL.String() + } + return &url.Error{ + Op: method[:1] + strings.ToLower(method[1:]), + URL: urlStr, + Err: err, + } + } + for { + // For all but the first request, create the next + // request hop and replace req. + if len(reqs) > 0 { + loc := resp.Header.Get("Location") + if loc == "" { + return nil, uerr(fmt.Errorf("%d response missing Location header", resp.StatusCode)) } - nreq.Header = make(Header) - nreq.URL, err = base.Parse(urlStr) + u, err := req.URL.Parse(loc) if err != nil { - break + return nil, uerr(fmt.Errorf("failed to parse Location header %q: %v", loc, err)) } - if len(via) > 0 { - // Add the Referer header. - lastReq := via[len(via)-1] - if ref := refererForURL(lastReq.URL, nreq.URL); ref != "" { - nreq.Header.Set("Referer", ref) - } - - err = redirectChecker(nreq, via) + ireq := reqs[0] + req = &Request{ + Method: redirectMethod, + Response: resp, + URL: u, + Header: make(Header), + Cancel: ireq.Cancel, + ctx: ireq.ctx, + } + if includeBody && ireq.GetBody != nil { + req.Body, err = ireq.GetBody() if err != nil { - redirectFailed = true - break + return nil, uerr(err) } + req.ContentLength = ireq.ContentLength } - req = nreq - } - urlStr = req.URL.String() - if resp, err = c.send(req, deadline); err != nil { - if !deadline.IsZero() && !time.Now().Before(deadline) { - err = &httpError{ - err: err.Error() + " (Client.Timeout exceeded while awaiting headers)", - timeout: true, - } + // Copy original headers before setting the Referer, + // in case the user set Referer on their first request. + // If they really want to override, they can do it in + // their CheckRedirect func. + copyHeaders(req) + + // Add the Referer header from the most recent + // request URL to the new one, if it's not https->http: + if ref := refererForURL(reqs[len(reqs)-1].URL, req.URL); ref != "" { + req.Header.Set("Referer", ref) + } + err = c.checkRedirect(req, reqs) + + // Sentinel error to let users select the + // previous response, without closing its + // body. See Issue 10069. + if err == ErrUseLastResponse { + return resp, nil } - break - } - if shouldRedirect(resp.StatusCode) { - // Read the body if small so underlying TCP connection will be re-used. - // No need to check for errors: if it fails, Transport won't reuse it anyway. + // Close the previous response's body. But + // read at least some of the body so if it's + // small the underlying TCP connection will be + // re-used. No need to check for errors: if it + // fails, the Transport won't reuse it anyway. const maxBodySlurpSize = 2 << 10 if resp.ContentLength == -1 || resp.ContentLength <= maxBodySlurpSize { io.CopyN(ioutil.Discard, resp.Body, maxBodySlurpSize) } resp.Body.Close() - if urlStr = resp.Header.Get("Location"); urlStr == "" { - err = fmt.Errorf("%d response missing Location header", resp.StatusCode) - break + + if err != nil { + // Special case for Go 1 compatibility: return both the response + // and an error if the CheckRedirect function failed. + // See https://golang.org/issue/3795 + // The resp.Body has already been closed. + ue := uerr(err) + ue.(*url.Error).URL = loc + return resp, ue } - base = req.URL - via = append(via, req) - continue } - return resp, nil - } - method := valueOrDefault(ireq.Method, "GET") - urlErr := &url.Error{ - Op: method[:1] + strings.ToLower(method[1:]), - URL: urlStr, - Err: err, + reqs = append(reqs, req) + var err error + var didTimeout func() bool + if resp, didTimeout, err = c.send(req, deadline); err != nil { + if !deadline.IsZero() && didTimeout() { + err = &httpError{ + err: err.Error() + " (Client.Timeout exceeded while awaiting headers)", + timeout: true, + } + } + return nil, uerr(err) + } + + var shouldRedirect bool + redirectMethod, shouldRedirect, includeBody = redirectBehavior(req.Method, resp, reqs[0]) + if !shouldRedirect { + return resp, nil + } + + req.closeBody() } +} - if redirectFailed { - // Special case for Go 1 compatibility: return both the response - // and an error if the CheckRedirect function failed. - // See https://golang.org/issue/3795 - return resp, urlErr +// makeHeadersCopier makes a function that copies headers from the +// initial Request, ireq. For every redirect, this function must be called +// so that it can copy headers into the upcoming Request. +func (c *Client) makeHeadersCopier(ireq *Request) func(*Request) { + // The headers to copy are from the very initial request. + // We use a closured callback to keep a reference to these original headers. + var ( + ireqhdr = ireq.Header.clone() + icookies map[string][]*Cookie + ) + if c.Jar != nil && ireq.Header.Get("Cookie") != "" { + icookies = make(map[string][]*Cookie) + for _, c := range ireq.Cookies() { + icookies[c.Name] = append(icookies[c.Name], c) + } } - if resp != nil { - resp.Body.Close() + preq := ireq // The previous request + return func(req *Request) { + // If Jar is present and there was some initial cookies provided + // via the request header, then we may need to alter the initial + // cookies as we follow redirects since each redirect may end up + // modifying a pre-existing cookie. + // + // Since cookies already set in the request header do not contain + // information about the original domain and path, the logic below + // assumes any new set cookies override the original cookie + // regardless of domain or path. + // + // See https://golang.org/issue/17494 + if c.Jar != nil && icookies != nil { + var changed bool + resp := req.Response // The response that caused the upcoming redirect + for _, c := range resp.Cookies() { + if _, ok := icookies[c.Name]; ok { + delete(icookies, c.Name) + changed = true + } + } + if changed { + ireqhdr.Del("Cookie") + var ss []string + for _, cs := range icookies { + for _, c := range cs { + ss = append(ss, c.Name+"="+c.Value) + } + } + sort.Strings(ss) // Ensure deterministic headers + ireqhdr.Set("Cookie", strings.Join(ss, "; ")) + } + } + + // Copy the initial request's Header values + // (at least the safe ones). + for k, vv := range ireqhdr { + if shouldCopyHeaderOnRedirect(k, preq.URL, req.URL) { + req.Header[k] = vv + } + } + + preq = req // Update previous Request with the current request } - return nil, urlErr } func defaultCheckRedirect(req *Request, via []*Request) error { @@ -538,8 +693,11 @@ func defaultCheckRedirect(req *Request, via []*Request) error { // Post is a wrapper around DefaultClient.Post. // // To set custom headers, use NewRequest and DefaultClient.Do. -func Post(url string, bodyType string, body io.Reader) (resp *Response, err error) { - return DefaultClient.Post(url, bodyType, body) +// +// See the Client.Do method documentation for details on how redirects +// are handled. +func Post(url string, contentType string, body io.Reader) (resp *Response, err error) { + return DefaultClient.Post(url, contentType, body) } // Post issues a POST to the specified URL. @@ -550,13 +708,16 @@ func Post(url string, bodyType string, body io.Reader) (resp *Response, err erro // request. // // To set custom headers, use NewRequest and Client.Do. -func (c *Client) Post(url string, bodyType string, body io.Reader) (resp *Response, err error) { +// +// See the Client.Do method documentation for details on how redirects +// are handled. +func (c *Client) Post(url string, contentType string, body io.Reader) (resp *Response, err error) { req, err := NewRequest("POST", url, body) if err != nil { return nil, err } - req.Header.Set("Content-Type", bodyType) - return c.doFollowingRedirects(req, shouldRedirectPost) + req.Header.Set("Content-Type", contentType) + return c.Do(req) } // PostForm issues a POST to the specified URL, with data's keys and @@ -569,6 +730,9 @@ func (c *Client) Post(url string, bodyType string, body io.Reader) (resp *Respon // Caller should close resp.Body when done reading from it. // // PostForm is a wrapper around DefaultClient.PostForm. +// +// See the Client.Do method documentation for details on how redirects +// are handled. func PostForm(url string, data url.Values) (resp *Response, err error) { return DefaultClient.PostForm(url, data) } @@ -581,11 +745,14 @@ func PostForm(url string, data url.Values) (resp *Response, err error) { // // When err is nil, resp always contains a non-nil resp.Body. // Caller should close resp.Body when done reading from it. +// +// See the Client.Do method documentation for details on how redirects +// are handled. func (c *Client) PostForm(url string, data url.Values) (resp *Response, err error) { return c.Post(url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode())) } -// Head issues a HEAD to the specified URL. If the response is one of +// Head issues a HEAD to the specified URL. If the response is one of // the following redirect codes, Head follows the redirect, up to a // maximum of 10 redirects: // @@ -593,13 +760,14 @@ func (c *Client) PostForm(url string, data url.Values) (resp *Response, err erro // 302 (Found) // 303 (See Other) // 307 (Temporary Redirect) +// 308 (Permanent Redirect) // // Head is a wrapper around DefaultClient.Head func Head(url string) (resp *Response, err error) { return DefaultClient.Head(url) } -// Head issues a HEAD to the specified URL. If the response is one of the +// Head issues a HEAD to the specified URL. If the response is one of the // following redirect codes, Head follows the redirect after calling the // Client's CheckRedirect function: // @@ -607,22 +775,23 @@ func Head(url string) (resp *Response, err error) { // 302 (Found) // 303 (See Other) // 307 (Temporary Redirect) +// 308 (Permanent Redirect) func (c *Client) Head(url string) (resp *Response, err error) { req, err := NewRequest("HEAD", url, nil) if err != nil { return nil, err } - return c.doFollowingRedirects(req, shouldRedirectGet) + return c.Do(req) } // cancelTimerBody is an io.ReadCloser that wraps rc with two features: // 1) on Read error or close, the stop func is called. -// 2) On Read failure, if reqWasCanceled is true, the error is wrapped and +// 2) On Read failure, if reqDidTimeout is true, the error is wrapped and // marked as net.Error that hit its timeout. type cancelTimerBody struct { - stop func() // stops the time.Timer waiting to cancel the request - rc io.ReadCloser - reqWasCanceled func() bool + stop func() // stops the time.Timer waiting to cancel the request + rc io.ReadCloser + reqDidTimeout func() bool } func (b *cancelTimerBody) Read(p []byte) (n int, err error) { @@ -634,7 +803,7 @@ func (b *cancelTimerBody) Read(p []byte) (n int, err error) { if err == io.EOF { return n, err } - if b.reqWasCanceled() { + if b.reqDidTimeout() { err = &httpError{ err: err.Error() + " (Client.Timeout exceeded while reading body)", timeout: true, @@ -648,3 +817,52 @@ func (b *cancelTimerBody) Close() error { b.stop() return err } + +func shouldCopyHeaderOnRedirect(headerKey string, initial, dest *url.URL) bool { + switch CanonicalHeaderKey(headerKey) { + case "Authorization", "Www-Authenticate", "Cookie", "Cookie2": + // Permit sending auth/cookie headers from "foo.com" + // to "sub.foo.com". + + // Note that we don't send all cookies to subdomains + // automatically. This function is only used for + // Cookies set explicitly on the initial outgoing + // client request. Cookies automatically added via the + // CookieJar mechanism continue to follow each + // cookie's scope as set by Set-Cookie. But for + // outgoing requests with the Cookie header set + // directly, we don't know their scope, so we assume + // it's for *.domain.com. + + // TODO(bradfitz): once issue 16142 is fixed, make + // this code use those URL accessors, and consider + // "http://foo.com" and "http://foo.com:80" as + // equivalent? + + // TODO(bradfitz): better hostname canonicalization, + // at least once we figure out IDNA/Punycode (issue + // 13835). + ihost := strings.ToLower(initial.Host) + dhost := strings.ToLower(dest.Host) + return isDomainOrSubdomain(dhost, ihost) + } + // All other headers are copied: + return true +} + +// isDomainOrSubdomain reports whether sub is a subdomain (or exact +// match) of the parent domain. +// +// Both domains must already be in canonical form. +func isDomainOrSubdomain(sub, parent string) bool { + if sub == parent { + return true + } + // If sub is "foo.example.com" and parent is "example.com", + // that means sub must end in "."+parent. + // Do it without allocating. + if !strings.HasSuffix(sub, parent) { + return false + } + return sub[len(sub)-len(parent)-1] == '.' +} diff --git a/libgo/go/net/http/client_test.go b/libgo/go/net/http/client_test.go index 8939dc8baf..4f674dd8d6 100644 --- a/libgo/go/net/http/client_test.go +++ b/libgo/go/net/http/client_test.go @@ -8,6 +8,7 @@ package http_test import ( "bytes" + "context" "crypto/tls" "crypto/x509" "encoding/base64" @@ -18,11 +19,14 @@ import ( "log" "net" . "net/http" + "net/http/cookiejar" "net/http/httptest" "net/url" + "reflect" "strconv" "strings" "sync" + "sync/atomic" "testing" "time" ) @@ -64,11 +68,13 @@ func (w chanWriter) Write(p []byte) (n int, err error) { } func TestClient(t *testing.T) { + setParallel(t) defer afterTest(t) ts := httptest.NewServer(robotsTxtHandler) defer ts.Close() - r, err := Get(ts.URL) + c := &Client{Transport: &Transport{DisableKeepAlives: true}} + r, err := c.Get(ts.URL) var b []byte if err == nil { b, err = pedanticReadAll(r.Body) @@ -108,6 +114,7 @@ func (t *recordingTransport) RoundTrip(req *Request) (resp *Response, err error) } func TestGetRequestFormat(t *testing.T) { + setParallel(t) defer afterTest(t) tr := &recordingTransport{} client := &Client{Transport: tr} @@ -194,6 +201,7 @@ func TestPostFormRequestFormat(t *testing.T) { } func TestClientRedirects(t *testing.T) { + setParallel(t) defer afterTest(t) var ts *httptest.Server ts = httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { @@ -205,14 +213,17 @@ func TestClientRedirects(t *testing.T) { } } if n < 15 { - Redirect(w, r, fmt.Sprintf("/?n=%d", n+1), StatusFound) + Redirect(w, r, fmt.Sprintf("/?n=%d", n+1), StatusTemporaryRedirect) return } fmt.Fprintf(w, "n=%d", n) })) defer ts.Close() - c := &Client{} + tr := &Transport{} + defer tr.CloseIdleConnections() + + c := &Client{Transport: tr} _, err := c.Get(ts.URL) if e, g := "Get /?n=10: stopped after 10 redirects", fmt.Sprintf("%v", err); e != g { t.Errorf("with default client Get, expected error %q, got %q", e, g) @@ -241,11 +252,14 @@ func TestClientRedirects(t *testing.T) { var checkErr error var lastVia []*Request var lastReq *Request - c = &Client{CheckRedirect: func(req *Request, via []*Request) error { - lastReq = req - lastVia = via - return checkErr - }} + c = &Client{ + Transport: tr, + CheckRedirect: func(req *Request, via []*Request) error { + lastReq = req + lastVia = via + return checkErr + }, + } res, err := c.Get(ts.URL) if err != nil { t.Fatalf("Get error: %v", err) @@ -273,7 +287,7 @@ func TestClientRedirects(t *testing.T) { t.Fatal("didn't see redirect") } if lastReq.Cancel != cancel { - t.Errorf("expected lastReq to have the cancel channel set on the inital req") + t.Errorf("expected lastReq to have the cancel channel set on the initial req") } checkErr = errors.New("no redirects allowed") @@ -290,7 +304,125 @@ func TestClientRedirects(t *testing.T) { } } +func TestClientRedirectContext(t *testing.T) { + setParallel(t) + defer afterTest(t) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + Redirect(w, r, "/", StatusTemporaryRedirect) + })) + defer ts.Close() + + tr := &Transport{} + defer tr.CloseIdleConnections() + + ctx, cancel := context.WithCancel(context.Background()) + c := &Client{ + Transport: tr, + CheckRedirect: func(req *Request, via []*Request) error { + cancel() + if len(via) > 2 { + return errors.New("too many redirects") + } + return nil + }, + } + req, _ := NewRequest("GET", ts.URL, nil) + req = req.WithContext(ctx) + _, err := c.Do(req) + ue, ok := err.(*url.Error) + if !ok { + t.Fatalf("got error %T; want *url.Error", err) + } + if ue.Err != context.Canceled { + t.Errorf("url.Error.Err = %v; want %v", ue.Err, context.Canceled) + } +} + +type redirectTest struct { + suffix string + want int // response code + redirectBody string +} + func TestPostRedirects(t *testing.T) { + postRedirectTests := []redirectTest{ + {"/", 200, "first"}, + {"/?code=301&next=302", 200, "c301"}, + {"/?code=302&next=302", 200, "c302"}, + {"/?code=303&next=301", 200, "c303wc301"}, // Issue 9348 + {"/?code=304", 304, "c304"}, + {"/?code=305", 305, "c305"}, + {"/?code=307&next=303,308,302", 200, "c307"}, + {"/?code=308&next=302,301", 200, "c308"}, + {"/?code=404", 404, "c404"}, + } + + wantSegments := []string{ + `POST / "first"`, + `POST /?code=301&next=302 "c301"`, + `GET /?code=302 ""`, + `GET / ""`, + `POST /?code=302&next=302 "c302"`, + `GET /?code=302 ""`, + `GET / ""`, + `POST /?code=303&next=301 "c303wc301"`, + `GET /?code=301 ""`, + `GET / ""`, + `POST /?code=304 "c304"`, + `POST /?code=305 "c305"`, + `POST /?code=307&next=303,308,302 "c307"`, + `POST /?code=303&next=308,302 "c307"`, + `GET /?code=308&next=302 ""`, + `GET /?code=302 "c307"`, + `GET / ""`, + `POST /?code=308&next=302,301 "c308"`, + `POST /?code=302&next=301 "c308"`, + `GET /?code=301 ""`, + `GET / ""`, + `POST /?code=404 "c404"`, + } + want := strings.Join(wantSegments, "\n") + testRedirectsByMethod(t, "POST", postRedirectTests, want) +} + +func TestDeleteRedirects(t *testing.T) { + deleteRedirectTests := []redirectTest{ + {"/", 200, "first"}, + {"/?code=301&next=302,308", 200, "c301"}, + {"/?code=302&next=302", 200, "c302"}, + {"/?code=303", 200, "c303"}, + {"/?code=307&next=301,308,303,302,304", 304, "c307"}, + {"/?code=308&next=307", 200, "c308"}, + {"/?code=404", 404, "c404"}, + } + + wantSegments := []string{ + `DELETE / "first"`, + `DELETE /?code=301&next=302,308 "c301"`, + `GET /?code=302&next=308 ""`, + `GET /?code=308 ""`, + `GET / "c301"`, + `DELETE /?code=302&next=302 "c302"`, + `GET /?code=302 ""`, + `GET / ""`, + `DELETE /?code=303 "c303"`, + `GET / ""`, + `DELETE /?code=307&next=301,308,303,302,304 "c307"`, + `DELETE /?code=301&next=308,303,302,304 "c307"`, + `GET /?code=308&next=303,302,304 ""`, + `GET /?code=303&next=302,304 "c307"`, + `GET /?code=302&next=304 ""`, + `GET /?code=304 ""`, + `DELETE /?code=308&next=307 "c308"`, + `DELETE /?code=307 "c308"`, + `DELETE / "c308"`, + `DELETE /?code=404 "c404"`, + } + want := strings.Join(wantSegments, "\n") + testRedirectsByMethod(t, "DELETE", deleteRedirectTests, want) +} + +func testRedirectsByMethod(t *testing.T, method string, table []redirectTest, want string) { defer afterTest(t) var log struct { sync.Mutex @@ -299,29 +431,39 @@ func TestPostRedirects(t *testing.T) { var ts *httptest.Server ts = httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { log.Lock() - fmt.Fprintf(&log.Buffer, "%s %s ", r.Method, r.RequestURI) + slurp, _ := ioutil.ReadAll(r.Body) + fmt.Fprintf(&log.Buffer, "%s %s %q", r.Method, r.RequestURI, slurp) + if cl := r.Header.Get("Content-Length"); r.Method == "GET" && len(slurp) == 0 && (r.ContentLength != 0 || cl != "") { + fmt.Fprintf(&log.Buffer, " (but with body=%T, content-length = %v, %q)", r.Body, r.ContentLength, cl) + } + log.WriteByte('\n') log.Unlock() - if v := r.URL.Query().Get("code"); v != "" { + urlQuery := r.URL.Query() + if v := urlQuery.Get("code"); v != "" { + location := ts.URL + if final := urlQuery.Get("next"); final != "" { + splits := strings.Split(final, ",") + first, rest := splits[0], splits[1:] + location = fmt.Sprintf("%s?code=%s", location, first) + if len(rest) > 0 { + location = fmt.Sprintf("%s&next=%s", location, strings.Join(rest, ",")) + } + } code, _ := strconv.Atoi(v) if code/100 == 3 { - w.Header().Set("Location", ts.URL) + w.Header().Set("Location", location) } w.WriteHeader(code) } })) defer ts.Close() - tests := []struct { - suffix string - want int // response code - }{ - {"/", 200}, - {"/?code=301", 301}, - {"/?code=302", 200}, - {"/?code=303", 200}, - {"/?code=404", 404}, - } - for _, tt := range tests { - res, err := Post(ts.URL+tt.suffix, "text/plain", strings.NewReader("Some content")) + + for _, tt := range table { + content := tt.redirectBody + req, _ := NewRequest(method, ts.URL+tt.suffix, strings.NewReader(content)) + req.GetBody = func() (io.ReadCloser, error) { return ioutil.NopCloser(strings.NewReader(content)), nil } + res, err := DefaultClient.Do(req) + if err != nil { t.Fatal(err) } @@ -332,9 +474,125 @@ func TestPostRedirects(t *testing.T) { log.Lock() got := log.String() log.Unlock() - want := "POST / POST /?code=301 POST /?code=302 GET / POST /?code=303 GET / POST /?code=404 " + + got = strings.TrimSpace(got) + want = strings.TrimSpace(want) + if got != want { - t.Errorf("Log differs.\n Got: %q\nWant: %q", got, want) + got, want, lines := removeCommonLines(got, want) + t.Errorf("Log differs after %d common lines.\n\nGot:\n%s\n\nWant:\n%s\n", lines, got, want) + } +} + +func removeCommonLines(a, b string) (asuffix, bsuffix string, commonLines int) { + for { + nl := strings.IndexByte(a, '\n') + if nl < 0 { + return a, b, commonLines + } + line := a[:nl+1] + if !strings.HasPrefix(b, line) { + return a, b, commonLines + } + commonLines++ + a = a[len(line):] + b = b[len(line):] + } +} + +func TestClientRedirectUseResponse(t *testing.T) { + setParallel(t) + defer afterTest(t) + const body = "Hello, world." + var ts *httptest.Server + ts = httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + if strings.Contains(r.URL.Path, "/other") { + io.WriteString(w, "wrong body") + } else { + w.Header().Set("Location", ts.URL+"/other") + w.WriteHeader(StatusFound) + io.WriteString(w, body) + } + })) + defer ts.Close() + + tr := &Transport{} + defer tr.CloseIdleConnections() + + c := &Client{ + Transport: tr, + CheckRedirect: func(req *Request, via []*Request) error { + if req.Response == nil { + t.Error("expected non-nil Request.Response") + } + return ErrUseLastResponse + }, + } + res, err := c.Get(ts.URL) + if err != nil { + t.Fatal(err) + } + if res.StatusCode != StatusFound { + t.Errorf("status = %d; want %d", res.StatusCode, StatusFound) + } + defer res.Body.Close() + slurp, err := ioutil.ReadAll(res.Body) + if err != nil { + t.Fatal(err) + } + if string(slurp) != body { + t.Errorf("body = %q; want %q", slurp, body) + } +} + +// Issue 17773: don't follow a 308 (or 307) if the response doesn't +// have a Location header. +func TestClientRedirect308NoLocation(t *testing.T) { + setParallel(t) + defer afterTest(t) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + w.Header().Set("Foo", "Bar") + w.WriteHeader(308) + })) + defer ts.Close() + res, err := Get(ts.URL) + if err != nil { + t.Fatal(err) + } + res.Body.Close() + if res.StatusCode != 308 { + t.Errorf("status = %d; want %d", res.StatusCode, 308) + } + if got := res.Header.Get("Foo"); got != "Bar" { + t.Errorf("Foo header = %q; want Bar", got) + } +} + +// Don't follow a 307/308 if we can't resent the request body. +func TestClientRedirect308NoGetBody(t *testing.T) { + setParallel(t) + defer afterTest(t) + const fakeURL = "https://localhost:1234/" // won't be hit + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + w.Header().Set("Location", fakeURL) + w.WriteHeader(308) + })) + defer ts.Close() + req, err := NewRequest("POST", ts.URL, strings.NewReader("some body")) + if err != nil { + t.Fatal(err) + } + req.GetBody = nil // so it can't rewind. + res, err := DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + res.Body.Close() + if res.StatusCode != 308 { + t.Errorf("status = %d; want %d", res.StatusCode, 308) + } + if got := res.Header.Get("Location"); got != fakeURL { + t.Errorf("Location header = %q; want %q", got, fakeURL) } } @@ -410,12 +668,16 @@ func (j *TestJar) Cookies(u *url.URL) []*Cookie { } func TestRedirectCookiesJar(t *testing.T) { + setParallel(t) defer afterTest(t) var ts *httptest.Server ts = httptest.NewServer(echoCookiesRedirectHandler) defer ts.Close() + tr := &Transport{} + defer tr.CloseIdleConnections() c := &Client{ - Jar: new(TestJar), + Transport: tr, + Jar: new(TestJar), } u, _ := url.Parse(ts.URL) c.Jar.SetCookies(u, []*Cookie{expectedCookies[0]}) @@ -599,6 +861,7 @@ func TestClientWrites(t *testing.T) { } func TestClientInsecureTransport(t *testing.T) { + setParallel(t) defer afterTest(t) ts := httptest.NewTLSServer(HandlerFunc(func(w ResponseWriter, r *Request) { w.Write([]byte("Hello")) @@ -776,6 +1039,7 @@ func TestResponseSetsTLSConnectionState(t *testing.T) { func TestHTTPSClientDetectsHTTPServer(t *testing.T) { defer afterTest(t) ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) {})) + ts.Config.ErrorLog = quietLog defer ts.Close() _, err := Get(strings.Replace(ts.URL, "http", "https", 1)) @@ -829,6 +1093,7 @@ func testClientHeadContentLength(t *testing.T, h2 bool) { } func TestEmptyPasswordAuth(t *testing.T) { + setParallel(t) defer afterTest(t) gopher := "gopher" ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { @@ -849,7 +1114,9 @@ func TestEmptyPasswordAuth(t *testing.T) { } })) defer ts.Close() - c := &Client{} + tr := &Transport{} + defer tr.CloseIdleConnections() + c := &Client{Transport: tr} req, err := NewRequest("GET", ts.URL, nil) if err != nil { t.Fatal(err) @@ -941,10 +1208,10 @@ func TestClientTimeout_h1(t *testing.T) { testClientTimeout(t, h1Mode) } func TestClientTimeout_h2(t *testing.T) { testClientTimeout(t, h2Mode) } func testClientTimeout(t *testing.T, h2 bool) { - if testing.Short() { - t.Skip("skipping in short mode") - } + setParallel(t) defer afterTest(t) + testDone := make(chan struct{}) // closed in defer below + sawRoot := make(chan bool, 1) sawSlow := make(chan bool, 1) cst := newClientServerTest(t, h2, HandlerFunc(func(w ResponseWriter, r *Request) { @@ -954,19 +1221,26 @@ func testClientTimeout(t *testing.T, h2 bool) { return } if r.URL.Path == "/slow" { + sawSlow <- true w.Write([]byte("Hello")) w.(Flusher).Flush() - sawSlow <- true - time.Sleep(2 * time.Second) + <-testDone return } })) defer cst.close() - const timeout = 500 * time.Millisecond + defer close(testDone) // before cst.close, to unblock /slow handler + + // 200ms should be long enough to get a normal request (the / + // handler), but not so long that it makes the test slow. + const timeout = 200 * time.Millisecond cst.c.Timeout = timeout res, err := cst.c.Get(cst.ts.URL) if err != nil { + if strings.Contains(err.Error(), "Client.Timeout") { + t.Skipf("host too slow to get fast resource in %v", timeout) + } t.Fatal(err) } @@ -991,7 +1265,7 @@ func testClientTimeout(t *testing.T, h2 bool) { res.Body.Close() }() - const failTime = timeout * 2 + const failTime = 5 * time.Second select { case err := <-errc: if err == nil { @@ -1016,11 +1290,9 @@ func TestClientTimeout_Headers_h2(t *testing.T) { testClientTimeout_Headers(t, h // Client.Timeout firing before getting to the body func testClientTimeout_Headers(t *testing.T, h2 bool) { - if testing.Short() { - t.Skip("skipping in short mode") - } + setParallel(t) defer afterTest(t) - donec := make(chan bool) + donec := make(chan bool, 1) cst := newClientServerTest(t, h2, HandlerFunc(func(w ResponseWriter, r *Request) { <-donec })) @@ -1034,9 +1306,10 @@ func testClientTimeout_Headers(t *testing.T, h2 bool) { // doesn't know this, so synchronize explicitly. defer func() { donec <- true }() - cst.c.Timeout = 500 * time.Millisecond - _, err := cst.c.Get(cst.ts.URL) + cst.c.Timeout = 5 * time.Millisecond + res, err := cst.c.Get(cst.ts.URL) if err == nil { + res.Body.Close() t.Fatal("got response from Get; expected error") } if _, ok := err.(*url.Error); !ok { @@ -1054,9 +1327,40 @@ func testClientTimeout_Headers(t *testing.T, h2 bool) { } } +// Issue 16094: if Client.Timeout is set but not hit, a Timeout error shouldn't be +// returned. +func TestClientTimeoutCancel(t *testing.T) { + setParallel(t) + defer afterTest(t) + + testDone := make(chan struct{}) + ctx, cancel := context.WithCancel(context.Background()) + + cst := newClientServerTest(t, h1Mode, HandlerFunc(func(w ResponseWriter, r *Request) { + w.(Flusher).Flush() + <-testDone + })) + defer cst.close() + defer close(testDone) + + cst.c.Timeout = 1 * time.Hour + req, _ := NewRequest("GET", cst.ts.URL, nil) + req.Cancel = ctx.Done() + res, err := cst.c.Do(req) + if err != nil { + t.Fatal(err) + } + cancel() + _, err = io.Copy(ioutil.Discard, res.Body) + if err != ExportErrRequestCanceled { + t.Fatalf("error = %v; want errRequestCanceled", err) + } +} + func TestClientRedirectEatsBody_h1(t *testing.T) { testClientRedirectEatsBody(t, h1Mode) } func TestClientRedirectEatsBody_h2(t *testing.T) { testClientRedirectEatsBody(t, h2Mode) } func testClientRedirectEatsBody(t *testing.T, h2 bool) { + setParallel(t) defer afterTest(t) saw := make(chan string, 2) cst := newClientServerTest(t, h2, HandlerFunc(func(w ResponseWriter, r *Request) { @@ -1072,10 +1376,10 @@ func testClientRedirectEatsBody(t *testing.T, h2 bool) { t.Fatal(err) } _, err = ioutil.ReadAll(res.Body) + res.Body.Close() if err != nil { t.Fatal(err) } - res.Body.Close() var first string select { @@ -1140,3 +1444,392 @@ func TestReferer(t *testing.T) { } } } + +// issue15577Tripper returns a Response with a redirect response +// header and doesn't populate its Response.Request field. +type issue15577Tripper struct{} + +func (issue15577Tripper) RoundTrip(*Request) (*Response, error) { + resp := &Response{ + StatusCode: 303, + Header: map[string][]string{"Location": {"http://www.example.com/"}}, + Body: ioutil.NopCloser(strings.NewReader("")), + } + return resp, nil +} + +// Issue 15577: don't assume the roundtripper's response populates its Request field. +func TestClientRedirectResponseWithoutRequest(t *testing.T) { + c := &Client{ + CheckRedirect: func(*Request, []*Request) error { return fmt.Errorf("no redirects!") }, + Transport: issue15577Tripper{}, + } + // Check that this doesn't crash: + c.Get("http://dummy.tld") +} + +// Issue 4800: copy (some) headers when Client follows a redirect +func TestClientCopyHeadersOnRedirect(t *testing.T) { + const ( + ua = "some-agent/1.2" + xfoo = "foo-val" + ) + var ts2URL string + ts1 := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + want := Header{ + "User-Agent": []string{ua}, + "X-Foo": []string{xfoo}, + "Referer": []string{ts2URL}, + "Accept-Encoding": []string{"gzip"}, + } + if !reflect.DeepEqual(r.Header, want) { + t.Errorf("Request.Header = %#v; want %#v", r.Header, want) + } + if t.Failed() { + w.Header().Set("Result", "got errors") + } else { + w.Header().Set("Result", "ok") + } + })) + defer ts1.Close() + ts2 := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + Redirect(w, r, ts1.URL, StatusFound) + })) + defer ts2.Close() + ts2URL = ts2.URL + + tr := &Transport{} + defer tr.CloseIdleConnections() + c := &Client{ + Transport: tr, + CheckRedirect: func(r *Request, via []*Request) error { + want := Header{ + "User-Agent": []string{ua}, + "X-Foo": []string{xfoo}, + "Referer": []string{ts2URL}, + } + if !reflect.DeepEqual(r.Header, want) { + t.Errorf("CheckRedirect Request.Header = %#v; want %#v", r.Header, want) + } + return nil + }, + } + + req, _ := NewRequest("GET", ts2.URL, nil) + req.Header.Add("User-Agent", ua) + req.Header.Add("X-Foo", xfoo) + req.Header.Add("Cookie", "foo=bar") + req.Header.Add("Authorization", "secretpassword") + res, err := c.Do(req) + if err != nil { + t.Fatal(err) + } + defer res.Body.Close() + if res.StatusCode != 200 { + t.Fatal(res.Status) + } + if got := res.Header.Get("Result"); got != "ok" { + t.Errorf("result = %q; want ok", got) + } +} + +// Issue 17494: cookies should be altered when Client follows redirects. +func TestClientAltersCookiesOnRedirect(t *testing.T) { + cookieMap := func(cs []*Cookie) map[string][]string { + m := make(map[string][]string) + for _, c := range cs { + m[c.Name] = append(m[c.Name], c.Value) + } + return m + } + + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + var want map[string][]string + got := cookieMap(r.Cookies()) + + c, _ := r.Cookie("Cycle") + switch c.Value { + case "0": + want = map[string][]string{ + "Cookie1": {"OldValue1a", "OldValue1b"}, + "Cookie2": {"OldValue2"}, + "Cookie3": {"OldValue3a", "OldValue3b"}, + "Cookie4": {"OldValue4"}, + "Cycle": {"0"}, + } + SetCookie(w, &Cookie{Name: "Cycle", Value: "1", Path: "/"}) + SetCookie(w, &Cookie{Name: "Cookie2", Path: "/", MaxAge: -1}) // Delete cookie from Header + Redirect(w, r, "/", StatusFound) + case "1": + want = map[string][]string{ + "Cookie1": {"OldValue1a", "OldValue1b"}, + "Cookie3": {"OldValue3a", "OldValue3b"}, + "Cookie4": {"OldValue4"}, + "Cycle": {"1"}, + } + SetCookie(w, &Cookie{Name: "Cycle", Value: "2", Path: "/"}) + SetCookie(w, &Cookie{Name: "Cookie3", Value: "NewValue3", Path: "/"}) // Modify cookie in Header + SetCookie(w, &Cookie{Name: "Cookie4", Value: "NewValue4", Path: "/"}) // Modify cookie in Jar + Redirect(w, r, "/", StatusFound) + case "2": + want = map[string][]string{ + "Cookie1": {"OldValue1a", "OldValue1b"}, + "Cookie3": {"NewValue3"}, + "Cookie4": {"NewValue4"}, + "Cycle": {"2"}, + } + SetCookie(w, &Cookie{Name: "Cycle", Value: "3", Path: "/"}) + SetCookie(w, &Cookie{Name: "Cookie5", Value: "NewValue5", Path: "/"}) // Insert cookie into Jar + Redirect(w, r, "/", StatusFound) + case "3": + want = map[string][]string{ + "Cookie1": {"OldValue1a", "OldValue1b"}, + "Cookie3": {"NewValue3"}, + "Cookie4": {"NewValue4"}, + "Cookie5": {"NewValue5"}, + "Cycle": {"3"}, + } + // Don't redirect to ensure the loop ends. + default: + t.Errorf("unexpected redirect cycle") + return + } + + if !reflect.DeepEqual(got, want) { + t.Errorf("redirect %s, Cookie = %v, want %v", c.Value, got, want) + } + })) + defer ts.Close() + + tr := &Transport{} + defer tr.CloseIdleConnections() + jar, _ := cookiejar.New(nil) + c := &Client{ + Transport: tr, + Jar: jar, + } + + u, _ := url.Parse(ts.URL) + req, _ := NewRequest("GET", ts.URL, nil) + req.AddCookie(&Cookie{Name: "Cookie1", Value: "OldValue1a"}) + req.AddCookie(&Cookie{Name: "Cookie1", Value: "OldValue1b"}) + req.AddCookie(&Cookie{Name: "Cookie2", Value: "OldValue2"}) + req.AddCookie(&Cookie{Name: "Cookie3", Value: "OldValue3a"}) + req.AddCookie(&Cookie{Name: "Cookie3", Value: "OldValue3b"}) + jar.SetCookies(u, []*Cookie{{Name: "Cookie4", Value: "OldValue4", Path: "/"}}) + jar.SetCookies(u, []*Cookie{{Name: "Cycle", Value: "0", Path: "/"}}) + res, err := c.Do(req) + if err != nil { + t.Fatal(err) + } + defer res.Body.Close() + if res.StatusCode != 200 { + t.Fatal(res.Status) + } +} + +// Part of Issue 4800 +func TestShouldCopyHeaderOnRedirect(t *testing.T) { + tests := []struct { + header string + initialURL string + destURL string + want bool + }{ + {"User-Agent", "http://foo.com/", "http://bar.com/", true}, + {"X-Foo", "http://foo.com/", "http://bar.com/", true}, + + // Sensitive headers: + {"cookie", "http://foo.com/", "http://bar.com/", false}, + {"cookie2", "http://foo.com/", "http://bar.com/", false}, + {"authorization", "http://foo.com/", "http://bar.com/", false}, + {"www-authenticate", "http://foo.com/", "http://bar.com/", false}, + + // But subdomains should work: + {"www-authenticate", "http://foo.com/", "http://foo.com/", true}, + {"www-authenticate", "http://foo.com/", "http://sub.foo.com/", true}, + {"www-authenticate", "http://foo.com/", "http://notfoo.com/", false}, + // TODO(bradfitz): make this test work, once issue 16142 is fixed: + // {"www-authenticate", "http://foo.com:80/", "http://foo.com/", true}, + } + for i, tt := range tests { + u0, err := url.Parse(tt.initialURL) + if err != nil { + t.Errorf("%d. initial URL %q parse error: %v", i, tt.initialURL, err) + continue + } + u1, err := url.Parse(tt.destURL) + if err != nil { + t.Errorf("%d. dest URL %q parse error: %v", i, tt.destURL, err) + continue + } + got := Export_shouldCopyHeaderOnRedirect(tt.header, u0, u1) + if got != tt.want { + t.Errorf("%d. shouldCopyHeaderOnRedirect(%q, %q => %q) = %v; want %v", + i, tt.header, tt.initialURL, tt.destURL, got, tt.want) + } + } +} + +func TestClientRedirectTypes(t *testing.T) { + setParallel(t) + defer afterTest(t) + + tests := [...]struct { + method string + serverStatus int + wantMethod string // desired subsequent client method + }{ + 0: {method: "POST", serverStatus: 301, wantMethod: "GET"}, + 1: {method: "POST", serverStatus: 302, wantMethod: "GET"}, + 2: {method: "POST", serverStatus: 303, wantMethod: "GET"}, + 3: {method: "POST", serverStatus: 307, wantMethod: "POST"}, + 4: {method: "POST", serverStatus: 308, wantMethod: "POST"}, + + 5: {method: "HEAD", serverStatus: 301, wantMethod: "HEAD"}, + 6: {method: "HEAD", serverStatus: 302, wantMethod: "HEAD"}, + 7: {method: "HEAD", serverStatus: 303, wantMethod: "HEAD"}, + 8: {method: "HEAD", serverStatus: 307, wantMethod: "HEAD"}, + 9: {method: "HEAD", serverStatus: 308, wantMethod: "HEAD"}, + + 10: {method: "GET", serverStatus: 301, wantMethod: "GET"}, + 11: {method: "GET", serverStatus: 302, wantMethod: "GET"}, + 12: {method: "GET", serverStatus: 303, wantMethod: "GET"}, + 13: {method: "GET", serverStatus: 307, wantMethod: "GET"}, + 14: {method: "GET", serverStatus: 308, wantMethod: "GET"}, + + 15: {method: "DELETE", serverStatus: 301, wantMethod: "GET"}, + 16: {method: "DELETE", serverStatus: 302, wantMethod: "GET"}, + 17: {method: "DELETE", serverStatus: 303, wantMethod: "GET"}, + 18: {method: "DELETE", serverStatus: 307, wantMethod: "DELETE"}, + 19: {method: "DELETE", serverStatus: 308, wantMethod: "DELETE"}, + + 20: {method: "PUT", serverStatus: 301, wantMethod: "GET"}, + 21: {method: "PUT", serverStatus: 302, wantMethod: "GET"}, + 22: {method: "PUT", serverStatus: 303, wantMethod: "GET"}, + 23: {method: "PUT", serverStatus: 307, wantMethod: "PUT"}, + 24: {method: "PUT", serverStatus: 308, wantMethod: "PUT"}, + + 25: {method: "MADEUPMETHOD", serverStatus: 301, wantMethod: "GET"}, + 26: {method: "MADEUPMETHOD", serverStatus: 302, wantMethod: "GET"}, + 27: {method: "MADEUPMETHOD", serverStatus: 303, wantMethod: "GET"}, + 28: {method: "MADEUPMETHOD", serverStatus: 307, wantMethod: "MADEUPMETHOD"}, + 29: {method: "MADEUPMETHOD", serverStatus: 308, wantMethod: "MADEUPMETHOD"}, + } + + handlerc := make(chan HandlerFunc, 1) + + ts := httptest.NewServer(HandlerFunc(func(rw ResponseWriter, req *Request) { + h := <-handlerc + h(rw, req) + })) + defer ts.Close() + + tr := &Transport{} + defer tr.CloseIdleConnections() + + for i, tt := range tests { + handlerc <- func(w ResponseWriter, r *Request) { + w.Header().Set("Location", ts.URL) + w.WriteHeader(tt.serverStatus) + } + + req, err := NewRequest(tt.method, ts.URL, nil) + if err != nil { + t.Errorf("#%d: NewRequest: %v", i, err) + continue + } + + c := &Client{Transport: tr} + c.CheckRedirect = func(req *Request, via []*Request) error { + if got, want := req.Method, tt.wantMethod; got != want { + return fmt.Errorf("#%d: got next method %q; want %q", i, got, want) + } + handlerc <- func(rw ResponseWriter, req *Request) { + // TODO: Check that the body is valid when we do 307 and 308 support + } + return nil + } + + res, err := c.Do(req) + if err != nil { + t.Errorf("#%d: Response: %v", i, err) + continue + } + + res.Body.Close() + } +} + +// issue18239Body is an io.ReadCloser for TestTransportBodyReadError. +// Its Read returns readErr and increments *readCalls atomically. +// Its Close returns nil and increments *closeCalls atomically. +type issue18239Body struct { + readCalls *int32 + closeCalls *int32 + readErr error +} + +func (b issue18239Body) Read([]byte) (int, error) { + atomic.AddInt32(b.readCalls, 1) + return 0, b.readErr +} + +func (b issue18239Body) Close() error { + atomic.AddInt32(b.closeCalls, 1) + return nil +} + +// Issue 18239: make sure the Transport doesn't retry requests with bodies. +// (Especially if Request.GetBody is not defined.) +func TestTransportBodyReadError(t *testing.T) { + setParallel(t) + defer afterTest(t) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + if r.URL.Path == "/ping" { + return + } + buf := make([]byte, 1) + n, err := r.Body.Read(buf) + w.Header().Set("X-Body-Read", fmt.Sprintf("%v, %v", n, err)) + })) + defer ts.Close() + tr := &Transport{} + defer tr.CloseIdleConnections() + c := &Client{Transport: tr} + + // Do one initial successful request to create an idle TCP connection + // for the subsequent request to reuse. (The Transport only retries + // requests on reused connections.) + res, err := c.Get(ts.URL + "/ping") + if err != nil { + t.Fatal(err) + } + res.Body.Close() + + var readCallsAtomic int32 + var closeCallsAtomic int32 // atomic + someErr := errors.New("some body read error") + body := issue18239Body{&readCallsAtomic, &closeCallsAtomic, someErr} + + req, err := NewRequest("POST", ts.URL, body) + if err != nil { + t.Fatal(err) + } + _, err = tr.RoundTrip(req) + if err != someErr { + t.Errorf("Got error: %v; want Request.Body read error: %v", err, someErr) + } + + // And verify that our Body wasn't used multiple times, which + // would indicate retries. (as it buggily was during part of + // Go 1.8's dev cycle) + readCalls := atomic.LoadInt32(&readCallsAtomic) + closeCalls := atomic.LoadInt32(&closeCallsAtomic) + if readCalls != 1 { + t.Errorf("read calls = %d; want 1", readCalls) + } + if closeCalls != 1 { + t.Errorf("close calls = %d; want 1", closeCalls) + } +} diff --git a/libgo/go/net/http/clientserver_test.go b/libgo/go/net/http/clientserver_test.go index aa2473a277..580115ca9c 100644 --- a/libgo/go/net/http/clientserver_test.go +++ b/libgo/go/net/http/clientserver_test.go @@ -17,6 +17,7 @@ import ( "net" . "net/http" "net/http/httptest" + "net/http/httputil" "net/url" "os" "reflect" @@ -43,11 +44,35 @@ func (t *clientServerTest) close() { t.ts.Close() } +func (t *clientServerTest) getURL(u string) string { + res, err := t.c.Get(u) + if err != nil { + t.t.Fatal(err) + } + defer res.Body.Close() + slurp, err := ioutil.ReadAll(res.Body) + if err != nil { + t.t.Fatal(err) + } + return string(slurp) +} + +func (t *clientServerTest) scheme() string { + if t.h2 { + return "https" + } + return "http" +} + const ( h1Mode = false h2Mode = true ) +var optQuietLog = func(ts *httptest.Server) { + ts.Config.ErrorLog = quietLog +} + func newClientServerTest(t *testing.T, h2 bool, h Handler, opts ...interface{}) *clientServerTest { cst := &clientServerTest{ t: t, @@ -56,21 +81,23 @@ func newClientServerTest(t *testing.T, h2 bool, h Handler, opts ...interface{}) tr: &Transport{}, } cst.c = &Client{Transport: cst.tr} + cst.ts = httptest.NewUnstartedServer(h) for _, opt := range opts { switch opt := opt.(type) { case func(*Transport): opt(cst.tr) + case func(*httptest.Server): + opt(cst.ts) default: t.Fatalf("unhandled option type %T", opt) } } if !h2 { - cst.ts = httptest.NewServer(h) + cst.ts.Start() return cst } - cst.ts = httptest.NewUnstartedServer(h) ExportHttp2ConfigureServer(cst.ts.Config, nil) cst.ts.TLS = cst.ts.Config.TLSConfig cst.ts.StartTLS() @@ -147,10 +174,11 @@ type reqFunc func(c *Client, url string) (*Response, error) // h12Compare is a test that compares HTTP/1 and HTTP/2 behavior // against each other. type h12Compare struct { - Handler func(ResponseWriter, *Request) // required - ReqFunc reqFunc // optional - CheckResponse func(proto string, res *Response) // optional - Opts []interface{} + Handler func(ResponseWriter, *Request) // required + ReqFunc reqFunc // optional + CheckResponse func(proto string, res *Response) // optional + EarlyCheckResponse func(proto string, res *Response) // optional; pre-normalize + Opts []interface{} } func (tt h12Compare) reqFunc() reqFunc { @@ -161,6 +189,7 @@ func (tt h12Compare) reqFunc() reqFunc { } func (tt h12Compare) run(t *testing.T) { + setParallel(t) cst1 := newClientServerTest(t, false, HandlerFunc(tt.Handler), tt.Opts...) defer cst1.close() cst2 := newClientServerTest(t, true, HandlerFunc(tt.Handler), tt.Opts...) @@ -176,6 +205,12 @@ func (tt h12Compare) run(t *testing.T) { t.Errorf("HTTP/2 request: %v", err) return } + + if fn := tt.EarlyCheckResponse; fn != nil { + fn("HTTP/1.1", res1) + fn("HTTP/2.0", res2) + } + tt.normalizeRes(t, res1, "HTTP/1.1") tt.normalizeRes(t, res2, "HTTP/2.0") res1body, res2body := res1.Body, res2.Body @@ -220,6 +255,7 @@ func (tt h12Compare) normalizeRes(t *testing.T, res *Response, wantProto string) t.Errorf("got %q response; want %q", res.Proto, wantProto) } slurp, err := ioutil.ReadAll(res.Body) + res.Body.Close() res.Body = slurpResult{ ReadCloser: ioutil.NopCloser(bytes.NewReader(slurp)), @@ -356,7 +392,7 @@ func TestH12_HandlerWritesTooLittle(t *testing.T) { } // Tests that the HTTP/1 and HTTP/2 servers prevent handlers from -// writing more than they declared. This test does not test whether +// writing more than they declared. This test does not test whether // the transport deals with too much data, though, since the server // doesn't make it possible to send bogus data. For those tests, see // transport_test.go (for HTTP/1) or x/net/http2/transport_test.go @@ -452,7 +488,7 @@ func TestH12_RequestContentLength_Known_NonZero(t *testing.T) { } func TestH12_RequestContentLength_Known_Zero(t *testing.T) { - h12requestContentLength(t, func() io.Reader { return strings.NewReader("") }, 0) + h12requestContentLength(t, func() io.Reader { return nil }, 0) } func TestH12_RequestContentLength_Unknown(t *testing.T) { @@ -922,6 +958,7 @@ func testStarRequest(t *testing.T, method string, h2 bool) { // Issue 13957 func TestTransportDiscardsUnneededConns(t *testing.T) { + setParallel(t) defer afterTest(t) cst := newClientServerTest(t, h2Mode, HandlerFunc(func(w ResponseWriter, r *Request) { fmt.Fprintf(w, "Hello, %v", r.RemoteAddr) @@ -1010,6 +1047,7 @@ func testTransportGCRequest(t *testing.T, h2, body bool) { t.Skip("skipping on gccgo because conservative GC means that finalizer may never run") } + setParallel(t) defer afterTest(t) cst := newClientServerTest(t, h2, HandlerFunc(func(w ResponseWriter, r *Request) { ioutil.ReadAll(r.Body) @@ -1049,6 +1087,228 @@ func testTransportGCRequest(t *testing.T, h2, body bool) { } } +func TestTransportRejectsInvalidHeaders_h1(t *testing.T) { + testTransportRejectsInvalidHeaders(t, h1Mode) +} +func TestTransportRejectsInvalidHeaders_h2(t *testing.T) { + testTransportRejectsInvalidHeaders(t, h2Mode) +} +func testTransportRejectsInvalidHeaders(t *testing.T, h2 bool) { + setParallel(t) + defer afterTest(t) + cst := newClientServerTest(t, h2, HandlerFunc(func(w ResponseWriter, r *Request) { + fmt.Fprintf(w, "Handler saw headers: %q", r.Header) + }), optQuietLog) + defer cst.close() + cst.tr.DisableKeepAlives = true + + tests := []struct { + key, val string + ok bool + }{ + {"Foo", "capital-key", true}, // verify h2 allows capital keys + {"Foo", "foo\x00bar", false}, // \x00 byte in value not allowed + {"Foo", "two\nlines", false}, // \n byte in value not allowed + {"bogus\nkey", "v", false}, // \n byte also not allowed in key + {"A space", "v", false}, // spaces in keys not allowed + {"имя", "v", false}, // key must be ascii + {"name", "валю", true}, // value may be non-ascii + {"", "v", false}, // key must be non-empty + {"k", "", true}, // value may be empty + } + for _, tt := range tests { + dialedc := make(chan bool, 1) + cst.tr.Dial = func(netw, addr string) (net.Conn, error) { + dialedc <- true + return net.Dial(netw, addr) + } + req, _ := NewRequest("GET", cst.ts.URL, nil) + req.Header[tt.key] = []string{tt.val} + res, err := cst.c.Do(req) + var body []byte + if err == nil { + body, _ = ioutil.ReadAll(res.Body) + res.Body.Close() + } + var dialed bool + select { + case <-dialedc: + dialed = true + default: + } + + if !tt.ok && dialed { + t.Errorf("For key %q, value %q, transport dialed. Expected local failure. Response was: (%v, %v)\nServer replied with: %s", tt.key, tt.val, res, err, body) + } else if (err == nil) != tt.ok { + t.Errorf("For key %q, value %q; got err = %v; want ok=%v", tt.key, tt.val, err, tt.ok) + } + } +} + +// Tests that we support bogus under-100 HTTP statuses, because we historically +// have. This might change at some point, but not yet in Go 1.6. +func TestBogusStatusWorks_h1(t *testing.T) { testBogusStatusWorks(t, h1Mode) } +func TestBogusStatusWorks_h2(t *testing.T) { testBogusStatusWorks(t, h2Mode) } +func testBogusStatusWorks(t *testing.T, h2 bool) { + defer afterTest(t) + const code = 7 + cst := newClientServerTest(t, h2, HandlerFunc(func(w ResponseWriter, r *Request) { + w.WriteHeader(code) + })) + defer cst.close() + + res, err := cst.c.Get(cst.ts.URL) + if err != nil { + t.Fatal(err) + } + if res.StatusCode != code { + t.Errorf("StatusCode = %d; want %d", res.StatusCode, code) + } +} + +func TestInterruptWithPanic_h1(t *testing.T) { testInterruptWithPanic(t, h1Mode, "boom") } +func TestInterruptWithPanic_h2(t *testing.T) { testInterruptWithPanic(t, h2Mode, "boom") } +func TestInterruptWithPanic_nil_h1(t *testing.T) { testInterruptWithPanic(t, h1Mode, nil) } +func TestInterruptWithPanic_nil_h2(t *testing.T) { testInterruptWithPanic(t, h2Mode, nil) } +func TestInterruptWithPanic_ErrAbortHandler_h1(t *testing.T) { + testInterruptWithPanic(t, h1Mode, ErrAbortHandler) +} +func TestInterruptWithPanic_ErrAbortHandler_h2(t *testing.T) { + testInterruptWithPanic(t, h2Mode, ErrAbortHandler) +} +func testInterruptWithPanic(t *testing.T, h2 bool, panicValue interface{}) { + setParallel(t) + const msg = "hello" + defer afterTest(t) + + testDone := make(chan struct{}) + defer close(testDone) + + var errorLog lockedBytesBuffer + gotHeaders := make(chan bool, 1) + cst := newClientServerTest(t, h2, HandlerFunc(func(w ResponseWriter, r *Request) { + io.WriteString(w, msg) + w.(Flusher).Flush() + + select { + case <-gotHeaders: + case <-testDone: + } + panic(panicValue) + }), func(ts *httptest.Server) { + ts.Config.ErrorLog = log.New(&errorLog, "", 0) + }) + defer cst.close() + res, err := cst.c.Get(cst.ts.URL) + if err != nil { + t.Fatal(err) + } + gotHeaders <- true + defer res.Body.Close() + slurp, err := ioutil.ReadAll(res.Body) + if string(slurp) != msg { + t.Errorf("client read %q; want %q", slurp, msg) + } + if err == nil { + t.Errorf("client read all successfully; want some error") + } + logOutput := func() string { + errorLog.Lock() + defer errorLog.Unlock() + return errorLog.String() + } + wantStackLogged := panicValue != nil && panicValue != ErrAbortHandler + + if err := waitErrCondition(5*time.Second, 10*time.Millisecond, func() error { + gotLog := logOutput() + if !wantStackLogged { + if gotLog == "" { + return nil + } + return fmt.Errorf("want no log output; got: %s", gotLog) + } + if gotLog == "" { + return fmt.Errorf("wanted a stack trace logged; got nothing") + } + if !strings.Contains(gotLog, "created by ") && strings.Count(gotLog, "\n") < 6 { + return fmt.Errorf("output doesn't look like a panic stack trace. Got: %s", gotLog) + } + return nil + }); err != nil { + t.Fatal(err) + } +} + +type lockedBytesBuffer struct { + sync.Mutex + bytes.Buffer +} + +func (b *lockedBytesBuffer) Write(p []byte) (int, error) { + b.Lock() + defer b.Unlock() + return b.Buffer.Write(p) +} + +// Issue 15366 +func TestH12_AutoGzipWithDumpResponse(t *testing.T) { + h12Compare{ + Handler: func(w ResponseWriter, r *Request) { + h := w.Header() + h.Set("Content-Encoding", "gzip") + h.Set("Content-Length", "23") + h.Set("Connection", "keep-alive") + io.WriteString(w, "\x1f\x8b\b\x00\x00\x00\x00\x00\x00\x00s\xf3\xf7\a\x00\xab'\xd4\x1a\x03\x00\x00\x00") + }, + EarlyCheckResponse: func(proto string, res *Response) { + if !res.Uncompressed { + t.Errorf("%s: expected Uncompressed to be set", proto) + } + dump, err := httputil.DumpResponse(res, true) + if err != nil { + t.Errorf("%s: DumpResponse: %v", proto, err) + return + } + if strings.Contains(string(dump), "Connection: close") { + t.Errorf("%s: should not see \"Connection: close\" in dump; got:\n%s", proto, dump) + } + if !strings.Contains(string(dump), "FOO") { + t.Errorf("%s: should see \"FOO\" in response; got:\n%s", proto, dump) + } + }, + }.run(t) +} + +// Issue 14607 +func TestCloseIdleConnections_h1(t *testing.T) { testCloseIdleConnections(t, h1Mode) } +func TestCloseIdleConnections_h2(t *testing.T) { testCloseIdleConnections(t, h2Mode) } +func testCloseIdleConnections(t *testing.T, h2 bool) { + setParallel(t) + defer afterTest(t) + cst := newClientServerTest(t, h2, HandlerFunc(func(w ResponseWriter, r *Request) { + w.Header().Set("X-Addr", r.RemoteAddr) + })) + defer cst.close() + get := func() string { + res, err := cst.c.Get(cst.ts.URL) + if err != nil { + t.Fatal(err) + } + res.Body.Close() + v := res.Header.Get("X-Addr") + if v == "" { + t.Fatal("didn't get X-Addr") + } + return v + } + a1 := get() + cst.tr.CloseIdleConnections() + a2 := get() + if a1 == a2 { + t.Errorf("didn't close connection") + } +} + type noteCloseConn struct { net.Conn closeFunc func() @@ -1058,3 +1318,70 @@ func (x noteCloseConn) Close() error { x.closeFunc() return x.Conn.Close() } + +type testErrorReader struct{ t *testing.T } + +func (r testErrorReader) Read(p []byte) (n int, err error) { + r.t.Error("unexpected Read call") + return 0, io.EOF +} + +func TestNoSniffExpectRequestBody_h1(t *testing.T) { testNoSniffExpectRequestBody(t, h1Mode) } +func TestNoSniffExpectRequestBody_h2(t *testing.T) { testNoSniffExpectRequestBody(t, h2Mode) } + +func testNoSniffExpectRequestBody(t *testing.T, h2 bool) { + defer afterTest(t) + cst := newClientServerTest(t, h2, HandlerFunc(func(w ResponseWriter, r *Request) { + w.WriteHeader(StatusUnauthorized) + })) + defer cst.close() + + // Set ExpectContinueTimeout non-zero so RoundTrip won't try to write it. + cst.tr.ExpectContinueTimeout = 10 * time.Second + + req, err := NewRequest("POST", cst.ts.URL, testErrorReader{t}) + if err != nil { + t.Fatal(err) + } + req.ContentLength = 0 // so transport is tempted to sniff it + req.Header.Set("Expect", "100-continue") + res, err := cst.tr.RoundTrip(req) + if err != nil { + t.Fatal(err) + } + defer res.Body.Close() + if res.StatusCode != StatusUnauthorized { + t.Errorf("status code = %v; want %v", res.StatusCode, StatusUnauthorized) + } +} + +func TestServerUndeclaredTrailers_h1(t *testing.T) { testServerUndeclaredTrailers(t, h1Mode) } +func TestServerUndeclaredTrailers_h2(t *testing.T) { testServerUndeclaredTrailers(t, h2Mode) } +func testServerUndeclaredTrailers(t *testing.T, h2 bool) { + defer afterTest(t) + cst := newClientServerTest(t, h2, HandlerFunc(func(w ResponseWriter, r *Request) { + w.Header().Set("Foo", "Bar") + w.Header().Set("Trailer:Foo", "Baz") + w.(Flusher).Flush() + w.Header().Add("Trailer:Foo", "Baz2") + w.Header().Set("Trailer:Bar", "Quux") + })) + defer cst.close() + res, err := cst.c.Get(cst.ts.URL) + if err != nil { + t.Fatal(err) + } + if _, err := io.Copy(ioutil.Discard, res.Body); err != nil { + t.Fatal(err) + } + res.Body.Close() + delete(res.Header, "Date") + delete(res.Header, "Content-Type") + + if want := (Header{"Foo": {"Bar"}}); !reflect.DeepEqual(res.Header, want) { + t.Errorf("Header = %#v; want %#v", res.Header, want) + } + if want := (Header{"Foo": {"Baz", "Baz2"}, "Bar": {"Quux"}}); !reflect.DeepEqual(res.Trailer, want) { + t.Errorf("Trailer = %#v; want %#v", res.Trailer, want) + } +} diff --git a/libgo/go/net/http/cookie.go b/libgo/go/net/http/cookie.go index 648709dd99..5a67476cd4 100644 --- a/libgo/go/net/http/cookie.go +++ b/libgo/go/net/http/cookie.go @@ -6,7 +6,6 @@ package http import ( "bytes" - "fmt" "log" "net" "strconv" @@ -40,7 +39,11 @@ type Cookie struct { // readSetCookies parses all "Set-Cookie" values from // the header h and returns the successfully parsed Cookies. func readSetCookies(h Header) []*Cookie { - cookies := []*Cookie{} + cookieCount := len(h["Set-Cookie"]) + if cookieCount == 0 { + return []*Cookie{} + } + cookies := make([]*Cookie, 0, cookieCount) for _, line := range h["Set-Cookie"] { parts := strings.Split(strings.TrimSpace(line), ";") if len(parts) == 1 && parts[0] == "" { @@ -55,8 +58,8 @@ func readSetCookies(h Header) []*Cookie { if !isCookieNameValid(name) { continue } - value, success := parseCookieValue(value, true) - if !success { + value, ok := parseCookieValue(value, true) + if !ok { continue } c := &Cookie{ @@ -75,8 +78,8 @@ func readSetCookies(h Header) []*Cookie { attr, val = attr[:j], attr[j+1:] } lowerAttr := strings.ToLower(attr) - val, success = parseCookieValue(val, false) - if !success { + val, ok = parseCookieValue(val, false) + if !ok { c.Unparsed = append(c.Unparsed, parts[i]) continue } @@ -96,10 +99,9 @@ func readSetCookies(h Header) []*Cookie { break } if secs <= 0 { - c.MaxAge = -1 - } else { - c.MaxAge = secs + secs = -1 } + c.MaxAge = secs continue case "expires": c.RawExpires = val @@ -142,9 +144,13 @@ func (c *Cookie) String() string { return "" } var b bytes.Buffer - fmt.Fprintf(&b, "%s=%s", sanitizeCookieName(c.Name), sanitizeCookieValue(c.Value)) + b.WriteString(sanitizeCookieName(c.Name)) + b.WriteRune('=') + b.WriteString(sanitizeCookieValue(c.Value)) + if len(c.Path) > 0 { - fmt.Fprintf(&b, "; Path=%s", sanitizeCookiePath(c.Path)) + b.WriteString("; Path=") + b.WriteString(sanitizeCookiePath(c.Path)) } if len(c.Domain) > 0 { if validCookieDomain(c.Domain) { @@ -156,25 +162,31 @@ func (c *Cookie) String() string { if d[0] == '.' { d = d[1:] } - fmt.Fprintf(&b, "; Domain=%s", d) + b.WriteString("; Domain=") + b.WriteString(d) } else { - log.Printf("net/http: invalid Cookie.Domain %q; dropping domain attribute", - c.Domain) + log.Printf("net/http: invalid Cookie.Domain %q; dropping domain attribute", c.Domain) } } - if c.Expires.Unix() > 0 { - fmt.Fprintf(&b, "; Expires=%s", c.Expires.UTC().Format(TimeFormat)) + if validCookieExpires(c.Expires) { + b.WriteString("; Expires=") + b2 := b.Bytes() + b.Reset() + b.Write(c.Expires.UTC().AppendFormat(b2, TimeFormat)) } if c.MaxAge > 0 { - fmt.Fprintf(&b, "; Max-Age=%d", c.MaxAge) + b.WriteString("; Max-Age=") + b2 := b.Bytes() + b.Reset() + b.Write(strconv.AppendInt(b2, int64(c.MaxAge), 10)) } else if c.MaxAge < 0 { - fmt.Fprintf(&b, "; Max-Age=0") + b.WriteString("; Max-Age=0") } if c.HttpOnly { - fmt.Fprintf(&b, "; HttpOnly") + b.WriteString("; HttpOnly") } if c.Secure { - fmt.Fprintf(&b, "; Secure") + b.WriteString("; Secure") } return b.String() } @@ -184,12 +196,12 @@ func (c *Cookie) String() string { // // if filter isn't empty, only cookies of that name are returned func readCookies(h Header, filter string) []*Cookie { - cookies := []*Cookie{} lines, ok := h["Cookie"] if !ok { - return cookies + return []*Cookie{} } + cookies := []*Cookie{} for _, line := range lines { parts := strings.Split(strings.TrimSpace(line), ";") if len(parts) == 1 && parts[0] == "" { @@ -212,8 +224,8 @@ func readCookies(h Header, filter string) []*Cookie { if filter != "" && filter != name { continue } - val, success := parseCookieValue(val, true) - if !success { + val, ok := parseCookieValue(val, true) + if !ok { continue } cookies = append(cookies, &Cookie{Name: name, Value: val}) @@ -223,7 +235,7 @@ func readCookies(h Header, filter string) []*Cookie { return cookies } -// validCookieDomain returns wheter v is a valid cookie domain-value. +// validCookieDomain returns whether v is a valid cookie domain-value. func validCookieDomain(v string) bool { if isCookieDomainName(v) { return true @@ -234,6 +246,12 @@ func validCookieDomain(v string) bool { return false } +// validCookieExpires returns whether v is a valid cookie expires-value. +func validCookieExpires(t time.Time) bool { + // IETF RFC 6265 Section 5.1.1.5, the year must not be less than 1601 + return t.Year() >= 1601 +} + // isCookieDomainName returns whether s is a valid domain name or a valid // domain name with a leading dot '.'. It is almost a direct copy of // package net's isDomainName. diff --git a/libgo/go/net/http/cookie_test.go b/libgo/go/net/http/cookie_test.go index d474f31347..b3e54f8db3 100644 --- a/libgo/go/net/http/cookie_test.go +++ b/libgo/go/net/http/cookie_test.go @@ -1,4 +1,4 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -56,6 +56,15 @@ var writeSetCookiesTests = []struct { &Cookie{Name: "cookie-9", Value: "expiring", Expires: time.Unix(1257894000, 0)}, "cookie-9=expiring; Expires=Tue, 10 Nov 2009 23:00:00 GMT", }, + // According to IETF 6265 Section 5.1.1.5, the year cannot be less than 1601 + { + &Cookie{Name: "cookie-10", Value: "expiring-1601", Expires: time.Date(1601, 1, 1, 1, 1, 1, 1, time.UTC)}, + "cookie-10=expiring-1601; Expires=Mon, 01 Jan 1601 01:01:01 GMT", + }, + { + &Cookie{Name: "cookie-11", Value: "invalid-expiry", Expires: time.Date(1600, 1, 1, 1, 1, 1, 1, time.UTC)}, + "cookie-11=invalid-expiry", + }, // The "special" cookies have values containing commas or spaces which // are disallowed by RFC 6265 but are common in the wild. { @@ -426,3 +435,92 @@ func TestCookieSanitizePath(t *testing.T) { t.Errorf("Expected substring %q in log output. Got:\n%s", sub, got) } } + +func BenchmarkCookieString(b *testing.B) { + const wantCookieString = `cookie-9=i3e01nf61b6t23bvfmplnanol3; Path=/restricted/; Domain=example.com; Expires=Tue, 10 Nov 2009 23:00:00 GMT; Max-Age=3600` + c := &Cookie{ + Name: "cookie-9", + Value: "i3e01nf61b6t23bvfmplnanol3", + Expires: time.Unix(1257894000, 0), + Path: "/restricted/", + Domain: ".example.com", + MaxAge: 3600, + } + var benchmarkCookieString string + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + benchmarkCookieString = c.String() + } + if have, want := benchmarkCookieString, wantCookieString; have != want { + b.Fatalf("Have: %v Want: %v", have, want) + } +} + +func BenchmarkReadSetCookies(b *testing.B) { + header := Header{ + "Set-Cookie": { + "NID=99=YsDT5i3E-CXax-; expires=Wed, 23-Nov-2011 01:05:03 GMT; path=/; domain=.google.ch; HttpOnly", + ".ASPXAUTH=7E3AA; expires=Wed, 07-Mar-2012 14:25:06 GMT; path=/; HttpOnly", + }, + } + wantCookies := []*Cookie{ + { + Name: "NID", + Value: "99=YsDT5i3E-CXax-", + Path: "/", + Domain: ".google.ch", + HttpOnly: true, + Expires: time.Date(2011, 11, 23, 1, 5, 3, 0, time.UTC), + RawExpires: "Wed, 23-Nov-2011 01:05:03 GMT", + Raw: "NID=99=YsDT5i3E-CXax-; expires=Wed, 23-Nov-2011 01:05:03 GMT; path=/; domain=.google.ch; HttpOnly", + }, + { + Name: ".ASPXAUTH", + Value: "7E3AA", + Path: "/", + Expires: time.Date(2012, 3, 7, 14, 25, 6, 0, time.UTC), + RawExpires: "Wed, 07-Mar-2012 14:25:06 GMT", + HttpOnly: true, + Raw: ".ASPXAUTH=7E3AA; expires=Wed, 07-Mar-2012 14:25:06 GMT; path=/; HttpOnly", + }, + } + var c []*Cookie + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + c = readSetCookies(header) + } + if !reflect.DeepEqual(c, wantCookies) { + b.Fatalf("readSetCookies:\nhave: %s\nwant: %s\n", toJSON(c), toJSON(wantCookies)) + } +} + +func BenchmarkReadCookies(b *testing.B) { + header := Header{ + "Cookie": { + `de=; client_region=0; rpld1=0:hispeed.ch|20:che|21:zh|22:zurich|23:47.36|24:8.53|; rpld0=1:08|; backplane-channel=newspaper.com:1471; devicetype=0; osfam=0; rplmct=2; s_pers=%20s_vmonthnum%3D1472680800496%2526vn%253D1%7C1472680800496%3B%20s_nr%3D1471686767664-New%7C1474278767664%3B%20s_lv%3D1471686767669%7C1566294767669%3B%20s_lv_s%3DFirst%2520Visit%7C1471688567669%3B%20s_monthinvisit%3Dtrue%7C1471688567677%3B%20gvp_p5%3Dsports%253Ablog%253Aearly-lead%2520-%2520184693%2520-%252020160820%2520-%2520u-s%7C1471688567681%3B%20gvp_p51%3Dwp%2520-%2520sports%7C1471688567684%3B; s_sess=%20s_wp_ep%3Dhomepage%3B%20s._ref%3Dhttps%253A%252F%252Fwww.google.ch%252F%3B%20s_cc%3Dtrue%3B%20s_ppvl%3Dsports%25253Ablog%25253Aearly-lead%252520-%252520184693%252520-%25252020160820%252520-%252520u-lawyer%252C12%252C12%252C502%252C1231%252C502%252C1680%252C1050%252C2%252CP%3B%20s_ppv%3Dsports%25253Ablog%25253Aearly-lead%252520-%252520184693%252520-%25252020160820%252520-%252520u-s-lawyer%252C12%252C12%252C502%252C1231%252C502%252C1680%252C1050%252C2%252CP%3B%20s_dslv%3DFirst%2520Visit%3B%20s_sq%3Dwpninewspapercom%253D%252526pid%25253Dsports%2525253Ablog%2525253Aearly-lead%25252520-%25252520184693%25252520-%2525252020160820%25252520-%25252520u-s%252526pidt%25253D1%252526oid%25253Dhttps%2525253A%2525252F%2525252Fwww.newspaper.com%2525252F%2525253Fnid%2525253Dmenu_nav_homepage%252526ot%25253DA%3B`, + }, + } + wantCookies := []*Cookie{ + {Name: "de", Value: ""}, + {Name: "client_region", Value: "0"}, + {Name: "rpld1", Value: "0:hispeed.ch|20:che|21:zh|22:zurich|23:47.36|24:8.53|"}, + {Name: "rpld0", Value: "1:08|"}, + {Name: "backplane-channel", Value: "newspaper.com:1471"}, + {Name: "devicetype", Value: "0"}, + {Name: "osfam", Value: "0"}, + {Name: "rplmct", Value: "2"}, + {Name: "s_pers", Value: "%20s_vmonthnum%3D1472680800496%2526vn%253D1%7C1472680800496%3B%20s_nr%3D1471686767664-New%7C1474278767664%3B%20s_lv%3D1471686767669%7C1566294767669%3B%20s_lv_s%3DFirst%2520Visit%7C1471688567669%3B%20s_monthinvisit%3Dtrue%7C1471688567677%3B%20gvp_p5%3Dsports%253Ablog%253Aearly-lead%2520-%2520184693%2520-%252020160820%2520-%2520u-s%7C1471688567681%3B%20gvp_p51%3Dwp%2520-%2520sports%7C1471688567684%3B"}, + {Name: "s_sess", Value: "%20s_wp_ep%3Dhomepage%3B%20s._ref%3Dhttps%253A%252F%252Fwww.google.ch%252F%3B%20s_cc%3Dtrue%3B%20s_ppvl%3Dsports%25253Ablog%25253Aearly-lead%252520-%252520184693%252520-%25252020160820%252520-%252520u-lawyer%252C12%252C12%252C502%252C1231%252C502%252C1680%252C1050%252C2%252CP%3B%20s_ppv%3Dsports%25253Ablog%25253Aearly-lead%252520-%252520184693%252520-%25252020160820%252520-%252520u-s-lawyer%252C12%252C12%252C502%252C1231%252C502%252C1680%252C1050%252C2%252CP%3B%20s_dslv%3DFirst%2520Visit%3B%20s_sq%3Dwpninewspapercom%253D%252526pid%25253Dsports%2525253Ablog%2525253Aearly-lead%25252520-%25252520184693%25252520-%2525252020160820%25252520-%25252520u-s%252526pidt%25253D1%252526oid%25253Dhttps%2525253A%2525252F%2525252Fwww.newspaper.com%2525252F%2525253Fnid%2525253Dmenu_nav_homepage%252526ot%25253DA%3B"}, + } + var c []*Cookie + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + c = readCookies(header, "") + } + if !reflect.DeepEqual(c, wantCookies) { + b.Fatalf("readCookies:\nhave: %s\nwant: %s\n", toJSON(c), toJSON(wantCookies)) + } +} diff --git a/libgo/go/net/http/cookiejar/dummy_publicsuffix_test.go b/libgo/go/net/http/cookiejar/dummy_publicsuffix_test.go new file mode 100644 index 0000000000..748ec5cc43 --- /dev/null +++ b/libgo/go/net/http/cookiejar/dummy_publicsuffix_test.go @@ -0,0 +1,23 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package cookiejar_test + +import "net/http/cookiejar" + +type dummypsl struct { + List cookiejar.PublicSuffixList +} + +func (dummypsl) PublicSuffix(domain string) string { + return domain +} + +func (dummypsl) String() string { + return "dummy" +} + +var publicsuffix = dummypsl{} diff --git a/libgo/go/net/http/cookiejar/example_test.go b/libgo/go/net/http/cookiejar/example_test.go new file mode 100644 index 0000000000..19a57465ff --- /dev/null +++ b/libgo/go/net/http/cookiejar/example_test.go @@ -0,0 +1,67 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package cookiejar_test + +import ( + "fmt" + "log" + "net/http" + "net/http/cookiejar" + "net/http/httptest" + "net/url" +) + +func ExampleNew() { + // Start a server to give us cookies. + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if cookie, err := r.Cookie("Flavor"); err != nil { + http.SetCookie(w, &http.Cookie{Name: "Flavor", Value: "Chocolate Chip"}) + } else { + cookie.Value = "Oatmeal Raisin" + http.SetCookie(w, cookie) + } + })) + defer ts.Close() + + u, err := url.Parse(ts.URL) + if err != nil { + log.Fatal(err) + } + + // All users of cookiejar should import "golang.org/x/net/publicsuffix" + jar, err := cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List}) + if err != nil { + log.Fatal(err) + } + + client := &http.Client{ + Jar: jar, + } + + if _, err = client.Get(u.String()); err != nil { + log.Fatal(err) + } + + fmt.Println("After 1st request:") + for _, cookie := range jar.Cookies(u) { + fmt.Printf(" %s: %s\n", cookie.Name, cookie.Value) + } + + if _, err = client.Get(u.String()); err != nil { + log.Fatal(err) + } + + fmt.Println("After 2nd request:") + for _, cookie := range jar.Cookies(u) { + fmt.Printf(" %s: %s\n", cookie.Name, cookie.Value) + } + // Output: + // After 1st request: + // Flavor: Chocolate Chip + // After 2nd request: + // Flavor: Oatmeal Raisin +} diff --git a/libgo/go/net/http/cookiejar/jar.go b/libgo/go/net/http/cookiejar/jar.go index 0e0fac9286..f89abbcd18 100644 --- a/libgo/go/net/http/cookiejar/jar.go +++ b/libgo/go/net/http/cookiejar/jar.go @@ -107,7 +107,7 @@ type entry struct { seqNum uint64 } -// Id returns the domain;path;name triple of e as an id. +// id returns the domain;path;name triple of e as an id. func (e *entry) id() string { return fmt.Sprintf("%s;%s;%s", e.Domain, e.Path, e.Name) } @@ -147,24 +147,6 @@ func hasDotSuffix(s, suffix string) bool { return len(s) > len(suffix) && s[len(s)-len(suffix)-1] == '.' && s[len(s)-len(suffix):] == suffix } -// byPathLength is a []entry sort.Interface that sorts according to RFC 6265 -// section 5.4 point 2: by longest path and then by earliest creation time. -type byPathLength []entry - -func (s byPathLength) Len() int { return len(s) } - -func (s byPathLength) Less(i, j int) bool { - if len(s[i].Path) != len(s[j].Path) { - return len(s[i].Path) > len(s[j].Path) - } - if !s[i].Creation.Equal(s[j].Creation) { - return s[i].Creation.Before(s[j].Creation) - } - return s[i].seqNum < s[j].seqNum -} - -func (s byPathLength) Swap(i, j int) { s[i], s[j] = s[j], s[i] } - // Cookies implements the Cookies method of the http.CookieJar interface. // // It returns an empty slice if the URL's scheme is not HTTP or HTTPS. @@ -221,7 +203,18 @@ func (j *Jar) cookies(u *url.URL, now time.Time) (cookies []*http.Cookie) { } } - sort.Sort(byPathLength(selected)) + // sort according to RFC 6265 section 5.4 point 2: by longest + // path and then by earliest creation time. + sort.Slice(selected, func(i, j int) bool { + s := selected + if len(s[i].Path) != len(s[j].Path) { + return len(s[i].Path) > len(s[j].Path) + } + if !s[i].Creation.Equal(s[j].Creation) { + return s[i].Creation.Before(s[j].Creation) + } + return s[i].seqNum < s[j].seqNum + }) for _, e := range selected { cookies = append(cookies, &http.Cookie{Name: e.Name, Value: e.Value}) } diff --git a/libgo/go/net/http/cookiejar/punycode.go b/libgo/go/net/http/cookiejar/punycode.go index ea7ceb5ef3..a9cc666e8c 100644 --- a/libgo/go/net/http/cookiejar/punycode.go +++ b/libgo/go/net/http/cookiejar/punycode.go @@ -37,7 +37,7 @@ func encode(prefix, s string) (string, error) { delta, n, bias := int32(0), initialN, initialBias b, remaining := int32(0), int32(0) for _, r := range s { - if r < 0x80 { + if r < utf8.RuneSelf { b++ output = append(output, byte(r)) } else { diff --git a/libgo/go/net/http/doc.go b/libgo/go/net/http/doc.go index 4ec8272f62..7855feaaa9 100644 --- a/libgo/go/net/http/doc.go +++ b/libgo/go/net/http/doc.go @@ -44,7 +44,8 @@ For control over proxies, TLS configuration, keep-alives, compression, and other settings, create a Transport: tr := &http.Transport{ - TLSClientConfig: &tls.Config{RootCAs: pool}, + MaxIdleConns: 10, + IdleConnTimeout: 30 * time.Second, DisableCompression: true, } client := &http.Client{Transport: tr} @@ -77,19 +78,30 @@ custom Server: } log.Fatal(s.ListenAndServe()) -The http package has transparent support for the HTTP/2 protocol when -using HTTPS. Programs that must disable HTTP/2 can do so by setting -Transport.TLSNextProto (for clients) or Server.TLSNextProto (for -servers) to a non-nil, empty map. Alternatively, the following GODEBUG -environment variables are currently supported: +Starting with Go 1.6, the http package has transparent support for the +HTTP/2 protocol when using HTTPS. Programs that must disable HTTP/2 +can do so by setting Transport.TLSNextProto (for clients) or +Server.TLSNextProto (for servers) to a non-nil, empty +map. Alternatively, the following GODEBUG environment variables are +currently supported: GODEBUG=http2client=0 # disable HTTP/2 client support GODEBUG=http2server=0 # disable HTTP/2 server support GODEBUG=http2debug=1 # enable verbose HTTP/2 debug logs GODEBUG=http2debug=2 # ... even more verbose, with frame dumps -The GODEBUG variables are not covered by Go's API compatibility promise. -HTTP/2 support was added in Go 1.6. Please report any issues instead of -disabling HTTP/2 support: https://golang.org/s/http2bug +The GODEBUG variables are not covered by Go's API compatibility +promise. Please report any issues before disabling HTTP/2 +support: https://golang.org/s/http2bug + +The http package's Transport and Server both automatically enable +HTTP/2 support for simple configurations. To enable HTTP/2 for more +complex configurations, to use lower-level HTTP/2 features, or to use +a newer version of Go's http2 package, import "golang.org/x/net/http2" +directly and use its ConfigureTransport and/or ConfigureServer +functions. Manually configuring HTTP/2 via the golang.org/x/net/http2 +package takes precedence over the net/http package's built-in HTTP/2 +support. + */ package http diff --git a/libgo/go/net/http/export_test.go b/libgo/go/net/http/export_test.go index 52bccbdce3..b61f58b2db 100644 --- a/libgo/go/net/http/export_test.go +++ b/libgo/go/net/http/export_test.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -9,22 +9,23 @@ package http import ( "net" + "sort" "sync" "time" ) var ( - DefaultUserAgent = defaultUserAgent - NewLoggingConn = newLoggingConn - ExportAppendTime = appendTime - ExportRefererForURL = refererForURL - ExportServerNewConn = (*Server).newConn - ExportCloseWriteAndWait = (*conn).closeWriteAndWait - ExportErrRequestCanceled = errRequestCanceled - ExportErrRequestCanceledConn = errRequestCanceledConn - ExportServeFile = serveFile - ExportHttp2ConfigureTransport = http2ConfigureTransport - ExportHttp2ConfigureServer = http2ConfigureServer + DefaultUserAgent = defaultUserAgent + NewLoggingConn = newLoggingConn + ExportAppendTime = appendTime + ExportRefererForURL = refererForURL + ExportServerNewConn = (*Server).newConn + ExportCloseWriteAndWait = (*conn).closeWriteAndWait + ExportErrRequestCanceled = errRequestCanceled + ExportErrRequestCanceledConn = errRequestCanceledConn + ExportServeFile = serveFile + ExportScanETag = scanETag + ExportHttp2ConfigureServer = http2ConfigureServer ) func init() { @@ -35,9 +36,8 @@ func init() { } var ( - SetEnterRoundTripHook = hookSetter(&testHookEnterRoundTrip) - SetTestHookWaitResLoop = hookSetter(&testHookWaitResLoop) - SetRoundTripRetried = hookSetter(&testHookRoundTripRetried) + SetEnterRoundTripHook = hookSetter(&testHookEnterRoundTrip) + SetRoundTripRetried = hookSetter(&testHookRoundTripRetried) ) func SetReadLoopBeforeNextReadHook(f func()) { @@ -59,9 +59,9 @@ func SetTestHookServerServe(fn func(*Server, net.Listener)) { testHookServerServ func NewTestTimeoutHandler(handler Handler, ch <-chan time.Time) Handler { return &timeoutHandler{ - handler: handler, - timeout: func() <-chan time.Time { return ch }, - // (no body and nil cancelTimer) + handler: handler, + testTimeout: ch, + // (no body) } } @@ -81,21 +81,53 @@ func (t *Transport) IdleConnKeysForTesting() (keys []string) { keys = make([]string, 0) t.idleMu.Lock() defer t.idleMu.Unlock() - if t.idleConn == nil { - return - } for key := range t.idleConn { keys = append(keys, key.String()) } + sort.Strings(keys) return } -func (t *Transport) IdleConnCountForTesting(cacheKey string) int { +func (t *Transport) IdleConnKeyCountForTesting() int { t.idleMu.Lock() defer t.idleMu.Unlock() - if t.idleConn == nil { - return 0 + return len(t.idleConn) +} + +func (t *Transport) IdleConnStrsForTesting() []string { + var ret []string + t.idleMu.Lock() + defer t.idleMu.Unlock() + for _, conns := range t.idleConn { + for _, pc := range conns { + ret = append(ret, pc.conn.LocalAddr().String()+"/"+pc.conn.RemoteAddr().String()) + } } + sort.Strings(ret) + return ret +} + +func (t *Transport) IdleConnStrsForTesting_h2() []string { + var ret []string + noDialPool := t.h2transport.ConnPool.(http2noDialClientConnPool) + pool := noDialPool.http2clientConnPool + + pool.mu.Lock() + defer pool.mu.Unlock() + + for k, cc := range pool.conns { + for range cc { + ret = append(ret, k) + } + } + + sort.Strings(ret) + return ret +} + +func (t *Transport) IdleConnCountForTesting(cacheKey string) int { + t.idleMu.Lock() + defer t.idleMu.Unlock() for k, conns := range t.idleConn { if k.String() == cacheKey { return len(conns) @@ -144,3 +176,26 @@ func hookSetter(dst *func()) func(func()) { *dst = fn } } + +func ExportHttp2ConfigureTransport(t *Transport) error { + t2, err := http2configureTransport(t) + if err != nil { + return err + } + t.h2transport = t2 + return nil +} + +var Export_shouldCopyHeaderOnRedirect = shouldCopyHeaderOnRedirect + +func (s *Server) ExportAllConnsIdle() bool { + s.mu.Lock() + defer s.mu.Unlock() + for c := range s.activeConn { + st, ok := c.curState.Load().(ConnState) + if !ok || st != StateIdle { + return false + } + } + return true +} diff --git a/libgo/go/net/http/fcgi/fcgi.go b/libgo/go/net/http/fcgi/fcgi.go index 06bba0488a..5057d70098 100644 --- a/libgo/go/net/http/fcgi/fcgi.go +++ b/libgo/go/net/http/fcgi/fcgi.go @@ -3,8 +3,12 @@ // license that can be found in the LICENSE file. // Package fcgi implements the FastCGI protocol. +// +// The protocol is not an official standard and the original +// documentation is no longer online. See the Internet Archive's +// mirror at: https://web.archive.org/web/20150420080736/http://www.fastcgi.com/drupal/node/6?q=node/22 +// // Currently only the responder role is supported. -// The protocol is defined at http://www.fastcgi.com/drupal/node/6?q=node/22 package fcgi // This file defines the raw protocol and some utilities used by the child and @@ -58,8 +62,6 @@ const ( statusUnknownRole ) -const headerLen = 8 - type header struct { Version uint8 Type recType @@ -158,11 +160,6 @@ func (c *conn) writeRecord(recType recType, reqId uint16, b []byte) error { return err } -func (c *conn) writeBeginRequest(reqId uint16, role uint16, flags uint8) error { - b := [8]byte{byte(role >> 8), byte(role), flags} - return c.writeRecord(typeBeginRequest, reqId, b[:]) -} - func (c *conn) writeEndRequest(reqId uint16, appStatus int, protocolStatus uint8) error { b := make([]byte, 8) binary.BigEndian.PutUint32(b, uint32(appStatus)) diff --git a/libgo/go/net/http/filetransport.go b/libgo/go/net/http/filetransport.go index 821787e0c4..32126d7ec0 100644 --- a/libgo/go/net/http/filetransport.go +++ b/libgo/go/net/http/filetransport.go @@ -33,7 +33,7 @@ func NewFileTransport(fs FileSystem) RoundTripper { func (t fileTransport) RoundTrip(req *Request) (resp *Response, err error) { // We start ServeHTTP in a goroutine, which may take a long - // time if the file is large. The newPopulateResponseWriter + // time if the file is large. The newPopulateResponseWriter // call returns a channel which either ServeHTTP or finish() // sends our *Response on, once the *Response itself has been // populated (even if the body itself is still being diff --git a/libgo/go/net/http/fs.go b/libgo/go/net/http/fs.go index f61c138c1d..bf63bb5441 100644 --- a/libgo/go/net/http/fs.go +++ b/libgo/go/net/http/fs.go @@ -34,7 +34,7 @@ import ( type Dir string func (d Dir) Open(name string) (File, error) { - if filepath.Separator != '/' && strings.IndexRune(name, filepath.Separator) >= 0 || + if filepath.Separator != '/' && strings.ContainsRune(name, filepath.Separator) || strings.Contains(name, "\x00") { return nil, errors.New("http: invalid character in file path") } @@ -77,7 +77,7 @@ func dirList(w ResponseWriter, f File) { Error(w, "Error reading directory", StatusInternalServerError) return } - sort.Sort(byName(dirs)) + sort.Slice(dirs, func(i, j int) bool { return dirs[i].Name() < dirs[j].Name() }) w.Header().Set("Content-Type", "text/html; charset=utf-8") fmt.Fprintf(w, "<pre>\n") @@ -96,9 +96,10 @@ func dirList(w ResponseWriter, f File) { } // ServeContent replies to the request using the content in the -// provided ReadSeeker. The main benefit of ServeContent over io.Copy +// provided ReadSeeker. The main benefit of ServeContent over io.Copy // is that it handles Range requests properly, sets the MIME type, and -// handles If-Modified-Since requests. +// handles If-Match, If-Unmodified-Since, If-None-Match, If-Modified-Since, +// and If-Range requests. // // If the response's Content-Type header is not set, ServeContent // first tries to deduce the type from name's file extension and, @@ -108,24 +109,24 @@ func dirList(w ResponseWriter, f File) { // never sent in the response. // // If modtime is not the zero time or Unix epoch, ServeContent -// includes it in a Last-Modified header in the response. If the +// includes it in a Last-Modified header in the response. If the // request includes an If-Modified-Since header, ServeContent uses // modtime to decide whether the content needs to be sent at all. // // The content's Seek method must work: ServeContent uses // a seek to the end of the content to determine its size. // -// If the caller has set w's ETag header, ServeContent uses it to -// handle requests using If-Range and If-None-Match. +// If the caller has set w's ETag header formatted per RFC 7232, section 2.3, +// ServeContent uses it to handle requests using If-Match, If-None-Match, or If-Range. // // Note that *os.File implements the io.ReadSeeker interface. func ServeContent(w ResponseWriter, req *Request, name string, modtime time.Time, content io.ReadSeeker) { sizeFunc := func() (int64, error) { - size, err := content.Seek(0, os.SEEK_END) + size, err := content.Seek(0, io.SeekEnd) if err != nil { return 0, errSeeker } - _, err = content.Seek(0, os.SEEK_SET) + _, err = content.Seek(0, io.SeekStart) if err != nil { return 0, errSeeker } @@ -140,15 +141,17 @@ func ServeContent(w ResponseWriter, req *Request, name string, modtime time.Time // users. var errSeeker = errors.New("seeker can't seek") +// errNoOverlap is returned by serveContent's parseRange if first-byte-pos of +// all of the byte-range-spec values is greater than the content size. +var errNoOverlap = errors.New("invalid range: failed to overlap") + // if name is empty, filename is unknown. (used for mime type, before sniffing) // if modtime.IsZero(), modtime is unknown. // content must be seeked to the beginning of the file. // The sizeFunc is called at most once. Its error, if any, is sent in the HTTP response. func serveContent(w ResponseWriter, r *Request, name string, modtime time.Time, sizeFunc func() (int64, error), content io.ReadSeeker) { - if checkLastModified(w, r, modtime) { - return - } - rangeReq, done := checkETag(w, r, modtime) + setLastModified(w, modtime) + done, rangeReq := checkPreconditions(w, r, modtime) if done { return } @@ -166,7 +169,7 @@ func serveContent(w ResponseWriter, r *Request, name string, modtime time.Time, var buf [sniffLen]byte n, _ := io.ReadFull(content, buf[:]) ctype = DetectContentType(buf[:n]) - _, err := content.Seek(0, os.SEEK_SET) // rewind to output whole file + _, err := content.Seek(0, io.SeekStart) // rewind to output whole file if err != nil { Error(w, "seeker can't seek", StatusInternalServerError) return @@ -189,6 +192,9 @@ func serveContent(w ResponseWriter, r *Request, name string, modtime time.Time, if size >= 0 { ranges, err := parseRange(rangeReq, size) if err != nil { + if err == errNoOverlap { + w.Header().Set("Content-Range", fmt.Sprintf("bytes */%d", size)) + } Error(w, err.Error(), StatusRequestedRangeNotSatisfiable) return } @@ -196,7 +202,7 @@ func serveContent(w ResponseWriter, r *Request, name string, modtime time.Time, // The total number of bytes in all the ranges // is larger than the size of the file by // itself, so this is probably an attack, or a - // dumb client. Ignore the range request. + // dumb client. Ignore the range request. ranges = nil } switch { @@ -213,7 +219,7 @@ func serveContent(w ResponseWriter, r *Request, name string, modtime time.Time, // A response to a request for a single range MUST NOT // be sent using the multipart/byteranges media type." ra := ranges[0] - if _, err := content.Seek(ra.start, os.SEEK_SET); err != nil { + if _, err := content.Seek(ra.start, io.SeekStart); err != nil { Error(w, err.Error(), StatusRequestedRangeNotSatisfiable) return } @@ -236,7 +242,7 @@ func serveContent(w ResponseWriter, r *Request, name string, modtime time.Time, pw.CloseWithError(err) return } - if _, err := content.Seek(ra.start, os.SEEK_SET); err != nil { + if _, err := content.Seek(ra.start, io.SeekStart); err != nil { pw.CloseWithError(err) return } @@ -263,90 +269,245 @@ func serveContent(w ResponseWriter, r *Request, name string, modtime time.Time, } } -var unixEpochTime = time.Unix(0, 0) - -// modtime is the modification time of the resource to be served, or IsZero(). -// return value is whether this request is now complete. -func checkLastModified(w ResponseWriter, r *Request, modtime time.Time) bool { - if modtime.IsZero() || modtime.Equal(unixEpochTime) { - // If the file doesn't have a modtime (IsZero), or the modtime - // is obviously garbage (Unix time == 0), then ignore modtimes - // and don't process the If-Modified-Since header. - return false +// scanETag determines if a syntactically valid ETag is present at s. If so, +// the ETag and remaining text after consuming ETag is returned. Otherwise, +// it returns "", "". +func scanETag(s string) (etag string, remain string) { + s = textproto.TrimString(s) + start := 0 + if strings.HasPrefix(s, "W/") { + start = 2 + } + if len(s[start:]) < 2 || s[start] != '"' { + return "", "" + } + // ETag is either W/"text" or "text". + // See RFC 7232 2.3. + for i := start + 1; i < len(s); i++ { + c := s[i] + switch { + // Character values allowed in ETags. + case c == 0x21 || c >= 0x23 && c <= 0x7E || c >= 0x80: + case c == '"': + return string(s[:i+1]), s[i+1:] + default: + break + } } + return "", "" +} - // The Date-Modified header truncates sub-second precision, so - // use mtime < t+1s instead of mtime <= t to check for unmodified. - if t, err := time.Parse(TimeFormat, r.Header.Get("If-Modified-Since")); err == nil && modtime.Before(t.Add(1*time.Second)) { - h := w.Header() - delete(h, "Content-Type") - delete(h, "Content-Length") - w.WriteHeader(StatusNotModified) - return true - } - w.Header().Set("Last-Modified", modtime.UTC().Format(TimeFormat)) - return false +// etagStrongMatch reports whether a and b match using strong ETag comparison. +// Assumes a and b are valid ETags. +func etagStrongMatch(a, b string) bool { + return a == b && a != "" && a[0] == '"' } -// checkETag implements If-None-Match and If-Range checks. -// -// The ETag or modtime must have been previously set in the -// ResponseWriter's headers. The modtime is only compared at second -// granularity and may be the zero value to mean unknown. -// -// The return value is the effective request "Range" header to use and -// whether this request is now considered done. -func checkETag(w ResponseWriter, r *Request, modtime time.Time) (rangeReq string, done bool) { - etag := w.Header().get("Etag") - rangeReq = r.Header.get("Range") - - // Invalidate the range request if the entity doesn't match the one - // the client was expecting. - // "If-Range: version" means "ignore the Range: header unless version matches the - // current file." - // We only support ETag versions. - // The caller must have set the ETag on the response already. - if ir := r.Header.get("If-Range"); ir != "" && ir != etag { - // The If-Range value is typically the ETag value, but it may also be - // the modtime date. See golang.org/issue/8367. - timeMatches := false - if !modtime.IsZero() { - if t, err := ParseTime(ir); err == nil && t.Unix() == modtime.Unix() { - timeMatches = true - } +// etagWeakMatch reports whether a and b match using weak ETag comparison. +// Assumes a and b are valid ETags. +func etagWeakMatch(a, b string) bool { + return strings.TrimPrefix(a, "W/") == strings.TrimPrefix(b, "W/") +} + +// condResult is the result of an HTTP request precondition check. +// See https://tools.ietf.org/html/rfc7232 section 3. +type condResult int + +const ( + condNone condResult = iota + condTrue + condFalse +) + +func checkIfMatch(w ResponseWriter, r *Request) condResult { + im := r.Header.Get("If-Match") + if im == "" { + return condNone + } + for { + im = textproto.TrimString(im) + if len(im) == 0 { + break + } + if im[0] == ',' { + im = im[1:] + continue + } + if im[0] == '*' { + return condTrue + } + etag, remain := scanETag(im) + if etag == "" { + break } - if !timeMatches { - rangeReq = "" + if etagStrongMatch(etag, w.Header().get("Etag")) { + return condTrue } + im = remain } - if inm := r.Header.get("If-None-Match"); inm != "" { - // Must know ETag. + return condFalse +} + +func checkIfUnmodifiedSince(w ResponseWriter, r *Request, modtime time.Time) condResult { + ius := r.Header.Get("If-Unmodified-Since") + if ius == "" || isZeroTime(modtime) { + return condNone + } + if t, err := ParseTime(ius); err == nil { + // The Date-Modified header truncates sub-second precision, so + // use mtime < t+1s instead of mtime <= t to check for unmodified. + if modtime.Before(t.Add(1 * time.Second)) { + return condTrue + } + return condFalse + } + return condNone +} + +func checkIfNoneMatch(w ResponseWriter, r *Request) condResult { + inm := r.Header.get("If-None-Match") + if inm == "" { + return condNone + } + buf := inm + for { + buf = textproto.TrimString(buf) + if len(buf) == 0 { + break + } + if buf[0] == ',' { + buf = buf[1:] + } + if buf[0] == '*' { + return condFalse + } + etag, remain := scanETag(buf) if etag == "" { - return rangeReq, false + break + } + if etagWeakMatch(etag, w.Header().get("Etag")) { + return condFalse } + buf = remain + } + return condTrue +} - // TODO(bradfitz): non-GET/HEAD requests require more work: - // sending a different status code on matches, and - // also can't use weak cache validators (those with a "W/ - // prefix). But most users of ServeContent will be using - // it on GET or HEAD, so only support those for now. - if r.Method != "GET" && r.Method != "HEAD" { - return rangeReq, false +func checkIfModifiedSince(w ResponseWriter, r *Request, modtime time.Time) condResult { + if r.Method != "GET" && r.Method != "HEAD" { + return condNone + } + ims := r.Header.Get("If-Modified-Since") + if ims == "" || isZeroTime(modtime) { + return condNone + } + t, err := ParseTime(ims) + if err != nil { + return condNone + } + // The Date-Modified header truncates sub-second precision, so + // use mtime < t+1s instead of mtime <= t to check for unmodified. + if modtime.Before(t.Add(1 * time.Second)) { + return condFalse + } + return condTrue +} + +func checkIfRange(w ResponseWriter, r *Request, modtime time.Time) condResult { + if r.Method != "GET" { + return condNone + } + ir := r.Header.get("If-Range") + if ir == "" { + return condNone + } + etag, _ := scanETag(ir) + if etag != "" { + if etagStrongMatch(etag, w.Header().Get("Etag")) { + return condTrue + } else { + return condFalse } + } + // The If-Range value is typically the ETag value, but it may also be + // the modtime date. See golang.org/issue/8367. + if modtime.IsZero() { + return condFalse + } + t, err := ParseTime(ir) + if err != nil { + return condFalse + } + if t.Unix() == modtime.Unix() { + return condTrue + } + return condFalse +} + +var unixEpochTime = time.Unix(0, 0) + +// isZeroTime reports whether t is obviously unspecified (either zero or Unix()=0). +func isZeroTime(t time.Time) bool { + return t.IsZero() || t.Equal(unixEpochTime) +} - // TODO(bradfitz): deal with comma-separated or multiple-valued - // list of If-None-match values. For now just handle the common - // case of a single item. - if inm == etag || inm == "*" { - h := w.Header() - delete(h, "Content-Type") - delete(h, "Content-Length") - w.WriteHeader(StatusNotModified) - return "", true +func setLastModified(w ResponseWriter, modtime time.Time) { + if !isZeroTime(modtime) { + w.Header().Set("Last-Modified", modtime.UTC().Format(TimeFormat)) + } +} + +func writeNotModified(w ResponseWriter) { + // RFC 7232 section 4.1: + // a sender SHOULD NOT generate representation metadata other than the + // above listed fields unless said metadata exists for the purpose of + // guiding cache updates (e.g., Last-Modified might be useful if the + // response does not have an ETag field). + h := w.Header() + delete(h, "Content-Type") + delete(h, "Content-Length") + if h.Get("Etag") != "" { + delete(h, "Last-Modified") + } + w.WriteHeader(StatusNotModified) +} + +// checkPreconditions evaluates request preconditions and reports whether a precondition +// resulted in sending StatusNotModified or StatusPreconditionFailed. +func checkPreconditions(w ResponseWriter, r *Request, modtime time.Time) (done bool, rangeHeader string) { + // This function carefully follows RFC 7232 section 6. + ch := checkIfMatch(w, r) + if ch == condNone { + ch = checkIfUnmodifiedSince(w, r, modtime) + } + if ch == condFalse { + w.WriteHeader(StatusPreconditionFailed) + return true, "" + } + switch checkIfNoneMatch(w, r) { + case condFalse: + if r.Method == "GET" || r.Method == "HEAD" { + writeNotModified(w) + return true, "" + } else { + w.WriteHeader(StatusPreconditionFailed) + return true, "" + } + case condNone: + if checkIfModifiedSince(w, r, modtime) == condFalse { + writeNotModified(w) + return true, "" + } + } + + rangeHeader = r.Header.get("Range") + if rangeHeader != "" { + if checkIfRange(w, r, modtime) == condFalse { + rangeHeader = "" } } - return rangeReq, false + return false, rangeHeader } // name is '/'-separated, not filepath.Separator. @@ -393,6 +554,15 @@ func serveFile(w ResponseWriter, r *Request, fs FileSystem, name string, redirec } } + // redirect if the directory name doesn't end in a slash + if d.IsDir() { + url := r.URL.Path + if url[len(url)-1] != '/' { + localRedirect(w, r, path.Base(url)+"/") + return + } + } + // use contents of index.html for directory, if present if d.IsDir() { index := strings.TrimSuffix(name, "/") + indexPage @@ -410,9 +580,11 @@ func serveFile(w ResponseWriter, r *Request, fs FileSystem, name string, redirec // Still a directory? (we didn't find an index.html file) if d.IsDir() { - if checkLastModified(w, r, d.ModTime()) { + if checkIfModifiedSince(w, r, d.ModTime()) == condFalse { + writeNotModified(w) return } + w.Header().Set("Last-Modified", d.ModTime().UTC().Format(TimeFormat)) dirList(w, f) return } @@ -451,7 +623,7 @@ func localRedirect(w ResponseWriter, r *Request, newPath string) { // ServeFile replies to the request with the contents of the named // file or directory. // -// If the provided file or direcory name is a relative path, it is +// If the provided file or directory name is a relative path, it is // interpreted relative to the current directory and may ascend to parent // directories. If the provided name is constructed from user input, it // should be sanitized before calling ServeFile. As a precaution, ServeFile @@ -534,6 +706,7 @@ func (r httpRange) mimeHeader(contentType string, size int64) textproto.MIMEHead } // parseRange parses a Range header string as per RFC 2616. +// errNoOverlap is returned if none of the ranges overlap. func parseRange(s string, size int64) ([]httpRange, error) { if s == "" { return nil, nil // header not present @@ -543,6 +716,7 @@ func parseRange(s string, size int64) ([]httpRange, error) { return nil, errors.New("invalid range") } var ranges []httpRange + noOverlap := false for _, ra := range strings.Split(s[len(b):], ",") { ra = strings.TrimSpace(ra) if ra == "" { @@ -568,9 +742,15 @@ func parseRange(s string, size int64) ([]httpRange, error) { r.length = size - r.start } else { i, err := strconv.ParseInt(start, 10, 64) - if err != nil || i >= size || i < 0 { + if err != nil || i < 0 { return nil, errors.New("invalid range") } + if i >= size { + // If the range begins after the size of the content, + // then it does not overlap. + noOverlap = true + continue + } r.start = i if end == "" { // If no end is specified, range extends to end of the file. @@ -588,6 +768,10 @@ func parseRange(s string, size int64) ([]httpRange, error) { } ranges = append(ranges, r) } + if noOverlap && len(ranges) == 0 { + // The specified ranges did not overlap with the content. + return nil, errNoOverlap + } return ranges, nil } @@ -619,9 +803,3 @@ func sumRangesSize(ranges []httpRange) (size int64) { } return } - -type byName []os.FileInfo - -func (s byName) Len() int { return len(s) } -func (s byName) Less(i, j int) bool { return s[i].Name() < s[j].Name() } -func (s byName) Swap(i, j int) { s[i], s[j] = s[j], s[i] } diff --git a/libgo/go/net/http/fs_test.go b/libgo/go/net/http/fs_test.go index cf5b63c9f7..bba5682115 100644 --- a/libgo/go/net/http/fs_test.go +++ b/libgo/go/net/http/fs_test.go @@ -24,7 +24,6 @@ import ( "reflect" "regexp" "runtime" - "strconv" "strings" "testing" "time" @@ -39,8 +38,6 @@ type wantRange struct { start, end int64 // range [start,end) } -var itoa = strconv.Itoa - var ServeFileRangeTests = []struct { r string code int @@ -71,6 +68,7 @@ var ServeFileRangeTests = []struct { } func TestServeFile(t *testing.T) { + setParallel(t) defer afterTest(t) ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { ServeFile(w, r, "testdata/file") @@ -277,6 +275,7 @@ func TestFileServerEscapesNames(t *testing.T) { {`"'<>&`, `<a href="%22%27%3C%3E&">"'<>&</a>`}, {`?foo=bar#baz`, `<a href="%3Ffoo=bar%23baz">?foo=bar#baz</a>`}, {`<combo>?foo`, `<a href="%3Ccombo%3E%3Ffoo"><combo>?foo</a>`}, + {`foo:bar`, `<a href="./foo:bar">foo:bar</a>`}, } // We put each test file in its own directory in the fakeFS so we can look at it in isolation. @@ -508,6 +507,24 @@ func TestServeFileFromCWD(t *testing.T) { } } +// Issue 13996 +func TestServeDirWithoutTrailingSlash(t *testing.T) { + e := "/testdata/" + defer afterTest(t) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + ServeFile(w, r, ".") + })) + defer ts.Close() + r, err := Get(ts.URL + "/testdata") + if err != nil { + t.Fatal(err) + } + r.Body.Close() + if g := r.Request.URL.Path; g != e { + t.Errorf("got %s, want %s", g, e) + } +} + // Tests that ServeFile doesn't add a Content-Length if a Content-Encoding is // specified. func TestServeFileWithContentEncoding_h1(t *testing.T) { testServeFileWithContentEncoding(t, h1Mode) } @@ -750,6 +767,7 @@ func TestServeContent(t *testing.T) { reqHeader map[string]string wantLastMod string wantContentType string + wantContentRange string wantStatus int } htmlModTime := mustStat(t, "testdata/index.html").ModTime() @@ -767,8 +785,9 @@ func TestServeContent(t *testing.T) { wantStatus: 200, }, "not_modified_modtime": { - file: "testdata/style.css", - modtime: htmlModTime, + file: "testdata/style.css", + serveETag: `"foo"`, // Last-Modified sent only when no ETag + modtime: htmlModTime, reqHeader: map[string]string{ "If-Modified-Since": htmlModTime.UTC().Format(TimeFormat), }, @@ -777,6 +796,7 @@ func TestServeContent(t *testing.T) { "not_modified_modtime_with_contenttype": { file: "testdata/style.css", serveContentType: "text/css", // explicit content type + serveETag: `"foo"`, // Last-Modified sent only when no ETag modtime: htmlModTime, reqHeader: map[string]string{ "If-Modified-Since": htmlModTime.UTC().Format(TimeFormat), @@ -793,21 +813,62 @@ func TestServeContent(t *testing.T) { }, "not_modified_etag_no_seek": { content: panicOnSeek{nil}, // should never be called - serveETag: `"foo"`, + serveETag: `W/"foo"`, // If-None-Match uses weak ETag comparison reqHeader: map[string]string{ - "If-None-Match": `"foo"`, + "If-None-Match": `"baz", W/"foo"`, }, wantStatus: 304, }, + "if_none_match_mismatch": { + file: "testdata/style.css", + serveETag: `"foo"`, + reqHeader: map[string]string{ + "If-None-Match": `"Foo"`, + }, + wantStatus: 200, + wantContentType: "text/css; charset=utf-8", + }, "range_good": { file: "testdata/style.css", serveETag: `"A"`, reqHeader: map[string]string{ "Range": "bytes=0-4", }, - wantStatus: StatusPartialContent, + wantStatus: StatusPartialContent, + wantContentType: "text/css; charset=utf-8", + wantContentRange: "bytes 0-4/8", + }, + "range_match": { + file: "testdata/style.css", + serveETag: `"A"`, + reqHeader: map[string]string{ + "Range": "bytes=0-4", + "If-Range": `"A"`, + }, + wantStatus: StatusPartialContent, + wantContentType: "text/css; charset=utf-8", + wantContentRange: "bytes 0-4/8", + }, + "range_match_weak_etag": { + file: "testdata/style.css", + serveETag: `W/"A"`, + reqHeader: map[string]string{ + "Range": "bytes=0-4", + "If-Range": `W/"A"`, + }, + wantStatus: 200, wantContentType: "text/css; charset=utf-8", }, + "range_no_overlap": { + file: "testdata/style.css", + serveETag: `"A"`, + reqHeader: map[string]string{ + "Range": "bytes=10-20", + }, + wantStatus: StatusRequestedRangeNotSatisfiable, + wantContentType: "text/plain; charset=utf-8", + wantContentRange: "bytes */8", + }, // An If-Range resource for entity "A", but entity "B" is now current. // The Range request should be ignored. "range_no_match": { @@ -827,9 +888,10 @@ func TestServeContent(t *testing.T) { "Range": "bytes=0-4", "If-Range": "Wed, 25 Jun 2014 17:12:18 GMT", }, - wantStatus: StatusPartialContent, - wantContentType: "text/css; charset=utf-8", - wantLastMod: "Wed, 25 Jun 2014 17:12:18 GMT", + wantStatus: StatusPartialContent, + wantContentType: "text/css; charset=utf-8", + wantContentRange: "bytes 0-4/8", + wantLastMod: "Wed, 25 Jun 2014 17:12:18 GMT", }, "range_with_modtime_nanos": { file: "testdata/style.css", @@ -838,9 +900,10 @@ func TestServeContent(t *testing.T) { "Range": "bytes=0-4", "If-Range": "Wed, 25 Jun 2014 17:12:18 GMT", }, - wantStatus: StatusPartialContent, - wantContentType: "text/css; charset=utf-8", - wantLastMod: "Wed, 25 Jun 2014 17:12:18 GMT", + wantStatus: StatusPartialContent, + wantContentType: "text/css; charset=utf-8", + wantContentRange: "bytes 0-4/8", + wantLastMod: "Wed, 25 Jun 2014 17:12:18 GMT", }, "unix_zero_modtime": { content: strings.NewReader("<html>foo"), @@ -848,6 +911,62 @@ func TestServeContent(t *testing.T) { wantStatus: StatusOK, wantContentType: "text/html; charset=utf-8", }, + "ifmatch_matches": { + file: "testdata/style.css", + serveETag: `"A"`, + reqHeader: map[string]string{ + "If-Match": `"Z", "A"`, + }, + wantStatus: 200, + wantContentType: "text/css; charset=utf-8", + }, + "ifmatch_star": { + file: "testdata/style.css", + serveETag: `"A"`, + reqHeader: map[string]string{ + "If-Match": `*`, + }, + wantStatus: 200, + wantContentType: "text/css; charset=utf-8", + }, + "ifmatch_failed": { + file: "testdata/style.css", + serveETag: `"A"`, + reqHeader: map[string]string{ + "If-Match": `"B"`, + }, + wantStatus: 412, + wantContentType: "text/plain; charset=utf-8", + }, + "ifmatch_fails_on_weak_etag": { + file: "testdata/style.css", + serveETag: `W/"A"`, + reqHeader: map[string]string{ + "If-Match": `W/"A"`, + }, + wantStatus: 412, + wantContentType: "text/plain; charset=utf-8", + }, + "if_unmodified_since_true": { + file: "testdata/style.css", + modtime: htmlModTime, + reqHeader: map[string]string{ + "If-Unmodified-Since": htmlModTime.UTC().Format(TimeFormat), + }, + wantStatus: 200, + wantContentType: "text/css; charset=utf-8", + wantLastMod: htmlModTime.UTC().Format(TimeFormat), + }, + "if_unmodified_since_false": { + file: "testdata/style.css", + modtime: htmlModTime, + reqHeader: map[string]string{ + "If-Unmodified-Since": htmlModTime.Add(-2 * time.Second).UTC().Format(TimeFormat), + }, + wantStatus: 412, + wantContentType: "text/plain; charset=utf-8", + wantLastMod: htmlModTime.UTC().Format(TimeFormat), + }, } for testName, tt := range tests { var content io.ReadSeeker @@ -888,6 +1007,9 @@ func TestServeContent(t *testing.T) { if g, e := res.Header.Get("Content-Type"), tt.wantContentType; g != e { t.Errorf("test %q: content-type = %q, want %q", testName, g, e) } + if g, e := res.Header.Get("Content-Range"), tt.wantContentRange; g != e { + t.Errorf("test %q: content-range = %q, want %q", testName, g, e) + } if g, e := res.Header.Get("Last-Modified"), tt.wantLastMod; g != e { t.Errorf("test %q: last-modified = %q, want %q", testName, g, e) } @@ -943,6 +1065,7 @@ func TestServeContentErrorMessages(t *testing.T) { // verifies that sendfile is being used on Linux func TestLinuxSendfile(t *testing.T) { + setParallel(t) defer afterTest(t) if runtime.GOOS != "linux" { t.Skip("skipping; linux-only test") @@ -963,10 +1086,12 @@ func TestLinuxSendfile(t *testing.T) { syscalls := "sendfile,sendfile64" switch runtime.GOARCH { - case "mips64", "mips64le", "alpha": - // mips64 strace doesn't support sendfile64 and will error out - // if we specify that with `-e trace='. + case "mips64", "mips64le", "s390x", "alpha": + // strace on the above platforms doesn't support sendfile64 + // and will error out if we specify that with `-e trace='. syscalls = "sendfile" + case "mips64": + t.Skip("TODO: update this test to be robust against various versions of strace on mips64. See golang.org/issue/33430") } var buf bytes.Buffer @@ -993,10 +1118,9 @@ func TestLinuxSendfile(t *testing.T) { Post(fmt.Sprintf("http://%s/quit", ln.Addr()), "", nil) child.Wait() - rx := regexp.MustCompile(`sendfile(64)?\(\d+,\s*\d+,\s*NULL,\s*\d+\)\s*=\s*\d+\s*\n`) - rxResume := regexp.MustCompile(`<\.\.\. sendfile(64)? resumed> \)\s*=\s*\d+\s*\n`) + rx := regexp.MustCompile(`sendfile(64)?\(\d+,\s*\d+,\s*NULL,\s*\d+`) out := buf.String() - if !rx.MatchString(out) && !rxResume.MatchString(out) { + if !rx.MatchString(out) { t.Errorf("no sendfile system call found in:\n%s", out) } } @@ -1075,3 +1199,26 @@ func (d fileServerCleanPathDir) Open(path string) (File, error) { } type panicOnSeek struct{ io.ReadSeeker } + +func Test_scanETag(t *testing.T) { + tests := []struct { + in string + wantETag string + wantRemain string + }{ + {`W/"etag-1"`, `W/"etag-1"`, ""}, + {`"etag-2"`, `"etag-2"`, ""}, + {`"etag-1", "etag-2"`, `"etag-1"`, `, "etag-2"`}, + {"", "", ""}, + {"", "", ""}, + {"W/", "", ""}, + {`W/"truc`, "", ""}, + {`w/"case-sensitive"`, "", ""}, + } + for _, test := range tests { + etag, remain := ExportScanETag(test.in) + if etag != test.wantETag || remain != test.wantRemain { + t.Errorf("scanETag(%q)=%q %q, want %q %q", test.in, etag, remain, test.wantETag, test.wantRemain) + } + } +} diff --git a/libgo/go/net/http/h2_bundle.go b/libgo/go/net/http/h2_bundle.go index 4e19b3e71f..25fdf09d92 100644 --- a/libgo/go/net/http/h2_bundle.go +++ b/libgo/go/net/http/h2_bundle.go @@ -1,5 +1,5 @@ // Code generated by golang.org/x/tools/cmd/bundle. -//go:generate bundle -o h2_bundle.go -prefix http2 -import golang.org/x/net/http2/hpack=internal/golang.org/x/net/http2/hpack golang.org/x/net/http2 +//go:generate bundle -o h2_bundle.go -prefix http2 -underscore golang.org/x/net/http2 // Package http2 implements the HTTP/2 protocol. // @@ -20,15 +20,18 @@ import ( "bufio" "bytes" "compress/gzip" + "context" + "crypto/rand" "crypto/tls" "encoding/binary" "errors" "fmt" - "internal/golang.org/x/net/http2/hpack" "io" "io/ioutil" "log" + "math" "net" + "net/http/httptrace" "net/textproto" "net/url" "os" @@ -39,6 +42,10 @@ import ( "strings" "sync" "time" + + "golang_org/x/net/http2/hpack" + "golang_org/x/net/idna" + "golang_org/x/net/lex/httplex" ) // ClientConnPool manages a pool of HTTP/2 client connections. @@ -47,6 +54,18 @@ type http2ClientConnPool interface { MarkDead(*http2ClientConn) } +// clientConnPoolIdleCloser is the interface implemented by ClientConnPool +// implementations which can close their idle connections. +type http2clientConnPoolIdleCloser interface { + http2ClientConnPool + closeIdleConnections() +} + +var ( + _ http2clientConnPoolIdleCloser = (*http2clientConnPool)(nil) + _ http2clientConnPoolIdleCloser = http2noDialClientConnPool{} +) + // TODO: use singleflight for dialing and addConnCalls? type http2clientConnPool struct { t *http2Transport @@ -69,7 +88,16 @@ const ( http2noDialOnMiss = false ) -func (p *http2clientConnPool) getClientConn(_ *Request, addr string, dialOnMiss bool) (*http2ClientConn, error) { +func (p *http2clientConnPool) getClientConn(req *Request, addr string, dialOnMiss bool) (*http2ClientConn, error) { + if http2isConnectionCloseRequest(req) && dialOnMiss { + // It gets its own connection. + const singleUse = true + cc, err := p.t.dialClientConn(addr, singleUse) + if err != nil { + return nil, err + } + return cc, nil + } p.mu.Lock() for _, cc := range p.conns[addr] { if cc.CanTakeNewRequest() { @@ -112,7 +140,8 @@ func (p *http2clientConnPool) getStartDialLocked(addr string) *http2dialCall { // run in its own goroutine. func (c *http2dialCall) dial(addr string) { - c.res, c.err = c.p.t.dialClientConn(addr) + const singleUse = false // shared conn + c.res, c.err = c.p.t.dialClientConn(addr, singleUse) close(c.done) c.p.mu.Lock() @@ -247,6 +276,15 @@ func http2filterOutClientConn(in []*http2ClientConn, exclude *http2ClientConn) [ return out } +// noDialClientConnPool is an implementation of http2.ClientConnPool +// which never dials. We let the HTTP/1.1 client dial and use its TLS +// connection instead. +type http2noDialClientConnPool struct{ *http2clientConnPool } + +func (p http2noDialClientConnPool) GetClientConn(req *Request, addr string) (*http2ClientConn, error) { + return p.getClientConn(req, addr, http2noDialOnMiss) +} + func http2configureTransport(t1 *Transport) (*http2Transport, error) { connPool := new(http2clientConnPool) t2 := &http2Transport{ @@ -267,7 +305,7 @@ func http2configureTransport(t1 *Transport) (*http2Transport, error) { t1.TLSClientConfig.NextProtos = append(t1.TLSClientConfig.NextProtos, "http/1.1") } upgradeFn := func(authority string, c *tls.Conn) RoundTripper { - addr := http2authorityAddr(authority) + addr := http2authorityAddr("https", authority) if used, err := connPool.addConnIfNeeded(addr, t2, c); err != nil { go c.Close() return http2erringRoundTripper{err} @@ -299,15 +337,6 @@ func http2registerHTTPSProtocol(t *Transport, rt RoundTripper) (err error) { return nil } -// noDialClientConnPool is an implementation of http2.ClientConnPool -// which never dials. We let the HTTP/1.1 client dial and use its TLS -// connection instead. -type http2noDialClientConnPool struct{ *http2clientConnPool } - -func (p http2noDialClientConnPool) GetClientConn(req *Request, addr string) (*http2ClientConn, error) { - return p.getClientConn(req, addr, http2noDialOnMiss) -} - // noDialH2RoundTripper is a RoundTripper which only tries to complete the request // if there's already has a cached connection to the host. type http2noDialH2RoundTripper struct{ t *http2Transport } @@ -377,9 +406,17 @@ func (e http2ConnectionError) Error() string { type http2StreamError struct { StreamID uint32 Code http2ErrCode + Cause error // optional additional detail +} + +func http2streamError(id uint32, code http2ErrCode) http2StreamError { + return http2StreamError{StreamID: id, Code: code} } func (e http2StreamError) Error() string { + if e.Cause != nil { + return fmt.Sprintf("stream error: stream ID %d; %v; %v", e.StreamID, e.Code, e.Cause) + } return fmt.Sprintf("stream error: stream ID %d; %v", e.StreamID, e.Code) } @@ -403,6 +440,35 @@ func (e http2connError) Error() string { return fmt.Sprintf("http2: connection error: %v: %v", e.Code, e.Reason) } +type http2pseudoHeaderError string + +func (e http2pseudoHeaderError) Error() string { + return fmt.Sprintf("invalid pseudo-header %q", string(e)) +} + +type http2duplicatePseudoHeaderError string + +func (e http2duplicatePseudoHeaderError) Error() string { + return fmt.Sprintf("duplicate pseudo-header %q", string(e)) +} + +type http2headerFieldNameError string + +func (e http2headerFieldNameError) Error() string { + return fmt.Sprintf("invalid header field name %q", string(e)) +} + +type http2headerFieldValueError string + +func (e http2headerFieldValueError) Error() string { + return fmt.Sprintf("invalid header field value %q", string(e)) +} + +var ( + http2errMixPseudoHeaderTypes = errors.New("mix of request and response pseudo headers") + http2errPseudoAfterRegular = errors.New("pseudo header field after regular") +) + // fixedBuffer is an io.ReadWriter backed by a fixed size buffer. // It never allocates, but moves old data as new data is written. type http2fixedBuffer struct { @@ -743,7 +809,7 @@ type http2Frame interface { type http2Framer struct { r io.Reader lastFrame http2Frame - errReason string + errDetail error // lastHeaderStream is non-zero if the last frame was an // unfinished HEADERS/CONTINUATION. @@ -775,12 +841,33 @@ type http2Framer struct { // to return non-compliant frames or frame orders. // This is for testing and permits using the Framer to test // other HTTP/2 implementations' conformance to the spec. + // It is not compatible with ReadMetaHeaders. AllowIllegalReads bool - logReads bool + // ReadMetaHeaders if non-nil causes ReadFrame to merge + // HEADERS and CONTINUATION frames together and return + // MetaHeadersFrame instead. + ReadMetaHeaders *hpack.Decoder - debugFramer *http2Framer // only use for logging written writes - debugFramerBuf *bytes.Buffer + // MaxHeaderListSize is the http2 MAX_HEADER_LIST_SIZE. + // It's used only if ReadMetaHeaders is set; 0 means a sane default + // (currently 16MB) + // If the limit is hit, MetaHeadersFrame.Truncated is set true. + MaxHeaderListSize uint32 + + logReads, logWrites bool + + debugFramer *http2Framer // only use for logging written writes + debugFramerBuf *bytes.Buffer + debugReadLoggerf func(string, ...interface{}) + debugWriteLoggerf func(string, ...interface{}) +} + +func (fr *http2Framer) maxHeaderListSize() uint32 { + if fr.MaxHeaderListSize == 0 { + return 16 << 20 + } + return fr.MaxHeaderListSize } func (f *http2Framer) startWrite(ftype http2FrameType, flags http2Flags, streamID uint32) { @@ -807,7 +894,7 @@ func (f *http2Framer) endWrite() error { byte(length>>16), byte(length>>8), byte(length)) - if http2logFrameWrites { + if f.logWrites { f.logWrite() } @@ -829,10 +916,10 @@ func (f *http2Framer) logWrite() { f.debugFramerBuf.Write(f.wbuf) fr, err := f.debugFramer.ReadFrame() if err != nil { - log.Printf("http2: Framer %p: failed to decode just-written frame", f) + f.debugWriteLoggerf("http2: Framer %p: failed to decode just-written frame", f) return } - log.Printf("http2: Framer %p: wrote %v", f, http2summarizeFrame(fr)) + f.debugWriteLoggerf("http2: Framer %p: wrote %v", f, http2summarizeFrame(fr)) } func (f *http2Framer) writeByte(v byte) { f.wbuf = append(f.wbuf, v) } @@ -853,9 +940,12 @@ const ( // NewFramer returns a Framer that writes frames to w and reads them from r. func http2NewFramer(w io.Writer, r io.Reader) *http2Framer { fr := &http2Framer{ - w: w, - r: r, - logReads: http2logFrameReads, + w: w, + r: r, + logReads: http2logFrameReads, + logWrites: http2logFrameWrites, + debugReadLoggerf: log.Printf, + debugWriteLoggerf: log.Printf, } fr.getReadBuf = func(size uint32) []byte { if cap(fr.readBuf) >= int(size) { @@ -879,6 +969,17 @@ func (fr *http2Framer) SetMaxReadFrameSize(v uint32) { fr.maxReadSize = v } +// ErrorDetail returns a more detailed error of the last error +// returned by Framer.ReadFrame. For instance, if ReadFrame +// returns a StreamError with code PROTOCOL_ERROR, ErrorDetail +// will say exactly what was invalid. ErrorDetail is not guaranteed +// to return a non-nil value and like the rest of the http2 package, +// its return value is not protected by an API compatibility promise. +// ErrorDetail is reset after the next call to ReadFrame. +func (fr *http2Framer) ErrorDetail() error { + return fr.errDetail +} + // ErrFrameTooLarge is returned from Framer.ReadFrame when the peer // sends a frame that is larger than declared with SetMaxReadFrameSize. var http2ErrFrameTooLarge = errors.New("http2: frame too large") @@ -897,9 +998,10 @@ func http2terminalReadFrameError(err error) bool { // // If the frame is larger than previously set with SetMaxReadFrameSize, the // returned error is ErrFrameTooLarge. Other errors may be of type -// ConnectionError, StreamError, or anything else from from the underlying +// ConnectionError, StreamError, or anything else from the underlying // reader. func (fr *http2Framer) ReadFrame() (http2Frame, error) { + fr.errDetail = nil if fr.lastFrame != nil { fr.lastFrame.invalidate() } @@ -925,7 +1027,10 @@ func (fr *http2Framer) ReadFrame() (http2Frame, error) { return nil, err } if fr.logReads { - log.Printf("http2: Framer %p: read %v", fr, http2summarizeFrame(f)) + fr.debugReadLoggerf("http2: Framer %p: read %v", fr, http2summarizeFrame(f)) + } + if fh.Type == http2FrameHeaders && fr.ReadMetaHeaders != nil { + return fr.readMetaFrame(f.(*http2HeadersFrame)) } return f, nil } @@ -935,7 +1040,7 @@ func (fr *http2Framer) ReadFrame() (http2Frame, error) { // to the peer before hanging up on them. This might help others debug // their implementations. func (fr *http2Framer) connError(code http2ErrCode, reason string) error { - fr.errReason = reason + fr.errDetail = errors.New(reason) return http2ConnectionError(code) } @@ -1023,7 +1128,15 @@ func http2parseDataFrame(fh http2FrameHeader, payload []byte) (http2Frame, error return f, nil } -var http2errStreamID = errors.New("invalid streamid") +var ( + http2errStreamID = errors.New("invalid stream ID") + http2errDepStreamID = errors.New("invalid dependent stream ID") + http2errPadLength = errors.New("pad length too large") +) + +func http2validStreamIDOrZero(streamID uint32) bool { + return streamID&(1<<31) == 0 +} func http2validStreamID(streamID uint32) bool { return streamID != 0 && streamID&(1<<31) == 0 @@ -1032,18 +1145,40 @@ func http2validStreamID(streamID uint32) bool { // WriteData writes a DATA frame. // // It will perform exactly one Write to the underlying Writer. -// It is the caller's responsibility to not call other Write methods concurrently. +// It is the caller's responsibility not to violate the maximum frame size +// and to not call other Write methods concurrently. func (f *http2Framer) WriteData(streamID uint32, endStream bool, data []byte) error { + return f.WriteDataPadded(streamID, endStream, data, nil) +} +// WriteData writes a DATA frame with optional padding. +// +// If pad is nil, the padding bit is not sent. +// The length of pad must not exceed 255 bytes. +// +// It will perform exactly one Write to the underlying Writer. +// It is the caller's responsibility not to violate the maximum frame size +// and to not call other Write methods concurrently. +func (f *http2Framer) WriteDataPadded(streamID uint32, endStream bool, data, pad []byte) error { if !http2validStreamID(streamID) && !f.AllowIllegalWrites { return http2errStreamID } + if len(pad) > 255 { + return http2errPadLength + } var flags http2Flags if endStream { flags |= http2FlagDataEndStream } + if pad != nil { + flags |= http2FlagDataPadded + } f.startWrite(http2FrameData, flags, streamID) + if pad != nil { + f.wbuf = append(f.wbuf, byte(len(pad))) + } f.wbuf = append(f.wbuf, data...) + f.wbuf = append(f.wbuf, pad...) return f.endWrite() } @@ -1126,7 +1261,7 @@ func (f *http2Framer) WriteSettings(settings ...http2Setting) error { return f.endWrite() } -// WriteSettings writes an empty SETTINGS frame with the ACK bit set. +// WriteSettingsAck writes an empty SETTINGS frame with the ACK bit set. // // It will perform exactly one Write to the underlying Writer. // It is the caller's responsibility to not call other Write methods concurrently. @@ -1247,7 +1382,7 @@ func http2parseWindowUpdateFrame(fh http2FrameHeader, p []byte) (http2Frame, err if fh.StreamID == 0 { return nil, http2ConnectionError(http2ErrCodeProtocol) } - return nil, http2StreamError{fh.StreamID, http2ErrCodeProtocol} + return nil, http2streamError(fh.StreamID, http2ErrCodeProtocol) } return &http2WindowUpdateFrame{ http2FrameHeader: fh, @@ -1325,7 +1460,7 @@ func http2parseHeadersFrame(fh http2FrameHeader, p []byte) (_ http2Frame, err er } } if len(p)-int(padLength) <= 0 { - return nil, http2StreamError{fh.StreamID, http2ErrCodeProtocol} + return nil, http2streamError(fh.StreamID, http2ErrCodeProtocol) } hf.headerFragBuf = p[:len(p)-int(padLength)] return hf, nil @@ -1389,8 +1524,8 @@ func (f *http2Framer) WriteHeaders(p http2HeadersFrameParam) error { } if !p.Priority.IsZero() { v := p.Priority.StreamDep - if !http2validStreamID(v) && !f.AllowIllegalWrites { - return errors.New("invalid dependent stream id") + if !http2validStreamIDOrZero(v) && !f.AllowIllegalWrites { + return http2errDepStreamID } if p.Priority.Exclusive { v |= 1 << 31 @@ -1458,6 +1593,9 @@ func (f *http2Framer) WritePriority(streamID uint32, p http2PriorityParam) error if !http2validStreamID(streamID) && !f.AllowIllegalWrites { return http2errStreamID } + if !http2validStreamIDOrZero(p.StreamDep) { + return http2errDepStreamID + } f.startWrite(http2FramePriority, 0, streamID) v := p.StreamDep if p.Exclusive { @@ -1669,6 +1807,202 @@ type http2headersEnder interface { HeadersEnded() bool } +type http2headersOrContinuation interface { + http2headersEnder + HeaderBlockFragment() []byte +} + +// A MetaHeadersFrame is the representation of one HEADERS frame and +// zero or more contiguous CONTINUATION frames and the decoding of +// their HPACK-encoded contents. +// +// This type of frame does not appear on the wire and is only returned +// by the Framer when Framer.ReadMetaHeaders is set. +type http2MetaHeadersFrame struct { + *http2HeadersFrame + + // Fields are the fields contained in the HEADERS and + // CONTINUATION frames. The underlying slice is owned by the + // Framer and must not be retained after the next call to + // ReadFrame. + // + // Fields are guaranteed to be in the correct http2 order and + // not have unknown pseudo header fields or invalid header + // field names or values. Required pseudo header fields may be + // missing, however. Use the MetaHeadersFrame.Pseudo accessor + // method access pseudo headers. + Fields []hpack.HeaderField + + // Truncated is whether the max header list size limit was hit + // and Fields is incomplete. The hpack decoder state is still + // valid, however. + Truncated bool +} + +// PseudoValue returns the given pseudo header field's value. +// The provided pseudo field should not contain the leading colon. +func (mh *http2MetaHeadersFrame) PseudoValue(pseudo string) string { + for _, hf := range mh.Fields { + if !hf.IsPseudo() { + return "" + } + if hf.Name[1:] == pseudo { + return hf.Value + } + } + return "" +} + +// RegularFields returns the regular (non-pseudo) header fields of mh. +// The caller does not own the returned slice. +func (mh *http2MetaHeadersFrame) RegularFields() []hpack.HeaderField { + for i, hf := range mh.Fields { + if !hf.IsPseudo() { + return mh.Fields[i:] + } + } + return nil +} + +// PseudoFields returns the pseudo header fields of mh. +// The caller does not own the returned slice. +func (mh *http2MetaHeadersFrame) PseudoFields() []hpack.HeaderField { + for i, hf := range mh.Fields { + if !hf.IsPseudo() { + return mh.Fields[:i] + } + } + return mh.Fields +} + +func (mh *http2MetaHeadersFrame) checkPseudos() error { + var isRequest, isResponse bool + pf := mh.PseudoFields() + for i, hf := range pf { + switch hf.Name { + case ":method", ":path", ":scheme", ":authority": + isRequest = true + case ":status": + isResponse = true + default: + return http2pseudoHeaderError(hf.Name) + } + + for _, hf2 := range pf[:i] { + if hf.Name == hf2.Name { + return http2duplicatePseudoHeaderError(hf.Name) + } + } + } + if isRequest && isResponse { + return http2errMixPseudoHeaderTypes + } + return nil +} + +func (fr *http2Framer) maxHeaderStringLen() int { + v := fr.maxHeaderListSize() + if uint32(int(v)) == v { + return int(v) + } + + return 0 +} + +// readMetaFrame returns 0 or more CONTINUATION frames from fr and +// merge them into into the provided hf and returns a MetaHeadersFrame +// with the decoded hpack values. +func (fr *http2Framer) readMetaFrame(hf *http2HeadersFrame) (*http2MetaHeadersFrame, error) { + if fr.AllowIllegalReads { + return nil, errors.New("illegal use of AllowIllegalReads with ReadMetaHeaders") + } + mh := &http2MetaHeadersFrame{ + http2HeadersFrame: hf, + } + var remainSize = fr.maxHeaderListSize() + var sawRegular bool + + var invalid error // pseudo header field errors + hdec := fr.ReadMetaHeaders + hdec.SetEmitEnabled(true) + hdec.SetMaxStringLength(fr.maxHeaderStringLen()) + hdec.SetEmitFunc(func(hf hpack.HeaderField) { + if http2VerboseLogs && fr.logReads { + fr.debugReadLoggerf("http2: decoded hpack field %+v", hf) + } + if !httplex.ValidHeaderFieldValue(hf.Value) { + invalid = http2headerFieldValueError(hf.Value) + } + isPseudo := strings.HasPrefix(hf.Name, ":") + if isPseudo { + if sawRegular { + invalid = http2errPseudoAfterRegular + } + } else { + sawRegular = true + if !http2validWireHeaderFieldName(hf.Name) { + invalid = http2headerFieldNameError(hf.Name) + } + } + + if invalid != nil { + hdec.SetEmitEnabled(false) + return + } + + size := hf.Size() + if size > remainSize { + hdec.SetEmitEnabled(false) + mh.Truncated = true + return + } + remainSize -= size + + mh.Fields = append(mh.Fields, hf) + }) + + defer hdec.SetEmitFunc(func(hf hpack.HeaderField) {}) + + var hc http2headersOrContinuation = hf + for { + frag := hc.HeaderBlockFragment() + if _, err := hdec.Write(frag); err != nil { + return nil, http2ConnectionError(http2ErrCodeCompression) + } + + if hc.HeadersEnded() { + break + } + if f, err := fr.ReadFrame(); err != nil { + return nil, err + } else { + hc = f.(*http2ContinuationFrame) + } + } + + mh.http2HeadersFrame.headerFragBuf = nil + mh.http2HeadersFrame.invalidate() + + if err := hdec.Close(); err != nil { + return nil, http2ConnectionError(http2ErrCodeCompression) + } + if invalid != nil { + fr.errDetail = invalid + if http2VerboseLogs { + log.Printf("http2: invalid header: %v", invalid) + } + return nil, http2StreamError{mh.StreamID, http2ErrCodeProtocol, invalid} + } + if err := mh.checkPseudos(); err != nil { + fr.errDetail = err + if http2VerboseLogs { + log.Printf("http2: invalid pseudo headers: %v", err) + } + return nil, http2StreamError{mh.StreamID, http2ErrCodeProtocol, err} + } + return mh, nil +} + func http2summarizeFrame(f http2Frame) string { var buf bytes.Buffer f.Header().writeDebug(&buf) @@ -1712,7 +2046,160 @@ func http2summarizeFrame(f http2Frame) string { return buf.String() } -func http2requestCancel(req *Request) <-chan struct{} { return req.Cancel } +func http2transportExpectContinueTimeout(t1 *Transport) time.Duration { + return t1.ExpectContinueTimeout +} + +// isBadCipher reports whether the cipher is blacklisted by the HTTP/2 spec. +func http2isBadCipher(cipher uint16) bool { + switch cipher { + case tls.TLS_RSA_WITH_RC4_128_SHA, + tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA, + tls.TLS_RSA_WITH_AES_128_CBC_SHA, + tls.TLS_RSA_WITH_AES_256_CBC_SHA, + tls.TLS_RSA_WITH_AES_128_GCM_SHA256, + tls.TLS_RSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_ECDSA_WITH_RC4_128_SHA, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, + tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, + tls.TLS_ECDHE_RSA_WITH_RC4_128_SHA, + tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA, + tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, + tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA: + + return true + default: + return false + } +} + +type http2contextContext interface { + context.Context +} + +func http2serverConnBaseContext(c net.Conn, opts *http2ServeConnOpts) (ctx http2contextContext, cancel func()) { + ctx, cancel = context.WithCancel(context.Background()) + ctx = context.WithValue(ctx, LocalAddrContextKey, c.LocalAddr()) + if hs := opts.baseConfig(); hs != nil { + ctx = context.WithValue(ctx, ServerContextKey, hs) + } + return +} + +func http2contextWithCancel(ctx http2contextContext) (_ http2contextContext, cancel func()) { + return context.WithCancel(ctx) +} + +func http2requestWithContext(req *Request, ctx http2contextContext) *Request { + return req.WithContext(ctx) +} + +type http2clientTrace httptrace.ClientTrace + +func http2reqContext(r *Request) context.Context { return r.Context() } + +func (t *http2Transport) idleConnTimeout() time.Duration { + if t.t1 != nil { + return t.t1.IdleConnTimeout + } + return 0 +} + +func http2setResponseUncompressed(res *Response) { res.Uncompressed = true } + +func http2traceGotConn(req *Request, cc *http2ClientConn) { + trace := httptrace.ContextClientTrace(req.Context()) + if trace == nil || trace.GotConn == nil { + return + } + ci := httptrace.GotConnInfo{Conn: cc.tconn} + cc.mu.Lock() + ci.Reused = cc.nextStreamID > 1 + ci.WasIdle = len(cc.streams) == 0 && ci.Reused + if ci.WasIdle && !cc.lastActive.IsZero() { + ci.IdleTime = time.Now().Sub(cc.lastActive) + } + cc.mu.Unlock() + + trace.GotConn(ci) +} + +func http2traceWroteHeaders(trace *http2clientTrace) { + if trace != nil && trace.WroteHeaders != nil { + trace.WroteHeaders() + } +} + +func http2traceGot100Continue(trace *http2clientTrace) { + if trace != nil && trace.Got100Continue != nil { + trace.Got100Continue() + } +} + +func http2traceWait100Continue(trace *http2clientTrace) { + if trace != nil && trace.Wait100Continue != nil { + trace.Wait100Continue() + } +} + +func http2traceWroteRequest(trace *http2clientTrace, err error) { + if trace != nil && trace.WroteRequest != nil { + trace.WroteRequest(httptrace.WroteRequestInfo{Err: err}) + } +} + +func http2traceFirstResponseByte(trace *http2clientTrace) { + if trace != nil && trace.GotFirstResponseByte != nil { + trace.GotFirstResponseByte() + } +} + +func http2requestTrace(req *Request) *http2clientTrace { + trace := httptrace.ContextClientTrace(req.Context()) + return (*http2clientTrace)(trace) +} + +// Ping sends a PING frame to the server and waits for the ack. +func (cc *http2ClientConn) Ping(ctx context.Context) error { + return cc.ping(ctx) +} + +func http2cloneTLSConfig(c *tls.Config) *tls.Config { return c.Clone() } + +var _ Pusher = (*http2responseWriter)(nil) + +// Push implements http.Pusher. +func (w *http2responseWriter) Push(target string, opts *PushOptions) error { + internalOpts := http2pushOptions{} + if opts != nil { + internalOpts.Method = opts.Method + internalOpts.Header = opts.Header + } + return w.push(target, internalOpts) +} + +func http2configureServer18(h1 *Server, h2 *http2Server) error { + if h2.IdleTimeout == 0 { + if h1.IdleTimeout != 0 { + h2.IdleTimeout = h1.IdleTimeout + } else { + h2.IdleTimeout = h1.ReadTimeout + } + } + return nil +} + +func http2shouldLogPanic(panicValue interface{}) bool { + return panicValue != nil && panicValue != ErrAbortHandler +} + +func http2reqGetBody(req *Request) func() (io.ReadCloser, error) { + return req.GetBody +} + +func http2reqBodyIsNoBody(body io.ReadCloser) bool { + return body == NoBody +} var http2DebugGoroutines = os.Getenv("DEBUG_HTTP2_GOROUTINES") == "1" @@ -1937,6 +2424,7 @@ var ( http2VerboseLogs bool http2logFrameWrites bool http2logFrameReads bool + http2inTests bool ) func init() { @@ -1978,13 +2466,23 @@ var ( type http2streamState int +// HTTP/2 stream states. +// +// See http://tools.ietf.org/html/rfc7540#section-5.1. +// +// For simplicity, the server code merges "reserved (local)" into +// "half-closed (remote)". This is one less state transition to track. +// The only downside is that we send PUSH_PROMISEs slightly less +// liberally than allowable. More discussion here: +// https://lists.w3.org/Archives/Public/ietf-http-wg/2016JulSep/0599.html +// +// "reserved (remote)" is omitted since the client code does not +// support server push. const ( http2stateIdle http2streamState = iota http2stateOpen http2stateHalfClosedLocal http2stateHalfClosedRemote - http2stateResvLocal - http2stateResvRemote http2stateClosed ) @@ -1993,8 +2491,6 @@ var http2stateName = [...]string{ http2stateOpen: "Open", http2stateHalfClosedLocal: "HalfClosedLocal", http2stateHalfClosedRemote: "HalfClosedRemote", - http2stateResvLocal: "ResvLocal", - http2stateResvRemote: "ResvRemote", http2stateClosed: "Closed", } @@ -2070,57 +2566,23 @@ var ( http2errInvalidHeaderFieldValue = errors.New("http2: invalid header field value") ) -// validHeaderFieldName reports whether v is a valid header field name (key). -// RFC 7230 says: -// header-field = field-name ":" OWS field-value OWS -// field-name = token -// tchar = "!" / "#" / "$" / "%" / "&" / "'" / "*" / "+" / "-" / "." / -// "^" / "_" / " +// validWireHeaderFieldName reports whether v is a valid header field +// name (key). See httplex.ValidHeaderName for the base rules. +// // Further, http2 says: // "Just as in HTTP/1.x, header field names are strings of ASCII // characters that are compared in a case-insensitive // fashion. However, header field names MUST be converted to // lowercase prior to their encoding in HTTP/2. " -func http2validHeaderFieldName(v string) bool { +func http2validWireHeaderFieldName(v string) bool { if len(v) == 0 { return false } for _, r := range v { - if int(r) >= len(http2isTokenTable) || ('A' <= r && r <= 'Z') { + if !httplex.IsTokenRune(r) { return false } - if !http2isTokenTable[byte(r)] { - return false - } - } - return true -} - -// validHeaderFieldValue reports whether v is a valid header field value. -// -// RFC 7230 says: -// field-value = *( field-content / obs-fold ) -// obj-fold = N/A to http2, and deprecated -// field-content = field-vchar [ 1*( SP / HTAB ) field-vchar ] -// field-vchar = VCHAR / obs-text -// obs-text = %x80-FF -// VCHAR = "any visible [USASCII] character" -// -// http2 further says: "Similarly, HTTP/2 allows header field values -// that are not valid. While most of the values that can be encoded -// will not alter header field parsing, carriage return (CR, ASCII -// 0xd), line feed (LF, ASCII 0xa), and the zero character (NUL, ASCII -// 0x0) might be exploited by an attacker if they are translated -// verbatim. Any request or response that contains a character not -// permitted in a header field value MUST be treated as malformed -// (Section 8.1.2.6). Valid characters are defined by the -// field-content ABNF rule in Section 3.2 of [RFC7230]." -// -// This function does not (yet?) properly handle the rejection of -// strings that begin or end with SP or HTAB. -func http2validHeaderFieldValue(v string) bool { - for i := 0; i < len(v); i++ { - if b := v[i]; b < ' ' && b != '\t' || b == 0x7f { + if 'A' <= r && r <= 'Z' { return false } } @@ -2189,13 +2651,27 @@ func http2newBufferedWriter(w io.Writer) *http2bufferedWriter { return &http2bufferedWriter{w: w} } +// bufWriterPoolBufferSize is the size of bufio.Writer's +// buffers created using bufWriterPool. +// +// TODO: pick a less arbitrary value? this is a bit under +// (3 x typical 1500 byte MTU) at least. Other than that, +// not much thought went into it. +const http2bufWriterPoolBufferSize = 4 << 10 + var http2bufWriterPool = sync.Pool{ New: func() interface{} { - - return bufio.NewWriterSize(nil, 4<<10) + return bufio.NewWriterSize(nil, http2bufWriterPoolBufferSize) }, } +func (w *http2bufferedWriter) Available() int { + if w.bw == nil { + return http2bufWriterPoolBufferSize + } + return w.bw.Available() +} + func (w *http2bufferedWriter) Write(p []byte) (n int, err error) { if w.bw == nil { bw := http2bufWriterPool.Get().(*bufio.Writer) @@ -2225,7 +2701,7 @@ func http2mustUint31(v int32) uint32 { } // bodyAllowedForStatus reports whether a given response status code -// permits a body. See RFC2616, section 4.4. +// permits a body. See RFC 2616, section 4.4. func http2bodyAllowedForStatus(status int) bool { switch { case status >= 100 && status <= 199: @@ -2251,90 +2727,57 @@ func (e *http2httpError) Temporary() bool { return true } var http2errTimeout error = &http2httpError{msg: "http2: timeout awaiting response headers", timeout: true} -var http2isTokenTable = [127]bool{ - '!': true, - '#': true, - '$': true, - '%': true, - '&': true, - '\'': true, - '*': true, - '+': true, - '-': true, - '.': true, - '0': true, - '1': true, - '2': true, - '3': true, - '4': true, - '5': true, - '6': true, - '7': true, - '8': true, - '9': true, - 'A': true, - 'B': true, - 'C': true, - 'D': true, - 'E': true, - 'F': true, - 'G': true, - 'H': true, - 'I': true, - 'J': true, - 'K': true, - 'L': true, - 'M': true, - 'N': true, - 'O': true, - 'P': true, - 'Q': true, - 'R': true, - 'S': true, - 'T': true, - 'U': true, - 'W': true, - 'V': true, - 'X': true, - 'Y': true, - 'Z': true, - '^': true, - '_': true, - '`': true, - 'a': true, - 'b': true, - 'c': true, - 'd': true, - 'e': true, - 'f': true, - 'g': true, - 'h': true, - 'i': true, - 'j': true, - 'k': true, - 'l': true, - 'm': true, - 'n': true, - 'o': true, - 'p': true, - 'q': true, - 'r': true, - 's': true, - 't': true, - 'u': true, - 'v': true, - 'w': true, - 'x': true, - 'y': true, - 'z': true, - '|': true, - '~': true, -} - type http2connectionStater interface { ConnectionState() tls.ConnectionState } +var http2sorterPool = sync.Pool{New: func() interface{} { return new(http2sorter) }} + +type http2sorter struct { + v []string // owned by sorter +} + +func (s *http2sorter) Len() int { return len(s.v) } + +func (s *http2sorter) Swap(i, j int) { s.v[i], s.v[j] = s.v[j], s.v[i] } + +func (s *http2sorter) Less(i, j int) bool { return s.v[i] < s.v[j] } + +// Keys returns the sorted keys of h. +// +// The returned slice is only valid until s used again or returned to +// its pool. +func (s *http2sorter) Keys(h Header) []string { + keys := s.v[:0] + for k := range h { + keys = append(keys, k) + } + s.v = keys + sort.Sort(s) + return keys +} + +func (s *http2sorter) SortStrings(ss []string) { + + save := s.v + s.v = ss + sort.Sort(s) + s.v = save +} + +// validPseudoPath reports whether v is a valid :path pseudo-header +// value. It must be either: +// +// *) a non-empty string starting with '/', but not with with "//", +// *) the string '*', for OPTIONS requests. +// +// For now this is only used a quick check for deciding when to clean +// up Opaque URLs before sending requests from the Transport. +// See golang.org/issue/16847 +func http2validPseudoPath(v string) bool { + return (len(v) > 0 && v[0] == '/' && (len(v) == 1 || v[1] != '/')) || v == "*" +} + // pipe is a goroutine-safe io.Reader/io.Writer pair. It's like // io.Pipe except there are no PipeReader/PipeWriter halves, and the // underlying buffer is an interface. (io.Pipe is always unbuffered) @@ -2354,6 +2797,12 @@ type http2pipeBuffer interface { io.Reader } +func (p *http2pipe) Len() int { + p.mu.Lock() + defer p.mu.Unlock() + return p.b.Len() +} + // Read waits until data is available and copies bytes // from the buffer into p. func (p *http2pipe) Read(d []byte) (n int, err error) { @@ -2525,6 +2974,15 @@ type http2Server struct { // PermitProhibitedCipherSuites, if true, permits the use of // cipher suites prohibited by the HTTP/2 spec. PermitProhibitedCipherSuites bool + + // IdleTimeout specifies how long until idle clients should be + // closed with a GOAWAY frame. PING frames are not considered + // activity for the purposes of IdleTimeout. + IdleTimeout time.Duration + + // NewWriteScheduler constructs a write scheduler for a connection. + // If nil, a default scheduler is chosen. + NewWriteScheduler func() http2WriteScheduler } func (s *http2Server) maxReadFrameSize() uint32 { @@ -2547,9 +3005,15 @@ func (s *http2Server) maxConcurrentStreams() uint32 { // // ConfigureServer must be called before s begins serving. func http2ConfigureServer(s *Server, conf *http2Server) error { + if s == nil { + panic("nil *http.Server") + } if conf == nil { conf = new(http2Server) } + if err := http2configureServer18(s, conf); err != nil { + return err + } if s.TLSConfig == nil { s.TLSConfig = new(tls.Config) @@ -2588,8 +3052,6 @@ func http2ConfigureServer(s *Server, conf *http2Server) error { s.TLSConfig.NextProtos = append(s.TLSConfig.NextProtos, http2NextProtoTLS) } - s.TLSConfig.NextProtos = append(s.TLSConfig.NextProtos, "h2-14") - if s.TLSNextProto == nil { s.TLSNextProto = map[string]func(*Server, *tls.Conn, Handler){} } @@ -2603,7 +3065,6 @@ func http2ConfigureServer(s *Server, conf *http2Server) error { }) } s.TLSNextProto[http2NextProtoTLS] = protoHandler - s.TLSNextProto["h2-14"] = protoHandler return nil } @@ -2653,35 +3114,50 @@ func (o *http2ServeConnOpts) handler() Handler { // // The opts parameter is optional. If nil, default values are used. func (s *http2Server) ServeConn(c net.Conn, opts *http2ServeConnOpts) { + baseCtx, cancel := http2serverConnBaseContext(c, opts) + defer cancel() + sc := &http2serverConn{ - srv: s, - hs: opts.baseConfig(), - conn: c, - remoteAddrStr: c.RemoteAddr().String(), - bw: http2newBufferedWriter(c), - handler: opts.handler(), - streams: make(map[uint32]*http2stream), - readFrameCh: make(chan http2readFrameResult), - wantWriteFrameCh: make(chan http2frameWriteMsg, 8), - wroteFrameCh: make(chan http2frameWriteResult, 1), - bodyReadCh: make(chan http2bodyReadMsg), - doneServing: make(chan struct{}), - advMaxStreams: s.maxConcurrentStreams(), - writeSched: http2writeScheduler{ - maxFrameSize: http2initialMaxFrameSize, - }, + srv: s, + hs: opts.baseConfig(), + conn: c, + baseCtx: baseCtx, + remoteAddrStr: c.RemoteAddr().String(), + bw: http2newBufferedWriter(c), + handler: opts.handler(), + streams: make(map[uint32]*http2stream), + readFrameCh: make(chan http2readFrameResult), + wantWriteFrameCh: make(chan http2FrameWriteRequest, 8), + wantStartPushCh: make(chan http2startPushRequest, 8), + wroteFrameCh: make(chan http2frameWriteResult, 1), + bodyReadCh: make(chan http2bodyReadMsg), + doneServing: make(chan struct{}), + clientMaxStreams: math.MaxUint32, + advMaxStreams: s.maxConcurrentStreams(), initialWindowSize: http2initialWindowSize, + maxFrameSize: http2initialMaxFrameSize, headerTableSize: http2initialHeaderTableSize, serveG: http2newGoroutineLock(), pushEnabled: true, } + + if sc.hs.WriteTimeout != 0 { + sc.conn.SetWriteDeadline(time.Time{}) + } + + if s.NewWriteScheduler != nil { + sc.writeSched = s.NewWriteScheduler() + } else { + sc.writeSched = http2NewRandomWriteScheduler() + } + sc.flow.add(http2initialWindowSize) sc.inflow.add(http2initialWindowSize) sc.hpackEncoder = hpack.NewEncoder(&sc.headerWriteBuf) - sc.hpackDecoder = hpack.NewDecoder(http2initialHeaderTableSize, nil) - sc.hpackDecoder.SetMaxStringLength(sc.maxHeaderStringLen()) fr := http2NewFramer(sc.bw, c) + fr.ReadMetaHeaders = hpack.NewDecoder(http2initialHeaderTableSize, nil) + fr.MaxHeaderListSize = sc.maxHeaderListSize() fr.SetMaxReadFrameSize(s.maxReadFrameSize()) sc.framer = fr @@ -2711,27 +3187,6 @@ func (s *http2Server) ServeConn(c net.Conn, opts *http2ServeConnOpts) { sc.serve() } -// isBadCipher reports whether the cipher is blacklisted by the HTTP/2 spec. -func http2isBadCipher(cipher uint16) bool { - switch cipher { - case tls.TLS_RSA_WITH_RC4_128_SHA, - tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA, - tls.TLS_RSA_WITH_AES_128_CBC_SHA, - tls.TLS_RSA_WITH_AES_256_CBC_SHA, - tls.TLS_ECDHE_ECDSA_WITH_RC4_128_SHA, - tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, - tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, - tls.TLS_ECDHE_RSA_WITH_RC4_128_SHA, - tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA, - tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, - tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA: - - return true - default: - return false - } -} - func (sc *http2serverConn) rejectConn(err http2ErrCode, debug string) { sc.vlogf("http2: server rejecting conn: %v, %s", err, debug) @@ -2747,18 +3202,20 @@ type http2serverConn struct { conn net.Conn bw *http2bufferedWriter // writing to conn handler Handler + baseCtx http2contextContext framer *http2Framer - hpackDecoder *hpack.Decoder - doneServing chan struct{} // closed when serverConn.serve ends - readFrameCh chan http2readFrameResult // written by serverConn.readFrames - wantWriteFrameCh chan http2frameWriteMsg // from handlers -> serve - wroteFrameCh chan http2frameWriteResult // from writeFrameAsync -> serve, tickles more frame writes - bodyReadCh chan http2bodyReadMsg // from handlers -> serve - testHookCh chan func(int) // code to run on the serve loop - flow http2flow // conn-wide (not stream-specific) outbound flow control - inflow http2flow // conn-wide inbound flow control - tlsState *tls.ConnectionState // shared by all handlers, like net/http + doneServing chan struct{} // closed when serverConn.serve ends + readFrameCh chan http2readFrameResult // written by serverConn.readFrames + wantWriteFrameCh chan http2FrameWriteRequest // from handlers -> serve + wantStartPushCh chan http2startPushRequest // from handlers -> serve + wroteFrameCh chan http2frameWriteResult // from writeFrameAsync -> serve, tickles more frame writes + bodyReadCh chan http2bodyReadMsg // from handlers -> serve + testHookCh chan func(int) // code to run on the serve loop + flow http2flow // conn-wide (not stream-specific) outbound flow control + inflow http2flow // conn-wide inbound flow control + tlsState *tls.ConnectionState // shared by all handlers, like net/http remoteAddrStr string + writeSched http2WriteScheduler // Everything following is owned by the serve loop; use serveG.check(): serveG http2goroutineLock // used to verify funcs are on serve() @@ -2768,37 +3225,33 @@ type http2serverConn struct { unackedSettings int // how many SETTINGS have we sent without ACKs? clientMaxStreams uint32 // SETTINGS_MAX_CONCURRENT_STREAMS from client (our PUSH_PROMISE limit) advMaxStreams uint32 // our SETTINGS_MAX_CONCURRENT_STREAMS advertised the client - curOpenStreams uint32 // client's number of open streams - maxStreamID uint32 // max ever seen + curClientStreams uint32 // number of open streams initiated by the client + curPushedStreams uint32 // number of open streams initiated by server push + maxClientStreamID uint32 // max ever seen from client (odd), or 0 if there have been no client requests + maxPushPromiseID uint32 // ID of the last push promise (even), or 0 if there have been no pushes streams map[uint32]*http2stream initialWindowSize int32 + maxFrameSize int32 headerTableSize uint32 peerMaxHeaderListSize uint32 // zero means unknown (default) canonHeader map[string]string // http2-lower-case -> Go-Canonical-Case - req http2requestParam // non-zero while reading request headers - writingFrame bool // started write goroutine but haven't heard back on wroteFrameCh + writingFrame bool // started writing a frame (on serve goroutine or separate) + writingFrameAsync bool // started a frame on its own goroutine but haven't heard back on wroteFrameCh needsFrameFlush bool // last frame write wasn't a flush - writeSched http2writeScheduler - inGoAway bool // we've started to or sent GOAWAY - needToSendGoAway bool // we need to schedule a GOAWAY frame write + inGoAway bool // we've started to or sent GOAWAY + inFrameScheduleLoop bool // whether we're in the scheduleFrameWrite loop + needToSendGoAway bool // we need to schedule a GOAWAY frame write goAwayCode http2ErrCode shutdownTimerCh <-chan time.Time // nil until used shutdownTimer *time.Timer // nil until used + idleTimer *time.Timer // nil if unused + idleTimerCh <-chan time.Time // nil if unused // Owned by the writeFrameAsync goroutine: headerWriteBuf bytes.Buffer hpackEncoder *hpack.Encoder } -func (sc *http2serverConn) maxHeaderStringLen() int { - v := sc.maxHeaderListSize() - if uint32(int(v)) == v { - return int(v) - } - - return 0 -} - func (sc *http2serverConn) maxHeaderListSize() uint32 { n := sc.hs.MaxHeaderBytes if n <= 0 { @@ -2811,19 +3264,9 @@ func (sc *http2serverConn) maxHeaderListSize() uint32 { return uint32(n + typicalHeaders*perFieldOverhead) } -// requestParam is the state of the next request, initialized over -// potentially several frames HEADERS + zero or more CONTINUATION -// frames. -type http2requestParam struct { - // stream is non-nil if we're reading (HEADER or CONTINUATION) - // frames for a request (but not DATA). - stream *http2stream - header Header - method, path string - scheme, authority string - sawRegularHeader bool // saw a non-pseudo header already - invalidHeader bool // an invalid header was seen - headerListSize int64 // actually uint32, but easier math this way +func (sc *http2serverConn) curOpenStreams() uint32 { + sc.serveG.check() + return sc.curClientStreams + sc.curPushedStreams } // stream represents a stream. This is the minimal metadata needed by @@ -2835,10 +3278,12 @@ type http2requestParam struct { // responseWriter's state field. type http2stream struct { // immutable: - sc *http2serverConn - id uint32 - body *http2pipe // non-nil if expecting DATA frames - cw http2closeWaiter // closed wait stream transitions to closed state + sc *http2serverConn + id uint32 + body *http2pipe // non-nil if expecting DATA frames + cw http2closeWaiter // closed wait stream transitions to closed state + ctx http2contextContext + cancelCtx func() // owned by serverConn's serve loop: bodyBytes int64 // body bytes seen so far @@ -2849,9 +3294,10 @@ type http2stream struct { numTrailerValues int64 weight uint8 state http2streamState - sentReset bool // only true once detached from streams map - gotReset bool // only true once detacted from streams map - gotTrailerHeader bool // HEADER frame for trailers was seen + resetQueued bool // RST_STREAM queued for write; set by sc.resetStream + gotTrailerHeader bool // HEADER frame for trailers was seen + wroteHeaders bool // whether we wrote headers (not status 100) + reqBuf []byte // if non-nil, body pipe buffer to return later at EOF trailer Header // accumulated trailers reqTrailer Header // handler's Request.Trailer @@ -2874,8 +3320,14 @@ func (sc *http2serverConn) state(streamID uint32) (http2streamState, *http2strea return st.state, st } - if streamID <= sc.maxStreamID { - return http2stateClosed, nil + if streamID%2 == 1 { + if streamID <= sc.maxClientStreamID { + return http2stateClosed, nil + } + } else { + if streamID <= sc.maxPushPromiseID { + return http2stateClosed, nil + } } return http2stateIdle, nil } @@ -2952,83 +3404,6 @@ func (sc *http2serverConn) condlogf(err error, format string, args ...interface{ } } -func (sc *http2serverConn) onNewHeaderField(f hpack.HeaderField) { - sc.serveG.check() - if http2VerboseLogs { - sc.vlogf("http2: server decoded %v", f) - } - switch { - case !http2validHeaderFieldValue(f.Value): - sc.req.invalidHeader = true - case strings.HasPrefix(f.Name, ":"): - if sc.req.sawRegularHeader { - sc.logf("pseudo-header after regular header") - sc.req.invalidHeader = true - return - } - var dst *string - switch f.Name { - case ":method": - dst = &sc.req.method - case ":path": - dst = &sc.req.path - case ":scheme": - dst = &sc.req.scheme - case ":authority": - dst = &sc.req.authority - default: - - sc.logf("invalid pseudo-header %q", f.Name) - sc.req.invalidHeader = true - return - } - if *dst != "" { - sc.logf("duplicate pseudo-header %q sent", f.Name) - sc.req.invalidHeader = true - return - } - *dst = f.Value - case !http2validHeaderFieldName(f.Name): - sc.req.invalidHeader = true - default: - sc.req.sawRegularHeader = true - sc.req.header.Add(sc.canonicalHeader(f.Name), f.Value) - const headerFieldOverhead = 32 // per spec - sc.req.headerListSize += int64(len(f.Name)) + int64(len(f.Value)) + headerFieldOverhead - if sc.req.headerListSize > int64(sc.maxHeaderListSize()) { - sc.hpackDecoder.SetEmitEnabled(false) - } - } -} - -func (st *http2stream) onNewTrailerField(f hpack.HeaderField) { - sc := st.sc - sc.serveG.check() - if http2VerboseLogs { - sc.vlogf("http2: server decoded trailer %v", f) - } - switch { - case strings.HasPrefix(f.Name, ":"): - sc.req.invalidHeader = true - return - case !http2validHeaderFieldName(f.Name) || !http2validHeaderFieldValue(f.Value): - sc.req.invalidHeader = true - return - default: - key := sc.canonicalHeader(f.Name) - if st.trailer != nil { - vv := append(st.trailer[key], f.Value) - st.trailer[key] = vv - - // arbitrary; TODO: read spec about header list size limits wrt trailers - const tooBig = 1000 - if len(vv) >= tooBig { - sc.hpackDecoder.SetEmitEnabled(false) - } - } - } -} - func (sc *http2serverConn) canonicalHeader(v string) string { sc.serveG.check() cv, ok := http2commonCanonHeader[v] @@ -3063,10 +3438,11 @@ type http2readFrameResult struct { // It's run on its own goroutine. func (sc *http2serverConn) readFrames() { gate := make(http2gate) + gateDone := gate.Done for { f, err := sc.framer.ReadFrame() select { - case sc.readFrameCh <- http2readFrameResult{f, err, gate.Done}: + case sc.readFrameCh <- http2readFrameResult{f, err, gateDone}: case <-sc.doneServing: return } @@ -3083,17 +3459,17 @@ func (sc *http2serverConn) readFrames() { // frameWriteResult is the message passed from writeFrameAsync to the serve goroutine. type http2frameWriteResult struct { - wm http2frameWriteMsg // what was written (or attempted) - err error // result of the writeFrame call + wr http2FrameWriteRequest // what was written (or attempted) + err error // result of the writeFrame call } // writeFrameAsync runs in its own goroutine and writes a single frame // and then reports when it's done. // At most one goroutine can be running writeFrameAsync at a time per // serverConn. -func (sc *http2serverConn) writeFrameAsync(wm http2frameWriteMsg) { - err := wm.write.writeFrame(sc) - sc.wroteFrameCh <- http2frameWriteResult{wm, err} +func (sc *http2serverConn) writeFrameAsync(wr http2FrameWriteRequest) { + err := wr.write.writeFrame(sc) + sc.wroteFrameCh <- http2frameWriteResult{wr, err} } func (sc *http2serverConn) closeAllStreamsOnConnClose() { @@ -3137,7 +3513,7 @@ func (sc *http2serverConn) serve() { sc.vlogf("http2: server connection from %v on %p", sc.conn.RemoteAddr(), sc.hs) } - sc.writeFrame(http2frameWriteMsg{ + sc.writeFrame(http2FrameWriteRequest{ write: http2writeSettings{ {http2SettingMaxFrameSize, sc.srv.maxReadFrameSize()}, {http2SettingMaxConcurrentStreams, sc.advMaxStreams}, @@ -3154,6 +3530,17 @@ func (sc *http2serverConn) serve() { sc.setConnState(StateActive) sc.setConnState(StateIdle) + if sc.srv.IdleTimeout != 0 { + sc.idleTimer = time.NewTimer(sc.srv.IdleTimeout) + defer sc.idleTimer.Stop() + sc.idleTimerCh = sc.idleTimer.C + } + + var gracefulShutdownCh <-chan struct{} + if sc.hs != nil { + gracefulShutdownCh = http2h1ServerShutdownChan(sc.hs) + } + go sc.readFrames() settingsTimer := time.NewTimer(http2firstSettingsTimeout) @@ -3161,8 +3548,10 @@ func (sc *http2serverConn) serve() { for { loopNum++ select { - case wm := <-sc.wantWriteFrameCh: - sc.writeFrame(wm) + case wr := <-sc.wantWriteFrameCh: + sc.writeFrame(wr) + case spr := <-sc.wantStartPushCh: + sc.startPush(spr) case res := <-sc.wroteFrameCh: sc.wroteFrame(res) case res := <-sc.readFrameCh: @@ -3179,12 +3568,22 @@ func (sc *http2serverConn) serve() { case <-settingsTimer.C: sc.logf("timeout waiting for SETTINGS frames from %v", sc.conn.RemoteAddr()) return + case <-gracefulShutdownCh: + gracefulShutdownCh = nil + sc.startGracefulShutdown() case <-sc.shutdownTimerCh: sc.vlogf("GOAWAY close timer fired; closing conn from %v", sc.conn.RemoteAddr()) return + case <-sc.idleTimerCh: + sc.vlogf("connection is idle") + sc.goAway(http2ErrCodeNo) case fn := <-sc.testHookCh: fn(loopNum) } + + if sc.inGoAway && sc.curOpenStreams() == 0 && !sc.needToSendGoAway && !sc.writingFrame { + return + } } } @@ -3232,7 +3631,7 @@ func (sc *http2serverConn) writeDataFromHandler(stream *http2stream, data []byte ch := http2errChanPool.Get().(chan error) writeArg := http2writeDataPool.Get().(*http2writeData) *writeArg = http2writeData{stream.id, data, endStream} - err := sc.writeFrameFromHandler(http2frameWriteMsg{ + err := sc.writeFrameFromHandler(http2FrameWriteRequest{ write: writeArg, stream: stream, done: ch, @@ -3262,17 +3661,17 @@ func (sc *http2serverConn) writeDataFromHandler(stream *http2stream, data []byte return err } -// writeFrameFromHandler sends wm to sc.wantWriteFrameCh, but aborts +// writeFrameFromHandler sends wr to sc.wantWriteFrameCh, but aborts // if the connection has gone away. // // This must not be run from the serve goroutine itself, else it might // deadlock writing to sc.wantWriteFrameCh (which is only mildly // buffered and is read by serve itself). If you're on the serve // goroutine, call writeFrame instead. -func (sc *http2serverConn) writeFrameFromHandler(wm http2frameWriteMsg) error { +func (sc *http2serverConn) writeFrameFromHandler(wr http2FrameWriteRequest) error { sc.serveG.checkNotOn() select { - case sc.wantWriteFrameCh <- wm: + case sc.wantWriteFrameCh <- wr: return nil case <-sc.doneServing: @@ -3288,39 +3687,81 @@ func (sc *http2serverConn) writeFrameFromHandler(wm http2frameWriteMsg) error { // make it onto the wire // // If you're not on the serve goroutine, use writeFrameFromHandler instead. -func (sc *http2serverConn) writeFrame(wm http2frameWriteMsg) { +func (sc *http2serverConn) writeFrame(wr http2FrameWriteRequest) { sc.serveG.check() - sc.writeSched.add(wm) + + // If true, wr will not be written and wr.done will not be signaled. + var ignoreWrite bool + + if wr.StreamID() != 0 { + _, isReset := wr.write.(http2StreamError) + if state, _ := sc.state(wr.StreamID()); state == http2stateClosed && !isReset { + ignoreWrite = true + } + } + + switch wr.write.(type) { + case *http2writeResHeaders: + wr.stream.wroteHeaders = true + case http2write100ContinueHeadersFrame: + if wr.stream.wroteHeaders { + + if wr.done != nil { + panic("wr.done != nil for write100ContinueHeadersFrame") + } + ignoreWrite = true + } + } + + if !ignoreWrite { + sc.writeSched.Push(wr) + } sc.scheduleFrameWrite() } -// startFrameWrite starts a goroutine to write wm (in a separate +// startFrameWrite starts a goroutine to write wr (in a separate // goroutine since that might block on the network), and updates the -// serve goroutine's state about the world, updated from info in wm. -func (sc *http2serverConn) startFrameWrite(wm http2frameWriteMsg) { +// serve goroutine's state about the world, updated from info in wr. +func (sc *http2serverConn) startFrameWrite(wr http2FrameWriteRequest) { sc.serveG.check() if sc.writingFrame { panic("internal error: can only be writing one frame at a time") } - st := wm.stream + st := wr.stream if st != nil { switch st.state { case http2stateHalfClosedLocal: - panic("internal error: attempt to send frame on half-closed-local stream") - case http2stateClosed: - if st.sentReset || st.gotReset { + switch wr.write.(type) { + case http2StreamError, http2handlerPanicRST, http2writeWindowUpdate: - sc.scheduleFrameWrite() - return + default: + panic(fmt.Sprintf("internal error: attempt to send frame on a half-closed-local stream: %v", wr)) } - panic(fmt.Sprintf("internal error: attempt to send a write %v on a closed stream", wm)) + case http2stateClosed: + panic(fmt.Sprintf("internal error: attempt to send frame on a closed stream: %v", wr)) + } + } + if wpp, ok := wr.write.(*http2writePushPromise); ok { + var err error + wpp.promisedID, err = wpp.allocatePromisedID() + if err != nil { + sc.writingFrameAsync = false + wr.replyToWriter(err) + return } } sc.writingFrame = true sc.needsFrameFlush = true - go sc.writeFrameAsync(wm) + if wr.write.staysWithinBuffer(sc.bw.Available()) { + sc.writingFrameAsync = false + err := wr.write.writeFrame(sc) + sc.wroteFrame(http2frameWriteResult{wr, err}) + } else { + sc.writingFrameAsync = true + go sc.writeFrameAsync(wr) + } } // errHandlerPanicked is the error given to any callers blocked in a read from @@ -3336,26 +3777,12 @@ func (sc *http2serverConn) wroteFrame(res http2frameWriteResult) { panic("internal error: expected to be already writing a frame") } sc.writingFrame = false + sc.writingFrameAsync = false - wm := res.wm - st := wm.stream - - closeStream := http2endsStream(wm.write) - - if _, ok := wm.write.(http2handlerPanicRST); ok { - sc.closeStream(st, http2errHandlerPanicked) - } + wr := res.wr - if ch := wm.done; ch != nil { - select { - case ch <- res.err: - default: - panic(fmt.Sprintf("unbuffered done channel passed in for type %T", wm.write)) - } - } - wm.write = nil - - if closeStream { + if http2writeEndsStream(wr.write) { + st := wr.stream if st == nil { panic("internal error: expecting non-nil stream") } @@ -3363,13 +3790,24 @@ func (sc *http2serverConn) wroteFrame(res http2frameWriteResult) { case http2stateOpen: st.state = http2stateHalfClosedLocal - errCancel := http2StreamError{st.id, http2ErrCodeCancel} - sc.resetStream(errCancel) + sc.resetStream(http2streamError(st.id, http2ErrCodeCancel)) case http2stateHalfClosedRemote: sc.closeStream(st, http2errHandlerComplete) } + } else { + switch v := wr.write.(type) { + case http2StreamError: + + if st, ok := sc.streams[v.StreamID]; ok { + sc.closeStream(st, v) + } + case http2handlerPanicRST: + sc.closeStream(wr.stream, http2errHandlerPanicked) + } } + wr.replyToWriter(res.err) + sc.scheduleFrameWrite() } @@ -3387,47 +3825,68 @@ func (sc *http2serverConn) wroteFrame(res http2frameWriteResult) { // flush the write buffer. func (sc *http2serverConn) scheduleFrameWrite() { sc.serveG.check() - if sc.writingFrame { + if sc.writingFrame || sc.inFrameScheduleLoop { return } - if sc.needToSendGoAway { - sc.needToSendGoAway = false - sc.startFrameWrite(http2frameWriteMsg{ - write: &http2writeGoAway{ - maxStreamID: sc.maxStreamID, - code: sc.goAwayCode, - }, - }) - return - } - if sc.needToSendSettingsAck { - sc.needToSendSettingsAck = false - sc.startFrameWrite(http2frameWriteMsg{write: http2writeSettingsAck{}}) - return - } - if !sc.inGoAway { - if wm, ok := sc.writeSched.take(); ok { - sc.startFrameWrite(wm) - return + sc.inFrameScheduleLoop = true + for !sc.writingFrameAsync { + if sc.needToSendGoAway { + sc.needToSendGoAway = false + sc.startFrameWrite(http2FrameWriteRequest{ + write: &http2writeGoAway{ + maxStreamID: sc.maxClientStreamID, + code: sc.goAwayCode, + }, + }) + continue } + if sc.needToSendSettingsAck { + sc.needToSendSettingsAck = false + sc.startFrameWrite(http2FrameWriteRequest{write: http2writeSettingsAck{}}) + continue + } + if !sc.inGoAway || sc.goAwayCode == http2ErrCodeNo { + if wr, ok := sc.writeSched.Pop(); ok { + sc.startFrameWrite(wr) + continue + } + } + if sc.needsFrameFlush { + sc.startFrameWrite(http2FrameWriteRequest{write: http2flushFrameWriter{}}) + sc.needsFrameFlush = false + continue + } + break } - if sc.needsFrameFlush { - sc.startFrameWrite(http2frameWriteMsg{write: http2flushFrameWriter{}}) - sc.needsFrameFlush = false - return - } + sc.inFrameScheduleLoop = false +} + +// startGracefulShutdown sends a GOAWAY with ErrCodeNo to tell the +// client we're gracefully shutting down. The connection isn't closed +// until all current streams are done. +func (sc *http2serverConn) startGracefulShutdown() { + sc.goAwayIn(http2ErrCodeNo, 0) } func (sc *http2serverConn) goAway(code http2ErrCode) { sc.serveG.check() - if sc.inGoAway { - return - } + var forceCloseIn time.Duration if code != http2ErrCodeNo { - sc.shutDownIn(250 * time.Millisecond) + forceCloseIn = 250 * time.Millisecond } else { - sc.shutDownIn(1 * time.Second) + forceCloseIn = 1 * time.Second + } + sc.goAwayIn(code, forceCloseIn) +} + +func (sc *http2serverConn) goAwayIn(code http2ErrCode, forceCloseIn time.Duration) { + sc.serveG.check() + if sc.inGoAway { + return + } + if forceCloseIn != 0 { + sc.shutDownIn(forceCloseIn) } sc.inGoAway = true sc.needToSendGoAway = true @@ -3443,10 +3902,9 @@ func (sc *http2serverConn) shutDownIn(d time.Duration) { func (sc *http2serverConn) resetStream(se http2StreamError) { sc.serveG.check() - sc.writeFrame(http2frameWriteMsg{write: se}) + sc.writeFrame(http2FrameWriteRequest{write: se}) if st, ok := sc.streams[se.StreamID]; ok { - st.sentReset = true - sc.closeStream(st, se) + st.resetQueued = true } } @@ -3511,10 +3969,8 @@ func (sc *http2serverConn) processFrame(f http2Frame) error { switch f := f.(type) { case *http2SettingsFrame: return sc.processSettings(f) - case *http2HeadersFrame: + case *http2MetaHeadersFrame: return sc.processHeaders(f) - case *http2ContinuationFrame: - return sc.processContinuation(f) case *http2WindowUpdateFrame: return sc.processWindowUpdate(f) case *http2PingFrame: @@ -3525,6 +3981,8 @@ func (sc *http2serverConn) processFrame(f http2Frame) error { return sc.processResetStream(f) case *http2PriorityFrame: return sc.processPriority(f) + case *http2GoAwayFrame: + return sc.processGoAway(f) case *http2PushPromiseFrame: return http2ConnectionError(http2ErrCodeProtocol) @@ -3544,7 +4002,10 @@ func (sc *http2serverConn) processPing(f *http2PingFrame) error { return http2ConnectionError(http2ErrCodeProtocol) } - sc.writeFrame(http2frameWriteMsg{write: http2writePingAck{f}}) + if sc.inGoAway && sc.goAwayCode != http2ErrCodeNo { + return nil + } + sc.writeFrame(http2FrameWriteRequest{write: http2writePingAck{f}}) return nil } @@ -3552,13 +4013,17 @@ func (sc *http2serverConn) processWindowUpdate(f *http2WindowUpdateFrame) error sc.serveG.check() switch { case f.StreamID != 0: - st := sc.streams[f.StreamID] + state, st := sc.state(f.StreamID) + if state == http2stateIdle { + + return http2ConnectionError(http2ErrCodeProtocol) + } if st == nil { return nil } if !st.flow.add(int32(f.Increment)) { - return http2StreamError{f.StreamID, http2ErrCodeFlowControl} + return http2streamError(f.StreamID, http2ErrCodeFlowControl) } default: if !sc.flow.add(int32(f.Increment)) { @@ -3578,8 +4043,8 @@ func (sc *http2serverConn) processResetStream(f *http2RSTStreamFrame) error { return http2ConnectionError(http2ErrCodeProtocol) } if st != nil { - st.gotReset = true - sc.closeStream(st, http2StreamError{f.StreamID, f.ErrCode}) + st.cancelCtx() + sc.closeStream(st, http2streamError(f.StreamID, f.ErrCode)) } return nil } @@ -3590,16 +4055,29 @@ func (sc *http2serverConn) closeStream(st *http2stream, err error) { panic(fmt.Sprintf("invariant; can't close stream in state %v", st.state)) } st.state = http2stateClosed - sc.curOpenStreams-- - if sc.curOpenStreams == 0 { - sc.setConnState(StateIdle) + if st.isPushed() { + sc.curPushedStreams-- + } else { + sc.curClientStreams-- } delete(sc.streams, st.id) + if len(sc.streams) == 0 { + sc.setConnState(StateIdle) + if sc.srv.IdleTimeout != 0 { + sc.idleTimer.Reset(sc.srv.IdleTimeout) + } + if http2h1ServerKeepAlivesDisabled(sc.hs) { + sc.startGracefulShutdown() + } + } if p := st.body; p != nil { + + sc.sendWindowUpdate(nil, p.Len()) + p.CloseWithError(err) } st.cw.Close() - sc.writeSched.forgetStream(st.id) + sc.writeSched.CloseStream(st.id) } func (sc *http2serverConn) processSettings(f *http2SettingsFrame) error { @@ -3639,7 +4117,7 @@ func (sc *http2serverConn) processSetting(s http2Setting) error { case http2SettingInitialWindowSize: return sc.processSettingInitialWindowSize(s.Val) case http2SettingMaxFrameSize: - sc.writeSched.maxFrameSize = s.Val + sc.maxFrameSize = int32(s.Val) case http2SettingMaxHeaderListSize: sc.peerMaxHeaderListSize = s.Val default: @@ -3668,36 +4146,62 @@ func (sc *http2serverConn) processSettingInitialWindowSize(val uint32) error { func (sc *http2serverConn) processData(f *http2DataFrame) error { sc.serveG.check() + if sc.inGoAway && sc.goAwayCode != http2ErrCodeNo { + return nil + } + data := f.Data() id := f.Header().StreamID - st, ok := sc.streams[id] - if !ok || st.state != http2stateOpen || st.gotTrailerHeader { + state, st := sc.state(id) + if id == 0 || state == http2stateIdle { + + return http2ConnectionError(http2ErrCodeProtocol) + } + if st == nil || state != http2stateOpen || st.gotTrailerHeader || st.resetQueued { - return http2StreamError{id, http2ErrCodeStreamClosed} + if sc.inflow.available() < int32(f.Length) { + return http2streamError(id, http2ErrCodeFlowControl) + } + + sc.inflow.take(int32(f.Length)) + sc.sendWindowUpdate(nil, int(f.Length)) + + if st != nil && st.resetQueued { + + return nil + } + return http2streamError(id, http2ErrCodeStreamClosed) } if st.body == nil { panic("internal error: should have a body in this state") } - data := f.Data() if st.declBodyBytes != -1 && st.bodyBytes+int64(len(data)) > st.declBodyBytes { st.body.CloseWithError(fmt.Errorf("sender tried to send more than declared Content-Length of %d bytes", st.declBodyBytes)) - return http2StreamError{id, http2ErrCodeStreamClosed} + return http2streamError(id, http2ErrCodeStreamClosed) } - if len(data) > 0 { + if f.Length > 0 { - if int(st.inflow.available()) < len(data) { - return http2StreamError{id, http2ErrCodeFlowControl} + if st.inflow.available() < int32(f.Length) { + return http2streamError(id, http2ErrCodeFlowControl) } - st.inflow.take(int32(len(data))) - wrote, err := st.body.Write(data) - if err != nil { - return http2StreamError{id, http2ErrCodeStreamClosed} + st.inflow.take(int32(f.Length)) + + if len(data) > 0 { + wrote, err := st.body.Write(data) + if err != nil { + return http2streamError(id, http2ErrCodeStreamClosed) + } + if wrote != len(data) { + panic("internal error: bad Writer") + } + st.bodyBytes += int64(len(data)) } - if wrote != len(data) { - panic("internal error: bad Writer") + + if pad := int32(f.Length) - int32(len(data)); pad > 0 { + sc.sendWindowUpdate32(nil, pad) + sc.sendWindowUpdate32(st, pad) } - st.bodyBytes += int64(len(data)) } if f.StreamEnded() { st.endStream() @@ -3705,6 +4209,24 @@ func (sc *http2serverConn) processData(f *http2DataFrame) error { return nil } +func (sc *http2serverConn) processGoAway(f *http2GoAwayFrame) error { + sc.serveG.check() + if f.ErrCode != http2ErrCodeNo { + sc.logf("http2: received GOAWAY %+v, starting graceful shutdown", f) + } else { + sc.vlogf("http2: received GOAWAY %+v, starting graceful shutdown", f) + } + sc.startGracefulShutdown() + + sc.pushEnabled = false + return nil +} + +// isPushed reports whether the stream is server-initiated. +func (st *http2stream) isPushed() bool { + return st.id%2 == 0 +} + // endStream closes a Request.Body's pipe. It is called when a DATA // frame says a request body is over (or after trailers). func (st *http2stream) endStream() { @@ -3732,9 +4254,9 @@ func (st *http2stream) copyTrailersToHandlerRequest() { } } -func (sc *http2serverConn) processHeaders(f *http2HeadersFrame) error { +func (sc *http2serverConn) processHeaders(f *http2MetaHeadersFrame) error { sc.serveG.check() - id := f.Header().StreamID + id := f.StreamID if sc.inGoAway { return nil @@ -3744,96 +4266,46 @@ func (sc *http2serverConn) processHeaders(f *http2HeadersFrame) error { return http2ConnectionError(http2ErrCodeProtocol) } - st := sc.streams[f.Header().StreamID] - if st != nil { + if st := sc.streams[f.StreamID]; st != nil { + if st.resetQueued { + + return nil + } return st.processTrailerHeaders(f) } - if id <= sc.maxStreamID || sc.req.stream != nil { + if id <= sc.maxClientStreamID { return http2ConnectionError(http2ErrCodeProtocol) } + sc.maxClientStreamID = id - if id > sc.maxStreamID { - sc.maxStreamID = id - } - st = &http2stream{ - sc: sc, - id: id, - state: http2stateOpen, + if sc.idleTimer != nil { + sc.idleTimer.Stop() } - if f.StreamEnded() { - st.state = http2stateHalfClosedRemote - } - st.cw.Init() - st.flow.conn = &sc.flow - st.flow.add(sc.initialWindowSize) - st.inflow.conn = &sc.inflow - st.inflow.add(http2initialWindowSize) - - sc.streams[id] = st - if f.HasPriority() { - http2adjustStreamPriority(sc.streams, st.id, f.Priority) - } - sc.curOpenStreams++ - if sc.curOpenStreams == 1 { - sc.setConnState(StateActive) - } - sc.req = http2requestParam{ - stream: st, - header: make(Header), - } - sc.hpackDecoder.SetEmitFunc(sc.onNewHeaderField) - sc.hpackDecoder.SetEmitEnabled(true) - return sc.processHeaderBlockFragment(st, f.HeaderBlockFragment(), f.HeadersEnded()) -} + if sc.curClientStreams+1 > sc.advMaxStreams { + if sc.unackedSettings == 0 { -func (st *http2stream) processTrailerHeaders(f *http2HeadersFrame) error { - sc := st.sc - sc.serveG.check() - if st.gotTrailerHeader { - return http2ConnectionError(http2ErrCodeProtocol) - } - st.gotTrailerHeader = true - if !f.StreamEnded() { - return http2StreamError{st.id, http2ErrCodeProtocol} - } - sc.resetPendingRequest() - return st.processTrailerHeaderBlockFragment(f.HeaderBlockFragment(), f.HeadersEnded()) -} + return http2streamError(id, http2ErrCodeProtocol) + } -func (sc *http2serverConn) processContinuation(f *http2ContinuationFrame) error { - sc.serveG.check() - st := sc.streams[f.Header().StreamID] - if st.gotTrailerHeader { - return st.processTrailerHeaderBlockFragment(f.HeaderBlockFragment(), f.HeadersEnded()) + return http2streamError(id, http2ErrCodeRefusedStream) } - return sc.processHeaderBlockFragment(st, f.HeaderBlockFragment(), f.HeadersEnded()) -} -func (sc *http2serverConn) processHeaderBlockFragment(st *http2stream, frag []byte, end bool) error { - sc.serveG.check() - if _, err := sc.hpackDecoder.Write(frag); err != nil { - return http2ConnectionError(http2ErrCodeCompression) - } - if !end { - return nil - } - if err := sc.hpackDecoder.Close(); err != nil { - return http2ConnectionError(http2ErrCodeCompression) + initialState := http2stateOpen + if f.StreamEnded() { + initialState = http2stateHalfClosedRemote } - defer sc.resetPendingRequest() - if sc.curOpenStreams > sc.advMaxStreams { - - if sc.unackedSettings == 0 { + st := sc.newStream(id, 0, initialState) - return http2StreamError{st.id, http2ErrCodeProtocol} + if f.HasPriority() { + if err := http2checkPriority(f.StreamID, f.Priority); err != nil { + return err } - - return http2StreamError{st.id, http2ErrCodeRefusedStream} + sc.writeSched.AdjustStream(st.id, f.Priority) } - rw, req, err := sc.newWriterAndRequest() + rw, req, err := sc.newWriterAndRequest(st, f) if err != nil { return err } @@ -3845,114 +4317,169 @@ func (sc *http2serverConn) processHeaderBlockFragment(st *http2stream, frag []by st.declBodyBytes = req.ContentLength handler := sc.handler.ServeHTTP - if !sc.hpackDecoder.EmitEnabled() { + if f.Truncated { handler = http2handleHeaderListTooLong + } else if err := http2checkValidHTTP2RequestHeaders(req.Header); err != nil { + handler = http2new400Handler(err) + } + + if sc.hs.ReadTimeout != 0 { + sc.conn.SetReadDeadline(time.Time{}) } go sc.runHandler(rw, req, handler) return nil } -func (st *http2stream) processTrailerHeaderBlockFragment(frag []byte, end bool) error { +func (st *http2stream) processTrailerHeaders(f *http2MetaHeadersFrame) error { sc := st.sc sc.serveG.check() - sc.hpackDecoder.SetEmitFunc(st.onNewTrailerField) - if _, err := sc.hpackDecoder.Write(frag); err != nil { - return http2ConnectionError(http2ErrCodeCompression) + if st.gotTrailerHeader { + return http2ConnectionError(http2ErrCodeProtocol) } - if !end { - return nil + st.gotTrailerHeader = true + if !f.StreamEnded() { + return http2streamError(st.id, http2ErrCodeProtocol) } - rp := &sc.req - if rp.invalidHeader { - return http2StreamError{rp.stream.id, http2ErrCodeProtocol} + if len(f.PseudoFields()) > 0 { + return http2streamError(st.id, http2ErrCodeProtocol) } + if st.trailer != nil { + for _, hf := range f.RegularFields() { + key := sc.canonicalHeader(hf.Name) + if !http2ValidTrailerHeader(key) { - err := sc.hpackDecoder.Close() + return http2streamError(st.id, http2ErrCodeProtocol) + } + st.trailer[key] = append(st.trailer[key], hf.Value) + } + } st.endStream() - if err != nil { - return http2ConnectionError(http2ErrCodeCompression) + return nil +} + +func http2checkPriority(streamID uint32, p http2PriorityParam) error { + if streamID == p.StreamDep { + + return http2streamError(streamID, http2ErrCodeProtocol) } return nil } func (sc *http2serverConn) processPriority(f *http2PriorityFrame) error { - http2adjustStreamPriority(sc.streams, f.StreamID, f.http2PriorityParam) + if sc.inGoAway { + return nil + } + if err := http2checkPriority(f.StreamID, f.http2PriorityParam); err != nil { + return err + } + sc.writeSched.AdjustStream(f.StreamID, f.http2PriorityParam) return nil } -func http2adjustStreamPriority(streams map[uint32]*http2stream, streamID uint32, priority http2PriorityParam) { - st, ok := streams[streamID] - if !ok { - - return +func (sc *http2serverConn) newStream(id, pusherID uint32, state http2streamState) *http2stream { + sc.serveG.check() + if id == 0 { + panic("internal error: cannot create stream with id 0") } - st.weight = priority.Weight - parent := streams[priority.StreamDep] - if parent == st { - return + ctx, cancelCtx := http2contextWithCancel(sc.baseCtx) + st := &http2stream{ + sc: sc, + id: id, + state: state, + ctx: ctx, + cancelCtx: cancelCtx, } + st.cw.Init() + st.flow.conn = &sc.flow + st.flow.add(sc.initialWindowSize) + st.inflow.conn = &sc.inflow + st.inflow.add(http2initialWindowSize) - for piter := parent; piter != nil; piter = piter.parent { - if piter == st { - parent.parent = st.parent - break - } + sc.streams[id] = st + sc.writeSched.OpenStream(st.id, http2OpenStreamOptions{PusherID: pusherID}) + if st.isPushed() { + sc.curPushedStreams++ + } else { + sc.curClientStreams++ } - st.parent = parent - if priority.Exclusive && (st.parent != nil || priority.StreamDep == 0) { - for _, openStream := range streams { - if openStream != st && openStream.parent == st.parent { - openStream.parent = st - } - } + if sc.curOpenStreams() == 1 { + sc.setConnState(StateActive) } -} -// resetPendingRequest zeros out all state related to a HEADERS frame -// and its zero or more CONTINUATION frames sent to start a new -// request. -func (sc *http2serverConn) resetPendingRequest() { - sc.serveG.check() - sc.req = http2requestParam{} + return st } -func (sc *http2serverConn) newWriterAndRequest() (*http2responseWriter, *Request, error) { +func (sc *http2serverConn) newWriterAndRequest(st *http2stream, f *http2MetaHeadersFrame) (*http2responseWriter, *Request, error) { sc.serveG.check() - rp := &sc.req - if rp.invalidHeader { - return nil, nil, http2StreamError{rp.stream.id, http2ErrCodeProtocol} + rp := http2requestParam{ + method: f.PseudoValue("method"), + scheme: f.PseudoValue("scheme"), + authority: f.PseudoValue("authority"), + path: f.PseudoValue("path"), } isConnect := rp.method == "CONNECT" if isConnect { if rp.path != "" || rp.scheme != "" || rp.authority == "" { - return nil, nil, http2StreamError{rp.stream.id, http2ErrCodeProtocol} + return nil, nil, http2streamError(f.StreamID, http2ErrCodeProtocol) } - } else if rp.method == "" || rp.path == "" || - (rp.scheme != "https" && rp.scheme != "http") { + } else if rp.method == "" || rp.path == "" || (rp.scheme != "https" && rp.scheme != "http") { - return nil, nil, http2StreamError{rp.stream.id, http2ErrCodeProtocol} + return nil, nil, http2streamError(f.StreamID, http2ErrCodeProtocol) } - bodyOpen := rp.stream.state == http2stateOpen + bodyOpen := !f.StreamEnded() if rp.method == "HEAD" && bodyOpen { - return nil, nil, http2StreamError{rp.stream.id, http2ErrCodeProtocol} + return nil, nil, http2streamError(f.StreamID, http2ErrCodeProtocol) + } + + rp.header = make(Header) + for _, hf := range f.RegularFields() { + rp.header.Add(sc.canonicalHeader(hf.Name), hf.Value) + } + if rp.authority == "" { + rp.authority = rp.header.Get("Host") } - var tlsState *tls.ConnectionState // nil if not scheme https + rw, req, err := sc.newWriterAndRequestNoBody(st, rp) + if err != nil { + return nil, nil, err + } + if bodyOpen { + st.reqBuf = http2getRequestBodyBuf() + req.Body.(*http2requestBody).pipe = &http2pipe{ + b: &http2fixedBuffer{buf: st.reqBuf}, + } + + if vv, ok := rp.header["Content-Length"]; ok { + req.ContentLength, _ = strconv.ParseInt(vv[0], 10, 64) + } else { + req.ContentLength = -1 + } + } + return rw, req, nil +} + +type http2requestParam struct { + method string + scheme, authority, path string + header Header +} + +func (sc *http2serverConn) newWriterAndRequestNoBody(st *http2stream, rp http2requestParam) (*http2responseWriter, *Request, error) { + sc.serveG.check() + + var tlsState *tls.ConnectionState // nil if not scheme https if rp.scheme == "https" { tlsState = sc.tlsState } - authority := rp.authority - if authority == "" { - authority = rp.header.Get("Host") - } + needsContinue := rp.header.Get("Expect") == "100-continue" if needsContinue { rp.header.Del("Expect") @@ -3980,24 +4507,25 @@ func (sc *http2serverConn) newWriterAndRequest() (*http2responseWriter, *Request } delete(rp.header, "Trailer") - body := &http2requestBody{ - conn: sc, - stream: rp.stream, - needsContinue: needsContinue, - } var url_ *url.URL var requestURI string - if isConnect { + if rp.method == "CONNECT" { url_ = &url.URL{Host: rp.authority} requestURI = rp.authority } else { var err error url_, err = url.ParseRequestURI(rp.path) if err != nil { - return nil, nil, http2StreamError{rp.stream.id, http2ErrCodeProtocol} + return nil, nil, http2streamError(st.id, http2ErrCodeProtocol) } requestURI = rp.path } + + body := &http2requestBody{ + conn: sc, + stream: st, + needsContinue: needsContinue, + } req := &Request{ Method: rp.method, URL: url_, @@ -4008,21 +4536,11 @@ func (sc *http2serverConn) newWriterAndRequest() (*http2responseWriter, *Request ProtoMajor: 2, ProtoMinor: 0, TLS: tlsState, - Host: authority, + Host: rp.authority, Body: body, Trailer: trailer, } - if bodyOpen { - body.pipe = &http2pipe{ - b: &http2fixedBuffer{buf: make([]byte, http2initialWindowSize)}, - } - - if vv, ok := rp.header["Content-Length"]; ok { - req.ContentLength, _ = strconv.ParseInt(vv[0], 10, 64) - } else { - req.ContentLength = -1 - } - } + req = http2requestWithContext(req, st.ctx) rws := http2responseWriterStatePool.Get().(*http2responseWriterState) bwSave := rws.bw @@ -4030,7 +4548,7 @@ func (sc *http2serverConn) newWriterAndRequest() (*http2responseWriter, *Request rws.conn = sc rws.bw = bwSave rws.bw.Reset(http2chunkWriter{rws}) - rws.stream = rp.stream + rws.stream = st rws.req = req rws.body = body @@ -4038,21 +4556,42 @@ func (sc *http2serverConn) newWriterAndRequest() (*http2responseWriter, *Request return rw, req, nil } +var http2reqBodyCache = make(chan []byte, 8) + +func http2getRequestBodyBuf() []byte { + select { + case b := <-http2reqBodyCache: + return b + default: + return make([]byte, http2initialWindowSize) + } +} + +func http2putRequestBodyBuf(b []byte) { + select { + case http2reqBodyCache <- b: + default: + } +} + // Run on its own goroutine. func (sc *http2serverConn) runHandler(rw *http2responseWriter, req *Request, handler func(ResponseWriter, *Request)) { didPanic := true defer func() { + rw.rws.stream.cancelCtx() if didPanic { e := recover() - // Same as net/http: - const size = 64 << 10 - buf := make([]byte, size) - buf = buf[:runtime.Stack(buf, false)] - sc.writeFrameFromHandler(http2frameWriteMsg{ + sc.writeFrameFromHandler(http2FrameWriteRequest{ write: http2handlerPanicRST{rw.rws.stream.id}, stream: rw.rws.stream, }) - sc.logf("http2: panic serving %v: %v\n%s", sc.conn.RemoteAddr(), e, buf) + + if http2shouldLogPanic(e) { + const size = 64 << 10 + buf := make([]byte, size) + buf = buf[:runtime.Stack(buf, false)] + sc.logf("http2: panic serving %v: %v\n%s", sc.conn.RemoteAddr(), e, buf) + } return } rw.handlerDone() @@ -4080,7 +4619,7 @@ func (sc *http2serverConn) writeHeaders(st *http2stream, headerData *http2writeR errc = http2errChanPool.Get().(chan error) } - if err := sc.writeFrameFromHandler(http2frameWriteMsg{ + if err := sc.writeFrameFromHandler(http2FrameWriteRequest{ write: headerData, stream: st, done: errc, @@ -4103,7 +4642,7 @@ func (sc *http2serverConn) writeHeaders(st *http2stream, headerData *http2writeR // called from handler goroutines. func (sc *http2serverConn) write100ContinueHeaders(st *http2stream) { - sc.writeFrameFromHandler(http2frameWriteMsg{ + sc.writeFrameFromHandler(http2FrameWriteRequest{ write: http2write100ContinueHeadersFrame{st.id}, stream: st, }) @@ -4119,11 +4658,19 @@ type http2bodyReadMsg struct { // called from handler goroutines. // Notes that the handler for the given stream ID read n bytes of its body // and schedules flow control tokens to be sent. -func (sc *http2serverConn) noteBodyReadFromHandler(st *http2stream, n int) { +func (sc *http2serverConn) noteBodyReadFromHandler(st *http2stream, n int, err error) { sc.serveG.checkNotOn() - select { - case sc.bodyReadCh <- http2bodyReadMsg{st, n}: - case <-sc.doneServing: + if n > 0 { + select { + case sc.bodyReadCh <- http2bodyReadMsg{st, n}: + case <-sc.doneServing: + } + } + if err == io.EOF { + if buf := st.reqBuf; buf != nil { + st.reqBuf = nil + http2putRequestBodyBuf(buf) + } } } @@ -4165,7 +4712,7 @@ func (sc *http2serverConn) sendWindowUpdate32(st *http2stream, n int32) { if st != nil { streamID = st.id } - sc.writeFrame(http2frameWriteMsg{ + sc.writeFrame(http2FrameWriteRequest{ write: http2writeWindowUpdate{streamID: streamID, n: uint32(n)}, stream: st, }) @@ -4180,17 +4727,20 @@ func (sc *http2serverConn) sendWindowUpdate32(st *http2stream, n int32) { } } +// requestBody is the Handler's Request.Body type. +// Read and Close may be called concurrently. type http2requestBody struct { stream *http2stream conn *http2serverConn - closed bool + closed bool // for use by Close only + sawEOF bool // for use by Read only pipe *http2pipe // non-nil if we have a HTTP entity message body needsContinue bool // need to send a 100-continue } func (b *http2requestBody) Close() error { - if b.pipe != nil { - b.pipe.CloseWithError(http2errClosedBody) + if b.pipe != nil && !b.closed { + b.pipe.BreakWithError(http2errClosedBody) } b.closed = true return nil @@ -4201,13 +4751,17 @@ func (b *http2requestBody) Read(p []byte) (n int, err error) { b.needsContinue = false b.conn.write100ContinueHeaders(b.stream) } - if b.pipe == nil { + if b.pipe == nil || b.sawEOF { return 0, io.EOF } n, err = b.pipe.Read(p) - if n > 0 { - b.conn.noteBodyReadFromHandler(b.stream, n) + if err == io.EOF { + b.sawEOF = true + } + if b.conn == nil && http2inTests { + return } + b.conn.noteBodyReadFromHandler(b.stream, n, err) return } @@ -4265,9 +4819,9 @@ func (rws *http2responseWriterState) hasTrailers() bool { return len(rws.trailer // written in the trailers at the end of the response. func (rws *http2responseWriterState) declareTrailer(k string) { k = CanonicalHeaderKey(k) - switch k { - case "Transfer-Encoding", "Content-Length", "Trailer": + if !http2ValidTrailerHeader(k) { + rws.conn.logf("ignoring invalid trailer %q", k) return } if !http2strSliceContains(rws.trailers, k) { @@ -4408,7 +4962,12 @@ func (rws *http2responseWriterState) promoteUndeclaredTrailers() { rws.declareTrailer(trailerKey) rws.handlerHeader[CanonicalHeaderKey(trailerKey)] = vv } - sort.Strings(rws.trailers) + + if len(rws.trailers) > 1 { + sorter := http2sorterPool.Get().(*http2sorter) + sorter.SortStrings(rws.trailers) + http2sorterPool.Put(sorter) + } } func (w *http2responseWriter) Flush() { @@ -4437,8 +4996,9 @@ func (w *http2responseWriter) CloseNotify() <-chan bool { if ch == nil { ch = make(chan bool, 1) rws.closeNotifierCh = ch + cw := rws.stream.cw go func() { - rws.stream.cw.Wait() + cw.Wait() ch <- true }() } @@ -4534,6 +5094,172 @@ func (w *http2responseWriter) handlerDone() { http2responseWriterStatePool.Put(rws) } +// Push errors. +var ( + http2ErrRecursivePush = errors.New("http2: recursive push not allowed") + http2ErrPushLimitReached = errors.New("http2: push would exceed peer's SETTINGS_MAX_CONCURRENT_STREAMS") +) + +// pushOptions is the internal version of http.PushOptions, which we +// cannot include here because it's only defined in Go 1.8 and later. +type http2pushOptions struct { + Method string + Header Header +} + +func (w *http2responseWriter) push(target string, opts http2pushOptions) error { + st := w.rws.stream + sc := st.sc + sc.serveG.checkNotOn() + + if st.isPushed() { + return http2ErrRecursivePush + } + + if opts.Method == "" { + opts.Method = "GET" + } + if opts.Header == nil { + opts.Header = Header{} + } + wantScheme := "http" + if w.rws.req.TLS != nil { + wantScheme = "https" + } + + u, err := url.Parse(target) + if err != nil { + return err + } + if u.Scheme == "" { + if !strings.HasPrefix(target, "/") { + return fmt.Errorf("target must be an absolute URL or an absolute path: %q", target) + } + u.Scheme = wantScheme + u.Host = w.rws.req.Host + } else { + if u.Scheme != wantScheme { + return fmt.Errorf("cannot push URL with scheme %q from request with scheme %q", u.Scheme, wantScheme) + } + if u.Host == "" { + return errors.New("URL must have a host") + } + } + for k := range opts.Header { + if strings.HasPrefix(k, ":") { + return fmt.Errorf("promised request headers cannot include pseudo header %q", k) + } + + switch strings.ToLower(k) { + case "content-length", "content-encoding", "trailer", "te", "expect", "host": + return fmt.Errorf("promised request headers cannot include %q", k) + } + } + if err := http2checkValidHTTP2RequestHeaders(opts.Header); err != nil { + return err + } + + if opts.Method != "GET" && opts.Method != "HEAD" { + return fmt.Errorf("method %q must be GET or HEAD", opts.Method) + } + + msg := http2startPushRequest{ + parent: st, + method: opts.Method, + url: u, + header: http2cloneHeader(opts.Header), + done: http2errChanPool.Get().(chan error), + } + + select { + case <-sc.doneServing: + return http2errClientDisconnected + case <-st.cw: + return http2errStreamClosed + case sc.wantStartPushCh <- msg: + } + + select { + case <-sc.doneServing: + return http2errClientDisconnected + case <-st.cw: + return http2errStreamClosed + case err := <-msg.done: + http2errChanPool.Put(msg.done) + return err + } +} + +type http2startPushRequest struct { + parent *http2stream + method string + url *url.URL + header Header + done chan error +} + +func (sc *http2serverConn) startPush(msg http2startPushRequest) { + sc.serveG.check() + + if msg.parent.state != http2stateOpen && msg.parent.state != http2stateHalfClosedRemote { + + msg.done <- http2errStreamClosed + return + } + + if !sc.pushEnabled { + msg.done <- ErrNotSupported + return + } + + allocatePromisedID := func() (uint32, error) { + sc.serveG.check() + + if !sc.pushEnabled { + return 0, ErrNotSupported + } + + if sc.curPushedStreams+1 > sc.clientMaxStreams { + return 0, http2ErrPushLimitReached + } + + if sc.maxPushPromiseID+2 >= 1<<31 { + sc.startGracefulShutdown() + return 0, http2ErrPushLimitReached + } + sc.maxPushPromiseID += 2 + promisedID := sc.maxPushPromiseID + + promised := sc.newStream(promisedID, msg.parent.id, http2stateHalfClosedRemote) + rw, req, err := sc.newWriterAndRequestNoBody(promised, http2requestParam{ + method: msg.method, + scheme: msg.url.Scheme, + authority: msg.url.Host, + path: msg.url.RequestURI(), + header: http2cloneHeader(msg.header), + }) + if err != nil { + + panic(fmt.Sprintf("newWriterAndRequestNoBody(%+v): %v", msg.url, err)) + } + + go sc.runHandler(rw, req, sc.handler.ServeHTTP) + return promisedID, nil + } + + sc.writeFrame(http2FrameWriteRequest{ + write: &http2writePushPromise{ + streamID: msg.parent.id, + method: msg.method, + url: msg.url, + h: msg.header, + allocatePromisedID: allocatePromisedID, + }, + stream: msg.parent, + done: msg.done, + }) +} + // foreachHeaderElement splits v according to the "#rule" construction // in RFC 2616 section 2.1 and calls fn for each non-empty element. func http2foreachHeaderElement(v string, fn func(string)) { @@ -4552,6 +5278,111 @@ func http2foreachHeaderElement(v string, fn func(string)) { } } +// From http://httpwg.org/specs/rfc7540.html#rfc.section.8.1.2.2 +var http2connHeaders = []string{ + "Connection", + "Keep-Alive", + "Proxy-Connection", + "Transfer-Encoding", + "Upgrade", +} + +// checkValidHTTP2RequestHeaders checks whether h is a valid HTTP/2 request, +// per RFC 7540 Section 8.1.2.2. +// The returned error is reported to users. +func http2checkValidHTTP2RequestHeaders(h Header) error { + for _, k := range http2connHeaders { + if _, ok := h[k]; ok { + return fmt.Errorf("request header %q is not valid in HTTP/2", k) + } + } + te := h["Te"] + if len(te) > 0 && (len(te) > 1 || (te[0] != "trailers" && te[0] != "")) { + return errors.New(`request header "TE" may only be "trailers" in HTTP/2`) + } + return nil +} + +func http2new400Handler(err error) HandlerFunc { + return func(w ResponseWriter, r *Request) { + Error(w, err.Error(), StatusBadRequest) + } +} + +// ValidTrailerHeader reports whether name is a valid header field name to appear +// in trailers. +// See: http://tools.ietf.org/html/rfc7230#section-4.1.2 +func http2ValidTrailerHeader(name string) bool { + name = CanonicalHeaderKey(name) + if strings.HasPrefix(name, "If-") || http2badTrailer[name] { + return false + } + return true +} + +var http2badTrailer = map[string]bool{ + "Authorization": true, + "Cache-Control": true, + "Connection": true, + "Content-Encoding": true, + "Content-Length": true, + "Content-Range": true, + "Content-Type": true, + "Expect": true, + "Host": true, + "Keep-Alive": true, + "Max-Forwards": true, + "Pragma": true, + "Proxy-Authenticate": true, + "Proxy-Authorization": true, + "Proxy-Connection": true, + "Range": true, + "Realm": true, + "Te": true, + "Trailer": true, + "Transfer-Encoding": true, + "Www-Authenticate": true, +} + +// h1ServerShutdownChan returns a channel that will be closed when the +// provided *http.Server wants to shut down. +// +// This is a somewhat hacky way to get at http1 innards. It works +// when the http2 code is bundled into the net/http package in the +// standard library. The alternatives ended up making the cmd/go tool +// depend on http Servers. This is the lightest option for now. +// This is tested via the TestServeShutdown* tests in net/http. +func http2h1ServerShutdownChan(hs *Server) <-chan struct{} { + if fn := http2testh1ServerShutdownChan; fn != nil { + return fn(hs) + } + var x interface{} = hs + type I interface { + getDoneChan() <-chan struct{} + } + if hs, ok := x.(I); ok { + return hs.getDoneChan() + } + return nil +} + +// optional test hook for h1ServerShutdownChan. +var http2testh1ServerShutdownChan func(hs *Server) <-chan struct{} + +// h1ServerKeepAlivesDisabled reports whether hs has its keep-alives +// disabled. See comments on h1ServerShutdownChan above for why +// the code is written this way. +func http2h1ServerKeepAlivesDisabled(hs *Server) bool { + var x interface{} = hs + type I interface { + doKeepAlives() bool + } + if hs, ok := x.(I); ok { + return !hs.doKeepAlives() + } + return false +} + const ( // transportDefaultConnFlow is how many connection-level flow control // tokens we give the server at start-up, past the default 64k. @@ -4601,6 +5432,10 @@ type http2Transport struct { // uncompressed. DisableCompression bool + // AllowHTTP, if true, permits HTTP/2 requests using the insecure, + // plain-text "http" scheme. Note that this does not enable h2c support. + AllowHTTP bool + // MaxHeaderListSize is the http2 SETTINGS_MAX_HEADER_LIST_SIZE to // send in the initial settings frame. It is how many bytes // of response headers are allow. Unlike the http2 spec, zero here @@ -4659,32 +5494,41 @@ func (t *http2Transport) initConnPool() { // ClientConn is the state of a single HTTP/2 client connection to an // HTTP/2 server. type http2ClientConn struct { - t *http2Transport - tconn net.Conn // usually *tls.Conn, except specialized impls - tlsState *tls.ConnectionState // nil only for specialized impls + t *http2Transport + tconn net.Conn // usually *tls.Conn, except specialized impls + tlsState *tls.ConnectionState // nil only for specialized impls + singleUse bool // whether being used for a single http.Request // readLoop goroutine fields: readerDone chan struct{} // closed on error readerErr error // set before readerDone is closed - mu sync.Mutex // guards following - cond *sync.Cond // hold mu; broadcast on flow/closed changes - flow http2flow // our conn-level flow control quota (cs.flow is per stream) - inflow http2flow // peer's conn-level flow control - closed bool - goAway *http2GoAwayFrame // if non-nil, the GoAwayFrame we received - streams map[uint32]*http2clientStream // client-initiated - nextStreamID uint32 - bw *bufio.Writer - br *bufio.Reader - fr *http2Framer - // Settings from peer: + idleTimeout time.Duration // or 0 for never + idleTimer *time.Timer + + mu sync.Mutex // guards following + cond *sync.Cond // hold mu; broadcast on flow/closed changes + flow http2flow // our conn-level flow control quota (cs.flow is per stream) + inflow http2flow // peer's conn-level flow control + closed bool + wantSettingsAck bool // we sent a SETTINGS frame and haven't heard back + goAway *http2GoAwayFrame // if non-nil, the GoAwayFrame we received + goAwayDebug string // goAway frame's debug data, retained as a string + streams map[uint32]*http2clientStream // client-initiated + nextStreamID uint32 + pings map[[8]byte]chan struct{} // in flight ping data to notification channel + bw *bufio.Writer + br *bufio.Reader + fr *http2Framer + lastActive time.Time + // Settings from peer: (also guarded by mu) maxFrameSize uint32 maxConcurrentStreams uint32 initialWindowSize uint32 - hbuf bytes.Buffer // HPACK encoder writes into this - henc *hpack.Encoder - freeBuf [][]byte + + hbuf bytes.Buffer // HPACK encoder writes into this + henc *hpack.Encoder + freeBuf [][]byte wmu sync.Mutex // held while writing; acquire AFTER mu if holding both werr error // first write error that has occurred @@ -4695,16 +5539,20 @@ type http2ClientConn struct { type http2clientStream struct { cc *http2ClientConn req *Request + trace *http2clientTrace // or nil ID uint32 resc chan http2resAndError bufPipe http2pipe // buffered pipe with the flow-controlled response payload + startedWrite bool // started request body write; guarded by cc.mu requestedGzip bool + on100 func() // optional code to run if get a 100 continue response flow http2flow // guarded by cc.mu inflow http2flow // guarded by cc.mu bytesRemain int64 // -1 means unknown; owned by transportResponseBody.Read readErr error // sticky read error; owned by transportResponseBody.Read stopReqBody error // if non-nil, stop writing req body; guarded by cc.mu + didReset bool // whether we sent a RST_STREAM to the server; guarded by cc.mu peerReset chan struct{} // closed on peer reset resetErr error // populated before peerReset is closed @@ -4712,36 +5560,54 @@ type http2clientStream struct { done chan struct{} // closed when stream remove from cc.streams map; close calls guarded by cc.mu // owned by clientConnReadLoop: - pastHeaders bool // got HEADERS w/ END_HEADERS - pastTrailers bool // got second HEADERS frame w/ END_HEADERS + firstByte bool // got the first response byte + pastHeaders bool // got first MetaHeadersFrame (actual headers) + pastTrailers bool // got optional second MetaHeadersFrame (trailers) trailer Header // accumulated trailers resTrailer *Header // client's Response.Trailer } // awaitRequestCancel runs in its own goroutine and waits for the user -// to either cancel a RoundTrip request (using the provided -// Request.Cancel channel), or for the request to be done (any way it -// might be removed from the cc.streams map: peer reset, successful -// completion, TCP connection breakage, etc) -func (cs *http2clientStream) awaitRequestCancel(cancel <-chan struct{}) { - if cancel == nil { +// to cancel a RoundTrip request, its context to expire, or for the +// request to be done (any way it might be removed from the cc.streams +// map: peer reset, successful completion, TCP connection breakage, +// etc) +func (cs *http2clientStream) awaitRequestCancel(req *Request) { + ctx := http2reqContext(req) + if req.Cancel == nil && ctx.Done() == nil { return } select { - case <-cancel: + case <-req.Cancel: + cs.cancelStream() cs.bufPipe.CloseWithError(http2errRequestCanceled) - cs.cc.writeStreamReset(cs.ID, http2ErrCodeCancel, nil) + case <-ctx.Done(): + cs.cancelStream() + cs.bufPipe.CloseWithError(ctx.Err()) case <-cs.done: } } -// checkReset reports any error sent in a RST_STREAM frame by the -// server. -func (cs *http2clientStream) checkReset() error { +func (cs *http2clientStream) cancelStream() { + cs.cc.mu.Lock() + didReset := cs.didReset + cs.didReset = true + cs.cc.mu.Unlock() + + if !didReset { + cs.cc.writeStreamReset(cs.ID, http2ErrCodeCancel, nil) + } +} + +// checkResetOrDone reports any error sent in a RST_STREAM frame by the +// server, or errStreamClosed if the stream is complete. +func (cs *http2clientStream) checkResetOrDone() error { select { case <-cs.peerReset: return cs.resetErr + case <-cs.done: + return http2errStreamClosed default: return nil } @@ -4789,29 +5655,44 @@ func (t *http2Transport) RoundTrip(req *Request) (*Response, error) { // authorityAddr returns a given authority (a host/IP, or host:port / ip:port) // and returns a host:port. The port 443 is added if needed. -func http2authorityAddr(authority string) (addr string) { - if _, _, err := net.SplitHostPort(authority); err == nil { - return authority +func http2authorityAddr(scheme string, authority string) (addr string) { + host, port, err := net.SplitHostPort(authority) + if err != nil { + port = "443" + if scheme == "http" { + port = "80" + } + host = authority + } + if a, err := idna.ToASCII(host); err == nil { + host = a } - return net.JoinHostPort(authority, "443") + + if strings.HasPrefix(host, "[") && strings.HasSuffix(host, "]") { + return host + ":" + port + } + return net.JoinHostPort(host, port) } // RoundTripOpt is like RoundTrip, but takes options. func (t *http2Transport) RoundTripOpt(req *Request, opt http2RoundTripOpt) (*Response, error) { - if req.URL.Scheme != "https" { + if !(req.URL.Scheme == "https" || (req.URL.Scheme == "http" && t.AllowHTTP)) { return nil, errors.New("http2: unsupported scheme") } - addr := http2authorityAddr(req.URL.Host) + addr := http2authorityAddr(req.URL.Scheme, req.URL.Host) for { cc, err := t.connPool().GetClientConn(req, addr) if err != nil { t.vlogf("http2: Transport failed to get client conn for %s: %v", addr, err) return nil, err } + http2traceGotConn(req, cc) res, err := cc.RoundTrip(req) - if http2shouldRetryRequest(req, err) { - continue + if err != nil { + if req, err = http2shouldRetryRequest(req, err); err == nil { + continue + } } if err != nil { t.vlogf("RoundTrip failure: %v", err) @@ -4825,7 +5706,7 @@ func (t *http2Transport) RoundTripOpt(req *Request, opt http2RoundTripOpt) (*Res // connected from previous requests but are now sitting idle. // It does not interrupt any connections currently in use. func (t *http2Transport) CloseIdleConnections() { - if cp, ok := t.connPool().(*http2clientConnPool); ok { + if cp, ok := t.connPool().(http2clientConnPoolIdleCloser); ok { cp.closeIdleConnections() } } @@ -4833,14 +5714,42 @@ func (t *http2Transport) CloseIdleConnections() { var ( http2errClientConnClosed = errors.New("http2: client conn is closed") http2errClientConnUnusable = errors.New("http2: client conn not usable") + + http2errClientConnGotGoAway = errors.New("http2: Transport received Server's graceful shutdown GOAWAY") + http2errClientConnGotGoAwayAfterSomeReqBody = errors.New("http2: Transport received Server's graceful shutdown GOAWAY; some request body already written") ) -func http2shouldRetryRequest(req *Request, err error) bool { +// shouldRetryRequest is called by RoundTrip when a request fails to get +// response headers. It is always called with a non-nil error. +// It returns either a request to retry (either the same request, or a +// modified clone), or an error if the request can't be replayed. +func http2shouldRetryRequest(req *Request, err error) (*Request, error) { + switch err { + default: + return nil, err + case http2errClientConnUnusable, http2errClientConnGotGoAway: + return req, nil + case http2errClientConnGotGoAwayAfterSomeReqBody: + + if req.Body == nil || http2reqBodyIsNoBody(req.Body) { + return req, nil + } - return err == http2errClientConnUnusable + getBody := http2reqGetBody(req) + if getBody == nil { + return nil, errors.New("http2: Transport: peer server initiated graceful shutdown after some of Request.Body was written; define Request.GetBody to avoid this error") + } + body, err := getBody() + if err != nil { + return nil, err + } + newReq := *req + newReq.Body = body + return &newReq, nil + } } -func (t *http2Transport) dialClientConn(addr string) (*http2ClientConn, error) { +func (t *http2Transport) dialClientConn(addr string, singleUse bool) (*http2ClientConn, error) { host, _, err := net.SplitHostPort(addr) if err != nil { return nil, err @@ -4849,16 +5758,20 @@ func (t *http2Transport) dialClientConn(addr string) (*http2ClientConn, error) { if err != nil { return nil, err } - return t.NewClientConn(tconn) + return t.newClientConn(tconn, singleUse) } func (t *http2Transport) newTLSConfig(host string) *tls.Config { cfg := new(tls.Config) if t.TLSClientConfig != nil { - *cfg = *t.TLSClientConfig + *cfg = *http2cloneTLSConfig(t.TLSClientConfig) + } + if !http2strSliceContains(cfg.NextProtos, http2NextProtoTLS) { + cfg.NextProtos = append([]string{http2NextProtoTLS}, cfg.NextProtos...) + } + if cfg.ServerName == "" { + cfg.ServerName = host } - cfg.NextProtos = []string{http2NextProtoTLS} - cfg.ServerName = host return cfg } @@ -4898,15 +5811,18 @@ func (t *http2Transport) disableKeepAlives() bool { return t.t1 != nil && t.t1.DisableKeepAlives } -func (t *http2Transport) NewClientConn(c net.Conn) (*http2ClientConn, error) { - if http2VerboseLogs { - t.vlogf("http2: Transport creating client conn to %v", c.RemoteAddr()) - } - if _, err := c.Write(http2clientPreface); err != nil { - t.vlogf("client preface write error: %v", err) - return nil, err +func (t *http2Transport) expectContinueTimeout() time.Duration { + if t.t1 == nil { + return 0 } + return http2transportExpectContinueTimeout(t.t1) +} + +func (t *http2Transport) NewClientConn(c net.Conn) (*http2ClientConn, error) { + return t.newClientConn(c, false) +} +func (t *http2Transport) newClientConn(c net.Conn, singleUse bool) (*http2ClientConn, error) { cc := &http2ClientConn{ t: t, tconn: c, @@ -4916,13 +5832,26 @@ func (t *http2Transport) NewClientConn(c net.Conn) (*http2ClientConn, error) { initialWindowSize: 65535, maxConcurrentStreams: 1000, streams: make(map[uint32]*http2clientStream), + singleUse: singleUse, + wantSettingsAck: true, + pings: make(map[[8]byte]chan struct{}), } + if d := t.idleConnTimeout(); d != 0 { + cc.idleTimeout = d + cc.idleTimer = time.AfterFunc(d, cc.onIdleTimeout) + } + if http2VerboseLogs { + t.vlogf("http2: Transport creating client conn %p to %v", cc, c.RemoteAddr()) + } + cc.cond = sync.NewCond(&cc.mu) cc.flow.add(int32(http2initialWindowSize)) cc.bw = bufio.NewWriter(http2stickyErrWriter{c, &cc.werr}) cc.br = bufio.NewReader(c) cc.fr = http2NewFramer(cc.bw, cc.br) + cc.fr.ReadMetaHeaders = hpack.NewDecoder(http2initialHeaderTableSize, nil) + cc.fr.MaxHeaderListSize = t.maxHeaderListSize() cc.henc = hpack.NewEncoder(&cc.hbuf) @@ -4932,12 +5861,14 @@ func (t *http2Transport) NewClientConn(c net.Conn) (*http2ClientConn, error) { } initialSettings := []http2Setting{ - http2Setting{ID: http2SettingEnablePush, Val: 0}, - http2Setting{ID: http2SettingInitialWindowSize, Val: http2transportDefaultStreamFlow}, + {ID: http2SettingEnablePush, Val: 0}, + {ID: http2SettingInitialWindowSize, Val: http2transportDefaultStreamFlow}, } if max := t.maxHeaderListSize(); max != 0 { initialSettings = append(initialSettings, http2Setting{ID: http2SettingMaxHeaderListSize, Val: max}) } + + cc.bw.Write(http2clientPreface) cc.fr.WriteSettings(initialSettings...) cc.fr.WriteWindowUpdate(0, http2transportDefaultConnFlow) cc.inflow.add(http2transportDefaultConnFlow + http2initialWindowSize) @@ -4946,32 +5877,6 @@ func (t *http2Transport) NewClientConn(c net.Conn) (*http2ClientConn, error) { return nil, cc.werr } - f, err := cc.fr.ReadFrame() - if err != nil { - return nil, err - } - sf, ok := f.(*http2SettingsFrame) - if !ok { - return nil, fmt.Errorf("expected settings frame, got: %T", f) - } - cc.fr.WriteSettingsAck() - cc.bw.Flush() - - sf.ForeachSetting(func(s http2Setting) error { - switch s.ID { - case http2SettingMaxFrameSize: - cc.maxFrameSize = s.Val - case http2SettingMaxConcurrentStreams: - cc.maxConcurrentStreams = s.Val - case http2SettingInitialWindowSize: - cc.initialWindowSize = s.Val - default: - - t.vlogf("Unhandled Setting: %v", s) - } - return nil - }) - go cc.readLoop() return cc, nil } @@ -4979,7 +5884,25 @@ func (t *http2Transport) NewClientConn(c net.Conn) (*http2ClientConn, error) { func (cc *http2ClientConn) setGoAway(f *http2GoAwayFrame) { cc.mu.Lock() defer cc.mu.Unlock() + + old := cc.goAway cc.goAway = f + + if cc.goAwayDebug == "" { + cc.goAwayDebug = string(f.DebugData()) + } + if old != nil && old.ErrCode != http2ErrCodeNo { + cc.goAway.ErrCode = old.ErrCode + } + last := f.LastStreamID + for streamID, cs := range cc.streams { + if streamID > last { + select { + case cs.resc <- http2resAndError{err: http2errClientConnGotGoAway}: + default: + } + } + } } func (cc *http2ClientConn) CanTakeNewRequest() bool { @@ -4989,9 +5912,22 @@ func (cc *http2ClientConn) CanTakeNewRequest() bool { } func (cc *http2ClientConn) canTakeNewRequestLocked() bool { + if cc.singleUse && cc.nextStreamID > 1 { + return false + } return cc.goAway == nil && !cc.closed && int64(len(cc.streams)+1) < int64(cc.maxConcurrentStreams) && - cc.nextStreamID < 2147483647 + cc.nextStreamID < math.MaxInt32 +} + +// onIdleTimeout is called from a time.AfterFunc goroutine. It will +// only be called when we're idle, but because we're coming from a new +// goroutine, there could be a new request coming in at the same time, +// so this simply calls the synchronized closeIfIdle to shut down this +// connection. The timer could just call closeIfIdle, but this is more +// clear. +func (cc *http2ClientConn) onIdleTimeout() { + cc.closeIfIdle() } func (cc *http2ClientConn) closeIfIdle() { @@ -5001,9 +5937,13 @@ func (cc *http2ClientConn) closeIfIdle() { return } cc.closed = true + nextID := cc.nextStreamID cc.mu.Unlock() + if http2VerboseLogs { + cc.vlogf("http2: Transport closing idle conn %p (forSingleUse=%v, maxStream=%v)", cc, cc.singleUse, nextID-2) + } cc.tconn.Close() } @@ -5082,21 +6022,37 @@ func (cc *http2ClientConn) responseHeaderTimeout() time.Duration { // Certain headers are special-cased as okay but not transmitted later. func http2checkConnHeaders(req *Request) error { if v := req.Header.Get("Upgrade"); v != "" { - return errors.New("http2: invalid Upgrade request header") + return fmt.Errorf("http2: invalid Upgrade request header: %q", req.Header["Upgrade"]) } - if v := req.Header.Get("Transfer-Encoding"); (v != "" && v != "chunked") || len(req.Header["Transfer-Encoding"]) > 1 { - return errors.New("http2: invalid Transfer-Encoding request header") + if vv := req.Header["Transfer-Encoding"]; len(vv) > 0 && (len(vv) > 1 || vv[0] != "" && vv[0] != "chunked") { + return fmt.Errorf("http2: invalid Transfer-Encoding request header: %q", vv) } - if v := req.Header.Get("Connection"); (v != "" && v != "close" && v != "keep-alive") || len(req.Header["Connection"]) > 1 { - return errors.New("http2: invalid Connection request header") + if vv := req.Header["Connection"]; len(vv) > 0 && (len(vv) > 1 || vv[0] != "" && vv[0] != "close" && vv[0] != "keep-alive") { + return fmt.Errorf("http2: invalid Connection request header: %q", vv) } return nil } +// actualContentLength returns a sanitized version of +// req.ContentLength, where 0 actually means zero (not unknown) and -1 +// means unknown. +func http2actualContentLength(req *Request) int64 { + if req.Body == nil { + return 0 + } + if req.ContentLength != 0 { + return req.ContentLength + } + return -1 +} + func (cc *http2ClientConn) RoundTrip(req *Request) (*Response, error) { if err := http2checkConnHeaders(req); err != nil { return nil, err } + if cc.idleTimer != nil { + cc.idleTimer.Stop() + } trailers, err := http2commaSeparatedTrailers(req) if err != nil { @@ -5104,67 +6060,63 @@ func (cc *http2ClientConn) RoundTrip(req *Request) (*Response, error) { } hasTrailers := trailers != "" - var body io.Reader = req.Body - contentLen := req.ContentLength - if req.Body != nil && contentLen == 0 { - // Test to see if it's actually zero or just unset. - var buf [1]byte - n, rerr := io.ReadFull(body, buf[:]) - if rerr != nil && rerr != io.EOF { - contentLen = -1 - body = http2errorReader{rerr} - } else if n == 1 { - - contentLen = -1 - body = io.MultiReader(bytes.NewReader(buf[:]), body) - } else { - - body = nil - } - } - cc.mu.Lock() + cc.lastActive = time.Now() if cc.closed || !cc.canTakeNewRequestLocked() { cc.mu.Unlock() return nil, http2errClientConnUnusable } - cs := cc.newStream() - cs.req = req + body := req.Body hasBody := body != nil + contentLen := http2actualContentLength(req) + // TODO(bradfitz): this is a copy of the logic in net/http. Unify somewhere? + var requestedGzip bool if !cc.t.disableCompression() && req.Header.Get("Accept-Encoding") == "" && req.Header.Get("Range") == "" && req.Method != "HEAD" { - cs.requestedGzip = true + requestedGzip = true } - hdrs := cc.encodeHeaders(req, cs.requestedGzip, trailers, contentLen) + hdrs, err := cc.encodeHeaders(req, requestedGzip, trailers, contentLen) + if err != nil { + cc.mu.Unlock() + return nil, err + } + + cs := cc.newStream() + cs.req = req + cs.trace = http2requestTrace(req) + cs.requestedGzip = requestedGzip + bodyWriter := cc.t.getBodyWriterState(cs, body) + cs.on100 = bodyWriter.on100 + cc.wmu.Lock() endStream := !hasBody && !hasTrailers werr := cc.writeHeaders(cs.ID, endStream, hdrs) cc.wmu.Unlock() + http2traceWroteHeaders(cs.trace) cc.mu.Unlock() if werr != nil { if hasBody { req.Body.Close() + bodyWriter.cancel() } cc.forgetStreamID(cs.ID) + http2traceWroteRequest(cs.trace, werr) return nil, werr } var respHeaderTimer <-chan time.Time - var bodyCopyErrc chan error // result of body copy if hasBody { - bodyCopyErrc = make(chan error, 1) - go func() { - bodyCopyErrc <- cs.writeRequestBody(body, req.Body) - }() + bodyWriter.scheduleBodyWrite() } else { + http2traceWroteRequest(cs.trace, nil) if d := cc.responseHeaderTimeout(); d != 0 { timer := time.NewTimer(d) defer timer.Stop() @@ -5173,44 +6125,87 @@ func (cc *http2ClientConn) RoundTrip(req *Request) (*Response, error) { } readLoopResCh := cs.resc - requestCanceledCh := http2requestCancel(req) bodyWritten := false + ctx := http2reqContext(req) + + handleReadLoopResponse := func(re http2resAndError) (*Response, error) { + res := re.res + if re.err != nil || res.StatusCode > 299 { + + bodyWriter.cancel() + cs.abortRequestBodyWrite(http2errStopReqBodyWrite) + } + if re.err != nil { + if re.err == http2errClientConnGotGoAway { + cc.mu.Lock() + if cs.startedWrite { + re.err = http2errClientConnGotGoAwayAfterSomeReqBody + } + cc.mu.Unlock() + } + cc.forgetStreamID(cs.ID) + return nil, re.err + } + res.Request = req + res.TLS = cc.tlsState + return res, nil + } for { select { case re := <-readLoopResCh: - res := re.res - if re.err != nil || res.StatusCode > 299 { - - cs.abortRequestBodyWrite(http2errStopReqBodyWrite) - } - if re.err != nil { - cc.forgetStreamID(cs.ID) - return nil, re.err - } - res.Request = req - res.TLS = cc.tlsState - return res, nil + return handleReadLoopResponse(re) case <-respHeaderTimer: cc.forgetStreamID(cs.ID) if !hasBody || bodyWritten { cc.writeStreamReset(cs.ID, http2ErrCodeCancel, nil) } else { + bodyWriter.cancel() cs.abortRequestBodyWrite(http2errStopReqBodyWriteAndCancel) } return nil, http2errTimeout - case <-requestCanceledCh: + case <-ctx.Done(): + select { + case re := <-readLoopResCh: + return handleReadLoopResponse(re) + default: + } + cc.forgetStreamID(cs.ID) + if !hasBody || bodyWritten { + cc.writeStreamReset(cs.ID, http2ErrCodeCancel, nil) + } else { + bodyWriter.cancel() + cs.abortRequestBodyWrite(http2errStopReqBodyWriteAndCancel) + } + return nil, ctx.Err() + case <-req.Cancel: + select { + case re := <-readLoopResCh: + return handleReadLoopResponse(re) + default: + } cc.forgetStreamID(cs.ID) if !hasBody || bodyWritten { cc.writeStreamReset(cs.ID, http2ErrCodeCancel, nil) } else { + bodyWriter.cancel() cs.abortRequestBodyWrite(http2errStopReqBodyWriteAndCancel) } return nil, http2errRequestCanceled case <-cs.peerReset: - + select { + case re := <-readLoopResCh: + return handleReadLoopResponse(re) + default: + } return nil, cs.resetErr - case err := <-bodyCopyErrc: + case err := <-bodyWriter.resc: + + select { + case re := <-readLoopResCh: + return handleReadLoopResponse(re) + default: + } if err != nil { return nil, err } @@ -5268,6 +6263,7 @@ func (cs *http2clientStream) writeRequestBody(body io.Reader, bodyCloser io.Clos defer cc.putFrameScratchBuffer(buf) defer func() { + http2traceWroteRequest(cs.trace, err) cerr := bodyCloser.Close() if err == nil { @@ -5317,26 +6313,29 @@ func (cs *http2clientStream) writeRequestBody(body io.Reader, bodyCloser io.Clos } } + if sentEnd { + + return nil + } + + var trls []byte + if hasTrailers { + cc.mu.Lock() + defer cc.mu.Unlock() + trls = cc.encodeTrailers(req) + } + cc.wmu.Lock() - if !sentEnd { - var trls []byte - if hasTrailers { - cc.mu.Lock() - trls = cc.encodeTrailers(req) - cc.mu.Unlock() - } + defer cc.wmu.Unlock() - if len(trls) > 0 { - err = cc.writeHeaders(cs.ID, true, trls) - } else { - err = cc.fr.WriteData(cs.ID, true, nil) - } + if len(trls) > 0 { + err = cc.writeHeaders(cs.ID, true, trls) + } else { + err = cc.fr.WriteData(cs.ID, true, nil) } if ferr := cc.bw.Flush(); ferr != nil && err == nil { err = ferr } - cc.wmu.Unlock() - return err } @@ -5355,7 +6354,7 @@ func (cs *http2clientStream) awaitFlowControl(maxBytes int) (taken int32, err er if cs.stopReqBody != nil { return 0, cs.stopReqBody } - if err := cs.checkReset(); err != nil { + if err := cs.checkResetOrDone(); err != nil { return 0, err } if a := cs.flow.available(); a > 0 { @@ -5382,19 +6381,50 @@ type http2badStringError struct { func (e *http2badStringError) Error() string { return fmt.Sprintf("%s %q", e.what, e.str) } // requires cc.mu be held. -func (cc *http2ClientConn) encodeHeaders(req *Request, addGzipHeader bool, trailers string, contentLength int64) []byte { +func (cc *http2ClientConn) encodeHeaders(req *Request, addGzipHeader bool, trailers string, contentLength int64) ([]byte, error) { cc.hbuf.Reset() host := req.Host if host == "" { host = req.URL.Host } + host, err := httplex.PunycodeHostPort(host) + if err != nil { + return nil, err + } + + var path string + if req.Method != "CONNECT" { + path = req.URL.RequestURI() + if !http2validPseudoPath(path) { + orig := path + path = strings.TrimPrefix(path, req.URL.Scheme+"://"+host) + if !http2validPseudoPath(path) { + if req.URL.Opaque != "" { + return nil, fmt.Errorf("invalid request :path %q from URL.Opaque = %q", orig, req.URL.Opaque) + } else { + return nil, fmt.Errorf("invalid request :path %q", orig) + } + } + } + } + + for k, vv := range req.Header { + if !httplex.ValidHeaderFieldName(k) { + return nil, fmt.Errorf("invalid HTTP header name %q", k) + } + for _, v := range vv { + if !httplex.ValidHeaderFieldValue(v) { + return nil, fmt.Errorf("invalid HTTP header value %q for header %q", v, k) + } + } + } cc.writeHeader(":authority", host) cc.writeHeader(":method", req.Method) if req.Method != "CONNECT" { - cc.writeHeader(":path", req.URL.RequestURI()) - cc.writeHeader(":scheme", "https") + cc.writeHeader(":path", path) + cc.writeHeader(":scheme", req.URL.Scheme) } if trailers != "" { cc.writeHeader("trailer", trailers) @@ -5407,7 +6437,7 @@ func (cc *http2ClientConn) encodeHeaders(req *Request, addGzipHeader bool, trail case "host", "content-length": continue - case "connection", "proxy-connection", "transfer-encoding", "upgrade": + case "connection", "proxy-connection", "transfer-encoding", "upgrade", "keep-alive": continue case "user-agent": @@ -5434,7 +6464,7 @@ func (cc *http2ClientConn) encodeHeaders(req *Request, addGzipHeader bool, trail if !didUA { cc.writeHeader("user-agent", http2defaultUserAgent) } - return cc.hbuf.Bytes() + return cc.hbuf.Bytes(), nil } // shouldSendReqContentLength reports whether the http2.Transport should send @@ -5510,8 +6540,13 @@ func (cc *http2ClientConn) streamByID(id uint32, andRemove bool) *http2clientStr defer cc.mu.Unlock() cs := cc.streams[id] if andRemove && cs != nil && !cc.closed { + cc.lastActive = time.Now() delete(cc.streams, id) + if len(cc.streams) == 0 && cc.idleTimer != nil { + cc.idleTimer.Reset(cc.idleTimeout) + } close(cs.done) + cc.cond.Broadcast() } return cs } @@ -5521,15 +6556,6 @@ type http2clientConnReadLoop struct { cc *http2ClientConn activeRes map[uint32]*http2clientStream // keyed by streamID closeWhenIdle bool - - hdec *hpack.Decoder - - // Fields reset on each HEADERS: - nextRes *Response - sawRegHeader bool // saw non-pseudo header - reqMalformed error // non-nil once known to be malformed - lastHeaderEndsStream bool - headerListSize int64 // actually uint32, but easier math this way } // readLoop runs in its own goroutine and reads and dispatches frames. @@ -5538,7 +6564,6 @@ func (cc *http2ClientConn) readLoop() { cc: cc, activeRes: make(map[uint32]*http2clientStream), } - rl.hdec = hpack.NewDecoder(http2initialHeaderTableSize, rl.onNewHeaderField) defer rl.cleanup() cc.readerErr = rl.run() @@ -5549,17 +6574,48 @@ func (cc *http2ClientConn) readLoop() { } } +// GoAwayError is returned by the Transport when the server closes the +// TCP connection after sending a GOAWAY frame. +type http2GoAwayError struct { + LastStreamID uint32 + ErrCode http2ErrCode + DebugData string +} + +func (e http2GoAwayError) Error() string { + return fmt.Sprintf("http2: server sent GOAWAY and closed the connection; LastStreamID=%v, ErrCode=%v, debug=%q", + e.LastStreamID, e.ErrCode, e.DebugData) +} + +func http2isEOFOrNetReadError(err error) bool { + if err == io.EOF { + return true + } + ne, ok := err.(*net.OpError) + return ok && ne.Op == "read" +} + func (rl *http2clientConnReadLoop) cleanup() { cc := rl.cc defer cc.tconn.Close() defer cc.t.connPool().MarkDead(cc) defer close(cc.readerDone) + if cc.idleTimer != nil { + cc.idleTimer.Stop() + } + err := cc.readerErr - if err == io.EOF { + cc.mu.Lock() + if cc.goAway != nil && http2isEOFOrNetReadError(err) { + err = http2GoAwayError{ + LastStreamID: cc.goAway.LastStreamID, + ErrCode: cc.goAway.ErrCode, + DebugData: cc.goAwayDebug, + } + } else if err == io.EOF { err = io.ErrUnexpectedEOF } - cc.mu.Lock() for _, cs := range rl.activeRes { cs.bufPipe.CloseWithError(err) } @@ -5577,32 +6633,43 @@ func (rl *http2clientConnReadLoop) cleanup() { func (rl *http2clientConnReadLoop) run() error { cc := rl.cc - rl.closeWhenIdle = cc.t.disableKeepAlives() + rl.closeWhenIdle = cc.t.disableKeepAlives() || cc.singleUse gotReply := false + gotSettings := false for { f, err := cc.fr.ReadFrame() if err != nil { - cc.vlogf("Transport readFrame error: (%T) %v", err, err) + cc.vlogf("http2: Transport readFrame error on conn %p: (%T) %v", cc, err, err) } if se, ok := err.(http2StreamError); ok { - - return se + if cs := cc.streamByID(se.StreamID, true); cs != nil { + cs.cc.writeStreamReset(cs.ID, se.Code, err) + if se.Cause == nil { + se.Cause = cc.fr.errDetail + } + rl.endStreamError(cs, se) + } + continue } else if err != nil { return err } if http2VerboseLogs { cc.vlogf("http2: Transport received %s", http2summarizeFrame(f)) } + if !gotSettings { + if _, ok := f.(*http2SettingsFrame); !ok { + cc.logf("protocol error: received %T before a SETTINGS frame", f) + return http2ConnectionError(http2ErrCodeProtocol) + } + gotSettings = true + } maybeIdle := false switch f := f.(type) { - case *http2HeadersFrame: + case *http2MetaHeadersFrame: err = rl.processHeaders(f) maybeIdle = true gotReply = true - case *http2ContinuationFrame: - err = rl.processContinuation(f) - maybeIdle = true case *http2DataFrame: err = rl.processData(f) maybeIdle = true @@ -5624,6 +6691,9 @@ func (rl *http2clientConnReadLoop) run() error { cc.logf("Transport: unhandled response frame type %T", f) } if err != nil { + if http2VerboseLogs { + cc.vlogf("http2: Transport conn %p received error from processing frame %v: %v", cc, http2summarizeFrame(f), err) + } return err } if rl.closeWhenIdle && gotReply && maybeIdle && len(rl.activeRes) == 0 { @@ -5632,83 +6702,105 @@ func (rl *http2clientConnReadLoop) run() error { } } -func (rl *http2clientConnReadLoop) processHeaders(f *http2HeadersFrame) error { - rl.sawRegHeader = false - rl.reqMalformed = nil - rl.lastHeaderEndsStream = f.StreamEnded() - rl.headerListSize = 0 - rl.nextRes = &Response{ - Proto: "HTTP/2.0", - ProtoMajor: 2, - Header: make(Header), - } - rl.hdec.SetEmitEnabled(true) - return rl.processHeaderBlockFragment(f.HeaderBlockFragment(), f.StreamID, f.HeadersEnded()) -} - -func (rl *http2clientConnReadLoop) processContinuation(f *http2ContinuationFrame) error { - return rl.processHeaderBlockFragment(f.HeaderBlockFragment(), f.StreamID, f.HeadersEnded()) -} - -func (rl *http2clientConnReadLoop) processHeaderBlockFragment(frag []byte, streamID uint32, finalFrag bool) error { +func (rl *http2clientConnReadLoop) processHeaders(f *http2MetaHeadersFrame) error { cc := rl.cc - streamEnded := rl.lastHeaderEndsStream - cs := cc.streamByID(streamID, streamEnded && finalFrag) + cs := cc.streamByID(f.StreamID, f.StreamEnded()) if cs == nil { return nil } - if cs.pastHeaders { - rl.hdec.SetEmitFunc(func(f hpack.HeaderField) { rl.onNewTrailerField(cs, f) }) - } else { - rl.hdec.SetEmitFunc(rl.onNewHeaderField) - } - _, err := rl.hdec.Write(frag) - if err != nil { - return http2ConnectionError(http2ErrCodeCompression) - } - if finalFrag { - if err := rl.hdec.Close(); err != nil { - return http2ConnectionError(http2ErrCodeCompression) - } - } + if !cs.firstByte { + if cs.trace != nil { - if !finalFrag { - return nil + http2traceFirstResponseByte(cs.trace) + } + cs.firstByte = true } - if !cs.pastHeaders { cs.pastHeaders = true } else { + return rl.processTrailers(cs, f) + } - if cs.pastTrailers { - - return http2ConnectionError(http2ErrCodeProtocol) + res, err := rl.handleResponse(cs, f) + if err != nil { + if _, ok := err.(http2ConnectionError); ok { + return err } - cs.pastTrailers = true - if !streamEnded { - return http2ConnectionError(http2ErrCodeProtocol) - } - rl.endStream(cs) + cs.cc.writeStreamReset(f.StreamID, http2ErrCodeProtocol, err) + cs.resc <- http2resAndError{err: err} return nil } + if res == nil { - if rl.reqMalformed != nil { - cs.resc <- http2resAndError{err: rl.reqMalformed} - rl.cc.writeStreamReset(cs.ID, http2ErrCodeProtocol, rl.reqMalformed) return nil } + if res.Body != http2noBody { + rl.activeRes[cs.ID] = cs + } + cs.resTrailer = &res.Trailer + cs.resc <- http2resAndError{res: res} + return nil +} - res := rl.nextRes +// may return error types nil, or ConnectionError. Any other error value +// is a StreamError of type ErrCodeProtocol. The returned error in that case +// is the detail. +// +// As a special case, handleResponse may return (nil, nil) to skip the +// frame (currently only used for 100 expect continue). This special +// case is going away after Issue 13851 is fixed. +func (rl *http2clientConnReadLoop) handleResponse(cs *http2clientStream, f *http2MetaHeadersFrame) (*Response, error) { + if f.Truncated { + return nil, http2errResponseHeaderListSize + } - if res.StatusCode == 100 { + status := f.PseudoValue("status") + if status == "" { + return nil, errors.New("missing status pseudo header") + } + statusCode, err := strconv.Atoi(status) + if err != nil { + return nil, errors.New("malformed non-numeric status pseudo header") + } + if statusCode == 100 { + http2traceGot100Continue(cs.trace) + if cs.on100 != nil { + cs.on100() + } cs.pastHeaders = false - return nil + return nil, nil } - if !streamEnded || cs.req.Method == "HEAD" { + header := make(Header) + res := &Response{ + Proto: "HTTP/2.0", + ProtoMajor: 2, + Header: header, + StatusCode: statusCode, + Status: status + " " + StatusText(statusCode), + } + for _, hf := range f.RegularFields() { + key := CanonicalHeaderKey(hf.Name) + if key == "Trailer" { + t := res.Trailer + if t == nil { + t = make(Header) + res.Trailer = t + } + http2foreachHeaderElement(hf.Value, func(v string) { + t[CanonicalHeaderKey(v)] = nil + }) + } else { + header[key] = append(header[key], hf.Value) + } + } + + streamEnded := f.StreamEnded() + isHead := cs.req.Method == "HEAD" + if !streamEnded || isHead { res.ContentLength = -1 if clens := res.Header["Content-Length"]; len(clens) == 1 { if clen64, err := strconv.ParseInt(clens[0], 10, 64); err == nil { @@ -5721,27 +6813,50 @@ func (rl *http2clientConnReadLoop) processHeaderBlockFragment(frag []byte, strea } } - if streamEnded { + if streamEnded || isHead { res.Body = http2noBody - } else { - buf := new(bytes.Buffer) - cs.bufPipe = http2pipe{b: buf} - cs.bytesRemain = res.ContentLength - res.Body = http2transportResponseBody{cs} - go cs.awaitRequestCancel(http2requestCancel(cs.req)) - - if cs.requestedGzip && res.Header.Get("Content-Encoding") == "gzip" { - res.Header.Del("Content-Encoding") - res.Header.Del("Content-Length") - res.ContentLength = -1 - res.Body = &http2gzipReader{body: res.Body} - } - rl.activeRes[cs.ID] = cs + return res, nil } - cs.resTrailer = &res.Trailer - cs.resc <- http2resAndError{res: res} - rl.nextRes = nil + buf := new(bytes.Buffer) + cs.bufPipe = http2pipe{b: buf} + cs.bytesRemain = res.ContentLength + res.Body = http2transportResponseBody{cs} + go cs.awaitRequestCancel(cs.req) + + if cs.requestedGzip && res.Header.Get("Content-Encoding") == "gzip" { + res.Header.Del("Content-Encoding") + res.Header.Del("Content-Length") + res.ContentLength = -1 + res.Body = &http2gzipReader{body: res.Body} + http2setResponseUncompressed(res) + } + return res, nil +} + +func (rl *http2clientConnReadLoop) processTrailers(cs *http2clientStream, f *http2MetaHeadersFrame) error { + if cs.pastTrailers { + + return http2ConnectionError(http2ErrCodeProtocol) + } + cs.pastTrailers = true + if !f.StreamEnded() { + + return http2ConnectionError(http2ErrCodeProtocol) + } + if len(f.PseudoFields()) > 0 { + + return http2ConnectionError(http2ErrCodeProtocol) + } + + trailer := make(Header) + for _, hf := range f.RegularFields() { + key := CanonicalHeaderKey(hf.Name) + trailer[key] = append(trailer[key], hf.Value) + } + cs.trailer = trailer + + rl.endStream(cs) return nil } @@ -5792,8 +6907,10 @@ func (b http2transportResponseBody) Read(p []byte) (n int, err error) { cc.inflow.add(connAdd) } if err == nil { - if v := cs.inflow.available(); v < http2transportDefaultStreamFlow-http2transportDefaultStreamMinRefresh { - streamAdd = http2transportDefaultStreamFlow - v + + v := int(cs.inflow.available()) + cs.bufPipe.Len() + if v < http2transportDefaultStreamFlow-http2transportDefaultStreamMinRefresh { + streamAdd = int32(http2transportDefaultStreamFlow - v) cs.inflow.add(streamAdd) } } @@ -5815,10 +6932,27 @@ var http2errClosedResponseBody = errors.New("http2: response body closed") func (b http2transportResponseBody) Close() error { cs := b.cs - if cs.bufPipe.Err() != io.EOF { + cc := cs.cc - cs.cc.writeStreamReset(cs.ID, http2ErrCodeCancel, nil) + serverSentStreamEnd := cs.bufPipe.Err() == io.EOF + unread := cs.bufPipe.Len() + + if unread > 0 || !serverSentStreamEnd { + cc.mu.Lock() + cc.wmu.Lock() + if !serverSentStreamEnd { + cc.fr.WriteRSTStream(cs.ID, http2ErrCodeCancel) + } + + if unread > 0 { + cc.inflow.add(int32(unread)) + cc.fr.WriteWindowUpdate(0, uint32(unread)) + } + cc.bw.Flush() + cc.wmu.Unlock() + cc.mu.Unlock() } + cs.bufPipe.BreakWithError(http2errClosedResponseBody) return nil } @@ -5826,6 +6960,7 @@ func (b http2transportResponseBody) Close() error { func (rl *http2clientConnReadLoop) processData(f *http2DataFrame) error { cc := rl.cc cs := cc.streamByID(f.StreamID, f.StreamEnded()) + data := f.Data() if cs == nil { cc.mu.Lock() neverSent := cc.nextStreamID @@ -5836,26 +6971,50 @@ func (rl *http2clientConnReadLoop) processData(f *http2DataFrame) error { return http2ConnectionError(http2ErrCodeProtocol) } + if f.Length > 0 { + cc.mu.Lock() + cc.inflow.add(int32(f.Length)) + cc.mu.Unlock() + + cc.wmu.Lock() + cc.fr.WriteWindowUpdate(0, uint32(f.Length)) + cc.bw.Flush() + cc.wmu.Unlock() + } return nil } - if data := f.Data(); len(data) > 0 { - if cs.bufPipe.b == nil { + if f.Length > 0 { + if len(data) > 0 && cs.bufPipe.b == nil { cc.logf("http2: Transport received DATA frame for closed stream; closing connection") return http2ConnectionError(http2ErrCodeProtocol) } cc.mu.Lock() - if cs.inflow.available() >= int32(len(data)) { - cs.inflow.take(int32(len(data))) + if cs.inflow.available() >= int32(f.Length) { + cs.inflow.take(int32(f.Length)) } else { cc.mu.Unlock() return http2ConnectionError(http2ErrCodeFlowControl) } + + if pad := int32(f.Length) - int32(len(data)); pad > 0 { + cs.inflow.add(pad) + cc.inflow.add(pad) + cc.wmu.Lock() + cc.fr.WriteWindowUpdate(0, uint32(pad)) + cc.fr.WriteWindowUpdate(cs.ID, uint32(pad)) + cc.bw.Flush() + cc.wmu.Unlock() + } + didReset := cs.didReset cc.mu.Unlock() - if _, err := cs.bufPipe.Write(data); err != nil { - return err + if len(data) > 0 && !didReset { + if _, err := cs.bufPipe.Write(data); err != nil { + rl.endStreamError(cs, err) + return err + } } } @@ -5869,17 +7028,25 @@ var http2errInvalidTrailers = errors.New("http2: invalid trailers") func (rl *http2clientConnReadLoop) endStream(cs *http2clientStream) { - err := io.EOF - code := cs.copyTrailers - if rl.reqMalformed != nil { - err = rl.reqMalformed - code = nil + rl.endStreamError(cs, nil) +} + +func (rl *http2clientConnReadLoop) endStreamError(cs *http2clientStream, err error) { + var code func() + if err == nil { + err = io.EOF + code = cs.copyTrailers } cs.bufPipe.closeWithErrorAndCode(err, code) delete(rl.activeRes, cs.ID) - if cs.req.Close || cs.req.Header.Get("Connection") == "close" { + if http2isConnectionCloseRequest(cs.req) { rl.closeWhenIdle = true } + + select { + case cs.resc <- http2resAndError{err: err}: + default: + } } func (cs *http2clientStream) copyTrailers() { @@ -5907,7 +7074,16 @@ func (rl *http2clientConnReadLoop) processSettings(f *http2SettingsFrame) error cc := rl.cc cc.mu.Lock() defer cc.mu.Unlock() - return f.ForeachSetting(func(s http2Setting) error { + + if f.IsAck() { + if cc.wantSettingsAck { + cc.wantSettingsAck = false + return nil + } + return http2ConnectionError(http2ErrCodeProtocol) + } + + err := f.ForeachSetting(func(s http2Setting) error { switch s.ID { case http2SettingMaxFrameSize: cc.maxFrameSize = s.Val @@ -5915,6 +7091,16 @@ func (rl *http2clientConnReadLoop) processSettings(f *http2SettingsFrame) error cc.maxConcurrentStreams = s.Val case http2SettingInitialWindowSize: + if s.Val > math.MaxInt32 { + return http2ConnectionError(http2ErrCodeFlowControl) + } + + delta := int32(s.Val) - int32(cc.initialWindowSize) + for _, cs := range cc.streams { + cs.flow.add(delta) + } + cc.cond.Broadcast() + cc.initialWindowSize = s.Val default: @@ -5922,6 +7108,16 @@ func (rl *http2clientConnReadLoop) processSettings(f *http2SettingsFrame) error } return nil }) + if err != nil { + return err + } + + cc.wmu.Lock() + defer cc.wmu.Unlock() + + cc.fr.WriteSettingsAck() + cc.bw.Flush() + return cc.werr } func (rl *http2clientConnReadLoop) processWindowUpdate(f *http2WindowUpdateFrame) error { @@ -5955,7 +7151,7 @@ func (rl *http2clientConnReadLoop) processResetStream(f *http2RSTStreamFrame) er case <-cs.peerReset: default: - err := http2StreamError{cs.ID, f.ErrCode} + err := http2streamError(cs.ID, f.ErrCode) cs.resetErr = err close(cs.peerReset) cs.bufPipe.CloseWithError(err) @@ -5965,9 +7161,56 @@ func (rl *http2clientConnReadLoop) processResetStream(f *http2RSTStreamFrame) er return nil } +// Ping sends a PING frame to the server and waits for the ack. +// Public implementation is in go17.go and not_go17.go +func (cc *http2ClientConn) ping(ctx http2contextContext) error { + c := make(chan struct{}) + // Generate a random payload + var p [8]byte + for { + if _, err := rand.Read(p[:]); err != nil { + return err + } + cc.mu.Lock() + + if _, found := cc.pings[p]; !found { + cc.pings[p] = c + cc.mu.Unlock() + break + } + cc.mu.Unlock() + } + cc.wmu.Lock() + if err := cc.fr.WritePing(false, p); err != nil { + cc.wmu.Unlock() + return err + } + if err := cc.bw.Flush(); err != nil { + cc.wmu.Unlock() + return err + } + cc.wmu.Unlock() + select { + case <-c: + return nil + case <-ctx.Done(): + return ctx.Err() + case <-cc.readerDone: + + return cc.readerErr + } +} + func (rl *http2clientConnReadLoop) processPing(f *http2PingFrame) error { if f.IsAck() { + cc := rl.cc + cc.mu.Lock() + defer cc.mu.Unlock() + if c, ok := cc.pings[f.Data]; ok { + close(c) + delete(cc.pings, f.Data) + } return nil } cc := rl.cc @@ -5997,113 +7240,6 @@ var ( http2errPseudoTrailers = errors.New("http2: invalid pseudo header in trailers") ) -func (rl *http2clientConnReadLoop) checkHeaderField(f hpack.HeaderField) bool { - if rl.reqMalformed != nil { - return false - } - - const headerFieldOverhead = 32 // per spec - rl.headerListSize += int64(len(f.Name)) + int64(len(f.Value)) + headerFieldOverhead - if max := rl.cc.t.maxHeaderListSize(); max != 0 && rl.headerListSize > int64(max) { - rl.hdec.SetEmitEnabled(false) - rl.reqMalformed = http2errResponseHeaderListSize - return false - } - - if !http2validHeaderFieldValue(f.Value) { - rl.reqMalformed = http2errInvalidHeaderFieldValue - return false - } - - isPseudo := strings.HasPrefix(f.Name, ":") - if isPseudo { - if rl.sawRegHeader { - rl.reqMalformed = errors.New("http2: invalid pseudo header after regular header") - return false - } - } else { - if !http2validHeaderFieldName(f.Name) { - rl.reqMalformed = http2errInvalidHeaderFieldName - return false - } - rl.sawRegHeader = true - } - - return true -} - -// onNewHeaderField runs on the readLoop goroutine whenever a new -// hpack header field is decoded. -func (rl *http2clientConnReadLoop) onNewHeaderField(f hpack.HeaderField) { - cc := rl.cc - if http2VerboseLogs { - cc.logf("http2: Transport decoded %v", f) - } - - if !rl.checkHeaderField(f) { - return - } - - isPseudo := strings.HasPrefix(f.Name, ":") - if isPseudo { - switch f.Name { - case ":status": - code, err := strconv.Atoi(f.Value) - if err != nil { - rl.reqMalformed = errors.New("http2: invalid :status") - return - } - rl.nextRes.Status = f.Value + " " + StatusText(code) - rl.nextRes.StatusCode = code - default: - - rl.reqMalformed = fmt.Errorf("http2: unknown response pseudo header %q", f.Name) - } - return - } - - key := CanonicalHeaderKey(f.Name) - if key == "Trailer" { - t := rl.nextRes.Trailer - if t == nil { - t = make(Header) - rl.nextRes.Trailer = t - } - http2foreachHeaderElement(f.Value, func(v string) { - t[CanonicalHeaderKey(v)] = nil - }) - } else { - rl.nextRes.Header.Add(key, f.Value) - } -} - -func (rl *http2clientConnReadLoop) onNewTrailerField(cs *http2clientStream, f hpack.HeaderField) { - if http2VerboseLogs { - rl.cc.logf("http2: Transport decoded trailer %v", f) - } - if !rl.checkHeaderField(f) { - return - } - if strings.HasPrefix(f.Name, ":") { - - rl.reqMalformed = http2errPseudoTrailers - return - } - - key := CanonicalHeaderKey(f.Name) - - // The spec says one must predeclare their trailers but in practice - // popular users (which is to say the only user we found) do not so we - // violate the spec and accept all of them. - const acceptAllTrailers = true - if _, ok := (*cs.resTrailer)[key]; ok || acceptAllTrailers { - if cs.trailer == nil { - cs.trailer = make(Header) - } - cs.trailer[key] = append(cs.trailer[key], f.Value) - } -} - func (cc *http2ClientConn) logf(format string, args ...interface{}) { cc.t.logf(format, args...) } @@ -6167,9 +7303,96 @@ type http2errorReader struct{ err error } func (r http2errorReader) Read(p []byte) (int, error) { return 0, r.err } +// bodyWriterState encapsulates various state around the Transport's writing +// of the request body, particularly regarding doing delayed writes of the body +// when the request contains "Expect: 100-continue". +type http2bodyWriterState struct { + cs *http2clientStream + timer *time.Timer // if non-nil, we're doing a delayed write + fnonce *sync.Once // to call fn with + fn func() // the code to run in the goroutine, writing the body + resc chan error // result of fn's execution + delay time.Duration // how long we should delay a delayed write for +} + +func (t *http2Transport) getBodyWriterState(cs *http2clientStream, body io.Reader) (s http2bodyWriterState) { + s.cs = cs + if body == nil { + return + } + resc := make(chan error, 1) + s.resc = resc + s.fn = func() { + cs.cc.mu.Lock() + cs.startedWrite = true + cs.cc.mu.Unlock() + resc <- cs.writeRequestBody(body, cs.req.Body) + } + s.delay = t.expectContinueTimeout() + if s.delay == 0 || + !httplex.HeaderValuesContainsToken( + cs.req.Header["Expect"], + "100-continue") { + return + } + s.fnonce = new(sync.Once) + + // Arm the timer with a very large duration, which we'll + // intentionally lower later. It has to be large now because + // we need a handle to it before writing the headers, but the + // s.delay value is defined to not start until after the + // request headers were written. + const hugeDuration = 365 * 24 * time.Hour + s.timer = time.AfterFunc(hugeDuration, func() { + s.fnonce.Do(s.fn) + }) + return +} + +func (s http2bodyWriterState) cancel() { + if s.timer != nil { + s.timer.Stop() + } +} + +func (s http2bodyWriterState) on100() { + if s.timer == nil { + + return + } + s.timer.Stop() + go func() { s.fnonce.Do(s.fn) }() +} + +// scheduleBodyWrite starts writing the body, either immediately (in +// the common case) or after the delay timeout. It should not be +// called until after the headers have been written. +func (s http2bodyWriterState) scheduleBodyWrite() { + if s.timer == nil { + + go s.fn() + return + } + http2traceWait100Continue(s.cs.trace) + if s.timer.Stop() { + s.timer.Reset(s.delay) + } +} + +// isConnectionCloseRequest reports whether req should use its own +// connection for a single request and then close the connection. +func http2isConnectionCloseRequest(req *Request) bool { + return req.Close || httplex.HeaderValuesContainsToken(req.Header["Connection"], "close") +} + // writeFramer is implemented by any type that is used to write frames. type http2writeFramer interface { writeFrame(http2writeContext) error + + // staysWithinBuffer reports whether this writer promises that + // it will only write less than or equal to size bytes, and it + // won't Flush the write context. + staysWithinBuffer(size int) bool } // writeContext is the interface needed by the various frame writer @@ -6191,9 +7414,10 @@ type http2writeContext interface { HeaderEncoder() (*hpack.Encoder, *bytes.Buffer) } -// endsStream reports whether the given frame writer w will locally -// close the stream. -func http2endsStream(w http2writeFramer) bool { +// writeEndsStream reports whether w writes a frame that will transition +// the stream to a half-closed local state. This returns false for RST_STREAM, +// which closes the entire stream (not just the local half). +func http2writeEndsStream(w http2writeFramer) bool { switch v := w.(type) { case *http2writeData: return v.endStream @@ -6201,7 +7425,7 @@ func http2endsStream(w http2writeFramer) bool { return v.endStream case nil: - panic("endsStream called on nil writeFramer") + panic("writeEndsStream called on nil writeFramer") } return false } @@ -6212,8 +7436,16 @@ func (http2flushFrameWriter) writeFrame(ctx http2writeContext) error { return ctx.Flush() } +func (http2flushFrameWriter) staysWithinBuffer(max int) bool { return false } + type http2writeSettings []http2Setting +func (s http2writeSettings) staysWithinBuffer(max int) bool { + const settingSize = 6 // uint16 + uint32 + return http2frameHeaderLen+settingSize*len(s) <= max + +} + func (s http2writeSettings) writeFrame(ctx http2writeContext) error { return ctx.Framer().WriteSettings([]http2Setting(s)...) } @@ -6233,6 +7465,8 @@ func (p *http2writeGoAway) writeFrame(ctx http2writeContext) error { return err } +func (*http2writeGoAway) staysWithinBuffer(max int) bool { return false } + type http2writeData struct { streamID uint32 p []byte @@ -6247,6 +7481,10 @@ func (w *http2writeData) writeFrame(ctx http2writeContext) error { return ctx.Framer().WriteData(w.streamID, w.endStream, w.p) } +func (w *http2writeData) staysWithinBuffer(max int) bool { + return http2frameHeaderLen+len(w.p) <= max +} + // handlerPanicRST is the message sent from handler goroutines when // the handler panics. type http2handlerPanicRST struct { @@ -6257,22 +7495,59 @@ func (hp http2handlerPanicRST) writeFrame(ctx http2writeContext) error { return ctx.Framer().WriteRSTStream(hp.StreamID, http2ErrCodeInternal) } +func (hp http2handlerPanicRST) staysWithinBuffer(max int) bool { return http2frameHeaderLen+4 <= max } + func (se http2StreamError) writeFrame(ctx http2writeContext) error { return ctx.Framer().WriteRSTStream(se.StreamID, se.Code) } +func (se http2StreamError) staysWithinBuffer(max int) bool { return http2frameHeaderLen+4 <= max } + type http2writePingAck struct{ pf *http2PingFrame } func (w http2writePingAck) writeFrame(ctx http2writeContext) error { return ctx.Framer().WritePing(true, w.pf.Data) } +func (w http2writePingAck) staysWithinBuffer(max int) bool { + return http2frameHeaderLen+len(w.pf.Data) <= max +} + type http2writeSettingsAck struct{} func (http2writeSettingsAck) writeFrame(ctx http2writeContext) error { return ctx.Framer().WriteSettingsAck() } +func (http2writeSettingsAck) staysWithinBuffer(max int) bool { return http2frameHeaderLen <= max } + +// splitHeaderBlock splits headerBlock into fragments so that each fragment fits +// in a single frame, then calls fn for each fragment. firstFrag/lastFrag are true +// for the first/last fragment, respectively. +func http2splitHeaderBlock(ctx http2writeContext, headerBlock []byte, fn func(ctx http2writeContext, frag []byte, firstFrag, lastFrag bool) error) error { + // For now we're lazy and just pick the minimum MAX_FRAME_SIZE + // that all peers must support (16KB). Later we could care + // more and send larger frames if the peer advertised it, but + // there's little point. Most headers are small anyway (so we + // generally won't have CONTINUATION frames), and extra frames + // only waste 9 bytes anyway. + const maxFrameSize = 16384 + + first := true + for len(headerBlock) > 0 { + frag := headerBlock + if len(frag) > maxFrameSize { + frag = frag[:maxFrameSize] + } + headerBlock = headerBlock[len(frag):] + if err := fn(ctx, frag, first, len(headerBlock) == 0); err != nil { + return err + } + first = false + } + return nil +} + // writeResHeaders is a request to write a HEADERS and 0+ CONTINUATION frames // for HTTP response headers or trailers from a server handler. type http2writeResHeaders struct { @@ -6294,6 +7569,11 @@ func http2encKV(enc *hpack.Encoder, k, v string) { enc.WriteField(hpack.HeaderField{Name: k, Value: v}) } +func (w *http2writeResHeaders) staysWithinBuffer(max int) bool { + + return false +} + func (w *http2writeResHeaders) writeFrame(ctx http2writeContext) error { enc, buf := ctx.HeaderEncoder() buf.Reset() @@ -6319,39 +7599,69 @@ func (w *http2writeResHeaders) writeFrame(ctx http2writeContext) error { panic("unexpected empty hpack") } - // For now we're lazy and just pick the minimum MAX_FRAME_SIZE - // that all peers must support (16KB). Later we could care - // more and send larger frames if the peer advertised it, but - // there's little point. Most headers are small anyway (so we - // generally won't have CONTINUATION frames), and extra frames - // only waste 9 bytes anyway. - const maxFrameSize = 16384 + return http2splitHeaderBlock(ctx, headerBlock, w.writeHeaderBlock) +} - first := true - for len(headerBlock) > 0 { - frag := headerBlock - if len(frag) > maxFrameSize { - frag = frag[:maxFrameSize] - } - headerBlock = headerBlock[len(frag):] - endHeaders := len(headerBlock) == 0 - var err error - if first { - first = false - err = ctx.Framer().WriteHeaders(http2HeadersFrameParam{ - StreamID: w.streamID, - BlockFragment: frag, - EndStream: w.endStream, - EndHeaders: endHeaders, - }) - } else { - err = ctx.Framer().WriteContinuation(w.streamID, endHeaders, frag) - } - if err != nil { - return err - } +func (w *http2writeResHeaders) writeHeaderBlock(ctx http2writeContext, frag []byte, firstFrag, lastFrag bool) error { + if firstFrag { + return ctx.Framer().WriteHeaders(http2HeadersFrameParam{ + StreamID: w.streamID, + BlockFragment: frag, + EndStream: w.endStream, + EndHeaders: lastFrag, + }) + } else { + return ctx.Framer().WriteContinuation(w.streamID, lastFrag, frag) + } +} + +// writePushPromise is a request to write a PUSH_PROMISE and 0+ CONTINUATION frames. +type http2writePushPromise struct { + streamID uint32 // pusher stream + method string // for :method + url *url.URL // for :scheme, :authority, :path + h Header + + // Creates an ID for a pushed stream. This runs on serveG just before + // the frame is written. The returned ID is copied to promisedID. + allocatePromisedID func() (uint32, error) + promisedID uint32 +} + +func (w *http2writePushPromise) staysWithinBuffer(max int) bool { + + return false +} + +func (w *http2writePushPromise) writeFrame(ctx http2writeContext) error { + enc, buf := ctx.HeaderEncoder() + buf.Reset() + + http2encKV(enc, ":method", w.method) + http2encKV(enc, ":scheme", w.url.Scheme) + http2encKV(enc, ":authority", w.url.Host) + http2encKV(enc, ":path", w.url.RequestURI()) + http2encodeHeaders(enc, w.h, nil) + + headerBlock := buf.Bytes() + if len(headerBlock) == 0 { + panic("unexpected empty hpack") + } + + return http2splitHeaderBlock(ctx, headerBlock, w.writeHeaderBlock) +} + +func (w *http2writePushPromise) writeHeaderBlock(ctx http2writeContext, frag []byte, firstFrag, lastFrag bool) error { + if firstFrag { + return ctx.Framer().WritePushPromise(http2PushPromiseParam{ + StreamID: w.streamID, + PromiseID: w.promisedID, + BlockFragment: frag, + EndHeaders: lastFrag, + }) + } else { + return ctx.Framer().WriteContinuation(w.streamID, lastFrag, frag) } - return nil } type http2write100ContinueHeadersFrame struct { @@ -6370,34 +7680,41 @@ func (w http2write100ContinueHeadersFrame) writeFrame(ctx http2writeContext) err }) } +func (w http2write100ContinueHeadersFrame) staysWithinBuffer(max int) bool { + + return 9+2*(len(":status")+len("100")) <= max +} + type http2writeWindowUpdate struct { streamID uint32 // or 0 for conn-level n uint32 } +func (wu http2writeWindowUpdate) staysWithinBuffer(max int) bool { return http2frameHeaderLen+4 <= max } + func (wu http2writeWindowUpdate) writeFrame(ctx http2writeContext) error { return ctx.Framer().WriteWindowUpdate(wu.streamID, wu.n) } +// encodeHeaders encodes an http.Header. If keys is not nil, then (k, h[k]) +// is encoded only only if k is in keys. func http2encodeHeaders(enc *hpack.Encoder, h Header, keys []string) { - if keys == nil { - keys = make([]string, 0, len(h)) - for k := range h { - keys = append(keys, k) - } - sort.Strings(keys) + sorter := http2sorterPool.Get().(*http2sorter) + + defer http2sorterPool.Put(sorter) + keys = sorter.Keys(h) } for _, k := range keys { vv := h[k] k = http2lowerHeader(k) - if !http2validHeaderFieldName(k) { + if !http2validWireHeaderFieldName(k) { continue } isTE := k == "transfer-encoding" for _, v := range vv { - if !http2validHeaderFieldValue(v) { + if !httplex.ValidHeaderFieldValue(v) { continue } @@ -6410,14 +7727,53 @@ func http2encodeHeaders(enc *hpack.Encoder, h Header, keys []string) { } } -// frameWriteMsg is a request to write a frame. -type http2frameWriteMsg struct { +// WriteScheduler is the interface implemented by HTTP/2 write schedulers. +// Methods are never called concurrently. +type http2WriteScheduler interface { + // OpenStream opens a new stream in the write scheduler. + // It is illegal to call this with streamID=0 or with a streamID that is + // already open -- the call may panic. + OpenStream(streamID uint32, options http2OpenStreamOptions) + + // CloseStream closes a stream in the write scheduler. Any frames queued on + // this stream should be discarded. It is illegal to call this on a stream + // that is not open -- the call may panic. + CloseStream(streamID uint32) + + // AdjustStream adjusts the priority of the given stream. This may be called + // on a stream that has not yet been opened or has been closed. Note that + // RFC 7540 allows PRIORITY frames to be sent on streams in any state. See: + // https://tools.ietf.org/html/rfc7540#section-5.1 + AdjustStream(streamID uint32, priority http2PriorityParam) + + // Push queues a frame in the scheduler. In most cases, this will not be + // called with wr.StreamID()!=0 unless that stream is currently open. The one + // exception is RST_STREAM frames, which may be sent on idle or closed streams. + Push(wr http2FrameWriteRequest) + + // Pop dequeues the next frame to write. Returns false if no frames can + // be written. Frames with a given wr.StreamID() are Pop'd in the same + // order they are Push'd. + Pop() (wr http2FrameWriteRequest, ok bool) +} + +// OpenStreamOptions specifies extra options for WriteScheduler.OpenStream. +type http2OpenStreamOptions struct { + // PusherID is zero if the stream was initiated by the client. Otherwise, + // PusherID names the stream that pushed the newly opened stream. + PusherID uint32 +} + +// FrameWriteRequest is a request to write a frame. +type http2FrameWriteRequest struct { // write is the interface value that does the writing, once the - // writeScheduler (below) has decided to select this frame - // to write. The write functions are all defined in write.go. + // WriteScheduler has selected this frame to write. The write + // functions are all defined in write.go. write http2writeFramer - stream *http2stream // used for prioritization. nil for non-stream frames. + // stream is the stream on which this frame will be written. + // nil for non-stream frames like PING and SETTINGS. + stream *http2stream // done, if non-nil, must be a buffered channel with space for // 1 message and is sent the return value from write (or an @@ -6425,247 +7781,644 @@ type http2frameWriteMsg struct { done chan error } -// for debugging only: -func (wm http2frameWriteMsg) String() string { - var streamID uint32 - if wm.stream != nil { - streamID = wm.stream.id +// StreamID returns the id of the stream this frame will be written to. +// 0 is used for non-stream frames such as PING and SETTINGS. +func (wr http2FrameWriteRequest) StreamID() uint32 { + if wr.stream == nil { + if se, ok := wr.write.(http2StreamError); ok { + + return se.StreamID + } + return 0 + } + return wr.stream.id +} + +// DataSize returns the number of flow control bytes that must be consumed +// to write this entire frame. This is 0 for non-DATA frames. +func (wr http2FrameWriteRequest) DataSize() int { + if wd, ok := wr.write.(*http2writeData); ok { + return len(wd.p) + } + return 0 +} + +// Consume consumes min(n, available) bytes from this frame, where available +// is the number of flow control bytes available on the stream. Consume returns +// 0, 1, or 2 frames, where the integer return value gives the number of frames +// returned. +// +// If flow control prevents consuming any bytes, this returns (_, _, 0). If +// the entire frame was consumed, this returns (wr, _, 1). Otherwise, this +// returns (consumed, rest, 2), where 'consumed' contains the consumed bytes and +// 'rest' contains the remaining bytes. The consumed bytes are deducted from the +// underlying stream's flow control budget. +func (wr http2FrameWriteRequest) Consume(n int32) (http2FrameWriteRequest, http2FrameWriteRequest, int) { + var empty http2FrameWriteRequest + + wd, ok := wr.write.(*http2writeData) + if !ok || len(wd.p) == 0 { + return wr, empty, 1 + } + + allowed := wr.stream.flow.available() + if n < allowed { + allowed = n + } + if wr.stream.sc.maxFrameSize < allowed { + allowed = wr.stream.sc.maxFrameSize + } + if allowed <= 0 { + return empty, empty, 0 + } + if len(wd.p) > int(allowed) { + wr.stream.flow.take(allowed) + consumed := http2FrameWriteRequest{ + stream: wr.stream, + write: &http2writeData{ + streamID: wd.streamID, + p: wd.p[:allowed], + + endStream: false, + }, + + done: nil, + } + rest := http2FrameWriteRequest{ + stream: wr.stream, + write: &http2writeData{ + streamID: wd.streamID, + p: wd.p[allowed:], + endStream: wd.endStream, + }, + done: wr.done, + } + return consumed, rest, 2 } + + wr.stream.flow.take(int32(len(wd.p))) + return wr, empty, 1 +} + +// String is for debugging only. +func (wr http2FrameWriteRequest) String() string { var des string - if s, ok := wm.write.(fmt.Stringer); ok { + if s, ok := wr.write.(fmt.Stringer); ok { des = s.String() } else { - des = fmt.Sprintf("%T", wm.write) + des = fmt.Sprintf("%T", wr.write) } - return fmt.Sprintf("[frameWriteMsg stream=%d, ch=%v, type: %v]", streamID, wm.done != nil, des) + return fmt.Sprintf("[FrameWriteRequest stream=%d, ch=%v, writer=%v]", wr.StreamID(), wr.done != nil, des) } -// writeScheduler tracks pending frames to write, priorities, and decides -// the next one to use. It is not thread-safe. -type http2writeScheduler struct { - // zero are frames not associated with a specific stream. - // They're sent before any stream-specific freams. - zero http2writeQueue +// replyToWriter sends err to wr.done and panics if the send must block +// This does nothing if wr.done is nil. +func (wr *http2FrameWriteRequest) replyToWriter(err error) { + if wr.done == nil { + return + } + select { + case wr.done <- err: + default: + panic(fmt.Sprintf("unbuffered done channel passed in for type %T", wr.write)) + } + wr.write = nil +} - // maxFrameSize is the maximum size of a DATA frame - // we'll write. Must be non-zero and between 16K-16M. - maxFrameSize uint32 +// writeQueue is used by implementations of WriteScheduler. +type http2writeQueue struct { + s []http2FrameWriteRequest +} - // sq contains the stream-specific queues, keyed by stream ID. - // when a stream is idle, it's deleted from the map. - sq map[uint32]*http2writeQueue +func (q *http2writeQueue) empty() bool { return len(q.s) == 0 } - // canSend is a slice of memory that's reused between frame - // scheduling decisions to hold the list of writeQueues (from sq) - // which have enough flow control data to send. After canSend is - // built, the best is selected. - canSend []*http2writeQueue +func (q *http2writeQueue) push(wr http2FrameWriteRequest) { + q.s = append(q.s, wr) +} - // pool of empty queues for reuse. - queuePool []*http2writeQueue +func (q *http2writeQueue) shift() http2FrameWriteRequest { + if len(q.s) == 0 { + panic("invalid use of queue") + } + wr := q.s[0] + + copy(q.s, q.s[1:]) + q.s[len(q.s)-1] = http2FrameWriteRequest{} + q.s = q.s[:len(q.s)-1] + return wr } -func (ws *http2writeScheduler) putEmptyQueue(q *http2writeQueue) { - if len(q.s) != 0 { - panic("queue must be empty") +// consume consumes up to n bytes from q.s[0]. If the frame is +// entirely consumed, it is removed from the queue. If the frame +// is partially consumed, the frame is kept with the consumed +// bytes removed. Returns true iff any bytes were consumed. +func (q *http2writeQueue) consume(n int32) (http2FrameWriteRequest, bool) { + if len(q.s) == 0 { + return http2FrameWriteRequest{}, false + } + consumed, rest, numresult := q.s[0].Consume(n) + switch numresult { + case 0: + return http2FrameWriteRequest{}, false + case 1: + q.shift() + case 2: + q.s[0] = rest } - ws.queuePool = append(ws.queuePool, q) + return consumed, true } -func (ws *http2writeScheduler) getEmptyQueue() *http2writeQueue { - ln := len(ws.queuePool) +type http2writeQueuePool []*http2writeQueue + +// put inserts an unused writeQueue into the pool. +func (p *http2writeQueuePool) put(q *http2writeQueue) { + for i := range q.s { + q.s[i] = http2FrameWriteRequest{} + } + q.s = q.s[:0] + *p = append(*p, q) +} + +// get returns an empty writeQueue. +func (p *http2writeQueuePool) get() *http2writeQueue { + ln := len(*p) if ln == 0 { return new(http2writeQueue) } - q := ws.queuePool[ln-1] - ws.queuePool = ws.queuePool[:ln-1] + x := ln - 1 + q := (*p)[x] + (*p)[x] = nil + *p = (*p)[:x] return q } -func (ws *http2writeScheduler) empty() bool { return ws.zero.empty() && len(ws.sq) == 0 } +// RFC 7540, Section 5.3.5: the default weight is 16. +const http2priorityDefaultWeight = 15 // 16 = 15 + 1 -func (ws *http2writeScheduler) add(wm http2frameWriteMsg) { - st := wm.stream - if st == nil { - ws.zero.push(wm) +// PriorityWriteSchedulerConfig configures a priorityWriteScheduler. +type http2PriorityWriteSchedulerConfig struct { + // MaxClosedNodesInTree controls the maximum number of closed streams to + // retain in the priority tree. Setting this to zero saves a small amount + // of memory at the cost of performance. + // + // See RFC 7540, Section 5.3.4: + // "It is possible for a stream to become closed while prioritization + // information ... is in transit. ... This potentially creates suboptimal + // prioritization, since the stream could be given a priority that is + // different from what is intended. To avoid these problems, an endpoint + // SHOULD retain stream prioritization state for a period after streams + // become closed. The longer state is retained, the lower the chance that + // streams are assigned incorrect or default priority values." + MaxClosedNodesInTree int + + // MaxIdleNodesInTree controls the maximum number of idle streams to + // retain in the priority tree. Setting this to zero saves a small amount + // of memory at the cost of performance. + // + // See RFC 7540, Section 5.3.4: + // Similarly, streams that are in the "idle" state can be assigned + // priority or become a parent of other streams. This allows for the + // creation of a grouping node in the dependency tree, which enables + // more flexible expressions of priority. Idle streams begin with a + // default priority (Section 5.3.5). + MaxIdleNodesInTree int + + // ThrottleOutOfOrderWrites enables write throttling to help ensure that + // data is delivered in priority order. This works around a race where + // stream B depends on stream A and both streams are about to call Write + // to queue DATA frames. If B wins the race, a naive scheduler would eagerly + // write as much data from B as possible, but this is suboptimal because A + // is a higher-priority stream. With throttling enabled, we write a small + // amount of data from B to minimize the amount of bandwidth that B can + // steal from A. + ThrottleOutOfOrderWrites bool +} + +// NewPriorityWriteScheduler constructs a WriteScheduler that schedules +// frames by following HTTP/2 priorities as described in RFC 7340 Section 5.3. +// If cfg is nil, default options are used. +func http2NewPriorityWriteScheduler(cfg *http2PriorityWriteSchedulerConfig) http2WriteScheduler { + if cfg == nil { + + cfg = &http2PriorityWriteSchedulerConfig{ + MaxClosedNodesInTree: 10, + MaxIdleNodesInTree: 10, + ThrottleOutOfOrderWrites: false, + } + } + + ws := &http2priorityWriteScheduler{ + nodes: make(map[uint32]*http2priorityNode), + maxClosedNodesInTree: cfg.MaxClosedNodesInTree, + maxIdleNodesInTree: cfg.MaxIdleNodesInTree, + enableWriteThrottle: cfg.ThrottleOutOfOrderWrites, + } + ws.nodes[0] = &ws.root + if cfg.ThrottleOutOfOrderWrites { + ws.writeThrottleLimit = 1024 } else { - ws.streamQueue(st.id).push(wm) + ws.writeThrottleLimit = math.MaxInt32 } + return ws +} + +type http2priorityNodeState int + +const ( + http2priorityNodeOpen http2priorityNodeState = iota + http2priorityNodeClosed + http2priorityNodeIdle +) + +// priorityNode is a node in an HTTP/2 priority tree. +// Each node is associated with a single stream ID. +// See RFC 7540, Section 5.3. +type http2priorityNode struct { + q http2writeQueue // queue of pending frames to write + id uint32 // id of the stream, or 0 for the root of the tree + weight uint8 // the actual weight is weight+1, so the value is in [1,256] + state http2priorityNodeState // open | closed | idle + bytes int64 // number of bytes written by this node, or 0 if closed + subtreeBytes int64 // sum(node.bytes) of all nodes in this subtree + + // These links form the priority tree. + parent *http2priorityNode + kids *http2priorityNode // start of the kids list + prev, next *http2priorityNode // doubly-linked list of siblings } -func (ws *http2writeScheduler) streamQueue(streamID uint32) *http2writeQueue { - if q, ok := ws.sq[streamID]; ok { - return q +func (n *http2priorityNode) setParent(parent *http2priorityNode) { + if n == parent { + panic("setParent to self") } - if ws.sq == nil { - ws.sq = make(map[uint32]*http2writeQueue) + if n.parent == parent { + return + } + + if parent := n.parent; parent != nil { + if n.prev == nil { + parent.kids = n.next + } else { + n.prev.next = n.next + } + if n.next != nil { + n.next.prev = n.prev + } + } + + n.parent = parent + if parent == nil { + n.next = nil + n.prev = nil + } else { + n.next = parent.kids + n.prev = nil + if n.next != nil { + n.next.prev = n + } + parent.kids = n } - q := ws.getEmptyQueue() - ws.sq[streamID] = q - return q } -// take returns the most important frame to write and removes it from the scheduler. -// It is illegal to call this if the scheduler is empty or if there are no connection-level -// flow control bytes available. -func (ws *http2writeScheduler) take() (wm http2frameWriteMsg, ok bool) { - if ws.maxFrameSize == 0 { - panic("internal error: ws.maxFrameSize not initialized or invalid") +func (n *http2priorityNode) addBytes(b int64) { + n.bytes += b + for ; n != nil; n = n.parent { + n.subtreeBytes += b } +} - if !ws.zero.empty() { - return ws.zero.shift(), true +// walkReadyInOrder iterates over the tree in priority order, calling f for each node +// with a non-empty write queue. When f returns true, this funcion returns true and the +// walk halts. tmp is used as scratch space for sorting. +// +// f(n, openParent) takes two arguments: the node to visit, n, and a bool that is true +// if any ancestor p of n is still open (ignoring the root node). +func (n *http2priorityNode) walkReadyInOrder(openParent bool, tmp *[]*http2priorityNode, f func(*http2priorityNode, bool) bool) bool { + if !n.q.empty() && f(n, openParent) { + return true } - if len(ws.sq) == 0 { - return + if n.kids == nil { + return false } - for id, q := range ws.sq { - if q.firstIsNoCost() { - return ws.takeFrom(id, q) + if n.id != 0 { + openParent = openParent || (n.state == http2priorityNodeOpen) + } + + w := n.kids.weight + needSort := false + for k := n.kids.next; k != nil; k = k.next { + if k.weight != w { + needSort = true + break } } + if !needSort { + for k := n.kids; k != nil; k = k.next { + if k.walkReadyInOrder(openParent, tmp, f) { + return true + } + } + return false + } - if len(ws.canSend) != 0 { - panic("should be empty") + *tmp = (*tmp)[:0] + for n.kids != nil { + *tmp = append(*tmp, n.kids) + n.kids.setParent(nil) } - for _, q := range ws.sq { - if n := ws.streamWritableBytes(q); n > 0 { - ws.canSend = append(ws.canSend, q) + sort.Sort(http2sortPriorityNodeSiblings(*tmp)) + for i := len(*tmp) - 1; i >= 0; i-- { + (*tmp)[i].setParent(n) + } + for k := n.kids; k != nil; k = k.next { + if k.walkReadyInOrder(openParent, tmp, f) { + return true } } - if len(ws.canSend) == 0 { - return + return false +} + +type http2sortPriorityNodeSiblings []*http2priorityNode + +func (z http2sortPriorityNodeSiblings) Len() int { return len(z) } + +func (z http2sortPriorityNodeSiblings) Swap(i, k int) { z[i], z[k] = z[k], z[i] } + +func (z http2sortPriorityNodeSiblings) Less(i, k int) bool { + + wi, bi := float64(z[i].weight+1), float64(z[i].subtreeBytes) + wk, bk := float64(z[k].weight+1), float64(z[k].subtreeBytes) + if bi == 0 && bk == 0 { + return wi >= wk + } + if bk == 0 { + return false } - defer ws.zeroCanSend() + return bi/bk <= wi/wk +} + +type http2priorityWriteScheduler struct { + // root is the root of the priority tree, where root.id = 0. + // The root queues control frames that are not associated with any stream. + root http2priorityNode + + // nodes maps stream ids to priority tree nodes. + nodes map[uint32]*http2priorityNode + + // maxID is the maximum stream id in nodes. + maxID uint32 + + // lists of nodes that have been closed or are idle, but are kept in + // the tree for improved prioritization. When the lengths exceed either + // maxClosedNodesInTree or maxIdleNodesInTree, old nodes are discarded. + closedNodes, idleNodes []*http2priorityNode - q := ws.canSend[0] + // From the config. + maxClosedNodesInTree int + maxIdleNodesInTree int + writeThrottleLimit int32 + enableWriteThrottle bool - return ws.takeFrom(q.streamID(), q) + // tmp is scratch space for priorityNode.walkReadyInOrder to reduce allocations. + tmp []*http2priorityNode + + // pool of empty queues for reuse. + queuePool http2writeQueuePool } -// zeroCanSend is defered from take. -func (ws *http2writeScheduler) zeroCanSend() { - for i := range ws.canSend { - ws.canSend[i] = nil +func (ws *http2priorityWriteScheduler) OpenStream(streamID uint32, options http2OpenStreamOptions) { + + if curr := ws.nodes[streamID]; curr != nil { + if curr.state != http2priorityNodeIdle { + panic(fmt.Sprintf("stream %d already opened", streamID)) + } + curr.state = http2priorityNodeOpen + return + } + + parent := ws.nodes[options.PusherID] + if parent == nil { + parent = &ws.root + } + n := &http2priorityNode{ + q: *ws.queuePool.get(), + id: streamID, + weight: http2priorityDefaultWeight, + state: http2priorityNodeOpen, + } + n.setParent(parent) + ws.nodes[streamID] = n + if streamID > ws.maxID { + ws.maxID = streamID } - ws.canSend = ws.canSend[:0] } -// streamWritableBytes returns the number of DATA bytes we could write -// from the given queue's stream, if this stream/queue were -// selected. It is an error to call this if q's head isn't a -// *writeData. -func (ws *http2writeScheduler) streamWritableBytes(q *http2writeQueue) int32 { - wm := q.head() - ret := wm.stream.flow.available() - if ret == 0 { - return 0 +func (ws *http2priorityWriteScheduler) CloseStream(streamID uint32) { + if streamID == 0 { + panic("violation of WriteScheduler interface: cannot close stream 0") } - if int32(ws.maxFrameSize) < ret { - ret = int32(ws.maxFrameSize) + if ws.nodes[streamID] == nil { + panic(fmt.Sprintf("violation of WriteScheduler interface: unknown stream %d", streamID)) } - if ret == 0 { - panic("internal error: ws.maxFrameSize not initialized or invalid") + if ws.nodes[streamID].state != http2priorityNodeOpen { + panic(fmt.Sprintf("violation of WriteScheduler interface: stream %d already closed", streamID)) } - wd := wm.write.(*http2writeData) - if len(wd.p) < int(ret) { - ret = int32(len(wd.p)) + + n := ws.nodes[streamID] + n.state = http2priorityNodeClosed + n.addBytes(-n.bytes) + + q := n.q + ws.queuePool.put(&q) + n.q.s = nil + if ws.maxClosedNodesInTree > 0 { + ws.addClosedOrIdleNode(&ws.closedNodes, ws.maxClosedNodesInTree, n) + } else { + ws.removeNode(n) } - return ret } -func (ws *http2writeScheduler) takeFrom(id uint32, q *http2writeQueue) (wm http2frameWriteMsg, ok bool) { - wm = q.head() - - if wd, ok := wm.write.(*http2writeData); ok && len(wd.p) > 0 { - allowed := wm.stream.flow.available() - if allowed == 0 { +func (ws *http2priorityWriteScheduler) AdjustStream(streamID uint32, priority http2PriorityParam) { + if streamID == 0 { + panic("adjustPriority on root") + } - return http2frameWriteMsg{}, false + n := ws.nodes[streamID] + if n == nil { + if streamID <= ws.maxID || ws.maxIdleNodesInTree == 0 { + return } - if int32(ws.maxFrameSize) < allowed { - allowed = int32(ws.maxFrameSize) + ws.maxID = streamID + n = &http2priorityNode{ + q: *ws.queuePool.get(), + id: streamID, + weight: http2priorityDefaultWeight, + state: http2priorityNodeIdle, } + n.setParent(&ws.root) + ws.nodes[streamID] = n + ws.addClosedOrIdleNode(&ws.idleNodes, ws.maxIdleNodesInTree, n) + } - if len(wd.p) > int(allowed) { - wm.stream.flow.take(allowed) - chunk := wd.p[:allowed] - wd.p = wd.p[allowed:] + parent := ws.nodes[priority.StreamDep] + if parent == nil { + n.setParent(&ws.root) + n.weight = http2priorityDefaultWeight + return + } - return http2frameWriteMsg{ - stream: wm.stream, - write: &http2writeData{ - streamID: wd.streamID, - p: chunk, + if n == parent { + return + } - endStream: false, - }, + for x := parent.parent; x != nil; x = x.parent { + if x == n { + parent.setParent(n.parent) + break + } + } - done: nil, - }, true + if priority.Exclusive { + k := parent.kids + for k != nil { + next := k.next + if k != n { + k.setParent(n) + } + k = next } - wm.stream.flow.take(int32(len(wd.p))) } - q.shift() - if q.empty() { - ws.putEmptyQueue(q) - delete(ws.sq, id) + n.setParent(parent) + n.weight = priority.Weight +} + +func (ws *http2priorityWriteScheduler) Push(wr http2FrameWriteRequest) { + var n *http2priorityNode + if id := wr.StreamID(); id == 0 { + n = &ws.root + } else { + n = ws.nodes[id] + if n == nil { + + if wr.DataSize() > 0 { + panic("add DATA on non-open stream") + } + n = &ws.root + } } - return wm, true + n.q.push(wr) } -func (ws *http2writeScheduler) forgetStream(id uint32) { - q, ok := ws.sq[id] - if !ok { +func (ws *http2priorityWriteScheduler) Pop() (wr http2FrameWriteRequest, ok bool) { + ws.root.walkReadyInOrder(false, &ws.tmp, func(n *http2priorityNode, openParent bool) bool { + limit := int32(math.MaxInt32) + if openParent { + limit = ws.writeThrottleLimit + } + wr, ok = n.q.consume(limit) + if !ok { + return false + } + n.addBytes(int64(wr.DataSize())) + + if openParent { + ws.writeThrottleLimit += 1024 + if ws.writeThrottleLimit < 0 { + ws.writeThrottleLimit = math.MaxInt32 + } + } else if ws.enableWriteThrottle { + ws.writeThrottleLimit = 1024 + } + return true + }) + return wr, ok +} + +func (ws *http2priorityWriteScheduler) addClosedOrIdleNode(list *[]*http2priorityNode, maxSize int, n *http2priorityNode) { + if maxSize == 0 { return } - delete(ws.sq, id) + if len(*list) == maxSize { - for i := range q.s { - q.s[i] = http2frameWriteMsg{} + ws.removeNode((*list)[0]) + x := (*list)[1:] + copy(*list, x) + *list = (*list)[:len(x)] } - q.s = q.s[:0] - ws.putEmptyQueue(q) + *list = append(*list, n) } -type http2writeQueue struct { - s []http2frameWriteMsg +func (ws *http2priorityWriteScheduler) removeNode(n *http2priorityNode) { + for k := n.kids; k != nil; k = k.next { + k.setParent(n.parent) + } + n.setParent(nil) + delete(ws.nodes, n.id) } -// streamID returns the stream ID for a non-empty stream-specific queue. -func (q *http2writeQueue) streamID() uint32 { return q.s[0].stream.id } +// NewRandomWriteScheduler constructs a WriteScheduler that ignores HTTP/2 +// priorities. Control frames like SETTINGS and PING are written before DATA +// frames, but if no control frames are queued and multiple streams have queued +// HEADERS or DATA frames, Pop selects a ready stream arbitrarily. +func http2NewRandomWriteScheduler() http2WriteScheduler { + return &http2randomWriteScheduler{sq: make(map[uint32]*http2writeQueue)} +} -func (q *http2writeQueue) empty() bool { return len(q.s) == 0 } +type http2randomWriteScheduler struct { + // zero are frames not associated with a specific stream. + zero http2writeQueue + + // sq contains the stream-specific queues, keyed by stream ID. + // When a stream is idle or closed, it's deleted from the map. + sq map[uint32]*http2writeQueue -func (q *http2writeQueue) push(wm http2frameWriteMsg) { - q.s = append(q.s, wm) + // pool of empty queues for reuse. + queuePool http2writeQueuePool } -// head returns the next item that would be removed by shift. -func (q *http2writeQueue) head() http2frameWriteMsg { - if len(q.s) == 0 { - panic("invalid use of queue") - } - return q.s[0] +func (ws *http2randomWriteScheduler) OpenStream(streamID uint32, options http2OpenStreamOptions) { + } -func (q *http2writeQueue) shift() http2frameWriteMsg { - if len(q.s) == 0 { - panic("invalid use of queue") +func (ws *http2randomWriteScheduler) CloseStream(streamID uint32) { + q, ok := ws.sq[streamID] + if !ok { + return } - wm := q.s[0] + delete(ws.sq, streamID) + ws.queuePool.put(q) +} + +func (ws *http2randomWriteScheduler) AdjustStream(streamID uint32, priority http2PriorityParam) { - copy(q.s, q.s[1:]) - q.s[len(q.s)-1] = http2frameWriteMsg{} - q.s = q.s[:len(q.s)-1] - return wm } -func (q *http2writeQueue) firstIsNoCost() bool { - if df, ok := q.s[0].write.(*http2writeData); ok { - return len(df.p) == 0 +func (ws *http2randomWriteScheduler) Push(wr http2FrameWriteRequest) { + id := wr.StreamID() + if id == 0 { + ws.zero.push(wr) + return } - return true + q, ok := ws.sq[id] + if !ok { + q = ws.queuePool.get() + ws.sq[id] = q + } + q.push(wr) +} + +func (ws *http2randomWriteScheduler) Pop() (http2FrameWriteRequest, bool) { + + if !ws.zero.empty() { + return ws.zero.shift(), true + } + + for _, q := range ws.sq { + if wr, ok := q.consume(math.MaxInt32); ok { + return wr, true + } + } + return http2FrameWriteRequest{}, false } diff --git a/libgo/go/net/http/header.go b/libgo/go/net/http/header.go index 049f32f27d..832169247f 100644 --- a/libgo/go/net/http/header.go +++ b/libgo/go/net/http/header.go @@ -1,4 +1,4 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -25,16 +25,18 @@ func (h Header) Add(key, value string) { } // Set sets the header entries associated with key to -// the single element value. It replaces any existing +// the single element value. It replaces any existing // values associated with key. func (h Header) Set(key, value string) { textproto.MIMEHeader(h).Set(key, value) } // Get gets the first value associated with the given key. +// It is case insensitive; textproto.CanonicalMIMEHeaderKey is used +// to canonicalize the provided key. // If there are no values associated with the key, Get returns "". -// To access multiple values of a key, access the map directly -// with CanonicalHeaderKey. +// To access multiple values of a key, or to use non-canonical keys, +// access the map directly. func (h Header) Get(key string) string { return textproto.MIMEHeader(h).Get(key) } @@ -164,9 +166,9 @@ func (h Header) WriteSubset(w io.Writer, exclude map[string]bool) error { } // CanonicalHeaderKey returns the canonical format of the -// header key s. The canonicalization converts the first +// header key s. The canonicalization converts the first // letter and any letter following a hyphen to upper case; -// the rest are converted to lowercase. For example, the +// the rest are converted to lowercase. For example, the // canonical key for "accept-encoding" is "Accept-Encoding". // If s contains a space or invalid header field bytes, it is // returned without modifications. @@ -186,7 +188,7 @@ func hasToken(v, token string) bool { for sp := 0; sp <= len(v)-len(token); sp++ { // Check that first character is good. // The token is ASCII, so checking only a single byte - // is sufficient. We skip this potential starting + // is sufficient. We skip this potential starting // position if both the first byte and its potential // ASCII uppercase equivalent (b|0x20) don't match. // False positives ('^' => '~') are caught by EqualFold. diff --git a/libgo/go/net/http/header_test.go b/libgo/go/net/http/header_test.go index 299576ba8c..5c0de15b73 100644 --- a/libgo/go/net/http/header_test.go +++ b/libgo/go/net/http/header_test.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/http/http.go b/libgo/go/net/http/http.go new file mode 100644 index 0000000000..b95ca89f40 --- /dev/null +++ b/libgo/go/net/http/http.go @@ -0,0 +1,141 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +import ( + "io" + "strconv" + "strings" + "time" + "unicode/utf8" + + "golang_org/x/net/lex/httplex" +) + +// maxInt64 is the effective "infinite" value for the Server and +// Transport's byte-limiting readers. +const maxInt64 = 1<<63 - 1 + +// aLongTimeAgo is a non-zero time, far in the past, used for +// immediate cancelation of network operations. +var aLongTimeAgo = time.Unix(1, 0) + +// TODO(bradfitz): move common stuff here. The other files have accumulated +// generic http stuff in random places. + +// contextKey is a value for use with context.WithValue. It's used as +// a pointer so it fits in an interface{} without allocation. +type contextKey struct { + name string +} + +func (k *contextKey) String() string { return "net/http context value " + k.name } + +// Given a string of the form "host", "host:port", or "[ipv6::address]:port", +// return true if the string includes a port. +func hasPort(s string) bool { return strings.LastIndex(s, ":") > strings.LastIndex(s, "]") } + +// removeEmptyPort strips the empty port in ":port" to "" +// as mandated by RFC 3986 Section 6.2.3. +func removeEmptyPort(host string) string { + if hasPort(host) { + return strings.TrimSuffix(host, ":") + } + return host +} + +func isNotToken(r rune) bool { + return !httplex.IsTokenRune(r) +} + +func isASCII(s string) bool { + for i := 0; i < len(s); i++ { + if s[i] >= utf8.RuneSelf { + return false + } + } + return true +} + +func hexEscapeNonASCII(s string) string { + newLen := 0 + for i := 0; i < len(s); i++ { + if s[i] >= utf8.RuneSelf { + newLen += 3 + } else { + newLen++ + } + } + if newLen == len(s) { + return s + } + b := make([]byte, 0, newLen) + for i := 0; i < len(s); i++ { + if s[i] >= utf8.RuneSelf { + b = append(b, '%') + b = strconv.AppendInt(b, int64(s[i]), 16) + } else { + b = append(b, s[i]) + } + } + return string(b) +} + +// NoBody is an io.ReadCloser with no bytes. Read always returns EOF +// and Close always returns nil. It can be used in an outgoing client +// request to explicitly signal that a request has zero bytes. +// An alternative, however, is to simply set Request.Body to nil. +var NoBody = noBody{} + +type noBody struct{} + +func (noBody) Read([]byte) (int, error) { return 0, io.EOF } +func (noBody) Close() error { return nil } +func (noBody) WriteTo(io.Writer) (int64, error) { return 0, nil } + +var ( + // verify that an io.Copy from NoBody won't require a buffer: + _ io.WriterTo = NoBody + _ io.ReadCloser = NoBody +) + +// PushOptions describes options for Pusher.Push. +type PushOptions struct { + // Method specifies the HTTP method for the promised request. + // If set, it must be "GET" or "HEAD". Empty means "GET". + Method string + + // Header specifies additional promised request headers. This cannot + // include HTTP/2 pseudo header fields like ":path" and ":scheme", + // which will be added automatically. + Header Header +} + +// Pusher is the interface implemented by ResponseWriters that support +// HTTP/2 server push. For more background, see +// https://tools.ietf.org/html/rfc7540#section-8.2. +type Pusher interface { + // Push initiates an HTTP/2 server push. This constructs a synthetic + // request using the given target and options, serializes that request + // into a PUSH_PROMISE frame, then dispatches that request using the + // server's request handler. If opts is nil, default options are used. + // + // The target must either be an absolute path (like "/path") or an absolute + // URL that contains a valid host and the same scheme as the parent request. + // If the target is a path, it will inherit the scheme and host of the + // parent request. + // + // The HTTP/2 spec disallows recursive pushes and cross-authority pushes. + // Push may or may not detect these invalid pushes; however, invalid + // pushes will be detected and canceled by conforming clients. + // + // Handlers that wish to push URL X should call Push before sending any + // data that may trigger a request for URL X. This avoids a race where the + // client issues requests for X before receiving the PUSH_PROMISE for X. + // + // Push returns ErrNotSupported if the client has disabled push or if push + // is not supported on the underlying connection. + Push(target string, opts *PushOptions) error +} diff --git a/libgo/go/net/http/http_test.go b/libgo/go/net/http/http_test.go index dead3b0454..8f466bb366 100644 --- a/libgo/go/net/http/http_test.go +++ b/libgo/go/net/http/http_test.go @@ -2,15 +2,23 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Tests of internal functions with no better homes. +// Tests of internal functions and things with no better homes. package http import ( + "bytes" + "internal/testenv" + "os/exec" "reflect" "testing" + "time" ) +func init() { + shutdownPollInterval = 5 * time.Millisecond +} + func TestForeachHeaderElement(t *testing.T) { tests := []struct { in string @@ -48,6 +56,18 @@ func TestCleanHost(t *testing.T) { {"www.google.com foo", "www.google.com"}, {"www.google.com/foo", "www.google.com"}, {" first character is a space", ""}, + {"[1::6]:8080", "[1::6]:8080"}, + + // Punycode: + {"гофер.рф/foo", "xn--c1ae0ajs.xn--p1ai"}, + {"bücher.de", "xn--bcher-kva.de"}, + {"bücher.de:8080", "xn--bcher-kva.de:8080"}, + // Verify we convert to lowercase before punycode: + {"BÜCHER.de", "xn--bcher-kva.de"}, + {"BÜCHER.de:8080", "xn--bcher-kva.de:8080"}, + // Verify we normalize to NFC before punycode: + {"gophér.nfc", "xn--gophr-esa.nfc"}, // NFC input; no work needed + {"goph\u0065\u0301r.nfd", "xn--gophr-esa.nfd"}, // NFD input } for _, tt := range tests { got := cleanHost(tt.in) @@ -56,3 +76,36 @@ func TestCleanHost(t *testing.T) { } } } + +// Test that cmd/go doesn't link in the HTTP server. +// +// This catches accidental dependencies between the HTTP transport and +// server code. +func TestCmdGoNoHTTPServer(t *testing.T) { + t.Parallel() + goBin := testenv.GoToolPath(t) + out, err := exec.Command(goBin, "tool", "nm", goBin).CombinedOutput() + if err != nil { + t.Fatalf("go tool nm: %v: %s", err, out) + } + wantSym := map[string]bool{ + // Verify these exist: (sanity checking this test) + "net/http.(*Client).Get": true, + "net/http.(*Transport).RoundTrip": true, + + // Verify these don't exist: + "net/http.http2Server": false, + "net/http.(*Server).Serve": false, + "net/http.(*ServeMux).ServeHTTP": false, + "net/http.DefaultServeMux": false, + } + for sym, want := range wantSym { + got := bytes.Contains(out, []byte(sym)) + if !want && got { + t.Errorf("cmd/go unexpectedly links in HTTP server code; found symbol %q in cmd/go", sym) + } + if want && !got { + t.Errorf("expected to find symbol %q in cmd/go; not found", sym) + } + } +} diff --git a/libgo/go/net/http/httptest/httptest.go b/libgo/go/net/http/httptest/httptest.go new file mode 100644 index 0000000000..f7202da92f --- /dev/null +++ b/libgo/go/net/http/httptest/httptest.go @@ -0,0 +1,91 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package httptest provides utilities for HTTP testing. +package httptest + +import ( + "bufio" + "bytes" + "crypto/tls" + "io" + "io/ioutil" + "net/http" + "strings" +) + +// NewRequest returns a new incoming server Request, suitable +// for passing to an http.Handler for testing. +// +// The target is the RFC 7230 "request-target": it may be either a +// path or an absolute URL. If target is an absolute URL, the host name +// from the URL is used. Otherwise, "example.com" is used. +// +// The TLS field is set to a non-nil dummy value if target has scheme +// "https". +// +// The Request.Proto is always HTTP/1.1. +// +// An empty method means "GET". +// +// The provided body may be nil. If the body is of type *bytes.Reader, +// *strings.Reader, or *bytes.Buffer, the Request.ContentLength is +// set. +// +// NewRequest panics on error for ease of use in testing, where a +// panic is acceptable. +// +// To generate a client HTTP request instead of a server request, see +// the NewRequest function in the net/http package. +func NewRequest(method, target string, body io.Reader) *http.Request { + if method == "" { + method = "GET" + } + req, err := http.ReadRequest(bufio.NewReader(strings.NewReader(method + " " + target + " HTTP/1.0\r\n\r\n"))) + if err != nil { + panic("invalid NewRequest arguments; " + err.Error()) + } + + // HTTP/1.0 was used above to avoid needing a Host field. Change it to 1.1 here. + req.Proto = "HTTP/1.1" + req.ProtoMinor = 1 + req.Close = false + + if body != nil { + switch v := body.(type) { + case *bytes.Buffer: + req.ContentLength = int64(v.Len()) + case *bytes.Reader: + req.ContentLength = int64(v.Len()) + case *strings.Reader: + req.ContentLength = int64(v.Len()) + default: + req.ContentLength = -1 + } + if rc, ok := body.(io.ReadCloser); ok { + req.Body = rc + } else { + req.Body = ioutil.NopCloser(body) + } + } + + // 192.0.2.0/24 is "TEST-NET" in RFC 5737 for use solely in + // documentation and example source code and should not be + // used publicly. + req.RemoteAddr = "192.0.2.1:1234" + + if req.Host == "" { + req.Host = "example.com" + } + + if strings.HasPrefix(target, "https://") { + req.TLS = &tls.ConnectionState{ + Version: tls.VersionTLS12, + HandshakeComplete: true, + ServerName: req.Host, + } + } + + return req +} diff --git a/libgo/go/net/http/httptest/httptest_test.go b/libgo/go/net/http/httptest/httptest_test.go new file mode 100644 index 0000000000..4f9ecbd8bb --- /dev/null +++ b/libgo/go/net/http/httptest/httptest_test.go @@ -0,0 +1,177 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package httptest + +import ( + "crypto/tls" + "io" + "io/ioutil" + "net/http" + "net/url" + "reflect" + "strings" + "testing" +) + +func TestNewRequest(t *testing.T) { + tests := [...]struct { + method, uri string + body io.Reader + + want *http.Request + wantBody string + }{ + // Empty method means GET: + 0: { + method: "", + uri: "/", + body: nil, + want: &http.Request{ + Method: "GET", + Host: "example.com", + URL: &url.URL{Path: "/"}, + Header: http.Header{}, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + RemoteAddr: "192.0.2.1:1234", + RequestURI: "/", + }, + wantBody: "", + }, + + // GET with full URL: + 1: { + method: "GET", + uri: "http://foo.com/path/%2f/bar/", + body: nil, + want: &http.Request{ + Method: "GET", + Host: "foo.com", + URL: &url.URL{ + Scheme: "http", + Path: "/path///bar/", + RawPath: "/path/%2f/bar/", + Host: "foo.com", + }, + Header: http.Header{}, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + RemoteAddr: "192.0.2.1:1234", + RequestURI: "http://foo.com/path/%2f/bar/", + }, + wantBody: "", + }, + + // GET with full https URL: + 2: { + method: "GET", + uri: "https://foo.com/path/", + body: nil, + want: &http.Request{ + Method: "GET", + Host: "foo.com", + URL: &url.URL{ + Scheme: "https", + Path: "/path/", + Host: "foo.com", + }, + Header: http.Header{}, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + RemoteAddr: "192.0.2.1:1234", + RequestURI: "https://foo.com/path/", + TLS: &tls.ConnectionState{ + Version: tls.VersionTLS12, + HandshakeComplete: true, + ServerName: "foo.com", + }, + }, + wantBody: "", + }, + + // Post with known length + 3: { + method: "POST", + uri: "/", + body: strings.NewReader("foo"), + want: &http.Request{ + Method: "POST", + Host: "example.com", + URL: &url.URL{Path: "/"}, + Header: http.Header{}, + Proto: "HTTP/1.1", + ContentLength: 3, + ProtoMajor: 1, + ProtoMinor: 1, + RemoteAddr: "192.0.2.1:1234", + RequestURI: "/", + }, + wantBody: "foo", + }, + + // Post with unknown length + 4: { + method: "POST", + uri: "/", + body: struct{ io.Reader }{strings.NewReader("foo")}, + want: &http.Request{ + Method: "POST", + Host: "example.com", + URL: &url.URL{Path: "/"}, + Header: http.Header{}, + Proto: "HTTP/1.1", + ContentLength: -1, + ProtoMajor: 1, + ProtoMinor: 1, + RemoteAddr: "192.0.2.1:1234", + RequestURI: "/", + }, + wantBody: "foo", + }, + + // OPTIONS * + 5: { + method: "OPTIONS", + uri: "*", + want: &http.Request{ + Method: "OPTIONS", + Host: "example.com", + URL: &url.URL{Path: "*"}, + Header: http.Header{}, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + RemoteAddr: "192.0.2.1:1234", + RequestURI: "*", + }, + }, + } + for i, tt := range tests { + got := NewRequest(tt.method, tt.uri, tt.body) + slurp, err := ioutil.ReadAll(got.Body) + if err != nil { + t.Errorf("%d. ReadAll: %v", i, err) + } + if string(slurp) != tt.wantBody { + t.Errorf("%d. Body = %q; want %q", i, slurp, tt.wantBody) + } + got.Body = nil // before DeepEqual + if !reflect.DeepEqual(got.URL, tt.want.URL) { + t.Errorf("%d. Request.URL mismatch:\n got: %#v\nwant: %#v", i, got.URL, tt.want.URL) + } + if !reflect.DeepEqual(got.Header, tt.want.Header) { + t.Errorf("%d. Request.Header mismatch:\n got: %#v\nwant: %#v", i, got.Header, tt.want.Header) + } + if !reflect.DeepEqual(got.TLS, tt.want.TLS) { + t.Errorf("%d. Request.TLS mismatch:\n got: %#v\nwant: %#v", i, got.TLS, tt.want.TLS) + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("%d. Request mismatch:\n got: %#v\nwant: %#v", i, got, tt.want) + } + } +} diff --git a/libgo/go/net/http/httptest/recorder.go b/libgo/go/net/http/httptest/recorder.go index 7c51af1867..5f1aa6af47 100644 --- a/libgo/go/net/http/httptest/recorder.go +++ b/libgo/go/net/http/httptest/recorder.go @@ -2,22 +2,42 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Package httptest provides utilities for HTTP testing. package httptest import ( "bytes" + "io/ioutil" "net/http" + "strconv" + "strings" ) // ResponseRecorder is an implementation of http.ResponseWriter that // records its mutations for later inspection in tests. type ResponseRecorder struct { - Code int // the HTTP response code from WriteHeader - HeaderMap http.Header // the HTTP response headers - Body *bytes.Buffer // if non-nil, the bytes.Buffer to append written data to - Flushed bool + // Code is the HTTP response code set by WriteHeader. + // + // Note that if a Handler never calls WriteHeader or Write, + // this might end up being 0, rather than the implicit + // http.StatusOK. To get the implicit value, use the Result + // method. + Code int + // HeaderMap contains the headers explicitly set by the Handler. + // + // To get the implicit headers set by the server (such as + // automatic Content-Type), use the Result method. + HeaderMap http.Header + + // Body is the buffer to which the Handler's Write calls are sent. + // If nil, the Writes are silently discarded. + Body *bytes.Buffer + + // Flushed is whether the Handler called Flush. + Flushed bool + + result *http.Response // cache of Result's return value + snapHeader http.Header // snapshot of HeaderMap at first Write wroteHeader bool } @@ -59,16 +79,15 @@ func (rw *ResponseRecorder) writeHeader(b []byte, str string) { str = str[:512] } - _, hasType := rw.HeaderMap["Content-Type"] - hasTE := rw.HeaderMap.Get("Transfer-Encoding") != "" + m := rw.Header() + + _, hasType := m["Content-Type"] + hasTE := m.Get("Transfer-Encoding") != "" if !hasType && !hasTE { if b == nil { b = []byte(str) } - if rw.HeaderMap == nil { - rw.HeaderMap = make(http.Header) - } - rw.HeaderMap.Set("Content-Type", http.DetectContentType(b)) + m.Set("Content-Type", http.DetectContentType(b)) } rw.WriteHeader(200) @@ -92,12 +111,28 @@ func (rw *ResponseRecorder) WriteString(str string) (int, error) { return len(str), nil } -// WriteHeader sets rw.Code. +// WriteHeader sets rw.Code. After it is called, changing rw.Header +// will not affect rw.HeaderMap. func (rw *ResponseRecorder) WriteHeader(code int) { - if !rw.wroteHeader { - rw.Code = code - rw.wroteHeader = true + if rw.wroteHeader { + return + } + rw.Code = code + rw.wroteHeader = true + if rw.HeaderMap == nil { + rw.HeaderMap = make(http.Header) } + rw.snapHeader = cloneHeader(rw.HeaderMap) +} + +func cloneHeader(h http.Header) http.Header { + h2 := make(http.Header, len(h)) + for k, vv := range h { + vv2 := make([]string, len(vv)) + copy(vv2, vv) + h2[k] = vv2 + } + return h2 } // Flush sets rw.Flushed to true. @@ -107,3 +142,94 @@ func (rw *ResponseRecorder) Flush() { } rw.Flushed = true } + +// Result returns the response generated by the handler. +// +// The returned Response will have at least its StatusCode, +// Header, Body, and optionally Trailer populated. +// More fields may be populated in the future, so callers should +// not DeepEqual the result in tests. +// +// The Response.Header is a snapshot of the headers at the time of the +// first write call, or at the time of this call, if the handler never +// did a write. +// +// The Response.Body is guaranteed to be non-nil and Body.Read call is +// guaranteed to not return any error other than io.EOF. +// +// Result must only be called after the handler has finished running. +func (rw *ResponseRecorder) Result() *http.Response { + if rw.result != nil { + return rw.result + } + if rw.snapHeader == nil { + rw.snapHeader = cloneHeader(rw.HeaderMap) + } + res := &http.Response{ + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + StatusCode: rw.Code, + Header: rw.snapHeader, + } + rw.result = res + if res.StatusCode == 0 { + res.StatusCode = 200 + } + res.Status = http.StatusText(res.StatusCode) + if rw.Body != nil { + res.Body = ioutil.NopCloser(bytes.NewReader(rw.Body.Bytes())) + } + res.ContentLength = parseContentLength(res.Header.Get("Content-Length")) + + if trailers, ok := rw.snapHeader["Trailer"]; ok { + res.Trailer = make(http.Header, len(trailers)) + for _, k := range trailers { + // TODO: use http2.ValidTrailerHeader, but we can't + // get at it easily because it's bundled into net/http + // unexported. This is good enough for now: + switch k { + case "Transfer-Encoding", "Content-Length", "Trailer": + // Ignore since forbidden by RFC 2616 14.40. + continue + } + k = http.CanonicalHeaderKey(k) + vv, ok := rw.HeaderMap[k] + if !ok { + continue + } + vv2 := make([]string, len(vv)) + copy(vv2, vv) + res.Trailer[k] = vv2 + } + } + for k, vv := range rw.HeaderMap { + if !strings.HasPrefix(k, http.TrailerPrefix) { + continue + } + if res.Trailer == nil { + res.Trailer = make(http.Header) + } + for _, v := range vv { + res.Trailer.Add(strings.TrimPrefix(k, http.TrailerPrefix), v) + } + } + return res +} + +// parseContentLength trims whitespace from s and returns -1 if no value +// is set, or the value if it's >= 0. +// +// This a modified version of same function found in net/http/transfer.go. This +// one just ignores an invalid header. +func parseContentLength(cl string) int64 { + cl = strings.TrimSpace(cl) + if cl == "" { + return -1 + } + n, err := strconv.ParseInt(cl, 10, 64) + if err != nil { + return -1 + } + return n +} diff --git a/libgo/go/net/http/httptest/recorder_test.go b/libgo/go/net/http/httptest/recorder_test.go index c29b6d4cf9..9afba4e556 100644 --- a/libgo/go/net/http/httptest/recorder_test.go +++ b/libgo/go/net/http/httptest/recorder_test.go @@ -23,6 +23,14 @@ func TestRecorder(t *testing.T) { return nil } } + hasResultStatus := func(wantCode int) checkFunc { + return func(rec *ResponseRecorder) error { + if rec.Result().StatusCode != wantCode { + return fmt.Errorf("Result().StatusCode = %d; want %d", rec.Result().StatusCode, wantCode) + } + return nil + } + } hasContents := func(want string) checkFunc { return func(rec *ResponseRecorder) error { if rec.Body.String() != want { @@ -39,10 +47,57 @@ func TestRecorder(t *testing.T) { return nil } } - hasHeader := func(key, want string) checkFunc { + hasOldHeader := func(key, want string) checkFunc { return func(rec *ResponseRecorder) error { if got := rec.HeaderMap.Get(key); got != want { - return fmt.Errorf("header %s = %q; want %q", key, got, want) + return fmt.Errorf("HeaderMap header %s = %q; want %q", key, got, want) + } + return nil + } + } + hasHeader := func(key, want string) checkFunc { + return func(rec *ResponseRecorder) error { + if got := rec.Result().Header.Get(key); got != want { + return fmt.Errorf("final header %s = %q; want %q", key, got, want) + } + return nil + } + } + hasNotHeaders := func(keys ...string) checkFunc { + return func(rec *ResponseRecorder) error { + for _, k := range keys { + v, ok := rec.Result().Header[http.CanonicalHeaderKey(k)] + if ok { + return fmt.Errorf("unexpected header %s with value %q", k, v) + } + } + return nil + } + } + hasTrailer := func(key, want string) checkFunc { + return func(rec *ResponseRecorder) error { + if got := rec.Result().Trailer.Get(key); got != want { + return fmt.Errorf("trailer %s = %q; want %q", key, got, want) + } + return nil + } + } + hasNotTrailers := func(keys ...string) checkFunc { + return func(rec *ResponseRecorder) error { + trailers := rec.Result().Trailer + for _, k := range keys { + _, ok := trailers[http.CanonicalHeaderKey(k)] + if ok { + return fmt.Errorf("unexpected trailer %s", k) + } + } + return nil + } + } + hasContentLength := func(length int64) checkFunc { + return func(rec *ResponseRecorder) error { + if got := rec.Result().ContentLength; got != length { + return fmt.Errorf("ContentLength = %d; want %d", got, length) } return nil } @@ -94,7 +149,7 @@ func TestRecorder(t *testing.T) { w.(http.Flusher).Flush() // also sends a 200 w.WriteHeader(201) }, - check(hasStatus(200), hasFlush(true)), + check(hasStatus(200), hasFlush(true), hasContentLength(-1)), }, { "Content-Type detection", @@ -130,6 +185,85 @@ func TestRecorder(t *testing.T) { }, check(hasHeader("Content-Type", "text/html; charset=utf-8")), }, + { + "Header is not changed after write", + func(w http.ResponseWriter, r *http.Request) { + hdr := w.Header() + hdr.Set("Key", "correct") + w.WriteHeader(200) + hdr.Set("Key", "incorrect") + }, + check(hasHeader("Key", "correct")), + }, + { + "Trailer headers are correctly recorded", + func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Non-Trailer", "correct") + w.Header().Set("Trailer", "Trailer-A") + w.Header().Add("Trailer", "Trailer-B") + w.Header().Add("Trailer", "Trailer-C") + io.WriteString(w, "<html>") + w.Header().Set("Non-Trailer", "incorrect") + w.Header().Set("Trailer-A", "valuea") + w.Header().Set("Trailer-C", "valuec") + w.Header().Set("Trailer-NotDeclared", "should be omitted") + w.Header().Set("Trailer:Trailer-D", "with prefix") + }, + check( + hasStatus(200), + hasHeader("Content-Type", "text/html; charset=utf-8"), + hasHeader("Non-Trailer", "correct"), + hasNotHeaders("Trailer-A", "Trailer-B", "Trailer-C", "Trailer-NotDeclared"), + hasTrailer("Trailer-A", "valuea"), + hasTrailer("Trailer-C", "valuec"), + hasNotTrailers("Non-Trailer", "Trailer-B", "Trailer-NotDeclared"), + hasTrailer("Trailer-D", "with prefix"), + ), + }, + { + "Header set without any write", // Issue 15560 + func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Foo", "1") + + // Simulate somebody using + // new(ResponseRecorder) instead of + // using the constructor which sets + // this to 200 + w.(*ResponseRecorder).Code = 0 + }, + check( + hasOldHeader("X-Foo", "1"), + hasStatus(0), + hasHeader("X-Foo", "1"), + hasResultStatus(200), + ), + }, + { + "HeaderMap vs FinalHeaders", // more for Issue 15560 + func(w http.ResponseWriter, r *http.Request) { + h := w.Header() + h.Set("X-Foo", "1") + w.Write([]byte("hi")) + h.Set("X-Foo", "2") + h.Set("X-Bar", "2") + }, + check( + hasOldHeader("X-Foo", "2"), + hasOldHeader("X-Bar", "2"), + hasHeader("X-Foo", "1"), + hasNotHeaders("X-Bar"), + ), + }, + { + "setting Content-Length header", + func(w http.ResponseWriter, r *http.Request) { + body := "Some body" + contentLength := fmt.Sprintf("%d", len(body)) + w.Header().Set("Content-Length", contentLength) + io.WriteString(w, body) + }, + check(hasStatus(200), hasContents("Some body"), hasContentLength(9)), + }, } r, _ := http.NewRequest("GET", "http://foo.com/", nil) for _, tt := range tests { diff --git a/libgo/go/net/http/httptest/server.go b/libgo/go/net/http/httptest/server.go index bbe323396f..5e9ace591f 100644 --- a/libgo/go/net/http/httptest/server.go +++ b/libgo/go/net/http/httptest/server.go @@ -16,7 +16,6 @@ import ( "net/http" "net/http/internal" "os" - "runtime" "sync" "time" ) @@ -114,9 +113,10 @@ func (s *Server) StartTLS() { } existingConfig := s.TLS - s.TLS = new(tls.Config) if existingConfig != nil { - *s.TLS = *existingConfig + s.TLS = existingConfig.Clone() + } else { + s.TLS = new(tls.Config) } if s.TLS.NextProtos == nil { s.TLS.NextProtos = []string{"http/1.1"} @@ -158,7 +158,7 @@ func (s *Server) Close() { // previously-flaky tests) in the case of // socket-late-binding races from the http Client // dialing this server and then getting an idle - // connection before the dial completed. There is thus + // connection before the dial completed. There is thus // a connected connection in StateNew with no // associated Request. We only close StateIdle and // StateNew because they're not doing anything. It's @@ -167,7 +167,7 @@ func (s *Server) Close() { // few milliseconds wasn't liked (early versions of // https://golang.org/cl/15151) so now we just // forcefully close StateNew. The docs for Server.Close say - // we wait for "oustanding requests", so we don't close things + // we wait for "outstanding requests", so we don't close things // in StateActive. if st == http.StateIdle || st == http.StateNew { s.closeConn(c) @@ -202,12 +202,10 @@ func (s *Server) logCloseHangDebugInfo() { // CloseClientConnections closes any open HTTP connections to the test Server. func (s *Server) CloseClientConnections() { - var conns int - ch := make(chan bool) - s.mu.Lock() + nconn := len(s.conns) + ch := make(chan struct{}, nconn) for c := range s.conns { - conns++ s.closeConnChan(c, ch) } s.mu.Unlock() @@ -220,7 +218,7 @@ func (s *Server) CloseClientConnections() { // in tests. timer := time.NewTimer(5 * time.Second) defer timer.Stop() - for i := 0; i < conns; i++ { + for i := 0; i < nconn; i++ { select { case <-ch: case <-timer.C: @@ -294,30 +292,11 @@ func (s *Server) closeConn(c net.Conn) { s.closeConnChan(c, nil) } // closeConnChan is like closeConn, but takes an optional channel to receive a value // when the goroutine closing c is done. -func (s *Server) closeConnChan(c net.Conn, done chan<- bool) { - if runtime.GOOS == "plan9" { - // Go's Plan 9 net package isn't great at unblocking reads when - // their underlying TCP connections are closed. Don't trust - // that that the ConnState state machine will get to - // StateClosed. Instead, just go there directly. Plan 9 may leak - // resources if the syscall doesn't end up returning. Oh well. - s.forgetConn(c) +func (s *Server) closeConnChan(c net.Conn, done chan<- struct{}) { + c.Close() + if done != nil { + done <- struct{}{} } - - // Somewhere in the chaos of https://golang.org/cl/15151 we found that - // some types of conns were blocking in Close too long (or deadlocking?) - // and we had to call Close in a goroutine. I (bradfitz) forget what - // that was at this point, but I suspect it was *tls.Conns, which - // were later fixed in https://golang.org/cl/18572, so this goroutine - // is _probably_ unnecessary now. But it's too late in Go 1.6 too remove - // it with confidence. - // TODO(bradfitz): try to remove it for Go 1.7. (golang.org/issue/14291) - go func() { - c.Close() - if done != nil { - done <- true - } - }() } // forgetConn removes c from the set of tracked conns and decrements it from the diff --git a/libgo/go/net/http/httptest/server_test.go b/libgo/go/net/http/httptest/server_test.go index c9606f2419..d032c5983b 100644 --- a/libgo/go/net/http/httptest/server_test.go +++ b/libgo/go/net/http/httptest/server_test.go @@ -53,7 +53,7 @@ func TestGetAfterClose(t *testing.T) { res, err = http.Get(ts.URL) if err == nil { body, _ := ioutil.ReadAll(res.Body) - t.Fatalf("Unexected response after close: %v, %v, %s", res.Status, res.Header, body) + t.Fatalf("Unexpected response after close: %v, %v, %s", res.Status, res.Header, body) } } @@ -95,6 +95,6 @@ func TestServerCloseClientConnections(t *testing.T) { res, err := http.Get(s.URL) if err == nil { res.Body.Close() - t.Fatal("Unexpected response: %#v", res) + t.Fatalf("Unexpected response: %#v", res) } } diff --git a/libgo/go/net/http/httptrace/example_test.go b/libgo/go/net/http/httptrace/example_test.go new file mode 100644 index 0000000000..27cdcdec31 --- /dev/null +++ b/libgo/go/net/http/httptrace/example_test.go @@ -0,0 +1,31 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package httptrace_test + +import ( + "fmt" + "log" + "net/http" + "net/http/httptrace" +) + +func Example() { + req, _ := http.NewRequest("GET", "http://example.com", nil) + trace := &httptrace.ClientTrace{ + GotConn: func(connInfo httptrace.GotConnInfo) { + fmt.Printf("Got Conn: %+v\n", connInfo) + }, + DNSDone: func(dnsInfo httptrace.DNSDoneInfo) { + fmt.Printf("DNS Info: %+v\n", dnsInfo) + }, + } + req = req.WithContext(httptrace.WithClientTrace(req.Context(), trace)) + _, err := http.DefaultTransport.RoundTrip(req) + if err != nil { + log.Fatal(err) + } +} diff --git a/libgo/go/net/http/httptrace/trace.go b/libgo/go/net/http/httptrace/trace.go new file mode 100644 index 0000000000..ea7b38c8fc --- /dev/null +++ b/libgo/go/net/http/httptrace/trace.go @@ -0,0 +1,243 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package httptrace provides mechanisms to trace the events within +// HTTP client requests. +package httptrace + +import ( + "context" + "crypto/tls" + "internal/nettrace" + "net" + "reflect" + "time" +) + +// unique type to prevent assignment. +type clientEventContextKey struct{} + +// ContextClientTrace returns the ClientTrace associated with the +// provided context. If none, it returns nil. +func ContextClientTrace(ctx context.Context) *ClientTrace { + trace, _ := ctx.Value(clientEventContextKey{}).(*ClientTrace) + return trace +} + +// WithClientTrace returns a new context based on the provided parent +// ctx. HTTP client requests made with the returned context will use +// the provided trace hooks, in addition to any previous hooks +// registered with ctx. Any hooks defined in the provided trace will +// be called first. +func WithClientTrace(ctx context.Context, trace *ClientTrace) context.Context { + if trace == nil { + panic("nil trace") + } + old := ContextClientTrace(ctx) + trace.compose(old) + + ctx = context.WithValue(ctx, clientEventContextKey{}, trace) + if trace.hasNetHooks() { + nt := &nettrace.Trace{ + ConnectStart: trace.ConnectStart, + ConnectDone: trace.ConnectDone, + } + if trace.DNSStart != nil { + nt.DNSStart = func(name string) { + trace.DNSStart(DNSStartInfo{Host: name}) + } + } + if trace.DNSDone != nil { + nt.DNSDone = func(netIPs []interface{}, coalesced bool, err error) { + addrs := make([]net.IPAddr, len(netIPs)) + for i, ip := range netIPs { + addrs[i] = ip.(net.IPAddr) + } + trace.DNSDone(DNSDoneInfo{ + Addrs: addrs, + Coalesced: coalesced, + Err: err, + }) + } + } + ctx = context.WithValue(ctx, nettrace.TraceKey{}, nt) + } + return ctx +} + +// ClientTrace is a set of hooks to run at various stages of an outgoing +// HTTP request. Any particular hook may be nil. Functions may be +// called concurrently from different goroutines and some may be called +// after the request has completed or failed. +// +// ClientTrace currently traces a single HTTP request & response +// during a single round trip and has no hooks that span a series +// of redirected requests. +// +// See https://blog.golang.org/http-tracing for more. +type ClientTrace struct { + // GetConn is called before a connection is created or + // retrieved from an idle pool. The hostPort is the + // "host:port" of the target or proxy. GetConn is called even + // if there's already an idle cached connection available. + GetConn func(hostPort string) + + // GotConn is called after a successful connection is + // obtained. There is no hook for failure to obtain a + // connection; instead, use the error from + // Transport.RoundTrip. + GotConn func(GotConnInfo) + + // PutIdleConn is called when the connection is returned to + // the idle pool. If err is nil, the connection was + // successfully returned to the idle pool. If err is non-nil, + // it describes why not. PutIdleConn is not called if + // connection reuse is disabled via Transport.DisableKeepAlives. + // PutIdleConn is called before the caller's Response.Body.Close + // call returns. + // For HTTP/2, this hook is not currently used. + PutIdleConn func(err error) + + // GotFirstResponseByte is called when the first byte of the response + // headers is available. + GotFirstResponseByte func() + + // Got100Continue is called if the server replies with a "100 + // Continue" response. + Got100Continue func() + + // DNSStart is called when a DNS lookup begins. + DNSStart func(DNSStartInfo) + + // DNSDone is called when a DNS lookup ends. + DNSDone func(DNSDoneInfo) + + // ConnectStart is called when a new connection's Dial begins. + // If net.Dialer.DualStack (IPv6 "Happy Eyeballs") support is + // enabled, this may be called multiple times. + ConnectStart func(network, addr string) + + // ConnectDone is called when a new connection's Dial + // completes. The provided err indicates whether the + // connection completedly successfully. + // If net.Dialer.DualStack ("Happy Eyeballs") support is + // enabled, this may be called multiple times. + ConnectDone func(network, addr string, err error) + + // TLSHandshakeStart is called when the TLS handshake is started. When + // connecting to a HTTPS site via a HTTP proxy, the handshake happens after + // the CONNECT request is processed by the proxy. + TLSHandshakeStart func() + + // TLSHandshakeDone is called after the TLS handshake with either the + // successful handshake's connection state, or a non-nil error on handshake + // failure. + TLSHandshakeDone func(tls.ConnectionState, error) + + // WroteHeaders is called after the Transport has written + // the request headers. + WroteHeaders func() + + // Wait100Continue is called if the Request specified + // "Expected: 100-continue" and the Transport has written the + // request headers but is waiting for "100 Continue" from the + // server before writing the request body. + Wait100Continue func() + + // WroteRequest is called with the result of writing the + // request and any body. It may be called multiple times + // in the case of retried requests. + WroteRequest func(WroteRequestInfo) +} + +// WroteRequestInfo contains information provided to the WroteRequest +// hook. +type WroteRequestInfo struct { + // Err is any error encountered while writing the Request. + Err error +} + +// compose modifies t such that it respects the previously-registered hooks in old, +// subject to the composition policy requested in t.Compose. +func (t *ClientTrace) compose(old *ClientTrace) { + if old == nil { + return + } + tv := reflect.ValueOf(t).Elem() + ov := reflect.ValueOf(old).Elem() + structType := tv.Type() + for i := 0; i < structType.NumField(); i++ { + tf := tv.Field(i) + hookType := tf.Type() + if hookType.Kind() != reflect.Func { + continue + } + of := ov.Field(i) + if of.IsNil() { + continue + } + if tf.IsNil() { + tf.Set(of) + continue + } + + // Make a copy of tf for tf to call. (Otherwise it + // creates a recursive call cycle and stack overflows) + tfCopy := reflect.ValueOf(tf.Interface()) + + // We need to call both tf and of in some order. + newFunc := reflect.MakeFunc(hookType, func(args []reflect.Value) []reflect.Value { + tfCopy.Call(args) + return of.Call(args) + }) + tv.Field(i).Set(newFunc) + } +} + +// DNSStartInfo contains information about a DNS request. +type DNSStartInfo struct { + Host string +} + +// DNSDoneInfo contains information about the results of a DNS lookup. +type DNSDoneInfo struct { + // Addrs are the IPv4 and/or IPv6 addresses found in the DNS + // lookup. The contents of the slice should not be mutated. + Addrs []net.IPAddr + + // Err is any error that occurred during the DNS lookup. + Err error + + // Coalesced is whether the Addrs were shared with another + // caller who was doing the same DNS lookup concurrently. + Coalesced bool +} + +func (t *ClientTrace) hasNetHooks() bool { + if t == nil { + return false + } + return t.DNSStart != nil || t.DNSDone != nil || t.ConnectStart != nil || t.ConnectDone != nil +} + +// GotConnInfo is the argument to the ClientTrace.GotConn function and +// contains information about the obtained connection. +type GotConnInfo struct { + // Conn is the connection that was obtained. It is owned by + // the http.Transport and should not be read, written or + // closed by users of ClientTrace. + Conn net.Conn + + // Reused is whether this connection has been previously + // used for another HTTP request. + Reused bool + + // WasIdle is whether this connection was obtained from an + // idle pool. + WasIdle bool + + // IdleTime reports how long the connection was previously + // idle, if WasIdle is true. + IdleTime time.Duration +} diff --git a/libgo/go/net/http/httptrace/trace_test.go b/libgo/go/net/http/httptrace/trace_test.go new file mode 100644 index 0000000000..bb57ada853 --- /dev/null +++ b/libgo/go/net/http/httptrace/trace_test.go @@ -0,0 +1,89 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package httptrace + +import ( + "bytes" + "context" + "testing" +) + +func TestWithClientTrace(t *testing.T) { + var buf bytes.Buffer + connectStart := func(b byte) func(network, addr string) { + return func(network, addr string) { + buf.WriteByte(b) + } + } + + ctx := context.Background() + oldtrace := &ClientTrace{ + ConnectStart: connectStart('O'), + } + ctx = WithClientTrace(ctx, oldtrace) + newtrace := &ClientTrace{ + ConnectStart: connectStart('N'), + } + ctx = WithClientTrace(ctx, newtrace) + trace := ContextClientTrace(ctx) + + buf.Reset() + trace.ConnectStart("net", "addr") + if got, want := buf.String(), "NO"; got != want { + t.Errorf("got %q; want %q", got, want) + } +} + +func TestCompose(t *testing.T) { + var buf bytes.Buffer + var testNum int + + connectStart := func(b byte) func(network, addr string) { + return func(network, addr string) { + if addr != "addr" { + t.Errorf(`%d. args for %q case = %q, %q; want addr of "addr"`, testNum, b, network, addr) + } + buf.WriteByte(b) + } + } + + tests := [...]struct { + trace, old *ClientTrace + want string + }{ + 0: { + want: "T", + trace: &ClientTrace{ + ConnectStart: connectStart('T'), + }, + }, + 1: { + want: "TO", + trace: &ClientTrace{ + ConnectStart: connectStart('T'), + }, + old: &ClientTrace{ConnectStart: connectStart('O')}, + }, + 2: { + want: "O", + trace: &ClientTrace{}, + old: &ClientTrace{ConnectStart: connectStart('O')}, + }, + } + for i, tt := range tests { + testNum = i + buf.Reset() + + tr := *tt.trace + tr.compose(tt.old) + if tr.ConnectStart != nil { + tr.ConnectStart("net", "addr") + } + if got := buf.String(); got != tt.want { + t.Errorf("%d. got = %q; want %q", i, got, tt.want) + } + } + +} diff --git a/libgo/go/net/http/httputil/dump.go b/libgo/go/net/http/httputil/dump.go index e22cc66dbf..7104c37454 100644 --- a/libgo/go/net/http/httputil/dump.go +++ b/libgo/go/net/http/httputil/dump.go @@ -18,11 +18,16 @@ import ( "time" ) -// One of the copies, say from b to r2, could be avoided by using a more -// elaborate trick where the other copy is made during Request/Response.Write. -// This would complicate things too much, given that these functions are for -// debugging only. +// drainBody reads all of b to memory and then returns two equivalent +// ReadClosers yielding the same bytes. +// +// It returns an error if the initial slurp of all bytes fails. It does not attempt +// to make the returned ReadClosers have identical error-matching behavior. func drainBody(b io.ReadCloser) (r1, r2 io.ReadCloser, err error) { + if b == http.NoBody { + // No copying needed. Preserve the magic sentinel meaning of NoBody. + return http.NoBody, http.NoBody, nil + } var buf bytes.Buffer if _, err = buf.ReadFrom(b); err != nil { return nil, b, err @@ -128,7 +133,7 @@ func DumpRequestOut(req *http.Request, body bool) ([]byte, error) { // If we used a dummy body above, remove it now. // TODO: if the req.ContentLength is large, we allocate memory - // unnecessarily just to slice it off here. But this is just + // unnecessarily just to slice it off here. But this is just // a debug function, so this is acceptable for now. We could // discard the body earlier if this matters. if dummyBody { @@ -163,18 +168,10 @@ func valueOrDefault(value, def string) string { var reqWriteExcludeHeaderDump = map[string]bool{ "Host": true, // not in Header map anyway - "Content-Length": true, "Transfer-Encoding": true, "Trailer": true, } -// dumpAsReceived writes req to w in the form as it was received, or -// at least as accurately as possible from the information retained in -// the request. -func dumpAsReceived(req *http.Request, w io.Writer) error { - return nil -} - // DumpRequest returns the given request in its HTTP/1.x wire // representation. It should only be used by servers to debug client // requests. The returned representation is an approximation only; @@ -191,7 +188,8 @@ func dumpAsReceived(req *http.Request, w io.Writer) error { // // The documentation for http.Request.Write details which fields // of req are included in the dump. -func DumpRequest(req *http.Request, body bool) (dump []byte, err error) { +func DumpRequest(req *http.Request, body bool) ([]byte, error) { + var err error save := req.Body if !body || req.Body == nil { req.Body = nil @@ -239,7 +237,7 @@ func DumpRequest(req *http.Request, body bool) (dump []byte, err error) { err = req.Header.WriteSubset(&b, reqWriteExcludeHeaderDump) if err != nil { - return + return nil, err } io.WriteString(&b, "\r\n") @@ -258,35 +256,42 @@ func DumpRequest(req *http.Request, body bool) (dump []byte, err error) { req.Body = save if err != nil { - return + return nil, err } - dump = b.Bytes() - return + return b.Bytes(), nil } -// errNoBody is a sentinel error value used by failureToReadBody so we can detect -// that the lack of body was intentional. +// errNoBody is a sentinel error value used by failureToReadBody so we +// can detect that the lack of body was intentional. var errNoBody = errors.New("sentinel error value") // failureToReadBody is a io.ReadCloser that just returns errNoBody on -// Read. It's swapped in when we don't actually want to consume the -// body, but need a non-nil one, and want to distinguish the error -// from reading the dummy body. +// Read. It's swapped in when we don't actually want to consume +// the body, but need a non-nil one, and want to distinguish the +// error from reading the dummy body. type failureToReadBody struct{} func (failureToReadBody) Read([]byte) (int, error) { return 0, errNoBody } func (failureToReadBody) Close() error { return nil } +// emptyBody is an instance of empty reader. var emptyBody = ioutil.NopCloser(strings.NewReader("")) // DumpResponse is like DumpRequest but dumps a response. -func DumpResponse(resp *http.Response, body bool) (dump []byte, err error) { +func DumpResponse(resp *http.Response, body bool) ([]byte, error) { var b bytes.Buffer + var err error save := resp.Body savecl := resp.ContentLength if !body { - resp.Body = failureToReadBody{} + // For content length of zero. Make sure the body is an empty + // reader, instead of returning error through failureToReadBody{}. + if resp.ContentLength == 0 { + resp.Body = emptyBody + } else { + resp.Body = failureToReadBody{} + } } else if resp.Body == nil { resp.Body = emptyBody } else { diff --git a/libgo/go/net/http/httputil/dump_test.go b/libgo/go/net/http/httputil/dump_test.go index 46bf521723..f881020fef 100644 --- a/libgo/go/net/http/httputil/dump_test.go +++ b/libgo/go/net/http/httputil/dump_test.go @@ -122,6 +122,10 @@ var dumpTests = []dumpTest{ Host: "post.tld", Path: "/", }, + Header: http.Header{ + "Content-Length": []string{"8193"}, + }, + ContentLength: 8193, ProtoMajor: 1, ProtoMinor: 1, @@ -135,6 +139,10 @@ var dumpTests = []dumpTest{ "Content-Length: 8193\r\n" + "Accept-Encoding: gzip\r\n\r\n" + strings.Repeat("a", 8193), + WantDump: "POST / HTTP/1.1\r\n" + + "Host: post.tld\r\n" + + "Content-Length: 8193\r\n\r\n" + + strings.Repeat("a", 8193), }, { @@ -144,6 +152,50 @@ var dumpTests = []dumpTest{ WantDump: "GET http://foo.com/ HTTP/1.1\r\n" + "User-Agent: blah\r\n\r\n", }, + + // Issue #7215. DumpRequest should return the "Content-Length" when set + { + Req: *mustReadRequest("POST /v2/api/?login HTTP/1.1\r\n" + + "Host: passport.myhost.com\r\n" + + "Content-Length: 3\r\n" + + "\r\nkey1=name1&key2=name2"), + WantDump: "POST /v2/api/?login HTTP/1.1\r\n" + + "Host: passport.myhost.com\r\n" + + "Content-Length: 3\r\n" + + "\r\nkey", + }, + + // Issue #7215. DumpRequest should return the "Content-Length" in ReadRequest + { + Req: *mustReadRequest("POST /v2/api/?login HTTP/1.1\r\n" + + "Host: passport.myhost.com\r\n" + + "Content-Length: 0\r\n" + + "\r\nkey1=name1&key2=name2"), + WantDump: "POST /v2/api/?login HTTP/1.1\r\n" + + "Host: passport.myhost.com\r\n" + + "Content-Length: 0\r\n\r\n", + }, + + // Issue #7215. DumpRequest should not return the "Content-Length" if unset + { + Req: *mustReadRequest("POST /v2/api/?login HTTP/1.1\r\n" + + "Host: passport.myhost.com\r\n" + + "\r\nkey1=name1&key2=name2"), + WantDump: "POST /v2/api/?login HTTP/1.1\r\n" + + "Host: passport.myhost.com\r\n\r\n", + }, + + // Issue 18506: make drainBody recognize NoBody. Otherwise + // this was turning into a chunked request. + { + Req: *mustNewRequest("POST", "http://example.com/foo", http.NoBody), + + WantDumpOut: "POST /foo HTTP/1.1\r\n" + + "Host: example.com\r\n" + + "User-Agent: Go-http-client/1.1\r\n" + + "Content-Length: 0\r\n" + + "Accept-Encoding: gzip\r\n\r\n", + }, } func TestDumpRequest(t *testing.T) { @@ -288,6 +340,27 @@ Transfer-Encoding: chunked foo 0`, }, + { + res: &http.Response{ + Status: "200 OK", + StatusCode: 200, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + ContentLength: 0, + Header: http.Header{ + // To verify if headers are not filtered out. + "Foo1": []string{"Bar1"}, + "Foo2": []string{"Bar2"}, + }, + Body: nil, + }, + body: false, // to verify we see 0, not empty. + want: `HTTP/1.1 200 OK +Foo1: Bar1 +Foo2: Bar2 +Content-Length: 0`, + }, } func TestDumpResponse(t *testing.T) { diff --git a/libgo/go/net/http/httputil/example_test.go b/libgo/go/net/http/httputil/example_test.go index 8fb1a2d279..e8dc962d3e 100644 --- a/libgo/go/net/http/httputil/example_test.go +++ b/libgo/go/net/http/httputil/example_test.go @@ -49,7 +49,7 @@ func ExampleDumpRequest() { fmt.Printf("%s", b) // Output: - // "POST / HTTP/1.1\r\nHost: www.example.org\r\nAccept-Encoding: gzip\r\nUser-Agent: Go-http-client/1.1\r\n\r\nGo is a general-purpose language designed with systems programming in mind." + // "POST / HTTP/1.1\r\nHost: www.example.org\r\nAccept-Encoding: gzip\r\nContent-Length: 75\r\nUser-Agent: Go-http-client/1.1\r\n\r\nGo is a general-purpose language designed with systems programming in mind." } func ExampleDumpRequestOut() { diff --git a/libgo/go/net/http/httputil/persist.go b/libgo/go/net/http/httputil/persist.go index 987bcc96ba..cbedf25ad1 100644 --- a/libgo/go/net/http/httputil/persist.go +++ b/libgo/go/net/http/httputil/persist.go @@ -15,26 +15,27 @@ import ( ) var ( + // Deprecated: No longer used. ErrPersistEOF = &http.ProtocolError{ErrorString: "persistent connection closed"} - ErrClosed = &http.ProtocolError{ErrorString: "connection closed by user"} - ErrPipeline = &http.ProtocolError{ErrorString: "pipeline error"} + + // Deprecated: No longer used. + ErrClosed = &http.ProtocolError{ErrorString: "connection closed by user"} + + // Deprecated: No longer used. + ErrPipeline = &http.ProtocolError{ErrorString: "pipeline error"} ) // This is an API usage error - the local side is closed. // ErrPersistEOF (above) reports that the remote side is closed. var errClosed = errors.New("i/o operation on closed connection") -// A ServerConn reads requests and sends responses over an underlying -// connection, until the HTTP keepalive logic commands an end. ServerConn -// also allows hijacking the underlying connection by calling Hijack -// to regain control over the connection. ServerConn supports pipe-lining, -// i.e. requests can be read out of sync (but in the same order) while the -// respective responses are sent. +// ServerConn is an artifact of Go's early HTTP implementation. +// It is low-level, old, and unused by Go's current HTTP stack. +// We should have deleted it before Go 1. // -// ServerConn is low-level and old. Applications should instead use Server -// in the net/http package. +// Deprecated: Use the Server in package net/http instead. type ServerConn struct { - lk sync.Mutex // read-write protects the following fields + mu sync.Mutex // read-write protects the following fields c net.Conn r *bufio.Reader re, we error // read/write errors @@ -45,11 +46,11 @@ type ServerConn struct { pipe textproto.Pipeline } -// NewServerConn returns a new ServerConn reading and writing c. If r is not -// nil, it is the buffer to use when reading c. +// NewServerConn is an artifact of Go's early HTTP implementation. +// It is low-level, old, and unused by Go's current HTTP stack. +// We should have deleted it before Go 1. // -// ServerConn is low-level and old. Applications should instead use Server -// in the net/http package. +// Deprecated: Use the Server in package net/http instead. func NewServerConn(c net.Conn, r *bufio.Reader) *ServerConn { if r == nil { r = bufio.NewReader(c) @@ -61,17 +62,17 @@ func NewServerConn(c net.Conn, r *bufio.Reader) *ServerConn { // as the read-side bufio which may have some left over data. Hijack may be // called before Read has signaled the end of the keep-alive logic. The user // should not call Hijack while Read or Write is in progress. -func (sc *ServerConn) Hijack() (c net.Conn, r *bufio.Reader) { - sc.lk.Lock() - defer sc.lk.Unlock() - c = sc.c - r = sc.r +func (sc *ServerConn) Hijack() (net.Conn, *bufio.Reader) { + sc.mu.Lock() + defer sc.mu.Unlock() + c := sc.c + r := sc.r sc.c = nil sc.r = nil - return + return c, r } -// Close calls Hijack and then also closes the underlying connection +// Close calls Hijack and then also closes the underlying connection. func (sc *ServerConn) Close() error { c, _ := sc.Hijack() if c != nil { @@ -84,7 +85,9 @@ func (sc *ServerConn) Close() error { // it is gracefully determined that there are no more requests (e.g. after the // first request on an HTTP/1.0 connection, or after a Connection:close on a // HTTP/1.1 connection). -func (sc *ServerConn) Read() (req *http.Request, err error) { +func (sc *ServerConn) Read() (*http.Request, error) { + var req *http.Request + var err error // Ensure ordered execution of Reads and Writes id := sc.pipe.Next() @@ -96,29 +99,29 @@ func (sc *ServerConn) Read() (req *http.Request, err error) { sc.pipe.EndResponse(id) } else { // Remember the pipeline id of this request - sc.lk.Lock() + sc.mu.Lock() sc.pipereq[req] = id - sc.lk.Unlock() + sc.mu.Unlock() } }() - sc.lk.Lock() + sc.mu.Lock() if sc.we != nil { // no point receiving if write-side broken or closed - defer sc.lk.Unlock() + defer sc.mu.Unlock() return nil, sc.we } if sc.re != nil { - defer sc.lk.Unlock() + defer sc.mu.Unlock() return nil, sc.re } if sc.r == nil { // connection closed by user in the meantime - defer sc.lk.Unlock() + defer sc.mu.Unlock() return nil, errClosed } r := sc.r lastbody := sc.lastbody sc.lastbody = nil - sc.lk.Unlock() + sc.mu.Unlock() // Make sure body is fully consumed, even if user does not call body.Close if lastbody != nil { @@ -127,16 +130,16 @@ func (sc *ServerConn) Read() (req *http.Request, err error) { // returned. err = lastbody.Close() if err != nil { - sc.lk.Lock() - defer sc.lk.Unlock() + sc.mu.Lock() + defer sc.mu.Unlock() sc.re = err return nil, err } } req, err = http.ReadRequest(r) - sc.lk.Lock() - defer sc.lk.Unlock() + sc.mu.Lock() + defer sc.mu.Unlock() if err != nil { if err == io.ErrUnexpectedEOF { // A close from the opposing client is treated as a @@ -161,8 +164,8 @@ func (sc *ServerConn) Read() (req *http.Request, err error) { // Pending returns the number of unanswered requests // that have been received on the connection. func (sc *ServerConn) Pending() int { - sc.lk.Lock() - defer sc.lk.Unlock() + sc.mu.Lock() + defer sc.mu.Unlock() return sc.nread - sc.nwritten } @@ -172,31 +175,31 @@ func (sc *ServerConn) Pending() int { func (sc *ServerConn) Write(req *http.Request, resp *http.Response) error { // Retrieve the pipeline ID of this request/response pair - sc.lk.Lock() + sc.mu.Lock() id, ok := sc.pipereq[req] delete(sc.pipereq, req) if !ok { - sc.lk.Unlock() + sc.mu.Unlock() return ErrPipeline } - sc.lk.Unlock() + sc.mu.Unlock() // Ensure pipeline order sc.pipe.StartResponse(id) defer sc.pipe.EndResponse(id) - sc.lk.Lock() + sc.mu.Lock() if sc.we != nil { - defer sc.lk.Unlock() + defer sc.mu.Unlock() return sc.we } if sc.c == nil { // connection closed by user in the meantime - defer sc.lk.Unlock() + defer sc.mu.Unlock() return ErrClosed } c := sc.c if sc.nread <= sc.nwritten { - defer sc.lk.Unlock() + defer sc.mu.Unlock() return errors.New("persist server pipe count") } if resp.Close { @@ -205,11 +208,11 @@ func (sc *ServerConn) Write(req *http.Request, resp *http.Response) error { // before signaling. sc.re = ErrPersistEOF } - sc.lk.Unlock() + sc.mu.Unlock() err := resp.Write(c) - sc.lk.Lock() - defer sc.lk.Unlock() + sc.mu.Lock() + defer sc.mu.Unlock() if err != nil { sc.we = err return err @@ -219,15 +222,13 @@ func (sc *ServerConn) Write(req *http.Request, resp *http.Response) error { return nil } -// A ClientConn sends request and receives headers over an underlying -// connection, while respecting the HTTP keepalive logic. ClientConn -// supports hijacking the connection calling Hijack to -// regain control of the underlying net.Conn and deal with it as desired. +// ClientConn is an artifact of Go's early HTTP implementation. +// It is low-level, old, and unused by Go's current HTTP stack. +// We should have deleted it before Go 1. // -// ClientConn is low-level and old. Applications should instead use -// Client or Transport in the net/http package. +// Deprecated: Use Client or Transport in package net/http instead. type ClientConn struct { - lk sync.Mutex // read-write protects the following fields + mu sync.Mutex // read-write protects the following fields c net.Conn r *bufio.Reader re, we error // read/write errors @@ -239,11 +240,11 @@ type ClientConn struct { writeReq func(*http.Request, io.Writer) error } -// NewClientConn returns a new ClientConn reading and writing c. If r is not -// nil, it is the buffer to use when reading c. +// NewClientConn is an artifact of Go's early HTTP implementation. +// It is low-level, old, and unused by Go's current HTTP stack. +// We should have deleted it before Go 1. // -// ClientConn is low-level and old. Applications should use Client or -// Transport in the net/http package. +// Deprecated: Use the Client or Transport in package net/http instead. func NewClientConn(c net.Conn, r *bufio.Reader) *ClientConn { if r == nil { r = bufio.NewReader(c) @@ -256,11 +257,11 @@ func NewClientConn(c net.Conn, r *bufio.Reader) *ClientConn { } } -// NewProxyClientConn works like NewClientConn but writes Requests -// using Request's WriteProxy method. +// NewProxyClientConn is an artifact of Go's early HTTP implementation. +// It is low-level, old, and unused by Go's current HTTP stack. +// We should have deleted it before Go 1. // -// New code should not use NewProxyClientConn. See Client or -// Transport in the net/http package instead. +// Deprecated: Use the Client or Transport in package net/http instead. func NewProxyClientConn(c net.Conn, r *bufio.Reader) *ClientConn { cc := NewClientConn(c, r) cc.writeReq = (*http.Request).WriteProxy @@ -272,8 +273,8 @@ func NewProxyClientConn(c net.Conn, r *bufio.Reader) *ClientConn { // called before the user or Read have signaled the end of the keep-alive // logic. The user should not call Hijack while Read or Write is in progress. func (cc *ClientConn) Hijack() (c net.Conn, r *bufio.Reader) { - cc.lk.Lock() - defer cc.lk.Unlock() + cc.mu.Lock() + defer cc.mu.Unlock() c = cc.c r = cc.r cc.c = nil @@ -281,7 +282,7 @@ func (cc *ClientConn) Hijack() (c net.Conn, r *bufio.Reader) { return } -// Close calls Hijack and then also closes the underlying connection +// Close calls Hijack and then also closes the underlying connection. func (cc *ClientConn) Close() error { c, _ := cc.Hijack() if c != nil { @@ -295,7 +296,8 @@ func (cc *ClientConn) Close() error { // keepalive connection is logically closed after this request and the opposing // server is informed. An ErrUnexpectedEOF indicates the remote closed the // underlying TCP connection, which is usually considered as graceful close. -func (cc *ClientConn) Write(req *http.Request) (err error) { +func (cc *ClientConn) Write(req *http.Request) error { + var err error // Ensure ordered execution of Writes id := cc.pipe.Next() @@ -307,23 +309,23 @@ func (cc *ClientConn) Write(req *http.Request) (err error) { cc.pipe.EndResponse(id) } else { // Remember the pipeline id of this request - cc.lk.Lock() + cc.mu.Lock() cc.pipereq[req] = id - cc.lk.Unlock() + cc.mu.Unlock() } }() - cc.lk.Lock() + cc.mu.Lock() if cc.re != nil { // no point sending if read-side closed or broken - defer cc.lk.Unlock() + defer cc.mu.Unlock() return cc.re } if cc.we != nil { - defer cc.lk.Unlock() + defer cc.mu.Unlock() return cc.we } if cc.c == nil { // connection closed by user in the meantime - defer cc.lk.Unlock() + defer cc.mu.Unlock() return errClosed } c := cc.c @@ -332,11 +334,11 @@ func (cc *ClientConn) Write(req *http.Request) (err error) { // still might be some pipelined reads cc.we = ErrPersistEOF } - cc.lk.Unlock() + cc.mu.Unlock() err = cc.writeReq(req, c) - cc.lk.Lock() - defer cc.lk.Unlock() + cc.mu.Lock() + defer cc.mu.Unlock() if err != nil { cc.we = err return err @@ -349,8 +351,8 @@ func (cc *ClientConn) Write(req *http.Request) (err error) { // Pending returns the number of unanswered requests // that have been sent on the connection. func (cc *ClientConn) Pending() int { - cc.lk.Lock() - defer cc.lk.Unlock() + cc.mu.Lock() + defer cc.mu.Unlock() return cc.nwritten - cc.nread } @@ -360,32 +362,32 @@ func (cc *ClientConn) Pending() int { // concurrently with Write, but not with another Read. func (cc *ClientConn) Read(req *http.Request) (resp *http.Response, err error) { // Retrieve the pipeline ID of this request/response pair - cc.lk.Lock() + cc.mu.Lock() id, ok := cc.pipereq[req] delete(cc.pipereq, req) if !ok { - cc.lk.Unlock() + cc.mu.Unlock() return nil, ErrPipeline } - cc.lk.Unlock() + cc.mu.Unlock() // Ensure pipeline order cc.pipe.StartResponse(id) defer cc.pipe.EndResponse(id) - cc.lk.Lock() + cc.mu.Lock() if cc.re != nil { - defer cc.lk.Unlock() + defer cc.mu.Unlock() return nil, cc.re } if cc.r == nil { // connection closed by user in the meantime - defer cc.lk.Unlock() + defer cc.mu.Unlock() return nil, errClosed } r := cc.r lastbody := cc.lastbody cc.lastbody = nil - cc.lk.Unlock() + cc.mu.Unlock() // Make sure body is fully consumed, even if user does not call body.Close if lastbody != nil { @@ -394,16 +396,16 @@ func (cc *ClientConn) Read(req *http.Request) (resp *http.Response, err error) { // returned. err = lastbody.Close() if err != nil { - cc.lk.Lock() - defer cc.lk.Unlock() + cc.mu.Lock() + defer cc.mu.Unlock() cc.re = err return nil, err } } resp, err = http.ReadResponse(r, req) - cc.lk.Lock() - defer cc.lk.Unlock() + cc.mu.Lock() + defer cc.mu.Unlock() if err != nil { cc.re = err return resp, err @@ -420,10 +422,10 @@ func (cc *ClientConn) Read(req *http.Request) (resp *http.Response, err error) { } // Do is convenience method that writes a request and reads a response. -func (cc *ClientConn) Do(req *http.Request) (resp *http.Response, err error) { - err = cc.Write(req) +func (cc *ClientConn) Do(req *http.Request) (*http.Response, error) { + err := cc.Write(req) if err != nil { - return + return nil, err } return cc.Read(req) } diff --git a/libgo/go/net/http/httputil/reverseproxy.go b/libgo/go/net/http/httputil/reverseproxy.go index 54411caeca..79c8fe2770 100644 --- a/libgo/go/net/http/httputil/reverseproxy.go +++ b/libgo/go/net/http/httputil/reverseproxy.go @@ -7,6 +7,7 @@ package httputil import ( + "context" "io" "log" "net" @@ -29,6 +30,8 @@ type ReverseProxy struct { // the request into a new request to be sent // using Transport. Its response is then copied // back to the original client unmodified. + // Director must not access the provided Request + // after returning. Director func(*http.Request) // The transport used to perform proxy requests. @@ -51,6 +54,11 @@ type ReverseProxy struct { // get byte slices for use by io.CopyBuffer when // copying HTTP response bodies. BufferPool BufferPool + + // ModifyResponse is an optional function that + // modifies the Response from the backend. + // If it returns an error, the proxy returns a StatusBadGateway error. + ModifyResponse func(*http.Response) error } // A BufferPool is an interface for getting and returning temporary @@ -90,6 +98,10 @@ func NewSingleHostReverseProxy(target *url.URL) *ReverseProxy { } else { req.URL.RawQuery = targetQuery + "&" + req.URL.RawQuery } + if _, ok := req.Header["User-Agent"]; !ok { + // explicitly disable User-Agent so it's not set to default value + req.Header.Set("User-Agent", "") + } } return &ReverseProxy{Director: director} } @@ -116,76 +128,59 @@ var hopHeaders = []string{ "Upgrade", } -type requestCanceler interface { - CancelRequest(*http.Request) -} - -type runOnFirstRead struct { - io.Reader // optional; nil means empty body - - fn func() // Run before first Read, then set to nil -} - -func (c *runOnFirstRead) Read(bs []byte) (int, error) { - if c.fn != nil { - c.fn() - c.fn = nil - } - if c.Reader == nil { - return 0, io.EOF - } - return c.Reader.Read(bs) -} - func (p *ReverseProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) { transport := p.Transport if transport == nil { transport = http.DefaultTransport } + ctx := req.Context() + if cn, ok := rw.(http.CloseNotifier); ok { + var cancel context.CancelFunc + ctx, cancel = context.WithCancel(ctx) + defer cancel() + notifyChan := cn.CloseNotify() + go func() { + select { + case <-notifyChan: + cancel() + case <-ctx.Done(): + } + }() + } + outreq := new(http.Request) *outreq = *req // includes shallow copies of maps, but okay - - if closeNotifier, ok := rw.(http.CloseNotifier); ok { - if requestCanceler, ok := transport.(requestCanceler); ok { - reqDone := make(chan struct{}) - defer close(reqDone) - - clientGone := closeNotifier.CloseNotify() - - outreq.Body = struct { - io.Reader - io.Closer - }{ - Reader: &runOnFirstRead{ - Reader: outreq.Body, - fn: func() { - go func() { - select { - case <-clientGone: - requestCanceler.CancelRequest(outreq) - case <-reqDone: - } - }() - }, - }, - Closer: outreq.Body, - } - } + if req.ContentLength == 0 { + outreq.Body = nil // Issue 16036: nil Body for http.Transport retries } + outreq = outreq.WithContext(ctx) p.Director(outreq) - outreq.Proto = "HTTP/1.1" - outreq.ProtoMajor = 1 - outreq.ProtoMinor = 1 outreq.Close = false - // Remove hop-by-hop headers to the backend. Especially - // important is "Connection" because we want a persistent - // connection, regardless of what the client sent to us. This - // is modifying the same underlying map from req (shallow + // We are modifying the same underlying map from req (shallow // copied above) so we only copy it if necessary. copiedHeaders := false + + // Remove hop-by-hop headers listed in the "Connection" header. + // See RFC 2616, section 14.10. + if c := outreq.Header.Get("Connection"); c != "" { + for _, f := range strings.Split(c, ",") { + if f = strings.TrimSpace(f); f != "" { + if !copiedHeaders { + outreq.Header = make(http.Header) + copyHeader(outreq.Header, req.Header) + copiedHeaders = true + } + outreq.Header.Del(f) + } + } + } + + // Remove hop-by-hop headers to the backend. Especially + // important is "Connection" because we want a persistent + // connection, regardless of what the client sent to us. for _, h := range hopHeaders { if outreq.Header.Get(h) != "" { if !copiedHeaders { @@ -210,20 +205,38 @@ func (p *ReverseProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) { res, err := transport.RoundTrip(outreq) if err != nil { p.logf("http: proxy error: %v", err) - rw.WriteHeader(http.StatusInternalServerError) + rw.WriteHeader(http.StatusBadGateway) return } + // Remove hop-by-hop headers listed in the + // "Connection" header of the response. + if c := res.Header.Get("Connection"); c != "" { + for _, f := range strings.Split(c, ",") { + if f = strings.TrimSpace(f); f != "" { + res.Header.Del(f) + } + } + } + for _, h := range hopHeaders { res.Header.Del(h) } + if p.ModifyResponse != nil { + if err := p.ModifyResponse(res); err != nil { + p.logf("http: proxy error: %v", err) + rw.WriteHeader(http.StatusBadGateway) + return + } + } + copyHeader(rw.Header(), res.Header) // The "Trailer" header isn't included in the Transport's response, // at least for *http.Transport. Build it up from Trailer. if len(res.Trailer) > 0 { - var trailerKeys []string + trailerKeys := make([]string, 0, len(res.Trailer)) for k := range res.Trailer { trailerKeys = append(trailerKeys, k) } @@ -262,12 +275,40 @@ func (p *ReverseProxy) copyResponse(dst io.Writer, src io.Reader) { if p.BufferPool != nil { buf = p.BufferPool.Get() } - io.CopyBuffer(dst, src, buf) + p.copyBuffer(dst, src, buf) if p.BufferPool != nil { p.BufferPool.Put(buf) } } +func (p *ReverseProxy) copyBuffer(dst io.Writer, src io.Reader, buf []byte) (int64, error) { + if len(buf) == 0 { + buf = make([]byte, 32*1024) + } + var written int64 + for { + nr, rerr := src.Read(buf) + if rerr != nil && rerr != io.EOF { + p.logf("httputil: ReverseProxy read error during body copy: %v", rerr) + } + if nr > 0 { + nw, werr := dst.Write(buf[:nr]) + if nw > 0 { + written += int64(nw) + } + if werr != nil { + return written, werr + } + if nr != nw { + return written, io.ErrShortWrite + } + } + if rerr != nil { + return written, rerr + } + } +} + func (p *ReverseProxy) logf(format string, args ...interface{}) { if p.ErrorLog != nil { p.ErrorLog.Printf(format, args...) @@ -285,13 +326,13 @@ type maxLatencyWriter struct { dst writeFlusher latency time.Duration - lk sync.Mutex // protects Write + Flush + mu sync.Mutex // protects Write + Flush done chan bool } func (m *maxLatencyWriter) Write(p []byte) (int, error) { - m.lk.Lock() - defer m.lk.Unlock() + m.mu.Lock() + defer m.mu.Unlock() return m.dst.Write(p) } @@ -306,9 +347,9 @@ func (m *maxLatencyWriter) flushLoop() { } return case <-t.C: - m.lk.Lock() + m.mu.Lock() m.dst.Flush() - m.lk.Unlock() + m.mu.Unlock() } } } diff --git a/libgo/go/net/http/httputil/reverseproxy_test.go b/libgo/go/net/http/httputil/reverseproxy_test.go index 0849427b85..20c4e16bcb 100644 --- a/libgo/go/net/http/httputil/reverseproxy_test.go +++ b/libgo/go/net/http/httputil/reverseproxy_test.go @@ -9,6 +9,8 @@ package httputil import ( "bufio" "bytes" + "errors" + "fmt" "io" "io/ioutil" "log" @@ -33,6 +35,11 @@ func TestReverseProxy(t *testing.T) { const backendResponse = "I am the backend" const backendStatus = 404 backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == "GET" && r.FormValue("mode") == "hangup" { + c, _, _ := w.(http.Hijacker).Hijack() + c.Close() + return + } if len(r.TransferEncoding) > 0 { t.Errorf("backend got unexpected TransferEncoding: %v", r.TransferEncoding) } @@ -69,6 +76,7 @@ func TestReverseProxy(t *testing.T) { t.Fatal(err) } proxyHandler := NewSingleHostReverseProxy(backendURL) + proxyHandler.ErrorLog = log.New(ioutil.Discard, "", 0) // quiet for tests frontend := httptest.NewServer(proxyHandler) defer frontend.Close() @@ -113,6 +121,75 @@ func TestReverseProxy(t *testing.T) { if g, e := res.Trailer.Get("X-Trailer"), "trailer_value"; g != e { t.Errorf("Trailer(X-Trailer) = %q ; want %q", g, e) } + + // Test that a backend failing to be reached or one which doesn't return + // a response results in a StatusBadGateway. + getReq, _ = http.NewRequest("GET", frontend.URL+"/?mode=hangup", nil) + getReq.Close = true + res, err = http.DefaultClient.Do(getReq) + if err != nil { + t.Fatal(err) + } + res.Body.Close() + if res.StatusCode != http.StatusBadGateway { + t.Errorf("request to bad proxy = %v; want 502 StatusBadGateway", res.Status) + } + +} + +// Issue 16875: remove any proxied headers mentioned in the "Connection" +// header value. +func TestReverseProxyStripHeadersPresentInConnection(t *testing.T) { + const fakeConnectionToken = "X-Fake-Connection-Token" + const backendResponse = "I am the backend" + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if c := r.Header.Get(fakeConnectionToken); c != "" { + t.Errorf("handler got header %q = %q; want empty", fakeConnectionToken, c) + } + if c := r.Header.Get("Upgrade"); c != "" { + t.Errorf("handler got header %q = %q; want empty", "Upgrade", c) + } + w.Header().Set("Connection", "Upgrade, "+fakeConnectionToken) + w.Header().Set("Upgrade", "should be deleted") + w.Header().Set(fakeConnectionToken, "should be deleted") + io.WriteString(w, backendResponse) + })) + defer backend.Close() + backendURL, err := url.Parse(backend.URL) + if err != nil { + t.Fatal(err) + } + proxyHandler := NewSingleHostReverseProxy(backendURL) + frontend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + proxyHandler.ServeHTTP(w, r) + if c := r.Header.Get("Upgrade"); c != "original value" { + t.Errorf("handler modified header %q = %q; want %q", "Upgrade", c, "original value") + } + })) + defer frontend.Close() + + getReq, _ := http.NewRequest("GET", frontend.URL, nil) + getReq.Header.Set("Connection", "Upgrade, "+fakeConnectionToken) + getReq.Header.Set("Upgrade", "original value") + getReq.Header.Set(fakeConnectionToken, "should be deleted") + res, err := http.DefaultClient.Do(getReq) + if err != nil { + t.Fatalf("Get: %v", err) + } + defer res.Body.Close() + bodyBytes, err := ioutil.ReadAll(res.Body) + if err != nil { + t.Fatalf("reading body: %v", err) + } + if got, want := string(bodyBytes), backendResponse; got != want { + t.Errorf("got body %q; want %q", got, want) + } + if c := res.Header.Get("Upgrade"); c != "" { + t.Errorf("handler got header %q = %q; want empty", "Upgrade", c) + } + if c := res.Header.Get(fakeConnectionToken); c != "" { + t.Errorf("handler got header %q = %q; want empty", fakeConnectionToken, c) + } } func TestXForwardedFor(t *testing.T) { @@ -240,14 +317,14 @@ func TestReverseProxyCancelation(t *testing.T) { reqInFlight := make(chan struct{}) backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - close(reqInFlight) + close(reqInFlight) // cause the client to cancel its request select { case <-time.After(10 * time.Second): // Note: this should only happen in broken implementations, and the // closenotify case should be instantaneous. - t.Log("Failed to close backend connection") - t.Fail() + t.Error("Handler never saw CloseNotify") + return case <-w.(http.CloseNotifier).CloseNotify(): } @@ -280,13 +357,13 @@ func TestReverseProxyCancelation(t *testing.T) { }() res, err := http.DefaultClient.Do(getReq) if res != nil { - t.Fatal("Non-nil response") + t.Errorf("got response %v; want nil", res.Status) } if err == nil { // This should be an error like: // Get http://127.0.0.1:58079: read tcp 127.0.0.1:58079: // use of closed network connection - t.Fatal("DefaultClient.Do() returned nil error") + t.Error("DefaultClient.Do() returned nil error; want non-nil error") } } @@ -328,6 +405,49 @@ func TestNilBody(t *testing.T) { } } +// Issue 15524 +func TestUserAgentHeader(t *testing.T) { + const explicitUA = "explicit UA" + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/noua" { + if c := r.Header.Get("User-Agent"); c != "" { + t.Errorf("handler got non-empty User-Agent header %q", c) + } + return + } + if c := r.Header.Get("User-Agent"); c != explicitUA { + t.Errorf("handler got unexpected User-Agent header %q", c) + } + })) + defer backend.Close() + backendURL, err := url.Parse(backend.URL) + if err != nil { + t.Fatal(err) + } + proxyHandler := NewSingleHostReverseProxy(backendURL) + proxyHandler.ErrorLog = log.New(ioutil.Discard, "", 0) // quiet for tests + frontend := httptest.NewServer(proxyHandler) + defer frontend.Close() + + getReq, _ := http.NewRequest("GET", frontend.URL, nil) + getReq.Header.Set("User-Agent", explicitUA) + getReq.Close = true + res, err := http.DefaultClient.Do(getReq) + if err != nil { + t.Fatalf("Get: %v", err) + } + res.Body.Close() + + getReq, _ = http.NewRequest("GET", frontend.URL+"/noua", nil) + getReq.Header.Set("User-Agent", "") + getReq.Close = true + res, err = http.DefaultClient.Do(getReq) + if err != nil { + t.Fatalf("Get: %v", err) + } + res.Body.Close() +} + type bufferPool struct { get func() []byte put func([]byte) @@ -432,3 +552,115 @@ func TestReverseProxy_Post(t *testing.T) { t.Errorf("got body %q; expected %q", g, e) } } + +type RoundTripperFunc func(*http.Request) (*http.Response, error) + +func (fn RoundTripperFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return fn(req) +} + +// Issue 16036: send a Request with a nil Body when possible +func TestReverseProxy_NilBody(t *testing.T) { + backendURL, _ := url.Parse("http://fake.tld/") + proxyHandler := NewSingleHostReverseProxy(backendURL) + proxyHandler.ErrorLog = log.New(ioutil.Discard, "", 0) // quiet for tests + proxyHandler.Transport = RoundTripperFunc(func(req *http.Request) (*http.Response, error) { + if req.Body != nil { + t.Error("Body != nil; want a nil Body") + } + return nil, errors.New("done testing the interesting part; so force a 502 Gateway error") + }) + frontend := httptest.NewServer(proxyHandler) + defer frontend.Close() + + res, err := http.DefaultClient.Get(frontend.URL) + if err != nil { + t.Fatal(err) + } + defer res.Body.Close() + if res.StatusCode != 502 { + t.Errorf("status code = %v; want 502 (Gateway Error)", res.Status) + } +} + +// Issue 14237. Test ModifyResponse and that an error from it +// causes the proxy to return StatusBadGateway, or StatusOK otherwise. +func TestReverseProxyModifyResponse(t *testing.T) { + backendServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Add("X-Hit-Mod", fmt.Sprintf("%v", r.URL.Path == "/mod")) + })) + defer backendServer.Close() + + rpURL, _ := url.Parse(backendServer.URL) + rproxy := NewSingleHostReverseProxy(rpURL) + rproxy.ErrorLog = log.New(ioutil.Discard, "", 0) // quiet for tests + rproxy.ModifyResponse = func(resp *http.Response) error { + if resp.Header.Get("X-Hit-Mod") != "true" { + return fmt.Errorf("tried to by-pass proxy") + } + return nil + } + + frontendProxy := httptest.NewServer(rproxy) + defer frontendProxy.Close() + + tests := []struct { + url string + wantCode int + }{ + {frontendProxy.URL + "/mod", http.StatusOK}, + {frontendProxy.URL + "/schedule", http.StatusBadGateway}, + } + + for i, tt := range tests { + resp, err := http.Get(tt.url) + if err != nil { + t.Fatalf("failed to reach proxy: %v", err) + } + if g, e := resp.StatusCode, tt.wantCode; g != e { + t.Errorf("#%d: got res.StatusCode %d; expected %d", i, g, e) + } + resp.Body.Close() + } +} + +// Issue 16659: log errors from short read +func TestReverseProxy_CopyBuffer(t *testing.T) { + backendServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + out := "this call was relayed by the reverse proxy" + // Coerce a wrong content length to induce io.UnexpectedEOF + w.Header().Set("Content-Length", fmt.Sprintf("%d", len(out)*2)) + fmt.Fprintln(w, out) + })) + defer backendServer.Close() + + rpURL, err := url.Parse(backendServer.URL) + if err != nil { + t.Fatal(err) + } + + var proxyLog bytes.Buffer + rproxy := NewSingleHostReverseProxy(rpURL) + rproxy.ErrorLog = log.New(&proxyLog, "", log.Lshortfile) + frontendProxy := httptest.NewServer(rproxy) + defer frontendProxy.Close() + + resp, err := http.Get(frontendProxy.URL) + if err != nil { + t.Fatalf("failed to reach proxy: %v", err) + } + defer resp.Body.Close() + + if _, err := ioutil.ReadAll(resp.Body); err == nil { + t.Fatalf("want non-nil error") + } + expected := []string{ + "EOF", + "read", + } + for _, phrase := range expected { + if !bytes.Contains(proxyLog.Bytes(), []byte(phrase)) { + t.Errorf("expected log to contain phrase %q", phrase) + } + } +} diff --git a/libgo/go/net/http/internal/chunked.go b/libgo/go/net/http/internal/chunked.go index 2e62c00d5d..63f321d03b 100644 --- a/libgo/go/net/http/internal/chunked.go +++ b/libgo/go/net/http/internal/chunked.go @@ -35,10 +35,11 @@ func NewChunkedReader(r io.Reader) io.Reader { } type chunkedReader struct { - r *bufio.Reader - n uint64 // unread bytes in chunk - err error - buf [2]byte + r *bufio.Reader + n uint64 // unread bytes in chunk + err error + buf [2]byte + checkEnd bool // whether need to check for \r\n chunk footer } func (cr *chunkedReader) beginChunk() { @@ -68,6 +69,21 @@ func (cr *chunkedReader) chunkHeaderAvailable() bool { func (cr *chunkedReader) Read(b []uint8) (n int, err error) { for cr.err == nil { + if cr.checkEnd { + if n > 0 && cr.r.Buffered() < 2 { + // We have some data. Return early (per the io.Reader + // contract) instead of potentially blocking while + // reading more. + break + } + if _, cr.err = io.ReadFull(cr.r, cr.buf[:2]); cr.err == nil { + if string(cr.buf[:]) != "\r\n" { + cr.err = errors.New("malformed chunked encoding") + break + } + } + cr.checkEnd = false + } if cr.n == 0 { if n > 0 && !cr.chunkHeaderAvailable() { // We've read enough. Don't potentially block @@ -92,11 +108,7 @@ func (cr *chunkedReader) Read(b []uint8) (n int, err error) { // If we're at the end of a chunk, read the next two // bytes to verify they are "\r\n". if cr.n == 0 && cr.err == nil { - if _, cr.err = io.ReadFull(cr.r, cr.buf[:2]); cr.err == nil { - if cr.buf[0] != '\r' || cr.buf[1] != '\n' { - cr.err = errors.New("malformed chunked encoding") - } - } + cr.checkEnd = true } } return n, cr.err diff --git a/libgo/go/net/http/internal/chunked_test.go b/libgo/go/net/http/internal/chunked_test.go index a136dc99a6..d06716591a 100644 --- a/libgo/go/net/http/internal/chunked_test.go +++ b/libgo/go/net/http/internal/chunked_test.go @@ -122,7 +122,7 @@ func TestChunkReaderAllocs(t *testing.T) { byter := bytes.NewReader(buf.Bytes()) bufr := bufio.NewReader(byter) mallocs := testing.AllocsPerRun(100, func() { - byter.Seek(0, 0) + byter.Seek(0, io.SeekStart) bufr.Reset(byter) r := NewChunkedReader(bufr) n, err := io.ReadFull(r, readBuf) @@ -185,3 +185,30 @@ func TestChunkReadingIgnoresExtensions(t *testing.T) { t.Errorf("read %q; want %q", g, e) } } + +// Issue 17355: ChunkedReader shouldn't block waiting for more data +// if it can return something. +func TestChunkReadPartial(t *testing.T) { + pr, pw := io.Pipe() + go func() { + pw.Write([]byte("7\r\n1234567")) + }() + cr := NewChunkedReader(pr) + readBuf := make([]byte, 7) + n, err := cr.Read(readBuf) + if err != nil { + t.Fatal(err) + } + want := "1234567" + if n != 7 || string(readBuf) != want { + t.Fatalf("Read: %v %q; want %d, %q", n, readBuf[:n], len(want), want) + } + go func() { + pw.Write([]byte("xx")) + }() + _, err = cr.Read(readBuf) + if got := fmt.Sprint(err); !strings.Contains(got, "malformed") { + t.Fatalf("second read = %v; want malformed error", err) + } + +} diff --git a/libgo/go/net/http/lex.go b/libgo/go/net/http/lex.go deleted file mode 100644 index 52b6481c14..0000000000 --- a/libgo/go/net/http/lex.go +++ /dev/null @@ -1,183 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package http - -import ( - "strings" - "unicode/utf8" -) - -// This file deals with lexical matters of HTTP - -var isTokenTable = [127]bool{ - '!': true, - '#': true, - '$': true, - '%': true, - '&': true, - '\'': true, - '*': true, - '+': true, - '-': true, - '.': true, - '0': true, - '1': true, - '2': true, - '3': true, - '4': true, - '5': true, - '6': true, - '7': true, - '8': true, - '9': true, - 'A': true, - 'B': true, - 'C': true, - 'D': true, - 'E': true, - 'F': true, - 'G': true, - 'H': true, - 'I': true, - 'J': true, - 'K': true, - 'L': true, - 'M': true, - 'N': true, - 'O': true, - 'P': true, - 'Q': true, - 'R': true, - 'S': true, - 'T': true, - 'U': true, - 'W': true, - 'V': true, - 'X': true, - 'Y': true, - 'Z': true, - '^': true, - '_': true, - '`': true, - 'a': true, - 'b': true, - 'c': true, - 'd': true, - 'e': true, - 'f': true, - 'g': true, - 'h': true, - 'i': true, - 'j': true, - 'k': true, - 'l': true, - 'm': true, - 'n': true, - 'o': true, - 'p': true, - 'q': true, - 'r': true, - 's': true, - 't': true, - 'u': true, - 'v': true, - 'w': true, - 'x': true, - 'y': true, - 'z': true, - '|': true, - '~': true, -} - -func isToken(r rune) bool { - i := int(r) - return i < len(isTokenTable) && isTokenTable[i] -} - -func isNotToken(r rune) bool { - return !isToken(r) -} - -// headerValuesContainsToken reports whether any string in values -// contains the provided token, ASCII case-insensitively. -func headerValuesContainsToken(values []string, token string) bool { - for _, v := range values { - if headerValueContainsToken(v, token) { - return true - } - } - return false -} - -// isOWS reports whether b is an optional whitespace byte, as defined -// by RFC 7230 section 3.2.3. -func isOWS(b byte) bool { return b == ' ' || b == '\t' } - -// trimOWS returns x with all optional whitespace removes from the -// beginning and end. -func trimOWS(x string) string { - // TODO: consider using strings.Trim(x, " \t") instead, - // if and when it's fast enough. See issue 10292. - // But this ASCII-only code will probably always beat UTF-8 - // aware code. - for len(x) > 0 && isOWS(x[0]) { - x = x[1:] - } - for len(x) > 0 && isOWS(x[len(x)-1]) { - x = x[:len(x)-1] - } - return x -} - -// headerValueContainsToken reports whether v (assumed to be a -// 0#element, in the ABNF extension described in RFC 7230 section 7) -// contains token amongst its comma-separated tokens, ASCII -// case-insensitively. -func headerValueContainsToken(v string, token string) bool { - v = trimOWS(v) - if comma := strings.IndexByte(v, ','); comma != -1 { - return tokenEqual(trimOWS(v[:comma]), token) || headerValueContainsToken(v[comma+1:], token) - } - return tokenEqual(v, token) -} - -// lowerASCII returns the ASCII lowercase version of b. -func lowerASCII(b byte) byte { - if 'A' <= b && b <= 'Z' { - return b + ('a' - 'A') - } - return b -} - -// tokenEqual reports whether t1 and t2 are equal, ASCII case-insensitively. -func tokenEqual(t1, t2 string) bool { - if len(t1) != len(t2) { - return false - } - for i, b := range t1 { - if b >= utf8.RuneSelf { - // No UTF-8 or non-ASCII allowed in tokens. - return false - } - if lowerASCII(byte(b)) != lowerASCII(t2[i]) { - return false - } - } - return true -} - -// isLWS reports whether b is linear white space, according -// to http://www.w3.org/Protocols/rfc2616/rfc2616-sec2.html#sec2.2 -// LWS = [CRLF] 1*( SP | HT ) -func isLWS(b byte) bool { return b == ' ' || b == '\t' } - -// isCTL reports whether b is a control byte, according -// to http://www.w3.org/Protocols/rfc2616/rfc2616-sec2.html#sec2.2 -// CTL = <any US-ASCII control character -// (octets 0 - 31) and DEL (127)> -func isCTL(b byte) bool { - const del = 0x7f // a CTL - return b < ' ' || b == del -} diff --git a/libgo/go/net/http/lex_test.go b/libgo/go/net/http/lex_test.go deleted file mode 100644 index 986fda17dc..0000000000 --- a/libgo/go/net/http/lex_test.go +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package http - -import ( - "testing" -) - -func isChar(c rune) bool { return c <= 127 } - -func isCtl(c rune) bool { return c <= 31 || c == 127 } - -func isSeparator(c rune) bool { - switch c { - case '(', ')', '<', '>', '@', ',', ';', ':', '\\', '"', '/', '[', ']', '?', '=', '{', '}', ' ', '\t': - return true - } - return false -} - -func TestIsToken(t *testing.T) { - for i := 0; i <= 130; i++ { - r := rune(i) - expected := isChar(r) && !isCtl(r) && !isSeparator(r) - if isToken(r) != expected { - t.Errorf("isToken(0x%x) = %v", r, !expected) - } - } -} - -func TestHeaderValuesContainsToken(t *testing.T) { - tests := []struct { - vals []string - token string - want bool - }{ - { - vals: []string{"foo"}, - token: "foo", - want: true, - }, - { - vals: []string{"bar", "foo"}, - token: "foo", - want: true, - }, - { - vals: []string{"foo"}, - token: "FOO", - want: true, - }, - { - vals: []string{"foo"}, - token: "bar", - want: false, - }, - { - vals: []string{" foo "}, - token: "FOO", - want: true, - }, - { - vals: []string{"foo,bar"}, - token: "FOO", - want: true, - }, - { - vals: []string{"bar,foo,bar"}, - token: "FOO", - want: true, - }, - { - vals: []string{"bar , foo"}, - token: "FOO", - want: true, - }, - { - vals: []string{"foo ,bar "}, - token: "FOO", - want: true, - }, - { - vals: []string{"bar, foo ,bar"}, - token: "FOO", - want: true, - }, - { - vals: []string{"bar , foo"}, - token: "FOO", - want: true, - }, - } - for _, tt := range tests { - got := headerValuesContainsToken(tt.vals, tt.token) - if got != tt.want { - t.Errorf("headerValuesContainsToken(%q, %q) = %v; want %v", tt.vals, tt.token, got, tt.want) - } - } -} diff --git a/libgo/go/net/http/main_test.go b/libgo/go/net/http/main_test.go index 299cd7b2d2..438bd2e58f 100644 --- a/libgo/go/net/http/main_test.go +++ b/libgo/go/net/http/main_test.go @@ -5,8 +5,9 @@ package http_test import ( - "flag" "fmt" + "io/ioutil" + "log" "net/http" "os" "runtime" @@ -16,7 +17,7 @@ import ( "time" ) -var flaky = flag.Bool("flaky", false, "run known-flaky tests too") +var quietLog = log.New(ioutil.Discard, "", 0) func TestMain(m *testing.M) { v := m.Run() @@ -91,12 +92,6 @@ func setParallel(t *testing.T) { } } -func setFlaky(t *testing.T, issue int) { - if !*flaky { - t.Skipf("skipping known flaky test; see golang.org/issue/%d", issue) - } -} - func afterTest(t testing.TB) { http.DefaultTransport.(*http.Transport).CloseIdleConnections() if testing.Short() { @@ -129,3 +124,34 @@ func afterTest(t testing.TB) { } t.Errorf("Test appears to have leaked %s:\n%s", bad, stacks) } + +// waitCondition reports whether fn eventually returned true, +// checking immediately and then every checkEvery amount, +// until waitFor has elapsed, at which point it returns false. +func waitCondition(waitFor, checkEvery time.Duration, fn func() bool) bool { + deadline := time.Now().Add(waitFor) + for time.Now().Before(deadline) { + if fn() { + return true + } + time.Sleep(checkEvery) + } + return false +} + +// waitErrCondition is like waitCondition but with errors instead of bools. +func waitErrCondition(waitFor, checkEvery time.Duration, fn func() error) error { + deadline := time.Now().Add(waitFor) + var err error + for time.Now().Before(deadline) { + if err = fn(); err == nil { + return nil + } + time.Sleep(checkEvery) + } + return err +} + +func closeClient(c *http.Client) { + c.Transport.(*http.Transport).CloseIdleConnections() +} diff --git a/libgo/go/net/http/method.go b/libgo/go/net/http/method.go index b74f9604d3..6f46155069 100644 --- a/libgo/go/net/http/method.go +++ b/libgo/go/net/http/method.go @@ -12,7 +12,7 @@ const ( MethodHead = "HEAD" MethodPost = "POST" MethodPut = "PUT" - MethodPatch = "PATCH" // RFC 5741 + MethodPatch = "PATCH" // RFC 5789 MethodDelete = "DELETE" MethodConnect = "CONNECT" MethodOptions = "OPTIONS" diff --git a/libgo/go/net/http/npn_test.go b/libgo/go/net/http/npn_test.go index e2e911d3dd..4c1f6b573d 100644 --- a/libgo/go/net/http/npn_test.go +++ b/libgo/go/net/http/npn_test.go @@ -18,6 +18,7 @@ import ( ) func TestNextProtoUpgrade(t *testing.T) { + setParallel(t) defer afterTest(t) ts := httptest.NewUnstartedServer(HandlerFunc(func(w ResponseWriter, r *Request) { fmt.Fprintf(w, "path=%s,proto=", r.URL.Path) diff --git a/libgo/go/net/http/pprof/pprof.go b/libgo/go/net/http/pprof/pprof.go index 7262c6c101..05d0890fdf 100644 --- a/libgo/go/net/http/pprof/pprof.go +++ b/libgo/go/net/http/pprof/pprof.go @@ -1,11 +1,9 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // Package pprof serves via its HTTP server runtime profiling data // in the format expected by the pprof visualization tool. -// For more information about pprof, see -// http://code.google.com/p/google-perftools/. // // The package is typically only imported for the side effect of // registering its HTTP handlers. @@ -15,7 +13,7 @@ // import _ "net/http/pprof" // // If your application is not already running an http server, you -// need to start one. Add "net/http" and "log" to your imports and +// need to start one. Add "net/http" and "log" to your imports and // the following code to your main function: // // go func() { @@ -30,7 +28,8 @@ // // go tool pprof http://localhost:6060/debug/pprof/profile // -// Or to look at the goroutine blocking profile: +// Or to look at the goroutine blocking profile, after calling +// runtime.SetBlockProfileRate in your program: // // go tool pprof http://localhost:6060/debug/pprof/block // @@ -118,8 +117,8 @@ func Profile(w http.ResponseWriter, r *http.Request) { // Tracing lasts for duration specified in seconds GET parameter, or for 1 second if not specified. // The package initialization registers it as /debug/pprof/trace. func Trace(w http.ResponseWriter, r *http.Request) { - sec, _ := strconv.ParseInt(r.FormValue("seconds"), 10, 64) - if sec == 0 { + sec, err := strconv.ParseFloat(r.FormValue("seconds"), 64) + if sec <= 0 || err != nil { sec = 1 } @@ -127,18 +126,16 @@ func Trace(w http.ResponseWriter, r *http.Request) { // because if it does it starts writing. w.Header().Set("Content-Type", "application/octet-stream") w.Write([]byte("tracing not yet supported with gccgo")) - /* - if err := trace.Start(w); err != nil { - // trace.Start failed, so no writes yet. - // Can change header back to text content and send error code. - w.Header().Set("Content-Type", "text/plain; charset=utf-8") - w.WriteHeader(http.StatusInternalServerError) - fmt.Fprintf(w, "Could not enable tracing: %s\n", err) - return - } - sleep(w, time.Duration(sec)*time.Second) - trace.Stop() - */ + // if err := trace.Start(w); err != nil { + // // trace.Start failed, so no writes yet. + // // Can change header back to text content and send error code. + // w.Header().Set("Content-Type", "text/plain; charset=utf-8") + // w.WriteHeader(http.StatusInternalServerError) + // fmt.Fprintf(w, "Could not enable tracing: %s\n", err) + // return + // } + // sleep(w, time.Duration(sec*float64(time.Second))) + // trace.Stop() } // Symbol looks up the program counters listed in the request, @@ -148,11 +145,11 @@ func Symbol(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/plain; charset=utf-8") // We have to read the whole POST body before - // writing any output. Buffer the output here. + // writing any output. Buffer the output here. var buf bytes.Buffer // We don't know how many symbols we have, but we - // do have symbol information. Pprof only cares whether + // do have symbol information. Pprof only cares whether // this number is 0 (no symbols available) or > 0. fmt.Fprintf(&buf, "num_symbols: 1\n") diff --git a/libgo/go/net/http/range_test.go b/libgo/go/net/http/range_test.go index ef911af7b0..114987ed2c 100644 --- a/libgo/go/net/http/range_test.go +++ b/libgo/go/net/http/range_test.go @@ -38,7 +38,7 @@ var ParseRangeTests = []struct { {"bytes=0-", 10, []httpRange{{0, 10}}}, {"bytes=5-", 10, []httpRange{{5, 5}}}, {"bytes=0-20", 10, []httpRange{{0, 10}}}, - {"bytes=15-,0-5", 10, nil}, + {"bytes=15-,0-5", 10, []httpRange{{0, 6}}}, {"bytes=1-2,5-", 10, []httpRange{{1, 2}, {5, 5}}}, {"bytes=-2 , 7-", 11, []httpRange{{9, 2}, {7, 4}}}, {"bytes=0-0 ,2-2, 7-", 11, []httpRange{{0, 1}, {2, 1}, {7, 4}}}, diff --git a/libgo/go/net/http/readrequest_test.go b/libgo/go/net/http/readrequest_test.go index 60e2be41d1..28a148b9ac 100644 --- a/libgo/go/net/http/readrequest_test.go +++ b/libgo/go/net/http/readrequest_test.go @@ -1,4 +1,4 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -25,7 +25,7 @@ type reqTest struct { } var noError = "" -var noBody = "" +var noBodyStr = "" var noTrailer Header = nil var reqTests = []reqTest{ @@ -95,7 +95,7 @@ var reqTests = []reqTest{ RequestURI: "/", }, - noBody, + noBodyStr, noTrailer, noError, }, @@ -121,7 +121,7 @@ var reqTests = []reqTest{ RequestURI: "//user@host/is/actually/a/path/", }, - noBody, + noBodyStr, noTrailer, noError, }, @@ -131,7 +131,7 @@ var reqTests = []reqTest{ "GET ../../../../etc/passwd HTTP/1.1\r\n" + "Host: test\r\n\r\n", nil, - noBody, + noBodyStr, noTrailer, "parse ../../../../etc/passwd: invalid URI for request", }, @@ -141,7 +141,7 @@ var reqTests = []reqTest{ "GET HTTP/1.1\r\n" + "Host: test\r\n\r\n", nil, - noBody, + noBodyStr, noTrailer, "parse : empty url", }, @@ -227,7 +227,7 @@ var reqTests = []reqTest{ RequestURI: "www.google.com:443", }, - noBody, + noBodyStr, noTrailer, noError, }, @@ -251,7 +251,7 @@ var reqTests = []reqTest{ RequestURI: "127.0.0.1:6060", }, - noBody, + noBodyStr, noTrailer, noError, }, @@ -275,7 +275,7 @@ var reqTests = []reqTest{ RequestURI: "/_goRPC_", }, - noBody, + noBodyStr, noTrailer, noError, }, @@ -299,7 +299,7 @@ var reqTests = []reqTest{ RequestURI: "*", }, - noBody, + noBodyStr, noTrailer, noError, }, @@ -323,7 +323,7 @@ var reqTests = []reqTest{ RequestURI: "*", }, - noBody, + noBodyStr, noTrailer, noError, }, @@ -350,7 +350,7 @@ var reqTests = []reqTest{ RequestURI: "/", }, - noBody, + noBodyStr, noTrailer, noError, }, @@ -376,7 +376,28 @@ var reqTests = []reqTest{ RequestURI: "/", }, - noBody, + noBodyStr, + noTrailer, + noError, + }, + + // http2 client preface: + { + "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n", + &Request{ + Method: "PRI", + URL: &url.URL{ + Path: "*", + }, + Header: Header{}, + Proto: "HTTP/2.0", + ProtoMajor: 2, + ProtoMinor: 0, + RequestURI: "*", + ContentLength: -1, + Close: true, + }, + noBodyStr, noTrailer, noError, }, diff --git a/libgo/go/net/http/request.go b/libgo/go/net/http/request.go index 8cdab02af5..fb6bb0aab5 100644 --- a/libgo/go/net/http/request.go +++ b/libgo/go/net/http/request.go @@ -9,6 +9,7 @@ package http import ( "bufio" "bytes" + "context" "crypto/tls" "encoding/base64" "errors" @@ -17,11 +18,17 @@ import ( "io/ioutil" "mime" "mime/multipart" + "net" + "net/http/httptrace" "net/textproto" "net/url" "strconv" "strings" "sync" + + "golang_org/x/net/idna" + "golang_org/x/text/unicode/norm" + "golang_org/x/text/width" ) const ( @@ -32,21 +39,40 @@ const ( // is either not present in the request or not a file field. var ErrMissingFile = errors.New("http: no such file") -// HTTP request parsing errors. +// ProtocolError represents an HTTP protocol error. +// +// Deprecated: Not all errors in the http package related to protocol errors +// are of type ProtocolError. type ProtocolError struct { ErrorString string } -func (err *ProtocolError) Error() string { return err.ErrorString } +func (pe *ProtocolError) Error() string { return pe.ErrorString } var ( - ErrHeaderTooLong = &ProtocolError{"header too long"} - ErrShortBody = &ProtocolError{"entity body too short"} - ErrNotSupported = &ProtocolError{"feature not supported"} - ErrUnexpectedTrailer = &ProtocolError{"trailer header without chunked transfer encoding"} + // ErrNotSupported is returned by the Push method of Pusher + // implementations to indicate that HTTP/2 Push support is not + // available. + ErrNotSupported = &ProtocolError{"feature not supported"} + + // ErrUnexpectedTrailer is returned by the Transport when a server + // replies with a Trailer header, but without a chunked reply. + ErrUnexpectedTrailer = &ProtocolError{"trailer header without chunked transfer encoding"} + + // ErrMissingBoundary is returned by Request.MultipartReader when the + // request's Content-Type does not include a "boundary" parameter. + ErrMissingBoundary = &ProtocolError{"no multipart boundary param in Content-Type"} + + // ErrNotMultipart is returned by Request.MultipartReader when the + // request's Content-Type is not multipart/form-data. + ErrNotMultipart = &ProtocolError{"request Content-Type isn't multipart/form-data"} + + // Deprecated: ErrHeaderTooLong is not used. + ErrHeaderTooLong = &ProtocolError{"header too long"} + // Deprecated: ErrShortBody is not used. + ErrShortBody = &ProtocolError{"entity body too short"} + // Deprecated: ErrMissingContentLength is not used. ErrMissingContentLength = &ProtocolError{"missing ContentLength in HEAD response"} - ErrNotMultipart = &ProtocolError{"request Content-Type isn't multipart/form-data"} - ErrMissingBoundary = &ProtocolError{"no multipart boundary param in Content-Type"} ) type badStringError struct { @@ -144,11 +170,20 @@ type Request struct { // Handler does not need to. Body io.ReadCloser + // GetBody defines an optional func to return a new copy of + // Body. It is used for client requests when a redirect requires + // reading the body more than once. Use of GetBody still + // requires setting Body. + // + // For server requests it is unused. + GetBody func() (io.ReadCloser, error) + // ContentLength records the length of the associated content. // The value -1 indicates that the length is unknown. // Values >= 0 indicate that the given number of bytes may // be read from Body. - // For client requests, a value of 0 means unknown if Body is not nil. + // For client requests, a value of 0 with a non-nil Body is + // also treated as unknown. ContentLength int64 // TransferEncoding lists the transfer encodings from outermost to @@ -173,11 +208,15 @@ type Request struct { // For server requests Host specifies the host on which the // URL is sought. Per RFC 2616, this is either the value of // the "Host" header or the host name given in the URL itself. - // It may be of the form "host:port". + // It may be of the form "host:port". For international domain + // names, Host may be in Punycode or Unicode form. Use + // golang.org/x/net/idna to convert it to either format if + // needed. // // For client requests Host optionally overrides the Host // header to send. If empty, the Request.Write method uses - // the value of URL.Host. + // the value of URL.Host. Host may contain an international + // domain name. Host string // Form contains the parsed form data, including both the URL @@ -247,7 +286,52 @@ type Request struct { // RoundTripper may support Cancel. // // For server requests, this field is not applicable. + // + // Deprecated: Use the Context and WithContext methods + // instead. If a Request's Cancel field and context are both + // set, it is undefined whether Cancel is respected. Cancel <-chan struct{} + + // Response is the redirect response which caused this request + // to be created. This field is only populated during client + // redirects. + Response *Response + + // ctx is either the client or server context. It should only + // be modified via copying the whole Request using WithContext. + // It is unexported to prevent people from using Context wrong + // and mutating the contexts held by callers of the same request. + ctx context.Context +} + +// Context returns the request's context. To change the context, use +// WithContext. +// +// The returned context is always non-nil; it defaults to the +// background context. +// +// For outgoing client requests, the context controls cancelation. +// +// For incoming server requests, the context is canceled when the +// client's connection closes, the request is canceled (with HTTP/2), +// or when the ServeHTTP method returns. +func (r *Request) Context() context.Context { + if r.ctx != nil { + return r.ctx + } + return context.Background() +} + +// WithContext returns a shallow copy of r with its context changed +// to ctx. The provided ctx must be non-nil. +func (r *Request) WithContext(ctx context.Context) *Request { + if ctx == nil { + panic("nil context") + } + r2 := new(Request) + *r2 = *r + r2.ctx = ctx + return r2 } // ProtoAtLeast reports whether the HTTP protocol used @@ -257,6 +341,18 @@ func (r *Request) ProtoAtLeast(major, minor int) bool { r.ProtoMajor == major && r.ProtoMinor >= minor } +// protoAtLeastOutgoing is like ProtoAtLeast, but is for outgoing +// requests (see issue 18407) where these fields aren't supposed to +// matter. As a minor fix for Go 1.8, at least treat (0, 0) as +// matching HTTP/1.1 or HTTP/1.0. Only HTTP/1.1 is used. +// TODO(bradfitz): ideally remove this whole method. It shouldn't be used. +func (r *Request) protoAtLeastOutgoing(major, minor int) bool { + if r.ProtoMajor == 0 && r.ProtoMinor == 0 && major == 1 && minor <= 1 { + return true + } + return r.ProtoAtLeast(major, minor) +} + // UserAgent returns the client's User-Agent, if sent in the request. func (r *Request) UserAgent() string { return r.Header.Get("User-Agent") @@ -272,6 +368,8 @@ var ErrNoCookie = errors.New("http: named cookie not present") // Cookie returns the named cookie provided in the request or // ErrNoCookie if not found. +// If multiple cookies match the given name, only one cookie will +// be returned. func (r *Request) Cookie(name string) (*Cookie, error) { for _, c := range readCookies(r.Header, name) { return c, nil @@ -279,8 +377,8 @@ func (r *Request) Cookie(name string) (*Cookie, error) { return nil, ErrNoCookie } -// AddCookie adds a cookie to the request. Per RFC 6265 section 5.4, -// AddCookie does not attach more than one Cookie header field. That +// AddCookie adds a cookie to the request. Per RFC 6265 section 5.4, +// AddCookie does not attach more than one Cookie header field. That // means all cookies, if any, are written into the same line, // separated by semicolon. func (r *Request) AddCookie(c *Cookie) { @@ -343,6 +441,12 @@ func (r *Request) multipartReader() (*multipart.Reader, error) { return multipart.NewReader(r.Body, boundary), nil } +// isH2Upgrade reports whether r represents the http2 "client preface" +// magic string. +func (r *Request) isH2Upgrade() bool { + return r.Method == "PRI" && len(r.Header) == 0 && r.URL.Path == "*" && r.Proto == "HTTP/2.0" +} + // Return value if nonempty, def otherwise. func valueOrDefault(value, def string) string { if value != "" { @@ -375,7 +479,7 @@ func (r *Request) Write(w io.Writer) error { } // WriteProxy is like Write but writes the request in the form -// expected by an HTTP proxy. In particular, WriteProxy writes the +// expected by an HTTP proxy. In particular, WriteProxy writes the // initial Request-URI line of the request with an absolute URI, per // section 5.1.2 of RFC 2616, including the scheme and host. // In either case, WriteProxy also writes a Host header, using @@ -390,7 +494,16 @@ var errMissingHost = errors.New("http: Request.Write on Request with no Host or // extraHeaders may be nil // waitForContinue may be nil -func (req *Request) write(w io.Writer, usingProxy bool, extraHeaders Header, waitForContinue func() bool) error { +func (req *Request) write(w io.Writer, usingProxy bool, extraHeaders Header, waitForContinue func() bool) (err error) { + trace := httptrace.ContextClientTrace(req.Context()) + if trace != nil && trace.WroteRequest != nil { + defer func() { + trace.WroteRequest(httptrace.WroteRequestInfo{ + Err: err, + }) + }() + } + // Find the target host. Prefer the Host: header, but if that // is not given, use the host from the request URL. // @@ -427,7 +540,7 @@ func (req *Request) write(w io.Writer, usingProxy bool, extraHeaders Header, wai w = bw } - _, err := fmt.Fprintf(w, "%s %s HTTP/1.1\r\n", valueOrDefault(req.Method, "GET"), ruri) + _, err = fmt.Fprintf(w, "%s %s HTTP/1.1\r\n", valueOrDefault(req.Method, "GET"), ruri) if err != nil { return err } @@ -478,6 +591,10 @@ func (req *Request) write(w io.Writer, usingProxy bool, extraHeaders Header, wai return err } + if trace != nil && trace.WroteHeaders != nil { + trace.WroteHeaders() + } + // Flush and wait for 100-continue if expected. if waitForContinue != nil { if bw, ok := w.(*bufio.Writer); ok { @@ -486,13 +603,21 @@ func (req *Request) write(w io.Writer, usingProxy bool, extraHeaders Header, wai return err } } - + if trace != nil && trace.Wait100Continue != nil { + trace.Wait100Continue() + } if !waitForContinue() { req.closeBody() return nil } } + if bw, ok := w.(*bufio.Writer); ok && tw.FlushHeaders { + if err := bw.Flush(); err != nil { + return err + } + } + // Write body and trailer err = tw.WriteBody(w) if err != nil { @@ -505,7 +630,24 @@ func (req *Request) write(w io.Writer, usingProxy bool, extraHeaders Header, wai return nil } -// cleanHost strips anything after '/' or ' '. +func idnaASCII(v string) (string, error) { + if isASCII(v) { + return v, nil + } + // The idna package doesn't do everything from + // https://tools.ietf.org/html/rfc5895 so we do it here. + // TODO(bradfitz): should the idna package do this instead? + v = strings.ToLower(v) + v = width.Fold.String(v) + v = norm.NFC.String(v) + return idna.ToASCII(v) +} + +// cleanHost cleans up the host sent in request's Host header. +// +// It both strips anything after '/' or ' ', and puts the value +// into Punycode form, if necessary. +// // Ideally we'd clean the Host header according to the spec: // https://tools.ietf.org/html/rfc7230#section-5.4 (Host = uri-host [ ":" port ]") // https://tools.ietf.org/html/rfc7230#section-2.7 (uri-host -> rfc3986's host) @@ -516,12 +658,24 @@ func (req *Request) write(w io.Writer, usingProxy bool, extraHeaders Header, wai // first offending character. func cleanHost(in string) string { if i := strings.IndexAny(in, " /"); i != -1 { - return in[:i] + in = in[:i] + } + host, port, err := net.SplitHostPort(in) + if err != nil { // input was just a host + a, err := idnaASCII(in) + if err != nil { + return in // garbage in, garbage out + } + return a + } + a, err := idnaASCII(host) + if err != nil { + return in // garbage in, garbage out } - return in + return net.JoinHostPort(a, port) } -// removeZone removes IPv6 zone identifer from host. +// removeZone removes IPv6 zone identifier from host. // E.g., "[fe80::1%en0]:8080" to "[fe80::1]:8080" func removeZone(host string) string { if !strings.HasPrefix(host, "[") { @@ -590,11 +744,17 @@ func validMethod(method string) bool { // methods Do, Post, and PostForm, and Transport.RoundTrip. // // NewRequest returns a Request suitable for use with Client.Do or -// Transport.RoundTrip. -// To create a request for use with testing a Server Handler use either -// ReadRequest or manually update the Request fields. See the Request -// type's documentation for the difference between inbound and outbound -// request fields. +// Transport.RoundTrip. To create a request for use with testing a +// Server Handler, either use the NewRequest function in the +// net/http/httptest package, use ReadRequest, or manually update the +// Request fields. See the Request type's documentation for the +// difference between inbound and outbound request fields. +// +// If body is of type *bytes.Buffer, *bytes.Reader, or +// *strings.Reader, the returned request's ContentLength is set to its +// exact value (instead of -1), GetBody is populated (so 307 and 308 +// redirects can replay the body), and Body is set to NoBody if the +// ContentLength is 0. func NewRequest(method, urlStr string, body io.Reader) (*Request, error) { if method == "" { // We document that "" means "GET" for Request.Method, and people have @@ -613,6 +773,8 @@ func NewRequest(method, urlStr string, body io.Reader) (*Request, error) { if !ok && body != nil { rc = ioutil.NopCloser(body) } + // The host's colon:port should be normalized. See Issue 14836. + u.Host = removeEmptyPort(u.Host) req := &Request{ Method: method, URL: u, @@ -627,10 +789,43 @@ func NewRequest(method, urlStr string, body io.Reader) (*Request, error) { switch v := body.(type) { case *bytes.Buffer: req.ContentLength = int64(v.Len()) + buf := v.Bytes() + req.GetBody = func() (io.ReadCloser, error) { + r := bytes.NewReader(buf) + return ioutil.NopCloser(r), nil + } case *bytes.Reader: req.ContentLength = int64(v.Len()) + snapshot := *v + req.GetBody = func() (io.ReadCloser, error) { + r := snapshot + return ioutil.NopCloser(&r), nil + } case *strings.Reader: req.ContentLength = int64(v.Len()) + snapshot := *v + req.GetBody = func() (io.ReadCloser, error) { + r := snapshot + return ioutil.NopCloser(&r), nil + } + default: + // This is where we'd set it to -1 (at least + // if body != NoBody) to mean unknown, but + // that broke people during the Go 1.8 testing + // period. People depend on it being 0 I + // guess. Maybe retry later. See Issue 18117. + } + // For client requests, Request.ContentLength of 0 + // means either actually 0, or unknown. The only way + // to explicitly say that the ContentLength is zero is + // to set the Body to nil. But turns out too much code + // depends on NewRequest returning a non-nil Body, + // so we use a well-known ReadCloser variable instead + // and have the http package also treat that sentinel + // variable to mean explicitly zero. + if req.GetBody != nil && req.ContentLength == 0 { + req.Body = NoBody + req.GetBody = func() (io.ReadCloser, error) { return NoBody, nil } } } @@ -704,7 +899,9 @@ func putTextprotoReader(r *textproto.Reader) { } // ReadRequest reads and parses an incoming request from b. -func ReadRequest(b *bufio.Reader) (req *Request, err error) { return readRequest(b, deleteHostHeader) } +func ReadRequest(b *bufio.Reader) (*Request, error) { + return readRequest(b, deleteHostHeader) +} // Constants for readRequest's deleteHostHeader parameter. const ( @@ -768,13 +965,13 @@ func readRequest(b *bufio.Reader, deleteHostHeader bool) (req *Request, err erro } req.Header = Header(mimeHeader) - // RFC2616: Must treat + // RFC 2616: Must treat // GET /index.html HTTP/1.1 // Host: www.google.com // and // GET http://www.google.com/index.html HTTP/1.1 // Host: doesntmatter - // the same. In the second case, any Host line is ignored. + // the same. In the second case, any Host line is ignored. req.Host = req.URL.Host if req.Host == "" { req.Host = req.Header.get("Host") @@ -792,6 +989,16 @@ func readRequest(b *bufio.Reader, deleteHostHeader bool) (req *Request, err erro return nil, err } + if req.isH2Upgrade() { + // Because it's neither chunked, nor declared: + req.ContentLength = -1 + + // We want to give handlers a chance to hijack the + // connection, but we need to prevent the Server from + // dealing with the connection further if it's not + // hijacked. Set Close to ensure that: + req.Close = true + } return req, nil } @@ -808,57 +1015,56 @@ func MaxBytesReader(w ResponseWriter, r io.ReadCloser, n int64) io.ReadCloser { } type maxBytesReader struct { - w ResponseWriter - r io.ReadCloser // underlying reader - n int64 // max bytes remaining - stopped bool - sawEOF bool + w ResponseWriter + r io.ReadCloser // underlying reader + n int64 // max bytes remaining + err error // sticky error } func (l *maxBytesReader) tooLarge() (n int, err error) { - if !l.stopped { - l.stopped = true - if res, ok := l.w.(*response); ok { - res.requestTooLarge() - } - } - return 0, errors.New("http: request body too large") + l.err = errors.New("http: request body too large") + return 0, l.err } func (l *maxBytesReader) Read(p []byte) (n int, err error) { - toRead := l.n - if l.n == 0 { - if l.sawEOF { - return l.tooLarge() - } - // The underlying io.Reader may not return (0, io.EOF) - // at EOF if the requested size is 0, so read 1 byte - // instead. The io.Reader docs are a bit ambiguous - // about the return value of Read when 0 bytes are - // requested, and {bytes,strings}.Reader gets it wrong - // too (it returns (0, nil) even at EOF). - toRead = 1 + if l.err != nil { + return 0, l.err + } + if len(p) == 0 { + return 0, nil } - if int64(len(p)) > toRead { - p = p[:toRead] + // If they asked for a 32KB byte read but only 5 bytes are + // remaining, no need to read 32KB. 6 bytes will answer the + // question of the whether we hit the limit or go past it. + if int64(len(p)) > l.n+1 { + p = p[:l.n+1] } n, err = l.r.Read(p) - if err == io.EOF { - l.sawEOF = true - } - if l.n == 0 { - // If we had zero bytes to read remaining (but hadn't seen EOF) - // and we get a byte here, that means we went over our limit. - if n > 0 { - return l.tooLarge() - } - return 0, err + + if int64(n) <= l.n { + l.n -= int64(n) + l.err = err + return n, err } - l.n -= int64(n) - if l.n < 0 { - l.n = 0 + + n = int(l.n) + l.n = 0 + + // The server code and client code both use + // maxBytesReader. This "requestTooLarge" check is + // only used by the server code. To prevent binaries + // which only using the HTTP Client code (such as + // cmd/go) from also linking in the HTTP server, don't + // use a static type assertion to the server + // "*response" type. Check this interface instead: + type requestTooLarger interface { + requestTooLarge() } - return + if res, ok := l.w.(requestTooLarger); ok { + res.requestTooLarge() + } + l.err = errors.New("http: request body too large") + return n, l.err } func (l *maxBytesReader) Close() error { @@ -919,18 +1125,24 @@ func parsePostForm(r *Request) (vs url.Values, err error) { return } -// ParseForm parses the raw query from the URL and updates r.Form. +// ParseForm populates r.Form and r.PostForm. +// +// For all requests, ParseForm parses the raw query from the URL and updates +// r.Form. // -// For POST or PUT requests, it also parses the request body as a form and -// put the results into both r.PostForm and r.Form. -// POST and PUT body parameters take precedence over URL query string values -// in r.Form. +// For POST, PUT, and PATCH requests, it also parses the request body as a form +// and puts the results into both r.PostForm and r.Form. Request body parameters +// take precedence over URL query string values in r.Form. +// +// For other HTTP methods, or when the Content-Type is not +// application/x-www-form-urlencoded, the request Body is not read, and +// r.PostForm is initialized to a non-nil, empty value. // // If the request Body's size has not already been limited by MaxBytesReader, // the size is capped at 10MB. // // ParseMultipartForm calls ParseForm automatically. -// It is idempotent. +// ParseForm is idempotent. func (r *Request) ParseForm() error { var err error if r.PostForm == nil { @@ -995,9 +1207,16 @@ func (r *Request) ParseMultipartForm(maxMemory int64) error { if err != nil { return err } + + if r.PostForm == nil { + r.PostForm = make(url.Values) + } for k, v := range f.Value { r.Form[k] = append(r.Form[k], v...) + // r.PostForm should also be populated. See Issue 9305. + r.PostForm[k] = append(r.PostForm[k], v...) } + r.MultipartForm = f return nil @@ -1087,91 +1306,29 @@ func (r *Request) isReplayable() bool { return false } -func validHostHeader(h string) bool { - // The latests spec is actually this: - // - // http://tools.ietf.org/html/rfc7230#section-5.4 - // Host = uri-host [ ":" port ] - // - // Where uri-host is: - // http://tools.ietf.org/html/rfc3986#section-3.2.2 - // - // But we're going to be much more lenient for now and just - // search for any byte that's not a valid byte in any of those - // expressions. - for i := 0; i < len(h); i++ { - if !validHostByte[h[i]] { - return false - } +// outgoingLength reports the Content-Length of this outgoing (Client) request. +// It maps 0 into -1 (unknown) when the Body is non-nil. +func (r *Request) outgoingLength() int64 { + if r.Body == nil || r.Body == NoBody { + return 0 } - return true -} - -// See the validHostHeader comment. -var validHostByte = [256]bool{ - '0': true, '1': true, '2': true, '3': true, '4': true, '5': true, '6': true, '7': true, - '8': true, '9': true, - - 'a': true, 'b': true, 'c': true, 'd': true, 'e': true, 'f': true, 'g': true, 'h': true, - 'i': true, 'j': true, 'k': true, 'l': true, 'm': true, 'n': true, 'o': true, 'p': true, - 'q': true, 'r': true, 's': true, 't': true, 'u': true, 'v': true, 'w': true, 'x': true, - 'y': true, 'z': true, - - 'A': true, 'B': true, 'C': true, 'D': true, 'E': true, 'F': true, 'G': true, 'H': true, - 'I': true, 'J': true, 'K': true, 'L': true, 'M': true, 'N': true, 'O': true, 'P': true, - 'Q': true, 'R': true, 'S': true, 'T': true, 'U': true, 'V': true, 'W': true, 'X': true, - 'Y': true, 'Z': true, - - '!': true, // sub-delims - '$': true, // sub-delims - '%': true, // pct-encoded (and used in IPv6 zones) - '&': true, // sub-delims - '(': true, // sub-delims - ')': true, // sub-delims - '*': true, // sub-delims - '+': true, // sub-delims - ',': true, // sub-delims - '-': true, // unreserved - '.': true, // unreserved - ':': true, // IPv6address + Host expression's optional port - ';': true, // sub-delims - '=': true, // sub-delims - '[': true, - '\'': true, // sub-delims - ']': true, - '_': true, // unreserved - '~': true, // unreserved -} - -func validHeaderName(v string) bool { - if len(v) == 0 { - return false + if r.ContentLength != 0 { + return r.ContentLength } - return strings.IndexFunc(v, isNotToken) == -1 + return -1 } -// validHeaderValue reports whether v is a valid "field-value" according to -// http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.2 : -// -// message-header = field-name ":" [ field-value ] -// field-value = *( field-content | LWS ) -// field-content = <the OCTETs making up the field-value -// and consisting of either *TEXT or combinations -// of token, separators, and quoted-string> -// -// http://www.w3.org/Protocols/rfc2616/rfc2616-sec2.html#sec2.2 : -// -// TEXT = <any OCTET except CTLs, -// but including LWS> -// LWS = [CRLF] 1*( SP | HT ) -// CTL = <any US-ASCII control character -// (octets 0 - 31) and DEL (127)> -func validHeaderValue(v string) bool { - for i := 0; i < len(v); i++ { - b := v[i] - if isCTL(b) && !isLWS(b) { - return false - } +// requestMethodUsuallyLacksBody reports whether the given request +// method is one that typically does not involve a request body. +// This is used by the Transport (via +// transferWriter.shouldSendChunkedRequestBody) to determine whether +// we try to test-read a byte from a non-nil Request.Body when +// Request.outgoingLength() returns -1. See the comments in +// shouldSendChunkedRequestBody. +func requestMethodUsuallyLacksBody(method string) bool { + switch method { + case "GET", "HEAD", "DELETE", "OPTIONS", "PROPFIND", "SEARCH": + return true } - return true + return false } diff --git a/libgo/go/net/http/request_test.go b/libgo/go/net/http/request_test.go index 0ecdf85a56..e6748375b5 100644 --- a/libgo/go/net/http/request_test.go +++ b/libgo/go/net/http/request_test.go @@ -29,9 +29,9 @@ func TestQuery(t *testing.T) { } } -func TestPostQuery(t *testing.T) { - req, _ := NewRequest("POST", "http://www.google.com/search?q=foo&q=bar&both=x&prio=1&empty=not", - strings.NewReader("z=post&both=y&prio=2&empty=")) +func TestParseFormQuery(t *testing.T) { + req, _ := NewRequest("POST", "http://www.google.com/search?q=foo&q=bar&both=x&prio=1&orphan=nope&empty=not", + strings.NewReader("z=post&both=y&prio=2&=nokey&orphan;empty=&")) req.Header.Set("Content-Type", "application/x-www-form-urlencoded; param=value") if q := req.FormValue("q"); q != "foo" { @@ -55,39 +55,30 @@ func TestPostQuery(t *testing.T) { if prio := req.FormValue("prio"); prio != "2" { t.Errorf(`req.FormValue("prio") = %q, want "2" (from body)`, prio) } - if empty := req.FormValue("empty"); empty != "" { + if orphan := req.Form["orphan"]; !reflect.DeepEqual(orphan, []string{"", "nope"}) { + t.Errorf(`req.FormValue("orphan") = %q, want "" (from body)`, orphan) + } + if empty := req.Form["empty"]; !reflect.DeepEqual(empty, []string{"", "not"}) { t.Errorf(`req.FormValue("empty") = %q, want "" (from body)`, empty) } + if nokey := req.Form[""]; !reflect.DeepEqual(nokey, []string{"nokey"}) { + t.Errorf(`req.FormValue("nokey") = %q, want "nokey" (from body)`, nokey) + } } -func TestPatchQuery(t *testing.T) { - req, _ := NewRequest("PATCH", "http://www.google.com/search?q=foo&q=bar&both=x&prio=1&empty=not", - strings.NewReader("z=post&both=y&prio=2&empty=")) - req.Header.Set("Content-Type", "application/x-www-form-urlencoded; param=value") - - if q := req.FormValue("q"); q != "foo" { - t.Errorf(`req.FormValue("q") = %q, want "foo"`, q) - } - if z := req.FormValue("z"); z != "post" { - t.Errorf(`req.FormValue("z") = %q, want "post"`, z) - } - if bq, found := req.PostForm["q"]; found { - t.Errorf(`req.PostForm["q"] = %q, want no entry in map`, bq) - } - if bz := req.PostFormValue("z"); bz != "post" { - t.Errorf(`req.PostFormValue("z") = %q, want "post"`, bz) - } - if qs := req.Form["q"]; !reflect.DeepEqual(qs, []string{"foo", "bar"}) { - t.Errorf(`req.Form["q"] = %q, want ["foo", "bar"]`, qs) - } - if both := req.Form["both"]; !reflect.DeepEqual(both, []string{"y", "x"}) { - t.Errorf(`req.Form["both"] = %q, want ["y", "x"]`, both) - } - if prio := req.FormValue("prio"); prio != "2" { - t.Errorf(`req.FormValue("prio") = %q, want "2" (from body)`, prio) - } - if empty := req.FormValue("empty"); empty != "" { - t.Errorf(`req.FormValue("empty") = %q, want "" (from body)`, empty) +// Tests that we only parse the form automatically for certain methods. +func TestParseFormQueryMethods(t *testing.T) { + for _, method := range []string{"POST", "PATCH", "PUT", "FOO"} { + req, _ := NewRequest(method, "http://www.google.com/search", + strings.NewReader("foo=bar")) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded; param=value") + want := "bar" + if method == "FOO" { + want = "" + } + if got := req.FormValue("foo"); got != want { + t.Errorf(`for method %s, FormValue("foo") = %q; want %q`, method, got, want) + } } } @@ -99,7 +90,7 @@ type parseContentTypeTest struct { var parseContentTypeTests = []parseContentTypeTest{ {false, stringMap{"Content-Type": {"text/plain"}}}, - // Empty content type is legal - shoult be treated as + // Empty content type is legal - should be treated as // application/octet-stream (RFC 2616, section 7.2.1) {false, stringMap{}}, {true, stringMap{"Content-Type": {"text/plain; boundary="}}}, @@ -158,6 +149,68 @@ func TestMultipartReader(t *testing.T) { } } +// Issue 9305: ParseMultipartForm should populate PostForm too +func TestParseMultipartFormPopulatesPostForm(t *testing.T) { + postData := + `--xxx +Content-Disposition: form-data; name="field1" + +value1 +--xxx +Content-Disposition: form-data; name="field2" + +value2 +--xxx +Content-Disposition: form-data; name="file"; filename="file" +Content-Type: application/octet-stream +Content-Transfer-Encoding: binary + +binary data +--xxx-- +` + req := &Request{ + Method: "POST", + Header: Header{"Content-Type": {`multipart/form-data; boundary=xxx`}}, + Body: ioutil.NopCloser(strings.NewReader(postData)), + } + + initialFormItems := map[string]string{ + "language": "Go", + "name": "gopher", + "skill": "go-ing", + "field2": "initial-value2", + } + + req.Form = make(url.Values) + for k, v := range initialFormItems { + req.Form.Add(k, v) + } + + err := req.ParseMultipartForm(10000) + if err != nil { + t.Fatalf("unexpected multipart error %v", err) + } + + wantForm := url.Values{ + "language": []string{"Go"}, + "name": []string{"gopher"}, + "skill": []string{"go-ing"}, + "field1": []string{"value1"}, + "field2": []string{"initial-value2", "value2"}, + } + if !reflect.DeepEqual(req.Form, wantForm) { + t.Fatalf("req.Form = %v, want %v", req.Form, wantForm) + } + + wantPostForm := url.Values{ + "field1": []string{"value1"}, + "field2": []string{"value2"}, + } + if !reflect.DeepEqual(req.PostForm, wantPostForm) { + t.Fatalf("req.PostForm = %v, want %v", req.PostForm, wantPostForm) + } +} + func TestParseMultipartForm(t *testing.T) { req := &Request{ Method: "POST", @@ -312,18 +365,68 @@ func TestFormFileOrder(t *testing.T) { var readRequestErrorTests = []struct { in string - err error + err string + + header Header }{ - {"GET / HTTP/1.1\r\nheader:foo\r\n\r\n", nil}, - {"GET / HTTP/1.1\r\nheader:foo\r\n", io.ErrUnexpectedEOF}, - {"", io.EOF}, + 0: {"GET / HTTP/1.1\r\nheader:foo\r\n\r\n", "", Header{"Header": {"foo"}}}, + 1: {"GET / HTTP/1.1\r\nheader:foo\r\n", io.ErrUnexpectedEOF.Error(), nil}, + 2: {"", io.EOF.Error(), nil}, + 3: { + in: "HEAD / HTTP/1.1\r\nContent-Length:4\r\n\r\n", + err: "http: method cannot contain a Content-Length", + }, + 4: { + in: "HEAD / HTTP/1.1\r\n\r\n", + header: Header{}, + }, + + // Multiple Content-Length values should either be + // deduplicated if same or reject otherwise + // See Issue 16490. + 5: { + in: "POST / HTTP/1.1\r\nContent-Length: 10\r\nContent-Length: 0\r\n\r\nGopher hey\r\n", + err: "cannot contain multiple Content-Length headers", + }, + 6: { + in: "POST / HTTP/1.1\r\nContent-Length: 10\r\nContent-Length: 6\r\n\r\nGopher\r\n", + err: "cannot contain multiple Content-Length headers", + }, + 7: { + in: "PUT / HTTP/1.1\r\nContent-Length: 6 \r\nContent-Length: 6\r\nContent-Length:6\r\n\r\nGopher\r\n", + err: "", + header: Header{"Content-Length": {"6"}}, + }, + 8: { + in: "PUT / HTTP/1.1\r\nContent-Length: 1\r\nContent-Length: 6 \r\n\r\n", + err: "cannot contain multiple Content-Length headers", + }, + 9: { + in: "POST / HTTP/1.1\r\nContent-Length:\r\nContent-Length: 3\r\n\r\n", + err: "cannot contain multiple Content-Length headers", + }, + 10: { + in: "HEAD / HTTP/1.1\r\nContent-Length:0\r\nContent-Length: 0\r\n\r\n", + header: Header{"Content-Length": {"0"}}, + }, } func TestReadRequestErrors(t *testing.T) { for i, tt := range readRequestErrorTests { - _, err := ReadRequest(bufio.NewReader(strings.NewReader(tt.in))) - if err != tt.err { - t.Errorf("%d. got error = %v; want %v", i, err, tt.err) + req, err := ReadRequest(bufio.NewReader(strings.NewReader(tt.in))) + if err == nil { + if tt.err != "" { + t.Errorf("#%d: got nil err; want %q", i, tt.err) + } + + if !reflect.DeepEqual(tt.header, req.Header) { + t.Errorf("#%d: gotHeader: %q wantHeader: %q", i, req.Header, tt.header) + } + continue + } + + if tt.err == "" || !strings.Contains(err.Error(), tt.err) { + t.Errorf("%d: got error = %v; want %v", i, err, tt.err) } } } @@ -336,11 +439,13 @@ var newRequestHostTests = []struct { {"http://192.168.0.1/", "192.168.0.1"}, {"http://192.168.0.1:8080/", "192.168.0.1:8080"}, + {"http://192.168.0.1:/", "192.168.0.1"}, {"http://[fe80::1]/", "[fe80::1]"}, {"http://[fe80::1]:8080/", "[fe80::1]:8080"}, {"http://[fe80::1%25en0]/", "[fe80::1%en0]"}, {"http://[fe80::1%25en0]:8080/", "[fe80::1%en0]:8080"}, + {"http://[fe80::1%25en0]:/", "[fe80::1%en0]"}, } func TestNewRequestHost(t *testing.T) { @@ -392,18 +497,23 @@ func TestNewRequestContentLength(t *testing.T) { {bytes.NewReader([]byte("123")), 3}, {bytes.NewBuffer([]byte("1234")), 4}, {strings.NewReader("12345"), 5}, - // Not detected: + {strings.NewReader(""), 0}, + {NoBody, 0}, + + // Not detected. During Go 1.8 we tried to make these set to -1, but + // due to Issue 18117, we keep these returning 0, even though they're + // unknown. {struct{ io.Reader }{strings.NewReader("xyz")}, 0}, {io.NewSectionReader(strings.NewReader("x"), 0, 6), 0}, {readByte(io.NewSectionReader(strings.NewReader("xy"), 0, 6)), 0}, } - for _, tt := range tests { + for i, tt := range tests { req, err := NewRequest("POST", "http://localhost/", tt.r) if err != nil { t.Fatal(err) } if req.ContentLength != tt.want { - t.Errorf("ContentLength(%T) = %d; want %d", tt.r, req.ContentLength, tt.want) + t.Errorf("test[%d]: ContentLength(%T) = %d; want %d", i, tt.r, req.ContentLength, tt.want) } } } @@ -562,11 +672,31 @@ func TestStarRequest(t *testing.T) { if err != nil { return } + if req.ContentLength != 0 { + t.Errorf("ContentLength = %d; want 0", req.ContentLength) + } + if req.Body == nil { + t.Errorf("Body = nil; want non-nil") + } + + // Request.Write has Client semantics for Body/ContentLength, + // where ContentLength 0 means unknown if Body is non-nil, and + // thus chunking will happen unless we change semantics and + // signal that we want to serialize it as exactly zero. The + // only way to do that for outbound requests is with a nil + // Body: + clientReq := *req + clientReq.Body = nil + var out bytes.Buffer - if err := req.Write(&out); err != nil { + if err := clientReq.Write(&out); err != nil { t.Fatal(err) } - back, err := ReadRequest(bufio.NewReader(&out)) + + if strings.Contains(out.String(), "chunked") { + t.Error("wrote chunked request; want no body") + } + back, err := ReadRequest(bufio.NewReader(bytes.NewReader(out.Bytes()))) if err != nil { t.Fatal(err) } @@ -615,6 +745,87 @@ func TestIssue10884_MaxBytesEOF(t *testing.T) { } } +// Issue 14981: MaxBytesReader's return error wasn't sticky. It +// doesn't technically need to be, but people expected it to be. +func TestMaxBytesReaderStickyError(t *testing.T) { + isSticky := func(r io.Reader) error { + var log bytes.Buffer + buf := make([]byte, 1000) + var firstErr error + for { + n, err := r.Read(buf) + fmt.Fprintf(&log, "Read(%d) = %d, %v\n", len(buf), n, err) + if err == nil { + continue + } + if firstErr == nil { + firstErr = err + continue + } + if !reflect.DeepEqual(err, firstErr) { + return fmt.Errorf("non-sticky error. got log:\n%s", log.Bytes()) + } + t.Logf("Got log: %s", log.Bytes()) + return nil + } + } + tests := [...]struct { + readable int + limit int64 + }{ + 0: {99, 100}, + 1: {100, 100}, + 2: {101, 100}, + } + for i, tt := range tests { + rc := MaxBytesReader(nil, ioutil.NopCloser(bytes.NewReader(make([]byte, tt.readable))), tt.limit) + if err := isSticky(rc); err != nil { + t.Errorf("%d. error: %v", i, err) + } + } +} + +// verify that NewRequest sets Request.GetBody and that it works +func TestNewRequestGetBody(t *testing.T) { + tests := []struct { + r io.Reader + }{ + {r: strings.NewReader("hello")}, + {r: bytes.NewReader([]byte("hello"))}, + {r: bytes.NewBuffer([]byte("hello"))}, + } + for i, tt := range tests { + req, err := NewRequest("POST", "http://foo.tld/", tt.r) + if err != nil { + t.Errorf("test[%d]: %v", i, err) + continue + } + if req.Body == nil { + t.Errorf("test[%d]: Body = nil", i) + continue + } + if req.GetBody == nil { + t.Errorf("test[%d]: GetBody = nil", i) + continue + } + slurp1, err := ioutil.ReadAll(req.Body) + if err != nil { + t.Errorf("test[%d]: ReadAll(Body) = %v", i, err) + } + newBody, err := req.GetBody() + if err != nil { + t.Errorf("test[%d]: GetBody = %v", i, err) + } + slurp2, err := ioutil.ReadAll(newBody) + if err != nil { + t.Errorf("test[%d]: ReadAll(GetBody()) = %v", i, err) + } + if string(slurp1) != string(slurp2) { + t.Errorf("test[%d]: Body %q != GetBody %q", i, slurp1, slurp2) + } + } +} + func testMissingFile(t *testing.T, req *Request) { f, fh, err := req.FormFile("missing") if f != nil { diff --git a/libgo/go/net/http/requestwrite_test.go b/libgo/go/net/http/requestwrite_test.go index cfb95b0a80..eb65b9f736 100644 --- a/libgo/go/net/http/requestwrite_test.go +++ b/libgo/go/net/http/requestwrite_test.go @@ -1,18 +1,21 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package http import ( + "bufio" "bytes" "errors" "fmt" "io" "io/ioutil" + "net" "net/url" "strings" "testing" + "time" ) type reqWriteTest struct { @@ -28,7 +31,7 @@ type reqWriteTest struct { var reqWriteTests = []reqWriteTest{ // HTTP/1.1 => chunked coding; no body; no trailer - { + 0: { Req: Request{ Method: "GET", URL: &url.URL{ @@ -75,7 +78,7 @@ var reqWriteTests = []reqWriteTest{ "Proxy-Connection: keep-alive\r\n\r\n", }, // HTTP/1.1 => chunked coding; body; empty trailer - { + 1: { Req: Request{ Method: "GET", URL: &url.URL{ @@ -104,7 +107,7 @@ var reqWriteTests = []reqWriteTest{ chunk("abcdef") + chunk(""), }, // HTTP/1.1 POST => chunked coding; body; empty trailer - { + 2: { Req: Request{ Method: "POST", URL: &url.URL{ @@ -137,7 +140,7 @@ var reqWriteTests = []reqWriteTest{ }, // HTTP/1.1 POST with Content-Length, no chunking - { + 3: { Req: Request{ Method: "POST", URL: &url.URL{ @@ -172,7 +175,7 @@ var reqWriteTests = []reqWriteTest{ }, // HTTP/1.1 POST with Content-Length in headers - { + 4: { Req: Request{ Method: "POST", URL: mustParseURL("http://example.com/"), @@ -201,7 +204,7 @@ var reqWriteTests = []reqWriteTest{ }, // default to HTTP/1.1 - { + 5: { Req: Request{ Method: "GET", URL: mustParseURL("/search"), @@ -215,7 +218,7 @@ var reqWriteTests = []reqWriteTest{ }, // Request with a 0 ContentLength and a 0 byte body. - { + 6: { Req: Request{ Method: "POST", URL: mustParseURL("/"), @@ -227,9 +230,32 @@ var reqWriteTests = []reqWriteTest{ Body: func() io.ReadCloser { return ioutil.NopCloser(io.LimitReader(strings.NewReader("xx"), 0)) }, - // RFC 2616 Section 14.13 says Content-Length should be specified - // unless body is prohibited by the request method. - // Also, nginx expects it for POST and PUT. + WantWrite: "POST / HTTP/1.1\r\n" + + "Host: example.com\r\n" + + "User-Agent: Go-http-client/1.1\r\n" + + "Transfer-Encoding: chunked\r\n" + + "\r\n0\r\n\r\n", + + WantProxy: "POST / HTTP/1.1\r\n" + + "Host: example.com\r\n" + + "User-Agent: Go-http-client/1.1\r\n" + + "Transfer-Encoding: chunked\r\n" + + "\r\n0\r\n\r\n", + }, + + // Request with a 0 ContentLength and a nil body. + 7: { + Req: Request{ + Method: "POST", + URL: mustParseURL("/"), + Host: "example.com", + ProtoMajor: 1, + ProtoMinor: 1, + ContentLength: 0, // as if unset by user + }, + + Body: func() io.ReadCloser { return nil }, + WantWrite: "POST / HTTP/1.1\r\n" + "Host: example.com\r\n" + "User-Agent: Go-http-client/1.1\r\n" + @@ -244,7 +270,7 @@ var reqWriteTests = []reqWriteTest{ }, // Request with a 0 ContentLength and a 1 byte body. - { + 8: { Req: Request{ Method: "POST", URL: mustParseURL("/"), @@ -270,7 +296,7 @@ var reqWriteTests = []reqWriteTest{ }, // Request with a ContentLength of 10 but a 5 byte body. - { + 9: { Req: Request{ Method: "POST", URL: mustParseURL("/"), @@ -284,7 +310,7 @@ var reqWriteTests = []reqWriteTest{ }, // Request with a ContentLength of 4 but an 8 byte body. - { + 10: { Req: Request{ Method: "POST", URL: mustParseURL("/"), @@ -298,7 +324,7 @@ var reqWriteTests = []reqWriteTest{ }, // Request with a 5 ContentLength and nil body. - { + 11: { Req: Request{ Method: "POST", URL: mustParseURL("/"), @@ -311,7 +337,7 @@ var reqWriteTests = []reqWriteTest{ }, // Request with a 0 ContentLength and a body with 1 byte content and an error. - { + 12: { Req: Request{ Method: "POST", URL: mustParseURL("/"), @@ -331,7 +357,7 @@ var reqWriteTests = []reqWriteTest{ }, // Request with a 0 ContentLength and a body without content and an error. - { + 13: { Req: Request{ Method: "POST", URL: mustParseURL("/"), @@ -352,7 +378,7 @@ var reqWriteTests = []reqWriteTest{ // Verify that DumpRequest preserves the HTTP version number, doesn't add a Host, // and doesn't add a User-Agent. - { + 14: { Req: Request{ Method: "GET", URL: mustParseURL("/foo"), @@ -373,7 +399,7 @@ var reqWriteTests = []reqWriteTest{ // an empty Host header, and don't use // Request.Header["Host"]. This is just testing that // we don't change Go 1.0 behavior. - { + 15: { Req: Request{ Method: "GET", Host: "", @@ -395,7 +421,7 @@ var reqWriteTests = []reqWriteTest{ }, // Opaque test #1 from golang.org/issue/4860 - { + 16: { Req: Request{ Method: "GET", URL: &url.URL{ @@ -414,7 +440,7 @@ var reqWriteTests = []reqWriteTest{ }, // Opaque test #2 from golang.org/issue/4860 - { + 17: { Req: Request{ Method: "GET", URL: &url.URL{ @@ -433,7 +459,7 @@ var reqWriteTests = []reqWriteTest{ }, // Testing custom case in header keys. Issue 5022. - { + 18: { Req: Request{ Method: "GET", URL: &url.URL{ @@ -457,7 +483,7 @@ var reqWriteTests = []reqWriteTest{ }, // Request with host header field; IPv6 address with zone identifier - { + 19: { Req: Request{ Method: "GET", URL: &url.URL{ @@ -472,7 +498,7 @@ var reqWriteTests = []reqWriteTest{ }, // Request with optional host header field; IPv6 address with zone identifier - { + 20: { Req: Request{ Method: "GET", URL: &url.URL{ @@ -543,6 +569,138 @@ func TestRequestWrite(t *testing.T) { } } +func TestRequestWriteTransport(t *testing.T) { + t.Parallel() + + matchSubstr := func(substr string) func(string) error { + return func(written string) error { + if !strings.Contains(written, substr) { + return fmt.Errorf("expected substring %q in request: %s", substr, written) + } + return nil + } + } + + noContentLengthOrTransferEncoding := func(req string) error { + if strings.Contains(req, "Content-Length: ") { + return fmt.Errorf("unexpected Content-Length in request: %s", req) + } + if strings.Contains(req, "Transfer-Encoding: ") { + return fmt.Errorf("unexpected Transfer-Encoding in request: %s", req) + } + return nil + } + + all := func(checks ...func(string) error) func(string) error { + return func(req string) error { + for _, c := range checks { + if err := c(req); err != nil { + return err + } + } + return nil + } + } + + type testCase struct { + method string + clen int64 // ContentLength + body io.ReadCloser + want func(string) error + + // optional: + init func(*testCase) + afterReqRead func() + } + + tests := []testCase{ + { + method: "GET", + want: noContentLengthOrTransferEncoding, + }, + { + method: "GET", + body: ioutil.NopCloser(strings.NewReader("")), + want: noContentLengthOrTransferEncoding, + }, + { + method: "GET", + clen: -1, + body: ioutil.NopCloser(strings.NewReader("")), + want: noContentLengthOrTransferEncoding, + }, + // A GET with a body, with explicit content length: + { + method: "GET", + clen: 7, + body: ioutil.NopCloser(strings.NewReader("foobody")), + want: all(matchSubstr("Content-Length: 7"), + matchSubstr("foobody")), + }, + // A GET with a body, sniffing the leading "f" from "foobody". + { + method: "GET", + clen: -1, + body: ioutil.NopCloser(strings.NewReader("foobody")), + want: all(matchSubstr("Transfer-Encoding: chunked"), + matchSubstr("\r\n1\r\nf\r\n"), + matchSubstr("oobody")), + }, + // But a POST request is expected to have a body, so + // no sniffing happens: + { + method: "POST", + clen: -1, + body: ioutil.NopCloser(strings.NewReader("foobody")), + want: all(matchSubstr("Transfer-Encoding: chunked"), + matchSubstr("foobody")), + }, + { + method: "POST", + clen: -1, + body: ioutil.NopCloser(strings.NewReader("")), + want: all(matchSubstr("Transfer-Encoding: chunked")), + }, + // Verify that a blocking Request.Body doesn't block forever. + { + method: "GET", + clen: -1, + init: func(tt *testCase) { + pr, pw := io.Pipe() + tt.afterReqRead = func() { + pw.Close() + } + tt.body = ioutil.NopCloser(pr) + }, + want: matchSubstr("Transfer-Encoding: chunked"), + }, + } + + for i, tt := range tests { + if tt.init != nil { + tt.init(&tt) + } + req := &Request{ + Method: tt.method, + URL: &url.URL{ + Scheme: "http", + Host: "example.com", + }, + Header: make(Header), + ContentLength: tt.clen, + Body: tt.body, + } + got, err := dumpRequestOut(req, tt.afterReqRead) + if err != nil { + t.Errorf("test[%d]: %v", i, err) + continue + } + if err := tt.want(string(got)); err != nil { + t.Errorf("test[%d]: %v", i, err) + } + } +} + type closeChecker struct { io.Reader closed bool @@ -553,17 +711,19 @@ func (rc *closeChecker) Close() error { return nil } -// TestRequestWriteClosesBody tests that Request.Write does close its request.Body. +// TestRequestWriteClosesBody tests that Request.Write closes its request.Body. // It also indirectly tests NewRequest and that it doesn't wrap an existing Closer // inside a NopCloser, and that it serializes it correctly. func TestRequestWriteClosesBody(t *testing.T) { rc := &closeChecker{Reader: strings.NewReader("my body")} - req, _ := NewRequest("POST", "http://foo.com/", rc) - if req.ContentLength != 0 { - t.Errorf("got req.ContentLength %d, want 0", req.ContentLength) + req, err := NewRequest("POST", "http://foo.com/", rc) + if err != nil { + t.Fatal(err) } buf := new(bytes.Buffer) - req.Write(buf) + if err := req.Write(buf); err != nil { + t.Error(err) + } if !rc.closed { t.Error("body not closed after write") } @@ -571,12 +731,7 @@ func TestRequestWriteClosesBody(t *testing.T) { "Host: foo.com\r\n" + "User-Agent: Go-http-client/1.1\r\n" + "Transfer-Encoding: chunked\r\n\r\n" + - // TODO: currently we don't buffer before chunking, so we get a - // single "m" chunk before the other chunks, as this was the 1-byte - // read from our MultiReader where we stiched the Body back together - // after sniffing whether the Body was 0 bytes or not. - chunk("m") + - chunk("y body") + + chunk("my body") + chunk("") if buf.String() != expected { t.Errorf("write:\n got: %s\nwant: %s", buf.String(), expected) @@ -604,7 +759,7 @@ func TestRequestWriteError(t *testing.T) { failAfter, writeCount := 0, 0 errFail := errors.New("fake write failure") - // w is the buffered io.Writer to write the request to. It + // w is the buffered io.Writer to write the request to. It // fails exactly once on its Nth Write call, as controlled by // failAfter. It also tracks the number of calls in // writeCount. @@ -652,3 +807,76 @@ func TestRequestWriteError(t *testing.T) { t.Fatalf("writeCalls constant is outdated in test") } } + +// dumpRequestOut is a modified copy of net/http/httputil.DumpRequestOut. +// Unlike the original, this version doesn't mutate the req.Body and +// try to restore it. It always dumps the whole body. +// And it doesn't support https. +func dumpRequestOut(req *Request, onReadHeaders func()) ([]byte, error) { + + // Use the actual Transport code to record what we would send + // on the wire, but not using TCP. Use a Transport with a + // custom dialer that returns a fake net.Conn that waits + // for the full input (and recording it), and then responds + // with a dummy response. + var buf bytes.Buffer // records the output + pr, pw := io.Pipe() + defer pr.Close() + defer pw.Close() + dr := &delegateReader{c: make(chan io.Reader)} + + t := &Transport{ + Dial: func(net, addr string) (net.Conn, error) { + return &dumpConn{io.MultiWriter(&buf, pw), dr}, nil + }, + } + defer t.CloseIdleConnections() + + // Wait for the request before replying with a dummy response: + go func() { + req, err := ReadRequest(bufio.NewReader(pr)) + if err == nil { + if onReadHeaders != nil { + onReadHeaders() + } + // Ensure all the body is read; otherwise + // we'll get a partial dump. + io.Copy(ioutil.Discard, req.Body) + req.Body.Close() + } + dr.c <- strings.NewReader("HTTP/1.1 204 No Content\r\nConnection: close\r\n\r\n") + }() + + _, err := t.RoundTrip(req) + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// delegateReader is a reader that delegates to another reader, +// once it arrives on a channel. +type delegateReader struct { + c chan io.Reader + r io.Reader // nil until received from c +} + +func (r *delegateReader) Read(p []byte) (int, error) { + if r.r == nil { + r.r = <-r.c + } + return r.r.Read(p) +} + +// dumpConn is a net.Conn that writes to Writer and reads from Reader. +type dumpConn struct { + io.Writer + io.Reader +} + +func (c *dumpConn) Close() error { return nil } +func (c *dumpConn) LocalAddr() net.Addr { return nil } +func (c *dumpConn) RemoteAddr() net.Addr { return nil } +func (c *dumpConn) SetDeadline(t time.Time) error { return nil } +func (c *dumpConn) SetReadDeadline(t time.Time) error { return nil } +func (c *dumpConn) SetWriteDeadline(t time.Time) error { return nil } diff --git a/libgo/go/net/http/response.go b/libgo/go/net/http/response.go index c424f61cd0..ae118fb386 100644 --- a/libgo/go/net/http/response.go +++ b/libgo/go/net/http/response.go @@ -11,6 +11,7 @@ import ( "bytes" "crypto/tls" "errors" + "fmt" "io" "net/textproto" "net/url" @@ -33,7 +34,7 @@ type Response struct { ProtoMajor int // e.g. 1 ProtoMinor int // e.g. 0 - // Header maps header keys to values. If the response had multiple + // Header maps header keys to values. If the response had multiple // headers with the same key, they may be concatenated, with comma // delimiters. (Section 4.2 of RFC 2616 requires that multiple headers // be semantically equivalent to a comma-delimited sequence.) Values @@ -57,8 +58,8 @@ type Response struct { // with a "chunked" Transfer-Encoding. Body io.ReadCloser - // ContentLength records the length of the associated content. The - // value -1 indicates that the length is unknown. Unless Request.Method + // ContentLength records the length of the associated content. The + // value -1 indicates that the length is unknown. Unless Request.Method // is "HEAD", values >= 0 indicate that the given number of bytes may // be read from Body. ContentLength int64 @@ -68,10 +69,19 @@ type Response struct { TransferEncoding []string // Close records whether the header directed that the connection be - // closed after reading Body. The value is advice for clients: neither + // closed after reading Body. The value is advice for clients: neither // ReadResponse nor Response.Write ever closes a connection. Close bool + // Uncompressed reports whether the response was sent compressed but + // was decompressed by the http package. When true, reading from + // Body yields the uncompressed content instead of the compressed + // content actually set from the server, ContentLength is set to -1, + // and the "Content-Length" and "Content-Encoding" fields are deleted + // from the responseHeader. To get the original response from + // the server, set Transport.DisableCompression to true. + Uncompressed bool + // Trailer maps trailer keys to values in the same // format as Header. // @@ -86,7 +96,7 @@ type Response struct { // any trailer values sent by the server. Trailer Header - // The Request that was sent to obtain this Response. + // Request is the request that was sent to obtain this Response. // Request's Body is nil (having already been consumed). // This is only populated for Client requests. Request *Request @@ -108,8 +118,8 @@ func (r *Response) Cookies() []*Cookie { var ErrNoLocation = errors.New("http: no Location header in response") // Location returns the URL of the response's "Location" header, -// if present. Relative redirects are resolved relative to -// the Response's Request. ErrNoLocation is returned if no +// if present. Relative redirects are resolved relative to +// the Response's Request. ErrNoLocation is returned if no // Location header is present. func (r *Response) Location() (*url.URL, error) { lv := r.Header.Get("Location") @@ -184,7 +194,7 @@ func ReadResponse(r *bufio.Reader, req *Request) (*Response, error) { return resp, nil } -// RFC2616: Should treat +// RFC 2616: Should treat // Pragma: no-cache // like // Cache-Control: no-cache @@ -203,7 +213,7 @@ func (r *Response) ProtoAtLeast(major, minor int) bool { r.ProtoMajor == major && r.ProtoMinor >= minor } -// Write writes r to w in the HTTP/1.n server response format, +// Write writes r to w in the HTTP/1.x server response format, // including the status line, headers, body, and optional trailer. // // This method consults the following fields of the response r: @@ -228,11 +238,13 @@ func (r *Response) Write(w io.Writer) error { if !ok { text = "status code " + strconv.Itoa(r.StatusCode) } + } else { + // Just to reduce stutter, if user set r.Status to "200 OK" and StatusCode to 200. + // Not important. + text = strings.TrimPrefix(text, strconv.Itoa(r.StatusCode)+" ") } - protoMajor, protoMinor := strconv.Itoa(r.ProtoMajor), strconv.Itoa(r.ProtoMinor) - statusCode := strconv.Itoa(r.StatusCode) + " " - text = strings.TrimPrefix(text, statusCode) - if _, err := io.WriteString(w, "HTTP/"+protoMajor+"."+protoMinor+" "+statusCode+text+"\r\n"); err != nil { + + if _, err := fmt.Fprintf(w, "HTTP/%d.%d %03d %s\r\n", r.ProtoMajor, r.ProtoMinor, r.StatusCode, text); err != nil { return err } @@ -249,7 +261,7 @@ func (r *Response) Write(w io.Writer) error { if n == 0 { // Reset it to a known zero reader, in case underlying one // is unhappy being read repeatedly. - r1.Body = eofReader + r1.Body = NoBody } else { r1.ContentLength = -1 r1.Body = struct { @@ -265,7 +277,7 @@ func (r *Response) Write(w io.Writer) error { // content-length, the only way to do that is the old HTTP/1.0 // way, by noting the EOF with a connection close, so we need // to set Close. - if r1.ContentLength == -1 && !r1.Close && r1.ProtoAtLeast(1, 1) && !chunked(r1.TransferEncoding) { + if r1.ContentLength == -1 && !r1.Close && r1.ProtoAtLeast(1, 1) && !chunked(r1.TransferEncoding) && !r1.Uncompressed { r1.Close = true } @@ -288,7 +300,7 @@ func (r *Response) Write(w io.Writer) error { // contentLengthAlreadySent may have been already sent for // POST/PUT requests, even if zero length. See Issue 8180. contentLengthAlreadySent := tw.shouldSendContentLength() - if r1.ContentLength == 0 && !chunked(r1.TransferEncoding) && !contentLengthAlreadySent { + if r1.ContentLength == 0 && !chunked(r1.TransferEncoding) && !contentLengthAlreadySent && bodyAllowedForStatus(r.StatusCode) { if _, err := io.WriteString(w, "Content-Length: 0\r\n"); err != nil { return err } diff --git a/libgo/go/net/http/response_test.go b/libgo/go/net/http/response_test.go index d8a53400cf..660d51791b 100644 --- a/libgo/go/net/http/response_test.go +++ b/libgo/go/net/http/response_test.go @@ -1,4 +1,4 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -10,6 +10,7 @@ import ( "compress/gzip" "crypto/rand" "fmt" + "go/ast" "io" "io/ioutil" "net/http/internal" @@ -505,6 +506,32 @@ some body`, "Body here\n", }, + + { + "HTTP/1.1 200 OK\r\n" + + "Content-Encoding: gzip\r\n" + + "Content-Length: 23\r\n" + + "Connection: keep-alive\r\n" + + "Keep-Alive: timeout=7200\r\n\r\n" + + "\x1f\x8b\b\x00\x00\x00\x00\x00\x00\x00s\xf3\xf7\a\x00\xab'\xd4\x1a\x03\x00\x00\x00", + Response{ + Status: "200 OK", + StatusCode: 200, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Request: dummyReq("GET"), + Header: Header{ + "Content-Length": {"23"}, + "Content-Encoding": {"gzip"}, + "Connection": {"keep-alive"}, + "Keep-Alive": {"timeout=7200"}, + }, + Close: false, + ContentLength: 23, + }, + "\x1f\x8b\b\x00\x00\x00\x00\x00\x00\x00s\xf3\xf7\a\x00\xab'\xd4\x1a\x03\x00\x00\x00", + }, } // tests successful calls to ReadResponse, and inspects the returned Response. @@ -562,6 +589,7 @@ var readResponseCloseInMiddleTests = []struct { // reading only part of its contents advances the read to the end of // the request, right up until the next request. func TestReadResponseCloseInMiddle(t *testing.T) { + t.Parallel() for _, test := range readResponseCloseInMiddleTests { fatalf := func(format string, args ...interface{}) { args = append([]interface{}{test.chunked, test.compressed}, args...) @@ -656,10 +684,14 @@ func diff(t *testing.T, prefix string, have, want interface{}) { t.Errorf("%s: type mismatch %v want %v", prefix, hv.Type(), wv.Type()) } for i := 0; i < hv.NumField(); i++ { + name := hv.Type().Field(i).Name + if !ast.IsExported(name) { + continue + } hf := hv.Field(i).Interface() wf := wv.Field(i).Interface() if !reflect.DeepEqual(hf, wf) { - t.Errorf("%s: %s = %v want %v", prefix, hv.Type().Field(i).Name, hf, wf) + t.Errorf("%s: %s = %v want %v", prefix, name, hf, wf) } } } @@ -761,6 +793,7 @@ func TestReadResponseErrors(t *testing.T) { type testCase struct { name string // optional, defaults to in in string + header Header wantErr interface{} // nil, err value, or string substring } @@ -786,11 +819,22 @@ func TestReadResponseErrors(t *testing.T) { } } + contentLength := func(status, body string, wantErr interface{}, header Header) testCase { + return testCase{ + name: fmt.Sprintf("status %q %q", status, body), + in: fmt.Sprintf("HTTP/1.1 %s\r\n%s", status, body), + wantErr: wantErr, + header: header, + } + } + + errMultiCL := "message cannot contain multiple Content-Length headers" + tests := []testCase{ - {"", "", io.ErrUnexpectedEOF}, - {"", "HTTP/1.1 301 Moved Permanently\r\nFoo: bar", io.ErrUnexpectedEOF}, - {"", "HTTP/1.1", "malformed HTTP response"}, - {"", "HTTP/2.0", "malformed HTTP response"}, + {"", "", nil, io.ErrUnexpectedEOF}, + {"", "HTTP/1.1 301 Moved Permanently\r\nFoo: bar", nil, io.ErrUnexpectedEOF}, + {"", "HTTP/1.1", nil, "malformed HTTP response"}, + {"", "HTTP/2.0", nil, "malformed HTTP response"}, status("20X Unknown", true), status("abcd Unknown", true), status("二百/两百 OK", true), @@ -815,7 +859,21 @@ func TestReadResponseErrors(t *testing.T) { version("HTTP/A.B", true), version("HTTP/1", true), version("http/1.1", true), + + contentLength("200 OK", "Content-Length: 10\r\nContent-Length: 7\r\n\r\nGopher hey\r\n", errMultiCL, nil), + contentLength("200 OK", "Content-Length: 7\r\nContent-Length: 7\r\n\r\nGophers\r\n", nil, Header{"Content-Length": {"7"}}), + contentLength("201 OK", "Content-Length: 0\r\nContent-Length: 7\r\n\r\nGophers\r\n", errMultiCL, nil), + contentLength("300 OK", "Content-Length: 0\r\nContent-Length: 0 \r\n\r\nGophers\r\n", nil, Header{"Content-Length": {"0"}}), + contentLength("200 OK", "Content-Length:\r\nContent-Length:\r\n\r\nGophers\r\n", nil, nil), + contentLength("206 OK", "Content-Length:\r\nContent-Length: 0 \r\nConnection: close\r\n\r\nGophers\r\n", errMultiCL, nil), + + // multiple content-length headers for 204 and 304 should still be checked + contentLength("204 OK", "Content-Length: 7\r\nContent-Length: 8\r\n\r\n", errMultiCL, nil), + contentLength("204 OK", "Content-Length: 3\r\nContent-Length: 3\r\n\r\n", nil, nil), + contentLength("304 OK", "Content-Length: 880\r\nContent-Length: 1\r\n\r\n", errMultiCL, nil), + contentLength("304 OK", "Content-Length: 961\r\nContent-Length: 961\r\n\r\n", nil, nil), } + for i, tt := range tests { br := bufio.NewReader(strings.NewReader(tt.in)) _, rerr := ReadResponse(br, nil) diff --git a/libgo/go/net/http/responsewrite_test.go b/libgo/go/net/http/responsewrite_test.go index 5b8d47ab58..d41d89896e 100644 --- a/libgo/go/net/http/responsewrite_test.go +++ b/libgo/go/net/http/responsewrite_test.go @@ -1,4 +1,4 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -222,6 +222,56 @@ func TestResponseWrite(t *testing.T) { }, "HTTP/1.1 200 OK\r\nConnection: close\r\n\r\nabcdef", }, + + // Status code under 100 should be zero-padded to + // three digits. Still bogus, but less bogus. (be + // consistent with generating three digits, since the + // Transport requires it) + { + Response{ + StatusCode: 7, + Status: "license to violate specs", + ProtoMajor: 1, + ProtoMinor: 0, + Request: dummyReq("GET"), + Header: Header{}, + Body: nil, + }, + + "HTTP/1.0 007 license to violate specs\r\nContent-Length: 0\r\n\r\n", + }, + + // No stutter. Status code in 1xx range response should + // not include a Content-Length header. See issue #16942. + { + Response{ + StatusCode: 123, + Status: "123 Sesame Street", + ProtoMajor: 1, + ProtoMinor: 0, + Request: dummyReq("GET"), + Header: Header{}, + Body: nil, + }, + + "HTTP/1.0 123 Sesame Street\r\n\r\n", + }, + + // Status code 204 (No content) response should not include a + // Content-Length header. See issue #16942. + { + Response{ + StatusCode: 204, + Status: "No Content", + ProtoMajor: 1, + ProtoMinor: 0, + Request: dummyReq("GET"), + Header: Header{}, + Body: nil, + }, + + "HTTP/1.0 204 No Content\r\n\r\n", + }, } for i := range respWriteTests { diff --git a/libgo/go/net/http/serve_test.go b/libgo/go/net/http/serve_test.go index 384b453ce0..73dd56e8c4 100644 --- a/libgo/go/net/http/serve_test.go +++ b/libgo/go/net/http/serve_test.go @@ -9,7 +9,10 @@ package http_test import ( "bufio" "bytes" + "compress/gzip" + "context" "crypto/tls" + "encoding/json" "errors" "fmt" "internal/testenv" @@ -153,6 +156,7 @@ func (ht handlerTest) rawResponse(req string) string { } func TestConsumingBodyOnNextConn(t *testing.T) { + t.Parallel() defer afterTest(t) conn := new(testConn) for i := 0; i < 2; i++ { @@ -234,6 +238,7 @@ var vtests = []struct { } func TestHostHandlers(t *testing.T) { + setParallel(t) defer afterTest(t) mux := NewServeMux() for _, h := range handlers { @@ -350,6 +355,7 @@ var serveMuxTests = []struct { } func TestServeMuxHandler(t *testing.T) { + setParallel(t) mux := NewServeMux() for _, e := range serveMuxRegister { mux.Handle(e.pattern, e.h) @@ -387,15 +393,16 @@ var serveMuxTests2 = []struct { // TestServeMuxHandlerRedirects tests that automatic redirects generated by // mux.Handler() shouldn't clear the request's query string. func TestServeMuxHandlerRedirects(t *testing.T) { + setParallel(t) mux := NewServeMux() for _, e := range serveMuxRegister { mux.Handle(e.pattern, e.h) } for _, tt := range serveMuxTests2 { - tries := 1 + tries := 1 // expect at most 1 redirection if redirOk is true. turl := tt.url - for tries > 0 { + for { u, e := url.Parse(turl) if e != nil { t.Fatal(e) @@ -429,6 +436,7 @@ func TestServeMuxHandlerRedirects(t *testing.T) { // Tests for https://golang.org/issue/900 func TestMuxRedirectLeadingSlashes(t *testing.T) { + setParallel(t) paths := []string{"//foo.txt", "///foo.txt", "/../../foo.txt"} for _, path := range paths { req, err := ReadRequest(bufio.NewReader(strings.NewReader("GET " + path + " HTTP/1.1\r\nHost: test\r\n\r\n"))) @@ -453,9 +461,6 @@ func TestMuxRedirectLeadingSlashes(t *testing.T) { } func TestServerTimeouts(t *testing.T) { - if runtime.GOOS == "plan9" { - t.Skip("skipping test; see https://golang.org/issue/7237") - } setParallel(t) defer afterTest(t) reqNum := 0 @@ -476,11 +481,11 @@ func TestServerTimeouts(t *testing.T) { if err != nil { t.Fatalf("http Get #1: %v", err) } - got, _ := ioutil.ReadAll(r.Body) + got, err := ioutil.ReadAll(r.Body) expected := "req=1" - if string(got) != expected { - t.Errorf("Unexpected response for request #1; got %q; expected %q", - string(got), expected) + if string(got) != expected || err != nil { + t.Errorf("Unexpected response for request #1; got %q ,%v; expected %q, nil", + string(got), err, expected) } // Slow client that should timeout. @@ -491,6 +496,7 @@ func TestServerTimeouts(t *testing.T) { } buf := make([]byte, 1) n, err := conn.Read(buf) + conn.Close() latency := time.Since(t1) if n != 0 || err != io.EOF { t.Errorf("Read = %v, %v, wanted %v, %v", n, err, 0, io.EOF) @@ -502,14 +508,14 @@ func TestServerTimeouts(t *testing.T) { // Hit the HTTP server successfully again, verifying that the // previous slow connection didn't run our handler. (that we // get "req=2", not "req=3") - r, err = Get(ts.URL) + r, err = c.Get(ts.URL) if err != nil { t.Fatalf("http Get #2: %v", err) } - got, _ = ioutil.ReadAll(r.Body) + got, err = ioutil.ReadAll(r.Body) expected = "req=2" - if string(got) != expected { - t.Errorf("Get #2 got %q, want %q", string(got), expected) + if string(got) != expected || err != nil { + t.Errorf("Get #2 got %q, %v, want %q, nil", string(got), err, expected) } if !testing.Short() { @@ -529,13 +535,61 @@ func TestServerTimeouts(t *testing.T) { } } +// Test that the HTTP/2 server handles Server.WriteTimeout (Issue 18437) +func TestHTTP2WriteDeadlineExtendedOnNewRequest(t *testing.T) { + if testing.Short() { + t.Skip("skipping in short mode") + } + setParallel(t) + defer afterTest(t) + ts := httptest.NewUnstartedServer(HandlerFunc(func(res ResponseWriter, req *Request) {})) + ts.Config.WriteTimeout = 250 * time.Millisecond + ts.TLS = &tls.Config{NextProtos: []string{"h2"}} + ts.StartTLS() + defer ts.Close() + + tr := newTLSTransport(t, ts) + defer tr.CloseIdleConnections() + if err := ExportHttp2ConfigureTransport(tr); err != nil { + t.Fatal(err) + } + c := &Client{Transport: tr} + + for i := 1; i <= 3; i++ { + req, err := NewRequest("GET", ts.URL, nil) + if err != nil { + t.Fatal(err) + } + + // fail test if no response after 1 second + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + req = req.WithContext(ctx) + + r, err := c.Do(req) + select { + case <-ctx.Done(): + if ctx.Err() == context.DeadlineExceeded { + t.Fatalf("http2 Get #%d response timed out", i) + } + default: + } + if err != nil { + t.Fatalf("http2 Get #%d: %v", i, err) + } + r.Body.Close() + if r.ProtoMajor != 2 { + t.Fatalf("http2 Get expected HTTP/2.0, got %q", r.Proto) + } + time.Sleep(ts.Config.WriteTimeout / 2) + } +} + // golang.org/issue/4741 -- setting only a write timeout that triggers // shouldn't cause a handler to block forever on reads (next HTTP // request) that will never happen. func TestOnlyWriteTimeout(t *testing.T) { - if runtime.GOOS == "plan9" { - t.Skip("skipping test; see https://golang.org/issue/7237") - } + setParallel(t) defer afterTest(t) var conn net.Conn var afterTimeoutErrc = make(chan error, 1) @@ -595,6 +649,7 @@ func (l trackLastConnListener) Accept() (c net.Conn, err error) { // TestIdentityResponse verifies that a handler can unset func TestIdentityResponse(t *testing.T) { + setParallel(t) defer afterTest(t) handler := HandlerFunc(func(rw ResponseWriter, req *Request) { rw.Header().Set("Content-Length", "3") @@ -616,13 +671,16 @@ func TestIdentityResponse(t *testing.T) { ts := httptest.NewServer(handler) defer ts.Close() + c := &Client{Transport: new(Transport)} + defer closeClient(c) + // Note: this relies on the assumption (which is true) that - // Get sends HTTP/1.1 or greater requests. Otherwise the + // Get sends HTTP/1.1 or greater requests. Otherwise the // server wouldn't have the choice to send back chunked // responses. for _, te := range []string{"", "identity"} { url := ts.URL + "/?te=" + te - res, err := Get(url) + res, err := c.Get(url) if err != nil { t.Fatalf("error with Get of %s: %v", url, err) } @@ -641,7 +699,7 @@ func TestIdentityResponse(t *testing.T) { // Verify that ErrContentLength is returned url := ts.URL + "/?overwrite=1" - res, err := Get(url) + res, err := c.Get(url) if err != nil { t.Fatalf("error with Get of %s: %v", url, err) } @@ -671,6 +729,7 @@ func TestIdentityResponse(t *testing.T) { } func testTCPConnectionCloses(t *testing.T, req string, h Handler) { + setParallel(t) defer afterTest(t) s := httptest.NewServer(h) defer s.Close() @@ -713,6 +772,32 @@ func testTCPConnectionCloses(t *testing.T, req string, h Handler) { } } +func testTCPConnectionStaysOpen(t *testing.T, req string, handler Handler) { + setParallel(t) + defer afterTest(t) + ts := httptest.NewServer(handler) + defer ts.Close() + conn, err := net.Dial("tcp", ts.Listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + br := bufio.NewReader(conn) + for i := 0; i < 2; i++ { + if _, err := io.WriteString(conn, req); err != nil { + t.Fatal(err) + } + res, err := ReadResponse(br, nil) + if err != nil { + t.Fatalf("res %d: %v", i+1, err) + } + if _, err := io.Copy(ioutil.Discard, res.Body); err != nil { + t.Fatalf("res %d body copy: %v", i+1, err) + } + res.Body.Close() + } +} + // TestServeHTTP10Close verifies that HTTP/1.0 requests won't be kept alive. func TestServeHTTP10Close(t *testing.T) { testTCPConnectionCloses(t, "GET / HTTP/1.0\r\n\r\n", HandlerFunc(func(w ResponseWriter, r *Request) { @@ -722,7 +807,7 @@ func TestServeHTTP10Close(t *testing.T) { // TestClientCanClose verifies that clients can also force a connection to close. func TestClientCanClose(t *testing.T) { - testTCPConnectionCloses(t, "GET / HTTP/1.1\r\nConnection: close\r\n\r\n", HandlerFunc(func(w ResponseWriter, r *Request) { + testTCPConnectionCloses(t, "GET / HTTP/1.1\r\nHost: foo\r\nConnection: close\r\n\r\n", HandlerFunc(func(w ResponseWriter, r *Request) { // Nothing. })) } @@ -730,7 +815,7 @@ func TestClientCanClose(t *testing.T) { // TestHandlersCanSetConnectionClose verifies that handlers can force a connection to close, // even for HTTP/1.1 requests. func TestHandlersCanSetConnectionClose11(t *testing.T) { - testTCPConnectionCloses(t, "GET / HTTP/1.1\r\n\r\n", HandlerFunc(func(w ResponseWriter, r *Request) { + testTCPConnectionCloses(t, "GET / HTTP/1.1\r\nHost: foo\r\n\r\n\r\n", HandlerFunc(func(w ResponseWriter, r *Request) { w.Header().Set("Connection", "close") })) } @@ -741,10 +826,67 @@ func TestHandlersCanSetConnectionClose10(t *testing.T) { })) } +func TestHTTP2UpgradeClosesConnection(t *testing.T) { + testTCPConnectionCloses(t, "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n", HandlerFunc(func(w ResponseWriter, r *Request) { + // Nothing. (if not hijacked, the server should close the connection + // afterwards) + })) +} + +func send204(w ResponseWriter, r *Request) { w.WriteHeader(204) } +func send304(w ResponseWriter, r *Request) { w.WriteHeader(304) } + +// Issue 15647: 204 responses can't have bodies, so HTTP/1.0 keep-alive conns should stay open. +func TestHTTP10KeepAlive204Response(t *testing.T) { + testTCPConnectionStaysOpen(t, "GET / HTTP/1.0\r\nConnection: keep-alive\r\n\r\n", HandlerFunc(send204)) +} + +func TestHTTP11KeepAlive204Response(t *testing.T) { + testTCPConnectionStaysOpen(t, "GET / HTTP/1.1\r\nHost: foo\r\n\r\n", HandlerFunc(send204)) +} + +func TestHTTP10KeepAlive304Response(t *testing.T) { + testTCPConnectionStaysOpen(t, + "GET / HTTP/1.0\r\nConnection: keep-alive\r\nIf-Modified-Since: Mon, 02 Jan 2006 15:04:05 GMT\r\n\r\n", + HandlerFunc(send304)) +} + +// Issue 15703 +func TestKeepAliveFinalChunkWithEOF(t *testing.T) { + setParallel(t) + defer afterTest(t) + cst := newClientServerTest(t, false /* h1 */, HandlerFunc(func(w ResponseWriter, r *Request) { + w.(Flusher).Flush() // force chunked encoding + w.Write([]byte("{\"Addr\": \"" + r.RemoteAddr + "\"}")) + })) + defer cst.close() + type data struct { + Addr string + } + var addrs [2]data + for i := range addrs { + res, err := cst.c.Get(cst.ts.URL) + if err != nil { + t.Fatal(err) + } + if err := json.NewDecoder(res.Body).Decode(&addrs[i]); err != nil { + t.Fatal(err) + } + if addrs[i].Addr == "" { + t.Fatal("no address") + } + res.Body.Close() + } + if addrs[0] != addrs[1] { + t.Fatalf("connection not reused") + } +} + func TestSetsRemoteAddr_h1(t *testing.T) { testSetsRemoteAddr(t, h1Mode) } func TestSetsRemoteAddr_h2(t *testing.T) { testSetsRemoteAddr(t, h2Mode) } func testSetsRemoteAddr(t *testing.T, h2 bool) { + setParallel(t) defer afterTest(t) cst := newClientServerTest(t, h2, HandlerFunc(func(w ResponseWriter, r *Request) { fmt.Fprintf(w, "%s", r.RemoteAddr) @@ -794,6 +936,7 @@ func (c *blockingRemoteAddrConn) RemoteAddr() net.Addr { // Issue 12943 func TestServerAllowsBlockingRemoteAddr(t *testing.T) { + setParallel(t) defer afterTest(t) ts := httptest.NewUnstartedServer(HandlerFunc(func(w ResponseWriter, r *Request) { fmt.Fprintf(w, "RA:%s", r.RemoteAddr) @@ -865,7 +1008,9 @@ func TestServerAllowsBlockingRemoteAddr(t *testing.T) { t.Fatalf("response 1 addr = %q; want %q", g, e) } } + func TestIdentityResponseHeaders(t *testing.T) { + // Not parallel; changes log output. defer afterTest(t) log.SetOutput(ioutil.Discard) // is noisy otherwise defer log.SetOutput(os.Stderr) @@ -877,7 +1022,10 @@ func TestIdentityResponseHeaders(t *testing.T) { })) defer ts.Close() - res, err := Get(ts.URL) + c := &Client{Transport: new(Transport)} + defer closeClient(c) + + res, err := c.Get(ts.URL) if err != nil { t.Fatalf("Get error: %v", err) } @@ -900,6 +1048,7 @@ func TestHeadResponses_h1(t *testing.T) { testHeadResponses(t, h1Mode) } func TestHeadResponses_h2(t *testing.T) { testHeadResponses(t, h2Mode) } func testHeadResponses(t *testing.T, h2 bool) { + setParallel(t) defer afterTest(t) cst := newClientServerTest(t, h2, HandlerFunc(func(w ResponseWriter, r *Request) { _, err := w.Write([]byte("<html>")) @@ -937,9 +1086,6 @@ func testHeadResponses(t *testing.T, h2 bool) { } func TestTLSHandshakeTimeout(t *testing.T) { - if runtime.GOOS == "plan9" { - t.Skip("skipping test; see https://golang.org/issue/7237") - } setParallel(t) defer afterTest(t) ts := httptest.NewUnstartedServer(HandlerFunc(func(w ResponseWriter, r *Request) {})) @@ -971,6 +1117,7 @@ func TestTLSHandshakeTimeout(t *testing.T) { } func TestTLSServer(t *testing.T) { + setParallel(t) defer afterTest(t) ts := httptest.NewTLSServer(HandlerFunc(func(w ResponseWriter, r *Request) { if r.TLS != nil { @@ -984,7 +1131,7 @@ func TestTLSServer(t *testing.T) { defer ts.Close() // Connect an idle TCP connection to this server before we run - // our real tests. This idle connection used to block forever + // our real tests. This idle connection used to block forever // in the TLS handshake, preventing future connections from // being accepted. It may prevent future accidental blocking // in newConn. @@ -1024,11 +1171,46 @@ func TestTLSServer(t *testing.T) { }) } -func TestAutomaticHTTP2_Serve(t *testing.T) { +// Issue 15908 +func TestAutomaticHTTP2_Serve_NoTLSConfig(t *testing.T) { + testAutomaticHTTP2_Serve(t, nil, true) +} + +func TestAutomaticHTTP2_Serve_NonH2TLSConfig(t *testing.T) { + testAutomaticHTTP2_Serve(t, &tls.Config{}, false) +} + +func TestAutomaticHTTP2_Serve_H2TLSConfig(t *testing.T) { + testAutomaticHTTP2_Serve(t, &tls.Config{NextProtos: []string{"h2"}}, true) +} + +func testAutomaticHTTP2_Serve(t *testing.T, tlsConf *tls.Config, wantH2 bool) { + setParallel(t) defer afterTest(t) ln := newLocalListener(t) ln.Close() // immediately (not a defer!) var s Server + s.TLSConfig = tlsConf + if err := s.Serve(ln); err == nil { + t.Fatal("expected an error") + } + gotH2 := s.TLSNextProto["h2"] != nil + if gotH2 != wantH2 { + t.Errorf("http2 configured = %v; want %v", gotH2, wantH2) + } +} + +func TestAutomaticHTTP2_Serve_WithTLSConfig(t *testing.T) { + setParallel(t) + defer afterTest(t) + ln := newLocalListener(t) + ln.Close() // immediately (not a defer!) + var s Server + // Set the TLSConfig. In reality, this would be the + // *tls.Config given to tls.NewListener. + s.TLSConfig = &tls.Config{ + NextProtos: []string{"h2"}, + } if err := s.Serve(ln); err == nil { t.Fatal("expected an error") } @@ -1061,6 +1243,7 @@ func TestAutomaticHTTP2_ListenAndServe_GetCertificate(t *testing.T) { } func testAutomaticHTTP2_ListenAndServe(t *testing.T, tlsConf *tls.Config) { + // Not parallel: uses global test hooks. defer afterTest(t) defer SetTestHookServerServe(nil) var ok bool @@ -1164,6 +1347,7 @@ var serverExpectTests = []serverExpectTest{ // correctly. // http2 test: TestServer_Response_Automatic100Continue func TestServerExpect(t *testing.T) { + setParallel(t) defer afterTest(t) ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { // Note using r.FormValue("readbody") because for POST @@ -1257,6 +1441,7 @@ func TestServerExpect(t *testing.T) { // Under a ~256KB (maxPostHandlerReadBytes) threshold, the server // should consume client request bodies that a handler didn't read. func TestServerUnreadRequestBodyLittle(t *testing.T) { + setParallel(t) defer afterTest(t) conn := new(testConn) body := strings.Repeat("x", 100<<10) @@ -1297,6 +1482,7 @@ func TestServerUnreadRequestBodyLittle(t *testing.T) { // should ignore client request bodies that a handler didn't read // and close the connection. func TestServerUnreadRequestBodyLarge(t *testing.T) { + setParallel(t) if testing.Short() && testenv.Builder() == "" { t.Log("skipping in short mode") } @@ -1430,6 +1616,7 @@ var handlerBodyCloseTests = [...]handlerBodyCloseTest{ } func TestHandlerBodyClose(t *testing.T) { + setParallel(t) if testing.Short() && testenv.Builder() == "" { t.Skip("skipping in -short mode") } @@ -1509,6 +1696,7 @@ var testHandlerBodyConsumers = []testHandlerBodyConsumer{ } func TestRequestBodyReadErrorClosesConnection(t *testing.T) { + setParallel(t) defer afterTest(t) for _, handler := range testHandlerBodyConsumers { conn := new(testConn) @@ -1539,6 +1727,7 @@ func TestRequestBodyReadErrorClosesConnection(t *testing.T) { } func TestInvalidTrailerClosesConnection(t *testing.T) { + setParallel(t) defer afterTest(t) for _, handler := range testHandlerBodyConsumers { conn := new(testConn) @@ -1621,7 +1810,7 @@ restart: if !c.rd.IsZero() { // If the deadline falls in the middle of our sleep window, deduct // part of the sleep, then return a timeout. - if remaining := c.rd.Sub(time.Now()); remaining < cue { + if remaining := time.Until(c.rd); remaining < cue { c.script[0] = cue - remaining time.Sleep(remaining) return 0, syscall.ETIMEDOUT @@ -1707,6 +1896,7 @@ func TestRequestBodyTimeoutClosesConnection(t *testing.T) { func TestTimeoutHandler_h1(t *testing.T) { testTimeoutHandler(t, h1Mode) } func TestTimeoutHandler_h2(t *testing.T) { testTimeoutHandler(t, h2Mode) } func testTimeoutHandler(t *testing.T, h2 bool) { + setParallel(t) defer afterTest(t) sendHi := make(chan bool, 1) writeErrors := make(chan error, 1) @@ -1760,6 +1950,7 @@ func testTimeoutHandler(t *testing.T, h2 bool) { // See issues 8209 and 8414. func TestTimeoutHandlerRace(t *testing.T) { + setParallel(t) defer afterTest(t) delayHi := HandlerFunc(func(w ResponseWriter, r *Request) { @@ -1776,6 +1967,9 @@ func TestTimeoutHandlerRace(t *testing.T) { ts := httptest.NewServer(TimeoutHandler(delayHi, 20*time.Millisecond, "")) defer ts.Close() + c := &Client{Transport: new(Transport)} + defer closeClient(c) + var wg sync.WaitGroup gate := make(chan bool, 10) n := 50 @@ -1789,7 +1983,7 @@ func TestTimeoutHandlerRace(t *testing.T) { go func() { defer wg.Done() defer func() { <-gate }() - res, err := Get(fmt.Sprintf("%s/%d", ts.URL, rand.Intn(50))) + res, err := c.Get(fmt.Sprintf("%s/%d", ts.URL, rand.Intn(50))) if err == nil { io.Copy(ioutil.Discard, res.Body) res.Body.Close() @@ -1801,6 +1995,7 @@ func TestTimeoutHandlerRace(t *testing.T) { // See issues 8209 and 8414. func TestTimeoutHandlerRaceHeader(t *testing.T) { + setParallel(t) defer afterTest(t) delay204 := HandlerFunc(func(w ResponseWriter, r *Request) { @@ -1816,13 +2011,15 @@ func TestTimeoutHandlerRaceHeader(t *testing.T) { if testing.Short() { n = 10 } + c := &Client{Transport: new(Transport)} + defer closeClient(c) for i := 0; i < n; i++ { gate <- true wg.Add(1) go func() { defer wg.Done() defer func() { <-gate }() - res, err := Get(ts.URL) + res, err := c.Get(ts.URL) if err != nil { t.Error(err) return @@ -1836,6 +2033,7 @@ func TestTimeoutHandlerRaceHeader(t *testing.T) { // Issue 9162 func TestTimeoutHandlerRaceHeaderTimeout(t *testing.T) { + setParallel(t) defer afterTest(t) sendHi := make(chan bool, 1) writeErrors := make(chan error, 1) @@ -1888,20 +2086,57 @@ func TestTimeoutHandlerRaceHeaderTimeout(t *testing.T) { } } -// Verifies we don't path.Clean() on the wrong parts in redirects. -func TestRedirectMunging(t *testing.T) { - req, _ := NewRequest("GET", "http://example.com/", nil) +// Issue 14568. +func TestTimeoutHandlerStartTimerWhenServing(t *testing.T) { + if testing.Short() { + t.Skip("skipping sleeping test in -short mode") + } + defer afterTest(t) + var handler HandlerFunc = func(w ResponseWriter, _ *Request) { + w.WriteHeader(StatusNoContent) + } + timeout := 300 * time.Millisecond + ts := httptest.NewServer(TimeoutHandler(handler, timeout, "")) + defer ts.Close() + + c := &Client{Transport: new(Transport)} + defer closeClient(c) + + // Issue was caused by the timeout handler starting the timer when + // was created, not when the request. So wait for more than the timeout + // to ensure that's not the case. + time.Sleep(2 * timeout) + res, err := c.Get(ts.URL) + if err != nil { + t.Fatal(err) + } + defer res.Body.Close() + if res.StatusCode != StatusNoContent { + t.Errorf("got res.StatusCode %d, want %v", res.StatusCode, StatusNoContent) + } +} - resp := httptest.NewRecorder() - Redirect(resp, req, "/foo?next=http://bar.com/", 302) - if g, e := resp.Header().Get("Location"), "/foo?next=http://bar.com/"; g != e { - t.Errorf("Location header was %q; want %q", g, e) +// https://golang.org/issue/15948 +func TestTimeoutHandlerEmptyResponse(t *testing.T) { + setParallel(t) + defer afterTest(t) + var handler HandlerFunc = func(w ResponseWriter, _ *Request) { + // No response. } + timeout := 300 * time.Millisecond + ts := httptest.NewServer(TimeoutHandler(handler, timeout, "")) + defer ts.Close() + + c := &Client{Transport: new(Transport)} + defer closeClient(c) - resp = httptest.NewRecorder() - Redirect(resp, req, "http://localhost:8080/_ah/login?continue=http://localhost:8080/", 302) - if g, e := resp.Header().Get("Location"), "http://localhost:8080/_ah/login?continue=http://localhost:8080/"; g != e { - t.Errorf("Location header was %q; want %q", g, e) + res, err := c.Get(ts.URL) + if err != nil { + t.Fatal(err) + } + defer res.Body.Close() + if res.StatusCode != StatusOK { + t.Errorf("got res.StatusCode %d, want %v", res.StatusCode, StatusOK) } } @@ -1923,7 +2158,7 @@ func TestRedirectBadPath(t *testing.T) { } // Test different URL formats and schemes -func TestRedirectURLFormat(t *testing.T) { +func TestRedirect(t *testing.T) { req, _ := NewRequest("GET", "http://example.com/qux/", nil) var tests = []struct { @@ -1946,6 +2181,14 @@ func TestRedirectURLFormat(t *testing.T) { {"../quux/foobar.com/baz", "/quux/foobar.com/baz"}, // incorrect number of slashes {"///foobar.com/baz", "/foobar.com/baz"}, + + // Verifies we don't path.Clean() on the wrong parts in redirects: + {"/foo?next=http://bar.com/", "/foo?next=http://bar.com/"}, + {"http://localhost:8080/_ah/login?continue=http://localhost:8080/", + "http://localhost:8080/_ah/login?continue=http://localhost:8080/"}, + + {"/фубар", "/%d1%84%d1%83%d0%b1%d0%b0%d1%80"}, + {"http://foo.com/фубар", "http://foo.com/%d1%84%d1%83%d0%b1%d0%b0%d1%80"}, } for _, tt := range tests { @@ -1971,6 +2214,7 @@ func TestZeroLengthPostAndResponse_h2(t *testing.T) { } func testZeroLengthPostAndResponse(t *testing.T, h2 bool) { + setParallel(t) defer afterTest(t) cst := newClientServerTest(t, h2, HandlerFunc(func(rw ResponseWriter, r *Request) { all, err := ioutil.ReadAll(r.Body) @@ -2027,7 +2271,7 @@ func TestHandlerPanicWithHijack(t *testing.T) { func testHandlerPanic(t *testing.T, withHijack, h2 bool, panicValue interface{}) { defer afterTest(t) // Unlike the other tests that set the log output to ioutil.Discard - // to quiet the output, this test uses a pipe. The pipe serves three + // to quiet the output, this test uses a pipe. The pipe serves three // purposes: // // 1) The log.Print from the http server (generated by the caught @@ -2060,7 +2304,7 @@ func testHandlerPanic(t *testing.T, withHijack, h2 bool, panicValue interface{}) defer cst.close() // Do a blocking read on the log output pipe so its logging - // doesn't bleed into the next test. But wait only 5 seconds + // doesn't bleed into the next test. But wait only 5 seconds // for it. done := make(chan bool, 1) go func() { @@ -2090,12 +2334,58 @@ func testHandlerPanic(t *testing.T, withHijack, h2 bool, panicValue interface{}) } } +type terrorWriter struct{ t *testing.T } + +func (w terrorWriter) Write(p []byte) (int, error) { + w.t.Errorf("%s", p) + return len(p), nil +} + +// Issue 16456: allow writing 0 bytes on hijacked conn to test hijack +// without any log spam. +func TestServerWriteHijackZeroBytes(t *testing.T) { + defer afterTest(t) + done := make(chan struct{}) + ts := httptest.NewUnstartedServer(HandlerFunc(func(w ResponseWriter, r *Request) { + defer close(done) + w.(Flusher).Flush() + conn, _, err := w.(Hijacker).Hijack() + if err != nil { + t.Errorf("Hijack: %v", err) + return + } + defer conn.Close() + _, err = w.Write(nil) + if err != ErrHijacked { + t.Errorf("Write error = %v; want ErrHijacked", err) + } + })) + ts.Config.ErrorLog = log.New(terrorWriter{t}, "Unexpected write: ", 0) + ts.Start() + defer ts.Close() + + tr := &Transport{} + defer tr.CloseIdleConnections() + c := &Client{Transport: tr} + res, err := c.Get(ts.URL) + if err != nil { + t.Fatal(err) + } + res.Body.Close() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("timeout") + } +} + func TestServerNoDate_h1(t *testing.T) { testServerNoHeader(t, h1Mode, "Date") } func TestServerNoDate_h2(t *testing.T) { testServerNoHeader(t, h2Mode, "Date") } func TestServerNoContentType_h1(t *testing.T) { testServerNoHeader(t, h1Mode, "Content-Type") } func TestServerNoContentType_h2(t *testing.T) { testServerNoHeader(t, h2Mode, "Content-Type") } func testServerNoHeader(t *testing.T, h2 bool, header string) { + setParallel(t) defer afterTest(t) cst := newClientServerTest(t, h2, HandlerFunc(func(w ResponseWriter, r *Request) { w.Header()[header] = nil @@ -2113,6 +2403,7 @@ func testServerNoHeader(t *testing.T, h2 bool, header string) { } func TestStripPrefix(t *testing.T) { + setParallel(t) defer afterTest(t) h := HandlerFunc(func(w ResponseWriter, r *Request) { w.Header().Set("X-Path", r.URL.Path) @@ -2120,7 +2411,10 @@ func TestStripPrefix(t *testing.T) { ts := httptest.NewServer(StripPrefix("/foo", h)) defer ts.Close() - res, err := Get(ts.URL + "/foo/bar") + c := &Client{Transport: new(Transport)} + defer closeClient(c) + + res, err := c.Get(ts.URL + "/foo/bar") if err != nil { t.Fatal(err) } @@ -2142,10 +2436,11 @@ func TestStripPrefix(t *testing.T) { func TestRequestLimit_h1(t *testing.T) { testRequestLimit(t, h1Mode) } func TestRequestLimit_h2(t *testing.T) { testRequestLimit(t, h2Mode) } func testRequestLimit(t *testing.T, h2 bool) { + setParallel(t) defer afterTest(t) cst := newClientServerTest(t, h2, HandlerFunc(func(w ResponseWriter, r *Request) { t.Fatalf("didn't expect to get request in Handler") - })) + }), optQuietLog) defer cst.close() req, _ := NewRequest("GET", cst.ts.URL, nil) var bytesPerHeader = len("header12345: val12345\r\n") @@ -2188,6 +2483,7 @@ func (cr countReader) Read(p []byte) (n int, err error) { func TestRequestBodyLimit_h1(t *testing.T) { testRequestBodyLimit(t, h1Mode) } func TestRequestBodyLimit_h2(t *testing.T) { testRequestBodyLimit(t, h2Mode) } func testRequestBodyLimit(t *testing.T, h2 bool) { + setParallel(t) defer afterTest(t) const limit = 1 << 20 cst := newClientServerTest(t, h2, HandlerFunc(func(w ResponseWriter, r *Request) { @@ -2205,10 +2501,10 @@ func testRequestBodyLimit(t *testing.T, h2 bool) { nWritten := new(int64) req, _ := NewRequest("POST", cst.ts.URL, io.LimitReader(countReader{neverEnding('a'), nWritten}, limit*200)) - // Send the POST, but don't care it succeeds or not. The + // Send the POST, but don't care it succeeds or not. The // remote side is going to reply and then close the TCP // connection, and HTTP doesn't really define if that's - // allowed or not. Some HTTP clients will get the response + // allowed or not. Some HTTP clients will get the response // and some (like ours, currently) will complain that the // request write failed, without reading the response. // @@ -2237,14 +2533,14 @@ func TestClientWriteShutdown(t *testing.T) { } err = conn.(*net.TCPConn).CloseWrite() if err != nil { - t.Fatalf("Dial: %v", err) + t.Fatalf("CloseWrite: %v", err) } donec := make(chan bool) go func() { defer close(donec) bs, err := ioutil.ReadAll(conn) if err != nil { - t.Fatalf("ReadAll: %v", err) + t.Errorf("ReadAll: %v", err) } got := string(bs) if got != "" { @@ -2283,6 +2579,7 @@ func TestServerBufferedChunking(t *testing.T) { // closing the TCP connection, causing the client to get a RST. // See https://golang.org/issue/3595 func TestServerGracefulClose(t *testing.T) { + setParallel(t) defer afterTest(t) ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { Error(w, "bye", StatusUnauthorized) @@ -2395,7 +2692,8 @@ func TestCloseNotifier(t *testing.T) { go func() { _, err = fmt.Fprintf(conn, "GET / HTTP/1.1\r\nConnection: keep-alive\r\nHost: foo\r\n\r\n") if err != nil { - t.Fatal(err) + t.Error(err) + return } <-diec conn.Close() @@ -2437,7 +2735,8 @@ func TestCloseNotifierPipelined(t *testing.T) { const req = "GET / HTTP/1.1\r\nConnection: keep-alive\r\nHost: foo\r\n\r\n" _, err = io.WriteString(conn, req+req) // two requests if err != nil { - t.Fatal(err) + t.Error(err) + return } <-diec conn.Close() @@ -2545,6 +2844,7 @@ func TestHijackAfterCloseNotifier(t *testing.T) { } func TestHijackBeforeRequestBodyRead(t *testing.T) { + setParallel(t) defer afterTest(t) var requestBody = bytes.Repeat([]byte("a"), 1<<20) bodyOkay := make(chan bool, 1) @@ -2650,7 +2950,7 @@ func TestOptions(t *testing.T) { } // Tests regarding the ordering of Write, WriteHeader, Header, and -// Flush calls. In Go 1.0, rw.WriteHeader immediately flushed the +// Flush calls. In Go 1.0, rw.WriteHeader immediately flushed the // (*response).header to the wire. In Go 1.1, the actual wire flush is // delayed, so we could maybe tack on a Content-Length and better // Content-Type after we see more (or all) of the output. To preserve @@ -2866,15 +3166,18 @@ func (l *errorListener) Addr() net.Addr { } func TestAcceptMaxFds(t *testing.T) { - log.SetOutput(ioutil.Discard) // is noisy otherwise - defer log.SetOutput(os.Stderr) + setParallel(t) ln := &errorListener{[]error{ &net.OpError{ Op: "accept", Err: syscall.EMFILE, }}} - err := Serve(ln, HandlerFunc(HandlerFunc(func(ResponseWriter, *Request) {}))) + server := &Server{ + Handler: HandlerFunc(HandlerFunc(func(ResponseWriter, *Request) {})), + ErrorLog: log.New(ioutil.Discard, "", 0), // noisy otherwise + } + err := server.Serve(ln) if err != io.EOF { t.Errorf("got error %v, want EOF", err) } @@ -2999,6 +3302,7 @@ func TestHTTP10ConnectionHeader(t *testing.T) { func TestServerReaderFromOrder_h1(t *testing.T) { testServerReaderFromOrder(t, h1Mode) } func TestServerReaderFromOrder_h2(t *testing.T) { testServerReaderFromOrder(t, h2Mode) } func testServerReaderFromOrder(t *testing.T, h2 bool) { + setParallel(t) defer afterTest(t) pr, pw := io.Pipe() const size = 3 << 20 @@ -3103,11 +3407,12 @@ func TestTransportAndServerSharedBodyRace_h2(t *testing.T) { testTransportAndServerSharedBodyRace(t, h2Mode) } func testTransportAndServerSharedBodyRace(t *testing.T, h2 bool) { + setParallel(t) defer afterTest(t) const bodySize = 1 << 20 - // errorf is like t.Errorf, but also writes to println. When + // errorf is like t.Errorf, but also writes to println. When // this test fails, it hangs. This helps debugging and I've // added this enough times "temporarily". It now gets added // full time. @@ -3291,6 +3596,7 @@ func TestAppendTime(t *testing.T) { } func TestServerConnState(t *testing.T) { + setParallel(t) defer afterTest(t) handler := map[string]func(w ResponseWriter, r *Request){ "/": func(w ResponseWriter, r *Request) { @@ -3338,14 +3644,39 @@ func TestServerConnState(t *testing.T) { } ts.Start() - mustGet(t, ts.URL+"/") - mustGet(t, ts.URL+"/close") + tr := &Transport{} + defer tr.CloseIdleConnections() + c := &Client{Transport: tr} + + mustGet := func(url string, headers ...string) { + req, err := NewRequest("GET", url, nil) + if err != nil { + t.Fatal(err) + } + for len(headers) > 0 { + req.Header.Add(headers[0], headers[1]) + headers = headers[2:] + } + res, err := c.Do(req) + if err != nil { + t.Errorf("Error fetching %s: %v", url, err) + return + } + _, err = ioutil.ReadAll(res.Body) + defer res.Body.Close() + if err != nil { + t.Errorf("Error reading %s: %v", url, err) + } + } + + mustGet(ts.URL + "/") + mustGet(ts.URL + "/close") - mustGet(t, ts.URL+"/") - mustGet(t, ts.URL+"/", "Connection", "close") + mustGet(ts.URL + "/") + mustGet(ts.URL+"/", "Connection", "close") - mustGet(t, ts.URL+"/hijack") - mustGet(t, ts.URL+"/hijack-panic") + mustGet(ts.URL + "/hijack") + mustGet(ts.URL + "/hijack-panic") // New->Closed { @@ -3425,31 +3756,10 @@ func TestServerConnState(t *testing.T) { } mu.Lock() - t.Errorf("Unexpected events.\nGot log: %s\n Want: %s\n", logString(stateLog), logString(want)) + t.Errorf("Unexpected events.\nGot log:\n%s\n Want:\n%s\n", logString(stateLog), logString(want)) mu.Unlock() } -func mustGet(t *testing.T, url string, headers ...string) { - req, err := NewRequest("GET", url, nil) - if err != nil { - t.Fatal(err) - } - for len(headers) > 0 { - req.Header.Add(headers[0], headers[1]) - headers = headers[2:] - } - res, err := DefaultClient.Do(req) - if err != nil { - t.Errorf("Error fetching %s: %v", url, err) - return - } - _, err = ioutil.ReadAll(res.Body) - defer res.Body.Close() - if err != nil { - t.Errorf("Error reading %s: %v", url, err) - } -} - func TestServerKeepAlivesEnabled(t *testing.T) { defer afterTest(t) ts := httptest.NewUnstartedServer(HandlerFunc(func(w ResponseWriter, r *Request) {})) @@ -3470,6 +3780,7 @@ func TestServerKeepAlivesEnabled(t *testing.T) { func TestServerEmptyBodyRace_h1(t *testing.T) { testServerEmptyBodyRace(t, h1Mode) } func TestServerEmptyBodyRace_h2(t *testing.T) { testServerEmptyBodyRace(t, h2Mode) } func testServerEmptyBodyRace(t *testing.T, h2 bool) { + setParallel(t) defer afterTest(t) var n int32 cst := newClientServerTest(t, h2, HandlerFunc(func(rw ResponseWriter, req *Request) { @@ -3533,6 +3844,7 @@ func (c *closeWriteTestConn) CloseWrite() error { } func TestCloseWrite(t *testing.T) { + setParallel(t) var srv Server var testConn closeWriteTestConn c := ExportServerNewConn(&srv, &testConn) @@ -3773,6 +4085,7 @@ Host: foo // If a Handler finishes and there's an unread request body, // verify the server try to do implicit read on it before replying. func TestHandlerFinishSkipBigContentLengthRead(t *testing.T) { + setParallel(t) conn := &testConn{closec: make(chan bool)} conn.readBuf.Write([]byte(fmt.Sprintf( "POST / HTTP/1.1\r\n" + @@ -3829,6 +4142,8 @@ func TestServerValidatesHostHeader(t *testing.T) { host string want int }{ + {"HTTP/0.9", "", 400}, + {"HTTP/1.1", "", 400}, {"HTTP/1.1", "Host: \r\n", 200}, {"HTTP/1.1", "Host: 1.2.3.4\r\n", 200}, @@ -3851,13 +4166,29 @@ func TestServerValidatesHostHeader(t *testing.T) { {"HTTP/1.0", "", 200}, {"HTTP/1.0", "Host: first\r\nHost: second\r\n", 400}, {"HTTP/1.0", "Host: \xff\r\n", 400}, + + // Make an exception for HTTP upgrade requests: + {"PRI * HTTP/2.0", "", 200}, + + // But not other HTTP/2 stuff: + {"PRI / HTTP/2.0", "", 400}, + {"GET / HTTP/2.0", "", 400}, + {"GET / HTTP/3.0", "", 400}, } for _, tt := range tests { conn := &testConn{closec: make(chan bool, 1)} - io.WriteString(&conn.readBuf, "GET / "+tt.proto+"\r\n"+tt.host+"\r\n") + methodTarget := "GET / " + if !strings.HasPrefix(tt.proto, "HTTP/") { + methodTarget = "" + } + io.WriteString(&conn.readBuf, methodTarget+tt.proto+"\r\n"+tt.host+"\r\n") ln := &oneConnListener{conn} - go Serve(ln, HandlerFunc(func(ResponseWriter, *Request) {})) + srv := Server{ + ErrorLog: quietLog, + Handler: HandlerFunc(func(ResponseWriter, *Request) {}), + } + go srv.Serve(ln) <-conn.closec res, err := ReadResponse(bufio.NewReader(&conn.writeBuf), nil) if err != nil { @@ -3870,9 +4201,49 @@ func TestServerValidatesHostHeader(t *testing.T) { } } +func TestServerHandlersCanHandleH2PRI(t *testing.T) { + const upgradeResponse = "upgrade here" + defer afterTest(t) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + conn, br, err := w.(Hijacker).Hijack() + defer conn.Close() + if r.Method != "PRI" || r.RequestURI != "*" { + t.Errorf("Got method/target %q %q; want PRI *", r.Method, r.RequestURI) + return + } + if !r.Close { + t.Errorf("Request.Close = true; want false") + } + const want = "SM\r\n\r\n" + buf := make([]byte, len(want)) + n, err := io.ReadFull(br, buf) + if err != nil || string(buf[:n]) != want { + t.Errorf("Read = %v, %v (%q), want %q", n, err, buf[:n], want) + return + } + io.WriteString(conn, upgradeResponse) + })) + defer ts.Close() + + c, err := net.Dial("tcp", ts.Listener.Addr().String()) + if err != nil { + t.Fatalf("Dial: %v", err) + } + defer c.Close() + io.WriteString(c, "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n") + slurp, err := ioutil.ReadAll(c) + if err != nil { + t.Fatal(err) + } + if string(slurp) != upgradeResponse { + t.Errorf("Handler response = %q; want %q", slurp, upgradeResponse) + } +} + // Test that we validate the valid bytes in HTTP/1 headers. // Issue 11207. func TestServerValidatesHeaders(t *testing.T) { + setParallel(t) tests := []struct { header string want int @@ -3882,9 +4253,10 @@ func TestServerValidatesHeaders(t *testing.T) { {"X-Foo: bar\r\n", 200}, {"Foo: a space\r\n", 200}, - {"A space: foo\r\n", 400}, // space in header - {"foo\xffbar: foo\r\n", 400}, // binary in header - {"foo\x00bar: foo\r\n", 400}, // binary in header + {"A space: foo\r\n", 400}, // space in header + {"foo\xffbar: foo\r\n", 400}, // binary in header + {"foo\x00bar: foo\r\n", 400}, // binary in header + {"Foo: " + strings.Repeat("x", 1<<21) + "\r\n", 431}, // header too large {"foo: foo foo\r\n", 200}, // LWS space is okay {"foo: foo\tfoo\r\n", 200}, // LWS tab is okay @@ -3897,7 +4269,11 @@ func TestServerValidatesHeaders(t *testing.T) { io.WriteString(&conn.readBuf, "GET / HTTP/1.1\r\nHost: foo\r\n"+tt.header+"\r\n") ln := &oneConnListener{conn} - go Serve(ln, HandlerFunc(func(ResponseWriter, *Request) {})) + srv := Server{ + ErrorLog: quietLog, + Handler: HandlerFunc(func(ResponseWriter, *Request) {}), + } + go srv.Serve(ln) <-conn.closec res, err := ReadResponse(bufio.NewReader(&conn.writeBuf), nil) if err != nil { @@ -3910,6 +4286,143 @@ func TestServerValidatesHeaders(t *testing.T) { } } +func TestServerRequestContextCancel_ServeHTTPDone_h1(t *testing.T) { + testServerRequestContextCancel_ServeHTTPDone(t, h1Mode) +} +func TestServerRequestContextCancel_ServeHTTPDone_h2(t *testing.T) { + testServerRequestContextCancel_ServeHTTPDone(t, h2Mode) +} +func testServerRequestContextCancel_ServeHTTPDone(t *testing.T, h2 bool) { + setParallel(t) + defer afterTest(t) + ctxc := make(chan context.Context, 1) + cst := newClientServerTest(t, h2, HandlerFunc(func(w ResponseWriter, r *Request) { + ctx := r.Context() + select { + case <-ctx.Done(): + t.Error("should not be Done in ServeHTTP") + default: + } + ctxc <- ctx + })) + defer cst.close() + res, err := cst.c.Get(cst.ts.URL) + if err != nil { + t.Fatal(err) + } + res.Body.Close() + ctx := <-ctxc + select { + case <-ctx.Done(): + default: + t.Error("context should be done after ServeHTTP completes") + } +} + +// Tests that the Request.Context available to the Handler is canceled +// if the peer closes their TCP connection. This requires that the server +// is always blocked in a Read call so it notices the EOF from the client. +// See issues 15927 and 15224. +func TestServerRequestContextCancel_ConnClose(t *testing.T) { + setParallel(t) + defer afterTest(t) + inHandler := make(chan struct{}) + handlerDone := make(chan struct{}) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + close(inHandler) + select { + case <-r.Context().Done(): + case <-time.After(3 * time.Second): + t.Errorf("timeout waiting for context to be done") + } + close(handlerDone) + })) + defer ts.Close() + c, err := net.Dial("tcp", ts.Listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + defer c.Close() + io.WriteString(c, "GET / HTTP/1.1\r\nHost: foo\r\n\r\n") + select { + case <-inHandler: + case <-time.After(3 * time.Second): + t.Fatalf("timeout waiting to see ServeHTTP get called") + } + c.Close() // this should trigger the context being done + + select { + case <-handlerDone: + case <-time.After(4 * time.Second): + t.Fatalf("timeout waiting to see ServeHTTP exit") + } +} + +func TestServerContext_ServerContextKey_h1(t *testing.T) { + testServerContext_ServerContextKey(t, h1Mode) +} +func TestServerContext_ServerContextKey_h2(t *testing.T) { + testServerContext_ServerContextKey(t, h2Mode) +} +func testServerContext_ServerContextKey(t *testing.T, h2 bool) { + setParallel(t) + defer afterTest(t) + cst := newClientServerTest(t, h2, HandlerFunc(func(w ResponseWriter, r *Request) { + ctx := r.Context() + got := ctx.Value(ServerContextKey) + if _, ok := got.(*Server); !ok { + t.Errorf("context value = %T; want *http.Server", got) + } + + got = ctx.Value(LocalAddrContextKey) + if addr, ok := got.(net.Addr); !ok { + t.Errorf("local addr value = %T; want net.Addr", got) + } else if fmt.Sprint(addr) != r.Host { + t.Errorf("local addr = %v; want %v", addr, r.Host) + } + })) + defer cst.close() + res, err := cst.c.Get(cst.ts.URL) + if err != nil { + t.Fatal(err) + } + res.Body.Close() +} + +// https://golang.org/issue/15960 +func TestHandlerSetTransferEncodingChunked(t *testing.T) { + setParallel(t) + defer afterTest(t) + ht := newHandlerTest(HandlerFunc(func(w ResponseWriter, r *Request) { + w.Header().Set("Transfer-Encoding", "chunked") + w.Write([]byte("hello")) + })) + resp := ht.rawResponse("GET / HTTP/1.1\nHost: foo") + const hdr = "Transfer-Encoding: chunked" + if n := strings.Count(resp, hdr); n != 1 { + t.Errorf("want 1 occurrence of %q in response, got %v\nresponse: %v", hdr, n, resp) + } +} + +// https://golang.org/issue/16063 +func TestHandlerSetTransferEncodingGzip(t *testing.T) { + setParallel(t) + defer afterTest(t) + ht := newHandlerTest(HandlerFunc(func(w ResponseWriter, r *Request) { + w.Header().Set("Transfer-Encoding", "gzip") + gz := gzip.NewWriter(w) + gz.Write([]byte("hello")) + gz.Close() + })) + resp := ht.rawResponse("GET / HTTP/1.1\nHost: foo") + for _, v := range []string{"gzip", "chunked"} { + hdr := "Transfer-Encoding: " + v + if n := strings.Count(resp, hdr); n != 1 { + t.Errorf("want 1 occurrence of %q in response, got %v\nresponse: %v", hdr, n, resp) + } + } +} + func BenchmarkClientServer(b *testing.B) { b.ReportAllocs() b.StopTimer() @@ -4067,13 +4580,19 @@ func BenchmarkClient(b *testing.B) { b.StopTimer() defer afterTest(b) - port := os.Getenv("TEST_BENCH_SERVER_PORT") // can be set by user - if port == "" { - port = "39207" - } var data = []byte("Hello world.\n") if server := os.Getenv("TEST_BENCH_SERVER"); server != "" { // Server process mode. + port := os.Getenv("TEST_BENCH_SERVER_PORT") // can be set by user + if port == "" { + port = "0" + } + ln, err := net.Listen("tcp", "localhost:"+port) + if err != nil { + fmt.Fprintln(os.Stderr, err.Error()) + os.Exit(1) + } + fmt.Println(ln.Addr().String()) HandleFunc("/", func(w ResponseWriter, r *Request) { r.ParseForm() if r.Form.Get("stop") != "" { @@ -4082,33 +4601,44 @@ func BenchmarkClient(b *testing.B) { w.Header().Set("Content-Type", "text/html; charset=utf-8") w.Write(data) }) - log.Fatal(ListenAndServe("localhost:"+port, nil)) + var srv Server + log.Fatal(srv.Serve(ln)) } // Start server process. cmd := exec.Command(os.Args[0], "-test.run=XXXX", "-test.bench=BenchmarkClient$") cmd.Env = append(os.Environ(), "TEST_BENCH_SERVER=yes") + cmd.Stderr = os.Stderr + stdout, err := cmd.StdoutPipe() + if err != nil { + b.Fatal(err) + } if err := cmd.Start(); err != nil { b.Fatalf("subprocess failed to start: %v", err) } defer cmd.Process.Kill() + + // Wait for the server in the child process to respond and tell us + // its listening address, once it's started listening: + timer := time.AfterFunc(10*time.Second, func() { + cmd.Process.Kill() + }) + defer timer.Stop() + bs := bufio.NewScanner(stdout) + if !bs.Scan() { + b.Fatalf("failed to read listening URL from child: %v", bs.Err()) + } + url := "http://" + strings.TrimSpace(bs.Text()) + "/" + timer.Stop() + if _, err := getNoBody(url); err != nil { + b.Fatalf("initial probe of child process failed: %v", err) + } + done := make(chan error) go func() { done <- cmd.Wait() }() - // Wait for the server process to respond. - url := "http://localhost:" + port + "/" - for i := 0; i < 100; i++ { - time.Sleep(50 * time.Millisecond) - if _, err := getNoBody(url); err == nil { - break - } - if i == 99 { - b.Fatalf("subprocess does not respond") - } - } - // Do b.N requests to the server. b.StartTimer() for i := 0; i < b.N; i++ { @@ -4121,7 +4651,7 @@ func BenchmarkClient(b *testing.B) { if err != nil { b.Fatalf("ReadAll: %v", err) } - if bytes.Compare(body, data) != 0 { + if !bytes.Equal(body, data) { b.Fatalf("Got body: %q", body) } } @@ -4367,3 +4897,418 @@ func BenchmarkCloseNotifier(b *testing.B) { } b.StopTimer() } + +// Verify this doesn't race (Issue 16505) +func TestConcurrentServerServe(t *testing.T) { + setParallel(t) + for i := 0; i < 100; i++ { + ln1 := &oneConnListener{conn: nil} + ln2 := &oneConnListener{conn: nil} + srv := Server{} + go func() { srv.Serve(ln1) }() + go func() { srv.Serve(ln2) }() + } +} + +func TestServerIdleTimeout(t *testing.T) { + if testing.Short() { + t.Skip("skipping in short mode") + } + setParallel(t) + defer afterTest(t) + ts := httptest.NewUnstartedServer(HandlerFunc(func(w ResponseWriter, r *Request) { + io.Copy(ioutil.Discard, r.Body) + io.WriteString(w, r.RemoteAddr) + })) + ts.Config.ReadHeaderTimeout = 1 * time.Second + ts.Config.IdleTimeout = 2 * time.Second + ts.Start() + defer ts.Close() + + tr := &Transport{} + defer tr.CloseIdleConnections() + c := &Client{Transport: tr} + + get := func() string { + res, err := c.Get(ts.URL) + if err != nil { + t.Fatal(err) + } + defer res.Body.Close() + slurp, err := ioutil.ReadAll(res.Body) + if err != nil { + t.Fatal(err) + } + return string(slurp) + } + + a1, a2 := get(), get() + if a1 != a2 { + t.Fatalf("did requests on different connections") + } + time.Sleep(3 * time.Second) + a3 := get() + if a2 == a3 { + t.Fatal("request three unexpectedly on same connection") + } + + // And test that ReadHeaderTimeout still works: + conn, err := net.Dial("tcp", ts.Listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + conn.Write([]byte("GET / HTTP/1.1\r\nHost: foo.com\r\n")) + time.Sleep(2 * time.Second) + if _, err := io.CopyN(ioutil.Discard, conn, 1); err == nil { + t.Fatal("copy byte succeeded; want err") + } +} + +func get(t *testing.T, c *Client, url string) string { + res, err := c.Get(url) + if err != nil { + t.Fatal(err) + } + defer res.Body.Close() + slurp, err := ioutil.ReadAll(res.Body) + if err != nil { + t.Fatal(err) + } + return string(slurp) +} + +// Tests that calls to Server.SetKeepAlivesEnabled(false) closes any +// currently-open connections. +func TestServerSetKeepAlivesEnabledClosesConns(t *testing.T) { + setParallel(t) + defer afterTest(t) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + io.WriteString(w, r.RemoteAddr) + })) + defer ts.Close() + + tr := &Transport{} + defer tr.CloseIdleConnections() + c := &Client{Transport: tr} + + get := func() string { return get(t, c, ts.URL) } + + a1, a2 := get(), get() + if a1 != a2 { + t.Fatal("expected first two requests on same connection") + } + var idle0 int + if !waitCondition(2*time.Second, 10*time.Millisecond, func() bool { + idle0 = tr.IdleConnKeyCountForTesting() + return idle0 == 1 + }) { + t.Fatalf("idle count before SetKeepAlivesEnabled called = %v; want 1", idle0) + } + + ts.Config.SetKeepAlivesEnabled(false) + + var idle1 int + if !waitCondition(2*time.Second, 10*time.Millisecond, func() bool { + idle1 = tr.IdleConnKeyCountForTesting() + return idle1 == 0 + }) { + t.Fatalf("idle count after SetKeepAlivesEnabled called = %v; want 0", idle1) + } + + a3 := get() + if a3 == a2 { + t.Fatal("expected third request on new connection") + } +} + +func TestServerShutdown_h1(t *testing.T) { testServerShutdown(t, h1Mode) } +func TestServerShutdown_h2(t *testing.T) { testServerShutdown(t, h2Mode) } + +func testServerShutdown(t *testing.T, h2 bool) { + setParallel(t) + defer afterTest(t) + var doShutdown func() // set later + var shutdownRes = make(chan error, 1) + cst := newClientServerTest(t, h2, HandlerFunc(func(w ResponseWriter, r *Request) { + go doShutdown() + // Shutdown is graceful, so it should not interrupt + // this in-flight response. Add a tiny sleep here to + // increase the odds of a failure if shutdown has + // bugs. + time.Sleep(20 * time.Millisecond) + io.WriteString(w, r.RemoteAddr) + })) + defer cst.close() + + doShutdown = func() { + shutdownRes <- cst.ts.Config.Shutdown(context.Background()) + } + get(t, cst.c, cst.ts.URL) // calls t.Fail on failure + + if err := <-shutdownRes; err != nil { + t.Fatalf("Shutdown: %v", err) + } + + res, err := cst.c.Get(cst.ts.URL) + if err == nil { + res.Body.Close() + t.Fatal("second request should fail. server should be shut down") + } +} + +// Issue 17878: tests that we can call Close twice. +func TestServerCloseDeadlock(t *testing.T) { + var s Server + s.Close() + s.Close() +} + +// Issue 17717: tests that Server.SetKeepAlivesEnabled is respected by +// both HTTP/1 and HTTP/2. +func TestServerKeepAlivesEnabled_h1(t *testing.T) { testServerKeepAlivesEnabled(t, h1Mode) } +func TestServerKeepAlivesEnabled_h2(t *testing.T) { testServerKeepAlivesEnabled(t, h2Mode) } +func testServerKeepAlivesEnabled(t *testing.T, h2 bool) { + setParallel(t) + defer afterTest(t) + cst := newClientServerTest(t, h2, HandlerFunc(func(w ResponseWriter, r *Request) { + fmt.Fprintf(w, "%v", r.RemoteAddr) + })) + defer cst.close() + srv := cst.ts.Config + srv.SetKeepAlivesEnabled(false) + a := cst.getURL(cst.ts.URL) + if !waitCondition(2*time.Second, 10*time.Millisecond, srv.ExportAllConnsIdle) { + t.Fatalf("test server has active conns") + } + b := cst.getURL(cst.ts.URL) + if a == b { + t.Errorf("got same connection between first and second requests") + } + if !waitCondition(2*time.Second, 10*time.Millisecond, srv.ExportAllConnsIdle) { + t.Fatalf("test server has active conns") + } +} + +// Issue 18447: test that the Server's ReadTimeout is stopped while +// the server's doing its 1-byte background read between requests, +// waiting for the connection to maybe close. +func TestServerCancelsReadTimeoutWhenIdle(t *testing.T) { + setParallel(t) + defer afterTest(t) + const timeout = 250 * time.Millisecond + ts := httptest.NewUnstartedServer(HandlerFunc(func(w ResponseWriter, r *Request) { + select { + case <-time.After(2 * timeout): + fmt.Fprint(w, "ok") + case <-r.Context().Done(): + fmt.Fprint(w, r.Context().Err()) + } + })) + ts.Config.ReadTimeout = timeout + ts.Start() + defer ts.Close() + + tr := &Transport{} + defer tr.CloseIdleConnections() + c := &Client{Transport: tr} + + res, err := c.Get(ts.URL) + if err != nil { + t.Fatal(err) + } + slurp, err := ioutil.ReadAll(res.Body) + res.Body.Close() + if err != nil { + t.Fatal(err) + } + if string(slurp) != "ok" { + t.Fatalf("Got: %q, want ok", slurp) + } +} + +// Issue 18535: test that the Server doesn't try to do a background +// read if it's already done one. +func TestServerDuplicateBackgroundRead(t *testing.T) { + setParallel(t) + defer afterTest(t) + + const goroutines = 5 + const requests = 2000 + + hts := httptest.NewServer(HandlerFunc(NotFound)) + defer hts.Close() + + reqBytes := []byte("GET / HTTP/1.1\r\nHost: e.com\r\n\r\n") + + var wg sync.WaitGroup + for i := 0; i < goroutines; i++ { + wg.Add(1) + go func() { + defer wg.Done() + cn, err := net.Dial("tcp", hts.Listener.Addr().String()) + if err != nil { + t.Error(err) + return + } + defer cn.Close() + + wg.Add(1) + go func() { + defer wg.Done() + io.Copy(ioutil.Discard, cn) + }() + + for j := 0; j < requests; j++ { + if t.Failed() { + return + } + _, err := cn.Write(reqBytes) + if err != nil { + t.Error(err) + return + } + } + }() + } + wg.Wait() +} + +// Test that the bufio.Reader returned by Hijack includes any buffered +// byte (from the Server's backgroundRead) in its buffer. We want the +// Handler code to be able to tell that a byte is available via +// bufio.Reader.Buffered(), without resorting to Reading it +// (potentially blocking) to get at it. +func TestServerHijackGetsBackgroundByte(t *testing.T) { + if runtime.GOOS == "plan9" { + t.Skip("skipping test; see https://golang.org/issue/18657") + } + setParallel(t) + defer afterTest(t) + done := make(chan struct{}) + inHandler := make(chan bool, 1) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + defer close(done) + + // Tell the client to send more data after the GET request. + inHandler <- true + + // Wait until the HTTP server sees the extra data + // after the GET request. The HTTP server fires the + // close notifier here, assuming it's a pipelined + // request, as documented. + select { + case <-w.(CloseNotifier).CloseNotify(): + case <-time.After(5 * time.Second): + t.Error("timeout") + return + } + + conn, buf, err := w.(Hijacker).Hijack() + if err != nil { + t.Error(err) + return + } + defer conn.Close() + n := buf.Reader.Buffered() + if n != 1 { + t.Errorf("buffered data = %d; want 1", n) + } + peek, err := buf.Reader.Peek(3) + if string(peek) != "foo" || err != nil { + t.Errorf("Peek = %q, %v; want foo, nil", peek, err) + } + })) + defer ts.Close() + + cn, err := net.Dial("tcp", ts.Listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + defer cn.Close() + if _, err := cn.Write([]byte("GET / HTTP/1.1\r\nHost: e.com\r\n\r\n")); err != nil { + t.Fatal(err) + } + <-inHandler + if _, err := cn.Write([]byte("foo")); err != nil { + t.Fatal(err) + } + + if err := cn.(*net.TCPConn).CloseWrite(); err != nil { + t.Fatal(err) + } + select { + case <-done: + case <-time.After(2 * time.Second): + t.Error("timeout") + } +} + +// Like TestServerHijackGetsBackgroundByte above but sending a +// immediate 1MB of data to the server to fill up the server's 4KB +// buffer. +func TestServerHijackGetsBackgroundByte_big(t *testing.T) { + if runtime.GOOS == "plan9" { + t.Skip("skipping test; see https://golang.org/issue/18657") + } + setParallel(t) + defer afterTest(t) + done := make(chan struct{}) + const size = 8 << 10 + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + defer close(done) + + // Wait until the HTTP server sees the extra data + // after the GET request. The HTTP server fires the + // close notifier here, assuming it's a pipelined + // request, as documented. + select { + case <-w.(CloseNotifier).CloseNotify(): + case <-time.After(5 * time.Second): + t.Error("timeout") + return + } + + conn, buf, err := w.(Hijacker).Hijack() + if err != nil { + t.Error(err) + return + } + defer conn.Close() + slurp, err := ioutil.ReadAll(buf.Reader) + if err != nil { + t.Errorf("Copy: %v", err) + } + allX := true + for _, v := range slurp { + if v != 'x' { + allX = false + } + } + if len(slurp) != size { + t.Errorf("read %d; want %d", len(slurp), size) + } else if !allX { + t.Errorf("read %q; want %d 'x'", slurp, size) + } + })) + defer ts.Close() + + cn, err := net.Dial("tcp", ts.Listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + defer cn.Close() + if _, err := fmt.Fprintf(cn, "GET / HTTP/1.1\r\nHost: e.com\r\n\r\n%s", + strings.Repeat("x", size)); err != nil { + t.Fatal(err) + } + if err := cn.(*net.TCPConn).CloseWrite(); err != nil { + t.Fatal(err) + } + + select { + case <-done: + case <-time.After(2 * time.Second): + t.Error("timeout") + } +} diff --git a/libgo/go/net/http/server.go b/libgo/go/net/http/server.go index 5e3b6084ae..df70a15193 100644 --- a/libgo/go/net/http/server.go +++ b/libgo/go/net/http/server.go @@ -2,13 +2,14 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// HTTP server. See RFC 2616. +// HTTP server. See RFC 2616. package http import ( "bufio" "bytes" + "context" "crypto/tls" "errors" "fmt" @@ -26,14 +27,32 @@ import ( "sync" "sync/atomic" "time" + + "golang_org/x/net/lex/httplex" ) -// Errors introduced by the HTTP server. +// Errors used by the HTTP server. var ( - ErrWriteAfterFlush = errors.New("Conn.Write called after Flush") - ErrBodyNotAllowed = errors.New("http: request method or response status code does not allow body") - ErrHijacked = errors.New("Conn has been hijacked") - ErrContentLength = errors.New("Conn.Write wrote more than the declared Content-Length") + // ErrBodyNotAllowed is returned by ResponseWriter.Write calls + // when the HTTP method or response code does not permit a + // body. + ErrBodyNotAllowed = errors.New("http: request method or response status code does not allow body") + + // ErrHijacked is returned by ResponseWriter.Write calls when + // the underlying connection has been hijacked using the + // Hijacker interface. A zero-byte write on a hijacked + // connection will return ErrHijacked without any other side + // effects. + ErrHijacked = errors.New("http: connection has been hijacked") + + // ErrContentLength is returned by ResponseWriter.Write calls + // when a Handler set a Content-Length response header with a + // declared size and then attempted to write more bytes than + // declared. + ErrContentLength = errors.New("http: wrote more than the declared Content-Length") + + // Deprecated: ErrWriteAfterFlush is no longer used. + ErrWriteAfterFlush = errors.New("unused") ) // A Handler responds to an HTTP request. @@ -50,10 +69,15 @@ var ( // ResponseWriter. Cautious handlers should read the Request.Body // first, and then reply. // +// Except for reading the body, handlers should not modify the +// provided Request. +// // If ServeHTTP panics, the server (the caller of ServeHTTP) assumes // that the effect of the panic was isolated to the active request. // It recovers the panic, logs a stack trace to the server error log, -// and hangs up the connection. +// and hangs up the connection. To abort a handler so the client sees +// an interrupted response but the server doesn't log an error, panic +// with the value ErrAbortHandler. type Handler interface { ServeHTTP(ResponseWriter, *Request) } @@ -65,18 +89,46 @@ type Handler interface { // has returned. type ResponseWriter interface { // Header returns the header map that will be sent by - // WriteHeader. Changing the header after a call to - // WriteHeader (or Write) has no effect unless the modified - // headers were declared as trailers by setting the - // "Trailer" header before the call to WriteHeader (see example). - // To suppress implicit response headers, set their value to nil. + // WriteHeader. The Header map also is the mechanism with which + // Handlers can set HTTP trailers. + // + // Changing the header map after a call to WriteHeader (or + // Write) has no effect unless the modified headers are + // trailers. + // + // There are two ways to set Trailers. The preferred way is to + // predeclare in the headers which trailers you will later + // send by setting the "Trailer" header to the names of the + // trailer keys which will come later. In this case, those + // keys of the Header map are treated as if they were + // trailers. See the example. The second way, for trailer + // keys not known to the Handler until after the first Write, + // is to prefix the Header map keys with the TrailerPrefix + // constant value. See TrailerPrefix. + // + // To suppress implicit response headers (such as "Date"), set + // their value to nil. Header() Header // Write writes the data to the connection as part of an HTTP reply. - // If WriteHeader has not yet been called, Write calls WriteHeader(http.StatusOK) - // before writing the data. If the Header does not contain a - // Content-Type line, Write adds a Content-Type set to the result of passing - // the initial 512 bytes of written data to DetectContentType. + // + // If WriteHeader has not yet been called, Write calls + // WriteHeader(http.StatusOK) before writing the data. If the Header + // does not contain a Content-Type line, Write adds a Content-Type set + // to the result of passing the initial 512 bytes of written data to + // DetectContentType. + // + // Depending on the HTTP protocol version and the client, calling + // Write or WriteHeader may prevent future reads on the + // Request.Body. For HTTP/1.x requests, handlers should read any + // needed request body data before writing the response. Once the + // headers have been flushed (due to either an explicit Flusher.Flush + // call or writing enough data to trigger a flush), the request body + // may be unavailable. For HTTP/2 requests, the Go HTTP server permits + // handlers to continue to read the request body while concurrently + // writing the response. However, such behavior may not be supported + // by all HTTP/2 clients. Handlers should read before writing if + // possible to maximize compatibility. Write([]byte) (int, error) // WriteHeader sends an HTTP response header with status code. @@ -90,6 +142,10 @@ type ResponseWriter interface { // The Flusher interface is implemented by ResponseWriters that allow // an HTTP handler to flush buffered data to the client. // +// The default HTTP/1.x and HTTP/2 ResponseWriter implementations +// support Flusher, but ResponseWriter wrappers may not. Handlers +// should always test for this ability at runtime. +// // Note that even for ResponseWriters that support Flush, // if the client is connected through an HTTP proxy, // the buffered data may not reach the client until the response @@ -101,9 +157,14 @@ type Flusher interface { // The Hijacker interface is implemented by ResponseWriters that allow // an HTTP handler to take over the connection. +// +// The default ResponseWriter for HTTP/1.x connections supports +// Hijacker, but HTTP/2 connections intentionally do not. +// ResponseWriter wrappers may also not support Hijacker. Handlers +// should always test for this ability at runtime. type Hijacker interface { // Hijack lets the caller take over the connection. - // After a call to Hijack(), the HTTP server library + // After a call to Hijack the HTTP server library // will not do anything else with the connection. // // It becomes the caller's responsibility to manage @@ -113,6 +174,9 @@ type Hijacker interface { // already set, depending on the configuration of the // Server. It is the caller's responsibility to set // or clear those deadlines as needed. + // + // The returned bufio.Reader may contain unprocessed buffered + // data from the client. Hijack() (net.Conn, *bufio.ReadWriter, error) } @@ -143,12 +207,29 @@ type CloseNotifier interface { CloseNotify() <-chan bool } +var ( + // ServerContextKey is a context key. It can be used in HTTP + // handlers with context.WithValue to access the server that + // started the handler. The associated value will be of + // type *Server. + ServerContextKey = &contextKey{"http-server"} + + // LocalAddrContextKey is a context key. It can be used in + // HTTP handlers with context.WithValue to access the address + // the local address the connection arrived on. + // The associated value will be of type net.Addr. + LocalAddrContextKey = &contextKey{"local-addr"} +) + // A conn represents the server side of an HTTP connection. type conn struct { // server is the server on which the connection arrived. // Immutable; never nil. server *Server + // cancelCtx cancels the connection-level context. + cancelCtx context.CancelFunc + // rwc is the underlying network connection. // This is never wrapped by other types and is the value given out // to CloseNotifier callers. It is usually of type *net.TCPConn or @@ -175,7 +256,6 @@ type conn struct { r *connReader // bufr reads from r. - // Users of bufr must hold mu. bufr *bufio.Reader // bufw writes to checkConnErrorWriter{c}, which populates werr on error. @@ -185,7 +265,11 @@ type conn struct { // on this connection, if any. lastMethod string - // mu guards hijackedv, use of bufr, (*response).closeNotifyCh. + curReq atomic.Value // of *response (which has a Request in it) + + curState atomic.Value // of ConnState + + // mu guards hijackedv mu sync.Mutex // hijackedv is whether this connection has been hijacked @@ -205,9 +289,18 @@ func (c *conn) hijackLocked() (rwc net.Conn, buf *bufio.ReadWriter, err error) { if c.hijackedv { return nil, nil, ErrHijacked } + c.r.abortPendingRead() + c.hijackedv = true rwc = c.rwc + rwc.SetDeadline(time.Time{}) + buf = bufio.NewReadWriter(c.bufr, bufio.NewWriter(rwc)) + if c.r.hasByte { + if _, err := c.bufr.Peek(c.bufr.Buffered() + 1); err != nil { + return nil, nil, fmt.Errorf("unexpected Peek failure reading buffered byte: %v", err) + } + } c.setState(rwc, StateHijacked) return } @@ -289,13 +382,7 @@ func (cw *chunkWriter) close() { bw := cw.res.conn.bufw // conn's bufio writer // zero chunk to mark EOF bw.WriteString("0\r\n") - if len(cw.res.trailers) > 0 { - trailers := make(Header) - for _, h := range cw.res.trailers { - if vv := cw.res.handlerHeader[h]; len(vv) > 0 { - trailers[h] = vv - } - } + if trailers := cw.res.finalTrailers(); trailers != nil { trailers.Write(bw) // the writer handles noting errors } // final blank line after the trailers (whether @@ -306,11 +393,14 @@ func (cw *chunkWriter) close() { // A response represents the server side of an HTTP response. type response struct { - conn *conn - req *Request // request for this response - reqBody io.ReadCloser - wroteHeader bool // reply header has been (logically) written - wroteContinue bool // 100 Continue response was written + conn *conn + req *Request // request for this response + reqBody io.ReadCloser + cancelCtx context.CancelFunc // when ServeHTTP exits + wroteHeader bool // reply header has been (logically) written + wroteContinue bool // 100 Continue response was written + wants10KeepAlive bool // HTTP/1.0 w/ Connection "keep-alive" + wantsClose bool // HTTP request has Connection "close" w *bufio.Writer // buffers output in chunks to chunkWriter cw chunkWriter @@ -342,7 +432,7 @@ type response struct { requestBodyLimitHit bool // trailers are the headers to be sent after the handler - // finishes writing the body. This field is initialized from + // finishes writing the body. This field is initialized from // the Trailer response header when the response header is // written. trailers []string @@ -353,9 +443,48 @@ type response struct { dateBuf [len(TimeFormat)]byte clenBuf [10]byte - // closeNotifyCh is non-nil once CloseNotify is called. - // Guarded by conn.mu - closeNotifyCh <-chan bool + // closeNotifyCh is the channel returned by CloseNotify. + // TODO(bradfitz): this is currently (for Go 1.8) always + // non-nil. Make this lazily-created again as it used to be? + closeNotifyCh chan bool + didCloseNotify int32 // atomic (only 0->1 winner should send) +} + +// TrailerPrefix is a magic prefix for ResponseWriter.Header map keys +// that, if present, signals that the map entry is actually for +// the response trailers, and not the response headers. The prefix +// is stripped after the ServeHTTP call finishes and the values are +// sent in the trailers. +// +// This mechanism is intended only for trailers that are not known +// prior to the headers being written. If the set of trailers is fixed +// or known before the header is written, the normal Go trailers mechanism +// is preferred: +// https://golang.org/pkg/net/http/#ResponseWriter +// https://golang.org/pkg/net/http/#example_ResponseWriter_trailers +const TrailerPrefix = "Trailer:" + +// finalTrailers is called after the Handler exits and returns a non-nil +// value if the Handler set any trailers. +func (w *response) finalTrailers() Header { + var t Header + for k, vv := range w.handlerHeader { + if strings.HasPrefix(k, TrailerPrefix) { + if t == nil { + t = make(Header) + } + t[strings.TrimPrefix(k, TrailerPrefix)] = vv + } + } + for _, k := range w.trailers { + if t == nil { + t = make(Header) + } + for _, v := range w.handlerHeader[k] { + t.Add(k, v) + } + } + return t } type atomicBool int32 @@ -488,60 +617,152 @@ type readResult struct { // call blocked in a background goroutine to wait for activity and // trigger a CloseNotifier channel. type connReader struct { - r io.Reader - remain int64 // bytes remaining + conn *conn - // ch is non-nil if a background read is in progress. - // It is guarded by conn.mu. - ch chan readResult + mu sync.Mutex // guards following + hasByte bool + byteBuf [1]byte + bgErr error // non-nil means error happened on background read + cond *sync.Cond + inRead bool + aborted bool // set true before conn.rwc deadline is set to past + remain int64 // bytes remaining +} + +func (cr *connReader) lock() { + cr.mu.Lock() + if cr.cond == nil { + cr.cond = sync.NewCond(&cr.mu) + } +} + +func (cr *connReader) unlock() { cr.mu.Unlock() } + +func (cr *connReader) startBackgroundRead() { + cr.lock() + defer cr.unlock() + if cr.inRead { + panic("invalid concurrent Body.Read call") + } + if cr.hasByte { + return + } + cr.inRead = true + cr.conn.rwc.SetReadDeadline(time.Time{}) + go cr.backgroundRead() +} + +func (cr *connReader) backgroundRead() { + n, err := cr.conn.rwc.Read(cr.byteBuf[:]) + cr.lock() + if n == 1 { + cr.hasByte = true + // We were at EOF already (since we wouldn't be in a + // background read otherwise), so this is a pipelined + // HTTP request. + cr.closeNotifyFromPipelinedRequest() + } + if ne, ok := err.(net.Error); ok && cr.aborted && ne.Timeout() { + // Ignore this error. It's the expected error from + // another goroutine calling abortPendingRead. + } else if err != nil { + cr.handleReadError(err) + } + cr.aborted = false + cr.inRead = false + cr.unlock() + cr.cond.Broadcast() +} + +func (cr *connReader) abortPendingRead() { + cr.lock() + defer cr.unlock() + if !cr.inRead { + return + } + cr.aborted = true + cr.conn.rwc.SetReadDeadline(aLongTimeAgo) + for cr.inRead { + cr.cond.Wait() + } + cr.conn.rwc.SetReadDeadline(time.Time{}) } func (cr *connReader) setReadLimit(remain int64) { cr.remain = remain } -func (cr *connReader) setInfiniteReadLimit() { cr.remain = 1<<63 - 1 } +func (cr *connReader) setInfiniteReadLimit() { cr.remain = maxInt64 } func (cr *connReader) hitReadLimit() bool { return cr.remain <= 0 } +// may be called from multiple goroutines. +func (cr *connReader) handleReadError(err error) { + cr.conn.cancelCtx() + cr.closeNotify() +} + +// closeNotifyFromPipelinedRequest simply calls closeNotify. +// +// This method wrapper is here for documentation. The callers are the +// cases where we send on the closenotify channel because of a +// pipelined HTTP request, per the previous Go behavior and +// documentation (that this "MAY" happen). +// +// TODO: consider changing this behavior and making context +// cancelation and closenotify work the same. +func (cr *connReader) closeNotifyFromPipelinedRequest() { + cr.closeNotify() +} + +// may be called from multiple goroutines. +func (cr *connReader) closeNotify() { + res, _ := cr.conn.curReq.Load().(*response) + if res != nil { + if atomic.CompareAndSwapInt32(&res.didCloseNotify, 0, 1) { + res.closeNotifyCh <- true + } + } +} + func (cr *connReader) Read(p []byte) (n int, err error) { + cr.lock() + if cr.inRead { + cr.unlock() + panic("invalid concurrent Body.Read call") + } if cr.hitReadLimit() { + cr.unlock() return 0, io.EOF } + if cr.bgErr != nil { + err = cr.bgErr + cr.unlock() + return 0, err + } if len(p) == 0 { - return + cr.unlock() + return 0, nil } if int64(len(p)) > cr.remain { p = p[:cr.remain] } - - // Is a background read (started by CloseNotifier) already in - // flight? If so, wait for it and use its result. - ch := cr.ch - if ch != nil { - cr.ch = nil - res := <-ch - if res.n == 1 { - p[0] = res.b - cr.remain -= 1 - } - return res.n, res.err + if cr.hasByte { + p[0] = cr.byteBuf[0] + cr.hasByte = false + cr.unlock() + return 1, nil } - n, err = cr.r.Read(p) - cr.remain -= int64(n) - return -} + cr.inRead = true + cr.unlock() + n, err = cr.conn.rwc.Read(p) -func (cr *connReader) startBackgroundRead(onReadComplete func()) { - if cr.ch != nil { - // Background read already started. - return + cr.lock() + cr.inRead = false + if err != nil { + cr.handleReadError(err) } - cr.ch = make(chan readResult, 1) - go cr.closeNotifyAwaitActivityRead(cr.ch, onReadComplete) -} + cr.remain -= int64(n) + cr.unlock() -func (cr *connReader) closeNotifyAwaitActivityRead(ch chan<- readResult, onReadComplete func()) { - var buf [1]byte - n, err := cr.r.Read(buf[:1]) - onReadComplete() - ch <- readResult{n, err, buf[0]} + cr.cond.Broadcast() + return n, err } var ( @@ -573,7 +794,7 @@ func newBufioReader(r io.Reader) *bufio.Reader { br.Reset(r) return br } - // Note: if this reader size is every changed, update + // Note: if this reader size is ever changed, update // TestHandlerBodyClose's assumptions. return bufio.NewReader(r) } @@ -681,14 +902,23 @@ func appendTime(b []byte, t time.Time) []byte { var errTooLarge = errors.New("http: request too large") // Read next request from connection. -func (c *conn) readRequest() (w *response, err error) { +func (c *conn) readRequest(ctx context.Context) (w *response, err error) { if c.hijacked() { return nil, ErrHijacked } + var ( + wholeReqDeadline time.Time // or zero if none + hdrDeadline time.Time // or zero if none + ) + t0 := time.Now() + if d := c.server.readHeaderTimeout(); d != 0 { + hdrDeadline = t0.Add(d) + } if d := c.server.ReadTimeout; d != 0 { - c.rwc.SetReadDeadline(time.Now().Add(d)) + wholeReqDeadline = t0.Add(d) } + c.rwc.SetReadDeadline(hdrDeadline) if d := c.server.WriteTimeout; d != 0 { defer func() { c.rwc.SetWriteDeadline(time.Now().Add(d)) @@ -696,63 +926,102 @@ func (c *conn) readRequest() (w *response, err error) { } c.r.setReadLimit(c.server.initialReadLimitSize()) - c.mu.Lock() // while using bufr if c.lastMethod == "POST" { // RFC 2616 section 4.1 tolerance for old buggy clients. peek, _ := c.bufr.Peek(4) // ReadRequest will get err below c.bufr.Discard(numLeadingCRorLF(peek)) } req, err := readRequest(c.bufr, keepHostHeader) - c.mu.Unlock() if err != nil { if c.r.hitReadLimit() { return nil, errTooLarge } return nil, err } + + if !http1ServerSupportsRequest(req) { + return nil, badRequestError("unsupported protocol version") + } + c.lastMethod = req.Method c.r.setInfiniteReadLimit() hosts, haveHost := req.Header["Host"] - if req.ProtoAtLeast(1, 1) && (!haveHost || len(hosts) == 0) { + isH2Upgrade := req.isH2Upgrade() + if req.ProtoAtLeast(1, 1) && (!haveHost || len(hosts) == 0) && !isH2Upgrade { return nil, badRequestError("missing required Host header") } if len(hosts) > 1 { return nil, badRequestError("too many Host headers") } - if len(hosts) == 1 && !validHostHeader(hosts[0]) { + if len(hosts) == 1 && !httplex.ValidHostHeader(hosts[0]) { return nil, badRequestError("malformed Host header") } for k, vv := range req.Header { - if !validHeaderName(k) { + if !httplex.ValidHeaderFieldName(k) { return nil, badRequestError("invalid header name") } for _, v := range vv { - if !validHeaderValue(v) { + if !httplex.ValidHeaderFieldValue(v) { return nil, badRequestError("invalid header value") } } } delete(req.Header, "Host") + ctx, cancelCtx := context.WithCancel(ctx) + req.ctx = ctx req.RemoteAddr = c.remoteAddr req.TLS = c.tlsState if body, ok := req.Body.(*body); ok { body.doEarlyClose = true } + // Adjust the read deadline if necessary. + if !hdrDeadline.Equal(wholeReqDeadline) { + c.rwc.SetReadDeadline(wholeReqDeadline) + } + w = &response{ conn: c, + cancelCtx: cancelCtx, req: req, reqBody: req.Body, handlerHeader: make(Header), contentLength: -1, + closeNotifyCh: make(chan bool, 1), + + // We populate these ahead of time so we're not + // reading from req.Header after their Handler starts + // and maybe mutates it (Issue 14940) + wants10KeepAlive: req.wantsHttp10KeepAlive(), + wantsClose: req.wantsClose(), + } + if isH2Upgrade { + w.closeAfterReply = true } w.cw.res = w w.w = newBufioWriterSize(&w.cw, bufferBeforeChunkingSize) return w, nil } +// http1ServerSupportsRequest reports whether Go's HTTP/1.x server +// supports the given request. +func http1ServerSupportsRequest(req *Request) bool { + if req.ProtoMajor == 1 { + return true + } + // Accept "PRI * HTTP/2.0" upgrade requests, so Handlers can + // wire up their own HTTP/2 upgrades. + if req.ProtoMajor == 2 && req.ProtoMinor == 0 && + req.Method == "PRI" && req.RequestURI == "*" { + return true + } + // Reject HTTP/0.x, and all other HTTP/2+ requests (which + // aren't encoded in ASCII anyway). + return false +} + func (w *response) Header() Header { if w.cw.header == nil && w.wroteHeader && !w.cw.wroteHeader { // Accessing the header between logically writing it @@ -766,7 +1035,7 @@ func (w *response) Header() Header { // maxPostHandlerReadBytes is the max number of Request.Body bytes not // consumed by a handler that the server will read from the client -// in order to keep a connection alive. If there are more bytes than +// in order to keep a connection alive. If there are more bytes than // this then the server to be paranoid instead sends a "Connection: // close" response. // @@ -855,8 +1124,8 @@ func (h extraHeader) Write(w *bufio.Writer) { // to cw.res.conn.bufw. // // p is not written by writeHeader, but is the first chunk of the body -// that will be written. It is sniffed for a Content-Type if none is -// set explicitly. It's also used to set the Content-Length, if the +// that will be written. It is sniffed for a Content-Type if none is +// set explicitly. It's also used to set the Content-Length, if the // total body size was small and the handler has already finished // running. func (cw *chunkWriter) writeHeader(p []byte) { @@ -895,7 +1164,17 @@ func (cw *chunkWriter) writeHeader(p []byte) { } var setHeader extraHeader + // Don't write out the fake "Trailer:foo" keys. See TrailerPrefix. trailers := false + for k := range cw.header { + if strings.HasPrefix(k, TrailerPrefix) { + if excludeHeader == nil { + excludeHeader = make(map[string]bool) + } + excludeHeader[k] = true + trailers = true + } + } for _, v := range cw.header["Trailer"] { trailers = true foreachHeaderElement(v, cw.res.declareTrailer) @@ -911,9 +1190,9 @@ func (cw *chunkWriter) writeHeader(p []byte) { // Exceptions: 304/204/1xx responses never get Content-Length, and if // it was a HEAD request, we don't know the difference between // 0 actual bytes and 0 bytes because the handler noticed it - // was a HEAD request and chose not to write anything. So for + // was a HEAD request and chose not to write anything. So for // HEAD, the handler should either write the Content-Length or - // write non-zero bytes. If it's actually 0 bytes and the + // write non-zero bytes. If it's actually 0 bytes and the // handler never looked at the Request.Method, we just don't // send a Content-Length header. // Further, we don't send an automatic Content-Length if they @@ -925,7 +1204,7 @@ func (cw *chunkWriter) writeHeader(p []byte) { // If this was an HTTP/1.0 request with keep-alive and we sent a // Content-Length back, we can make this a keep-alive response ... - if w.req.wantsHttp10KeepAlive() && keepAlivesEnabled { + if w.wants10KeepAlive && keepAlivesEnabled { sentLength := header.get("Content-Length") != "" if sentLength && header.get("Connection") == "keep-alive" { w.closeAfterReply = false @@ -935,12 +1214,12 @@ func (cw *chunkWriter) writeHeader(p []byte) { // Check for a explicit (and valid) Content-Length header. hasCL := w.contentLength != -1 - if w.req.wantsHttp10KeepAlive() && (isHEAD || hasCL) { + if w.wants10KeepAlive && (isHEAD || hasCL || !bodyAllowedForStatus(w.status)) { _, connectionHeaderSet := header["Connection"] if !connectionHeaderSet { setHeader.connection = "keep-alive" } - } else if !w.req.ProtoAtLeast(1, 1) || w.req.wantsClose() { + } else if !w.req.ProtoAtLeast(1, 1) || w.wantsClose { w.closeAfterReply = true } @@ -965,9 +1244,12 @@ func (cw *chunkWriter) writeHeader(p []byte) { } // Per RFC 2616, we should consume the request body before - // replying, if the handler hasn't already done so. But we + // replying, if the handler hasn't already done so. But we // don't want to do an unbounded amount of reading here for // DoS reasons, so we only try up to a threshold. + // TODO(bradfitz): where does RFC 2616 say that? See Issue 15527 + // about HTTP/1.x Handlers concurrently reading and writing, like + // HTTP/2 handlers can do. Maybe this code should be relaxed? if w.req.ContentLength != 0 && !w.closeAfterReply { var discard, tooBig bool @@ -1009,7 +1291,7 @@ func (cw *chunkWriter) writeHeader(p []byte) { w.closeAfterReply = true } default: - // Some other kind of error occured, like a read timeout, or + // Some other kind of error occurred, like a read timeout, or // corrupt chunked encoding. In any case, whatever remains // on the wire must not be parsed as another HTTP request. w.closeAfterReply = true @@ -1069,6 +1351,10 @@ func (cw *chunkWriter) writeHeader(p []byte) { // to avoid closing the connection at EOF. cw.chunking = true setHeader.transferEncoding = "chunked" + if hasTE && te == "chunked" { + // We will send the chunked Transfer-Encoding header later. + delHeader("Transfer-Encoding") + } } } else { // HTTP version < 1.1: cannot do chunked transfer @@ -1148,7 +1434,7 @@ func statusLine(req *Request, code int) string { if proto11 { proto = "HTTP/1.1" } - codestring := strconv.Itoa(code) + codestring := fmt.Sprintf("%03d", code) text, ok := statusText[code] if !ok { text = "status code " + codestring @@ -1174,7 +1460,7 @@ func (w *response) bodyAllowed() bool { // The Life Of A Write is like this: // // Handler starts. No header has been sent. The handler can either -// write a header, or just start writing. Writing before sending a header +// write a header, or just start writing. Writing before sending a header // sends an implicitly empty 200 OK header. // // If the handler didn't declare a Content-Length up front, we either @@ -1200,7 +1486,7 @@ func (w *response) bodyAllowed() bool { // initial header contains both a Content-Type and Content-Length. // Also short-circuit in (1) when the header's been sent and not in // chunking mode, writing directly to (4) instead, if (2) has no -// buffered data. More generally, we could short-circuit from (1) to +// buffered data. More generally, we could short-circuit from (1) to // (3) even in chunking mode if the write size from (1) is over some // threshold and nothing is in (2). The answer might be mostly making // bufferBeforeChunkingSize smaller and having bufio's fast-paths deal @@ -1216,7 +1502,9 @@ func (w *response) WriteString(data string) (n int, err error) { // either dataB or dataS is non-zero. func (w *response) write(lenData int, dataB []byte, dataS string) (n int, err error) { if w.conn.hijacked() { - w.conn.server.logf("http: response.Write on hijacked connection") + if lenData > 0 { + w.conn.server.logf("http: response.Write on hijacked connection") + } return 0, ErrHijacked } if !w.wroteHeader { @@ -1252,6 +1540,8 @@ func (w *response) finishRequest() { w.cw.close() w.conn.bufw.Flush() + w.conn.r.abortPendingRead() + // Close the body (regardless of w.closeAfterReply) so we can // re-use its bufio.Reader later safely. w.reqBody.Close() @@ -1341,7 +1631,7 @@ type closeWriter interface { var _ closeWriter = (*net.TCPConn)(nil) // closeWrite flushes any outstanding data and sends a FIN packet (if -// client is connected via TCP), signalling that we're done. We then +// client is connected via TCP), signalling that we're done. We then // pause for a bit, hoping the client processes it before any // subsequent RST. // @@ -1355,7 +1645,7 @@ func (c *conn) closeWriteAndWait() { } // validNPN reports whether the proto is not a blacklisted Next -// Protocol Negotiation protocol. Empty and built-in protocol types +// Protocol Negotiation protocol. Empty and built-in protocol types // are blacklisted and can't be overridden with alternate // implementations. func validNPN(proto string) bool { @@ -1367,23 +1657,65 @@ func validNPN(proto string) bool { } func (c *conn) setState(nc net.Conn, state ConnState) { - if hook := c.server.ConnState; hook != nil { + srv := c.server + switch state { + case StateNew: + srv.trackConn(c, true) + case StateHijacked, StateClosed: + srv.trackConn(c, false) + } + c.curState.Store(connStateInterface[state]) + if hook := srv.ConnState; hook != nil { hook(nc, state) } } +// connStateInterface is an array of the interface{} versions of +// ConnState values, so we can use them in atomic.Values later without +// paying the cost of shoving their integers in an interface{}. +var connStateInterface = [...]interface{}{ + StateNew: StateNew, + StateActive: StateActive, + StateIdle: StateIdle, + StateHijacked: StateHijacked, + StateClosed: StateClosed, +} + // badRequestError is a literal string (used by in the server in HTML, // unescaped) to tell the user why their request was bad. It should -// be plain text without user info or other embeddded errors. +// be plain text without user info or other embedded errors. type badRequestError string func (e badRequestError) Error() string { return "Bad Request: " + string(e) } +// ErrAbortHandler is a sentinel panic value to abort a handler. +// While any panic from ServeHTTP aborts the response to the client, +// panicking with ErrAbortHandler also suppresses logging of a stack +// trace to the server's error log. +var ErrAbortHandler = errors.New("net/http: abort Handler") + +// isCommonNetReadError reports whether err is a common error +// encountered during reading a request off the network when the +// client has gone away or had its read fail somehow. This is used to +// determine which logs are interesting enough to log about. +func isCommonNetReadError(err error) bool { + if err == io.EOF { + return true + } + if neterr, ok := err.(net.Error); ok && neterr.Timeout() { + return true + } + if oe, ok := err.(*net.OpError); ok && oe.Op == "read" { + return true + } + return false +} + // Serve a new connection. -func (c *conn) serve() { +func (c *conn) serve(ctx context.Context) { c.remoteAddr = c.rwc.RemoteAddr().String() defer func() { - if err := recover(); err != nil { + if err := recover(); err != nil && err != ErrAbortHandler { const size = 64 << 10 buf := make([]byte, size) buf = buf[:runtime.Stack(buf, false)] @@ -1417,38 +1749,46 @@ func (c *conn) serve() { } } - c.r = &connReader{r: c.rwc} + // HTTP/1.x from here on. + + ctx, cancelCtx := context.WithCancel(ctx) + c.cancelCtx = cancelCtx + defer cancelCtx() + + c.r = &connReader{conn: c} c.bufr = newBufioReader(c.r) c.bufw = newBufioWriterSize(checkConnErrorWriter{c}, 4<<10) for { - w, err := c.readRequest() + w, err := c.readRequest(ctx) if c.r.remain != c.server.initialReadLimitSize() { // If we read any bytes off the wire, we're active. c.setState(c.rwc, StateActive) } if err != nil { + const errorHeaders = "\r\nContent-Type: text/plain; charset=utf-8\r\nConnection: close\r\n\r\n" + if err == errTooLarge { // Their HTTP client may or may not be // able to read this if we're // responding to them and hanging up // while they're still writing their - // request. Undefined behavior. - io.WriteString(c.rwc, "HTTP/1.1 431 Request Header Fields Too Large\r\nContent-Type: text/plain\r\nConnection: close\r\n\r\n431 Request Header Fields Too Large") + // request. Undefined behavior. + const publicErr = "431 Request Header Fields Too Large" + fmt.Fprintf(c.rwc, "HTTP/1.1 "+publicErr+errorHeaders+publicErr) c.closeWriteAndWait() return } - if err == io.EOF { - return // don't reply - } - if neterr, ok := err.(net.Error); ok && neterr.Timeout() { + if isCommonNetReadError(err) { return // don't reply } - var publicErr string + + publicErr := "400 Bad Request" if v, ok := err.(badRequestError); ok { - publicErr = ": " + string(v) + publicErr = publicErr + ": " + string(v) } - io.WriteString(c.rwc, "HTTP/1.1 400 Bad Request\r\nContent-Type: text/plain\r\nConnection: close\r\n\r\n400 Bad Request"+publicErr) + + fmt.Fprintf(c.rwc, "HTTP/1.1 "+publicErr+errorHeaders+publicErr) return } @@ -1464,12 +1804,26 @@ func (c *conn) serve() { return } + c.curReq.Store(w) + + if requestBodyRemains(req.Body) { + registerOnHitEOF(req.Body, w.conn.r.startBackgroundRead) + } else { + if w.conn.bufr.Buffered() > 0 { + w.conn.r.closeNotifyFromPipelinedRequest() + } + w.conn.r.startBackgroundRead() + } + // HTTP cannot have multiple simultaneous active requests.[*] // Until the server replies to this request, it can't read another, // so we might as well run the handler in this goroutine. - // [*] Not strictly true: HTTP pipelining. We could let them all process + // [*] Not strictly true: HTTP pipelining. We could let them all process // in parallel even if their responses need to be serialized. + // But we're not going to implement HTTP pipelining because it + // was never deployed in the wild and the answer is HTTP/2. serverHandler{c.server}.ServeHTTP(w, w.req) + w.cancelCtx() if c.hijacked() { return } @@ -1481,6 +1835,23 @@ func (c *conn) serve() { return } c.setState(c.rwc, StateIdle) + c.curReq.Store((*response)(nil)) + + if !w.conn.server.doKeepAlives() { + // We're in shutdown mode. We might've replied + // to the user without "Connection: close" and + // they might think they can send another + // request, but such is life with HTTP/1.1. + return + } + + if d := c.server.idleTimeout(); d != 0 { + c.rwc.SetReadDeadline(time.Now().Add(d)) + if _, err := c.bufr.Peek(4); err != nil { + return + } + } + c.rwc.SetReadDeadline(time.Time{}) } } @@ -1488,7 +1859,7 @@ func (w *response) sendExpectationFailed() { // TODO(bradfitz): let ServeHTTP handlers handle // requests with non-standard expectation[s]? Seems // theoretical at best, and doesn't fit into the - // current ServeHTTP model anyway. We'd need to + // current ServeHTTP model anyway. We'd need to // make the ResponseWriter an optional // "ExpectReplier" interface or something. // @@ -1516,10 +1887,6 @@ func (w *response) Hijack() (rwc net.Conn, buf *bufio.ReadWriter, err error) { c.mu.Lock() defer c.mu.Unlock() - if w.closeNotifyCh != nil { - return nil, nil, errors.New("http: Hijack is incompatible with use of CloseNotifier in same ServeHTTP call") - } - // Release the bufioWriter that writes to the chunk writer, it is not // used after a connection has been hijacked. rwc, buf, err = c.hijackLocked() @@ -1534,50 +1901,7 @@ func (w *response) CloseNotify() <-chan bool { if w.handlerDone.isSet() { panic("net/http: CloseNotify called after ServeHTTP finished") } - c := w.conn - c.mu.Lock() - defer c.mu.Unlock() - - if w.closeNotifyCh != nil { - return w.closeNotifyCh - } - ch := make(chan bool, 1) - w.closeNotifyCh = ch - - if w.conn.hijackedv { - // CloseNotify is undefined after a hijack, but we have - // no place to return an error, so just return a channel, - // even though it'll never receive a value. - return ch - } - - var once sync.Once - notify := func() { once.Do(func() { ch <- true }) } - - if requestBodyRemains(w.reqBody) { - // They're still consuming the request body, so we - // shouldn't notify yet. - registerOnHitEOF(w.reqBody, func() { - c.mu.Lock() - defer c.mu.Unlock() - startCloseNotifyBackgroundRead(c, notify) - }) - } else { - startCloseNotifyBackgroundRead(c, notify) - } - return ch -} - -// c.mu must be held. -func startCloseNotifyBackgroundRead(c *conn, notify func()) { - if c.bufr.Buffered() > 0 { - // They've consumed the request body, so anything - // remaining is a pipelined request, which we - // document as firing on. - notify() - } else { - c.r.startBackgroundRead(notify) - } + return w.closeNotifyCh } func registerOnHitEOF(rc io.ReadCloser, fn func()) { @@ -1594,7 +1918,7 @@ func registerOnHitEOF(rc io.ReadCloser, fn func()) { // requestBodyRemains reports whether future calls to Read // on rc might yield more data. func requestBodyRemains(rc io.ReadCloser) bool { - if rc == eofReader { + if rc == NoBody { return false } switch v := rc.(type) { @@ -1608,7 +1932,7 @@ func requestBodyRemains(rc io.ReadCloser) bool { } // The HandlerFunc type is an adapter to allow the use of -// ordinary functions as HTTP handlers. If f is a function +// ordinary functions as HTTP handlers. If f is a function // with the appropriate signature, HandlerFunc(f) is a // Handler that calls f. type HandlerFunc func(ResponseWriter, *Request) @@ -1621,6 +1945,8 @@ func (f HandlerFunc) ServeHTTP(w ResponseWriter, r *Request) { // Helper handlers // Error replies to the request with the specified error message and HTTP code. +// It does not otherwise end the request; the caller should ensure no further +// writes are done to w. // The error message should be plain text. func Error(w ResponseWriter, error string, code int) { w.Header().Set("Content-Type", "text/plain; charset=utf-8") @@ -1706,10 +2032,10 @@ func Redirect(w ResponseWriter, r *Request, urlStr string, code int) { } } - w.Header().Set("Location", urlStr) + w.Header().Set("Location", hexEscapeNonASCII(urlStr)) w.WriteHeader(code) - // RFC2616 recommends that a short note "SHOULD" be included in the + // RFC 2616 recommends that a short note "SHOULD" be included in the // response because older user agents may not understand 301/307. // Shouldn't send the response for POST or HEAD; that leaves GET. if r.Method == "GET" { @@ -1779,7 +2105,7 @@ func RedirectHandler(url string, code int) Handler { // been registered separately. // // Patterns may optionally begin with a host name, restricting matches to -// URLs on that host only. Host-specific patterns take precedence over +// URLs on that host only. Host-specific patterns take precedence over // general patterns, so that a handler might register for the two patterns // "/codesearch" and "codesearch.google.com/" without also taking over // requests for "http://www.google.com/". @@ -1800,10 +2126,12 @@ type muxEntry struct { } // NewServeMux allocates and returns a new ServeMux. -func NewServeMux() *ServeMux { return &ServeMux{m: make(map[string]muxEntry)} } +func NewServeMux() *ServeMux { return new(ServeMux) } // DefaultServeMux is the default ServeMux used by Serve. -var DefaultServeMux = NewServeMux() +var DefaultServeMux = &defaultServeMux + +var defaultServeMux ServeMux // Does path match pattern? func pathMatch(pattern, path string) bool { @@ -1926,6 +2254,9 @@ func (mux *ServeMux) Handle(pattern string, handler Handler) { panic("http: multiple registrations for " + pattern) } + if mux.m == nil { + mux.m = make(map[string]muxEntry) + } mux.m[pattern] = muxEntry{explicit: true, h: handler, pattern: pattern} if pattern[0] != '/' { @@ -1968,7 +2299,7 @@ func HandleFunc(pattern string, handler func(ResponseWriter, *Request)) { } // Serve accepts incoming HTTP connections on the listener l, -// creating a new service goroutine for each. The service goroutines +// creating a new service goroutine for each. The service goroutines // read requests and then call handler to reply to them. // Handler is typically nil, in which case the DefaultServeMux is used. func Serve(l net.Listener, handler Handler) error { @@ -1979,21 +2310,53 @@ func Serve(l net.Listener, handler Handler) error { // A Server defines parameters for running an HTTP server. // The zero value for Server is a valid configuration. type Server struct { - Addr string // TCP address to listen on, ":http" if empty - Handler Handler // handler to invoke, http.DefaultServeMux if nil - ReadTimeout time.Duration // maximum duration before timing out read of the request - WriteTimeout time.Duration // maximum duration before timing out write of the response - MaxHeaderBytes int // maximum size of request headers, DefaultMaxHeaderBytes if 0 - TLSConfig *tls.Config // optional TLS config, used by ListenAndServeTLS + Addr string // TCP address to listen on, ":http" if empty + Handler Handler // handler to invoke, http.DefaultServeMux if nil + TLSConfig *tls.Config // optional TLS config, used by ListenAndServeTLS + + // ReadTimeout is the maximum duration for reading the entire + // request, including the body. + // + // Because ReadTimeout does not let Handlers make per-request + // decisions on each request body's acceptable deadline or + // upload rate, most users will prefer to use + // ReadHeaderTimeout. It is valid to use them both. + ReadTimeout time.Duration + + // ReadHeaderTimeout is the amount of time allowed to read + // request headers. The connection's read deadline is reset + // after reading the headers and the Handler can decide what + // is considered too slow for the body. + ReadHeaderTimeout time.Duration + + // WriteTimeout is the maximum duration before timing out + // writes of the response. It is reset whenever a new + // request's header is read. Like ReadTimeout, it does not + // let Handlers make decisions on a per-request basis. + WriteTimeout time.Duration + + // IdleTimeout is the maximum amount of time to wait for the + // next request when keep-alives are enabled. If IdleTimeout + // is zero, the value of ReadTimeout is used. If both are + // zero, there is no timeout. + IdleTimeout time.Duration + + // MaxHeaderBytes controls the maximum number of bytes the + // server will read parsing the request header's keys and + // values, including the request line. It does not limit the + // size of the request body. + // If zero, DefaultMaxHeaderBytes is used. + MaxHeaderBytes int // TLSNextProto optionally specifies a function to take over - // ownership of the provided TLS connection when an NPN - // protocol upgrade has occurred. The map key is the protocol + // ownership of the provided TLS connection when an NPN/ALPN + // protocol upgrade has occurred. The map key is the protocol // name negotiated. The Handler argument should be used to // handle HTTP requests and will initialize the Request's TLS - // and RemoteAddr if not already set. The connection is + // and RemoteAddr if not already set. The connection is // automatically closed when the function returns. - // If TLSNextProto is nil, HTTP/2 support is enabled automatically. + // If TLSNextProto is not nil, HTTP/2 support is not enabled + // automatically. TLSNextProto map[string]func(*Server, *tls.Conn, Handler) // ConnState specifies an optional callback function that is @@ -2008,8 +2371,132 @@ type Server struct { ErrorLog *log.Logger disableKeepAlives int32 // accessed atomically. - nextProtoOnce sync.Once // guards initialization of TLSNextProto in Serve - nextProtoErr error + inShutdown int32 // accessed atomically (non-zero means we're in Shutdown) + nextProtoOnce sync.Once // guards setupHTTP2_* init + nextProtoErr error // result of http2.ConfigureServer if used + + mu sync.Mutex + listeners map[net.Listener]struct{} + activeConn map[*conn]struct{} + doneChan chan struct{} +} + +func (s *Server) getDoneChan() <-chan struct{} { + s.mu.Lock() + defer s.mu.Unlock() + return s.getDoneChanLocked() +} + +func (s *Server) getDoneChanLocked() chan struct{} { + if s.doneChan == nil { + s.doneChan = make(chan struct{}) + } + return s.doneChan +} + +func (s *Server) closeDoneChanLocked() { + ch := s.getDoneChanLocked() + select { + case <-ch: + // Already closed. Don't close again. + default: + // Safe to close here. We're the only closer, guarded + // by s.mu. + close(ch) + } +} + +// Close immediately closes all active net.Listeners and any +// connections in state StateNew, StateActive, or StateIdle. For a +// graceful shutdown, use Shutdown. +// +// Close does not attempt to close (and does not even know about) +// any hijacked connections, such as WebSockets. +// +// Close returns any error returned from closing the Server's +// underlying Listener(s). +func (srv *Server) Close() error { + srv.mu.Lock() + defer srv.mu.Unlock() + srv.closeDoneChanLocked() + err := srv.closeListenersLocked() + for c := range srv.activeConn { + c.rwc.Close() + delete(srv.activeConn, c) + } + return err +} + +// shutdownPollInterval is how often we poll for quiescence +// during Server.Shutdown. This is lower during tests, to +// speed up tests. +// Ideally we could find a solution that doesn't involve polling, +// but which also doesn't have a high runtime cost (and doesn't +// involve any contentious mutexes), but that is left as an +// exercise for the reader. +var shutdownPollInterval = 500 * time.Millisecond + +// Shutdown gracefully shuts down the server without interrupting any +// active connections. Shutdown works by first closing all open +// listeners, then closing all idle connections, and then waiting +// indefinitely for connections to return to idle and then shut down. +// If the provided context expires before the shutdown is complete, +// then the context's error is returned. +// +// Shutdown does not attempt to close nor wait for hijacked +// connections such as WebSockets. The caller of Shutdown should +// separately notify such long-lived connections of shutdown and wait +// for them to close, if desired. +func (srv *Server) Shutdown(ctx context.Context) error { + atomic.AddInt32(&srv.inShutdown, 1) + defer atomic.AddInt32(&srv.inShutdown, -1) + + srv.mu.Lock() + lnerr := srv.closeListenersLocked() + srv.closeDoneChanLocked() + srv.mu.Unlock() + + ticker := time.NewTicker(shutdownPollInterval) + defer ticker.Stop() + for { + if srv.closeIdleConns() { + return lnerr + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + } + } +} + +// closeIdleConns closes all idle connections and reports whether the +// server is quiescent. +func (s *Server) closeIdleConns() bool { + s.mu.Lock() + defer s.mu.Unlock() + quiescent := true + for c := range s.activeConn { + st, ok := c.curState.Load().(ConnState) + if !ok || st != StateIdle { + quiescent = false + continue + } + c.rwc.Close() + delete(s.activeConn, c) + } + return quiescent +} + +func (s *Server) closeListenersLocked() error { + var err error + for ln := range s.listeners { + if cerr := ln.Close(); cerr != nil && err == nil { + err = cerr + } + delete(s.listeners, ln) + } + return err } // A ConnState represents the state of a client connection to a server. @@ -2032,7 +2519,7 @@ const ( // For HTTP/2, StateActive fires on the transition from zero // to one active request, and only transitions away once all // active requests are complete. That means that ConnState - // can not be used to do per-request work; ConnState only notes + // cannot be used to do per-request work; ConnState only notes // the overall state of the connection. StateActive @@ -2100,22 +2587,66 @@ func (srv *Server) ListenAndServe() error { var testHookServerServe func(*Server, net.Listener) // used if non-nil +// shouldDoServeHTTP2 reports whether Server.Serve should configure +// automatic HTTP/2. (which sets up the srv.TLSNextProto map) +func (srv *Server) shouldConfigureHTTP2ForServe() bool { + if srv.TLSConfig == nil { + // Compatibility with Go 1.6: + // If there's no TLSConfig, it's possible that the user just + // didn't set it on the http.Server, but did pass it to + // tls.NewListener and passed that listener to Serve. + // So we should configure HTTP/2 (to set up srv.TLSNextProto) + // in case the listener returns an "h2" *tls.Conn. + return true + } + // The user specified a TLSConfig on their http.Server. + // In this, case, only configure HTTP/2 if their tls.Config + // explicitly mentions "h2". Otherwise http2.ConfigureServer + // would modify the tls.Config to add it, but they probably already + // passed this tls.Config to tls.NewListener. And if they did, + // it's too late anyway to fix it. It would only be potentially racy. + // See Issue 15908. + return strSliceContains(srv.TLSConfig.NextProtos, http2NextProtoTLS) +} + +var ErrServerClosed = errors.New("http: Server closed") + // Serve accepts incoming connections on the Listener l, creating a // new service goroutine for each. The service goroutines read requests and // then call srv.Handler to reply to them. -// Serve always returns a non-nil error. +// +// For HTTP/2 support, srv.TLSConfig should be initialized to the +// provided listener's TLS Config before calling Serve. If +// srv.TLSConfig is non-nil and doesn't include the string "h2" in +// Config.NextProtos, HTTP/2 support is not enabled. +// +// Serve always returns a non-nil error. After Shutdown or Close, the +// returned error is ErrServerClosed. func (srv *Server) Serve(l net.Listener) error { defer l.Close() if fn := testHookServerServe; fn != nil { fn(srv, l) } var tempDelay time.Duration // how long to sleep on accept failure - if err := srv.setupHTTP2(); err != nil { + + if err := srv.setupHTTP2_Serve(); err != nil { return err } + + srv.trackListener(l, true) + defer srv.trackListener(l, false) + + baseCtx := context.Background() // base is always background, per Issue 16220 + ctx := context.WithValue(baseCtx, ServerContextKey, srv) + ctx = context.WithValue(ctx, LocalAddrContextKey, l.Addr()) for { rw, e := l.Accept() if e != nil { + select { + case <-srv.getDoneChan(): + return ErrServerClosed + default: + } if ne, ok := e.(net.Error); ok && ne.Temporary() { if tempDelay == 0 { tempDelay = 5 * time.Millisecond @@ -2134,12 +2665,61 @@ func (srv *Server) Serve(l net.Listener) error { tempDelay = 0 c := srv.newConn(rw) c.setState(c.rwc, StateNew) // before Serve can return - go c.serve() + go c.serve(ctx) + } +} + +func (s *Server) trackListener(ln net.Listener, add bool) { + s.mu.Lock() + defer s.mu.Unlock() + if s.listeners == nil { + s.listeners = make(map[net.Listener]struct{}) + } + if add { + // If the *Server is being reused after a previous + // Close or Shutdown, reset its doneChan: + if len(s.listeners) == 0 && len(s.activeConn) == 0 { + s.doneChan = nil + } + s.listeners[ln] = struct{}{} + } else { + delete(s.listeners, ln) } } +func (s *Server) trackConn(c *conn, add bool) { + s.mu.Lock() + defer s.mu.Unlock() + if s.activeConn == nil { + s.activeConn = make(map[*conn]struct{}) + } + if add { + s.activeConn[c] = struct{}{} + } else { + delete(s.activeConn, c) + } +} + +func (s *Server) idleTimeout() time.Duration { + if s.IdleTimeout != 0 { + return s.IdleTimeout + } + return s.ReadTimeout +} + +func (s *Server) readHeaderTimeout() time.Duration { + if s.ReadHeaderTimeout != 0 { + return s.ReadHeaderTimeout + } + return s.ReadTimeout +} + func (s *Server) doKeepAlives() bool { - return atomic.LoadInt32(&s.disableKeepAlives) == 0 + return atomic.LoadInt32(&s.disableKeepAlives) == 0 && !s.shuttingDown() +} + +func (s *Server) shuttingDown() bool { + return atomic.LoadInt32(&s.inShutdown) != 0 } // SetKeepAlivesEnabled controls whether HTTP keep-alives are enabled. @@ -2149,9 +2729,21 @@ func (s *Server) doKeepAlives() bool { func (srv *Server) SetKeepAlivesEnabled(v bool) { if v { atomic.StoreInt32(&srv.disableKeepAlives, 0) - } else { - atomic.StoreInt32(&srv.disableKeepAlives, 1) + return } + atomic.StoreInt32(&srv.disableKeepAlives, 1) + + // Close idle HTTP/1 conns: + srv.closeIdleConns() + + // Close HTTP/2 conns, as soon as they become idle, but reset + // the chan so future conns (if the listener is still active) + // still work and don't get a GOAWAY immediately, before their + // first request: + srv.mu.Lock() + defer srv.mu.Unlock() + srv.closeDoneChanLocked() // closes http2 conns + srv.doneChan = nil } func (s *Server) logf(format string, args ...interface{}) { @@ -2250,7 +2842,7 @@ func (srv *Server) ListenAndServeTLS(certFile, keyFile string) error { // Setup HTTP/2 before srv.Serve, to initialize srv.TLSConfig // before we clone it and create the TLS Listener. - if err := srv.setupHTTP2(); err != nil { + if err := srv.setupHTTP2_ListenAndServeTLS(); err != nil { return err } @@ -2278,14 +2870,36 @@ func (srv *Server) ListenAndServeTLS(certFile, keyFile string) error { return srv.Serve(tlsListener) } -func (srv *Server) setupHTTP2() error { +// setupHTTP2_ListenAndServeTLS conditionally configures HTTP/2 on +// srv and returns whether there was an error setting it up. If it is +// not configured for policy reasons, nil is returned. +func (srv *Server) setupHTTP2_ListenAndServeTLS() error { srv.nextProtoOnce.Do(srv.onceSetNextProtoDefaults) return srv.nextProtoErr } +// setupHTTP2_Serve is called from (*Server).Serve and conditionally +// configures HTTP/2 on srv using a more conservative policy than +// setupHTTP2_ListenAndServeTLS because Serve may be called +// concurrently. +// +// The tests named TestTransportAutomaticHTTP2* and +// TestConcurrentServerServe in server_test.go demonstrate some +// of the supported use cases and motivations. +func (srv *Server) setupHTTP2_Serve() error { + srv.nextProtoOnce.Do(srv.onceSetNextProtoDefaults_Serve) + return srv.nextProtoErr +} + +func (srv *Server) onceSetNextProtoDefaults_Serve() { + if srv.shouldConfigureHTTP2ForServe() { + srv.onceSetNextProtoDefaults() + } +} + // onceSetNextProtoDefaults configures HTTP/2, if the user hasn't // configured otherwise. (by setting srv.TLSNextProto non-nil) -// It must only be called via srv.nextProtoOnce (use srv.setupHTTP2). +// It must only be called via srv.nextProtoOnce (use srv.setupHTTP2_*). func (srv *Server) onceSetNextProtoDefaults() { if strings.Contains(os.Getenv("GODEBUG"), "http2server=0") { return @@ -2309,15 +2923,10 @@ func (srv *Server) onceSetNextProtoDefaults() { // TimeoutHandler buffers all Handler writes to memory and does not // support the Hijacker or Flusher interfaces. func TimeoutHandler(h Handler, dt time.Duration, msg string) Handler { - t := time.NewTimer(dt) return &timeoutHandler{ handler: h, body: msg, - - // Effectively storing a *time.Timer, but decomposed - // for testing: - timeout: func() <-chan time.Time { return t.C }, - cancelTimer: t.Stop, + dt: dt, } } @@ -2328,12 +2937,11 @@ var ErrHandlerTimeout = errors.New("http: Handler timeout") type timeoutHandler struct { handler Handler body string + dt time.Duration - // timeout returns the channel of a *time.Timer and - // cancelTimer cancels it. They're stored separately for - // testing purposes. - timeout func() <-chan time.Time // returns channel producing a timeout - cancelTimer func() bool // optional + // When set, no timer will be created and this channel will + // be used instead. + testTimeout <-chan time.Time } func (h *timeoutHandler) errorBody() string { @@ -2344,6 +2952,12 @@ func (h *timeoutHandler) errorBody() string { } func (h *timeoutHandler) ServeHTTP(w ResponseWriter, r *Request) { + var t *time.Timer + timeout := h.testTimeout + if timeout == nil { + t = time.NewTimer(h.dt) + timeout = t.C + } done := make(chan struct{}) tw := &timeoutWriter{ w: w, @@ -2361,12 +2975,15 @@ func (h *timeoutHandler) ServeHTTP(w ResponseWriter, r *Request) { for k, vv := range tw.h { dst[k] = vv } + if !tw.wroteHeader { + tw.code = StatusOK + } w.WriteHeader(tw.code) w.Write(tw.wbuf.Bytes()) - if h.cancelTimer != nil { - h.cancelTimer() + if t != nil { + t.Stop() } - case <-h.timeout(): + case <-timeout: tw.mu.Lock() defer tw.mu.Unlock() w.WriteHeader(StatusServiceUnavailable) @@ -2449,24 +3066,6 @@ func (globalOptionsHandler) ServeHTTP(w ResponseWriter, r *Request) { } } -type eofReaderWithWriteTo struct{} - -func (eofReaderWithWriteTo) WriteTo(io.Writer) (int64, error) { return 0, nil } -func (eofReaderWithWriteTo) Read([]byte) (int, error) { return 0, io.EOF } - -// eofReader is a non-nil io.ReadCloser that always returns EOF. -// It has a WriteTo method so io.Copy won't need a buffer. -var eofReader = &struct { - eofReaderWithWriteTo - io.Closer -}{ - eofReaderWithWriteTo{}, - ioutil.NopCloser(nil), -} - -// Verify that an io.Copy from an eofReader won't require a buffer. -var _ io.WriterTo = eofReader - // initNPNRequest is an HTTP handler that initializes certain // uninitialized fields in its *Request. Such partially-initialized // Requests come from NPN protocol handlers. @@ -2481,7 +3080,7 @@ func (h initNPNRequest) ServeHTTP(rw ResponseWriter, req *Request) { *req.TLS = h.c.ConnectionState() } if req.Body == nil { - req.Body = eofReader + req.Body = NoBody } if req.RemoteAddr == "" { req.RemoteAddr = h.c.RemoteAddr().String() @@ -2542,6 +3141,7 @@ func (w checkConnErrorWriter) Write(p []byte) (n int, err error) { n, err = w.c.rwc.Write(p) if err != nil && w.c.werr == nil { w.c.werr = err + w.c.cancelCtx() } return } diff --git a/libgo/go/net/http/sniff.go b/libgo/go/net/http/sniff.go index 18810bad06..0d21b44a56 100644 --- a/libgo/go/net/http/sniff.go +++ b/libgo/go/net/http/sniff.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -14,8 +14,8 @@ const sniffLen = 512 // DetectContentType implements the algorithm described // at http://mimesniff.spec.whatwg.org/ to determine the -// Content-Type of the given data. It considers at most the -// first 512 bytes of data. DetectContentType always returns +// Content-Type of the given data. It considers at most the +// first 512 bytes of data. DetectContentType always returns // a valid MIME type: if it cannot determine a more specific one, it // returns "application/octet-stream". func DetectContentType(data []byte) string { @@ -91,12 +91,41 @@ var sniffSignatures = []sniffSig{ ct: "image/webp", }, &exactSig{[]byte("\x00\x00\x01\x00"), "image/vnd.microsoft.icon"}, - &exactSig{[]byte("\x4F\x67\x67\x53\x00"), "application/ogg"}, &maskedSig{ mask: []byte("\xFF\xFF\xFF\xFF\x00\x00\x00\x00\xFF\xFF\xFF\xFF"), pat: []byte("RIFF\x00\x00\x00\x00WAVE"), ct: "audio/wave", }, + &maskedSig{ + mask: []byte("\xFF\xFF\xFF\xFF\x00\x00\x00\x00\xFF\xFF\xFF\xFF"), + pat: []byte("FORM\x00\x00\x00\x00AIFF"), + ct: "audio/aiff", + }, + &maskedSig{ + mask: []byte("\xFF\xFF\xFF\xFF"), + pat: []byte(".snd"), + ct: "audio/basic", + }, + &maskedSig{ + mask: []byte("OggS\x00"), + pat: []byte("\x4F\x67\x67\x53\x00"), + ct: "application/ogg", + }, + &maskedSig{ + mask: []byte("\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF"), + pat: []byte("MThd\x00\x00\x00\x06"), + ct: "audio/midi", + }, + &maskedSig{ + mask: []byte("\xFF\xFF\xFF"), + pat: []byte("ID3"), + ct: "audio/mpeg", + }, + &maskedSig{ + mask: []byte("\xFF\xFF\xFF\xFF\x00\x00\x00\x00\xFF\xFF\xFF\xFF"), + pat: []byte("RIFF\x00\x00\x00\x00AVI "), + ct: "video/avi", + }, &exactSig{[]byte("\x1A\x45\xDF\xA3"), "video/webm"}, &exactSig{[]byte("\x52\x61\x72\x20\x1A\x07\x00"), "application/x-rar-compressed"}, &exactSig{[]byte("\x50\x4B\x03\x04"), "application/zip"}, @@ -126,9 +155,15 @@ type maskedSig struct { } func (m *maskedSig) match(data []byte, firstNonWS int) string { + // pattern matching algorithm section 6 + // https://mimesniff.spec.whatwg.org/#pattern-matching-algorithm + if m.skipWS { data = data[firstNonWS:] } + if len(m.pat) != len(m.mask) { + return "" + } if len(data) < len(m.mask) { return "" } diff --git a/libgo/go/net/http/sniff_test.go b/libgo/go/net/http/sniff_test.go index e0085516da..38f3f8197e 100644 --- a/libgo/go/net/http/sniff_test.go +++ b/libgo/go/net/http/sniff_test.go @@ -39,7 +39,18 @@ var sniffTests = []struct { {"GIF 87a", []byte(`GIF87a`), "image/gif"}, {"GIF 89a", []byte(`GIF89a...`), "image/gif"}, + // Audio types. + {"MIDI audio", []byte("MThd\x00\x00\x00\x06\x00\x01"), "audio/midi"}, + {"MP3 audio/MPEG audio", []byte("ID3\x03\x00\x00\x00\x00\x0f"), "audio/mpeg"}, + {"WAV audio #1", []byte("RIFFb\xb8\x00\x00WAVEfmt \x12\x00\x00\x00\x06"), "audio/wave"}, + {"WAV audio #2", []byte("RIFF,\x00\x00\x00WAVEfmt \x12\x00\x00\x00\x06"), "audio/wave"}, + {"AIFF audio #1", []byte("FORM\x00\x00\x00\x00AIFFCOMM\x00\x00\x00\x12\x00\x01\x00\x00\x57\x55\x00\x10\x40\x0d\xf3\x34"), "audio/aiff"}, + {"OGG audio", []byte("OggS\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x7e\x46\x00\x00\x00\x00\x00\x00\x1f\xf6\xb4\xfc\x01\x1e\x01\x76\x6f\x72"), "application/ogg"}, + + // Video types. {"MP4 video", []byte("\x00\x00\x00\x18ftypmp42\x00\x00\x00\x00mp42isom<\x06t\xbfmdat"), "video/mp4"}, + {"AVI video #1", []byte("RIFF,O\n\x00AVI LISTÀ"), "video/avi"}, + {"AVI video #2", []byte("RIFF,\n\x00\x00AVI LISTÀ"), "video/avi"}, } func TestDetectContentType(t *testing.T) { @@ -55,6 +66,7 @@ func TestServerContentType_h1(t *testing.T) { testServerContentType(t, h1Mode) } func TestServerContentType_h2(t *testing.T) { testServerContentType(t, h2Mode) } func testServerContentType(t *testing.T, h2 bool) { + setParallel(t) defer afterTest(t) cst := newClientServerTest(t, h2, HandlerFunc(func(w ResponseWriter, r *Request) { i, _ := strconv.Atoi(r.FormValue("i")) @@ -149,6 +161,7 @@ func testContentTypeWithCopy(t *testing.T, h2 bool) { func TestSniffWriteSize_h1(t *testing.T) { testSniffWriteSize(t, h1Mode) } func TestSniffWriteSize_h2(t *testing.T) { testSniffWriteSize(t, h2Mode) } func testSniffWriteSize(t *testing.T, h2 bool) { + setParallel(t) defer afterTest(t) cst := newClientServerTest(t, h2, HandlerFunc(func(w ResponseWriter, r *Request) { size, _ := strconv.Atoi(r.FormValue("size")) diff --git a/libgo/go/net/http/status.go b/libgo/go/net/http/status.go index f3dacab6a9..98645b7d74 100644 --- a/libgo/go/net/http/status.go +++ b/libgo/go/net/http/status.go @@ -4,63 +4,79 @@ package http -// HTTP status codes, defined in RFC 2616. +// HTTP status codes as registered with IANA. +// See: http://www.iana.org/assignments/http-status-codes/http-status-codes.xhtml const ( - StatusContinue = 100 - StatusSwitchingProtocols = 101 + StatusContinue = 100 // RFC 7231, 6.2.1 + StatusSwitchingProtocols = 101 // RFC 7231, 6.2.2 + StatusProcessing = 102 // RFC 2518, 10.1 - StatusOK = 200 - StatusCreated = 201 - StatusAccepted = 202 - StatusNonAuthoritativeInfo = 203 - StatusNoContent = 204 - StatusResetContent = 205 - StatusPartialContent = 206 + StatusOK = 200 // RFC 7231, 6.3.1 + StatusCreated = 201 // RFC 7231, 6.3.2 + StatusAccepted = 202 // RFC 7231, 6.3.3 + StatusNonAuthoritativeInfo = 203 // RFC 7231, 6.3.4 + StatusNoContent = 204 // RFC 7231, 6.3.5 + StatusResetContent = 205 // RFC 7231, 6.3.6 + StatusPartialContent = 206 // RFC 7233, 4.1 + StatusMultiStatus = 207 // RFC 4918, 11.1 + StatusAlreadyReported = 208 // RFC 5842, 7.1 + StatusIMUsed = 226 // RFC 3229, 10.4.1 - StatusMultipleChoices = 300 - StatusMovedPermanently = 301 - StatusFound = 302 - StatusSeeOther = 303 - StatusNotModified = 304 - StatusUseProxy = 305 - StatusTemporaryRedirect = 307 + StatusMultipleChoices = 300 // RFC 7231, 6.4.1 + StatusMovedPermanently = 301 // RFC 7231, 6.4.2 + StatusFound = 302 // RFC 7231, 6.4.3 + StatusSeeOther = 303 // RFC 7231, 6.4.4 + StatusNotModified = 304 // RFC 7232, 4.1 + StatusUseProxy = 305 // RFC 7231, 6.4.5 + _ = 306 // RFC 7231, 6.4.6 (Unused) + StatusTemporaryRedirect = 307 // RFC 7231, 6.4.7 + StatusPermanentRedirect = 308 // RFC 7538, 3 - StatusBadRequest = 400 - StatusUnauthorized = 401 - StatusPaymentRequired = 402 - StatusForbidden = 403 - StatusNotFound = 404 - StatusMethodNotAllowed = 405 - StatusNotAcceptable = 406 - StatusProxyAuthRequired = 407 - StatusRequestTimeout = 408 - StatusConflict = 409 - StatusGone = 410 - StatusLengthRequired = 411 - StatusPreconditionFailed = 412 - StatusRequestEntityTooLarge = 413 - StatusRequestURITooLong = 414 - StatusUnsupportedMediaType = 415 - StatusRequestedRangeNotSatisfiable = 416 - StatusExpectationFailed = 417 - StatusTeapot = 418 - StatusPreconditionRequired = 428 - StatusTooManyRequests = 429 - StatusRequestHeaderFieldsTooLarge = 431 - StatusUnavailableForLegalReasons = 451 + StatusBadRequest = 400 // RFC 7231, 6.5.1 + StatusUnauthorized = 401 // RFC 7235, 3.1 + StatusPaymentRequired = 402 // RFC 7231, 6.5.2 + StatusForbidden = 403 // RFC 7231, 6.5.3 + StatusNotFound = 404 // RFC 7231, 6.5.4 + StatusMethodNotAllowed = 405 // RFC 7231, 6.5.5 + StatusNotAcceptable = 406 // RFC 7231, 6.5.6 + StatusProxyAuthRequired = 407 // RFC 7235, 3.2 + StatusRequestTimeout = 408 // RFC 7231, 6.5.7 + StatusConflict = 409 // RFC 7231, 6.5.8 + StatusGone = 410 // RFC 7231, 6.5.9 + StatusLengthRequired = 411 // RFC 7231, 6.5.10 + StatusPreconditionFailed = 412 // RFC 7232, 4.2 + StatusRequestEntityTooLarge = 413 // RFC 7231, 6.5.11 + StatusRequestURITooLong = 414 // RFC 7231, 6.5.12 + StatusUnsupportedMediaType = 415 // RFC 7231, 6.5.13 + StatusRequestedRangeNotSatisfiable = 416 // RFC 7233, 4.4 + StatusExpectationFailed = 417 // RFC 7231, 6.5.14 + StatusTeapot = 418 // RFC 7168, 2.3.3 + StatusUnprocessableEntity = 422 // RFC 4918, 11.2 + StatusLocked = 423 // RFC 4918, 11.3 + StatusFailedDependency = 424 // RFC 4918, 11.4 + StatusUpgradeRequired = 426 // RFC 7231, 6.5.15 + StatusPreconditionRequired = 428 // RFC 6585, 3 + StatusTooManyRequests = 429 // RFC 6585, 4 + StatusRequestHeaderFieldsTooLarge = 431 // RFC 6585, 5 + StatusUnavailableForLegalReasons = 451 // RFC 7725, 3 - StatusInternalServerError = 500 - StatusNotImplemented = 501 - StatusBadGateway = 502 - StatusServiceUnavailable = 503 - StatusGatewayTimeout = 504 - StatusHTTPVersionNotSupported = 505 - StatusNetworkAuthenticationRequired = 511 + StatusInternalServerError = 500 // RFC 7231, 6.6.1 + StatusNotImplemented = 501 // RFC 7231, 6.6.2 + StatusBadGateway = 502 // RFC 7231, 6.6.3 + StatusServiceUnavailable = 503 // RFC 7231, 6.6.4 + StatusGatewayTimeout = 504 // RFC 7231, 6.6.5 + StatusHTTPVersionNotSupported = 505 // RFC 7231, 6.6.6 + StatusVariantAlsoNegotiates = 506 // RFC 2295, 8.1 + StatusInsufficientStorage = 507 // RFC 4918, 11.5 + StatusLoopDetected = 508 // RFC 5842, 7.2 + StatusNotExtended = 510 // RFC 2774, 7 + StatusNetworkAuthenticationRequired = 511 // RFC 6585, 6 ) var statusText = map[int]string{ StatusContinue: "Continue", StatusSwitchingProtocols: "Switching Protocols", + StatusProcessing: "Processing", StatusOK: "OK", StatusCreated: "Created", @@ -69,6 +85,9 @@ var statusText = map[int]string{ StatusNoContent: "No Content", StatusResetContent: "Reset Content", StatusPartialContent: "Partial Content", + StatusMultiStatus: "Multi-Status", + StatusAlreadyReported: "Already Reported", + StatusIMUsed: "IM Used", StatusMultipleChoices: "Multiple Choices", StatusMovedPermanently: "Moved Permanently", @@ -77,6 +96,7 @@ var statusText = map[int]string{ StatusNotModified: "Not Modified", StatusUseProxy: "Use Proxy", StatusTemporaryRedirect: "Temporary Redirect", + StatusPermanentRedirect: "Permanent Redirect", StatusBadRequest: "Bad Request", StatusUnauthorized: "Unauthorized", @@ -97,6 +117,10 @@ var statusText = map[int]string{ StatusRequestedRangeNotSatisfiable: "Requested Range Not Satisfiable", StatusExpectationFailed: "Expectation Failed", StatusTeapot: "I'm a teapot", + StatusUnprocessableEntity: "Unprocessable Entity", + StatusLocked: "Locked", + StatusFailedDependency: "Failed Dependency", + StatusUpgradeRequired: "Upgrade Required", StatusPreconditionRequired: "Precondition Required", StatusTooManyRequests: "Too Many Requests", StatusRequestHeaderFieldsTooLarge: "Request Header Fields Too Large", @@ -108,6 +132,10 @@ var statusText = map[int]string{ StatusServiceUnavailable: "Service Unavailable", StatusGatewayTimeout: "Gateway Timeout", StatusHTTPVersionNotSupported: "HTTP Version Not Supported", + StatusVariantAlsoNegotiates: "Variant Also Negotiates", + StatusInsufficientStorage: "Insufficient Storage", + StatusLoopDetected: "Loop Detected", + StatusNotExtended: "Not Extended", StatusNetworkAuthenticationRequired: "Network Authentication Required", } diff --git a/libgo/go/net/http/transfer.go b/libgo/go/net/http/transfer.go index 6e59af8f6f..4f47637aa7 100644 --- a/libgo/go/net/http/transfer.go +++ b/libgo/go/net/http/transfer.go @@ -17,6 +17,9 @@ import ( "strconv" "strings" "sync" + "time" + + "golang_org/x/net/lex/httplex" ) // ErrLineTooLong is returned when reading request or response bodies @@ -31,6 +34,23 @@ func (r errorReader) Read(p []byte) (n int, err error) { return 0, r.err } +type byteReader struct { + b byte + done bool +} + +func (br *byteReader) Read(p []byte) (n int, err error) { + if br.done { + return 0, io.EOF + } + if len(p) == 0 { + return 0, nil + } + br.done = true + p[0] = br.b + return 1, io.EOF +} + // transferWriter inspects the fields of a user-supplied Request or Response, // sanitizes them without changing the user object and provides methods for // writing the respective header, body and trailer in wire format. @@ -44,6 +64,9 @@ type transferWriter struct { TransferEncoding []string Trailer Header IsResponse bool + + FlushHeaders bool // flush headers to network before body + ByteReadCh chan readResult // non-nil if probeRequestBody called } func newTransferWriter(r interface{}) (t *transferWriter, err error) { @@ -57,37 +80,15 @@ func newTransferWriter(r interface{}) (t *transferWriter, err error) { return nil, fmt.Errorf("http: Request.ContentLength=%d with nil Body", rr.ContentLength) } t.Method = valueOrDefault(rr.Method, "GET") - t.Body = rr.Body - t.BodyCloser = rr.Body - t.ContentLength = rr.ContentLength t.Close = rr.Close t.TransferEncoding = rr.TransferEncoding t.Trailer = rr.Trailer - atLeastHTTP11 = rr.ProtoAtLeast(1, 1) - if t.Body != nil && len(t.TransferEncoding) == 0 && atLeastHTTP11 { - if t.ContentLength == 0 { - // Test to see if it's actually zero or just unset. - var buf [1]byte - n, rerr := io.ReadFull(t.Body, buf[:]) - if rerr != nil && rerr != io.EOF { - t.ContentLength = -1 - t.Body = errorReader{rerr} - } else if n == 1 { - // Oh, guess there is data in this Body Reader after all. - // The ContentLength field just wasn't set. - // Stich the Body back together again, re-attaching our - // consumed byte. - t.ContentLength = -1 - t.Body = io.MultiReader(bytes.NewReader(buf[:]), t.Body) - } else { - // Body is actually empty. - t.Body = nil - t.BodyCloser = nil - } - } - if t.ContentLength < 0 { - t.TransferEncoding = []string{"chunked"} - } + atLeastHTTP11 = rr.protoAtLeastOutgoing(1, 1) + t.Body = rr.Body + t.BodyCloser = rr.Body + t.ContentLength = rr.outgoingLength() + if t.ContentLength < 0 && len(t.TransferEncoding) == 0 && atLeastHTTP11 && t.shouldSendChunkedRequestBody() { + t.TransferEncoding = []string{"chunked"} } case *Response: t.IsResponse = true @@ -101,7 +102,7 @@ func newTransferWriter(r interface{}) (t *transferWriter, err error) { t.TransferEncoding = rr.TransferEncoding t.Trailer = rr.Trailer atLeastHTTP11 = rr.ProtoAtLeast(1, 1) - t.ResponseToHEAD = noBodyExpected(t.Method) + t.ResponseToHEAD = noResponseBodyExpected(t.Method) } // Sanitize Body,ContentLength,TransferEncoding @@ -129,7 +130,100 @@ func newTransferWriter(r interface{}) (t *transferWriter, err error) { return t, nil } -func noBodyExpected(requestMethod string) bool { +// shouldSendChunkedRequestBody reports whether we should try to send a +// chunked request body to the server. In particular, the case we really +// want to prevent is sending a GET or other typically-bodyless request to a +// server with a chunked body when the body has zero bytes, since GETs with +// bodies (while acceptable according to specs), even zero-byte chunked +// bodies, are approximately never seen in the wild and confuse most +// servers. See Issue 18257, as one example. +// +// The only reason we'd send such a request is if the user set the Body to a +// non-nil value (say, ioutil.NopCloser(bytes.NewReader(nil))) and didn't +// set ContentLength, or NewRequest set it to -1 (unknown), so then we assume +// there's bytes to send. +// +// This code tries to read a byte from the Request.Body in such cases to see +// whether the body actually has content (super rare) or is actually just +// a non-nil content-less ReadCloser (the more common case). In that more +// common case, we act as if their Body were nil instead, and don't send +// a body. +func (t *transferWriter) shouldSendChunkedRequestBody() bool { + // Note that t.ContentLength is the corrected content length + // from rr.outgoingLength, so 0 actually means zero, not unknown. + if t.ContentLength >= 0 || t.Body == nil { // redundant checks; caller did them + return false + } + if requestMethodUsuallyLacksBody(t.Method) { + // Only probe the Request.Body for GET/HEAD/DELETE/etc + // requests, because it's only those types of requests + // that confuse servers. + t.probeRequestBody() // adjusts t.Body, t.ContentLength + return t.Body != nil + } + // For all other request types (PUT, POST, PATCH, or anything + // made-up we've never heard of), assume it's normal and the server + // can deal with a chunked request body. Maybe we'll adjust this + // later. + return true +} + +// probeRequestBody reads a byte from t.Body to see whether it's empty +// (returns io.EOF right away). +// +// But because we've had problems with this blocking users in the past +// (issue 17480) when the body is a pipe (perhaps waiting on the response +// headers before the pipe is fed data), we need to be careful and bound how +// long we wait for it. This delay will only affect users if all the following +// are true: +// * the request body blocks +// * the content length is not set (or set to -1) +// * the method doesn't usually have a body (GET, HEAD, DELETE, ...) +// * there is no transfer-encoding=chunked already set. +// In other words, this delay will not normally affect anybody, and there +// are workarounds if it does. +func (t *transferWriter) probeRequestBody() { + t.ByteReadCh = make(chan readResult, 1) + go func(body io.Reader) { + var buf [1]byte + var rres readResult + rres.n, rres.err = body.Read(buf[:]) + if rres.n == 1 { + rres.b = buf[0] + } + t.ByteReadCh <- rres + }(t.Body) + timer := time.NewTimer(200 * time.Millisecond) + select { + case rres := <-t.ByteReadCh: + timer.Stop() + if rres.n == 0 && rres.err == io.EOF { + // It was empty. + t.Body = nil + t.ContentLength = 0 + } else if rres.n == 1 { + if rres.err != nil { + t.Body = io.MultiReader(&byteReader{b: rres.b}, errorReader{rres.err}) + } else { + t.Body = io.MultiReader(&byteReader{b: rres.b}, t.Body) + } + } else if rres.err != nil { + t.Body = errorReader{rres.err} + } + case <-timer.C: + // Too slow. Don't wait. Read it later, and keep + // assuming that this is ContentLength == -1 + // (unknown), which means we'll send a + // "Transfer-Encoding: chunked" header. + t.Body = io.MultiReader(finishAsyncByteRead{t}, t.Body) + // Request that Request.Write flush the headers to the + // network before writing the body, since our body may not + // become readable until it's seen the response headers. + t.FlushHeaders = true + } +} + +func noResponseBodyExpected(requestMethod string) bool { return requestMethod == "HEAD" } @@ -212,7 +306,7 @@ func (t *transferWriter) WriteBody(w io.Writer) error { if t.Body != nil { if chunked(t.TransferEncoding) { if bw, ok := w.(*bufio.Writer); ok && !t.IsResponse { - w = &internal.FlushAfterChunkWriter{bw} + w = &internal.FlushAfterChunkWriter{Writer: bw} } cw := internal.NewChunkedWriter(w) _, err = io.Copy(cw, t.Body) @@ -233,7 +327,9 @@ func (t *transferWriter) WriteBody(w io.Writer) error { if err != nil { return err } - if err = t.BodyCloser.Close(); err != nil { + } + if t.BodyCloser != nil { + if err := t.BodyCloser.Close(); err != nil { return err } } @@ -276,7 +372,7 @@ func (t *transferReader) protoAtLeast(m, n int) bool { } // bodyAllowedForStatus reports whether a given response status code -// permits a body. See RFC2616, section 4.4. +// permits a body. See RFC 2616, section 4.4. func bodyAllowedForStatus(status int) bool { switch { case status >= 100 && status <= 199: @@ -368,7 +464,7 @@ func readTransfer(msg interface{}, r *bufio.Reader) (err error) { // If there is no Content-Length or chunked Transfer-Encoding on a *Response // and the status is not 1xx, 204 or 304, then the body is unbounded. - // See RFC2616, section 4.4. + // See RFC 2616, section 4.4. switch msg.(type) { case *Response: if realLength == -1 && @@ -379,17 +475,17 @@ func readTransfer(msg interface{}, r *bufio.Reader) (err error) { } } - // Prepare body reader. ContentLength < 0 means chunked encoding + // Prepare body reader. ContentLength < 0 means chunked encoding // or close connection when finished, since multipart is not supported yet switch { case chunked(t.TransferEncoding): - if noBodyExpected(t.RequestMethod) { - t.Body = eofReader + if noResponseBodyExpected(t.RequestMethod) { + t.Body = NoBody } else { t.Body = &body{src: internal.NewChunkedReader(r), hdr: msg, r: r, closing: t.Close} } case realLength == 0: - t.Body = eofReader + t.Body = NoBody case realLength > 0: t.Body = &body{src: io.LimitReader(r, realLength), closing: t.Close} default: @@ -399,7 +495,7 @@ func readTransfer(msg interface{}, r *bufio.Reader) (err error) { t.Body = &body{src: r, closing: t.Close} } else { // Persistent connection (i.e. HTTP/1.1) - t.Body = eofReader + t.Body = NoBody } } @@ -491,10 +587,31 @@ func (t *transferReader) fixTransferEncoding() error { // function is not a method, because ultimately it should be shared by // ReadResponse and ReadRequest. func fixLength(isResponse bool, status int, requestMethod string, header Header, te []string) (int64, error) { - contentLens := header["Content-Length"] isRequest := !isResponse + contentLens := header["Content-Length"] + + // Hardening against HTTP request smuggling + if len(contentLens) > 1 { + // Per RFC 7230 Section 3.3.2, prevent multiple + // Content-Length headers if they differ in value. + // If there are dups of the value, remove the dups. + // See Issue 16490. + first := strings.TrimSpace(contentLens[0]) + for _, ct := range contentLens[1:] { + if first != strings.TrimSpace(ct) { + return 0, fmt.Errorf("http: message cannot contain multiple Content-Length headers; got %q", contentLens) + } + } + + // deduplicate Content-Length + header.Del("Content-Length") + header.Add("Content-Length", first) + + contentLens = header["Content-Length"] + } + // Logic based on response type or status - if noBodyExpected(requestMethod) { + if noResponseBodyExpected(requestMethod) { // For HTTP requests, as part of hardening against request // smuggling (RFC 7230), don't allow a Content-Length header for // methods which don't permit bodies. As an exception, allow @@ -512,11 +629,6 @@ func fixLength(isResponse bool, status int, requestMethod string, header Header, return 0, nil } - if len(contentLens) > 1 { - // harden against HTTP request smuggling. See RFC 7230. - return 0, errors.New("http: message cannot contain multiple Content-Length headers") - } - // Logic based on Transfer-Encoding if chunked(te) { return -1, nil @@ -537,7 +649,7 @@ func fixLength(isResponse bool, status int, requestMethod string, header Header, header.Del("Content-Length") } - if !isResponse { + if isRequest { // RFC 2616 neither explicitly permits nor forbids an // entity-body on a GET request so we permit one if // declared, but we default to 0 here (not -1 below) @@ -558,21 +670,19 @@ func fixLength(isResponse bool, status int, requestMethod string, header Header, func shouldClose(major, minor int, header Header, removeCloseHeader bool) bool { if major < 1 { return true - } else if major == 1 && minor == 0 { - vv := header["Connection"] - if headerValuesContainsToken(vv, "close") || !headerValuesContainsToken(vv, "keep-alive") { - return true - } - return false - } else { - if headerValuesContainsToken(header["Connection"], "close") { - if removeCloseHeader { - header.Del("Connection") - } - return true - } } - return false + + conv := header["Connection"] + hasClose := httplex.HeaderValuesContainsToken(conv, "close") + if major == 1 && minor == 0 { + return hasClose || !httplex.HeaderValuesContainsToken(conv, "keep-alive") + } + + if hasClose && removeCloseHeader { + header.Del("Connection") + } + + return hasClose } // Parse the trailer header @@ -729,11 +839,11 @@ func (b *body) readTrailer() error { } // Make sure there's a header terminator coming up, to prevent - // a DoS with an unbounded size Trailer. It's not easy to + // a DoS with an unbounded size Trailer. It's not easy to // slip in a LimitReader here, as textproto.NewReader requires - // a concrete *bufio.Reader. Also, we can't get all the way + // a concrete *bufio.Reader. Also, we can't get all the way // back up to our conn's LimitedReader that *might* be backing - // this bufio.Reader. Instead, a hack: we iteratively Peek up + // this bufio.Reader. Instead, a hack: we iteratively Peek up // to the bufio.Reader's max size, looking for a double CRLF. // This limits the trailer to the underlying buffer size, typically 4kB. if !seeUpcomingDoubleCRLF(b.r) { @@ -864,3 +974,21 @@ func parseContentLength(cl string) (int64, error) { return n, nil } + +// finishAsyncByteRead finishes reading the 1-byte sniff +// from the ContentLength==0, Body!=nil case. +type finishAsyncByteRead struct { + tw *transferWriter +} + +func (fr finishAsyncByteRead) Read(p []byte) (n int, err error) { + if len(p) == 0 { + return + } + rres := <-fr.tw.ByteReadCh + n, err = rres.n, rres.err + if n == 1 { + p[0] = rres.b + } + return +} diff --git a/libgo/go/net/http/transport.go b/libgo/go/net/http/transport.go index baf71d5e85..571943d6e5 100644 --- a/libgo/go/net/http/transport.go +++ b/libgo/go/net/http/transport.go @@ -12,17 +12,23 @@ package http import ( "bufio" "compress/gzip" + "container/list" + "context" "crypto/tls" "errors" "fmt" "io" "log" "net" + "net/http/httptrace" "net/url" "os" "strings" "sync" + "sync/atomic" "time" + + "golang_org/x/net/lex/httplex" ) // DefaultTransport is the default implementation of Transport and is @@ -32,10 +38,13 @@ import ( // $no_proxy) environment variables. var DefaultTransport RoundTripper = &Transport{ Proxy: ProxyFromEnvironment, - Dial: (&net.Dialer{ + DialContext: (&net.Dialer{ Timeout: 30 * time.Second, KeepAlive: 30 * time.Second, - }).Dial, + DualStack: true, + }).DialContext, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, TLSHandshakeTimeout: 10 * time.Second, ExpectContinueTimeout: 1 * time.Second, } @@ -59,19 +68,22 @@ const DefaultMaxIdleConnsPerHost = 2 // For high-level functionality, such as cookies and redirects, see Client. // // Transport uses HTTP/1.1 for HTTP URLs and either HTTP/1.1 or HTTP/2 -// for HTTPS URLs, depending on whether the server supports HTTP/2. -// See the package docs for more about HTTP/2. +// for HTTPS URLs, depending on whether the server supports HTTP/2, +// and how the Transport is configured. The DefaultTransport supports HTTP/2. +// To explicitly enable HTTP/2 on a transport, use golang.org/x/net/http2 +// and call ConfigureTransport. See the package docs for more about HTTP/2. type Transport struct { idleMu sync.Mutex - wantIdle bool // user has requested to close all idle conns - idleConn map[connectMethodKey][]*persistConn + wantIdle bool // user has requested to close all idle conns + idleConn map[connectMethodKey][]*persistConn // most recently used at end idleConnCh map[connectMethodKey]chan *persistConn + idleLRU connLRU reqMu sync.Mutex - reqCanceler map[*Request]func() + reqCanceler map[*Request]func(error) - altMu sync.RWMutex - altProto map[string]RoundTripper // nil or map of URI scheme => RoundTripper + altMu sync.Mutex // guards changing altProto only + altProto atomic.Value // of nil or map[string]RoundTripper, key is URI scheme // Proxy specifies a function to return a proxy for a given // Request. If the function returns a non-nil error, the @@ -79,9 +91,16 @@ type Transport struct { // If Proxy is nil or returns a nil *URL, no proxy is used. Proxy func(*Request) (*url.URL, error) - // Dial specifies the dial function for creating unencrypted - // TCP connections. - // If Dial is nil, net.Dial is used. + // DialContext specifies the dial function for creating unencrypted TCP connections. + // If DialContext is nil (and the deprecated Dial below is also nil), + // then the transport dials using package net. + DialContext func(ctx context.Context, network, addr string) (net.Conn, error) + + // Dial specifies the dial function for creating unencrypted TCP connections. + // + // Deprecated: Use DialContext instead, which allows the transport + // to cancel dials as soon as they are no longer needed. + // If both are set, DialContext takes priority. Dial func(network, addr string) (net.Conn, error) // DialTLS specifies an optional dial function for creating @@ -96,7 +115,9 @@ type Transport struct { DialTLS func(network, addr string) (net.Conn, error) // TLSClientConfig specifies the TLS configuration to use with - // tls.Client. If nil, the default configuration is used. + // tls.Client. + // If nil, the default configuration is used. + // If non-nil, HTTP/2 support may not be enabled by default. TLSClientConfig *tls.Config // TLSHandshakeTimeout specifies the maximum amount of time waiting to @@ -117,11 +138,21 @@ type Transport struct { // uncompressed. DisableCompression bool + // MaxIdleConns controls the maximum number of idle (keep-alive) + // connections across all hosts. Zero means no limit. + MaxIdleConns int + // MaxIdleConnsPerHost, if non-zero, controls the maximum idle - // (keep-alive) to keep per-host. If zero, + // (keep-alive) connections to keep per-host. If zero, // DefaultMaxIdleConnsPerHost is used. MaxIdleConnsPerHost int + // IdleConnTimeout is the maximum amount of time an idle + // (keep-alive) connection will remain idle before closing + // itself. + // Zero means no limit. + IdleConnTimeout time.Duration + // ResponseHeaderTimeout, if non-zero, specifies the amount of // time to wait for a server's response headers after fully // writing the request (including its body, if any). This @@ -131,28 +162,40 @@ type Transport struct { // ExpectContinueTimeout, if non-zero, specifies the amount of // time to wait for a server's first response headers after fully // writing the request headers if the request has an - // "Expect: 100-continue" header. Zero means no timeout. + // "Expect: 100-continue" header. Zero means no timeout and + // causes the body to be sent immediately, without + // waiting for the server to approve. // This time does not include the time to send the request header. ExpectContinueTimeout time.Duration // TLSNextProto specifies how the Transport switches to an // alternate protocol (such as HTTP/2) after a TLS NPN/ALPN - // protocol negotiation. If Transport dials an TLS connection + // protocol negotiation. If Transport dials an TLS connection // with a non-empty protocol name and TLSNextProto contains a // map entry for that key (such as "h2"), then the func is // called with the request's authority (such as "example.com" // or "example.com:1234") and the TLS connection. The function // must return a RoundTripper that then handles the request. - // If TLSNextProto is nil, HTTP/2 support is enabled automatically. + // If TLSNextProto is not nil, HTTP/2 support is not enabled + // automatically. TLSNextProto map[string]func(authority string, c *tls.Conn) RoundTripper + // ProxyConnectHeader optionally specifies headers to send to + // proxies during CONNECT requests. + ProxyConnectHeader Header + + // MaxResponseHeaderBytes specifies a limit on how many + // response bytes are allowed in the server's response + // header. + // + // Zero means to use a default limit. + MaxResponseHeaderBytes int64 + // nextProtoOnce guards initialization of TLSNextProto and // h2transport (via onceSetNextProtoDefaults) nextProtoOnce sync.Once h2transport *http2Transport // non-nil if http2 wired up - // TODO: tunable on global max cached connections - // TODO: tunable on timeout on cached connections // TODO: tunable on max per-host TCP dials in flight (Issue 13957) } @@ -167,25 +210,34 @@ func (t *Transport) onceSetNextProtoDefaults() { // Transport. return } - if t.TLSClientConfig != nil { - // Be conservative for now (for Go 1.6) at least and - // don't automatically enable http2 if they've - // specified a custom TLS config. Let them opt-in - // themselves via http2.ConfigureTransport so we don't - // surprise them by modifying their tls.Config. - // Issue 14275. - return - } - if t.ExpectContinueTimeout != 0 { - // Unsupported in http2, so disable http2 for now. - // Issue 13851. + if t.TLSClientConfig != nil || t.Dial != nil || t.DialTLS != nil { + // Be conservative and don't automatically enable + // http2 if they've specified a custom TLS config or + // custom dialers. Let them opt-in themselves via + // http2.ConfigureTransport so we don't surprise them + // by modifying their tls.Config. Issue 14275. return } t2, err := http2configureTransport(t) if err != nil { log.Printf("Error enabling Transport HTTP/2 support: %v", err) - } else { - t.h2transport = t2 + return + } + t.h2transport = t2 + + // Auto-configure the http2.Transport's MaxHeaderListSize from + // the http.Transport's MaxResponseHeaderBytes. They don't + // exactly mean the same thing, but they're close. + // + // TODO: also add this to x/net/http2.Configure Transport, behind + // a +build go1.7 build tag: + if limit1 := t.MaxResponseHeaderBytes; limit1 != 0 && t2.MaxHeaderListSize == 0 { + const h2max = 1<<32 - 1 + if limit1 >= h2max { + t2.MaxHeaderListSize = h2max + } else { + t2.MaxHeaderListSize = uint32(limit1) + } } } @@ -212,6 +264,9 @@ func ProxyFromEnvironment(req *Request) (*url.URL, error) { } if proxy == "" { proxy = httpProxyEnv.Get() + if proxy != "" && os.Getenv("REQUEST_METHOD") != "" { + return nil, errors.New("net/http: refusing to use HTTP_PROXY value in CGI environment; see golang.org/s/cgihttpproxy") + } } if proxy == "" { return nil, nil @@ -245,8 +300,9 @@ func ProxyURL(fixedURL *url.URL) func(*Request) (*url.URL, error) { // transportRequest is a wrapper around a *Request that adds // optional extra headers to write. type transportRequest struct { - *Request // original request, not to be mutated - extra Header // extra headers to write, or nil + *Request // original request, not to be mutated + extra Header // extra headers to write, or nil + trace *httptrace.ClientTrace // optional } func (tr *transportRequest) extraHeaders() Header { @@ -262,6 +318,9 @@ func (tr *transportRequest) extraHeaders() Header { // and redirects), see Get, Post, and the Client type. func (t *Transport) RoundTrip(req *Request) (*Response, error) { t.nextProtoOnce.Do(t.onceSetNextProtoDefaults) + ctx := req.Context() + trace := httptrace.ContextClientTrace(ctx) + if req.URL == nil { req.closeBody() return nil, errors.New("http: nil Request.URL") @@ -270,18 +329,30 @@ func (t *Transport) RoundTrip(req *Request) (*Response, error) { req.closeBody() return nil, errors.New("http: nil Request.Header") } - // TODO(bradfitz): switch to atomic.Value for this map instead of RWMutex - t.altMu.RLock() - altRT := t.altProto[req.URL.Scheme] - t.altMu.RUnlock() - if altRT != nil { + scheme := req.URL.Scheme + isHTTP := scheme == "http" || scheme == "https" + if isHTTP { + for k, vv := range req.Header { + if !httplex.ValidHeaderFieldName(k) { + return nil, fmt.Errorf("net/http: invalid header field name %q", k) + } + for _, v := range vv { + if !httplex.ValidHeaderFieldValue(v) { + return nil, fmt.Errorf("net/http: invalid header field value %q for key %v", v, k) + } + } + } + } + + altProto, _ := t.altProto.Load().(map[string]RoundTripper) + if altRT := altProto[scheme]; altRT != nil { if resp, err := altRT.RoundTrip(req); err != ErrSkipAltProtocol { return resp, err } } - if s := req.URL.Scheme; s != "http" && s != "https" { + if !isHTTP { req.closeBody() - return nil, &badStringError{"unsupported protocol scheme", s} + return nil, &badStringError{"unsupported protocol scheme", scheme} } if req.Method != "" && !validMethod(req.Method) { return nil, fmt.Errorf("net/http: invalid method %q", req.Method) @@ -293,7 +364,7 @@ func (t *Transport) RoundTrip(req *Request) (*Response, error) { for { // treq gets modified by roundTrip, so we need to recreate for each retry. - treq := &transportRequest{Request: req} + treq := &transportRequest{Request: req, trace: trace} cm, err := t.connectMethodForRequest(treq) if err != nil { req.closeBody() @@ -302,9 +373,9 @@ func (t *Transport) RoundTrip(req *Request) (*Response, error) { // Get the cached or newly-created connection to either the // host (for http or https), the http proxy, or the http proxy - // pre-CONNECTed to https server. In any case, we'll be ready + // pre-CONNECTed to https server. In any case, we'll be ready // to send it requests. - pconn, err := t.getConn(req, cm) + pconn, err := t.getConn(treq, cm) if err != nil { t.setReqCanceler(req, nil) req.closeBody() @@ -322,46 +393,65 @@ func (t *Transport) RoundTrip(req *Request) (*Response, error) { if err == nil { return resp, nil } - if err := checkTransportResend(err, req, pconn); err != nil { + if !pconn.shouldRetryRequest(req, err) { + // Issue 16465: return underlying net.Conn.Read error from peek, + // as we've historically done. + if e, ok := err.(transportReadFromServerError); ok { + err = e.err + } return nil, err } testHookRoundTripRetried() } } -// checkTransportResend checks whether a failed HTTP request can be -// resent on a new connection. The non-nil input error is the error from -// roundTrip, which might be wrapped in a beforeRespHeaderError error. -// -// The return value is err or the unwrapped error inside a -// beforeRespHeaderError. -func checkTransportResend(err error, req *Request, pconn *persistConn) error { - brhErr, ok := err.(beforeRespHeaderError) - if !ok { - return err +// shouldRetryRequest reports whether we should retry sending a failed +// HTTP request on a new connection. The non-nil input error is the +// error from roundTrip. +func (pc *persistConn) shouldRetryRequest(req *Request, err error) bool { + if err == http2ErrNoCachedConn { + // Issue 16582: if the user started a bunch of + // requests at once, they can all pick the same conn + // and violate the server's max concurrent streams. + // Instead, match the HTTP/1 behavior for now and dial + // again to get a new TCP connection, rather than failing + // this request. + return true } - err = brhErr.error // unwrap the custom error in case we return it - if err != errMissingHost && pconn.isReused() && req.isReplayable() { - // If we try to reuse a connection that the server is in the process of - // closing, we may end up successfully writing out our request (or a - // portion of our request) only to find a connection error when we try to - // read from (or finish writing to) the socket. - - // There can be a race between the socket pool checking whether a socket - // is still connected, receiving the FIN, and sending/reading data on a - // reused socket. If we receive the FIN between the connectedness check - // and writing/reading from the socket, we may first learn the socket is - // disconnected when we get a ERR_SOCKET_NOT_CONNECTED. This will most - // likely happen when trying to retrieve its IP address. See - // http://crbug.com/105824 for more details. - - // We resend a request only if we reused a keep-alive connection and did - // not yet receive any header data. This automatically prevents an - // infinite resend loop because we'll run out of the cached keep-alive - // connections eventually. - return nil + if err == errMissingHost { + // User error. + return false } - return err + if !pc.isReused() { + // This was a fresh connection. There's no reason the server + // should've hung up on us. + // + // Also, if we retried now, we could loop forever + // creating new connections and retrying if the server + // is just hanging up on us because it doesn't like + // our request (as opposed to sending an error). + return false + } + if _, ok := err.(nothingWrittenError); ok { + // We never wrote anything, so it's safe to retry. + return true + } + if !req.isReplayable() { + // Don't retry non-idempotent requests. + return false + } + if _, ok := err.(transportReadFromServerError); ok { + // We got some non-EOF net.Conn.Read failure reading + // the 1st response byte from the server. + return true + } + if err == errServerClosedIdle { + // The server replied with io.EOF while we were trying to + // read the response. Probably an unfortunately keep-alive + // timeout, just as the client was writing a request. + return true + } + return false // conservatively } // ErrSkipAltProtocol is a sentinel error value defined by Transport.RegisterProtocol. @@ -380,13 +470,16 @@ var ErrSkipAltProtocol = errors.New("net/http: skip alternate protocol") func (t *Transport) RegisterProtocol(scheme string, rt RoundTripper) { t.altMu.Lock() defer t.altMu.Unlock() - if t.altProto == nil { - t.altProto = make(map[string]RoundTripper) - } - if _, exists := t.altProto[scheme]; exists { + oldMap, _ := t.altProto.Load().(map[string]RoundTripper) + if _, exists := oldMap[scheme]; exists { panic("protocol " + scheme + " already registered") } - t.altProto[scheme] = rt + newMap := make(map[string]RoundTripper) + for k, v := range oldMap { + newMap[k] = v + } + newMap[scheme] = rt + t.altProto.Store(newMap) } // CloseIdleConnections closes any connections which were previously @@ -400,6 +493,7 @@ func (t *Transport) CloseIdleConnections() { t.idleConn = nil t.idleConnCh = nil t.wantIdle = true + t.idleLRU = connLRU{} t.idleMu.Unlock() for _, conns := range m { for _, pconn := range conns { @@ -414,15 +508,21 @@ func (t *Transport) CloseIdleConnections() { // CancelRequest cancels an in-flight request by closing its connection. // CancelRequest should only be called after RoundTrip has returned. // -// Deprecated: Use Request.Cancel instead. CancelRequest can not cancel -// HTTP/2 requests. +// Deprecated: Use Request.WithContext to create a request with a +// cancelable context instead. CancelRequest cannot cancel HTTP/2 +// requests. func (t *Transport) CancelRequest(req *Request) { + t.cancelRequest(req, errRequestCanceled) +} + +// Cancel an in-flight request, recording the error value. +func (t *Transport) cancelRequest(req *Request, err error) { t.reqMu.Lock() cancel := t.reqCanceler[req] delete(t.reqCanceler, req) t.reqMu.Unlock() if cancel != nil { - cancel() + cancel(err) } } @@ -472,10 +572,18 @@ func (e *envOnce) reset() { } func (t *Transport) connectMethodForRequest(treq *transportRequest) (cm connectMethod, err error) { + if port := treq.URL.Port(); !validPort(port) { + return cm, fmt.Errorf("invalid URL port %q", port) + } cm.targetScheme = treq.URL.Scheme cm.targetAddr = canonicalAddr(treq.URL) if t.Proxy != nil { cm.proxyURL, err = t.Proxy(treq.Request) + if err == nil && cm.proxyURL != nil { + if port := cm.proxyURL.Port(); !validPort(port) { + return cm, fmt.Errorf("invalid proxy URL port %q", port) + } + } } return cm, err } @@ -500,17 +608,43 @@ var ( errConnBroken = errors.New("http: putIdleConn: connection is in bad state") errWantIdle = errors.New("http: putIdleConn: CloseIdleConnections was called") errTooManyIdle = errors.New("http: putIdleConn: too many idle connections") + errTooManyIdleHost = errors.New("http: putIdleConn: too many idle connections for host") errCloseIdleConns = errors.New("http: CloseIdleConnections called") errReadLoopExiting = errors.New("http: persistConn.readLoop exiting") - errServerClosedIdle = errors.New("http: server closed idle conn") + errServerClosedIdle = errors.New("http: server closed idle connection") + errIdleConnTimeout = errors.New("http: idle connection timeout") + errNotCachingH2Conn = errors.New("http: not caching alternate protocol's connections") ) +// transportReadFromServerError is used by Transport.readLoop when the +// 1 byte peek read fails and we're actually anticipating a response. +// Usually this is just due to the inherent keep-alive shut down race, +// where the server closed the connection at the same time the client +// wrote. The underlying err field is usually io.EOF or some +// ECONNRESET sort of thing which varies by platform. But it might be +// the user's custom net.Conn.Read error too, so we carry it along for +// them to return from Transport.RoundTrip. +type transportReadFromServerError struct { + err error +} + +func (e transportReadFromServerError) Error() string { + return fmt.Sprintf("net/http: Transport failed to read from server: %v", e.err) +} + func (t *Transport) putOrCloseIdleConn(pconn *persistConn) { if err := t.tryPutIdleConn(pconn); err != nil { pconn.close(err) } } +func (t *Transport) maxIdleConnsPerHost() int { + if v := t.MaxIdleConnsPerHost; v != 0 { + return v + } + return DefaultMaxIdleConnsPerHost +} + // tryPutIdleConn adds pconn to the list of idle persistent connections awaiting // a new request. // If pconn is no longer needed or not in a good state, tryPutIdleConn returns @@ -523,13 +657,14 @@ func (t *Transport) tryPutIdleConn(pconn *persistConn) error { if pconn.isBroken() { return errConnBroken } - key := pconn.cacheKey - max := t.MaxIdleConnsPerHost - if max == 0 { - max = DefaultMaxIdleConnsPerHost + if pconn.alt != nil { + return errNotCachingH2Conn } pconn.markReused() + key := pconn.cacheKey + t.idleMu.Lock() + defer t.idleMu.Unlock() waitingDialer := t.idleConnCh[key] select { @@ -537,9 +672,8 @@ func (t *Transport) tryPutIdleConn(pconn *persistConn) error { // We're done with this pconn and somebody else is // currently waiting for a conn of this type (they're // actively dialing, but this conn is ready - // first). Chrome calls this socket late binding. See + // first). Chrome calls this socket late binding. See // https://insouciant.org/tech/connection-management-in-chromium/ - t.idleMu.Unlock() return nil default: if waitingDialer != nil { @@ -549,23 +683,35 @@ func (t *Transport) tryPutIdleConn(pconn *persistConn) error { } } if t.wantIdle { - t.idleMu.Unlock() return errWantIdle } if t.idleConn == nil { t.idleConn = make(map[connectMethodKey][]*persistConn) } - if len(t.idleConn[key]) >= max { - t.idleMu.Unlock() - return errTooManyIdle + idles := t.idleConn[key] + if len(idles) >= t.maxIdleConnsPerHost() { + return errTooManyIdleHost } - for _, exist := range t.idleConn[key] { + for _, exist := range idles { if exist == pconn { log.Fatalf("dup idle pconn %p in freelist", pconn) } } - t.idleConn[key] = append(t.idleConn[key], pconn) - t.idleMu.Unlock() + t.idleConn[key] = append(idles, pconn) + t.idleLRU.add(pconn) + if t.MaxIdleConns != 0 && t.idleLRU.len() > t.MaxIdleConns { + oldest := t.idleLRU.removeOldest() + oldest.close(errTooManyIdle) + t.removeIdleConnLocked(oldest) + } + if t.IdleConnTimeout > 0 { + if pconn.idleTimer != nil { + pconn.idleTimer.Reset(t.IdleConnTimeout) + } else { + pconn.idleTimer = time.AfterFunc(t.IdleConnTimeout, pconn.closeConnIfStillIdle) + } + } + pconn.idleAt = time.Now() return nil } @@ -591,38 +737,84 @@ func (t *Transport) getIdleConnCh(cm connectMethod) chan *persistConn { return ch } -func (t *Transport) getIdleConn(cm connectMethod) (pconn *persistConn) { +func (t *Transport) getIdleConn(cm connectMethod) (pconn *persistConn, idleSince time.Time) { key := cm.key() t.idleMu.Lock() defer t.idleMu.Unlock() - if t.idleConn == nil { - return nil - } for { pconns, ok := t.idleConn[key] if !ok { - return nil + return nil, time.Time{} } if len(pconns) == 1 { pconn = pconns[0] delete(t.idleConn, key) } else { - // 2 or more cached connections; pop last - // TODO: queue? + // 2 or more cached connections; use the most + // recently used one at the end. pconn = pconns[len(pconns)-1] t.idleConn[key] = pconns[:len(pconns)-1] } - if !pconn.isBroken() { - return + t.idleLRU.remove(pconn) + if pconn.isBroken() { + // There is a tiny window where this is + // possible, between the connecting dying and + // the persistConn readLoop calling + // Transport.removeIdleConn. Just skip it and + // carry on. + continue } + if pconn.idleTimer != nil && !pconn.idleTimer.Stop() { + // We picked this conn at the ~same time it + // was expiring and it's trying to close + // itself in another goroutine. Don't use it. + continue + } + return pconn, pconn.idleAt } } -func (t *Transport) setReqCanceler(r *Request, fn func()) { +// removeIdleConn marks pconn as dead. +func (t *Transport) removeIdleConn(pconn *persistConn) { + t.idleMu.Lock() + defer t.idleMu.Unlock() + t.removeIdleConnLocked(pconn) +} + +// t.idleMu must be held. +func (t *Transport) removeIdleConnLocked(pconn *persistConn) { + if pconn.idleTimer != nil { + pconn.idleTimer.Stop() + } + t.idleLRU.remove(pconn) + key := pconn.cacheKey + pconns, _ := t.idleConn[key] + switch len(pconns) { + case 0: + // Nothing + case 1: + if pconns[0] == pconn { + delete(t.idleConn, key) + } + default: + for i, v := range pconns { + if v != pconn { + continue + } + // Slide down, keeping most recently-used + // conns at the end. + copy(pconns[i:], pconns[i+1:]) + t.idleConn[key] = pconns[:len(pconns)-1] + break + } + } +} + +func (t *Transport) setReqCanceler(r *Request, fn func(error)) { t.reqMu.Lock() defer t.reqMu.Unlock() if t.reqCanceler == nil { - t.reqCanceler = make(map[*Request]func()) + t.reqCanceler = make(map[*Request]func(error)) } if fn != nil { t.reqCanceler[r] = fn @@ -635,7 +827,7 @@ func (t *Transport) setReqCanceler(r *Request, fn func()) { // for the request, we don't set the function and return false. // Since CancelRequest will clear the canceler, we can use the return value to detect if // the request was canceled since the last setReqCancel call. -func (t *Transport) replaceReqCanceler(r *Request, fn func()) bool { +func (t *Transport) replaceReqCanceler(r *Request, fn func(error)) bool { t.reqMu.Lock() defer t.reqMu.Unlock() _, ok := t.reqCanceler[r] @@ -650,7 +842,12 @@ func (t *Transport) replaceReqCanceler(r *Request, fn func()) bool { return true } -func (t *Transport) dial(network, addr string) (net.Conn, error) { +var zeroDialer net.Dialer + +func (t *Transport) dial(ctx context.Context, network, addr string) (net.Conn, error) { + if t.DialContext != nil { + return t.DialContext(ctx, network, addr) + } if t.Dial != nil { c, err := t.Dial(network, addr) if c == nil && err == nil { @@ -658,19 +855,28 @@ func (t *Transport) dial(network, addr string) (net.Conn, error) { } return c, err } - return net.Dial(network, addr) + return zeroDialer.DialContext(ctx, network, addr) } // getConn dials and creates a new persistConn to the target as -// specified in the connectMethod. This includes doing a proxy CONNECT +// specified in the connectMethod. This includes doing a proxy CONNECT // and/or setting up TLS. If this doesn't return an error, the persistConn // is ready to write requests to. -func (t *Transport) getConn(req *Request, cm connectMethod) (*persistConn, error) { - if pc := t.getIdleConn(cm); pc != nil { +func (t *Transport) getConn(treq *transportRequest, cm connectMethod) (*persistConn, error) { + req := treq.Request + trace := treq.trace + ctx := req.Context() + if trace != nil && trace.GetConn != nil { + trace.GetConn(cm.addr()) + } + if pc, idleSince := t.getIdleConn(cm); pc != nil { + if trace != nil && trace.GotConn != nil { + trace.GotConn(pc.gotIdleConnTrace(idleSince)) + } // set request canceler to some non-nil function so we // can detect whether it was cleared between now and when // we enter roundTrip - t.setReqCanceler(req, func() {}) + t.setReqCanceler(req, func(error) {}) return pc, nil } @@ -695,11 +901,11 @@ func (t *Transport) getConn(req *Request, cm connectMethod) (*persistConn, error }() } - cancelc := make(chan struct{}) - t.setReqCanceler(req, func() { close(cancelc) }) + cancelc := make(chan error, 1) + t.setReqCanceler(req, func(err error) { cancelc <- err }) go func() { - pc, err := t.dialConn(cm) + pc, err := t.dialConn(ctx, cm) dialc <- dialRes{pc, err} }() @@ -707,7 +913,31 @@ func (t *Transport) getConn(req *Request, cm connectMethod) (*persistConn, error select { case v := <-dialc: // Our dial finished. - return v.pc, v.err + if v.pc != nil { + if trace != nil && trace.GotConn != nil && v.pc.alt == nil { + trace.GotConn(httptrace.GotConnInfo{Conn: v.pc.conn}) + } + return v.pc, nil + } + // Our dial failed. See why to return a nicer error + // value. + select { + case <-req.Cancel: + // It was an error due to cancelation, so prioritize that + // error value. (Issue 16049) + return nil, errRequestCanceledConn + case <-req.Context().Done(): + return nil, req.Context().Err() + case err := <-cancelc: + if err == errRequestCanceled { + err = errRequestCanceledConn + } + return nil, err + default: + // It wasn't an error due to cancelation, so + // return the original error message: + return nil, v.err + } case pc := <-idleConnCh: // Another request finished first and its net.Conn // became available before our dial. Or somebody @@ -715,25 +945,36 @@ func (t *Transport) getConn(req *Request, cm connectMethod) (*persistConn, error // But our dial is still going, so give it away // when it finishes: handlePendingDial() + if trace != nil && trace.GotConn != nil { + trace.GotConn(httptrace.GotConnInfo{Conn: pc.conn, Reused: pc.isReused()}) + } return pc, nil case <-req.Cancel: handlePendingDial() return nil, errRequestCanceledConn - case <-cancelc: + case <-req.Context().Done(): handlePendingDial() - return nil, errRequestCanceledConn + return nil, req.Context().Err() + case err := <-cancelc: + handlePendingDial() + if err == errRequestCanceled { + err = errRequestCanceledConn + } + return nil, err } } -func (t *Transport) dialConn(cm connectMethod) (*persistConn, error) { +func (t *Transport) dialConn(ctx context.Context, cm connectMethod) (*persistConn, error) { pconn := &persistConn{ - t: t, - cacheKey: cm.key(), - reqch: make(chan requestAndChan, 1), - writech: make(chan writeRequest, 1), - closech: make(chan struct{}), - writeErrCh: make(chan error, 1), - } + t: t, + cacheKey: cm.key(), + reqch: make(chan requestAndChan, 1), + writech: make(chan writeRequest, 1), + closech: make(chan struct{}), + writeErrCh: make(chan error, 1), + writeLoopDone: make(chan struct{}), + } + trace := httptrace.ContextClientTrace(ctx) tlsDial := t.DialTLS != nil && cm.targetScheme == "https" && cm.proxyURL == nil if tlsDial { var err error @@ -747,18 +988,28 @@ func (t *Transport) dialConn(cm connectMethod) (*persistConn, error) { if tc, ok := pconn.conn.(*tls.Conn); ok { // Handshake here, in case DialTLS didn't. TLSNextProto below // depends on it for knowing the connection state. + if trace != nil && trace.TLSHandshakeStart != nil { + trace.TLSHandshakeStart() + } if err := tc.Handshake(); err != nil { go pconn.conn.Close() + if trace != nil && trace.TLSHandshakeDone != nil { + trace.TLSHandshakeDone(tls.ConnectionState{}, err) + } return nil, err } cs := tc.ConnectionState() + if trace != nil && trace.TLSHandshakeDone != nil { + trace.TLSHandshakeDone(cs, nil) + } pconn.tlsState = &cs } } else { - conn, err := t.dial("tcp", cm.addr()) + conn, err := t.dial(ctx, "tcp", cm.addr()) if err != nil { if cm.proxyURL != nil { - err = fmt.Errorf("http: error connecting to proxy %s: %v", cm.proxyURL, err) + // Return a typed error, per Issue 16997: + err = &net.OpError{Op: "proxyconnect", Net: "tcp", Err: err} } return nil, err } @@ -778,11 +1029,15 @@ func (t *Transport) dialConn(cm connectMethod) (*persistConn, error) { } case cm.targetScheme == "https": conn := pconn.conn + hdr := t.ProxyConnectHeader + if hdr == nil { + hdr = make(Header) + } connectReq := &Request{ Method: "CONNECT", URL: &url.URL{Opaque: cm.targetAddr}, Host: cm.targetAddr, - Header: make(Header), + Header: hdr, } if pa := cm.proxyAuth(); pa != "" { connectReq.Header.Set("Proxy-Authorization", pa) @@ -807,7 +1062,7 @@ func (t *Transport) dialConn(cm connectMethod) (*persistConn, error) { if cm.targetScheme == "https" && !tlsDial { // Initiate TLS and check remote host name against certificate. - cfg := cloneTLSClientConfig(t.TLSClientConfig) + cfg := cloneTLSConfig(t.TLSClientConfig) if cfg.ServerName == "" { cfg.ServerName = cm.tlsHost() } @@ -821,6 +1076,9 @@ func (t *Transport) dialConn(cm connectMethod) (*persistConn, error) { }) } go func() { + if trace != nil && trace.TLSHandshakeStart != nil { + trace.TLSHandshakeStart() + } err := tlsConn.Handshake() if timer != nil { timer.Stop() @@ -829,6 +1087,9 @@ func (t *Transport) dialConn(cm connectMethod) (*persistConn, error) { }() if err := <-errc; err != nil { plainConn.Close() + if trace != nil && trace.TLSHandshakeDone != nil { + trace.TLSHandshakeDone(tls.ConnectionState{}, err) + } return nil, err } if !cfg.InsecureSkipVerify { @@ -838,6 +1099,9 @@ func (t *Transport) dialConn(cm connectMethod) (*persistConn, error) { } } cs := tlsConn.ConnectionState() + if trace != nil && trace.TLSHandshakeDone != nil { + trace.TLSHandshakeDone(cs, nil) + } pconn.tlsState = &cs pconn.conn = tlsConn } @@ -848,13 +1112,29 @@ func (t *Transport) dialConn(cm connectMethod) (*persistConn, error) { } } - pconn.br = bufio.NewReader(noteEOFReader{pconn.conn, &pconn.sawEOF}) - pconn.bw = bufio.NewWriter(pconn.conn) + pconn.br = bufio.NewReader(pconn) + pconn.bw = bufio.NewWriter(persistConnWriter{pconn}) go pconn.readLoop() go pconn.writeLoop() return pconn, nil } +// persistConnWriter is the io.Writer written to by pc.bw. +// It accumulates the number of bytes written to the underlying conn, +// so the retry logic can determine whether any bytes made it across +// the wire. +// This is exactly 1 pointer field wide so it can go into an interface +// without allocation. +type persistConnWriter struct { + pc *persistConn +} + +func (w persistConnWriter) Write(p []byte) (n int, err error) { + n, err = w.pc.conn.Write(p) + w.pc.nwrite += int64(n) + return +} + // useProxy reports whether requests to addr should use a proxy, // according to the NO_PROXY or no_proxy environment variable. // addr is always a canonicalAddr with a host and port. @@ -978,32 +1258,40 @@ func (k connectMethodKey) String() string { // (but may be used for non-keep-alive requests as well) type persistConn struct { // alt optionally specifies the TLS NextProto RoundTripper. - // This is used for HTTP/2 today and future protocol laters. + // This is used for HTTP/2 today and future protocols later. // If it's non-nil, the rest of the fields are unused. alt RoundTripper - t *Transport - cacheKey connectMethodKey - conn net.Conn - tlsState *tls.ConnectionState - br *bufio.Reader // from conn - sawEOF bool // whether we've seen EOF from conn; owned by readLoop - bw *bufio.Writer // to conn - reqch chan requestAndChan // written by roundTrip; read by readLoop - writech chan writeRequest // written by roundTrip; read by writeLoop - closech chan struct{} // closed when conn closed - isProxy bool + t *Transport + cacheKey connectMethodKey + conn net.Conn + tlsState *tls.ConnectionState + br *bufio.Reader // from conn + bw *bufio.Writer // to conn + nwrite int64 // bytes written + reqch chan requestAndChan // written by roundTrip; read by readLoop + writech chan writeRequest // written by roundTrip; read by writeLoop + closech chan struct{} // closed when conn closed + isProxy bool + sawEOF bool // whether we've seen EOF from conn; owned by readLoop + readLimit int64 // bytes allowed to be read; owned by readLoop // writeErrCh passes the request write error (usually nil) // from the writeLoop goroutine to the readLoop which passes // it off to the res.Body reader, which then uses it to decide // whether or not a connection can be reused. Issue 7569. writeErrCh chan error - lk sync.Mutex // guards following fields + writeLoopDone chan struct{} // closed when write loop ends + + // Both guarded by Transport.idleMu: + idleAt time.Time // time it last become idle + idleTimer *time.Timer // holding an AfterFunc to close it + + mu sync.Mutex // guards following fields numExpectedResponses int closed error // set non-nil when conn is closed, before closech is closed + canceledErr error // set non-nil if conn is canceled broken bool // an error has happened on this connection; marked broken so it's not reused. - canceled bool // whether this conn was broken due a CancelRequest reused bool // whether conn has had successful request/response and is being reused. // mutateHeaderFunc is an optional func to modify extra // headers on each outbound request before it's written. (the @@ -1011,45 +1299,161 @@ type persistConn struct { mutateHeaderFunc func(Header) } +func (pc *persistConn) maxHeaderResponseSize() int64 { + if v := pc.t.MaxResponseHeaderBytes; v != 0 { + return v + } + return 10 << 20 // conservative default; same as http2 +} + +func (pc *persistConn) Read(p []byte) (n int, err error) { + if pc.readLimit <= 0 { + return 0, fmt.Errorf("read limit of %d bytes exhausted", pc.maxHeaderResponseSize()) + } + if int64(len(p)) > pc.readLimit { + p = p[:pc.readLimit] + } + n, err = pc.conn.Read(p) + if err == io.EOF { + pc.sawEOF = true + } + pc.readLimit -= int64(n) + return +} + // isBroken reports whether this connection is in a known broken state. func (pc *persistConn) isBroken() bool { - pc.lk.Lock() - b := pc.broken - pc.lk.Unlock() + pc.mu.Lock() + b := pc.closed != nil + pc.mu.Unlock() return b } -// isCanceled reports whether this connection was closed due to CancelRequest. -func (pc *persistConn) isCanceled() bool { - pc.lk.Lock() - defer pc.lk.Unlock() - return pc.canceled +// canceled returns non-nil if the connection was closed due to +// CancelRequest or due to context cancelation. +func (pc *persistConn) canceled() error { + pc.mu.Lock() + defer pc.mu.Unlock() + return pc.canceledErr } // isReused reports whether this connection is in a known broken state. func (pc *persistConn) isReused() bool { - pc.lk.Lock() + pc.mu.Lock() r := pc.reused - pc.lk.Unlock() + pc.mu.Unlock() return r } -func (pc *persistConn) cancelRequest() { - pc.lk.Lock() - defer pc.lk.Unlock() - pc.canceled = true +func (pc *persistConn) gotIdleConnTrace(idleAt time.Time) (t httptrace.GotConnInfo) { + pc.mu.Lock() + defer pc.mu.Unlock() + t.Reused = pc.reused + t.Conn = pc.conn + t.WasIdle = true + if !idleAt.IsZero() { + t.IdleTime = time.Since(idleAt) + } + return +} + +func (pc *persistConn) cancelRequest(err error) { + pc.mu.Lock() + defer pc.mu.Unlock() + pc.canceledErr = err pc.closeLocked(errRequestCanceled) } +// closeConnIfStillIdle closes the connection if it's still sitting idle. +// This is what's called by the persistConn's idleTimer, and is run in its +// own goroutine. +func (pc *persistConn) closeConnIfStillIdle() { + t := pc.t + t.idleMu.Lock() + defer t.idleMu.Unlock() + if _, ok := t.idleLRU.m[pc]; !ok { + // Not idle. + return + } + t.removeIdleConnLocked(pc) + pc.close(errIdleConnTimeout) +} + +// mapRoundTripErrorFromReadLoop maps the provided readLoop error into +// the error value that should be returned from persistConn.roundTrip. +// +// The startBytesWritten value should be the value of pc.nwrite before the roundTrip +// started writing the request. +func (pc *persistConn) mapRoundTripErrorFromReadLoop(req *Request, startBytesWritten int64, err error) (out error) { + if err == nil { + return nil + } + if err := pc.canceled(); err != nil { + return err + } + if err == errServerClosedIdle { + return err + } + if _, ok := err.(transportReadFromServerError); ok { + return err + } + if pc.isBroken() { + <-pc.writeLoopDone + if pc.nwrite == startBytesWritten && req.outgoingLength() == 0 { + return nothingWrittenError{err} + } + } + return err +} + +// mapRoundTripErrorAfterClosed returns the error value to be propagated +// up to Transport.RoundTrip method when persistConn.roundTrip sees +// its pc.closech channel close, indicating the persistConn is dead. +// (after closech is closed, pc.closed is valid). +func (pc *persistConn) mapRoundTripErrorAfterClosed(req *Request, startBytesWritten int64) error { + if err := pc.canceled(); err != nil { + return err + } + err := pc.closed + if err == errServerClosedIdle { + // Don't decorate + return err + } + if _, ok := err.(transportReadFromServerError); ok { + // Don't decorate + return err + } + + // Wait for the writeLoop goroutine to terminated, and then + // see if we actually managed to write anything. If not, we + // can retry the request. + <-pc.writeLoopDone + if pc.nwrite == startBytesWritten && req.outgoingLength() == 0 { + return nothingWrittenError{err} + } + + return fmt.Errorf("net/http: HTTP/1.x transport connection broken: %v", err) + +} + func (pc *persistConn) readLoop() { closeErr := errReadLoopExiting // default value, if not changed below - defer func() { pc.close(closeErr) }() + defer func() { + pc.close(closeErr) + pc.t.removeIdleConn(pc) + }() - tryPutIdleConn := func() bool { + tryPutIdleConn := func(trace *httptrace.ClientTrace) bool { if err := pc.t.tryPutIdleConn(pc); err != nil { closeErr = err + if trace != nil && trace.PutIdleConn != nil && err != errKeepAlivesDisabled { + trace.PutIdleConn(err) + } return false } + if trace != nil && trace.PutIdleConn != nil { + trace.PutIdleConn(nil) + } return true } @@ -1066,27 +1470,33 @@ func (pc *persistConn) readLoop() { alive := true for alive { + pc.readLimit = pc.maxHeaderResponseSize() _, err := pc.br.Peek(1) - if err != nil { - err = beforeRespHeaderError{err} - } - pc.lk.Lock() + pc.mu.Lock() if pc.numExpectedResponses == 0 { pc.readLoopPeekFailLocked(err) - pc.lk.Unlock() + pc.mu.Unlock() return } - pc.lk.Unlock() + pc.mu.Unlock() rc := <-pc.reqch + trace := httptrace.ContextClientTrace(rc.req.Context()) var resp *Response if err == nil { - resp, err = pc.readResponse(rc) + resp, err = pc.readResponse(rc, trace) + } else { + err = transportReadFromServerError{err} + closeErr = err } if err != nil { + if pc.readLimit <= 0 { + err = fmt.Errorf("net/http: server response headers exceeded %d bytes; aborted", pc.maxHeaderResponseSize()) + } + // If we won't be able to retry this request later (from the // roundTrip goroutine), mark it as done now. // BEFORE the send on rc.ch, as the client might re-use the @@ -1094,7 +1504,7 @@ func (pc *persistConn) readLoop() { // t.setReqCanceler from this persistConn while the Transport // potentially spins up a different persistConn for the // caller's subsequent request. - if checkTransportResend(err, rc.req, pc) != nil { + if !pc.shouldRetryRequest(rc.req, err) { pc.t.setReqCanceler(rc.req, nil) } select { @@ -1104,10 +1514,11 @@ func (pc *persistConn) readLoop() { } return } + pc.readLimit = maxInt64 // effictively no limit for response bodies - pc.lk.Lock() + pc.mu.Lock() pc.numExpectedResponses-- - pc.lk.Unlock() + pc.mu.Unlock() hasBody := rc.req.Method != "HEAD" && resp.ContentLength != 0 @@ -1130,7 +1541,7 @@ func (pc *persistConn) readLoop() { alive = alive && !pc.sawEOF && pc.wroteRequest() && - tryPutIdleConn() + tryPutIdleConn(trace) select { case rc.ch <- responseAndError{res: resp}: @@ -1145,25 +1556,35 @@ func (pc *persistConn) readLoop() { continue } - if rc.addedGzip { - maybeUngzipResponse(resp) + waitForBodyRead := make(chan bool, 2) + body := &bodyEOFSignal{ + body: resp.Body, + earlyCloseFn: func() error { + waitForBodyRead <- false + return nil + + }, + fn: func(err error) error { + isEOF := err == io.EOF + waitForBodyRead <- isEOF + if isEOF { + <-eofc // see comment above eofc declaration + } else if err != nil { + if cerr := pc.canceled(); cerr != nil { + return cerr + } + } + return err + }, } - resp.Body = &bodyEOFSignal{body: resp.Body} - waitForBodyRead := make(chan bool, 2) - resp.Body.(*bodyEOFSignal).earlyCloseFn = func() error { - waitForBodyRead <- false - return nil - } - resp.Body.(*bodyEOFSignal).fn = func(err error) error { - isEOF := err == io.EOF - waitForBodyRead <- isEOF - if isEOF { - <-eofc // see comment above eofc declaration - } else if err != nil && pc.isCanceled() { - return errRequestCanceled - } - return err + resp.Body = body + if rc.addedGzip && resp.Header.Get("Content-Encoding") == "gzip" { + resp.Body = &gzipReader{body: body} + resp.Header.Del("Content-Encoding") + resp.Header.Del("Content-Length") + resp.ContentLength = -1 + resp.Uncompressed = true } select { @@ -1182,13 +1603,16 @@ func (pc *persistConn) readLoop() { bodyEOF && !pc.sawEOF && pc.wroteRequest() && - tryPutIdleConn() + tryPutIdleConn(trace) if bodyEOF { eofc <- struct{}{} } case <-rc.req.Cancel: alive = false pc.t.CancelRequest(rc.req) + case <-rc.req.Context().Done(): + alive = false + pc.t.cancelRequest(rc.req, rc.req.Context().Err()) case <-pc.closech: alive = false } @@ -1197,15 +1621,6 @@ func (pc *persistConn) readLoop() { } } -func maybeUngzipResponse(resp *Response) { - if resp.Header.Get("Content-Encoding") == "gzip" { - resp.Header.Del("Content-Encoding") - resp.Header.Del("Content-Length") - resp.ContentLength = -1 - resp.Body = &gzipReader{body: resp.Body} - } -} - func (pc *persistConn) readLoopPeekFailLocked(peekErr error) { if pc.closed != nil { return @@ -1224,19 +1639,29 @@ func (pc *persistConn) readLoopPeekFailLocked(peekErr error) { // readResponse reads an HTTP response (or two, in the case of "Expect: // 100-continue") from the server. It returns the final non-100 one. -func (pc *persistConn) readResponse(rc requestAndChan) (resp *Response, err error) { +// trace is optional. +func (pc *persistConn) readResponse(rc requestAndChan, trace *httptrace.ClientTrace) (resp *Response, err error) { + if trace != nil && trace.GotFirstResponseByte != nil { + if peek, err := pc.br.Peek(1); err == nil && len(peek) == 1 { + trace.GotFirstResponseByte() + } + } resp, err = ReadResponse(pc.br, rc.req) if err != nil { return } if rc.continueCh != nil { if resp.StatusCode == 100 { + if trace != nil && trace.Got100Continue != nil { + trace.Got100Continue() + } rc.continueCh <- struct{}{} } else { close(rc.continueCh) } } if resp.StatusCode == 100 { + pc.readLimit = pc.maxHeaderResponseSize() // reset the limit resp, err = ReadResponse(pc.br, rc.req) if err != nil { return @@ -1268,24 +1693,33 @@ func (pc *persistConn) waitForContinue(continueCh <-chan struct{}) func() bool { } } +// nothingWrittenError wraps a write errors which ended up writing zero bytes. +type nothingWrittenError struct { + error +} + func (pc *persistConn) writeLoop() { + defer close(pc.writeLoopDone) for { select { case wr := <-pc.writech: - if pc.isBroken() { - wr.ch <- errors.New("http: can't write HTTP request on broken connection") - continue - } + startBytesWritten := pc.nwrite err := wr.req.Request.write(pc.bw, pc.isProxy, wr.req.extra, pc.waitForContinue(wr.continueCh)) if err == nil { err = pc.bw.Flush() } if err != nil { - pc.markBroken() wr.req.Request.closeBody() + if pc.nwrite == startBytesWritten && wr.req.outgoingLength() == 0 { + err = nothingWrittenError{err} + } } pc.writeErrCh <- err // to the body reader, which might recycle us wr.ch <- err // to the roundTrip function + if err != nil { + pc.close(err) + return + } case <-pc.closech: return } @@ -1331,9 +1765,9 @@ type requestAndChan struct { req *Request ch chan responseAndError // unbuffered; always send in select on callerGone - // did the Transport (as opposed to the client code) add an - // Accept-Encoding gzip header? only if it we set it do - // we transparently decode the gzip. + // whether the Transport (as opposed to the user client code) + // added the Accept-Encoding gzip header. If the Transport + // set it, only then do we transparently decode the gzip. addedGzip bool // Optional blocking chan for Expect: 100-continue (for send). @@ -1353,7 +1787,7 @@ type writeRequest struct { req *transportRequest ch chan<- error - // Optional blocking chan for Expect: 100-continue (for recieve). + // Optional blocking chan for Expect: 100-continue (for receive). // If not nil, writeLoop blocks sending request body until // it receives from this chan. continueCh <-chan struct{} @@ -1369,7 +1803,6 @@ func (e *httpError) Timeout() bool { return e.timeout } func (e *httpError) Temporary() bool { return true } var errTimeout error = &httpError{err: "net/http: timeout awaiting response headers", timeout: true} -var errClosed error = &httpError{err: "net/http: server closed connection before response was received"} var errRequestCanceled = errors.New("net/http: request canceled") var errRequestCanceledConn = errors.New("net/http: request canceled while waiting for connection") // TODO: unify? @@ -1387,22 +1820,16 @@ var ( testHookReadLoopBeforeNextRead = nop ) -// beforeRespHeaderError is used to indicate when an IO error has occurred before -// any header data was received. -type beforeRespHeaderError struct { - error -} - func (pc *persistConn) roundTrip(req *transportRequest) (resp *Response, err error) { testHookEnterRoundTrip() if !pc.t.replaceReqCanceler(req.Request, pc.cancelRequest) { pc.t.putOrCloseIdleConn(pc) return nil, errRequestCanceled } - pc.lk.Lock() + pc.mu.Lock() pc.numExpectedResponses++ headerFn := pc.mutateHeaderFunc - pc.lk.Unlock() + pc.mu.Unlock() if headerFn != nil { headerFn(req.extraHeaders()) @@ -1448,6 +1875,7 @@ func (pc *persistConn) roundTrip(req *transportRequest) (resp *Response, err err // Write the request concurrently with waiting for a response, // in case the server decides to reply before reading our full // request body. + startBytesWritten := pc.nwrite writeErrCh := make(chan error, 1) pc.writech <- writeRequest{req, writeErrCh, continueCh} @@ -1463,16 +1891,17 @@ func (pc *persistConn) roundTrip(req *transportRequest) (resp *Response, err err var re responseAndError var respHeaderTimer <-chan time.Time cancelChan := req.Request.Cancel + ctxDoneChan := req.Context().Done() WaitResponse: for { testHookWaitResLoop() select { case err := <-writeErrCh: if err != nil { - if pc.isCanceled() { - err = errRequestCanceled + if cerr := pc.canceled(); cerr != nil { + err = cerr } - re = responseAndError{err: beforeRespHeaderError{err}} + re = responseAndError{err: err} pc.close(fmt.Errorf("write error: %v", err)) break WaitResponse } @@ -1482,26 +1911,22 @@ WaitResponse: respHeaderTimer = timer.C } case <-pc.closech: - var err error - if pc.isCanceled() { - err = errRequestCanceled - } else { - err = beforeRespHeaderError{fmt.Errorf("net/http: HTTP/1 transport connection broken: %v", pc.closed)} - } - re = responseAndError{err: err} + re = responseAndError{err: pc.mapRoundTripErrorAfterClosed(req.Request, startBytesWritten)} break WaitResponse case <-respHeaderTimer: pc.close(errTimeout) re = responseAndError{err: errTimeout} break WaitResponse case re = <-resc: - if re.err != nil && pc.isCanceled() { - re.err = errRequestCanceled - } + re.err = pc.mapRoundTripErrorFromReadLoop(req.Request, startBytesWritten, re.err) break WaitResponse case <-cancelChan: pc.t.CancelRequest(req.Request) cancelChan = nil + case <-ctxDoneChan: + pc.t.cancelRequest(req.Request, req.Context().Err()) + cancelChan = nil + ctxDoneChan = nil } } @@ -1514,21 +1939,12 @@ WaitResponse: return re.res, re.err } -// markBroken marks a connection as broken (so it's not reused). -// It differs from close in that it doesn't close the underlying -// connection for use when it's still being read. -func (pc *persistConn) markBroken() { - pc.lk.Lock() - defer pc.lk.Unlock() - pc.broken = true -} - // markReused marks this connection as having been successfully used for a // request and response. func (pc *persistConn) markReused() { - pc.lk.Lock() + pc.mu.Lock() pc.reused = true - pc.lk.Unlock() + pc.mu.Unlock() } // close closes the underlying TCP connection and closes @@ -1537,8 +1953,8 @@ func (pc *persistConn) markReused() { // The provided err is only for testing and debugging; in normal // circumstances it should never be seen by users. func (pc *persistConn) close(err error) { - pc.lk.Lock() - defer pc.lk.Unlock() + pc.mu.Lock() + defer pc.mu.Unlock() pc.closeLocked(err) } @@ -1554,7 +1970,7 @@ func (pc *persistConn) closeLocked(err error) { // handlePendingDial's putOrCloseIdleConn when // it turns out the abandoned connection in // flight ended up negotiating an alternate - // protocol. We don't use the connection + // protocol. We don't use the connection // freelist for http2. That's done by the // alternate protocol's RoundTripper. } else { @@ -1572,14 +1988,22 @@ var portMap = map[string]string{ // canonicalAddr returns url.Host but always with a ":port" suffix func canonicalAddr(url *url.URL) string { - addr := url.Host - if !hasPort(addr) { - return addr + ":" + portMap[url.Scheme] + addr := url.Hostname() + if v, err := idnaASCII(addr); err == nil { + addr = v + } + port := url.Port() + if port == "" { + port = portMap[url.Scheme] } - return addr + return net.JoinHostPort(addr, port) } -// bodyEOFSignal wraps a ReadCloser but runs fn (if non-nil) at most +// bodyEOFSignal is used by the HTTP/1 transport when reading response +// bodies to make sure we see the end of a response body before +// proceeding and reading on the connection again. +// +// It wraps a ReadCloser but runs fn (if non-nil) at most // once, right before its final (error-producing) Read or Close call // returns. fn should return the new error to return from Read or Close. // @@ -1595,12 +2019,14 @@ type bodyEOFSignal struct { earlyCloseFn func() error // optional alt Close func used if io.EOF not seen } +var errReadOnClosedResBody = errors.New("http: read on closed response body") + func (es *bodyEOFSignal) Read(p []byte) (n int, err error) { es.mu.Lock() closed, rerr := es.closed, es.rerr es.mu.Unlock() if closed { - return 0, errors.New("http: read on closed response body") + return 0, errReadOnClosedResBody } if rerr != nil { return 0, rerr @@ -1645,17 +2071,30 @@ func (es *bodyEOFSignal) condfn(err error) error { // gzipReader wraps a response body so it can lazily // call gzip.NewReader on the first call to Read type gzipReader struct { - body io.ReadCloser // underlying Response.Body - zr io.Reader // lazily-initialized gzip reader + body *bodyEOFSignal // underlying HTTP/1 response body framing + zr *gzip.Reader // lazily-initialized gzip reader + zerr error // any error from gzip.NewReader; sticky } func (gz *gzipReader) Read(p []byte) (n int, err error) { if gz.zr == nil { - gz.zr, err = gzip.NewReader(gz.body) - if err != nil { - return 0, err + if gz.zerr == nil { + gz.zr, gz.zerr = gzip.NewReader(gz.body) + } + if gz.zerr != nil { + return 0, gz.zerr } } + + gz.body.mu.Lock() + if gz.body.closed { + err = errReadOnClosedResBody + } + gz.body.mu.Unlock() + + if err != nil { + return 0, err + } return gz.zr.Read(p) } @@ -1674,19 +2113,6 @@ func (tlsHandshakeTimeoutError) Timeout() bool { return true } func (tlsHandshakeTimeoutError) Temporary() bool { return true } func (tlsHandshakeTimeoutError) Error() string { return "net/http: TLS handshake timeout" } -type noteEOFReader struct { - r io.Reader - sawEOF *bool -} - -func (nr noteEOFReader) Read(p []byte) (n int, err error) { - n, err = nr.r.Read(p) - if err == io.EOF { - *nr.sawEOF = true - } - return -} - // fakeLocker is a sync.Locker which does nothing. It's used to guard // test-only fields when not under test, to avoid runtime atomic // overhead. @@ -1695,80 +2121,63 @@ type fakeLocker struct{} func (fakeLocker) Lock() {} func (fakeLocker) Unlock() {} -func isNetWriteError(err error) bool { - switch e := err.(type) { - case *url.Error: - return isNetWriteError(e.Err) - case *net.OpError: - return e.Op == "write" - default: - return false - } -} - -// cloneTLSConfig returns a shallow clone of the exported -// fields of cfg, ignoring the unexported sync.Once, which -// contains a mutex and must not be copied. -// -// The cfg must not be in active use by tls.Server, or else -// there can still be a race with tls.Server updating SessionTicketKey -// and our copying it, and also a race with the server setting -// SessionTicketsDisabled=false on failure to set the random -// ticket key. -// -// If cfg is nil, a new zero tls.Config is returned. +// clneTLSConfig returns a shallow clone of cfg, or a new zero tls.Config if +// cfg is nil. This is safe to call even if cfg is in active use by a TLS +// client or server. func cloneTLSConfig(cfg *tls.Config) *tls.Config { if cfg == nil { return &tls.Config{} } - return &tls.Config{ - Rand: cfg.Rand, - Time: cfg.Time, - Certificates: cfg.Certificates, - NameToCertificate: cfg.NameToCertificate, - GetCertificate: cfg.GetCertificate, - RootCAs: cfg.RootCAs, - NextProtos: cfg.NextProtos, - ServerName: cfg.ServerName, - ClientAuth: cfg.ClientAuth, - ClientCAs: cfg.ClientCAs, - InsecureSkipVerify: cfg.InsecureSkipVerify, - CipherSuites: cfg.CipherSuites, - PreferServerCipherSuites: cfg.PreferServerCipherSuites, - SessionTicketsDisabled: cfg.SessionTicketsDisabled, - SessionTicketKey: cfg.SessionTicketKey, - ClientSessionCache: cfg.ClientSessionCache, - MinVersion: cfg.MinVersion, - MaxVersion: cfg.MaxVersion, - CurvePreferences: cfg.CurvePreferences, - } -} - -// cloneTLSClientConfig is like cloneTLSConfig but omits -// the fields SessionTicketsDisabled and SessionTicketKey. -// This makes it safe to call cloneTLSClientConfig on a config -// in active use by a server. -func cloneTLSClientConfig(cfg *tls.Config) *tls.Config { - if cfg == nil { - return &tls.Config{} + return cfg.Clone() +} + +type connLRU struct { + ll *list.List // list.Element.Value type is of *persistConn + m map[*persistConn]*list.Element +} + +// add adds pc to the head of the linked list. +func (cl *connLRU) add(pc *persistConn) { + if cl.ll == nil { + cl.ll = list.New() + cl.m = make(map[*persistConn]*list.Element) } - return &tls.Config{ - Rand: cfg.Rand, - Time: cfg.Time, - Certificates: cfg.Certificates, - NameToCertificate: cfg.NameToCertificate, - GetCertificate: cfg.GetCertificate, - RootCAs: cfg.RootCAs, - NextProtos: cfg.NextProtos, - ServerName: cfg.ServerName, - ClientAuth: cfg.ClientAuth, - ClientCAs: cfg.ClientCAs, - InsecureSkipVerify: cfg.InsecureSkipVerify, - CipherSuites: cfg.CipherSuites, - PreferServerCipherSuites: cfg.PreferServerCipherSuites, - ClientSessionCache: cfg.ClientSessionCache, - MinVersion: cfg.MinVersion, - MaxVersion: cfg.MaxVersion, - CurvePreferences: cfg.CurvePreferences, + ele := cl.ll.PushFront(pc) + if _, ok := cl.m[pc]; ok { + panic("persistConn was already in LRU") + } + cl.m[pc] = ele +} + +func (cl *connLRU) removeOldest() *persistConn { + ele := cl.ll.Back() + pc := ele.Value.(*persistConn) + cl.ll.Remove(ele) + delete(cl.m, pc) + return pc +} + +// remove removes pc from cl. +func (cl *connLRU) remove(pc *persistConn) { + if ele, ok := cl.m[pc]; ok { + cl.ll.Remove(ele) + delete(cl.m, pc) } } + +// len returns the number of items in the cache. +func (cl *connLRU) len() int { + return len(cl.m) +} + +// validPort reports whether p (without the colon) is a valid port in +// a URL, per RFC 3986 Section 3.2.3, which says the port may be +// empty, or only contain digits. +func validPort(p string) bool { + for _, r := range []byte(p) { + if r < '0' || r > '9' { + return false + } + } + return true +} diff --git a/libgo/go/net/http/transport_internal_test.go b/libgo/go/net/http/transport_internal_test.go new file mode 100644 index 0000000000..3d24fc127d --- /dev/null +++ b/libgo/go/net/http/transport_internal_test.go @@ -0,0 +1,141 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// White-box tests for transport.go (in package http instead of http_test). + +package http + +import ( + "errors" + "net" + "testing" +) + +// Issue 15446: incorrect wrapping of errors when server closes an idle connection. +func TestTransportPersistConnReadLoopEOF(t *testing.T) { + ln := newLocalListener(t) + defer ln.Close() + + connc := make(chan net.Conn, 1) + go func() { + defer close(connc) + c, err := ln.Accept() + if err != nil { + t.Error(err) + return + } + connc <- c + }() + + tr := new(Transport) + req, _ := NewRequest("GET", "http://"+ln.Addr().String(), nil) + treq := &transportRequest{Request: req} + cm := connectMethod{targetScheme: "http", targetAddr: ln.Addr().String()} + pc, err := tr.getConn(treq, cm) + if err != nil { + t.Fatal(err) + } + defer pc.close(errors.New("test over")) + + conn := <-connc + if conn == nil { + // Already called t.Error in the accept goroutine. + return + } + conn.Close() // simulate the server hanging up on the client + + _, err = pc.roundTrip(treq) + if !isTransportReadFromServerError(err) && err != errServerClosedIdle { + t.Fatalf("roundTrip = %#v, %v; want errServerClosedConn or errServerClosedIdle", err, err) + } + + <-pc.closech + err = pc.closed + if !isTransportReadFromServerError(err) && err != errServerClosedIdle { + t.Fatalf("pc.closed = %#v, %v; want errServerClosedConn or errServerClosedIdle", err, err) + } +} + +func isTransportReadFromServerError(err error) bool { + _, ok := err.(transportReadFromServerError) + return ok +} + +func newLocalListener(t *testing.T) net.Listener { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + ln, err = net.Listen("tcp6", "[::1]:0") + } + if err != nil { + t.Fatal(err) + } + return ln +} + +func dummyRequest(method string) *Request { + req, err := NewRequest(method, "http://fake.tld/", nil) + if err != nil { + panic(err) + } + return req +} + +func TestTransportShouldRetryRequest(t *testing.T) { + tests := []struct { + pc *persistConn + req *Request + + err error + want bool + }{ + 0: { + pc: &persistConn{reused: false}, + req: dummyRequest("POST"), + err: nothingWrittenError{}, + want: false, + }, + 1: { + pc: &persistConn{reused: true}, + req: dummyRequest("POST"), + err: nothingWrittenError{}, + want: true, + }, + 2: { + pc: &persistConn{reused: true}, + req: dummyRequest("POST"), + err: http2ErrNoCachedConn, + want: true, + }, + 3: { + pc: &persistConn{reused: true}, + req: dummyRequest("POST"), + err: errMissingHost, + want: false, + }, + 4: { + pc: &persistConn{reused: true}, + req: dummyRequest("POST"), + err: transportReadFromServerError{}, + want: false, + }, + 5: { + pc: &persistConn{reused: true}, + req: dummyRequest("GET"), + err: transportReadFromServerError{}, + want: true, + }, + 6: { + pc: &persistConn{reused: true}, + req: dummyRequest("GET"), + err: errServerClosedIdle, + want: true, + }, + } + for i, tt := range tests { + got := tt.pc.shouldRetryRequest(tt.req, tt.err) + if got != tt.want { + t.Errorf("%d. shouldRetryRequest = %v; want %v", i, got, tt.want) + } + } +} diff --git a/libgo/go/net/http/transport_test.go b/libgo/go/net/http/transport_test.go index 0c901b30a4..a58b1839cc 100644 --- a/libgo/go/net/http/transport_test.go +++ b/libgo/go/net/http/transport_test.go @@ -13,16 +13,20 @@ import ( "bufio" "bytes" "compress/gzip" + "context" "crypto/rand" "crypto/tls" "errors" "fmt" + "internal/nettrace" + "internal/testenv" "io" "io/ioutil" "log" "net" . "net/http" "net/http/httptest" + "net/http/httptrace" "net/http/httputil" "net/http/internal" "net/url" @@ -32,6 +36,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "testing" "time" ) @@ -379,8 +384,8 @@ func TestTransportMaxPerHostIdleConns(t *testing.T) { } })) defer ts.Close() - maxIdleConns := 2 - tr := &Transport{DisableKeepAlives: false, MaxIdleConnsPerHost: maxIdleConns} + maxIdleConnsPerHost := 2 + tr := &Transport{DisableKeepAlives: false, MaxIdleConnsPerHost: maxIdleConnsPerHost} c := &Client{Transport: tr} // Start 3 outstanding requests and wait for the server to get them. @@ -425,14 +430,63 @@ func TestTransportMaxPerHostIdleConns(t *testing.T) { resch <- "res2" <-donech - if e, g := 2, tr.IdleConnCountForTesting(cacheKey); e != g { - t.Errorf("after second response, expected %d idle conns; got %d", e, g) + if g, w := tr.IdleConnCountForTesting(cacheKey), 2; g != w { + t.Errorf("after second response, idle conns = %d; want %d", g, w) } resch <- "res3" <-donech - if e, g := maxIdleConns, tr.IdleConnCountForTesting(cacheKey); e != g { - t.Errorf("after third response, still expected %d idle conns; got %d", e, g) + if g, w := tr.IdleConnCountForTesting(cacheKey), maxIdleConnsPerHost; g != w { + t.Errorf("after third response, idle conns = %d; want %d", g, w) + } +} + +func TestTransportRemovesDeadIdleConnections(t *testing.T) { + setParallel(t) + defer afterTest(t) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + io.WriteString(w, r.RemoteAddr) + })) + defer ts.Close() + + tr := &Transport{} + defer tr.CloseIdleConnections() + c := &Client{Transport: tr} + + doReq := func(name string) string { + // Do a POST instead of a GET to prevent the Transport's + // idempotent request retry logic from kicking in... + res, err := c.Post(ts.URL, "", nil) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + if res.StatusCode != 200 { + t.Fatalf("%s: %v", name, res.Status) + } + defer res.Body.Close() + slurp, err := ioutil.ReadAll(res.Body) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + return string(slurp) + } + + first := doReq("first") + keys1 := tr.IdleConnKeysForTesting() + + ts.CloseClientConnections() + + var keys2 []string + if !waitCondition(3*time.Second, 50*time.Millisecond, func() bool { + keys2 = tr.IdleConnKeysForTesting() + return len(keys2) == 0 + }) { + t.Fatalf("Transport didn't notice idle connection's death.\nbefore: %q\n after: %q\n", keys1, keys2) + } + + second := doReq("second") + if first == second { + t.Errorf("expected a different connection between requests. got %q both times", first) } } @@ -478,7 +532,7 @@ func TestTransportServerClosingUnexpectedly(t *testing.T) { // This test has an expected race. Sleeping for 25 ms prevents // it on most fast machines, causing the next fetch() call to - // succeed quickly. But if we do get errors, fetch() will retry 5 + // succeed quickly. But if we do get errors, fetch() will retry 5 // times with some delays between. time.Sleep(25 * time.Millisecond) @@ -518,7 +572,7 @@ func TestStressSurpriseServerCloses(t *testing.T) { // after each request completes, regardless of whether it failed. // If these are too high, OS X exhausts its ephemeral ports // and hangs waiting for them to transition TCP states. That's - // not what we want to test. TODO(bradfitz): use an io.Pipe + // not what we want to test. TODO(bradfitz): use an io.Pipe // dialer for this test instead? const ( numClients = 20 @@ -645,6 +699,7 @@ var roundTripTests = []struct { // Test that the modification made to the Request by the RoundTripper is cleaned up func TestRoundTripGzip(t *testing.T) { + setParallel(t) defer afterTest(t) const responseBody = "test response body" ts := httptest.NewServer(HandlerFunc(func(rw ResponseWriter, req *Request) { @@ -703,6 +758,7 @@ func TestRoundTripGzip(t *testing.T) { } func TestTransportGzip(t *testing.T) { + setParallel(t) defer afterTest(t) const testString = "The test string aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" const nRandBytes = 1024 * 1024 @@ -801,6 +857,7 @@ func TestTransportGzip(t *testing.T) { // If a request has Expect:100-continue header, the request blocks sending body until the first response. // Premature consumption of the request body should not be occurred. func TestTransportExpect100Continue(t *testing.T) { + setParallel(t) defer afterTest(t) ts := httptest.NewServer(HandlerFunc(func(rw ResponseWriter, req *Request) { @@ -853,7 +910,7 @@ func TestTransportExpect100Continue(t *testing.T) { {path: "/100", body: []byte("hello"), sent: 5, status: 200}, // Got 100 followed by 200, entire body is sent. {path: "/200", body: []byte("hello"), sent: 0, status: 200}, // Got 200 without 100. body isn't sent. {path: "/500", body: []byte("hello"), sent: 0, status: 500}, // Got 500 without 100. body isn't sent. - {path: "/keepalive", body: []byte("hello"), sent: 0, status: 500}, // Althogh without Connection:close, body isn't sent. + {path: "/keepalive", body: []byte("hello"), sent: 0, status: 500}, // Although without Connection:close, body isn't sent. {path: "/timeout", body: []byte("hello"), sent: 5, status: 200}, // Timeout exceeded and entire body is sent. } @@ -911,6 +968,48 @@ func TestTransportProxy(t *testing.T) { } } +// Issue 16997: test transport dial preserves typed errors +func TestTransportDialPreservesNetOpProxyError(t *testing.T) { + defer afterTest(t) + + var errDial = errors.New("some dial error") + + tr := &Transport{ + Proxy: func(*Request) (*url.URL, error) { + return url.Parse("http://proxy.fake.tld/") + }, + Dial: func(string, string) (net.Conn, error) { + return nil, errDial + }, + } + defer tr.CloseIdleConnections() + + c := &Client{Transport: tr} + req, _ := NewRequest("GET", "http://fake.tld", nil) + res, err := c.Do(req) + if err == nil { + res.Body.Close() + t.Fatal("wanted a non-nil error") + } + + uerr, ok := err.(*url.Error) + if !ok { + t.Fatalf("got %T, want *url.Error", err) + } + oe, ok := uerr.Err.(*net.OpError) + if !ok { + t.Fatalf("url.Error.Err = %T; want *net.OpError", uerr.Err) + } + want := &net.OpError{ + Op: "proxyconnect", + Net: "tcp", + Err: errDial, // original error, unwrapped. + } + if !reflect.DeepEqual(oe, want) { + t.Errorf("Got error %#v; want %#v", oe, want) + } +} + // TestTransportGzipRecursive sends a gzip quine and checks that the // client gets the same value back. This is more cute than anything, // but checks that we don't recurse forever, and checks that @@ -923,7 +1022,9 @@ func TestTransportGzipRecursive(t *testing.T) { })) defer ts.Close() - c := &Client{Transport: &Transport{}} + tr := &Transport{} + defer tr.CloseIdleConnections() + c := &Client{Transport: tr} res, err := c.Get(ts.URL) if err != nil { t.Fatal(err) @@ -968,12 +1069,25 @@ func TestTransportGzipShort(t *testing.T) { } } +// Wait until number of goroutines is no greater than nmax, or time out. +func waitNumGoroutine(nmax int) int { + nfinal := runtime.NumGoroutine() + for ntries := 10; ntries > 0 && nfinal > nmax; ntries-- { + time.Sleep(50 * time.Millisecond) + runtime.GC() + nfinal = runtime.NumGoroutine() + } + return nfinal +} + // tests that persistent goroutine connections shut down when no longer desired. func TestTransportPersistConnLeak(t *testing.T) { - setParallel(t) + // Not parallel: counts goroutines defer afterTest(t) - gotReqCh := make(chan bool) - unblockCh := make(chan bool) + + const numReq = 25 + gotReqCh := make(chan bool, numReq) + unblockCh := make(chan bool, numReq) ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { gotReqCh <- true <-unblockCh @@ -987,14 +1101,15 @@ func TestTransportPersistConnLeak(t *testing.T) { n0 := runtime.NumGoroutine() - const numReq = 25 - didReqCh := make(chan bool) + didReqCh := make(chan bool, numReq) + failed := make(chan bool, numReq) for i := 0; i < numReq; i++ { go func() { res, err := c.Get(ts.URL) didReqCh <- true if err != nil { t.Errorf("client fetch error: %v", err) + failed <- true return } res.Body.Close() @@ -1003,7 +1118,13 @@ func TestTransportPersistConnLeak(t *testing.T) { // Wait for all goroutines to be stuck in the Handler. for i := 0; i < numReq; i++ { - <-gotReqCh + select { + case <-gotReqCh: + // ok + case <-failed: + close(unblockCh) + return + } } nhigh := runtime.NumGoroutine() @@ -1019,14 +1140,11 @@ func TestTransportPersistConnLeak(t *testing.T) { } tr.CloseIdleConnections() - time.Sleep(100 * time.Millisecond) - runtime.GC() - runtime.GC() // even more. - nfinal := runtime.NumGoroutine() + nfinal := waitNumGoroutine(n0 + 5) growth := nfinal - n0 - // We expect 0 or 1 extra goroutine, empirically. Allow up to 5. + // We expect 0 or 1 extra goroutine, empirically. Allow up to 5. // Previously we were leaking one per numReq. if int(growth) > 5 { t.Logf("goroutine growth: %d -> %d -> %d (delta: %d)", n0, nhigh, nfinal, growth) @@ -1037,7 +1155,7 @@ func TestTransportPersistConnLeak(t *testing.T) { // golang.org/issue/4531: Transport leaks goroutines when // request.ContentLength is explicitly short func TestTransportPersistConnLeakShortBody(t *testing.T) { - setParallel(t) + // Not parallel: measures goroutines. defer afterTest(t) ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { })) @@ -1061,13 +1179,11 @@ func TestTransportPersistConnLeakShortBody(t *testing.T) { } nhigh := runtime.NumGoroutine() tr.CloseIdleConnections() - time.Sleep(400 * time.Millisecond) - runtime.GC() - nfinal := runtime.NumGoroutine() + nfinal := waitNumGoroutine(n0 + 5) growth := nfinal - n0 - // We expect 0 or 1 extra goroutine, empirically. Allow up to 5. + // We expect 0 or 1 extra goroutine, empirically. Allow up to 5. // Previously we were leaking one per numReq. t.Logf("goroutine growth: %d -> %d -> %d (delta: %d)", n0, nhigh, nfinal, growth) if int(growth) > 5 { @@ -1103,8 +1219,8 @@ func TestTransportIdleConnCrash(t *testing.T) { } // Test that the transport doesn't close the TCP connection early, -// before the response body has been read. This was a regression -// which sadly lacked a triggering test. The large response body made +// before the response body has been read. This was a regression +// which sadly lacked a triggering test. The large response body made // the old race easier to trigger. func TestIssue3644(t *testing.T) { defer afterTest(t) @@ -1135,6 +1251,7 @@ func TestIssue3644(t *testing.T) { // Test that a client receives a server's reply, even if the server doesn't read // the entire request body. func TestIssue3595(t *testing.T) { + setParallel(t) defer afterTest(t) const deniedMsg = "sorry, denied." ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { @@ -1183,6 +1300,7 @@ func TestChunkedNoContent(t *testing.T) { } func TestTransportConcurrency(t *testing.T) { + // Not parallel: uses global test hooks. defer afterTest(t) maxProcs, numReqs := 16, 500 if testing.Short() { @@ -1199,7 +1317,7 @@ func TestTransportConcurrency(t *testing.T) { // Due to the Transport's "socket late binding" (see // idleConnCh in transport.go), the numReqs HTTP requests - // below can finish with a dial still outstanding. To keep + // below can finish with a dial still outstanding. To keep // the leak checker happy, keep track of pending dials and // wait for them to finish (and be closed or returned to the // idle pool) before we close idle connections. @@ -1243,9 +1361,7 @@ func TestTransportConcurrency(t *testing.T) { } func TestIssue4191_InfiniteGetTimeout(t *testing.T) { - if runtime.GOOS == "plan9" { - t.Skip("skipping test; see https://golang.org/issue/7237") - } + setParallel(t) defer afterTest(t) const debug = false mux := NewServeMux() @@ -1307,9 +1423,7 @@ func TestIssue4191_InfiniteGetTimeout(t *testing.T) { } func TestIssue4191_InfiniteGetToPutTimeout(t *testing.T) { - if runtime.GOOS == "plan9" { - t.Skip("skipping test; see https://golang.org/issue/7237") - } + setParallel(t) defer afterTest(t) const debug = false mux := NewServeMux() @@ -1617,7 +1731,13 @@ func TestCancelRequestWithChannel(t *testing.T) { } } -func TestCancelRequestWithChannelBeforeDo(t *testing.T) { +func TestCancelRequestWithChannelBeforeDo_Cancel(t *testing.T) { + testCancelRequestWithChannelBeforeDo(t, false) +} +func TestCancelRequestWithChannelBeforeDo_Context(t *testing.T) { + testCancelRequestWithChannelBeforeDo(t, true) +} +func testCancelRequestWithChannelBeforeDo(t *testing.T, withCtx bool) { setParallel(t) defer afterTest(t) unblockc := make(chan bool) @@ -1627,24 +1747,33 @@ func TestCancelRequestWithChannelBeforeDo(t *testing.T) { defer ts.Close() defer close(unblockc) - // Don't interfere with the next test on plan9. - // Cf. https://golang.org/issues/11476 - if runtime.GOOS == "plan9" { - defer time.Sleep(500 * time.Millisecond) - } - tr := &Transport{} defer tr.CloseIdleConnections() c := &Client{Transport: tr} req, _ := NewRequest("GET", ts.URL, nil) - ch := make(chan struct{}) - req.Cancel = ch - close(ch) + if withCtx { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + req = req.WithContext(ctx) + } else { + ch := make(chan struct{}) + req.Cancel = ch + close(ch) + } _, err := c.Do(req) - if err == nil || !strings.Contains(err.Error(), "canceled") { - t.Errorf("Do error = %v; want cancelation", err) + if ue, ok := err.(*url.Error); ok { + err = ue.Err + } + if withCtx { + if err != context.Canceled { + t.Errorf("Do error = %v; want %v", err, context.Canceled) + } + } else { + if err == nil || !strings.Contains(err.Error(), "canceled") { + t.Errorf("Do error = %v; want cancelation", err) + } } } @@ -1813,6 +1942,7 @@ func TestTransportEmptyMethod(t *testing.T) { } func TestTransportSocketLateBinding(t *testing.T) { + setParallel(t) defer afterTest(t) mux := NewServeMux() @@ -1985,7 +2115,8 @@ type proxyFromEnvTest struct { env string // HTTP_PROXY httpsenv string // HTTPS_PROXY - noenv string // NO_RPXY + noenv string // NO_PROXY + reqmeth string // REQUEST_METHOD want string wanterr error @@ -2009,6 +2140,10 @@ func (t proxyFromEnvTest) String() string { space() fmt.Fprintf(&buf, "no_proxy=%q", t.noenv) } + if t.reqmeth != "" { + space() + fmt.Fprintf(&buf, "request_method=%q", t.reqmeth) + } req := "http://example.com" if t.req != "" { req = t.req @@ -2032,6 +2167,12 @@ var proxyFromEnvTests = []proxyFromEnvTest{ {req: "https://secure.tld/", env: "http.proxy.tld", httpsenv: "secure.proxy.tld", want: "http://secure.proxy.tld"}, {req: "https://secure.tld/", env: "http.proxy.tld", httpsenv: "https://secure.proxy.tld", want: "https://secure.proxy.tld"}, + // Issue 16405: don't use HTTP_PROXY in a CGI environment, + // where HTTP_PROXY can be attacker-controlled. + {env: "http://10.1.2.3:8080", reqmeth: "POST", + want: "<nil>", + wanterr: errors.New("net/http: refusing to use HTTP_PROXY value in CGI environment; see golang.org/s/cgihttpproxy")}, + {want: "<nil>"}, {noenv: "example.com", req: "http://example.com/", env: "proxy", want: "<nil>"}, @@ -2047,6 +2188,7 @@ func TestProxyFromEnvironment(t *testing.T) { os.Setenv("HTTP_PROXY", tt.env) os.Setenv("HTTPS_PROXY", tt.httpsenv) os.Setenv("NO_PROXY", tt.noenv) + os.Setenv("REQUEST_METHOD", tt.reqmeth) ResetCachedEnvironment() reqURL := tt.req if reqURL == "" { @@ -2065,6 +2207,7 @@ func TestProxyFromEnvironment(t *testing.T) { } func TestIdleConnChannelLeak(t *testing.T) { + // Not parallel: uses global test hooks. var mu sync.Mutex var n int @@ -2208,7 +2351,7 @@ func TestTransportTLSHandshakeTimeout(t *testing.T) { // Trying to repro golang.org/issue/3514 func TestTLSServerClosesConnection(t *testing.T) { defer afterTest(t) - setFlaky(t, 7634) + testenv.SkipFlaky(t, 7634) closedc := make(chan bool, 1) ts := httptest.NewTLSServer(HandlerFunc(func(w ResponseWriter, r *Request) { @@ -2273,7 +2416,7 @@ func TestTLSServerClosesConnection(t *testing.T) { } // byteFromChanReader is an io.Reader that reads a single byte at a -// time from the channel. When the channel is closed, the reader +// time from the channel. When the channel is closed, the reader // returns io.EOF. type byteFromChanReader chan byte @@ -2296,6 +2439,7 @@ func (c byteFromChanReader) Read(p []byte) (n int, err error) { // questionable state. // golang.org/issue/7569 func TestTransportNoReuseAfterEarlyResponse(t *testing.T) { + setParallel(t) defer afterTest(t) var sconn struct { sync.Mutex @@ -2398,98 +2542,101 @@ type errorReader struct { func (e errorReader) Read(p []byte) (int, error) { return 0, e.err } -type plan9SleepReader struct{} - -func (plan9SleepReader) Read(p []byte) (int, error) { - if runtime.GOOS == "plan9" { - // After the fix to unblock TCP Reads in - // https://golang.org/cl/15941, this sleep is required - // on plan9 to make sure TCP Writes before an - // immediate TCP close go out on the wire. On Plan 9, - // it seems that a hangup of a TCP connection with - // queued data doesn't send the queued data first. - // https://golang.org/issue/9554 - time.Sleep(50 * time.Millisecond) - } - return 0, io.EOF -} - type closerFunc func() error func (f closerFunc) Close() error { return f() } +type writerFuncConn struct { + net.Conn + write func(p []byte) (n int, err error) +} + +func (c writerFuncConn) Write(p []byte) (n int, err error) { return c.write(p) } + // Issue 4677. If we try to reuse a connection that the server is in the // process of closing, we may end up successfully writing out our request (or a // portion of our request) only to find a connection error when we try to read // from (or finish writing to) the socket. // // NOTE: we resend a request only if the request is idempotent, we reused a -// keep-alive connection, and we haven't yet received any header data. This +// keep-alive connection, and we haven't yet received any header data. This // automatically prevents an infinite resend loop because we'll run out of the // cached keep-alive connections eventually. func TestRetryIdempotentRequestsOnError(t *testing.T) { defer afterTest(t) + var ( + mu sync.Mutex + logbuf bytes.Buffer + ) + logf := func(format string, args ...interface{}) { + mu.Lock() + defer mu.Unlock() + fmt.Fprintf(&logbuf, format, args...) + logbuf.WriteByte('\n') + } + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + logf("Handler") + w.Header().Set("X-Status", "ok") })) defer ts.Close() - tr := &Transport{} + var writeNumAtomic int32 + tr := &Transport{ + Dial: func(network, addr string) (net.Conn, error) { + logf("Dial") + c, err := net.Dial(network, ts.Listener.Addr().String()) + if err != nil { + logf("Dial error: %v", err) + return nil, err + } + return &writerFuncConn{ + Conn: c, + write: func(p []byte) (n int, err error) { + if atomic.AddInt32(&writeNumAtomic, 1) == 2 { + logf("intentional write failure") + return 0, errors.New("second write fails") + } + logf("Write(%q)", p) + return c.Write(p) + }, + }, nil + }, + } + defer tr.CloseIdleConnections() c := &Client{Transport: tr} - const N = 2 - retryc := make(chan struct{}, N) SetRoundTripRetried(func() { - retryc <- struct{}{} + logf("Retried.") }) defer SetRoundTripRetried(nil) - for n := 0; n < 100; n++ { - // open 2 conns - errc := make(chan error, N) - for i := 0; i < N; i++ { - // start goroutines, send on errc - go func() { - res, err := c.Get(ts.URL) - if err == nil { - res.Body.Close() - } - errc <- err - }() - } - for i := 0; i < N; i++ { - if err := <-errc; err != nil { - t.Fatal(err) - } - } - - ts.CloseClientConnections() - for i := 0; i < N; i++ { - go func() { - res, err := c.Get(ts.URL) - if err == nil { - res.Body.Close() - } - errc <- err - }() + for i := 0; i < 3; i++ { + res, err := c.Get("http://fake.golang/") + if err != nil { + t.Fatalf("i=%d: Get = %v", i, err) } + res.Body.Close() + } - for i := 0; i < N; i++ { - if err := <-errc; err != nil { - t.Fatal(err) - } - } - for i := 0; i < N; i++ { - select { - case <-retryc: - // we triggered a retry, test was successful - t.Logf("finished after %d runs\n", n) - return - default: - } - } + mu.Lock() + got := logbuf.String() + mu.Unlock() + const want = `Dial +Write("GET / HTTP/1.1\r\nHost: fake.golang\r\nUser-Agent: Go-http-client/1.1\r\nAccept-Encoding: gzip\r\n\r\n") +Handler +intentional write failure +Retried. +Dial +Write("GET / HTTP/1.1\r\nHost: fake.golang\r\nUser-Agent: Go-http-client/1.1\r\nAccept-Encoding: gzip\r\n\r\n") +Handler +Write("GET / HTTP/1.1\r\nHost: fake.golang\r\nUser-Agent: Go-http-client/1.1\r\nAccept-Encoding: gzip\r\n\r\n") +Handler +` + if got != want { + t.Errorf("Log of events differs. Got:\n%s\nWant:\n%s", got, want) } - t.Fatal("did not trigger any retries") } // Issue 6981 @@ -2508,7 +2655,7 @@ func TestTransportClosesBodyOnError(t *testing.T) { io.Reader io.Closer }{ - io.MultiReader(io.LimitReader(neverEnding('x'), 1<<20), plan9SleepReader{}, errorReader{fakeErr}), + io.MultiReader(io.LimitReader(neverEnding('x'), 1<<20), errorReader{fakeErr}), closerFunc(func() error { select { case didClose <- true: @@ -2540,6 +2687,8 @@ func TestTransportClosesBodyOnError(t *testing.T) { } func TestTransportDialTLS(t *testing.T) { + setParallel(t) + defer afterTest(t) var mu sync.Mutex // guards following var gotReq, didDial bool @@ -2817,14 +2966,8 @@ func TestTransportFlushesBodyChunks(t *testing.T) { defer res.Body.Close() want := []string{ - // Because Request.ContentLength = 0, the body is sniffed for 1 byte to determine whether there's content. - // That explains the initial "num0" being split into "n" and "um0". - // The first byte is included with the request headers Write. Perhaps in the future - // we will want to flush the headers out early if the first byte of the request body is - // taking a long time to arrive. But not yet. "POST / HTTP/1.1\r\nHost: localhost:8080\r\nUser-Agent: x\r\nTransfer-Encoding: chunked\r\nAccept-Encoding: gzip\r\n\r\n" + - "1\r\nn\r\n", - "4\r\num0\n\r\n", + "5\r\nnum0\n\r\n", "5\r\nnum1\n\r\n", "5\r\nnum2\n\r\n", "0\r\n\r\n", @@ -2888,6 +3031,11 @@ func TestTransportAutomaticHTTP2(t *testing.T) { testTransportAutoHTTP(t, &Transport{}, true) } +// golang.org/issue/14391: also check DefaultTransport +func TestTransportAutomaticHTTP2_DefaultTransport(t *testing.T) { + testTransportAutoHTTP(t, DefaultTransport.(*Transport), true) +} + func TestTransportAutomaticHTTP2_TLSNextProto(t *testing.T) { testTransportAutoHTTP(t, &Transport{ TLSNextProto: make(map[string]func(string, *tls.Conn) RoundTripper), @@ -2903,6 +3051,21 @@ func TestTransportAutomaticHTTP2_TLSConfig(t *testing.T) { func TestTransportAutomaticHTTP2_ExpectContinueTimeout(t *testing.T) { testTransportAutoHTTP(t, &Transport{ ExpectContinueTimeout: 1 * time.Second, + }, true) +} + +func TestTransportAutomaticHTTP2_Dial(t *testing.T) { + var d net.Dialer + testTransportAutoHTTP(t, &Transport{ + Dial: d.Dial, + }, false) +} + +func TestTransportAutomaticHTTP2_DialTLS(t *testing.T) { + testTransportAutoHTTP(t, &Transport{ + DialTLS: func(network, addr string) (net.Conn, error) { + panic("unused") + }, }, false) } @@ -3033,6 +3196,701 @@ func TestNoCrashReturningTransportAltConn(t *testing.T) { <-handledPendingDial } +func TestTransportReuseConnection_Gzip_Chunked(t *testing.T) { + testTransportReuseConnection_Gzip(t, true) +} + +func TestTransportReuseConnection_Gzip_ContentLength(t *testing.T) { + testTransportReuseConnection_Gzip(t, false) +} + +// Make sure we re-use underlying TCP connection for gzipped responses too. +func testTransportReuseConnection_Gzip(t *testing.T, chunked bool) { + setParallel(t) + defer afterTest(t) + addr := make(chan string, 2) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + addr <- r.RemoteAddr + w.Header().Set("Content-Encoding", "gzip") + if chunked { + w.(Flusher).Flush() + } + w.Write(rgz) // arbitrary gzip response + })) + defer ts.Close() + + tr := &Transport{} + defer tr.CloseIdleConnections() + c := &Client{Transport: tr} + for i := 0; i < 2; i++ { + res, err := c.Get(ts.URL) + if err != nil { + t.Fatal(err) + } + buf := make([]byte, len(rgz)) + if n, err := io.ReadFull(res.Body, buf); err != nil { + t.Errorf("%d. ReadFull = %v, %v", i, n, err) + } + // Note: no res.Body.Close call. It should work without it, + // since the flate.Reader's internal buffering will hit EOF + // and that should be sufficient. + } + a1, a2 := <-addr, <-addr + if a1 != a2 { + t.Fatalf("didn't reuse connection") + } +} + +func TestTransportResponseHeaderLength(t *testing.T) { + setParallel(t) + defer afterTest(t) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + if r.URL.Path == "/long" { + w.Header().Set("Long", strings.Repeat("a", 1<<20)) + } + })) + defer ts.Close() + + tr := &Transport{ + MaxResponseHeaderBytes: 512 << 10, + } + defer tr.CloseIdleConnections() + c := &Client{Transport: tr} + if res, err := c.Get(ts.URL); err != nil { + t.Fatal(err) + } else { + res.Body.Close() + } + + res, err := c.Get(ts.URL + "/long") + if err == nil { + defer res.Body.Close() + var n int64 + for k, vv := range res.Header { + for _, v := range vv { + n += int64(len(k)) + int64(len(v)) + } + } + t.Fatalf("Unexpected success. Got %v and %d bytes of response headers", res.Status, n) + } + if want := "server response headers exceeded 524288 bytes"; !strings.Contains(err.Error(), want) { + t.Errorf("got error: %v; want %q", err, want) + } +} + +func TestTransportEventTrace(t *testing.T) { testTransportEventTrace(t, h1Mode, false) } +func TestTransportEventTrace_h2(t *testing.T) { testTransportEventTrace(t, h2Mode, false) } + +// test a non-nil httptrace.ClientTrace but with all hooks set to zero. +func TestTransportEventTrace_NoHooks(t *testing.T) { testTransportEventTrace(t, h1Mode, true) } +func TestTransportEventTrace_NoHooks_h2(t *testing.T) { testTransportEventTrace(t, h2Mode, true) } + +func testTransportEventTrace(t *testing.T, h2 bool, noHooks bool) { + defer afterTest(t) + const resBody = "some body" + gotWroteReqEvent := make(chan struct{}) + cst := newClientServerTest(t, h2, HandlerFunc(func(w ResponseWriter, r *Request) { + if _, err := ioutil.ReadAll(r.Body); err != nil { + t.Error(err) + } + if !noHooks { + select { + case <-gotWroteReqEvent: + case <-time.After(5 * time.Second): + t.Error("timeout waiting for WroteRequest event") + } + } + io.WriteString(w, resBody) + })) + defer cst.close() + + cst.tr.ExpectContinueTimeout = 1 * time.Second + + var mu sync.Mutex // guards buf + var buf bytes.Buffer + logf := func(format string, args ...interface{}) { + mu.Lock() + defer mu.Unlock() + fmt.Fprintf(&buf, format, args...) + buf.WriteByte('\n') + } + + addrStr := cst.ts.Listener.Addr().String() + ip, port, err := net.SplitHostPort(addrStr) + if err != nil { + t.Fatal(err) + } + + // Install a fake DNS server. + ctx := context.WithValue(context.Background(), nettrace.LookupIPAltResolverKey{}, func(ctx context.Context, host string) ([]net.IPAddr, error) { + if host != "dns-is-faked.golang" { + t.Errorf("unexpected DNS host lookup for %q", host) + return nil, nil + } + return []net.IPAddr{{IP: net.ParseIP(ip)}}, nil + }) + + req, _ := NewRequest("POST", cst.scheme()+"://dns-is-faked.golang:"+port, strings.NewReader("some body")) + trace := &httptrace.ClientTrace{ + GetConn: func(hostPort string) { logf("Getting conn for %v ...", hostPort) }, + GotConn: func(ci httptrace.GotConnInfo) { logf("got conn: %+v", ci) }, + GotFirstResponseByte: func() { logf("first response byte") }, + PutIdleConn: func(err error) { logf("PutIdleConn = %v", err) }, + DNSStart: func(e httptrace.DNSStartInfo) { logf("DNS start: %+v", e) }, + DNSDone: func(e httptrace.DNSDoneInfo) { logf("DNS done: %+v", e) }, + ConnectStart: func(network, addr string) { logf("ConnectStart: Connecting to %s %s ...", network, addr) }, + ConnectDone: func(network, addr string, err error) { + if err != nil { + t.Errorf("ConnectDone: %v", err) + } + logf("ConnectDone: connected to %s %s = %v", network, addr, err) + }, + Wait100Continue: func() { logf("Wait100Continue") }, + Got100Continue: func() { logf("Got100Continue") }, + WroteRequest: func(e httptrace.WroteRequestInfo) { + logf("WroteRequest: %+v", e) + close(gotWroteReqEvent) + }, + } + if h2 { + trace.TLSHandshakeStart = func() { logf("tls handshake start") } + trace.TLSHandshakeDone = func(s tls.ConnectionState, err error) { + logf("tls handshake done. ConnectionState = %v \n err = %v", s, err) + } + } + if noHooks { + // zero out all func pointers, trying to get some path to crash + *trace = httptrace.ClientTrace{} + } + req = req.WithContext(httptrace.WithClientTrace(ctx, trace)) + + req.Header.Set("Expect", "100-continue") + res, err := cst.c.Do(req) + if err != nil { + t.Fatal(err) + } + logf("got roundtrip.response") + slurp, err := ioutil.ReadAll(res.Body) + if err != nil { + t.Fatal(err) + } + logf("consumed body") + if string(slurp) != resBody || res.StatusCode != 200 { + t.Fatalf("Got %q, %v; want %q, 200 OK", slurp, res.Status, resBody) + } + res.Body.Close() + + if noHooks { + // Done at this point. Just testing a full HTTP + // requests can happen with a trace pointing to a zero + // ClientTrace, full of nil func pointers. + return + } + + mu.Lock() + got := buf.String() + mu.Unlock() + + wantOnce := func(sub string) { + if strings.Count(got, sub) != 1 { + t.Errorf("expected substring %q exactly once in output.", sub) + } + } + wantOnceOrMore := func(sub string) { + if strings.Count(got, sub) == 0 { + t.Errorf("expected substring %q at least once in output.", sub) + } + } + wantOnce("Getting conn for dns-is-faked.golang:" + port) + wantOnce("DNS start: {Host:dns-is-faked.golang}") + wantOnce("DNS done: {Addrs:[{IP:" + ip + " Zone:}] Err:<nil> Coalesced:false}") + wantOnce("got conn: {") + wantOnceOrMore("Connecting to tcp " + addrStr) + wantOnceOrMore("connected to tcp " + addrStr + " = <nil>") + wantOnce("Reused:false WasIdle:false IdleTime:0s") + wantOnce("first response byte") + if h2 { + wantOnce("tls handshake start") + wantOnce("tls handshake done") + } else { + wantOnce("PutIdleConn = <nil>") + } + wantOnce("Wait100Continue") + wantOnce("Got100Continue") + wantOnce("WroteRequest: {Err:<nil>}") + if strings.Contains(got, " to udp ") { + t.Errorf("should not see UDP (DNS) connections") + } + if t.Failed() { + t.Errorf("Output:\n%s", got) + } +} + +func TestTransportEventTraceRealDNS(t *testing.T) { + if testing.Short() && testenv.Builder() == "" { + // Skip this test in short mode (the default for + // all.bash), in case the user is using a shady/ISP + // DNS server hijacking queries. + // See issues 16732, 16716. + // Our builders use 8.8.8.8, though, which correctly + // returns NXDOMAIN, so still run this test there. + t.Skip("skipping in short mode") + } + defer afterTest(t) + tr := &Transport{} + defer tr.CloseIdleConnections() + c := &Client{Transport: tr} + + var mu sync.Mutex // guards buf + var buf bytes.Buffer + logf := func(format string, args ...interface{}) { + mu.Lock() + defer mu.Unlock() + fmt.Fprintf(&buf, format, args...) + buf.WriteByte('\n') + } + + req, _ := NewRequest("GET", "http://dns-should-not-resolve.golang:80", nil) + trace := &httptrace.ClientTrace{ + DNSStart: func(e httptrace.DNSStartInfo) { logf("DNSStart: %+v", e) }, + DNSDone: func(e httptrace.DNSDoneInfo) { logf("DNSDone: %+v", e) }, + ConnectStart: func(network, addr string) { logf("ConnectStart: %s %s", network, addr) }, + ConnectDone: func(network, addr string, err error) { logf("ConnectDone: %s %s %v", network, addr, err) }, + } + req = req.WithContext(httptrace.WithClientTrace(context.Background(), trace)) + + resp, err := c.Do(req) + if err == nil { + resp.Body.Close() + t.Fatal("expected error during DNS lookup") + } + + mu.Lock() + got := buf.String() + mu.Unlock() + + wantSub := func(sub string) { + if !strings.Contains(got, sub) { + t.Errorf("expected substring %q in output.", sub) + } + } + wantSub("DNSStart: {Host:dns-should-not-resolve.golang}") + wantSub("DNSDone: {Addrs:[] Err:") + if strings.Contains(got, "ConnectStart") || strings.Contains(got, "ConnectDone") { + t.Errorf("should not see Connect events") + } + if t.Failed() { + t.Errorf("Output:\n%s", got) + } +} + +// Issue 14353: port can only contain digits. +func TestTransportRejectsAlphaPort(t *testing.T) { + res, err := Get("http://dummy.tld:123foo/bar") + if err == nil { + res.Body.Close() + t.Fatal("unexpected success") + } + ue, ok := err.(*url.Error) + if !ok { + t.Fatalf("got %#v; want *url.Error", err) + } + got := ue.Err.Error() + want := `invalid URL port "123foo"` + if got != want { + t.Errorf("got error %q; want %q", got, want) + } +} + +// Test the httptrace.TLSHandshake{Start,Done} hooks with a https http1 +// connections. The http2 test is done in TestTransportEventTrace_h2 +func TestTLSHandshakeTrace(t *testing.T) { + defer afterTest(t) + s := httptest.NewTLSServer(HandlerFunc(func(w ResponseWriter, r *Request) {})) + defer s.Close() + + var mu sync.Mutex + var start, done bool + trace := &httptrace.ClientTrace{ + TLSHandshakeStart: func() { + mu.Lock() + defer mu.Unlock() + start = true + }, + TLSHandshakeDone: func(s tls.ConnectionState, err error) { + mu.Lock() + defer mu.Unlock() + done = true + if err != nil { + t.Fatal("Expected error to be nil but was:", err) + } + }, + } + + tr := &Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} + defer tr.CloseIdleConnections() + c := &Client{Transport: tr} + req, err := NewRequest("GET", s.URL, nil) + if err != nil { + t.Fatal("Unable to construct test request:", err) + } + req = req.WithContext(httptrace.WithClientTrace(req.Context(), trace)) + + r, err := c.Do(req) + if err != nil { + t.Fatal("Unexpected error making request:", err) + } + r.Body.Close() + mu.Lock() + defer mu.Unlock() + if !start { + t.Fatal("Expected TLSHandshakeStart to be called, but wasn't") + } + if !done { + t.Fatal("Expected TLSHandshakeDone to be called, but wasnt't") + } +} + +func TestTransportMaxIdleConns(t *testing.T) { + defer afterTest(t) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + // No body for convenience. + })) + defer ts.Close() + tr := &Transport{ + MaxIdleConns: 4, + } + defer tr.CloseIdleConnections() + + ip, port, err := net.SplitHostPort(ts.Listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + c := &Client{Transport: tr} + ctx := context.WithValue(context.Background(), nettrace.LookupIPAltResolverKey{}, func(ctx context.Context, host string) ([]net.IPAddr, error) { + return []net.IPAddr{{IP: net.ParseIP(ip)}}, nil + }) + + hitHost := func(n int) { + req, _ := NewRequest("GET", fmt.Sprintf("http://host-%d.dns-is-faked.golang:"+port, n), nil) + req = req.WithContext(ctx) + res, err := c.Do(req) + if err != nil { + t.Fatal(err) + } + res.Body.Close() + } + for i := 0; i < 4; i++ { + hitHost(i) + } + want := []string{ + "|http|host-0.dns-is-faked.golang:" + port, + "|http|host-1.dns-is-faked.golang:" + port, + "|http|host-2.dns-is-faked.golang:" + port, + "|http|host-3.dns-is-faked.golang:" + port, + } + if got := tr.IdleConnKeysForTesting(); !reflect.DeepEqual(got, want) { + t.Fatalf("idle conn keys mismatch.\n got: %q\nwant: %q\n", got, want) + } + + // Now hitting the 5th host should kick out the first host: + hitHost(4) + want = []string{ + "|http|host-1.dns-is-faked.golang:" + port, + "|http|host-2.dns-is-faked.golang:" + port, + "|http|host-3.dns-is-faked.golang:" + port, + "|http|host-4.dns-is-faked.golang:" + port, + } + if got := tr.IdleConnKeysForTesting(); !reflect.DeepEqual(got, want) { + t.Fatalf("idle conn keys mismatch after 5th host.\n got: %q\nwant: %q\n", got, want) + } +} + +func TestTransportIdleConnTimeout_h1(t *testing.T) { testTransportIdleConnTimeout(t, h1Mode) } +func TestTransportIdleConnTimeout_h2(t *testing.T) { testTransportIdleConnTimeout(t, h2Mode) } +func testTransportIdleConnTimeout(t *testing.T, h2 bool) { + if testing.Short() { + t.Skip("skipping in short mode") + } + defer afterTest(t) + + const timeout = 1 * time.Second + + cst := newClientServerTest(t, h2, HandlerFunc(func(w ResponseWriter, r *Request) { + // No body for convenience. + })) + defer cst.close() + tr := cst.tr + tr.IdleConnTimeout = timeout + defer tr.CloseIdleConnections() + c := &Client{Transport: tr} + + idleConns := func() []string { + if h2 { + return tr.IdleConnStrsForTesting_h2() + } else { + return tr.IdleConnStrsForTesting() + } + } + + var conn string + doReq := func(n int) { + req, _ := NewRequest("GET", cst.ts.URL, nil) + req = req.WithContext(httptrace.WithClientTrace(context.Background(), &httptrace.ClientTrace{ + PutIdleConn: func(err error) { + if err != nil { + t.Errorf("failed to keep idle conn: %v", err) + } + }, + })) + res, err := c.Do(req) + if err != nil { + t.Fatal(err) + } + res.Body.Close() + conns := idleConns() + if len(conns) != 1 { + t.Fatalf("req %v: unexpected number of idle conns: %q", n, conns) + } + if conn == "" { + conn = conns[0] + } + if conn != conns[0] { + t.Fatalf("req %v: cached connection changed; expected the same one throughout the test", n) + } + } + for i := 0; i < 3; i++ { + doReq(i) + time.Sleep(timeout / 2) + } + time.Sleep(timeout * 3 / 2) + if got := idleConns(); len(got) != 0 { + t.Errorf("idle conns = %q; want none", got) + } +} + +// Issue 16208: Go 1.7 crashed after Transport.IdleConnTimeout if an +// HTTP/2 connection was established but but its caller no longer +// wanted it. (Assuming the connection cache was enabled, which it is +// by default) +// +// This test reproduced the crash by setting the IdleConnTimeout low +// (to make the test reasonable) and then making a request which is +// canceled by the DialTLS hook, which then also waits to return the +// real connection until after the RoundTrip saw the error. Then we +// know the successful tls.Dial from DialTLS will need to go into the +// idle pool. Then we give it a of time to explode. +func TestIdleConnH2Crash(t *testing.T) { + setParallel(t) + cst := newClientServerTest(t, h2Mode, HandlerFunc(func(w ResponseWriter, r *Request) { + // nothing + })) + defer cst.close() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + sawDoErr := make(chan bool, 1) + testDone := make(chan struct{}) + defer close(testDone) + + cst.tr.IdleConnTimeout = 5 * time.Millisecond + cst.tr.DialTLS = func(network, addr string) (net.Conn, error) { + c, err := tls.Dial(network, addr, &tls.Config{ + InsecureSkipVerify: true, + NextProtos: []string{"h2"}, + }) + if err != nil { + t.Error(err) + return nil, err + } + if cs := c.ConnectionState(); cs.NegotiatedProtocol != "h2" { + t.Errorf("protocol = %q; want %q", cs.NegotiatedProtocol, "h2") + c.Close() + return nil, errors.New("bogus") + } + + cancel() + + failTimer := time.NewTimer(5 * time.Second) + defer failTimer.Stop() + select { + case <-sawDoErr: + case <-testDone: + case <-failTimer.C: + t.Error("timeout in DialTLS, waiting too long for cst.c.Do to fail") + } + return c, nil + } + + req, _ := NewRequest("GET", cst.ts.URL, nil) + req = req.WithContext(ctx) + res, err := cst.c.Do(req) + if err == nil { + res.Body.Close() + t.Fatal("unexpected success") + } + sawDoErr <- true + + // Wait for the explosion. + time.Sleep(cst.tr.IdleConnTimeout * 10) +} + +type funcConn struct { + net.Conn + read func([]byte) (int, error) + write func([]byte) (int, error) +} + +func (c funcConn) Read(p []byte) (int, error) { return c.read(p) } +func (c funcConn) Write(p []byte) (int, error) { return c.write(p) } +func (c funcConn) Close() error { return nil } + +// Issue 16465: Transport.RoundTrip should return the raw net.Conn.Read error from Peek +// back to the caller. +func TestTransportReturnsPeekError(t *testing.T) { + errValue := errors.New("specific error value") + + wrote := make(chan struct{}) + var wroteOnce sync.Once + + tr := &Transport{ + Dial: func(network, addr string) (net.Conn, error) { + c := funcConn{ + read: func([]byte) (int, error) { + <-wrote + return 0, errValue + }, + write: func(p []byte) (int, error) { + wroteOnce.Do(func() { close(wrote) }) + return len(p), nil + }, + } + return c, nil + }, + } + _, err := tr.RoundTrip(httptest.NewRequest("GET", "http://fake.tld/", nil)) + if err != errValue { + t.Errorf("error = %#v; want %v", err, errValue) + } +} + +// Issue 13835: international domain names should work +func TestTransportIDNA_h1(t *testing.T) { testTransportIDNA(t, h1Mode) } +func TestTransportIDNA_h2(t *testing.T) { testTransportIDNA(t, h2Mode) } +func testTransportIDNA(t *testing.T, h2 bool) { + defer afterTest(t) + + const uniDomain = "гофер.го" + const punyDomain = "xn--c1ae0ajs.xn--c1aw" + + var port string + cst := newClientServerTest(t, h2, HandlerFunc(func(w ResponseWriter, r *Request) { + want := punyDomain + ":" + port + if r.Host != want { + t.Errorf("Host header = %q; want %q", r.Host, want) + } + if h2 { + if r.TLS == nil { + t.Errorf("r.TLS == nil") + } else if r.TLS.ServerName != punyDomain { + t.Errorf("TLS.ServerName = %q; want %q", r.TLS.ServerName, punyDomain) + } + } + w.Header().Set("Hit-Handler", "1") + })) + defer cst.close() + + ip, port, err := net.SplitHostPort(cst.ts.Listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + + // Install a fake DNS server. + ctx := context.WithValue(context.Background(), nettrace.LookupIPAltResolverKey{}, func(ctx context.Context, host string) ([]net.IPAddr, error) { + if host != punyDomain { + t.Errorf("got DNS host lookup for %q; want %q", host, punyDomain) + return nil, nil + } + return []net.IPAddr{{IP: net.ParseIP(ip)}}, nil + }) + + req, _ := NewRequest("GET", cst.scheme()+"://"+uniDomain+":"+port, nil) + trace := &httptrace.ClientTrace{ + GetConn: func(hostPort string) { + want := net.JoinHostPort(punyDomain, port) + if hostPort != want { + t.Errorf("getting conn for %q; want %q", hostPort, want) + } + }, + DNSStart: func(e httptrace.DNSStartInfo) { + if e.Host != punyDomain { + t.Errorf("DNSStart Host = %q; want %q", e.Host, punyDomain) + } + }, + } + req = req.WithContext(httptrace.WithClientTrace(ctx, trace)) + + res, err := cst.tr.RoundTrip(req) + if err != nil { + t.Fatal(err) + } + defer res.Body.Close() + if res.Header.Get("Hit-Handler") != "1" { + out, err := httputil.DumpResponse(res, true) + if err != nil { + t.Fatal(err) + } + t.Errorf("Response body wasn't from Handler. Got:\n%s\n", out) + } +} + +// Issue 13290: send User-Agent in proxy CONNECT +func TestTransportProxyConnectHeader(t *testing.T) { + defer afterTest(t) + reqc := make(chan *Request, 1) + ts := httptest.NewServer(HandlerFunc(func(w ResponseWriter, r *Request) { + if r.Method != "CONNECT" { + t.Errorf("method = %q; want CONNECT", r.Method) + } + reqc <- r + c, _, err := w.(Hijacker).Hijack() + if err != nil { + t.Errorf("Hijack: %v", err) + return + } + c.Close() + })) + defer ts.Close() + tr := &Transport{ + ProxyConnectHeader: Header{ + "User-Agent": {"foo"}, + "Other": {"bar"}, + }, + Proxy: func(r *Request) (*url.URL, error) { + return url.Parse(ts.URL) + }, + } + defer tr.CloseIdleConnections() + c := &Client{Transport: tr} + res, err := c.Get("https://dummy.tld/") // https to force a CONNECT + if err == nil { + res.Body.Close() + t.Errorf("unexpected success") + } + select { + case <-time.After(3 * time.Second): + t.Fatal("timeout") + case r := <-reqc: + if got, want := r.Header.Get("User-Agent"), "foo"; got != want { + t.Errorf("CONNECT request User-Agent = %q; want %q", got, want) + } + if got, want := r.Header.Get("Other"), "bar"; got != want { + t.Errorf("CONNECT request Other = %q; want %q", got, want) + } + } +} + var errFakeRoundTrip = errors.New("fake roundtrip") type funcRoundTripper func() diff --git a/libgo/go/net/interface.go b/libgo/go/net/interface.go index 9c7b5da8fe..b3297f249d 100644 --- a/libgo/go/net/interface.go +++ b/libgo/go/net/interface.go @@ -1,10 +1,20 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package net -import "errors" +import ( + "errors" + "sync" + "time" +) + +// BUG(mikio): On NaCl, methods and functions related to +// Interface are not implemented. + +// BUG(mikio): On DragonFly BSD, NetBSD, OpenBSD, Plan 9 and Solaris, +// the MulticastAddrs method of Interface is not implemented. var ( errInvalidInterface = errors.New("invalid network interface") @@ -15,7 +25,7 @@ var ( ) // Interface represents a mapping between network interface name -// and index. It also represents network interface facility +// and index. It also represents network interface facility // information. type Interface struct { Index int // positive integer that starts at one, zero is never used @@ -59,7 +69,8 @@ func (f Flags) String() string { return s } -// Addrs returns interface addresses for a specific interface. +// Addrs returns a list of unicast interface addresses for a specific +// interface. func (ifi *Interface) Addrs() ([]Addr, error) { if ifi == nil { return nil, &OpError{Op: "route", Net: "ip+net", Source: nil, Addr: nil, Err: errInvalidInterface} @@ -71,8 +82,8 @@ func (ifi *Interface) Addrs() ([]Addr, error) { return ifat, err } -// MulticastAddrs returns multicast, joined group addresses for -// a specific interface. +// MulticastAddrs returns a list of multicast, joined group addresses +// for a specific interface. func (ifi *Interface) MulticastAddrs() ([]Addr, error) { if ifi == nil { return nil, &OpError{Op: "route", Net: "ip+net", Source: nil, Addr: nil, Err: errInvalidInterface} @@ -88,13 +99,19 @@ func (ifi *Interface) MulticastAddrs() ([]Addr, error) { func Interfaces() ([]Interface, error) { ift, err := interfaceTable(0) if err != nil { - err = &OpError{Op: "route", Net: "ip+net", Source: nil, Addr: nil, Err: err} + return nil, &OpError{Op: "route", Net: "ip+net", Source: nil, Addr: nil, Err: err} + } + if len(ift) != 0 { + zoneCache.update(ift) } - return ift, err + return ift, nil } -// InterfaceAddrs returns a list of the system's network interface +// InterfaceAddrs returns a list of the system's unicast interface // addresses. +// +// The returned list does not identify the associated interface; use +// Interfaces and Interface.Addrs for more detail. func InterfaceAddrs() ([]Addr, error) { ifat, err := interfaceAddrTable(nil) if err != nil { @@ -104,6 +121,10 @@ func InterfaceAddrs() ([]Addr, error) { } // InterfaceByIndex returns the interface specified by index. +// +// On Solaris, it returns one of the logical network interfaces +// sharing the logical data link; for more precision use +// InterfaceByName. func InterfaceByIndex(index int) (*Interface, error) { if index <= 0 { return nil, &OpError{Op: "route", Net: "ip+net", Source: nil, Addr: nil, Err: errInvalidInterfaceIndex} @@ -137,6 +158,9 @@ func InterfaceByName(name string) (*Interface, error) { if err != nil { return nil, &OpError{Op: "route", Net: "ip+net", Source: nil, Addr: nil, Err: err} } + if len(ift) != 0 { + zoneCache.update(ift) + } for _, ifi := range ift { if name == ifi.Name { return &ifi, nil @@ -144,3 +168,73 @@ func InterfaceByName(name string) (*Interface, error) { } return nil, &OpError{Op: "route", Net: "ip+net", Source: nil, Addr: nil, Err: errNoSuchInterface} } + +// An ipv6ZoneCache represents a cache holding partial network +// interface information. It is used for reducing the cost of IPv6 +// addressing scope zone resolution. +// +// Multiple names sharing the index are managed by first-come +// first-served basis for consistency. +type ipv6ZoneCache struct { + sync.RWMutex // guard the following + lastFetched time.Time // last time routing information was fetched + toIndex map[string]int // interface name to its index + toName map[int]string // interface index to its name +} + +var zoneCache = ipv6ZoneCache{ + toIndex: make(map[string]int), + toName: make(map[int]string), +} + +func (zc *ipv6ZoneCache) update(ift []Interface) { + zc.Lock() + defer zc.Unlock() + now := time.Now() + if zc.lastFetched.After(now.Add(-60 * time.Second)) { + return + } + zc.lastFetched = now + if len(ift) == 0 { + var err error + if ift, err = interfaceTable(0); err != nil { + return + } + } + zc.toIndex = make(map[string]int, len(ift)) + zc.toName = make(map[int]string, len(ift)) + for _, ifi := range ift { + zc.toIndex[ifi.Name] = ifi.Index + if _, ok := zc.toName[ifi.Index]; !ok { + zc.toName[ifi.Index] = ifi.Name + } + } +} + +func zoneToString(zone int) string { + if zone == 0 { + return "" + } + zoneCache.update(nil) + zoneCache.RLock() + defer zoneCache.RUnlock() + name, ok := zoneCache.toName[zone] + if !ok { + name = uitoa(uint(zone)) + } + return name +} + +func zoneToInt(zone string) int { + if zone == "" { + return 0 + } + zoneCache.update(nil) + zoneCache.RLock() + defer zoneCache.RUnlock() + index, ok := zoneCache.toIndex[zone] + if !ok { + index, _, _ = dtoi(zone) + } + return index +} diff --git a/libgo/go/net/interface_bsd.go b/libgo/go/net/interface_bsd.go index 208f37f9fd..35b1c26815 100644 --- a/libgo/go/net/interface_bsd.go +++ b/libgo/go/net/interface_bsd.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -7,74 +7,54 @@ package net import ( - "os" "syscall" - "unsafe" + + "golang_org/x/net/route" ) // If the ifindex is zero, interfaceTable returns mappings of all -// network interfaces. Otherwise it returns a mapping of a specific +// network interfaces. Otherwise it returns a mapping of a specific // interface. func interfaceTable(ifindex int) ([]Interface, error) { - tab, err := syscall.RouteRIB(syscall.NET_RT_IFLIST, ifindex) + msgs, err := interfaceMessages(ifindex) if err != nil { - return nil, os.NewSyscallError("routerib", err) + return nil, err } - msgs, err := syscall.ParseRoutingMessage(tab) - if err != nil { - return nil, os.NewSyscallError("parseroutingmessage", err) + n := len(msgs) + if ifindex != 0 { + n = 1 } - return parseInterfaceTable(ifindex, msgs) -} - -func parseInterfaceTable(ifindex int, msgs []syscall.RoutingMessage) ([]Interface, error) { - var ift []Interface -loop: + ift := make([]Interface, n) + n = 0 for _, m := range msgs { switch m := m.(type) { - case *syscall.InterfaceMessage: - if ifindex == 0 || ifindex == int(m.Header.Index) { - ifi, err := newLink(m) - if err != nil { - return nil, err - } - ift = append(ift, *ifi) - if ifindex == int(m.Header.Index) { - break loop + case *route.InterfaceMessage: + if ifindex != 0 && ifindex != m.Index { + continue + } + ift[n].Index = m.Index + ift[n].Name = m.Name + ift[n].Flags = linkFlags(m.Flags) + if sa, ok := m.Addrs[syscall.RTAX_IFP].(*route.LinkAddr); ok && len(sa.Addr) > 0 { + ift[n].HardwareAddr = make([]byte, len(sa.Addr)) + copy(ift[n].HardwareAddr, sa.Addr) + } + for _, sys := range m.Sys() { + if imx, ok := sys.(*route.InterfaceMetrics); ok { + ift[n].MTU = imx.MTU + break } } + n++ + if ifindex == m.Index { + return ift[:n], nil + } } } - return ift, nil -} - -func newLink(m *syscall.InterfaceMessage) (*Interface, error) { - sas, err := syscall.ParseRoutingSockaddr(m) - if err != nil { - return nil, os.NewSyscallError("parseroutingsockaddr", err) - } - ifi := &Interface{Index: int(m.Header.Index), Flags: linkFlags(m.Header.Flags)} - sa, _ := sas[syscall.RTAX_IFP].(*syscall.SockaddrDatalink) - if sa != nil { - // NOTE: SockaddrDatalink.Data is minimum work area, - // can be larger. - m.Data = m.Data[unsafe.Offsetof(sa.Data):] - var name [syscall.IFNAMSIZ]byte - for i := 0; i < int(sa.Nlen); i++ { - name[i] = byte(m.Data[i]) - } - ifi.Name = string(name[:sa.Nlen]) - ifi.MTU = int(m.Header.Data.Mtu) - addr := make([]byte, sa.Alen) - for i := 0; i < int(sa.Alen); i++ { - addr[i] = byte(m.Data[int(sa.Nlen)+i]) - } - ifi.HardwareAddr = addr[:sa.Alen] - } - return ifi, nil + return ift[:n], nil } -func linkFlags(rawFlags int32) Flags { +func linkFlags(rawFlags int) Flags { var f Flags if rawFlags&syscall.IFF_UP != 0 { f |= FlagUp @@ -95,81 +75,44 @@ func linkFlags(rawFlags int32) Flags { } // If the ifi is nil, interfaceAddrTable returns addresses for all -// network interfaces. Otherwise it returns addresses for a specific +// network interfaces. Otherwise it returns addresses for a specific // interface. func interfaceAddrTable(ifi *Interface) ([]Addr, error) { index := 0 if ifi != nil { index = ifi.Index } - tab, err := syscall.RouteRIB(syscall.NET_RT_IFLIST, index) - if err != nil { - return nil, os.NewSyscallError("routerib", err) - } - msgs, err := syscall.ParseRoutingMessage(tab) + msgs, err := interfaceMessages(index) if err != nil { - return nil, os.NewSyscallError("parseroutingmessage", err) + return nil, err } - var ift []Interface - if index == 0 { - ift, err = parseInterfaceTable(index, msgs) - if err != nil { - return nil, err - } - } - var ifat []Addr + ifat := make([]Addr, 0, len(msgs)) for _, m := range msgs { switch m := m.(type) { - case *syscall.InterfaceAddrMessage: - if index == 0 || index == int(m.Header.Index) { - if index == 0 { - var err error - ifi, err = interfaceByIndex(ift, int(m.Header.Index)) - if err != nil { - return nil, err - } - } - ifa, err := newAddr(ifi, m) - if err != nil { - return nil, err - } - if ifa != nil { - ifat = append(ifat, ifa) - } + case *route.InterfaceAddrMessage: + if index != 0 && index != m.Index { + continue + } + var mask IPMask + switch sa := m.Addrs[syscall.RTAX_NETMASK].(type) { + case *route.Inet4Addr: + mask = IPv4Mask(sa.IP[0], sa.IP[1], sa.IP[2], sa.IP[3]) + case *route.Inet6Addr: + mask = make(IPMask, IPv6len) + copy(mask, sa.IP[:]) + } + var ip IP + switch sa := m.Addrs[syscall.RTAX_IFA].(type) { + case *route.Inet4Addr: + ip = IPv4(sa.IP[0], sa.IP[1], sa.IP[2], sa.IP[3]) + case *route.Inet6Addr: + ip = make(IP, IPv6len) + copy(ip, sa.IP[:]) + } + if ip != nil && mask != nil { // NetBSD may contain route.LinkAddr + ifat = append(ifat, &IPNet{IP: ip, Mask: mask}) } } } return ifat, nil } - -func newAddr(ifi *Interface, m *syscall.InterfaceAddrMessage) (*IPNet, error) { - sas, err := syscall.ParseRoutingSockaddr(m) - if err != nil { - return nil, os.NewSyscallError("parseroutingsockaddr", err) - } - ifa := &IPNet{} - switch sa := sas[syscall.RTAX_NETMASK].(type) { - case *syscall.SockaddrInet4: - ifa.Mask = IPv4Mask(sa.Addr[0], sa.Addr[1], sa.Addr[2], sa.Addr[3]) - case *syscall.SockaddrInet6: - ifa.Mask = make(IPMask, IPv6len) - copy(ifa.Mask, sa.Addr[:]) - } - switch sa := sas[syscall.RTAX_IFA].(type) { - case *syscall.SockaddrInet4: - ifa.IP = IPv4(sa.Addr[0], sa.Addr[1], sa.Addr[2], sa.Addr[3]) - case *syscall.SockaddrInet6: - ifa.IP = make(IP, IPv6len) - copy(ifa.IP, sa.Addr[:]) - // NOTE: KAME based IPv6 protcol stack usually embeds - // the interface index in the interface-local or - // link-local address as the kernel-internal form. - if ifa.IP.IsLinkLocalUnicast() { - ifa.IP[2], ifa.IP[3] = 0, 0 - } - } - if ifa.IP == nil || ifa.Mask == nil { - return nil, nil // Sockaddrs contain syscall.SockaddrDatalink on NetBSD - } - return ifa, nil -} diff --git a/libgo/go/net/interface_bsdvar.go b/libgo/go/net/interface_bsdvar.go new file mode 100644 index 0000000000..0b84ca37d4 --- /dev/null +++ b/libgo/go/net/interface_bsdvar.go @@ -0,0 +1,28 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build dragonfly netbsd openbsd + +package net + +import ( + "syscall" + + "golang_org/x/net/route" +) + +func interfaceMessages(ifindex int) ([]route.Message, error) { + rib, err := route.FetchRIB(syscall.AF_UNSPEC, syscall.NET_RT_IFLIST, ifindex) + if err != nil { + return nil, err + } + return route.ParseRIB(syscall.NET_RT_IFLIST, rib) +} + +// interfaceMulticastAddrTable returns addresses for a specific +// interface. +func interfaceMulticastAddrTable(ifi *Interface) ([]Addr, error) { + // TODO(mikio): Implement this like other platforms. + return nil, nil +} diff --git a/libgo/go/net/interface_darwin.go b/libgo/go/net/interface_darwin.go index b7a333849d..2ec8e1cc6e 100644 --- a/libgo/go/net/interface_darwin.go +++ b/libgo/go/net/interface_darwin.go @@ -1,62 +1,53 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package net import ( - "os" "syscall" + + "golang_org/x/net/route" ) +func interfaceMessages(ifindex int) ([]route.Message, error) { + rib, err := route.FetchRIB(syscall.AF_UNSPEC, syscall.NET_RT_IFLIST, ifindex) + if err != nil { + return nil, err + } + return route.ParseRIB(syscall.NET_RT_IFLIST, rib) +} + // interfaceMulticastAddrTable returns addresses for a specific // interface. func interfaceMulticastAddrTable(ifi *Interface) ([]Addr, error) { - tab, err := syscall.RouteRIB(syscall.NET_RT_IFLIST2, ifi.Index) + rib, err := route.FetchRIB(syscall.AF_UNSPEC, syscall.NET_RT_IFLIST2, ifi.Index) if err != nil { - return nil, os.NewSyscallError("routerib", err) + return nil, err } - msgs, err := syscall.ParseRoutingMessage(tab) + msgs, err := route.ParseRIB(syscall.NET_RT_IFLIST2, rib) if err != nil { - return nil, os.NewSyscallError("parseroutingmessage", err) + return nil, err } - var ifmat []Addr + ifmat := make([]Addr, 0, len(msgs)) for _, m := range msgs { switch m := m.(type) { - case *syscall.InterfaceMulticastAddrMessage: - if ifi.Index == int(m.Header.Index) { - ifma, err := newMulticastAddr(ifi, m) - if err != nil { - return nil, err - } - if ifma != nil { - ifmat = append(ifmat, ifma) - } + case *route.InterfaceMulticastAddrMessage: + if ifi.Index != m.Index { + continue + } + var ip IP + switch sa := m.Addrs[syscall.RTAX_IFA].(type) { + case *route.Inet4Addr: + ip = IPv4(sa.IP[0], sa.IP[1], sa.IP[2], sa.IP[3]) + case *route.Inet6Addr: + ip = make(IP, IPv6len) + copy(ip, sa.IP[:]) + } + if ip != nil { + ifmat = append(ifmat, &IPAddr{IP: ip}) } } } return ifmat, nil } - -func newMulticastAddr(ifi *Interface, m *syscall.InterfaceMulticastAddrMessage) (*IPAddr, error) { - sas, err := syscall.ParseRoutingSockaddr(m) - if err != nil { - return nil, os.NewSyscallError("parseroutingsockaddr", err) - } - switch sa := sas[syscall.RTAX_IFA].(type) { - case *syscall.SockaddrInet4: - return &IPAddr{IP: IPv4(sa.Addr[0], sa.Addr[1], sa.Addr[2], sa.Addr[3])}, nil - case *syscall.SockaddrInet6: - ifma := IPAddr{IP: make(IP, IPv6len)} - copy(ifma.IP, sa.Addr[:]) - // NOTE: KAME based IPv6 protcol stack usually embeds - // the interface index in the interface-local or - // link-local address as the kernel-internal form. - if ifma.IP.IsInterfaceLocalMulticast() || ifma.IP.IsLinkLocalMulticast() { - ifma.IP[2], ifma.IP[3] = 0, 0 - } - return &ifma, nil - default: - return nil, nil - } -} diff --git a/libgo/go/net/interface_dragonfly.go b/libgo/go/net/interface_dragonfly.go deleted file mode 100644 index c9ce5a7ac1..0000000000 --- a/libgo/go/net/interface_dragonfly.go +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package net - -// interfaceMulticastAddrTable returns addresses for a specific -// interface. -func interfaceMulticastAddrTable(ifi *Interface) ([]Addr, error) { - // TODO(mikio): Implement this like other platforms. - return nil, nil -} diff --git a/libgo/go/net/interface_freebsd.go b/libgo/go/net/interface_freebsd.go index c42d90b740..8a7d6f67c0 100644 --- a/libgo/go/net/interface_freebsd.go +++ b/libgo/go/net/interface_freebsd.go @@ -1,62 +1,58 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package net import ( - "os" "syscall" + + "golang_org/x/net/route" ) +func interfaceMessages(ifindex int) ([]route.Message, error) { + typ := route.RIBType(syscall.NET_RT_IFLISTL) + rib, err := route.FetchRIB(syscall.AF_UNSPEC, typ, ifindex) + if err != nil { + typ = route.RIBType(syscall.NET_RT_IFLIST) + rib, err = route.FetchRIB(syscall.AF_UNSPEC, typ, ifindex) + } + if err != nil { + return nil, err + } + return route.ParseRIB(typ, rib) +} + // interfaceMulticastAddrTable returns addresses for a specific // interface. func interfaceMulticastAddrTable(ifi *Interface) ([]Addr, error) { - tab, err := syscall.RouteRIB(syscall.NET_RT_IFMALIST, ifi.Index) + rib, err := route.FetchRIB(syscall.AF_UNSPEC, syscall.NET_RT_IFMALIST, ifi.Index) if err != nil { - return nil, os.NewSyscallError("routerib", err) + return nil, err } - msgs, err := syscall.ParseRoutingMessage(tab) + msgs, err := route.ParseRIB(syscall.NET_RT_IFMALIST, rib) if err != nil { - return nil, os.NewSyscallError("parseroutingmessage", err) + return nil, err } - var ifmat []Addr + ifmat := make([]Addr, 0, len(msgs)) for _, m := range msgs { switch m := m.(type) { - case *syscall.InterfaceMulticastAddrMessage: - if ifi.Index == int(m.Header.Index) { - ifma, err := newMulticastAddr(ifi, m) - if err != nil { - return nil, err - } - if ifma != nil { - ifmat = append(ifmat, ifma) - } + case *route.InterfaceMulticastAddrMessage: + if ifi.Index != m.Index { + continue + } + var ip IP + switch sa := m.Addrs[syscall.RTAX_IFA].(type) { + case *route.Inet4Addr: + ip = IPv4(sa.IP[0], sa.IP[1], sa.IP[2], sa.IP[3]) + case *route.Inet6Addr: + ip = make(IP, IPv6len) + copy(ip, sa.IP[:]) + } + if ip != nil { + ifmat = append(ifmat, &IPAddr{IP: ip}) } } } return ifmat, nil } - -func newMulticastAddr(ifi *Interface, m *syscall.InterfaceMulticastAddrMessage) (*IPAddr, error) { - sas, err := syscall.ParseRoutingSockaddr(m) - if err != nil { - return nil, os.NewSyscallError("parseroutingsockaddr", err) - } - switch sa := sas[syscall.RTAX_IFA].(type) { - case *syscall.SockaddrInet4: - return &IPAddr{IP: IPv4(sa.Addr[0], sa.Addr[1], sa.Addr[2], sa.Addr[3])}, nil - case *syscall.SockaddrInet6: - ifma := IPAddr{IP: make(IP, IPv6len)} - copy(ifma.IP, sa.Addr[:]) - // NOTE: KAME based IPv6 protcol stack usually embeds - // the interface index in the interface-local or - // link-local address as the kernel-internal form. - if ifma.IP.IsInterfaceLocalMulticast() || ifma.IP.IsLinkLocalMulticast() { - ifma.IP[2], ifma.IP[3] = 0, 0 - } - return &ifma, nil - default: - return nil, nil - } -} diff --git a/libgo/go/net/interface_linux.go b/libgo/go/net/interface_linux.go index ef2042920e..5e391b28b0 100644 --- a/libgo/go/net/interface_linux.go +++ b/libgo/go/net/interface_linux.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -11,7 +11,7 @@ import ( ) // If the ifindex is zero, interfaceTable returns mappings of all -// network interfaces. Otherwise it returns a mapping of a specific +// network interfaces. Otherwise it returns a mapping of a specific // interface. func interfaceTable(ifindex int) ([]Interface, error) { tab, err := syscall.NetlinkRIB(syscall.RTM_GETLINK, syscall.AF_UNSPEC) @@ -115,7 +115,7 @@ func linkFlags(rawFlags uint32) Flags { } // If the ifi is nil, interfaceAddrTable returns addresses for all -// network interfaces. Otherwise it returns addresses for a specific +// network interfaces. Otherwise it returns addresses for a specific // interface. func interfaceAddrTable(ifi *Interface) ([]Addr, error) { tab, err := syscall.NetlinkRIB(syscall.RTM_GETADDR, syscall.AF_UNSPEC) diff --git a/libgo/go/net/interface_netbsd.go b/libgo/go/net/interface_netbsd.go deleted file mode 100644 index c9ce5a7ac1..0000000000 --- a/libgo/go/net/interface_netbsd.go +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package net - -// interfaceMulticastAddrTable returns addresses for a specific -// interface. -func interfaceMulticastAddrTable(ifi *Interface) ([]Addr, error) { - // TODO(mikio): Implement this like other platforms. - return nil, nil -} diff --git a/libgo/go/net/interface_openbsd.go b/libgo/go/net/interface_openbsd.go deleted file mode 100644 index c9ce5a7ac1..0000000000 --- a/libgo/go/net/interface_openbsd.go +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package net - -// interfaceMulticastAddrTable returns addresses for a specific -// interface. -func interfaceMulticastAddrTable(ifi *Interface) ([]Addr, error) { - // TODO(mikio): Implement this like other platforms. - return nil, nil -} diff --git a/libgo/go/net/interface_plan9.go b/libgo/go/net/interface_plan9.go new file mode 100644 index 0000000000..e5d77390f8 --- /dev/null +++ b/libgo/go/net/interface_plan9.go @@ -0,0 +1,198 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "errors" + "os" +) + +// If the ifindex is zero, interfaceTable returns mappings of all +// network interfaces. Otherwise it returns a mapping of a specific +// interface. +func interfaceTable(ifindex int) ([]Interface, error) { + if ifindex == 0 { + n, err := interfaceCount() + if err != nil { + return nil, err + } + ifcs := make([]Interface, n) + for i := range ifcs { + ifc, err := readInterface(i) + if err != nil { + return nil, err + } + ifcs[i] = *ifc + } + return ifcs, nil + } + + ifc, err := readInterface(ifindex - 1) + if err != nil { + return nil, err + } + return []Interface{*ifc}, nil +} + +func readInterface(i int) (*Interface, error) { + ifc := &Interface{ + Index: i + 1, // Offset the index by one to suit the contract + Name: netdir + "/ipifc/" + itoa(i), // Name is the full path to the interface path in plan9 + } + + ifcstat := ifc.Name + "/status" + ifcstatf, err := open(ifcstat) + if err != nil { + return nil, err + } + defer ifcstatf.close() + + line, ok := ifcstatf.readLine() + if !ok { + return nil, errors.New("invalid interface status file: " + ifcstat) + } + + fields := getFields(line) + if len(fields) < 4 { + return nil, errors.New("invalid interface status file: " + ifcstat) + } + + device := fields[1] + mtustr := fields[3] + + mtu, _, ok := dtoi(mtustr) + if !ok { + return nil, errors.New("invalid status file of interface: " + ifcstat) + } + ifc.MTU = mtu + + // Not a loopback device + if device != "/dev/null" { + deviceaddrf, err := open(device + "/addr") + if err != nil { + return nil, err + } + defer deviceaddrf.close() + + line, ok = deviceaddrf.readLine() + if !ok { + return nil, errors.New("invalid address file for interface: " + device + "/addr") + } + + if len(line) > 0 && len(line)%2 == 0 { + ifc.HardwareAddr = make([]byte, len(line)/2) + var ok bool + for i := range ifc.HardwareAddr { + j := (i + 1) * 2 + ifc.HardwareAddr[i], ok = xtoi2(line[i*2:j], 0) + if !ok { + ifc.HardwareAddr = ifc.HardwareAddr[:i] + break + } + } + } + + ifc.Flags = FlagUp | FlagBroadcast | FlagMulticast + } else { + ifc.Flags = FlagUp | FlagMulticast | FlagLoopback + } + + return ifc, nil +} + +func interfaceCount() (int, error) { + d, err := os.Open(netdir + "/ipifc") + if err != nil { + return -1, err + } + defer d.Close() + + names, err := d.Readdirnames(0) + if err != nil { + return -1, err + } + + // Assumes that numbered files in ipifc are strictly + // the incrementing numbered directories for the + // interfaces + c := 0 + for _, name := range names { + if _, _, ok := dtoi(name); !ok { + continue + } + c++ + } + + return c, nil +} + +// If the ifi is nil, interfaceAddrTable returns addresses for all +// network interfaces. Otherwise it returns addresses for a specific +// interface. +func interfaceAddrTable(ifi *Interface) ([]Addr, error) { + var ifcs []Interface + if ifi == nil { + var err error + ifcs, err = interfaceTable(0) + if err != nil { + return nil, err + } + } else { + ifcs = []Interface{*ifi} + } + + addrs := make([]Addr, len(ifcs)) + for i, ifc := range ifcs { + status := ifc.Name + "/status" + statusf, err := open(status) + if err != nil { + return nil, err + } + defer statusf.close() + + line, ok := statusf.readLine() + line, ok = statusf.readLine() + if !ok { + return nil, errors.New("cannot parse IP address for interface: " + status) + } + + // This assumes only a single address for the interface. + fields := getFields(line) + if len(fields) < 1 { + return nil, errors.New("cannot parse IP address for interface: " + status) + } + addr := fields[0] + ip := ParseIP(addr) + if ip == nil { + return nil, errors.New("cannot parse IP address for interface: " + status) + } + + // The mask is represented as CIDR relative to the IPv6 address. + // Plan 9 internal representation is always IPv6. + maskfld := fields[1] + maskfld = maskfld[1:] + pfxlen, _, ok := dtoi(maskfld) + if !ok { + return nil, errors.New("cannot parse network mask for interface: " + status) + } + var mask IPMask + if ip.To4() != nil { // IPv4 or IPv6 IPv4-mapped address + mask = CIDRMask(pfxlen-8*len(v4InV6Prefix), 8*IPv4len) + } + if ip.To16() != nil && ip.To4() == nil { // IPv6 address + mask = CIDRMask(pfxlen, 8*IPv6len) + } + + addrs[i] = &IPNet{IP: ip, Mask: mask} + } + + return addrs, nil +} + +// interfaceMulticastAddrTable returns addresses for a specific +// interface. +func interfaceMulticastAddrTable(ifi *Interface) ([]Addr, error) { + return nil, nil +} diff --git a/libgo/go/net/interface_solaris.go b/libgo/go/net/interface_solaris.go new file mode 100644 index 0000000000..dc8ffbfcb8 --- /dev/null +++ b/libgo/go/net/interface_solaris.go @@ -0,0 +1,107 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "syscall" + + "golang_org/x/net/lif" +) + +// If the ifindex is zero, interfaceTable returns mappings of all +// network interfaces. Otherwise it returns a mapping of a specific +// interface. +func interfaceTable(ifindex int) ([]Interface, error) { + lls, err := lif.Links(syscall.AF_UNSPEC, "") + if err != nil { + return nil, err + } + var ift []Interface + for _, ll := range lls { + if ifindex != 0 && ifindex != ll.Index { + continue + } + ifi := Interface{Index: ll.Index, MTU: ll.MTU, Name: ll.Name, Flags: linkFlags(ll.Flags)} + if len(ll.Addr) > 0 { + ifi.HardwareAddr = HardwareAddr(ll.Addr) + } + ift = append(ift, ifi) + } + return ift, nil +} + +const ( + sysIFF_UP = 0x1 + sysIFF_BROADCAST = 0x2 + sysIFF_DEBUG = 0x4 + sysIFF_LOOPBACK = 0x8 + sysIFF_POINTOPOINT = 0x10 + sysIFF_NOTRAILERS = 0x20 + sysIFF_RUNNING = 0x40 + sysIFF_NOARP = 0x80 + sysIFF_PROMISC = 0x100 + sysIFF_ALLMULTI = 0x200 + sysIFF_INTELLIGENT = 0x400 + sysIFF_MULTICAST = 0x800 + sysIFF_MULTI_BCAST = 0x1000 + sysIFF_UNNUMBERED = 0x2000 + sysIFF_PRIVATE = 0x8000 +) + +func linkFlags(rawFlags int) Flags { + var f Flags + if rawFlags&sysIFF_UP != 0 { + f |= FlagUp + } + if rawFlags&sysIFF_BROADCAST != 0 { + f |= FlagBroadcast + } + if rawFlags&sysIFF_LOOPBACK != 0 { + f |= FlagLoopback + } + if rawFlags&sysIFF_POINTOPOINT != 0 { + f |= FlagPointToPoint + } + if rawFlags&sysIFF_MULTICAST != 0 { + f |= FlagMulticast + } + return f +} + +// If the ifi is nil, interfaceAddrTable returns addresses for all +// network interfaces. Otherwise it returns addresses for a specific +// interface. +func interfaceAddrTable(ifi *Interface) ([]Addr, error) { + var name string + if ifi != nil { + name = ifi.Name + } + as, err := lif.Addrs(syscall.AF_UNSPEC, name) + if err != nil { + return nil, err + } + var ifat []Addr + for _, a := range as { + var ip IP + var mask IPMask + switch a := a.(type) { + case *lif.Inet4Addr: + ip = IPv4(a.IP[0], a.IP[1], a.IP[2], a.IP[3]) + mask = CIDRMask(a.PrefixLen, 8*IPv4len) + case *lif.Inet6Addr: + ip = make(IP, IPv6len) + copy(ip, a.IP[:]) + mask = CIDRMask(a.PrefixLen, 8*IPv6len) + } + ifat = append(ifat, &IPNet{IP: ip, Mask: mask}) + } + return ifat, nil +} + +// interfaceMulticastAddrTable returns addresses for a specific +// interface. +func interfaceMulticastAddrTable(ifi *Interface) ([]Addr, error) { + return nil, nil +} diff --git a/libgo/go/net/interface_stub.go b/libgo/go/net/interface_stub.go index c38fb7f765..3b0a1aeacf 100644 --- a/libgo/go/net/interface_stub.go +++ b/libgo/go/net/interface_stub.go @@ -1,20 +1,20 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build nacl plan9 solaris +// +build nacl package net // If the ifindex is zero, interfaceTable returns mappings of all -// network interfaces. Otherwise it returns a mapping of a specific +// network interfaces. Otherwise it returns a mapping of a specific // interface. func interfaceTable(ifindex int) ([]Interface, error) { return nil, nil } // If the ifi is nil, interfaceAddrTable returns addresses for all -// network interfaces. Otherwise it returns addresses for a specific +// network interfaces. Otherwise it returns addresses for a specific // interface. func interfaceAddrTable(ifi *Interface) ([]Addr, error) { return nil, nil diff --git a/libgo/go/net/interface_test.go b/libgo/go/net/interface_test.go index 7bdd924150..38a2ca4656 100644 --- a/libgo/go/net/interface_test.go +++ b/libgo/go/net/interface_test.go @@ -1,17 +1,18 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package net import ( + "fmt" "reflect" "runtime" "testing" ) // loopbackInterface returns an available logical network interface -// for loopback tests. It returns nil if no suitable interface is +// for loopback tests. It returns nil if no suitable interface is // found. func loopbackInterface() *Interface { ift, err := Interfaces() @@ -47,27 +48,25 @@ func ipv6LinkLocalUnicastAddr(ifi *Interface) string { return "" } -type routeStats struct { - loop int // # of active loopback interfaces - other int // # of active other interfaces - - uni4, uni6 int // # of active connected unicast, anycast routes - multi4, multi6 int // # of active connected multicast route clones -} - func TestInterfaces(t *testing.T) { ift, err := Interfaces() if err != nil { t.Fatal(err) } - var stats routeStats for _, ifi := range ift { ifxi, err := InterfaceByIndex(ifi.Index) if err != nil { t.Fatal(err) } - if !reflect.DeepEqual(ifxi, &ifi) { - t.Errorf("got %v; want %v", ifxi, ifi) + switch runtime.GOOS { + case "solaris": + if ifxi.Index != ifi.Index { + t.Errorf("got %v; want %v", ifxi, ifi) + } + default: + if !reflect.DeepEqual(ifxi, &ifi) { + t.Errorf("got %v; want %v", ifxi, ifi) + } } ifxn, err := InterfaceByName(ifi.Name) if err != nil { @@ -76,56 +75,7 @@ func TestInterfaces(t *testing.T) { if !reflect.DeepEqual(ifxn, &ifi) { t.Errorf("got %v; want %v", ifxn, ifi) } - t.Logf("%q: flags %q, ifindex %v, mtu %v", ifi.Name, ifi.Flags.String(), ifi.Index, ifi.MTU) - t.Logf("hardware address %q", ifi.HardwareAddr.String()) - if ifi.Flags&FlagUp != 0 { - if ifi.Flags&FlagLoopback != 0 { - stats.loop++ - } else { - stats.other++ - } - } - n4, n6 := testInterfaceAddrs(t, &ifi) - stats.uni4 += n4 - stats.uni6 += n6 - n4, n6 = testInterfaceMulticastAddrs(t, &ifi) - stats.multi4 += n4 - stats.multi6 += n6 - } - switch runtime.GOOS { - case "nacl", "plan9", "solaris": - default: - // Test the existence of connected unicast routes for - // IPv4. - if supportsIPv4 && stats.loop+stats.other > 0 && stats.uni4 == 0 { - t.Errorf("num IPv4 unicast routes = 0; want >0; summary: %+v", stats) - } - // Test the existence of connected unicast routes for - // IPv6. We can assume the existence of ::1/128 when - // at least one looopback interface is installed. - if supportsIPv6 && stats.loop > 0 && stats.uni6 == 0 { - t.Errorf("num IPv6 unicast routes = 0; want >0; summary: %+v", stats) - } - } - switch runtime.GOOS { - case "dragonfly", "nacl", "netbsd", "openbsd", "plan9", "solaris": - default: - // Test the existence of connected multicast route - // clones for IPv4. Unlike IPv6, IPv4 multicast - // capability is not a mandatory feature, and so this - // test is disabled. - //if supportsIPv4 && stats.loop > 0 && stats.uni4 > 1 && stats.multi4 == 0 { - // t.Errorf("num IPv4 multicast route clones = 0; want >0; summary: %+v", stats) - //} - // Test the existence of connected multicast route - // clones for IPv6. Some platform never uses loopback - // interface as the nexthop for multicast routing. - // We can assume the existence of connected multicast - // route clones when at least two connected unicast - // routes, ::1/128 and other, are installed. - if supportsIPv6 && stats.loop > 0 && stats.uni6 > 1 && stats.multi6 == 0 { - t.Errorf("num IPv6 multicast route clones = 0; want >0; summary: %+v", stats) - } + t.Logf("%s: flags=%v index=%d mtu=%d hwaddr=%v", ifi.Name, ifi.Flags, ifi.Index, ifi.MTU, ifi.HardwareAddr) } } @@ -134,132 +84,217 @@ func TestInterfaceAddrs(t *testing.T) { if err != nil { t.Fatal(err) } - var stats routeStats - for _, ifi := range ift { - if ifi.Flags&FlagUp != 0 { - if ifi.Flags&FlagLoopback != 0 { - stats.loop++ - } else { - stats.other++ - } - } - } + ifStats := interfaceStats(ift) ifat, err := InterfaceAddrs() if err != nil { t.Fatal(err) } - stats.uni4, stats.uni6 = testAddrs(t, ifat) - // Test the existence of connected unicast routes for IPv4. - if supportsIPv4 && stats.loop+stats.other > 0 && stats.uni4 == 0 { - t.Errorf("num IPv4 unicast routes = 0; want >0; summary: %+v", stats) + uniStats, err := validateInterfaceUnicastAddrs(ifat) + if err != nil { + t.Fatal(err) } - // Test the existence of connected unicast routes for IPv6. - // We can assume the existence of ::1/128 when at least one - // looopback interface is installed. - if supportsIPv6 && stats.loop > 0 && stats.uni6 == 0 { - t.Errorf("num IPv6 unicast routes = 0; want >0; summary: %+v", stats) + if err := checkUnicastStats(ifStats, uniStats); err != nil { + t.Fatal(err) } } -func testInterfaceAddrs(t *testing.T, ifi *Interface) (naf4, naf6 int) { - ifat, err := ifi.Addrs() +func TestInterfaceUnicastAddrs(t *testing.T) { + ift, err := Interfaces() + if err != nil { + t.Fatal(err) + } + ifStats := interfaceStats(ift) if err != nil { t.Fatal(err) } - return testAddrs(t, ifat) + var uniStats routeStats + for _, ifi := range ift { + ifat, err := ifi.Addrs() + if err != nil { + t.Fatal(ifi, err) + } + stats, err := validateInterfaceUnicastAddrs(ifat) + if err != nil { + t.Fatal(ifi, err) + } + uniStats.ipv4 += stats.ipv4 + uniStats.ipv6 += stats.ipv6 + } + if err := checkUnicastStats(ifStats, &uniStats); err != nil { + t.Fatal(err) + } } -func testInterfaceMulticastAddrs(t *testing.T, ifi *Interface) (nmaf4, nmaf6 int) { - ifmat, err := ifi.MulticastAddrs() +func TestInterfaceMulticastAddrs(t *testing.T) { + ift, err := Interfaces() if err != nil { t.Fatal(err) } - return testMulticastAddrs(t, ifmat) + ifStats := interfaceStats(ift) + ifat, err := InterfaceAddrs() + if err != nil { + t.Fatal(err) + } + uniStats, err := validateInterfaceUnicastAddrs(ifat) + if err != nil { + t.Fatal(err) + } + var multiStats routeStats + for _, ifi := range ift { + ifmat, err := ifi.MulticastAddrs() + if err != nil { + t.Fatal(ifi, err) + } + stats, err := validateInterfaceMulticastAddrs(ifmat) + if err != nil { + t.Fatal(ifi, err) + } + multiStats.ipv4 += stats.ipv4 + multiStats.ipv6 += stats.ipv6 + } + if err := checkMulticastStats(ifStats, uniStats, &multiStats); err != nil { + t.Fatal(err) + } +} + +type ifStats struct { + loop int // # of active loopback interfaces + other int // # of active other interfaces +} + +func interfaceStats(ift []Interface) *ifStats { + var stats ifStats + for _, ifi := range ift { + if ifi.Flags&FlagUp != 0 { + if ifi.Flags&FlagLoopback != 0 { + stats.loop++ + } else { + stats.other++ + } + } + } + return &stats +} + +type routeStats struct { + ipv4, ipv6 int // # of active connected unicast, anycast or multicast routes } -func testAddrs(t *testing.T, ifat []Addr) (naf4, naf6 int) { +func validateInterfaceUnicastAddrs(ifat []Addr) (*routeStats, error) { + // Note: BSD variants allow assigning any IPv4/IPv6 address + // prefix to IP interface. For example, + // - 0.0.0.0/0 through 255.255.255.255/32 + // - ::/0 through ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/128 + // In other words, there is no tightly-coupled combination of + // interface address prefixes and connected routes. + stats := new(routeStats) for _, ifa := range ifat { switch ifa := ifa.(type) { case *IPNet: - if ifa == nil || ifa.IP == nil || ifa.IP.IsUnspecified() || ifa.IP.IsMulticast() || ifa.Mask == nil { - t.Errorf("unexpected value: %#v", ifa) - continue + if ifa == nil || ifa.IP == nil || ifa.IP.IsMulticast() || ifa.Mask == nil { + return nil, fmt.Errorf("unexpected value: %#v", ifa) } if len(ifa.IP) != IPv6len { - t.Errorf("should be internal representation either IPv6 or IPv6 IPv4-mapped address: %#v", ifa) - continue + return nil, fmt.Errorf("should be internal representation either IPv6 or IPv4-mapped IPv6 address: %#v", ifa) } prefixLen, maxPrefixLen := ifa.Mask.Size() if ifa.IP.To4() != nil { if 0 >= prefixLen || prefixLen > 8*IPv4len || maxPrefixLen != 8*IPv4len { - t.Errorf("unexpected prefix length: %d/%d", prefixLen, maxPrefixLen) - continue + return nil, fmt.Errorf("unexpected prefix length: %d/%d for %#v", prefixLen, maxPrefixLen, ifa) } if ifa.IP.IsLoopback() && (prefixLen != 8 && prefixLen != 8*IPv4len) { // see RFC 1122 - t.Errorf("unexpected prefix length for IPv4 loopback: %d/%d", prefixLen, maxPrefixLen) - continue + return nil, fmt.Errorf("unexpected prefix length: %d/%d for %#v", prefixLen, maxPrefixLen, ifa) } - naf4++ + stats.ipv4++ } if ifa.IP.To16() != nil && ifa.IP.To4() == nil { if 0 >= prefixLen || prefixLen > 8*IPv6len || maxPrefixLen != 8*IPv6len { - t.Errorf("unexpected prefix length: %d/%d", prefixLen, maxPrefixLen) - continue + return nil, fmt.Errorf("unexpected prefix length: %d/%d for %#v", prefixLen, maxPrefixLen, ifa) } if ifa.IP.IsLoopback() && prefixLen != 8*IPv6len { // see RFC 4291 - t.Errorf("unexpected prefix length for IPv6 loopback: %d/%d", prefixLen, maxPrefixLen) - continue + return nil, fmt.Errorf("unexpected prefix length: %d/%d for %#v", prefixLen, maxPrefixLen, ifa) } - naf6++ + stats.ipv6++ } - t.Logf("interface address %q", ifa.String()) case *IPAddr: - if ifa == nil || ifa.IP == nil || ifa.IP.IsUnspecified() || ifa.IP.IsMulticast() { - t.Errorf("unexpected value: %#v", ifa) - continue + if ifa == nil || ifa.IP == nil || ifa.IP.IsMulticast() { + return nil, fmt.Errorf("unexpected value: %#v", ifa) } if len(ifa.IP) != IPv6len { - t.Errorf("should be internal representation either IPv6 or IPv6 IPv4-mapped address: %#v", ifa) - continue + return nil, fmt.Errorf("should be internal representation either IPv6 or IPv4-mapped IPv6 address: %#v", ifa) } if ifa.IP.To4() != nil { - naf4++ + stats.ipv4++ } if ifa.IP.To16() != nil && ifa.IP.To4() == nil { - naf6++ + stats.ipv6++ } - t.Logf("interface address %s", ifa.String()) default: - t.Errorf("unexpected type: %T", ifa) + return nil, fmt.Errorf("unexpected type: %T", ifa) } } - return + return stats, nil } -func testMulticastAddrs(t *testing.T, ifmat []Addr) (nmaf4, nmaf6 int) { - for _, ifma := range ifmat { - switch ifma := ifma.(type) { +func validateInterfaceMulticastAddrs(ifat []Addr) (*routeStats, error) { + stats := new(routeStats) + for _, ifa := range ifat { + switch ifa := ifa.(type) { case *IPAddr: - if ifma == nil || ifma.IP == nil || ifma.IP.IsUnspecified() || !ifma.IP.IsMulticast() { - t.Errorf("unexpected value: %+v", ifma) - continue + if ifa == nil || ifa.IP == nil || ifa.IP.IsUnspecified() || !ifa.IP.IsMulticast() { + return nil, fmt.Errorf("unexpected value: %#v", ifa) } - if len(ifma.IP) != IPv6len { - t.Errorf("should be internal representation either IPv6 or IPv6 IPv4-mapped address: %#v", ifma) - continue + if len(ifa.IP) != IPv6len { + return nil, fmt.Errorf("should be internal representation either IPv6 or IPv4-mapped IPv6 address: %#v", ifa) } - if ifma.IP.To4() != nil { - nmaf4++ + if ifa.IP.To4() != nil { + stats.ipv4++ } - if ifma.IP.To16() != nil && ifma.IP.To4() == nil { - nmaf6++ + if ifa.IP.To16() != nil && ifa.IP.To4() == nil { + stats.ipv6++ } - t.Logf("joined group address %q", ifma.String()) default: - t.Errorf("unexpected type: %T", ifma) + return nil, fmt.Errorf("unexpected type: %T", ifa) } } - return + return stats, nil +} + +func checkUnicastStats(ifStats *ifStats, uniStats *routeStats) error { + // Test the existence of connected unicast routes for IPv4. + if supportsIPv4 && ifStats.loop+ifStats.other > 0 && uniStats.ipv4 == 0 { + return fmt.Errorf("num IPv4 unicast routes = 0; want >0; summary: %+v, %+v", ifStats, uniStats) + } + // Test the existence of connected unicast routes for IPv6. + // We can assume the existence of ::1/128 when at least one + // loopback interface is installed. + if supportsIPv6 && ifStats.loop > 0 && uniStats.ipv6 == 0 { + return fmt.Errorf("num IPv6 unicast routes = 0; want >0; summary: %+v, %+v", ifStats, uniStats) + } + return nil +} + +func checkMulticastStats(ifStats *ifStats, uniStats, multiStats *routeStats) error { + switch runtime.GOOS { + case "dragonfly", "nacl", "netbsd", "openbsd", "plan9", "solaris": + default: + // Test the existence of connected multicast route + // clones for IPv4. Unlike IPv6, IPv4 multicast + // capability is not a mandatory feature, and so IPv4 + // multicast validation is ignored and we only check + // IPv6 below. + // + // Test the existence of connected multicast route + // clones for IPv6. Some platform never uses loopback + // interface as the nexthop for multicast routing. + // We can assume the existence of connected multicast + // route clones when at least two connected unicast + // routes, ::1/128 and other, are installed. + if supportsIPv6 && ifStats.loop > 0 && uniStats.ipv6 > 1 && multiStats.ipv6 == 0 { + return fmt.Errorf("num IPv6 multicast route clones = 0; want >0; summary: %+v, %+v, %+v", ifStats, uniStats, multiStats) + } + } + return nil } func BenchmarkInterfaces(b *testing.B) { diff --git a/libgo/go/net/interface_windows.go b/libgo/go/net/interface_windows.go index 4d6bcdf4c7..8b976e585f 100644 --- a/libgo/go/net/interface_windows.go +++ b/libgo/go/net/interface_windows.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -61,7 +61,7 @@ func adapterAddresses() ([]*windows.IpAdapterAddresses, error) { } // If the ifindex is zero, interfaceTable returns mappings of all -// network interfaces. Otherwise it returns a mapping of a specific +// network interfaces. Otherwise it returns a mapping of a specific // interface. func interfaceTable(ifindex int) ([]Interface, error) { aas, err := adapterAddresses() @@ -116,7 +116,7 @@ func interfaceTable(ifindex int) ([]Interface, error) { } // If the ifi is nil, interfaceAddrTable returns addresses for all -// network interfaces. Otherwise it returns addresses for a specific +// network interfaces. Otherwise it returns addresses for a specific // interface. func interfaceAddrTable(ifi *Interface) ([]Addr, error) { aas, err := adapterAddresses() diff --git a/libgo/go/net/internal/socktest/switch.go b/libgo/go/net/internal/socktest/switch.go index 8bef06b97c..3c37b6ff80 100644 --- a/libgo/go/net/internal/socktest/switch.go +++ b/libgo/go/net/internal/socktest/switch.go @@ -121,7 +121,7 @@ const ( FilterSocket FilterType = iota // for Socket FilterConnect // for Connect or ConnectEx FilterListen // for Listen - FilterAccept // for Accept or Accept4 + FilterAccept // for Accept, Accept4 or AcceptEx FilterGetsockoptInt // for GetsockoptInt FilterClose // for Close or Closesocket ) diff --git a/libgo/go/net/internal/socktest/sys_windows.go b/libgo/go/net/internal/socktest/sys_windows.go index e61bf2be60..2e3d2bc7fc 100644 --- a/libgo/go/net/internal/socktest/sys_windows.go +++ b/libgo/go/net/internal/socktest/sys_windows.go @@ -154,3 +154,33 @@ func (sw *Switch) Listen(s syscall.Handle, backlog int) (err error) { sw.stats.getLocked(so.Cookie).Listened++ return nil } + +// AcceptEx wraps syscall.AcceptEx. +func (sw *Switch) AcceptEx(ls syscall.Handle, as syscall.Handle, b *byte, rxdatalen uint32, laddrlen uint32, raddrlen uint32, rcvd *uint32, overlapped *syscall.Overlapped) error { + so := sw.sockso(ls) + if so == nil { + return syscall.AcceptEx(ls, as, b, rxdatalen, laddrlen, raddrlen, rcvd, overlapped) + } + sw.fmu.RLock() + f, _ := sw.fltab[FilterAccept] + sw.fmu.RUnlock() + + af, err := f.apply(so) + if err != nil { + return err + } + so.Err = syscall.AcceptEx(ls, as, b, rxdatalen, laddrlen, raddrlen, rcvd, overlapped) + if err = af.apply(so); err != nil { + return err + } + + sw.smu.Lock() + defer sw.smu.Unlock() + if so.Err != nil { + sw.stats.getLocked(so.Cookie).AcceptFailed++ + return so.Err + } + nso := sw.addLocked(as, so.Cookie.Family(), so.Cookie.Type(), so.Cookie.Protocol()) + sw.stats.getLocked(nso.Cookie).Accepted++ + return nil +} diff --git a/libgo/go/net/ip.go b/libgo/go/net/ip.go index cc004d6072..db3364c1b3 100644 --- a/libgo/go/net/ip.go +++ b/libgo/go/net/ip.go @@ -90,7 +90,7 @@ func CIDRMask(ones, bits int) IPMask { // Well-known IPv4 addresses var ( - IPv4bcast = IPv4(255, 255, 255, 255) // broadcast + IPv4bcast = IPv4(255, 255, 255, 255) // limited broadcast IPv4allsys = IPv4(224, 0, 0, 1) // all systems IPv4allrouter = IPv4(224, 0, 0, 2) // all routers IPv4zero = IPv4(0, 0, 0, 0) // all zeros @@ -153,6 +153,12 @@ func (ip IP) IsLinkLocalUnicast() bool { // IsGlobalUnicast reports whether ip is a global unicast // address. +// +// The identification of global unicast addresses uses address type +// identification as defined in RFC 1122, RFC 4632 and RFC 4291 with +// the exception of IPv4 directed broadcast addresses. +// It returns true even if ip is in IPv4 private address space or +// local IPv6 unicast address space. func (ip IP) IsGlobalUnicast() bool { return (len(ip) == IPv4len || len(ip) == IPv6len) && !ip.Equal(IPv4bcast) && @@ -252,9 +258,11 @@ func (ip IP) Mask(mask IPMask) IP { } // String returns the string form of the IP address ip. -// If the address is an IPv4 address, the string representation -// is dotted decimal ("74.125.19.99"). Otherwise the representation -// is IPv6 ("2001:4860:0:2001::68"). +// It returns one of 4 forms: +// - "<nil>", if ip has length 0 +// - dotted decimal ("192.0.2.1"), if ip is an IPv4 or IP4-mapped IPv6 address +// - IPv6 ("2001:db8::1"), if ip is a valid IPv6 address +// - the hexadecimal form of ip, without punctuation, if no other cases apply func (ip IP) String() string { p := ip @@ -270,7 +278,7 @@ func (ip IP) String() string { uitoa(uint(p4[3])) } if len(p) != IPv6len { - return "?" + return "?" + hexString(ip) } // Find longest run of zeros. @@ -312,6 +320,14 @@ func (ip IP) String() string { return string(b) } +func hexString(b []byte) string { + s := make([]byte, len(b)*2) + for i, tn := range b { + s[i*2], s[i*2+1] = hexDigit[tn>>4], hexDigit[tn&0xf] + } + return string(s) +} + // ipEmptyString is like ip.String except that it returns // an empty string when ip is unset. func ipEmptyString(ip IP) string { @@ -328,7 +344,7 @@ func (ip IP) MarshalText() ([]byte, error) { return []byte(""), nil } if len(ip) != IPv4len && len(ip) != IPv6len { - return nil, &AddrError{Err: "invalid IP address", Addr: ip.String()} + return nil, &AddrError{Err: "invalid IP address", Addr: hexString(ip)} } return []byte(ip.String()), nil } @@ -377,6 +393,10 @@ func bytesEqual(x, y []byte) bool { return true } +func (ip IP) matchAddrFamily(x IP) bool { + return ip.To4() != nil && x.To4() != nil || ip.To16() != nil && ip.To4() == nil && x.To16() != nil && x.To4() == nil +} + // If mask is a sequence of 1 bits followed by 0 bits, // return the number of 1 bits. func simpleMaskLength(mask IPMask) int { @@ -422,11 +442,7 @@ func (m IPMask) String() string { if len(m) == 0 { return "<nil>" } - buf := make([]byte, len(m)*2) - for i, b := range m { - buf[i*2], buf[i*2+1] = hexDigit[b>>4], hexDigit[b&0xf] - } - return string(buf) + return hexString(m) } func networkNumberAndMask(n *IPNet) (ip IP, m IPMask) { @@ -473,12 +489,12 @@ func (n *IPNet) Contains(ip IP) bool { // Network returns the address's network name, "ip+net". func (n *IPNet) Network() string { return "ip+net" } -// String returns the CIDR notation of n like "192.168.100.1/24" -// or "2001:DB8::/48" as defined in RFC 4632 and RFC 4291. +// String returns the CIDR notation of n like "192.0.2.1/24" +// or "2001:db8::/48" as defined in RFC 4632 and RFC 4291. // If the mask is not in the canonical form, it returns the // string which consists of an IP address, followed by a slash // character and a mask expressed as hexadecimal form with no -// punctuation like "192.168.100.1/c000ff00". +// punctuation like "198.51.100.1/c000ff00". func (n *IPNet) String() string { nn, m := networkNumberAndMask(n) if nn == nil || m == nil { @@ -494,29 +510,25 @@ func (n *IPNet) String() string { // Parse IPv4 address (d.d.d.d). func parseIPv4(s string) IP { var p [IPv4len]byte - i := 0 - for j := 0; j < IPv4len; j++ { - if i >= len(s) { + for i := 0; i < IPv4len; i++ { + if len(s) == 0 { // Missing octets. return nil } - if j > 0 { - if s[i] != '.' { + if i > 0 { + if s[0] != '.' { return nil } - i++ + s = s[1:] } - var ( - n int - ok bool - ) - n, i, ok = dtoi(s, i) + n, c, ok := dtoi(s) if !ok || n > 0xFF { return nil } - p[j] = byte(n) + s = s[c:] + p[i] = byte(n) } - if i != len(s) { + if len(s) != 0 { return nil } return IPv4(p[0], p[1], p[2], p[3]) @@ -528,8 +540,7 @@ func parseIPv4(s string) IP { // true. func parseIPv6(s string, zoneAllowed bool) (ip IP, zone string) { ip = make(IP, IPv6len) - ellipsis := -1 // position of ellipsis in p - i := 0 // index in string s + ellipsis := -1 // position of ellipsis in ip if zoneAllowed { s, zone = splitHostZone(s) @@ -538,90 +549,91 @@ func parseIPv6(s string, zoneAllowed bool) (ip IP, zone string) { // Might have leading ellipsis if len(s) >= 2 && s[0] == ':' && s[1] == ':' { ellipsis = 0 - i = 2 + s = s[2:] // Might be only ellipsis - if i == len(s) { + if len(s) == 0 { return ip, zone } } // Loop, parsing hex numbers followed by colon. - j := 0 - for j < IPv6len { + i := 0 + for i < IPv6len { // Hex number. - n, i1, ok := xtoi(s, i) + n, c, ok := xtoi(s) if !ok || n > 0xFFFF { return nil, zone } // If followed by dot, might be in trailing IPv4. - if i1 < len(s) && s[i1] == '.' { - if ellipsis < 0 && j != IPv6len-IPv4len { + if c < len(s) && s[c] == '.' { + if ellipsis < 0 && i != IPv6len-IPv4len { // Not the right place. return nil, zone } - if j+IPv4len > IPv6len { + if i+IPv4len > IPv6len { // Not enough room. return nil, zone } - ip4 := parseIPv4(s[i:]) + ip4 := parseIPv4(s) if ip4 == nil { return nil, zone } - ip[j] = ip4[12] - ip[j+1] = ip4[13] - ip[j+2] = ip4[14] - ip[j+3] = ip4[15] - i = len(s) - j += IPv4len + ip[i] = ip4[12] + ip[i+1] = ip4[13] + ip[i+2] = ip4[14] + ip[i+3] = ip4[15] + s = "" + i += IPv4len break } // Save this 16-bit chunk. - ip[j] = byte(n >> 8) - ip[j+1] = byte(n) - j += 2 + ip[i] = byte(n >> 8) + ip[i+1] = byte(n) + i += 2 // Stop at end of string. - i = i1 - if i == len(s) { + s = s[c:] + if len(s) == 0 { break } // Otherwise must be followed by colon and more. - if s[i] != ':' || i+1 == len(s) { + if s[0] != ':' || len(s) == 1 { return nil, zone } - i++ + s = s[1:] // Look for ellipsis. - if s[i] == ':' { + if s[0] == ':' { if ellipsis >= 0 { // already have one return nil, zone } - ellipsis = j - if i++; i == len(s) { // can be at end + ellipsis = i + s = s[1:] + if len(s) == 0 { // can be at end break } } } // Must have used entire string. - if i != len(s) { + if len(s) != 0 { return nil, zone } // If didn't parse enough, expand ellipsis. - if j < IPv6len { + if i < IPv6len { if ellipsis < 0 { return nil, zone } - n := IPv6len - j - for k := j - 1; k >= ellipsis; k-- { - ip[k+n] = ip[k] + n := IPv6len - i + for j := i - 1; j >= ellipsis; j-- { + ip[j+n] = ip[j] } - for k := ellipsis + n - 1; k >= ellipsis; k-- { - ip[k] = 0 + for j := ellipsis + n - 1; j >= ellipsis; j-- { + ip[j] = 0 } } else if ellipsis >= 0 { // Ellipsis must represent at least one 0 group. @@ -631,8 +643,8 @@ func parseIPv6(s string, zoneAllowed bool) (ip IP, zone string) { } // ParseIP parses s as an IP address, returning the result. -// The string s can be in dotted decimal ("74.125.19.99") -// or IPv6 ("2001:4860:0:2001::68") form. +// The string s can be in dotted decimal ("192.0.2.1") +// or IPv6 ("2001:db8::68") form. // If s is not a valid textual representation of an IP address, // ParseIP returns nil. func ParseIP(s string) IP { @@ -648,13 +660,14 @@ func ParseIP(s string) IP { return nil } -// ParseCIDR parses s as a CIDR notation IP address and mask, -// like "192.168.100.1/24" or "2001:DB8::/48", as defined in +// ParseCIDR parses s as a CIDR notation IP address and prefix length, +// like "192.0.2.0/24" or "2001:db8::/32", as defined in // RFC 4632 and RFC 4291. // -// It returns the IP address and the network implied by the IP -// and mask. For example, ParseCIDR("192.168.100.1/16") returns -// the IP address 192.168.100.1 and the network 192.168.0.0/16. +// It returns the IP address and the network implied by the IP and +// prefix length. +// For example, ParseCIDR("192.0.2.1/24") returns the IP address +// 198.0.2.1 and the network 198.0.2.0/24. func ParseCIDR(s string) (IP, *IPNet, error) { i := byteIndex(s, '/') if i < 0 { @@ -667,7 +680,7 @@ func ParseCIDR(s string) (IP, *IPNet, error) { iplen = IPv6len ip, _ = parseIPv6(addr, false) } - n, i, ok := dtoi(mask, 0) + n, i, ok := dtoi(mask) if ip == nil || !ok || i != len(mask) || n < 0 || n > 8*iplen { return nil, nil, &ParseError{Type: "CIDR address", Text: s} } diff --git a/libgo/go/net/ip_test.go b/libgo/go/net/ip_test.go index 3d95a73c09..46551633ce 100644 --- a/libgo/go/net/ip_test.go +++ b/libgo/go/net/ip_test.go @@ -5,6 +5,7 @@ package net import ( + "bytes" "reflect" "runtime" "testing" @@ -27,6 +28,10 @@ var parseIPTests = []struct { {"2001:4860:0:2001::68", IP{0x20, 0x01, 0x48, 0x60, 0, 0, 0x20, 0x01, 0, 0, 0, 0, 0, 0, 0x00, 0x68}}, {"2001:4860:0000:2001:0000:0000:0000:0068", IP{0x20, 0x01, 0x48, 0x60, 0, 0, 0x20, 0x01, 0, 0, 0, 0, 0, 0, 0x00, 0x68}}, + {"-0.0.0.0", nil}, + {"0.-1.0.0", nil}, + {"0.0.-2.0", nil}, + {"0.0.0.-3", nil}, {"127.0.0.256", nil}, {"abc", nil}, {"123:", nil}, @@ -124,41 +129,132 @@ func TestMarshalEmptyIP(t *testing.T) { } var ipStringTests = []struct { - in IP - out string // see RFC 5952 + in IP // see RFC 791 and RFC 4291 + str string // see RFC 791, RFC 4291 and RFC 5952 + byt []byte + error }{ - {IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0, 0, 0, 0x1, 0x23, 0, 0x12, 0, 0x1}, "2001:db8::123:12:1"}, - {IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x1}, "2001:db8::1"}, - {IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0x1, 0, 0, 0, 0x1, 0, 0, 0, 0x1}, "2001:db8:0:1:0:1:0:1"}, - {IP{0x20, 0x1, 0xd, 0xb8, 0, 0x1, 0, 0, 0, 0x1, 0, 0, 0, 0x1, 0, 0}, "2001:db8:1:0:1:0:1:0"}, - {IP{0x20, 0x1, 0, 0, 0, 0, 0, 0, 0, 0x1, 0, 0, 0, 0, 0, 0x1}, "2001::1:0:0:1"}, - {IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0, 0, 0x1, 0, 0, 0, 0, 0, 0}, "2001:db8:0:0:1::"}, - {IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0, 0, 0x1, 0, 0, 0, 0, 0, 0x1}, "2001:db8::1:0:0:1"}, - {IP{0x20, 0x1, 0xD, 0xB8, 0, 0, 0, 0, 0, 0xA, 0, 0xB, 0, 0xC, 0, 0xD}, "2001:db8::a:b:c:d"}, - {IPv4(192, 168, 0, 1), "192.168.0.1"}, - {nil, ""}, + // IPv4 address + { + IP{192, 0, 2, 1}, + "192.0.2.1", + []byte("192.0.2.1"), + nil, + }, + { + IP{0, 0, 0, 0}, + "0.0.0.0", + []byte("0.0.0.0"), + nil, + }, + + // IPv4-mapped IPv6 address + { + IP{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, 192, 0, 2, 1}, + "192.0.2.1", + []byte("192.0.2.1"), + nil, + }, + { + IP{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff, 0, 0, 0, 0}, + "0.0.0.0", + []byte("0.0.0.0"), + nil, + }, + + // IPv6 address + { + IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0, 0, 0, 0x1, 0x23, 0, 0x12, 0, 0x1}, + "2001:db8::123:12:1", + []byte("2001:db8::123:12:1"), + nil, + }, + { + IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x1}, + "2001:db8::1", + []byte("2001:db8::1"), + nil, + }, + { + IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0x1, 0, 0, 0, 0x1, 0, 0, 0, 0x1}, + "2001:db8:0:1:0:1:0:1", + []byte("2001:db8:0:1:0:1:0:1"), + nil, + }, + { + IP{0x20, 0x1, 0xd, 0xb8, 0, 0x1, 0, 0, 0, 0x1, 0, 0, 0, 0x1, 0, 0}, + "2001:db8:1:0:1:0:1:0", + []byte("2001:db8:1:0:1:0:1:0"), + nil, + }, + { + IP{0x20, 0x1, 0, 0, 0, 0, 0, 0, 0, 0x1, 0, 0, 0, 0, 0, 0x1}, + "2001::1:0:0:1", + []byte("2001::1:0:0:1"), + nil, + }, + { + IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0, 0, 0x1, 0, 0, 0, 0, 0, 0}, + "2001:db8:0:0:1::", + []byte("2001:db8:0:0:1::"), + nil, + }, + { + IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0, 0, 0x1, 0, 0, 0, 0, 0, 0x1}, + "2001:db8::1:0:0:1", + []byte("2001:db8::1:0:0:1"), + nil, + }, + { + IP{0x20, 0x1, 0xd, 0xb8, 0, 0, 0, 0, 0, 0xa, 0, 0xb, 0, 0xc, 0, 0xd}, + "2001:db8::a:b:c:d", + []byte("2001:db8::a:b:c:d"), + nil, + }, + { + IPv6unspecified, + "::", + []byte("::"), + nil, + }, + + // IP wildcard equivalent address in Dial/Listen API + { + nil, + "<nil>", + nil, + nil, + }, + + // Opaque byte sequence + { + IP{0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef}, + "?0123456789abcdef", + nil, + &AddrError{Err: "invalid IP address", Addr: "0123456789abcdef"}, + }, } func TestIPString(t *testing.T) { for _, tt := range ipStringTests { - if tt.in != nil { - if out := tt.in.String(); out != tt.out { - t.Errorf("IP.String(%v) = %q, want %q", tt.in, out, tt.out) - } + if out := tt.in.String(); out != tt.str { + t.Errorf("IP.String(%v) = %q, want %q", tt.in, out, tt.str) } - if out, err := tt.in.MarshalText(); string(out) != tt.out || err != nil { - t.Errorf("IP.MarshalText(%v) = %q, %v, want %q, nil", tt.in, out, err, tt.out) + if out, err := tt.in.MarshalText(); !bytes.Equal(out, tt.byt) || !reflect.DeepEqual(err, tt.error) { + t.Errorf("IP.MarshalText(%v) = %v, %v, want %v, %v", tt.in, out, err, tt.byt, tt.error) } } } +var sink string + func BenchmarkIPString(b *testing.B) { testHookUninstaller.Do(uninstallTestHooks) for i := 0; i < b.N; i++ { for _, tt := range ipStringTests { if tt.in != nil { - tt.in.String() + sink = tt.in.String() } } } @@ -209,7 +305,7 @@ func BenchmarkIPMaskString(b *testing.B) { for i := 0; i < b.N; i++ { for _, tt := range ipMaskStringTests { - tt.in.String() + sink = tt.in.String() } } } @@ -240,6 +336,12 @@ var parseCIDRTests = []struct { {"192.168.1.1/255.255.255.0", nil, nil, &ParseError{Type: "CIDR address", Text: "192.168.1.1/255.255.255.0"}}, {"192.168.1.1/35", nil, nil, &ParseError{Type: "CIDR address", Text: "192.168.1.1/35"}}, {"2001:db8::1/-1", nil, nil, &ParseError{Type: "CIDR address", Text: "2001:db8::1/-1"}}, + {"2001:db8::1/-0", nil, nil, &ParseError{Type: "CIDR address", Text: "2001:db8::1/-0"}}, + {"-0.0.0.0/32", nil, nil, &ParseError{Type: "CIDR address", Text: "-0.0.0.0/32"}}, + {"0.-1.0.0/32", nil, nil, &ParseError{Type: "CIDR address", Text: "0.-1.0.0/32"}}, + {"0.0.-2.0/32", nil, nil, &ParseError{Type: "CIDR address", Text: "0.0.-2.0/32"}}, + {"0.0.0.-3/32", nil, nil, &ParseError{Type: "CIDR address", Text: "0.0.0.-3/32"}}, + {"0.0.0.0/-0", nil, nil, &ParseError{Type: "CIDR address", Text: "0.0.0.0/-0"}}, {"", nil, nil, &ParseError{Type: "CIDR address", Text: ""}}, } @@ -379,8 +481,8 @@ var splitJoinTests = []struct { {"", "0", ":0"}, {"google.com", "https%foo", "google.com:https%foo"}, // Go 1.0 behavior - {"127.0.0.1", "", "127.0.0.1:"}, // Go 1.0 behaviour - {"www.google.com", "", "www.google.com:"}, // Go 1.0 behaviour + {"127.0.0.1", "", "127.0.0.1:"}, // Go 1.0 behavior + {"www.google.com", "", "www.google.com:"}, // Go 1.0 behavior } var splitFailureTests = []struct { @@ -421,13 +523,16 @@ func TestSplitHostPort(t *testing.T) { } } for _, tt := range splitFailureTests { - if _, _, err := SplitHostPort(tt.hostPort); err == nil { + if host, port, err := SplitHostPort(tt.hostPort); err == nil { t.Errorf("SplitHostPort(%q) should have failed", tt.hostPort) } else { e := err.(*AddrError) if e.Err != tt.err { t.Errorf("SplitHostPort(%q) = _, _, %q; want %q", tt.hostPort, e.Err, tt.err) } + if host != "" || port != "" { + t.Errorf("SplitHostPort(%q) = %q, %q, err; want %q, %q, err on failure", tt.hostPort, host, port, "", "") + } } } } diff --git a/libgo/go/net/iprawsock.go b/libgo/go/net/iprawsock.go index f02df7fa8d..d994fc67c6 100644 --- a/libgo/go/net/iprawsock.go +++ b/libgo/go/net/iprawsock.go @@ -1,9 +1,32 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package net +import ( + "context" + "syscall" +) + +// BUG(mikio): On every POSIX platform, reads from the "ip4" network +// using the ReadFrom or ReadFromIP method might not return a complete +// IPv4 packet, including its header, even if there is space +// available. This can occur even in cases where Read or ReadMsgIP +// could return a complete packet. For this reason, it is recommended +// that you do not use these methods if it is important to receive a +// full packet. +// +// The Go 1 compatibility guidelines make it impossible for us to +// change the behavior of these methods; use Read or ReadMsgIP +// instead. + +// BUG(mikio): On NaCl, Plan 9 and Windows, the ReadMsgIP and +// WriteMsgIP methods of IPConn are not implemented. + +// BUG(mikio): On Windows, the File method of IPConn is not +// implemented. + // IPAddr represents the address of an IP end point. type IPAddr struct { IP IP @@ -41,11 +64,14 @@ func (a *IPAddr) opAddr() Addr { // ResolveIPAddr parses addr as an IP address of the form "host" or // "ipv6-host%zone" and resolves the domain name on the network net, // which must be "ip", "ip4" or "ip6". +// +// Resolving a hostname is not recommended because this returns at most +// one of its IP addresses. func ResolveIPAddr(net, addr string) (*IPAddr, error) { if net == "" { // a hint wildcard for Go 1.0 undocumented behavior net = "ip" } - afnet, _, err := parseNetwork(net) + afnet, _, err := parseNetwork(context.Background(), net) if err != nil { return nil, err } @@ -54,9 +80,136 @@ func ResolveIPAddr(net, addr string) (*IPAddr, error) { default: return nil, UnknownNetworkError(net) } - addrs, err := internetAddrList(afnet, addr, noDeadline) + addrs, err := DefaultResolver.internetAddrList(context.Background(), afnet, addr) if err != nil { return nil, err } return addrs.first(isIPv4).(*IPAddr), nil } + +// IPConn is the implementation of the Conn and PacketConn interfaces +// for IP network connections. +type IPConn struct { + conn +} + +// ReadFromIP reads an IP packet from c, copying the payload into b. +// It returns the number of bytes copied into b and the return address +// that was on the packet. +// +// ReadFromIP can be made to time out and return an error with +// Timeout() == true after a fixed time limit; see SetDeadline and +// SetReadDeadline. +func (c *IPConn) ReadFromIP(b []byte) (int, *IPAddr, error) { + if !c.ok() { + return 0, nil, syscall.EINVAL + } + n, addr, err := c.readFrom(b) + if err != nil { + err = &OpError{Op: "read", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} + } + return n, addr, err +} + +// ReadFrom implements the PacketConn ReadFrom method. +func (c *IPConn) ReadFrom(b []byte) (int, Addr, error) { + if !c.ok() { + return 0, nil, syscall.EINVAL + } + n, addr, err := c.readFrom(b) + if err != nil { + err = &OpError{Op: "read", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} + } + if addr == nil { + return n, nil, err + } + return n, addr, err +} + +// ReadMsgIP reads a packet from c, copying the payload into b and the +// associated out-of-band data into oob. It returns the number of +// bytes copied into b, the number of bytes copied into oob, the flags +// that were set on the packet and the source address of the packet. +func (c *IPConn) ReadMsgIP(b, oob []byte) (n, oobn, flags int, addr *IPAddr, err error) { + if !c.ok() { + return 0, 0, 0, nil, syscall.EINVAL + } + n, oobn, flags, addr, err = c.readMsg(b, oob) + if err != nil { + err = &OpError{Op: "read", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} + } + return +} + +// WriteToIP writes an IP packet to addr via c, copying the payload +// from b. +// +// WriteToIP can be made to time out and return an error with +// Timeout() == true after a fixed time limit; see SetDeadline and +// SetWriteDeadline. On packet-oriented connections, write timeouts +// are rare. +func (c *IPConn) WriteToIP(b []byte, addr *IPAddr) (int, error) { + if !c.ok() { + return 0, syscall.EINVAL + } + n, err := c.writeTo(b, addr) + if err != nil { + err = &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: err} + } + return n, err +} + +// WriteTo implements the PacketConn WriteTo method. +func (c *IPConn) WriteTo(b []byte, addr Addr) (int, error) { + if !c.ok() { + return 0, syscall.EINVAL + } + a, ok := addr.(*IPAddr) + if !ok { + return 0, &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr, Err: syscall.EINVAL} + } + n, err := c.writeTo(b, a) + if err != nil { + err = &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: a.opAddr(), Err: err} + } + return n, err +} + +// WriteMsgIP writes a packet to addr via c, copying the payload from +// b and the associated out-of-band data from oob. It returns the +// number of payload and out-of-band bytes written. +func (c *IPConn) WriteMsgIP(b, oob []byte, addr *IPAddr) (n, oobn int, err error) { + if !c.ok() { + return 0, 0, syscall.EINVAL + } + n, oobn, err = c.writeMsg(b, oob, addr) + if err != nil { + err = &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: err} + } + return +} + +func newIPConn(fd *netFD) *IPConn { return &IPConn{conn{fd}} } + +// DialIP connects to the remote address raddr on the network protocol +// netProto, which must be "ip", "ip4", or "ip6" followed by a colon +// and a protocol number or name. +func DialIP(netProto string, laddr, raddr *IPAddr) (*IPConn, error) { + c, err := dialIP(context.Background(), netProto, laddr, raddr) + if err != nil { + return nil, &OpError{Op: "dial", Net: netProto, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: err} + } + return c, nil +} + +// ListenIP listens for incoming IP packets addressed to the local +// address laddr. The returned connection's ReadFrom and WriteTo +// methods can be used to receive and send IP packets with per-packet +// addressing. +func ListenIP(netProto string, laddr *IPAddr) (*IPConn, error) { + c, err := listenIP(context.Background(), netProto, laddr) + if err != nil { + return nil, &OpError{Op: "listen", Net: netProto, Source: nil, Addr: laddr.opAddr(), Err: err} + } + return c, nil +} diff --git a/libgo/go/net/iprawsock_plan9.go b/libgo/go/net/iprawsock_plan9.go index b027adc53a..6aebea169c 100644 --- a/libgo/go/net/iprawsock_plan9.go +++ b/libgo/go/net/iprawsock_plan9.go @@ -1,82 +1,34 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package net import ( + "context" "syscall" - "time" ) -// IPConn is the implementation of the Conn and PacketConn interfaces -// for IP network connections. -type IPConn struct { - conn +func (c *IPConn) readFrom(b []byte) (int, *IPAddr, error) { + return 0, nil, syscall.EPLAN9 } -// ReadFromIP reads an IP packet from c, copying the payload into b. -// It returns the number of bytes copied into b and the return address -// that was on the packet. -// -// ReadFromIP can be made to time out and return an error with -// Timeout() == true after a fixed time limit; see SetDeadline and -// SetReadDeadline. -func (c *IPConn) ReadFromIP(b []byte) (int, *IPAddr, error) { - return 0, nil, &OpError{Op: "read", Net: c.fd.dir, Source: c.fd.laddr, Addr: c.fd.raddr, Err: syscall.EPLAN9} +func (c *IPConn) readMsg(b, oob []byte) (n, oobn, flags int, addr *IPAddr, err error) { + return 0, 0, 0, nil, syscall.EPLAN9 } -// ReadFrom implements the PacketConn ReadFrom method. -func (c *IPConn) ReadFrom(b []byte) (int, Addr, error) { - return 0, nil, &OpError{Op: "read", Net: c.fd.dir, Source: c.fd.laddr, Addr: c.fd.raddr, Err: syscall.EPLAN9} +func (c *IPConn) writeTo(b []byte, addr *IPAddr) (int, error) { + return 0, syscall.EPLAN9 } -// ReadMsgIP reads a packet from c, copying the payload into b and the -// associated out-of-band data into oob. It returns the number of -// bytes copied into b, the number of bytes copied into oob, the flags -// that were set on the packet and the source address of the packet. -func (c *IPConn) ReadMsgIP(b, oob []byte) (n, oobn, flags int, addr *IPAddr, err error) { - return 0, 0, 0, nil, &OpError{Op: "read", Net: c.fd.dir, Source: c.fd.laddr, Addr: c.fd.raddr, Err: syscall.EPLAN9} +func (c *IPConn) writeMsg(b, oob []byte, addr *IPAddr) (n, oobn int, err error) { + return 0, 0, syscall.EPLAN9 } -// WriteToIP writes an IP packet to addr via c, copying the payload -// from b. -// -// WriteToIP can be made to time out and return an error with -// Timeout() == true after a fixed time limit; see SetDeadline and -// SetWriteDeadline. On packet-oriented connections, write timeouts -// are rare. -func (c *IPConn) WriteToIP(b []byte, addr *IPAddr) (int, error) { - return 0, &OpError{Op: "write", Net: c.fd.dir, Source: c.fd.laddr, Addr: addr.opAddr(), Err: syscall.EPLAN9} +func dialIP(ctx context.Context, netProto string, laddr, raddr *IPAddr) (*IPConn, error) { + return nil, syscall.EPLAN9 } -// WriteTo implements the PacketConn WriteTo method. -func (c *IPConn) WriteTo(b []byte, addr Addr) (int, error) { - return 0, &OpError{Op: "write", Net: c.fd.dir, Source: c.fd.laddr, Addr: addr, Err: syscall.EPLAN9} -} - -// WriteMsgIP writes a packet to addr via c, copying the payload from -// b and the associated out-of-band data from oob. It returns the -// number of payload and out-of-band bytes written. -func (c *IPConn) WriteMsgIP(b, oob []byte, addr *IPAddr) (n, oobn int, err error) { - return 0, 0, &OpError{Op: "write", Net: c.fd.dir, Source: c.fd.laddr, Addr: addr.opAddr(), Err: syscall.EPLAN9} -} - -// DialIP connects to the remote address raddr on the network protocol -// netProto, which must be "ip", "ip4", or "ip6" followed by a colon -// and a protocol number or name. -func DialIP(netProto string, laddr, raddr *IPAddr) (*IPConn, error) { - return dialIP(netProto, laddr, raddr, noDeadline) -} - -func dialIP(netProto string, laddr, raddr *IPAddr, deadline time.Time) (*IPConn, error) { - return nil, &OpError{Op: "dial", Net: netProto, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: syscall.EPLAN9} -} - -// ListenIP listens for incoming IP packets addressed to the local -// address laddr. The returned connection's ReadFrom and WriteTo -// methods can be used to receive and send IP packets with per-packet -// addressing. -func ListenIP(netProto string, laddr *IPAddr) (*IPConn, error) { - return nil, &OpError{Op: "listen", Net: netProto, Source: nil, Addr: laddr.opAddr(), Err: syscall.EPLAN9} +func listenIP(ctx context.Context, netProto string, laddr *IPAddr) (*IPConn, error) { + return nil, syscall.EPLAN9 } diff --git a/libgo/go/net/iprawsock_posix.go b/libgo/go/net/iprawsock_posix.go index 93fee3e232..8f4b702e48 100644 --- a/libgo/go/net/iprawsock_posix.go +++ b/libgo/go/net/iprawsock_posix.go @@ -1,4 +1,4 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -7,22 +7,10 @@ package net import ( + "context" "syscall" - "time" ) -// BUG(mikio): On every POSIX platform, reads from the "ip4" network -// using the ReadFrom or ReadFromIP method might not return a complete -// IPv4 packet, including its header, even if there is space -// available. This can occur even in cases where Read or ReadMsgIP -// could return a complete packet. For this reason, it is recommended -// that you do not uses these methods if it is important to receive a -// full packet. -// -// The Go 1 compatibility guidelines make it impossible for us to -// change the behavior of these methods; use Read or ReadMsgIP -// instead. - func sockaddrToIP(sa syscall.Sockaddr) Addr { switch sa := sa.(type) { case *syscall.SockaddrInet4: @@ -50,25 +38,11 @@ func (a *IPAddr) sockaddr(family int) (syscall.Sockaddr, error) { return ipToSockaddr(family, a.IP, 0, a.Zone) } -// IPConn is the implementation of the Conn and PacketConn interfaces -// for IP network connections. -type IPConn struct { - conn +func (a *IPAddr) toLocal(net string) sockaddr { + return &IPAddr{loopbackIP(net), a.Zone} } -func newIPConn(fd *netFD) *IPConn { return &IPConn{conn{fd}} } - -// ReadFromIP reads an IP packet from c, copying the payload into b. -// It returns the number of bytes copied into b and the return address -// that was on the packet. -// -// ReadFromIP can be made to time out and return an error with -// Timeout() == true after a fixed time limit; see SetDeadline and -// SetReadDeadline. -func (c *IPConn) ReadFromIP(b []byte) (int, *IPAddr, error) { - if !c.ok() { - return 0, nil, syscall.EINVAL - } +func (c *IPConn) readFrom(b []byte) (int, *IPAddr, error) { // TODO(cw,rsc): consider using readv if we know the family // type to avoid the header trim/copy var addr *IPAddr @@ -80,9 +54,6 @@ func (c *IPConn) ReadFromIP(b []byte) (int, *IPAddr, error) { case *syscall.SockaddrInet6: addr = &IPAddr{IP: sa.Addr[0:], Zone: zoneToString(int(sa.ZoneId))} } - if err != nil { - err = &OpError{Op: "read", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} - } return n, addr, err } @@ -101,26 +72,7 @@ func stripIPv4Header(n int, b []byte) int { return n - l } -// ReadFrom implements the PacketConn ReadFrom method. -func (c *IPConn) ReadFrom(b []byte) (int, Addr, error) { - if !c.ok() { - return 0, nil, syscall.EINVAL - } - n, addr, err := c.ReadFromIP(b) - if addr == nil { - return n, nil, err - } - return n, addr, err -} - -// ReadMsgIP reads a packet from c, copying the payload into b and the -// associated out-of-band data into oob. It returns the number of -// bytes copied into b, the number of bytes copied into oob, the flags -// that were set on the packet and the source address of the packet. -func (c *IPConn) ReadMsgIP(b, oob []byte) (n, oobn, flags int, addr *IPAddr, err error) { - if !c.ok() { - return 0, 0, 0, nil, syscall.EINVAL - } +func (c *IPConn) readMsg(b, oob []byte) (n, oobn, flags int, addr *IPAddr, err error) { var sa syscall.Sockaddr n, oobn, flags, sa, err = c.fd.readMsg(b, oob) switch sa := sa.(type) { @@ -129,121 +81,70 @@ func (c *IPConn) ReadMsgIP(b, oob []byte) (n, oobn, flags int, addr *IPAddr, err case *syscall.SockaddrInet6: addr = &IPAddr{IP: sa.Addr[0:], Zone: zoneToString(int(sa.ZoneId))} } - if err != nil { - err = &OpError{Op: "read", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} - } return } -// WriteToIP writes an IP packet to addr via c, copying the payload -// from b. -// -// WriteToIP can be made to time out and return an error with -// Timeout() == true after a fixed time limit; see SetDeadline and -// SetWriteDeadline. On packet-oriented connections, write timeouts -// are rare. -func (c *IPConn) WriteToIP(b []byte, addr *IPAddr) (int, error) { - if !c.ok() { - return 0, syscall.EINVAL - } +func (c *IPConn) writeTo(b []byte, addr *IPAddr) (int, error) { if c.fd.isConnected { - return 0, &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: ErrWriteToConnected} + return 0, ErrWriteToConnected } if addr == nil { - return 0, &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: nil, Err: errMissingAddress} + return 0, errMissingAddress } sa, err := addr.sockaddr(c.fd.family) if err != nil { - return 0, &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: err} + return 0, err } - n, err := c.fd.writeTo(b, sa) - if err != nil { - err = &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: err} - } - return n, err + return c.fd.writeTo(b, sa) } -// WriteTo implements the PacketConn WriteTo method. -func (c *IPConn) WriteTo(b []byte, addr Addr) (int, error) { - if !c.ok() { - return 0, syscall.EINVAL - } - a, ok := addr.(*IPAddr) - if !ok { - return 0, &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr, Err: syscall.EINVAL} - } - return c.WriteToIP(b, a) -} - -// WriteMsgIP writes a packet to addr via c, copying the payload from -// b and the associated out-of-band data from oob. It returns the -// number of payload and out-of-band bytes written. -func (c *IPConn) WriteMsgIP(b, oob []byte, addr *IPAddr) (n, oobn int, err error) { - if !c.ok() { - return 0, 0, syscall.EINVAL - } +func (c *IPConn) writeMsg(b, oob []byte, addr *IPAddr) (n, oobn int, err error) { if c.fd.isConnected { - return 0, 0, &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: ErrWriteToConnected} + return 0, 0, ErrWriteToConnected } if addr == nil { - return 0, 0, &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: nil, Err: errMissingAddress} + return 0, 0, errMissingAddress } - var sa syscall.Sockaddr - sa, err = addr.sockaddr(c.fd.family) - if err != nil { - return 0, 0, &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: err} - } - n, oobn, err = c.fd.writeMsg(b, oob, sa) + sa, err := addr.sockaddr(c.fd.family) if err != nil { - err = &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: err} + return 0, 0, err } - return -} - -// DialIP connects to the remote address raddr on the network protocol -// netProto, which must be "ip", "ip4", or "ip6" followed by a colon -// and a protocol number or name. -func DialIP(netProto string, laddr, raddr *IPAddr) (*IPConn, error) { - return dialIP(netProto, laddr, raddr, noDeadline) + return c.fd.writeMsg(b, oob, sa) } -func dialIP(netProto string, laddr, raddr *IPAddr, deadline time.Time) (*IPConn, error) { - net, proto, err := parseNetwork(netProto) +func dialIP(ctx context.Context, netProto string, laddr, raddr *IPAddr) (*IPConn, error) { + network, proto, err := parseNetwork(ctx, netProto) if err != nil { - return nil, &OpError{Op: "dial", Net: netProto, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: err} + return nil, err } - switch net { + switch network { case "ip", "ip4", "ip6": default: - return nil, &OpError{Op: "dial", Net: netProto, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: UnknownNetworkError(netProto)} + return nil, UnknownNetworkError(netProto) } if raddr == nil { - return nil, &OpError{Op: "dial", Net: netProto, Source: laddr.opAddr(), Addr: nil, Err: errMissingAddress} + return nil, errMissingAddress } - fd, err := internetSocket(net, laddr, raddr, deadline, syscall.SOCK_RAW, proto, "dial", noCancel) + fd, err := internetSocket(ctx, network, laddr, raddr, syscall.SOCK_RAW, proto, "dial") if err != nil { - return nil, &OpError{Op: "dial", Net: netProto, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: err} + return nil, err } return newIPConn(fd), nil } -// ListenIP listens for incoming IP packets addressed to the local -// address laddr. The returned connection's ReadFrom and WriteTo -// methods can be used to receive and send IP packets with per-packet -// addressing. -func ListenIP(netProto string, laddr *IPAddr) (*IPConn, error) { - net, proto, err := parseNetwork(netProto) +func listenIP(ctx context.Context, netProto string, laddr *IPAddr) (*IPConn, error) { + network, proto, err := parseNetwork(ctx, netProto) if err != nil { - return nil, &OpError{Op: "listen", Net: netProto, Source: nil, Addr: laddr.opAddr(), Err: err} + return nil, err } - switch net { + switch network { case "ip", "ip4", "ip6": default: - return nil, &OpError{Op: "listen", Net: netProto, Source: nil, Addr: laddr.opAddr(), Err: UnknownNetworkError(netProto)} + return nil, UnknownNetworkError(netProto) } - fd, err := internetSocket(net, laddr, nil, noDeadline, syscall.SOCK_RAW, proto, "listen", noCancel) + fd, err := internetSocket(ctx, network, laddr, nil, syscall.SOCK_RAW, proto, "listen") if err != nil { - return nil, &OpError{Op: "listen", Net: netProto, Source: nil, Addr: laddr.opAddr(), Err: err} + return nil, err } return newIPConn(fd), nil } diff --git a/libgo/go/net/ipraw_test.go b/libgo/go/net/iprawsock_test.go index 5d86a9d031..5d33b26a91 100644 --- a/libgo/go/net/ipraw_test.go +++ b/libgo/go/net/iprawsock_test.go @@ -1,4 +1,4 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -43,6 +43,13 @@ var resolveIPAddrTests = []resolveIPAddrTest{ {"l2tp", "127.0.0.1", nil, UnknownNetworkError("l2tp")}, {"l2tp:gre", "127.0.0.1", nil, UnknownNetworkError("l2tp:gre")}, {"tcp", "1.2.3.4:123", nil, UnknownNetworkError("tcp")}, + + {"ip4", "2001:db8::1", nil, &AddrError{Err: errNoSuitableAddress.Error(), Addr: "2001:db8::1"}}, + {"ip4:icmp", "2001:db8::1", nil, &AddrError{Err: errNoSuitableAddress.Error(), Addr: "2001:db8::1"}}, + {"ip6", "127.0.0.1", nil, &AddrError{Err: errNoSuitableAddress.Error(), Addr: "127.0.0.1"}}, + {"ip6", "::ffff:127.0.0.1", nil, &AddrError{Err: errNoSuitableAddress.Error(), Addr: "::ffff:127.0.0.1"}}, + {"ip6:ipv6-icmp", "127.0.0.1", nil, &AddrError{Err: errNoSuitableAddress.Error(), Addr: "127.0.0.1"}}, + {"ip6:ipv6-icmp", "::ffff:127.0.0.1", nil, &AddrError{Err: errNoSuitableAddress.Error(), Addr: "::ffff:127.0.0.1"}}, } func TestResolveIPAddr(t *testing.T) { @@ -54,21 +61,17 @@ func TestResolveIPAddr(t *testing.T) { defer func() { testHookLookupIP = origTestHookLookupIP }() testHookLookupIP = lookupLocalhost - for i, tt := range resolveIPAddrTests { + for _, tt := range resolveIPAddrTests { addr, err := ResolveIPAddr(tt.network, tt.litAddrOrName) - if err != tt.err { - t.Errorf("#%d: %v", i, err) - } else if !reflect.DeepEqual(addr, tt.addr) { - t.Errorf("#%d: got %#v; want %#v", i, addr, tt.addr) - } - if err != nil { + if !reflect.DeepEqual(addr, tt.addr) || !reflect.DeepEqual(err, tt.err) { + t.Errorf("ResolveIPAddr(%q, %q) = %#v, %v, want %#v, %v", tt.network, tt.litAddrOrName, addr, err, tt.addr, tt.err) continue } - rtaddr, err := ResolveIPAddr(addr.Network(), addr.String()) - if err != nil { - t.Errorf("#%d: %v", i, err) - } else if !reflect.DeepEqual(rtaddr, addr) { - t.Errorf("#%d: got %#v; want %#v", i, rtaddr, addr) + if err == nil { + addr2, err := ResolveIPAddr(addr.Network(), addr.String()) + if !reflect.DeepEqual(addr2, tt.addr) || err != tt.err { + t.Errorf("(%q, %q): ResolveIPAddr(%q, %q) = %#v, %v, want %#v, %v", tt.network, tt.litAddrOrName, addr.Network(), addr.String(), addr2, err, tt.addr, tt.err) + } } } } diff --git a/libgo/go/net/ipsock.go b/libgo/go/net/ipsock.go index 55f697f622..f1394a7ed8 100644 --- a/libgo/go/net/ipsock.go +++ b/libgo/go/net/ipsock.go @@ -1,4 +1,4 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -7,10 +7,16 @@ package net import ( - "errors" - "time" + "context" ) +// BUG(rsc,mikio): On DragonFly BSD and OpenBSD, listening on the +// "tcp" and "udp" networks does not listen for both IPv4 and IPv6 +// connections. This is due to the fact that IPv4 traffic will not be +// routed to an IPv6 socket - two separate sockets are required if +// both address families are to be supported. +// See inet6(4) for details. + var ( // supportsIPv4 reports whether the platform supports IPv4 // networking functionality. @@ -22,7 +28,7 @@ var ( // supportsIPv4map reports whether the platform supports // mapping an IPv4 address inside an IPv6 address at transport - // layer protocols. See RFC 4291, RFC 4038 and RFC 3493. + // layer protocols. See RFC 4291, RFC 4038 and RFC 3493. supportsIPv4map bool ) @@ -73,13 +79,11 @@ func (addrs addrList) partition(strategy func(Addr) bool) (primaries, fallbacks return } -var errNoSuitableAddress = errors.New("no suitable address found") - // filterAddrList applies a filter to a list of IP addresses, // yielding a list of Addr objects. Known filters are nil, ipv4only, // and ipv6only. It returns every address when the filter is nil. // The result contains at least one address when error is nil. -func filterAddrList(filter func(IPAddr) bool, ips []IPAddr, inetaddr func(IPAddr) Addr) (addrList, error) { +func filterAddrList(filter func(IPAddr) bool, ips []IPAddr, inetaddr func(IPAddr) Addr, originalAddr string) (addrList, error) { var addrs addrList for _, ip := range ips { if filter == nil || filter(ip) { @@ -87,92 +91,82 @@ func filterAddrList(filter func(IPAddr) bool, ips []IPAddr, inetaddr func(IPAddr } } if len(addrs) == 0 { - return nil, errNoSuitableAddress + return nil, &AddrError{Err: errNoSuitableAddress.Error(), Addr: originalAddr} } return addrs, nil } -// ipv4only reports whether the kernel supports IPv4 addressing mode -// and addr is an IPv4 address. +// ipv4only reports whether addr is an IPv4 address. func ipv4only(addr IPAddr) bool { - return supportsIPv4 && addr.IP.To4() != nil + return addr.IP.To4() != nil } -// ipv6only reports whether the kernel supports IPv6 addressing mode -// and addr is an IPv6 address except IPv4-mapped IPv6 address. +// ipv6only reports whether addr is an IPv6 address except IPv4-mapped IPv6 address. func ipv6only(addr IPAddr) bool { - return supportsIPv6 && len(addr.IP) == IPv6len && addr.IP.To4() == nil + return len(addr.IP) == IPv6len && addr.IP.To4() == nil } // SplitHostPort splits a network address of the form "host:port", // "[host]:port" or "[ipv6-host%zone]:port" into host or -// ipv6-host%zone and port. A literal address or host name for IPv6 +// ipv6-host%zone and port. A literal address or host name for IPv6 // must be enclosed in square brackets, as in "[::1]:80", // "[ipv6-host]:http" or "[ipv6-host%zone]:80". func SplitHostPort(hostport string) (host, port string, err error) { + const ( + missingPort = "missing port in address" + tooManyColons = "too many colons in address" + ) + addrErr := func(addr, why string) (host, port string, err error) { + return "", "", &AddrError{Err: why, Addr: addr} + } j, k := 0, 0 // The port starts after the last colon. i := last(hostport, ':') if i < 0 { - goto missingPort + return addrErr(hostport, missingPort) } if hostport[0] == '[' { // Expect the first ']' just before the last ':'. end := byteIndex(hostport, ']') if end < 0 { - err = &AddrError{Err: "missing ']' in address", Addr: hostport} - return + return addrErr(hostport, "missing ']' in address") } switch end + 1 { case len(hostport): // There can't be a ':' behind the ']' now. - goto missingPort + return addrErr(hostport, missingPort) case i: // The expected result. default: // Either ']' isn't followed by a colon, or it is // followed by a colon that is not the last one. if hostport[end+1] == ':' { - goto tooManyColons + return addrErr(hostport, tooManyColons) } - goto missingPort + return addrErr(hostport, missingPort) } host = hostport[1:end] j, k = 1, end+1 // there can't be a '[' resp. ']' before these positions } else { host = hostport[:i] if byteIndex(host, ':') >= 0 { - goto tooManyColons + return addrErr(hostport, tooManyColons) } if byteIndex(host, '%') >= 0 { - goto missingBrackets + return addrErr(hostport, "missing brackets in address") } } if byteIndex(hostport[j:], '[') >= 0 { - err = &AddrError{Err: "unexpected '[' in address", Addr: hostport} - return + return addrErr(hostport, "unexpected '[' in address") } if byteIndex(hostport[k:], ']') >= 0 { - err = &AddrError{Err: "unexpected ']' in address", Addr: hostport} - return + return addrErr(hostport, "unexpected ']' in address") } port = hostport[i+1:] - return - -missingPort: - err = &AddrError{Err: "missing port in address", Addr: hostport} - return - -tooManyColons: - err = &AddrError{Err: "too many colons in address", Addr: hostport} - return - -missingBrackets: - err = &AddrError{Err: "missing brackets in address", Addr: hostport} - return + return host, port, nil } func splitHostZone(s string) (host, zone string) { @@ -201,7 +195,7 @@ func JoinHostPort(host, port string) string { // address or a DNS name, and returns a list of internet protocol // family addresses. The result contains at least one address when // error is nil. -func internetAddrList(net, addr string, deadline time.Time) (addrList, error) { +func (r *Resolver) internetAddrList(ctx context.Context, net, addr string) (addrList, error) { var ( err error host, port string @@ -213,7 +207,7 @@ func internetAddrList(net, addr string, deadline time.Time) (addrList, error) { if host, port, err = SplitHostPort(addr); err != nil { return nil, err } - if portnum, err = LookupPort(net, port); err != nil { + if portnum, err = r.LookupPort(ctx, net, port); err != nil { return nil, err } } @@ -239,20 +233,21 @@ func internetAddrList(net, addr string, deadline time.Time) (addrList, error) { if host == "" { return addrList{inetaddr(IPAddr{})}, nil } - // Try as a literal IP address. - var ip IP - if ip = parseIPv4(host); ip != nil { - return addrList{inetaddr(IPAddr{IP: ip})}, nil - } - var zone string - if ip, zone = parseIPv6(host, true); ip != nil { - return addrList{inetaddr(IPAddr{IP: ip, Zone: zone})}, nil - } - // Try as a DNS name. - ips, err := lookupIPDeadline(host, deadline) - if err != nil { - return nil, err + + // Try as a literal IP address, then as a DNS name. + var ips []IPAddr + if ip := parseIPv4(host); ip != nil { + ips = []IPAddr{{IP: ip}} + } else if ip, zone := parseIPv6(host, true); ip != nil { + ips = []IPAddr{{IP: ip, Zone: zone}} + } else { + // Try as a DNS name. + ips, err = r.LookupIPAddr(ctx, host) + if err != nil { + return nil, err + } } + var filter func(IPAddr) bool if net != "" && net[len(net)-1] == '4' { filter = ipv4only @@ -260,26 +255,12 @@ func internetAddrList(net, addr string, deadline time.Time) (addrList, error) { if net != "" && net[len(net)-1] == '6' { filter = ipv6only } - return filterAddrList(filter, ips, inetaddr) + return filterAddrList(filter, ips, inetaddr, host) } -func zoneToString(zone int) string { - if zone == 0 { - return "" - } - if ifi, err := InterfaceByIndex(zone); err == nil { - return ifi.Name - } - return uitoa(uint(zone)) -} - -func zoneToInt(zone string) int { - if zone == "" { - return 0 - } - if ifi, err := InterfaceByName(zone); err == nil { - return ifi.Index +func loopbackIP(net string) IP { + if net != "" && net[len(net)-1] == '6' { + return IPv6loopback } - n, _, _ := dtoi(zone, 0) - return n + return IP{127, 0, 0, 1} } diff --git a/libgo/go/net/ipsock_plan9.go b/libgo/go/net/ipsock_plan9.go index 9da6ec3053..b7fd344c8a 100644 --- a/libgo/go/net/ipsock_plan9.go +++ b/libgo/go/net/ipsock_plan9.go @@ -1,4 +1,4 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -7,6 +7,7 @@ package net import ( + "context" "os" "syscall" ) @@ -39,8 +40,8 @@ func probeIPv4Stack() bool { return probe(netdir+"/iproute", "4i") } -// probeIPv6Stack returns two boolean values. If the first boolean -// value is true, kernel supports basic IPv6 functionality. If the +// probeIPv6Stack returns two boolean values. If the first boolean +// value is true, kernel supports basic IPv6 functionality. If the // second boolean value is true, kernel supports IPv6 IPv4-mapping. func probeIPv6Stack() (supportsIPv6, supportsIPv4map bool) { // Plan 9 uses IPv6 natively, see ip(3). @@ -62,7 +63,7 @@ func parsePlan9Addr(s string) (ip IP, iport int, err error) { return nil, 0, &ParseError{Type: "IP address", Text: s} } } - p, _, ok := dtoi(s[i+1:], 0) + p, _, ok := dtoi(s[i+1:]) if !ok { return nil, 0, &ParseError{Type: "port", Text: s} } @@ -99,7 +100,7 @@ func readPlan9Addr(proto, filename string) (addr Addr, err error) { return addr, nil } -func startPlan9(net string, addr Addr) (ctl *os.File, dest, proto, name string, err error) { +func startPlan9(ctx context.Context, net string, addr Addr) (ctl *os.File, dest, proto, name string, err error) { var ( ip IP port int @@ -118,7 +119,12 @@ func startPlan9(net string, addr Addr) (ctl *os.File, dest, proto, name string, return } - clone, dest, err := queryCS1(proto, ip, port) + if port > 65535 { + err = InvalidAddrError("port should be < 65536") + return + } + + clone, dest, err := queryCS1(ctx, proto, ip, port) if err != nil { return } @@ -135,8 +141,8 @@ func startPlan9(net string, addr Addr) (ctl *os.File, dest, proto, name string, return f, dest, proto, string(buf[:n]), nil } -func netErr(e error) { - oe, ok := e.(*OpError) +func fixErr(err error) { + oe, ok := err.(*OpError) if !ok { return } @@ -165,81 +171,141 @@ func netErr(e error) { } } -func dialPlan9(net string, laddr, raddr Addr) (fd *netFD, err error) { - defer func() { netErr(err) }() - f, dest, proto, name, err := startPlan9(net, raddr) +func dialPlan9(ctx context.Context, net string, laddr, raddr Addr) (fd *netFD, err error) { + defer func() { fixErr(err) }() + type res struct { + fd *netFD + err error + } + resc := make(chan res) + go func() { + testHookDialChannel() + fd, err := dialPlan9Blocking(ctx, net, laddr, raddr) + select { + case resc <- res{fd, err}: + case <-ctx.Done(): + if fd != nil { + fd.Close() + } + } + }() + select { + case res := <-resc: + return res.fd, res.err + case <-ctx.Done(): + return nil, mapErr(ctx.Err()) + } +} + +func dialPlan9Blocking(ctx context.Context, net string, laddr, raddr Addr) (fd *netFD, err error) { + if isWildcard(raddr) { + raddr = toLocal(raddr, net) + } + f, dest, proto, name, err := startPlan9(ctx, net, raddr) if err != nil { - return nil, &OpError{Op: "dial", Net: net, Source: laddr, Addr: raddr, Err: err} + return nil, err } _, err = f.WriteString("connect " + dest) if err != nil { f.Close() - return nil, &OpError{Op: "dial", Net: f.Name(), Source: laddr, Addr: raddr, Err: err} + return nil, err } data, err := os.OpenFile(netdir+"/"+proto+"/"+name+"/data", os.O_RDWR, 0) if err != nil { f.Close() - return nil, &OpError{Op: "dial", Net: net, Source: laddr, Addr: raddr, Err: err} + return nil, err } laddr, err = readPlan9Addr(proto, netdir+"/"+proto+"/"+name+"/local") if err != nil { data.Close() f.Close() - return nil, &OpError{Op: "dial", Net: proto, Source: laddr, Addr: raddr, Err: err} + return nil, err } - return newFD(proto, name, f, data, laddr, raddr) + return newFD(proto, name, nil, f, data, laddr, raddr) } -func listenPlan9(net string, laddr Addr) (fd *netFD, err error) { - defer func() { netErr(err) }() - f, dest, proto, name, err := startPlan9(net, laddr) +func listenPlan9(ctx context.Context, net string, laddr Addr) (fd *netFD, err error) { + defer func() { fixErr(err) }() + f, dest, proto, name, err := startPlan9(ctx, net, laddr) if err != nil { - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr, Err: err} + return nil, err } _, err = f.WriteString("announce " + dest) if err != nil { f.Close() - return nil, &OpError{Op: "announce", Net: proto, Source: nil, Addr: laddr, Err: err} + return nil, err } laddr, err = readPlan9Addr(proto, netdir+"/"+proto+"/"+name+"/local") if err != nil { f.Close() - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr, Err: err} + return nil, err } - return newFD(proto, name, f, nil, laddr, nil) + return newFD(proto, name, nil, f, nil, laddr, nil) } func (fd *netFD) netFD() (*netFD, error) { - return newFD(fd.net, fd.n, fd.ctl, fd.data, fd.laddr, fd.raddr) + return newFD(fd.net, fd.n, fd.listen, fd.ctl, fd.data, fd.laddr, fd.raddr) } func (fd *netFD) acceptPlan9() (nfd *netFD, err error) { - defer func() { netErr(err) }() + defer func() { fixErr(err) }() if err := fd.readLock(); err != nil { return nil, err } defer fd.readUnlock() - f, err := os.Open(fd.dir + "/listen") + listen, err := os.Open(fd.dir + "/listen") if err != nil { - return nil, &OpError{Op: "accept", Net: fd.dir + "/listen", Source: nil, Addr: fd.laddr, Err: err} + return nil, err } var buf [16]byte - n, err := f.Read(buf[:]) + n, err := listen.Read(buf[:]) if err != nil { - f.Close() - return nil, &OpError{Op: "accept", Net: fd.dir + "/listen", Source: nil, Addr: fd.laddr, Err: err} + listen.Close() + return nil, err } name := string(buf[:n]) + ctl, err := os.OpenFile(netdir+"/"+fd.net+"/"+name+"/ctl", os.O_RDWR, 0) + if err != nil { + listen.Close() + return nil, err + } data, err := os.OpenFile(netdir+"/"+fd.net+"/"+name+"/data", os.O_RDWR, 0) if err != nil { - f.Close() - return nil, &OpError{Op: "accept", Net: fd.net, Source: nil, Addr: fd.laddr, Err: err} + listen.Close() + ctl.Close() + return nil, err } raddr, err := readPlan9Addr(fd.net, netdir+"/"+fd.net+"/"+name+"/remote") if err != nil { + listen.Close() + ctl.Close() data.Close() - f.Close() - return nil, &OpError{Op: "accept", Net: fd.net, Source: nil, Addr: fd.laddr, Err: err} + return nil, err + } + return newFD(fd.net, name, listen, ctl, data, fd.laddr, raddr) +} + +func isWildcard(a Addr) bool { + var wildcard bool + switch a := a.(type) { + case *TCPAddr: + wildcard = a.isWildcard() + case *UDPAddr: + wildcard = a.isWildcard() + case *IPAddr: + wildcard = a.isWildcard() + } + return wildcard +} + +func toLocal(a Addr, net string) Addr { + switch a := a.(type) { + case *TCPAddr: + a.IP = loopbackIP(net) + case *UDPAddr: + a.IP = loopbackIP(net) + case *IPAddr: + a.IP = loopbackIP(net) } - return newFD(fd.net, name, f, data, fd.laddr, raddr) + return a } diff --git a/libgo/go/net/ipsock_posix.go b/libgo/go/net/ipsock_posix.go index 2bddd46a15..ff280c3e4e 100644 --- a/libgo/go/net/ipsock_posix.go +++ b/libgo/go/net/ipsock_posix.go @@ -1,26 +1,17 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris windows -// Internet protocol family sockets for POSIX - package net import ( + "context" "runtime" "syscall" - "time" ) -// BUG(rsc,mikio): On DragonFly BSD and OpenBSD, listening on the -// "tcp" and "udp" networks does not listen for both IPv4 and IPv6 -// connections. This is due to the fact that IPv4 traffic will not be -// routed to an IPv6 socket - two separate sockets are required if -// both address families are to be supported. -// See inet6(4) for details. - func probeIPv4Stack() bool { s, err := socketFunc(syscall.AF_INET, syscall.SOCK_STREAM, syscall.IPPROTO_TCP) switch err { @@ -35,15 +26,15 @@ func probeIPv4Stack() bool { // Should we try to use the IPv4 socket interface if we're // only dealing with IPv4 sockets? As long as the host system // understands IPv6, it's okay to pass IPv4 addresses to the IPv6 -// interface. That simplifies our code and is most general. +// interface. That simplifies our code and is most general. // Unfortunately, we need to run on kernels built without IPv6 -// support too. So probe the kernel to figure it out. +// support too. So probe the kernel to figure it out. // // probeIPv6Stack probes both basic IPv6 capability and IPv6 IPv4- // mapping capability which is controlled by IPV6_V6ONLY socket // option and/or kernel state "net.inet6.ip6.v6only". -// It returns two boolean values. If the first boolean value is -// true, kernel supports basic IPv6 functionality. If the second +// It returns two boolean values. If the first boolean value is +// true, kernel supports basic IPv6 functionality. If the second // boolean value is true, kernel supports IPv6 IPv4-mapping. func probeIPv6Stack() (supportsIPv6, supportsIPv4map bool) { var probes = []struct { @@ -52,7 +43,7 @@ func probeIPv6Stack() (supportsIPv6, supportsIPv4map bool) { }{ // IPv6 communication capability {laddr: TCPAddr{IP: ParseIP("::1")}, value: 1}, - // IPv6 IPv4-mapped address communication capability + // IPv4-mapped IPv6 address communication capability {laddr: TCPAddr{IP: IPv4(127, 0, 0, 1)}, value: 0}, } var supps [2]bool @@ -61,7 +52,7 @@ func probeIPv6Stack() (supportsIPv6, supportsIPv4map bool) { // Some released versions of DragonFly BSD pretend to // accept IPV6_V6ONLY=0 successfully, but the state // still stays IPV6_V6ONLY=1. Eventually DragonFly BSD - // stops preteding, but the transition period would + // stops pretending, but the transition period would // cause unpredictable behavior and we need to avoid // it. // @@ -93,8 +84,8 @@ func probeIPv6Stack() (supportsIPv6, supportsIPv4map bool) { } // favoriteAddrFamily returns the appropriate address family to -// the given net, laddr, raddr and mode. At first it figures -// address family out from the net. If mode indicates "listen" +// the given net, laddr, raddr and mode. At first it figures +// address family out from the net. If mode indicates "listen" // and laddr is a wildcard, it assumes that the user wants to // make a passive connection with a wildcard address family, both // AF_INET and AF_INET6, and a wildcard address like following: @@ -155,10 +146,12 @@ func favoriteAddrFamily(net string, laddr, raddr sockaddr, mode string) (family } // Internet sockets (TCP, UDP, IP) - -func internetSocket(net string, laddr, raddr sockaddr, deadline time.Time, sotype, proto int, mode string, cancel <-chan struct{}) (fd *netFD, err error) { +func internetSocket(ctx context.Context, net string, laddr, raddr sockaddr, sotype, proto int, mode string) (fd *netFD, err error) { + if (runtime.GOOS == "windows" || runtime.GOOS == "openbsd" || runtime.GOOS == "nacl") && mode == "dial" && raddr.isWildcard() { + raddr = raddr.toLocal(net) + } family, ipv6only := favoriteAddrFamily(net, laddr, raddr, mode) - return socket(net, family, sotype, proto, ipv6only, laddr, raddr, deadline, cancel) + return socket(ctx, net, family, sotype, proto, ipv6only, laddr, raddr) } func ipToSockaddr(family int, ip IP, port int, zone string) (syscall.Sockaddr, error) { @@ -167,34 +160,35 @@ func ipToSockaddr(family int, ip IP, port int, zone string) (syscall.Sockaddr, e if len(ip) == 0 { ip = IPv4zero } - if ip = ip.To4(); ip == nil { + ip4 := ip.To4() + if ip4 == nil { return nil, &AddrError{Err: "non-IPv4 address", Addr: ip.String()} } - sa := new(syscall.SockaddrInet4) - for i := 0; i < IPv4len; i++ { - sa.Addr[i] = ip[i] - } - sa.Port = port + sa := &syscall.SockaddrInet4{Port: port} + copy(sa.Addr[:], ip4) return sa, nil case syscall.AF_INET6: - if len(ip) == 0 { - ip = IPv6zero - } - // IPv4 callers use 0.0.0.0 to mean "announce on any available address". - // In IPv6 mode, Linux treats that as meaning "announce on 0.0.0.0", - // which it refuses to do. Rewrite to the IPv6 unspecified address. - if ip.Equal(IPv4zero) { + // In general, an IP wildcard address, which is either + // "0.0.0.0" or "::", means the entire IP addressing + // space. For some historical reason, it is used to + // specify "any available address" on some operations + // of IP node. + // + // When the IP node supports IPv4-mapped IPv6 address, + // we allow an listener to listen to the wildcard + // address of both IP addressing spaces by specifying + // IPv6 wildcard address. + if len(ip) == 0 || ip.Equal(IPv4zero) { ip = IPv6zero } - if ip = ip.To16(); ip == nil { + // We accept any IPv6 address including IPv4-mapped + // IPv6 address. + ip6 := ip.To16() + if ip6 == nil { return nil, &AddrError{Err: "non-IPv6 address", Addr: ip.String()} } - sa := new(syscall.SockaddrInet6) - for i := 0; i < IPv6len; i++ { - sa.Addr[i] = ip[i] - } - sa.Port = port - sa.ZoneId = uint32(zoneToInt(zone)) + sa := &syscall.SockaddrInet6{Port: port, ZoneId: uint32(zoneToInt(zone))} + copy(sa.Addr[:], ip6) return sa, nil } return nil, &AddrError{Err: "invalid address family", Addr: ip.String()} diff --git a/libgo/go/net/ipsock_test.go b/libgo/go/net/ipsock_test.go index b36557a157..1d0f00ff5e 100644 --- a/libgo/go/net/ipsock_test.go +++ b/libgo/go/net/ipsock_test.go @@ -205,13 +205,13 @@ var addrListTests = []struct { nil, }, - {nil, nil, testInetaddr, nil, nil, nil, errNoSuitableAddress}, + {nil, nil, testInetaddr, nil, nil, nil, &AddrError{errNoSuitableAddress.Error(), "ADDR"}}, - {ipv4only, nil, testInetaddr, nil, nil, nil, errNoSuitableAddress}, - {ipv4only, []IPAddr{{IP: IPv6loopback}}, testInetaddr, nil, nil, nil, errNoSuitableAddress}, + {ipv4only, nil, testInetaddr, nil, nil, nil, &AddrError{errNoSuitableAddress.Error(), "ADDR"}}, + {ipv4only, []IPAddr{{IP: IPv6loopback}}, testInetaddr, nil, nil, nil, &AddrError{errNoSuitableAddress.Error(), "ADDR"}}, - {ipv6only, nil, testInetaddr, nil, nil, nil, errNoSuitableAddress}, - {ipv6only, []IPAddr{{IP: IPv4(127, 0, 0, 1)}}, testInetaddr, nil, nil, nil, errNoSuitableAddress}, + {ipv6only, nil, testInetaddr, nil, nil, nil, &AddrError{errNoSuitableAddress.Error(), "ADDR"}}, + {ipv6only, []IPAddr{{IP: IPv4(127, 0, 0, 1)}}, testInetaddr, nil, nil, nil, &AddrError{errNoSuitableAddress.Error(), "ADDR"}}, } func TestAddrList(t *testing.T) { @@ -220,8 +220,8 @@ func TestAddrList(t *testing.T) { } for i, tt := range addrListTests { - addrs, err := filterAddrList(tt.filter, tt.ips, tt.inetaddr) - if err != tt.err { + addrs, err := filterAddrList(tt.filter, tt.ips, tt.inetaddr, "ADDR") + if !reflect.DeepEqual(err, tt.err) { t.Errorf("#%v: got %v; want %v", i, err, tt.err) } if tt.err != nil { diff --git a/libgo/go/net/listen_test.go b/libgo/go/net/listen_test.go index 51ffe67238..6037f3600d 100644 --- a/libgo/go/net/listen_test.go +++ b/libgo/go/net/listen_test.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -8,6 +8,7 @@ package net import ( "fmt" + "internal/testenv" "os" "runtime" "syscall" @@ -157,7 +158,7 @@ var dualStackTCPListenerTests = []struct { network2, address2 string // second listener xerr error // expected error value, nil or other }{ - // Test cases and expected results for the attemping 2nd listen on the same port + // Test cases and expected results for the attempting 2nd listen on the same port // 1st listen 2nd listen darwin freebsd linux openbsd // ------------------------------------------------------------------------------------ // "tcp" "" "tcp" "" - - - - @@ -216,9 +217,12 @@ var dualStackTCPListenerTests = []struct { // TestDualStackTCPListener tests both single and double listen // to a test listener with various address families, different // listening address and same port. +// +// On DragonFly BSD, we expect the kernel version of node under test +// to be greater than or equal to 4.4. func TestDualStackTCPListener(t *testing.T) { switch runtime.GOOS { - case "dragonfly", "nacl", "plan9": // re-enable on dragonfly once the new IP control block management has landed + case "nacl", "plan9": t.Skipf("not supported on %s", runtime.GOOS) } if !supportsIPv4 || !supportsIPv6 { @@ -301,11 +305,14 @@ var dualStackUDPListenerTests = []struct { } // TestDualStackUDPListener tests both single and double listen -// to a test listener with various address families, differnet +// to a test listener with various address families, different // listening address and same port. +// +// On DragonFly BSD, we expect the kernel version of node under test +// to be greater than or equal to 4.4. func TestDualStackUDPListener(t *testing.T) { switch runtime.GOOS { - case "dragonfly", "nacl", "plan9": // re-enable on dragonfly once the new IP control block management has landed + case "nacl", "plan9": t.Skipf("not supported on %s", runtime.GOOS) } if !supportsIPv4 || !supportsIPv6 { @@ -477,13 +484,12 @@ func checkDualStackAddrFamily(fd *netFD) error { } func TestWildWildcardListener(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + switch runtime.GOOS { case "plan9": t.Skipf("not supported on %s", runtime.GOOS) } - if testing.Short() || !*testExternal { - t.Skip("avoid external network") - } defer func() { if p := recover(); p != nil { @@ -521,12 +527,17 @@ var ipv4MulticastListenerTests = []struct { // test listener with same address family, same group address and same // port. func TestIPv4MulticastListener(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + switch runtime.GOOS { case "android", "nacl", "plan9": t.Skipf("not supported on %s", runtime.GOOS) case "solaris": t.Skipf("not supported on solaris, see golang.org/issue/7399") } + if !supportsIPv4 { + t.Skip("IPv4 is not supported") + } closer := func(cs []*UDPConn) { for _, c := range cs { @@ -542,7 +553,7 @@ func TestIPv4MulticastListener(t *testing.T) { // routing stuff for finding out an appropriate // nexthop containing both network and link layer // adjacencies. - if ifi == nil && (testing.Short() || !*testExternal) { + if ifi == nil || !*testIPv4 { continue } for _, tt := range ipv4MulticastListenerTests { @@ -591,6 +602,8 @@ var ipv6MulticastListenerTests = []struct { // test listener with same address family, same group address and same // port. func TestIPv6MulticastListener(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + switch runtime.GOOS { case "plan9": t.Skipf("not supported on %s", runtime.GOOS) @@ -598,7 +611,7 @@ func TestIPv6MulticastListener(t *testing.T) { t.Skipf("not supported on solaris, see issue 7399") } if !supportsIPv6 { - t.Skip("ipv6 is not supported") + t.Skip("IPv6 is not supported") } if os.Getuid() != 0 { t.Skip("must be root") @@ -618,7 +631,7 @@ func TestIPv6MulticastListener(t *testing.T) { // routing stuff for finding out an appropriate // nexthop containing both network and link layer // adjacencies. - if ifi == nil && (testing.Short() || !*testExternal || !*testIPv6) { + if ifi == nil && !*testIPv6 { continue } for _, tt := range ipv6MulticastListenerTests { diff --git a/libgo/go/net/lookup.go b/libgo/go/net/lookup.go index 7aa111ba92..cc2013e432 100644 --- a/libgo/go/net/lookup.go +++ b/libgo/go/net/lookup.go @@ -1,12 +1,13 @@ -// Copyright 2012 The Go Authors. All rights reserved. +// Copyright 2012 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package net import ( + "context" + "internal/nettrace" "internal/singleflight" - "time" ) // protocols contains minimal mappings between internet protocol @@ -14,17 +15,101 @@ import ( // protocol numbers. // // See http://www.iana.org/assignments/protocol-numbers +// +// On Unix, this map is augmented by readProtocols via lookupProtocol. var protocols = map[string]int{ - "icmp": 1, "ICMP": 1, - "igmp": 2, "IGMP": 2, - "tcp": 6, "TCP": 6, - "udp": 17, "UDP": 17, - "ipv6-icmp": 58, "IPV6-ICMP": 58, "IPv6-ICMP": 58, + "icmp": 1, + "igmp": 2, + "tcp": 6, + "udp": 17, + "ipv6-icmp": 58, +} + +// services contains minimal mappings between services names and port +// numbers for platforms that don't have a complete list of port numbers +// (some Solaris distros, nacl, etc). +// On Unix, this map is augmented by readServices via goLookupPort. +var services = map[string]map[string]int{ + "udp": { + "domain": 53, + }, + "tcp": { + "ftp": 21, + "ftps": 990, + "gopher": 70, // ʕ◔ϖ◔ʔ + "http": 80, + "https": 443, + "imap2": 143, + "imap3": 220, + "imaps": 993, + "pop3": 110, + "pop3s": 995, + "smtp": 25, + "ssh": 22, + "telnet": 23, + }, +} + +const maxProtoLength = len("RSVP-E2E-IGNORE") + 10 // with room to grow + +func lookupProtocolMap(name string) (int, error) { + var lowerProtocol [maxProtoLength]byte + n := copy(lowerProtocol[:], name) + lowerASCIIBytes(lowerProtocol[:n]) + proto, found := protocols[string(lowerProtocol[:n])] + if !found || n != len(name) { + return 0, &AddrError{Err: "unknown IP protocol specified", Addr: name} + } + return proto, nil +} + +const maxServiceLength = len("mobility-header") + 10 // with room to grow + +func lookupPortMap(network, service string) (port int, error error) { + switch network { + case "tcp4", "tcp6": + network = "tcp" + case "udp4", "udp6": + network = "udp" + } + + if m, ok := services[network]; ok { + var lowerService [maxServiceLength]byte + n := copy(lowerService[:], service) + lowerASCIIBytes(lowerService[:n]) + if port, ok := m[string(lowerService[:n])]; ok && n == len(service) { + return port, nil + } + } + return 0, &AddrError{Err: "unknown port", Addr: network + "/" + service} +} + +// DefaultResolver is the resolver used by the package-level Lookup +// functions and by Dialers without a specified Resolver. +var DefaultResolver = &Resolver{} + +// A Resolver looks up names and numbers. +// +// A nil *Resolver is equivalent to a zero Resolver. +type Resolver struct { + // PreferGo controls whether Go's built-in DNS resolver is preferred + // on platforms where it's available. It is equivalent to setting + // GODEBUG=netdns=go, but scoped to just this resolver. + PreferGo bool + + // TODO(bradfitz): optional interface impl override hook + // TODO(bradfitz): Timeout time.Duration? } // LookupHost looks up the given host using the local resolver. -// It returns an array of that host's addresses. +// It returns a slice of that host's addresses. func LookupHost(host string) (addrs []string, err error) { + return DefaultResolver.LookupHost(context.Background(), host) +} + +// LookupHost looks up the given host using the local resolver. +// It returns a slice of that host's addresses. +func (r *Resolver) LookupHost(ctx context.Context, host string) (addrs []string, err error) { // Make sure that no matter what we do later, host=="" is rejected. // ParseIP, for example, does accept empty strings. if host == "" { @@ -33,43 +118,77 @@ func LookupHost(host string) (addrs []string, err error) { if ip := ParseIP(host); ip != nil { return []string{host}, nil } - return lookupHost(host) + return r.lookupHost(ctx, host) } // LookupIP looks up host using the local resolver. -// It returns an array of that host's IPv4 and IPv6 addresses. -func LookupIP(host string) (ips []IP, err error) { +// It returns a slice of that host's IPv4 and IPv6 addresses. +func LookupIP(host string) ([]IP, error) { + addrs, err := DefaultResolver.LookupIPAddr(context.Background(), host) + if err != nil { + return nil, err + } + ips := make([]IP, len(addrs)) + for i, ia := range addrs { + ips[i] = ia.IP + } + return ips, nil +} + +// LookupIPAddr looks up host using the local resolver. +// It returns a slice of that host's IPv4 and IPv6 addresses. +func (r *Resolver) LookupIPAddr(ctx context.Context, host string) ([]IPAddr, error) { // Make sure that no matter what we do later, host=="" is rejected. // ParseIP, for example, does accept empty strings. if host == "" { return nil, &DNSError{Err: errNoSuchHost.Error(), Name: host} } if ip := ParseIP(host); ip != nil { - return []IP{ip}, nil + return []IPAddr{{IP: ip}}, nil } - addrs, err := lookupIPMerge(host) - if err != nil { - return + trace, _ := ctx.Value(nettrace.TraceKey{}).(*nettrace.Trace) + if trace != nil && trace.DNSStart != nil { + trace.DNSStart(host) } - ips = make([]IP, len(addrs)) - for i, addr := range addrs { - ips[i] = addr.IP + // The underlying resolver func is lookupIP by default but it + // can be overridden by tests. This is needed by net/http, so it + // uses a context key instead of unexported variables. + resolverFunc := r.lookupIP + if alt, _ := ctx.Value(nettrace.LookupIPAltResolverKey{}).(func(context.Context, string) ([]IPAddr, error)); alt != nil { + resolverFunc = alt } - return -} -var lookupGroup singleflight.Group - -// lookupIPMerge wraps lookupIP, but makes sure that for any given -// host, only one lookup is in-flight at a time. The returned memory -// is always owned by the caller. -func lookupIPMerge(host string) (addrs []IPAddr, err error) { - addrsi, err, shared := lookupGroup.Do(host, func() (interface{}, error) { - return testHookLookupIP(lookupIP, host) + ch := lookupGroup.DoChan(host, func() (interface{}, error) { + return testHookLookupIP(ctx, resolverFunc, host) }) - return lookupIPReturn(addrsi, err, shared) + + select { + case <-ctx.Done(): + // The DNS lookup timed out for some reason. Force + // future requests to start the DNS lookup again + // rather than waiting for the current lookup to + // complete. See issue 8602. + err := mapErr(ctx.Err()) + lookupGroup.Forget(host) + if trace != nil && trace.DNSDone != nil { + trace.DNSDone(nil, false, err) + } + return nil, err + case r := <-ch: + if trace != nil && trace.DNSDone != nil { + addrs, _ := r.Val.([]IPAddr) + trace.DNSDone(ipAddrsEface(addrs), r.Shared, r.Err) + } + return lookupIPReturn(r.Val, r.Err, r.Shared) + } } +// lookupGroup merges LookupIPAddr calls together for lookups +// for the same host. The lookupGroup key is is the LookupIPAddr.host +// argument. +// The return values are ([]IPAddr, error). +var lookupGroup singleflight.Group + // lookupIPReturn turns the return values from singleflight.Do into // the return values from LookupIP. func lookupIPReturn(addrsi interface{}, err error, shared bool) ([]IPAddr, error) { @@ -85,52 +204,25 @@ func lookupIPReturn(addrsi interface{}, err error, shared bool) ([]IPAddr, error return addrs, nil } -// lookupIPDeadline looks up a hostname with a deadline. -func lookupIPDeadline(host string, deadline time.Time) (addrs []IPAddr, err error) { - if deadline.IsZero() { - return lookupIPMerge(host) - } - - // We could push the deadline down into the name resolution - // functions. However, the most commonly used implementation - // calls getaddrinfo, which has no timeout. - - timeout := deadline.Sub(time.Now()) - if timeout <= 0 { - return nil, errTimeout - } - t := time.NewTimer(timeout) - defer t.Stop() - - ch := lookupGroup.DoChan(host, func() (interface{}, error) { - return testHookLookupIP(lookupIP, host) - }) - - select { - case <-t.C: - // The DNS lookup timed out for some reason. Force - // future requests to start the DNS lookup again - // rather than waiting for the current lookup to - // complete. See issue 8602. - lookupGroup.Forget(host) - - return nil, errTimeout - - case r := <-ch: - return lookupIPReturn(r.Val, r.Err, r.Shared) +// ipAddrsEface returns an empty interface slice of addrs. +func ipAddrsEface(addrs []IPAddr) []interface{} { + s := make([]interface{}, len(addrs)) + for i, v := range addrs { + s[i] = v } + return s } // LookupPort looks up the port for the given network and service. func LookupPort(network, service string) (port int, err error) { - if service == "" { - // Lock in the legacy behavior that an empty string - // means port 0. See Issue 13610. - return 0, nil - } - port, _, ok := dtoi(service, 0) - if !ok && port != big && port != -big { - port, err = lookupPort(network, service) + return DefaultResolver.LookupPort(context.Background(), network, service) +} + +// LookupPort looks up the port for the given network and service. +func (r *Resolver) LookupPort(ctx context.Context, network, service string) (port int, err error) { + port, needsLookup := parsePort(service) + if needsLookup { + port, err = r.lookupPort(ctx, network, service) if err != nil { return 0, err } @@ -141,44 +233,101 @@ func LookupPort(network, service string) (port int, err error) { return port, nil } -// LookupCNAME returns the canonical DNS host for the given name. +// LookupCNAME returns the canonical name for the given host. +// Callers that do not care about the canonical name can call +// LookupHost or LookupIP directly; both take care of resolving +// the canonical name as part of the lookup. +// +// A canonical name is the final name after following zero +// or more CNAME records. +// LookupCNAME does not return an error if host does not +// contain DNS "CNAME" records, as long as host resolves to +// address records. +func LookupCNAME(host string) (cname string, err error) { + return DefaultResolver.lookupCNAME(context.Background(), host) +} + +// LookupCNAME returns the canonical name for the given host. // Callers that do not care about the canonical name can call // LookupHost or LookupIP directly; both take care of resolving // the canonical name as part of the lookup. -func LookupCNAME(name string) (cname string, err error) { - return lookupCNAME(name) +// +// A canonical name is the final name after following zero +// or more CNAME records. +// LookupCNAME does not return an error if host does not +// contain DNS "CNAME" records, as long as host resolves to +// address records. +func (r *Resolver) LookupCNAME(ctx context.Context, host string) (cname string, err error) { + return r.lookupCNAME(ctx, host) } // LookupSRV tries to resolve an SRV query of the given service, -// protocol, and domain name. The proto is "tcp" or "udp". +// protocol, and domain name. The proto is "tcp" or "udp". // The returned records are sorted by priority and randomized // by weight within a priority. // // LookupSRV constructs the DNS name to look up following RFC 2782. -// That is, it looks up _service._proto.name. To accommodate services +// That is, it looks up _service._proto.name. To accommodate services // publishing SRV records under non-standard names, if both service // and proto are empty strings, LookupSRV looks up name directly. func LookupSRV(service, proto, name string) (cname string, addrs []*SRV, err error) { - return lookupSRV(service, proto, name) + return DefaultResolver.lookupSRV(context.Background(), service, proto, name) +} + +// LookupSRV tries to resolve an SRV query of the given service, +// protocol, and domain name. The proto is "tcp" or "udp". +// The returned records are sorted by priority and randomized +// by weight within a priority. +// +// LookupSRV constructs the DNS name to look up following RFC 2782. +// That is, it looks up _service._proto.name. To accommodate services +// publishing SRV records under non-standard names, if both service +// and proto are empty strings, LookupSRV looks up name directly. +func (r *Resolver) LookupSRV(ctx context.Context, service, proto, name string) (cname string, addrs []*SRV, err error) { + return r.lookupSRV(ctx, service, proto, name) +} + +// LookupMX returns the DNS MX records for the given domain name sorted by preference. +func LookupMX(name string) ([]*MX, error) { + return DefaultResolver.lookupMX(context.Background(), name) } // LookupMX returns the DNS MX records for the given domain name sorted by preference. -func LookupMX(name string) (mxs []*MX, err error) { - return lookupMX(name) +func (r *Resolver) LookupMX(ctx context.Context, name string) ([]*MX, error) { + return r.lookupMX(ctx, name) } // LookupNS returns the DNS NS records for the given domain name. -func LookupNS(name string) (nss []*NS, err error) { - return lookupNS(name) +func LookupNS(name string) ([]*NS, error) { + return DefaultResolver.lookupNS(context.Background(), name) +} + +// LookupNS returns the DNS NS records for the given domain name. +func (r *Resolver) LookupNS(ctx context.Context, name string) ([]*NS, error) { + return r.lookupNS(ctx, name) +} + +// LookupTXT returns the DNS TXT records for the given domain name. +func LookupTXT(name string) ([]string, error) { + return DefaultResolver.lookupTXT(context.Background(), name) } // LookupTXT returns the DNS TXT records for the given domain name. -func LookupTXT(name string) (txts []string, err error) { - return lookupTXT(name) +func (r *Resolver) LookupTXT(ctx context.Context, name string) ([]string, error) { + return r.lookupTXT(ctx, name) } // LookupAddr performs a reverse lookup for the given address, returning a list // of names mapping to that address. +// +// When using the host C library resolver, at most one result will be +// returned. To bypass the host resolver, use a custom Resolver. func LookupAddr(addr string) (names []string, err error) { - return lookupAddr(addr) + return DefaultResolver.lookupAddr(context.Background(), addr) +} + +// LookupAddr performs a reverse lookup for the given address, returning a list +// of names mapping to that address. +func (r *Resolver) LookupAddr(ctx context.Context, addr string) (names []string, err error) { + return r.lookupAddr(ctx, addr) } diff --git a/libgo/go/net/lookup_nacl.go b/libgo/go/net/lookup_nacl.go new file mode 100644 index 0000000000..43cebad760 --- /dev/null +++ b/libgo/go/net/lookup_nacl.go @@ -0,0 +1,52 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build nacl + +package net + +import ( + "context" + "syscall" +) + +func lookupProtocol(ctx context.Context, name string) (proto int, err error) { + return lookupProtocolMap(name) +} + +func (*Resolver) lookupHost(ctx context.Context, host string) (addrs []string, err error) { + return nil, syscall.ENOPROTOOPT +} + +func (*Resolver) lookupIP(ctx context.Context, host string) (addrs []IPAddr, err error) { + return nil, syscall.ENOPROTOOPT +} + +func (*Resolver) lookupPort(ctx context.Context, network, service string) (port int, err error) { + return goLookupPort(network, service) +} + +func (*Resolver) lookupCNAME(ctx context.Context, name string) (cname string, err error) { + return "", syscall.ENOPROTOOPT +} + +func (*Resolver) lookupSRV(ctx context.Context, service, proto, name string) (cname string, srvs []*SRV, err error) { + return "", nil, syscall.ENOPROTOOPT +} + +func (*Resolver) lookupMX(ctx context.Context, name string) (mxs []*MX, err error) { + return nil, syscall.ENOPROTOOPT +} + +func (*Resolver) lookupNS(ctx context.Context, name string) (nss []*NS, err error) { + return nil, syscall.ENOPROTOOPT +} + +func (*Resolver) lookupTXT(ctx context.Context, name string) (txts []string, err error) { + return nil, syscall.ENOPROTOOPT +} + +func (*Resolver) lookupAddr(ctx context.Context, addr string) (ptrs []string, err error) { + return nil, syscall.ENOPROTOOPT +} diff --git a/libgo/go/net/lookup_plan9.go b/libgo/go/net/lookup_plan9.go index a33162882b..f81e220fc8 100644 --- a/libgo/go/net/lookup_plan9.go +++ b/libgo/go/net/lookup_plan9.go @@ -1,22 +1,24 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package net import ( + "context" "errors" + "io" "os" ) -func query(filename, query string, bufSize int) (res []string, err error) { +func query(ctx context.Context, filename, query string, bufSize int) (res []string, err error) { file, err := os.OpenFile(filename, os.O_RDWR, 0) if err != nil { return } defer file.Close() - _, err = file.Seek(0, 0) + _, err = file.Seek(0, io.SeekStart) if err != nil { return } @@ -24,7 +26,7 @@ func query(filename, query string, bufSize int) (res []string, err error) { if err != nil { return } - _, err = file.Seek(0, 0) + _, err = file.Seek(0, io.SeekStart) if err != nil { return } @@ -39,7 +41,7 @@ func query(filename, query string, bufSize int) (res []string, err error) { return } -func queryCS(net, host, service string) (res []string, err error) { +func queryCS(ctx context.Context, net, host, service string) (res []string, err error) { switch net { case "tcp4", "tcp6": net = "tcp" @@ -49,15 +51,15 @@ func queryCS(net, host, service string) (res []string, err error) { if host == "" { host = "*" } - return query(netdir+"/cs", net+"!"+host+"!"+service, 128) + return query(ctx, netdir+"/cs", net+"!"+host+"!"+service, 128) } -func queryCS1(net string, ip IP, port int) (clone, dest string, err error) { +func queryCS1(ctx context.Context, net string, ip IP, port int) (clone, dest string, err error) { ips := "*" if len(ip) != 0 && !ip.IsUnspecified() { ips = ip.String() } - lines, err := queryCS(net, ips, itoa(port)) + lines, err := queryCS(ctx, net, ips, itoa(port)) if err != nil { return } @@ -69,8 +71,8 @@ func queryCS1(net string, ip IP, port int) (clone, dest string, err error) { return } -func queryDNS(addr string, typ string) (res []string, err error) { - return query(netdir+"/dns", addr+" "+typ, 1024) +func queryDNS(ctx context.Context, addr string, typ string) (res []string, err error) { + return query(ctx, netdir+"/dns", addr+" "+typ, 1024) } // toLower returns a lower-case version of in. Restricting us to @@ -96,8 +98,8 @@ func toLower(in string) string { // lookupProtocol looks up IP protocol name and returns // the corresponding protocol number. -func lookupProtocol(name string) (proto int, err error) { - lines, err := query(netdir+"/cs", "!protocol="+toLower(name), 128) +func lookupProtocol(ctx context.Context, name string) (proto int, err error) { + lines, err := query(ctx, netdir+"/cs", "!protocol="+toLower(name), 128) if err != nil { return 0, err } @@ -109,17 +111,20 @@ func lookupProtocol(name string) (proto int, err error) { return 0, UnknownNetworkError(name) } s := f[1] - if n, _, ok := dtoi(s, byteIndex(s, '=')+1); ok { + if n, _, ok := dtoi(s[byteIndex(s, '=')+1:]); ok { return n, nil } return 0, UnknownNetworkError(name) } -func lookupHost(host string) (addrs []string, err error) { +func (*Resolver) lookupHost(ctx context.Context, host string) (addrs []string, err error) { // Use netdir/cs instead of netdir/dns because cs knows about // host names in local network (e.g. from /lib/ndb/local) - lines, err := queryCS("net", host, "1") + lines, err := queryCS(ctx, "net", host, "1") if err != nil { + if stringsHasSuffix(err.Error(), "dns failure") { + err = errNoSuchHost + } return } loop: @@ -146,8 +151,8 @@ loop: return } -func lookupIP(host string) (addrs []IPAddr, err error) { - lits, err := LookupHost(host) +func (r *Resolver) lookupIP(ctx context.Context, host string) (addrs []IPAddr, err error) { + lits, err := r.lookupHost(ctx, host) if err != nil { return } @@ -161,14 +166,14 @@ func lookupIP(host string) (addrs []IPAddr, err error) { return } -func lookupPort(network, service string) (port int, err error) { +func (*Resolver) lookupPort(ctx context.Context, network, service string) (port int, err error) { switch network { case "tcp4", "tcp6": network = "tcp" case "udp4", "udp6": network = "udp" } - lines, err := queryCS(network, "127.0.0.1", service) + lines, err := queryCS(ctx, network, "127.0.0.1", toLower(service)) if err != nil { return } @@ -184,15 +189,19 @@ func lookupPort(network, service string) (port int, err error) { if i := byteIndex(s, '!'); i >= 0 { s = s[i+1:] // remove address } - if n, _, ok := dtoi(s, 0); ok { + if n, _, ok := dtoi(s); ok { return n, nil } return 0, unknownPortError } -func lookupCNAME(name string) (cname string, err error) { - lines, err := queryDNS(name, "cname") +func (*Resolver) lookupCNAME(ctx context.Context, name string) (cname string, err error) { + lines, err := queryDNS(ctx, name, "cname") if err != nil { + if stringsHasSuffix(err.Error(), "dns failure") { + cname = name + "." + err = nil + } return } if len(lines) > 0 { @@ -203,14 +212,14 @@ func lookupCNAME(name string) (cname string, err error) { return "", errors.New("bad response from ndb/dns") } -func lookupSRV(service, proto, name string) (cname string, addrs []*SRV, err error) { +func (*Resolver) lookupSRV(ctx context.Context, service, proto, name string) (cname string, addrs []*SRV, err error) { var target string if service == "" && proto == "" { target = name } else { target = "_" + service + "._" + proto + "." + name } - lines, err := queryDNS(target, "srv") + lines, err := queryDNS(ctx, target, "srv") if err != nil { return } @@ -219,9 +228,9 @@ func lookupSRV(service, proto, name string) (cname string, addrs []*SRV, err err if len(f) < 6 { continue } - port, _, portOk := dtoi(f[4], 0) - priority, _, priorityOk := dtoi(f[3], 0) - weight, _, weightOk := dtoi(f[2], 0) + port, _, portOk := dtoi(f[4]) + priority, _, priorityOk := dtoi(f[3]) + weight, _, weightOk := dtoi(f[2]) if !(portOk && priorityOk && weightOk) { continue } @@ -232,8 +241,8 @@ func lookupSRV(service, proto, name string) (cname string, addrs []*SRV, err err return } -func lookupMX(name string) (mx []*MX, err error) { - lines, err := queryDNS(name, "mx") +func (*Resolver) lookupMX(ctx context.Context, name string) (mx []*MX, err error) { + lines, err := queryDNS(ctx, name, "mx") if err != nil { return } @@ -242,7 +251,7 @@ func lookupMX(name string) (mx []*MX, err error) { if len(f) < 4 { continue } - if pref, _, ok := dtoi(f[2], 0); ok { + if pref, _, ok := dtoi(f[2]); ok { mx = append(mx, &MX{absDomainName([]byte(f[3])), uint16(pref)}) } } @@ -250,8 +259,8 @@ func lookupMX(name string) (mx []*MX, err error) { return } -func lookupNS(name string) (ns []*NS, err error) { - lines, err := queryDNS(name, "ns") +func (*Resolver) lookupNS(ctx context.Context, name string) (ns []*NS, err error) { + lines, err := queryDNS(ctx, name, "ns") if err != nil { return } @@ -265,8 +274,8 @@ func lookupNS(name string) (ns []*NS, err error) { return } -func lookupTXT(name string) (txt []string, err error) { - lines, err := queryDNS(name, "txt") +func (*Resolver) lookupTXT(ctx context.Context, name string) (txt []string, err error) { + lines, err := queryDNS(ctx, name, "txt") if err != nil { return } @@ -278,12 +287,12 @@ func lookupTXT(name string) (txt []string, err error) { return } -func lookupAddr(addr string) (name []string, err error) { +func (*Resolver) lookupAddr(ctx context.Context, addr string) (name []string, err error) { arpa, err := reverseaddr(addr) if err != nil { return } - lines, err := queryDNS(arpa, "ptr") + lines, err := queryDNS(ctx, arpa, "ptr") if err != nil { return } diff --git a/libgo/go/net/lookup_stub.go b/libgo/go/net/lookup_stub.go deleted file mode 100644 index 5636198f88..0000000000 --- a/libgo/go/net/lookup_stub.go +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright 2011 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build nacl - -package net - -import "syscall" - -func lookupProtocol(name string) (proto int, err error) { - return 0, syscall.ENOPROTOOPT -} - -func lookupHost(host string) (addrs []string, err error) { - return nil, syscall.ENOPROTOOPT -} - -func lookupIP(host string) (addrs []IPAddr, err error) { - return nil, syscall.ENOPROTOOPT -} - -func lookupPort(network, service string) (port int, err error) { - return 0, syscall.ENOPROTOOPT -} - -func lookupCNAME(name string) (cname string, err error) { - return "", syscall.ENOPROTOOPT -} - -func lookupSRV(service, proto, name string) (cname string, srvs []*SRV, err error) { - return "", nil, syscall.ENOPROTOOPT -} - -func lookupMX(name string) (mxs []*MX, err error) { - return nil, syscall.ENOPROTOOPT -} - -func lookupNS(name string) (nss []*NS, err error) { - return nil, syscall.ENOPROTOOPT -} - -func lookupTXT(name string) (txts []string, err error) { - return nil, syscall.ENOPROTOOPT -} - -func lookupAddr(addr string) (ptrs []string, err error) { - return nil, syscall.ENOPROTOOPT -} diff --git a/libgo/go/net/lookup_test.go b/libgo/go/net/lookup_test.go index 439496ac81..36db56acd0 100644 --- a/libgo/go/net/lookup_test.go +++ b/libgo/go/net/lookup_test.go @@ -6,6 +6,7 @@ package net import ( "bytes" + "context" "fmt" "internal/testenv" "runtime" @@ -14,7 +15,7 @@ import ( "time" ) -func lookupLocalhost(fn func(string) ([]IPAddr, error), host string) ([]IPAddr, error) { +func lookupLocalhost(ctx context.Context, fn func(context.Context, string) ([]IPAddr, error), host string) ([]IPAddr, error) { switch host { case "localhost": return []IPAddr{ @@ -22,7 +23,7 @@ func lookupLocalhost(fn func(string) ([]IPAddr, error), host string) ([]IPAddr, {IP: IPv6loopback}, }, nil default: - return fn(host) + return fn(ctx, host) } } @@ -58,9 +59,10 @@ var lookupGoogleSRVTests = []struct { } func TestLookupGoogleSRV(t *testing.T) { - if testing.Short() && testenv.Builder() == "" || !*testExternal { - t.Skip("avoid external network") + if testenv.Builder() == "" { + testenv.MustHaveExternalNetwork(t) } + if !supportsIPv4 || !*testIPv4 { t.Skip("IPv4 is required") } @@ -68,6 +70,7 @@ func TestLookupGoogleSRV(t *testing.T) { for _, tt := range lookupGoogleSRVTests { cname, srvs, err := LookupSRV(tt.service, tt.proto, tt.name) if err != nil { + testenv.SkipFlakyNet(t) t.Fatal(err) } if len(srvs) == 0 { @@ -92,9 +95,10 @@ var lookupGmailMXTests = []struct { } func TestLookupGmailMX(t *testing.T) { - if testing.Short() && testenv.Builder() == "" || !*testExternal { - t.Skip("avoid external network") + if testenv.Builder() == "" { + testenv.MustHaveExternalNetwork(t) } + if !supportsIPv4 || !*testIPv4 { t.Skip("IPv4 is required") } @@ -123,9 +127,10 @@ var lookupGmailNSTests = []struct { } func TestLookupGmailNS(t *testing.T) { - if testing.Short() && testenv.Builder() == "" || !*testExternal { - t.Skip("avoid external network") + if testenv.Builder() == "" { + testenv.MustHaveExternalNetwork(t) } + if !supportsIPv4 || !*testIPv4 { t.Skip("IPv4 is required") } @@ -133,6 +138,7 @@ func TestLookupGmailNS(t *testing.T) { for _, tt := range lookupGmailNSTests { nss, err := LookupNS(tt.name) if err != nil { + testenv.SkipFlakyNet(t) t.Fatal(err) } if len(nss) == 0 { @@ -154,9 +160,10 @@ var lookupGmailTXTTests = []struct { } func TestLookupGmailTXT(t *testing.T) { - if testing.Short() && testenv.Builder() == "" || !*testExternal { - t.Skip("avoid external network") + if testenv.Builder() == "" { + testenv.MustHaveExternalNetwork(t) } + if !supportsIPv4 || !*testIPv4 { t.Skip("IPv4 is required") } @@ -188,9 +195,10 @@ var lookupGooglePublicDNSAddrTests = []struct { } func TestLookupGooglePublicDNSAddr(t *testing.T) { - if testing.Short() && testenv.Builder() == "" || !*testExternal { - t.Skip("avoid external network") + if testenv.Builder() == "" { + testenv.MustHaveExternalNetwork(t) } + if !supportsIPv4 || !supportsIPv6 || !*testIPv4 || !*testIPv6 { t.Skip("both IPv4 and IPv6 are required") } @@ -235,22 +243,24 @@ func TestLookupIPv6LinkLocalAddr(t *testing.T) { } } -var lookupIANACNAMETests = []struct { +var lookupCNAMETests = []struct { name, cname string }{ {"www.iana.org", "icann.org."}, {"www.iana.org.", "icann.org."}, + {"www.google.com", "google.com."}, } -func TestLookupIANACNAME(t *testing.T) { - if testing.Short() && testenv.Builder() == "" || !*testExternal { - t.Skip("avoid external network") +func TestLookupCNAME(t *testing.T) { + if testenv.Builder() == "" { + testenv.MustHaveExternalNetwork(t) } + if !supportsIPv4 || !*testIPv4 { t.Skip("IPv4 is required") } - for _, tt := range lookupIANACNAMETests { + for _, tt := range lookupCNAMETests { cname, err := LookupCNAME(tt.name) if err != nil { t.Fatal(err) @@ -269,9 +279,10 @@ var lookupGoogleHostTests = []struct { } func TestLookupGoogleHost(t *testing.T) { - if testing.Short() && testenv.Builder() == "" || !*testExternal { - t.Skip("avoid external network") + if testenv.Builder() == "" { + testenv.MustHaveExternalNetwork(t) } + if !supportsIPv4 || !*testIPv4 { t.Skip("IPv4 is required") } @@ -300,9 +311,10 @@ var lookupGoogleIPTests = []struct { } func TestLookupGoogleIP(t *testing.T) { - if testing.Short() && testenv.Builder() == "" || !*testExternal { - t.Skip("avoid external network") + if testenv.Builder() == "" { + testenv.MustHaveExternalNetwork(t) } + if !supportsIPv4 || !*testIPv4 { t.Skip("IPv4 is required") } @@ -360,22 +372,38 @@ func TestReverseAddress(t *testing.T) { } } -func TestLookupIPDeadline(t *testing.T) { +func TestDNSFlood(t *testing.T) { if !*testDNSFlood { t.Skip("test disabled; use -dnsflood to enable") } - const N = 5000 + var N = 5000 + if runtime.GOOS == "darwin" { + // On Darwin this test consumes kernel threads much + // than other platforms for some reason. + // When we monitor the number of allocated Ms by + // observing on runtime.newm calls, we can see that it + // easily reaches the per process ceiling + // kern.num_threads when CGO_ENABLED=1 and + // GODEBUG=netdns=go. + N = 500 + } + const timeout = 3 * time.Second + ctxHalfTimeout, cancel := context.WithTimeout(context.Background(), timeout/2) + defer cancel() + ctxTimeout, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + c := make(chan error, 2*N) for i := 0; i < N; i++ { name := fmt.Sprintf("%d.net-test.golang.org", i) go func() { - _, err := lookupIPDeadline(name, time.Now().Add(timeout/2)) + _, err := DefaultResolver.LookupIPAddr(ctxHalfTimeout, name) c <- err }() go func() { - _, err := lookupIPDeadline(name, time.Now().Add(timeout)) + _, err := DefaultResolver.LookupIPAddr(ctxTimeout, name) c <- err }() } @@ -426,6 +454,10 @@ func TestLookupDotsWithLocalSource(t *testing.T) { t.Skip("IPv4 is required") } + if testenv.Builder() == "" { + testenv.MustHaveExternalNetwork(t) + } + for i, fn := range []func() func(){forceGoDNS, forceCgoDNS} { fixup := fn() if fixup == nil { @@ -463,9 +495,10 @@ func TestLookupDotsWithLocalSource(t *testing.T) { } func TestLookupDotsWithRemoteSource(t *testing.T) { - if testing.Short() && testenv.Builder() == "" || !*testExternal { - t.Skip("avoid external network") + if testenv.Builder() == "" { + testenv.MustHaveExternalNetwork(t) } + if !supportsIPv4 || !*testIPv4 { t.Skip("IPv4 is required") } @@ -483,6 +516,7 @@ func TestLookupDotsWithRemoteSource(t *testing.T) { func testDots(t *testing.T, mode string) { names, err := LookupAddr("8.8.8.8") // Google dns server if err != nil { + testenv.SkipFlakyNet(t) t.Errorf("LookupAddr(8.8.8.8): %v (mode=%v)", err, mode) } else { for _, name := range names { @@ -494,12 +528,16 @@ func testDots(t *testing.T, mode string) { } cname, err := LookupCNAME("www.mit.edu") - if err != nil || !strings.HasSuffix(cname, ".") { - t.Errorf("LookupCNAME(www.mit.edu) = %v, %v, want cname ending in . with trailing dot (mode=%v)", cname, err, mode) + if err != nil { + testenv.SkipFlakyNet(t) + t.Errorf("LookupCNAME(www.mit.edu, mode=%v): %v", mode, err) + } else if !strings.HasSuffix(cname, ".") { + t.Errorf("LookupCNAME(www.mit.edu) = %v, want cname ending in . with trailing dot (mode=%v)", cname, mode) } mxs, err := LookupMX("google.com") if err != nil { + testenv.SkipFlakyNet(t) t.Errorf("LookupMX(google.com): %v (mode=%v)", err, mode) } else { for _, mx := range mxs { @@ -512,6 +550,7 @@ func testDots(t *testing.T, mode string) { nss, err := LookupNS("google.com") if err != nil { + testenv.SkipFlakyNet(t) t.Errorf("LookupNS(google.com): %v (mode=%v)", err, mode) } else { for _, ns := range nss { @@ -524,6 +563,7 @@ func testDots(t *testing.T, mode string) { cname, srvs, err := LookupSRV("xmpp-server", "tcp", "google.com") if err != nil { + testenv.SkipFlakyNet(t) t.Errorf("LookupSRV(xmpp-server, tcp, google.com): %v (mode=%v)", err, mode) } else { if !strings.HasSuffix(cname, ".google.com.") { @@ -574,61 +614,128 @@ func srvString(srvs []*SRV) string { return buf.String() } -var lookupPortTests = []struct { - network string - name string - port int - ok bool -}{ - {"tcp", "0", 0, true}, - {"tcp", "echo", 7, true}, - {"tcp", "discard", 9, true}, - {"tcp", "systat", 11, true}, - {"tcp", "daytime", 13, true}, - {"tcp", "chargen", 19, true}, - {"tcp", "ftp-data", 20, true}, - {"tcp", "ftp", 21, true}, - {"tcp", "telnet", 23, true}, - {"tcp", "smtp", 25, true}, - {"tcp", "time", 37, true}, - {"tcp", "domain", 53, true}, - {"tcp", "finger", 79, true}, - {"tcp", "42", 42, true}, - - {"udp", "0", 0, true}, - {"udp", "echo", 7, true}, - {"udp", "tftp", 69, true}, - {"udp", "bootpc", 68, true}, - {"udp", "bootps", 67, true}, - {"udp", "domain", 53, true}, - {"udp", "ntp", 123, true}, - {"udp", "snmp", 161, true}, - {"udp", "syslog", 514, true}, - {"udp", "42", 42, true}, - - {"--badnet--", "zzz", 0, false}, - {"tcp", "--badport--", 0, false}, - {"tcp", "-1", 0, false}, - {"tcp", "65536", 0, false}, - {"udp", "-1", 0, false}, - {"udp", "65536", 0, false}, - - // Issue 13610: LookupPort("tcp", "") - {"tcp", "", 0, true}, - {"tcp6", "", 0, true}, - {"tcp4", "", 0, true}, - {"udp", "", 0, true}, -} - func TestLookupPort(t *testing.T) { + // See http://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.xhtml + // + // Please be careful about adding new test cases. + // There are platforms having incomplete mappings for + // restricted resource access and security reasons. + type test struct { + network string + name string + port int + ok bool + } + var tests = []test{ + {"tcp", "0", 0, true}, + {"udp", "0", 0, true}, + {"udp", "domain", 53, true}, + + {"--badnet--", "zzz", 0, false}, + {"tcp", "--badport--", 0, false}, + {"tcp", "-1", 0, false}, + {"tcp", "65536", 0, false}, + {"udp", "-1", 0, false}, + {"udp", "65536", 0, false}, + {"tcp", "123456789", 0, false}, + + // Issue 13610: LookupPort("tcp", "") + {"tcp", "", 0, true}, + {"tcp4", "", 0, true}, + {"tcp6", "", 0, true}, + {"udp", "", 0, true}, + {"udp4", "", 0, true}, + {"udp6", "", 0, true}, + } + switch runtime.GOOS { - case "nacl": - t.Skipf("not supported on %s", runtime.GOOS) + case "android": + if netGo { + t.Skipf("not supported on %s without cgo; see golang.org/issues/14576", runtime.GOOS) + } + default: + tests = append(tests, test{"tcp", "http", 80, true}) } - for _, tt := range lookupPortTests { - if port, err := LookupPort(tt.network, tt.name); port != tt.port || (err == nil) != tt.ok { - t.Errorf("LookupPort(%q, %q) = %d, %v; want %d", tt.network, tt.name, port, err, tt.port) + for _, tt := range tests { + port, err := LookupPort(tt.network, tt.name) + if port != tt.port || (err == nil) != tt.ok { + t.Errorf("LookupPort(%q, %q) = %d, %v; want %d, error=%t", tt.network, tt.name, port, err, tt.port, !tt.ok) } + if err != nil { + if perr := parseLookupPortError(err); perr != nil { + t.Error(perr) + } + } + } +} + +// Like TestLookupPort but with minimal tests that should always pass +// because the answers are baked-in to the net package. +func TestLookupPort_Minimal(t *testing.T) { + type test struct { + network string + name string + port int + } + var tests = []test{ + {"tcp", "http", 80}, + {"tcp", "HTTP", 80}, // case shouldn't matter + {"tcp", "https", 443}, + {"tcp", "ssh", 22}, + {"tcp", "gopher", 70}, + {"tcp4", "http", 80}, + {"tcp6", "http", 80}, + } + + for _, tt := range tests { + port, err := LookupPort(tt.network, tt.name) + if port != tt.port || err != nil { + t.Errorf("LookupPort(%q, %q) = %d, %v; want %d, error=nil", tt.network, tt.name, port, err, tt.port) + } + } +} + +func TestLookupProtocol_Minimal(t *testing.T) { + type test struct { + name string + want int + } + var tests = []test{ + {"tcp", 6}, + {"TcP", 6}, // case shouldn't matter + {"icmp", 1}, + {"igmp", 2}, + {"udp", 17}, + {"ipv6-icmp", 58}, + } + + for _, tt := range tests { + got, err := lookupProtocol(context.Background(), tt.name) + if got != tt.want || err != nil { + t.Errorf("LookupProtocol(%q) = %d, %v; want %d, error=nil", tt.name, got, err, tt.want) + } + } + +} + +func TestLookupNonLDH(t *testing.T) { + if runtime.GOOS == "nacl" { + t.Skip("skip on nacl") + } + if fixup := forceGoDNS(); fixup != nil { + defer fixup() + } + + // "LDH" stands for letters, digits, and hyphens and is the usual + // description of standard DNS names. + // This test is checking that other kinds of names are reported + // as not found, not reported as invalid names. + addrs, err := LookupHost("!!!.###.bogus..domain.") + if err == nil { + t.Fatalf("lookup succeeded: %v", addrs) + } + if !strings.HasSuffix(err.Error(), errNoSuchHost.Error()) { + t.Fatalf("lookup error = %v, want %v", err, errNoSuchHost) } } diff --git a/libgo/go/net/lookup_unix.go b/libgo/go/net/lookup_unix.go index a64da8bcb5..be2ced9c39 100644 --- a/libgo/go/net/lookup_unix.go +++ b/libgo/go/net/lookup_unix.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -6,7 +6,10 @@ package net -import "sync" +import ( + "context" + "sync" +) var onceReadProtocols sync.Once @@ -23,7 +26,7 @@ func readProtocols() { if len(f) < 2 { continue } - if proto, _, ok := dtoi(f[1], 0); ok { + if proto, _, ok := dtoi(f[1]); ok { if _, ok := protocols[f[0]]; !ok { protocols[f[0]] = proto } @@ -40,65 +43,72 @@ func readProtocols() { // lookupProtocol looks up IP protocol name in /etc/protocols and // returns correspondent protocol number. -func lookupProtocol(name string) (int, error) { +func lookupProtocol(_ context.Context, name string) (int, error) { onceReadProtocols.Do(readProtocols) - proto, found := protocols[name] - if !found { - return 0, &AddrError{Err: "unknown IP protocol specified", Addr: name} - } - return proto, nil + return lookupProtocolMap(name) } -func lookupHost(host string) (addrs []string, err error) { +func (r *Resolver) lookupHost(ctx context.Context, host string) (addrs []string, err error) { order := systemConf().hostLookupOrder(host) - if order == hostLookupCgo { - if addrs, err, ok := cgoLookupHost(host); ok { + if !r.PreferGo && order == hostLookupCgo { + if addrs, err, ok := cgoLookupHost(ctx, host); ok { return addrs, err } // cgo not available (or netgo); fall back to Go's DNS resolver order = hostLookupFilesDNS } - return goLookupHostOrder(host, order) + return goLookupHostOrder(ctx, host, order) } -func lookupIP(host string) (addrs []IPAddr, err error) { +func (r *Resolver) lookupIP(ctx context.Context, host string) (addrs []IPAddr, err error) { + if r.PreferGo { + return goLookupIP(ctx, host) + } order := systemConf().hostLookupOrder(host) if order == hostLookupCgo { - if addrs, err, ok := cgoLookupIP(host); ok { + if addrs, err, ok := cgoLookupIP(ctx, host); ok { return addrs, err } // cgo not available (or netgo); fall back to Go's DNS resolver order = hostLookupFilesDNS } - return goLookupIPOrder(host, order) + addrs, _, err = goLookupIPCNAMEOrder(ctx, host, order) + return } -func lookupPort(network, service string) (int, error) { - if systemConf().canUseCgo() { - if port, err, ok := cgoLookupPort(network, service); ok { +func (r *Resolver) lookupPort(ctx context.Context, network, service string) (int, error) { + if !r.PreferGo && systemConf().canUseCgo() { + if port, err, ok := cgoLookupPort(ctx, network, service); ok { + if err != nil { + // Issue 18213: if cgo fails, first check to see whether we + // have the answer baked-in to the net package. + if port, err := goLookupPort(network, service); err == nil { + return port, nil + } + } return port, err } } return goLookupPort(network, service) } -func lookupCNAME(name string) (string, error) { - if systemConf().canUseCgo() { - if cname, err, ok := cgoLookupCNAME(name); ok { +func (r *Resolver) lookupCNAME(ctx context.Context, name string) (string, error) { + if !r.PreferGo && systemConf().canUseCgo() { + if cname, err, ok := cgoLookupCNAME(ctx, name); ok { return cname, err } } - return goLookupCNAME(name) + return goLookupCNAME(ctx, name) } -func lookupSRV(service, proto, name string) (string, []*SRV, error) { +func (*Resolver) lookupSRV(ctx context.Context, service, proto, name string) (string, []*SRV, error) { var target string if service == "" && proto == "" { target = name } else { target = "_" + service + "._" + proto + "." + name } - cname, rrs, err := lookup(target, dnsTypeSRV) + cname, rrs, err := lookup(ctx, target, dnsTypeSRV) if err != nil { return "", nil, err } @@ -111,8 +121,8 @@ func lookupSRV(service, proto, name string) (string, []*SRV, error) { return cname, srvs, nil } -func lookupMX(name string) ([]*MX, error) { - _, rrs, err := lookup(name, dnsTypeMX) +func (*Resolver) lookupMX(ctx context.Context, name string) ([]*MX, error) { + _, rrs, err := lookup(ctx, name, dnsTypeMX) if err != nil { return nil, err } @@ -125,8 +135,8 @@ func lookupMX(name string) ([]*MX, error) { return mxs, nil } -func lookupNS(name string) ([]*NS, error) { - _, rrs, err := lookup(name, dnsTypeNS) +func (*Resolver) lookupNS(ctx context.Context, name string) ([]*NS, error) { + _, rrs, err := lookup(ctx, name, dnsTypeNS) if err != nil { return nil, err } @@ -137,8 +147,8 @@ func lookupNS(name string) ([]*NS, error) { return nss, nil } -func lookupTXT(name string) ([]string, error) { - _, rrs, err := lookup(name, dnsTypeTXT) +func (r *Resolver) lookupTXT(ctx context.Context, name string) ([]string, error) { + _, rrs, err := lookup(ctx, name, dnsTypeTXT) if err != nil { return nil, err } @@ -149,11 +159,11 @@ func lookupTXT(name string) ([]string, error) { return txts, nil } -func lookupAddr(addr string) ([]string, error) { - if systemConf().canUseCgo() { - if ptrs, err, ok := cgoLookupPTR(addr); ok { +func (r *Resolver) lookupAddr(ctx context.Context, addr string) ([]string, error) { + if !r.PreferGo && systemConf().canUseCgo() { + if ptrs, err, ok := cgoLookupPTR(ctx, addr); ok { return ptrs, err } } - return goLookupPTR(addr) + return goLookupPTR(ctx, addr) } diff --git a/libgo/go/net/lookup_windows.go b/libgo/go/net/lookup_windows.go index 13edc264e8..5808293058 100644 --- a/libgo/go/net/lookup_windows.go +++ b/libgo/go/net/lookup_windows.go @@ -5,54 +5,67 @@ package net import ( + "context" "os" "runtime" "syscall" "unsafe" ) -var ( - lookupPort = oldLookupPort - lookupIP = oldLookupIP -) +const _WSAHOST_NOT_FOUND = syscall.Errno(11001) + +func winError(call string, err error) error { + switch err { + case _WSAHOST_NOT_FOUND: + return errNoSuchHost + } + return os.NewSyscallError(call, err) +} func getprotobyname(name string) (proto int, err error) { p, err := syscall.GetProtoByName(name) if err != nil { - return 0, os.NewSyscallError("getprotobyname", err) + return 0, winError("getprotobyname", err) } return int(p.Proto), nil } // lookupProtocol looks up IP protocol name and returns correspondent protocol number. -func lookupProtocol(name string) (int, error) { +func lookupProtocol(ctx context.Context, name string) (int, error) { // GetProtoByName return value is stored in thread local storage. // Start new os thread before the call to prevent races. type result struct { proto int err error } - ch := make(chan result) + ch := make(chan result) // unbuffered go func() { acquireThread() defer releaseThread() runtime.LockOSThread() defer runtime.UnlockOSThread() proto, err := getprotobyname(name) - ch <- result{proto: proto, err: err} + select { + case ch <- result{proto: proto, err: err}: + case <-ctx.Done(): + } }() - r := <-ch - if r.err != nil { - if proto, ok := protocols[name]; ok { - return proto, nil + select { + case r := <-ch: + if r.err != nil { + if proto, err := lookupProtocolMap(name); err == nil { + return proto, nil + } + r.err = &DNSError{Err: r.err.Error(), Name: name} } - r.err = &DNSError{Err: r.err.Error(), Name: name} + return r.proto, r.err + case <-ctx.Done(): + return 0, mapErr(ctx.Err()) } - return r.proto, r.err } -func lookupHost(name string) ([]string, error) { - ips, err := LookupIP(name) +func (r *Resolver) lookupHost(ctx context.Context, name string) ([]string, error) { + ips, err := r.lookupIP(ctx, name) if err != nil { return nil, err } @@ -63,121 +76,71 @@ func lookupHost(name string) ([]string, error) { return addrs, nil } -func gethostbyname(name string) (addrs []IPAddr, err error) { - // caller already acquired thread - h, err := syscall.GetHostByName(name) - if err != nil { - return nil, os.NewSyscallError("gethostbyname", err) - } - switch h.AddrType { - case syscall.AF_INET: - i := 0 - addrs = make([]IPAddr, 100) // plenty of room to grow - for p := (*[100](*[4]byte))(unsafe.Pointer(h.AddrList)); i < cap(addrs) && p[i] != nil; i++ { - addrs[i] = IPAddr{IP: IPv4(p[i][0], p[i][1], p[i][2], p[i][3])} - } - addrs = addrs[0:i] - default: // TODO(vcc): Implement non IPv4 address lookups. - return nil, syscall.EWINDOWS - } - return addrs, nil -} +func (r *Resolver) lookupIP(ctx context.Context, name string) ([]IPAddr, error) { + // TODO(bradfitz,brainman): use ctx more. See TODO below. -func oldLookupIP(name string) ([]IPAddr, error) { - // GetHostByName return value is stored in thread local storage. - // Start new os thread before the call to prevent races. - type result struct { + type ret struct { addrs []IPAddr err error } - ch := make(chan result) + ch := make(chan ret, 1) go func() { acquireThread() defer releaseThread() - runtime.LockOSThread() - defer runtime.UnlockOSThread() - addrs, err := gethostbyname(name) - ch <- result{addrs: addrs, err: err} + hints := syscall.AddrinfoW{ + Family: syscall.AF_UNSPEC, + Socktype: syscall.SOCK_STREAM, + Protocol: syscall.IPPROTO_IP, + } + var result *syscall.AddrinfoW + e := syscall.GetAddrInfoW(syscall.StringToUTF16Ptr(name), nil, &hints, &result) + if e != nil { + ch <- ret{err: &DNSError{Err: winError("getaddrinfow", e).Error(), Name: name}} + } + defer syscall.FreeAddrInfoW(result) + addrs := make([]IPAddr, 0, 5) + for ; result != nil; result = result.Next { + addr := unsafe.Pointer(result.Addr) + switch result.Family { + case syscall.AF_INET: + a := (*syscall.RawSockaddrInet4)(addr).Addr + addrs = append(addrs, IPAddr{IP: IPv4(a[0], a[1], a[2], a[3])}) + case syscall.AF_INET6: + a := (*syscall.RawSockaddrInet6)(addr).Addr + zone := zoneToString(int((*syscall.RawSockaddrInet6)(addr).Scope_id)) + addrs = append(addrs, IPAddr{IP: IP{a[0], a[1], a[2], a[3], a[4], a[5], a[6], a[7], a[8], a[9], a[10], a[11], a[12], a[13], a[14], a[15]}, Zone: zone}) + default: + ch <- ret{err: &DNSError{Err: syscall.EWINDOWS.Error(), Name: name}} + } + } + ch <- ret{addrs: addrs} }() - r := <-ch - if r.err != nil { - r.err = &DNSError{Err: r.err.Error(), Name: name} - } - return r.addrs, r.err -} - -func newLookupIP(name string) ([]IPAddr, error) { - acquireThread() - defer releaseThread() - hints := syscall.AddrinfoW{ - Family: syscall.AF_UNSPEC, - Socktype: syscall.SOCK_STREAM, - Protocol: syscall.IPPROTO_IP, - } - var result *syscall.AddrinfoW - e := syscall.GetAddrInfoW(syscall.StringToUTF16Ptr(name), nil, &hints, &result) - if e != nil { - return nil, &DNSError{Err: os.NewSyscallError("getaddrinfow", e).Error(), Name: name} - } - defer syscall.FreeAddrInfoW(result) - addrs := make([]IPAddr, 0, 5) - for ; result != nil; result = result.Next { - addr := unsafe.Pointer(result.Addr) - switch result.Family { - case syscall.AF_INET: - a := (*syscall.RawSockaddrInet4)(addr).Addr - addrs = append(addrs, IPAddr{IP: IPv4(a[0], a[1], a[2], a[3])}) - case syscall.AF_INET6: - a := (*syscall.RawSockaddrInet6)(addr).Addr - zone := zoneToString(int((*syscall.RawSockaddrInet6)(addr).Scope_id)) - addrs = append(addrs, IPAddr{IP: IP{a[0], a[1], a[2], a[3], a[4], a[5], a[6], a[7], a[8], a[9], a[10], a[11], a[12], a[13], a[14], a[15]}, Zone: zone}) - default: - return nil, &DNSError{Err: syscall.EWINDOWS.Error(), Name: name} + select { + case r := <-ch: + return r.addrs, r.err + case <-ctx.Done(): + // TODO(bradfitz,brainman): cancel the ongoing + // GetAddrInfoW? It would require conditionally using + // GetAddrInfoEx with lpOverlapped, which requires + // Windows 8 or newer. I guess we'll need oldLookupIP, + // newLookupIP, and newerLookUP. + // + // For now we just let it finish and write to the + // buffered channel. + return nil, &DNSError{ + Name: name, + Err: ctx.Err().Error(), + IsTimeout: ctx.Err() == context.DeadlineExceeded, } } - return addrs, nil -} - -func getservbyname(network, service string) (int, error) { - acquireThread() - defer releaseThread() - switch network { - case "tcp4", "tcp6": - network = "tcp" - case "udp4", "udp6": - network = "udp" - } - s, err := syscall.GetServByName(service, network) - if err != nil { - return 0, os.NewSyscallError("getservbyname", err) - } - return int(syscall.Ntohs(s.Port)), nil } -func oldLookupPort(network, service string) (int, error) { - // GetServByName return value is stored in thread local storage. - // Start new os thread before the call to prevent races. - type result struct { - port int - err error - } - ch := make(chan result) - go func() { - acquireThread() - defer releaseThread() - runtime.LockOSThread() - defer runtime.UnlockOSThread() - port, err := getservbyname(network, service) - ch <- result{port: port, err: err} - }() - r := <-ch - if r.err != nil { - r.err = &DNSError{Err: r.err.Error(), Name: network + "/" + service} +func (r *Resolver) lookupPort(ctx context.Context, network, service string) (int, error) { + if r.PreferGo { + return lookupPortMap(network, service) } - return r.port, r.err -} -func newLookupPort(network, service string) (int, error) { + // TODO(bradfitz): finish ctx plumbing. Nothing currently depends on this. acquireThread() defer releaseThread() var stype int32 @@ -195,7 +158,10 @@ func newLookupPort(network, service string) (int, error) { var result *syscall.AddrinfoW e := syscall.GetAddrInfoW(nil, syscall.StringToUTF16Ptr(service), &hints, &result) if e != nil { - return 0, &DNSError{Err: os.NewSyscallError("getaddrinfow", e).Error(), Name: network + "/" + service} + if port, err := lookupPortMap(network, service); err == nil { + return port, nil + } + return 0, &DNSError{Err: winError("getaddrinfow", e).Error(), Name: network + "/" + service} } defer syscall.FreeAddrInfoW(result) if result == nil { @@ -213,7 +179,8 @@ func newLookupPort(network, service string) (int, error) { return 0, &DNSError{Err: syscall.EINVAL.Error(), Name: network + "/" + service} } -func lookupCNAME(name string) (string, error) { +func (*Resolver) lookupCNAME(ctx context.Context, name string) (string, error) { + // TODO(bradfitz): finish ctx plumbing. Nothing currently depends on this. acquireThread() defer releaseThread() var r *syscall.DNSRecord @@ -224,7 +191,7 @@ func lookupCNAME(name string) (string, error) { return absDomainName([]byte(name)), nil } if e != nil { - return "", &DNSError{Err: os.NewSyscallError("dnsquery", e).Error(), Name: name} + return "", &DNSError{Err: winError("dnsquery", e).Error(), Name: name} } defer syscall.DnsRecordListFree(r, 1) @@ -233,7 +200,8 @@ func lookupCNAME(name string) (string, error) { return absDomainName([]byte(cname)), nil } -func lookupSRV(service, proto, name string) (string, []*SRV, error) { +func (*Resolver) lookupSRV(ctx context.Context, service, proto, name string) (string, []*SRV, error) { + // TODO(bradfitz): finish ctx plumbing. Nothing currently depends on this. acquireThread() defer releaseThread() var target string @@ -245,7 +213,7 @@ func lookupSRV(service, proto, name string) (string, []*SRV, error) { var r *syscall.DNSRecord e := syscall.DnsQuery(target, syscall.DNS_TYPE_SRV, 0, nil, &r, nil) if e != nil { - return "", nil, &DNSError{Err: os.NewSyscallError("dnsquery", e).Error(), Name: target} + return "", nil, &DNSError{Err: winError("dnsquery", e).Error(), Name: target} } defer syscall.DnsRecordListFree(r, 1) @@ -258,13 +226,14 @@ func lookupSRV(service, proto, name string) (string, []*SRV, error) { return absDomainName([]byte(target)), srvs, nil } -func lookupMX(name string) ([]*MX, error) { +func (*Resolver) lookupMX(ctx context.Context, name string) ([]*MX, error) { + // TODO(bradfitz): finish ctx plumbing. Nothing currently depends on this. acquireThread() defer releaseThread() var r *syscall.DNSRecord e := syscall.DnsQuery(name, syscall.DNS_TYPE_MX, 0, nil, &r, nil) if e != nil { - return nil, &DNSError{Err: os.NewSyscallError("dnsquery", e).Error(), Name: name} + return nil, &DNSError{Err: winError("dnsquery", e).Error(), Name: name} } defer syscall.DnsRecordListFree(r, 1) @@ -277,13 +246,14 @@ func lookupMX(name string) ([]*MX, error) { return mxs, nil } -func lookupNS(name string) ([]*NS, error) { +func (*Resolver) lookupNS(ctx context.Context, name string) ([]*NS, error) { + // TODO(bradfitz): finish ctx plumbing. Nothing currently depends on this. acquireThread() defer releaseThread() var r *syscall.DNSRecord e := syscall.DnsQuery(name, syscall.DNS_TYPE_NS, 0, nil, &r, nil) if e != nil { - return nil, &DNSError{Err: os.NewSyscallError("dnsquery", e).Error(), Name: name} + return nil, &DNSError{Err: winError("dnsquery", e).Error(), Name: name} } defer syscall.DnsRecordListFree(r, 1) @@ -295,13 +265,14 @@ func lookupNS(name string) ([]*NS, error) { return nss, nil } -func lookupTXT(name string) ([]string, error) { +func (*Resolver) lookupTXT(ctx context.Context, name string) ([]string, error) { + // TODO(bradfitz): finish ctx plumbing. Nothing currently depends on this. acquireThread() defer releaseThread() var r *syscall.DNSRecord e := syscall.DnsQuery(name, syscall.DNS_TYPE_TEXT, 0, nil, &r, nil) if e != nil { - return nil, &DNSError{Err: os.NewSyscallError("dnsquery", e).Error(), Name: name} + return nil, &DNSError{Err: winError("dnsquery", e).Error(), Name: name} } defer syscall.DnsRecordListFree(r, 1) @@ -316,7 +287,8 @@ func lookupTXT(name string) ([]string, error) { return txts, nil } -func lookupAddr(addr string) ([]string, error) { +func (*Resolver) lookupAddr(ctx context.Context, addr string) ([]string, error) { + // TODO(bradfitz): finish ctx plumbing. Nothing currently depends on this. acquireThread() defer releaseThread() arpa, err := reverseaddr(addr) @@ -326,7 +298,7 @@ func lookupAddr(addr string) ([]string, error) { var r *syscall.DNSRecord e := syscall.DnsQuery(arpa, syscall.DNS_TYPE_PTR, 0, nil, &r, nil) if e != nil { - return nil, &DNSError{Err: os.NewSyscallError("dnsquery", e).Error(), Name: addr} + return nil, &DNSError{Err: winError("dnsquery", e).Error(), Name: addr} } defer syscall.DnsRecordListFree(r, 1) diff --git a/libgo/go/net/mac.go b/libgo/go/net/mac.go index 93f0b09121..f3b1694735 100644 --- a/libgo/go/net/mac.go +++ b/libgo/go/net/mac.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/mac_test.go b/libgo/go/net/mac_test.go index 1ec6b287ac..2630d19047 100644 --- a/libgo/go/net/mac_test.go +++ b/libgo/go/net/mac_test.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/mail/message.go b/libgo/go/net/mail/message.go index 923630c49c..702b765c34 100644 --- a/libgo/go/net/mail/message.go +++ b/libgo/go/net/mail/message.go @@ -5,13 +5,15 @@ /* Package mail implements parsing of mail messages. -For the most part, this package follows the syntax as specified by RFC 5322. +For the most part, this package follows the syntax as specified by RFC 5322 and +extended by RFC 6532. Notable divergences: * Obsolete address formats are not parsed, including addresses with embedded route information. * Group addresses are not parsed. * The full range of spacing (the CFWS syntax element) is not supported, such as breaking addresses across lines. + * No unicode normalization is performed. */ package mail @@ -26,6 +28,7 @@ import ( "net/textproto" "strings" "time" + "unicode/utf8" ) var debug = debugT(false) @@ -89,7 +92,8 @@ func init() { } } -func parseDate(date string) (time.Time, error) { +// ParseDate parses an RFC 5322 date string. +func ParseDate(date string) (time.Time, error) { for _, layout := range dateLayouts { t, err := time.Parse(layout, date) if err == nil { @@ -103,7 +107,11 @@ func parseDate(date string) (time.Time, error) { type Header map[string][]string // Get gets the first value associated with the given key. +// It is case insensitive; CanonicalMIMEHeaderKey is used +// to canonicalize the provided key. // If there are no values associated with the key, Get returns "". +// To access multiple values of a key, or to use non-canonical keys, +// access the map directly. func (h Header) Get(key string) string { return textproto.MIMEHeader(h).Get(key) } @@ -116,7 +124,7 @@ func (h Header) Date() (time.Time, error) { if hdr == "" { return time.Time{}, ErrHeaderNotPresent } - return parseDate(hdr) + return ParseDate(hdr) } // AddressList parses the named header field as a list of addresses. @@ -138,7 +146,7 @@ type Address struct { // Parses a single RFC 5322 address, e.g. "Barry Gibbs <bg@example.com>" func ParseAddress(address string) (*Address, error) { - return (&addrParser{s: address}).parseAddress() + return (&addrParser{s: address}).parseSingleAddress() } // ParseAddressList parses the given string as a list of addresses. @@ -155,7 +163,7 @@ type AddressParser struct { // Parse parses a single RFC 5322 address of the // form "Gogh Fir <gf@example.com>" or "foo@example.com". func (p *AddressParser) Parse(address string) (*Address, error) { - return (&addrParser{s: address, dec: p.WordDecoder}).parseAddress() + return (&addrParser{s: address, dec: p.WordDecoder}).parseSingleAddress() } // ParseList parses the given string as a list of comma-separated addresses @@ -168,7 +176,6 @@ func (p *AddressParser) ParseList(list string) ([]*Address, error) { // If the address's name contains non-ASCII characters // the name will be rendered according to RFC 2047. func (a *Address) String() string { - // Format address local@domain at := strings.LastIndex(a.Address, "@") var local, domain string @@ -181,15 +188,12 @@ func (a *Address) String() string { } // Add quotes if needed - // TODO: rendering quoted local part and rendering printable name - // should be merged in helper function. quoteLocal := false - for i := 0; i < len(local); i++ { - ch := local[i] - if isAtext(ch, false) { + for i, r := range local { + if isAtext(r, false) { continue } - if ch == '.' { + if r == '.' { // Dots are okay if they are surrounded by atext. // We only need to check that the previous byte is // not a dot, and this isn't the end of the string. @@ -213,25 +217,16 @@ func (a *Address) String() string { // If every character is printable ASCII, quoting is simple. allPrintable := true - for i := 0; i < len(a.Name); i++ { + for _, r := range a.Name { // isWSP here should actually be isFWS, // but we don't support folding yet. - if !isVchar(a.Name[i]) && !isWSP(a.Name[i]) { + if !isVchar(r) && !isWSP(r) || isMultibyte(r) { allPrintable = false break } } if allPrintable { - b := bytes.NewBufferString(`"`) - for i := 0; i < len(a.Name); i++ { - if !isQtext(a.Name[i]) && !isWSP(a.Name[i]) { - b.WriteByte('\\') - } - b.WriteByte(a.Name[i]) - } - b.WriteString(`" `) - b.WriteString(s) - return b.String() + return quoteString(a.Name) + " " + s } // Text in an encoded-word in a display-name must not contain certain @@ -269,6 +264,18 @@ func (p *addrParser) parseAddressList() ([]*Address, error) { return list, nil } +func (p *addrParser) parseSingleAddress() (*Address, error) { + addr, err := p.parseAddress() + if err != nil { + return nil, err + } + p.skipSpace() + if !p.empty() { + return nil, fmt.Errorf("mail: expected single address, got %q", p.s) + } + return addr, nil +} + // parseAddress parses a single RFC 5322 address at the start of p. func (p *addrParser) parseAddress() (addr *Address, err error) { debug.Printf("parseAddress: %q", p.s) @@ -343,6 +350,9 @@ func (p *addrParser) consumeAddrSpec() (spec string, err error) { // quoted-string debug.Printf("consumeAddrSpec: parsing quoted-string") localPart, err = p.consumeQuotedString() + if localPart == "" { + err = errors.New("mail: empty quoted string in addr-spec") + } } else { // dot-atom debug.Printf("consumeAddrSpec: parsing dot-atom") @@ -416,57 +426,81 @@ func (p *addrParser) consumePhrase() (phrase string, err error) { func (p *addrParser) consumeQuotedString() (qs string, err error) { // Assume first byte is '"'. i := 1 - qsb := make([]byte, 0, 10) + qsb := make([]rune, 0, 10) + + escaped := false + Loop: for { - if i >= p.len() { + r, size := utf8.DecodeRuneInString(p.s[i:]) + + switch { + case size == 0: return "", errors.New("mail: unclosed quoted-string") - } - switch c := p.s[i]; { - case c == '"': - break Loop - case c == '\\': - if i+1 == p.len() { - return "", errors.New("mail: unclosed quoted-string") + + case size == 1 && r == utf8.RuneError: + return "", fmt.Errorf("mail: invalid utf-8 in quoted-string: %q", p.s) + + case escaped: + // quoted-pair = ("\" (VCHAR / WSP)) + + if !isVchar(r) && !isWSP(r) { + return "", fmt.Errorf("mail: bad character in quoted-string: %q", r) } - qsb = append(qsb, p.s[i+1]) - i += 2 - case isQtext(c), c == ' ': + + qsb = append(qsb, r) + escaped = false + + case isQtext(r) || isWSP(r): // qtext (printable US-ASCII excluding " and \), or // FWS (almost; we're ignoring CRLF) - qsb = append(qsb, c) - i++ + qsb = append(qsb, r) + + case r == '"': + break Loop + + case r == '\\': + escaped = true + default: - return "", fmt.Errorf("mail: bad character in quoted-string: %q", c) + return "", fmt.Errorf("mail: bad character in quoted-string: %q", r) + } + + i += size } p.s = p.s[i+1:] - if len(qsb) == 0 { - return "", errors.New("mail: empty quoted-string") - } return string(qsb), nil } -var errNonASCII = errors.New("mail: unencoded non-ASCII text in address") - // consumeAtom parses an RFC 5322 atom at the start of p. // If dot is true, consumeAtom parses an RFC 5322 dot-atom instead. // If permissive is true, consumeAtom will not fail on // leading/trailing/double dots in the atom (see golang.org/issue/4938). func (p *addrParser) consumeAtom(dot bool, permissive bool) (atom string, err error) { - if c := p.peek(); !isAtext(c, false) { - if c > 127 { - return "", errNonASCII + i := 0 + +Loop: + for { + r, size := utf8.DecodeRuneInString(p.s[i:]) + + switch { + case size == 1 && r == utf8.RuneError: + return "", fmt.Errorf("mail: invalid utf-8 in address: %q", p.s) + + case size == 0 || !isAtext(r, dot): + break Loop + + default: + i += size + } - return "", errors.New("mail: invalid string") } - i := 1 - for ; i < p.len() && isAtext(p.s[i], dot); i++ { - } - if i < p.len() && p.s[i] > 127 { - return "", errNonASCII + + if i == 0 { + return "", errors.New("mail: invalid string") } - atom, p.s = string(p.s[:i]), p.s[i:] + atom, p.s = p.s[:i], p.s[i:] if !permissive { if strings.HasPrefix(atom, ".") { return "", errors.New("mail: leading dot in atom") @@ -536,54 +570,58 @@ func (e charsetError) Error() string { return fmt.Sprintf("charset not supported: %q", string(e)) } -var atextChars = []byte("ABCDEFGHIJKLMNOPQRSTUVWXYZ" + - "abcdefghijklmnopqrstuvwxyz" + - "0123456789" + - "!#$%&'*+-/=?^_`{|}~") - -// isAtext reports whether c is an RFC 5322 atext character. +// isAtext reports whether r is an RFC 5322 atext character. // If dot is true, period is included. -func isAtext(c byte, dot bool) bool { - if dot && c == '.' { - return true +func isAtext(r rune, dot bool) bool { + switch r { + case '.': + return dot + + case '(', ')', '<', '>', '[', ']', ':', ';', '@', '\\', ',', '"': // RFC 5322 3.2.3. specials + return false } - return bytes.IndexByte(atextChars, c) >= 0 + return isVchar(r) } -// isQtext reports whether c is an RFC 5322 qtext character. -func isQtext(c byte) bool { +// isQtext reports whether r is an RFC 5322 qtext character. +func isQtext(r rune) bool { // Printable US-ASCII, excluding backslash or quote. - if c == '\\' || c == '"' { + if r == '\\' || r == '"' { return false } - return '!' <= c && c <= '~' + return isVchar(r) } -// quoteString renders a string as a RFC5322 quoted-string. +// quoteString renders a string as an RFC 5322 quoted-string. func quoteString(s string) string { var buf bytes.Buffer buf.WriteByte('"') - for _, c := range s { - ch := byte(c) - if isQtext(ch) || isWSP(ch) { - buf.WriteByte(ch) - } else if isVchar(ch) { + for _, r := range s { + if isQtext(r) || isWSP(r) { + buf.WriteRune(r) + } else if isVchar(r) { buf.WriteByte('\\') - buf.WriteByte(ch) + buf.WriteRune(r) } } buf.WriteByte('"') return buf.String() } -// isVchar reports whether c is an RFC 5322 VCHAR character. -func isVchar(c byte) bool { +// isVchar reports whether r is an RFC 5322 VCHAR character. +func isVchar(r rune) bool { // Visible (printing) characters. - return '!' <= c && c <= '~' + return '!' <= r && r <= '~' || isMultibyte(r) +} + +// isMultibyte reports whether r is a multi-byte UTF-8 character +// as supported by RFC 6532 +func isMultibyte(r rune) bool { + return r >= utf8.RuneSelf } -// isWSP reports whether c is a WSP (white space). -// WSP is a space or horizontal tab (RFC5234 Appendix B). -func isWSP(c byte) bool { - return c == ' ' || c == '\t' +// isWSP reports whether r is a WSP (white space). +// WSP is a space or horizontal tab (RFC 5234 Appendix B). +func isWSP(r rune) bool { + return r == ' ' || r == '\t' } diff --git a/libgo/go/net/mail/message_test.go b/libgo/go/net/mail/message_test.go index 4e718e2636..f0761ab09f 100644 --- a/libgo/go/net/mail/message_test.go +++ b/libgo/go/net/mail/message_test.go @@ -92,7 +92,7 @@ func TestDateParsing(t *testing.T) { "Fri, 21 Nov 1997 09:55:06 -0600", time.Date(1997, 11, 21, 9, 55, 6, 0, time.FixedZone("", -6*60*60)), }, - // RFC5322, Appendix A.6.2 + // RFC 5322, Appendix A.6.2 // Obsolete date. { "21 Nov 97 09:55:06 GMT", @@ -110,28 +110,39 @@ func TestDateParsing(t *testing.T) { } date, err := hdr.Date() if err != nil { - t.Errorf("Failed parsing %q: %v", test.dateStr, err) - continue + t.Errorf("Header(Date: %s).Date(): %v", test.dateStr, err) + } else if !date.Equal(test.exp) { + t.Errorf("Header(Date: %s).Date() = %+v, want %+v", test.dateStr, date, test.exp) } - if !date.Equal(test.exp) { - t.Errorf("Parse of %q: got %+v, want %+v", test.dateStr, date, test.exp) + + date, err = ParseDate(test.dateStr) + if err != nil { + t.Errorf("ParseDate(%s): %v", test.dateStr, err) + } else if !date.Equal(test.exp) { + t.Errorf("ParseDate(%s) = %+v, want %+v", test.dateStr, date, test.exp) } } } func TestAddressParsingError(t *testing.T) { - const txt = "=?iso-8859-2?Q?Bogl=E1rka_Tak=E1cs?= <unknown@gmail.com>" - _, err := ParseAddress(txt) - if err == nil || !strings.Contains(err.Error(), "charset not supported") { - t.Errorf(`mail.ParseAddress(%q) err: %q, want ".*charset not supported.*"`, txt, err) + mustErrTestCases := [...]struct { + text string + wantErrText string + }{ + 0: {"=?iso-8859-2?Q?Bogl=E1rka_Tak=E1cs?= <unknown@gmail.com>", "charset not supported"}, + 1: {"a@gmail.com b@gmail.com", "expected single address"}, + 2: {string([]byte{0xed, 0xa0, 0x80}) + " <micro@example.net>", "invalid utf-8 in address"}, + 3: {"\"" + string([]byte{0xed, 0xa0, 0x80}) + "\" <half-surrogate@example.com>", "invalid utf-8 in quoted-string"}, + 4: {"\"\\" + string([]byte{0x80}) + "\" <escaped-invalid-unicode@example.net>", "invalid utf-8 in quoted-string"}, + 5: {"\"\x00\" <null@example.net>", "bad character in quoted-string"}, + 6: {"\"\\\x00\" <escaped-null@example.net>", "bad character in quoted-string"}, } -} -func TestAddressParsingErrorUnquotedNonASCII(t *testing.T) { - const txt = "µ <micro@example.net>" - _, err := ParseAddress(txt) - if err == nil || !strings.Contains(err.Error(), "unencoded non-ASCII text in address") { - t.Errorf(`mail.ParseAddress(%q) err: %q, want ".*unencoded non-ASCII text in address.*"`, txt, err) + for i, tc := range mustErrTestCases { + _, err := ParseAddress(tc.text) + if err == nil || !strings.Contains(err.Error(), tc.wantErrText) { + t.Errorf(`mail.ParseAddress(%q) #%d want %q, got %v`, tc.text, i, tc.wantErrText, err) + } } } @@ -264,6 +275,56 @@ func TestAddressParsing(t *testing.T) { }, }, }, + // RFC 6532 3.2.3, qtext /= UTF8-non-ascii + { + `"Gø Pher" <gopher@example.com>`, + []*Address{ + { + Name: `Gø Pher`, + Address: "gopher@example.com", + }, + }, + }, + // RFC 6532 3.2, atext /= UTF8-non-ascii + { + `µ <micro@example.com>`, + []*Address{ + { + Name: `µ`, + Address: "micro@example.com", + }, + }, + }, + // RFC 6532 3.2.2, local address parts allow UTF-8 + { + `Micro <µ@example.com>`, + []*Address{ + { + Name: `Micro`, + Address: "µ@example.com", + }, + }, + }, + // RFC 6532 3.2.4, domains parts allow UTF-8 + { + `Micro <micro@µ.example.com>`, + []*Address{ + { + Name: `Micro`, + Address: "micro@µ.example.com", + }, + }, + }, + // Issue 14866 + { + `"" <emptystring@example.com>`, + []*Address{ + { + Name: "", + Address: "emptystring@example.com", + }, + }, + }, } for _, test := range tests { if len(test.exp) == 1 { @@ -515,6 +576,11 @@ func TestAddressString(t *testing.T) { &Address{Name: "world?=", Address: "hello@world.com"}, `"world?=" <hello@world.com>`, }, + { + // should q-encode even for invalid utf-8. + &Address{Name: string([]byte{0xed, 0xa0, 0x80}), Address: "invalid-utf8@example.net"}, + "=?utf-8?q?=ED=A0=80?= <invalid-utf8@example.net>", + }, } for _, test := range tests { s := test.addr.String() @@ -610,7 +676,6 @@ func TestAddressParsingAndFormatting(t *testing.T) { `< @example.com>`, `<""test""blah""@example.com>`, `<""@0>`, - "<\"\t0\"@0>", } for _, test := range badTests { diff --git a/libgo/go/net/main_conf_test.go b/libgo/go/net/main_conf_test.go new file mode 100644 index 0000000000..9875ceaf1f --- /dev/null +++ b/libgo/go/net/main_conf_test.go @@ -0,0 +1,38 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !nacl,!plan9,!windows + +package net + +// forceGoDNS forces the resolver configuration to use the pure Go resolver +// and returns a fixup function to restore the old settings. +func forceGoDNS() func() { + c := systemConf() + oldGo := c.netGo + oldCgo := c.netCgo + fixup := func() { + c.netGo = oldGo + c.netCgo = oldCgo + } + c.netGo = true + c.netCgo = false + return fixup +} + +// forceCgoDNS forces the resolver configuration to use the cgo resolver +// and returns a fixup function to restore the old settings. +// (On non-Unix systems forceCgoDNS returns nil.) +func forceCgoDNS() func() { + c := systemConf() + oldGo := c.netGo + oldCgo := c.netCgo + fixup := func() { + c.netGo = oldGo + c.netCgo = oldCgo + } + c.netGo = false + c.netCgo = true + return fixup +} diff --git a/libgo/go/net/non_unix_test.go b/libgo/go/net/main_noconf_test.go index db3427e7cb..489477bc97 100644 --- a/libgo/go/net/non_unix_test.go +++ b/libgo/go/net/main_noconf_test.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Go Authors. All rights reserved. +// Copyright 2015 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -8,7 +8,7 @@ package net import "runtime" -// See unix_test.go for what these (don't) do. +// See main_conf_test.go for what these (don't) do. func forceGoDNS() func() { switch runtime.GOOS { case "plan9", "windows": @@ -18,5 +18,5 @@ func forceGoDNS() func() { } } -// See unix_test.go for what these (don't) do. +// See main_conf_test.go for what these (don't) do. func forceCgoDNS() func() { return nil } diff --git a/libgo/go/net/main_plan9_test.go b/libgo/go/net/main_plan9_test.go index 94501cada9..2bc5be88be 100644 --- a/libgo/go/net/main_plan9_test.go +++ b/libgo/go/net/main_plan9_test.go @@ -8,6 +8,7 @@ func installTestHooks() {} func uninstallTestHooks() {} +// forceCloseSockets must be called only from TestMain. func forceCloseSockets() {} func enableSocketConnect() {} diff --git a/libgo/go/net/main_test.go b/libgo/go/net/main_test.go index f3f8b1a900..28a8ff66d6 100644 --- a/libgo/go/net/main_test.go +++ b/libgo/go/net/main_test.go @@ -24,9 +24,9 @@ var ( ) var ( - testDNSFlood = flag.Bool("dnsflood", false, "whether to test DNS query flooding") + testTCPBig = flag.Bool("tcpbig", false, "whether to test massive size of data per read or write call on TCP connection") - testExternal = flag.Bool("external", true, "allow use of external networks during long test") + testDNSFlood = flag.Bool("dnsflood", false, "whether to test DNS query flooding") // If external IPv4 connectivity exists, we can try dialing // non-node/interface local scope IPv4 addresses. diff --git a/libgo/go/net/main_unix_test.go b/libgo/go/net/main_unix_test.go index bfb4cd0065..0cc129f34d 100644 --- a/libgo/go/net/main_unix_test.go +++ b/libgo/go/net/main_unix_test.go @@ -45,6 +45,7 @@ func uninstallTestHooks() { } } +// forceCloseSockets must be called only from TestMain. func forceCloseSockets() { for s := range sw.Sockets() { closeFunc(s) diff --git a/libgo/go/net/main_windows_test.go b/libgo/go/net/main_windows_test.go index 2d829743ec..6ea318c2a5 100644 --- a/libgo/go/net/main_windows_test.go +++ b/libgo/go/net/main_windows_test.go @@ -11,6 +11,7 @@ var ( origConnect = connectFunc origConnectEx = connectExFunc origListen = listenFunc + origAccept = acceptFunc ) func installTestHooks() { @@ -19,6 +20,7 @@ func installTestHooks() { connectFunc = sw.Connect connectExFunc = sw.ConnectEx listenFunc = sw.Listen + acceptFunc = sw.AcceptEx } func uninstallTestHooks() { @@ -27,8 +29,10 @@ func uninstallTestHooks() { connectFunc = origConnect connectExFunc = origConnectEx listenFunc = origListen + acceptFunc = origAccept } +// forceCloseSockets must be called only from TestMain. func forceCloseSockets() { for s := range sw.Sockets() { closeFunc(s) diff --git a/libgo/go/net/mockserver_test.go b/libgo/go/net/mockserver_test.go index dd6f4df3b9..766de6a815 100644 --- a/libgo/go/net/mockserver_test.go +++ b/libgo/go/net/mockserver_test.go @@ -30,10 +30,20 @@ func testUnixAddr() string { func newLocalListener(network string) (Listener, error) { switch network { - case "tcp", "tcp4", "tcp6": + case "tcp": + if supportsIPv4 { + if ln, err := Listen("tcp4", "127.0.0.1:0"); err == nil { + return ln, nil + } + } + if supportsIPv6 { + return Listen("tcp6", "[::1]:0") + } + case "tcp4": if supportsIPv4 { return Listen("tcp4", "127.0.0.1:0") } + case "tcp6": if supportsIPv6 { return Listen("tcp6", "[::1]:0") } @@ -142,13 +152,6 @@ func (dss *dualStackServer) buildup(handler func(*dualStackServer, Listener)) er return nil } -func (dss *dualStackServer) putConn(c Conn) error { - dss.cmu.Lock() - dss.cs = append(dss.cs, c) - dss.cmu.Unlock() - return nil -} - func (dss *dualStackServer) teardownNetwork(network string) error { dss.lnmu.Lock() for i := range dss.lns { @@ -181,28 +184,24 @@ func (dss *dualStackServer) teardown() error { return nil } -func newDualStackServer(lns []streamListener) (*dualStackServer, error) { - dss := &dualStackServer{lns: lns, port: "0"} - for i := range dss.lns { - ln, err := Listen(dss.lns[i].network, JoinHostPort(dss.lns[i].address, dss.port)) - if err != nil { - for _, ln := range dss.lns[:i] { - ln.Listener.Close() - } - return nil, err - } - dss.lns[i].Listener = ln - dss.lns[i].done = make(chan bool) - if dss.port == "0" { - if _, dss.port, err = SplitHostPort(ln.Addr().String()); err != nil { - for _, ln := range dss.lns { - ln.Listener.Close() - } - return nil, err - } - } +func newDualStackServer() (*dualStackServer, error) { + lns, err := newDualStackListener() + if err != nil { + return nil, err + } + _, port, err := SplitHostPort(lns[0].Addr().String()) + if err != nil { + lns[0].Close() + lns[1].Close() + return nil, err } - return dss, nil + return &dualStackServer{ + lns: []streamListener{ + {network: "tcp4", address: lns[0].Addr().String(), Listener: lns[0], done: make(chan bool)}, + {network: "tcp6", address: lns[1].Addr().String(), Listener: lns[1], done: make(chan bool)}, + }, + port: port, + }, nil } func transponder(ln Listener, ch chan<- error) { @@ -225,7 +224,7 @@ func transponder(ln Listener, ch chan<- error) { defer c.Close() network := ln.Addr().Network() - if c.LocalAddr().Network() != network || c.LocalAddr().Network() != network { + if c.LocalAddr().Network() != network || c.RemoteAddr().Network() != network { ch <- fmt.Errorf("got %v->%v; expected %v->%v", c.LocalAddr().Network(), c.RemoteAddr().Network(), network, network) return } @@ -333,10 +332,18 @@ func timeoutTransmitter(c Conn, d, min, max time.Duration, ch chan<- error) { func newLocalPacketListener(network string) (PacketConn, error) { switch network { - case "udp", "udp4", "udp6": + case "udp": + if supportsIPv4 { + return ListenPacket("udp4", "127.0.0.1:0") + } + if supportsIPv6 { + return ListenPacket("udp6", "[::1]:0") + } + case "udp4": if supportsIPv4 { return ListenPacket("udp4", "127.0.0.1:0") } + case "udp6": if supportsIPv6 { return ListenPacket("udp6", "[::1]:0") } diff --git a/libgo/go/net/net.go b/libgo/go/net/net.go index d9d23fae8f..a8b57361e6 100644 --- a/libgo/go/net/net.go +++ b/libgo/go/net/net.go @@ -14,7 +14,7 @@ the same interfaces and similar Dial and Listen functions. The Dial function connects to a server: - conn, err := net.Dial("tcp", "google.com:80") + conn, err := net.Dial("tcp", "golang.org:80") if err != nil { // handle error } @@ -79,6 +79,7 @@ On Windows, the resolver always uses C library functions, such as GetAddrInfo an package net import ( + "context" "errors" "io" "os" @@ -101,9 +102,13 @@ func init() { } // Addr represents a network end point address. +// +// The two methods Network and String conventionally return strings +// that can be passed as the arguments to Dial, but the exact form +// and meaning of the strings is up to the implementation. type Addr interface { - Network() string // name of the network - String() string // string form of address + Network() string // name of the network (for example, "tcp", "udp") + String() string // string form of address (for example, "192.0.2.1:25", "[2001:db8::1]:80") } // Conn is a generic stream-oriented network connection. @@ -111,12 +116,12 @@ type Addr interface { // Multiple goroutines may invoke methods on a Conn simultaneously. type Conn interface { // Read reads data from the connection. - // Read can be made to time out and return a Error with Timeout() == true + // Read can be made to time out and return an Error with Timeout() == true // after a fixed time limit; see SetDeadline and SetReadDeadline. Read(b []byte) (n int, err error) // Write writes data to the connection. - // Write can be made to time out and return a Error with Timeout() == true + // Write can be made to time out and return an Error with Timeout() == true // after a fixed time limit; see SetDeadline and SetWriteDeadline. Write(b []byte) (n int, err error) @@ -136,8 +141,10 @@ type Conn interface { // // A deadline is an absolute time after which I/O operations // fail with a timeout (see type Error) instead of - // blocking. The deadline applies to all future I/O, not just - // the immediately following call to Read or Write. + // blocking. The deadline applies to all future and pending + // I/O, not just the immediately following call to Read or + // Write. After a deadline has been exceeded, the connection + // can be refreshed by setting a deadline in the future. // // An idle timeout can be implemented by repeatedly extending // the deadline after successful Read or Write calls. @@ -145,11 +152,13 @@ type Conn interface { // A zero value for t means I/O operations will not time out. SetDeadline(t time.Time) error - // SetReadDeadline sets the deadline for future Read calls. + // SetReadDeadline sets the deadline for future Read calls + // and any currently-blocked Read call. // A zero value for t means Read will not time out. SetReadDeadline(t time.Time) error - // SetWriteDeadline sets the deadline for future Write calls. + // SetWriteDeadline sets the deadline for future Write calls + // and any currently-blocked Write call. // Even if write times out, it may return n > 0, indicating that // some of the data was successfully written. // A zero value for t means Write will not time out. @@ -297,17 +306,17 @@ func (c *conn) File() (f *os.File, err error) { // Multiple goroutines may invoke methods on a PacketConn simultaneously. type PacketConn interface { // ReadFrom reads a packet from the connection, - // copying the payload into b. It returns the number of + // copying the payload into b. It returns the number of // bytes copied into b and the return address that // was on the packet. // ReadFrom can be made to time out and return - // an error with Timeout() == true after a fixed time limit; + // an Error with Timeout() == true after a fixed time limit; // see SetDeadline and SetReadDeadline. ReadFrom(b []byte) (n int, addr Addr, err error) // WriteTo writes a packet with payload b to addr. // WriteTo can be made to time out and return - // an error with Timeout() == true after a fixed time limit; + // an Error with Timeout() == true after a fixed time limit; // see SetDeadline and SetWriteDeadline. // On packet-oriented connections, write timeouts are rare. WriteTo(b []byte, addr Addr) (n int, err error) @@ -320,21 +329,32 @@ type PacketConn interface { LocalAddr() Addr // SetDeadline sets the read and write deadlines associated - // with the connection. + // with the connection. It is equivalent to calling both + // SetReadDeadline and SetWriteDeadline. + // + // A deadline is an absolute time after which I/O operations + // fail with a timeout (see type Error) instead of + // blocking. The deadline applies to all future and pending + // I/O, not just the immediately following call to ReadFrom or + // WriteTo. After a deadline has been exceeded, the connection + // can be refreshed by setting a deadline in the future. + // + // An idle timeout can be implemented by repeatedly extending + // the deadline after successful ReadFrom or WriteTo calls. + // + // A zero value for t means I/O operations will not time out. SetDeadline(t time.Time) error - // SetReadDeadline sets the deadline for future Read calls. - // If the deadline is reached, Read will fail with a timeout - // (see type Error) instead of blocking. - // A zero value for t means Read will not time out. + // SetReadDeadline sets the deadline for future ReadFrom calls + // and any currently-blocked ReadFrom call. + // A zero value for t means ReadFrom will not time out. SetReadDeadline(t time.Time) error - // SetWriteDeadline sets the deadline for future Write calls. - // If the deadline is reached, Write will fail with a timeout - // (see type Error) instead of blocking. - // A zero value for t means Write will not time out. + // SetWriteDeadline sets the deadline for future WriteTo calls + // and any currently-blocked WriteTo call. // Even if write times out, it may return n > 0, indicating that // some of the data was successfully written. + // A zero value for t means WriteTo will not time out. SetWriteDeadline(t time.Time) error } @@ -364,6 +384,9 @@ type Error interface { // Various errors contained in OpError. var ( + // For connection setup operations. + errNoSuitableAddress = errors.New("no suitable address found") + // For connection setup and write operations. errMissingAddress = errors.New("missing address") @@ -374,6 +397,22 @@ var ( ErrWriteToConnected = errors.New("use of WriteTo with pre-connected connection") ) +// mapErr maps from the context errors to the historical internal net +// error values. +// +// TODO(bradfitz): get rid of this after adjusting tests and making +// context.DeadlineExceeded implement net.Error? +func mapErr(err error) error { + switch err { + case context.Canceled: + return errCanceled + case context.DeadlineExceeded: + return errTimeout + default: + return err + } +} + // OpError is the error type usually returned by functions in the net // package. It describes the operation, network type, and address of // an error. @@ -429,7 +468,7 @@ func (e *OpError) Error() string { var ( // aLongTimeAgo is a non-zero time, far in the past, used for // immediate cancelation of dials. - aLongTimeAgo = time.Unix(233431200, 0) + aLongTimeAgo = time.Unix(1, 0) // nonDeadline and noCancel are just zero values for // readability with functions taking too many parameters. @@ -492,7 +531,7 @@ func (e *AddrError) Error() string { } s := e.Err if e.Addr != "" { - s += " " + e.Addr + s = "address " + e.Addr + ": " + s } return s } @@ -584,3 +623,66 @@ func acquireThread() { func releaseThread() { <-threadLimit } + +// buffersWriter is the interface implemented by Conns that support a +// "writev"-like batch write optimization. +// writeBuffers should fully consume and write all chunks from the +// provided Buffers, else it should report a non-nil error. +type buffersWriter interface { + writeBuffers(*Buffers) (int64, error) +} + +var testHookDidWritev = func(wrote int) {} + +// Buffers contains zero or more runs of bytes to write. +// +// On certain machines, for certain types of connections, this is +// optimized into an OS-specific batch write operation (such as +// "writev"). +type Buffers [][]byte + +var ( + _ io.WriterTo = (*Buffers)(nil) + _ io.Reader = (*Buffers)(nil) +) + +func (v *Buffers) WriteTo(w io.Writer) (n int64, err error) { + if wv, ok := w.(buffersWriter); ok { + return wv.writeBuffers(v) + } + for _, b := range *v { + nb, err := w.Write(b) + n += int64(nb) + if err != nil { + v.consume(n) + return n, err + } + } + v.consume(n) + return n, nil +} + +func (v *Buffers) Read(p []byte) (n int, err error) { + for len(p) > 0 && len(*v) > 0 { + n0 := copy(p, (*v)[0]) + v.consume(int64(n0)) + p = p[n0:] + n += n0 + } + if len(*v) == 0 { + err = io.EOF + } + return +} + +func (v *Buffers) consume(n int64) { + for len(*v) > 0 { + ln0 := int64(len((*v)[0])) + if ln0 > n { + (*v)[0] = (*v)[0][n:] + return + } + n -= ln0 + *v = (*v)[1:] + } +} diff --git a/libgo/go/net/net_test.go b/libgo/go/net/net_test.go index cd62b4373e..9a9a7e552c 100644 --- a/libgo/go/net/net_test.go +++ b/libgo/go/net/net_test.go @@ -5,7 +5,10 @@ package net import ( + "errors" + "fmt" "io" + "net/internal/socktest" "os" "runtime" "testing" @@ -14,7 +17,7 @@ import ( func TestCloseRead(t *testing.T) { switch runtime.GOOS { - case "nacl", "plan9": + case "plan9": t.Skipf("not supported on %s", runtime.GOOS) } @@ -304,3 +307,212 @@ func TestListenCloseListen(t *testing.T) { } t.Fatalf("failed to listen/close/listen on same address after %d tries", maxTries) } + +// See golang.org/issue/6163, golang.org/issue/6987. +func TestAcceptIgnoreAbortedConnRequest(t *testing.T) { + switch runtime.GOOS { + case "plan9": + t.Skipf("%s does not have full support of socktest", runtime.GOOS) + } + + syserr := make(chan error) + go func() { + defer close(syserr) + for _, err := range abortedConnRequestErrors { + syserr <- err + } + }() + sw.Set(socktest.FilterAccept, func(so *socktest.Status) (socktest.AfterFilter, error) { + if err, ok := <-syserr; ok { + return nil, err + } + return nil, nil + }) + defer sw.Set(socktest.FilterAccept, nil) + + operr := make(chan error, 1) + handler := func(ls *localServer, ln Listener) { + defer close(operr) + c, err := ln.Accept() + if err != nil { + if perr := parseAcceptError(err); perr != nil { + operr <- perr + } + operr <- err + return + } + c.Close() + } + ls, err := newLocalServer("tcp") + if err != nil { + t.Fatal(err) + } + defer ls.teardown() + if err := ls.buildup(handler); err != nil { + t.Fatal(err) + } + + c, err := Dial(ls.Listener.Addr().Network(), ls.Listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + c.Close() + + for err := range operr { + t.Error(err) + } +} + +func TestZeroByteRead(t *testing.T) { + for _, network := range []string{"tcp", "unix", "unixpacket"} { + if !testableNetwork(network) { + t.Logf("skipping %s test", network) + continue + } + + ln, err := newLocalListener(network) + if err != nil { + t.Fatal(err) + } + connc := make(chan Conn, 1) + go func() { + defer ln.Close() + c, err := ln.Accept() + if err != nil { + t.Error(err) + } + connc <- c // might be nil + }() + c, err := Dial(network, ln.Addr().String()) + if err != nil { + t.Fatal(err) + } + defer c.Close() + sc := <-connc + if sc == nil { + continue + } + defer sc.Close() + + if runtime.GOOS == "windows" { + // A zero byte read on Windows caused a wait for readability first. + // Rather than change that behavior, satisfy it in this test. + // See Issue 15735. + go io.WriteString(sc, "a") + } + + n, err := c.Read(nil) + if n != 0 || err != nil { + t.Errorf("%s: zero byte client read = %v, %v; want 0, nil", network, n, err) + } + + if runtime.GOOS == "windows" { + // Same as comment above. + go io.WriteString(c, "a") + } + n, err = sc.Read(nil) + if n != 0 || err != nil { + t.Errorf("%s: zero byte server read = %v, %v; want 0, nil", network, n, err) + } + } +} + +// withTCPConnPair sets up a TCP connection between two peers, then +// runs peer1 and peer2 concurrently. withTCPConnPair returns when +// both have completed. +func withTCPConnPair(t *testing.T, peer1, peer2 func(c *TCPConn) error) { + ln, err := newLocalListener("tcp") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + errc := make(chan error, 2) + go func() { + c1, err := ln.Accept() + if err != nil { + errc <- err + return + } + defer c1.Close() + errc <- peer1(c1.(*TCPConn)) + }() + go func() { + c2, err := Dial("tcp", ln.Addr().String()) + if err != nil { + errc <- err + return + } + defer c2.Close() + errc <- peer2(c2.(*TCPConn)) + }() + for i := 0; i < 2; i++ { + if err := <-errc; err != nil { + t.Fatal(err) + } + } +} + +// Tests that a blocked Read is interrupted by a concurrent SetReadDeadline +// modifying that Conn's read deadline to the past. +// See golang.org/cl/30164 which documented this. The net/http package +// depends on this. +func TestReadTimeoutUnblocksRead(t *testing.T) { + serverDone := make(chan struct{}) + server := func(cs *TCPConn) error { + defer close(serverDone) + errc := make(chan error, 1) + go func() { + defer close(errc) + go func() { + // TODO: find a better way to wait + // until we're blocked in the cs.Read + // call below. Sleep is lame. + time.Sleep(100 * time.Millisecond) + + // Interrupt the upcoming Read, unblocking it: + cs.SetReadDeadline(time.Unix(123, 0)) // time in the past + }() + var buf [1]byte + n, err := cs.Read(buf[:1]) + if n != 0 || err == nil { + errc <- fmt.Errorf("Read = %v, %v; want 0, non-nil", n, err) + } + }() + select { + case err := <-errc: + return err + case <-time.After(5 * time.Second): + buf := make([]byte, 2<<20) + buf = buf[:runtime.Stack(buf, true)] + println("Stacks at timeout:\n", string(buf)) + return errors.New("timeout waiting for Read to finish") + } + + } + // Do nothing in the client. Never write. Just wait for the + // server's half to be done. + client := func(*TCPConn) error { + <-serverDone + return nil + } + withTCPConnPair(t, client, server) +} + +// Issue 17695: verify that a blocked Read is woken up by a Close. +func TestCloseUnblocksRead(t *testing.T) { + t.Parallel() + server := func(cs *TCPConn) error { + // Give the client time to get stuck in a Read: + time.Sleep(20 * time.Millisecond) + cs.Close() + return nil + } + client := func(ss *TCPConn) error { + n, err := ss.Read([]byte{0}) + if n != 0 || err != io.EOF { + return fmt.Errorf("Read = %v, %v; want 0, EOF", n, err) + } + return nil + } + withTCPConnPair(t, client, server) +} diff --git a/libgo/go/net/newpollserver_rtems.go b/libgo/go/net/newpollserver_rtems.go index 410f9321d3..2add81467b 100644 --- a/libgo/go/net/newpollserver_rtems.go +++ b/libgo/go/net/newpollserver_rtems.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +// +build rtems + package net import ( diff --git a/libgo/go/net/packetconn_test.go b/libgo/go/net/packetconn_test.go index 7f3ea8a2d0..7d50489021 100644 --- a/libgo/go/net/packetconn_test.go +++ b/libgo/go/net/packetconn_test.go @@ -1,4 +1,4 @@ -// Copyright 2012 The Go Authors. All rights reserved. +// Copyright 2012 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/parse.go b/libgo/go/net/parse.go index 80836a108c..b270159cd8 100644 --- a/libgo/go/net/parse.go +++ b/libgo/go/net/parse.go @@ -106,14 +106,14 @@ func splitAtBytes(s string, t string) []string { for i := 0; i < len(s); i++ { if byteIndex(t, s[i]) >= 0 { if last < i { - a[n] = string(s[last:i]) + a[n] = s[last:i] n++ } last = i + 1 } } if last < len(s) { - a[n] = string(s[last:]) + a[n] = s[last:] n++ } return a[0:n] @@ -124,39 +124,27 @@ func getFields(s string) []string { return splitAtBytes(s, " \r\t\n") } // Bigger than we need, not too big to worry about overflow const big = 0xFFFFFF -// Decimal to integer starting at &s[i0]. -// Returns number, new offset, success. -func dtoi(s string, i0 int) (n int, i int, ok bool) { +// Decimal to integer. +// Returns number, characters consumed, success. +func dtoi(s string) (n int, i int, ok bool) { n = 0 - neg := false - if len(s) > 0 && s[0] == '-' { - neg = true - s = s[1:] - } - for i = i0; i < len(s) && '0' <= s[i] && s[i] <= '9'; i++ { + for i = 0; i < len(s) && '0' <= s[i] && s[i] <= '9'; i++ { n = n*10 + int(s[i]-'0') if n >= big { - if neg { - return -big, i + 1, false - } return big, i, false } } - if i == i0 { - return 0, i, false - } - if neg { - n = -n - i++ + if i == 0 { + return 0, 0, false } return n, i, true } -// Hexadecimal to integer starting at &s[i0]. -// Returns number, new offset, success. -func xtoi(s string, i0 int) (n int, i int, ok bool) { +// Hexadecimal to integer. +// Returns number, characters consumed, success. +func xtoi(s string) (n int, i int, ok bool) { n = 0 - for i = i0; i < len(s); i++ { + for i = 0; i < len(s); i++ { if '0' <= s[i] && s[i] <= '9' { n *= 16 n += int(s[i] - '0') @@ -173,7 +161,7 @@ func xtoi(s string, i0 int) (n int, i int, ok bool) { return 0, i, false } } - if i == i0 { + if i == 0 { return 0, i, false } return n, i, true @@ -187,7 +175,7 @@ func xtoi2(s string, e byte) (byte, bool) { if len(s) > 2 && s[2] != e { return 0, false } - n, ei, ok := xtoi(s[:2], 0) + n, ei, ok := xtoi(s[:2]) return byte(n), ok && ei == 2 } @@ -348,22 +336,28 @@ func stringsHasSuffix(s, suffix string) bool { // stringsHasSuffixFold reports whether s ends in suffix, // ASCII-case-insensitively. func stringsHasSuffixFold(s, suffix string) bool { - if len(suffix) > len(s) { + return len(s) >= len(suffix) && stringsEqualFold(s[len(s)-len(suffix):], suffix) +} + +// stringsHasPrefix is strings.HasPrefix. It reports whether s begins with prefix. +func stringsHasPrefix(s, prefix string) bool { + return len(s) >= len(prefix) && s[:len(prefix)] == prefix +} + +// stringsEqualFold is strings.EqualFold, ASCII only. It reports whether s and t +// are equal, ASCII-case-insensitively. +func stringsEqualFold(s, t string) bool { + if len(s) != len(t) { return false } - for i := 0; i < len(suffix); i++ { - if lowerASCII(suffix[i]) != lowerASCII(s[len(s)-len(suffix)+i]) { + for i := 0; i < len(s); i++ { + if lowerASCII(s[i]) != lowerASCII(t[i]) { return false } } return true } -// stringsHasPrefix is strings.HasPrefix. It reports whether s begins with prefix. -func stringsHasPrefix(s, prefix string) bool { - return len(s) >= len(prefix) && s[:len(prefix)] == prefix -} - func readFull(r io.Reader) (all []byte, err error) { buf := make([]byte, 1024) for { diff --git a/libgo/go/net/parse_test.go b/libgo/go/net/parse_test.go index fec9200946..c5f8bfd198 100644 --- a/libgo/go/net/parse_test.go +++ b/libgo/go/net/parse_test.go @@ -86,14 +86,13 @@ func TestDtoi(t *testing.T) { ok bool }{ {"", 0, 0, false}, - - {"-123456789", -big, 9, false}, - {"-1", -1, 2, true}, {"0", 0, 1, true}, {"65536", 65536, 5, true}, {"123456789", big, 8, false}, + {"-0", 0, 0, false}, + {"-1234", 0, 0, false}, } { - n, i, ok := dtoi(tt.in, 0) + n, i, ok := dtoi(tt.in) if n != tt.out || i != tt.off || ok != tt.ok { t.Errorf("got %d, %d, %v; want %d, %d, %v", n, i, ok, tt.out, tt.off, tt.ok) } diff --git a/libgo/go/net/pipe.go b/libgo/go/net/pipe.go index 5fc830b740..37e552f54e 100644 --- a/libgo/go/net/pipe.go +++ b/libgo/go/net/pipe.go @@ -1,4 +1,4 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/pipe_test.go b/libgo/go/net/pipe_test.go index 60c3920593..e3172d882f 100644 --- a/libgo/go/net/pipe_test.go +++ b/libgo/go/net/pipe_test.go @@ -1,4 +1,4 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/platform_test.go b/libgo/go/net/platform_test.go index 76c53138cd..2a14095cc2 100644 --- a/libgo/go/net/platform_test.go +++ b/libgo/go/net/platform_test.go @@ -5,6 +5,7 @@ package net import ( + "internal/testenv" "os" "runtime" "strings" @@ -110,7 +111,7 @@ func testableListenArgs(network, address, client string) bool { } // Test wildcard IP addresses. - if wildcard && (testing.Short() || !*testExternal) { + if wildcard && !testenv.HasExternalNetwork() { return false } diff --git a/libgo/go/net/port.go b/libgo/go/net/port.go new file mode 100644 index 0000000000..8e1321afa4 --- /dev/null +++ b/libgo/go/net/port.go @@ -0,0 +1,62 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +// parsePort parses service as a decimal interger and returns the +// corresponding value as port. It is the caller's responsibility to +// parse service as a non-decimal integer when needsLookup is true. +// +// Some system resolvers will return a valid port number when given a number +// over 65536 (see https://github.com/golang/go/issues/11715). Alas, the parser +// can't bail early on numbers > 65536. Therefore reasonably large/small +// numbers are parsed in full and rejected if invalid. +func parsePort(service string) (port int, needsLookup bool) { + if service == "" { + // Lock in the legacy behavior that an empty string + // means port 0. See golang.org/issue/13610. + return 0, false + } + const ( + max = uint32(1<<32 - 1) + cutoff = uint32(1 << 30) + ) + neg := false + if service[0] == '+' { + service = service[1:] + } else if service[0] == '-' { + neg = true + service = service[1:] + } + var n uint32 + for _, d := range service { + if '0' <= d && d <= '9' { + d -= '0' + } else { + return 0, true + } + if n >= cutoff { + n = max + break + } + n *= 10 + nn := n + uint32(d) + if nn < n || nn > max { + n = max + break + } + n = nn + } + if !neg && n >= cutoff { + port = int(cutoff - 1) + } else if neg && n > cutoff { + port = int(cutoff) + } else { + port = int(n) + } + if neg { + port = -port + } + return port, false +} diff --git a/libgo/go/net/port_test.go b/libgo/go/net/port_test.go new file mode 100644 index 0000000000..e0bdb4247d --- /dev/null +++ b/libgo/go/net/port_test.go @@ -0,0 +1,52 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import "testing" + +var parsePortTests = []struct { + service string + port int + needsLookup bool +}{ + {"", 0, false}, + + // Decimal number literals + {"-1073741825", -1 << 30, false}, + {"-1073741824", -1 << 30, false}, + {"-1073741823", -(1<<30 - 1), false}, + {"-123456789", -123456789, false}, + {"-1", -1, false}, + {"-0", 0, false}, + {"0", 0, false}, + {"+0", 0, false}, + {"+1", 1, false}, + {"65535", 65535, false}, + {"65536", 65536, false}, + {"123456789", 123456789, false}, + {"1073741822", 1<<30 - 2, false}, + {"1073741823", 1<<30 - 1, false}, + {"1073741824", 1<<30 - 1, false}, + {"1073741825", 1<<30 - 1, false}, + + // Others + {"abc", 0, true}, + {"9pfs", 0, true}, + {"123badport", 0, true}, + {"bad123port", 0, true}, + {"badport123", 0, true}, + {"123456789badport", 0, true}, + {"-2147483649badport", 0, true}, + {"2147483649badport", 0, true}, +} + +func TestParsePort(t *testing.T) { + // The following test cases are cribbed from the strconv + for _, tt := range parsePortTests { + if port, needsLookup := parsePort(tt.service); port != tt.port || needsLookup != tt.needsLookup { + t.Errorf("parsePort(%q) = %d, %t; want %d, %t", tt.service, port, needsLookup, tt.port, tt.needsLookup) + } + } +} diff --git a/libgo/go/net/port_unix.go b/libgo/go/net/port_unix.go index badf8abc79..868d1e4784 100644 --- a/libgo/go/net/port_unix.go +++ b/libgo/go/net/port_unix.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build darwin dragonfly freebsd linux netbsd openbsd solaris +// +build darwin dragonfly freebsd linux netbsd openbsd solaris nacl // Read system port mappings from /etc/services @@ -10,31 +10,24 @@ package net import "sync" -// services contains minimal mappings between services names and port -// numbers for platforms that don't have a complete list of port numbers -// (some Solaris distros). -var services = map[string]map[string]int{ - "tcp": {"http": 80}, -} -var servicesError error var onceReadServices sync.Once func readServices() { - var file *file - if file, servicesError = open("/etc/services"); servicesError != nil { + file, err := open("/etc/services") + if err != nil { return } for line, ok := file.readLine(); ok; line, ok = file.readLine() { // "http 80/tcp www www-http # World Wide Web HTTP" if i := byteIndex(line, '#'); i >= 0 { - line = line[0:i] + line = line[:i] } f := getFields(line) if len(f) < 2 { continue } portnet := f[1] // "80/tcp" - port, j, ok := dtoi(portnet, 0) + port, j, ok := dtoi(portnet) if !ok || port <= 0 || j >= len(portnet) || portnet[j] != '/' { continue } @@ -56,18 +49,5 @@ func readServices() { // goLookupPort is the native Go implementation of LookupPort. func goLookupPort(network, service string) (port int, err error) { onceReadServices.Do(readServices) - - switch network { - case "tcp4", "tcp6": - network = "tcp" - case "udp4", "udp6": - network = "udp" - } - - if m, ok := services[network]; ok { - if port, ok = m[service]; ok { - return - } - } - return 0, &AddrError{Err: "unknown port", Addr: network + "/" + service} + return lookupPortMap(network, service) } diff --git a/libgo/go/net/protoconn_test.go b/libgo/go/net/protoconn_test.go index c6ef23b0e1..23589d3ca8 100644 --- a/libgo/go/net/protoconn_test.go +++ b/libgo/go/net/protoconn_test.go @@ -1,4 +1,4 @@ -// Copyright 2012 The Go Authors. All rights reserved. +// Copyright 2012 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/rpc/client.go b/libgo/go/net/rpc/client.go index d0c4a69214..fce6a4866c 100644 --- a/libgo/go/net/rpc/client.go +++ b/libgo/go/net/rpc/client.go @@ -55,7 +55,7 @@ type Client struct { // reading of RPC responses for the client side of an RPC session. // The client calls WriteRequest to write a request to the connection // and calls ReadResponseHeader and ReadResponseBody in pairs -// to read responses. The client calls Close when finished with the +// to read responses. The client calls Close when finished with the // connection. ReadResponseBody may be called with a nil // argument to force the body of the response to be read and then // discarded. @@ -173,7 +173,7 @@ func (call *Call) done() { case call.Done <- call: // ok default: - // We don't want to block here. It is the caller's responsibility to make + // We don't want to block here. It is the caller's responsibility to make // sure the channel has enough buffer space. See comment in Go(). if debugLog { log.Println("rpc: discarding Call reply due to insufficient Done chan capacity") @@ -274,6 +274,8 @@ func Dial(network, address string) (*Client, error) { return NewClient(conn), nil } +// Close calls the underlying codec's Close method. If the connection is already +// shutting down, ErrShutdown is returned. func (client *Client) Close() error { client.mutex.Lock() if client.closing { @@ -285,9 +287,9 @@ func (client *Client) Close() error { return client.codec.Close() } -// Go invokes the function asynchronously. It returns the Call structure representing -// the invocation. The done channel will signal when the call is complete by returning -// the same Call object. If done is nil, Go will allocate a new channel. +// Go invokes the function asynchronously. It returns the Call structure representing +// the invocation. The done channel will signal when the call is complete by returning +// the same Call object. If done is nil, Go will allocate a new channel. // If non-nil, done must be buffered or Go will deliberately crash. func (client *Client) Go(serviceMethod string, args interface{}, reply interface{}, done chan *Call) *Call { call := new(Call) @@ -299,7 +301,7 @@ func (client *Client) Go(serviceMethod string, args interface{}, reply interface } else { // If caller passes done != nil, it must arrange that // done has enough buffer for the number of simultaneous - // RPCs that will be using that channel. If the channel + // RPCs that will be using that channel. If the channel // is totally unbuffered, it's best not to run at all. if cap(done) == 0 { log.Panic("rpc: done channel is unbuffered") diff --git a/libgo/go/net/rpc/client_test.go b/libgo/go/net/rpc/client_test.go index ba11ff8586..d116d2acc9 100644 --- a/libgo/go/net/rpc/client_test.go +++ b/libgo/go/net/rpc/client_test.go @@ -8,7 +8,6 @@ import ( "errors" "fmt" "net" - "runtime" "strings" "testing" ) @@ -53,9 +52,6 @@ func (s *S) Recv(nul *struct{}, reply *R) error { } func TestGobError(t *testing.T) { - if runtime.GOOS == "plan9" { - t.Skip("skipping test; see https://golang.org/issue/8908") - } defer func() { err := recover() if err == nil { diff --git a/libgo/go/net/rpc/jsonrpc/all_test.go b/libgo/go/net/rpc/jsonrpc/all_test.go index a433a365e8..b811d3c0c7 100644 --- a/libgo/go/net/rpc/jsonrpc/all_test.go +++ b/libgo/go/net/rpc/jsonrpc/all_test.go @@ -1,4 +1,4 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/rpc/jsonrpc/client.go b/libgo/go/net/rpc/jsonrpc/client.go index 2194f21257..da1b8165fc 100644 --- a/libgo/go/net/rpc/jsonrpc/client.go +++ b/libgo/go/net/rpc/jsonrpc/client.go @@ -1,4 +1,4 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/rpc/jsonrpc/server.go b/libgo/go/net/rpc/jsonrpc/server.go index e6d37cfa64..40e4e6f2aa 100644 --- a/libgo/go/net/rpc/jsonrpc/server.go +++ b/libgo/go/net/rpc/jsonrpc/server.go @@ -1,4 +1,4 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -110,7 +110,7 @@ func (c *serverCodec) WriteResponse(r *rpc.Response, x interface{}) error { c.mutex.Unlock() if b == nil { - // Invalid request so no id. Use JSON null. + // Invalid request so no id. Use JSON null. b = &null } resp := serverResponse{Id: b} diff --git a/libgo/go/net/rpc/server.go b/libgo/go/net/rpc/server.go index c4d4479958..18ea629b0d 100644 --- a/libgo/go/net/rpc/server.go +++ b/libgo/go/net/rpc/server.go @@ -23,7 +23,7 @@ func (t *T) MethodName(argType T1, replyType *T2) error - where T, T1 and T2 can be marshaled by encoding/gob. + where T1 and T2 can be marshaled by encoding/gob. These requirements apply even if a different codec is used. (In the future, these requirements may soften for custom codecs.) @@ -55,6 +55,8 @@ package server + import "errors" + type Args struct { A, B int } @@ -119,6 +121,8 @@ A server implementation will often provide a simple, type-safe wrapper for the client. + + The net/rpc package is frozen and is not accepting new features. */ package rpc @@ -143,8 +147,8 @@ const ( DefaultDebugPath = "/debug/rpc" ) -// Precompute the reflect type for error. Can't use error directly -// because Typeof takes an empty interface value. This is annoying. +// Precompute the reflect type for error. Can't use error directly +// because Typeof takes an empty interface value. This is annoying. var typeOfError = reflect.TypeOf((*error)(nil)).Elem() type methodType struct { @@ -162,7 +166,7 @@ type service struct { method map[string]*methodType // registered methods } -// Request is a header written before every RPC call. It is used internally +// Request is a header written before every RPC call. It is used internally // but documented here as an aid to debugging, such as when analyzing // network traffic. type Request struct { @@ -171,7 +175,7 @@ type Request struct { next *Request // for free list in Server } -// Response is a header written before every RPC return. It is used internally +// Response is a header written before every RPC return. It is used internally // but documented here as an aid to debugging, such as when analyzing // network traffic. type Response struct { @@ -442,7 +446,7 @@ func (c *gobServerCodec) Close() error { // ServeConn blocks, serving the connection until the client hangs up. // The caller typically invokes ServeConn in a go statement. // ServeConn uses the gob wire format (see package gob) on the -// connection. To use an alternate codec, use ServeCodec. +// connection. To use an alternate codec, use ServeCodec. func (server *Server) ServeConn(conn io.ReadWriteCloser) { buf := bufio.NewWriter(conn) srv := &gobServerCodec{ @@ -583,7 +587,7 @@ func (server *Server) readRequestHeader(codec ServerCodec) (service *service, mt return } - // We read the header successfully. If we see an error now, + // We read the header successfully. If we see an error now, // we can still recover and move on to the next request. keepReading = true @@ -638,7 +642,7 @@ func RegisterName(name string, rcvr interface{}) error { // RPC responses for the server side of an RPC session. // The server calls ReadRequestHeader and ReadRequestBody in pairs // to read requests from the connection, and it calls WriteResponse to -// write a response back. The server calls Close when finished with the +// write a response back. The server calls Close when finished with the // connection. ReadRequestBody may be called with a nil // argument to force the body of the request to be read and discarded. type ServerCodec interface { @@ -654,7 +658,7 @@ type ServerCodec interface { // ServeConn blocks, serving the connection until the client hangs up. // The caller typically invokes ServeConn in a go statement. // ServeConn uses the gob wire format (see package gob) on the -// connection. To use an alternate codec, use ServeCodec. +// connection. To use an alternate codec, use ServeCodec. func ServeConn(conn io.ReadWriteCloser) { DefaultServer.ServeConn(conn) } diff --git a/libgo/go/net/rpc/server_test.go b/libgo/go/net/rpc/server_test.go index 8871c88133..8369c9dec7 100644 --- a/libgo/go/net/rpc/server_test.go +++ b/libgo/go/net/rpc/server_test.go @@ -183,7 +183,7 @@ func testRPC(t *testing.T, addr string) { err = client.Call("Arith.Unknown", args, reply) if err == nil { t.Error("expected error calling unknown service") - } else if strings.Index(err.Error(), "method") < 0 { + } else if !strings.Contains(err.Error(), "method") { t.Error("expected error about method; got", err) } @@ -226,7 +226,7 @@ func testRPC(t *testing.T, addr string) { err = client.Call("Arith.Add", reply, reply) // args, reply would be the correct thing to use if err == nil { t.Error("expected error calling Arith.Add with wrong arg type") - } else if strings.Index(err.Error(), "type") < 0 { + } else if !strings.Contains(err.Error(), "type") { t.Error("expected error about type; got", err) } @@ -657,6 +657,9 @@ func benchmarkEndToEnd(dial func() (*Client, error), b *testing.B) { } func benchmarkEndToEndAsync(dial func() (*Client, error), b *testing.B) { + if b.N == 0 { + return + } const MaxConcurrentCalls = 100 once.Do(startServer) client, err := dial() @@ -690,7 +693,8 @@ func benchmarkEndToEndAsync(dial func() (*Client, error), b *testing.B) { B := call.Args.(*Args).B C := call.Reply.(*Reply).C if A+B != C { - b.Fatalf("incorrect reply: Add: expected %d got %d", A+B, C) + b.Errorf("incorrect reply: Add: expected %d got %d", A+B, C) + return } <-gate if atomic.AddInt32(&recv, -1) == 0 { diff --git a/libgo/go/net/sendfile_dragonfly.go b/libgo/go/net/sendfile_dragonfly.go index a9cf3fe951..d4b825c370 100644 --- a/libgo/go/net/sendfile_dragonfly.go +++ b/libgo/go/net/sendfile_dragonfly.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -53,7 +53,7 @@ func sendFile(c *netFD, r io.Reader) (written int64, err error, handled bool) { // use the current position of the file -- if you pass it offset 0, it starts // from offset 0. There's no way to tell it "start from current position", so // we have to manage that explicitly. - pos, err := f.Seek(0, os.SEEK_CUR) + pos, err := f.Seek(0, io.SeekCurrent) if err != nil { return 0, err, false } @@ -81,7 +81,7 @@ func sendFile(c *netFD, r io.Reader) (written int64, err error, handled bool) { break } if err1 == syscall.EAGAIN { - if err1 = c.pd.WaitWrite(); err1 == nil { + if err1 = c.pd.waitWrite(); err1 == nil { continue } } diff --git a/libgo/go/net/sendfile_freebsd.go b/libgo/go/net/sendfile_freebsd.go index d0bf6034c1..18cbb27b53 100644 --- a/libgo/go/net/sendfile_freebsd.go +++ b/libgo/go/net/sendfile_freebsd.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -53,7 +53,7 @@ func sendFile(c *netFD, r io.Reader) (written int64, err error, handled bool) { // use the current position of the file -- if you pass it offset 0, it starts // from offset 0. There's no way to tell it "start from current position", so // we have to manage that explicitly. - pos, err := f.Seek(0, os.SEEK_CUR) + pos, err := f.Seek(0, io.SeekCurrent) if err != nil { return 0, err, false } @@ -81,7 +81,7 @@ func sendFile(c *netFD, r io.Reader) (written int64, err error, handled bool) { break } if err1 == syscall.EAGAIN { - if err1 = c.pd.WaitWrite(); err1 == nil { + if err1 = c.pd.waitWrite(); err1 == nil { continue } } diff --git a/libgo/go/net/sendfile_linux.go b/libgo/go/net/sendfile_linux.go index 5ca41c39eb..7e741f9794 100644 --- a/libgo/go/net/sendfile_linux.go +++ b/libgo/go/net/sendfile_linux.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -57,7 +57,7 @@ func sendFile(c *netFD, r io.Reader) (written int64, err error, handled bool) { break } if err1 == syscall.EAGAIN { - if err1 = c.pd.WaitWrite(); err1 == nil { + if err1 = c.pd.waitWrite(); err1 == nil { continue } } diff --git a/libgo/go/net/sendfile_solaris.go b/libgo/go/net/sendfile_solaris.go index f6833813fd..add70c3147 100644 --- a/libgo/go/net/sendfile_solaris.go +++ b/libgo/go/net/sendfile_solaris.go @@ -1,4 +1,4 @@ -// Copyright 2015 The Go Authors. All rights reserved. +// Copyright 2015 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -26,8 +26,6 @@ const maxSendfileSize int = 4 << 20 // // if handled == false, sendFile performed no work. func sendFile(c *netFD, r io.Reader) (written int64, err error, handled bool) { - return // Solaris sendfile is disabled until Issue 13892 is understood and fixed - // Solaris uses 0 as the "until EOF" value. If you pass in more bytes than the // file contains, it will loop back to the beginning ad nauseam until it's sent // exactly the number of bytes told to. As such, we need to know exactly how many @@ -59,7 +57,7 @@ func sendFile(c *netFD, r io.Reader) (written int64, err error, handled bool) { // use the current position of the file -- if you pass it offset 0, it starts // from offset 0. There's no way to tell it "start from current position", so // we have to manage that explicitly. - pos, err := f.Seek(0, os.SEEK_CUR) + pos, err := f.Seek(0, io.SeekCurrent) if err != nil { return 0, err, false } @@ -78,6 +76,13 @@ func sendFile(c *netFD, r io.Reader) (written int64, err error, handled bool) { } pos1 := pos n, err1 := syscall.Sendfile(dst, src, &pos1, n) + if err1 == syscall.EAGAIN || err1 == syscall.EINTR { + // partial write may have occurred + if n = int(pos1 - pos); n == 0 { + // nothing more to write + err1 = nil + } + } if n > 0 { pos += int64(n) written += int64(n) @@ -87,7 +92,7 @@ func sendFile(c *netFD, r io.Reader) (written int64, err error, handled bool) { break } if err1 == syscall.EAGAIN { - if err1 = c.pd.WaitWrite(); err1 == nil { + if err1 = c.pd.waitWrite(); err1 == nil { continue } } diff --git a/libgo/go/net/sendfile_stub.go b/libgo/go/net/sendfile_stub.go index a0760b4e52..905f1d6cef 100644 --- a/libgo/go/net/sendfile_stub.go +++ b/libgo/go/net/sendfile_stub.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/sendfile_test.go b/libgo/go/net/sendfile_test.go new file mode 100644 index 0000000000..2255e7c478 --- /dev/null +++ b/libgo/go/net/sendfile_test.go @@ -0,0 +1,90 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" + "testing" +) + +const ( + twain = "testdata/Mark.Twain-Tom.Sawyer.txt" + twainLen = 387851 + twainSHA256 = "461eb7cb2d57d293fc680c836464c9125e4382be3596f7d415093ae9db8fcb0e" +) + +func TestSendfile(t *testing.T) { + ln, err := newLocalListener("tcp") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + + errc := make(chan error, 1) + go func(ln Listener) { + // Wait for a connection. + conn, err := ln.Accept() + if err != nil { + errc <- err + close(errc) + return + } + + go func() { + defer close(errc) + defer conn.Close() + + f, err := os.Open(twain) + if err != nil { + errc <- err + return + } + defer f.Close() + + // Return file data using io.Copy, which should use + // sendFile if available. + sbytes, err := io.Copy(conn, f) + if err != nil { + errc <- err + return + } + + if sbytes != twainLen { + errc <- fmt.Errorf("sent %d bytes; expected %d", sbytes, twainLen) + return + } + }() + }(ln) + + // Connect to listener to retrieve file and verify digest matches + // expected. + c, err := Dial("tcp", ln.Addr().String()) + if err != nil { + t.Fatal(err) + } + defer c.Close() + + h := sha256.New() + rbytes, err := io.Copy(h, c) + if err != nil { + t.Error(err) + } + + if rbytes != twainLen { + t.Errorf("received %d bytes; expected %d", rbytes, twainLen) + } + + if res := hex.EncodeToString(h.Sum(nil)); res != twainSHA256 { + t.Error("retrieved data hash did not match") + } + + for err := range errc { + t.Error(err) + } +} diff --git a/libgo/go/net/sendfile_windows.go b/libgo/go/net/sendfile_windows.go index f3f3b54b88..bc0b7fb5b2 100644 --- a/libgo/go/net/sendfile_windows.go +++ b/libgo/go/net/sendfile_windows.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -18,7 +18,7 @@ import ( // // if handled == false, sendFile performed no work. // -// Note that sendfile for windows does not suppport >2GB file. +// Note that sendfile for windows does not support >2GB file. func sendFile(fd *netFD, r io.Reader) (written int64, err error, handled bool) { var n int64 = 0 // by default, copy until EOF diff --git a/libgo/go/net/smtp/smtp.go b/libgo/go/net/smtp/smtp.go index 0988350322..a408fa5336 100644 --- a/libgo/go/net/smtp/smtp.go +++ b/libgo/go/net/smtp/smtp.go @@ -8,12 +8,18 @@ // AUTH RFC 2554 // STARTTLS RFC 3207 // Additional extensions may be handled by clients. +// +// The smtp package is frozen and is not accepting new features. +// Some external packages provide more functionality. See: +// +// https://godoc.org/?q=smtp package smtp import ( "crypto/tls" "encoding/base64" "errors" + "fmt" "io" "net" "net/textproto" @@ -83,8 +89,8 @@ func (c *Client) hello() error { // Hello sends a HELO or EHLO to the server as the given host name. // Calling this method is only necessary if the client needs control -// over the host name used. The client will introduce itself as "localhost" -// automatically otherwise. If Hello is called, it must be called before +// over the host name used. The client will introduce itself as "localhost" +// automatically otherwise. If Hello is called, it must be called before // any of the other methods. func (c *Client) Hello(localName string) error { if c.didHello { @@ -195,7 +201,7 @@ func (c *Client) Auth(a Auth) error { } resp64 := make([]byte, encoding.EncodedLen(len(resp))) encoding.Encode(resp64, resp) - code, msg64, err := c.cmd(0, "AUTH %s %s", mech, resp64) + code, msg64, err := c.cmd(0, strings.TrimSpace(fmt.Sprintf("AUTH %s %s", mech, resp64))) for err == nil { var msg []byte switch code { @@ -265,7 +271,7 @@ func (d *dataCloser) Close() error { // Data issues a DATA command to the server and returns a writer that // can be used to write the mail headers and body. The caller should -// close the writer before calling any more methods on c. A call to +// close the writer before calling any more methods on c. A call to // Data must be preceded by one or more calls to Rcpt. func (c *Client) Data() (io.WriteCloser, error) { _, _, err := c.cmd(354, "DATA") @@ -287,7 +293,7 @@ var testHookStartTLS func(*tls.Config) // nil, except for tests // // The msg parameter should be an RFC 822-style email with headers // first, a blank line, and then the message body. The lines of msg -// should be CRLF terminated. The msg headers should usually include +// should be CRLF terminated. The msg headers should usually include // fields such as "From", "To", "Subject", and "Cc". Sending "Bcc" // messages is accomplished by including an email address in the to // parameter but not including it in the msg headers. diff --git a/libgo/go/net/smtp/smtp_test.go b/libgo/go/net/smtp/smtp_test.go index 3ae0d5bf1d..c48fae6d5a 100644 --- a/libgo/go/net/smtp/smtp_test.go +++ b/libgo/go/net/smtp/smtp_test.go @@ -94,6 +94,46 @@ func TestAuthPlain(t *testing.T) { } } +// Issue 17794: don't send a trailing space on AUTH command when there's no password. +func TestClientAuthTrimSpace(t *testing.T) { + server := "220 hello world\r\n" + + "200 some more" + var wrote bytes.Buffer + var fake faker + fake.ReadWriter = struct { + io.Reader + io.Writer + }{ + strings.NewReader(server), + &wrote, + } + c, err := NewClient(fake, "fake.host") + if err != nil { + t.Fatalf("NewClient: %v", err) + } + c.tls = true + c.didHello = true + c.Auth(toServerEmptyAuth{}) + c.Close() + if got, want := wrote.String(), "AUTH FOOAUTH\r\n*\r\nQUIT\r\n"; got != want { + t.Errorf("wrote %q; want %q", got, want) + } +} + +// toServerEmptyAuth is an implementation of Auth that only implements +// the Start method, and returns "FOOAUTH", nil, nil. Notably, it returns +// zero bytes for "toServer" so we can test that we don't send spaces at +// the end of the line. See TestClientAuthTrimSpace. +type toServerEmptyAuth struct{} + +func (toServerEmptyAuth) Start(server *ServerInfo) (proto string, toServer []byte, err error) { + return "FOOAUTH", nil, nil +} + +func (toServerEmptyAuth) Next(fromServer []byte, more bool) (toServer []byte, err error) { + panic("unexpected call") +} + type faker struct { io.ReadWriter } @@ -716,23 +756,24 @@ func sendMail(hostPort string) error { // generated from src/crypto/tls: // go run generate_cert.go --rsa-bits 512 --host 127.0.0.1,::1,example.com --ca --start-date "Jan 1 00:00:00 1970" --duration=1000000h var localhostCert = []byte(`-----BEGIN CERTIFICATE----- -MIIBdzCCASOgAwIBAgIBADALBgkqhkiG9w0BAQUwEjEQMA4GA1UEChMHQWNtZSBD -bzAeFw03MDAxMDEwMDAwMDBaFw00OTEyMzEyMzU5NTlaMBIxEDAOBgNVBAoTB0Fj -bWUgQ28wWjALBgkqhkiG9w0BAQEDSwAwSAJBAN55NcYKZeInyTuhcCwFMhDHCmwa -IUSdtXdcbItRB/yfXGBhiex00IaLXQnSU+QZPRZWYqeTEbFSgihqi1PUDy8CAwEA -AaNoMGYwDgYDVR0PAQH/BAQDAgCkMBMGA1UdJQQMMAoGCCsGAQUFBwMBMA8GA1Ud -EwEB/wQFMAMBAf8wLgYDVR0RBCcwJYILZXhhbXBsZS5jb22HBH8AAAGHEAAAAAAA -AAAAAAAAAAAAAAEwCwYJKoZIhvcNAQEFA0EAAoQn/ytgqpiLcZu9XKbCJsJcvkgk -Se6AbGXgSlq+ZCEVo0qIwSgeBqmsJxUu7NCSOwVJLYNEBO2DtIxoYVk+MA== +MIIBjjCCATigAwIBAgIQMon9v0s3pDFXvAMnPgelpzANBgkqhkiG9w0BAQsFADAS +MRAwDgYDVQQKEwdBY21lIENvMCAXDTcwMDEwMTAwMDAwMFoYDzIwODQwMTI5MTYw +MDAwWjASMRAwDgYDVQQKEwdBY21lIENvMFwwDQYJKoZIhvcNAQEBBQADSwAwSAJB +AM0u/mNXKkhAzNsFkwKZPSpC4lZZaePQ55IyaJv3ovMM2smvthnlqaUfVKVmz7FF +wLP9csX6vGtvkZg1uWAtvfkCAwEAAaNoMGYwDgYDVR0PAQH/BAQDAgKkMBMGA1Ud +JQQMMAoGCCsGAQUFBwMBMA8GA1UdEwEB/wQFMAMBAf8wLgYDVR0RBCcwJYILZXhh +bXBsZS5jb22HBH8AAAGHEAAAAAAAAAAAAAAAAAAAAAEwDQYJKoZIhvcNAQELBQAD +QQBOZsFVC7IwX+qibmSbt2IPHkUgXhfbq0a9MYhD6tHcj4gbDcTXh4kZCbgHCz22 +gfSj2/G2wxzopoISVDucuncj -----END CERTIFICATE-----`) // localhostKey is the private key for localhostCert. var localhostKey = []byte(`-----BEGIN RSA PRIVATE KEY----- -MIIBPAIBAAJBAN55NcYKZeInyTuhcCwFMhDHCmwaIUSdtXdcbItRB/yfXGBhiex0 -0IaLXQnSU+QZPRZWYqeTEbFSgihqi1PUDy8CAwEAAQJBAQdUx66rfh8sYsgfdcvV -NoafYpnEcB5s4m/vSVe6SU7dCK6eYec9f9wpT353ljhDUHq3EbmE4foNzJngh35d -AekCIQDhRQG5Li0Wj8TM4obOnnXUXf1jRv0UkzE9AHWLG5q3AwIhAPzSjpYUDjVW -MCUXgckTpKCuGwbJk7424Nb8bLzf3kllAiA5mUBgjfr/WtFSJdWcPQ4Zt9KTMNKD -EUO0ukpTwEIl6wIhAMbGqZK3zAAFdq8DD2jPx+UJXnh0rnOkZBzDtJ6/iN69AiEA -1Aq8MJgTaYsDQWyU/hDq5YkDJc9e9DSCvUIzqxQWMQE= +MIIBOwIBAAJBAM0u/mNXKkhAzNsFkwKZPSpC4lZZaePQ55IyaJv3ovMM2smvthnl +qaUfVKVmz7FFwLP9csX6vGtvkZg1uWAtvfkCAwEAAQJART2qkxODLUbQ2siSx7m2 +rmBLyR/7X+nLe8aPDrMOxj3heDNl4YlaAYLexbcY8d7VDfCRBKYoAOP0UCP1Vhuf +UQIhAO6PEI55K3SpNIdc2k5f0xz+9rodJCYzu51EwWX7r8ufAiEA3C9EkLiU2NuK +3L3DHCN5IlUSN1Nr/lw8NIt50Yorj2cCIQCDw1VbvCV6bDLtSSXzAA51B4ZzScE7 +sHtB5EYF9Dwm9QIhAJuCquuH4mDzVjUntXjXOQPdj7sRqVGCNWdrJwOukat7AiAy +LXLEwb77DIPoI5ZuaXQC+MnyyJj1ExC9RFcGz+bexA== -----END RSA PRIVATE KEY-----`) diff --git a/libgo/go/net/sock_bsd.go b/libgo/go/net/sock_bsd.go index 6c37109f5e..4e0e9e01f2 100644 --- a/libgo/go/net/sock_bsd.go +++ b/libgo/go/net/sock_bsd.go @@ -1,4 +1,4 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/sock_linux.go b/libgo/go/net/sock_linux.go index cc5ce153b3..7bca37605e 100644 --- a/libgo/go/net/sock_linux.go +++ b/libgo/go/net/sock_linux.go @@ -1,4 +1,4 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -17,7 +17,7 @@ func maxListenerBacklog() int { return syscall.SOMAXCONN } f := getFields(l) - n, _, ok := dtoi(f[0], 0) + n, _, ok := dtoi(f[0]) if n == 0 || !ok { return syscall.SOMAXCONN } diff --git a/libgo/go/net/sock_plan9.go b/libgo/go/net/sock_plan9.go index 88d9ed15cf..9367ad8fa7 100644 --- a/libgo/go/net/sock_plan9.go +++ b/libgo/go/net/sock_plan9.go @@ -1,4 +1,4 @@ -// Copyright 2013 The Go Authors. All rights reserved. +// Copyright 2013 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/sock_posix.go b/libgo/go/net/sock_posix.go index 4676721567..16351e1f14 100644 --- a/libgo/go/net/sock_posix.go +++ b/libgo/go/net/sock_posix.go @@ -1,4 +1,4 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -7,9 +7,9 @@ package net import ( + "context" "os" "syscall" - "time" ) // A sockaddr represents a TCP, UDP, IP or Unix network endpoint @@ -30,11 +30,14 @@ type sockaddr interface { // interface. It returns a nil interface when the address is // nil. sockaddr(family int) (syscall.Sockaddr, error) + + // toLocal maps the zero address to a local system address (127.0.0.1 or ::1) + toLocal(net string) sockaddr } // socket returns a network file descriptor that is ready for // asynchronous I/O using the network poller. -func socket(net string, family, sotype, proto int, ipv6only bool, laddr, raddr sockaddr, deadline time.Time, cancel <-chan struct{}) (fd *netFD, err error) { +func socket(ctx context.Context, net string, family, sotype, proto int, ipv6only bool, laddr, raddr sockaddr) (fd *netFD, err error) { s, err := sysSocket(family, sotype, proto) if err != nil { return nil, err @@ -86,7 +89,7 @@ func socket(net string, family, sotype, proto int, ipv6only bool, laddr, raddr s return fd, nil } } - if err := fd.dial(laddr, raddr, deadline, cancel); err != nil { + if err := fd.dial(ctx, laddr, raddr); err != nil { fd.Close() return nil, err } @@ -117,7 +120,7 @@ func (fd *netFD) addrFunc() func(syscall.Sockaddr) Addr { return func(syscall.Sockaddr) Addr { return nil } } -func (fd *netFD) dial(laddr, raddr sockaddr, deadline time.Time, cancel <-chan struct{}) error { +func (fd *netFD) dial(ctx context.Context, laddr, raddr sockaddr) error { var err error var lsa syscall.Sockaddr if laddr != nil { @@ -134,7 +137,7 @@ func (fd *netFD) dial(laddr, raddr sockaddr, deadline time.Time, cancel <-chan s if rsa, err = raddr.sockaddr(fd.family); err != nil { return err } - if err := fd.connect(lsa, rsa, deadline, cancel); err != nil { + if err := fd.connect(ctx, lsa, rsa); err != nil { return err } fd.isConnected = true diff --git a/libgo/go/net/sock_stub.go b/libgo/go/net/sock_stub.go index ed6b089489..5ac1e864f7 100644 --- a/libgo/go/net/sock_stub.go +++ b/libgo/go/net/sock_stub.go @@ -1,4 +1,4 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/sock_windows.go b/libgo/go/net/sock_windows.go index 888e70bb8e..89a3ca4258 100644 --- a/libgo/go/net/sock_windows.go +++ b/libgo/go/net/sock_windows.go @@ -1,4 +1,4 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/sockopt_bsd.go b/libgo/go/net/sockopt_bsd.go index 52b2a5debc..734a1093d6 100644 --- a/libgo/go/net/sockopt_bsd.go +++ b/libgo/go/net/sockopt_bsd.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -27,7 +27,7 @@ func setDefaultSockopts(s, family, sotype int, ipv6only bool) error { } if supportsIPv4map && family == syscall.AF_INET6 && sotype != syscall.SOCK_RAW { // Allow both IP versions even if the OS default - // is otherwise. Note that some operating systems + // is otherwise. Note that some operating systems // never admit this option. syscall.SetsockoptInt(s, syscall.IPPROTO_IPV6, syscall.IPV6_V6ONLY, boolint(ipv6only)) } diff --git a/libgo/go/net/sockopt_linux.go b/libgo/go/net/sockopt_linux.go index 54c20b1409..0f70b12407 100644 --- a/libgo/go/net/sockopt_linux.go +++ b/libgo/go/net/sockopt_linux.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -12,7 +12,7 @@ import ( func setDefaultSockopts(s, family, sotype int, ipv6only bool) error { if family == syscall.AF_INET6 && sotype != syscall.SOCK_RAW { // Allow both IP versions even if the OS default - // is otherwise. Note that some operating systems + // is otherwise. Note that some operating systems // never admit this option. syscall.SetsockoptInt(s, syscall.IPPROTO_IPV6, syscall.IPV6_V6ONLY, boolint(ipv6only)) } diff --git a/libgo/go/net/sockopt_plan9.go b/libgo/go/net/sockopt_plan9.go index 8bc689b6c2..02468cda97 100644 --- a/libgo/go/net/sockopt_plan9.go +++ b/libgo/go/net/sockopt_plan9.go @@ -1,9 +1,11 @@ -// Copyright 2014 The Go Authors. All rights reserved. +// Copyright 2014 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package net +import "syscall" + func setKeepAlive(fd *netFD, keepalive bool) error { if keepalive { _, e := fd.ctl.WriteAt([]byte("keepalive"), 0) @@ -11,3 +13,7 @@ func setKeepAlive(fd *netFD, keepalive bool) error { } return nil } + +func setLinger(fd *netFD, sec int) error { + return syscall.EPLAN9 +} diff --git a/libgo/go/net/sockopt_posix.go b/libgo/go/net/sockopt_posix.go index 1654d1b85e..cd3d562289 100644 --- a/libgo/go/net/sockopt_posix.go +++ b/libgo/go/net/sockopt_posix.go @@ -1,4 +1,4 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/sockopt_solaris.go b/libgo/go/net/sockopt_solaris.go index 54c20b1409..0f70b12407 100644 --- a/libgo/go/net/sockopt_solaris.go +++ b/libgo/go/net/sockopt_solaris.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -12,7 +12,7 @@ import ( func setDefaultSockopts(s, family, sotype int, ipv6only bool) error { if family == syscall.AF_INET6 && sotype != syscall.SOCK_RAW { // Allow both IP versions even if the OS default - // is otherwise. Note that some operating systems + // is otherwise. Note that some operating systems // never admit this option. syscall.SetsockoptInt(s, syscall.IPPROTO_IPV6, syscall.IPV6_V6ONLY, boolint(ipv6only)) } diff --git a/libgo/go/net/sockopt_stub.go b/libgo/go/net/sockopt_stub.go index de5ee0bb63..7e9e560e05 100644 --- a/libgo/go/net/sockopt_stub.go +++ b/libgo/go/net/sockopt_stub.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/sockopt_windows.go b/libgo/go/net/sockopt_windows.go index cb64a40c69..8017426521 100644 --- a/libgo/go/net/sockopt_windows.go +++ b/libgo/go/net/sockopt_windows.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -12,7 +12,7 @@ import ( func setDefaultSockopts(s syscall.Handle, family, sotype int, ipv6only bool) error { if family == syscall.AF_INET6 && sotype != syscall.SOCK_RAW { // Allow both IP versions even if the OS default - // is otherwise. Note that some operating systems + // is otherwise. Note that some operating systems // never admit this option. syscall.SetsockoptInt(s, syscall.IPPROTO_IPV6, syscall.IPV6_V6ONLY, boolint(ipv6only)) } diff --git a/libgo/go/net/sockoptip_bsd.go b/libgo/go/net/sockoptip_bsd.go index 2199e480d4..b15c6396ba 100644 --- a/libgo/go/net/sockoptip_bsd.go +++ b/libgo/go/net/sockoptip_bsd.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/sockoptip_linux.go b/libgo/go/net/sockoptip_linux.go index a69b778e4d..c1dcc911c7 100644 --- a/libgo/go/net/sockoptip_linux.go +++ b/libgo/go/net/sockoptip_linux.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/sockoptip_posix.go b/libgo/go/net/sockoptip_posix.go index c2579be911..d50886003e 100644 --- a/libgo/go/net/sockoptip_posix.go +++ b/libgo/go/net/sockoptip_posix.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/sockoptip_stub.go b/libgo/go/net/sockoptip_stub.go index 32ec5ddb85..f698687514 100644 --- a/libgo/go/net/sockoptip_stub.go +++ b/libgo/go/net/sockoptip_stub.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/sockoptip_windows.go b/libgo/go/net/sockoptip_windows.go index 7b11f207aa..916debebc6 100644 --- a/libgo/go/net/sockoptip_windows.go +++ b/libgo/go/net/sockoptip_windows.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/tcpsock.go b/libgo/go/net/tcpsock.go index 8765affd46..69731ebc91 100644 --- a/libgo/go/net/tcpsock.go +++ b/libgo/go/net/tcpsock.go @@ -1,9 +1,20 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package net +import ( + "context" + "io" + "os" + "syscall" + "time" +) + +// BUG(mikio): On Windows, the File method of TCPListener is not +// implemented. + // TCPAddr represents the address of a TCP end point. type TCPAddr struct { IP IP @@ -45,6 +56,9 @@ func (a *TCPAddr) opAddr() Addr { // "tcp6". A literal address or host name for IPv6 must be enclosed // in square brackets, as in "[::1]:80", "[ipv6-host]:http" or // "[ipv6-host%zone]:80". +// +// Resolving a hostname is not recommended because this returns at most +// one of its IP addresses. func ResolveTCPAddr(net, addr string) (*TCPAddr, error) { switch net { case "tcp", "tcp4", "tcp6": @@ -53,9 +67,234 @@ func ResolveTCPAddr(net, addr string) (*TCPAddr, error) { default: return nil, UnknownNetworkError(net) } - addrs, err := internetAddrList(net, addr, noDeadline) + addrs, err := DefaultResolver.internetAddrList(context.Background(), net, addr) if err != nil { return nil, err } return addrs.first(isIPv4).(*TCPAddr), nil } + +// TCPConn is an implementation of the Conn interface for TCP network +// connections. +type TCPConn struct { + conn +} + +// ReadFrom implements the io.ReaderFrom ReadFrom method. +func (c *TCPConn) ReadFrom(r io.Reader) (int64, error) { + if !c.ok() { + return 0, syscall.EINVAL + } + n, err := c.readFrom(r) + if err != nil && err != io.EOF { + err = &OpError{Op: "readfrom", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} + } + return n, err +} + +// CloseRead shuts down the reading side of the TCP connection. +// Most callers should just use Close. +func (c *TCPConn) CloseRead() error { + if !c.ok() { + return syscall.EINVAL + } + if err := c.fd.closeRead(); err != nil { + return &OpError{Op: "close", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} + } + return nil +} + +// CloseWrite shuts down the writing side of the TCP connection. +// Most callers should just use Close. +func (c *TCPConn) CloseWrite() error { + if !c.ok() { + return syscall.EINVAL + } + if err := c.fd.closeWrite(); err != nil { + return &OpError{Op: "close", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} + } + return nil +} + +// SetLinger sets the behavior of Close on a connection which still +// has data waiting to be sent or to be acknowledged. +// +// If sec < 0 (the default), the operating system finishes sending the +// data in the background. +// +// If sec == 0, the operating system discards any unsent or +// unacknowledged data. +// +// If sec > 0, the data is sent in the background as with sec < 0. On +// some operating systems after sec seconds have elapsed any remaining +// unsent data may be discarded. +func (c *TCPConn) SetLinger(sec int) error { + if !c.ok() { + return syscall.EINVAL + } + if err := setLinger(c.fd, sec); err != nil { + return &OpError{Op: "set", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} + } + return nil +} + +// SetKeepAlive sets whether the operating system should send +// keepalive messages on the connection. +func (c *TCPConn) SetKeepAlive(keepalive bool) error { + if !c.ok() { + return syscall.EINVAL + } + if err := setKeepAlive(c.fd, keepalive); err != nil { + return &OpError{Op: "set", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} + } + return nil +} + +// SetKeepAlivePeriod sets period between keep alives. +func (c *TCPConn) SetKeepAlivePeriod(d time.Duration) error { + if !c.ok() { + return syscall.EINVAL + } + if err := setKeepAlivePeriod(c.fd, d); err != nil { + return &OpError{Op: "set", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} + } + return nil +} + +// SetNoDelay controls whether the operating system should delay +// packet transmission in hopes of sending fewer packets (Nagle's +// algorithm). The default is true (no delay), meaning that data is +// sent as soon as possible after a Write. +func (c *TCPConn) SetNoDelay(noDelay bool) error { + if !c.ok() { + return syscall.EINVAL + } + if err := setNoDelay(c.fd, noDelay); err != nil { + return &OpError{Op: "set", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} + } + return nil +} + +func newTCPConn(fd *netFD) *TCPConn { + c := &TCPConn{conn{fd}} + setNoDelay(c.fd, true) + return c +} + +// DialTCP connects to the remote address raddr on the network net, +// which must be "tcp", "tcp4", or "tcp6". If laddr is not nil, it is +// used as the local address for the connection. +func DialTCP(net string, laddr, raddr *TCPAddr) (*TCPConn, error) { + switch net { + case "tcp", "tcp4", "tcp6": + default: + return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: UnknownNetworkError(net)} + } + if raddr == nil { + return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: nil, Err: errMissingAddress} + } + c, err := dialTCP(context.Background(), net, laddr, raddr) + if err != nil { + return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: err} + } + return c, nil +} + +// TCPListener is a TCP network listener. Clients should typically +// use variables of type Listener instead of assuming TCP. +type TCPListener struct { + fd *netFD +} + +// AcceptTCP accepts the next incoming call and returns the new +// connection. +func (l *TCPListener) AcceptTCP() (*TCPConn, error) { + if !l.ok() { + return nil, syscall.EINVAL + } + c, err := l.accept() + if err != nil { + return nil, &OpError{Op: "accept", Net: l.fd.net, Source: nil, Addr: l.fd.laddr, Err: err} + } + return c, nil +} + +// Accept implements the Accept method in the Listener interface; it +// waits for the next call and returns a generic Conn. +func (l *TCPListener) Accept() (Conn, error) { + if !l.ok() { + return nil, syscall.EINVAL + } + c, err := l.accept() + if err != nil { + return nil, &OpError{Op: "accept", Net: l.fd.net, Source: nil, Addr: l.fd.laddr, Err: err} + } + return c, nil +} + +// Close stops listening on the TCP address. +// Already Accepted connections are not closed. +func (l *TCPListener) Close() error { + if !l.ok() { + return syscall.EINVAL + } + if err := l.close(); err != nil { + return &OpError{Op: "close", Net: l.fd.net, Source: nil, Addr: l.fd.laddr, Err: err} + } + return nil +} + +// Addr returns the listener's network address, a *TCPAddr. +// The Addr returned is shared by all invocations of Addr, so +// do not modify it. +func (l *TCPListener) Addr() Addr { return l.fd.laddr } + +// SetDeadline sets the deadline associated with the listener. +// A zero time value disables the deadline. +func (l *TCPListener) SetDeadline(t time.Time) error { + if !l.ok() { + return syscall.EINVAL + } + if err := l.fd.setDeadline(t); err != nil { + return &OpError{Op: "set", Net: l.fd.net, Source: nil, Addr: l.fd.laddr, Err: err} + } + return nil +} + +// File returns a copy of the underlying os.File, set to blocking +// mode. It is the caller's responsibility to close f when finished. +// Closing l does not affect f, and closing f does not affect l. +// +// The returned os.File's file descriptor is different from the +// connection's. Attempting to change properties of the original +// using this duplicate may or may not have the desired effect. +func (l *TCPListener) File() (f *os.File, err error) { + if !l.ok() { + return nil, syscall.EINVAL + } + f, err = l.file() + if err != nil { + return nil, &OpError{Op: "file", Net: l.fd.net, Source: nil, Addr: l.fd.laddr, Err: err} + } + return +} + +// ListenTCP announces on the TCP address laddr and returns a TCP +// listener. Net must be "tcp", "tcp4", or "tcp6". If laddr has a +// port of 0, ListenTCP will choose an available port. The caller can +// use the Addr method of TCPListener to retrieve the chosen address. +func ListenTCP(net string, laddr *TCPAddr) (*TCPListener, error) { + switch net { + case "tcp", "tcp4", "tcp6": + default: + return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: UnknownNetworkError(net)} + } + if laddr == nil { + laddr = &TCPAddr{} + } + ln, err := listenTCP(context.Background(), net, laddr) + if err != nil { + return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: err} + } + return ln, nil +} diff --git a/libgo/go/net/tcpsock_plan9.go b/libgo/go/net/tcpsock_plan9.go index afccbfe8a7..d2860607f8 100644 --- a/libgo/go/net/tcpsock_plan9.go +++ b/libgo/go/net/tcpsock_plan9.go @@ -1,230 +1,73 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package net import ( + "context" "io" "os" - "syscall" - "time" ) -// TCPConn is an implementation of the Conn interface for TCP network -// connections. -type TCPConn struct { - conn +func (c *TCPConn) readFrom(r io.Reader) (int64, error) { + return genericReadFrom(c, r) } -func newTCPConn(fd *netFD) *TCPConn { - return &TCPConn{conn{fd}} -} - -// ReadFrom implements the io.ReaderFrom ReadFrom method. -func (c *TCPConn) ReadFrom(r io.Reader) (int64, error) { - n, err := genericReadFrom(c, r) - if err != nil && err != io.EOF { - err = &OpError{Op: "read", Net: c.fd.dir, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} - } - return n, err -} - -// CloseRead shuts down the reading side of the TCP connection. -// Most callers should just use Close. -func (c *TCPConn) CloseRead() error { - if !c.ok() { - return syscall.EINVAL - } - err := c.fd.closeRead() - if err != nil { - err = &OpError{Op: "close", Net: c.fd.dir, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} +func dialTCP(ctx context.Context, net string, laddr, raddr *TCPAddr) (*TCPConn, error) { + if testHookDialTCP != nil { + return testHookDialTCP(ctx, net, laddr, raddr) } - return err -} - -// CloseWrite shuts down the writing side of the TCP connection. -// Most callers should just use Close. -func (c *TCPConn) CloseWrite() error { - if !c.ok() { - return syscall.EINVAL - } - err := c.fd.closeWrite() - if err != nil { - err = &OpError{Op: "close", Net: c.fd.dir, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} - } - return err -} - -// SetLinger sets the behavior of Close on a connection which still -// has data waiting to be sent or to be acknowledged. -// -// If sec < 0 (the default), the operating system finishes sending the -// data in the background. -// -// If sec == 0, the operating system discards any unsent or -// unacknowledged data. -// -// If sec > 0, the data is sent in the background as with sec < 0. On -// some operating systems after sec seconds have elapsed any remaining -// unsent data may be discarded. -func (c *TCPConn) SetLinger(sec int) error { - return &OpError{Op: "set", Net: c.fd.dir, Source: c.fd.laddr, Addr: c.fd.raddr, Err: syscall.EPLAN9} -} - -// SetKeepAlive sets whether the operating system should send -// keepalive messages on the connection. -func (c *TCPConn) SetKeepAlive(keepalive bool) error { - if !c.ok() { - return syscall.EPLAN9 - } - if err := setKeepAlive(c.fd, keepalive); err != nil { - return &OpError{Op: "set", Net: c.fd.dir, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} - } - return nil -} - -// SetKeepAlivePeriod sets period between keep alives. -func (c *TCPConn) SetKeepAlivePeriod(d time.Duration) error { - if !c.ok() { - return syscall.EPLAN9 - } - if err := setKeepAlivePeriod(c.fd, d); err != nil { - return &OpError{Op: "set", Net: c.fd.dir, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} - } - return nil -} - -// SetNoDelay controls whether the operating system should delay -// packet transmission in hopes of sending fewer packets (Nagle's -// algorithm). The default is true (no delay), meaning that data is -// sent as soon as possible after a Write. -func (c *TCPConn) SetNoDelay(noDelay bool) error { - return &OpError{Op: "set", Net: c.fd.dir, Source: c.fd.laddr, Addr: c.fd.raddr, Err: syscall.EPLAN9} -} - -// DialTCP connects to the remote address raddr on the network net, -// which must be "tcp", "tcp4", or "tcp6". If laddr is not nil, it is -// used as the local address for the connection. -func DialTCP(net string, laddr, raddr *TCPAddr) (*TCPConn, error) { - return dialTCP(net, laddr, raddr, noDeadline, noCancel) + return doDialTCP(ctx, net, laddr, raddr) } -func dialTCP(net string, laddr, raddr *TCPAddr, deadline time.Time, cancel <-chan struct{}) (*TCPConn, error) { - if !deadline.IsZero() { - panic("net.dialTCP: deadline not implemented on Plan 9") - } - // TODO(bradfitz,0intro): also use the cancel channel. +func doDialTCP(ctx context.Context, net string, laddr, raddr *TCPAddr) (*TCPConn, error) { switch net { case "tcp", "tcp4", "tcp6": default: - return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: UnknownNetworkError(net)} + return nil, UnknownNetworkError(net) } if raddr == nil { - return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: nil, Err: errMissingAddress} + return nil, errMissingAddress } - fd, err := dialPlan9(net, laddr, raddr) + fd, err := dialPlan9(ctx, net, laddr, raddr) if err != nil { return nil, err } return newTCPConn(fd), nil } -// TCPListener is a TCP network listener. Clients should typically -// use variables of type Listener instead of assuming TCP. -type TCPListener struct { - fd *netFD -} +func (ln *TCPListener) ok() bool { return ln != nil && ln.fd != nil && ln.fd.ctl != nil } -// AcceptTCP accepts the next incoming call and returns the new -// connection. -func (l *TCPListener) AcceptTCP() (*TCPConn, error) { - if l == nil || l.fd == nil || l.fd.ctl == nil { - return nil, syscall.EINVAL - } - fd, err := l.fd.acceptPlan9() +func (ln *TCPListener) accept() (*TCPConn, error) { + fd, err := ln.fd.acceptPlan9() if err != nil { return nil, err } return newTCPConn(fd), nil } -// Accept implements the Accept method in the Listener interface; it -// waits for the next call and returns a generic Conn. -func (l *TCPListener) Accept() (Conn, error) { - if l == nil || l.fd == nil || l.fd.ctl == nil { - return nil, syscall.EINVAL +func (ln *TCPListener) close() error { + if _, err := ln.fd.ctl.WriteString("hangup"); err != nil { + ln.fd.ctl.Close() + return err } - c, err := l.AcceptTCP() - if err != nil { - return nil, err - } - return c, nil -} - -// Close stops listening on the TCP address. -// Already Accepted connections are not closed. -func (l *TCPListener) Close() error { - if l == nil || l.fd == nil || l.fd.ctl == nil { - return syscall.EINVAL - } - if _, err := l.fd.ctl.WriteString("hangup"); err != nil { - l.fd.ctl.Close() - return &OpError{Op: "close", Net: l.fd.dir, Source: nil, Addr: l.fd.laddr, Err: err} - } - err := l.fd.ctl.Close() - if err != nil { - err = &OpError{Op: "close", Net: l.fd.dir, Source: nil, Addr: l.fd.laddr, Err: err} - } - return err -} - -// Addr returns the listener's network address, a *TCPAddr. -// The Addr returned is shared by all invocations of Addr, so -// do not modify it. -func (l *TCPListener) Addr() Addr { return l.fd.laddr } - -// SetDeadline sets the deadline associated with the listener. -// A zero time value disables the deadline. -func (l *TCPListener) SetDeadline(t time.Time) error { - if l == nil || l.fd == nil || l.fd.ctl == nil { - return syscall.EINVAL - } - if err := l.fd.setDeadline(t); err != nil { - return &OpError{Op: "set", Net: l.fd.dir, Source: nil, Addr: l.fd.laddr, Err: err} + if err := ln.fd.ctl.Close(); err != nil { + return err } return nil } -// File returns a copy of the underlying os.File, set to blocking -// mode. It is the caller's responsibility to close f when finished. -// Closing l does not affect f, and closing f does not affect l. -// -// The returned os.File's file descriptor is different from the -// connection's. Attempting to change properties of the original -// using this duplicate may or may not have the desired effect. -func (l *TCPListener) File() (f *os.File, err error) { - f, err = l.dup() +func (ln *TCPListener) file() (*os.File, error) { + f, err := ln.dup() if err != nil { - err = &OpError{Op: "file", Net: l.fd.dir, Source: nil, Addr: l.fd.laddr, Err: err} + return nil, err } - return + return f, nil } -// ListenTCP announces on the TCP address laddr and returns a TCP -// listener. Net must be "tcp", "tcp4", or "tcp6". If laddr has a -// port of 0, ListenTCP will choose an available port. The caller can -// use the Addr method of TCPListener to retrieve the chosen address. -func ListenTCP(net string, laddr *TCPAddr) (*TCPListener, error) { - switch net { - case "tcp", "tcp4", "tcp6": - default: - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: UnknownNetworkError(net)} - } - if laddr == nil { - laddr = &TCPAddr{} - } - fd, err := listenPlan9(net, laddr) +func listenTCP(ctx context.Context, network string, laddr *TCPAddr) (*TCPListener, error) { + fd, err := listenPlan9(ctx, network, laddr) if err != nil { return nil, err } diff --git a/libgo/go/net/tcpsock_posix.go b/libgo/go/net/tcpsock_posix.go index 0e12d54300..9641e5c028 100644 --- a/libgo/go/net/tcpsock_posix.go +++ b/libgo/go/net/tcpsock_posix.go @@ -1,4 +1,4 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -7,10 +7,10 @@ package net import ( + "context" "io" "os" "syscall" - "time" ) func sockaddrToTCP(sa syscall.Sockaddr) Addr { @@ -40,151 +40,42 @@ func (a *TCPAddr) sockaddr(family int) (syscall.Sockaddr, error) { return ipToSockaddr(family, a.IP, a.Port, a.Zone) } -// TCPConn is an implementation of the Conn interface for TCP network -// connections. -type TCPConn struct { - conn +func (a *TCPAddr) toLocal(net string) sockaddr { + return &TCPAddr{loopbackIP(net), a.Port, a.Zone} } -func newTCPConn(fd *netFD) *TCPConn { - c := &TCPConn{conn{fd}} - setNoDelay(c.fd, true) - return c -} - -// ReadFrom implements the io.ReaderFrom ReadFrom method. -func (c *TCPConn) ReadFrom(r io.Reader) (int64, error) { +func (c *TCPConn) readFrom(r io.Reader) (int64, error) { if n, err, handled := sendFile(c.fd, r); handled { - if err != nil && err != io.EOF { - err = &OpError{Op: "read", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} - } return n, err } - n, err := genericReadFrom(c, r) - if err != nil && err != io.EOF { - err = &OpError{Op: "read", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} - } - return n, err + return genericReadFrom(c, r) } -// CloseRead shuts down the reading side of the TCP connection. -// Most callers should just use Close. -func (c *TCPConn) CloseRead() error { - if !c.ok() { - return syscall.EINVAL +func dialTCP(ctx context.Context, net string, laddr, raddr *TCPAddr) (*TCPConn, error) { + if testHookDialTCP != nil { + return testHookDialTCP(ctx, net, laddr, raddr) } - err := c.fd.closeRead() - if err != nil { - err = &OpError{Op: "close", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} - } - return err + return doDialTCP(ctx, net, laddr, raddr) } -// CloseWrite shuts down the writing side of the TCP connection. -// Most callers should just use Close. -func (c *TCPConn) CloseWrite() error { - if !c.ok() { - return syscall.EINVAL - } - err := c.fd.closeWrite() - if err != nil { - err = &OpError{Op: "close", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} - } - return err -} - -// SetLinger sets the behavior of Close on a connection which still -// has data waiting to be sent or to be acknowledged. -// -// If sec < 0 (the default), the operating system finishes sending the -// data in the background. -// -// If sec == 0, the operating system discards any unsent or -// unacknowledged data. -// -// If sec > 0, the data is sent in the background as with sec < 0. On -// some operating systems after sec seconds have elapsed any remaining -// unsent data may be discarded. -func (c *TCPConn) SetLinger(sec int) error { - if !c.ok() { - return syscall.EINVAL - } - if err := setLinger(c.fd, sec); err != nil { - return &OpError{Op: "set", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} - } - return nil -} - -// SetKeepAlive sets whether the operating system should send -// keepalive messages on the connection. -func (c *TCPConn) SetKeepAlive(keepalive bool) error { - if !c.ok() { - return syscall.EINVAL - } - if err := setKeepAlive(c.fd, keepalive); err != nil { - return &OpError{Op: "set", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} - } - return nil -} - -// SetKeepAlivePeriod sets period between keep alives. -func (c *TCPConn) SetKeepAlivePeriod(d time.Duration) error { - if !c.ok() { - return syscall.EINVAL - } - if err := setKeepAlivePeriod(c.fd, d); err != nil { - return &OpError{Op: "set", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} - } - return nil -} - -// SetNoDelay controls whether the operating system should delay -// packet transmission in hopes of sending fewer packets (Nagle's -// algorithm). The default is true (no delay), meaning that data is -// sent as soon as possible after a Write. -func (c *TCPConn) SetNoDelay(noDelay bool) error { - if !c.ok() { - return syscall.EINVAL - } - if err := setNoDelay(c.fd, noDelay); err != nil { - return &OpError{Op: "set", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} - } - return nil -} - -// DialTCP connects to the remote address raddr on the network net, -// which must be "tcp", "tcp4", or "tcp6". If laddr is not nil, it is -// used as the local address for the connection. -func DialTCP(net string, laddr, raddr *TCPAddr) (*TCPConn, error) { - switch net { - case "tcp", "tcp4", "tcp6": - default: - return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: UnknownNetworkError(net)} - } - if raddr == nil { - return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: nil, Err: errMissingAddress} - } - return dialTCP(net, laddr, raddr, noDeadline, noCancel) -} - -func dialTCP(net string, laddr, raddr *TCPAddr, deadline time.Time, cancel <-chan struct{}) (*TCPConn, error) { - fd, err := internetSocket(net, laddr, raddr, deadline, syscall.SOCK_STREAM, 0, "dial", cancel) +func doDialTCP(ctx context.Context, net string, laddr, raddr *TCPAddr) (*TCPConn, error) { + fd, err := internetSocket(ctx, net, laddr, raddr, syscall.SOCK_STREAM, 0, "dial") // TCP has a rarely used mechanism called a 'simultaneous connection' in // which Dial("tcp", addr1, addr2) run on the machine at addr1 can // connect to a simultaneous Dial("tcp", addr2, addr1) run on the machine - // at addr2, without either machine executing Listen. If laddr == nil, + // at addr2, without either machine executing Listen. If laddr == nil, // it means we want the kernel to pick an appropriate originating local - // address. Some Linux kernels cycle blindly through a fixed range of - // local ports, regardless of destination port. If a kernel happens to + // address. Some Linux kernels cycle blindly through a fixed range of + // local ports, regardless of destination port. If a kernel happens to // pick local port 50001 as the source for a Dial("tcp", "", "localhost:50001"), // then the Dial will succeed, having simultaneously connected to itself. // This can only happen when we are letting the kernel pick a port (laddr == nil) // and when there is no listener for the destination address. - // It's hard to argue this is anything other than a kernel bug. If we + // It's hard to argue this is anything other than a kernel bug. If we // see this happen, rather than expose the buggy effect to users, we - // close the fd and try again. If it happens twice more, we relent and - // use the result. See also: + // close the fd and try again. If it happens twice more, we relent and + // use the result. See also: // https://golang.org/issue/2690 // http://stackoverflow.com/questions/4949858/ // @@ -198,11 +89,11 @@ func dialTCP(net string, laddr, raddr *TCPAddr, deadline time.Time, cancel <-cha if err == nil { fd.Close() } - fd, err = internetSocket(net, laddr, raddr, deadline, syscall.SOCK_STREAM, 0, "dial", cancel) + fd, err = internetSocket(ctx, net, laddr, raddr, syscall.SOCK_STREAM, 0, "dial") } if err != nil { - return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: err} + return nil, err } return newTCPConn(fd), nil } @@ -239,96 +130,32 @@ func spuriousENOTAVAIL(err error) bool { return err == syscall.EADDRNOTAVAIL } -// TCPListener is a TCP network listener. Clients should typically -// use variables of type Listener instead of assuming TCP. -type TCPListener struct { - fd *netFD -} - -// AcceptTCP accepts the next incoming call and returns the new -// connection. -func (l *TCPListener) AcceptTCP() (*TCPConn, error) { - if l == nil || l.fd == nil { - return nil, syscall.EINVAL - } - fd, err := l.fd.accept() - if err != nil { - return nil, &OpError{Op: "accept", Net: l.fd.net, Source: nil, Addr: l.fd.laddr, Err: err} - } - return newTCPConn(fd), nil -} +func (ln *TCPListener) ok() bool { return ln != nil && ln.fd != nil } -// Accept implements the Accept method in the Listener interface; it -// waits for the next call and returns a generic Conn. -func (l *TCPListener) Accept() (Conn, error) { - c, err := l.AcceptTCP() +func (ln *TCPListener) accept() (*TCPConn, error) { + fd, err := ln.fd.accept() if err != nil { return nil, err } - return c, nil -} - -// Close stops listening on the TCP address. -// Already Accepted connections are not closed. -func (l *TCPListener) Close() error { - if l == nil || l.fd == nil { - return syscall.EINVAL - } - err := l.fd.Close() - if err != nil { - err = &OpError{Op: "close", Net: l.fd.net, Source: nil, Addr: l.fd.laddr, Err: err} - } - return err + return newTCPConn(fd), nil } -// Addr returns the listener's network address, a *TCPAddr. -// The Addr returned is shared by all invocations of Addr, so -// do not modify it. -func (l *TCPListener) Addr() Addr { return l.fd.laddr } - -// SetDeadline sets the deadline associated with the listener. -// A zero time value disables the deadline. -func (l *TCPListener) SetDeadline(t time.Time) error { - if l == nil || l.fd == nil { - return syscall.EINVAL - } - if err := l.fd.setDeadline(t); err != nil { - return &OpError{Op: "set", Net: l.fd.net, Source: nil, Addr: l.fd.laddr, Err: err} - } - return nil +func (ln *TCPListener) close() error { + return ln.fd.Close() } -// File returns a copy of the underlying os.File, set to blocking -// mode. It is the caller's responsibility to close f when finished. -// Closing l does not affect f, and closing f does not affect l. -// -// The returned os.File's file descriptor is different from the -// connection's. Attempting to change properties of the original -// using this duplicate may or may not have the desired effect. -func (l *TCPListener) File() (f *os.File, err error) { - f, err = l.fd.dup() +func (ln *TCPListener) file() (*os.File, error) { + f, err := ln.fd.dup() if err != nil { - err = &OpError{Op: "file", Net: l.fd.net, Source: nil, Addr: l.fd.laddr, Err: err} + return nil, err } - return + return f, nil } -// ListenTCP announces on the TCP address laddr and returns a TCP -// listener. Net must be "tcp", "tcp4", or "tcp6". If laddr has a -// port of 0, ListenTCP will choose an available port. The caller can -// use the Addr method of TCPListener to retrieve the chosen address. -func ListenTCP(net string, laddr *TCPAddr) (*TCPListener, error) { - switch net { - case "tcp", "tcp4", "tcp6": - default: - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: UnknownNetworkError(net)} - } - if laddr == nil { - laddr = &TCPAddr{} - } - fd, err := internetSocket(net, laddr, nil, noDeadline, syscall.SOCK_STREAM, 0, "listen", noCancel) +func listenTCP(ctx context.Context, network string, laddr *TCPAddr) (*TCPListener, error) { + fd, err := internetSocket(ctx, network, laddr, nil, syscall.SOCK_STREAM, 0, "listen") if err != nil { - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr, Err: err} + return nil, err } return &TCPListener{fd}, nil } diff --git a/libgo/go/net/tcp_test.go b/libgo/go/net/tcpsock_test.go index 9da6f6cdd0..51154221d0 100644 --- a/libgo/go/net/tcp_test.go +++ b/libgo/go/net/tcpsock_test.go @@ -5,6 +5,8 @@ package net import ( + "fmt" + "internal/testenv" "io" "reflect" "runtime" @@ -309,6 +311,17 @@ var resolveTCPAddrTests = []resolveTCPAddrTest{ {"tcp", ":12345", &TCPAddr{Port: 12345}, nil}, {"http", "127.0.0.1:0", nil, UnknownNetworkError("http")}, + + {"tcp", "127.0.0.1:http", &TCPAddr{IP: ParseIP("127.0.0.1"), Port: 80}, nil}, + {"tcp", "[::ffff:127.0.0.1]:http", &TCPAddr{IP: ParseIP("::ffff:127.0.0.1"), Port: 80}, nil}, + {"tcp", "[2001:db8::1]:http", &TCPAddr{IP: ParseIP("2001:db8::1"), Port: 80}, nil}, + {"tcp4", "127.0.0.1:http", &TCPAddr{IP: ParseIP("127.0.0.1"), Port: 80}, nil}, + {"tcp4", "[::ffff:127.0.0.1]:http", &TCPAddr{IP: ParseIP("127.0.0.1"), Port: 80}, nil}, + {"tcp6", "[2001:db8::1]:http", &TCPAddr{IP: ParseIP("2001:db8::1"), Port: 80}, nil}, + + {"tcp4", "[2001:db8::1]:http", nil, &AddrError{Err: errNoSuitableAddress.Error(), Addr: "2001:db8::1"}}, + {"tcp6", "127.0.0.1:http", nil, &AddrError{Err: errNoSuitableAddress.Error(), Addr: "127.0.0.1"}}, + {"tcp6", "[::ffff:127.0.0.1]:http", nil, &AddrError{Err: errNoSuitableAddress.Error(), Addr: "::ffff:127.0.0.1"}}, } func TestResolveTCPAddr(t *testing.T) { @@ -316,21 +329,17 @@ func TestResolveTCPAddr(t *testing.T) { defer func() { testHookLookupIP = origTestHookLookupIP }() testHookLookupIP = lookupLocalhost - for i, tt := range resolveTCPAddrTests { + for _, tt := range resolveTCPAddrTests { addr, err := ResolveTCPAddr(tt.network, tt.litAddrOrName) - if err != tt.err { - t.Errorf("#%d: %v", i, err) - } else if !reflect.DeepEqual(addr, tt.addr) { - t.Errorf("#%d: got %#v; want %#v", i, addr, tt.addr) - } - if err != nil { + if !reflect.DeepEqual(addr, tt.addr) || !reflect.DeepEqual(err, tt.err) { + t.Errorf("ResolveTCPAddr(%q, %q) = %#v, %v, want %#v, %v", tt.network, tt.litAddrOrName, addr, err, tt.addr, tt.err) continue } - rtaddr, err := ResolveTCPAddr(addr.Network(), addr.String()) - if err != nil { - t.Errorf("#%d: %v", i, err) - } else if !reflect.DeepEqual(rtaddr, addr) { - t.Errorf("#%d: got %#v; want %#v", i, rtaddr, addr) + if err == nil { + addr2, err := ResolveTCPAddr(addr.Network(), addr.String()) + if !reflect.DeepEqual(addr2, tt.addr) || err != tt.err { + t.Errorf("(%q, %q): ResolveTCPAddr(%q, %q) = %#v, %v, want %#v, %v", tt.network, tt.litAddrOrName, addr.Network(), addr.String(), addr2, err, tt.addr, tt.err) + } } } } @@ -345,9 +354,7 @@ var tcpListenerNameTests = []struct { } func TestTCPListenerName(t *testing.T) { - if testing.Short() || !*testExternal { - t.Skip("avoid external network") - } + testenv.MustHaveExternalNetwork(t) for _, tt := range tcpListenerNameTests { ln, err := ListenTCP(tt.net, tt.laddr) @@ -363,9 +370,8 @@ func TestTCPListenerName(t *testing.T) { } func TestIPv6LinkLocalUnicastTCP(t *testing.T) { - if testing.Short() || !*testExternal { - t.Skip("avoid external network") - } + testenv.MustHaveExternalNetwork(t) + if !supportsIPv6 { t.Skip("IPv6 is not supported") } @@ -461,13 +467,19 @@ func TestTCPConcurrentAccept(t *testing.T) { } func TestTCPReadWriteAllocs(t *testing.T) { - t.Skip("skipping test on gccgo until escape analysis is turned on") + if runtime.Compiler == "gccgo" { + t.Skip("skipping for gccgo until escape analysis is enabled") + } + switch runtime.GOOS { - case "nacl", "windows": + case "plan9": + // The implementation of asynchronous cancelable + // I/O on Plan 9 allocates memory. + // See net/fd_io_plan9.go. + t.Skipf("not supported on %s", runtime.GOOS) + case "nacl": // NaCl needs to allocate pseudo file descriptor // stuff. See syscall/fd_nacl.go. - // Windows uses closures and channels for IO - // completion port-based netpoll. See fd_windows.go. t.Skipf("not supported on %s", runtime.GOOS) } @@ -477,7 +489,7 @@ func TestTCPReadWriteAllocs(t *testing.T) { } defer ln.Close() var server Conn - errc := make(chan error) + errc := make(chan error, 1) go func() { var err error server, err = ln.Accept() @@ -492,6 +504,7 @@ func TestTCPReadWriteAllocs(t *testing.T) { t.Fatal(err) } defer server.Close() + var buf [128]byte allocs := testing.AllocsPerRun(1000, func() { _, err := server.Write(buf[:]) @@ -503,6 +516,29 @@ func TestTCPReadWriteAllocs(t *testing.T) { t.Fatal(err) } }) + // For gccgo changed "> 0" to "> 7". + if allocs > 7 { + t.Fatalf("got %v; want 0", allocs) + } + + var bufwrt [128]byte + ch := make(chan bool) + defer close(ch) + go func() { + for <-ch { + _, err := server.Write(bufwrt[:]) + errc <- err + } + }() + allocs = testing.AllocsPerRun(1000, func() { + ch <- true + if _, err = io.ReadFull(client, buf[:]); err != nil { + t.Fatal(err) + } + if err := <-errc; err != nil { + t.Fatal(err) + } + }) if allocs > 0 { t.Fatalf("got %v; want 0", allocs) } @@ -589,3 +625,105 @@ func TestTCPStress(t *testing.T) { ln.Close() <-done } + +func TestTCPSelfConnect(t *testing.T) { + if runtime.GOOS == "windows" { + // TODO(brainman): do not know why it hangs. + t.Skip("known-broken test on windows") + } + + ln, err := newLocalListener("tcp") + if err != nil { + t.Fatal(err) + } + var d Dialer + c, err := d.Dial(ln.Addr().Network(), ln.Addr().String()) + if err != nil { + ln.Close() + t.Fatal(err) + } + network := c.LocalAddr().Network() + laddr := *c.LocalAddr().(*TCPAddr) + c.Close() + ln.Close() + + // Try to connect to that address repeatedly. + n := 100000 + if testing.Short() { + n = 1000 + } + switch runtime.GOOS { + case "darwin", "dragonfly", "freebsd", "netbsd", "openbsd", "plan9", "solaris", "windows": + // Non-Linux systems take a long time to figure + // out that there is nothing listening on localhost. + n = 100 + } + for i := 0; i < n; i++ { + d.Timeout = time.Millisecond + c, err := d.Dial(network, laddr.String()) + if err == nil { + addr := c.LocalAddr().(*TCPAddr) + if addr.Port == laddr.Port || addr.IP.Equal(laddr.IP) { + t.Errorf("Dial %v should fail", addr) + } else { + t.Logf("Dial %v succeeded - possibly racing with other listener", addr) + } + c.Close() + } + } +} + +// Test that >32-bit reads work on 64-bit systems. +// On 32-bit systems this tests that maxint reads work. +func TestTCPBig(t *testing.T) { + if !*testTCPBig { + t.Skip("test disabled; use -tcpbig to enable") + } + + for _, writev := range []bool{false, true} { + t.Run(fmt.Sprintf("writev=%v", writev), func(t *testing.T) { + ln, err := newLocalListener("tcp") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + + x := int(1 << 30) + x = x*5 + 1<<20 // just over 5 GB on 64-bit, just over 1GB on 32-bit + done := make(chan int) + go func() { + defer close(done) + c, err := ln.Accept() + if err != nil { + t.Error(err) + return + } + buf := make([]byte, x) + var n int + if writev { + var n64 int64 + n64, err = (&Buffers{buf}).WriteTo(c) + n = int(n64) + } else { + n, err = c.Write(buf) + } + if n != len(buf) || err != nil { + t.Errorf("Write(buf) = %d, %v, want %d, nil", n, err, x) + } + c.Close() + }() + + c, err := Dial("tcp", ln.Addr().String()) + if err != nil { + t.Fatal(err) + } + buf := make([]byte, x) + n, err := io.ReadFull(c, buf) + if n != len(buf) || err != nil { + t.Errorf("Read(buf) = %d, %v, want %d, nil", n, err, x) + } + c.Close() + <-done + }) + } +} diff --git a/libgo/go/net/tcpsock_unix_test.go b/libgo/go/net/tcpsock_unix_test.go new file mode 100644 index 0000000000..2375fe24dc --- /dev/null +++ b/libgo/go/net/tcpsock_unix_test.go @@ -0,0 +1,79 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin + +package net + +import ( + "runtime" + "sync" + "syscall" + "testing" + "time" +) + +// See golang.org/issue/14548. +func TestTCPSpuriousConnSetupCompletion(t *testing.T) { + if testing.Short() { + t.Skip("skipping in short mode") + } + + ln, err := newLocalListener("tcp") + if err != nil { + t.Fatal(err) + } + var wg sync.WaitGroup + wg.Add(1) + go func(ln Listener) { + defer wg.Done() + for { + c, err := ln.Accept() + if err != nil { + return + } + wg.Add(1) + go func(c Conn) { + var b [1]byte + c.Read(b[:]) + c.Close() + wg.Done() + }(c) + } + }(ln) + + attempts := int(1e4) // larger is better + wg.Add(attempts) + throttle := make(chan struct{}, runtime.GOMAXPROCS(-1)*2) + for i := 0; i < attempts; i++ { + throttle <- struct{}{} + go func(i int) { + defer func() { + <-throttle + wg.Done() + }() + d := Dialer{Timeout: 50 * time.Millisecond} + c, err := d.Dial(ln.Addr().Network(), ln.Addr().String()) + if err != nil { + if perr := parseDialError(err); perr != nil { + t.Errorf("#%d: %v (original error: %v)", i, perr, err) + } + return + } + var b [1]byte + if _, err := c.Write(b[:]); err != nil { + if perr := parseWriteError(err); perr != nil { + t.Errorf("#%d: %v", i, err) + } + if samePlatformError(err, syscall.ENOTCONN) { + t.Errorf("#%d: %v", i, err) + } + } + c.Close() + }(i) + } + + ln.Close() + wg.Wait() +} diff --git a/libgo/go/net/tcpsockopt_darwin.go b/libgo/go/net/tcpsockopt_darwin.go index 1f1609088b..0d1310eaf9 100644 --- a/libgo/go/net/tcpsockopt_darwin.go +++ b/libgo/go/net/tcpsockopt_darwin.go @@ -1,4 +1,4 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/tcpsockopt_dragonfly.go b/libgo/go/net/tcpsockopt_dragonfly.go index 0aa213239d..7cc716bad1 100644 --- a/libgo/go/net/tcpsockopt_dragonfly.go +++ b/libgo/go/net/tcpsockopt_dragonfly.go @@ -1,4 +1,4 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/tcpsockopt_openbsd.go b/libgo/go/net/tcpsockopt_openbsd.go index 041e1786a9..10e1bef3e5 100644 --- a/libgo/go/net/tcpsockopt_openbsd.go +++ b/libgo/go/net/tcpsockopt_openbsd.go @@ -1,4 +1,4 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/tcpsockopt_plan9.go b/libgo/go/net/tcpsockopt_plan9.go index 157282abd3..fb56871857 100644 --- a/libgo/go/net/tcpsockopt_plan9.go +++ b/libgo/go/net/tcpsockopt_plan9.go @@ -1,4 +1,4 @@ -// Copyright 2014 The Go Authors. All rights reserved. +// Copyright 2014 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -7,9 +7,14 @@ package net import ( + "syscall" "time" ) +func setNoDelay(fd *netFD, noDelay bool) error { + return syscall.EPLAN9 +} + // Set keep alive period. func setKeepAlivePeriod(fd *netFD, d time.Duration) error { cmd := "keepalive " + itoa(int(d/time.Millisecond)) diff --git a/libgo/go/net/tcpsockopt_posix.go b/libgo/go/net/tcpsockopt_posix.go index 0abf3f97f6..805b56b5c7 100644 --- a/libgo/go/net/tcpsockopt_posix.go +++ b/libgo/go/net/tcpsockopt_posix.go @@ -1,4 +1,4 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/tcpsockopt_solaris.go b/libgo/go/net/tcpsockopt_solaris.go index eaab6b6787..347c17d7e2 100644 --- a/libgo/go/net/tcpsockopt_solaris.go +++ b/libgo/go/net/tcpsockopt_solaris.go @@ -1,4 +1,4 @@ -// Copyright 2013 The Go Authors. All rights reserved. +// Copyright 2015 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/tcpsockopt_stub.go b/libgo/go/net/tcpsockopt_stub.go index b413a764d8..19c83e6a5f 100644 --- a/libgo/go/net/tcpsockopt_stub.go +++ b/libgo/go/net/tcpsockopt_stub.go @@ -1,4 +1,4 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/tcpsockopt_unix.go b/libgo/go/net/tcpsockopt_unix.go index c8970d1b57..8d44fb2095 100644 --- a/libgo/go/net/tcpsockopt_unix.go +++ b/libgo/go/net/tcpsockopt_unix.go @@ -1,4 +1,4 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/tcpsockopt_windows.go b/libgo/go/net/tcpsockopt_windows.go index ae2d7c8f18..45a4dca525 100644 --- a/libgo/go/net/tcpsockopt_windows.go +++ b/libgo/go/net/tcpsockopt_windows.go @@ -1,4 +1,4 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/testdata/Mark.Twain-Tom.Sawyer.txt b/libgo/go/net/testdata/Mark.Twain-Tom.Sawyer.txt new file mode 100644 index 0000000000..c9106fd522 --- /dev/null +++ b/libgo/go/net/testdata/Mark.Twain-Tom.Sawyer.txt @@ -0,0 +1,8465 @@ +Produced by David Widger. The previous edition was updated by Jose +Menendez. + + + + + + THE ADVENTURES OF TOM SAWYER + BY + MARK TWAIN + (Samuel Langhorne Clemens) + + + + + P R E F A C E + +MOST of the adventures recorded in this book really occurred; one or +two were experiences of my own, the rest those of boys who were +schoolmates of mine. Huck Finn is drawn from life; Tom Sawyer also, but +not from an individual--he is a combination of the characteristics of +three boys whom I knew, and therefore belongs to the composite order of +architecture. + +The odd superstitions touched upon were all prevalent among children +and slaves in the West at the period of this story--that is to say, +thirty or forty years ago. + +Although my book is intended mainly for the entertainment of boys and +girls, I hope it will not be shunned by men and women on that account, +for part of my plan has been to try to pleasantly remind adults of what +they once were themselves, and of how they felt and thought and talked, +and what queer enterprises they sometimes engaged in. + + THE AUTHOR. + +HARTFORD, 1876. + + + + T O M S A W Y E R + + + +CHAPTER I + +"TOM!" + +No answer. + +"TOM!" + +No answer. + +"What's gone with that boy, I wonder? You TOM!" + +No answer. + +The old lady pulled her spectacles down and looked over them about the +room; then she put them up and looked out under them. She seldom or +never looked THROUGH them for so small a thing as a boy; they were her +state pair, the pride of her heart, and were built for "style," not +service--she could have seen through a pair of stove-lids just as well. +She looked perplexed for a moment, and then said, not fiercely, but +still loud enough for the furniture to hear: + +"Well, I lay if I get hold of you I'll--" + +She did not finish, for by this time she was bending down and punching +under the bed with the broom, and so she needed breath to punctuate the +punches with. She resurrected nothing but the cat. + +"I never did see the beat of that boy!" + +She went to the open door and stood in it and looked out among the +tomato vines and "jimpson" weeds that constituted the garden. No Tom. +So she lifted up her voice at an angle calculated for distance and +shouted: + +"Y-o-u-u TOM!" + +There was a slight noise behind her and she turned just in time to +seize a small boy by the slack of his roundabout and arrest his flight. + +"There! I might 'a' thought of that closet. What you been doing in +there?" + +"Nothing." + +"Nothing! Look at your hands. And look at your mouth. What IS that +truck?" + +"I don't know, aunt." + +"Well, I know. It's jam--that's what it is. Forty times I've said if +you didn't let that jam alone I'd skin you. Hand me that switch." + +The switch hovered in the air--the peril was desperate-- + +"My! Look behind you, aunt!" + +The old lady whirled round, and snatched her skirts out of danger. The +lad fled on the instant, scrambled up the high board-fence, and +disappeared over it. + +His aunt Polly stood surprised a moment, and then broke into a gentle +laugh. + +"Hang the boy, can't I never learn anything? Ain't he played me tricks +enough like that for me to be looking out for him by this time? But old +fools is the biggest fools there is. Can't learn an old dog new tricks, +as the saying is. But my goodness, he never plays them alike, two days, +and how is a body to know what's coming? He 'pears to know just how +long he can torment me before I get my dander up, and he knows if he +can make out to put me off for a minute or make me laugh, it's all down +again and I can't hit him a lick. I ain't doing my duty by that boy, +and that's the Lord's truth, goodness knows. Spare the rod and spile +the child, as the Good Book says. I'm a laying up sin and suffering for +us both, I know. He's full of the Old Scratch, but laws-a-me! he's my +own dead sister's boy, poor thing, and I ain't got the heart to lash +him, somehow. Every time I let him off, my conscience does hurt me so, +and every time I hit him my old heart most breaks. Well-a-well, man +that is born of woman is of few days and full of trouble, as the +Scripture says, and I reckon it's so. He'll play hookey this evening, * +and [* Southwestern for "afternoon"] I'll just be obleeged to make him +work, to-morrow, to punish him. It's mighty hard to make him work +Saturdays, when all the boys is having holiday, but he hates work more +than he hates anything else, and I've GOT to do some of my duty by him, +or I'll be the ruination of the child." + +Tom did play hookey, and he had a very good time. He got back home +barely in season to help Jim, the small colored boy, saw next-day's +wood and split the kindlings before supper--at least he was there in +time to tell his adventures to Jim while Jim did three-fourths of the +work. Tom's younger brother (or rather half-brother) Sid was already +through with his part of the work (picking up chips), for he was a +quiet boy, and had no adventurous, troublesome ways. + +While Tom was eating his supper, and stealing sugar as opportunity +offered, Aunt Polly asked him questions that were full of guile, and +very deep--for she wanted to trap him into damaging revealments. Like +many other simple-hearted souls, it was her pet vanity to believe she +was endowed with a talent for dark and mysterious diplomacy, and she +loved to contemplate her most transparent devices as marvels of low +cunning. Said she: + +"Tom, it was middling warm in school, warn't it?" + +"Yes'm." + +"Powerful warm, warn't it?" + +"Yes'm." + +"Didn't you want to go in a-swimming, Tom?" + +A bit of a scare shot through Tom--a touch of uncomfortable suspicion. +He searched Aunt Polly's face, but it told him nothing. So he said: + +"No'm--well, not very much." + +The old lady reached out her hand and felt Tom's shirt, and said: + +"But you ain't too warm now, though." And it flattered her to reflect +that she had discovered that the shirt was dry without anybody knowing +that that was what she had in her mind. But in spite of her, Tom knew +where the wind lay, now. So he forestalled what might be the next move: + +"Some of us pumped on our heads--mine's damp yet. See?" + +Aunt Polly was vexed to think she had overlooked that bit of +circumstantial evidence, and missed a trick. Then she had a new +inspiration: + +"Tom, you didn't have to undo your shirt collar where I sewed it, to +pump on your head, did you? Unbutton your jacket!" + +The trouble vanished out of Tom's face. He opened his jacket. His +shirt collar was securely sewed. + +"Bother! Well, go 'long with you. I'd made sure you'd played hookey +and been a-swimming. But I forgive ye, Tom. I reckon you're a kind of a +singed cat, as the saying is--better'n you look. THIS time." + +She was half sorry her sagacity had miscarried, and half glad that Tom +had stumbled into obedient conduct for once. + +But Sidney said: + +"Well, now, if I didn't think you sewed his collar with white thread, +but it's black." + +"Why, I did sew it with white! Tom!" + +But Tom did not wait for the rest. As he went out at the door he said: + +"Siddy, I'll lick you for that." + +In a safe place Tom examined two large needles which were thrust into +the lapels of his jacket, and had thread bound about them--one needle +carried white thread and the other black. He said: + +"She'd never noticed if it hadn't been for Sid. Confound it! sometimes +she sews it with white, and sometimes she sews it with black. I wish to +geeminy she'd stick to one or t'other--I can't keep the run of 'em. But +I bet you I'll lam Sid for that. I'll learn him!" + +He was not the Model Boy of the village. He knew the model boy very +well though--and loathed him. + +Within two minutes, or even less, he had forgotten all his troubles. +Not because his troubles were one whit less heavy and bitter to him +than a man's are to a man, but because a new and powerful interest bore +them down and drove them out of his mind for the time--just as men's +misfortunes are forgotten in the excitement of new enterprises. This +new interest was a valued novelty in whistling, which he had just +acquired from a negro, and he was suffering to practise it undisturbed. +It consisted in a peculiar bird-like turn, a sort of liquid warble, +produced by touching the tongue to the roof of the mouth at short +intervals in the midst of the music--the reader probably remembers how +to do it, if he has ever been a boy. Diligence and attention soon gave +him the knack of it, and he strode down the street with his mouth full +of harmony and his soul full of gratitude. He felt much as an +astronomer feels who has discovered a new planet--no doubt, as far as +strong, deep, unalloyed pleasure is concerned, the advantage was with +the boy, not the astronomer. + +The summer evenings were long. It was not dark, yet. Presently Tom +checked his whistle. A stranger was before him--a boy a shade larger +than himself. A new-comer of any age or either sex was an impressive +curiosity in the poor little shabby village of St. Petersburg. This boy +was well dressed, too--well dressed on a week-day. This was simply +astounding. His cap was a dainty thing, his close-buttoned blue cloth +roundabout was new and natty, and so were his pantaloons. He had shoes +on--and it was only Friday. He even wore a necktie, a bright bit of +ribbon. He had a citified air about him that ate into Tom's vitals. The +more Tom stared at the splendid marvel, the higher he turned up his +nose at his finery and the shabbier and shabbier his own outfit seemed +to him to grow. Neither boy spoke. If one moved, the other moved--but +only sidewise, in a circle; they kept face to face and eye to eye all +the time. Finally Tom said: + +"I can lick you!" + +"I'd like to see you try it." + +"Well, I can do it." + +"No you can't, either." + +"Yes I can." + +"No you can't." + +"I can." + +"You can't." + +"Can!" + +"Can't!" + +An uncomfortable pause. Then Tom said: + +"What's your name?" + +"'Tisn't any of your business, maybe." + +"Well I 'low I'll MAKE it my business." + +"Well why don't you?" + +"If you say much, I will." + +"Much--much--MUCH. There now." + +"Oh, you think you're mighty smart, DON'T you? I could lick you with +one hand tied behind me, if I wanted to." + +"Well why don't you DO it? You SAY you can do it." + +"Well I WILL, if you fool with me." + +"Oh yes--I've seen whole families in the same fix." + +"Smarty! You think you're SOME, now, DON'T you? Oh, what a hat!" + +"You can lump that hat if you don't like it. I dare you to knock it +off--and anybody that'll take a dare will suck eggs." + +"You're a liar!" + +"You're another." + +"You're a fighting liar and dasn't take it up." + +"Aw--take a walk!" + +"Say--if you give me much more of your sass I'll take and bounce a +rock off'n your head." + +"Oh, of COURSE you will." + +"Well I WILL." + +"Well why don't you DO it then? What do you keep SAYING you will for? +Why don't you DO it? It's because you're afraid." + +"I AIN'T afraid." + +"You are." + +"I ain't." + +"You are." + +Another pause, and more eying and sidling around each other. Presently +they were shoulder to shoulder. Tom said: + +"Get away from here!" + +"Go away yourself!" + +"I won't." + +"I won't either." + +So they stood, each with a foot placed at an angle as a brace, and +both shoving with might and main, and glowering at each other with +hate. But neither could get an advantage. After struggling till both +were hot and flushed, each relaxed his strain with watchful caution, +and Tom said: + +"You're a coward and a pup. I'll tell my big brother on you, and he +can thrash you with his little finger, and I'll make him do it, too." + +"What do I care for your big brother? I've got a brother that's bigger +than he is--and what's more, he can throw him over that fence, too." +[Both brothers were imaginary.] + +"That's a lie." + +"YOUR saying so don't make it so." + +Tom drew a line in the dust with his big toe, and said: + +"I dare you to step over that, and I'll lick you till you can't stand +up. Anybody that'll take a dare will steal sheep." + +The new boy stepped over promptly, and said: + +"Now you said you'd do it, now let's see you do it." + +"Don't you crowd me now; you better look out." + +"Well, you SAID you'd do it--why don't you do it?" + +"By jingo! for two cents I WILL do it." + +The new boy took two broad coppers out of his pocket and held them out +with derision. Tom struck them to the ground. In an instant both boys +were rolling and tumbling in the dirt, gripped together like cats; and +for the space of a minute they tugged and tore at each other's hair and +clothes, punched and scratched each other's nose, and covered +themselves with dust and glory. Presently the confusion took form, and +through the fog of battle Tom appeared, seated astride the new boy, and +pounding him with his fists. "Holler 'nuff!" said he. + +The boy only struggled to free himself. He was crying--mainly from rage. + +"Holler 'nuff!"--and the pounding went on. + +At last the stranger got out a smothered "'Nuff!" and Tom let him up +and said: + +"Now that'll learn you. Better look out who you're fooling with next +time." + +The new boy went off brushing the dust from his clothes, sobbing, +snuffling, and occasionally looking back and shaking his head and +threatening what he would do to Tom the "next time he caught him out." +To which Tom responded with jeers, and started off in high feather, and +as soon as his back was turned the new boy snatched up a stone, threw +it and hit him between the shoulders and then turned tail and ran like +an antelope. Tom chased the traitor home, and thus found out where he +lived. He then held a position at the gate for some time, daring the +enemy to come outside, but the enemy only made faces at him through the +window and declined. At last the enemy's mother appeared, and called +Tom a bad, vicious, vulgar child, and ordered him away. So he went +away; but he said he "'lowed" to "lay" for that boy. + +He got home pretty late that night, and when he climbed cautiously in +at the window, he uncovered an ambuscade, in the person of his aunt; +and when she saw the state his clothes were in her resolution to turn +his Saturday holiday into captivity at hard labor became adamantine in +its firmness. + + + +CHAPTER II + +SATURDAY morning was come, and all the summer world was bright and +fresh, and brimming with life. There was a song in every heart; and if +the heart was young the music issued at the lips. There was cheer in +every face and a spring in every step. The locust-trees were in bloom +and the fragrance of the blossoms filled the air. Cardiff Hill, beyond +the village and above it, was green with vegetation and it lay just far +enough away to seem a Delectable Land, dreamy, reposeful, and inviting. + +Tom appeared on the sidewalk with a bucket of whitewash and a +long-handled brush. He surveyed the fence, and all gladness left him and +a deep melancholy settled down upon his spirit. Thirty yards of board +fence nine feet high. Life to him seemed hollow, and existence but a +burden. Sighing, he dipped his brush and passed it along the topmost +plank; repeated the operation; did it again; compared the insignificant +whitewashed streak with the far-reaching continent of unwhitewashed +fence, and sat down on a tree-box discouraged. Jim came skipping out at +the gate with a tin pail, and singing Buffalo Gals. Bringing water from +the town pump had always been hateful work in Tom's eyes, before, but +now it did not strike him so. He remembered that there was company at +the pump. White, mulatto, and negro boys and girls were always there +waiting their turns, resting, trading playthings, quarrelling, +fighting, skylarking. And he remembered that although the pump was only +a hundred and fifty yards off, Jim never got back with a bucket of +water under an hour--and even then somebody generally had to go after +him. Tom said: + +"Say, Jim, I'll fetch the water if you'll whitewash some." + +Jim shook his head and said: + +"Can't, Mars Tom. Ole missis, she tole me I got to go an' git dis +water an' not stop foolin' roun' wid anybody. She say she spec' Mars +Tom gwine to ax me to whitewash, an' so she tole me go 'long an' 'tend +to my own business--she 'lowed SHE'D 'tend to de whitewashin'." + +"Oh, never you mind what she said, Jim. That's the way she always +talks. Gimme the bucket--I won't be gone only a a minute. SHE won't +ever know." + +"Oh, I dasn't, Mars Tom. Ole missis she'd take an' tar de head off'n +me. 'Deed she would." + +"SHE! She never licks anybody--whacks 'em over the head with her +thimble--and who cares for that, I'd like to know. She talks awful, but +talk don't hurt--anyways it don't if she don't cry. Jim, I'll give you +a marvel. I'll give you a white alley!" + +Jim began to waver. + +"White alley, Jim! And it's a bully taw." + +"My! Dat's a mighty gay marvel, I tell you! But Mars Tom I's powerful +'fraid ole missis--" + +"And besides, if you will I'll show you my sore toe." + +Jim was only human--this attraction was too much for him. He put down +his pail, took the white alley, and bent over the toe with absorbing +interest while the bandage was being unwound. In another moment he was +flying down the street with his pail and a tingling rear, Tom was +whitewashing with vigor, and Aunt Polly was retiring from the field +with a slipper in her hand and triumph in her eye. + +But Tom's energy did not last. He began to think of the fun he had +planned for this day, and his sorrows multiplied. Soon the free boys +would come tripping along on all sorts of delicious expeditions, and +they would make a world of fun of him for having to work--the very +thought of it burnt him like fire. He got out his worldly wealth and +examined it--bits of toys, marbles, and trash; enough to buy an +exchange of WORK, maybe, but not half enough to buy so much as half an +hour of pure freedom. So he returned his straitened means to his +pocket, and gave up the idea of trying to buy the boys. At this dark +and hopeless moment an inspiration burst upon him! Nothing less than a +great, magnificent inspiration. + +He took up his brush and went tranquilly to work. Ben Rogers hove in +sight presently--the very boy, of all boys, whose ridicule he had been +dreading. Ben's gait was the hop-skip-and-jump--proof enough that his +heart was light and his anticipations high. He was eating an apple, and +giving a long, melodious whoop, at intervals, followed by a deep-toned +ding-dong-dong, ding-dong-dong, for he was personating a steamboat. As +he drew near, he slackened speed, took the middle of the street, leaned +far over to starboard and rounded to ponderously and with laborious +pomp and circumstance--for he was personating the Big Missouri, and +considered himself to be drawing nine feet of water. He was boat and +captain and engine-bells combined, so he had to imagine himself +standing on his own hurricane-deck giving the orders and executing them: + +"Stop her, sir! Ting-a-ling-ling!" The headway ran almost out, and he +drew up slowly toward the sidewalk. + +"Ship up to back! Ting-a-ling-ling!" His arms straightened and +stiffened down his sides. + +"Set her back on the stabboard! Ting-a-ling-ling! Chow! ch-chow-wow! +Chow!" His right hand, meantime, describing stately circles--for it was +representing a forty-foot wheel. + +"Let her go back on the labboard! Ting-a-lingling! Chow-ch-chow-chow!" +The left hand began to describe circles. + +"Stop the stabboard! Ting-a-ling-ling! Stop the labboard! Come ahead +on the stabboard! Stop her! Let your outside turn over slow! +Ting-a-ling-ling! Chow-ow-ow! Get out that head-line! LIVELY now! +Come--out with your spring-line--what're you about there! Take a turn +round that stump with the bight of it! Stand by that stage, now--let her +go! Done with the engines, sir! Ting-a-ling-ling! SH'T! S'H'T! SH'T!" +(trying the gauge-cocks). + +Tom went on whitewashing--paid no attention to the steamboat. Ben +stared a moment and then said: "Hi-YI! YOU'RE up a stump, ain't you!" + +No answer. Tom surveyed his last touch with the eye of an artist, then +he gave his brush another gentle sweep and surveyed the result, as +before. Ben ranged up alongside of him. Tom's mouth watered for the +apple, but he stuck to his work. Ben said: + +"Hello, old chap, you got to work, hey?" + +Tom wheeled suddenly and said: + +"Why, it's you, Ben! I warn't noticing." + +"Say--I'm going in a-swimming, I am. Don't you wish you could? But of +course you'd druther WORK--wouldn't you? Course you would!" + +Tom contemplated the boy a bit, and said: + +"What do you call work?" + +"Why, ain't THAT work?" + +Tom resumed his whitewashing, and answered carelessly: + +"Well, maybe it is, and maybe it ain't. All I know, is, it suits Tom +Sawyer." + +"Oh come, now, you don't mean to let on that you LIKE it?" + +The brush continued to move. + +"Like it? Well, I don't see why I oughtn't to like it. Does a boy get +a chance to whitewash a fence every day?" + +That put the thing in a new light. Ben stopped nibbling his apple. Tom +swept his brush daintily back and forth--stepped back to note the +effect--added a touch here and there--criticised the effect again--Ben +watching every move and getting more and more interested, more and more +absorbed. Presently he said: + +"Say, Tom, let ME whitewash a little." + +Tom considered, was about to consent; but he altered his mind: + +"No--no--I reckon it wouldn't hardly do, Ben. You see, Aunt Polly's +awful particular about this fence--right here on the street, you know +--but if it was the back fence I wouldn't mind and SHE wouldn't. Yes, +she's awful particular about this fence; it's got to be done very +careful; I reckon there ain't one boy in a thousand, maybe two +thousand, that can do it the way it's got to be done." + +"No--is that so? Oh come, now--lemme just try. Only just a little--I'd +let YOU, if you was me, Tom." + +"Ben, I'd like to, honest injun; but Aunt Polly--well, Jim wanted to +do it, but she wouldn't let him; Sid wanted to do it, and she wouldn't +let Sid. Now don't you see how I'm fixed? If you was to tackle this +fence and anything was to happen to it--" + +"Oh, shucks, I'll be just as careful. Now lemme try. Say--I'll give +you the core of my apple." + +"Well, here--No, Ben, now don't. I'm afeard--" + +"I'll give you ALL of it!" + +Tom gave up the brush with reluctance in his face, but alacrity in his +heart. And while the late steamer Big Missouri worked and sweated in +the sun, the retired artist sat on a barrel in the shade close by, +dangled his legs, munched his apple, and planned the slaughter of more +innocents. There was no lack of material; boys happened along every +little while; they came to jeer, but remained to whitewash. By the time +Ben was fagged out, Tom had traded the next chance to Billy Fisher for +a kite, in good repair; and when he played out, Johnny Miller bought in +for a dead rat and a string to swing it with--and so on, and so on, +hour after hour. And when the middle of the afternoon came, from being +a poor poverty-stricken boy in the morning, Tom was literally rolling +in wealth. He had besides the things before mentioned, twelve marbles, +part of a jews-harp, a piece of blue bottle-glass to look through, a +spool cannon, a key that wouldn't unlock anything, a fragment of chalk, +a glass stopper of a decanter, a tin soldier, a couple of tadpoles, six +fire-crackers, a kitten with only one eye, a brass doorknob, a +dog-collar--but no dog--the handle of a knife, four pieces of +orange-peel, and a dilapidated old window sash. + +He had had a nice, good, idle time all the while--plenty of company +--and the fence had three coats of whitewash on it! If he hadn't run out +of whitewash he would have bankrupted every boy in the village. + +Tom said to himself that it was not such a hollow world, after all. He +had discovered a great law of human action, without knowing it--namely, +that in order to make a man or a boy covet a thing, it is only +necessary to make the thing difficult to attain. If he had been a great +and wise philosopher, like the writer of this book, he would now have +comprehended that Work consists of whatever a body is OBLIGED to do, +and that Play consists of whatever a body is not obliged to do. And +this would help him to understand why constructing artificial flowers +or performing on a tread-mill is work, while rolling ten-pins or +climbing Mont Blanc is only amusement. There are wealthy gentlemen in +England who drive four-horse passenger-coaches twenty or thirty miles +on a daily line, in the summer, because the privilege costs them +considerable money; but if they were offered wages for the service, +that would turn it into work and then they would resign. + +The boy mused awhile over the substantial change which had taken place +in his worldly circumstances, and then wended toward headquarters to +report. + + + +CHAPTER III + +TOM presented himself before Aunt Polly, who was sitting by an open +window in a pleasant rearward apartment, which was bedroom, +breakfast-room, dining-room, and library, combined. The balmy summer +air, the restful quiet, the odor of the flowers, and the drowsing murmur +of the bees had had their effect, and she was nodding over her knitting +--for she had no company but the cat, and it was asleep in her lap. Her +spectacles were propped up on her gray head for safety. She had thought +that of course Tom had deserted long ago, and she wondered at seeing him +place himself in her power again in this intrepid way. He said: "Mayn't +I go and play now, aunt?" + +"What, a'ready? How much have you done?" + +"It's all done, aunt." + +"Tom, don't lie to me--I can't bear it." + +"I ain't, aunt; it IS all done." + +Aunt Polly placed small trust in such evidence. She went out to see +for herself; and she would have been content to find twenty per cent. +of Tom's statement true. When she found the entire fence whitewashed, +and not only whitewashed but elaborately coated and recoated, and even +a streak added to the ground, her astonishment was almost unspeakable. +She said: + +"Well, I never! There's no getting round it, you can work when you're +a mind to, Tom." And then she diluted the compliment by adding, "But +it's powerful seldom you're a mind to, I'm bound to say. Well, go 'long +and play; but mind you get back some time in a week, or I'll tan you." + +She was so overcome by the splendor of his achievement that she took +him into the closet and selected a choice apple and delivered it to +him, along with an improving lecture upon the added value and flavor a +treat took to itself when it came without sin through virtuous effort. +And while she closed with a happy Scriptural flourish, he "hooked" a +doughnut. + +Then he skipped out, and saw Sid just starting up the outside stairway +that led to the back rooms on the second floor. Clods were handy and +the air was full of them in a twinkling. They raged around Sid like a +hail-storm; and before Aunt Polly could collect her surprised faculties +and sally to the rescue, six or seven clods had taken personal effect, +and Tom was over the fence and gone. There was a gate, but as a general +thing he was too crowded for time to make use of it. His soul was at +peace, now that he had settled with Sid for calling attention to his +black thread and getting him into trouble. + +Tom skirted the block, and came round into a muddy alley that led by +the back of his aunt's cow-stable. He presently got safely beyond the +reach of capture and punishment, and hastened toward the public square +of the village, where two "military" companies of boys had met for +conflict, according to previous appointment. Tom was General of one of +these armies, Joe Harper (a bosom friend) General of the other. These +two great commanders did not condescend to fight in person--that being +better suited to the still smaller fry--but sat together on an eminence +and conducted the field operations by orders delivered through +aides-de-camp. Tom's army won a great victory, after a long and +hard-fought battle. Then the dead were counted, prisoners exchanged, +the terms of the next disagreement agreed upon, and the day for the +necessary battle appointed; after which the armies fell into line and +marched away, and Tom turned homeward alone. + +As he was passing by the house where Jeff Thatcher lived, he saw a new +girl in the garden--a lovely little blue-eyed creature with yellow hair +plaited into two long-tails, white summer frock and embroidered +pantalettes. The fresh-crowned hero fell without firing a shot. A +certain Amy Lawrence vanished out of his heart and left not even a +memory of herself behind. He had thought he loved her to distraction; +he had regarded his passion as adoration; and behold it was only a poor +little evanescent partiality. He had been months winning her; she had +confessed hardly a week ago; he had been the happiest and the proudest +boy in the world only seven short days, and here in one instant of time +she had gone out of his heart like a casual stranger whose visit is +done. + +He worshipped this new angel with furtive eye, till he saw that she +had discovered him; then he pretended he did not know she was present, +and began to "show off" in all sorts of absurd boyish ways, in order to +win her admiration. He kept up this grotesque foolishness for some +time; but by-and-by, while he was in the midst of some dangerous +gymnastic performances, he glanced aside and saw that the little girl +was wending her way toward the house. Tom came up to the fence and +leaned on it, grieving, and hoping she would tarry yet awhile longer. +She halted a moment on the steps and then moved toward the door. Tom +heaved a great sigh as she put her foot on the threshold. But his face +lit up, right away, for she tossed a pansy over the fence a moment +before she disappeared. + +The boy ran around and stopped within a foot or two of the flower, and +then shaded his eyes with his hand and began to look down street as if +he had discovered something of interest going on in that direction. +Presently he picked up a straw and began trying to balance it on his +nose, with his head tilted far back; and as he moved from side to side, +in his efforts, he edged nearer and nearer toward the pansy; finally +his bare foot rested upon it, his pliant toes closed upon it, and he +hopped away with the treasure and disappeared round the corner. But +only for a minute--only while he could button the flower inside his +jacket, next his heart--or next his stomach, possibly, for he was not +much posted in anatomy, and not hypercritical, anyway. + +He returned, now, and hung about the fence till nightfall, "showing +off," as before; but the girl never exhibited herself again, though Tom +comforted himself a little with the hope that she had been near some +window, meantime, and been aware of his attentions. Finally he strode +home reluctantly, with his poor head full of visions. + +All through supper his spirits were so high that his aunt wondered +"what had got into the child." He took a good scolding about clodding +Sid, and did not seem to mind it in the least. He tried to steal sugar +under his aunt's very nose, and got his knuckles rapped for it. He said: + +"Aunt, you don't whack Sid when he takes it." + +"Well, Sid don't torment a body the way you do. You'd be always into +that sugar if I warn't watching you." + +Presently she stepped into the kitchen, and Sid, happy in his +immunity, reached for the sugar-bowl--a sort of glorying over Tom which +was wellnigh unbearable. But Sid's fingers slipped and the bowl dropped +and broke. Tom was in ecstasies. In such ecstasies that he even +controlled his tongue and was silent. He said to himself that he would +not speak a word, even when his aunt came in, but would sit perfectly +still till she asked who did the mischief; and then he would tell, and +there would be nothing so good in the world as to see that pet model +"catch it." He was so brimful of exultation that he could hardly hold +himself when the old lady came back and stood above the wreck +discharging lightnings of wrath from over her spectacles. He said to +himself, "Now it's coming!" And the next instant he was sprawling on +the floor! The potent palm was uplifted to strike again when Tom cried +out: + +"Hold on, now, what 'er you belting ME for?--Sid broke it!" + +Aunt Polly paused, perplexed, and Tom looked for healing pity. But +when she got her tongue again, she only said: + +"Umf! Well, you didn't get a lick amiss, I reckon. You been into some +other audacious mischief when I wasn't around, like enough." + +Then her conscience reproached her, and she yearned to say something +kind and loving; but she judged that this would be construed into a +confession that she had been in the wrong, and discipline forbade that. +So she kept silence, and went about her affairs with a troubled heart. +Tom sulked in a corner and exalted his woes. He knew that in her heart +his aunt was on her knees to him, and he was morosely gratified by the +consciousness of it. He would hang out no signals, he would take notice +of none. He knew that a yearning glance fell upon him, now and then, +through a film of tears, but he refused recognition of it. He pictured +himself lying sick unto death and his aunt bending over him beseeching +one little forgiving word, but he would turn his face to the wall, and +die with that word unsaid. Ah, how would she feel then? And he pictured +himself brought home from the river, dead, with his curls all wet, and +his sore heart at rest. How she would throw herself upon him, and how +her tears would fall like rain, and her lips pray God to give her back +her boy and she would never, never abuse him any more! But he would lie +there cold and white and make no sign--a poor little sufferer, whose +griefs were at an end. He so worked upon his feelings with the pathos +of these dreams, that he had to keep swallowing, he was so like to +choke; and his eyes swam in a blur of water, which overflowed when he +winked, and ran down and trickled from the end of his nose. And such a +luxury to him was this petting of his sorrows, that he could not bear +to have any worldly cheeriness or any grating delight intrude upon it; +it was too sacred for such contact; and so, presently, when his cousin +Mary danced in, all alive with the joy of seeing home again after an +age-long visit of one week to the country, he got up and moved in +clouds and darkness out at one door as she brought song and sunshine in +at the other. + +He wandered far from the accustomed haunts of boys, and sought +desolate places that were in harmony with his spirit. A log raft in the +river invited him, and he seated himself on its outer edge and +contemplated the dreary vastness of the stream, wishing, the while, +that he could only be drowned, all at once and unconsciously, without +undergoing the uncomfortable routine devised by nature. Then he thought +of his flower. He got it out, rumpled and wilted, and it mightily +increased his dismal felicity. He wondered if she would pity him if she +knew? Would she cry, and wish that she had a right to put her arms +around his neck and comfort him? Or would she turn coldly away like all +the hollow world? This picture brought such an agony of pleasurable +suffering that he worked it over and over again in his mind and set it +up in new and varied lights, till he wore it threadbare. At last he +rose up sighing and departed in the darkness. + +About half-past nine or ten o'clock he came along the deserted street +to where the Adored Unknown lived; he paused a moment; no sound fell +upon his listening ear; a candle was casting a dull glow upon the +curtain of a second-story window. Was the sacred presence there? He +climbed the fence, threaded his stealthy way through the plants, till +he stood under that window; he looked up at it long, and with emotion; +then he laid him down on the ground under it, disposing himself upon +his back, with his hands clasped upon his breast and holding his poor +wilted flower. And thus he would die--out in the cold world, with no +shelter over his homeless head, no friendly hand to wipe the +death-damps from his brow, no loving face to bend pityingly over him +when the great agony came. And thus SHE would see him when she looked +out upon the glad morning, and oh! would she drop one little tear upon +his poor, lifeless form, would she heave one little sigh to see a bright +young life so rudely blighted, so untimely cut down? + +The window went up, a maid-servant's discordant voice profaned the +holy calm, and a deluge of water drenched the prone martyr's remains! + +The strangling hero sprang up with a relieving snort. There was a whiz +as of a missile in the air, mingled with the murmur of a curse, a sound +as of shivering glass followed, and a small, vague form went over the +fence and shot away in the gloom. + +Not long after, as Tom, all undressed for bed, was surveying his +drenched garments by the light of a tallow dip, Sid woke up; but if he +had any dim idea of making any "references to allusions," he thought +better of it and held his peace, for there was danger in Tom's eye. + +Tom turned in without the added vexation of prayers, and Sid made +mental note of the omission. + + + +CHAPTER IV + +THE sun rose upon a tranquil world, and beamed down upon the peaceful +village like a benediction. Breakfast over, Aunt Polly had family +worship: it began with a prayer built from the ground up of solid +courses of Scriptural quotations, welded together with a thin mortar of +originality; and from the summit of this she delivered a grim chapter +of the Mosaic Law, as from Sinai. + +Then Tom girded up his loins, so to speak, and went to work to "get +his verses." Sid had learned his lesson days before. Tom bent all his +energies to the memorizing of five verses, and he chose part of the +Sermon on the Mount, because he could find no verses that were shorter. +At the end of half an hour Tom had a vague general idea of his lesson, +but no more, for his mind was traversing the whole field of human +thought, and his hands were busy with distracting recreations. Mary +took his book to hear him recite, and he tried to find his way through +the fog: + +"Blessed are the--a--a--" + +"Poor"-- + +"Yes--poor; blessed are the poor--a--a--" + +"In spirit--" + +"In spirit; blessed are the poor in spirit, for they--they--" + +"THEIRS--" + +"For THEIRS. Blessed are the poor in spirit, for theirs is the kingdom +of heaven. Blessed are they that mourn, for they--they--" + +"Sh--" + +"For they--a--" + +"S, H, A--" + +"For they S, H--Oh, I don't know what it is!" + +"SHALL!" + +"Oh, SHALL! for they shall--for they shall--a--a--shall mourn--a--a-- +blessed are they that shall--they that--a--they that shall mourn, for +they shall--a--shall WHAT? Why don't you tell me, Mary?--what do you +want to be so mean for?" + +"Oh, Tom, you poor thick-headed thing, I'm not teasing you. I wouldn't +do that. You must go and learn it again. Don't you be discouraged, Tom, +you'll manage it--and if you do, I'll give you something ever so nice. +There, now, that's a good boy." + +"All right! What is it, Mary, tell me what it is." + +"Never you mind, Tom. You know if I say it's nice, it is nice." + +"You bet you that's so, Mary. All right, I'll tackle it again." + +And he did "tackle it again"--and under the double pressure of +curiosity and prospective gain he did it with such spirit that he +accomplished a shining success. Mary gave him a brand-new "Barlow" +knife worth twelve and a half cents; and the convulsion of delight that +swept his system shook him to his foundations. True, the knife would +not cut anything, but it was a "sure-enough" Barlow, and there was +inconceivable grandeur in that--though where the Western boys ever got +the idea that such a weapon could possibly be counterfeited to its +injury is an imposing mystery and will always remain so, perhaps. Tom +contrived to scarify the cupboard with it, and was arranging to begin +on the bureau, when he was called off to dress for Sunday-school. + +Mary gave him a tin basin of water and a piece of soap, and he went +outside the door and set the basin on a little bench there; then he +dipped the soap in the water and laid it down; turned up his sleeves; +poured out the water on the ground, gently, and then entered the +kitchen and began to wipe his face diligently on the towel behind the +door. But Mary removed the towel and said: + +"Now ain't you ashamed, Tom. You mustn't be so bad. Water won't hurt +you." + +Tom was a trifle disconcerted. The basin was refilled, and this time +he stood over it a little while, gathering resolution; took in a big +breath and began. When he entered the kitchen presently, with both eyes +shut and groping for the towel with his hands, an honorable testimony +of suds and water was dripping from his face. But when he emerged from +the towel, he was not yet satisfactory, for the clean territory stopped +short at his chin and his jaws, like a mask; below and beyond this line +there was a dark expanse of unirrigated soil that spread downward in +front and backward around his neck. Mary took him in hand, and when she +was done with him he was a man and a brother, without distinction of +color, and his saturated hair was neatly brushed, and its short curls +wrought into a dainty and symmetrical general effect. [He privately +smoothed out the curls, with labor and difficulty, and plastered his +hair close down to his head; for he held curls to be effeminate, and +his own filled his life with bitterness.] Then Mary got out a suit of +his clothing that had been used only on Sundays during two years--they +were simply called his "other clothes"--and so by that we know the +size of his wardrobe. The girl "put him to rights" after he had dressed +himself; she buttoned his neat roundabout up to his chin, turned his +vast shirt collar down over his shoulders, brushed him off and crowned +him with his speckled straw hat. He now looked exceedingly improved and +uncomfortable. He was fully as uncomfortable as he looked; for there +was a restraint about whole clothes and cleanliness that galled him. He +hoped that Mary would forget his shoes, but the hope was blighted; she +coated them thoroughly with tallow, as was the custom, and brought them +out. He lost his temper and said he was always being made to do +everything he didn't want to do. But Mary said, persuasively: + +"Please, Tom--that's a good boy." + +So he got into the shoes snarling. Mary was soon ready, and the three +children set out for Sunday-school--a place that Tom hated with his +whole heart; but Sid and Mary were fond of it. + +Sabbath-school hours were from nine to half-past ten; and then church +service. Two of the children always remained for the sermon +voluntarily, and the other always remained too--for stronger reasons. +The church's high-backed, uncushioned pews would seat about three +hundred persons; the edifice was but a small, plain affair, with a sort +of pine board tree-box on top of it for a steeple. At the door Tom +dropped back a step and accosted a Sunday-dressed comrade: + +"Say, Billy, got a yaller ticket?" + +"Yes." + +"What'll you take for her?" + +"What'll you give?" + +"Piece of lickrish and a fish-hook." + +"Less see 'em." + +Tom exhibited. They were satisfactory, and the property changed hands. +Then Tom traded a couple of white alleys for three red tickets, and +some small trifle or other for a couple of blue ones. He waylaid other +boys as they came, and went on buying tickets of various colors ten or +fifteen minutes longer. He entered the church, now, with a swarm of +clean and noisy boys and girls, proceeded to his seat and started a +quarrel with the first boy that came handy. The teacher, a grave, +elderly man, interfered; then turned his back a moment and Tom pulled a +boy's hair in the next bench, and was absorbed in his book when the boy +turned around; stuck a pin in another boy, presently, in order to hear +him say "Ouch!" and got a new reprimand from his teacher. Tom's whole +class were of a pattern--restless, noisy, and troublesome. When they +came to recite their lessons, not one of them knew his verses +perfectly, but had to be prompted all along. However, they worried +through, and each got his reward--in small blue tickets, each with a +passage of Scripture on it; each blue ticket was pay for two verses of +the recitation. Ten blue tickets equalled a red one, and could be +exchanged for it; ten red tickets equalled a yellow one; for ten yellow +tickets the superintendent gave a very plainly bound Bible (worth forty +cents in those easy times) to the pupil. How many of my readers would +have the industry and application to memorize two thousand verses, even +for a Dore Bible? And yet Mary had acquired two Bibles in this way--it +was the patient work of two years--and a boy of German parentage had +won four or five. He once recited three thousand verses without +stopping; but the strain upon his mental faculties was too great, and +he was little better than an idiot from that day forth--a grievous +misfortune for the school, for on great occasions, before company, the +superintendent (as Tom expressed it) had always made this boy come out +and "spread himself." Only the older pupils managed to keep their +tickets and stick to their tedious work long enough to get a Bible, and +so the delivery of one of these prizes was a rare and noteworthy +circumstance; the successful pupil was so great and conspicuous for +that day that on the spot every scholar's heart was fired with a fresh +ambition that often lasted a couple of weeks. It is possible that Tom's +mental stomach had never really hungered for one of those prizes, but +unquestionably his entire being had for many a day longed for the glory +and the eclat that came with it. + +In due course the superintendent stood up in front of the pulpit, with +a closed hymn-book in his hand and his forefinger inserted between its +leaves, and commanded attention. When a Sunday-school superintendent +makes his customary little speech, a hymn-book in the hand is as +necessary as is the inevitable sheet of music in the hand of a singer +who stands forward on the platform and sings a solo at a concert +--though why, is a mystery: for neither the hymn-book nor the sheet of +music is ever referred to by the sufferer. This superintendent was a +slim creature of thirty-five, with a sandy goatee and short sandy hair; +he wore a stiff standing-collar whose upper edge almost reached his +ears and whose sharp points curved forward abreast the corners of his +mouth--a fence that compelled a straight lookout ahead, and a turning +of the whole body when a side view was required; his chin was propped +on a spreading cravat which was as broad and as long as a bank-note, +and had fringed ends; his boot toes were turned sharply up, in the +fashion of the day, like sleigh-runners--an effect patiently and +laboriously produced by the young men by sitting with their toes +pressed against a wall for hours together. Mr. Walters was very earnest +of mien, and very sincere and honest at heart; and he held sacred +things and places in such reverence, and so separated them from worldly +matters, that unconsciously to himself his Sunday-school voice had +acquired a peculiar intonation which was wholly absent on week-days. He +began after this fashion: + +"Now, children, I want you all to sit up just as straight and pretty +as you can and give me all your attention for a minute or two. There +--that is it. That is the way good little boys and girls should do. I see +one little girl who is looking out of the window--I am afraid she +thinks I am out there somewhere--perhaps up in one of the trees making +a speech to the little birds. [Applausive titter.] I want to tell you +how good it makes me feel to see so many bright, clean little faces +assembled in a place like this, learning to do right and be good." And +so forth and so on. It is not necessary to set down the rest of the +oration. It was of a pattern which does not vary, and so it is familiar +to us all. + +The latter third of the speech was marred by the resumption of fights +and other recreations among certain of the bad boys, and by fidgetings +and whisperings that extended far and wide, washing even to the bases +of isolated and incorruptible rocks like Sid and Mary. But now every +sound ceased suddenly, with the subsidence of Mr. Walters' voice, and +the conclusion of the speech was received with a burst of silent +gratitude. + +A good part of the whispering had been occasioned by an event which +was more or less rare--the entrance of visitors: lawyer Thatcher, +accompanied by a very feeble and aged man; a fine, portly, middle-aged +gentleman with iron-gray hair; and a dignified lady who was doubtless +the latter's wife. The lady was leading a child. Tom had been restless +and full of chafings and repinings; conscience-smitten, too--he could +not meet Amy Lawrence's eye, he could not brook her loving gaze. But +when he saw this small new-comer his soul was all ablaze with bliss in +a moment. The next moment he was "showing off" with all his might +--cuffing boys, pulling hair, making faces--in a word, using every art +that seemed likely to fascinate a girl and win her applause. His +exaltation had but one alloy--the memory of his humiliation in this +angel's garden--and that record in sand was fast washing out, under +the waves of happiness that were sweeping over it now. + +The visitors were given the highest seat of honor, and as soon as Mr. +Walters' speech was finished, he introduced them to the school. The +middle-aged man turned out to be a prodigious personage--no less a one +than the county judge--altogether the most august creation these +children had ever looked upon--and they wondered what kind of material +he was made of--and they half wanted to hear him roar, and were half +afraid he might, too. He was from Constantinople, twelve miles away--so +he had travelled, and seen the world--these very eyes had looked upon +the county court-house--which was said to have a tin roof. The awe +which these reflections inspired was attested by the impressive silence +and the ranks of staring eyes. This was the great Judge Thatcher, +brother of their own lawyer. Jeff Thatcher immediately went forward, to +be familiar with the great man and be envied by the school. It would +have been music to his soul to hear the whisperings: + +"Look at him, Jim! He's a going up there. Say--look! he's a going to +shake hands with him--he IS shaking hands with him! By jings, don't you +wish you was Jeff?" + +Mr. Walters fell to "showing off," with all sorts of official +bustlings and activities, giving orders, delivering judgments, +discharging directions here, there, everywhere that he could find a +target. The librarian "showed off"--running hither and thither with his +arms full of books and making a deal of the splutter and fuss that +insect authority delights in. The young lady teachers "showed off" +--bending sweetly over pupils that were lately being boxed, lifting +pretty warning fingers at bad little boys and patting good ones +lovingly. The young gentlemen teachers "showed off" with small +scoldings and other little displays of authority and fine attention to +discipline--and most of the teachers, of both sexes, found business up +at the library, by the pulpit; and it was business that frequently had +to be done over again two or three times (with much seeming vexation). +The little girls "showed off" in various ways, and the little boys +"showed off" with such diligence that the air was thick with paper wads +and the murmur of scufflings. And above it all the great man sat and +beamed a majestic judicial smile upon all the house, and warmed himself +in the sun of his own grandeur--for he was "showing off," too. + +There was only one thing wanting to make Mr. Walters' ecstasy +complete, and that was a chance to deliver a Bible-prize and exhibit a +prodigy. Several pupils had a few yellow tickets, but none had enough +--he had been around among the star pupils inquiring. He would have given +worlds, now, to have that German lad back again with a sound mind. + +And now at this moment, when hope was dead, Tom Sawyer came forward +with nine yellow tickets, nine red tickets, and ten blue ones, and +demanded a Bible. This was a thunderbolt out of a clear sky. Walters +was not expecting an application from this source for the next ten +years. But there was no getting around it--here were the certified +checks, and they were good for their face. Tom was therefore elevated +to a place with the Judge and the other elect, and the great news was +announced from headquarters. It was the most stunning surprise of the +decade, and so profound was the sensation that it lifted the new hero +up to the judicial one's altitude, and the school had two marvels to +gaze upon in place of one. The boys were all eaten up with envy--but +those that suffered the bitterest pangs were those who perceived too +late that they themselves had contributed to this hated splendor by +trading tickets to Tom for the wealth he had amassed in selling +whitewashing privileges. These despised themselves, as being the dupes +of a wily fraud, a guileful snake in the grass. + +The prize was delivered to Tom with as much effusion as the +superintendent could pump up under the circumstances; but it lacked +somewhat of the true gush, for the poor fellow's instinct taught him +that there was a mystery here that could not well bear the light, +perhaps; it was simply preposterous that this boy had warehoused two +thousand sheaves of Scriptural wisdom on his premises--a dozen would +strain his capacity, without a doubt. + +Amy Lawrence was proud and glad, and she tried to make Tom see it in +her face--but he wouldn't look. She wondered; then she was just a grain +troubled; next a dim suspicion came and went--came again; she watched; +a furtive glance told her worlds--and then her heart broke, and she was +jealous, and angry, and the tears came and she hated everybody. Tom +most of all (she thought). + +Tom was introduced to the Judge; but his tongue was tied, his breath +would hardly come, his heart quaked--partly because of the awful +greatness of the man, but mainly because he was her parent. He would +have liked to fall down and worship him, if it were in the dark. The +Judge put his hand on Tom's head and called him a fine little man, and +asked him what his name was. The boy stammered, gasped, and got it out: + +"Tom." + +"Oh, no, not Tom--it is--" + +"Thomas." + +"Ah, that's it. I thought there was more to it, maybe. That's very +well. But you've another one I daresay, and you'll tell it to me, won't +you?" + +"Tell the gentleman your other name, Thomas," said Walters, "and say +sir. You mustn't forget your manners." + +"Thomas Sawyer--sir." + +"That's it! That's a good boy. Fine boy. Fine, manly little fellow. +Two thousand verses is a great many--very, very great many. And you +never can be sorry for the trouble you took to learn them; for +knowledge is worth more than anything there is in the world; it's what +makes great men and good men; you'll be a great man and a good man +yourself, some day, Thomas, and then you'll look back and say, It's all +owing to the precious Sunday-school privileges of my boyhood--it's all +owing to my dear teachers that taught me to learn--it's all owing to +the good superintendent, who encouraged me, and watched over me, and +gave me a beautiful Bible--a splendid elegant Bible--to keep and have +it all for my own, always--it's all owing to right bringing up! That is +what you will say, Thomas--and you wouldn't take any money for those +two thousand verses--no indeed you wouldn't. And now you wouldn't mind +telling me and this lady some of the things you've learned--no, I know +you wouldn't--for we are proud of little boys that learn. Now, no +doubt you know the names of all the twelve disciples. Won't you tell us +the names of the first two that were appointed?" + +Tom was tugging at a button-hole and looking sheepish. He blushed, +now, and his eyes fell. Mr. Walters' heart sank within him. He said to +himself, it is not possible that the boy can answer the simplest +question--why DID the Judge ask him? Yet he felt obliged to speak up +and say: + +"Answer the gentleman, Thomas--don't be afraid." + +Tom still hung fire. + +"Now I know you'll tell me," said the lady. "The names of the first +two disciples were--" + +"DAVID AND GOLIAH!" + +Let us draw the curtain of charity over the rest of the scene. + + + +CHAPTER V + +ABOUT half-past ten the cracked bell of the small church began to +ring, and presently the people began to gather for the morning sermon. +The Sunday-school children distributed themselves about the house and +occupied pews with their parents, so as to be under supervision. Aunt +Polly came, and Tom and Sid and Mary sat with her--Tom being placed +next the aisle, in order that he might be as far away from the open +window and the seductive outside summer scenes as possible. The crowd +filed up the aisles: the aged and needy postmaster, who had seen better +days; the mayor and his wife--for they had a mayor there, among other +unnecessaries; the justice of the peace; the widow Douglass, fair, +smart, and forty, a generous, good-hearted soul and well-to-do, her +hill mansion the only palace in the town, and the most hospitable and +much the most lavish in the matter of festivities that St. Petersburg +could boast; the bent and venerable Major and Mrs. Ward; lawyer +Riverson, the new notable from a distance; next the belle of the +village, followed by a troop of lawn-clad and ribbon-decked young +heart-breakers; then all the young clerks in town in a body--for they +had stood in the vestibule sucking their cane-heads, a circling wall of +oiled and simpering admirers, till the last girl had run their gantlet; +and last of all came the Model Boy, Willie Mufferson, taking as heedful +care of his mother as if she were cut glass. He always brought his +mother to church, and was the pride of all the matrons. The boys all +hated him, he was so good. And besides, he had been "thrown up to them" +so much. His white handkerchief was hanging out of his pocket behind, as +usual on Sundays--accidentally. Tom had no handkerchief, and he looked +upon boys who had as snobs. + +The congregation being fully assembled, now, the bell rang once more, +to warn laggards and stragglers, and then a solemn hush fell upon the +church which was only broken by the tittering and whispering of the +choir in the gallery. The choir always tittered and whispered all +through service. There was once a church choir that was not ill-bred, +but I have forgotten where it was, now. It was a great many years ago, +and I can scarcely remember anything about it, but I think it was in +some foreign country. + +The minister gave out the hymn, and read it through with a relish, in +a peculiar style which was much admired in that part of the country. +His voice began on a medium key and climbed steadily up till it reached +a certain point, where it bore with strong emphasis upon the topmost +word and then plunged down as if from a spring-board: + + Shall I be car-ri-ed toe the skies, on flow'ry BEDS of ease, + + Whilst others fight to win the prize, and sail thro' BLOODY seas? + +He was regarded as a wonderful reader. At church "sociables" he was +always called upon to read poetry; and when he was through, the ladies +would lift up their hands and let them fall helplessly in their laps, +and "wall" their eyes, and shake their heads, as much as to say, "Words +cannot express it; it is too beautiful, TOO beautiful for this mortal +earth." + +After the hymn had been sung, the Rev. Mr. Sprague turned himself into +a bulletin-board, and read off "notices" of meetings and societies and +things till it seemed that the list would stretch out to the crack of +doom--a queer custom which is still kept up in America, even in cities, +away here in this age of abundant newspapers. Often, the less there is +to justify a traditional custom, the harder it is to get rid of it. + +And now the minister prayed. A good, generous prayer it was, and went +into details: it pleaded for the church, and the little children of the +church; for the other churches of the village; for the village itself; +for the county; for the State; for the State officers; for the United +States; for the churches of the United States; for Congress; for the +President; for the officers of the Government; for poor sailors, tossed +by stormy seas; for the oppressed millions groaning under the heel of +European monarchies and Oriental despotisms; for such as have the light +and the good tidings, and yet have not eyes to see nor ears to hear +withal; for the heathen in the far islands of the sea; and closed with +a supplication that the words he was about to speak might find grace +and favor, and be as seed sown in fertile ground, yielding in time a +grateful harvest of good. Amen. + +There was a rustling of dresses, and the standing congregation sat +down. The boy whose history this book relates did not enjoy the prayer, +he only endured it--if he even did that much. He was restive all +through it; he kept tally of the details of the prayer, unconsciously +--for he was not listening, but he knew the ground of old, and the +clergyman's regular route over it--and when a little trifle of new +matter was interlarded, his ear detected it and his whole nature +resented it; he considered additions unfair, and scoundrelly. In the +midst of the prayer a fly had lit on the back of the pew in front of +him and tortured his spirit by calmly rubbing its hands together, +embracing its head with its arms, and polishing it so vigorously that +it seemed to almost part company with the body, and the slender thread +of a neck was exposed to view; scraping its wings with its hind legs +and smoothing them to its body as if they had been coat-tails; going +through its whole toilet as tranquilly as if it knew it was perfectly +safe. As indeed it was; for as sorely as Tom's hands itched to grab for +it they did not dare--he believed his soul would be instantly destroyed +if he did such a thing while the prayer was going on. But with the +closing sentence his hand began to curve and steal forward; and the +instant the "Amen" was out the fly was a prisoner of war. His aunt +detected the act and made him let it go. + +The minister gave out his text and droned along monotonously through +an argument that was so prosy that many a head by and by began to nod +--and yet it was an argument that dealt in limitless fire and brimstone +and thinned the predestined elect down to a company so small as to be +hardly worth the saving. Tom counted the pages of the sermon; after +church he always knew how many pages there had been, but he seldom knew +anything else about the discourse. However, this time he was really +interested for a little while. The minister made a grand and moving +picture of the assembling together of the world's hosts at the +millennium when the lion and the lamb should lie down together and a +little child should lead them. But the pathos, the lesson, the moral of +the great spectacle were lost upon the boy; he only thought of the +conspicuousness of the principal character before the on-looking +nations; his face lit with the thought, and he said to himself that he +wished he could be that child, if it was a tame lion. + +Now he lapsed into suffering again, as the dry argument was resumed. +Presently he bethought him of a treasure he had and got it out. It was +a large black beetle with formidable jaws--a "pinchbug," he called it. +It was in a percussion-cap box. The first thing the beetle did was to +take him by the finger. A natural fillip followed, the beetle went +floundering into the aisle and lit on its back, and the hurt finger +went into the boy's mouth. The beetle lay there working its helpless +legs, unable to turn over. Tom eyed it, and longed for it; but it was +safe out of his reach. Other people uninterested in the sermon found +relief in the beetle, and they eyed it too. Presently a vagrant poodle +dog came idling along, sad at heart, lazy with the summer softness and +the quiet, weary of captivity, sighing for change. He spied the beetle; +the drooping tail lifted and wagged. He surveyed the prize; walked +around it; smelt at it from a safe distance; walked around it again; +grew bolder, and took a closer smell; then lifted his lip and made a +gingerly snatch at it, just missing it; made another, and another; +began to enjoy the diversion; subsided to his stomach with the beetle +between his paws, and continued his experiments; grew weary at last, +and then indifferent and absent-minded. His head nodded, and little by +little his chin descended and touched the enemy, who seized it. There +was a sharp yelp, a flirt of the poodle's head, and the beetle fell a +couple of yards away, and lit on its back once more. The neighboring +spectators shook with a gentle inward joy, several faces went behind +fans and handkerchiefs, and Tom was entirely happy. The dog looked +foolish, and probably felt so; but there was resentment in his heart, +too, and a craving for revenge. So he went to the beetle and began a +wary attack on it again; jumping at it from every point of a circle, +lighting with his fore-paws within an inch of the creature, making even +closer snatches at it with his teeth, and jerking his head till his +ears flapped again. But he grew tired once more, after a while; tried +to amuse himself with a fly but found no relief; followed an ant +around, with his nose close to the floor, and quickly wearied of that; +yawned, sighed, forgot the beetle entirely, and sat down on it. Then +there was a wild yelp of agony and the poodle went sailing up the +aisle; the yelps continued, and so did the dog; he crossed the house in +front of the altar; he flew down the other aisle; he crossed before the +doors; he clamored up the home-stretch; his anguish grew with his +progress, till presently he was but a woolly comet moving in its orbit +with the gleam and the speed of light. At last the frantic sufferer +sheered from its course, and sprang into its master's lap; he flung it +out of the window, and the voice of distress quickly thinned away and +died in the distance. + +By this time the whole church was red-faced and suffocating with +suppressed laughter, and the sermon had come to a dead standstill. The +discourse was resumed presently, but it went lame and halting, all +possibility of impressiveness being at an end; for even the gravest +sentiments were constantly being received with a smothered burst of +unholy mirth, under cover of some remote pew-back, as if the poor +parson had said a rarely facetious thing. It was a genuine relief to +the whole congregation when the ordeal was over and the benediction +pronounced. + +Tom Sawyer went home quite cheerful, thinking to himself that there +was some satisfaction about divine service when there was a bit of +variety in it. He had but one marring thought; he was willing that the +dog should play with his pinchbug, but he did not think it was upright +in him to carry it off. + + + +CHAPTER VI + +MONDAY morning found Tom Sawyer miserable. Monday morning always found +him so--because it began another week's slow suffering in school. He +generally began that day with wishing he had had no intervening +holiday, it made the going into captivity and fetters again so much +more odious. + +Tom lay thinking. Presently it occurred to him that he wished he was +sick; then he could stay home from school. Here was a vague +possibility. He canvassed his system. No ailment was found, and he +investigated again. This time he thought he could detect colicky +symptoms, and he began to encourage them with considerable hope. But +they soon grew feeble, and presently died wholly away. He reflected +further. Suddenly he discovered something. One of his upper front teeth +was loose. This was lucky; he was about to begin to groan, as a +"starter," as he called it, when it occurred to him that if he came +into court with that argument, his aunt would pull it out, and that +would hurt. So he thought he would hold the tooth in reserve for the +present, and seek further. Nothing offered for some little time, and +then he remembered hearing the doctor tell about a certain thing that +laid up a patient for two or three weeks and threatened to make him +lose a finger. So the boy eagerly drew his sore toe from under the +sheet and held it up for inspection. But now he did not know the +necessary symptoms. However, it seemed well worth while to chance it, +so he fell to groaning with considerable spirit. + +But Sid slept on unconscious. + +Tom groaned louder, and fancied that he began to feel pain in the toe. + +No result from Sid. + +Tom was panting with his exertions by this time. He took a rest and +then swelled himself up and fetched a succession of admirable groans. + +Sid snored on. + +Tom was aggravated. He said, "Sid, Sid!" and shook him. This course +worked well, and Tom began to groan again. Sid yawned, stretched, then +brought himself up on his elbow with a snort, and began to stare at +Tom. Tom went on groaning. Sid said: + +"Tom! Say, Tom!" [No response.] "Here, Tom! TOM! What is the matter, +Tom?" And he shook him and looked in his face anxiously. + +Tom moaned out: + +"Oh, don't, Sid. Don't joggle me." + +"Why, what's the matter, Tom? I must call auntie." + +"No--never mind. It'll be over by and by, maybe. Don't call anybody." + +"But I must! DON'T groan so, Tom, it's awful. How long you been this +way?" + +"Hours. Ouch! Oh, don't stir so, Sid, you'll kill me." + +"Tom, why didn't you wake me sooner? Oh, Tom, DON'T! It makes my +flesh crawl to hear you. Tom, what is the matter?" + +"I forgive you everything, Sid. [Groan.] Everything you've ever done +to me. When I'm gone--" + +"Oh, Tom, you ain't dying, are you? Don't, Tom--oh, don't. Maybe--" + +"I forgive everybody, Sid. [Groan.] Tell 'em so, Sid. And Sid, you +give my window-sash and my cat with one eye to that new girl that's +come to town, and tell her--" + +But Sid had snatched his clothes and gone. Tom was suffering in +reality, now, so handsomely was his imagination working, and so his +groans had gathered quite a genuine tone. + +Sid flew down-stairs and said: + +"Oh, Aunt Polly, come! Tom's dying!" + +"Dying!" + +"Yes'm. Don't wait--come quick!" + +"Rubbage! I don't believe it!" + +But she fled up-stairs, nevertheless, with Sid and Mary at her heels. +And her face grew white, too, and her lip trembled. When she reached +the bedside she gasped out: + +"You, Tom! Tom, what's the matter with you?" + +"Oh, auntie, I'm--" + +"What's the matter with you--what is the matter with you, child?" + +"Oh, auntie, my sore toe's mortified!" + +The old lady sank down into a chair and laughed a little, then cried a +little, then did both together. This restored her and she said: + +"Tom, what a turn you did give me. Now you shut up that nonsense and +climb out of this." + +The groans ceased and the pain vanished from the toe. The boy felt a +little foolish, and he said: + +"Aunt Polly, it SEEMED mortified, and it hurt so I never minded my +tooth at all." + +"Your tooth, indeed! What's the matter with your tooth?" + +"One of them's loose, and it aches perfectly awful." + +"There, there, now, don't begin that groaning again. Open your mouth. +Well--your tooth IS loose, but you're not going to die about that. +Mary, get me a silk thread, and a chunk of fire out of the kitchen." + +Tom said: + +"Oh, please, auntie, don't pull it out. It don't hurt any more. I wish +I may never stir if it does. Please don't, auntie. I don't want to stay +home from school." + +"Oh, you don't, don't you? So all this row was because you thought +you'd get to stay home from school and go a-fishing? Tom, Tom, I love +you so, and you seem to try every way you can to break my old heart +with your outrageousness." By this time the dental instruments were +ready. The old lady made one end of the silk thread fast to Tom's tooth +with a loop and tied the other to the bedpost. Then she seized the +chunk of fire and suddenly thrust it almost into the boy's face. The +tooth hung dangling by the bedpost, now. + +But all trials bring their compensations. As Tom wended to school +after breakfast, he was the envy of every boy he met because the gap in +his upper row of teeth enabled him to expectorate in a new and +admirable way. He gathered quite a following of lads interested in the +exhibition; and one that had cut his finger and had been a centre of +fascination and homage up to this time, now found himself suddenly +without an adherent, and shorn of his glory. His heart was heavy, and +he said with a disdain which he did not feel that it wasn't anything to +spit like Tom Sawyer; but another boy said, "Sour grapes!" and he +wandered away a dismantled hero. + +Shortly Tom came upon the juvenile pariah of the village, Huckleberry +Finn, son of the town drunkard. Huckleberry was cordially hated and +dreaded by all the mothers of the town, because he was idle and lawless +and vulgar and bad--and because all their children admired him so, and +delighted in his forbidden society, and wished they dared to be like +him. Tom was like the rest of the respectable boys, in that he envied +Huckleberry his gaudy outcast condition, and was under strict orders +not to play with him. So he played with him every time he got a chance. +Huckleberry was always dressed in the cast-off clothes of full-grown +men, and they were in perennial bloom and fluttering with rags. His hat +was a vast ruin with a wide crescent lopped out of its brim; his coat, +when he wore one, hung nearly to his heels and had the rearward buttons +far down the back; but one suspender supported his trousers; the seat +of the trousers bagged low and contained nothing, the fringed legs +dragged in the dirt when not rolled up. + +Huckleberry came and went, at his own free will. He slept on doorsteps +in fine weather and in empty hogsheads in wet; he did not have to go to +school or to church, or call any being master or obey anybody; he could +go fishing or swimming when and where he chose, and stay as long as it +suited him; nobody forbade him to fight; he could sit up as late as he +pleased; he was always the first boy that went barefoot in the spring +and the last to resume leather in the fall; he never had to wash, nor +put on clean clothes; he could swear wonderfully. In a word, everything +that goes to make life precious that boy had. So thought every +harassed, hampered, respectable boy in St. Petersburg. + +Tom hailed the romantic outcast: + +"Hello, Huckleberry!" + +"Hello yourself, and see how you like it." + +"What's that you got?" + +"Dead cat." + +"Lemme see him, Huck. My, he's pretty stiff. Where'd you get him?" + +"Bought him off'n a boy." + +"What did you give?" + +"I give a blue ticket and a bladder that I got at the slaughter-house." + +"Where'd you get the blue ticket?" + +"Bought it off'n Ben Rogers two weeks ago for a hoop-stick." + +"Say--what is dead cats good for, Huck?" + +"Good for? Cure warts with." + +"No! Is that so? I know something that's better." + +"I bet you don't. What is it?" + +"Why, spunk-water." + +"Spunk-water! I wouldn't give a dern for spunk-water." + +"You wouldn't, wouldn't you? D'you ever try it?" + +"No, I hain't. But Bob Tanner did." + +"Who told you so!" + +"Why, he told Jeff Thatcher, and Jeff told Johnny Baker, and Johnny +told Jim Hollis, and Jim told Ben Rogers, and Ben told a nigger, and +the nigger told me. There now!" + +"Well, what of it? They'll all lie. Leastways all but the nigger. I +don't know HIM. But I never see a nigger that WOULDN'T lie. Shucks! Now +you tell me how Bob Tanner done it, Huck." + +"Why, he took and dipped his hand in a rotten stump where the +rain-water was." + +"In the daytime?" + +"Certainly." + +"With his face to the stump?" + +"Yes. Least I reckon so." + +"Did he say anything?" + +"I don't reckon he did. I don't know." + +"Aha! Talk about trying to cure warts with spunk-water such a blame +fool way as that! Why, that ain't a-going to do any good. You got to go +all by yourself, to the middle of the woods, where you know there's a +spunk-water stump, and just as it's midnight you back up against the +stump and jam your hand in and say: + + 'Barley-corn, barley-corn, injun-meal shorts, + Spunk-water, spunk-water, swaller these warts,' + +and then walk away quick, eleven steps, with your eyes shut, and then +turn around three times and walk home without speaking to anybody. +Because if you speak the charm's busted." + +"Well, that sounds like a good way; but that ain't the way Bob Tanner +done." + +"No, sir, you can bet he didn't, becuz he's the wartiest boy in this +town; and he wouldn't have a wart on him if he'd knowed how to work +spunk-water. I've took off thousands of warts off of my hands that way, +Huck. I play with frogs so much that I've always got considerable many +warts. Sometimes I take 'em off with a bean." + +"Yes, bean's good. I've done that." + +"Have you? What's your way?" + +"You take and split the bean, and cut the wart so as to get some +blood, and then you put the blood on one piece of the bean and take and +dig a hole and bury it 'bout midnight at the crossroads in the dark of +the moon, and then you burn up the rest of the bean. You see that piece +that's got the blood on it will keep drawing and drawing, trying to +fetch the other piece to it, and so that helps the blood to draw the +wart, and pretty soon off she comes." + +"Yes, that's it, Huck--that's it; though when you're burying it if you +say 'Down bean; off wart; come no more to bother me!' it's better. +That's the way Joe Harper does, and he's been nearly to Coonville and +most everywheres. But say--how do you cure 'em with dead cats?" + +"Why, you take your cat and go and get in the graveyard 'long about +midnight when somebody that was wicked has been buried; and when it's +midnight a devil will come, or maybe two or three, but you can't see +'em, you can only hear something like the wind, or maybe hear 'em talk; +and when they're taking that feller away, you heave your cat after 'em +and say, 'Devil follow corpse, cat follow devil, warts follow cat, I'm +done with ye!' That'll fetch ANY wart." + +"Sounds right. D'you ever try it, Huck?" + +"No, but old Mother Hopkins told me." + +"Well, I reckon it's so, then. Becuz they say she's a witch." + +"Say! Why, Tom, I KNOW she is. She witched pap. Pap says so his own +self. He come along one day, and he see she was a-witching him, so he +took up a rock, and if she hadn't dodged, he'd a got her. Well, that +very night he rolled off'n a shed wher' he was a layin drunk, and broke +his arm." + +"Why, that's awful. How did he know she was a-witching him?" + +"Lord, pap can tell, easy. Pap says when they keep looking at you +right stiddy, they're a-witching you. Specially if they mumble. Becuz +when they mumble they're saying the Lord's Prayer backards." + +"Say, Hucky, when you going to try the cat?" + +"To-night. I reckon they'll come after old Hoss Williams to-night." + +"But they buried him Saturday. Didn't they get him Saturday night?" + +"Why, how you talk! How could their charms work till midnight?--and +THEN it's Sunday. Devils don't slosh around much of a Sunday, I don't +reckon." + +"I never thought of that. That's so. Lemme go with you?" + +"Of course--if you ain't afeard." + +"Afeard! 'Tain't likely. Will you meow?" + +"Yes--and you meow back, if you get a chance. Last time, you kep' me +a-meowing around till old Hays went to throwing rocks at me and says +'Dern that cat!' and so I hove a brick through his window--but don't +you tell." + +"I won't. I couldn't meow that night, becuz auntie was watching me, +but I'll meow this time. Say--what's that?" + +"Nothing but a tick." + +"Where'd you get him?" + +"Out in the woods." + +"What'll you take for him?" + +"I don't know. I don't want to sell him." + +"All right. It's a mighty small tick, anyway." + +"Oh, anybody can run a tick down that don't belong to them. I'm +satisfied with it. It's a good enough tick for me." + +"Sho, there's ticks a plenty. I could have a thousand of 'em if I +wanted to." + +"Well, why don't you? Becuz you know mighty well you can't. This is a +pretty early tick, I reckon. It's the first one I've seen this year." + +"Say, Huck--I'll give you my tooth for him." + +"Less see it." + +Tom got out a bit of paper and carefully unrolled it. Huckleberry +viewed it wistfully. The temptation was very strong. At last he said: + +"Is it genuwyne?" + +Tom lifted his lip and showed the vacancy. + +"Well, all right," said Huckleberry, "it's a trade." + +Tom enclosed the tick in the percussion-cap box that had lately been +the pinchbug's prison, and the boys separated, each feeling wealthier +than before. + +When Tom reached the little isolated frame schoolhouse, he strode in +briskly, with the manner of one who had come with all honest speed. +He hung his hat on a peg and flung himself into his seat with +business-like alacrity. The master, throned on high in his great +splint-bottom arm-chair, was dozing, lulled by the drowsy hum of study. +The interruption roused him. + +"Thomas Sawyer!" + +Tom knew that when his name was pronounced in full, it meant trouble. + +"Sir!" + +"Come up here. Now, sir, why are you late again, as usual?" + +Tom was about to take refuge in a lie, when he saw two long tails of +yellow hair hanging down a back that he recognized by the electric +sympathy of love; and by that form was THE ONLY VACANT PLACE on the +girls' side of the schoolhouse. He instantly said: + +"I STOPPED TO TALK WITH HUCKLEBERRY FINN!" + +The master's pulse stood still, and he stared helplessly. The buzz of +study ceased. The pupils wondered if this foolhardy boy had lost his +mind. The master said: + +"You--you did what?" + +"Stopped to talk with Huckleberry Finn." + +There was no mistaking the words. + +"Thomas Sawyer, this is the most astounding confession I have ever +listened to. No mere ferule will answer for this offence. Take off your +jacket." + +The master's arm performed until it was tired and the stock of +switches notably diminished. Then the order followed: + +"Now, sir, go and sit with the girls! And let this be a warning to you." + +The titter that rippled around the room appeared to abash the boy, but +in reality that result was caused rather more by his worshipful awe of +his unknown idol and the dread pleasure that lay in his high good +fortune. He sat down upon the end of the pine bench and the girl +hitched herself away from him with a toss of her head. Nudges and winks +and whispers traversed the room, but Tom sat still, with his arms upon +the long, low desk before him, and seemed to study his book. + +By and by attention ceased from him, and the accustomed school murmur +rose upon the dull air once more. Presently the boy began to steal +furtive glances at the girl. She observed it, "made a mouth" at him and +gave him the back of her head for the space of a minute. When she +cautiously faced around again, a peach lay before her. She thrust it +away. Tom gently put it back. She thrust it away again, but with less +animosity. Tom patiently returned it to its place. Then she let it +remain. Tom scrawled on his slate, "Please take it--I got more." The +girl glanced at the words, but made no sign. Now the boy began to draw +something on the slate, hiding his work with his left hand. For a time +the girl refused to notice; but her human curiosity presently began to +manifest itself by hardly perceptible signs. The boy worked on, +apparently unconscious. The girl made a sort of noncommittal attempt to +see, but the boy did not betray that he was aware of it. At last she +gave in and hesitatingly whispered: + +"Let me see it." + +Tom partly uncovered a dismal caricature of a house with two gable +ends to it and a corkscrew of smoke issuing from the chimney. Then the +girl's interest began to fasten itself upon the work and she forgot +everything else. When it was finished, she gazed a moment, then +whispered: + +"It's nice--make a man." + +The artist erected a man in the front yard, that resembled a derrick. +He could have stepped over the house; but the girl was not +hypercritical; she was satisfied with the monster, and whispered: + +"It's a beautiful man--now make me coming along." + +Tom drew an hour-glass with a full moon and straw limbs to it and +armed the spreading fingers with a portentous fan. The girl said: + +"It's ever so nice--I wish I could draw." + +"It's easy," whispered Tom, "I'll learn you." + +"Oh, will you? When?" + +"At noon. Do you go home to dinner?" + +"I'll stay if you will." + +"Good--that's a whack. What's your name?" + +"Becky Thatcher. What's yours? Oh, I know. It's Thomas Sawyer." + +"That's the name they lick me by. I'm Tom when I'm good. You call me +Tom, will you?" + +"Yes." + +Now Tom began to scrawl something on the slate, hiding the words from +the girl. But she was not backward this time. She begged to see. Tom +said: + +"Oh, it ain't anything." + +"Yes it is." + +"No it ain't. You don't want to see." + +"Yes I do, indeed I do. Please let me." + +"You'll tell." + +"No I won't--deed and deed and double deed won't." + +"You won't tell anybody at all? Ever, as long as you live?" + +"No, I won't ever tell ANYbody. Now let me." + +"Oh, YOU don't want to see!" + +"Now that you treat me so, I WILL see." And she put her small hand +upon his and a little scuffle ensued, Tom pretending to resist in +earnest but letting his hand slip by degrees till these words were +revealed: "I LOVE YOU." + +"Oh, you bad thing!" And she hit his hand a smart rap, but reddened +and looked pleased, nevertheless. + +Just at this juncture the boy felt a slow, fateful grip closing on his +ear, and a steady lifting impulse. In that wise he was borne across the +house and deposited in his own seat, under a peppering fire of giggles +from the whole school. Then the master stood over him during a few +awful moments, and finally moved away to his throne without saying a +word. But although Tom's ear tingled, his heart was jubilant. + +As the school quieted down Tom made an honest effort to study, but the +turmoil within him was too great. In turn he took his place in the +reading class and made a botch of it; then in the geography class and +turned lakes into mountains, mountains into rivers, and rivers into +continents, till chaos was come again; then in the spelling class, and +got "turned down," by a succession of mere baby words, till he brought +up at the foot and yielded up the pewter medal which he had worn with +ostentation for months. + + + +CHAPTER VII + +THE harder Tom tried to fasten his mind on his book, the more his +ideas wandered. So at last, with a sigh and a yawn, he gave it up. It +seemed to him that the noon recess would never come. The air was +utterly dead. There was not a breath stirring. It was the sleepiest of +sleepy days. The drowsing murmur of the five and twenty studying +scholars soothed the soul like the spell that is in the murmur of bees. +Away off in the flaming sunshine, Cardiff Hill lifted its soft green +sides through a shimmering veil of heat, tinted with the purple of +distance; a few birds floated on lazy wing high in the air; no other +living thing was visible but some cows, and they were asleep. Tom's +heart ached to be free, or else to have something of interest to do to +pass the dreary time. His hand wandered into his pocket and his face +lit up with a glow of gratitude that was prayer, though he did not know +it. Then furtively the percussion-cap box came out. He released the +tick and put him on the long flat desk. The creature probably glowed +with a gratitude that amounted to prayer, too, at this moment, but it +was premature: for when he started thankfully to travel off, Tom turned +him aside with a pin and made him take a new direction. + +Tom's bosom friend sat next him, suffering just as Tom had been, and +now he was deeply and gratefully interested in this entertainment in an +instant. This bosom friend was Joe Harper. The two boys were sworn +friends all the week, and embattled enemies on Saturdays. Joe took a +pin out of his lapel and began to assist in exercising the prisoner. +The sport grew in interest momently. Soon Tom said that they were +interfering with each other, and neither getting the fullest benefit of +the tick. So he put Joe's slate on the desk and drew a line down the +middle of it from top to bottom. + +"Now," said he, "as long as he is on your side you can stir him up and +I'll let him alone; but if you let him get away and get on my side, +you're to leave him alone as long as I can keep him from crossing over." + +"All right, go ahead; start him up." + +The tick escaped from Tom, presently, and crossed the equator. Joe +harassed him awhile, and then he got away and crossed back again. This +change of base occurred often. While one boy was worrying the tick with +absorbing interest, the other would look on with interest as strong, +the two heads bowed together over the slate, and the two souls dead to +all things else. At last luck seemed to settle and abide with Joe. The +tick tried this, that, and the other course, and got as excited and as +anxious as the boys themselves, but time and again just as he would +have victory in his very grasp, so to speak, and Tom's fingers would be +twitching to begin, Joe's pin would deftly head him off, and keep +possession. At last Tom could stand it no longer. The temptation was +too strong. So he reached out and lent a hand with his pin. Joe was +angry in a moment. Said he: + +"Tom, you let him alone." + +"I only just want to stir him up a little, Joe." + +"No, sir, it ain't fair; you just let him alone." + +"Blame it, I ain't going to stir him much." + +"Let him alone, I tell you." + +"I won't!" + +"You shall--he's on my side of the line." + +"Look here, Joe Harper, whose is that tick?" + +"I don't care whose tick he is--he's on my side of the line, and you +sha'n't touch him." + +"Well, I'll just bet I will, though. He's my tick and I'll do what I +blame please with him, or die!" + +A tremendous whack came down on Tom's shoulders, and its duplicate on +Joe's; and for the space of two minutes the dust continued to fly from +the two jackets and the whole school to enjoy it. The boys had been too +absorbed to notice the hush that had stolen upon the school awhile +before when the master came tiptoeing down the room and stood over +them. He had contemplated a good part of the performance before he +contributed his bit of variety to it. + +When school broke up at noon, Tom flew to Becky Thatcher, and +whispered in her ear: + +"Put on your bonnet and let on you're going home; and when you get to +the corner, give the rest of 'em the slip, and turn down through the +lane and come back. I'll go the other way and come it over 'em the same +way." + +So the one went off with one group of scholars, and the other with +another. In a little while the two met at the bottom of the lane, and +when they reached the school they had it all to themselves. Then they +sat together, with a slate before them, and Tom gave Becky the pencil +and held her hand in his, guiding it, and so created another surprising +house. When the interest in art began to wane, the two fell to talking. +Tom was swimming in bliss. He said: + +"Do you love rats?" + +"No! I hate them!" + +"Well, I do, too--LIVE ones. But I mean dead ones, to swing round your +head with a string." + +"No, I don't care for rats much, anyway. What I like is chewing-gum." + +"Oh, I should say so! I wish I had some now." + +"Do you? I've got some. I'll let you chew it awhile, but you must give +it back to me." + +That was agreeable, so they chewed it turn about, and dangled their +legs against the bench in excess of contentment. + +"Was you ever at a circus?" said Tom. + +"Yes, and my pa's going to take me again some time, if I'm good." + +"I been to the circus three or four times--lots of times. Church ain't +shucks to a circus. There's things going on at a circus all the time. +I'm going to be a clown in a circus when I grow up." + +"Oh, are you! That will be nice. They're so lovely, all spotted up." + +"Yes, that's so. And they get slathers of money--most a dollar a day, +Ben Rogers says. Say, Becky, was you ever engaged?" + +"What's that?" + +"Why, engaged to be married." + +"No." + +"Would you like to?" + +"I reckon so. I don't know. What is it like?" + +"Like? Why it ain't like anything. You only just tell a boy you won't +ever have anybody but him, ever ever ever, and then you kiss and that's +all. Anybody can do it." + +"Kiss? What do you kiss for?" + +"Why, that, you know, is to--well, they always do that." + +"Everybody?" + +"Why, yes, everybody that's in love with each other. Do you remember +what I wrote on the slate?" + +"Ye--yes." + +"What was it?" + +"I sha'n't tell you." + +"Shall I tell YOU?" + +"Ye--yes--but some other time." + +"No, now." + +"No, not now--to-morrow." + +"Oh, no, NOW. Please, Becky--I'll whisper it, I'll whisper it ever so +easy." + +Becky hesitating, Tom took silence for consent, and passed his arm +about her waist and whispered the tale ever so softly, with his mouth +close to her ear. And then he added: + +"Now you whisper it to me--just the same." + +She resisted, for a while, and then said: + +"You turn your face away so you can't see, and then I will. But you +mustn't ever tell anybody--WILL you, Tom? Now you won't, WILL you?" + +"No, indeed, indeed I won't. Now, Becky." + +He turned his face away. She bent timidly around till her breath +stirred his curls and whispered, "I--love--you!" + +Then she sprang away and ran around and around the desks and benches, +with Tom after her, and took refuge in a corner at last, with her +little white apron to her face. Tom clasped her about her neck and +pleaded: + +"Now, Becky, it's all done--all over but the kiss. Don't you be afraid +of that--it ain't anything at all. Please, Becky." And he tugged at her +apron and the hands. + +By and by she gave up, and let her hands drop; her face, all glowing +with the struggle, came up and submitted. Tom kissed the red lips and +said: + +"Now it's all done, Becky. And always after this, you know, you ain't +ever to love anybody but me, and you ain't ever to marry anybody but +me, ever never and forever. Will you?" + +"No, I'll never love anybody but you, Tom, and I'll never marry +anybody but you--and you ain't to ever marry anybody but me, either." + +"Certainly. Of course. That's PART of it. And always coming to school +or when we're going home, you're to walk with me, when there ain't +anybody looking--and you choose me and I choose you at parties, because +that's the way you do when you're engaged." + +"It's so nice. I never heard of it before." + +"Oh, it's ever so gay! Why, me and Amy Lawrence--" + +The big eyes told Tom his blunder and he stopped, confused. + +"Oh, Tom! Then I ain't the first you've ever been engaged to!" + +The child began to cry. Tom said: + +"Oh, don't cry, Becky, I don't care for her any more." + +"Yes, you do, Tom--you know you do." + +Tom tried to put his arm about her neck, but she pushed him away and +turned her face to the wall, and went on crying. Tom tried again, with +soothing words in his mouth, and was repulsed again. Then his pride was +up, and he strode away and went outside. He stood about, restless and +uneasy, for a while, glancing at the door, every now and then, hoping +she would repent and come to find him. But she did not. Then he began +to feel badly and fear that he was in the wrong. It was a hard struggle +with him to make new advances, now, but he nerved himself to it and +entered. She was still standing back there in the corner, sobbing, with +her face to the wall. Tom's heart smote him. He went to her and stood a +moment, not knowing exactly how to proceed. Then he said hesitatingly: + +"Becky, I--I don't care for anybody but you." + +No reply--but sobs. + +"Becky"--pleadingly. "Becky, won't you say something?" + +More sobs. + +Tom got out his chiefest jewel, a brass knob from the top of an +andiron, and passed it around her so that she could see it, and said: + +"Please, Becky, won't you take it?" + +She struck it to the floor. Then Tom marched out of the house and over +the hills and far away, to return to school no more that day. Presently +Becky began to suspect. She ran to the door; he was not in sight; she +flew around to the play-yard; he was not there. Then she called: + +"Tom! Come back, Tom!" + +She listened intently, but there was no answer. She had no companions +but silence and loneliness. So she sat down to cry again and upbraid +herself; and by this time the scholars began to gather again, and she +had to hide her griefs and still her broken heart and take up the cross +of a long, dreary, aching afternoon, with none among the strangers +about her to exchange sorrows with. + + + +CHAPTER VIII + +TOM dodged hither and thither through lanes until he was well out of +the track of returning scholars, and then fell into a moody jog. He +crossed a small "branch" two or three times, because of a prevailing +juvenile superstition that to cross water baffled pursuit. Half an hour +later he was disappearing behind the Douglas mansion on the summit of +Cardiff Hill, and the schoolhouse was hardly distinguishable away off +in the valley behind him. He entered a dense wood, picked his pathless +way to the centre of it, and sat down on a mossy spot under a spreading +oak. There was not even a zephyr stirring; the dead noonday heat had +even stilled the songs of the birds; nature lay in a trance that was +broken by no sound but the occasional far-off hammering of a +woodpecker, and this seemed to render the pervading silence and sense +of loneliness the more profound. The boy's soul was steeped in +melancholy; his feelings were in happy accord with his surroundings. He +sat long with his elbows on his knees and his chin in his hands, +meditating. It seemed to him that life was but a trouble, at best, and +he more than half envied Jimmy Hodges, so lately released; it must be +very peaceful, he thought, to lie and slumber and dream forever and +ever, with the wind whispering through the trees and caressing the +grass and the flowers over the grave, and nothing to bother and grieve +about, ever any more. If he only had a clean Sunday-school record he +could be willing to go, and be done with it all. Now as to this girl. +What had he done? Nothing. He had meant the best in the world, and been +treated like a dog--like a very dog. She would be sorry some day--maybe +when it was too late. Ah, if he could only die TEMPORARILY! + +But the elastic heart of youth cannot be compressed into one +constrained shape long at a time. Tom presently began to drift +insensibly back into the concerns of this life again. What if he turned +his back, now, and disappeared mysteriously? What if he went away--ever +so far away, into unknown countries beyond the seas--and never came +back any more! How would she feel then! The idea of being a clown +recurred to him now, only to fill him with disgust. For frivolity and +jokes and spotted tights were an offense, when they intruded themselves +upon a spirit that was exalted into the vague august realm of the +romantic. No, he would be a soldier, and return after long years, all +war-worn and illustrious. No--better still, he would join the Indians, +and hunt buffaloes and go on the warpath in the mountain ranges and the +trackless great plains of the Far West, and away in the future come +back a great chief, bristling with feathers, hideous with paint, and +prance into Sunday-school, some drowsy summer morning, with a +bloodcurdling war-whoop, and sear the eyeballs of all his companions +with unappeasable envy. But no, there was something gaudier even than +this. He would be a pirate! That was it! NOW his future lay plain +before him, and glowing with unimaginable splendor. How his name would +fill the world, and make people shudder! How gloriously he would go +plowing the dancing seas, in his long, low, black-hulled racer, the +Spirit of the Storm, with his grisly flag flying at the fore! And at +the zenith of his fame, how he would suddenly appear at the old village +and stalk into church, brown and weather-beaten, in his black velvet +doublet and trunks, his great jack-boots, his crimson sash, his belt +bristling with horse-pistols, his crime-rusted cutlass at his side, his +slouch hat with waving plumes, his black flag unfurled, with the skull +and crossbones on it, and hear with swelling ecstasy the whisperings, +"It's Tom Sawyer the Pirate!--the Black Avenger of the Spanish Main!" + +Yes, it was settled; his career was determined. He would run away from +home and enter upon it. He would start the very next morning. Therefore +he must now begin to get ready. He would collect his resources +together. He went to a rotten log near at hand and began to dig under +one end of it with his Barlow knife. He soon struck wood that sounded +hollow. He put his hand there and uttered this incantation impressively: + +"What hasn't come here, come! What's here, stay here!" + +Then he scraped away the dirt, and exposed a pine shingle. He took it +up and disclosed a shapely little treasure-house whose bottom and sides +were of shingles. In it lay a marble. Tom's astonishment was boundless! +He scratched his head with a perplexed air, and said: + +"Well, that beats anything!" + +Then he tossed the marble away pettishly, and stood cogitating. The +truth was, that a superstition of his had failed, here, which he and +all his comrades had always looked upon as infallible. If you buried a +marble with certain necessary incantations, and left it alone a +fortnight, and then opened the place with the incantation he had just +used, you would find that all the marbles you had ever lost had +gathered themselves together there, meantime, no matter how widely they +had been separated. But now, this thing had actually and unquestionably +failed. Tom's whole structure of faith was shaken to its foundations. +He had many a time heard of this thing succeeding but never of its +failing before. It did not occur to him that he had tried it several +times before, himself, but could never find the hiding-places +afterward. He puzzled over the matter some time, and finally decided +that some witch had interfered and broken the charm. He thought he +would satisfy himself on that point; so he searched around till he +found a small sandy spot with a little funnel-shaped depression in it. +He laid himself down and put his mouth close to this depression and +called-- + +"Doodle-bug, doodle-bug, tell me what I want to know! Doodle-bug, +doodle-bug, tell me what I want to know!" + +The sand began to work, and presently a small black bug appeared for a +second and then darted under again in a fright. + +"He dasn't tell! So it WAS a witch that done it. I just knowed it." + +He well knew the futility of trying to contend against witches, so he +gave up discouraged. But it occurred to him that he might as well have +the marble he had just thrown away, and therefore he went and made a +patient search for it. But he could not find it. Now he went back to +his treasure-house and carefully placed himself just as he had been +standing when he tossed the marble away; then he took another marble +from his pocket and tossed it in the same way, saying: + +"Brother, go find your brother!" + +He watched where it stopped, and went there and looked. But it must +have fallen short or gone too far; so he tried twice more. The last +repetition was successful. The two marbles lay within a foot of each +other. + +Just here the blast of a toy tin trumpet came faintly down the green +aisles of the forest. Tom flung off his jacket and trousers, turned a +suspender into a belt, raked away some brush behind the rotten log, +disclosing a rude bow and arrow, a lath sword and a tin trumpet, and in +a moment had seized these things and bounded away, barelegged, with +fluttering shirt. He presently halted under a great elm, blew an +answering blast, and then began to tiptoe and look warily out, this way +and that. He said cautiously--to an imaginary company: + +"Hold, my merry men! Keep hid till I blow." + +Now appeared Joe Harper, as airily clad and elaborately armed as Tom. +Tom called: + +"Hold! Who comes here into Sherwood Forest without my pass?" + +"Guy of Guisborne wants no man's pass. Who art thou that--that--" + +"Dares to hold such language," said Tom, prompting--for they talked +"by the book," from memory. + +"Who art thou that dares to hold such language?" + +"I, indeed! I am Robin Hood, as thy caitiff carcase soon shall know." + +"Then art thou indeed that famous outlaw? Right gladly will I dispute +with thee the passes of the merry wood. Have at thee!" + +They took their lath swords, dumped their other traps on the ground, +struck a fencing attitude, foot to foot, and began a grave, careful +combat, "two up and two down." Presently Tom said: + +"Now, if you've got the hang, go it lively!" + +So they "went it lively," panting and perspiring with the work. By and +by Tom shouted: + +"Fall! fall! Why don't you fall?" + +"I sha'n't! Why don't you fall yourself? You're getting the worst of +it." + +"Why, that ain't anything. I can't fall; that ain't the way it is in +the book. The book says, 'Then with one back-handed stroke he slew poor +Guy of Guisborne.' You're to turn around and let me hit you in the +back." + +There was no getting around the authorities, so Joe turned, received +the whack and fell. + +"Now," said Joe, getting up, "you got to let me kill YOU. That's fair." + +"Why, I can't do that, it ain't in the book." + +"Well, it's blamed mean--that's all." + +"Well, say, Joe, you can be Friar Tuck or Much the miller's son, and +lam me with a quarter-staff; or I'll be the Sheriff of Nottingham and +you be Robin Hood a little while and kill me." + +This was satisfactory, and so these adventures were carried out. Then +Tom became Robin Hood again, and was allowed by the treacherous nun to +bleed his strength away through his neglected wound. And at last Joe, +representing a whole tribe of weeping outlaws, dragged him sadly forth, +gave his bow into his feeble hands, and Tom said, "Where this arrow +falls, there bury poor Robin Hood under the greenwood tree." Then he +shot the arrow and fell back and would have died, but he lit on a +nettle and sprang up too gaily for a corpse. + +The boys dressed themselves, hid their accoutrements, and went off +grieving that there were no outlaws any more, and wondering what modern +civilization could claim to have done to compensate for their loss. +They said they would rather be outlaws a year in Sherwood Forest than +President of the United States forever. + + + +CHAPTER IX + +AT half-past nine, that night, Tom and Sid were sent to bed, as usual. +They said their prayers, and Sid was soon asleep. Tom lay awake and +waited, in restless impatience. When it seemed to him that it must be +nearly daylight, he heard the clock strike ten! This was despair. He +would have tossed and fidgeted, as his nerves demanded, but he was +afraid he might wake Sid. So he lay still, and stared up into the dark. +Everything was dismally still. By and by, out of the stillness, little, +scarcely perceptible noises began to emphasize themselves. The ticking +of the clock began to bring itself into notice. Old beams began to +crack mysteriously. The stairs creaked faintly. Evidently spirits were +abroad. A measured, muffled snore issued from Aunt Polly's chamber. And +now the tiresome chirping of a cricket that no human ingenuity could +locate, began. Next the ghastly ticking of a deathwatch in the wall at +the bed's head made Tom shudder--it meant that somebody's days were +numbered. Then the howl of a far-off dog rose on the night air, and was +answered by a fainter howl from a remoter distance. Tom was in an +agony. At last he was satisfied that time had ceased and eternity +begun; he began to doze, in spite of himself; the clock chimed eleven, +but he did not hear it. And then there came, mingling with his +half-formed dreams, a most melancholy caterwauling. The raising of a +neighboring window disturbed him. A cry of "Scat! you devil!" and the +crash of an empty bottle against the back of his aunt's woodshed +brought him wide awake, and a single minute later he was dressed and +out of the window and creeping along the roof of the "ell" on all +fours. He "meow'd" with caution once or twice, as he went; then jumped +to the roof of the woodshed and thence to the ground. Huckleberry Finn +was there, with his dead cat. The boys moved off and disappeared in the +gloom. At the end of half an hour they were wading through the tall +grass of the graveyard. + +It was a graveyard of the old-fashioned Western kind. It was on a +hill, about a mile and a half from the village. It had a crazy board +fence around it, which leaned inward in places, and outward the rest of +the time, but stood upright nowhere. Grass and weeds grew rank over the +whole cemetery. All the old graves were sunken in, there was not a +tombstone on the place; round-topped, worm-eaten boards staggered over +the graves, leaning for support and finding none. "Sacred to the memory +of" So-and-So had been painted on them once, but it could no longer +have been read, on the most of them, now, even if there had been light. + +A faint wind moaned through the trees, and Tom feared it might be the +spirits of the dead, complaining at being disturbed. The boys talked +little, and only under their breath, for the time and the place and the +pervading solemnity and silence oppressed their spirits. They found the +sharp new heap they were seeking, and ensconced themselves within the +protection of three great elms that grew in a bunch within a few feet +of the grave. + +Then they waited in silence for what seemed a long time. The hooting +of a distant owl was all the sound that troubled the dead stillness. +Tom's reflections grew oppressive. He must force some talk. So he said +in a whisper: + +"Hucky, do you believe the dead people like it for us to be here?" + +Huckleberry whispered: + +"I wisht I knowed. It's awful solemn like, AIN'T it?" + +"I bet it is." + +There was a considerable pause, while the boys canvassed this matter +inwardly. Then Tom whispered: + +"Say, Hucky--do you reckon Hoss Williams hears us talking?" + +"O' course he does. Least his sperrit does." + +Tom, after a pause: + +"I wish I'd said Mister Williams. But I never meant any harm. +Everybody calls him Hoss." + +"A body can't be too partic'lar how they talk 'bout these-yer dead +people, Tom." + +This was a damper, and conversation died again. + +Presently Tom seized his comrade's arm and said: + +"Sh!" + +"What is it, Tom?" And the two clung together with beating hearts. + +"Sh! There 'tis again! Didn't you hear it?" + +"I--" + +"There! Now you hear it." + +"Lord, Tom, they're coming! They're coming, sure. What'll we do?" + +"I dono. Think they'll see us?" + +"Oh, Tom, they can see in the dark, same as cats. I wisht I hadn't +come." + +"Oh, don't be afeard. I don't believe they'll bother us. We ain't +doing any harm. If we keep perfectly still, maybe they won't notice us +at all." + +"I'll try to, Tom, but, Lord, I'm all of a shiver." + +"Listen!" + +The boys bent their heads together and scarcely breathed. A muffled +sound of voices floated up from the far end of the graveyard. + +"Look! See there!" whispered Tom. "What is it?" + +"It's devil-fire. Oh, Tom, this is awful." + +Some vague figures approached through the gloom, swinging an +old-fashioned tin lantern that freckled the ground with innumerable +little spangles of light. Presently Huckleberry whispered with a +shudder: + +"It's the devils sure enough. Three of 'em! Lordy, Tom, we're goners! +Can you pray?" + +"I'll try, but don't you be afeard. They ain't going to hurt us. 'Now +I lay me down to sleep, I--'" + +"Sh!" + +"What is it, Huck?" + +"They're HUMANS! One of 'em is, anyway. One of 'em's old Muff Potter's +voice." + +"No--'tain't so, is it?" + +"I bet I know it. Don't you stir nor budge. He ain't sharp enough to +notice us. Drunk, the same as usual, likely--blamed old rip!" + +"All right, I'll keep still. Now they're stuck. Can't find it. Here +they come again. Now they're hot. Cold again. Hot again. Red hot! +They're p'inted right, this time. Say, Huck, I know another o' them +voices; it's Injun Joe." + +"That's so--that murderin' half-breed! I'd druther they was devils a +dern sight. What kin they be up to?" + +The whisper died wholly out, now, for the three men had reached the +grave and stood within a few feet of the boys' hiding-place. + +"Here it is," said the third voice; and the owner of it held the +lantern up and revealed the face of young Doctor Robinson. + +Potter and Injun Joe were carrying a handbarrow with a rope and a +couple of shovels on it. They cast down their load and began to open +the grave. The doctor put the lantern at the head of the grave and came +and sat down with his back against one of the elm trees. He was so +close the boys could have touched him. + +"Hurry, men!" he said, in a low voice; "the moon might come out at any +moment." + +They growled a response and went on digging. For some time there was +no noise but the grating sound of the spades discharging their freight +of mould and gravel. It was very monotonous. Finally a spade struck +upon the coffin with a dull woody accent, and within another minute or +two the men had hoisted it out on the ground. They pried off the lid +with their shovels, got out the body and dumped it rudely on the +ground. The moon drifted from behind the clouds and exposed the pallid +face. The barrow was got ready and the corpse placed on it, covered +with a blanket, and bound to its place with the rope. Potter took out a +large spring-knife and cut off the dangling end of the rope and then +said: + +"Now the cussed thing's ready, Sawbones, and you'll just out with +another five, or here she stays." + +"That's the talk!" said Injun Joe. + +"Look here, what does this mean?" said the doctor. "You required your +pay in advance, and I've paid you." + +"Yes, and you done more than that," said Injun Joe, approaching the +doctor, who was now standing. "Five years ago you drove me away from +your father's kitchen one night, when I come to ask for something to +eat, and you said I warn't there for any good; and when I swore I'd get +even with you if it took a hundred years, your father had me jailed for +a vagrant. Did you think I'd forget? The Injun blood ain't in me for +nothing. And now I've GOT you, and you got to SETTLE, you know!" + +He was threatening the doctor, with his fist in his face, by this +time. The doctor struck out suddenly and stretched the ruffian on the +ground. Potter dropped his knife, and exclaimed: + +"Here, now, don't you hit my pard!" and the next moment he had +grappled with the doctor and the two were struggling with might and +main, trampling the grass and tearing the ground with their heels. +Injun Joe sprang to his feet, his eyes flaming with passion, snatched +up Potter's knife, and went creeping, catlike and stooping, round and +round about the combatants, seeking an opportunity. All at once the +doctor flung himself free, seized the heavy headboard of Williams' +grave and felled Potter to the earth with it--and in the same instant +the half-breed saw his chance and drove the knife to the hilt in the +young man's breast. He reeled and fell partly upon Potter, flooding him +with his blood, and in the same moment the clouds blotted out the +dreadful spectacle and the two frightened boys went speeding away in +the dark. + +Presently, when the moon emerged again, Injun Joe was standing over +the two forms, contemplating them. The doctor murmured inarticulately, +gave a long gasp or two and was still. The half-breed muttered: + +"THAT score is settled--damn you." + +Then he robbed the body. After which he put the fatal knife in +Potter's open right hand, and sat down on the dismantled coffin. Three +--four--five minutes passed, and then Potter began to stir and moan. His +hand closed upon the knife; he raised it, glanced at it, and let it +fall, with a shudder. Then he sat up, pushing the body from him, and +gazed at it, and then around him, confusedly. His eyes met Joe's. + +"Lord, how is this, Joe?" he said. + +"It's a dirty business," said Joe, without moving. + +"What did you do it for?" + +"I! I never done it!" + +"Look here! That kind of talk won't wash." + +Potter trembled and grew white. + +"I thought I'd got sober. I'd no business to drink to-night. But it's +in my head yet--worse'n when we started here. I'm all in a muddle; +can't recollect anything of it, hardly. Tell me, Joe--HONEST, now, old +feller--did I do it? Joe, I never meant to--'pon my soul and honor, I +never meant to, Joe. Tell me how it was, Joe. Oh, it's awful--and him +so young and promising." + +"Why, you two was scuffling, and he fetched you one with the headboard +and you fell flat; and then up you come, all reeling and staggering +like, and snatched the knife and jammed it into him, just as he fetched +you another awful clip--and here you've laid, as dead as a wedge til +now." + +"Oh, I didn't know what I was a-doing. I wish I may die this minute if +I did. It was all on account of the whiskey and the excitement, I +reckon. I never used a weepon in my life before, Joe. I've fought, but +never with weepons. They'll all say that. Joe, don't tell! Say you +won't tell, Joe--that's a good feller. I always liked you, Joe, and +stood up for you, too. Don't you remember? You WON'T tell, WILL you, +Joe?" And the poor creature dropped on his knees before the stolid +murderer, and clasped his appealing hands. + +"No, you've always been fair and square with me, Muff Potter, and I +won't go back on you. There, now, that's as fair as a man can say." + +"Oh, Joe, you're an angel. I'll bless you for this the longest day I +live." And Potter began to cry. + +"Come, now, that's enough of that. This ain't any time for blubbering. +You be off yonder way and I'll go this. Move, now, and don't leave any +tracks behind you." + +Potter started on a trot that quickly increased to a run. The +half-breed stood looking after him. He muttered: + +"If he's as much stunned with the lick and fuddled with the rum as he +had the look of being, he won't think of the knife till he's gone so +far he'll be afraid to come back after it to such a place by himself +--chicken-heart!" + +Two or three minutes later the murdered man, the blanketed corpse, the +lidless coffin, and the open grave were under no inspection but the +moon's. The stillness was complete again, too. + + + +CHAPTER X + +THE two boys flew on and on, toward the village, speechless with +horror. They glanced backward over their shoulders from time to time, +apprehensively, as if they feared they might be followed. Every stump +that started up in their path seemed a man and an enemy, and made them +catch their breath; and as they sped by some outlying cottages that lay +near the village, the barking of the aroused watch-dogs seemed to give +wings to their feet. + +"If we can only get to the old tannery before we break down!" +whispered Tom, in short catches between breaths. "I can't stand it much +longer." + +Huckleberry's hard pantings were his only reply, and the boys fixed +their eyes on the goal of their hopes and bent to their work to win it. +They gained steadily on it, and at last, breast to breast, they burst +through the open door and fell grateful and exhausted in the sheltering +shadows beyond. By and by their pulses slowed down, and Tom whispered: + +"Huckleberry, what do you reckon'll come of this?" + +"If Doctor Robinson dies, I reckon hanging'll come of it." + +"Do you though?" + +"Why, I KNOW it, Tom." + +Tom thought a while, then he said: + +"Who'll tell? We?" + +"What are you talking about? S'pose something happened and Injun Joe +DIDN'T hang? Why, he'd kill us some time or other, just as dead sure as +we're a laying here." + +"That's just what I was thinking to myself, Huck." + +"If anybody tells, let Muff Potter do it, if he's fool enough. He's +generally drunk enough." + +Tom said nothing--went on thinking. Presently he whispered: + +"Huck, Muff Potter don't know it. How can he tell?" + +"What's the reason he don't know it?" + +"Because he'd just got that whack when Injun Joe done it. D'you reckon +he could see anything? D'you reckon he knowed anything?" + +"By hokey, that's so, Tom!" + +"And besides, look-a-here--maybe that whack done for HIM!" + +"No, 'taint likely, Tom. He had liquor in him; I could see that; and +besides, he always has. Well, when pap's full, you might take and belt +him over the head with a church and you couldn't phase him. He says so, +his own self. So it's the same with Muff Potter, of course. But if a +man was dead sober, I reckon maybe that whack might fetch him; I dono." + +After another reflective silence, Tom said: + +"Hucky, you sure you can keep mum?" + +"Tom, we GOT to keep mum. You know that. That Injun devil wouldn't +make any more of drownding us than a couple of cats, if we was to +squeak 'bout this and they didn't hang him. Now, look-a-here, Tom, less +take and swear to one another--that's what we got to do--swear to keep +mum." + +"I'm agreed. It's the best thing. Would you just hold hands and swear +that we--" + +"Oh no, that wouldn't do for this. That's good enough for little +rubbishy common things--specially with gals, cuz THEY go back on you +anyway, and blab if they get in a huff--but there orter be writing +'bout a big thing like this. And blood." + +Tom's whole being applauded this idea. It was deep, and dark, and +awful; the hour, the circumstances, the surroundings, were in keeping +with it. He picked up a clean pine shingle that lay in the moonlight, +took a little fragment of "red keel" out of his pocket, got the moon on +his work, and painfully scrawled these lines, emphasizing each slow +down-stroke by clamping his tongue between his teeth, and letting up +the pressure on the up-strokes. [See next page.] + + "Huck Finn and + Tom Sawyer swears + they will keep mum + about This and They + wish They may Drop + down dead in Their + Tracks if They ever + Tell and Rot." + +Huckleberry was filled with admiration of Tom's facility in writing, +and the sublimity of his language. He at once took a pin from his lapel +and was going to prick his flesh, but Tom said: + +"Hold on! Don't do that. A pin's brass. It might have verdigrease on +it." + +"What's verdigrease?" + +"It's p'ison. That's what it is. You just swaller some of it once +--you'll see." + +So Tom unwound the thread from one of his needles, and each boy +pricked the ball of his thumb and squeezed out a drop of blood. In +time, after many squeezes, Tom managed to sign his initials, using the +ball of his little finger for a pen. Then he showed Huckleberry how to +make an H and an F, and the oath was complete. They buried the shingle +close to the wall, with some dismal ceremonies and incantations, and +the fetters that bound their tongues were considered to be locked and +the key thrown away. + +A figure crept stealthily through a break in the other end of the +ruined building, now, but they did not notice it. + +"Tom," whispered Huckleberry, "does this keep us from EVER telling +--ALWAYS?" + +"Of course it does. It don't make any difference WHAT happens, we got +to keep mum. We'd drop down dead--don't YOU know that?" + +"Yes, I reckon that's so." + +They continued to whisper for some little time. Presently a dog set up +a long, lugubrious howl just outside--within ten feet of them. The boys +clasped each other suddenly, in an agony of fright. + +"Which of us does he mean?" gasped Huckleberry. + +"I dono--peep through the crack. Quick!" + +"No, YOU, Tom!" + +"I can't--I can't DO it, Huck!" + +"Please, Tom. There 'tis again!" + +"Oh, lordy, I'm thankful!" whispered Tom. "I know his voice. It's Bull +Harbison." * + +[* If Mr. Harbison owned a slave named Bull, Tom would have spoken of +him as "Harbison's Bull," but a son or a dog of that name was "Bull +Harbison."] + +"Oh, that's good--I tell you, Tom, I was most scared to death; I'd a +bet anything it was a STRAY dog." + +The dog howled again. The boys' hearts sank once more. + +"Oh, my! that ain't no Bull Harbison!" whispered Huckleberry. "DO, Tom!" + +Tom, quaking with fear, yielded, and put his eye to the crack. His +whisper was hardly audible when he said: + +"Oh, Huck, IT S A STRAY DOG!" + +"Quick, Tom, quick! Who does he mean?" + +"Huck, he must mean us both--we're right together." + +"Oh, Tom, I reckon we're goners. I reckon there ain't no mistake 'bout +where I'LL go to. I been so wicked." + +"Dad fetch it! This comes of playing hookey and doing everything a +feller's told NOT to do. I might a been good, like Sid, if I'd a tried +--but no, I wouldn't, of course. But if ever I get off this time, I lay +I'll just WALLER in Sunday-schools!" And Tom began to snuffle a little. + +"YOU bad!" and Huckleberry began to snuffle too. "Consound it, Tom +Sawyer, you're just old pie, 'longside o' what I am. Oh, LORDY, lordy, +lordy, I wisht I only had half your chance." + +Tom choked off and whispered: + +"Look, Hucky, look! He's got his BACK to us!" + +Hucky looked, with joy in his heart. + +"Well, he has, by jingoes! Did he before?" + +"Yes, he did. But I, like a fool, never thought. Oh, this is bully, +you know. NOW who can he mean?" + +The howling stopped. Tom pricked up his ears. + +"Sh! What's that?" he whispered. + +"Sounds like--like hogs grunting. No--it's somebody snoring, Tom." + +"That IS it! Where 'bouts is it, Huck?" + +"I bleeve it's down at 'tother end. Sounds so, anyway. Pap used to +sleep there, sometimes, 'long with the hogs, but laws bless you, he +just lifts things when HE snores. Besides, I reckon he ain't ever +coming back to this town any more." + +The spirit of adventure rose in the boys' souls once more. + +"Hucky, do you das't to go if I lead?" + +"I don't like to, much. Tom, s'pose it's Injun Joe!" + +Tom quailed. But presently the temptation rose up strong again and the +boys agreed to try, with the understanding that they would take to +their heels if the snoring stopped. So they went tiptoeing stealthily +down, the one behind the other. When they had got to within five steps +of the snorer, Tom stepped on a stick, and it broke with a sharp snap. +The man moaned, writhed a little, and his face came into the moonlight. +It was Muff Potter. The boys' hearts had stood still, and their hopes +too, when the man moved, but their fears passed away now. They tiptoed +out, through the broken weather-boarding, and stopped at a little +distance to exchange a parting word. That long, lugubrious howl rose on +the night air again! They turned and saw the strange dog standing +within a few feet of where Potter was lying, and FACING Potter, with +his nose pointing heavenward. + +"Oh, geeminy, it's HIM!" exclaimed both boys, in a breath. + +"Say, Tom--they say a stray dog come howling around Johnny Miller's +house, 'bout midnight, as much as two weeks ago; and a whippoorwill +come in and lit on the banisters and sung, the very same evening; and +there ain't anybody dead there yet." + +"Well, I know that. And suppose there ain't. Didn't Gracie Miller fall +in the kitchen fire and burn herself terrible the very next Saturday?" + +"Yes, but she ain't DEAD. And what's more, she's getting better, too." + +"All right, you wait and see. She's a goner, just as dead sure as Muff +Potter's a goner. That's what the niggers say, and they know all about +these kind of things, Huck." + +Then they separated, cogitating. When Tom crept in at his bedroom +window the night was almost spent. He undressed with excessive caution, +and fell asleep congratulating himself that nobody knew of his +escapade. He was not aware that the gently-snoring Sid was awake, and +had been so for an hour. + +When Tom awoke, Sid was dressed and gone. There was a late look in the +light, a late sense in the atmosphere. He was startled. Why had he not +been called--persecuted till he was up, as usual? The thought filled +him with bodings. Within five minutes he was dressed and down-stairs, +feeling sore and drowsy. The family were still at table, but they had +finished breakfast. There was no voice of rebuke; but there were +averted eyes; there was a silence and an air of solemnity that struck a +chill to the culprit's heart. He sat down and tried to seem gay, but it +was up-hill work; it roused no smile, no response, and he lapsed into +silence and let his heart sink down to the depths. + +After breakfast his aunt took him aside, and Tom almost brightened in +the hope that he was going to be flogged; but it was not so. His aunt +wept over him and asked him how he could go and break her old heart so; +and finally told him to go on, and ruin himself and bring her gray +hairs with sorrow to the grave, for it was no use for her to try any +more. This was worse than a thousand whippings, and Tom's heart was +sorer now than his body. He cried, he pleaded for forgiveness, promised +to reform over and over again, and then received his dismissal, feeling +that he had won but an imperfect forgiveness and established but a +feeble confidence. + +He left the presence too miserable to even feel revengeful toward Sid; +and so the latter's prompt retreat through the back gate was +unnecessary. He moped to school gloomy and sad, and took his flogging, +along with Joe Harper, for playing hookey the day before, with the air +of one whose heart was busy with heavier woes and wholly dead to +trifles. Then he betook himself to his seat, rested his elbows on his +desk and his jaws in his hands, and stared at the wall with the stony +stare of suffering that has reached the limit and can no further go. +His elbow was pressing against some hard substance. After a long time +he slowly and sadly changed his position, and took up this object with +a sigh. It was in a paper. He unrolled it. A long, lingering, colossal +sigh followed, and his heart broke. It was his brass andiron knob! + +This final feather broke the camel's back. + + + +CHAPTER XI + +CLOSE upon the hour of noon the whole village was suddenly electrified +with the ghastly news. No need of the as yet undreamed-of telegraph; +the tale flew from man to man, from group to group, from house to +house, with little less than telegraphic speed. Of course the +schoolmaster gave holiday for that afternoon; the town would have +thought strangely of him if he had not. + +A gory knife had been found close to the murdered man, and it had been +recognized by somebody as belonging to Muff Potter--so the story ran. +And it was said that a belated citizen had come upon Potter washing +himself in the "branch" about one or two o'clock in the morning, and +that Potter had at once sneaked off--suspicious circumstances, +especially the washing which was not a habit with Potter. It was also +said that the town had been ransacked for this "murderer" (the public +are not slow in the matter of sifting evidence and arriving at a +verdict), but that he could not be found. Horsemen had departed down +all the roads in every direction, and the Sheriff "was confident" that +he would be captured before night. + +All the town was drifting toward the graveyard. Tom's heartbreak +vanished and he joined the procession, not because he would not a +thousand times rather go anywhere else, but because an awful, +unaccountable fascination drew him on. Arrived at the dreadful place, +he wormed his small body through the crowd and saw the dismal +spectacle. It seemed to him an age since he was there before. Somebody +pinched his arm. He turned, and his eyes met Huckleberry's. Then both +looked elsewhere at once, and wondered if anybody had noticed anything +in their mutual glance. But everybody was talking, and intent upon the +grisly spectacle before them. + +"Poor fellow!" "Poor young fellow!" "This ought to be a lesson to +grave robbers!" "Muff Potter'll hang for this if they catch him!" This +was the drift of remark; and the minister said, "It was a judgment; His +hand is here." + +Now Tom shivered from head to heel; for his eye fell upon the stolid +face of Injun Joe. At this moment the crowd began to sway and struggle, +and voices shouted, "It's him! it's him! he's coming himself!" + +"Who? Who?" from twenty voices. + +"Muff Potter!" + +"Hallo, he's stopped!--Look out, he's turning! Don't let him get away!" + +People in the branches of the trees over Tom's head said he wasn't +trying to get away--he only looked doubtful and perplexed. + +"Infernal impudence!" said a bystander; "wanted to come and take a +quiet look at his work, I reckon--didn't expect any company." + +The crowd fell apart, now, and the Sheriff came through, +ostentatiously leading Potter by the arm. The poor fellow's face was +haggard, and his eyes showed the fear that was upon him. When he stood +before the murdered man, he shook as with a palsy, and he put his face +in his hands and burst into tears. + +"I didn't do it, friends," he sobbed; "'pon my word and honor I never +done it." + +"Who's accused you?" shouted a voice. + +This shot seemed to carry home. Potter lifted his face and looked +around him with a pathetic hopelessness in his eyes. He saw Injun Joe, +and exclaimed: + +"Oh, Injun Joe, you promised me you'd never--" + +"Is that your knife?" and it was thrust before him by the Sheriff. + +Potter would have fallen if they had not caught him and eased him to +the ground. Then he said: + +"Something told me 't if I didn't come back and get--" He shuddered; +then waved his nerveless hand with a vanquished gesture and said, "Tell +'em, Joe, tell 'em--it ain't any use any more." + +Then Huckleberry and Tom stood dumb and staring, and heard the +stony-hearted liar reel off his serene statement, they expecting every +moment that the clear sky would deliver God's lightnings upon his head, +and wondering to see how long the stroke was delayed. And when he had +finished and still stood alive and whole, their wavering impulse to +break their oath and save the poor betrayed prisoner's life faded and +vanished away, for plainly this miscreant had sold himself to Satan and +it would be fatal to meddle with the property of such a power as that. + +"Why didn't you leave? What did you want to come here for?" somebody +said. + +"I couldn't help it--I couldn't help it," Potter moaned. "I wanted to +run away, but I couldn't seem to come anywhere but here." And he fell +to sobbing again. + +Injun Joe repeated his statement, just as calmly, a few minutes +afterward on the inquest, under oath; and the boys, seeing that the +lightnings were still withheld, were confirmed in their belief that Joe +had sold himself to the devil. He was now become, to them, the most +balefully interesting object they had ever looked upon, and they could +not take their fascinated eyes from his face. + +They inwardly resolved to watch him nights, when opportunity should +offer, in the hope of getting a glimpse of his dread master. + +Injun Joe helped to raise the body of the murdered man and put it in a +wagon for removal; and it was whispered through the shuddering crowd +that the wound bled a little! The boys thought that this happy +circumstance would turn suspicion in the right direction; but they were +disappointed, for more than one villager remarked: + +"It was within three feet of Muff Potter when it done it." + +Tom's fearful secret and gnawing conscience disturbed his sleep for as +much as a week after this; and at breakfast one morning Sid said: + +"Tom, you pitch around and talk in your sleep so much that you keep me +awake half the time." + +Tom blanched and dropped his eyes. + +"It's a bad sign," said Aunt Polly, gravely. "What you got on your +mind, Tom?" + +"Nothing. Nothing 't I know of." But the boy's hand shook so that he +spilled his coffee. + +"And you do talk such stuff," Sid said. "Last night you said, 'It's +blood, it's blood, that's what it is!' You said that over and over. And +you said, 'Don't torment me so--I'll tell!' Tell WHAT? What is it +you'll tell?" + +Everything was swimming before Tom. There is no telling what might +have happened, now, but luckily the concern passed out of Aunt Polly's +face and she came to Tom's relief without knowing it. She said: + +"Sho! It's that dreadful murder. I dream about it most every night +myself. Sometimes I dream it's me that done it." + +Mary said she had been affected much the same way. Sid seemed +satisfied. Tom got out of the presence as quick as he plausibly could, +and after that he complained of toothache for a week, and tied up his +jaws every night. He never knew that Sid lay nightly watching, and +frequently slipped the bandage free and then leaned on his elbow +listening a good while at a time, and afterward slipped the bandage +back to its place again. Tom's distress of mind wore off gradually and +the toothache grew irksome and was discarded. If Sid really managed to +make anything out of Tom's disjointed mutterings, he kept it to himself. + +It seemed to Tom that his schoolmates never would get done holding +inquests on dead cats, and thus keeping his trouble present to his +mind. Sid noticed that Tom never was coroner at one of these inquiries, +though it had been his habit to take the lead in all new enterprises; +he noticed, too, that Tom never acted as a witness--and that was +strange; and Sid did not overlook the fact that Tom even showed a +marked aversion to these inquests, and always avoided them when he +could. Sid marvelled, but said nothing. However, even inquests went out +of vogue at last, and ceased to torture Tom's conscience. + +Every day or two, during this time of sorrow, Tom watched his +opportunity and went to the little grated jail-window and smuggled such +small comforts through to the "murderer" as he could get hold of. The +jail was a trifling little brick den that stood in a marsh at the edge +of the village, and no guards were afforded for it; indeed, it was +seldom occupied. These offerings greatly helped to ease Tom's +conscience. + +The villagers had a strong desire to tar-and-feather Injun Joe and +ride him on a rail, for body-snatching, but so formidable was his +character that nobody could be found who was willing to take the lead +in the matter, so it was dropped. He had been careful to begin both of +his inquest-statements with the fight, without confessing the +grave-robbery that preceded it; therefore it was deemed wisest not +to try the case in the courts at present. + + + +CHAPTER XII + +ONE of the reasons why Tom's mind had drifted away from its secret +troubles was, that it had found a new and weighty matter to interest +itself about. Becky Thatcher had stopped coming to school. Tom had +struggled with his pride a few days, and tried to "whistle her down the +wind," but failed. He began to find himself hanging around her father's +house, nights, and feeling very miserable. She was ill. What if she +should die! There was distraction in the thought. He no longer took an +interest in war, nor even in piracy. The charm of life was gone; there +was nothing but dreariness left. He put his hoop away, and his bat; +there was no joy in them any more. His aunt was concerned. She began to +try all manner of remedies on him. She was one of those people who are +infatuated with patent medicines and all new-fangled methods of +producing health or mending it. She was an inveterate experimenter in +these things. When something fresh in this line came out she was in a +fever, right away, to try it; not on herself, for she was never ailing, +but on anybody else that came handy. She was a subscriber for all the +"Health" periodicals and phrenological frauds; and the solemn ignorance +they were inflated with was breath to her nostrils. All the "rot" they +contained about ventilation, and how to go to bed, and how to get up, +and what to eat, and what to drink, and how much exercise to take, and +what frame of mind to keep one's self in, and what sort of clothing to +wear, was all gospel to her, and she never observed that her +health-journals of the current month customarily upset everything they +had recommended the month before. She was as simple-hearted and honest +as the day was long, and so she was an easy victim. She gathered +together her quack periodicals and her quack medicines, and thus armed +with death, went about on her pale horse, metaphorically speaking, with +"hell following after." But she never suspected that she was not an +angel of healing and the balm of Gilead in disguise, to the suffering +neighbors. + +The water treatment was new, now, and Tom's low condition was a +windfall to her. She had him out at daylight every morning, stood him +up in the woodshed and drowned him with a deluge of cold water; then +she scrubbed him down with a towel like a file, and so brought him to; +then she rolled him up in a wet sheet and put him away under blankets +till she sweated his soul clean and "the yellow stains of it came +through his pores"--as Tom said. + +Yet notwithstanding all this, the boy grew more and more melancholy +and pale and dejected. She added hot baths, sitz baths, shower baths, +and plunges. The boy remained as dismal as a hearse. She began to +assist the water with a slim oatmeal diet and blister-plasters. She +calculated his capacity as she would a jug's, and filled him up every +day with quack cure-alls. + +Tom had become indifferent to persecution by this time. This phase +filled the old lady's heart with consternation. This indifference must +be broken up at any cost. Now she heard of Pain-killer for the first +time. She ordered a lot at once. She tasted it and was filled with +gratitude. It was simply fire in a liquid form. She dropped the water +treatment and everything else, and pinned her faith to Pain-killer. She +gave Tom a teaspoonful and watched with the deepest anxiety for the +result. Her troubles were instantly at rest, her soul at peace again; +for the "indifference" was broken up. The boy could not have shown a +wilder, heartier interest, if she had built a fire under him. + +Tom felt that it was time to wake up; this sort of life might be +romantic enough, in his blighted condition, but it was getting to have +too little sentiment and too much distracting variety about it. So he +thought over various plans for relief, and finally hit pon that of +professing to be fond of Pain-killer. He asked for it so often that he +became a nuisance, and his aunt ended by telling him to help himself +and quit bothering her. If it had been Sid, she would have had no +misgivings to alloy her delight; but since it was Tom, she watched the +bottle clandestinely. She found that the medicine did really diminish, +but it did not occur to her that the boy was mending the health of a +crack in the sitting-room floor with it. + +One day Tom was in the act of dosing the crack when his aunt's yellow +cat came along, purring, eying the teaspoon avariciously, and begging +for a taste. Tom said: + +"Don't ask for it unless you want it, Peter." + +But Peter signified that he did want it. + +"You better make sure." + +Peter was sure. + +"Now you've asked for it, and I'll give it to you, because there ain't +anything mean about me; but if you find you don't like it, you mustn't +blame anybody but your own self." + +Peter was agreeable. So Tom pried his mouth open and poured down the +Pain-killer. Peter sprang a couple of yards in the air, and then +delivered a war-whoop and set off round and round the room, banging +against furniture, upsetting flower-pots, and making general havoc. +Next he rose on his hind feet and pranced around, in a frenzy of +enjoyment, with his head over his shoulder and his voice proclaiming +his unappeasable happiness. Then he went tearing around the house again +spreading chaos and destruction in his path. Aunt Polly entered in time +to see him throw a few double summersets, deliver a final mighty +hurrah, and sail through the open window, carrying the rest of the +flower-pots with him. The old lady stood petrified with astonishment, +peering over her glasses; Tom lay on the floor expiring with laughter. + +"Tom, what on earth ails that cat?" + +"I don't know, aunt," gasped the boy. + +"Why, I never see anything like it. What did make him act so?" + +"Deed I don't know, Aunt Polly; cats always act so when they're having +a good time." + +"They do, do they?" There was something in the tone that made Tom +apprehensive. + +"Yes'm. That is, I believe they do." + +"You DO?" + +"Yes'm." + +The old lady was bending down, Tom watching, with interest emphasized +by anxiety. Too late he divined her "drift." The handle of the telltale +teaspoon was visible under the bed-valance. Aunt Polly took it, held it +up. Tom winced, and dropped his eyes. Aunt Polly raised him by the +usual handle--his ear--and cracked his head soundly with her thimble. + +"Now, sir, what did you want to treat that poor dumb beast so, for?" + +"I done it out of pity for him--because he hadn't any aunt." + +"Hadn't any aunt!--you numskull. What has that got to do with it?" + +"Heaps. Because if he'd had one she'd a burnt him out herself! She'd a +roasted his bowels out of him 'thout any more feeling than if he was a +human!" + +Aunt Polly felt a sudden pang of remorse. This was putting the thing +in a new light; what was cruelty to a cat MIGHT be cruelty to a boy, +too. She began to soften; she felt sorry. Her eyes watered a little, +and she put her hand on Tom's head and said gently: + +"I was meaning for the best, Tom. And, Tom, it DID do you good." + +Tom looked up in her face with just a perceptible twinkle peeping +through his gravity. + +"I know you was meaning for the best, aunty, and so was I with Peter. +It done HIM good, too. I never see him get around so since--" + +"Oh, go 'long with you, Tom, before you aggravate me again. And you +try and see if you can't be a good boy, for once, and you needn't take +any more medicine." + +Tom reached school ahead of time. It was noticed that this strange +thing had been occurring every day latterly. And now, as usual of late, +he hung about the gate of the schoolyard instead of playing with his +comrades. He was sick, he said, and he looked it. He tried to seem to +be looking everywhere but whither he really was looking--down the road. +Presently Jeff Thatcher hove in sight, and Tom's face lighted; he gazed +a moment, and then turned sorrowfully away. When Jeff arrived, Tom +accosted him; and "led up" warily to opportunities for remark about +Becky, but the giddy lad never could see the bait. Tom watched and +watched, hoping whenever a frisking frock came in sight, and hating the +owner of it as soon as he saw she was not the right one. At last frocks +ceased to appear, and he dropped hopelessly into the dumps; he entered +the empty schoolhouse and sat down to suffer. Then one more frock +passed in at the gate, and Tom's heart gave a great bound. The next +instant he was out, and "going on" like an Indian; yelling, laughing, +chasing boys, jumping over the fence at risk of life and limb, throwing +handsprings, standing on his head--doing all the heroic things he could +conceive of, and keeping a furtive eye out, all the while, to see if +Becky Thatcher was noticing. But she seemed to be unconscious of it +all; she never looked. Could it be possible that she was not aware that +he was there? He carried his exploits to her immediate vicinity; came +war-whooping around, snatched a boy's cap, hurled it to the roof of the +schoolhouse, broke through a group of boys, tumbling them in every +direction, and fell sprawling, himself, under Becky's nose, almost +upsetting her--and she turned, with her nose in the air, and he heard +her say: "Mf! some people think they're mighty smart--always showing +off!" + +Tom's cheeks burned. He gathered himself up and sneaked off, crushed +and crestfallen. + + + +CHAPTER XIII + +TOM'S mind was made up now. He was gloomy and desperate. He was a +forsaken, friendless boy, he said; nobody loved him; when they found +out what they had driven him to, perhaps they would be sorry; he had +tried to do right and get along, but they would not let him; since +nothing would do them but to be rid of him, let it be so; and let them +blame HIM for the consequences--why shouldn't they? What right had the +friendless to complain? Yes, they had forced him to it at last: he +would lead a life of crime. There was no choice. + +By this time he was far down Meadow Lane, and the bell for school to +"take up" tinkled faintly upon his ear. He sobbed, now, to think he +should never, never hear that old familiar sound any more--it was very +hard, but it was forced on him; since he was driven out into the cold +world, he must submit--but he forgave them. Then the sobs came thick +and fast. + +Just at this point he met his soul's sworn comrade, Joe Harper +--hard-eyed, and with evidently a great and dismal purpose in his heart. +Plainly here were "two souls with but a single thought." Tom, wiping +his eyes with his sleeve, began to blubber out something about a +resolution to escape from hard usage and lack of sympathy at home by +roaming abroad into the great world never to return; and ended by +hoping that Joe would not forget him. + +But it transpired that this was a request which Joe had just been +going to make of Tom, and had come to hunt him up for that purpose. His +mother had whipped him for drinking some cream which he had never +tasted and knew nothing about; it was plain that she was tired of him +and wished him to go; if she felt that way, there was nothing for him +to do but succumb; he hoped she would be happy, and never regret having +driven her poor boy out into the unfeeling world to suffer and die. + +As the two boys walked sorrowing along, they made a new compact to +stand by each other and be brothers and never separate till death +relieved them of their troubles. Then they began to lay their plans. +Joe was for being a hermit, and living on crusts in a remote cave, and +dying, some time, of cold and want and grief; but after listening to +Tom, he conceded that there were some conspicuous advantages about a +life of crime, and so he consented to be a pirate. + +Three miles below St. Petersburg, at a point where the Mississippi +River was a trifle over a mile wide, there was a long, narrow, wooded +island, with a shallow bar at the head of it, and this offered well as +a rendezvous. It was not inhabited; it lay far over toward the further +shore, abreast a dense and almost wholly unpeopled forest. So Jackson's +Island was chosen. Who were to be the subjects of their piracies was a +matter that did not occur to them. Then they hunted up Huckleberry +Finn, and he joined them promptly, for all careers were one to him; he +was indifferent. They presently separated to meet at a lonely spot on +the river-bank two miles above the village at the favorite hour--which +was midnight. There was a small log raft there which they meant to +capture. Each would bring hooks and lines, and such provision as he +could steal in the most dark and mysterious way--as became outlaws. And +before the afternoon was done, they had all managed to enjoy the sweet +glory of spreading the fact that pretty soon the town would "hear +something." All who got this vague hint were cautioned to "be mum and +wait." + +About midnight Tom arrived with a boiled ham and a few trifles, +and stopped in a dense undergrowth on a small bluff overlooking the +meeting-place. It was starlight, and very still. The mighty river lay +like an ocean at rest. Tom listened a moment, but no sound disturbed the +quiet. Then he gave a low, distinct whistle. It was answered from under +the bluff. Tom whistled twice more; these signals were answered in the +same way. Then a guarded voice said: + +"Who goes there?" + +"Tom Sawyer, the Black Avenger of the Spanish Main. Name your names." + +"Huck Finn the Red-Handed, and Joe Harper the Terror of the Seas." Tom +had furnished these titles, from his favorite literature. + +"'Tis well. Give the countersign." + +Two hoarse whispers delivered the same awful word simultaneously to +the brooding night: + +"BLOOD!" + +Then Tom tumbled his ham over the bluff and let himself down after it, +tearing both skin and clothes to some extent in the effort. There was +an easy, comfortable path along the shore under the bluff, but it +lacked the advantages of difficulty and danger so valued by a pirate. + +The Terror of the Seas had brought a side of bacon, and had about worn +himself out with getting it there. Finn the Red-Handed had stolen a +skillet and a quantity of half-cured leaf tobacco, and had also brought +a few corn-cobs to make pipes with. But none of the pirates smoked or +"chewed" but himself. The Black Avenger of the Spanish Main said it +would never do to start without some fire. That was a wise thought; +matches were hardly known there in that day. They saw a fire +smouldering upon a great raft a hundred yards above, and they went +stealthily thither and helped themselves to a chunk. They made an +imposing adventure of it, saying, "Hist!" every now and then, and +suddenly halting with finger on lip; moving with hands on imaginary +dagger-hilts; and giving orders in dismal whispers that if "the foe" +stirred, to "let him have it to the hilt," because "dead men tell no +tales." They knew well enough that the raftsmen were all down at the +village laying in stores or having a spree, but still that was no +excuse for their conducting this thing in an unpiratical way. + +They shoved off, presently, Tom in command, Huck at the after oar and +Joe at the forward. Tom stood amidships, gloomy-browed, and with folded +arms, and gave his orders in a low, stern whisper: + +"Luff, and bring her to the wind!" + +"Aye-aye, sir!" + +"Steady, steady-y-y-y!" + +"Steady it is, sir!" + +"Let her go off a point!" + +"Point it is, sir!" + +As the boys steadily and monotonously drove the raft toward mid-stream +it was no doubt understood that these orders were given only for +"style," and were not intended to mean anything in particular. + +"What sail's she carrying?" + +"Courses, tops'ls, and flying-jib, sir." + +"Send the r'yals up! Lay out aloft, there, half a dozen of ye +--foretopmaststuns'l! Lively, now!" + +"Aye-aye, sir!" + +"Shake out that maintogalans'l! Sheets and braces! NOW my hearties!" + +"Aye-aye, sir!" + +"Hellum-a-lee--hard a port! Stand by to meet her when she comes! Port, +port! NOW, men! With a will! Stead-y-y-y!" + +"Steady it is, sir!" + +The raft drew beyond the middle of the river; the boys pointed her +head right, and then lay on their oars. The river was not high, so +there was not more than a two or three mile current. Hardly a word was +said during the next three-quarters of an hour. Now the raft was +passing before the distant town. Two or three glimmering lights showed +where it lay, peacefully sleeping, beyond the vague vast sweep of +star-gemmed water, unconscious of the tremendous event that was happening. +The Black Avenger stood still with folded arms, "looking his last" upon +the scene of his former joys and his later sufferings, and wishing +"she" could see him now, abroad on the wild sea, facing peril and death +with dauntless heart, going to his doom with a grim smile on his lips. +It was but a small strain on his imagination to remove Jackson's Island +beyond eyeshot of the village, and so he "looked his last" with a +broken and satisfied heart. The other pirates were looking their last, +too; and they all looked so long that they came near letting the +current drift them out of the range of the island. But they discovered +the danger in time, and made shift to avert it. About two o'clock in +the morning the raft grounded on the bar two hundred yards above the +head of the island, and they waded back and forth until they had landed +their freight. Part of the little raft's belongings consisted of an old +sail, and this they spread over a nook in the bushes for a tent to +shelter their provisions; but they themselves would sleep in the open +air in good weather, as became outlaws. + +They built a fire against the side of a great log twenty or thirty +steps within the sombre depths of the forest, and then cooked some +bacon in the frying-pan for supper, and used up half of the corn "pone" +stock they had brought. It seemed glorious sport to be feasting in that +wild, free way in the virgin forest of an unexplored and uninhabited +island, far from the haunts of men, and they said they never would +return to civilization. The climbing fire lit up their faces and threw +its ruddy glare upon the pillared tree-trunks of their forest temple, +and upon the varnished foliage and festooning vines. + +When the last crisp slice of bacon was gone, and the last allowance of +corn pone devoured, the boys stretched themselves out on the grass, +filled with contentment. They could have found a cooler place, but they +would not deny themselves such a romantic feature as the roasting +camp-fire. + +"AIN'T it gay?" said Joe. + +"It's NUTS!" said Tom. "What would the boys say if they could see us?" + +"Say? Well, they'd just die to be here--hey, Hucky!" + +"I reckon so," said Huckleberry; "anyways, I'm suited. I don't want +nothing better'n this. I don't ever get enough to eat, gen'ally--and +here they can't come and pick at a feller and bullyrag him so." + +"It's just the life for me," said Tom. "You don't have to get up, +mornings, and you don't have to go to school, and wash, and all that +blame foolishness. You see a pirate don't have to do ANYTHING, Joe, +when he's ashore, but a hermit HE has to be praying considerable, and +then he don't have any fun, anyway, all by himself that way." + +"Oh yes, that's so," said Joe, "but I hadn't thought much about it, +you know. I'd a good deal rather be a pirate, now that I've tried it." + +"You see," said Tom, "people don't go much on hermits, nowadays, like +they used to in old times, but a pirate's always respected. And a +hermit's got to sleep on the hardest place he can find, and put +sackcloth and ashes on his head, and stand out in the rain, and--" + +"What does he put sackcloth and ashes on his head for?" inquired Huck. + +"I dono. But they've GOT to do it. Hermits always do. You'd have to do +that if you was a hermit." + +"Dern'd if I would," said Huck. + +"Well, what would you do?" + +"I dono. But I wouldn't do that." + +"Why, Huck, you'd HAVE to. How'd you get around it?" + +"Why, I just wouldn't stand it. I'd run away." + +"Run away! Well, you WOULD be a nice old slouch of a hermit. You'd be +a disgrace." + +The Red-Handed made no response, being better employed. He had +finished gouging out a cob, and now he fitted a weed stem to it, loaded +it with tobacco, and was pressing a coal to the charge and blowing a +cloud of fragrant smoke--he was in the full bloom of luxurious +contentment. The other pirates envied him this majestic vice, and +secretly resolved to acquire it shortly. Presently Huck said: + +"What does pirates have to do?" + +Tom said: + +"Oh, they have just a bully time--take ships and burn them, and get +the money and bury it in awful places in their island where there's +ghosts and things to watch it, and kill everybody in the ships--make +'em walk a plank." + +"And they carry the women to the island," said Joe; "they don't kill +the women." + +"No," assented Tom, "they don't kill the women--they're too noble. And +the women's always beautiful, too. + +"And don't they wear the bulliest clothes! Oh no! All gold and silver +and di'monds," said Joe, with enthusiasm. + +"Who?" said Huck. + +"Why, the pirates." + +Huck scanned his own clothing forlornly. + +"I reckon I ain't dressed fitten for a pirate," said he, with a +regretful pathos in his voice; "but I ain't got none but these." + +But the other boys told him the fine clothes would come fast enough, +after they should have begun their adventures. They made him understand +that his poor rags would do to begin with, though it was customary for +wealthy pirates to start with a proper wardrobe. + +Gradually their talk died out and drowsiness began to steal upon the +eyelids of the little waifs. The pipe dropped from the fingers of the +Red-Handed, and he slept the sleep of the conscience-free and the +weary. The Terror of the Seas and the Black Avenger of the Spanish Main +had more difficulty in getting to sleep. They said their prayers +inwardly, and lying down, since there was nobody there with authority +to make them kneel and recite aloud; in truth, they had a mind not to +say them at all, but they were afraid to proceed to such lengths as +that, lest they might call down a sudden and special thunderbolt from +heaven. Then at once they reached and hovered upon the imminent verge +of sleep--but an intruder came, now, that would not "down." It was +conscience. They began to feel a vague fear that they had been doing +wrong to run away; and next they thought of the stolen meat, and then +the real torture came. They tried to argue it away by reminding +conscience that they had purloined sweetmeats and apples scores of +times; but conscience was not to be appeased by such thin +plausibilities; it seemed to them, in the end, that there was no +getting around the stubborn fact that taking sweetmeats was only +"hooking," while taking bacon and hams and such valuables was plain +simple stealing--and there was a command against that in the Bible. So +they inwardly resolved that so long as they remained in the business, +their piracies should not again be sullied with the crime of stealing. +Then conscience granted a truce, and these curiously inconsistent +pirates fell peacefully to sleep. + + + +CHAPTER XIV + +WHEN Tom awoke in the morning, he wondered where he was. He sat up and +rubbed his eyes and looked around. Then he comprehended. It was the +cool gray dawn, and there was a delicious sense of repose and peace in +the deep pervading calm and silence of the woods. Not a leaf stirred; +not a sound obtruded upon great Nature's meditation. Beaded dewdrops +stood upon the leaves and grasses. A white layer of ashes covered the +fire, and a thin blue breath of smoke rose straight into the air. Joe +and Huck still slept. + +Now, far away in the woods a bird called; another answered; presently +the hammering of a woodpecker was heard. Gradually the cool dim gray of +the morning whitened, and as gradually sounds multiplied and life +manifested itself. The marvel of Nature shaking off sleep and going to +work unfolded itself to the musing boy. A little green worm came +crawling over a dewy leaf, lifting two-thirds of his body into the air +from time to time and "sniffing around," then proceeding again--for he +was measuring, Tom said; and when the worm approached him, of its own +accord, he sat as still as a stone, with his hopes rising and falling, +by turns, as the creature still came toward him or seemed inclined to +go elsewhere; and when at last it considered a painful moment with its +curved body in the air and then came decisively down upon Tom's leg and +began a journey over him, his whole heart was glad--for that meant that +he was going to have a new suit of clothes--without the shadow of a +doubt a gaudy piratical uniform. Now a procession of ants appeared, +from nowhere in particular, and went about their labors; one struggled +manfully by with a dead spider five times as big as itself in its arms, +and lugged it straight up a tree-trunk. A brown spotted lady-bug +climbed the dizzy height of a grass blade, and Tom bent down close to +it and said, "Lady-bug, lady-bug, fly away home, your house is on fire, +your children's alone," and she took wing and went off to see about it +--which did not surprise the boy, for he knew of old that this insect was +credulous about conflagrations, and he had practised upon its +simplicity more than once. A tumblebug came next, heaving sturdily at +its ball, and Tom touched the creature, to see it shut its legs against +its body and pretend to be dead. The birds were fairly rioting by this +time. A catbird, the Northern mocker, lit in a tree over Tom's head, +and trilled out her imitations of her neighbors in a rapture of +enjoyment; then a shrill jay swept down, a flash of blue flame, and +stopped on a twig almost within the boy's reach, cocked his head to one +side and eyed the strangers with a consuming curiosity; a gray squirrel +and a big fellow of the "fox" kind came skurrying along, sitting up at +intervals to inspect and chatter at the boys, for the wild things had +probably never seen a human being before and scarcely knew whether to +be afraid or not. All Nature was wide awake and stirring, now; long +lances of sunlight pierced down through the dense foliage far and near, +and a few butterflies came fluttering upon the scene. + +Tom stirred up the other pirates and they all clattered away with a +shout, and in a minute or two were stripped and chasing after and +tumbling over each other in the shallow limpid water of the white +sandbar. They felt no longing for the little village sleeping in the +distance beyond the majestic waste of water. A vagrant current or a +slight rise in the river had carried off their raft, but this only +gratified them, since its going was something like burning the bridge +between them and civilization. + +They came back to camp wonderfully refreshed, glad-hearted, and +ravenous; and they soon had the camp-fire blazing up again. Huck found +a spring of clear cold water close by, and the boys made cups of broad +oak or hickory leaves, and felt that water, sweetened with such a +wildwood charm as that, would be a good enough substitute for coffee. +While Joe was slicing bacon for breakfast, Tom and Huck asked him to +hold on a minute; they stepped to a promising nook in the river-bank +and threw in their lines; almost immediately they had reward. Joe had +not had time to get impatient before they were back again with some +handsome bass, a couple of sun-perch and a small catfish--provisions +enough for quite a family. They fried the fish with the bacon, and were +astonished; for no fish had ever seemed so delicious before. They did +not know that the quicker a fresh-water fish is on the fire after he is +caught the better he is; and they reflected little upon what a sauce +open-air sleeping, open-air exercise, bathing, and a large ingredient +of hunger make, too. + +They lay around in the shade, after breakfast, while Huck had a smoke, +and then went off through the woods on an exploring expedition. They +tramped gayly along, over decaying logs, through tangled underbrush, +among solemn monarchs of the forest, hung from their crowns to the +ground with a drooping regalia of grape-vines. Now and then they came +upon snug nooks carpeted with grass and jeweled with flowers. + +They found plenty of things to be delighted with, but nothing to be +astonished at. They discovered that the island was about three miles +long and a quarter of a mile wide, and that the shore it lay closest to +was only separated from it by a narrow channel hardly two hundred yards +wide. They took a swim about every hour, so it was close upon the +middle of the afternoon when they got back to camp. They were too +hungry to stop to fish, but they fared sumptuously upon cold ham, and +then threw themselves down in the shade to talk. But the talk soon +began to drag, and then died. The stillness, the solemnity that brooded +in the woods, and the sense of loneliness, began to tell upon the +spirits of the boys. They fell to thinking. A sort of undefined longing +crept upon them. This took dim shape, presently--it was budding +homesickness. Even Finn the Red-Handed was dreaming of his doorsteps +and empty hogsheads. But they were all ashamed of their weakness, and +none was brave enough to speak his thought. + +For some time, now, the boys had been dully conscious of a peculiar +sound in the distance, just as one sometimes is of the ticking of a +clock which he takes no distinct note of. But now this mysterious sound +became more pronounced, and forced a recognition. The boys started, +glanced at each other, and then each assumed a listening attitude. +There was a long silence, profound and unbroken; then a deep, sullen +boom came floating down out of the distance. + +"What is it!" exclaimed Joe, under his breath. + +"I wonder," said Tom in a whisper. + +"'Tain't thunder," said Huckleberry, in an awed tone, "becuz thunder--" + +"Hark!" said Tom. "Listen--don't talk." + +They waited a time that seemed an age, and then the same muffled boom +troubled the solemn hush. + +"Let's go and see." + +They sprang to their feet and hurried to the shore toward the town. +They parted the bushes on the bank and peered out over the water. The +little steam ferryboat was about a mile below the village, drifting +with the current. Her broad deck seemed crowded with people. There were +a great many skiffs rowing about or floating with the stream in the +neighborhood of the ferryboat, but the boys could not determine what +the men in them were doing. Presently a great jet of white smoke burst +from the ferryboat's side, and as it expanded and rose in a lazy cloud, +that same dull throb of sound was borne to the listeners again. + +"I know now!" exclaimed Tom; "somebody's drownded!" + +"That's it!" said Huck; "they done that last summer, when Bill Turner +got drownded; they shoot a cannon over the water, and that makes him +come up to the top. Yes, and they take loaves of bread and put +quicksilver in 'em and set 'em afloat, and wherever there's anybody +that's drownded, they'll float right there and stop." + +"Yes, I've heard about that," said Joe. "I wonder what makes the bread +do that." + +"Oh, it ain't the bread, so much," said Tom; "I reckon it's mostly +what they SAY over it before they start it out." + +"But they don't say anything over it," said Huck. "I've seen 'em and +they don't." + +"Well, that's funny," said Tom. "But maybe they say it to themselves. +Of COURSE they do. Anybody might know that." + +The other boys agreed that there was reason in what Tom said, because +an ignorant lump of bread, uninstructed by an incantation, could not be +expected to act very intelligently when set upon an errand of such +gravity. + +"By jings, I wish I was over there, now," said Joe. + +"I do too" said Huck "I'd give heaps to know who it is." + +The boys still listened and watched. Presently a revealing thought +flashed through Tom's mind, and he exclaimed: + +"Boys, I know who's drownded--it's us!" + +They felt like heroes in an instant. Here was a gorgeous triumph; they +were missed; they were mourned; hearts were breaking on their account; +tears were being shed; accusing memories of unkindness to these poor +lost lads were rising up, and unavailing regrets and remorse were being +indulged; and best of all, the departed were the talk of the whole +town, and the envy of all the boys, as far as this dazzling notoriety +was concerned. This was fine. It was worth while to be a pirate, after +all. + +As twilight drew on, the ferryboat went back to her accustomed +business and the skiffs disappeared. The pirates returned to camp. They +were jubilant with vanity over their new grandeur and the illustrious +trouble they were making. They caught fish, cooked supper and ate it, +and then fell to guessing at what the village was thinking and saying +about them; and the pictures they drew of the public distress on their +account were gratifying to look upon--from their point of view. But +when the shadows of night closed them in, they gradually ceased to +talk, and sat gazing into the fire, with their minds evidently +wandering elsewhere. The excitement was gone, now, and Tom and Joe +could not keep back thoughts of certain persons at home who were not +enjoying this fine frolic as much as they were. Misgivings came; they +grew troubled and unhappy; a sigh or two escaped, unawares. By and by +Joe timidly ventured upon a roundabout "feeler" as to how the others +might look upon a return to civilization--not right now, but-- + +Tom withered him with derision! Huck, being uncommitted as yet, joined +in with Tom, and the waverer quickly "explained," and was glad to get +out of the scrape with as little taint of chicken-hearted homesickness +clinging to his garments as he could. Mutiny was effectually laid to +rest for the moment. + +As the night deepened, Huck began to nod, and presently to snore. Joe +followed next. Tom lay upon his elbow motionless, for some time, +watching the two intently. At last he got up cautiously, on his knees, +and went searching among the grass and the flickering reflections flung +by the camp-fire. He picked up and inspected several large +semi-cylinders of the thin white bark of a sycamore, and finally chose +two which seemed to suit him. Then he knelt by the fire and painfully +wrote something upon each of these with his "red keel"; one he rolled up +and put in his jacket pocket, and the other he put in Joe's hat and +removed it to a little distance from the owner. And he also put into the +hat certain schoolboy treasures of almost inestimable value--among them +a lump of chalk, an India-rubber ball, three fishhooks, and one of that +kind of marbles known as a "sure 'nough crystal." Then he tiptoed his +way cautiously among the trees till he felt that he was out of hearing, +and straightway broke into a keen run in the direction of the sandbar. + + + +CHAPTER XV + +A FEW minutes later Tom was in the shoal water of the bar, wading +toward the Illinois shore. Before the depth reached his middle he was +half-way over; the current would permit no more wading, now, so he +struck out confidently to swim the remaining hundred yards. He swam +quartering upstream, but still was swept downward rather faster than he +had expected. However, he reached the shore finally, and drifted along +till he found a low place and drew himself out. He put his hand on his +jacket pocket, found his piece of bark safe, and then struck through +the woods, following the shore, with streaming garments. Shortly before +ten o'clock he came out into an open place opposite the village, and +saw the ferryboat lying in the shadow of the trees and the high bank. +Everything was quiet under the blinking stars. He crept down the bank, +watching with all his eyes, slipped into the water, swam three or four +strokes and climbed into the skiff that did "yawl" duty at the boat's +stern. He laid himself down under the thwarts and waited, panting. + +Presently the cracked bell tapped and a voice gave the order to "cast +off." A minute or two later the skiff's head was standing high up, +against the boat's swell, and the voyage was begun. Tom felt happy in +his success, for he knew it was the boat's last trip for the night. At +the end of a long twelve or fifteen minutes the wheels stopped, and Tom +slipped overboard and swam ashore in the dusk, landing fifty yards +downstream, out of danger of possible stragglers. + +He flew along unfrequented alleys, and shortly found himself at his +aunt's back fence. He climbed over, approached the "ell," and looked in +at the sitting-room window, for a light was burning there. There sat +Aunt Polly, Sid, Mary, and Joe Harper's mother, grouped together, +talking. They were by the bed, and the bed was between them and the +door. Tom went to the door and began to softly lift the latch; then he +pressed gently and the door yielded a crack; he continued pushing +cautiously, and quaking every time it creaked, till he judged he might +squeeze through on his knees; so he put his head through and began, +warily. + +"What makes the candle blow so?" said Aunt Polly. Tom hurried up. +"Why, that door's open, I believe. Why, of course it is. No end of +strange things now. Go 'long and shut it, Sid." + +Tom disappeared under the bed just in time. He lay and "breathed" +himself for a time, and then crept to where he could almost touch his +aunt's foot. + +"But as I was saying," said Aunt Polly, "he warn't BAD, so to say +--only mischEEvous. Only just giddy, and harum-scarum, you know. He +warn't any more responsible than a colt. HE never meant any harm, and +he was the best-hearted boy that ever was"--and she began to cry. + +"It was just so with my Joe--always full of his devilment, and up to +every kind of mischief, but he was just as unselfish and kind as he +could be--and laws bless me, to think I went and whipped him for taking +that cream, never once recollecting that I throwed it out myself +because it was sour, and I never to see him again in this world, never, +never, never, poor abused boy!" And Mrs. Harper sobbed as if her heart +would break. + +"I hope Tom's better off where he is," said Sid, "but if he'd been +better in some ways--" + +"SID!" Tom felt the glare of the old lady's eye, though he could not +see it. "Not a word against my Tom, now that he's gone! God'll take +care of HIM--never you trouble YOURself, sir! Oh, Mrs. Harper, I don't +know how to give him up! I don't know how to give him up! He was such a +comfort to me, although he tormented my old heart out of me, 'most." + +"The Lord giveth and the Lord hath taken away--Blessed be the name of +the Lord! But it's so hard--Oh, it's so hard! Only last Saturday my +Joe busted a firecracker right under my nose and I knocked him +sprawling. Little did I know then, how soon--Oh, if it was to do over +again I'd hug him and bless him for it." + +"Yes, yes, yes, I know just how you feel, Mrs. Harper, I know just +exactly how you feel. No longer ago than yesterday noon, my Tom took +and filled the cat full of Pain-killer, and I did think the cretur +would tear the house down. And God forgive me, I cracked Tom's head +with my thimble, poor boy, poor dead boy. But he's out of all his +troubles now. And the last words I ever heard him say was to reproach--" + +But this memory was too much for the old lady, and she broke entirely +down. Tom was snuffling, now, himself--and more in pity of himself than +anybody else. He could hear Mary crying, and putting in a kindly word +for him from time to time. He began to have a nobler opinion of himself +than ever before. Still, he was sufficiently touched by his aunt's +grief to long to rush out from under the bed and overwhelm her with +joy--and the theatrical gorgeousness of the thing appealed strongly to +his nature, too, but he resisted and lay still. + +He went on listening, and gathered by odds and ends that it was +conjectured at first that the boys had got drowned while taking a swim; +then the small raft had been missed; next, certain boys said the +missing lads had promised that the village should "hear something" +soon; the wise-heads had "put this and that together" and decided that +the lads had gone off on that raft and would turn up at the next town +below, presently; but toward noon the raft had been found, lodged +against the Missouri shore some five or six miles below the village +--and then hope perished; they must be drowned, else hunger would have +driven them home by nightfall if not sooner. It was believed that the +search for the bodies had been a fruitless effort merely because the +drowning must have occurred in mid-channel, since the boys, being good +swimmers, would otherwise have escaped to shore. This was Wednesday +night. If the bodies continued missing until Sunday, all hope would be +given over, and the funerals would be preached on that morning. Tom +shuddered. + +Mrs. Harper gave a sobbing good-night and turned to go. Then with a +mutual impulse the two bereaved women flung themselves into each +other's arms and had a good, consoling cry, and then parted. Aunt Polly +was tender far beyond her wont, in her good-night to Sid and Mary. Sid +snuffled a bit and Mary went off crying with all her heart. + +Aunt Polly knelt down and prayed for Tom so touchingly, so +appealingly, and with such measureless love in her words and her old +trembling voice, that he was weltering in tears again, long before she +was through. + +He had to keep still long after she went to bed, for she kept making +broken-hearted ejaculations from time to time, tossing unrestfully, and +turning over. But at last she was still, only moaning a little in her +sleep. Now the boy stole out, rose gradually by the bedside, shaded the +candle-light with his hand, and stood regarding her. His heart was full +of pity for her. He took out his sycamore scroll and placed it by the +candle. But something occurred to him, and he lingered considering. His +face lighted with a happy solution of his thought; he put the bark +hastily in his pocket. Then he bent over and kissed the faded lips, and +straightway made his stealthy exit, latching the door behind him. + +He threaded his way back to the ferry landing, found nobody at large +there, and walked boldly on board the boat, for he knew she was +tenantless except that there was a watchman, who always turned in and +slept like a graven image. He untied the skiff at the stern, slipped +into it, and was soon rowing cautiously upstream. When he had pulled a +mile above the village, he started quartering across and bent himself +stoutly to his work. He hit the landing on the other side neatly, for +this was a familiar bit of work to him. He was moved to capture the +skiff, arguing that it might be considered a ship and therefore +legitimate prey for a pirate, but he knew a thorough search would be +made for it and that might end in revelations. So he stepped ashore and +entered the woods. + +He sat down and took a long rest, torturing himself meanwhile to keep +awake, and then started warily down the home-stretch. The night was far +spent. It was broad daylight before he found himself fairly abreast the +island bar. He rested again until the sun was well up and gilding the +great river with its splendor, and then he plunged into the stream. A +little later he paused, dripping, upon the threshold of the camp, and +heard Joe say: + +"No, Tom's true-blue, Huck, and he'll come back. He won't desert. He +knows that would be a disgrace to a pirate, and Tom's too proud for +that sort of thing. He's up to something or other. Now I wonder what?" + +"Well, the things is ours, anyway, ain't they?" + +"Pretty near, but not yet, Huck. The writing says they are if he ain't +back here to breakfast." + +"Which he is!" exclaimed Tom, with fine dramatic effect, stepping +grandly into camp. + +A sumptuous breakfast of bacon and fish was shortly provided, and as +the boys set to work upon it, Tom recounted (and adorned) his +adventures. They were a vain and boastful company of heroes when the +tale was done. Then Tom hid himself away in a shady nook to sleep till +noon, and the other pirates got ready to fish and explore. + + + +CHAPTER XVI + +AFTER dinner all the gang turned out to hunt for turtle eggs on the +bar. They went about poking sticks into the sand, and when they found a +soft place they went down on their knees and dug with their hands. +Sometimes they would take fifty or sixty eggs out of one hole. They +were perfectly round white things a trifle smaller than an English +walnut. They had a famous fried-egg feast that night, and another on +Friday morning. + +After breakfast they went whooping and prancing out on the bar, and +chased each other round and round, shedding clothes as they went, until +they were naked, and then continued the frolic far away up the shoal +water of the bar, against the stiff current, which latter tripped their +legs from under them from time to time and greatly increased the fun. +And now and then they stooped in a group and splashed water in each +other's faces with their palms, gradually approaching each other, with +averted faces to avoid the strangling sprays, and finally gripping and +struggling till the best man ducked his neighbor, and then they all +went under in a tangle of white legs and arms and came up blowing, +sputtering, laughing, and gasping for breath at one and the same time. + +When they were well exhausted, they would run out and sprawl on the +dry, hot sand, and lie there and cover themselves up with it, and by +and by break for the water again and go through the original +performance once more. Finally it occurred to them that their naked +skin represented flesh-colored "tights" very fairly; so they drew a +ring in the sand and had a circus--with three clowns in it, for none +would yield this proudest post to his neighbor. + +Next they got their marbles and played "knucks" and "ring-taw" and +"keeps" till that amusement grew stale. Then Joe and Huck had another +swim, but Tom would not venture, because he found that in kicking off +his trousers he had kicked his string of rattlesnake rattles off his +ankle, and he wondered how he had escaped cramp so long without the +protection of this mysterious charm. He did not venture again until he +had found it, and by that time the other boys were tired and ready to +rest. They gradually wandered apart, dropped into the "dumps," and fell +to gazing longingly across the wide river to where the village lay +drowsing in the sun. Tom found himself writing "BECKY" in the sand with +his big toe; he scratched it out, and was angry with himself for his +weakness. But he wrote it again, nevertheless; he could not help it. He +erased it once more and then took himself out of temptation by driving +the other boys together and joining them. + +But Joe's spirits had gone down almost beyond resurrection. He was so +homesick that he could hardly endure the misery of it. The tears lay +very near the surface. Huck was melancholy, too. Tom was downhearted, +but tried hard not to show it. He had a secret which he was not ready +to tell, yet, but if this mutinous depression was not broken up soon, +he would have to bring it out. He said, with a great show of +cheerfulness: + +"I bet there's been pirates on this island before, boys. We'll explore +it again. They've hid treasures here somewhere. How'd you feel to light +on a rotten chest full of gold and silver--hey?" + +But it roused only faint enthusiasm, which faded out, with no reply. +Tom tried one or two other seductions; but they failed, too. It was +discouraging work. Joe sat poking up the sand with a stick and looking +very gloomy. Finally he said: + +"Oh, boys, let's give it up. I want to go home. It's so lonesome." + +"Oh no, Joe, you'll feel better by and by," said Tom. "Just think of +the fishing that's here." + +"I don't care for fishing. I want to go home." + +"But, Joe, there ain't such another swimming-place anywhere." + +"Swimming's no good. I don't seem to care for it, somehow, when there +ain't anybody to say I sha'n't go in. I mean to go home." + +"Oh, shucks! Baby! You want to see your mother, I reckon." + +"Yes, I DO want to see my mother--and you would, too, if you had one. +I ain't any more baby than you are." And Joe snuffled a little. + +"Well, we'll let the cry-baby go home to his mother, won't we, Huck? +Poor thing--does it want to see its mother? And so it shall. You like +it here, don't you, Huck? We'll stay, won't we?" + +Huck said, "Y-e-s"--without any heart in it. + +"I'll never speak to you again as long as I live," said Joe, rising. +"There now!" And he moved moodily away and began to dress himself. + +"Who cares!" said Tom. "Nobody wants you to. Go 'long home and get +laughed at. Oh, you're a nice pirate. Huck and me ain't cry-babies. +We'll stay, won't we, Huck? Let him go if he wants to. I reckon we can +get along without him, per'aps." + +But Tom was uneasy, nevertheless, and was alarmed to see Joe go +sullenly on with his dressing. And then it was discomforting to see +Huck eying Joe's preparations so wistfully, and keeping up such an +ominous silence. Presently, without a parting word, Joe began to wade +off toward the Illinois shore. Tom's heart began to sink. He glanced at +Huck. Huck could not bear the look, and dropped his eyes. Then he said: + +"I want to go, too, Tom. It was getting so lonesome anyway, and now +it'll be worse. Let's us go, too, Tom." + +"I won't! You can all go, if you want to. I mean to stay." + +"Tom, I better go." + +"Well, go 'long--who's hendering you." + +Huck began to pick up his scattered clothes. He said: + +"Tom, I wisht you'd come, too. Now you think it over. We'll wait for +you when we get to shore." + +"Well, you'll wait a blame long time, that's all." + +Huck started sorrowfully away, and Tom stood looking after him, with a +strong desire tugging at his heart to yield his pride and go along too. +He hoped the boys would stop, but they still waded slowly on. It +suddenly dawned on Tom that it was become very lonely and still. He +made one final struggle with his pride, and then darted after his +comrades, yelling: + +"Wait! Wait! I want to tell you something!" + +They presently stopped and turned around. When he got to where they +were, he began unfolding his secret, and they listened moodily till at +last they saw the "point" he was driving at, and then they set up a +war-whoop of applause and said it was "splendid!" and said if he had +told them at first, they wouldn't have started away. He made a plausible +excuse; but his real reason had been the fear that not even the secret +would keep them with him any very great length of time, and so he had +meant to hold it in reserve as a last seduction. + +The lads came gayly back and went at their sports again with a will, +chattering all the time about Tom's stupendous plan and admiring the +genius of it. After a dainty egg and fish dinner, Tom said he wanted to +learn to smoke, now. Joe caught at the idea and said he would like to +try, too. So Huck made pipes and filled them. These novices had never +smoked anything before but cigars made of grape-vine, and they "bit" +the tongue, and were not considered manly anyway. + +Now they stretched themselves out on their elbows and began to puff, +charily, and with slender confidence. The smoke had an unpleasant +taste, and they gagged a little, but Tom said: + +"Why, it's just as easy! If I'd a knowed this was all, I'd a learnt +long ago." + +"So would I," said Joe. "It's just nothing." + +"Why, many a time I've looked at people smoking, and thought well I +wish I could do that; but I never thought I could," said Tom. + +"That's just the way with me, hain't it, Huck? You've heard me talk +just that way--haven't you, Huck? I'll leave it to Huck if I haven't." + +"Yes--heaps of times," said Huck. + +"Well, I have too," said Tom; "oh, hundreds of times. Once down by the +slaughter-house. Don't you remember, Huck? Bob Tanner was there, and +Johnny Miller, and Jeff Thatcher, when I said it. Don't you remember, +Huck, 'bout me saying that?" + +"Yes, that's so," said Huck. "That was the day after I lost a white +alley. No, 'twas the day before." + +"There--I told you so," said Tom. "Huck recollects it." + +"I bleeve I could smoke this pipe all day," said Joe. "I don't feel +sick." + +"Neither do I," said Tom. "I could smoke it all day. But I bet you +Jeff Thatcher couldn't." + +"Jeff Thatcher! Why, he'd keel over just with two draws. Just let him +try it once. HE'D see!" + +"I bet he would. And Johnny Miller--I wish could see Johnny Miller +tackle it once." + +"Oh, don't I!" said Joe. "Why, I bet you Johnny Miller couldn't any +more do this than nothing. Just one little snifter would fetch HIM." + +"'Deed it would, Joe. Say--I wish the boys could see us now." + +"So do I." + +"Say--boys, don't say anything about it, and some time when they're +around, I'll come up to you and say, 'Joe, got a pipe? I want a smoke.' +And you'll say, kind of careless like, as if it warn't anything, you'll +say, 'Yes, I got my OLD pipe, and another one, but my tobacker ain't +very good.' And I'll say, 'Oh, that's all right, if it's STRONG +enough.' And then you'll out with the pipes, and we'll light up just as +ca'm, and then just see 'em look!" + +"By jings, that'll be gay, Tom! I wish it was NOW!" + +"So do I! And when we tell 'em we learned when we was off pirating, +won't they wish they'd been along?" + +"Oh, I reckon not! I'll just BET they will!" + +So the talk ran on. But presently it began to flag a trifle, and grow +disjointed. The silences widened; the expectoration marvellously +increased. Every pore inside the boys' cheeks became a spouting +fountain; they could scarcely bail out the cellars under their tongues +fast enough to prevent an inundation; little overflowings down their +throats occurred in spite of all they could do, and sudden retchings +followed every time. Both boys were looking very pale and miserable, +now. Joe's pipe dropped from his nerveless fingers. Tom's followed. +Both fountains were going furiously and both pumps bailing with might +and main. Joe said feebly: + +"I've lost my knife. I reckon I better go and find it." + +Tom said, with quivering lips and halting utterance: + +"I'll help you. You go over that way and I'll hunt around by the +spring. No, you needn't come, Huck--we can find it." + +So Huck sat down again, and waited an hour. Then he found it lonesome, +and went to find his comrades. They were wide apart in the woods, both +very pale, both fast asleep. But something informed him that if they +had had any trouble they had got rid of it. + +They were not talkative at supper that night. They had a humble look, +and when Huck prepared his pipe after the meal and was going to prepare +theirs, they said no, they were not feeling very well--something they +ate at dinner had disagreed with them. + +About midnight Joe awoke, and called the boys. There was a brooding +oppressiveness in the air that seemed to bode something. The boys +huddled themselves together and sought the friendly companionship of +the fire, though the dull dead heat of the breathless atmosphere was +stifling. They sat still, intent and waiting. The solemn hush +continued. Beyond the light of the fire everything was swallowed up in +the blackness of darkness. Presently there came a quivering glow that +vaguely revealed the foliage for a moment and then vanished. By and by +another came, a little stronger. Then another. Then a faint moan came +sighing through the branches of the forest and the boys felt a fleeting +breath upon their cheeks, and shuddered with the fancy that the Spirit +of the Night had gone by. There was a pause. Now a weird flash turned +night into day and showed every little grass-blade, separate and +distinct, that grew about their feet. And it showed three white, +startled faces, too. A deep peal of thunder went rolling and tumbling +down the heavens and lost itself in sullen rumblings in the distance. A +sweep of chilly air passed by, rustling all the leaves and snowing the +flaky ashes broadcast about the fire. Another fierce glare lit up the +forest and an instant crash followed that seemed to rend the tree-tops +right over the boys' heads. They clung together in terror, in the thick +gloom that followed. A few big rain-drops fell pattering upon the +leaves. + +"Quick! boys, go for the tent!" exclaimed Tom. + +They sprang away, stumbling over roots and among vines in the dark, no +two plunging in the same direction. A furious blast roared through the +trees, making everything sing as it went. One blinding flash after +another came, and peal on peal of deafening thunder. And now a +drenching rain poured down and the rising hurricane drove it in sheets +along the ground. The boys cried out to each other, but the roaring +wind and the booming thunder-blasts drowned their voices utterly. +However, one by one they straggled in at last and took shelter under +the tent, cold, scared, and streaming with water; but to have company +in misery seemed something to be grateful for. They could not talk, the +old sail flapped so furiously, even if the other noises would have +allowed them. The tempest rose higher and higher, and presently the +sail tore loose from its fastenings and went winging away on the blast. +The boys seized each others' hands and fled, with many tumblings and +bruises, to the shelter of a great oak that stood upon the river-bank. +Now the battle was at its highest. Under the ceaseless conflagration of +lightning that flamed in the skies, everything below stood out in +clean-cut and shadowless distinctness: the bending trees, the billowy +river, white with foam, the driving spray of spume-flakes, the dim +outlines of the high bluffs on the other side, glimpsed through the +drifting cloud-rack and the slanting veil of rain. Every little while +some giant tree yielded the fight and fell crashing through the younger +growth; and the unflagging thunder-peals came now in ear-splitting +explosive bursts, keen and sharp, and unspeakably appalling. The storm +culminated in one matchless effort that seemed likely to tear the island +to pieces, burn it up, drown it to the tree-tops, blow it away, and +deafen every creature in it, all at one and the same moment. It was a +wild night for homeless young heads to be out in. + +But at last the battle was done, and the forces retired with weaker +and weaker threatenings and grumblings, and peace resumed her sway. The +boys went back to camp, a good deal awed; but they found there was +still something to be thankful for, because the great sycamore, the +shelter of their beds, was a ruin, now, blasted by the lightnings, and +they were not under it when the catastrophe happened. + +Everything in camp was drenched, the camp-fire as well; for they were +but heedless lads, like their generation, and had made no provision +against rain. Here was matter for dismay, for they were soaked through +and chilled. They were eloquent in their distress; but they presently +discovered that the fire had eaten so far up under the great log it had +been built against (where it curved upward and separated itself from +the ground), that a handbreadth or so of it had escaped wetting; so +they patiently wrought until, with shreds and bark gathered from the +under sides of sheltered logs, they coaxed the fire to burn again. Then +they piled on great dead boughs till they had a roaring furnace, and +were glad-hearted once more. They dried their boiled ham and had a +feast, and after that they sat by the fire and expanded and glorified +their midnight adventure until morning, for there was not a dry spot to +sleep on, anywhere around. + +As the sun began to steal in upon the boys, drowsiness came over them, +and they went out on the sandbar and lay down to sleep. They got +scorched out by and by, and drearily set about getting breakfast. After +the meal they felt rusty, and stiff-jointed, and a little homesick once +more. Tom saw the signs, and fell to cheering up the pirates as well as +he could. But they cared nothing for marbles, or circus, or swimming, +or anything. He reminded them of the imposing secret, and raised a ray +of cheer. While it lasted, he got them interested in a new device. This +was to knock off being pirates, for a while, and be Indians for a +change. They were attracted by this idea; so it was not long before +they were stripped, and striped from head to heel with black mud, like +so many zebras--all of them chiefs, of course--and then they went +tearing through the woods to attack an English settlement. + +By and by they separated into three hostile tribes, and darted upon +each other from ambush with dreadful war-whoops, and killed and scalped +each other by thousands. It was a gory day. Consequently it was an +extremely satisfactory one. + +They assembled in camp toward supper-time, hungry and happy; but now a +difficulty arose--hostile Indians could not break the bread of +hospitality together without first making peace, and this was a simple +impossibility without smoking a pipe of peace. There was no other +process that ever they had heard of. Two of the savages almost wished +they had remained pirates. However, there was no other way; so with +such show of cheerfulness as they could muster they called for the pipe +and took their whiff as it passed, in due form. + +And behold, they were glad they had gone into savagery, for they had +gained something; they found that they could now smoke a little without +having to go and hunt for a lost knife; they did not get sick enough to +be seriously uncomfortable. They were not likely to fool away this high +promise for lack of effort. No, they practised cautiously, after +supper, with right fair success, and so they spent a jubilant evening. +They were prouder and happier in their new acquirement than they would +have been in the scalping and skinning of the Six Nations. We will +leave them to smoke and chatter and brag, since we have no further use +for them at present. + + + +CHAPTER XVII + +BUT there was no hilarity in the little town that same tranquil +Saturday afternoon. The Harpers, and Aunt Polly's family, were being +put into mourning, with great grief and many tears. An unusual quiet +possessed the village, although it was ordinarily quiet enough, in all +conscience. The villagers conducted their concerns with an absent air, +and talked little; but they sighed often. The Saturday holiday seemed a +burden to the children. They had no heart in their sports, and +gradually gave them up. + +In the afternoon Becky Thatcher found herself moping about the +deserted schoolhouse yard, and feeling very melancholy. But she found +nothing there to comfort her. She soliloquized: + +"Oh, if I only had a brass andiron-knob again! But I haven't got +anything now to remember him by." And she choked back a little sob. + +Presently she stopped, and said to herself: + +"It was right here. Oh, if it was to do over again, I wouldn't say +that--I wouldn't say it for the whole world. But he's gone now; I'll +never, never, never see him any more." + +This thought broke her down, and she wandered away, with tears rolling +down her cheeks. Then quite a group of boys and girls--playmates of +Tom's and Joe's--came by, and stood looking over the paling fence and +talking in reverent tones of how Tom did so-and-so the last time they +saw him, and how Joe said this and that small trifle (pregnant with +awful prophecy, as they could easily see now!)--and each speaker +pointed out the exact spot where the lost lads stood at the time, and +then added something like "and I was a-standing just so--just as I am +now, and as if you was him--I was as close as that--and he smiled, just +this way--and then something seemed to go all over me, like--awful, you +know--and I never thought what it meant, of course, but I can see now!" + +Then there was a dispute about who saw the dead boys last in life, and +many claimed that dismal distinction, and offered evidences, more or +less tampered with by the witness; and when it was ultimately decided +who DID see the departed last, and exchanged the last words with them, +the lucky parties took upon themselves a sort of sacred importance, and +were gaped at and envied by all the rest. One poor chap, who had no +other grandeur to offer, said with tolerably manifest pride in the +remembrance: + +"Well, Tom Sawyer he licked me once." + +But that bid for glory was a failure. Most of the boys could say that, +and so that cheapened the distinction too much. The group loitered +away, still recalling memories of the lost heroes, in awed voices. + +When the Sunday-school hour was finished, the next morning, the bell +began to toll, instead of ringing in the usual way. It was a very still +Sabbath, and the mournful sound seemed in keeping with the musing hush +that lay upon nature. The villagers began to gather, loitering a moment +in the vestibule to converse in whispers about the sad event. But there +was no whispering in the house; only the funereal rustling of dresses +as the women gathered to their seats disturbed the silence there. None +could remember when the little church had been so full before. There +was finally a waiting pause, an expectant dumbness, and then Aunt Polly +entered, followed by Sid and Mary, and they by the Harper family, all +in deep black, and the whole congregation, the old minister as well, +rose reverently and stood until the mourners were seated in the front +pew. There was another communing silence, broken at intervals by +muffled sobs, and then the minister spread his hands abroad and prayed. +A moving hymn was sung, and the text followed: "I am the Resurrection +and the Life." + +As the service proceeded, the clergyman drew such pictures of the +graces, the winning ways, and the rare promise of the lost lads that +every soul there, thinking he recognized these pictures, felt a pang in +remembering that he had persistently blinded himself to them always +before, and had as persistently seen only faults and flaws in the poor +boys. The minister related many a touching incident in the lives of the +departed, too, which illustrated their sweet, generous natures, and the +people could easily see, now, how noble and beautiful those episodes +were, and remembered with grief that at the time they occurred they had +seemed rank rascalities, well deserving of the cowhide. The +congregation became more and more moved, as the pathetic tale went on, +till at last the whole company broke down and joined the weeping +mourners in a chorus of anguished sobs, the preacher himself giving way +to his feelings, and crying in the pulpit. + +There was a rustle in the gallery, which nobody noticed; a moment +later the church door creaked; the minister raised his streaming eyes +above his handkerchief, and stood transfixed! First one and then +another pair of eyes followed the minister's, and then almost with one +impulse the congregation rose and stared while the three dead boys came +marching up the aisle, Tom in the lead, Joe next, and Huck, a ruin of +drooping rags, sneaking sheepishly in the rear! They had been hid in +the unused gallery listening to their own funeral sermon! + +Aunt Polly, Mary, and the Harpers threw themselves upon their restored +ones, smothered them with kisses and poured out thanksgivings, while +poor Huck stood abashed and uncomfortable, not knowing exactly what to +do or where to hide from so many unwelcoming eyes. He wavered, and +started to slink away, but Tom seized him and said: + +"Aunt Polly, it ain't fair. Somebody's got to be glad to see Huck." + +"And so they shall. I'm glad to see him, poor motherless thing!" And +the loving attentions Aunt Polly lavished upon him were the one thing +capable of making him more uncomfortable than he was before. + +Suddenly the minister shouted at the top of his voice: "Praise God +from whom all blessings flow--SING!--and put your hearts in it!" + +And they did. Old Hundred swelled up with a triumphant burst, and +while it shook the rafters Tom Sawyer the Pirate looked around upon the +envying juveniles about him and confessed in his heart that this was +the proudest moment of his life. + +As the "sold" congregation trooped out they said they would almost be +willing to be made ridiculous again to hear Old Hundred sung like that +once more. + +Tom got more cuffs and kisses that day--according to Aunt Polly's +varying moods--than he had earned before in a year; and he hardly knew +which expressed the most gratefulness to God and affection for himself. + + + +CHAPTER XVIII + +THAT was Tom's great secret--the scheme to return home with his +brother pirates and attend their own funerals. They had paddled over to +the Missouri shore on a log, at dusk on Saturday, landing five or six +miles below the village; they had slept in the woods at the edge of the +town till nearly daylight, and had then crept through back lanes and +alleys and finished their sleep in the gallery of the church among a +chaos of invalided benches. + +At breakfast, Monday morning, Aunt Polly and Mary were very loving to +Tom, and very attentive to his wants. There was an unusual amount of +talk. In the course of it Aunt Polly said: + +"Well, I don't say it wasn't a fine joke, Tom, to keep everybody +suffering 'most a week so you boys had a good time, but it is a pity +you could be so hard-hearted as to let me suffer so. If you could come +over on a log to go to your funeral, you could have come over and give +me a hint some way that you warn't dead, but only run off." + +"Yes, you could have done that, Tom," said Mary; "and I believe you +would if you had thought of it." + +"Would you, Tom?" said Aunt Polly, her face lighting wistfully. "Say, +now, would you, if you'd thought of it?" + +"I--well, I don't know. 'Twould 'a' spoiled everything." + +"Tom, I hoped you loved me that much," said Aunt Polly, with a grieved +tone that discomforted the boy. "It would have been something if you'd +cared enough to THINK of it, even if you didn't DO it." + +"Now, auntie, that ain't any harm," pleaded Mary; "it's only Tom's +giddy way--he is always in such a rush that he never thinks of +anything." + +"More's the pity. Sid would have thought. And Sid would have come and +DONE it, too. Tom, you'll look back, some day, when it's too late, and +wish you'd cared a little more for me when it would have cost you so +little." + +"Now, auntie, you know I do care for you," said Tom. + +"I'd know it better if you acted more like it." + +"I wish now I'd thought," said Tom, with a repentant tone; "but I +dreamt about you, anyway. That's something, ain't it?" + +"It ain't much--a cat does that much--but it's better than nothing. +What did you dream?" + +"Why, Wednesday night I dreamt that you was sitting over there by the +bed, and Sid was sitting by the woodbox, and Mary next to him." + +"Well, so we did. So we always do. I'm glad your dreams could take +even that much trouble about us." + +"And I dreamt that Joe Harper's mother was here." + +"Why, she was here! Did you dream any more?" + +"Oh, lots. But it's so dim, now." + +"Well, try to recollect--can't you?" + +"Somehow it seems to me that the wind--the wind blowed the--the--" + +"Try harder, Tom! The wind did blow something. Come!" + +Tom pressed his fingers on his forehead an anxious minute, and then +said: + +"I've got it now! I've got it now! It blowed the candle!" + +"Mercy on us! Go on, Tom--go on!" + +"And it seems to me that you said, 'Why, I believe that that door--'" + +"Go ON, Tom!" + +"Just let me study a moment--just a moment. Oh, yes--you said you +believed the door was open." + +"As I'm sitting here, I did! Didn't I, Mary! Go on!" + +"And then--and then--well I won't be certain, but it seems like as if +you made Sid go and--and--" + +"Well? Well? What did I make him do, Tom? What did I make him do?" + +"You made him--you--Oh, you made him shut it." + +"Well, for the land's sake! I never heard the beat of that in all my +days! Don't tell ME there ain't anything in dreams, any more. Sereny +Harper shall know of this before I'm an hour older. I'd like to see her +get around THIS with her rubbage 'bout superstition. Go on, Tom!" + +"Oh, it's all getting just as bright as day, now. Next you said I +warn't BAD, only mischeevous and harum-scarum, and not any more +responsible than--than--I think it was a colt, or something." + +"And so it was! Well, goodness gracious! Go on, Tom!" + +"And then you began to cry." + +"So I did. So I did. Not the first time, neither. And then--" + +"Then Mrs. Harper she began to cry, and said Joe was just the same, +and she wished she hadn't whipped him for taking cream when she'd +throwed it out her own self--" + +"Tom! The sperrit was upon you! You was a prophesying--that's what you +was doing! Land alive, go on, Tom!" + +"Then Sid he said--he said--" + +"I don't think I said anything," said Sid. + +"Yes you did, Sid," said Mary. + +"Shut your heads and let Tom go on! What did he say, Tom?" + +"He said--I THINK he said he hoped I was better off where I was gone +to, but if I'd been better sometimes--" + +"THERE, d'you hear that! It was his very words!" + +"And you shut him up sharp." + +"I lay I did! There must 'a' been an angel there. There WAS an angel +there, somewheres!" + +"And Mrs. Harper told about Joe scaring her with a firecracker, and +you told about Peter and the Painkiller--" + +"Just as true as I live!" + +"And then there was a whole lot of talk 'bout dragging the river for +us, and 'bout having the funeral Sunday, and then you and old Miss +Harper hugged and cried, and she went." + +"It happened just so! It happened just so, as sure as I'm a-sitting in +these very tracks. Tom, you couldn't told it more like if you'd 'a' +seen it! And then what? Go on, Tom!" + +"Then I thought you prayed for me--and I could see you and hear every +word you said. And you went to bed, and I was so sorry that I took and +wrote on a piece of sycamore bark, 'We ain't dead--we are only off +being pirates,' and put it on the table by the candle; and then you +looked so good, laying there asleep, that I thought I went and leaned +over and kissed you on the lips." + +"Did you, Tom, DID you! I just forgive you everything for that!" And +she seized the boy in a crushing embrace that made him feel like the +guiltiest of villains. + +"It was very kind, even though it was only a--dream," Sid soliloquized +just audibly. + +"Shut up, Sid! A body does just the same in a dream as he'd do if he +was awake. Here's a big Milum apple I've been saving for you, Tom, if +you was ever found again--now go 'long to school. I'm thankful to the +good God and Father of us all I've got you back, that's long-suffering +and merciful to them that believe on Him and keep His word, though +goodness knows I'm unworthy of it, but if only the worthy ones got His +blessings and had His hand to help them over the rough places, there's +few enough would smile here or ever enter into His rest when the long +night comes. Go 'long Sid, Mary, Tom--take yourselves off--you've +hendered me long enough." + +The children left for school, and the old lady to call on Mrs. Harper +and vanquish her realism with Tom's marvellous dream. Sid had better +judgment than to utter the thought that was in his mind as he left the +house. It was this: "Pretty thin--as long a dream as that, without any +mistakes in it!" + +What a hero Tom was become, now! He did not go skipping and prancing, +but moved with a dignified swagger as became a pirate who felt that the +public eye was on him. And indeed it was; he tried not to seem to see +the looks or hear the remarks as he passed along, but they were food +and drink to him. Smaller boys than himself flocked at his heels, as +proud to be seen with him, and tolerated by him, as if he had been the +drummer at the head of a procession or the elephant leading a menagerie +into town. Boys of his own size pretended not to know he had been away +at all; but they were consuming with envy, nevertheless. They would +have given anything to have that swarthy suntanned skin of his, and his +glittering notoriety; and Tom would not have parted with either for a +circus. + +At school the children made so much of him and of Joe, and delivered +such eloquent admiration from their eyes, that the two heroes were not +long in becoming insufferably "stuck-up." They began to tell their +adventures to hungry listeners--but they only began; it was not a thing +likely to have an end, with imaginations like theirs to furnish +material. And finally, when they got out their pipes and went serenely +puffing around, the very summit of glory was reached. + +Tom decided that he could be independent of Becky Thatcher now. Glory +was sufficient. He would live for glory. Now that he was distinguished, +maybe she would be wanting to "make up." Well, let her--she should see +that he could be as indifferent as some other people. Presently she +arrived. Tom pretended not to see her. He moved away and joined a group +of boys and girls and began to talk. Soon he observed that she was +tripping gayly back and forth with flushed face and dancing eyes, +pretending to be busy chasing schoolmates, and screaming with laughter +when she made a capture; but he noticed that she always made her +captures in his vicinity, and that she seemed to cast a conscious eye +in his direction at such times, too. It gratified all the vicious +vanity that was in him; and so, instead of winning him, it only "set +him up" the more and made him the more diligent to avoid betraying that +he knew she was about. Presently she gave over skylarking, and moved +irresolutely about, sighing once or twice and glancing furtively and +wistfully toward Tom. Then she observed that now Tom was talking more +particularly to Amy Lawrence than to any one else. She felt a sharp +pang and grew disturbed and uneasy at once. She tried to go away, but +her feet were treacherous, and carried her to the group instead. She +said to a girl almost at Tom's elbow--with sham vivacity: + +"Why, Mary Austin! you bad girl, why didn't you come to Sunday-school?" + +"I did come--didn't you see me?" + +"Why, no! Did you? Where did you sit?" + +"I was in Miss Peters' class, where I always go. I saw YOU." + +"Did you? Why, it's funny I didn't see you. I wanted to tell you about +the picnic." + +"Oh, that's jolly. Who's going to give it?" + +"My ma's going to let me have one." + +"Oh, goody; I hope she'll let ME come." + +"Well, she will. The picnic's for me. She'll let anybody come that I +want, and I want you." + +"That's ever so nice. When is it going to be?" + +"By and by. Maybe about vacation." + +"Oh, won't it be fun! You going to have all the girls and boys?" + +"Yes, every one that's friends to me--or wants to be"; and she glanced +ever so furtively at Tom, but he talked right along to Amy Lawrence +about the terrible storm on the island, and how the lightning tore the +great sycamore tree "all to flinders" while he was "standing within +three feet of it." + +"Oh, may I come?" said Grace Miller. + +"Yes." + +"And me?" said Sally Rogers. + +"Yes." + +"And me, too?" said Susy Harper. "And Joe?" + +"Yes." + +And so on, with clapping of joyful hands till all the group had begged +for invitations but Tom and Amy. Then Tom turned coolly away, still +talking, and took Amy with him. Becky's lips trembled and the tears +came to her eyes; she hid these signs with a forced gayety and went on +chattering, but the life had gone out of the picnic, now, and out of +everything else; she got away as soon as she could and hid herself and +had what her sex call "a good cry." Then she sat moody, with wounded +pride, till the bell rang. She roused up, now, with a vindictive cast +in her eye, and gave her plaited tails a shake and said she knew what +SHE'D do. + +At recess Tom continued his flirtation with Amy with jubilant +self-satisfaction. And he kept drifting about to find Becky and lacerate +her with the performance. At last he spied her, but there was a sudden +falling of his mercury. She was sitting cosily on a little bench behind +the schoolhouse looking at a picture-book with Alfred Temple--and so +absorbed were they, and their heads so close together over the book, +that they did not seem to be conscious of anything in the world besides. +Jealousy ran red-hot through Tom's veins. He began to hate himself for +throwing away the chance Becky had offered for a reconciliation. He +called himself a fool, and all the hard names he could think of. He +wanted to cry with vexation. Amy chatted happily along, as they walked, +for her heart was singing, but Tom's tongue had lost its function. He +did not hear what Amy was saying, and whenever she paused expectantly he +could only stammer an awkward assent, which was as often misplaced as +otherwise. He kept drifting to the rear of the schoolhouse, again and +again, to sear his eyeballs with the hateful spectacle there. He could +not help it. And it maddened him to see, as he thought he saw, that +Becky Thatcher never once suspected that he was even in the land of the +living. But she did see, nevertheless; and she knew she was winning her +fight, too, and was glad to see him suffer as she had suffered. + +Amy's happy prattle became intolerable. Tom hinted at things he had to +attend to; things that must be done; and time was fleeting. But in +vain--the girl chirped on. Tom thought, "Oh, hang her, ain't I ever +going to get rid of her?" At last he must be attending to those +things--and she said artlessly that she would be "around" when school +let out. And he hastened away, hating her for it. + +"Any other boy!" Tom thought, grating his teeth. "Any boy in the whole +town but that Saint Louis smarty that thinks he dresses so fine and is +aristocracy! Oh, all right, I licked you the first day you ever saw +this town, mister, and I'll lick you again! You just wait till I catch +you out! I'll just take and--" + +And he went through the motions of thrashing an imaginary boy +--pummelling the air, and kicking and gouging. "Oh, you do, do you? You +holler 'nough, do you? Now, then, let that learn you!" And so the +imaginary flogging was finished to his satisfaction. + +Tom fled home at noon. His conscience could not endure any more of +Amy's grateful happiness, and his jealousy could bear no more of the +other distress. Becky resumed her picture inspections with Alfred, but +as the minutes dragged along and no Tom came to suffer, her triumph +began to cloud and she lost interest; gravity and absent-mindedness +followed, and then melancholy; two or three times she pricked up her +ear at a footstep, but it was a false hope; no Tom came. At last she +grew entirely miserable and wished she hadn't carried it so far. When +poor Alfred, seeing that he was losing her, he did not know how, kept +exclaiming: "Oh, here's a jolly one! look at this!" she lost patience +at last, and said, "Oh, don't bother me! I don't care for them!" and +burst into tears, and got up and walked away. + +Alfred dropped alongside and was going to try to comfort her, but she +said: + +"Go away and leave me alone, can't you! I hate you!" + +So the boy halted, wondering what he could have done--for she had said +she would look at pictures all through the nooning--and she walked on, +crying. Then Alfred went musing into the deserted schoolhouse. He was +humiliated and angry. He easily guessed his way to the truth--the girl +had simply made a convenience of him to vent her spite upon Tom Sawyer. +He was far from hating Tom the less when this thought occurred to him. +He wished there was some way to get that boy into trouble without much +risk to himself. Tom's spelling-book fell under his eye. Here was his +opportunity. He gratefully opened to the lesson for the afternoon and +poured ink upon the page. + +Becky, glancing in at a window behind him at the moment, saw the act, +and moved on, without discovering herself. She started homeward, now, +intending to find Tom and tell him; Tom would be thankful and their +troubles would be healed. Before she was half way home, however, she +had changed her mind. The thought of Tom's treatment of her when she +was talking about her picnic came scorching back and filled her with +shame. She resolved to let him get whipped on the damaged +spelling-book's account, and to hate him forever, into the bargain. + + + +CHAPTER XIX + +TOM arrived at home in a dreary mood, and the first thing his aunt +said to him showed him that he had brought his sorrows to an +unpromising market: + +"Tom, I've a notion to skin you alive!" + +"Auntie, what have I done?" + +"Well, you've done enough. Here I go over to Sereny Harper, like an +old softy, expecting I'm going to make her believe all that rubbage +about that dream, when lo and behold you she'd found out from Joe that +you was over here and heard all the talk we had that night. Tom, I +don't know what is to become of a boy that will act like that. It makes +me feel so bad to think you could let me go to Sereny Harper and make +such a fool of myself and never say a word." + +This was a new aspect of the thing. His smartness of the morning had +seemed to Tom a good joke before, and very ingenious. It merely looked +mean and shabby now. He hung his head and could not think of anything +to say for a moment. Then he said: + +"Auntie, I wish I hadn't done it--but I didn't think." + +"Oh, child, you never think. You never think of anything but your own +selfishness. You could think to come all the way over here from +Jackson's Island in the night to laugh at our troubles, and you could +think to fool me with a lie about a dream; but you couldn't ever think +to pity us and save us from sorrow." + +"Auntie, I know now it was mean, but I didn't mean to be mean. I +didn't, honest. And besides, I didn't come over here to laugh at you +that night." + +"What did you come for, then?" + +"It was to tell you not to be uneasy about us, because we hadn't got +drownded." + +"Tom, Tom, I would be the thankfullest soul in this world if I could +believe you ever had as good a thought as that, but you know you never +did--and I know it, Tom." + +"Indeed and 'deed I did, auntie--I wish I may never stir if I didn't." + +"Oh, Tom, don't lie--don't do it. It only makes things a hundred times +worse." + +"It ain't a lie, auntie; it's the truth. I wanted to keep you from +grieving--that was all that made me come." + +"I'd give the whole world to believe that--it would cover up a power +of sins, Tom. I'd 'most be glad you'd run off and acted so bad. But it +ain't reasonable; because, why didn't you tell me, child?" + +"Why, you see, when you got to talking about the funeral, I just got +all full of the idea of our coming and hiding in the church, and I +couldn't somehow bear to spoil it. So I just put the bark back in my +pocket and kept mum." + +"What bark?" + +"The bark I had wrote on to tell you we'd gone pirating. I wish, now, +you'd waked up when I kissed you--I do, honest." + +The hard lines in his aunt's face relaxed and a sudden tenderness +dawned in her eyes. + +"DID you kiss me, Tom?" + +"Why, yes, I did." + +"Are you sure you did, Tom?" + +"Why, yes, I did, auntie--certain sure." + +"What did you kiss me for, Tom?" + +"Because I loved you so, and you laid there moaning and I was so sorry." + +The words sounded like truth. The old lady could not hide a tremor in +her voice when she said: + +"Kiss me again, Tom!--and be off with you to school, now, and don't +bother me any more." + +The moment he was gone, she ran to a closet and got out the ruin of a +jacket which Tom had gone pirating in. Then she stopped, with it in her +hand, and said to herself: + +"No, I don't dare. Poor boy, I reckon he's lied about it--but it's a +blessed, blessed lie, there's such a comfort come from it. I hope the +Lord--I KNOW the Lord will forgive him, because it was such +goodheartedness in him to tell it. But I don't want to find out it's a +lie. I won't look." + +She put the jacket away, and stood by musing a minute. Twice she put +out her hand to take the garment again, and twice she refrained. Once +more she ventured, and this time she fortified herself with the +thought: "It's a good lie--it's a good lie--I won't let it grieve me." +So she sought the jacket pocket. A moment later she was reading Tom's +piece of bark through flowing tears and saying: "I could forgive the +boy, now, if he'd committed a million sins!" + + + +CHAPTER XX + +THERE was something about Aunt Polly's manner, when she kissed Tom, +that swept away his low spirits and made him lighthearted and happy +again. He started to school and had the luck of coming upon Becky +Thatcher at the head of Meadow Lane. His mood always determined his +manner. Without a moment's hesitation he ran to her and said: + +"I acted mighty mean to-day, Becky, and I'm so sorry. I won't ever, +ever do that way again, as long as ever I live--please make up, won't +you?" + +The girl stopped and looked him scornfully in the face: + +"I'll thank you to keep yourself TO yourself, Mr. Thomas Sawyer. I'll +never speak to you again." + +She tossed her head and passed on. Tom was so stunned that he had not +even presence of mind enough to say "Who cares, Miss Smarty?" until the +right time to say it had gone by. So he said nothing. But he was in a +fine rage, nevertheless. He moped into the schoolyard wishing she were +a boy, and imagining how he would trounce her if she were. He presently +encountered her and delivered a stinging remark as he passed. She +hurled one in return, and the angry breach was complete. It seemed to +Becky, in her hot resentment, that she could hardly wait for school to +"take in," she was so impatient to see Tom flogged for the injured +spelling-book. If she had had any lingering notion of exposing Alfred +Temple, Tom's offensive fling had driven it entirely away. + +Poor girl, she did not know how fast she was nearing trouble herself. +The master, Mr. Dobbins, had reached middle age with an unsatisfied +ambition. The darling of his desires was, to be a doctor, but poverty +had decreed that he should be nothing higher than a village +schoolmaster. Every day he took a mysterious book out of his desk and +absorbed himself in it at times when no classes were reciting. He kept +that book under lock and key. There was not an urchin in school but was +perishing to have a glimpse of it, but the chance never came. Every boy +and girl had a theory about the nature of that book; but no two +theories were alike, and there was no way of getting at the facts in +the case. Now, as Becky was passing by the desk, which stood near the +door, she noticed that the key was in the lock! It was a precious +moment. She glanced around; found herself alone, and the next instant +she had the book in her hands. The title-page--Professor Somebody's +ANATOMY--carried no information to her mind; so she began to turn the +leaves. She came at once upon a handsomely engraved and colored +frontispiece--a human figure, stark naked. At that moment a shadow fell +on the page and Tom Sawyer stepped in at the door and caught a glimpse +of the picture. Becky snatched at the book to close it, and had the +hard luck to tear the pictured page half down the middle. She thrust +the volume into the desk, turned the key, and burst out crying with +shame and vexation. + +"Tom Sawyer, you are just as mean as you can be, to sneak up on a +person and look at what they're looking at." + +"How could I know you was looking at anything?" + +"You ought to be ashamed of yourself, Tom Sawyer; you know you're +going to tell on me, and oh, what shall I do, what shall I do! I'll be +whipped, and I never was whipped in school." + +Then she stamped her little foot and said: + +"BE so mean if you want to! I know something that's going to happen. +You just wait and you'll see! Hateful, hateful, hateful!"--and she +flung out of the house with a new explosion of crying. + +Tom stood still, rather flustered by this onslaught. Presently he said +to himself: + +"What a curious kind of a fool a girl is! Never been licked in school! +Shucks! What's a licking! That's just like a girl--they're so +thin-skinned and chicken-hearted. Well, of course I ain't going to tell +old Dobbins on this little fool, because there's other ways of getting +even on her, that ain't so mean; but what of it? Old Dobbins will ask +who it was tore his book. Nobody'll answer. Then he'll do just the way +he always does--ask first one and then t'other, and when he comes to the +right girl he'll know it, without any telling. Girls' faces always tell +on them. They ain't got any backbone. She'll get licked. Well, it's a +kind of a tight place for Becky Thatcher, because there ain't any way +out of it." Tom conned the thing a moment longer, and then added: "All +right, though; she'd like to see me in just such a fix--let her sweat it +out!" + +Tom joined the mob of skylarking scholars outside. In a few moments +the master arrived and school "took in." Tom did not feel a strong +interest in his studies. Every time he stole a glance at the girls' +side of the room Becky's face troubled him. Considering all things, he +did not want to pity her, and yet it was all he could do to help it. He +could get up no exultation that was really worthy the name. Presently +the spelling-book discovery was made, and Tom's mind was entirely full +of his own matters for a while after that. Becky roused up from her +lethargy of distress and showed good interest in the proceedings. She +did not expect that Tom could get out of his trouble by denying that he +spilt the ink on the book himself; and she was right. The denial only +seemed to make the thing worse for Tom. Becky supposed she would be +glad of that, and she tried to believe she was glad of it, but she +found she was not certain. When the worst came to the worst, she had an +impulse to get up and tell on Alfred Temple, but she made an effort and +forced herself to keep still--because, said she to herself, "he'll tell +about me tearing the picture sure. I wouldn't say a word, not to save +his life!" + +Tom took his whipping and went back to his seat not at all +broken-hearted, for he thought it was possible that he had unknowingly +upset the ink on the spelling-book himself, in some skylarking bout--he +had denied it for form's sake and because it was custom, and had stuck +to the denial from principle. + +A whole hour drifted by, the master sat nodding in his throne, the air +was drowsy with the hum of study. By and by, Mr. Dobbins straightened +himself up, yawned, then unlocked his desk, and reached for his book, +but seemed undecided whether to take it out or leave it. Most of the +pupils glanced up languidly, but there were two among them that watched +his movements with intent eyes. Mr. Dobbins fingered his book absently +for a while, then took it out and settled himself in his chair to read! +Tom shot a glance at Becky. He had seen a hunted and helpless rabbit +look as she did, with a gun levelled at its head. Instantly he forgot +his quarrel with her. Quick--something must be done! done in a flash, +too! But the very imminence of the emergency paralyzed his invention. +Good!--he had an inspiration! He would run and snatch the book, spring +through the door and fly. But his resolution shook for one little +instant, and the chance was lost--the master opened the volume. If Tom +only had the wasted opportunity back again! Too late. There was no help +for Becky now, he said. The next moment the master faced the school. +Every eye sank under his gaze. There was that in it which smote even +the innocent with fear. There was silence while one might count ten +--the master was gathering his wrath. Then he spoke: "Who tore this book?" + +There was not a sound. One could have heard a pin drop. The stillness +continued; the master searched face after face for signs of guilt. + +"Benjamin Rogers, did you tear this book?" + +A denial. Another pause. + +"Joseph Harper, did you?" + +Another denial. Tom's uneasiness grew more and more intense under the +slow torture of these proceedings. The master scanned the ranks of +boys--considered a while, then turned to the girls: + +"Amy Lawrence?" + +A shake of the head. + +"Gracie Miller?" + +The same sign. + +"Susan Harper, did you do this?" + +Another negative. The next girl was Becky Thatcher. Tom was trembling +from head to foot with excitement and a sense of the hopelessness of +the situation. + +"Rebecca Thatcher" [Tom glanced at her face--it was white with terror] +--"did you tear--no, look me in the face" [her hands rose in appeal] +--"did you tear this book?" + +A thought shot like lightning through Tom's brain. He sprang to his +feet and shouted--"I done it!" + +The school stared in perplexity at this incredible folly. Tom stood a +moment, to gather his dismembered faculties; and when he stepped +forward to go to his punishment the surprise, the gratitude, the +adoration that shone upon him out of poor Becky's eyes seemed pay +enough for a hundred floggings. Inspired by the splendor of his own +act, he took without an outcry the most merciless flaying that even Mr. +Dobbins had ever administered; and also received with indifference the +added cruelty of a command to remain two hours after school should be +dismissed--for he knew who would wait for him outside till his +captivity was done, and not count the tedious time as loss, either. + +Tom went to bed that night planning vengeance against Alfred Temple; +for with shame and repentance Becky had told him all, not forgetting +her own treachery; but even the longing for vengeance had to give way, +soon, to pleasanter musings, and he fell asleep at last with Becky's +latest words lingering dreamily in his ear-- + +"Tom, how COULD you be so noble!" + + + +CHAPTER XXI + +VACATION was approaching. The schoolmaster, always severe, grew +severer and more exacting than ever, for he wanted the school to make a +good showing on "Examination" day. His rod and his ferule were seldom +idle now--at least among the smaller pupils. Only the biggest boys, and +young ladies of eighteen and twenty, escaped lashing. Mr. Dobbins' +lashings were very vigorous ones, too; for although he carried, under +his wig, a perfectly bald and shiny head, he had only reached middle +age, and there was no sign of feebleness in his muscle. As the great +day approached, all the tyranny that was in him came to the surface; he +seemed to take a vindictive pleasure in punishing the least +shortcomings. The consequence was, that the smaller boys spent their +days in terror and suffering and their nights in plotting revenge. They +threw away no opportunity to do the master a mischief. But he kept +ahead all the time. The retribution that followed every vengeful +success was so sweeping and majestic that the boys always retired from +the field badly worsted. At last they conspired together and hit upon a +plan that promised a dazzling victory. They swore in the sign-painter's +boy, told him the scheme, and asked his help. He had his own reasons +for being delighted, for the master boarded in his father's family and +had given the boy ample cause to hate him. The master's wife would go +on a visit to the country in a few days, and there would be nothing to +interfere with the plan; the master always prepared himself for great +occasions by getting pretty well fuddled, and the sign-painter's boy +said that when the dominie had reached the proper condition on +Examination Evening he would "manage the thing" while he napped in his +chair; then he would have him awakened at the right time and hurried +away to school. + +In the fulness of time the interesting occasion arrived. At eight in +the evening the schoolhouse was brilliantly lighted, and adorned with +wreaths and festoons of foliage and flowers. The master sat throned in +his great chair upon a raised platform, with his blackboard behind him. +He was looking tolerably mellow. Three rows of benches on each side and +six rows in front of him were occupied by the dignitaries of the town +and by the parents of the pupils. To his left, back of the rows of +citizens, was a spacious temporary platform upon which were seated the +scholars who were to take part in the exercises of the evening; rows of +small boys, washed and dressed to an intolerable state of discomfort; +rows of gawky big boys; snowbanks of girls and young ladies clad in +lawn and muslin and conspicuously conscious of their bare arms, their +grandmothers' ancient trinkets, their bits of pink and blue ribbon and +the flowers in their hair. All the rest of the house was filled with +non-participating scholars. + +The exercises began. A very little boy stood up and sheepishly +recited, "You'd scarce expect one of my age to speak in public on the +stage," etc.--accompanying himself with the painfully exact and +spasmodic gestures which a machine might have used--supposing the +machine to be a trifle out of order. But he got through safely, though +cruelly scared, and got a fine round of applause when he made his +manufactured bow and retired. + +A little shamefaced girl lisped, "Mary had a little lamb," etc., +performed a compassion-inspiring curtsy, got her meed of applause, and +sat down flushed and happy. + +Tom Sawyer stepped forward with conceited confidence and soared into +the unquenchable and indestructible "Give me liberty or give me death" +speech, with fine fury and frantic gesticulation, and broke down in the +middle of it. A ghastly stage-fright seized him, his legs quaked under +him and he was like to choke. True, he had the manifest sympathy of the +house but he had the house's silence, too, which was even worse than +its sympathy. The master frowned, and this completed the disaster. Tom +struggled awhile and then retired, utterly defeated. There was a weak +attempt at applause, but it died early. + +"The Boy Stood on the Burning Deck" followed; also "The Assyrian Came +Down," and other declamatory gems. Then there were reading exercises, +and a spelling fight. The meagre Latin class recited with honor. The +prime feature of the evening was in order, now--original "compositions" +by the young ladies. Each in her turn stepped forward to the edge of +the platform, cleared her throat, held up her manuscript (tied with +dainty ribbon), and proceeded to read, with labored attention to +"expression" and punctuation. The themes were the same that had been +illuminated upon similar occasions by their mothers before them, their +grandmothers, and doubtless all their ancestors in the female line +clear back to the Crusades. "Friendship" was one; "Memories of Other +Days"; "Religion in History"; "Dream Land"; "The Advantages of +Culture"; "Forms of Political Government Compared and Contrasted"; +"Melancholy"; "Filial Love"; "Heart Longings," etc., etc. + +A prevalent feature in these compositions was a nursed and petted +melancholy; another was a wasteful and opulent gush of "fine language"; +another was a tendency to lug in by the ears particularly prized words +and phrases until they were worn entirely out; and a peculiarity that +conspicuously marked and marred them was the inveterate and intolerable +sermon that wagged its crippled tail at the end of each and every one +of them. No matter what the subject might be, a brain-racking effort +was made to squirm it into some aspect or other that the moral and +religious mind could contemplate with edification. The glaring +insincerity of these sermons was not sufficient to compass the +banishment of the fashion from the schools, and it is not sufficient +to-day; it never will be sufficient while the world stands, perhaps. +There is no school in all our land where the young ladies do not feel +obliged to close their compositions with a sermon; and you will find +that the sermon of the most frivolous and the least religious girl in +the school is always the longest and the most relentlessly pious. But +enough of this. Homely truth is unpalatable. + +Let us return to the "Examination." The first composition that was +read was one entitled "Is this, then, Life?" Perhaps the reader can +endure an extract from it: + + "In the common walks of life, with what delightful + emotions does the youthful mind look forward to some + anticipated scene of festivity! Imagination is busy + sketching rose-tinted pictures of joy. In fancy, the + voluptuous votary of fashion sees herself amid the + festive throng, 'the observed of all observers.' Her + graceful form, arrayed in snowy robes, is whirling + through the mazes of the joyous dance; her eye is + brightest, her step is lightest in the gay assembly. + + "In such delicious fancies time quickly glides by, + and the welcome hour arrives for her entrance into + the Elysian world, of which she has had such bright + dreams. How fairy-like does everything appear to + her enchanted vision! Each new scene is more charming + than the last. But after a while she finds that + beneath this goodly exterior, all is vanity, the + flattery which once charmed her soul, now grates + harshly upon her ear; the ball-room has lost its + charms; and with wasted health and imbittered heart, + she turns away with the conviction that earthly + pleasures cannot satisfy the longings of the soul!" + +And so forth and so on. There was a buzz of gratification from time to +time during the reading, accompanied by whispered ejaculations of "How +sweet!" "How eloquent!" "So true!" etc., and after the thing had closed +with a peculiarly afflicting sermon the applause was enthusiastic. + +Then arose a slim, melancholy girl, whose face had the "interesting" +paleness that comes of pills and indigestion, and read a "poem." Two +stanzas of it will do: + + "A MISSOURI MAIDEN'S FAREWELL TO ALABAMA + + "Alabama, good-bye! I love thee well! + But yet for a while do I leave thee now! + Sad, yes, sad thoughts of thee my heart doth swell, + And burning recollections throng my brow! + For I have wandered through thy flowery woods; + Have roamed and read near Tallapoosa's stream; + Have listened to Tallassee's warring floods, + And wooed on Coosa's side Aurora's beam. + + "Yet shame I not to bear an o'er-full heart, + Nor blush to turn behind my tearful eyes; + 'Tis from no stranger land I now must part, + 'Tis to no strangers left I yield these sighs. + Welcome and home were mine within this State, + Whose vales I leave--whose spires fade fast from me + And cold must be mine eyes, and heart, and tete, + When, dear Alabama! they turn cold on thee!" + +There were very few there who knew what "tete" meant, but the poem was +very satisfactory, nevertheless. + +Next appeared a dark-complexioned, black-eyed, black-haired young +lady, who paused an impressive moment, assumed a tragic expression, and +began to read in a measured, solemn tone: + + "A VISION + + "Dark and tempestuous was night. Around the + throne on high not a single star quivered; but + the deep intonations of the heavy thunder + constantly vibrated upon the ear; whilst the + terrific lightning revelled in angry mood + through the cloudy chambers of heaven, seeming + to scorn the power exerted over its terror by + the illustrious Franklin! Even the boisterous + winds unanimously came forth from their mystic + homes, and blustered about as if to enhance by + their aid the wildness of the scene. + + "At such a time, so dark, so dreary, for human + sympathy my very spirit sighed; but instead thereof, + + "'My dearest friend, my counsellor, my comforter + and guide--My joy in grief, my second bliss + in joy,' came to my side. She moved like one of + those bright beings pictured in the sunny walks + of fancy's Eden by the romantic and young, a + queen of beauty unadorned save by her own + transcendent loveliness. So soft was her step, it + failed to make even a sound, and but for the + magical thrill imparted by her genial touch, as + other unobtrusive beauties, she would have glided + away un-perceived--unsought. A strange sadness + rested upon her features, like icy tears upon + the robe of December, as she pointed to the + contending elements without, and bade me contemplate + the two beings presented." + +This nightmare occupied some ten pages of manuscript and wound up with +a sermon so destructive of all hope to non-Presbyterians that it took +the first prize. This composition was considered to be the very finest +effort of the evening. The mayor of the village, in delivering the +prize to the author of it, made a warm speech in which he said that it +was by far the most "eloquent" thing he had ever listened to, and that +Daniel Webster himself might well be proud of it. + +It may be remarked, in passing, that the number of compositions in +which the word "beauteous" was over-fondled, and human experience +referred to as "life's page," was up to the usual average. + +Now the master, mellow almost to the verge of geniality, put his chair +aside, turned his back to the audience, and began to draw a map of +America on the blackboard, to exercise the geography class upon. But he +made a sad business of it with his unsteady hand, and a smothered +titter rippled over the house. He knew what the matter was, and set +himself to right it. He sponged out lines and remade them; but he only +distorted them more than ever, and the tittering was more pronounced. +He threw his entire attention upon his work, now, as if determined not +to be put down by the mirth. He felt that all eyes were fastened upon +him; he imagined he was succeeding, and yet the tittering continued; it +even manifestly increased. And well it might. There was a garret above, +pierced with a scuttle over his head; and down through this scuttle +came a cat, suspended around the haunches by a string; she had a rag +tied about her head and jaws to keep her from mewing; as she slowly +descended she curved upward and clawed at the string, she swung +downward and clawed at the intangible air. The tittering rose higher +and higher--the cat was within six inches of the absorbed teacher's +head--down, down, a little lower, and she grabbed his wig with her +desperate claws, clung to it, and was snatched up into the garret in an +instant with her trophy still in her possession! And how the light did +blaze abroad from the master's bald pate--for the sign-painter's boy +had GILDED it! + +That broke up the meeting. The boys were avenged. Vacation had come. + + NOTE:--The pretended "compositions" quoted in + this chapter are taken without alteration from a + volume entitled "Prose and Poetry, by a Western + Lady"--but they are exactly and precisely after + the schoolgirl pattern, and hence are much + happier than any mere imitations could be. + + + +CHAPTER XXII + +TOM joined the new order of Cadets of Temperance, being attracted by +the showy character of their "regalia." He promised to abstain from +smoking, chewing, and profanity as long as he remained a member. Now he +found out a new thing--namely, that to promise not to do a thing is the +surest way in the world to make a body want to go and do that very +thing. Tom soon found himself tormented with a desire to drink and +swear; the desire grew to be so intense that nothing but the hope of a +chance to display himself in his red sash kept him from withdrawing +from the order. Fourth of July was coming; but he soon gave that up +--gave it up before he had worn his shackles over forty-eight hours--and +fixed his hopes upon old Judge Frazer, justice of the peace, who was +apparently on his deathbed and would have a big public funeral, since +he was so high an official. During three days Tom was deeply concerned +about the Judge's condition and hungry for news of it. Sometimes his +hopes ran high--so high that he would venture to get out his regalia +and practise before the looking-glass. But the Judge had a most +discouraging way of fluctuating. At last he was pronounced upon the +mend--and then convalescent. Tom was disgusted; and felt a sense of +injury, too. He handed in his resignation at once--and that night the +Judge suffered a relapse and died. Tom resolved that he would never +trust a man like that again. + +The funeral was a fine thing. The Cadets paraded in a style calculated +to kill the late member with envy. Tom was a free boy again, however +--there was something in that. He could drink and swear, now--but found +to his surprise that he did not want to. The simple fact that he could, +took the desire away, and the charm of it. + +Tom presently wondered to find that his coveted vacation was beginning +to hang a little heavily on his hands. + +He attempted a diary--but nothing happened during three days, and so +he abandoned it. + +The first of all the negro minstrel shows came to town, and made a +sensation. Tom and Joe Harper got up a band of performers and were +happy for two days. + +Even the Glorious Fourth was in some sense a failure, for it rained +hard, there was no procession in consequence, and the greatest man in +the world (as Tom supposed), Mr. Benton, an actual United States +Senator, proved an overwhelming disappointment--for he was not +twenty-five feet high, nor even anywhere in the neighborhood of it. + +A circus came. The boys played circus for three days afterward in +tents made of rag carpeting--admission, three pins for boys, two for +girls--and then circusing was abandoned. + +A phrenologist and a mesmerizer came--and went again and left the +village duller and drearier than ever. + +There were some boys-and-girls' parties, but they were so few and so +delightful that they only made the aching voids between ache the harder. + +Becky Thatcher was gone to her Constantinople home to stay with her +parents during vacation--so there was no bright side to life anywhere. + +The dreadful secret of the murder was a chronic misery. It was a very +cancer for permanency and pain. + +Then came the measles. + +During two long weeks Tom lay a prisoner, dead to the world and its +happenings. He was very ill, he was interested in nothing. When he got +upon his feet at last and moved feebly down-town, a melancholy change +had come over everything and every creature. There had been a +"revival," and everybody had "got religion," not only the adults, but +even the boys and girls. Tom went about, hoping against hope for the +sight of one blessed sinful face, but disappointment crossed him +everywhere. He found Joe Harper studying a Testament, and turned sadly +away from the depressing spectacle. He sought Ben Rogers, and found him +visiting the poor with a basket of tracts. He hunted up Jim Hollis, who +called his attention to the precious blessing of his late measles as a +warning. Every boy he encountered added another ton to his depression; +and when, in desperation, he flew for refuge at last to the bosom of +Huckleberry Finn and was received with a Scriptural quotation, his +heart broke and he crept home and to bed realizing that he alone of all +the town was lost, forever and forever. + +And that night there came on a terrific storm, with driving rain, +awful claps of thunder and blinding sheets of lightning. He covered his +head with the bedclothes and waited in a horror of suspense for his +doom; for he had not the shadow of a doubt that all this hubbub was +about him. He believed he had taxed the forbearance of the powers above +to the extremity of endurance and that this was the result. It might +have seemed to him a waste of pomp and ammunition to kill a bug with a +battery of artillery, but there seemed nothing incongruous about the +getting up such an expensive thunderstorm as this to knock the turf +from under an insect like himself. + +By and by the tempest spent itself and died without accomplishing its +object. The boy's first impulse was to be grateful, and reform. His +second was to wait--for there might not be any more storms. + +The next day the doctors were back; Tom had relapsed. The three weeks +he spent on his back this time seemed an entire age. When he got abroad +at last he was hardly grateful that he had been spared, remembering how +lonely was his estate, how companionless and forlorn he was. He drifted +listlessly down the street and found Jim Hollis acting as judge in a +juvenile court that was trying a cat for murder, in the presence of her +victim, a bird. He found Joe Harper and Huck Finn up an alley eating a +stolen melon. Poor lads! they--like Tom--had suffered a relapse. + + + +CHAPTER XXIII + +AT last the sleepy atmosphere was stirred--and vigorously: the murder +trial came on in the court. It became the absorbing topic of village +talk immediately. Tom could not get away from it. Every reference to +the murder sent a shudder to his heart, for his troubled conscience and +fears almost persuaded him that these remarks were put forth in his +hearing as "feelers"; he did not see how he could be suspected of +knowing anything about the murder, but still he could not be +comfortable in the midst of this gossip. It kept him in a cold shiver +all the time. He took Huck to a lonely place to have a talk with him. +It would be some relief to unseal his tongue for a little while; to +divide his burden of distress with another sufferer. Moreover, he +wanted to assure himself that Huck had remained discreet. + +"Huck, have you ever told anybody about--that?" + +"'Bout what?" + +"You know what." + +"Oh--'course I haven't." + +"Never a word?" + +"Never a solitary word, so help me. What makes you ask?" + +"Well, I was afeard." + +"Why, Tom Sawyer, we wouldn't be alive two days if that got found out. +YOU know that." + +Tom felt more comfortable. After a pause: + +"Huck, they couldn't anybody get you to tell, could they?" + +"Get me to tell? Why, if I wanted that half-breed devil to drownd me +they could get me to tell. They ain't no different way." + +"Well, that's all right, then. I reckon we're safe as long as we keep +mum. But let's swear again, anyway. It's more surer." + +"I'm agreed." + +So they swore again with dread solemnities. + +"What is the talk around, Huck? I've heard a power of it." + +"Talk? Well, it's just Muff Potter, Muff Potter, Muff Potter all the +time. It keeps me in a sweat, constant, so's I want to hide som'ers." + +"That's just the same way they go on round me. I reckon he's a goner. +Don't you feel sorry for him, sometimes?" + +"Most always--most always. He ain't no account; but then he hain't +ever done anything to hurt anybody. Just fishes a little, to get money +to get drunk on--and loafs around considerable; but lord, we all do +that--leastways most of us--preachers and such like. But he's kind of +good--he give me half a fish, once, when there warn't enough for two; +and lots of times he's kind of stood by me when I was out of luck." + +"Well, he's mended kites for me, Huck, and knitted hooks on to my +line. I wish we could get him out of there." + +"My! we couldn't get him out, Tom. And besides, 'twouldn't do any +good; they'd ketch him again." + +"Yes--so they would. But I hate to hear 'em abuse him so like the +dickens when he never done--that." + +"I do too, Tom. Lord, I hear 'em say he's the bloodiest looking +villain in this country, and they wonder he wasn't ever hung before." + +"Yes, they talk like that, all the time. I've heard 'em say that if he +was to get free they'd lynch him." + +"And they'd do it, too." + +The boys had a long talk, but it brought them little comfort. As the +twilight drew on, they found themselves hanging about the neighborhood +of the little isolated jail, perhaps with an undefined hope that +something would happen that might clear away their difficulties. But +nothing happened; there seemed to be no angels or fairies interested in +this luckless captive. + +The boys did as they had often done before--went to the cell grating +and gave Potter some tobacco and matches. He was on the ground floor +and there were no guards. + +His gratitude for their gifts had always smote their consciences +before--it cut deeper than ever, this time. They felt cowardly and +treacherous to the last degree when Potter said: + +"You've been mighty good to me, boys--better'n anybody else in this +town. And I don't forget it, I don't. Often I says to myself, says I, +'I used to mend all the boys' kites and things, and show 'em where the +good fishin' places was, and befriend 'em what I could, and now they've +all forgot old Muff when he's in trouble; but Tom don't, and Huck +don't--THEY don't forget him, says I, 'and I don't forget them.' Well, +boys, I done an awful thing--drunk and crazy at the time--that's the +only way I account for it--and now I got to swing for it, and it's +right. Right, and BEST, too, I reckon--hope so, anyway. Well, we won't +talk about that. I don't want to make YOU feel bad; you've befriended +me. But what I want to say, is, don't YOU ever get drunk--then you won't +ever get here. Stand a litter furder west--so--that's it; it's a prime +comfort to see faces that's friendly when a body's in such a muck of +trouble, and there don't none come here but yourn. Good friendly +faces--good friendly faces. Git up on one another's backs and let me +touch 'em. That's it. Shake hands--yourn'll come through the bars, but +mine's too big. Little hands, and weak--but they've helped Muff Potter +a power, and they'd help him more if they could." + +Tom went home miserable, and his dreams that night were full of +horrors. The next day and the day after, he hung about the court-room, +drawn by an almost irresistible impulse to go in, but forcing himself +to stay out. Huck was having the same experience. They studiously +avoided each other. Each wandered away, from time to time, but the same +dismal fascination always brought them back presently. Tom kept his +ears open when idlers sauntered out of the court-room, but invariably +heard distressing news--the toils were closing more and more +relentlessly around poor Potter. At the end of the second day the +village talk was to the effect that Injun Joe's evidence stood firm and +unshaken, and that there was not the slightest question as to what the +jury's verdict would be. + +Tom was out late, that night, and came to bed through the window. He +was in a tremendous state of excitement. It was hours before he got to +sleep. All the village flocked to the court-house the next morning, for +this was to be the great day. Both sexes were about equally represented +in the packed audience. After a long wait the jury filed in and took +their places; shortly afterward, Potter, pale and haggard, timid and +hopeless, was brought in, with chains upon him, and seated where all +the curious eyes could stare at him; no less conspicuous was Injun Joe, +stolid as ever. There was another pause, and then the judge arrived and +the sheriff proclaimed the opening of the court. The usual whisperings +among the lawyers and gathering together of papers followed. These +details and accompanying delays worked up an atmosphere of preparation +that was as impressive as it was fascinating. + +Now a witness was called who testified that he found Muff Potter +washing in the brook, at an early hour of the morning that the murder +was discovered, and that he immediately sneaked away. After some +further questioning, counsel for the prosecution said: + +"Take the witness." + +The prisoner raised his eyes for a moment, but dropped them again when +his own counsel said: + +"I have no questions to ask him." + +The next witness proved the finding of the knife near the corpse. +Counsel for the prosecution said: + +"Take the witness." + +"I have no questions to ask him," Potter's lawyer replied. + +A third witness swore he had often seen the knife in Potter's +possession. + +"Take the witness." + +Counsel for Potter declined to question him. The faces of the audience +began to betray annoyance. Did this attorney mean to throw away his +client's life without an effort? + +Several witnesses deposed concerning Potter's guilty behavior when +brought to the scene of the murder. They were allowed to leave the +stand without being cross-questioned. + +Every detail of the damaging circumstances that occurred in the +graveyard upon that morning which all present remembered so well was +brought out by credible witnesses, but none of them were cross-examined +by Potter's lawyer. The perplexity and dissatisfaction of the house +expressed itself in murmurs and provoked a reproof from the bench. +Counsel for the prosecution now said: + +"By the oaths of citizens whose simple word is above suspicion, we +have fastened this awful crime, beyond all possibility of question, +upon the unhappy prisoner at the bar. We rest our case here." + +A groan escaped from poor Potter, and he put his face in his hands and +rocked his body softly to and fro, while a painful silence reigned in +the court-room. Many men were moved, and many women's compassion +testified itself in tears. Counsel for the defence rose and said: + +"Your honor, in our remarks at the opening of this trial, we +foreshadowed our purpose to prove that our client did this fearful deed +while under the influence of a blind and irresponsible delirium +produced by drink. We have changed our mind. We shall not offer that +plea." [Then to the clerk:] "Call Thomas Sawyer!" + +A puzzled amazement awoke in every face in the house, not even +excepting Potter's. Every eye fastened itself with wondering interest +upon Tom as he rose and took his place upon the stand. The boy looked +wild enough, for he was badly scared. The oath was administered. + +"Thomas Sawyer, where were you on the seventeenth of June, about the +hour of midnight?" + +Tom glanced at Injun Joe's iron face and his tongue failed him. The +audience listened breathless, but the words refused to come. After a +few moments, however, the boy got a little of his strength back, and +managed to put enough of it into his voice to make part of the house +hear: + +"In the graveyard!" + +"A little bit louder, please. Don't be afraid. You were--" + +"In the graveyard." + +A contemptuous smile flitted across Injun Joe's face. + +"Were you anywhere near Horse Williams' grave?" + +"Yes, sir." + +"Speak up--just a trifle louder. How near were you?" + +"Near as I am to you." + +"Were you hidden, or not?" + +"I was hid." + +"Where?" + +"Behind the elms that's on the edge of the grave." + +Injun Joe gave a barely perceptible start. + +"Any one with you?" + +"Yes, sir. I went there with--" + +"Wait--wait a moment. Never mind mentioning your companion's name. We +will produce him at the proper time. Did you carry anything there with +you." + +Tom hesitated and looked confused. + +"Speak out, my boy--don't be diffident. The truth is always +respectable. What did you take there?" + +"Only a--a--dead cat." + +There was a ripple of mirth, which the court checked. + +"We will produce the skeleton of that cat. Now, my boy, tell us +everything that occurred--tell it in your own way--don't skip anything, +and don't be afraid." + +Tom began--hesitatingly at first, but as he warmed to his subject his +words flowed more and more easily; in a little while every sound ceased +but his own voice; every eye fixed itself upon him; with parted lips +and bated breath the audience hung upon his words, taking no note of +time, rapt in the ghastly fascinations of the tale. The strain upon +pent emotion reached its climax when the boy said: + +"--and as the doctor fetched the board around and Muff Potter fell, +Injun Joe jumped with the knife and--" + +Crash! Quick as lightning the half-breed sprang for a window, tore his +way through all opposers, and was gone! + + + +CHAPTER XXIV + +TOM was a glittering hero once more--the pet of the old, the envy of +the young. His name even went into immortal print, for the village +paper magnified him. There were some that believed he would be +President, yet, if he escaped hanging. + +As usual, the fickle, unreasoning world took Muff Potter to its bosom +and fondled him as lavishly as it had abused him before. But that sort +of conduct is to the world's credit; therefore it is not well to find +fault with it. + +Tom's days were days of splendor and exultation to him, but his nights +were seasons of horror. Injun Joe infested all his dreams, and always +with doom in his eye. Hardly any temptation could persuade the boy to +stir abroad after nightfall. Poor Huck was in the same state of +wretchedness and terror, for Tom had told the whole story to the lawyer +the night before the great day of the trial, and Huck was sore afraid +that his share in the business might leak out, yet, notwithstanding +Injun Joe's flight had saved him the suffering of testifying in court. +The poor fellow had got the attorney to promise secrecy, but what of +that? Since Tom's harassed conscience had managed to drive him to the +lawyer's house by night and wring a dread tale from lips that had been +sealed with the dismalest and most formidable of oaths, Huck's +confidence in the human race was well-nigh obliterated. + +Daily Muff Potter's gratitude made Tom glad he had spoken; but nightly +he wished he had sealed up his tongue. + +Half the time Tom was afraid Injun Joe would never be captured; the +other half he was afraid he would be. He felt sure he never could draw +a safe breath again until that man was dead and he had seen the corpse. + +Rewards had been offered, the country had been scoured, but no Injun +Joe was found. One of those omniscient and awe-inspiring marvels, a +detective, came up from St. Louis, moused around, shook his head, +looked wise, and made that sort of astounding success which members of +that craft usually achieve. That is to say, he "found a clew." But you +can't hang a "clew" for murder, and so after that detective had got +through and gone home, Tom felt just as insecure as he was before. + +The slow days drifted on, and each left behind it a slightly lightened +weight of apprehension. + + + +CHAPTER XXV + +THERE comes a time in every rightly-constructed boy's life when he has +a raging desire to go somewhere and dig for hidden treasure. This +desire suddenly came upon Tom one day. He sallied out to find Joe +Harper, but failed of success. Next he sought Ben Rogers; he had gone +fishing. Presently he stumbled upon Huck Finn the Red-Handed. Huck +would answer. Tom took him to a private place and opened the matter to +him confidentially. Huck was willing. Huck was always willing to take a +hand in any enterprise that offered entertainment and required no +capital, for he had a troublesome superabundance of that sort of time +which is not money. "Where'll we dig?" said Huck. + +"Oh, most anywhere." + +"Why, is it hid all around?" + +"No, indeed it ain't. It's hid in mighty particular places, Huck +--sometimes on islands, sometimes in rotten chests under the end of a +limb of an old dead tree, just where the shadow falls at midnight; but +mostly under the floor in ha'nted houses." + +"Who hides it?" + +"Why, robbers, of course--who'd you reckon? Sunday-school +sup'rintendents?" + +"I don't know. If 'twas mine I wouldn't hide it; I'd spend it and have +a good time." + +"So would I. But robbers don't do that way. They always hide it and +leave it there." + +"Don't they come after it any more?" + +"No, they think they will, but they generally forget the marks, or +else they die. Anyway, it lays there a long time and gets rusty; and by +and by somebody finds an old yellow paper that tells how to find the +marks--a paper that's got to be ciphered over about a week because it's +mostly signs and hy'roglyphics." + +"Hyro--which?" + +"Hy'roglyphics--pictures and things, you know, that don't seem to mean +anything." + +"Have you got one of them papers, Tom?" + +"No." + +"Well then, how you going to find the marks?" + +"I don't want any marks. They always bury it under a ha'nted house or +on an island, or under a dead tree that's got one limb sticking out. +Well, we've tried Jackson's Island a little, and we can try it again +some time; and there's the old ha'nted house up the Still-House branch, +and there's lots of dead-limb trees--dead loads of 'em." + +"Is it under all of them?" + +"How you talk! No!" + +"Then how you going to know which one to go for?" + +"Go for all of 'em!" + +"Why, Tom, it'll take all summer." + +"Well, what of that? Suppose you find a brass pot with a hundred +dollars in it, all rusty and gray, or rotten chest full of di'monds. +How's that?" + +Huck's eyes glowed. + +"That's bully. Plenty bully enough for me. Just you gimme the hundred +dollars and I don't want no di'monds." + +"All right. But I bet you I ain't going to throw off on di'monds. Some +of 'em's worth twenty dollars apiece--there ain't any, hardly, but's +worth six bits or a dollar." + +"No! Is that so?" + +"Cert'nly--anybody'll tell you so. Hain't you ever seen one, Huck?" + +"Not as I remember." + +"Oh, kings have slathers of them." + +"Well, I don' know no kings, Tom." + +"I reckon you don't. But if you was to go to Europe you'd see a raft +of 'em hopping around." + +"Do they hop?" + +"Hop?--your granny! No!" + +"Well, what did you say they did, for?" + +"Shucks, I only meant you'd SEE 'em--not hopping, of course--what do +they want to hop for?--but I mean you'd just see 'em--scattered around, +you know, in a kind of a general way. Like that old humpbacked Richard." + +"Richard? What's his other name?" + +"He didn't have any other name. Kings don't have any but a given name." + +"No?" + +"But they don't." + +"Well, if they like it, Tom, all right; but I don't want to be a king +and have only just a given name, like a nigger. But say--where you +going to dig first?" + +"Well, I don't know. S'pose we tackle that old dead-limb tree on the +hill t'other side of Still-House branch?" + +"I'm agreed." + +So they got a crippled pick and a shovel, and set out on their +three-mile tramp. They arrived hot and panting, and threw themselves +down in the shade of a neighboring elm to rest and have a smoke. + +"I like this," said Tom. + +"So do I." + +"Say, Huck, if we find a treasure here, what you going to do with your +share?" + +"Well, I'll have pie and a glass of soda every day, and I'll go to +every circus that comes along. I bet I'll have a gay time." + +"Well, ain't you going to save any of it?" + +"Save it? What for?" + +"Why, so as to have something to live on, by and by." + +"Oh, that ain't any use. Pap would come back to thish-yer town some +day and get his claws on it if I didn't hurry up, and I tell you he'd +clean it out pretty quick. What you going to do with yourn, Tom?" + +"I'm going to buy a new drum, and a sure-'nough sword, and a red +necktie and a bull pup, and get married." + +"Married!" + +"That's it." + +"Tom, you--why, you ain't in your right mind." + +"Wait--you'll see." + +"Well, that's the foolishest thing you could do. Look at pap and my +mother. Fight! Why, they used to fight all the time. I remember, mighty +well." + +"That ain't anything. The girl I'm going to marry won't fight." + +"Tom, I reckon they're all alike. They'll all comb a body. Now you +better think 'bout this awhile. I tell you you better. What's the name +of the gal?" + +"It ain't a gal at all--it's a girl." + +"It's all the same, I reckon; some says gal, some says girl--both's +right, like enough. Anyway, what's her name, Tom?" + +"I'll tell you some time--not now." + +"All right--that'll do. Only if you get married I'll be more lonesomer +than ever." + +"No you won't. You'll come and live with me. Now stir out of this and +we'll go to digging." + +They worked and sweated for half an hour. No result. They toiled +another half-hour. Still no result. Huck said: + +"Do they always bury it as deep as this?" + +"Sometimes--not always. Not generally. I reckon we haven't got the +right place." + +So they chose a new spot and began again. The labor dragged a little, +but still they made progress. They pegged away in silence for some +time. Finally Huck leaned on his shovel, swabbed the beaded drops from +his brow with his sleeve, and said: + +"Where you going to dig next, after we get this one?" + +"I reckon maybe we'll tackle the old tree that's over yonder on +Cardiff Hill back of the widow's." + +"I reckon that'll be a good one. But won't the widow take it away from +us, Tom? It's on her land." + +"SHE take it away! Maybe she'd like to try it once. Whoever finds one +of these hid treasures, it belongs to him. It don't make any difference +whose land it's on." + +That was satisfactory. The work went on. By and by Huck said: + +"Blame it, we must be in the wrong place again. What do you think?" + +"It is mighty curious, Huck. I don't understand it. Sometimes witches +interfere. I reckon maybe that's what's the trouble now." + +"Shucks! Witches ain't got no power in the daytime." + +"Well, that's so. I didn't think of that. Oh, I know what the matter +is! What a blamed lot of fools we are! You got to find out where the +shadow of the limb falls at midnight, and that's where you dig!" + +"Then consound it, we've fooled away all this work for nothing. Now +hang it all, we got to come back in the night. It's an awful long way. +Can you get out?" + +"I bet I will. We've got to do it to-night, too, because if somebody +sees these holes they'll know in a minute what's here and they'll go +for it." + +"Well, I'll come around and maow to-night." + +"All right. Let's hide the tools in the bushes." + +The boys were there that night, about the appointed time. They sat in +the shadow waiting. It was a lonely place, and an hour made solemn by +old traditions. Spirits whispered in the rustling leaves, ghosts lurked +in the murky nooks, the deep baying of a hound floated up out of the +distance, an owl answered with his sepulchral note. The boys were +subdued by these solemnities, and talked little. By and by they judged +that twelve had come; they marked where the shadow fell, and began to +dig. Their hopes commenced to rise. Their interest grew stronger, and +their industry kept pace with it. The hole deepened and still deepened, +but every time their hearts jumped to hear the pick strike upon +something, they only suffered a new disappointment. It was only a stone +or a chunk. At last Tom said: + +"It ain't any use, Huck, we're wrong again." + +"Well, but we CAN'T be wrong. We spotted the shadder to a dot." + +"I know it, but then there's another thing." + +"What's that?". + +"Why, we only guessed at the time. Like enough it was too late or too +early." + +Huck dropped his shovel. + +"That's it," said he. "That's the very trouble. We got to give this +one up. We can't ever tell the right time, and besides this kind of +thing's too awful, here this time of night with witches and ghosts +a-fluttering around so. I feel as if something's behind me all the time; +and I'm afeard to turn around, becuz maybe there's others in front +a-waiting for a chance. I been creeping all over, ever since I got here." + +"Well, I've been pretty much so, too, Huck. They most always put in a +dead man when they bury a treasure under a tree, to look out for it." + +"Lordy!" + +"Yes, they do. I've always heard that." + +"Tom, I don't like to fool around much where there's dead people. A +body's bound to get into trouble with 'em, sure." + +"I don't like to stir 'em up, either. S'pose this one here was to +stick his skull out and say something!" + +"Don't Tom! It's awful." + +"Well, it just is. Huck, I don't feel comfortable a bit." + +"Say, Tom, let's give this place up, and try somewheres else." + +"All right, I reckon we better." + +"What'll it be?" + +Tom considered awhile; and then said: + +"The ha'nted house. That's it!" + +"Blame it, I don't like ha'nted houses, Tom. Why, they're a dern sight +worse'n dead people. Dead people might talk, maybe, but they don't come +sliding around in a shroud, when you ain't noticing, and peep over your +shoulder all of a sudden and grit their teeth, the way a ghost does. I +couldn't stand such a thing as that, Tom--nobody could." + +"Yes, but, Huck, ghosts don't travel around only at night. They won't +hender us from digging there in the daytime." + +"Well, that's so. But you know mighty well people don't go about that +ha'nted house in the day nor the night." + +"Well, that's mostly because they don't like to go where a man's been +murdered, anyway--but nothing's ever been seen around that house except +in the night--just some blue lights slipping by the windows--no regular +ghosts." + +"Well, where you see one of them blue lights flickering around, Tom, +you can bet there's a ghost mighty close behind it. It stands to +reason. Becuz you know that they don't anybody but ghosts use 'em." + +"Yes, that's so. But anyway they don't come around in the daytime, so +what's the use of our being afeard?" + +"Well, all right. We'll tackle the ha'nted house if you say so--but I +reckon it's taking chances." + +They had started down the hill by this time. There in the middle of +the moonlit valley below them stood the "ha'nted" house, utterly +isolated, its fences gone long ago, rank weeds smothering the very +doorsteps, the chimney crumbled to ruin, the window-sashes vacant, a +corner of the roof caved in. The boys gazed awhile, half expecting to +see a blue light flit past a window; then talking in a low tone, as +befitted the time and the circumstances, they struck far off to the +right, to give the haunted house a wide berth, and took their way +homeward through the woods that adorned the rearward side of Cardiff +Hill. + + + +CHAPTER XXVI + +ABOUT noon the next day the boys arrived at the dead tree; they had +come for their tools. Tom was impatient to go to the haunted house; +Huck was measurably so, also--but suddenly said: + +"Lookyhere, Tom, do you know what day it is?" + +Tom mentally ran over the days of the week, and then quickly lifted +his eyes with a startled look in them-- + +"My! I never once thought of it, Huck!" + +"Well, I didn't neither, but all at once it popped onto me that it was +Friday." + +"Blame it, a body can't be too careful, Huck. We might 'a' got into an +awful scrape, tackling such a thing on a Friday." + +"MIGHT! Better say we WOULD! There's some lucky days, maybe, but +Friday ain't." + +"Any fool knows that. I don't reckon YOU was the first that found it +out, Huck." + +"Well, I never said I was, did I? And Friday ain't all, neither. I had +a rotten bad dream last night--dreampt about rats." + +"No! Sure sign of trouble. Did they fight?" + +"No." + +"Well, that's good, Huck. When they don't fight it's only a sign that +there's trouble around, you know. All we got to do is to look mighty +sharp and keep out of it. We'll drop this thing for to-day, and play. +Do you know Robin Hood, Huck?" + +"No. Who's Robin Hood?" + +"Why, he was one of the greatest men that was ever in England--and the +best. He was a robber." + +"Cracky, I wisht I was. Who did he rob?" + +"Only sheriffs and bishops and rich people and kings, and such like. +But he never bothered the poor. He loved 'em. He always divided up with +'em perfectly square." + +"Well, he must 'a' been a brick." + +"I bet you he was, Huck. Oh, he was the noblest man that ever was. +They ain't any such men now, I can tell you. He could lick any man in +England, with one hand tied behind him; and he could take his yew bow +and plug a ten-cent piece every time, a mile and a half." + +"What's a YEW bow?" + +"I don't know. It's some kind of a bow, of course. And if he hit that +dime only on the edge he would set down and cry--and curse. But we'll +play Robin Hood--it's nobby fun. I'll learn you." + +"I'm agreed." + +So they played Robin Hood all the afternoon, now and then casting a +yearning eye down upon the haunted house and passing a remark about the +morrow's prospects and possibilities there. As the sun began to sink +into the west they took their way homeward athwart the long shadows of +the trees and soon were buried from sight in the forests of Cardiff +Hill. + +On Saturday, shortly after noon, the boys were at the dead tree again. +They had a smoke and a chat in the shade, and then dug a little in +their last hole, not with great hope, but merely because Tom said there +were so many cases where people had given up a treasure after getting +down within six inches of it, and then somebody else had come along and +turned it up with a single thrust of a shovel. The thing failed this +time, however, so the boys shouldered their tools and went away feeling +that they had not trifled with fortune, but had fulfilled all the +requirements that belong to the business of treasure-hunting. + +When they reached the haunted house there was something so weird and +grisly about the dead silence that reigned there under the baking sun, +and something so depressing about the loneliness and desolation of the +place, that they were afraid, for a moment, to venture in. Then they +crept to the door and took a trembling peep. They saw a weed-grown, +floorless room, unplastered, an ancient fireplace, vacant windows, a +ruinous staircase; and here, there, and everywhere hung ragged and +abandoned cobwebs. They presently entered, softly, with quickened +pulses, talking in whispers, ears alert to catch the slightest sound, +and muscles tense and ready for instant retreat. + +In a little while familiarity modified their fears and they gave the +place a critical and interested examination, rather admiring their own +boldness, and wondering at it, too. Next they wanted to look up-stairs. +This was something like cutting off retreat, but they got to daring +each other, and of course there could be but one result--they threw +their tools into a corner and made the ascent. Up there were the same +signs of decay. In one corner they found a closet that promised +mystery, but the promise was a fraud--there was nothing in it. Their +courage was up now and well in hand. They were about to go down and +begin work when-- + +"Sh!" said Tom. + +"What is it?" whispered Huck, blanching with fright. + +"Sh!... There!... Hear it?" + +"Yes!... Oh, my! Let's run!" + +"Keep still! Don't you budge! They're coming right toward the door." + +The boys stretched themselves upon the floor with their eyes to +knot-holes in the planking, and lay waiting, in a misery of fear. + +"They've stopped.... No--coming.... Here they are. Don't whisper +another word, Huck. My goodness, I wish I was out of this!" + +Two men entered. Each boy said to himself: "There's the old deaf and +dumb Spaniard that's been about town once or twice lately--never saw +t'other man before." + +"T'other" was a ragged, unkempt creature, with nothing very pleasant +in his face. The Spaniard was wrapped in a serape; he had bushy white +whiskers; long white hair flowed from under his sombrero, and he wore +green goggles. When they came in, "t'other" was talking in a low voice; +they sat down on the ground, facing the door, with their backs to the +wall, and the speaker continued his remarks. His manner became less +guarded and his words more distinct as he proceeded: + +"No," said he, "I've thought it all over, and I don't like it. It's +dangerous." + +"Dangerous!" grunted the "deaf and dumb" Spaniard--to the vast +surprise of the boys. "Milksop!" + +This voice made the boys gasp and quake. It was Injun Joe's! There was +silence for some time. Then Joe said: + +"What's any more dangerous than that job up yonder--but nothing's come +of it." + +"That's different. Away up the river so, and not another house about. +'Twon't ever be known that we tried, anyway, long as we didn't succeed." + +"Well, what's more dangerous than coming here in the daytime!--anybody +would suspicion us that saw us." + +"I know that. But there warn't any other place as handy after that +fool of a job. I want to quit this shanty. I wanted to yesterday, only +it warn't any use trying to stir out of here, with those infernal boys +playing over there on the hill right in full view." + +"Those infernal boys" quaked again under the inspiration of this +remark, and thought how lucky it was that they had remembered it was +Friday and concluded to wait a day. They wished in their hearts they +had waited a year. + +The two men got out some food and made a luncheon. After a long and +thoughtful silence, Injun Joe said: + +"Look here, lad--you go back up the river where you belong. Wait there +till you hear from me. I'll take the chances on dropping into this town +just once more, for a look. We'll do that 'dangerous' job after I've +spied around a little and think things look well for it. Then for +Texas! We'll leg it together!" + +This was satisfactory. Both men presently fell to yawning, and Injun +Joe said: + +"I'm dead for sleep! It's your turn to watch." + +He curled down in the weeds and soon began to snore. His comrade +stirred him once or twice and he became quiet. Presently the watcher +began to nod; his head drooped lower and lower, both men began to snore +now. + +The boys drew a long, grateful breath. Tom whispered: + +"Now's our chance--come!" + +Huck said: + +"I can't--I'd die if they was to wake." + +Tom urged--Huck held back. At last Tom rose slowly and softly, and +started alone. But the first step he made wrung such a hideous creak +from the crazy floor that he sank down almost dead with fright. He +never made a second attempt. The boys lay there counting the dragging +moments till it seemed to them that time must be done and eternity +growing gray; and then they were grateful to note that at last the sun +was setting. + +Now one snore ceased. Injun Joe sat up, stared around--smiled grimly +upon his comrade, whose head was drooping upon his knees--stirred him +up with his foot and said: + +"Here! YOU'RE a watchman, ain't you! All right, though--nothing's +happened." + +"My! have I been asleep?" + +"Oh, partly, partly. Nearly time for us to be moving, pard. What'll we +do with what little swag we've got left?" + +"I don't know--leave it here as we've always done, I reckon. No use to +take it away till we start south. Six hundred and fifty in silver's +something to carry." + +"Well--all right--it won't matter to come here once more." + +"No--but I'd say come in the night as we used to do--it's better." + +"Yes: but look here; it may be a good while before I get the right +chance at that job; accidents might happen; 'tain't in such a very good +place; we'll just regularly bury it--and bury it deep." + +"Good idea," said the comrade, who walked across the room, knelt down, +raised one of the rearward hearth-stones and took out a bag that +jingled pleasantly. He subtracted from it twenty or thirty dollars for +himself and as much for Injun Joe, and passed the bag to the latter, +who was on his knees in the corner, now, digging with his bowie-knife. + +The boys forgot all their fears, all their miseries in an instant. +With gloating eyes they watched every movement. Luck!--the splendor of +it was beyond all imagination! Six hundred dollars was money enough to +make half a dozen boys rich! Here was treasure-hunting under the +happiest auspices--there would not be any bothersome uncertainty as to +where to dig. They nudged each other every moment--eloquent nudges and +easily understood, for they simply meant--"Oh, but ain't you glad NOW +we're here!" + +Joe's knife struck upon something. + +"Hello!" said he. + +"What is it?" said his comrade. + +"Half-rotten plank--no, it's a box, I believe. Here--bear a hand and +we'll see what it's here for. Never mind, I've broke a hole." + +He reached his hand in and drew it out-- + +"Man, it's money!" + +The two men examined the handful of coins. They were gold. The boys +above were as excited as themselves, and as delighted. + +Joe's comrade said: + +"We'll make quick work of this. There's an old rusty pick over amongst +the weeds in the corner the other side of the fireplace--I saw it a +minute ago." + +He ran and brought the boys' pick and shovel. Injun Joe took the pick, +looked it over critically, shook his head, muttered something to +himself, and then began to use it. The box was soon unearthed. It was +not very large; it was iron bound and had been very strong before the +slow years had injured it. The men contemplated the treasure awhile in +blissful silence. + +"Pard, there's thousands of dollars here," said Injun Joe. + +"'Twas always said that Murrel's gang used to be around here one +summer," the stranger observed. + +"I know it," said Injun Joe; "and this looks like it, I should say." + +"Now you won't need to do that job." + +The half-breed frowned. Said he: + +"You don't know me. Least you don't know all about that thing. 'Tain't +robbery altogether--it's REVENGE!" and a wicked light flamed in his +eyes. "I'll need your help in it. When it's finished--then Texas. Go +home to your Nance and your kids, and stand by till you hear from me." + +"Well--if you say so; what'll we do with this--bury it again?" + +"Yes. [Ravishing delight overhead.] NO! by the great Sachem, no! +[Profound distress overhead.] I'd nearly forgot. That pick had fresh +earth on it! [The boys were sick with terror in a moment.] What +business has a pick and a shovel here? What business with fresh earth +on them? Who brought them here--and where are they gone? Have you heard +anybody?--seen anybody? What! bury it again and leave them to come and +see the ground disturbed? Not exactly--not exactly. We'll take it to my +den." + +"Why, of course! Might have thought of that before. You mean Number +One?" + +"No--Number Two--under the cross. The other place is bad--too common." + +"All right. It's nearly dark enough to start." + +Injun Joe got up and went about from window to window cautiously +peeping out. Presently he said: + +"Who could have brought those tools here? Do you reckon they can be +up-stairs?" + +The boys' breath forsook them. Injun Joe put his hand on his knife, +halted a moment, undecided, and then turned toward the stairway. The +boys thought of the closet, but their strength was gone. The steps came +creaking up the stairs--the intolerable distress of the situation woke +the stricken resolution of the lads--they were about to spring for the +closet, when there was a crash of rotten timbers and Injun Joe landed +on the ground amid the debris of the ruined stairway. He gathered +himself up cursing, and his comrade said: + +"Now what's the use of all that? If it's anybody, and they're up +there, let them STAY there--who cares? If they want to jump down, now, +and get into trouble, who objects? It will be dark in fifteen minutes +--and then let them follow us if they want to. I'm willing. In my +opinion, whoever hove those things in here caught a sight of us and +took us for ghosts or devils or something. I'll bet they're running +yet." + +Joe grumbled awhile; then he agreed with his friend that what daylight +was left ought to be economized in getting things ready for leaving. +Shortly afterward they slipped out of the house in the deepening +twilight, and moved toward the river with their precious box. + +Tom and Huck rose up, weak but vastly relieved, and stared after them +through the chinks between the logs of the house. Follow? Not they. +They were content to reach ground again without broken necks, and take +the townward track over the hill. They did not talk much. They were too +much absorbed in hating themselves--hating the ill luck that made them +take the spade and the pick there. But for that, Injun Joe never would +have suspected. He would have hidden the silver with the gold to wait +there till his "revenge" was satisfied, and then he would have had the +misfortune to find that money turn up missing. Bitter, bitter luck that +the tools were ever brought there! + +They resolved to keep a lookout for that Spaniard when he should come +to town spying out for chances to do his revengeful job, and follow him +to "Number Two," wherever that might be. Then a ghastly thought +occurred to Tom. + +"Revenge? What if he means US, Huck!" + +"Oh, don't!" said Huck, nearly fainting. + +They talked it all over, and as they entered town they agreed to +believe that he might possibly mean somebody else--at least that he +might at least mean nobody but Tom, since only Tom had testified. + +Very, very small comfort it was to Tom to be alone in danger! Company +would be a palpable improvement, he thought. + + + +CHAPTER XXVII + +THE adventure of the day mightily tormented Tom's dreams that night. +Four times he had his hands on that rich treasure and four times it +wasted to nothingness in his fingers as sleep forsook him and +wakefulness brought back the hard reality of his misfortune. As he lay +in the early morning recalling the incidents of his great adventure, he +noticed that they seemed curiously subdued and far away--somewhat as if +they had happened in another world, or in a time long gone by. Then it +occurred to him that the great adventure itself must be a dream! There +was one very strong argument in favor of this idea--namely, that the +quantity of coin he had seen was too vast to be real. He had never seen +as much as fifty dollars in one mass before, and he was like all boys +of his age and station in life, in that he imagined that all references +to "hundreds" and "thousands" were mere fanciful forms of speech, and +that no such sums really existed in the world. He never had supposed +for a moment that so large a sum as a hundred dollars was to be found +in actual money in any one's possession. If his notions of hidden +treasure had been analyzed, they would have been found to consist of a +handful of real dimes and a bushel of vague, splendid, ungraspable +dollars. + +But the incidents of his adventure grew sensibly sharper and clearer +under the attrition of thinking them over, and so he presently found +himself leaning to the impression that the thing might not have been a +dream, after all. This uncertainty must be swept away. He would snatch +a hurried breakfast and go and find Huck. Huck was sitting on the +gunwale of a flatboat, listlessly dangling his feet in the water and +looking very melancholy. Tom concluded to let Huck lead up to the +subject. If he did not do it, then the adventure would be proved to +have been only a dream. + +"Hello, Huck!" + +"Hello, yourself." + +Silence, for a minute. + +"Tom, if we'd 'a' left the blame tools at the dead tree, we'd 'a' got +the money. Oh, ain't it awful!" + +"'Tain't a dream, then, 'tain't a dream! Somehow I most wish it was. +Dog'd if I don't, Huck." + +"What ain't a dream?" + +"Oh, that thing yesterday. I been half thinking it was." + +"Dream! If them stairs hadn't broke down you'd 'a' seen how much dream +it was! I've had dreams enough all night--with that patch-eyed Spanish +devil going for me all through 'em--rot him!" + +"No, not rot him. FIND him! Track the money!" + +"Tom, we'll never find him. A feller don't have only one chance for +such a pile--and that one's lost. I'd feel mighty shaky if I was to see +him, anyway." + +"Well, so'd I; but I'd like to see him, anyway--and track him out--to +his Number Two." + +"Number Two--yes, that's it. I been thinking 'bout that. But I can't +make nothing out of it. What do you reckon it is?" + +"I dono. It's too deep. Say, Huck--maybe it's the number of a house!" + +"Goody!... No, Tom, that ain't it. If it is, it ain't in this +one-horse town. They ain't no numbers here." + +"Well, that's so. Lemme think a minute. Here--it's the number of a +room--in a tavern, you know!" + +"Oh, that's the trick! They ain't only two taverns. We can find out +quick." + +"You stay here, Huck, till I come." + +Tom was off at once. He did not care to have Huck's company in public +places. He was gone half an hour. He found that in the best tavern, No. +2 had long been occupied by a young lawyer, and was still so occupied. +In the less ostentatious house, No. 2 was a mystery. The +tavern-keeper's young son said it was kept locked all the time, and he +never saw anybody go into it or come out of it except at night; he did +not know any particular reason for this state of things; had had some +little curiosity, but it was rather feeble; had made the most of the +mystery by entertaining himself with the idea that that room was +"ha'nted"; had noticed that there was a light in there the night before. + +"That's what I've found out, Huck. I reckon that's the very No. 2 +we're after." + +"I reckon it is, Tom. Now what you going to do?" + +"Lemme think." + +Tom thought a long time. Then he said: + +"I'll tell you. The back door of that No. 2 is the door that comes out +into that little close alley between the tavern and the old rattle trap +of a brick store. Now you get hold of all the door-keys you can find, +and I'll nip all of auntie's, and the first dark night we'll go there +and try 'em. And mind you, keep a lookout for Injun Joe, because he +said he was going to drop into town and spy around once more for a +chance to get his revenge. If you see him, you just follow him; and if +he don't go to that No. 2, that ain't the place." + +"Lordy, I don't want to foller him by myself!" + +"Why, it'll be night, sure. He mightn't ever see you--and if he did, +maybe he'd never think anything." + +"Well, if it's pretty dark I reckon I'll track him. I dono--I dono. +I'll try." + +"You bet I'll follow him, if it's dark, Huck. Why, he might 'a' found +out he couldn't get his revenge, and be going right after that money." + +"It's so, Tom, it's so. I'll foller him; I will, by jingoes!" + +"Now you're TALKING! Don't you ever weaken, Huck, and I won't." + + + +CHAPTER XXVIII + +THAT night Tom and Huck were ready for their adventure. They hung +about the neighborhood of the tavern until after nine, one watching the +alley at a distance and the other the tavern door. Nobody entered the +alley or left it; nobody resembling the Spaniard entered or left the +tavern door. The night promised to be a fair one; so Tom went home with +the understanding that if a considerable degree of darkness came on, +Huck was to come and "maow," whereupon he would slip out and try the +keys. But the night remained clear, and Huck closed his watch and +retired to bed in an empty sugar hogshead about twelve. + +Tuesday the boys had the same ill luck. Also Wednesday. But Thursday +night promised better. Tom slipped out in good season with his aunt's +old tin lantern, and a large towel to blindfold it with. He hid the +lantern in Huck's sugar hogshead and the watch began. An hour before +midnight the tavern closed up and its lights (the only ones +thereabouts) were put out. No Spaniard had been seen. Nobody had +entered or left the alley. Everything was auspicious. The blackness of +darkness reigned, the perfect stillness was interrupted only by +occasional mutterings of distant thunder. + +Tom got his lantern, lit it in the hogshead, wrapped it closely in the +towel, and the two adventurers crept in the gloom toward the tavern. +Huck stood sentry and Tom felt his way into the alley. Then there was a +season of waiting anxiety that weighed upon Huck's spirits like a +mountain. He began to wish he could see a flash from the lantern--it +would frighten him, but it would at least tell him that Tom was alive +yet. It seemed hours since Tom had disappeared. Surely he must have +fainted; maybe he was dead; maybe his heart had burst under terror and +excitement. In his uneasiness Huck found himself drawing closer and +closer to the alley; fearing all sorts of dreadful things, and +momentarily expecting some catastrophe to happen that would take away +his breath. There was not much to take away, for he seemed only able to +inhale it by thimblefuls, and his heart would soon wear itself out, the +way it was beating. Suddenly there was a flash of light and Tom came +tearing by him: "Run!" said he; "run, for your life!" + +He needn't have repeated it; once was enough; Huck was making thirty +or forty miles an hour before the repetition was uttered. The boys +never stopped till they reached the shed of a deserted slaughter-house +at the lower end of the village. Just as they got within its shelter +the storm burst and the rain poured down. As soon as Tom got his breath +he said: + +"Huck, it was awful! I tried two of the keys, just as soft as I could; +but they seemed to make such a power of racket that I couldn't hardly +get my breath I was so scared. They wouldn't turn in the lock, either. +Well, without noticing what I was doing, I took hold of the knob, and +open comes the door! It warn't locked! I hopped in, and shook off the +towel, and, GREAT CAESAR'S GHOST!" + +"What!--what'd you see, Tom?" + +"Huck, I most stepped onto Injun Joe's hand!" + +"No!" + +"Yes! He was lying there, sound asleep on the floor, with his old +patch on his eye and his arms spread out." + +"Lordy, what did you do? Did he wake up?" + +"No, never budged. Drunk, I reckon. I just grabbed that towel and +started!" + +"I'd never 'a' thought of the towel, I bet!" + +"Well, I would. My aunt would make me mighty sick if I lost it." + +"Say, Tom, did you see that box?" + +"Huck, I didn't wait to look around. I didn't see the box, I didn't +see the cross. I didn't see anything but a bottle and a tin cup on the +floor by Injun Joe; yes, I saw two barrels and lots more bottles in the +room. Don't you see, now, what's the matter with that ha'nted room?" + +"How?" + +"Why, it's ha'nted with whiskey! Maybe ALL the Temperance Taverns have +got a ha'nted room, hey, Huck?" + +"Well, I reckon maybe that's so. Who'd 'a' thought such a thing? But +say, Tom, now's a mighty good time to get that box, if Injun Joe's +drunk." + +"It is, that! You try it!" + +Huck shuddered. + +"Well, no--I reckon not." + +"And I reckon not, Huck. Only one bottle alongside of Injun Joe ain't +enough. If there'd been three, he'd be drunk enough and I'd do it." + +There was a long pause for reflection, and then Tom said: + +"Lookyhere, Huck, less not try that thing any more till we know Injun +Joe's not in there. It's too scary. Now, if we watch every night, we'll +be dead sure to see him go out, some time or other, and then we'll +snatch that box quicker'n lightning." + +"Well, I'm agreed. I'll watch the whole night long, and I'll do it +every night, too, if you'll do the other part of the job." + +"All right, I will. All you got to do is to trot up Hooper Street a +block and maow--and if I'm asleep, you throw some gravel at the window +and that'll fetch me." + +"Agreed, and good as wheat!" + +"Now, Huck, the storm's over, and I'll go home. It'll begin to be +daylight in a couple of hours. You go back and watch that long, will +you?" + +"I said I would, Tom, and I will. I'll ha'nt that tavern every night +for a year! I'll sleep all day and I'll stand watch all night." + +"That's all right. Now, where you going to sleep?" + +"In Ben Rogers' hayloft. He lets me, and so does his pap's nigger man, +Uncle Jake. I tote water for Uncle Jake whenever he wants me to, and +any time I ask him he gives me a little something to eat if he can +spare it. That's a mighty good nigger, Tom. He likes me, becuz I don't +ever act as if I was above him. Sometime I've set right down and eat +WITH him. But you needn't tell that. A body's got to do things when +he's awful hungry he wouldn't want to do as a steady thing." + +"Well, if I don't want you in the daytime, I'll let you sleep. I won't +come bothering around. Any time you see something's up, in the night, +just skip right around and maow." + + + +CHAPTER XXIX + +THE first thing Tom heard on Friday morning was a glad piece of news +--Judge Thatcher's family had come back to town the night before. Both +Injun Joe and the treasure sunk into secondary importance for a moment, +and Becky took the chief place in the boy's interest. He saw her and +they had an exhausting good time playing "hi-spy" and "gully-keeper" +with a crowd of their school-mates. The day was completed and crowned +in a peculiarly satisfactory way: Becky teased her mother to appoint +the next day for the long-promised and long-delayed picnic, and she +consented. The child's delight was boundless; and Tom's not more +moderate. The invitations were sent out before sunset, and straightway +the young folks of the village were thrown into a fever of preparation +and pleasurable anticipation. Tom's excitement enabled him to keep +awake until a pretty late hour, and he had good hopes of hearing Huck's +"maow," and of having his treasure to astonish Becky and the picnickers +with, next day; but he was disappointed. No signal came that night. + +Morning came, eventually, and by ten or eleven o'clock a giddy and +rollicking company were gathered at Judge Thatcher's, and everything +was ready for a start. It was not the custom for elderly people to mar +the picnics with their presence. The children were considered safe +enough under the wings of a few young ladies of eighteen and a few +young gentlemen of twenty-three or thereabouts. The old steam ferryboat +was chartered for the occasion; presently the gay throng filed up the +main street laden with provision-baskets. Sid was sick and had to miss +the fun; Mary remained at home to entertain him. The last thing Mrs. +Thatcher said to Becky, was: + +"You'll not get back till late. Perhaps you'd better stay all night +with some of the girls that live near the ferry-landing, child." + +"Then I'll stay with Susy Harper, mamma." + +"Very well. And mind and behave yourself and don't be any trouble." + +Presently, as they tripped along, Tom said to Becky: + +"Say--I'll tell you what we'll do. 'Stead of going to Joe Harper's +we'll climb right up the hill and stop at the Widow Douglas'. She'll +have ice-cream! She has it most every day--dead loads of it. And she'll +be awful glad to have us." + +"Oh, that will be fun!" + +Then Becky reflected a moment and said: + +"But what will mamma say?" + +"How'll she ever know?" + +The girl turned the idea over in her mind, and said reluctantly: + +"I reckon it's wrong--but--" + +"But shucks! Your mother won't know, and so what's the harm? All she +wants is that you'll be safe; and I bet you she'd 'a' said go there if +she'd 'a' thought of it. I know she would!" + +The Widow Douglas' splendid hospitality was a tempting bait. It and +Tom's persuasions presently carried the day. So it was decided to say +nothing anybody about the night's programme. Presently it occurred to +Tom that maybe Huck might come this very night and give the signal. The +thought took a deal of the spirit out of his anticipations. Still he +could not bear to give up the fun at Widow Douglas'. And why should he +give it up, he reasoned--the signal did not come the night before, so +why should it be any more likely to come to-night? The sure fun of the +evening outweighed the uncertain treasure; and, boy-like, he determined +to yield to the stronger inclination and not allow himself to think of +the box of money another time that day. + +Three miles below town the ferryboat stopped at the mouth of a woody +hollow and tied up. The crowd swarmed ashore and soon the forest +distances and craggy heights echoed far and near with shoutings and +laughter. All the different ways of getting hot and tired were gone +through with, and by-and-by the rovers straggled back to camp fortified +with responsible appetites, and then the destruction of the good things +began. After the feast there was a refreshing season of rest and chat +in the shade of spreading oaks. By-and-by somebody shouted: + +"Who's ready for the cave?" + +Everybody was. Bundles of candles were procured, and straightway there +was a general scamper up the hill. The mouth of the cave was up the +hillside--an opening shaped like a letter A. Its massive oaken door +stood unbarred. Within was a small chamber, chilly as an ice-house, and +walled by Nature with solid limestone that was dewy with a cold sweat. +It was romantic and mysterious to stand here in the deep gloom and look +out upon the green valley shining in the sun. But the impressiveness of +the situation quickly wore off, and the romping began again. The moment +a candle was lighted there was a general rush upon the owner of it; a +struggle and a gallant defence followed, but the candle was soon +knocked down or blown out, and then there was a glad clamor of laughter +and a new chase. But all things have an end. By-and-by the procession +went filing down the steep descent of the main avenue, the flickering +rank of lights dimly revealing the lofty walls of rock almost to their +point of junction sixty feet overhead. This main avenue was not more +than eight or ten feet wide. Every few steps other lofty and still +narrower crevices branched from it on either hand--for McDougal's cave +was but a vast labyrinth of crooked aisles that ran into each other and +out again and led nowhere. It was said that one might wander days and +nights together through its intricate tangle of rifts and chasms, and +never find the end of the cave; and that he might go down, and down, +and still down, into the earth, and it was just the same--labyrinth +under labyrinth, and no end to any of them. No man "knew" the cave. +That was an impossible thing. Most of the young men knew a portion of +it, and it was not customary to venture much beyond this known portion. +Tom Sawyer knew as much of the cave as any one. + +The procession moved along the main avenue some three-quarters of a +mile, and then groups and couples began to slip aside into branch +avenues, fly along the dismal corridors, and take each other by +surprise at points where the corridors joined again. Parties were able +to elude each other for the space of half an hour without going beyond +the "known" ground. + +By-and-by, one group after another came straggling back to the mouth +of the cave, panting, hilarious, smeared from head to foot with tallow +drippings, daubed with clay, and entirely delighted with the success of +the day. Then they were astonished to find that they had been taking no +note of time and that night was about at hand. The clanging bell had +been calling for half an hour. However, this sort of close to the day's +adventures was romantic and therefore satisfactory. When the ferryboat +with her wild freight pushed into the stream, nobody cared sixpence for +the wasted time but the captain of the craft. + +Huck was already upon his watch when the ferryboat's lights went +glinting past the wharf. He heard no noise on board, for the young +people were as subdued and still as people usually are who are nearly +tired to death. He wondered what boat it was, and why she did not stop +at the wharf--and then he dropped her out of his mind and put his +attention upon his business. The night was growing cloudy and dark. Ten +o'clock came, and the noise of vehicles ceased, scattered lights began +to wink out, all straggling foot-passengers disappeared, the village +betook itself to its slumbers and left the small watcher alone with the +silence and the ghosts. Eleven o'clock came, and the tavern lights were +put out; darkness everywhere, now. Huck waited what seemed a weary long +time, but nothing happened. His faith was weakening. Was there any use? +Was there really any use? Why not give it up and turn in? + +A noise fell upon his ear. He was all attention in an instant. The +alley door closed softly. He sprang to the corner of the brick store. +The next moment two men brushed by him, and one seemed to have +something under his arm. It must be that box! So they were going to +remove the treasure. Why call Tom now? It would be absurd--the men +would get away with the box and never be found again. No, he would +stick to their wake and follow them; he would trust to the darkness for +security from discovery. So communing with himself, Huck stepped out +and glided along behind the men, cat-like, with bare feet, allowing +them to keep just far enough ahead not to be invisible. + +They moved up the river street three blocks, then turned to the left +up a cross-street. They went straight ahead, then, until they came to +the path that led up Cardiff Hill; this they took. They passed by the +old Welshman's house, half-way up the hill, without hesitating, and +still climbed upward. Good, thought Huck, they will bury it in the old +quarry. But they never stopped at the quarry. They passed on, up the +summit. They plunged into the narrow path between the tall sumach +bushes, and were at once hidden in the gloom. Huck closed up and +shortened his distance, now, for they would never be able to see him. +He trotted along awhile; then slackened his pace, fearing he was +gaining too fast; moved on a piece, then stopped altogether; listened; +no sound; none, save that he seemed to hear the beating of his own +heart. The hooting of an owl came over the hill--ominous sound! But no +footsteps. Heavens, was everything lost! He was about to spring with +winged feet, when a man cleared his throat not four feet from him! +Huck's heart shot into his throat, but he swallowed it again; and then +he stood there shaking as if a dozen agues had taken charge of him at +once, and so weak that he thought he must surely fall to the ground. He +knew where he was. He knew he was within five steps of the stile +leading into Widow Douglas' grounds. Very well, he thought, let them +bury it there; it won't be hard to find. + +Now there was a voice--a very low voice--Injun Joe's: + +"Damn her, maybe she's got company--there's lights, late as it is." + +"I can't see any." + +This was that stranger's voice--the stranger of the haunted house. A +deadly chill went to Huck's heart--this, then, was the "revenge" job! +His thought was, to fly. Then he remembered that the Widow Douglas had +been kind to him more than once, and maybe these men were going to +murder her. He wished he dared venture to warn her; but he knew he +didn't dare--they might come and catch him. He thought all this and +more in the moment that elapsed between the stranger's remark and Injun +Joe's next--which was-- + +"Because the bush is in your way. Now--this way--now you see, don't +you?" + +"Yes. Well, there IS company there, I reckon. Better give it up." + +"Give it up, and I just leaving this country forever! Give it up and +maybe never have another chance. I tell you again, as I've told you +before, I don't care for her swag--you may have it. But her husband was +rough on me--many times he was rough on me--and mainly he was the +justice of the peace that jugged me for a vagrant. And that ain't all. +It ain't a millionth part of it! He had me HORSEWHIPPED!--horsewhipped +in front of the jail, like a nigger!--with all the town looking on! +HORSEWHIPPED!--do you understand? He took advantage of me and died. But +I'll take it out of HER." + +"Oh, don't kill her! Don't do that!" + +"Kill? Who said anything about killing? I would kill HIM if he was +here; but not her. When you want to get revenge on a woman you don't +kill her--bosh! you go for her looks. You slit her nostrils--you notch +her ears like a sow!" + +"By God, that's--" + +"Keep your opinion to yourself! It will be safest for you. I'll tie +her to the bed. If she bleeds to death, is that my fault? I'll not cry, +if she does. My friend, you'll help me in this thing--for MY sake +--that's why you're here--I mightn't be able alone. If you flinch, I'll +kill you. Do you understand that? And if I have to kill you, I'll kill +her--and then I reckon nobody'll ever know much about who done this +business." + +"Well, if it's got to be done, let's get at it. The quicker the +better--I'm all in a shiver." + +"Do it NOW? And company there? Look here--I'll get suspicious of you, +first thing you know. No--we'll wait till the lights are out--there's +no hurry." + +Huck felt that a silence was going to ensue--a thing still more awful +than any amount of murderous talk; so he held his breath and stepped +gingerly back; planted his foot carefully and firmly, after balancing, +one-legged, in a precarious way and almost toppling over, first on one +side and then on the other. He took another step back, with the same +elaboration and the same risks; then another and another, and--a twig +snapped under his foot! His breath stopped and he listened. There was +no sound--the stillness was perfect. His gratitude was measureless. Now +he turned in his tracks, between the walls of sumach bushes--turned +himself as carefully as if he were a ship--and then stepped quickly but +cautiously along. When he emerged at the quarry he felt secure, and so +he picked up his nimble heels and flew. Down, down he sped, till he +reached the Welshman's. He banged at the door, and presently the heads +of the old man and his two stalwart sons were thrust from windows. + +"What's the row there? Who's banging? What do you want?" + +"Let me in--quick! I'll tell everything." + +"Why, who are you?" + +"Huckleberry Finn--quick, let me in!" + +"Huckleberry Finn, indeed! It ain't a name to open many doors, I +judge! But let him in, lads, and let's see what's the trouble." + +"Please don't ever tell I told you," were Huck's first words when he +got in. "Please don't--I'd be killed, sure--but the widow's been good +friends to me sometimes, and I want to tell--I WILL tell if you'll +promise you won't ever say it was me." + +"By George, he HAS got something to tell, or he wouldn't act so!" +exclaimed the old man; "out with it and nobody here'll ever tell, lad." + +Three minutes later the old man and his sons, well armed, were up the +hill, and just entering the sumach path on tiptoe, their weapons in +their hands. Huck accompanied them no further. He hid behind a great +bowlder and fell to listening. There was a lagging, anxious silence, +and then all of a sudden there was an explosion of firearms and a cry. + +Huck waited for no particulars. He sprang away and sped down the hill +as fast as his legs could carry him. + + + +CHAPTER XXX + +AS the earliest suspicion of dawn appeared on Sunday morning, Huck +came groping up the hill and rapped gently at the old Welshman's door. +The inmates were asleep, but it was a sleep that was set on a +hair-trigger, on account of the exciting episode of the night. A call +came from a window: + +"Who's there!" + +Huck's scared voice answered in a low tone: + +"Please let me in! It's only Huck Finn!" + +"It's a name that can open this door night or day, lad!--and welcome!" + +These were strange words to the vagabond boy's ears, and the +pleasantest he had ever heard. He could not recollect that the closing +word had ever been applied in his case before. The door was quickly +unlocked, and he entered. Huck was given a seat and the old man and his +brace of tall sons speedily dressed themselves. + +"Now, my boy, I hope you're good and hungry, because breakfast will be +ready as soon as the sun's up, and we'll have a piping hot one, too +--make yourself easy about that! I and the boys hoped you'd turn up and +stop here last night." + +"I was awful scared," said Huck, "and I run. I took out when the +pistols went off, and I didn't stop for three mile. I've come now becuz +I wanted to know about it, you know; and I come before daylight becuz I +didn't want to run across them devils, even if they was dead." + +"Well, poor chap, you do look as if you'd had a hard night of it--but +there's a bed here for you when you've had your breakfast. No, they +ain't dead, lad--we are sorry enough for that. You see we knew right +where to put our hands on them, by your description; so we crept along +on tiptoe till we got within fifteen feet of them--dark as a cellar +that sumach path was--and just then I found I was going to sneeze. It +was the meanest kind of luck! I tried to keep it back, but no use +--'twas bound to come, and it did come! I was in the lead with my pistol +raised, and when the sneeze started those scoundrels a-rustling to get +out of the path, I sung out, 'Fire boys!' and blazed away at the place +where the rustling was. So did the boys. But they were off in a jiffy, +those villains, and we after them, down through the woods. I judge we +never touched them. They fired a shot apiece as they started, but their +bullets whizzed by and didn't do us any harm. As soon as we lost the +sound of their feet we quit chasing, and went down and stirred up the +constables. They got a posse together, and went off to guard the river +bank, and as soon as it is light the sheriff and a gang are going to +beat up the woods. My boys will be with them presently. I wish we had +some sort of description of those rascals--'twould help a good deal. +But you couldn't see what they were like, in the dark, lad, I suppose?" + +"Oh yes; I saw them down-town and follered them." + +"Splendid! Describe them--describe them, my boy!" + +"One's the old deaf and dumb Spaniard that's ben around here once or +twice, and t'other's a mean-looking, ragged--" + +"That's enough, lad, we know the men! Happened on them in the woods +back of the widow's one day, and they slunk away. Off with you, boys, +and tell the sheriff--get your breakfast to-morrow morning!" + +The Welshman's sons departed at once. As they were leaving the room +Huck sprang up and exclaimed: + +"Oh, please don't tell ANYbody it was me that blowed on them! Oh, +please!" + +"All right if you say it, Huck, but you ought to have the credit of +what you did." + +"Oh no, no! Please don't tell!" + +When the young men were gone, the old Welshman said: + +"They won't tell--and I won't. But why don't you want it known?" + +Huck would not explain, further than to say that he already knew too +much about one of those men and would not have the man know that he +knew anything against him for the whole world--he would be killed for +knowing it, sure. + +The old man promised secrecy once more, and said: + +"How did you come to follow these fellows, lad? Were they looking +suspicious?" + +Huck was silent while he framed a duly cautious reply. Then he said: + +"Well, you see, I'm a kind of a hard lot,--least everybody says so, +and I don't see nothing agin it--and sometimes I can't sleep much, on +account of thinking about it and sort of trying to strike out a new way +of doing. That was the way of it last night. I couldn't sleep, and so I +come along up-street 'bout midnight, a-turning it all over, and when I +got to that old shackly brick store by the Temperance Tavern, I backed +up agin the wall to have another think. Well, just then along comes +these two chaps slipping along close by me, with something under their +arm, and I reckoned they'd stole it. One was a-smoking, and t'other one +wanted a light; so they stopped right before me and the cigars lit up +their faces and I see that the big one was the deaf and dumb Spaniard, +by his white whiskers and the patch on his eye, and t'other one was a +rusty, ragged-looking devil." + +"Could you see the rags by the light of the cigars?" + +This staggered Huck for a moment. Then he said: + +"Well, I don't know--but somehow it seems as if I did." + +"Then they went on, and you--" + +"Follered 'em--yes. That was it. I wanted to see what was up--they +sneaked along so. I dogged 'em to the widder's stile, and stood in the +dark and heard the ragged one beg for the widder, and the Spaniard +swear he'd spile her looks just as I told you and your two--" + +"What! The DEAF AND DUMB man said all that!" + +Huck had made another terrible mistake! He was trying his best to keep +the old man from getting the faintest hint of who the Spaniard might +be, and yet his tongue seemed determined to get him into trouble in +spite of all he could do. He made several efforts to creep out of his +scrape, but the old man's eye was upon him and he made blunder after +blunder. Presently the Welshman said: + +"My boy, don't be afraid of me. I wouldn't hurt a hair of your head +for all the world. No--I'd protect you--I'd protect you. This Spaniard +is not deaf and dumb; you've let that slip without intending it; you +can't cover that up now. You know something about that Spaniard that +you want to keep dark. Now trust me--tell me what it is, and trust me +--I won't betray you." + +Huck looked into the old man's honest eyes a moment, then bent over +and whispered in his ear: + +"'Tain't a Spaniard--it's Injun Joe!" + +The Welshman almost jumped out of his chair. In a moment he said: + +"It's all plain enough, now. When you talked about notching ears and +slitting noses I judged that that was your own embellishment, because +white men don't take that sort of revenge. But an Injun! That's a +different matter altogether." + +During breakfast the talk went on, and in the course of it the old man +said that the last thing which he and his sons had done, before going +to bed, was to get a lantern and examine the stile and its vicinity for +marks of blood. They found none, but captured a bulky bundle of-- + +"Of WHAT?" + +If the words had been lightning they could not have leaped with a more +stunning suddenness from Huck's blanched lips. His eyes were staring +wide, now, and his breath suspended--waiting for the answer. The +Welshman started--stared in return--three seconds--five seconds--ten +--then replied: + +"Of burglar's tools. Why, what's the MATTER with you?" + +Huck sank back, panting gently, but deeply, unutterably grateful. The +Welshman eyed him gravely, curiously--and presently said: + +"Yes, burglar's tools. That appears to relieve you a good deal. But +what did give you that turn? What were YOU expecting we'd found?" + +Huck was in a close place--the inquiring eye was upon him--he would +have given anything for material for a plausible answer--nothing +suggested itself--the inquiring eye was boring deeper and deeper--a +senseless reply offered--there was no time to weigh it, so at a venture +he uttered it--feebly: + +"Sunday-school books, maybe." + +Poor Huck was too distressed to smile, but the old man laughed loud +and joyously, shook up the details of his anatomy from head to foot, +and ended by saying that such a laugh was money in a-man's pocket, +because it cut down the doctor's bill like everything. Then he added: + +"Poor old chap, you're white and jaded--you ain't well a bit--no +wonder you're a little flighty and off your balance. But you'll come +out of it. Rest and sleep will fetch you out all right, I hope." + +Huck was irritated to think he had been such a goose and betrayed such +a suspicious excitement, for he had dropped the idea that the parcel +brought from the tavern was the treasure, as soon as he had heard the +talk at the widow's stile. He had only thought it was not the treasure, +however--he had not known that it wasn't--and so the suggestion of a +captured bundle was too much for his self-possession. But on the whole +he felt glad the little episode had happened, for now he knew beyond +all question that that bundle was not THE bundle, and so his mind was +at rest and exceedingly comfortable. In fact, everything seemed to be +drifting just in the right direction, now; the treasure must be still +in No. 2, the men would be captured and jailed that day, and he and Tom +could seize the gold that night without any trouble or any fear of +interruption. + +Just as breakfast was completed there was a knock at the door. Huck +jumped for a hiding-place, for he had no mind to be connected even +remotely with the late event. The Welshman admitted several ladies and +gentlemen, among them the Widow Douglas, and noticed that groups of +citizens were climbing up the hill--to stare at the stile. So the news +had spread. The Welshman had to tell the story of the night to the +visitors. The widow's gratitude for her preservation was outspoken. + +"Don't say a word about it, madam. There's another that you're more +beholden to than you are to me and my boys, maybe, but he don't allow +me to tell his name. We wouldn't have been there but for him." + +Of course this excited a curiosity so vast that it almost belittled +the main matter--but the Welshman allowed it to eat into the vitals of +his visitors, and through them be transmitted to the whole town, for he +refused to part with his secret. When all else had been learned, the +widow said: + +"I went to sleep reading in bed and slept straight through all that +noise. Why didn't you come and wake me?" + +"We judged it warn't worth while. Those fellows warn't likely to come +again--they hadn't any tools left to work with, and what was the use of +waking you up and scaring you to death? My three negro men stood guard +at your house all the rest of the night. They've just come back." + +More visitors came, and the story had to be told and retold for a +couple of hours more. + +There was no Sabbath-school during day-school vacation, but everybody +was early at church. The stirring event was well canvassed. News came +that not a sign of the two villains had been yet discovered. When the +sermon was finished, Judge Thatcher's wife dropped alongside of Mrs. +Harper as she moved down the aisle with the crowd and said: + +"Is my Becky going to sleep all day? I just expected she would be +tired to death." + +"Your Becky?" + +"Yes," with a startled look--"didn't she stay with you last night?" + +"Why, no." + +Mrs. Thatcher turned pale, and sank into a pew, just as Aunt Polly, +talking briskly with a friend, passed by. Aunt Polly said: + +"Good-morning, Mrs. Thatcher. Good-morning, Mrs. Harper. I've got a +boy that's turned up missing. I reckon my Tom stayed at your house last +night--one of you. And now he's afraid to come to church. I've got to +settle with him." + +Mrs. Thatcher shook her head feebly and turned paler than ever. + +"He didn't stay with us," said Mrs. Harper, beginning to look uneasy. +A marked anxiety came into Aunt Polly's face. + +"Joe Harper, have you seen my Tom this morning?" + +"No'm." + +"When did you see him last?" + +Joe tried to remember, but was not sure he could say. The people had +stopped moving out of church. Whispers passed along, and a boding +uneasiness took possession of every countenance. Children were +anxiously questioned, and young teachers. They all said they had not +noticed whether Tom and Becky were on board the ferryboat on the +homeward trip; it was dark; no one thought of inquiring if any one was +missing. One young man finally blurted out his fear that they were +still in the cave! Mrs. Thatcher swooned away. Aunt Polly fell to +crying and wringing her hands. + +The alarm swept from lip to lip, from group to group, from street to +street, and within five minutes the bells were wildly clanging and the +whole town was up! The Cardiff Hill episode sank into instant +insignificance, the burglars were forgotten, horses were saddled, +skiffs were manned, the ferryboat ordered out, and before the horror +was half an hour old, two hundred men were pouring down highroad and +river toward the cave. + +All the long afternoon the village seemed empty and dead. Many women +visited Aunt Polly and Mrs. Thatcher and tried to comfort them. They +cried with them, too, and that was still better than words. All the +tedious night the town waited for news; but when the morning dawned at +last, all the word that came was, "Send more candles--and send food." +Mrs. Thatcher was almost crazed; and Aunt Polly, also. Judge Thatcher +sent messages of hope and encouragement from the cave, but they +conveyed no real cheer. + +The old Welshman came home toward daylight, spattered with +candle-grease, smeared with clay, and almost worn out. He found Huck +still in the bed that had been provided for him, and delirious with +fever. The physicians were all at the cave, so the Widow Douglas came +and took charge of the patient. She said she would do her best by him, +because, whether he was good, bad, or indifferent, he was the Lord's, +and nothing that was the Lord's was a thing to be neglected. The +Welshman said Huck had good spots in him, and the widow said: + +"You can depend on it. That's the Lord's mark. He don't leave it off. +He never does. Puts it somewhere on every creature that comes from his +hands." + +Early in the forenoon parties of jaded men began to straggle into the +village, but the strongest of the citizens continued searching. All the +news that could be gained was that remotenesses of the cavern were +being ransacked that had never been visited before; that every corner +and crevice was going to be thoroughly searched; that wherever one +wandered through the maze of passages, lights were to be seen flitting +hither and thither in the distance, and shoutings and pistol-shots sent +their hollow reverberations to the ear down the sombre aisles. In one +place, far from the section usually traversed by tourists, the names +"BECKY & TOM" had been found traced upon the rocky wall with +candle-smoke, and near at hand a grease-soiled bit of ribbon. Mrs. +Thatcher recognized the ribbon and cried over it. She said it was the +last relic she should ever have of her child; and that no other memorial +of her could ever be so precious, because this one parted latest from +the living body before the awful death came. Some said that now and +then, in the cave, a far-away speck of light would glimmer, and then a +glorious shout would burst forth and a score of men go trooping down the +echoing aisle--and then a sickening disappointment always followed; the +children were not there; it was only a searcher's light. + +Three dreadful days and nights dragged their tedious hours along, and +the village sank into a hopeless stupor. No one had heart for anything. +The accidental discovery, just made, that the proprietor of the +Temperance Tavern kept liquor on his premises, scarcely fluttered the +public pulse, tremendous as the fact was. In a lucid interval, Huck +feebly led up to the subject of taverns, and finally asked--dimly +dreading the worst--if anything had been discovered at the Temperance +Tavern since he had been ill. + +"Yes," said the widow. + +Huck started up in bed, wild-eyed: + +"What? What was it?" + +"Liquor!--and the place has been shut up. Lie down, child--what a turn +you did give me!" + +"Only tell me just one thing--only just one--please! Was it Tom Sawyer +that found it?" + +The widow burst into tears. "Hush, hush, child, hush! I've told you +before, you must NOT talk. You are very, very sick!" + +Then nothing but liquor had been found; there would have been a great +powwow if it had been the gold. So the treasure was gone forever--gone +forever! But what could she be crying about? Curious that she should +cry. + +These thoughts worked their dim way through Huck's mind, and under the +weariness they gave him he fell asleep. The widow said to herself: + +"There--he's asleep, poor wreck. Tom Sawyer find it! Pity but somebody +could find Tom Sawyer! Ah, there ain't many left, now, that's got hope +enough, or strength enough, either, to go on searching." + + + +CHAPTER XXXI + +NOW to return to Tom and Becky's share in the picnic. They tripped +along the murky aisles with the rest of the company, visiting the +familiar wonders of the cave--wonders dubbed with rather +over-descriptive names, such as "The Drawing-Room," "The Cathedral," +"Aladdin's Palace," and so on. Presently the hide-and-seek frolicking +began, and Tom and Becky engaged in it with zeal until the exertion +began to grow a trifle wearisome; then they wandered down a sinuous +avenue holding their candles aloft and reading the tangled web-work of +names, dates, post-office addresses, and mottoes with which the rocky +walls had been frescoed (in candle-smoke). Still drifting along and +talking, they scarcely noticed that they were now in a part of the cave +whose walls were not frescoed. They smoked their own names under an +overhanging shelf and moved on. Presently they came to a place where a +little stream of water, trickling over a ledge and carrying a limestone +sediment with it, had, in the slow-dragging ages, formed a laced and +ruffled Niagara in gleaming and imperishable stone. Tom squeezed his +small body behind it in order to illuminate it for Becky's +gratification. He found that it curtained a sort of steep natural +stairway which was enclosed between narrow walls, and at once the +ambition to be a discoverer seized him. Becky responded to his call, +and they made a smoke-mark for future guidance, and started upon their +quest. They wound this way and that, far down into the secret depths of +the cave, made another mark, and branched off in search of novelties to +tell the upper world about. In one place they found a spacious cavern, +from whose ceiling depended a multitude of shining stalactites of the +length and circumference of a man's leg; they walked all about it, +wondering and admiring, and presently left it by one of the numerous +passages that opened into it. This shortly brought them to a bewitching +spring, whose basin was incrusted with a frostwork of glittering +crystals; it was in the midst of a cavern whose walls were supported by +many fantastic pillars which had been formed by the joining of great +stalactites and stalagmites together, the result of the ceaseless +water-drip of centuries. Under the roof vast knots of bats had packed +themselves together, thousands in a bunch; the lights disturbed the +creatures and they came flocking down by hundreds, squeaking and +darting furiously at the candles. Tom knew their ways and the danger of +this sort of conduct. He seized Becky's hand and hurried her into the +first corridor that offered; and none too soon, for a bat struck +Becky's light out with its wing while she was passing out of the +cavern. The bats chased the children a good distance; but the fugitives +plunged into every new passage that offered, and at last got rid of the +perilous things. Tom found a subterranean lake, shortly, which +stretched its dim length away until its shape was lost in the shadows. +He wanted to explore its borders, but concluded that it would be best +to sit down and rest awhile, first. Now, for the first time, the deep +stillness of the place laid a clammy hand upon the spirits of the +children. Becky said: + +"Why, I didn't notice, but it seems ever so long since I heard any of +the others." + +"Come to think, Becky, we are away down below them--and I don't know +how far away north, or south, or east, or whichever it is. We couldn't +hear them here." + +Becky grew apprehensive. + +"I wonder how long we've been down here, Tom? We better start back." + +"Yes, I reckon we better. P'raps we better." + +"Can you find the way, Tom? It's all a mixed-up crookedness to me." + +"I reckon I could find it--but then the bats. If they put our candles +out it will be an awful fix. Let's try some other way, so as not to go +through there." + +"Well. But I hope we won't get lost. It would be so awful!" and the +girl shuddered at the thought of the dreadful possibilities. + +They started through a corridor, and traversed it in silence a long +way, glancing at each new opening, to see if there was anything +familiar about the look of it; but they were all strange. Every time +Tom made an examination, Becky would watch his face for an encouraging +sign, and he would say cheerily: + +"Oh, it's all right. This ain't the one, but we'll come to it right +away!" + +But he felt less and less hopeful with each failure, and presently +began to turn off into diverging avenues at sheer random, in desperate +hope of finding the one that was wanted. He still said it was "all +right," but there was such a leaden dread at his heart that the words +had lost their ring and sounded just as if he had said, "All is lost!" +Becky clung to his side in an anguish of fear, and tried hard to keep +back the tears, but they would come. At last she said: + +"Oh, Tom, never mind the bats, let's go back that way! We seem to get +worse and worse off all the time." + +"Listen!" said he. + +Profound silence; silence so deep that even their breathings were +conspicuous in the hush. Tom shouted. The call went echoing down the +empty aisles and died out in the distance in a faint sound that +resembled a ripple of mocking laughter. + +"Oh, don't do it again, Tom, it is too horrid," said Becky. + +"It is horrid, but I better, Becky; they might hear us, you know," and +he shouted again. + +The "might" was even a chillier horror than the ghostly laughter, it +so confessed a perishing hope. The children stood still and listened; +but there was no result. Tom turned upon the back track at once, and +hurried his steps. It was but a little while before a certain +indecision in his manner revealed another fearful fact to Becky--he +could not find his way back! + +"Oh, Tom, you didn't make any marks!" + +"Becky, I was such a fool! Such a fool! I never thought we might want +to come back! No--I can't find the way. It's all mixed up." + +"Tom, Tom, we're lost! we're lost! We never can get out of this awful +place! Oh, why DID we ever leave the others!" + +She sank to the ground and burst into such a frenzy of crying that Tom +was appalled with the idea that she might die, or lose her reason. He +sat down by her and put his arms around her; she buried her face in his +bosom, she clung to him, she poured out her terrors, her unavailing +regrets, and the far echoes turned them all to jeering laughter. Tom +begged her to pluck up hope again, and she said she could not. He fell +to blaming and abusing himself for getting her into this miserable +situation; this had a better effect. She said she would try to hope +again, she would get up and follow wherever he might lead if only he +would not talk like that any more. For he was no more to blame than +she, she said. + +So they moved on again--aimlessly--simply at random--all they could do +was to move, keep moving. For a little while, hope made a show of +reviving--not with any reason to back it, but only because it is its +nature to revive when the spring has not been taken out of it by age +and familiarity with failure. + +By-and-by Tom took Becky's candle and blew it out. This economy meant +so much! Words were not needed. Becky understood, and her hope died +again. She knew that Tom had a whole candle and three or four pieces in +his pockets--yet he must economize. + +By-and-by, fatigue began to assert its claims; the children tried to +pay attention, for it was dreadful to think of sitting down when time +was grown to be so precious, moving, in some direction, in any +direction, was at least progress and might bear fruit; but to sit down +was to invite death and shorten its pursuit. + +At last Becky's frail limbs refused to carry her farther. She sat +down. Tom rested with her, and they talked of home, and the friends +there, and the comfortable beds and, above all, the light! Becky cried, +and Tom tried to think of some way of comforting her, but all his +encouragements were grown threadbare with use, and sounded like +sarcasms. Fatigue bore so heavily upon Becky that she drowsed off to +sleep. Tom was grateful. He sat looking into her drawn face and saw it +grow smooth and natural under the influence of pleasant dreams; and +by-and-by a smile dawned and rested there. The peaceful face reflected +somewhat of peace and healing into his own spirit, and his thoughts +wandered away to bygone times and dreamy memories. While he was deep in +his musings, Becky woke up with a breezy little laugh--but it was +stricken dead upon her lips, and a groan followed it. + +"Oh, how COULD I sleep! I wish I never, never had waked! No! No, I +don't, Tom! Don't look so! I won't say it again." + +"I'm glad you've slept, Becky; you'll feel rested, now, and we'll find +the way out." + +"We can try, Tom; but I've seen such a beautiful country in my dream. +I reckon we are going there." + +"Maybe not, maybe not. Cheer up, Becky, and let's go on trying." + +They rose up and wandered along, hand in hand and hopeless. They tried +to estimate how long they had been in the cave, but all they knew was +that it seemed days and weeks, and yet it was plain that this could not +be, for their candles were not gone yet. A long time after this--they +could not tell how long--Tom said they must go softly and listen for +dripping water--they must find a spring. They found one presently, and +Tom said it was time to rest again. Both were cruelly tired, yet Becky +said she thought she could go a little farther. She was surprised to +hear Tom dissent. She could not understand it. They sat down, and Tom +fastened his candle to the wall in front of them with some clay. +Thought was soon busy; nothing was said for some time. Then Becky broke +the silence: + +"Tom, I am so hungry!" + +Tom took something out of his pocket. + +"Do you remember this?" said he. + +Becky almost smiled. + +"It's our wedding-cake, Tom." + +"Yes--I wish it was as big as a barrel, for it's all we've got." + +"I saved it from the picnic for us to dream on, Tom, the way grown-up +people do with wedding-cake--but it'll be our--" + +She dropped the sentence where it was. Tom divided the cake and Becky +ate with good appetite, while Tom nibbled at his moiety. There was +abundance of cold water to finish the feast with. By-and-by Becky +suggested that they move on again. Tom was silent a moment. Then he +said: + +"Becky, can you bear it if I tell you something?" + +Becky's face paled, but she thought she could. + +"Well, then, Becky, we must stay here, where there's water to drink. +That little piece is our last candle!" + +Becky gave loose to tears and wailings. Tom did what he could to +comfort her, but with little effect. At length Becky said: + +"Tom!" + +"Well, Becky?" + +"They'll miss us and hunt for us!" + +"Yes, they will! Certainly they will!" + +"Maybe they're hunting for us now, Tom." + +"Why, I reckon maybe they are. I hope they are." + +"When would they miss us, Tom?" + +"When they get back to the boat, I reckon." + +"Tom, it might be dark then--would they notice we hadn't come?" + +"I don't know. But anyway, your mother would miss you as soon as they +got home." + +A frightened look in Becky's face brought Tom to his senses and he saw +that he had made a blunder. Becky was not to have gone home that night! +The children became silent and thoughtful. In a moment a new burst of +grief from Becky showed Tom that the thing in his mind had struck hers +also--that the Sabbath morning might be half spent before Mrs. Thatcher +discovered that Becky was not at Mrs. Harper's. + +The children fastened their eyes upon their bit of candle and watched +it melt slowly and pitilessly away; saw the half inch of wick stand +alone at last; saw the feeble flame rise and fall, climb the thin +column of smoke, linger at its top a moment, and then--the horror of +utter darkness reigned! + +How long afterward it was that Becky came to a slow consciousness that +she was crying in Tom's arms, neither could tell. All that they knew +was, that after what seemed a mighty stretch of time, both awoke out of +a dead stupor of sleep and resumed their miseries once more. Tom said +it might be Sunday, now--maybe Monday. He tried to get Becky to talk, +but her sorrows were too oppressive, all her hopes were gone. Tom said +that they must have been missed long ago, and no doubt the search was +going on. He would shout and maybe some one would come. He tried it; +but in the darkness the distant echoes sounded so hideously that he +tried it no more. + +The hours wasted away, and hunger came to torment the captives again. +A portion of Tom's half of the cake was left; they divided and ate it. +But they seemed hungrier than before. The poor morsel of food only +whetted desire. + +By-and-by Tom said: + +"SH! Did you hear that?" + +Both held their breath and listened. There was a sound like the +faintest, far-off shout. Instantly Tom answered it, and leading Becky +by the hand, started groping down the corridor in its direction. +Presently he listened again; again the sound was heard, and apparently +a little nearer. + +"It's them!" said Tom; "they're coming! Come along, Becky--we're all +right now!" + +The joy of the prisoners was almost overwhelming. Their speed was +slow, however, because pitfalls were somewhat common, and had to be +guarded against. They shortly came to one and had to stop. It might be +three feet deep, it might be a hundred--there was no passing it at any +rate. Tom got down on his breast and reached as far down as he could. +No bottom. They must stay there and wait until the searchers came. They +listened; evidently the distant shoutings were growing more distant! a +moment or two more and they had gone altogether. The heart-sinking +misery of it! Tom whooped until he was hoarse, but it was of no use. He +talked hopefully to Becky; but an age of anxious waiting passed and no +sounds came again. + +The children groped their way back to the spring. The weary time +dragged on; they slept again, and awoke famished and woe-stricken. Tom +believed it must be Tuesday by this time. + +Now an idea struck him. There were some side passages near at hand. It +would be better to explore some of these than bear the weight of the +heavy time in idleness. He took a kite-line from his pocket, tied it to +a projection, and he and Becky started, Tom in the lead, unwinding the +line as he groped along. At the end of twenty steps the corridor ended +in a "jumping-off place." Tom got down on his knees and felt below, and +then as far around the corner as he could reach with his hands +conveniently; he made an effort to stretch yet a little farther to the +right, and at that moment, not twenty yards away, a human hand, holding +a candle, appeared from behind a rock! Tom lifted up a glorious shout, +and instantly that hand was followed by the body it belonged to--Injun +Joe's! Tom was paralyzed; he could not move. He was vastly gratified +the next moment, to see the "Spaniard" take to his heels and get +himself out of sight. Tom wondered that Joe had not recognized his +voice and come over and killed him for testifying in court. But the +echoes must have disguised the voice. Without doubt, that was it, he +reasoned. Tom's fright weakened every muscle in his body. He said to +himself that if he had strength enough to get back to the spring he +would stay there, and nothing should tempt him to run the risk of +meeting Injun Joe again. He was careful to keep from Becky what it was +he had seen. He told her he had only shouted "for luck." + +But hunger and wretchedness rise superior to fears in the long run. +Another tedious wait at the spring and another long sleep brought +changes. The children awoke tortured with a raging hunger. Tom believed +that it must be Wednesday or Thursday or even Friday or Saturday, now, +and that the search had been given over. He proposed to explore another +passage. He felt willing to risk Injun Joe and all other terrors. But +Becky was very weak. She had sunk into a dreary apathy and would not be +roused. She said she would wait, now, where she was, and die--it would +not be long. She told Tom to go with the kite-line and explore if he +chose; but she implored him to come back every little while and speak +to her; and she made him promise that when the awful time came, he +would stay by her and hold her hand until all was over. + +Tom kissed her, with a choking sensation in his throat, and made a +show of being confident of finding the searchers or an escape from the +cave; then he took the kite-line in his hand and went groping down one +of the passages on his hands and knees, distressed with hunger and sick +with bodings of coming doom. + + + +CHAPTER XXXII + +TUESDAY afternoon came, and waned to the twilight. The village of St. +Petersburg still mourned. The lost children had not been found. Public +prayers had been offered up for them, and many and many a private +prayer that had the petitioner's whole heart in it; but still no good +news came from the cave. The majority of the searchers had given up the +quest and gone back to their daily avocations, saying that it was plain +the children could never be found. Mrs. Thatcher was very ill, and a +great part of the time delirious. People said it was heartbreaking to +hear her call her child, and raise her head and listen a whole minute +at a time, then lay it wearily down again with a moan. Aunt Polly had +drooped into a settled melancholy, and her gray hair had grown almost +white. The village went to its rest on Tuesday night, sad and forlorn. + +Away in the middle of the night a wild peal burst from the village +bells, and in a moment the streets were swarming with frantic half-clad +people, who shouted, "Turn out! turn out! they're found! they're +found!" Tin pans and horns were added to the din, the population massed +itself and moved toward the river, met the children coming in an open +carriage drawn by shouting citizens, thronged around it, joined its +homeward march, and swept magnificently up the main street roaring +huzzah after huzzah! + +The village was illuminated; nobody went to bed again; it was the +greatest night the little town had ever seen. During the first half-hour +a procession of villagers filed through Judge Thatcher's house, seized +the saved ones and kissed them, squeezed Mrs. Thatcher's hand, tried to +speak but couldn't--and drifted out raining tears all over the place. + +Aunt Polly's happiness was complete, and Mrs. Thatcher's nearly so. It +would be complete, however, as soon as the messenger dispatched with +the great news to the cave should get the word to her husband. Tom lay +upon a sofa with an eager auditory about him and told the history of +the wonderful adventure, putting in many striking additions to adorn it +withal; and closed with a description of how he left Becky and went on +an exploring expedition; how he followed two avenues as far as his +kite-line would reach; how he followed a third to the fullest stretch of +the kite-line, and was about to turn back when he glimpsed a far-off +speck that looked like daylight; dropped the line and groped toward it, +pushed his head and shoulders through a small hole, and saw the broad +Mississippi rolling by! And if it had only happened to be night he would +not have seen that speck of daylight and would not have explored that +passage any more! He told how he went back for Becky and broke the good +news and she told him not to fret her with such stuff, for she was +tired, and knew she was going to die, and wanted to. He described how he +labored with her and convinced her; and how she almost died for joy when +she had groped to where she actually saw the blue speck of daylight; how +he pushed his way out at the hole and then helped her out; how they sat +there and cried for gladness; how some men came along in a skiff and Tom +hailed them and told them their situation and their famished condition; +how the men didn't believe the wild tale at first, "because," said they, +"you are five miles down the river below the valley the cave is in" +--then took them aboard, rowed to a house, gave them supper, made them +rest till two or three hours after dark and then brought them home. + +Before day-dawn, Judge Thatcher and the handful of searchers with him +were tracked out, in the cave, by the twine clews they had strung +behind them, and informed of the great news. + +Three days and nights of toil and hunger in the cave were not to be +shaken off at once, as Tom and Becky soon discovered. They were +bedridden all of Wednesday and Thursday, and seemed to grow more and +more tired and worn, all the time. Tom got about, a little, on +Thursday, was down-town Friday, and nearly as whole as ever Saturday; +but Becky did not leave her room until Sunday, and then she looked as +if she had passed through a wasting illness. + +Tom learned of Huck's sickness and went to see him on Friday, but +could not be admitted to the bedroom; neither could he on Saturday or +Sunday. He was admitted daily after that, but was warned to keep still +about his adventure and introduce no exciting topic. The Widow Douglas +stayed by to see that he obeyed. At home Tom learned of the Cardiff +Hill event; also that the "ragged man's" body had eventually been found +in the river near the ferry-landing; he had been drowned while trying +to escape, perhaps. + +About a fortnight after Tom's rescue from the cave, he started off to +visit Huck, who had grown plenty strong enough, now, to hear exciting +talk, and Tom had some that would interest him, he thought. Judge +Thatcher's house was on Tom's way, and he stopped to see Becky. The +Judge and some friends set Tom to talking, and some one asked him +ironically if he wouldn't like to go to the cave again. Tom said he +thought he wouldn't mind it. The Judge said: + +"Well, there are others just like you, Tom, I've not the least doubt. +But we have taken care of that. Nobody will get lost in that cave any +more." + +"Why?" + +"Because I had its big door sheathed with boiler iron two weeks ago, +and triple-locked--and I've got the keys." + +Tom turned as white as a sheet. + +"What's the matter, boy! Here, run, somebody! Fetch a glass of water!" + +The water was brought and thrown into Tom's face. + +"Ah, now you're all right. What was the matter with you, Tom?" + +"Oh, Judge, Injun Joe's in the cave!" + + + +CHAPTER XXXIII + +WITHIN a few minutes the news had spread, and a dozen skiff-loads of +men were on their way to McDougal's cave, and the ferryboat, well +filled with passengers, soon followed. Tom Sawyer was in the skiff that +bore Judge Thatcher. + +When the cave door was unlocked, a sorrowful sight presented itself in +the dim twilight of the place. Injun Joe lay stretched upon the ground, +dead, with his face close to the crack of the door, as if his longing +eyes had been fixed, to the latest moment, upon the light and the cheer +of the free world outside. Tom was touched, for he knew by his own +experience how this wretch had suffered. His pity was moved, but +nevertheless he felt an abounding sense of relief and security, now, +which revealed to him in a degree which he had not fully appreciated +before how vast a weight of dread had been lying upon him since the day +he lifted his voice against this bloody-minded outcast. + +Injun Joe's bowie-knife lay close by, its blade broken in two. The +great foundation-beam of the door had been chipped and hacked through, +with tedious labor; useless labor, too, it was, for the native rock +formed a sill outside it, and upon that stubborn material the knife had +wrought no effect; the only damage done was to the knife itself. But if +there had been no stony obstruction there the labor would have been +useless still, for if the beam had been wholly cut away Injun Joe could +not have squeezed his body under the door, and he knew it. So he had +only hacked that place in order to be doing something--in order to pass +the weary time--in order to employ his tortured faculties. Ordinarily +one could find half a dozen bits of candle stuck around in the crevices +of this vestibule, left there by tourists; but there were none now. The +prisoner had searched them out and eaten them. He had also contrived to +catch a few bats, and these, also, he had eaten, leaving only their +claws. The poor unfortunate had starved to death. In one place, near at +hand, a stalagmite had been slowly growing up from the ground for ages, +builded by the water-drip from a stalactite overhead. The captive had +broken off the stalagmite, and upon the stump had placed a stone, +wherein he had scooped a shallow hollow to catch the precious drop +that fell once in every three minutes with the dreary regularity of a +clock-tick--a dessertspoonful once in four and twenty hours. That drop +was falling when the Pyramids were new; when Troy fell; when the +foundations of Rome were laid; when Christ was crucified; when the +Conqueror created the British empire; when Columbus sailed; when the +massacre at Lexington was "news." It is falling now; it will still be +falling when all these things shall have sunk down the afternoon of +history, and the twilight of tradition, and been swallowed up in the +thick night of oblivion. Has everything a purpose and a mission? Did +this drop fall patiently during five thousand years to be ready for +this flitting human insect's need? and has it another important object +to accomplish ten thousand years to come? No matter. It is many and +many a year since the hapless half-breed scooped out the stone to catch +the priceless drops, but to this day the tourist stares longest at that +pathetic stone and that slow-dropping water when he comes to see the +wonders of McDougal's cave. Injun Joe's cup stands first in the list of +the cavern's marvels; even "Aladdin's Palace" cannot rival it. + +Injun Joe was buried near the mouth of the cave; and people flocked +there in boats and wagons from the towns and from all the farms and +hamlets for seven miles around; they brought their children, and all +sorts of provisions, and confessed that they had had almost as +satisfactory a time at the funeral as they could have had at the +hanging. + +This funeral stopped the further growth of one thing--the petition to +the governor for Injun Joe's pardon. The petition had been largely +signed; many tearful and eloquent meetings had been held, and a +committee of sappy women been appointed to go in deep mourning and wail +around the governor, and implore him to be a merciful ass and trample +his duty under foot. Injun Joe was believed to have killed five +citizens of the village, but what of that? If he had been Satan himself +there would have been plenty of weaklings ready to scribble their names +to a pardon-petition, and drip a tear on it from their permanently +impaired and leaky water-works. + +The morning after the funeral Tom took Huck to a private place to have +an important talk. Huck had learned all about Tom's adventure from the +Welshman and the Widow Douglas, by this time, but Tom said he reckoned +there was one thing they had not told him; that thing was what he +wanted to talk about now. Huck's face saddened. He said: + +"I know what it is. You got into No. 2 and never found anything but +whiskey. Nobody told me it was you; but I just knowed it must 'a' ben +you, soon as I heard 'bout that whiskey business; and I knowed you +hadn't got the money becuz you'd 'a' got at me some way or other and +told me even if you was mum to everybody else. Tom, something's always +told me we'd never get holt of that swag." + +"Why, Huck, I never told on that tavern-keeper. YOU know his tavern +was all right the Saturday I went to the picnic. Don't you remember you +was to watch there that night?" + +"Oh yes! Why, it seems 'bout a year ago. It was that very night that I +follered Injun Joe to the widder's." + +"YOU followed him?" + +"Yes--but you keep mum. I reckon Injun Joe's left friends behind him, +and I don't want 'em souring on me and doing me mean tricks. If it +hadn't ben for me he'd be down in Texas now, all right." + +Then Huck told his entire adventure in confidence to Tom, who had only +heard of the Welshman's part of it before. + +"Well," said Huck, presently, coming back to the main question, +"whoever nipped the whiskey in No. 2, nipped the money, too, I reckon +--anyways it's a goner for us, Tom." + +"Huck, that money wasn't ever in No. 2!" + +"What!" Huck searched his comrade's face keenly. "Tom, have you got on +the track of that money again?" + +"Huck, it's in the cave!" + +Huck's eyes blazed. + +"Say it again, Tom." + +"The money's in the cave!" + +"Tom--honest injun, now--is it fun, or earnest?" + +"Earnest, Huck--just as earnest as ever I was in my life. Will you go +in there with me and help get it out?" + +"I bet I will! I will if it's where we can blaze our way to it and not +get lost." + +"Huck, we can do that without the least little bit of trouble in the +world." + +"Good as wheat! What makes you think the money's--" + +"Huck, you just wait till we get in there. If we don't find it I'll +agree to give you my drum and every thing I've got in the world. I +will, by jings." + +"All right--it's a whiz. When do you say?" + +"Right now, if you say it. Are you strong enough?" + +"Is it far in the cave? I ben on my pins a little, three or four days, +now, but I can't walk more'n a mile, Tom--least I don't think I could." + +"It's about five mile into there the way anybody but me would go, +Huck, but there's a mighty short cut that they don't anybody but me +know about. Huck, I'll take you right to it in a skiff. I'll float the +skiff down there, and I'll pull it back again all by myself. You +needn't ever turn your hand over." + +"Less start right off, Tom." + +"All right. We want some bread and meat, and our pipes, and a little +bag or two, and two or three kite-strings, and some of these +new-fangled things they call lucifer matches. I tell you, many's +the time I wished I had some when I was in there before." + +A trifle after noon the boys borrowed a small skiff from a citizen who +was absent, and got under way at once. When they were several miles +below "Cave Hollow," Tom said: + +"Now you see this bluff here looks all alike all the way down from the +cave hollow--no houses, no wood-yards, bushes all alike. But do you see +that white place up yonder where there's been a landslide? Well, that's +one of my marks. We'll get ashore, now." + +They landed. + +"Now, Huck, where we're a-standing you could touch that hole I got out +of with a fishing-pole. See if you can find it." + +Huck searched all the place about, and found nothing. Tom proudly +marched into a thick clump of sumach bushes and said: + +"Here you are! Look at it, Huck; it's the snuggest hole in this +country. You just keep mum about it. All along I've been wanting to be +a robber, but I knew I'd got to have a thing like this, and where to +run across it was the bother. We've got it now, and we'll keep it +quiet, only we'll let Joe Harper and Ben Rogers in--because of course +there's got to be a Gang, or else there wouldn't be any style about it. +Tom Sawyer's Gang--it sounds splendid, don't it, Huck?" + +"Well, it just does, Tom. And who'll we rob?" + +"Oh, most anybody. Waylay people--that's mostly the way." + +"And kill them?" + +"No, not always. Hive them in the cave till they raise a ransom." + +"What's a ransom?" + +"Money. You make them raise all they can, off'n their friends; and +after you've kept them a year, if it ain't raised then you kill them. +That's the general way. Only you don't kill the women. You shut up the +women, but you don't kill them. They're always beautiful and rich, and +awfully scared. You take their watches and things, but you always take +your hat off and talk polite. They ain't anybody as polite as robbers +--you'll see that in any book. Well, the women get to loving you, and +after they've been in the cave a week or two weeks they stop crying and +after that you couldn't get them to leave. If you drove them out they'd +turn right around and come back. It's so in all the books." + +"Why, it's real bully, Tom. I believe it's better'n to be a pirate." + +"Yes, it's better in some ways, because it's close to home and +circuses and all that." + +By this time everything was ready and the boys entered the hole, Tom +in the lead. They toiled their way to the farther end of the tunnel, +then made their spliced kite-strings fast and moved on. A few steps +brought them to the spring, and Tom felt a shudder quiver all through +him. He showed Huck the fragment of candle-wick perched on a lump of +clay against the wall, and described how he and Becky had watched the +flame struggle and expire. + +The boys began to quiet down to whispers, now, for the stillness and +gloom of the place oppressed their spirits. They went on, and presently +entered and followed Tom's other corridor until they reached the +"jumping-off place." The candles revealed the fact that it was not +really a precipice, but only a steep clay hill twenty or thirty feet +high. Tom whispered: + +"Now I'll show you something, Huck." + +He held his candle aloft and said: + +"Look as far around the corner as you can. Do you see that? There--on +the big rock over yonder--done with candle-smoke." + +"Tom, it's a CROSS!" + +"NOW where's your Number Two? 'UNDER THE CROSS,' hey? Right yonder's +where I saw Injun Joe poke up his candle, Huck!" + +Huck stared at the mystic sign awhile, and then said with a shaky voice: + +"Tom, less git out of here!" + +"What! and leave the treasure?" + +"Yes--leave it. Injun Joe's ghost is round about there, certain." + +"No it ain't, Huck, no it ain't. It would ha'nt the place where he +died--away out at the mouth of the cave--five mile from here." + +"No, Tom, it wouldn't. It would hang round the money. I know the ways +of ghosts, and so do you." + +Tom began to fear that Huck was right. Misgivings gathered in his +mind. But presently an idea occurred to him-- + +"Lookyhere, Huck, what fools we're making of ourselves! Injun Joe's +ghost ain't a going to come around where there's a cross!" + +The point was well taken. It had its effect. + +"Tom, I didn't think of that. But that's so. It's luck for us, that +cross is. I reckon we'll climb down there and have a hunt for that box." + +Tom went first, cutting rude steps in the clay hill as he descended. +Huck followed. Four avenues opened out of the small cavern which the +great rock stood in. The boys examined three of them with no result. +They found a small recess in the one nearest the base of the rock, with +a pallet of blankets spread down in it; also an old suspender, some +bacon rind, and the well-gnawed bones of two or three fowls. But there +was no money-box. The lads searched and researched this place, but in +vain. Tom said: + +"He said UNDER the cross. Well, this comes nearest to being under the +cross. It can't be under the rock itself, because that sets solid on +the ground." + +They searched everywhere once more, and then sat down discouraged. +Huck could suggest nothing. By-and-by Tom said: + +"Lookyhere, Huck, there's footprints and some candle-grease on the +clay about one side of this rock, but not on the other sides. Now, +what's that for? I bet you the money IS under the rock. I'm going to +dig in the clay." + +"That ain't no bad notion, Tom!" said Huck with animation. + +Tom's "real Barlow" was out at once, and he had not dug four inches +before he struck wood. + +"Hey, Huck!--you hear that?" + +Huck began to dig and scratch now. Some boards were soon uncovered and +removed. They had concealed a natural chasm which led under the rock. +Tom got into this and held his candle as far under the rock as he +could, but said he could not see to the end of the rift. He proposed to +explore. He stooped and passed under; the narrow way descended +gradually. He followed its winding course, first to the right, then to +the left, Huck at his heels. Tom turned a short curve, by-and-by, and +exclaimed: + +"My goodness, Huck, lookyhere!" + +It was the treasure-box, sure enough, occupying a snug little cavern, +along with an empty powder-keg, a couple of guns in leather cases, two +or three pairs of old moccasins, a leather belt, and some other rubbish +well soaked with the water-drip. + +"Got it at last!" said Huck, ploughing among the tarnished coins with +his hand. "My, but we're rich, Tom!" + +"Huck, I always reckoned we'd get it. It's just too good to believe, +but we HAVE got it, sure! Say--let's not fool around here. Let's snake +it out. Lemme see if I can lift the box." + +It weighed about fifty pounds. Tom could lift it, after an awkward +fashion, but could not carry it conveniently. + +"I thought so," he said; "THEY carried it like it was heavy, that day +at the ha'nted house. I noticed that. I reckon I was right to think of +fetching the little bags along." + +The money was soon in the bags and the boys took it up to the cross +rock. + +"Now less fetch the guns and things," said Huck. + +"No, Huck--leave them there. They're just the tricks to have when we +go to robbing. We'll keep them there all the time, and we'll hold our +orgies there, too. It's an awful snug place for orgies." + +"What orgies?" + +"I dono. But robbers always have orgies, and of course we've got to +have them, too. Come along, Huck, we've been in here a long time. It's +getting late, I reckon. I'm hungry, too. We'll eat and smoke when we +get to the skiff." + +They presently emerged into the clump of sumach bushes, looked warily +out, found the coast clear, and were soon lunching and smoking in the +skiff. As the sun dipped toward the horizon they pushed out and got +under way. Tom skimmed up the shore through the long twilight, chatting +cheerily with Huck, and landed shortly after dark. + +"Now, Huck," said Tom, "we'll hide the money in the loft of the +widow's woodshed, and I'll come up in the morning and we'll count it +and divide, and then we'll hunt up a place out in the woods for it +where it will be safe. Just you lay quiet here and watch the stuff till +I run and hook Benny Taylor's little wagon; I won't be gone a minute." + +He disappeared, and presently returned with the wagon, put the two +small sacks into it, threw some old rags on top of them, and started +off, dragging his cargo behind him. When the boys reached the +Welshman's house, they stopped to rest. Just as they were about to move +on, the Welshman stepped out and said: + +"Hallo, who's that?" + +"Huck and Tom Sawyer." + +"Good! Come along with me, boys, you are keeping everybody waiting. +Here--hurry up, trot ahead--I'll haul the wagon for you. Why, it's not +as light as it might be. Got bricks in it?--or old metal?" + +"Old metal," said Tom. + +"I judged so; the boys in this town will take more trouble and fool +away more time hunting up six bits' worth of old iron to sell to the +foundry than they would to make twice the money at regular work. But +that's human nature--hurry along, hurry along!" + +The boys wanted to know what the hurry was about. + +"Never mind; you'll see, when we get to the Widow Douglas'." + +Huck said with some apprehension--for he was long used to being +falsely accused: + +"Mr. Jones, we haven't been doing nothing." + +The Welshman laughed. + +"Well, I don't know, Huck, my boy. I don't know about that. Ain't you +and the widow good friends?" + +"Yes. Well, she's ben good friends to me, anyway." + +"All right, then. What do you want to be afraid for?" + +This question was not entirely answered in Huck's slow mind before he +found himself pushed, along with Tom, into Mrs. Douglas' drawing-room. +Mr. Jones left the wagon near the door and followed. + +The place was grandly lighted, and everybody that was of any +consequence in the village was there. The Thatchers were there, the +Harpers, the Rogerses, Aunt Polly, Sid, Mary, the minister, the editor, +and a great many more, and all dressed in their best. The widow +received the boys as heartily as any one could well receive two such +looking beings. They were covered with clay and candle-grease. Aunt +Polly blushed crimson with humiliation, and frowned and shook her head +at Tom. Nobody suffered half as much as the two boys did, however. Mr. +Jones said: + +"Tom wasn't at home, yet, so I gave him up; but I stumbled on him and +Huck right at my door, and so I just brought them along in a hurry." + +"And you did just right," said the widow. "Come with me, boys." + +She took them to a bedchamber and said: + +"Now wash and dress yourselves. Here are two new suits of clothes +--shirts, socks, everything complete. They're Huck's--no, no thanks, +Huck--Mr. Jones bought one and I the other. But they'll fit both of you. +Get into them. We'll wait--come down when you are slicked up enough." + +Then she left. + + + +CHAPTER XXXIV + +HUCK said: "Tom, we can slope, if we can find a rope. The window ain't +high from the ground." + +"Shucks! what do you want to slope for?" + +"Well, I ain't used to that kind of a crowd. I can't stand it. I ain't +going down there, Tom." + +"Oh, bother! It ain't anything. I don't mind it a bit. I'll take care +of you." + +Sid appeared. + +"Tom," said he, "auntie has been waiting for you all the afternoon. +Mary got your Sunday clothes ready, and everybody's been fretting about +you. Say--ain't this grease and clay, on your clothes?" + +"Now, Mr. Siddy, you jist 'tend to your own business. What's all this +blow-out about, anyway?" + +"It's one of the widow's parties that she's always having. This time +it's for the Welshman and his sons, on account of that scrape they +helped her out of the other night. And say--I can tell you something, +if you want to know." + +"Well, what?" + +"Why, old Mr. Jones is going to try to spring something on the people +here to-night, but I overheard him tell auntie to-day about it, as a +secret, but I reckon it's not much of a secret now. Everybody knows +--the widow, too, for all she tries to let on she don't. Mr. Jones was +bound Huck should be here--couldn't get along with his grand secret +without Huck, you know!" + +"Secret about what, Sid?" + +"About Huck tracking the robbers to the widow's. I reckon Mr. Jones +was going to make a grand time over his surprise, but I bet you it will +drop pretty flat." + +Sid chuckled in a very contented and satisfied way. + +"Sid, was it you that told?" + +"Oh, never mind who it was. SOMEBODY told--that's enough." + +"Sid, there's only one person in this town mean enough to do that, and +that's you. If you had been in Huck's place you'd 'a' sneaked down the +hill and never told anybody on the robbers. You can't do any but mean +things, and you can't bear to see anybody praised for doing good ones. +There--no thanks, as the widow says"--and Tom cuffed Sid's ears and +helped him to the door with several kicks. "Now go and tell auntie if +you dare--and to-morrow you'll catch it!" + +Some minutes later the widow's guests were at the supper-table, and a +dozen children were propped up at little side-tables in the same room, +after the fashion of that country and that day. At the proper time Mr. +Jones made his little speech, in which he thanked the widow for the +honor she was doing himself and his sons, but said that there was +another person whose modesty-- + +And so forth and so on. He sprung his secret about Huck's share in the +adventure in the finest dramatic manner he was master of, but the +surprise it occasioned was largely counterfeit and not as clamorous and +effusive as it might have been under happier circumstances. However, +the widow made a pretty fair show of astonishment, and heaped so many +compliments and so much gratitude upon Huck that he almost forgot the +nearly intolerable discomfort of his new clothes in the entirely +intolerable discomfort of being set up as a target for everybody's gaze +and everybody's laudations. + +The widow said she meant to give Huck a home under her roof and have +him educated; and that when she could spare the money she would start +him in business in a modest way. Tom's chance was come. He said: + +"Huck don't need it. Huck's rich." + +Nothing but a heavy strain upon the good manners of the company kept +back the due and proper complimentary laugh at this pleasant joke. But +the silence was a little awkward. Tom broke it: + +"Huck's got money. Maybe you don't believe it, but he's got lots of +it. Oh, you needn't smile--I reckon I can show you. You just wait a +minute." + +Tom ran out of doors. The company looked at each other with a +perplexed interest--and inquiringly at Huck, who was tongue-tied. + +"Sid, what ails Tom?" said Aunt Polly. "He--well, there ain't ever any +making of that boy out. I never--" + +Tom entered, struggling with the weight of his sacks, and Aunt Polly +did not finish her sentence. Tom poured the mass of yellow coin upon +the table and said: + +"There--what did I tell you? Half of it's Huck's and half of it's mine!" + +The spectacle took the general breath away. All gazed, nobody spoke +for a moment. Then there was a unanimous call for an explanation. Tom +said he could furnish it, and he did. The tale was long, but brimful of +interest. There was scarcely an interruption from any one to break the +charm of its flow. When he had finished, Mr. Jones said: + +"I thought I had fixed up a little surprise for this occasion, but it +don't amount to anything now. This one makes it sing mighty small, I'm +willing to allow." + +The money was counted. The sum amounted to a little over twelve +thousand dollars. It was more than any one present had ever seen at one +time before, though several persons were there who were worth +considerably more than that in property. + + + +CHAPTER XXXV + +THE reader may rest satisfied that Tom's and Huck's windfall made a +mighty stir in the poor little village of St. Petersburg. So vast a +sum, all in actual cash, seemed next to incredible. It was talked +about, gloated over, glorified, until the reason of many of the +citizens tottered under the strain of the unhealthy excitement. Every +"haunted" house in St. Petersburg and the neighboring villages was +dissected, plank by plank, and its foundations dug up and ransacked for +hidden treasure--and not by boys, but men--pretty grave, unromantic +men, too, some of them. Wherever Tom and Huck appeared they were +courted, admired, stared at. The boys were not able to remember that +their remarks had possessed weight before; but now their sayings were +treasured and repeated; everything they did seemed somehow to be +regarded as remarkable; they had evidently lost the power of doing and +saying commonplace things; moreover, their past history was raked up +and discovered to bear marks of conspicuous originality. The village +paper published biographical sketches of the boys. + +The Widow Douglas put Huck's money out at six per cent., and Judge +Thatcher did the same with Tom's at Aunt Polly's request. Each lad had +an income, now, that was simply prodigious--a dollar for every week-day +in the year and half of the Sundays. It was just what the minister got +--no, it was what he was promised--he generally couldn't collect it. A +dollar and a quarter a week would board, lodge, and school a boy in +those old simple days--and clothe him and wash him, too, for that +matter. + +Judge Thatcher had conceived a great opinion of Tom. He said that no +commonplace boy would ever have got his daughter out of the cave. When +Becky told her father, in strict confidence, how Tom had taken her +whipping at school, the Judge was visibly moved; and when she pleaded +grace for the mighty lie which Tom had told in order to shift that +whipping from her shoulders to his own, the Judge said with a fine +outburst that it was a noble, a generous, a magnanimous lie--a lie that +was worthy to hold up its head and march down through history breast to +breast with George Washington's lauded Truth about the hatchet! Becky +thought her father had never looked so tall and so superb as when he +walked the floor and stamped his foot and said that. She went straight +off and told Tom about it. + +Judge Thatcher hoped to see Tom a great lawyer or a great soldier some +day. He said he meant to look to it that Tom should be admitted to the +National Military Academy and afterward trained in the best law school +in the country, in order that he might be ready for either career or +both. + +Huck Finn's wealth and the fact that he was now under the Widow +Douglas' protection introduced him into society--no, dragged him into +it, hurled him into it--and his sufferings were almost more than he +could bear. The widow's servants kept him clean and neat, combed and +brushed, and they bedded him nightly in unsympathetic sheets that had +not one little spot or stain which he could press to his heart and know +for a friend. He had to eat with a knife and fork; he had to use +napkin, cup, and plate; he had to learn his book, he had to go to +church; he had to talk so properly that speech was become insipid in +his mouth; whithersoever he turned, the bars and shackles of +civilization shut him in and bound him hand and foot. + +He bravely bore his miseries three weeks, and then one day turned up +missing. For forty-eight hours the widow hunted for him everywhere in +great distress. The public were profoundly concerned; they searched +high and low, they dragged the river for his body. Early the third +morning Tom Sawyer wisely went poking among some old empty hogsheads +down behind the abandoned slaughter-house, and in one of them he found +the refugee. Huck had slept there; he had just breakfasted upon some +stolen odds and ends of food, and was lying off, now, in comfort, with +his pipe. He was unkempt, uncombed, and clad in the same old ruin of +rags that had made him picturesque in the days when he was free and +happy. Tom routed him out, told him the trouble he had been causing, +and urged him to go home. Huck's face lost its tranquil content, and +took a melancholy cast. He said: + +"Don't talk about it, Tom. I've tried it, and it don't work; it don't +work, Tom. It ain't for me; I ain't used to it. The widder's good to +me, and friendly; but I can't stand them ways. She makes me get up just +at the same time every morning; she makes me wash, they comb me all to +thunder; she won't let me sleep in the woodshed; I got to wear them +blamed clothes that just smothers me, Tom; they don't seem to any air +git through 'em, somehow; and they're so rotten nice that I can't set +down, nor lay down, nor roll around anywher's; I hain't slid on a +cellar-door for--well, it 'pears to be years; I got to go to church and +sweat and sweat--I hate them ornery sermons! I can't ketch a fly in +there, I can't chaw. I got to wear shoes all Sunday. The widder eats by +a bell; she goes to bed by a bell; she gits up by a bell--everything's +so awful reg'lar a body can't stand it." + +"Well, everybody does that way, Huck." + +"Tom, it don't make no difference. I ain't everybody, and I can't +STAND it. It's awful to be tied up so. And grub comes too easy--I don't +take no interest in vittles, that way. I got to ask to go a-fishing; I +got to ask to go in a-swimming--dern'd if I hain't got to ask to do +everything. Well, I'd got to talk so nice it wasn't no comfort--I'd got +to go up in the attic and rip out awhile, every day, to git a taste in +my mouth, or I'd a died, Tom. The widder wouldn't let me smoke; she +wouldn't let me yell, she wouldn't let me gape, nor stretch, nor +scratch, before folks--" [Then with a spasm of special irritation and +injury]--"And dad fetch it, she prayed all the time! I never see such a +woman! I HAD to shove, Tom--I just had to. And besides, that school's +going to open, and I'd a had to go to it--well, I wouldn't stand THAT, +Tom. Looky here, Tom, being rich ain't what it's cracked up to be. It's +just worry and worry, and sweat and sweat, and a-wishing you was dead +all the time. Now these clothes suits me, and this bar'l suits me, and +I ain't ever going to shake 'em any more. Tom, I wouldn't ever got into +all this trouble if it hadn't 'a' ben for that money; now you just take +my sheer of it along with your'n, and gimme a ten-center sometimes--not +many times, becuz I don't give a dern for a thing 'thout it's tollable +hard to git--and you go and beg off for me with the widder." + +"Oh, Huck, you know I can't do that. 'Tain't fair; and besides if +you'll try this thing just a while longer you'll come to like it." + +"Like it! Yes--the way I'd like a hot stove if I was to set on it long +enough. No, Tom, I won't be rich, and I won't live in them cussed +smothery houses. I like the woods, and the river, and hogsheads, and +I'll stick to 'em, too. Blame it all! just as we'd got guns, and a +cave, and all just fixed to rob, here this dern foolishness has got to +come up and spile it all!" + +Tom saw his opportunity-- + +"Lookyhere, Huck, being rich ain't going to keep me back from turning +robber." + +"No! Oh, good-licks; are you in real dead-wood earnest, Tom?" + +"Just as dead earnest as I'm sitting here. But Huck, we can't let you +into the gang if you ain't respectable, you know." + +Huck's joy was quenched. + +"Can't let me in, Tom? Didn't you let me go for a pirate?" + +"Yes, but that's different. A robber is more high-toned than what a +pirate is--as a general thing. In most countries they're awful high up +in the nobility--dukes and such." + +"Now, Tom, hain't you always ben friendly to me? You wouldn't shet me +out, would you, Tom? You wouldn't do that, now, WOULD you, Tom?" + +"Huck, I wouldn't want to, and I DON'T want to--but what would people +say? Why, they'd say, 'Mph! Tom Sawyer's Gang! pretty low characters in +it!' They'd mean you, Huck. You wouldn't like that, and I wouldn't." + +Huck was silent for some time, engaged in a mental struggle. Finally +he said: + +"Well, I'll go back to the widder for a month and tackle it and see if +I can come to stand it, if you'll let me b'long to the gang, Tom." + +"All right, Huck, it's a whiz! Come along, old chap, and I'll ask the +widow to let up on you a little, Huck." + +"Will you, Tom--now will you? That's good. If she'll let up on some of +the roughest things, I'll smoke private and cuss private, and crowd +through or bust. When you going to start the gang and turn robbers?" + +"Oh, right off. We'll get the boys together and have the initiation +to-night, maybe." + +"Have the which?" + +"Have the initiation." + +"What's that?" + +"It's to swear to stand by one another, and never tell the gang's +secrets, even if you're chopped all to flinders, and kill anybody and +all his family that hurts one of the gang." + +"That's gay--that's mighty gay, Tom, I tell you." + +"Well, I bet it is. And all that swearing's got to be done at +midnight, in the lonesomest, awfulest place you can find--a ha'nted +house is the best, but they're all ripped up now." + +"Well, midnight's good, anyway, Tom." + +"Yes, so it is. And you've got to swear on a coffin, and sign it with +blood." + +"Now, that's something LIKE! Why, it's a million times bullier than +pirating. I'll stick to the widder till I rot, Tom; and if I git to be +a reg'lar ripper of a robber, and everybody talking 'bout it, I reckon +she'll be proud she snaked me in out of the wet." + + + +CONCLUSION + +SO endeth this chronicle. It being strictly a history of a BOY, it +must stop here; the story could not go much further without becoming +the history of a MAN. When one writes a novel about grown people, he +knows exactly where to stop--that is, with a marriage; but when he +writes of juveniles, he must stop where he best can. + +Most of the characters that perform in this book still live, and are +prosperous and happy. Some day it may seem worth while to take up the +story of the younger ones again and see what sort of men and women they +turned out to be; therefore it will be wisest not to reveal any of that +part of their lives at present. diff --git a/libgo/go/net/testdata/invalid-ndots-resolv.conf b/libgo/go/net/testdata/invalid-ndots-resolv.conf new file mode 100644 index 0000000000..084c1643de --- /dev/null +++ b/libgo/go/net/testdata/invalid-ndots-resolv.conf @@ -0,0 +1 @@ +options ndots:invalid
\ No newline at end of file diff --git a/libgo/go/net/testdata/large-ndots-resolv.conf b/libgo/go/net/testdata/large-ndots-resolv.conf new file mode 100644 index 0000000000..72968eee91 --- /dev/null +++ b/libgo/go/net/testdata/large-ndots-resolv.conf @@ -0,0 +1 @@ +options ndots:16
\ No newline at end of file diff --git a/libgo/go/net/testdata/negative-ndots-resolv.conf b/libgo/go/net/testdata/negative-ndots-resolv.conf new file mode 100644 index 0000000000..c11e0cc403 --- /dev/null +++ b/libgo/go/net/testdata/negative-ndots-resolv.conf @@ -0,0 +1 @@ +options ndots:-1
\ No newline at end of file diff --git a/libgo/go/net/textproto/header.go b/libgo/go/net/textproto/header.go index 7fb32f8045..ed096d9a3c 100644 --- a/libgo/go/net/textproto/header.go +++ b/libgo/go/net/textproto/header.go @@ -1,4 +1,4 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -16,15 +16,17 @@ func (h MIMEHeader) Add(key, value string) { } // Set sets the header entries associated with key to -// the single element value. It replaces any existing +// the single element value. It replaces any existing // values associated with key. func (h MIMEHeader) Set(key, value string) { h[CanonicalMIMEHeaderKey(key)] = []string{value} } // Get gets the first value associated with the given key. +// It is case insensitive; CanonicalMIMEHeaderKey is used +// to canonicalize the provided key. // If there are no values associated with the key, Get returns "". -// Get is a convenience method. For more complex queries, +// To access multiple values of a key, or to use non-canonical keys, // access the map directly. func (h MIMEHeader) Get(key string) string { if h == nil { diff --git a/libgo/go/net/textproto/pipeline.go b/libgo/go/net/textproto/pipeline.go index ca50eddac3..2e283218b5 100644 --- a/libgo/go/net/textproto/pipeline.go +++ b/libgo/go/net/textproto/pipeline.go @@ -1,4 +1,4 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -66,8 +66,8 @@ func (p *Pipeline) EndResponse(id uint) { } // A sequencer schedules a sequence of numbered events that must -// happen in order, one after the other. The event numbering must start -// at 0 and increment without skipping. The event number wraps around +// happen in order, one after the other. The event numbering must start +// at 0 and increment without skipping. The event number wraps around // safely as long as there are not 2^32 simultaneous events pending. type sequencer struct { mu sync.Mutex diff --git a/libgo/go/net/textproto/reader.go b/libgo/go/net/textproto/reader.go index 91bbb57304..e07d1d62e0 100644 --- a/libgo/go/net/textproto/reader.go +++ b/libgo/go/net/textproto/reader.go @@ -1,4 +1,4 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -71,7 +71,7 @@ func (r *Reader) readLineSlice() ([]byte, error) { // ReadContinuedLine reads a possibly continued line from r, // eliding the final trailing ASCII white space. // Lines after the first are considered continuations if they -// begin with a space or tab character. In the returned data, +// begin with a space or tab character. In the returned data, // continuation lines are separated from the previous line // only by a single space: the newline and leading white space // are removed. @@ -204,7 +204,7 @@ func parseCodeLine(line string, expectCode int) (code int, continued bool, messa // ReadCodeLine reads a response code line of the form // code message // where code is a three-digit status code and the message -// extends to the rest of the line. An example of such a line is: +// extends to the rest of the line. An example of such a line is: // 220 plan9.bell-labs.com ESMTP // // If the prefix of the status does not match the digits in expectCode, @@ -366,7 +366,7 @@ func (d *dotReader) Read(b []byte) (n int, err error) { d.state = stateBeginLine break } - // Not part of \r\n. Emit saved \r + // Not part of \r\n. Emit saved \r br.UnreadByte() c = '\r' d.state = stateData @@ -552,9 +552,9 @@ func (r *Reader) upcomingHeaderNewlines() (n int) { } // CanonicalMIMEHeaderKey returns the canonical format of the -// MIME header key s. The canonicalization converts the first +// MIME header key s. The canonicalization converts the first // letter and any letter following a hyphen to upper case; -// the rest are converted to lowercase. For example, the +// the rest are converted to lowercase. For example, the // canonical key for "accept-encoding" is "Accept-Encoding". // MIME header keys are assumed to be ASCII only. // If s contains a space or invalid header field bytes, it is @@ -581,18 +581,14 @@ func CanonicalMIMEHeaderKey(s string) string { const toLower = 'a' - 'A' // validHeaderFieldByte reports whether b is a valid byte in a header -// field key. This is actually stricter than RFC 7230, which says: +// field name. RFC 7230 says: +// header-field = field-name ":" OWS field-value OWS +// field-name = token // tchar = "!" / "#" / "$" / "%" / "&" / "'" / "*" / "+" / "-" / "." / // "^" / "_" / "`" / "|" / "~" / DIGIT / ALPHA // token = 1*tchar -// TODO: revisit in Go 1.6+ and possibly expand this. But note that many -// servers have historically dropped '_' to prevent ambiguities when mapping -// to CGI environment variables. func validHeaderFieldByte(b byte) bool { - return ('A' <= b && b <= 'Z') || - ('a' <= b && b <= 'z') || - ('0' <= b && b <= '9') || - b == '-' + return int(b) < len(isTokenTable) && isTokenTable[b] } // canonicalMIMEHeaderKey is like CanonicalMIMEHeaderKey but is @@ -682,3 +678,85 @@ func init() { commonHeader[v] = v } } + +// isTokenTable is a copy of net/http/lex.go's isTokenTable. +// See https://httpwg.github.io/specs/rfc7230.html#rule.token.separators +var isTokenTable = [127]bool{ + '!': true, + '#': true, + '$': true, + '%': true, + '&': true, + '\'': true, + '*': true, + '+': true, + '-': true, + '.': true, + '0': true, + '1': true, + '2': true, + '3': true, + '4': true, + '5': true, + '6': true, + '7': true, + '8': true, + '9': true, + 'A': true, + 'B': true, + 'C': true, + 'D': true, + 'E': true, + 'F': true, + 'G': true, + 'H': true, + 'I': true, + 'J': true, + 'K': true, + 'L': true, + 'M': true, + 'N': true, + 'O': true, + 'P': true, + 'Q': true, + 'R': true, + 'S': true, + 'T': true, + 'U': true, + 'W': true, + 'V': true, + 'X': true, + 'Y': true, + 'Z': true, + '^': true, + '_': true, + '`': true, + 'a': true, + 'b': true, + 'c': true, + 'd': true, + 'e': true, + 'f': true, + 'g': true, + 'h': true, + 'i': true, + 'j': true, + 'k': true, + 'l': true, + 'm': true, + 'n': true, + 'o': true, + 'p': true, + 'q': true, + 'r': true, + 's': true, + 't': true, + 'u': true, + 'v': true, + 'w': true, + 'x': true, + 'y': true, + 'z': true, + '|': true, + '~': true, +} diff --git a/libgo/go/net/textproto/reader_test.go b/libgo/go/net/textproto/reader_test.go index 93d7939bb5..0c53d48b74 100644 --- a/libgo/go/net/textproto/reader_test.go +++ b/libgo/go/net/textproto/reader_test.go @@ -1,4 +1,4 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -25,6 +25,12 @@ var canonicalHeaderKeyTests = []canonicalHeaderKeyTest{ {"user-agent", "User-Agent"}, {"USER-AGENT", "User-Agent"}, + // Other valid tchar bytes in tokens: + {"foo-bar_baz", "Foo-Bar_baz"}, + {"foo-bar$baz", "Foo-Bar$baz"}, + {"foo-bar~baz", "Foo-Bar~baz"}, + {"foo-bar*baz", "Foo-Bar*baz"}, + // Non-ASCII or anything with spaces or non-token chars is unchanged: {"üser-agenT", "üser-agenT"}, {"a B", "a B"}, diff --git a/libgo/go/net/textproto/textproto.go b/libgo/go/net/textproto/textproto.go index 026eb026b1..8fd781e777 100644 --- a/libgo/go/net/textproto/textproto.go +++ b/libgo/go/net/textproto/textproto.go @@ -1,4 +1,4 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -87,7 +87,7 @@ func Dial(network, addr string) (*Conn, error) { } // Cmd is a convenience method that sends a command after -// waiting its turn in the pipeline. The command text is the +// waiting its turn in the pipeline. The command text is the // result of formatting format with args and appending \r\n. // Cmd returns the id of the command, for use with StartResponse and EndResponse. // diff --git a/libgo/go/net/textproto/writer.go b/libgo/go/net/textproto/writer.go index 03e2fd658e..1bc5974c6c 100644 --- a/libgo/go/net/textproto/writer.go +++ b/libgo/go/net/textproto/writer.go @@ -1,4 +1,4 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -36,7 +36,7 @@ func (w *Writer) PrintfLine(format string, args ...interface{}) error { // DotWriter returns a writer that can be used to write a dot-encoding to w. // It takes care of inserting leading dots when necessary, // translating line-ending \n into \r\n, and adding the final .\r\n line -// when the DotWriter is closed. The caller should close the +// when the DotWriter is closed. The caller should close the // DotWriter before the next call to a method on w. // // See the documentation for Reader's DotReader method for details about dot-encoding. diff --git a/libgo/go/net/textproto/writer_test.go b/libgo/go/net/textproto/writer_test.go index e03ab5e15f..ac03669fa2 100644 --- a/libgo/go/net/textproto/writer_test.go +++ b/libgo/go/net/textproto/writer_test.go @@ -1,4 +1,4 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/net/timeout_test.go b/libgo/go/net/timeout_test.go index 98e3164fb9..55bbf4402d 100644 --- a/libgo/go/net/timeout_test.go +++ b/libgo/go/net/timeout_test.go @@ -6,6 +6,7 @@ package net import ( "fmt" + "internal/testenv" "io" "io/ioutil" "net/internal/socktest" @@ -26,6 +27,8 @@ var dialTimeoutTests = []struct { {-5 * time.Second, 0, -5 * time.Second, 100 * time.Millisecond}, {0, -5 * time.Second, -5 * time.Second, 100 * time.Millisecond}, {-5 * time.Second, 5 * time.Second, -5 * time.Second, 100 * time.Millisecond}, // timeout over deadline + {-1 << 63, 0, time.Second, 100 * time.Millisecond}, + {0, -1 << 63, time.Second, 100 * time.Millisecond}, {50 * time.Millisecond, 0, 100 * time.Millisecond, time.Second}, {0, 50 * time.Millisecond, 100 * time.Millisecond, time.Second}, @@ -38,19 +41,6 @@ func TestDialTimeout(t *testing.T) { defer func() { testHookDialChannel = origTestHookDialChannel }() defer sw.Set(socktest.FilterConnect, nil) - // Avoid tracking open-close jitterbugs between netFD and - // socket that leads to confusion of information inside - // socktest.Switch. - // It may happen when the Dial call bumps against TCP - // simultaneous open. See selfConnect in tcpsock_posix.go. - defer func() { - sw.Set(socktest.FilterClose, nil) - forceCloseSockets() - }() - sw.Set(socktest.FilterClose, func(so *socktest.Status) (socktest.AfterFilter, error) { - return nil, errTimedout - }) - for i, tt := range dialTimeoutTests { switch runtime.GOOS { case "plan9", "windows": @@ -99,6 +89,56 @@ func TestDialTimeout(t *testing.T) { } } +var dialTimeoutMaxDurationTests = []struct { + timeout time.Duration + delta time.Duration // for deadline +}{ + // Large timeouts that will overflow an int64 unix nanos. + {1<<63 - 1, 0}, + {0, 1<<63 - 1}, +} + +func TestDialTimeoutMaxDuration(t *testing.T) { + if runtime.GOOS == "openbsd" { + testenv.SkipFlaky(t, 15157) + } + + ln, err := newLocalListener("tcp") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + + for i, tt := range dialTimeoutMaxDurationTests { + ch := make(chan error) + max := time.NewTimer(250 * time.Millisecond) + defer max.Stop() + go func() { + d := Dialer{Timeout: tt.timeout} + if tt.delta != 0 { + d.Deadline = time.Now().Add(tt.delta) + } + c, err := d.Dial(ln.Addr().Network(), ln.Addr().String()) + if err == nil { + c.Close() + } + ch <- err + }() + + select { + case <-max.C: + t.Fatalf("#%d: Dial didn't return in an expected time", i) + case err := <-ch: + if perr := parseDialError(err); perr != nil { + t.Error(perr) + } + if err != nil { + t.Errorf("#%d: %v", i, err) + } + } + } +} + var acceptTimeoutTests = []struct { timeout time.Duration xerrs [2]error // expected errors in transition @@ -111,6 +151,7 @@ var acceptTimeoutTests = []struct { } func TestAcceptTimeout(t *testing.T) { + testenv.SkipFlaky(t, 17948) t.Parallel() switch runtime.GOOS { @@ -124,16 +165,18 @@ func TestAcceptTimeout(t *testing.T) { } defer ln.Close() + var wg sync.WaitGroup for i, tt := range acceptTimeoutTests { if tt.timeout < 0 { + wg.Add(1) go func() { - c, err := Dial(ln.Addr().Network(), ln.Addr().String()) + defer wg.Done() + d := Dialer{Timeout: 100 * time.Millisecond} + c, err := d.Dial(ln.Addr().Network(), ln.Addr().String()) if err != nil { t.Error(err) return } - var b [1]byte - c.Read(b[:]) c.Close() }() } @@ -154,13 +197,14 @@ func TestAcceptTimeout(t *testing.T) { } if err == nil { c.Close() - time.Sleep(tt.timeout / 3) + time.Sleep(10 * time.Millisecond) continue } break } } } + wg.Wait() } func TestAcceptTimeoutMustReturn(t *testing.T) { @@ -261,13 +305,6 @@ var readTimeoutTests = []struct { } func TestReadTimeout(t *testing.T) { - t.Parallel() - - switch runtime.GOOS { - case "plan9": - t.Skipf("not supported on %s", runtime.GOOS) - } - handler := func(ls *localServer, ln Listener) { c, err := ln.Accept() if err != nil { @@ -393,7 +430,7 @@ var readFromTimeoutTests = []struct { func TestReadFromTimeout(t *testing.T) { switch runtime.GOOS { - case "nacl", "plan9": + case "nacl": t.Skipf("not supported on %s", runtime.GOOS) // see golang.org/issue/8916 } @@ -467,11 +504,6 @@ var writeTimeoutTests = []struct { func TestWriteTimeout(t *testing.T) { t.Parallel() - switch runtime.GOOS { - case "plan9": - t.Skipf("not supported on %s", runtime.GOOS) - } - ln, err := newLocalListener("tcp") if err != nil { t.Fatal(err) @@ -587,7 +619,7 @@ func TestWriteToTimeout(t *testing.T) { t.Parallel() switch runtime.GOOS { - case "nacl", "plan9": + case "nacl": t.Skipf("not supported on %s", runtime.GOOS) } @@ -639,11 +671,6 @@ func TestWriteToTimeout(t *testing.T) { func TestReadTimeoutFluctuation(t *testing.T) { t.Parallel() - switch runtime.GOOS { - case "plan9": - t.Skipf("not supported on %s", runtime.GOOS) - } - ln, err := newLocalListener("tcp") if err != nil { t.Fatal(err) @@ -677,11 +704,6 @@ func TestReadTimeoutFluctuation(t *testing.T) { func TestReadFromTimeoutFluctuation(t *testing.T) { t.Parallel() - switch runtime.GOOS { - case "plan9": - t.Skipf("not supported on %s", runtime.GOOS) - } - c1, err := newLocalPacketListener("udp") if err != nil { t.Fatal(err) @@ -787,11 +809,6 @@ func (b neverEnding) Read(p []byte) (int, error) { } func testVariousDeadlines(t *testing.T) { - switch runtime.GOOS { - case "plan9": - t.Skipf("not supported on %s", runtime.GOOS) - } - type result struct { n int64 err error @@ -988,7 +1005,7 @@ func TestReadWriteDeadlineRace(t *testing.T) { t.Parallel() switch runtime.GOOS { - case "nacl", "plan9": + case "nacl": t.Skipf("not supported on %s", runtime.GOOS) } diff --git a/libgo/go/net/udpsock.go b/libgo/go/net/udpsock.go index 9292133aeb..841ef53359 100644 --- a/libgo/go/net/udpsock.go +++ b/libgo/go/net/udpsock.go @@ -1,9 +1,23 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package net +import ( + "context" + "syscall" +) + +// BUG(mikio): On NaCl, Plan 9 and Windows, the ReadMsgUDP and +// WriteMsgUDP methods of UDPConn are not implemented. + +// BUG(mikio): On Windows, the File method of UDPConn is not +// implemented. + +// BUG(mikio): On NaCl, the ListenMulticastUDP function is not +// implemented. + // UDPAddr represents the address of a UDP end point. type UDPAddr struct { IP IP @@ -45,6 +59,9 @@ func (a *UDPAddr) opAddr() Addr { // "udp6". A literal address or host name for IPv6 must be enclosed // in square brackets, as in "[::1]:80", "[ipv6-host]:http" or // "[ipv6-host%zone]:80". +// +// Resolving a hostname is not recommended because this returns at most +// one of its IP addresses. func ResolveUDPAddr(net, addr string) (*UDPAddr, error) { switch net { case "udp", "udp4", "udp6": @@ -53,9 +70,185 @@ func ResolveUDPAddr(net, addr string) (*UDPAddr, error) { default: return nil, UnknownNetworkError(net) } - addrs, err := internetAddrList(net, addr, noDeadline) + addrs, err := DefaultResolver.internetAddrList(context.Background(), net, addr) if err != nil { return nil, err } return addrs.first(isIPv4).(*UDPAddr), nil } + +// UDPConn is the implementation of the Conn and PacketConn interfaces +// for UDP network connections. +type UDPConn struct { + conn +} + +// ReadFromUDP reads a UDP packet from c, copying the payload into b. +// It returns the number of bytes copied into b and the return address +// that was on the packet. +// +// ReadFromUDP can be made to time out and return an error with +// Timeout() == true after a fixed time limit; see SetDeadline and +// SetReadDeadline. +func (c *UDPConn) ReadFromUDP(b []byte) (int, *UDPAddr, error) { + if !c.ok() { + return 0, nil, syscall.EINVAL + } + n, addr, err := c.readFrom(b) + if err != nil { + err = &OpError{Op: "read", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} + } + return n, addr, err +} + +// ReadFrom implements the PacketConn ReadFrom method. +func (c *UDPConn) ReadFrom(b []byte) (int, Addr, error) { + if !c.ok() { + return 0, nil, syscall.EINVAL + } + n, addr, err := c.readFrom(b) + if err != nil { + err = &OpError{Op: "read", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} + } + if addr == nil { + return n, nil, err + } + return n, addr, err +} + +// ReadMsgUDP reads a packet from c, copying the payload into b and +// the associated out-of-band data into oob. It returns the number +// of bytes copied into b, the number of bytes copied into oob, the +// flags that were set on the packet and the source address of the +// packet. +func (c *UDPConn) ReadMsgUDP(b, oob []byte) (n, oobn, flags int, addr *UDPAddr, err error) { + if !c.ok() { + return 0, 0, 0, nil, syscall.EINVAL + } + n, oobn, flags, addr, err = c.readMsg(b, oob) + if err != nil { + err = &OpError{Op: "read", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} + } + return +} + +// WriteToUDP writes a UDP packet to addr via c, copying the payload +// from b. +// +// WriteToUDP can be made to time out and return an error with +// Timeout() == true after a fixed time limit; see SetDeadline and +// SetWriteDeadline. On packet-oriented connections, write timeouts +// are rare. +func (c *UDPConn) WriteToUDP(b []byte, addr *UDPAddr) (int, error) { + if !c.ok() { + return 0, syscall.EINVAL + } + n, err := c.writeTo(b, addr) + if err != nil { + err = &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: err} + } + return n, err +} + +// WriteTo implements the PacketConn WriteTo method. +func (c *UDPConn) WriteTo(b []byte, addr Addr) (int, error) { + if !c.ok() { + return 0, syscall.EINVAL + } + a, ok := addr.(*UDPAddr) + if !ok { + return 0, &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr, Err: syscall.EINVAL} + } + n, err := c.writeTo(b, a) + if err != nil { + err = &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: a.opAddr(), Err: err} + } + return n, err +} + +// WriteMsgUDP writes a packet to addr via c if c isn't connected, or +// to c's remote destination address if c is connected (in which case +// addr must be nil). The payload is copied from b and the associated +// out-of-band data is copied from oob. It returns the number of +// payload and out-of-band bytes written. +func (c *UDPConn) WriteMsgUDP(b, oob []byte, addr *UDPAddr) (n, oobn int, err error) { + if !c.ok() { + return 0, 0, syscall.EINVAL + } + n, oobn, err = c.writeMsg(b, oob, addr) + if err != nil { + err = &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: err} + } + return +} + +func newUDPConn(fd *netFD) *UDPConn { return &UDPConn{conn{fd}} } + +// DialUDP connects to the remote address raddr on the network net, +// which must be "udp", "udp4", or "udp6". If laddr is not nil, it is +// used as the local address for the connection. +func DialUDP(net string, laddr, raddr *UDPAddr) (*UDPConn, error) { + switch net { + case "udp", "udp4", "udp6": + default: + return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: UnknownNetworkError(net)} + } + if raddr == nil { + return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: nil, Err: errMissingAddress} + } + c, err := dialUDP(context.Background(), net, laddr, raddr) + if err != nil { + return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: err} + } + return c, nil +} + +// ListenUDP listens for incoming UDP packets addressed to the local +// address laddr. Net must be "udp", "udp4", or "udp6". If laddr has +// a port of 0, ListenUDP will choose an available port. +// The LocalAddr method of the returned UDPConn can be used to +// discover the port. The returned connection's ReadFrom and WriteTo +// methods can be used to receive and send UDP packets with per-packet +// addressing. +func ListenUDP(net string, laddr *UDPAddr) (*UDPConn, error) { + switch net { + case "udp", "udp4", "udp6": + default: + return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: UnknownNetworkError(net)} + } + if laddr == nil { + laddr = &UDPAddr{} + } + c, err := listenUDP(context.Background(), net, laddr) + if err != nil { + return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: err} + } + return c, nil +} + +// ListenMulticastUDP listens for incoming multicast UDP packets +// addressed to the group address gaddr on the interface ifi. +// Network must be "udp", "udp4" or "udp6". +// ListenMulticastUDP uses the system-assigned multicast interface +// when ifi is nil, although this is not recommended because the +// assignment depends on platforms and sometimes it might require +// routing configuration. +// +// ListenMulticastUDP is just for convenience of simple, small +// applications. There are golang.org/x/net/ipv4 and +// golang.org/x/net/ipv6 packages for general purpose uses. +func ListenMulticastUDP(network string, ifi *Interface, gaddr *UDPAddr) (*UDPConn, error) { + switch network { + case "udp", "udp4", "udp6": + default: + return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: gaddr.opAddr(), Err: UnknownNetworkError(network)} + } + if gaddr == nil || gaddr.IP == nil { + return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: gaddr.opAddr(), Err: errMissingAddress} + } + c, err := listenMulticastUDP(context.Background(), network, ifi, gaddr) + if err != nil { + return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: gaddr.opAddr(), Err: err} + } + return c, nil +} diff --git a/libgo/go/net/udpsock_plan9.go b/libgo/go/net/udpsock_plan9.go index 1ba57a227e..1ce7f88c62 100644 --- a/libgo/go/net/udpsock_plan9.go +++ b/libgo/go/net/udpsock_plan9.go @@ -1,42 +1,24 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package net import ( + "context" "errors" "os" "syscall" - "time" ) -// UDPConn is the implementation of the Conn and PacketConn interfaces -// for UDP network connections. -type UDPConn struct { - conn -} - -func newUDPConn(fd *netFD) *UDPConn { return &UDPConn{conn{fd}} } - -// ReadFromUDP reads a UDP packet from c, copying the payload into b. -// It returns the number of bytes copied into b and the return address -// that was on the packet. -// -// ReadFromUDP can be made to time out and return an error with -// Timeout() == true after a fixed time limit; see SetDeadline and -// SetReadDeadline. -func (c *UDPConn) ReadFromUDP(b []byte) (n int, addr *UDPAddr, err error) { - if !c.ok() || c.fd.data == nil { - return 0, nil, syscall.EINVAL - } +func (c *UDPConn) readFrom(b []byte) (n int, addr *UDPAddr, err error) { buf := make([]byte, udpHeaderSize+len(b)) - m, err := c.fd.data.Read(buf) + m, err := c.fd.Read(buf) if err != nil { - return 0, nil, &OpError{Op: "read", Net: c.fd.dir, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} + return 0, nil, err } if m < udpHeaderSize { - return 0, nil, &OpError{Op: "read", Net: c.fd.dir, Source: c.fd.laddr, Addr: c.fd.raddr, Err: errors.New("short read reading UDP header")} + return 0, nil, errors.New("short read reading UDP header") } buf = buf[:m] @@ -45,36 +27,13 @@ func (c *UDPConn) ReadFromUDP(b []byte) (n int, addr *UDPAddr, err error) { return n, &UDPAddr{IP: h.raddr, Port: int(h.rport)}, nil } -// ReadFrom implements the PacketConn ReadFrom method. -func (c *UDPConn) ReadFrom(b []byte) (int, Addr, error) { - if !c.ok() { - return 0, nil, syscall.EINVAL - } - return c.ReadFromUDP(b) -} - -// ReadMsgUDP reads a packet from c, copying the payload into b and -// the associated out-of-band data into oob. It returns the number -// of bytes copied into b, the number of bytes copied into oob, the -// flags that were set on the packet and the source address of the -// packet. -func (c *UDPConn) ReadMsgUDP(b, oob []byte) (n, oobn, flags int, addr *UDPAddr, err error) { - return 0, 0, 0, nil, &OpError{Op: "read", Net: c.fd.dir, Source: c.fd.laddr, Addr: c.fd.raddr, Err: syscall.EPLAN9} +func (c *UDPConn) readMsg(b, oob []byte) (n, oobn, flags int, addr *UDPAddr, err error) { + return 0, 0, 0, nil, syscall.EPLAN9 } -// WriteToUDP writes a UDP packet to addr via c, copying the payload -// from b. -// -// WriteToUDP can be made to time out and return an error with -// Timeout() == true after a fixed time limit; see SetDeadline and -// SetWriteDeadline. On packet-oriented connections, write timeouts -// are rare. -func (c *UDPConn) WriteToUDP(b []byte, addr *UDPAddr) (int, error) { - if !c.ok() || c.fd.data == nil { - return 0, syscall.EINVAL - } +func (c *UDPConn) writeTo(b []byte, addr *UDPAddr) (int, error) { if addr == nil { - return 0, &OpError{Op: "write", Net: c.fd.dir, Source: c.fd.laddr, Addr: nil, Err: errMissingAddress} + return 0, errMissingAddress } h := new(udpHeader) h.raddr = addr.IP.To16() @@ -86,53 +45,18 @@ func (c *UDPConn) WriteToUDP(b []byte, addr *UDPAddr) (int, error) { buf := make([]byte, udpHeaderSize+len(b)) i := copy(buf, h.Bytes()) copy(buf[i:], b) - if _, err := c.fd.data.Write(buf); err != nil { - return 0, &OpError{Op: "write", Net: c.fd.dir, Source: c.fd.laddr, Addr: addr.opAddr(), Err: err} + if _, err := c.fd.Write(buf); err != nil { + return 0, err } return len(b), nil } -// WriteTo implements the PacketConn WriteTo method. -func (c *UDPConn) WriteTo(b []byte, addr Addr) (int, error) { - if !c.ok() { - return 0, syscall.EINVAL - } - a, ok := addr.(*UDPAddr) - if !ok { - return 0, &OpError{Op: "write", Net: c.fd.dir, Source: c.fd.laddr, Addr: addr, Err: syscall.EINVAL} - } - return c.WriteToUDP(b, a) +func (c *UDPConn) writeMsg(b, oob []byte, addr *UDPAddr) (n, oobn int, err error) { + return 0, 0, syscall.EPLAN9 } -// WriteMsgUDP writes a packet to addr via c if c isn't connected, or -// to c's remote destination address if c is connected (in which case -// addr must be nil). The payload is copied from b and the associated -// out-of-band data is copied from oob. It returns the number of -// payload and out-of-band bytes written. -func (c *UDPConn) WriteMsgUDP(b, oob []byte, addr *UDPAddr) (n, oobn int, err error) { - return 0, 0, &OpError{Op: "write", Net: c.fd.dir, Source: c.fd.laddr, Addr: addr.opAddr(), Err: syscall.EPLAN9} -} - -// DialUDP connects to the remote address raddr on the network net, -// which must be "udp", "udp4", or "udp6". If laddr is not nil, it is -// used as the local address for the connection. -func DialUDP(net string, laddr, raddr *UDPAddr) (*UDPConn, error) { - return dialUDP(net, laddr, raddr, noDeadline) -} - -func dialUDP(net string, laddr, raddr *UDPAddr, deadline time.Time) (*UDPConn, error) { - if !deadline.IsZero() { - panic("net.dialUDP: deadline not implemented on Plan 9") - } - switch net { - case "udp", "udp4", "udp6": - default: - return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: UnknownNetworkError(net)} - } - if raddr == nil { - return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: nil, Err: errMissingAddress} - } - fd, err := dialPlan9(net, laddr, raddr) +func dialUDP(ctx context.Context, net string, laddr, raddr *UDPAddr) (*UDPConn, error) { + fd, err := dialPlan9(ctx, net, laddr, raddr) if err != nil { return nil, err } @@ -167,49 +91,59 @@ func unmarshalUDPHeader(b []byte) (*udpHeader, []byte) { return h, b } -// ListenUDP listens for incoming UDP packets addressed to the local -// address laddr. Net must be "udp", "udp4", or "udp6". If laddr has -// a port of 0, ListenUDP will choose an available port. -// The LocalAddr method of the returned UDPConn can be used to -// discover the port. The returned connection's ReadFrom and WriteTo -// methods can be used to receive and send UDP packets with per-packet -// addressing. -func ListenUDP(net string, laddr *UDPAddr) (*UDPConn, error) { - switch net { - case "udp", "udp4", "udp6": - default: - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: UnknownNetworkError(net)} - } - if laddr == nil { - laddr = &UDPAddr{} - } - l, err := listenPlan9(net, laddr) +func listenUDP(ctx context.Context, network string, laddr *UDPAddr) (*UDPConn, error) { + l, err := listenPlan9(ctx, network, laddr) if err != nil { return nil, err } _, err = l.ctl.WriteString("headers") if err != nil { - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr, Err: err} + return nil, err } l.data, err = os.OpenFile(l.dir+"/data", os.O_RDWR, 0) if err != nil { - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr, Err: err} + return nil, err } fd, err := l.netFD() return newUDPConn(fd), err } -// ListenMulticastUDP listens for incoming multicast UDP packets -// addressed to the group address gaddr on the interface ifi. -// Network must be "udp", "udp4" or "udp6". -// ListenMulticastUDP uses the system-assigned multicast interface -// when ifi is nil, although this is not recommended because the -// assignment depends on platforms and sometimes it might require -// routing configuration. -// -// ListenMulticastUDP is just for convenience of simple, small -// applications. There are golang.org/x/net/ipv4 and -// golang.org/x/net/ipv6 packages for general purpose uses. -func ListenMulticastUDP(network string, ifi *Interface, gaddr *UDPAddr) (*UDPConn, error) { - return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: gaddr.opAddr(), Err: syscall.EPLAN9} +func listenMulticastUDP(ctx context.Context, network string, ifi *Interface, gaddr *UDPAddr) (*UDPConn, error) { + l, err := listenPlan9(ctx, network, gaddr) + if err != nil { + return nil, err + } + _, err = l.ctl.WriteString("headers") + if err != nil { + return nil, err + } + var addrs []Addr + if ifi != nil { + addrs, err = ifi.Addrs() + if err != nil { + return nil, err + } + } else { + addrs, err = InterfaceAddrs() + if err != nil { + return nil, err + } + } + for _, addr := range addrs { + if ipnet, ok := addr.(*IPNet); ok { + _, err = l.ctl.WriteString("addmulti " + ipnet.IP.String() + " " + gaddr.IP.String()) + if err != nil { + return nil, err + } + } + } + l.data, err = os.OpenFile(l.dir+"/data", os.O_RDWR, 0) + if err != nil { + return nil, err + } + fd, err := l.netFD() + if err != nil { + return nil, err + } + return newUDPConn(fd), nil } diff --git a/libgo/go/net/udpsock_plan9_test.go b/libgo/go/net/udpsock_plan9_test.go new file mode 100644 index 0000000000..09f5a5dc65 --- /dev/null +++ b/libgo/go/net/udpsock_plan9_test.go @@ -0,0 +1,69 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "internal/testenv" + "runtime" + "testing" +) + +func TestListenMulticastUDP(t *testing.T) { + testenv.MustHaveExternalNetwork(t) + + ifcs, err := Interfaces() + if err != nil { + t.Skip(err.Error()) + } + if len(ifcs) == 0 { + t.Skip("no network interfaces found") + } + + var mifc *Interface + for _, ifc := range ifcs { + if ifc.Flags&FlagUp|FlagMulticast != FlagUp|FlagMulticast { + continue + } + mifc = &ifc + break + } + + if mifc == nil { + t.Skipf("no multicast interfaces found") + } + + c1, err := ListenMulticastUDP("udp4", mifc, &UDPAddr{IP: ParseIP("224.0.0.254")}) + if err != nil { + t.Fatalf("multicast not working on %s", runtime.GOOS) + } + c1addr := c1.LocalAddr().(*UDPAddr) + if err != nil { + t.Fatal(err) + } + defer c1.Close() + + c2, err := ListenUDP("udp4", &UDPAddr{IP: IPv4zero, Port: 0}) + c2addr := c2.LocalAddr().(*UDPAddr) + if err != nil { + t.Fatal(err) + } + defer c2.Close() + + n, err := c2.WriteToUDP([]byte("data"), c1addr) + if err != nil { + t.Fatal(err) + } + if n != 4 { + t.Fatalf("got %d; want 4", n) + } + + n, err = c1.WriteToUDP([]byte("data"), c2addr) + if err != nil { + t.Fatal(err) + } + if n != 4 { + t.Fatalf("got %d; want 4", n) + } +} diff --git a/libgo/go/net/udpsock_posix.go b/libgo/go/net/udpsock_posix.go index 932c6ce713..72aadca5dc 100644 --- a/libgo/go/net/udpsock_posix.go +++ b/libgo/go/net/udpsock_posix.go @@ -1,4 +1,4 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -7,8 +7,8 @@ package net import ( + "context" "syscall" - "time" ) func sockaddrToUDP(sa syscall.Sockaddr) Addr { @@ -38,25 +38,11 @@ func (a *UDPAddr) sockaddr(family int) (syscall.Sockaddr, error) { return ipToSockaddr(family, a.IP, a.Port, a.Zone) } -// UDPConn is the implementation of the Conn and PacketConn interfaces -// for UDP network connections. -type UDPConn struct { - conn +func (a *UDPAddr) toLocal(net string) sockaddr { + return &UDPAddr{loopbackIP(net), a.Port, a.Zone} } -func newUDPConn(fd *netFD) *UDPConn { return &UDPConn{conn{fd}} } - -// ReadFromUDP reads a UDP packet from c, copying the payload into b. -// It returns the number of bytes copied into b and the return address -// that was on the packet. -// -// ReadFromUDP can be made to time out and return an error with -// Timeout() == true after a fixed time limit; see SetDeadline and -// SetReadDeadline. -func (c *UDPConn) ReadFromUDP(b []byte) (int, *UDPAddr, error) { - if !c.ok() { - return 0, nil, syscall.EINVAL - } +func (c *UDPConn) readFrom(b []byte) (int, *UDPAddr, error) { var addr *UDPAddr n, sa, err := c.fd.readFrom(b) switch sa := sa.(type) { @@ -65,33 +51,10 @@ func (c *UDPConn) ReadFromUDP(b []byte) (int, *UDPAddr, error) { case *syscall.SockaddrInet6: addr = &UDPAddr{IP: sa.Addr[0:], Port: sa.Port, Zone: zoneToString(int(sa.ZoneId))} } - if err != nil { - err = &OpError{Op: "read", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} - } return n, addr, err } -// ReadFrom implements the PacketConn ReadFrom method. -func (c *UDPConn) ReadFrom(b []byte) (int, Addr, error) { - if !c.ok() { - return 0, nil, syscall.EINVAL - } - n, addr, err := c.ReadFromUDP(b) - if addr == nil { - return n, nil, err - } - return n, addr, err -} - -// ReadMsgUDP reads a packet from c, copying the payload into b and -// the associated out-of-band data into oob. It returns the number -// of bytes copied into b, the number of bytes copied into oob, the -// flags that were set on the packet and the source address of the -// packet. -func (c *UDPConn) ReadMsgUDP(b, oob []byte) (n, oobn, flags int, addr *UDPAddr, err error) { - if !c.ok() { - return 0, 0, 0, nil, syscall.EINVAL - } +func (c *UDPConn) readMsg(b, oob []byte) (n, oobn, flags int, addr *UDPAddr, err error) { var sa syscall.Sockaddr n, oobn, flags, sa, err = c.fd.readMsg(b, oob) switch sa := sa.(type) { @@ -100,159 +63,68 @@ func (c *UDPConn) ReadMsgUDP(b, oob []byte) (n, oobn, flags int, addr *UDPAddr, case *syscall.SockaddrInet6: addr = &UDPAddr{IP: sa.Addr[0:], Port: sa.Port, Zone: zoneToString(int(sa.ZoneId))} } - if err != nil { - err = &OpError{Op: "read", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} - } return } -// WriteToUDP writes a UDP packet to addr via c, copying the payload -// from b. -// -// WriteToUDP can be made to time out and return an error with -// Timeout() == true after a fixed time limit; see SetDeadline and -// SetWriteDeadline. On packet-oriented connections, write timeouts -// are rare. -func (c *UDPConn) WriteToUDP(b []byte, addr *UDPAddr) (int, error) { - if !c.ok() { - return 0, syscall.EINVAL - } +func (c *UDPConn) writeTo(b []byte, addr *UDPAddr) (int, error) { if c.fd.isConnected { - return 0, &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: ErrWriteToConnected} + return 0, ErrWriteToConnected } if addr == nil { - return 0, &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: nil, Err: errMissingAddress} + return 0, errMissingAddress } sa, err := addr.sockaddr(c.fd.family) if err != nil { - return 0, &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: err} - } - n, err := c.fd.writeTo(b, sa) - if err != nil { - err = &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: err} - } - return n, err -} - -// WriteTo implements the PacketConn WriteTo method. -func (c *UDPConn) WriteTo(b []byte, addr Addr) (int, error) { - if !c.ok() { - return 0, syscall.EINVAL - } - a, ok := addr.(*UDPAddr) - if !ok { - return 0, &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr, Err: syscall.EINVAL} + return 0, err } - return c.WriteToUDP(b, a) + return c.fd.writeTo(b, sa) } -// WriteMsgUDP writes a packet to addr via c if c isn't connected, or -// to c's remote destination address if c is connected (in which case -// addr must be nil). The payload is copied from b and the associated -// out-of-band data is copied from oob. It returns the number of -// payload and out-of-band bytes written. -func (c *UDPConn) WriteMsgUDP(b, oob []byte, addr *UDPAddr) (n, oobn int, err error) { - if !c.ok() { - return 0, 0, syscall.EINVAL - } +func (c *UDPConn) writeMsg(b, oob []byte, addr *UDPAddr) (n, oobn int, err error) { if c.fd.isConnected && addr != nil { - return 0, 0, &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: ErrWriteToConnected} + return 0, 0, ErrWriteToConnected } if !c.fd.isConnected && addr == nil { - return 0, 0, &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: errMissingAddress} + return 0, 0, errMissingAddress } - var sa syscall.Sockaddr - sa, err = addr.sockaddr(c.fd.family) - if err != nil { - return 0, 0, &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: err} - } - n, oobn, err = c.fd.writeMsg(b, oob, sa) + sa, err := addr.sockaddr(c.fd.family) if err != nil { - err = &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: err} + return 0, 0, err } - return + return c.fd.writeMsg(b, oob, sa) } -// DialUDP connects to the remote address raddr on the network net, -// which must be "udp", "udp4", or "udp6". If laddr is not nil, it is -// used as the local address for the connection. -func DialUDP(net string, laddr, raddr *UDPAddr) (*UDPConn, error) { - switch net { - case "udp", "udp4", "udp6": - default: - return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: UnknownNetworkError(net)} - } - if raddr == nil { - return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: nil, Err: errMissingAddress} - } - return dialUDP(net, laddr, raddr, noDeadline) -} - -func dialUDP(net string, laddr, raddr *UDPAddr, deadline time.Time) (*UDPConn, error) { - fd, err := internetSocket(net, laddr, raddr, deadline, syscall.SOCK_DGRAM, 0, "dial", noCancel) +func dialUDP(ctx context.Context, net string, laddr, raddr *UDPAddr) (*UDPConn, error) { + fd, err := internetSocket(ctx, net, laddr, raddr, syscall.SOCK_DGRAM, 0, "dial") if err != nil { - return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: err} + return nil, err } return newUDPConn(fd), nil } -// ListenUDP listens for incoming UDP packets addressed to the local -// address laddr. Net must be "udp", "udp4", or "udp6". If laddr has -// a port of 0, ListenUDP will choose an available port. -// The LocalAddr method of the returned UDPConn can be used to -// discover the port. The returned connection's ReadFrom and WriteTo -// methods can be used to receive and send UDP packets with per-packet -// addressing. -func ListenUDP(net string, laddr *UDPAddr) (*UDPConn, error) { - switch net { - case "udp", "udp4", "udp6": - default: - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: UnknownNetworkError(net)} - } - if laddr == nil { - laddr = &UDPAddr{} - } - fd, err := internetSocket(net, laddr, nil, noDeadline, syscall.SOCK_DGRAM, 0, "listen", noCancel) +func listenUDP(ctx context.Context, network string, laddr *UDPAddr) (*UDPConn, error) { + fd, err := internetSocket(ctx, network, laddr, nil, syscall.SOCK_DGRAM, 0, "listen") if err != nil { - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr, Err: err} + return nil, err } return newUDPConn(fd), nil } -// ListenMulticastUDP listens for incoming multicast UDP packets -// addressed to the group address gaddr on the interface ifi. -// Network must be "udp", "udp4" or "udp6". -// ListenMulticastUDP uses the system-assigned multicast interface -// when ifi is nil, although this is not recommended because the -// assignment depends on platforms and sometimes it might require -// routing configuration. -// -// ListenMulticastUDP is just for convenience of simple, small -// applications. There are golang.org/x/net/ipv4 and -// golang.org/x/net/ipv6 packages for general purpose uses. -func ListenMulticastUDP(network string, ifi *Interface, gaddr *UDPAddr) (*UDPConn, error) { - switch network { - case "udp", "udp4", "udp6": - default: - return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: gaddr.opAddr(), Err: UnknownNetworkError(network)} - } - if gaddr == nil || gaddr.IP == nil { - return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: gaddr.opAddr(), Err: errMissingAddress} - } - fd, err := internetSocket(network, gaddr, nil, noDeadline, syscall.SOCK_DGRAM, 0, "listen", noCancel) +func listenMulticastUDP(ctx context.Context, network string, ifi *Interface, gaddr *UDPAddr) (*UDPConn, error) { + fd, err := internetSocket(ctx, network, gaddr, nil, syscall.SOCK_DGRAM, 0, "listen") if err != nil { - return nil, &OpError{Op: "listen", Net: network, Source: nil, Addr: gaddr, Err: err} + return nil, err } c := newUDPConn(fd) if ip4 := gaddr.IP.To4(); ip4 != nil { if err := listenIPv4MulticastUDP(c, ifi, ip4); err != nil { c.Close() - return nil, &OpError{Op: "listen", Net: network, Source: c.fd.laddr, Addr: &IPAddr{IP: ip4}, Err: err} + return nil, err } } else { if err := listenIPv6MulticastUDP(c, ifi, gaddr.IP); err != nil { c.Close() - return nil, &OpError{Op: "listen", Net: network, Source: c.fd.laddr, Addr: &IPAddr{IP: gaddr.IP}, Err: err} + return nil, err } } return c, nil diff --git a/libgo/go/net/udp_test.go b/libgo/go/net/udpsock_test.go index b25f96a3fd..708cc10120 100644 --- a/libgo/go/net/udp_test.go +++ b/libgo/go/net/udpsock_test.go @@ -1,16 +1,54 @@ -// Copyright 2012 The Go Authors. All rights reserved. +// Copyright 2012 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package net import ( + "internal/testenv" "reflect" "runtime" "testing" "time" ) +func BenchmarkUDP6LinkLocalUnicast(b *testing.B) { + testHookUninstaller.Do(uninstallTestHooks) + + if !supportsIPv6 { + b.Skip("IPv6 is not supported") + } + ifi := loopbackInterface() + if ifi == nil { + b.Skip("loopback interface not found") + } + lla := ipv6LinkLocalUnicastAddr(ifi) + if lla == "" { + b.Skip("IPv6 link-local unicast address not found") + } + + c1, err := ListenPacket("udp6", JoinHostPort(lla+"%"+ifi.Name, "0")) + if err != nil { + b.Fatal(err) + } + defer c1.Close() + c2, err := ListenPacket("udp6", JoinHostPort(lla+"%"+ifi.Name, "0")) + if err != nil { + b.Fatal(err) + } + defer c2.Close() + + var buf [1]byte + for i := 0; i < b.N; i++ { + if _, err := c1.WriteTo(buf[:], c2.LocalAddr()); err != nil { + b.Fatal(err) + } + if _, _, err := c2.ReadFrom(buf[:]); err != nil { + b.Fatal(err) + } + } +} + type resolveUDPAddrTest struct { network string litAddrOrName string @@ -34,6 +72,17 @@ var resolveUDPAddrTests = []resolveUDPAddrTest{ {"udp", ":12345", &UDPAddr{Port: 12345}, nil}, {"http", "127.0.0.1:0", nil, UnknownNetworkError("http")}, + + {"udp", "127.0.0.1:domain", &UDPAddr{IP: ParseIP("127.0.0.1"), Port: 53}, nil}, + {"udp", "[::ffff:127.0.0.1]:domain", &UDPAddr{IP: ParseIP("::ffff:127.0.0.1"), Port: 53}, nil}, + {"udp", "[2001:db8::1]:domain", &UDPAddr{IP: ParseIP("2001:db8::1"), Port: 53}, nil}, + {"udp4", "127.0.0.1:domain", &UDPAddr{IP: ParseIP("127.0.0.1"), Port: 53}, nil}, + {"udp4", "[::ffff:127.0.0.1]:domain", &UDPAddr{IP: ParseIP("127.0.0.1"), Port: 53}, nil}, + {"udp6", "[2001:db8::1]:domain", &UDPAddr{IP: ParseIP("2001:db8::1"), Port: 53}, nil}, + + {"udp4", "[2001:db8::1]:domain", nil, &AddrError{Err: errNoSuitableAddress.Error(), Addr: "2001:db8::1"}}, + {"udp6", "127.0.0.1:domain", nil, &AddrError{Err: errNoSuitableAddress.Error(), Addr: "127.0.0.1"}}, + {"udp6", "[::ffff:127.0.0.1]:domain", nil, &AddrError{Err: errNoSuitableAddress.Error(), Addr: "::ffff:127.0.0.1"}}, } func TestResolveUDPAddr(t *testing.T) { @@ -41,21 +90,17 @@ func TestResolveUDPAddr(t *testing.T) { defer func() { testHookLookupIP = origTestHookLookupIP }() testHookLookupIP = lookupLocalhost - for i, tt := range resolveUDPAddrTests { + for _, tt := range resolveUDPAddrTests { addr, err := ResolveUDPAddr(tt.network, tt.litAddrOrName) - if err != tt.err { - t.Errorf("#%d: %v", i, err) - } else if !reflect.DeepEqual(addr, tt.addr) { - t.Errorf("#%d: got %#v; want %#v", i, addr, tt.addr) - } - if err != nil { + if !reflect.DeepEqual(addr, tt.addr) || !reflect.DeepEqual(err, tt.err) { + t.Errorf("ResolveUDPAddr(%q, %q) = %#v, %v, want %#v, %v", tt.network, tt.litAddrOrName, addr, err, tt.addr, tt.err) continue } - rtaddr, err := ResolveUDPAddr(addr.Network(), addr.String()) - if err != nil { - t.Errorf("#%d: %v", i, err) - } else if !reflect.DeepEqual(rtaddr, addr) { - t.Errorf("#%d: got %#v; want %#v", i, rtaddr, addr) + if err == nil { + addr2, err := ResolveUDPAddr(addr.Network(), addr.String()) + if !reflect.DeepEqual(addr2, tt.addr) || err != tt.err { + t.Errorf("(%q, %q): ResolveUDPAddr(%q, %q) = %#v, %v, want %#v, %v", tt.network, tt.litAddrOrName, addr.Network(), addr.String(), addr2, err, tt.addr, tt.err) + } } } } @@ -178,9 +223,7 @@ var udpConnLocalNameTests = []struct { } func TestUDPConnLocalName(t *testing.T) { - if testing.Short() || !*testExternal { - t.Skip("avoid external network") - } + testenv.MustHaveExternalNetwork(t) for _, tt := range udpConnLocalNameTests { c, err := ListenUDP(tt.net, tt.laddr) @@ -234,9 +277,8 @@ func TestUDPConnLocalAndRemoteNames(t *testing.T) { } func TestIPv6LinkLocalUnicastUDP(t *testing.T) { - if testing.Short() || !*testExternal { - t.Skip("avoid external network") - } + testenv.MustHaveExternalNetwork(t) + if !supportsIPv6 { t.Skip("IPv6 is not supported") } @@ -356,7 +398,7 @@ func TestUDPZeroByteBuffer(t *testing.T) { switch err { case nil: // ReadFrom succeeds default: // Read may timeout, it depends on the platform - if nerr, ok := err.(Error); (!ok || !nerr.Timeout()) && runtime.GOOS != "windows" { // Windows retruns WSAEMSGSIZ + if nerr, ok := err.(Error); (!ok || !nerr.Timeout()) && runtime.GOOS != "windows" { // Windows returns WSAEMSGSIZ t.Fatal(err) } } diff --git a/libgo/go/net/unixsock.go b/libgo/go/net/unixsock.go index eb91d0d630..b25d492f59 100644 --- a/libgo/go/net/unixsock.go +++ b/libgo/go/net/unixsock.go @@ -1,9 +1,17 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package net +import ( + "context" + "os" + "sync" + "syscall" + "time" +) + // UnixAddr represents the address of a Unix domain socket end point. type UnixAddr struct { Name string @@ -45,3 +53,275 @@ func ResolveUnixAddr(net, addr string) (*UnixAddr, error) { return nil, UnknownNetworkError(net) } } + +// UnixConn is an implementation of the Conn interface for connections +// to Unix domain sockets. +type UnixConn struct { + conn +} + +// CloseRead shuts down the reading side of the Unix domain connection. +// Most callers should just use Close. +func (c *UnixConn) CloseRead() error { + if !c.ok() { + return syscall.EINVAL + } + if err := c.fd.closeRead(); err != nil { + return &OpError{Op: "close", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} + } + return nil +} + +// CloseWrite shuts down the writing side of the Unix domain connection. +// Most callers should just use Close. +func (c *UnixConn) CloseWrite() error { + if !c.ok() { + return syscall.EINVAL + } + if err := c.fd.closeWrite(); err != nil { + return &OpError{Op: "close", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} + } + return nil +} + +// ReadFromUnix reads a packet from c, copying the payload into b. It +// returns the number of bytes copied into b and the source address of +// the packet. +// +// ReadFromUnix can be made to time out and return an error with +// Timeout() == true after a fixed time limit; see SetDeadline and +// SetReadDeadline. +func (c *UnixConn) ReadFromUnix(b []byte) (int, *UnixAddr, error) { + if !c.ok() { + return 0, nil, syscall.EINVAL + } + n, addr, err := c.readFrom(b) + if err != nil { + err = &OpError{Op: "read", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} + } + return n, addr, err +} + +// ReadFrom implements the PacketConn ReadFrom method. +func (c *UnixConn) ReadFrom(b []byte) (int, Addr, error) { + if !c.ok() { + return 0, nil, syscall.EINVAL + } + n, addr, err := c.readFrom(b) + if err != nil { + err = &OpError{Op: "read", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} + } + if addr == nil { + return n, nil, err + } + return n, addr, err +} + +// ReadMsgUnix reads a packet from c, copying the payload into b and +// the associated out-of-band data into oob. It returns the number of +// bytes copied into b, the number of bytes copied into oob, the flags +// that were set on the packet, and the source address of the packet. +// +// Note that if len(b) == 0 and len(oob) > 0, this function will still +// read (and discard) 1 byte from the connection. +func (c *UnixConn) ReadMsgUnix(b, oob []byte) (n, oobn, flags int, addr *UnixAddr, err error) { + if !c.ok() { + return 0, 0, 0, nil, syscall.EINVAL + } + n, oobn, flags, addr, err = c.readMsg(b, oob) + if err != nil { + err = &OpError{Op: "read", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} + } + return +} + +// WriteToUnix writes a packet to addr via c, copying the payload from b. +// +// WriteToUnix can be made to time out and return an error with +// Timeout() == true after a fixed time limit; see SetDeadline and +// SetWriteDeadline. On packet-oriented connections, write timeouts +// are rare. +func (c *UnixConn) WriteToUnix(b []byte, addr *UnixAddr) (int, error) { + if !c.ok() { + return 0, syscall.EINVAL + } + n, err := c.writeTo(b, addr) + if err != nil { + err = &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: err} + } + return n, err +} + +// WriteTo implements the PacketConn WriteTo method. +func (c *UnixConn) WriteTo(b []byte, addr Addr) (int, error) { + if !c.ok() { + return 0, syscall.EINVAL + } + a, ok := addr.(*UnixAddr) + if !ok { + return 0, &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr, Err: syscall.EINVAL} + } + n, err := c.writeTo(b, a) + if err != nil { + err = &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: a.opAddr(), Err: err} + } + return n, err +} + +// WriteMsgUnix writes a packet to addr via c, copying the payload +// from b and the associated out-of-band data from oob. It returns +// the number of payload and out-of-band bytes written. +// +// Note that if len(b) == 0 and len(oob) > 0, this function will still +// write 1 byte to the connection. +func (c *UnixConn) WriteMsgUnix(b, oob []byte, addr *UnixAddr) (n, oobn int, err error) { + if !c.ok() { + return 0, 0, syscall.EINVAL + } + n, oobn, err = c.writeMsg(b, oob, addr) + if err != nil { + err = &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: err} + } + return +} + +func newUnixConn(fd *netFD) *UnixConn { return &UnixConn{conn{fd}} } + +// DialUnix connects to the remote address raddr on the network net, +// which must be "unix", "unixgram" or "unixpacket". If laddr is not +// nil, it is used as the local address for the connection. +func DialUnix(net string, laddr, raddr *UnixAddr) (*UnixConn, error) { + switch net { + case "unix", "unixgram", "unixpacket": + default: + return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: UnknownNetworkError(net)} + } + c, err := dialUnix(context.Background(), net, laddr, raddr) + if err != nil { + return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: err} + } + return c, nil +} + +// UnixListener is a Unix domain socket listener. Clients should +// typically use variables of type Listener instead of assuming Unix +// domain sockets. +type UnixListener struct { + fd *netFD + path string + unlink bool + unlinkOnce sync.Once +} + +func (ln *UnixListener) ok() bool { return ln != nil && ln.fd != nil } + +// AcceptUnix accepts the next incoming call and returns the new +// connection. +func (l *UnixListener) AcceptUnix() (*UnixConn, error) { + if !l.ok() { + return nil, syscall.EINVAL + } + c, err := l.accept() + if err != nil { + return nil, &OpError{Op: "accept", Net: l.fd.net, Source: nil, Addr: l.fd.laddr, Err: err} + } + return c, nil +} + +// Accept implements the Accept method in the Listener interface. +// Returned connections will be of type *UnixConn. +func (l *UnixListener) Accept() (Conn, error) { + if !l.ok() { + return nil, syscall.EINVAL + } + c, err := l.accept() + if err != nil { + return nil, &OpError{Op: "accept", Net: l.fd.net, Source: nil, Addr: l.fd.laddr, Err: err} + } + return c, nil +} + +// Close stops listening on the Unix address. Already accepted +// connections are not closed. +func (l *UnixListener) Close() error { + if !l.ok() { + return syscall.EINVAL + } + if err := l.close(); err != nil { + return &OpError{Op: "close", Net: l.fd.net, Source: nil, Addr: l.fd.laddr, Err: err} + } + return nil +} + +// Addr returns the listener's network address. +// The Addr returned is shared by all invocations of Addr, so +// do not modify it. +func (l *UnixListener) Addr() Addr { return l.fd.laddr } + +// SetDeadline sets the deadline associated with the listener. +// A zero time value disables the deadline. +func (l *UnixListener) SetDeadline(t time.Time) error { + if !l.ok() { + return syscall.EINVAL + } + if err := l.fd.setDeadline(t); err != nil { + return &OpError{Op: "set", Net: l.fd.net, Source: nil, Addr: l.fd.laddr, Err: err} + } + return nil +} + +// File returns a copy of the underlying os.File, set to blocking +// mode. It is the caller's responsibility to close f when finished. +// Closing l does not affect f, and closing f does not affect l. +// +// The returned os.File's file descriptor is different from the +// connection's. Attempting to change properties of the original +// using this duplicate may or may not have the desired effect. +func (l *UnixListener) File() (f *os.File, err error) { + if !l.ok() { + return nil, syscall.EINVAL + } + f, err = l.file() + if err != nil { + err = &OpError{Op: "file", Net: l.fd.net, Source: nil, Addr: l.fd.laddr, Err: err} + } + return +} + +// ListenUnix announces on the Unix domain socket laddr and returns a +// Unix listener. The network net must be "unix" or "unixpacket". +func ListenUnix(net string, laddr *UnixAddr) (*UnixListener, error) { + switch net { + case "unix", "unixpacket": + default: + return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: UnknownNetworkError(net)} + } + if laddr == nil { + return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: errMissingAddress} + } + ln, err := listenUnix(context.Background(), net, laddr) + if err != nil { + return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: err} + } + return ln, nil +} + +// ListenUnixgram listens for incoming Unix datagram packets addressed +// to the local address laddr. The network net must be "unixgram". +// The returned connection's ReadFrom and WriteTo methods can be used +// to receive and send packets with per-packet addressing. +func ListenUnixgram(net string, laddr *UnixAddr) (*UnixConn, error) { + switch net { + case "unixgram": + default: + return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: UnknownNetworkError(net)} + } + if laddr == nil { + return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: nil, Err: errMissingAddress} + } + c, err := listenUnixgram(context.Background(), net, laddr) + if err != nil { + return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: err} + } + return c, nil +} diff --git a/libgo/go/net/unixsock_plan9.go b/libgo/go/net/unixsock_plan9.go index 84b6b600f6..e70eb211bb 100644 --- a/libgo/go/net/unixsock_plan9.go +++ b/libgo/go/net/unixsock_plan9.go @@ -1,147 +1,51 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. package net import ( + "context" "os" "syscall" - "time" ) -// UnixConn is an implementation of the Conn interface for connections -// to Unix domain sockets. -type UnixConn struct { - conn +func (c *UnixConn) readFrom(b []byte) (int, *UnixAddr, error) { + return 0, nil, syscall.EPLAN9 } -// ReadFromUnix reads a packet from c, copying the payload into b. It -// returns the number of bytes copied into b and the source address of -// the packet. -// -// ReadFromUnix can be made to time out and return an error with -// Timeout() == true after a fixed time limit; see SetDeadline and -// SetReadDeadline. -func (c *UnixConn) ReadFromUnix(b []byte) (int, *UnixAddr, error) { - return 0, nil, &OpError{Op: "read", Net: c.fd.dir, Source: c.fd.laddr, Addr: c.fd.raddr, Err: syscall.EPLAN9} +func (c *UnixConn) readMsg(b, oob []byte) (n, oobn, flags int, addr *UnixAddr, err error) { + return 0, 0, 0, nil, syscall.EPLAN9 } -// ReadFrom implements the PacketConn ReadFrom method. -func (c *UnixConn) ReadFrom(b []byte) (int, Addr, error) { - return 0, nil, &OpError{Op: "read", Net: c.fd.dir, Source: c.fd.laddr, Addr: c.fd.raddr, Err: syscall.EPLAN9} +func (c *UnixConn) writeTo(b []byte, addr *UnixAddr) (int, error) { + return 0, syscall.EPLAN9 } -// ReadMsgUnix reads a packet from c, copying the payload into b and -// the associated out-of-band data into oob. It returns the number of -// bytes copied into b, the number of bytes copied into oob, the flags -// that were set on the packet, and the source address of the packet. -func (c *UnixConn) ReadMsgUnix(b, oob []byte) (n, oobn, flags int, addr *UnixAddr, err error) { - return 0, 0, 0, nil, &OpError{Op: "read", Net: c.fd.dir, Source: c.fd.laddr, Addr: c.fd.raddr, Err: syscall.EPLAN9} +func (c *UnixConn) writeMsg(b, oob []byte, addr *UnixAddr) (n, oobn int, err error) { + return 0, 0, syscall.EPLAN9 } -// WriteToUnix writes a packet to addr via c, copying the payload from b. -// -// WriteToUnix can be made to time out and return an error with -// Timeout() == true after a fixed time limit; see SetDeadline and -// SetWriteDeadline. On packet-oriented connections, write timeouts -// are rare. -func (c *UnixConn) WriteToUnix(b []byte, addr *UnixAddr) (int, error) { - return 0, &OpError{Op: "write", Net: c.fd.dir, Source: c.fd.laddr, Addr: addr.opAddr(), Err: syscall.EPLAN9} +func dialUnix(ctx context.Context, network string, laddr, raddr *UnixAddr) (*UnixConn, error) { + return nil, syscall.EPLAN9 } -// WriteTo implements the PacketConn WriteTo method. -func (c *UnixConn) WriteTo(b []byte, addr Addr) (int, error) { - return 0, &OpError{Op: "write", Net: c.fd.dir, Source: c.fd.laddr, Addr: addr, Err: syscall.EPLAN9} +func (ln *UnixListener) accept() (*UnixConn, error) { + return nil, syscall.EPLAN9 } -// WriteMsgUnix writes a packet to addr via c, copying the payload -// from b and the associated out-of-band data from oob. It returns -// the number of payload and out-of-band bytes written. -func (c *UnixConn) WriteMsgUnix(b, oob []byte, addr *UnixAddr) (n, oobn int, err error) { - return 0, 0, &OpError{Op: "write", Net: c.fd.dir, Source: c.fd.laddr, Addr: addr.opAddr(), Err: syscall.EPLAN9} +func (ln *UnixListener) close() error { + return syscall.EPLAN9 } -// CloseRead shuts down the reading side of the Unix domain connection. -// Most callers should just use Close. -func (c *UnixConn) CloseRead() error { - return &OpError{Op: "close", Net: c.fd.dir, Source: c.fd.laddr, Addr: c.fd.raddr, Err: syscall.EPLAN9} +func (ln *UnixListener) file() (*os.File, error) { + return nil, syscall.EPLAN9 } -// CloseWrite shuts down the writing side of the Unix domain connection. -// Most callers should just use Close. -func (c *UnixConn) CloseWrite() error { - return &OpError{Op: "close", Net: c.fd.dir, Source: c.fd.laddr, Addr: c.fd.raddr, Err: syscall.EPLAN9} +func listenUnix(ctx context.Context, network string, laddr *UnixAddr) (*UnixListener, error) { + return nil, syscall.EPLAN9 } -// DialUnix connects to the remote address raddr on the network net, -// which must be "unix", "unixgram" or "unixpacket". If laddr is not -// nil, it is used as the local address for the connection. -func DialUnix(net string, laddr, raddr *UnixAddr) (*UnixConn, error) { - return dialUnix(net, laddr, raddr, noDeadline) -} - -func dialUnix(net string, laddr, raddr *UnixAddr, deadline time.Time) (*UnixConn, error) { - return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: syscall.EPLAN9} -} - -// UnixListener is a Unix domain socket listener. Clients should -// typically use variables of type Listener instead of assuming Unix -// domain sockets. -type UnixListener struct { - fd *netFD -} - -// ListenUnix announces on the Unix domain socket laddr and returns a -// Unix listener. The network net must be "unix" or "unixpacket". -func ListenUnix(net string, laddr *UnixAddr) (*UnixListener, error) { - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: syscall.EPLAN9} -} - -// AcceptUnix accepts the next incoming call and returns the new -// connection. -func (l *UnixListener) AcceptUnix() (*UnixConn, error) { - return nil, &OpError{Op: "accept", Net: l.fd.dir, Source: nil, Addr: l.fd.laddr, Err: syscall.EPLAN9} -} - -// Accept implements the Accept method in the Listener interface; it -// waits for the next call and returns a generic Conn. -func (l *UnixListener) Accept() (Conn, error) { - return nil, &OpError{Op: "accept", Net: l.fd.dir, Source: nil, Addr: l.fd.laddr, Err: syscall.EPLAN9} -} - -// Close stops listening on the Unix address. Already accepted -// connections are not closed. -func (l *UnixListener) Close() error { - return &OpError{Op: "close", Net: l.fd.dir, Source: nil, Addr: l.fd.laddr, Err: syscall.EPLAN9} -} - -// Addr returns the listener's network address. -// The Addr returned is shared by all invocations of Addr, so -// do not modify it. -func (l *UnixListener) Addr() Addr { return nil } - -// SetDeadline sets the deadline associated with the listener. -// A zero time value disables the deadline. -func (l *UnixListener) SetDeadline(t time.Time) error { - return &OpError{Op: "set", Net: l.fd.dir, Source: nil, Addr: l.fd.laddr, Err: syscall.EPLAN9} -} - -// File returns a copy of the underlying os.File, set to blocking -// mode. It is the caller's responsibility to close f when finished. -// Closing l does not affect f, and closing f does not affect l. -// -// The returned os.File's file descriptor is different from the -// connection's. Attempting to change properties of the original -// using this duplicate may or may not have the desired effect. -func (l *UnixListener) File() (*os.File, error) { - return nil, &OpError{Op: "file", Net: l.fd.dir, Source: nil, Addr: l.fd.laddr, Err: syscall.EPLAN9} -} - -// ListenUnixgram listens for incoming Unix datagram packets addressed -// to the local address laddr. The network net must be "unixgram". -// The returned connection's ReadFrom and WriteTo methods can be used -// to receive and send packets with per-packet addressing. -func ListenUnixgram(net string, laddr *UnixAddr) (*UnixConn, error) { - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: syscall.EPLAN9} +func listenUnixgram(ctx context.Context, network string, laddr *UnixAddr) (*UnixConn, error) { + return nil, syscall.EPLAN9 } diff --git a/libgo/go/net/unixsock_posix.go b/libgo/go/net/unixsock_posix.go index fb2397e26f..a8f892e6c4 100644 --- a/libgo/go/net/unixsock_posix.go +++ b/libgo/go/net/unixsock_posix.go @@ -1,4 +1,4 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -7,13 +7,13 @@ package net import ( + "context" "errors" "os" "syscall" - "time" ) -func unixSocket(net string, laddr, raddr sockaddr, mode string, deadline time.Time) (*netFD, error) { +func unixSocket(ctx context.Context, net string, laddr, raddr sockaddr, mode string) (*netFD, error) { var sotype int switch net { case "unix": @@ -42,7 +42,7 @@ func unixSocket(net string, laddr, raddr sockaddr, mode string, deadline time.Ti return nil, errors.New("unknown mode: " + mode) } - fd, err := socket(net, syscall.AF_UNIX, sotype, 0, false, laddr, raddr, deadline, noCancel) + fd, err := socket(ctx, net, syscall.AF_UNIX, sotype, 0, false, laddr, raddr) if err != nil { return nil, err } @@ -94,25 +94,11 @@ func (a *UnixAddr) sockaddr(family int) (syscall.Sockaddr, error) { return &syscall.SockaddrUnix{Name: a.Name}, nil } -// UnixConn is an implementation of the Conn interface for connections -// to Unix domain sockets. -type UnixConn struct { - conn +func (a *UnixAddr) toLocal(net string) sockaddr { + return a } -func newUnixConn(fd *netFD) *UnixConn { return &UnixConn{conn{fd}} } - -// ReadFromUnix reads a packet from c, copying the payload into b. It -// returns the number of bytes copied into b and the source address of -// the packet. -// -// ReadFromUnix can be made to time out and return an error with -// Timeout() == true after a fixed time limit; see SetDeadline and -// SetReadDeadline. -func (c *UnixConn) ReadFromUnix(b []byte) (int, *UnixAddr, error) { - if !c.ok() { - return 0, nil, syscall.EINVAL - } +func (c *UnixConn) readFrom(b []byte) (int, *UnixAddr, error) { var addr *UnixAddr n, sa, err := c.fd.readFrom(b) switch sa := sa.(type) { @@ -121,211 +107,66 @@ func (c *UnixConn) ReadFromUnix(b []byte) (int, *UnixAddr, error) { addr = &UnixAddr{Name: sa.Name, Net: sotypeToNet(c.fd.sotype)} } } - if err != nil { - err = &OpError{Op: "read", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} - } - return n, addr, err -} - -// ReadFrom implements the PacketConn ReadFrom method. -func (c *UnixConn) ReadFrom(b []byte) (int, Addr, error) { - if !c.ok() { - return 0, nil, syscall.EINVAL - } - n, addr, err := c.ReadFromUnix(b) - if addr == nil { - return n, nil, err - } return n, addr, err } -// ReadMsgUnix reads a packet from c, copying the payload into b and -// the associated out-of-band data into oob. It returns the number of -// bytes copied into b, the number of bytes copied into oob, the flags -// that were set on the packet, and the source address of the packet. -func (c *UnixConn) ReadMsgUnix(b, oob []byte) (n, oobn, flags int, addr *UnixAddr, err error) { - if !c.ok() { - return 0, 0, 0, nil, syscall.EINVAL - } - n, oobn, flags, sa, err := c.fd.readMsg(b, oob) +func (c *UnixConn) readMsg(b, oob []byte) (n, oobn, flags int, addr *UnixAddr, err error) { + var sa syscall.Sockaddr + n, oobn, flags, sa, err = c.fd.readMsg(b, oob) switch sa := sa.(type) { case *syscall.SockaddrUnix: if sa.Name != "" { addr = &UnixAddr{Name: sa.Name, Net: sotypeToNet(c.fd.sotype)} } } - if err != nil { - err = &OpError{Op: "read", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} - } return } -// WriteToUnix writes a packet to addr via c, copying the payload from b. -// -// WriteToUnix can be made to time out and return an error with -// Timeout() == true after a fixed time limit; see SetDeadline and -// SetWriteDeadline. On packet-oriented connections, write timeouts -// are rare. -func (c *UnixConn) WriteToUnix(b []byte, addr *UnixAddr) (int, error) { - if !c.ok() { - return 0, syscall.EINVAL - } +func (c *UnixConn) writeTo(b []byte, addr *UnixAddr) (int, error) { if c.fd.isConnected { - return 0, &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: ErrWriteToConnected} + return 0, ErrWriteToConnected } if addr == nil { - return 0, &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: nil, Err: errMissingAddress} + return 0, errMissingAddress } if addr.Net != sotypeToNet(c.fd.sotype) { - return 0, &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: syscall.EAFNOSUPPORT} + return 0, syscall.EAFNOSUPPORT } sa := &syscall.SockaddrUnix{Name: addr.Name} - n, err := c.fd.writeTo(b, sa) - if err != nil { - err = &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: err} - } - return n, err -} - -// WriteTo implements the PacketConn WriteTo method. -func (c *UnixConn) WriteTo(b []byte, addr Addr) (n int, err error) { - if !c.ok() { - return 0, syscall.EINVAL - } - a, ok := addr.(*UnixAddr) - if !ok { - return 0, &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr, Err: syscall.EINVAL} - } - return c.WriteToUnix(b, a) + return c.fd.writeTo(b, sa) } -// WriteMsgUnix writes a packet to addr via c, copying the payload -// from b and the associated out-of-band data from oob. It returns -// the number of payload and out-of-band bytes written. -func (c *UnixConn) WriteMsgUnix(b, oob []byte, addr *UnixAddr) (n, oobn int, err error) { - if !c.ok() { - return 0, 0, syscall.EINVAL - } +func (c *UnixConn) writeMsg(b, oob []byte, addr *UnixAddr) (n, oobn int, err error) { if c.fd.sotype == syscall.SOCK_DGRAM && c.fd.isConnected { - return 0, 0, &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: ErrWriteToConnected} + return 0, 0, ErrWriteToConnected } var sa syscall.Sockaddr if addr != nil { if addr.Net != sotypeToNet(c.fd.sotype) { - return 0, 0, &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: syscall.EAFNOSUPPORT} + return 0, 0, syscall.EAFNOSUPPORT } sa = &syscall.SockaddrUnix{Name: addr.Name} } - n, oobn, err = c.fd.writeMsg(b, oob, sa) - if err != nil { - err = &OpError{Op: "write", Net: c.fd.net, Source: c.fd.laddr, Addr: addr.opAddr(), Err: err} - } - return -} - -// CloseRead shuts down the reading side of the Unix domain connection. -// Most callers should just use Close. -func (c *UnixConn) CloseRead() error { - if !c.ok() { - return syscall.EINVAL - } - err := c.fd.closeRead() - if err != nil { - err = &OpError{Op: "close", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} - } - return err -} - -// CloseWrite shuts down the writing side of the Unix domain connection. -// Most callers should just use Close. -func (c *UnixConn) CloseWrite() error { - if !c.ok() { - return syscall.EINVAL - } - err := c.fd.closeWrite() - if err != nil { - err = &OpError{Op: "close", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} - } - return err -} - -// DialUnix connects to the remote address raddr on the network net, -// which must be "unix", "unixgram" or "unixpacket". If laddr is not -// nil, it is used as the local address for the connection. -func DialUnix(net string, laddr, raddr *UnixAddr) (*UnixConn, error) { - switch net { - case "unix", "unixgram", "unixpacket": - default: - return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: UnknownNetworkError(net)} - } - return dialUnix(net, laddr, raddr, noDeadline) + return c.fd.writeMsg(b, oob, sa) } -func dialUnix(net string, laddr, raddr *UnixAddr, deadline time.Time) (*UnixConn, error) { - fd, err := unixSocket(net, laddr, raddr, "dial", deadline) +func dialUnix(ctx context.Context, net string, laddr, raddr *UnixAddr) (*UnixConn, error) { + fd, err := unixSocket(ctx, net, laddr, raddr, "dial") if err != nil { - return nil, &OpError{Op: "dial", Net: net, Source: laddr.opAddr(), Addr: raddr.opAddr(), Err: err} - } - return newUnixConn(fd), nil -} - -// UnixListener is a Unix domain socket listener. Clients should -// typically use variables of type Listener instead of assuming Unix -// domain sockets. -type UnixListener struct { - fd *netFD - path string - unlink bool -} - -// ListenUnix announces on the Unix domain socket laddr and returns a -// Unix listener. The network net must be "unix" or "unixpacket". -func ListenUnix(net string, laddr *UnixAddr) (*UnixListener, error) { - switch net { - case "unix", "unixpacket": - default: - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: UnknownNetworkError(net)} - } - if laddr == nil { - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: errMissingAddress} - } - fd, err := unixSocket(net, laddr, nil, "listen", noDeadline) - if err != nil { - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: err} - } - return &UnixListener{fd: fd, path: fd.laddr.String(), unlink: true}, nil -} - -// AcceptUnix accepts the next incoming call and returns the new -// connection. -func (l *UnixListener) AcceptUnix() (*UnixConn, error) { - if l == nil || l.fd == nil { - return nil, syscall.EINVAL - } - fd, err := l.fd.accept() - if err != nil { - return nil, &OpError{Op: "accept", Net: l.fd.net, Source: nil, Addr: l.fd.laddr, Err: err} + return nil, err } return newUnixConn(fd), nil } -// Accept implements the Accept method in the Listener interface; it -// waits for the next call and returns a generic Conn. -func (l *UnixListener) Accept() (c Conn, err error) { - c1, err := l.AcceptUnix() +func (ln *UnixListener) accept() (*UnixConn, error) { + fd, err := ln.fd.accept() if err != nil { return nil, err } - return c1, nil + return newUnixConn(fd), nil } -// Close stops listening on the Unix address. Already accepted -// connections are not closed. -func (l *UnixListener) Close() error { - if l == nil || l.fd == nil { - return syscall.EINVAL - } - +func (ln *UnixListener) close() error { // The operating system doesn't clean up // the file that announcing created, so // we have to clean it up ourselves. @@ -334,66 +175,49 @@ func (l *UnixListener) Close() error { // and replaced our socket name already-- // but this sequence (remove then close) // is at least compatible with the auto-remove - // sequence in ListenUnix. It's only non-Go + // sequence in ListenUnix. It's only non-Go // programs that can mess us up. - if l.path[0] != '@' && l.unlink { - syscall.Unlink(l.path) - } - err := l.fd.Close() - if err != nil { - err = &OpError{Op: "close", Net: l.fd.net, Source: l.fd.laddr, Addr: l.fd.raddr, Err: err} - } - return err + // Even if there are racy calls to Close, we want to unlink only for the first one. + ln.unlinkOnce.Do(func() { + if ln.path[0] != '@' && ln.unlink { + syscall.Unlink(ln.path) + } + }) + return ln.fd.Close() } -// Addr returns the listener's network address. -// The Addr returned is shared by all invocations of Addr, so -// do not modify it. -func (l *UnixListener) Addr() Addr { return l.fd.laddr } - -// SetDeadline sets the deadline associated with the listener. -// A zero time value disables the deadline. -func (l *UnixListener) SetDeadline(t time.Time) error { - if l == nil || l.fd == nil { - return syscall.EINVAL - } - if err := l.fd.setDeadline(t); err != nil { - return &OpError{Op: "set", Net: l.fd.net, Source: nil, Addr: l.fd.laddr, Err: err} +func (ln *UnixListener) file() (*os.File, error) { + f, err := ln.fd.dup() + if err != nil { + return nil, err } - return nil + return f, nil } -// File returns a copy of the underlying os.File, set to blocking -// mode. It is the caller's responsibility to close f when finished. -// Closing l does not affect f, and closing f does not affect l. +// SetUnlinkOnClose sets whether the underlying socket file should be removed +// from the file system when the listener is closed. // -// The returned os.File's file descriptor is different from the -// connection's. Attempting to change properties of the original -// using this duplicate may or may not have the desired effect. -func (l *UnixListener) File() (f *os.File, err error) { - f, err = l.fd.dup() +// The default behavior is to unlink the socket file only when package net created it. +// That is, when the listener and the underlying socket file were created by a call to +// Listen or ListenUnix, then by default closing the listener will remove the socket file. +// but if the listener was created by a call to FileListener to use an already existing +// socket file, then by default closing the listener will not remove the socket file. +func (l *UnixListener) SetUnlinkOnClose(unlink bool) { + l.unlink = unlink +} + +func listenUnix(ctx context.Context, network string, laddr *UnixAddr) (*UnixListener, error) { + fd, err := unixSocket(ctx, network, laddr, nil, "listen") if err != nil { - err = &OpError{Op: "file", Net: l.fd.net, Source: nil, Addr: l.fd.laddr, Err: err} + return nil, err } - return + return &UnixListener{fd: fd, path: fd.laddr.String(), unlink: true}, nil } -// ListenUnixgram listens for incoming Unix datagram packets addressed -// to the local address laddr. The network net must be "unixgram". -// The returned connection's ReadFrom and WriteTo methods can be used -// to receive and send packets with per-packet addressing. -func ListenUnixgram(net string, laddr *UnixAddr) (*UnixConn, error) { - switch net { - case "unixgram": - default: - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: UnknownNetworkError(net)} - } - if laddr == nil { - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: nil, Err: errMissingAddress} - } - fd, err := unixSocket(net, laddr, nil, "listen", noDeadline) +func listenUnixgram(ctx context.Context, network string, laddr *UnixAddr) (*UnixConn, error) { + fd, err := unixSocket(ctx, network, laddr, nil, "listen") if err != nil { - return nil, &OpError{Op: "listen", Net: net, Source: nil, Addr: laddr.opAddr(), Err: err} + return nil, err } return newUnixConn(fd), nil } diff --git a/libgo/go/net/unix_test.go b/libgo/go/net/unixsock_test.go index f0c583068e..489a29bc7d 100644 --- a/libgo/go/net/unix_test.go +++ b/libgo/go/net/unixsock_test.go @@ -1,4 +1,4 @@ -// Copyright 2013 The Go Authors. All rights reserved. +// Copyright 2013 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -8,6 +8,8 @@ package net import ( "bytes" + "internal/testenv" + "io/ioutil" "os" "reflect" "runtime" @@ -20,6 +22,9 @@ func TestReadUnixgramWithUnnamedSocket(t *testing.T) { if !testableNetwork("unixgram") { t.Skip("unixgram test") } + if runtime.GOOS == "openbsd" { + testenv.SkipFlaky(t, 15157) + } addr := testUnixAddr() la, err := ResolveUnixAddr("unixgram", addr) @@ -410,64 +415,104 @@ func TestUnixUnlink(t *testing.T) { t.Skip("unix test") } name := testUnixAddr() - l, err := Listen("unix", name) - if err != nil { - t.Fatal(err) - } - if _, err := os.Stat(name); err != nil { - t.Fatalf("cannot stat unix socket after ListenUnix: %v", err) - } - f, _ := l.(*UnixListener).File() - l1, err := FileListener(f) - if err != nil { - t.Fatal(err) - } - if _, err := os.Stat(name); err != nil { - t.Fatalf("cannot stat unix socket after FileListener: %v", err) - } - if err := l1.Close(); err != nil { - t.Fatalf("closing file listener: %v", err) - } - if _, err := os.Stat(name); err != nil { - t.Fatalf("cannot stat unix socket after closing FileListener: %v", err) + + listen := func(t *testing.T) *UnixListener { + l, err := Listen("unix", name) + if err != nil { + t.Fatal(err) + } + return l.(*UnixListener) } - f.Close() - if _, err := os.Stat(name); err != nil { - t.Fatalf("cannot stat unix socket after closing FileListener and fd: %v", err) + checkExists := func(t *testing.T, desc string) { + if _, err := os.Stat(name); err != nil { + t.Fatalf("unix socket does not exist %s: %v", desc, err) + } } - l.Close() - if _, err := os.Stat(name); err == nil { - t.Fatal("closing unix listener did not remove unix socket") + checkNotExists := func(t *testing.T, desc string) { + if _, err := os.Stat(name); err == nil { + t.Fatalf("unix socket does exist %s: %v", desc, err) + } } -} -// forceGoDNS forces the resolver configuration to use the pure Go resolver -// and returns a fixup function to restore the old settings. -func forceGoDNS() func() { - c := systemConf() - oldGo := c.netGo - oldCgo := c.netCgo - fixup := func() { - c.netGo = oldGo - c.netCgo = oldCgo - } - c.netGo = true - c.netCgo = false - return fixup -} - -// forceCgoDNS forces the resolver configuration to use the cgo resolver -// and returns a fixup function to restore the old settings. -// (On non-Unix systems forceCgoDNS returns nil.) -func forceCgoDNS() func() { - c := systemConf() - oldGo := c.netGo - oldCgo := c.netCgo - fixup := func() { - c.netGo = oldGo - c.netCgo = oldCgo - } - c.netGo = false - c.netCgo = true - return fixup + // Listener should remove on close. + t.Run("Listen", func(t *testing.T) { + l := listen(t) + checkExists(t, "after Listen") + l.Close() + checkNotExists(t, "after Listener close") + }) + + // FileListener should not. + t.Run("FileListener", func(t *testing.T) { + l := listen(t) + f, _ := l.File() + l1, _ := FileListener(f) + checkExists(t, "after FileListener") + f.Close() + checkExists(t, "after File close") + l1.Close() + checkExists(t, "after FileListener close") + l.Close() + checkNotExists(t, "after Listener close") + }) + + // Only first call to l.Close should remove. + t.Run("SecondClose", func(t *testing.T) { + l := listen(t) + checkExists(t, "after Listen") + l.Close() + checkNotExists(t, "after Listener close") + if err := ioutil.WriteFile(name, []byte("hello world"), 0666); err != nil { + t.Fatalf("cannot recreate socket file: %v", err) + } + checkExists(t, "after writing temp file") + l.Close() + checkExists(t, "after second Listener close") + os.Remove(name) + }) + + // SetUnlinkOnClose should do what it says. + + t.Run("Listen/SetUnlinkOnClose(true)", func(t *testing.T) { + l := listen(t) + checkExists(t, "after Listen") + l.SetUnlinkOnClose(true) + l.Close() + checkNotExists(t, "after Listener close") + }) + + t.Run("Listen/SetUnlinkOnClose(false)", func(t *testing.T) { + l := listen(t) + checkExists(t, "after Listen") + l.SetUnlinkOnClose(false) + l.Close() + checkExists(t, "after Listener close") + os.Remove(name) + }) + + t.Run("FileListener/SetUnlinkOnClose(true)", func(t *testing.T) { + l := listen(t) + f, _ := l.File() + l1, _ := FileListener(f) + checkExists(t, "after FileListener") + l1.(*UnixListener).SetUnlinkOnClose(true) + f.Close() + checkExists(t, "after File close") + l1.Close() + checkNotExists(t, "after FileListener close") + l.Close() + }) + + t.Run("FileListener/SetUnlinkOnClose(false)", func(t *testing.T) { + l := listen(t) + f, _ := l.File() + l1, _ := FileListener(f) + checkExists(t, "after FileListener") + l1.(*UnixListener).SetUnlinkOnClose(false) + f.Close() + checkExists(t, "after File close") + l1.Close() + checkExists(t, "after FileListener close") + l.Close() + }) } diff --git a/libgo/go/net/url/url.go b/libgo/go/net/url/url.go index 1a93e3496e..42a514bbc1 100644 --- a/libgo/go/net/url/url.go +++ b/libgo/go/net/url/url.go @@ -3,9 +3,13 @@ // license that can be found in the LICENSE file. // Package url parses URLs and implements query escaping. -// See RFC 3986. package url +// See RFC 3986. This package generally follows RFC 3986, except where +// it deviates for compatibility reasons. When sending changes, first +// search old issues for history on decisions. Unit tests should also +// contain references to issue numbers with details. + import ( "bytes" "errors" @@ -70,6 +74,7 @@ type encoding int const ( encodePath encoding = 1 + iota + encodePathSegment encodeHost encodeZone encodeUserPassword @@ -128,9 +133,14 @@ func shouldEscape(c byte, mode encoding) bool { // The RFC allows : @ & = + $ but saves / ; , for assigning // meaning to individual path segments. This package // only manipulates the path as a whole, so we allow those - // last two as well. That leaves only ? to escape. + // last three as well. That leaves only ? to escape. return c == '?' + case encodePathSegment: // §3.3 + // The RFC allows : @ & = + $ but saves / ; , for assigning + // meaning to individual path segments. + return c == '/' || c == ';' || c == ',' || c == '?' + case encodeUserPassword: // §3.2.1 // The RFC allows ';', ':', '&', '=', '+', '$', and ',' in // userinfo, so we must escape only '@', '/', and '?'. @@ -160,6 +170,15 @@ func QueryUnescape(s string) (string, error) { return unescape(s, encodeQueryComponent) } +// PathUnescape does the inverse transformation of PathEscape, converting +// %AB into the byte 0xAB. It returns an error if any % is not followed by +// two hexadecimal digits. +// +// PathUnescape is identical to QueryUnescape except that it does not unescape '+' to ' ' (space). +func PathUnescape(s string) (string, error) { + return unescape(s, encodePathSegment) +} + // unescape unescapes a string; the mode specifies // which section of the URL string is being unescaped. func unescape(s string, mode encoding) (string, error) { @@ -246,6 +265,12 @@ func QueryEscape(s string) string { return escape(s, encodeQueryComponent) } +// PathEscape escapes the string so it can be safely placed +// inside a URL path segment. +func PathEscape(s string) string { + return escape(s, encodePathSegment) +} + func escape(s string, mode encoding) string { spaceCount, hexCount := 0, 0 for i := 0; i < len(s); i++ { @@ -307,14 +332,15 @@ func escape(s string, mode encoding) string { // construct a URL struct directly and set the Opaque field instead of Path. // These still work as well. type URL struct { - Scheme string - Opaque string // encoded opaque data - User *Userinfo // username and password information - Host string // host or host:port - Path string - RawPath string // encoded path hint (Go 1.5 and later only; see EscapedPath method) - RawQuery string // encoded query values, without '?' - Fragment string // fragment for references, without '#' + Scheme string + Opaque string // encoded opaque data + User *Userinfo // username and password information + Host string // host or host:port + Path string + RawPath string // encoded path hint (Go 1.5 and later only; see EscapedPath method) + ForceQuery bool // append a query ('?') even if RawQuery is empty + RawQuery string // encoded query values, without '?' + Fragment string // fragment for references, without '#' } // User returns a Userinfo containing the provided username @@ -351,10 +377,7 @@ func (u *Userinfo) Username() string { // Password returns the password in case it is set, and whether it is set. func (u *Userinfo) Password() (string, bool) { - if u.passwordSet { - return u.password, true - } - return "", false + return u.password, u.passwordSet } // String returns the encoded userinfo information in the standard form @@ -410,11 +433,12 @@ func split(s string, c string, cutc bool) (string, string) { // Parse parses rawurl into a URL structure. // The rawurl may be relative or absolute. -func Parse(rawurl string) (url *URL, err error) { +func Parse(rawurl string) (*URL, error) { // Cut off #frag u, frag := split(rawurl, "#", true) - if url, err = parse(u, false); err != nil { - return nil, err + url, err := parse(u, false) + if err != nil { + return nil, &Error{"parse", u, err} } if frag == "" { return url, nil @@ -425,41 +449,50 @@ func Parse(rawurl string) (url *URL, err error) { return url, nil } -// ParseRequestURI parses rawurl into a URL structure. It assumes that +// ParseRequestURI parses rawurl into a URL structure. It assumes that // rawurl was received in an HTTP request, so the rawurl is interpreted // only as an absolute URI or an absolute path. // The string rawurl is assumed not to have a #fragment suffix. // (Web browsers strip #fragment before sending the URL to a web server.) -func ParseRequestURI(rawurl string) (url *URL, err error) { - return parse(rawurl, true) +func ParseRequestURI(rawurl string) (*URL, error) { + url, err := parse(rawurl, true) + if err != nil { + return nil, &Error{"parse", rawurl, err} + } + return url, nil } -// parse parses a URL from a string in one of two contexts. If +// parse parses a URL from a string in one of two contexts. If // viaRequest is true, the URL is assumed to have arrived via an HTTP request, // in which case only absolute URLs or path-absolute relative URLs are allowed. // If viaRequest is false, all forms of relative URLs are allowed. -func parse(rawurl string, viaRequest bool) (url *URL, err error) { +func parse(rawurl string, viaRequest bool) (*URL, error) { var rest string + var err error if rawurl == "" && viaRequest { - err = errors.New("empty url") - goto Error + return nil, errors.New("empty url") } - url = new(URL) + url := new(URL) if rawurl == "*" { url.Path = "*" - return + return url, nil } // Split off possible leading "http:", "mailto:", etc. // Cannot contain escaped characters. if url.Scheme, rest, err = getscheme(rawurl); err != nil { - goto Error + return nil, err } url.Scheme = strings.ToLower(url.Scheme) - rest, url.RawQuery = split(rest, "?", true) + if strings.HasSuffix(rest, "?") && strings.Count(rest, "?") == 1 { + url.ForceQuery = true + rest = rest[:len(rest)-1] + } else { + rest, url.RawQuery = split(rest, "?", true) + } if !strings.HasPrefix(rest, "/") { if url.Scheme != "" { @@ -468,8 +501,20 @@ func parse(rawurl string, viaRequest bool) (url *URL, err error) { return url, nil } if viaRequest { - err = errors.New("invalid URI for request") - goto Error + return nil, errors.New("invalid URI for request") + } + + // Avoid confusion with malformed schemes, like cache_object:foo/bar. + // See golang.org/issue/16822. + // + // RFC 3986, §3.3: + // In addition, a URI reference (Section 4.1) may be a relative-path reference, + // in which case the first path segment cannot contain a colon (":") character. + colon := strings.Index(rest, ":") + slash := strings.Index(rest, "/") + if colon >= 0 && (slash < 0 || colon < slash) { + // First path segment has colon. Not allowed in relative URL. + return nil, errors.New("first path segment in URL cannot contain colon") } } @@ -478,23 +523,17 @@ func parse(rawurl string, viaRequest bool) (url *URL, err error) { authority, rest = split(rest[2:], "/", false) url.User, url.Host, err = parseAuthority(authority) if err != nil { - goto Error + return nil, err } } - if url.Path, err = unescape(rest, encodePath); err != nil { - goto Error - } - // RawPath is a hint as to the encoding of Path to use - // in url.EscapedPath. If that method already gets the - // right answer without RawPath, leave it empty. - // This will help make sure that people don't rely on it in general. - if url.EscapedPath() != rest && validEncodedPath(rest) { - url.RawPath = rest + // Set Path and, optionally, RawPath. + // RawPath is a hint of the encoding of Path. We don't want to set it if + // the default escaping of Path is equivalent, to help make sure that people + // don't rely on it in general. + if err := url.setPath(rest); err != nil { + return nil, err } return url, nil - -Error: - return nil, &Error{"parse", rawurl, err} } func parseAuthority(authority string) (user *Userinfo, host string, err error) { @@ -511,7 +550,7 @@ func parseAuthority(authority string) (user *Userinfo, host string, err error) { return nil, host, nil } userinfo := authority[:i] - if strings.Index(userinfo, ":") < 0 { + if !strings.Contains(userinfo, ":") { if userinfo, err = unescape(userinfo, encodeUserPassword); err != nil { return nil, "", err } @@ -575,6 +614,29 @@ func parseHost(host string) (string, error) { return host, nil } +// setPath sets the Path and RawPath fields of the URL based on the provided +// escaped path p. It maintains the invariant that RawPath is only specified +// when it differs from the default encoding of the path. +// For example: +// - setPath("/foo/bar") will set Path="/foo/bar" and RawPath="" +// - setPath("/foo%2fbar") will set Path="/foo/bar" and RawPath="/foo%2fbar" +// setPath will return an error only if the provided path contains an invalid +// escaping. +func (u *URL) setPath(p string) error { + path, err := unescape(p, encodePath) + if err != nil { + return err + } + u.Path = path + if escp := escape(path, encodePath); p == escp { + // Default encoding is fine. + u.RawPath = "" + } else { + u.RawPath = p + } + return nil +} + // EscapedPath returns the escaped form of u.Path. // In general there are multiple possible escaped forms of any path. // EscapedPath returns u.RawPath when it is a valid escaping of u.Path. @@ -682,9 +744,20 @@ func (u *URL) String() string { if path != "" && path[0] != '/' && u.Host != "" { buf.WriteByte('/') } + if buf.Len() == 0 { + // RFC 3986 §4.2 + // A path segment that contains a colon character (e.g., "this:that") + // cannot be used as the first segment of a relative-path reference, as + // it would be mistaken for a scheme name. Such a segment must be + // preceded by a dot-segment (e.g., "./this:that") to make a relative- + // path reference. + if i := strings.IndexByte(path, ':'); i > -1 && strings.IndexByte(path[:i], '/') == -1 { + buf.WriteString("./") + } + } buf.WriteString(path) } - if u.RawQuery != "" { + if u.ForceQuery || u.RawQuery != "" { buf.WriteByte('?') buf.WriteString(u.RawQuery) } @@ -709,8 +782,8 @@ func (v Values) Get(key string) string { if v == nil { return "" } - vs, ok := v[key] - if !ok || len(vs) == 0 { + vs := v[key] + if len(vs) == 0 { return "" } return vs[0] @@ -738,10 +811,14 @@ func (v Values) Del(key string) { // ParseQuery always returns a non-nil map containing all the // valid query parameters found; err describes the first decoding error // encountered, if any. -func ParseQuery(query string) (m Values, err error) { - m = make(Values) - err = parseQuery(m, query) - return +// +// Query is expected to be a list of key=value settings separated by +// ampersands or semicolons. A setting without an equals sign is +// interpreted as a key set to an empty value. +func ParseQuery(query string) (Values, error) { + m := make(Values) + err := parseQuery(m, query) + return m, err } func parseQuery(m Values, query string) (err error) { @@ -841,12 +918,13 @@ func resolvePath(base, ref string) string { } // IsAbs reports whether the URL is absolute. +// Absolute means that it has a non-empty scheme. func (u *URL) IsAbs() bool { return u.Scheme != "" } -// Parse parses a URL in the context of the receiver. The provided URL -// may be relative or absolute. Parse returns nil, err on parse +// Parse parses a URL in the context of the receiver. The provided URL +// may be relative or absolute. Parse returns nil, err on parse // failure, otherwise its return value is the same as ResolveReference. func (u *URL) Parse(ref string) (*URL, error) { refurl, err := Parse(ref) @@ -858,7 +936,7 @@ func (u *URL) Parse(ref string) (*URL, error) { // ResolveReference resolves a URI reference to an absolute URI from // an absolute base URI, per RFC 3986 Section 5.2. The URI reference -// may be relative or absolute. ResolveReference always returns a new +// may be relative or absolute. ResolveReference always returns a new // URL instance, even if the returned URL is identical to either the // base or reference. If ref is an absolute URL, then ResolveReference // ignores base and returns a copy of ref. @@ -869,7 +947,9 @@ func (u *URL) ResolveReference(ref *URL) *URL { } if ref.Scheme != "" || ref.Host != "" || ref.User != nil { // The "absoluteURI" or "net_path" cases. - url.Path = resolvePath(ref.Path, "") + // We can ignore the error from setPath since we know we provided a + // validly-escaped path. + url.setPath(resolvePath(ref.EscapedPath(), "")) return &url } if ref.Opaque != "" { @@ -889,7 +969,7 @@ func (u *URL) ResolveReference(ref *URL) *URL { // The "abs_path" or "rel_path" cases. url.Host = u.Host url.User = u.User - url.Path = resolvePath(u.Path, ref.Path) + url.setPath(resolvePath(u.EscapedPath(), ref.EscapedPath())) return &url } @@ -913,8 +993,64 @@ func (u *URL) RequestURI() string { result = u.Scheme + ":" + result } } - if u.RawQuery != "" { + if u.ForceQuery || u.RawQuery != "" { result += "?" + u.RawQuery } return result } + +// Hostname returns u.Host, without any port number. +// +// If Host is an IPv6 literal with a port number, Hostname returns the +// IPv6 literal without the square brackets. IPv6 literals may include +// a zone identifier. +func (u *URL) Hostname() string { + return stripPort(u.Host) +} + +// Port returns the port part of u.Host, without the leading colon. +// If u.Host doesn't contain a port, Port returns an empty string. +func (u *URL) Port() string { + return portOnly(u.Host) +} + +func stripPort(hostport string) string { + colon := strings.IndexByte(hostport, ':') + if colon == -1 { + return hostport + } + if i := strings.IndexByte(hostport, ']'); i != -1 { + return strings.TrimPrefix(hostport[:i], "[") + } + return hostport[:colon] +} + +func portOnly(hostport string) string { + colon := strings.IndexByte(hostport, ':') + if colon == -1 { + return "" + } + if i := strings.Index(hostport, "]:"); i != -1 { + return hostport[i+len("]:"):] + } + if strings.Contains(hostport, "]") { + return "" + } + return hostport[colon+len(":"):] +} + +// Marshaling interface implementations. +// Would like to implement MarshalText/UnmarshalText but that will change the JSON representation of URLs. + +func (u *URL) MarshalBinary() (text []byte, err error) { + return []byte(u.String()), nil +} + +func (u *URL) UnmarshalBinary(text []byte) error { + u1, err := Parse(string(text)) + if err != nil { + return err + } + *u = *u1 + return nil +} diff --git a/libgo/go/net/url/url_test.go b/libgo/go/net/url/url_test.go index d3f8487bd7..6c3bb21d20 100644 --- a/libgo/go/net/url/url_test.go +++ b/libgo/go/net/url/url_test.go @@ -5,6 +5,10 @@ package url import ( + "bytes" + encodingPkg "encoding" + "encoding/gob" + "encoding/json" "fmt" "io" "net" @@ -72,6 +76,28 @@ var urltests = []URLTest{ }, "ftp://john%20doe@www.google.com/", }, + // empty query + { + "http://www.google.com/?", + &URL{ + Scheme: "http", + Host: "www.google.com", + Path: "/", + ForceQuery: true, + }, + "", + }, + // query ending in question mark (Issue 14573) + { + "http://www.google.com/?foo=bar?", + &URL{ + Scheme: "http", + Host: "www.google.com", + Path: "/", + RawQuery: "foo=bar?", + }, + "", + }, // query { "http://www.google.com/?q=go+language", @@ -553,22 +579,8 @@ func ufmt(u *URL) string { pass = p } } - return fmt.Sprintf("opaque=%q, scheme=%q, user=%#v, pass=%#v, host=%q, path=%q, rawpath=%q, rawq=%q, frag=%q", - u.Opaque, u.Scheme, user, pass, u.Host, u.Path, u.RawPath, u.RawQuery, u.Fragment) -} - -func DoTest(t *testing.T, parse func(string) (*URL, error), name string, tests []URLTest) { - for _, tt := range tests { - u, err := parse(tt.in) - if err != nil { - t.Errorf("%s(%q) returned error %s", name, tt.in, err) - continue - } - if !reflect.DeepEqual(u, tt.out) { - t.Errorf("%s(%q):\n\thave %v\n\twant %v\n", - name, tt.in, ufmt(u), ufmt(tt.out)) - } - } + return fmt.Sprintf("opaque=%q, scheme=%q, user=%#v, pass=%#v, host=%q, path=%q, rawpath=%q, rawq=%q, frag=%q, forcequery=%v", + u.Opaque, u.Scheme, user, pass, u.Host, u.Path, u.RawPath, u.RawQuery, u.Fragment, u.ForceQuery) } func BenchmarkString(b *testing.B) { @@ -589,14 +601,23 @@ func BenchmarkString(b *testing.B) { g = u.String() } b.StopTimer() - if w := tt.roundtrip; g != w { + if w := tt.roundtrip; b.N > 0 && g != w { b.Errorf("Parse(%q).String() == %q, want %q", tt.in, g, w) } } } func TestParse(t *testing.T) { - DoTest(t, Parse, "Parse", urltests) + for _, tt := range urltests { + u, err := Parse(tt.in) + if err != nil { + t.Errorf("Parse(%q) returned error %v", tt.in, err) + continue + } + if !reflect.DeepEqual(u, tt.out) { + t.Errorf("Parse(%q):\n\tgot %v\n\twant %v\n", tt.in, ufmt(u), ufmt(tt.out)) + } + } } const pathThatLooksSchemeRelative = "//not.a.user@not.a.host/just/a/path" @@ -643,9 +664,10 @@ var parseRequestURLTests = []struct { func TestParseRequestURI(t *testing.T) { for _, test := range parseRequestURLTests { _, err := ParseRequestURI(test.url) - valid := err == nil - if valid != test.expectedValid { - t.Errorf("Expected valid=%v for %q; got %v", test.expectedValid, test.url, valid) + if test.expectedValid && err != nil { + t.Errorf("ParseRequestURI(%q) gave err %v; want no error", test.url, err) + } else if !test.expectedValid && err == nil { + t.Errorf("ParseRequestURI(%q) gave nil error; want some error", test.url) } } @@ -654,45 +676,69 @@ func TestParseRequestURI(t *testing.T) { t.Fatalf("Unexpected error %v", err) } if url.Path != pathThatLooksSchemeRelative { - t.Errorf("Expected path %q; got %q", pathThatLooksSchemeRelative, url.Path) + t.Errorf("ParseRequestURI path:\ngot %q\nwant %q", url.Path, pathThatLooksSchemeRelative) } } -func DoTestString(t *testing.T, parse func(string) (*URL, error), name string, tests []URLTest) { - for _, tt := range tests { - u, err := parse(tt.in) +var stringURLTests = []struct { + url URL + want string +}{ + // No leading slash on path should prepend slash on String() call + { + url: URL{ + Scheme: "http", + Host: "www.google.com", + Path: "search", + }, + want: "http://www.google.com/search", + }, + // Relative path with first element containing ":" should be prepended with "./", golang.org/issue/17184 + { + url: URL{ + Path: "this:that", + }, + want: "./this:that", + }, + // Relative path with second element containing ":" should not be prepended with "./" + { + url: URL{ + Path: "here/this:that", + }, + want: "here/this:that", + }, + // Non-relative path with first element containing ":" should not be prepended with "./" + { + url: URL{ + Scheme: "http", + Host: "www.google.com", + Path: "this:that", + }, + want: "http://www.google.com/this:that", + }, +} + +func TestURLString(t *testing.T) { + for _, tt := range urltests { + u, err := Parse(tt.in) if err != nil { - t.Errorf("%s(%q) returned error %s", name, tt.in, err) + t.Errorf("Parse(%q) returned error %s", tt.in, err) continue } expected := tt.in - if len(tt.roundtrip) > 0 { + if tt.roundtrip != "" { expected = tt.roundtrip } s := u.String() if s != expected { - t.Errorf("%s(%q).String() == %q (expected %q)", name, tt.in, s, expected) + t.Errorf("Parse(%q).String() == %q (expected %q)", tt.in, s, expected) } } -} -func TestURLString(t *testing.T) { - DoTestString(t, Parse, "Parse", urltests) - - // no leading slash on path should prepend - // slash on String() call - noslash := URLTest{ - "http://www.google.com/search", - &URL{ - Scheme: "http", - Host: "www.google.com", - Path: "search", - }, - "", - } - s := noslash.out.String() - if s != noslash.in { - t.Errorf("Expected %s; go %s", noslash.in, s) + for _, tt := range stringURLTests { + if got := tt.url.String(); got != tt.want { + t.Errorf("%+v.String() = %q; want %q", tt.url, got, tt.want) + } } } @@ -758,6 +804,16 @@ var unescapeTests = []EscapeTest{ "", EscapeError("%zz"), }, + { + "a+b", + "a b", + nil, + }, + { + "a%20b", + "a b", + nil, + }, } func TestUnescape(t *testing.T) { @@ -766,10 +822,33 @@ func TestUnescape(t *testing.T) { if actual != tt.out || (err != nil) != (tt.err != nil) { t.Errorf("QueryUnescape(%q) = %q, %s; want %q, %s", tt.in, actual, err, tt.out, tt.err) } + + in := tt.in + out := tt.out + if strings.Contains(tt.in, "+") { + in = strings.Replace(tt.in, "+", "%20", -1) + actual, err := PathUnescape(in) + if actual != tt.out || (err != nil) != (tt.err != nil) { + t.Errorf("PathUnescape(%q) = %q, %s; want %q, %s", in, actual, err, tt.out, tt.err) + } + if tt.err == nil { + s, err := QueryUnescape(strings.Replace(tt.in, "+", "XXX", -1)) + if err != nil { + continue + } + in = tt.in + out = strings.Replace(s, "XXX", "+", -1) + } + } + + actual, err = PathUnescape(in) + if actual != out || (err != nil) != (tt.err != nil) { + t.Errorf("PathUnescape(%q) = %q, %s; want %q, %s", in, actual, err, out, tt.err) + } } } -var escapeTests = []EscapeTest{ +var queryEscapeTests = []EscapeTest{ { "", "", @@ -797,8 +876,8 @@ var escapeTests = []EscapeTest{ }, } -func TestEscape(t *testing.T) { - for _, tt := range escapeTests { +func TestQueryEscape(t *testing.T) { + for _, tt := range queryEscapeTests { actual := QueryEscape(tt.in) if tt.out != actual { t.Errorf("QueryEscape(%q) = %q, want %q", tt.in, actual, tt.out) @@ -812,6 +891,54 @@ func TestEscape(t *testing.T) { } } +var pathEscapeTests = []EscapeTest{ + { + "", + "", + nil, + }, + { + "abc", + "abc", + nil, + }, + { + "abc+def", + "abc+def", + nil, + }, + { + "one two", + "one%20two", + nil, + }, + { + "10%", + "10%25", + nil, + }, + { + " ?&=#+%!<>#\"{}|\\^[]`☺\t:/@$'()*,;", + "%20%3F&=%23+%25%21%3C%3E%23%22%7B%7D%7C%5C%5E%5B%5D%60%E2%98%BA%09:%2F@$%27%28%29%2A%2C%3B", + nil, + }, +} + +func TestPathEscape(t *testing.T) { + for _, tt := range pathEscapeTests { + actual := PathEscape(tt.in) + if tt.out != actual { + t.Errorf("PathEscape(%q) = %q, want %q", tt.in, actual, tt.out) + } + + // for bonus points, verify that escape:unescape is an identity. + roundtrip, err := PathUnescape(actual) + if roundtrip != tt.in || err != nil { + t.Errorf("PathUnescape(%q) = %q, %s; want %q, %s", actual, roundtrip, err, tt.in, "[no error]") + } + } +} + //var userinfoTests = []UserinfoTest{ // {"user", "password", "user:password"}, // {"foo:bar", "~!@#$%^&*()_+{}|[]\\-=`:;'\"<>?,./", @@ -874,11 +1001,13 @@ var resolveReferenceTests = []struct { // Absolute URL references {"http://foo.com?a=b", "https://bar.com/", "https://bar.com/"}, {"http://foo.com/", "https://bar.com/?a=b", "https://bar.com/?a=b"}, + {"http://foo.com/", "https://bar.com/?", "https://bar.com/?"}, {"http://foo.com/bar", "mailto:foo@example.com", "mailto:foo@example.com"}, // Path-absolute references {"http://foo.com/bar", "/baz", "http://foo.com/baz"}, {"http://foo.com/bar?a=b#f", "/baz", "http://foo.com/baz"}, + {"http://foo.com/bar?a=b", "/baz?", "http://foo.com/baz?"}, {"http://foo.com/bar?a=b", "/baz?c=d", "http://foo.com/baz?c=d"}, // Scheme-relative @@ -921,6 +1050,15 @@ var resolveReferenceTests = []struct { // Fragment {"http://foo.com/bar", ".#frag", "http://foo.com/#frag"}, + // Paths with escaping (issue 16947). + {"http://foo.com/foo%2fbar/", "../baz", "http://foo.com/baz"}, + {"http://foo.com/1/2%2f/3%2f4/5", "../../a/b/c", "http://foo.com/1/a/b/c"}, + {"http://foo.com/1/2/3", "./a%2f../../b/..%2fc", "http://foo.com/1/2/b/..%2fc"}, + {"http://foo.com/1/2%2f/3%2f4/5", "./a%2f../b/../c", "http://foo.com/1/2%2f/3%2f4/a%2f../c"}, + {"http://foo.com/foo%20bar/", "../baz", "http://foo.com/baz"}, + {"http://foo.com/foo", "../bar%2fbaz", "http://foo.com/bar%2fbaz"}, + {"http://foo.com/foo%2dbar/", "./baz-quux", "http://foo.com/foo%2dbar/baz-quux"}, + // RFC 3986: Normal Examples // http://tools.ietf.org/html/rfc3986#section-5.4.1 {"http://a/b/c/d;p?q", "g:h", "g:h"}, @@ -980,7 +1118,7 @@ func TestResolveReference(t *testing.T) { mustParse := func(url string) *URL { u, err := Parse(url) if err != nil { - t.Fatalf("Expected URL to parse: %q, got error: %v", url, err) + t.Fatalf("Parse(%q) got err %v", url, err) } return u } @@ -989,8 +1127,8 @@ func TestResolveReference(t *testing.T) { base := mustParse(test.base) rel := mustParse(test.rel) url := base.ResolveReference(rel) - if url.String() != test.expected { - t.Errorf("URL(%q).ResolveReference(%q) == %q, got %q", test.base, test.rel, test.expected, url.String()) + if got := url.String(); got != test.expected { + t.Errorf("URL(%q).ResolveReference(%q)\ngot %q\nwant %q", test.base, test.rel, got, test.expected) } // Ensure that new instances are returned. if base == url { @@ -1000,8 +1138,8 @@ func TestResolveReference(t *testing.T) { url, err := base.Parse(test.rel) if err != nil { t.Errorf("URL(%q).Parse(%q) failed: %v", test.base, test.rel, err) - } else if url.String() != test.expected { - t.Errorf("URL(%q).Parse(%q) == %q, got %q", test.base, test.rel, test.expected, url.String()) + } else if got := url.String(); got != test.expected { + t.Errorf("URL(%q).Parse(%q)\ngot %q\nwant %q", test.base, test.rel, got, test.expected) } else if base == url { // Ensure that new instances are returned for the wrapper too. t.Errorf("Expected URL.Parse to return new URL instance.") @@ -1009,14 +1147,14 @@ func TestResolveReference(t *testing.T) { // Ensure Opaque resets the URL. url = base.ResolveReference(opaque) if *url != *opaque { - t.Errorf("ResolveReference failed to resolve opaque URL: want %#v, got %#v", url, opaque) + t.Errorf("ResolveReference failed to resolve opaque URL:\ngot %#v\nwant %#v", url, opaque) } // Test the convenience wrapper with an opaque URL too. url, err = base.Parse("scheme:opaque") if err != nil { t.Errorf(`URL(%q).Parse("scheme:opaque") failed: %v`, test.base, err) } else if *url != *opaque { - t.Errorf("Parse failed to resolve opaque URL: want %#v, got %#v", url, opaque) + t.Errorf("Parse failed to resolve opaque URL:\ngot %#v\nwant %#v", opaque, url) } else if base == url { // Ensure that new instances are returned, again. t.Errorf("Expected URL.Parse to return new URL instance.") @@ -1217,6 +1355,15 @@ var requritests = []RequestURITest{ }, "//foo", }, + { + &URL{ + Scheme: "http", + Host: "example.com", + Path: "/foo", + ForceQuery: true, + }, + "/foo?", + }, } func TestRequestURI(t *testing.T) { @@ -1238,7 +1385,7 @@ func TestParseFailure(t *testing.T) { } } -func TestParseAuthority(t *testing.T) { +func TestParseErrors(t *testing.T) { tests := []struct { in string wantErr bool @@ -1258,9 +1405,13 @@ func TestParseAuthority(t *testing.T) { {"http://%41:8080/", true}, // not allowed: % encoding only for non-ASCII {"mysql://x@y(z:123)/foo", false}, // golang.org/issue/12023 {"mysql://x@y(1.2.3.4:123)/foo", false}, - {"mysql://x@y([2001:db8::1]:123)/foo", false}, + {"http://[]%20%48%54%54%50%2f%31%2e%31%0a%4d%79%48%65%61%64%65%72%3a%20%31%32%33%0a%0a/", true}, // golang.org/issue/11208 {"http://a b.com/", true}, // no space in host name please + {"cache_object://foo", true}, // scheme cannot have _, relative path cannot have : in first segment + {"cache_object:foo", true}, + {"cache_object:foo/bar", true}, + {"cache_object/:foo/bar", false}, } for _, tt := range tests { u, err := Parse(tt.in) @@ -1429,11 +1580,106 @@ func TestURLErrorImplementsNetError(t *testing.T) { continue } if err.Timeout() != tt.timeout { - t.Errorf("%d: err.Timeout(): want %v, have %v", i+1, tt.timeout, err.Timeout()) + t.Errorf("%d: err.Timeout(): got %v, want %v", i+1, err.Timeout(), tt.timeout) continue } if err.Temporary() != tt.temporary { - t.Errorf("%d: err.Temporary(): want %v, have %v", i+1, tt.temporary, err.Temporary()) + t.Errorf("%d: err.Temporary(): got %v, want %v", i+1, err.Temporary(), tt.temporary) + } + } +} + +func TestURLHostname(t *testing.T) { + tests := []struct { + host string // URL.Host field + want string + }{ + {"foo.com:80", "foo.com"}, + {"foo.com", "foo.com"}, + {"FOO.COM", "FOO.COM"}, // no canonicalization (yet?) + {"1.2.3.4", "1.2.3.4"}, + {"1.2.3.4:80", "1.2.3.4"}, + {"[1:2:3:4]", "1:2:3:4"}, + {"[1:2:3:4]:80", "1:2:3:4"}, + {"[::1]:80", "::1"}, + } + for _, tt := range tests { + u := &URL{Host: tt.host} + got := u.Hostname() + if got != tt.want { + t.Errorf("Hostname for Host %q = %q; want %q", tt.host, got, tt.want) } } } + +func TestURLPort(t *testing.T) { + tests := []struct { + host string // URL.Host field + want string + }{ + {"foo.com", ""}, + {"foo.com:80", "80"}, + {"1.2.3.4", ""}, + {"1.2.3.4:80", "80"}, + {"[1:2:3:4]", ""}, + {"[1:2:3:4]:80", "80"}, + } + for _, tt := range tests { + u := &URL{Host: tt.host} + got := u.Port() + if got != tt.want { + t.Errorf("Port for Host %q = %q; want %q", tt.host, got, tt.want) + } + } +} + +var _ encodingPkg.BinaryMarshaler = (*URL)(nil) +var _ encodingPkg.BinaryUnmarshaler = (*URL)(nil) + +func TestJSON(t *testing.T) { + u, err := Parse("https://www.google.com/x?y=z") + if err != nil { + t.Fatal(err) + } + js, err := json.Marshal(u) + if err != nil { + t.Fatal(err) + } + + // If only we could implement TextMarshaler/TextUnmarshaler, + // this would work: + // + // if string(js) != strconv.Quote(u.String()) { + // t.Errorf("json encoding: %s\nwant: %s\n", js, strconv.Quote(u.String())) + // } + + u1 := new(URL) + err = json.Unmarshal(js, u1) + if err != nil { + t.Fatal(err) + } + if u1.String() != u.String() { + t.Errorf("json decoded to: %s\nwant: %s\n", u1, u) + } +} + +func TestGob(t *testing.T) { + u, err := Parse("https://www.google.com/x?y=z") + if err != nil { + t.Fatal(err) + } + var w bytes.Buffer + err = gob.NewEncoder(&w).Encode(u) + if err != nil { + t.Fatal(err) + } + + u1 := new(URL) + err = gob.NewDecoder(&w).Decode(u1) + if err != nil { + t.Fatal(err) + } + if u1.String() != u.String() { + t.Errorf("json decoded to: %s\nwant: %s\n", u1, u) + } +} diff --git a/libgo/go/net/writev_test.go b/libgo/go/net/writev_test.go new file mode 100644 index 0000000000..7160d28c3a --- /dev/null +++ b/libgo/go/net/writev_test.go @@ -0,0 +1,225 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package net + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "reflect" + "runtime" + "sync" + "testing" +) + +func TestBuffers_read(t *testing.T) { + const story = "once upon a time in Gopherland ... " + buffers := Buffers{ + []byte("once "), + []byte("upon "), + []byte("a "), + []byte("time "), + []byte("in "), + []byte("Gopherland ... "), + } + got, err := ioutil.ReadAll(&buffers) + if err != nil { + t.Fatal(err) + } + if string(got) != story { + t.Errorf("read %q; want %q", got, story) + } + if len(buffers) != 0 { + t.Errorf("len(buffers) = %d; want 0", len(buffers)) + } +} + +func TestBuffers_consume(t *testing.T) { + tests := []struct { + in Buffers + consume int64 + want Buffers + }{ + { + in: Buffers{[]byte("foo"), []byte("bar")}, + consume: 0, + want: Buffers{[]byte("foo"), []byte("bar")}, + }, + { + in: Buffers{[]byte("foo"), []byte("bar")}, + consume: 2, + want: Buffers{[]byte("o"), []byte("bar")}, + }, + { + in: Buffers{[]byte("foo"), []byte("bar")}, + consume: 3, + want: Buffers{[]byte("bar")}, + }, + { + in: Buffers{[]byte("foo"), []byte("bar")}, + consume: 4, + want: Buffers{[]byte("ar")}, + }, + { + in: Buffers{nil, nil, nil, []byte("bar")}, + consume: 1, + want: Buffers{[]byte("ar")}, + }, + { + in: Buffers{nil, nil, nil, []byte("foo")}, + consume: 0, + want: Buffers{[]byte("foo")}, + }, + { + in: Buffers{nil, nil, nil}, + consume: 0, + want: Buffers{}, + }, + } + for i, tt := range tests { + in := tt.in + in.consume(tt.consume) + if !reflect.DeepEqual(in, tt.want) { + t.Errorf("%d. after consume(%d) = %+v, want %+v", i, tt.consume, in, tt.want) + } + } +} + +func TestBuffers_WriteTo(t *testing.T) { + for _, name := range []string{"WriteTo", "Copy"} { + for _, size := range []int{0, 10, 1023, 1024, 1025} { + t.Run(fmt.Sprintf("%s/%d", name, size), func(t *testing.T) { + testBuffer_writeTo(t, size, name == "Copy") + }) + } + } +} + +func testBuffer_writeTo(t *testing.T, chunks int, useCopy bool) { + oldHook := testHookDidWritev + defer func() { testHookDidWritev = oldHook }() + var writeLog struct { + sync.Mutex + log []int + } + testHookDidWritev = func(size int) { + writeLog.Lock() + writeLog.log = append(writeLog.log, size) + writeLog.Unlock() + } + var want bytes.Buffer + for i := 0; i < chunks; i++ { + want.WriteByte(byte(i)) + } + + withTCPConnPair(t, func(c *TCPConn) error { + buffers := make(Buffers, chunks) + for i := range buffers { + buffers[i] = want.Bytes()[i : i+1] + } + var n int64 + var err error + if useCopy { + n, err = io.Copy(c, &buffers) + } else { + n, err = buffers.WriteTo(c) + } + if err != nil { + return err + } + if len(buffers) != 0 { + return fmt.Errorf("len(buffers) = %d; want 0", len(buffers)) + } + if n != int64(want.Len()) { + return fmt.Errorf("Buffers.WriteTo returned %d; want %d", n, want.Len()) + } + return nil + }, func(c *TCPConn) error { + all, err := ioutil.ReadAll(c) + if !bytes.Equal(all, want.Bytes()) || err != nil { + return fmt.Errorf("client read %q, %v; want %q, nil", all, err, want.Bytes()) + } + + writeLog.Lock() // no need to unlock + var gotSum int + for _, v := range writeLog.log { + gotSum += v + } + + var wantSum int + switch runtime.GOOS { + case "android", "darwin", "dragonfly", "freebsd", "linux", "netbsd", "openbsd": + var wantMinCalls int + wantSum = want.Len() + v := chunks + for v > 0 { + wantMinCalls++ + v -= 1024 + } + if len(writeLog.log) < wantMinCalls { + t.Errorf("write calls = %v < wanted min %v", len(writeLog.log), wantMinCalls) + } + case "windows": + var wantCalls int + wantSum = want.Len() + if wantSum > 0 { + wantCalls = 1 // windows will always do 1 syscall, unless sending empty buffer + } + if len(writeLog.log) != wantCalls { + t.Errorf("write calls = %v; want %v", len(writeLog.log), wantCalls) + } + } + if gotSum != wantSum { + t.Errorf("writev call sum = %v; want %v", gotSum, wantSum) + } + return nil + }) +} + +func TestWritevError(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skipf("skipping the test: windows does not have problem sending large chunks of data") + } + + ln, err := newLocalListener("tcp") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + + ch := make(chan Conn, 1) + go func() { + defer close(ch) + c, err := ln.Accept() + if err != nil { + t.Error(err) + return + } + ch <- c + }() + c1, err := Dial("tcp", ln.Addr().String()) + if err != nil { + t.Fatal(err) + } + defer c1.Close() + c2 := <-ch + if c2 == nil { + t.Fatal("no server side connection") + } + c2.Close() + + // 1 GB of data should be enough to notice the connection is gone. + // Just a few bytes is not enough. + // Arrange to reuse the same 1 MB buffer so that we don't allocate much. + buf := make([]byte, 1<<20) + buffers := make(Buffers, 1<<10) + for i := range buffers { + buffers[i] = buf + } + if _, err := buffers.WriteTo(c1); err == nil { + t.Fatal("Buffers.WriteTo(closed conn) succeeded, want error") + } +} diff --git a/libgo/go/net/writev_unix.go b/libgo/go/net/writev_unix.go new file mode 100644 index 0000000000..174e6bc51e --- /dev/null +++ b/libgo/go/net/writev_unix.go @@ -0,0 +1,95 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build darwin dragonfly freebsd linux netbsd openbsd + +package net + +import ( + "io" + "os" + "syscall" + "unsafe" +) + +func (c *conn) writeBuffers(v *Buffers) (int64, error) { + if !c.ok() { + return 0, syscall.EINVAL + } + n, err := c.fd.writeBuffers(v) + if err != nil { + return n, &OpError{Op: "writev", Net: c.fd.net, Source: c.fd.laddr, Addr: c.fd.raddr, Err: err} + } + return n, nil +} + +func (fd *netFD) writeBuffers(v *Buffers) (n int64, err error) { + if err := fd.writeLock(); err != nil { + return 0, err + } + defer fd.writeUnlock() + if err := fd.pd.prepareWrite(); err != nil { + return 0, err + } + + var iovecs []syscall.Iovec + if fd.iovecs != nil { + iovecs = *fd.iovecs + } + // TODO: read from sysconf(_SC_IOV_MAX)? The Linux default is + // 1024 and this seems conservative enough for now. Darwin's + // UIO_MAXIOV also seems to be 1024. + maxVec := 1024 + + for len(*v) > 0 { + iovecs = iovecs[:0] + for _, chunk := range *v { + if len(chunk) == 0 { + continue + } + iovecs = append(iovecs, syscall.Iovec{Base: &chunk[0]}) + if fd.isStream && len(chunk) > 1<<30 { + iovecs[len(iovecs)-1].SetLen(1 << 30) + break // continue chunk on next writev + } + iovecs[len(iovecs)-1].SetLen(len(chunk)) + if len(iovecs) == maxVec { + break + } + } + if len(iovecs) == 0 { + break + } + fd.iovecs = &iovecs // cache + + wrote, _, e0 := syscall.Syscall(syscall.SYS_WRITEV, + uintptr(fd.sysfd), + uintptr(unsafe.Pointer(&iovecs[0])), + uintptr(len(iovecs))) + if wrote == ^uintptr(0) { + wrote = 0 + } + testHookDidWritev(int(wrote)) + n += int64(wrote) + v.consume(int64(wrote)) + if e0 == syscall.EAGAIN { + if err = fd.pd.waitWrite(); err == nil { + continue + } + } else if e0 != 0 { + err = syscall.Errno(e0) + } + if err != nil { + break + } + if n == 0 { + err = io.ErrUnexpectedEOF + break + } + } + if _, ok := err.(syscall.Errno); ok { + err = os.NewSyscallError("writev", err) + } + return n, err +} |
