diff options
Diffstat (limited to 'libgo/go/net/http/request.go')
| -rw-r--r-- | libgo/go/net/http/request.go | 471 |
1 files changed, 314 insertions, 157 deletions
diff --git a/libgo/go/net/http/request.go b/libgo/go/net/http/request.go index 8cdab02af5..fb6bb0aab5 100644 --- a/libgo/go/net/http/request.go +++ b/libgo/go/net/http/request.go @@ -9,6 +9,7 @@ package http import ( "bufio" "bytes" + "context" "crypto/tls" "encoding/base64" "errors" @@ -17,11 +18,17 @@ import ( "io/ioutil" "mime" "mime/multipart" + "net" + "net/http/httptrace" "net/textproto" "net/url" "strconv" "strings" "sync" + + "golang_org/x/net/idna" + "golang_org/x/text/unicode/norm" + "golang_org/x/text/width" ) const ( @@ -32,21 +39,40 @@ const ( // is either not present in the request or not a file field. var ErrMissingFile = errors.New("http: no such file") -// HTTP request parsing errors. +// ProtocolError represents an HTTP protocol error. +// +// Deprecated: Not all errors in the http package related to protocol errors +// are of type ProtocolError. type ProtocolError struct { ErrorString string } -func (err *ProtocolError) Error() string { return err.ErrorString } +func (pe *ProtocolError) Error() string { return pe.ErrorString } var ( - ErrHeaderTooLong = &ProtocolError{"header too long"} - ErrShortBody = &ProtocolError{"entity body too short"} - ErrNotSupported = &ProtocolError{"feature not supported"} - ErrUnexpectedTrailer = &ProtocolError{"trailer header without chunked transfer encoding"} + // ErrNotSupported is returned by the Push method of Pusher + // implementations to indicate that HTTP/2 Push support is not + // available. + ErrNotSupported = &ProtocolError{"feature not supported"} + + // ErrUnexpectedTrailer is returned by the Transport when a server + // replies with a Trailer header, but without a chunked reply. + ErrUnexpectedTrailer = &ProtocolError{"trailer header without chunked transfer encoding"} + + // ErrMissingBoundary is returned by Request.MultipartReader when the + // request's Content-Type does not include a "boundary" parameter. + ErrMissingBoundary = &ProtocolError{"no multipart boundary param in Content-Type"} + + // ErrNotMultipart is returned by Request.MultipartReader when the + // request's Content-Type is not multipart/form-data. + ErrNotMultipart = &ProtocolError{"request Content-Type isn't multipart/form-data"} + + // Deprecated: ErrHeaderTooLong is not used. + ErrHeaderTooLong = &ProtocolError{"header too long"} + // Deprecated: ErrShortBody is not used. + ErrShortBody = &ProtocolError{"entity body too short"} + // Deprecated: ErrMissingContentLength is not used. ErrMissingContentLength = &ProtocolError{"missing ContentLength in HEAD response"} - ErrNotMultipart = &ProtocolError{"request Content-Type isn't multipart/form-data"} - ErrMissingBoundary = &ProtocolError{"no multipart boundary param in Content-Type"} ) type badStringError struct { @@ -144,11 +170,20 @@ type Request struct { // Handler does not need to. Body io.ReadCloser + // GetBody defines an optional func to return a new copy of + // Body. It is used for client requests when a redirect requires + // reading the body more than once. Use of GetBody still + // requires setting Body. + // + // For server requests it is unused. + GetBody func() (io.ReadCloser, error) + // ContentLength records the length of the associated content. // The value -1 indicates that the length is unknown. // Values >= 0 indicate that the given number of bytes may // be read from Body. - // For client requests, a value of 0 means unknown if Body is not nil. + // For client requests, a value of 0 with a non-nil Body is + // also treated as unknown. ContentLength int64 // TransferEncoding lists the transfer encodings from outermost to @@ -173,11 +208,15 @@ type Request struct { // For server requests Host specifies the host on which the // URL is sought. Per RFC 2616, this is either the value of // the "Host" header or the host name given in the URL itself. - // It may be of the form "host:port". + // It may be of the form "host:port". For international domain + // names, Host may be in Punycode or Unicode form. Use + // golang.org/x/net/idna to convert it to either format if + // needed. // // For client requests Host optionally overrides the Host // header to send. If empty, the Request.Write method uses - // the value of URL.Host. + // the value of URL.Host. Host may contain an international + // domain name. Host string // Form contains the parsed form data, including both the URL @@ -247,7 +286,52 @@ type Request struct { // RoundTripper may support Cancel. // // For server requests, this field is not applicable. + // + // Deprecated: Use the Context and WithContext methods + // instead. If a Request's Cancel field and context are both + // set, it is undefined whether Cancel is respected. Cancel <-chan struct{} + + // Response is the redirect response which caused this request + // to be created. This field is only populated during client + // redirects. + Response *Response + + // ctx is either the client or server context. It should only + // be modified via copying the whole Request using WithContext. + // It is unexported to prevent people from using Context wrong + // and mutating the contexts held by callers of the same request. + ctx context.Context +} + +// Context returns the request's context. To change the context, use +// WithContext. +// +// The returned context is always non-nil; it defaults to the +// background context. +// +// For outgoing client requests, the context controls cancelation. +// +// For incoming server requests, the context is canceled when the +// client's connection closes, the request is canceled (with HTTP/2), +// or when the ServeHTTP method returns. +func (r *Request) Context() context.Context { + if r.ctx != nil { + return r.ctx + } + return context.Background() +} + +// WithContext returns a shallow copy of r with its context changed +// to ctx. The provided ctx must be non-nil. +func (r *Request) WithContext(ctx context.Context) *Request { + if ctx == nil { + panic("nil context") + } + r2 := new(Request) + *r2 = *r + r2.ctx = ctx + return r2 } // ProtoAtLeast reports whether the HTTP protocol used @@ -257,6 +341,18 @@ func (r *Request) ProtoAtLeast(major, minor int) bool { r.ProtoMajor == major && r.ProtoMinor >= minor } +// protoAtLeastOutgoing is like ProtoAtLeast, but is for outgoing +// requests (see issue 18407) where these fields aren't supposed to +// matter. As a minor fix for Go 1.8, at least treat (0, 0) as +// matching HTTP/1.1 or HTTP/1.0. Only HTTP/1.1 is used. +// TODO(bradfitz): ideally remove this whole method. It shouldn't be used. +func (r *Request) protoAtLeastOutgoing(major, minor int) bool { + if r.ProtoMajor == 0 && r.ProtoMinor == 0 && major == 1 && minor <= 1 { + return true + } + return r.ProtoAtLeast(major, minor) +} + // UserAgent returns the client's User-Agent, if sent in the request. func (r *Request) UserAgent() string { return r.Header.Get("User-Agent") @@ -272,6 +368,8 @@ var ErrNoCookie = errors.New("http: named cookie not present") // Cookie returns the named cookie provided in the request or // ErrNoCookie if not found. +// If multiple cookies match the given name, only one cookie will +// be returned. func (r *Request) Cookie(name string) (*Cookie, error) { for _, c := range readCookies(r.Header, name) { return c, nil @@ -279,8 +377,8 @@ func (r *Request) Cookie(name string) (*Cookie, error) { return nil, ErrNoCookie } -// AddCookie adds a cookie to the request. Per RFC 6265 section 5.4, -// AddCookie does not attach more than one Cookie header field. That +// AddCookie adds a cookie to the request. Per RFC 6265 section 5.4, +// AddCookie does not attach more than one Cookie header field. That // means all cookies, if any, are written into the same line, // separated by semicolon. func (r *Request) AddCookie(c *Cookie) { @@ -343,6 +441,12 @@ func (r *Request) multipartReader() (*multipart.Reader, error) { return multipart.NewReader(r.Body, boundary), nil } +// isH2Upgrade reports whether r represents the http2 "client preface" +// magic string. +func (r *Request) isH2Upgrade() bool { + return r.Method == "PRI" && len(r.Header) == 0 && r.URL.Path == "*" && r.Proto == "HTTP/2.0" +} + // Return value if nonempty, def otherwise. func valueOrDefault(value, def string) string { if value != "" { @@ -375,7 +479,7 @@ func (r *Request) Write(w io.Writer) error { } // WriteProxy is like Write but writes the request in the form -// expected by an HTTP proxy. In particular, WriteProxy writes the +// expected by an HTTP proxy. In particular, WriteProxy writes the // initial Request-URI line of the request with an absolute URI, per // section 5.1.2 of RFC 2616, including the scheme and host. // In either case, WriteProxy also writes a Host header, using @@ -390,7 +494,16 @@ var errMissingHost = errors.New("http: Request.Write on Request with no Host or // extraHeaders may be nil // waitForContinue may be nil -func (req *Request) write(w io.Writer, usingProxy bool, extraHeaders Header, waitForContinue func() bool) error { +func (req *Request) write(w io.Writer, usingProxy bool, extraHeaders Header, waitForContinue func() bool) (err error) { + trace := httptrace.ContextClientTrace(req.Context()) + if trace != nil && trace.WroteRequest != nil { + defer func() { + trace.WroteRequest(httptrace.WroteRequestInfo{ + Err: err, + }) + }() + } + // Find the target host. Prefer the Host: header, but if that // is not given, use the host from the request URL. // @@ -427,7 +540,7 @@ func (req *Request) write(w io.Writer, usingProxy bool, extraHeaders Header, wai w = bw } - _, err := fmt.Fprintf(w, "%s %s HTTP/1.1\r\n", valueOrDefault(req.Method, "GET"), ruri) + _, err = fmt.Fprintf(w, "%s %s HTTP/1.1\r\n", valueOrDefault(req.Method, "GET"), ruri) if err != nil { return err } @@ -478,6 +591,10 @@ func (req *Request) write(w io.Writer, usingProxy bool, extraHeaders Header, wai return err } + if trace != nil && trace.WroteHeaders != nil { + trace.WroteHeaders() + } + // Flush and wait for 100-continue if expected. if waitForContinue != nil { if bw, ok := w.(*bufio.Writer); ok { @@ -486,13 +603,21 @@ func (req *Request) write(w io.Writer, usingProxy bool, extraHeaders Header, wai return err } } - + if trace != nil && trace.Wait100Continue != nil { + trace.Wait100Continue() + } if !waitForContinue() { req.closeBody() return nil } } + if bw, ok := w.(*bufio.Writer); ok && tw.FlushHeaders { + if err := bw.Flush(); err != nil { + return err + } + } + // Write body and trailer err = tw.WriteBody(w) if err != nil { @@ -505,7 +630,24 @@ func (req *Request) write(w io.Writer, usingProxy bool, extraHeaders Header, wai return nil } -// cleanHost strips anything after '/' or ' '. +func idnaASCII(v string) (string, error) { + if isASCII(v) { + return v, nil + } + // The idna package doesn't do everything from + // https://tools.ietf.org/html/rfc5895 so we do it here. + // TODO(bradfitz): should the idna package do this instead? + v = strings.ToLower(v) + v = width.Fold.String(v) + v = norm.NFC.String(v) + return idna.ToASCII(v) +} + +// cleanHost cleans up the host sent in request's Host header. +// +// It both strips anything after '/' or ' ', and puts the value +// into Punycode form, if necessary. +// // Ideally we'd clean the Host header according to the spec: // https://tools.ietf.org/html/rfc7230#section-5.4 (Host = uri-host [ ":" port ]") // https://tools.ietf.org/html/rfc7230#section-2.7 (uri-host -> rfc3986's host) @@ -516,12 +658,24 @@ func (req *Request) write(w io.Writer, usingProxy bool, extraHeaders Header, wai // first offending character. func cleanHost(in string) string { if i := strings.IndexAny(in, " /"); i != -1 { - return in[:i] + in = in[:i] + } + host, port, err := net.SplitHostPort(in) + if err != nil { // input was just a host + a, err := idnaASCII(in) + if err != nil { + return in // garbage in, garbage out + } + return a + } + a, err := idnaASCII(host) + if err != nil { + return in // garbage in, garbage out } - return in + return net.JoinHostPort(a, port) } -// removeZone removes IPv6 zone identifer from host. +// removeZone removes IPv6 zone identifier from host. // E.g., "[fe80::1%en0]:8080" to "[fe80::1]:8080" func removeZone(host string) string { if !strings.HasPrefix(host, "[") { @@ -590,11 +744,17 @@ func validMethod(method string) bool { // methods Do, Post, and PostForm, and Transport.RoundTrip. // // NewRequest returns a Request suitable for use with Client.Do or -// Transport.RoundTrip. -// To create a request for use with testing a Server Handler use either -// ReadRequest or manually update the Request fields. See the Request -// type's documentation for the difference between inbound and outbound -// request fields. +// Transport.RoundTrip. To create a request for use with testing a +// Server Handler, either use the NewRequest function in the +// net/http/httptest package, use ReadRequest, or manually update the +// Request fields. See the Request type's documentation for the +// difference between inbound and outbound request fields. +// +// If body is of type *bytes.Buffer, *bytes.Reader, or +// *strings.Reader, the returned request's ContentLength is set to its +// exact value (instead of -1), GetBody is populated (so 307 and 308 +// redirects can replay the body), and Body is set to NoBody if the +// ContentLength is 0. func NewRequest(method, urlStr string, body io.Reader) (*Request, error) { if method == "" { // We document that "" means "GET" for Request.Method, and people have @@ -613,6 +773,8 @@ func NewRequest(method, urlStr string, body io.Reader) (*Request, error) { if !ok && body != nil { rc = ioutil.NopCloser(body) } + // The host's colon:port should be normalized. See Issue 14836. + u.Host = removeEmptyPort(u.Host) req := &Request{ Method: method, URL: u, @@ -627,10 +789,43 @@ func NewRequest(method, urlStr string, body io.Reader) (*Request, error) { switch v := body.(type) { case *bytes.Buffer: req.ContentLength = int64(v.Len()) + buf := v.Bytes() + req.GetBody = func() (io.ReadCloser, error) { + r := bytes.NewReader(buf) + return ioutil.NopCloser(r), nil + } case *bytes.Reader: req.ContentLength = int64(v.Len()) + snapshot := *v + req.GetBody = func() (io.ReadCloser, error) { + r := snapshot + return ioutil.NopCloser(&r), nil + } case *strings.Reader: req.ContentLength = int64(v.Len()) + snapshot := *v + req.GetBody = func() (io.ReadCloser, error) { + r := snapshot + return ioutil.NopCloser(&r), nil + } + default: + // This is where we'd set it to -1 (at least + // if body != NoBody) to mean unknown, but + // that broke people during the Go 1.8 testing + // period. People depend on it being 0 I + // guess. Maybe retry later. See Issue 18117. + } + // For client requests, Request.ContentLength of 0 + // means either actually 0, or unknown. The only way + // to explicitly say that the ContentLength is zero is + // to set the Body to nil. But turns out too much code + // depends on NewRequest returning a non-nil Body, + // so we use a well-known ReadCloser variable instead + // and have the http package also treat that sentinel + // variable to mean explicitly zero. + if req.GetBody != nil && req.ContentLength == 0 { + req.Body = NoBody + req.GetBody = func() (io.ReadCloser, error) { return NoBody, nil } } } @@ -704,7 +899,9 @@ func putTextprotoReader(r *textproto.Reader) { } // ReadRequest reads and parses an incoming request from b. -func ReadRequest(b *bufio.Reader) (req *Request, err error) { return readRequest(b, deleteHostHeader) } +func ReadRequest(b *bufio.Reader) (*Request, error) { + return readRequest(b, deleteHostHeader) +} // Constants for readRequest's deleteHostHeader parameter. const ( @@ -768,13 +965,13 @@ func readRequest(b *bufio.Reader, deleteHostHeader bool) (req *Request, err erro } req.Header = Header(mimeHeader) - // RFC2616: Must treat + // RFC 2616: Must treat // GET /index.html HTTP/1.1 // Host: www.google.com // and // GET http://www.google.com/index.html HTTP/1.1 // Host: doesntmatter - // the same. In the second case, any Host line is ignored. + // the same. In the second case, any Host line is ignored. req.Host = req.URL.Host if req.Host == "" { req.Host = req.Header.get("Host") @@ -792,6 +989,16 @@ func readRequest(b *bufio.Reader, deleteHostHeader bool) (req *Request, err erro return nil, err } + if req.isH2Upgrade() { + // Because it's neither chunked, nor declared: + req.ContentLength = -1 + + // We want to give handlers a chance to hijack the + // connection, but we need to prevent the Server from + // dealing with the connection further if it's not + // hijacked. Set Close to ensure that: + req.Close = true + } return req, nil } @@ -808,57 +1015,56 @@ func MaxBytesReader(w ResponseWriter, r io.ReadCloser, n int64) io.ReadCloser { } type maxBytesReader struct { - w ResponseWriter - r io.ReadCloser // underlying reader - n int64 // max bytes remaining - stopped bool - sawEOF bool + w ResponseWriter + r io.ReadCloser // underlying reader + n int64 // max bytes remaining + err error // sticky error } func (l *maxBytesReader) tooLarge() (n int, err error) { - if !l.stopped { - l.stopped = true - if res, ok := l.w.(*response); ok { - res.requestTooLarge() - } - } - return 0, errors.New("http: request body too large") + l.err = errors.New("http: request body too large") + return 0, l.err } func (l *maxBytesReader) Read(p []byte) (n int, err error) { - toRead := l.n - if l.n == 0 { - if l.sawEOF { - return l.tooLarge() - } - // The underlying io.Reader may not return (0, io.EOF) - // at EOF if the requested size is 0, so read 1 byte - // instead. The io.Reader docs are a bit ambiguous - // about the return value of Read when 0 bytes are - // requested, and {bytes,strings}.Reader gets it wrong - // too (it returns (0, nil) even at EOF). - toRead = 1 + if l.err != nil { + return 0, l.err + } + if len(p) == 0 { + return 0, nil } - if int64(len(p)) > toRead { - p = p[:toRead] + // If they asked for a 32KB byte read but only 5 bytes are + // remaining, no need to read 32KB. 6 bytes will answer the + // question of the whether we hit the limit or go past it. + if int64(len(p)) > l.n+1 { + p = p[:l.n+1] } n, err = l.r.Read(p) - if err == io.EOF { - l.sawEOF = true - } - if l.n == 0 { - // If we had zero bytes to read remaining (but hadn't seen EOF) - // and we get a byte here, that means we went over our limit. - if n > 0 { - return l.tooLarge() - } - return 0, err + + if int64(n) <= l.n { + l.n -= int64(n) + l.err = err + return n, err } - l.n -= int64(n) - if l.n < 0 { - l.n = 0 + + n = int(l.n) + l.n = 0 + + // The server code and client code both use + // maxBytesReader. This "requestTooLarge" check is + // only used by the server code. To prevent binaries + // which only using the HTTP Client code (such as + // cmd/go) from also linking in the HTTP server, don't + // use a static type assertion to the server + // "*response" type. Check this interface instead: + type requestTooLarger interface { + requestTooLarge() } - return + if res, ok := l.w.(requestTooLarger); ok { + res.requestTooLarge() + } + l.err = errors.New("http: request body too large") + return n, l.err } func (l *maxBytesReader) Close() error { @@ -919,18 +1125,24 @@ func parsePostForm(r *Request) (vs url.Values, err error) { return } -// ParseForm parses the raw query from the URL and updates r.Form. +// ParseForm populates r.Form and r.PostForm. +// +// For all requests, ParseForm parses the raw query from the URL and updates +// r.Form. // -// For POST or PUT requests, it also parses the request body as a form and -// put the results into both r.PostForm and r.Form. -// POST and PUT body parameters take precedence over URL query string values -// in r.Form. +// For POST, PUT, and PATCH requests, it also parses the request body as a form +// and puts the results into both r.PostForm and r.Form. Request body parameters +// take precedence over URL query string values in r.Form. +// +// For other HTTP methods, or when the Content-Type is not +// application/x-www-form-urlencoded, the request Body is not read, and +// r.PostForm is initialized to a non-nil, empty value. // // If the request Body's size has not already been limited by MaxBytesReader, // the size is capped at 10MB. // // ParseMultipartForm calls ParseForm automatically. -// It is idempotent. +// ParseForm is idempotent. func (r *Request) ParseForm() error { var err error if r.PostForm == nil { @@ -995,9 +1207,16 @@ func (r *Request) ParseMultipartForm(maxMemory int64) error { if err != nil { return err } + + if r.PostForm == nil { + r.PostForm = make(url.Values) + } for k, v := range f.Value { r.Form[k] = append(r.Form[k], v...) + // r.PostForm should also be populated. See Issue 9305. + r.PostForm[k] = append(r.PostForm[k], v...) } + r.MultipartForm = f return nil @@ -1087,91 +1306,29 @@ func (r *Request) isReplayable() bool { return false } -func validHostHeader(h string) bool { - // The latests spec is actually this: - // - // http://tools.ietf.org/html/rfc7230#section-5.4 - // Host = uri-host [ ":" port ] - // - // Where uri-host is: - // http://tools.ietf.org/html/rfc3986#section-3.2.2 - // - // But we're going to be much more lenient for now and just - // search for any byte that's not a valid byte in any of those - // expressions. - for i := 0; i < len(h); i++ { - if !validHostByte[h[i]] { - return false - } +// outgoingLength reports the Content-Length of this outgoing (Client) request. +// It maps 0 into -1 (unknown) when the Body is non-nil. +func (r *Request) outgoingLength() int64 { + if r.Body == nil || r.Body == NoBody { + return 0 } - return true -} - -// See the validHostHeader comment. -var validHostByte = [256]bool{ - '0': true, '1': true, '2': true, '3': true, '4': true, '5': true, '6': true, '7': true, - '8': true, '9': true, - - 'a': true, 'b': true, 'c': true, 'd': true, 'e': true, 'f': true, 'g': true, 'h': true, - 'i': true, 'j': true, 'k': true, 'l': true, 'm': true, 'n': true, 'o': true, 'p': true, - 'q': true, 'r': true, 's': true, 't': true, 'u': true, 'v': true, 'w': true, 'x': true, - 'y': true, 'z': true, - - 'A': true, 'B': true, 'C': true, 'D': true, 'E': true, 'F': true, 'G': true, 'H': true, - 'I': true, 'J': true, 'K': true, 'L': true, 'M': true, 'N': true, 'O': true, 'P': true, - 'Q': true, 'R': true, 'S': true, 'T': true, 'U': true, 'V': true, 'W': true, 'X': true, - 'Y': true, 'Z': true, - - '!': true, // sub-delims - '$': true, // sub-delims - '%': true, // pct-encoded (and used in IPv6 zones) - '&': true, // sub-delims - '(': true, // sub-delims - ')': true, // sub-delims - '*': true, // sub-delims - '+': true, // sub-delims - ',': true, // sub-delims - '-': true, // unreserved - '.': true, // unreserved - ':': true, // IPv6address + Host expression's optional port - ';': true, // sub-delims - '=': true, // sub-delims - '[': true, - '\'': true, // sub-delims - ']': true, - '_': true, // unreserved - '~': true, // unreserved -} - -func validHeaderName(v string) bool { - if len(v) == 0 { - return false + if r.ContentLength != 0 { + return r.ContentLength } - return strings.IndexFunc(v, isNotToken) == -1 + return -1 } -// validHeaderValue reports whether v is a valid "field-value" according to -// http://www.w3.org/Protocols/rfc2616/rfc2616-sec4.html#sec4.2 : -// -// message-header = field-name ":" [ field-value ] -// field-value = *( field-content | LWS ) -// field-content = <the OCTETs making up the field-value -// and consisting of either *TEXT or combinations -// of token, separators, and quoted-string> -// -// http://www.w3.org/Protocols/rfc2616/rfc2616-sec2.html#sec2.2 : -// -// TEXT = <any OCTET except CTLs, -// but including LWS> -// LWS = [CRLF] 1*( SP | HT ) -// CTL = <any US-ASCII control character -// (octets 0 - 31) and DEL (127)> -func validHeaderValue(v string) bool { - for i := 0; i < len(v); i++ { - b := v[i] - if isCTL(b) && !isLWS(b) { - return false - } +// requestMethodUsuallyLacksBody reports whether the given request +// method is one that typically does not involve a request body. +// This is used by the Transport (via +// transferWriter.shouldSendChunkedRequestBody) to determine whether +// we try to test-read a byte from a non-nil Request.Body when +// Request.outgoingLength() returns -1. See the comments in +// shouldSendChunkedRequestBody. +func requestMethodUsuallyLacksBody(method string) bool { + switch method { + case "GET", "HEAD", "DELETE", "OPTIONS", "PROPFIND", "SEARCH": + return true } - return true + return false } |