diff options
Diffstat (limited to 'libgo/go/encoding')
56 files changed, 3830 insertions, 990 deletions
diff --git a/libgo/go/encoding/ascii85/ascii85.go b/libgo/go/encoding/ascii85/ascii85.go index 4d7193873a..d42eb0ab00 100644 --- a/libgo/go/encoding/ascii85/ascii85.go +++ b/libgo/go/encoding/ascii85/ascii85.go @@ -20,7 +20,7 @@ import ( // // The encoding handles 4-byte chunks, using a special encoding // for the last fragment, so Encode is not appropriate for use on -// individual blocks of a large data stream. Use NewEncoder() instead. +// individual blocks of a large data stream. Use NewEncoder() instead. // // Often, ascii85-encoded data is wrapped in <~ and ~> symbols. // Encode does not add these. @@ -85,7 +85,7 @@ func Encode(dst, src []byte) int { // MaxEncodedLen returns the maximum length of an encoding of n source bytes. func MaxEncodedLen(n int) int { return (n + 3) / 4 * 5 } -// NewEncoder returns a new ascii85 stream encoder. Data written to +// NewEncoder returns a new ascii85 stream encoder. Data written to // the returned writer will be encoded and then written to w. // Ascii85 encodings operate in 32-bit blocks; when finished // writing, the caller must Close the returned encoder to flush any @@ -294,7 +294,7 @@ func (d *decoder) Read(p []byte) (n int, err error) { } } - // Out of input, out of decoded output. Check errors. + // Out of input, out of decoded output. Check errors. if d.err != nil { return 0, d.err } diff --git a/libgo/go/encoding/asn1/asn1.go b/libgo/go/encoding/asn1/asn1.go index 8bafefd52b..044f74ab46 100644 --- a/libgo/go/encoding/asn1/asn1.go +++ b/libgo/go/encoding/asn1/asn1.go @@ -393,7 +393,7 @@ func isPrintable(b byte) bool { // byte slice and returns it. func parseIA5String(bytes []byte) (ret string, err error) { for _, b := range bytes { - if b >= 0x80 { + if b >= utf8.RuneSelf { err = SyntaxError{"IA5String contains invalid character"} return } @@ -461,6 +461,11 @@ func parseTagAndLength(bytes []byte, initOffset int) (ret tagAndLength, offset i if err != nil { return } + // Tags should be encoded in minimal form. + if ret.tag < 0x1f { + err = SyntaxError{"non-minimal tag"} + return + } } if offset >= len(bytes) { err = SyntaxError{"truncated tag or length"} @@ -836,6 +841,13 @@ func parseField(v reflect.Value, bytes []byte, initOffset int, params fieldParam case reflect.Struct: structType := fieldType + for i := 0; i < structType.NumField(); i++ { + if structType.Field(i).PkgPath != "" { + err = StructuralError{"struct contains unexported fields"} + return + } + } + if structType.NumField() > 0 && structType.Field(0).Type == rawContentsType { bytes := bytes[initOffset:offset] @@ -964,7 +976,7 @@ func setDefaultValue(v reflect.Value, params fieldParameters) (ok bool) { // The following tags on struct fields have special meaning to Unmarshal: // // application specifies that a APPLICATION tag is used -// default:x sets the default value for optional integer fields +// default:x sets the default value for optional integer fields (only used if optional is also present) // explicit specifies that an additional, explicit tag wraps the implicit one // optional marks the field as ASN.1 OPTIONAL // set causes a SET, rather than a SEQUENCE type to be expected diff --git a/libgo/go/encoding/asn1/asn1_test.go b/libgo/go/encoding/asn1/asn1_test.go index e0e833123b..9976656df8 100644 --- a/libgo/go/encoding/asn1/asn1_test.go +++ b/libgo/go/encoding/asn1/asn1_test.go @@ -132,9 +132,13 @@ func TestParseBigInt(t *testing.T) { if ret.String() != test.base10 { t.Errorf("#%d: bad result from %x, got %s want %s", i, test.in, ret.String(), test.base10) } - fw := newForkableWriter() - marshalBigInt(fw, ret) - result := fw.Bytes() + e, err := makeBigInt(ret) + if err != nil { + t.Errorf("%d: err=%q", i, err) + continue + } + result := make([]byte, e.Len()) + e.Encode(result) if !bytes.Equal(result, test.in) { t.Errorf("#%d: got %x from marshaling %s, want %x", i, result, ret, test.in) } @@ -364,7 +368,7 @@ var tagAndLengthData = []tagAndLengthTest{ {[]byte{0xa0, 0x01}, true, tagAndLength{2, 0, 1, true}}, {[]byte{0x02, 0x00}, true, tagAndLength{0, 2, 0, false}}, {[]byte{0xfe, 0x00}, true, tagAndLength{3, 30, 0, true}}, - {[]byte{0x1f, 0x01, 0x00}, true, tagAndLength{0, 1, 0, false}}, + {[]byte{0x1f, 0x1f, 0x00}, true, tagAndLength{0, 31, 0, false}}, {[]byte{0x1f, 0x81, 0x00, 0x00}, true, tagAndLength{0, 128, 0, false}}, {[]byte{0x1f, 0x81, 0x80, 0x01, 0x00}, true, tagAndLength{0, 0x4001, 0, false}}, {[]byte{0x00, 0x81, 0x80}, true, tagAndLength{0, 0, 128, false}}, @@ -382,6 +386,8 @@ var tagAndLengthData = []tagAndLengthTest{ {[]byte{0xa0, 0x81, 0x7f}, false, tagAndLength{}}, // Tag numbers which would overflow int32 are rejected. (The value below is 2^31.) {[]byte{0x1f, 0x88, 0x80, 0x80, 0x80, 0x00, 0x00}, false, tagAndLength{}}, + // Long tag number form may not be used for tags that fit in short form. + {[]byte{0x1f, 0x1e, 0x00}, false, tagAndLength{}}, } func TestParseTagAndLength(t *testing.T) { @@ -961,7 +967,7 @@ func TestUnmarshalInvalidUTF8(t *testing.T) { func TestMarshalNilValue(t *testing.T) { nilValueTestData := []interface{}{ nil, - struct{ v interface{} }{}, + struct{ V interface{} }{}, } for i, test := range nilValueTestData { if _, err := Marshal(test); err == nil { @@ -969,3 +975,32 @@ func TestMarshalNilValue(t *testing.T) { } } } + +type unexported struct { + X int + y int +} + +type exported struct { + X int + Y int +} + +func TestUnexportedStructField(t *testing.T) { + want := StructuralError{"struct contains unexported fields"} + + _, err := Marshal(unexported{X: 5, y: 1}) + if err != want { + t.Errorf("got %v, want %v", err, want) + } + + bs, err := Marshal(exported{X: 5, Y: 1}) + if err != nil { + t.Fatal(err) + } + var u unexported + _, err = Unmarshal(bs, &u) + if err != want { + t.Errorf("got %v, want %v", err, want) + } +} diff --git a/libgo/go/encoding/asn1/marshal.go b/libgo/go/encoding/asn1/marshal.go index 6e858584a6..225fd0849c 100644 --- a/libgo/go/encoding/asn1/marshal.go +++ b/libgo/go/encoding/asn1/marshal.go @@ -5,77 +5,125 @@ package asn1 import ( - "bytes" "errors" "fmt" - "io" "math/big" "reflect" "time" "unicode/utf8" ) -// A forkableWriter is an in-memory buffer that can be -// 'forked' to create new forkableWriters that bracket the -// original. After -// pre, post := w.fork() -// the overall sequence of bytes represented is logically w+pre+post. -type forkableWriter struct { - *bytes.Buffer - pre, post *forkableWriter +var ( + byte00Encoder encoder = byteEncoder(0x00) + byteFFEncoder encoder = byteEncoder(0xff) +) + +// encoder represents a ASN.1 element that is waiting to be marshaled. +type encoder interface { + // Len returns the number of bytes needed to marshal this element. + Len() int + // Encode encodes this element by writing Len() bytes to dst. + Encode(dst []byte) +} + +type byteEncoder byte + +func (c byteEncoder) Len() int { + return 1 } -func newForkableWriter() *forkableWriter { - return &forkableWriter{new(bytes.Buffer), nil, nil} +func (c byteEncoder) Encode(dst []byte) { + dst[0] = byte(c) } -func (f *forkableWriter) fork() (pre, post *forkableWriter) { - if f.pre != nil || f.post != nil { - panic("have already forked") +type bytesEncoder []byte + +func (b bytesEncoder) Len() int { + return len(b) +} + +func (b bytesEncoder) Encode(dst []byte) { + if copy(dst, b) != len(b) { + panic("internal error") } - f.pre = newForkableWriter() - f.post = newForkableWriter() - return f.pre, f.post } -func (f *forkableWriter) Len() (l int) { - l += f.Buffer.Len() - if f.pre != nil { - l += f.pre.Len() +type stringEncoder string + +func (s stringEncoder) Len() int { + return len(s) +} + +func (s stringEncoder) Encode(dst []byte) { + if copy(dst, s) != len(s) { + panic("internal error") } - if f.post != nil { - l += f.post.Len() +} + +type multiEncoder []encoder + +func (m multiEncoder) Len() int { + var size int + for _, e := range m { + size += e.Len() } - return + return size } -func (f *forkableWriter) writeTo(out io.Writer) (n int, err error) { - n, err = out.Write(f.Bytes()) - if err != nil { - return +func (m multiEncoder) Encode(dst []byte) { + var off int + for _, e := range m { + e.Encode(dst[off:]) + off += e.Len() } +} - var nn int +type taggedEncoder struct { + // scratch contains temporary space for encoding the tag and length of + // an element in order to avoid extra allocations. + scratch [8]byte + tag encoder + body encoder +} - if f.pre != nil { - nn, err = f.pre.writeTo(out) - n += nn - if err != nil { - return - } +func (t *taggedEncoder) Len() int { + return t.tag.Len() + t.body.Len() +} + +func (t *taggedEncoder) Encode(dst []byte) { + t.tag.Encode(dst) + t.body.Encode(dst[t.tag.Len():]) +} + +type int64Encoder int64 + +func (i int64Encoder) Len() int { + n := 1 + + for i > 127 { + n++ + i >>= 8 } - if f.post != nil { - nn, err = f.post.writeTo(out) - n += nn + for i < -128 { + n++ + i >>= 8 } - return + + return n } -func marshalBase128Int(out *forkableWriter, n int64) (err error) { +func (i int64Encoder) Encode(dst []byte) { + n := i.Len() + + for j := 0; j < n; j++ { + dst[j] = byte(i >> uint((n-1-j)*8)) + } +} + +func base128IntLength(n int64) int { if n == 0 { - err = out.WriteByte(0) - return + return 1 } l := 0 @@ -83,54 +131,33 @@ func marshalBase128Int(out *forkableWriter, n int64) (err error) { l++ } + return l +} + +func appendBase128Int(dst []byte, n int64) []byte { + l := base128IntLength(n) + for i := l - 1; i >= 0; i-- { o := byte(n >> uint(i*7)) o &= 0x7f if i != 0 { o |= 0x80 } - err = out.WriteByte(o) - if err != nil { - return - } - } - - return nil -} -func marshalInt64(out *forkableWriter, i int64) (err error) { - n := int64Length(i) - - for ; n > 0; n-- { - err = out.WriteByte(byte(i >> uint((n-1)*8))) - if err != nil { - return - } + dst = append(dst, o) } - return nil + return dst } -func int64Length(i int64) (numBytes int) { - numBytes = 1 - - for i > 127 { - numBytes++ - i >>= 8 +func makeBigInt(n *big.Int) (encoder, error) { + if n == nil { + return nil, StructuralError{"empty integer"} } - for i < -128 { - numBytes++ - i >>= 8 - } - - return -} - -func marshalBigInt(out *forkableWriter, n *big.Int) (err error) { if n.Sign() < 0 { // A negative number has to be converted to two's-complement - // form. So we'll subtract 1 and invert. If the + // form. So we'll invert and subtract 1. If the // most-significant-bit isn't set then we'll need to pad the // beginning with 0xff in order to keep the number negative. nMinus1 := new(big.Int).Neg(n) @@ -140,41 +167,31 @@ func marshalBigInt(out *forkableWriter, n *big.Int) (err error) { bytes[i] ^= 0xff } if len(bytes) == 0 || bytes[0]&0x80 == 0 { - err = out.WriteByte(0xff) - if err != nil { - return - } + return multiEncoder([]encoder{byteFFEncoder, bytesEncoder(bytes)}), nil } - _, err = out.Write(bytes) + return bytesEncoder(bytes), nil } else if n.Sign() == 0 { // Zero is written as a single 0 zero rather than no bytes. - err = out.WriteByte(0x00) + return byte00Encoder, nil } else { bytes := n.Bytes() if len(bytes) > 0 && bytes[0]&0x80 != 0 { // We'll have to pad this with 0x00 in order to stop it // looking like a negative number. - err = out.WriteByte(0) - if err != nil { - return - } + return multiEncoder([]encoder{byte00Encoder, bytesEncoder(bytes)}), nil } - _, err = out.Write(bytes) + return bytesEncoder(bytes), nil } - return } -func marshalLength(out *forkableWriter, i int) (err error) { +func appendLength(dst []byte, i int) []byte { n := lengthLength(i) for ; n > 0; n-- { - err = out.WriteByte(byte(i >> uint((n-1)*8))) - if err != nil { - return - } + dst = append(dst, byte(i>>uint((n-1)*8))) } - return nil + return dst } func lengthLength(i int) (numBytes int) { @@ -186,123 +203,104 @@ func lengthLength(i int) (numBytes int) { return } -func marshalTagAndLength(out *forkableWriter, t tagAndLength) (err error) { +func appendTagAndLength(dst []byte, t tagAndLength) []byte { b := uint8(t.class) << 6 if t.isCompound { b |= 0x20 } if t.tag >= 31 { b |= 0x1f - err = out.WriteByte(b) - if err != nil { - return - } - err = marshalBase128Int(out, int64(t.tag)) - if err != nil { - return - } + dst = append(dst, b) + dst = appendBase128Int(dst, int64(t.tag)) } else { b |= uint8(t.tag) - err = out.WriteByte(b) - if err != nil { - return - } + dst = append(dst, b) } if t.length >= 128 { l := lengthLength(t.length) - err = out.WriteByte(0x80 | byte(l)) - if err != nil { - return - } - err = marshalLength(out, t.length) - if err != nil { - return - } + dst = append(dst, 0x80|byte(l)) + dst = appendLength(dst, t.length) } else { - err = out.WriteByte(byte(t.length)) - if err != nil { - return - } + dst = append(dst, byte(t.length)) } - return nil + return dst } -func marshalBitString(out *forkableWriter, b BitString) (err error) { - paddingBits := byte((8 - b.BitLength%8) % 8) - err = out.WriteByte(paddingBits) - if err != nil { - return - } - _, err = out.Write(b.Bytes) - return +type bitStringEncoder BitString + +func (b bitStringEncoder) Len() int { + return len(b.Bytes) + 1 } -func marshalObjectIdentifier(out *forkableWriter, oid []int) (err error) { - if len(oid) < 2 || oid[0] > 2 || (oid[0] < 2 && oid[1] >= 40) { - return StructuralError{"invalid object identifier"} +func (b bitStringEncoder) Encode(dst []byte) { + dst[0] = byte((8 - b.BitLength%8) % 8) + if copy(dst[1:], b.Bytes) != len(b.Bytes) { + panic("internal error") } +} - err = marshalBase128Int(out, int64(oid[0]*40+oid[1])) - if err != nil { - return +type oidEncoder []int + +func (oid oidEncoder) Len() int { + l := base128IntLength(int64(oid[0]*40 + oid[1])) + for i := 2; i < len(oid); i++ { + l += base128IntLength(int64(oid[i])) } + return l +} + +func (oid oidEncoder) Encode(dst []byte) { + dst = appendBase128Int(dst[:0], int64(oid[0]*40+oid[1])) for i := 2; i < len(oid); i++ { - err = marshalBase128Int(out, int64(oid[i])) - if err != nil { - return - } + dst = appendBase128Int(dst, int64(oid[i])) } +} - return +func makeObjectIdentifier(oid []int) (e encoder, err error) { + if len(oid) < 2 || oid[0] > 2 || (oid[0] < 2 && oid[1] >= 40) { + return nil, StructuralError{"invalid object identifier"} + } + + return oidEncoder(oid), nil } -func marshalPrintableString(out *forkableWriter, s string) (err error) { - b := []byte(s) - for _, c := range b { - if !isPrintable(c) { - return StructuralError{"PrintableString contains invalid character"} +func makePrintableString(s string) (e encoder, err error) { + for i := 0; i < len(s); i++ { + if !isPrintable(s[i]) { + return nil, StructuralError{"PrintableString contains invalid character"} } } - _, err = out.Write(b) - return + return stringEncoder(s), nil } -func marshalIA5String(out *forkableWriter, s string) (err error) { - b := []byte(s) - for _, c := range b { - if c > 127 { - return StructuralError{"IA5String contains invalid character"} +func makeIA5String(s string) (e encoder, err error) { + for i := 0; i < len(s); i++ { + if s[i] > 127 { + return nil, StructuralError{"IA5String contains invalid character"} } } - _, err = out.Write(b) - return + return stringEncoder(s), nil } -func marshalUTF8String(out *forkableWriter, s string) (err error) { - _, err = out.Write([]byte(s)) - return +func makeUTF8String(s string) encoder { + return stringEncoder(s) } -func marshalTwoDigits(out *forkableWriter, v int) (err error) { - err = out.WriteByte(byte('0' + (v/10)%10)) - if err != nil { - return - } - return out.WriteByte(byte('0' + v%10)) +func appendTwoDigits(dst []byte, v int) []byte { + return append(dst, byte('0'+(v/10)%10), byte('0'+v%10)) } -func marshalFourDigits(out *forkableWriter, v int) (err error) { +func appendFourDigits(dst []byte, v int) []byte { var bytes [4]byte for i := range bytes { bytes[3-i] = '0' + byte(v%10) v /= 10 } - _, err = out.Write(bytes[:]) - return + return append(dst, bytes[:]...) } func outsideUTCRange(t time.Time) bool { @@ -310,80 +308,75 @@ func outsideUTCRange(t time.Time) bool { return year < 1950 || year >= 2050 } -func marshalUTCTime(out *forkableWriter, t time.Time) (err error) { +func makeUTCTime(t time.Time) (e encoder, err error) { + dst := make([]byte, 0, 18) + + dst, err = appendUTCTime(dst, t) + if err != nil { + return nil, err + } + + return bytesEncoder(dst), nil +} + +func makeGeneralizedTime(t time.Time) (e encoder, err error) { + dst := make([]byte, 0, 20) + + dst, err = appendGeneralizedTime(dst, t) + if err != nil { + return nil, err + } + + return bytesEncoder(dst), nil +} + +func appendUTCTime(dst []byte, t time.Time) (ret []byte, err error) { year := t.Year() switch { case 1950 <= year && year < 2000: - err = marshalTwoDigits(out, int(year-1900)) + dst = appendTwoDigits(dst, year-1900) case 2000 <= year && year < 2050: - err = marshalTwoDigits(out, int(year-2000)) + dst = appendTwoDigits(dst, year-2000) default: - return StructuralError{"cannot represent time as UTCTime"} - } - if err != nil { - return + return nil, StructuralError{"cannot represent time as UTCTime"} } - return marshalTimeCommon(out, t) + return appendTimeCommon(dst, t), nil } -func marshalGeneralizedTime(out *forkableWriter, t time.Time) (err error) { +func appendGeneralizedTime(dst []byte, t time.Time) (ret []byte, err error) { year := t.Year() if year < 0 || year > 9999 { - return StructuralError{"cannot represent time as GeneralizedTime"} - } - if err = marshalFourDigits(out, year); err != nil { - return + return nil, StructuralError{"cannot represent time as GeneralizedTime"} } - return marshalTimeCommon(out, t) + dst = appendFourDigits(dst, year) + + return appendTimeCommon(dst, t), nil } -func marshalTimeCommon(out *forkableWriter, t time.Time) (err error) { +func appendTimeCommon(dst []byte, t time.Time) []byte { _, month, day := t.Date() - err = marshalTwoDigits(out, int(month)) - if err != nil { - return - } - - err = marshalTwoDigits(out, day) - if err != nil { - return - } + dst = appendTwoDigits(dst, int(month)) + dst = appendTwoDigits(dst, day) hour, min, sec := t.Clock() - err = marshalTwoDigits(out, hour) - if err != nil { - return - } - - err = marshalTwoDigits(out, min) - if err != nil { - return - } - - err = marshalTwoDigits(out, sec) - if err != nil { - return - } + dst = appendTwoDigits(dst, hour) + dst = appendTwoDigits(dst, min) + dst = appendTwoDigits(dst, sec) _, offset := t.Zone() switch { case offset/60 == 0: - err = out.WriteByte('Z') - return + return append(dst, 'Z') case offset > 0: - err = out.WriteByte('+') + dst = append(dst, '+') case offset < 0: - err = out.WriteByte('-') - } - - if err != nil { - return + dst = append(dst, '-') } offsetMinutes := offset / 60 @@ -391,13 +384,10 @@ func marshalTimeCommon(out *forkableWriter, t time.Time) (err error) { offsetMinutes = -offsetMinutes } - err = marshalTwoDigits(out, offsetMinutes/60) - if err != nil { - return - } + dst = appendTwoDigits(dst, offsetMinutes/60) + dst = appendTwoDigits(dst, offsetMinutes%60) - err = marshalTwoDigits(out, offsetMinutes%60) - return + return dst } func stripTagAndLength(in []byte) []byte { @@ -408,114 +398,130 @@ func stripTagAndLength(in []byte) []byte { return in[offset:] } -func marshalBody(out *forkableWriter, value reflect.Value, params fieldParameters) (err error) { +func makeBody(value reflect.Value, params fieldParameters) (e encoder, err error) { switch value.Type() { case flagType: - return nil + return bytesEncoder(nil), nil case timeType: t := value.Interface().(time.Time) if params.timeType == TagGeneralizedTime || outsideUTCRange(t) { - return marshalGeneralizedTime(out, t) - } else { - return marshalUTCTime(out, t) + return makeGeneralizedTime(t) } + return makeUTCTime(t) case bitStringType: - return marshalBitString(out, value.Interface().(BitString)) + return bitStringEncoder(value.Interface().(BitString)), nil case objectIdentifierType: - return marshalObjectIdentifier(out, value.Interface().(ObjectIdentifier)) + return makeObjectIdentifier(value.Interface().(ObjectIdentifier)) case bigIntType: - return marshalBigInt(out, value.Interface().(*big.Int)) + return makeBigInt(value.Interface().(*big.Int)) } switch v := value; v.Kind() { case reflect.Bool: if v.Bool() { - return out.WriteByte(255) - } else { - return out.WriteByte(0) + return byteFFEncoder, nil } + return byte00Encoder, nil case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - return marshalInt64(out, int64(v.Int())) + return int64Encoder(v.Int()), nil case reflect.Struct: t := v.Type() + for i := 0; i < t.NumField(); i++ { + if t.Field(i).PkgPath != "" { + return nil, StructuralError{"struct contains unexported fields"} + } + } + startingField := 0 + n := t.NumField() + if n == 0 { + return bytesEncoder(nil), nil + } + // If the first element of the structure is a non-empty // RawContents, then we don't bother serializing the rest. - if t.NumField() > 0 && t.Field(0).Type == rawContentsType { + if t.Field(0).Type == rawContentsType { s := v.Field(0) if s.Len() > 0 { - bytes := make([]byte, s.Len()) - for i := 0; i < s.Len(); i++ { - bytes[i] = uint8(s.Index(i).Uint()) - } + bytes := s.Bytes() /* The RawContents will contain the tag and * length fields but we'll also be writing * those ourselves, so we strip them out of * bytes */ - _, err = out.Write(stripTagAndLength(bytes)) - return - } else { - startingField = 1 + return bytesEncoder(stripTagAndLength(bytes)), nil } + + startingField = 1 } - for i := startingField; i < t.NumField(); i++ { - var pre *forkableWriter - pre, out = out.fork() - err = marshalField(pre, v.Field(i), parseFieldParameters(t.Field(i).Tag.Get("asn1"))) - if err != nil { - return + switch n1 := n - startingField; n1 { + case 0: + return bytesEncoder(nil), nil + case 1: + return makeField(v.Field(startingField), parseFieldParameters(t.Field(startingField).Tag.Get("asn1"))) + default: + m := make([]encoder, n1) + for i := 0; i < n1; i++ { + m[i], err = makeField(v.Field(i+startingField), parseFieldParameters(t.Field(i+startingField).Tag.Get("asn1"))) + if err != nil { + return nil, err + } } + + return multiEncoder(m), nil } - return case reflect.Slice: sliceType := v.Type() if sliceType.Elem().Kind() == reflect.Uint8 { - bytes := make([]byte, v.Len()) - for i := 0; i < v.Len(); i++ { - bytes[i] = uint8(v.Index(i).Uint()) - } - _, err = out.Write(bytes) - return + return bytesEncoder(v.Bytes()), nil } var fp fieldParameters - for i := 0; i < v.Len(); i++ { - var pre *forkableWriter - pre, out = out.fork() - err = marshalField(pre, v.Index(i), fp) - if err != nil { - return + + switch l := v.Len(); l { + case 0: + return bytesEncoder(nil), nil + case 1: + return makeField(v.Index(0), fp) + default: + m := make([]encoder, l) + + for i := 0; i < l; i++ { + m[i], err = makeField(v.Index(i), fp) + if err != nil { + return nil, err + } } + + return multiEncoder(m), nil } - return case reflect.String: switch params.stringType { case TagIA5String: - return marshalIA5String(out, v.String()) + return makeIA5String(v.String()) case TagPrintableString: - return marshalPrintableString(out, v.String()) + return makePrintableString(v.String()) default: - return marshalUTF8String(out, v.String()) + return makeUTF8String(v.String()), nil } } - return StructuralError{"unknown Go type"} + return nil, StructuralError{"unknown Go type"} } -func marshalField(out *forkableWriter, v reflect.Value, params fieldParameters) (err error) { +func makeField(v reflect.Value, params fieldParameters) (e encoder, err error) { if !v.IsValid() { - return fmt.Errorf("asn1: cannot marshal nil value") + return nil, fmt.Errorf("asn1: cannot marshal nil value") } // If the field is an interface{} then recurse into it. if v.Kind() == reflect.Interface && v.Type().NumMethod() == 0 { - return marshalField(out, v.Elem(), params) + return makeField(v.Elem(), params) } if v.Kind() == reflect.Slice && v.Len() == 0 && params.omitEmpty { - return + return bytesEncoder(nil), nil } if params.optional && params.defaultValue != nil && canHaveDefaultValue(v.Kind()) { @@ -523,46 +529,45 @@ func marshalField(out *forkableWriter, v reflect.Value, params fieldParameters) defaultValue.SetInt(*params.defaultValue) if reflect.DeepEqual(v.Interface(), defaultValue.Interface()) { - return + return bytesEncoder(nil), nil } } // If no default value is given then the zero value for the type is // assumed to be the default value. This isn't obviously the correct - // behaviour, but it's what Go has traditionally done. + // behavior, but it's what Go has traditionally done. if params.optional && params.defaultValue == nil { if reflect.DeepEqual(v.Interface(), reflect.Zero(v.Type()).Interface()) { - return + return bytesEncoder(nil), nil } } if v.Type() == rawValueType { rv := v.Interface().(RawValue) if len(rv.FullBytes) != 0 { - _, err = out.Write(rv.FullBytes) - } else { - err = marshalTagAndLength(out, tagAndLength{rv.Class, rv.Tag, len(rv.Bytes), rv.IsCompound}) - if err != nil { - return - } - _, err = out.Write(rv.Bytes) + return bytesEncoder(rv.FullBytes), nil } - return + + t := new(taggedEncoder) + + t.tag = bytesEncoder(appendTagAndLength(t.scratch[:0], tagAndLength{rv.Class, rv.Tag, len(rv.Bytes), rv.IsCompound})) + t.body = bytesEncoder(rv.Bytes) + + return t, nil } tag, isCompound, ok := getUniversalType(v.Type()) if !ok { - err = StructuralError{fmt.Sprintf("unknown Go type: %v", v.Type())} - return + return nil, StructuralError{fmt.Sprintf("unknown Go type: %v", v.Type())} } class := ClassUniversal if params.timeType != 0 && tag != TagUTCTime { - return StructuralError{"explicit time type given to non-time member"} + return nil, StructuralError{"explicit time type given to non-time member"} } if params.stringType != 0 && tag != TagPrintableString { - return StructuralError{"explicit string type given to non-string member"} + return nil, StructuralError{"explicit string type given to non-string member"} } switch tag { @@ -574,7 +579,7 @@ func marshalField(out *forkableWriter, v reflect.Value, params fieldParameters) for _, r := range v.String() { if r >= utf8.RuneSelf || !isPrintable(byte(r)) { if !utf8.ValidString(v.String()) { - return errors.New("asn1: string not valid UTF-8") + return nil, errors.New("asn1: string not valid UTF-8") } tag = TagUTF8String break @@ -591,46 +596,46 @@ func marshalField(out *forkableWriter, v reflect.Value, params fieldParameters) if params.set { if tag != TagSequence { - return StructuralError{"non sequence tagged as set"} + return nil, StructuralError{"non sequence tagged as set"} } tag = TagSet } - tags, body := out.fork() + t := new(taggedEncoder) - err = marshalBody(body, v, params) + t.body, err = makeBody(v, params) if err != nil { - return + return nil, err } - bodyLen := body.Len() + bodyLen := t.body.Len() - var explicitTag *forkableWriter if params.explicit { - explicitTag, tags = tags.fork() - } + t.tag = bytesEncoder(appendTagAndLength(t.scratch[:0], tagAndLength{class, tag, bodyLen, isCompound})) - if !params.explicit && params.tag != nil { - // implicit tag. - tag = *params.tag - class = ClassContextSpecific - } + tt := new(taggedEncoder) - err = marshalTagAndLength(tags, tagAndLength{class, tag, bodyLen, isCompound}) - if err != nil { - return - } + tt.body = t - if params.explicit { - err = marshalTagAndLength(explicitTag, tagAndLength{ + tt.tag = bytesEncoder(appendTagAndLength(tt.scratch[:0], tagAndLength{ class: ClassContextSpecific, tag: *params.tag, - length: bodyLen + tags.Len(), + length: bodyLen + t.tag.Len(), isCompound: true, - }) + })) + + return tt, nil + } + + if params.tag != nil { + // implicit tag. + tag = *params.tag + class = ClassContextSpecific } - return err + t.tag = bytesEncoder(appendTagAndLength(t.scratch[:0], tagAndLength{class, tag, bodyLen, isCompound})) + + return t, nil } // Marshal returns the ASN.1 encoding of val. @@ -643,13 +648,11 @@ func marshalField(out *forkableWriter, v reflect.Value, params fieldParameters) // printable: causes strings to be marshaled as ASN.1, PrintableString strings. // utf8: causes strings to be marshaled as ASN.1, UTF8 strings func Marshal(val interface{}) ([]byte, error) { - var out bytes.Buffer - v := reflect.ValueOf(val) - f := newForkableWriter() - err := marshalField(f, v, fieldParameters{}) + e, err := makeField(reflect.ValueOf(val), fieldParameters{}) if err != nil { return nil, err } - _, err = f.writeTo(&out) - return out.Bytes(), err + b := make([]byte, e.Len()) + e.Encode(b) + return b, nil } diff --git a/libgo/go/encoding/asn1/marshal_test.go b/libgo/go/encoding/asn1/marshal_test.go index cdca8aa336..10db1aa575 100644 --- a/libgo/go/encoding/asn1/marshal_test.go +++ b/libgo/go/encoding/asn1/marshal_test.go @@ -8,6 +8,7 @@ import ( "bytes" "encoding/hex" "math/big" + "strings" "testing" "time" ) @@ -167,9 +168,42 @@ func TestMarshal(t *testing.T) { } } +type marshalErrTest struct { + in interface{} + err string +} + +var marshalErrTests = []marshalErrTest{ + {bigIntStruct{nil}, "empty integer"}, +} + +func TestMarshalError(t *testing.T) { + for i, test := range marshalErrTests { + _, err := Marshal(test.in) + if err == nil { + t.Errorf("#%d should fail, but success", i) + continue + } + + if !strings.Contains(err.Error(), test.err) { + t.Errorf("#%d got: %v want %v", i, err, test.err) + } + } +} + func TestInvalidUTF8(t *testing.T) { _, err := Marshal(string([]byte{0xff, 0xff})) if err == nil { t.Errorf("invalid UTF8 string was accepted") } } + +func BenchmarkMarshal(b *testing.B) { + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + for _, test := range marshalTests { + Marshal(test.in) + } + } +} diff --git a/libgo/go/encoding/base32/base32.go b/libgo/go/encoding/base32/base32.go index 5a9e86919d..c193e65e1b 100644 --- a/libgo/go/encoding/base32/base32.go +++ b/libgo/go/encoding/base32/base32.go @@ -17,7 +17,7 @@ import ( */ // An Encoding is a radix 32 encoding/decoding scheme, defined by a -// 32-character alphabet. The most common is the "base32" encoding +// 32-character alphabet. The most common is the "base32" encoding // introduced for SASL GSSAPI and standardized in RFC 4648. // The alternate "base32hex" encoding is used in DNSSEC. type Encoding struct { @@ -66,7 +66,7 @@ var removeNewlinesMapper = func(r rune) rune { // // The encoding pads the output to a multiple of 8 bytes, // so Encode is not appropriate for use on individual blocks -// of a large data stream. Use NewEncoder() instead. +// of a large data stream. Use NewEncoder() instead. func (enc *Encoding) Encode(dst, src []byte) { if len(src) == 0 { return @@ -208,7 +208,7 @@ func (e *encoder) Close() error { return e.err } -// NewEncoder returns a new base32 stream encoder. Data written to +// NewEncoder returns a new base32 stream encoder. Data written to // the returned writer will be encoded using enc and then written to w. // Base32 encodings operate in 5-byte blocks; when finished // writing, the caller must Close the returned encoder to flush any @@ -313,9 +313,9 @@ func (enc *Encoding) decode(dst, src []byte) (n int, end bool, err error) { return n, end, nil } -// Decode decodes src using the encoding enc. It writes at most +// Decode decodes src using the encoding enc. It writes at most // DecodedLen(len(src)) bytes to dst and returns the number of bytes -// written. If src contains invalid base32 data, it will return the +// written. If src contains invalid base32 data, it will return the // number of bytes successfully written and CorruptInputError. // New line characters (\r and \n) are ignored. func (enc *Encoding) Decode(dst, src []byte) (n int, err error) { diff --git a/libgo/go/encoding/base32/base32_test.go b/libgo/go/encoding/base32/base32_test.go index 5a68f06e1c..66a48a3f6f 100644 --- a/libgo/go/encoding/base32/base32_test.go +++ b/libgo/go/encoding/base32/base32_test.go @@ -171,7 +171,7 @@ func TestDecodeCorrupt(t *testing.T) { _, err := StdEncoding.Decode(dbuf, []byte(tc.input)) if tc.offset == -1 { if err != nil { - t.Error("Decoder wrongly detected coruption in", tc.input) + t.Error("Decoder wrongly detected corruption in", tc.input) } continue } diff --git a/libgo/go/encoding/base64/base64.go b/libgo/go/encoding/base64/base64.go index 1bda804c38..d2efad4518 100644 --- a/libgo/go/encoding/base64/base64.go +++ b/libgo/go/encoding/base64/base64.go @@ -15,7 +15,7 @@ import ( */ // An Encoding is a radix 64 encoding/decoding scheme, defined by a -// 64-character alphabet. The most common encoding is the "base64" +// 64-character alphabet. The most common encoding is the "base64" // encoding defined in RFC 4648 and used in MIME (RFC 2045) and PEM // (RFC 1421). RFC 4648 also defines an alternate encoding, which is // the standard encoding with - and _ substituted for + and /. @@ -23,6 +23,7 @@ type Encoding struct { encode [64]byte decodeMap [256]byte padChar rune + strict bool } const ( @@ -62,6 +63,14 @@ func (enc Encoding) WithPadding(padding rune) *Encoding { return &enc } +// Strict creates a new encoding identical to enc except with +// strict decoding enabled. In this mode, the decoder requires that +// trailing padding bits are zero, as described in RFC 4648 section 3.5. +func (enc Encoding) Strict() *Encoding { + enc.strict = true + return &enc +} + // StdEncoding is the standard base64 encoding, as defined in // RFC 4648. var StdEncoding = NewEncoding(encodeStd) @@ -89,7 +98,7 @@ var RawURLEncoding = URLEncoding.WithPadding(NoPadding) // // The encoding pads the output to a multiple of 4 bytes, // so Encode is not appropriate for use on individual blocks -// of a large data stream. Use NewEncoder() instead. +// of a large data stream. Use NewEncoder() instead. func (enc *Encoding) Encode(dst, src []byte) { if len(src) == 0 { return @@ -213,7 +222,7 @@ func (e *encoder) Close() error { return e.err } -// NewEncoder returns a new base64 stream encoder. Data written to +// NewEncoder returns a new base64 stream encoder. Data written to // the returned writer will be encoded using enc and then written to w. // Base64 encodings operate in 4-byte blocks; when finished // writing, the caller must Close the returned encoder to flush any @@ -311,15 +320,24 @@ func (enc *Encoding) decode(dst, src []byte) (n int, end bool, err error) { // Convert 4x 6bit source bytes into 3 bytes val := uint(dbuf[0])<<18 | uint(dbuf[1])<<12 | uint(dbuf[2])<<6 | uint(dbuf[3]) + dbuf[2], dbuf[1], dbuf[0] = byte(val>>0), byte(val>>8), byte(val>>16) switch dlen { case 4: - dst[2] = byte(val >> 0) + dst[2] = dbuf[2] + dbuf[2] = 0 fallthrough case 3: - dst[1] = byte(val >> 8) + dst[1] = dbuf[1] + if enc.strict && dbuf[2] != 0 { + return n, end, CorruptInputError(si - 1) + } + dbuf[1] = 0 fallthrough case 2: - dst[0] = byte(val >> 16) + dst[0] = dbuf[0] + if enc.strict && (dbuf[1] != 0 || dbuf[2] != 0) { + return n, end, CorruptInputError(si - 2) + } } dst = dst[dinc:] n += dlen - 1 @@ -328,9 +346,9 @@ func (enc *Encoding) decode(dst, src []byte) (n int, end bool, err error) { return n, end, err } -// Decode decodes src using the encoding enc. It writes at most +// Decode decodes src using the encoding enc. It writes at most // DecodedLen(len(src)) bytes to dst and returns the number of bytes -// written. If src contains invalid base64 data, it will return the +// written. If src contains invalid base64 data, it will return the // number of bytes successfully written and CorruptInputError. // New line characters (\r and \n) are ignored. func (enc *Encoding) Decode(dst, src []byte) (n int, err error) { @@ -459,7 +477,7 @@ func NewDecoder(enc *Encoding, r io.Reader) io.Reader { func (enc *Encoding) DecodedLen(n int) int { if enc.padChar == NoPadding { // Unpadded data may end with partial block of 2-3 characters. - return (n*6 + 7) / 8 + return n * 6 / 8 } // Padded base64 should always be a multiple of 4 characters in length. return n / 4 * 3 diff --git a/libgo/go/encoding/base64/base64_test.go b/libgo/go/encoding/base64/base64_test.go index fc6a1ea654..e2e1d59f3c 100644 --- a/libgo/go/encoding/base64/base64_test.go +++ b/libgo/go/encoding/base64/base64_test.go @@ -85,6 +85,11 @@ var encodingTests = []encodingTest{ {RawStdEncoding, rawRef}, {RawURLEncoding, rawUrlRef}, {funnyEncoding, funnyRef}, + {StdEncoding.Strict(), stdRef}, + {URLEncoding.Strict(), urlRef}, + {RawStdEncoding.Strict(), rawRef}, + {RawURLEncoding.Strict(), rawUrlRef}, + {funnyEncoding.Strict(), funnyRef}, } var bigtest = testpair{ @@ -221,7 +226,7 @@ func TestDecodeCorrupt(t *testing.T) { _, err := StdEncoding.Decode(dbuf, []byte(tc.input)) if tc.offset == -1 { if err != nil { - t.Error("Decoder wrongly detected coruption in", tc.input) + t.Error("Decoder wrongly detected corruption in", tc.input) } continue } @@ -234,6 +239,51 @@ func TestDecodeCorrupt(t *testing.T) { } } +func TestEncodedLen(t *testing.T) { + for _, tt := range []struct { + enc *Encoding + n int + want int + }{ + {RawStdEncoding, 0, 0}, + {RawStdEncoding, 1, 2}, + {RawStdEncoding, 2, 3}, + {RawStdEncoding, 3, 4}, + {RawStdEncoding, 7, 10}, + {StdEncoding, 0, 0}, + {StdEncoding, 1, 4}, + {StdEncoding, 2, 4}, + {StdEncoding, 3, 4}, + {StdEncoding, 4, 8}, + {StdEncoding, 7, 12}, + } { + if got := tt.enc.EncodedLen(tt.n); got != tt.want { + t.Errorf("EncodedLen(%d): got %d, want %d", tt.n, got, tt.want) + } + } +} + +func TestDecodedLen(t *testing.T) { + for _, tt := range []struct { + enc *Encoding + n int + want int + }{ + {RawStdEncoding, 0, 0}, + {RawStdEncoding, 2, 1}, + {RawStdEncoding, 3, 2}, + {RawStdEncoding, 4, 3}, + {RawStdEncoding, 10, 7}, + {StdEncoding, 0, 0}, + {StdEncoding, 4, 3}, + {StdEncoding, 8, 6}, + } { + if got := tt.enc.DecodedLen(tt.n); got != tt.want { + t.Errorf("DecodedLen(%d): got %d, want %d", tt.n, got, tt.want) + } + } +} + func TestBig(t *testing.T) { n := 3*1000 + 1 raw := make([]byte, n) @@ -391,6 +441,22 @@ func TestDecoderIssue7733(t *testing.T) { } } +func TestDecoderIssue15656(t *testing.T) { + _, err := StdEncoding.Strict().DecodeString("WvLTlMrX9NpYDQlEIFlnDB==") + want := CorruptInputError(22) + if !reflect.DeepEqual(want, err) { + t.Errorf("Error = %v; want CorruptInputError(22)", err) + } + _, err = StdEncoding.Strict().DecodeString("WvLTlMrX9NpYDQlEIFlnDA==") + if err != nil { + t.Errorf("Error = %v; want nil", err) + } + _, err = StdEncoding.DecodeString("WvLTlMrX9NpYDQlEIFlnDB==") + if err != nil { + t.Errorf("Error = %v; want nil", err) + } +} + func BenchmarkEncodeToString(b *testing.B) { data := make([]byte, 8192) b.SetBytes(int64(len(data))) diff --git a/libgo/go/encoding/binary/binary.go b/libgo/go/encoding/binary/binary.go index 1c2577b68d..3834254596 100644 --- a/libgo/go/encoding/binary/binary.go +++ b/libgo/go/encoding/binary/binary.go @@ -1,4 +1,4 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -7,7 +7,7 @@ // // Numbers are translated by reading and writing fixed-size values. // A fixed-size value is either a fixed-size arithmetic -// type (int8, uint8, int16, float32, complex64, ...) +// type (bool, int8, uint8, int16, float32, complex64, ...) // or an array or struct containing only fixed-size values. // // The varint functions encode and decode single integer values using @@ -48,18 +48,24 @@ var BigEndian bigEndian type littleEndian struct{} -func (littleEndian) Uint16(b []byte) uint16 { return uint16(b[0]) | uint16(b[1])<<8 } +func (littleEndian) Uint16(b []byte) uint16 { + _ = b[1] // bounds check hint to compiler; see golang.org/issue/14808 + return uint16(b[0]) | uint16(b[1])<<8 +} func (littleEndian) PutUint16(b []byte, v uint16) { + _ = b[1] // early bounds check to guarantee safety of writes below b[0] = byte(v) b[1] = byte(v >> 8) } func (littleEndian) Uint32(b []byte) uint32 { + _ = b[3] // bounds check hint to compiler; see golang.org/issue/14808 return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24 } func (littleEndian) PutUint32(b []byte, v uint32) { + _ = b[3] // early bounds check to guarantee safety of writes below b[0] = byte(v) b[1] = byte(v >> 8) b[2] = byte(v >> 16) @@ -67,11 +73,13 @@ func (littleEndian) PutUint32(b []byte, v uint32) { } func (littleEndian) Uint64(b []byte) uint64 { + _ = b[7] // bounds check hint to compiler; see golang.org/issue/14808 return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 } func (littleEndian) PutUint64(b []byte, v uint64) { + _ = b[7] // early bounds check to guarantee safety of writes below b[0] = byte(v) b[1] = byte(v >> 8) b[2] = byte(v >> 16) @@ -88,18 +96,24 @@ func (littleEndian) GoString() string { return "binary.LittleEndian" } type bigEndian struct{} -func (bigEndian) Uint16(b []byte) uint16 { return uint16(b[1]) | uint16(b[0])<<8 } +func (bigEndian) Uint16(b []byte) uint16 { + _ = b[1] // bounds check hint to compiler; see golang.org/issue/14808 + return uint16(b[1]) | uint16(b[0])<<8 +} func (bigEndian) PutUint16(b []byte, v uint16) { + _ = b[1] // early bounds check to guarantee safety of writes below b[0] = byte(v >> 8) b[1] = byte(v) } func (bigEndian) Uint32(b []byte) uint32 { + _ = b[3] // bounds check hint to compiler; see golang.org/issue/14808 return uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24 } func (bigEndian) PutUint32(b []byte, v uint32) { + _ = b[3] // early bounds check to guarantee safety of writes below b[0] = byte(v >> 24) b[1] = byte(v >> 16) b[2] = byte(v >> 8) @@ -107,11 +121,13 @@ func (bigEndian) PutUint32(b []byte, v uint32) { } func (bigEndian) Uint64(b []byte) uint64 { + _ = b[7] // bounds check hint to compiler; see golang.org/issue/14808 return uint64(b[7]) | uint64(b[6])<<8 | uint64(b[5])<<16 | uint64(b[4])<<24 | uint64(b[3])<<32 | uint64(b[2])<<40 | uint64(b[1])<<48 | uint64(b[0])<<56 } func (bigEndian) PutUint64(b []byte, v uint64) { + _ = b[7] // early bounds check to guarantee safety of writes below b[0] = byte(v >> 56) b[1] = byte(v >> 48) b[2] = byte(v >> 40) @@ -131,6 +147,8 @@ func (bigEndian) GoString() string { return "binary.BigEndian" } // of fixed-size values. // Bytes read from r are decoded using the specified byte order // and written to successive fields of the data. +// When decoding boolean values, a zero byte is decoded as false, and +// any other non-zero byte is decoded as true. // When reading into structs, the field data for fields with // blank (_) field names is skipped; i.e., blank field names // may be used for padding. @@ -153,6 +171,8 @@ func Read(r io.Reader, order ByteOrder, data interface{}) error { return err } switch data := data.(type) { + case *bool: + *data = b[0] != 0 case *int8: *data = int8(b[0]) case *uint8: @@ -169,8 +189,12 @@ func Read(r io.Reader, order ByteOrder, data interface{}) error { *data = int64(order.Uint64(bs)) case *uint64: *data = order.Uint64(bs) - case []int8: + case []bool: for i, x := range bs { // Easier to loop over the input for 8-bit values. + data[i] = x != 0 + } + case []int8: + for i, x := range bs { data[i] = int8(x) } case []uint8: @@ -227,6 +251,7 @@ func Read(r io.Reader, order ByteOrder, data interface{}) error { // Write writes the binary representation of data into w. // Data must be a fixed-size value or a slice of fixed-size // values, or a pointer to such data. +// Boolean values encode as one byte: 1 for true, and 0 for false. // Bytes written to w are encoded using the specified byte order // and read from successive fields of the data. // When writing structs, zero values are written for fields @@ -242,6 +267,26 @@ func Write(w io.Writer, order ByteOrder, data interface{}) error { bs = b[:n] } switch v := data.(type) { + case *bool: + if *v { + b[0] = 1 + } else { + b[0] = 0 + } + case bool: + if v { + b[0] = 1 + } else { + b[0] = 0 + } + case []bool: + for i, x := range v { + if x { + bs[i] = 1 + } else { + bs[i] = 0 + } + } case *int8: b[0] = byte(*v) case int8: @@ -253,7 +298,7 @@ func Write(w io.Writer, order ByteOrder, data interface{}) error { case *uint8: b[0] = *v case uint8: - b[0] = byte(v) + b[0] = v case []uint8: bs = v case *int16: @@ -362,7 +407,8 @@ func sizeof(t reflect.Type) int { } return sum - case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, + case reflect.Bool, + reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Float32, reflect.Float64, reflect.Complex64, reflect.Complex128: return int(t.Size()) @@ -379,6 +425,21 @@ type coder struct { type decoder coder type encoder coder +func (d *decoder) bool() bool { + x := d.buf[0] + d.buf = d.buf[1:] + return x != 0 +} + +func (e *encoder) bool(x bool) { + if x { + e.buf[0] = 1 + } else { + e.buf[0] = 0 + } + e.buf = e.buf[1:] +} + func (d *decoder) uint8() uint8 { x := d.buf[0] d.buf = d.buf[1:] @@ -469,6 +530,9 @@ func (d *decoder) value(v reflect.Value) { d.value(v.Index(i)) } + case reflect.Bool: + v.SetBool(d.bool()) + case reflect.Int8: v.SetInt(int64(d.int8())) case reflect.Int16: @@ -531,6 +595,9 @@ func (e *encoder) value(v reflect.Value) { e.value(v.Index(i)) } + case reflect.Bool: + e.bool(v.Bool()) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: switch v.Type().Kind() { case reflect.Int8: @@ -593,7 +660,7 @@ func (e *encoder) skip(v reflect.Value) { // It returns zero if the type cannot be implemented by the fast path in Read or Write. func intDataSize(data interface{}) int { switch data := data.(type) { - case int8, uint8, *int8, *uint8: + case bool, int8, uint8, *bool, *int8, *uint8: return 1 case []int8: return len(data) diff --git a/libgo/go/encoding/binary/binary_test.go b/libgo/go/encoding/binary/binary_test.go index 7fd36fa4ef..fc7f2765ef 100644 --- a/libgo/go/encoding/binary/binary_test.go +++ b/libgo/go/encoding/binary/binary_test.go @@ -1,4 +1,4 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -27,6 +27,8 @@ type Struct struct { Complex64 complex64 Complex128 complex128 Array [4]uint8 + Bool bool + BoolArray [4]bool } type T struct { @@ -58,6 +60,9 @@ var s = Struct{ ), [4]uint8{0x43, 0x44, 0x45, 0x46}, + + true, + [4]bool{true, false, true, false}, } var big = []byte{ @@ -76,6 +81,9 @@ var big = []byte{ 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, + + 1, + 1, 0, 1, 0, } var little = []byte{ @@ -94,6 +102,9 @@ var little = []byte{ 58, 57, 56, 55, 54, 53, 52, 51, 66, 65, 64, 63, 62, 61, 60, 59, 67, 68, 69, 70, + + 1, + 1, 0, 1, 0, } var src = []byte{1, 2, 3, 4, 5, 6, 7, 8} @@ -141,6 +152,25 @@ func TestWriteSlice(t *testing.T) { checkResult(t, "WriteSlice", BigEndian, err, buf.Bytes(), src) } +func TestReadBool(t *testing.T) { + var res bool + var err error + err = Read(bytes.NewReader([]byte{0}), BigEndian, &res) + checkResult(t, "ReadBool", BigEndian, err, res, false) + res = false + err = Read(bytes.NewReader([]byte{1}), BigEndian, &res) + checkResult(t, "ReadBool", BigEndian, err, res, true) + res = false + err = Read(bytes.NewReader([]byte{2}), BigEndian, &res) + checkResult(t, "ReadBool", BigEndian, err, res, true) +} + +func TestReadBoolSlice(t *testing.T) { + slice := make([]bool, 4) + err := Read(bytes.NewReader([]byte{0, 1, 2, 255}), BigEndian, slice) + checkResult(t, "ReadBoolSlice", BigEndian, err, slice, []bool{false, true, true, true}) +} + // Addresses of arrays are easier to manipulate with reflection than are slices. var intArrays = []interface{}{ &[100]int8{}, @@ -266,7 +296,7 @@ func TestBlankFields(t *testing.T) { } // An attempt to read into a struct with an unexported field will -// panic. This is probably not the best choice, but at this point +// panic. This is probably not the best choice, but at this point // anything else would be an API change. type Unexported struct { @@ -339,6 +369,33 @@ func TestReadTruncated(t *testing.T) { } } +func testUint64SmallSliceLengthPanics() (panicked bool) { + defer func() { + panicked = recover() != nil + }() + b := [8]byte{1, 2, 3, 4, 5, 6, 7, 8} + LittleEndian.Uint64(b[:4]) + return false +} + +func testPutUint64SmallSliceLengthPanics() (panicked bool) { + defer func() { + panicked = recover() != nil + }() + b := [8]byte{} + LittleEndian.PutUint64(b[:4], 0x0102030405060708) + return false +} + +func TestEarlyBoundsChecks(t *testing.T) { + if testUint64SmallSliceLengthPanics() != true { + t.Errorf("binary.LittleEndian.Uint64 expected to panic for small slices, but didn't") + } + if testPutUint64SmallSliceLengthPanics() != true { + t.Errorf("binary.LittleEndian.PutUint64 expected to panic for small slices, but didn't") + } +} + type byteSliceReader struct { remain []byte } @@ -373,8 +430,8 @@ func BenchmarkReadStruct(b *testing.B) { Read(bsr, BigEndian, &t) } b.StopTimer() - if !reflect.DeepEqual(s, t) { - b.Fatal("no match") + if b.N > 0 && !reflect.DeepEqual(s, t) { + b.Fatalf("struct doesn't match:\ngot %v;\nwant %v", t, s) } } @@ -395,18 +452,17 @@ func BenchmarkReadInts(b *testing.B) { Read(r, BigEndian, &ls.Uint32) Read(r, BigEndian, &ls.Uint64) } - + b.StopTimer() want := s want.Float32 = 0 want.Float64 = 0 want.Complex64 = 0 want.Complex128 = 0 - for i := range want.Array { - want.Array[i] = 0 - } - b.StopTimer() - if !reflect.DeepEqual(ls, want) { - panic("no match") + want.Array = [4]uint8{0, 0, 0, 0} + want.Bool = false + want.BoolArray = [4]bool{false, false, false, false} + if b.N > 0 && !reflect.DeepEqual(ls, want) { + b.Fatalf("struct doesn't match:\ngot %v;\nwant %v", ls, want) } } @@ -427,7 +483,7 @@ func BenchmarkWriteInts(b *testing.B) { Write(w, BigEndian, s.Uint64) } b.StopTimer() - if !bytes.Equal(buf.Bytes(), big[:30]) { + if b.N > 0 && !bytes.Equal(buf.Bytes(), big[:30]) { b.Fatalf("first half doesn't match: %x %x", buf.Bytes(), big[:30]) } } diff --git a/libgo/go/encoding/binary/varint.go b/libgo/go/encoding/binary/varint.go index 3a2dfa3c74..d7a75f99b1 100644 --- a/libgo/go/encoding/binary/varint.go +++ b/libgo/go/encoding/binary/varint.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/encoding/csv/reader.go b/libgo/go/encoding/csv/reader.go index a6bb780bf2..c8c4ca7758 100644 --- a/libgo/go/encoding/csv/reader.go +++ b/libgo/go/encoding/csv/reader.go @@ -3,6 +3,8 @@ // license that can be found in the LICENSE file. // Package csv reads and writes comma-separated values (CSV) files. +// There are many kinds of CSV files; this package supports the format +// described in RFC 4180. // // A csv file contains zero or more records of one or more fields per record. // Each record is separated by the newline character. The final record may @@ -14,11 +16,11 @@ // // Carriage returns before newline characters are silently removed. // -// Blank lines are ignored. A line with only whitespace characters (excluding +// Blank lines are ignored. A line with only whitespace characters (excluding // the ending newline character) is not considered a blank line. // // Fields which start and stop with the quote character " are called -// quoted-fields. The beginning and ending quote are not part of the +// quoted-fields. The beginning and ending quote are not part of the // field. // // The source: @@ -84,32 +86,42 @@ var ( // The exported fields can be changed to customize the details before the // first call to Read or ReadAll. // -// Comma is the field delimiter. It defaults to ','. // -// Comment, if not 0, is the comment character. Lines beginning with the -// Comment character are ignored. -// -// If FieldsPerRecord is positive, Read requires each record to -// have the given number of fields. If FieldsPerRecord is 0, Read sets it to -// the number of fields in the first record, so that future records must -// have the same field count. If FieldsPerRecord is negative, no check is -// made and records may have a variable number of fields. -// -// If LazyQuotes is true, a quote may appear in an unquoted field and a -// non-doubled quote may appear in a quoted field. -// -// If TrimLeadingSpace is true, leading white space in a field is ignored. type Reader struct { - Comma rune // field delimiter (set to ',' by NewReader) - Comment rune // comment character for start of line - FieldsPerRecord int // number of expected fields per record - LazyQuotes bool // allow lazy quotes - TrailingComma bool // ignored; here for backwards compatibility - TrimLeadingSpace bool // trim leading space - line int - column int - r *bufio.Reader - field bytes.Buffer + // Comma is the field delimiter. + // It is set to comma (',') by NewReader. + Comma rune + // Comment, if not 0, is the comment character. Lines beginning with the + // Comment character without preceding whitespace are ignored. + // With leading whitespace the Comment character becomes part of the + // field, even if TrimLeadingSpace is true. + Comment rune + // FieldsPerRecord is the number of expected fields per record. + // If FieldsPerRecord is positive, Read requires each record to + // have the given number of fields. If FieldsPerRecord is 0, Read sets it to + // the number of fields in the first record, so that future records must + // have the same field count. If FieldsPerRecord is negative, no check is + // made and records may have a variable number of fields. + FieldsPerRecord int + // If LazyQuotes is true, a quote may appear in an unquoted field and a + // non-doubled quote may appear in a quoted field. + LazyQuotes bool + TrailingComma bool // ignored; here for backwards compatibility + // If TrimLeadingSpace is true, leading white space in a field is ignored. + // This is done even if the field delimiter, Comma, is white space. + TrimLeadingSpace bool + + line int + column int + r *bufio.Reader + // lineBuffer holds the unescaped fields read by readField, one after another. + // The fields can be accessed by using the indexes in fieldIndexes. + // Example: for the row `a,"b","c""d",e` lineBuffer will contain `abc"de` and + // fieldIndexes will contain the indexes 0, 1, 2, 5. + lineBuffer bytes.Buffer + // Indexes of fields inside lineBuffer + // The i'th field starts at offset fieldIndexes[i] in lineBuffer. + fieldIndexes []int } // NewReader returns a new Reader that reads from r. @@ -129,8 +141,12 @@ func (r *Reader) error(err error) error { } } -// Read reads one record from r. The record is a slice of strings with each -// string representing one field. +// Read reads one record (a slice of fields) from r. +// If the record has an unexpected number of fields, +// Read returns the record along with the error ErrFieldCount. +// Except for that case, Read always returns either a non-nil +// record or a non-nil error, but not both. +// If there is no data left to be read, Read returns nil, io.EOF. func (r *Reader) Read() (record []string, err error) { for { record, err = r.parseRecord() @@ -177,7 +193,7 @@ func (r *Reader) ReadAll() (records [][]string, err error) { func (r *Reader) readRune() (rune, error) { r1, _, err := r.r.ReadRune() - // Handle \r\n here. We make the simplifying assumption that + // Handle \r\n here. We make the simplifying assumption that // anytime \r is followed by \n that it can be folded to \n. // We will not detect files which contain both \r\n and bare \n. if r1 == '\r' { @@ -208,13 +224,13 @@ func (r *Reader) skip(delim rune) error { // parseRecord reads and parses a single csv record from r. func (r *Reader) parseRecord() (fields []string, err error) { - // Each record starts on a new line. We increment our line + // Each record starts on a new line. We increment our line // number (lines start at 1, not 0) and set column to -1 // so as we increment in readRune it points to the character we read. r.line++ r.column = -1 - // Peek at the first rune. If it is an error we are done. + // Peek at the first rune. If it is an error we are done. // If we support comments and it is the comment character // then skip to the end of line. @@ -228,31 +244,54 @@ func (r *Reader) parseRecord() (fields []string, err error) { } r.r.UnreadRune() + r.lineBuffer.Reset() + r.fieldIndexes = r.fieldIndexes[:0] + // At this point we have at least one field. for { + idx := r.lineBuffer.Len() + haveField, delim, err := r.parseField() if haveField { - // If FieldsPerRecord is greater then 0 we can assume the final - // length of fields to be equal to FieldsPerRecord. - if r.FieldsPerRecord > 0 && fields == nil { - fields = make([]string, 0, r.FieldsPerRecord) - } - fields = append(fields, r.field.String()) + r.fieldIndexes = append(r.fieldIndexes, idx) } + if delim == '\n' || err == io.EOF { - return fields, err - } else if err != nil { + if len(r.fieldIndexes) == 0 { + return nil, err + } + break + } + + if err != nil { return nil, err } } + + fieldCount := len(r.fieldIndexes) + // Using this approach (creating a single string and taking slices of it) + // means that a single reference to any of the fields will retain the whole + // string. The risk of a nontrivial space leak caused by this is considered + // minimal and a tradeoff for better performance through the combined + // allocations. + line := r.lineBuffer.String() + fields = make([]string, fieldCount) + + for i, idx := range r.fieldIndexes { + if i == fieldCount-1 { + fields[i] = line[idx:] + } else { + fields[i] = line[idx:r.fieldIndexes[i+1]] + } + } + + return fields, nil } -// parseField parses the next field in the record. The read field is -// located in r.field. Delim is the first character not part of the field +// parseField parses the next field in the record. The read field is +// appended to r.lineBuffer. Delim is the first character not part of the field // (r.Comma or '\n'). func (r *Reader) parseField() (haveField bool, delim rune, err error) { - r.field.Reset() - r1, err := r.readRune() for err == nil && r.TrimLeadingSpace && r1 != '\n' && unicode.IsSpace(r1) { r1, err = r.readRune() @@ -305,19 +344,19 @@ func (r *Reader) parseField() (haveField bool, delim rune, err error) { return false, 0, r.error(ErrQuote) } // accept the bare quote - r.field.WriteRune('"') + r.lineBuffer.WriteRune('"') } case '\n': r.line++ r.column = -1 } - r.field.WriteRune(r1) + r.lineBuffer.WriteRune(r1) } default: // unquoted field for { - r.field.WriteRune(r1) + r.lineBuffer.WriteRune(r1) r1, err = r.readRune() if err != nil || r1 == r.Comma { break diff --git a/libgo/go/encoding/csv/reader_test.go b/libgo/go/encoding/csv/reader_test.go index be1002d034..7b3aca4c5f 100644 --- a/libgo/go/encoding/csv/reader_test.go +++ b/libgo/go/encoding/csv/reader_test.go @@ -5,6 +5,7 @@ package csv import ( + "io" "reflect" "strings" "testing" @@ -292,8 +293,52 @@ func TestRead(t *testing.T) { } } -func BenchmarkRead(b *testing.B) { - data := `x,y,z,w +// nTimes is an io.Reader which yields the string s n times. +type nTimes struct { + s string + n int + off int +} + +func (r *nTimes) Read(p []byte) (n int, err error) { + for { + if r.n <= 0 || r.s == "" { + return n, io.EOF + } + n0 := copy(p, r.s[r.off:]) + p = p[n0:] + n += n0 + r.off += n0 + if r.off == len(r.s) { + r.off = 0 + r.n-- + } + if len(p) == 0 { + return + } + } +} + +// benchmarkRead measures reading the provided CSV rows data. +// initReader, if non-nil, modifies the Reader before it's used. +func benchmarkRead(b *testing.B, initReader func(*Reader), rows string) { + b.ReportAllocs() + r := NewReader(&nTimes{s: rows, n: b.N}) + if initReader != nil { + initReader(r) + } + for { + _, err := r.Read() + if err == io.EOF { + break + } + if err != nil { + b.Fatal(err) + } + } +} + +const benchmarkCSVData = `x,y,z,w x,y,z, x,y,, x,,, @@ -305,11 +350,22 @@ x,,, "","","","" ` - for i := 0; i < b.N; i++ { - _, err := NewReader(strings.NewReader(data)).ReadAll() +func BenchmarkRead(b *testing.B) { + benchmarkRead(b, nil, benchmarkCSVData) +} - if err != nil { - b.Fatalf("could not read data: %s", err) - } - } +func BenchmarkReadWithFieldsPerRecord(b *testing.B) { + benchmarkRead(b, func(r *Reader) { r.FieldsPerRecord = 4 }, benchmarkCSVData) +} + +func BenchmarkReadWithoutFieldsPerRecord(b *testing.B) { + benchmarkRead(b, func(r *Reader) { r.FieldsPerRecord = -1 }, benchmarkCSVData) +} + +func BenchmarkReadLargeFields(b *testing.B) { + benchmarkRead(b, nil, strings.Repeat(`xxxxxxxxxxxxxxxx,yyyyyyyyyyyyyyyy,zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz,wwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww,vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv +xxxxxxxxxxxxxxxxxxxxxxxx,yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy,zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz,wwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww,vvvv +,,zzzz,wwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww,vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx,yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy,zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz,wwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwwww,vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv +`, 3)) } diff --git a/libgo/go/encoding/csv/writer.go b/libgo/go/encoding/csv/writer.go index 353d91f238..84b7aa1ed1 100644 --- a/libgo/go/encoding/csv/writer.go +++ b/libgo/go/encoding/csv/writer.go @@ -15,7 +15,7 @@ import ( // A Writer writes records to a CSV encoded file. // // As returned by NewWriter, a Writer writes records terminated by a -// newline and uses ',' as the field delimiter. The exported fields can be +// newline and uses ',' as the field delimiter. The exported fields can be // changed to customize the details before the first call to Write or WriteAll. // // Comma is the field delimiter. @@ -37,27 +37,28 @@ func NewWriter(w io.Writer) *Writer { // Writer writes a single CSV record to w along with any necessary quoting. // A record is a slice of strings with each string being one field. -func (w *Writer) Write(record []string) (err error) { +func (w *Writer) Write(record []string) error { for n, field := range record { if n > 0 { - if _, err = w.w.WriteRune(w.Comma); err != nil { - return + if _, err := w.w.WriteRune(w.Comma); err != nil { + return err } } // If we don't have to have a quoted field then just // write out the field and continue to the next field. if !w.fieldNeedsQuotes(field) { - if _, err = w.w.WriteString(field); err != nil { - return + if _, err := w.w.WriteString(field); err != nil { + return err } continue } - if err = w.w.WriteByte('"'); err != nil { - return + if err := w.w.WriteByte('"'); err != nil { + return err } for _, r1 := range field { + var err error switch r1 { case '"': _, err = w.w.WriteString(`""`) @@ -75,20 +76,21 @@ func (w *Writer) Write(record []string) (err error) { _, err = w.w.WriteRune(r1) } if err != nil { - return + return err } } - if err = w.w.WriteByte('"'); err != nil { - return + if err := w.w.WriteByte('"'); err != nil { + return err } } + var err error if w.UseCRLF { _, err = w.w.WriteString("\r\n") } else { err = w.w.WriteByte('\n') } - return + return err } // Flush writes any buffered data to the underlying io.Writer. @@ -104,9 +106,9 @@ func (w *Writer) Error() error { } // WriteAll writes multiple CSV records to w using Write and then calls Flush. -func (w *Writer) WriteAll(records [][]string) (err error) { +func (w *Writer) WriteAll(records [][]string) error { for _, record := range records { - err = w.Write(record) + err := w.Write(record) if err != nil { return err } @@ -130,7 +132,7 @@ func (w *Writer) fieldNeedsQuotes(field string) bool { if field == "" { return false } - if field == `\.` || strings.IndexRune(field, w.Comma) >= 0 || strings.IndexAny(field, "\"\r\n") >= 0 { + if field == `\.` || strings.ContainsRune(field, w.Comma) || strings.ContainsAny(field, "\"\r\n") { return true } diff --git a/libgo/go/encoding/encoding.go b/libgo/go/encoding/encoding.go index 6d218071b7..cc5a536996 100644 --- a/libgo/go/encoding/encoding.go +++ b/libgo/go/encoding/encoding.go @@ -1,4 +1,4 @@ -// Copyright 2013 The Go Authors. All rights reserved. +// Copyright 2013 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/encoding/gob/codec_test.go b/libgo/go/encoding/gob/codec_test.go index 8efcdc78ff..d4002cbcca 100644 --- a/libgo/go/encoding/gob/codec_test.go +++ b/libgo/go/encoding/gob/codec_test.go @@ -970,7 +970,7 @@ func TestBadRecursiveType(t *testing.T) { err := NewEncoder(b).Encode(&rec) if err == nil { t.Error("expected error; got none") - } else if strings.Index(err.Error(), "recursive") < 0 { + } else if !strings.Contains(err.Error(), "recursive") { t.Error("expected recursive type error; got", err) } // Can't test decode easily because we can't encode one, so we can't pass one to a Decoder. @@ -1253,7 +1253,7 @@ func TestIgnoreInterface(t *testing.T) { if item2.I != item1.I { t.Error("normal int did not decode correctly") } - if item2.F != item2.F { + if item2.F != item1.F { t.Error("normal float did not decode correctly") } } @@ -1280,7 +1280,7 @@ func TestUnexportedFields(t *testing.T) { if err != nil { t.Fatal("decode error:", err) } - if u0.A != u0.A || u0.B != u1.B || u0.D != u1.D { + if u0.A != u1.A || u0.B != u1.B || u0.D != u1.D { t.Errorf("u1->u0: expected %v; got %v", u0, u1) } if u1.c != 1234. { diff --git a/libgo/go/encoding/gob/debug.go b/libgo/go/encoding/gob/debug.go index 536bbdb5ac..d69d36f516 100644 --- a/libgo/go/encoding/gob/debug.go +++ b/libgo/go/encoding/gob/debug.go @@ -7,7 +7,7 @@ package gob -// This file is not normally included in the gob package. Used only for debugging the package itself. +// This file is not normally included in the gob package. Used only for debugging the package itself. // Except for reading uints, it is an implementation of a reader that is independent of // the one implemented by Decoder. // To enable the Debug function, delete the +build ignore line above and do @@ -241,7 +241,7 @@ func (deb *debugger) delimitedMessage(indent tab) bool { // loadBlock preps us to read a message // of the length specified next in the input. It returns // the length of the block. The argument tells whether -// an EOF is acceptable now. If it is and one is found, +// an EOF is acceptable now. If it is and one is found, // the return value is negative. func (deb *debugger) loadBlock(eofOK bool) int { n64, w, err := decodeUintReader(deb.r, deb.tmp) // deb.uint64 will error at EOF @@ -339,7 +339,7 @@ func (deb *debugger) string() string { return string(b) } -// delta returns the field delta at the input point. The expect argument, +// delta returns the field delta at the input point. The expect argument, // if non-negative, identifies what the value should be. func (deb *debugger) delta(expect int) int { delta := int(deb.uint64()) diff --git a/libgo/go/encoding/gob/decode.go b/libgo/go/encoding/gob/decode.go index 3b0dca86f3..9645dc5790 100644 --- a/libgo/go/encoding/gob/decode.go +++ b/libgo/go/encoding/gob/decode.go @@ -216,10 +216,10 @@ func ignoreTwoUints(i *decInstr, state *decoderState, v reflect.Value) { } // Since the encoder writes no zeros, if we arrive at a decoder we have -// a value to extract and store. The field number has already been read +// a value to extract and store. The field number has already been read // (it's how we knew to call this decoder). // Each decoder is responsible for handling any indirections associated -// with the data structure. If any pointer so reached is nil, allocation must +// with the data structure. If any pointer so reached is nil, allocation must // be done. // decAlloc takes a value and returns a settable value that can @@ -308,9 +308,9 @@ func decUint64(i *decInstr, state *decoderState, value reflect.Value) { } // Floating-point numbers are transmitted as uint64s holding the bits -// of the underlying representation. They are sent byte-reversed, with +// of the underlying representation. They are sent byte-reversed, with // the exponent end coming out first, so integer floating point numbers -// (for example) transmit more compactly. This routine does the +// (for example) transmit more compactly. This routine does the // unswizzling. func float64FromBits(u uint64) float64 { var v uint64 @@ -332,7 +332,7 @@ func float32FromBits(u uint64, ovfl error) float64 { if av < 0 { av = -av } - // +Inf is OK in both 32- and 64-bit floats. Underflow is always OK. + // +Inf is OK in both 32- and 64-bit floats. Underflow is always OK. if math.MaxFloat32 < av && av <= math.MaxFloat64 { error_(ovfl) } @@ -421,7 +421,7 @@ func ignoreUint8Array(i *decInstr, state *decoderState, value reflect.Value) { // Execution engine // The encoder engine is an array of instructions indexed by field number of the incoming -// decoder. It is executed with random access according to field number. +// decoder. It is executed with random access according to field number. type decEngine struct { instr []decInstr numInstr int // the number of active instructions @@ -442,7 +442,7 @@ func (dec *Decoder) decodeSingle(engine *decEngine, ut *userTypeInfo, value refl } // decodeStruct decodes a top-level struct and stores it in value. -// Indir is for the value, not the type. At the time of the call it may +// Indir is for the value, not the type. At the time of the call it may // differ from ut.indir, which was computed when the engine was built. // This state cannot arise for decodeSingle, which is called directly // from the user's value, not from the innards of an engine. @@ -536,7 +536,7 @@ func (dec *Decoder) decodeArrayHelper(state *decoderState, value reflect.Value, } // decodeArray decodes an array and stores it in value. -// The length is an unsigned integer preceding the elements. Even though the length is redundant +// The length is an unsigned integer preceding the elements. Even though the length is redundant // (it's part of the type), it's a useful check and is included in the encoding. func (dec *Decoder) decodeArray(atyp reflect.Type, state *decoderState, value reflect.Value, elemOp decOp, length int, ovfl error, helper decHelper) { if n := state.decodeUint(); n != uint64(length) { @@ -645,10 +645,10 @@ func (dec *Decoder) decodeInterface(ityp reflect.Type, state *decoderState, valu errorf("invalid type name length %d: exceeds input size", nr) } n := int(nr) - name := string(state.b.Bytes()[:n]) + name := state.b.Bytes()[:n] state.b.Drop(n) // Allocate the destination interface value. - if name == "" { + if len(name) == 0 { // Copy the nil interface value to the target. value.Set(reflect.Zero(value.Type())) return @@ -658,7 +658,7 @@ func (dec *Decoder) decodeInterface(ityp reflect.Type, state *decoderState, valu } // The concrete type must be registered. registerLock.RLock() - typ, ok := nameToConcreteType[name] + typ, ok := nameToConcreteType[string(name)] registerLock.RUnlock() if !ok { errorf("name not registered for interface: %q", name) @@ -1075,7 +1075,7 @@ func (dec *Decoder) compileIgnoreSingle(remoteId typeId) (engine *decEngine, err return } -// compileDec compiles the decoder engine for a value. If the value is not a struct, +// compileDec compiles the decoder engine for a value. If the value is not a struct, // it calls out to compileSingle. func (dec *Decoder) compileDec(remoteId typeId, ut *userTypeInfo) (engine *decEngine, err error) { defer catchError(&err) diff --git a/libgo/go/encoding/gob/decoder.go b/libgo/go/encoding/gob/decoder.go index c453e9ba39..c182941773 100644 --- a/libgo/go/encoding/gob/decoder.go +++ b/libgo/go/encoding/gob/decoder.go @@ -130,9 +130,9 @@ func (dec *Decoder) nextUint() uint64 { // decodeTypeSequence parses: // TypeSequence // (TypeDefinition DelimitedTypeDefinition*)? -// and returns the type id of the next value. It returns -1 at +// and returns the type id of the next value. It returns -1 at // EOF. Upon return, the remainder of dec.buf is the value to be -// decoded. If this is an interface value, it can be ignored by +// decoded. If this is an interface value, it can be ignored by // resetting that buffer. func (dec *Decoder) decodeTypeSequence(isInterface bool) typeId { for dec.err == nil { @@ -150,7 +150,7 @@ func (dec *Decoder) decodeTypeSequence(isInterface bool) typeId { // Type definition for (-id) follows. dec.recvType(-id) // When decoding an interface, after a type there may be a - // DelimitedValue still in the buffer. Skip its count. + // DelimitedValue still in the buffer. Skip its count. // (Alternatively, the buffer is empty and the byte count // will be absorbed by recvMessage.) if dec.buf.Len() > 0 { @@ -177,7 +177,7 @@ func (dec *Decoder) Decode(e interface{}) error { } value := reflect.ValueOf(e) // If e represents a value as opposed to a pointer, the answer won't - // get back to the caller. Make sure it's a pointer. + // get back to the caller. Make sure it's a pointer. if value.Type().Kind() != reflect.Ptr { dec.err = errors.New("gob: attempt to decode into a non-pointer") return dec.err @@ -187,7 +187,7 @@ func (dec *Decoder) Decode(e interface{}) error { // DecodeValue reads the next value from the input stream. // If v is the zero reflect.Value (v.Kind() == Invalid), DecodeValue discards the value. -// Otherwise, it stores the value into v. In that case, v must represent +// Otherwise, it stores the value into v. In that case, v must represent // a non-nil pointer to data or be an assignable reflect.Value (v.CanSet()) // If the input is at EOF, DecodeValue returns io.EOF and // does not modify v. diff --git a/libgo/go/encoding/gob/doc.go b/libgo/go/encoding/gob/doc.go index cf878f4502..1536574fed 100644 --- a/libgo/go/encoding/gob/doc.go +++ b/libgo/go/encoding/gob/doc.go @@ -17,7 +17,8 @@ Basics A stream of gobs is self-describing. Each data item in the stream is preceded by a specification of its type, expressed in terms of a small set of predefined types. Pointers are not transmitted, but the things they point to are -transmitted; that is, the values are flattened. Recursive types work fine, but +transmitted; that is, the values are flattened. Nil pointers are not permitted, +as they have no value. Recursive types work fine, but recursive values (data with cycles) are problematic. This may change. To use gobs, create an Encoder and present it with a series of data items as @@ -254,6 +255,12 @@ In summary, a gob stream looks like where * signifies zero or more repetitions and the type id of a value must be predefined or be defined before the value in the stream. +Compatibility: Any future changes to the package will endeavor to maintain +compatibility with streams encoded using previous versions. That is, any released +version of this package should be able to decode data written with any previously +released version, subject to issues such as security fixes. See the Go compatibility +document for background: https://golang.org/doc/go1compat + See "Gobs of data" for a design discussion of the gob wire format: https://blog.golang.org/gobs-of-data */ diff --git a/libgo/go/encoding/gob/encode.go b/libgo/go/encoding/gob/encode.go index 96052ef33b..50cd6adb46 100644 --- a/libgo/go/encoding/gob/encode.go +++ b/libgo/go/encoding/gob/encode.go @@ -96,7 +96,7 @@ func (enc *Encoder) freeEncoderState(e *encoderState) { enc.freeList = e } -// Unsigned integers have a two-state encoding. If the number is less +// Unsigned integers have a two-state encoding. If the number is less // than 128 (0 through 0x7F), its value is written directly. // Otherwise the value is written in big-endian byte order preceded // by the byte length, negated. @@ -127,7 +127,7 @@ func (state *encoderState) encodeInt(i int64) { } else { x = uint64(i << 1) } - state.encodeUint(uint64(x)) + state.encodeUint(x) } // encOp is the signature of an encoding operator for a given type. @@ -152,8 +152,8 @@ func (state *encoderState) update(instr *encInstr) { // Each encoder for a composite is responsible for handling any // indirections associated with the elements of the data structure. -// If any pointer so reached is nil, no bytes are written. If the -// data item is zero, no bytes are written. Single values - ints, +// If any pointer so reached is nil, no bytes are written. If the +// data item is zero, no bytes are written. Single values - ints, // strings etc. - are indirected before calling their encoders. // Otherwise, the output (for a scalar) is the field number, as an // encoded integer, followed by the field data in its appropriate @@ -203,9 +203,9 @@ func encUint(i *encInstr, state *encoderState, v reflect.Value) { // floatBits returns a uint64 holding the bits of a floating-point number. // Floating-point numbers are transmitted as uint64s holding the bits -// of the underlying representation. They are sent byte-reversed, with +// of the underlying representation. They are sent byte-reversed, with // the exponent end coming out first, so integer floating point numbers -// (for example) transmit more compactly. This routine does the +// (for example) transmit more compactly. This routine does the // swizzling. func floatBits(f float64) uint64 { u := math.Float64bits(f) @@ -272,7 +272,7 @@ func encStructTerminator(i *encInstr, state *encoderState, v reflect.Value) { // Execution engine // encEngine an array of instructions indexed by field number of the encoding -// data, typically a struct. It is executed top to bottom, walking the struct. +// data, typically a struct. It is executed top to bottom, walking the struct. type encEngine struct { instr []encInstr } @@ -297,7 +297,7 @@ func (enc *Encoder) encodeSingle(b *encBuffer, engine *encEngine, value reflect. defer enc.freeEncoderState(state) state.fieldnum = singletonField // There is no surrounding struct to frame the transmission, so we must - // generate data even if the item is zero. To do this, set sendZero. + // generate data even if the item is zero. To do this, set sendZero. state.sendZero = true instr := &engine.instr[singletonField] if instr.indir > 0 { @@ -386,7 +386,7 @@ func (enc *Encoder) encodeMap(b *encBuffer, mv reflect.Value, keyOp, elemOp encO // encodeInterface encodes the interface value iv. // To send an interface, we send a string identifying the concrete type, followed // by the type identifier (which might require defining that type right now), followed -// by the concrete value. A nil value gets sent as the empty string for the name, +// by the concrete value. A nil value gets sent as the empty string for the name, // followed by no value. func (enc *Encoder) encodeInterface(b *encBuffer, iv reflect.Value) { // Gobs can encode nil interface values but not typed interface @@ -417,7 +417,7 @@ func (enc *Encoder) encodeInterface(b *encBuffer, iv reflect.Value) { enc.sendTypeDescriptor(enc.writer(), state, ut) // Send the type id. enc.sendTypeId(state, ut) - // Encode the value into a new buffer. Any nested type definitions + // Encode the value into a new buffer. Any nested type definitions // should be written to b, before the encoded value. enc.pushWriter(b) data := encBufferPool.Get().(*encBuffer) diff --git a/libgo/go/encoding/gob/encoder.go b/libgo/go/encoding/gob/encoder.go index 62d0f42e81..40ec81b6e6 100644 --- a/libgo/go/encoding/gob/encoder.go +++ b/libgo/go/encoding/gob/encoder.go @@ -170,6 +170,7 @@ func (enc *Encoder) sendType(w io.Writer, state *encoderState, origt reflect.Typ // Encode transmits the data item represented by the empty interface value, // guaranteeing that all necessary type information has been transmitted first. +// Passing a nil pointer to Encoder will panic, as they cannot be transmitted by gob. func (enc *Encoder) Encode(e interface{}) error { return enc.EncodeValue(reflect.ValueOf(e)) } @@ -191,7 +192,7 @@ func (enc *Encoder) sendTypeDescriptor(w io.Writer, state *encoderState, ut *use return } // If the type info has still not been transmitted, it means we have - // a singleton basic type (int, []byte etc.) at top level. We don't + // a singleton basic type (int, []byte etc.) at top level. We don't // need to send the type info but we do need to update enc.sent. if !sent { info, err := getTypeInfo(ut) @@ -212,9 +213,11 @@ func (enc *Encoder) sendTypeId(state *encoderState, ut *userTypeInfo) { // EncodeValue transmits the data item represented by the reflection value, // guaranteeing that all necessary type information has been transmitted first. +// Passing a nil pointer to EncodeValue will panic, as they cannot be transmitted by gob. func (enc *Encoder) EncodeValue(value reflect.Value) error { - // Gobs contain values. They cannot represent nil pointers, which - // have no value to encode. + if value.Kind() == reflect.Invalid { + return errors.New("gob: cannot encode nil value") + } if value.Kind() == reflect.Ptr && value.IsNil() { panic("gob: cannot encode nil pointer of type " + value.Type().String()) } diff --git a/libgo/go/encoding/gob/encoder_test.go b/libgo/go/encoding/gob/encoder_test.go index 570d79696b..9256848b50 100644 --- a/libgo/go/encoding/gob/encoder_test.go +++ b/libgo/go/encoding/gob/encoder_test.go @@ -8,6 +8,7 @@ import ( "bytes" "encoding/hex" "fmt" + "io/ioutil" "reflect" "strings" "testing" @@ -280,7 +281,7 @@ func TestValueError(t *testing.T) { } t4p := &Type4{3} var t4 Type4 // note: not a pointer. - if err := encAndDec(t4p, t4); err == nil || strings.Index(err.Error(), "pointer") < 0 { + if err := encAndDec(t4p, t4); err == nil || !strings.Contains(err.Error(), "pointer") { t.Error("expected error about pointer; got", err) } } @@ -388,7 +389,7 @@ func TestSingletons(t *testing.T) { t.Errorf("expected error decoding %v: %s", test.in, test.err) continue case err != nil && test.err != "": - if strings.Index(err.Error(), test.err) < 0 { + if !strings.Contains(err.Error(), test.err) { t.Errorf("wrong error decoding %v: wanted %s, got %v", test.in, test.err, err) } continue @@ -414,7 +415,7 @@ func TestStructNonStruct(t *testing.T) { var ns NonStruct if err := encAndDec(s, &ns); err == nil { t.Error("should get error for struct/non-struct") - } else if strings.Index(err.Error(), "type") < 0 { + } else if !strings.Contains(err.Error(), "type") { t.Error("for struct/non-struct expected type error; got", err) } // Now try the other way @@ -424,7 +425,7 @@ func TestStructNonStruct(t *testing.T) { } if err := encAndDec(ns, &s); err == nil { t.Error("should get error for non-struct/struct") - } else if strings.Index(err.Error(), "type") < 0 { + } else if !strings.Contains(err.Error(), "type") { t.Error("for non-struct/struct expected type error; got", err) } } @@ -439,8 +440,8 @@ func (this *interfaceIndirectTestT) F() bool { return true } -// A version of a bug reported on golang-nuts. Also tests top-level -// slice of interfaces. The issue was registering *T caused T to be +// A version of a bug reported on golang-nuts. Also tests top-level +// slice of interfaces. The issue was registering *T caused T to be // stored as the concrete type. func TestInterfaceIndirect(t *testing.T) { Register(&interfaceIndirectTestT{}) @@ -463,7 +464,7 @@ func TestInterfaceIndirect(t *testing.T) { // Also, when the ignored object contains an interface value, it may define // types. Make sure that skipping the value still defines the types by using -// the encoder/decoder pair to send a value afterwards. If an interface +// the encoder/decoder pair to send a value afterwards. If an interface // is sent, its type in the test is always NewType0, so this checks that the // encoder and decoder don't skew with respect to type definitions. @@ -603,10 +604,6 @@ type Bug1Elem struct { type Bug1StructMap map[string]Bug1Elem -func bug1EncDec(in Bug1StructMap, out *Bug1StructMap) error { - return nil -} - func TestMapBug1(t *testing.T) { in := make(Bug1StructMap) in["val1"] = Bug1Elem{"elem1", 1} @@ -833,32 +830,97 @@ func TestPtrToMapOfMap(t *testing.T) { } } +// Test that untyped nils generate an error, not a panic. +// See Issue 16204. +func TestCatchInvalidNilValue(t *testing.T) { + encodeErr, panicErr := encodeAndRecover(nil) + if panicErr != nil { + t.Fatalf("panicErr=%v, should not panic encoding untyped nil", panicErr) + } + if encodeErr == nil { + t.Errorf("got err=nil, want non-nil error when encoding untyped nil value") + } else if !strings.Contains(encodeErr.Error(), "nil value") { + t.Errorf("expected 'nil value' error; got err=%v", encodeErr) + } +} + // A top-level nil pointer generates a panic with a helpful string-valued message. func TestTopLevelNilPointer(t *testing.T) { - errMsg := topLevelNilPanic(t) - if errMsg == "" { + var ip *int + encodeErr, panicErr := encodeAndRecover(ip) + if encodeErr != nil { + t.Fatal("error in encode:", encodeErr) + } + if panicErr == nil { t.Fatal("top-level nil pointer did not panic") } + errMsg := panicErr.Error() if !strings.Contains(errMsg, "nil pointer") { t.Fatal("expected nil pointer error, got:", errMsg) } } -func topLevelNilPanic(t *testing.T) (panicErr string) { +func encodeAndRecover(value interface{}) (encodeErr, panicErr error) { defer func() { e := recover() - if err, ok := e.(string); ok { - panicErr = err + if e != nil { + switch err := e.(type) { + case error: + panicErr = err + default: + panicErr = fmt.Errorf("%v", err) + } } }() - var ip *int - buf := new(bytes.Buffer) - if err := NewEncoder(buf).Encode(ip); err != nil { - t.Fatal("error in encode:", err) - } + + encodeErr = NewEncoder(ioutil.Discard).Encode(value) return } +func TestNilPointerPanics(t *testing.T) { + var ( + nilStringPtr *string + intMap = make(map[int]int) + intMapPtr = &intMap + nilIntMapPtr *map[int]int + zero int + nilBoolChannel chan bool + nilBoolChannelPtr *chan bool + nilStringSlice []string + stringSlice = make([]string, 1) + nilStringSlicePtr *[]string + ) + + testCases := []struct { + value interface{} + mustPanic bool + }{ + {nilStringPtr, true}, + {intMap, false}, + {intMapPtr, false}, + {nilIntMapPtr, true}, + {zero, false}, + {nilStringSlice, false}, + {stringSlice, false}, + {nilStringSlicePtr, true}, + {nilBoolChannel, false}, + {nilBoolChannelPtr, true}, + } + + for _, tt := range testCases { + _, panicErr := encodeAndRecover(tt.value) + if tt.mustPanic { + if panicErr == nil { + t.Errorf("expected panic with input %#v, did not panic", tt.value) + } + continue + } + if panicErr != nil { + t.Fatalf("expected no panic with input %#v, got panic=%v", tt.value, panicErr) + } + } +} + func TestNilPointerInsideInterface(t *testing.T) { var ip *int si := struct { @@ -913,7 +975,7 @@ func TestMutipleEncodingsOfBadType(t *testing.T) { // There was an error check comparing the length of the input with the // length of the slice being decoded. It was wrong because the next // thing in the input might be a type definition, which would lead to -// an incorrect length check. This test reproduces the corner case. +// an incorrect length check. This test reproduces the corner case. type Z struct { } diff --git a/libgo/go/encoding/gob/error.go b/libgo/go/encoding/gob/error.go index 92cc0c615e..8b5265c278 100644 --- a/libgo/go/encoding/gob/error.go +++ b/libgo/go/encoding/gob/error.go @@ -9,7 +9,7 @@ import "fmt" // Errors in decoding and encoding are handled using panic and recover. // Panics caused by user error (that is, everything except run-time panics // such as "index out of bounds" errors) do not leave the file that caused -// them, but are instead turned into plain error returns. Encoding and +// them, but are instead turned into plain error returns. Encoding and // decoding functions and methods that do not return an error either use // panic to report an error or are guaranteed error-free. @@ -30,7 +30,7 @@ func error_(err error) { } // catchError is meant to be used as a deferred function to turn a panic(gobError) into a -// plain error. It overwrites the error return of the function that deferred its call. +// plain error. It overwrites the error return of the function that deferred its call. func catchError(err *error) { if e := recover(); e != nil { ge, ok := e.(gobError) diff --git a/libgo/go/encoding/gob/gobencdec_test.go b/libgo/go/encoding/gob/gobencdec_test.go index eb76b481d1..ecc91eef1f 100644 --- a/libgo/go/encoding/gob/gobencdec_test.go +++ b/libgo/go/encoding/gob/gobencdec_test.go @@ -376,7 +376,7 @@ func TestGobEncoderIndirectArrayField(t *testing.T) { } // As long as the fields have the same name and implement the -// interface, we can cross-connect them. Not sure it's useful +// interface, we can cross-connect them. Not sure it's useful // and may even be bad but it works and it's hard to prevent // without exposing the contents of the object, which would // defeat the purpose. @@ -434,7 +434,7 @@ func TestGobEncoderValueEncoder(t *testing.T) { } // Test that we can use a value then a pointer type of a GobEncoder -// in the same encoded value. Bug 4647. +// in the same encoded value. Bug 4647. func TestGobEncoderValueThenPointer(t *testing.T) { v := ValueGobber("forty-two") w := ValueGobber("six-by-nine") @@ -548,7 +548,7 @@ func TestGobEncoderFieldTypeError(t *testing.T) { if err == nil { t.Fatal("expected decode error for mismatched fields (encoder to non-decoder)") } - if strings.Index(err.Error(), "type") < 0 { + if !strings.Contains(err.Error(), "type") { t.Fatal("expected type error; got", err) } // Non-encoder to GobDecoder: error @@ -562,7 +562,7 @@ func TestGobEncoderFieldTypeError(t *testing.T) { if err == nil { t.Fatal("expected decode error for mismatched fields (non-encoder to decoder)") } - if strings.Index(err.Error(), "type") < 0 { + if !strings.Contains(err.Error(), "type") { t.Fatal("expected type error; got", err) } } diff --git a/libgo/go/encoding/gob/type.go b/libgo/go/encoding/gob/type.go index cf5cec0703..c27f7e9707 100644 --- a/libgo/go/encoding/gob/type.go +++ b/libgo/go/encoding/gob/type.go @@ -17,7 +17,7 @@ import ( ) // userTypeInfo stores the information associated with a type the user has handed -// to the package. It's computed once and stored in a map keyed by reflection +// to the package. It's computed once and stored in a map keyed by reflection // type. type userTypeInfo struct { user reflect.Type // the type the user handed us @@ -44,7 +44,7 @@ var ( ) // validType returns, and saves, the information associated with user-provided type rt. -// If the user type is not valid, err will be non-nil. To be used when the error handler +// If the user type is not valid, err will be non-nil. To be used when the error handler // is not set up. func validUserType(rt reflect.Type) (ut *userTypeInfo, err error) { userTypeLock.RLock() @@ -64,7 +64,7 @@ func validUserType(rt reflect.Type) (ut *userTypeInfo, err error) { ut.base = rt ut.user = rt // A type that is just a cycle of pointers (such as type T *T) cannot - // be represented in gobs, which need some concrete data. We use a + // be represented in gobs, which need some concrete data. We use a // cycle detection algorithm from Knuth, Vol 2, Section 3.1, Ex 6, // pp 539-540. As we step through indirections, run another type at // half speed. If they meet up, there's a cycle. @@ -493,7 +493,7 @@ func newTypeObject(name string, ut *userTypeInfo, rt reflect.Type) (gobType, err // For arrays, maps, and slices, we set the type id after the elements // are constructed. This is to retain the order of type id allocation after // a fix made to handle recursive types, which changed the order in - // which types are built. Delaying the setting in this way preserves + // which types are built. Delaying the setting in this way preserves // type ids while allowing recursive types to be described. Structs, // done below, were already handling recursion correctly so they // assign the top-level id before those of the field. @@ -597,7 +597,7 @@ func getBaseType(name string, rt reflect.Type) (gobType, error) { // getType returns the Gob type describing the given reflect.Type. // Should be called only when handling GobEncoders/Decoders, -// which may be pointers. All other types are handled through the +// which may be pointers. All other types are handled through the // base type, never a pointer. // typeLock must be held. func getType(name string, ut *userTypeInfo, rt reflect.Type) (gobType, error) { @@ -642,7 +642,7 @@ func bootstrapType(name string, e interface{}, expect typeId) typeId { // For bootstrapping purposes, we assume that the recipient knows how // to decode a wireType; it is exactly the wireType struct here, interpreted // using the gob rules for sending a structure, except that we assume the -// ids for wireType and structType etc. are known. The relevant pieces +// ids for wireType and structType etc. are known. The relevant pieces // are built in encode.go's init() function. // To maintain binary compatibility, if you extend this type, always put // the new fields last. @@ -789,7 +789,7 @@ func mustGetTypeInfo(rt reflect.Type) *typeInfo { // // Note: Since gobs can be stored permanently, it is good design // to guarantee the encoding used by a GobEncoder is stable as the -// software evolves. For instance, it might make sense for GobEncode +// software evolves. For instance, it might make sense for GobEncode // to include a version number in the encoding. type GobEncoder interface { // GobEncode returns a byte slice representing the encoding of the @@ -838,8 +838,8 @@ func RegisterName(name string, value interface{}) { } // Register records a type, identified by a value for that type, under its -// internal type name. That name will identify the concrete type of a value -// sent or received as an interface variable. Only types that will be +// internal type name. That name will identify the concrete type of a value +// sent or received as an interface variable. Only types that will be // transferred as implementations of interface values need to be registered. // Expecting to be used only during initialization, it panics if the mapping // between types and names is not a bijection. diff --git a/libgo/go/encoding/hex/example_test.go b/libgo/go/encoding/hex/example_test.go new file mode 100644 index 0000000000..fb1554eba7 --- /dev/null +++ b/libgo/go/encoding/hex/example_test.go @@ -0,0 +1,100 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package hex_test + +import ( + "encoding/hex" + "fmt" + "log" + "os" +) + +func ExampleEncode() { + src := []byte("Hello Gopher!") + + dst := make([]byte, hex.EncodedLen(len(src))) + hex.Encode(dst, src) + + fmt.Printf("%s\n", dst) + + // Output: + // 48656c6c6f20476f7068657221 +} + +func ExampleDecode() { + src := []byte("48656c6c6f20476f7068657221") + + dst := make([]byte, hex.DecodedLen(len(src))) + n, err := hex.Decode(dst, src) + if err != nil { + log.Fatal(err) + } + + fmt.Printf("%s\n", dst[:n]) + + // Output: + // Hello Gopher! +} + +func ExampleDecodeString() { + const s = "48656c6c6f20476f7068657221" + decoded, err := hex.DecodeString(s) + if err != nil { + log.Fatal(err) + } + + fmt.Printf("%s\n", decoded) + + // Output: + // Hello Gopher! +} + +func ExampleDump() { + content := []byte("Go is an open source programming language.") + + fmt.Printf("%s", hex.Dump(content)) + + // Output: + // 00000000 47 6f 20 69 73 20 61 6e 20 6f 70 65 6e 20 73 6f |Go is an open so| + // 00000010 75 72 63 65 20 70 72 6f 67 72 61 6d 6d 69 6e 67 |urce programming| + // 00000020 20 6c 61 6e 67 75 61 67 65 2e | language.| +} + +func ExampleDumper() { + lines := []string{ + "Go is an open source programming language.", + "\n", + "We encourage all Go users to subscribe to golang-announce.", + } + + stdoutDumper := hex.Dumper(os.Stdout) + + defer stdoutDumper.Close() + + for _, line := range lines { + stdoutDumper.Write([]byte(line)) + } + + // Output: + // 00000000 47 6f 20 69 73 20 61 6e 20 6f 70 65 6e 20 73 6f |Go is an open so| + // 00000010 75 72 63 65 20 70 72 6f 67 72 61 6d 6d 69 6e 67 |urce programming| + // 00000020 20 6c 61 6e 67 75 61 67 65 2e 0a 57 65 20 65 6e | language..We en| + // 00000030 63 6f 75 72 61 67 65 20 61 6c 6c 20 47 6f 20 75 |courage all Go u| + // 00000040 73 65 72 73 20 74 6f 20 73 75 62 73 63 72 69 62 |sers to subscrib| + // 00000050 65 20 74 6f 20 67 6f 6c 61 6e 67 2d 61 6e 6e 6f |e to golang-anno| + // 00000060 75 6e 63 65 2e |unce.| +} + +func ExampleEncodeToString() { + src := []byte("Hello") + encodedStr := hex.EncodeToString(src) + + fmt.Printf("%s\n", encodedStr) + + // Output: + // 48656c6c6f +} diff --git a/libgo/go/encoding/hex/hex.go b/libgo/go/encoding/hex/hex.go index d1fc7024a9..b43c1c4b45 100644 --- a/libgo/go/encoding/hex/hex.go +++ b/libgo/go/encoding/hex/hex.go @@ -12,13 +12,17 @@ import ( "io" ) -const hextable = "0123456789abcdef" +var hextable = [16]byte{ + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', + 'a', 'b', 'c', 'd', 'e', 'f', +} // EncodedLen returns the length of an encoding of n source bytes. +// Specifically, it returns n * 2. func EncodedLen(n int) int { return n * 2 } // Encode encodes src into EncodedLen(len(src)) -// bytes of dst. As a convenience, it returns the number +// bytes of dst. As a convenience, it returns the number // of bytes written to dst, but this value is always EncodedLen(len(src)). // Encode implements hexadecimal encoding. func Encode(dst, src []byte) int { @@ -40,12 +44,15 @@ func (e InvalidByteError) Error() string { return fmt.Sprintf("encoding/hex: invalid byte: %#U", rune(e)) } +// DecodedLen returns the length of a decoding of x source bytes. +// Specifically, it returns x / 2. func DecodedLen(x int) int { return x / 2 } -// Decode decodes src into DecodedLen(len(src)) bytes, returning the actual -// number of bytes written to dst. +// Decode decodes src into DecodedLen(len(src)) bytes, +// returning the actual number of bytes written to dst. // -// If Decode encounters invalid input, it returns an error describing the failure. +// Decode expects that src contain only hexadecimal +// characters and that src should have an even length. func Decode(dst, src []byte) (int, error) { if len(src)%2 == 1 { return 0, ErrLength @@ -105,7 +112,7 @@ func Dump(data []byte) string { dumper := Dumper(&buf) dumper.Write(data) dumper.Close() - return string(buf.Bytes()) + return buf.String() } // Dumper returns a WriteCloser that writes a hex dump of all written data to diff --git a/libgo/go/encoding/hex/hex_test.go b/libgo/go/encoding/hex/hex_test.go index b969636cd5..64dabbd10a 100644 --- a/libgo/go/encoding/hex/hex_test.go +++ b/libgo/go/encoding/hex/hex_test.go @@ -6,6 +6,7 @@ package hex import ( "bytes" + "fmt" "testing" ) @@ -151,3 +152,18 @@ var expectedHexDump = []byte(`00000000 1e 1f 20 21 22 23 24 25 26 27 28 29 2a 00000010 2e 2f 30 31 32 33 34 35 36 37 38 39 3a 3b 3c 3d |./0123456789:;<=| 00000020 3e 3f 40 41 42 43 44 45 |>?@ABCDE| `) + +var sink []byte + +func BenchmarkEncode(b *testing.B) { + for _, size := range []int{256, 1024, 4096, 16384} { + src := bytes.Repeat([]byte{2, 3, 5, 7, 9, 11, 13, 17}, size/8) + sink = make([]byte, 2*size) + + b.Run(fmt.Sprintf("%v", size), func(b *testing.B) { + for i := 0; i < b.N; i++ { + Encode(sink, src) + } + }) + } +} diff --git a/libgo/go/encoding/json/bench_test.go b/libgo/go/encoding/json/bench_test.go index ed89d1156e..cd7380b1ef 100644 --- a/libgo/go/encoding/json/bench_test.go +++ b/libgo/go/encoding/json/bench_test.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -158,7 +158,7 @@ func BenchmarkCodeUnmarshal(b *testing.B) { for i := 0; i < b.N; i++ { var r codeResponse if err := Unmarshal(codeJSON, &r); err != nil { - b.Fatal("Unmmarshal:", err) + b.Fatal("Unmarshal:", err) } } b.SetBytes(int64(len(codeJSON))) @@ -173,7 +173,7 @@ func BenchmarkCodeUnmarshalReuse(b *testing.B) { var r codeResponse for i := 0; i < b.N; i++ { if err := Unmarshal(codeJSON, &r); err != nil { - b.Fatal("Unmmarshal:", err) + b.Fatal("Unmarshal:", err) } } } diff --git a/libgo/go/encoding/json/decode.go b/libgo/go/encoding/json/decode.go index 539d952ad6..77fc4607db 100644 --- a/libgo/go/encoding/json/decode.go +++ b/libgo/go/encoding/json/decode.go @@ -29,11 +29,18 @@ import ( // with the following additional rules: // // To unmarshal JSON into a pointer, Unmarshal first handles the case of -// the JSON being the JSON literal null. In that case, Unmarshal sets -// the pointer to nil. Otherwise, Unmarshal unmarshals the JSON into -// the value pointed at by the pointer. If the pointer is nil, Unmarshal +// the JSON being the JSON literal null. In that case, Unmarshal sets +// the pointer to nil. Otherwise, Unmarshal unmarshals the JSON into +// the value pointed at by the pointer. If the pointer is nil, Unmarshal // allocates a new value for it to point to. // +// To unmarshal JSON into a value implementing the Unmarshaler interface, +// Unmarshal calls that value's UnmarshalJSON method, including +// when the input is a JSON null. +// Otherwise, if the value implements encoding.TextUnmarshaler +// and the input is a JSON quoted string, Unmarshal calls that value's +// UnmarshalText method with the unquoted form of the string. +// // To unmarshal JSON into a struct, Unmarshal matches incoming object // keys to the keys used by Marshal (either the struct field name or its tag), // preferring an exact match but also accepting a case-insensitive match. @@ -61,10 +68,11 @@ import ( // If the JSON array is smaller than the Go array, // the additional Go array elements are set to zero values. // -// To unmarshal a JSON object into a string-keyed map, Unmarshal first -// establishes a map to use, If the map is nil, Unmarshal allocates a new map. -// Otherwise Unmarshal reuses the existing map, keeping existing entries. -// Unmarshal then stores key-value pairs from the JSON object into the map. +// To unmarshal a JSON object into a map, Unmarshal first establishes a map to +// use. If the map is nil, Unmarshal allocates a new map. Otherwise Unmarshal +// reuses the existing map, keeping existing entries. Unmarshal then stores +// key-value pairs from the JSON object into the map. The map's key type must +// either be a string, an integer, or implement encoding.TextUnmarshaler. // // If a JSON value is not appropriate for a given target type, // or if a JSON number overflows the target type, Unmarshal @@ -96,11 +104,14 @@ func Unmarshal(data []byte, v interface{}) error { return d.unmarshal(v) } -// Unmarshaler is the interface implemented by objects +// Unmarshaler is the interface implemented by types // that can unmarshal a JSON description of themselves. // The input can be assumed to be a valid encoding of // a JSON value. UnmarshalJSON must copy the JSON data // if it wishes to retain the data after returning. +// +// By convention, to approximate the behavior of Unmarshal itself, +// Unmarshalers implement UnmarshalJSON([]byte("null")) as a no-op. type Unmarshaler interface { UnmarshalJSON([]byte) error } @@ -111,9 +122,14 @@ type UnmarshalTypeError struct { Value string // description of JSON value - "bool", "array", "number -5" Type reflect.Type // type of Go value it could not be assigned to Offset int64 // error occurred after reading Offset bytes + Struct string // name of the struct type containing the field + Field string // name of the field holding the Go value } func (e *UnmarshalTypeError) Error() string { + if e.Struct != "" || e.Field != "" { + return "json: cannot unmarshal " + e.Value + " into Go struct field " + e.Struct + "." + e.Field + " of type " + e.Type.String() + } return "json: cannot unmarshal " + e.Value + " into Go value of type " + e.Type.String() } @@ -247,10 +263,14 @@ func isValidNumber(s string) bool { // decodeState represents the state while decoding a JSON value. type decodeState struct { - data []byte - off int // read offset in data - scan scanner - nextscan scanner // for calls to nextValue + data []byte + off int // read offset in data + scan scanner + nextscan scanner // for calls to nextValue + errorContext struct { // provides context for type errors + Struct string + Field string + } savedError error useNumber bool } @@ -264,22 +284,37 @@ func (d *decodeState) init(data []byte) *decodeState { d.data = data d.off = 0 d.savedError = nil + d.errorContext.Struct = "" + d.errorContext.Field = "" return d } // error aborts the decoding by panicking with err. func (d *decodeState) error(err error) { - panic(err) + panic(d.addErrorContext(err)) } // saveError saves the first err it is called with, // for reporting at the end of the unmarshal. func (d *decodeState) saveError(err error) { if d.savedError == nil { - d.savedError = err + d.savedError = d.addErrorContext(err) } } +// addErrorContext returns a new error enhanced with information from d.errorContext +func (d *decodeState) addErrorContext(err error) error { + if d.errorContext.Struct != "" || d.errorContext.Field != "" { + switch err := err.(type) { + case *UnmarshalTypeError: + err.Struct = d.errorContext.Struct + err.Field = d.errorContext.Field + return err + } + } + return err +} + // next cuts off and returns the next full JSON value in d.data[d.off:]. // The next value is known to be an object or array, not a literal. func (d *decodeState) next() []byte { @@ -433,8 +468,10 @@ func (d *decodeState) indirect(v reflect.Value, decodingNull bool) (Unmarshaler, if u, ok := v.Interface().(Unmarshaler); ok { return u, nil, reflect.Value{} } - if u, ok := v.Interface().(encoding.TextUnmarshaler); ok { - return nil, u, reflect.Value{} + if !decodingNull { + if u, ok := v.Interface().(encoding.TextUnmarshaler); ok { + return nil, u, reflect.Value{} + } } } v = v.Elem() @@ -456,7 +493,7 @@ func (d *decodeState) array(v reflect.Value) { return } if ut != nil { - d.saveError(&UnmarshalTypeError{"array", v.Type(), int64(d.off)}) + d.saveError(&UnmarshalTypeError{Value: "array", Type: v.Type(), Offset: int64(d.off)}) d.off-- d.next() return @@ -475,7 +512,7 @@ func (d *decodeState) array(v reflect.Value) { // Otherwise it's invalid. fallthrough default: - d.saveError(&UnmarshalTypeError{"array", v.Type(), int64(d.off)}) + d.saveError(&UnmarshalTypeError{Value: "array", Type: v.Type(), Offset: int64(d.off)}) d.off-- d.next() return @@ -534,7 +571,7 @@ func (d *decodeState) array(v reflect.Value) { if i < v.Len() { if v.Kind() == reflect.Array { - // Array. Zero the rest. + // Array. Zero the rest. z := reflect.Zero(v.Type().Elem()) for ; i < v.Len(); i++ { v.Index(i).Set(z) @@ -549,6 +586,7 @@ func (d *decodeState) array(v reflect.Value) { } var nullLiteral = []byte("null") +var textUnmarshalerType = reflect.TypeOf(new(encoding.TextUnmarshaler)).Elem() // object consumes an object from d.data[d.off-1:], decoding into the value v. // the first byte ('{') of the object has been read already. @@ -564,7 +602,7 @@ func (d *decodeState) object(v reflect.Value) { return } if ut != nil { - d.saveError(&UnmarshalTypeError{"object", v.Type(), int64(d.off)}) + d.saveError(&UnmarshalTypeError{Value: "object", Type: v.Type(), Offset: int64(d.off)}) d.off-- d.next() // skip over { } in input return @@ -577,24 +615,34 @@ func (d *decodeState) object(v reflect.Value) { return } - // Check type of target: struct or map[string]T + // Check type of target: + // struct or + // map[T1]T2 where T1 is string, an integer type, + // or an encoding.TextUnmarshaler switch v.Kind() { case reflect.Map: - // map must have string kind + // Map key must either have string kind, have an integer kind, + // or be an encoding.TextUnmarshaler. t := v.Type() - if t.Key().Kind() != reflect.String { - d.saveError(&UnmarshalTypeError{"object", v.Type(), int64(d.off)}) - d.off-- - d.next() // skip over { } in input - return + switch t.Key().Kind() { + case reflect.String, + reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + default: + if !reflect.PtrTo(t.Key()).Implements(textUnmarshalerType) { + d.saveError(&UnmarshalTypeError{Value: "object", Type: v.Type(), Offset: int64(d.off)}) + d.off-- + d.next() // skip over { } in input + return + } } if v.IsNil() { v.Set(reflect.MakeMap(t)) } case reflect.Struct: - + // ok default: - d.saveError(&UnmarshalTypeError{"object", v.Type(), int64(d.off)}) + d.saveError(&UnmarshalTypeError{Value: "object", Type: v.Type(), Offset: int64(d.off)}) d.off-- d.next() // skip over { } in input return @@ -659,6 +707,8 @@ func (d *decodeState) object(v reflect.Value) { } subv = subv.Field(i) } + d.errorContext.Field = f.name + d.errorContext.Struct = v.Type().Name() } } @@ -670,7 +720,6 @@ func (d *decodeState) object(v reflect.Value) { d.error(errPhase) } - // Read value. if destring { switch qv := d.valueQuoted().(type) { case nil: @@ -687,7 +736,37 @@ func (d *decodeState) object(v reflect.Value) { // Write value back to map; // if using struct, subv points into struct already. if v.Kind() == reflect.Map { - kv := reflect.ValueOf(key).Convert(v.Type().Key()) + kt := v.Type().Key() + var kv reflect.Value + switch { + case kt.Kind() == reflect.String: + kv = reflect.ValueOf(key).Convert(kt) + case reflect.PtrTo(kt).Implements(textUnmarshalerType): + kv = reflect.New(v.Type().Key()) + d.literalStore(item, kv, true) + kv = kv.Elem() + default: + switch kt.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + s := string(key) + n, err := strconv.ParseInt(s, 10, 64) + if err != nil || reflect.Zero(kt).OverflowInt(n) { + d.saveError(&UnmarshalTypeError{Value: "number " + s, Type: kt, Offset: int64(start + 1)}) + return + } + kv = reflect.ValueOf(n).Convert(kt) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + s := string(key) + n, err := strconv.ParseUint(s, 10, 64) + if err != nil || reflect.Zero(kt).OverflowUint(n) { + d.saveError(&UnmarshalTypeError{Value: "number " + s, Type: kt, Offset: int64(start + 1)}) + return + } + kv = reflect.ValueOf(n).Convert(kt) + default: + panic("json: Unexpected key type") // should never occur + } + } v.SetMapIndex(kv, subv) } @@ -699,6 +778,9 @@ func (d *decodeState) object(v reflect.Value) { if op != scanObjectValue { d.error(errPhase) } + + d.errorContext.Struct = "" + d.errorContext.Field = "" } } @@ -725,7 +807,7 @@ func (d *decodeState) convertNumber(s string) (interface{}, error) { } f, err := strconv.ParseFloat(s, 64) if err != nil { - return nil, &UnmarshalTypeError{"number " + s, reflect.TypeOf(0.0), int64(d.off)} + return nil, &UnmarshalTypeError{Value: "number " + s, Type: reflect.TypeOf(0.0), Offset: int64(d.off)} } return f, nil } @@ -744,8 +826,8 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) return } - wantptr := item[0] == 'n' // null - u, ut, pv := d.indirect(v, wantptr) + isNull := item[0] == 'n' // null + u, ut, pv := d.indirect(v, isNull) if u != nil { err := u.UnmarshalJSON(item) if err != nil { @@ -758,7 +840,16 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool if fromQuoted { d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) } else { - d.saveError(&UnmarshalTypeError{"string", v.Type(), int64(d.off)}) + var val string + switch item[0] { + case 'n': + val = "null" + case 't', 'f': + val = "bool" + default: + val = "number" + } + d.saveError(&UnmarshalTypeError{Value: val, Type: v.Type(), Offset: int64(d.off)}) } return } @@ -781,19 +872,31 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool switch c := item[0]; c { case 'n': // null + // The main parser checks that only true and false can reach here, + // but if this was a quoted string input, it could be anything. + if fromQuoted && string(item) != "null" { + d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) + break + } switch v.Kind() { case reflect.Interface, reflect.Ptr, reflect.Map, reflect.Slice: v.Set(reflect.Zero(v.Type())) // otherwise, ignore null for primitives/string } case 't', 'f': // true, false - value := c == 't' + value := item[0] == 't' + // The main parser checks that only true and false can reach here, + // but if this was a quoted string input, it could be anything. + if fromQuoted && string(item) != "true" && string(item) != "false" { + d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) + break + } switch v.Kind() { default: if fromQuoted { d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) } else { - d.saveError(&UnmarshalTypeError{"bool", v.Type(), int64(d.off)}) + d.saveError(&UnmarshalTypeError{Value: "bool", Type: v.Type(), Offset: int64(d.off)}) } case reflect.Bool: v.SetBool(value) @@ -801,7 +904,7 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool if v.NumMethod() == 0 { v.Set(reflect.ValueOf(value)) } else { - d.saveError(&UnmarshalTypeError{"bool", v.Type(), int64(d.off)}) + d.saveError(&UnmarshalTypeError{Value: "bool", Type: v.Type(), Offset: int64(d.off)}) } } @@ -816,10 +919,10 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool } switch v.Kind() { default: - d.saveError(&UnmarshalTypeError{"string", v.Type(), int64(d.off)}) + d.saveError(&UnmarshalTypeError{Value: "string", Type: v.Type(), Offset: int64(d.off)}) case reflect.Slice: if v.Type().Elem().Kind() != reflect.Uint8 { - d.saveError(&UnmarshalTypeError{"string", v.Type(), int64(d.off)}) + d.saveError(&UnmarshalTypeError{Value: "string", Type: v.Type(), Offset: int64(d.off)}) break } b := make([]byte, base64.StdEncoding.DecodedLen(len(s))) @@ -835,7 +938,7 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool if v.NumMethod() == 0 { v.Set(reflect.ValueOf(string(s))) } else { - d.saveError(&UnmarshalTypeError{"string", v.Type(), int64(d.off)}) + d.saveError(&UnmarshalTypeError{Value: "string", Type: v.Type(), Offset: int64(d.off)}) } } @@ -860,7 +963,7 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool if fromQuoted { d.error(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) } else { - d.error(&UnmarshalTypeError{"number", v.Type(), int64(d.off)}) + d.error(&UnmarshalTypeError{Value: "number", Type: v.Type(), Offset: int64(d.off)}) } case reflect.Interface: n, err := d.convertNumber(s) @@ -869,7 +972,7 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool break } if v.NumMethod() != 0 { - d.saveError(&UnmarshalTypeError{"number", v.Type(), int64(d.off)}) + d.saveError(&UnmarshalTypeError{Value: "number", Type: v.Type(), Offset: int64(d.off)}) break } v.Set(reflect.ValueOf(n)) @@ -877,7 +980,7 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: n, err := strconv.ParseInt(s, 10, 64) if err != nil || v.OverflowInt(n) { - d.saveError(&UnmarshalTypeError{"number " + s, v.Type(), int64(d.off)}) + d.saveError(&UnmarshalTypeError{Value: "number " + s, Type: v.Type(), Offset: int64(d.off)}) break } v.SetInt(n) @@ -885,7 +988,7 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: n, err := strconv.ParseUint(s, 10, 64) if err != nil || v.OverflowUint(n) { - d.saveError(&UnmarshalTypeError{"number " + s, v.Type(), int64(d.off)}) + d.saveError(&UnmarshalTypeError{Value: "number " + s, Type: v.Type(), Offset: int64(d.off)}) break } v.SetUint(n) @@ -893,7 +996,7 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool case reflect.Float32, reflect.Float64: n, err := strconv.ParseFloat(s, v.Type().Bits()) if err != nil || v.OverflowFloat(n) { - d.saveError(&UnmarshalTypeError{"number " + s, v.Type(), int64(d.off)}) + d.saveError(&UnmarshalTypeError{Value: "number " + s, Type: v.Type(), Offset: int64(d.off)}) break } v.SetFloat(n) @@ -902,7 +1005,7 @@ func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool } // The xxxInterface routines build up a value to be stored -// in an empty interface. They are not strictly necessary, +// in an empty interface. They are not strictly necessary, // but they avoid the weight of reflection in this common case. // valueInterface is like value but returns interface{} diff --git a/libgo/go/encoding/json/decode_test.go b/libgo/go/encoding/json/decode_test.go index 9546ae459c..bd38ddd319 100644 --- a/libgo/go/encoding/json/decode_test.go +++ b/libgo/go/encoding/json/decode_test.go @@ -7,10 +7,14 @@ package json import ( "bytes" "encoding" + "errors" "fmt" "image" + "math" + "math/big" "net" "reflect" + "strconv" "strings" "testing" "time" @@ -30,6 +34,11 @@ type V struct { F1 interface{} F2 int32 F3 Number + F4 *VOuter +} + +type VOuter struct { + V V } // ifaceNumAsFloat64/ifaceNumAsNumber are used to test unmarshaling with and @@ -52,6 +61,8 @@ type tx struct { x int } +type u8 uint8 + // A type that can unmarshal itself. type unmarshaler struct { @@ -68,16 +79,20 @@ type ustruct struct { } type unmarshalerText struct { - T bool + A, B string } // needed for re-marshaling tests -func (u *unmarshalerText) MarshalText() ([]byte, error) { - return []byte(""), nil +func (u unmarshalerText) MarshalText() ([]byte, error) { + return []byte(u.A + ":" + u.B), nil } func (u *unmarshalerText) UnmarshalText(b []byte) error { - *u = unmarshalerText{true} // All we need to see that UnmarshalText is called. + pos := bytes.Index(b, []byte(":")) + if pos == -1 { + return errors.New("missing separator") + } + u.A, u.B = string(b[:pos]), string(b[pos+1:]) return nil } @@ -87,6 +102,29 @@ type ustructText struct { M unmarshalerText } +// u8marshal is an integer type that can marshal/unmarshal itself. +type u8marshal uint8 + +func (u8 u8marshal) MarshalText() ([]byte, error) { + return []byte(fmt.Sprintf("u%d", u8)), nil +} + +var errMissingU8Prefix = errors.New("missing 'u' prefix") + +func (u8 *u8marshal) UnmarshalText(b []byte) error { + if !bytes.HasPrefix(b, []byte{'u'}) { + return errMissingU8Prefix + } + n, err := strconv.Atoi(string(b[1:])) + if err != nil { + return err + } + *u8 = u8marshal(n) + return nil +} + +var _ encoding.TextUnmarshaler = (*u8marshal)(nil) + var ( um0, um1 unmarshaler // target2 of unmarshaling ump = &um1 @@ -95,12 +133,16 @@ var ( umslicep = new([]unmarshaler) umstruct = ustruct{unmarshaler{true}} - um0T, um1T unmarshalerText // target2 of unmarshaling - umpT = &um1T - umtrueT = unmarshalerText{true} - umsliceT = []unmarshalerText{{true}} - umslicepT = new([]unmarshalerText) - umstructT = ustructText{unmarshalerText{true}} + um0T, um1T unmarshalerText // target2 of unmarshaling + umpType = &um1T + umtrueXY = unmarshalerText{"x", "y"} + umsliceXY = []unmarshalerText{{"x", "y"}} + umslicepType = new([]unmarshalerText) + umstructType = new(ustructText) + umstructXY = ustructText{unmarshalerText{"x", "y"}} + + ummapType = map[unmarshalerText]bool{} + ummapXY = map[unmarshalerText]bool{unmarshalerText{"x", "y"}: true} ) // Test data structures for anonymous fields. @@ -202,14 +244,6 @@ type S13 struct { S8 } -type unmarshalTest struct { - in string - ptr interface{} - out interface{} - err error - useNumber bool -} - type Ambig struct { // Given "hello", the first match should win. First int `json:"HELLO"` @@ -225,6 +259,131 @@ type XYZ struct { func sliceAddr(x []int) *[]int { return &x } func mapAddr(x map[string]int) *map[string]int { return &x } +type byteWithMarshalJSON byte + +func (b byteWithMarshalJSON) MarshalJSON() ([]byte, error) { + return []byte(fmt.Sprintf(`"Z%.2x"`, byte(b))), nil +} + +func (b *byteWithMarshalJSON) UnmarshalJSON(data []byte) error { + if len(data) != 5 || data[0] != '"' || data[1] != 'Z' || data[4] != '"' { + return fmt.Errorf("bad quoted string") + } + i, err := strconv.ParseInt(string(data[2:4]), 16, 8) + if err != nil { + return fmt.Errorf("bad hex") + } + *b = byteWithMarshalJSON(i) + return nil +} + +type byteWithPtrMarshalJSON byte + +func (b *byteWithPtrMarshalJSON) MarshalJSON() ([]byte, error) { + return byteWithMarshalJSON(*b).MarshalJSON() +} + +func (b *byteWithPtrMarshalJSON) UnmarshalJSON(data []byte) error { + return (*byteWithMarshalJSON)(b).UnmarshalJSON(data) +} + +type byteWithMarshalText byte + +func (b byteWithMarshalText) MarshalText() ([]byte, error) { + return []byte(fmt.Sprintf(`Z%.2x`, byte(b))), nil +} + +func (b *byteWithMarshalText) UnmarshalText(data []byte) error { + if len(data) != 3 || data[0] != 'Z' { + return fmt.Errorf("bad quoted string") + } + i, err := strconv.ParseInt(string(data[1:3]), 16, 8) + if err != nil { + return fmt.Errorf("bad hex") + } + *b = byteWithMarshalText(i) + return nil +} + +type byteWithPtrMarshalText byte + +func (b *byteWithPtrMarshalText) MarshalText() ([]byte, error) { + return byteWithMarshalText(*b).MarshalText() +} + +func (b *byteWithPtrMarshalText) UnmarshalText(data []byte) error { + return (*byteWithMarshalText)(b).UnmarshalText(data) +} + +type intWithMarshalJSON int + +func (b intWithMarshalJSON) MarshalJSON() ([]byte, error) { + return []byte(fmt.Sprintf(`"Z%.2x"`, int(b))), nil +} + +func (b *intWithMarshalJSON) UnmarshalJSON(data []byte) error { + if len(data) != 5 || data[0] != '"' || data[1] != 'Z' || data[4] != '"' { + return fmt.Errorf("bad quoted string") + } + i, err := strconv.ParseInt(string(data[2:4]), 16, 8) + if err != nil { + return fmt.Errorf("bad hex") + } + *b = intWithMarshalJSON(i) + return nil +} + +type intWithPtrMarshalJSON int + +func (b *intWithPtrMarshalJSON) MarshalJSON() ([]byte, error) { + return intWithMarshalJSON(*b).MarshalJSON() +} + +func (b *intWithPtrMarshalJSON) UnmarshalJSON(data []byte) error { + return (*intWithMarshalJSON)(b).UnmarshalJSON(data) +} + +type intWithMarshalText int + +func (b intWithMarshalText) MarshalText() ([]byte, error) { + return []byte(fmt.Sprintf(`Z%.2x`, int(b))), nil +} + +func (b *intWithMarshalText) UnmarshalText(data []byte) error { + if len(data) != 3 || data[0] != 'Z' { + return fmt.Errorf("bad quoted string") + } + i, err := strconv.ParseInt(string(data[1:3]), 16, 8) + if err != nil { + return fmt.Errorf("bad hex") + } + *b = intWithMarshalText(i) + return nil +} + +type intWithPtrMarshalText int + +func (b *intWithPtrMarshalText) MarshalText() ([]byte, error) { + return intWithMarshalText(*b).MarshalText() +} + +func (b *intWithPtrMarshalText) UnmarshalText(data []byte) error { + return (*intWithMarshalText)(b).UnmarshalText(data) +} + +type unmarshalTest struct { + in string + ptr interface{} + out interface{} + err error + useNumber bool + golden bool +} + +type B struct { + B bool `json:",string"` +} + var unmarshalTests = []unmarshalTest{ // basic types {in: `true`, ptr: new(bool), out: true}, @@ -240,7 +399,7 @@ var unmarshalTests = []unmarshalTest{ {in: `"g-clef: \uD834\uDD1E"`, ptr: new(string), out: "g-clef: \U0001D11E"}, {in: `"invalid: \uD834x\uDD1E"`, ptr: new(string), out: "invalid: \uFFFDx\uFFFD"}, {in: "null", ptr: new(interface{}), out: nil}, - {in: `{"X": [1,2,3], "Y": 4}`, ptr: new(T), out: T{Y: 4}, err: &UnmarshalTypeError{"array", reflect.TypeOf(""), 7}}, + {in: `{"X": [1,2,3], "Y": 4}`, ptr: new(T), out: T{Y: 4}, err: &UnmarshalTypeError{"array", reflect.TypeOf(""), 7, "T", "X"}}, {in: `{"x": 1}`, ptr: new(tx), out: tx{}}, {in: `{"F1":1,"F2":2,"F3":3}`, ptr: new(V), out: V{F1: float64(1), F2: int32(2), F3: Number("3")}}, {in: `{"F1":1,"F2":2,"F3":3}`, ptr: new(V), out: V{F1: Number("1"), F2: int32(2), F3: Number("3")}, useNumber: true}, @@ -302,14 +461,81 @@ var unmarshalTests = []unmarshalTest{ {in: `{"T":false}`, ptr: &ump, out: &umtrue}, {in: `[{"T":false}]`, ptr: &umslice, out: umslice}, {in: `[{"T":false}]`, ptr: &umslicep, out: &umslice}, - {in: `{"M":{"T":false}}`, ptr: &umstruct, out: umstruct}, + {in: `{"M":{"T":"x:y"}}`, ptr: &umstruct, out: umstruct}, // UnmarshalText interface test - {in: `"X"`, ptr: &um0T, out: umtrueT}, // use "false" so test will fail if custom unmarshaler is not called - {in: `"X"`, ptr: &umpT, out: &umtrueT}, - {in: `["X"]`, ptr: &umsliceT, out: umsliceT}, - {in: `["X"]`, ptr: &umslicepT, out: &umsliceT}, - {in: `{"M":"X"}`, ptr: &umstructT, out: umstructT}, + {in: `"x:y"`, ptr: &um0T, out: umtrueXY}, + {in: `"x:y"`, ptr: &umpType, out: &umtrueXY}, + {in: `["x:y"]`, ptr: &umsliceXY, out: umsliceXY}, + {in: `["x:y"]`, ptr: &umslicepType, out: &umsliceXY}, + {in: `{"M":"x:y"}`, ptr: umstructType, out: umstructXY}, + + // integer-keyed map test + { + in: `{"-1":"a","0":"b","1":"c"}`, + ptr: new(map[int]string), + out: map[int]string{-1: "a", 0: "b", 1: "c"}, + }, + { + in: `{"0":"a","10":"c","9":"b"}`, + ptr: new(map[u8]string), + out: map[u8]string{0: "a", 9: "b", 10: "c"}, + }, + { + in: `{"-9223372036854775808":"min","9223372036854775807":"max"}`, + ptr: new(map[int64]string), + out: map[int64]string{math.MinInt64: "min", math.MaxInt64: "max"}, + }, + { + in: `{"18446744073709551615":"max"}`, + ptr: new(map[uint64]string), + out: map[uint64]string{math.MaxUint64: "max"}, + }, + { + in: `{"0":false,"10":true}`, + ptr: new(map[uintptr]bool), + out: map[uintptr]bool{0: false, 10: true}, + }, + + // Check that MarshalText and UnmarshalText take precedence + // over default integer handling in map keys. + { + in: `{"u2":4}`, + ptr: new(map[u8marshal]int), + out: map[u8marshal]int{2: 4}, + }, + { + in: `{"2":4}`, + ptr: new(map[u8marshal]int), + err: errMissingU8Prefix, + }, + + // integer-keyed map errors + { + in: `{"abc":"abc"}`, + ptr: new(map[int]string), + err: &UnmarshalTypeError{Value: "number abc", Type: reflect.TypeOf(0), Offset: 2}, + }, + { + in: `{"256":"abc"}`, + ptr: new(map[uint8]string), + err: &UnmarshalTypeError{Value: "number 256", Type: reflect.TypeOf(uint8(0)), Offset: 2}, + }, + { + in: `{"128":"abc"}`, + ptr: new(map[int8]string), + err: &UnmarshalTypeError{Value: "number 128", Type: reflect.TypeOf(int8(0)), Offset: 2}, + }, + { + in: `{"-1":"abc"}`, + ptr: new(map[uint8]string), + err: &UnmarshalTypeError{Value: "number -1", Type: reflect.TypeOf(uint8(0)), Offset: 2}, + }, + + // Map keys can be encoding.TextUnmarshalers. + {in: `{"x:y":true}`, ptr: &ummapType, out: ummapXY}, + // If multiple values for the same key exists, only the most recent value is used. + {in: `{"x:y":false,"x:y":true}`, ptr: &ummapType, out: ummapXY}, // Overwriting of data. // This is different from package xml, but it's what we've always done. @@ -426,12 +652,147 @@ var unmarshalTests = []unmarshalTest{ out: "hello\ufffd\ufffd\ufffd\ufffd\ufffd\ufffdworld", }, - // issue 8305 + // Used to be issue 8305, but time.Time implements encoding.TextUnmarshaler so this works now. { in: `{"2009-11-10T23:00:00Z": "hello world"}`, ptr: &map[time.Time]string{}, - err: &UnmarshalTypeError{"object", reflect.TypeOf(map[time.Time]string{}), 1}, + out: map[time.Time]string{time.Date(2009, 11, 10, 23, 0, 0, 0, time.UTC): "hello world"}, + }, + + // issue 8305 + { + in: `{"2009-11-10T23:00:00Z": "hello world"}`, + ptr: &map[Point]string{}, + err: &UnmarshalTypeError{Value: "object", Type: reflect.TypeOf(map[Point]string{}), Offset: 1}, + }, + { + in: `{"asdf": "hello world"}`, + ptr: &map[unmarshaler]string{}, + err: &UnmarshalTypeError{Value: "object", Type: reflect.TypeOf(map[unmarshaler]string{}), Offset: 1}, + }, + + // related to issue 13783. + // Go 1.7 changed marshaling a slice of typed byte to use the methods on the byte type, + // similar to marshaling a slice of typed int. + // These tests check that, assuming the byte type also has valid decoding methods, + // either the old base64 string encoding or the new per-element encoding can be + // successfully unmarshaled. The custom unmarshalers were accessible in earlier + // versions of Go, even though the custom marshaler was not. + { + in: `"AQID"`, + ptr: new([]byteWithMarshalJSON), + out: []byteWithMarshalJSON{1, 2, 3}, + }, + { + in: `["Z01","Z02","Z03"]`, + ptr: new([]byteWithMarshalJSON), + out: []byteWithMarshalJSON{1, 2, 3}, + golden: true, + }, + { + in: `"AQID"`, + ptr: new([]byteWithMarshalText), + out: []byteWithMarshalText{1, 2, 3}, + }, + { + in: `["Z01","Z02","Z03"]`, + ptr: new([]byteWithMarshalText), + out: []byteWithMarshalText{1, 2, 3}, + golden: true, + }, + { + in: `"AQID"`, + ptr: new([]byteWithPtrMarshalJSON), + out: []byteWithPtrMarshalJSON{1, 2, 3}, + }, + { + in: `["Z01","Z02","Z03"]`, + ptr: new([]byteWithPtrMarshalJSON), + out: []byteWithPtrMarshalJSON{1, 2, 3}, + golden: true, + }, + { + in: `"AQID"`, + ptr: new([]byteWithPtrMarshalText), + out: []byteWithPtrMarshalText{1, 2, 3}, }, + { + in: `["Z01","Z02","Z03"]`, + ptr: new([]byteWithPtrMarshalText), + out: []byteWithPtrMarshalText{1, 2, 3}, + golden: true, + }, + + // ints work with the marshaler but not the base64 []byte case + { + in: `["Z01","Z02","Z03"]`, + ptr: new([]intWithMarshalJSON), + out: []intWithMarshalJSON{1, 2, 3}, + golden: true, + }, + { + in: `["Z01","Z02","Z03"]`, + ptr: new([]intWithMarshalText), + out: []intWithMarshalText{1, 2, 3}, + golden: true, + }, + { + in: `["Z01","Z02","Z03"]`, + ptr: new([]intWithPtrMarshalJSON), + out: []intWithPtrMarshalJSON{1, 2, 3}, + golden: true, + }, + { + in: `["Z01","Z02","Z03"]`, + ptr: new([]intWithPtrMarshalText), + out: []intWithPtrMarshalText{1, 2, 3}, + golden: true, + }, + + {in: `0.000001`, ptr: new(float64), out: 0.000001, golden: true}, + {in: `1e-7`, ptr: new(float64), out: 1e-7, golden: true}, + {in: `100000000000000000000`, ptr: new(float64), out: 100000000000000000000.0, golden: true}, + {in: `1e+21`, ptr: new(float64), out: 1e21, golden: true}, + {in: `-0.000001`, ptr: new(float64), out: -0.000001, golden: true}, + {in: `-1e-7`, ptr: new(float64), out: -1e-7, golden: true}, + {in: `-100000000000000000000`, ptr: new(float64), out: -100000000000000000000.0, golden: true}, + {in: `-1e+21`, ptr: new(float64), out: -1e21, golden: true}, + {in: `999999999999999900000`, ptr: new(float64), out: 999999999999999900000.0, golden: true}, + {in: `9007199254740992`, ptr: new(float64), out: 9007199254740992.0, golden: true}, + {in: `9007199254740993`, ptr: new(float64), out: 9007199254740992.0, golden: false}, + + { + in: `{"V": {"F2": "hello"}}`, + ptr: new(VOuter), + err: &UnmarshalTypeError{ + Value: "string", + Struct: "V", + Field: "F2", + Type: reflect.TypeOf(int32(0)), + Offset: 20, + }, + }, + { + in: `{"V": {"F4": {}, "F2": "hello"}}`, + ptr: new(VOuter), + err: &UnmarshalTypeError{ + Value: "string", + Struct: "V", + Field: "F2", + Type: reflect.TypeOf(int32(0)), + Offset: 30, + }, + }, + + // issue 15146. + // invalid inputs in wrongStringTests below. + {in: `{"B":"true"}`, ptr: new(B), out: B{true}, golden: true}, + {in: `{"B":"false"}`, ptr: new(B), out: B{false}, golden: true}, + {in: `{"B": "maybe"}`, ptr: new(B), err: errors.New(`json: invalid use of ,string struct tag, trying to unmarshal "maybe" into bool`)}, + {in: `{"B": "tru"}`, ptr: new(B), err: errors.New(`json: invalid use of ,string struct tag, trying to unmarshal "tru" into bool`)}, + {in: `{"B": "False"}`, ptr: new(B), err: errors.New(`json: invalid use of ,string struct tag, trying to unmarshal "False" into bool`)}, + {in: `{"B": "null"}`, ptr: new(B), out: B{false}}, + {in: `{"B": "nul"}`, ptr: new(B), err: errors.New(`json: invalid use of ,string struct tag, trying to unmarshal "nul" into bool`)}, } func TestMarshal(t *testing.T) { @@ -565,13 +926,16 @@ func TestUnmarshal(t *testing.T) { continue } - // Check round trip. + // Check round trip also decodes correctly. if tt.err == nil { enc, err := Marshal(v.Interface()) if err != nil { t.Errorf("#%d: error re-marshaling: %v", i, err) continue } + if tt.golden && !bytes.Equal(enc, in) { + t.Errorf("#%d: remarshal mismatch:\nhave: %s\nwant: %s", i, enc, in) + } vv := reflect.New(reflect.TypeOf(tt.ptr).Elem()) dec = NewDecoder(bytes.NewReader(enc)) if tt.useNumber { @@ -1161,40 +1525,148 @@ func TestInterfaceSet(t *testing.T) { } } +type NullTest struct { + Bool bool + Int int + Int8 int8 + Int16 int16 + Int32 int32 + Int64 int64 + Uint uint + Uint8 uint8 + Uint16 uint16 + Uint32 uint32 + Uint64 uint64 + Float32 float32 + Float64 float64 + String string + PBool *bool + Map map[string]string + Slice []string + Interface interface{} + + PRaw *RawMessage + PTime *time.Time + PBigInt *big.Int + PText *MustNotUnmarshalText + PBuffer *bytes.Buffer // has methods, just not relevant ones + PStruct *struct{} + + Raw RawMessage + Time time.Time + BigInt big.Int + Text MustNotUnmarshalText + Buffer bytes.Buffer + Struct struct{} +} + +type NullTestStrings struct { + Bool bool `json:",string"` + Int int `json:",string"` + Int8 int8 `json:",string"` + Int16 int16 `json:",string"` + Int32 int32 `json:",string"` + Int64 int64 `json:",string"` + Uint uint `json:",string"` + Uint8 uint8 `json:",string"` + Uint16 uint16 `json:",string"` + Uint32 uint32 `json:",string"` + Uint64 uint64 `json:",string"` + Float32 float32 `json:",string"` + Float64 float64 `json:",string"` + String string `json:",string"` + PBool *bool `json:",string"` + Map map[string]string `json:",string"` + Slice []string `json:",string"` + Interface interface{} `json:",string"` + + PRaw *RawMessage `json:",string"` + PTime *time.Time `json:",string"` + PBigInt *big.Int `json:",string"` + PText *MustNotUnmarshalText `json:",string"` + PBuffer *bytes.Buffer `json:",string"` + PStruct *struct{} `json:",string"` + + Raw RawMessage `json:",string"` + Time time.Time `json:",string"` + BigInt big.Int `json:",string"` + Text MustNotUnmarshalText `json:",string"` + Buffer bytes.Buffer `json:",string"` + Struct struct{} `json:",string"` +} + // JSON null values should be ignored for primitives and string values instead of resulting in an error. // Issue 2540 func TestUnmarshalNulls(t *testing.T) { + // Unmarshal docs: + // The JSON null value unmarshals into an interface, map, pointer, or slice + // by setting that Go value to nil. Because null is often used in JSON to mean + // ``not present,'' unmarshaling a JSON null into any other Go type has no effect + // on the value and produces no error. + jsonData := []byte(`{ - "Bool" : null, - "Int" : null, - "Int8" : null, - "Int16" : null, - "Int32" : null, - "Int64" : null, - "Uint" : null, - "Uint8" : null, - "Uint16" : null, - "Uint32" : null, - "Uint64" : null, - "Float32" : null, - "Float64" : null, - "String" : null}`) - - nulls := All{ - Bool: true, - Int: 2, - Int8: 3, - Int16: 4, - Int32: 5, - Int64: 6, - Uint: 7, - Uint8: 8, - Uint16: 9, - Uint32: 10, - Uint64: 11, - Float32: 12.1, - Float64: 13.1, - String: "14"} + "Bool" : null, + "Int" : null, + "Int8" : null, + "Int16" : null, + "Int32" : null, + "Int64" : null, + "Uint" : null, + "Uint8" : null, + "Uint16" : null, + "Uint32" : null, + "Uint64" : null, + "Float32" : null, + "Float64" : null, + "String" : null, + "PBool": null, + "Map": null, + "Slice": null, + "Interface": null, + "PRaw": null, + "PTime": null, + "PBigInt": null, + "PText": null, + "PBuffer": null, + "PStruct": null, + "Raw": null, + "Time": null, + "BigInt": null, + "Text": null, + "Buffer": null, + "Struct": null + }`) + nulls := NullTest{ + Bool: true, + Int: 2, + Int8: 3, + Int16: 4, + Int32: 5, + Int64: 6, + Uint: 7, + Uint8: 8, + Uint16: 9, + Uint32: 10, + Uint64: 11, + Float32: 12.1, + Float64: 13.1, + String: "14", + PBool: new(bool), + Map: map[string]string{}, + Slice: []string{}, + Interface: new(MustNotUnmarshalJSON), + PRaw: new(RawMessage), + PTime: new(time.Time), + PBigInt: new(big.Int), + PText: new(MustNotUnmarshalText), + PStruct: new(struct{}), + PBuffer: new(bytes.Buffer), + Raw: RawMessage("123"), + Time: time.Unix(123456789, 0), + BigInt: *big.NewInt(123), + } + + before := nulls.Time.String() err := Unmarshal(jsonData, &nulls) if err != nil { @@ -1203,9 +1675,61 @@ func TestUnmarshalNulls(t *testing.T) { if !nulls.Bool || nulls.Int != 2 || nulls.Int8 != 3 || nulls.Int16 != 4 || nulls.Int32 != 5 || nulls.Int64 != 6 || nulls.Uint != 7 || nulls.Uint8 != 8 || nulls.Uint16 != 9 || nulls.Uint32 != 10 || nulls.Uint64 != 11 || nulls.Float32 != 12.1 || nulls.Float64 != 13.1 || nulls.String != "14" { - t.Errorf("Unmarshal of null values affected primitives") } + + if nulls.PBool != nil { + t.Errorf("Unmarshal of null did not clear nulls.PBool") + } + if nulls.Map != nil { + t.Errorf("Unmarshal of null did not clear nulls.Map") + } + if nulls.Slice != nil { + t.Errorf("Unmarshal of null did not clear nulls.Slice") + } + if nulls.Interface != nil { + t.Errorf("Unmarshal of null did not clear nulls.Interface") + } + if nulls.PRaw != nil { + t.Errorf("Unmarshal of null did not clear nulls.PRaw") + } + if nulls.PTime != nil { + t.Errorf("Unmarshal of null did not clear nulls.PTime") + } + if nulls.PBigInt != nil { + t.Errorf("Unmarshal of null did not clear nulls.PBigInt") + } + if nulls.PText != nil { + t.Errorf("Unmarshal of null did not clear nulls.PText") + } + if nulls.PBuffer != nil { + t.Errorf("Unmarshal of null did not clear nulls.PBuffer") + } + if nulls.PStruct != nil { + t.Errorf("Unmarshal of null did not clear nulls.PStruct") + } + + if string(nulls.Raw) != "null" { + t.Errorf("Unmarshal of RawMessage null did not record null: %v", string(nulls.Raw)) + } + if nulls.Time.String() != before { + t.Errorf("Unmarshal of time.Time null set time to %v", nulls.Time.String()) + } + if nulls.BigInt.String() != "123" { + t.Errorf("Unmarshal of big.Int null set int to %v", nulls.BigInt.String()) + } +} + +type MustNotUnmarshalJSON struct{} + +func (x MustNotUnmarshalJSON) UnmarshalJSON(data []byte) error { + return errors.New("MustNotUnmarshalJSON was used") +} + +type MustNotUnmarshalText struct{} + +func (x MustNotUnmarshalText) UnmarshalText(text []byte) error { + return errors.New("MustNotUnmarshalText was used") } func TestStringKind(t *testing.T) { @@ -1231,8 +1755,8 @@ func TestStringKind(t *testing.T) { } } -// Custom types with []byte as underlying type could not be marshalled -// and then unmarshalled. +// Custom types with []byte as underlying type could not be marshaled +// and then unmarshaled. // Issue 8962. func TestByteKind(t *testing.T) { type byteKind []byte @@ -1270,7 +1794,7 @@ func TestSliceOfCustomByte(t *testing.T) { t.Fatal(err) } if !reflect.DeepEqual(a, b) { - t.Fatal("expected %v == %v", a, b) + t.Fatalf("expected %v == %v", a, b) } } @@ -1444,7 +1968,7 @@ var invalidUnmarshalTextTests = []struct { {nil, "json: Unmarshal(nil)"}, {struct{}{}, "json: Unmarshal(non-pointer struct {})"}, {(*int)(nil), "json: Unmarshal(nil *int)"}, - {new(net.IP), "json: cannot unmarshal string into Go value of type *net.IP"}, + {new(net.IP), "json: cannot unmarshal number into Go value of type *net.IP"}, } func TestInvalidUnmarshalText(t *testing.T) { diff --git a/libgo/go/encoding/json/encode.go b/libgo/go/encoding/json/encode.go index 69ac7e03c8..8f21ddaed9 100644 --- a/libgo/go/encoding/json/encode.go +++ b/libgo/go/encoding/json/encode.go @@ -1,9 +1,9 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Package json implements encoding and decoding of JSON objects as defined in -// RFC 4627. The mapping between JSON objects and Go values is described +// Package json implements encoding and decoding of JSON as defined in +// RFC 4627. The mapping between JSON and Go values is described // in the documentation for the Marshal and Unmarshal functions. // // See "JSON and Go" for an introduction to this package: @@ -22,6 +22,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "unicode" "unicode/utf8" ) @@ -33,7 +34,7 @@ import ( // and is not a nil pointer, Marshal calls its MarshalJSON method // to produce JSON. If no MarshalJSON method is present but the // value implements encoding.TextMarshaler instead, Marshal calls -// its MarshalText method. +// its MarshalText method and encodes the result as a JSON string. // The nil pointer exception is not strictly necessary // but mimics a similar, necessary exception in the behavior of // UnmarshalJSON. @@ -49,24 +50,33 @@ import ( // The angle brackets "<" and ">" are escaped to "\u003c" and "\u003e" // to keep some browsers from misinterpreting JSON output as HTML. // Ampersand "&" is also escaped to "\u0026" for the same reason. +// This escaping can be disabled using an Encoder that had SetEscapeHTML(false) +// called on it. // // Array and slice values encode as JSON arrays, except that // []byte encodes as a base64-encoded string, and a nil slice -// encodes as the null JSON object. +// encodes as the null JSON value. // -// Struct values encode as JSON objects. Each exported struct field -// becomes a member of the object unless -// - the field's tag is "-", or -// - the field is empty and its tag specifies the "omitempty" option. -// The empty values are false, 0, any -// nil pointer or interface value, and any array, slice, map, or string of -// length zero. The object's default key string is the struct field name -// but can be specified in the struct field's tag value. The "json" key in -// the struct field's tag value is the key name, followed by an optional comma -// and options. Examples: +// Struct values encode as JSON objects. +// Each exported struct field becomes a member of the object, using the +// field name as the object key, unless the field is omitted for one of the +// reasons given below. // -// // Field is ignored by this package. -// Field int `json:"-"` +// The encoding of each struct field can be customized by the format string +// stored under the "json" key in the struct field's tag. +// The format string gives the name of the field, possibly followed by a +// comma-separated list of options. The name may be empty in order to +// specify options without overriding the default field name. +// +// The "omitempty" option specifies that the field should be omitted +// from the encoding if the field has an empty value, defined as +// false, 0, a nil pointer, a nil interface value, and any empty array, +// slice, map, or string. +// +// As a special case, if the field tag is "-", the field is always omitted. +// Note that a field with name "-" can still be generated using the tag "-,". +// +// Examples of struct field tags and their meanings: // // // Field appears in JSON as key "myName". // Field int `json:"myName"` @@ -81,6 +91,12 @@ import ( // // Note the leading comma. // Field int `json:",omitempty"` // +// // Field is ignored by this package. +// Field int `json:"-"` +// +// // Field appears in JSON as key "-". +// Field int `json:"-,"` +// // The "string" option signals that a field is stored as JSON inside a // JSON-encoded string. It applies only to fields of string, floating point, // integer, or boolean types. This extra level of encoding is sometimes used @@ -89,8 +105,8 @@ import ( // Int64String int64 `json:",string"` // // The key name will be used if it's a non-empty string consisting of -// only Unicode letters, digits, dollar signs, percent signs, hyphens, -// underscores and slashes. +// only Unicode letters, digits, and ASCII punctuation except quotation +// marks, backslash, and comma. // // Anonymous struct fields are usually marshaled as if their inner exported fields // were fields in the outer struct, subject to the usual Go visibility rules amended @@ -108,7 +124,9 @@ import ( // // 1) Of those fields, if any are JSON-tagged, only tagged fields are considered, // even if there are multiple untagged fields that would otherwise conflict. +// // 2) If there is exactly one field (tagged or not according to the first rule), that is selected. +// // 3) Otherwise there are multiple fields, and all are ignored; no error occurs. // // Handling of anonymous struct fields is new in Go 1.1. @@ -116,27 +134,31 @@ import ( // an anonymous struct field in both current and earlier versions, give the field // a JSON tag of "-". // -// Map values encode as JSON objects. -// The map's key type must be string; the map keys are used as JSON object -// keys, subject to the UTF-8 coercion described for string values above. +// Map values encode as JSON objects. The map's key type must either be a +// string, an integer type, or implement encoding.TextMarshaler. The map keys +// are sorted and used as JSON object keys by applying the following rules, +// subject to the UTF-8 coercion described for string values above: +// - string keys are used directly +// - encoding.TextMarshalers are marshaled +// - integer keys are converted to strings // // Pointer values encode as the value pointed to. -// A nil pointer encodes as the null JSON object. +// A nil pointer encodes as the null JSON value. // // Interface values encode as the value contained in the interface. -// A nil interface value encodes as the null JSON object. +// A nil interface value encodes as the null JSON value. // // Channel, complex, and function values cannot be encoded in JSON. // Attempting to encode such a value causes Marshal to return // an UnsupportedTypeError. // // JSON cannot represent cyclic data structures and Marshal does not -// handle them. Passing cyclic structures to Marshal will result in +// handle them. Passing cyclic structures to Marshal will result in // an infinite recursion. // func Marshal(v interface{}) ([]byte, error) { e := &encodeState{} - err := e.marshal(v) + err := e.marshal(v, encOpts{escapeHTML: true}) if err != nil { return nil, err } @@ -192,7 +214,7 @@ func HTMLEscape(dst *bytes.Buffer, src []byte) { } } -// Marshaler is the interface implemented by objects that +// Marshaler is the interface implemented by types that // can marshal themselves into valid JSON. type Marshaler interface { MarshalJSON() ([]byte, error) @@ -259,7 +281,7 @@ func newEncodeState() *encodeState { return new(encodeState) } -func (e *encodeState) marshal(v interface{}) (err error) { +func (e *encodeState) marshal(v interface{}, opts encOpts) (err error) { defer func() { if r := recover(); r != nil { if _, ok := r.(runtime.Error); ok { @@ -271,7 +293,7 @@ func (e *encodeState) marshal(v interface{}) (err error) { err = r.(error) } }() - e.reflectValue(reflect.ValueOf(v)) + e.reflectValue(reflect.ValueOf(v), opts) return nil } @@ -297,11 +319,18 @@ func isEmptyValue(v reflect.Value) bool { return false } -func (e *encodeState) reflectValue(v reflect.Value) { - valueEncoder(v)(e, v, false) +func (e *encodeState) reflectValue(v reflect.Value, opts encOpts) { + valueEncoder(v)(e, v, opts) +} + +type encOpts struct { + // quoted causes primitive fields to be encoded inside JSON strings. + quoted bool + // escapeHTML causes '<', '>', and '&' to be escaped in JSON strings. + escapeHTML bool } -type encoderFunc func(e *encodeState, v reflect.Value, quoted bool) +type encoderFunc func(e *encodeState, v reflect.Value, opts encOpts) var encoderCache struct { sync.RWMutex @@ -325,7 +354,7 @@ func typeEncoder(t reflect.Type) encoderFunc { // To deal with recursive types, populate the map with an // indirect func before we build it. This type waits on the - // real func (f) to be ready and then calls it. This indirect + // real func (f) to be ready and then calls it. This indirect // func is only used for recursive types. encoderCache.Lock() if encoderCache.m == nil { @@ -333,9 +362,9 @@ func typeEncoder(t reflect.Type) encoderFunc { } var wg sync.WaitGroup wg.Add(1) - encoderCache.m[t] = func(e *encodeState, v reflect.Value, quoted bool) { + encoderCache.m[t] = func(e *encodeState, v reflect.Value, opts encOpts) { wg.Wait() - f(e, v, quoted) + f(e, v, opts) } encoderCache.Unlock() @@ -405,27 +434,31 @@ func newTypeEncoder(t reflect.Type, allowAddr bool) encoderFunc { } } -func invalidValueEncoder(e *encodeState, v reflect.Value, quoted bool) { +func invalidValueEncoder(e *encodeState, v reflect.Value, _ encOpts) { e.WriteString("null") } -func marshalerEncoder(e *encodeState, v reflect.Value, quoted bool) { +func marshalerEncoder(e *encodeState, v reflect.Value, opts encOpts) { if v.Kind() == reflect.Ptr && v.IsNil() { e.WriteString("null") return } - m := v.Interface().(Marshaler) + m, ok := v.Interface().(Marshaler) + if !ok { + e.WriteString("null") + return + } b, err := m.MarshalJSON() if err == nil { // copy JSON into buffer, checking validity. - err = compact(&e.Buffer, b, true) + err = compact(&e.Buffer, b, opts.escapeHTML) } if err != nil { e.error(&MarshalerError{v.Type(), err}) } } -func addrMarshalerEncoder(e *encodeState, v reflect.Value, quoted bool) { +func addrMarshalerEncoder(e *encodeState, v reflect.Value, _ encOpts) { va := v.Addr() if va.IsNil() { e.WriteString("null") @@ -442,7 +475,7 @@ func addrMarshalerEncoder(e *encodeState, v reflect.Value, quoted bool) { } } -func textMarshalerEncoder(e *encodeState, v reflect.Value, quoted bool) { +func textMarshalerEncoder(e *encodeState, v reflect.Value, opts encOpts) { if v.Kind() == reflect.Ptr && v.IsNil() { e.WriteString("null") return @@ -452,10 +485,10 @@ func textMarshalerEncoder(e *encodeState, v reflect.Value, quoted bool) { if err != nil { e.error(&MarshalerError{v.Type(), err}) } - e.stringBytes(b) + e.stringBytes(b, opts.escapeHTML) } -func addrTextMarshalerEncoder(e *encodeState, v reflect.Value, quoted bool) { +func addrTextMarshalerEncoder(e *encodeState, v reflect.Value, opts encOpts) { va := v.Addr() if va.IsNil() { e.WriteString("null") @@ -466,11 +499,11 @@ func addrTextMarshalerEncoder(e *encodeState, v reflect.Value, quoted bool) { if err != nil { e.error(&MarshalerError{v.Type(), err}) } - e.stringBytes(b) + e.stringBytes(b, opts.escapeHTML) } -func boolEncoder(e *encodeState, v reflect.Value, quoted bool) { - if quoted { +func boolEncoder(e *encodeState, v reflect.Value, opts encOpts) { + if opts.quoted { e.WriteByte('"') } if v.Bool() { @@ -478,46 +511,70 @@ func boolEncoder(e *encodeState, v reflect.Value, quoted bool) { } else { e.WriteString("false") } - if quoted { + if opts.quoted { e.WriteByte('"') } } -func intEncoder(e *encodeState, v reflect.Value, quoted bool) { +func intEncoder(e *encodeState, v reflect.Value, opts encOpts) { b := strconv.AppendInt(e.scratch[:0], v.Int(), 10) - if quoted { + if opts.quoted { e.WriteByte('"') } e.Write(b) - if quoted { + if opts.quoted { e.WriteByte('"') } } -func uintEncoder(e *encodeState, v reflect.Value, quoted bool) { +func uintEncoder(e *encodeState, v reflect.Value, opts encOpts) { b := strconv.AppendUint(e.scratch[:0], v.Uint(), 10) - if quoted { + if opts.quoted { e.WriteByte('"') } e.Write(b) - if quoted { + if opts.quoted { e.WriteByte('"') } } type floatEncoder int // number of bits -func (bits floatEncoder) encode(e *encodeState, v reflect.Value, quoted bool) { +func (bits floatEncoder) encode(e *encodeState, v reflect.Value, opts encOpts) { f := v.Float() if math.IsInf(f, 0) || math.IsNaN(f) { e.error(&UnsupportedValueError{v, strconv.FormatFloat(f, 'g', -1, int(bits))}) } - b := strconv.AppendFloat(e.scratch[:0], f, 'g', -1, int(bits)) - if quoted { + + // Convert as if by ES6 number to string conversion. + // This matches most other JSON generators. + // See golang.org/issue/6384 and golang.org/issue/14135. + // Like fmt %g, but the exponent cutoffs are different + // and exponents themselves are not padded to two digits. + b := e.scratch[:0] + abs := math.Abs(f) + fmt := byte('f') + // Note: Must use float32 comparisons for underlying float32 value to get precise cutoffs right. + if abs != 0 { + if bits == 64 && (abs < 1e-6 || abs >= 1e21) || bits == 32 && (float32(abs) < 1e-6 || float32(abs) >= 1e21) { + fmt = 'e' + } + } + b = strconv.AppendFloat(b, f, fmt, -1, int(bits)) + if fmt == 'e' { + // clean up e-09 to e-9 + n := len(b) + if n >= 4 && b[n-4] == 'e' && b[n-3] == '-' && b[n-2] == '0' { + b[n-2] = b[n-1] + b = b[:n-1] + } + } + + if opts.quoted { e.WriteByte('"') } e.Write(b) - if quoted { + if opts.quoted { e.WriteByte('"') } } @@ -527,7 +584,7 @@ var ( float64Encoder = (floatEncoder(64)).encode ) -func stringEncoder(e *encodeState, v reflect.Value, quoted bool) { +func stringEncoder(e *encodeState, v reflect.Value, opts encOpts) { if v.Type() == numberType { numStr := v.String() // In Go1.5 the empty string encodes to "0", while this is not a valid number literal @@ -541,26 +598,26 @@ func stringEncoder(e *encodeState, v reflect.Value, quoted bool) { e.WriteString(numStr) return } - if quoted { + if opts.quoted { sb, err := Marshal(v.String()) if err != nil { e.error(err) } - e.string(string(sb)) + e.string(string(sb), opts.escapeHTML) } else { - e.string(v.String()) + e.string(v.String(), opts.escapeHTML) } } -func interfaceEncoder(e *encodeState, v reflect.Value, quoted bool) { +func interfaceEncoder(e *encodeState, v reflect.Value, opts encOpts) { if v.IsNil() { e.WriteString("null") return } - e.reflectValue(v.Elem()) + e.reflectValue(v.Elem(), opts) } -func unsupportedTypeEncoder(e *encodeState, v reflect.Value, quoted bool) { +func unsupportedTypeEncoder(e *encodeState, v reflect.Value, _ encOpts) { e.error(&UnsupportedTypeError{v.Type()}) } @@ -569,7 +626,7 @@ type structEncoder struct { fieldEncs []encoderFunc } -func (se *structEncoder) encode(e *encodeState, v reflect.Value, quoted bool) { +func (se *structEncoder) encode(e *encodeState, v reflect.Value, opts encOpts) { e.WriteByte('{') first := true for i, f := range se.fields { @@ -582,9 +639,10 @@ func (se *structEncoder) encode(e *encodeState, v reflect.Value, quoted bool) { } else { e.WriteByte(',') } - e.string(f.name) + e.string(f.name, opts.escapeHTML) e.WriteByte(':') - se.fieldEncs[i](e, fv, f.quoted) + opts.quoted = f.quoted + se.fieldEncs[i](e, fv, opts) } e.WriteByte('}') } @@ -605,34 +663,50 @@ type mapEncoder struct { elemEnc encoderFunc } -func (me *mapEncoder) encode(e *encodeState, v reflect.Value, _ bool) { +func (me *mapEncoder) encode(e *encodeState, v reflect.Value, opts encOpts) { if v.IsNil() { e.WriteString("null") return } e.WriteByte('{') - var sv stringValues = v.MapKeys() - sort.Sort(sv) - for i, k := range sv { + + // Extract and sort the keys. + keys := v.MapKeys() + sv := make([]reflectWithString, len(keys)) + for i, v := range keys { + sv[i].v = v + if err := sv[i].resolve(); err != nil { + e.error(&MarshalerError{v.Type(), err}) + } + } + sort.Slice(sv, func(i, j int) bool { return sv[i].s < sv[j].s }) + + for i, kv := range sv { if i > 0 { e.WriteByte(',') } - e.string(k.String()) + e.string(kv.s, opts.escapeHTML) e.WriteByte(':') - me.elemEnc(e, v.MapIndex(k), false) + me.elemEnc(e, v.MapIndex(kv.v), opts) } e.WriteByte('}') } func newMapEncoder(t reflect.Type) encoderFunc { - if t.Key().Kind() != reflect.String { - return unsupportedTypeEncoder + switch t.Key().Kind() { + case reflect.String, + reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + default: + if !t.Key().Implements(textMarshalerType) { + return unsupportedTypeEncoder + } } me := &mapEncoder{typeEncoder(t.Elem())} return me.encode } -func encodeByteSlice(e *encodeState, v reflect.Value, _ bool) { +func encodeByteSlice(e *encodeState, v reflect.Value, _ encOpts) { if v.IsNil() { e.WriteString("null") return @@ -659,18 +733,21 @@ type sliceEncoder struct { arrayEnc encoderFunc } -func (se *sliceEncoder) encode(e *encodeState, v reflect.Value, _ bool) { +func (se *sliceEncoder) encode(e *encodeState, v reflect.Value, opts encOpts) { if v.IsNil() { e.WriteString("null") return } - se.arrayEnc(e, v, false) + se.arrayEnc(e, v, opts) } func newSliceEncoder(t reflect.Type) encoderFunc { // Byte slices get special treatment; arrays don't. if t.Elem().Kind() == reflect.Uint8 { - return encodeByteSlice + p := reflect.PtrTo(t.Elem()) + if !p.Implements(marshalerType) && !p.Implements(textMarshalerType) { + return encodeByteSlice + } } enc := &sliceEncoder{newArrayEncoder(t)} return enc.encode @@ -680,14 +757,14 @@ type arrayEncoder struct { elemEnc encoderFunc } -func (ae *arrayEncoder) encode(e *encodeState, v reflect.Value, _ bool) { +func (ae *arrayEncoder) encode(e *encodeState, v reflect.Value, opts encOpts) { e.WriteByte('[') n := v.Len() for i := 0; i < n; i++ { if i > 0 { e.WriteByte(',') } - ae.elemEnc(e, v.Index(i), false) + ae.elemEnc(e, v.Index(i), opts) } e.WriteByte(']') } @@ -701,12 +778,12 @@ type ptrEncoder struct { elemEnc encoderFunc } -func (pe *ptrEncoder) encode(e *encodeState, v reflect.Value, quoted bool) { +func (pe *ptrEncoder) encode(e *encodeState, v reflect.Value, opts encOpts) { if v.IsNil() { e.WriteString("null") return } - pe.elemEnc(e, v.Elem(), quoted) + pe.elemEnc(e, v.Elem(), opts) } func newPtrEncoder(t reflect.Type) encoderFunc { @@ -718,11 +795,11 @@ type condAddrEncoder struct { canAddrEnc, elseEnc encoderFunc } -func (ce *condAddrEncoder) encode(e *encodeState, v reflect.Value, quoted bool) { +func (ce *condAddrEncoder) encode(e *encodeState, v reflect.Value, opts encOpts) { if v.CanAddr() { - ce.canAddrEnc(e, v, quoted) + ce.canAddrEnc(e, v, opts) } else { - ce.elseEnc(e, v, quoted) + ce.elseEnc(e, v, opts) } } @@ -775,23 +852,40 @@ func typeByIndex(t reflect.Type, index []int) reflect.Type { return t } -// stringValues is a slice of reflect.Value holding *reflect.StringValue. -// It implements the methods to sort by string. -type stringValues []reflect.Value +type reflectWithString struct { + v reflect.Value + s string +} -func (sv stringValues) Len() int { return len(sv) } -func (sv stringValues) Swap(i, j int) { sv[i], sv[j] = sv[j], sv[i] } -func (sv stringValues) Less(i, j int) bool { return sv.get(i) < sv.get(j) } -func (sv stringValues) get(i int) string { return sv[i].String() } +func (w *reflectWithString) resolve() error { + if w.v.Kind() == reflect.String { + w.s = w.v.String() + return nil + } + if tm, ok := w.v.Interface().(encoding.TextMarshaler); ok { + buf, err := tm.MarshalText() + w.s = string(buf) + return err + } + switch w.v.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + w.s = strconv.FormatInt(w.v.Int(), 10) + return nil + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + w.s = strconv.FormatUint(w.v.Uint(), 10) + return nil + } + panic("unexpected map key type") +} // NOTE: keep in sync with stringBytes below. -func (e *encodeState) string(s string) int { +func (e *encodeState) string(s string, escapeHTML bool) int { len0 := e.Len() e.WriteByte('"') start := 0 for i := 0; i < len(s); { if b := s[i]; b < utf8.RuneSelf { - if 0x20 <= b && b != '\\' && b != '"' && b != '<' && b != '>' && b != '&' { + if htmlSafeSet[b] || (!escapeHTML && safeSet[b]) { i++ continue } @@ -812,10 +906,11 @@ func (e *encodeState) string(s string) int { e.WriteByte('\\') e.WriteByte('t') default: - // This encodes bytes < 0x20 except for \n and \r, - // as well as <, > and &. The latter are escaped because they - // can lead to security holes when user-controlled strings - // are rendered into JSON and served to some browsers. + // This encodes bytes < 0x20 except for \t, \n and \r. + // If escapeHTML is set, it also escapes <, >, and & + // because they can lead to security holes when + // user-controlled strings are rendered into JSON + // and served to some browsers. e.WriteString(`\u00`) e.WriteByte(hex[b>>4]) e.WriteByte(hex[b&0xF]) @@ -861,13 +956,13 @@ func (e *encodeState) string(s string) int { } // NOTE: keep in sync with string above. -func (e *encodeState) stringBytes(s []byte) int { +func (e *encodeState) stringBytes(s []byte, escapeHTML bool) int { len0 := e.Len() e.WriteByte('"') start := 0 for i := 0; i < len(s); { if b := s[i]; b < utf8.RuneSelf { - if 0x20 <= b && b != '\\' && b != '"' && b != '<' && b != '>' && b != '&' { + if htmlSafeSet[b] || (!escapeHTML && safeSet[b]) { i++ continue } @@ -888,10 +983,11 @@ func (e *encodeState) stringBytes(s []byte) int { e.WriteByte('\\') e.WriteByte('t') default: - // This encodes bytes < 0x20 except for \n and \r, - // as well as <, >, and &. The latter are escaped because they - // can lead to security holes when user-controlled strings - // are rendered into JSON and served to some browsers. + // This encodes bytes < 0x20 except for \t, \n and \r. + // If escapeHTML is set, it also escapes <, >, and & + // because they can lead to security holes when + // user-controlled strings are rendered into JSON + // and served to some browsers. e.WriteString(`\u00`) e.WriteByte(hex[b>>4]) e.WriteByte(hex[b&0xF]) @@ -955,28 +1051,6 @@ func fillField(f field) field { return f } -// byName sorts field by name, breaking ties with depth, -// then breaking ties with "name came from json tag", then -// breaking ties with index sequence. -type byName []field - -func (x byName) Len() int { return len(x) } - -func (x byName) Swap(i, j int) { x[i], x[j] = x[j], x[i] } - -func (x byName) Less(i, j int) bool { - if x[i].name != x[j].name { - return x[i].name < x[j].name - } - if len(x[i].index) != len(x[j].index) { - return len(x[i].index) < len(x[j].index) - } - if x[i].tag != x[j].tag { - return x[i].tag - } - return byIndex(x).Less(i, j) -} - // byIndex sorts field by index sequence. type byIndex []field @@ -1094,7 +1168,22 @@ func typeFields(t reflect.Type) []field { } } - sort.Sort(byName(fields)) + sort.Slice(fields, func(i, j int) bool { + x := fields + // sort field by name, breaking ties with depth, then + // breaking ties with "name came from json tag", then + // breaking ties with index sequence. + if x[i].name != x[j].name { + return x[i].name < x[j].name + } + if len(x[i].index) != len(x[j].index) { + return len(x[i].index) < len(x[j].index) + } + if x[i].tag != x[j].tag { + return x[i].tag + } + return byIndex(x).Less(i, j) + }) // Delete all fields that are hidden by the Go rules for embedded fields, // except that fields with JSON tags are promoted. @@ -1169,15 +1258,14 @@ func dominantField(fields []field) (field, bool) { } var fieldCache struct { - sync.RWMutex - m map[reflect.Type][]field + value atomic.Value // map[reflect.Type][]field + mu sync.Mutex // used only by writers } // cachedTypeFields is like typeFields but uses a cache to avoid repeated work. func cachedTypeFields(t reflect.Type) []field { - fieldCache.RLock() - f := fieldCache.m[t] - fieldCache.RUnlock() + m, _ := fieldCache.value.Load().(map[reflect.Type][]field) + f := m[t] if f != nil { return f } @@ -1189,11 +1277,14 @@ func cachedTypeFields(t reflect.Type) []field { f = []field{} } - fieldCache.Lock() - if fieldCache.m == nil { - fieldCache.m = map[reflect.Type][]field{} + fieldCache.mu.Lock() + m, _ = fieldCache.value.Load().(map[reflect.Type][]field) + newM := make(map[reflect.Type][]field, len(m)+1) + for k, v := range m { + newM[k] = v } - fieldCache.m[t] = f - fieldCache.Unlock() + newM[t] = f + fieldCache.value.Store(newM) + fieldCache.mu.Unlock() return f } diff --git a/libgo/go/encoding/json/encode_test.go b/libgo/go/encoding/json/encode_test.go index c00491e00c..6d574cfc47 100644 --- a/libgo/go/encoding/json/encode_test.go +++ b/libgo/go/encoding/json/encode_test.go @@ -6,8 +6,12 @@ package json import ( "bytes" + "fmt" + "log" "math" "reflect" + "regexp" + "strconv" "testing" "unicode" ) @@ -289,6 +293,44 @@ type BugX struct { BugB } +// Issue 16042. Even if a nil interface value is passed in +// as long as it implements MarshalJSON, it should be marshaled. +type nilMarshaler string + +func (nm *nilMarshaler) MarshalJSON() ([]byte, error) { + if nm == nil { + return Marshal("0zenil0") + } + return Marshal("zenil:" + string(*nm)) +} + +// Issue 16042. +func TestNilMarshal(t *testing.T) { + testCases := []struct { + v interface{} + want string + }{ + {v: nil, want: `null`}, + {v: new(float64), want: `0`}, + {v: []interface{}(nil), want: `null`}, + {v: []string(nil), want: `null`}, + {v: map[string]string(nil), want: `null`}, + {v: []byte(nil), want: `null`}, + {v: struct{ M string }{"gopher"}, want: `{"M":"gopher"}`}, + {v: struct{ M Marshaler }{}, want: `{"M":null}`}, + {v: struct{ M Marshaler }{(*nilMarshaler)(nil)}, want: `{"M":"0zenil0"}`}, + {v: struct{ M interface{} }{(*nilMarshaler)(nil)}, want: `{"M":null}`}, + } + + for _, tt := range testCases { + out, err := Marshal(tt.v) + if err != nil || string(out) != tt.want { + t.Errorf("Marshal(%#v) = %#q, %#v, want %#q, nil", tt.v, out, err, tt.want) + continue + } + } +} + // Issue 5245. func TestEmbeddedBug(t *testing.T) { v := BugB{ @@ -374,66 +416,47 @@ func TestDuplicatedFieldDisappears(t *testing.T) { } func TestStringBytes(t *testing.T) { + t.Parallel() // Test that encodeState.stringBytes and encodeState.string use the same encoding. - es := &encodeState{} var r []rune for i := '\u0000'; i <= unicode.MaxRune; i++ { r = append(r, i) } s := string(r) + "\xff\xff\xffhello" // some invalid UTF-8 too - es.string(s) - esBytes := &encodeState{} - esBytes.stringBytes([]byte(s)) + for _, escapeHTML := range []bool{true, false} { + es := &encodeState{} + es.string(s, escapeHTML) - enc := es.Buffer.String() - encBytes := esBytes.Buffer.String() - if enc != encBytes { - i := 0 - for i < len(enc) && i < len(encBytes) && enc[i] == encBytes[i] { - i++ - } - enc = enc[i:] - encBytes = encBytes[i:] - i = 0 - for i < len(enc) && i < len(encBytes) && enc[len(enc)-i-1] == encBytes[len(encBytes)-i-1] { - i++ - } - enc = enc[:len(enc)-i] - encBytes = encBytes[:len(encBytes)-i] - - if len(enc) > 20 { - enc = enc[:20] + "..." - } - if len(encBytes) > 20 { - encBytes = encBytes[:20] + "..." - } - - t.Errorf("encodings differ at %#q vs %#q", enc, encBytes) - } -} - -func TestIssue6458(t *testing.T) { - type Foo struct { - M RawMessage - } - x := Foo{RawMessage(`"foo"`)} + esBytes := &encodeState{} + esBytes.stringBytes([]byte(s), escapeHTML) - b, err := Marshal(&x) - if err != nil { - t.Fatal(err) - } - if want := `{"M":"foo"}`; string(b) != want { - t.Errorf("Marshal(&x) = %#q; want %#q", b, want) - } + enc := es.Buffer.String() + encBytes := esBytes.Buffer.String() + if enc != encBytes { + i := 0 + for i < len(enc) && i < len(encBytes) && enc[i] == encBytes[i] { + i++ + } + enc = enc[i:] + encBytes = encBytes[i:] + i = 0 + for i < len(enc) && i < len(encBytes) && enc[len(enc)-i-1] == encBytes[len(encBytes)-i-1] { + i++ + } + enc = enc[:len(enc)-i] + encBytes = encBytes[:len(encBytes)-i] - b, err = Marshal(x) - if err != nil { - t.Fatal(err) - } + if len(enc) > 20 { + enc = enc[:20] + "..." + } + if len(encBytes) > 20 { + encBytes = encBytes[:20] + "..." + } - if want := `{"M":"ImZvbyI="}`; string(b) != want { - t.Errorf("Marshal(x) = %#q; want %#q", b, want) + t.Errorf("with escapeHTML=%t, encodings differ at %#q vs %#q", + escapeHTML, enc, encBytes) + } } } @@ -478,7 +501,7 @@ func TestEncodePointerString(t *testing.T) { t.Fatalf("Unmarshal: %v", err) } if back.N == nil { - t.Fatalf("Unmarshalled nil N field") + t.Fatalf("Unmarshaled nil N field") } if *back.N != 42 { t.Fatalf("*N = %d; want 42", *back.N) @@ -536,3 +559,278 @@ func TestEncodeString(t *testing.T) { } } } + +type jsonbyte byte + +func (b jsonbyte) MarshalJSON() ([]byte, error) { return tenc(`{"JB":%d}`, b) } + +type textbyte byte + +func (b textbyte) MarshalText() ([]byte, error) { return tenc(`TB:%d`, b) } + +type jsonint int + +func (i jsonint) MarshalJSON() ([]byte, error) { return tenc(`{"JI":%d}`, i) } + +type textint int + +func (i textint) MarshalText() ([]byte, error) { return tenc(`TI:%d`, i) } + +func tenc(format string, a ...interface{}) ([]byte, error) { + var buf bytes.Buffer + fmt.Fprintf(&buf, format, a...) + return buf.Bytes(), nil +} + +// Issue 13783 +func TestEncodeBytekind(t *testing.T) { + testdata := []struct { + data interface{} + want string + }{ + {byte(7), "7"}, + {jsonbyte(7), `{"JB":7}`}, + {textbyte(4), `"TB:4"`}, + {jsonint(5), `{"JI":5}`}, + {textint(1), `"TI:1"`}, + {[]byte{0, 1}, `"AAE="`}, + {[]jsonbyte{0, 1}, `[{"JB":0},{"JB":1}]`}, + {[][]jsonbyte{{0, 1}, {3}}, `[[{"JB":0},{"JB":1}],[{"JB":3}]]`}, + {[]textbyte{2, 3}, `["TB:2","TB:3"]`}, + {[]jsonint{5, 4}, `[{"JI":5},{"JI":4}]`}, + {[]textint{9, 3}, `["TI:9","TI:3"]`}, + {[]int{9, 3}, `[9,3]`}, + } + for _, d := range testdata { + js, err := Marshal(d.data) + if err != nil { + t.Error(err) + continue + } + got, want := string(js), d.want + if got != want { + t.Errorf("got %s, want %s", got, want) + } + } +} + +func TestTextMarshalerMapKeysAreSorted(t *testing.T) { + b, err := Marshal(map[unmarshalerText]int{ + {"x", "y"}: 1, + {"y", "x"}: 2, + {"a", "z"}: 3, + {"z", "a"}: 4, + }) + if err != nil { + t.Fatalf("Failed to Marshal text.Marshaler: %v", err) + } + const want = `{"a:z":3,"x:y":1,"y:x":2,"z:a":4}` + if string(b) != want { + t.Errorf("Marshal map with text.Marshaler keys: got %#q, want %#q", b, want) + } +} + +var re = regexp.MustCompile + +// syntactic checks on form of marshaled floating point numbers. +var badFloatREs = []*regexp.Regexp{ + re(`p`), // no binary exponential notation + re(`^\+`), // no leading + sign + re(`^-?0[^.]`), // no unnecessary leading zeros + re(`^-?\.`), // leading zero required before decimal point + re(`\.(e|$)`), // no trailing decimal + re(`\.[0-9]+0(e|$)`), // no trailing zero in fraction + re(`^-?(0|[0-9]{2,})\..*e`), // exponential notation must have normalized mantissa + re(`e[0-9]`), // positive exponent must be signed + re(`e[+-]0`), // exponent must not have leading zeros + re(`e-[1-6]$`), // not tiny enough for exponential notation + re(`e+(.|1.|20)$`), // not big enough for exponential notation + re(`^-?0\.0000000`), // too tiny, should use exponential notation + re(`^-?[0-9]{22}`), // too big, should use exponential notation + re(`[1-9][0-9]{16}[1-9]`), // too many significant digits in integer + re(`[1-9][0-9.]{17}[1-9]`), // too many significant digits in decimal + // below here for float32 only + re(`[1-9][0-9]{8}[1-9]`), // too many significant digits in integer + re(`[1-9][0-9.]{9}[1-9]`), // too many significant digits in decimal +} + +func TestMarshalFloat(t *testing.T) { + t.Parallel() + nfail := 0 + test := func(f float64, bits int) { + vf := interface{}(f) + if bits == 32 { + f = float64(float32(f)) // round + vf = float32(f) + } + bout, err := Marshal(vf) + if err != nil { + t.Errorf("Marshal(%T(%g)): %v", vf, vf, err) + nfail++ + return + } + out := string(bout) + + // result must convert back to the same float + g, err := strconv.ParseFloat(out, bits) + if err != nil { + t.Errorf("Marshal(%T(%g)) = %q, cannot parse back: %v", vf, vf, out, err) + nfail++ + return + } + if f != g || fmt.Sprint(f) != fmt.Sprint(g) { // fmt.Sprint handles ±0 + t.Errorf("Marshal(%T(%g)) = %q (is %g, not %g)", vf, vf, out, float32(g), vf) + nfail++ + return + } + + bad := badFloatREs + if bits == 64 { + bad = bad[:len(bad)-2] + } + for _, re := range bad { + if re.MatchString(out) { + t.Errorf("Marshal(%T(%g)) = %q, must not match /%s/", vf, vf, out, re) + nfail++ + return + } + } + } + + var ( + bigger = math.Inf(+1) + smaller = math.Inf(-1) + ) + + var digits = "1.2345678901234567890123" + for i := len(digits); i >= 2; i-- { + for exp := -30; exp <= 30; exp++ { + for _, sign := range "+-" { + for bits := 32; bits <= 64; bits += 32 { + s := fmt.Sprintf("%c%se%d", sign, digits[:i], exp) + f, err := strconv.ParseFloat(s, bits) + if err != nil { + log.Fatal(err) + } + next := math.Nextafter + if bits == 32 { + next = func(g, h float64) float64 { + return float64(math.Nextafter32(float32(g), float32(h))) + } + } + test(f, bits) + test(next(f, bigger), bits) + test(next(f, smaller), bits) + if nfail > 50 { + t.Fatalf("stopping test early") + } + } + } + } + } + test(0, 64) + test(math.Copysign(0, -1), 64) + test(0, 32) + test(math.Copysign(0, -1), 32) +} + +func TestMarshalRawMessageValue(t *testing.T) { + type ( + T1 struct { + M RawMessage `json:",omitempty"` + } + T2 struct { + M *RawMessage `json:",omitempty"` + } + ) + + var ( + rawNil = RawMessage(nil) + rawEmpty = RawMessage([]byte{}) + rawText = RawMessage([]byte(`"foo"`)) + ) + + tests := []struct { + in interface{} + want string + ok bool + }{ + // Test with nil RawMessage. + {rawNil, "null", true}, + {&rawNil, "null", true}, + {[]interface{}{rawNil}, "[null]", true}, + {&[]interface{}{rawNil}, "[null]", true}, + {[]interface{}{&rawNil}, "[null]", true}, + {&[]interface{}{&rawNil}, "[null]", true}, + {struct{ M RawMessage }{rawNil}, `{"M":null}`, true}, + {&struct{ M RawMessage }{rawNil}, `{"M":null}`, true}, + {struct{ M *RawMessage }{&rawNil}, `{"M":null}`, true}, + {&struct{ M *RawMessage }{&rawNil}, `{"M":null}`, true}, + {map[string]interface{}{"M": rawNil}, `{"M":null}`, true}, + {&map[string]interface{}{"M": rawNil}, `{"M":null}`, true}, + {map[string]interface{}{"M": &rawNil}, `{"M":null}`, true}, + {&map[string]interface{}{"M": &rawNil}, `{"M":null}`, true}, + {T1{rawNil}, "{}", true}, + {T2{&rawNil}, `{"M":null}`, true}, + {&T1{rawNil}, "{}", true}, + {&T2{&rawNil}, `{"M":null}`, true}, + + // Test with empty, but non-nil, RawMessage. + {rawEmpty, "", false}, + {&rawEmpty, "", false}, + {[]interface{}{rawEmpty}, "", false}, + {&[]interface{}{rawEmpty}, "", false}, + {[]interface{}{&rawEmpty}, "", false}, + {&[]interface{}{&rawEmpty}, "", false}, + {struct{ X RawMessage }{rawEmpty}, "", false}, + {&struct{ X RawMessage }{rawEmpty}, "", false}, + {struct{ X *RawMessage }{&rawEmpty}, "", false}, + {&struct{ X *RawMessage }{&rawEmpty}, "", false}, + {map[string]interface{}{"nil": rawEmpty}, "", false}, + {&map[string]interface{}{"nil": rawEmpty}, "", false}, + {map[string]interface{}{"nil": &rawEmpty}, "", false}, + {&map[string]interface{}{"nil": &rawEmpty}, "", false}, + {T1{rawEmpty}, "{}", true}, + {T2{&rawEmpty}, "", false}, + {&T1{rawEmpty}, "{}", true}, + {&T2{&rawEmpty}, "", false}, + + // Test with RawMessage with some text. + // + // The tests below marked with Issue6458 used to generate "ImZvbyI=" instead "foo". + // This behavior was intentionally changed in Go 1.8. + // See https://github.com/golang/go/issues/14493#issuecomment-255857318 + {rawText, `"foo"`, true}, // Issue6458 + {&rawText, `"foo"`, true}, + {[]interface{}{rawText}, `["foo"]`, true}, // Issue6458 + {&[]interface{}{rawText}, `["foo"]`, true}, // Issue6458 + {[]interface{}{&rawText}, `["foo"]`, true}, + {&[]interface{}{&rawText}, `["foo"]`, true}, + {struct{ M RawMessage }{rawText}, `{"M":"foo"}`, true}, // Issue6458 + {&struct{ M RawMessage }{rawText}, `{"M":"foo"}`, true}, + {struct{ M *RawMessage }{&rawText}, `{"M":"foo"}`, true}, + {&struct{ M *RawMessage }{&rawText}, `{"M":"foo"}`, true}, + {map[string]interface{}{"M": rawText}, `{"M":"foo"}`, true}, // Issue6458 + {&map[string]interface{}{"M": rawText}, `{"M":"foo"}`, true}, // Issue6458 + {map[string]interface{}{"M": &rawText}, `{"M":"foo"}`, true}, + {&map[string]interface{}{"M": &rawText}, `{"M":"foo"}`, true}, + {T1{rawText}, `{"M":"foo"}`, true}, // Issue6458 + {T2{&rawText}, `{"M":"foo"}`, true}, + {&T1{rawText}, `{"M":"foo"}`, true}, + {&T2{&rawText}, `{"M":"foo"}`, true}, + } + + for i, tt := range tests { + b, err := Marshal(tt.in) + if ok := (err == nil); ok != tt.ok { + if err != nil { + t.Errorf("test %d, unexpected failure: %v", i, err) + } else { + t.Errorf("test %d, unexpected success", i) + } + } + if got := string(b); got != tt.want { + t.Errorf("test %d, Marshal(%#v) = %q, want %q", i, tt.in, got, tt.want) + } + } +} diff --git a/libgo/go/encoding/json/example_marshaling_test.go b/libgo/go/encoding/json/example_marshaling_test.go new file mode 100644 index 0000000000..1c4f783a69 --- /dev/null +++ b/libgo/go/encoding/json/example_marshaling_test.go @@ -0,0 +1,75 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package json_test + +import ( + "encoding/json" + "fmt" + "log" + "strings" +) + +type Animal int + +const ( + Unknown Animal = iota + Gopher + Zebra +) + +func (a *Animal) UnmarshalJSON(b []byte) error { + var s string + if err := json.Unmarshal(b, &s); err != nil { + return err + } + switch strings.ToLower(s) { + default: + *a = Unknown + case "gopher": + *a = Gopher + case "zebra": + *a = Zebra + } + + return nil +} + +func (a Animal) MarshalJSON() ([]byte, error) { + var s string + switch a { + default: + s = "unknown" + case Gopher: + s = "gopher" + case Zebra: + s = "zebra" + } + + return json.Marshal(s) +} + +func Example_customMarshalJSON() { + blob := `["gopher","armadillo","zebra","unknown","gopher","bee","gopher","zebra"]` + var zoo []Animal + if err := json.Unmarshal([]byte(blob), &zoo); err != nil { + log.Fatal(err) + } + + census := make(map[Animal]int) + for _, animal := range zoo { + census[animal] += 1 + } + + fmt.Printf("Zoo Census:\n* Gophers: %d\n* Zebras: %d\n* Unknown: %d\n", + census[Gopher], census[Zebra], census[Unknown]) + + // Output: + // Zoo Census: + // * Gophers: 3 + // * Zebras: 2 + // * Unknown: 3 +} diff --git a/libgo/go/encoding/json/indent.go b/libgo/go/encoding/json/indent.go index 7cd9f4db18..fba19548c9 100644 --- a/libgo/go/encoding/json/indent.go +++ b/libgo/go/encoding/json/indent.go @@ -1,4 +1,4 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/encoding/json/number_test.go b/libgo/go/encoding/json/number_test.go index 4e63cf9c74..4b86999638 100644 --- a/libgo/go/encoding/json/number_test.go +++ b/libgo/go/encoding/json/number_test.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/encoding/json/scanner.go b/libgo/go/encoding/json/scanner.go index ee6622e8cf..a6d8706c73 100644 --- a/libgo/go/encoding/json/scanner.go +++ b/libgo/go/encoding/json/scanner.go @@ -1,4 +1,4 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -132,7 +132,7 @@ const ( // These values are stored in the parseState stack. // They give the current state of a composite value -// being scanned. If the parser is inside a nested value +// being scanned. If the parser is inside a nested value // the parseState describes the nested state, outermost at entry 0. const ( parseObjectKey = iota // parsing object key (before colon) diff --git a/libgo/go/encoding/json/scanner_test.go b/libgo/go/encoding/json/scanner_test.go index 66383ef0ef..c5c1be31f1 100644 --- a/libgo/go/encoding/json/scanner_test.go +++ b/libgo/go/encoding/json/scanner_test.go @@ -1,4 +1,4 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -119,6 +119,7 @@ func TestCompactBig(t *testing.T) { } func TestIndentBig(t *testing.T) { + t.Parallel() initBig() var buf bytes.Buffer if err := Indent(&buf, jsonBig, "", "\t"); err != nil { diff --git a/libgo/go/encoding/json/stream.go b/libgo/go/encoding/json/stream.go index 8ddcf4d279..95e30ce36d 100644 --- a/libgo/go/encoding/json/stream.go +++ b/libgo/go/encoding/json/stream.go @@ -1,4 +1,4 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -10,7 +10,7 @@ import ( "io" ) -// A Decoder reads and decodes JSON objects from an input stream. +// A Decoder reads and decodes JSON values from an input stream. type Decoder struct { r io.Reader buf []byte @@ -148,7 +148,7 @@ func (dec *Decoder) refill() error { dec.buf = newBuf } - // Read. Delay error for next iteration (after scan). + // Read. Delay error for next iteration (after scan). n, err := dec.r.Read(dec.buf[len(dec.buf):cap(dec.buf)]) dec.buf = dec.buf[0 : len(dec.buf)+n] @@ -164,15 +164,20 @@ func nonSpace(b []byte) bool { return false } -// An Encoder writes JSON objects to an output stream. +// An Encoder writes JSON values to an output stream. type Encoder struct { - w io.Writer - err error + w io.Writer + err error + escapeHTML bool + + indentBuf *bytes.Buffer + indentPrefix string + indentValue string } // NewEncoder returns a new encoder that writes to w. func NewEncoder(w io.Writer) *Encoder { - return &Encoder{w: w} + return &Encoder{w: w, escapeHTML: true} } // Encode writes the JSON encoding of v to the stream, @@ -185,7 +190,7 @@ func (enc *Encoder) Encode(v interface{}) error { return enc.err } e := newEncodeState() - err := e.marshal(v) + err := e.marshal(v, encOpts{escapeHTML: enc.escapeHTML}) if err != nil { return err } @@ -198,21 +203,55 @@ func (enc *Encoder) Encode(v interface{}) error { // digits coming. e.WriteByte('\n') - if _, err = enc.w.Write(e.Bytes()); err != nil { + b := e.Bytes() + if enc.indentPrefix != "" || enc.indentValue != "" { + if enc.indentBuf == nil { + enc.indentBuf = new(bytes.Buffer) + } + enc.indentBuf.Reset() + err = Indent(enc.indentBuf, b, enc.indentPrefix, enc.indentValue) + if err != nil { + return err + } + b = enc.indentBuf.Bytes() + } + if _, err = enc.w.Write(b); err != nil { enc.err = err } encodeStatePool.Put(e) return err } -// RawMessage is a raw encoded JSON object. +// SetIndent instructs the encoder to format each subsequent encoded +// value as if indented by the package-level function Indent(dst, src, prefix, indent). +// Calling SetIndent("", "") disables indentation. +func (enc *Encoder) SetIndent(prefix, indent string) { + enc.indentPrefix = prefix + enc.indentValue = indent +} + +// SetEscapeHTML specifies whether problematic HTML characters +// should be escaped inside JSON quoted strings. +// The default behavior is to escape &, <, and > to \u0026, \u003c, and \u003e +// to avoid certain safety problems that can arise when embedding JSON in HTML. +// +// In non-HTML settings where the escaping interferes with the readability +// of the output, SetEscapeHTML(false) disables this behavior. +func (enc *Encoder) SetEscapeHTML(on bool) { + enc.escapeHTML = on +} + +// RawMessage is a raw encoded JSON value. // It implements Marshaler and Unmarshaler and can // be used to delay JSON decoding or precompute a JSON encoding. type RawMessage []byte -// MarshalJSON returns *m as the JSON encoding of m. -func (m *RawMessage) MarshalJSON() ([]byte, error) { - return *m, nil +// MarshalJSON returns m as the JSON encoding of m. +func (m RawMessage) MarshalJSON() ([]byte, error) { + if m == nil { + return []byte("null"), nil + } + return m, nil } // UnmarshalJSON sets *m to a copy of data. diff --git a/libgo/go/encoding/json/stream_test.go b/libgo/go/encoding/json/stream_test.go index c2e30408cd..84edeb187c 100644 --- a/libgo/go/encoding/json/stream_test.go +++ b/libgo/go/encoding/json/stream_test.go @@ -1,4 +1,4 @@ -// Copyright 2010 The Go Authors. All rights reserved. +// Copyright 2010 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -44,6 +44,9 @@ func TestEncoder(t *testing.T) { for i := 0; i <= len(streamTest); i++ { var buf bytes.Buffer enc := NewEncoder(&buf) + // Check that enc.SetIndent("", "") turns off indentation. + enc.SetIndent(">", ".") + enc.SetIndent("", "") for j, v := range streamTest[0:i] { if err := enc.Encode(v); err != nil { t.Fatalf("encode #%d: %v", j, err) @@ -57,6 +60,69 @@ func TestEncoder(t *testing.T) { } } +var streamEncodedIndent = `0.1 +"hello" +null +true +false +[ +>."a", +>."b", +>."c" +>] +{ +>."ß": "long s", +>."K": "Kelvin" +>} +3.14 +` + +func TestEncoderIndent(t *testing.T) { + var buf bytes.Buffer + enc := NewEncoder(&buf) + enc.SetIndent(">", ".") + for _, v := range streamTest { + enc.Encode(v) + } + if have, want := buf.String(), streamEncodedIndent; have != want { + t.Error("indented encoding mismatch") + diff(t, []byte(have), []byte(want)) + } +} + +func TestEncoderSetEscapeHTML(t *testing.T) { + var c C + var ct CText + for _, tt := range []struct { + name string + v interface{} + wantEscape string + want string + }{ + {"c", c, `"\u003c\u0026\u003e"`, `"<&>"`}, + {"ct", ct, `"\"\u003c\u0026\u003e\""`, `"\"<&>\""`}, + {`"<&>"`, "<&>", `"\u003c\u0026\u003e"`, `"<&>"`}, + } { + var buf bytes.Buffer + enc := NewEncoder(&buf) + if err := enc.Encode(tt.v); err != nil { + t.Fatalf("Encode(%s): %s", tt.name, err) + } + if got := strings.TrimSpace(buf.String()); got != tt.wantEscape { + t.Errorf("Encode(%s) = %#q, want %#q", tt.name, got, tt.wantEscape) + } + buf.Reset() + enc.SetEscapeHTML(false) + if err := enc.Encode(tt.v); err != nil { + t.Fatalf("SetEscapeHTML(false) Encode(%s): %s", tt.name, err) + } + if got := strings.TrimSpace(buf.String()); got != tt.want { + t.Errorf("SetEscapeHTML(false) Encode(%s) = %#q, want %#q", + tt.name, got, tt.want) + } + } +} + func TestDecoder(t *testing.T) { for i := 0; i <= len(streamTest); i++ { // Use stream without newlines as input, diff --git a/libgo/go/encoding/json/tables.go b/libgo/go/encoding/json/tables.go new file mode 100644 index 0000000000..10acdc18c6 --- /dev/null +++ b/libgo/go/encoding/json/tables.go @@ -0,0 +1,218 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package json + +import "unicode/utf8" + +// safeSet holds the value true if the ASCII character with the given array +// position can be represented inside a JSON string without any further +// escaping. +// +// All values are true except for the ASCII control characters (0-31), the +// double quote ("), and the backslash character ("\"). +var safeSet = [utf8.RuneSelf]bool{ + ' ': true, + '!': true, + '"': false, + '#': true, + '$': true, + '%': true, + '&': true, + '\'': true, + '(': true, + ')': true, + '*': true, + '+': true, + ',': true, + '-': true, + '.': true, + '/': true, + '0': true, + '1': true, + '2': true, + '3': true, + '4': true, + '5': true, + '6': true, + '7': true, + '8': true, + '9': true, + ':': true, + ';': true, + '<': true, + '=': true, + '>': true, + '?': true, + '@': true, + 'A': true, + 'B': true, + 'C': true, + 'D': true, + 'E': true, + 'F': true, + 'G': true, + 'H': true, + 'I': true, + 'J': true, + 'K': true, + 'L': true, + 'M': true, + 'N': true, + 'O': true, + 'P': true, + 'Q': true, + 'R': true, + 'S': true, + 'T': true, + 'U': true, + 'V': true, + 'W': true, + 'X': true, + 'Y': true, + 'Z': true, + '[': true, + '\\': false, + ']': true, + '^': true, + '_': true, + '`': true, + 'a': true, + 'b': true, + 'c': true, + 'd': true, + 'e': true, + 'f': true, + 'g': true, + 'h': true, + 'i': true, + 'j': true, + 'k': true, + 'l': true, + 'm': true, + 'n': true, + 'o': true, + 'p': true, + 'q': true, + 'r': true, + 's': true, + 't': true, + 'u': true, + 'v': true, + 'w': true, + 'x': true, + 'y': true, + 'z': true, + '{': true, + '|': true, + '}': true, + '~': true, + '\u007f': true, +} + +// htmlSafeSet holds the value true if the ASCII character with the given +// array position can be safely represented inside a JSON string, embedded +// inside of HTML <script> tags, without any additional escaping. +// +// All values are true except for the ASCII control characters (0-31), the +// double quote ("), the backslash character ("\"), HTML opening and closing +// tags ("<" and ">"), and the ampersand ("&"). +var htmlSafeSet = [utf8.RuneSelf]bool{ + ' ': true, + '!': true, + '"': false, + '#': true, + '$': true, + '%': true, + '&': false, + '\'': true, + '(': true, + ')': true, + '*': true, + '+': true, + ',': true, + '-': true, + '.': true, + '/': true, + '0': true, + '1': true, + '2': true, + '3': true, + '4': true, + '5': true, + '6': true, + '7': true, + '8': true, + '9': true, + ':': true, + ';': true, + '<': false, + '=': true, + '>': false, + '?': true, + '@': true, + 'A': true, + 'B': true, + 'C': true, + 'D': true, + 'E': true, + 'F': true, + 'G': true, + 'H': true, + 'I': true, + 'J': true, + 'K': true, + 'L': true, + 'M': true, + 'N': true, + 'O': true, + 'P': true, + 'Q': true, + 'R': true, + 'S': true, + 'T': true, + 'U': true, + 'V': true, + 'W': true, + 'X': true, + 'Y': true, + 'Z': true, + '[': true, + '\\': false, + ']': true, + '^': true, + '_': true, + '`': true, + 'a': true, + 'b': true, + 'c': true, + 'd': true, + 'e': true, + 'f': true, + 'g': true, + 'h': true, + 'i': true, + 'j': true, + 'k': true, + 'l': true, + 'm': true, + 'n': true, + 'o': true, + 'p': true, + 'q': true, + 'r': true, + 's': true, + 't': true, + 'u': true, + 'v': true, + 'w': true, + 'x': true, + 'y': true, + 'z': true, + '{': true, + '|': true, + '}': true, + '~': true, + '\u007f': true, +} diff --git a/libgo/go/encoding/json/tagkey_test.go b/libgo/go/encoding/json/tagkey_test.go index 85bb4ba837..f77c49c764 100644 --- a/libgo/go/encoding/json/tagkey_test.go +++ b/libgo/go/encoding/json/tagkey_test.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -44,6 +44,10 @@ type punctuationTag struct { V string `json:"!#$%&()*+-./:<=>?@[]^_{|}~"` // https://golang.org/issue/3546 } +type dashTag struct { + V string `json:"-,"` +} + type emptyTag struct { W string } @@ -80,6 +84,7 @@ var structTagObjectKeyTests = []struct { {basicLatin6xTag{"6x"}, "6x", "abcdefghijklmno"}, {basicLatin7xTag{"7x"}, "7x", "pqrstuvwxyz"}, {miscPlaneTag{"いろはにほへと"}, "いろはにほへと", "色は匂へど"}, + {dashTag{"foo"}, "foo", "-"}, {emptyTag{"Pour Moi"}, "Pour Moi", "W"}, {misnamedTag{"Animal Kingdom"}, "Animal Kingdom", "X"}, {badFormatTag{"Orfevre"}, "Orfevre", "Y"}, diff --git a/libgo/go/encoding/json/tags_test.go b/libgo/go/encoding/json/tags_test.go index 91fb18831e..8ba8ddd5f8 100644 --- a/libgo/go/encoding/json/tags_test.go +++ b/libgo/go/encoding/json/tags_test.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/libgo/go/encoding/json/testdata/code.json.gz b/libgo/go/encoding/json/testdata/code.json.gz Binary files differindex 0e2895b53a..1572a92bfb 100644 --- a/libgo/go/encoding/json/testdata/code.json.gz +++ b/libgo/go/encoding/json/testdata/code.json.gz diff --git a/libgo/go/encoding/pem/example_test.go b/libgo/go/encoding/pem/example_test.go new file mode 100644 index 0000000000..ffd962bd68 --- /dev/null +++ b/libgo/go/encoding/pem/example_test.go @@ -0,0 +1,46 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package pem_test + +import ( + "crypto/x509" + "encoding/pem" + "fmt" + "log" +) + +func ExampleDecode() { + var pubPEMData = []byte(` +-----BEGIN PUBLIC KEY----- +MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAlRuRnThUjU8/prwYxbty +WPT9pURI3lbsKMiB6Fn/VHOKE13p4D8xgOCADpdRagdT6n4etr9atzDKUSvpMtR3 +CP5noNc97WiNCggBjVWhs7szEe8ugyqF23XwpHQ6uV1LKH50m92MbOWfCtjU9p/x +qhNpQQ1AZhqNy5Gevap5k8XzRmjSldNAFZMY7Yv3Gi+nyCwGwpVtBUwhuLzgNFK/ +yDtw2WcWmUU7NuC8Q6MWvPebxVtCfVp/iQU6q60yyt6aGOBkhAX0LpKAEhKidixY +nP9PNVBvxgu3XZ4P36gZV6+ummKdBVnc3NqwBLu5+CcdRdusmHPHd5pHf4/38Z3/ +6qU2a/fPvWzceVTEgZ47QjFMTCTmCwNt29cvi7zZeQzjtwQgn4ipN9NibRH/Ax/q +TbIzHfrJ1xa2RteWSdFjwtxi9C20HUkjXSeI4YlzQMH0fPX6KCE7aVePTOnB69I/ +a9/q96DiXZajwlpq3wFctrs1oXqBp5DVrCIj8hU2wNgB7LtQ1mCtsYz//heai0K9 +PhE4X6hiE0YmeAZjR0uHl8M/5aW9xCoJ72+12kKpWAa0SFRWLy6FejNYCYpkupVJ +yecLk/4L1W0l6jQQZnWErXZYe0PNFcmwGXy1Rep83kfBRNKRy5tvocalLlwXLdUk +AIU+2GKjyT3iMuzZxxFxPFMCAwEAAQ== +-----END PUBLIC KEY----- +and some more`) + + block, rest := pem.Decode(pubPEMData) + if block == nil || block.Type != "PUBLIC KEY" { + log.Fatal("failed to decode PEM block containing public key") + } + + pub, err := x509.ParsePKIXPublicKey(block.Bytes) + if err != nil { + log.Fatal(err) + } + + fmt.Printf("Got a %T, with remaining data: %q", pub, rest) + // Output: Got a *rsa.PublicKey, with remaining data: "and some more" +} diff --git a/libgo/go/encoding/pem/pem.go b/libgo/go/encoding/pem/pem.go index 506196b1db..fbf49997d5 100644 --- a/libgo/go/encoding/pem/pem.go +++ b/libgo/go/encoding/pem/pem.go @@ -119,19 +119,36 @@ func Decode(data []byte) (p *Block, rest []byte) { rest = next } - var endIndex int + var endIndex, endTrailerIndex int + // If there were no headers, the END line might occur // immediately, without a leading newline. if len(p.Headers) == 0 && bytes.HasPrefix(rest, pemEnd[1:]) { endIndex = 0 + endTrailerIndex = len(pemEnd) - 1 } else { endIndex = bytes.Index(rest, pemEnd) + endTrailerIndex = endIndex + len(pemEnd) } if endIndex < 0 { return decodeError(data, rest) } + // After the "-----" of the ending line should be the same type and a + // final five dashes. + endTrailer := rest[endTrailerIndex:] + endTrailerLen := len(typeLine) + len(pemEndOfLine) + if len(endTrailer) < endTrailerLen { + return decodeError(data, rest) + } + + endTrailer = endTrailer[:endTrailerLen] + if !bytes.HasPrefix(endTrailer, typeLine) || + !bytes.HasSuffix(endTrailer, pemEndOfLine) { + return decodeError(data, rest) + } + base64Data := removeWhitespace(rest[:endIndex]) p.Bytes = make([]byte, base64.StdEncoding.DecodedLen(len(base64Data))) n, err := base64.StdEncoding.Decode(p.Bytes, base64Data) @@ -150,7 +167,7 @@ func Decode(data []byte) (p *Block, rest []byte) { func decodeError(data, rest []byte) (*Block, []byte) { // If we get here then we have rejected a likely looking, but // ultimately invalid PEM block. We need to start over from a new - // position. We have consumed the preamble line and will have consumed + // position. We have consumed the preamble line and will have consumed // any lines which could be header lines. However, a valid preamble // line is not a valid header line, therefore we cannot have consumed // the preamble line for the any subsequent block. Thus, we will always diff --git a/libgo/go/encoding/pem/pem_test.go b/libgo/go/encoding/pem/pem_test.go index 958dbc1a3a..6321dec382 100644 --- a/libgo/go/encoding/pem/pem_test.go +++ b/libgo/go/encoding/pem/pem_test.go @@ -78,6 +78,48 @@ func TestDecode(t *testing.T) { } } +const pemTooFewEndingDashes = ` +-----BEGIN FOO----- +dGVzdA== +-----END FOO----` + +const pemWrongEndingType = ` +-----BEGIN FOO----- +dGVzdA== +-----END BAR-----` + +const pemMissingEndingSpace = ` +-----BEGIN FOO----- +dGVzdA== +-----ENDBAR-----` + +var badPEMTests = []struct { + name string + input string +}{ + { + "too few trailing dashes", + pemTooFewEndingDashes, + }, + { + "incorrect ending type", + pemWrongEndingType, + }, + { + "missing ending space", + pemMissingEndingSpace, + }, +} + +func TestBadDecode(t *testing.T) { + for _, test := range badPEMTests { + result, _ := Decode([]byte(test.input)) + if result != nil { + t.Errorf("unexpected success while parsing %q", test.name) + } + } +} + func TestEncode(t *testing.T) { r := EncodeToMemory(privateKey2) if string(r) != pemPrivateKey2 { diff --git a/libgo/go/encoding/xml/marshal.go b/libgo/go/encoding/xml/marshal.go index 8ebd693030..4c6ba8c1a5 100644 --- a/libgo/go/encoding/xml/marshal.go +++ b/libgo/go/encoding/xml/marshal.go @@ -24,21 +24,21 @@ const ( // Marshal returns the XML encoding of v. // -// Marshal handles an array or slice by marshalling each of the elements. -// Marshal handles a pointer by marshalling the value it points at or, if the -// pointer is nil, by writing nothing. Marshal handles an interface value by -// marshalling the value it contains or, if the interface value is nil, by -// writing nothing. Marshal handles all other data by writing one or more XML +// Marshal handles an array or slice by marshaling each of the elements. +// Marshal handles a pointer by marshaling the value it points at or, if the +// pointer is nil, by writing nothing. Marshal handles an interface value by +// marshaling the value it contains or, if the interface value is nil, by +// writing nothing. Marshal handles all other data by writing one or more XML // elements containing the data. // // The name for the XML elements is taken from, in order of preference: // - the tag on the XMLName field, if the data is a struct -// - the value of the XMLName field of type xml.Name +// - the value of the XMLName field of type Name // - the tag of the struct field used to obtain the data // - the name of the struct field used to obtain the data -// - the name of the marshalled type +// - the name of the marshaled type // -// The XML element for a struct contains marshalled elements for each of the +// The XML element for a struct contains marshaled elements for each of the // exported fields of the struct, with these exceptions: // - the XMLName field, described above, is omitted. // - a field with tag "-" is omitted. @@ -51,9 +51,9 @@ const ( // - a field with tag ",cdata" is written as character data // wrapped in one or more <![CDATA[ ... ]]> tags, not as an XML element. // - a field with tag ",innerxml" is written verbatim, not subject -// to the usual marshalling procedure. +// to the usual marshaling procedure. // - a field with tag ",comment" is written as an XML comment, not -// subject to the usual marshalling procedure. It must not contain +// subject to the usual marshaling procedure. It must not contain // the "--" string within it. // - a field with a tag including the "omitempty" option is omitted // if the field value is empty. The empty values are false, 0, any @@ -63,7 +63,7 @@ const ( // value were part of the outer struct. // // If a field uses a tag "a>b>c", then the element c will be nested inside -// parent elements a and b. Fields that appear next to each other that name +// parent elements a and b. Fields that appear next to each other that name // the same parent will be enclosed in one XML element. // // See MarshalIndent for an example. @@ -175,10 +175,9 @@ func (enc *Encoder) EncodeElement(v interface{}, start StartElement) error { } var ( - begComment = []byte("<!--") - endComment = []byte("-->") - endProcInst = []byte("?>") - endDirective = []byte(">") + begComment = []byte("<!--") + endComment = []byte("-->") + endProcInst = []byte("?>") ) // EncodeToken writes the given XML token to the stream. @@ -217,7 +216,7 @@ func (enc *Encoder) EncodeToken(t Token) error { return p.cachedWriteError() case ProcInst: // First token to be encoded which is also a ProcInst with target of xml - // is the xml declaration. The only ProcInst where target of xml is allowed. + // is the xml declaration. The only ProcInst where target of xml is allowed. if t.Target == "xml" && p.Buffered() != 0 { return fmt.Errorf("xml: EncodeToken of ProcInst xml target only valid for xml declaration, first token encoded") } @@ -495,7 +494,6 @@ func (p *printer) marshalValue(val reflect.Value, finfo *fieldInfo, startTemplat continue } fv := finfo.value(val) - name := Name{Space: finfo.xmlns, Local: finfo.name} if finfo.flags&fOmitEmpty != 0 && isEmptyValue(fv) { continue @@ -505,69 +503,10 @@ func (p *printer) marshalValue(val reflect.Value, finfo *fieldInfo, startTemplat continue } - if fv.CanInterface() && fv.Type().Implements(marshalerAttrType) { - attr, err := fv.Interface().(MarshalerAttr).MarshalXMLAttr(name) - if err != nil { - return err - } - if attr.Name.Local != "" { - start.Attr = append(start.Attr, attr) - } - continue - } - - if fv.CanAddr() { - pv := fv.Addr() - if pv.CanInterface() && pv.Type().Implements(marshalerAttrType) { - attr, err := pv.Interface().(MarshalerAttr).MarshalXMLAttr(name) - if err != nil { - return err - } - if attr.Name.Local != "" { - start.Attr = append(start.Attr, attr) - } - continue - } - } - - if fv.CanInterface() && fv.Type().Implements(textMarshalerType) { - text, err := fv.Interface().(encoding.TextMarshaler).MarshalText() - if err != nil { - return err - } - start.Attr = append(start.Attr, Attr{name, string(text)}) - continue - } - - if fv.CanAddr() { - pv := fv.Addr() - if pv.CanInterface() && pv.Type().Implements(textMarshalerType) { - text, err := pv.Interface().(encoding.TextMarshaler).MarshalText() - if err != nil { - return err - } - start.Attr = append(start.Attr, Attr{name, string(text)}) - continue - } - } - - // Dereference or skip nil pointer, interface values. - switch fv.Kind() { - case reflect.Ptr, reflect.Interface: - if fv.IsNil() { - continue - } - fv = fv.Elem() - } - - s, b, err := p.marshalSimple(fv.Type(), fv) - if err != nil { + name := Name{Space: finfo.xmlns, Local: finfo.name} + if err := p.marshalAttr(&start, name, fv); err != nil { return err } - if b != nil { - s = string(b) - } - start.Attr = append(start.Attr, Attr{name, s}) } if err := p.writeStart(&start); err != nil { @@ -597,6 +536,90 @@ func (p *printer) marshalValue(val reflect.Value, finfo *fieldInfo, startTemplat return p.cachedWriteError() } +// marshalAttr marshals an attribute with the given name and value, adding to start.Attr. +func (p *printer) marshalAttr(start *StartElement, name Name, val reflect.Value) error { + if val.CanInterface() && val.Type().Implements(marshalerAttrType) { + attr, err := val.Interface().(MarshalerAttr).MarshalXMLAttr(name) + if err != nil { + return err + } + if attr.Name.Local != "" { + start.Attr = append(start.Attr, attr) + } + return nil + } + + if val.CanAddr() { + pv := val.Addr() + if pv.CanInterface() && pv.Type().Implements(marshalerAttrType) { + attr, err := pv.Interface().(MarshalerAttr).MarshalXMLAttr(name) + if err != nil { + return err + } + if attr.Name.Local != "" { + start.Attr = append(start.Attr, attr) + } + return nil + } + } + + if val.CanInterface() && val.Type().Implements(textMarshalerType) { + text, err := val.Interface().(encoding.TextMarshaler).MarshalText() + if err != nil { + return err + } + start.Attr = append(start.Attr, Attr{name, string(text)}) + return nil + } + + if val.CanAddr() { + pv := val.Addr() + if pv.CanInterface() && pv.Type().Implements(textMarshalerType) { + text, err := pv.Interface().(encoding.TextMarshaler).MarshalText() + if err != nil { + return err + } + start.Attr = append(start.Attr, Attr{name, string(text)}) + return nil + } + } + + // Dereference or skip nil pointer, interface values. + switch val.Kind() { + case reflect.Ptr, reflect.Interface: + if val.IsNil() { + return nil + } + val = val.Elem() + } + + // Walk slices. + if val.Kind() == reflect.Slice && val.Type().Elem().Kind() != reflect.Uint8 { + n := val.Len() + for i := 0; i < n; i++ { + if err := p.marshalAttr(start, name, val.Index(i)); err != nil { + return err + } + } + return nil + } + + if val.Type() == attrType { + start.Attr = append(start.Attr, val.Interface().(Attr)) + return nil + } + + s, b, err := p.marshalSimple(val.Type(), val) + if err != nil { + return err + } + if b != nil { + s = string(b) + } + start.Attr = append(start.Attr, Attr{name, s}) + return nil +} + // defaultStart returns the default start element to use, // given the reflect type, field info, and start template. func defaultStart(typ reflect.Type, finfo *fieldInfo, startTemplate *StartElement) StartElement { @@ -752,6 +775,20 @@ func (p *printer) marshalSimple(typ reflect.Type, val reflect.Value) (string, [] var ddBytes = []byte("--") +// indirect drills into interfaces and pointers, returning the pointed-at value. +// If it encounters a nil interface or pointer, indirect returns that nil value. +// This can turn into an infinite loop given a cyclic chain, +// but it matches the Go 1 behavior. +func indirect(vf reflect.Value) reflect.Value { + for vf.Kind() == reflect.Interface || vf.Kind() == reflect.Ptr { + if vf.IsNil() { + return vf + } + vf = vf.Elem() + } + return vf +} + func (p *printer) marshalStruct(tinfo *typeInfo, val reflect.Value) error { s := parentStack{p: p} for i := range tinfo.fields { @@ -761,14 +798,6 @@ func (p *printer) marshalStruct(tinfo *typeInfo, val reflect.Value) error { } vf := finfo.value(val) - // Dereference or skip nil pointer, interface values. - switch vf.Kind() { - case reflect.Ptr, reflect.Interface: - if !vf.IsNil() { - vf = vf.Elem() - } - } - switch finfo.flags & fMode { case fCDATA, fCharData: emit := EscapeText @@ -801,7 +830,9 @@ func (p *printer) marshalStruct(tinfo *typeInfo, val reflect.Value) error { continue } } + var scratch [64]byte + vf = indirect(vf) switch vf.Kind() { case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: if err := emit(p, strconv.AppendInt(scratch[:0], vf.Int(), 10)); err != nil { @@ -836,6 +867,7 @@ func (p *printer) marshalStruct(tinfo *typeInfo, val reflect.Value) error { if err := s.trim(finfo.parents); err != nil { return err } + vf = indirect(vf) k := vf.Kind() if !(k == reflect.String || k == reflect.Slice && vf.Type().Elem().Kind() == reflect.Uint8) { return fmt.Errorf("xml: bad type for comment field of %s", val.Type()) @@ -850,14 +882,14 @@ func (p *printer) marshalStruct(tinfo *typeInfo, val reflect.Value) error { switch k { case reflect.String: s := vf.String() - dashDash = strings.Index(s, "--") >= 0 + dashDash = strings.Contains(s, "--") dashLast = s[len(s)-1] == '-' if !dashDash { p.WriteString(s) } case reflect.Slice: b := vf.Bytes() - dashDash = bytes.Index(b, ddBytes) >= 0 + dashDash = bytes.Contains(b, ddBytes) dashLast = b[len(b)-1] == '-' if !dashDash { p.Write(b) @@ -876,6 +908,7 @@ func (p *printer) marshalStruct(tinfo *typeInfo, val reflect.Value) error { continue case fInnerXml: + vf = indirect(vf) iface := vf.Interface() switch raw := iface.(type) { case []byte: @@ -949,8 +982,8 @@ type parentStack struct { } // trim updates the XML context to match the longest common prefix of the stack -// and the given parents. A closing tag will be written for every parent -// popped. Passing a zero slice or nil will close all the elements. +// and the given parents. A closing tag will be written for every parent +// popped. Passing a zero slice or nil will close all the elements. func (s *parentStack) trim(parents []string) error { split := 0 for ; split < len(parents) && split < len(s.stack); split++ { diff --git a/libgo/go/encoding/xml/marshal_test.go b/libgo/go/encoding/xml/marshal_test.go index fe8b16fe43..0126146d33 100644 --- a/libgo/go/encoding/xml/marshal_test.go +++ b/libgo/go/encoding/xml/marshal_test.go @@ -199,6 +199,17 @@ type AttrTest struct { Bytes []byte `xml:",attr"` } +type AttrsTest struct { + Attrs []Attr `xml:",any,attr"` + Int int `xml:",attr"` + Named int `xml:"int,attr"` + Float float64 `xml:",attr"` + Uint8 uint8 `xml:",attr"` + Bool bool `xml:",attr"` + Str string `xml:",attr"` + Bytes []byte `xml:",attr"` +} + type OmitAttrTest struct { Int int `xml:",attr,omitempty"` Named int `xml:"int,attr,omitempty"` @@ -207,6 +218,7 @@ type OmitAttrTest struct { Bool bool `xml:",attr,omitempty"` Str string `xml:",attr,omitempty"` Bytes []byte `xml:",attr,omitempty"` + PStr *string `xml:",attr,omitempty"` } type OmitFieldTest struct { @@ -217,6 +229,7 @@ type OmitFieldTest struct { Bool bool `xml:",omitempty"` Str string `xml:",omitempty"` Bytes []byte `xml:",omitempty"` + PStr *string `xml:",omitempty"` Ptr *PresenceTest `xml:",omitempty"` } @@ -317,6 +330,10 @@ func (m *MyMarshalerAttrTest) MarshalXMLAttr(name Name) (Attr, error) { return Attr{name, "hello world"}, nil } +func (m *MyMarshalerAttrTest) UnmarshalXMLAttr(attr Attr) error { + return nil +} + type MarshalerStruct struct { Foo MyMarshalerAttrTest `xml:",attr"` } @@ -369,21 +386,158 @@ func ifaceptr(x interface{}) interface{} { return &x } +func stringptr(x string) *string { + return &x +} + +type T1 struct{} +type T2 struct{} +type T3 struct{} + +type IndirComment struct { + T1 T1 + Comment *string `xml:",comment"` + T2 T2 +} + +type DirectComment struct { + T1 T1 + Comment string `xml:",comment"` + T2 T2 +} + +type IfaceComment struct { + T1 T1 + Comment interface{} `xml:",comment"` + T2 T2 +} + +type IndirChardata struct { + T1 T1 + Chardata *string `xml:",chardata"` + T2 T2 +} + +type DirectChardata struct { + T1 T1 + Chardata string `xml:",chardata"` + T2 T2 +} + +type IfaceChardata struct { + T1 T1 + Chardata interface{} `xml:",chardata"` + T2 T2 +} + +type IndirCDATA struct { + T1 T1 + CDATA *string `xml:",cdata"` + T2 T2 +} + +type DirectCDATA struct { + T1 T1 + CDATA string `xml:",cdata"` + T2 T2 +} + +type IfaceCDATA struct { + T1 T1 + CDATA interface{} `xml:",cdata"` + T2 T2 +} + +type IndirInnerXML struct { + T1 T1 + InnerXML *string `xml:",innerxml"` + T2 T2 +} + +type DirectInnerXML struct { + T1 T1 + InnerXML string `xml:",innerxml"` + T2 T2 +} + +type IfaceInnerXML struct { + T1 T1 + InnerXML interface{} `xml:",innerxml"` + T2 T2 +} + +type IndirElement struct { + T1 T1 + Element *string + T2 T2 +} + +type DirectElement struct { + T1 T1 + Element string + T2 T2 +} + +type IfaceElement struct { + T1 T1 + Element interface{} + T2 T2 +} + +type IndirOmitEmpty struct { + T1 T1 + OmitEmpty *string `xml:",omitempty"` + T2 T2 +} + +type DirectOmitEmpty struct { + T1 T1 + OmitEmpty string `xml:",omitempty"` + T2 T2 +} + +type IfaceOmitEmpty struct { + T1 T1 + OmitEmpty interface{} `xml:",omitempty"` + T2 T2 +} + +type IndirAny struct { + T1 T1 + Any *string `xml:",any"` + T2 T2 +} + +type DirectAny struct { + T1 T1 + Any string `xml:",any"` + T2 T2 +} + +type IfaceAny struct { + T1 T1 + Any interface{} `xml:",any"` + T2 T2 +} + var ( nameAttr = "Sarah" ageAttr = uint(12) contentsAttr = "lorem ipsum" + empty = "" ) // Unless explicitly stated as such (or *Plain), all of the // tests below are two-way tests. When introducing new tests, // please try to make them two-way as well to ensure that -// marshalling and unmarshalling are as symmetrical as feasible. +// marshaling and unmarshaling are as symmetrical as feasible. var marshalTests = []struct { - Value interface{} - ExpectXML string - MarshalOnly bool - UnmarshalOnly bool + Value interface{} + ExpectXML string + MarshalOnly bool + MarshalError string + UnmarshalOnly bool + UnmarshalError string }{ // Test nil marshals to nothing {Value: nil, ExpectXML: ``, MarshalOnly: true}, @@ -823,6 +977,53 @@ var marshalTests = []struct { ` Bool="false" Str="" Bytes=""></AttrTest>`, }, { + Value: &AttrsTest{ + Attrs: []Attr{ + {Name: Name{Local: "Answer"}, Value: "42"}, + {Name: Name{Local: "Int"}, Value: "8"}, + {Name: Name{Local: "int"}, Value: "9"}, + {Name: Name{Local: "Float"}, Value: "23.5"}, + {Name: Name{Local: "Uint8"}, Value: "255"}, + {Name: Name{Local: "Bool"}, Value: "true"}, + {Name: Name{Local: "Str"}, Value: "str"}, + {Name: Name{Local: "Bytes"}, Value: "byt"}, + }, + }, + ExpectXML: `<AttrsTest Answer="42" Int="8" int="9" Float="23.5" Uint8="255" Bool="true" Str="str" Bytes="byt" Int="0" int="0" Float="0" Uint8="0" Bool="false" Str="" Bytes=""></AttrsTest>`, + MarshalOnly: true, + }, + { + Value: &AttrsTest{ + Attrs: []Attr{ + {Name: Name{Local: "Answer"}, Value: "42"}, + }, + Int: 8, + Named: 9, + Float: 23.5, + Uint8: 255, + Bool: true, + Str: "str", + Bytes: []byte("byt"), + }, + ExpectXML: `<AttrsTest Answer="42" Int="8" int="9" Float="23.5" Uint8="255" Bool="true" Str="str" Bytes="byt"></AttrsTest>`, + }, + { + Value: &AttrsTest{ + Attrs: []Attr{ + {Name: Name{Local: "Int"}, Value: "0"}, + {Name: Name{Local: "int"}, Value: "0"}, + {Name: Name{Local: "Float"}, Value: "0"}, + {Name: Name{Local: "Uint8"}, Value: "0"}, + {Name: Name{Local: "Bool"}, Value: "false"}, + {Name: Name{Local: "Str"}}, + {Name: Name{Local: "Bytes"}}, + }, + Bytes: []byte{}, + }, + ExpectXML: `<AttrsTest Int="0" int="0" Float="0" Uint8="0" Bool="false" Str="" Bytes="" Int="0" int="0" Float="0" Uint8="0" Bool="false" Str="" Bytes=""></AttrsTest>`, + MarshalOnly: true, + }, + { Value: &OmitAttrTest{ Int: 8, Named: 9, @@ -831,9 +1032,10 @@ var marshalTests = []struct { Bool: true, Str: "str", Bytes: []byte("byt"), + PStr: &empty, }, ExpectXML: `<OmitAttrTest Int="8" int="9" Float="23.5" Uint8="255"` + - ` Bool="true" Str="str" Bytes="byt"></OmitAttrTest>`, + ` Bool="true" Str="str" Bytes="byt" PStr=""></OmitAttrTest>`, }, { Value: &OmitAttrTest{}, @@ -864,6 +1066,7 @@ var marshalTests = []struct { Bool: true, Str: "str", Bytes: []byte("byt"), + PStr: &empty, Ptr: &PresenceTest{}, }, ExpectXML: `<OmitFieldTest>` + @@ -874,6 +1077,7 @@ var marshalTests = []struct { `<Bool>true</Bool>` + `<Str>str</Str>` + `<Bytes>byt</Bytes>` + + `<PStr></PStr>` + `<Ptr></Ptr>` + `</OmitFieldTest>`, }, @@ -1065,6 +1269,382 @@ var marshalTests = []struct { ExpectXML: `<NestedAndCData><A><B></B><B></B></A><![CDATA[test]]></NestedAndCData>`, Value: &NestedAndCData{AB: make([]string, 2), CDATA: "test"}, }, + // Test pointer indirection in various kinds of fields. + // https://golang.org/issue/19063 + { + ExpectXML: `<IndirComment><T1></T1><!--hi--><T2></T2></IndirComment>`, + Value: &IndirComment{Comment: stringptr("hi")}, + MarshalOnly: true, + }, + { + ExpectXML: `<IndirComment><T1></T1><T2></T2></IndirComment>`, + Value: &IndirComment{Comment: stringptr("")}, + MarshalOnly: true, + }, + { + ExpectXML: `<IndirComment><T1></T1><T2></T2></IndirComment>`, + Value: &IndirComment{Comment: nil}, + MarshalError: "xml: bad type for comment field of xml.IndirComment", + }, + { + ExpectXML: `<IndirComment><T1></T1><!--hi--><T2></T2></IndirComment>`, + Value: &IndirComment{Comment: nil}, + UnmarshalOnly: true, + }, + { + ExpectXML: `<IfaceComment><T1></T1><!--hi--><T2></T2></IfaceComment>`, + Value: &IfaceComment{Comment: "hi"}, + MarshalOnly: true, + }, + { + ExpectXML: `<IfaceComment><T1></T1><!--hi--><T2></T2></IfaceComment>`, + Value: &IfaceComment{Comment: nil}, + UnmarshalOnly: true, + }, + { + ExpectXML: `<IfaceComment><T1></T1><T2></T2></IfaceComment>`, + Value: &IfaceComment{Comment: nil}, + MarshalError: "xml: bad type for comment field of xml.IfaceComment", + }, + { + ExpectXML: `<IfaceComment><T1></T1><T2></T2></IfaceComment>`, + Value: &IfaceComment{Comment: nil}, + UnmarshalOnly: true, + }, + { + ExpectXML: `<DirectComment><T1></T1><!--hi--><T2></T2></DirectComment>`, + Value: &DirectComment{Comment: string("hi")}, + }, + { + ExpectXML: `<DirectComment><T1></T1><T2></T2></DirectComment>`, + Value: &DirectComment{Comment: string("")}, + }, + { + ExpectXML: `<IndirChardata><T1></T1>hi<T2></T2></IndirChardata>`, + Value: &IndirChardata{Chardata: stringptr("hi")}, + }, + { + ExpectXML: `<IndirChardata><T1></T1><![CDATA[hi]]><T2></T2></IndirChardata>`, + Value: &IndirChardata{Chardata: stringptr("hi")}, + UnmarshalOnly: true, // marshals without CDATA + }, + { + ExpectXML: `<IndirChardata><T1></T1><T2></T2></IndirChardata>`, + Value: &IndirChardata{Chardata: stringptr("")}, + }, + { + ExpectXML: `<IndirChardata><T1></T1><T2></T2></IndirChardata>`, + Value: &IndirChardata{Chardata: nil}, + MarshalOnly: true, // unmarshal leaves Chardata=stringptr("") + }, + { + ExpectXML: `<IfaceChardata><T1></T1>hi<T2></T2></IfaceChardata>`, + Value: &IfaceChardata{Chardata: string("hi")}, + UnmarshalError: "cannot unmarshal into interface {}", + }, + { + ExpectXML: `<IfaceChardata><T1></T1><![CDATA[hi]]><T2></T2></IfaceChardata>`, + Value: &IfaceChardata{Chardata: string("hi")}, + UnmarshalOnly: true, // marshals without CDATA + UnmarshalError: "cannot unmarshal into interface {}", + }, + { + ExpectXML: `<IfaceChardata><T1></T1><T2></T2></IfaceChardata>`, + Value: &IfaceChardata{Chardata: string("")}, + UnmarshalError: "cannot unmarshal into interface {}", + }, + { + ExpectXML: `<IfaceChardata><T1></T1><T2></T2></IfaceChardata>`, + Value: &IfaceChardata{Chardata: nil}, + UnmarshalError: "cannot unmarshal into interface {}", + }, + { + ExpectXML: `<DirectChardata><T1></T1>hi<T2></T2></DirectChardata>`, + Value: &DirectChardata{Chardata: string("hi")}, + }, + { + ExpectXML: `<DirectChardata><T1></T1><![CDATA[hi]]><T2></T2></DirectChardata>`, + Value: &DirectChardata{Chardata: string("hi")}, + UnmarshalOnly: true, // marshals without CDATA + }, + { + ExpectXML: `<DirectChardata><T1></T1><T2></T2></DirectChardata>`, + Value: &DirectChardata{Chardata: string("")}, + }, + { + ExpectXML: `<IndirCDATA><T1></T1><![CDATA[hi]]><T2></T2></IndirCDATA>`, + Value: &IndirCDATA{CDATA: stringptr("hi")}, + }, + { + ExpectXML: `<IndirCDATA><T1></T1>hi<T2></T2></IndirCDATA>`, + Value: &IndirCDATA{CDATA: stringptr("hi")}, + UnmarshalOnly: true, // marshals with CDATA + }, + { + ExpectXML: `<IndirCDATA><T1></T1><T2></T2></IndirCDATA>`, + Value: &IndirCDATA{CDATA: stringptr("")}, + }, + { + ExpectXML: `<IndirCDATA><T1></T1><T2></T2></IndirCDATA>`, + Value: &IndirCDATA{CDATA: nil}, + MarshalOnly: true, // unmarshal leaves CDATA=stringptr("") + }, + { + ExpectXML: `<IfaceCDATA><T1></T1><![CDATA[hi]]><T2></T2></IfaceCDATA>`, + Value: &IfaceCDATA{CDATA: string("hi")}, + UnmarshalError: "cannot unmarshal into interface {}", + }, + { + ExpectXML: `<IfaceCDATA><T1></T1>hi<T2></T2></IfaceCDATA>`, + Value: &IfaceCDATA{CDATA: string("hi")}, + UnmarshalOnly: true, // marshals with CDATA + UnmarshalError: "cannot unmarshal into interface {}", + }, + { + ExpectXML: `<IfaceCDATA><T1></T1><T2></T2></IfaceCDATA>`, + Value: &IfaceCDATA{CDATA: string("")}, + UnmarshalError: "cannot unmarshal into interface {}", + }, + { + ExpectXML: `<IfaceCDATA><T1></T1><T2></T2></IfaceCDATA>`, + Value: &IfaceCDATA{CDATA: nil}, + UnmarshalError: "cannot unmarshal into interface {}", + }, + { + ExpectXML: `<DirectCDATA><T1></T1><![CDATA[hi]]><T2></T2></DirectCDATA>`, + Value: &DirectCDATA{CDATA: string("hi")}, + }, + { + ExpectXML: `<DirectCDATA><T1></T1>hi<T2></T2></DirectCDATA>`, + Value: &DirectCDATA{CDATA: string("hi")}, + UnmarshalOnly: true, // marshals with CDATA + }, + { + ExpectXML: `<DirectCDATA><T1></T1><T2></T2></DirectCDATA>`, + Value: &DirectCDATA{CDATA: string("")}, + }, + { + ExpectXML: `<IndirInnerXML><T1></T1><hi/><T2></T2></IndirInnerXML>`, + Value: &IndirInnerXML{InnerXML: stringptr("<hi/>")}, + MarshalOnly: true, + }, + { + ExpectXML: `<IndirInnerXML><T1></T1><T2></T2></IndirInnerXML>`, + Value: &IndirInnerXML{InnerXML: stringptr("")}, + MarshalOnly: true, + }, + { + ExpectXML: `<IndirInnerXML><T1></T1><T2></T2></IndirInnerXML>`, + Value: &IndirInnerXML{InnerXML: nil}, + }, + { + ExpectXML: `<IndirInnerXML><T1></T1><hi/><T2></T2></IndirInnerXML>`, + Value: &IndirInnerXML{InnerXML: nil}, + UnmarshalOnly: true, + }, + { + ExpectXML: `<IfaceInnerXML><T1></T1><hi/><T2></T2></IfaceInnerXML>`, + Value: &IfaceInnerXML{InnerXML: "<hi/>"}, + MarshalOnly: true, + }, + { + ExpectXML: `<IfaceInnerXML><T1></T1><hi/><T2></T2></IfaceInnerXML>`, + Value: &IfaceInnerXML{InnerXML: nil}, + UnmarshalOnly: true, + }, + { + ExpectXML: `<IfaceInnerXML><T1></T1><T2></T2></IfaceInnerXML>`, + Value: &IfaceInnerXML{InnerXML: nil}, + }, + { + ExpectXML: `<IfaceInnerXML><T1></T1><T2></T2></IfaceInnerXML>`, + Value: &IfaceInnerXML{InnerXML: nil}, + UnmarshalOnly: true, + }, + { + ExpectXML: `<DirectInnerXML><T1></T1><hi/><T2></T2></DirectInnerXML>`, + Value: &DirectInnerXML{InnerXML: string("<hi/>")}, + MarshalOnly: true, + }, + { + ExpectXML: `<DirectInnerXML><T1></T1><hi/><T2></T2></DirectInnerXML>`, + Value: &DirectInnerXML{InnerXML: string("<T1></T1><hi/><T2></T2>")}, + UnmarshalOnly: true, + }, + { + ExpectXML: `<DirectInnerXML><T1></T1><T2></T2></DirectInnerXML>`, + Value: &DirectInnerXML{InnerXML: string("")}, + MarshalOnly: true, + }, + { + ExpectXML: `<DirectInnerXML><T1></T1><T2></T2></DirectInnerXML>`, + Value: &DirectInnerXML{InnerXML: string("<T1></T1><T2></T2>")}, + UnmarshalOnly: true, + }, + { + ExpectXML: `<IndirElement><T1></T1><Element>hi</Element><T2></T2></IndirElement>`, + Value: &IndirElement{Element: stringptr("hi")}, + }, + { + ExpectXML: `<IndirElement><T1></T1><Element></Element><T2></T2></IndirElement>`, + Value: &IndirElement{Element: stringptr("")}, + }, + { + ExpectXML: `<IndirElement><T1></T1><T2></T2></IndirElement>`, + Value: &IndirElement{Element: nil}, + }, + { + ExpectXML: `<IfaceElement><T1></T1><Element>hi</Element><T2></T2></IfaceElement>`, + Value: &IfaceElement{Element: "hi"}, + MarshalOnly: true, + }, + { + ExpectXML: `<IfaceElement><T1></T1><Element>hi</Element><T2></T2></IfaceElement>`, + Value: &IfaceElement{Element: nil}, + UnmarshalOnly: true, + }, + { + ExpectXML: `<IfaceElement><T1></T1><T2></T2></IfaceElement>`, + Value: &IfaceElement{Element: nil}, + }, + { + ExpectXML: `<IfaceElement><T1></T1><T2></T2></IfaceElement>`, + Value: &IfaceElement{Element: nil}, + UnmarshalOnly: true, + }, + { + ExpectXML: `<DirectElement><T1></T1><Element>hi</Element><T2></T2></DirectElement>`, + Value: &DirectElement{Element: string("hi")}, + }, + { + ExpectXML: `<DirectElement><T1></T1><Element></Element><T2></T2></DirectElement>`, + Value: &DirectElement{Element: string("")}, + }, + { + ExpectXML: `<IndirOmitEmpty><T1></T1><OmitEmpty>hi</OmitEmpty><T2></T2></IndirOmitEmpty>`, + Value: &IndirOmitEmpty{OmitEmpty: stringptr("hi")}, + }, + { + // Note: Changed in Go 1.8 to include <OmitEmpty> element (because x.OmitEmpty != nil). + ExpectXML: `<IndirOmitEmpty><T1></T1><OmitEmpty></OmitEmpty><T2></T2></IndirOmitEmpty>`, + Value: &IndirOmitEmpty{OmitEmpty: stringptr("")}, + MarshalOnly: true, + }, + { + ExpectXML: `<IndirOmitEmpty><T1></T1><OmitEmpty></OmitEmpty><T2></T2></IndirOmitEmpty>`, + Value: &IndirOmitEmpty{OmitEmpty: stringptr("")}, + UnmarshalOnly: true, + }, + { + ExpectXML: `<IndirOmitEmpty><T1></T1><T2></T2></IndirOmitEmpty>`, + Value: &IndirOmitEmpty{OmitEmpty: nil}, + }, + { + ExpectXML: `<IfaceOmitEmpty><T1></T1><OmitEmpty>hi</OmitEmpty><T2></T2></IfaceOmitEmpty>`, + Value: &IfaceOmitEmpty{OmitEmpty: "hi"}, + MarshalOnly: true, + }, + { + ExpectXML: `<IfaceOmitEmpty><T1></T1><OmitEmpty>hi</OmitEmpty><T2></T2></IfaceOmitEmpty>`, + Value: &IfaceOmitEmpty{OmitEmpty: nil}, + UnmarshalOnly: true, + }, + { + ExpectXML: `<IfaceOmitEmpty><T1></T1><T2></T2></IfaceOmitEmpty>`, + Value: &IfaceOmitEmpty{OmitEmpty: nil}, + }, + { + ExpectXML: `<IfaceOmitEmpty><T1></T1><T2></T2></IfaceOmitEmpty>`, + Value: &IfaceOmitEmpty{OmitEmpty: nil}, + UnmarshalOnly: true, + }, + { + ExpectXML: `<DirectOmitEmpty><T1></T1><OmitEmpty>hi</OmitEmpty><T2></T2></DirectOmitEmpty>`, + Value: &DirectOmitEmpty{OmitEmpty: string("hi")}, + }, + { + ExpectXML: `<DirectOmitEmpty><T1></T1><T2></T2></DirectOmitEmpty>`, + Value: &DirectOmitEmpty{OmitEmpty: string("")}, + }, + { + ExpectXML: `<IndirAny><T1></T1><Any>hi</Any><T2></T2></IndirAny>`, + Value: &IndirAny{Any: stringptr("hi")}, + }, + { + ExpectXML: `<IndirAny><T1></T1><Any></Any><T2></T2></IndirAny>`, + Value: &IndirAny{Any: stringptr("")}, + }, + { + ExpectXML: `<IndirAny><T1></T1><T2></T2></IndirAny>`, + Value: &IndirAny{Any: nil}, + }, + { + ExpectXML: `<IfaceAny><T1></T1><Any>hi</Any><T2></T2></IfaceAny>`, + Value: &IfaceAny{Any: "hi"}, + MarshalOnly: true, + }, + { + ExpectXML: `<IfaceAny><T1></T1><Any>hi</Any><T2></T2></IfaceAny>`, + Value: &IfaceAny{Any: nil}, + UnmarshalOnly: true, + }, + { + ExpectXML: `<IfaceAny><T1></T1><T2></T2></IfaceAny>`, + Value: &IfaceAny{Any: nil}, + }, + { + ExpectXML: `<IfaceAny><T1></T1><T2></T2></IfaceAny>`, + Value: &IfaceAny{Any: nil}, + UnmarshalOnly: true, + }, + { + ExpectXML: `<DirectAny><T1></T1><Any>hi</Any><T2></T2></DirectAny>`, + Value: &DirectAny{Any: string("hi")}, + }, + { + ExpectXML: `<DirectAny><T1></T1><Any></Any><T2></T2></DirectAny>`, + Value: &DirectAny{Any: string("")}, + }, + { + ExpectXML: `<IndirFoo><T1></T1><Foo>hi</Foo><T2></T2></IndirFoo>`, + Value: &IndirAny{Any: stringptr("hi")}, + UnmarshalOnly: true, + }, + { + ExpectXML: `<IndirFoo><T1></T1><Foo></Foo><T2></T2></IndirFoo>`, + Value: &IndirAny{Any: stringptr("")}, + UnmarshalOnly: true, + }, + { + ExpectXML: `<IndirFoo><T1></T1><T2></T2></IndirFoo>`, + Value: &IndirAny{Any: nil}, + UnmarshalOnly: true, + }, + { + ExpectXML: `<IfaceFoo><T1></T1><Foo>hi</Foo><T2></T2></IfaceFoo>`, + Value: &IfaceAny{Any: nil}, + UnmarshalOnly: true, + }, + { + ExpectXML: `<IfaceFoo><T1></T1><T2></T2></IfaceFoo>`, + Value: &IfaceAny{Any: nil}, + UnmarshalOnly: true, + }, + { + ExpectXML: `<IfaceFoo><T1></T1><T2></T2></IfaceFoo>`, + Value: &IfaceAny{Any: nil}, + UnmarshalOnly: true, + }, + { + ExpectXML: `<DirectFoo><T1></T1><Foo>hi</Foo><T2></T2></DirectFoo>`, + Value: &DirectAny{Any: string("hi")}, + UnmarshalOnly: true, + }, + { + ExpectXML: `<DirectFoo><T1></T1><Foo></Foo><T2></T2></DirectFoo>`, + Value: &DirectAny{Any: string("")}, + UnmarshalOnly: true, + }, } func TestMarshal(t *testing.T) { @@ -1074,7 +1654,17 @@ func TestMarshal(t *testing.T) { } data, err := Marshal(test.Value) if err != nil { - t.Errorf("#%d: marshal(%#v): %s", idx, test.Value, err) + if test.MarshalError == "" { + t.Errorf("#%d: marshal(%#v): %s", idx, test.Value, err) + continue + } + if !strings.Contains(err.Error(), test.MarshalError) { + t.Errorf("#%d: marshal(%#v): %s, want %q", idx, test.Value, err, test.MarshalError) + } + continue + } + if test.MarshalError != "" { + t.Errorf("#%d: Marshal succeeded, want error %q", idx, test.MarshalError) continue } if got, want := string(data), test.ExpectXML; got != want { @@ -1092,7 +1682,7 @@ type AttrParent struct { } type BadAttr struct { - Name []string `xml:"name,attr"` + Name map[string]string `xml:"name,attr"` } var marshalErrorTests = []struct { @@ -1128,8 +1718,8 @@ var marshalErrorTests = []struct { Err: `xml: X>Y chain not valid with attr flag`, }, { - Value: BadAttr{[]string{"X", "Y"}}, - Err: `xml: unsupported type: []string`, + Value: BadAttr{map[string]string{"X": "Y"}}, + Err: `xml: unsupported type: map[string]string`, }, } @@ -1200,8 +1790,16 @@ func TestUnmarshal(t *testing.T) { } if err != nil { - t.Errorf("#%d: unexpected error: %#v", i, err) - } else if got, want := dest, test.Value; !reflect.DeepEqual(got, want) { + if test.UnmarshalError == "" { + t.Errorf("#%d: unmarshal(%#v): %s", i, test.ExpectXML, err) + continue + } + if !strings.Contains(err.Error(), test.UnmarshalError) { + t.Errorf("#%d: unmarshal(%#v): %s, want %q", i, test.ExpectXML, err, test.UnmarshalError) + } + continue + } + if got, want := dest, test.Value; !reflect.DeepEqual(got, want) { t.Errorf("#%d: unmarshal(%q):\nhave %#v\nwant %#v", i, test.ExpectXML, got, want) } } @@ -1732,7 +2330,7 @@ func TestDecodeEncode(t *testing.T) { in.WriteString(`<?xml version="1.0" encoding="UTF-8"?> <?Target Instruction?> <root> -</root> +</root> `) dec := NewDecoder(&in) enc := NewEncoder(&out) @@ -1823,3 +2421,17 @@ func TestSimpleUseOfEncodeToken(t *testing.T) { t.Errorf("enc.EncodeToken: expected %q; got %q", want, buf.String()) } } + +// Issue 16158. Decoder.unmarshalAttr ignores the return value of copyValue. +func TestIssue16158(t *testing.T) { + const data = `<foo b="HELLOWORLD"></foo>` + err := Unmarshal([]byte(data), &struct { + B byte `xml:"b,attr,omitempty"` + }{}) + + // For Go 1.8.1 we've restored the old "no errors reported" behavior. + // We'll try again in Go 1.9 to report errors. + if err != nil { + t.Errorf("Unmarshal: expected nil, got error") + } +} diff --git a/libgo/go/encoding/xml/read.go b/libgo/go/encoding/xml/read.go index 77b4c7b495..799b57e9d1 100644 --- a/libgo/go/encoding/xml/read.go +++ b/libgo/go/encoding/xml/read.go @@ -1,4 +1,4 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -27,7 +27,7 @@ import ( // discarded. // // Because Unmarshal uses the reflect package, it can only assign -// to exported (upper case) fields. Unmarshal uses a case-sensitive +// to exported (upper case) fields. Unmarshal uses a case-sensitive // comparison to match XML element names to tag values and struct // field names. // @@ -37,9 +37,9 @@ import ( // // * If the struct has a field of type []byte or string with tag // ",innerxml", Unmarshal accumulates the raw XML nested inside the -// element in that field. The rest of the rules still apply. +// element in that field. The rest of the rules still apply. // -// * If the struct has a field named XMLName of type xml.Name, +// * If the struct has a field named XMLName of type Name, // Unmarshal records the element name in that field. // // * If the XMLName field has an associated tag of the form @@ -52,6 +52,11 @@ import ( // the explicit name in a struct field tag of the form "name,attr", // Unmarshal records the attribute value in that field. // +// * If the XML element has an attribute not handled by the previous +// rule and the struct has a field with an associated tag containing +// ",any,attr", Unmarshal records the attribute value in the first +// such field. +// // * If the XML element contains character data, that data is // accumulated in the first struct field that has tag ",chardata". // The struct field may have type []byte or string. @@ -59,7 +64,7 @@ import ( // // * If the XML element contains comments, they are accumulated in // the first struct field that has tag ",comment". The struct -// field may have type []byte or string. If there is no such +// field may have type []byte or string. If there is no such // field, the comments are discarded. // // * If the XML element contains a sub-element whose name matches @@ -85,7 +90,7 @@ import ( // * An anonymous struct field is handled as if the fields of its // value were part of the outer struct. // -// * A struct field with tag "-" is never unmarshalled into. +// * A struct field with tag "-" is never unmarshaled into. // // Unmarshal maps an XML element to a string or []byte by saving the // concatenation of that element's character data in the string or @@ -94,19 +99,23 @@ import ( // Unmarshal maps an attribute value to a string or []byte by saving // the value in the string or slice. // -// Unmarshal maps an XML element to a slice by extending the length of -// the slice and mapping the element to the newly created value. +// Unmarshal maps an attribute value to an Attr by saving the attribute, +// including its name, in the Attr. +// +// Unmarshal maps an XML element or attribute value to a slice by +// extending the length of the slice and mapping the element or attribute +// to the newly created value. // // Unmarshal maps an XML element or attribute value to a bool by // setting it to the boolean value represented by the string. // // Unmarshal maps an XML element or attribute value to an integer or // floating-point field by setting the field to the result of -// interpreting the string value in decimal. There is no check for +// interpreting the string value in decimal. There is no check for // overflow. // -// Unmarshal maps an XML element to an xml.Name by recording the -// element name. +// Unmarshal maps an XML element to a Name by recording the element +// name. // // Unmarshal maps an XML element to a pointer by setting the pointer // to a freshly allocated value and then mapping the element to that value. @@ -115,13 +124,13 @@ func Unmarshal(data []byte, v interface{}) error { return NewDecoder(bytes.NewReader(data)).Decode(v) } -// Decode works like xml.Unmarshal, except it reads the decoder +// Decode works like Unmarshal, except it reads the decoder // stream to find the start element. func (d *Decoder) Decode(v interface{}) error { return d.DecodeElement(v, nil) } -// DecodeElement works like xml.Unmarshal except that it takes +// DecodeElement works like Unmarshal except that it takes // a pointer to the start XML element to decode into v. // It is useful when a client reads some raw XML tokens itself // but also wants to defer to Unmarshal for some elements. @@ -133,7 +142,7 @@ func (d *Decoder) DecodeElement(v interface{}, start *StartElement) error { return d.unmarshal(val.Elem(), start) } -// An UnmarshalError represents an error in the unmarshalling process. +// An UnmarshalError represents an error in the unmarshaling process. type UnmarshalError string func (e UnmarshalError) Error() string { return string(e) } @@ -232,7 +241,6 @@ func (p *Decoder) unmarshalAttr(val reflect.Value, attr Attr) error { } val = val.Elem() } - if val.CanInterface() && val.Type().Implements(unmarshalerAttrType) { // This is an unmarshaler with a non-pointer receiver, // so it's likely to be incorrect, but we do what we're told. @@ -258,11 +266,31 @@ func (p *Decoder) unmarshalAttr(val reflect.Value, attr Attr) error { } } + if val.Type().Kind() == reflect.Slice && val.Type().Elem().Kind() != reflect.Uint8 { + // Slice of element values. + // Grow slice. + n := val.Len() + val.Set(reflect.Append(val, reflect.Zero(val.Type().Elem()))) + + // Recur to read element into slice. + if err := p.unmarshalAttr(val.Index(n), attr); err != nil { + val.SetLen(n) + return err + } + return nil + } + + if val.Type() == attrType { + val.Set(reflect.ValueOf(attr)) + return nil + } + copyValue(val, []byte(attr.Value)) return nil } var ( + attrType = reflect.TypeOf(Attr{}) unmarshalerType = reflect.TypeOf((*Unmarshaler)(nil)).Elem() unmarshalerAttrType = reflect.TypeOf((*UnmarshalerAttr)(nil)).Elem() textUnmarshalerType = reflect.TypeOf((*encoding.TextUnmarshaler)(nil)).Elem() @@ -359,16 +387,7 @@ func (p *Decoder) unmarshal(val reflect.Value, start *StartElement) error { // Slice of element values. // Grow slice. n := v.Len() - if n >= v.Cap() { - ncap := 2 * n - if ncap < 4 { - ncap = 4 - } - new := reflect.MakeSlice(typ, n, ncap) - reflect.Copy(new, v) - v.Set(new) - } - v.SetLen(n + 1) + v.Set(reflect.Append(val, reflect.Zero(v.Type().Elem()))) // Recur to read element into slice. if err := p.unmarshal(v.Index(n), start); err != nil { @@ -415,22 +434,40 @@ func (p *Decoder) unmarshal(val reflect.Value, start *StartElement) error { } // Assign attributes. - // Also, determine whether we need to save character data or comments. - for i := range tinfo.fields { - finfo := &tinfo.fields[i] - switch finfo.flags & fMode { - case fAttr: - strv := finfo.value(sv) - // Look for attribute. - for _, a := range start.Attr { + for _, a := range start.Attr { + handled := false + any := -1 + for i := range tinfo.fields { + finfo := &tinfo.fields[i] + switch finfo.flags & fMode { + case fAttr: + strv := finfo.value(sv) if a.Name.Local == finfo.name && (finfo.xmlns == "" || finfo.xmlns == a.Name.Space) { if err := p.unmarshalAttr(strv, a); err != nil { return err } - break + handled = true } + + case fAny | fAttr: + if any == -1 { + any = i + } + } + } + if !handled && any >= 0 { + finfo := &tinfo.fields[any] + strv := finfo.value(sv) + if err := p.unmarshalAttr(strv, a); err != nil { + return err } + } + } + // Determine whether we need to save character data or comments. + for i := range tinfo.fields { + finfo := &tinfo.fields[i] + switch finfo.flags & fMode { case fCDATA, fCharData: if !saveData.IsValid() { saveData = finfo.value(sv) @@ -546,7 +583,9 @@ Loop: case reflect.String: t.SetString(string(saveXMLData)) case reflect.Slice: - t.Set(reflect.ValueOf(saveXMLData)) + if t.Type().Elem().Kind() == reflect.Uint8 { + t.Set(reflect.ValueOf(saveXMLData)) + } } return nil diff --git a/libgo/go/encoding/xml/read_test.go b/libgo/go/encoding/xml/read_test.go index 7a98092803..273c303d16 100644 --- a/libgo/go/encoding/xml/read_test.go +++ b/libgo/go/encoding/xml/read_test.go @@ -705,7 +705,7 @@ func TestUnmarshalIntoInterface(t *testing.T) { } pea, ok := pod.Pea.(*Pea) if !ok { - t.Fatalf("unmarshalled into wrong type: have %T want *Pea", pod.Pea) + t.Fatalf("unmarshaled into wrong type: have %T want *Pea", pod.Pea) } have, want := pea.Cotelydon, "Green stuff" if have != want { @@ -733,3 +733,22 @@ func TestMalformedComment(t *testing.T) { } } } + +type IXField struct { + Five int `xml:"five"` + NotInnerXML []string `xml:",innerxml"` +} + +// Issue 15600. ",innerxml" on a field that can't hold it. +func TestInvalidInnerXMLType(t *testing.T) { + v := new(IXField) + if err := Unmarshal([]byte(`<tag><five>5</five><innertag/></tag>`), v); err != nil { + t.Errorf("Unmarshal failed: got %v", err) + } + if v.Five != 5 { + t.Errorf("Five = %v, want 5", v.Five) + } + if v.NotInnerXML != nil { + t.Errorf("NotInnerXML = %v, want nil", v.NotInnerXML) + } +} diff --git a/libgo/go/encoding/xml/typeinfo.go b/libgo/go/encoding/xml/typeinfo.go index 6483c8dbe6..6623c78308 100644 --- a/libgo/go/encoding/xml/typeinfo.go +++ b/libgo/go/encoding/xml/typeinfo.go @@ -1,4 +1,4 @@ -// Copyright 2011 The Go Authors. All rights reserved. +// Copyright 2011 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -48,7 +48,7 @@ var tinfoLock sync.RWMutex var nameType = reflect.TypeOf(Name{}) // getTypeInfo returns the typeInfo structure with details necessary -// for marshalling and unmarshalling typ. +// for marshaling and unmarshaling typ. func getTypeInfo(typ reflect.Type) (*typeInfo, error) { tinfoLock.RLock() tinfo, ok := tinfoMap[typ] @@ -151,7 +151,7 @@ func structFieldInfo(typ reflect.Type, f *reflect.StructField) (*fieldInfo, erro switch mode := finfo.flags & fMode; mode { case 0: finfo.flags |= fElement - case fAttr, fCDATA, fCharData, fInnerXml, fComment, fAny: + case fAttr, fCDATA, fCharData, fInnerXml, fComment, fAny, fAny | fAttr: if f.Name == "XMLName" || tag != "" && mode != fAttr { valid = false } @@ -214,7 +214,7 @@ func structFieldInfo(typ reflect.Type, f *reflect.StructField) (*fieldInfo, erro } // If the field type has an XMLName field, the names must match - // so that the behavior of both marshalling and unmarshalling + // so that the behavior of both marshaling and unmarshaling // is straightforward and unambiguous. if finfo.flags&fElement != 0 { ftyp := f.Type @@ -334,7 +334,7 @@ Loop: return nil } -// A TagPathError represents an error in the unmarshalling process +// A TagPathError represents an error in the unmarshaling process // caused by the use of field tags with conflicting paths. type TagPathError struct { Struct reflect.Type diff --git a/libgo/go/encoding/xml/xml.go b/libgo/go/encoding/xml/xml.go index 45f4157318..9a3b792955 100644 --- a/libgo/go/encoding/xml/xml.go +++ b/libgo/go/encoding/xml/xml.go @@ -219,7 +219,7 @@ func NewDecoder(r io.Reader) *Decoder { // // Slices of bytes in the returned token data refer to the // parser's internal buffer and remain valid only until the next -// call to Token. To acquire a copy of the bytes, call CopyToken +// call to Token. To acquire a copy of the bytes, call CopyToken // or the token's Copy method. // // Token expands self-closing elements such as <br/> @@ -237,10 +237,11 @@ func NewDecoder(r io.Reader) *Decoder { // set to the URL identifying its name space when known. // If Token encounters an unrecognized name space prefix, // it uses the prefix as the Space rather than report an error. -func (d *Decoder) Token() (t Token, err error) { +func (d *Decoder) Token() (Token, error) { + var t Token + var err error if d.stk != nil && d.stk.kind == stkEOF { - err = io.EOF - return + return nil, io.EOF } if d.nextToken != nil { t = d.nextToken @@ -249,7 +250,7 @@ func (d *Decoder) Token() (t Token, err error) { if err == io.EOF && d.stk != nil && d.stk.kind != stkEOF { err = d.syntaxError("unexpected EOF") } - return + return t, err } if !d.Strict { @@ -292,7 +293,7 @@ func (d *Decoder) Token() (t Token, err error) { } t = t1 } - return + return t, err } const xmlURL = "http://www.w3.org/XML/1998/namespace" @@ -331,7 +332,7 @@ func (d *Decoder) switchToReader(r io.Reader) { } // Parsing state - stack holds old name space translations -// and the current set of open elements. The translations to pop when +// and the current set of open elements. The translations to pop when // ending a given tag are *below* it on the stack, which is // more work but forced on us by XML. type stack struct { @@ -1229,7 +1230,7 @@ func isNameString(s string) bool { // These tables were generated by cut and paste from Appendix B of // the XML spec at http://www.xml.com/axml/testaxml.htm -// and then reformatting. First corresponds to (Letter | '_' | ':') +// and then reformatting. First corresponds to (Letter | '_' | ':') // and second corresponds to NameChar. var first = &unicode.RangeTable{ diff --git a/libgo/go/encoding/xml/xml_test.go b/libgo/go/encoding/xml/xml_test.go index 5d5e4bf970..f43a5e7eeb 100644 --- a/libgo/go/encoding/xml/xml_test.go +++ b/libgo/go/encoding/xml/xml_test.go @@ -1,4 +1,4 @@ -// Copyright 2009 The Go Authors. All rights reserved. +// Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -184,8 +184,6 @@ const nonStrictInput = ` <tag>&0a;</tag> ` -var nonStringEntity = map[string]string{"": "oops!", "0a": "oops!"} - var nonStrictTokens = []Token{ CharData("\n"), StartElement{Name{"", "tag"}, []Attr{}}, @@ -652,10 +650,6 @@ func TestDisallowedCharacters(t *testing.T) { } } -type procInstEncodingTest struct { - expect, got string -} - var procInstTests = []struct { input string expect [2]string @@ -803,3 +797,37 @@ func TestIssue12417(t *testing.T) { } } } + +func TestIssue19333(t *testing.T) { + type X struct { + XMLName Name `xml:"X"` + A int `xml:",attr"` + C int + } + + var tests = []struct { + input string + ok bool + }{ + {`<X></X>`, true}, + {`<X A=""></X>`, true}, + {`<X A="bad"></X>`, true}, + {`<X></X>`, true}, + {`<X><C></C></X>`, false}, + {`<X><C/></X>`, false}, + {`<X><C>bad</C></X>`, false}, + } + + for _, tt := range tests { + err := Unmarshal([]byte(tt.input), new(X)) + if tt.ok { + if err != nil { + t.Errorf("%s: unexpected error: %v", tt.input, err) + } + } else { + if err == nil { + t.Errorf("%s: unexpected success", tt.input) + } + } + } +} |