diff options
author | David Symonds <dsymonds@golang.org> | 2014-11-28 11:43:44 +1100 |
---|---|---|
committer | David Symonds <dsymonds@golang.org> | 2014-11-28 11:43:51 +1100 |
commit | abd3b412d3c2460d848b6b81478fcb4e542d6327 (patch) | |
tree | b468782c4380c52ff08851ce4802db8eae80c35f | |
parent | aec3ce103c5571c237126cf8e3e12b58c1af635f (diff) | |
download | protobuf-abd3b412d3c2460d848b6b81478fcb4e542d6327.tar.gz |
Support proto3.
-rw-r--r-- | Makefile | 1 | ||||
-rw-r--r-- | proto/Makefile | 5 | ||||
-rw-r--r-- | proto/decode.go | 36 | ||||
-rw-r--r-- | proto/encode.go | 124 | ||||
-rw-r--r-- | proto/pointer_reflect.go | 90 | ||||
-rw-r--r-- | proto/pointer_unsafe.go | 43 | ||||
-rw-r--r-- | proto/properties.go | 41 | ||||
-rw-r--r-- | proto/proto3_proto/Makefile | 44 | ||||
-rw-r--r-- | proto/proto3_proto/proto3.proto | 58 | ||||
-rw-r--r-- | proto/proto3_test.go | 93 | ||||
-rw-r--r-- | proto/size_test.go | 11 | ||||
-rw-r--r-- | proto/testdata/Makefile | 4 | ||||
-rw-r--r-- | proto/text.go | 29 | ||||
-rw-r--r-- | proto/text_parser.go | 4 | ||||
-rw-r--r-- | proto/text_parser_test.go | 16 | ||||
-rw-r--r-- | proto/text_test.go | 21 | ||||
-rw-r--r-- | protoc-gen-go/generator/generator.go | 77 | ||||
-rw-r--r-- | protoc-gen-go/testdata/proto3.proto | 52 |
18 files changed, 724 insertions, 25 deletions
@@ -50,3 +50,4 @@ regenerate: make -C protoc-gen-go/descriptor regenerate make -C protoc-gen-go/plugin regenerate make -C proto/testdata regenerate + make -C proto/proto3_proto regenerate diff --git a/proto/Makefile b/proto/Makefile index 66bd2f2..fb838ed 100644 --- a/proto/Makefile +++ b/proto/Makefile @@ -37,4 +37,7 @@ test: install generate-test-pbs generate-test-pbs: - make install && cd testdata && make + make install + make -C testdata + make -C proto3_proto + make diff --git a/proto/decode.go b/proto/decode.go index 6a09e29..6166dd4 100644 --- a/proto/decode.go +++ b/proto/decode.go @@ -465,6 +465,15 @@ func (o *Buffer) dec_bool(p *Properties, base structPointer) error { return nil } +func (o *Buffer) dec_proto3_bool(p *Properties, base structPointer) error { + u, err := p.valDec(o) + if err != nil { + return err + } + *structPointer_BoolVal(base, p.field) = u != 0 + return nil +} + // Decode an int32. func (o *Buffer) dec_int32(p *Properties, base structPointer) error { u, err := p.valDec(o) @@ -475,6 +484,15 @@ func (o *Buffer) dec_int32(p *Properties, base structPointer) error { return nil } +func (o *Buffer) dec_proto3_int32(p *Properties, base structPointer) error { + u, err := p.valDec(o) + if err != nil { + return err + } + word32Val_Set(structPointer_Word32Val(base, p.field), uint32(u)) + return nil +} + // Decode an int64. func (o *Buffer) dec_int64(p *Properties, base structPointer) error { u, err := p.valDec(o) @@ -485,6 +503,15 @@ func (o *Buffer) dec_int64(p *Properties, base structPointer) error { return nil } +func (o *Buffer) dec_proto3_int64(p *Properties, base structPointer) error { + u, err := p.valDec(o) + if err != nil { + return err + } + word64Val_Set(structPointer_Word64Val(base, p.field), o, u) + return nil +} + // Decode a string. func (o *Buffer) dec_string(p *Properties, base structPointer) error { s, err := o.DecodeStringBytes() @@ -497,6 +524,15 @@ func (o *Buffer) dec_string(p *Properties, base structPointer) error { return nil } +func (o *Buffer) dec_proto3_string(p *Properties, base structPointer) error { + s, err := o.DecodeStringBytes() + if err != nil { + return err + } + *structPointer_StringVal(base, p.field) = s + return nil +} + // Decode a slice of bytes ([]byte). func (o *Buffer) dec_slice_byte(p *Properties, base structPointer) error { b, err := o.DecodeRawBytes(true) diff --git a/proto/encode.go b/proto/encode.go index cbe4242..cc202cd 100644 --- a/proto/encode.go +++ b/proto/encode.go @@ -298,6 +298,16 @@ func (o *Buffer) enc_bool(p *Properties, base structPointer) error { return nil } +func (o *Buffer) enc_proto3_bool(p *Properties, base structPointer) error { + v := *structPointer_BoolVal(base, p.field) + if !v { + return ErrNil + } + o.buf = append(o.buf, p.tagcode...) + p.valEnc(o, 1) + return nil +} + func size_bool(p *Properties, base structPointer) int { v := *structPointer_Bool(base, p.field) if v == nil { @@ -306,6 +316,14 @@ func size_bool(p *Properties, base structPointer) int { return len(p.tagcode) + 1 // each bool takes exactly one byte } +func size_proto3_bool(p *Properties, base structPointer) int { + v := *structPointer_BoolVal(base, p.field) + if !v { + return 0 + } + return len(p.tagcode) + 1 // each bool takes exactly one byte +} + // Encode an int32. func (o *Buffer) enc_int32(p *Properties, base structPointer) error { v := structPointer_Word32(base, p.field) @@ -318,6 +336,17 @@ func (o *Buffer) enc_int32(p *Properties, base structPointer) error { return nil } +func (o *Buffer) enc_proto3_int32(p *Properties, base structPointer) error { + v := structPointer_Word32Val(base, p.field) + x := int32(word32Val_Get(v)) // permit sign extension to use full 64-bit range + if x == 0 { + return ErrNil + } + o.buf = append(o.buf, p.tagcode...) + p.valEnc(o, uint64(x)) + return nil +} + func size_int32(p *Properties, base structPointer) (n int) { v := structPointer_Word32(base, p.field) if word32_IsNil(v) { @@ -329,6 +358,17 @@ func size_int32(p *Properties, base structPointer) (n int) { return } +func size_proto3_int32(p *Properties, base structPointer) (n int) { + v := structPointer_Word32Val(base, p.field) + x := int32(word32Val_Get(v)) // permit sign extension to use full 64-bit range + if x == 0 { + return 0 + } + n += len(p.tagcode) + n += p.valSize(uint64(x)) + return +} + // Encode a uint32. // Exactly the same as int32, except for no sign extension. func (o *Buffer) enc_uint32(p *Properties, base structPointer) error { @@ -342,6 +382,17 @@ func (o *Buffer) enc_uint32(p *Properties, base structPointer) error { return nil } +func (o *Buffer) enc_proto3_uint32(p *Properties, base structPointer) error { + v := structPointer_Word32Val(base, p.field) + x := word32Val_Get(v) + if x == 0 { + return ErrNil + } + o.buf = append(o.buf, p.tagcode...) + p.valEnc(o, uint64(x)) + return nil +} + func size_uint32(p *Properties, base structPointer) (n int) { v := structPointer_Word32(base, p.field) if word32_IsNil(v) { @@ -353,6 +404,17 @@ func size_uint32(p *Properties, base structPointer) (n int) { return } +func size_proto3_uint32(p *Properties, base structPointer) (n int) { + v := structPointer_Word32Val(base, p.field) + x := word32Val_Get(v) + if x == 0 { + return 0 + } + n += len(p.tagcode) + n += p.valSize(uint64(x)) + return +} + // Encode an int64. func (o *Buffer) enc_int64(p *Properties, base structPointer) error { v := structPointer_Word64(base, p.field) @@ -365,6 +427,17 @@ func (o *Buffer) enc_int64(p *Properties, base structPointer) error { return nil } +func (o *Buffer) enc_proto3_int64(p *Properties, base structPointer) error { + v := structPointer_Word64Val(base, p.field) + x := word64Val_Get(v) + if x == 0 { + return ErrNil + } + o.buf = append(o.buf, p.tagcode...) + p.valEnc(o, x) + return nil +} + func size_int64(p *Properties, base structPointer) (n int) { v := structPointer_Word64(base, p.field) if word64_IsNil(v) { @@ -376,6 +449,17 @@ func size_int64(p *Properties, base structPointer) (n int) { return } +func size_proto3_int64(p *Properties, base structPointer) (n int) { + v := structPointer_Word64Val(base, p.field) + x := word64Val_Get(v) + if x == 0 { + return 0 + } + n += len(p.tagcode) + n += p.valSize(x) + return +} + // Encode a string. func (o *Buffer) enc_string(p *Properties, base structPointer) error { v := *structPointer_String(base, p.field) @@ -388,6 +472,16 @@ func (o *Buffer) enc_string(p *Properties, base structPointer) error { return nil } +func (o *Buffer) enc_proto3_string(p *Properties, base structPointer) error { + v := *structPointer_StringVal(base, p.field) + if v == "" { + return ErrNil + } + o.buf = append(o.buf, p.tagcode...) + o.EncodeStringBytes(v) + return nil +} + func size_string(p *Properties, base structPointer) (n int) { v := *structPointer_String(base, p.field) if v == nil { @@ -399,6 +493,16 @@ func size_string(p *Properties, base structPointer) (n int) { return } +func size_proto3_string(p *Properties, base structPointer) (n int) { + v := *structPointer_StringVal(base, p.field) + if v == "" { + return 0 + } + n += len(p.tagcode) + n += sizeStringBytes(v) + return +} + // All protocol buffer fields are nillable, but be careful. func isNil(v reflect.Value) bool { switch v.Kind() { @@ -551,6 +655,16 @@ func (o *Buffer) enc_slice_byte(p *Properties, base structPointer) error { return nil } +func (o *Buffer) enc_proto3_slice_byte(p *Properties, base structPointer) error { + s := *structPointer_Bytes(base, p.field) + if len(s) == 0 { + return ErrNil + } + o.buf = append(o.buf, p.tagcode...) + o.EncodeRawBytes(s) + return nil +} + func size_slice_byte(p *Properties, base structPointer) (n int) { s := *structPointer_Bytes(base, p.field) if s == nil { @@ -561,6 +675,16 @@ func size_slice_byte(p *Properties, base structPointer) (n int) { return } +func size_proto3_slice_byte(p *Properties, base structPointer) (n int) { + s := *structPointer_Bytes(base, p.field) + if len(s) == 0 { + return 0 + } + n += len(p.tagcode) + n += sizeRawBytes(s) + return +} + // Encode a slice of int32s ([]int32). func (o *Buffer) enc_slice_int32(p *Properties, base structPointer) error { s := structPointer_Word32Slice(base, p.field) diff --git a/proto/pointer_reflect.go b/proto/pointer_reflect.go index 0c53417..42c387a 100644 --- a/proto/pointer_reflect.go +++ b/proto/pointer_reflect.go @@ -114,6 +114,11 @@ func structPointer_Bool(p structPointer, f field) **bool { return structPointer_ifield(p, f).(**bool) } +// BoolVal returns the address of a bool field in the struct. +func structPointer_BoolVal(p structPointer, f field) *bool { + return structPointer_ifield(p, f).(*bool) +} + // BoolSlice returns the address of a []bool field in the struct. func structPointer_BoolSlice(p structPointer, f field) *[]bool { return structPointer_ifield(p, f).(*[]bool) @@ -124,6 +129,11 @@ func structPointer_String(p structPointer, f field) **string { return structPointer_ifield(p, f).(**string) } +// StringVal returns the address of a string field in the struct. +func structPointer_StringVal(p structPointer, f field) *string { + return structPointer_ifield(p, f).(*string) +} + // StringSlice returns the address of a []string field in the struct. func structPointer_StringSlice(p structPointer, f field) *[]string { return structPointer_ifield(p, f).(*[]string) @@ -235,6 +245,49 @@ func structPointer_Word32(p structPointer, f field) word32 { return word32{structPointer_field(p, f)} } +// A word32Val represents a field of type int32, uint32, float32, or enum. +// That is, v.Type() is int32, uint32, float32, or enum and v is assignable. +type word32Val struct { + v reflect.Value +} + +// Set sets *p to x. +func word32Val_Set(p word32Val, x uint32) { + switch p.v.Type() { + case int32Type: + p.v.SetInt(int64(x)) + return + case uint32Type: + p.v.SetUint(uint64(x)) + return + case float32Type: + p.v.SetFloat(float64(math.Float32frombits(x))) + return + } + + // must be enum + p.v.SetInt(int64(int32(x))) +} + +// Get gets the bits pointed at by p, as a uint32. +func word32Val_Get(p word32Val) uint32 { + elem := p.v + switch elem.Kind() { + case reflect.Int32: + return uint32(elem.Int()) + case reflect.Uint32: + return uint32(elem.Uint()) + case reflect.Float32: + return math.Float32bits(float32(elem.Float())) + } + panic("unreachable") +} + +// Word32Val returns a reference to a int32, uint32, float32, or enum field in the struct. +func structPointer_Word32Val(p structPointer, f field) word32Val { + return word32Val{structPointer_field(p, f)} +} + // A word32Slice is a slice of 32-bit values. // That is, v.Type() is []int32, []uint32, []float32, or []enum. type word32Slice struct { @@ -339,6 +392,43 @@ func structPointer_Word64(p structPointer, f field) word64 { return word64{structPointer_field(p, f)} } +// word64Val is like word32Val but for 64-bit values. +type word64Val struct { + v reflect.Value +} + +func word64Val_Set(p word64Val, o *Buffer, x uint64) { + switch p.v.Type() { + case int64Type: + p.v.SetInt(int64(x)) + return + case uint64Type: + p.v.SetUint(x) + return + case float64Type: + p.v.SetFloat(math.Float64frombits(x)) + return + } + panic("unreachable") +} + +func word64Val_Get(p word64Val) uint64 { + elem := p.v + switch elem.Kind() { + case reflect.Int64: + return uint64(elem.Int()) + case reflect.Uint64: + return elem.Uint() + case reflect.Float64: + return math.Float64bits(elem.Float()) + } + panic("unreachable") +} + +func structPointer_Word64Val(p structPointer, f field) word64Val { + return word64Val{structPointer_field(p, f)} +} + type word64Slice struct { v reflect.Value } diff --git a/proto/pointer_unsafe.go b/proto/pointer_unsafe.go index 7f3473e..cf9fc9a 100644 --- a/proto/pointer_unsafe.go +++ b/proto/pointer_unsafe.go @@ -100,6 +100,11 @@ func structPointer_Bool(p structPointer, f field) **bool { return (**bool)(unsafe.Pointer(uintptr(p) + uintptr(f))) } +// BoolVal returns the address of a bool field in the struct. +func structPointer_BoolVal(p structPointer, f field) *bool { + return (*bool)(unsafe.Pointer(uintptr(p) + uintptr(f))) +} + // BoolSlice returns the address of a []bool field in the struct. func structPointer_BoolSlice(p structPointer, f field) *[]bool { return (*[]bool)(unsafe.Pointer(uintptr(p) + uintptr(f))) @@ -110,6 +115,11 @@ func structPointer_String(p structPointer, f field) **string { return (**string)(unsafe.Pointer(uintptr(p) + uintptr(f))) } +// StringVal returns the address of a string field in the struct. +func structPointer_StringVal(p structPointer, f field) *string { + return (*string)(unsafe.Pointer(uintptr(p) + uintptr(f))) +} + // StringSlice returns the address of a []string field in the struct. func structPointer_StringSlice(p structPointer, f field) *[]string { return (*[]string)(unsafe.Pointer(uintptr(p) + uintptr(f))) @@ -170,6 +180,24 @@ func structPointer_Word32(p structPointer, f field) word32 { return word32((**uint32)(unsafe.Pointer(uintptr(p) + uintptr(f)))) } +// A word32Val is the address of a 32-bit value field. +type word32Val *uint32 + +// Set sets *p to x. +func word32Val_Set(p word32Val, x uint32) { + *p = x +} + +// Get gets the value pointed at by p. +func word32Val_Get(p word32Val) uint32 { + return *p +} + +// Word32Val returns the address of a *int32, *uint32, *float32, or *enum field in the struct. +func structPointer_Word32Val(p structPointer, f field) word32Val { + return word32Val((*uint32)(unsafe.Pointer(uintptr(p) + uintptr(f)))) +} + // A word32Slice is a slice of 32-bit values. type word32Slice []uint32 @@ -206,6 +234,21 @@ func structPointer_Word64(p structPointer, f field) word64 { return word64((**uint64)(unsafe.Pointer(uintptr(p) + uintptr(f)))) } +// word64Val is like word32Val but for 64-bit values. +type word64Val *uint64 + +func word64Val_Set(p word64Val, o *Buffer, x uint64) { + *p = x +} + +func word64Val_Get(p word64Val) uint64 { + return *p +} + +func structPointer_Word64Val(p structPointer, f field) word64Val { + return word64Val((*uint64)(unsafe.Pointer(uintptr(p) + uintptr(f)))) +} + // word64Slice is like word32Slice but for 64-bit values. type word64Slice []uint64 diff --git a/proto/properties.go b/proto/properties.go index 7262888..4420881 100644 --- a/proto/properties.go +++ b/proto/properties.go @@ -155,6 +155,7 @@ type Properties struct { Repeated bool Packed bool // relevant for repeated primitives only Enum string // set for enum types only + proto3 bool // whether this is known to be a proto3 field; set for []byte only Default string // default value HasDefault bool // whether an explicit default was provided @@ -200,6 +201,9 @@ func (p *Properties) String() string { if p.OrigName != p.Name { s += ",name=" + p.OrigName } + if p.proto3 { + s += ",proto3" + } if len(p.Enum) > 0 { s += ",enum=" + p.Enum } @@ -274,6 +278,8 @@ func (p *Properties) Parse(s string) { p.OrigName = f[5:] case strings.HasPrefix(f, "enum="): p.Enum = f[5:] + case f == "proto3": + p.proto3 = true case strings.HasPrefix(f, "def="): p.HasDefault = true p.Default = f[4:] // rest of string @@ -302,6 +308,37 @@ func (p *Properties) setEncAndDec(typ reflect.Type, lockGetProp bool) { default: fmt.Fprintf(os.Stderr, "proto: no coders for %v\n", t1) + // proto3 scalar types + + case reflect.Bool: + p.enc = (*Buffer).enc_proto3_bool + p.dec = (*Buffer).dec_proto3_bool + p.size = size_proto3_bool + case reflect.Int32: + p.enc = (*Buffer).enc_proto3_int32 + p.dec = (*Buffer).dec_proto3_int32 + p.size = size_proto3_int32 + case reflect.Uint32: + p.enc = (*Buffer).enc_proto3_uint32 + p.dec = (*Buffer).dec_proto3_int32 // can reuse + p.size = size_proto3_uint32 + case reflect.Int64, reflect.Uint64: + p.enc = (*Buffer).enc_proto3_int64 + p.dec = (*Buffer).dec_proto3_int64 + p.size = size_proto3_int64 + case reflect.Float32: + p.enc = (*Buffer).enc_proto3_uint32 // can just treat them as bits + p.dec = (*Buffer).dec_proto3_int32 + p.size = size_proto3_uint32 + case reflect.Float64: + p.enc = (*Buffer).enc_proto3_int64 // can just treat them as bits + p.dec = (*Buffer).dec_proto3_int64 + p.size = size_proto3_int64 + case reflect.String: + p.enc = (*Buffer).enc_proto3_string + p.dec = (*Buffer).dec_proto3_string + p.size = size_proto3_string + case reflect.Ptr: switch t2 := t1.Elem(); t2.Kind() { default: @@ -399,6 +436,10 @@ func (p *Properties) setEncAndDec(typ reflect.Type, lockGetProp bool) { p.enc = (*Buffer).enc_slice_byte p.dec = (*Buffer).dec_slice_byte p.size = size_slice_byte + if p.proto3 { + p.enc = (*Buffer).enc_proto3_slice_byte + p.size = size_proto3_slice_byte + } case reflect.Float32, reflect.Float64: switch t2.Bits() { case 32: diff --git a/proto/proto3_proto/Makefile b/proto/proto3_proto/Makefile new file mode 100644 index 0000000..75144b5 --- /dev/null +++ b/proto/proto3_proto/Makefile @@ -0,0 +1,44 @@ +# Go support for Protocol Buffers - Google's data interchange format +# +# Copyright 2014 The Go Authors. All rights reserved. +# https://github.com/golang/protobuf +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +include ../../Make.protobuf + +all: regenerate + +regenerate: + rm -f proto3.pb.go + make proto3.pb.go + +# The following rules are just aids to development. Not needed for typical testing. + +diff: regenerate + git diff proto3.pb.go diff --git a/proto/proto3_proto/proto3.proto b/proto/proto3_proto/proto3.proto new file mode 100644 index 0000000..3e327de --- /dev/null +++ b/proto/proto3_proto/proto3.proto @@ -0,0 +1,58 @@ +// Go support for Protocol Buffers - Google's data interchange format +// +// Copyright 2014 The Go Authors. All rights reserved. +// https://github.com/golang/protobuf +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package proto3_proto; + +message Message { + enum Humour { + UNKNOWN = 0; + PUNS = 1; + SLAPSTICK = 2; + BILL_BAILEY = 3; + } + + string name = 1; + Humour hilarity = 2; + uint32 height_in_cm = 3; + bytes data = 4; + int64 result_count = 7; + bool true_scotsman = 8; + float score = 9; + + repeated uint64 key = 5; + Nested nested = 6; +} + +message Nested { + string bunny = 1; +} diff --git a/proto/proto3_test.go b/proto/proto3_test.go new file mode 100644 index 0000000..d4c96a9 --- /dev/null +++ b/proto/proto3_test.go @@ -0,0 +1,93 @@ +// Go support for Protocol Buffers - Google's data interchange format +// +// Copyright 2014 The Go Authors. All rights reserved. +// https://github.com/golang/protobuf +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package proto_test + +import ( + "testing" + + pb "./proto3_proto" + "github.com/golang/protobuf/proto" +) + +func TestProto3ZeroValues(t *testing.T) { + tests := []struct { + desc string + m proto.Message + }{ + {"zero message", &pb.Message{}}, + {"empty bytes field", &pb.Message{Data: []byte{}}}, + } + for _, test := range tests { + b, err := proto.Marshal(test.m) + if err != nil { + t.Errorf("%s: proto.Marshal: %v", test.desc, err) + continue + } + if len(b) > 0 { + t.Errorf("%s: Encoding is non-empty: %q", test.desc, b) + } + } +} + +func TestRoundTripProto3(t *testing.T) { + m := &pb.Message{ + Name: "David", // (2 | 1<<3): 0x0a 0x05 "David" + Hilarity: pb.Message_PUNS, // (0 | 2<<3): 0x10 0x01 + HeightInCm: 178, // (0 | 3<<3): 0x18 0xb2 0x01 + Data: []byte("roboto"), // (2 | 4<<3): 0x20 0x06 "roboto" + ResultCount: 47, // (0 | 7<<3): 0x38 0x2f + TrueScotsman: true, // (0 | 8<<3): 0x40 0x01 + Score: 8.1, // (5 | 9<<3): 0x4d <8.1> + + Key: []uint64{1, 0xdeadbeef}, + Nested: &pb.Nested{ + Bunny: "Monty", + }, + } + t.Logf(" m: %v", m) + + b, err := proto.Marshal(m) + if err != nil { + t.Fatalf("proto.Marshal: %v", err) + } + t.Logf(" b: %q", b) + + m2 := new(pb.Message) + if err := proto.Unmarshal(b, m2); err != nil { + t.Fatalf("proto.Unmarshal: %v", err) + } + t.Logf("m2: %v", m2) + + if !proto.Equal(m, m2) { + t.Errorf("proto.Equal returned false:\n m: %v\nm2: %v", m, m2) + } +} diff --git a/proto/size_test.go b/proto/size_test.go index 69d6481..4f87f3b 100644 --- a/proto/size_test.go +++ b/proto/size_test.go @@ -35,6 +35,7 @@ import ( "log" "testing" + proto3pb "./proto3_proto" pb "./testdata" . "github.com/golang/protobuf/proto" ) @@ -102,6 +103,16 @@ var SizeTests = []struct { {"unrecognized", &pb.MoreRepeated{XXX_unrecognized: []byte{13<<3 | 0, 4}}}, {"extension (unencoded)", messageWithExtension1}, {"extension (encoded)", messageWithExtension3}, + // proto3 message + {"proto3 empty", &proto3pb.Message{}}, + {"proto3 bool", &proto3pb.Message{TrueScotsman: true}}, + {"proto3 int64", &proto3pb.Message{ResultCount: 1}}, + {"proto3 uint32", &proto3pb.Message{HeightInCm: 123}}, + {"proto3 float", &proto3pb.Message{Score: 12.6}}, + {"proto3 string", &proto3pb.Message{Name: "Snezana"}}, + {"proto3 bytes", &proto3pb.Message{Data: []byte("wowsa")}}, + {"proto3 bytes, empty", &proto3pb.Message{Data: []byte{}}}, + {"proto3 enum", &proto3pb.Message{Hilarity: proto3pb.Message_PUNS}}, } func TestSize(t *testing.T) { diff --git a/proto/testdata/Makefile b/proto/testdata/Makefile index a4cd82a..fc28862 100644 --- a/proto/testdata/Makefile +++ b/proto/testdata/Makefile @@ -37,11 +37,11 @@ all: regenerate regenerate: rm -f test.pb.go make test.pb.go - + # The following rules are just aids to development. Not needed for typical testing. diff: regenerate - hg diff test.pb.go + git diff test.pb.go restore: cp test.pb.go.golden test.pb.go diff --git a/proto/text.go b/proto/text.go index 279c976..426db1e 100644 --- a/proto/text.go +++ b/proto/text.go @@ -244,6 +244,35 @@ func writeStruct(w *textWriter, sv reflect.Value) error { } continue } + if props.proto3 && fv.Kind() == reflect.Slice && fv.Len() == 0 { + // empty bytes field + continue + } + if fv.Kind() != reflect.Ptr && fv.Kind() != reflect.Slice { + // proto3 non-repeated scalar field; skip if zero value + switch fv.Kind() { + case reflect.Bool: + if !fv.Bool() { + continue + } + case reflect.Int32, reflect.Int64: + if fv.Int() == 0 { + continue + } + case reflect.Uint32, reflect.Uint64: + if fv.Uint() == 0 { + continue + } + case reflect.Float32, reflect.Float64: + if fv.Float() == 0 { + continue + } + case reflect.String: + if fv.String() == "" { + continue + } + } + } if err := writeName(w, props); err != nil { return err diff --git a/proto/text_parser.go b/proto/text_parser.go index 4885bd7..f733f30 100644 --- a/proto/text_parser.go +++ b/proto/text_parser.go @@ -409,6 +409,10 @@ func (p *textParser) checkForColon(props *Properties, typ reflect.Type) *ParseEr if typ.Elem().Kind() != reflect.Ptr { break } + } else if typ.Kind() == reflect.String { + // The proto3 exception is for a string field, + // which requires a colon. + break } needColon = false } diff --git a/proto/text_parser_test.go b/proto/text_parser_test.go index b9268ee..89ab106 100644 --- a/proto/text_parser_test.go +++ b/proto/text_parser_test.go @@ -36,6 +36,7 @@ import ( "reflect" "testing" + proto3pb "./proto3_proto" . "./testdata" . "github.com/golang/protobuf/proto" ) @@ -443,6 +444,21 @@ func TestRepeatedEnum(t *testing.T) { } } +func TestProto3TextParsing(t *testing.T) { + m := new(proto3pb.Message) + const in = `name: "Wallace" true_scotsman: true` + want := &proto3pb.Message{ + Name: "Wallace", + TrueScotsman: true, + } + if err := UnmarshalText(in, m); err != nil { + t.Fatal(err) + } + if !Equal(m, want) { + t.Errorf("\n got %v\nwant %v", m, want) + } +} + var benchInput string func init() { diff --git a/proto/text_test.go b/proto/text_test.go index 900ba6a..404920e 100644 --- a/proto/text_test.go +++ b/proto/text_test.go @@ -41,6 +41,7 @@ import ( "github.com/golang/protobuf/proto" + proto3pb "./proto3_proto" pb "./testdata" ) @@ -406,3 +407,23 @@ Message <nil> t.Errorf(" got: %s\nwant: %s", s, want) } } + +func TestProto3Text(t *testing.T) { + tests := []struct { + m proto.Message + want string + }{ + // zero message + {&proto3pb.Message{}, ``}, + // zero message except for an empty byte slice + {&proto3pb.Message{Data: []byte{}}, ``}, + // trivial case + {&proto3pb.Message{Name: "Rob", HeightInCm: 175}, `name:"Rob" height_in_cm:175`}, + } + for _, test := range tests { + got := strings.TrimSpace(test.m.String()) + if got != test.want { + t.Errorf("\n got %s\nwant %s", got, test.want) + } + } +} diff --git a/protoc-gen-go/generator/generator.go b/protoc-gen-go/generator/generator.go index 8882ed6..4b309cb 100644 --- a/protoc-gen-go/generator/generator.go +++ b/protoc-gen-go/generator/generator.go @@ -96,6 +96,12 @@ func (c *common) PackageName() string { return uniquePackageOf(c.file) } func (c *common) File() *descriptor.FileDescriptorProto { return c.file } +func fileIsProto3(file *descriptor.FileDescriptorProto) bool { + return file.GetSyntax() == "proto3" +} + +func (c *common) proto3() bool { return fileIsProto3(c.file) } + // Descriptor represents a protocol buffer message. type Descriptor struct { common @@ -243,6 +249,7 @@ type FileDescriptor struct { index int // The index of this file in the list of files to generate code for + proto3 bool // whether to generate proto3 code for this file } // PackageName is the package name we'll use in the generated code to refer to this file. @@ -668,6 +675,7 @@ func (g *Generator) WrapTypes() { ext: exts, imp: imps, exported: make(map[Object][]symbol), + proto3: fileIsProto3(f), } extractComments(fd) g.allFiles[i] = fd @@ -1116,7 +1124,9 @@ func (g *Generator) generateImports() { // reference it later. The same argument applies to the math package, // for handling bit patterns for floating-point numbers. g.P("import " + g.Pkg["proto"] + " " + strconv.Quote(g.ImportPrefix+"github.com/golang/protobuf/proto")) - g.P("import " + g.Pkg["math"] + ` "math"`) + if !g.file.proto3 { + g.P("import " + g.Pkg["math"] + ` "math"`) + } for i, s := range g.file.Dependency { fd := g.fileByName(s) // Do not import our own package. @@ -1154,7 +1164,9 @@ func (g *Generator) generateImports() { } g.P("// Reference imports to suppress errors if they are not otherwise used.") g.P("var _ = ", g.Pkg["proto"], ".Marshal") - g.P("var _ = ", g.Pkg["math"], ".Inf") + if !g.file.proto3 { + g.P("var _ = ", g.Pkg["math"], ".Inf") + } g.P() } @@ -1227,13 +1239,15 @@ func (g *Generator) generateEnum(enum *EnumDescriptor) { g.Out() g.P("}") - g.P("func (x ", ccTypeName, ") Enum() *", ccTypeName, " {") - g.In() - g.P("p := new(", ccTypeName, ")") - g.P("*p = x") - g.P("return p") - g.Out() - g.P("}") + if !enum.proto3() { + g.P("func (x ", ccTypeName, ") Enum() *", ccTypeName, " {") + g.In() + g.P("p := new(", ccTypeName, ")") + g.P("*p = x") + g.P("return p") + g.Out() + g.P("}") + } g.P("func (x ", ccTypeName, ") String() string {") g.In() @@ -1241,18 +1255,20 @@ func (g *Generator) generateEnum(enum *EnumDescriptor) { g.Out() g.P("}") - g.P("func (x *", ccTypeName, ") UnmarshalJSON(data []byte) error {") - g.In() - g.P("value, err := ", g.Pkg["proto"], ".UnmarshalJSONEnum(", ccTypeName, `_value, data, "`, ccTypeName, `")`) - g.P("if err != nil {") - g.In() - g.P("return err") - g.Out() - g.P("}") - g.P("*x = ", ccTypeName, "(value)") - g.P("return nil") - g.Out() - g.P("}") + if !enum.proto3() { + g.P("func (x *", ccTypeName, ") UnmarshalJSON(data []byte) error {") + g.In() + g.P("value, err := ", g.Pkg["proto"], ".UnmarshalJSONEnum(", ccTypeName, `_value, data, "`, ccTypeName, `")`) + g.P("if err != nil {") + g.In() + g.P("return err") + g.Out() + g.P("}") + g.P("*x = ", ccTypeName, "(value)") + g.P("return nil") + g.Out() + g.P("}") + } g.P() } @@ -1266,6 +1282,7 @@ func (g *Generator) generateEnum(enum *EnumDescriptor) { // packed whether the encoding is "packed" (optional; repeated primitives only) // name= the original declared name // enum= the name of the enum type if it is an enum-typed field. +// proto3 if this field is in a proto3 message // def= string representation of the default value, if any. // The default value must be in a representation that can be used at run-time // to generate the default value. Thus bools become 0 and 1, for instance. @@ -1348,6 +1365,13 @@ func (g *Generator) goTag(message *Descriptor, field *descriptor.FieldDescriptor } else { name = ",name=" + name } + if message.proto3() { + // We only need the extra tag for []byte fields; + // no need to add noise for the others. + if *field.Type == descriptor.FieldDescriptorProto_TYPE_BYTES { + name += ",proto3" + } + } return strconv.Quote(fmt.Sprintf("%s,%d,%s%s%s%s%s", wiretype, field.GetNumber(), @@ -1433,6 +1457,8 @@ func (g *Generator) GoType(message *Descriptor, field *descriptor.FieldDescripto } if isRepeated(field) { typ = "[]" + typ + } else if message != nil && message.proto3() { + return } else if needsStar(*field.Type) { typ = "*" + typ } @@ -1498,7 +1524,9 @@ func (g *Generator) generateMessage(message *Descriptor) { if len(message.ExtensionRange) > 0 { g.P("XXX_extensions\t\tmap[int32]", g.Pkg["proto"], ".Extension `json:\"-\"`") } - g.P("XXX_unrecognized\t[]byte `json:\"-\"`") + if !message.proto3() { + g.P("XXX_unrecognized\t[]byte `json:\"-\"`") + } g.Out() g.P("}") @@ -1634,6 +1662,11 @@ func (g *Generator) generateMessage(message *Descriptor) { star = "*" } + // In proto3, only generate getters for message fields. + if message.proto3() && *field.Type != descriptor.FieldDescriptorProto_TYPE_MESSAGE { + continue + } + // Only export getter symbols for basic types, // and for messages and enums in the same package. // Groups are not exported. diff --git a/protoc-gen-go/testdata/proto3.proto b/protoc-gen-go/testdata/proto3.proto new file mode 100644 index 0000000..ce192ae --- /dev/null +++ b/protoc-gen-go/testdata/proto3.proto @@ -0,0 +1,52 @@ +// Go support for Protocol Buffers - Google's data interchange format +// +// Copyright 2014 The Go Authors. All rights reserved. +// https://github.com/golang/protobuf +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package proto3; + +message Request { + enum Flavour { + SWEET = 0; + SOUR = 1; + UMAMI = 2; + GOPHERLICIOUS = 3; + } + string name = 1; // "optional" may be omitted + repeated int64 key = 2; + optional Flavour taste = 3; + Book book = 4; +} + +message Book { + string title = 1; + bytes raw_data = 2; +} |