Sync from Piper @304070343 PROTOBUF_SYNC_PIPER
diff --git a/python/google/protobuf/descriptor.py b/python/google/protobuf/descriptor.py index 204f89b..5ef81a2 100755 --- a/python/google/protobuf/descriptor.py +++ b/python/google/protobuf/descriptor.py
@@ -35,6 +35,7 @@ __author__ = 'robinson@google.com (Will Robinson)' import threading +import warnings import six from google.protobuf.internal import api_implementation @@ -91,6 +92,25 @@ _lock = threading.Lock() +def _Deprecated(name): + if _Deprecated.count > 0: + _Deprecated.count -= 1 + warnings.warn( + 'Call to deprecated create function %s(). Note: Create unlinked ' + 'descriptors is going to go away. Please use get/find descriptors from ' + 'generated code or query the descriptor_pool.' + % name, + category=DeprecationWarning, stacklevel=3) + + +# Deprecated warnings will print 100 times at most which should be enough for +# users to notice and do not cause timeout. +_Deprecated.count = 100 + + +_internal_create_key = object() + + class DescriptorBase(six.with_metaclass(DescriptorMetaclass)): """Descriptors base class. @@ -271,7 +291,7 @@ serialized_options=None, is_extendable=True, extension_ranges=None, oneofs=None, file=None, serialized_start=None, serialized_end=None, # pylint: disable=redefined-builtin - syntax=None): + syntax=None, create_key=None): _message.Message._CheckCalledFromGeneratedFile() return _message.default_pool.FindMessageTypeByName(full_name) @@ -283,13 +303,16 @@ serialized_options=None, is_extendable=True, extension_ranges=None, oneofs=None, file=None, serialized_start=None, serialized_end=None, # pylint: disable=redefined-builtin - syntax=None): + syntax=None, create_key=None): """Arguments to __init__() are as described in the description of Descriptor fields above. Note that filename is an obsolete argument, that is not used anymore. Please use file.name to access this as an attribute. """ + if create_key is not _internal_create_key: + _Deprecated('Descriptor') + super(Descriptor, self).__init__( options, 'MessageOptions', name, full_name, file, containing_type, serialized_start=serialized_start, @@ -515,7 +538,7 @@ is_extension, extension_scope, options=None, serialized_options=None, has_default_value=True, containing_oneof=None, json_name=None, - file=None): # pylint: disable=redefined-builtin + file=None, create_key=None): # pylint: disable=redefined-builtin _message.Message._CheckCalledFromGeneratedFile() if is_extension: return _message.default_pool.FindExtensionByName(full_name) @@ -527,7 +550,7 @@ is_extension, extension_scope, options=None, serialized_options=None, has_default_value=True, containing_oneof=None, json_name=None, - file=None): # pylint: disable=redefined-builtin + file=None, create_key=None): # pylint: disable=redefined-builtin """The arguments are as described in the description of FieldDescriptor attributes above. @@ -535,6 +558,9 @@ (to deal with circular references between message types, for example). Likewise for extension_scope. """ + if create_key is not _internal_create_key: + _Deprecated('FieldDescriptor') + super(FieldDescriptor, self).__init__( options, serialized_options, 'FieldOptions') self.name = name @@ -628,19 +654,22 @@ def __new__(cls, name, full_name, filename, values, containing_type=None, options=None, serialized_options=None, file=None, # pylint: disable=redefined-builtin - serialized_start=None, serialized_end=None): + serialized_start=None, serialized_end=None, create_key=None): _message.Message._CheckCalledFromGeneratedFile() return _message.default_pool.FindEnumTypeByName(full_name) def __init__(self, name, full_name, filename, values, containing_type=None, options=None, serialized_options=None, file=None, # pylint: disable=redefined-builtin - serialized_start=None, serialized_end=None): + serialized_start=None, serialized_end=None, create_key=None): """Arguments are as described in the attribute description above. Note that filename is an obsolete argument, that is not used anymore. Please use file.name to access this as an attribute. """ + if create_key is not _internal_create_key: + _Deprecated('EnumDescriptor') + super(EnumDescriptor, self).__init__( options, 'EnumOptions', name, full_name, file, containing_type, serialized_start=serialized_start, @@ -684,7 +713,7 @@ def __new__(cls, name, index, number, type=None, # pylint: disable=redefined-builtin - options=None, serialized_options=None): + options=None, serialized_options=None, create_key=None): _message.Message._CheckCalledFromGeneratedFile() # There is no way we can build a complete EnumValueDescriptor with the # given parameters (the name of the Enum is not known, for example). @@ -694,8 +723,11 @@ def __init__(self, name, index, number, type=None, # pylint: disable=redefined-builtin - options=None, serialized_options=None): + options=None, serialized_options=None, create_key=None): """Arguments are as described in the attribute description above.""" + if create_key is not _internal_create_key: + _Deprecated('EnumValueDescriptor') + super(EnumValueDescriptor, self).__init__( options, serialized_options, 'EnumValueOptions') self.name = name @@ -724,14 +756,17 @@ def __new__( cls, name, full_name, index, containing_type, fields, options=None, - serialized_options=None): + serialized_options=None, create_key=None): _message.Message._CheckCalledFromGeneratedFile() return _message.default_pool.FindOneofByName(full_name) def __init__( self, name, full_name, index, containing_type, fields, options=None, - serialized_options=None): + serialized_options=None, create_key=None): """Arguments are as described in the attribute description above.""" + if create_key is not _internal_create_key: + _Deprecated('OneofDescriptor') + super(OneofDescriptor, self).__init__( options, serialized_options, 'OneofOptions') self.name = name @@ -765,13 +800,16 @@ def __new__(cls, name, full_name, index, methods, options=None, serialized_options=None, file=None, # pylint: disable=redefined-builtin - serialized_start=None, serialized_end=None): + serialized_start=None, serialized_end=None, create_key=None): _message.Message._CheckCalledFromGeneratedFile() # pylint: disable=protected-access return _message.default_pool.FindServiceByName(full_name) def __init__(self, name, full_name, index, methods, options=None, serialized_options=None, file=None, # pylint: disable=redefined-builtin - serialized_start=None, serialized_end=None): + serialized_start=None, serialized_end=None, create_key=None): + if create_key is not _internal_create_key: + _Deprecated('ServiceDescriptor') + super(ServiceDescriptor, self).__init__( options, 'ServiceOptions', name, full_name, file, None, serialized_start=serialized_start, @@ -826,17 +864,22 @@ _C_DESCRIPTOR_CLASS = _message.MethodDescriptor def __new__(cls, name, full_name, index, containing_service, - input_type, output_type, options=None, serialized_options=None): + input_type, output_type, options=None, serialized_options=None, + create_key=None): _message.Message._CheckCalledFromGeneratedFile() # pylint: disable=protected-access return _message.default_pool.FindMethodByName(full_name) def __init__(self, name, full_name, index, containing_service, - input_type, output_type, options=None, serialized_options=None): + input_type, output_type, options=None, serialized_options=None, + create_key=None): """The arguments are as described in the description of MethodDescriptor attributes above. Note that containing_service may be None, and may be set later if necessary. """ + if create_key is not _internal_create_key: + _Deprecated('MethodDescriptor') + super(MethodDescriptor, self).__init__( options, serialized_options, 'MethodOptions') self.name = name @@ -884,11 +927,11 @@ def __new__(cls, name, package, options=None, serialized_options=None, serialized_pb=None, dependencies=None, public_dependencies=None, - syntax=None, pool=None): + syntax=None, pool=None, create_key=None): # FileDescriptor() is called from various places, not only from generated # files, to register dynamic proto files and messages. # pylint: disable=g-explicit-bool-comparison - if serialized_pb == '': + if serialized_pb == b'': # Cpp generated code must be linked in if serialized_pb is '' try: return _message.default_pool.FindFileByName(name) @@ -902,8 +945,11 @@ def __init__(self, name, package, options=None, serialized_options=None, serialized_pb=None, dependencies=None, public_dependencies=None, - syntax=None, pool=None): + syntax=None, pool=None, create_key=None): """Constructor.""" + if create_key is not _internal_create_key: + _Deprecated('FileDescriptor') + super(FileDescriptor, self).__init__( options, serialized_options, 'FileOptions') @@ -1042,9 +1088,11 @@ for enum_proto in desc_proto.enum_type: full_name = '.'.join(full_message_name + [enum_proto.name]) enum_desc = EnumDescriptor( - enum_proto.name, full_name, None, [ - EnumValueDescriptor(enum_val.name, ii, enum_val.number) - for ii, enum_val in enumerate(enum_proto.value)]) + enum_proto.name, full_name, None, [ + EnumValueDescriptor(enum_val.name, ii, enum_val.number, + create_key=_internal_create_key) + for ii, enum_val in enumerate(enum_proto.value)], + create_key=_internal_create_key) enum_types[full_name] = enum_desc # Create Descriptors for nested types @@ -1083,10 +1131,11 @@ FieldDescriptor.ProtoTypeToCppProtoType(field_proto.type), field_proto.label, None, nested_desc, enum_desc, None, False, None, options=_OptionsOrNone(field_proto), has_default_value=False, - json_name=json_name) + json_name=json_name, create_key=_internal_create_key) fields.append(field) desc_name = '.'.join(full_message_name) return Descriptor(desc_proto.name, desc_name, None, None, fields, list(nested_types.values()), list(enum_types.values()), [], - options=_OptionsOrNone(desc_proto)) + options=_OptionsOrNone(desc_proto), + create_key=_internal_create_key)
diff --git a/python/google/protobuf/descriptor_pool.py b/python/google/protobuf/descriptor_pool.py index 078d39a..de9100b 100644 --- a/python/google/protobuf/descriptor_pool.py +++ b/python/google/protobuf/descriptor_pool.py
@@ -619,7 +619,7 @@ def FindAllExtensions(self, message_descriptor): """Gets all the known extensions of a given message. - Extensions have to be registered to this pool by calling + Extensions have to be registered to this pool by build related :func:`Add` or :func:`AddExtensionDescriptor`. Args: @@ -662,18 +662,7 @@ return try: - file_desc = self._ConvertFileProtoToFileDescriptor(file_proto) - for extension in file_desc.extensions_by_name.values(): - self._extensions_by_number[extension.containing_type][ - extension.number] = extension - self._extensions_by_name[extension.containing_type][ - extension.full_name] = extension - for message_type in file_desc.message_types_by_name.values(): - for extension in message_type.extensions: - self._extensions_by_number[extension.containing_type][ - extension.number] = extension - self._extensions_by_name[extension.containing_type][ - extension.full_name] = extension + self._ConvertFileProtoToFileDescriptor(file_proto) except: warn_msg = ('Unable to load proto file %s for extension number %d.' % (file_proto.name, number)) @@ -761,7 +750,9 @@ options=_OptionsOrNone(file_proto), serialized_pb=file_proto.SerializeToString(), dependencies=direct_deps, - public_dependencies=public_deps) + public_dependencies=public_deps, + # pylint: disable=protected-access + create_key=descriptor._internal_create_key) scope = {} # This loop extracts all the message and enum types from all the @@ -820,7 +811,15 @@ self.Add(file_proto) self._file_descriptors[file_proto.name] = file_descriptor - return self._file_descriptors[file_proto.name] + # Add extensions to the pool + file_desc = self._file_descriptors[file_proto.name] + for extension in file_desc.extensions_by_name.values(): + self._AddExtensionDescriptor(extension) + for message_type in file_desc.message_types_by_name.values(): + for extension in message_type.extensions: + self._AddExtensionDescriptor(extension) + + return file_desc def _ConvertMessageDescriptor(self, desc_proto, package=None, file_desc=None, scope=None, syntax=None): @@ -865,8 +864,11 @@ is_extension=True) for index, extension in enumerate(desc_proto.extension)] oneofs = [ + # pylint: disable=g-complex-comprehension descriptor.OneofDescriptor(desc.name, '.'.join((desc_name, desc.name)), - index, None, [], desc.options) + index, None, [], desc.options, + # pylint: disable=protected-access + create_key=descriptor._internal_create_key) for index, desc in enumerate(desc_proto.oneof_decl)] extension_ranges = [(r.start, r.end) for r in desc_proto.extension_range] if extension_ranges: @@ -889,7 +891,9 @@ file=file_desc, serialized_start=None, serialized_end=None, - syntax=syntax) + syntax=syntax, + # pylint: disable=protected-access + create_key=descriptor._internal_create_key) for nested in desc.nested_types: nested.containing_type = desc for enum in desc.enum_types: @@ -940,7 +944,9 @@ file=file_desc, values=values, containing_type=containing_type, - options=_OptionsOrNone(enum_proto)) + options=_OptionsOrNone(enum_proto), + # pylint: disable=protected-access + create_key=descriptor._internal_create_key) scope['.%s' % enum_name] = desc self._CheckConflictRegister(desc, desc.full_name, desc.file.name) self._enum_descriptors[enum_name] = desc @@ -997,7 +1003,9 @@ is_extension=is_extension, extension_scope=None, options=_OptionsOrNone(field_proto), - file=file_desc) + file=file_desc, + # pylint: disable=protected-access + create_key=descriptor._internal_create_key) def _SetAllFieldTypes(self, package, desc_proto, scope): """Sets all the descriptor's fields's types. @@ -1121,7 +1129,9 @@ index=index, number=value_proto.number, options=_OptionsOrNone(value_proto), - type=None) + type=None, + # pylint: disable=protected-access + create_key=descriptor._internal_create_key) def _MakeServiceDescriptor(self, service_proto, service_index, scope, package, file_desc): @@ -1146,12 +1156,15 @@ methods = [self._MakeMethodDescriptor(method_proto, service_name, package, scope, index) for index, method_proto in enumerate(service_proto.method)] - desc = descriptor.ServiceDescriptor(name=service_proto.name, - full_name=service_name, - index=service_index, - methods=methods, - options=_OptionsOrNone(service_proto), - file=file_desc) + desc = descriptor.ServiceDescriptor( + name=service_proto.name, + full_name=service_name, + index=service_index, + methods=methods, + options=_OptionsOrNone(service_proto), + file=file_desc, + # pylint: disable=protected-access + create_key=descriptor._internal_create_key) self._CheckConflictRegister(desc, desc.full_name, desc.file.name) self._service_descriptors[service_name] = desc return desc @@ -1175,13 +1188,16 @@ package, method_proto.input_type, scope) output_type = self._GetTypeFromScope( package, method_proto.output_type, scope) - return descriptor.MethodDescriptor(name=method_proto.name, - full_name=full_name, - index=index, - containing_service=None, - input_type=input_type, - output_type=output_type, - options=_OptionsOrNone(method_proto)) + return descriptor.MethodDescriptor( + name=method_proto.name, + full_name=full_name, + index=index, + containing_service=None, + input_type=input_type, + output_type=output_type, + options=_OptionsOrNone(method_proto), + # pylint: disable=protected-access + create_key=descriptor._internal_create_key) def _ExtractSymbols(self, descriptors): """Pulls out all the symbols from descriptor protos.
diff --git a/python/google/protobuf/internal/decoder.py b/python/google/protobuf/internal/decoder.py index c04df8e..092dab5 100755 --- a/python/google/protobuf/internal/decoder.py +++ b/python/google/protobuf/internal/decoder.py
@@ -816,8 +816,12 @@ if extension is not None: value = field_dict.get(extension) if value is None: + message_type = extension.message_type + if not hasattr(message_type, '_concrete_class'): + # pylint: disable=protected-access + message._FACTORY.GetPrototype(message_type) value = field_dict.setdefault( - extension, extension.message_type._concrete_class()) + extension, message_type._concrete_class()) if value._InternalParse(buffer, message_start,message_end) != message_end: # The only reason _InternalParse would return early is if it encountered # an end-group tag.
diff --git a/python/google/protobuf/internal/descriptor_pool_test.py b/python/google/protobuf/internal/descriptor_pool_test.py index ad1eb65..fed82bf 100644 --- a/python/google/protobuf/internal/descriptor_pool_test.py +++ b/python/google/protobuf/internal/descriptor_pool_test.py
@@ -36,6 +36,7 @@ import copy import os +import warnings try: import unittest2 as unittest #PY26 @@ -63,6 +64,9 @@ +warnings.simplefilter('error', DeprecationWarning) + + class DescriptorPoolTestBase(object): def testFindFileByName(self): @@ -336,12 +340,10 @@ 'google.protobuf.python.internal.Factory2Message') # An extension defined in a message. one_more_field = factory2_message.extensions_by_name['one_more_field'] - self.pool.AddExtensionDescriptor(one_more_field) # An extension defined at file scope. factory_test2 = self.pool.FindFileByName( 'google/protobuf/internal/factory_test2.proto') another_field = factory_test2.extensions_by_name['another_field'] - self.pool.AddExtensionDescriptor(another_field) extensions = self.pool.FindAllExtensions(factory1_message) expected_extension_numbers = set([one_more_field, another_field]) @@ -356,16 +358,9 @@ def testFindExtensionByNumber(self): factory1_message = self.pool.FindMessageTypeByName( 'google.protobuf.python.internal.Factory1Message') - factory2_message = self.pool.FindMessageTypeByName( - 'google.protobuf.python.internal.Factory2Message') - # An extension defined in a message. - one_more_field = factory2_message.extensions_by_name['one_more_field'] - self.pool.AddExtensionDescriptor(one_more_field) - # An extension defined at file scope. - factory_test2 = self.pool.FindFileByName( + # Build factory_test2.proto which will put extensions to the pool + self.pool.FindFileByName( 'google/protobuf/internal/factory_test2.proto') - another_field = factory_test2.extensions_by_name['another_field'] - self.pool.AddExtensionDescriptor(another_field) # An extension defined in a message. extension = self.pool.FindExtensionByNumber(factory1_message, 1001) @@ -533,13 +528,13 @@ else: pool = copy.deepcopy(self.pool) file_descriptor = unittest_pb2.DESCRIPTOR - pool.AddDescriptor( + pool._AddDescriptor( file_descriptor.message_types_by_name['TestAllTypes']) - pool.AddEnumDescriptor( + pool._AddEnumDescriptor( file_descriptor.enum_types_by_name['ForeignEnum']) - pool.AddServiceDescriptor( + pool._AddServiceDescriptor( file_descriptor.services_by_name['TestService']) - pool.AddExtensionDescriptor( + pool._AddExtensionDescriptor( file_descriptor.extensions_by_name['optional_int32_extension']) pool.Add(unittest_fd) pool.Add(conflict_fd) @@ -873,7 +868,7 @@ def _TestMessage(self, prefix): pool = descriptor_pool.DescriptorPool() - pool.AddDescriptor(unittest_pb2.TestAllTypes.DESCRIPTOR) + pool._AddDescriptor(unittest_pb2.TestAllTypes.DESCRIPTOR) self.assertEqual( 'protobuf_unittest.TestAllTypes', pool.FindMessageTypeByName( @@ -884,7 +879,7 @@ pool.FindMessageTypeByName( prefix + 'protobuf_unittest.TestAllTypes.NestedMessage') - pool.AddDescriptor(unittest_pb2.TestAllTypes.NestedMessage.DESCRIPTOR) + pool._AddDescriptor(unittest_pb2.TestAllTypes.NestedMessage.DESCRIPTOR) self.assertEqual( 'protobuf_unittest.TestAllTypes.NestedMessage', pool.FindMessageTypeByName( @@ -909,7 +904,10 @@ def _TestEnum(self, prefix): pool = descriptor_pool.DescriptorPool() - pool.AddEnumDescriptor(unittest_pb2.ForeignEnum.DESCRIPTOR) + if api_implementation.Type() == 'cpp': + pool.AddEnumDescriptor(unittest_pb2.ForeignEnum.DESCRIPTOR) + else: + pool._AddEnumDescriptor(unittest_pb2.ForeignEnum.DESCRIPTOR) self.assertEqual( 'protobuf_unittest.ForeignEnum', pool.FindEnumTypeByName( @@ -920,7 +918,10 @@ pool.FindEnumTypeByName( prefix + 'protobuf_unittest.ForeignEnum.NestedEnum') - pool.AddEnumDescriptor(unittest_pb2.TestAllTypes.NestedEnum.DESCRIPTOR) + if api_implementation.Type() == 'cpp': + pool.AddEnumDescriptor(unittest_pb2.TestAllTypes.NestedEnum.DESCRIPTOR) + else: + pool._AddEnumDescriptor(unittest_pb2.TestAllTypes.NestedEnum.DESCRIPTOR) self.assertEqual( 'protobuf_unittest.TestAllTypes.NestedEnum', pool.FindEnumTypeByName( @@ -949,7 +950,7 @@ pool = descriptor_pool.DescriptorPool() with self.assertRaises(KeyError): pool.FindServiceByName('protobuf_unittest.TestService') - pool.AddServiceDescriptor(unittest_pb2._TESTSERVICE) + pool._AddServiceDescriptor(unittest_pb2._TESTSERVICE) self.assertEqual( 'protobuf_unittest.TestService', pool.FindServiceByName('protobuf_unittest.TestService').full_name) @@ -958,7 +959,7 @@ 'With the cpp implementation, Add() must be called first') def testFile(self): pool = descriptor_pool.DescriptorPool() - pool.AddFileDescriptor(unittest_pb2.DESCRIPTOR) + pool._AddFileDescriptor(unittest_pb2.DESCRIPTOR) self.assertEqual( 'google/protobuf/unittest.proto', pool.FindFileByName( @@ -1032,16 +1033,28 @@ def testAddTypeError(self): pool = descriptor_pool.DescriptorPool() - with self.assertRaises(TypeError): - pool.AddDescriptor(0) - with self.assertRaises(TypeError): - pool.AddEnumDescriptor(0) - with self.assertRaises(TypeError): - pool.AddServiceDescriptor(0) - with self.assertRaises(TypeError): - pool.AddExtensionDescriptor(0) - with self.assertRaises(TypeError): - pool.AddFileDescriptor(0) + if api_implementation.Type() == 'cpp': + with self.assertRaises(TypeError): + pool.AddDescriptor(0) + with self.assertRaises(TypeError): + pool.AddEnumDescriptor(0) + with self.assertRaises(TypeError): + pool.AddServiceDescriptor(0) + with self.assertRaises(TypeError): + pool.AddExtensionDescriptor(0) + with self.assertRaises(TypeError): + pool.AddFileDescriptor(0) + else: + with self.assertRaises(TypeError): + pool._AddDescriptor(0) + with self.assertRaises(TypeError): + pool._AddEnumDescriptor(0) + with self.assertRaises(TypeError): + pool._AddServiceDescriptor(0) + with self.assertRaises(TypeError): + pool._AddExtensionDescriptor(0) + with self.assertRaises(TypeError): + pool._AddFileDescriptor(0) TEST1_FILE = ProtoFile(
diff --git a/python/google/protobuf/internal/descriptor_test.py b/python/google/protobuf/internal/descriptor_test.py index bff2d5f..5e3b0a9 100755 --- a/python/google/protobuf/internal/descriptor_test.py +++ b/python/google/protobuf/internal/descriptor_test.py
@@ -35,6 +35,7 @@ __author__ = 'robinson@google.com (Will Robinson)' import sys +import warnings try: import unittest2 as unittest #PY26 @@ -58,6 +59,9 @@ """ +warnings.simplefilter('error', DeprecationWarning) + + class DescriptorTest(unittest.TestCase): def setUp(self):
diff --git a/python/google/protobuf/internal/extension_dict.py b/python/google/protobuf/internal/extension_dict.py index d08df4a..b346cf2 100644 --- a/python/google/protobuf/internal/extension_dict.py +++ b/python/google/protobuf/internal/extension_dict.py
@@ -87,6 +87,10 @@ if extension_handle.label == FieldDescriptor.LABEL_REPEATED: result = extension_handle._default_constructor(self._extended_message) elif extension_handle.cpp_type == FieldDescriptor.CPPTYPE_MESSAGE: + message_type = extension_handle.message_type + if not hasattr(message_type, '_concrete_class'): + # pylint: disable=protected-access + self._extended_message._FACTORY.GetPrototype(message_type) assert getattr(extension_handle.message_type, '_concrete_class', None), ( 'Uninitialized concrete class found for field %r (message type %r)' % (extension_handle.full_name,
diff --git a/python/google/protobuf/internal/json_format_test.py b/python/google/protobuf/internal/json_format_test.py index e3992f9..d5245f4 100644 --- a/python/google/protobuf/internal/json_format_test.py +++ b/python/google/protobuf/internal/json_format_test.py
@@ -821,6 +821,10 @@ def testFloatPrecision(self): message = json_format_proto3_pb2.TestMessage() message.float_value = 1.123456789 + # Default to 8 valid digits. + text = '{\n "floatValue": 1.1234568\n}' + self.assertEqual( + json_format.MessageToJson(message), text) # Set to 7 valid digits. text = '{\n "floatValue": 1.123457\n}' self.assertEqual(
diff --git a/python/google/protobuf/internal/message_test.py b/python/google/protobuf/internal/message_test.py index f12b531..097c082 100755 --- a/python/google/protobuf/internal/message_test.py +++ b/python/google/protobuf/internal/message_test.py
@@ -106,6 +106,9 @@ return isinf(val) and (val < 0) +warnings.simplefilter('error', DeprecationWarning) + + @_parameterized.named_parameters( ('_proto2', unittest_pb2), ('_proto3', unittest_proto3_arena_pb2)) @@ -438,12 +441,19 @@ self.assertEqual(str(message), 'optional_float: 2.0\n') def testHighPrecisionFloatPrinting(self, message_module): - message = message_module.TestAllTypes() - message.optional_double = 0.12345678912345678 + msg = message_module.TestAllTypes() + msg.optional_float = 0.12345678912345678 + old_float = msg.optional_float + msg.ParseFromString(msg.SerializeToString()) + self.assertEqual(old_float, msg.optional_float) + + def testHighPrecisionDoublePrinting(self, message_module): + msg = message_module.TestAllTypes() + msg.optional_double = 0.12345678912345678 if sys.version_info >= (3,): - self.assertEqual(str(message), 'optional_double: 0.12345678912345678\n') + self.assertEqual(str(msg), 'optional_double: 0.12345678912345678\n') else: - self.assertEqual(str(message), 'optional_double: 0.123456789123\n') + self.assertEqual(str(msg), 'optional_double: 0.123456789123\n') def testUnknownFieldPrinting(self, message_module): populated = message_module.TestAllTypes() @@ -885,11 +895,13 @@ def testOneofDefaultValues(self, message_module): m = message_module.TestAllTypes() self.assertIs(None, m.WhichOneof('oneof_field')) + self.assertFalse(m.HasField('oneof_field')) self.assertFalse(m.HasField('oneof_uint32')) # Oneof is set even when setting it to a default value. m.oneof_uint32 = 0 self.assertEqual('oneof_uint32', m.WhichOneof('oneof_field')) + self.assertTrue(m.HasField('oneof_field')) self.assertTrue(m.HasField('oneof_uint32')) self.assertFalse(m.HasField('oneof_string'))
diff --git a/python/google/protobuf/internal/python_message.py b/python/google/protobuf/internal/python_message.py index 8fb1946..0691a4c 100755 --- a/python/google/protobuf/internal/python_message.py +++ b/python/google/protobuf/internal/python_message.py
@@ -124,9 +124,16 @@ Returns: Newly-allocated class. + + Raises: + RuntimeError: Generated code only work with python cpp extension. """ descriptor = dictionary[GeneratedProtocolMessageType._DESCRIPTOR_KEY] + if isinstance(descriptor, str): + raise RuntimeError('The generated code only work with python cpp ' + 'extension, but it is using pure python runtime.') + # If a concrete class already exists for this descriptor, don't try to # create another. Doing so will break any messages that already exist with # the existing class. @@ -838,10 +845,9 @@ continue hassable_fields[field.name] = field - if not is_proto3: - # Fields inside oneofs are never repeated (enforced by the compiler). - for oneof in message_descriptor.oneofs: - hassable_fields[oneof.name] = oneof + # Has methods are supported for oneof descriptors. + for oneof in message_descriptor.oneofs: + hassable_fields[oneof.name] = oneof def HasField(self, field_name): try:
diff --git a/python/google/protobuf/internal/reflection_test.py b/python/google/protobuf/internal/reflection_test.py index dace154..e67248f 100755 --- a/python/google/protobuf/internal/reflection_test.py +++ b/python/google/protobuf/internal/reflection_test.py
@@ -40,6 +40,7 @@ import operator import six import struct +import warnings try: import unittest2 as unittest #PY26 @@ -70,6 +71,9 @@ long = int # pylint: disable=redefined-builtin,invalid-name +warnings.simplefilter('error', DeprecationWarning) + + class _MiniDecoder(object): """Decodes a stream of values from a string. @@ -1433,12 +1437,16 @@ label=FieldDescriptor.LABEL_OPTIONAL, default_value=0, containing_type=None, message_type=None, enum_type=None, is_extension=False, extension_scope=None, - options=descriptor_pb2.FieldOptions()) + options=descriptor_pb2.FieldOptions(), + # pylint: disable=protected-access + create_key=descriptor._internal_create_key) mydescriptor = descriptor.Descriptor( name='MyProto', full_name='MyProto', filename='ignored', containing_type=None, nested_types=[], enum_types=[], fields=[foo_field_descriptor], extensions=[], - options=descriptor_pb2.MessageOptions()) + options=descriptor_pb2.MessageOptions(), + # pylint: disable=protected-access + create_key=descriptor._internal_create_key) class MyProtoClass(six.with_metaclass(reflection.GeneratedProtocolMessageType, message.Message)): DESCRIPTOR = mydescriptor myproto_instance = MyProtoClass() @@ -3168,22 +3176,34 @@ 'C++ implementation requires a call to MakeDescriptor()') @testing_refleaks.SkipReferenceLeakChecker('MakeClass is not repeatable') def testMakeClassWithNestedDescriptor(self): - leaf_desc = descriptor.Descriptor('leaf', 'package.parent.child.leaf', '', - containing_type=None, fields=[], - nested_types=[], enum_types=[], - extensions=[]) - child_desc = descriptor.Descriptor('child', 'package.parent.child', '', - containing_type=None, fields=[], - nested_types=[leaf_desc], enum_types=[], - extensions=[]) - sibling_desc = descriptor.Descriptor('sibling', 'package.parent.sibling', - '', containing_type=None, fields=[], - nested_types=[], enum_types=[], - extensions=[]) - parent_desc = descriptor.Descriptor('parent', 'package.parent', '', - containing_type=None, fields=[], - nested_types=[child_desc, sibling_desc], - enum_types=[], extensions=[]) + leaf_desc = descriptor.Descriptor( + 'leaf', 'package.parent.child.leaf', '', + containing_type=None, fields=[], + nested_types=[], enum_types=[], + extensions=[], + # pylint: disable=protected-access + create_key=descriptor._internal_create_key) + child_desc = descriptor.Descriptor( + 'child', 'package.parent.child', '', + containing_type=None, fields=[], + nested_types=[leaf_desc], enum_types=[], + extensions=[], + # pylint: disable=protected-access + create_key=descriptor._internal_create_key) + sibling_desc = descriptor.Descriptor( + 'sibling', 'package.parent.sibling', + '', containing_type=None, fields=[], + nested_types=[], enum_types=[], + extensions=[], + # pylint: disable=protected-access + create_key=descriptor._internal_create_key) + parent_desc = descriptor.Descriptor( + 'parent', 'package.parent', '', + containing_type=None, fields=[], + nested_types=[child_desc, sibling_desc], + enum_types=[], extensions=[], + # pylint: disable=protected-access + create_key=descriptor._internal_create_key) reflection.MakeClass(parent_desc) def _GetSerializedFileDescriptor(self, name): @@ -3305,4 +3325,3 @@ if __name__ == '__main__': unittest.main() -
diff --git a/python/google/protobuf/internal/text_format_test.py b/python/google/protobuf/internal/text_format_test.py index 8952088..b7d2333 100755 --- a/python/google/protobuf/internal/text_format_test.py +++ b/python/google/protobuf/internal/text_format_test.py
@@ -2194,5 +2194,23 @@ }""")) +class OptionalColonMessageToStringTest(unittest.TestCase): + + def testForcePrintOptionalColon(self): + packed_message = unittest_pb2.OneString() + packed_message.data = 'string' + message = any_test_pb2.TestAny() + message.any_value.Pack(packed_message) + output = text_format.MessageToString( + message, + force_colon=True) + expected = ('any_value: {\n' + ' [type.googleapis.com/protobuf_unittest.OneString]: {\n' + ' data: "string"\n' + ' }\n' + '}\n') + self.assertEqual(expected, output) + + if __name__ == '__main__': unittest.main()
diff --git a/python/google/protobuf/internal/type_checkers.py b/python/google/protobuf/internal/type_checkers.py index 8deba47..f769008 100755 --- a/python/google/protobuf/internal/type_checkers.py +++ b/python/google/protobuf/internal/type_checkers.py
@@ -45,6 +45,11 @@ __author__ = 'robinson@google.com (Will Robinson)' +try: + import ctypes +except Exception: # pylint: disable=broad-except + ctypes = None + import struct import numbers import six @@ -257,9 +262,10 @@ if converted_value < _FLOAT_MIN: return _NEG_INF - return converted_value - # TODO(jieluo): convert to 4 bytes float (c style float) at setters: - # return struct.unpack('f', struct.pack('f', converted_value)) + if ctypes: + return ctypes.c_float(converted_value).value + else: + return struct.unpack('<f', struct.pack('<f', converted_value))[0] def DefaultValue(self): return 0.0
diff --git a/python/google/protobuf/json_format.py b/python/google/protobuf/json_format.py index 5c8ef9e..865ffdd 100644 --- a/python/google/protobuf/json_format.py +++ b/python/google/protobuf/json_format.py
@@ -104,7 +104,7 @@ sort_keys=False, use_integers_for_enums=False, descriptor_pool=None, - float_precision=None): + float_precision=8): """Converts protobuf message to JSON format. Args: @@ -123,6 +123,7 @@ descriptor_pool: A Descriptor Pool for resolving types. If None use the default. float_precision: If set, use this to specify float field valid digits. + Otherwise, 8 valid digits is used (default '.8g'). Returns: A string containing the JSON formatted protocol buffer message. @@ -142,7 +143,7 @@ preserving_proto_field_name=False, use_integers_for_enums=False, descriptor_pool=None, - float_precision=None): + float_precision=8): """Converts protobuf message to a dictionary. When the dictionary is encoded to JSON, it conforms to proto3 JSON spec. @@ -160,6 +161,7 @@ descriptor_pool: A Descriptor Pool for resolving types. If None use the default. float_precision: If set, use this to specify float field valid digits. + Otherwise, 8 valid digits is used (default '.8g'). Returns: A dict representation of the protocol buffer message.
diff --git a/python/google/protobuf/message_factory.py b/python/google/protobuf/message_factory.py index f3ab0a5..24a524a 100644 --- a/python/google/protobuf/message_factory.py +++ b/python/google/protobuf/message_factory.py
@@ -83,6 +83,8 @@ descriptor_name, (message.Message,), {'DESCRIPTOR': descriptor, '__module__': None}) + # pylint: disable=protected-access + result_class._FACTORY = self # If module not set, it wrongly points to message_factory module. self._classes[descriptor] = result_class for field in descriptor.fields:
diff --git a/python/google/protobuf/pyext/message.cc b/python/google/protobuf/pyext/message.cc index 419f8f5..e4d6c7c 100644 --- a/python/google/protobuf/pyext/message.cc +++ b/python/google/protobuf/pyext/message.cc
@@ -1425,16 +1425,7 @@ } if (field_descriptor->file()->syntax() == FileDescriptor::SYNTAX_PROTO3) { - // HasField() for a oneof *itself* isn't supported. - if (in_oneof) { - PyErr_Format(PyExc_ValueError, - "Can't test oneof field \"%s.%s\" for presence in proto3, " - "use WhichOneof instead.", message_name.c_str(), - field_descriptor->containing_oneof()->name().c_str()); - return false; - } - - // ...but HasField() for fields *in* a oneof is supported. + // HasField() is supported for oneof fields. if (field_descriptor->containing_oneof() != NULL) { return true; }
diff --git a/python/google/protobuf/pyext/message_module.cc b/python/google/protobuf/pyext/message_module.cc index 4bb35b3..63d60b7 100644 --- a/python/google/protobuf/pyext/message_module.cc +++ b/python/google/protobuf/pyext/message_module.cc
@@ -107,9 +107,12 @@ } // Adds the C++ API - if (PyObject* api = - PyCapsule_New(new ApiImplementation(), - google::protobuf::python::PyProtoAPICapsuleName(), NULL)) { + if (PyObject* api = PyCapsule_New( + new ApiImplementation(), google::protobuf::python::PyProtoAPICapsuleName(), + [](PyObject* o) { + delete (ApiImplementation*)PyCapsule_GetPointer( + o, google::protobuf::python::PyProtoAPICapsuleName()); + })) { PyModule_AddObject(m, "proto_API", api); } else { return INITFUNC_ERRORVAL;
diff --git a/python/google/protobuf/text_format.py b/python/google/protobuf/text_format.py index bc9060b..9c2b267 100755 --- a/python/google/protobuf/text_format.py +++ b/python/google/protobuf/text_format.py
@@ -132,7 +132,8 @@ descriptor_pool=None, indent=0, message_formatter=None, - print_unknown_fields=False): + print_unknown_fields=False, + force_colon=False): # type: (...) -> str """Convert protobuf message to text format. @@ -170,17 +171,28 @@ Custom formatter for selected sub-messages (usually based on message type). Use to pretty print parts of the protobuf for easier diffing. print_unknown_fields: If True, unknown fields will be printed. + force_colon: If set, a colon will be added after the field name even if the + field is a proto message. Returns: str: A string of the text formatted protocol buffer message. """ out = TextWriter(as_utf8) - printer = _Printer(out, indent, as_utf8, as_one_line, - use_short_repeated_primitives, pointy_brackets, - use_index_order, float_format, double_format, - use_field_number, - descriptor_pool, message_formatter, - print_unknown_fields=print_unknown_fields) + printer = _Printer( + out, + indent, + as_utf8, + as_one_line, + use_short_repeated_primitives, + pointy_brackets, + use_index_order, + float_format, + double_format, + use_field_number, + descriptor_pool, + message_formatter, + print_unknown_fields=print_unknown_fields, + force_colon=force_colon) printer.PrintMessage(message) result = out.getvalue() out.close() @@ -218,7 +230,8 @@ use_field_number=False, descriptor_pool=None, message_formatter=None, - print_unknown_fields=False): + print_unknown_fields=False, + force_colon=False): printer = _Printer( out=out, indent=indent, as_utf8=as_utf8, as_one_line=as_one_line, @@ -230,7 +243,8 @@ use_field_number=use_field_number, descriptor_pool=descriptor_pool, message_formatter=message_formatter, - print_unknown_fields=print_unknown_fields) + print_unknown_fields=print_unknown_fields, + force_colon=force_colon) printer.PrintMessage(message) @@ -246,13 +260,15 @@ float_format=None, double_format=None, message_formatter=None, - print_unknown_fields=False): + print_unknown_fields=False, + force_colon=False): """Print a single field name/value pair.""" printer = _Printer(out, indent, as_utf8, as_one_line, use_short_repeated_primitives, pointy_brackets, use_index_order, float_format, double_format, message_formatter=message_formatter, - print_unknown_fields=print_unknown_fields) + print_unknown_fields=print_unknown_fields, + force_colon=force_colon) printer.PrintField(field, value) @@ -268,13 +284,15 @@ float_format=None, double_format=None, message_formatter=None, - print_unknown_fields=False): + print_unknown_fields=False, + force_colon=False): """Print a single field value (not including name).""" printer = _Printer(out, indent, as_utf8, as_one_line, use_short_repeated_primitives, pointy_brackets, use_index_order, float_format, double_format, message_formatter=message_formatter, - print_unknown_fields=print_unknown_fields) + print_unknown_fields=print_unknown_fields, + force_colon=force_colon) printer.PrintFieldValue(field, value) @@ -324,7 +342,8 @@ use_field_number=False, descriptor_pool=None, message_formatter=None, - print_unknown_fields=False): + print_unknown_fields=False, + force_colon=False): """Initialize the Printer. Double values can be formatted compactly with 15 digits of precision @@ -358,6 +377,8 @@ to custom format selected sub-messages (usually based on message type). Use to pretty print parts of the protobuf for easier diffing. print_unknown_fields: If True, unknown fields will be printed. + force_colon: If set, a colon will be added after the field name even if + the field is a proto message. """ self.out = out self.indent = indent @@ -375,6 +396,7 @@ self.descriptor_pool = descriptor_pool self.message_formatter = message_formatter self.print_unknown_fields = print_unknown_fields + self.force_colon = force_colon def _TryPrintAsAnyMessage(self, message): """Serializes if message is a google.protobuf.Any field.""" @@ -384,7 +406,8 @@ self.descriptor_pool) if packed_message: packed_message.MergeFromString(message.value) - self.out.write('%s[%s] ' % (self.indent * ' ', message.type_url)) + colon = ':' if self.force_colon else '' + self.out.write('%s[%s]%s ' % (self.indent * ' ', message.type_url, colon)) self._PrintMessageFieldValue(packed_message) self.out.write(' ' if self.as_one_line else '\n') return True @@ -518,9 +541,11 @@ else: out.write(field.name) - if field.cpp_type != descriptor.FieldDescriptor.CPPTYPE_MESSAGE: + if (self.force_colon or + field.cpp_type != descriptor.FieldDescriptor.CPPTYPE_MESSAGE): # The colon is optional in this case, but our cross-language golden files - # don't include it. + # don't include it. Here, the colon is only included if force_colon is + # set to True out.write(':') def PrintField(self, field, value): @@ -531,6 +556,7 @@ self.out.write(' ' if self.as_one_line else '\n') def _PrintShortRepeatedPrimitivesValue(self, field, value): + """"Prints short repeated primitives value.""" # Note: this is called only when value has at least one element. self._PrintFieldName(field) self.out.write(' [') @@ -539,6 +565,8 @@ self.out.write(', ') self.PrintFieldValue(field, value[-1]) self.out.write(']') + if self.force_colon: + self.out.write(':') self.out.write(' ' if self.as_one_line else '\n') def _PrintMessageFieldValue(self, value):
diff --git a/python/setup.py b/python/setup.py index 327658a..9aabbf7 100755 --- a/python/setup.py +++ b/python/setup.py
@@ -19,23 +19,17 @@ from distutils.command.clean import clean as _clean from distutils.spawn import find_executable - -current_dir = (os.path.dirname(__file__) or os.curdir) -current_dir_relative = os.path.relpath(current_dir) -src_dir = os.path.abspath(os.path.join(current_dir, "..", "src")) -vsprojects_dir = os.path.abspath(os.path.join(current_dir, "..", "vsprojects")) - # Find the Protocol Compiler. if 'PROTOC' in os.environ and os.path.exists(os.environ['PROTOC']): protoc = os.environ['PROTOC'] -elif os.path.exists(os.path.join(src_dir, "protoc")): - protoc = os.path.join(src_dir, "protoc") -elif os.path.exists(os.path.join(src_dir, "protoc.exe")): - protoc = os.path.join(src_dir, "protoc.exe") -elif os.path.exists(os.path.join(vsprojects_dir, "Debug", "protoc.exe")): - protoc = os.path.join(vsprojects_dir, "Debug", "protoc.exe") -elif os.path.exists(os.path.join(vsprojects_dir, "Release", "protoc.exe")): - protoc = os.path.join(vsprojects_dir, "Release", "protoc.exe") +elif os.path.exists("../src/protoc"): + protoc = "../src/protoc" +elif os.path.exists("../src/protoc.exe"): + protoc = "../src/protoc.exe" +elif os.path.exists("../vsprojects/Debug/protoc.exe"): + protoc = "../vsprojects/Debug/protoc.exe" +elif os.path.exists("../vsprojects/Release/protoc.exe"): + protoc = "../vsprojects/Release/protoc.exe" else: protoc = find_executable("protoc") @@ -46,7 +40,7 @@ Do not import google.protobuf.__init__ directly, because an installed protobuf library may be loaded instead.""" - with open(os.path.join(current_dir, 'google', 'protobuf', '__init__.py')) as version_file: + with open(os.path.join('google', 'protobuf', '__init__.py')) as version_file: exec(version_file.read(), globals()) global __version__ return __version__ @@ -57,21 +51,15 @@ .proto file. Does nothing if the output already exists and is newer than the input.""" - original_source = source - source = source.replace("../src", src_dir) - if not require and not os.path.exists(source): return - output = os.path.join( - current_dir, - original_source.replace(".proto", "_pb2.py").replace("../src/", "") - ) + output = source.replace(".proto", "_pb2.py").replace("../src/", "") if (not os.path.exists(output) or (os.path.exists(source) and os.path.getmtime(source) > os.path.getmtime(output))): - print("Generating %s..." % os.path.relpath(output)) + print("Generating %s..." % output) if not os.path.exists(source): sys.stderr.write("Can't find required file: %s\n" % source) @@ -83,13 +71,7 @@ "or install the binary package.\n") sys.exit(-1) - protoc_command = [ - protoc, - "-I{}".format(src_dir), - "-I{}".format(current_dir), - "--python_out={}".format(current_dir), - source, - ] + protoc_command = [ protoc, "-I../src", "-I.", "--python_out=.", source ] if subprocess.call(protoc_command) != 0: sys.exit(-1) @@ -134,7 +116,7 @@ class clean(_clean): def run(self): # Delete generated files in the code tree. - for (dirpath, dirnames, filenames) in os.walk(current_dir): + for (dirpath, dirnames, filenames) in os.walk("."): for filename in filenames: filepath = os.path.join(dirpath, filename) if filepath.endswith("_pb2.py") or filepath.endswith(".pyc") or \ @@ -287,14 +269,7 @@ "Programming Language :: Python :: 3.7", ], namespace_packages=['google'], - # package_dir is required when setup.py is not run from the python/ - # directory (such as from the ReadTheDocs build). See - # https://setuptools.readthedocs.io/en/latest/setuptools.html#using-find-packages - # package_dir must be a relative path. See: - # https://stackoverflow.com/a/53547931/101923 - package_dir={"": current_dir_relative}, packages=find_packages( - where=current_dir, exclude=[ 'import_test_package', ],