Python 2.7 protobuf .py file generation issue

Question

I am following this guide (https://developers.google.com/protocol-buffers/docs/pythontutorial) and using the exact sample of addressbook.proto.

Here is the content of generated addressbook_pb2.py file, question is where is the definition of class Person, PhoneNumber and AddressBook? I see the sample code of the guide refer them as classes.

Here is the sample code I refer to,

import addressbook_pb2
person = addressbook_pb2.Person()
person.id = 1234
person.name = "John Doe"
person.email = "jdoe@example.com"
phone = person.phone.add()
phone.number = "555-4321"
phone.type = addressbook_pb2.Person.HOME

Another quick question is, where should I put file addressbook_pb2.py, so that my other python file could refer it to use Person, PhoneNumber and AddressBook classes?

Here is the automated generated file addressbook_pb2.py,

# Generated by the protocol buffer compiler.  DO NOT EDIT!
# source: addressbook.proto

import sys
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
from google.protobuf import descriptor as _descriptor
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
from google.protobuf import descriptor_pb2
# @@protoc_insertion_point(imports)

_sym_db = _symbol_database.Default()




DESCRIPTOR = _descriptor.FileDescriptor(
  name='addressbook.proto',
  package='tutorial',
  syntax='proto2',
  serialized_pb=_b('\n\x11\x61\x64\x64ressbook.proto\x12\x08tutorial\"\xda\x01\n\x06Person\x12\x0c\n\x04name\x18\x01 \x02(\t\x12\n\n\x02id\x18\x02 \x02(\x05\x12\r\n\x05\x65mail\x18\x03 \x01(\t\x12+\n\x05phone\x18\x04 \x03(\x0b\x32\x1c.tutorial.Person.PhoneNumber\x1aM\n\x0bPhoneNumber\x12\x0e\n\x06number\x18\x01 \x02(\t\x12.\n\x04type\x18\x02 \x01(\x0e\x32\x1a.tutorial.Person.PhoneType:\x04HOME\"+\n\tPhoneType\x12\n\n\x06MOBILE\x10\x00\x12\x08\n\x04HOME\x10\x01\x12\x08\n\x04WORK\x10\x02\"/\n\x0b\x41\x64\x64ressBook\x12 \n\x06person\x18\x01 \x03(\x0b\x32\x10.tutorial.Person')
)
_sym_db.RegisterFileDescriptor(DESCRIPTOR)



_PERSON_PHONETYPE = _descriptor.EnumDescriptor(
  name='PhoneType',
  full_name='tutorial.Person.PhoneType',
  filename=None,
  file=DESCRIPTOR,
  values=[
    _descriptor.EnumValueDescriptor(
      name='MOBILE', index=0, number=0,
      options=None,
      type=None),
    _descriptor.EnumValueDescriptor(
      name='HOME', index=1, number=1,
      options=None,
      type=None),
    _descriptor.EnumValueDescriptor(
      name='WORK', index=2, number=2,
      options=None,
      type=None),
  ],
  containing_type=None,
  options=None,
  serialized_start=207,
  serialized_end=250,
)
_sym_db.RegisterEnumDescriptor(_PERSON_PHONETYPE)


_PERSON_PHONENUMBER = _descriptor.Descriptor(
  name='PhoneNumber',
  full_name='tutorial.Person.PhoneNumber',
  filename=None,
  file=DESCRIPTOR,
  containing_type=None,
  fields=[
    _descriptor.FieldDescriptor(
      name='number', full_name='tutorial.Person.PhoneNumber.number', index=0,
      number=1, type=9, cpp_type=9, label=2,
      has_default_value=False, default_value=_b("").decode('utf-8'),
      message_type=None, enum_type=None, containing_type=None,
      is_extension=False, extension_scope=None,
      options=None),
    _descriptor.FieldDescriptor(
      name='type', full_name='tutorial.Person.PhoneNumber.type', index=1,
      number=2, type=14, cpp_type=8, label=1,
      has_default_value=True, default_value=1,
      message_type=None, enum_type=None, containing_type=None,
      is_extension=False, extension_scope=None,
      options=None),
  ],
  extensions=[
  ],
  nested_types=[],
  enum_types=[
  ],
  options=None,
  is_extendable=False,
  syntax='proto2',
  extension_ranges=[],
  oneofs=[
  ],
  serialized_start=128,
  serialized_end=205,
)

_PERSON = _descriptor.Descriptor(
  name='Person',
  full_name='tutorial.Person',
  filename=None,
  file=DESCRIPTOR,
  containing_type=None,
  fields=[
    _descriptor.FieldDescriptor(
      name='name', full_name='tutorial.Person.name', index=0,
      number=1, type=9, cpp_type=9, label=2,
      has_default_value=False, default_value=_b("").decode('utf-8'),
      message_type=None, enum_type=None, containing_type=None,
      is_extension=False, extension_scope=None,
      options=None),
    _descriptor.FieldDescriptor(
      name='id', full_name='tutorial.Person.id', index=1,
      number=2, type=5, cpp_type=1, label=2,
      has_default_value=False, default_value=0,
      message_type=None, enum_type=None, containing_type=None,
      is_extension=False, extension_scope=None,
      options=None),
    _descriptor.FieldDescriptor(
      name='email', full_name='tutorial.Person.email', index=2,
      number=3, type=9, cpp_type=9, label=1,
      has_default_value=False, default_value=_b("").decode('utf-8'),
      message_type=None, enum_type=None, containing_type=None,
      is_extension=False, extension_scope=None,
      options=None),
    _descriptor.FieldDescriptor(
      name='phone', full_name='tutorial.Person.phone', index=3,
      number=4, type=11, cpp_type=10, label=3,
      has_default_value=False, default_value=[],
      message_type=None, enum_type=None, containing_type=None,
      is_extension=False, extension_scope=None,
      options=None),
  ],
  extensions=[
  ],
  nested_types=[_PERSON_PHONENUMBER, ],
  enum_types=[
    _PERSON_PHONETYPE,
  ],
  options=None,
  is_extendable=False,
  syntax='proto2',
  extension_ranges=[],
  oneofs=[
  ],
  serialized_start=32,
  serialized_end=250,
)


_ADDRESSBOOK = _descriptor.Descriptor(
  name='AddressBook',
  full_name='tutorial.AddressBook',
  filename=None,
  file=DESCRIPTOR,
  containing_type=None,
  fields=[
    _descriptor.FieldDescriptor(
      name='person', full_name='tutorial.AddressBook.person', index=0,
      number=1, type=11, cpp_type=10, label=3,
      has_default_value=False, default_value=[],
      message_type=None, enum_type=None, containing_type=None,
      is_extension=False, extension_scope=None,
      options=None),
  ],
  extensions=[
  ],
  nested_types=[],
  enum_types=[
  ],
  options=None,
  is_extendable=False,
  syntax='proto2',
  extension_ranges=[],
  oneofs=[
  ],
  serialized_start=252,
  serialized_end=299,
)

_PERSON_PHONENUMBER.fields_by_name['type'].enum_type = _PERSON_PHONETYPE
_PERSON_PHONENUMBER.containing_type = _PERSON
_PERSON.fields_by_name['phone'].message_type = _PERSON_PHONENUMBER
_PERSON_PHONETYPE.containing_type = _PERSON
_ADDRESSBOOK.fields_by_name['person'].message_type = _PERSON
DESCRIPTOR.message_types_by_name['Person'] = _PERSON
DESCRIPTOR.message_types_by_name['AddressBook'] = _ADDRESSBOOK

Person = _reflection.GeneratedProtocolMessageType('Person', (_message.Message,), dict(

  PhoneNumber = _reflection.GeneratedProtocolMessageType('PhoneNumber', (_message.Message,), dict(
    DESCRIPTOR = _PERSON_PHONENUMBER,
    __module__ = 'addressbook_pb2'
    # @@protoc_insertion_point(class_scope:tutorial.Person.PhoneNumber)
    ))
  ,
  DESCRIPTOR = _PERSON,
  __module__ = 'addressbook_pb2'
  # @@protoc_insertion_point(class_scope:tutorial.Person)
  ))
_sym_db.RegisterMessage(Person)
_sym_db.RegisterMessage(Person.PhoneNumber)

AddressBook = _reflection.GeneratedProtocolMessageType('AddressBook', (_message.Message,), dict(
  DESCRIPTOR = _ADDRESSBOOK,
  __module__ = 'addressbook_pb2'
  # @@protoc_insertion_point(class_scope:tutorial.AddressBook)
  ))
_sym_db.RegisterMessage(AddressBook)


# @@protoc_insertion_point(module_scope)

score 1 · Answer 1 · answered Jul 27 '16 at 23:31

1

The Person class is defined in this line of your addressbook_pb2.py:

Person = _reflection.GeneratedProtocolMessageType('Person', (_message.Message,), dict( ... ) )

And similarly for the AddressBook class. Note that the PhoneNumber class you refer to is actually Person.PhoneNumber.

What is happening here? GeneratedProtocolMessageType is a metaclass. A metaclass is a type whose instances are simple classes, and is invoked with the form:

metaclass(<class name>, <tuple of base classes for this new class>, <namespace of new class>)

As to where you should put addressbook_pb2.py, that depends entirely on how you are structuring your project. Place it either somewhere on your PYTHONPATH, or beside your application.

answered Jul 27 '16 at 23:31

donkopotamus

18,536
2
36
52

Thanks donkopotamus, vote up and will try. Another quick question is, where should I put `addressbook_pb2.py` if I need to refer it (e.g. refer `Person`) in another python file in the same directory? Is there a simple way without rewrite `PYTHONPATH`? – Lin Ma Jul 27 '16 at 23:49
1

If you are not using "packages" at all, simply put it beside your other modules. – donkopotamus Jul 27 '16 at 23:51
Thanks donkopotamus, vote up for your reply, do you mean I should just put `addressbook_pb2.py` in the same directory as `foo.py`, suppose I want to use Person class defined in `addressbook_pb2.py` from `foo.py`? – Lin Ma Jul 28 '16 at 00:03

score 1 · Accepted Answer · edited May 23 '17 at 12:33

1

The Python classes are generated dynamically at runtime.

Quoting from the documentation page that you have linked to:

The important line in each class is metaclass = reflection.GeneratedProtocolMessageType. While the details of how Python metaclasses work is beyond the scope of this tutorial, you can think of them as like a template for creating classes. At load time, the GeneratedProtocolMessageType metaclass uses the specified descriptors to create all the Python methods you need to work with each message type and adds them to the relevant classes. You can then use the fully-populated classes in your code.

More specifically the line:

Person = _reflection.GeneratedProtocolMessageType('Person', (_message.Message,), dict(...

creates the Person class in Python.

In Python, something like:

>>> Person = type('Person', (), {})
>>> Person
<class '__main__.Person'>

is same as defining a regular Person class. This is what is being used under the hood here.

You may want to look at the SO question or this blog entry,if you are not very familiar with the concept of metaclasses.

Edited in reply to last comment

I am not sure if I correctly understood your last remark but If you want to use the Person class from addressbook_pb2.py in foo.py, you place addressbook_pb2.py adjacent to foo.py and do the following import in foo.py:

from addressbook_pb2 import Person

edited May 23 '17 at 12:33

Community

1
1

answered Jul 27 '16 at 23:31

bhaskarc

7,796
10
58
80

Thanks tao, vote up and will try. Another quick question is, where should I put `addressbook_pb2.py` if I need to refer it (e.g. refer `Person`) in another python file in the same directory? – Lin Ma Jul 27 '16 at 23:49
1

It is just another regular Python file and so you may put it anywhere in your application directory structure and then import it specifying the file directory hierarchy as in `import directory.path.to.your.file.addressbook_pb2.py`. You may have to add a blank `__init__.py` file to the directory where you keep this file for Python to recognise it as a package – bhaskarc Jul 27 '16 at 23:55
Thanks tao, vote up. In the code I posted for `addressbook_pb2.py`, it does not have any package definition, so, I should be safe to skip the step of add a blank `__init__.py` file? – Lin Ma Jul 28 '16 at 00:05
1

The answer to this is a bit more lengthy so I would suggest you read this answer: http://stackoverflow.com/a/21166205/2348704 – bhaskarc Jul 28 '16 at 16:12
Thanks tao, vote up, and for the addressbook_pb2.py I posted in my original post, for `Person`, it is in any package? I have the confusion since I see you refer `Person` by `Person = _reflection.GeneratedProtocolMessageType`, it seems `Person` is not in any packages from your code, but from the code `full_name='tutorial.Person'`, it seems it is in `tutorial` package? – Lin Ma Jul 28 '16 at 22:34
1

edited my answer to post reply to your last question – bhaskarc Jul 29 '16 at 09:52
Thanks tao for update, and vote up. I think you understand my question correctly. I am wondering for your code you `from addressbook_pb2`, and `addressbook_pb2` is a package name or? I have this confusion since I do not find a package called `addressbook_pb2` defined in protobuf compiled file. Thanks. – Lin Ma Jul 29 '16 at 18:06
1

No addressbook_pb2 is not the package name, it is the filename(a file is called a module in Python). A Python package is simply a directory of Python module(s) (read files). Check this answer http://programmers.stackexchange.com/a/111882/121746 – bhaskarc Jul 29 '16 at 18:40

Python 2.7 protobuf .py file generation issue

2 Answers2